summaryrefslogtreecommitdiff
path: root/arch
diff options
context:
space:
mode:
Diffstat (limited to 'arch')
-rw-r--r--arch/arm/configs/sdm660-perf_defconfig1
-rw-r--r--arch/arm/configs/sdm660_defconfig2
-rw-r--r--arch/arm/include/asm/cacheflush.h3
-rw-r--r--arch/arm/include/asm/device.h2
-rw-r--r--arch/arm/include/asm/dma-iommu.h3
-rw-r--r--arch/arm/include/asm/dma-mapping.h7
-rw-r--r--arch/arm/include/asm/glue-cache.h6
-rw-r--r--arch/arm/mm/dma-mapping.c26
-rw-r--r--arch/arm/mm/dma.h3
9 files changed, 38 insertions, 15 deletions
diff --git a/arch/arm/configs/sdm660-perf_defconfig b/arch/arm/configs/sdm660-perf_defconfig
index 013bada10783..73c12f90016c 100644
--- a/arch/arm/configs/sdm660-perf_defconfig
+++ b/arch/arm/configs/sdm660-perf_defconfig
@@ -498,6 +498,7 @@ CONFIG_MSM_MMCC_660=y
CONFIG_CLOCK_CPU_OSM=y
CONFIG_QCOM_MDSS_PLL=y
CONFIG_REMOTE_SPINLOCK_MSM=y
+CONFIG_IOMMU_IO_PGTABLE_FAST=y
CONFIG_ARM_SMMU=y
CONFIG_IOMMU_DEBUG=y
CONFIG_IOMMU_DEBUG_TRACKING=y
diff --git a/arch/arm/configs/sdm660_defconfig b/arch/arm/configs/sdm660_defconfig
index 22da2099accc..5c8dbe8e8302 100644
--- a/arch/arm/configs/sdm660_defconfig
+++ b/arch/arm/configs/sdm660_defconfig
@@ -499,6 +499,8 @@ CONFIG_MSM_MMCC_660=y
CONFIG_CLOCK_CPU_OSM=y
CONFIG_QCOM_MDSS_PLL=y
CONFIG_REMOTE_SPINLOCK_MSM=y
+CONFIG_IOMMU_IO_PGTABLE_FAST=y
+CONFIG_IOMMU_IO_PGTABLE_FAST_SELFTEST=y
CONFIG_ARM_SMMU=y
CONFIG_IOMMU_DEBUG=y
CONFIG_IOMMU_DEBUG_TRACKING=y
diff --git a/arch/arm/include/asm/cacheflush.h b/arch/arm/include/asm/cacheflush.h
index 2e2d1657e604..012a3aafcf33 100644
--- a/arch/arm/include/asm/cacheflush.h
+++ b/arch/arm/include/asm/cacheflush.h
@@ -178,6 +178,9 @@ extern void __cpuc_flush_dcache_area(void *, size_t);
* is visible to DMA, or data written by DMA to system memory is
* visible to the CPU.
*/
+extern void __dma_map_area(const void *, size_t, int);
+extern void __dma_unmap_area(const void *, size_t, int);
+
extern void dmac_inv_range(const void *, const void *);
extern void dmac_clean_range(const void *, const void *);
extern void dmac_flush_range(const void *, const void *);
diff --git a/arch/arm/include/asm/device.h b/arch/arm/include/asm/device.h
index 4111592f0130..d8a572f9c187 100644
--- a/arch/arm/include/asm/device.h
+++ b/arch/arm/include/asm/device.h
@@ -7,7 +7,7 @@
#define ASMARM_DEVICE_H
struct dev_archdata {
- struct dma_map_ops *dma_ops;
+ const struct dma_map_ops *dma_ops;
#ifdef CONFIG_DMABOUNCE
struct dmabounce_device_info *dmabounce;
#endif
diff --git a/arch/arm/include/asm/dma-iommu.h b/arch/arm/include/asm/dma-iommu.h
index b4e74af0abb7..74643f5b41c4 100644
--- a/arch/arm/include/asm/dma-iommu.h
+++ b/arch/arm/include/asm/dma-iommu.h
@@ -8,6 +8,7 @@
#include <linux/dma-debug.h>
#include <linux/kmemcheck.h>
#include <linux/kref.h>
+#include <linux/dma-mapping-fast.h>
struct dma_iommu_mapping {
/* iommu specific data */
@@ -22,6 +23,8 @@ struct dma_iommu_mapping {
spinlock_t lock;
struct kref kref;
+
+ struct dma_fast_smmu_mapping *fast;
};
#ifdef CONFIG_ARM_DMA_USE_IOMMU
diff --git a/arch/arm/include/asm/dma-mapping.h b/arch/arm/include/asm/dma-mapping.h
index ccb3aa64640d..05ff03aead43 100644
--- a/arch/arm/include/asm/dma-mapping.h
+++ b/arch/arm/include/asm/dma-mapping.h
@@ -17,14 +17,14 @@
extern struct dma_map_ops arm_dma_ops;
extern struct dma_map_ops arm_coherent_dma_ops;
-static inline struct dma_map_ops *__generic_dma_ops(struct device *dev)
+static inline const struct dma_map_ops *__generic_dma_ops(struct device *dev)
{
if (dev && dev->archdata.dma_ops)
return dev->archdata.dma_ops;
return &arm_dma_ops;
}
-static inline struct dma_map_ops *get_dma_ops(struct device *dev)
+static inline const struct dma_map_ops *get_dma_ops(struct device *dev)
{
if (xen_initial_domain())
return xen_dma_ops;
@@ -32,7 +32,8 @@ static inline struct dma_map_ops *get_dma_ops(struct device *dev)
return __generic_dma_ops(dev);
}
-static inline void set_dma_ops(struct device *dev, struct dma_map_ops *ops)
+static inline void set_dma_ops(struct device *dev,
+ const struct dma_map_ops *ops)
{
BUG_ON(!dev);
dev->archdata.dma_ops = ops;
diff --git a/arch/arm/include/asm/glue-cache.h b/arch/arm/include/asm/glue-cache.h
index 1f442995cdbf..7a4893e61866 100644
--- a/arch/arm/include/asm/glue-cache.h
+++ b/arch/arm/include/asm/glue-cache.h
@@ -161,6 +161,12 @@ static inline void nop_dma_unmap_area(const void *s, size_t l, int f) { }
#define dmac_flush_range __glue(_CACHE,_dma_flush_range)
#define dmac_inv_range __glue(_CACHE, _dma_inv_range)
#define dmac_clean_range __glue(_CACHE, _dma_clean_range)
+#define dmac_map_area __glue(_CACHE, _dma_map_area)
+#define dmac_unmap_area __glue(_CACHE, _dma_unmap_area)
+
+#define __dma_map_area dmac_map_area
+#define __dma_unmap_area dmac_unmap_area
+#define __dma_flush_range dmac_flush_range
#endif
#endif
diff --git a/arch/arm/mm/dma-mapping.c b/arch/arm/mm/dma-mapping.c
index 330061d3d081..b58d8e0d9ed1 100644
--- a/arch/arm/mm/dma-mapping.c
+++ b/arch/arm/mm/dma-mapping.c
@@ -29,6 +29,7 @@
#include <linux/sizes.h>
#include <linux/cma.h>
#include <linux/msm_dma_iommu_mapping.h>
+#include <linux/dma-mapping-fast.h>
#include <asm/memory.h>
#include <asm/highmem.h>
@@ -1019,7 +1020,7 @@ static void __dma_page_dev_to_cpu(struct page *page, unsigned long off,
int arm_dma_map_sg(struct device *dev, struct scatterlist *sg, int nents,
enum dma_data_direction dir, struct dma_attrs *attrs)
{
- struct dma_map_ops *ops = get_dma_ops(dev);
+ const struct dma_map_ops *ops = get_dma_ops(dev);
struct scatterlist *s;
int i, j;
@@ -1053,7 +1054,7 @@ int arm_dma_map_sg(struct device *dev, struct scatterlist *sg, int nents,
void arm_dma_unmap_sg(struct device *dev, struct scatterlist *sg, int nents,
enum dma_data_direction dir, struct dma_attrs *attrs)
{
- struct dma_map_ops *ops = get_dma_ops(dev);
+ const struct dma_map_ops *ops = get_dma_ops(dev);
struct scatterlist *s;
int i;
@@ -1072,7 +1073,7 @@ void arm_dma_unmap_sg(struct device *dev, struct scatterlist *sg, int nents,
void arm_dma_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg,
int nents, enum dma_data_direction dir)
{
- struct dma_map_ops *ops = get_dma_ops(dev);
+ const struct dma_map_ops *ops = get_dma_ops(dev);
struct scatterlist *s;
int i;
@@ -1091,7 +1092,7 @@ void arm_dma_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg,
void arm_dma_sync_sg_for_device(struct device *dev, struct scatterlist *sg,
int nents, enum dma_data_direction dir)
{
- struct dma_map_ops *ops = get_dma_ops(dev);
+ const struct dma_map_ops *ops = get_dma_ops(dev);
struct scatterlist *s;
int i;
@@ -1988,7 +1989,7 @@ static void arm_iommu_sync_single_for_device(struct device *dev,
__dma_page_cpu_to_dev(page, offset, size, dir);
}
-struct dma_map_ops iommu_ops = {
+const struct dma_map_ops iommu_ops = {
.alloc = arm_iommu_alloc_attrs,
.free = arm_iommu_free_attrs,
.mmap = arm_iommu_mmap_attrs,
@@ -2007,7 +2008,7 @@ struct dma_map_ops iommu_ops = {
.set_dma_mask = arm_dma_set_mask,
};
-struct dma_map_ops iommu_coherent_ops = {
+const struct dma_map_ops iommu_coherent_ops = {
.alloc = arm_iommu_alloc_attrs,
.free = arm_iommu_free_attrs,
.mmap = arm_iommu_mmap_attrs,
@@ -2166,6 +2167,11 @@ int arm_iommu_attach_device(struct device *dev,
{
int err;
int s1_bypass = 0;
+ int is_fast = 0;
+
+ iommu_domain_get_attr(mapping->domain, DOMAIN_ATTR_FAST, &is_fast);
+ if (is_fast)
+ return fast_smmu_attach_device(dev, mapping);
err = __arm_iommu_attach_device(dev, mapping);
if (err)
@@ -2182,6 +2188,7 @@ EXPORT_SYMBOL_GPL(arm_iommu_attach_device);
static void __arm_iommu_detach_device(struct device *dev)
{
struct dma_iommu_mapping *mapping;
+ int is_fast;
mapping = to_dma_iommu_mapping(dev);
if (!mapping) {
@@ -2191,6 +2198,9 @@ static void __arm_iommu_detach_device(struct device *dev)
if (msm_dma_unmap_all_for_dev(dev))
dev_warn(dev, "IOMMU detach with outstanding mappings\n");
+ iommu_domain_get_attr(mapping->domain, DOMAIN_ATTR_FAST, &is_fast);
+ if (is_fast)
+ return fast_smmu_detach_device(dev, mapping);
iommu_detach_device(mapping->domain, dev);
kref_put(&mapping->kref, release_iommu_mapping);
@@ -2226,7 +2236,7 @@ void arm_iommu_detach_device(struct device *dev)
}
EXPORT_SYMBOL_GPL(arm_iommu_detach_device);
-static struct dma_map_ops *arm_get_iommu_dma_map_ops(bool coherent)
+static const struct dma_map_ops *arm_get_iommu_dma_map_ops(bool coherent)
{
return coherent ? &iommu_coherent_ops : &iommu_ops;
}
@@ -2289,7 +2299,7 @@ static struct dma_map_ops *arm_get_dma_map_ops(bool coherent)
void arch_setup_dma_ops(struct device *dev, u64 dma_base, u64 size,
struct iommu_ops *iommu, bool coherent)
{
- struct dma_map_ops *dma_ops;
+ const struct dma_map_ops *dma_ops;
dev->archdata.dma_coherent = coherent;
if (arm_setup_iommu_dma_ops(dev, dma_base, size, iommu))
diff --git a/arch/arm/mm/dma.h b/arch/arm/mm/dma.h
index 70ea6852f94e..29c54f7d81f3 100644
--- a/arch/arm/mm/dma.h
+++ b/arch/arm/mm/dma.h
@@ -4,9 +4,6 @@
#include <asm/glue-cache.h>
#ifndef MULTI_CACHE
-#define dmac_map_area __glue(_CACHE,_dma_map_area)
-#define dmac_unmap_area __glue(_CACHE,_dma_unmap_area)
-
/*
* These are private to the dma-mapping API. Do not use directly.
* Their sole purpose is to ensure that data held in the cache
generated by cgit v1.2.3 (git 2.39.1) at 2024-10-18 15:31:08 +0000