summaryrefslogtreecommitdiff
path: root/drivers/iommu
diff options
context:
space:
mode:
Diffstat (limited to 'drivers/iommu')
-rw-r--r--drivers/iommu/Kconfig85
-rw-r--r--drivers/iommu/Makefile5
-rw-r--r--drivers/iommu/arm-smmu.c3086
-rw-r--r--drivers/iommu/dma-iommu.c10
-rw-r--r--drivers/iommu/dma-mapping-fast.c860
-rw-r--r--drivers/iommu/io-pgtable-arm.c832
-rw-r--r--drivers/iommu/io-pgtable-fast.c751
-rw-r--r--drivers/iommu/io-pgtable-msm-secure.c243
-rw-r--r--drivers/iommu/io-pgtable.c75
-rw-r--r--drivers/iommu/io-pgtable.h69
-rw-r--r--drivers/iommu/iommu-debug.c2386
-rw-r--r--drivers/iommu/iommu-debug.h34
-rw-r--r--drivers/iommu/iommu.c165
-rw-r--r--drivers/iommu/msm_dma_iommu_mapping.c423
-rw-r--r--drivers/iommu/msm_iommu.c735
-rw-r--r--drivers/iommu/msm_iommu.h120
-rw-r--r--drivers/iommu/msm_iommu_dev.c392
-rw-r--r--drivers/iommu/msm_iommu_hw-8xxx.h1865
18 files changed, 8626 insertions, 3510 deletions
diff --git a/drivers/iommu/Kconfig b/drivers/iommu/Kconfig
index b9094e9da537..7a504b1ad94d 100644
--- a/drivers/iommu/Kconfig
+++ b/drivers/iommu/Kconfig
@@ -39,6 +39,42 @@ config IOMMU_IO_PGTABLE_LPAE_SELFTEST
If unsure, say N here.
+config IOMMU_IO_PGTABLE_FAST
+ bool "Fast ARMv7/v8 Long Descriptor Format"
+ depends on ARM64_DMA_USE_IOMMU
+ help
+ Enable support for a subset of the ARM long descriptor pagetable
+ format. This allocator achieves fast performance by
+ pre-allocating and pre-populating page table memory up front.
+ only supports a 32 bit virtual address space.
+
+ This implementation is mainly optimized for use cases where the
+ buffers are small (<= 64K) since it only supports 4K page sizes.
+
+config IOMMU_IO_PGTABLE_FAST_SELFTEST
+ bool "Fast IO pgtable selftests"
+ depends on IOMMU_IO_PGTABLE_FAST
+ help
+ Enable self-tests for "fast" page table allocator. This performs
+ a series of page-table consistency checks during boot.
+
+ If unsure, say N here.
+
+config IOMMU_IO_PGTABLE_FAST_PROVE_TLB
+ bool "Prove correctness of TLB maintenance in the Fast DMA mapper"
+ depends on IOMMU_IO_PGTABLE_FAST
+ help
+ Enables some debug features that help prove correctness of TLB
+ maintenance routines in the Fast DMA mapper. This option will
+ slow things down considerably, so should only be used in a debug
+ configuration. This relies on the ability to set bits in an
+ invalid page table entry, which is disallowed on some hardware
+ due to errata. If you're running on such a platform then this
+ option can only be used with unit tests. It will break real use
+ cases.
+
+ If unsure, say N here.
+
endmenu
config IOMMU_IOVA
@@ -66,24 +102,6 @@ config FSL_PAMU
PAMU can authorize memory access, remap the memory address, and remap I/O
transaction types.
-# MSM IOMMU support
-config MSM_IOMMU
- bool "MSM IOMMU Support"
- depends on ARM
- depends on ARCH_MSM8X60 || ARCH_MSM8960 || COMPILE_TEST
- depends on BROKEN
- select IOMMU_API
- help
- Support for the IOMMUs found on certain Qualcomm SOCs.
- These IOMMUs allow virtualization of the address space used by most
- cores within the multimedia subsystem.
-
- If unsure, say N here.
-
-config IOMMU_PGTABLES_L2
- def_bool y
- depends on MSM_IOMMU && MMU && SMP && CPU_DCACHE_DISABLE=n
-
# AMD IOMMU support
config AMD_IOMMU
bool "AMD IOMMU support"
@@ -366,6 +384,7 @@ config ARM_SMMU
select IOMMU_API
select IOMMU_IO_PGTABLE_LPAE
select ARM_DMA_USE_IOMMU if ARM
+ select ARM64_DMA_USE_IOMMU if ARM64
help
Support for implementations of the ARM System MMU architecture
versions 1 and 2.
@@ -393,4 +412,34 @@ config S390_IOMMU
help
Support for the IOMMU API for s390 PCI devices.
+menuconfig IOMMU_DEBUG
+ bool "IOMMU Profiling and Debugging"
+ help
+ Makes available some additional IOMMU profiling and debugging
+ options.
+
+if IOMMU_DEBUG
+
+config IOMMU_DEBUG_TRACKING
+ bool "Track key IOMMU events"
+ select IOMMU_API
+ help
+ Enables additional debug tracking in the IOMMU framework code.
+ Tracking information and tests can be accessed through various
+ debugfs files.
+
+ Say Y here if you need to debug IOMMU issues and are okay with
+ the performance penalty of the tracking.
+
+config IOMMU_TESTS
+ bool "Interactive IOMMU performance/functional tests"
+ select IOMMU_API
+ help
+ Enables a suite of IOMMU unit tests. The tests are runnable
+ through debugfs. Unlike the IOMMU_DEBUG_TRACKING option, the
+ impact of enabling this option to overal system performance
+ should be minimal.
+
+endif # IOMMU_DEBUG
+
endif # IOMMU_SUPPORT
diff --git a/drivers/iommu/Makefile b/drivers/iommu/Makefile
index 68faca02225d..fe78a84c845e 100644
--- a/drivers/iommu/Makefile
+++ b/drivers/iommu/Makefile
@@ -2,11 +2,14 @@ obj-$(CONFIG_IOMMU_API) += iommu.o
obj-$(CONFIG_IOMMU_API) += iommu-traces.o
obj-$(CONFIG_IOMMU_API) += iommu-sysfs.o
obj-$(CONFIG_IOMMU_DMA) += dma-iommu.o
+obj-$(CONFIG_IOMMU_API) += msm_dma_iommu_mapping.o
obj-$(CONFIG_IOMMU_IO_PGTABLE) += io-pgtable.o
obj-$(CONFIG_IOMMU_IO_PGTABLE_LPAE) += io-pgtable-arm.o
obj-$(CONFIG_IOMMU_IOVA) += iova.o
+obj-$(CONFIG_MSM_TZ_SMMU) += io-pgtable-msm-secure.o
+obj-$(CONFIG_IOMMU_IO_PGTABLE_FAST) += io-pgtable-fast.o dma-mapping-fast.o
obj-$(CONFIG_OF_IOMMU) += of_iommu.o
-obj-$(CONFIG_MSM_IOMMU) += msm_iommu.o msm_iommu_dev.o
+obj-$(CONFIG_IOMMU_DEBUG) += iommu-debug.o
obj-$(CONFIG_AMD_IOMMU) += amd_iommu.o amd_iommu_init.o
obj-$(CONFIG_AMD_IOMMU_V2) += amd_iommu_v2.o
obj-$(CONFIG_ARM_SMMU) += arm-smmu.o
diff --git a/drivers/iommu/arm-smmu.c b/drivers/iommu/arm-smmu.c
index 47dc7a793f5c..90306a0e2164 100644
--- a/drivers/iommu/arm-smmu.c
+++ b/drivers/iommu/arm-smmu.c
@@ -42,13 +42,21 @@
#include <linux/platform_device.h>
#include <linux/slab.h>
#include <linux/spinlock.h>
+#include <linux/notifier.h>
#include <linux/amba/bus.h>
+#include <soc/qcom/msm_tz_smmu.h>
+#include <soc/qcom/scm.h>
+#include <soc/qcom/secure_buffer.h>
+#include <asm/cacheflush.h>
+#include <linux/msm-bus.h>
+#include <dt-bindings/msm/msm-bus-ids.h>
+#include <linux/msm_pcie.h>
#include "io-pgtable.h"
/* Maximum number of stream IDs assigned to a single device */
-#define MAX_MASTER_STREAMIDS MAX_PHANDLE_ARGS
+#define MAX_MASTER_STREAMIDS 45
/* Maximum number of context banks per SMMU */
#define ARM_SMMU_MAX_CBS 128
@@ -148,7 +156,7 @@
#define ARM_SMMU_GR0_sTLBGSYNC 0x70
#define ARM_SMMU_GR0_sTLBGSTATUS 0x74
#define sTLBGSTATUS_GSACTIVE (1 << 0)
-#define TLB_LOOP_TIMEOUT 1000000 /* 1s! */
+#define TLB_LOOP_TIMEOUT 500000 /* 500ms */
/* Stream mapping registers */
#define ARM_SMMU_GR0_SMR(n) (0x800 + ((n) << 2))
@@ -195,29 +203,38 @@
#define ARM_SMMU_CB(smmu, n) ((n) * (1 << (smmu)->pgshift))
#define ARM_SMMU_CB_SCTLR 0x0
+#define ARM_SMMU_CB_ACTLR 0x4
#define ARM_SMMU_CB_RESUME 0x8
#define ARM_SMMU_CB_TTBCR2 0x10
#define ARM_SMMU_CB_TTBR0 0x20
#define ARM_SMMU_CB_TTBR1 0x28
#define ARM_SMMU_CB_TTBCR 0x30
+#define ARM_SMMU_CB_CONTEXTIDR 0x34
#define ARM_SMMU_CB_S1_MAIR0 0x38
#define ARM_SMMU_CB_S1_MAIR1 0x3c
#define ARM_SMMU_CB_PAR_LO 0x50
#define ARM_SMMU_CB_PAR_HI 0x54
#define ARM_SMMU_CB_FSR 0x58
+#define ARM_SMMU_CB_FSRRESTORE 0x5c
#define ARM_SMMU_CB_FAR_LO 0x60
#define ARM_SMMU_CB_FAR_HI 0x64
#define ARM_SMMU_CB_FSYNR0 0x68
#define ARM_SMMU_CB_S1_TLBIVA 0x600
#define ARM_SMMU_CB_S1_TLBIASID 0x610
+#define ARM_SMMU_CB_S1_TLBIALL 0x618
#define ARM_SMMU_CB_S1_TLBIVAL 0x620
#define ARM_SMMU_CB_S2_TLBIIPAS2 0x630
#define ARM_SMMU_CB_S2_TLBIIPAS2L 0x638
+#define ARM_SMMU_CB_TLBSYNC 0x7f0
+#define ARM_SMMU_CB_TLBSTATUS 0x7f4
+#define TLBSTATUS_SACTIVE (1 << 0)
#define ARM_SMMU_CB_ATS1PR 0x800
#define ARM_SMMU_CB_ATSR 0x8f0
+#define ARM_SMMU_GR1_CBFRSYNRA(n) (0x400 + ((n) << 2))
#define SCTLR_S1_ASIDPNE (1 << 12)
#define SCTLR_CFCFG (1 << 7)
+#define SCTLR_HUPCF (1 << 8)
#define SCTLR_CFIE (1 << 6)
#define SCTLR_CFRE (1 << 5)
#define SCTLR_E (1 << 4)
@@ -233,9 +250,6 @@
#define RESUME_RETRY (0 << 0)
#define RESUME_TERMINATE (1 << 0)
-#define TTBCR2_SEP_SHIFT 15
-#define TTBCR2_SEP_UPSTREAM (0x7 << TTBCR2_SEP_SHIFT)
-
#define TTBRn_ASID_SHIFT 48
#define FSR_MULTI (1 << 31)
@@ -249,12 +263,46 @@
#define FSR_AFF (1 << 2)
#define FSR_TF (1 << 1)
+/* Definitions for implementation-defined registers */
+#define ACTLR_QCOM_OSH_SHIFT 28
+#define ACTLR_QCOM_OSH 1
+
+#define ACTLR_QCOM_ISH_SHIFT 29
+#define ACTLR_QCOM_ISH 1
+
+#define ACTLR_QCOM_NSH_SHIFT 30
+#define ACTLR_QCOM_NSH 1
+
+#define ARM_SMMU_IMPL_DEF0(smmu) \
+ ((smmu)->base + (2 * (1 << (smmu)->pgshift)))
+#define ARM_SMMU_IMPL_DEF1(smmu) \
+ ((smmu)->base + (6 * (1 << (smmu)->pgshift)))
+#define IMPL_DEF1_MICRO_MMU_CTRL 0
+#define MICRO_MMU_CTRL_LOCAL_HALT_REQ (1 << 2)
+#define MICRO_MMU_CTRL_IDLE (1 << 3)
+
#define FSR_IGN (FSR_AFF | FSR_ASF | \
FSR_TLBMCF | FSR_TLBLKF)
#define FSR_FAULT (FSR_MULTI | FSR_SS | FSR_UUT | \
FSR_EF | FSR_PF | FSR_TF | FSR_IGN)
#define FSYNR0_WNR (1 << 4)
+#define MAX_GLOBAL_REG_SAVE_ENTRIES (2 * ARM_SMMU_MAX_SMRS + 1)
+
+enum arm_smmu_save_ctx {
+ SAVE_ARM_SMMU_CB_SCTLR,
+ SAVE_ARM_SMMU_CB_ACTLR,
+ SAVE_ARM_SMMU_CB_TTBCR2,
+ SAVE_ARM_SMMU_CB_TTBR0,
+ SAVE_ARM_SMMU_CB_TTBR1,
+ SAVE_ARM_SMMU_CB_TTBCR,
+ SAVE_ARM_SMMU_CB_CONTEXTIDR,
+ SAVE_ARM_SMMU_CB_S1_MAIR0,
+ SAVE_ARM_SMMU_CB_S1_MAIR1,
+ SAVE_ARM_SMMU_GR1_CBA2R,
+ SAVE_ARM_SMMU_GR1_CBAR,
+ SAVE_ARM_SMMU_MAX_CNT,
+};
static int force_stage;
module_param_named(force_stage, force_stage, int, S_IRUGO);
@@ -284,11 +332,24 @@ struct arm_smmu_master {
struct arm_smmu_master_cfg cfg;
};
+enum smmu_model_id {
+ SMMU_MODEL_DEFAULT,
+ SMMU_MODEL_QCOM_V2,
+};
+
+struct arm_smmu_impl_def_reg {
+ u32 offset;
+ u32 value;
+};
+
struct arm_smmu_device {
struct device *dev;
+ enum smmu_model_id model;
+
void __iomem *base;
unsigned long size;
+ phys_addr_t phys_addr;
unsigned long pgshift;
#define ARM_SMMU_FEAT_COHERENT_WALK (1 << 0)
@@ -300,6 +361,17 @@ struct arm_smmu_device {
u32 features;
#define ARM_SMMU_OPT_SECURE_CFG_ACCESS (1 << 0)
+#define ARM_SMMU_OPT_INVALIDATE_ON_MAP (1 << 1)
+#define ARM_SMMU_OPT_HALT_AND_TLB_ON_ATOS (1 << 2)
+#define ARM_SMMU_OPT_REGISTER_SAVE (1 << 3)
+#define ARM_SMMU_OPT_SKIP_INIT (1 << 4)
+#define ARM_SMMU_OPT_ERRATA_CTX_FAULT_HANG (1 << 5)
+#define ARM_SMMU_OPT_FATAL_ASF (1 << 6)
+#define ARM_SMMU_OPT_ERRATA_TZ_ATOS (1 << 7)
+#define ARM_SMMU_OPT_NO_SMR_CHECK (1 << 9)
+#define ARM_SMMU_OPT_DYNAMIC (1 << 10)
+#define ARM_SMMU_OPT_HALT (1 << 11)
+#define ARM_SMMU_OPT_STATIC_CB (1 << 12)
u32 options;
enum arm_smmu_arch_version version;
@@ -311,6 +383,8 @@ struct arm_smmu_device {
u32 num_mapping_groups;
DECLARE_BITMAP(smr_map, ARM_SMMU_MAX_SMRS);
+ u32 ubs;
+
unsigned long va_size;
unsigned long ipa_size;
unsigned long pa_size;
@@ -320,18 +394,59 @@ struct arm_smmu_device {
unsigned int *irqs;
struct list_head list;
+ struct list_head static_cbndx_list;
struct rb_root masters;
+
+ int num_clocks;
+ struct clk **clocks;
+
+ struct regulator *gdsc;
+ struct notifier_block regulator_nb;
+
+ /* Protects against domains attaching to the same SMMU concurrently */
+ struct mutex attach_lock;
+ unsigned int attach_count;
+ struct idr asid_idr;
+
+ struct arm_smmu_impl_def_reg *impl_def_attach_registers;
+ unsigned int num_impl_def_attach_registers;
+
+ spinlock_t atos_lock;
+ unsigned int clock_refs_count;
+ spinlock_t clock_refs_lock;
+
+ struct mutex power_lock;
+ unsigned int power_count;
+
+ u32 bus_client;
+ struct msm_bus_scale_pdata *bus_pdata;
+
+ enum tz_smmu_device_id sec_id;
+ int regulator_defer;
+ u64 regs[ARM_SMMU_MAX_CBS*(SAVE_ARM_SMMU_MAX_CNT)];
+ u64 reg_global[MAX_GLOBAL_REG_SAVE_ENTRIES];
};
struct arm_smmu_cfg {
u8 cbndx;
u8 irptndx;
u32 cbar;
+ u32 procid;
+ u16 asid;
+ u8 vmid;
};
#define INVALID_IRPTNDX 0xff
+#define INVALID_CBNDX 0xff
+#define INVALID_ASID 0xffff
+#define INVALID_VMID 0xff
+/*
+ * In V7L and V8L with TTBCR2.AS == 0, ASID is 8 bits.
+ * V8L 16 with TTBCR2.AS == 1 (16 bit ASID) isn't supported yet.
+ */
+#define MAX_ASID 0xff
-#define ARM_SMMU_CB_ASID(cfg) ((cfg)->cbndx)
-#define ARM_SMMU_CB_VMID(cfg) ((cfg)->cbndx + 1)
+#define ARM_SMMU_CB_ASID(cfg) ((cfg)->asid)
+#define ARM_SMMU_CB_VMID(cfg) ((cfg)->vmid)
enum arm_smmu_domain_stage {
ARM_SMMU_DOMAIN_S1 = 0,
@@ -339,13 +454,29 @@ enum arm_smmu_domain_stage {
ARM_SMMU_DOMAIN_NESTED,
};
+struct arm_smmu_pte_info {
+ void *virt_addr;
+ size_t size;
+ struct list_head entry;
+};
+
struct arm_smmu_domain {
struct arm_smmu_device *smmu;
struct io_pgtable_ops *pgtbl_ops;
- spinlock_t pgtbl_lock;
+ struct io_pgtable_cfg pgtbl_cfg;
+ spinlock_t pgtbl_spin_lock;
+ struct mutex pgtbl_mutex_lock;
struct arm_smmu_cfg cfg;
enum arm_smmu_domain_stage stage;
struct mutex init_mutex; /* Protects smmu pointer */
+ u32 attributes;
+ bool slave_side_secure;
+ u32 secure_vmid;
+ struct list_head pte_info_list;
+ struct list_head unassign_list;
+ struct mutex assign_lock;
+ struct list_head secure_pool_list;
+ bool non_fatal_faults;
struct iommu_domain domain;
};
@@ -361,9 +492,75 @@ struct arm_smmu_option_prop {
static struct arm_smmu_option_prop arm_smmu_options[] = {
{ ARM_SMMU_OPT_SECURE_CFG_ACCESS, "calxeda,smmu-secure-config-access" },
+ { ARM_SMMU_OPT_INVALIDATE_ON_MAP, "qcom,smmu-invalidate-on-map" },
+ { ARM_SMMU_OPT_HALT_AND_TLB_ON_ATOS, "qcom,halt-and-tlb-on-atos" },
+ { ARM_SMMU_OPT_REGISTER_SAVE, "qcom,register-save" },
+ { ARM_SMMU_OPT_SKIP_INIT, "qcom,skip-init" },
+ { ARM_SMMU_OPT_ERRATA_CTX_FAULT_HANG, "qcom,errata-ctx-fault-hang" },
+ { ARM_SMMU_OPT_FATAL_ASF, "qcom,fatal-asf" },
+ { ARM_SMMU_OPT_ERRATA_TZ_ATOS, "qcom,errata-tz-atos" },
+ { ARM_SMMU_OPT_NO_SMR_CHECK, "qcom,no-smr-check" },
+ { ARM_SMMU_OPT_DYNAMIC, "qcom,dynamic" },
+ { ARM_SMMU_OPT_HALT, "qcom,enable-smmu-halt"},
+ { ARM_SMMU_OPT_STATIC_CB, "qcom,enable-static-cb"},
{ 0, NULL},
};
+#define TYPE_TRANS (S2CR_TYPE_TRANS >> S2CR_TYPE_SHIFT)
+#define TYPE_BYPASS (S2CR_TYPE_BYPASS >> S2CR_TYPE_SHIFT)
+#define TYPE_FAULT (S2CR_TYPE_FAULT >> S2CR_TYPE_SHIFT)
+
+struct static_cbndx_entry {
+ struct list_head list;
+ u8 cbndx;
+ u8 smr_idx;
+ u16 sid;
+ u8 type;
+};
+
+struct arm_iommus_node {
+ struct device_node *master;
+ struct list_head list;
+ struct list_head iommuspec_list;
+};
+
+struct arm_iommus_spec {
+ struct of_phandle_args iommu_spec;
+ struct list_head list;
+};
+
+static LIST_HEAD(iommus_nodes);
+
+static int arm_smmu_enable_clocks_atomic(struct arm_smmu_device *smmu);
+static void arm_smmu_disable_clocks_atomic(struct arm_smmu_device *smmu);
+static void arm_smmu_prepare_pgtable(void *addr, void *cookie);
+static void arm_smmu_unprepare_pgtable(void *cookie, void *addr, size_t size);
+static phys_addr_t arm_smmu_iova_to_phys_hard(struct iommu_domain *domain,
+ dma_addr_t iova);
+static phys_addr_t arm_smmu_iova_to_phys_hard_no_halt(
+ struct iommu_domain *domain, dma_addr_t iova);
+static int arm_smmu_wait_for_halt(struct arm_smmu_device *smmu);
+static int arm_smmu_halt_nowait(struct arm_smmu_device *smmu);
+static void arm_smmu_resume(struct arm_smmu_device *smmu);
+static phys_addr_t arm_smmu_iova_to_phys(struct iommu_domain *domain,
+ dma_addr_t iova);
+static int arm_smmu_assign_table(struct arm_smmu_domain *smmu_domain);
+static void arm_smmu_unassign_table(struct arm_smmu_domain *smmu_domain);
+static int arm_smmu_halt(struct arm_smmu_device *smmu);
+static void arm_smmu_device_reset(struct arm_smmu_device *smmu);
+static size_t arm_smmu_unmap(struct iommu_domain *domain, unsigned long iova,
+ size_t size);
+static bool arm_smmu_is_master_side_secure(struct arm_smmu_domain *smmu_domain);
+static bool arm_smmu_is_static_cb(struct arm_smmu_device *smmu);
+static bool arm_smmu_is_slave_side_secure(struct arm_smmu_domain *smmu_domain);
+static bool arm_smmu_has_secure_vmid(struct arm_smmu_domain *smmu_domain);
+static bool arm_smmu_is_iova_coherent(struct iommu_domain *domain,
+ dma_addr_t iova);
+static uint64_t arm_smmu_iova_to_pte(struct iommu_domain *domain,
+ dma_addr_t iova);
+
+static int arm_smmu_enable_s1_translations(struct arm_smmu_domain *smmu_domain);
+
static struct arm_smmu_domain *to_smmu_domain(struct iommu_domain *dom)
{
return container_of(dom, struct arm_smmu_domain, domain);
@@ -377,7 +574,7 @@ static void parse_driver_options(struct arm_smmu_device *smmu)
if (of_property_read_bool(smmu->dev->of_node,
arm_smmu_options[i].prop)) {
smmu->options |= arm_smmu_options[i].opt;
- dev_notice(smmu->dev, "option %s\n",
+ dev_dbg(smmu->dev, "option %s\n",
arm_smmu_options[i].prop);
}
} while (arm_smmu_options[++i].opt);
@@ -417,6 +614,28 @@ static struct arm_smmu_master *find_smmu_master(struct arm_smmu_device *smmu,
return NULL;
}
+static struct arm_smmu_master *find_smmu_master_by_sid(
+ struct arm_smmu_device *smmu, u32 sid)
+{
+ struct rb_node *next;
+ struct arm_smmu_master *master;
+ struct arm_smmu_master_cfg *cfg;
+ int i;
+
+ next = rb_first(&smmu->masters);
+ for (; next; next = rb_next(next)) {
+ master = container_of(next, struct arm_smmu_master, node);
+ cfg = &master->cfg;
+
+ for (i = 0; i < cfg->num_streamids; i++) {
+ if (cfg->streamids[i] == sid)
+ return master;
+ }
+ }
+
+ return NULL;
+}
+
static struct arm_smmu_master_cfg *
find_smmu_master_cfg(struct device *dev)
{
@@ -456,25 +675,32 @@ static int insert_smmu_master(struct arm_smmu_device *smmu,
return 0;
}
+struct iommus_entry {
+ struct list_head list;
+ struct device_node *node;
+ u16 streamids[MAX_MASTER_STREAMIDS];
+ int num_sids;
+};
+
static int register_smmu_master(struct arm_smmu_device *smmu,
- struct device *dev,
- struct of_phandle_args *masterspec)
+ struct iommus_entry *entry)
{
int i;
struct arm_smmu_master *master;
+ struct device *dev = smmu->dev;
- master = find_smmu_master(smmu, masterspec->np);
+ master = find_smmu_master(smmu, entry->node);
if (master) {
dev_err(dev,
"rejecting multiple registrations for master device %s\n",
- masterspec->np->name);
+ entry->node->name);
return -EBUSY;
}
- if (masterspec->args_count > MAX_MASTER_STREAMIDS) {
+ if (entry->num_sids > MAX_MASTER_STREAMIDS) {
dev_err(dev,
"reached maximum number (%d) of stream IDs for master device %s\n",
- MAX_MASTER_STREAMIDS, masterspec->np->name);
+ MAX_MASTER_STREAMIDS, entry->node->name);
return -ENOSPC;
}
@@ -482,22 +708,84 @@ static int register_smmu_master(struct arm_smmu_device *smmu,
if (!master)
return -ENOMEM;
- master->of_node = masterspec->np;
- master->cfg.num_streamids = masterspec->args_count;
+ master->of_node = entry->node;
+ master->cfg.num_streamids = entry->num_sids;
- for (i = 0; i < master->cfg.num_streamids; ++i) {
- u16 streamid = masterspec->args[i];
+ for (i = 0; i < master->cfg.num_streamids; ++i)
+ master->cfg.streamids[i] = entry->streamids[i];
- if (!(smmu->features & ARM_SMMU_FEAT_STREAM_MATCH) &&
- (streamid >= smmu->num_mapping_groups)) {
- dev_err(dev,
- "stream ID for master device %s greater than maximum allowed (%d)\n",
- masterspec->np->name, smmu->num_mapping_groups);
- return -ERANGE;
+ return insert_smmu_master(smmu, master);
+}
+
+static int arm_smmu_parse_iommus_properties(struct arm_smmu_device *smmu)
+{
+ struct arm_iommus_node *node, *nex;
+
+ list_for_each_entry_safe(node, nex, &iommus_nodes, list) {
+ struct iommus_entry *entry, *next;
+ struct arm_iommus_spec *iommuspec_node, *n;
+ LIST_HEAD(iommus);
+ int node_found = 0;
+
+ list_for_each_entry_safe(iommuspec_node, n,
+ &node->iommuspec_list, list) {
+ if (iommuspec_node->iommu_spec.np != smmu->dev->of_node)
+ continue;
+
+ /*
+ * Since each master node will have iommu spec(s) of the
+ * same device, we can delete this master node after
+ * the devices are registered.
+ */
+ node_found = 1;
+
+ list_for_each_entry(entry, &iommus, list)
+ if (entry->node == node->master)
+ break;
+ if (&entry->list == &iommus) {
+ entry = devm_kzalloc(smmu->dev, sizeof(*entry),
+ GFP_KERNEL);
+ if (!entry)
+ return -ENOMEM;
+ entry->node = node->master;
+ list_add(&entry->list, &iommus);
+ }
+ switch (iommuspec_node->iommu_spec.args_count) {
+ case 0:
+ /*
+ * For pci-e devices the SIDs are provided
+ * at device attach time.
+ */
+ break;
+ case 1:
+ entry->num_sids++;
+ entry->streamids[entry->num_sids - 1]
+ = iommuspec_node->iommu_spec.args[0];
+ break;
+ default:
+ BUG();
+ }
+ list_del(&iommuspec_node->list);
+ kfree(iommuspec_node);
+ }
+
+ list_for_each_entry_safe(entry, next, &iommus, list) {
+ int rc = register_smmu_master(smmu, entry);
+
+ if (rc)
+ dev_err(smmu->dev, "Couldn't register %s\n",
+ entry->node->name);
+ list_del(&entry->list);
+ devm_kfree(smmu->dev, entry);
+ }
+
+ if (node_found) {
+ list_del(&node->list);
+ kfree(node);
}
- master->cfg.streamids[i] = streamid;
}
- return insert_smmu_master(smmu, master);
+
+ return 0;
}
static struct arm_smmu_device *find_smmu_for_device(struct device *dev)
@@ -530,11 +818,272 @@ static int __arm_smmu_alloc_bitmap(unsigned long *map, int start, int end)
return idx;
}
+static int __arm_smmu_set_bitmap(unsigned long *map, int idx)
+{
+ return test_and_set_bit(idx, map);
+}
+
+static struct static_cbndx_entry *arm_smmu_get_static_entry_from_sid(
+ struct arm_smmu_device *smmu, int sid)
+{
+ struct static_cbndx_entry *entry;
+
+ list_for_each_entry(entry, &smmu->static_cbndx_list, list) {
+ if (entry->sid == sid)
+ return entry;
+ }
+
+ return NULL;
+}
+
+static struct static_cbndx_entry *arm_smmu_get_static_entry_from_context(
+ struct arm_smmu_device *smmu, int idx)
+{
+ struct static_cbndx_entry *entry;
+
+ list_for_each_entry(entry, &smmu->static_cbndx_list, list) {
+ if (entry->type == TYPE_TRANS && entry->cbndx == idx)
+ return entry;
+ }
+
+ return NULL;
+}
+
+static struct static_cbndx_entry *arm_smmu_get_static_entry_from_smr(
+ struct arm_smmu_device *smmu, int idx)
+{
+ struct static_cbndx_entry *entry;
+
+ list_for_each_entry(entry, &smmu->static_cbndx_list, list) {
+ if (entry->smr_idx == idx)
+ return entry;
+ }
+
+ return NULL;
+}
+
+static int arm_smmu_alloc_smr_idx(struct arm_smmu_device *smmu, int start,
+ int end, int sid)
+{
+ struct static_cbndx_entry *entry = arm_smmu_get_static_entry_from_sid(
+ smmu, sid);
+
+ if (entry)
+ return entry->smr_idx;
+ else
+ return __arm_smmu_alloc_bitmap(smmu->smr_map, start, end);
+}
+
+static int arm_smmu_alloc_context_idx(struct arm_smmu_device *smmu, int start,
+ int end, u16 *streamids, int num_streamids)
+{
+ struct static_cbndx_entry *entry = NULL;
+ int i;
+
+ for (i = 0; i < num_streamids; ++i) {
+ entry = arm_smmu_get_static_entry_from_sid(smmu, streamids[i]);
+ if (entry && entry->type == TYPE_TRANS)
+ break;
+ }
+
+ if (entry && entry->type == TYPE_TRANS)
+ return entry->cbndx;
+ else
+ return __arm_smmu_alloc_bitmap(smmu->context_map, start, end);
+}
+
static void __arm_smmu_free_bitmap(unsigned long *map, int idx)
{
clear_bit(idx, map);
}
+static void arm_smmu_free_smr_idx(struct arm_smmu_device *smmu, int idx)
+{
+ struct static_cbndx_entry *entry = arm_smmu_get_static_entry_from_smr(
+ smmu, idx);
+
+ if (!entry)
+ __arm_smmu_free_bitmap(smmu->smr_map, idx);
+}
+
+static void arm_smmu_free_context_idx(struct arm_smmu_device *smmu, int idx)
+{
+ struct static_cbndx_entry *entry =
+ arm_smmu_get_static_entry_from_context(smmu, idx);
+
+ if (!entry)
+ __arm_smmu_free_bitmap(smmu->context_map, idx);
+}
+
+static void arm_smmu_unprepare_clocks(struct arm_smmu_device *smmu)
+{
+ int i;
+
+ for (i = smmu->num_clocks; i; --i)
+ clk_unprepare(smmu->clocks[i - 1]);
+}
+
+static int arm_smmu_prepare_clocks(struct arm_smmu_device *smmu)
+{
+ int i, ret = 0;
+
+ for (i = 0; i < smmu->num_clocks; ++i) {
+ ret = clk_prepare(smmu->clocks[i]);
+ if (ret) {
+ dev_err(smmu->dev, "Couldn't prepare clock #%d\n", i);
+ while (i--)
+ clk_unprepare(smmu->clocks[i]);
+ break;
+ }
+ }
+ return ret;
+}
+
+static int arm_smmu_request_bus(struct arm_smmu_device *smmu)
+{
+ if (!smmu->bus_client)
+ return 0;
+ return msm_bus_scale_client_update_request(smmu->bus_client, 1);
+}
+
+static int arm_smmu_unrequest_bus(struct arm_smmu_device *smmu)
+{
+ if (!smmu->bus_client)
+ return 0;
+ return msm_bus_scale_client_update_request(smmu->bus_client, 0);
+}
+
+static int arm_smmu_disable_regulators(struct arm_smmu_device *smmu)
+{
+ int ret = 0;
+
+ mutex_lock(&smmu->power_lock);
+ if (smmu->power_count == 0) {
+ WARN(1, "%s: Mismatched power count\n", dev_name(smmu->dev));
+ mutex_unlock(&smmu->power_lock);
+ return -EINVAL;
+ } else if (smmu->power_count > 1) {
+ smmu->power_count -= 1;
+ mutex_unlock(&smmu->power_lock);
+ return 0;
+ }
+
+ arm_smmu_unprepare_clocks(smmu);
+ arm_smmu_unrequest_bus(smmu);
+ if (smmu->gdsc) {
+ ret = regulator_disable_deferred(smmu->gdsc,
+ smmu->regulator_defer);
+ WARN(ret, "%s: Regulator disable failed\n",
+ dev_name(smmu->dev));
+ }
+
+ smmu->power_count = 0;
+ mutex_unlock(&smmu->power_lock);
+ return ret;
+}
+
+static int arm_smmu_enable_regulators(struct arm_smmu_device *smmu)
+{
+ int ret;
+
+ mutex_lock(&smmu->power_lock);
+ if (smmu->power_count) {
+ smmu->power_count++;
+ mutex_unlock(&smmu->power_lock);
+ return 0;
+ }
+
+ if (smmu->gdsc) {
+ ret = regulator_enable(smmu->gdsc);
+ if (WARN_ON_ONCE(ret))
+ goto out;
+ }
+
+ ret = arm_smmu_request_bus(smmu);
+ if (WARN_ON_ONCE(ret))
+ goto out_reg;
+
+ ret = arm_smmu_prepare_clocks(smmu);
+ if (WARN_ON_ONCE(ret))
+ goto out_bus;
+
+ smmu->power_count = 1;
+ mutex_unlock(&smmu->power_lock);
+ return ret;
+
+out_bus:
+ arm_smmu_unrequest_bus(smmu);
+out_reg:
+ if (smmu->gdsc)
+ regulator_disable(smmu->gdsc);
+out:
+ mutex_unlock(&smmu->power_lock);
+ return ret;
+}
+
+static int arm_smmu_enable_clocks(struct arm_smmu_device *smmu)
+{
+ int ret = 0;
+
+ ret = arm_smmu_enable_regulators(smmu);
+ if (unlikely(ret))
+ return ret;
+ ret = arm_smmu_enable_clocks_atomic(smmu);
+ if (unlikely(ret))
+ arm_smmu_disable_regulators(smmu);
+
+ return ret;
+}
+
+static void arm_smmu_disable_clocks(struct arm_smmu_device *smmu)
+{
+ arm_smmu_disable_clocks_atomic(smmu);
+ arm_smmu_disable_regulators(smmu);
+}
+
+/* Clocks must be prepared before this (arm_smmu_prepare_clocks) */
+static int arm_smmu_enable_clocks_atomic(struct arm_smmu_device *smmu)
+{
+ int i, ret = 0;
+ unsigned long flags;
+
+ spin_lock_irqsave(&smmu->clock_refs_lock, flags);
+ if (smmu->clock_refs_count++ > 0) {
+ spin_unlock_irqrestore(&smmu->clock_refs_lock, flags);
+ return 0;
+ }
+
+ for (i = 0; i < smmu->num_clocks; ++i) {
+ ret = clk_enable(smmu->clocks[i]);
+ if (WARN_ON_ONCE(ret)) {
+ dev_err(smmu->dev, "Couldn't enable clock #%d\n", i);
+ while (i--)
+ clk_disable(smmu->clocks[i]);
+ smmu->clock_refs_count--;
+ break;
+ }
+ }
+ spin_unlock_irqrestore(&smmu->clock_refs_lock, flags);
+ return ret;
+}
+
+/* Clocks should be unprepared after this (arm_smmu_unprepare_clocks) */
+static void arm_smmu_disable_clocks_atomic(struct arm_smmu_device *smmu)
+{
+ int i;
+ unsigned long flags;
+
+ spin_lock_irqsave(&smmu->clock_refs_lock, flags);
+ if (smmu->clock_refs_count-- > 1) {
+ spin_unlock_irqrestore(&smmu->clock_refs_lock, flags);
+ return;
+ }
+
+ for (i = smmu->num_clocks; i; --i)
+ clk_disable(smmu->clocks[i - 1]);
+ spin_unlock_irqrestore(&smmu->clock_refs_lock, flags);
+}
+
/* Wait for any pending TLB invalidations to complete */
static void __arm_smmu_tlb_sync(struct arm_smmu_device *smmu)
{
@@ -554,12 +1103,30 @@ static void __arm_smmu_tlb_sync(struct arm_smmu_device *smmu)
}
}
+static void arm_smmu_tlb_sync_cb(struct arm_smmu_device *smmu,
+ int cbndx)
+{
+ void __iomem *base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cbndx);
+ u32 val;
+
+ writel_relaxed(0, base + ARM_SMMU_CB_TLBSYNC);
+ if (readl_poll_timeout_atomic(base + ARM_SMMU_CB_TLBSTATUS, val,
+ !(val & TLBSTATUS_SACTIVE),
+ 0, TLB_LOOP_TIMEOUT))
+ dev_err(smmu->dev, "TLBSYNC timeout!\n");
+}
+
static void arm_smmu_tlb_sync(void *cookie)
{
struct arm_smmu_domain *smmu_domain = cookie;
- __arm_smmu_tlb_sync(smmu_domain->smmu);
+
+ if (smmu_domain->smmu == NULL)
+ return;
+
+ arm_smmu_tlb_sync_cb(smmu_domain->smmu, smmu_domain->cfg.cbndx);
}
+/* Must be called with clocks/regulators enabled */
static void arm_smmu_tlb_inv_context(void *cookie)
{
struct arm_smmu_domain *smmu_domain = cookie;
@@ -568,19 +1135,23 @@ static void arm_smmu_tlb_inv_context(void *cookie)
bool stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS;
void __iomem *base;
+ if (!smmu)
+ return;
+
if (stage1) {
base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
writel_relaxed(ARM_SMMU_CB_ASID(cfg),
base + ARM_SMMU_CB_S1_TLBIASID);
+ arm_smmu_tlb_sync_cb(smmu, cfg->cbndx);
} else {
base = ARM_SMMU_GR0(smmu);
writel_relaxed(ARM_SMMU_CB_VMID(cfg),
base + ARM_SMMU_GR0_TLBIVMID);
+ __arm_smmu_tlb_sync(smmu);
}
-
- __arm_smmu_tlb_sync(smmu);
}
+/* Must be called with clocks/regulators enabled */
static void arm_smmu_tlb_inv_range_nosync(unsigned long iova, size_t size,
bool leaf, void *cookie)
{
@@ -589,6 +1160,11 @@ static void arm_smmu_tlb_inv_range_nosync(unsigned long iova, size_t size,
struct arm_smmu_device *smmu = smmu_domain->smmu;
bool stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS;
void __iomem *reg;
+ int atomic_ctx = smmu_domain->attributes & (1 << DOMAIN_ATTR_ATOMIC);
+
+ BUG_ON(atomic_ctx && !smmu);
+ if (!smmu)
+ return;
if (stage1) {
reg = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
@@ -618,61 +1194,331 @@ static void arm_smmu_tlb_inv_range_nosync(unsigned long iova, size_t size,
}
}
+static void arm_smmu_tlbi_domain(struct iommu_domain *domain)
+{
+ arm_smmu_tlb_inv_context(to_smmu_domain(domain));
+}
+
+static int arm_smmu_enable_config_clocks(struct iommu_domain *domain)
+{
+ struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
+
+ return arm_smmu_enable_clocks(smmu_domain->smmu);
+}
+
+static void arm_smmu_disable_config_clocks(struct iommu_domain *domain)
+{
+ struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
+
+ arm_smmu_disable_clocks(smmu_domain->smmu);
+}
+
+struct arm_smmu_secure_pool_chunk {
+ void *addr;
+ size_t size;
+ struct list_head list;
+};
+
+static void *arm_smmu_secure_pool_remove(struct arm_smmu_domain *smmu_domain,
+ size_t size)
+{
+ struct arm_smmu_secure_pool_chunk *it;
+
+ list_for_each_entry(it, &smmu_domain->secure_pool_list, list) {
+ if (it->size == size) {
+ void *addr = it->addr;
+
+ list_del(&it->list);
+ kfree(it);
+ return addr;
+ }
+ }
+
+ return NULL;
+}
+
+static int arm_smmu_secure_pool_add(struct arm_smmu_domain *smmu_domain,
+ void *addr, size_t size)
+{
+ struct arm_smmu_secure_pool_chunk *chunk;
+
+ chunk = kmalloc(sizeof(*chunk), GFP_ATOMIC);
+ if (!chunk)
+ return -ENOMEM;
+
+ chunk->addr = addr;
+ chunk->size = size;
+ memset(addr, 0, size);
+ list_add(&chunk->list, &smmu_domain->secure_pool_list);
+
+ return 0;
+}
+
+static void arm_smmu_secure_pool_destroy(struct arm_smmu_domain *smmu_domain)
+{
+ struct arm_smmu_secure_pool_chunk *it, *i;
+
+ list_for_each_entry_safe(it, i, &smmu_domain->secure_pool_list, list) {
+ arm_smmu_unprepare_pgtable(smmu_domain, it->addr, it->size);
+ /* pages will be freed later (after being unassigned) */
+ list_del(&it->list);
+ kfree(it);
+ }
+}
+
+static void *arm_smmu_alloc_pages_exact(void *cookie,
+ size_t size, gfp_t gfp_mask)
+{
+ void *ret;
+ struct arm_smmu_domain *smmu_domain = cookie;
+
+ if (!arm_smmu_is_master_side_secure(smmu_domain))
+ return alloc_pages_exact(size, gfp_mask);
+
+ ret = arm_smmu_secure_pool_remove(smmu_domain, size);
+ if (ret)
+ return ret;
+
+ ret = alloc_pages_exact(size, gfp_mask);
+ if (ret)
+ arm_smmu_prepare_pgtable(ret, cookie);
+
+ return ret;
+}
+
+static void arm_smmu_free_pages_exact(void *cookie, void *virt, size_t size)
+{
+ struct arm_smmu_domain *smmu_domain = cookie;
+
+ if (!arm_smmu_is_master_side_secure(smmu_domain)) {
+ free_pages_exact(virt, size);
+ return;
+ }
+
+ if (arm_smmu_secure_pool_add(smmu_domain, virt, size))
+ arm_smmu_unprepare_pgtable(smmu_domain, virt, size);
+}
+
static struct iommu_gather_ops arm_smmu_gather_ops = {
.tlb_flush_all = arm_smmu_tlb_inv_context,
.tlb_add_flush = arm_smmu_tlb_inv_range_nosync,
.tlb_sync = arm_smmu_tlb_sync,
+ .alloc_pages_exact = arm_smmu_alloc_pages_exact,
+ .free_pages_exact = arm_smmu_free_pages_exact,
};
+static phys_addr_t arm_smmu_verify_fault(struct iommu_domain *domain,
+ dma_addr_t iova, u32 fsr)
+{
+ struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
+ struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
+ struct arm_smmu_device *smmu;
+ void __iomem *cb_base;
+ u64 sctlr, sctlr_orig;
+ phys_addr_t phys;
+
+ smmu = smmu_domain->smmu;
+ cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
+
+ arm_smmu_halt_nowait(smmu);
+
+ writel_relaxed(RESUME_TERMINATE, cb_base + ARM_SMMU_CB_RESUME);
+
+ arm_smmu_wait_for_halt(smmu);
+
+ /* clear FSR to allow ATOS to log any faults */
+ writel_relaxed(fsr, cb_base + ARM_SMMU_CB_FSR);
+
+ /* disable stall mode momentarily */
+ sctlr_orig = readl_relaxed(cb_base + ARM_SMMU_CB_SCTLR);
+ sctlr = sctlr_orig & ~SCTLR_CFCFG;
+ writel_relaxed(sctlr, cb_base + ARM_SMMU_CB_SCTLR);
+
+ phys = arm_smmu_iova_to_phys_hard_no_halt(domain, iova);
+
+ if (!phys) {
+ dev_err(smmu->dev,
+ "ATOS failed. Will issue a TLBIALL and try again...\n");
+ arm_smmu_tlb_inv_context(smmu_domain);
+ phys = arm_smmu_iova_to_phys_hard_no_halt(domain, iova);
+ if (phys)
+ dev_err(smmu->dev,
+ "ATOS succeeded this time. Maybe we missed a TLB invalidation while messing with page tables earlier??\n");
+ else
+ dev_err(smmu->dev,
+ "ATOS still failed. If the page tables look good (check the software table walk) then hardware might be misbehaving.\n");
+ }
+
+ /* restore SCTLR */
+ writel_relaxed(sctlr_orig, cb_base + ARM_SMMU_CB_SCTLR);
+
+ arm_smmu_resume(smmu);
+
+ return phys;
+}
+
static irqreturn_t arm_smmu_context_fault(int irq, void *dev)
{
- int flags, ret;
- u32 fsr, far, fsynr, resume;
- unsigned long iova;
+ int flags, ret, tmp;
+ u32 fsr, fsynr, resume;
+ unsigned long iova, far;
struct iommu_domain *domain = dev;
struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
- struct arm_smmu_device *smmu = smmu_domain->smmu;
+ struct arm_smmu_device *smmu;
void __iomem *cb_base;
+ bool ctx_hang_errata;
+ bool fatal_asf;
+ void __iomem *gr1_base;
+ phys_addr_t phys_soft;
+ u32 sid;
+ bool non_fatal_fault = smmu_domain->non_fatal_faults;
+ struct arm_smmu_master *master;
+
+ static DEFINE_RATELIMIT_STATE(_rs,
+ DEFAULT_RATELIMIT_INTERVAL,
+ DEFAULT_RATELIMIT_BURST);
+
+ mutex_lock(&smmu_domain->init_mutex);
+ smmu = smmu_domain->smmu;
+ if (!smmu) {
+ ret = IRQ_HANDLED;
+ pr_err("took a fault on a detached domain (%p)\n", domain);
+ goto out_unlock;
+ }
+ ctx_hang_errata = smmu->options & ARM_SMMU_OPT_ERRATA_CTX_FAULT_HANG;
+ fatal_asf = smmu->options & ARM_SMMU_OPT_FATAL_ASF;
+ if (arm_smmu_enable_clocks(smmu)) {
+ ret = IRQ_NONE;
+ goto out_unlock;
+ }
+
+ gr1_base = ARM_SMMU_GR1(smmu);
cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
fsr = readl_relaxed(cb_base + ARM_SMMU_CB_FSR);
- if (!(fsr & FSR_FAULT))
- return IRQ_NONE;
+ if (!(fsr & FSR_FAULT)) {
+ arm_smmu_disable_clocks(smmu);
+ ret = IRQ_NONE;
+ goto out_unlock;
+ }
- if (fsr & FSR_IGN)
- dev_err_ratelimited(smmu->dev,
- "Unexpected context fault (fsr 0x%x)\n",
- fsr);
+ if (fatal_asf && (fsr & FSR_ASF)) {
+ dev_err(smmu->dev,
+ "Took an address size fault. Refusing to recover.\n");
+ BUG();
+ }
fsynr = readl_relaxed(cb_base + ARM_SMMU_CB_FSYNR0);
flags = fsynr & FSYNR0_WNR ? IOMMU_FAULT_WRITE : IOMMU_FAULT_READ;
+ if (fsr & FSR_TF)
+ flags |= IOMMU_FAULT_TRANSLATION;
+ if (fsr & FSR_PF)
+ flags |= IOMMU_FAULT_PERMISSION;
+ if (fsr & FSR_EF)
+ flags |= IOMMU_FAULT_EXTERNAL;
+ if (fsr & FSR_SS)
+ flags |= IOMMU_FAULT_TRANSACTION_STALLED;
far = readl_relaxed(cb_base + ARM_SMMU_CB_FAR_LO);
- iova = far;
#ifdef CONFIG_64BIT
- far = readl_relaxed(cb_base + ARM_SMMU_CB_FAR_HI);
- iova |= ((unsigned long)far << 32);
+ far |= ((u64)readl_relaxed(cb_base + ARM_SMMU_CB_FAR_HI)) << 32;
#endif
+ iova = far;
- if (!report_iommu_fault(domain, smmu->dev, iova, flags)) {
+ phys_soft = arm_smmu_iova_to_phys(domain, iova);
+ sid = readl_relaxed(gr1_base + ARM_SMMU_GR1_CBFRSYNRA(cfg->cbndx));
+ sid &= 0xffff;
+ master = find_smmu_master_by_sid(smmu, sid);
+ tmp = report_iommu_fault(domain, smmu->dev, iova, flags);
+ if (!tmp || (tmp == -EBUSY)) {
+ dev_dbg(smmu->dev,
+ "Context fault handled by client: iova=0x%08lx, fsr=0x%x, fsynr=0x%x, cb=%d\n",
+ iova, fsr, fsynr, cfg->cbndx);
+ dev_dbg(smmu->dev,
+ "soft iova-to-phys=%pa\n", &phys_soft);
ret = IRQ_HANDLED;
- resume = RESUME_RETRY;
+ resume = RESUME_TERMINATE;
} else {
- dev_err_ratelimited(smmu->dev,
- "Unhandled context fault: iova=0x%08lx, fsynr=0x%x, cb=%d\n",
- iova, fsynr, cfg->cbndx);
+ phys_addr_t phys_atos = arm_smmu_verify_fault(domain, iova,
+ fsr);
+
+ if (__ratelimit(&_rs)) {
+ dev_err(smmu->dev, "Context Fault for %s\n",
+ master ? master->of_node->name : "Unknown SID");
+
+ dev_err(smmu->dev,
+ "Unhandled context fault: iova=0x%08lx, fsr=0x%x, fsynr=0x%x, cb=%d\n",
+ iova, fsr, fsynr, cfg->cbndx);
+ dev_err(smmu->dev, "FAR = %016lx\n",
+ (unsigned long)far);
+ dev_err(smmu->dev,
+ "FSR = %08x [%s%s%s%s%s%s%s%s%s]\n",
+ fsr,
+ (fsr & 0x02) ? "TF " : "",
+ (fsr & 0x04) ? "AFF " : "",
+ (fsr & 0x08) ? "PF " : "",
+ (fsr & 0x10) ? "EF " : "",
+ (fsr & 0x20) ? "TLBMCF " : "",
+ (fsr & 0x40) ? "TLBLKF " : "",
+ (fsr & 0x80) ? "MHF " : "",
+ (fsr & 0x40000000) ? "SS " : "",
+ (fsr & 0x80000000) ? "MULTI " : "");
+ dev_err(smmu->dev,
+ "soft iova-to-phys=%pa\n", &phys_soft);
+ if (!phys_soft)
+ dev_err(smmu->dev,
+ "SOFTWARE TABLE WALK FAILED! Looks like %s accessed an unmapped address!\n",
+ dev_name(smmu->dev));
+ dev_err(smmu->dev,
+ "hard iova-to-phys (ATOS)=%pa\n", &phys_atos);
+ dev_err(smmu->dev, "SID=0x%x\n", sid);
+ }
ret = IRQ_NONE;
resume = RESUME_TERMINATE;
+ if (!non_fatal_fault) {
+ dev_err(smmu->dev,
+ "Unhandled context faults are fatal on this domain. Going down now...\n");
+ BUG();
+ }
}
- /* Clear the faulting FSR */
- writel(fsr, cb_base + ARM_SMMU_CB_FSR);
+ /*
+ * If the client returns -EBUSY, do not clear FSR and do not RESUME
+ * if stalled. This is required to keep the IOMMU client stalled on
+ * the outstanding fault. This gives the client a chance to take any
+ * debug action and then terminate the stalled transaction.
+ * So, the sequence in case of stall on fault should be:
+ * 1) Do not clear FSR or write to RESUME here
+ * 2) Client takes any debug action
+ * 3) Client terminates the stalled transaction and resumes the IOMMU
+ * 4) Client clears FSR. The FSR should only be cleared after 3) and
+ * not before so that the fault remains outstanding. This ensures
+ * SCTLR.HUPCF has the desired effect if subsequent transactions also
+ * need to be terminated.
+ */
+ if (tmp != -EBUSY) {
+ /* Clear the faulting FSR */
+ writel_relaxed(fsr, cb_base + ARM_SMMU_CB_FSR);
- /* Retry or terminate any stalled transactions */
- if (fsr & FSR_SS)
- writel_relaxed(resume, cb_base + ARM_SMMU_CB_RESUME);
+ /*
+ * Barrier required to ensure that the FSR is cleared
+ * before resuming SMMU operation
+ */
+ wmb();
+
+ /* Retry or terminate any stalled transactions */
+ if (fsr & FSR_SS) {
+ if (ctx_hang_errata)
+ arm_smmu_tlb_sync_cb(smmu, cfg->cbndx);
+ writel_relaxed(resume, cb_base + ARM_SMMU_CB_RESUME);
+ }
+ }
+
+ arm_smmu_disable_clocks(smmu);
+out_unlock:
+ mutex_unlock(&smmu_domain->init_mutex);
return ret;
}
@@ -683,13 +1529,18 @@ static irqreturn_t arm_smmu_global_fault(int irq, void *dev)
struct arm_smmu_device *smmu = dev;
void __iomem *gr0_base = ARM_SMMU_GR0_NS(smmu);
+ if (arm_smmu_enable_clocks(smmu))
+ return IRQ_NONE;
+
gfsr = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSR);
gfsynr0 = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSYNR0);
gfsynr1 = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSYNR1);
gfsynr2 = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSYNR2);
- if (!gfsr)
+ if (!gfsr) {
+ arm_smmu_disable_clocks(smmu);
return IRQ_NONE;
+ }
dev_err_ratelimited(smmu->dev,
"Unexpected global fault, this could be serious\n");
@@ -698,9 +1549,36 @@ static irqreturn_t arm_smmu_global_fault(int irq, void *dev)
gfsr, gfsynr0, gfsynr1, gfsynr2);
writel(gfsr, gr0_base + ARM_SMMU_GR0_sGFSR);
+ arm_smmu_disable_clocks(smmu);
return IRQ_HANDLED;
}
+static void arm_smmu_trigger_fault(struct iommu_domain *domain,
+ unsigned long flags)
+{
+ struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
+ struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
+ struct arm_smmu_device *smmu;
+ void __iomem *cb_base;
+
+ if (!smmu_domain->smmu) {
+ pr_err("Can't trigger faults on non-attached domains\n");
+ return;
+ }
+
+ smmu = smmu_domain->smmu;
+
+ cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
+ if (arm_smmu_enable_clocks(smmu))
+ return;
+ dev_err(smmu->dev, "Writing 0x%lx to FSRRESTORE on cb %d\n",
+ flags, cfg->cbndx);
+ writel_relaxed(flags, cb_base + ARM_SMMU_CB_FSRRESTORE);
+ /* give the interrupt time to fire... */
+ msleep(1000);
+ arm_smmu_disable_clocks(smmu);
+}
+
static void arm_smmu_init_context_bank(struct arm_smmu_domain *smmu_domain,
struct io_pgtable_cfg *pgtbl_cfg)
{
@@ -723,6 +1601,9 @@ static void arm_smmu_init_context_bank(struct arm_smmu_domain *smmu_domain,
*/
#ifdef CONFIG_64BIT
reg = CBA2R_RW64_64BIT;
+ if (!arm_smmu_has_secure_vmid(smmu_domain) &&
+ arm_smmu_is_static_cb(smmu))
+ msm_tz_set_cb_format(smmu->sec_id, cfg->cbndx);
#else
reg = CBA2R_RW64_32BIT;
#endif
@@ -741,9 +1622,8 @@ static void arm_smmu_init_context_bank(struct arm_smmu_domain *smmu_domain,
if (stage1) {
reg |= (CBAR_S1_BPSHCFG_NSH << CBAR_S1_BPSHCFG_SHIFT) |
(CBAR_S1_MEMATTR_WB << CBAR_S1_MEMATTR_SHIFT);
- } else {
- reg |= ARM_SMMU_CB_VMID(cfg) << CBAR_VMID_SHIFT;
}
+ reg |= ARM_SMMU_CB_VMID(cfg) << CBAR_VMID_SHIFT;
writel_relaxed(reg, gr1_base + ARM_SMMU_GR1_CBAR(cfg->cbndx));
/* TTBRs */
@@ -767,7 +1647,6 @@ static void arm_smmu_init_context_bank(struct arm_smmu_domain *smmu_domain,
writel_relaxed(reg, cb_base + ARM_SMMU_CB_TTBCR);
if (smmu->version > ARM_SMMU_V1) {
reg = pgtbl_cfg->arm_lpae_s1_cfg.tcr >> 32;
- reg |= TTBCR2_SEP_UPSTREAM;
writel_relaxed(reg, cb_base + ARM_SMMU_CB_TTBCR2);
}
} else {
@@ -783,8 +1662,25 @@ static void arm_smmu_init_context_bank(struct arm_smmu_domain *smmu_domain,
writel_relaxed(reg, cb_base + ARM_SMMU_CB_S1_MAIR1);
}
+ if (smmu->model == SMMU_MODEL_QCOM_V2) {
+ reg = ACTLR_QCOM_ISH << ACTLR_QCOM_ISH_SHIFT |
+ ACTLR_QCOM_OSH << ACTLR_QCOM_OSH_SHIFT |
+ ACTLR_QCOM_NSH << ACTLR_QCOM_NSH_SHIFT;
+ writel_relaxed(reg, cb_base + ARM_SMMU_CB_ACTLR);
+ }
+
/* SCTLR */
- reg = SCTLR_CFCFG | SCTLR_CFIE | SCTLR_CFRE | SCTLR_M | SCTLR_EAE_SBOP;
+ reg = SCTLR_CFCFG | SCTLR_CFIE | SCTLR_CFRE | SCTLR_EAE_SBOP;
+
+ if (smmu_domain->attributes & (1 << DOMAIN_ATTR_CB_STALL_DISABLE)) {
+ reg &= ~SCTLR_CFCFG;
+ reg |= SCTLR_HUPCF;
+ }
+
+ if ((!(smmu_domain->attributes & (1 << DOMAIN_ATTR_S1_BYPASS)) &&
+ !(smmu_domain->attributes & (1 << DOMAIN_ATTR_EARLY_MAP))) ||
+ !stage1)
+ reg |= SCTLR_M;
if (stage1)
reg |= SCTLR_S1_ASIDPNE;
#ifdef __BIG_ENDIAN
@@ -793,20 +1689,107 @@ static void arm_smmu_init_context_bank(struct arm_smmu_domain *smmu_domain,
writel_relaxed(reg, cb_base + ARM_SMMU_CB_SCTLR);
}
+static bool arm_smmu_is_static_cb(struct arm_smmu_device *smmu)
+{
+ return smmu->options & ARM_SMMU_OPT_STATIC_CB;
+}
+
+static bool arm_smmu_has_secure_vmid(struct arm_smmu_domain *smmu_domain)
+{
+ return smmu_domain->secure_vmid != VMID_INVAL;
+}
+
+static bool arm_smmu_is_slave_side_secure(struct arm_smmu_domain *smmu_domain)
+{
+ return arm_smmu_has_secure_vmid(smmu_domain)
+ && smmu_domain->slave_side_secure;
+}
+
+static bool arm_smmu_is_master_side_secure(struct arm_smmu_domain *smmu_domain)
+{
+ return arm_smmu_has_secure_vmid(smmu_domain)
+ && !smmu_domain->slave_side_secure;
+}
+
+static void arm_smmu_secure_domain_lock(struct arm_smmu_domain *smmu_domain)
+{
+ if (arm_smmu_is_master_side_secure(smmu_domain))
+ mutex_lock(&smmu_domain->assign_lock);
+}
+
+static void arm_smmu_secure_domain_unlock(struct arm_smmu_domain *smmu_domain)
+{
+ if (arm_smmu_is_master_side_secure(smmu_domain))
+ mutex_unlock(&smmu_domain->assign_lock);
+}
+
+static unsigned long arm_smmu_pgtbl_lock(struct arm_smmu_domain *smmu_domain)
+{
+ unsigned long flags = 0;
+
+ if (arm_smmu_is_slave_side_secure(smmu_domain))
+ mutex_lock(&smmu_domain->pgtbl_mutex_lock);
+ else
+ spin_lock_irqsave(&smmu_domain->pgtbl_spin_lock, flags);
+
+ return flags;
+}
+
+static void arm_smmu_pgtbl_unlock(struct arm_smmu_domain *smmu_domain,
+ unsigned long flags)
+{
+ if (arm_smmu_is_slave_side_secure(smmu_domain))
+ mutex_unlock(&smmu_domain->pgtbl_mutex_lock);
+ else
+ spin_unlock_irqrestore(&smmu_domain->pgtbl_spin_lock, flags);
+}
+
+static int arm_smmu_restore_sec_cfg(struct arm_smmu_device *smmu)
+{
+ int ret;
+ u64 scm_ret = 0;
+
+ if (!arm_smmu_is_static_cb(smmu))
+ return 0;
+
+ ret = scm_restore_sec_cfg(smmu->sec_id, 0x0, &scm_ret);
+ if (ret || scm_ret) {
+ pr_err("scm call IOMMU_SECURE_CFG failed\n");
+ return -EINVAL;
+ }
+
+ return 0;
+}
+
+static bool is_iommu_pt_coherent(struct arm_smmu_domain *smmu_domain)
+{
+ if (smmu_domain->attributes &
+ (1 << DOMAIN_ATTR_PAGE_TABLE_FORCE_COHERENT))
+ return true;
+ else if (smmu_domain->smmu && smmu_domain->smmu->dev)
+ return smmu_domain->smmu->dev->archdata.dma_coherent;
+ else
+ return false;
+}
+
static int arm_smmu_init_domain_context(struct iommu_domain *domain,
- struct arm_smmu_device *smmu)
+ struct arm_smmu_device *smmu,
+ struct arm_smmu_master_cfg *master_cfg)
{
int irq, start, ret = 0;
unsigned long ias, oas;
+ int sep = 0;
struct io_pgtable_ops *pgtbl_ops;
- struct io_pgtable_cfg pgtbl_cfg;
enum io_pgtable_fmt fmt;
struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
+ bool is_fast = smmu_domain->attributes & (1 << DOMAIN_ATTR_FAST);
+ unsigned long quirks =
+ smmu_domain->attributes & (1 << DOMAIN_ATTR_ENABLE_TTBR1) ?
+ IO_PGTABLE_QUIRK_ARM_TTBR1 : 0;
- mutex_lock(&smmu_domain->init_mutex);
if (smmu_domain->smmu)
- goto out_unlock;
+ goto out;
/*
* Mapping the requested stage onto what we support is surprisingly
@@ -837,9 +1820,27 @@ static int arm_smmu_init_domain_context(struct iommu_domain *domain,
start = smmu->num_s2_context_banks;
ias = smmu->va_size;
oas = smmu->ipa_size;
- if (IS_ENABLED(CONFIG_64BIT))
+ if (IS_ENABLED(CONFIG_64BIT)) {
fmt = ARM_64_LPAE_S1;
- else
+
+ if (quirks & IO_PGTABLE_QUIRK_ARM_TTBR1) {
+
+ /*
+ * When the UBS id is 5 we know that the bus
+ * size is 49 bits and that bit 48 is the fixed
+ * sign extension bit. For any other bus size
+ * we need to specify the sign extension bit
+ * and adjust the input size accordingly
+ */
+
+ if (smmu->ubs == 5) {
+ sep = 48;
+ } else {
+ sep = ias - 1;
+ ias--;
+ }
+ }
+ } else
fmt = ARM_32_LPAE_S1;
break;
case ARM_SMMU_DOMAIN_NESTED:
@@ -859,15 +1860,18 @@ static int arm_smmu_init_domain_context(struct iommu_domain *domain,
break;
default:
ret = -EINVAL;
- goto out_unlock;
+ goto out;
}
- ret = __arm_smmu_alloc_bitmap(smmu->context_map, start,
- smmu->num_context_banks);
- if (IS_ERR_VALUE(ret))
- goto out_unlock;
+ if (cfg->cbndx == INVALID_CBNDX) {
+ ret = arm_smmu_alloc_context_idx(smmu, start,
+ smmu->num_context_banks, master_cfg->streamids,
+ master_cfg->num_streamids);
+ if (IS_ERR_VALUE(ret))
+ goto out;
+ cfg->cbndx = ret;
+ }
- cfg->cbndx = ret;
if (smmu->version == ARM_SMMU_V1) {
cfg->irptndx = atomic_inc_return(&smmu->irptndx);
cfg->irptndx %= smmu->num_context_irqs;
@@ -875,50 +1879,82 @@ static int arm_smmu_init_domain_context(struct iommu_domain *domain,
cfg->irptndx = cfg->cbndx;
}
- pgtbl_cfg = (struct io_pgtable_cfg) {
- .pgsize_bitmap = arm_smmu_ops.pgsize_bitmap,
- .ias = ias,
- .oas = oas,
- .tlb = &arm_smmu_gather_ops,
- .iommu_dev = smmu->dev,
- };
-
smmu_domain->smmu = smmu;
- pgtbl_ops = alloc_io_pgtable_ops(fmt, &pgtbl_cfg, smmu_domain);
+
+ if (is_iommu_pt_coherent(smmu_domain))
+ quirks |= IO_PGTABLE_QUIRK_PAGE_TABLE_COHERENT;
+
+ if (arm_smmu_is_slave_side_secure(smmu_domain)) {
+ smmu_domain->pgtbl_cfg = (struct io_pgtable_cfg) {
+ .quirks = quirks,
+ .pgsize_bitmap = arm_smmu_ops.pgsize_bitmap,
+ .arm_msm_secure_cfg = {
+ .sec_id = smmu->sec_id,
+ .cbndx = cfg->cbndx,
+ },
+ .iommu_dev = smmu->dev,
+ };
+ fmt = ARM_MSM_SECURE;
+ } else {
+
+ smmu_domain->pgtbl_cfg = (struct io_pgtable_cfg) {
+ .quirks = quirks,
+ .pgsize_bitmap = arm_smmu_ops.pgsize_bitmap,
+ .ias = ias,
+ .oas = oas,
+ .sep = sep,
+ .tlb = &arm_smmu_gather_ops,
+ .iommu_dev = smmu->dev,
+ .iova_base = domain->geometry.aperture_start,
+ .iova_end = domain->geometry.aperture_end,
+ };
+ }
+
+ if (is_fast)
+ fmt = ARM_V8L_FAST;
+
+ cfg->asid = cfg->cbndx + 1;
+ cfg->vmid = cfg->cbndx + 2;
+ pgtbl_ops = alloc_io_pgtable_ops(fmt, &smmu_domain->pgtbl_cfg,
+ smmu_domain);
if (!pgtbl_ops) {
ret = -ENOMEM;
goto out_clear_smmu;
}
-
- /* Update our support page sizes to reflect the page table format */
- arm_smmu_ops.pgsize_bitmap = pgtbl_cfg.pgsize_bitmap;
+ /*
+ * assign any page table memory that might have been allocated
+ * during alloc_io_pgtable_ops
+ */
+ if (arm_smmu_is_master_side_secure(smmu_domain)) {
+ arm_smmu_secure_domain_lock(smmu_domain);
+ arm_smmu_assign_table(smmu_domain);
+ arm_smmu_secure_domain_unlock(smmu_domain);
+ }
/* Initialise the context bank with our page table cfg */
- arm_smmu_init_context_bank(smmu_domain, &pgtbl_cfg);
+ arm_smmu_init_context_bank(smmu_domain, &smmu_domain->pgtbl_cfg);
/*
* Request context fault interrupt. Do this last to avoid the
* handler seeing a half-initialised domain state.
*/
irq = smmu->irqs[smmu->num_global_irqs + cfg->irptndx];
- ret = request_irq(irq, arm_smmu_context_fault, IRQF_SHARED,
- "arm-smmu-context-fault", domain);
+ ret = request_threaded_irq(irq, NULL, arm_smmu_context_fault,
+ IRQF_ONESHOT | IRQF_SHARED,
+ "arm-smmu-context-fault", domain);
if (IS_ERR_VALUE(ret)) {
dev_err(smmu->dev, "failed to request context IRQ %d (%u)\n",
cfg->irptndx, irq);
cfg->irptndx = INVALID_IRPTNDX;
}
- mutex_unlock(&smmu_domain->init_mutex);
-
/* Publish page table ops for map/unmap */
smmu_domain->pgtbl_ops = pgtbl_ops;
return 0;
out_clear_smmu:
smmu_domain->smmu = NULL;
-out_unlock:
- mutex_unlock(&smmu_domain->init_mutex);
+out:
return ret;
}
@@ -930,9 +1966,8 @@ static void arm_smmu_destroy_domain_context(struct iommu_domain *domain)
void __iomem *cb_base;
int irq;
- if (!smmu)
- return;
-
+ if (arm_smmu_enable_clocks(smmu_domain->smmu))
+ goto free_irqs;
/*
* Disable the context bank and free the page tables before freeing
* it.
@@ -940,15 +1975,32 @@ static void arm_smmu_destroy_domain_context(struct iommu_domain *domain)
cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
writel_relaxed(0, cb_base + ARM_SMMU_CB_SCTLR);
+ arm_smmu_disable_clocks(smmu_domain->smmu);
+
+ if (smmu_domain->pgtbl_ops) {
+ free_io_pgtable_ops(smmu_domain->pgtbl_ops);
+ /* unassign any freed page table memory */
+ if (arm_smmu_is_master_side_secure(smmu_domain)) {
+ arm_smmu_secure_domain_lock(smmu_domain);
+ arm_smmu_secure_pool_destroy(smmu_domain);
+ arm_smmu_unassign_table(smmu_domain);
+ arm_smmu_secure_domain_unlock(smmu_domain);
+ }
+ smmu_domain->pgtbl_ops = NULL;
+ }
+
+free_irqs:
if (cfg->irptndx != INVALID_IRPTNDX) {
irq = smmu->irqs[smmu->num_global_irqs + cfg->irptndx];
free_irq(irq, domain);
}
- if (smmu_domain->pgtbl_ops)
- free_io_pgtable_ops(smmu_domain->pgtbl_ops);
-
- __arm_smmu_free_bitmap(smmu->context_map, cfg->cbndx);
+ arm_smmu_free_context_idx(smmu, cfg->cbndx);
+ smmu_domain->smmu = NULL;
+ cfg->cbndx = INVALID_CBNDX;
+ cfg->irptndx = INVALID_IRPTNDX;
+ cfg->asid = INVALID_ASID;
+ cfg->vmid = INVALID_VMID;
}
static struct iommu_domain *arm_smmu_domain_alloc(unsigned type)
@@ -966,8 +2018,19 @@ static struct iommu_domain *arm_smmu_domain_alloc(unsigned type)
if (!smmu_domain)
return NULL;
+ smmu_domain->secure_vmid = VMID_INVAL;
+ INIT_LIST_HEAD(&smmu_domain->pte_info_list);
+ INIT_LIST_HEAD(&smmu_domain->unassign_list);
+ INIT_LIST_HEAD(&smmu_domain->secure_pool_list);
+ smmu_domain->cfg.cbndx = INVALID_CBNDX;
+ smmu_domain->cfg.irptndx = INVALID_IRPTNDX;
+ smmu_domain->cfg.asid = INVALID_ASID;
+ smmu_domain->cfg.vmid = INVALID_VMID;
+
mutex_init(&smmu_domain->init_mutex);
- spin_lock_init(&smmu_domain->pgtbl_lock);
+ spin_lock_init(&smmu_domain->pgtbl_spin_lock);
+ mutex_init(&smmu_domain->assign_lock);
+ mutex_init(&smmu_domain->pgtbl_mutex_lock);
return &smmu_domain->domain;
}
@@ -980,7 +2043,18 @@ static void arm_smmu_domain_free(struct iommu_domain *domain)
* Free the domain resources. We assume that all devices have
* already been detached.
*/
- arm_smmu_destroy_domain_context(domain);
+ if (smmu_domain->pgtbl_ops) {
+ free_io_pgtable_ops(smmu_domain->pgtbl_ops);
+ /* unassign any freed page table memory */
+ if (arm_smmu_is_master_side_secure(smmu_domain)) {
+ arm_smmu_secure_domain_lock(smmu_domain);
+ arm_smmu_secure_pool_destroy(smmu_domain);
+ arm_smmu_unassign_table(smmu_domain);
+ arm_smmu_secure_domain_unlock(smmu_domain);
+ }
+ smmu_domain->pgtbl_ops = NULL;
+ }
+
kfree(smmu_domain);
}
@@ -1006,8 +2080,8 @@ static int arm_smmu_master_configure_smrs(struct arm_smmu_device *smmu,
/* Allocate the SMRs on the SMMU */
for (i = 0; i < cfg->num_streamids; ++i) {
- int idx = __arm_smmu_alloc_bitmap(smmu->smr_map, 0,
- smmu->num_mapping_groups);
+ int idx = arm_smmu_alloc_smr_idx(smmu, 0,
+ smmu->num_mapping_groups, cfg->streamids[i]);
if (IS_ERR_VALUE(idx)) {
dev_err(smmu->dev, "failed to allocate free SMR\n");
goto err_free_smrs;
@@ -1032,7 +2106,7 @@ static int arm_smmu_master_configure_smrs(struct arm_smmu_device *smmu,
err_free_smrs:
while (--i >= 0)
- __arm_smmu_free_bitmap(smmu->smr_map, smrs[i].idx);
+ arm_smmu_free_smr_idx(smmu, smrs[i].idx);
kfree(smrs);
return -ENOSPC;
}
@@ -1052,7 +2126,7 @@ static void arm_smmu_master_free_smrs(struct arm_smmu_device *smmu,
u8 idx = smrs[i].idx;
writel_relaxed(~SMR_VALID, gr0_base + ARM_SMMU_GR0_SMR(idx));
- __arm_smmu_free_bitmap(smmu->smr_map, idx);
+ arm_smmu_free_smr_idx(smmu, idx);
}
cfg->smrs = NULL;
@@ -1098,6 +2172,8 @@ static void arm_smmu_domain_remove_master(struct arm_smmu_domain *smmu_domain,
* We *must* clear the S2CR first, because freeing the SMR means
* that it can be re-allocated immediately.
*/
+ if (arm_smmu_enable_clocks(smmu))
+ return;
for (i = 0; i < cfg->num_streamids; ++i) {
u32 idx = cfg->smrs ? cfg->smrs[i].idx : cfg->streamids[i];
@@ -1106,6 +2182,132 @@ static void arm_smmu_domain_remove_master(struct arm_smmu_domain *smmu_domain,
}
arm_smmu_master_free_smrs(smmu, cfg);
+ arm_smmu_disable_clocks(smmu);
+}
+
+static void arm_smmu_impl_def_programming(struct arm_smmu_device *smmu)
+{
+ int i;
+ struct arm_smmu_impl_def_reg *regs = smmu->impl_def_attach_registers;
+
+ arm_smmu_halt(smmu);
+ for (i = 0; i < smmu->num_impl_def_attach_registers; ++i)
+ writel_relaxed(regs[i].value,
+ ARM_SMMU_GR0(smmu) + regs[i].offset);
+ arm_smmu_resume(smmu);
+}
+
+static int arm_smmu_attach_dynamic(struct iommu_domain *domain,
+ struct arm_smmu_device *smmu)
+{
+ int ret;
+ struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
+ enum io_pgtable_fmt fmt;
+ struct io_pgtable_ops *pgtbl_ops = NULL;
+ struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
+
+ if (!(smmu->options & ARM_SMMU_OPT_DYNAMIC)) {
+ dev_err(smmu->dev, "dynamic domains not supported\n");
+ return -EPERM;
+ }
+
+ if (smmu_domain->smmu != NULL) {
+ dev_err(smmu->dev, "domain is already attached\n");
+ return -EBUSY;
+ }
+
+ if (smmu_domain->cfg.cbndx >= smmu->num_context_banks) {
+ dev_err(smmu->dev, "invalid context bank\n");
+ return -ENODEV;
+ }
+
+ if (smmu->features & ARM_SMMU_FEAT_TRANS_NESTED) {
+ smmu_domain->cfg.cbar = CBAR_TYPE_S1_TRANS_S2_BYPASS;
+ } else if (smmu->features & ARM_SMMU_FEAT_TRANS_S1) {
+ smmu_domain->cfg.cbar = CBAR_TYPE_S1_TRANS_S2_BYPASS;
+ } else {
+ /* dynamic only makes sense for S1. */
+ return -EINVAL;
+ }
+
+ smmu_domain->pgtbl_cfg = (struct io_pgtable_cfg) {
+ .pgsize_bitmap = arm_smmu_ops.pgsize_bitmap,
+ .ias = smmu->va_size,
+ .oas = smmu->ipa_size,
+ .tlb = &arm_smmu_gather_ops,
+ .iommu_dev = smmu->dev,
+ };
+
+ fmt = IS_ENABLED(CONFIG_64BIT) ? ARM_64_LPAE_S1 : ARM_32_LPAE_S1;
+
+ pgtbl_ops = alloc_io_pgtable_ops(fmt, &smmu_domain->pgtbl_cfg,
+ smmu_domain);
+ if (!pgtbl_ops)
+ return -ENOMEM;
+
+ /*
+ * assign any page table memory that might have been allocated
+ * during alloc_io_pgtable_ops
+ */
+ if (arm_smmu_is_master_side_secure(smmu_domain)) {
+ arm_smmu_secure_domain_lock(smmu_domain);
+ arm_smmu_assign_table(smmu_domain);
+ arm_smmu_secure_domain_unlock(smmu_domain);
+ }
+
+ cfg->vmid = cfg->cbndx + 2;
+ smmu_domain->smmu = smmu;
+
+ mutex_lock(&smmu->attach_lock);
+ /* try to avoid reusing an old ASID right away */
+ ret = idr_alloc_cyclic(&smmu->asid_idr, domain,
+ smmu->num_context_banks + 2,
+ MAX_ASID + 1, GFP_KERNEL);
+ if (ret < 0) {
+ dev_err_ratelimited(smmu->dev,
+ "dynamic ASID allocation failed: %d\n", ret);
+ goto out;
+ }
+
+ smmu_domain->cfg.asid = ret;
+ smmu_domain->smmu = smmu;
+ smmu_domain->pgtbl_ops = pgtbl_ops;
+ ret = 0;
+out:
+ if (ret) {
+ free_io_pgtable_ops(pgtbl_ops);
+ /* unassign any freed page table memory */
+ if (arm_smmu_is_master_side_secure(smmu_domain)) {
+ arm_smmu_secure_domain_lock(smmu_domain);
+ arm_smmu_secure_pool_destroy(smmu_domain);
+ arm_smmu_unassign_table(smmu_domain);
+ arm_smmu_secure_domain_unlock(smmu_domain);
+ }
+ smmu_domain->pgtbl_ops = NULL;
+ }
+ mutex_unlock(&smmu->attach_lock);
+
+ return ret;
+}
+
+static int arm_smmu_populate_cb(struct arm_smmu_device *smmu,
+ struct arm_smmu_domain *smmu_domain, struct device *dev)
+{
+ struct arm_smmu_master_cfg *cfg;
+ struct arm_smmu_cfg *smmu_cfg = &smmu_domain->cfg;
+ struct static_cbndx_entry *entry;
+
+ cfg = find_smmu_master_cfg(dev);
+ if (!cfg)
+ return -ENODEV;
+
+ entry = arm_smmu_get_static_entry_from_sid(smmu, cfg->streamids[0]);
+ if (entry && entry->type == TYPE_TRANS) {
+ smmu_cfg->cbndx = entry->cbndx;
+ return 0;
+ }
+
+ return -EINVAL;
}
static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev)
@@ -1114,22 +2316,83 @@ static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev)
struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
struct arm_smmu_device *smmu;
struct arm_smmu_master_cfg *cfg;
+ int atomic_ctx = smmu_domain->attributes & (1 << DOMAIN_ATTR_ATOMIC);
+ mutex_lock(&smmu_domain->init_mutex);
smmu = find_smmu_for_device(dev);
if (!smmu) {
dev_err(dev, "cannot attach to SMMU, is it on the same bus?\n");
+ mutex_unlock(&smmu_domain->init_mutex);
return -ENXIO;
}
+ if (smmu_domain->attributes & (1 << DOMAIN_ATTR_DYNAMIC)) {
+ ret = arm_smmu_attach_dynamic(domain, smmu);
+ mutex_unlock(&smmu_domain->init_mutex);
+ return ret;
+ }
+
+ mutex_lock(&smmu->attach_lock);
+
if (dev->archdata.iommu) {
dev_err(dev, "already attached to IOMMU domain\n");
- return -EEXIST;
+ ret = -EEXIST;
+ goto err_unlock;
+ }
+
+ if (!smmu->attach_count) {
+ /*
+ * We need an extra power vote if we can't retain register
+ * settings across a power collapse, or if this is an
+ * atomic domain (since atomic domains can't sleep during
+ * unmap, so regulators already need to be on to enable tlb
+ * invalidation). The result (due to regulator
+ * refcounting) is that we never disable regulators while a
+ * client is attached in these cases.
+ */
+ if (!(smmu->options & ARM_SMMU_OPT_REGISTER_SAVE)) {
+ ret = arm_smmu_enable_regulators(smmu);
+ if (ret)
+ goto err_unlock;
+ }
+ ret = arm_smmu_enable_clocks(smmu);
+ if (ret)
+ goto err_disable_regulators;
+ arm_smmu_device_reset(smmu);
+ arm_smmu_impl_def_programming(smmu);
+ } else {
+ ret = arm_smmu_enable_clocks(smmu);
+ if (ret)
+ goto err_unlock;
+ }
+ smmu->attach_count++;
+
+ if (atomic_ctx) {
+ ret = arm_smmu_enable_regulators(smmu);
+ if (ret)
+ goto err_disable_clocks;
+ }
+
+ if (arm_smmu_is_static_cb(smmu)) {
+ ret = arm_smmu_populate_cb(smmu, smmu_domain, dev);
+
+ if (ret) {
+ dev_err(dev, "Failed to get valid context bank\n");
+ goto err_atomic_ctx;
+ }
+ smmu_domain->slave_side_secure = true;
+ }
+
+ cfg = find_smmu_master_cfg(dev);
+ if (!cfg) {
+ ret = -ENODEV;
+ goto err_atomic_ctx;
}
/* Ensure that the domain is finalised */
- ret = arm_smmu_init_domain_context(domain, smmu);
+ ret = arm_smmu_init_domain_context(domain, smmu, cfg);
if (IS_ERR_VALUE(ret))
- return ret;
+ goto err_atomic_ctx;
/*
* Sanity check the domain. We don't support domains across
@@ -1139,31 +2402,196 @@ static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev)
dev_err(dev,
"cannot attach to SMMU %s whilst already attached to domain on SMMU %s\n",
dev_name(smmu_domain->smmu->dev), dev_name(smmu->dev));
- return -EINVAL;
+ ret = -EINVAL;
+ goto err_destroy_domain_context;
}
/* Looks ok, so add the device to the domain */
- cfg = find_smmu_master_cfg(dev);
- if (!cfg)
- return -ENODEV;
-
ret = arm_smmu_domain_add_master(smmu_domain, cfg);
- if (!ret)
- dev->archdata.iommu = domain;
+ if (ret)
+ goto err_destroy_domain_context;
+ dev->archdata.iommu = domain;
+ arm_smmu_disable_clocks(smmu);
+ mutex_unlock(&smmu->attach_lock);
+ mutex_unlock(&smmu_domain->init_mutex);
+ return ret;
+
+err_destroy_domain_context:
+ arm_smmu_destroy_domain_context(domain);
+err_atomic_ctx:
+ if (atomic_ctx)
+ arm_smmu_disable_regulators(smmu);
+err_disable_clocks:
+ arm_smmu_disable_clocks(smmu);
+ --smmu->attach_count;
+err_disable_regulators:
+ if (!smmu->attach_count &&
+ (!(smmu->options & ARM_SMMU_OPT_REGISTER_SAVE)))
+ arm_smmu_disable_regulators(smmu);
+err_unlock:
+ mutex_unlock(&smmu->attach_lock);
+ mutex_unlock(&smmu_domain->init_mutex);
return ret;
}
+static void arm_smmu_power_off(struct arm_smmu_device *smmu)
+{
+ /* Turn the thing off */
+ if (arm_smmu_enable_clocks(smmu))
+ return;
+ writel_relaxed(sCR0_CLIENTPD,
+ ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sCR0);
+ arm_smmu_disable_clocks(smmu);
+ if (!(smmu->options & ARM_SMMU_OPT_REGISTER_SAVE))
+ arm_smmu_disable_regulators(smmu);
+}
+
+static void arm_smmu_detach_dynamic(struct iommu_domain *domain,
+ struct arm_smmu_device *smmu)
+{
+ struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
+
+ mutex_lock(&smmu->attach_lock);
+ if (smmu->attach_count > 0) {
+ if (arm_smmu_enable_clocks(smmu_domain->smmu))
+ goto idr_remove;
+ arm_smmu_tlb_inv_context(smmu_domain);
+ arm_smmu_disable_clocks(smmu_domain->smmu);
+ }
+idr_remove:
+ idr_remove(&smmu->asid_idr, smmu_domain->cfg.asid);
+ smmu_domain->cfg.asid = INVALID_ASID;
+ smmu_domain->smmu = NULL;
+ mutex_unlock(&smmu->attach_lock);
+}
+
static void arm_smmu_detach_dev(struct iommu_domain *domain, struct device *dev)
{
struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
struct arm_smmu_master_cfg *cfg;
+ struct arm_smmu_device *smmu;
+ int atomic_ctx = smmu_domain->attributes & (1 << DOMAIN_ATTR_ATOMIC);
+
+ mutex_lock(&smmu_domain->init_mutex);
+ smmu = smmu_domain->smmu;
+ if (!smmu) {
+ dev_err(dev, "Domain already detached!\n");
+ mutex_unlock(&smmu_domain->init_mutex);
+ return;
+ }
+
+
+ if (smmu_domain->attributes & (1 << DOMAIN_ATTR_DYNAMIC)) {
+ arm_smmu_detach_dynamic(domain, smmu);
+ mutex_unlock(&smmu_domain->init_mutex);
+ if (atomic_ctx)
+ arm_smmu_disable_regulators(smmu);
+ return;
+ }
+
+ mutex_lock(&smmu->attach_lock);
cfg = find_smmu_master_cfg(dev);
if (!cfg)
- return;
+ goto unlock;
dev->archdata.iommu = NULL;
arm_smmu_domain_remove_master(smmu_domain, cfg);
+ arm_smmu_destroy_domain_context(domain);
+ if (!--smmu->attach_count)
+ arm_smmu_power_off(smmu);
+ if (atomic_ctx)
+ arm_smmu_disable_regulators(smmu);
+unlock:
+ mutex_unlock(&smmu->attach_lock);
+ mutex_unlock(&smmu_domain->init_mutex);
+}
+
+
+static int arm_smmu_assign_table(struct arm_smmu_domain *smmu_domain)
+{
+ int ret = 0;
+ int dest_vmids[2] = {VMID_HLOS, smmu_domain->secure_vmid};
+ int dest_perms[2] = {PERM_READ | PERM_WRITE, PERM_READ};
+ int source_vmid = VMID_HLOS;
+ struct arm_smmu_pte_info *pte_info, *temp;
+
+ if (!arm_smmu_is_master_side_secure(smmu_domain))
+ return ret;
+
+ list_for_each_entry(pte_info, &smmu_domain->pte_info_list, entry) {
+ ret = hyp_assign_phys(virt_to_phys(pte_info->virt_addr),
+ PAGE_SIZE, &source_vmid, 1,
+ dest_vmids, dest_perms, 2);
+ if (WARN_ON(ret))
+ break;
+ }
+
+ list_for_each_entry_safe(pte_info, temp, &smmu_domain->pte_info_list,
+ entry) {
+ list_del(&pte_info->entry);
+ kfree(pte_info);
+ }
+ return ret;
+}
+
+static void arm_smmu_unassign_table(struct arm_smmu_domain *smmu_domain)
+{
+ int ret;
+ int dest_vmids = VMID_HLOS;
+ int dest_perms = PERM_READ | PERM_WRITE | PERM_EXEC;
+ int source_vmlist[2] = {smmu_domain->secure_vmid, VMID_HLOS};
+ struct arm_smmu_pte_info *pte_info, *temp;
+
+ if (!arm_smmu_is_master_side_secure(smmu_domain))
+ return;
+
+ list_for_each_entry(pte_info, &smmu_domain->unassign_list, entry) {
+ ret = hyp_assign_phys(virt_to_phys(pte_info->virt_addr),
+ PAGE_SIZE, source_vmlist, 2,
+ &dest_vmids, &dest_perms, 1);
+ if (WARN_ON(ret))
+ break;
+
+ free_pages_exact(pte_info->virt_addr, pte_info->size);
+ }
+
+ list_for_each_entry_safe(pte_info, temp, &smmu_domain->unassign_list,
+ entry) {
+ list_del(&pte_info->entry);
+ kfree(pte_info);
+ }
+ return;
+}
+
+static void arm_smmu_unprepare_pgtable(void *cookie, void *addr, size_t size)
+{
+ struct arm_smmu_domain *smmu_domain = cookie;
+ struct arm_smmu_pte_info *pte_info;
+
+ BUG_ON(!arm_smmu_is_master_side_secure(smmu_domain));
+
+ pte_info = kzalloc(sizeof(struct arm_smmu_pte_info), GFP_ATOMIC);
+ if (!pte_info)
+ return;
+
+ pte_info->virt_addr = addr;
+ pte_info->size = size;
+ list_add_tail(&pte_info->entry, &smmu_domain->unassign_list);
+}
+
+static void arm_smmu_prepare_pgtable(void *addr, void *cookie)
+{
+ struct arm_smmu_domain *smmu_domain = cookie;
+ struct arm_smmu_pte_info *pte_info;
+
+ BUG_ON(!arm_smmu_is_master_side_secure(smmu_domain));
+
+ pte_info = kzalloc(sizeof(struct arm_smmu_pte_info), GFP_ATOMIC);
+ if (!pte_info)
+ return;
+ pte_info->virt_addr = addr;
+ list_add_tail(&pte_info->entry, &smmu_domain->pte_info_list);
}
static int arm_smmu_map(struct iommu_domain *domain, unsigned long iova,
@@ -1177,9 +2605,87 @@ static int arm_smmu_map(struct iommu_domain *domain, unsigned long iova,
if (!ops)
return -ENODEV;
- spin_lock_irqsave(&smmu_domain->pgtbl_lock, flags);
+ arm_smmu_secure_domain_lock(smmu_domain);
+
+ flags = arm_smmu_pgtbl_lock(smmu_domain);
ret = ops->map(ops, iova, paddr, size, prot);
- spin_unlock_irqrestore(&smmu_domain->pgtbl_lock, flags);
+ arm_smmu_pgtbl_unlock(smmu_domain, flags);
+
+ if (!ret)
+ ret = arm_smmu_assign_table(smmu_domain);
+
+ arm_smmu_secure_domain_unlock(smmu_domain);
+
+ return ret;
+}
+
+static uint64_t arm_smmu_iova_to_pte(struct iommu_domain *domain,
+ dma_addr_t iova)
+{
+ uint64_t ret;
+ unsigned long flags;
+ struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
+ struct io_pgtable_ops *ops = smmu_domain->pgtbl_ops;
+
+ if (!ops)
+ return 0;
+
+ flags = arm_smmu_pgtbl_lock(smmu_domain);
+ ret = ops->iova_to_pte(ops, iova);
+ arm_smmu_pgtbl_unlock(smmu_domain, flags);
+ return ret;
+}
+
+static size_t arm_smmu_map_sg(struct iommu_domain *domain, unsigned long iova,
+ struct scatterlist *sg, unsigned int nents, int prot)
+{
+ int ret;
+ size_t size;
+ unsigned long flags;
+ struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
+ struct io_pgtable_ops *ops = smmu_domain->pgtbl_ops;
+ struct arm_smmu_device *smmu = smmu_domain->smmu;
+ int atomic_ctx = smmu_domain->attributes & (1 << DOMAIN_ATTR_ATOMIC);
+
+ if (!ops)
+ return -ENODEV;
+
+ if (arm_smmu_is_slave_side_secure(smmu_domain) && atomic_ctx) {
+ dev_err(smmu->dev, "Slave side atomic context not supported\n");
+ return 0;
+ }
+
+ if (arm_smmu_is_slave_side_secure(smmu_domain)) {
+ mutex_lock(&smmu_domain->init_mutex);
+
+ if (arm_smmu_enable_clocks(smmu)) {
+ mutex_unlock(&smmu_domain->init_mutex);
+ return 0;
+ }
+ }
+
+ arm_smmu_secure_domain_lock(smmu_domain);
+
+ flags = arm_smmu_pgtbl_lock(smmu_domain);
+ ret = ops->map_sg(ops, iova, sg, nents, prot, &size);
+ arm_smmu_pgtbl_unlock(smmu_domain, flags);
+
+ if (ret) {
+ if (arm_smmu_assign_table(smmu_domain)) {
+ ret = 0;
+ goto out;
+ }
+ } else {
+ arm_smmu_secure_domain_unlock(smmu_domain);
+ arm_smmu_unmap(domain, iova, size);
+ }
+
+out:
+ arm_smmu_secure_domain_unlock(smmu_domain);
+ if (arm_smmu_is_slave_side_secure(smmu_domain)) {
+ arm_smmu_disable_clocks(smmu_domain->smmu);
+ mutex_unlock(&smmu_domain->init_mutex);
+ }
return ret;
}
@@ -1190,31 +2696,206 @@ static size_t arm_smmu_unmap(struct iommu_domain *domain, unsigned long iova,
unsigned long flags;
struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops;
+ int atomic_ctx = smmu_domain->attributes & (1 << DOMAIN_ATTR_ATOMIC);
if (!ops)
return 0;
- spin_lock_irqsave(&smmu_domain->pgtbl_lock, flags);
+ if (arm_smmu_is_slave_side_secure(smmu_domain) && atomic_ctx) {
+ dev_err(smmu_domain->smmu->dev,
+ "Slave side atomic context not supported\n");
+ return 0;
+ }
+
+ /*
+ * The contract here is that if you set DOMAIN_ATTR_ATOMIC your
+ * domain *must* must be attached an SMMU during unmap. This
+ * function calls other functions that try to use smmu_domain->smmu
+ * if it's not NULL (like the tlb invalidation routines). So if
+ * the client sets DOMAIN_ATTR_ATOMIC and detaches in the middle of
+ * the unmap the smmu instance could go away and we could
+ * dereference NULL. This little BUG_ON should catch most gross
+ * offenders but if atomic clients violate this contract then this
+ * code is racy.
+ */
+ BUG_ON(atomic_ctx && !smmu_domain->smmu);
+
+ if (atomic_ctx) {
+ if (arm_smmu_enable_clocks_atomic(smmu_domain->smmu))
+ return 0;
+ } else {
+ mutex_lock(&smmu_domain->init_mutex);
+ arm_smmu_secure_domain_lock(smmu_domain);
+ if (smmu_domain->smmu &&
+ arm_smmu_enable_clocks(smmu_domain->smmu)) {
+ arm_smmu_secure_domain_unlock(smmu_domain);
+ mutex_unlock(&smmu_domain->init_mutex);
+ return 0;
+ }
+ }
+
+ flags = arm_smmu_pgtbl_lock(smmu_domain);
ret = ops->unmap(ops, iova, size);
- spin_unlock_irqrestore(&smmu_domain->pgtbl_lock, flags);
+ arm_smmu_pgtbl_unlock(smmu_domain, flags);
+
+ /*
+ * While splitting up block mappings, we might allocate page table
+ * memory during unmap, so the vmids needs to be assigned to the
+ * memory here as well.
+ */
+ if (arm_smmu_assign_table(smmu_domain)) {
+ arm_smmu_unassign_table(smmu_domain);
+ arm_smmu_secure_domain_unlock(smmu_domain);
+ mutex_unlock(&smmu_domain->init_mutex);
+ return 0;
+ }
+
+ /* Also unassign any pages that were free'd during unmap */
+ arm_smmu_unassign_table(smmu_domain);
+
+ if (atomic_ctx) {
+ arm_smmu_disable_clocks_atomic(smmu_domain->smmu);
+ } else {
+ if (smmu_domain->smmu)
+ arm_smmu_disable_clocks(smmu_domain->smmu);
+ arm_smmu_secure_domain_unlock(smmu_domain);
+ mutex_unlock(&smmu_domain->init_mutex);
+ }
+
return ret;
}
-static phys_addr_t arm_smmu_iova_to_phys_hard(struct iommu_domain *domain,
- dma_addr_t iova)
+static phys_addr_t arm_smmu_iova_to_phys(struct iommu_domain *domain,
+ dma_addr_t iova)
+{
+ phys_addr_t ret;
+ unsigned long flags;
+ struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
+ struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops;
+
+ if (!ops)
+ return 0;
+
+ flags = arm_smmu_pgtbl_lock(smmu_domain);
+ ret = ops->iova_to_phys(ops, iova);
+ arm_smmu_pgtbl_unlock(smmu_domain, flags);
+ return ret;
+}
+
+static bool arm_smmu_is_iova_coherent(struct iommu_domain *domain,
+ dma_addr_t iova)
+{
+ bool ret;
+ unsigned long flags;
+ struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
+ struct io_pgtable_ops *ops = smmu_domain->pgtbl_ops;
+
+ if (!ops)
+ return false;
+
+ flags = arm_smmu_pgtbl_lock(smmu_domain);
+ ret = ops->is_iova_coherent(ops, iova);
+ arm_smmu_pgtbl_unlock(smmu_domain, flags);
+ return ret;
+}
+
+static int arm_smmu_wait_for_halt(struct arm_smmu_device *smmu)
+{
+ void __iomem *impl_def1_base = ARM_SMMU_IMPL_DEF1(smmu);
+ u32 tmp;
+
+ if (readl_poll_timeout_atomic(impl_def1_base + IMPL_DEF1_MICRO_MMU_CTRL,
+ tmp, (tmp & MICRO_MMU_CTRL_IDLE),
+ 0, 30000)) {
+ dev_err(smmu->dev, "Couldn't halt SMMU!\n");
+ return -EBUSY;
+ }
+
+ return 0;
+}
+
+static int __arm_smmu_halt(struct arm_smmu_device *smmu, bool wait)
+{
+ u32 reg;
+ void __iomem *impl_def1_base = ARM_SMMU_IMPL_DEF1(smmu);
+
+ reg = readl_relaxed(impl_def1_base + IMPL_DEF1_MICRO_MMU_CTRL);
+ reg |= MICRO_MMU_CTRL_LOCAL_HALT_REQ;
+
+ if (arm_smmu_is_static_cb(smmu)) {
+ phys_addr_t impl_def1_base_phys = impl_def1_base - smmu->base +
+ smmu->phys_addr;
+
+ if (scm_io_write(impl_def1_base_phys +
+ IMPL_DEF1_MICRO_MMU_CTRL, reg)) {
+ dev_err(smmu->dev,
+ "scm_io_write fail. SMMU might not be halted");
+ return -EINVAL;
+ }
+ } else {
+ writel_relaxed(reg, impl_def1_base + IMPL_DEF1_MICRO_MMU_CTRL);
+ }
+
+ return wait ? arm_smmu_wait_for_halt(smmu) : 0;
+}
+
+static int arm_smmu_halt(struct arm_smmu_device *smmu)
+{
+ return __arm_smmu_halt(smmu, true);
+}
+
+static int arm_smmu_halt_nowait(struct arm_smmu_device *smmu)
+{
+ return __arm_smmu_halt(smmu, false);
+}
+
+static void arm_smmu_resume(struct arm_smmu_device *smmu)
+{
+ void __iomem *impl_def1_base = ARM_SMMU_IMPL_DEF1(smmu);
+ u32 reg;
+
+ if (arm_smmu_restore_sec_cfg(smmu))
+ return;
+
+ reg = readl_relaxed(impl_def1_base + IMPL_DEF1_MICRO_MMU_CTRL);
+ reg &= ~MICRO_MMU_CTRL_LOCAL_HALT_REQ;
+
+ if (arm_smmu_is_static_cb(smmu)) {
+ phys_addr_t impl_def1_base_phys = impl_def1_base - smmu->base +
+ smmu->phys_addr;
+
+ if (scm_io_write(impl_def1_base_phys +
+ IMPL_DEF1_MICRO_MMU_CTRL, reg))
+ dev_err(smmu->dev,
+ "scm_io_write fail. SMMU might not be resumed");
+ } else {
+ writel_relaxed(reg, impl_def1_base + IMPL_DEF1_MICRO_MMU_CTRL);
+ }
+}
+
+static phys_addr_t __arm_smmu_iova_to_phys_hard(struct iommu_domain *domain,
+ dma_addr_t iova, bool do_halt)
{
struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
struct arm_smmu_device *smmu = smmu_domain->smmu;
struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
- struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops;
struct device *dev = smmu->dev;
void __iomem *cb_base;
u32 tmp;
u64 phys;
unsigned long va;
+ unsigned long flags;
+
+ if (arm_smmu_enable_clocks(smmu))
+ return 0;
cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
+ spin_lock_irqsave(&smmu->atos_lock, flags);
+
+ if (do_halt && arm_smmu_halt(smmu))
+ goto err_unlock;
+
/* ATS1 registers can only be written atomically */
va = iova & ~0xfffUL;
if (smmu->version == ARM_SMMU_V2)
@@ -1224,46 +2905,115 @@ static phys_addr_t arm_smmu_iova_to_phys_hard(struct iommu_domain *domain,
if (readl_poll_timeout_atomic(cb_base + ARM_SMMU_CB_ATSR, tmp,
!(tmp & ATSR_ACTIVE), 5, 50)) {
- dev_err(dev,
- "iova to phys timed out on %pad. Falling back to software table walk.\n",
- &iova);
- return ops->iova_to_phys(ops, iova);
+ dev_err(dev, "iova to phys timed out\n");
+ goto err_resume;
}
phys = readl_relaxed(cb_base + ARM_SMMU_CB_PAR_LO);
- phys |= ((u64)readl_relaxed(cb_base + ARM_SMMU_CB_PAR_HI)) << 32;
+ phys |= ((u64) readl_relaxed(cb_base + ARM_SMMU_CB_PAR_HI)) << 32;
+
+ if (do_halt)
+ arm_smmu_resume(smmu);
+ spin_unlock_irqrestore(&smmu->atos_lock, flags);
if (phys & CB_PAR_F) {
- dev_err(dev, "translation fault!\n");
+ dev_err(dev, "translation fault on %s!\n", dev_name(dev));
dev_err(dev, "PAR = 0x%llx\n", phys);
- return 0;
+ phys = 0;
+ } else {
+ phys = (phys & (PHYS_MASK & ~0xfffULL)) | (iova & 0xfff);
}
- return (phys & GENMASK_ULL(39, 12)) | (iova & 0xfff);
+ arm_smmu_disable_clocks(smmu);
+ return phys;
+
+err_resume:
+ if (do_halt)
+ arm_smmu_resume(smmu);
+err_unlock:
+ spin_unlock_irqrestore(&smmu->atos_lock, flags);
+ arm_smmu_disable_clocks(smmu);
+ phys = arm_smmu_iova_to_phys(domain, iova);
+ dev_err(dev,
+ "iova to phys failed 0x%pa. software table walk result=%pa.\n",
+ &iova, &phys);
+ return 0;
}
-static phys_addr_t arm_smmu_iova_to_phys(struct iommu_domain *domain,
- dma_addr_t iova)
+static phys_addr_t arm_smmu_iova_to_phys_hard(struct iommu_domain *domain,
+ dma_addr_t iova)
+{
+ return __arm_smmu_iova_to_phys_hard(domain, iova, true);
+}
+
+static phys_addr_t arm_smmu_iova_to_phys_hard_no_halt(
+ struct iommu_domain *domain, dma_addr_t iova)
+{
+ return __arm_smmu_iova_to_phys_hard(domain, iova, false);
+}
+
+static unsigned long arm_smmu_reg_read(struct iommu_domain *domain,
+ unsigned long offset)
{
- phys_addr_t ret;
- unsigned long flags;
struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
- struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops;
+ struct arm_smmu_device *smmu;
+ struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
+ void __iomem *cb_base;
+ unsigned long val;
- if (!ops)
+ if (offset >= SZ_4K) {
+ pr_err("Invalid offset: 0x%lx\n", offset);
return 0;
+ }
- spin_lock_irqsave(&smmu_domain->pgtbl_lock, flags);
- if (smmu_domain->smmu->features & ARM_SMMU_FEAT_TRANS_OPS &&
- smmu_domain->stage == ARM_SMMU_DOMAIN_S1) {
- ret = arm_smmu_iova_to_phys_hard(domain, iova);
- } else {
- ret = ops->iova_to_phys(ops, iova);
+ mutex_lock(&smmu_domain->init_mutex);
+ smmu = smmu_domain->smmu;
+ if (!smmu) {
+ WARN(1, "Can't read registers of a detached domain\n");
+ val = 0;
+ goto unlock;
}
- spin_unlock_irqrestore(&smmu_domain->pgtbl_lock, flags);
+ cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
+ if (arm_smmu_enable_clocks(smmu)) {
+ val = 0;
+ goto unlock;
+ }
+ val = readl_relaxed(cb_base + offset);
+ arm_smmu_disable_clocks(smmu);
- return ret;
+unlock:
+ mutex_unlock(&smmu_domain->init_mutex);
+ return val;
+}
+
+static void arm_smmu_reg_write(struct iommu_domain *domain,
+ unsigned long offset, unsigned long val)
+{
+ struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
+ struct arm_smmu_device *smmu;
+ struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
+ void __iomem *cb_base;
+
+ if (offset >= SZ_4K) {
+ pr_err("Invalid offset: 0x%lx\n", offset);
+ return;
+ }
+
+ mutex_lock(&smmu_domain->init_mutex);
+ smmu = smmu_domain->smmu;
+ if (!smmu) {
+ WARN(1, "Can't read registers of a detached domain\n");
+ goto unlock;
+ }
+
+ cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
+ if (arm_smmu_enable_clocks(smmu))
+ goto unlock;
+ writel_relaxed(val, cb_base + offset);
+ arm_smmu_disable_clocks(smmu);
+unlock:
+ mutex_unlock(&smmu_domain->init_mutex);
}
static bool arm_smmu_capable(enum iommu_cap cap)
@@ -1284,12 +3034,6 @@ static bool arm_smmu_capable(enum iommu_cap cap)
}
}
-static int __arm_smmu_get_pci_sid(struct pci_dev *pdev, u16 alias, void *data)
-{
- *((u16 *)data) = alias;
- return 0; /* Continue walking */
-}
-
static void __arm_smmu_release_pci_iommudata(void *data)
{
kfree(data);
@@ -1299,8 +3043,9 @@ static int arm_smmu_init_pci_device(struct pci_dev *pdev,
struct iommu_group *group)
{
struct arm_smmu_master_cfg *cfg;
- u16 sid;
- int i;
+ u32 sid;
+ int tmp, ret;
+ struct device *dev = &pdev->dev;
cfg = iommu_group_get_iommudata(group);
if (!cfg) {
@@ -1315,18 +3060,14 @@ static int arm_smmu_init_pci_device(struct pci_dev *pdev,
if (cfg->num_streamids >= MAX_MASTER_STREAMIDS)
return -ENOSPC;
- /*
- * Assume Stream ID == Requester ID for now.
- * We need a way to describe the ID mappings in FDT.
- */
- pci_for_each_dma_alias(pdev, __arm_smmu_get_pci_sid, &sid);
- for (i = 0; i < cfg->num_streamids; ++i)
- if (cfg->streamids[i] == sid)
- break;
-
- /* Avoid duplicate SIDs, as this can lead to SMR conflicts */
- if (i == cfg->num_streamids)
- cfg->streamids[cfg->num_streamids++] = sid;
+ ret = msm_pcie_configure_sid(dev, &sid, &tmp);
+ if (ret) {
+ dev_err(dev,
+ "Couldn't configure SID through PCI-e driver: %d\n",
+ ret);
+ return ret;
+ }
+ cfg->streamids[cfg->num_streamids++] = sid;
return 0;
}
@@ -1370,12 +3111,18 @@ static struct iommu_group *arm_smmu_device_group(struct device *dev)
struct iommu_group *group;
int ret;
- if (dev_is_pci(dev))
- group = pci_device_group(dev);
- else
- group = generic_device_group(dev);
+ /*
+ * We used to call pci_device_group here for dev_is_pci(dev)
+ * devices. However, that causes the root complex device to be
+ * placed in the same group as endpoint devices (and probably puts
+ * all endpoint devices in the same group as well), which makes
+ * things tricky in the DMA layer since we don't actually want to
+ * attach *everybody* in the group when one client calls attach.
+ * Instead, we'll just allocate a new group for everybody here.
+ */
+ group = generic_device_group(dev);
- if (IS_ERR(group))
+ if (IS_ERR_OR_NULL(group))
return group;
if (dev_is_pci(dev))
@@ -1394,15 +3141,120 @@ static struct iommu_group *arm_smmu_device_group(struct device *dev)
static int arm_smmu_domain_get_attr(struct iommu_domain *domain,
enum iommu_attr attr, void *data)
{
+ int ret;
struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
+ mutex_lock(&smmu_domain->init_mutex);
switch (attr) {
case DOMAIN_ATTR_NESTING:
*(int *)data = (smmu_domain->stage == ARM_SMMU_DOMAIN_NESTED);
- return 0;
+ ret = 0;
+ break;
+ case DOMAIN_ATTR_SECURE_VMID:
+ *((int *)data) = smmu_domain->secure_vmid;
+ ret = 0;
+ break;
+ case DOMAIN_ATTR_PT_BASE_ADDR:
+ *((phys_addr_t *)data) =
+ smmu_domain->pgtbl_cfg.arm_lpae_s1_cfg.ttbr[0];
+ ret = 0;
+ break;
+ case DOMAIN_ATTR_CONTEXT_BANK:
+ /* context bank index isn't valid until we are attached */
+ if (smmu_domain->smmu == NULL)
+ return -ENODEV;
+
+ *((unsigned int *) data) = smmu_domain->cfg.cbndx;
+ ret = 0;
+ break;
+ case DOMAIN_ATTR_TTBR0: {
+ u64 val;
+ /* not valid until we are attached */
+ if (smmu_domain->smmu == NULL)
+ return -ENODEV;
+
+ val = smmu_domain->pgtbl_cfg.arm_lpae_s1_cfg.ttbr[0];
+ if (smmu_domain->cfg.cbar != CBAR_TYPE_S2_TRANS)
+ val |= (u64)ARM_SMMU_CB_ASID(&smmu_domain->cfg)
+ << TTBRn_ASID_SHIFT;
+ *((u64 *)data) = val;
+ ret = 0;
+ break;
+ }
+ case DOMAIN_ATTR_CONTEXTIDR:
+ /* not valid until attached */
+ if (smmu_domain->smmu == NULL)
+ return -ENODEV;
+ *((u32 *)data) = smmu_domain->cfg.procid;
+ ret = 0;
+ break;
+ case DOMAIN_ATTR_PROCID:
+ *((u32 *)data) = smmu_domain->cfg.procid;
+ ret = 0;
+ break;
+ case DOMAIN_ATTR_DYNAMIC:
+ *((int *)data) = !!(smmu_domain->attributes
+ & (1 << DOMAIN_ATTR_DYNAMIC));
+ ret = 0;
+ break;
+ case DOMAIN_ATTR_NON_FATAL_FAULTS:
+ *((int *)data) = !!(smmu_domain->attributes
+ & (1 << DOMAIN_ATTR_NON_FATAL_FAULTS));
+ ret = 0;
+ break;
+ case DOMAIN_ATTR_S1_BYPASS:
+ *((int *)data) = !!(smmu_domain->attributes
+ & (1 << DOMAIN_ATTR_S1_BYPASS));
+ ret = 0;
+ break;
+ case DOMAIN_ATTR_FAST:
+ *((int *)data) = !!(smmu_domain->attributes
+ & (1 << DOMAIN_ATTR_FAST));
+ ret = 0;
+ break;
+ case DOMAIN_ATTR_PGTBL_INFO: {
+ struct iommu_pgtbl_info *info = data;
+
+ if (!(smmu_domain->attributes & (1 << DOMAIN_ATTR_FAST))) {
+ ret = -ENODEV;
+ break;
+ }
+ info->pmds = smmu_domain->pgtbl_cfg.av8l_fast_cfg.pmds;
+ ret = 0;
+ break;
+ }
+ case DOMAIN_ATTR_EARLY_MAP:
+ *((int *)data) = !!(smmu_domain->attributes
+ & (1 << DOMAIN_ATTR_EARLY_MAP));
+ ret = 0;
+ break;
+ case DOMAIN_ATTR_PAGE_TABLE_IS_COHERENT:
+ if (!smmu_domain->smmu)
+ return -ENODEV;
+ *((int *)data) = is_iommu_pt_coherent(smmu_domain);
+ ret = 0;
+ break;
+ case DOMAIN_ATTR_PAGE_TABLE_FORCE_COHERENT:
+ *((int *)data) = !!(smmu_domain->attributes
+ & (1 << DOMAIN_ATTR_PAGE_TABLE_FORCE_COHERENT));
+ ret = 0;
+ break;
+ case DOMAIN_ATTR_ENABLE_TTBR1:
+ *((int *)data) = !!(smmu_domain->attributes
+ & (1 << DOMAIN_ATTR_ENABLE_TTBR1));
+ ret = 0;
+ break;
+ case DOMAIN_ATTR_CB_STALL_DISABLE:
+ *((int *)data) = !!(smmu_domain->attributes
+ & (1 << DOMAIN_ATTR_CB_STALL_DISABLE));
+ ret = 0;
+ break;
default:
- return -ENODEV;
+ ret = -ENODEV;
+ break;
}
+ mutex_unlock(&smmu_domain->init_mutex);
+ return ret;
}
static int arm_smmu_domain_set_attr(struct iommu_domain *domain,
@@ -1424,10 +3276,174 @@ static int arm_smmu_domain_set_attr(struct iommu_domain *domain,
smmu_domain->stage = ARM_SMMU_DOMAIN_NESTED;
else
smmu_domain->stage = ARM_SMMU_DOMAIN_S1;
+ break;
+ case DOMAIN_ATTR_SECURE_VMID:
+ BUG_ON(smmu_domain->secure_vmid != VMID_INVAL);
+ smmu_domain->secure_vmid = *((int *)data);
+ break;
+ case DOMAIN_ATTR_ATOMIC:
+ {
+ int atomic_ctx = *((int *)data);
+ if (atomic_ctx)
+ smmu_domain->attributes |= (1 << DOMAIN_ATTR_ATOMIC);
+ else
+ smmu_domain->attributes &= ~(1 << DOMAIN_ATTR_ATOMIC);
+ break;
+ }
+ case DOMAIN_ATTR_PROCID:
+ if (smmu_domain->smmu != NULL) {
+ dev_err(smmu_domain->smmu->dev,
+ "cannot change procid attribute while attached\n");
+ ret = -EBUSY;
+ break;
+ }
+ smmu_domain->cfg.procid = *((u32 *)data);
+ ret = 0;
+ break;
+ case DOMAIN_ATTR_DYNAMIC: {
+ int dynamic = *((int *)data);
+
+ if (smmu_domain->smmu != NULL) {
+ dev_err(smmu_domain->smmu->dev,
+ "cannot change dynamic attribute while attached\n");
+ ret = -EBUSY;
+ break;
+ }
+
+ if (dynamic)
+ smmu_domain->attributes |= 1 << DOMAIN_ATTR_DYNAMIC;
+ else
+ smmu_domain->attributes &= ~(1 << DOMAIN_ATTR_DYNAMIC);
+ ret = 0;
+ break;
+ }
+ case DOMAIN_ATTR_CONTEXT_BANK:
+ /* context bank can't be set while attached */
+ if (smmu_domain->smmu != NULL) {
+ ret = -EBUSY;
+ break;
+ }
+ /* ... and it can only be set for dynamic contexts. */
+ if (!(smmu_domain->attributes & (1 << DOMAIN_ATTR_DYNAMIC))) {
+ ret = -EINVAL;
+ break;
+ }
+
+ /* this will be validated during attach */
+ smmu_domain->cfg.cbndx = *((unsigned int *)data);
+ ret = 0;
+ break;
+ case DOMAIN_ATTR_NON_FATAL_FAULTS:
+ smmu_domain->non_fatal_faults = *((int *)data);
+ ret = 0;
+ break;
+ case DOMAIN_ATTR_S1_BYPASS: {
+ int bypass = *((int *)data);
+
+ if (bypass)
+ smmu_domain->attributes |= 1 << DOMAIN_ATTR_S1_BYPASS;
+ else
+ smmu_domain->attributes &=
+ ~(1 << DOMAIN_ATTR_S1_BYPASS);
+
+ ret = 0;
+ break;
+ }
+ case DOMAIN_ATTR_FAST:
+ if (*((int *)data))
+ smmu_domain->attributes |= 1 << DOMAIN_ATTR_FAST;
+ ret = 0;
+ break;
+ case DOMAIN_ATTR_EARLY_MAP: {
+ int early_map = *((int *)data);
+
+ ret = 0;
+ if (early_map) {
+ smmu_domain->attributes |=
+ 1 << DOMAIN_ATTR_EARLY_MAP;
+ } else {
+ if (smmu_domain->smmu)
+ ret = arm_smmu_enable_s1_translations(
+ smmu_domain);
+
+ if (!ret)
+ smmu_domain->attributes &=
+ ~(1 << DOMAIN_ATTR_EARLY_MAP);
+ }
+ break;
+ }
+ case DOMAIN_ATTR_PAGE_TABLE_FORCE_COHERENT: {
+ int force_coherent = *((int *)data);
+
+ if (smmu_domain->smmu != NULL) {
+ dev_err(smmu_domain->smmu->dev,
+ "cannot change force coherent attribute while attached\n");
+ ret = -EBUSY;
+ break;
+ }
+
+ if (force_coherent)
+ smmu_domain->attributes |=
+ 1 << DOMAIN_ATTR_PAGE_TABLE_FORCE_COHERENT;
+ else
+ smmu_domain->attributes &=
+ ~(1 << DOMAIN_ATTR_PAGE_TABLE_FORCE_COHERENT);
+ ret = 0;
+ break;
+ }
+ case DOMAIN_ATTR_ENABLE_TTBR1:
+ if (*((int *)data))
+ smmu_domain->attributes |=
+ 1 << DOMAIN_ATTR_ENABLE_TTBR1;
+ ret = 0;
+ break;
+ case DOMAIN_ATTR_GEOMETRY: {
+ struct iommu_domain_geometry *geometry =
+ (struct iommu_domain_geometry *)data;
+
+ if (smmu_domain->smmu != NULL) {
+ dev_err(smmu_domain->smmu->dev,
+ "cannot set geometry attribute while attached\n");
+ ret = -EBUSY;
+ break;
+ }
+
+ if (geometry->aperture_start >= SZ_1G * 4ULL ||
+ geometry->aperture_end >= SZ_1G * 4ULL) {
+ pr_err("fastmap does not support IOVAs >= 4GB\n");
+ ret = -EINVAL;
+ break;
+ }
+ if (smmu_domain->attributes
+ & (1 << DOMAIN_ATTR_GEOMETRY)) {
+ if (geometry->aperture_start
+ < domain->geometry.aperture_start)
+ domain->geometry.aperture_start =
+ geometry->aperture_start;
+
+ if (geometry->aperture_end
+ > domain->geometry.aperture_end)
+ domain->geometry.aperture_end =
+ geometry->aperture_end;
+ } else {
+ smmu_domain->attributes |= 1 << DOMAIN_ATTR_GEOMETRY;
+ domain->geometry.aperture_start =
+ geometry->aperture_start;
+ domain->geometry.aperture_end = geometry->aperture_end;
+ }
+ ret = 0;
+ break;
+ }
+ case DOMAIN_ATTR_CB_STALL_DISABLE:
+ if (*((int *)data))
+ smmu_domain->attributes |=
+ 1 << DOMAIN_ATTR_CB_STALL_DISABLE;
+ ret = 0;
break;
default:
ret = -ENODEV;
+ break;
}
out_unlock:
@@ -1435,6 +3451,66 @@ out_unlock:
return ret;
}
+
+static int arm_smmu_enable_s1_translations(struct arm_smmu_domain *smmu_domain)
+{
+ struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
+ struct arm_smmu_device *smmu = smmu_domain->smmu;
+ void __iomem *cb_base;
+ u32 reg;
+ int ret;
+
+ cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
+ ret = arm_smmu_enable_clocks(smmu);
+ if (ret)
+ return ret;
+
+ reg = readl_relaxed(cb_base + ARM_SMMU_CB_SCTLR);
+ reg |= SCTLR_M;
+
+ writel_relaxed(reg, cb_base + ARM_SMMU_CB_SCTLR);
+ arm_smmu_disable_clocks(smmu);
+ return ret;
+}
+
+static int arm_smmu_dma_supported(struct iommu_domain *domain,
+ struct device *dev, u64 mask)
+{
+ struct arm_smmu_device *smmu;
+ struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
+ int ret;
+
+ mutex_lock(&smmu_domain->init_mutex);
+ smmu = smmu_domain->smmu;
+ if (!smmu) {
+ dev_err(dev,
+ "Can't call dma_supported on an unattached domain\n");
+ mutex_unlock(&smmu_domain->init_mutex);
+ return 0;
+ }
+
+ ret = ((1ULL << smmu->va_size) - 1) <= mask ? 0 : 1;
+ mutex_unlock(&smmu_domain->init_mutex);
+ return ret;
+}
+
+static unsigned long arm_smmu_get_pgsize_bitmap(struct iommu_domain *domain)
+{
+ struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
+
+ /*
+ * if someone is calling map before attach just return the
+ * supported page sizes for the hardware itself.
+ */
+ if (!smmu_domain->pgtbl_cfg.pgsize_bitmap)
+ return arm_smmu_ops.pgsize_bitmap;
+ /*
+ * otherwise return the page sizes supported by this specific page
+ * table configuration
+ */
+ return smmu_domain->pgtbl_cfg.pgsize_bitmap;
+}
+
static struct iommu_ops arm_smmu_ops = {
.capable = arm_smmu_capable,
.domain_alloc = arm_smmu_domain_alloc,
@@ -1443,14 +3519,25 @@ static struct iommu_ops arm_smmu_ops = {
.detach_dev = arm_smmu_detach_dev,
.map = arm_smmu_map,
.unmap = arm_smmu_unmap,
- .map_sg = default_iommu_map_sg,
+ .map_sg = arm_smmu_map_sg,
.iova_to_phys = arm_smmu_iova_to_phys,
+ .iova_to_phys_hard = arm_smmu_iova_to_phys_hard,
.add_device = arm_smmu_add_device,
.remove_device = arm_smmu_remove_device,
.device_group = arm_smmu_device_group,
.domain_get_attr = arm_smmu_domain_get_attr,
.domain_set_attr = arm_smmu_domain_set_attr,
.pgsize_bitmap = -1UL, /* Restricted during device attach */
+ .get_pgsize_bitmap = arm_smmu_get_pgsize_bitmap,
+ .dma_supported = arm_smmu_dma_supported,
+ .trigger_fault = arm_smmu_trigger_fault,
+ .reg_read = arm_smmu_reg_read,
+ .reg_write = arm_smmu_reg_write,
+ .tlbi_domain = arm_smmu_tlbi_domain,
+ .enable_config_clocks = arm_smmu_enable_config_clocks,
+ .disable_config_clocks = arm_smmu_disable_config_clocks,
+ .is_iova_coherent = arm_smmu_is_iova_coherent,
+ .iova_to_pte = arm_smmu_iova_to_pte,
};
static void arm_smmu_device_reset(struct arm_smmu_device *smmu)
@@ -1464,18 +3551,21 @@ static void arm_smmu_device_reset(struct arm_smmu_device *smmu)
reg = readl_relaxed(ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sGFSR);
writel(reg, ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sGFSR);
- /* Mark all SMRn as invalid and all S2CRn as bypass */
- for (i = 0; i < smmu->num_mapping_groups; ++i) {
- writel_relaxed(0, gr0_base + ARM_SMMU_GR0_SMR(i));
- writel_relaxed(S2CR_TYPE_BYPASS,
- gr0_base + ARM_SMMU_GR0_S2CR(i));
- }
+ if (!(smmu->options & ARM_SMMU_OPT_SKIP_INIT)) {
+ /* Mark all SMRn as invalid and all S2CRn as bypass */
+ for (i = 0; i < smmu->num_mapping_groups; ++i) {
+ writel_relaxed(0,
+ gr0_base + ARM_SMMU_GR0_SMR(i));
+ writel_relaxed(S2CR_TYPE_BYPASS,
+ gr0_base + ARM_SMMU_GR0_S2CR(i));
+ }
- /* Make sure all context banks are disabled and clear CB_FSR */
- for (i = 0; i < smmu->num_context_banks; ++i) {
- cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, i);
- writel_relaxed(0, cb_base + ARM_SMMU_CB_SCTLR);
- writel_relaxed(FSR_FAULT, cb_base + ARM_SMMU_CB_FSR);
+ /* Make sure all context banks are disabled and clear CB_FSR */
+ for (i = 0; i < smmu->num_context_banks; ++i) {
+ cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, i);
+ writel_relaxed(0, cb_base + ARM_SMMU_CB_SCTLR);
+ writel_relaxed(FSR_FAULT, cb_base + ARM_SMMU_CB_FSR);
+ }
}
/* Invalidate the TLB, just in case */
@@ -1490,8 +3580,11 @@ static void arm_smmu_device_reset(struct arm_smmu_device *smmu)
/* Disable TLB broadcasting. */
reg |= (sCR0_VMIDPNE | sCR0_PTM);
- /* Enable client access, but bypass when no mapping is found */
- reg &= ~(sCR0_CLIENTPD | sCR0_USFCFG);
+ /* Enable client access */
+ reg &= ~sCR0_CLIENTPD;
+
+ /* Raise an unidentified stream fault on unmapped access */
+ reg |= sCR0_USFCFG;
/* Disable forced broadcasting */
reg &= ~sCR0_FB;
@@ -1523,6 +3616,203 @@ static int arm_smmu_id_size_to_bits(int size)
}
}
+static int regulator_notifier(struct notifier_block *nb,
+ unsigned long event, void *data)
+{
+ int ret = 0;
+ struct arm_smmu_device *smmu = container_of(nb,
+ struct arm_smmu_device, regulator_nb);
+
+ /* Ignore EVENT DISABLE as no clocks could be turned on
+ * at this notification.
+ */
+ if (event != REGULATOR_EVENT_PRE_DISABLE &&
+ event != REGULATOR_EVENT_ENABLE)
+ return NOTIFY_OK;
+
+ ret = arm_smmu_prepare_clocks(smmu);
+ if (ret)
+ goto out;
+
+ ret = arm_smmu_enable_clocks_atomic(smmu);
+ if (ret)
+ goto unprepare_clock;
+
+ if (event == REGULATOR_EVENT_PRE_DISABLE)
+ arm_smmu_halt(smmu);
+ else if (event == REGULATOR_EVENT_ENABLE)
+ arm_smmu_resume(smmu);
+
+ arm_smmu_disable_clocks_atomic(smmu);
+unprepare_clock:
+ arm_smmu_unprepare_clocks(smmu);
+out:
+ return NOTIFY_OK;
+}
+
+static int register_regulator_notifier(struct arm_smmu_device *smmu)
+{
+ struct device *dev = smmu->dev;
+ int ret = 0;
+
+ if (smmu->options & ARM_SMMU_OPT_HALT) {
+ smmu->regulator_nb.notifier_call = regulator_notifier;
+ ret = regulator_register_notifier(smmu->gdsc,
+ &smmu->regulator_nb);
+
+ if (ret)
+ dev_err(dev, "Regulator notifier request failed\n");
+ }
+ return ret;
+}
+
+static int arm_smmu_init_regulators(struct arm_smmu_device *smmu)
+{
+ struct device *dev = smmu->dev;
+
+ if (!of_get_property(dev->of_node, "vdd-supply", NULL))
+ return 0;
+
+ if (!of_property_read_u32(dev->of_node,
+ "qcom,deferred-regulator-disable-delay",
+ &(smmu->regulator_defer)))
+ dev_info(dev, "regulator defer delay %d\n",
+ smmu->regulator_defer);
+
+ smmu->gdsc = devm_regulator_get(dev, "vdd");
+ if (IS_ERR(smmu->gdsc))
+ return PTR_ERR(smmu->gdsc);
+
+ return 0;
+}
+
+static int arm_smmu_init_clocks(struct arm_smmu_device *smmu)
+{
+ const char *cname;
+ struct property *prop;
+ int i;
+ struct device *dev = smmu->dev;
+
+ smmu->num_clocks =
+ of_property_count_strings(dev->of_node, "clock-names");
+
+ if (smmu->num_clocks < 1) {
+ smmu->num_clocks = 0;
+ return 0;
+ }
+
+ smmu->clocks = devm_kzalloc(
+ dev, sizeof(*smmu->clocks) * smmu->num_clocks,
+ GFP_KERNEL);
+
+ if (!smmu->clocks) {
+ dev_err(dev,
+ "Failed to allocate memory for clocks\n");
+ return -ENODEV;
+ }
+
+ i = 0;
+ of_property_for_each_string(dev->of_node, "clock-names",
+ prop, cname) {
+ struct clk *c = devm_clk_get(dev, cname);
+ if (IS_ERR(c)) {
+ dev_err(dev, "Couldn't get clock: %s",
+ cname);
+ return PTR_ERR(c);
+ }
+
+ if (clk_get_rate(c) == 0) {
+ long rate = clk_round_rate(c, 1000);
+ clk_set_rate(c, rate);
+ }
+
+ smmu->clocks[i] = c;
+
+ ++i;
+ }
+ return 0;
+}
+
+static int arm_smmu_init_bus_scaling(struct platform_device *pdev,
+ struct arm_smmu_device *smmu)
+{
+ if (!of_find_property(pdev->dev.of_node, "qcom,msm-bus,name", NULL)) {
+ dev_dbg(&pdev->dev, "No bus scaling info\n");
+ return 0;
+ }
+
+ smmu->bus_pdata = msm_bus_cl_get_pdata(pdev);
+ if (!smmu->bus_pdata) {
+ dev_err(&pdev->dev, "Unable to read bus-scaling from DT\n");
+ return -EINVAL;
+ }
+
+ smmu->bus_client = msm_bus_scale_register_client(smmu->bus_pdata);
+ if (!smmu->bus_client) {
+ dev_err(&pdev->dev, "Bus client registration failed\n");
+ return -EINVAL;
+ }
+
+ return 0;
+}
+
+static void arm_smmu_exit_bus_scaling(struct arm_smmu_device *smmu)
+{
+ if (smmu->bus_client)
+ msm_bus_scale_unregister_client(smmu->bus_client);
+ if (smmu->bus_pdata)
+ msm_bus_cl_clear_pdata(smmu->bus_pdata);
+
+ smmu->bus_client = 0;
+ smmu->bus_pdata = NULL;
+}
+
+static int arm_smmu_parse_impl_def_registers(struct arm_smmu_device *smmu)
+{
+ struct device *dev = smmu->dev;
+ int i, ntuples, ret;
+ u32 *tuples;
+ struct arm_smmu_impl_def_reg *regs, *regit;
+
+ if (!of_find_property(dev->of_node, "attach-impl-defs", &ntuples))
+ return 0;
+
+ ntuples /= sizeof(u32);
+ if (ntuples % 2) {
+ dev_err(dev,
+ "Invalid number of attach-impl-defs registers: %d\n",
+ ntuples);
+ return -EINVAL;
+ }
+
+ regs = devm_kmalloc(
+ dev, sizeof(*smmu->impl_def_attach_registers) * ntuples,
+ GFP_KERNEL);
+ if (!regs)
+ return -ENOMEM;
+
+ tuples = devm_kmalloc(dev, sizeof(u32) * ntuples * 2, GFP_KERNEL);
+ if (!tuples)
+ return -ENOMEM;
+
+ ret = of_property_read_u32_array(dev->of_node, "attach-impl-defs",
+ tuples, ntuples);
+ if (ret)
+ return ret;
+
+ for (i = 0, regit = regs; i < ntuples; i += 2, ++regit) {
+ regit->offset = tuples[i];
+ regit->value = tuples[i + 1];
+ }
+
+ devm_kfree(dev, tuples);
+
+ smmu->impl_def_attach_registers = regs;
+ smmu->num_impl_def_attach_registers = ntuples / 2;
+
+ return 0;
+}
+
static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu)
{
unsigned long size;
@@ -1530,8 +3820,11 @@ static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu)
u32 id;
bool cttw_dt, cttw_reg;
- dev_notice(smmu->dev, "probing hardware configuration...\n");
- dev_notice(smmu->dev, "SMMUv%d with:\n", smmu->version);
+ if (arm_smmu_restore_sec_cfg(smmu))
+ return -ENODEV;
+
+ dev_dbg(smmu->dev, "probing hardware configuration...\n");
+ dev_dbg(smmu->dev, "SMMUv%d with:\n", smmu->version);
/* ID0 */
id = readl_relaxed(gr0_base + ARM_SMMU_GR0_ID0);
@@ -1544,28 +3837,28 @@ static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu)
if (id & ID0_S1TS) {
smmu->features |= ARM_SMMU_FEAT_TRANS_S1;
- dev_notice(smmu->dev, "\tstage 1 translation\n");
+ dev_dbg(smmu->dev, "\tstage 1 translation\n");
}
if (id & ID0_S2TS) {
smmu->features |= ARM_SMMU_FEAT_TRANS_S2;
- dev_notice(smmu->dev, "\tstage 2 translation\n");
+ dev_dbg(smmu->dev, "\tstage 2 translation\n");
}
if (id & ID0_NTS) {
smmu->features |= ARM_SMMU_FEAT_TRANS_NESTED;
- dev_notice(smmu->dev, "\tnested translation\n");
+ dev_dbg(smmu->dev, "\tnested translation\n");
}
if (!(smmu->features &
(ARM_SMMU_FEAT_TRANS_S1 | ARM_SMMU_FEAT_TRANS_S2))) {
- dev_err(smmu->dev, "\tno translation support!\n");
+ dev_err(smmu->dev, "\tno translation support (id0=%x)!\n", id);
return -ENODEV;
}
if ((id & ID0_S1TS) && ((smmu->version == 1) || !(id & ID0_ATOSNS))) {
smmu->features |= ARM_SMMU_FEAT_TRANS_OPS;
- dev_notice(smmu->dev, "\taddress translation ops\n");
+ dev_dbg(smmu->dev, "\taddress translation ops\n");
}
/*
@@ -1579,14 +3872,14 @@ static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu)
if (cttw_dt)
smmu->features |= ARM_SMMU_FEAT_COHERENT_WALK;
if (cttw_dt || cttw_reg)
- dev_notice(smmu->dev, "\t%scoherent table walk\n",
+ dev_dbg(smmu->dev, "\t%scoherent table walk\n",
cttw_dt ? "" : "non-");
if (cttw_dt != cttw_reg)
- dev_notice(smmu->dev,
+ dev_dbg(smmu->dev,
"\t(IDR0.CTTW overridden by dma-coherent property)\n");
if (id & ID0_SMS) {
- u32 smr, sid, mask;
+ u32 smr, sid, mask = 0;
smmu->features |= ARM_SMMU_FEAT_STREAM_MATCH;
smmu->num_mapping_groups = (id >> ID0_NUMSMRG_SHIFT) &
@@ -1597,23 +3890,25 @@ static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu)
return -ENODEV;
}
- smr = SMR_MASK_MASK << SMR_MASK_SHIFT;
- smr |= (SMR_ID_MASK << SMR_ID_SHIFT);
- writel_relaxed(smr, gr0_base + ARM_SMMU_GR0_SMR(0));
- smr = readl_relaxed(gr0_base + ARM_SMMU_GR0_SMR(0));
-
- mask = (smr >> SMR_MASK_SHIFT) & SMR_MASK_MASK;
- sid = (smr >> SMR_ID_SHIFT) & SMR_ID_MASK;
- if ((mask & sid) != sid) {
- dev_err(smmu->dev,
- "SMR mask bits (0x%x) insufficient for ID field (0x%x)\n",
- mask, sid);
- return -ENODEV;
+ if (!(smmu->options & ARM_SMMU_OPT_NO_SMR_CHECK)) {
+ smr = SMR_MASK_MASK << SMR_MASK_SHIFT;
+ smr |= (SMR_ID_MASK << SMR_ID_SHIFT);
+ writel_relaxed(smr, gr0_base + ARM_SMMU_GR0_SMR(0));
+ smr = readl_relaxed(gr0_base + ARM_SMMU_GR0_SMR(0));
+
+ mask = (smr >> SMR_MASK_SHIFT) & SMR_MASK_MASK;
+ sid = (smr >> SMR_ID_SHIFT) & SMR_ID_MASK;
+ if ((mask & sid) != sid) {
+ dev_err(smmu->dev,
+ "SMR mask bits (0x%x) insufficient for ID field (0x%x)\n",
+ mask, sid);
+ return -ENODEV;
+ }
}
- dev_notice(smmu->dev,
- "\tstream matching with %u register groups, mask 0x%x",
- smmu->num_mapping_groups, mask);
+ dev_dbg(smmu->dev,
+ "\tstream matching with %u register groups, mask 0x%x",
+ smmu->num_mapping_groups, mask);
} else {
smmu->num_mapping_groups = (id >> ID0_NUMSIDB_SHIFT) &
ID0_NUMSIDB_MASK;
@@ -1637,8 +3932,8 @@ static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu)
dev_err(smmu->dev, "impossible number of S2 context banks!\n");
return -ENODEV;
}
- dev_notice(smmu->dev, "\t%u context banks (%u stage-2 only)\n",
- smmu->num_context_banks, smmu->num_s2_context_banks);
+ dev_dbg(smmu->dev, "\t%u context banks (%u stage-2 only)\n",
+ smmu->num_context_banks, smmu->num_s2_context_banks);
/* ID2 */
id = readl_relaxed(gr0_base + ARM_SMMU_GR0_ID2);
@@ -1662,11 +3957,13 @@ static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu)
smmu->va_size = smmu->ipa_size;
size = SZ_4K | SZ_2M | SZ_1G;
} else {
- size = (id >> ID2_UBS_SHIFT) & ID2_UBS_MASK;
- smmu->va_size = arm_smmu_id_size_to_bits(size);
+ smmu->ubs = (id >> ID2_UBS_SHIFT) & ID2_UBS_MASK;
+
+ smmu->va_size = arm_smmu_id_size_to_bits(smmu->ubs);
#ifndef CONFIG_64BIT
smmu->va_size = min(32UL, smmu->va_size);
#endif
+ smmu->va_size = min(39UL, smmu->va_size);
size = 0;
if (id & ID2_PTFS_4K)
size |= SZ_4K | SZ_2M | SZ_1G;
@@ -1677,25 +3974,82 @@ static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu)
}
arm_smmu_ops.pgsize_bitmap &= size;
- dev_notice(smmu->dev, "\tSupported page sizes: 0x%08lx\n", size);
+ dev_dbg(smmu->dev, "\tSupported page sizes: 0x%08lx\n", size);
if (smmu->features & ARM_SMMU_FEAT_TRANS_S1)
- dev_notice(smmu->dev, "\tStage-1: %lu-bit VA -> %lu-bit IPA\n",
- smmu->va_size, smmu->ipa_size);
+ dev_dbg(smmu->dev, "\tStage-1: %lu-bit VA -> %lu-bit IPA\n",
+ smmu->va_size, smmu->ipa_size);
if (smmu->features & ARM_SMMU_FEAT_TRANS_S2)
- dev_notice(smmu->dev, "\tStage-2: %lu-bit IPA -> %lu-bit PA\n",
- smmu->ipa_size, smmu->pa_size);
+ dev_dbg(smmu->dev, "\tStage-2: %lu-bit IPA -> %lu-bit PA\n",
+ smmu->ipa_size, smmu->pa_size);
return 0;
}
+static int arm_smmu_add_static_cbndx(struct arm_smmu_device *smmu, int sid,
+ int smr_idx)
+{
+ void __iomem *gr0_base;
+ u32 s2cr_reg;
+ struct static_cbndx_entry *entry;
+
+ entry = devm_kzalloc(smmu->dev, sizeof(*entry), GFP_KERNEL);
+ if (!entry)
+ return -ENOMEM;
+
+ gr0_base = ARM_SMMU_GR0(smmu);
+ s2cr_reg = readl_relaxed(gr0_base + ARM_SMMU_GR0_S2CR(smr_idx));
+ entry->type = (s2cr_reg >> S2CR_TYPE_SHIFT) & S2CR_TYPE_MASK;
+ entry->smr_idx = smr_idx;
+ entry->sid = sid;
+
+ if (entry->type == TYPE_TRANS) {
+ entry->cbndx = (s2cr_reg >> S2CR_CBNDX_SHIFT) &
+ S2CR_CBNDX_MASK;
+ __arm_smmu_set_bitmap(smmu->context_map, entry->cbndx);
+ pr_debug("Static context bank: smr:%d, sid:%d, cbndx:%d\n",
+ smr_idx, sid, entry->cbndx);
+ }
+ __arm_smmu_set_bitmap(smmu->smr_map, smr_idx);
+ list_add(&entry->list, &smmu->static_cbndx_list);
+
+ return 0;
+}
+
+static int arm_smmu_init_static_cbndx_list(struct arm_smmu_device *smmu)
+{
+ int i, ret = 0;
+ void __iomem *gr0_base = ARM_SMMU_GR0(smmu);
+
+ for (i = 0; i < smmu->num_mapping_groups; ++i) {
+ u32 smr_reg, sid;
+
+ smr_reg = readl_relaxed(gr0_base + ARM_SMMU_GR0_SMR(i));
+ if (smr_reg & SMR_VALID) {
+ u32 smr_mask = (smr_reg >> SMR_MASK_SHIFT) &
+ SMR_MASK_MASK;
+
+ if (smr_mask != 0)
+ dev_warn(smmu->dev,
+ "Static smr mask not supported\n");
+ sid = ((smr_reg >> SMR_ID_SHIFT) & SMR_ID_MASK);
+ ret = arm_smmu_add_static_cbndx(smmu, sid, i);
+ if (ret)
+ break;
+ }
+ }
+
+ return ret;
+}
+
static const struct of_device_id arm_smmu_of_match[] = {
{ .compatible = "arm,smmu-v1", .data = (void *)ARM_SMMU_V1 },
{ .compatible = "arm,smmu-v2", .data = (void *)ARM_SMMU_V2 },
{ .compatible = "arm,mmu-400", .data = (void *)ARM_SMMU_V1 },
{ .compatible = "arm,mmu-401", .data = (void *)ARM_SMMU_V1 },
{ .compatible = "arm,mmu-500", .data = (void *)ARM_SMMU_V2 },
+ { .compatible = "qcom,smmu-v2", .data = (void *)ARM_SMMU_V2 },
{ },
};
MODULE_DEVICE_TABLE(of, arm_smmu_of_match);
@@ -1707,7 +4061,6 @@ static int arm_smmu_device_dt_probe(struct platform_device *pdev)
struct arm_smmu_device *smmu;
struct device *dev = &pdev->dev;
struct rb_node *node;
- struct of_phandle_args masterspec;
int num_irqs, i, err;
smmu = devm_kzalloc(dev, sizeof(*smmu), GFP_KERNEL);
@@ -1716,11 +4069,19 @@ static int arm_smmu_device_dt_probe(struct platform_device *pdev)
return -ENOMEM;
}
smmu->dev = dev;
+ mutex_init(&smmu->attach_lock);
+ mutex_init(&smmu->power_lock);
+ spin_lock_init(&smmu->atos_lock);
+ spin_lock_init(&smmu->clock_refs_lock);
+ INIT_LIST_HEAD(&smmu->static_cbndx_list);
of_id = of_match_node(arm_smmu_of_match, dev->of_node);
+ if (!of_id)
+ return -ENODEV;
smmu->version = (enum arm_smmu_arch_version)of_id->data;
res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+ smmu->phys_addr = res->start;
smmu->base = devm_ioremap_resource(dev, res);
if (IS_ERR(smmu->base))
return PTR_ERR(smmu->base);
@@ -1762,43 +4123,62 @@ static int arm_smmu_device_dt_probe(struct platform_device *pdev)
smmu->irqs[i] = irq;
}
- err = arm_smmu_device_cfg_probe(smmu);
+ i = 0;
+
+ err = arm_smmu_parse_impl_def_registers(smmu);
if (err)
- return err;
+ goto out;
- i = 0;
- smmu->masters = RB_ROOT;
- while (!of_parse_phandle_with_args(dev->of_node, "mmu-masters",
- "#stream-id-cells", i,
- &masterspec)) {
- err = register_smmu_master(smmu, dev, &masterspec);
- if (err) {
- dev_err(dev, "failed to add master %s\n",
- masterspec.np->name);
- goto out_put_masters;
- }
+ err = arm_smmu_init_regulators(smmu);
+ if (err)
+ goto out;
- i++;
- }
- dev_notice(dev, "registered %d master devices\n", i);
+ err = arm_smmu_init_clocks(smmu);
+ if (err)
+ goto out;
+
+ err = arm_smmu_init_bus_scaling(pdev, smmu);
+ if (err)
+ goto out;
parse_driver_options(smmu);
+ err = arm_smmu_enable_clocks(smmu);
+ if (err)
+ goto out;
+
+ /* No probe deferral occurred! Proceed with iommu property parsing. */
+ smmu->masters = RB_ROOT;
+ err = arm_smmu_parse_iommus_properties(smmu);
+ if (err)
+ goto out_put_masters;
+
+ smmu->sec_id = msm_dev_to_device_id(dev);
+ err = arm_smmu_device_cfg_probe(smmu);
+ if (!err)
+ err = arm_smmu_init_static_cbndx_list(smmu);
+
+ arm_smmu_disable_clocks(smmu);
+ if (err)
+ goto out_put_masters;
+
+ if (of_device_is_compatible(dev->of_node, "qcom,smmu-v2"))
+ smmu->model = SMMU_MODEL_QCOM_V2;
+
if (smmu->version > ARM_SMMU_V1 &&
smmu->num_context_banks != smmu->num_context_irqs) {
dev_err(dev,
- "found only %d context interrupt(s) but %d required\n",
- smmu->num_context_irqs, smmu->num_context_banks);
- err = -ENODEV;
- goto out_put_masters;
+ "found %d context interrupt(s) but have %d context banks. assuming %d context interrupts.\n",
+ smmu->num_context_irqs, smmu->num_context_banks,
+ smmu->num_context_banks);
+ smmu->num_context_irqs = smmu->num_context_banks;
}
for (i = 0; i < smmu->num_global_irqs; ++i) {
- err = request_irq(smmu->irqs[i],
- arm_smmu_global_fault,
- IRQF_SHARED,
- "arm-smmu global fault",
- smmu);
+ err = request_threaded_irq(smmu->irqs[i],
+ NULL, arm_smmu_global_fault,
+ IRQF_ONESHOT | IRQF_SHARED,
+ "arm-smmu global fault", smmu);
if (err) {
dev_err(dev, "failed to request global IRQ %d (%u)\n",
i, smmu->irqs[i]);
@@ -1806,12 +4186,19 @@ static int arm_smmu_device_dt_probe(struct platform_device *pdev)
}
}
+ idr_init(&smmu->asid_idr);
+
+ platform_set_drvdata(pdev, smmu);
+
+ err = register_regulator_notifier(smmu);
+ if (err)
+ goto out_free_irqs;
+
INIT_LIST_HEAD(&smmu->list);
spin_lock(&arm_smmu_devices_lock);
list_add(&smmu->list, &arm_smmu_devices);
spin_unlock(&arm_smmu_devices_lock);
- arm_smmu_device_reset(smmu);
return 0;
out_free_irqs:
@@ -1819,12 +4206,13 @@ out_free_irqs:
free_irq(smmu->irqs[i], smmu);
out_put_masters:
+ arm_smmu_exit_bus_scaling(smmu);
for (node = rb_first(&smmu->masters); node; node = rb_next(node)) {
struct arm_smmu_master *master
= container_of(node, struct arm_smmu_master, node);
of_node_put(master->of_node);
}
-
+out:
return err;
}
@@ -1860,15 +4248,212 @@ static int arm_smmu_device_remove(struct platform_device *pdev)
for (i = 0; i < smmu->num_global_irqs; ++i)
free_irq(smmu->irqs[i], smmu);
- /* Turn the thing off */
- writel(sCR0_CLIENTPD, ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sCR0);
+ mutex_lock(&smmu->attach_lock);
+ idr_destroy(&smmu->asid_idr);
+ /*
+ * If all devices weren't detached for some reason, we're
+ * still powered on. Power off now.
+ */
+ if (smmu->attach_count)
+ arm_smmu_power_off(smmu);
+ mutex_unlock(&smmu->attach_lock);
+
+ arm_smmu_exit_bus_scaling(smmu);
+
+ return 0;
+}
+
+static void arm_smmu_free_master_nodes(void)
+{
+ struct arm_iommus_node *node, *nex;
+ struct arm_iommus_spec *entry, *n;
+
+ list_for_each_entry_safe(node, nex, &iommus_nodes, list) {
+ list_for_each_entry_safe(entry, n,
+ &node->iommuspec_list, list) {
+ list_del(&entry->list);
+ kfree(entry);
+ }
+ list_del(&node->list);
+ kfree(node);
+ }
+}
+
+static int arm_smmu_get_master_nodes(void)
+{
+ struct arm_iommus_node *node;
+ struct device_node *master;
+ struct of_phandle_args iommuspec;
+ struct arm_iommus_spec *entry;
+
+ for_each_node_with_property(master, "iommus") {
+ int arg_ind = 0;
+
+ node = kzalloc(sizeof(*node), GFP_KERNEL);
+ if (!node)
+ goto release_memory;
+ node->master = master;
+ list_add(&node->list, &iommus_nodes);
+
+ INIT_LIST_HEAD(&node->iommuspec_list);
+
+ while (!of_parse_phandle_with_args(master, "iommus",
+ "#iommu-cells", arg_ind, &iommuspec)) {
+ entry = kzalloc(sizeof(*entry), GFP_KERNEL);
+ if (!entry)
+ goto release_memory;
+ entry->iommu_spec = iommuspec;
+ list_add(&entry->list, &node->iommuspec_list);
+ arg_ind++;
+ }
+ }
+
return 0;
+
+release_memory:
+ arm_smmu_free_master_nodes();
+ return -ENOMEM;
}
+#if CONFIG_PM
+static int arm_smmu_pm_suspend(struct device *dev)
+{
+ struct arm_smmu_device *smmu = dev_get_drvdata(dev);
+ u64 *regs, *reg_global;
+ int j, k = 0;
+ u32 cb_count = 0;
+ void __iomem *base, *gr0_base, *gr1_base;
+
+ if (!smmu)
+ return -ENODEV;
+
+ if (!smmu->attach_count)
+ return 0;
+
+ if (arm_smmu_enable_clocks(smmu)) {
+ dev_err(smmu->dev, "failed to enable clocks for smmu");
+ return -EINVAL;
+ }
+
+ regs = &smmu->regs[0];
+ reg_global = &smmu->reg_global[0];
+ cb_count = smmu->num_context_banks;
+
+ gr0_base = ARM_SMMU_GR0(smmu);
+ gr1_base = ARM_SMMU_GR1(smmu);
+
+ for (j = 0; j < cb_count; j++) {
+ base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, j);
+ regs[k++] = readl_relaxed(base + ARM_SMMU_CB_SCTLR);
+ regs[k++] = readl_relaxed(base + ARM_SMMU_CB_ACTLR);
+ regs[k++] = readl_relaxed(base + ARM_SMMU_CB_TTBCR2);
+ regs[k++] = readq_relaxed(base + ARM_SMMU_CB_TTBR0);
+ regs[k++] = readq_relaxed(base + ARM_SMMU_CB_TTBR1);
+ regs[k++] = readl_relaxed(base + ARM_SMMU_CB_TTBCR);
+ regs[k++] = readl_relaxed(base + ARM_SMMU_CB_CONTEXTIDR);
+ regs[k++] = readl_relaxed(base + ARM_SMMU_CB_S1_MAIR0);
+ regs[k++] = readl_relaxed(base + ARM_SMMU_CB_S1_MAIR1);
+ regs[k++] = readl_relaxed(gr1_base + ARM_SMMU_GR1_CBA2R(j));
+ regs[k++] = readl_relaxed(gr1_base + ARM_SMMU_GR1_CBAR(j));
+ }
+
+ for (j = 0, k = 0; j < smmu->num_mapping_groups; j++) {
+ reg_global[k++] = readl_relaxed(
+ gr0_base + ARM_SMMU_GR0_S2CR(j));
+ reg_global[k++] = readl_relaxed(
+ gr0_base + ARM_SMMU_GR0_SMR(j));
+ }
+ reg_global[k++] = readl_relaxed(ARM_SMMU_GR0_NS(smmu)
+ + ARM_SMMU_GR0_sCR0);
+
+ arm_smmu_disable_clocks(smmu);
+
+ return 0;
+}
+static int arm_smmu_pm_resume(struct device *dev)
+{
+ struct arm_smmu_device *smmu = dev_get_drvdata(dev);
+ u64 *regs, *reg_global;
+ int j, k = 0;
+ u32 cb_count = 0;
+ void __iomem *base, *gr0_base, *gr1_base;
+
+ if (!smmu)
+ return -ENODEV;
+
+ if (!smmu->attach_count)
+ return 0;
+
+ if (arm_smmu_enable_clocks(smmu)) {
+ dev_err(smmu->dev, "failed to enable clocks for smmu");
+ return -EINVAL;
+ }
+
+ regs = &smmu->regs[0];
+ reg_global = &smmu->reg_global[0];
+ cb_count = smmu->num_context_banks;
+
+ gr0_base = ARM_SMMU_GR0(smmu);
+ gr1_base = ARM_SMMU_GR1(smmu);
+
+ for (j = 0; j < cb_count; j++) {
+ base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, j);
+ writel_relaxed(regs[k++], base + ARM_SMMU_CB_SCTLR);
+ writel_relaxed(regs[k++], base + ARM_SMMU_CB_ACTLR);
+ writel_relaxed(regs[k++], base + ARM_SMMU_CB_TTBCR2);
+ writeq_relaxed(regs[k++], base + ARM_SMMU_CB_TTBR0);
+ writeq_relaxed(regs[k++], base + ARM_SMMU_CB_TTBR1);
+ writel_relaxed(regs[k++], base + ARM_SMMU_CB_TTBCR);
+ writel_relaxed(regs[k++], base + ARM_SMMU_CB_CONTEXTIDR);
+ writel_relaxed(regs[k++], base + ARM_SMMU_CB_S1_MAIR0);
+ writel_relaxed(regs[k++], base + ARM_SMMU_CB_S1_MAIR1);
+ writel_relaxed(regs[k++], gr1_base + ARM_SMMU_GR1_CBA2R(j));
+ writel_relaxed(regs[k++], gr1_base + ARM_SMMU_GR1_CBAR(j));
+ }
+
+ for (j = 0, k = 0; j < smmu->num_mapping_groups; j++) {
+ writel_relaxed(reg_global[k++],
+ gr0_base + ARM_SMMU_GR0_S2CR(j));
+ writel_relaxed(reg_global[k++],
+ gr0_base + ARM_SMMU_GR0_SMR(j));
+ }
+ writel_relaxed(reg_global[k++],
+ ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sCR0);
+
+ /* Do a tlb flush */
+ writel_relaxed(0, gr0_base + ARM_SMMU_GR0_TLBIALLH);
+ writel_relaxed(0, gr0_base + ARM_SMMU_GR0_TLBIALLNSNH);
+ __arm_smmu_tlb_sync(smmu);
+
+ arm_smmu_disable_clocks(smmu);
+
+ return 0;
+}
+#else
+static inline int arm_smmu_pm_suspend(struct device *dev)
+{
+ return 0;
+}
+
+static inline int arm_smmu_pm_resume(struct device *dev)
+{
+ return 0;
+}
+#endif
+
+static const struct dev_pm_ops arm_smmu_pm_ops = {
+#ifdef CONFIG_PM
+ .freeze_late = arm_smmu_pm_suspend,
+ .thaw_early = arm_smmu_pm_resume,
+ .restore_early = arm_smmu_pm_resume,
+#endif
+};
+
static struct platform_driver arm_smmu_driver = {
.driver = {
.name = "arm-smmu",
.of_match_table = of_match_ptr(arm_smmu_of_match),
+ .pm = &arm_smmu_pm_ops,
},
.probe = arm_smmu_device_dt_probe,
.remove = arm_smmu_device_remove,
@@ -1890,10 +4475,15 @@ static int __init arm_smmu_init(void)
of_node_put(np);
- ret = platform_driver_register(&arm_smmu_driver);
+ ret = arm_smmu_get_master_nodes();
if (ret)
return ret;
+ ret = platform_driver_register(&arm_smmu_driver);
+ if (ret) {
+ arm_smmu_free_master_nodes();
+ return ret;
+ }
/* Oh, for a proper bus abstraction */
if (!iommu_present(&platform_bus_type))
bus_set_iommu(&platform_bus_type, &arm_smmu_ops);
diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c
index 087a092a6e6e..899cee22f535 100644
--- a/drivers/iommu/dma-iommu.c
+++ b/drivers/iommu/dma-iommu.c
@@ -517,16 +517,6 @@ void iommu_dma_unmap_sg(struct device *dev, struct scatterlist *sg, int nents,
__iommu_dma_unmap(iommu_get_domain_for_dev(dev), sg_dma_address(sg));
}
-int iommu_dma_supported(struct device *dev, u64 mask)
-{
- /*
- * 'Special' IOMMUs which don't have the same addressing capability
- * as the CPU will have to wait until we have some way to query that
- * before they'll be able to use this framework.
- */
- return 1;
-}
-
int iommu_dma_mapping_error(struct device *dev, dma_addr_t dma_addr)
{
return dma_addr == DMA_ERROR_CODE;
diff --git a/drivers/iommu/dma-mapping-fast.c b/drivers/iommu/dma-mapping-fast.c
new file mode 100644
index 000000000000..66c2abd358f8
--- /dev/null
+++ b/drivers/iommu/dma-mapping-fast.c
@@ -0,0 +1,860 @@
+/* Copyright (c) 2016-2017,2019, The Linux Foundation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 and
+ * only version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ */
+
+#include <linux/dma-contiguous.h>
+#include <linux/dma-mapping.h>
+#include <linux/dma-mapping-fast.h>
+#include <linux/io-pgtable-fast.h>
+#include <linux/vmalloc.h>
+#include <asm/cacheflush.h>
+#include <asm/dma-iommu.h>
+#include <linux/slab.h>
+#include <linux/vmalloc.h>
+
+/* some redundant definitions... :( TODO: move to io-pgtable-fast.h */
+#define FAST_PAGE_SHIFT 12
+#define FAST_PAGE_SIZE (1UL << FAST_PAGE_SHIFT)
+#define FAST_PAGE_MASK (~(PAGE_SIZE - 1))
+#define FAST_PTE_ADDR_MASK ((av8l_fast_iopte)0xfffffffff000)
+#define FAST_MAIR_ATTR_IDX_CACHE 1
+#define FAST_PTE_ATTRINDX_SHIFT 2
+#define FAST_PTE_ATTRINDX_MASK 0x7
+#define FAST_PTE_SH_SHIFT 8
+#define FAST_PTE_SH_MASK (((av8l_fast_iopte)0x3) << FAST_PTE_SH_SHIFT)
+#define FAST_PTE_SH_OS (((av8l_fast_iopte)2) << FAST_PTE_SH_SHIFT)
+#define FAST_PTE_SH_IS (((av8l_fast_iopte)3) << FAST_PTE_SH_SHIFT)
+
+static pgprot_t __get_dma_pgprot(struct dma_attrs *attrs, pgprot_t prot,
+ bool coherent)
+{
+ if (dma_get_attr(DMA_ATTR_STRONGLY_ORDERED, attrs))
+ return pgprot_noncached(prot);
+ else if (!coherent || dma_get_attr(DMA_ATTR_WRITE_COMBINE, attrs))
+ return pgprot_writecombine(prot);
+ return prot;
+}
+
+static int __get_iommu_pgprot(struct dma_attrs *attrs, int prot,
+ bool coherent)
+{
+ if (!dma_get_attr(DMA_ATTR_EXEC_MAPPING, attrs))
+ prot |= IOMMU_NOEXEC;
+ if (dma_get_attr(DMA_ATTR_STRONGLY_ORDERED, attrs))
+ prot |= IOMMU_DEVICE;
+ if (coherent)
+ prot |= IOMMU_CACHE;
+
+ return prot;
+}
+
+static void fast_dmac_clean_range(struct dma_fast_smmu_mapping *mapping,
+ void *start, void *end)
+{
+ if (!mapping->is_smmu_pt_coherent)
+ dmac_clean_range(start, end);
+}
+
+static bool __fast_is_pte_coherent(av8l_fast_iopte *ptep)
+{
+ int attr_idx = (*ptep & (FAST_PTE_ATTRINDX_MASK <<
+ FAST_PTE_ATTRINDX_SHIFT)) >>
+ FAST_PTE_ATTRINDX_SHIFT;
+
+ if ((attr_idx == FAST_MAIR_ATTR_IDX_CACHE) &&
+ (((*ptep & FAST_PTE_SH_MASK) == FAST_PTE_SH_IS) ||
+ (*ptep & FAST_PTE_SH_MASK) == FAST_PTE_SH_OS))
+ return true;
+
+ return false;
+}
+
+static bool is_dma_coherent(struct device *dev, struct dma_attrs *attrs)
+{
+ bool is_coherent;
+
+ if (dma_get_attr(DMA_ATTR_FORCE_COHERENT, attrs))
+ is_coherent = true;
+ else if (dma_get_attr(DMA_ATTR_FORCE_NON_COHERENT, attrs))
+ is_coherent = false;
+ else if (is_device_dma_coherent(dev))
+ is_coherent = true;
+ else
+ is_coherent = false;
+
+ return is_coherent;
+}
+
+/*
+ * Checks if the allocated range (ending at @end) covered the upcoming
+ * stale bit. We don't need to know exactly where the range starts since
+ * we already know where the candidate search range started. If, starting
+ * from the beginning of the candidate search range, we had to step over
+ * (or landed directly on top of) the upcoming stale bit, then we return
+ * true.
+ *
+ * Due to wrapping, there are two scenarios we'll need to check: (1) if the
+ * range [search_start, upcoming_stale] spans 0 (i.e. search_start >
+ * upcoming_stale), and, (2) if the range: [search_start, upcoming_stale]
+ * does *not* span 0 (i.e. search_start <= upcoming_stale). And for each
+ * of those two scenarios we need to handle three cases: (1) the bit was
+ * found before wrapping or
+ */
+static bool __bit_covered_stale(unsigned long upcoming_stale,
+ unsigned long search_start,
+ unsigned long end)
+{
+ if (search_start > upcoming_stale) {
+ if (end >= search_start) {
+ /*
+ * We started searching above upcoming_stale and we
+ * didn't wrap, so we couldn't have crossed
+ * upcoming_stale.
+ */
+ return false;
+ }
+ /*
+ * We wrapped. Did we cross (or land on top of)
+ * upcoming_stale?
+ */
+ return end >= upcoming_stale;
+ }
+
+ if (search_start <= upcoming_stale) {
+ if (end >= search_start) {
+ /*
+ * We didn't wrap. Did we cross (or land on top
+ * of) upcoming_stale?
+ */
+ return end >= upcoming_stale;
+ }
+ /*
+ * We wrapped. So we must have crossed upcoming_stale
+ * (since we started searching below it).
+ */
+ return true;
+ }
+
+ /* we should have covered all logical combinations... */
+ WARN_ON(1);
+ return true;
+}
+
+static dma_addr_t __fast_smmu_alloc_iova(struct dma_fast_smmu_mapping *mapping,
+ struct dma_attrs *attrs,
+ size_t size)
+{
+ unsigned long bit, prev_search_start, nbits = size >> FAST_PAGE_SHIFT;
+ unsigned long align = (1 << get_order(size)) - 1;
+
+ bit = bitmap_find_next_zero_area(
+ mapping->bitmap, mapping->num_4k_pages, mapping->next_start,
+ nbits, align);
+ if (unlikely(bit > mapping->num_4k_pages)) {
+ /* try wrapping */
+ mapping->next_start = 0; /* TODO: SHOULD I REALLY DO THIS?!? */
+ bit = bitmap_find_next_zero_area(
+ mapping->bitmap, mapping->num_4k_pages, 0, nbits,
+ align);
+ if (unlikely(bit > mapping->num_4k_pages))
+ return DMA_ERROR_CODE;
+ }
+
+ bitmap_set(mapping->bitmap, bit, nbits);
+ prev_search_start = mapping->next_start;
+ mapping->next_start = bit + nbits;
+ if (unlikely(mapping->next_start >= mapping->num_4k_pages))
+ mapping->next_start = 0;
+
+ /*
+ * If we just re-allocated a VA whose TLB hasn't been invalidated
+ * since it was last used and unmapped, we need to invalidate it
+ * here. We actually invalidate the entire TLB so that we don't
+ * have to invalidate the TLB again until we wrap back around.
+ */
+ if (mapping->have_stale_tlbs &&
+ __bit_covered_stale(mapping->upcoming_stale_bit,
+ prev_search_start,
+ bit + nbits - 1)) {
+ bool skip_sync = dma_get_attr(DMA_ATTR_SKIP_CPU_SYNC, attrs);
+
+ iommu_tlbiall(mapping->domain);
+ mapping->have_stale_tlbs = false;
+ av8l_fast_clear_stale_ptes(mapping->pgtbl_pmds,
+ mapping->domain->geometry.aperture_start,
+ mapping->base,
+ mapping->base + mapping->size - 1,
+ skip_sync);
+ }
+
+ return (bit << FAST_PAGE_SHIFT) + mapping->base;
+}
+
+/*
+ * Checks whether the candidate bit will be allocated sooner than the
+ * current upcoming stale bit. We can say candidate will be upcoming
+ * sooner than the current upcoming stale bit if it lies between the
+ * starting bit of the next search range and the upcoming stale bit
+ * (allowing for wrap-around).
+ *
+ * Stated differently, we're checking the relative ordering of three
+ * unsigned numbers. So we need to check all 6 (i.e. 3!) permutations,
+ * namely:
+ *
+ * 0 |---A---B---C---| TOP (Case 1)
+ * 0 |---A---C---B---| TOP (Case 2)
+ * 0 |---B---A---C---| TOP (Case 3)
+ * 0 |---B---C---A---| TOP (Case 4)
+ * 0 |---C---A---B---| TOP (Case 5)
+ * 0 |---C---B---A---| TOP (Case 6)
+ *
+ * Note that since we're allowing numbers to wrap, the following three
+ * scenarios are all equivalent for Case 1:
+ *
+ * 0 |---A---B---C---| TOP
+ * 0 |---C---A---B---| TOP (C has wrapped. This is Case 5.)
+ * 0 |---B---C---A---| TOP (C and B have wrapped. This is Case 4.)
+ *
+ * In any of these cases, if we start searching from A, we will find B
+ * before we find C.
+ *
+ * We can also find two equivalent cases for Case 2:
+ *
+ * 0 |---A---C---B---| TOP
+ * 0 |---B---A---C---| TOP (B has wrapped. This is Case 3.)
+ * 0 |---C---B---A---| TOP (B and C have wrapped. This is Case 6.)
+ *
+ * In any of these cases, if we start searching from A, we will find C
+ * before we find B.
+ */
+static bool __bit_is_sooner(unsigned long candidate,
+ struct dma_fast_smmu_mapping *mapping)
+{
+ unsigned long A = mapping->next_start;
+ unsigned long B = candidate;
+ unsigned long C = mapping->upcoming_stale_bit;
+
+ if ((A < B && B < C) || /* Case 1 */
+ (C < A && A < B) || /* Case 5 */
+ (B < C && C < A)) /* Case 4 */
+ return true;
+
+ if ((A < C && C < B) || /* Case 2 */
+ (B < A && A < C) || /* Case 3 */
+ (C < B && B < A)) /* Case 6 */
+ return false;
+
+ /*
+ * For simplicity, we've been ignoring the possibility of any of
+ * our three numbers being equal. Handle those cases here (they
+ * shouldn't happen very often, (I think?)).
+ */
+
+ /*
+ * If candidate is the next bit to be searched then it's definitely
+ * sooner.
+ */
+ if (A == B)
+ return true;
+
+ /*
+ * If candidate is the next upcoming stale bit we'll return false
+ * to avoid doing `upcoming = candidate' in the caller (which would
+ * be useless since they're already equal)
+ */
+ if (B == C)
+ return false;
+
+ /*
+ * If next start is the upcoming stale bit then candidate can't
+ * possibly be sooner. The "soonest" bit is already selected.
+ */
+ if (A == C)
+ return false;
+
+ /* We should have covered all logical combinations. */
+ WARN(1, "Well, that's awkward. A=%ld, B=%ld, C=%ld\n", A, B, C);
+ return true;
+}
+
+static void __fast_smmu_free_iova(struct dma_fast_smmu_mapping *mapping,
+ dma_addr_t iova, size_t size)
+{
+ unsigned long start_bit = (iova - mapping->base) >> FAST_PAGE_SHIFT;
+ unsigned long nbits = size >> FAST_PAGE_SHIFT;
+
+ /*
+ * We don't invalidate TLBs on unmap. We invalidate TLBs on map
+ * when we're about to re-allocate a VA that was previously
+ * unmapped but hasn't yet been invalidated. So we need to keep
+ * track of which bit is the closest to being re-allocated here.
+ */
+ if (__bit_is_sooner(start_bit, mapping))
+ mapping->upcoming_stale_bit = start_bit;
+
+ bitmap_clear(mapping->bitmap, start_bit, nbits);
+ mapping->have_stale_tlbs = true;
+}
+
+
+static void __fast_dma_page_cpu_to_dev(struct page *page, unsigned long off,
+ size_t size, enum dma_data_direction dir)
+{
+ __dma_map_area(page_address(page) + off, size, dir);
+}
+
+static void __fast_dma_page_dev_to_cpu(struct page *page, unsigned long off,
+ size_t size, enum dma_data_direction dir)
+{
+ __dma_unmap_area(page_address(page) + off, size, dir);
+
+ /* TODO: WHAT IS THIS? */
+ /*
+ * Mark the D-cache clean for this page to avoid extra flushing.
+ */
+ if (dir != DMA_TO_DEVICE && off == 0 && size >= PAGE_SIZE)
+ set_bit(PG_dcache_clean, &page->flags);
+}
+
+static int __fast_dma_direction_to_prot(enum dma_data_direction dir)
+{
+ switch (dir) {
+ case DMA_BIDIRECTIONAL:
+ return IOMMU_READ | IOMMU_WRITE;
+ case DMA_TO_DEVICE:
+ return IOMMU_READ;
+ case DMA_FROM_DEVICE:
+ return IOMMU_WRITE;
+ default:
+ return 0;
+ }
+}
+
+static dma_addr_t fast_smmu_map_page(struct device *dev, struct page *page,
+ unsigned long offset, size_t size,
+ enum dma_data_direction dir,
+ struct dma_attrs *attrs)
+{
+ struct dma_fast_smmu_mapping *mapping = dev->archdata.mapping->fast;
+ dma_addr_t iova;
+ unsigned long flags;
+ av8l_fast_iopte *pmd;
+ phys_addr_t phys_plus_off = page_to_phys(page) + offset;
+ phys_addr_t phys_to_map = round_down(phys_plus_off, FAST_PAGE_SIZE);
+ unsigned long offset_from_phys_to_map = phys_plus_off & ~FAST_PAGE_MASK;
+ size_t len = ALIGN(size + offset_from_phys_to_map, FAST_PAGE_SIZE);
+ int nptes = len >> FAST_PAGE_SHIFT;
+ bool skip_sync = dma_get_attr(DMA_ATTR_SKIP_CPU_SYNC, attrs);
+ int prot = __fast_dma_direction_to_prot(dir);
+ bool is_coherent = is_dma_coherent(dev, attrs);
+
+ prot = __get_iommu_pgprot(attrs, prot, is_coherent);
+
+ if (!skip_sync && !is_coherent)
+ __fast_dma_page_cpu_to_dev(phys_to_page(phys_to_map),
+ offset_from_phys_to_map, size, dir);
+
+ spin_lock_irqsave(&mapping->lock, flags);
+
+ iova = __fast_smmu_alloc_iova(mapping, attrs, len);
+
+ if (unlikely(iova == DMA_ERROR_CODE))
+ goto fail;
+
+ pmd = iopte_pmd_offset(mapping->pgtbl_pmds,
+ mapping->domain->geometry.aperture_start, iova);
+
+ if (unlikely(av8l_fast_map_public(pmd, phys_to_map, len, prot)))
+ goto fail_free_iova;
+
+ fast_dmac_clean_range(mapping, pmd, pmd + nptes);
+
+ spin_unlock_irqrestore(&mapping->lock, flags);
+ return iova + offset_from_phys_to_map;
+
+fail_free_iova:
+ __fast_smmu_free_iova(mapping, iova, size);
+fail:
+ spin_unlock_irqrestore(&mapping->lock, flags);
+ return DMA_ERROR_CODE;
+}
+
+static void fast_smmu_unmap_page(struct device *dev, dma_addr_t iova,
+ size_t size, enum dma_data_direction dir,
+ struct dma_attrs *attrs)
+{
+ struct dma_fast_smmu_mapping *mapping = dev->archdata.mapping->fast;
+ unsigned long flags;
+ av8l_fast_iopte *pmd = iopte_pmd_offset(mapping->pgtbl_pmds,
+ mapping->domain->geometry.aperture_start,
+ iova);
+ unsigned long offset = iova & ~FAST_PAGE_MASK;
+ size_t len = ALIGN(size + offset, FAST_PAGE_SIZE);
+ int nptes = len >> FAST_PAGE_SHIFT;
+ struct page *page = phys_to_page((*pmd & FAST_PTE_ADDR_MASK));
+ bool skip_sync = dma_get_attr(DMA_ATTR_SKIP_CPU_SYNC, attrs);
+ bool is_coherent = is_dma_coherent(dev, attrs);
+
+ if (!skip_sync && !is_coherent)
+ __fast_dma_page_dev_to_cpu(page, offset, size, dir);
+
+ spin_lock_irqsave(&mapping->lock, flags);
+ av8l_fast_unmap_public(pmd, len);
+ fast_dmac_clean_range(mapping, pmd, pmd + nptes);
+ __fast_smmu_free_iova(mapping, iova, len);
+ spin_unlock_irqrestore(&mapping->lock, flags);
+}
+
+static void fast_smmu_sync_single_for_cpu(struct device *dev,
+ dma_addr_t iova, size_t size, enum dma_data_direction dir)
+{
+ struct dma_fast_smmu_mapping *mapping = dev->archdata.mapping->fast;
+ av8l_fast_iopte *pmd = iopte_pmd_offset(mapping->pgtbl_pmds,
+ mapping->domain->geometry.aperture_start,
+ iova);
+ unsigned long offset = iova & ~FAST_PAGE_MASK;
+ struct page *page = phys_to_page((*pmd & FAST_PTE_ADDR_MASK));
+
+ if (!__fast_is_pte_coherent(pmd))
+ __fast_dma_page_dev_to_cpu(page, offset, size, dir);
+}
+
+static void fast_smmu_sync_single_for_device(struct device *dev,
+ dma_addr_t iova, size_t size, enum dma_data_direction dir)
+{
+ struct dma_fast_smmu_mapping *mapping = dev->archdata.mapping->fast;
+ av8l_fast_iopte *pmd = iopte_pmd_offset(mapping->pgtbl_pmds,
+ mapping->domain->geometry.aperture_start,
+ iova);
+ unsigned long offset = iova & ~FAST_PAGE_MASK;
+ struct page *page = phys_to_page((*pmd & FAST_PTE_ADDR_MASK));
+
+ if (!__fast_is_pte_coherent(pmd))
+ __fast_dma_page_cpu_to_dev(page, offset, size, dir);
+}
+
+static int fast_smmu_map_sg(struct device *dev, struct scatterlist *sg,
+ int nents, enum dma_data_direction dir,
+ struct dma_attrs *attrs)
+{
+ return -EINVAL;
+}
+
+static void fast_smmu_unmap_sg(struct device *dev,
+ struct scatterlist *sg, int nents,
+ enum dma_data_direction dir,
+ struct dma_attrs *attrs)
+{
+ WARN_ON_ONCE(1);
+}
+
+static void fast_smmu_sync_sg_for_cpu(struct device *dev,
+ struct scatterlist *sg, int nents, enum dma_data_direction dir)
+{
+ WARN_ON_ONCE(1);
+}
+
+static void fast_smmu_sync_sg_for_device(struct device *dev,
+ struct scatterlist *sg, int nents, enum dma_data_direction dir)
+{
+ WARN_ON_ONCE(1);
+}
+
+static void __fast_smmu_free_pages(struct page **pages, int count)
+{
+ while (count--)
+ __free_page(pages[count]);
+ kvfree(pages);
+}
+
+static struct page **__fast_smmu_alloc_pages(unsigned int count, gfp_t gfp)
+{
+ struct page **pages;
+ unsigned int i = 0, array_size = count * sizeof(*pages);
+
+ if (array_size <= PAGE_SIZE)
+ pages = kzalloc(array_size, GFP_KERNEL);
+ else
+ pages = vzalloc(array_size);
+ if (!pages)
+ return NULL;
+
+ /* IOMMU can map any pages, so himem can also be used here */
+ gfp |= __GFP_NOWARN | __GFP_HIGHMEM;
+
+ for (i = 0; i < count; ++i) {
+ struct page *page = alloc_page(gfp);
+
+ if (!page) {
+ __fast_smmu_free_pages(pages, i - 1);
+ return NULL;
+ }
+ pages[i] = page;
+ }
+ return pages;
+}
+
+static void *fast_smmu_alloc(struct device *dev, size_t size,
+ dma_addr_t *handle, gfp_t gfp,
+ struct dma_attrs *attrs)
+{
+ struct dma_fast_smmu_mapping *mapping = dev->archdata.mapping->fast;
+ struct sg_table sgt;
+ dma_addr_t dma_addr, iova_iter;
+ void *addr;
+ av8l_fast_iopte *ptep;
+ unsigned long flags;
+ struct sg_mapping_iter miter;
+ size_t count = ALIGN(size, SZ_4K) >> PAGE_SHIFT;
+ int prot = IOMMU_READ | IOMMU_WRITE; /* TODO: extract from attrs */
+ bool is_coherent = is_dma_coherent(dev, attrs);
+ pgprot_t remap_prot = __get_dma_pgprot(attrs, PAGE_KERNEL, is_coherent);
+ struct page **pages;
+
+ /*
+ * sg_alloc_table_from_pages accepts unsigned int value for count
+ * so check count doesn't exceed UINT_MAX.
+ */
+
+ if (count > UINT_MAX) {
+ dev_err(dev, "count: %zx exceeds UNIT_MAX\n", count);
+ return NULL;
+ }
+
+ prot = __get_iommu_pgprot(attrs, prot, is_coherent);
+
+ *handle = DMA_ERROR_CODE;
+
+ pages = __fast_smmu_alloc_pages(count, gfp);
+ if (!pages) {
+ dev_err(dev, "no pages\n");
+ return NULL;
+ }
+
+ size = ALIGN(size, SZ_4K);
+ if (sg_alloc_table_from_pages(&sgt, pages, count, 0, size, gfp)) {
+ dev_err(dev, "no sg tablen\n");
+ goto out_free_pages;
+ }
+
+ if (!is_coherent) {
+ /*
+ * The CPU-centric flushing implied by SG_MITER_TO_SG isn't
+ * sufficient here, so skip it by using the "wrong" direction.
+ */
+ sg_miter_start(&miter, sgt.sgl, sgt.orig_nents,
+ SG_MITER_FROM_SG);
+ while (sg_miter_next(&miter))
+ __dma_flush_range(miter.addr,
+ miter.addr + miter.length);
+ sg_miter_stop(&miter);
+ }
+
+ spin_lock_irqsave(&mapping->lock, flags);
+ dma_addr = __fast_smmu_alloc_iova(mapping, attrs, size);
+ if (dma_addr == DMA_ERROR_CODE) {
+ dev_err(dev, "no iova\n");
+ spin_unlock_irqrestore(&mapping->lock, flags);
+ goto out_free_sg;
+ }
+ iova_iter = dma_addr;
+ sg_miter_start(&miter, sgt.sgl, sgt.orig_nents,
+ SG_MITER_FROM_SG | SG_MITER_ATOMIC);
+ while (sg_miter_next(&miter)) {
+ int nptes = miter.length >> FAST_PAGE_SHIFT;
+
+ ptep = iopte_pmd_offset(mapping->pgtbl_pmds,
+ mapping->domain->geometry.aperture_start,
+ iova_iter);
+ if (unlikely(av8l_fast_map_public(
+ ptep, page_to_phys(miter.page),
+ miter.length, prot))) {
+ dev_err(dev, "no map public\n");
+ /* TODO: unwind previously successful mappings */
+ goto out_free_iova;
+ }
+ fast_dmac_clean_range(mapping, ptep, ptep + nptes);
+ iova_iter += miter.length;
+ }
+ sg_miter_stop(&miter);
+ spin_unlock_irqrestore(&mapping->lock, flags);
+
+ addr = dma_common_pages_remap(pages, size, VM_USERMAP, remap_prot,
+ __builtin_return_address(0));
+ if (!addr) {
+ dev_err(dev, "no common pages\n");
+ goto out_unmap;
+ }
+
+ *handle = dma_addr;
+ sg_free_table(&sgt);
+ return addr;
+
+out_unmap:
+ /* need to take the lock again for page tables and iova */
+ spin_lock_irqsave(&mapping->lock, flags);
+ ptep = iopte_pmd_offset(mapping->pgtbl_pmds,
+ mapping->domain->geometry.aperture_start,
+ dma_addr);
+ av8l_fast_unmap_public(ptep, size);
+ fast_dmac_clean_range(mapping, ptep, ptep + count);
+out_free_iova:
+ __fast_smmu_free_iova(mapping, dma_addr, size);
+ spin_unlock_irqrestore(&mapping->lock, flags);
+out_free_sg:
+ sg_free_table(&sgt);
+out_free_pages:
+ __fast_smmu_free_pages(pages, count);
+ return NULL;
+}
+
+static void fast_smmu_free(struct device *dev, size_t size,
+ void *vaddr, dma_addr_t dma_handle,
+ struct dma_attrs *attrs)
+{
+ struct dma_fast_smmu_mapping *mapping = dev->archdata.mapping->fast;
+ struct vm_struct *area;
+ struct page **pages;
+ size_t count = ALIGN(size, SZ_4K) >> FAST_PAGE_SHIFT;
+ av8l_fast_iopte *ptep;
+ unsigned long flags;
+
+ size = ALIGN(size, SZ_4K);
+
+ area = find_vm_area(vaddr);
+ if (WARN_ON_ONCE(!area))
+ return;
+
+ pages = area->pages;
+ dma_common_free_remap(vaddr, size, VM_USERMAP, false);
+ ptep = iopte_pmd_offset(mapping->pgtbl_pmds,
+ mapping->domain->geometry.aperture_start, dma_handle);
+ spin_lock_irqsave(&mapping->lock, flags);
+ av8l_fast_unmap_public(ptep, size);
+ fast_dmac_clean_range(mapping, ptep, ptep + count);
+ __fast_smmu_free_iova(mapping, dma_handle, size);
+ spin_unlock_irqrestore(&mapping->lock, flags);
+ __fast_smmu_free_pages(pages, count);
+}
+
+static int fast_smmu_mmap_attrs(struct device *dev, struct vm_area_struct *vma,
+ void *cpu_addr, dma_addr_t dma_addr,
+ size_t size, struct dma_attrs *attrs)
+{
+ struct vm_struct *area;
+ unsigned long uaddr = vma->vm_start;
+ struct page **pages;
+ int i, nr_pages, ret = 0;
+ bool coherent = is_dma_coherent(dev, attrs);
+
+ vma->vm_page_prot = __get_dma_pgprot(attrs, vma->vm_page_prot,
+ coherent);
+ area = find_vm_area(cpu_addr);
+ if (!area)
+ return -EINVAL;
+
+ pages = area->pages;
+ nr_pages = PAGE_ALIGN(size) >> PAGE_SHIFT;
+ for (i = vma->vm_pgoff; i < nr_pages && uaddr < vma->vm_end; i++) {
+ ret = vm_insert_page(vma, uaddr, pages[i]);
+ if (ret)
+ break;
+ uaddr += PAGE_SIZE;
+ }
+
+ return ret;
+}
+
+static int fast_smmu_dma_supported(struct device *dev, u64 mask)
+{
+ return mask <= 0xffffffff;
+}
+
+static int fast_smmu_mapping_error(struct device *dev,
+ dma_addr_t dma_addr)
+{
+ return dma_addr == DMA_ERROR_CODE;
+}
+
+static void __fast_smmu_mapped_over_stale(struct dma_fast_smmu_mapping *fast,
+ void *data)
+{
+ av8l_fast_iopte *ptep = data;
+ dma_addr_t iova;
+ unsigned long bitmap_idx;
+
+ bitmap_idx = (unsigned long)(ptep - fast->pgtbl_pmds);
+ iova = bitmap_idx << FAST_PAGE_SHIFT;
+ dev_err(fast->dev, "Mapped over stale tlb at %pa\n", &iova);
+ dev_err(fast->dev, "bitmap (failure at idx %lu):\n", bitmap_idx);
+ dev_err(fast->dev, "ptep: %p pmds: %p diff: %lu\n", ptep,
+ fast->pgtbl_pmds, bitmap_idx);
+ print_hex_dump(KERN_ERR, "bmap: ", DUMP_PREFIX_ADDRESS,
+ 32, 8, fast->bitmap, fast->bitmap_size, false);
+}
+
+static int fast_smmu_notify(struct notifier_block *self,
+ unsigned long action, void *data)
+{
+ struct dma_fast_smmu_mapping *fast = container_of(
+ self, struct dma_fast_smmu_mapping, notifier);
+
+ switch (action) {
+ case MAPPED_OVER_STALE_TLB:
+ __fast_smmu_mapped_over_stale(fast, data);
+ return NOTIFY_OK;
+ default:
+ WARN(1, "Unhandled notifier action");
+ return NOTIFY_DONE;
+ }
+}
+
+static const struct dma_map_ops fast_smmu_dma_ops = {
+ .alloc = fast_smmu_alloc,
+ .free = fast_smmu_free,
+ .mmap = fast_smmu_mmap_attrs,
+ .map_page = fast_smmu_map_page,
+ .unmap_page = fast_smmu_unmap_page,
+ .sync_single_for_cpu = fast_smmu_sync_single_for_cpu,
+ .sync_single_for_device = fast_smmu_sync_single_for_device,
+ .map_sg = fast_smmu_map_sg,
+ .unmap_sg = fast_smmu_unmap_sg,
+ .sync_sg_for_cpu = fast_smmu_sync_sg_for_cpu,
+ .sync_sg_for_device = fast_smmu_sync_sg_for_device,
+ .dma_supported = fast_smmu_dma_supported,
+ .mapping_error = fast_smmu_mapping_error,
+};
+
+/**
+ * __fast_smmu_create_mapping_sized
+ * @base: bottom of the VA range
+ * @size: size of the VA range in bytes
+ *
+ * Creates a mapping structure which holds information about used/unused IO
+ * address ranges, which is required to perform mapping with IOMMU aware
+ * functions. The only VA range supported is [0, 4GB].
+ *
+ * The client device need to be attached to the mapping with
+ * fast_smmu_attach_device function.
+ */
+static struct dma_fast_smmu_mapping *__fast_smmu_create_mapping_sized(
+ dma_addr_t base, u64 size)
+{
+ struct dma_fast_smmu_mapping *fast;
+
+ fast = kzalloc(sizeof(struct dma_fast_smmu_mapping), GFP_KERNEL);
+ if (!fast)
+ goto err;
+
+ fast->base = base;
+ fast->size = size;
+ fast->num_4k_pages = size >> FAST_PAGE_SHIFT;
+ fast->bitmap_size = BITS_TO_LONGS(fast->num_4k_pages) * sizeof(long);
+
+ fast->bitmap = kzalloc(fast->bitmap_size, GFP_KERNEL | __GFP_NOWARN |
+ __GFP_NORETRY);
+ if (!fast->bitmap)
+ fast->bitmap = vzalloc(fast->bitmap_size);
+
+ if (!fast->bitmap)
+ goto err2;
+
+ spin_lock_init(&fast->lock);
+
+ return fast;
+err2:
+ kfree(fast);
+err:
+ return ERR_PTR(-ENOMEM);
+}
+
+/**
+ * fast_smmu_attach_device
+ * @dev: valid struct device pointer
+ * @mapping: io address space mapping structure (returned from
+ * fast_smmu_create_mapping)
+ *
+ * Attaches specified io address space mapping to the provided device,
+ * this replaces the dma operations (dma_map_ops pointer) with the
+ * IOMMU aware version. More than one client might be attached to
+ * the same io address space mapping.
+ */
+int fast_smmu_attach_device(struct device *dev,
+ struct dma_iommu_mapping *mapping)
+{
+ int atomic_domain = 1;
+ struct iommu_domain *domain = mapping->domain;
+ struct iommu_pgtbl_info info;
+ u64 size = (u64)mapping->bits << PAGE_SHIFT;
+ struct iommu_domain_geometry geometry;
+
+ if (mapping->base + size > (SZ_1G * 4ULL))
+ return -EINVAL;
+
+ if (iommu_domain_set_attr(domain, DOMAIN_ATTR_ATOMIC,
+ &atomic_domain))
+ return -EINVAL;
+
+ mapping->fast = __fast_smmu_create_mapping_sized(mapping->base, size);
+ if (IS_ERR(mapping->fast))
+ return -ENOMEM;
+ mapping->fast->domain = domain;
+ mapping->fast->dev = dev;
+
+ geometry.aperture_start = mapping->base;
+ geometry.aperture_end = mapping->base + size - 1;
+ if (iommu_domain_set_attr(domain, DOMAIN_ATTR_GEOMETRY,
+ &geometry))
+ return -EINVAL;
+
+ if (iommu_attach_device(domain, dev))
+ return -EINVAL;
+
+ if (iommu_domain_get_attr(domain, DOMAIN_ATTR_PGTBL_INFO,
+ &info)) {
+ dev_err(dev, "Couldn't get page table info\n");
+ fast_smmu_detach_device(dev, mapping);
+ return -EINVAL;
+ }
+ mapping->fast->pgtbl_pmds = info.pmds;
+
+ if (iommu_domain_get_attr(domain, DOMAIN_ATTR_PAGE_TABLE_IS_COHERENT,
+ &mapping->fast->is_smmu_pt_coherent))
+ return -EINVAL;
+
+ mapping->fast->notifier.notifier_call = fast_smmu_notify;
+ av8l_register_notify(&mapping->fast->notifier);
+
+ dev->archdata.mapping = mapping;
+ set_dma_ops(dev, &fast_smmu_dma_ops);
+
+ return 0;
+}
+EXPORT_SYMBOL(fast_smmu_attach_device);
+
+/**
+ * fast_smmu_detach_device
+ * @dev: valid struct device pointer
+ *
+ * Detaches the provided device from a previously attached map.
+ * This voids the dma operations (dma_map_ops pointer)
+ */
+void fast_smmu_detach_device(struct device *dev,
+ struct dma_iommu_mapping *mapping)
+{
+ iommu_detach_device(mapping->domain, dev);
+ dev->archdata.mapping = NULL;
+ set_dma_ops(dev, NULL);
+
+ kvfree(mapping->fast->bitmap);
+ kfree(mapping->fast);
+}
+EXPORT_SYMBOL(fast_smmu_detach_device);
diff --git a/drivers/iommu/io-pgtable-arm.c b/drivers/iommu/io-pgtable-arm.c
index 18751b1dfd3d..3f1617ca2fc0 100644
--- a/drivers/iommu/io-pgtable-arm.c
+++ b/drivers/iommu/io-pgtable-arm.c
@@ -22,6 +22,7 @@
#include <linux/iommu.h>
#include <linux/kernel.h>
+#include <linux/scatterlist.h>
#include <linux/sizes.h>
#include <linux/slab.h>
#include <linux/types.h>
@@ -68,9 +69,12 @@
#define ARM_LPAE_PGD_IDX(l,d) \
((l) == ARM_LPAE_START_LVL(d) ? ilog2(ARM_LPAE_PAGES_PER_PGD(d)) : 0)
+#define ARM_LPAE_LVL_MASK(l, d) \
+ ((l) == ARM_LPAE_START_LVL(d) ? (1 << (d)->pgd_bits) - 1 : \
+ (1 << (d)->bits_per_level) - 1)
#define ARM_LPAE_LVL_IDX(a,l,d) \
(((u64)(a) >> ARM_LPAE_LVL_SHIFT(l,d)) & \
- ((1 << ((d)->bits_per_level + ARM_LPAE_PGD_IDX(l,d))) - 1))
+ ARM_LPAE_LVL_MASK(l, d))
/* Calculate the block/page mapping size at level l for pagetable in d. */
#define ARM_LPAE_BLOCK_SIZE(l,d) \
@@ -85,6 +89,7 @@
#define ARM_LPAE_PTE_TYPE_TABLE 3
#define ARM_LPAE_PTE_TYPE_PAGE 3
+#define ARM_LPAE_PTE_SH_MASK (((arm_lpae_iopte)0x3) << 8)
#define ARM_LPAE_PTE_NSTABLE (((arm_lpae_iopte)1) << 63)
#define ARM_LPAE_PTE_XN (((arm_lpae_iopte)3) << 53)
#define ARM_LPAE_PTE_AF (((arm_lpae_iopte)1) << 10)
@@ -101,8 +106,11 @@
ARM_LPAE_PTE_ATTR_HI_MASK)
/* Stage-1 PTE */
-#define ARM_LPAE_PTE_AP_UNPRIV (((arm_lpae_iopte)1) << 6)
-#define ARM_LPAE_PTE_AP_RDONLY (((arm_lpae_iopte)2) << 6)
+#define ARM_LPAE_PTE_AP_PRIV_RW (((arm_lpae_iopte)0) << 6)
+#define ARM_LPAE_PTE_AP_RW (((arm_lpae_iopte)1) << 6)
+#define ARM_LPAE_PTE_AP_PRIV_RO (((arm_lpae_iopte)2) << 6)
+#define ARM_LPAE_PTE_AP_RO (((arm_lpae_iopte)3) << 6)
+#define ARM_LPAE_PTE_ATTRINDX_MASK 0x7
#define ARM_LPAE_PTE_ATTRINDX_SHIFT 2
#define ARM_LPAE_PTE_nG (((arm_lpae_iopte)1) << 11)
@@ -124,14 +132,21 @@
#define ARM_LPAE_TCR_TG0_64K (1 << 14)
#define ARM_LPAE_TCR_TG0_16K (2 << 14)
+#define ARM_LPAE_TCR_TG1_16K 1ULL
+#define ARM_LPAE_TCR_TG1_4K 2ULL
+#define ARM_LPAE_TCR_TG1_64K 3ULL
+
#define ARM_LPAE_TCR_SH0_SHIFT 12
#define ARM_LPAE_TCR_SH0_MASK 0x3
+#define ARM_LPAE_TCR_SH1_SHIFT 28
#define ARM_LPAE_TCR_SH_NS 0
#define ARM_LPAE_TCR_SH_OS 2
#define ARM_LPAE_TCR_SH_IS 3
#define ARM_LPAE_TCR_ORGN0_SHIFT 10
+#define ARM_LPAE_TCR_ORGN1_SHIFT 26
#define ARM_LPAE_TCR_IRGN0_SHIFT 8
+#define ARM_LPAE_TCR_IRGN1_SHIFT 24
#define ARM_LPAE_TCR_RGN_MASK 0x3
#define ARM_LPAE_TCR_RGN_NC 0
#define ARM_LPAE_TCR_RGN_WBWA 1
@@ -144,6 +159,9 @@
#define ARM_LPAE_TCR_T0SZ_SHIFT 0
#define ARM_LPAE_TCR_SZ_MASK 0xf
+#define ARM_LPAE_TCR_T1SZ_SHIFT 16
+#define ARM_LPAE_TCR_T1SZ_MASK 0x3f
+
#define ARM_LPAE_TCR_PS_SHIFT 16
#define ARM_LPAE_TCR_PS_MASK 0x7
@@ -157,6 +175,19 @@
#define ARM_LPAE_TCR_PS_44_BIT 0x4ULL
#define ARM_LPAE_TCR_PS_48_BIT 0x5ULL
+#define ARM_LPAE_TCR_EPD1_SHIFT 23
+#define ARM_LPAE_TCR_EPD1_FAULT 1
+
+#define ARM_LPAE_TCR_SEP_SHIFT (15 + 32)
+
+#define ARM_LPAE_TCR_SEP_31 0ULL
+#define ARM_LPAE_TCR_SEP_35 1ULL
+#define ARM_LPAE_TCR_SEP_39 2ULL
+#define ARM_LPAE_TCR_SEP_41 3ULL
+#define ARM_LPAE_TCR_SEP_43 4ULL
+#define ARM_LPAE_TCR_SEP_47 5ULL
+#define ARM_LPAE_TCR_SEP_UPSTREAM 7ULL
+
#define ARM_LPAE_MAIR_ATTR_SHIFT(n) ((n) << 3)
#define ARM_LPAE_MAIR_ATTR_MASK 0xff
#define ARM_LPAE_MAIR_ATTR_DEVICE 0x04
@@ -167,8 +198,8 @@
#define ARM_LPAE_MAIR_ATTR_IDX_DEV 2
/* IOPTE accessors */
-#define iopte_deref(pte,d) \
- (__va((pte) & ((1ULL << ARM_LPAE_MAX_ADDR_BITS) - 1) \
+#define iopte_deref(pte, d) \
+ (__va(iopte_val(pte) & ((1ULL << ARM_LPAE_MAX_ADDR_BITS) - 1) \
& ~((1ULL << (d)->pg_shift) - 1)))
#define iopte_type(pte,l) \
@@ -191,28 +222,113 @@ struct arm_lpae_io_pgtable {
struct io_pgtable iop;
int levels;
+ unsigned int pgd_bits;
size_t pgd_size;
unsigned long pg_shift;
unsigned long bits_per_level;
- void *pgd;
+ void *pgd[2];
};
typedef u64 arm_lpae_iopte;
static bool selftest_running = false;
+/*
+ * We'll use some ignored bits in table entries to keep track of the number
+ * of page mappings beneath the table. The maximum number of entries
+ * beneath any table mapping in armv8 is 8192 (which is possible at the
+ * 2nd- and 3rd-level when using a 64K granule size). The bits at our
+ * disposal are:
+ *
+ * 4k granule: [58..52], [11..2]
+ * 64k granule: [58..52], [15..2]
+ *
+ * [58..52], [11..2] is enough bits for tracking table mappings at any
+ * level for any granule, so we'll use those.
+ */
+#define BOTTOM_IGNORED_MASK 0x3ff
+#define BOTTOM_IGNORED_SHIFT 2
+#define BOTTOM_IGNORED_NUM_BITS 10
+#define TOP_IGNORED_MASK 0x7fULL
+#define TOP_IGNORED_SHIFT 52
+#define IOPTE_RESERVED_MASK ((BOTTOM_IGNORED_MASK << BOTTOM_IGNORED_SHIFT) | \
+ (TOP_IGNORED_MASK << TOP_IGNORED_SHIFT))
+
+static arm_lpae_iopte iopte_val(arm_lpae_iopte table_pte)
+{
+ return table_pte & ~IOPTE_RESERVED_MASK;
+}
+
+static arm_lpae_iopte _iopte_bottom_ignored_val(arm_lpae_iopte table_pte)
+{
+ return (table_pte & (BOTTOM_IGNORED_MASK << BOTTOM_IGNORED_SHIFT))
+ >> BOTTOM_IGNORED_SHIFT;
+}
+
+static arm_lpae_iopte _iopte_top_ignored_val(arm_lpae_iopte table_pte)
+{
+ return (table_pte & (TOP_IGNORED_MASK << TOP_IGNORED_SHIFT))
+ >> TOP_IGNORED_SHIFT;
+}
+
+static int iopte_tblcnt(arm_lpae_iopte table_pte)
+{
+ return (_iopte_bottom_ignored_val(table_pte) |
+ (_iopte_top_ignored_val(table_pte) << BOTTOM_IGNORED_NUM_BITS));
+}
+
+static void iopte_tblcnt_set(arm_lpae_iopte *table_pte, int val)
+{
+ arm_lpae_iopte pte = iopte_val(*table_pte);
+
+ pte |= ((val & BOTTOM_IGNORED_MASK) << BOTTOM_IGNORED_SHIFT) |
+ (((val & (TOP_IGNORED_MASK << BOTTOM_IGNORED_NUM_BITS))
+ >> BOTTOM_IGNORED_NUM_BITS) << TOP_IGNORED_SHIFT);
+ *table_pte = pte;
+}
+
+static void iopte_tblcnt_sub(arm_lpae_iopte *table_ptep, int cnt)
+{
+ arm_lpae_iopte current_cnt = iopte_tblcnt(*table_ptep);
+
+ current_cnt -= cnt;
+ iopte_tblcnt_set(table_ptep, current_cnt);
+}
+
+static void iopte_tblcnt_add(arm_lpae_iopte *table_ptep, int cnt)
+{
+ arm_lpae_iopte current_cnt = iopte_tblcnt(*table_ptep);
+
+ current_cnt += cnt;
+ iopte_tblcnt_set(table_ptep, current_cnt);
+}
+
+static bool suppress_map_failures;
+
static dma_addr_t __arm_lpae_dma_addr(void *pages)
{
return (dma_addr_t)virt_to_phys(pages);
}
+static inline void pgtable_dma_sync_single_for_device(
+ struct io_pgtable_cfg *cfg,
+ dma_addr_t addr, size_t size,
+ enum dma_data_direction dir)
+{
+ if (!(cfg->quirks & IO_PGTABLE_QUIRK_PAGE_TABLE_COHERENT))
+ dma_sync_single_for_device(cfg->iommu_dev, addr, size,
+ dir);
+}
+
static void *__arm_lpae_alloc_pages(size_t size, gfp_t gfp,
- struct io_pgtable_cfg *cfg)
+ struct io_pgtable_cfg *cfg,
+ void *cookie)
{
struct device *dev = cfg->iommu_dev;
dma_addr_t dma;
- void *pages = alloc_pages_exact(size, gfp | __GFP_ZERO);
+ void *pages = io_pgtable_alloc_pages_exact(cfg, cookie,
+ size, gfp | __GFP_ZERO);
if (!pages)
return NULL;
@@ -236,17 +352,17 @@ out_unmap:
dev_err(dev, "Cannot accommodate DMA translation for IOMMU page tables\n");
dma_unmap_single(dev, dma, size, DMA_TO_DEVICE);
out_free:
- free_pages_exact(pages, size);
+ io_pgtable_free_pages_exact(cfg, cookie, pages, size);
return NULL;
}
static void __arm_lpae_free_pages(void *pages, size_t size,
- struct io_pgtable_cfg *cfg)
+ struct io_pgtable_cfg *cfg, void *cookie)
{
if (!selftest_running)
dma_unmap_single(cfg->iommu_dev, __arm_lpae_dma_addr(pages),
size, DMA_TO_DEVICE);
- free_pages_exact(pages, size);
+ io_pgtable_free_pages_exact(cfg, cookie, pages, size);
}
static void __arm_lpae_set_pte(arm_lpae_iopte *ptep, arm_lpae_iopte pte,
@@ -255,38 +371,24 @@ static void __arm_lpae_set_pte(arm_lpae_iopte *ptep, arm_lpae_iopte pte,
*ptep = pte;
if (!selftest_running)
- dma_sync_single_for_device(cfg->iommu_dev,
+ pgtable_dma_sync_single_for_device(cfg,
__arm_lpae_dma_addr(ptep),
sizeof(pte), DMA_TO_DEVICE);
}
-static int __arm_lpae_unmap(struct arm_lpae_io_pgtable *data,
- unsigned long iova, size_t size, int lvl,
- arm_lpae_iopte *ptep);
-
static int arm_lpae_init_pte(struct arm_lpae_io_pgtable *data,
unsigned long iova, phys_addr_t paddr,
arm_lpae_iopte prot, int lvl,
- arm_lpae_iopte *ptep)
+ arm_lpae_iopte *ptep, arm_lpae_iopte *prev_ptep,
+ bool flush)
{
arm_lpae_iopte pte = prot;
struct io_pgtable_cfg *cfg = &data->iop.cfg;
- if (iopte_leaf(*ptep, lvl)) {
- /* We require an unmap first */
- WARN_ON(!selftest_running);
+ /* We require an unmap first */
+ if (*ptep & ARM_LPAE_PTE_VALID) {
+ BUG_ON(!suppress_map_failures);
return -EEXIST;
- } else if (iopte_type(*ptep, lvl) == ARM_LPAE_PTE_TYPE_TABLE) {
- /*
- * We need to unmap and free the old table before
- * overwriting it with a block entry.
- */
- arm_lpae_iopte *tblp;
- size_t sz = ARM_LPAE_BLOCK_SIZE(lvl, data);
-
- tblp = ptep - ARM_LPAE_LVL_IDX(iova, lvl, data);
- if (WARN_ON(__arm_lpae_unmap(data, iova, sz, lvl, tblp) != sz))
- return -EINVAL;
}
if (cfg->quirks & IO_PGTABLE_QUIRK_ARM_NS)
@@ -297,27 +399,82 @@ static int arm_lpae_init_pte(struct arm_lpae_io_pgtable *data,
else
pte |= ARM_LPAE_PTE_TYPE_BLOCK;
- pte |= ARM_LPAE_PTE_AF | ARM_LPAE_PTE_SH_IS;
+ pte |= ARM_LPAE_PTE_AF | ARM_LPAE_PTE_SH_OS;
pte |= pfn_to_iopte(paddr >> data->pg_shift, data);
- __arm_lpae_set_pte(ptep, pte, cfg);
+ *ptep = pte;
+
+ if (flush)
+ __arm_lpae_set_pte(ptep, pte, cfg);
+
+ if (prev_ptep)
+ iopte_tblcnt_add(prev_ptep, 1);
+
return 0;
}
+struct map_state {
+ unsigned long iova_end;
+ unsigned int pgsize;
+ arm_lpae_iopte *pgtable;
+ arm_lpae_iopte *prev_pgtable;
+ arm_lpae_iopte *pte_start;
+ unsigned int num_pte;
+};
+/* map state optimization works at level 3 (the 2nd-to-last level) */
+#define MAP_STATE_LVL 3
+
static int __arm_lpae_map(struct arm_lpae_io_pgtable *data, unsigned long iova,
phys_addr_t paddr, size_t size, arm_lpae_iopte prot,
- int lvl, arm_lpae_iopte *ptep)
+ int lvl, arm_lpae_iopte *ptep,
+ arm_lpae_iopte *prev_ptep, struct map_state *ms)
{
arm_lpae_iopte *cptep, pte;
+ void *cookie = data->iop.cookie;
size_t block_size = ARM_LPAE_BLOCK_SIZE(lvl, data);
+ arm_lpae_iopte *pgtable = ptep;
struct io_pgtable_cfg *cfg = &data->iop.cfg;
/* Find our entry at the current level */
ptep += ARM_LPAE_LVL_IDX(iova, lvl, data);
/* If we can install a leaf entry at this level, then do so */
- if (size == block_size && (size & cfg->pgsize_bitmap))
- return arm_lpae_init_pte(data, iova, paddr, prot, lvl, ptep);
+ if (size == block_size && (size & cfg->pgsize_bitmap)) {
+ if (!ms)
+ return arm_lpae_init_pte(data, iova, paddr, prot, lvl,
+ ptep, prev_ptep, true);
+
+ if (lvl == MAP_STATE_LVL) {
+ if (ms->pgtable)
+ pgtable_dma_sync_single_for_device(cfg,
+ __arm_lpae_dma_addr(ms->pte_start),
+ ms->num_pte * sizeof(*ptep),
+ DMA_TO_DEVICE);
+
+ ms->iova_end = round_down(iova, SZ_2M) + SZ_2M;
+ ms->pgtable = pgtable;
+ ms->prev_pgtable = prev_ptep;
+ ms->pgsize = size;
+ ms->pte_start = ptep;
+ ms->num_pte = 1;
+ } else {
+ /*
+ * We have some map state from previous page
+ * mappings, but we're about to set up a block
+ * mapping. Flush out the previous page mappings.
+ */
+ if (ms->pgtable)
+ pgtable_dma_sync_single_for_device(cfg,
+ __arm_lpae_dma_addr(ms->pte_start),
+ ms->num_pte * sizeof(*ptep),
+ DMA_TO_DEVICE);
+ memset(ms, 0, sizeof(*ms));
+ ms = NULL;
+ }
+
+ return arm_lpae_init_pte(data, iova, paddr, prot, lvl, ptep,
+ prev_ptep, ms == NULL);
+ }
/* We can't allocate tables at the final level */
if (WARN_ON(lvl >= ARM_LPAE_MAX_LEVELS - 1))
@@ -327,7 +484,7 @@ static int __arm_lpae_map(struct arm_lpae_io_pgtable *data, unsigned long iova,
pte = *ptep;
if (!pte) {
cptep = __arm_lpae_alloc_pages(1UL << data->pg_shift,
- GFP_ATOMIC, cfg);
+ GFP_ATOMIC, cfg, cookie);
if (!cptep)
return -ENOMEM;
@@ -344,7 +501,8 @@ static int __arm_lpae_map(struct arm_lpae_io_pgtable *data, unsigned long iova,
}
/* Rinse, repeat */
- return __arm_lpae_map(data, iova, paddr, size, prot, lvl + 1, cptep);
+ return __arm_lpae_map(data, iova, paddr, size, prot, lvl + 1, cptep,
+ ptep, ms);
}
static arm_lpae_iopte arm_lpae_prot_to_pte(struct arm_lpae_io_pgtable *data,
@@ -354,14 +512,22 @@ static arm_lpae_iopte arm_lpae_prot_to_pte(struct arm_lpae_io_pgtable *data,
if (data->iop.fmt == ARM_64_LPAE_S1 ||
data->iop.fmt == ARM_32_LPAE_S1) {
- pte = ARM_LPAE_PTE_AP_UNPRIV | ARM_LPAE_PTE_nG;
+ pte = ARM_LPAE_PTE_nG;
- if (!(prot & IOMMU_WRITE) && (prot & IOMMU_READ))
- pte |= ARM_LPAE_PTE_AP_RDONLY;
+ if (prot & IOMMU_WRITE)
+ pte |= (prot & IOMMU_PRIV) ? ARM_LPAE_PTE_AP_PRIV_RW
+ : ARM_LPAE_PTE_AP_RW;
+ else
+ pte |= (prot & IOMMU_PRIV) ? ARM_LPAE_PTE_AP_PRIV_RO
+ : ARM_LPAE_PTE_AP_RO;
if (prot & IOMMU_CACHE)
pte |= (ARM_LPAE_MAIR_ATTR_IDX_CACHE
<< ARM_LPAE_PTE_ATTRINDX_SHIFT);
+
+ if (prot & IOMMU_DEVICE)
+ pte |= (ARM_LPAE_MAIR_ATTR_IDX_DEV <<
+ ARM_LPAE_PTE_ATTRINDX_SHIFT);
} else {
pte = ARM_LPAE_PTE_HAP_FAULT;
if (prot & IOMMU_READ)
@@ -372,6 +538,9 @@ static arm_lpae_iopte arm_lpae_prot_to_pte(struct arm_lpae_io_pgtable *data,
pte |= ARM_LPAE_PTE_MEMATTR_OIWB;
else
pte |= ARM_LPAE_PTE_MEMATTR_NC;
+
+ if (prot & IOMMU_DEVICE)
+ pte |= ARM_LPAE_PTE_MEMATTR_DEV;
}
if (prot & IOMMU_NOEXEC)
@@ -380,20 +549,42 @@ static arm_lpae_iopte arm_lpae_prot_to_pte(struct arm_lpae_io_pgtable *data,
return pte;
}
+static inline arm_lpae_iopte *arm_lpae_get_table(
+ struct arm_lpae_io_pgtable *data, unsigned long iova)
+{
+ struct io_pgtable_cfg *cfg = &data->iop.cfg;
+
+ /*
+ * iovas for TTBR1 will have all the bits set between the input address
+ * region and the sign extension bit
+ */
+ if (unlikely(cfg->quirks & IO_PGTABLE_QUIRK_ARM_TTBR1)) {
+ unsigned long mask = GENMASK(cfg->sep, cfg->ias);
+
+ if ((iova & mask) == mask)
+ return data->pgd[1];
+ }
+
+ return data->pgd[0];
+}
+
static int arm_lpae_map(struct io_pgtable_ops *ops, unsigned long iova,
phys_addr_t paddr, size_t size, int iommu_prot)
{
struct arm_lpae_io_pgtable *data = io_pgtable_ops_to_data(ops);
- arm_lpae_iopte *ptep = data->pgd;
+ arm_lpae_iopte *ptep;
int ret, lvl = ARM_LPAE_START_LVL(data);
arm_lpae_iopte prot;
+ ptep = arm_lpae_get_table(data, iova);
+
/* If no access, then nothing to do */
if (!(iommu_prot & (IOMMU_READ | IOMMU_WRITE)))
return 0;
prot = arm_lpae_prot_to_pte(data, iommu_prot);
- ret = __arm_lpae_map(data, iova, paddr, size, prot, lvl, ptep);
+ ret = __arm_lpae_map(data, iova, paddr, size, prot, lvl, ptep, NULL,
+ NULL);
/*
* Synchronise all PTE updates for the new mapping before there's
* a chance for anything to kick off a table walk for the new iova.
@@ -403,6 +594,91 @@ static int arm_lpae_map(struct io_pgtable_ops *ops, unsigned long iova,
return ret;
}
+static int arm_lpae_map_sg(struct io_pgtable_ops *ops, unsigned long iova,
+ struct scatterlist *sg, unsigned int nents,
+ int iommu_prot, size_t *size)
+{
+ struct arm_lpae_io_pgtable *data = io_pgtable_ops_to_data(ops);
+ struct io_pgtable_cfg *cfg = &data->iop.cfg;
+ arm_lpae_iopte *ptep;
+ int lvl = ARM_LPAE_START_LVL(data);
+ arm_lpae_iopte prot;
+ struct scatterlist *s;
+ size_t mapped = 0;
+ int i, ret;
+ unsigned int min_pagesz;
+ struct map_state ms;
+
+ ptep = arm_lpae_get_table(data, iova);
+
+ /* If no access, then nothing to do */
+ if (!(iommu_prot & (IOMMU_READ | IOMMU_WRITE)))
+ goto out_err;
+
+ prot = arm_lpae_prot_to_pte(data, iommu_prot);
+
+ min_pagesz = 1 << __ffs(data->iop.cfg.pgsize_bitmap);
+
+ memset(&ms, 0, sizeof(ms));
+
+ for_each_sg(sg, s, nents, i) {
+ phys_addr_t phys = page_to_phys(sg_page(s)) + s->offset;
+ size_t size = s->length;
+
+ /*
+ * We are mapping on IOMMU page boundaries, so offset within
+ * the page must be 0. However, the IOMMU may support pages
+ * smaller than PAGE_SIZE, so s->offset may still represent
+ * an offset of that boundary within the CPU page.
+ */
+ if (!IS_ALIGNED(s->offset, min_pagesz))
+ goto out_err;
+
+ while (size) {
+ size_t pgsize = iommu_pgsize(
+ data->iop.cfg.pgsize_bitmap, iova | phys, size);
+
+ if (ms.pgtable && (iova < ms.iova_end)) {
+ arm_lpae_iopte *ptep = ms.pgtable +
+ ARM_LPAE_LVL_IDX(iova, MAP_STATE_LVL,
+ data);
+ arm_lpae_init_pte(
+ data, iova, phys, prot, MAP_STATE_LVL,
+ ptep, ms.prev_pgtable, false);
+ ms.num_pte++;
+ } else {
+ ret = __arm_lpae_map(data, iova, phys, pgsize,
+ prot, lvl, ptep, NULL, &ms);
+ if (ret)
+ goto out_err;
+ }
+
+ iova += pgsize;
+ mapped += pgsize;
+ phys += pgsize;
+ size -= pgsize;
+ }
+ }
+
+ if (ms.pgtable)
+ pgtable_dma_sync_single_for_device(cfg,
+ __arm_lpae_dma_addr(ms.pte_start),
+ ms.num_pte * sizeof(*ms.pte_start),
+ DMA_TO_DEVICE);
+ /*
+ * Synchronise all PTE updates for the new mapping before there's
+ * a chance for anything to kick off a table walk for the new iova.
+ */
+ wmb();
+
+ return mapped;
+
+out_err:
+ /* Return the size of the partial mapping so that they can be undone */
+ *size = mapped;
+ return 0;
+}
+
static void __arm_lpae_free_pgtable(struct arm_lpae_io_pgtable *data, int lvl,
arm_lpae_iopte *ptep)
{
@@ -422,6 +698,10 @@ static void __arm_lpae_free_pgtable(struct arm_lpae_io_pgtable *data, int lvl,
else
end = (void *)ptep + table_size;
+ /* Only leaf entries at the last level */
+ if (lvl == ARM_LPAE_MAX_LEVELS - 1)
+ goto end;
+
while (ptep != end) {
arm_lpae_iopte pte = *ptep++;
@@ -431,21 +711,27 @@ static void __arm_lpae_free_pgtable(struct arm_lpae_io_pgtable *data, int lvl,
__arm_lpae_free_pgtable(data, lvl + 1, iopte_deref(pte, data));
}
- __arm_lpae_free_pages(start, table_size, &data->iop.cfg);
+end:
+ __arm_lpae_free_pages(start, table_size, &data->iop.cfg,
+ data->iop.cookie);
}
static void arm_lpae_free_pgtable(struct io_pgtable *iop)
{
struct arm_lpae_io_pgtable *data = io_pgtable_to_data(iop);
- __arm_lpae_free_pgtable(data, ARM_LPAE_START_LVL(data), data->pgd);
+ __arm_lpae_free_pgtable(data, ARM_LPAE_START_LVL(data), data->pgd[0]);
+ if (data->pgd[1])
+ __arm_lpae_free_pgtable(data, ARM_LPAE_START_LVL(data),
+ data->pgd[1]);
kfree(data);
}
static int arm_lpae_split_blk_unmap(struct arm_lpae_io_pgtable *data,
unsigned long iova, size_t size,
arm_lpae_iopte prot, int lvl,
- arm_lpae_iopte *ptep, size_t blk_size)
+ arm_lpae_iopte *ptep,
+ arm_lpae_iopte *prev_ptep, size_t blk_size)
{
unsigned long blk_start, blk_end;
phys_addr_t blk_paddr;
@@ -455,6 +741,7 @@ static int arm_lpae_split_blk_unmap(struct arm_lpae_io_pgtable *data,
blk_start = iova & ~(blk_size - 1);
blk_end = blk_start + blk_size;
blk_paddr = iopte_to_pfn(*ptep, data) << data->pg_shift;
+ size = ARM_LPAE_BLOCK_SIZE(lvl + 1, data);
for (; blk_start < blk_end; blk_start += size, blk_paddr += size) {
arm_lpae_iopte *tablep;
@@ -466,7 +753,7 @@ static int arm_lpae_split_blk_unmap(struct arm_lpae_io_pgtable *data,
/* __arm_lpae_map expects a pointer to the start of the table */
tablep = &table - ARM_LPAE_LVL_IDX(blk_start, lvl, data);
if (__arm_lpae_map(data, blk_start, blk_paddr, size, prot, lvl,
- tablep) < 0) {
+ tablep, prev_ptep, NULL) < 0) {
if (table) {
/* Free the table we allocated */
tablep = iopte_deref(table, data);
@@ -477,17 +764,15 @@ static int arm_lpae_split_blk_unmap(struct arm_lpae_io_pgtable *data,
}
__arm_lpae_set_pte(ptep, table, cfg);
- iova &= ~(blk_size - 1);
- cfg->tlb->tlb_add_flush(iova, blk_size, true, data->iop.cookie);
return size;
}
static int __arm_lpae_unmap(struct arm_lpae_io_pgtable *data,
unsigned long iova, size_t size, int lvl,
- arm_lpae_iopte *ptep)
+ arm_lpae_iopte *ptep, arm_lpae_iopte *prev_ptep)
{
arm_lpae_iopte pte;
- const struct iommu_gather_ops *tlb = data->iop.cfg.tlb;
+ struct io_pgtable_cfg *cfg = &data->iop.cfg;
void *cookie = data->iop.cookie;
size_t blk_size = ARM_LPAE_BLOCK_SIZE(lvl, data);
@@ -504,15 +789,45 @@ static int __arm_lpae_unmap(struct arm_lpae_io_pgtable *data,
if (!iopte_leaf(pte, lvl)) {
/* Also flush any partial walks */
- tlb->tlb_add_flush(iova, size, false, cookie);
- tlb->tlb_sync(cookie);
ptep = iopte_deref(pte, data);
__arm_lpae_free_pgtable(data, lvl + 1, ptep);
- } else {
- tlb->tlb_add_flush(iova, size, true, cookie);
}
return size;
+ } else if ((lvl == ARM_LPAE_MAX_LEVELS - 2) && !iopte_leaf(pte, lvl)) {
+ arm_lpae_iopte *table = iopte_deref(pte, data);
+ arm_lpae_iopte *table_base = table;
+ int tl_offset = ARM_LPAE_LVL_IDX(iova, lvl + 1, data);
+ int entry_size = (1 << data->pg_shift);
+ int max_entries = ARM_LPAE_BLOCK_SIZE(lvl, data) / entry_size;
+ int entries = min_t(int, size / entry_size,
+ max_entries - tl_offset);
+ int table_len = entries * sizeof(*table);
+
+ /*
+ * This isn't a block mapping so it must be a table mapping
+ * and since it's the 2nd-to-last level the next level has
+ * to be all page mappings. Zero them all out in one fell
+ * swoop.
+ */
+
+ table += tl_offset;
+
+ memset(table, 0, table_len);
+ pgtable_dma_sync_single_for_device(cfg,
+ __arm_lpae_dma_addr(table),
+ table_len, DMA_TO_DEVICE);
+
+ iopte_tblcnt_sub(ptep, entries);
+ if (!iopte_tblcnt(*ptep)) {
+ /* no valid mappings left under this table. free it. */
+ __arm_lpae_set_pte(ptep, 0, cfg);
+ io_pgtable_free_pages_exact(
+ &data->iop.cfg, cookie, table_base,
+ max_entries * sizeof(*table_base));
+ }
+
+ return entries * entry_size;
} else if (iopte_leaf(pte, lvl)) {
/*
* Insert a table at the next level to map the old region,
@@ -520,63 +835,150 @@ static int __arm_lpae_unmap(struct arm_lpae_io_pgtable *data,
*/
return arm_lpae_split_blk_unmap(data, iova, size,
iopte_prot(pte), lvl, ptep,
+ prev_ptep,
blk_size);
}
/* Keep on walkin' */
+ prev_ptep = ptep;
ptep = iopte_deref(pte, data);
- return __arm_lpae_unmap(data, iova, size, lvl + 1, ptep);
+ return __arm_lpae_unmap(data, iova, size, lvl + 1, ptep, prev_ptep);
}
-static int arm_lpae_unmap(struct io_pgtable_ops *ops, unsigned long iova,
+static size_t arm_lpae_unmap(struct io_pgtable_ops *ops, unsigned long iova,
size_t size)
{
- size_t unmapped;
+ size_t unmapped = 0;
struct arm_lpae_io_pgtable *data = io_pgtable_ops_to_data(ops);
struct io_pgtable *iop = &data->iop;
- arm_lpae_iopte *ptep = data->pgd;
+ arm_lpae_iopte *ptep;
int lvl = ARM_LPAE_START_LVL(data);
- unmapped = __arm_lpae_unmap(data, iova, size, lvl, ptep);
+ ptep = arm_lpae_get_table(data, iova);
+
+ while (unmapped < size) {
+ size_t ret, size_to_unmap, remaining;
+
+ remaining = (size - unmapped);
+ size_to_unmap = remaining < SZ_2M
+ ? remaining
+ : iommu_pgsize(data->iop.cfg.pgsize_bitmap, iova,
+ remaining);
+ ret = __arm_lpae_unmap(data, iova, size_to_unmap, lvl, ptep,
+ NULL);
+ if (ret == 0)
+ break;
+ unmapped += ret;
+ iova += ret;
+ }
if (unmapped)
- iop->cfg.tlb->tlb_sync(iop->cookie);
+ iop->cfg.tlb->tlb_flush_all(iop->cookie);
return unmapped;
}
-static phys_addr_t arm_lpae_iova_to_phys(struct io_pgtable_ops *ops,
- unsigned long iova)
+static int arm_lpae_iova_to_pte(struct arm_lpae_io_pgtable *data,
+ unsigned long iova, int *plvl_ret,
+ arm_lpae_iopte *ptep_ret)
{
- struct arm_lpae_io_pgtable *data = io_pgtable_ops_to_data(ops);
- arm_lpae_iopte pte, *ptep = data->pgd;
- int lvl = ARM_LPAE_START_LVL(data);
+ arm_lpae_iopte pte, *ptep;
+
+ ptep = arm_lpae_get_table(data, iova);
+
+ *plvl_ret = ARM_LPAE_START_LVL(data);
+ *ptep_ret = 0;
do {
/* Valid IOPTE pointer? */
if (!ptep)
- return 0;
+ return -EINVAL;
/* Grab the IOPTE we're interested in */
- pte = *(ptep + ARM_LPAE_LVL_IDX(iova, lvl, data));
+ pte = *(ptep + ARM_LPAE_LVL_IDX(iova, *plvl_ret, data));
/* Valid entry? */
if (!pte)
- return 0;
+ return -EINVAL;
/* Leaf entry? */
- if (iopte_leaf(pte,lvl))
+ if (iopte_leaf(pte, *plvl_ret))
goto found_translation;
/* Take it to the next level */
ptep = iopte_deref(pte, data);
- } while (++lvl < ARM_LPAE_MAX_LEVELS);
+ } while (++(*plvl_ret) < ARM_LPAE_MAX_LEVELS);
/* Ran out of page tables to walk */
- return 0;
+ return -EINVAL;
found_translation:
- iova &= ((1 << data->pg_shift) - 1);
- return ((phys_addr_t)iopte_to_pfn(pte,data) << data->pg_shift) | iova;
+ *ptep_ret = pte;
+ return 0;
+}
+
+static uint64_t arm_lpae_iova_get_pte(struct io_pgtable_ops *ops,
+ unsigned long iova)
+{
+ struct arm_lpae_io_pgtable *data = io_pgtable_ops_to_data(ops);
+ arm_lpae_iopte pte;
+ int lvl;
+
+ if (!arm_lpae_iova_to_pte(data, iova, &lvl, &pte))
+ return pte;
+
+ return 0;
+}
+
+static phys_addr_t arm_lpae_iova_to_phys(struct io_pgtable_ops *ops,
+ unsigned long iova)
+{
+ struct arm_lpae_io_pgtable *data = io_pgtable_ops_to_data(ops);
+ arm_lpae_iopte pte;
+ int lvl;
+ phys_addr_t phys = 0;
+
+ if (!arm_lpae_iova_to_pte(data, iova, &lvl, &pte)) {
+ iova &= ((1 << ARM_LPAE_LVL_SHIFT(lvl, data)) - 1);
+ phys = ((phys_addr_t)iopte_to_pfn(pte, data)
+ << data->pg_shift) | iova;
+ }
+
+ return phys;
+}
+
+static bool __arm_lpae_is_iova_coherent(struct arm_lpae_io_pgtable *data,
+ arm_lpae_iopte *ptep)
+{
+ if (data->iop.fmt == ARM_64_LPAE_S1 ||
+ data->iop.fmt == ARM_32_LPAE_S1) {
+ int attr_idx = (*ptep & (ARM_LPAE_PTE_ATTRINDX_MASK <<
+ ARM_LPAE_PTE_ATTRINDX_SHIFT)) >>
+ ARM_LPAE_PTE_ATTRINDX_SHIFT;
+ if ((attr_idx == ARM_LPAE_MAIR_ATTR_IDX_CACHE) &&
+ (((*ptep & ARM_LPAE_PTE_SH_MASK) == ARM_LPAE_PTE_SH_IS)
+ ||
+ (*ptep & ARM_LPAE_PTE_SH_MASK) == ARM_LPAE_PTE_SH_OS))
+ return true;
+ } else {
+ if (*ptep & ARM_LPAE_PTE_MEMATTR_OIWB)
+ return true;
+ }
+
+ return false;
+}
+
+static bool arm_lpae_is_iova_coherent(struct io_pgtable_ops *ops,
+ unsigned long iova)
+{
+ struct arm_lpae_io_pgtable *data = io_pgtable_ops_to_data(ops);
+ arm_lpae_iopte pte;
+ int lvl;
+ bool ret = false;
+
+ if (!arm_lpae_iova_to_pte(data, iova, &lvl, &pte))
+ ret = __arm_lpae_is_iova_coherent(data, &pte);
+
+ return ret;
}
static void arm_lpae_restrict_pgsizes(struct io_pgtable_cfg *cfg)
@@ -648,17 +1050,86 @@ arm_lpae_alloc_pgtable(struct io_pgtable_cfg *cfg)
/* Calculate the actual size of our pgd (without concatenation) */
pgd_bits = va_bits - (data->bits_per_level * (data->levels - 1));
+ data->pgd_bits = pgd_bits;
data->pgd_size = 1UL << (pgd_bits + ilog2(sizeof(arm_lpae_iopte)));
data->iop.ops = (struct io_pgtable_ops) {
.map = arm_lpae_map,
+ .map_sg = arm_lpae_map_sg,
.unmap = arm_lpae_unmap,
.iova_to_phys = arm_lpae_iova_to_phys,
+ .is_iova_coherent = arm_lpae_is_iova_coherent,
+ .iova_to_pte = arm_lpae_iova_get_pte,
};
return data;
}
+static u64 arm64_lpae_setup_ttbr1(struct io_pgtable_cfg *cfg,
+ struct arm_lpae_io_pgtable *data)
+
+{
+ u64 reg;
+
+ /* If TTBR1 is disabled, disable speculative walks through the TTBR1 */
+ if (!(cfg->quirks & IO_PGTABLE_QUIRK_ARM_TTBR1)) {
+ reg = ARM_LPAE_TCR_EPD1;
+ reg |= (ARM_LPAE_TCR_SEP_UPSTREAM << ARM_LPAE_TCR_SEP_SHIFT);
+ return reg;
+ }
+
+ if (cfg->iommu_dev && cfg->iommu_dev->archdata.dma_coherent)
+ reg = (ARM_LPAE_TCR_SH_OS << ARM_LPAE_TCR_SH1_SHIFT) |
+ (ARM_LPAE_TCR_RGN_WBWA << ARM_LPAE_TCR_IRGN1_SHIFT) |
+ (ARM_LPAE_TCR_RGN_WBWA << ARM_LPAE_TCR_ORGN1_SHIFT);
+ else
+ reg = (ARM_LPAE_TCR_SH_OS << ARM_LPAE_TCR_SH1_SHIFT) |
+ (ARM_LPAE_TCR_RGN_NC << ARM_LPAE_TCR_IRGN1_SHIFT) |
+ (ARM_LPAE_TCR_RGN_NC << ARM_LPAE_TCR_ORGN1_SHIFT);
+
+ switch (1 << data->pg_shift) {
+ case SZ_4K:
+ reg |= (ARM_LPAE_TCR_TG1_4K << 30);
+ break;
+ case SZ_16K:
+ reg |= (ARM_LPAE_TCR_TG1_16K << 30);
+ break;
+ case SZ_64K:
+ reg |= (ARM_LPAE_TCR_TG1_64K << 30);
+ break;
+ }
+
+ /* Set T1SZ */
+ reg |= (64ULL - cfg->ias) << ARM_LPAE_TCR_T1SZ_SHIFT;
+
+ switch (cfg->sep) {
+ case 31:
+ reg |= (ARM_LPAE_TCR_SEP_31 << ARM_LPAE_TCR_SEP_SHIFT);
+ break;
+ case 35:
+ reg |= (ARM_LPAE_TCR_SEP_35 << ARM_LPAE_TCR_SEP_SHIFT);
+ break;
+ case 39:
+ reg |= (ARM_LPAE_TCR_SEP_39 << ARM_LPAE_TCR_SEP_SHIFT);
+ break;
+ case 41:
+ reg |= (ARM_LPAE_TCR_SEP_41 << ARM_LPAE_TCR_SEP_SHIFT);
+ break;
+ case 43:
+ reg |= (ARM_LPAE_TCR_SEP_43 << ARM_LPAE_TCR_SEP_SHIFT);
+ break;
+ case 47:
+ reg |= (ARM_LPAE_TCR_SEP_47 << ARM_LPAE_TCR_SEP_SHIFT);
+ break;
+ case 48:
+ default:
+ reg |= (ARM_LPAE_TCR_SEP_UPSTREAM << ARM_LPAE_TCR_SEP_SHIFT);
+ break;
+ }
+
+ return reg;
+}
+
static struct io_pgtable *
arm_64_lpae_alloc_pgtable_s1(struct io_pgtable_cfg *cfg, void *cookie)
{
@@ -669,9 +1140,14 @@ arm_64_lpae_alloc_pgtable_s1(struct io_pgtable_cfg *cfg, void *cookie)
return NULL;
/* TCR */
- reg = (ARM_LPAE_TCR_SH_IS << ARM_LPAE_TCR_SH0_SHIFT) |
- (ARM_LPAE_TCR_RGN_WBWA << ARM_LPAE_TCR_IRGN0_SHIFT) |
- (ARM_LPAE_TCR_RGN_WBWA << ARM_LPAE_TCR_ORGN0_SHIFT);
+ if (cfg->quirks & IO_PGTABLE_QUIRK_PAGE_TABLE_COHERENT)
+ reg = (ARM_LPAE_TCR_SH_OS << ARM_LPAE_TCR_SH0_SHIFT) |
+ (ARM_LPAE_TCR_RGN_WBWA << ARM_LPAE_TCR_IRGN0_SHIFT) |
+ (ARM_LPAE_TCR_RGN_WBWA << ARM_LPAE_TCR_ORGN0_SHIFT);
+ else
+ reg = (ARM_LPAE_TCR_SH_OS << ARM_LPAE_TCR_SH0_SHIFT) |
+ (ARM_LPAE_TCR_RGN_NC << ARM_LPAE_TCR_IRGN0_SHIFT) |
+ (ARM_LPAE_TCR_RGN_NC << ARM_LPAE_TCR_ORGN0_SHIFT);
switch (1 << data->pg_shift) {
case SZ_4K:
@@ -710,8 +1186,9 @@ arm_64_lpae_alloc_pgtable_s1(struct io_pgtable_cfg *cfg, void *cookie)
reg |= (64ULL - cfg->ias) << ARM_LPAE_TCR_T0SZ_SHIFT;
- /* Disable speculative walks through TTBR1 */
- reg |= ARM_LPAE_TCR_EPD1;
+ /* Bring in the TTBR1 configuration */
+ reg |= arm64_lpae_setup_ttbr1(cfg, data);
+
cfg->arm_lpae_s1_cfg.tcr = reg;
/* MAIRs */
@@ -726,16 +1203,33 @@ arm_64_lpae_alloc_pgtable_s1(struct io_pgtable_cfg *cfg, void *cookie)
cfg->arm_lpae_s1_cfg.mair[1] = 0;
/* Looking good; allocate a pgd */
- data->pgd = __arm_lpae_alloc_pages(data->pgd_size, GFP_KERNEL, cfg);
- if (!data->pgd)
+ data->pgd[0] = __arm_lpae_alloc_pages(data->pgd_size, GFP_KERNEL, cfg,
+ cookie);
+ if (!data->pgd[0])
goto out_free_data;
+
+ if (cfg->quirks & IO_PGTABLE_QUIRK_ARM_TTBR1) {
+ data->pgd[1] = __arm_lpae_alloc_pages(data->pgd_size,
+ GFP_KERNEL, cfg, cookie);
+ if (!data->pgd[1]) {
+ __arm_lpae_free_pages(data->pgd[0], data->pgd_size, cfg,
+ cookie);
+ goto out_free_data;
+ }
+ } else {
+ data->pgd[1] = NULL;
+ }
+
/* Ensure the empty pgd is visible before any actual TTBR write */
wmb();
/* TTBRs */
- cfg->arm_lpae_s1_cfg.ttbr[0] = virt_to_phys(data->pgd);
- cfg->arm_lpae_s1_cfg.ttbr[1] = 0;
+ cfg->arm_lpae_s1_cfg.ttbr[0] = virt_to_phys(data->pgd[0]);
+
+ if (data->pgd[1])
+ cfg->arm_lpae_s1_cfg.ttbr[1] = virt_to_phys(data->pgd[1]);
+
return &data->iop;
out_free_data:
@@ -815,15 +1309,16 @@ arm_64_lpae_alloc_pgtable_s2(struct io_pgtable_cfg *cfg, void *cookie)
cfg->arm_lpae_s2_cfg.vtcr = reg;
/* Allocate pgd pages */
- data->pgd = __arm_lpae_alloc_pages(data->pgd_size, GFP_KERNEL, cfg);
- if (!data->pgd)
+ data->pgd[0] = __arm_lpae_alloc_pages(data->pgd_size, GFP_KERNEL, cfg,
+ cookie);
+ if (!data->pgd[0])
goto out_free_data;
/* Ensure the empty pgd is visible before any actual TTBR write */
wmb();
/* VTTBR */
- cfg->arm_lpae_s2_cfg.vttbr = virt_to_phys(data->pgd);
+ cfg->arm_lpae_s2_cfg.vttbr = virt_to_phys(data->pgd[0]);
return &data->iop;
out_free_data:
@@ -921,16 +1416,54 @@ static void __init arm_lpae_dump_ops(struct io_pgtable_ops *ops)
cfg->pgsize_bitmap, cfg->ias);
pr_err("data: %d levels, 0x%zx pgd_size, %lu pg_shift, %lu bits_per_level, pgd @ %p\n",
data->levels, data->pgd_size, data->pg_shift,
- data->bits_per_level, data->pgd);
+ data->bits_per_level, data->pgd[0]);
}
#define __FAIL(ops, i) ({ \
WARN(1, "selftest: test failed for fmt idx %d\n", (i)); \
arm_lpae_dump_ops(ops); \
- selftest_running = false; \
+ selftest_running = false; \
+ suppress_map_failures = false; \
-EFAULT; \
})
+/*
+ * Returns true if there's any mapping in the given iova range in ops.
+ */
+static bool arm_lpae_range_has_mapping(struct io_pgtable_ops *ops,
+ unsigned long iova_start, size_t size)
+{
+ unsigned long iova = iova_start;
+
+ while (iova < (iova_start + size)) {
+ if (ops->iova_to_phys(ops, iova + 42))
+ return true;
+ iova += SZ_4K;
+ }
+ return false;
+}
+
+/*
+ * Returns true if the iova range is successfully mapped to the contiguous
+ * phys range in ops.
+ */
+static bool arm_lpae_range_has_specific_mapping(struct io_pgtable_ops *ops,
+ const unsigned long iova_start,
+ const phys_addr_t phys_start,
+ const size_t size)
+{
+ unsigned long iova = iova_start;
+ phys_addr_t phys = phys_start;
+
+ while (iova < (iova_start + size)) {
+ if (ops->iova_to_phys(ops, iova + 42) != (phys + 42))
+ return false;
+ iova += SZ_4K;
+ phys += SZ_4K;
+ }
+ return true;
+}
+
static int __init arm_lpae_run_tests(struct io_pgtable_cfg *cfg)
{
static const enum io_pgtable_fmt fmts[] = {
@@ -938,7 +1471,7 @@ static int __init arm_lpae_run_tests(struct io_pgtable_cfg *cfg)
ARM_64_LPAE_S2,
};
- int i, j;
+ int i, j, k;
unsigned long iova;
size_t size;
struct io_pgtable_ops *ops;
@@ -946,6 +1479,9 @@ static int __init arm_lpae_run_tests(struct io_pgtable_cfg *cfg)
selftest_running = true;
for (i = 0; i < ARRAY_SIZE(fmts); ++i) {
+ unsigned long test_sg_sizes[] = { SZ_4K, SZ_64K, SZ_2M,
+ SZ_1M * 12, SZ_1M * 20 };
+
cfg_cookie = cfg;
ops = alloc_io_pgtable_ops(fmts[i], cfg, cfg);
if (!ops) {
@@ -954,16 +1490,11 @@ static int __init arm_lpae_run_tests(struct io_pgtable_cfg *cfg)
}
/*
- * Initial sanity checks.
- * Empty page tables shouldn't provide any translations.
+ * Initial sanity checks. Empty page tables shouldn't
+ * provide any translations. TODO: check entire supported
+ * range for these ops rather than first 2G
*/
- if (ops->iova_to_phys(ops, 42))
- return __FAIL(ops, i);
-
- if (ops->iova_to_phys(ops, SZ_1G + 42))
- return __FAIL(ops, i);
-
- if (ops->iova_to_phys(ops, SZ_2G + 42))
+ if (arm_lpae_range_has_mapping(ops, 0, SZ_2G))
return __FAIL(ops, i);
/*
@@ -980,12 +1511,15 @@ static int __init arm_lpae_run_tests(struct io_pgtable_cfg *cfg)
IOMMU_CACHE))
return __FAIL(ops, i);
+ suppress_map_failures = true;
/* Overlapping mappings */
if (!ops->map(ops, iova, iova + size, size,
IOMMU_READ | IOMMU_NOEXEC))
return __FAIL(ops, i);
+ suppress_map_failures = false;
- if (ops->iova_to_phys(ops, iova + 42) != (iova + 42))
+ if (!arm_lpae_range_has_specific_mapping(ops, iova,
+ iova, size))
return __FAIL(ops, i);
iova += SZ_1G;
@@ -998,11 +1532,15 @@ static int __init arm_lpae_run_tests(struct io_pgtable_cfg *cfg)
if (ops->unmap(ops, SZ_1G + size, size) != size)
return __FAIL(ops, i);
+ if (arm_lpae_range_has_mapping(ops, SZ_1G + size, size))
+ return __FAIL(ops, i);
+
/* Remap of partial unmap */
if (ops->map(ops, SZ_1G + size, size, size, IOMMU_READ))
return __FAIL(ops, i);
- if (ops->iova_to_phys(ops, SZ_1G + size + 42) != (size + 42))
+ if (!arm_lpae_range_has_specific_mapping(ops, SZ_1G + size,
+ size, size))
return __FAIL(ops, i);
/* Full unmap */
@@ -1024,15 +1562,107 @@ static int __init arm_lpae_run_tests(struct io_pgtable_cfg *cfg)
if (ops->iova_to_phys(ops, iova + 42) != (iova + 42))
return __FAIL(ops, i);
+ if (ops->unmap(ops, iova, size) != size)
+ return __FAIL(ops, i);
+
iova += SZ_1G;
j++;
j = find_next_bit(&cfg->pgsize_bitmap, BITS_PER_LONG, j);
}
+ if (arm_lpae_range_has_mapping(ops, 0, SZ_2G))
+ return __FAIL(ops, i);
+
+ if ((cfg->pgsize_bitmap & SZ_2M) &&
+ (cfg->pgsize_bitmap & SZ_4K)) {
+ /* mixed block + page mappings */
+ iova = 0;
+ if (ops->map(ops, iova, iova, SZ_2M, IOMMU_READ))
+ return __FAIL(ops, i);
+
+ if (ops->map(ops, iova + SZ_2M, iova + SZ_2M, SZ_4K,
+ IOMMU_READ))
+ return __FAIL(ops, i);
+
+ if (ops->iova_to_phys(ops, iova + 42) != (iova + 42))
+ return __FAIL(ops, i);
+
+ if (ops->iova_to_phys(ops, iova + SZ_2M + 42) !=
+ (iova + SZ_2M + 42))
+ return __FAIL(ops, i);
+
+ /* unmap both mappings at once */
+ if (ops->unmap(ops, iova, SZ_2M + SZ_4K) !=
+ (SZ_2M + SZ_4K))
+ return __FAIL(ops, i);
+
+ if (arm_lpae_range_has_mapping(ops, 0, SZ_2G))
+ return __FAIL(ops, i);
+ }
+
+ /* map_sg */
+ for (j = 0; j < ARRAY_SIZE(test_sg_sizes); ++j) {
+ size_t mapped;
+ size_t unused;
+ struct page *page;
+ phys_addr_t page_phys;
+ struct sg_table table;
+ struct scatterlist *sg;
+ unsigned long total_size = test_sg_sizes[j];
+ int chunk_size = 1UL << find_first_bit(
+ &cfg->pgsize_bitmap, BITS_PER_LONG);
+ int nents = total_size / chunk_size;
+
+ if (total_size < chunk_size)
+ continue;
+
+ page = alloc_pages(GFP_KERNEL, get_order(chunk_size));
+ page_phys = page_to_phys(page);
+
+ iova = 0;
+ BUG_ON(sg_alloc_table(&table, nents, GFP_KERNEL));
+ BUG_ON(!page);
+ for_each_sg(table.sgl, sg, table.nents, k)
+ sg_set_page(sg, page, chunk_size, 0);
+
+ mapped = ops->map_sg(ops, iova, table.sgl, table.nents,
+ IOMMU_READ | IOMMU_WRITE, &unused);
+
+ if (mapped != total_size)
+ return __FAIL(ops, i);
+
+ if (!arm_lpae_range_has_mapping(ops, iova, total_size))
+ return __FAIL(ops, i);
+
+ if (arm_lpae_range_has_mapping(ops, iova + total_size,
+ SZ_2G - (iova + total_size)))
+ return __FAIL(ops, i);
+
+ for_each_sg(table.sgl, sg, table.nents, k) {
+ dma_addr_t newphys =
+ ops->iova_to_phys(ops, iova + 42);
+ if (newphys != (page_phys + 42))
+ return __FAIL(ops, i);
+ iova += chunk_size;
+ }
+
+ if (ops->unmap(ops, 0, total_size) != total_size)
+ return __FAIL(ops, i);
+
+ if (arm_lpae_range_has_mapping(ops, 0, SZ_2G))
+ return __FAIL(ops, i);
+
+ sg_free_table(&table);
+ __free_pages(page, get_order(chunk_size));
+ }
+
+ if (arm_lpae_range_has_mapping(ops, 0, SZ_2G))
+ return __FAIL(ops, i);
+
free_io_pgtable_ops(ops);
}
- selftest_running = false;
+ suppress_map_failures = false;
return 0;
}
diff --git a/drivers/iommu/io-pgtable-fast.c b/drivers/iommu/io-pgtable-fast.c
new file mode 100644
index 000000000000..5378e95c4627
--- /dev/null
+++ b/drivers/iommu/io-pgtable-fast.c
@@ -0,0 +1,751 @@
+/* Copyright (c) 2016-2017, The Linux Foundation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 and
+ * only version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ */
+
+#define pr_fmt(fmt) "io-pgtable-fast: " fmt
+
+#include <linux/iommu.h>
+#include <linux/kernel.h>
+#include <linux/scatterlist.h>
+#include <linux/sizes.h>
+#include <linux/slab.h>
+#include <linux/types.h>
+#include <linux/io-pgtable-fast.h>
+#include <linux/mm.h>
+#include <asm/cacheflush.h>
+#include <linux/vmalloc.h>
+
+#include "io-pgtable.h"
+
+#define AV8L_FAST_MAX_ADDR_BITS 48
+
+/* Struct accessors */
+#define iof_pgtable_to_data(x) \
+ container_of((x), struct av8l_fast_io_pgtable, iop)
+
+#define iof_pgtable_ops_to_pgtable(x) \
+ container_of((x), struct io_pgtable, ops)
+
+#define iof_pgtable_ops_to_data(x) \
+ iof_pgtable_to_data(iof_pgtable_ops_to_pgtable(x))
+
+struct av8l_fast_io_pgtable {
+ struct io_pgtable iop;
+ av8l_fast_iopte *pgd;
+ av8l_fast_iopte *puds[4];
+ av8l_fast_iopte *pmds;
+ struct page **pages; /* page table memory */
+ int nr_pages;
+ dma_addr_t base;
+ dma_addr_t end;
+};
+
+/* Page table bits */
+#define AV8L_FAST_PTE_TYPE_SHIFT 0
+#define AV8L_FAST_PTE_TYPE_MASK 0x3
+
+#define AV8L_FAST_PTE_TYPE_BLOCK 1
+#define AV8L_FAST_PTE_TYPE_TABLE 3
+#define AV8L_FAST_PTE_TYPE_PAGE 3
+
+#define AV8L_FAST_PTE_NSTABLE (((av8l_fast_iopte)1) << 63)
+#define AV8L_FAST_PTE_XN (((av8l_fast_iopte)3) << 53)
+#define AV8L_FAST_PTE_AF (((av8l_fast_iopte)1) << 10)
+#define AV8L_FAST_PTE_SH_NS (((av8l_fast_iopte)0) << 8)
+#define AV8L_FAST_PTE_SH_OS (((av8l_fast_iopte)2) << 8)
+#define AV8L_FAST_PTE_SH_IS (((av8l_fast_iopte)3) << 8)
+#define AV8L_FAST_PTE_NS (((av8l_fast_iopte)1) << 5)
+#define AV8L_FAST_PTE_VALID (((av8l_fast_iopte)1) << 0)
+
+#define AV8L_FAST_PTE_ATTR_LO_MASK (((av8l_fast_iopte)0x3ff) << 2)
+/* Ignore the contiguous bit for block splitting */
+#define AV8L_FAST_PTE_ATTR_HI_MASK (((av8l_fast_iopte)6) << 52)
+#define AV8L_FAST_PTE_ATTR_MASK (AV8L_FAST_PTE_ATTR_LO_MASK | \
+ AV8L_FAST_PTE_ATTR_HI_MASK)
+#define AV8L_FAST_PTE_ADDR_MASK ((av8l_fast_iopte)0xfffffffff000)
+
+
+/* Stage-1 PTE */
+#define AV8L_FAST_PTE_AP_PRIV_RW (((av8l_fast_iopte)0) << 6)
+#define AV8L_FAST_PTE_AP_RW (((av8l_fast_iopte)1) << 6)
+#define AV8L_FAST_PTE_AP_PRIV_RO (((av8l_fast_iopte)2) << 6)
+#define AV8L_FAST_PTE_AP_RO (((av8l_fast_iopte)3) << 6)
+#define AV8L_FAST_PTE_ATTRINDX_SHIFT 2
+#define AV8L_FAST_PTE_nG (((av8l_fast_iopte)1) << 11)
+
+/* Stage-2 PTE */
+#define AV8L_FAST_PTE_HAP_FAULT (((av8l_fast_iopte)0) << 6)
+#define AV8L_FAST_PTE_HAP_READ (((av8l_fast_iopte)1) << 6)
+#define AV8L_FAST_PTE_HAP_WRITE (((av8l_fast_iopte)2) << 6)
+#define AV8L_FAST_PTE_MEMATTR_OIWB (((av8l_fast_iopte)0xf) << 2)
+#define AV8L_FAST_PTE_MEMATTR_NC (((av8l_fast_iopte)0x5) << 2)
+#define AV8L_FAST_PTE_MEMATTR_DEV (((av8l_fast_iopte)0x1) << 2)
+
+/* Register bits */
+#define ARM_32_LPAE_TCR_EAE (1 << 31)
+#define ARM_64_LPAE_S2_TCR_RES1 (1 << 31)
+
+#define AV8L_FAST_TCR_TG0_4K (0 << 14)
+#define AV8L_FAST_TCR_TG0_64K (1 << 14)
+#define AV8L_FAST_TCR_TG0_16K (2 << 14)
+
+#define AV8L_FAST_TCR_SH0_SHIFT 12
+#define AV8L_FAST_TCR_SH0_MASK 0x3
+#define AV8L_FAST_TCR_SH_NS 0
+#define AV8L_FAST_TCR_SH_OS 2
+#define AV8L_FAST_TCR_SH_IS 3
+
+#define AV8L_FAST_TCR_ORGN0_SHIFT 10
+#define AV8L_FAST_TCR_IRGN0_SHIFT 8
+#define AV8L_FAST_TCR_RGN_MASK 0x3
+#define AV8L_FAST_TCR_RGN_NC 0
+#define AV8L_FAST_TCR_RGN_WBWA 1
+#define AV8L_FAST_TCR_RGN_WT 2
+#define AV8L_FAST_TCR_RGN_WB 3
+
+#define AV8L_FAST_TCR_SL0_SHIFT 6
+#define AV8L_FAST_TCR_SL0_MASK 0x3
+
+#define AV8L_FAST_TCR_T0SZ_SHIFT 0
+#define AV8L_FAST_TCR_SZ_MASK 0xf
+
+#define AV8L_FAST_TCR_PS_SHIFT 16
+#define AV8L_FAST_TCR_PS_MASK 0x7
+
+#define AV8L_FAST_TCR_IPS_SHIFT 32
+#define AV8L_FAST_TCR_IPS_MASK 0x7
+
+#define AV8L_FAST_TCR_PS_32_BIT 0x0ULL
+#define AV8L_FAST_TCR_PS_36_BIT 0x1ULL
+#define AV8L_FAST_TCR_PS_40_BIT 0x2ULL
+#define AV8L_FAST_TCR_PS_42_BIT 0x3ULL
+#define AV8L_FAST_TCR_PS_44_BIT 0x4ULL
+#define AV8L_FAST_TCR_PS_48_BIT 0x5ULL
+
+#define AV8L_FAST_TCR_EPD1_SHIFT 23
+#define AV8L_FAST_TCR_EPD1_FAULT 1
+
+#define AV8L_FAST_TCR_SEP_SHIFT (15 + 32)
+#define AV8L_FAST_TCR_SEP_UPSTREAM 7ULL
+
+#define AV8L_FAST_MAIR_ATTR_SHIFT(n) ((n) << 3)
+#define AV8L_FAST_MAIR_ATTR_MASK 0xff
+#define AV8L_FAST_MAIR_ATTR_DEVICE 0x04
+#define AV8L_FAST_MAIR_ATTR_NC 0x44
+#define AV8L_FAST_MAIR_ATTR_WBRWA 0xff
+#define AV8L_FAST_MAIR_ATTR_IDX_NC 0
+#define AV8L_FAST_MAIR_ATTR_IDX_CACHE 1
+#define AV8L_FAST_MAIR_ATTR_IDX_DEV 2
+
+#define AV8L_FAST_PAGE_SHIFT 12
+
+
+#ifdef CONFIG_IOMMU_IO_PGTABLE_FAST_PROVE_TLB
+
+#include <asm/cacheflush.h>
+#include <linux/notifier.h>
+
+static ATOMIC_NOTIFIER_HEAD(av8l_notifier_list);
+
+void av8l_register_notify(struct notifier_block *nb)
+{
+ atomic_notifier_chain_register(&av8l_notifier_list, nb);
+}
+EXPORT_SYMBOL(av8l_register_notify);
+
+static void __av8l_check_for_stale_tlb(av8l_fast_iopte *ptep)
+{
+ if (unlikely(*ptep)) {
+ atomic_notifier_call_chain(
+ &av8l_notifier_list, MAPPED_OVER_STALE_TLB,
+ (void *) ptep);
+ pr_err("Tried to map over a non-vacant pte: 0x%llx @ %p\n",
+ *ptep, ptep);
+ pr_err("Nearby memory:\n");
+ print_hex_dump(KERN_ERR, "pgtbl: ", DUMP_PREFIX_ADDRESS,
+ 32, 8, ptep - 16, 32 * sizeof(*ptep), false);
+ }
+}
+
+void av8l_fast_clear_stale_ptes(av8l_fast_iopte *pmds, u64 base,
+ u64 start, u64 end, bool skip_sync)
+{
+ int i;
+ av8l_fast_iopte *pmdp = iopte_pmd_offset(pmds, base, start);
+
+ for (i = start >> AV8L_FAST_PAGE_SHIFT;
+ i <= (end >> AV8L_FAST_PAGE_SHIFT); ++i) {
+ if (!(*pmdp & AV8L_FAST_PTE_VALID)) {
+ *pmdp = 0;
+ if (!skip_sync)
+ dmac_clean_range(pmdp, pmdp + 1);
+ }
+ pmdp++;
+ }
+}
+#else
+static void __av8l_check_for_stale_tlb(av8l_fast_iopte *ptep)
+{
+}
+#endif
+
+/* caller must take care of cache maintenance on *ptep */
+int av8l_fast_map_public(av8l_fast_iopte *ptep, phys_addr_t paddr, size_t size,
+ int prot)
+{
+ int i, nptes = size >> AV8L_FAST_PAGE_SHIFT;
+ av8l_fast_iopte pte = AV8L_FAST_PTE_XN
+ | AV8L_FAST_PTE_TYPE_PAGE
+ | AV8L_FAST_PTE_AF
+ | AV8L_FAST_PTE_nG
+ | AV8L_FAST_PTE_SH_OS;
+
+ if (prot & IOMMU_DEVICE)
+ pte |= (AV8L_FAST_MAIR_ATTR_IDX_DEV
+ << AV8L_FAST_PTE_ATTRINDX_SHIFT);
+ else if (prot & IOMMU_CACHE)
+ pte |= (AV8L_FAST_MAIR_ATTR_IDX_CACHE
+ << AV8L_FAST_PTE_ATTRINDX_SHIFT);
+
+ if (!(prot & IOMMU_WRITE))
+ pte |= AV8L_FAST_PTE_AP_RO;
+ else
+ pte |= AV8L_FAST_PTE_AP_RW;
+
+ paddr &= AV8L_FAST_PTE_ADDR_MASK;
+ for (i = 0; i < nptes; i++, paddr += SZ_4K) {
+ __av8l_check_for_stale_tlb(ptep + i);
+ *(ptep + i) = pte | paddr;
+ }
+
+ return 0;
+}
+
+static int av8l_fast_map(struct io_pgtable_ops *ops, unsigned long iova,
+ phys_addr_t paddr, size_t size, int prot)
+{
+ struct av8l_fast_io_pgtable *data = iof_pgtable_ops_to_data(ops);
+ av8l_fast_iopte *ptep = iopte_pmd_offset(data->pmds, data->base, iova);
+ unsigned long nptes = size >> AV8L_FAST_PAGE_SHIFT;
+
+ av8l_fast_map_public(ptep, paddr, size, prot);
+ dmac_clean_range(ptep, ptep + nptes);
+
+ return 0;
+}
+
+static void __av8l_fast_unmap(av8l_fast_iopte *ptep, size_t size,
+ bool need_stale_tlb_tracking)
+{
+ unsigned long nptes = size >> AV8L_FAST_PAGE_SHIFT;
+ int val = need_stale_tlb_tracking
+ ? AV8L_FAST_PTE_UNMAPPED_NEED_TLBI
+ : 0;
+
+ memset(ptep, val, sizeof(*ptep) * nptes);
+}
+
+/* caller must take care of cache maintenance on *ptep */
+void av8l_fast_unmap_public(av8l_fast_iopte *ptep, size_t size)
+{
+ __av8l_fast_unmap(ptep, size, true);
+}
+
+static size_t av8l_fast_unmap(struct io_pgtable_ops *ops, unsigned long iova,
+ size_t size)
+{
+ struct av8l_fast_io_pgtable *data = iof_pgtable_ops_to_data(ops);
+ struct io_pgtable *iop = &data->iop;
+ av8l_fast_iopte *ptep = iopte_pmd_offset(data->pmds, data->base, iova);
+ unsigned long nptes = size >> AV8L_FAST_PAGE_SHIFT;
+
+ __av8l_fast_unmap(ptep, size, false);
+ dmac_clean_range(ptep, ptep + nptes);
+ iop->cfg.tlb->tlb_flush_all(iop->cookie);
+
+ return size;
+}
+
+#if defined(CONFIG_ARM64)
+#define FAST_PGDNDX(va) (((va) & 0x7fc0000000) >> 27)
+#elif defined(CONFIG_ARM)
+#define FAST_PGDNDX(va) (((va) & 0xc0000000) >> 27)
+#endif
+
+static phys_addr_t av8l_fast_iova_to_phys(struct io_pgtable_ops *ops,
+ unsigned long iova)
+{
+ struct av8l_fast_io_pgtable *data = iof_pgtable_ops_to_data(ops);
+ av8l_fast_iopte pte, *pgdp, *pudp, *pmdp;
+ unsigned long pgd;
+ phys_addr_t phys;
+ const unsigned long pts = AV8L_FAST_PTE_TYPE_SHIFT;
+ const unsigned long ptm = AV8L_FAST_PTE_TYPE_MASK;
+ const unsigned long ptt = AV8L_FAST_PTE_TYPE_TABLE;
+ const unsigned long ptp = AV8L_FAST_PTE_TYPE_PAGE;
+ const av8l_fast_iopte am = AV8L_FAST_PTE_ADDR_MASK;
+
+ /* TODO: clean up some of these magic numbers... */
+
+ pgd = (unsigned long)data->pgd | FAST_PGDNDX(iova);
+ pgdp = (av8l_fast_iopte *)pgd;
+
+ pte = *pgdp;
+ if (((pte >> pts) & ptm) != ptt)
+ return 0;
+ pudp = phys_to_virt((pte & am) | ((iova & 0x3fe00000) >> 18));
+
+ pte = *pudp;
+ if (((pte >> pts) & ptm) != ptt)
+ return 0;
+ pmdp = phys_to_virt((pte & am) | ((iova & 0x1ff000) >> 9));
+
+ pte = *pmdp;
+ if (((pte >> pts) & ptm) != ptp)
+ return 0;
+ phys = pte & am;
+
+ return phys | (iova & 0xfff);
+}
+
+static int av8l_fast_map_sg(struct io_pgtable_ops *ops, unsigned long iova,
+ struct scatterlist *sg, unsigned int nents,
+ int prot, size_t *size)
+{
+ return -ENODEV;
+}
+
+static struct av8l_fast_io_pgtable *
+av8l_fast_alloc_pgtable_data(struct io_pgtable_cfg *cfg)
+{
+ struct av8l_fast_io_pgtable *data;
+
+ data = kmalloc(sizeof(*data), GFP_KERNEL);
+ if (!data)
+ return NULL;
+
+ data->iop.ops = (struct io_pgtable_ops) {
+ .map = av8l_fast_map,
+ .map_sg = av8l_fast_map_sg,
+ .unmap = av8l_fast_unmap,
+ .iova_to_phys = av8l_fast_iova_to_phys,
+ };
+
+ return data;
+}
+
+/*
+ * We need max 1 page for the pgd, 4 pages for puds (1GB VA per pud page) and
+ * 2048 pages for pmds (each pud page contains 512 table entries, each
+ * pointing to a pmd).
+ */
+#define NUM_PGD_PAGES 1
+#define NUM_PUD_PAGES 4
+#define NUM_PMD_PAGES 2048
+#define NUM_PGTBL_PAGES (NUM_PGD_PAGES + NUM_PUD_PAGES + NUM_PMD_PAGES)
+
+/* undefine arch specific definitions which depends on page table format */
+#undef pud_index
+#undef pud_mask
+#undef pud_next
+#undef pmd_index
+#undef pmd_mask
+#undef pmd_next
+
+#define pud_index(addr) (((addr) >> 30) & 0x3)
+#define pud_mask(addr) ((addr) & ~((1UL << 30) - 1))
+#define pud_next(addr, end) \
+({ unsigned long __boundary = pud_mask(addr + (1UL << 30));\
+ (__boundary - 1 < (end) - 1) ? __boundary : (end); \
+})
+
+#define pmd_index(addr) (((addr) >> 21) & 0x1ff)
+#define pmd_mask(addr) ((addr) & ~((1UL << 21) - 1))
+#define pmd_next(addr, end) \
+({ unsigned long __boundary = pmd_mask(addr + (1UL << 21));\
+ (__boundary - 1 < (end) - 1) ? __boundary : (end); \
+})
+
+static int
+av8l_fast_prepopulate_pgtables(struct av8l_fast_io_pgtable *data,
+ struct io_pgtable_cfg *cfg, void *cookie)
+{
+ int i, j, pg = 0;
+ struct page **pages, *page;
+ dma_addr_t base = cfg->iova_base;
+ dma_addr_t end = cfg->iova_end;
+ dma_addr_t pud, pmd;
+ int pmd_pg_index;
+
+ pages = kmalloc(sizeof(*pages) * NUM_PGTBL_PAGES, __GFP_NOWARN |
+ __GFP_NORETRY);
+
+ if (!pages)
+ pages = vmalloc(sizeof(*pages) * NUM_PGTBL_PAGES);
+
+ if (!pages)
+ return -ENOMEM;
+
+ page = alloc_page(GFP_KERNEL | __GFP_ZERO);
+ if (!page)
+ goto err_free_pages_arr;
+ pages[pg++] = page;
+ data->pgd = page_address(page);
+
+ /*
+ * We need max 2048 entries at level 2 to map 4GB of VA space. A page
+ * can hold 512 entries, so we need max 4 pages.
+ */
+ for (i = pud_index(base), pud = base; pud < end;
+ ++i, pud = pud_next(pud, end)) {
+ av8l_fast_iopte pte, *ptep;
+
+ page = alloc_page(GFP_KERNEL | __GFP_ZERO);
+ if (!page)
+ goto err_free_pages;
+ pages[pg++] = page;
+ data->puds[i] = page_address(page);
+ pte = page_to_phys(page) | AV8L_FAST_PTE_TYPE_TABLE;
+ ptep = ((av8l_fast_iopte *)data->pgd) + i;
+ *ptep = pte;
+ }
+ dmac_clean_range(data->pgd, data->pgd + 4);
+
+ /*
+ * We have max 4 puds, each of which can point to 512 pmds, so we'll
+ * have max 2048 pmds, each of which can hold 512 ptes, for a grand
+ * total of 2048*512=1048576 PTEs.
+ */
+ pmd_pg_index = pg;
+ for (i = pud_index(base), pud = base; pud < end;
+ ++i, pud = pud_next(pud, end)) {
+ for (j = pmd_index(pud), pmd = pud; pmd < pud_next(pud, end);
+ ++j, pmd = pmd_next(pmd, end)) {
+ av8l_fast_iopte pte, *pudp;
+ void *addr;
+
+ page = alloc_page(GFP_KERNEL | __GFP_ZERO);
+ if (!page)
+ goto err_free_pages;
+ pages[pg++] = page;
+
+ addr = page_address(page);
+ dmac_clean_range(addr, addr + SZ_4K);
+
+ pte = page_to_phys(page) | AV8L_FAST_PTE_TYPE_TABLE;
+ pudp = data->puds[i] + j;
+ *pudp = pte;
+ }
+ dmac_clean_range(data->puds[i], data->puds[i] + 512);
+ }
+
+ /*
+ * We map the pmds into a virtually contiguous space so that we
+ * don't have to traverse the first two levels of the page tables
+ * to find the appropriate pud. Instead, it will be a simple
+ * offset from the virtual base of the pmds.
+ */
+ data->pmds = vmap(&pages[pmd_pg_index], pg - pmd_pg_index,
+ VM_IOREMAP, PAGE_KERNEL);
+ if (!data->pmds)
+ goto err_free_pages;
+
+ data->pages = pages;
+ data->nr_pages = pg;
+ data->base = base;
+ data->end = end;
+ return 0;
+
+err_free_pages:
+ for (i = 0; i < pg; ++i)
+ __free_page(pages[i]);
+err_free_pages_arr:
+ kvfree(pages);
+ return -ENOMEM;
+}
+
+static struct io_pgtable *
+av8l_fast_alloc_pgtable(struct io_pgtable_cfg *cfg, void *cookie)
+{
+ u64 reg;
+ struct av8l_fast_io_pgtable *data =
+ av8l_fast_alloc_pgtable_data(cfg);
+
+ if (!data)
+ return NULL;
+
+ /* restrict according to the fast map requirements */
+ cfg->ias = 32;
+ cfg->pgsize_bitmap = SZ_4K;
+
+ /* TCR */
+ if (cfg->quirks & IO_PGTABLE_QUIRK_PAGE_TABLE_COHERENT)
+ reg = (AV8L_FAST_TCR_SH_OS << AV8L_FAST_TCR_SH0_SHIFT) |
+ (AV8L_FAST_TCR_RGN_WBWA << AV8L_FAST_TCR_IRGN0_SHIFT) |
+ (AV8L_FAST_TCR_RGN_WBWA << AV8L_FAST_TCR_ORGN0_SHIFT);
+ else
+ reg = (AV8L_FAST_TCR_SH_OS << AV8L_FAST_TCR_SH0_SHIFT) |
+ (AV8L_FAST_TCR_RGN_NC << AV8L_FAST_TCR_IRGN0_SHIFT) |
+ (AV8L_FAST_TCR_RGN_NC << AV8L_FAST_TCR_ORGN0_SHIFT);
+
+ reg |= AV8L_FAST_TCR_TG0_4K;
+
+ switch (cfg->oas) {
+ case 32:
+ reg |= (AV8L_FAST_TCR_PS_32_BIT << AV8L_FAST_TCR_IPS_SHIFT);
+ break;
+ case 36:
+ reg |= (AV8L_FAST_TCR_PS_36_BIT << AV8L_FAST_TCR_IPS_SHIFT);
+ break;
+ case 40:
+ reg |= (AV8L_FAST_TCR_PS_40_BIT << AV8L_FAST_TCR_IPS_SHIFT);
+ break;
+ case 42:
+ reg |= (AV8L_FAST_TCR_PS_42_BIT << AV8L_FAST_TCR_IPS_SHIFT);
+ break;
+ case 44:
+ reg |= (AV8L_FAST_TCR_PS_44_BIT << AV8L_FAST_TCR_IPS_SHIFT);
+ break;
+ case 48:
+ reg |= (AV8L_FAST_TCR_PS_48_BIT << AV8L_FAST_TCR_IPS_SHIFT);
+ break;
+ default:
+ goto out_free_data;
+ }
+
+ reg |= (64ULL - cfg->ias) << AV8L_FAST_TCR_T0SZ_SHIFT;
+ reg |= AV8L_FAST_TCR_EPD1_FAULT << AV8L_FAST_TCR_EPD1_SHIFT;
+#if defined(CONFIG_ARM)
+ reg |= ARM_32_LPAE_TCR_EAE;
+#endif
+ reg |= AV8L_FAST_TCR_SEP_UPSTREAM << AV8L_FAST_TCR_SEP_SHIFT;
+ cfg->av8l_fast_cfg.tcr = reg;
+
+ /* MAIRs */
+ reg = (AV8L_FAST_MAIR_ATTR_NC
+ << AV8L_FAST_MAIR_ATTR_SHIFT(AV8L_FAST_MAIR_ATTR_IDX_NC)) |
+ (AV8L_FAST_MAIR_ATTR_WBRWA
+ << AV8L_FAST_MAIR_ATTR_SHIFT(AV8L_FAST_MAIR_ATTR_IDX_CACHE)) |
+ (AV8L_FAST_MAIR_ATTR_DEVICE
+ << AV8L_FAST_MAIR_ATTR_SHIFT(AV8L_FAST_MAIR_ATTR_IDX_DEV));
+
+ cfg->av8l_fast_cfg.mair[0] = reg;
+ cfg->av8l_fast_cfg.mair[1] = 0;
+
+ /* Allocate all page table memory! */
+ if (av8l_fast_prepopulate_pgtables(data, cfg, cookie))
+ goto out_free_data;
+
+ cfg->av8l_fast_cfg.pmds = data->pmds;
+
+ /* TTBRs */
+ cfg->av8l_fast_cfg.ttbr[0] = virt_to_phys(data->pgd);
+ cfg->av8l_fast_cfg.ttbr[1] = 0;
+ return &data->iop;
+
+out_free_data:
+ kfree(data);
+ return NULL;
+}
+
+static void av8l_fast_free_pgtable(struct io_pgtable *iop)
+{
+ int i;
+ struct av8l_fast_io_pgtable *data = iof_pgtable_to_data(iop);
+
+ vunmap(data->pmds);
+ for (i = 0; i < data->nr_pages; ++i)
+ __free_page(data->pages[i]);
+ kvfree(data->pages);
+ kfree(data);
+}
+
+struct io_pgtable_init_fns io_pgtable_av8l_fast_init_fns = {
+ .alloc = av8l_fast_alloc_pgtable,
+ .free = av8l_fast_free_pgtable,
+};
+
+
+#ifdef CONFIG_IOMMU_IO_PGTABLE_FAST_SELFTEST
+
+#include <linux/dma-contiguous.h>
+
+static struct io_pgtable_cfg *cfg_cookie;
+
+static void dummy_tlb_flush_all(void *cookie)
+{
+ WARN_ON(cookie != cfg_cookie);
+}
+
+static void dummy_tlb_add_flush(unsigned long iova, size_t size, bool leaf,
+ void *cookie)
+{
+ WARN_ON(cookie != cfg_cookie);
+ WARN_ON(!(size & cfg_cookie->pgsize_bitmap));
+}
+
+static void dummy_tlb_sync(void *cookie)
+{
+ WARN_ON(cookie != cfg_cookie);
+}
+
+static struct iommu_gather_ops dummy_tlb_ops __initdata = {
+ .tlb_flush_all = dummy_tlb_flush_all,
+ .tlb_add_flush = dummy_tlb_add_flush,
+ .tlb_sync = dummy_tlb_sync,
+};
+
+/*
+ * Returns true if the iova range is successfully mapped to the contiguous
+ * phys range in ops.
+ */
+static bool av8l_fast_range_has_specific_mapping(struct io_pgtable_ops *ops,
+ const unsigned long iova_start,
+ const phys_addr_t phys_start,
+ const size_t size)
+{
+ u64 iova = iova_start;
+ phys_addr_t phys = phys_start;
+
+ while (iova < (iova_start + size)) {
+ /* + 42 just to make sure offsetting is working */
+ if (ops->iova_to_phys(ops, iova + 42) != (phys + 42))
+ return false;
+ iova += SZ_4K;
+ phys += SZ_4K;
+ }
+ return true;
+}
+
+static int __init av8l_fast_positive_testing(void)
+{
+ int failed = 0;
+ u64 iova;
+ struct io_pgtable_ops *ops;
+ struct io_pgtable_cfg cfg;
+ struct av8l_fast_io_pgtable *data;
+ av8l_fast_iopte *pmds;
+ u64 max = SZ_1G * 4ULL - 1;
+ u64 base = 0;
+
+ cfg = (struct io_pgtable_cfg) {
+ .quirks = 0,
+ .tlb = &dummy_tlb_ops,
+ .ias = 32,
+ .oas = 32,
+ .pgsize_bitmap = SZ_4K,
+ .iova_base = base,
+ .iova_end = max,
+ };
+
+ cfg_cookie = &cfg;
+ ops = alloc_io_pgtable_ops(ARM_V8L_FAST, &cfg, &cfg);
+
+ if (WARN_ON(!ops))
+ return 1;
+
+ data = iof_pgtable_ops_to_data(ops);
+ pmds = data->pmds;
+
+ /* map the entire 4GB VA space with 4K map calls */
+ for (iova = base; iova < max; iova += SZ_4K) {
+ if (WARN_ON(ops->map(ops, iova, iova, SZ_4K, IOMMU_READ))) {
+ failed++;
+ continue;
+ }
+ }
+ if (WARN_ON(!av8l_fast_range_has_specific_mapping(ops, base,
+ base, max - base)))
+ failed++;
+
+ /* unmap it all */
+ for (iova = base; iova < max; iova += SZ_4K) {
+ if (WARN_ON(ops->unmap(ops, iova, SZ_4K) != SZ_4K))
+ failed++;
+ }
+
+ /* sweep up TLB proving PTEs */
+ av8l_fast_clear_stale_ptes(pmds, base, base, max, false);
+
+ /* map the entire 4GB VA space with 8K map calls */
+ for (iova = base; iova < max; iova += SZ_8K) {
+ if (WARN_ON(ops->map(ops, iova, iova, SZ_8K, IOMMU_READ))) {
+ failed++;
+ continue;
+ }
+ }
+
+ if (WARN_ON(!av8l_fast_range_has_specific_mapping(ops, base,
+ base, max - base)))
+ failed++;
+
+ /* unmap it all with 8K unmap calls */
+ for (iova = base; iova < max; iova += SZ_8K) {
+ if (WARN_ON(ops->unmap(ops, iova, SZ_8K) != SZ_8K))
+ failed++;
+ }
+
+ /* sweep up TLB proving PTEs */
+ av8l_fast_clear_stale_ptes(pmds, base, base, max, false);
+
+ /* map the entire 4GB VA space with 16K map calls */
+ for (iova = base; iova < max; iova += SZ_16K) {
+ if (WARN_ON(ops->map(ops, iova, iova, SZ_16K, IOMMU_READ))) {
+ failed++;
+ continue;
+ }
+ }
+
+ if (WARN_ON(!av8l_fast_range_has_specific_mapping(ops, base,
+ base, max - base)))
+ failed++;
+
+ /* unmap it all */
+ for (iova = base; iova < max; iova += SZ_16K) {
+ if (WARN_ON(ops->unmap(ops, iova, SZ_16K) != SZ_16K))
+ failed++;
+ }
+
+ /* sweep up TLB proving PTEs */
+ av8l_fast_clear_stale_ptes(pmds, base, base, max, false);
+
+ /* map the entire 4GB VA space with 64K map calls */
+ for (iova = base; iova < max; iova += SZ_64K) {
+ if (WARN_ON(ops->map(ops, iova, iova, SZ_64K, IOMMU_READ))) {
+ failed++;
+ continue;
+ }
+ }
+
+ if (WARN_ON(!av8l_fast_range_has_specific_mapping(ops, base,
+ base, max - base)))
+ failed++;
+
+ /* unmap it all at once */
+ if (WARN_ON(ops->unmap(ops, base, max - base) != (max - base)))
+ failed++;
+
+ free_io_pgtable_ops(ops);
+ return failed;
+}
+
+static int __init av8l_fast_do_selftests(void)
+{
+ int failed = 0;
+
+ failed += av8l_fast_positive_testing();
+
+ pr_err("selftest: completed with %d failures\n", failed);
+
+ return 0;
+}
+subsys_initcall(av8l_fast_do_selftests);
+#endif
diff --git a/drivers/iommu/io-pgtable-msm-secure.c b/drivers/iommu/io-pgtable-msm-secure.c
new file mode 100644
index 000000000000..336a43a1ed1f
--- /dev/null
+++ b/drivers/iommu/io-pgtable-msm-secure.c
@@ -0,0 +1,243 @@
+/* Copyright (c) 2016, The Linux Foundation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 and
+ * only version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ */
+
+#define pr_fmt(fmt) "io-pgtable-msm-secure: " fmt
+
+#include <linux/iommu.h>
+#include <linux/kernel.h>
+#include <linux/scatterlist.h>
+#include <linux/sizes.h>
+#include <linux/slab.h>
+#include <linux/types.h>
+#include <soc/qcom/scm.h>
+#include <asm/cacheflush.h>
+
+#include "io-pgtable.h"
+
+#define IOMMU_SECURE_MAP2_FLAT 0x12
+#define IOMMU_SECURE_UNMAP2_FLAT 0x13
+#define IOMMU_TLBINVAL_FLAG 0x00000001
+
+#define io_pgtable_to_data(x) \
+ container_of((x), struct msm_secure_io_pgtable, iop)
+
+#define io_pgtable_ops_to_pgtable(x) \
+ container_of((x), struct io_pgtable, ops)
+
+#define io_pgtable_ops_to_data(x) \
+ io_pgtable_to_data(io_pgtable_ops_to_pgtable(x))
+
+struct msm_secure_io_pgtable {
+ struct io_pgtable iop;
+};
+
+static int msm_secure_map(struct io_pgtable_ops *ops, unsigned long iova,
+ phys_addr_t paddr, size_t size, int iommu_prot)
+{
+ return -EINVAL;
+}
+
+static dma_addr_t msm_secure_get_phys_addr(struct scatterlist *sg)
+{
+ /*
+ * Try sg_dma_address first so that we can
+ * map carveout regions that do not have a
+ * struct page associated with them.
+ */
+ dma_addr_t pa = sg_dma_address(sg);
+
+ if (pa == 0)
+ pa = sg_phys(sg);
+ return pa;
+}
+
+static int msm_secure_map_sg(struct io_pgtable_ops *ops, unsigned long iova,
+ struct scatterlist *sg, unsigned int nents,
+ int iommu_prot, size_t *size)
+{
+ struct msm_secure_io_pgtable *data = io_pgtable_ops_to_data(ops);
+ struct io_pgtable_cfg *cfg = &data->iop.cfg;
+ int ret = -EINVAL;
+ struct scatterlist *tmp, *sgiter;
+ dma_addr_t *pa_list = 0;
+ unsigned int cnt, offset = 0, chunk_offset = 0;
+ dma_addr_t pa;
+ void *flush_va, *flush_va_end;
+ unsigned long len = 0;
+ struct scm_desc desc = {0};
+ int i;
+ u32 resp;
+
+ for_each_sg(sg, tmp, nents, i)
+ len += tmp->length;
+
+ if (!IS_ALIGNED(iova, SZ_1M) || !IS_ALIGNED(len, SZ_1M))
+ return -EINVAL;
+
+ if (sg->length == len) {
+ cnt = 1;
+ pa = msm_secure_get_phys_addr(sg);
+ if (!IS_ALIGNED(pa, SZ_1M))
+ return -EINVAL;
+
+ desc.args[0] = virt_to_phys(&pa);
+ desc.args[1] = cnt;
+ desc.args[2] = len;
+ flush_va = &pa;
+ } else {
+ sgiter = sg;
+ if (!IS_ALIGNED(sgiter->length, SZ_1M))
+ return -EINVAL;
+ cnt = sg->length / SZ_1M;
+ while ((sgiter = sg_next(sgiter))) {
+ if (!IS_ALIGNED(sgiter->length, SZ_1M))
+ return -EINVAL;
+ cnt += sgiter->length / SZ_1M;
+ }
+
+ pa_list = kmalloc_array(cnt, sizeof(*pa_list), GFP_KERNEL);
+ if (!pa_list)
+ return -ENOMEM;
+
+ sgiter = sg;
+ cnt = 0;
+ pa = msm_secure_get_phys_addr(sgiter);
+ while (offset < len) {
+
+ if (!IS_ALIGNED(pa, SZ_1M)) {
+ kfree(pa_list);
+ return -EINVAL;
+ }
+
+ pa_list[cnt] = pa + chunk_offset;
+ chunk_offset += SZ_1M;
+ offset += SZ_1M;
+ cnt++;
+
+ if (chunk_offset >= sgiter->length && offset < len) {
+ chunk_offset = 0;
+ sgiter = sg_next(sgiter);
+ pa = msm_secure_get_phys_addr(sgiter);
+ }
+ }
+
+ desc.args[0] = virt_to_phys(pa_list);
+ desc.args[1] = cnt;
+ desc.args[2] = SZ_1M;
+ flush_va = pa_list;
+ }
+
+ desc.args[3] = cfg->arm_msm_secure_cfg.sec_id;
+ desc.args[4] = cfg->arm_msm_secure_cfg.cbndx;
+ desc.args[5] = iova;
+ desc.args[6] = len;
+ desc.args[7] = 0;
+
+ desc.arginfo = SCM_ARGS(8, SCM_RW, SCM_VAL, SCM_VAL, SCM_VAL, SCM_VAL,
+ SCM_VAL, SCM_VAL, SCM_VAL);
+
+ /*
+ * Ensure that the buffer is in RAM by the time it gets to TZ
+ */
+
+ flush_va_end = (void *) (((unsigned long) flush_va) +
+ (cnt * sizeof(*pa_list)));
+ dmac_clean_range(flush_va, flush_va_end);
+
+ if (is_scm_armv8()) {
+ ret = scm_call2(SCM_SIP_FNID(SCM_SVC_MP,
+ IOMMU_SECURE_MAP2_FLAT), &desc);
+ resp = desc.ret[0];
+
+ if (ret || resp)
+ ret = -EINVAL;
+ else
+ ret = len;
+ }
+
+ kfree(pa_list);
+ return ret;
+}
+
+static size_t msm_secure_unmap(struct io_pgtable_ops *ops, unsigned long iova,
+ size_t len)
+{
+ struct msm_secure_io_pgtable *data = io_pgtable_ops_to_data(ops);
+ struct io_pgtable_cfg *cfg = &data->iop.cfg;
+ int ret = -EINVAL;
+ struct scm_desc desc = {0};
+
+ if (!IS_ALIGNED(iova, SZ_1M) || !IS_ALIGNED(len, SZ_1M))
+ return ret;
+
+ desc.args[0] = cfg->arm_msm_secure_cfg.sec_id;
+ desc.args[1] = cfg->arm_msm_secure_cfg.cbndx;
+ desc.args[2] = iova;
+ desc.args[3] = len;
+ desc.args[4] = IOMMU_TLBINVAL_FLAG;
+ desc.arginfo = SCM_ARGS(5);
+
+ if (is_scm_armv8()) {
+ ret = scm_call2(SCM_SIP_FNID(SCM_SVC_MP,
+ IOMMU_SECURE_UNMAP2_FLAT), &desc);
+
+ if (!ret)
+ ret = len;
+ }
+ return ret;
+}
+
+static phys_addr_t msm_secure_iova_to_phys(struct io_pgtable_ops *ops,
+ unsigned long iova)
+{
+ return -EINVAL;
+}
+
+static struct msm_secure_io_pgtable *
+msm_secure_alloc_pgtable_data(struct io_pgtable_cfg *cfg)
+{
+ struct msm_secure_io_pgtable *data;
+
+ data = kmalloc(sizeof(*data), GFP_KERNEL);
+ if (!data)
+ return NULL;
+
+ data->iop.ops = (struct io_pgtable_ops) {
+ .map = msm_secure_map,
+ .map_sg = msm_secure_map_sg,
+ .unmap = msm_secure_unmap,
+ .iova_to_phys = msm_secure_iova_to_phys,
+ };
+
+ return data;
+}
+
+static struct io_pgtable *
+msm_secure_alloc_pgtable(struct io_pgtable_cfg *cfg, void *cookie)
+{
+ struct msm_secure_io_pgtable *data =
+ msm_secure_alloc_pgtable_data(cfg);
+
+ return &data->iop;
+}
+
+static void msm_secure_free_pgtable(struct io_pgtable *iop)
+{
+ struct msm_secure_io_pgtable *data = io_pgtable_to_data(iop);
+
+ kfree(data);
+}
+
+struct io_pgtable_init_fns io_pgtable_arm_msm_secure_init_fns = {
+ .alloc = msm_secure_alloc_pgtable,
+ .free = msm_secure_free_pgtable,
+};
diff --git a/drivers/iommu/io-pgtable.c b/drivers/iommu/io-pgtable.c
index 6f2e319d4f04..be11f5714750 100644
--- a/drivers/iommu/io-pgtable.c
+++ b/drivers/iommu/io-pgtable.c
@@ -18,12 +18,25 @@
* Author: Will Deacon <will.deacon@arm.com>
*/
+#define pr_fmt(fmt) "io-pgtable: " fmt
+
#include <linux/bug.h>
#include <linux/kernel.h>
#include <linux/types.h>
+#include <linux/iommu.h>
+#include <linux/debugfs.h>
+#include <linux/atomic.h>
+#include <linux/module.h>
#include "io-pgtable.h"
+extern struct io_pgtable_init_fns io_pgtable_arm_32_lpae_s1_init_fns;
+extern struct io_pgtable_init_fns io_pgtable_arm_32_lpae_s2_init_fns;
+extern struct io_pgtable_init_fns io_pgtable_arm_64_lpae_s1_init_fns;
+extern struct io_pgtable_init_fns io_pgtable_arm_64_lpae_s2_init_fns;
+extern struct io_pgtable_init_fns io_pgtable_arm_msm_secure_init_fns;
+extern struct io_pgtable_init_fns io_pgtable_av8l_fast_init_fns;
+
static const struct io_pgtable_init_fns *
io_pgtable_init_table[IO_PGTABLE_NUM_FMTS] =
{
@@ -33,8 +46,16 @@ io_pgtable_init_table[IO_PGTABLE_NUM_FMTS] =
[ARM_64_LPAE_S1] = &io_pgtable_arm_64_lpae_s1_init_fns,
[ARM_64_LPAE_S2] = &io_pgtable_arm_64_lpae_s2_init_fns,
#endif
+#ifdef CONFIG_MSM_TZ_SMMU
+ [ARM_MSM_SECURE] = &io_pgtable_arm_msm_secure_init_fns,
+#endif
+#ifdef CONFIG_IOMMU_IO_PGTABLE_FAST
+ [ARM_V8L_FAST] = &io_pgtable_av8l_fast_init_fns,
+#endif
};
+static struct dentry *io_pgtable_top;
+
struct io_pgtable_ops *alloc_io_pgtable_ops(enum io_pgtable_fmt fmt,
struct io_pgtable_cfg *cfg,
void *cookie)
@@ -72,6 +93,58 @@ void free_io_pgtable_ops(struct io_pgtable_ops *ops)
return;
iop = container_of(ops, struct io_pgtable, ops);
- iop->cfg.tlb->tlb_flush_all(iop->cookie);
io_pgtable_init_table[iop->fmt]->free(iop);
}
+
+static atomic_t pages_allocated;
+
+void *io_pgtable_alloc_pages_exact(struct io_pgtable_cfg *cfg, void *cookie,
+ size_t size, gfp_t gfp_mask)
+{
+ void *ret;
+
+ if (cfg->tlb->alloc_pages_exact)
+ ret = cfg->tlb->alloc_pages_exact(cookie, size, gfp_mask);
+ else
+ ret = alloc_pages_exact(size, gfp_mask);
+
+ if (likely(ret))
+ atomic_add(1 << get_order(size), &pages_allocated);
+
+ return ret;
+}
+
+void io_pgtable_free_pages_exact(struct io_pgtable_cfg *cfg, void *cookie,
+ void *virt, size_t size)
+{
+ if (cfg->tlb->free_pages_exact)
+ cfg->tlb->free_pages_exact(cookie, virt, size);
+ else
+ free_pages_exact(virt, size);
+
+ atomic_sub(1 << get_order(size), &pages_allocated);
+}
+
+static int io_pgtable_init(void)
+{
+ io_pgtable_top = debugfs_create_dir("io-pgtable", iommu_debugfs_top);
+
+ if (!io_pgtable_top)
+ return -ENODEV;
+
+ if (!debugfs_create_atomic_t("pages", 0600,
+ io_pgtable_top, &pages_allocated)) {
+ debugfs_remove_recursive(io_pgtable_top);
+ return -ENODEV;
+ }
+
+ return 0;
+}
+
+static void io_pgtable_exit(void)
+{
+ debugfs_remove_recursive(io_pgtable_top);
+}
+
+module_init(io_pgtable_init);
+module_exit(io_pgtable_exit);
diff --git a/drivers/iommu/io-pgtable.h b/drivers/iommu/io-pgtable.h
index ac9e2341a633..0326bb6a4afa 100644
--- a/drivers/iommu/io-pgtable.h
+++ b/drivers/iommu/io-pgtable.h
@@ -1,6 +1,9 @@
#ifndef __IO_PGTABLE_H
#define __IO_PGTABLE_H
+#include <linux/scatterlist.h>
+#include <soc/qcom/msm_tz_smmu.h>
+
/*
* Public API for use by IOMMU drivers
*/
@@ -9,6 +12,8 @@ enum io_pgtable_fmt {
ARM_32_LPAE_S2,
ARM_64_LPAE_S1,
ARM_64_LPAE_S2,
+ ARM_MSM_SECURE,
+ ARM_V8L_FAST,
IO_PGTABLE_NUM_FMTS,
};
@@ -20,6 +25,10 @@ enum io_pgtable_fmt {
* @tlb_sync: Ensure any queued TLB invalidation has taken effect, and
* any corresponding page table updates are visible to the
* IOMMU.
+ * @alloc_pages_exact: Allocate page table memory (optional, defaults to
+ * alloc_pages_exact)
+ * @free_pages_exact: Free page table memory (optional, defaults to
+ * free_pages_exact)
*
* Note that these can all be called in atomic context and must therefore
* not block.
@@ -29,6 +38,8 @@ struct iommu_gather_ops {
void (*tlb_add_flush)(unsigned long iova, size_t size, bool leaf,
void *cookie);
void (*tlb_sync)(void *cookie);
+ void *(*alloc_pages_exact)(void *cookie, size_t size, gfp_t gfp_mask);
+ void (*free_pages_exact)(void *cookie, void *virt, size_t size);
};
/**
@@ -45,13 +56,22 @@ struct iommu_gather_ops {
* page table walker.
*/
struct io_pgtable_cfg {
+ /*
+ * IO_PGTABLE_QUIRK_PAGE_TABLE_COHERENT: Set the page table as
+ * coherent.
+ */
#define IO_PGTABLE_QUIRK_ARM_NS (1 << 0) /* Set NS bit in PTEs */
+ #define IO_PGTABLE_QUIRK_PAGE_TABLE_COHERENT (1 << 1)
+ #define IO_PGTABLE_QUIRK_ARM_TTBR1 (1 << 2) /* Allocate TTBR1 PT */
int quirks;
unsigned long pgsize_bitmap;
unsigned int ias;
unsigned int oas;
+ int sep;
const struct iommu_gather_ops *tlb;
struct device *iommu_dev;
+ dma_addr_t iova_base;
+ dma_addr_t iova_end;
/* Low-level data specific to the table format */
union {
@@ -65,6 +85,18 @@ struct io_pgtable_cfg {
u64 vttbr;
u64 vtcr;
} arm_lpae_s2_cfg;
+
+ struct {
+ enum tz_smmu_device_id sec_id;
+ int cbndx;
+ } arm_msm_secure_cfg;
+
+ struct {
+ u64 ttbr[2];
+ u64 tcr;
+ u64 mair[2];
+ void *pmds;
+ } av8l_fast_cfg;
};
};
@@ -72,6 +104,9 @@ struct io_pgtable_cfg {
* struct io_pgtable_ops - Page table manipulation API for IOMMU drivers.
*
* @map: Map a physically contiguous memory region.
+ * @map_sg: Map a scatterlist. Returns the number of bytes mapped,
+ * or 0 on failure. The size parameter contains the size
+ * of the partial mapping in case of failure.
* @unmap: Unmap a physically contiguous memory region.
* @iova_to_phys: Translate iova to physical address.
*
@@ -81,10 +116,18 @@ struct io_pgtable_cfg {
struct io_pgtable_ops {
int (*map)(struct io_pgtable_ops *ops, unsigned long iova,
phys_addr_t paddr, size_t size, int prot);
- int (*unmap)(struct io_pgtable_ops *ops, unsigned long iova,
- size_t size);
+ int (*map_sg)(struct io_pgtable_ops *ops, unsigned long iova,
+ struct scatterlist *sg, unsigned int nents,
+ int prot, size_t *size);
+ size_t (*unmap)(struct io_pgtable_ops *ops, unsigned long iova,
+ size_t size);
phys_addr_t (*iova_to_phys)(struct io_pgtable_ops *ops,
unsigned long iova);
+ bool (*is_iova_coherent)(struct io_pgtable_ops *ops,
+ unsigned long iova);
+ uint64_t (*iova_to_pte)(struct io_pgtable_ops *ops,
+ unsigned long iova);
+
};
/**
@@ -143,6 +186,28 @@ struct io_pgtable_init_fns {
void (*free)(struct io_pgtable *iop);
};
+/**
+ * io_pgtable_alloc_pages_exact - allocate an exact number physically-contiguous pages.
+ * @size: the number of bytes to allocate
+ * @gfp_mask: GFP flags for the allocation
+ *
+ * Like alloc_pages_exact(), but with some additional accounting for debug
+ * purposes.
+ */
+void *io_pgtable_alloc_pages_exact(struct io_pgtable_cfg *cfg, void *cookie,
+ size_t size, gfp_t gfp_mask);
+
+/**
+ * io_pgtable_free_pages_exact - release memory allocated via io_pgtable_alloc_pages_exact()
+ * @virt: the value returned by alloc_pages_exact.
+ * @size: size of allocation, same value as passed to alloc_pages_exact().
+ *
+ * Like free_pages_exact(), but with some additional accounting for debug
+ * purposes.
+ */
+void io_pgtable_free_pages_exact(struct io_pgtable_cfg *cfg, void *cookie,
+ void *virt, size_t size);
+
extern struct io_pgtable_init_fns io_pgtable_arm_32_lpae_s1_init_fns;
extern struct io_pgtable_init_fns io_pgtable_arm_32_lpae_s2_init_fns;
extern struct io_pgtable_init_fns io_pgtable_arm_64_lpae_s1_init_fns;
diff --git a/drivers/iommu/iommu-debug.c b/drivers/iommu/iommu-debug.c
new file mode 100644
index 000000000000..108976f795b5
--- /dev/null
+++ b/drivers/iommu/iommu-debug.c
@@ -0,0 +1,2386 @@
+/*
+ * Copyright (c) 2015-2019, The Linux Foundation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 and
+ * only version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ */
+
+#define pr_fmt(fmt) "iommu-debug: %s: " fmt, __func__
+
+#include <linux/debugfs.h>
+#include <linux/device.h>
+#include <linux/iommu.h>
+#include <linux/of.h>
+#include <linux/platform_device.h>
+#include <linux/slab.h>
+#include <linux/uaccess.h>
+#include <linux/dma-contiguous.h>
+#include <soc/qcom/secure_buffer.h>
+#include <linux/qcom_iommu.h>
+#include <linux/module.h>
+#include <linux/dma-mapping.h>
+#include <asm/cacheflush.h>
+#include <asm/dma-iommu.h>
+#include "iommu-debug.h"
+
+#if defined(CONFIG_IOMMU_TESTS)
+
+static const char *iommu_debug_attr_to_string(enum iommu_attr attr)
+{
+ switch (attr) {
+ case DOMAIN_ATTR_GEOMETRY:
+ return "DOMAIN_ATTR_GEOMETRY";
+ case DOMAIN_ATTR_PAGING:
+ return "DOMAIN_ATTR_PAGING";
+ case DOMAIN_ATTR_WINDOWS:
+ return "DOMAIN_ATTR_WINDOWS";
+ case DOMAIN_ATTR_FSL_PAMU_STASH:
+ return "DOMAIN_ATTR_FSL_PAMU_STASH";
+ case DOMAIN_ATTR_FSL_PAMU_ENABLE:
+ return "DOMAIN_ATTR_FSL_PAMU_ENABLE";
+ case DOMAIN_ATTR_FSL_PAMUV1:
+ return "DOMAIN_ATTR_FSL_PAMUV1";
+ case DOMAIN_ATTR_NESTING:
+ return "DOMAIN_ATTR_NESTING";
+ case DOMAIN_ATTR_PT_BASE_ADDR:
+ return "DOMAIN_ATTR_PT_BASE_ADDR";
+ case DOMAIN_ATTR_SECURE_VMID:
+ return "DOMAIN_ATTR_SECURE_VMID";
+ case DOMAIN_ATTR_ATOMIC:
+ return "DOMAIN_ATTR_ATOMIC";
+ case DOMAIN_ATTR_CONTEXT_BANK:
+ return "DOMAIN_ATTR_CONTEXT_BANK";
+ case DOMAIN_ATTR_TTBR0:
+ return "DOMAIN_ATTR_TTBR0";
+ case DOMAIN_ATTR_CONTEXTIDR:
+ return "DOMAIN_ATTR_CONTEXTIDR";
+ case DOMAIN_ATTR_PROCID:
+ return "DOMAIN_ATTR_PROCID";
+ case DOMAIN_ATTR_DYNAMIC:
+ return "DOMAIN_ATTR_DYNAMIC";
+ case DOMAIN_ATTR_NON_FATAL_FAULTS:
+ return "DOMAIN_ATTR_NON_FATAL_FAULTS";
+ case DOMAIN_ATTR_S1_BYPASS:
+ return "DOMAIN_ATTR_S1_BYPASS";
+ case DOMAIN_ATTR_FAST:
+ return "DOMAIN_ATTR_FAST";
+ case DOMAIN_ATTR_EARLY_MAP:
+ return "DOMAIN_ATTR_EARLY_MAP";
+ case DOMAIN_ATTR_CB_STALL_DISABLE:
+ return "DOMAIN_ATTR_CB_STALL_DISABLE";
+ default:
+ return "Unknown attr!";
+ }
+}
+#endif
+
+#ifdef CONFIG_IOMMU_DEBUG_TRACKING
+
+static DEFINE_MUTEX(iommu_debug_attachments_lock);
+static LIST_HEAD(iommu_debug_attachments);
+
+/*
+ * Each group may have more than one domain; but each domain may
+ * only have one group.
+ * Used by debug tools to display the name of the device(s) associated
+ * with a particular domain.
+ */
+struct iommu_debug_attachment {
+ struct iommu_domain *domain;
+ struct iommu_group *group;
+ struct list_head list;
+};
+
+void iommu_debug_attach_device(struct iommu_domain *domain,
+ struct device *dev)
+{
+ struct iommu_debug_attachment *attach;
+ struct iommu_group *group;
+
+ group = iommu_group_get(dev);
+ if (!group)
+ return;
+
+ attach = kzalloc(sizeof(*attach), GFP_KERNEL);
+ if (!attach)
+ return;
+
+ attach->domain = domain;
+ attach->group = group;
+ INIT_LIST_HEAD(&attach->list);
+
+ mutex_lock(&iommu_debug_attachments_lock);
+ list_add(&attach->list, &iommu_debug_attachments);
+ mutex_unlock(&iommu_debug_attachments_lock);
+}
+
+void iommu_debug_domain_remove(struct iommu_domain *domain)
+{
+ struct iommu_debug_attachment *it, *tmp;
+
+ mutex_lock(&iommu_debug_attachments_lock);
+ list_for_each_entry_safe(it, tmp, &iommu_debug_attachments, list) {
+ if (it->domain != domain)
+ continue;
+ list_del(&it->list);
+ iommu_group_put(it->group);
+ kfree(it);
+ }
+
+ mutex_unlock(&iommu_debug_attachments_lock);
+}
+
+#endif
+
+#ifdef CONFIG_IOMMU_TESTS
+
+#ifdef CONFIG_64BIT
+
+#define kstrtoux kstrtou64
+#define kstrtox_from_user kstrtoull_from_user
+#define kstrtosize_t kstrtoul
+
+#else
+
+#define kstrtoux kstrtou32
+#define kstrtox_from_user kstrtouint_from_user
+#define kstrtosize_t kstrtouint
+
+#endif
+
+static LIST_HEAD(iommu_debug_devices);
+static struct dentry *debugfs_tests_dir;
+static u32 iters_per_op = 1;
+static void *virt_addr;
+
+struct iommu_debug_device {
+ struct device *dev;
+ struct iommu_domain *domain;
+ u64 iova;
+ u64 phys;
+ size_t len;
+ struct list_head list;
+ struct mutex clk_lock;
+ struct mutex dev_lock;
+ unsigned int clk_count;
+};
+
+static int iommu_debug_build_phoney_sg_table(struct device *dev,
+ struct sg_table *table,
+ unsigned long total_size,
+ unsigned long chunk_size)
+{
+ unsigned long nents = total_size / chunk_size;
+ struct scatterlist *sg;
+ int i;
+ struct page *page;
+
+ BUG_ON(!IS_ALIGNED(total_size, PAGE_SIZE));
+ BUG_ON(!IS_ALIGNED(total_size, chunk_size));
+ BUG_ON(sg_alloc_table(table, nents, GFP_KERNEL));
+ page = alloc_pages(GFP_KERNEL, get_order(chunk_size));
+ if (!page)
+ goto free_table;
+
+ /* all the same page... why not. */
+ for_each_sg(table->sgl, sg, table->nents, i)
+ sg_set_page(sg, page, chunk_size, 0);
+
+ return 0;
+
+free_table:
+ sg_free_table(table);
+ return -ENOMEM;
+}
+
+static void iommu_debug_destroy_phoney_sg_table(struct device *dev,
+ struct sg_table *table,
+ unsigned long chunk_size)
+{
+ __free_pages(sg_page(table->sgl), get_order(chunk_size));
+ sg_free_table(table);
+}
+
+static const char * const _size_to_string(unsigned long size)
+{
+ switch (size) {
+ case SZ_4K:
+ return "4K";
+ case SZ_8K:
+ return "8K";
+ case SZ_16K:
+ return "16K";
+ case SZ_64K:
+ return "64K";
+ case SZ_2M:
+ return "2M";
+ case SZ_1M * 12:
+ return "12M";
+ case SZ_1M * 20:
+ return "20M";
+ }
+ return "unknown size, please add to _size_to_string";
+}
+
+static int nr_iters_set(void *data, u64 val)
+{
+ if (!val)
+ val = 1;
+ if (val > 10000)
+ val = 10000;
+ *(u32 *)data = val;
+ return 0;
+}
+
+static int nr_iters_get(void *data, u64 *val)
+{
+ *val = *(u32 *)data;
+ return 0;
+}
+
+DEFINE_SIMPLE_ATTRIBUTE(iommu_debug_nr_iters_ops,
+ nr_iters_get, nr_iters_set, "%llu\n");
+
+static void iommu_debug_device_profiling(struct seq_file *s, struct device *dev,
+ enum iommu_attr attrs[],
+ void *attr_values[], int nattrs,
+ const size_t sizes[])
+{
+ int i;
+ const size_t *sz;
+ struct iommu_domain *domain;
+ struct bus_type *bus;
+ unsigned long iova = 0x10000;
+ phys_addr_t paddr = 0xa000;
+
+ bus = msm_iommu_get_bus(dev);
+ if (!bus)
+ return;
+
+ domain = iommu_domain_alloc(bus);
+ if (!domain) {
+ seq_puts(s, "Couldn't allocate domain\n");
+ return;
+ }
+
+ seq_puts(s, "Domain attributes: [ ");
+ for (i = 0; i < nattrs; ++i) {
+ /* not all attrs are ints, but this will get us by for now */
+ seq_printf(s, "%s=%d%s", iommu_debug_attr_to_string(attrs[i]),
+ *((int *)attr_values[i]),
+ i < nattrs ? " " : "");
+ }
+ seq_puts(s, "]\n");
+ for (i = 0; i < nattrs; ++i) {
+ if (iommu_domain_set_attr(domain, attrs[i], attr_values[i])) {
+ seq_printf(s, "Couldn't set %d to the value at %p\n",
+ attrs[i], attr_values[i]);
+ goto out_domain_free;
+ }
+ }
+
+ if (iommu_attach_device(domain, dev)) {
+ seq_puts(s,
+ "Couldn't attach new domain to device. Is it already attached?\n");
+ goto out_domain_free;
+ }
+
+ seq_printf(s, "(average over %d iterations)\n", iters_per_op);
+ seq_printf(s, "%8s %19s %16s\n", "size", "iommu_map", "iommu_unmap");
+ for (sz = sizes; *sz; ++sz) {
+ size_t size = *sz;
+ size_t unmapped;
+ u64 map_elapsed_ns = 0, unmap_elapsed_ns = 0;
+ u64 map_elapsed_us = 0, unmap_elapsed_us = 0;
+ u32 map_elapsed_rem = 0, unmap_elapsed_rem = 0;
+ struct timespec tbefore, tafter, diff;
+ int i;
+
+ for (i = 0; i < iters_per_op; ++i) {
+ getnstimeofday(&tbefore);
+ if (iommu_map(domain, iova, paddr, size,
+ IOMMU_READ | IOMMU_WRITE)) {
+ seq_puts(s, "Failed to map\n");
+ continue;
+ }
+ getnstimeofday(&tafter);
+ diff = timespec_sub(tafter, tbefore);
+ map_elapsed_ns += timespec_to_ns(&diff);
+
+ getnstimeofday(&tbefore);
+ unmapped = iommu_unmap(domain, iova, size);
+ if (unmapped != size) {
+ seq_printf(s,
+ "Only unmapped %zx instead of %zx\n",
+ unmapped, size);
+ continue;
+ }
+ getnstimeofday(&tafter);
+ diff = timespec_sub(tafter, tbefore);
+ unmap_elapsed_ns += timespec_to_ns(&diff);
+ }
+
+ map_elapsed_ns = div_u64_rem(map_elapsed_ns, iters_per_op,
+ &map_elapsed_rem);
+ unmap_elapsed_ns = div_u64_rem(unmap_elapsed_ns, iters_per_op,
+ &unmap_elapsed_rem);
+
+ map_elapsed_us = div_u64_rem(map_elapsed_ns, 1000,
+ &map_elapsed_rem);
+ unmap_elapsed_us = div_u64_rem(unmap_elapsed_ns, 1000,
+ &unmap_elapsed_rem);
+
+ seq_printf(s, "%8s %12lld.%03d us %9lld.%03d us\n",
+ _size_to_string(size),
+ map_elapsed_us, map_elapsed_rem,
+ unmap_elapsed_us, unmap_elapsed_rem);
+ }
+
+ seq_putc(s, '\n');
+ seq_printf(s, "%8s %19s %16s\n", "size", "iommu_map_sg", "iommu_unmap");
+ for (sz = sizes; *sz; ++sz) {
+ size_t size = *sz;
+ size_t unmapped;
+ u64 map_elapsed_ns = 0, unmap_elapsed_ns = 0;
+ u64 map_elapsed_us = 0, unmap_elapsed_us = 0;
+ u32 map_elapsed_rem = 0, unmap_elapsed_rem = 0;
+ struct timespec tbefore, tafter, diff;
+ struct sg_table table;
+ unsigned long chunk_size = SZ_4K;
+ int i;
+
+ if (iommu_debug_build_phoney_sg_table(dev, &table, size,
+ chunk_size)) {
+ seq_puts(s,
+ "couldn't build phoney sg table! bailing...\n");
+ goto out_detach;
+ }
+
+ for (i = 0; i < iters_per_op; ++i) {
+ getnstimeofday(&tbefore);
+ if (iommu_map_sg(domain, iova, table.sgl, table.nents,
+ IOMMU_READ | IOMMU_WRITE) != size) {
+ seq_puts(s, "Failed to map_sg\n");
+ goto next;
+ }
+ getnstimeofday(&tafter);
+ diff = timespec_sub(tafter, tbefore);
+ map_elapsed_ns += timespec_to_ns(&diff);
+
+ getnstimeofday(&tbefore);
+ unmapped = iommu_unmap(domain, iova, size);
+ if (unmapped != size) {
+ seq_printf(s,
+ "Only unmapped %zx instead of %zx\n",
+ unmapped, size);
+ goto next;
+ }
+ getnstimeofday(&tafter);
+ diff = timespec_sub(tafter, tbefore);
+ unmap_elapsed_ns += timespec_to_ns(&diff);
+ }
+
+ map_elapsed_ns = div_u64_rem(map_elapsed_ns, iters_per_op,
+ &map_elapsed_rem);
+ unmap_elapsed_ns = div_u64_rem(unmap_elapsed_ns, iters_per_op,
+ &unmap_elapsed_rem);
+
+ map_elapsed_us = div_u64_rem(map_elapsed_ns, 1000,
+ &map_elapsed_rem);
+ unmap_elapsed_us = div_u64_rem(unmap_elapsed_ns, 1000,
+ &unmap_elapsed_rem);
+
+ seq_printf(s, "%8s %12lld.%03d us %9lld.%03d us\n",
+ _size_to_string(size),
+ map_elapsed_us, map_elapsed_rem,
+ unmap_elapsed_us, unmap_elapsed_rem);
+
+next:
+ iommu_debug_destroy_phoney_sg_table(dev, &table, chunk_size);
+ }
+
+out_detach:
+ iommu_detach_device(domain, dev);
+out_domain_free:
+ iommu_domain_free(domain);
+}
+
+static int iommu_debug_profiling_show(struct seq_file *s, void *ignored)
+{
+ struct iommu_debug_device *ddev = s->private;
+ const size_t sizes[] = { SZ_4K, SZ_64K, SZ_2M, SZ_1M * 12,
+ SZ_1M * 20, 0 };
+ enum iommu_attr attrs[] = {
+ DOMAIN_ATTR_ATOMIC,
+ };
+ int htw_disable = 1, atomic = 1;
+ void *attr_values[] = { &htw_disable, &atomic };
+
+ iommu_debug_device_profiling(s, ddev->dev, attrs, attr_values,
+ ARRAY_SIZE(attrs), sizes);
+
+ return 0;
+}
+
+static int iommu_debug_profiling_open(struct inode *inode, struct file *file)
+{
+ return single_open(file, iommu_debug_profiling_show, inode->i_private);
+}
+
+static const struct file_operations iommu_debug_profiling_fops = {
+ .open = iommu_debug_profiling_open,
+ .read = seq_read,
+ .llseek = seq_lseek,
+ .release = single_release,
+};
+
+static int iommu_debug_secure_profiling_show(struct seq_file *s, void *ignored)
+{
+ struct iommu_debug_device *ddev = s->private;
+ const size_t sizes[] = { SZ_4K, SZ_64K, SZ_2M, SZ_1M * 12,
+ SZ_1M * 20, 0 };
+
+ enum iommu_attr attrs[] = {
+ DOMAIN_ATTR_ATOMIC,
+ DOMAIN_ATTR_SECURE_VMID,
+ };
+ int one = 1, secure_vmid = VMID_CP_PIXEL;
+ void *attr_values[] = { &one, &secure_vmid };
+
+ iommu_debug_device_profiling(s, ddev->dev, attrs, attr_values,
+ ARRAY_SIZE(attrs), sizes);
+
+ return 0;
+}
+
+static int iommu_debug_secure_profiling_open(struct inode *inode,
+ struct file *file)
+{
+ return single_open(file, iommu_debug_secure_profiling_show,
+ inode->i_private);
+}
+
+static const struct file_operations iommu_debug_secure_profiling_fops = {
+ .open = iommu_debug_secure_profiling_open,
+ .read = seq_read,
+ .llseek = seq_lseek,
+ .release = single_release,
+};
+
+static int iommu_debug_profiling_fast_show(struct seq_file *s, void *ignored)
+{
+ struct iommu_debug_device *ddev = s->private;
+ size_t sizes[] = {SZ_4K, SZ_8K, SZ_16K, SZ_64K, 0};
+ enum iommu_attr attrs[] = {
+ DOMAIN_ATTR_FAST,
+ DOMAIN_ATTR_ATOMIC,
+ DOMAIN_ATTR_GEOMETRY,
+ };
+ int one = 1;
+ struct iommu_domain_geometry geometry = {0, 0, 0};
+ void *attr_values[] = { &one, &one, &geometry};
+
+ geometry.aperture_end = (dma_addr_t)(SZ_1G * 4ULL - 1);
+
+ iommu_debug_device_profiling(s, ddev->dev, attrs, attr_values,
+ ARRAY_SIZE(attrs), sizes);
+
+ return 0;
+}
+
+static int iommu_debug_profiling_fast_open(struct inode *inode,
+ struct file *file)
+{
+ return single_open(file, iommu_debug_profiling_fast_show,
+ inode->i_private);
+}
+
+static const struct file_operations iommu_debug_profiling_fast_fops = {
+ .open = iommu_debug_profiling_fast_open,
+ .read = seq_read,
+ .llseek = seq_lseek,
+ .release = single_release,
+};
+
+static int iommu_debug_profiling_fast_dma_api_show(struct seq_file *s,
+ void *ignored)
+{
+ int i, experiment;
+ struct iommu_debug_device *ddev = s->private;
+ struct device *dev = ddev->dev;
+ u64 map_elapsed_ns[10], unmap_elapsed_ns[10];
+ struct dma_iommu_mapping *mapping;
+ dma_addr_t dma_addr;
+ void *virt;
+ int fast = 1;
+ const char * const extra_labels[] = {
+ "not coherent",
+ "coherent",
+ };
+ struct dma_attrs coherent_attrs;
+ struct dma_attrs *extra_attrs[] = {
+ NULL,
+ &coherent_attrs,
+ };
+
+ init_dma_attrs(&coherent_attrs);
+ dma_set_attr(DMA_ATTR_SKIP_CPU_SYNC, &coherent_attrs);
+
+ virt = kmalloc(1518, GFP_KERNEL);
+ if (!virt)
+ goto out;
+
+ mapping = arm_iommu_create_mapping(&platform_bus_type, 0, SZ_1G * 4ULL);
+ if (!mapping) {
+ seq_puts(s, "fast_smmu_create_mapping failed\n");
+ goto out_kfree;
+ }
+
+ if (iommu_domain_set_attr(mapping->domain, DOMAIN_ATTR_FAST, &fast)) {
+ seq_puts(s, "iommu_domain_set_attr failed\n");
+ goto out_release_mapping;
+ }
+
+ if (arm_iommu_attach_device(dev, mapping)) {
+ seq_puts(s, "fast_smmu_attach_device failed\n");
+ goto out_release_mapping;
+ }
+
+ if (iommu_enable_config_clocks(mapping->domain)) {
+ seq_puts(s, "Couldn't enable clocks\n");
+ goto out_detach;
+ }
+ for (experiment = 0; experiment < 2; ++experiment) {
+ size_t map_avg = 0, unmap_avg = 0;
+
+ for (i = 0; i < 10; ++i) {
+ struct timespec tbefore, tafter, diff;
+ u64 ns;
+
+ getnstimeofday(&tbefore);
+ dma_addr = dma_map_single_attrs(
+ dev, virt, SZ_4K, DMA_TO_DEVICE,
+ extra_attrs[experiment]);
+ getnstimeofday(&tafter);
+ diff = timespec_sub(tafter, tbefore);
+ ns = timespec_to_ns(&diff);
+ if (dma_mapping_error(dev, dma_addr)) {
+ seq_puts(s, "dma_map_single failed\n");
+ goto out_disable_config_clocks;
+ }
+ map_elapsed_ns[i] = ns;
+
+ getnstimeofday(&tbefore);
+ dma_unmap_single_attrs(
+ dev, dma_addr, SZ_4K, DMA_TO_DEVICE,
+ extra_attrs[experiment]);
+ getnstimeofday(&tafter);
+ diff = timespec_sub(tafter, tbefore);
+ ns = timespec_to_ns(&diff);
+ unmap_elapsed_ns[i] = ns;
+ }
+
+ seq_printf(s, "%13s %24s (ns): [", extra_labels[experiment],
+ "dma_map_single_attrs");
+ for (i = 0; i < 10; ++i) {
+ map_avg += map_elapsed_ns[i];
+ seq_printf(s, "%5llu%s", map_elapsed_ns[i],
+ i < 9 ? ", " : "");
+ }
+ map_avg /= 10;
+ seq_printf(s, "] (avg: %zu)\n", map_avg);
+
+ seq_printf(s, "%13s %24s (ns): [", extra_labels[experiment],
+ "dma_unmap_single_attrs");
+ for (i = 0; i < 10; ++i) {
+ unmap_avg += unmap_elapsed_ns[i];
+ seq_printf(s, "%5llu%s", unmap_elapsed_ns[i],
+ i < 9 ? ", " : "");
+ }
+ unmap_avg /= 10;
+ seq_printf(s, "] (avg: %zu)\n", unmap_avg);
+ }
+
+out_disable_config_clocks:
+ iommu_disable_config_clocks(mapping->domain);
+out_detach:
+ arm_iommu_detach_device(dev);
+out_release_mapping:
+ arm_iommu_release_mapping(mapping);
+out_kfree:
+ kfree(virt);
+out:
+ return 0;
+}
+
+static int iommu_debug_profiling_fast_dma_api_open(struct inode *inode,
+ struct file *file)
+{
+ return single_open(file, iommu_debug_profiling_fast_dma_api_show,
+ inode->i_private);
+}
+
+static const struct file_operations iommu_debug_profiling_fast_dma_api_fops = {
+ .open = iommu_debug_profiling_fast_dma_api_open,
+ .read = seq_read,
+ .llseek = seq_lseek,
+ .release = single_release,
+};
+
+static int __tlb_stress_sweep(struct device *dev, struct seq_file *s)
+{
+ int i, ret = 0;
+ u64 iova;
+ const u64 max = SZ_1G * 4ULL - 1;
+ void *virt;
+ phys_addr_t phys;
+ dma_addr_t dma_addr;
+
+ /*
+ * we'll be doing 4K and 8K mappings. Need to own an entire 8K
+ * chunk that we can work with.
+ */
+ virt = (void *)__get_free_pages(GFP_KERNEL, get_order(SZ_8K));
+ phys = virt_to_phys(virt);
+
+ /* fill the whole 4GB space */
+ for (iova = 0, i = 0; iova < max; iova += SZ_8K, ++i) {
+ dma_addr = dma_map_single(dev, virt, SZ_8K, DMA_TO_DEVICE);
+ if (dma_addr == DMA_ERROR_CODE) {
+ dev_err(dev, "Failed map on iter %d\n", i);
+ ret = -EINVAL;
+ goto out;
+ }
+ }
+
+ if (dma_map_single(dev, virt, SZ_4K, DMA_TO_DEVICE) != DMA_ERROR_CODE) {
+ dev_err(dev,
+ "dma_map_single unexpectedly (VA should have been exhausted)\n");
+ ret = -EINVAL;
+ goto out;
+ }
+
+ /*
+ * free up 4K at the very beginning, then leave one 4K mapping,
+ * then free up 8K. This will result in the next 8K map to skip
+ * over the 4K hole and take the 8K one.
+ */
+ dma_unmap_single(dev, 0, SZ_4K, DMA_TO_DEVICE);
+ dma_unmap_single(dev, SZ_8K, SZ_4K, DMA_TO_DEVICE);
+ dma_unmap_single(dev, SZ_8K + SZ_4K, SZ_4K, DMA_TO_DEVICE);
+
+ /* remap 8K */
+ dma_addr = dma_map_single(dev, virt, SZ_8K, DMA_TO_DEVICE);
+ if (dma_addr != SZ_8K) {
+ dma_addr_t expected = SZ_8K;
+
+ dev_err(dev, "Unexpected dma_addr. got: %pa expected: %pa\n",
+ &dma_addr, &expected);
+ ret = -EINVAL;
+ goto out;
+ }
+
+ /*
+ * now remap 4K. We should get the first 4K chunk that was skipped
+ * over during the previous 8K map. If we missed a TLB invalidate
+ * at that point this should explode.
+ */
+ dma_addr = dma_map_single(dev, virt, SZ_4K, DMA_TO_DEVICE);
+ if (dma_addr != 0) {
+ dma_addr_t expected = 0;
+
+ dev_err(dev, "Unexpected dma_addr. got: %pa expected: %pa\n",
+ &dma_addr, &expected);
+ ret = -EINVAL;
+ goto out;
+ }
+
+ if (dma_map_single(dev, virt, SZ_4K, DMA_TO_DEVICE) != DMA_ERROR_CODE) {
+ dev_err(dev,
+ "dma_map_single unexpectedly after remaps (VA should have been exhausted)\n");
+ ret = -EINVAL;
+ goto out;
+ }
+
+ /* we're all full again. unmap everything. */
+ for (iova = 0; iova < max; iova += SZ_8K)
+ dma_unmap_single(dev, (dma_addr_t)iova, SZ_8K, DMA_TO_DEVICE);
+
+out:
+ free_pages((unsigned long)virt, get_order(SZ_8K));
+ return ret;
+}
+
+struct fib_state {
+ unsigned long cur;
+ unsigned long prev;
+};
+
+static void fib_init(struct fib_state *f)
+{
+ f->cur = f->prev = 1;
+}
+
+static unsigned long get_next_fib(struct fib_state *f)
+{
+ int next = f->cur + f->prev;
+
+ f->prev = f->cur;
+ f->cur = next;
+ return next;
+}
+
+/*
+ * Not actually random. Just testing the fibs (and max - the fibs).
+ */
+static int __rand_va_sweep(struct device *dev, struct seq_file *s,
+ const size_t size)
+{
+ u64 iova;
+ const u64 max = SZ_1G * 4ULL - 1;
+ int i, remapped, unmapped, ret = 0;
+ void *virt;
+ dma_addr_t dma_addr, dma_addr2;
+ struct fib_state fib;
+
+ virt = (void *)__get_free_pages(GFP_KERNEL, get_order(size));
+ if (!virt) {
+ if (size > SZ_8K) {
+ dev_err(dev,
+ "Failed to allocate %s of memory, which is a lot. Skipping test for this size\n",
+ _size_to_string(size));
+ return 0;
+ }
+ return -ENOMEM;
+ }
+
+ /* fill the whole 4GB space */
+ for (iova = 0, i = 0; iova < max; iova += size, ++i) {
+ dma_addr = dma_map_single(dev, virt, size, DMA_TO_DEVICE);
+ if (dma_addr == DMA_ERROR_CODE) {
+ dev_err(dev, "Failed map on iter %d\n", i);
+ ret = -EINVAL;
+ goto out;
+ }
+ }
+
+ /* now unmap "random" iovas */
+ unmapped = 0;
+ fib_init(&fib);
+ for (iova = get_next_fib(&fib) * size;
+ iova < max - size;
+ iova = (u64)get_next_fib(&fib) * size) {
+ dma_addr = (dma_addr_t)(iova);
+ dma_addr2 = (dma_addr_t)((max + 1) - size - iova);
+ if (dma_addr == dma_addr2) {
+ WARN(1,
+ "%s test needs update! The random number sequence is folding in on itself and should be changed.\n",
+ __func__);
+ return -EINVAL;
+ }
+ dma_unmap_single(dev, dma_addr, size, DMA_TO_DEVICE);
+ dma_unmap_single(dev, dma_addr2, size, DMA_TO_DEVICE);
+ unmapped += 2;
+ }
+
+ /* and map until everything fills back up */
+ for (remapped = 0; ; ++remapped) {
+ dma_addr = dma_map_single(dev, virt, size, DMA_TO_DEVICE);
+ if (dma_addr == DMA_ERROR_CODE)
+ break;
+ }
+
+ if (unmapped != remapped) {
+ dev_err(dev,
+ "Unexpected random remap count! Unmapped %d but remapped %d\n",
+ unmapped, remapped);
+ ret = -EINVAL;
+ }
+
+ for (iova = 0; iova < max; iova += size)
+ dma_unmap_single(dev, (dma_addr_t)iova, size, DMA_TO_DEVICE);
+
+out:
+ free_pages((unsigned long)virt, get_order(size));
+ return ret;
+}
+
+static int __check_mapping(struct device *dev, struct iommu_domain *domain,
+ dma_addr_t iova, phys_addr_t expected)
+{
+ phys_addr_t res = iommu_iova_to_phys_hard(domain, iova);
+ phys_addr_t res2 = iommu_iova_to_phys(domain, iova);
+
+ WARN(res != res2, "hard/soft iova_to_phys fns don't agree...");
+
+ if (res != expected) {
+ dev_err_ratelimited(dev,
+ "Bad translation for %pa! Expected: %pa Got: %pa\n",
+ &iova, &expected, &res);
+ return -EINVAL;
+ }
+
+ return 0;
+}
+
+static int __full_va_sweep(struct device *dev, struct seq_file *s,
+ const size_t size, struct iommu_domain *domain)
+{
+ u64 iova;
+ dma_addr_t dma_addr;
+ void *virt;
+ phys_addr_t phys;
+ const u64 max = SZ_1G * 4ULL - 1;
+ int ret = 0, i;
+
+ virt = (void *)__get_free_pages(GFP_KERNEL, get_order(size));
+ if (!virt) {
+ if (size > SZ_8K) {
+ dev_err(dev,
+ "Failed to allocate %s of memory, which is a lot. Skipping test for this size\n",
+ _size_to_string(size));
+ return 0;
+ }
+ return -ENOMEM;
+ }
+ phys = virt_to_phys(virt);
+
+ for (iova = 0, i = 0; iova < max; iova += size, ++i) {
+ unsigned long expected = iova;
+
+ dma_addr = dma_map_single(dev, virt, size, DMA_TO_DEVICE);
+ if (dma_addr != expected) {
+ dev_err_ratelimited(dev,
+ "Unexpected iova on iter %d (expected: 0x%lx got: 0x%lx)\n",
+ i, expected,
+ (unsigned long)dma_addr);
+ ret = -EINVAL;
+ goto out;
+ }
+ }
+
+ if (domain) {
+ /* check every mapping from 0..6M */
+ for (iova = 0, i = 0; iova < SZ_2M * 3; iova += size, ++i) {
+ phys_addr_t expected = phys;
+
+ if (__check_mapping(dev, domain, iova, expected)) {
+ dev_err(dev, "iter: %d\n", i);
+ ret = -EINVAL;
+ goto out;
+ }
+ }
+ /* and from 4G..4G-6M */
+ for (iova = 0, i = 0; iova < SZ_2M * 3; iova += size, ++i) {
+ phys_addr_t expected = phys;
+ unsigned long theiova = ((SZ_1G * 4ULL) - size) - iova;
+
+ if (__check_mapping(dev, domain, theiova, expected)) {
+ dev_err(dev, "iter: %d\n", i);
+ ret = -EINVAL;
+ goto out;
+ }
+ }
+ }
+
+ /* at this point, our VA space should be full */
+ dma_addr = dma_map_single(dev, virt, size, DMA_TO_DEVICE);
+ if (dma_addr != DMA_ERROR_CODE) {
+ dev_err_ratelimited(dev,
+ "dma_map_single succeeded when it should have failed. Got iova: 0x%lx\n",
+ (unsigned long)dma_addr);
+ ret = -EINVAL;
+ }
+
+out:
+ for (iova = 0; iova < max; iova += size)
+ dma_unmap_single(dev, (dma_addr_t)iova, size, DMA_TO_DEVICE);
+
+ free_pages((unsigned long)virt, get_order(size));
+ return ret;
+}
+
+#define ds_printf(d, s, fmt, ...) ({ \
+ dev_err(d, fmt, ##__VA_ARGS__); \
+ seq_printf(s, fmt, ##__VA_ARGS__); \
+ })
+
+static int __functional_dma_api_va_test(struct device *dev, struct seq_file *s,
+ struct iommu_domain *domain, void *priv)
+{
+ int i, j, ret = 0;
+ size_t *sz, *sizes = priv;
+
+ for (j = 0; j < 1; ++j) {
+ for (sz = sizes; *sz; ++sz) {
+ for (i = 0; i < 2; ++i) {
+ ds_printf(dev, s, "Full VA sweep @%s %d",
+ _size_to_string(*sz), i);
+ if (__full_va_sweep(dev, s, *sz, domain)) {
+ ds_printf(dev, s, " -> FAILED\n");
+ ret = -EINVAL;
+ } else {
+ ds_printf(dev, s, " -> SUCCEEDED\n");
+ }
+ }
+ }
+ }
+
+ ds_printf(dev, s, "bonus map:");
+ if (__full_va_sweep(dev, s, SZ_4K, domain)) {
+ ds_printf(dev, s, " -> FAILED\n");
+ ret = -EINVAL;
+ } else {
+ ds_printf(dev, s, " -> SUCCEEDED\n");
+ }
+
+ for (sz = sizes; *sz; ++sz) {
+ for (i = 0; i < 2; ++i) {
+ ds_printf(dev, s, "Rand VA sweep @%s %d",
+ _size_to_string(*sz), i);
+ if (__rand_va_sweep(dev, s, *sz)) {
+ ds_printf(dev, s, " -> FAILED\n");
+ ret = -EINVAL;
+ } else {
+ ds_printf(dev, s, " -> SUCCEEDED\n");
+ }
+ }
+ }
+
+ ds_printf(dev, s, "TLB stress sweep");
+ if (__tlb_stress_sweep(dev, s)) {
+ ds_printf(dev, s, " -> FAILED\n");
+ ret = -EINVAL;
+ } else {
+ ds_printf(dev, s, " -> SUCCEEDED\n");
+ }
+
+ ds_printf(dev, s, "second bonus map:");
+ if (__full_va_sweep(dev, s, SZ_4K, domain)) {
+ ds_printf(dev, s, " -> FAILED\n");
+ ret = -EINVAL;
+ } else {
+ ds_printf(dev, s, " -> SUCCEEDED\n");
+ }
+
+ return ret;
+}
+
+static int __functional_dma_api_alloc_test(struct device *dev,
+ struct seq_file *s,
+ struct iommu_domain *domain,
+ void *ignored)
+{
+ size_t size = SZ_1K * 742;
+ int ret = 0;
+ u8 *data;
+ dma_addr_t iova;
+
+ /* Make sure we can allocate and use a buffer */
+ ds_printf(dev, s, "Allocating coherent buffer");
+ data = dma_alloc_coherent(dev, size, &iova, GFP_KERNEL);
+ if (!data) {
+ ds_printf(dev, s, " -> FAILED\n");
+ ret = -EINVAL;
+ } else {
+ int i;
+
+ ds_printf(dev, s, " -> SUCCEEDED\n");
+ ds_printf(dev, s, "Using coherent buffer");
+ for (i = 0; i < 742; ++i) {
+ int ind = SZ_1K * i;
+ u8 *p = data + ind;
+ u8 val = i % 255;
+
+ memset(data, 0xa5, size);
+ *p = val;
+ (*p)++;
+ if ((*p) != val + 1) {
+ ds_printf(dev, s,
+ " -> FAILED on iter %d since %d != %d\n",
+ i, *p, val + 1);
+ ret = -EINVAL;
+ }
+ }
+ if (!ret)
+ ds_printf(dev, s, " -> SUCCEEDED\n");
+ dma_free_coherent(dev, size, data, iova);
+ }
+
+ return ret;
+}
+
+static int __functional_dma_api_basic_test(struct device *dev,
+ struct seq_file *s,
+ struct iommu_domain *domain,
+ void *ignored)
+{
+ size_t size = 1518;
+ int i, j, ret = 0;
+ u8 *data;
+ dma_addr_t iova;
+ phys_addr_t pa, pa2;
+
+ ds_printf(dev, s, "Basic DMA API test");
+ /* Make sure we can allocate and use a buffer */
+ for (i = 0; i < 1000; ++i) {
+ data = kmalloc(size, GFP_KERNEL);
+ if (!data) {
+ ds_printf(dev, s, " -> FAILED\n");
+ ret = -EINVAL;
+ goto out;
+ }
+ memset(data, 0xa5, size);
+ iova = dma_map_single(dev, data, size, DMA_TO_DEVICE);
+ pa = iommu_iova_to_phys(domain, iova);
+ pa2 = iommu_iova_to_phys_hard(domain, iova);
+ if (pa != pa2) {
+ dev_err(dev,
+ "iova_to_phys doesn't match iova_to_phys_hard: %pa != %pa\n",
+ &pa, &pa2);
+ ret = -EINVAL;
+ goto out;
+ }
+ pa2 = virt_to_phys(data);
+ if (pa != pa2) {
+ dev_err(dev,
+ "iova_to_phys doesn't match virt_to_phys: %pa != %pa\n",
+ &pa, &pa2);
+ ret = -EINVAL;
+ goto out;
+ }
+ dma_unmap_single(dev, iova, size, DMA_TO_DEVICE);
+ for (j = 0; j < size; ++j) {
+ if (data[j] != 0xa5) {
+ dev_err(dev, "data[%d] != 0xa5\n", data[j]);
+ ret = -EINVAL;
+ goto out;
+ }
+ }
+ kfree(data);
+ }
+
+out:
+ if (ret)
+ ds_printf(dev, s, " -> FAILED\n");
+ else
+ ds_printf(dev, s, " -> SUCCEEDED\n");
+
+ return ret;
+}
+
+/* Creates a fresh fast mapping and applies @fn to it */
+static int __apply_to_new_mapping(struct seq_file *s,
+ int (*fn)(struct device *dev,
+ struct seq_file *s,
+ struct iommu_domain *domain,
+ void *priv),
+ void *priv)
+{
+ struct dma_iommu_mapping *mapping;
+ struct iommu_debug_device *ddev = s->private;
+ struct device *dev = ddev->dev;
+ int ret = -EINVAL, fast = 1;
+ phys_addr_t pt_phys;
+
+ mapping = arm_iommu_create_mapping(&platform_bus_type, 0,
+ (SZ_1G * 4ULL));
+ if (!mapping)
+ goto out;
+
+ if (iommu_domain_set_attr(mapping->domain, DOMAIN_ATTR_FAST, &fast)) {
+ seq_puts(s, "iommu_domain_set_attr failed\n");
+ goto out_release_mapping;
+ }
+
+ if (arm_iommu_attach_device(dev, mapping))
+ goto out_release_mapping;
+
+ if (iommu_domain_get_attr(mapping->domain, DOMAIN_ATTR_PT_BASE_ADDR,
+ &pt_phys)) {
+ ds_printf(dev, s, "Couldn't get page table base address\n");
+ goto out_release_mapping;
+ }
+
+ dev_err(dev, "testing with pgtables at %pa\n", &pt_phys);
+ if (iommu_enable_config_clocks(mapping->domain)) {
+ ds_printf(dev, s, "Couldn't enable clocks\n");
+ goto out_release_mapping;
+ }
+ ret = fn(dev, s, mapping->domain, priv);
+ iommu_disable_config_clocks(mapping->domain);
+
+ arm_iommu_detach_device(dev);
+out_release_mapping:
+ arm_iommu_release_mapping(mapping);
+out:
+ seq_printf(s, "%s\n", ret ? "FAIL" : "SUCCESS");
+ return 0;
+}
+
+static int iommu_debug_functional_fast_dma_api_show(struct seq_file *s,
+ void *ignored)
+{
+ size_t sizes[] = {SZ_4K, SZ_8K, SZ_16K, SZ_64K, 0};
+ int ret = 0;
+
+ ret |= __apply_to_new_mapping(s, __functional_dma_api_alloc_test, NULL);
+ ret |= __apply_to_new_mapping(s, __functional_dma_api_basic_test, NULL);
+ ret |= __apply_to_new_mapping(s, __functional_dma_api_va_test, sizes);
+ return ret;
+}
+
+static int iommu_debug_functional_fast_dma_api_open(struct inode *inode,
+ struct file *file)
+{
+ return single_open(file, iommu_debug_functional_fast_dma_api_show,
+ inode->i_private);
+}
+
+static const struct file_operations iommu_debug_functional_fast_dma_api_fops = {
+ .open = iommu_debug_functional_fast_dma_api_open,
+ .read = seq_read,
+ .llseek = seq_lseek,
+ .release = single_release,
+};
+
+static int iommu_debug_functional_arm_dma_api_show(struct seq_file *s,
+ void *ignored)
+{
+ struct dma_iommu_mapping *mapping;
+ struct iommu_debug_device *ddev = s->private;
+ struct device *dev = ddev->dev;
+ size_t sizes[] = {SZ_4K, SZ_64K, SZ_2M, SZ_1M * 12, 0};
+ int ret = -EINVAL;
+
+ /* Make the size equal to MAX_ULONG */
+ mapping = arm_iommu_create_mapping(&platform_bus_type, 0,
+ (SZ_1G * 4ULL - 1));
+ if (!mapping)
+ goto out;
+
+ if (arm_iommu_attach_device(dev, mapping))
+ goto out_release_mapping;
+
+ ret = __functional_dma_api_alloc_test(dev, s, mapping->domain, sizes);
+ ret |= __functional_dma_api_basic_test(dev, s, mapping->domain, sizes);
+
+ arm_iommu_detach_device(dev);
+out_release_mapping:
+ arm_iommu_release_mapping(mapping);
+out:
+ seq_printf(s, "%s\n", ret ? "FAIL" : "SUCCESS");
+ return 0;
+}
+
+static int iommu_debug_functional_arm_dma_api_open(struct inode *inode,
+ struct file *file)
+{
+ return single_open(file, iommu_debug_functional_arm_dma_api_show,
+ inode->i_private);
+}
+
+static const struct file_operations iommu_debug_functional_arm_dma_api_fops = {
+ .open = iommu_debug_functional_arm_dma_api_open,
+ .read = seq_read,
+ .llseek = seq_lseek,
+ .release = single_release,
+};
+
+static int iommu_debug_attach_do_attach(struct iommu_debug_device *ddev,
+ int val, bool is_secure)
+{
+ struct bus_type *bus;
+
+ bus = msm_iommu_get_bus(ddev->dev);
+ if (!bus)
+ return -EINVAL;
+
+ ddev->domain = iommu_domain_alloc(bus);
+ if (!ddev->domain) {
+ pr_err("Couldn't allocate domain\n");
+ return -ENOMEM;
+ }
+
+ val = VMID_CP_CAMERA;
+ if (is_secure && iommu_domain_set_attr(ddev->domain,
+ DOMAIN_ATTR_SECURE_VMID,
+ &val)) {
+ pr_err("Couldn't set secure vmid to %d\n", val);
+ goto out_domain_free;
+ }
+
+ if (iommu_attach_device(ddev->domain, ddev->dev)) {
+ pr_err("Couldn't attach new domain to device. Is it already attached?\n");
+ goto out_domain_free;
+ }
+
+ return 0;
+
+out_domain_free:
+ iommu_domain_free(ddev->domain);
+ ddev->domain = NULL;
+ return -EIO;
+}
+
+static ssize_t __iommu_debug_dma_attach_write(struct file *file,
+ const char __user *ubuf,
+ size_t count, loff_t *offset)
+{
+ struct iommu_debug_device *ddev = file->private_data;
+ struct device *dev = ddev->dev;
+ struct dma_iommu_mapping *dma_mapping;
+ ssize_t retval = -EINVAL;
+ int val;
+
+ mutex_lock(&ddev->dev_lock);
+ if (kstrtoint_from_user(ubuf, count, 0, &val)) {
+ pr_err("Invalid format. Expected a hex or decimal integer");
+ retval = -EFAULT;
+ goto out;
+ }
+
+ if (val) {
+ if (dev->archdata.mapping)
+ if (dev->archdata.mapping->domain) {
+ pr_err("Already attached.\n");
+ retval = -EINVAL;
+ goto out;
+ }
+ if (WARN(dev->archdata.iommu,
+ "Attachment tracking out of sync with device\n")) {
+ retval = -EINVAL;
+ goto out;
+ }
+
+ dma_mapping = arm_iommu_create_mapping(&platform_bus_type, 0,
+ (SZ_1G * 4ULL));
+
+ if (!dma_mapping)
+ goto out;
+
+ if (arm_iommu_attach_device(dev, dma_mapping))
+ goto out_release_mapping;
+ pr_err("Attached\n");
+ } else {
+ if (!dev->archdata.mapping) {
+ pr_err("No mapping. Did you already attach?\n");
+ retval = -EINVAL;
+ goto out;
+ }
+ if (!dev->archdata.mapping->domain) {
+ pr_err("No domain. Did you already attach?\n");
+ retval = -EINVAL;
+ goto out;
+ }
+ arm_iommu_detach_device(dev);
+ arm_iommu_release_mapping(dev->archdata.mapping);
+ pr_err("Detached\n");
+ }
+ mutex_unlock(&ddev->dev_lock);
+ retval = count;
+ return retval;
+
+out_release_mapping:
+ arm_iommu_release_mapping(dma_mapping);
+out:
+ mutex_unlock(&ddev->dev_lock);
+ return retval;
+}
+
+static ssize_t __iommu_debug_attach_write(struct file *file,
+ const char __user *ubuf,
+ size_t count, loff_t *offset,
+ bool is_secure)
+{
+ struct iommu_debug_device *ddev = file->private_data;
+ ssize_t retval;
+ int val;
+
+ mutex_lock(&ddev->dev_lock);
+ if (kstrtoint_from_user(ubuf, count, 0, &val)) {
+ pr_err("Invalid format. Expected a hex or decimal integer");
+ retval = -EFAULT;
+ goto out;
+ }
+
+ if (val) {
+ if (ddev->domain) {
+ pr_err("Already attached.\n");
+ retval = -EINVAL;
+ goto out;
+ }
+ if (WARN(ddev->dev->archdata.iommu,
+ "Attachment tracking out of sync with device\n")) {
+ retval = -EINVAL;
+ goto out;
+ }
+ if (iommu_debug_attach_do_attach(ddev, val, is_secure)) {
+ retval = -EIO;
+ goto out;
+ }
+ pr_err("Attached\n");
+ } else {
+ if (!ddev->domain) {
+ pr_err("No domain. Did you already attach?\n");
+ retval = -EINVAL;
+ goto out;
+ }
+ iommu_detach_device(ddev->domain, ddev->dev);
+ iommu_domain_free(ddev->domain);
+ ddev->domain = NULL;
+ pr_err("Detached\n");
+ }
+
+ retval = count;
+out:
+ mutex_unlock(&ddev->dev_lock);
+ return retval;
+}
+
+static ssize_t iommu_debug_dma_attach_write(struct file *file,
+ const char __user *ubuf,
+ size_t count, loff_t *offset)
+{
+ return __iommu_debug_dma_attach_write(file, ubuf, count, offset);
+
+}
+
+static ssize_t iommu_debug_dma_attach_read(struct file *file, char __user *ubuf,
+ size_t count, loff_t *offset)
+{
+ struct iommu_debug_device *ddev = file->private_data;
+ struct device *dev = ddev->dev;
+ char c[2];
+ size_t buflen = sizeof(c);
+
+ if (*offset)
+ return 0;
+
+ if (!dev->archdata.mapping)
+ c[0] = '0';
+ else
+ c[0] = dev->archdata.mapping->domain ? '1' : '0';
+
+ c[1] = '\n';
+ buflen = min(count, buflen);
+ if (copy_to_user(ubuf, &c, buflen)) {
+ pr_err("copy_to_user failed\n");
+ return -EFAULT;
+ }
+ *offset = 1; /* non-zero means we're done */
+
+ return buflen;
+}
+
+static const struct file_operations iommu_debug_dma_attach_fops = {
+ .open = simple_open,
+ .write = iommu_debug_dma_attach_write,
+ .read = iommu_debug_dma_attach_read,
+};
+
+static ssize_t iommu_debug_virt_addr_read(struct file *file, char __user *ubuf,
+ size_t count, loff_t *offset)
+{
+ char buf[100];
+ ssize_t retval;
+ size_t buflen;
+
+ if (*offset)
+ return 0;
+
+ memset(buf, 0, 100);
+
+ if (!virt_addr)
+ strlcpy(buf, "FAIL\n", 100);
+ else
+ snprintf(buf, 100, "0x%pK\n", virt_addr);
+
+ buflen = min(count, strlen(buf));
+ if (copy_to_user(ubuf, buf, buflen)) {
+ pr_err("Couldn't copy_to_user\n");
+ retval = -EFAULT;
+ } else {
+ *offset = 1; /* non-zero means we're done */
+ retval = buflen;
+ }
+
+ return retval;
+}
+
+static const struct file_operations iommu_debug_virt_addr_fops = {
+ .open = simple_open,
+ .read = iommu_debug_virt_addr_read,
+};
+
+static ssize_t iommu_debug_attach_write(struct file *file,
+ const char __user *ubuf,
+ size_t count, loff_t *offset)
+{
+ return __iommu_debug_attach_write(file, ubuf, count, offset,
+ false);
+
+}
+
+static ssize_t iommu_debug_attach_read(struct file *file, char __user *ubuf,
+ size_t count, loff_t *offset)
+{
+ struct iommu_debug_device *ddev = file->private_data;
+ char c[2];
+ size_t buflen = sizeof(c);
+
+ if (*offset)
+ return 0;
+
+ c[0] = ddev->domain ? '1' : '0';
+ c[1] = '\n';
+ buflen = min(count, buflen);
+ if (copy_to_user(ubuf, &c, buflen)) {
+ pr_err("copy_to_user failed\n");
+ return -EFAULT;
+ }
+ *offset = 1; /* non-zero means we're done */
+
+ return buflen;
+}
+
+static const struct file_operations iommu_debug_attach_fops = {
+ .open = simple_open,
+ .write = iommu_debug_attach_write,
+ .read = iommu_debug_attach_read,
+};
+
+static ssize_t iommu_debug_attach_write_secure(struct file *file,
+ const char __user *ubuf,
+ size_t count, loff_t *offset)
+{
+ return __iommu_debug_attach_write(file, ubuf, count, offset,
+ true);
+
+}
+
+static const struct file_operations iommu_debug_secure_attach_fops = {
+ .open = simple_open,
+ .write = iommu_debug_attach_write_secure,
+ .read = iommu_debug_attach_read,
+};
+
+static ssize_t iommu_debug_pte_write(struct file *file,
+ const char __user *ubuf,
+ size_t count, loff_t *offset)
+{
+ struct iommu_debug_device *ddev = file->private_data;
+ dma_addr_t iova;
+
+ if (kstrtox_from_user(ubuf, count, 0, &iova)) {
+ pr_err("Invalid format for iova\n");
+ ddev->iova = 0;
+ return -EINVAL;
+ }
+
+ ddev->iova = iova;
+ pr_err("Saved iova=%pa for future PTE commands\n", &iova);
+ return count;
+}
+
+
+static ssize_t iommu_debug_pte_read(struct file *file, char __user *ubuf,
+ size_t count, loff_t *offset)
+{
+ struct iommu_debug_device *ddev = file->private_data;
+ struct device *dev = ddev->dev;
+ uint64_t pte;
+ char buf[100];
+ ssize_t retval;
+ size_t buflen;
+
+ if (kptr_restrict != 0) {
+ pr_err("kptr_restrict needs to be disabled.\n");
+ return -EPERM;
+ }
+ if (!dev->archdata.mapping) {
+ pr_err("No mapping. Did you already attach?\n");
+ return -EINVAL;
+ }
+ if (!dev->archdata.mapping->domain) {
+ pr_err("No domain. Did you already attach?\n");
+ return -EINVAL;
+ }
+
+ if (*offset)
+ return 0;
+
+ memset(buf, 0, 100);
+
+ pte = iommu_iova_to_pte(dev->archdata.mapping->domain,
+ ddev->iova);
+
+ if (!pte)
+ strlcpy(buf, "FAIL\n", 100);
+ else
+ snprintf(buf, 100, "pte=%016llx\n", pte);
+
+ buflen = min(count, strlen(buf));
+ if (copy_to_user(ubuf, buf, buflen)) {
+ pr_err("Couldn't copy_to_user\n");
+ retval = -EFAULT;
+ } else {
+ *offset = 1; /* non-zero means we're done */
+ retval = buflen;
+ }
+
+ return retval;
+}
+
+static const struct file_operations iommu_debug_pte_fops = {
+ .open = simple_open,
+ .write = iommu_debug_pte_write,
+ .read = iommu_debug_pte_read,
+};
+
+static ssize_t iommu_debug_atos_write(struct file *file,
+ const char __user *ubuf,
+ size_t count, loff_t *offset)
+{
+ struct iommu_debug_device *ddev = file->private_data;
+ dma_addr_t iova;
+
+ if (kstrtox_from_user(ubuf, count, 0, &iova)) {
+ pr_err("Invalid format for iova\n");
+ ddev->iova = 0;
+ return -EINVAL;
+ }
+
+ ddev->iova = iova;
+ pr_err("Saved iova=%pa for future ATOS commands\n", &iova);
+ return count;
+}
+
+static ssize_t iommu_debug_atos_read(struct file *file, char __user *ubuf,
+ size_t count, loff_t *offset)
+{
+ struct iommu_debug_device *ddev = file->private_data;
+ phys_addr_t phys;
+ char buf[100];
+ ssize_t retval;
+ size_t buflen;
+
+ if (kptr_restrict != 0) {
+ pr_err("kptr_restrict needs to be disabled.\n");
+ return -EPERM;
+ }
+ if (!ddev->domain) {
+ pr_err("No domain. Did you already attach?\n");
+ return -EINVAL;
+ }
+
+ if (*offset)
+ return 0;
+
+ memset(buf, 0, 100);
+
+ phys = iommu_iova_to_phys_hard(ddev->domain, ddev->iova);
+ if (!phys) {
+ strlcpy(buf, "FAIL\n", 100);
+ phys = iommu_iova_to_phys(ddev->domain, ddev->iova);
+ dev_err(ddev->dev, "ATOS for %pa failed. Software walk returned: %pa\n",
+ &ddev->iova, &phys);
+ } else {
+ snprintf(buf, 100, "%pa\n", &phys);
+ }
+
+ buflen = min(count, strlen(buf));
+ if (copy_to_user(ubuf, buf, buflen)) {
+ pr_err("Couldn't copy_to_user\n");
+ retval = -EFAULT;
+ } else {
+ *offset = 1; /* non-zero means we're done */
+ retval = buflen;
+ }
+
+ return retval;
+}
+
+static const struct file_operations iommu_debug_atos_fops = {
+ .open = simple_open,
+ .write = iommu_debug_atos_write,
+ .read = iommu_debug_atos_read,
+};
+
+static ssize_t iommu_debug_dma_atos_read(struct file *file, char __user *ubuf,
+ size_t count, loff_t *offset)
+{
+ struct iommu_debug_device *ddev = file->private_data;
+ struct device *dev = ddev->dev;
+ phys_addr_t phys;
+ char buf[100];
+ ssize_t retval;
+ size_t buflen;
+
+ if (kptr_restrict != 0) {
+ pr_err("kptr_restrict needs to be disabled.\n");
+ return -EPERM;
+ }
+ if (!dev->archdata.mapping) {
+ pr_err("No mapping. Did you already attach?\n");
+ return -EINVAL;
+ }
+ if (!dev->archdata.mapping->domain) {
+ pr_err("No domain. Did you already attach?\n");
+ return -EINVAL;
+ }
+
+ if (*offset)
+ return 0;
+
+ memset(buf, 0, 100);
+
+ phys = iommu_iova_to_phys_hard(dev->archdata.mapping->domain,
+ ddev->iova);
+ if (!phys)
+ strlcpy(buf, "FAIL\n", 100);
+ else
+ snprintf(buf, 100, "%pa\n", &phys);
+
+ buflen = min(count, strlen(buf));
+ if (copy_to_user(ubuf, buf, buflen)) {
+ pr_err("Couldn't copy_to_user\n");
+ retval = -EFAULT;
+ } else {
+ *offset = 1; /* non-zero means we're done */
+ retval = buflen;
+ }
+
+ return retval;
+}
+
+static const struct file_operations iommu_debug_dma_atos_fops = {
+ .open = simple_open,
+ .write = iommu_debug_atos_write,
+ .read = iommu_debug_dma_atos_read,
+};
+
+static ssize_t iommu_debug_map_write(struct file *file, const char __user *ubuf,
+ size_t count, loff_t *offset)
+{
+ ssize_t retval = -EINVAL;
+ int ret;
+ char *comma1, *comma2, *comma3;
+ char buf[100];
+ dma_addr_t iova;
+ phys_addr_t phys;
+ size_t size;
+ int prot;
+ struct iommu_debug_device *ddev = file->private_data;
+
+ if (count >= 100) {
+ pr_err("Value too large\n");
+ return -EINVAL;
+ }
+
+ if (!ddev->domain) {
+ pr_err("No domain. Did you already attach?\n");
+ return -EINVAL;
+ }
+
+ memset(buf, 0, 100);
+
+ if (copy_from_user(buf, ubuf, count)) {
+ pr_err("Couldn't copy from user\n");
+ retval = -EFAULT;
+ }
+
+ comma1 = strnchr(buf, count, ',');
+ if (!comma1)
+ goto invalid_format;
+
+ comma2 = strnchr(comma1 + 1, count, ',');
+ if (!comma2)
+ goto invalid_format;
+
+ comma3 = strnchr(comma2 + 1, count, ',');
+ if (!comma3)
+ goto invalid_format;
+
+ /* split up the words */
+ *comma1 = *comma2 = *comma3 = '\0';
+
+ if (kstrtoux(buf, 0, &iova))
+ goto invalid_format;
+
+ if (kstrtoux(comma1 + 1, 0, &phys))
+ goto invalid_format;
+
+ if (kstrtosize_t(comma2 + 1, 0, &size))
+ goto invalid_format;
+
+ if (kstrtoint(comma3 + 1, 0, &prot))
+ goto invalid_format;
+
+ mutex_lock(&ddev->dev_lock);
+ ret = iommu_map(ddev->domain, iova, phys, size, prot);
+ if (ret) {
+ pr_err("iommu_map failed with %d\n", ret);
+ retval = -EIO;
+ goto out;
+ }
+
+ retval = count;
+ pr_err("Mapped %pa to %pa (len=0x%zx, prot=0x%x)\n",
+ &iova, &phys, size, prot);
+out:
+ mutex_unlock(&ddev->dev_lock);
+ return retval;
+
+invalid_format:
+ pr_err("Invalid format. Expected: iova,phys,len,prot where `prot' is the bitwise OR of IOMMU_READ, IOMMU_WRITE, etc.\n");
+ return retval;
+}
+
+static const struct file_operations iommu_debug_map_fops = {
+ .open = simple_open,
+ .write = iommu_debug_map_write,
+};
+
+static ssize_t iommu_debug_dma_map_write(struct file *file,
+ const char __user *ubuf, size_t count, loff_t *offset)
+{
+ ssize_t retval = -EINVAL;
+ int ret;
+ char *comma1, *comma2;
+ char buf[100];
+ unsigned long addr;
+ void *v_addr;
+ dma_addr_t iova;
+ size_t size;
+ unsigned int attr;
+ struct dma_attrs coherent_attr;
+ struct dma_attrs *dma_attrs = &coherent_attr;
+ struct iommu_debug_device *ddev = file->private_data;
+ struct device *dev = ddev->dev;
+
+ init_dma_attrs(dma_attrs);
+
+ if (count >= 100) {
+ pr_err("Value too large\n");
+ return -EINVAL;
+ }
+
+ if (!dev->archdata.mapping) {
+ pr_err("No mapping. Did you already attach?\n");
+ retval = -EINVAL;
+ goto out;
+ }
+ if (!dev->archdata.mapping->domain) {
+ pr_err("No domain. Did you already attach?\n");
+ retval = -EINVAL;
+ goto out;
+ }
+
+ memset(buf, 0, 100);
+
+ if (copy_from_user(buf, ubuf, count)) {
+ pr_err("Couldn't copy from user\n");
+ retval = -EFAULT;
+ goto out;
+ }
+
+ comma1 = strnchr(buf, count, ',');
+ if (!comma1)
+ goto invalid_format;
+
+ comma2 = strnchr(comma1 + 1, count, ',');
+ if (!comma2)
+ goto invalid_format;
+
+ *comma1 = *comma2 = '\0';
+
+ if (kstrtoul(buf, 0, &addr))
+ goto invalid_format;
+ v_addr = (void *)addr;
+
+ if (kstrtosize_t(comma1 + 1, 0, &size))
+ goto invalid_format;
+
+ if (kstrtouint(comma2 + 1, 0, &attr))
+ goto invalid_format;
+
+ if (v_addr < virt_addr || v_addr > (virt_addr + SZ_1M - 1))
+ goto invalid_addr;
+
+ if (attr == 0)
+ dma_attrs = NULL;
+ else if (attr == 1)
+ dma_set_attr(DMA_ATTR_FORCE_COHERENT, dma_attrs);
+ else if (attr == 2)
+ dma_set_attr(DMA_ATTR_FORCE_NON_COHERENT, dma_attrs);
+ else
+ goto invalid_format;
+
+ mutex_lock(&ddev->dev_lock);
+ iova = dma_map_single_attrs(dev, v_addr, size,
+ DMA_TO_DEVICE, dma_attrs);
+
+ if (dma_mapping_error(dev, iova)) {
+ pr_err("Failed to perform dma_map_single\n");
+ ret = -EINVAL;
+ mutex_unlock(&ddev->dev_lock);
+ goto out;
+ }
+ mutex_unlock(&ddev->dev_lock);
+
+ retval = count;
+ pr_err("Mapped 0x%p to %pa (len=0x%zx)\n",
+ v_addr, &iova, size);
+ ddev->iova = iova;
+ pr_err("Saved iova=%pa for future PTE commands\n", &iova);
+out:
+ return retval;
+
+invalid_format:
+ pr_err("Invalid format. Expected: addr,len,dma attr where 'dma attr' is\n0: normal mapping\n1: force coherent\n2: force non-cohernet\n");
+ return retval;
+
+invalid_addr:
+ pr_err("Invalid addr given! Address should be within 1MB size from start addr returned by doing 'cat virt_addr'.\n");
+ return retval;
+}
+
+static ssize_t iommu_debug_dma_map_read(struct file *file, char __user *ubuf,
+ size_t count, loff_t *offset)
+{
+ struct iommu_debug_device *ddev = file->private_data;
+ struct device *dev = ddev->dev;
+ char buf[100];
+ ssize_t retval;
+ size_t buflen;
+ dma_addr_t iova;
+
+ if (!dev->archdata.mapping) {
+ pr_err("No mapping. Did you already attach?\n");
+ return -EINVAL;
+ }
+ if (!dev->archdata.mapping->domain) {
+ pr_err("No domain. Did you already attach?\n");
+ return -EINVAL;
+ }
+
+ if (*offset)
+ return 0;
+
+ memset(buf, 0, 100);
+
+ iova = ddev->iova;
+ snprintf(buf, 100, "%pa\n", &iova);
+
+ buflen = min(count, strlen(buf));
+ if (copy_to_user(ubuf, buf, buflen)) {
+ pr_err("Couldn't copy_to_user\n");
+ retval = -EFAULT;
+ } else {
+ *offset = 1; /* non-zero means we're done */
+ retval = buflen;
+ }
+
+ return retval;
+}
+
+static const struct file_operations iommu_debug_dma_map_fops = {
+ .open = simple_open,
+ .write = iommu_debug_dma_map_write,
+ .read = iommu_debug_dma_map_read,
+};
+
+static ssize_t iommu_debug_unmap_write(struct file *file,
+ const char __user *ubuf,
+ size_t count, loff_t *offset)
+{
+ ssize_t retval = 0;
+ char *comma1;
+ char buf[100];
+ dma_addr_t iova;
+ size_t size;
+ size_t unmapped;
+ struct iommu_debug_device *ddev = file->private_data;
+
+ if (count >= 100) {
+ pr_err("Value too large\n");
+ return -EINVAL;
+ }
+
+ if (!ddev->domain) {
+ pr_err("No domain. Did you already attach?\n");
+ return -EINVAL;
+ }
+
+ memset(buf, 0, 100);
+
+ if (copy_from_user(buf, ubuf, count)) {
+ pr_err("Couldn't copy from user\n");
+ retval = -EFAULT;
+ goto out;
+ }
+
+ comma1 = strnchr(buf, count, ',');
+ if (!comma1)
+ goto invalid_format;
+
+ /* split up the words */
+ *comma1 = '\0';
+
+ if (kstrtoux(buf, 0, &iova))
+ goto invalid_format;
+
+ if (kstrtosize_t(comma1 + 1, 0, &size))
+ goto invalid_format;
+
+ mutex_lock(&ddev->dev_lock);
+ unmapped = iommu_unmap(ddev->domain, iova, size);
+ if (unmapped != size) {
+ pr_err("iommu_unmap failed. Expected to unmap: 0x%zx, unmapped: 0x%zx",
+ size, unmapped);
+ mutex_unlock(&ddev->dev_lock);
+ return -EIO;
+ }
+ mutex_unlock(&ddev->dev_lock);
+
+ retval = count;
+ pr_err("Unmapped %pa (len=0x%zx)\n", &iova, size);
+out:
+ return retval;
+
+invalid_format:
+ pr_err("Invalid format. Expected: iova,len\n");
+ return retval;
+}
+
+static const struct file_operations iommu_debug_unmap_fops = {
+ .open = simple_open,
+ .write = iommu_debug_unmap_write,
+};
+
+static ssize_t iommu_debug_dma_unmap_write(struct file *file,
+ const char __user *ubuf,
+ size_t count, loff_t *offset)
+{
+ ssize_t retval = 0;
+ char *comma1, *comma2;
+ char buf[100];
+ size_t size;
+ unsigned int attr;
+ dma_addr_t iova;
+ struct dma_attrs coherent_attr;
+ struct dma_attrs *dma_attrs = &coherent_attr;
+ struct iommu_debug_device *ddev = file->private_data;
+ struct device *dev = ddev->dev;
+
+ init_dma_attrs(dma_attrs);
+
+ if (count >= 100) {
+ pr_err("Value too large\n");
+ return -EINVAL;
+ }
+
+ if (!dev->archdata.mapping) {
+ pr_err("No mapping. Did you already attach?\n");
+ retval = -EINVAL;
+ goto out;
+ }
+ if (!dev->archdata.mapping->domain) {
+ pr_err("No domain. Did you already attach?\n");
+ retval = -EINVAL;
+ goto out;
+ }
+
+ memset(buf, 0, 100);
+
+ if (copy_from_user(buf, ubuf, count)) {
+ pr_err("Couldn't copy from user\n");
+ retval = -EFAULT;
+ goto out;
+ }
+
+ comma1 = strnchr(buf, count, ',');
+ if (!comma1)
+ goto invalid_format;
+
+ comma2 = strnchr(comma1 + 1, count, ',');
+ if (!comma2)
+ goto invalid_format;
+
+ *comma1 = *comma2 = '\0';
+
+ if (kstrtoux(buf, 0, &iova))
+ goto invalid_format;
+
+ if (kstrtosize_t(comma1 + 1, 0, &size))
+ goto invalid_format;
+
+ if (kstrtouint(comma2 + 1, 0, &attr))
+ goto invalid_format;
+
+ if (attr == 0)
+ dma_attrs = NULL;
+ else if (attr == 1)
+ dma_set_attr(DMA_ATTR_FORCE_COHERENT, dma_attrs);
+ else if (attr == 2)
+ dma_set_attr(DMA_ATTR_FORCE_NON_COHERENT, dma_attrs);
+ else
+ goto invalid_format;
+
+ mutex_lock(&ddev->dev_lock);
+ dma_unmap_single_attrs(dev, iova, size, DMA_TO_DEVICE, dma_attrs);
+ mutex_unlock(&ddev->dev_lock);
+
+ retval = count;
+ pr_err("Unmapped %pa (len=0x%zx)\n", &iova, size);
+out:
+ return retval;
+
+invalid_format:
+ pr_err("Invalid format. Expected: iova,len, dma attr\n");
+ return retval;
+}
+
+static const struct file_operations iommu_debug_dma_unmap_fops = {
+ .open = simple_open,
+ .write = iommu_debug_dma_unmap_write,
+};
+
+static ssize_t iommu_debug_config_clocks_write(struct file *file,
+ const char __user *ubuf,
+ size_t count, loff_t *offset)
+{
+ char buf;
+ struct iommu_debug_device *ddev = file->private_data;
+ struct device *dev = ddev->dev;
+
+ /* we're expecting a single character plus (optionally) a newline */
+ if (count > 2) {
+ dev_err(dev, "Invalid value\n");
+ return -EINVAL;
+ }
+
+ if (!ddev->domain) {
+ dev_err(dev, "No domain. Did you already attach?\n");
+ return -EINVAL;
+ }
+
+ if (copy_from_user(&buf, ubuf, 1)) {
+ dev_err(dev, "Couldn't copy from user\n");
+ return -EFAULT;
+ }
+
+ mutex_lock(&ddev->clk_lock);
+ switch (buf) {
+ case '0':
+ if (ddev->clk_count == 0) {
+ dev_err(dev, "Config clocks already disabled\n");
+ break;
+ }
+
+ if (--ddev->clk_count > 0)
+ break;
+
+ dev_err(dev, "Disabling config clocks\n");
+ iommu_disable_config_clocks(ddev->domain);
+ break;
+ case '1':
+ if (ddev->clk_count++ > 0)
+ break;
+
+ dev_err(dev, "Enabling config clocks\n");
+ if (iommu_enable_config_clocks(ddev->domain))
+ dev_err(dev, "Failed!\n");
+ break;
+ default:
+ dev_err(dev, "Invalid value. Should be 0 or 1.\n");
+ mutex_unlock(&ddev->clk_lock);
+ return -EINVAL;
+ }
+ mutex_unlock(&ddev->clk_lock);
+
+ return count;
+}
+
+static const struct file_operations iommu_debug_config_clocks_fops = {
+ .open = simple_open,
+ .write = iommu_debug_config_clocks_write,
+};
+
+/*
+ * The following will only work for drivers that implement the generic
+ * device tree bindings described in
+ * Documentation/devicetree/bindings/iommu/iommu.txt
+ */
+static int snarf_iommu_devices(struct device *dev, const char *name)
+{
+ struct iommu_debug_device *ddev;
+ struct dentry *dir;
+
+ if (IS_ERR_OR_NULL(dev))
+ return -EINVAL;
+
+ ddev = kzalloc(sizeof(*ddev), GFP_KERNEL);
+ if (!ddev)
+ return -ENODEV;
+ mutex_init(&ddev->clk_lock);
+ mutex_init(&ddev->dev_lock);
+ ddev->dev = dev;
+ dir = debugfs_create_dir(name, debugfs_tests_dir);
+ if (!dir) {
+ pr_err("Couldn't create iommu/devices/%s debugfs dir\n",
+ name);
+ goto err;
+ }
+
+ if (!debugfs_create_file("nr_iters", S_IRUSR, dir, &iters_per_op,
+ &iommu_debug_nr_iters_ops)) {
+ pr_err("Couldn't create iommu/devices/%s/nr_iters debugfs file\n",
+ name);
+ goto err_rmdir;
+ }
+
+ if (!debugfs_create_file("virt_addr", S_IRUSR, dir, ddev,
+ &iommu_debug_virt_addr_fops)) {
+ pr_err("Couldn't create iommu/devices/%s/virt_addr debugfs file\n",
+ name);
+ goto err_rmdir;
+ }
+
+ if (!debugfs_create_file("profiling", S_IRUSR, dir, ddev,
+ &iommu_debug_profiling_fops)) {
+ pr_err("Couldn't create iommu/devices/%s/profiling debugfs file\n",
+ name);
+ goto err_rmdir;
+ }
+
+ if (!debugfs_create_file("secure_profiling", S_IRUSR, dir, ddev,
+ &iommu_debug_secure_profiling_fops)) {
+ pr_err("Couldn't create iommu/devices/%s/secure_profiling debugfs file\n",
+ name);
+ goto err_rmdir;
+ }
+
+ if (!debugfs_create_file("profiling_fast", S_IRUSR, dir, ddev,
+ &iommu_debug_profiling_fast_fops)) {
+ pr_err("Couldn't create iommu/devices/%s/profiling_fast debugfs file\n",
+ name);
+ goto err_rmdir;
+ }
+
+ if (!debugfs_create_file("profiling_fast_dma_api", S_IRUSR, dir, ddev,
+ &iommu_debug_profiling_fast_dma_api_fops)) {
+ pr_err("Couldn't create iommu/devices/%s/profiling_fast_dma_api debugfs file\n",
+ name);
+ goto err_rmdir;
+ }
+
+ if (!debugfs_create_file("functional_fast_dma_api", S_IRUSR, dir, ddev,
+ &iommu_debug_functional_fast_dma_api_fops)) {
+ pr_err("Couldn't create iommu/devices/%s/functional_fast_dma_api debugfs file\n",
+ name);
+ goto err_rmdir;
+ }
+
+ if (!debugfs_create_file("functional_arm_dma_api", S_IRUSR, dir, ddev,
+ &iommu_debug_functional_arm_dma_api_fops)) {
+ pr_err("Couldn't create iommu/devices/%s/functional_arm_dma_api debugfs file\n",
+ name);
+ goto err_rmdir;
+ }
+
+ if (!debugfs_create_file("dma_attach", S_IRUSR, dir, ddev,
+ &iommu_debug_dma_attach_fops)) {
+ pr_err("Couldn't create iommu/devices/%s/dma_attach debugfs file\n",
+ name);
+ goto err_rmdir;
+ }
+
+ if (!debugfs_create_file("attach", S_IRUSR, dir, ddev,
+ &iommu_debug_attach_fops)) {
+ pr_err("Couldn't create iommu/devices/%s/attach debugfs file\n",
+ name);
+ goto err_rmdir;
+ }
+
+ if (!debugfs_create_file("secure_attach", S_IRUSR, dir, ddev,
+ &iommu_debug_secure_attach_fops)) {
+ pr_err("Couldn't create iommu/devices/%s/secure_attach debugfs file\n",
+ name);
+ goto err_rmdir;
+ }
+
+ if (!debugfs_create_file("atos", S_IWUSR, dir, ddev,
+ &iommu_debug_atos_fops)) {
+ pr_err("Couldn't create iommu/devices/%s/atos debugfs file\n",
+ name);
+ goto err_rmdir;
+ }
+
+ if (!debugfs_create_file("dma_atos", S_IWUSR, dir, ddev,
+ &iommu_debug_dma_atos_fops)) {
+ pr_err("Couldn't create iommu/devices/%s/dma_atos debugfs file\n",
+ name);
+ goto err_rmdir;
+ }
+
+ if (!debugfs_create_file("map", S_IWUSR, dir, ddev,
+ &iommu_debug_map_fops)) {
+ pr_err("Couldn't create iommu/devices/%s/map debugfs file\n",
+ name);
+ goto err_rmdir;
+ }
+
+ if (!debugfs_create_file("dma_map", S_IWUSR, dir, ddev,
+ &iommu_debug_dma_map_fops)) {
+ pr_err("Couldn't create iommu/devices/%s/dma_map debugfs file\n",
+ name);
+ goto err_rmdir;
+ }
+
+ if (!debugfs_create_file("unmap", S_IWUSR, dir, ddev,
+ &iommu_debug_unmap_fops)) {
+ pr_err("Couldn't create iommu/devices/%s/unmap debugfs file\n",
+ name);
+ goto err_rmdir;
+ }
+
+ if (!debugfs_create_file("dma_unmap", S_IWUSR, dir, ddev,
+ &iommu_debug_dma_unmap_fops)) {
+ pr_err("Couldn't create iommu/devices/%s/dma_unmap debugfs file\n",
+ name);
+ goto err_rmdir;
+ }
+
+ if (!debugfs_create_file("pte", S_IWUSR, dir, ddev,
+ &iommu_debug_pte_fops)) {
+ pr_err("Couldn't create iommu/devices/%s/pte debugfs file\n",
+ name);
+ goto err_rmdir;
+ }
+
+ if (!debugfs_create_file("config_clocks", S_IWUSR, dir, ddev,
+ &iommu_debug_config_clocks_fops)) {
+ pr_err("Couldn't create iommu/devices/%s/config_clocks debugfs file\n",
+ name);
+ goto err_rmdir;
+ }
+
+ list_add(&ddev->list, &iommu_debug_devices);
+ return 0;
+
+err_rmdir:
+ debugfs_remove_recursive(dir);
+err:
+ kfree(ddev);
+ return 0;
+}
+
+static int pass_iommu_devices(struct device *dev, void *ignored)
+{
+ if (!of_device_is_compatible(dev->of_node, "iommu-debug-test"))
+ return 0;
+
+ if (!of_find_property(dev->of_node, "iommus", NULL))
+ return 0;
+
+ return snarf_iommu_devices(dev, dev_name(dev));
+}
+
+static int iommu_debug_populate_devices(void)
+{
+ int ret;
+ struct device_node *np;
+ const char *cb_name;
+
+ for_each_compatible_node(np, NULL, "qcom,msm-smmu-v2-ctx") {
+ if (!of_device_is_compatible(np, "iommu-debug-test"))
+ continue;
+
+ ret = of_property_read_string(np, "label", &cb_name);
+ if (ret)
+ return ret;
+
+ ret = snarf_iommu_devices(msm_iommu_get_ctx(cb_name), cb_name);
+ if (ret)
+ return ret;
+ }
+
+ return bus_for_each_dev(&platform_bus_type, NULL, NULL,
+ pass_iommu_devices);
+}
+
+static int iommu_debug_init_tests(void)
+{
+ debugfs_tests_dir = debugfs_create_dir("tests",
+ iommu_debugfs_top);
+ if (!debugfs_tests_dir) {
+ pr_err("Couldn't create iommu/tests debugfs directory\n");
+ return -ENODEV;
+ }
+
+ virt_addr = kzalloc(SZ_1M, GFP_KERNEL);
+
+ if (!virt_addr)
+ return -ENOMEM;
+
+ return iommu_debug_populate_devices();
+}
+
+static void iommu_debug_destroy_tests(void)
+{
+ debugfs_remove_recursive(debugfs_tests_dir);
+}
+#else
+static inline int iommu_debug_init_tests(void) { return 0; }
+static inline void iommu_debug_destroy_tests(void) { }
+#endif
+
+/*
+ * This isn't really a "driver", we just need something in the device tree
+ * so that our tests can run without any client drivers, and our tests rely
+ * on parsing the device tree for nodes with the `iommus' property.
+ */
+static int iommu_debug_pass(struct platform_device *pdev)
+{
+ return 0;
+}
+
+static const struct of_device_id iommu_debug_of_match[] = {
+ { .compatible = "iommu-debug-test" },
+ { },
+};
+
+static struct platform_driver iommu_debug_driver = {
+ .probe = iommu_debug_pass,
+ .remove = iommu_debug_pass,
+ .driver = {
+ .name = "iommu-debug",
+ .of_match_table = iommu_debug_of_match,
+ },
+};
+
+static int iommu_debug_init(void)
+{
+ if (iommu_debug_init_tests())
+ return -ENODEV;
+
+ return platform_driver_register(&iommu_debug_driver);
+}
+
+static void iommu_debug_exit(void)
+{
+ platform_driver_unregister(&iommu_debug_driver);
+ iommu_debug_destroy_tests();
+}
+
+module_init(iommu_debug_init);
+module_exit(iommu_debug_exit);
diff --git a/drivers/iommu/iommu-debug.h b/drivers/iommu/iommu-debug.h
new file mode 100644
index 000000000000..91c418d9e37f
--- /dev/null
+++ b/drivers/iommu/iommu-debug.h
@@ -0,0 +1,34 @@
+/* Copyright (c) 2015-2017, The Linux Foundation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 and
+ * only version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ */
+
+#ifndef IOMMU_DEBUG_H
+#define IOMMU_DEBUG_H
+
+#ifdef CONFIG_IOMMU_DEBUG_TRACKING
+
+void iommu_debug_attach_device(struct iommu_domain *domain, struct device *dev);
+void iommu_debug_domain_remove(struct iommu_domain *domain);
+
+#else /* !CONFIG_IOMMU_DEBUG_TRACKING */
+
+static inline void iommu_debug_attach_device(struct iommu_domain *domain,
+ struct device *dev)
+{
+}
+
+static inline void iommu_debug_domain_remove(struct iommu_domain *domain)
+{
+}
+
+#endif /* CONFIG_IOMMU_DEBUG_TRACKING */
+
+#endif /* IOMMU_DEBUG_H */
diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c
index a1e7a73930fa..413c038b13ba 100644
--- a/drivers/iommu/iommu.c
+++ b/drivers/iommu/iommu.c
@@ -31,8 +31,11 @@
#include <linux/err.h>
#include <linux/pci.h>
#include <linux/bitops.h>
+#include <linux/debugfs.h>
#include <trace/events/iommu.h>
+#include "iommu-debug.h"
+
static struct kset *iommu_group_kset;
static struct ida iommu_group_ida;
static struct mutex iommu_group_mutex;
@@ -1035,6 +1038,8 @@ EXPORT_SYMBOL_GPL(bus_set_iommu);
bool iommu_present(struct bus_type *bus)
{
+ if (!bus)
+ return false;
return bus->iommu_ops != NULL;
}
EXPORT_SYMBOL_GPL(iommu_present);
@@ -1071,6 +1076,45 @@ void iommu_set_fault_handler(struct iommu_domain *domain,
}
EXPORT_SYMBOL_GPL(iommu_set_fault_handler);
+/**
+ * iommu_trigger_fault() - trigger an IOMMU fault
+ * @domain: iommu domain
+ *
+ * Triggers a fault on the device to which this domain is attached.
+ *
+ * This function should only be used for debugging purposes, for obvious
+ * reasons.
+ */
+void iommu_trigger_fault(struct iommu_domain *domain, unsigned long flags)
+{
+ if (domain->ops->trigger_fault)
+ domain->ops->trigger_fault(domain, flags);
+}
+
+/**
+ * iommu_reg_read() - read an IOMMU register
+ *
+ * Reads the IOMMU register at the given offset.
+ */
+unsigned long iommu_reg_read(struct iommu_domain *domain, unsigned long offset)
+{
+ if (domain->ops->reg_read)
+ return domain->ops->reg_read(domain, offset);
+ return 0;
+}
+
+/**
+ * iommu_reg_write() - write an IOMMU register
+ *
+ * Writes the given value to the IOMMU register at the given offset.
+ */
+void iommu_reg_write(struct iommu_domain *domain, unsigned long offset,
+ unsigned long val)
+{
+ if (domain->ops->reg_write)
+ domain->ops->reg_write(domain, offset, val);
+}
+
static struct iommu_domain *__iommu_domain_alloc(struct bus_type *bus,
unsigned type)
{
@@ -1097,6 +1141,7 @@ EXPORT_SYMBOL_GPL(iommu_domain_alloc);
void iommu_domain_free(struct iommu_domain *domain)
{
+ iommu_debug_domain_remove(domain);
domain->ops->domain_free(domain);
}
EXPORT_SYMBOL_GPL(iommu_domain_free);
@@ -1109,8 +1154,10 @@ static int __iommu_attach_device(struct iommu_domain *domain,
return -ENODEV;
ret = domain->ops->attach_dev(domain, dev);
- if (!ret)
+ if (!ret) {
trace_attach_device_to_domain(dev);
+ iommu_debug_attach_device(domain, dev);
+ }
return ret;
}
@@ -1289,8 +1336,40 @@ phys_addr_t iommu_iova_to_phys(struct iommu_domain *domain, dma_addr_t iova)
}
EXPORT_SYMBOL_GPL(iommu_iova_to_phys);
-static size_t iommu_pgsize(struct iommu_domain *domain,
- unsigned long addr_merge, size_t size)
+phys_addr_t iommu_iova_to_phys_hard(struct iommu_domain *domain,
+ dma_addr_t iova)
+{
+ if (unlikely(domain->ops->iova_to_phys_hard == NULL))
+ return 0;
+
+ return domain->ops->iova_to_phys_hard(domain, iova);
+}
+
+uint64_t iommu_iova_to_pte(struct iommu_domain *domain,
+ dma_addr_t iova)
+{
+ if (unlikely(domain->ops->iova_to_pte == NULL))
+ return 0;
+
+ return domain->ops->iova_to_pte(domain, iova);
+}
+
+bool iommu_is_iova_coherent(struct iommu_domain *domain, dma_addr_t iova)
+{
+ if (unlikely(domain->ops->is_iova_coherent == NULL))
+ return 0;
+
+ return domain->ops->is_iova_coherent(domain, iova);
+}
+static unsigned long iommu_get_pgsize_bitmap(struct iommu_domain *domain)
+{
+ if (domain->ops->get_pgsize_bitmap)
+ return domain->ops->get_pgsize_bitmap(domain);
+ return domain->ops->pgsize_bitmap;
+}
+
+size_t iommu_pgsize(unsigned long pgsize_bitmap,
+ unsigned long addr_merge, size_t size)
{
unsigned int pgsize_idx;
size_t pgsize;
@@ -1309,10 +1388,14 @@ static size_t iommu_pgsize(struct iommu_domain *domain,
pgsize = (1UL << (pgsize_idx + 1)) - 1;
/* throw away page sizes not supported by the hardware */
- pgsize &= domain->ops->pgsize_bitmap;
+ pgsize &= pgsize_bitmap;
/* make sure we're still sane */
- BUG_ON(!pgsize);
+ if (!pgsize) {
+ pr_err("invalid pgsize/addr/size! 0x%lx 0x%lx 0x%zx\n",
+ pgsize_bitmap, addr_merge, size);
+ BUG();
+ }
/* pick the biggest page */
pgsize_idx = __fls(pgsize);
@@ -1324,20 +1407,25 @@ static size_t iommu_pgsize(struct iommu_domain *domain,
int iommu_map(struct iommu_domain *domain, unsigned long iova,
phys_addr_t paddr, size_t size, int prot)
{
- unsigned long orig_iova = iova;
+ unsigned long orig_iova = iova, pgsize_bitmap;
unsigned int min_pagesz;
size_t orig_size = size;
int ret = 0;
+ trace_map_start(iova, paddr, size);
if (unlikely(domain->ops->map == NULL ||
- domain->ops->pgsize_bitmap == 0UL))
+ (domain->ops->pgsize_bitmap == 0UL &&
+ !domain->ops->get_pgsize_bitmap))) {
+ trace_map_end(iova, paddr, size);
return -ENODEV;
+ }
if (unlikely(!(domain->type & __IOMMU_DOMAIN_PAGING)))
return -EINVAL;
+ pgsize_bitmap = iommu_get_pgsize_bitmap(domain);
/* find out the minimum page size supported */
- min_pagesz = 1 << __ffs(domain->ops->pgsize_bitmap);
+ min_pagesz = 1 << __ffs(pgsize_bitmap);
/*
* both the virtual address and the physical one, as well as
@@ -1347,13 +1435,14 @@ int iommu_map(struct iommu_domain *domain, unsigned long iova,
if (!IS_ALIGNED(iova | paddr | size, min_pagesz)) {
pr_err("unaligned: iova 0x%lx pa %pa size 0x%zx min_pagesz 0x%x\n",
iova, &paddr, size, min_pagesz);
+ trace_map_end(iova, paddr, size);
return -EINVAL;
}
pr_debug("map: iova 0x%lx pa %pa size 0x%zx\n", iova, &paddr, size);
while (size) {
- size_t pgsize = iommu_pgsize(domain, iova | paddr, size);
+ size_t pgsize = iommu_pgsize(pgsize_bitmap, iova | paddr, size);
pr_debug("mapping: iova 0x%lx pa %pa pgsize 0x%zx\n",
iova, &paddr, pgsize);
@@ -1373,6 +1462,7 @@ int iommu_map(struct iommu_domain *domain, unsigned long iova,
else
trace_map(orig_iova, paddr, orig_size);
+ trace_map_end(iova, paddr, size);
return ret;
}
EXPORT_SYMBOL_GPL(iommu_map);
@@ -1383,15 +1473,21 @@ size_t iommu_unmap(struct iommu_domain *domain, unsigned long iova, size_t size)
unsigned int min_pagesz;
unsigned long orig_iova = iova;
+ trace_unmap_start(iova, 0, size);
if (unlikely(domain->ops->unmap == NULL ||
- domain->ops->pgsize_bitmap == 0UL))
+ (domain->ops->pgsize_bitmap == 0UL &&
+ !domain->ops->get_pgsize_bitmap))) {
+ trace_unmap_end(iova, 0, size);
return -ENODEV;
+ }
- if (unlikely(!(domain->type & __IOMMU_DOMAIN_PAGING)))
+ if (unlikely(!(domain->type & __IOMMU_DOMAIN_PAGING))) {
+ trace_unmap_end(iova, 0, size);
return -EINVAL;
+ }
/* find out the minimum page size supported */
- min_pagesz = 1 << __ffs(domain->ops->pgsize_bitmap);
+ min_pagesz = 1 << __ffs(iommu_get_pgsize_bitmap(domain));
/*
* The virtual address, as well as the size of the mapping, must be
@@ -1401,6 +1497,7 @@ size_t iommu_unmap(struct iommu_domain *domain, unsigned long iova, size_t size)
if (!IS_ALIGNED(iova | size, min_pagesz)) {
pr_err("unaligned: iova 0x%lx size 0x%zx min_pagesz 0x%x\n",
iova, size, min_pagesz);
+ trace_unmap_end(iova, 0, size);
return -EINVAL;
}
@@ -1411,9 +1508,9 @@ size_t iommu_unmap(struct iommu_domain *domain, unsigned long iova, size_t size)
* or we hit an area that isn't mapped.
*/
while (unmapped < size) {
- size_t pgsize = iommu_pgsize(domain, iova, size - unmapped);
+ size_t left = size - unmapped;
- unmapped_page = domain->ops->unmap(domain, iova, pgsize);
+ unmapped_page = domain->ops->unmap(domain, iova, left);
if (!unmapped_page)
break;
@@ -1425,6 +1522,7 @@ size_t iommu_unmap(struct iommu_domain *domain, unsigned long iova, size_t size)
}
trace_unmap(orig_iova, size, unmapped);
+ trace_unmap_end(orig_iova, 0, size);
return unmapped;
}
EXPORT_SYMBOL_GPL(iommu_unmap);
@@ -1436,11 +1534,14 @@ size_t default_iommu_map_sg(struct iommu_domain *domain, unsigned long iova,
size_t mapped = 0;
unsigned int i, min_pagesz;
int ret;
+ unsigned long pgsize_bitmap;
- if (unlikely(domain->ops->pgsize_bitmap == 0UL))
+ if (unlikely(domain->ops->pgsize_bitmap == 0UL &&
+ !domain->ops->get_pgsize_bitmap))
return 0;
- min_pagesz = 1 << __ffs(domain->ops->pgsize_bitmap);
+ pgsize_bitmap = iommu_get_pgsize_bitmap(domain);
+ min_pagesz = 1 << __ffs(pgsize_bitmap);
for_each_sg(sg, s, nents, i) {
phys_addr_t phys = page_to_phys(sg_page(s)) + s->offset;
@@ -1472,6 +1573,20 @@ out_err:
}
EXPORT_SYMBOL_GPL(default_iommu_map_sg);
+/* DEPRECATED */
+int iommu_map_range(struct iommu_domain *domain, unsigned int iova,
+ struct scatterlist *sg, unsigned int len, int opt)
+{
+ return -ENODEV;
+}
+
+/* DEPRECATED */
+int iommu_unmap_range(struct iommu_domain *domain, unsigned int iova,
+ unsigned int len)
+{
+ return -ENODEV;
+}
+
int iommu_domain_window_enable(struct iommu_domain *domain, u32 wnd_nr,
phys_addr_t paddr, u64 size, int prot)
{
@@ -1492,6 +1607,8 @@ void iommu_domain_window_disable(struct iommu_domain *domain, u32 wnd_nr)
}
EXPORT_SYMBOL_GPL(iommu_domain_window_disable);
+struct dentry *iommu_debugfs_top;
+
static int __init iommu_init(void)
{
iommu_group_kset = kset_create_and_add("iommu_groups",
@@ -1501,6 +1618,12 @@ static int __init iommu_init(void)
BUG_ON(!iommu_group_kset);
+ iommu_debugfs_top = debugfs_create_dir("iommu", NULL);
+ if (!iommu_debugfs_top) {
+ pr_err("Couldn't create iommu debugfs directory\n");
+ return -ENODEV;
+ }
+
return 0;
}
core_initcall(iommu_init);
@@ -1521,7 +1644,7 @@ int iommu_domain_get_attr(struct iommu_domain *domain,
break;
case DOMAIN_ATTR_PAGING:
paging = data;
- *paging = (domain->ops->pgsize_bitmap != 0UL);
+ *paging = (iommu_get_pgsize_bitmap(domain) != 0UL);
break;
case DOMAIN_ATTR_WINDOWS:
count = data;
@@ -1570,6 +1693,14 @@ int iommu_domain_set_attr(struct iommu_domain *domain,
}
EXPORT_SYMBOL_GPL(iommu_domain_set_attr);
+int iommu_dma_supported(struct iommu_domain *domain, struct device *dev,
+ u64 mask)
+{
+ if (domain->ops->dma_supported)
+ return domain->ops->dma_supported(domain, dev, mask);
+ return 0;
+}
+
void iommu_get_dm_regions(struct device *dev, struct list_head *list)
{
const struct iommu_ops *ops = dev->bus->iommu_ops;
diff --git a/drivers/iommu/msm_dma_iommu_mapping.c b/drivers/iommu/msm_dma_iommu_mapping.c
new file mode 100644
index 000000000000..25fe36ab6339
--- /dev/null
+++ b/drivers/iommu/msm_dma_iommu_mapping.c
@@ -0,0 +1,423 @@
+/* Copyright (c) 2015-2016, The Linux Foundation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 and
+ * only version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ */
+
+#include <linux/kernel.h>
+#include <linux/kref.h>
+#include <linux/slab.h>
+#include <linux/rbtree.h>
+#include <linux/mutex.h>
+#include <linux/err.h>
+#include <asm/barrier.h>
+
+#include <linux/msm_dma_iommu_mapping.h>
+
+/**
+ * struct msm_iommu_map - represents a mapping of an ion buffer to an iommu
+ * @lnode - list node to exist in the buffer's list of iommu mappings
+ * @dev - Device this is mapped to. Used as key
+ * @sgl - The scatterlist for this mapping
+ * @nents - Number of entries in sgl
+ * @dir - The direction for the unmap.
+ * @meta - Backpointer to the meta this guy belongs to.
+ * @ref - for reference counting this mapping
+ *
+ * Represents a mapping of one dma_buf buffer to a particular device
+ * and address range. There may exist other mappings of this buffer in
+ * different devices. All mappings will have the same cacheability and security.
+ */
+struct msm_iommu_map {
+ struct list_head lnode;
+ struct rb_node node;
+ struct device *dev;
+ struct scatterlist sgl;
+ unsigned int nents;
+ enum dma_data_direction dir;
+ struct msm_iommu_meta *meta;
+ struct kref ref;
+};
+
+struct msm_iommu_meta {
+ struct rb_node node;
+ struct list_head iommu_maps;
+ struct kref ref;
+ struct mutex lock;
+ void *buffer;
+};
+
+static struct rb_root iommu_root;
+static DEFINE_MUTEX(msm_iommu_map_mutex);
+
+static void msm_iommu_meta_add(struct msm_iommu_meta *meta)
+{
+ struct rb_root *root = &iommu_root;
+ struct rb_node **p = &root->rb_node;
+ struct rb_node *parent = NULL;
+ struct msm_iommu_meta *entry;
+
+ while (*p) {
+ parent = *p;
+ entry = rb_entry(parent, struct msm_iommu_meta, node);
+
+ if (meta->buffer < entry->buffer) {
+ p = &(*p)->rb_left;
+ } else if (meta->buffer > entry->buffer) {
+ p = &(*p)->rb_right;
+ } else {
+ pr_err("%s: dma_buf %p already exists\n", __func__,
+ entry->buffer);
+ BUG();
+ }
+ }
+
+ rb_link_node(&meta->node, parent, p);
+ rb_insert_color(&meta->node, root);
+}
+
+static struct msm_iommu_meta *msm_iommu_meta_lookup(void *buffer)
+{
+ struct rb_root *root = &iommu_root;
+ struct rb_node **p = &root->rb_node;
+ struct rb_node *parent = NULL;
+ struct msm_iommu_meta *entry = NULL;
+
+ while (*p) {
+ parent = *p;
+ entry = rb_entry(parent, struct msm_iommu_meta, node);
+
+ if (buffer < entry->buffer)
+ p = &(*p)->rb_left;
+ else if (buffer > entry->buffer)
+ p = &(*p)->rb_right;
+ else
+ return entry;
+ }
+
+ return NULL;
+}
+
+static void msm_iommu_add(struct msm_iommu_meta *meta,
+ struct msm_iommu_map *iommu)
+{
+ struct msm_iommu_map *entry;
+
+ list_for_each_entry(entry, &meta->iommu_maps, lnode) {
+ if (entry->dev == iommu->dev) {
+ pr_err("%s: dma_buf %p already has mapping to device %p\n",
+ __func__, meta->buffer, iommu->dev);
+ BUG();
+ }
+ }
+ INIT_LIST_HEAD(&iommu->lnode);
+ list_add(&iommu->lnode, &meta->iommu_maps);
+}
+
+
+static struct msm_iommu_map *msm_iommu_lookup(struct msm_iommu_meta *meta,
+ struct device *dev)
+{
+ struct msm_iommu_map *entry;
+
+ list_for_each_entry(entry, &meta->iommu_maps, lnode) {
+ if (entry->dev == dev)
+ return entry;
+ }
+
+ return NULL;
+}
+
+static struct msm_iommu_meta *msm_iommu_meta_create(struct dma_buf *dma_buf)
+{
+ struct msm_iommu_meta *meta;
+
+ meta = kzalloc(sizeof(*meta), GFP_KERNEL);
+
+ if (!meta)
+ return ERR_PTR(-ENOMEM);
+
+ INIT_LIST_HEAD(&meta->iommu_maps);
+ meta->buffer = dma_buf->priv;
+ kref_init(&meta->ref);
+ mutex_init(&meta->lock);
+ msm_iommu_meta_add(meta);
+
+ return meta;
+}
+
+static void msm_iommu_meta_put(struct msm_iommu_meta *meta);
+
+static inline int __msm_dma_map_sg(struct device *dev, struct scatterlist *sg,
+ int nents, enum dma_data_direction dir,
+ struct dma_buf *dma_buf,
+ struct dma_attrs *attrs)
+{
+ struct msm_iommu_map *iommu_map;
+ struct msm_iommu_meta *iommu_meta = NULL;
+ int ret = 0;
+ bool extra_meta_ref_taken = false;
+ int late_unmap = !dma_get_attr(DMA_ATTR_NO_DELAYED_UNMAP, attrs);
+
+ mutex_lock(&msm_iommu_map_mutex);
+ iommu_meta = msm_iommu_meta_lookup(dma_buf->priv);
+
+ if (!iommu_meta) {
+ iommu_meta = msm_iommu_meta_create(dma_buf);
+
+ if (IS_ERR(iommu_meta)) {
+ mutex_unlock(&msm_iommu_map_mutex);
+ ret = PTR_ERR(iommu_meta);
+ goto out;
+ }
+ if (late_unmap) {
+ kref_get(&iommu_meta->ref);
+ extra_meta_ref_taken = true;
+ }
+ } else {
+ kref_get(&iommu_meta->ref);
+ }
+
+ mutex_unlock(&msm_iommu_map_mutex);
+
+ mutex_lock(&iommu_meta->lock);
+ iommu_map = msm_iommu_lookup(iommu_meta, dev);
+ if (!iommu_map) {
+ iommu_map = kmalloc(sizeof(*iommu_map), GFP_ATOMIC);
+
+ if (!iommu_map) {
+ ret = -ENOMEM;
+ goto out_unlock;
+ }
+
+ ret = dma_map_sg_attrs(dev, sg, nents, dir, attrs);
+ if (ret != nents) {
+ kfree(iommu_map);
+ goto out_unlock;
+ }
+
+ kref_init(&iommu_map->ref);
+ if (late_unmap)
+ kref_get(&iommu_map->ref);
+ iommu_map->meta = iommu_meta;
+ iommu_map->sgl.dma_address = sg->dma_address;
+ iommu_map->sgl.dma_length = sg->dma_length;
+ iommu_map->dev = dev;
+ msm_iommu_add(iommu_meta, iommu_map);
+
+ } else {
+ sg->dma_address = iommu_map->sgl.dma_address;
+ sg->dma_length = iommu_map->sgl.dma_length;
+
+ kref_get(&iommu_map->ref);
+ if (is_device_dma_coherent(dev))
+ /*
+ * Ensure all outstanding changes for coherent
+ * buffers are applied to the cache before any
+ * DMA occurs.
+ */
+ dmb(ish);
+ ret = nents;
+ }
+ mutex_unlock(&iommu_meta->lock);
+ return ret;
+
+out_unlock:
+ mutex_unlock(&iommu_meta->lock);
+out:
+ if (!IS_ERR(iommu_meta)) {
+ if (extra_meta_ref_taken)
+ msm_iommu_meta_put(iommu_meta);
+ msm_iommu_meta_put(iommu_meta);
+ }
+ return ret;
+
+}
+
+/*
+ * We are not taking a reference to the dma_buf here. It is expected that
+ * clients hold reference to the dma_buf until they are done with mapping and
+ * unmapping.
+ */
+int msm_dma_map_sg_attrs(struct device *dev, struct scatterlist *sg, int nents,
+ enum dma_data_direction dir, struct dma_buf *dma_buf,
+ struct dma_attrs *attrs)
+{
+ int ret;
+
+ if (IS_ERR_OR_NULL(dev)) {
+ pr_err("%s: dev pointer is invalid\n", __func__);
+ return -EINVAL;
+ }
+
+ if (IS_ERR_OR_NULL(sg)) {
+ pr_err("%s: sg table pointer is invalid\n", __func__);
+ return -EINVAL;
+ }
+
+ if (IS_ERR_OR_NULL(dma_buf)) {
+ pr_err("%s: dma_buf pointer is invalid\n", __func__);
+ return -EINVAL;
+ }
+
+ ret = __msm_dma_map_sg(dev, sg, nents, dir, dma_buf, attrs);
+
+ return ret;
+}
+EXPORT_SYMBOL(msm_dma_map_sg_attrs);
+
+static void msm_iommu_meta_destroy(struct kref *kref)
+{
+ struct msm_iommu_meta *meta = container_of(kref, struct msm_iommu_meta,
+ ref);
+
+ if (!list_empty(&meta->iommu_maps)) {
+ WARN(1, "%s: DMA Buffer %p being destroyed with outstanding iommu mappins!\n", __func__,
+ meta->buffer);
+ }
+ rb_erase(&meta->node, &iommu_root);
+ kfree(meta);
+}
+
+static void msm_iommu_meta_put(struct msm_iommu_meta *meta)
+{
+ /*
+ * Need to lock here to prevent race against map/unmap
+ */
+ mutex_lock(&msm_iommu_map_mutex);
+ kref_put(&meta->ref, msm_iommu_meta_destroy);
+ mutex_unlock(&msm_iommu_map_mutex);
+}
+
+static void msm_iommu_map_release(struct kref *kref)
+{
+ struct msm_iommu_map *map = container_of(kref, struct msm_iommu_map,
+ ref);
+
+ list_del(&map->lnode);
+ dma_unmap_sg(map->dev, &map->sgl, map->nents, map->dir);
+ kfree(map);
+}
+
+void msm_dma_unmap_sg(struct device *dev, struct scatterlist *sgl, int nents,
+ enum dma_data_direction dir, struct dma_buf *dma_buf)
+{
+ struct msm_iommu_map *iommu_map;
+ struct msm_iommu_meta *meta;
+
+ mutex_lock(&msm_iommu_map_mutex);
+ meta = msm_iommu_meta_lookup(dma_buf->priv);
+ if (!meta) {
+ WARN(1, "%s: (%p) was never mapped\n", __func__, dma_buf);
+ mutex_unlock(&msm_iommu_map_mutex);
+ goto out;
+
+ }
+ mutex_unlock(&msm_iommu_map_mutex);
+
+ mutex_lock(&meta->lock);
+ iommu_map = msm_iommu_lookup(meta, dev);
+
+ if (!iommu_map) {
+ WARN(1, "%s: (%p) was never mapped for device %p\n", __func__,
+ dma_buf, dev);
+ mutex_unlock(&meta->lock);
+ goto out;
+ }
+
+ /*
+ * Save direction for later use when we actually unmap.
+ * Not used right now but in the future if we go to coherent mapping
+ * API we might want to call the appropriate API when client asks
+ * to unmap
+ */
+ iommu_map->dir = dir;
+
+ kref_put(&iommu_map->ref, msm_iommu_map_release);
+ mutex_unlock(&meta->lock);
+
+ msm_iommu_meta_put(meta);
+
+out:
+ return;
+}
+EXPORT_SYMBOL(msm_dma_unmap_sg);
+
+int msm_dma_unmap_all_for_dev(struct device *dev)
+{
+ int ret = 0;
+ struct msm_iommu_meta *meta;
+ struct rb_root *root;
+ struct rb_node *meta_node;
+
+ mutex_lock(&msm_iommu_map_mutex);
+ root = &iommu_root;
+ meta_node = rb_first(root);
+ while (meta_node) {
+ struct msm_iommu_map *iommu_map;
+ struct msm_iommu_map *iommu_map_next;
+
+ meta = rb_entry(meta_node, struct msm_iommu_meta, node);
+ mutex_lock(&meta->lock);
+ list_for_each_entry_safe(iommu_map, iommu_map_next,
+ &meta->iommu_maps, lnode)
+ if (iommu_map->dev == dev)
+ if (!kref_put(&iommu_map->ref,
+ msm_iommu_map_release))
+ ret = -EINVAL;
+
+ mutex_unlock(&meta->lock);
+ meta_node = rb_next(meta_node);
+ }
+ mutex_unlock(&msm_iommu_map_mutex);
+
+ return ret;
+}
+EXPORT_SYMBOL(msm_dma_unmap_all_for_dev);
+
+/*
+ * Only to be called by ION code when a buffer is freed
+ */
+void msm_dma_buf_freed(void *buffer)
+{
+ struct msm_iommu_map *iommu_map;
+ struct msm_iommu_map *iommu_map_next;
+ struct msm_iommu_meta *meta;
+
+ mutex_lock(&msm_iommu_map_mutex);
+ meta = msm_iommu_meta_lookup(buffer);
+ if (!meta) {
+ /* Already unmapped (assuming no late unmapping) */
+ mutex_unlock(&msm_iommu_map_mutex);
+ goto out;
+
+ }
+ mutex_unlock(&msm_iommu_map_mutex);
+
+ mutex_lock(&meta->lock);
+
+ list_for_each_entry_safe(iommu_map, iommu_map_next, &meta->iommu_maps,
+ lnode)
+ kref_put(&iommu_map->ref, msm_iommu_map_release);
+
+ if (!list_empty(&meta->iommu_maps)) {
+ WARN(1, "%s: DMA Buffer %p being destroyed with outstanding iommu mappins!\n", __func__,
+ meta->buffer);
+ }
+
+ INIT_LIST_HEAD(&meta->iommu_maps);
+ mutex_unlock(&meta->lock);
+
+ msm_iommu_meta_put(meta);
+out:
+ return;
+
+}
+
diff --git a/drivers/iommu/msm_iommu.c b/drivers/iommu/msm_iommu.c
deleted file mode 100644
index e321fa517a45..000000000000
--- a/drivers/iommu/msm_iommu.c
+++ /dev/null
@@ -1,735 +0,0 @@
-/* Copyright (c) 2010-2011, Code Aurora Forum. All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 and
- * only version 2 as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
- * 02110-1301, USA.
- */
-
-#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
-#include <linux/kernel.h>
-#include <linux/module.h>
-#include <linux/platform_device.h>
-#include <linux/errno.h>
-#include <linux/io.h>
-#include <linux/interrupt.h>
-#include <linux/list.h>
-#include <linux/spinlock.h>
-#include <linux/slab.h>
-#include <linux/iommu.h>
-#include <linux/clk.h>
-
-#include <asm/cacheflush.h>
-#include <asm/sizes.h>
-
-#include "msm_iommu_hw-8xxx.h"
-#include "msm_iommu.h"
-
-#define MRC(reg, processor, op1, crn, crm, op2) \
-__asm__ __volatile__ ( \
-" mrc " #processor "," #op1 ", %0," #crn "," #crm "," #op2 "\n" \
-: "=r" (reg))
-
-#define RCP15_PRRR(reg) MRC(reg, p15, 0, c10, c2, 0)
-#define RCP15_NMRR(reg) MRC(reg, p15, 0, c10, c2, 1)
-
-/* bitmap of the page sizes currently supported */
-#define MSM_IOMMU_PGSIZES (SZ_4K | SZ_64K | SZ_1M | SZ_16M)
-
-static int msm_iommu_tex_class[4];
-
-DEFINE_SPINLOCK(msm_iommu_lock);
-
-struct msm_priv {
- unsigned long *pgtable;
- struct list_head list_attached;
- struct iommu_domain domain;
-};
-
-static struct msm_priv *to_msm_priv(struct iommu_domain *dom)
-{
- return container_of(dom, struct msm_priv, domain);
-}
-
-static int __enable_clocks(struct msm_iommu_drvdata *drvdata)
-{
- int ret;
-
- ret = clk_enable(drvdata->pclk);
- if (ret)
- goto fail;
-
- if (drvdata->clk) {
- ret = clk_enable(drvdata->clk);
- if (ret)
- clk_disable(drvdata->pclk);
- }
-fail:
- return ret;
-}
-
-static void __disable_clocks(struct msm_iommu_drvdata *drvdata)
-{
- clk_disable(drvdata->clk);
- clk_disable(drvdata->pclk);
-}
-
-static int __flush_iotlb(struct iommu_domain *domain)
-{
- struct msm_priv *priv = to_msm_priv(domain);
- struct msm_iommu_drvdata *iommu_drvdata;
- struct msm_iommu_ctx_drvdata *ctx_drvdata;
- int ret = 0;
-#ifndef CONFIG_IOMMU_PGTABLES_L2
- unsigned long *fl_table = priv->pgtable;
- int i;
-
- if (!list_empty(&priv->list_attached)) {
- dmac_flush_range(fl_table, fl_table + SZ_16K);
-
- for (i = 0; i < NUM_FL_PTE; i++)
- if ((fl_table[i] & 0x03) == FL_TYPE_TABLE) {
- void *sl_table = __va(fl_table[i] &
- FL_BASE_MASK);
- dmac_flush_range(sl_table, sl_table + SZ_4K);
- }
- }
-#endif
-
- list_for_each_entry(ctx_drvdata, &priv->list_attached, attached_elm) {
-
- BUG_ON(!ctx_drvdata->pdev || !ctx_drvdata->pdev->dev.parent);
-
- iommu_drvdata = dev_get_drvdata(ctx_drvdata->pdev->dev.parent);
- BUG_ON(!iommu_drvdata);
-
- ret = __enable_clocks(iommu_drvdata);
- if (ret)
- goto fail;
-
- SET_CTX_TLBIALL(iommu_drvdata->base, ctx_drvdata->num, 0);
- __disable_clocks(iommu_drvdata);
- }
-fail:
- return ret;
-}
-
-static void __reset_context(void __iomem *base, int ctx)
-{
- SET_BPRCOSH(base, ctx, 0);
- SET_BPRCISH(base, ctx, 0);
- SET_BPRCNSH(base, ctx, 0);
- SET_BPSHCFG(base, ctx, 0);
- SET_BPMTCFG(base, ctx, 0);
- SET_ACTLR(base, ctx, 0);
- SET_SCTLR(base, ctx, 0);
- SET_FSRRESTORE(base, ctx, 0);
- SET_TTBR0(base, ctx, 0);
- SET_TTBR1(base, ctx, 0);
- SET_TTBCR(base, ctx, 0);
- SET_BFBCR(base, ctx, 0);
- SET_PAR(base, ctx, 0);
- SET_FAR(base, ctx, 0);
- SET_CTX_TLBIALL(base, ctx, 0);
- SET_TLBFLPTER(base, ctx, 0);
- SET_TLBSLPTER(base, ctx, 0);
- SET_TLBLKCR(base, ctx, 0);
- SET_PRRR(base, ctx, 0);
- SET_NMRR(base, ctx, 0);
-}
-
-static void __program_context(void __iomem *base, int ctx, phys_addr_t pgtable)
-{
- unsigned int prrr, nmrr;
- __reset_context(base, ctx);
-
- /* Set up HTW mode */
- /* TLB miss configuration: perform HTW on miss */
- SET_TLBMCFG(base, ctx, 0x3);
-
- /* V2P configuration: HTW for access */
- SET_V2PCFG(base, ctx, 0x3);
-
- SET_TTBCR(base, ctx, 0);
- SET_TTBR0_PA(base, ctx, (pgtable >> 14));
-
- /* Invalidate the TLB for this context */
- SET_CTX_TLBIALL(base, ctx, 0);
-
- /* Set interrupt number to "secure" interrupt */
- SET_IRPTNDX(base, ctx, 0);
-
- /* Enable context fault interrupt */
- SET_CFEIE(base, ctx, 1);
-
- /* Stall access on a context fault and let the handler deal with it */
- SET_CFCFG(base, ctx, 1);
-
- /* Redirect all cacheable requests to L2 slave port. */
- SET_RCISH(base, ctx, 1);
- SET_RCOSH(base, ctx, 1);
- SET_RCNSH(base, ctx, 1);
-
- /* Turn on TEX Remap */
- SET_TRE(base, ctx, 1);
-
- /* Set TEX remap attributes */
- RCP15_PRRR(prrr);
- RCP15_NMRR(nmrr);
- SET_PRRR(base, ctx, prrr);
- SET_NMRR(base, ctx, nmrr);
-
- /* Turn on BFB prefetch */
- SET_BFBDFE(base, ctx, 1);
-
-#ifdef CONFIG_IOMMU_PGTABLES_L2
- /* Configure page tables as inner-cacheable and shareable to reduce
- * the TLB miss penalty.
- */
- SET_TTBR0_SH(base, ctx, 1);
- SET_TTBR1_SH(base, ctx, 1);
-
- SET_TTBR0_NOS(base, ctx, 1);
- SET_TTBR1_NOS(base, ctx, 1);
-
- SET_TTBR0_IRGNH(base, ctx, 0); /* WB, WA */
- SET_TTBR0_IRGNL(base, ctx, 1);
-
- SET_TTBR1_IRGNH(base, ctx, 0); /* WB, WA */
- SET_TTBR1_IRGNL(base, ctx, 1);
-
- SET_TTBR0_ORGN(base, ctx, 1); /* WB, WA */
- SET_TTBR1_ORGN(base, ctx, 1); /* WB, WA */
-#endif
-
- /* Enable the MMU */
- SET_M(base, ctx, 1);
-}
-
-static struct iommu_domain *msm_iommu_domain_alloc(unsigned type)
-{
- struct msm_priv *priv;
-
- if (type != IOMMU_DOMAIN_UNMANAGED)
- return NULL;
-
- priv = kzalloc(sizeof(*priv), GFP_KERNEL);
- if (!priv)
- goto fail_nomem;
-
- INIT_LIST_HEAD(&priv->list_attached);
- priv->pgtable = (unsigned long *)__get_free_pages(GFP_KERNEL,
- get_order(SZ_16K));
-
- if (!priv->pgtable)
- goto fail_nomem;
-
- memset(priv->pgtable, 0, SZ_16K);
-
- priv->domain.geometry.aperture_start = 0;
- priv->domain.geometry.aperture_end = (1ULL << 32) - 1;
- priv->domain.geometry.force_aperture = true;
-
- return &priv->domain;
-
-fail_nomem:
- kfree(priv);
- return NULL;
-}
-
-static void msm_iommu_domain_free(struct iommu_domain *domain)
-{
- struct msm_priv *priv;
- unsigned long flags;
- unsigned long *fl_table;
- int i;
-
- spin_lock_irqsave(&msm_iommu_lock, flags);
- priv = to_msm_priv(domain);
-
- fl_table = priv->pgtable;
-
- for (i = 0; i < NUM_FL_PTE; i++)
- if ((fl_table[i] & 0x03) == FL_TYPE_TABLE)
- free_page((unsigned long) __va(((fl_table[i]) &
- FL_BASE_MASK)));
-
- free_pages((unsigned long)priv->pgtable, get_order(SZ_16K));
- priv->pgtable = NULL;
-
- kfree(priv);
- spin_unlock_irqrestore(&msm_iommu_lock, flags);
-}
-
-static int msm_iommu_attach_dev(struct iommu_domain *domain, struct device *dev)
-{
- struct msm_priv *priv;
- struct msm_iommu_ctx_dev *ctx_dev;
- struct msm_iommu_drvdata *iommu_drvdata;
- struct msm_iommu_ctx_drvdata *ctx_drvdata;
- struct msm_iommu_ctx_drvdata *tmp_drvdata;
- int ret = 0;
- unsigned long flags;
-
- spin_lock_irqsave(&msm_iommu_lock, flags);
-
- priv = to_msm_priv(domain);
-
- if (!dev) {
- ret = -EINVAL;
- goto fail;
- }
-
- iommu_drvdata = dev_get_drvdata(dev->parent);
- ctx_drvdata = dev_get_drvdata(dev);
- ctx_dev = dev->platform_data;
-
- if (!iommu_drvdata || !ctx_drvdata || !ctx_dev) {
- ret = -EINVAL;
- goto fail;
- }
-
- if (!list_empty(&ctx_drvdata->attached_elm)) {
- ret = -EBUSY;
- goto fail;
- }
-
- list_for_each_entry(tmp_drvdata, &priv->list_attached, attached_elm)
- if (tmp_drvdata == ctx_drvdata) {
- ret = -EBUSY;
- goto fail;
- }
-
- ret = __enable_clocks(iommu_drvdata);
- if (ret)
- goto fail;
-
- __program_context(iommu_drvdata->base, ctx_dev->num,
- __pa(priv->pgtable));
-
- __disable_clocks(iommu_drvdata);
- list_add(&(ctx_drvdata->attached_elm), &priv->list_attached);
- ret = __flush_iotlb(domain);
-
-fail:
- spin_unlock_irqrestore(&msm_iommu_lock, flags);
- return ret;
-}
-
-static void msm_iommu_detach_dev(struct iommu_domain *domain,
- struct device *dev)
-{
- struct msm_priv *priv;
- struct msm_iommu_ctx_dev *ctx_dev;
- struct msm_iommu_drvdata *iommu_drvdata;
- struct msm_iommu_ctx_drvdata *ctx_drvdata;
- unsigned long flags;
- int ret;
-
- spin_lock_irqsave(&msm_iommu_lock, flags);
- priv = to_msm_priv(domain);
-
- if (!dev)
- goto fail;
-
- iommu_drvdata = dev_get_drvdata(dev->parent);
- ctx_drvdata = dev_get_drvdata(dev);
- ctx_dev = dev->platform_data;
-
- if (!iommu_drvdata || !ctx_drvdata || !ctx_dev)
- goto fail;
-
- ret = __flush_iotlb(domain);
- if (ret)
- goto fail;
-
- ret = __enable_clocks(iommu_drvdata);
- if (ret)
- goto fail;
-
- __reset_context(iommu_drvdata->base, ctx_dev->num);
- __disable_clocks(iommu_drvdata);
- list_del_init(&ctx_drvdata->attached_elm);
-
-fail:
- spin_unlock_irqrestore(&msm_iommu_lock, flags);
-}
-
-static int msm_iommu_map(struct iommu_domain *domain, unsigned long va,
- phys_addr_t pa, size_t len, int prot)
-{
- struct msm_priv *priv;
- unsigned long flags;
- unsigned long *fl_table;
- unsigned long *fl_pte;
- unsigned long fl_offset;
- unsigned long *sl_table;
- unsigned long *sl_pte;
- unsigned long sl_offset;
- unsigned int pgprot;
- int ret = 0, tex, sh;
-
- spin_lock_irqsave(&msm_iommu_lock, flags);
-
- sh = (prot & MSM_IOMMU_ATTR_SH) ? 1 : 0;
- tex = msm_iommu_tex_class[prot & MSM_IOMMU_CP_MASK];
-
- if (tex < 0 || tex > NUM_TEX_CLASS - 1) {
- ret = -EINVAL;
- goto fail;
- }
-
- priv = to_msm_priv(domain);
-
- fl_table = priv->pgtable;
-
- if (len != SZ_16M && len != SZ_1M &&
- len != SZ_64K && len != SZ_4K) {
- pr_debug("Bad size: %d\n", len);
- ret = -EINVAL;
- goto fail;
- }
-
- if (!fl_table) {
- pr_debug("Null page table\n");
- ret = -EINVAL;
- goto fail;
- }
-
- if (len == SZ_16M || len == SZ_1M) {
- pgprot = sh ? FL_SHARED : 0;
- pgprot |= tex & 0x01 ? FL_BUFFERABLE : 0;
- pgprot |= tex & 0x02 ? FL_CACHEABLE : 0;
- pgprot |= tex & 0x04 ? FL_TEX0 : 0;
- } else {
- pgprot = sh ? SL_SHARED : 0;
- pgprot |= tex & 0x01 ? SL_BUFFERABLE : 0;
- pgprot |= tex & 0x02 ? SL_CACHEABLE : 0;
- pgprot |= tex & 0x04 ? SL_TEX0 : 0;
- }
-
- fl_offset = FL_OFFSET(va); /* Upper 12 bits */
- fl_pte = fl_table + fl_offset; /* int pointers, 4 bytes */
-
- if (len == SZ_16M) {
- int i = 0;
- for (i = 0; i < 16; i++)
- *(fl_pte+i) = (pa & 0xFF000000) | FL_SUPERSECTION |
- FL_AP_READ | FL_AP_WRITE | FL_TYPE_SECT |
- FL_SHARED | FL_NG | pgprot;
- }
-
- if (len == SZ_1M)
- *fl_pte = (pa & 0xFFF00000) | FL_AP_READ | FL_AP_WRITE | FL_NG |
- FL_TYPE_SECT | FL_SHARED | pgprot;
-
- /* Need a 2nd level table */
- if ((len == SZ_4K || len == SZ_64K) && (*fl_pte) == 0) {
- unsigned long *sl;
- sl = (unsigned long *) __get_free_pages(GFP_ATOMIC,
- get_order(SZ_4K));
-
- if (!sl) {
- pr_debug("Could not allocate second level table\n");
- ret = -ENOMEM;
- goto fail;
- }
-
- memset(sl, 0, SZ_4K);
- *fl_pte = ((((int)__pa(sl)) & FL_BASE_MASK) | FL_TYPE_TABLE);
- }
-
- sl_table = (unsigned long *) __va(((*fl_pte) & FL_BASE_MASK));
- sl_offset = SL_OFFSET(va);
- sl_pte = sl_table + sl_offset;
-
-
- if (len == SZ_4K)
- *sl_pte = (pa & SL_BASE_MASK_SMALL) | SL_AP0 | SL_AP1 | SL_NG |
- SL_SHARED | SL_TYPE_SMALL | pgprot;
-
- if (len == SZ_64K) {
- int i;
-
- for (i = 0; i < 16; i++)
- *(sl_pte+i) = (pa & SL_BASE_MASK_LARGE) | SL_AP0 |
- SL_NG | SL_AP1 | SL_SHARED | SL_TYPE_LARGE | pgprot;
- }
-
- ret = __flush_iotlb(domain);
-fail:
- spin_unlock_irqrestore(&msm_iommu_lock, flags);
- return ret;
-}
-
-static size_t msm_iommu_unmap(struct iommu_domain *domain, unsigned long va,
- size_t len)
-{
- struct msm_priv *priv;
- unsigned long flags;
- unsigned long *fl_table;
- unsigned long *fl_pte;
- unsigned long fl_offset;
- unsigned long *sl_table;
- unsigned long *sl_pte;
- unsigned long sl_offset;
- int i, ret = 0;
-
- spin_lock_irqsave(&msm_iommu_lock, flags);
-
- priv = to_msm_priv(domain);
-
- fl_table = priv->pgtable;
-
- if (len != SZ_16M && len != SZ_1M &&
- len != SZ_64K && len != SZ_4K) {
- pr_debug("Bad length: %d\n", len);
- goto fail;
- }
-
- if (!fl_table) {
- pr_debug("Null page table\n");
- goto fail;
- }
-
- fl_offset = FL_OFFSET(va); /* Upper 12 bits */
- fl_pte = fl_table + fl_offset; /* int pointers, 4 bytes */
-
- if (*fl_pte == 0) {
- pr_debug("First level PTE is 0\n");
- goto fail;
- }
-
- /* Unmap supersection */
- if (len == SZ_16M)
- for (i = 0; i < 16; i++)
- *(fl_pte+i) = 0;
-
- if (len == SZ_1M)
- *fl_pte = 0;
-
- sl_table = (unsigned long *) __va(((*fl_pte) & FL_BASE_MASK));
- sl_offset = SL_OFFSET(va);
- sl_pte = sl_table + sl_offset;
-
- if (len == SZ_64K) {
- for (i = 0; i < 16; i++)
- *(sl_pte+i) = 0;
- }
-
- if (len == SZ_4K)
- *sl_pte = 0;
-
- if (len == SZ_4K || len == SZ_64K) {
- int used = 0;
-
- for (i = 0; i < NUM_SL_PTE; i++)
- if (sl_table[i])
- used = 1;
- if (!used) {
- free_page((unsigned long)sl_table);
- *fl_pte = 0;
- }
- }
-
- ret = __flush_iotlb(domain);
-
-fail:
- spin_unlock_irqrestore(&msm_iommu_lock, flags);
-
- /* the IOMMU API requires us to return how many bytes were unmapped */
- len = ret ? 0 : len;
- return len;
-}
-
-static phys_addr_t msm_iommu_iova_to_phys(struct iommu_domain *domain,
- dma_addr_t va)
-{
- struct msm_priv *priv;
- struct msm_iommu_drvdata *iommu_drvdata;
- struct msm_iommu_ctx_drvdata *ctx_drvdata;
- unsigned int par;
- unsigned long flags;
- void __iomem *base;
- phys_addr_t ret = 0;
- int ctx;
-
- spin_lock_irqsave(&msm_iommu_lock, flags);
-
- priv = to_msm_priv(domain);
- if (list_empty(&priv->list_attached))
- goto fail;
-
- ctx_drvdata = list_entry(priv->list_attached.next,
- struct msm_iommu_ctx_drvdata, attached_elm);
- iommu_drvdata = dev_get_drvdata(ctx_drvdata->pdev->dev.parent);
-
- base = iommu_drvdata->base;
- ctx = ctx_drvdata->num;
-
- ret = __enable_clocks(iommu_drvdata);
- if (ret)
- goto fail;
-
- /* Invalidate context TLB */
- SET_CTX_TLBIALL(base, ctx, 0);
- SET_V2PPR(base, ctx, va & V2Pxx_VA);
-
- par = GET_PAR(base, ctx);
-
- /* We are dealing with a supersection */
- if (GET_NOFAULT_SS(base, ctx))
- ret = (par & 0xFF000000) | (va & 0x00FFFFFF);
- else /* Upper 20 bits from PAR, lower 12 from VA */
- ret = (par & 0xFFFFF000) | (va & 0x00000FFF);
-
- if (GET_FAULT(base, ctx))
- ret = 0;
-
- __disable_clocks(iommu_drvdata);
-fail:
- spin_unlock_irqrestore(&msm_iommu_lock, flags);
- return ret;
-}
-
-static bool msm_iommu_capable(enum iommu_cap cap)
-{
- return false;
-}
-
-static void print_ctx_regs(void __iomem *base, int ctx)
-{
- unsigned int fsr = GET_FSR(base, ctx);
- pr_err("FAR = %08x PAR = %08x\n",
- GET_FAR(base, ctx), GET_PAR(base, ctx));
- pr_err("FSR = %08x [%s%s%s%s%s%s%s%s%s%s]\n", fsr,
- (fsr & 0x02) ? "TF " : "",
- (fsr & 0x04) ? "AFF " : "",
- (fsr & 0x08) ? "APF " : "",
- (fsr & 0x10) ? "TLBMF " : "",
- (fsr & 0x20) ? "HTWDEEF " : "",
- (fsr & 0x40) ? "HTWSEEF " : "",
- (fsr & 0x80) ? "MHF " : "",
- (fsr & 0x10000) ? "SL " : "",
- (fsr & 0x40000000) ? "SS " : "",
- (fsr & 0x80000000) ? "MULTI " : "");
-
- pr_err("FSYNR0 = %08x FSYNR1 = %08x\n",
- GET_FSYNR0(base, ctx), GET_FSYNR1(base, ctx));
- pr_err("TTBR0 = %08x TTBR1 = %08x\n",
- GET_TTBR0(base, ctx), GET_TTBR1(base, ctx));
- pr_err("SCTLR = %08x ACTLR = %08x\n",
- GET_SCTLR(base, ctx), GET_ACTLR(base, ctx));
- pr_err("PRRR = %08x NMRR = %08x\n",
- GET_PRRR(base, ctx), GET_NMRR(base, ctx));
-}
-
-irqreturn_t msm_iommu_fault_handler(int irq, void *dev_id)
-{
- struct msm_iommu_drvdata *drvdata = dev_id;
- void __iomem *base;
- unsigned int fsr;
- int i, ret;
-
- spin_lock(&msm_iommu_lock);
-
- if (!drvdata) {
- pr_err("Invalid device ID in context interrupt handler\n");
- goto fail;
- }
-
- base = drvdata->base;
-
- pr_err("Unexpected IOMMU page fault!\n");
- pr_err("base = %08x\n", (unsigned int) base);
-
- ret = __enable_clocks(drvdata);
- if (ret)
- goto fail;
-
- for (i = 0; i < drvdata->ncb; i++) {
- fsr = GET_FSR(base, i);
- if (fsr) {
- pr_err("Fault occurred in context %d.\n", i);
- pr_err("Interesting registers:\n");
- print_ctx_regs(base, i);
- SET_FSR(base, i, 0x4000000F);
- }
- }
- __disable_clocks(drvdata);
-fail:
- spin_unlock(&msm_iommu_lock);
- return 0;
-}
-
-static const struct iommu_ops msm_iommu_ops = {
- .capable = msm_iommu_capable,
- .domain_alloc = msm_iommu_domain_alloc,
- .domain_free = msm_iommu_domain_free,
- .attach_dev = msm_iommu_attach_dev,
- .detach_dev = msm_iommu_detach_dev,
- .map = msm_iommu_map,
- .unmap = msm_iommu_unmap,
- .map_sg = default_iommu_map_sg,
- .iova_to_phys = msm_iommu_iova_to_phys,
- .pgsize_bitmap = MSM_IOMMU_PGSIZES,
-};
-
-static int __init get_tex_class(int icp, int ocp, int mt, int nos)
-{
- int i = 0;
- unsigned int prrr = 0;
- unsigned int nmrr = 0;
- int c_icp, c_ocp, c_mt, c_nos;
-
- RCP15_PRRR(prrr);
- RCP15_NMRR(nmrr);
-
- for (i = 0; i < NUM_TEX_CLASS; i++) {
- c_nos = PRRR_NOS(prrr, i);
- c_mt = PRRR_MT(prrr, i);
- c_icp = NMRR_ICP(nmrr, i);
- c_ocp = NMRR_OCP(nmrr, i);
-
- if (icp == c_icp && ocp == c_ocp && c_mt == mt && c_nos == nos)
- return i;
- }
-
- return -ENODEV;
-}
-
-static void __init setup_iommu_tex_classes(void)
-{
- msm_iommu_tex_class[MSM_IOMMU_ATTR_NONCACHED] =
- get_tex_class(CP_NONCACHED, CP_NONCACHED, MT_NORMAL, 1);
-
- msm_iommu_tex_class[MSM_IOMMU_ATTR_CACHED_WB_WA] =
- get_tex_class(CP_WB_WA, CP_WB_WA, MT_NORMAL, 1);
-
- msm_iommu_tex_class[MSM_IOMMU_ATTR_CACHED_WB_NWA] =
- get_tex_class(CP_WB_NWA, CP_WB_NWA, MT_NORMAL, 1);
-
- msm_iommu_tex_class[MSM_IOMMU_ATTR_CACHED_WT] =
- get_tex_class(CP_WT, CP_WT, MT_NORMAL, 1);
-}
-
-static int __init msm_iommu_init(void)
-{
- setup_iommu_tex_classes();
- bus_set_iommu(&platform_bus_type, &msm_iommu_ops);
- return 0;
-}
-
-subsys_initcall(msm_iommu_init);
-
-MODULE_LICENSE("GPL v2");
-MODULE_AUTHOR("Stepan Moskovchenko <stepanm@codeaurora.org>");
diff --git a/drivers/iommu/msm_iommu.h b/drivers/iommu/msm_iommu.h
deleted file mode 100644
index 5c7c955e6d25..000000000000
--- a/drivers/iommu/msm_iommu.h
+++ /dev/null
@@ -1,120 +0,0 @@
-/* Copyright (c) 2010-2011, Code Aurora Forum. All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 and
- * only version 2 as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
- * 02110-1301, USA.
- */
-
-#ifndef MSM_IOMMU_H
-#define MSM_IOMMU_H
-
-#include <linux/interrupt.h>
-#include <linux/clk.h>
-
-/* Sharability attributes of MSM IOMMU mappings */
-#define MSM_IOMMU_ATTR_NON_SH 0x0
-#define MSM_IOMMU_ATTR_SH 0x4
-
-/* Cacheability attributes of MSM IOMMU mappings */
-#define MSM_IOMMU_ATTR_NONCACHED 0x0
-#define MSM_IOMMU_ATTR_CACHED_WB_WA 0x1
-#define MSM_IOMMU_ATTR_CACHED_WB_NWA 0x2
-#define MSM_IOMMU_ATTR_CACHED_WT 0x3
-
-/* Mask for the cache policy attribute */
-#define MSM_IOMMU_CP_MASK 0x03
-
-/* Maximum number of Machine IDs that we are allowing to be mapped to the same
- * context bank. The number of MIDs mapped to the same CB does not affect
- * performance, but there is a practical limit on how many distinct MIDs may
- * be present. These mappings are typically determined at design time and are
- * not expected to change at run time.
- */
-#define MAX_NUM_MIDS 32
-
-/**
- * struct msm_iommu_dev - a single IOMMU hardware instance
- * name Human-readable name given to this IOMMU HW instance
- * ncb Number of context banks present on this IOMMU HW instance
- */
-struct msm_iommu_dev {
- const char *name;
- int ncb;
-};
-
-/**
- * struct msm_iommu_ctx_dev - an IOMMU context bank instance
- * name Human-readable name given to this context bank
- * num Index of this context bank within the hardware
- * mids List of Machine IDs that are to be mapped into this context
- * bank, terminated by -1. The MID is a set of signals on the
- * AXI bus that identifies the function associated with a specific
- * memory request. (See ARM spec).
- */
-struct msm_iommu_ctx_dev {
- const char *name;
- int num;
- int mids[MAX_NUM_MIDS];
-};
-
-
-/**
- * struct msm_iommu_drvdata - A single IOMMU hardware instance
- * @base: IOMMU config port base address (VA)
- * @ncb The number of contexts on this IOMMU
- * @irq: Interrupt number
- * @clk: The bus clock for this IOMMU hardware instance
- * @pclk: The clock for the IOMMU bus interconnect
- *
- * A msm_iommu_drvdata holds the global driver data about a single piece
- * of an IOMMU hardware instance.
- */
-struct msm_iommu_drvdata {
- void __iomem *base;
- int irq;
- int ncb;
- struct clk *clk;
- struct clk *pclk;
-};
-
-/**
- * struct msm_iommu_ctx_drvdata - an IOMMU context bank instance
- * @num: Hardware context number of this context
- * @pdev: Platform device associated wit this HW instance
- * @attached_elm: List element for domains to track which devices are
- * attached to them
- *
- * A msm_iommu_ctx_drvdata holds the driver data for a single context bank
- * within each IOMMU hardware instance
- */
-struct msm_iommu_ctx_drvdata {
- int num;
- struct platform_device *pdev;
- struct list_head attached_elm;
-};
-
-/*
- * Look up an IOMMU context device by its context name. NULL if none found.
- * Useful for testing and drivers that do not yet fully have IOMMU stuff in
- * their platform devices.
- */
-struct device *msm_iommu_get_ctx(const char *ctx_name);
-
-/*
- * Interrupt handler for the IOMMU context fault interrupt. Hooking the
- * interrupt is not supported in the API yet, but this will print an error
- * message and dump useful IOMMU registers.
- */
-irqreturn_t msm_iommu_fault_handler(int irq, void *dev_id);
-
-#endif
diff --git a/drivers/iommu/msm_iommu_dev.c b/drivers/iommu/msm_iommu_dev.c
deleted file mode 100644
index b6d01f97e537..000000000000
--- a/drivers/iommu/msm_iommu_dev.c
+++ /dev/null
@@ -1,392 +0,0 @@
-/* Copyright (c) 2010-2011, Code Aurora Forum. All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 and
- * only version 2 as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
- * 02110-1301, USA.
- */
-
-#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
-
-#include <linux/kernel.h>
-#include <linux/module.h>
-#include <linux/platform_device.h>
-#include <linux/io.h>
-#include <linux/clk.h>
-#include <linux/iommu.h>
-#include <linux/interrupt.h>
-#include <linux/err.h>
-#include <linux/slab.h>
-
-#include "msm_iommu_hw-8xxx.h"
-#include "msm_iommu.h"
-
-struct iommu_ctx_iter_data {
- /* input */
- const char *name;
-
- /* output */
- struct device *dev;
-};
-
-static struct platform_device *msm_iommu_root_dev;
-
-static int each_iommu_ctx(struct device *dev, void *data)
-{
- struct iommu_ctx_iter_data *res = data;
- struct msm_iommu_ctx_dev *c = dev->platform_data;
-
- if (!res || !c || !c->name || !res->name)
- return -EINVAL;
-
- if (!strcmp(res->name, c->name)) {
- res->dev = dev;
- return 1;
- }
- return 0;
-}
-
-static int each_iommu(struct device *dev, void *data)
-{
- return device_for_each_child(dev, data, each_iommu_ctx);
-}
-
-struct device *msm_iommu_get_ctx(const char *ctx_name)
-{
- struct iommu_ctx_iter_data r;
- int found;
-
- if (!msm_iommu_root_dev) {
- pr_err("No root IOMMU device.\n");
- goto fail;
- }
-
- r.name = ctx_name;
- found = device_for_each_child(&msm_iommu_root_dev->dev, &r, each_iommu);
-
- if (!found) {
- pr_err("Could not find context <%s>\n", ctx_name);
- goto fail;
- }
-
- return r.dev;
-fail:
- return NULL;
-}
-EXPORT_SYMBOL(msm_iommu_get_ctx);
-
-static void msm_iommu_reset(void __iomem *base, int ncb)
-{
- int ctx;
-
- SET_RPUE(base, 0);
- SET_RPUEIE(base, 0);
- SET_ESRRESTORE(base, 0);
- SET_TBE(base, 0);
- SET_CR(base, 0);
- SET_SPDMBE(base, 0);
- SET_TESTBUSCR(base, 0);
- SET_TLBRSW(base, 0);
- SET_GLOBAL_TLBIALL(base, 0);
- SET_RPU_ACR(base, 0);
- SET_TLBLKCRWE(base, 1);
-
- for (ctx = 0; ctx < ncb; ctx++) {
- SET_BPRCOSH(base, ctx, 0);
- SET_BPRCISH(base, ctx, 0);
- SET_BPRCNSH(base, ctx, 0);
- SET_BPSHCFG(base, ctx, 0);
- SET_BPMTCFG(base, ctx, 0);
- SET_ACTLR(base, ctx, 0);
- SET_SCTLR(base, ctx, 0);
- SET_FSRRESTORE(base, ctx, 0);
- SET_TTBR0(base, ctx, 0);
- SET_TTBR1(base, ctx, 0);
- SET_TTBCR(base, ctx, 0);
- SET_BFBCR(base, ctx, 0);
- SET_PAR(base, ctx, 0);
- SET_FAR(base, ctx, 0);
- SET_CTX_TLBIALL(base, ctx, 0);
- SET_TLBFLPTER(base, ctx, 0);
- SET_TLBSLPTER(base, ctx, 0);
- SET_TLBLKCR(base, ctx, 0);
- SET_PRRR(base, ctx, 0);
- SET_NMRR(base, ctx, 0);
- SET_CONTEXTIDR(base, ctx, 0);
- }
-}
-
-static int msm_iommu_probe(struct platform_device *pdev)
-{
- struct resource *r;
- struct clk *iommu_clk;
- struct clk *iommu_pclk;
- struct msm_iommu_drvdata *drvdata;
- struct msm_iommu_dev *iommu_dev = dev_get_platdata(&pdev->dev);
- void __iomem *regs_base;
- int ret, irq, par;
-
- if (pdev->id == -1) {
- msm_iommu_root_dev = pdev;
- return 0;
- }
-
- drvdata = kzalloc(sizeof(*drvdata), GFP_KERNEL);
-
- if (!drvdata) {
- ret = -ENOMEM;
- goto fail;
- }
-
- if (!iommu_dev) {
- ret = -ENODEV;
- goto fail;
- }
-
- iommu_pclk = clk_get(NULL, "smmu_pclk");
- if (IS_ERR(iommu_pclk)) {
- ret = -ENODEV;
- goto fail;
- }
-
- ret = clk_prepare_enable(iommu_pclk);
- if (ret)
- goto fail_enable;
-
- iommu_clk = clk_get(&pdev->dev, "iommu_clk");
-
- if (!IS_ERR(iommu_clk)) {
- if (clk_get_rate(iommu_clk) == 0)
- clk_set_rate(iommu_clk, 1);
-
- ret = clk_prepare_enable(iommu_clk);
- if (ret) {
- clk_put(iommu_clk);
- goto fail_pclk;
- }
- } else
- iommu_clk = NULL;
-
- r = platform_get_resource_byname(pdev, IORESOURCE_MEM, "physbase");
- regs_base = devm_ioremap_resource(&pdev->dev, r);
- if (IS_ERR(regs_base)) {
- ret = PTR_ERR(regs_base);
- goto fail_clk;
- }
-
- irq = platform_get_irq_byname(pdev, "secure_irq");
- if (irq < 0) {
- ret = -ENODEV;
- goto fail_clk;
- }
-
- msm_iommu_reset(regs_base, iommu_dev->ncb);
-
- SET_M(regs_base, 0, 1);
- SET_PAR(regs_base, 0, 0);
- SET_V2PCFG(regs_base, 0, 1);
- SET_V2PPR(regs_base, 0, 0);
- par = GET_PAR(regs_base, 0);
- SET_V2PCFG(regs_base, 0, 0);
- SET_M(regs_base, 0, 0);
-
- if (!par) {
- pr_err("%s: Invalid PAR value detected\n", iommu_dev->name);
- ret = -ENODEV;
- goto fail_clk;
- }
-
- ret = request_irq(irq, msm_iommu_fault_handler, 0,
- "msm_iommu_secure_irpt_handler", drvdata);
- if (ret) {
- pr_err("Request IRQ %d failed with ret=%d\n", irq, ret);
- goto fail_clk;
- }
-
-
- drvdata->pclk = iommu_pclk;
- drvdata->clk = iommu_clk;
- drvdata->base = regs_base;
- drvdata->irq = irq;
- drvdata->ncb = iommu_dev->ncb;
-
- pr_info("device %s mapped at %p, irq %d with %d ctx banks\n",
- iommu_dev->name, regs_base, irq, iommu_dev->ncb);
-
- platform_set_drvdata(pdev, drvdata);
-
- clk_disable(iommu_clk);
-
- clk_disable(iommu_pclk);
-
- return 0;
-fail_clk:
- if (iommu_clk) {
- clk_disable(iommu_clk);
- clk_put(iommu_clk);
- }
-fail_pclk:
- clk_disable_unprepare(iommu_pclk);
-fail_enable:
- clk_put(iommu_pclk);
-fail:
- kfree(drvdata);
- return ret;
-}
-
-static int msm_iommu_remove(struct platform_device *pdev)
-{
- struct msm_iommu_drvdata *drv = NULL;
-
- drv = platform_get_drvdata(pdev);
- if (drv) {
- if (drv->clk) {
- clk_unprepare(drv->clk);
- clk_put(drv->clk);
- }
- clk_unprepare(drv->pclk);
- clk_put(drv->pclk);
- memset(drv, 0, sizeof(*drv));
- kfree(drv);
- }
- return 0;
-}
-
-static int msm_iommu_ctx_probe(struct platform_device *pdev)
-{
- struct msm_iommu_ctx_dev *c = dev_get_platdata(&pdev->dev);
- struct msm_iommu_drvdata *drvdata;
- struct msm_iommu_ctx_drvdata *ctx_drvdata;
- int i, ret;
-
- if (!c || !pdev->dev.parent)
- return -EINVAL;
-
- drvdata = dev_get_drvdata(pdev->dev.parent);
- if (!drvdata)
- return -ENODEV;
-
- ctx_drvdata = kzalloc(sizeof(*ctx_drvdata), GFP_KERNEL);
- if (!ctx_drvdata)
- return -ENOMEM;
-
- ctx_drvdata->num = c->num;
- ctx_drvdata->pdev = pdev;
-
- INIT_LIST_HEAD(&ctx_drvdata->attached_elm);
- platform_set_drvdata(pdev, ctx_drvdata);
-
- ret = clk_prepare_enable(drvdata->pclk);
- if (ret)
- goto fail;
-
- if (drvdata->clk) {
- ret = clk_prepare_enable(drvdata->clk);
- if (ret) {
- clk_disable_unprepare(drvdata->pclk);
- goto fail;
- }
- }
-
- /* Program the M2V tables for this context */
- for (i = 0; i < MAX_NUM_MIDS; i++) {
- int mid = c->mids[i];
- if (mid == -1)
- break;
-
- SET_M2VCBR_N(drvdata->base, mid, 0);
- SET_CBACR_N(drvdata->base, c->num, 0);
-
- /* Set VMID = 0 */
- SET_VMID(drvdata->base, mid, 0);
-
- /* Set the context number for that MID to this context */
- SET_CBNDX(drvdata->base, mid, c->num);
-
- /* Set MID associated with this context bank to 0*/
- SET_CBVMID(drvdata->base, c->num, 0);
-
- /* Set the ASID for TLB tagging for this context */
- SET_CONTEXTIDR_ASID(drvdata->base, c->num, c->num);
-
- /* Set security bit override to be Non-secure */
- SET_NSCFG(drvdata->base, mid, 3);
- }
-
- clk_disable(drvdata->clk);
- clk_disable(drvdata->pclk);
-
- dev_info(&pdev->dev, "context %s using bank %d\n", c->name, c->num);
- return 0;
-fail:
- kfree(ctx_drvdata);
- return ret;
-}
-
-static int msm_iommu_ctx_remove(struct platform_device *pdev)
-{
- struct msm_iommu_ctx_drvdata *drv = NULL;
- drv = platform_get_drvdata(pdev);
- if (drv) {
- memset(drv, 0, sizeof(struct msm_iommu_ctx_drvdata));
- kfree(drv);
- }
- return 0;
-}
-
-static struct platform_driver msm_iommu_driver = {
- .driver = {
- .name = "msm_iommu",
- },
- .probe = msm_iommu_probe,
- .remove = msm_iommu_remove,
-};
-
-static struct platform_driver msm_iommu_ctx_driver = {
- .driver = {
- .name = "msm_iommu_ctx",
- },
- .probe = msm_iommu_ctx_probe,
- .remove = msm_iommu_ctx_remove,
-};
-
-static int __init msm_iommu_driver_init(void)
-{
- int ret;
- ret = platform_driver_register(&msm_iommu_driver);
- if (ret != 0) {
- pr_err("Failed to register IOMMU driver\n");
- goto error;
- }
-
- ret = platform_driver_register(&msm_iommu_ctx_driver);
- if (ret != 0) {
- platform_driver_unregister(&msm_iommu_driver);
- pr_err("Failed to register IOMMU context driver\n");
- goto error;
- }
-
-error:
- return ret;
-}
-
-static void __exit msm_iommu_driver_exit(void)
-{
- platform_driver_unregister(&msm_iommu_ctx_driver);
- platform_driver_unregister(&msm_iommu_driver);
-}
-
-subsys_initcall(msm_iommu_driver_init);
-module_exit(msm_iommu_driver_exit);
-
-MODULE_LICENSE("GPL v2");
-MODULE_AUTHOR("Stepan Moskovchenko <stepanm@codeaurora.org>");
diff --git a/drivers/iommu/msm_iommu_hw-8xxx.h b/drivers/iommu/msm_iommu_hw-8xxx.h
deleted file mode 100644
index fc160101dead..000000000000
--- a/drivers/iommu/msm_iommu_hw-8xxx.h
+++ /dev/null
@@ -1,1865 +0,0 @@
-/* Copyright (c) 2010-2011, Code Aurora Forum. All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 and
- * only version 2 as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
- * 02110-1301, USA.
- */
-
-#ifndef __ARCH_ARM_MACH_MSM_IOMMU_HW_8XXX_H
-#define __ARCH_ARM_MACH_MSM_IOMMU_HW_8XXX_H
-
-#define CTX_SHIFT 12
-
-#define GET_GLOBAL_REG(reg, base) (readl((base) + (reg)))
-#define GET_CTX_REG(reg, base, ctx) \
- (readl((base) + (reg) + ((ctx) << CTX_SHIFT)))
-
-#define SET_GLOBAL_REG(reg, base, val) writel((val), ((base) + (reg)))
-
-#define SET_CTX_REG(reg, base, ctx, val) \
- writel((val), ((base) + (reg) + ((ctx) << CTX_SHIFT)))
-
-/* Wrappers for numbered registers */
-#define SET_GLOBAL_REG_N(b, n, r, v) SET_GLOBAL_REG(b, ((r) + (n << 2)), (v))
-#define GET_GLOBAL_REG_N(b, n, r) GET_GLOBAL_REG(b, ((r) + (n << 2)))
-
-/* Field wrappers */
-#define GET_GLOBAL_FIELD(b, r, F) GET_FIELD(((b) + (r)), F##_MASK, F##_SHIFT)
-#define GET_CONTEXT_FIELD(b, c, r, F) \
- GET_FIELD(((b) + (r) + ((c) << CTX_SHIFT)), F##_MASK, F##_SHIFT)
-
-#define SET_GLOBAL_FIELD(b, r, F, v) \
- SET_FIELD(((b) + (r)), F##_MASK, F##_SHIFT, (v))
-#define SET_CONTEXT_FIELD(b, c, r, F, v) \
- SET_FIELD(((b) + (r) + ((c) << CTX_SHIFT)), F##_MASK, F##_SHIFT, (v))
-
-#define GET_FIELD(addr, mask, shift) ((readl(addr) >> (shift)) & (mask))
-
-#define SET_FIELD(addr, mask, shift, v) \
-do { \
- int t = readl(addr); \
- writel((t & ~((mask) << (shift))) + (((v) & (mask)) << (shift)), addr);\
-} while (0)
-
-
-#define NUM_FL_PTE 4096
-#define NUM_SL_PTE 256
-#define NUM_TEX_CLASS 8
-
-/* First-level page table bits */
-#define FL_BASE_MASK 0xFFFFFC00
-#define FL_TYPE_TABLE (1 << 0)
-#define FL_TYPE_SECT (2 << 0)
-#define FL_SUPERSECTION (1 << 18)
-#define FL_AP_WRITE (1 << 10)
-#define FL_AP_READ (1 << 11)
-#define FL_SHARED (1 << 16)
-#define FL_BUFFERABLE (1 << 2)
-#define FL_CACHEABLE (1 << 3)
-#define FL_TEX0 (1 << 12)
-#define FL_OFFSET(va) (((va) & 0xFFF00000) >> 20)
-#define FL_NG (1 << 17)
-
-/* Second-level page table bits */
-#define SL_BASE_MASK_LARGE 0xFFFF0000
-#define SL_BASE_MASK_SMALL 0xFFFFF000
-#define SL_TYPE_LARGE (1 << 0)
-#define SL_TYPE_SMALL (2 << 0)
-#define SL_AP0 (1 << 4)
-#define SL_AP1 (2 << 4)
-#define SL_SHARED (1 << 10)
-#define SL_BUFFERABLE (1 << 2)
-#define SL_CACHEABLE (1 << 3)
-#define SL_TEX0 (1 << 6)
-#define SL_OFFSET(va) (((va) & 0xFF000) >> 12)
-#define SL_NG (1 << 11)
-
-/* Memory type and cache policy attributes */
-#define MT_SO 0
-#define MT_DEV 1
-#define MT_NORMAL 2
-#define CP_NONCACHED 0
-#define CP_WB_WA 1
-#define CP_WT 2
-#define CP_WB_NWA 3
-
-/* Global register setters / getters */
-#define SET_M2VCBR_N(b, N, v) SET_GLOBAL_REG_N(M2VCBR_N, N, (b), (v))
-#define SET_CBACR_N(b, N, v) SET_GLOBAL_REG_N(CBACR_N, N, (b), (v))
-#define SET_TLBRSW(b, v) SET_GLOBAL_REG(TLBRSW, (b), (v))
-#define SET_TLBTR0(b, v) SET_GLOBAL_REG(TLBTR0, (b), (v))
-#define SET_TLBTR1(b, v) SET_GLOBAL_REG(TLBTR1, (b), (v))
-#define SET_TLBTR2(b, v) SET_GLOBAL_REG(TLBTR2, (b), (v))
-#define SET_TESTBUSCR(b, v) SET_GLOBAL_REG(TESTBUSCR, (b), (v))
-#define SET_GLOBAL_TLBIALL(b, v) SET_GLOBAL_REG(GLOBAL_TLBIALL, (b), (v))
-#define SET_TLBIVMID(b, v) SET_GLOBAL_REG(TLBIVMID, (b), (v))
-#define SET_CR(b, v) SET_GLOBAL_REG(CR, (b), (v))
-#define SET_EAR(b, v) SET_GLOBAL_REG(EAR, (b), (v))
-#define SET_ESR(b, v) SET_GLOBAL_REG(ESR, (b), (v))
-#define SET_ESRRESTORE(b, v) SET_GLOBAL_REG(ESRRESTORE, (b), (v))
-#define SET_ESYNR0(b, v) SET_GLOBAL_REG(ESYNR0, (b), (v))
-#define SET_ESYNR1(b, v) SET_GLOBAL_REG(ESYNR1, (b), (v))
-#define SET_RPU_ACR(b, v) SET_GLOBAL_REG(RPU_ACR, (b), (v))
-
-#define GET_M2VCBR_N(b, N) GET_GLOBAL_REG_N(M2VCBR_N, N, (b))
-#define GET_CBACR_N(b, N) GET_GLOBAL_REG_N(CBACR_N, N, (b))
-#define GET_TLBTR0(b) GET_GLOBAL_REG(TLBTR0, (b))
-#define GET_TLBTR1(b) GET_GLOBAL_REG(TLBTR1, (b))
-#define GET_TLBTR2(b) GET_GLOBAL_REG(TLBTR2, (b))
-#define GET_TESTBUSCR(b) GET_GLOBAL_REG(TESTBUSCR, (b))
-#define GET_GLOBAL_TLBIALL(b) GET_GLOBAL_REG(GLOBAL_TLBIALL, (b))
-#define GET_TLBIVMID(b) GET_GLOBAL_REG(TLBIVMID, (b))
-#define GET_CR(b) GET_GLOBAL_REG(CR, (b))
-#define GET_EAR(b) GET_GLOBAL_REG(EAR, (b))
-#define GET_ESR(b) GET_GLOBAL_REG(ESR, (b))
-#define GET_ESRRESTORE(b) GET_GLOBAL_REG(ESRRESTORE, (b))
-#define GET_ESYNR0(b) GET_GLOBAL_REG(ESYNR0, (b))
-#define GET_ESYNR1(b) GET_GLOBAL_REG(ESYNR1, (b))
-#define GET_REV(b) GET_GLOBAL_REG(REV, (b))
-#define GET_IDR(b) GET_GLOBAL_REG(IDR, (b))
-#define GET_RPU_ACR(b) GET_GLOBAL_REG(RPU_ACR, (b))
-
-
-/* Context register setters/getters */
-#define SET_SCTLR(b, c, v) SET_CTX_REG(SCTLR, (b), (c), (v))
-#define SET_ACTLR(b, c, v) SET_CTX_REG(ACTLR, (b), (c), (v))
-#define SET_CONTEXTIDR(b, c, v) SET_CTX_REG(CONTEXTIDR, (b), (c), (v))
-#define SET_TTBR0(b, c, v) SET_CTX_REG(TTBR0, (b), (c), (v))
-#define SET_TTBR1(b, c, v) SET_CTX_REG(TTBR1, (b), (c), (v))
-#define SET_TTBCR(b, c, v) SET_CTX_REG(TTBCR, (b), (c), (v))
-#define SET_PAR(b, c, v) SET_CTX_REG(PAR, (b), (c), (v))
-#define SET_FSR(b, c, v) SET_CTX_REG(FSR, (b), (c), (v))
-#define SET_FSRRESTORE(b, c, v) SET_CTX_REG(FSRRESTORE, (b), (c), (v))
-#define SET_FAR(b, c, v) SET_CTX_REG(FAR, (b), (c), (v))
-#define SET_FSYNR0(b, c, v) SET_CTX_REG(FSYNR0, (b), (c), (v))
-#define SET_FSYNR1(b, c, v) SET_CTX_REG(FSYNR1, (b), (c), (v))
-#define SET_PRRR(b, c, v) SET_CTX_REG(PRRR, (b), (c), (v))
-#define SET_NMRR(b, c, v) SET_CTX_REG(NMRR, (b), (c), (v))
-#define SET_TLBLKCR(b, c, v) SET_CTX_REG(TLBLCKR, (b), (c), (v))
-#define SET_V2PSR(b, c, v) SET_CTX_REG(V2PSR, (b), (c), (v))
-#define SET_TLBFLPTER(b, c, v) SET_CTX_REG(TLBFLPTER, (b), (c), (v))
-#define SET_TLBSLPTER(b, c, v) SET_CTX_REG(TLBSLPTER, (b), (c), (v))
-#define SET_BFBCR(b, c, v) SET_CTX_REG(BFBCR, (b), (c), (v))
-#define SET_CTX_TLBIALL(b, c, v) SET_CTX_REG(CTX_TLBIALL, (b), (c), (v))
-#define SET_TLBIASID(b, c, v) SET_CTX_REG(TLBIASID, (b), (c), (v))
-#define SET_TLBIVA(b, c, v) SET_CTX_REG(TLBIVA, (b), (c), (v))
-#define SET_TLBIVAA(b, c, v) SET_CTX_REG(TLBIVAA, (b), (c), (v))
-#define SET_V2PPR(b, c, v) SET_CTX_REG(V2PPR, (b), (c), (v))
-#define SET_V2PPW(b, c, v) SET_CTX_REG(V2PPW, (b), (c), (v))
-#define SET_V2PUR(b, c, v) SET_CTX_REG(V2PUR, (b), (c), (v))
-#define SET_V2PUW(b, c, v) SET_CTX_REG(V2PUW, (b), (c), (v))
-#define SET_RESUME(b, c, v) SET_CTX_REG(RESUME, (b), (c), (v))
-
-#define GET_SCTLR(b, c) GET_CTX_REG(SCTLR, (b), (c))
-#define GET_ACTLR(b, c) GET_CTX_REG(ACTLR, (b), (c))
-#define GET_CONTEXTIDR(b, c) GET_CTX_REG(CONTEXTIDR, (b), (c))
-#define GET_TTBR0(b, c) GET_CTX_REG(TTBR0, (b), (c))
-#define GET_TTBR1(b, c) GET_CTX_REG(TTBR1, (b), (c))
-#define GET_TTBCR(b, c) GET_CTX_REG(TTBCR, (b), (c))
-#define GET_PAR(b, c) GET_CTX_REG(PAR, (b), (c))
-#define GET_FSR(b, c) GET_CTX_REG(FSR, (b), (c))
-#define GET_FSRRESTORE(b, c) GET_CTX_REG(FSRRESTORE, (b), (c))
-#define GET_FAR(b, c) GET_CTX_REG(FAR, (b), (c))
-#define GET_FSYNR0(b, c) GET_CTX_REG(FSYNR0, (b), (c))
-#define GET_FSYNR1(b, c) GET_CTX_REG(FSYNR1, (b), (c))
-#define GET_PRRR(b, c) GET_CTX_REG(PRRR, (b), (c))
-#define GET_NMRR(b, c) GET_CTX_REG(NMRR, (b), (c))
-#define GET_TLBLCKR(b, c) GET_CTX_REG(TLBLCKR, (b), (c))
-#define GET_V2PSR(b, c) GET_CTX_REG(V2PSR, (b), (c))
-#define GET_TLBFLPTER(b, c) GET_CTX_REG(TLBFLPTER, (b), (c))
-#define GET_TLBSLPTER(b, c) GET_CTX_REG(TLBSLPTER, (b), (c))
-#define GET_BFBCR(b, c) GET_CTX_REG(BFBCR, (b), (c))
-#define GET_CTX_TLBIALL(b, c) GET_CTX_REG(CTX_TLBIALL, (b), (c))
-#define GET_TLBIASID(b, c) GET_CTX_REG(TLBIASID, (b), (c))
-#define GET_TLBIVA(b, c) GET_CTX_REG(TLBIVA, (b), (c))
-#define GET_TLBIVAA(b, c) GET_CTX_REG(TLBIVAA, (b), (c))
-#define GET_V2PPR(b, c) GET_CTX_REG(V2PPR, (b), (c))
-#define GET_V2PPW(b, c) GET_CTX_REG(V2PPW, (b), (c))
-#define GET_V2PUR(b, c) GET_CTX_REG(V2PUR, (b), (c))
-#define GET_V2PUW(b, c) GET_CTX_REG(V2PUW, (b), (c))
-#define GET_RESUME(b, c) GET_CTX_REG(RESUME, (b), (c))
-
-
-/* Global field setters / getters */
-/* Global Field Setters: */
-/* CBACR_N */
-#define SET_RWVMID(b, n, v) SET_GLOBAL_FIELD(b, (n<<2)|(CBACR_N), RWVMID, v)
-#define SET_RWE(b, n, v) SET_GLOBAL_FIELD(b, (n<<2)|(CBACR_N), RWE, v)
-#define SET_RWGE(b, n, v) SET_GLOBAL_FIELD(b, (n<<2)|(CBACR_N), RWGE, v)
-#define SET_CBVMID(b, n, v) SET_GLOBAL_FIELD(b, (n<<2)|(CBACR_N), CBVMID, v)
-#define SET_IRPTNDX(b, n, v) SET_GLOBAL_FIELD(b, (n<<2)|(CBACR_N), IRPTNDX, v)
-
-
-/* M2VCBR_N */
-#define SET_VMID(b, n, v) SET_GLOBAL_FIELD(b, (n<<2)|(M2VCBR_N), VMID, v)
-#define SET_CBNDX(b, n, v) SET_GLOBAL_FIELD(b, (n<<2)|(M2VCBR_N), CBNDX, v)
-#define SET_BYPASSD(b, n, v) SET_GLOBAL_FIELD(b, (n<<2)|(M2VCBR_N), BYPASSD, v)
-#define SET_BPRCOSH(b, n, v) SET_GLOBAL_FIELD(b, (n<<2)|(M2VCBR_N), BPRCOSH, v)
-#define SET_BPRCISH(b, n, v) SET_GLOBAL_FIELD(b, (n<<2)|(M2VCBR_N), BPRCISH, v)
-#define SET_BPRCNSH(b, n, v) SET_GLOBAL_FIELD(b, (n<<2)|(M2VCBR_N), BPRCNSH, v)
-#define SET_BPSHCFG(b, n, v) SET_GLOBAL_FIELD(b, (n<<2)|(M2VCBR_N), BPSHCFG, v)
-#define SET_NSCFG(b, n, v) SET_GLOBAL_FIELD(b, (n<<2)|(M2VCBR_N), NSCFG, v)
-#define SET_BPMTCFG(b, n, v) SET_GLOBAL_FIELD(b, (n<<2)|(M2VCBR_N), BPMTCFG, v)
-#define SET_BPMEMTYPE(b, n, v) \
- SET_GLOBAL_FIELD(b, (n<<2)|(M2VCBR_N), BPMEMTYPE, v)
-
-
-/* CR */
-#define SET_RPUE(b, v) SET_GLOBAL_FIELD(b, CR, RPUE, v)
-#define SET_RPUERE(b, v) SET_GLOBAL_FIELD(b, CR, RPUERE, v)
-#define SET_RPUEIE(b, v) SET_GLOBAL_FIELD(b, CR, RPUEIE, v)
-#define SET_DCDEE(b, v) SET_GLOBAL_FIELD(b, CR, DCDEE, v)
-#define SET_CLIENTPD(b, v) SET_GLOBAL_FIELD(b, CR, CLIENTPD, v)
-#define SET_STALLD(b, v) SET_GLOBAL_FIELD(b, CR, STALLD, v)
-#define SET_TLBLKCRWE(b, v) SET_GLOBAL_FIELD(b, CR, TLBLKCRWE, v)
-#define SET_CR_TLBIALLCFG(b, v) SET_GLOBAL_FIELD(b, CR, CR_TLBIALLCFG, v)
-#define SET_TLBIVMIDCFG(b, v) SET_GLOBAL_FIELD(b, CR, TLBIVMIDCFG, v)
-#define SET_CR_HUME(b, v) SET_GLOBAL_FIELD(b, CR, CR_HUME, v)
-
-
-/* ESR */
-#define SET_CFG(b, v) SET_GLOBAL_FIELD(b, ESR, CFG, v)
-#define SET_BYPASS(b, v) SET_GLOBAL_FIELD(b, ESR, BYPASS, v)
-#define SET_ESR_MULTI(b, v) SET_GLOBAL_FIELD(b, ESR, ESR_MULTI, v)
-
-
-/* ESYNR0 */
-#define SET_ESYNR0_AMID(b, v) SET_GLOBAL_FIELD(b, ESYNR0, ESYNR0_AMID, v)
-#define SET_ESYNR0_APID(b, v) SET_GLOBAL_FIELD(b, ESYNR0, ESYNR0_APID, v)
-#define SET_ESYNR0_ABID(b, v) SET_GLOBAL_FIELD(b, ESYNR0, ESYNR0_ABID, v)
-#define SET_ESYNR0_AVMID(b, v) SET_GLOBAL_FIELD(b, ESYNR0, ESYNR0_AVMID, v)
-#define SET_ESYNR0_ATID(b, v) SET_GLOBAL_FIELD(b, ESYNR0, ESYNR0_ATID, v)
-
-
-/* ESYNR1 */
-#define SET_ESYNR1_AMEMTYPE(b, v) \
- SET_GLOBAL_FIELD(b, ESYNR1, ESYNR1_AMEMTYPE, v)
-#define SET_ESYNR1_ASHARED(b, v) SET_GLOBAL_FIELD(b, ESYNR1, ESYNR1_ASHARED, v)
-#define SET_ESYNR1_AINNERSHARED(b, v) \
- SET_GLOBAL_FIELD(b, ESYNR1, ESYNR1_AINNERSHARED, v)
-#define SET_ESYNR1_APRIV(b, v) SET_GLOBAL_FIELD(b, ESYNR1, ESYNR1_APRIV, v)
-#define SET_ESYNR1_APROTNS(b, v) SET_GLOBAL_FIELD(b, ESYNR1, ESYNR1_APROTNS, v)
-#define SET_ESYNR1_AINST(b, v) SET_GLOBAL_FIELD(b, ESYNR1, ESYNR1_AINST, v)
-#define SET_ESYNR1_AWRITE(b, v) SET_GLOBAL_FIELD(b, ESYNR1, ESYNR1_AWRITE, v)
-#define SET_ESYNR1_ABURST(b, v) SET_GLOBAL_FIELD(b, ESYNR1, ESYNR1_ABURST, v)
-#define SET_ESYNR1_ALEN(b, v) SET_GLOBAL_FIELD(b, ESYNR1, ESYNR1_ALEN, v)
-#define SET_ESYNR1_ASIZE(b, v) SET_GLOBAL_FIELD(b, ESYNR1, ESYNR1_ASIZE, v)
-#define SET_ESYNR1_ALOCK(b, v) SET_GLOBAL_FIELD(b, ESYNR1, ESYNR1_ALOCK, v)
-#define SET_ESYNR1_AOOO(b, v) SET_GLOBAL_FIELD(b, ESYNR1, ESYNR1_AOOO, v)
-#define SET_ESYNR1_AFULL(b, v) SET_GLOBAL_FIELD(b, ESYNR1, ESYNR1_AFULL, v)
-#define SET_ESYNR1_AC(b, v) SET_GLOBAL_FIELD(b, ESYNR1, ESYNR1_AC, v)
-#define SET_ESYNR1_DCD(b, v) SET_GLOBAL_FIELD(b, ESYNR1, ESYNR1_DCD, v)
-
-
-/* TESTBUSCR */
-#define SET_TBE(b, v) SET_GLOBAL_FIELD(b, TESTBUSCR, TBE, v)
-#define SET_SPDMBE(b, v) SET_GLOBAL_FIELD(b, TESTBUSCR, SPDMBE, v)
-#define SET_WGSEL(b, v) SET_GLOBAL_FIELD(b, TESTBUSCR, WGSEL, v)
-#define SET_TBLSEL(b, v) SET_GLOBAL_FIELD(b, TESTBUSCR, TBLSEL, v)
-#define SET_TBHSEL(b, v) SET_GLOBAL_FIELD(b, TESTBUSCR, TBHSEL, v)
-#define SET_SPDM0SEL(b, v) SET_GLOBAL_FIELD(b, TESTBUSCR, SPDM0SEL, v)
-#define SET_SPDM1SEL(b, v) SET_GLOBAL_FIELD(b, TESTBUSCR, SPDM1SEL, v)
-#define SET_SPDM2SEL(b, v) SET_GLOBAL_FIELD(b, TESTBUSCR, SPDM2SEL, v)
-#define SET_SPDM3SEL(b, v) SET_GLOBAL_FIELD(b, TESTBUSCR, SPDM3SEL, v)
-
-
-/* TLBIVMID */
-#define SET_TLBIVMID_VMID(b, v) SET_GLOBAL_FIELD(b, TLBIVMID, TLBIVMID_VMID, v)
-
-
-/* TLBRSW */
-#define SET_TLBRSW_INDEX(b, v) SET_GLOBAL_FIELD(b, TLBRSW, TLBRSW_INDEX, v)
-#define SET_TLBBFBS(b, v) SET_GLOBAL_FIELD(b, TLBRSW, TLBBFBS, v)
-
-
-/* TLBTR0 */
-#define SET_PR(b, v) SET_GLOBAL_FIELD(b, TLBTR0, PR, v)
-#define SET_PW(b, v) SET_GLOBAL_FIELD(b, TLBTR0, PW, v)
-#define SET_UR(b, v) SET_GLOBAL_FIELD(b, TLBTR0, UR, v)
-#define SET_UW(b, v) SET_GLOBAL_FIELD(b, TLBTR0, UW, v)
-#define SET_XN(b, v) SET_GLOBAL_FIELD(b, TLBTR0, XN, v)
-#define SET_NSDESC(b, v) SET_GLOBAL_FIELD(b, TLBTR0, NSDESC, v)
-#define SET_ISH(b, v) SET_GLOBAL_FIELD(b, TLBTR0, ISH, v)
-#define SET_SH(b, v) SET_GLOBAL_FIELD(b, TLBTR0, SH, v)
-#define SET_MT(b, v) SET_GLOBAL_FIELD(b, TLBTR0, MT, v)
-#define SET_DPSIZR(b, v) SET_GLOBAL_FIELD(b, TLBTR0, DPSIZR, v)
-#define SET_DPSIZC(b, v) SET_GLOBAL_FIELD(b, TLBTR0, DPSIZC, v)
-
-
-/* TLBTR1 */
-#define SET_TLBTR1_VMID(b, v) SET_GLOBAL_FIELD(b, TLBTR1, TLBTR1_VMID, v)
-#define SET_TLBTR1_PA(b, v) SET_GLOBAL_FIELD(b, TLBTR1, TLBTR1_PA, v)
-
-
-/* TLBTR2 */
-#define SET_TLBTR2_ASID(b, v) SET_GLOBAL_FIELD(b, TLBTR2, TLBTR2_ASID, v)
-#define SET_TLBTR2_V(b, v) SET_GLOBAL_FIELD(b, TLBTR2, TLBTR2_V, v)
-#define SET_TLBTR2_NSTID(b, v) SET_GLOBAL_FIELD(b, TLBTR2, TLBTR2_NSTID, v)
-#define SET_TLBTR2_NV(b, v) SET_GLOBAL_FIELD(b, TLBTR2, TLBTR2_NV, v)
-#define SET_TLBTR2_VA(b, v) SET_GLOBAL_FIELD(b, TLBTR2, TLBTR2_VA, v)
-
-
-/* Global Field Getters */
-/* CBACR_N */
-#define GET_RWVMID(b, n) GET_GLOBAL_FIELD(b, (n<<2)|(CBACR_N), RWVMID)
-#define GET_RWE(b, n) GET_GLOBAL_FIELD(b, (n<<2)|(CBACR_N), RWE)
-#define GET_RWGE(b, n) GET_GLOBAL_FIELD(b, (n<<2)|(CBACR_N), RWGE)
-#define GET_CBVMID(b, n) GET_GLOBAL_FIELD(b, (n<<2)|(CBACR_N), CBVMID)
-#define GET_IRPTNDX(b, n) GET_GLOBAL_FIELD(b, (n<<2)|(CBACR_N), IRPTNDX)
-
-
-/* M2VCBR_N */
-#define GET_VMID(b, n) GET_GLOBAL_FIELD(b, (n<<2)|(M2VCBR_N), VMID)
-#define GET_CBNDX(b, n) GET_GLOBAL_FIELD(b, (n<<2)|(M2VCBR_N), CBNDX)
-#define GET_BYPASSD(b, n) GET_GLOBAL_FIELD(b, (n<<2)|(M2VCBR_N), BYPASSD)
-#define GET_BPRCOSH(b, n) GET_GLOBAL_FIELD(b, (n<<2)|(M2VCBR_N), BPRCOSH)
-#define GET_BPRCISH(b, n) GET_GLOBAL_FIELD(b, (n<<2)|(M2VCBR_N), BPRCISH)
-#define GET_BPRCNSH(b, n) GET_GLOBAL_FIELD(b, (n<<2)|(M2VCBR_N), BPRCNSH)
-#define GET_BPSHCFG(b, n) GET_GLOBAL_FIELD(b, (n<<2)|(M2VCBR_N), BPSHCFG)
-#define GET_NSCFG(b, n) GET_GLOBAL_FIELD(b, (n<<2)|(M2VCBR_N), NSCFG)
-#define GET_BPMTCFG(b, n) GET_GLOBAL_FIELD(b, (n<<2)|(M2VCBR_N), BPMTCFG)
-#define GET_BPMEMTYPE(b, n) GET_GLOBAL_FIELD(b, (n<<2)|(M2VCBR_N), BPMEMTYPE)
-
-
-/* CR */
-#define GET_RPUE(b) GET_GLOBAL_FIELD(b, CR, RPUE)
-#define GET_RPUERE(b) GET_GLOBAL_FIELD(b, CR, RPUERE)
-#define GET_RPUEIE(b) GET_GLOBAL_FIELD(b, CR, RPUEIE)
-#define GET_DCDEE(b) GET_GLOBAL_FIELD(b, CR, DCDEE)
-#define GET_CLIENTPD(b) GET_GLOBAL_FIELD(b, CR, CLIENTPD)
-#define GET_STALLD(b) GET_GLOBAL_FIELD(b, CR, STALLD)
-#define GET_TLBLKCRWE(b) GET_GLOBAL_FIELD(b, CR, TLBLKCRWE)
-#define GET_CR_TLBIALLCFG(b) GET_GLOBAL_FIELD(b, CR, CR_TLBIALLCFG)
-#define GET_TLBIVMIDCFG(b) GET_GLOBAL_FIELD(b, CR, TLBIVMIDCFG)
-#define GET_CR_HUME(b) GET_GLOBAL_FIELD(b, CR, CR_HUME)
-
-
-/* ESR */
-#define GET_CFG(b) GET_GLOBAL_FIELD(b, ESR, CFG)
-#define GET_BYPASS(b) GET_GLOBAL_FIELD(b, ESR, BYPASS)
-#define GET_ESR_MULTI(b) GET_GLOBAL_FIELD(b, ESR, ESR_MULTI)
-
-
-/* ESYNR0 */
-#define GET_ESYNR0_AMID(b) GET_GLOBAL_FIELD(b, ESYNR0, ESYNR0_AMID)
-#define GET_ESYNR0_APID(b) GET_GLOBAL_FIELD(b, ESYNR0, ESYNR0_APID)
-#define GET_ESYNR0_ABID(b) GET_GLOBAL_FIELD(b, ESYNR0, ESYNR0_ABID)
-#define GET_ESYNR0_AVMID(b) GET_GLOBAL_FIELD(b, ESYNR0, ESYNR0_AVMID)
-#define GET_ESYNR0_ATID(b) GET_GLOBAL_FIELD(b, ESYNR0, ESYNR0_ATID)
-
-
-/* ESYNR1 */
-#define GET_ESYNR1_AMEMTYPE(b) GET_GLOBAL_FIELD(b, ESYNR1, ESYNR1_AMEMTYPE)
-#define GET_ESYNR1_ASHARED(b) GET_GLOBAL_FIELD(b, ESYNR1, ESYNR1_ASHARED)
-#define GET_ESYNR1_AINNERSHARED(b) \
- GET_GLOBAL_FIELD(b, ESYNR1, ESYNR1_AINNERSHARED)
-#define GET_ESYNR1_APRIV(b) GET_GLOBAL_FIELD(b, ESYNR1, ESYNR1_APRIV)
-#define GET_ESYNR1_APROTNS(b) GET_GLOBAL_FIELD(b, ESYNR1, ESYNR1_APROTNS)
-#define GET_ESYNR1_AINST(b) GET_GLOBAL_FIELD(b, ESYNR1, ESYNR1_AINST)
-#define GET_ESYNR1_AWRITE(b) GET_GLOBAL_FIELD(b, ESYNR1, ESYNR1_AWRITE)
-#define GET_ESYNR1_ABURST(b) GET_GLOBAL_FIELD(b, ESYNR1, ESYNR1_ABURST)
-#define GET_ESYNR1_ALEN(b) GET_GLOBAL_FIELD(b, ESYNR1, ESYNR1_ALEN)
-#define GET_ESYNR1_ASIZE(b) GET_GLOBAL_FIELD(b, ESYNR1, ESYNR1_ASIZE)
-#define GET_ESYNR1_ALOCK(b) GET_GLOBAL_FIELD(b, ESYNR1, ESYNR1_ALOCK)
-#define GET_ESYNR1_AOOO(b) GET_GLOBAL_FIELD(b, ESYNR1, ESYNR1_AOOO)
-#define GET_ESYNR1_AFULL(b) GET_GLOBAL_FIELD(b, ESYNR1, ESYNR1_AFULL)
-#define GET_ESYNR1_AC(b) GET_GLOBAL_FIELD(b, ESYNR1, ESYNR1_AC)
-#define GET_ESYNR1_DCD(b) GET_GLOBAL_FIELD(b, ESYNR1, ESYNR1_DCD)
-
-
-/* IDR */
-#define GET_NM2VCBMT(b) GET_GLOBAL_FIELD(b, IDR, NM2VCBMT)
-#define GET_HTW(b) GET_GLOBAL_FIELD(b, IDR, HTW)
-#define GET_HUM(b) GET_GLOBAL_FIELD(b, IDR, HUM)
-#define GET_TLBSIZE(b) GET_GLOBAL_FIELD(b, IDR, TLBSIZE)
-#define GET_NCB(b) GET_GLOBAL_FIELD(b, IDR, NCB)
-#define GET_NIRPT(b) GET_GLOBAL_FIELD(b, IDR, NIRPT)
-
-
-/* REV */
-#define GET_MAJOR(b) GET_GLOBAL_FIELD(b, REV, MAJOR)
-#define GET_MINOR(b) GET_GLOBAL_FIELD(b, REV, MINOR)
-
-
-/* TESTBUSCR */
-#define GET_TBE(b) GET_GLOBAL_FIELD(b, TESTBUSCR, TBE)
-#define GET_SPDMBE(b) GET_GLOBAL_FIELD(b, TESTBUSCR, SPDMBE)
-#define GET_WGSEL(b) GET_GLOBAL_FIELD(b, TESTBUSCR, WGSEL)
-#define GET_TBLSEL(b) GET_GLOBAL_FIELD(b, TESTBUSCR, TBLSEL)
-#define GET_TBHSEL(b) GET_GLOBAL_FIELD(b, TESTBUSCR, TBHSEL)
-#define GET_SPDM0SEL(b) GET_GLOBAL_FIELD(b, TESTBUSCR, SPDM0SEL)
-#define GET_SPDM1SEL(b) GET_GLOBAL_FIELD(b, TESTBUSCR, SPDM1SEL)
-#define GET_SPDM2SEL(b) GET_GLOBAL_FIELD(b, TESTBUSCR, SPDM2SEL)
-#define GET_SPDM3SEL(b) GET_GLOBAL_FIELD(b, TESTBUSCR, SPDM3SEL)
-
-
-/* TLBIVMID */
-#define GET_TLBIVMID_VMID(b) GET_GLOBAL_FIELD(b, TLBIVMID, TLBIVMID_VMID)
-
-
-/* TLBTR0 */
-#define GET_PR(b) GET_GLOBAL_FIELD(b, TLBTR0, PR)
-#define GET_PW(b) GET_GLOBAL_FIELD(b, TLBTR0, PW)
-#define GET_UR(b) GET_GLOBAL_FIELD(b, TLBTR0, UR)
-#define GET_UW(b) GET_GLOBAL_FIELD(b, TLBTR0, UW)
-#define GET_XN(b) GET_GLOBAL_FIELD(b, TLBTR0, XN)
-#define GET_NSDESC(b) GET_GLOBAL_FIELD(b, TLBTR0, NSDESC)
-#define GET_ISH(b) GET_GLOBAL_FIELD(b, TLBTR0, ISH)
-#define GET_SH(b) GET_GLOBAL_FIELD(b, TLBTR0, SH)
-#define GET_MT(b) GET_GLOBAL_FIELD(b, TLBTR0, MT)
-#define GET_DPSIZR(b) GET_GLOBAL_FIELD(b, TLBTR0, DPSIZR)
-#define GET_DPSIZC(b) GET_GLOBAL_FIELD(b, TLBTR0, DPSIZC)
-
-
-/* TLBTR1 */
-#define GET_TLBTR1_VMID(b) GET_GLOBAL_FIELD(b, TLBTR1, TLBTR1_VMID)
-#define GET_TLBTR1_PA(b) GET_GLOBAL_FIELD(b, TLBTR1, TLBTR1_PA)
-
-
-/* TLBTR2 */
-#define GET_TLBTR2_ASID(b) GET_GLOBAL_FIELD(b, TLBTR2, TLBTR2_ASID)
-#define GET_TLBTR2_V(b) GET_GLOBAL_FIELD(b, TLBTR2, TLBTR2_V)
-#define GET_TLBTR2_NSTID(b) GET_GLOBAL_FIELD(b, TLBTR2, TLBTR2_NSTID)
-#define GET_TLBTR2_NV(b) GET_GLOBAL_FIELD(b, TLBTR2, TLBTR2_NV)
-#define GET_TLBTR2_VA(b) GET_GLOBAL_FIELD(b, TLBTR2, TLBTR2_VA)
-
-
-/* Context Register setters / getters */
-/* Context Register setters */
-/* ACTLR */
-#define SET_CFERE(b, c, v) SET_CONTEXT_FIELD(b, c, ACTLR, CFERE, v)
-#define SET_CFEIE(b, c, v) SET_CONTEXT_FIELD(b, c, ACTLR, CFEIE, v)
-#define SET_PTSHCFG(b, c, v) SET_CONTEXT_FIELD(b, c, ACTLR, PTSHCFG, v)
-#define SET_RCOSH(b, c, v) SET_CONTEXT_FIELD(b, c, ACTLR, RCOSH, v)
-#define SET_RCISH(b, c, v) SET_CONTEXT_FIELD(b, c, ACTLR, RCISH, v)
-#define SET_RCNSH(b, c, v) SET_CONTEXT_FIELD(b, c, ACTLR, RCNSH, v)
-#define SET_PRIVCFG(b, c, v) SET_CONTEXT_FIELD(b, c, ACTLR, PRIVCFG, v)
-#define SET_DNA(b, c, v) SET_CONTEXT_FIELD(b, c, ACTLR, DNA, v)
-#define SET_DNLV2PA(b, c, v) SET_CONTEXT_FIELD(b, c, ACTLR, DNLV2PA, v)
-#define SET_TLBMCFG(b, c, v) SET_CONTEXT_FIELD(b, c, ACTLR, TLBMCFG, v)
-#define SET_CFCFG(b, c, v) SET_CONTEXT_FIELD(b, c, ACTLR, CFCFG, v)
-#define SET_TIPCF(b, c, v) SET_CONTEXT_FIELD(b, c, ACTLR, TIPCF, v)
-#define SET_V2PCFG(b, c, v) SET_CONTEXT_FIELD(b, c, ACTLR, V2PCFG, v)
-#define SET_HUME(b, c, v) SET_CONTEXT_FIELD(b, c, ACTLR, HUME, v)
-#define SET_PTMTCFG(b, c, v) SET_CONTEXT_FIELD(b, c, ACTLR, PTMTCFG, v)
-#define SET_PTMEMTYPE(b, c, v) SET_CONTEXT_FIELD(b, c, ACTLR, PTMEMTYPE, v)
-
-
-/* BFBCR */
-#define SET_BFBDFE(b, c, v) SET_CONTEXT_FIELD(b, c, BFBCR, BFBDFE, v)
-#define SET_BFBSFE(b, c, v) SET_CONTEXT_FIELD(b, c, BFBCR, BFBSFE, v)
-#define SET_SFVS(b, c, v) SET_CONTEXT_FIELD(b, c, BFBCR, SFVS, v)
-#define SET_FLVIC(b, c, v) SET_CONTEXT_FIELD(b, c, BFBCR, FLVIC, v)
-#define SET_SLVIC(b, c, v) SET_CONTEXT_FIELD(b, c, BFBCR, SLVIC, v)
-
-
-/* CONTEXTIDR */
-#define SET_CONTEXTIDR_ASID(b, c, v) \
- SET_CONTEXT_FIELD(b, c, CONTEXTIDR, CONTEXTIDR_ASID, v)
-#define SET_CONTEXTIDR_PROCID(b, c, v) \
- SET_CONTEXT_FIELD(b, c, CONTEXTIDR, PROCID, v)
-
-
-/* FSR */
-#define SET_TF(b, c, v) SET_CONTEXT_FIELD(b, c, FSR, TF, v)
-#define SET_AFF(b, c, v) SET_CONTEXT_FIELD(b, c, FSR, AFF, v)
-#define SET_APF(b, c, v) SET_CONTEXT_FIELD(b, c, FSR, APF, v)
-#define SET_TLBMF(b, c, v) SET_CONTEXT_FIELD(b, c, FSR, TLBMF, v)
-#define SET_HTWDEEF(b, c, v) SET_CONTEXT_FIELD(b, c, FSR, HTWDEEF, v)
-#define SET_HTWSEEF(b, c, v) SET_CONTEXT_FIELD(b, c, FSR, HTWSEEF, v)
-#define SET_MHF(b, c, v) SET_CONTEXT_FIELD(b, c, FSR, MHF, v)
-#define SET_SL(b, c, v) SET_CONTEXT_FIELD(b, c, FSR, SL, v)
-#define SET_SS(b, c, v) SET_CONTEXT_FIELD(b, c, FSR, SS, v)
-#define SET_MULTI(b, c, v) SET_CONTEXT_FIELD(b, c, FSR, MULTI, v)
-
-
-/* FSYNR0 */
-#define SET_AMID(b, c, v) SET_CONTEXT_FIELD(b, c, FSYNR0, AMID, v)
-#define SET_APID(b, c, v) SET_CONTEXT_FIELD(b, c, FSYNR0, APID, v)
-#define SET_ABID(b, c, v) SET_CONTEXT_FIELD(b, c, FSYNR0, ABID, v)
-#define SET_ATID(b, c, v) SET_CONTEXT_FIELD(b, c, FSYNR0, ATID, v)
-
-
-/* FSYNR1 */
-#define SET_AMEMTYPE(b, c, v) SET_CONTEXT_FIELD(b, c, FSYNR1, AMEMTYPE, v)
-#define SET_ASHARED(b, c, v) SET_CONTEXT_FIELD(b, c, FSYNR1, ASHARED, v)
-#define SET_AINNERSHARED(b, c, v) \
- SET_CONTEXT_FIELD(b, c, FSYNR1, AINNERSHARED, v)
-#define SET_APRIV(b, c, v) SET_CONTEXT_FIELD(b, c, FSYNR1, APRIV, v)
-#define SET_APROTNS(b, c, v) SET_CONTEXT_FIELD(b, c, FSYNR1, APROTNS, v)
-#define SET_AINST(b, c, v) SET_CONTEXT_FIELD(b, c, FSYNR1, AINST, v)
-#define SET_AWRITE(b, c, v) SET_CONTEXT_FIELD(b, c, FSYNR1, AWRITE, v)
-#define SET_ABURST(b, c, v) SET_CONTEXT_FIELD(b, c, FSYNR1, ABURST, v)
-#define SET_ALEN(b, c, v) SET_CONTEXT_FIELD(b, c, FSYNR1, ALEN, v)
-#define SET_FSYNR1_ASIZE(b, c, v) \
- SET_CONTEXT_FIELD(b, c, FSYNR1, FSYNR1_ASIZE, v)
-#define SET_ALOCK(b, c, v) SET_CONTEXT_FIELD(b, c, FSYNR1, ALOCK, v)
-#define SET_AFULL(b, c, v) SET_CONTEXT_FIELD(b, c, FSYNR1, AFULL, v)
-
-
-/* NMRR */
-#define SET_ICPC0(b, c, v) SET_CONTEXT_FIELD(b, c, NMRR, ICPC0, v)
-#define SET_ICPC1(b, c, v) SET_CONTEXT_FIELD(b, c, NMRR, ICPC1, v)
-#define SET_ICPC2(b, c, v) SET_CONTEXT_FIELD(b, c, NMRR, ICPC2, v)
-#define SET_ICPC3(b, c, v) SET_CONTEXT_FIELD(b, c, NMRR, ICPC3, v)
-#define SET_ICPC4(b, c, v) SET_CONTEXT_FIELD(b, c, NMRR, ICPC4, v)
-#define SET_ICPC5(b, c, v) SET_CONTEXT_FIELD(b, c, NMRR, ICPC5, v)
-#define SET_ICPC6(b, c, v) SET_CONTEXT_FIELD(b, c, NMRR, ICPC6, v)
-#define SET_ICPC7(b, c, v) SET_CONTEXT_FIELD(b, c, NMRR, ICPC7, v)
-#define SET_OCPC0(b, c, v) SET_CONTEXT_FIELD(b, c, NMRR, OCPC0, v)
-#define SET_OCPC1(b, c, v) SET_CONTEXT_FIELD(b, c, NMRR, OCPC1, v)
-#define SET_OCPC2(b, c, v) SET_CONTEXT_FIELD(b, c, NMRR, OCPC2, v)
-#define SET_OCPC3(b, c, v) SET_CONTEXT_FIELD(b, c, NMRR, OCPC3, v)
-#define SET_OCPC4(b, c, v) SET_CONTEXT_FIELD(b, c, NMRR, OCPC4, v)
-#define SET_OCPC5(b, c, v) SET_CONTEXT_FIELD(b, c, NMRR, OCPC5, v)
-#define SET_OCPC6(b, c, v) SET_CONTEXT_FIELD(b, c, NMRR, OCPC6, v)
-#define SET_OCPC7(b, c, v) SET_CONTEXT_FIELD(b, c, NMRR, OCPC7, v)
-
-
-/* PAR */
-#define SET_FAULT(b, c, v) SET_CONTEXT_FIELD(b, c, PAR, FAULT, v)
-
-#define SET_FAULT_TF(b, c, v) SET_CONTEXT_FIELD(b, c, PAR, FAULT_TF, v)
-#define SET_FAULT_AFF(b, c, v) SET_CONTEXT_FIELD(b, c, PAR, FAULT_AFF, v)
-#define SET_FAULT_APF(b, c, v) SET_CONTEXT_FIELD(b, c, PAR, FAULT_APF, v)
-#define SET_FAULT_TLBMF(b, c, v) SET_CONTEXT_FIELD(b, c, PAR, FAULT_TLBMF, v)
-#define SET_FAULT_HTWDEEF(b, c, v) \
- SET_CONTEXT_FIELD(b, c, PAR, FAULT_HTWDEEF, v)
-#define SET_FAULT_HTWSEEF(b, c, v) \
- SET_CONTEXT_FIELD(b, c, PAR, FAULT_HTWSEEF, v)
-#define SET_FAULT_MHF(b, c, v) SET_CONTEXT_FIELD(b, c, PAR, FAULT_MHF, v)
-#define SET_FAULT_SL(b, c, v) SET_CONTEXT_FIELD(b, c, PAR, FAULT_SL, v)
-#define SET_FAULT_SS(b, c, v) SET_CONTEXT_FIELD(b, c, PAR, FAULT_SS, v)
-
-#define SET_NOFAULT_SS(b, c, v) SET_CONTEXT_FIELD(b, c, PAR, NOFAULT_SS, v)
-#define SET_NOFAULT_MT(b, c, v) SET_CONTEXT_FIELD(b, c, PAR, NOFAULT_MT, v)
-#define SET_NOFAULT_SH(b, c, v) SET_CONTEXT_FIELD(b, c, PAR, NOFAULT_SH, v)
-#define SET_NOFAULT_NS(b, c, v) SET_CONTEXT_FIELD(b, c, PAR, NOFAULT_NS, v)
-#define SET_NOFAULT_NOS(b, c, v) SET_CONTEXT_FIELD(b, c, PAR, NOFAULT_NOS, v)
-#define SET_NPFAULT_PA(b, c, v) SET_CONTEXT_FIELD(b, c, PAR, NPFAULT_PA, v)
-
-
-/* PRRR */
-#define SET_MTC0(b, c, v) SET_CONTEXT_FIELD(b, c, PRRR, MTC0, v)
-#define SET_MTC1(b, c, v) SET_CONTEXT_FIELD(b, c, PRRR, MTC1, v)
-#define SET_MTC2(b, c, v) SET_CONTEXT_FIELD(b, c, PRRR, MTC2, v)
-#define SET_MTC3(b, c, v) SET_CONTEXT_FIELD(b, c, PRRR, MTC3, v)
-#define SET_MTC4(b, c, v) SET_CONTEXT_FIELD(b, c, PRRR, MTC4, v)
-#define SET_MTC5(b, c, v) SET_CONTEXT_FIELD(b, c, PRRR, MTC5, v)
-#define SET_MTC6(b, c, v) SET_CONTEXT_FIELD(b, c, PRRR, MTC6, v)
-#define SET_MTC7(b, c, v) SET_CONTEXT_FIELD(b, c, PRRR, MTC7, v)
-#define SET_SHDSH0(b, c, v) SET_CONTEXT_FIELD(b, c, PRRR, SHDSH0, v)
-#define SET_SHDSH1(b, c, v) SET_CONTEXT_FIELD(b, c, PRRR, SHDSH1, v)
-#define SET_SHNMSH0(b, c, v) SET_CONTEXT_FIELD(b, c, PRRR, SHNMSH0, v)
-#define SET_SHNMSH1(b, c, v) SET_CONTEXT_FIELD(b, c, PRRR, SHNMSH1, v)
-#define SET_NOS0(b, c, v) SET_CONTEXT_FIELD(b, c, PRRR, NOS0, v)
-#define SET_NOS1(b, c, v) SET_CONTEXT_FIELD(b, c, PRRR, NOS1, v)
-#define SET_NOS2(b, c, v) SET_CONTEXT_FIELD(b, c, PRRR, NOS2, v)
-#define SET_NOS3(b, c, v) SET_CONTEXT_FIELD(b, c, PRRR, NOS3, v)
-#define SET_NOS4(b, c, v) SET_CONTEXT_FIELD(b, c, PRRR, NOS4, v)
-#define SET_NOS5(b, c, v) SET_CONTEXT_FIELD(b, c, PRRR, NOS5, v)
-#define SET_NOS6(b, c, v) SET_CONTEXT_FIELD(b, c, PRRR, NOS6, v)
-#define SET_NOS7(b, c, v) SET_CONTEXT_FIELD(b, c, PRRR, NOS7, v)
-
-
-/* RESUME */
-#define SET_TNR(b, c, v) SET_CONTEXT_FIELD(b, c, RESUME, TNR, v)
-
-
-/* SCTLR */
-#define SET_M(b, c, v) SET_CONTEXT_FIELD(b, c, SCTLR, M, v)
-#define SET_TRE(b, c, v) SET_CONTEXT_FIELD(b, c, SCTLR, TRE, v)
-#define SET_AFE(b, c, v) SET_CONTEXT_FIELD(b, c, SCTLR, AFE, v)
-#define SET_HAF(b, c, v) SET_CONTEXT_FIELD(b, c, SCTLR, HAF, v)
-#define SET_BE(b, c, v) SET_CONTEXT_FIELD(b, c, SCTLR, BE, v)
-#define SET_AFFD(b, c, v) SET_CONTEXT_FIELD(b, c, SCTLR, AFFD, v)
-
-
-/* TLBLKCR */
-#define SET_LKE(b, c, v) SET_CONTEXT_FIELD(b, c, TLBLKCR, LKE, v)
-#define SET_TLBLKCR_TLBIALLCFG(b, c, v) \
- SET_CONTEXT_FIELD(b, c, TLBLKCR, TLBLCKR_TLBIALLCFG, v)
-#define SET_TLBIASIDCFG(b, c, v) \
- SET_CONTEXT_FIELD(b, c, TLBLKCR, TLBIASIDCFG, v)
-#define SET_TLBIVAACFG(b, c, v) SET_CONTEXT_FIELD(b, c, TLBLKCR, TLBIVAACFG, v)
-#define SET_FLOOR(b, c, v) SET_CONTEXT_FIELD(b, c, TLBLKCR, FLOOR, v)
-#define SET_VICTIM(b, c, v) SET_CONTEXT_FIELD(b, c, TLBLKCR, VICTIM, v)
-
-
-/* TTBCR */
-#define SET_N(b, c, v) SET_CONTEXT_FIELD(b, c, TTBCR, N, v)
-#define SET_PD0(b, c, v) SET_CONTEXT_FIELD(b, c, TTBCR, PD0, v)
-#define SET_PD1(b, c, v) SET_CONTEXT_FIELD(b, c, TTBCR, PD1, v)
-
-
-/* TTBR0 */
-#define SET_TTBR0_IRGNH(b, c, v) SET_CONTEXT_FIELD(b, c, TTBR0, TTBR0_IRGNH, v)
-#define SET_TTBR0_SH(b, c, v) SET_CONTEXT_FIELD(b, c, TTBR0, TTBR0_SH, v)
-#define SET_TTBR0_ORGN(b, c, v) SET_CONTEXT_FIELD(b, c, TTBR0, TTBR0_ORGN, v)
-#define SET_TTBR0_NOS(b, c, v) SET_CONTEXT_FIELD(b, c, TTBR0, TTBR0_NOS, v)
-#define SET_TTBR0_IRGNL(b, c, v) SET_CONTEXT_FIELD(b, c, TTBR0, TTBR0_IRGNL, v)
-#define SET_TTBR0_PA(b, c, v) SET_CONTEXT_FIELD(b, c, TTBR0, TTBR0_PA, v)
-
-
-/* TTBR1 */
-#define SET_TTBR1_IRGNH(b, c, v) SET_CONTEXT_FIELD(b, c, TTBR1, TTBR1_IRGNH, v)
-#define SET_TTBR1_SH(b, c, v) SET_CONTEXT_FIELD(b, c, TTBR1, TTBR1_SH, v)
-#define SET_TTBR1_ORGN(b, c, v) SET_CONTEXT_FIELD(b, c, TTBR1, TTBR1_ORGN, v)
-#define SET_TTBR1_NOS(b, c, v) SET_CONTEXT_FIELD(b, c, TTBR1, TTBR1_NOS, v)
-#define SET_TTBR1_IRGNL(b, c, v) SET_CONTEXT_FIELD(b, c, TTBR1, TTBR1_IRGNL, v)
-#define SET_TTBR1_PA(b, c, v) SET_CONTEXT_FIELD(b, c, TTBR1, TTBR1_PA, v)
-
-
-/* V2PSR */
-#define SET_HIT(b, c, v) SET_CONTEXT_FIELD(b, c, V2PSR, HIT, v)
-#define SET_INDEX(b, c, v) SET_CONTEXT_FIELD(b, c, V2PSR, INDEX, v)
-
-
-/* Context Register getters */
-/* ACTLR */
-#define GET_CFERE(b, c) GET_CONTEXT_FIELD(b, c, ACTLR, CFERE)
-#define GET_CFEIE(b, c) GET_CONTEXT_FIELD(b, c, ACTLR, CFEIE)
-#define GET_PTSHCFG(b, c) GET_CONTEXT_FIELD(b, c, ACTLR, PTSHCFG)
-#define GET_RCOSH(b, c) GET_CONTEXT_FIELD(b, c, ACTLR, RCOSH)
-#define GET_RCISH(b, c) GET_CONTEXT_FIELD(b, c, ACTLR, RCISH)
-#define GET_RCNSH(b, c) GET_CONTEXT_FIELD(b, c, ACTLR, RCNSH)
-#define GET_PRIVCFG(b, c) GET_CONTEXT_FIELD(b, c, ACTLR, PRIVCFG)
-#define GET_DNA(b, c) GET_CONTEXT_FIELD(b, c, ACTLR, DNA)
-#define GET_DNLV2PA(b, c) GET_CONTEXT_FIELD(b, c, ACTLR, DNLV2PA)
-#define GET_TLBMCFG(b, c) GET_CONTEXT_FIELD(b, c, ACTLR, TLBMCFG)
-#define GET_CFCFG(b, c) GET_CONTEXT_FIELD(b, c, ACTLR, CFCFG)
-#define GET_TIPCF(b, c) GET_CONTEXT_FIELD(b, c, ACTLR, TIPCF)
-#define GET_V2PCFG(b, c) GET_CONTEXT_FIELD(b, c, ACTLR, V2PCFG)
-#define GET_HUME(b, c) GET_CONTEXT_FIELD(b, c, ACTLR, HUME)
-#define GET_PTMTCFG(b, c) GET_CONTEXT_FIELD(b, c, ACTLR, PTMTCFG)
-#define GET_PTMEMTYPE(b, c) GET_CONTEXT_FIELD(b, c, ACTLR, PTMEMTYPE)
-
-/* BFBCR */
-#define GET_BFBDFE(b, c) GET_CONTEXT_FIELD(b, c, BFBCR, BFBDFE)
-#define GET_BFBSFE(b, c) GET_CONTEXT_FIELD(b, c, BFBCR, BFBSFE)
-#define GET_SFVS(b, c) GET_CONTEXT_FIELD(b, c, BFBCR, SFVS)
-#define GET_FLVIC(b, c) GET_CONTEXT_FIELD(b, c, BFBCR, FLVIC)
-#define GET_SLVIC(b, c) GET_CONTEXT_FIELD(b, c, BFBCR, SLVIC)
-
-
-/* CONTEXTIDR */
-#define GET_CONTEXTIDR_ASID(b, c) \
- GET_CONTEXT_FIELD(b, c, CONTEXTIDR, CONTEXTIDR_ASID)
-#define GET_CONTEXTIDR_PROCID(b, c) GET_CONTEXT_FIELD(b, c, CONTEXTIDR, PROCID)
-
-
-/* FSR */
-#define GET_TF(b, c) GET_CONTEXT_FIELD(b, c, FSR, TF)
-#define GET_AFF(b, c) GET_CONTEXT_FIELD(b, c, FSR, AFF)
-#define GET_APF(b, c) GET_CONTEXT_FIELD(b, c, FSR, APF)
-#define GET_TLBMF(b, c) GET_CONTEXT_FIELD(b, c, FSR, TLBMF)
-#define GET_HTWDEEF(b, c) GET_CONTEXT_FIELD(b, c, FSR, HTWDEEF)
-#define GET_HTWSEEF(b, c) GET_CONTEXT_FIELD(b, c, FSR, HTWSEEF)
-#define GET_MHF(b, c) GET_CONTEXT_FIELD(b, c, FSR, MHF)
-#define GET_SL(b, c) GET_CONTEXT_FIELD(b, c, FSR, SL)
-#define GET_SS(b, c) GET_CONTEXT_FIELD(b, c, FSR, SS)
-#define GET_MULTI(b, c) GET_CONTEXT_FIELD(b, c, FSR, MULTI)
-
-
-/* FSYNR0 */
-#define GET_AMID(b, c) GET_CONTEXT_FIELD(b, c, FSYNR0, AMID)
-#define GET_APID(b, c) GET_CONTEXT_FIELD(b, c, FSYNR0, APID)
-#define GET_ABID(b, c) GET_CONTEXT_FIELD(b, c, FSYNR0, ABID)
-#define GET_ATID(b, c) GET_CONTEXT_FIELD(b, c, FSYNR0, ATID)
-
-
-/* FSYNR1 */
-#define GET_AMEMTYPE(b, c) GET_CONTEXT_FIELD(b, c, FSYNR1, AMEMTYPE)
-#define GET_ASHARED(b, c) GET_CONTEXT_FIELD(b, c, FSYNR1, ASHARED)
-#define GET_AINNERSHARED(b, c) GET_CONTEXT_FIELD(b, c, FSYNR1, AINNERSHARED)
-#define GET_APRIV(b, c) GET_CONTEXT_FIELD(b, c, FSYNR1, APRIV)
-#define GET_APROTNS(b, c) GET_CONTEXT_FIELD(b, c, FSYNR1, APROTNS)
-#define GET_AINST(b, c) GET_CONTEXT_FIELD(b, c, FSYNR1, AINST)
-#define GET_AWRITE(b, c) GET_CONTEXT_FIELD(b, c, FSYNR1, AWRITE)
-#define GET_ABURST(b, c) GET_CONTEXT_FIELD(b, c, FSYNR1, ABURST)
-#define GET_ALEN(b, c) GET_CONTEXT_FIELD(b, c, FSYNR1, ALEN)
-#define GET_FSYNR1_ASIZE(b, c) GET_CONTEXT_FIELD(b, c, FSYNR1, FSYNR1_ASIZE)
-#define GET_ALOCK(b, c) GET_CONTEXT_FIELD(b, c, FSYNR1, ALOCK)
-#define GET_AFULL(b, c) GET_CONTEXT_FIELD(b, c, FSYNR1, AFULL)
-
-
-/* NMRR */
-#define GET_ICPC0(b, c) GET_CONTEXT_FIELD(b, c, NMRR, ICPC0)
-#define GET_ICPC1(b, c) GET_CONTEXT_FIELD(b, c, NMRR, ICPC1)
-#define GET_ICPC2(b, c) GET_CONTEXT_FIELD(b, c, NMRR, ICPC2)
-#define GET_ICPC3(b, c) GET_CONTEXT_FIELD(b, c, NMRR, ICPC3)
-#define GET_ICPC4(b, c) GET_CONTEXT_FIELD(b, c, NMRR, ICPC4)
-#define GET_ICPC5(b, c) GET_CONTEXT_FIELD(b, c, NMRR, ICPC5)
-#define GET_ICPC6(b, c) GET_CONTEXT_FIELD(b, c, NMRR, ICPC6)
-#define GET_ICPC7(b, c) GET_CONTEXT_FIELD(b, c, NMRR, ICPC7)
-#define GET_OCPC0(b, c) GET_CONTEXT_FIELD(b, c, NMRR, OCPC0)
-#define GET_OCPC1(b, c) GET_CONTEXT_FIELD(b, c, NMRR, OCPC1)
-#define GET_OCPC2(b, c) GET_CONTEXT_FIELD(b, c, NMRR, OCPC2)
-#define GET_OCPC3(b, c) GET_CONTEXT_FIELD(b, c, NMRR, OCPC3)
-#define GET_OCPC4(b, c) GET_CONTEXT_FIELD(b, c, NMRR, OCPC4)
-#define GET_OCPC5(b, c) GET_CONTEXT_FIELD(b, c, NMRR, OCPC5)
-#define GET_OCPC6(b, c) GET_CONTEXT_FIELD(b, c, NMRR, OCPC6)
-#define GET_OCPC7(b, c) GET_CONTEXT_FIELD(b, c, NMRR, OCPC7)
-#define NMRR_ICP(nmrr, n) (((nmrr) & (3 << ((n) * 2))) >> ((n) * 2))
-#define NMRR_OCP(nmrr, n) (((nmrr) & (3 << ((n) * 2 + 16))) >> \
- ((n) * 2 + 16))
-
-/* PAR */
-#define GET_FAULT(b, c) GET_CONTEXT_FIELD(b, c, PAR, FAULT)
-
-#define GET_FAULT_TF(b, c) GET_CONTEXT_FIELD(b, c, PAR, FAULT_TF)
-#define GET_FAULT_AFF(b, c) GET_CONTEXT_FIELD(b, c, PAR, FAULT_AFF)
-#define GET_FAULT_APF(b, c) GET_CONTEXT_FIELD(b, c, PAR, FAULT_APF)
-#define GET_FAULT_TLBMF(b, c) GET_CONTEXT_FIELD(b, c, PAR, FAULT_TLBMF)
-#define GET_FAULT_HTWDEEF(b, c) GET_CONTEXT_FIELD(b, c, PAR, FAULT_HTWDEEF)
-#define GET_FAULT_HTWSEEF(b, c) GET_CONTEXT_FIELD(b, c, PAR, FAULT_HTWSEEF)
-#define GET_FAULT_MHF(b, c) GET_CONTEXT_FIELD(b, c, PAR, FAULT_MHF)
-#define GET_FAULT_SL(b, c) GET_CONTEXT_FIELD(b, c, PAR, FAULT_SL)
-#define GET_FAULT_SS(b, c) GET_CONTEXT_FIELD(b, c, PAR, FAULT_SS)
-
-#define GET_NOFAULT_SS(b, c) GET_CONTEXT_FIELD(b, c, PAR, PAR_NOFAULT_SS)
-#define GET_NOFAULT_MT(b, c) GET_CONTEXT_FIELD(b, c, PAR, PAR_NOFAULT_MT)
-#define GET_NOFAULT_SH(b, c) GET_CONTEXT_FIELD(b, c, PAR, PAR_NOFAULT_SH)
-#define GET_NOFAULT_NS(b, c) GET_CONTEXT_FIELD(b, c, PAR, PAR_NOFAULT_NS)
-#define GET_NOFAULT_NOS(b, c) GET_CONTEXT_FIELD(b, c, PAR, PAR_NOFAULT_NOS)
-#define GET_NPFAULT_PA(b, c) GET_CONTEXT_FIELD(b, c, PAR, PAR_NPFAULT_PA)
-
-
-/* PRRR */
-#define GET_MTC0(b, c) GET_CONTEXT_FIELD(b, c, PRRR, MTC0)
-#define GET_MTC1(b, c) GET_CONTEXT_FIELD(b, c, PRRR, MTC1)
-#define GET_MTC2(b, c) GET_CONTEXT_FIELD(b, c, PRRR, MTC2)
-#define GET_MTC3(b, c) GET_CONTEXT_FIELD(b, c, PRRR, MTC3)
-#define GET_MTC4(b, c) GET_CONTEXT_FIELD(b, c, PRRR, MTC4)
-#define GET_MTC5(b, c) GET_CONTEXT_FIELD(b, c, PRRR, MTC5)
-#define GET_MTC6(b, c) GET_CONTEXT_FIELD(b, c, PRRR, MTC6)
-#define GET_MTC7(b, c) GET_CONTEXT_FIELD(b, c, PRRR, MTC7)
-#define GET_SHDSH0(b, c) GET_CONTEXT_FIELD(b, c, PRRR, SHDSH0)
-#define GET_SHDSH1(b, c) GET_CONTEXT_FIELD(b, c, PRRR, SHDSH1)
-#define GET_SHNMSH0(b, c) GET_CONTEXT_FIELD(b, c, PRRR, SHNMSH0)
-#define GET_SHNMSH1(b, c) GET_CONTEXT_FIELD(b, c, PRRR, SHNMSH1)
-#define GET_NOS0(b, c) GET_CONTEXT_FIELD(b, c, PRRR, NOS0)
-#define GET_NOS1(b, c) GET_CONTEXT_FIELD(b, c, PRRR, NOS1)
-#define GET_NOS2(b, c) GET_CONTEXT_FIELD(b, c, PRRR, NOS2)
-#define GET_NOS3(b, c) GET_CONTEXT_FIELD(b, c, PRRR, NOS3)
-#define GET_NOS4(b, c) GET_CONTEXT_FIELD(b, c, PRRR, NOS4)
-#define GET_NOS5(b, c) GET_CONTEXT_FIELD(b, c, PRRR, NOS5)
-#define GET_NOS6(b, c) GET_CONTEXT_FIELD(b, c, PRRR, NOS6)
-#define GET_NOS7(b, c) GET_CONTEXT_FIELD(b, c, PRRR, NOS7)
-#define PRRR_NOS(prrr, n) ((prrr) & (1 << ((n) + 24)) ? 1 : 0)
-#define PRRR_MT(prrr, n) ((((prrr) & (3 << ((n) * 2))) >> ((n) * 2)))
-
-
-/* RESUME */
-#define GET_TNR(b, c) GET_CONTEXT_FIELD(b, c, RESUME, TNR)
-
-
-/* SCTLR */
-#define GET_M(b, c) GET_CONTEXT_FIELD(b, c, SCTLR, M)
-#define GET_TRE(b, c) GET_CONTEXT_FIELD(b, c, SCTLR, TRE)
-#define GET_AFE(b, c) GET_CONTEXT_FIELD(b, c, SCTLR, AFE)
-#define GET_HAF(b, c) GET_CONTEXT_FIELD(b, c, SCTLR, HAF)
-#define GET_BE(b, c) GET_CONTEXT_FIELD(b, c, SCTLR, BE)
-#define GET_AFFD(b, c) GET_CONTEXT_FIELD(b, c, SCTLR, AFFD)
-
-
-/* TLBLKCR */
-#define GET_LKE(b, c) GET_CONTEXT_FIELD(b, c, TLBLKCR, LKE)
-#define GET_TLBLCKR_TLBIALLCFG(b, c) \
- GET_CONTEXT_FIELD(b, c, TLBLKCR, TLBLCKR_TLBIALLCFG)
-#define GET_TLBIASIDCFG(b, c) GET_CONTEXT_FIELD(b, c, TLBLKCR, TLBIASIDCFG)
-#define GET_TLBIVAACFG(b, c) GET_CONTEXT_FIELD(b, c, TLBLKCR, TLBIVAACFG)
-#define GET_FLOOR(b, c) GET_CONTEXT_FIELD(b, c, TLBLKCR, FLOOR)
-#define GET_VICTIM(b, c) GET_CONTEXT_FIELD(b, c, TLBLKCR, VICTIM)
-
-
-/* TTBCR */
-#define GET_N(b, c) GET_CONTEXT_FIELD(b, c, TTBCR, N)
-#define GET_PD0(b, c) GET_CONTEXT_FIELD(b, c, TTBCR, PD0)
-#define GET_PD1(b, c) GET_CONTEXT_FIELD(b, c, TTBCR, PD1)
-
-
-/* TTBR0 */
-#define GET_TTBR0_IRGNH(b, c) GET_CONTEXT_FIELD(b, c, TTBR0, TTBR0_IRGNH)
-#define GET_TTBR0_SH(b, c) GET_CONTEXT_FIELD(b, c, TTBR0, TTBR0_SH)
-#define GET_TTBR0_ORGN(b, c) GET_CONTEXT_FIELD(b, c, TTBR0, TTBR0_ORGN)
-#define GET_TTBR0_NOS(b, c) GET_CONTEXT_FIELD(b, c, TTBR0, TTBR0_NOS)
-#define GET_TTBR0_IRGNL(b, c) GET_CONTEXT_FIELD(b, c, TTBR0, TTBR0_IRGNL)
-#define GET_TTBR0_PA(b, c) GET_CONTEXT_FIELD(b, c, TTBR0, TTBR0_PA)
-
-
-/* TTBR1 */
-#define GET_TTBR1_IRGNH(b, c) GET_CONTEXT_FIELD(b, c, TTBR1, TTBR1_IRGNH)
-#define GET_TTBR1_SH(b, c) GET_CONTEXT_FIELD(b, c, TTBR1, TTBR1_SH)
-#define GET_TTBR1_ORGN(b, c) GET_CONTEXT_FIELD(b, c, TTBR1, TTBR1_ORGN)
-#define GET_TTBR1_NOS(b, c) GET_CONTEXT_FIELD(b, c, TTBR1, TTBR1_NOS)
-#define GET_TTBR1_IRGNL(b, c) GET_CONTEXT_FIELD(b, c, TTBR1, TTBR1_IRGNL)
-#define GET_TTBR1_PA(b, c) GET_CONTEXT_FIELD(b, c, TTBR1, TTBR1_PA)
-
-
-/* V2PSR */
-#define GET_HIT(b, c) GET_CONTEXT_FIELD(b, c, V2PSR, HIT)
-#define GET_INDEX(b, c) GET_CONTEXT_FIELD(b, c, V2PSR, INDEX)
-
-
-/* Global Registers */
-#define M2VCBR_N (0xFF000)
-#define CBACR_N (0xFF800)
-#define TLBRSW (0xFFE00)
-#define TLBTR0 (0xFFE80)
-#define TLBTR1 (0xFFE84)
-#define TLBTR2 (0xFFE88)
-#define TESTBUSCR (0xFFE8C)
-#define GLOBAL_TLBIALL (0xFFF00)
-#define TLBIVMID (0xFFF04)
-#define CR (0xFFF80)
-#define EAR (0xFFF84)
-#define ESR (0xFFF88)
-#define ESRRESTORE (0xFFF8C)
-#define ESYNR0 (0xFFF90)
-#define ESYNR1 (0xFFF94)
-#define REV (0xFFFF4)
-#define IDR (0xFFFF8)
-#define RPU_ACR (0xFFFFC)
-
-
-/* Context Bank Registers */
-#define SCTLR (0x000)
-#define ACTLR (0x004)
-#define CONTEXTIDR (0x008)
-#define TTBR0 (0x010)
-#define TTBR1 (0x014)
-#define TTBCR (0x018)
-#define PAR (0x01C)
-#define FSR (0x020)
-#define FSRRESTORE (0x024)
-#define FAR (0x028)
-#define FSYNR0 (0x02C)
-#define FSYNR1 (0x030)
-#define PRRR (0x034)
-#define NMRR (0x038)
-#define TLBLCKR (0x03C)
-#define V2PSR (0x040)
-#define TLBFLPTER (0x044)
-#define TLBSLPTER (0x048)
-#define BFBCR (0x04C)
-#define CTX_TLBIALL (0x800)
-#define TLBIASID (0x804)
-#define TLBIVA (0x808)
-#define TLBIVAA (0x80C)
-#define V2PPR (0x810)
-#define V2PPW (0x814)
-#define V2PUR (0x818)
-#define V2PUW (0x81C)
-#define RESUME (0x820)
-
-
-/* Global Register Fields */
-/* CBACRn */
-#define RWVMID (RWVMID_MASK << RWVMID_SHIFT)
-#define RWE (RWE_MASK << RWE_SHIFT)
-#define RWGE (RWGE_MASK << RWGE_SHIFT)
-#define CBVMID (CBVMID_MASK << CBVMID_SHIFT)
-#define IRPTNDX (IRPTNDX_MASK << IRPTNDX_SHIFT)
-
-
-/* CR */
-#define RPUE (RPUE_MASK << RPUE_SHIFT)
-#define RPUERE (RPUERE_MASK << RPUERE_SHIFT)
-#define RPUEIE (RPUEIE_MASK << RPUEIE_SHIFT)
-#define DCDEE (DCDEE_MASK << DCDEE_SHIFT)
-#define CLIENTPD (CLIENTPD_MASK << CLIENTPD_SHIFT)
-#define STALLD (STALLD_MASK << STALLD_SHIFT)
-#define TLBLKCRWE (TLBLKCRWE_MASK << TLBLKCRWE_SHIFT)
-#define CR_TLBIALLCFG (CR_TLBIALLCFG_MASK << CR_TLBIALLCFG_SHIFT)
-#define TLBIVMIDCFG (TLBIVMIDCFG_MASK << TLBIVMIDCFG_SHIFT)
-#define CR_HUME (CR_HUME_MASK << CR_HUME_SHIFT)
-
-
-/* ESR */
-#define CFG (CFG_MASK << CFG_SHIFT)
-#define BYPASS (BYPASS_MASK << BYPASS_SHIFT)
-#define ESR_MULTI (ESR_MULTI_MASK << ESR_MULTI_SHIFT)
-
-
-/* ESYNR0 */
-#define ESYNR0_AMID (ESYNR0_AMID_MASK << ESYNR0_AMID_SHIFT)
-#define ESYNR0_APID (ESYNR0_APID_MASK << ESYNR0_APID_SHIFT)
-#define ESYNR0_ABID (ESYNR0_ABID_MASK << ESYNR0_ABID_SHIFT)
-#define ESYNR0_AVMID (ESYNR0_AVMID_MASK << ESYNR0_AVMID_SHIFT)
-#define ESYNR0_ATID (ESYNR0_ATID_MASK << ESYNR0_ATID_SHIFT)
-
-
-/* ESYNR1 */
-#define ESYNR1_AMEMTYPE (ESYNR1_AMEMTYPE_MASK << ESYNR1_AMEMTYPE_SHIFT)
-#define ESYNR1_ASHARED (ESYNR1_ASHARED_MASK << ESYNR1_ASHARED_SHIFT)
-#define ESYNR1_AINNERSHARED (ESYNR1_AINNERSHARED_MASK<< \
- ESYNR1_AINNERSHARED_SHIFT)
-#define ESYNR1_APRIV (ESYNR1_APRIV_MASK << ESYNR1_APRIV_SHIFT)
-#define ESYNR1_APROTNS (ESYNR1_APROTNS_MASK << ESYNR1_APROTNS_SHIFT)
-#define ESYNR1_AINST (ESYNR1_AINST_MASK << ESYNR1_AINST_SHIFT)
-#define ESYNR1_AWRITE (ESYNR1_AWRITE_MASK << ESYNR1_AWRITE_SHIFT)
-#define ESYNR1_ABURST (ESYNR1_ABURST_MASK << ESYNR1_ABURST_SHIFT)
-#define ESYNR1_ALEN (ESYNR1_ALEN_MASK << ESYNR1_ALEN_SHIFT)
-#define ESYNR1_ASIZE (ESYNR1_ASIZE_MASK << ESYNR1_ASIZE_SHIFT)
-#define ESYNR1_ALOCK (ESYNR1_ALOCK_MASK << ESYNR1_ALOCK_SHIFT)
-#define ESYNR1_AOOO (ESYNR1_AOOO_MASK << ESYNR1_AOOO_SHIFT)
-#define ESYNR1_AFULL (ESYNR1_AFULL_MASK << ESYNR1_AFULL_SHIFT)
-#define ESYNR1_AC (ESYNR1_AC_MASK << ESYNR1_AC_SHIFT)
-#define ESYNR1_DCD (ESYNR1_DCD_MASK << ESYNR1_DCD_SHIFT)
-
-
-/* IDR */
-#define NM2VCBMT (NM2VCBMT_MASK << NM2VCBMT_SHIFT)
-#define HTW (HTW_MASK << HTW_SHIFT)
-#define HUM (HUM_MASK << HUM_SHIFT)
-#define TLBSIZE (TLBSIZE_MASK << TLBSIZE_SHIFT)
-#define NCB (NCB_MASK << NCB_SHIFT)
-#define NIRPT (NIRPT_MASK << NIRPT_SHIFT)
-
-
-/* M2VCBRn */
-#define VMID (VMID_MASK << VMID_SHIFT)
-#define CBNDX (CBNDX_MASK << CBNDX_SHIFT)
-#define BYPASSD (BYPASSD_MASK << BYPASSD_SHIFT)
-#define BPRCOSH (BPRCOSH_MASK << BPRCOSH_SHIFT)
-#define BPRCISH (BPRCISH_MASK << BPRCISH_SHIFT)
-#define BPRCNSH (BPRCNSH_MASK << BPRCNSH_SHIFT)
-#define BPSHCFG (BPSHCFG_MASK << BPSHCFG_SHIFT)
-#define NSCFG (NSCFG_MASK << NSCFG_SHIFT)
-#define BPMTCFG (BPMTCFG_MASK << BPMTCFG_SHIFT)
-#define BPMEMTYPE (BPMEMTYPE_MASK << BPMEMTYPE_SHIFT)
-
-
-/* REV */
-#define IDR_MINOR (MINOR_MASK << MINOR_SHIFT)
-#define IDR_MAJOR (MAJOR_MASK << MAJOR_SHIFT)
-
-
-/* TESTBUSCR */
-#define TBE (TBE_MASK << TBE_SHIFT)
-#define SPDMBE (SPDMBE_MASK << SPDMBE_SHIFT)
-#define WGSEL (WGSEL_MASK << WGSEL_SHIFT)
-#define TBLSEL (TBLSEL_MASK << TBLSEL_SHIFT)
-#define TBHSEL (TBHSEL_MASK << TBHSEL_SHIFT)
-#define SPDM0SEL (SPDM0SEL_MASK << SPDM0SEL_SHIFT)
-#define SPDM1SEL (SPDM1SEL_MASK << SPDM1SEL_SHIFT)
-#define SPDM2SEL (SPDM2SEL_MASK << SPDM2SEL_SHIFT)
-#define SPDM3SEL (SPDM3SEL_MASK << SPDM3SEL_SHIFT)
-
-
-/* TLBIVMID */
-#define TLBIVMID_VMID (TLBIVMID_VMID_MASK << TLBIVMID_VMID_SHIFT)
-
-
-/* TLBRSW */
-#define TLBRSW_INDEX (TLBRSW_INDEX_MASK << TLBRSW_INDEX_SHIFT)
-#define TLBBFBS (TLBBFBS_MASK << TLBBFBS_SHIFT)
-
-
-/* TLBTR0 */
-#define PR (PR_MASK << PR_SHIFT)
-#define PW (PW_MASK << PW_SHIFT)
-#define UR (UR_MASK << UR_SHIFT)
-#define UW (UW_MASK << UW_SHIFT)
-#define XN (XN_MASK << XN_SHIFT)
-#define NSDESC (NSDESC_MASK << NSDESC_SHIFT)
-#define ISH (ISH_MASK << ISH_SHIFT)
-#define SH (SH_MASK << SH_SHIFT)
-#define MT (MT_MASK << MT_SHIFT)
-#define DPSIZR (DPSIZR_MASK << DPSIZR_SHIFT)
-#define DPSIZC (DPSIZC_MASK << DPSIZC_SHIFT)
-
-
-/* TLBTR1 */
-#define TLBTR1_VMID (TLBTR1_VMID_MASK << TLBTR1_VMID_SHIFT)
-#define TLBTR1_PA (TLBTR1_PA_MASK << TLBTR1_PA_SHIFT)
-
-
-/* TLBTR2 */
-#define TLBTR2_ASID (TLBTR2_ASID_MASK << TLBTR2_ASID_SHIFT)
-#define TLBTR2_V (TLBTR2_V_MASK << TLBTR2_V_SHIFT)
-#define TLBTR2_NSTID (TLBTR2_NSTID_MASK << TLBTR2_NSTID_SHIFT)
-#define TLBTR2_NV (TLBTR2_NV_MASK << TLBTR2_NV_SHIFT)
-#define TLBTR2_VA (TLBTR2_VA_MASK << TLBTR2_VA_SHIFT)
-
-
-/* Context Register Fields */
-/* ACTLR */
-#define CFERE (CFERE_MASK << CFERE_SHIFT)
-#define CFEIE (CFEIE_MASK << CFEIE_SHIFT)
-#define PTSHCFG (PTSHCFG_MASK << PTSHCFG_SHIFT)
-#define RCOSH (RCOSH_MASK << RCOSH_SHIFT)
-#define RCISH (RCISH_MASK << RCISH_SHIFT)
-#define RCNSH (RCNSH_MASK << RCNSH_SHIFT)
-#define PRIVCFG (PRIVCFG_MASK << PRIVCFG_SHIFT)
-#define DNA (DNA_MASK << DNA_SHIFT)
-#define DNLV2PA (DNLV2PA_MASK << DNLV2PA_SHIFT)
-#define TLBMCFG (TLBMCFG_MASK << TLBMCFG_SHIFT)
-#define CFCFG (CFCFG_MASK << CFCFG_SHIFT)
-#define TIPCF (TIPCF_MASK << TIPCF_SHIFT)
-#define V2PCFG (V2PCFG_MASK << V2PCFG_SHIFT)
-#define HUME (HUME_MASK << HUME_SHIFT)
-#define PTMTCFG (PTMTCFG_MASK << PTMTCFG_SHIFT)
-#define PTMEMTYPE (PTMEMTYPE_MASK << PTMEMTYPE_SHIFT)
-
-
-/* BFBCR */
-#define BFBDFE (BFBDFE_MASK << BFBDFE_SHIFT)
-#define BFBSFE (BFBSFE_MASK << BFBSFE_SHIFT)
-#define SFVS (SFVS_MASK << SFVS_SHIFT)
-#define FLVIC (FLVIC_MASK << FLVIC_SHIFT)
-#define SLVIC (SLVIC_MASK << SLVIC_SHIFT)
-
-
-/* CONTEXTIDR */
-#define CONTEXTIDR_ASID (CONTEXTIDR_ASID_MASK << CONTEXTIDR_ASID_SHIFT)
-#define PROCID (PROCID_MASK << PROCID_SHIFT)
-
-
-/* FSR */
-#define TF (TF_MASK << TF_SHIFT)
-#define AFF (AFF_MASK << AFF_SHIFT)
-#define APF (APF_MASK << APF_SHIFT)
-#define TLBMF (TLBMF_MASK << TLBMF_SHIFT)
-#define HTWDEEF (HTWDEEF_MASK << HTWDEEF_SHIFT)
-#define HTWSEEF (HTWSEEF_MASK << HTWSEEF_SHIFT)
-#define MHF (MHF_MASK << MHF_SHIFT)
-#define SL (SL_MASK << SL_SHIFT)
-#define SS (SS_MASK << SS_SHIFT)
-#define MULTI (MULTI_MASK << MULTI_SHIFT)
-
-
-/* FSYNR0 */
-#define AMID (AMID_MASK << AMID_SHIFT)
-#define APID (APID_MASK << APID_SHIFT)
-#define ABID (ABID_MASK << ABID_SHIFT)
-#define ATID (ATID_MASK << ATID_SHIFT)
-
-
-/* FSYNR1 */
-#define AMEMTYPE (AMEMTYPE_MASK << AMEMTYPE_SHIFT)
-#define ASHARED (ASHARED_MASK << ASHARED_SHIFT)
-#define AINNERSHARED (AINNERSHARED_MASK << AINNERSHARED_SHIFT)
-#define APRIV (APRIV_MASK << APRIV_SHIFT)
-#define APROTNS (APROTNS_MASK << APROTNS_SHIFT)
-#define AINST (AINST_MASK << AINST_SHIFT)
-#define AWRITE (AWRITE_MASK << AWRITE_SHIFT)
-#define ABURST (ABURST_MASK << ABURST_SHIFT)
-#define ALEN (ALEN_MASK << ALEN_SHIFT)
-#define FSYNR1_ASIZE (FSYNR1_ASIZE_MASK << FSYNR1_ASIZE_SHIFT)
-#define ALOCK (ALOCK_MASK << ALOCK_SHIFT)
-#define AFULL (AFULL_MASK << AFULL_SHIFT)
-
-
-/* NMRR */
-#define ICPC0 (ICPC0_MASK << ICPC0_SHIFT)
-#define ICPC1 (ICPC1_MASK << ICPC1_SHIFT)
-#define ICPC2 (ICPC2_MASK << ICPC2_SHIFT)
-#define ICPC3 (ICPC3_MASK << ICPC3_SHIFT)
-#define ICPC4 (ICPC4_MASK << ICPC4_SHIFT)
-#define ICPC5 (ICPC5_MASK << ICPC5_SHIFT)
-#define ICPC6 (ICPC6_MASK << ICPC6_SHIFT)
-#define ICPC7 (ICPC7_MASK << ICPC7_SHIFT)
-#define OCPC0 (OCPC0_MASK << OCPC0_SHIFT)
-#define OCPC1 (OCPC1_MASK << OCPC1_SHIFT)
-#define OCPC2 (OCPC2_MASK << OCPC2_SHIFT)
-#define OCPC3 (OCPC3_MASK << OCPC3_SHIFT)
-#define OCPC4 (OCPC4_MASK << OCPC4_SHIFT)
-#define OCPC5 (OCPC5_MASK << OCPC5_SHIFT)
-#define OCPC6 (OCPC6_MASK << OCPC6_SHIFT)
-#define OCPC7 (OCPC7_MASK << OCPC7_SHIFT)
-
-
-/* PAR */
-#define FAULT (FAULT_MASK << FAULT_SHIFT)
-/* If a fault is present, these are the
-same as the fault fields in the FAR */
-#define FAULT_TF (FAULT_TF_MASK << FAULT_TF_SHIFT)
-#define FAULT_AFF (FAULT_AFF_MASK << FAULT_AFF_SHIFT)
-#define FAULT_APF (FAULT_APF_MASK << FAULT_APF_SHIFT)
-#define FAULT_TLBMF (FAULT_TLBMF_MASK << FAULT_TLBMF_SHIFT)
-#define FAULT_HTWDEEF (FAULT_HTWDEEF_MASK << FAULT_HTWDEEF_SHIFT)
-#define FAULT_HTWSEEF (FAULT_HTWSEEF_MASK << FAULT_HTWSEEF_SHIFT)
-#define FAULT_MHF (FAULT_MHF_MASK << FAULT_MHF_SHIFT)
-#define FAULT_SL (FAULT_SL_MASK << FAULT_SL_SHIFT)
-#define FAULT_SS (FAULT_SS_MASK << FAULT_SS_SHIFT)
-
-/* If NO fault is present, the following fields are in effect */
-/* (FAULT remains as before) */
-#define PAR_NOFAULT_SS (PAR_NOFAULT_SS_MASK << PAR_NOFAULT_SS_SHIFT)
-#define PAR_NOFAULT_MT (PAR_NOFAULT_MT_MASK << PAR_NOFAULT_MT_SHIFT)
-#define PAR_NOFAULT_SH (PAR_NOFAULT_SH_MASK << PAR_NOFAULT_SH_SHIFT)
-#define PAR_NOFAULT_NS (PAR_NOFAULT_NS_MASK << PAR_NOFAULT_NS_SHIFT)
-#define PAR_NOFAULT_NOS (PAR_NOFAULT_NOS_MASK << PAR_NOFAULT_NOS_SHIFT)
-#define PAR_NPFAULT_PA (PAR_NPFAULT_PA_MASK << PAR_NPFAULT_PA_SHIFT)
-
-
-/* PRRR */
-#define MTC0 (MTC0_MASK << MTC0_SHIFT)
-#define MTC1 (MTC1_MASK << MTC1_SHIFT)
-#define MTC2 (MTC2_MASK << MTC2_SHIFT)
-#define MTC3 (MTC3_MASK << MTC3_SHIFT)
-#define MTC4 (MTC4_MASK << MTC4_SHIFT)
-#define MTC5 (MTC5_MASK << MTC5_SHIFT)
-#define MTC6 (MTC6_MASK << MTC6_SHIFT)
-#define MTC7 (MTC7_MASK << MTC7_SHIFT)
-#define SHDSH0 (SHDSH0_MASK << SHDSH0_SHIFT)
-#define SHDSH1 (SHDSH1_MASK << SHDSH1_SHIFT)
-#define SHNMSH0 (SHNMSH0_MASK << SHNMSH0_SHIFT)
-#define SHNMSH1 (SHNMSH1_MASK << SHNMSH1_SHIFT)
-#define NOS0 (NOS0_MASK << NOS0_SHIFT)
-#define NOS1 (NOS1_MASK << NOS1_SHIFT)
-#define NOS2 (NOS2_MASK << NOS2_SHIFT)
-#define NOS3 (NOS3_MASK << NOS3_SHIFT)
-#define NOS4 (NOS4_MASK << NOS4_SHIFT)
-#define NOS5 (NOS5_MASK << NOS5_SHIFT)
-#define NOS6 (NOS6_MASK << NOS6_SHIFT)
-#define NOS7 (NOS7_MASK << NOS7_SHIFT)
-
-
-/* RESUME */
-#define TNR (TNR_MASK << TNR_SHIFT)
-
-
-/* SCTLR */
-#define M (M_MASK << M_SHIFT)
-#define TRE (TRE_MASK << TRE_SHIFT)
-#define AFE (AFE_MASK << AFE_SHIFT)
-#define HAF (HAF_MASK << HAF_SHIFT)
-#define BE (BE_MASK << BE_SHIFT)
-#define AFFD (AFFD_MASK << AFFD_SHIFT)
-
-
-/* TLBIASID */
-#define TLBIASID_ASID (TLBIASID_ASID_MASK << TLBIASID_ASID_SHIFT)
-
-
-/* TLBIVA */
-#define TLBIVA_ASID (TLBIVA_ASID_MASK << TLBIVA_ASID_SHIFT)
-#define TLBIVA_VA (TLBIVA_VA_MASK << TLBIVA_VA_SHIFT)
-
-
-/* TLBIVAA */
-#define TLBIVAA_VA (TLBIVAA_VA_MASK << TLBIVAA_VA_SHIFT)
-
-
-/* TLBLCKR */
-#define LKE (LKE_MASK << LKE_SHIFT)
-#define TLBLCKR_TLBIALLCFG (TLBLCKR_TLBIALLCFG_MASK<<TLBLCKR_TLBIALLCFG_SHIFT)
-#define TLBIASIDCFG (TLBIASIDCFG_MASK << TLBIASIDCFG_SHIFT)
-#define TLBIVAACFG (TLBIVAACFG_MASK << TLBIVAACFG_SHIFT)
-#define FLOOR (FLOOR_MASK << FLOOR_SHIFT)
-#define VICTIM (VICTIM_MASK << VICTIM_SHIFT)
-
-
-/* TTBCR */
-#define N (N_MASK << N_SHIFT)
-#define PD0 (PD0_MASK << PD0_SHIFT)
-#define PD1 (PD1_MASK << PD1_SHIFT)
-
-
-/* TTBR0 */
-#define TTBR0_IRGNH (TTBR0_IRGNH_MASK << TTBR0_IRGNH_SHIFT)
-#define TTBR0_SH (TTBR0_SH_MASK << TTBR0_SH_SHIFT)
-#define TTBR0_ORGN (TTBR0_ORGN_MASK << TTBR0_ORGN_SHIFT)
-#define TTBR0_NOS (TTBR0_NOS_MASK << TTBR0_NOS_SHIFT)
-#define TTBR0_IRGNL (TTBR0_IRGNL_MASK << TTBR0_IRGNL_SHIFT)
-#define TTBR0_PA (TTBR0_PA_MASK << TTBR0_PA_SHIFT)
-
-
-/* TTBR1 */
-#define TTBR1_IRGNH (TTBR1_IRGNH_MASK << TTBR1_IRGNH_SHIFT)
-#define TTBR1_SH (TTBR1_SH_MASK << TTBR1_SH_SHIFT)
-#define TTBR1_ORGN (TTBR1_ORGN_MASK << TTBR1_ORGN_SHIFT)
-#define TTBR1_NOS (TTBR1_NOS_MASK << TTBR1_NOS_SHIFT)
-#define TTBR1_IRGNL (TTBR1_IRGNL_MASK << TTBR1_IRGNL_SHIFT)
-#define TTBR1_PA (TTBR1_PA_MASK << TTBR1_PA_SHIFT)
-
-
-/* V2PSR */
-#define HIT (HIT_MASK << HIT_SHIFT)
-#define INDEX (INDEX_MASK << INDEX_SHIFT)
-
-
-/* V2Pxx */
-#define V2Pxx_INDEX (V2Pxx_INDEX_MASK << V2Pxx_INDEX_SHIFT)
-#define V2Pxx_VA (V2Pxx_VA_MASK << V2Pxx_VA_SHIFT)
-
-
-/* Global Register Masks */
-/* CBACRn */
-#define RWVMID_MASK 0x1F
-#define RWE_MASK 0x01
-#define RWGE_MASK 0x01
-#define CBVMID_MASK 0x1F
-#define IRPTNDX_MASK 0xFF
-
-
-/* CR */
-#define RPUE_MASK 0x01
-#define RPUERE_MASK 0x01
-#define RPUEIE_MASK 0x01
-#define DCDEE_MASK 0x01
-#define CLIENTPD_MASK 0x01
-#define STALLD_MASK 0x01
-#define TLBLKCRWE_MASK 0x01
-#define CR_TLBIALLCFG_MASK 0x01
-#define TLBIVMIDCFG_MASK 0x01
-#define CR_HUME_MASK 0x01
-
-
-/* ESR */
-#define CFG_MASK 0x01
-#define BYPASS_MASK 0x01
-#define ESR_MULTI_MASK 0x01
-
-
-/* ESYNR0 */
-#define ESYNR0_AMID_MASK 0xFF
-#define ESYNR0_APID_MASK 0x1F
-#define ESYNR0_ABID_MASK 0x07
-#define ESYNR0_AVMID_MASK 0x1F
-#define ESYNR0_ATID_MASK 0xFF
-
-
-/* ESYNR1 */
-#define ESYNR1_AMEMTYPE_MASK 0x07
-#define ESYNR1_ASHARED_MASK 0x01
-#define ESYNR1_AINNERSHARED_MASK 0x01
-#define ESYNR1_APRIV_MASK 0x01
-#define ESYNR1_APROTNS_MASK 0x01
-#define ESYNR1_AINST_MASK 0x01
-#define ESYNR1_AWRITE_MASK 0x01
-#define ESYNR1_ABURST_MASK 0x01
-#define ESYNR1_ALEN_MASK 0x0F
-#define ESYNR1_ASIZE_MASK 0x01
-#define ESYNR1_ALOCK_MASK 0x03
-#define ESYNR1_AOOO_MASK 0x01
-#define ESYNR1_AFULL_MASK 0x01
-#define ESYNR1_AC_MASK 0x01
-#define ESYNR1_DCD_MASK 0x01
-
-
-/* IDR */
-#define NM2VCBMT_MASK 0x1FF
-#define HTW_MASK 0x01
-#define HUM_MASK 0x01
-#define TLBSIZE_MASK 0x0F
-#define NCB_MASK 0xFF
-#define NIRPT_MASK 0xFF
-
-
-/* M2VCBRn */
-#define VMID_MASK 0x1F
-#define CBNDX_MASK 0xFF
-#define BYPASSD_MASK 0x01
-#define BPRCOSH_MASK 0x01
-#define BPRCISH_MASK 0x01
-#define BPRCNSH_MASK 0x01
-#define BPSHCFG_MASK 0x03
-#define NSCFG_MASK 0x03
-#define BPMTCFG_MASK 0x01
-#define BPMEMTYPE_MASK 0x07
-
-
-/* REV */
-#define MINOR_MASK 0x0F
-#define MAJOR_MASK 0x0F
-
-
-/* TESTBUSCR */
-#define TBE_MASK 0x01
-#define SPDMBE_MASK 0x01
-#define WGSEL_MASK 0x03
-#define TBLSEL_MASK 0x03
-#define TBHSEL_MASK 0x03
-#define SPDM0SEL_MASK 0x0F
-#define SPDM1SEL_MASK 0x0F
-#define SPDM2SEL_MASK 0x0F
-#define SPDM3SEL_MASK 0x0F
-
-
-/* TLBIMID */
-#define TLBIVMID_VMID_MASK 0x1F
-
-
-/* TLBRSW */
-#define TLBRSW_INDEX_MASK 0xFF
-#define TLBBFBS_MASK 0x03
-
-
-/* TLBTR0 */
-#define PR_MASK 0x01
-#define PW_MASK 0x01
-#define UR_MASK 0x01
-#define UW_MASK 0x01
-#define XN_MASK 0x01
-#define NSDESC_MASK 0x01
-#define ISH_MASK 0x01
-#define SH_MASK 0x01
-#define MT_MASK 0x07
-#define DPSIZR_MASK 0x07
-#define DPSIZC_MASK 0x07
-
-
-/* TLBTR1 */
-#define TLBTR1_VMID_MASK 0x1F
-#define TLBTR1_PA_MASK 0x000FFFFF
-
-
-/* TLBTR2 */
-#define TLBTR2_ASID_MASK 0xFF
-#define TLBTR2_V_MASK 0x01
-#define TLBTR2_NSTID_MASK 0x01
-#define TLBTR2_NV_MASK 0x01
-#define TLBTR2_VA_MASK 0x000FFFFF
-
-
-/* Global Register Shifts */
-/* CBACRn */
-#define RWVMID_SHIFT 0
-#define RWE_SHIFT 8
-#define RWGE_SHIFT 9
-#define CBVMID_SHIFT 16
-#define IRPTNDX_SHIFT 24
-
-
-/* CR */
-#define RPUE_SHIFT 0
-#define RPUERE_SHIFT 1
-#define RPUEIE_SHIFT 2
-#define DCDEE_SHIFT 3
-#define CLIENTPD_SHIFT 4
-#define STALLD_SHIFT 5
-#define TLBLKCRWE_SHIFT 6
-#define CR_TLBIALLCFG_SHIFT 7
-#define TLBIVMIDCFG_SHIFT 8
-#define CR_HUME_SHIFT 9
-
-
-/* ESR */
-#define CFG_SHIFT 0
-#define BYPASS_SHIFT 1
-#define ESR_MULTI_SHIFT 31
-
-
-/* ESYNR0 */
-#define ESYNR0_AMID_SHIFT 0
-#define ESYNR0_APID_SHIFT 8
-#define ESYNR0_ABID_SHIFT 13
-#define ESYNR0_AVMID_SHIFT 16
-#define ESYNR0_ATID_SHIFT 24
-
-
-/* ESYNR1 */
-#define ESYNR1_AMEMTYPE_SHIFT 0
-#define ESYNR1_ASHARED_SHIFT 3
-#define ESYNR1_AINNERSHARED_SHIFT 4
-#define ESYNR1_APRIV_SHIFT 5
-#define ESYNR1_APROTNS_SHIFT 6
-#define ESYNR1_AINST_SHIFT 7
-#define ESYNR1_AWRITE_SHIFT 8
-#define ESYNR1_ABURST_SHIFT 10
-#define ESYNR1_ALEN_SHIFT 12
-#define ESYNR1_ASIZE_SHIFT 16
-#define ESYNR1_ALOCK_SHIFT 20
-#define ESYNR1_AOOO_SHIFT 22
-#define ESYNR1_AFULL_SHIFT 24
-#define ESYNR1_AC_SHIFT 30
-#define ESYNR1_DCD_SHIFT 31
-
-
-/* IDR */
-#define NM2VCBMT_SHIFT 0
-#define HTW_SHIFT 9
-#define HUM_SHIFT 10
-#define TLBSIZE_SHIFT 12
-#define NCB_SHIFT 16
-#define NIRPT_SHIFT 24
-
-
-/* M2VCBRn */
-#define VMID_SHIFT 0
-#define CBNDX_SHIFT 8
-#define BYPASSD_SHIFT 16
-#define BPRCOSH_SHIFT 17
-#define BPRCISH_SHIFT 18
-#define BPRCNSH_SHIFT 19
-#define BPSHCFG_SHIFT 20
-#define NSCFG_SHIFT 22
-#define BPMTCFG_SHIFT 24
-#define BPMEMTYPE_SHIFT 25
-
-
-/* REV */
-#define MINOR_SHIFT 0
-#define MAJOR_SHIFT 4
-
-
-/* TESTBUSCR */
-#define TBE_SHIFT 0
-#define SPDMBE_SHIFT 1
-#define WGSEL_SHIFT 8
-#define TBLSEL_SHIFT 12
-#define TBHSEL_SHIFT 14
-#define SPDM0SEL_SHIFT 16
-#define SPDM1SEL_SHIFT 20
-#define SPDM2SEL_SHIFT 24
-#define SPDM3SEL_SHIFT 28
-
-
-/* TLBIMID */
-#define TLBIVMID_VMID_SHIFT 0
-
-
-/* TLBRSW */
-#define TLBRSW_INDEX_SHIFT 0
-#define TLBBFBS_SHIFT 8
-
-
-/* TLBTR0 */
-#define PR_SHIFT 0
-#define PW_SHIFT 1
-#define UR_SHIFT 2
-#define UW_SHIFT 3
-#define XN_SHIFT 4
-#define NSDESC_SHIFT 6
-#define ISH_SHIFT 7
-#define SH_SHIFT 8
-#define MT_SHIFT 9
-#define DPSIZR_SHIFT 16
-#define DPSIZC_SHIFT 20
-
-
-/* TLBTR1 */
-#define TLBTR1_VMID_SHIFT 0
-#define TLBTR1_PA_SHIFT 12
-
-
-/* TLBTR2 */
-#define TLBTR2_ASID_SHIFT 0
-#define TLBTR2_V_SHIFT 8
-#define TLBTR2_NSTID_SHIFT 9
-#define TLBTR2_NV_SHIFT 10
-#define TLBTR2_VA_SHIFT 12
-
-
-/* Context Register Masks */
-/* ACTLR */
-#define CFERE_MASK 0x01
-#define CFEIE_MASK 0x01
-#define PTSHCFG_MASK 0x03
-#define RCOSH_MASK 0x01
-#define RCISH_MASK 0x01
-#define RCNSH_MASK 0x01
-#define PRIVCFG_MASK 0x03
-#define DNA_MASK 0x01
-#define DNLV2PA_MASK 0x01
-#define TLBMCFG_MASK 0x03
-#define CFCFG_MASK 0x01
-#define TIPCF_MASK 0x01
-#define V2PCFG_MASK 0x03
-#define HUME_MASK 0x01
-#define PTMTCFG_MASK 0x01
-#define PTMEMTYPE_MASK 0x07
-
-
-/* BFBCR */
-#define BFBDFE_MASK 0x01
-#define BFBSFE_MASK 0x01
-#define SFVS_MASK 0x01
-#define FLVIC_MASK 0x0F
-#define SLVIC_MASK 0x0F
-
-
-/* CONTEXTIDR */
-#define CONTEXTIDR_ASID_MASK 0xFF
-#define PROCID_MASK 0x00FFFFFF
-
-
-/* FSR */
-#define TF_MASK 0x01
-#define AFF_MASK 0x01
-#define APF_MASK 0x01
-#define TLBMF_MASK 0x01
-#define HTWDEEF_MASK 0x01
-#define HTWSEEF_MASK 0x01
-#define MHF_MASK 0x01
-#define SL_MASK 0x01
-#define SS_MASK 0x01
-#define MULTI_MASK 0x01
-
-
-/* FSYNR0 */
-#define AMID_MASK 0xFF
-#define APID_MASK 0x1F
-#define ABID_MASK 0x07
-#define ATID_MASK 0xFF
-
-
-/* FSYNR1 */
-#define AMEMTYPE_MASK 0x07
-#define ASHARED_MASK 0x01
-#define AINNERSHARED_MASK 0x01
-#define APRIV_MASK 0x01
-#define APROTNS_MASK 0x01
-#define AINST_MASK 0x01
-#define AWRITE_MASK 0x01
-#define ABURST_MASK 0x01
-#define ALEN_MASK 0x0F
-#define FSYNR1_ASIZE_MASK 0x07
-#define ALOCK_MASK 0x03
-#define AFULL_MASK 0x01
-
-
-/* NMRR */
-#define ICPC0_MASK 0x03
-#define ICPC1_MASK 0x03
-#define ICPC2_MASK 0x03
-#define ICPC3_MASK 0x03
-#define ICPC4_MASK 0x03
-#define ICPC5_MASK 0x03
-#define ICPC6_MASK 0x03
-#define ICPC7_MASK 0x03
-#define OCPC0_MASK 0x03
-#define OCPC1_MASK 0x03
-#define OCPC2_MASK 0x03
-#define OCPC3_MASK 0x03
-#define OCPC4_MASK 0x03
-#define OCPC5_MASK 0x03
-#define OCPC6_MASK 0x03
-#define OCPC7_MASK 0x03
-
-
-/* PAR */
-#define FAULT_MASK 0x01
-/* If a fault is present, these are the
-same as the fault fields in the FAR */
-#define FAULT_TF_MASK 0x01
-#define FAULT_AFF_MASK 0x01
-#define FAULT_APF_MASK 0x01
-#define FAULT_TLBMF_MASK 0x01
-#define FAULT_HTWDEEF_MASK 0x01
-#define FAULT_HTWSEEF_MASK 0x01
-#define FAULT_MHF_MASK 0x01
-#define FAULT_SL_MASK 0x01
-#define FAULT_SS_MASK 0x01
-
-/* If NO fault is present, the following
- * fields are in effect
- * (FAULT remains as before) */
-#define PAR_NOFAULT_SS_MASK 0x01
-#define PAR_NOFAULT_MT_MASK 0x07
-#define PAR_NOFAULT_SH_MASK 0x01
-#define PAR_NOFAULT_NS_MASK 0x01
-#define PAR_NOFAULT_NOS_MASK 0x01
-#define PAR_NPFAULT_PA_MASK 0x000FFFFF
-
-
-/* PRRR */
-#define MTC0_MASK 0x03
-#define MTC1_MASK 0x03
-#define MTC2_MASK 0x03
-#define MTC3_MASK 0x03
-#define MTC4_MASK 0x03
-#define MTC5_MASK 0x03
-#define MTC6_MASK 0x03
-#define MTC7_MASK 0x03
-#define SHDSH0_MASK 0x01
-#define SHDSH1_MASK 0x01
-#define SHNMSH0_MASK 0x01
-#define SHNMSH1_MASK 0x01
-#define NOS0_MASK 0x01
-#define NOS1_MASK 0x01
-#define NOS2_MASK 0x01
-#define NOS3_MASK 0x01
-#define NOS4_MASK 0x01
-#define NOS5_MASK 0x01
-#define NOS6_MASK 0x01
-#define NOS7_MASK 0x01
-
-
-/* RESUME */
-#define TNR_MASK 0x01
-
-
-/* SCTLR */
-#define M_MASK 0x01
-#define TRE_MASK 0x01
-#define AFE_MASK 0x01
-#define HAF_MASK 0x01
-#define BE_MASK 0x01
-#define AFFD_MASK 0x01
-
-
-/* TLBIASID */
-#define TLBIASID_ASID_MASK 0xFF
-
-
-/* TLBIVA */
-#define TLBIVA_ASID_MASK 0xFF
-#define TLBIVA_VA_MASK 0x000FFFFF
-
-
-/* TLBIVAA */
-#define TLBIVAA_VA_MASK 0x000FFFFF
-
-
-/* TLBLCKR */
-#define LKE_MASK 0x01
-#define TLBLCKR_TLBIALLCFG_MASK 0x01
-#define TLBIASIDCFG_MASK 0x01
-#define TLBIVAACFG_MASK 0x01
-#define FLOOR_MASK 0xFF
-#define VICTIM_MASK 0xFF
-
-
-/* TTBCR */
-#define N_MASK 0x07
-#define PD0_MASK 0x01
-#define PD1_MASK 0x01
-
-
-/* TTBR0 */
-#define TTBR0_IRGNH_MASK 0x01
-#define TTBR0_SH_MASK 0x01
-#define TTBR0_ORGN_MASK 0x03
-#define TTBR0_NOS_MASK 0x01
-#define TTBR0_IRGNL_MASK 0x01
-#define TTBR0_PA_MASK 0x0003FFFF
-
-
-/* TTBR1 */
-#define TTBR1_IRGNH_MASK 0x01
-#define TTBR1_SH_MASK 0x01
-#define TTBR1_ORGN_MASK 0x03
-#define TTBR1_NOS_MASK 0x01
-#define TTBR1_IRGNL_MASK 0x01
-#define TTBR1_PA_MASK 0x0003FFFF
-
-
-/* V2PSR */
-#define HIT_MASK 0x01
-#define INDEX_MASK 0xFF
-
-
-/* V2Pxx */
-#define V2Pxx_INDEX_MASK 0xFF
-#define V2Pxx_VA_MASK 0x000FFFFF
-
-
-/* Context Register Shifts */
-/* ACTLR */
-#define CFERE_SHIFT 0
-#define CFEIE_SHIFT 1
-#define PTSHCFG_SHIFT 2
-#define RCOSH_SHIFT 4
-#define RCISH_SHIFT 5
-#define RCNSH_SHIFT 6
-#define PRIVCFG_SHIFT 8
-#define DNA_SHIFT 10
-#define DNLV2PA_SHIFT 11
-#define TLBMCFG_SHIFT 12
-#define CFCFG_SHIFT 14
-#define TIPCF_SHIFT 15
-#define V2PCFG_SHIFT 16
-#define HUME_SHIFT 18
-#define PTMTCFG_SHIFT 20
-#define PTMEMTYPE_SHIFT 21
-
-
-/* BFBCR */
-#define BFBDFE_SHIFT 0
-#define BFBSFE_SHIFT 1
-#define SFVS_SHIFT 2
-#define FLVIC_SHIFT 4
-#define SLVIC_SHIFT 8
-
-
-/* CONTEXTIDR */
-#define CONTEXTIDR_ASID_SHIFT 0
-#define PROCID_SHIFT 8
-
-
-/* FSR */
-#define TF_SHIFT 1
-#define AFF_SHIFT 2
-#define APF_SHIFT 3
-#define TLBMF_SHIFT 4
-#define HTWDEEF_SHIFT 5
-#define HTWSEEF_SHIFT 6
-#define MHF_SHIFT 7
-#define SL_SHIFT 16
-#define SS_SHIFT 30
-#define MULTI_SHIFT 31
-
-
-/* FSYNR0 */
-#define AMID_SHIFT 0
-#define APID_SHIFT 8
-#define ABID_SHIFT 13
-#define ATID_SHIFT 24
-
-
-/* FSYNR1 */
-#define AMEMTYPE_SHIFT 0
-#define ASHARED_SHIFT 3
-#define AINNERSHARED_SHIFT 4
-#define APRIV_SHIFT 5
-#define APROTNS_SHIFT 6
-#define AINST_SHIFT 7
-#define AWRITE_SHIFT 8
-#define ABURST_SHIFT 10
-#define ALEN_SHIFT 12
-#define FSYNR1_ASIZE_SHIFT 16
-#define ALOCK_SHIFT 20
-#define AFULL_SHIFT 24
-
-
-/* NMRR */
-#define ICPC0_SHIFT 0
-#define ICPC1_SHIFT 2
-#define ICPC2_SHIFT 4
-#define ICPC3_SHIFT 6
-#define ICPC4_SHIFT 8
-#define ICPC5_SHIFT 10
-#define ICPC6_SHIFT 12
-#define ICPC7_SHIFT 14
-#define OCPC0_SHIFT 16
-#define OCPC1_SHIFT 18
-#define OCPC2_SHIFT 20
-#define OCPC3_SHIFT 22
-#define OCPC4_SHIFT 24
-#define OCPC5_SHIFT 26
-#define OCPC6_SHIFT 28
-#define OCPC7_SHIFT 30
-
-
-/* PAR */
-#define FAULT_SHIFT 0
-/* If a fault is present, these are the
-same as the fault fields in the FAR */
-#define FAULT_TF_SHIFT 1
-#define FAULT_AFF_SHIFT 2
-#define FAULT_APF_SHIFT 3
-#define FAULT_TLBMF_SHIFT 4
-#define FAULT_HTWDEEF_SHIFT 5
-#define FAULT_HTWSEEF_SHIFT 6
-#define FAULT_MHF_SHIFT 7
-#define FAULT_SL_SHIFT 16
-#define FAULT_SS_SHIFT 30
-
-/* If NO fault is present, the following
- * fields are in effect
- * (FAULT remains as before) */
-#define PAR_NOFAULT_SS_SHIFT 1
-#define PAR_NOFAULT_MT_SHIFT 4
-#define PAR_NOFAULT_SH_SHIFT 7
-#define PAR_NOFAULT_NS_SHIFT 9
-#define PAR_NOFAULT_NOS_SHIFT 10
-#define PAR_NPFAULT_PA_SHIFT 12
-
-
-/* PRRR */
-#define MTC0_SHIFT 0
-#define MTC1_SHIFT 2
-#define MTC2_SHIFT 4
-#define MTC3_SHIFT 6
-#define MTC4_SHIFT 8
-#define MTC5_SHIFT 10
-#define MTC6_SHIFT 12
-#define MTC7_SHIFT 14
-#define SHDSH0_SHIFT 16
-#define SHDSH1_SHIFT 17
-#define SHNMSH0_SHIFT 18
-#define SHNMSH1_SHIFT 19
-#define NOS0_SHIFT 24
-#define NOS1_SHIFT 25
-#define NOS2_SHIFT 26
-#define NOS3_SHIFT 27
-#define NOS4_SHIFT 28
-#define NOS5_SHIFT 29
-#define NOS6_SHIFT 30
-#define NOS7_SHIFT 31
-
-
-/* RESUME */
-#define TNR_SHIFT 0
-
-
-/* SCTLR */
-#define M_SHIFT 0
-#define TRE_SHIFT 1
-#define AFE_SHIFT 2
-#define HAF_SHIFT 3
-#define BE_SHIFT 4
-#define AFFD_SHIFT 5
-
-
-/* TLBIASID */
-#define TLBIASID_ASID_SHIFT 0
-
-
-/* TLBIVA */
-#define TLBIVA_ASID_SHIFT 0
-#define TLBIVA_VA_SHIFT 12
-
-
-/* TLBIVAA */
-#define TLBIVAA_VA_SHIFT 12
-
-
-/* TLBLCKR */
-#define LKE_SHIFT 0
-#define TLBLCKR_TLBIALLCFG_SHIFT 1
-#define TLBIASIDCFG_SHIFT 2
-#define TLBIVAACFG_SHIFT 3
-#define FLOOR_SHIFT 8
-#define VICTIM_SHIFT 8
-
-
-/* TTBCR */
-#define N_SHIFT 3
-#define PD0_SHIFT 4
-#define PD1_SHIFT 5
-
-
-/* TTBR0 */
-#define TTBR0_IRGNH_SHIFT 0
-#define TTBR0_SH_SHIFT 1
-#define TTBR0_ORGN_SHIFT 3
-#define TTBR0_NOS_SHIFT 5
-#define TTBR0_IRGNL_SHIFT 6
-#define TTBR0_PA_SHIFT 14
-
-
-/* TTBR1 */
-#define TTBR1_IRGNH_SHIFT 0
-#define TTBR1_SH_SHIFT 1
-#define TTBR1_ORGN_SHIFT 3
-#define TTBR1_NOS_SHIFT 5
-#define TTBR1_IRGNL_SHIFT 6
-#define TTBR1_PA_SHIFT 14
-
-
-/* V2PSR */
-#define HIT_SHIFT 0
-#define INDEX_SHIFT 8
-
-
-/* V2Pxx */
-#define V2Pxx_INDEX_SHIFT 0
-#define V2Pxx_VA_SHIFT 12
-
-#endif
generated by cgit v1.2.3 (git 2.39.1) at 2024-09-28 21:36:25 +0000