summaryrefslogtreecommitdiff
path: root/arch/arm/mm
diff options
context:
space:
mode:
Diffstat (limited to 'arch/arm/mm')
-rw-r--r--arch/arm/mm/Kconfig21
-rw-r--r--arch/arm/mm/cache-l2x0.c31
-rw-r--r--arch/arm/mm/cache-v6.S18
-rw-r--r--arch/arm/mm/dma-mapping.c18
-rw-r--r--arch/arm/mm/highmem.c13
5 files changed, 68 insertions, 33 deletions
diff --git a/arch/arm/mm/Kconfig b/arch/arm/mm/Kconfig
index 346ae14824a5..101105e52610 100644
--- a/arch/arm/mm/Kconfig
+++ b/arch/arm/mm/Kconfig
@@ -735,6 +735,25 @@ config NEEDS_SYSCALL_FOR_CMPXCHG
Forget about fast user space cmpxchg support.
It is just not possible.
+config DMA_CACHE_RWFO
+ bool "Enable read/write for ownership DMA cache maintenance"
+ depends on CPU_V6 && SMP
+ default y
+ help
+ The Snoop Control Unit on ARM11MPCore does not detect the
+ cache maintenance operations and the dma_{map,unmap}_area()
+ functions may leave stale cache entries on other CPUs. By
+ enabling this option, Read or Write For Ownership in the ARMv6
+ DMA cache maintenance functions is performed. These LDR/STR
+ instructions change the cache line state to shared or modified
+ so that the cache operation has the desired effect.
+
+ Note that the workaround is only valid on processors that do
+ not perform speculative loads into the D-cache. For such
+ processors, if cache maintenance operations are not broadcast
+ in hardware, other workarounds are needed (e.g. cache
+ maintenance broadcasting in software via FIQ).
+
config OUTER_CACHE
bool
@@ -794,6 +813,8 @@ config ARM_L1_CACHE_SHIFT
config ARM_DMA_MEM_BUFFERABLE
bool "Use non-cacheable memory for DMA" if CPU_V6 && !CPU_V7
+ depends on !(MACH_REALVIEW_PB1176 || REALVIEW_EB_ARM11MP || \
+ MACH_REALVIEW_PB11MP)
default y if CPU_V6 || CPU_V7
help
Historically, the kernel has used strongly ordered mappings to
diff --git a/arch/arm/mm/cache-l2x0.c b/arch/arm/mm/cache-l2x0.c
index 9819869d2bc9..9982eb385c0f 100644
--- a/arch/arm/mm/cache-l2x0.c
+++ b/arch/arm/mm/cache-l2x0.c
@@ -32,14 +32,14 @@ static uint32_t l2x0_way_mask; /* Bitmask of active ways */
static inline void cache_wait(void __iomem *reg, unsigned long mask)
{
/* wait for the operation to complete */
- while (readl(reg) & mask)
+ while (readl_relaxed(reg) & mask)
;
}
static inline void cache_sync(void)
{
void __iomem *base = l2x0_base;
- writel(0, base + L2X0_CACHE_SYNC);
+ writel_relaxed(0, base + L2X0_CACHE_SYNC);
cache_wait(base + L2X0_CACHE_SYNC, 1);
}
@@ -47,14 +47,14 @@ static inline void l2x0_clean_line(unsigned long addr)
{
void __iomem *base = l2x0_base;
cache_wait(base + L2X0_CLEAN_LINE_PA, 1);
- writel(addr, base + L2X0_CLEAN_LINE_PA);
+ writel_relaxed(addr, base + L2X0_CLEAN_LINE_PA);
}
static inline void l2x0_inv_line(unsigned long addr)
{
void __iomem *base = l2x0_base;
cache_wait(base + L2X0_INV_LINE_PA, 1);
- writel(addr, base + L2X0_INV_LINE_PA);
+ writel_relaxed(addr, base + L2X0_INV_LINE_PA);
}
#ifdef CONFIG_PL310_ERRATA_588369
@@ -75,9 +75,9 @@ static inline void l2x0_flush_line(unsigned long addr)
/* Clean by PA followed by Invalidate by PA */
cache_wait(base + L2X0_CLEAN_LINE_PA, 1);
- writel(addr, base + L2X0_CLEAN_LINE_PA);
+ writel_relaxed(addr, base + L2X0_CLEAN_LINE_PA);
cache_wait(base + L2X0_INV_LINE_PA, 1);
- writel(addr, base + L2X0_INV_LINE_PA);
+ writel_relaxed(addr, base + L2X0_INV_LINE_PA);
}
#else
@@ -90,7 +90,7 @@ static inline void l2x0_flush_line(unsigned long addr)
{
void __iomem *base = l2x0_base;
cache_wait(base + L2X0_CLEAN_INV_LINE_PA, 1);
- writel(addr, base + L2X0_CLEAN_INV_LINE_PA);
+ writel_relaxed(addr, base + L2X0_CLEAN_INV_LINE_PA);
}
#endif
@@ -109,7 +109,7 @@ static inline void l2x0_inv_all(void)
/* invalidate all ways */
spin_lock_irqsave(&l2x0_lock, flags);
- writel(l2x0_way_mask, l2x0_base + L2X0_INV_WAY);
+ writel_relaxed(l2x0_way_mask, l2x0_base + L2X0_INV_WAY);
cache_wait(l2x0_base + L2X0_INV_WAY, l2x0_way_mask);
cache_sync();
spin_unlock_irqrestore(&l2x0_lock, flags);
@@ -215,8 +215,11 @@ void __init l2x0_init(void __iomem *base, __u32 aux_val, __u32 aux_mask)
l2x0_base = base;
- cache_id = readl(l2x0_base + L2X0_CACHE_ID);
- aux = readl(l2x0_base + L2X0_AUX_CTRL);
+ cache_id = readl_relaxed(l2x0_base + L2X0_CACHE_ID);
+ aux = readl_relaxed(l2x0_base + L2X0_AUX_CTRL);
+
+ aux &= aux_mask;
+ aux |= aux_val;
/* Determine the number of ways */
switch (cache_id & L2X0_CACHE_ID_PART_MASK) {
@@ -245,17 +248,15 @@ void __init l2x0_init(void __iomem *base, __u32 aux_val, __u32 aux_mask)
* If you are booting from non-secure mode
* accessing the below registers will fault.
*/
- if (!(readl(l2x0_base + L2X0_CTRL) & 1)) {
+ if (!(readl_relaxed(l2x0_base + L2X0_CTRL) & 1)) {
/* l2x0 controller is disabled */
- aux &= aux_mask;
- aux |= aux_val;
- writel(aux, l2x0_base + L2X0_AUX_CTRL);
+ writel_relaxed(aux, l2x0_base + L2X0_AUX_CTRL);
l2x0_inv_all();
/* enable L2X0 */
- writel(1, l2x0_base + L2X0_CTRL);
+ writel_relaxed(1, l2x0_base + L2X0_CTRL);
}
outer_cache.inv_range = l2x0_inv_range;
diff --git a/arch/arm/mm/cache-v6.S b/arch/arm/mm/cache-v6.S
index e46ecd847138..86aa689ef1aa 100644
--- a/arch/arm/mm/cache-v6.S
+++ b/arch/arm/mm/cache-v6.S
@@ -211,8 +211,9 @@ v6_dma_inv_range:
mcrne p15, 0, r1, c7, c15, 1 @ clean & invalidate unified line
#endif
1:
-#ifdef CONFIG_SMP
- str r0, [r0] @ write for ownership
+#ifdef CONFIG_DMA_CACHE_RWFO
+ ldr r2, [r0] @ read for ownership
+ str r2, [r0] @ write for ownership
#endif
#ifdef HARVARD_CACHE
mcr p15, 0, r0, c7, c6, 1 @ invalidate D line
@@ -234,7 +235,7 @@ v6_dma_inv_range:
v6_dma_clean_range:
bic r0, r0, #D_CACHE_LINE_SIZE - 1
1:
-#ifdef CONFIG_SMP
+#ifdef CONFIG_DMA_CACHE_RWFO
ldr r2, [r0] @ read for ownership
#endif
#ifdef HARVARD_CACHE
@@ -257,7 +258,7 @@ v6_dma_clean_range:
ENTRY(v6_dma_flush_range)
bic r0, r0, #D_CACHE_LINE_SIZE - 1
1:
-#ifdef CONFIG_SMP
+#ifdef CONFIG_DMA_CACHE_RWFO
ldr r2, [r0] @ read for ownership
str r2, [r0] @ write for ownership
#endif
@@ -283,9 +284,13 @@ ENTRY(v6_dma_map_area)
add r1, r1, r0
teq r2, #DMA_FROM_DEVICE
beq v6_dma_inv_range
+#ifndef CONFIG_DMA_CACHE_RWFO
+ b v6_dma_clean_range
+#else
teq r2, #DMA_TO_DEVICE
beq v6_dma_clean_range
b v6_dma_flush_range
+#endif
ENDPROC(v6_dma_map_area)
/*
@@ -295,6 +300,11 @@ ENDPROC(v6_dma_map_area)
* - dir - DMA direction
*/
ENTRY(v6_dma_unmap_area)
+#ifndef CONFIG_DMA_CACHE_RWFO
+ add r1, r1, r0
+ teq r2, #DMA_TO_DEVICE
+ bne v6_dma_inv_range
+#endif
mov pc, lr
ENDPROC(v6_dma_unmap_area)
diff --git a/arch/arm/mm/dma-mapping.c b/arch/arm/mm/dma-mapping.c
index 13fa536d82e6..9e7742f0a102 100644
--- a/arch/arm/mm/dma-mapping.c
+++ b/arch/arm/mm/dma-mapping.c
@@ -24,15 +24,6 @@
#include <asm/tlbflush.h>
#include <asm/sizes.h>
-/* Sanity check size */
-#if (CONSISTENT_DMA_SIZE % SZ_2M)
-#error "CONSISTENT_DMA_SIZE must be multiple of 2MiB"
-#endif
-
-#define CONSISTENT_OFFSET(x) (((unsigned long)(x) - CONSISTENT_BASE) >> PAGE_SHIFT)
-#define CONSISTENT_PTE_INDEX(x) (((unsigned long)(x) - CONSISTENT_BASE) >> PGDIR_SHIFT)
-#define NUM_CONSISTENT_PTES (CONSISTENT_DMA_SIZE >> PGDIR_SHIFT)
-
static u64 get_coherent_dma_mask(struct device *dev)
{
u64 mask = ISA_DMA_THRESHOLD;
@@ -123,6 +114,15 @@ static void __dma_free_buffer(struct page *page, size_t size)
}
#ifdef CONFIG_MMU
+/* Sanity check size */
+#if (CONSISTENT_DMA_SIZE % SZ_2M)
+#error "CONSISTENT_DMA_SIZE must be multiple of 2MiB"
+#endif
+
+#define CONSISTENT_OFFSET(x) (((unsigned long)(x) - CONSISTENT_BASE) >> PAGE_SHIFT)
+#define CONSISTENT_PTE_INDEX(x) (((unsigned long)(x) - CONSISTENT_BASE) >> PGDIR_SHIFT)
+#define NUM_CONSISTENT_PTES (CONSISTENT_DMA_SIZE >> PGDIR_SHIFT)
+
/*
* These are the page tables (2MB each) covering uncached, DMA consistent allocations
*/
diff --git a/arch/arm/mm/highmem.c b/arch/arm/mm/highmem.c
index 086816b205b8..6ab244062b4a 100644
--- a/arch/arm/mm/highmem.c
+++ b/arch/arm/mm/highmem.c
@@ -163,19 +163,22 @@ static DEFINE_PER_CPU(int, kmap_high_l1_vipt_depth);
void *kmap_high_l1_vipt(struct page *page, pte_t *saved_pte)
{
- unsigned int idx, cpu = smp_processor_id();
- int *depth = &per_cpu(kmap_high_l1_vipt_depth, cpu);
+ unsigned int idx, cpu;
+ int *depth;
unsigned long vaddr, flags;
pte_t pte, *ptep;
+ if (!in_interrupt())
+ preempt_disable();
+
+ cpu = smp_processor_id();
+ depth = &per_cpu(kmap_high_l1_vipt_depth, cpu);
+
idx = KM_L1_CACHE + KM_TYPE_NR * cpu;
vaddr = __fix_to_virt(FIX_KMAP_BEGIN + idx);
ptep = TOP_PTE(vaddr);
pte = mk_pte(page, kmap_prot);
- if (!in_interrupt())
- preempt_disable();
-
raw_local_irq_save(flags);
(*depth)++;
if (pte_val(*ptep) == pte_val(pte)) {
generated by cgit v1.2.3 (git 2.39.1) at 2024-10-27 19:34:22 +0000