From: Arnd Bergmann <arnd@xxxxxxxx> As the final step of the conversion to generic arch_sync_dma_* helpers, change the Arm implementation to look the same as the new generic version, by calling the dmac_{clean,inv,flush}_area low-level functions instead of the abstracted dmac_{map,unmap}_area version. On ARMv6/v7, this invalidates the caches after a DMA transfer from a device because of speculative prefetching, while on earlier versions it only needs to do this before the transfer. This should not change any of the current behavior. FIXME: address CONFIG_DMA_CACHE_RWFO properly. Signed-off-by: Arnd Bergmann <arnd@xxxxxxxx> --- arch/arm/mm/dma-mapping-nommu.c | 11 +++---- arch/arm/mm/dma-mapping.c | 53 +++++++++++++++++++++++---------- 2 files changed, 43 insertions(+), 21 deletions(-) diff --git a/arch/arm/mm/dma-mapping-nommu.c b/arch/arm/mm/dma-mapping-nommu.c index cfd9c933d2f0..12b5c6ae93fc 100644 --- a/arch/arm/mm/dma-mapping-nommu.c +++ b/arch/arm/mm/dma-mapping-nommu.c @@ -16,12 +16,13 @@ void arch_sync_dma_for_device(phys_addr_t paddr, size_t size, enum dma_data_direction dir) { - dmac_map_area(__va(paddr), size, dir); - - if (dir == DMA_FROM_DEVICE) + if (dir == DMA_FROM_DEVICE) { + dmac_inv_range(__va(paddr), __va(paddr + size)); outer_inv_range(paddr, paddr + size); - else + } else { + dmac_clean_range(__va(paddr), __va(paddr + size)); outer_clean_range(paddr, paddr + size); + } } void arch_sync_dma_for_cpu(phys_addr_t paddr, size_t size, @@ -29,7 +30,7 @@ void arch_sync_dma_for_cpu(phys_addr_t paddr, size_t size, { if (dir != DMA_TO_DEVICE) { outer_inv_range(paddr, paddr + size); - dmac_unmap_area(__va(paddr), size, dir); + dmac_inv_range(__va(paddr), __va(paddr)); } } diff --git a/arch/arm/mm/dma-mapping.c b/arch/arm/mm/dma-mapping.c index ce4b74f34a58..cc702cb27ae7 100644 --- a/arch/arm/mm/dma-mapping.c +++ b/arch/arm/mm/dma-mapping.c @@ -623,8 +623,7 @@ static void __arm_dma_free(struct device *dev, size_t size, void *cpu_addr, } static void dma_cache_maint(phys_addr_t paddr, - size_t size, enum dma_data_direction dir, - void (*op)(const void *, size_t, int)) + size_t size, void (*op)(const void *, const void *)) { unsigned long pfn = PFN_DOWN(paddr); unsigned long offset = paddr % PAGE_SIZE; @@ -647,18 +646,18 @@ static void dma_cache_maint(phys_addr_t paddr, if (cache_is_vipt_nonaliasing()) { vaddr = kmap_atomic(page); - op(vaddr + offset, len, dir); + op(vaddr + offset, vaddr + offset + len); kunmap_atomic(vaddr); } else { vaddr = kmap_high_get(page); if (vaddr) { - op(vaddr + offset, len, dir); + op(vaddr + offset, vaddr + offset + len); kunmap_high(page); } } } else { vaddr = page_address(page) + offset; - op(vaddr, len, dir); + op(vaddr, vaddr + len); } offset = 0; pfn++; @@ -666,6 +665,18 @@ static void dma_cache_maint(phys_addr_t paddr, } while (left); } +static bool arch_sync_dma_cpu_needs_post_dma_flush(void) +{ + if (IS_ENABLED(CONFIG_CPU_V6) || + IS_ENABLED(CONFIG_CPU_V6K) || + IS_ENABLED(CONFIG_CPU_V7) || + IS_ENABLED(CONFIG_CPU_V7M)) + return true; + + /* FIXME: runtime detection */ + return false; +} + /* * Make an area consistent for devices. * Note: Drivers should NOT use this function directly. @@ -674,25 +685,35 @@ static void dma_cache_maint(phys_addr_t paddr, void arch_sync_dma_for_device(phys_addr_t paddr, size_t size, enum dma_data_direction dir) { - dma_cache_maint(paddr, size, dir, dmac_map_area); - - if (dir == DMA_FROM_DEVICE) { - outer_inv_range(paddr, paddr + size); - } else { + switch (dir) { + case DMA_TO_DEVICE: + dma_cache_maint(paddr, size, dmac_clean_range); outer_clean_range(paddr, paddr + size); + break; + case DMA_FROM_DEVICE: + dma_cache_maint(paddr, size, dmac_inv_range); + outer_inv_range(paddr, paddr + size); + break; + case DMA_BIDIRECTIONAL: + if (arch_sync_dma_cpu_needs_post_dma_flush()) { + dma_cache_maint(paddr, size, dmac_clean_range); + outer_clean_range(paddr, paddr + size); + } else { + dma_cache_maint(paddr, size, dmac_flush_range); + outer_flush_range(paddr, paddr + size); + } + break; + default: + break; } - /* FIXME: non-speculating: flush on bidirectional mappings? */ } void arch_sync_dma_for_cpu(phys_addr_t paddr, size_t size, enum dma_data_direction dir) { - /* FIXME: non-speculating: not required */ - /* in any case, don't bother invalidating if DMA to device */ - if (dir != DMA_TO_DEVICE) { + if (dir != DMA_TO_DEVICE && arch_sync_dma_cpu_needs_post_dma_flush()) { outer_inv_range(paddr, paddr + size); - - dma_cache_maint(paddr, size, dir, dmac_unmap_area); + dma_cache_maint(paddr, size, dmac_inv_range); } /* -- 2.39.2