From: Petr Tesarik <petr.tesarik.ext@xxxxxxxxxx>
Do not walk the list of dynamically allocated bounce buffers if the
list is empty. This avoids taking dma_io_tlb_dyn_lock for devices
which do not use any dynamically allocated bounce buffers.
When unmapping the last dynamically allocated bounce buffer, the
flag is set to false as soon as possible to allow skipping the
spinlock even before the list itself is updated.
Signed-off-by: Petr Tesarik <petr.tesarik.ext@xxxxxxxxxx>
---
include/linux/device.h | 4 ++++
include/linux/swiotlb.h | 6 +++++-
kernel/dma/swiotlb.c | 6 ++++++
3 files changed, 15 insertions(+), 1 deletion(-)
diff --git a/include/linux/device.h b/include/linux/device.h
index d1d2b8557b30..e340e0f06dce 100644
--- a/include/linux/device.h
+++ b/include/linux/device.h
@@ -516,6 +516,9 @@ struct device_physical_location {
* @dma_io_tlb_dyn_slots:
* Dynamically allocated bounce buffers for this device.
* Not for driver use.
+ * @dma_io_tlb_have_dyn:
+ * Does this device have any dynamically allocated bounce
+ * buffers? Not for driver use.
* @archdata: For arch-specific additions.
* @of_node: Associated device tree node.
* @fwnode: Associated device node supplied by platform firmware.
@@ -623,6 +626,7 @@ struct device {
struct io_tlb_mem *dma_io_tlb_mem;
spinlock_t dma_io_tlb_dyn_lock;
struct list_head dma_io_tlb_dyn_slots;
+ bool dma_io_tlb_have_dyn;
#endif
/* arch specific additions */
struct dev_archdata archdata;
diff --git a/include/linux/swiotlb.h b/include/linux/swiotlb.h
index daa2064f2ede..8cbb0bebb0bc 100644
--- a/include/linux/swiotlb.h
+++ b/include/linux/swiotlb.h
@@ -152,7 +152,11 @@ static inline bool is_swiotlb_buffer(struct device *dev, phys_addr_t paddr)
return mem &&
(is_swiotlb_fixed(mem, paddr) ||
- (mem->allow_dyn && is_swiotlb_dyn(dev, paddr)));
+ /* Pairs with smp_store_release() in swiotlb_dyn_map()
+ * and swiotlb_dyn_unmap().
+ */
+ (smp_load_acquire(&dev->dma_io_tlb_have_dyn) &&
+ is_swiotlb_dyn(dev, paddr)));
}
static inline bool is_swiotlb_force_bounce(struct device *dev)
diff --git a/kernel/dma/swiotlb.c b/kernel/dma/swiotlb.c
index 81eab1c72c50..e8be3ee50f18 100644
--- a/kernel/dma/swiotlb.c
+++ b/kernel/dma/swiotlb.c
@@ -642,6 +642,9 @@ static phys_addr_t swiotlb_dyn_map(struct device *dev, phys_addr_t orig_addr,
spin_lock_irqsave(&dev->dma_io_tlb_dyn_lock, flags);
list_add(&slot->node, &dev->dma_io_tlb_dyn_slots);
+ if (!dev->dma_io_tlb_have_dyn)
+ /* Pairs with smp_load_acquire() in is_swiotlb_buffer() */
+ smp_store_release(&dev->dma_io_tlb_have_dyn, true);
spin_unlock_irqrestore(&dev->dma_io_tlb_dyn_lock, flags);
return page_to_phys(slot->page);
@@ -668,6 +671,9 @@ static void swiotlb_dyn_unmap(struct device *dev, phys_addr_t tlb_addr,
unsigned long flags;
spin_lock_irqsave(&dev->dma_io_tlb_dyn_lock, flags);
+ if (list_is_singular(&dev->dma_io_tlb_dyn_slots))
+ /* Pairs with smp_load_acquire() in is_swiotlb_buffer() */
+ smp_store_release(&dev->dma_io_tlb_have_dyn, false);
slot = lookup_dyn_slot_locked(dev, tlb_addr);
list_del(&slot->node);
spin_unlock_irqrestore(&dev->dma_io_tlb_dyn_lock, flags);
--
2.25.1
