This patch adds DMA_ATTR_ALLOC_EXACT_SIZE attribute to the DMA-mapping subsystem
By default dma coherent alloc/free functions allocates/release memory in
order of 2^pages. By specifying this attribute, allocation/release can
be done for exact size of memory there by reducing internal memory
fragmentation when allocation is in large chunks of MBs
Added attr version of dma_alloc/release_from_coherent()
Signed-off-by: Vandana Salve <vsalve@xxxxxxxxxx>
---
arch/arm/mm/dma-mapping.c | 6 ++---
drivers/base/dma-coherent.c | 51 ++++++++++++++++++++++++++++----------
include/asm-generic/dma-coherent.h | 15 ++++++++---
include/linux/dma-attrs.h | 1 +
4 files changed, 54 insertions(+), 19 deletions(-)
diff --git a/arch/arm/mm/dma-mapping.c b/arch/arm/mm/dma-mapping.c
index 79f8b39..a39c1c1 100644
--- a/arch/arm/mm/dma-mapping.c
+++ b/arch/arm/mm/dma-mapping.c
@@ -710,7 +710,7 @@ void *arm_dma_alloc(struct device *dev, size_t size, dma_addr_t *handle,
pgprot_t prot = __get_dma_pgprot(attrs, PAGE_KERNEL);
void *memory;
- if (dma_alloc_from_coherent(dev, size, handle, &memory))
+ if (dma_alloc_from_coherent_attr(dev, size, handle, &memory, attrs))
return memory;
return __dma_alloc(dev, size, handle, gfp, prot, false,
@@ -723,7 +723,7 @@ static void *arm_coherent_dma_alloc(struct device *dev, size_t size,
pgprot_t prot = __get_dma_pgprot(attrs, PAGE_KERNEL);
void *memory;
- if (dma_alloc_from_coherent(dev, size, handle, &memory))
+ if (dma_alloc_from_coherent_attr(dev, size, handle, &memory, attrs))
return memory;
return __dma_alloc(dev, size, handle, gfp, prot, true,
@@ -769,7 +769,7 @@ static void __arm_dma_free(struct device *dev, size_t size, void *cpu_addr,
{
struct page *page = pfn_to_page(dma_to_pfn(dev, handle));
- if (dma_release_from_coherent(dev, get_order(size), cpu_addr))
+ if (dma_release_from_coherent_attr(dev, size, cpu_addr, attrs))
return;
size = PAGE_ALIGN(size);
diff --git a/drivers/base/dma-coherent.c b/drivers/base/dma-coherent.c
index bc256b6..51e8ff6 100644
--- a/drivers/base/dma-coherent.c
+++ b/drivers/base/dma-coherent.c
@@ -96,26 +96,31 @@ void *dma_mark_declared_memory_occupied(struct device *dev,
EXPORT_SYMBOL(dma_mark_declared_memory_occupied);
/**
- * dma_alloc_from_coherent() - try to allocate memory from the per-device coherent area
+ * dma_alloc_from_coherent_attr() - try to allocate memory from the per-device
+ * coherent area
*
* @dev: device from which we allocate memory
* @size: size of requested memory area
* @dma_handle: This will be filled with the correct dma handle
* @ret: This pointer will be filled with the virtual address
* to allocated area.
+ * @attrs: DMA Attribute
*
* This function should be only called from per-arch dma_alloc_coherent()
* to support allocation from per-device coherent memory pools.
*
- * Returns 0 if dma_alloc_coherent should continue with allocating from
+ * Returns 0 if dma_alloc_coherent_attr should continue with allocating from
* generic memory areas, or !0 if dma_alloc_coherent should return @ret.
*/
-int dma_alloc_from_coherent(struct device *dev, ssize_t size,
- dma_addr_t *dma_handle, void **ret)
+int dma_alloc_from_coherent_attr(struct device *dev, ssize_t size,
+ dma_addr_t *dma_handle, void **ret,
+ struct dma_attrs *attrs)
{
struct dma_coherent_mem *mem;
int order = get_order(size);
int pageno;
+ unsigned int count;
+ unsigned long align;
if (!dev)
return 0;
@@ -127,11 +132,22 @@ int dma_alloc_from_coherent(struct device *dev, ssize_t size,
if (unlikely(size > (mem->size << PAGE_SHIFT)))
goto err;
+ if (dma_get_attr(DMA_ATTR_ALLOC_EXACT_SIZE, attrs)) {
+ align = 0;
+ count = PAGE_ALIGN(size) >> PAGE_SHIFT;
+ } else {
+ align = (1 << order) - 1;
+ count = 1 << order;
+ }
+
+ pageno = bitmap_find_next_zero_area(mem->bitmap, mem->size,
+ 0, count, align);
- pageno = bitmap_find_free_region(mem->bitmap, mem->size, order);
- if (unlikely(pageno < 0))
+ if (pageno >= mem->size)
goto err;
+ bitmap_set(mem->bitmap, pageno, count);
+
/*
* Memory was found in the per-device area.
*/
@@ -149,35 +165,44 @@ err:
*/
return mem->flags & DMA_MEMORY_EXCLUSIVE;
}
-EXPORT_SYMBOL(dma_alloc_from_coherent);
+EXPORT_SYMBOL(dma_alloc_from_coherent_attr);
/**
- * dma_release_from_coherent() - try to free the memory allocated from per-device coherent memory pool
+ * dma_release_from_coherent_attr() - try to free the memory allocated from
+ * per-device coherent memory pool
* @dev: device from which the memory was allocated
- * @order: the order of pages allocated
+ * @size: size of the memory area to free
* @vaddr: virtual address of allocated pages
+ * @attrs: DMA Attribute
*
* This checks whether the memory was allocated from the per-device
* coherent memory pool and if so, releases that memory.
*
* Returns 1 if we correctly released the memory, or 0 if
- * dma_release_coherent() should proceed with releasing memory from
+ * dma_release_coherent_attr() should proceed with releasing memory from
* generic pools.
*/
-int dma_release_from_coherent(struct device *dev, int order, void *vaddr)
+int dma_release_from_coherent_attr(struct device *dev, size_t size, void *vaddr,
+ struct dma_attrs *attrs)
{
struct dma_coherent_mem *mem = dev ? dev->dma_mem : NULL;
+ unsigned int count;
if (mem && vaddr >= mem->virt_base && vaddr <
(mem->virt_base + (mem->size << PAGE_SHIFT))) {
int page = (vaddr - mem->virt_base) >> PAGE_SHIFT;
- bitmap_release_region(mem->bitmap, page, order);
+ if (dma_get_attr(DMA_ATTR_ALLOC_EXACT_SIZE, attrs))
+ count = PAGE_ALIGN(size) >> PAGE_SHIFT;
+ else
+ count = 1 << size;
+
+ bitmap_clear(mem->bitmap, page, count);
return 1;
}
return 0;
}
-EXPORT_SYMBOL(dma_release_from_coherent);
+EXPORT_SYMBOL(dma_release_from_coherent_attr);
/**
* dma_mmap_from_coherent() - try to mmap the memory allocated from
diff --git a/include/asm-generic/dma-coherent.h b/include/asm-generic/dma-coherent.h
index 2be8a2d..55e49bd 100644
--- a/include/asm-generic/dma-coherent.h
+++ b/include/asm-generic/dma-coherent.h
@@ -6,9 +6,16 @@
* These three functions are only for dma allocator.
* Don't use them in device drivers.
*/
-int dma_alloc_from_coherent(struct device *dev, ssize_t size,
- dma_addr_t *dma_handle, void **ret);
-int dma_release_from_coherent(struct device *dev, int order, void *vaddr);
+int dma_alloc_from_coherent_attr(struct device *dev, ssize_t size,
+ dma_addr_t *dma_handle, void **ret,
+ struct dma_attrs *attrs);
+int dma_release_from_coherent_attr(struct device *dev, size_t size, void *vaddr,
+ struct dma_attrs *attrs);
+#define dma_alloc_from_coherent(d, s, h, r) \
+ dma_alloc_from_coherent_attr(d, s, h, r, NULL)
+
+#define dma_release_from_coherent(d, s, v) \
+ dma_release_from_coherent_attr(d, s, v, NULL)
int dma_mmap_from_coherent(struct device *dev, struct vm_area_struct *vma,
void *cpu_addr, size_t size, int *ret);
@@ -27,6 +34,8 @@ extern void *
dma_mark_declared_memory_occupied(struct device *dev,
dma_addr_t device_addr, size_t size);
#else
+#define dma_alloc_from_coherent_attr(dev, size, handle, ret, attr) (0)
+#define dma_release_from_coherent_attr(dev, size, vaddr, attr) (0)
#define dma_alloc_from_coherent(dev, size, handle, ret) (0)
#define dma_release_from_coherent(dev, order, vaddr) (0)
#define dma_mmap_from_coherent(dev, vma, vaddr, order, ret) (0)
diff --git a/include/linux/dma-attrs.h b/include/linux/dma-attrs.h
index c8e1831..d23f28f 100644
--- a/include/linux/dma-attrs.h
+++ b/include/linux/dma-attrs.h
@@ -18,6 +18,7 @@ enum dma_attr {
DMA_ATTR_NO_KERNEL_MAPPING,
DMA_ATTR_SKIP_CPU_SYNC,
DMA_ATTR_FORCE_CONTIGUOUS,
+ DMA_ATTR_ALLOC_EXACT_SIZE,
DMA_ATTR_MAX,
};
--
1.8.1.5
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@xxxxxxxxx. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@xxxxxxxxx";> email@xxxxxxxxx </a>