Add initial proof of concept implementation of DMA-mapping API for
devices that have IOMMU support. Right now only dma_alloc_coherent,
dma_free_coherent and dma_mmap_coherent functions are supported.
Signed-off-by: Marek Szyprowski <m.szyprowski@xxxxxxxxxxx>
Signed-off-by: Kyungmin Park <kyungmin.park@xxxxxxxxxxx>
---
arch/arm/Kconfig | 1 +
arch/arm/include/asm/device.h | 2 +
arch/arm/include/asm/dma-iommu.h | 30 ++++
arch/arm/mm/dma-mapping.c | 326 ++++++++++++++++++++++++++++++++++++++
4 files changed, 359 insertions(+), 0 deletions(-)
create mode 100644 arch/arm/include/asm/dma-iommu.h
diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig
index 377a7a5..61900f1 100644
--- a/arch/arm/Kconfig
+++ b/arch/arm/Kconfig
@@ -8,6 +8,8 @@ config ARM
select RTC_LIB
select SYS_SUPPORTS_APM_EMULATION
select GENERIC_ATOMIC64 if (CPU_V6 || !CPU_32v6K || !AEABI)
+ select GENERIC_ALLOCATOR
+ select HAVE_DMA_ATTRS
select HAVE_OPROFILE if (HAVE_PERF_EVENTS)
select HAVE_ARCH_KGDB
select HAVE_KPROBES if (!XIP_KERNEL && !THUMB2_KERNEL)
diff --git a/arch/arm/include/asm/device.h b/arch/arm/include/asm/device.h
index 005791a..d3ec1e9 100644
--- a/arch/arm/include/asm/device.h
+++ b/arch/arm/include/asm/device.h
@@ -11,6 +11,8 @@ struct dev_archdata {
#ifdef CONFIG_DMABOUNCE
struct dmabounce_device_info *dmabounce;
#endif
+ void *iommu_priv;
+ struct dma_iommu_mapping *mapping;
};
struct pdev_archdata {
diff --git a/arch/arm/include/asm/dma-iommu.h b/arch/arm/include/asm/dma-iommu.h
new file mode 100644
index 0000000..c246ff3
--- /dev/null
+++ b/arch/arm/include/asm/dma-iommu.h
@@ -0,0 +1,30 @@
+#ifndef ASMARM_DMA_IOMMU_H
+#define ASMARM_DMA_IOMMU_H
+
+#ifdef __KERNEL__
+
+#include <linux/mm_types.h>
+#include <linux/scatterlist.h>
+#include <linux/dma-debug.h>
+#include <linux/kmemcheck.h>
+
+#include <asm/memory.h>
+
+struct dma_iommu_mapping {
+ /* iommu specific data */
+ struct iommu_domain *domain;
+
+ /* address space data */
+ struct gen_pool *pool;
+
+ dma_addr_t base;
+ size_t size;
+
+ atomic_t ref_count;
+ struct mutex lock;
+};
+
+int __init arm_iommu_assign_device(struct device *dev, dma_addr_t base, dma_addr_t size);
+
+#endif /* __KERNEL__ */
+#endif
diff --git a/arch/arm/mm/dma-mapping.c b/arch/arm/mm/dma-mapping.c
index f8c6972..b6397c1 100644
--- a/arch/arm/mm/dma-mapping.c
+++ b/arch/arm/mm/dma-mapping.c
@@ -19,6 +19,7 @@
#include <linux/dma-mapping.h>
#include <linux/highmem.h>
#include <linux/slab.h>
+#include <linux/genalloc.h>
#include <asm/memory.h>
#include <asm/highmem.h>
@@ -26,6 +27,9 @@
#include <asm/tlbflush.h>
#include <asm/sizes.h>
+#include <linux/iommu.h>
+#include <asm/dma-iommu.h>
+
#ifdef __arch_page_to_dma
#error Please update to __arch_pfn_to_dma
#endif
@@ -1040,3 +1044,325 @@ static int __init dma_debug_do_init(void)
return 0;
}
fs_initcall(dma_debug_do_init);
+
+
+/* IOMMU */
+
+/*
+ * Allocate a DMA buffer for 'dev' of size 'size' using the
+ * specified gfp mask. Note that 'size' must be page aligned.
+ */
+static struct page **__iommu_alloc_buffer(struct device *dev, size_t size, gfp_t gfp)
+{
+ struct page **pages;
+ int count = size >> PAGE_SHIFT;
+ void *ptr;
+ int i;
+
+ pages = kzalloc(count * sizeof(struct page*), gfp);
+ if (!pages)
+ return NULL;
+
+ printk("IOMMU: page table allocated\n");
+
+ for (i=0; i<count; i++) {
+ pages[i] = alloc_page(gfp); //alloc_pages(gfp, 0);
+
+ if (!pages[i])
+ goto error;
+
+ /*
+ * Ensure that the allocated pages are zeroed, and that any data
+ * lurking in the kernel direct-mapped region is invalidated.
+ */
+ ptr = page_address(pages[i]);
+ memset(ptr, 0, PAGE_SIZE);
+ dmac_flush_range(ptr, ptr + PAGE_SIZE);
+ outer_flush_range(__pa(ptr), __pa(ptr) + PAGE_SIZE);
+ }
+ printk("IOMMU: pages allocated\n");
+ return pages;
+error:
+ printk("IOMMU: error allocating pages\n");
+ while (--i)
+ if (pages[i])
+ __free_pages(pages[i], 0);
+ kfree(pages);
+ return NULL;
+}
+
+static int __iommu_free_buffer(struct device *dev, struct page **pages, size_t size)
+{
+ int count = size >> PAGE_SHIFT;
+ int i;
+ for (i=0; i< count; i++)
+ if (pages[i])
+ __free_pages(pages[i], 0);
+ kfree(pages);
+ return 0;
+}
+
+static void *
+__iommu_alloc_remap(struct page **pages, size_t size, gfp_t gfp, pgprot_t prot)
+{
+ struct arm_vmregion *c;
+ size_t align;
+ size_t count = size >> PAGE_SHIFT;
+ int bit;
+
+ if (!consistent_pte[0]) {
+ printk(KERN_ERR "%s: not initialised\n", __func__);
+ dump_stack();
+ return NULL;
+ }
+
+ /*
+ * Align the virtual region allocation - maximum alignment is
+ * a section size, minimum is a page size. This helps reduce
+ * fragmentation of the DMA space, and also prevents allocations
+ * smaller than a section from crossing a section boundary.
+ */
+ bit = fls(size - 1);
+ if (bit > SECTION_SHIFT)
+ bit = SECTION_SHIFT;
+ align = 1 << bit;
+
+ /*
+ * Allocate a virtual address in the consistent mapping region.
+ */
+ c = arm_vmregion_alloc(&consistent_head, align, size,
+ gfp & ~(__GFP_DMA | __GFP_HIGHMEM));
+ if (c) {
+ pte_t *pte;
+ int idx = CONSISTENT_PTE_INDEX(c->vm_start);
+ int i = 0;
+ u32 off = CONSISTENT_OFFSET(c->vm_start) & (PTRS_PER_PTE-1);
+
+ pte = consistent_pte[idx] + off;
+ c->priv = pages;
+
+ do {
+ BUG_ON(!pte_none(*pte));
+
+ set_pte_ext(pte, mk_pte(pages[i], prot), 0);
+ pte++;
+ off++;
+ i++;
+ if (off >= PTRS_PER_PTE) {
+ off = 0;
+ pte = consistent_pte[++idx];
+ }
+ } while (i < count);
+
+ dsb();
+
+ return (void *)c->vm_start;
+ }
+ return NULL;
+}
+
+static dma_addr_t __iommu_create_mapping(struct device *dev, struct page **pages, size_t size)
+{
+ struct dma_iommu_mapping *mapping = dev->archdata.mapping;
+ unsigned int count = size >> PAGE_SHIFT;
+ dma_addr_t dma_addr, iova;
+ int i, ret = 0;
+
+ printk("IOMMU: mapping %p\n", mapping);
+
+ iova = gen_pool_alloc(mapping->pool, size);
+
+ printk("IOMMU: gen_alloc res %x\n", iova);
+
+ if (iova == 0)
+ goto fail;
+
+ dma_addr = iova;
+
+ for (i=0; i<count; i++) {
+ unsigned int phys = page_to_phys(pages[i]);
+ ret = iommu_map(mapping->domain, iova, phys, 0, 0);
+ if (ret < 0)
+ goto fail;
+ iova += PAGE_SIZE;
+ }
+
+ return dma_addr;
+fail:
+ return 0;
+}
+
+static int __iommu_remove_mapping(struct device *dev, dma_addr_t iova, size_t size)
+{
+ struct dma_iommu_mapping *mapping = dev->archdata.mapping;
+ unsigned int count = size >> PAGE_SHIFT;
+ int i;
+
+ gen_pool_free(mapping->pool, iova, size);
+
+ for (i=0; i<count; i++) {
+ iommu_unmap(mapping->domain, iova, 0);
+ iova += PAGE_SIZE;
+ }
+ return 0;
+}
+
+int arm_iommu_init(struct device *dev);
+
+static void *arm_iommu_alloc_attrs(struct device *dev, size_t size,
+ dma_addr_t *handle, gfp_t gfp, struct dma_attrs *attrs)
+{
+ struct dma_iommu_mapping *mapping = dev->archdata.mapping;
+ struct page **pages;
+ void *addr = NULL;
+ pgprot_t prot;
+
+ if (dma_get_attr(DMA_ATTR_WRITE_COMBINE, attrs))
+ prot = pgprot_writecombine(pgprot_kernel);
+ else
+ prot = pgprot_dmacoherent(pgprot_kernel);
+
+ arm_iommu_init(dev);
+
+ mutex_lock(&mapping->lock);
+
+ *handle = ~0;
+ size = PAGE_ALIGN(size);
+
+ printk("IOMMU: requested size %d\n", size);
+
+ pages = __iommu_alloc_buffer(dev, size, gfp);
+ if (!pages)
+ return NULL;
+
+ printk("IOMMU: allocated pages: %p\n", pages);
+
+ *handle = __iommu_create_mapping(dev, pages, size);
+
+ printk("IOMMU: created iova: %08x\n", *handle);
+
+ if (!*handle)
+ goto err_buffer;
+
+ addr = __iommu_alloc_remap(pages, size, gfp, prot);
+ if (!addr)
+ goto err_iommu;
+
+ printk("IOMMU: allocated iova %08x, virt %p\n", *handle, addr);
+
+err_iommu:
+err_buffer:
+ mutex_unlock(&mapping->lock);
+ return addr;
+}
+
+static int arm_iommu_mmap_attrs(struct device *dev, struct vm_area_struct *vma,
+ void *cpu_addr, dma_addr_t dma_addr, size_t size,
+ struct dma_attrs *attrs)
+{
+ unsigned long user_size;
+ struct arm_vmregion *c;
+
+ if (dma_get_attr(DMA_ATTR_WRITE_COMBINE, attrs))
+ vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot);
+ else
+ vma->vm_page_prot = pgprot_dmacoherent(vma->vm_page_prot);
+
+ printk("IOMMU: mmap virt %p, dma %08x, size %d\n", cpu_addr, dma_addr, size);
+
+ user_size = (vma->vm_end - vma->vm_start) >> PAGE_SHIFT;
+
+ c = arm_vmregion_find(&consistent_head, (unsigned long)cpu_addr);
+ if (c) {
+ struct page **pages = c->priv;
+
+ unsigned long uaddr = vma->vm_start;
+ unsigned long usize = vma->vm_end - vma->vm_start;
+ int i = 0;
+
+ do {
+ int ret;
+
+ ret = vm_insert_page(vma, uaddr, pages[i++]);
+ if (ret) {
+ printk(KERN_ERR "Remapping memory, error: %d\n", ret);
+ return ret;
+ }
+
+ uaddr += PAGE_SIZE;
+ usize -= PAGE_SIZE;
+ } while (usize > 0);
+ }
+ return 0;
+}
+
+/*
+ * free a page as defined by the above mapping.
+ * Must not be called with IRQs disabled.
+ */
+void arm_iommu_free_attrs(struct device *dev, size_t size, void *cpu_addr,
+ dma_addr_t handle, struct dma_attrs *attrs)
+{
+ struct dma_iommu_mapping *mapping = dev->archdata.mapping;
+ struct arm_vmregion *c;
+ size = PAGE_ALIGN(size);
+
+ mutex_lock(&mapping->lock);
+ c = arm_vmregion_find(&consistent_head, (unsigned long)cpu_addr);
+ if (c) {
+ struct page **pages = c->priv;
+ __dma_free_remap(cpu_addr, size);
+ __iommu_remove_mapping(dev, handle, size);
+ __iommu_free_buffer(dev, pages, size);
+ }
+ mutex_unlock(&mapping->lock);
+}
+
+struct arm_dma_map_ops iommu_ops = {
+ .alloc_attrs = arm_iommu_alloc_attrs,
+ .free_attrs = arm_iommu_free_attrs,
+ .mmap_attrs = arm_iommu_mmap_attrs,
+};
+EXPORT_SYMBOL_GPL(iommu_ops);
+
+int arm_iommu_init(struct device *dev)
+{
+ struct dma_iommu_mapping *mapping = dev->archdata.mapping;
+
+ if (mapping->pool)
+ return 0;
+
+ mutex_init(&mapping->lock);
+
+ mapping->pool = gen_pool_create(16, -1);
+ if (!mapping->pool)
+ return -ENOMEM;
+
+ if (gen_pool_add(mapping->pool, mapping->base, mapping->size, -1) != 0)
+ return -ENOMEM;
+
+ mapping->domain = iommu_domain_alloc();
+ if (!mapping->domain)
+ return -ENOMEM;
+
+ if (iommu_attach_device(mapping->domain, dev) != 0)
+ return -ENOMEM;
+
+ return 0;
+}
+
+int __init arm_iommu_assign_device(struct device *dev, dma_addr_t base, dma_addr_t size)
+{
+ struct dma_iommu_mapping *mapping;
+ mapping = kzalloc(sizeof(struct dma_iommu_mapping), GFP_KERNEL);
+ if (!mapping)
+ return -ENOMEM;
+ mapping->base = base;
+ mapping->size = size;
+
+ dev->archdata.mapping = mapping;
+ set_dma_ops(dev, &iommu_ops);
+ printk(KERN_INFO "Assigned IOMMU device to %s\n", dev_name(dev));
+
+ return 0;
+}
--
1.7.1.569.g6f426
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@xxxxxxxxxx For more info on Linux MM,
see: http://www.linux-mm.org/ .
Fight unfair telecom internet charges in Canada: sign http://stopthemeter.ca/
Don't email: <a href=mailto:"dont@xxxxxxxxx";> email@xxxxxxxxx </a>