All other parts are now in place, so implement the actual translation
step and hook it up, so the driver claims support for cores with
the new MMU.
Signed-off-by: Lucas Stach <l.stach@xxxxxxxxxxxxxx>
---
drivers/gpu/drm/etnaviv/etnaviv_iommu.h | 3 +-
drivers/gpu/drm/etnaviv/etnaviv_iommu_v2.c | 255 ++++++++++++++++++++++++++++-
drivers/gpu/drm/etnaviv/etnaviv_mmu.c | 2 +-
3 files changed, 256 insertions(+), 4 deletions(-)
diff --git a/drivers/gpu/drm/etnaviv/etnaviv_iommu.h b/drivers/gpu/drm/etnaviv/etnaviv_iommu.h
index 4b6212999c1e..8b51e7c16feb 100644
--- a/drivers/gpu/drm/etnaviv/etnaviv_iommu.h
+++ b/drivers/gpu/drm/etnaviv/etnaviv_iommu.h
@@ -17,11 +17,12 @@
#ifndef __ETNAVIV_IOMMU_H__
#define __ETNAVIV_IOMMU_H__
-#include <linux/iommu.h>
struct etnaviv_gpu;
struct iommu_domain *etnaviv_iommuv1_domain_alloc(struct etnaviv_gpu *gpu);
void etnaviv_iommuv1_restore(struct etnaviv_gpu *gpu);
+
struct iommu_domain *etnaviv_iommuv2_domain_alloc(struct etnaviv_gpu *gpu);
+void etnaviv_iommuv2_restore(struct etnaviv_gpu *gpu);
#endif /* __ETNAVIV_IOMMU_H__ */
diff --git a/drivers/gpu/drm/etnaviv/etnaviv_iommu_v2.c b/drivers/gpu/drm/etnaviv/etnaviv_iommu_v2.c
index 8c913c83ac5e..dea2e8c8b8a0 100644
--- a/drivers/gpu/drm/etnaviv/etnaviv_iommu_v2.c
+++ b/drivers/gpu/drm/etnaviv/etnaviv_iommu_v2.c
@@ -1,5 +1,5 @@
/*
- * Copyright (C) 2014 Christian Gmeiner <christian.gmeiner@xxxxxxxxx>
+ * Copyright (C) 2016 Etnaviv Project
*
* This program is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License version 2 as published by
@@ -22,12 +22,263 @@
#include <linux/bitops.h>
#include "etnaviv_gpu.h"
+#include "etnaviv_mmu.h"
#include "etnaviv_iommu.h"
+#include "state.xml.h"
#include "state_hi.xml.h"
+#define MMUv2_PTE_PRESENT BIT(0)
+#define MMUv2_PTE_EXCEPTION BIT(1)
+#define MMUv2_PTE_WRITEABLE BIT(2)
+#define MMUv2_MTLB_MASK 0xffc00000
+#define MMUv2_MTLB_SHIFT 22
+#define MMUv2_STLB_MASK 0x003ff000
+#define MMUv2_STLB_SHIFT 12
+
+#define MMUv2_MAX_STLB_ENTRIES 1024
+
+struct etnaviv_iommuv2_domain {
+ struct iommu_domain domain;
+ struct device *dev;
+ void *bad_page_cpu;
+ dma_addr_t bad_page_dma;
+ /* M(aster) TLB aka first level pagetable */
+ u32 *mtlb_cpu;
+ dma_addr_t mtlb_dma;
+ /* S(lave) TLB aka second level pagetable */
+ u32 *stlb_cpu[1024];
+ dma_addr_t stlb_dma[1024];
+};
+
+static struct etnaviv_iommuv2_domain *to_etnaviv_domain(struct iommu_domain *domain)
+{
+ return container_of(domain, struct etnaviv_iommuv2_domain, domain);
+}
+
+static int etnaviv_iommuv2_map(struct iommu_domain *domain, unsigned long iova,
+ phys_addr_t paddr, size_t size, int prot)
+{
+ struct etnaviv_iommuv2_domain *etnaviv_domain =
+ to_etnaviv_domain(domain);
+ int mtlb_entry, stlb_entry;
+ u32 entry = (u32)paddr | MMUv2_PTE_PRESENT;
+
+ if (size != SZ_4K)
+ return -EINVAL;
+
+ if (prot & IOMMU_WRITE)
+ entry |= MMUv2_PTE_WRITEABLE;
+
+ mtlb_entry = (iova & MMUv2_MTLB_MASK) >> MMUv2_MTLB_SHIFT;
+ stlb_entry = (iova & MMUv2_STLB_MASK) >> MMUv2_STLB_SHIFT;
+
+ etnaviv_domain->stlb_cpu[mtlb_entry][stlb_entry] = entry;
+
+ return 0;
+}
+
+static size_t etnaviv_iommuv2_unmap(struct iommu_domain *domain,
+ unsigned long iova, size_t size)
+{
+ struct etnaviv_iommuv2_domain *etnaviv_domain =
+ to_etnaviv_domain(domain);
+ int mtlb_entry, stlb_entry;
+
+ if (size != SZ_4K)
+ return -EINVAL;
+
+ mtlb_entry = (iova & MMUv2_MTLB_MASK) >> MMUv2_MTLB_SHIFT;
+ stlb_entry = (iova & MMUv2_STLB_MASK) >> MMUv2_STLB_SHIFT;
+
+ etnaviv_domain->stlb_cpu[mtlb_entry][stlb_entry] = MMUv2_PTE_EXCEPTION;
+
+ return SZ_4K;
+}
+
+static phys_addr_t etnaviv_iommuv2_iova_to_phys(struct iommu_domain *domain,
+ dma_addr_t iova)
+{
+ struct etnaviv_iommuv2_domain *etnaviv_domain =
+ to_etnaviv_domain(domain);
+ int mtlb_entry, stlb_entry;
+
+ mtlb_entry = (iova & MMUv2_MTLB_MASK) >> MMUv2_MTLB_SHIFT;
+ stlb_entry = (iova & MMUv2_STLB_MASK) >> MMUv2_STLB_SHIFT;
+
+ return etnaviv_domain->stlb_cpu[mtlb_entry][stlb_entry] & ~(SZ_4K - 1);
+}
+
+static int etnaviv_iommuv2_init(struct etnaviv_iommuv2_domain *etnaviv_domain)
+{
+ u32 *p;
+ int ret, i, j;
+
+ /* allocate scratch page */
+ etnaviv_domain->bad_page_cpu = dma_alloc_coherent(etnaviv_domain->dev,
+ SZ_4K,
+ &etnaviv_domain->bad_page_dma,
+ GFP_KERNEL);
+ if (!etnaviv_domain->bad_page_cpu) {
+ ret = -ENOMEM;
+ goto fail_mem;
+ }
+ p = etnaviv_domain->bad_page_cpu;
+ for (i = 0; i < SZ_4K / 4; i++)
+ *p++ = 0xdead55aa;
+
+ etnaviv_domain->mtlb_cpu = dma_alloc_coherent(etnaviv_domain->dev,
+ SZ_4K,
+ &etnaviv_domain->mtlb_dma,
+ GFP_KERNEL);
+ if (!etnaviv_domain->mtlb_cpu) {
+ ret = -ENOMEM;
+ goto fail_mem;
+ }
+
+ /* pre-populate STLB pages (may want to switch to on-demand later) */
+ for (i = 0; i < MMUv2_MAX_STLB_ENTRIES; i++) {
+ etnaviv_domain->stlb_cpu[i] =
+ dma_alloc_coherent(etnaviv_domain->dev,
+ SZ_4K,
+ &etnaviv_domain->stlb_dma[i],
+ GFP_KERNEL);
+ if (!etnaviv_domain->stlb_cpu[i]) {
+ ret = -ENOMEM;
+ goto fail_mem;
+ }
+ p = etnaviv_domain->stlb_cpu[i];
+ for (j = 0; j < SZ_4K / 4; j++)
+ *p++ = MMUv2_PTE_EXCEPTION;
+
+ etnaviv_domain->mtlb_cpu[i] = etnaviv_domain->stlb_dma[i] |
+ MMUv2_PTE_PRESENT;
+ }
+
+ return 0;
+
+fail_mem:
+ if (etnaviv_domain->bad_page_cpu)
+ dma_free_coherent(etnaviv_domain->dev, SZ_4K,
+ etnaviv_domain->bad_page_cpu,
+ etnaviv_domain->bad_page_dma);
+
+ if (etnaviv_domain->mtlb_cpu)
+ dma_free_coherent(etnaviv_domain->dev, SZ_4K,
+ etnaviv_domain->mtlb_cpu,
+ etnaviv_domain->mtlb_dma);
+
+ for (i = 0; i < MMUv2_MAX_STLB_ENTRIES; i++) {
+ if (etnaviv_domain->stlb_cpu[i])
+ dma_free_coherent(etnaviv_domain->dev, SZ_4K,
+ etnaviv_domain->stlb_cpu[i],
+ etnaviv_domain->stlb_dma[i]);
+ }
+
+ return ret;
+}
+
+static void etnaviv_iommuv2_domain_free(struct iommu_domain *domain)
+{
+ struct etnaviv_iommuv2_domain *etnaviv_domain =
+ to_etnaviv_domain(domain);
+ int i;
+
+ dma_free_coherent(etnaviv_domain->dev, SZ_4K,
+ etnaviv_domain->bad_page_cpu,
+ etnaviv_domain->bad_page_dma);
+
+ dma_free_coherent(etnaviv_domain->dev, SZ_4K,
+ etnaviv_domain->mtlb_cpu,
+ etnaviv_domain->mtlb_dma);
+
+ for (i = 0; i < MMUv2_MAX_STLB_ENTRIES; i++) {
+ if (etnaviv_domain->stlb_cpu[i])
+ dma_free_coherent(etnaviv_domain->dev, SZ_4K,
+ etnaviv_domain->stlb_cpu[i],
+ etnaviv_domain->stlb_dma[i]);
+ }
+
+ vfree(etnaviv_domain);
+}
+
+static size_t etnaviv_iommuv2_dump_size(struct iommu_domain *domain)
+{
+ struct etnaviv_iommuv2_domain *etnaviv_domain =
+ to_etnaviv_domain(domain);
+ size_t dump_size = SZ_4K;
+ int i;
+
+ for (i = 0; i < MMUv2_MAX_STLB_ENTRIES; i++)
+ if (etnaviv_domain->mtlb_cpu[i] & MMUv2_PTE_PRESENT)
+ dump_size += SZ_4K;
+
+ return dump_size;
+}
+
+static void etnaviv_iommuv2_dump(struct iommu_domain *domain, void *buf)
+{
+ struct etnaviv_iommuv2_domain *etnaviv_domain =
+ to_etnaviv_domain(domain);
+ int i;
+
+ memcpy(buf, etnaviv_domain->mtlb_cpu, SZ_4K);
+ buf += SZ_4K;
+ for (i = 0; i < MMUv2_MAX_STLB_ENTRIES; i++, buf += SZ_4K)
+ if (etnaviv_domain->mtlb_cpu[i] & MMUv2_PTE_PRESENT)
+ memcpy(buf, etnaviv_domain->stlb_cpu[i], SZ_4K);
+}
+
+static struct etnaviv_iommu_ops etnaviv_iommu_ops = {
+ .ops = {
+ .domain_free = etnaviv_iommuv2_domain_free,
+ .map = etnaviv_iommuv2_map,
+ .unmap = etnaviv_iommuv2_unmap,
+ .iova_to_phys = etnaviv_iommuv2_iova_to_phys,
+ .pgsize_bitmap = SZ_4K,
+ },
+ .dump_size = etnaviv_iommuv2_dump_size,
+ .dump = etnaviv_iommuv2_dump,
+};
+
+void etnaviv_iommuv2_restore(struct etnaviv_gpu *gpu)
+{
+ struct etnaviv_iommuv2_domain *etnaviv_domain =
+ to_etnaviv_domain(gpu->mmu->domain);
+ u16 prefetch;
+
+ prefetch = etnaviv_buffer_config_mmuv2(gpu,
+ (u32)etnaviv_domain->mtlb_dma,
+ (u32)etnaviv_domain->bad_page_dma);
+ etnaviv_gpu_start_fe(gpu, gpu->buffer->paddr, prefetch);
+ etnaviv_gpu_wait_idle(gpu, 100);
+
+ gpu_write(gpu, VIVS_MMUv2_CONTROL, VIVS_MMUv2_CONTROL_ENABLE);
+}
struct iommu_domain *etnaviv_iommuv2_domain_alloc(struct etnaviv_gpu *gpu)
{
- /* TODO */
+ struct etnaviv_iommuv2_domain *etnaviv_domain;
+ int ret;
+
+ etnaviv_domain = vzalloc(sizeof(*etnaviv_domain));
+ if (!etnaviv_domain)
+ return NULL;
+
+ etnaviv_domain->dev = gpu->dev;
+
+ etnaviv_domain->domain.type = __IOMMU_DOMAIN_PAGING;
+ etnaviv_domain->domain.ops = &etnaviv_iommu_ops.ops;
+ etnaviv_domain->domain.pgsize_bitmap = SZ_4K;
+ etnaviv_domain->domain.geometry.aperture_start = 0;
+ etnaviv_domain->domain.geometry.aperture_end = ~0UL & ~(SZ_4K - 1);
+
+ ret = etnaviv_iommuv2_init(etnaviv_domain);
+ if (ret)
+ goto out_free;
+
+ return &etnaviv_domain->domain;
+
+out_free:
+ vfree(etnaviv_domain);
return NULL;
}
diff --git a/drivers/gpu/drm/etnaviv/etnaviv_mmu.c b/drivers/gpu/drm/etnaviv/etnaviv_mmu.c
index 98c84ef862c7..304c0b4a9d58 100644
--- a/drivers/gpu/drm/etnaviv/etnaviv_mmu.c
+++ b/drivers/gpu/drm/etnaviv/etnaviv_mmu.c
@@ -313,7 +313,7 @@ void etnaviv_iommu_restore(struct etnaviv_gpu *gpu)
if (gpu->mmu->version == ETNAVIV_IOMMU_V1)
etnaviv_iommuv1_restore(gpu);
else
- dev_err(gpu->dev, "IOMMUv2 restore not implemented\n");
+ etnaviv_iommuv2_restore(gpu);
}
u32 etnaviv_iommu_get_cmdbuf_va(struct etnaviv_gpu *gpu,
--
2.8.1
_______________________________________________
dri-devel mailing list
dri-devel@xxxxxxxxxxxxxxxxxxxxx
https://lists.freedesktop.org/mailman/listinfo/dri-devel
