+};
+
+/* If needed, make unmap functionality part of ttm_kmap_iter_ops */
+static void kunmap_local_iter(struct ttm_kmap_iter *iter,
+ struct dma_buf_map *map)
{
- if (virtual && mem->bus.addr == NULL)
- iounmap(virtual);
- ttm_mem_io_free(bdev, mem);
+ if (!iter->ops->needs_unmap)
+ return;
+
+ if (map->is_iomem)
+ io_mapping_unmap_local(map->vaddr_iomem);
+ else
+ kunmap_local(map->vaddr);
}
-static int ttm_copy_io_page(void *dst, void *src, unsigned long page)
+/**
+ * ttm_move_memcpy - Helper to perform a memcpy ttm move operation.
+ * @bo: The struct ttm_buffer_object.
+ * @new_mem: The struct ttm_resource we're moving to (copy destination).
+ * @new_iter: A struct ttm_kmap_iter representing the destination resource.
+ * @old_iter: A struct ttm_kmap_iter representing the source resource.
+ *
+ * This function is intended to be able to move out async under a
+ * dma-fence if desired.
+ */
+void ttm_move_memcpy(struct ttm_buffer_object *bo,
+ struct ttm_resource *new_mem,
+ struct ttm_kmap_iter *new_iter,
+ struct ttm_kmap_iter *old_iter)
{
- uint32_t *dstP =
- (uint32_t *) ((unsigned long)dst + (page << PAGE_SHIFT));
- uint32_t *srcP =
- (uint32_t *) ((unsigned long)src + (page << PAGE_SHIFT));
-
- int i;
- for (i = 0; i < PAGE_SIZE / sizeof(uint32_t); ++i)
- iowrite32(ioread32(srcP++), dstP++);
- return 0;
+ struct ttm_device *bdev = bo->bdev;
+ struct ttm_resource_manager *man = ttm_manager_type(bdev, new_mem->mem_type);
+ struct ttm_tt *ttm = bo->ttm;
+ struct ttm_resource *old_mem = &bo->mem;
+ struct ttm_resource_manager *old_man = ttm_manager_type(bdev, old_mem->mem_type);
+ struct dma_buf_map old_map, new_map;
+ pgoff_t i;
+
+ /* Single TTM move. NOP */
+ if (old_man->use_tt && man->use_tt)
+ return;
+
+ /* Don't move nonexistent data. Clear destination instead. */
+ if (old_man->use_tt && !man->use_tt &&
+ (!ttm || !ttm_tt_is_populated(ttm))) {
+ if (ttm && !(ttm->page_flags & TTM_PAGE_FLAG_ZERO_ALLOC))
+ return;
+
+ for (i = 0; i < new_mem->num_pages; ++i) {
+ new_iter->ops->kmap_local(new_iter, &new_map, i);
+ if (new_map.is_iomem)
+ memset_io(new_map.vaddr_iomem, 0, PAGE_SIZE);
+ else
+ memset(new_map.vaddr, 0, PAGE_SIZE);
+ kunmap_local_iter(new_iter, &new_map);
+ }
+ return;
+ }
+
+ for (i = 0; i < new_mem->num_pages; ++i) {
+ new_iter->ops->kmap_local(new_iter, &new_map, i);
+ old_iter->ops->kmap_local(old_iter, &old_map, i);
+
+ if (!old_map.is_iomem && !new_map.is_iomem) {
+ memcpy(new_map.vaddr, old_map.vaddr, PAGE_SIZE);
+ } else if (!old_map.is_iomem) {
+ dma_buf_map_memcpy_to(&new_map, old_map.vaddr,
+ PAGE_SIZE);
+ } else if (!new_map.is_iomem) {
+ memcpy_fromio(new_map.vaddr, old_map.vaddr_iomem,
+ PAGE_SIZE);
+ } else {
+ int j;
+ u32 __iomem *src = old_map.vaddr_iomem;
+ u32 __iomem *dst = new_map.vaddr_iomem;
+
+ for (j = 0; j < (PAGE_SIZE >> 2); ++j)
+ iowrite32(ioread32(src++), dst++);
+ }
+ kunmap_local_iter(old_iter, &old_map);
+ kunmap_local_iter(new_iter, &new_map);
+ }
}
+EXPORT_SYMBOL(ttm_move_memcpy);
-static int ttm_copy_io_ttm_page(struct ttm_tt *ttm, void *src,
- unsigned long page,
- pgprot_t prot)
+/**
+ * ttm_kmap_iter_iomap_init - Initialize a struct ttm_kmap_iter_iomap
+ * @iter_io: The struct ttm_kmap_iter_iomap to initialize.
+ * @iomap: The struct io_mapping representing the underlying linear io_memory.
+ * @st: sg_table into @iomap, representing the memory of the struct
+ * ttm_resource.
+ * @start: Offset that needs to be subtracted from @st to make
+ * sg_dma_address(st->sgl) - @start == 0 for @iomap start.
+ *
+ * Return: Pointer to the embedded struct ttm_kmap_iter.
+ */
+struct ttm_kmap_iter *
+ttm_kmap_iter_iomap_init(struct ttm_kmap_iter_iomap *iter_io,
+ struct io_mapping *iomap,
+ struct sg_table *st,
+ resource_size_t start)
{
- struct page *d = ttm->pages[page];
- void *dst;
+ iter_io->base.ops = &ttm_kmap_iter_io_ops;
+ iter_io->iomap = iomap;
+ iter_io->st = st;
+ iter_io->start = start;
+ memset(&iter_io->cache, 0, sizeof(iter_io->cache));
- if (!d)
- return -ENOMEM;
+ return &iter_io->base;
+}
+EXPORT_SYMBOL(ttm_kmap_iter_iomap_init);
- src = (void *)((unsigned long)src + (page << PAGE_SHIFT));
- dst = kmap_atomic_prot(d, prot);
- if (!dst)
- return -ENOMEM;
+/**
+ * ttm_kmap_iter_tt_init - Initialize a struct ttm_kmap_iter_tt
+ * @iter_tt: The struct ttm_kmap_iter_tt to initialize.
+ * @tt: Struct ttm_tt holding page pointers of the struct ttm_resource.
+ *
+ * Return: Pointer to the embedded struct ttm_kmap_iter.
+ */
+struct ttm_kmap_iter *
+ttm_kmap_iter_tt_init(struct ttm_kmap_iter_tt *iter_tt,
+ struct ttm_tt *tt)
+{
+ iter_tt->base.ops = &ttm_kmap_iter_tt_ops;
+ iter_tt->tt = tt;
+ iter_tt->prot = ttm_prot_from_caching(tt->caching, PAGE_KERNEL);
- memcpy_fromio(dst, src, PAGE_SIZE);
+ return &iter_tt->base;
+}
+EXPORT_SYMBOL(ttm_kmap_iter_tt_init);
- kunmap_atomic(dst);
+/**
+ * DOC: Linear io iterator
+ *
+ * This code should die in the not too near future. Best would be if we could
+ * make io-mapping use memremap for all io memory, and have memremap
+ * implement a kmap_local functionality. We could then strip a huge amount of
+ * code. These linear io iterators are implemented to mimic old functionality,
+ * and they don't use kmap_local semantics at all internally. Rather ioremap or
+ * friends, and at least on 32-bit they add global TLB flushes and points
+ * of failure.
+ */
- return 0;
+/**
+ * struct ttm_kmap_iter_linear_io - Iterator specialization for linear io
+ * @base: The base iterator
+ * @dmap: Points to the starting address of the region
+ * @needs_unmap: Whether we need to unmap on fini
+ */
+struct ttm_kmap_iter_linear_io {
+ struct ttm_kmap_iter base;
+ struct dma_buf_map dmap;
+ bool needs_unmap;
+};
+
+static void ttm_kmap_iter_linear_io_kmap_local(struct ttm_kmap_iter *iter,
+ struct dma_buf_map *dmap,
+ pgoff_t i)
+{
+ struct ttm_kmap_iter_linear_io *iter_io =
+ container_of(iter, typeof(*iter_io), base);
+
+ *dmap = iter_io->dmap;
+ dma_buf_map_incr(dmap, i * PAGE_SIZE);
}
-static int ttm_copy_ttm_io_page(struct ttm_tt *ttm, void *dst,
- unsigned long page,
- pgprot_t prot)
+static const struct ttm_kmap_iter_ops ttm_kmap_iter_linear_io_ops = {
+ .kmap_local = ttm_kmap_iter_linear_io_kmap_local
+};
+
+static struct ttm_kmap_iter *
+ttm_kmap_iter_linear_io_init(struct ttm_kmap_iter_linear_io *iter_io,
+ struct ttm_device *bdev,
+ struct ttm_resource *mem)
{
- struct page *s = ttm->pages[page];
- void *src;
+ int ret;
- if (!s)
- return -ENOMEM;
+ ret = ttm_mem_io_reserve(bdev, mem);
+ if (ret)
+ goto out_err;
+ if (!mem->bus.is_iomem) {
+ ret = -EINVAL;
+ goto out_io_free;
+ }
- dst = (void *)((unsigned long)dst + (page << PAGE_SHIFT));
- src = kmap_atomic_prot(s, prot);
- if (!src)
- return -ENOMEM;
+ if (mem->bus.addr) {
+ dma_buf_map_set_vaddr(&iter_io->dmap, mem->bus.addr);
+ iter_io->needs_unmap = false;
+ } else {
+ size_t bus_size = (size_t)mem->num_pages << PAGE_SHIFT;
- memcpy_toio(dst, src, PAGE_SIZE);
+ iter_io->needs_unmap = true;
+ if (mem->bus.caching == ttm_write_combined)
+ dma_buf_map_set_vaddr_iomem(&iter_io->dmap,
+ ioremap_wc(mem->bus.offset,
+ bus_size));
+ else if (mem->bus.caching == ttm_cached)
+ dma_buf_map_set_vaddr(&iter_io->dmap,
+ memremap(mem->bus.offset, bus_size,
+ MEMREMAP_WB));
+ else
+ dma_buf_map_set_vaddr_iomem(&iter_io->dmap,
+ ioremap(mem->bus.offset,
+ bus_size));
+ if (dma_buf_map_is_null(&iter_io->dmap)) {
+ ret = -ENOMEM;
+ goto out_io_free;
+ }
+ }
- kunmap_atomic(src);
+ iter_io->base.ops = &ttm_kmap_iter_linear_io_ops;
+ return &iter_io->base;
- return 0;
+out_io_free:
+ ttm_mem_io_free(bdev, mem);
+out_err:
+ return ERR_PTR(ret);
+}
+
+static void
+ttm_kmap_iter_linear_io_fini(struct ttm_kmap_iter_linear_io *iter_io,
+ struct ttm_device *bdev,
+ struct ttm_resource *mem)
+{
+ if (iter_io->needs_unmap && dma_buf_map_is_set(&iter_io->dmap)) {
+ if (iter_io->dmap.is_iomem)
+ iounmap(iter_io->dmap.vaddr_iomem);
+ else
+ memunmap(iter_io->dmap.vaddr);
+ }
+
+ ttm_mem_io_free(bdev, mem);
}
+static int ttm_bo_wait_free_node(struct ttm_buffer_object *bo,
+ bool dst_use_tt);
+
int ttm_bo_move_memcpy(struct ttm_buffer_object *bo,
struct ttm_operation_ctx *ctx,
struct ttm_resource *new_mem)
{
struct ttm_device *bdev = bo->bdev;
struct ttm_resource_manager *man = ttm_manager_type(bdev, new_mem->mem_type);
+ struct ttm_resource_manager *new_man =
+ ttm_manager_type(bo->bdev, new_mem->mem_type);
struct ttm_tt *ttm = bo->ttm;
struct ttm_resource *old_mem = &bo->mem;
struct ttm_resource old_copy = *old_mem;
- void *old_iomap;
- void *new_iomap;
+ union {
+ struct ttm_kmap_iter_tt tt;
+ struct ttm_kmap_iter_linear_io io;
+ } _new_iter, _old_iter;
+ struct ttm_kmap_iter *new_iter, *old_iter;
int ret;
- unsigned long i;
-
- ret = ttm_bo_wait_ctx(bo, ctx);
- if (ret)
- return ret;
-
- ret = ttm_resource_ioremap(bdev, old_mem, &old_iomap);
- if (ret)
- return ret;
- ret = ttm_resource_ioremap(bdev, new_mem, &new_iomap);
- if (ret)
- goto out;
-
- /*
- * Single TTM move. NOP.
- */
- if (old_iomap == NULL && new_iomap == NULL)
- goto out2;
- /*
- * Don't move nonexistent data. Clear destination instead.
- */
- if (old_iomap == NULL &&
- (ttm == NULL || (!ttm_tt_is_populated(ttm) &&
- !(ttm->page_flags & TTM_PAGE_FLAG_SWAPPED)))) {
- memset_io(new_iomap, 0, new_mem->num_pages*PAGE_SIZE);
- goto out2;
- }
-
- /*
- * TTM might be null for moves within the same region.
- */
if (ttm) {
ret = ttm_tt_populate(bdev, ttm, ctx);
if (ret)
- goto out1;
+ return ret;
}
- for (i = 0; i < new_mem->num_pages; ++i) {
- if (old_iomap == NULL) {
- pgprot_t prot = ttm_io_prot(bo, old_mem, PAGE_KERNEL);
- ret = ttm_copy_ttm_io_page(ttm, new_iomap, i,
- prot);
- } else if (new_iomap == NULL) {
- pgprot_t prot = ttm_io_prot(bo, new_mem, PAGE_KERNEL);
- ret = ttm_copy_io_ttm_page(ttm, old_iomap, i,
- prot);
- } else {
- ret = ttm_copy_io_page(new_iomap, old_iomap, i);
- }
- if (ret)
- goto out1;
+ new_iter = new_man->use_tt ?
+ ttm_kmap_iter_tt_init(&_new_iter.tt, bo->ttm) :
+ ttm_kmap_iter_linear_io_init(&_new_iter.io, bdev, new_mem);
+ if (IS_ERR(new_iter))
+ return PTR_ERR(new_iter);
+
+ old_iter = man->use_tt ?
+ ttm_kmap_iter_tt_init(&_old_iter.tt, bo->ttm) :
+ ttm_kmap_iter_linear_io_init(&_old_iter.io, bdev, old_mem);
+ if (IS_ERR(old_iter)) {
+ ret = PTR_ERR(old_iter);
+ goto out_old_iter;
}
- mb();
-out2:
- old_copy = *old_mem;
- ttm_bo_assign_mem(bo, new_mem);
+ ttm_move_memcpy(bo, new_mem, new_iter, old_iter);
+ old_copy = *old_mem;
+ ret = ttm_bo_wait_free_node(bo, new_man->use_tt);
if (!man->use_tt)
- ttm_bo_tt_destroy(bo);
+ ttm_kmap_iter_linear_io_fini(&_old_iter.io, bdev, &old_copy);
+out_old_iter:
+ if (!new_man->use_tt)
+ ttm_kmap_iter_linear_io_fini(&_new_iter.io, bdev, new_mem);
-out1:
- ttm_resource_iounmap(bdev, old_mem, new_iomap);
-out:
- ttm_resource_iounmap(bdev, &old_copy, old_iomap);
-
- /*
- * On error, keep the mm node!
- */
- if (!ret)
- ttm_resource_free(bo, &old_copy);
return ret;
}
EXPORT_SYMBOL(ttm_bo_move_memcpy);
@@ -336,27 +514,7 @@ pgprot_t ttm_io_prot(struct ttm_buffer_object *bo, struct ttm_resource *res,
man = ttm_manager_type(bo->bdev, res->mem_type);
caching = man->use_tt ? bo->ttm->caching : res->bus.caching;
- /* Cached mappings need no adjustment */
- if (caching == ttm_cached)
- return tmp;
-
-#if defined(__i386__) || defined(__x86_64__)
- if (caching == ttm_write_combined)
- tmp = pgprot_writecombine(tmp);
- else if (boot_cpu_data.x86 > 3)
- tmp = pgprot_noncached(tmp);
-#endif
-#if defined(__ia64__) || defined(__arm__) || defined(__aarch64__) || \
- defined(__powerpc__) || defined(__mips__)
- if (caching == ttm_write_combined)
- tmp = pgprot_writecombine(tmp);
- else
- tmp = pgprot_noncached(tmp);
-#endif
-#if defined(__sparc__)
- tmp = pgprot_noncached(tmp);
-#endif
- return tmp;
+ return ttm_prot_from_caching(caching, tmp);
}
EXPORT_SYMBOL(ttm_io_prot);
diff --git a/include/drm/ttm/ttm_bo_driver.h b/include/drm/ttm/ttm_bo_driver.h
index dbccac957f8f..30f1dce987b2 100644
--- a/include/drm/ttm/ttm_bo_driver.h
+++ b/include/drm/ttm/ttm_bo_driver.h
@@ -332,4 +332,98 @@ int ttm_range_man_init(struct ttm_device *bdev,
int ttm_range_man_fini(struct ttm_device *bdev,
unsigned type);
+struct dma_buf_map;
+struct io_mapping;
+struct sg_table;
+struct scatterlist;
+struct ttm_kmap_iter;
+
+/**
+ * struct ttm_kmap_iter_ops - Ops structure for a struct
+ * ttm_kmap_iter.
+ * @needs_unmap - Whether a kunmap_local is needed to balance @kmap_local.
+ */
+struct ttm_kmap_iter_ops {
+ /**
+ * kmap_local - Map a PAGE_SIZE part of the resource using
+ * kmap_local semantics.
+ * @res_kmap: Pointer to the struct ttm_kmap_iter representing
+ * the resource.
+ * @dmap: The struct dma_buf_map holding the virtual address after
+ * the operation.
+ * @i: The location within the resource to map. PAGE_SIZE granularity.
+ */
+ void (*kmap_local)(struct ttm_kmap_iter *res_kmap,
+ struct dma_buf_map *dmap, pgoff_t i);
+ bool needs_unmap;
+};
+
+/**
+ * struct ttm_kmap_iter - Iterator for kmap_local type operations on a
+ * resource.
+ * @ops: Pointer to the operations struct.
+ *
+ * This struct is intended to be embedded in a resource-specific specialization
+ * implementing operations for the resource.
+ *
+ * Nothing stops us from extending the operations to vmap, vmap_pfn etc,
+ * replacing some or parts of the ttm_bo_util. cpu-map functionality.
+ */
+struct ttm_kmap_iter {
+ const struct ttm_kmap_iter_ops *ops;
+};
+
+/**
+ * struct ttm_kmap_iter_tt - Specialization for a tt (page) backed struct
+ * ttm_resource.
+ * @base: Embedded struct ttm_kmap_iter providing the usage interface
+ * @tt: Cached struct ttm_tt.
+ */
+struct ttm_kmap_iter_tt {
+ struct ttm_kmap_iter base;
+ struct ttm_tt *tt;
+ pgprot_t prot;
+};
+
+/**
+ * struct ttm_kmap_iter_iomap - Specialization for a struct io_mapping +
+ * struct sg_table backed struct ttm_resource.
+ * @base: Embedded struct ttm_kmap_iter providing the usage interface.
+ * @iomap: struct io_mapping representing the underlying linear io_memory.
+ * @st: sg_table into @iomap, representing the memory of the struct ttm_resource.
+ * @start: Offset that needs to be subtracted from @st to make
+ * sg_dma_address(st->sgl) - @start == 0 for @iomap start.
+ * @cache: Scatterlist traversal cache for fast lookups.
+ * @cache.sg: Pointer to the currently cached scatterlist segment.
+ * @cache.i: First index of @sg. PAGE_SIZE granularity.
+ * @cache.end: Last index + 1 of @sg. PAGE_SIZE granularity.
+ * @cache.offs: First offset into @iomap of @sg. PAGE_SIZE granularity.
+ */
+struct ttm_kmap_iter_iomap {
+ struct ttm_kmap_iter base;
+ struct io_mapping *iomap;
+ struct sg_table *st;
+ resource_size_t start;
+ struct {
+ struct scatterlist *sg;
+ pgoff_t i;
+ pgoff_t end;
+ pgoff_t offs;
+ } cache;
+};
+
+void ttm_move_memcpy(struct ttm_buffer_object *bo,
+ struct ttm_resource *new_mem,
+ struct ttm_kmap_iter *new_iter,
+ struct ttm_kmap_iter *old_iter);
+
+struct ttm_kmap_iter *
+ttm_kmap_iter_tt_init(struct ttm_kmap_iter_tt *iter_tt,
+ struct ttm_tt *tt);
+
+struct ttm_kmap_iter *
+ttm_kmap_iter_iomap_init(struct ttm_kmap_iter_iomap *iter_io,
+ struct io_mapping *iomap,
+ struct sg_table *st,
+ resource_size_t start);
#endif
