If we pipeline the PTE updates and then do the copy of those pages
within a single unpreemptible command packet, we can submit the copies
and leave them to be scheduled without having to synchronously wait
under a global lock.
Signed-off-by: Chris Wilson <chris@xxxxxxxxxxxxxxxxxx>
---
drivers/gpu/drm/i915/Makefile | 1 +
drivers/gpu/drm/i915/gt/intel_engine.h | 1 +
drivers/gpu/drm/i915/gt/intel_migrate.c | 370 ++++++++++++++++++
drivers/gpu/drm/i915/gt/intel_migrate.h | 33 ++
drivers/gpu/drm/i915/gt/selftest_migrate.c | 105 +++++
.../drm/i915/selftests/i915_live_selftests.h | 1 +
6 files changed, 511 insertions(+)
create mode 100644 drivers/gpu/drm/i915/gt/intel_migrate.c
create mode 100644 drivers/gpu/drm/i915/gt/intel_migrate.h
create mode 100644 drivers/gpu/drm/i915/gt/selftest_migrate.c
diff --git a/drivers/gpu/drm/i915/Makefile b/drivers/gpu/drm/i915/Makefile
index e5574e506a5c..0b2e12c87f9d 100644
--- a/drivers/gpu/drm/i915/Makefile
+++ b/drivers/gpu/drm/i915/Makefile
@@ -103,6 +103,7 @@ gt-y += \
gt/intel_gtt.o \
gt/intel_llc.o \
gt/intel_lrc.o \
+ gt/intel_migrate.o \
gt/intel_mocs.o \
gt/intel_ppgtt.o \
gt/intel_rc6.o \
diff --git a/drivers/gpu/drm/i915/gt/intel_engine.h b/drivers/gpu/drm/i915/gt/intel_engine.h
index ac58fcda4927..079d26b47a97 100644
--- a/drivers/gpu/drm/i915/gt/intel_engine.h
+++ b/drivers/gpu/drm/i915/gt/intel_engine.h
@@ -188,6 +188,7 @@ intel_write_status_page(struct intel_engine_cs *engine, int reg, u32 value)
#define I915_GEM_HWS_PREEMPT_ADDR (I915_GEM_HWS_PREEMPT * sizeof(u32))
#define I915_GEM_HWS_SEQNO 0x40
#define I915_GEM_HWS_SEQNO_ADDR (I915_GEM_HWS_SEQNO * sizeof(u32))
+#define I915_GEM_HWS_MIGRATE (0x42 * sizeof(u32))
#define I915_GEM_HWS_SCRATCH 0x80
#define I915_HWS_CSB_BUF0_INDEX 0x10
diff --git a/drivers/gpu/drm/i915/gt/intel_migrate.c b/drivers/gpu/drm/i915/gt/intel_migrate.c
new file mode 100644
index 000000000000..4d7bd32eb8d4
--- /dev/null
+++ b/drivers/gpu/drm/i915/gt/intel_migrate.c
@@ -0,0 +1,370 @@
+// SPDX-License-Identifier: MIT
+/*
+ * Copyright © 2020 Intel Corporation
+ */
+
+#include "i915_drv.h"
+#include "intel_context.h"
+#include "intel_gt.h"
+#include "intel_gtt.h"
+#include "intel_lrc.h" /* virtual engine */
+#include "intel_migrate.h"
+#include "intel_ring.h"
+
+#define CHUNK_SZ SZ_8M /* ~1ms at 8GiB/s preemption delay */
+
+static void insert_pte(struct i915_address_space *vm,
+ struct i915_page_table *pt,
+ void *data)
+{
+ u64 *offset = data;
+
+ vm->insert_page(vm, px_dma(pt), *offset, I915_CACHE_NONE, 0);
+ *offset += PAGE_SIZE;
+}
+
+static struct i915_address_space *migrate_vm(struct intel_gt *gt)
+{
+ struct i915_vm_pt_stash stash = {};
+ struct i915_ppgtt *vm;
+ u64 offset, sz;
+ int err;
+
+ vm = i915_ppgtt_create(gt);
+ if (IS_ERR(vm))
+ return ERR_CAST(vm);
+
+ if (!vm->vm.allocate_va_range || !vm->vm.foreach) {
+ err = -ENODEV;
+ goto err_vm;
+ }
+
+ /*
+ * We copy in 8MiB chunks. Each PDE covers 2MiB, so we need
+ * 4x2 page directories for source/destination.
+ */
+ sz = 2 * CHUNK_SZ;
+ offset = sz;
+
+ /*
+ * We need another page directory setup so that we can write
+ * the 8x512 PTE in each chunk.
+ */
+ sz += (sz >> 12) * sizeof(u64);
+
+ err = i915_vm_alloc_pt_stash(&vm->vm, &stash, sz);
+ if (err)
+ goto err_vm;
+
+ err = i915_vm_pin_pt_stash(&vm->vm, &stash);
+ if (err) {
+ i915_vm_free_pt_stash(&vm->vm, &stash);
+ goto err_vm;
+ }
+
+ vm->vm.allocate_va_range(&vm->vm, &stash, 0, sz);
+ i915_vm_free_pt_stash(&vm->vm, &stash);
+
+ /* Now allow the GPU to rewrite the PTE via its own ppGTT */
+ vm->vm.foreach(&vm->vm, 0, sz, insert_pte, &offset);
+
+ return &vm->vm;
+
+err_vm:
+ i915_vm_put(&vm->vm);
+ return ERR_PTR(err);
+}
+
+static struct intel_engine_cs *first_copy_engine(struct intel_gt *gt)
+{
+ struct intel_engine_cs *engine;
+ int i;
+
+ for (i = 0; i < ARRAY_SIZE(gt->engine_class[COPY_ENGINE_CLASS]); i++) {
+ engine = gt->engine_class[COPY_ENGINE_CLASS][i];
+ if (engine)
+ return engine;
+ }
+
+ return NULL;
+}
+
+static struct intel_context *pinned_context(struct intel_gt *gt)
+{
+ static struct lock_class_key key;
+ struct intel_engine_cs *engine;
+ struct i915_address_space *vm;
+ struct intel_context *ce;
+ int err;
+
+ engine = first_copy_engine(gt);
+ if (!engine)
+ return ERR_PTR(-ENODEV);
+
+ ce = intel_engine_create_pinned_context(engine, SZ_512K,
+ I915_GEM_HWS_MIGRATE,
+ &key, "migrate");
+ if (IS_ERR(ce))
+ return ce;
+
+ vm = migrate_vm(gt);
+ if (IS_ERR(vm)) {
+ err = PTR_ERR(vm);
+ goto err_ce;
+ }
+ i915_vm_put(ce->vm);
+ ce->vm = vm;
+
+ return ce;
+
+err_ce:
+ intel_context_put(ce);
+ return ERR_PTR(err);
+}
+
+int intel_migrate_init(struct intel_migrate *m, struct intel_gt *gt)
+{
+ struct intel_context *ce;
+
+ ce = pinned_context(gt);
+ if (IS_ERR(ce))
+ return PTR_ERR(ce);
+
+ m->ce = ce;
+ return 0;
+}
+
+static struct intel_context *__migrate_engines(struct intel_gt *gt)
+{
+ struct intel_engine_cs *engines[MAX_ENGINE_INSTANCE];
+ struct intel_engine_cs *engine;
+ unsigned int count, i;
+
+ count = 0;
+ for (i = 0; i < ARRAY_SIZE(gt->engine_class[COPY_ENGINE_CLASS]); i++) {
+ engine = gt->engine_class[COPY_ENGINE_CLASS][i];
+ if (engine)
+ engines[count++] = engine;
+ }
+
+ return intel_execlists_create_virtual(engines, count);
+}
+
+struct intel_context *intel_migrate_create_context(struct intel_migrate *m)
+{
+ struct intel_context *ce;
+
+ ce = __migrate_engines(m->ce->engine->gt);
+ if (IS_ERR(ce))
+ return ce;
+
+ ce->ring = __intel_context_ring_size(SZ_512K);
+
+ i915_vm_put(ce->vm);
+ ce->vm = i915_vm_get(m->ce->vm);
+
+ return ce;
+}
+
+static inline struct sgt_dma sg_sgt(struct scatterlist *sg)
+{
+ dma_addr_t addr = sg_dma_address(sg);
+
+ return (struct sgt_dma){ sg, addr, addr + sg_dma_len(sg) };
+}
+
+static int emit_pte(struct i915_request *rq,
+ struct sgt_dma *it,
+ u64 encode,
+ int offset,
+ int length)
+{
+ int total = 0;
+
+ offset >>= 12;
+ offset *= sizeof(u64);
+ offset += 2 * CHUNK_SZ;
+
+ do {
+ u32 *cs;
+
+ cs = intel_ring_begin(rq, 8);
+ if (IS_ERR(cs))
+ return PTR_ERR(cs);
+
+ *cs++ = MI_STORE_DWORD_IMM_GEN4;
+ *cs++ = offset;
+ *cs++ = 0;
+ *cs++ = lower_32_bits(encode | it->dma);
+ *cs++ = MI_STORE_DWORD_IMM_GEN4;
+ *cs++ = offset + 4;
+ *cs++ = 0;
+ *cs++ = upper_32_bits(encode | it->dma);
+ intel_ring_advance(rq, cs);
+
+ offset += 8;
+ total += I915_GTT_PAGE_SIZE;
+
+ it->dma += I915_GTT_PAGE_SIZE;
+ if (it->dma >= it->max) {
+ it->sg = __sg_next(it->sg);
+ if (!it->sg || sg_dma_len(it->sg) == 0)
+ break;
+
+ it->dma = sg_dma_address(it->sg);
+ it->max = it->dma + sg_dma_len(it->sg);
+ }
+
+ if (total == length)
+ break;
+ } while (1);
+
+ return total;
+}
+
+static bool wa_1209644611_applies(int gen, u32 size)
+{
+ u32 height = size >> PAGE_SHIFT;
+
+ if (gen != 11)
+ return false;
+
+ return height % 4 == 3 && height <= 8;
+}
+
+static int emit_copy(struct i915_request *rq, int size)
+{
+ const int gen = INTEL_GEN(rq->engine->i915);
+ u32 *cs;
+
+ cs = intel_ring_begin(rq, gen >= 8 ? 10 : 6);
+ if (IS_ERR(cs))
+ return PTR_ERR(cs);
+
+ if (gen >= 9 && !wa_1209644611_applies(gen, size)) {
+ *cs++ = GEN9_XY_FAST_COPY_BLT_CMD | (10 - 2);
+ *cs++ = BLT_DEPTH_32 | PAGE_SIZE;
+ *cs++ = 0;
+ *cs++ = size >> PAGE_SHIFT << 16 | PAGE_SIZE / 4;
+ *cs++ = CHUNK_SZ; /* dst offset */
+ *cs++ = 0;
+ *cs++ = 0;
+ *cs++ = PAGE_SIZE;
+ *cs++ = 0; /* src offset */
+ *cs++ = 0;
+ } else if (gen >= 8) {
+ *cs++ = XY_SRC_COPY_BLT_CMD | BLT_WRITE_RGBA | (10 - 2);
+ *cs++ = BLT_DEPTH_32 | BLT_ROP_SRC_COPY | PAGE_SIZE;
+ *cs++ = 0;
+ *cs++ = size >> PAGE_SHIFT << 16 | PAGE_SIZE / 4;
+ *cs++ = CHUNK_SZ; /* dst offset */
+ *cs++ = 0;
+ *cs++ = 0;
+ *cs++ = PAGE_SIZE;
+ *cs++ = 0; /* src offset */
+ *cs++ = 0;
+ } else {
+ *cs++ = SRC_COPY_BLT_CMD | BLT_WRITE_RGBA | (6 - 2);
+ *cs++ = BLT_DEPTH_32 | BLT_ROP_SRC_COPY | PAGE_SIZE;
+ *cs++ = size >> PAGE_SHIFT << 16 | PAGE_SIZE;
+ *cs++ = CHUNK_SZ; /* dst offset */
+ *cs++ = PAGE_SIZE;
+ *cs++ = 0; /* src offset */
+ }
+
+ intel_ring_advance(rq, cs);
+ return 0;
+}
+
+struct i915_request *
+intel_context_migrate_pages(struct intel_context *ce,
+ struct scatterlist *src,
+ struct scatterlist *dst)
+{
+ struct sgt_dma it_s = sg_sgt(src), it_d = sg_sgt(dst);
+ u64 encode = ce->vm->pte_encode(0, I915_CACHE_LLC, 0); /* flags */
+ struct i915_request *rq;
+ int len;
+ int err;
+
+ /* GEM_BUG_ON(ce->vm != migrate_vm); */
+
+ err = intel_context_pin(ce);
+ if (err)
+ return ERR_PTR(err);
+
+ GEM_BUG_ON(ce->ring->size < SZ_64K);
+
+ do {
+ rq = i915_request_create(ce);
+ if (IS_ERR(rq)) {
+ err = PTR_ERR(rq);
+ goto out_ce;
+ }
+
+ len = emit_pte(rq, &it_s, encode, 0, CHUNK_SZ);
+ if (len <= 0) {
+ err = len;
+ goto out_rq;
+ }
+
+ if (emit_pte(rq, &it_d, encode, CHUNK_SZ, len) < len) {
+ err = -EINVAL;
+ goto out_rq;
+ }
+
+ err = rq->engine->emit_flush(rq, EMIT_INVALIDATE);
+ if (err)
+ goto out_rq;
+
+ err = emit_copy(rq, len);
+ if (err)
+ goto out_rq;
+
+ if (!it_s.sg)
+ i915_request_get(rq);
+out_rq:
+ i915_request_add(rq);
+ if (it_s.sg)
+ cond_resched();
+ } while (err == 0 && it_s.sg);
+
+out_ce:
+ intel_context_unpin(ce);
+ return err ? ERR_PTR(err) : rq;
+}
+
+struct i915_request *
+intel_migrate_pages(struct intel_migrate *m,
+ struct scatterlist *src,
+ struct scatterlist *dst)
+{
+ struct intel_context *ce;
+ struct i915_request *rq;
+
+ if (!m->ce)
+ return ERR_PTR(-ENODEV);
+
+ ce = intel_migrate_create_context(m);
+ if (IS_ERR(ce))
+ ce = intel_context_get(m->ce);
+ GEM_BUG_ON(IS_ERR(ce));
+
+ rq = intel_context_migrate_pages(ce, src, dst);
+
+ intel_context_put(ce);
+ return rq;
+}
+
+void intel_migrate_fini(struct intel_migrate *m)
+{
+ if (!m->ce)
+ return;
+
+ intel_context_unpin(m->ce);
+ intel_context_put(m->ce);
+}
+
+#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
+#include "selftest_migrate.c"
+#endif
diff --git a/drivers/gpu/drm/i915/gt/intel_migrate.h b/drivers/gpu/drm/i915/gt/intel_migrate.h
new file mode 100644
index 000000000000..8c3c446fbd33
--- /dev/null
+++ b/drivers/gpu/drm/i915/gt/intel_migrate.h
@@ -0,0 +1,33 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright © 2020 Intel Corporation
+ */
+
+#ifndef __INTEL_MIGRATE__
+#define __INTEL_MIGRATE__
+
+struct i915_request;
+struct intel_context;
+struct intel_gt;
+struct scatterlist;
+
+struct intel_migrate {
+ struct intel_context *ce;
+};
+
+int intel_migrate_init(struct intel_migrate *m, struct intel_gt *gt);
+
+struct i915_request *
+intel_migrate_pages(struct intel_migrate *m,
+ struct scatterlist *src,
+ struct scatterlist *dst);
+
+struct intel_context *intel_migrate_create_context(struct intel_migrate *m);
+struct i915_request *
+intel_context_migrate_pages(struct intel_context *ce,
+ struct scatterlist *src,
+ struct scatterlist *dst);
+
+void intel_migrate_fini(struct intel_migrate *m);
+
+#endif /* __INTEL_MIGRATE__ */
diff --git a/drivers/gpu/drm/i915/gt/selftest_migrate.c b/drivers/gpu/drm/i915/gt/selftest_migrate.c
new file mode 100644
index 000000000000..d5102058fe3b
--- /dev/null
+++ b/drivers/gpu/drm/i915/gt/selftest_migrate.c
@@ -0,0 +1,105 @@
+// SPDX-License-Identifier: MIT
+/*
+ * Copyright © 2020 Intel Corporation
+ */
+
+static int live_migrate(void *arg)
+{
+ struct intel_migrate *m = arg;
+ struct drm_i915_private *i915 = m->ce->engine->i915;
+ const unsigned int sizes[] = {
+ SZ_4K,
+ SZ_64K,
+ SZ_2M,
+ SZ_64M,
+ //SZ_2G,
+ };
+ int err = 0;
+ int i, j;
+
+ for (i = 0; i < ARRAY_SIZE(sizes); i++) {
+ struct drm_i915_gem_object *src, *dst;
+ struct i915_request *rq;
+ u32 *vaddr;
+
+ src = i915_gem_object_create_internal(i915, sizes[i]);
+ if (IS_ERR(src))
+ break;
+
+ vaddr = i915_gem_object_pin_map(src, I915_MAP_WC);
+ if (IS_ERR(vaddr)) {
+ i915_gem_object_put(src);
+ break;
+ }
+
+ for (j = 0; j < sizes[i] / sizeof(u32); j++)
+ vaddr[j] = j;
+ i915_gem_object_flush_map(src);
+
+ dst = i915_gem_object_create_internal(i915, sizes[i]);
+ if (IS_ERR(dst)) {
+ i915_gem_object_put(dst);
+ break;
+ }
+
+ vaddr = i915_gem_object_pin_map(dst, I915_MAP_WC);
+ if (IS_ERR(vaddr)) {
+ i915_gem_object_put(dst);
+ i915_gem_object_put(src);
+ break;
+ }
+
+ for (j = 0; j < sizes[i] / sizeof(u32); j++)
+ vaddr[j] = ~j;
+ i915_gem_object_flush_map(dst);
+
+ rq = intel_migrate_pages(m,
+ src->mm.pages->sgl,
+ dst->mm.pages->sgl);
+ if (IS_ERR(rq)) {
+ pr_err("Migration failed, size: %u\n", sizes[i]);
+ err = PTR_ERR(rq);
+ }
+
+ if (i915_request_wait(rq, 0, HZ) < 0) {
+ pr_err("Migration timed out, size: %u\n", sizes[i]);
+ err = -ETIME;
+ }
+ i915_request_put(rq);
+
+ for (j = 0; j < sizes[i] / sizeof(u32); j++) {
+ if (vaddr[j] != j) {
+ pr_err("Copy failed, size: %u, offset: %zu\n",
+ sizes[i], j * sizeof(u32));
+ igt_hexdump(vaddr + round_down(j, 1024), 4096);
+ err = -EINVAL;
+ break;
+ }
+ }
+
+ i915_gem_object_put(dst);
+ i915_gem_object_put(src);
+ i915_gem_drain_freed_objects(i915);
+ if (err)
+ break;
+ }
+
+ return err;
+}
+
+int intel_migrate_live_selftests(struct drm_i915_private *i915)
+{
+ static const struct i915_subtest tests[] = {
+ SUBTEST(live_migrate),
+ };
+ struct intel_migrate m;
+ int err;
+
+ if (intel_migrate_init(&m, &i915->gt))
+ return 0;
+
+ err = i915_subtests(tests, &m);
+ intel_migrate_fini(&m);
+
+ return err;
+}
diff --git a/drivers/gpu/drm/i915/selftests/i915_live_selftests.h b/drivers/gpu/drm/i915/selftests/i915_live_selftests.h
index a92c0e9b7e6b..be5e0191eaea 100644
--- a/drivers/gpu/drm/i915/selftests/i915_live_selftests.h
+++ b/drivers/gpu/drm/i915/selftests/i915_live_selftests.h
@@ -26,6 +26,7 @@ selftest(gt_mocs, intel_mocs_live_selftests)
selftest(gt_pm, intel_gt_pm_live_selftests)
selftest(gt_heartbeat, intel_heartbeat_live_selftests)
selftest(requests, i915_request_live_selftests)
+selftest(migrate, intel_migrate_live_selftests)
selftest(active, i915_active_live_selftests)
selftest(objects, i915_gem_object_live_selftests)
selftest(mman, i915_gem_mman_live_selftests)
--
2.20.1
_______________________________________________
Intel-gfx mailing list
Intel-gfx@xxxxxxxxxxxxxxxxxxxxx
https://lists.freedesktop.org/mailman/listinfo/intel-gfx
