To be secure, when a userptr is invalidated the pages should be dma
unmapped ensuring the device can no longer touch the invalidated pages.
v2:
- Don't free sg table in MMU notifer, just dma unmap pages
Fixes: dd08ebf6c352 ("drm/xe: Introduce a new DRM driver for Intel GPUs")
Fixes: 12f4b58a37f4 ("drm/xe: Use hmm_range_fault to populate user pages")
Cc: Thomas Hellström <thomas.hellstrom@xxxxxxxxxxxxxxx>
Cc: stable@xxxxxxxxxxxxxxx # 6.8
Signed-off-by: Matthew Brost <matthew.brost@xxxxxxxxx>
---
drivers/gpu/drm/xe/xe_hmm.c | 42 ++++++++++++++++++++++++++------
drivers/gpu/drm/xe/xe_hmm.h | 1 +
drivers/gpu/drm/xe/xe_pt.c | 2 +-
drivers/gpu/drm/xe/xe_vm.c | 5 +++-
drivers/gpu/drm/xe/xe_vm_types.h | 5 +++-
5 files changed, 45 insertions(+), 10 deletions(-)
diff --git a/drivers/gpu/drm/xe/xe_hmm.c b/drivers/gpu/drm/xe/xe_hmm.c
index 2c32dc46f7d4..baf42514e1f9 100644
--- a/drivers/gpu/drm/xe/xe_hmm.c
+++ b/drivers/gpu/drm/xe/xe_hmm.c
@@ -112,16 +112,20 @@ static int xe_build_sg(struct xe_device *xe, struct hmm_range *range,
return ret;
}
-/*
- * xe_hmm_userptr_free_sg() - Free the scatter gather table of userptr
+#define need_unmap(__sg) ((u64)(__sg) & 0x1ull)
+#define clear_need_unmap(__sg) (__sg) = (struct sg_table *)((u64)(__sg) & ~0x1ull)
+#define set_need_unmap(__sg) (__sg) = (struct sg_table *)((u64)(__sg) | 0x1ull)
+
+/**
+ * xe_hmm_userptr_unmap_sg() - Unmap the scatter gather table of userptr
*
* @uvma: the userptr vma which hold the scatter gather table
*
* With function xe_userptr_populate_range, we allocate storage of
- * the userptr sg table. This is a helper function to free this
- * sg table, and dma unmap the address in the table.
+ * the userptr sg table. This is a helper function to dma unmap the address in
+ * the table.
*/
-void xe_hmm_userptr_free_sg(struct xe_userptr_vma *uvma)
+void xe_hmm_userptr_unmap_sg(struct xe_userptr_vma *uvma)
{
struct xe_userptr *userptr = &uvma->userptr;
struct xe_vma *vma = &uvma->vma;
@@ -129,11 +133,34 @@ void xe_hmm_userptr_free_sg(struct xe_userptr_vma *uvma)
struct xe_vm *vm = xe_vma_vm(vma);
struct xe_device *xe = vm->xe;
struct device *dev = xe->drm.dev;
+ bool do_unmap;
xe_assert(xe, userptr->sg);
- dma_unmap_sgtable(dev, userptr->sg,
- write ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE, 0);
+ spin_lock(&vm->userptr.invalidated_lock);
+ do_unmap = need_unmap(userptr->sg);
+ clear_need_unmap(userptr->sg);
+ spin_unlock(&vm->userptr.invalidated_lock);
+
+ if (do_unmap)
+ dma_unmap_sgtable(dev, userptr->sg,
+ write ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE, 0);
+}
+
+/*
+ * xe_hmm_userptr_free_sg() - Free the scatter gather table of userptr
+ *
+ * @uvma: the userptr vma which hold the scatter gather table
+ *
+ * With function xe_userptr_populate_range, we allocate storage of
+ * the userptr sg table. This is a helper function to free this
+ * sg table, and dma unmap the address in the table.
+ */
+void xe_hmm_userptr_free_sg(struct xe_userptr_vma *uvma)
+{
+ struct xe_userptr *userptr = &uvma->userptr;
+
+ xe_hmm_userptr_unmap_sg(uvma);
sg_free_table(userptr->sg);
userptr->sg = NULL;
}
@@ -244,6 +271,7 @@ int xe_hmm_userptr_populate_range(struct xe_userptr_vma *uvma,
xe_mark_range_accessed(&hmm_range, write);
userptr->sg = &userptr->sgt;
+ set_need_unmap(userptr->sg);
userptr->notifier_seq = hmm_range.notifier_seq;
free_pfns:
diff --git a/drivers/gpu/drm/xe/xe_hmm.h b/drivers/gpu/drm/xe/xe_hmm.h
index 909dc2bdcd97..56e653dc9fa2 100644
--- a/drivers/gpu/drm/xe/xe_hmm.h
+++ b/drivers/gpu/drm/xe/xe_hmm.h
@@ -9,3 +9,4 @@ struct xe_userptr_vma;
int xe_hmm_userptr_populate_range(struct xe_userptr_vma *uvma, bool is_mm_mmap_locked);
void xe_hmm_userptr_free_sg(struct xe_userptr_vma *uvma);
+void xe_hmm_userptr_unmap_sg(struct xe_userptr_vma *uvma);
diff --git a/drivers/gpu/drm/xe/xe_pt.c b/drivers/gpu/drm/xe/xe_pt.c
index 8d3765d3351e..b095257dc684 100644
--- a/drivers/gpu/drm/xe/xe_pt.c
+++ b/drivers/gpu/drm/xe/xe_pt.c
@@ -635,7 +635,7 @@ xe_pt_stage_bind(struct xe_tile *tile, struct xe_vma *vma,
if (!xe_vma_is_null(vma)) {
if (xe_vma_is_userptr(vma))
- xe_res_first_sg(to_userptr_vma(vma)->userptr.sg, 0,
+ xe_res_first_sg(vma_to_userptr_sg(vma), 0,
xe_vma_size(vma), &curs);
else if (xe_bo_is_vram(bo) || xe_bo_is_stolen(bo))
xe_res_first(bo->ttm.resource, xe_vma_bo_offset(vma),
diff --git a/drivers/gpu/drm/xe/xe_vm.c b/drivers/gpu/drm/xe/xe_vm.c
index dfd31b346021..c3d54dcf2a3e 100644
--- a/drivers/gpu/drm/xe/xe_vm.c
+++ b/drivers/gpu/drm/xe/xe_vm.c
@@ -637,6 +637,9 @@ static bool vma_userptr_invalidate(struct mmu_interval_notifier *mni,
XE_WARN_ON(err);
}
+ if (userptr->sg)
+ xe_hmm_userptr_unmap_sg(uvma);
+
trace_xe_vma_userptr_invalidate_complete(vma);
return true;
@@ -3405,7 +3408,7 @@ int xe_analyze_vm(struct drm_printer *p, struct xe_vm *vm, int gt_id)
if (is_null) {
addr = 0;
} else if (is_userptr) {
- struct sg_table *sg = to_userptr_vma(vma)->userptr.sg;
+ struct sg_table *sg = vma_to_userptr_sg(vma);
struct xe_res_cursor cur;
if (sg) {
diff --git a/drivers/gpu/drm/xe/xe_vm_types.h b/drivers/gpu/drm/xe/xe_vm_types.h
index ce1a63a5e3e7..0478a2235076 100644
--- a/drivers/gpu/drm/xe/xe_vm_types.h
+++ b/drivers/gpu/drm/xe/xe_vm_types.h
@@ -34,6 +34,9 @@ struct xe_vm;
#define XE_VMA_PTE_COMPACT (DRM_GPUVA_USERBITS << 9)
#define XE_VMA_DUMPABLE (DRM_GPUVA_USERBITS << 10)
+#define vma_to_userptr_sg(__vma) \
+ (struct sg_table *)((u64)to_userptr_vma((__vma))->userptr.sg & ~0x1ull)
+
/** struct xe_userptr - User pointer */
struct xe_userptr {
/** @invalidate_link: Link for the vm::userptr.invalidated list */
@@ -206,7 +209,7 @@ struct xe_vm {
struct rw_semaphore notifier_lock;
/**
* @userptr.invalidated_lock: Protects the
- * @userptr.invalidated list.
+ * @userptr.invalidated list and dma mapped pages of userptrs
*/
spinlock_t invalidated_lock;
/**
--
2.34.1
