Use an rwsem as the notifier lock, and take it in read mode during
command submission.
This avoids the device-wide serialization of command submission in
the absence of userptr invalidations.
Cc: Christian König <christian.koenig@xxxxxxx>
Signed-off-by: Thomas Hellström <thomas.hellstrom@xxxxxxxxxxxxxxx>
---
drivers/gpu/drm/amd/amdgpu/amdgpu.h | 2 +-
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c | 2 +-
drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c | 12 ++++++++----
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 2 +-
drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c | 10 +++++-----
drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c | 2 +-
6 files changed, 17 insertions(+), 13 deletions(-)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
index d356e329e6f8..04fec2299c02 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
@@ -1051,7 +1051,7 @@ struct amdgpu_device {
struct rw_semaphore reset_sem;
struct amdgpu_doorbell_index doorbell_index;
- struct mutex notifier_lock;
+ struct rw_semaphore notifier_sem;
int asic_reset_res;
struct work_struct xgmi_reset_work;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
index 2d6b2d77b738..14be51692539 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
@@ -2049,7 +2049,7 @@ static int update_invalid_user_pages(struct amdkfd_process_info *process_info,
/*
* FIXME: Cannot ignore the return code, must hold
- * notifier_lock
+ * notifier_sem in read mode.
*/
amdgpu_ttm_tt_get_user_pages_done(bo->tbo.ttm);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
index 0311d799a010..26f447c49bdd 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
@@ -1224,11 +1224,13 @@ static int amdgpu_cs_submit(struct amdgpu_cs_parser *p,
drm_sched_job_arm(&job->base);
- /* No memory allocation is allowed while holding the notifier lock.
+ /* No memory allocation is allowed while holding the notifier sem.
* The lock is held until amdgpu_cs_submit is finished and fence is
* added to BOs.
*/
- mutex_lock(&p->adev->notifier_lock);
+ r = down_read_interruptible(&p->adev->notifier_sem);
+ if (r)
+ goto error_notifier;
/* If userptr are invalidated after amdgpu_cs_parser_bos(), return
* -EAGAIN, drmIoctl in libdrm will restart the amdgpu_cs_ioctl.
@@ -1288,13 +1290,15 @@ static int amdgpu_cs_submit(struct amdgpu_cs_parser *p,
}
ttm_eu_fence_buffer_objects(&p->ticket, &p->validated, p->fence);
- mutex_unlock(&p->adev->notifier_lock);
+ up_read(&p->adev->notifier_sem);
return 0;
error_abort:
+ up_read(&p->adev->notifier_sem);
+
+error_notifier:
drm_sched_job_cleanup(&job->base);
- mutex_unlock(&p->adev->notifier_lock);
error_unlock:
amdgpu_job_free(job);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
index 48089dc0180b..aa27b462152d 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
@@ -3476,7 +3476,7 @@ int amdgpu_device_init(struct amdgpu_device *adev,
atomic_set(&adev->in_gpu_reset, 0);
init_rwsem(&adev->reset_sem);
mutex_init(&adev->psp.mutex);
- mutex_init(&adev->notifier_lock);
+ init_rwsem(&adev->notifier_sem);
r = amdgpu_device_init_apu_flags(adev);
if (r)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c
index d3d340a6129c..e0b1b6e11bf5 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c
@@ -71,11 +71,11 @@ static bool amdgpu_mn_invalidate_gfx(struct mmu_interval_notifier *mni,
if (!mmu_notifier_range_blockable(range))
return false;
- mutex_lock(&adev->notifier_lock);
+ down_write(&adev->notifier_sem);
mmu_interval_set_seq(mni, cur_seq);
- mutex_unlock(&adev->notifier_lock);
+ up_write(&adev->notifier_sem);
r = dma_resv_wait_timeout(bo->tbo.base.resv, true, false,
MAX_SCHEDULE_TIMEOUT);
@@ -108,12 +108,12 @@ static bool amdgpu_mn_invalidate_hsa(struct mmu_interval_notifier *mni,
if (!mmu_notifier_range_blockable(range))
return false;
- mutex_lock(&adev->notifier_lock);
+ down_write(&adev->notifier_sem);
mmu_interval_set_seq(mni, cur_seq);
amdgpu_amdkfd_evict_userptr(bo->kfd_bo, bo->notifier.mm);
- mutex_unlock(&adev->notifier_lock);
+ up_write(&adev->notifier_sem);
return true;
}
@@ -215,7 +215,7 @@ int amdgpu_hmm_range_get_pages(struct mmu_interval_notifier *notifier,
/*
* Due to default_flags, all pages are HMM_PFN_VALID or
* hmm_range_fault() fails. FIXME: The pages cannot be touched outside
- * the notifier_lock, and mmu_interval_read_retry() must be done first.
+ * the notifier_sem, and mmu_interval_read_retry() must be done first.
*/
for (i = 0; pages && i < npages; i++)
pages[i] = hmm_pfn_to_page(pfns[i]);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
index 0cf94421665f..b2af53a05fb5 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
@@ -723,7 +723,7 @@ bool amdgpu_ttm_tt_get_user_pages_done(struct ttm_tt *ttm)
if (gtt->range) {
/*
- * FIXME: Must always hold notifier_lock for this, and must
+ * FIXME: Must always hold notifier_sem for this, and must
* not ignore the return code.
*/
r = amdgpu_hmm_range_get_pages_done(gtt->range);
--
2.31.1
