From: Vitaly Prosyak <vitaly.prosyak@xxxxxxx>
The current implementation of drm_sched_start uses a hardcoded -ECANCELED to dispose of a job when
the parent/hw fence is NULL. This results in drm_sched_job_done being called with -ECANCELED for
each job with a NULL parent in the pending list, making it difficult to distinguish between recovery
methods, whether a queue reset or a full GPU reset was used.
To improve this, we first try a soft recovery for timeout jobs and use the error code -ENODATA.
If soft recovery fails, we proceed with a queue reset, where the error code remains -ENODATA for
the job. Finally, for a full GPU reset, we use error codes -ECANCELED or -ETIME. This patch adds
an error code parameter to drm_sched_start, allowing us to differentiate between queue reset and
GPU reset failures. This enables user mode and test applications to validate the expected
correctness of the requested operation. After a successful queue reset, the only way to continue
normal operation is to call drm_sched_job_done with the specific error code -ENODATA.
v1: Initial implementation by Jesse utilized amdgpu_device_lock_reset_domain and
amdgpu_device_unlock_reset_domain to allow user mode to track the queue reset status and distinguish
between queue reset and GPU reset.
v2: Christian suggested using the error codes -ENODATA for queue reset and -ECANCELED or -ETIME for GPU
reset, returned to amdgpu_cs_wait_ioctl.
v3: To meet the requirements, we introduce a new function drm_sched_start_ex with an additional parameter
to set dma_fence_set_error, allowing us to handle the specific error codes appropriately and dispose
of bad jobs with the selected error code depending on whether it was a queue reset or GPU reset.
v4: Alex suggested using a new name, drm_sched_start_with_recovery_error, which more accurately describes
the function's purpose. Additionally, it was recommended to add documentation details about the new method.
v5: Fixed declaration of new function drm_sched_start_with_recovery_error.(Alex)
Cc: Alex Deucher <alexander.deucher@xxxxxxx>
Cc: Christian Koenig <christian.koenig@xxxxxxx>
Signed-off-by: Jesse Zhang <Jesse.Zhang@xxxxxxx>
Signed-off-by: Vitaly Prosyak <vitaly.prosyak@xxxxxxx>
---
drivers/gpu/drm/scheduler/sched_main.c | 30 +++++++++++++++++++++++---
include/drm/gpu_scheduler.h | 1 +
2 files changed, 28 insertions(+), 3 deletions(-)
diff --git a/drivers/gpu/drm/scheduler/sched_main.c b/drivers/gpu/drm/scheduler/sched_main.c
index 7e90c9f95611..c42449358b3f 100644
--- a/drivers/gpu/drm/scheduler/sched_main.c
+++ b/drivers/gpu/drm/scheduler/sched_main.c
@@ -671,13 +671,24 @@ void drm_sched_stop(struct drm_gpu_scheduler *sched, struct drm_sched_job *bad)
EXPORT_SYMBOL(drm_sched_stop);
/**
- * drm_sched_start - recover jobs after a reset
+ * drm_sched_start_with_recovery_error - recover jobs after a reset with
+ * custom error
*
* @sched: scheduler instance
* @full_recovery: proceed with complete sched restart
+ * @error : err code for set dma_fence_set_error
+ *
+ * Starts the scheduler and allows setting a custom dma_fence_set_error,
+ * which can be used to identify the recovery mechanism actually used.
*
+ * For example:
+ * - If a soft or queue reset was used, dma_fence_set_error is set to -ENODATA.
+ * - If an entire GPU reset was used, the error code is set to -ECANCELED.
+ *
+ * This approach enables user mode and test applications to know which
+ * recovery method was used for a given bad job.
*/
-void drm_sched_start(struct drm_gpu_scheduler *sched, bool full_recovery)
+void drm_sched_start_with_recovery_error(struct drm_gpu_scheduler *sched, bool full_recovery, int error)
{
struct drm_sched_job *s_job, *tmp;
int r;
@@ -704,7 +715,7 @@ void drm_sched_start(struct drm_gpu_scheduler *sched, bool full_recovery)
DRM_DEV_ERROR(sched->dev, "fence add callback failed (%d)\n",
r);
} else
- drm_sched_job_done(s_job, -ECANCELED);
+ drm_sched_job_done(s_job, error);
}
if (full_recovery)
@@ -712,6 +723,19 @@ void drm_sched_start(struct drm_gpu_scheduler *sched, bool full_recovery)
drm_sched_wqueue_start(sched);
}
+EXPORT_SYMBOL(drm_sched_start_with_recovery_error);
+
+/**
+ * drm_sched_start - recover jobs after a reset
+ *
+ * @sched: scheduler instance
+ * @full_recovery: proceed with complete sched restart
+ *
+ */
+void drm_sched_start(struct drm_gpu_scheduler *sched, bool full_recovery)
+{
+ drm_sched_start_with_recovery_error(sched, full_recovery, -ECANCELED);
+}
EXPORT_SYMBOL(drm_sched_start);
/**
diff --git a/include/drm/gpu_scheduler.h b/include/drm/gpu_scheduler.h
index 5acc64954a88..e80ea947a864 100644
--- a/include/drm/gpu_scheduler.h
+++ b/include/drm/gpu_scheduler.h
@@ -580,6 +580,7 @@ void drm_sched_wqueue_stop(struct drm_gpu_scheduler *sched);
void drm_sched_wqueue_start(struct drm_gpu_scheduler *sched);
void drm_sched_stop(struct drm_gpu_scheduler *sched, struct drm_sched_job *bad);
void drm_sched_start(struct drm_gpu_scheduler *sched, bool full_recovery);
+void drm_sched_start_with_recovery_error(struct drm_gpu_scheduler *sched, bool full_recovery, int error);
void drm_sched_resubmit_jobs(struct drm_gpu_scheduler *sched);
void drm_sched_increase_karma(struct drm_sched_job *bad);
void drm_sched_reset_karma(struct drm_sched_job *bad);
--
2.25.1
