There is no reason to queue just a single job if scheduler can take more
and re-queue the worker to queue more. We can simply feed the hardware
with as much as it can take in one go and hopefully win some latency.
Signed-off-by: Tvrtko Ursulin <tvrtko.ursulin@xxxxxxxxxx>
Cc: Christian König <christian.koenig@xxxxxxx>
Cc: Danilo Krummrich <dakr@xxxxxxxxxx>
Cc: Matthew Brost <matthew.brost@xxxxxxxxx>
Cc: Philipp Stanner <pstanner@xxxxxxxxxx>
---
drivers/gpu/drm/scheduler/sched_main.c | 109 ++++++++++++-------------
drivers/gpu/drm/scheduler/sched_rq.c | 17 +---
include/drm/gpu_scheduler.h | 2 -
3 files changed, 54 insertions(+), 74 deletions(-)
diff --git a/drivers/gpu/drm/scheduler/sched_main.c b/drivers/gpu/drm/scheduler/sched_main.c
index f748dcd06774..e3d885678b9b 100644
--- a/drivers/gpu/drm/scheduler/sched_main.c
+++ b/drivers/gpu/drm/scheduler/sched_main.c
@@ -98,33 +98,6 @@ static u32 drm_sched_available_credits(struct drm_gpu_scheduler *sched)
return credits;
}
-/**
- * drm_sched_can_queue -- Can we queue more to the hardware?
- * @sched: scheduler instance
- * @entity: the scheduler entity
- *
- * Return true if we can push at least one more job from @entity, false
- * otherwise.
- */
-bool drm_sched_can_queue(struct drm_gpu_scheduler *sched,
- struct drm_sched_entity *entity)
-{
- struct drm_sched_job *s_job;
-
- s_job = to_drm_sched_job(spsc_queue_peek(&entity->job_queue));
- if (!s_job)
- return false;
-
- /* If a job exceeds the credit limit, truncate it to the credit limit
- * itself to guarantee forward progress.
- */
- if (drm_WARN(sched, s_job->credits > sched->credit_limit,
- "Jobs may not exceed the credit limit, truncate.\n"))
- s_job->credits = sched->credit_limit;
-
- return drm_sched_available_credits(sched) >= s_job->credits;
-}
-
/**
* drm_sched_run_job_queue - enqueue run-job work
* @sched: scheduler instance
@@ -928,48 +901,68 @@ static void drm_sched_run_job_work(struct work_struct *w)
{
struct drm_gpu_scheduler *sched =
container_of(w, struct drm_gpu_scheduler, work_run_job);
+ u32 job_credits, submitted_credits = 0;
struct drm_sched_entity *entity;
- struct dma_fence *fence;
struct drm_sched_job *sched_job;
- int r;
+ struct dma_fence *fence;
- /* Find entity with a ready job */
- entity = drm_sched_rq_select_entity(sched, sched->rq);
- if (IS_ERR_OR_NULL(entity))
- return; /* No more work */
+ for (;;) {
+ /* Find entity with a ready job */
+ entity = drm_sched_rq_select_entity(sched, sched->rq);
+ if (!entity)
+ break; /* No more work */
- sched_job = drm_sched_entity_pop_job(entity);
- if (!sched_job) {
+ /*
+ * If a job exceeds the credit limit truncate it to guarantee
+ * forward progress.
+ */
+ sched_job = to_drm_sched_job(spsc_queue_peek(&entity->job_queue));
+ job_credits = sched_job->credits;
+ if (drm_WARN_ONCE(sched, job_credits > sched->credit_limit,
+ "Jobs may not exceed the credit limit, truncating.\n"))
+ job_credits = sched_job->credits = sched->credit_limit;
+
+ if (job_credits > drm_sched_available_credits(sched)) {
+ complete_all(&entity->entity_idle);
+ break;
+ }
+
+ sched_job = drm_sched_entity_pop_job(entity);
complete_all(&entity->entity_idle);
- drm_sched_run_job_queue(sched);
- return;
- }
+ if (!sched_job) {
+ /* Top entity is not yet runnable after all */
+ continue;
+ }
+
+ drm_sched_job_begin(sched_job);
+ trace_drm_run_job(sched_job, entity);
+ submitted_credits += job_credits;
+ atomic_add(job_credits, &sched->credit_count);
- atomic_add(sched_job->credits, &sched->credit_count);
- drm_sched_job_begin(sched_job);
+ fence = sched->ops->run_job(sched_job);
+ drm_sched_fence_scheduled(sched_job->s_fence, fence);
- trace_drm_run_job(sched_job, entity);
- fence = sched->ops->run_job(sched_job);
- complete_all(&entity->entity_idle);
- drm_sched_fence_scheduled(sched_job->s_fence, fence);
+ if (!IS_ERR_OR_NULL(fence)) {
+ int r;
- if (!IS_ERR_OR_NULL(fence)) {
- /* Drop for original kref_init of the fence */
- dma_fence_put(fence);
+ /* Drop for original kref_init of the fence */
+ dma_fence_put(fence);
- r = dma_fence_add_callback(fence, &sched_job->cb,
- drm_sched_job_done_cb);
- if (r == -ENOENT)
- drm_sched_job_done(sched_job, fence->error);
- else if (r)
- DRM_DEV_ERROR(sched->dev, "fence add callback failed (%d)\n", r);
- } else {
- drm_sched_job_done(sched_job, IS_ERR(fence) ?
- PTR_ERR(fence) : 0);
+ r = dma_fence_add_callback(fence, &sched_job->cb,
+ drm_sched_job_done_cb);
+ if (r == -ENOENT)
+ drm_sched_job_done(sched_job, fence->error);
+ else if (r)
+ DRM_DEV_ERROR(sched->dev,
+ "fence add callback failed (%d)\n", r);
+ } else {
+ drm_sched_job_done(sched_job, IS_ERR(fence) ?
+ PTR_ERR(fence) : 0);
+ }
}
- wake_up(&sched->job_scheduled);
- drm_sched_run_job_queue(sched);
+ if (submitted_credits)
+ wake_up(&sched->job_scheduled);
}
/**
diff --git a/drivers/gpu/drm/scheduler/sched_rq.c b/drivers/gpu/drm/scheduler/sched_rq.c
index d9c854b2e495..647734d3ed73 100644
--- a/drivers/gpu/drm/scheduler/sched_rq.c
+++ b/drivers/gpu/drm/scheduler/sched_rq.c
@@ -153,9 +153,7 @@ void drm_sched_rq_pop_entity(struct drm_sched_rq *rq,
*
* Find oldest waiting ready entity.
*
- * Return an entity if one is found; return an error-pointer (!NULL) if an
- * entity was ready, but the scheduler had insufficient credits to accommodate
- * its job; return NULL, if no ready entity was found.
+ * Return an entity if one is found or NULL if no ready entity was found.
*/
struct drm_sched_entity *
drm_sched_rq_select_entity(struct drm_gpu_scheduler *sched,
@@ -174,17 +172,8 @@ drm_sched_rq_select_entity(struct drm_gpu_scheduler *sched,
}
spin_unlock(&rq->lock);
- if (!entity)
- return NULL;
-
- /*
- * If scheduler cannot take more jobs signal the caller to not consider
- * lower priority queues.
- */
- if (!drm_sched_can_queue(sched, entity))
- return ERR_PTR(-ENOSPC);
-
- reinit_completion(&entity->entity_idle);
+ if (entity)
+ reinit_completion(&entity->entity_idle);
return entity;
}
diff --git a/include/drm/gpu_scheduler.h b/include/drm/gpu_scheduler.h
index 9f952574efe2..6d3a38772e72 100644
--- a/include/drm/gpu_scheduler.h
+++ b/include/drm/gpu_scheduler.h
@@ -543,8 +543,6 @@ int drm_sched_init(struct drm_gpu_scheduler *sched,
atomic_t *score, const char *name, struct device *dev);
void drm_sched_fini(struct drm_gpu_scheduler *sched);
-bool drm_sched_can_queue(struct drm_gpu_scheduler *sched,
- struct drm_sched_entity *entity);
int drm_sched_job_init(struct drm_sched_job *job,
struct drm_sched_entity *entity,
--
2.47.1
