Add an initial framework for changing the HW priorities of rings. The
framework allows requesting priority changes for the lifetime of an
amdgpu_job. After the job completes the priority will decay to the next
lowest priority for which a request is still valid.
A new ring function set_priority() can now be populated to take care of
the HW specific programming sequence for priority changes.
v2: set priority before emitting IB, and take a ref on amdgpu_job
Signed-off-by: Andres Rodriguez <andresx7 at gmail.com>
---
drivers/gpu/drm/amd/amdgpu/amdgpu.h | 4 +-
drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c | 2 +-
drivers/gpu/drm/amd/amdgpu/amdgpu_job.c | 10 ++++-
drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c | 73 ++++++++++++++++++++++++++++++++
drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h | 11 +++++
5 files changed, 96 insertions(+), 4 deletions(-)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
index 366f6d3..0676495 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
@@ -643,7 +643,7 @@ struct amdgpu_ib {
extern const struct amd_sched_backend_ops amdgpu_sched_ops;
-int amdgpu_job_alloc(struct amdgpu_device *adev, unsigned num_ibs,
+int amdgpu_job_alloc(struct amdgpu_device *adev, unsigned num_ibs, int priority,
struct amdgpu_job **job, struct amdgpu_vm *vm);
int amdgpu_job_alloc_with_ib(struct amdgpu_device *adev, unsigned size,
struct amdgpu_job **job);
@@ -997,6 +997,8 @@ struct amdgpu_job {
struct amdgpu_sync sync;
struct amdgpu_ib *ibs;
struct dma_fence *fence; /* the hw fence */
+ struct dma_fence_cb cb;
+ int priority;
uint32_t preamble_status;
uint32_t num_ibs;
void *owner;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
index 605d40e..19ce202 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
@@ -186,7 +186,7 @@ int amdgpu_cs_parser_init(struct amdgpu_cs_parser *p, void *data)
}
}
- ret = amdgpu_job_alloc(p->adev, num_ibs, &p->job, vm);
+ ret = amdgpu_job_alloc(p->adev, num_ibs, p->ctx->priority, &p->job, vm);
if (ret)
goto free_all_kdata;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c
index 86a1242..b445996 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c
@@ -39,7 +39,7 @@ static void amdgpu_job_timedout(struct amd_sched_job *s_job)
amdgpu_gpu_reset(job->adev);
}
-int amdgpu_job_alloc(struct amdgpu_device *adev, unsigned num_ibs,
+int amdgpu_job_alloc(struct amdgpu_device *adev, unsigned num_ibs, int priority,
struct amdgpu_job **job, struct amdgpu_vm *vm)
{
size_t size = sizeof(struct amdgpu_job);
@@ -55,6 +55,7 @@ int amdgpu_job_alloc(struct amdgpu_device *adev, unsigned num_ibs,
(*job)->adev = adev;
(*job)->vm = vm;
+ (*job)->priority = priority;
(*job)->ibs = (void *)&(*job)[1];
(*job)->num_ibs = num_ibs;
@@ -68,7 +69,7 @@ int amdgpu_job_alloc_with_ib(struct amdgpu_device *adev, unsigned size,
{
int r;
- r = amdgpu_job_alloc(adev, 1, job, NULL);
+ r = amdgpu_job_alloc(adev, 1, AMDGPU_CTX_PRIORITY_NORMAL, job, NULL);
if (r)
return r;
@@ -169,6 +170,10 @@ static struct dma_fence *amdgpu_job_run(struct amd_sched_job *sched_job)
BUG_ON(amdgpu_sync_peek_fence(&job->sync, NULL));
+ r = amdgpu_ring_elevate_priority(job->ring, job->priority, job);
+ if (r)
+ DRM_ERROR("Failed to set job priority (%d)\n", r);
+
trace_amdgpu_sched_run_job(job);
r = amdgpu_ib_schedule(job->ring, job->num_ibs, job->ibs, job, &fence);
if (r)
@@ -177,6 +182,7 @@ static struct dma_fence *amdgpu_job_run(struct amd_sched_job *sched_job)
/* if gpu reset, hw fence will be replaced here */
dma_fence_put(job->fence);
job->fence = dma_fence_get(fence);
+
amdgpu_job_free_resources(job);
return fence;
}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c
index 31c6274..1bda215 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c
@@ -199,6 +199,77 @@ void amdgpu_ring_undo(struct amdgpu_ring *ring)
ring->funcs->end_use(ring);
}
+static void amdgpu_ring_restore_priority_cb(struct dma_fence *f,
+ struct dma_fence_cb *cb)
+{
+ int i;
+ struct amdgpu_job *cb_job =
+ container_of(cb, struct amdgpu_job, cb);
+ struct amdgpu_ring *ring = cb_job->ring;
+
+ spin_lock(&ring->priority_lock);
+
+ /* remove ourselves from the list if necessary */
+ if (cb_job == ring->last_job[cb_job->priority])
+ ring->last_job[cb_job->priority] = NULL;
+
+ /* something higher prio is executing, no need to decay */
+ if (ring->priority > cb_job->priority)
+ goto out_unlock;
+
+ /* decay priority to the next level with a job available */
+ for (i = cb_job->priority; i >= 0; i--) {
+ if (i == AMDGPU_CTX_PRIORITY_NORMAL || ring->last_job[i]) {
+ ring->priority = i;
+ if (ring->funcs->set_priority)
+ ring->funcs->set_priority(ring, i);
+
+ break;
+ }
+ }
+ amdgpu_job_put(&cb_job);
+
+out_unlock:
+ spin_unlock(&ring->priority_lock);
+}
+
+/**
+ * amdgpu_ring_elevate_priority - change the ring's priority
+ *
+ * @ring: amdgpu_ring structure holding the information
+ * @priority: target priority
+ * @job: priority should remain elevated for the duration of this job
+ *
+ * Use HW specific mechanism's to elevate the ring's priority while @job
+ * is executing. Once @job finishes executing, the ring will reset back
+ * to normal priority.
+ * Returns 0 on success, error otherwise
+ */
+int amdgpu_ring_elevate_priority(struct amdgpu_ring *ring, int priority,
+ struct amdgpu_job *job)
+{
+ if (priority < 0 || priority >= AMD_SCHED_MAX_PRIORITY)
+ return -EINVAL;
+
+ spin_lock(&ring->priority_lock);
+ ring->last_job[priority] = job;
+
+ if (priority <= ring->priority)
+ goto out_unlock;
+
+ ring->priority = priority;
+ if (ring->funcs->set_priority)
+ ring->funcs->set_priority(ring, priority);
+
+ amdgpu_job_get(job);
+ dma_fence_add_callback(&job->base.s_fence->finished, &job->cb,
+ amdgpu_ring_restore_priority_cb);
+
+out_unlock:
+ spin_unlock(&ring->priority_lock);
+ return 0;
+}
+
/**
* amdgpu_ring_init - init driver ring struct.
*
@@ -281,6 +352,8 @@ int amdgpu_ring_init(struct amdgpu_device *adev, struct amdgpu_ring *ring,
}
ring->ptr_mask = (ring->ring_size / 4) - 1;
ring->max_dw = max_dw;
+ ring->priority = AMDGPU_CTX_PRIORITY_NORMAL;
+ spin_lock_init(&ring->priority_lock);
INIT_LIST_HEAD(&ring->lru_list);
amdgpu_ring_lru_touch(adev, ring);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
index b51bdcd..aa1a8ff 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
@@ -24,6 +24,7 @@
#ifndef __AMDGPU_RING_H__
#define __AMDGPU_RING_H__
+#include <drm/amdgpu_drm.h>
#include "gpu_scheduler.h"
/* max number of rings */
@@ -137,6 +138,8 @@ struct amdgpu_ring_funcs {
void (*emit_cntxcntl) (struct amdgpu_ring *ring, uint32_t flags);
void (*emit_rreg)(struct amdgpu_ring *ring, uint32_t reg);
void (*emit_wreg)(struct amdgpu_ring *ring, uint32_t reg, uint32_t val);
+ /* priority functions */
+ void (*set_priority) (struct amdgpu_ring *ring, int priority);
};
struct amdgpu_ring {
@@ -171,6 +174,12 @@ struct amdgpu_ring {
unsigned cond_exe_offs;
u64 cond_exe_gpu_addr;
volatile u32 *cond_exe_cpu_addr;
+
+ spinlock_t priority_lock;
+ /* protected by priority_lock */
+ struct amdgpu_job *last_job[AMDGPU_CTX_PRIORITY_NUM];
+ int priority;
+
#if defined(CONFIG_DEBUG_FS)
struct dentry *ent;
#endif
@@ -183,6 +192,8 @@ void amdgpu_ring_insert_nop(struct amdgpu_ring *ring, uint32_t count);
void amdgpu_ring_generic_pad_ib(struct amdgpu_ring *ring, struct amdgpu_ib *ib);
void amdgpu_ring_commit(struct amdgpu_ring *ring);
void amdgpu_ring_undo(struct amdgpu_ring *ring);
+int amdgpu_ring_elevate_priority(struct amdgpu_ring *ring, int priority,
+ struct amdgpu_job *job);
int amdgpu_ring_init(struct amdgpu_device *adev, struct amdgpu_ring *ring,
unsigned ring_size, struct amdgpu_irq_src *irq_src,
unsigned irq_type);
--
2.9.3
