If we convert the tracing over from direct use of ring->irq_get() and over to the breadcrumb infrastructure, we only have a single user of the ring->irq_get and so we will be able to simplify the driver routines (eliminating the redundant validation and irq refcounting). v2: Move to a signaling framework based upon the waiter. v3: Track the first-signal to avoid having to walk the rbtree everytime. v4: Mark the signaler thread as RT priority to reduce latency in the indirect wakeups. Signed-off-by: Chris Wilson <chris@xxxxxxxxxxxxxxxxxx> --- drivers/gpu/drm/i915/i915_drv.h | 8 -- drivers/gpu/drm/i915/i915_gem.c | 6 -- drivers/gpu/drm/i915/i915_trace.h | 2 +- drivers/gpu/drm/i915/intel_breadcrumbs.c | 176 +++++++++++++++++++++++++++++++ drivers/gpu/drm/i915/intel_ringbuffer.h | 7 +- 5 files changed, 183 insertions(+), 16 deletions(-) diff --git a/drivers/gpu/drm/i915/i915_drv.h b/drivers/gpu/drm/i915/i915_drv.h index ce685a83bb03..4468fff349d9 100644 --- a/drivers/gpu/drm/i915/i915_drv.h +++ b/drivers/gpu/drm/i915/i915_drv.h @@ -3807,14 +3807,6 @@ wait_remaining_ms_from_jiffies(unsigned long timestamp_jiffies, int to_wait_ms) schedule_timeout_uninterruptible(remaining_jiffies); } } - -static inline void i915_trace_irq_get(struct intel_engine_cs *engine, - struct drm_i915_gem_request *req) -{ - if (engine->trace_irq_req == NULL && engine->irq_get(engine)) - i915_gem_request_assign(&engine->trace_irq_req, req); -} - static inline bool __i915_request_irq_complete(struct drm_i915_gem_request *req) { struct intel_engine_cs *engine = req->engine; diff --git a/drivers/gpu/drm/i915/i915_gem.c b/drivers/gpu/drm/i915/i915_gem.c index 2c254cf49c15..0ec9ee8fd941 100644 --- a/drivers/gpu/drm/i915/i915_gem.c +++ b/drivers/gpu/drm/i915/i915_gem.c @@ -2935,12 +2935,6 @@ i915_gem_retire_requests_ring(struct intel_engine_cs *engine) i915_gem_object_retire__read(obj, engine->id); } - if (unlikely(engine->trace_irq_req && - i915_gem_request_completed(engine->trace_irq_req))) { - engine->irq_put(engine); - i915_gem_request_assign(&engine->trace_irq_req, NULL); - } - WARN_ON(i915_verify_lists(engine->dev)); } diff --git a/drivers/gpu/drm/i915/i915_trace.h b/drivers/gpu/drm/i915/i915_trace.h index 09bbb71e9ec5..f40ca977321c 100644 --- a/drivers/gpu/drm/i915/i915_trace.h +++ b/drivers/gpu/drm/i915/i915_trace.h @@ -490,7 +490,7 @@ TRACE_EVENT(i915_gem_ring_dispatch, __entry->ring = req->engine->id; __entry->seqno = req->seqno; __entry->flags = flags; - i915_trace_irq_get(req->engine, req); + intel_engine_enable_signaling(req); ), TP_printk("dev=%u, ring=%u, seqno=%u, flags=%x", diff --git a/drivers/gpu/drm/i915/intel_breadcrumbs.c b/drivers/gpu/drm/i915/intel_breadcrumbs.c index 578de43cb07e..14fb9fdcde3a 100644 --- a/drivers/gpu/drm/i915/intel_breadcrumbs.c +++ b/drivers/gpu/drm/i915/intel_breadcrumbs.c @@ -22,6 +22,8 @@ * */ +#include <linux/kthread.h> + #include "i915_drv.h" static void intel_breadcrumbs_fake_irq(unsigned long data) @@ -315,10 +317,184 @@ void intel_engine_init_breadcrumbs(struct intel_engine_cs *engine) (unsigned long)engine); } +struct signal { + struct rb_node node; + struct intel_wait wait; + struct drm_i915_gem_request *request; +}; + +static bool signal_complete(struct signal *signal) +{ + if (signal == NULL) + return false; + + /* If another process served as the bottom-half it may have already + * signalled that this wait is already completed. + */ + if (intel_wait_complete(&signal->wait)) + return true; + + /* Carefully check if the request is complete, giving time for the + * seqno to be visible or if the GPU hung. + */ + if (__i915_request_irq_complete(signal->request)) + return true; + + return false; +} + +static struct signal *to_signal(struct rb_node *rb) +{ + return container_of(rb, struct signal, node); +} + +static void signaler_set_rtpriority(void) +{ + struct sched_param param = { .sched_priority = 1 }; + sched_setscheduler_nocheck(current, SCHED_FIFO, ¶m); +} + +static int intel_breadcrumbs_signaler(void *arg) +{ + struct intel_engine_cs *engine = arg; + struct intel_breadcrumbs *b = &engine->breadcrumbs; + struct signal *signal; + + /* Install ourselves with high priority to reduce signalling latency */ + signaler_set_rtpriority(); + + do { + set_current_state(TASK_INTERRUPTIBLE); + + /* We are either woken up by the interrupt bottom-half, + * or by a client adding a new signaller. In both cases, + * the GPU seqno may have advanced beyond our oldest signal. + * If it has, propagate the signal, remove the waiter and + * check again with the next oldest signal. Otherwise we + * need to wait for a new interrupt from the GPU or for + * a new client. + */ + signal = READ_ONCE(b->first_signal); + if (signal_complete(signal)) { + /* Wake up all other completed waiters and select the + * next bottom-half for the next user interrupt. + */ + intel_engine_remove_wait(engine, &signal->wait); + + i915_gem_request_unreference(signal->request); + + /* Find the next oldest signal. Note that as we have + * not been holding the lock, another client may + * have installed an even older signal than the one + * we just completed - so double check we are still + * the oldest before picking the next one. + */ + spin_lock(&b->lock); + if (signal == b->first_signal) + b->first_signal = rb_next(&signal->node); + rb_erase(&signal->node, &b->signals); + spin_unlock(&b->lock); + + kfree(signal); + } else { + if (kthread_should_stop()) + break; + + schedule(); + } + } while (1); + + return 0; +} + +int intel_engine_enable_signaling(struct drm_i915_gem_request *request) +{ + struct intel_engine_cs *engine = request->engine; + struct intel_breadcrumbs *b = &engine->breadcrumbs; + struct rb_node *parent, **p; + struct task_struct *task; + struct signal *signal; + bool first; + + signal = kmalloc(sizeof(*signal), GFP_ATOMIC); + if (unlikely(signal == NULL)) + return -ENOMEM; + + /* Spawn a thread to provide a common bottom-half for all signals. + * As this is an asynchronous interface we cannot steal the current + * task for handling the bottom-half to the user interrupt, therefore + * we create a thread to do the coherent seqno dance after the + * interrupt and then signal the waitqueue (via the dma-buf/fence). + */ + task = READ_ONCE(b->signaler); + if (unlikely(task == NULL)) { + spin_lock(&b->lock); + task = b->signaler; + if (task == NULL) { + task = kthread_create(intel_breadcrumbs_signaler, + engine, + "irq/i915:%d", + engine->id); + if (!IS_ERR(task)) + b->signaler = task; + } + spin_unlock(&b->lock); + + if (IS_ERR(task)) { + kfree(signal); + return PTR_ERR(task); + } + } + + signal->wait.task = task; + signal->wait.seqno = request->seqno; + + signal->request = i915_gem_request_reference(request); + + /* Insert ourselves into the retirement ordered list of signals + * on this engine. We track the oldest seqno as that will be the + * first signal to complete. + */ + spin_lock(&b->lock); + parent = NULL; + first = true; + p = &b->signals.rb_node; + while (*p) { + parent = *p; + if (i915_seqno_passed(signal->wait.seqno, + to_signal(parent)->wait.seqno)) { + p = &parent->rb_right; + first = false; + } else + p = &parent->rb_left; + } + rb_link_node(&signal->node, parent, p); + rb_insert_color(&signal->node, &b->signals); + if (first) + smp_store_mb(b->first_signal, signal); + spin_unlock(&b->lock); + + /* Now add ourselves into the list of waiters, but register our + * bottom-half as the signaller thread. As per usual, only the oldest + * waiter (not just signaller) is tasked as the bottom-half waking + * up all completed waiters after the user interrupt. + * + * If we are the oldest waiter, enable the irq (after which we + * must double check that the seqno did not complete). + */ + if (intel_engine_add_wait(engine, &signal->wait)) + wake_up_process(task); + + return 0; +} + void intel_engine_fini_breadcrumbs(struct intel_engine_cs *engine) { struct intel_breadcrumbs *b = &engine->breadcrumbs; + if (b->signaler) + kthread_stop(b->signaler); + del_timer_sync(&b->fake_irq); } diff --git a/drivers/gpu/drm/i915/intel_ringbuffer.h b/drivers/gpu/drm/i915/intel_ringbuffer.h index a2e988068cb4..ca611fe6997e 100644 --- a/drivers/gpu/drm/i915/intel_ringbuffer.h +++ b/drivers/gpu/drm/i915/intel_ringbuffer.h @@ -141,6 +141,8 @@ struct i915_ctx_workarounds { struct drm_i915_gem_object *obj; }; +struct drm_i915_gem_request; + struct intel_engine_cs { struct drm_i915_private *i915; const char *name; @@ -179,7 +181,10 @@ struct intel_engine_cs { struct intel_breadcrumbs { spinlock_t lock; /* protects the lists of requests */ struct rb_root waiters; /* sorted by retirement, priority */ + struct rb_root signals; /* sorted by retirement */ struct task_struct *first_waiter; /* bh for user interrupts */ + struct task_struct *signaler; /* used for fence signalling */ + void *first_signal; struct timer_list fake_irq; /* used after a missed interrupt */ bool irq_enabled; bool rpm_wakelock; @@ -198,7 +203,6 @@ struct intel_engine_cs { unsigned irq_refcount; /* protected by dev_priv->irq_lock */ bool irq_posted; u32 irq_enable_mask; /* bitmask to enable ring interrupt */ - struct drm_i915_gem_request *trace_irq_req; bool __must_check (*irq_get)(struct intel_engine_cs *ring); void (*irq_put)(struct intel_engine_cs *ring); @@ -539,6 +543,7 @@ bool intel_engine_add_wait(struct intel_engine_cs *engine, struct intel_wait *wait); void intel_engine_remove_wait(struct intel_engine_cs *engine, struct intel_wait *wait); +int intel_engine_enable_signaling(struct drm_i915_gem_request *request); static inline bool intel_engine_has_waiter(struct intel_engine_cs *engine) { return READ_ONCE(engine->breadcrumbs.first_waiter); -- 2.8.1 _______________________________________________ Intel-gfx mailing list Intel-gfx@xxxxxxxxxxxxxxxxxxxxx https://lists.freedesktop.org/mailman/listinfo/intel-gfx