If we convert the tracing over from direct use of ring->irq_get() and
over to the breadcrumb infrastructure, we only have a single user of the
ring->irq_get and so we will be able to simplify the driver routines
(eliminating the redundant validation and irq refcounting).
v2: Move to a signaling framework based upon the waiter.
v3: Track the first-signal to avoid having to walk the rbtree everytime.
v4: Mark the signaler thread as RT priority to reduce latency in the
indirect wakeups.
Signed-off-by: Chris Wilson <chris@xxxxxxxxxxxxxxxxxx>
---
drivers/gpu/drm/i915/i915_drv.h | 8 --
drivers/gpu/drm/i915/i915_gem.c | 6 --
drivers/gpu/drm/i915/i915_irq.c | 7 +-
drivers/gpu/drm/i915/i915_trace.h | 2 +-
drivers/gpu/drm/i915/intel_breadcrumbs.c | 177 +++++++++++++++++++++++++++++++
drivers/gpu/drm/i915/intel_ringbuffer.h | 7 +-
6 files changed, 186 insertions(+), 21 deletions(-)
diff --git a/drivers/gpu/drm/i915/i915_drv.h b/drivers/gpu/drm/i915/i915_drv.h
index 8940b8d3fa59..7f021505e32f 100644
--- a/drivers/gpu/drm/i915/i915_drv.h
+++ b/drivers/gpu/drm/i915/i915_drv.h
@@ -3620,14 +3620,6 @@ wait_remaining_ms_from_jiffies(unsigned long timestamp_jiffies, int to_wait_ms)
schedule_timeout_uninterruptible(remaining_jiffies);
}
}
-
-static inline void i915_trace_irq_get(struct intel_engine_cs *ring,
- struct drm_i915_gem_request *req)
-{
- if (ring->trace_irq_req == NULL && ring->irq_get(ring))
- i915_gem_request_assign(&ring->trace_irq_req, req);
-}
-
static inline bool __i915_request_irq_complete(struct drm_i915_gem_request *req)
{
struct intel_engine_cs *engine = req->ring;
diff --git a/drivers/gpu/drm/i915/i915_gem.c b/drivers/gpu/drm/i915/i915_gem.c
index a713e8a6cb36..5ddb2ed0f785 100644
--- a/drivers/gpu/drm/i915/i915_gem.c
+++ b/drivers/gpu/drm/i915/i915_gem.c
@@ -2889,12 +2889,6 @@ i915_gem_retire_requests_ring(struct intel_engine_cs *ring)
i915_gem_object_retire__read(obj, ring->id);
}
- if (unlikely(ring->trace_irq_req &&
- i915_gem_request_completed(ring->trace_irq_req))) {
- ring->irq_put(ring);
- i915_gem_request_assign(&ring->trace_irq_req, NULL);
- }
-
WARN_ON(i915_verify_lists(ring->dev));
}
diff --git a/drivers/gpu/drm/i915/i915_irq.c b/drivers/gpu/drm/i915/i915_irq.c
index 738edd7fbf8d..bf48fa63127a 100644
--- a/drivers/gpu/drm/i915/i915_irq.c
+++ b/drivers/gpu/drm/i915/i915_irq.c
@@ -996,12 +996,9 @@ static void ironlake_rps_change_irq_handler(struct drm_device *dev)
static void notify_ring(struct intel_engine_cs *ring)
{
- if (!intel_ring_initialized(ring))
- return;
-
- trace_i915_gem_request_notify(ring);
ring->irq_posted = true; /* paired with mb() in wake_up_process() */
- intel_engine_wakeup(ring);
+ if (intel_engine_wakeup(ring))
+ trace_i915_gem_request_notify(ring);
}
static void vlv_c0_read(struct drm_i915_private *dev_priv,
diff --git a/drivers/gpu/drm/i915/i915_trace.h b/drivers/gpu/drm/i915/i915_trace.h
index efca75bcace3..43bb2e0bb949 100644
--- a/drivers/gpu/drm/i915/i915_trace.h
+++ b/drivers/gpu/drm/i915/i915_trace.h
@@ -503,7 +503,7 @@ TRACE_EVENT(i915_gem_ring_dispatch,
__entry->ring = ring->id;
__entry->seqno = i915_gem_request_get_seqno(req);
__entry->flags = flags;
- i915_trace_irq_get(ring, req);
+ intel_engine_enable_signaling(req);
),
TP_printk("dev=%u, ring=%u, seqno=%u, flags=%x",
diff --git a/drivers/gpu/drm/i915/intel_breadcrumbs.c b/drivers/gpu/drm/i915/intel_breadcrumbs.c
index d689bd61534e..cf9cbcc2d5d7 100644
--- a/drivers/gpu/drm/i915/intel_breadcrumbs.c
+++ b/drivers/gpu/drm/i915/intel_breadcrumbs.c
@@ -22,6 +22,8 @@
*
*/
+#include <linux/kthread.h>
+
#include "i915_drv.h"
static void intel_breadcrumbs_fake_irq(unsigned long data)
@@ -320,10 +322,185 @@ void intel_engine_init_breadcrumbs(struct intel_engine_cs *engine)
(unsigned long)engine);
}
+struct signal {
+ struct rb_node node;
+ struct intel_wait wait;
+ struct drm_i915_gem_request *request;
+};
+
+static bool signal_complete(struct signal *signal)
+{
+ if (signal == NULL)
+ return false;
+
+ /* If another process served as the bottom-half it may have already
+ * signalled that this wait is already completed.
+ */
+ if (intel_wait_complete(&signal->wait))
+ return true;
+
+ /* Carefully check if the request is complete, giving time for the
+ * seqno to be visible or if the GPU hung.
+ */
+ if (__i915_request_irq_complete(signal->request))
+ return true;
+
+ return false;
+}
+
+static struct signal *to_signal(struct rb_node *rb)
+{
+ return container_of(rb, struct signal, node);
+}
+
+static void signaler_set_rtpriority(void)
+{
+ struct sched_param param = { .sched_priority = 1 };
+ sched_setscheduler_nocheck(current, SCHED_FIFO, ¶m);
+}
+
+static int intel_breadcrumbs_signaler(void *arg)
+{
+ struct intel_engine_cs *engine = arg;
+ struct intel_breadcrumbs *b = &engine->breadcrumbs;
+ struct signal *signal;
+
+ /* Install ourselves with high priority to reduce signalling latency */
+ signaler_set_rtpriority();
+
+ do {
+ set_current_state(TASK_INTERRUPTIBLE);
+
+ /* We are either woken up by the interrupt bottom-half,
+ * or by a client adding a new signaller. In both cases,
+ * the GPU seqno may have advanced beyond our oldest signal.
+ * If it has, propagate the signal, remove the waiter and
+ * check again with the next oldest signal. Otherwise we
+ * need to wait for a new interrupt from the GPU or for
+ * a new client.
+ */
+ signal = READ_ONCE(b->first_signal);
+ if (signal_complete(signal)) {
+ /* Wake up all other completed waiters and select the
+ * next bottom-half for the next user interrupt.
+ */
+ intel_engine_remove_wait(engine, &signal->wait);
+
+ i915_gem_request_unreference__unlocked(signal->request);
+
+ /* Find the next oldest signal. Note that as we have
+ * not been holding the lock, another client may
+ * have installed an even older signal than the one
+ * we just completed - so double check we are still
+ * the oldest before picking the next one.
+ */
+ spin_lock(&b->lock);
+ if (signal == b->first_signal)
+ b->first_signal = rb_next(&signal->node);
+ rb_erase(&signal->node, &b->signals);
+ spin_unlock(&b->lock);
+
+ kfree(signal);
+ } else {
+ if (kthread_should_stop())
+ break;
+
+ schedule();
+ }
+ } while (1);
+
+ return 0;
+}
+
+int intel_engine_enable_signaling(struct drm_i915_gem_request *request)
+{
+ struct intel_engine_cs *engine = request->ring;
+ struct intel_breadcrumbs *b = &engine->breadcrumbs;
+ struct rb_node *parent, **p;
+ struct task_struct *task;
+ struct signal *signal;
+ bool first;
+
+ signal = kmalloc(sizeof(*signal), GFP_ATOMIC);
+ if (unlikely(signal == NULL))
+ return -ENOMEM;
+
+ /* Spawn a thread to provide a common bottom-half for all signals.
+ * As this is an asynchronous interface we cannot steal the current
+ * task for handling the bottom-half to the user interrupt, therefore
+ * we create a thread to do the coherent seqno dance after the
+ * interrupt and then signal the waitqueue (via the dma-buf/fence).
+ */
+ task = READ_ONCE(b->signaler);
+ if (unlikely(task == NULL)) {
+ spin_lock(&b->lock);
+ task = b->signaler;
+ if (task == NULL) {
+ task = kthread_create(intel_breadcrumbs_signaler,
+ engine,
+ "irq/i915:%d",
+ engine->id);
+ if (!IS_ERR(task))
+ b->signaler = task;
+ }
+ spin_unlock(&b->lock);
+
+ if (IS_ERR(task)) {
+ kfree(signal);
+ return PTR_ERR(task);
+ }
+ }
+
+ signal->wait.task = task;
+ signal->wait.seqno = request->seqno;
+
+ signal->request = i915_gem_request_reference(request);
+
+ /* Insert ourselves into the retirement ordered list of signals
+ * on this engine. We track the oldest seqno as that will be the
+ * first signal to complete.
+ */
+ spin_lock(&b->lock);
+ parent = NULL;
+ first = true;
+ p = &b->signals.rb_node;
+ while (*p) {
+ parent = *p;
+ if (i915_seqno_passed(signal->wait.seqno,
+ to_signal(parent)->wait.seqno)) {
+ p = &parent->rb_right;
+ first = false;
+ } else
+ p = &parent->rb_left;
+ }
+ rb_link_node(&signal->node, parent, p);
+ rb_insert_color(&signal->node, &b->signals);
+ if (first)
+ smp_store_mb(b->first_signal, signal);
+ spin_unlock(&b->lock);
+
+ /* Now add ourselves into the list of waiters, but register our
+ * bottom-half as the signaller thread. As per usual, only the oldest
+ * waiter (not just signaller) is tasked as the bottom-half waking
+ * up all completed waiters after the user interrupt.
+ *
+ * If we are the oldest waiter, enable the irq (after which we
+ * must double check that the seqno did not complete).
+ */
+ if (intel_engine_add_wait(engine, &signal->wait) &&
+ intel_engine_enable_wait_irq(engine, &signal->wait))
+ wake_up_process(task);
+
+ return 0;
+}
+
void intel_engine_fini_breadcrumbs(struct intel_engine_cs *engine)
{
struct intel_breadcrumbs *b = &engine->breadcrumbs;
+ if (b->signaler)
+ kthread_stop(b->signaler);
+
del_timer_sync(&b->fake_irq);
}
diff --git a/drivers/gpu/drm/i915/intel_ringbuffer.h b/drivers/gpu/drm/i915/intel_ringbuffer.h
index 8f305ce253ae..ba81052999fa 100644
--- a/drivers/gpu/drm/i915/intel_ringbuffer.h
+++ b/drivers/gpu/drm/i915/intel_ringbuffer.h
@@ -145,6 +145,8 @@ struct i915_ctx_workarounds {
struct drm_i915_gem_object *obj;
};
+struct drm_i915_gem_request;
+
struct intel_engine_cs {
const char *name;
enum intel_ring_id {
@@ -181,7 +183,10 @@ struct intel_engine_cs {
struct intel_breadcrumbs {
spinlock_t lock; /* protects the lists of requests */
struct rb_root waiters; /* sorted by retirement, priority */
+ struct rb_root signals; /* sorted by retirement */
struct task_struct *first_waiter; /* bh for user interrupts */
+ struct task_struct *signaler; /* used for fence signalling */
+ void *first_signal;
struct timer_list fake_irq; /* used after a missed interrupt */
bool irq_enabled;
bool rpm_wakelock;
@@ -200,7 +205,6 @@ struct intel_engine_cs {
unsigned irq_refcount; /* protected by dev_priv->irq_lock */
bool irq_posted;
u32 irq_enable_mask; /* bitmask to enable ring interrupt */
- struct drm_i915_gem_request *trace_irq_req;
bool __must_check (*irq_get)(struct intel_engine_cs *ring);
void (*irq_put)(struct intel_engine_cs *ring);
@@ -558,6 +562,7 @@ bool intel_engine_enable_wait_irq(struct intel_engine_cs *engine,
const struct intel_wait *wait);
void intel_engine_remove_wait(struct intel_engine_cs *engine,
struct intel_wait *wait);
+int intel_engine_enable_signaling(struct drm_i915_gem_request *request);
static inline bool intel_engine_has_waiter(struct intel_engine_cs *engine)
{
return READ_ONCE(engine->breadcrumbs.first_waiter);
--
2.7.0.rc3
_______________________________________________
Intel-gfx mailing list
Intel-gfx@xxxxxxxxxxxxxxxxxxxxx
http://lists.freedesktop.org/mailman/listinfo/intel-gfx
