This adds GuC backend support for i915_request_cancel(), which in turn
makes CONFIG_DRM_I915_REQUEST_TIMEOUT work.
Signed-off-by: Matthew Brost <matthew.brost@xxxxxxxxx>
Cc: Tvrtko Ursulin <tvrtko.ursulin@xxxxxxxxx>
---
drivers/gpu/drm/i915/gt/intel_context.c | 9 +
drivers/gpu/drm/i915/gt/intel_context.h | 7 +
drivers/gpu/drm/i915/gt/intel_context_types.h | 7 +
.../drm/i915/gt/intel_execlists_submission.c | 18 ++
drivers/gpu/drm/i915/gt/intel_gt_requests.c | 1 +
.../gpu/drm/i915/gt/uc/intel_guc_submission.c | 168 ++++++++++++++++++
drivers/gpu/drm/i915/i915_request.c | 14 +-
7 files changed, 211 insertions(+), 13 deletions(-)
diff --git a/drivers/gpu/drm/i915/gt/intel_context.c b/drivers/gpu/drm/i915/gt/intel_context.c
index 3fe7794b2bfd..b633fea684d4 100644
--- a/drivers/gpu/drm/i915/gt/intel_context.c
+++ b/drivers/gpu/drm/i915/gt/intel_context.c
@@ -366,6 +366,12 @@ static int __intel_context_active(struct i915_active *active)
return 0;
}
+static int sw_fence_dummy_notify(struct i915_sw_fence *sf,
+ enum i915_sw_fence_notify state)
+{
+ return NOTIFY_DONE;
+}
+
void
intel_context_init(struct intel_context *ce, struct intel_engine_cs *engine)
{
@@ -398,6 +404,9 @@ intel_context_init(struct intel_context *ce, struct intel_engine_cs *engine)
ce->guc_id = GUC_INVALID_LRC_ID;
INIT_LIST_HEAD(&ce->guc_id_link);
+ i915_sw_fence_init(&ce->guc_blocked, sw_fence_dummy_notify);
+ i915_sw_fence_commit(&ce->guc_blocked);
+
i915_active_init(&ce->active,
__intel_context_active, __intel_context_retire, 0);
}
diff --git a/drivers/gpu/drm/i915/gt/intel_context.h b/drivers/gpu/drm/i915/gt/intel_context.h
index 11fa7700dc9e..1b208daee72b 100644
--- a/drivers/gpu/drm/i915/gt/intel_context.h
+++ b/drivers/gpu/drm/i915/gt/intel_context.h
@@ -71,6 +71,13 @@ intel_context_is_pinned(struct intel_context *ce)
return atomic_read(&ce->pin_count);
}
+static inline void intel_context_cancel_request(struct intel_context *ce,
+ struct i915_request *rq)
+{
+ GEM_BUG_ON(!ce->ops->cancel_request);
+ return ce->ops->cancel_request(ce, rq);
+}
+
/**
* intel_context_unlock_pinned - Releases the earlier locking of 'pinned' status
* @ce - the context
diff --git a/drivers/gpu/drm/i915/gt/intel_context_types.h b/drivers/gpu/drm/i915/gt/intel_context_types.h
index 217761b27b6c..cd2ea5b98fc3 100644
--- a/drivers/gpu/drm/i915/gt/intel_context_types.h
+++ b/drivers/gpu/drm/i915/gt/intel_context_types.h
@@ -13,6 +13,7 @@
#include <linux/types.h>
#include "i915_active_types.h"
+#include "i915_sw_fence.h"
#include "i915_utils.h"
#include "intel_engine_types.h"
#include "intel_sseu.h"
@@ -43,6 +44,9 @@ struct intel_context_ops {
void (*unpin)(struct intel_context *ce);
void (*post_unpin)(struct intel_context *ce);
+ void (*cancel_request)(struct intel_context *ce,
+ struct i915_request *rq);
+
void (*enter)(struct intel_context *ce);
void (*exit)(struct intel_context *ce);
@@ -200,6 +204,9 @@ struct intel_context {
*/
u8 guc_prio;
u32 guc_prio_count[GUC_CLIENT_PRIORITY_NUM];
+
+ /* GuC context blocked fence */
+ struct i915_sw_fence guc_blocked;
};
#endif /* __INTEL_CONTEXT_TYPES__ */
diff --git a/drivers/gpu/drm/i915/gt/intel_execlists_submission.c b/drivers/gpu/drm/i915/gt/intel_execlists_submission.c
index 54518b64bdbd..16606cdfc2f5 100644
--- a/drivers/gpu/drm/i915/gt/intel_execlists_submission.c
+++ b/drivers/gpu/drm/i915/gt/intel_execlists_submission.c
@@ -114,6 +114,7 @@
#include "gen8_engine_cs.h"
#include "intel_breadcrumbs.h"
#include "intel_context.h"
+#include "intel_engine_heartbeat.h"
#include "intel_engine_pm.h"
#include "intel_engine_stats.h"
#include "intel_execlists_submission.h"
@@ -2545,11 +2546,26 @@ static int execlists_context_alloc(struct intel_context *ce)
return lrc_alloc(ce, ce->engine);
}
+static void execlists_context_cancel_request(struct intel_context *ce,
+ struct i915_request *rq)
+{
+ struct intel_engine_cs *engine = NULL;
+
+ i915_request_active_engine(rq, &engine);
+
+ if (engine && intel_engine_pulse(engine))
+ intel_gt_handle_error(engine->gt, engine->mask, 0,
+ "request cancellation by %s",
+ current->comm);
+}
+
static const struct intel_context_ops execlists_context_ops = {
.flags = COPS_HAS_INFLIGHT,
.alloc = execlists_context_alloc,
+ .cancel_request = execlists_context_cancel_request,
+
.pre_pin = execlists_context_pre_pin,
.pin = execlists_context_pin,
.unpin = lrc_unpin,
@@ -3649,6 +3665,8 @@ static const struct intel_context_ops virtual_context_ops = {
.alloc = virtual_context_alloc,
+ .cancel_request = execlists_context_cancel_request,
+
.pre_pin = virtual_context_pre_pin,
.pin = virtual_context_pin,
.unpin = lrc_unpin,
diff --git a/drivers/gpu/drm/i915/gt/intel_gt_requests.c b/drivers/gpu/drm/i915/gt/intel_gt_requests.c
index c6c702f236fa..3d3043d4bf98 100644
--- a/drivers/gpu/drm/i915/gt/intel_gt_requests.c
+++ b/drivers/gpu/drm/i915/gt/intel_gt_requests.c
@@ -13,6 +13,7 @@
#include "intel_gt_pm.h"
#include "intel_gt_requests.h"
#include "intel_timeline.h"
+#include "intel_context.h"
#include "uc/intel_uc.h"
static bool retire_requests(struct intel_timeline *tl)
diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.c b/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.c
index 6d2ae6390299..b3157eeb2599 100644
--- a/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.c
+++ b/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.c
@@ -181,6 +181,11 @@ static void clr_guc_ids_exhausted(struct guc_submit_engine *gse)
#define SCHED_STATE_NO_LOCK_GUC_ID_STOLEN BIT(3)
#define SCHED_STATE_NO_LOCK_NEEDS_REGISTER BIT(4)
#define SCHED_STATE_NO_LOCK_REGISTERED BIT(5)
+#define SCHED_STATE_NO_LOCK_BLOCKED_SHIFT 6
+#define SCHED_STATE_NO_LOCK_BLOCKED \
+ BIT(SCHED_STATE_NO_LOCK_BLOCKED_SHIFT)
+#define SCHED_STATE_NO_LOCK_BLOCKED_MASK \
+ (0xffff << SCHED_STATE_NO_LOCK_BLOCKED_SHIFT)
static inline bool context_enabled(struct intel_context *ce)
{
return (atomic_read(&ce->guc_sched_state_no_lock) &
@@ -288,6 +293,27 @@ static inline void clr_context_registered(struct intel_context *ce)
&ce->guc_sched_state_no_lock);
}
+static inline u32 context_blocked(struct intel_context *ce)
+{
+ return (atomic_read(&ce->guc_sched_state_no_lock) &
+ SCHED_STATE_NO_LOCK_BLOCKED_MASK) >>
+ SCHED_STATE_NO_LOCK_BLOCKED_SHIFT;
+}
+
+static inline void incr_context_blocked(struct intel_context *ce)
+{
+ lockdep_assert_held(&ce_to_gse(ce)->sched_engine.lock);
+ atomic_add(SCHED_STATE_NO_LOCK_BLOCKED,
+ &ce->guc_sched_state_no_lock);
+}
+
+static inline void decr_context_blocked(struct intel_context *ce)
+{
+ lockdep_assert_held(&ce_to_gse(ce)->sched_engine.lock);
+ atomic_sub(SCHED_STATE_NO_LOCK_BLOCKED,
+ &ce->guc_sched_state_no_lock);
+}
+
/*
* Below is a set of functions which control the GuC scheduling state which
* require a lock, aside from the special case where the functions are called
@@ -593,6 +619,9 @@ static int __guc_add_request(struct intel_guc *guc, struct i915_request *rq)
goto out;
}
+ if (unlikely(context_blocked(ce)))
+ goto out;
+
enabled = context_enabled(ce);
if (!enabled) {
@@ -853,6 +882,7 @@ static void __guc_context_destroy(struct intel_context *ce);
static void release_guc_id(struct intel_guc *guc, struct intel_context *ce);
static void guc_signal_context_fence(struct intel_context *ce);
static void guc_cancel_context_requests(struct intel_context *ce);
+static void guc_blocked_fence_complete(struct intel_context *ce);
static void scrub_guc_desc_for_outstanding_g2h(struct intel_guc *guc)
{
@@ -900,6 +930,10 @@ static void scrub_guc_desc_for_outstanding_g2h(struct intel_guc *guc)
}
intel_context_sched_disable_unpin(ce);
atomic_dec(&guc->outstanding_submission_g2h);
+ spin_lock_irqsave(&ce->guc_state.lock, flags);
+ guc_blocked_fence_complete(ce);
+ spin_unlock_irqrestore(&ce->guc_state.lock, flags);
+
intel_context_put(ce);
}
}
@@ -1917,6 +1951,21 @@ static void guc_context_post_unpin(struct intel_context *ce)
lrc_post_unpin(ce);
}
+static void __guc_context_sched_enable(struct intel_guc *guc,
+ struct intel_context *ce)
+{
+ u32 action[] = {
+ INTEL_GUC_ACTION_SCHED_CONTEXT_MODE_SET,
+ ce->guc_id,
+ GUC_CONTEXT_ENABLE
+ };
+
+ trace_intel_context_sched_enable(ce);
+
+ guc_submission_busy_loop(guc, action, ARRAY_SIZE(action),
+ G2H_LEN_DW_SCHED_CONTEXT_MODE_SET, true);
+}
+
static void __guc_context_sched_disable(struct intel_guc *guc,
struct intel_context *ce,
u16 guc_id)
@@ -1935,15 +1984,129 @@ static void __guc_context_sched_disable(struct intel_guc *guc,
G2H_LEN_DW_SCHED_CONTEXT_MODE_SET, true);
}
+static void guc_blocked_fence_complete(struct intel_context *ce)
+{
+ lockdep_assert_held(&ce->guc_state.lock);
+
+ if (!i915_sw_fence_done(&ce->guc_blocked))
+ i915_sw_fence_complete(&ce->guc_blocked);
+}
+
+static void guc_blocked_fence_reinit(struct intel_context *ce)
+{
+ lockdep_assert_held(&ce->guc_state.lock);
+ GEM_BUG_ON(!i915_sw_fence_done(&ce->guc_blocked));
+ i915_sw_fence_fini(&ce->guc_blocked);
+ i915_sw_fence_reinit(&ce->guc_blocked);
+ i915_sw_fence_await(&ce->guc_blocked);
+ i915_sw_fence_commit(&ce->guc_blocked);
+}
+
static u16 prep_context_pending_disable(struct intel_context *ce)
{
set_context_pending_disable(ce);
clr_context_enabled(ce);
+ guc_blocked_fence_reinit(ce);
intel_context_get(ce);
return ce->guc_id;
}
+static struct i915_sw_fence *guc_context_block(struct intel_context *ce)
+{
+ struct intel_guc *guc = ce_to_guc(ce);
+ struct i915_sched_engine *sched_engine = ce_to_sched_engine(ce);
+ unsigned long flags;
+ struct intel_runtime_pm *runtime_pm = &ce->engine->gt->i915->runtime_pm;
+ intel_wakeref_t wakeref;
+ u16 guc_id;
+ bool enabled;
+
+ spin_lock_irqsave(&sched_engine->lock, flags);
+ incr_context_blocked(ce);
+ spin_unlock_irqrestore(&sched_engine->lock, flags);
+
+ spin_lock_irqsave(&ce->guc_state.lock, flags);
+ enabled = context_enabled(ce);
+ if (unlikely(!enabled || submission_disabled(guc))) {
+ if (!enabled)
+ clr_context_enabled(ce);
+ spin_unlock_irqrestore(&ce->guc_state.lock, flags);
+ return &ce->guc_blocked;
+ }
+
+ /*
+ * We add +2 here as the schedule disable complete CTB handler calls
+ * intel_context_sched_disable_unpin (-2 to pin_count).
+ */
+ atomic_add(2, &ce->pin_count);
+
+ guc_id = prep_context_pending_disable(ce);
+ spin_unlock_irqrestore(&ce->guc_state.lock, flags);
+
+ with_intel_runtime_pm(runtime_pm, wakeref)
+ __guc_context_sched_disable(guc, ce, guc_id);
+
+ return &ce->guc_blocked;
+}
+
+static void guc_context_unblock(struct intel_context *ce)
+{
+ struct intel_guc *guc = ce_to_guc(ce);
+ struct i915_sched_engine *sched_engine = ce_to_sched_engine(ce);
+ unsigned long flags;
+ struct intel_runtime_pm *runtime_pm = &ce->engine->gt->i915->runtime_pm;
+ intel_wakeref_t wakeref;
+
+ GEM_BUG_ON(context_enabled(ce));
+
+ if (unlikely(context_blocked(ce) > 1)) {
+ spin_lock_irqsave(&sched_engine->lock, flags);
+ if (likely(context_blocked(ce) > 1))
+ goto decrement;
+ spin_unlock_irqrestore(&sched_engine->lock, flags);
+ }
+
+ spin_lock_irqsave(&ce->guc_state.lock, flags);
+ if (unlikely(submission_disabled(guc) ||
+ !intel_context_is_pinned(ce) ||
+ context_pending_disable(ce) ||
+ context_blocked(ce) > 1)) {
+ spin_unlock_irqrestore(&ce->guc_state.lock, flags);
+ goto out;
+ }
+
+ set_context_pending_enable(ce);
+ set_context_enabled(ce);
+ intel_context_get(ce);
+ spin_unlock_irqrestore(&ce->guc_state.lock, flags);
+
+ with_intel_runtime_pm(runtime_pm, wakeref)
+ __guc_context_sched_enable(guc, ce);
+
+out:
+ spin_lock_irqsave(&sched_engine->lock, flags);
+decrement:
+ decr_context_blocked(ce);
+ spin_unlock_irqrestore(&sched_engine->lock, flags);
+}
+
+static void guc_context_cancel_request(struct intel_context *ce,
+ struct i915_request *rq)
+{
+ if (i915_sw_fence_signaled(&rq->submit)) {
+ struct i915_sw_fence *fence = guc_context_block(ce);
+
+ i915_sw_fence_wait(fence);
+ if (!i915_request_completed(rq)) {
+ __i915_request_skip(rq);
+ guc_reset_state(ce, intel_ring_wrap(ce->ring, rq->head),
+ true);
+ }
+ guc_context_unblock(ce);
+ }
+}
+
static void guc_context_ban(struct intel_context *ce, struct i915_request *rq)
{
struct intel_guc *guc = ce_to_guc(ce);
@@ -2294,6 +2457,8 @@ static const struct intel_context_ops guc_context_ops = {
.ban = guc_context_ban,
+ .cancel_request = guc_context_cancel_request,
+
.enter = intel_context_enter_engine,
.exit = intel_context_exit_engine,
@@ -2661,6 +2826,8 @@ static const struct intel_context_ops virtual_guc_context_ops = {
.ban = guc_context_ban,
+ .cancel_request = guc_context_cancel_request,
+
.enter = guc_virtual_context_enter,
.exit = guc_virtual_context_exit,
@@ -3212,6 +3379,7 @@ int intel_guc_sched_done_process_msg(struct intel_guc *guc,
clr_context_banned(ce);
clr_context_pending_disable(ce);
__guc_signal_context_fence(ce);
+ guc_blocked_fence_complete(ce);
spin_unlock_irqrestore(&ce->guc_state.lock, flags);
if (context_block_tasklet(ce)) {
diff --git a/drivers/gpu/drm/i915/i915_request.c b/drivers/gpu/drm/i915/i915_request.c
index 4bf10f0ac34d..71965fb4f3ab 100644
--- a/drivers/gpu/drm/i915/i915_request.c
+++ b/drivers/gpu/drm/i915/i915_request.c
@@ -734,18 +734,6 @@ void i915_request_unsubmit(struct i915_request *request)
spin_unlock_irqrestore(&engine->sched_engine->lock, flags);
}
-static void __cancel_request(struct i915_request *rq)
-{
- struct intel_engine_cs *engine = NULL;
-
- i915_request_active_engine(rq, &engine);
-
- if (engine && intel_engine_pulse(engine))
- intel_gt_handle_error(engine->gt, engine->mask, 0,
- "request cancellation by %s",
- current->comm);
-}
-
void i915_request_cancel(struct i915_request *rq, int error)
{
if (!i915_request_set_error_once(rq, error))
@@ -753,7 +741,7 @@ void i915_request_cancel(struct i915_request *rq, int error)
set_bit(I915_FENCE_FLAG_SENTINEL, &rq->fence.flags);
- __cancel_request(rq);
+ intel_context_cancel_request(rq->context, rq);
}
static int __i915_sw_fence_call
--
2.28.0
