Assign contexts in parent-child relationship consecutive guc_ids. This
is accomplished by partitioning guc_id space between ones that need to
be consecutive (1/16 available guc_ids) and ones that do not (15/16 of
available guc_ids). The consecutive search is implemented via the bitmap
API.
This is a precursor to the full GuC multi-lrc implementation but aligns
to how GuC mutli-lrc interface is defined - guc_ids must be consecutive
when using the GuC multi-lrc interface.
Signed-off-by: Matthew Brost <matthew.brost@xxxxxxxxx>
---
drivers/gpu/drm/i915/gt/intel_context.h | 6 +
drivers/gpu/drm/i915/gt/intel_reset.c | 3 +-
drivers/gpu/drm/i915/gt/uc/intel_guc.h | 7 +-
.../gpu/drm/i915/gt/uc/intel_guc_submission.c | 222 ++++++++++++------
.../i915/gt/uc/intel_guc_submission_types.h | 10 +
5 files changed, 178 insertions(+), 70 deletions(-)
diff --git a/drivers/gpu/drm/i915/gt/intel_context.h b/drivers/gpu/drm/i915/gt/intel_context.h
index e78dfe9d2344..e18f16de1158 100644
--- a/drivers/gpu/drm/i915/gt/intel_context.h
+++ b/drivers/gpu/drm/i915/gt/intel_context.h
@@ -51,6 +51,12 @@ static inline bool intel_context_is_parent(struct intel_context *ce)
return !!ce->guc_number_children;
}
+static inline struct intel_context *
+intel_context_to_parent(struct intel_context *ce)
+{
+ return intel_context_is_child(ce) ? ce->parent : ce;
+}
+
void intel_context_bind_parent_child(struct intel_context *parent,
struct intel_context *child);
diff --git a/drivers/gpu/drm/i915/gt/intel_reset.c b/drivers/gpu/drm/i915/gt/intel_reset.c
index 880ca753fdd8..ebc978679b19 100644
--- a/drivers/gpu/drm/i915/gt/intel_reset.c
+++ b/drivers/gpu/drm/i915/gt/intel_reset.c
@@ -843,6 +843,7 @@ static void reset_finish(struct intel_gt *gt, intel_engine_mask_t awake)
static void nop_submit_request(struct i915_request *request)
{
+ struct intel_context *ce = intel_context_to_parent(request->context);
RQ_TRACE(request, "-EIO\n");
/*
@@ -851,7 +852,7 @@ static void nop_submit_request(struct i915_request *request)
* this for now.
*/
if (intel_engine_uses_guc(request->engine))
- intel_guc_decr_num_rq_not_ready(request->context);
+ intel_guc_decr_num_rq_not_ready(ce);
request = i915_request_mark_eio(request);
if (request) {
diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc.h b/drivers/gpu/drm/i915/gt/uc/intel_guc.h
index 466cc6227503..f79ae5796fd6 100644
--- a/drivers/gpu/drm/i915/gt/uc/intel_guc.h
+++ b/drivers/gpu/drm/i915/gt/uc/intel_guc.h
@@ -24,6 +24,7 @@ struct __guc_ads_blob;
enum {
GUC_SUBMIT_ENGINE_SINGLE_LRC,
+ GUC_SUBMIT_ENGINE_MULTI_LRC,
GUC_SUBMIT_ENGINE_MAX
};
@@ -59,8 +60,10 @@ struct intel_guc {
struct ida guc_ids;
u32 num_guc_ids;
u32 max_guc_ids;
- struct list_head guc_id_list_no_ref;
- struct list_head guc_id_list_unpinned;
+ unsigned long *guc_ids_bitmap;
+#define MAX_GUC_ID_ORDER 3
+ struct list_head guc_id_list_no_ref[MAX_GUC_ID_ORDER + 1];
+ struct list_head guc_id_list_unpinned[MAX_GUC_ID_ORDER + 1];
spinlock_t destroy_lock;
struct list_head destroyed_contexts;
diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.c b/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.c
index f60a46704ac5..d8be5a41d0ca 100644
--- a/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.c
+++ b/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.c
@@ -169,6 +169,15 @@ static void clr_guc_ids_exhausted(struct guc_submit_engine *gse)
clear_bit(GSE_STATE_GUC_IDS_EXHAUSTED, &gse->flags);
}
+/*
+ * We reserve 1/16 of the guc_ids for multi-lrc as these need to be contiguous
+ * and a different allocation algorithm is used (bitmap vs. ida). We believe the
+ * number of multi-lrc contexts in use should be low and 1/16 should be
+ * sufficient. Minimum of 32 ids for multi-lrc.
+ */
+#define NUMBER_MULTI_LRC_GUC_ID(guc) \
+ ((guc)->num_guc_ids / 16 > 32 ? (guc)->num_guc_ids / 16 : 32)
+
/*
* Below is a set of functions which control the GuC scheduling state which do
* not require a lock as all state transitions are mutually exclusive. i.e. It
@@ -402,16 +411,10 @@ static inline void clr_context_banned(struct intel_context *ce)
ce->guc_state.sched_state &= ~SCHED_STATE_BANNED;
}
-static inline struct intel_context *
-to_parent(struct intel_context *ce)
-{
- return intel_context_is_child(ce) ? ce->parent : ce;
-}
-
static inline struct intel_context *
request_to_scheduling_context(struct i915_request *rq)
{
- return to_parent(rq->context);
+ return intel_context_to_parent(rq->context);
}
static inline bool context_guc_id_invalid(struct intel_context *ce)
@@ -1426,7 +1429,7 @@ static void destroy_worker_func(struct work_struct *w);
*/
int intel_guc_submission_init(struct intel_guc *guc)
{
- int ret;
+ int ret, i;
if (guc_submission_initialized(guc))
return 0;
@@ -1438,9 +1441,13 @@ int intel_guc_submission_init(struct intel_guc *guc)
xa_init_flags(&guc->context_lookup, XA_FLAGS_LOCK_IRQ);
spin_lock_init(&guc->contexts_lock);
- INIT_LIST_HEAD(&guc->guc_id_list_no_ref);
- INIT_LIST_HEAD(&guc->guc_id_list_unpinned);
+ for (i = 0; i < MAX_GUC_ID_ORDER + 1; ++i) {
+ INIT_LIST_HEAD(&guc->guc_id_list_no_ref[i]);
+ INIT_LIST_HEAD(&guc->guc_id_list_unpinned[i]);
+ }
ida_init(&guc->guc_ids);
+ guc->guc_ids_bitmap =
+ bitmap_zalloc(NUMBER_MULTI_LRC_GUC_ID(guc), GFP_KERNEL);
spin_lock_init(&guc->destroy_lock);
@@ -1466,6 +1473,8 @@ void intel_guc_submission_fini(struct intel_guc *guc)
i915_sched_engine_put(sched_engine);
}
+
+ bitmap_free(guc->guc_ids_bitmap);
}
static inline void queue_request(struct i915_sched_engine *sched_engine,
@@ -1489,11 +1498,13 @@ static inline void queue_request(struct i915_sched_engine *sched_engine,
static bool too_many_guc_ids_not_ready(struct guc_submit_engine *gse,
struct intel_context *ce)
{
- u32 available_guc_ids, guc_ids_consumed;
struct intel_guc *guc = gse->sched_engine.private_data;
+ u32 available_guc_ids = intel_context_is_parent(ce) ?
+ NUMBER_MULTI_LRC_GUC_ID(guc) :
+ guc->num_guc_ids - NUMBER_MULTI_LRC_GUC_ID(guc);
+ u32 guc_ids_consumed = atomic_read(&gse->num_guc_ids_not_ready);
- available_guc_ids = guc->num_guc_ids;
- guc_ids_consumed = atomic_read(&gse->num_guc_ids_not_ready);
+ GEM_BUG_ON(intel_context_is_child(ce));
if (TOO_MANY_GUC_IDS_NOT_READY(available_guc_ids, guc_ids_consumed)) {
set_and_update_guc_ids_exhausted(gse);
@@ -1507,17 +1518,26 @@ static void incr_num_rq_not_ready(struct intel_context *ce)
{
struct guc_submit_engine *gse = ce_to_gse(ce);
+ GEM_BUG_ON(intel_context_is_child(ce));
+ GEM_BUG_ON(!intel_context_is_parent(ce) &&
+ ce->guc_number_children);
+
if (!atomic_fetch_add(1, &ce->guc_num_rq_not_ready))
- atomic_inc(&gse->num_guc_ids_not_ready);
+ atomic_add(ce->guc_number_children + 1,
+ &gse->num_guc_ids_not_ready);
}
void intel_guc_decr_num_rq_not_ready(struct intel_context *ce)
{
struct guc_submit_engine *gse = ce_to_gse(ce);
+ GEM_BUG_ON(intel_context_is_child(ce));
+
if (atomic_fetch_add(-1, &ce->guc_num_rq_not_ready) == 1) {
GEM_BUG_ON(!atomic_read(&gse->num_guc_ids_not_ready));
- atomic_dec(&gse->num_guc_ids_not_ready);
+
+ atomic_sub(ce->guc_number_children + 1,
+ &gse->num_guc_ids_not_ready);
}
}
@@ -1569,20 +1589,42 @@ static void guc_submit_request(struct i915_request *rq)
spin_unlock_irqrestore(&sched_engine->lock, flags);
- intel_guc_decr_num_rq_not_ready(rq->context);
+ intel_guc_decr_num_rq_not_ready(request_to_scheduling_context(rq));
}
-static int new_guc_id(struct intel_guc *guc)
+static int new_guc_id(struct intel_guc *guc, struct intel_context *ce)
{
- return ida_simple_get(&guc->guc_ids, 0,
- guc->num_guc_ids, GFP_KERNEL |
- __GFP_RETRY_MAYFAIL | __GFP_NOWARN);
+ int ret;
+
+ GEM_BUG_ON(intel_context_is_child(ce));
+
+ if (intel_context_is_parent(ce))
+ ret = bitmap_find_free_region(guc->guc_ids_bitmap,
+ NUMBER_MULTI_LRC_GUC_ID(guc),
+ order_base_2(ce->guc_number_children
+ + 1));
+ else
+ ret = ida_simple_get(&guc->guc_ids,
+ NUMBER_MULTI_LRC_GUC_ID(guc),
+ guc->num_guc_ids, GFP_KERNEL |
+ __GFP_RETRY_MAYFAIL | __GFP_NOWARN);
+ if (unlikely(ret < 0))
+ return ret;
+
+ ce->guc_id = ret;
+ return 0;
}
static void __release_guc_id(struct intel_guc *guc, struct intel_context *ce)
{
+ GEM_BUG_ON(intel_context_is_child(ce));
if (!context_guc_id_invalid(ce)) {
- ida_simple_remove(&guc->guc_ids, ce->guc_id);
+ if (intel_context_is_parent(ce))
+ bitmap_release_region(guc->guc_ids_bitmap, ce->guc_id,
+ order_base_2(ce->guc_number_children
+ + 1));
+ else
+ ida_simple_remove(&guc->guc_ids, ce->guc_id);
clr_lrc_desc_registered(guc, ce->guc_id);
set_context_guc_id_invalid(ce);
}
@@ -1594,6 +1636,8 @@ static void release_guc_id(struct intel_guc *guc, struct intel_context *ce)
{
unsigned long flags;
+ GEM_BUG_ON(intel_context_is_child(ce));
+
spin_lock_irqsave(&guc->contexts_lock, flags);
__release_guc_id(guc, ce);
spin_unlock_irqrestore(&guc->contexts_lock, flags);
@@ -1608,54 +1652,91 @@ static void release_guc_id(struct intel_guc *guc, struct intel_context *ce)
* schedule disable H2G + a deregister H2G.
*/
static struct list_head *get_guc_id_list(struct intel_guc *guc,
+ u8 number_children,
bool unpinned)
{
+ GEM_BUG_ON(order_base_2(number_children + 1) > MAX_GUC_ID_ORDER);
+
if (unpinned)
- return &guc->guc_id_list_unpinned;
+ return &guc->guc_id_list_unpinned[order_base_2(number_children + 1)];
else
- return &guc->guc_id_list_no_ref;
+ return &guc->guc_id_list_no_ref[order_base_2(number_children + 1)];
}
-static int steal_guc_id(struct intel_guc *guc, bool unpinned)
+static int steal_guc_id(struct intel_guc *guc, struct intel_context *ce,
+ bool unpinned)
{
- struct intel_context *ce;
- int guc_id;
- struct list_head *guc_id_list = get_guc_id_list(guc, unpinned);
+ struct intel_context *cn;
+ u8 number_children = ce->guc_number_children;
lockdep_assert_held(&guc->contexts_lock);
+ GEM_BUG_ON(intel_context_is_child(ce));
- if (!list_empty(guc_id_list)) {
- ce = list_first_entry(guc_id_list,
- struct intel_context,
- guc_id_link);
+ do {
+ struct list_head *guc_id_list =
+ get_guc_id_list(guc, number_children, unpinned);
- /* Ensure context getting stolen in expected state */
- GEM_BUG_ON(atomic_read(&ce->guc_id_ref));
- GEM_BUG_ON(context_guc_id_invalid(ce));
- GEM_BUG_ON(context_guc_id_stolen(ce));
+ if (!list_empty(guc_id_list)) {
+ u8 cn_o2, ce_o2 = order_base_2(ce->guc_number_children);
- list_del_init(&ce->guc_id_link);
- guc_id = ce->guc_id;
- clr_context_registered(ce);
+ cn = list_first_entry(guc_id_list,
+ struct intel_context,
+ guc_id_link);
+ cn_o2 = order_base_2(cn->guc_number_children);
- /*
- * If stealing from the pinned list, defer invalidating
- * the guc_id until the retire workqueue processes this
- * context.
- */
- if (!unpinned) {
- GEM_BUG_ON(ce_to_gse(ce)->stalled_context);
+ /*
+ * Corner case where a multi-lrc context steals a guc_id
+ * from another context that has more guc_id that itself.
+ */
+ if (cn_o2 != ce_o2) {
+ bitmap_release_region(guc->guc_ids_bitmap,
+ cn->guc_id,
+ cn_o2);
+ bitmap_allocate_region(guc->guc_ids_bitmap,
+ ce->guc_id,
+ ce_o2);
+ }
- ce_to_gse(ce)->stalled_context = intel_context_get(ce);
- set_context_guc_id_stolen(ce);
- } else {
- set_context_guc_id_invalid(ce);
+ /* Ensure context getting stolen in expected state */
+ GEM_BUG_ON(atomic_read(&cn->guc_id_ref));
+ GEM_BUG_ON(context_guc_id_invalid(cn));
+ GEM_BUG_ON(context_guc_id_stolen(cn));
+ GEM_BUG_ON(ce_to_gse(ce) != ce_to_gse(cn));
+
+ list_del_init(&cn->guc_id_link);
+ ce->guc_id = cn->guc_id;
+
+ /*
+ * If stealing from the pinned list, defer invalidating
+ * the guc_id until the retire workqueue processes this
+ * context.
+ */
+ clr_context_registered(cn);
+ if (!unpinned) {
+ GEM_BUG_ON(ce_to_gse(cn)->stalled_context);
+ ce_to_gse(cn)->stalled_context =
+ intel_context_get(cn);
+ set_context_guc_id_stolen(cn);
+ } else {
+ set_context_guc_id_invalid(cn);
+ }
+
+ return 0;
}
- return guc_id;
- } else {
- return -EAGAIN;
- }
+ /*
+ * When using multi-lrc we search the guc_id_lists with the
+ * least amount of guc_ids required first but will consume a
+ * block larger of guc_ids if necessary. +2 to is sufficient to
+ * move the search from 2 wide to 4 wide and 4 wide to 8 wide.
+ */
+ if (!number_children || number_children > 3)
+ break;
+ else
+ number_children += 2;
+ } while (true);
+
+ return -EAGAIN;
}
enum { /* Return values for pin_guc_id / assign_guc_id */
@@ -1664,17 +1745,18 @@ enum { /* Return values for pin_guc_id / assign_guc_id */
NEW_GUC_ID_ENABLED =2,
};
-static int assign_guc_id(struct intel_guc *guc, u16 *out, bool tasklet)
+static int assign_guc_id(struct intel_guc *guc, struct intel_context *ce,
+ bool tasklet)
{
int ret;
lockdep_assert_held(&guc->contexts_lock);
+ GEM_BUG_ON(intel_context_is_child(ce));
- ret = new_guc_id(guc);
+ ret = new_guc_id(guc, ce);
if (unlikely(ret < 0)) {
- ret = steal_guc_id(guc, true);
- if (ret >= 0) {
- *out = ret;
+ ret = steal_guc_id(guc, ce, true);
+ if (!ret) {
ret = NEW_GUC_ID_DISABLED;
} else if (ret < 0 && tasklet) {
/*
@@ -1682,15 +1764,18 @@ static int assign_guc_id(struct intel_guc *guc, u16 *out, bool tasklet)
* enabled if guc_ids are exhausted and we are submitting
* from the tasklet.
*/
- ret = steal_guc_id(guc, false);
- if (ret >= 0) {
- *out = ret;
+ ret = steal_guc_id(guc, ce, false);
+ if (!ret)
ret = NEW_GUC_ID_ENABLED;
- }
}
- } else {
- *out = ret;
- ret = SAME_GUC_ID;
+ }
+
+ if (!(ret < 0) && intel_context_is_parent(ce)) {
+ struct intel_context *child;
+ int i = 1;
+
+ for_each_child(ce, child)
+ child->guc_id = ce->guc_id + i++;
}
return ret;
@@ -1703,6 +1788,7 @@ static int pin_guc_id(struct intel_guc *guc, struct intel_context *ce,
int ret = 0;
unsigned long flags, tries = PIN_GUC_ID_TRIES;
+ GEM_BUG_ON(intel_context_is_child(ce));
GEM_BUG_ON(atomic_read(&ce->guc_id_ref));
try_again:
@@ -1714,7 +1800,7 @@ static int pin_guc_id(struct intel_guc *guc, struct intel_context *ce,
}
if (context_guc_id_invalid(ce)) {
- ret = assign_guc_id(guc, &ce->guc_id, tasklet);
+ ret = assign_guc_id(guc, ce, tasklet);
if (unlikely(ret < 0))
goto out_unlock;
}
@@ -1760,6 +1846,7 @@ static void unpin_guc_id(struct intel_guc *guc,
unsigned long flags;
GEM_BUG_ON(atomic_read(&ce->guc_id_ref) < 0);
+ GEM_BUG_ON(intel_context_is_child(ce));
spin_lock_irqsave(&guc->contexts_lock, flags);
@@ -1768,7 +1855,8 @@ static void unpin_guc_id(struct intel_guc *guc,
if (!context_guc_id_invalid(ce) && !context_guc_id_stolen(ce) &&
!atomic_read(&ce->guc_id_ref)) {
- struct list_head *head = get_guc_id_list(guc, unpinned);
+ struct list_head *head =
+ get_guc_id_list(guc, ce->guc_number_children, unpinned);
list_add_tail(&ce->guc_id_link, head);
}
diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc_submission_types.h b/drivers/gpu/drm/i915/gt/uc/intel_guc_submission_types.h
index 7069b7248f55..a5933e07bdd2 100644
--- a/drivers/gpu/drm/i915/gt/uc/intel_guc_submission_types.h
+++ b/drivers/gpu/drm/i915/gt/uc/intel_guc_submission_types.h
@@ -22,6 +22,16 @@ struct guc_virtual_engine {
/*
* Object which encapsulates the globally operated on i915_sched_engine +
* the GuC submission state machine described in intel_guc_submission.c.
+ *
+ * Currently we have two instances of these per GuC. One for single-lrc and one
+ * for multi-lrc submission. We split these into two submission engines as they
+ * can operate in parallel allowing a blocking condition on one not to affect
+ * the other. i.e. guc_ids are statically allocated between these two submission
+ * modes. One mode may have guc_ids exhausted which requires blocking while the
+ * other has plenty of guc_ids and can make forward progres.
+ *
+ * In the future if different submission use cases arise we can simply
+ * instantiate another of these objects and assign it to the context.
*/
struct guc_submit_engine {
struct i915_sched_engine sched_engine;
--
2.28.0
