Quoting Chris Wilson (2019-08-06 10:05:16) > As we need to acquire a mutex to serialise the final > intel_wakeref_put, we need to ensure that we are in process context at > that time. However, we want to allow operation on the intel_wakeref from > inside timer and other hardirq context, which means that need to defer > that final put to a workqueue. > > Inside the final wakeref puts, we are safe to operate in any context, as > we are simply marking up the HW and state tracking for the potential > sleep. It's only the serialisation with the potential sleeping getting > that requires careful wait avoidance. This allows us to retain the > immediate processing as before (we only need to sleep over the same > races as the current mutex_lock). > > v2: Add a selftest to ensure we exercise the code while lockdep watches. > v3: That test was extremely loud and complained about many things! > > Bugzilla: https://bugs.freedesktop.org/show_bug.cgi?id=111295 > References: https://bugs.freedesktop.org/show_bug.cgi?id=111245 Probably also merits References: https://bugs.freedesktop.org/show_bug.cgi?id=111256 > Fixes: 18398904ca9e ("drm/i915: Only recover active engines") > Fixes: 51fbd8de87dc ("drm/i915/pmu: Atomically acquire the gt_pm wakeref") > Signed-off-by: Chris Wilson <chris@xxxxxxxxxxxxxxxxxx> > Cc: Tvrtko Ursulin <tvrtko.ursulin@xxxxxxxxx> > Cc: Mika Kuoppala <mika.kuoppala@xxxxxxxxxxxxxxx> Mika, can I twist your arm some more into looking at this sooner rather than later :) > --- > drivers/gpu/drm/i915/gem/i915_gem_pm.c | 13 +-- > drivers/gpu/drm/i915/gt/intel_engine_cs.c | 1 + > drivers/gpu/drm/i915/gt/intel_engine_pm.c | 38 +++------ > drivers/gpu/drm/i915/gt/intel_engine_pm.h | 18 ++-- > drivers/gpu/drm/i915/gt/intel_gt_pm.c | 28 +++---- > drivers/gpu/drm/i915/gt/intel_gt_pm.h | 22 ++++- > drivers/gpu/drm/i915/gt/intel_lrc.c | 4 +- > drivers/gpu/drm/i915/gt/selftest_engine.c | 28 +++++++ > drivers/gpu/drm/i915/gt/selftest_engine.h | 14 ++++ > drivers/gpu/drm/i915/gt/selftest_engine_pm.c | 83 +++++++++++++++++++ > .../gpu/drm/i915/gt/uc/intel_guc_submission.c | 4 +- > drivers/gpu/drm/i915/i915_debugfs.c | 4 + > drivers/gpu/drm/i915/i915_gem.c | 26 +----- > drivers/gpu/drm/i915/intel_wakeref.c | 82 +++++++++++++----- > drivers/gpu/drm/i915/intel_wakeref.h | 62 +++++++++----- > .../drm/i915/selftests/i915_live_selftests.h | 5 +- > 16 files changed, 301 insertions(+), 131 deletions(-) > create mode 100644 drivers/gpu/drm/i915/gt/selftest_engine.c > create mode 100644 drivers/gpu/drm/i915/gt/selftest_engine.h > create mode 100644 drivers/gpu/drm/i915/gt/selftest_engine_pm.c > > diff --git a/drivers/gpu/drm/i915/gem/i915_gem_pm.c b/drivers/gpu/drm/i915/gem/i915_gem_pm.c > index 72922703af49..17e3618241c5 100644 > --- a/drivers/gpu/drm/i915/gem/i915_gem_pm.c > +++ b/drivers/gpu/drm/i915/gem/i915_gem_pm.c > @@ -130,7 +130,9 @@ static bool switch_to_kernel_context_sync(struct intel_gt *gt) > } > } while (i915_retire_requests(gt->i915) && result); > > - GEM_BUG_ON(gt->awake); > + if (intel_gt_pm_wait_for_idle(gt)) > + result = false; > + > return result; > } > > @@ -161,13 +163,6 @@ void i915_gem_suspend(struct drm_i915_private *i915) > > mutex_unlock(&i915->drm.struct_mutex); > > - /* > - * Assert that we successfully flushed all the work and > - * reset the GPU back to its idle, low power state. > - */ > - GEM_BUG_ON(i915->gt.awake); > - flush_work(&i915->gem.idle_work); > - > cancel_delayed_work_sync(&i915->gt.hangcheck.work); > > i915_gem_drain_freed_objects(i915); > @@ -244,8 +239,6 @@ void i915_gem_resume(struct drm_i915_private *i915) > { > GEM_TRACE("\n"); > > - WARN_ON(i915->gt.awake); > - > mutex_lock(&i915->drm.struct_mutex); > intel_uncore_forcewake_get(&i915->uncore, FORCEWAKE_ALL); > > diff --git a/drivers/gpu/drm/i915/gt/intel_engine_cs.c b/drivers/gpu/drm/i915/gt/intel_engine_cs.c > index d0befd6c023a..5748079ebce2 100644 > --- a/drivers/gpu/drm/i915/gt/intel_engine_cs.c > +++ b/drivers/gpu/drm/i915/gt/intel_engine_cs.c > @@ -1593,5 +1593,6 @@ intel_engine_find_active_request(struct intel_engine_cs *engine) > } > > #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) > +#include "selftest_engine.c" > #include "selftest_engine_cs.c" > #endif > diff --git a/drivers/gpu/drm/i915/gt/intel_engine_pm.c b/drivers/gpu/drm/i915/gt/intel_engine_pm.c > index 0336204988d6..6b15e3335dd6 100644 > --- a/drivers/gpu/drm/i915/gt/intel_engine_pm.c > +++ b/drivers/gpu/drm/i915/gt/intel_engine_pm.c > @@ -37,28 +37,6 @@ static int __engine_unpark(struct intel_wakeref *wf) > return 0; > } > > -void intel_engine_pm_get(struct intel_engine_cs *engine) > -{ > - intel_wakeref_get(&engine->i915->runtime_pm, &engine->wakeref, __engine_unpark); > -} > - > -void intel_engine_park(struct intel_engine_cs *engine) > -{ > - /* > - * We are committed now to parking this engine, make sure there > - * will be no more interrupts arriving later and the engine > - * is truly idle. > - */ > - if (wait_for(intel_engine_is_idle(engine), 10)) { > - struct drm_printer p = drm_debug_printer(__func__); > - > - dev_err(engine->i915->drm.dev, > - "%s is not idle before parking\n", > - engine->name); > - intel_engine_dump(engine, &p, NULL); > - } > -} > - > static bool switch_to_kernel_context(struct intel_engine_cs *engine) > { > struct i915_request *rq; > @@ -136,12 +114,18 @@ static int __engine_park(struct intel_wakeref *wf) > return 0; > } > > -void intel_engine_pm_put(struct intel_engine_cs *engine) > -{ > - intel_wakeref_put(&engine->i915->runtime_pm, &engine->wakeref, __engine_park); > -} > +static const struct intel_wakeref_ops wf_ops = { > + .get = __engine_unpark, > + .put = __engine_park, > +}; > > void intel_engine_init__pm(struct intel_engine_cs *engine) > { > - intel_wakeref_init(&engine->wakeref); > + struct intel_runtime_pm *rpm = &engine->i915->runtime_pm; > + > + intel_wakeref_init(&engine->wakeref, rpm, &wf_ops); > } > + > +#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) > +#include "selftest_engine_pm.c" > +#endif > diff --git a/drivers/gpu/drm/i915/gt/intel_engine_pm.h b/drivers/gpu/drm/i915/gt/intel_engine_pm.h > index 015ac72d7ad0..739c50fefcef 100644 > --- a/drivers/gpu/drm/i915/gt/intel_engine_pm.h > +++ b/drivers/gpu/drm/i915/gt/intel_engine_pm.h > @@ -10,24 +10,26 @@ > #include "intel_engine_types.h" > #include "intel_wakeref.h" > > -struct drm_i915_private; > - > -void intel_engine_pm_get(struct intel_engine_cs *engine); > -void intel_engine_pm_put(struct intel_engine_cs *engine); > - > static inline bool > intel_engine_pm_is_awake(const struct intel_engine_cs *engine) > { > return intel_wakeref_is_active(&engine->wakeref); > } > > -static inline bool > -intel_engine_pm_get_if_awake(struct intel_engine_cs *engine) > +static inline void intel_engine_pm_get(struct intel_engine_cs *engine) > +{ > + intel_wakeref_get(&engine->wakeref); > +} > + > +static inline bool intel_engine_pm_get_if_awake(struct intel_engine_cs *engine) > { > return intel_wakeref_get_if_active(&engine->wakeref); > } > > -void intel_engine_park(struct intel_engine_cs *engine); > +static inline void intel_engine_pm_put(struct intel_engine_cs *engine) > +{ > + intel_wakeref_put(&engine->wakeref); > +} > > void intel_engine_init__pm(struct intel_engine_cs *engine); > > diff --git a/drivers/gpu/drm/i915/gt/intel_gt_pm.c b/drivers/gpu/drm/i915/gt/intel_gt_pm.c > index 6c8970271a7f..1363e069ec83 100644 > --- a/drivers/gpu/drm/i915/gt/intel_gt_pm.c > +++ b/drivers/gpu/drm/i915/gt/intel_gt_pm.c > @@ -17,7 +17,7 @@ static void pm_notify(struct drm_i915_private *i915, int state) > blocking_notifier_call_chain(&i915->gt.pm_notifications, state, i915); > } > > -static int intel_gt_unpark(struct intel_wakeref *wf) > +static int __gt_unpark(struct intel_wakeref *wf) > { > struct intel_gt *gt = container_of(wf, typeof(*gt), wakeref); > struct drm_i915_private *i915 = gt->i915; > @@ -53,14 +53,7 @@ static int intel_gt_unpark(struct intel_wakeref *wf) > return 0; > } > > -void intel_gt_pm_get(struct intel_gt *gt) > -{ > - struct intel_runtime_pm *rpm = >->i915->runtime_pm; > - > - intel_wakeref_get(rpm, >->wakeref, intel_gt_unpark); > -} > - > -static int intel_gt_park(struct intel_wakeref *wf) > +static int __gt_park(struct intel_wakeref *wf) > { > struct drm_i915_private *i915 = > container_of(wf, typeof(*i915), gt.wakeref); > @@ -74,22 +67,25 @@ static int intel_gt_park(struct intel_wakeref *wf) > if (INTEL_GEN(i915) >= 6) > gen6_rps_idle(i915); > > + /* Everything switched off, flush any residual interrupt just in case */ > + intel_synchronize_irq(i915); > + > GEM_BUG_ON(!wakeref); > intel_display_power_put(i915, POWER_DOMAIN_GT_IRQ, wakeref); > > return 0; > } > > -void intel_gt_pm_put(struct intel_gt *gt) > -{ > - struct intel_runtime_pm *rpm = >->i915->runtime_pm; > - > - intel_wakeref_put(rpm, >->wakeref, intel_gt_park); > -} > +static const struct intel_wakeref_ops wf_ops = { > + .get = __gt_unpark, > + .put = __gt_park, > + .flags = INTEL_WAKEREF_PUT_ASYNC, > +}; > > void intel_gt_pm_init_early(struct intel_gt *gt) > { > - intel_wakeref_init(>->wakeref); > + intel_wakeref_init(>->wakeref, >->i915->runtime_pm, &wf_ops); > + > BLOCKING_INIT_NOTIFIER_HEAD(>->pm_notifications); > } > > diff --git a/drivers/gpu/drm/i915/gt/intel_gt_pm.h b/drivers/gpu/drm/i915/gt/intel_gt_pm.h > index e8a18d4b27c9..57633f538ddb 100644 > --- a/drivers/gpu/drm/i915/gt/intel_gt_pm.h > +++ b/drivers/gpu/drm/i915/gt/intel_gt_pm.h > @@ -17,14 +17,32 @@ enum { > INTEL_GT_PARK, > }; > > -void intel_gt_pm_get(struct intel_gt *gt); > -void intel_gt_pm_put(struct intel_gt *gt); > +static inline bool > +intel_gt_pm_is_awake(const struct intel_gt *gt) > +{ > + return intel_wakeref_is_active(>->wakeref); > +} > + > +static inline void intel_gt_pm_get(struct intel_gt *gt) > +{ > + intel_wakeref_get(>->wakeref); > +} > > static inline bool intel_gt_pm_get_if_awake(struct intel_gt *gt) > { > return intel_wakeref_get_if_active(>->wakeref); > } > > +static inline void intel_gt_pm_put(struct intel_gt *gt) > +{ > + intel_wakeref_put(>->wakeref); > +} > + > +static inline int intel_gt_pm_wait_for_idle(struct intel_gt *gt) > +{ > + return intel_wakeref_wait_for_idle(>->wakeref); > +} > + > void intel_gt_pm_init_early(struct intel_gt *gt); > > void intel_gt_sanitize(struct intel_gt *gt, bool force); > diff --git a/drivers/gpu/drm/i915/gt/intel_lrc.c b/drivers/gpu/drm/i915/gt/intel_lrc.c > index 232f40fcb490..1a0116751d19 100644 > --- a/drivers/gpu/drm/i915/gt/intel_lrc.c > +++ b/drivers/gpu/drm/i915/gt/intel_lrc.c > @@ -139,6 +139,7 @@ > #include "i915_vgpu.h" > #include "intel_engine_pm.h" > #include "intel_gt.h" > +#include "intel_gt_pm.h" > #include "intel_lrc_reg.h" > #include "intel_mocs.h" > #include "intel_reset.h" > @@ -556,6 +557,7 @@ execlists_schedule_in(struct i915_request *rq, int idx) > intel_context_get(ce); > ce->inflight = rq->engine; > > + intel_gt_pm_get(ce->inflight->gt); > execlists_context_status_change(rq, INTEL_CONTEXT_SCHEDULE_IN); > intel_engine_context_in(ce->inflight); > } > @@ -588,6 +590,7 @@ execlists_schedule_out(struct i915_request *rq) > if (!intel_context_inflight_count(ce)) { > intel_engine_context_out(ce->inflight); > execlists_context_status_change(rq, INTEL_CONTEXT_SCHEDULE_OUT); > + intel_gt_pm_put(ce->inflight->gt); > > /* > * If this is part of a virtual engine, its next request may > @@ -2719,7 +2722,6 @@ static u32 *gen8_emit_fini_breadcrumb_rcs(struct i915_request *request, u32 *cs) > static void execlists_park(struct intel_engine_cs *engine) > { > del_timer_sync(&engine->execlists.timer); > - intel_engine_park(engine); > } > > void intel_execlists_set_default_submission(struct intel_engine_cs *engine) > diff --git a/drivers/gpu/drm/i915/gt/selftest_engine.c b/drivers/gpu/drm/i915/gt/selftest_engine.c > new file mode 100644 > index 000000000000..f65b118e261d > --- /dev/null > +++ b/drivers/gpu/drm/i915/gt/selftest_engine.c > @@ -0,0 +1,28 @@ > +/* > + * SPDX-License-Identifier: GPL-2.0 > + * > + * Copyright © 2018 Intel Corporation > + */ > + > +#include "i915_selftest.h" > +#include "selftest_engine.h" > + > +int intel_engine_live_selftests(struct drm_i915_private *i915) > +{ > + static int (* const tests[])(struct intel_gt *) = { > + live_engine_pm_selftests, > + NULL, > + }; > + struct intel_gt *gt = &i915->gt; > + typeof(*tests) *fn; > + > + for (fn = tests; *fn; fn++) { > + int err; > + > + err = (*fn)(gt); > + if (err) > + return err; > + } > + > + return 0; > +} > diff --git a/drivers/gpu/drm/i915/gt/selftest_engine.h b/drivers/gpu/drm/i915/gt/selftest_engine.h > new file mode 100644 > index 000000000000..ab32d09ec5a1 > --- /dev/null > +++ b/drivers/gpu/drm/i915/gt/selftest_engine.h > @@ -0,0 +1,14 @@ > +/* > + * SPDX-License-Identifier: GPL-2.0 > + * > + * Copyright © 2019 Intel Corporation > + */ > + > +#ifndef SELFTEST_ENGINE_H > +#define SELFTEST_ENGINE_H > + > +struct intel_gt; > + > +int live_engine_pm_selftests(struct intel_gt *gt); > + > +#endif > diff --git a/drivers/gpu/drm/i915/gt/selftest_engine_pm.c b/drivers/gpu/drm/i915/gt/selftest_engine_pm.c > new file mode 100644 > index 000000000000..75c197612705 > --- /dev/null > +++ b/drivers/gpu/drm/i915/gt/selftest_engine_pm.c > @@ -0,0 +1,83 @@ > +/* > + * SPDX-License-Identifier: GPL-2.0 > + * > + * Copyright © 2018 Intel Corporation > + */ > + > +#include "i915_selftest.h" > +#include "selftest_engine.h" > +#include "selftests/igt_atomic.h" > + > +static int live_engine_pm(void *arg) > +{ > + struct intel_gt *gt = arg; > + struct intel_engine_cs *engine; > + enum intel_engine_id id; > + > + /* > + * Check we can call intel_engine_pm_put from any context. No > + * failures are reported directly, but if we mess up lockdep should > + * tell us. > + */ > + if (intel_gt_pm_wait_for_idle(gt)) { > + pr_err("Unable to flush GT pm before test\n"); > + return -EBUSY; > + } > + > + GEM_BUG_ON(intel_gt_pm_is_awake(gt)); > + for_each_engine(engine, gt->i915, id) { > + const typeof(*igt_atomic_phases) *p; > + > + for (p = igt_atomic_phases; p->name; p++) { > + /* > + * Acquisition is always synchronous, except if we > + * know that the engine is already awale, in which > + * we should use intel_engine_pm_get_if_awake() to > + * atomically grab the wakeref. > + * > + * In practice, > + * intel_engine_pm_get(); > + * intel_engine_pm_put(); > + * occurs in one thread, while simultaneously > + * intel_engine_pm_get_if_awake(); > + * intel_engine_pm_put(); > + * occurs in atomic context in another. > + */ > + GEM_BUG_ON(intel_engine_pm_is_awake(engine)); > + intel_engine_pm_get(engine); > + > + p->critical_section_begin(); > + if (!intel_engine_pm_get_if_awake(engine)) > + pr_err("intel_engine_pm_get_if_awake(%s) failed under %s\n", > + engine->name, p->name); > + else > + intel_engine_pm_put(engine); > + intel_engine_pm_put(engine); > + p->critical_section_end(); > + > + /* engine wakeref is sync (instant) */ > + if (intel_engine_pm_is_awake(engine)) { > + pr_err("%s is still awake after flushing pm\n", > + engine->name); > + return -EINVAL; > + } > + > + /* gt wakeref is async (deferred to workqueue) */ > + if (intel_gt_pm_wait_for_idle(gt)) { > + pr_err("GT failed to idle\n"); > + return -EINVAL; > + } > + } > + } > + > + return 0; > +} > + > +int live_engine_pm_selftests(struct intel_gt *gt) > +{ > + static const struct i915_subtest tests[] = { > + SUBTEST(live_engine_pm), > + }; > + > + return intel_gt_live_subtests(tests, gt); > +} > diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.c b/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.c > index a78bd99bc6cb..c9119b90d6aa 100644 > --- a/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.c > +++ b/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.c > @@ -29,6 +29,7 @@ > #include "gt/intel_context.h" > #include "gt/intel_engine_pm.h" > #include "gt/intel_gt.h" > +#include "gt/intel_gt_pm.h" > #include "gt/intel_lrc_reg.h" > #include "intel_guc_submission.h" > > @@ -537,6 +538,7 @@ static struct i915_request *schedule_in(struct i915_request *rq, int idx) > if (!rq->hw_context->inflight) > rq->hw_context->inflight = rq->engine; > intel_context_inflight_inc(rq->hw_context); > + intel_gt_pm_get(rq->engine->gt); > > return i915_request_get(rq); > } > @@ -549,6 +551,7 @@ static void schedule_out(struct i915_request *rq) > if (!intel_context_inflight_count(rq->hw_context)) > rq->hw_context->inflight = NULL; > > + intel_gt_pm_put(rq->engine->gt); > i915_request_put(rq); > } > > @@ -1076,7 +1079,6 @@ static void guc_interrupts_release(struct intel_gt *gt) > > static void guc_submission_park(struct intel_engine_cs *engine) > { > - intel_engine_park(engine); > intel_engine_unpin_breadcrumbs_irq(engine); > engine->flags &= ~I915_ENGINE_NEEDS_BREADCRUMB_TASKLET; > } > diff --git a/drivers/gpu/drm/i915/i915_debugfs.c b/drivers/gpu/drm/i915/i915_debugfs.c > index 63b599f584db..68a6b002c9ba 100644 > --- a/drivers/gpu/drm/i915/i915_debugfs.c > +++ b/drivers/gpu/drm/i915/i915_debugfs.c > @@ -39,6 +39,7 @@ > #include "display/intel_psr.h" > > #include "gem/i915_gem_context.h" > +#include "gt/intel_gt_pm.h" > #include "gt/intel_reset.h" > #include "gt/uc/intel_guc_submission.h" > > @@ -3664,6 +3665,9 @@ i915_drop_caches_set(void *data, u64 val) > i915_retire_requests(i915); > > mutex_unlock(&i915->drm.struct_mutex); > + > + if (ret == 0 && val & DROP_IDLE) > + ret = intel_gt_pm_wait_for_idle(&i915->gt); > } > > if (val & DROP_RESET_ACTIVE && intel_gt_terminally_wedged(&i915->gt)) > diff --git a/drivers/gpu/drm/i915/i915_gem.c b/drivers/gpu/drm/i915/i915_gem.c > index 7efff14b9137..f67197336ee7 100644 > --- a/drivers/gpu/drm/i915/i915_gem.c > +++ b/drivers/gpu/drm/i915/i915_gem.c > @@ -894,19 +894,6 @@ void i915_gem_runtime_suspend(struct drm_i915_private *i915) > } > } > > -static int wait_for_engines(struct intel_gt *gt) > -{ > - if (wait_for(intel_engines_are_idle(gt), I915_IDLE_ENGINES_TIMEOUT)) { > - dev_err(gt->i915->drm.dev, > - "Failed to idle engines, declaring wedged!\n"); > - GEM_TRACE_DUMP(); > - intel_gt_set_wedged(gt); > - return -EIO; > - } > - > - return 0; > -} > - > static long > wait_for_timelines(struct drm_i915_private *i915, > unsigned int flags, long timeout) > @@ -954,27 +941,20 @@ int i915_gem_wait_for_idle(struct drm_i915_private *i915, > unsigned int flags, long timeout) > { > /* If the device is asleep, we have no requests outstanding */ > - if (!READ_ONCE(i915->gt.awake)) > + if (!intel_gt_pm_is_awake(&i915->gt)) > return 0; > > - GEM_TRACE("flags=%x (%s), timeout=%ld%s, awake?=%s\n", > + GEM_TRACE("flags=%x (%s), timeout=%ld%s\n", > flags, flags & I915_WAIT_LOCKED ? "locked" : "unlocked", > - timeout, timeout == MAX_SCHEDULE_TIMEOUT ? " (forever)" : "", > - yesno(i915->gt.awake)); > + timeout, timeout == MAX_SCHEDULE_TIMEOUT ? " (forever)" : ""); > > timeout = wait_for_timelines(i915, flags, timeout); > if (timeout < 0) > return timeout; > > if (flags & I915_WAIT_LOCKED) { > - int err; > - > lockdep_assert_held(&i915->drm.struct_mutex); > > - err = wait_for_engines(&i915->gt); > - if (err) > - return err; > - > i915_retire_requests(i915); > } > > diff --git a/drivers/gpu/drm/i915/intel_wakeref.c b/drivers/gpu/drm/i915/intel_wakeref.c > index 06bd8b215cc2..d4443e81c1c8 100644 > --- a/drivers/gpu/drm/i915/intel_wakeref.c > +++ b/drivers/gpu/drm/i915/intel_wakeref.c > @@ -4,25 +4,25 @@ > * Copyright © 2019 Intel Corporation > */ > > +#include <linux/wait_bit.h> > + > #include "intel_runtime_pm.h" > #include "intel_wakeref.h" > > -static void rpm_get(struct intel_runtime_pm *rpm, struct intel_wakeref *wf) > +static void rpm_get(struct intel_wakeref *wf) > { > - wf->wakeref = intel_runtime_pm_get(rpm); > + wf->wakeref = intel_runtime_pm_get(wf->rpm); > } > > -static void rpm_put(struct intel_runtime_pm *rpm, struct intel_wakeref *wf) > +static void rpm_put(struct intel_wakeref *wf) > { > intel_wakeref_t wakeref = fetch_and_zero(&wf->wakeref); > > - intel_runtime_pm_put(rpm, wakeref); > + intel_runtime_pm_put(wf->rpm, wakeref); > INTEL_WAKEREF_BUG_ON(!wakeref); > } > > -int __intel_wakeref_get_first(struct intel_runtime_pm *rpm, > - struct intel_wakeref *wf, > - int (*fn)(struct intel_wakeref *wf)) > +int __intel_wakeref_get_first(struct intel_wakeref *wf) > { > /* > * Treat get/put as different subclasses, as we may need to run > @@ -34,11 +34,11 @@ int __intel_wakeref_get_first(struct intel_runtime_pm *rpm, > if (!atomic_read(&wf->count)) { > int err; > > - rpm_get(rpm, wf); > + rpm_get(wf); > > - err = fn(wf); > + err = wf->ops->get(wf); > if (unlikely(err)) { > - rpm_put(rpm, wf); > + rpm_put(wf); > mutex_unlock(&wf->mutex); > return err; > } > @@ -52,27 +52,67 @@ int __intel_wakeref_get_first(struct intel_runtime_pm *rpm, > return 0; > } > > -int __intel_wakeref_put_last(struct intel_runtime_pm *rpm, > - struct intel_wakeref *wf, > - int (*fn)(struct intel_wakeref *wf)) > +static void ____intel_wakeref_put_last(struct intel_wakeref *wf) > { > - int err; > + if (!atomic_dec_and_test(&wf->count)) > + goto unlock; > + > + if (likely(!wf->ops->put(wf))) { > + rpm_put(wf); > + wake_up_var(&wf->wakeref); > + } else { > + /* ops->put() must schedule its own release on deferral */ > + atomic_set_release(&wf->count, 1); > + } > > - err = fn(wf); > - if (likely(!err)) > - rpm_put(rpm, wf); > - else > - atomic_inc(&wf->count); > +unlock: > mutex_unlock(&wf->mutex); > +} > + > +void __intel_wakeref_put_last(struct intel_wakeref *wf) > +{ > + INTEL_WAKEREF_BUG_ON(work_pending(&wf->work)); > > - return err; > + /* Assume we are not in process context and so cannot sleep. */ > + if (wf->ops->flags & INTEL_WAKEREF_PUT_ASYNC || > + !mutex_trylock(&wf->mutex)) { > + schedule_work(&wf->work); > + return; > + } > + > + ____intel_wakeref_put_last(wf); > } > > -void __intel_wakeref_init(struct intel_wakeref *wf, struct lock_class_key *key) > +static void __intel_wakeref_put_work(struct work_struct *wrk) > { > + struct intel_wakeref *wf = container_of(wrk, typeof(*wf), work); > + > + if (atomic_add_unless(&wf->count, -1, 1)) > + return; > + > + mutex_lock(&wf->mutex); > + ____intel_wakeref_put_last(wf); > +} > + > +void __intel_wakeref_init(struct intel_wakeref *wf, > + struct intel_runtime_pm *rpm, > + const struct intel_wakeref_ops *ops, > + struct lock_class_key *key) > +{ > + wf->rpm = rpm; > + wf->ops = ops; > + > __mutex_init(&wf->mutex, "wakeref", key); > atomic_set(&wf->count, 0); > wf->wakeref = 0; > + > + INIT_WORK(&wf->work, __intel_wakeref_put_work); > +} > + > +int intel_wakeref_wait_for_idle(struct intel_wakeref *wf) > +{ > + return wait_var_event_killable(&wf->wakeref, > + !intel_wakeref_is_active(wf)); > } > > static void wakeref_auto_timeout(struct timer_list *t) > diff --git a/drivers/gpu/drm/i915/intel_wakeref.h b/drivers/gpu/drm/i915/intel_wakeref.h > index 1d6f5986e4e5..535a3a12864b 100644 > --- a/drivers/gpu/drm/i915/intel_wakeref.h > +++ b/drivers/gpu/drm/i915/intel_wakeref.h > @@ -8,10 +8,12 @@ > #define INTEL_WAKEREF_H > > #include <linux/atomic.h> > +#include <linux/bits.h> > #include <linux/mutex.h> > #include <linux/refcount.h> > #include <linux/stackdepot.h> > #include <linux/timer.h> > +#include <linux/workqueue.h> > > #if IS_ENABLED(CONFIG_DRM_I915_DEBUG) > #define INTEL_WAKEREF_BUG_ON(expr) BUG_ON(expr) > @@ -20,29 +22,42 @@ > #endif > > struct intel_runtime_pm; > +struct intel_wakeref; > > typedef depot_stack_handle_t intel_wakeref_t; > > +struct intel_wakeref_ops { > + int (*get)(struct intel_wakeref *wf); > + int (*put)(struct intel_wakeref *wf); > + > + unsigned long flags; > +#define INTEL_WAKEREF_PUT_ASYNC BIT(0) > +}; > + > struct intel_wakeref { > atomic_t count; > struct mutex mutex; > + > intel_wakeref_t wakeref; > + > + struct intel_runtime_pm *rpm; > + const struct intel_wakeref_ops *ops; > + > + struct work_struct work; > }; > > void __intel_wakeref_init(struct intel_wakeref *wf, > + struct intel_runtime_pm *rpm, > + const struct intel_wakeref_ops *ops, > struct lock_class_key *key); > -#define intel_wakeref_init(wf) do { \ > +#define intel_wakeref_init(wf, rpm, ops) do { \ > static struct lock_class_key __key; \ > \ > - __intel_wakeref_init((wf), &__key); \ > + __intel_wakeref_init((wf), (rpm), (ops), &__key); \ > } while (0) > > -int __intel_wakeref_get_first(struct intel_runtime_pm *rpm, > - struct intel_wakeref *wf, > - int (*fn)(struct intel_wakeref *wf)); > -int __intel_wakeref_put_last(struct intel_runtime_pm *rpm, > - struct intel_wakeref *wf, > - int (*fn)(struct intel_wakeref *wf)); > +int __intel_wakeref_get_first(struct intel_wakeref *wf); > +void __intel_wakeref_put_last(struct intel_wakeref *wf); > > /** > * intel_wakeref_get: Acquire the wakeref > @@ -61,12 +76,10 @@ int __intel_wakeref_put_last(struct intel_runtime_pm *rpm, > * code otherwise. > */ > static inline int > -intel_wakeref_get(struct intel_runtime_pm *rpm, > - struct intel_wakeref *wf, > - int (*fn)(struct intel_wakeref *wf)) > +intel_wakeref_get(struct intel_wakeref *wf) > { > if (unlikely(!atomic_inc_not_zero(&wf->count))) > - return __intel_wakeref_get_first(rpm, wf, fn); > + return __intel_wakeref_get_first(wf); > > return 0; > } > @@ -102,16 +115,12 @@ intel_wakeref_get_if_active(struct intel_wakeref *wf) > * Returns: 0 if the wakeref was released successfully, or a negative error > * code otherwise. > */ > -static inline int > -intel_wakeref_put(struct intel_runtime_pm *rpm, > - struct intel_wakeref *wf, > - int (*fn)(struct intel_wakeref *wf)) > +static inline void > +intel_wakeref_put(struct intel_wakeref *wf) > { > INTEL_WAKEREF_BUG_ON(atomic_read(&wf->count) <= 0); > - if (atomic_dec_and_mutex_lock(&wf->count, &wf->mutex)) > - return __intel_wakeref_put_last(rpm, wf, fn); > - > - return 0; > + if (unlikely(!atomic_add_unless(&wf->count, -1, 1))) > + __intel_wakeref_put_last(wf); > } > > /** > @@ -154,6 +163,19 @@ intel_wakeref_is_active(const struct intel_wakeref *wf) > return READ_ONCE(wf->wakeref); > } > > +/** > + * intel_wakeref_wait_for_idle: Wait until the wakeref is idle > + * @wf: the wakeref > + * > + * Wait for the earlier asynchronous release of the wakeref. Note > + * this will wait for any third party as well, so make sure you only wait > + * when you have control over the wakeref and trust no one else is acquiring > + * it. > + * > + * Return: 0 on success, error code if killed. > + */ > +int intel_wakeref_wait_for_idle(struct intel_wakeref *wf); > + > struct intel_wakeref_auto { > struct intel_runtime_pm *rpm; > struct timer_list timer; > diff --git a/drivers/gpu/drm/i915/selftests/i915_live_selftests.h b/drivers/gpu/drm/i915/selftests/i915_live_selftests.h > index a841d3f9bedc..1ccf0f731ac0 100644 > --- a/drivers/gpu/drm/i915/selftests/i915_live_selftests.h > +++ b/drivers/gpu/drm/i915/selftests/i915_live_selftests.h > @@ -12,10 +12,11 @@ > selftest(sanitycheck, i915_live_sanitycheck) /* keep first (igt selfcheck) */ > selftest(uncore, intel_uncore_live_selftests) > selftest(workarounds, intel_workarounds_live_selftests) > -selftest(timelines, intel_timeline_live_selftests) > +selftest(gt_engines, intel_engine_live_selftests) > +selftest(gt_timelines, intel_timeline_live_selftests) > +selftest(gt_contexts, intel_context_live_selftests) > selftest(requests, i915_request_live_selftests) > selftest(active, i915_active_live_selftests) > -selftest(gt_contexts, intel_context_live_selftests) > selftest(objects, i915_gem_object_live_selftests) > selftest(mman, i915_gem_mman_live_selftests) > selftest(dmabuf, i915_gem_dmabuf_live_selftests) > -- > 2.23.0.rc1 > _______________________________________________ Intel-gfx mailing list Intel-gfx@xxxxxxxxxxxxxxxxxxxxx https://lists.freedesktop.org/mailman/listinfo/intel-gfx
