From: Tvrtko Ursulin <tvrtko.ursulin@xxxxxxxxx> Track how many callers are explicity waiting on a fence to signal and allow querying that via new dma_fence_wait_count() API. This provides infrastructure on top of which generic "waitboost" concepts can be implemented by individual drivers. Wait-boosting is any reactive activity, such as raising the GPU clocks, which happens while there are active external waiters. Signed-off-by: Tvrtko Ursulin <tvrtko.ursulin@xxxxxxxxx> --- drivers/dma-buf/dma-fence.c | 102 ++++++++++++++++------ drivers/gpu/drm/i915/gt/intel_engine_pm.c | 1 - include/linux/dma-fence.h | 14 +++ 3 files changed, 88 insertions(+), 29 deletions(-) diff --git a/drivers/dma-buf/dma-fence.c b/drivers/dma-buf/dma-fence.c index 0de0482cd36e..ed43290c0bdf 100644 --- a/drivers/dma-buf/dma-fence.c +++ b/drivers/dma-buf/dma-fence.c @@ -344,6 +344,24 @@ void __dma_fence_might_wait(void) } #endif +static void incr_wait_count(struct dma_fence *fence, struct dma_fence_cb *cb) +{ + lockdep_assert_held(fence->lock); + + __set_bit(DMA_FENCE_CB_FLAG_WAITCOUNT_BIT, &cb->flags); + fence->waitcount++; + WARN_ON_ONCE(!fence->waitcount); +} + +static void decr_wait_count(struct dma_fence *fence, struct dma_fence_cb *cb) +{ + lockdep_assert_held(fence->lock); + + if (__test_and_clear_bit(DMA_FENCE_CB_FLAG_WAITCOUNT_BIT, &cb->flags)) { + WARN_ON_ONCE(!fence->waitcount); + fence->waitcount--; + } +} /** * dma_fence_signal_timestamp_locked - signal completion of a fence @@ -384,6 +402,7 @@ int dma_fence_signal_timestamp_locked(struct dma_fence *fence, list_for_each_entry_safe(cur, tmp, &cb_list, node) { INIT_LIST_HEAD(&cur->node); + decr_wait_count(fence, cur); cur->func(fence, cur); } @@ -612,35 +631,15 @@ void dma_fence_enable_sw_signaling(struct dma_fence *fence) unsigned long flags; spin_lock_irqsave(fence->lock, flags); + fence->waitcount++; + WARN_ON_ONCE(!fence->waitcount); __dma_fence_enable_signaling(fence); spin_unlock_irqrestore(fence->lock, flags); } EXPORT_SYMBOL(dma_fence_enable_sw_signaling); -/** - * dma_fence_add_callback - add a callback to be called when the fence - * is signaled - * @fence: the fence to wait on - * @cb: the callback to register - * @func: the function to call - * - * Add a software callback to the fence. The caller should keep a reference to - * the fence. - * - * @cb will be initialized by dma_fence_add_callback(), no initialization - * by the caller is required. Any number of callbacks can be registered - * to a fence, but a callback can only be registered to one fence at a time. - * - * If fence is already signaled, this function will return -ENOENT (and - * *not* call the callback). - * - * Note that the callback can be called from an atomic context or irq context. - * - * Returns 0 in case of success, -ENOENT if the fence is already signaled - * and -EINVAL in case of error. - */ -int dma_fence_add_callback(struct dma_fence *fence, struct dma_fence_cb *cb, - dma_fence_func_t func) +static int add_callback(struct dma_fence *fence, struct dma_fence_cb *cb, + dma_fence_func_t func, bool wait) { unsigned long flags; int ret = 0; @@ -655,10 +654,15 @@ int dma_fence_add_callback(struct dma_fence *fence, struct dma_fence_cb *cb, spin_lock_irqsave(fence->lock, flags); + if (wait) + incr_wait_count(fence, cb); + if (__dma_fence_enable_signaling(fence)) { cb->func = func; list_add_tail(&cb->node, &fence->cb_list); } else { + if (test_bit(DMA_FENCE_CB_FLAG_WAITCOUNT_BIT, &cb->flags)) + decr_wait_count(fence, cb); INIT_LIST_HEAD(&cb->node); ret = -ENOENT; } @@ -667,8 +671,44 @@ int dma_fence_add_callback(struct dma_fence *fence, struct dma_fence_cb *cb, return ret; } + +/** + * dma_fence_add_callback - add a callback to be called when the fence + * is signaled + * @fence: the fence to wait on + * @cb: the callback to register + * @func: the function to call + * + * Add a software callback to the fence. The caller should keep a reference to + * the fence. + * + * @cb will be initialized by dma_fence_add_callback(), no initialization + * by the caller is required. Any number of callbacks can be registered + * to a fence, but a callback can only be registered to one fence at a time. + * + * If fence is already signaled, this function will return -ENOENT (and + * *not* call the callback). + * + * Note that the callback can be called from an atomic context or irq context. + * + * Returns 0 in case of success, -ENOENT if the fence is already signaled + * and -EINVAL in case of error. + */ +int dma_fence_add_callback(struct dma_fence *fence, struct dma_fence_cb *cb, + dma_fence_func_t func) +{ + return add_callback(fence, cb, func, false); +} EXPORT_SYMBOL(dma_fence_add_callback); +int dma_fence_add_wait_callback(struct dma_fence *fence, + struct dma_fence_cb *cb, + dma_fence_func_t func) +{ + return add_callback(fence, cb, func, true); +} +EXPORT_SYMBOL(dma_fence_add_wait_callback); + /** * dma_fence_get_status - returns the status upon completion * @fence: the dma_fence to query @@ -721,8 +761,10 @@ dma_fence_remove_callback(struct dma_fence *fence, struct dma_fence_cb *cb) spin_lock_irqsave(fence->lock, flags); ret = !list_empty(&cb->node); - if (ret) + if (ret) { + decr_wait_count(fence, cb); list_del_init(&cb->node); + } spin_unlock_irqrestore(fence->lock, flags); @@ -780,6 +822,7 @@ dma_fence_default_wait(struct dma_fence *fence, bool intr, signed long timeout) cb.base.func = dma_fence_default_wait_cb; cb.task = current; + incr_wait_count(fence, &cb.base); list_add(&cb.base.node, &fence->cb_list); while (!test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags) && ret > 0) { @@ -796,8 +839,10 @@ dma_fence_default_wait(struct dma_fence *fence, bool intr, signed long timeout) ret = -ERESTARTSYS; } - if (!list_empty(&cb.base.node)) + if (!list_empty(&cb.base.node)) { + decr_wait_count(fence, &cb.base); list_del(&cb.base.node); + } __set_current_state(TASK_RUNNING); out: @@ -875,8 +920,8 @@ dma_fence_wait_any_timeout(struct dma_fence **fences, uint32_t count, struct dma_fence *fence = fences[i]; cb[i].task = current; - if (dma_fence_add_callback(fence, &cb[i].base, - dma_fence_default_wait_cb)) { + if (dma_fence_add_wait_callback(fence, &cb[i].base, + dma_fence_default_wait_cb)) { /* This fence is already signaled */ if (idx) *idx = i; @@ -957,6 +1002,7 @@ dma_fence_init(struct dma_fence *fence, const struct dma_fence_ops *ops, fence->context = context; fence->seqno = seqno; fence->flags = 0UL; + fence->waitcount = 0; fence->error = 0; trace_dma_fence_init(fence); diff --git a/drivers/gpu/drm/i915/gt/intel_engine_pm.c b/drivers/gpu/drm/i915/gt/intel_engine_pm.c index e971b153fda9..2693a0151a6b 100644 --- a/drivers/gpu/drm/i915/gt/intel_engine_pm.c +++ b/drivers/gpu/drm/i915/gt/intel_engine_pm.c @@ -218,7 +218,6 @@ static bool switch_to_kernel_context(struct intel_engine_cs *engine) * until the background request retirement running every * second or two). */ - BUILD_BUG_ON(sizeof(rq->duration) > sizeof(rq->submitq)); dma_fence_add_callback(&rq->fence, &rq->duration.cb, duration); rq->duration.emitted = ktime_get(); } diff --git a/include/linux/dma-fence.h b/include/linux/dma-fence.h index 775cdc0b4f24..d0ed923e4545 100644 --- a/include/linux/dma-fence.h +++ b/include/linux/dma-fence.h @@ -92,6 +92,7 @@ struct dma_fence { u64 seqno; unsigned long flags; struct kref refcount; + unsigned int waitcount; int error; }; @@ -116,6 +117,11 @@ typedef void (*dma_fence_func_t)(struct dma_fence *fence, struct dma_fence_cb { struct list_head node; dma_fence_func_t func; + unsigned long flags; +}; + +enum dma_fence_cb_flag_bits { + DMA_FENCE_CB_FLAG_WAITCOUNT_BIT, }; /** @@ -377,6 +383,9 @@ signed long dma_fence_default_wait(struct dma_fence *fence, int dma_fence_add_callback(struct dma_fence *fence, struct dma_fence_cb *cb, dma_fence_func_t func); +int dma_fence_add_wait_callback(struct dma_fence *fence, + struct dma_fence_cb *cb, + dma_fence_func_t func); bool dma_fence_remove_callback(struct dma_fence *fence, struct dma_fence_cb *cb); void dma_fence_enable_sw_signaling(struct dma_fence *fence); @@ -528,6 +537,11 @@ static inline int dma_fence_get_status_locked(struct dma_fence *fence) int dma_fence_get_status(struct dma_fence *fence); +static inline unsigned int dma_fence_wait_count(struct dma_fence *fence) +{ + return fence->waitcount; +} + /** * dma_fence_set_error - flag an error condition on the fence * @fence: the dma_fence -- 2.34.1