From: John Harrison <John.C.Harrison@xxxxxxxxx>
When requesting that all GPU work is completed, it is now necessary to
get the scheduler involved in order to flush out work that queued and
not yet submitted.
v2: Updated to add support for flushing the scheduler queue by time
stamp rather than just doing a blanket flush.
v3: Moved submit_max_priority() to this patch from an earlier patch
is it is no longer required in the other.
Change-Id: I95dcc2a2ee5c1a844748621c333994ddd6cf6a66
For: VIZ-1587
Signed-off-by: John Harrison <John.C.Harrison@xxxxxxxxx>
---
drivers/gpu/drm/i915/i915_gem.c | 24 ++++++-
drivers/gpu/drm/i915/i915_scheduler.c | 132 ++++++++++++++++++++++++++++++++++
drivers/gpu/drm/i915/i915_scheduler.h | 3 +
3 files changed, 158 insertions(+), 1 deletion(-)
diff --git a/drivers/gpu/drm/i915/i915_gem.c b/drivers/gpu/drm/i915/i915_gem.c
index 1a05c97..541ed9a 100644
--- a/drivers/gpu/drm/i915/i915_gem.c
+++ b/drivers/gpu/drm/i915/i915_gem.c
@@ -3765,6 +3765,10 @@ int i915_gpu_idle(struct drm_device *dev)
/* Flush everything onto the inactive list. */
for_each_ring(ring, dev_priv, i) {
+ ret = i915_scheduler_flush(ring, true);
+ if (ret < 0)
+ return ret;
+
if (!i915.enable_execlists) {
struct drm_i915_gem_request *req;
@@ -4478,7 +4482,8 @@ i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file)
unsigned long recent_enough = jiffies - DRM_I915_THROTTLE_JIFFIES;
struct drm_i915_gem_request *request, *target = NULL;
unsigned reset_counter;
- int ret;
+ int i, ret;
+ struct intel_engine_cs *ring;
ret = i915_gem_wait_for_error(&dev_priv->gpu_error);
if (ret)
@@ -4488,6 +4493,23 @@ i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file)
if (ret)
return ret;
+ for_each_ring(ring, dev_priv, i) {
+ /*
+ * Flush out scheduler entries that are getting 'stale'. Note
+ * that the following recent_enough test will only check
+ * against the time at which the request was submitted to the
+ * hardware (i.e. when it left the scheduler) not the time it
+ * was submitted to the driver.
+ *
+ * Also, there is not much point worring about busy return
+ * codes from the scheduler flush call. Even if more work
+ * cannot be submitted right now for whatever reason, we
+ * still want to throttle against stale work that has already
+ * been submitted.
+ */
+ i915_scheduler_flush_stamp(ring, recent_enough, false);
+ }
+
spin_lock(&file_priv->mm.lock);
list_for_each_entry(request, &file_priv->mm.request_list, client_list) {
if (time_after_eq(request->emitted_jiffies, recent_enough))
diff --git a/drivers/gpu/drm/i915/i915_scheduler.c b/drivers/gpu/drm/i915/i915_scheduler.c
index 386f157..c13dbc3 100644
--- a/drivers/gpu/drm/i915/i915_scheduler.c
+++ b/drivers/gpu/drm/i915/i915_scheduler.c
@@ -31,6 +31,8 @@ static int i915_scheduler_remove_dependent(struct i915_scheduler *schedu
struct i915_scheduler_queue_entry *remove);
static int i915_scheduler_submit(struct intel_engine_cs *ring,
bool is_locked);
+static int i915_scheduler_submit_max_priority(struct intel_engine_cs *ring,
+ bool is_locked);
static uint32_t i915_scheduler_count_flying(struct i915_scheduler *scheduler,
struct intel_engine_cs *ring);
static void i915_scheduler_priority_bump_clear(struct i915_scheduler *scheduler);
@@ -580,6 +582,98 @@ void i915_gem_scheduler_work_handler(struct work_struct *work)
}
}
+int i915_scheduler_flush_stamp(struct intel_engine_cs *ring,
+ unsigned long target,
+ bool is_locked)
+{
+ struct i915_scheduler_queue_entry *node;
+ struct drm_i915_private *dev_priv;
+ struct i915_scheduler *scheduler;
+ unsigned long flags;
+ int flush_count = 0;
+
+ if (!ring)
+ return -EINVAL;
+
+ dev_priv = ring->dev->dev_private;
+ scheduler = dev_priv->scheduler;
+
+ if (!scheduler)
+ return 0;
+
+ if (is_locked && (scheduler->flags[ring->id] & i915_sf_submitting)) {
+ /* Scheduler is busy already submitting another batch,
+ * come back later rather than going recursive... */
+ return -EAGAIN;
+ }
+
+ spin_lock_irqsave(&scheduler->lock, flags);
+ i915_scheduler_priority_bump_clear(scheduler);
+ list_for_each_entry(node, &scheduler->node_queue[ring->id], link) {
+ if (!I915_SQS_IS_QUEUED(node))
+ continue;
+
+ if (node->stamp > target)
+ continue;
+
+ flush_count = i915_scheduler_priority_bump(scheduler,
+ node, scheduler->priority_level_max);
+ }
+ spin_unlock_irqrestore(&scheduler->lock, flags);
+
+ if (flush_count) {
+ DRM_DEBUG_DRIVER("<%s> Bumped %d entries\n", ring->name, flush_count);
+ flush_count = i915_scheduler_submit_max_priority(ring, is_locked);
+ }
+
+ return flush_count;
+}
+
+int i915_scheduler_flush(struct intel_engine_cs *ring, bool is_locked)
+{
+ struct i915_scheduler_queue_entry *node;
+ struct drm_i915_private *dev_priv;
+ struct i915_scheduler *scheduler;
+ unsigned long flags;
+ bool found;
+ int ret;
+ uint32_t count = 0;
+
+ if (!ring)
+ return -EINVAL;
+
+ dev_priv = ring->dev->dev_private;
+ scheduler = dev_priv->scheduler;
+
+ if (!scheduler)
+ return 0;
+
+ BUG_ON(is_locked && (scheduler->flags[ring->id] & i915_sf_submitting));
+
+ do {
+ found = false;
+ spin_lock_irqsave(&scheduler->lock, flags);
+ list_for_each_entry(node, &scheduler->node_queue[ring->id], link) {
+ if (!I915_SQS_IS_QUEUED(node))
+ continue;
+
+ found = true;
+ break;
+ }
+ spin_unlock_irqrestore(&scheduler->lock, flags);
+
+ if (found) {
+ ret = i915_scheduler_submit(ring, is_locked);
+ if (ret < 0)
+ return ret;
+
+ count += ret;
+ }
+ } while (found);
+
+ return count;
+}
+
static void i915_scheduler_priority_bump_clear(struct i915_scheduler *scheduler)
{
struct i915_scheduler_queue_entry *node;
@@ -633,6 +727,44 @@ static int i915_scheduler_priority_bump(struct i915_scheduler *scheduler,
return count;
}
+static int i915_scheduler_submit_max_priority(struct intel_engine_cs *ring,
+ bool is_locked)
+{
+ struct i915_scheduler_queue_entry *node;
+ struct drm_i915_private *dev_priv = ring->dev->dev_private;
+ struct i915_scheduler *scheduler = dev_priv->scheduler;
+ unsigned long flags;
+ int ret, count = 0;
+ bool found;
+
+ do {
+ found = false;
+ spin_lock_irqsave(&scheduler->lock, flags);
+ list_for_each_entry(node, &scheduler->node_queue[ring->id], link) {
+ if (!I915_SQS_IS_QUEUED(node))
+ continue;
+
+ if (node->priority < scheduler->priority_level_max)
+ continue;
+
+ found = true;
+ break;
+ }
+ spin_unlock_irqrestore(&scheduler->lock, flags);
+
+ if (!found)
+ break;
+
+ ret = i915_scheduler_submit(ring, is_locked);
+ if (ret < 0)
+ return ret;
+
+ count += ret;
+ } while (found);
+
+ return count;
+}
+
static int i915_scheduler_pop_from_queue_locked(struct intel_engine_cs *ring,
struct i915_scheduler_queue_entry **pop_node,
unsigned long *flags)
diff --git a/drivers/gpu/drm/i915/i915_scheduler.h b/drivers/gpu/drm/i915/i915_scheduler.h
index c3e7ac6..7654013 100644
--- a/drivers/gpu/drm/i915/i915_scheduler.h
+++ b/drivers/gpu/drm/i915/i915_scheduler.h
@@ -93,6 +93,9 @@ int i915_scheduler_queue_execbuffer(struct i915_scheduler_queue_entry *q
bool i915_scheduler_notify_request(struct drm_i915_gem_request *req);
void i915_scheduler_wakeup(struct drm_device *dev);
void i915_gem_scheduler_work_handler(struct work_struct *work);
+int i915_scheduler_flush(struct intel_engine_cs *ring, bool is_locked);
+int i915_scheduler_flush_stamp(struct intel_engine_cs *ring,
+ unsigned long stamp, bool is_locked);
bool i915_scheduler_is_request_tracked(struct drm_i915_gem_request *req,
bool *completed, bool *busy);
--
1.9.1
_______________________________________________
Intel-gfx mailing list
Intel-gfx@xxxxxxxxxxxxxxxxxxxxx
http://lists.freedesktop.org/mailman/listinfo/intel-gfx
