Currently, we only track the last request globally across all engines.
This prevents us from issuing concurrent read requests on e.g. the RCS
and BCS engines (or more likely the render and media engines). Without
semaphores, we incur costly stalls as we synchronise between rings -
greatly impacting the current performance of Broadwell versus Haswell in
certain workloads (like video decode). With the introduction of
reference counted requests, it is much easier to track the last request
per ring, as well as the last global write request so that we can
optimise inter-engine read read requests (as well as better optimise
certain CPU waits).
v2: Fix inverted readonly condition for nonblocking waits.
Benchmark: igt/gem_read_read_speed
hsw:gt3e (with semaphores):
Before: Time to read-read 1024k: 275.794µs
After: Time to read-read 1024k: 123.260µs
hsw:gt3e (w/o semaphores):
Before: Time to read-read 1024k: 230.433µs
After: Time to read-read 1024k: 124.593µs
Testcase: igt/gem_concurrent_blit
Cc: Lionel Landwerlin <lionel.g.landwerlin@xxxxxxxxxxxxxxx>
Signed-off-by: Chris Wilson <chris@xxxxxxxxxxxxxxxxxx>
---
drivers/gpu/drm/i915/i915_debugfs.c | 16 +-
drivers/gpu/drm/i915/i915_drv.h | 11 +-
drivers/gpu/drm/i915/i915_gem.c | 351 ++++++++++++++++++--------------
drivers/gpu/drm/i915/i915_gem_context.c | 2 -
drivers/gpu/drm/i915/i915_gpu_error.c | 19 +-
drivers/gpu/drm/i915/intel_display.c | 4 +-
drivers/gpu/drm/i915/intel_ringbuffer.h | 1 +
7 files changed, 228 insertions(+), 176 deletions(-)
diff --git a/drivers/gpu/drm/i915/i915_debugfs.c b/drivers/gpu/drm/i915/i915_debugfs.c
index ad4fc2bf70b6..aacada84b537 100644
--- a/drivers/gpu/drm/i915/i915_debugfs.c
+++ b/drivers/gpu/drm/i915/i915_debugfs.c
@@ -120,10 +120,13 @@ static inline const char *get_global_flag(struct drm_i915_gem_object *obj)
static void
describe_obj(struct seq_file *m, struct drm_i915_gem_object *obj)
{
+ struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
+ struct intel_engine_cs *ring;
struct i915_vma *vma;
int pin_count = 0;
+ int i;
- seq_printf(m, "%pK: %s%s%s%s %8zdKiB %02x %02x %x %x %x%s%s%s",
+ seq_printf(m, "%pK: %s%s%s%s %8zdKiB %02x %02x [ ",
&obj->base,
obj->active ? "*" : " ",
get_pin_flag(obj),
@@ -131,8 +134,11 @@ describe_obj(struct seq_file *m, struct drm_i915_gem_object *obj)
get_global_flag(obj),
obj->base.size / 1024,
obj->base.read_domains,
- obj->base.write_domain,
- i915_gem_request_get_seqno(obj->last_read_req),
+ obj->base.write_domain);
+ for_each_ring(ring, dev_priv, i)
+ seq_printf(m, "%x ",
+ i915_gem_request_get_seqno(obj->last_read_req[i]));
+ seq_printf(m, "] %x %x%s%s%s",
i915_gem_request_get_seqno(obj->last_write_req),
i915_gem_request_get_seqno(obj->last_fenced_req),
i915_cache_level_str(to_i915(obj->base.dev), obj->cache_level),
@@ -169,9 +175,9 @@ describe_obj(struct seq_file *m, struct drm_i915_gem_object *obj)
*t = '\0';
seq_printf(m, " (%s mappable)", s);
}
- if (obj->last_read_req != NULL)
+ if (obj->last_write_req != NULL)
seq_printf(m, " (%s)",
- i915_gem_request_get_ring(obj->last_read_req)->name);
+ i915_gem_request_get_ring(obj->last_write_req)->name);
if (obj->frontbuffer_bits)
seq_printf(m, " (frontbuffer: 0x%03x)", obj->frontbuffer_bits);
}
diff --git a/drivers/gpu/drm/i915/i915_drv.h b/drivers/gpu/drm/i915/i915_drv.h
index a0f7784be589..a5faf669f7bb 100644
--- a/drivers/gpu/drm/i915/i915_drv.h
+++ b/drivers/gpu/drm/i915/i915_drv.h
@@ -498,7 +498,7 @@ struct drm_i915_error_state {
struct drm_i915_error_buffer {
u32 size;
u32 name;
- u32 rseqno, wseqno;
+ u32 rseqno[I915_NUM_RINGS], wseqno;
u32 gtt_offset;
u32 read_domains;
u32 write_domain;
@@ -1903,7 +1903,7 @@ struct drm_i915_gem_object {
struct drm_mm_node *stolen;
struct list_head global_list;
- struct list_head ring_list;
+ struct list_head ring_list[I915_NUM_RINGS];
/** Used in execbuf to temporarily hold a ref */
struct list_head obj_exec_link;
@@ -1914,7 +1914,7 @@ struct drm_i915_gem_object {
* rendering and so a non-zero seqno), and is not set if it i s on
* inactive (ready to be unbound) list.
*/
- unsigned int active:1;
+ unsigned int active:I915_MAX_RING_BIT;
/**
* This is set if the object has been written to since last bound
@@ -1986,7 +1986,7 @@ struct drm_i915_gem_object {
int vmapping_count;
/** Breadcrumb of last rendering to the buffer. */
- struct drm_i915_gem_request *last_read_req;
+ struct drm_i915_gem_request *last_read_req[I915_NUM_RINGS];
struct drm_i915_gem_request *last_write_req;
/** Breadcrumb of last fenced GPU access to the buffer. */
struct drm_i915_gem_request *last_fenced_req;
@@ -2124,10 +2124,11 @@ i915_gem_request_get_ring(struct drm_i915_gem_request *req)
return req ? req->ring : NULL;
}
-static inline void
+static inline struct drm_i915_gem_request *
i915_gem_request_reference(struct drm_i915_gem_request *req)
{
kref_get(&req->ref);
+ return req;
}
static inline void
diff --git a/drivers/gpu/drm/i915/i915_gem.c b/drivers/gpu/drm/i915/i915_gem.c
index 9cae7c7cd562..3ab8dea23734 100644
--- a/drivers/gpu/drm/i915/i915_gem.c
+++ b/drivers/gpu/drm/i915/i915_gem.c
@@ -41,12 +41,12 @@
static void i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj);
static void i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj);
+static void
+i915_gem_object_move_to_inactive(struct drm_i915_gem_object *obj,
+ int ring);
static __must_check int
i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj,
bool readonly);
-static void
-i915_gem_object_retire(struct drm_i915_gem_object *obj);
-
static void i915_gem_write_fence(struct drm_device *dev, int reg,
struct drm_i915_gem_object *obj);
static void i915_gem_object_update_fence(struct drm_i915_gem_object *obj,
@@ -528,8 +528,6 @@ int i915_gem_obj_prepare_shmem_read(struct drm_i915_gem_object *obj,
ret = i915_gem_object_wait_rendering(obj, true);
if (ret)
return ret;
-
- i915_gem_object_retire(obj);
}
ret = i915_gem_object_get_pages(obj);
@@ -949,8 +947,6 @@ i915_gem_shmem_pwrite(struct drm_device *dev,
ret = i915_gem_object_wait_rendering(obj, false);
if (ret)
return ret;
-
- i915_gem_object_retire(obj);
}
/* Same trick applies to invalidate partially written cachelines read
* before writing. */
@@ -1389,24 +1385,6 @@ i915_wait_request(struct drm_i915_gem_request *req)
return ret;
}
-static int
-i915_gem_object_wait_rendering__tail(struct drm_i915_gem_object *obj)
-{
- if (!obj->active)
- return 0;
-
- /* Manually manage the write flush as we may have not yet
- * retired the buffer.
- *
- * Note that the last_write_req is always the earlier of
- * the two (read/write) requests, so if we haved successfully waited,
- * we know we have passed the last write.
- */
- i915_gem_request_assign(&obj->last_write_req, NULL);
-
- return 0;
-}
-
/**
* Ensures that all rendering to the object has completed and the object is
* safe to unbind from the GTT or access from the CPU.
@@ -1415,18 +1393,38 @@ static __must_check int
i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj,
bool readonly)
{
- struct drm_i915_gem_request *req;
- int ret;
+ int ret, i;
- req = readonly ? obj->last_write_req : obj->last_read_req;
- if (!req)
+ if (!obj->active)
return 0;
- ret = i915_wait_request(req);
- if (ret)
- return ret;
+ if (readonly) {
+ if (obj->last_write_req != NULL) {
+ ret = i915_wait_request(obj->last_write_req);
+ if (ret)
+ return ret;
+
+ i = obj->last_write_req->ring->id;
+ if (obj->last_read_req[i] == obj->last_write_req)
+ i915_gem_object_move_to_inactive(obj, i);
+
+ i915_gem_request_assign(&obj->last_write_req, NULL);
+ }
+ } else {
+ for (i = 0; i < I915_NUM_RINGS; i++) {
+ if (obj->last_read_req[i] == NULL)
+ continue;
+
+ ret = i915_wait_request(obj->last_read_req[i]);
+ if (ret)
+ return ret;
+
+ i915_gem_object_move_to_inactive(obj, i);
+ }
+ }
+
+ return 0;
- return i915_gem_object_wait_rendering__tail(obj);
}
/* A nonblocking variant of the above wait. This is a highly dangerous routine
@@ -1437,37 +1435,63 @@ i915_gem_object_wait_rendering__nonblocking(struct drm_i915_gem_object *obj,
struct drm_i915_file_private *file_priv,
bool readonly)
{
- struct drm_i915_gem_request *req;
struct drm_device *dev = obj->base.dev;
struct drm_i915_private *dev_priv = dev->dev_private;
+ struct drm_i915_gem_request *req[I915_NUM_RINGS];
unsigned reset_counter;
- int ret;
+ int ret, i, n = 0;
BUG_ON(!mutex_is_locked(&dev->struct_mutex));
BUG_ON(!dev_priv->mm.interruptible);
- req = readonly ? obj->last_write_req : obj->last_read_req;
- if (!req)
+ if (!obj->active)
return 0;
ret = i915_gem_check_wedge(&dev_priv->gpu_error, true);
if (ret)
return ret;
- ret = i915_gem_check_olr(req);
- if (ret)
- return ret;
-
reset_counter = atomic_read(&dev_priv->gpu_error.reset_counter);
- i915_gem_request_reference(req);
+
+ if (readonly) {
+ if (obj->last_write_req == NULL)
+ return 0;
+
+ req[n++] = i915_gem_request_reference(obj->last_write_req);
+
+ ret = i915_gem_check_olr(req[n-1]);
+ if (ret)
+ goto err;
+ } else {
+ for (i = 0; i < I915_NUM_RINGS; i++) {
+ if (obj->last_read_req[i] == NULL)
+ continue;
+
+ req[n++] = i915_gem_request_reference(obj->last_read_req[i]);
+ ret = i915_gem_check_olr(req[n-1]);
+ if (ret)
+ goto err;
+ }
+ }
+
mutex_unlock(&dev->struct_mutex);
- ret = __i915_wait_request(req, reset_counter, true, NULL, file_priv);
+ for (i = 0; ret == 0 && i < n; i++)
+ ret = __i915_wait_request(req[i], reset_counter, true,
+ NULL, file_priv);
mutex_lock(&dev->struct_mutex);
- i915_gem_request_unreference(req);
- if (ret)
- return ret;
- return i915_gem_object_wait_rendering__tail(obj);
+err:
+ for (i = 0; i < n; i++) {
+ if (ret == 0) {
+ if (obj->last_read_req[i] == req[i])
+ i915_gem_object_move_to_inactive(obj, i);
+ if (obj->last_write_req == req[i])
+ i915_gem_request_assign(&obj->last_write_req, NULL);
+ }
+ i915_gem_request_unreference(req[i]);
+ }
+
+ return ret;
}
/**
@@ -2303,49 +2327,44 @@ i915_gem_object_get_pages(struct drm_i915_gem_object *obj)
return 0;
}
-static void
-i915_gem_object_move_to_active(struct drm_i915_gem_object *obj,
- struct intel_engine_cs *ring)
+void i915_vma_move_to_active(struct i915_vma *vma,
+ struct intel_engine_cs *ring)
{
- struct drm_i915_gem_request *req;
- struct intel_engine_cs *old_ring;
-
- BUG_ON(ring == NULL);
-
- req = intel_ring_get_request(ring);
- old_ring = i915_gem_request_get_ring(obj->last_read_req);
+ struct drm_i915_gem_object *obj = vma->obj;
+ struct drm_i915_gem_request *req = intel_ring_get_request(ring);
- if (old_ring != ring && obj->last_write_req) {
- /* Keep the request relative to the current ring */
- i915_gem_request_assign(&obj->last_write_req, req);
- }
+ list_move_tail(&vma->mm_list, &vma->vm->active_list);
/* Add a reference if we're newly entering the active list. */
- if (!obj->active) {
+ if (obj->last_read_req[ring->id] == NULL && obj->active++ == 0)
drm_gem_object_reference(&obj->base);
- obj->active = 1;
- }
- list_move_tail(&obj->ring_list, &ring->active_list);
-
- i915_gem_request_assign(&obj->last_read_req, req);
-}
-
-void i915_vma_move_to_active(struct i915_vma *vma,
- struct intel_engine_cs *ring)
-{
- list_move_tail(&vma->mm_list, &vma->vm->active_list);
- return i915_gem_object_move_to_active(vma->obj, ring);
+ list_move_tail(&obj->ring_list[ring->id], &ring->active_list);
+ i915_gem_request_assign(&obj->last_read_req[ring->id], req);
}
static void
-i915_gem_object_move_to_inactive(struct drm_i915_gem_object *obj)
+i915_gem_object_move_to_inactive(struct drm_i915_gem_object *obj,
+ int ring)
{
struct i915_vma *vma;
- BUG_ON(obj->base.write_domain & ~I915_GEM_GPU_DOMAINS);
+ WARN_ONCE(obj->base.write_domain & ~I915_GEM_GPU_DOMAINS,
+ "invalid write domains: %x\n",
+ obj->base.write_domain & ~I915_GEM_GPU_DOMAINS);
BUG_ON(!obj->active);
+ list_del_init(&obj->ring_list[ring]);
+ i915_gem_request_assign(&obj->last_read_req[ring], NULL);
+
+ if (obj->last_write_req && obj->last_write_req->ring->id == ring) {
+ i915_gem_request_assign(&obj->last_write_req, NULL);
+ obj->base.write_domain = 0;
+ }
+
+ if (--obj->active)
+ return;
+
list_for_each_entry(vma, &obj->vma_list, vma_link) {
if (!list_empty(&vma->mm_list))
list_move_tail(&vma->mm_list, &vma->vm->inactive_list);
@@ -2353,30 +2372,12 @@ i915_gem_object_move_to_inactive(struct drm_i915_gem_object *obj)
intel_fb_obj_flush(obj, true);
- list_del_init(&obj->ring_list);
-
- i915_gem_request_assign(&obj->last_read_req, NULL);
- i915_gem_request_assign(&obj->last_write_req, NULL);
- obj->base.write_domain = 0;
-
i915_gem_request_assign(&obj->last_fenced_req, NULL);
- obj->active = 0;
drm_gem_object_unreference(&obj->base);
-
WARN_ON(i915_verify_lists(dev));
}
-static void
-i915_gem_object_retire(struct drm_i915_gem_object *obj)
-{
- if (obj->last_read_req == NULL)
- return;
-
- if (i915_gem_request_completed(obj->last_read_req, true))
- i915_gem_object_move_to_inactive(obj);
-}
-
static int
i915_gem_init_seqno(struct drm_device *dev, u32 seqno)
{
@@ -2682,9 +2683,9 @@ static void i915_gem_reset_ring_cleanup(struct drm_i915_private *dev_priv,
obj = list_first_entry(&ring->active_list,
struct drm_i915_gem_object,
- ring_list);
+ ring_list[ring->id]);
- i915_gem_object_move_to_inactive(obj);
+ i915_gem_object_move_to_inactive(obj, ring->id);
}
/*
@@ -2791,12 +2792,13 @@ i915_gem_retire_requests_ring(struct intel_engine_cs *ring)
obj = list_first_entry(&ring->active_list,
struct drm_i915_gem_object,
- ring_list);
+ ring_list[ring->id]);
- if (!i915_gem_request_completed(obj->last_read_req, true))
+ if (!i915_gem_request_completed(obj->last_read_req[ring->id],
+ true))
break;
- i915_gem_object_move_to_inactive(obj);
+ i915_gem_object_move_to_inactive(obj, ring->id);
}
@@ -2908,17 +2910,20 @@ i915_gem_idle_work_handler(struct work_struct *work)
static int
i915_gem_object_flush_active(struct drm_i915_gem_object *obj)
{
- struct intel_engine_cs *ring;
- int ret;
+ int ret, i;
- if (obj->active) {
- ring = i915_gem_request_get_ring(obj->last_read_req);
+ if (!obj->active)
+ return 0;
+
+ for (i = 0; i < I915_NUM_RINGS; i++) {
+ if (obj->last_read_req[i] == NULL)
+ continue;
- ret = i915_gem_check_olr(obj->last_read_req);
+ ret = i915_gem_check_olr(obj->last_read_req[i]);
if (ret)
return ret;
- i915_gem_retire_requests_ring(ring);
+ i915_gem_retire_requests_ring(&to_i915(obj->base.dev)->ring[i]);
}
return 0;
@@ -2952,9 +2957,10 @@ i915_gem_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
struct drm_i915_private *dev_priv = dev->dev_private;
struct drm_i915_gem_wait *args = data;
struct drm_i915_gem_object *obj;
- struct drm_i915_gem_request *req;
+ struct drm_i915_gem_request *req[I915_NUM_RINGS];
unsigned reset_counter;
- int ret = 0;
+ int i, n = 0;
+ int ret;
if (args->flags != 0)
return -EINVAL;
@@ -2974,11 +2980,9 @@ i915_gem_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
if (ret)
goto out;
- if (!obj->active || !obj->last_read_req)
+ if (!obj->active)
goto out;
- req = obj->last_read_req;
-
/* Do this after OLR check to make sure we make forward progress polling
* on this IOCTL with a timeout == 0 (like busy ioctl)
*/
@@ -2989,13 +2993,23 @@ i915_gem_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
drm_gem_object_unreference(&obj->base);
reset_counter = atomic_read(&dev_priv->gpu_error.reset_counter);
- i915_gem_request_reference(req);
+
+ for (i = 0; i < I915_NUM_RINGS; i++) {
+ if (obj->last_read_req[i] == NULL)
+ continue;
+
+ req[n++] = i915_gem_request_reference(obj->last_read_req[i]);
+ }
+
mutex_unlock(&dev->struct_mutex);
- ret = __i915_wait_request(req, reset_counter, true,
- args->timeout_ns > 0 ? &args->timeout_ns : NULL,
- file->driver_priv);
- i915_gem_request_unreference__unlocked(req);
+ for (i = 0; i < n; i++) {
+ if (ret == 0)
+ ret = __i915_wait_request(req[i], reset_counter, true,
+ args->timeout_ns > 0 ? &args->timeout_ns : NULL,
+ file->driver_priv);
+ i915_gem_request_unreference__unlocked(req[i]);
+ }
return ret;
out:
@@ -3004,55 +3018,80 @@ out:
return ret;
}
-/**
- * i915_gem_object_sync - sync an object to a ring.
- *
- * @obj: object which may be in use on another ring.
- * @to: ring we wish to use the object on. May be NULL.
- *
- * This code is meant to abstract object synchronization with the GPU.
- * Calling with NULL implies synchronizing the object with the CPU
- * rather than a particular GPU ring.
- *
- * Returns 0 if successful, else propagates up the lower layer error.
- */
-int
-i915_gem_object_sync(struct drm_i915_gem_object *obj,
- struct intel_engine_cs *to)
+static int
+__i915_gem_object_sync(struct drm_i915_gem_object *obj,
+ struct intel_engine_cs *to,
+ struct drm_i915_gem_request *rq)
{
- struct intel_engine_cs *from;
+ struct intel_engine_cs *from = i915_gem_request_get_ring(rq);
u32 seqno;
int ret, idx;
- from = i915_gem_request_get_ring(obj->last_read_req);
-
if (from == NULL || to == from)
return 0;
+ if (i915_gem_request_completed(rq, true))
+ return 0;
+
if (to == NULL || !i915_semaphore_is_enabled(obj->base.dev))
- return i915_gem_object_wait_rendering(obj, false);
+ return i915_gem_object_wait_rendering(obj, obj->base.pending_write_domain == 0);
idx = intel_ring_sync_index(from, to);
- seqno = i915_gem_request_get_seqno(obj->last_read_req);
+ seqno = i915_gem_request_get_seqno(rq);
/* Optimization: Avoid semaphore sync when we are sure we already
* waited for an object with higher seqno */
if (seqno <= from->semaphore.sync_seqno[idx])
return 0;
- ret = i915_gem_check_olr(obj->last_read_req);
+ ret = i915_gem_check_olr(rq);
if (ret)
return ret;
- trace_i915_gem_ring_sync_to(from, to, obj->last_read_req);
+ trace_i915_gem_ring_sync_to(from, to, rq);
ret = to->semaphore.sync_to(to, from, seqno);
if (!ret)
/* We use last_read_req because sync_to()
* might have just caused seqno wrap under
* the radar.
*/
- from->semaphore.sync_seqno[idx] =
- i915_gem_request_get_seqno(obj->last_read_req);
+ from->semaphore.sync_seqno[idx] = i915_gem_request_get_seqno(rq);
+
+ return ret;
+}
+
+/**
+ * i915_gem_object_sync - sync an object to a ring.
+ *
+ * @obj: object which may be in use on another ring.
+ * @to: ring we wish to use the object on. May be NULL.
+ *
+ * This code is meant to abstract object synchronization with the GPU.
+ * Calling with NULL implies synchronizing the object with the CPU
+ * rather than a particular GPU ring.
+ *
+ * Returns 0 if successful, else propagates up the lower layer error.
+ */
+int
+i915_gem_object_sync(struct drm_i915_gem_object *obj,
+ struct intel_engine_cs *to)
+{
+ int ret = 0, i;
+
+ if (obj->base.pending_write_domain == 0) {
+ if (obj->last_write_req)
+ ret = __i915_gem_object_sync(obj, to,
+ obj->last_write_req);
+ } else {
+ for (i = 0; i < I915_NUM_RINGS; i++) {
+ if (obj->last_read_req[i] == NULL)
+ continue;
+
+ ret = __i915_gem_object_sync(obj, to, obj->last_read_req[i]);
+ if (ret)
+ break;
+ }
+ }
return ret;
}
@@ -3139,10 +3178,6 @@ int i915_vma_unbind(struct i915_vma *vma)
/* Since the unbound list is global, only move to that list if
* no more VMAs exist. */
if (list_empty(&obj->vma_list)) {
- /* Throw away the active reference before
- * moving to the unbound list. */
- i915_gem_object_retire(obj);
-
i915_gem_gtt_finish_object(obj);
list_move_tail(&obj->global_list, &dev_priv->mm.unbound_list);
}
@@ -3792,8 +3827,6 @@ i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write)
if (ret)
return ret;
- i915_gem_object_retire(obj);
-
/* Flush and acquire obj->pages so that we are coherent through
* direct access in memory with previous cached writes through
* shmemfs and that our cache domain tracking remains valid.
@@ -4018,11 +4051,9 @@ i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj,
bool was_pin_display;
int ret;
- if (pipelined != i915_gem_request_get_ring(obj->last_read_req)) {
- ret = i915_gem_object_sync(obj, pipelined);
- if (ret)
- return ret;
- }
+ ret = i915_gem_object_sync(obj, pipelined);
+ if (ret)
+ return ret;
/* Mark the pin_display early so that we account for the
* display coherency whilst setting up the cache domains.
@@ -4118,7 +4149,6 @@ i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write)
if (ret)
return ret;
- i915_gem_object_retire(obj);
i915_gem_object_flush_gtt_write_domain(obj);
old_write_domain = obj->base.write_domain;
@@ -4385,11 +4415,17 @@ i915_gem_busy_ioctl(struct drm_device *dev, void *data,
* necessary flushes here.
*/
ret = i915_gem_object_flush_active(obj);
- if (obj->last_read_req) {
- struct intel_engine_cs *ring;
+ if (obj->active) {
+ int i;
+
BUILD_BUG_ON(I915_NUM_RINGS > 16);
- ring = i915_gem_request_get_ring(obj->last_read_req);
- args->busy = intel_ring_flag(ring) << 16 | 1;
+
+ for (i = 0; i < I915_NUM_RINGS; i++) {
+ if (obj->last_read_req[i] == NULL)
+ continue;
+
+ args->busy |= 1 << (16 + i) | 1;
+ }
}
drm_gem_object_unreference(&obj->base);
@@ -4468,8 +4504,11 @@ unlock:
void i915_gem_object_init(struct drm_i915_gem_object *obj,
const struct drm_i915_gem_object_ops *ops)
{
+ int i;
+
INIT_LIST_HEAD(&obj->global_list);
- INIT_LIST_HEAD(&obj->ring_list);
+ for (i = 0; i < I915_NUM_RINGS; i++)
+ INIT_LIST_HEAD(&obj->ring_list[i]);
INIT_LIST_HEAD(&obj->obj_exec_link);
INIT_LIST_HEAD(&obj->vma_list);
INIT_LIST_HEAD(&obj->batch_pool_link);
diff --git a/drivers/gpu/drm/i915/i915_gem_context.c b/drivers/gpu/drm/i915/i915_gem_context.c
index 70346b0028f9..31826e7e904c 100644
--- a/drivers/gpu/drm/i915/i915_gem_context.c
+++ b/drivers/gpu/drm/i915/i915_gem_context.c
@@ -673,8 +673,6 @@ static int do_switch(struct intel_engine_cs *ring,
* swapped, but there is no way to do that yet.
*/
from->legacy_hw_ctx.rcs_state->dirty = 1;
- BUG_ON(i915_gem_request_get_ring(
- from->legacy_hw_ctx.rcs_state->last_read_req) != ring);
/* obj is kept alive until the next request by its active ref */
i915_gem_object_ggtt_unpin(from->legacy_hw_ctx.rcs_state);
diff --git a/drivers/gpu/drm/i915/i915_gpu_error.c b/drivers/gpu/drm/i915/i915_gpu_error.c
index a982849a5edd..fa5d53f1240f 100644
--- a/drivers/gpu/drm/i915/i915_gpu_error.c
+++ b/drivers/gpu/drm/i915/i915_gpu_error.c
@@ -192,15 +192,20 @@ static void print_error_buffers(struct drm_i915_error_state_buf *m,
struct drm_i915_error_buffer *err,
int count)
{
+ int i;
+
err_printf(m, " %s [%d]:\n", name, count);
while (count--) {
- err_printf(m, " %08x %8u %02x %02x %x %x",
+ err_printf(m, " %08x %8u %02x %02x [ ",
err->gtt_offset,
err->size,
err->read_domains,
- err->write_domain,
- err->rseqno, err->wseqno);
+ err->write_domain);
+ for (i = 0; i < I915_NUM_RINGS; i++)
+ err_printf(m, "%02x ", err->rseqno[i]);
+
+ err_printf(m, "] %02x", err->wseqno);
err_puts(m, pin_flag(err->pinned));
err_puts(m, tiling_flag(err->tiling));
err_puts(m, dirty_flag(err->dirty));
@@ -667,10 +672,12 @@ static void capture_bo(struct drm_i915_error_buffer *err,
struct i915_vma *vma)
{
struct drm_i915_gem_object *obj = vma->obj;
+ int i;
err->size = obj->base.size;
err->name = obj->base.name;
- err->rseqno = i915_gem_request_get_seqno(obj->last_read_req);
+ for (i = 0; i < I915_NUM_RINGS; i++)
+ err->rseqno[i] = i915_gem_request_get_seqno(obj->last_read_req[i]);
err->wseqno = i915_gem_request_get_seqno(obj->last_write_req);
err->gtt_offset = vma->node.start;
err->read_domains = obj->base.read_domains;
@@ -683,8 +690,8 @@ static void capture_bo(struct drm_i915_error_buffer *err,
err->dirty = obj->dirty;
err->purgeable = obj->madv != I915_MADV_WILLNEED;
err->userptr = obj->userptr.mm != NULL;
- err->ring = obj->last_read_req ?
- i915_gem_request_get_ring(obj->last_read_req)->id : -1;
+ err->ring = obj->last_write_req ?
+ i915_gem_request_get_ring(obj->last_write_req)->id : -1;
err->cache_level = obj->cache_level;
}
diff --git a/drivers/gpu/drm/i915/intel_display.c b/drivers/gpu/drm/i915/intel_display.c
index d36ec6b8fd0a..d621ebecd33e 100644
--- a/drivers/gpu/drm/i915/intel_display.c
+++ b/drivers/gpu/drm/i915/intel_display.c
@@ -9674,7 +9674,7 @@ static bool use_mmio_flip(struct intel_engine_cs *ring,
else if (i915.enable_execlists)
return true;
else
- return ring != i915_gem_request_get_ring(obj->last_read_req);
+ return ring != i915_gem_request_get_ring(obj->last_write_req);
}
static void skl_do_mmio_flip(struct intel_crtc *intel_crtc)
@@ -9977,7 +9977,7 @@ static int intel_crtc_page_flip(struct drm_crtc *crtc,
} else if (IS_IVYBRIDGE(dev) || IS_HASWELL(dev)) {
ring = &dev_priv->ring[BCS];
} else if (INTEL_INFO(dev)->gen >= 7) {
- ring = i915_gem_request_get_ring(obj->last_read_req);
+ ring = i915_gem_request_get_ring(obj->last_write_req);
if (ring == NULL || ring->id != RCS)
ring = &dev_priv->ring[BCS];
} else {
diff --git a/drivers/gpu/drm/i915/intel_ringbuffer.h b/drivers/gpu/drm/i915/intel_ringbuffer.h
index 1d08d8f9149d..a4a25c932b0b 100644
--- a/drivers/gpu/drm/i915/intel_ringbuffer.h
+++ b/drivers/gpu/drm/i915/intel_ringbuffer.h
@@ -129,6 +129,7 @@ struct intel_engine_cs {
VCS2
} id;
#define I915_NUM_RINGS 5
+#define I915_MAX_RING_BIT 3
#define LAST_USER_RING (VECS + 1)
u32 mmio_base;
struct drm_device *dev;
--
2.1.4
_______________________________________________
Intel-gfx mailing list
Intel-gfx@xxxxxxxxxxxxxxxxxxxxx
http://lists.freedesktop.org/mailman/listinfo/intel-gfx
