From: "Siluvery, Arun" <arun.siluvery@xxxxxxxxx>
Instead of full GPU reset, where possible a single ring can be reset
individually. This patch adds functions to save ring's current state and
it will be restored with the same state after reset. The state comprises
of a set of ring specific registers. The actual hang detection
and recovery changes are in subsequent patches.
Signed-off-by: Siluvery, Arun <arun.siluvery@xxxxxxxxx>
---
drivers/gpu/drm/i915/i915_drv.h | 9 +-
drivers/gpu/drm/i915/i915_reg.h | 7 +
drivers/gpu/drm/i915/intel_ringbuffer.c | 595 ++++++++++++++++++++++++++++++++
drivers/gpu/drm/i915/intel_ringbuffer.h | 49 +++
drivers/gpu/drm/i915/intel_uncore.c | 25 ++
5 files changed, 681 insertions(+), 4 deletions(-)
diff --git a/drivers/gpu/drm/i915/i915_drv.h b/drivers/gpu/drm/i915/i915_drv.h
index b98a7c8..b0a244d 100644
--- a/drivers/gpu/drm/i915/i915_drv.h
+++ b/drivers/gpu/drm/i915/i915_drv.h
@@ -2423,6 +2423,7 @@ extern void intel_display_print_error_state(struct drm_i915_error_state_buf *e,
*/
void gen6_gt_force_wake_get(struct drm_i915_private *dev_priv);
void gen6_gt_force_wake_put(struct drm_i915_private *dev_priv);
+void gen6_gt_force_wake_restore(struct drm_device *dev);
int sandybridge_pcode_read(struct drm_i915_private *dev_priv, u8 mbox, u32 *val);
int sandybridge_pcode_write(struct drm_i915_private *dev_priv, u8 mbox, u32 val);
@@ -2456,13 +2457,13 @@ int vlv_freq_opcode(struct drm_i915_private *dev_priv, int val);
#define I915_READ16(reg) dev_priv->uncore.funcs.mmio_readw(dev_priv, (reg), true)
#define I915_WRITE16(reg, val) dev_priv->uncore.funcs.mmio_writew(dev_priv, (reg), (val), true)
-#define I915_READ16_NOTRACE(reg) dev_priv->uncore.funcs.mmio_readw(dev_priv, (reg), false)
-#define I915_WRITE16_NOTRACE(reg, val) dev_priv->uncore.funcs.mmio_writew(dev_priv, (reg), (val), false)
+#define I915_READ16_NOTRACE(reg) dev_priv->uncore.funcs.mmio_readw(dev_priv, (reg), false)
+#define I915_WRITE16_NOTRACE(reg, val) dev_priv->uncore.funcs.mmio_writew(dev_priv, (reg), (val), false)
#define I915_READ(reg) dev_priv->uncore.funcs.mmio_readl(dev_priv, (reg), true)
#define I915_WRITE(reg, val) dev_priv->uncore.funcs.mmio_writel(dev_priv, (reg), (val), true)
-#define I915_READ_NOTRACE(reg) dev_priv->uncore.funcs.mmio_readl(dev_priv, (reg), false)
-#define I915_WRITE_NOTRACE(reg, val) dev_priv->uncore.funcs.mmio_writel(dev_priv, (reg), (val), false)
+#define I915_READ_NOTRACE(reg) dev_priv->uncore.funcs.mmio_readl(dev_priv, (reg), false)
+#define I915_WRITE_NOTRACE(reg, val) dev_priv->uncore.funcs.mmio_writel(dev_priv, (reg), (val), false)
#define I915_WRITE64(reg, val) dev_priv->uncore.funcs.mmio_writeq(dev_priv, (reg), (val), true)
#define I915_READ64(reg) dev_priv->uncore.funcs.mmio_readq(dev_priv, (reg), true)
diff --git a/drivers/gpu/drm/i915/i915_reg.h b/drivers/gpu/drm/i915/i915_reg.h
index 849e595..02e6ec6 100644
--- a/drivers/gpu/drm/i915/i915_reg.h
+++ b/drivers/gpu/drm/i915/i915_reg.h
@@ -33,6 +33,7 @@
#define _MASKED_BIT_ENABLE(a) (((a) << 16) | (a))
#define _MASKED_BIT_DISABLE(a) ((a) << 16)
+#define _MASKED_BIT_ENABLE_ALL(a) (0xFFFF0000 | (a))
/* PCI config space */
@@ -104,6 +105,7 @@
#define GEN6_GRDOM_RENDER (1 << 1)
#define GEN6_GRDOM_MEDIA (1 << 2)
#define GEN6_GRDOM_BLT (1 << 3)
+#define GEN6_GRDOM_VEBOX (1 << 4)
#define RING_PP_DIR_BASE(ring) ((ring)->mmio_base+0x228)
#define RING_PP_DIR_BASE_READ(ring) ((ring)->mmio_base+0x518)
@@ -642,6 +644,8 @@
#define RING_SYNC_0(base) ((base)+0x40)
#define RING_SYNC_1(base) ((base)+0x44)
#define RING_SYNC_2(base) ((base)+0x48)
+#define RING_MI_MODE(base) ((base)+0x9c)
+#define RING_UHPTR(base) ((base)+0x134)
#define GEN6_RVSYNC (RING_SYNC_0(RENDER_RING_BASE))
#define GEN6_RBSYNC (RING_SYNC_1(RENDER_RING_BASE))
#define GEN6_RVESYNC (RING_SYNC_2(RENDER_RING_BASE))
@@ -789,6 +793,8 @@
#define MI_MODE 0x0209c
# define VS_TIMER_DISPATCH (1 << 6)
+# define MODE_STOP (1 << 8)
+# define MODE_IDLE (1 << 9)
# define MI_FLUSH_ENABLE (1 << 12)
# define ASYNC_FLIP_PERF_DISABLE (1 << 14)
@@ -799,6 +805,7 @@
#define GFX_MODE 0x02520
#define GFX_MODE_GEN7 0x0229c
#define RING_MODE_GEN7(ring) ((ring)->mmio_base+0x29c)
+#define RING_EXCC_GEN7(ring) ((ring)->mmio_base+0x028)
#define GFX_RUN_LIST_ENABLE (1<<15)
#define GFX_TLB_INVALIDATE_ALWAYS (1<<13)
#define GFX_SURFACE_FAULT_ENABLE (1<<12)
diff --git a/drivers/gpu/drm/i915/intel_ringbuffer.c b/drivers/gpu/drm/i915/intel_ringbuffer.c
index b620337..cce29d0 100644
--- a/drivers/gpu/drm/i915/intel_ringbuffer.c
+++ b/drivers/gpu/drm/i915/intel_ringbuffer.c
@@ -46,6 +46,7 @@ void __intel_ring_advance(struct intel_ring_buffer *ring)
struct drm_i915_private *dev_priv = ring->dev->dev_private;
ring->tail &= ring->size - 1;
+
if (dev_priv->gpu_error.stop_rings & intel_ring_flag(ring))
return;
ring->write_tail(ring, ring->tail);
@@ -428,6 +429,21 @@ static void ring_setup_phys_status_page(struct intel_ring_buffer *ring)
I915_WRITE(HWS_PGA, addr);
}
+void intel_ring_resample(struct intel_ring_buffer *ring)
+{
+ struct drm_device *dev = ring->dev;
+ drm_i915_private_t *dev_priv = dev->dev_private;
+
+ if (!drm_core_check_feature(ring->dev, DRIVER_MODESET))
+ i915_kernel_lost_context(ring->dev);
+ else {
+ ring->head = I915_READ_HEAD(ring);
+ ring->tail = I915_READ_TAIL(ring) & TAIL_ADDR;
+ ring->space = ring_space(ring);
+ ring->last_retired_head = -1;
+ }
+}
+
static int init_ring_common(struct intel_ring_buffer *ring)
{
struct drm_device *dev = ring->dev;
@@ -508,6 +524,9 @@ static int init_ring_common(struct intel_ring_buffer *ring)
memset(&ring->hangcheck, 0, sizeof(ring->hangcheck));
+ if (ring->invalidate_tlb)
+ ring->invalidate_tlb(ring);
+
out:
gen6_gt_force_wake_put(dev_priv);
@@ -1318,6 +1337,40 @@ static int init_phys_status_page(struct intel_ring_buffer *ring)
return 0;
}
+/* gen6_ring_invalidate_tlb
+ * GFX soft resets do not invalidate TLBs, it is up to
+ * GFX driver to explicitly invalidate TLBs post reset.
+ */
+static int gen6_ring_invalidate_tlb(struct intel_ring_buffer *ring)
+{
+ struct drm_device *dev = ring->dev;
+ drm_i915_private_t *dev_priv = dev->dev_private;
+ u32 reg;
+ int ret;
+
+ if ((INTEL_INFO(dev)->gen < 6) || (!ring->stop) || (!ring->start))
+ return -EINVAL;
+
+ /* stop the ring before sync_flush */
+ ret = ring->stop(ring);
+ if ((ret) && (ret != -EALREADY))
+ DRM_ERROR("%s: unable to stop the ring\n", ring->name);
+
+ /* Invalidate TLB */
+ reg = RING_INSTPM(ring->mmio_base);
+ I915_WRITE(reg, _MASKED_BIT_ENABLE(INSTPM_TLB_INVALIDATE |
+ INSTPM_SYNC_FLUSH));
+ if (wait_for((I915_READ(reg) & INSTPM_SYNC_FLUSH) == 0, 1000))
+ DRM_ERROR("%s: wait for SyncFlush to complete timed out\n",
+ ring->name);
+
+ /* only start if stop was sucessfull */
+ if (!ret)
+ ring->start(ring);
+
+ return 0;
+}
+
static int intel_init_ring_buffer(struct drm_device *dev,
struct intel_ring_buffer *ring)
{
@@ -1801,6 +1854,516 @@ gen6_ring_dispatch_execbuffer(struct intel_ring_buffer *ring,
return 0;
}
+static int
+gen6_ring_stop(struct intel_ring_buffer *ring)
+{
+ struct drm_device *dev = ring->dev;
+ drm_i915_private_t *dev_priv = dev->dev_private;
+
+ /* check if ring is already stopped */
+ if (I915_READ_MODE(ring) & MODE_STOP)
+ return -EALREADY;
+
+ /* Request the ring to go idle */
+ I915_WRITE_MODE(ring, _MASKED_BIT_ENABLE(MODE_STOP));
+
+ /* Wait for idle */
+ if (wait_for_atomic((I915_READ_MODE(ring) & MODE_IDLE) != 0, 1000)) {
+ DRM_ERROR("%s :timed out trying to stop ring", ring->name);
+ return -ETIMEDOUT;
+ }
+
+ return 0;
+}
+
+static int
+gen6_ring_start(struct intel_ring_buffer *ring)
+{
+ struct drm_device *dev = ring->dev;
+ drm_i915_private_t *dev_priv = dev->dev_private;
+ uint32_t mode;
+
+ /* Clear the MI_MODE stop bit */
+ I915_WRITE_MODE(ring, _MASKED_BIT_DISABLE(MODE_STOP));
+ mode = I915_READ_MODE(ring); /* Barrier read */
+
+ return 0;
+}
+
+int intel_ring_disable(struct intel_ring_buffer *ring)
+{
+ if (ring && ring->disable)
+ return ring->disable(ring);
+ else {
+ DRM_ERROR("ring disable not supported\n");
+ return -EINVAL;
+ }
+}
+
+static int
+gen6_ring_disable(struct intel_ring_buffer *ring)
+{
+ struct drm_device *dev = ring->dev;
+ drm_i915_private_t *dev_priv = dev->dev_private;
+ uint32_t ring_ctl;
+ uint32_t mi_mode;
+ uint32_t retries = 10000;
+
+ /* Request the ring to go idle */
+ I915_WRITE_MODE(ring, _MASKED_BIT_ENABLE(MODE_STOP));
+
+ /* Wait for idle */
+ do {
+ mi_mode = I915_READ_MODE(ring);
+ } while (retries-- && !(mi_mode & MODE_IDLE));
+
+ if (retries == 0) {
+ DRM_ERROR("timed out trying to disable ring %d\n", ring->id);
+ return -ETIMEDOUT;
+ }
+
+ /* Disable the ring */
+ ring_ctl = I915_READ_CTL(ring);
+ ring_ctl &= (RING_NR_PAGES | RING_REPORT_MASK);
+ I915_WRITE_CTL(ring, ring_ctl);
+ ring_ctl = I915_READ_CTL(ring); /* Barrier read */
+
+ return ((ring_ctl & RING_VALID) == 0) ? 0 : -EIO;
+}
+
+int intel_ring_enable(struct intel_ring_buffer *ring)
+{
+ if (ring && ring->enable)
+ return ring->enable(ring);
+ else {
+ DRM_ERROR("ring enable not supported\n");
+ return -EINVAL;
+ }
+}
+
+static int
+gen6_ring_enable(struct intel_ring_buffer *ring)
+{
+ struct drm_device *dev = ring->dev;
+ drm_i915_private_t *dev_priv = dev->dev_private;
+ uint32_t ring_ctl;
+ uint32_t mode;
+
+ /* Clear the MI_MODE stop bit */
+ I915_WRITE_MODE(ring, _MASKED_BIT_DISABLE(MODE_STOP));
+ mode = I915_READ_MODE(ring); /* Barrier read */
+
+ /* Enable the ring */
+ ring_ctl = I915_READ_CTL(ring);
+ ring_ctl &= (RING_NR_PAGES | RING_REPORT_MASK);
+ I915_WRITE_CTL(ring, ring_ctl | RING_VALID);
+ ring_ctl = I915_READ_CTL(ring); /* Barrier read */
+
+ return ((ring_ctl & RING_VALID) == 0) ? -EIO : 0;
+}
+
+
+/* raw read/write, no need for forcewake etc. */
+#define __raw_i915_read32(dev_priv__, reg__) readl((dev_priv__)->regs + (reg__))
+#define __raw_i915_write32(dev_priv__, reg__, val__) writel(val__, (dev_priv__)->regs + (reg__))
+
+int intel_ring_reset(struct intel_ring_buffer *ring)
+{
+ if (ring && ring->reset) {
+ int ret = ring->reset(ring);
+ /* invalidate TLB, if we got reset due to TLB giving
+ * stale addr (after soft reset) for HW status page,
+ * resulting in seqno not getting updated.
+ */
+ if (ring->invalidate_tlb)
+ ring->invalidate_tlb(ring);
+ return ret;
+ } else {
+ DRM_ERROR("ring reset not supported\n");
+ return -EINVAL;
+ }
+}
+
+static int
+gen6_ring_reset(struct intel_ring_buffer *ring)
+{
+ struct drm_device *dev = ring->dev;
+ drm_i915_private_t *dev_priv = dev->dev_private;
+ int ret = 0;
+ char *reset_event[2];
+ unsigned long irqflags;
+ reset_event[1] = NULL;
+
+ /* Hold uncore.lock across reset to prevent any register access
+ * with forcewake not set correctly
+ */
+ spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
+
+ switch (ring->id) {
+ case RCS:
+ /* GEN6_GDRST is not in the gt power well, no need to check
+ * for fifo space for the write or forcewake the chip for
+ * the read
+ */
+ __raw_i915_write32(dev_priv, GEN6_GDRST, GEN6_GRDOM_RENDER);
+
+ /* Spin waiting for the device to ack the reset request */
+ ret = wait_for((__raw_i915_read32(dev_priv, GEN6_GDRST)
+ & GEN6_GRDOM_RENDER) == 0, 500);
+ DRM_DEBUG("RCS Reset\n");
+ break;
+
+
+ case BCS:
+ __raw_i915_write32(dev_priv, GEN6_GDRST, GEN6_GRDOM_BLT);
+
+ /* Spin waiting for the device to ack the reset request */
+ ret = wait_for((__raw_i915_read32(dev_priv, GEN6_GDRST)
+ & GEN6_GRDOM_BLT) == 0, 500);
+ DRM_DEBUG("BCS Reset\n");
+ break;
+
+
+ case VCS:
+ __raw_i915_write32(dev_priv, GEN6_GDRST, GEN6_GRDOM_MEDIA);
+
+ /* Spin waiting for the device to ack the reset request */
+ ret = wait_for((__raw_i915_read32(dev_priv, GEN6_GDRST)
+ & GEN6_GRDOM_MEDIA) == 0, 500);
+ DRM_DEBUG("VCS Reset\n");
+ break;
+
+ case VECS:
+ __raw_i915_write32(dev_priv, GEN6_GDRST, GEN6_GRDOM_VEBOX);
+
+ /* Spin waiting for the device to ack the reset request */
+ ret = wait_for((__raw_i915_read32(dev_priv, GEN6_GDRST)
+ & GEN6_GRDOM_VEBOX) == 0, 500);
+ DRM_DEBUG("VECS Reset\n");
+ break;
+
+ default:
+ DRM_ERROR("Unexpected ring ID\n");
+ break;
+ }
+
+ /* Request power management to restore the power state based
+ * on the current reference count(s)*/
+ gen6_gt_force_wake_restore(dev);
+
+ spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
+
+ /* Do uevent outside of spinlock as uevent can sleep */
+ reset_event[0] = kasprintf(GFP_KERNEL, "RESET RING=%d", ring->id);
+ kobject_uevent_env(&dev->primary->kdev->kobj,
+ KOBJ_CHANGE, reset_event);
+ kfree(reset_event[0]);
+
+ return ret;
+}
+
+int intel_ring_save(struct intel_ring_buffer *ring, u32 flags)
+{
+ if (ring && ring->save)
+ return ring->save(ring, ring->saved_state,
+ I915_RING_CONTEXT_SIZE, flags);
+ else {
+ DRM_ERROR("ring save not supported\n");
+ return -EINVAL;
+ }
+}
+
+static int
+gen6_ring_save(struct intel_ring_buffer *ring, uint32_t *data, uint32_t max,
+ u32 flags)
+{
+ struct drm_device *dev = ring->dev;
+ drm_i915_private_t *dev_priv = dev->dev_private;
+ uint32_t idx = 0;
+ uint32_t gen = INTEL_INFO(dev)->gen;
+ uint32_t head;
+ uint32_t tail;
+ uint32_t head_addr;
+ uint32_t tail_addr;
+ int clamp_to_tail = 0;
+
+ /* Ring save only added for gen >= 7 */
+ WARN_ON(gen < 7);
+
+ /* Save common registers */
+ if (max < COMMON_RING_CTX_SIZE)
+ return -EINVAL;
+
+ head = I915_READ_HEAD(ring);
+ tail = I915_READ_TAIL(ring);
+
+ head_addr = head & HEAD_ADDR;
+ tail_addr = tail & TAIL_ADDR;
+
+ if (flags & FORCE_ADVANCE) {
+ /* The head must always chase the tail.
+ * If the tail is beyond the head then do not allow
+ * the head to overtake it. If the tail is less than
+ * the head then the tail has already wrapped and
+ * there is no problem in advancing the head or even
+ * wrapping the head back to 0 as worst case it will
+ * become equal to tail */
+ if (head_addr <= tail_addr)
+ clamp_to_tail = 1;
+
+ /* Force head to next QWORD boundary */
+ head_addr &= ~0x7;
+ head_addr += 8;
+
+ if (clamp_to_tail && (head_addr > tail_addr)) {
+ head_addr = tail_addr;
+ } else if (head_addr >= ring->size) {
+ /* Wrap head back to start if it exceeds ring size*/
+ head_addr = 0;
+ }
+
+ /* Update the register */
+ head &= ~HEAD_ADDR;
+ head |= (head_addr & HEAD_ADDR);
+
+ DRM_DEBUG("Forced head to 0x%08x\n", head);
+ } else if (head & 0x7) {
+ /* Ensure head pointer is pointing to a QWORD boundary */
+ DRM_DEBUG("Rounding up head 0x%08x\n", head);
+ head += 0x7;
+ head &= ~0x7;
+ }
+
+ /* Saved with enable = 0 */
+ data[idx++] = I915_READ_CTL(ring) & (RING_NR_PAGES | RING_REPORT_MASK);
+
+ data[idx++] = (flags & RESET_HEAD_TAIL) ? 0 : tail;
+
+
+ if (flags & RESET_HEAD_TAIL) {
+ /* Save head as 0 so head is reset on restore */
+ data[idx++] = 0;
+ } else {
+ /* Head will already have advanced to next instruction location
+ * even if the current instruction caused a hang, so we just
+ * save the current value as the value to restart at */
+ data[idx++] = head;
+ }
+
+ data[idx++] = I915_READ_START(ring);
+
+ /* Workaround for reading DCLV registers for gen < 8 */
+ data[idx++] = (gen < 8) ?
+ I915_READ(RING_PP_DIR_DCLV(&dev_priv->ring[VCS]))
+ : I915_READ(RING_PP_DIR_DCLV(ring));
+ data[idx++] = (gen < 8) ?
+ I915_READ(RING_PP_DIR_BASE(&dev_priv->ring[VCS]))
+ : I915_READ(RING_PP_DIR_BASE(ring));
+
+ switch (ring->id) {
+ case RCS:
+ if (max < (COMMON_RING_CTX_SIZE + RCS_RING_CTX_SIZE))
+ return -EINVAL;
+
+ data[idx++] = I915_READ(RENDER_HWS_PGA_GEN7);
+ data[idx++] = I915_READ(RING_UHPTR(ring->mmio_base));
+ data[idx++] = I915_READ(RING_INSTPM(ring->mmio_base));
+ data[idx++] = I915_READ(RING_IMR(ring->mmio_base));
+ data[idx++] = I915_READ(CACHE_MODE_1);
+ data[idx++] = I915_READ(RING_MI_MODE(ring->mmio_base));
+ data[idx++] = I915_READ(FW_BLC2);
+ data[idx++] = I915_READ(_3D_CHICKEN3);
+ data[idx++] = I915_READ(GAM_ECOCHK);
+ data[idx++] = I915_READ(RING_MODE_GEN7(ring));
+ data[idx++] = I915_READ(GEN6_RBSYNC);
+ data[idx++] = I915_READ(GEN7_FF_THREAD_MODE);
+ data[idx++] = I915_READ(MI_ARB_STATE);
+ break;
+
+ case VCS:
+ if (max < (COMMON_RING_CTX_SIZE + VCS_RING_CTX_SIZE))
+ return -EINVAL;
+
+ data[idx++] = I915_READ(BSD_HWS_PGA_GEN7);
+ data[idx++] = I915_READ(RING_MI_MODE(ring->mmio_base));
+ data[idx++] = I915_READ(RING_IMR(ring->mmio_base));
+ data[idx++] = I915_READ(RING_UHPTR(ring->mmio_base));
+ data[idx++] = I915_READ(RING_INSTPM(ring->mmio_base));
+ data[idx++] = I915_READ(RING_EXCC_GEN7(ring));
+ data[idx++] = I915_READ(GAC_ECO_BITS);
+ data[idx++] = I915_READ(RING_MODE_GEN7(ring));
+ data[idx++] = I915_READ(GEN6_VRSYNC);
+ data[idx++] = I915_READ(RING_MAX_IDLE(ring->mmio_base));
+ break;
+
+ case BCS:
+ if (max < (COMMON_RING_CTX_SIZE + BCS_RING_CTX_SIZE))
+ return -EINVAL;
+
+ data[idx++] = I915_READ(BLT_HWS_PGA_GEN7);
+ data[idx++] = I915_READ(RING_MI_MODE(ring->mmio_base));
+ data[idx++] = I915_READ(RING_IMR(ring->mmio_base));
+ data[idx++] = I915_READ(RING_UHPTR(ring->mmio_base));
+ data[idx++] = I915_READ(RING_INSTPM(ring->mmio_base));
+ data[idx++] = I915_READ(RING_EXCC_GEN7(ring));
+ data[idx++] = I915_READ(GAB_CTL);
+ data[idx++] = I915_READ(RING_MODE_GEN7(ring));
+ data[idx++] = I915_READ(GEN6_BRSYNC);
+ data[idx++] = I915_READ(GEN6_BVSYNC);
+ data[idx++] = I915_READ(RING_MAX_IDLE(ring->mmio_base));
+ break;
+
+ case VECS:
+ if (max < (COMMON_RING_CTX_SIZE + VECS_RING_CTX_SIZE))
+ return -EINVAL;
+
+ data[idx++] = I915_READ(VEBOX_HWS_PGA_GEN7);
+ data[idx++] = I915_READ(RING_MI_MODE(ring->mmio_base));
+ data[idx++] = I915_READ(RING_IMR(ring->mmio_base));
+ data[idx++] = I915_READ(RING_UHPTR(ring->mmio_base));
+ data[idx++] = I915_READ(RING_INSTPM(ring->mmio_base));
+ data[idx++] = I915_READ(RING_EXCC_GEN7(ring));
+ data[idx++] = I915_READ(RING_MODE_GEN7(ring));
+ data[idx++] = I915_READ(GEN6_VEVSYNC);
+ break;
+ }
+
+ return 0;
+}
+
+int intel_ring_restore(struct intel_ring_buffer *ring)
+{
+ if (ring && ring->restore)
+ return ring->restore(ring, ring->saved_state,
+ I915_RING_CONTEXT_SIZE);
+ else {
+ DRM_ERROR("ring restore not supported\n");
+ return -EINVAL;
+ }
+}
+
+
+static int
+gen6_ring_restore(struct intel_ring_buffer *ring, uint32_t *data,
+ uint32_t max)
+{
+ struct drm_device *dev = ring->dev;
+ drm_i915_private_t *dev_priv = dev->dev_private;
+ uint32_t idx = 0;
+ uint32_t x;
+
+ /* NOTE: Registers are restored in reverse order from when
+ * they were saved. */
+ switch (ring->id) {
+ case RCS:
+ if (max < (COMMON_RING_CTX_SIZE + RCS_RING_CTX_SIZE))
+ return -EINVAL;
+
+ idx = COMMON_RING_CTX_SIZE + RCS_RING_CTX_SIZE - 1;
+
+ I915_WRITE(MI_ARB_STATE, _MASKED_BIT_ENABLE_ALL(data[idx--]));
+ I915_WRITE(GEN7_FF_THREAD_MODE, data[idx--]);
+ I915_WRITE(GEN6_RBSYNC, data[idx--]);
+ I915_WRITE(RING_MODE_GEN7(ring),
+ _MASKED_BIT_ENABLE_ALL(data[idx--]));
+ I915_WRITE(GAM_ECOCHK, data[idx--]);
+ I915_WRITE(_3D_CHICKEN3, _MASKED_BIT_ENABLE_ALL(data[idx--]));
+ I915_WRITE(FW_BLC2, _MASKED_BIT_ENABLE_ALL(data[idx--]));
+ I915_WRITE(RING_MI_MODE(ring->mmio_base),
+ _MASKED_BIT_ENABLE_ALL(data[idx--]));
+ I915_WRITE(CACHE_MODE_1, _MASKED_BIT_ENABLE_ALL(data[idx--]));
+ I915_WRITE(RING_IMR(ring->mmio_base), data[idx--]);
+ I915_WRITE(RING_INSTPM(ring->mmio_base),
+ _MASKED_BIT_ENABLE_ALL(data[idx--]));
+ I915_WRITE(RING_UHPTR(ring->mmio_base), data[idx--]);
+ I915_WRITE(RENDER_HWS_PGA_GEN7, data[idx--]);
+ break;
+
+ case VCS:
+ if (max < (COMMON_RING_CTX_SIZE + VCS_RING_CTX_SIZE))
+ return -EINVAL;
+
+ idx = COMMON_RING_CTX_SIZE + VCS_RING_CTX_SIZE - 1;
+
+ I915_WRITE(RING_MAX_IDLE(ring->mmio_base), data[idx--]);
+ I915_WRITE(GEN6_VRSYNC, data[idx--]);
+ I915_WRITE(RING_MODE_GEN7(ring),
+ _MASKED_BIT_ENABLE_ALL(data[idx--]));
+ I915_WRITE(GAC_ECO_BITS, data[idx--]);
+ I915_WRITE(RING_EXCC_GEN7(ring),
+ _MASKED_BIT_ENABLE_ALL(data[idx--]));
+ I915_WRITE(RING_INSTPM(ring->mmio_base),
+ _MASKED_BIT_ENABLE_ALL(data[idx--]));
+ I915_WRITE(RING_UHPTR(ring->mmio_base), data[idx--]);
+ I915_WRITE(RING_IMR(ring->mmio_base), data[idx--]);
+ I915_WRITE(RING_MI_MODE(ring->mmio_base),
+ _MASKED_BIT_ENABLE_ALL(data[idx--]));
+ I915_WRITE(BSD_HWS_PGA_GEN7, data[idx--]);
+ break;
+
+ case BCS:
+ if (max < (COMMON_RING_CTX_SIZE + BCS_RING_CTX_SIZE))
+ return -EINVAL;
+
+ idx = COMMON_RING_CTX_SIZE + BCS_RING_CTX_SIZE - 1;
+
+ I915_WRITE(RING_MAX_IDLE(ring->mmio_base), data[idx--]);
+ I915_WRITE(GEN6_BVSYNC, data[idx--]);
+ I915_WRITE(GEN6_BRSYNC, data[idx--]);
+ I915_WRITE(RING_MODE_GEN7(ring),
+ _MASKED_BIT_ENABLE_ALL(data[idx--]));
+ I915_WRITE(GAB_CTL, data[idx--]);
+ I915_WRITE(RING_EXCC_GEN7(ring),
+ _MASKED_BIT_ENABLE_ALL(data[idx--]));
+ I915_WRITE(RING_INSTPM(ring->mmio_base),
+ _MASKED_BIT_ENABLE_ALL(data[idx--]));
+ I915_WRITE(RING_UHPTR(ring->mmio_base), data[idx--]);
+ I915_WRITE(RING_IMR(ring->mmio_base), data[idx--]);
+ I915_WRITE(RING_MI_MODE(ring->mmio_base),
+ _MASKED_BIT_ENABLE_ALL(data[idx--]));
+ I915_WRITE(BLT_HWS_PGA_GEN7, data[idx--]);
+ break;
+
+ case VECS:
+ if (max < (COMMON_RING_CTX_SIZE + VECS_RING_CTX_SIZE))
+ return -EINVAL;
+
+ idx = COMMON_RING_CTX_SIZE + VECS_RING_CTX_SIZE - 1;
+
+ I915_WRITE(GEN6_VEVSYNC, data[idx--]);
+ I915_WRITE(RING_MODE_GEN7(ring),
+ _MASKED_BIT_ENABLE_ALL(data[idx--]));
+ I915_WRITE(RING_EXCC_GEN7(ring),
+ _MASKED_BIT_ENABLE_ALL(data[idx--]));
+ I915_WRITE(RING_INSTPM(ring->mmio_base),
+ _MASKED_BIT_ENABLE_ALL(data[idx--]));
+ I915_WRITE(RING_UHPTR(ring->mmio_base), data[idx--]);
+ I915_WRITE(RING_IMR(ring->mmio_base), data[idx--]);
+ I915_WRITE(RING_MI_MODE(ring->mmio_base),
+ _MASKED_BIT_ENABLE_ALL(data[idx--]));
+ I915_WRITE(VEBOX_HWS_PGA_GEN7, data[idx--]);
+ break;
+ }
+
+ /* Restore common registers */
+ if (max < COMMON_RING_CTX_SIZE)
+ return -EINVAL;
+
+ idx = COMMON_RING_CTX_SIZE - 1;
+
+ I915_WRITE(RING_PP_DIR_BASE(ring), data[idx--]);
+ I915_WRITE(RING_PP_DIR_DCLV(ring), data[idx--]);
+
+ /* Write ring base address before head/tail as it clears head to 0 */
+ I915_WRITE_START(ring, data[idx--]);
+ x = I915_READ_START(ring);
+ I915_WRITE_HEAD(ring, data[idx--]);
+ I915_WRITE_TAIL(ring, data[idx--]);
+ I915_WRITE_CTL(ring, data[idx--]);
+
+ return 0;
+}
+
/* Blitter support (SandyBridge+) */
static int gen6_ring_flush(struct intel_ring_buffer *ring,
@@ -1869,6 +2432,14 @@ int intel_init_render_ring_buffer(struct drm_device *dev)
ring->get_seqno = gen6_ring_get_seqno;
ring->set_seqno = ring_set_seqno;
ring->sync_to = gen6_ring_sync;
+ ring->enable = gen6_ring_enable;
+ ring->disable = gen6_ring_disable;
+ ring->start = gen6_ring_start;
+ ring->stop = gen6_ring_stop;
+ ring->reset = gen6_ring_reset;
+ ring->save = gen6_ring_save;
+ ring->restore = gen6_ring_restore;
+ ring->invalidate_tlb = gen6_ring_invalidate_tlb;
ring->semaphore_register[RCS] = MI_SEMAPHORE_SYNC_INVALID;
ring->semaphore_register[VCS] = MI_SEMAPHORE_SYNC_RV;
ring->semaphore_register[BCS] = MI_SEMAPHORE_SYNC_RB;
@@ -2045,6 +2616,14 @@ int intel_init_bsd_ring_buffer(struct drm_device *dev)
gen6_ring_dispatch_execbuffer;
}
ring->sync_to = gen6_ring_sync;
+ ring->enable = gen6_ring_enable;
+ ring->disable = gen6_ring_disable;
+ ring->start = gen6_ring_start;
+ ring->stop = gen6_ring_stop;
+ ring->reset = gen6_ring_reset;
+ ring->save = gen6_ring_save;
+ ring->restore = gen6_ring_restore;
+ ring->invalidate_tlb = gen6_ring_invalidate_tlb;
ring->semaphore_register[RCS] = MI_SEMAPHORE_SYNC_VR;
ring->semaphore_register[VCS] = MI_SEMAPHORE_SYNC_INVALID;
ring->semaphore_register[BCS] = MI_SEMAPHORE_SYNC_VB;
@@ -2102,6 +2681,14 @@ int intel_init_blt_ring_buffer(struct drm_device *dev)
ring->dispatch_execbuffer = gen6_ring_dispatch_execbuffer;
}
ring->sync_to = gen6_ring_sync;
+ ring->enable = gen6_ring_enable;
+ ring->disable = gen6_ring_disable;
+ ring->start = gen6_ring_start;
+ ring->stop = gen6_ring_stop;
+ ring->reset = gen6_ring_reset;
+ ring->save = gen6_ring_save;
+ ring->restore = gen6_ring_restore;
+ ring->invalidate_tlb = gen6_ring_invalidate_tlb;
ring->semaphore_register[RCS] = MI_SEMAPHORE_SYNC_BR;
ring->semaphore_register[VCS] = MI_SEMAPHORE_SYNC_BV;
ring->semaphore_register[BCS] = MI_SEMAPHORE_SYNC_INVALID;
@@ -2143,6 +2730,14 @@ int intel_init_vebox_ring_buffer(struct drm_device *dev)
ring->dispatch_execbuffer = gen6_ring_dispatch_execbuffer;
}
ring->sync_to = gen6_ring_sync;
+ ring->enable = gen6_ring_enable;
+ ring->disable = gen6_ring_disable;
+ ring->start = gen6_ring_start;
+ ring->stop = gen6_ring_stop;
+ ring->reset = gen6_ring_reset;
+ ring->save = gen6_ring_save;
+ ring->restore = gen6_ring_restore;
+ ring->invalidate_tlb = gen6_ring_invalidate_tlb;
ring->semaphore_register[RCS] = MI_SEMAPHORE_SYNC_VER;
ring->semaphore_register[VCS] = MI_SEMAPHORE_SYNC_VEV;
ring->semaphore_register[BCS] = MI_SEMAPHORE_SYNC_VEB;
diff --git a/drivers/gpu/drm/i915/intel_ringbuffer.h b/drivers/gpu/drm/i915/intel_ringbuffer.h
index 71a73f4..cd96ad9 100644
--- a/drivers/gpu/drm/i915/intel_ringbuffer.h
+++ b/drivers/gpu/drm/i915/intel_ringbuffer.h
@@ -18,6 +18,25 @@ struct intel_hw_status_page {
struct drm_i915_gem_object *obj;
};
+/* These values must match the requirements of the ring save/restore functions
+ * which may need to change for different versions of the chip */
+#define COMMON_RING_CTX_SIZE 6
+
+#define RCS_RING_CTX_SIZE 13
+#define VCS_RING_CTX_SIZE 10
+#define BCS_RING_CTX_SIZE 11
+#define VECS_RING_CTX_SIZE 8
+
+#define MAX_CTX(a, b) (((a) > (b)) ? (a) : (b))
+
+/* Largest of individual rings + common*/
+#define I915_RING_CONTEXT_SIZE (COMMON_RING_CTX_SIZE + \
+ MAX_CTX(MAX_CTX(RCS_RING_CTX_SIZE, \
+ VCS_RING_CTX_SIZE), \
+ MAX_CTX(BCS_RING_CTX_SIZE, \
+ VECS_RING_CTX_SIZE)))
+
+
#define I915_READ_TAIL(ring) I915_READ(RING_TAIL((ring)->mmio_base))
#define I915_WRITE_TAIL(ring, val) I915_WRITE(RING_TAIL((ring)->mmio_base), val)
@@ -33,6 +52,13 @@ struct intel_hw_status_page {
#define I915_READ_IMR(ring) I915_READ(RING_IMR((ring)->mmio_base))
#define I915_WRITE_IMR(ring, val) I915_WRITE(RING_IMR((ring)->mmio_base), val)
+#define I915_READ_MODE(ring) I915_READ(RING_MI_MODE((ring)->mmio_base))
+#define I915_WRITE_MODE(ring, val) \
+ I915_WRITE(RING_MI_MODE((ring)->mmio_base), val)
+
+#define RESET_HEAD_TAIL 0x1
+#define FORCE_ADVANCE 0x2
+
enum intel_ring_hangcheck_action {
HANGCHECK_IDLE = 0,
HANGCHECK_WAIT,
@@ -115,6 +141,18 @@ struct intel_ring_buffer {
struct intel_ring_buffer *to,
u32 seqno);
+ int (*enable)(struct intel_ring_buffer *ring);
+ int (*disable)(struct intel_ring_buffer *ring);
+ int (*start)(struct intel_ring_buffer *ring);
+ int (*stop)(struct intel_ring_buffer *ring);
+ int (*reset)(struct intel_ring_buffer *ring);
+ int (*save)(struct intel_ring_buffer *ring,
+ uint32_t *data, uint32_t max,
+ u32 flags);
+ int (*restore)(struct intel_ring_buffer *ring,
+ uint32_t *data, uint32_t max);
+ int (*invalidate_tlb)(struct intel_ring_buffer *ring);
+
/* our mbox written by others */
u32 semaphore_register[I915_NUM_RINGS];
/* mboxes this ring signals to */
@@ -155,6 +193,9 @@ struct intel_ring_buffer {
struct i915_hw_context *default_context;
struct i915_hw_context *last_context;
+ /* Area large enough to store all the register
+ * data associated with this ring */
+ u32 saved_state[I915_RING_CONTEXT_SIZE];
struct intel_ring_hangcheck hangcheck;
struct {
@@ -229,6 +270,7 @@ intel_write_status_page(struct intel_ring_buffer *ring,
#define I915_GEM_HWS_INDEX 0x20
#define I915_GEM_HWS_SCRATCH_INDEX 0x30
#define I915_GEM_HWS_SCRATCH_ADDR (I915_GEM_HWS_SCRATCH_INDEX << MI_STORE_DWORD_INDEX_SHIFT)
+#define I915_GEM_PGFLIP_INDEX 0x38
void intel_cleanup_ring_buffer(struct intel_ring_buffer *ring);
@@ -278,4 +320,11 @@ static inline void i915_trace_irq_get(struct intel_ring_buffer *ring, u32 seqno)
/* DRI warts */
int intel_render_ring_init_dri(struct drm_device *dev, u64 start, u32 size);
+void intel_ring_resample(struct intel_ring_buffer *ring);
+int intel_ring_disable(struct intel_ring_buffer *ring);
+int intel_ring_enable(struct intel_ring_buffer *ring);
+int intel_ring_reset(struct intel_ring_buffer *ring);
+int intel_ring_save(struct intel_ring_buffer *ring, u32 flags);
+int intel_ring_restore(struct intel_ring_buffer *ring);
+
#endif /* _INTEL_RINGBUFFER_H_ */
diff --git a/drivers/gpu/drm/i915/intel_uncore.c b/drivers/gpu/drm/i915/intel_uncore.c
index f9883ce..5349215 100644
--- a/drivers/gpu/drm/i915/intel_uncore.c
+++ b/drivers/gpu/drm/i915/intel_uncore.c
@@ -312,6 +312,31 @@ void gen6_gt_force_wake_put(struct drm_i915_private *dev_priv)
spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
}
+void gen6_gt_force_wake_restore(struct drm_device *dev)
+{
+ struct drm_i915_private *dev_priv = dev->dev_private;
+
+ /* Restore the current expected force wake state with the
+ * hardware. This may be required following a reset.
+ *
+ * WARNING: Caller *MUST* hold uncore.lock whilst calling this.
+ *
+ * uncore.lock isn't taken in this function to allow the caller the
+ * flexibility to do other work immediately before/after
+ * whilst holding the lock */
+
+ intel_uncore_forcewake_reset(dev);
+
+ /* If reset with a user forcewake, try to restore, otherwise turn it off */
+ if (dev_priv->uncore.forcewake_count)
+ dev_priv->uncore.funcs.force_wake_get(dev_priv);
+ else
+ dev_priv->uncore.funcs.force_wake_put(dev_priv);
+
+ /* Restore fifo count */
+ dev_priv->uncore.fifo_count = __raw_i915_read32(dev_priv, GT_FIFO_FREE_ENTRIES);
+}
+
/* We give fast paths for the really cool registers */
#define NEEDS_FORCE_WAKE(dev_priv, reg) \
((reg) < 0x40000 && (reg) != FORCEWAKE)
--
1.8.4
_______________________________________________
Intel-gfx mailing list
Intel-gfx@xxxxxxxxxxxxxxxxxxxxx
http://lists.freedesktop.org/mailman/listinfo/intel-gfx
