Signed-off-by: Ben Widawsky <ben@xxxxxxxxxxxx>
---
drivers/gpu/drm/i915/i915_drv.h | 4 ++++
drivers/gpu/drm/i915/i915_gem.c | 32 ++++++++++++++++++++++++++++----
drivers/gpu/drm/i915/i915_gem_gtt.c | 13 ++++++++++---
3 files changed, 42 insertions(+), 7 deletions(-)
diff --git a/drivers/gpu/drm/i915/i915_drv.h b/drivers/gpu/drm/i915/i915_drv.h
index 5d2f62d..dfecdfd 100644
--- a/drivers/gpu/drm/i915/i915_drv.h
+++ b/drivers/gpu/drm/i915/i915_drv.h
@@ -2818,6 +2818,10 @@ static inline bool cpu_cache_is_coherent(struct drm_device *dev,
{
return HAS_LLC(dev) || level != I915_CACHE_NONE;
}
+static inline bool i915_gem_obj_should_clflush(struct drm_i915_gem_object *obj)
+{
+ return obj->base.size >= to_i915(obj->base.dev)->wbinvd_threshold;
+}
int __must_check i915_gem_object_finish_gpu(struct drm_i915_gem_object *obj);
int __must_check i915_gem_init(struct drm_device *dev);
int i915_gem_init_rings(struct drm_device *dev);
diff --git a/drivers/gpu/drm/i915/i915_gem.c b/drivers/gpu/drm/i915/i915_gem.c
index 4d5a69d..59be709 100644
--- a/drivers/gpu/drm/i915/i915_gem.c
+++ b/drivers/gpu/drm/i915/i915_gem.c
@@ -204,6 +204,7 @@ i915_gem_object_get_pages_phys(struct drm_i915_gem_object *obj)
char *vaddr = obj->phys_handle->vaddr;
struct sg_table *st;
struct scatterlist *sg;
+ const bool do_wbinvd = i915_gem_obj_should_clflush(obj);
int i;
if (WARN_ON(i915_gem_object_needs_bit17_swizzle(obj)))
@@ -219,12 +220,15 @@ i915_gem_object_get_pages_phys(struct drm_i915_gem_object *obj)
src = kmap_atomic(page);
memcpy(vaddr, src, PAGE_SIZE);
- drm_clflush_virt_range(vaddr, PAGE_SIZE);
+ if (!do_wbinvd)
+ drm_clflush_virt_range(vaddr, PAGE_SIZE);
kunmap_atomic(src);
page_cache_release(page);
vaddr += PAGE_SIZE;
}
+ if (do_wbinvd)
+ wbinvd();
i915_gem_chipset_flush(obj->base.dev);
@@ -252,6 +256,7 @@ i915_gem_object_get_pages_phys(struct drm_i915_gem_object *obj)
static void
i915_gem_object_put_pages_phys(struct drm_i915_gem_object *obj)
{
+ const bool do_wbinvd = i915_gem_obj_should_clflush(obj);
int ret;
BUG_ON(obj->madv == __I915_MADV_PURGED);
@@ -282,7 +287,8 @@ i915_gem_object_put_pages_phys(struct drm_i915_gem_object *obj)
continue;
dst = kmap_atomic(page);
- drm_clflush_virt_range(vaddr, PAGE_SIZE);
+ if (!do_wbinvd)
+ drm_clflush_virt_range(vaddr, PAGE_SIZE);
memcpy(dst, vaddr, PAGE_SIZE);
kunmap_atomic(dst);
@@ -295,6 +301,9 @@ i915_gem_object_put_pages_phys(struct drm_i915_gem_object *obj)
obj->dirty = 0;
}
+ if (do_wbinvd && !ret)
+ wbinvd();
+
sg_free_table(obj->pages);
kfree(obj->pages);
@@ -396,7 +405,10 @@ i915_gem_phys_pwrite(struct drm_i915_gem_object *obj,
return -EFAULT;
}
- drm_clflush_virt_range(vaddr, args->size);
+ if (args->size >= to_i915(obj->base.dev)->wbinvd_threshold)
+ wbinvd();
+ else
+ drm_clflush_virt_range(vaddr, args->size);
i915_gem_chipset_flush(dev);
return 0;
}
@@ -647,6 +659,7 @@ i915_gem_shmem_pread(struct drm_device *dev,
int obj_do_bit17_swizzling, page_do_bit17_swizzling;
int prefaulted = 0;
int needs_clflush = 0;
+ bool do_wbinvd = false;
struct sg_page_iter sg_iter;
user_data = to_user_ptr(args->data_ptr);
@@ -658,6 +671,9 @@ i915_gem_shmem_pread(struct drm_device *dev,
if (ret)
return ret;
+ if (needs_clflush && i915_gem_obj_should_clflush(obj))
+ do_wbinvd = true;
+
offset = args->offset;
for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents,
@@ -714,6 +730,9 @@ next_page:
}
out:
+ if (do_wbinvd && !ret)
+ wbinvd();
+
i915_gem_object_unpin_pages(obj);
return ret;
@@ -4061,7 +4080,12 @@ i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write)
/* Flush the CPU cache if it's still invalid. */
if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0) {
- i915_gem_clflush_object(obj, false);
+ struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
+ if (is_cpu_flush_required(obj) &&
+ obj->base.size >= dev_priv->wbinvd_threshold)
+ wbinvd();
+ else
+ i915_gem_clflush_object(obj, false);
obj->base.read_domains |= I915_GEM_DOMAIN_CPU;
}
diff --git a/drivers/gpu/drm/i915/i915_gem_gtt.c b/drivers/gpu/drm/i915/i915_gem_gtt.c
index 746f77f..13cc493 100644
--- a/drivers/gpu/drm/i915/i915_gem_gtt.c
+++ b/drivers/gpu/drm/i915/i915_gem_gtt.c
@@ -371,6 +371,9 @@ static void gen8_ppgtt_insert_entries(struct i915_address_space *vm,
unsigned pde = start >> GEN8_PDE_SHIFT & GEN8_PDE_MASK;
unsigned pte = start >> GEN8_PTE_SHIFT & GEN8_PTE_MASK;
struct sg_page_iter sg_iter;
+ const bool needs_flush = !HAS_LLC(ppgtt->base.dev);
+ bool do_wbinvd = needs_flush &&
+ pages->nents * PAGE_SIZE >= to_i915(vm->dev)->wbinvd_threshold;
pt_vaddr = NULL;
@@ -385,7 +388,7 @@ static void gen8_ppgtt_insert_entries(struct i915_address_space *vm,
gen8_pte_encode(sg_page_iter_dma_address(&sg_iter),
cache_level, true);
if (++pte == GEN8_PTES_PER_PAGE) {
- if (!HAS_LLC(ppgtt->base.dev))
+ if (needs_flush && !do_wbinvd)
drm_clflush_virt_range(pt_vaddr, PAGE_SIZE);
kunmap_atomic(pt_vaddr);
pt_vaddr = NULL;
@@ -401,6 +404,9 @@ static void gen8_ppgtt_insert_entries(struct i915_address_space *vm,
drm_clflush_virt_range(pt_vaddr, PAGE_SIZE);
kunmap_atomic(pt_vaddr);
}
+
+ if (do_wbinvd)
+ wbinvd();
}
static void gen8_free_page_tables(struct page **pt_pages)
@@ -660,11 +666,12 @@ static int gen8_ppgtt_init(struct i915_hw_ppgtt *ppgtt, uint64_t size)
pd_vaddr[j] = gen8_pde_encode(ppgtt->base.dev, addr,
I915_CACHE_LLC);
}
- if (!HAS_LLC(ppgtt->base.dev))
- drm_clflush_virt_range(pd_vaddr, PAGE_SIZE);
kunmap_atomic(pd_vaddr);
}
+ if (!HAS_LLC(ppgtt->base.dev))
+ wbinvd();
+
ppgtt->switch_mm = gen8_mm_switch;
ppgtt->base.clear_range = gen8_ppgtt_clear_range;
ppgtt->base.insert_entries = gen8_ppgtt_insert_entries;
--
2.3.0
_______________________________________________
Intel-gfx mailing list
Intel-gfx@xxxxxxxxxxxxxxxxxxxxx
http://lists.freedesktop.org/mailman/listinfo/intel-gfx
