This transitional patch doesn't do much for the existing code. However,
it should make upcoming patches to use the full 48b address space a bit
easier. The patch also introduces the PML4, ie. the new top level structure
of the page tables.
v2: Renamed pdp_free to be similar to pd/pt (unmap_and_free_pdp).
v3: To facilitate testing, 48b mode will be available on Broadwell and
GEN9+, when i915.enable_ppgtt = 3.
v4: Rebase after s/page_tables/page_table/ and added extra information
about 4-level page table formats.
Signed-off-by: Ben Widawsky <ben@xxxxxxxxxxxx>
Signed-off-by: Michel Thierry <michel.thierry@xxxxxxxxx> (v2+)
---
drivers/gpu/drm/i915/i915_drv.h | 7 ++-
drivers/gpu/drm/i915/i915_gem_gtt.c | 107 +++++++++++++++++++++++++++++-------
drivers/gpu/drm/i915/i915_gem_gtt.h | 45 ++++++++++++---
3 files changed, 129 insertions(+), 30 deletions(-)
diff --git a/drivers/gpu/drm/i915/i915_drv.h b/drivers/gpu/drm/i915/i915_drv.h
index 9adfd12..9eea844 100644
--- a/drivers/gpu/drm/i915/i915_drv.h
+++ b/drivers/gpu/drm/i915/i915_drv.h
@@ -2436,7 +2436,12 @@ struct drm_i915_cmd_table {
#define HAS_HW_CONTEXTS(dev) (INTEL_INFO(dev)->gen >= 6)
#define HAS_LOGICAL_RING_CONTEXTS(dev) (INTEL_INFO(dev)->gen >= 8)
#define USES_PPGTT(dev) (i915.enable_ppgtt)
-#define USES_FULL_PPGTT(dev) (i915.enable_ppgtt == 2)
+#define USES_FULL_PPGTT(dev) (i915.enable_ppgtt >= 2)
+#ifdef CONFIG_64BIT
+# define USES_FULL_48BIT_PPGTT(dev) (i915.enable_ppgtt == 3)
+#else
+# define USES_FULL_48BIT_PPGTT(dev) false
+#endif
#define HAS_OVERLAY(dev) (INTEL_INFO(dev)->has_overlay)
#define OVERLAY_NEEDS_PHYSICAL(dev) (INTEL_INFO(dev)->overlay_needs_physical)
diff --git a/drivers/gpu/drm/i915/i915_gem_gtt.c b/drivers/gpu/drm/i915/i915_gem_gtt.c
index 5036ca0..a288f6b 100644
--- a/drivers/gpu/drm/i915/i915_gem_gtt.c
+++ b/drivers/gpu/drm/i915/i915_gem_gtt.c
@@ -104,10 +104,18 @@ static int sanitize_enable_ppgtt(struct drm_device *dev, int enable_ppgtt)
{
bool has_aliasing_ppgtt;
bool has_full_ppgtt;
+ bool has_full_64bit_ppgtt;
has_aliasing_ppgtt = INTEL_INFO(dev)->gen >= 6;
has_full_ppgtt = INTEL_INFO(dev)->gen >= 7;
+#ifdef CONFIG_64BIT
+ has_full_64bit_ppgtt = (IS_BROADWELL(dev) ||
+ INTEL_INFO(dev)->gen >= 9) && false; /* FIXME: 64b */
+#else
+ has_full_64bit_ppgtt = false;
+#endif
+
if (intel_vgpu_active(dev))
has_full_ppgtt = false; /* emulation is too hard */
@@ -125,6 +133,9 @@ static int sanitize_enable_ppgtt(struct drm_device *dev, int enable_ppgtt)
if (enable_ppgtt == 2 && has_full_ppgtt)
return 2;
+ if (enable_ppgtt == 3 && has_full_64bit_ppgtt)
+ return 3;
+
#ifdef CONFIG_INTEL_IOMMU
/* Disable ppgtt on SNB if VT-d is on. */
if (INTEL_INFO(dev)->gen == 6 && intel_iommu_gfx_mapped) {
@@ -451,6 +462,45 @@ free_pd:
return ERR_PTR(ret);
}
+static void __pdp_fini(struct i915_page_directory_pointer *pdp)
+{
+ kfree(pdp->used_pdpes);
+ kfree(pdp->page_directory);
+ /* HACK */
+ pdp->page_directory = NULL;
+}
+
+static void unmap_and_free_pdp(struct i915_page_directory_pointer *pdp,
+ struct drm_device *dev)
+{
+ __pdp_fini(pdp);
+ if (USES_FULL_48BIT_PPGTT(dev))
+ kfree(pdp);
+}
+
+static int __pdp_init(struct i915_page_directory_pointer *pdp,
+ struct drm_device *dev)
+{
+ size_t pdpes = I915_PDPES_PER_PDP(dev);
+
+ pdp->used_pdpes = kcalloc(BITS_TO_LONGS(pdpes),
+ sizeof(unsigned long),
+ GFP_KERNEL);
+ if (!pdp->used_pdpes)
+ return -ENOMEM;
+
+ pdp->page_directory = kcalloc(pdpes, sizeof(*pdp->page_directory), GFP_KERNEL);
+ if (!pdp->page_directory) {
+ kfree(pdp->used_pdpes);
+ /* the PDP might be the statically allocated top level. Keep it
+ * as clean as possible */
+ pdp->used_pdpes = NULL;
+ return -ENOMEM;
+ }
+
+ return 0;
+}
+
/* Broadwell Page Directory Pointer Descriptors */
static int gen8_write_pdp(struct intel_engine_cs *ring,
unsigned entry,
@@ -480,7 +530,7 @@ static int gen8_mm_switch(struct i915_hw_ppgtt *ppgtt,
{
int i, ret;
- for (i = GEN8_LEGACY_PDPES - 1; i >= 0; i--) {
+ for (i = 3; i >= 0; i--) {
struct i915_page_directory *pd = ppgtt->pdp.page_directory[i];
dma_addr_t pd_daddr = pd ? pd->daddr : ppgtt->scratch_pd->daddr;
/* The page directory might be NULL, but we need to clear out
@@ -569,9 +619,6 @@ static void gen8_ppgtt_insert_entries(struct i915_address_space *vm,
pt_vaddr = NULL;
for_each_sg_page(pages->sgl, &sg_iter, pages->nents, 0) {
- if (WARN_ON(pdpe >= GEN8_LEGACY_PDPES))
- break;
-
if (pt_vaddr == NULL) {
struct i915_page_directory *pd = ppgtt->pdp.page_directory[pdpe];
struct i915_page_table *pt = pd->page_table[pde];
@@ -653,7 +700,8 @@ static void gen8_ppgtt_cleanup(struct i915_address_space *vm)
container_of(vm, struct i915_hw_ppgtt, base);
int i;
- for_each_set_bit(i, ppgtt->pdp.used_pdpes, GEN8_LEGACY_PDPES) {
+ for_each_set_bit(i, ppgtt->pdp.used_pdpes,
+ I915_PDPES_PER_PDP(ppgtt->base.dev)) {
if (WARN_ON(!ppgtt->pdp.page_directory[i]))
continue;
@@ -661,6 +709,7 @@ static void gen8_ppgtt_cleanup(struct i915_address_space *vm)
unmap_and_free_pd(ppgtt->pdp.page_directory[i], ppgtt->base.dev);
}
+ unmap_and_free_pdp(&ppgtt->pdp, ppgtt->base.dev);
unmap_and_free_pd(ppgtt->scratch_pd, ppgtt->base.dev);
unmap_and_free_pt(ppgtt->scratch_pt, ppgtt->base.dev);
}
@@ -753,8 +802,9 @@ static int gen8_ppgtt_alloc_page_directories(struct i915_hw_ppgtt *ppgtt,
struct i915_page_directory *pd;
uint64_t temp;
uint32_t pdpe;
+ size_t pdpes = I915_PDPES_PER_PDP(ppgtt->base.dev);
- WARN_ON(!bitmap_empty(new_pds, GEN8_LEGACY_PDPES));
+ WARN_ON(!bitmap_empty(new_pds, pdpes));
/* FIXME: upper bound must not overflow 32 bits */
WARN_ON((start + length) > (1ULL << 32));
@@ -775,18 +825,19 @@ static int gen8_ppgtt_alloc_page_directories(struct i915_hw_ppgtt *ppgtt,
return 0;
unwind_out:
- for_each_set_bit(pdpe, new_pds, GEN8_LEGACY_PDPES)
+ for_each_set_bit(pdpe, new_pds, pdpes)
unmap_and_free_pd(pdp->page_directory[pdpe], dev);
return -ENOMEM;
}
static void
-free_gen8_temp_bitmaps(unsigned long *new_pds, unsigned long **new_pts)
+free_gen8_temp_bitmaps(unsigned long *new_pds, unsigned long **new_pts,
+ size_t pdpes)
{
int i;
- for (i = 0; i < GEN8_LEGACY_PDPES; i++)
+ for (i = 0; i < pdpes; i++)
kfree(new_pts[i]);
kfree(new_pts);
kfree(new_pds);
@@ -797,23 +848,24 @@ free_gen8_temp_bitmaps(unsigned long *new_pds, unsigned long **new_pts)
*/
static
int __must_check alloc_gen8_temp_bitmaps(unsigned long **new_pds,
- unsigned long ***new_pts)
+ unsigned long ***new_pts,
+ size_t pdpes)
{
int i;
unsigned long *pds;
unsigned long **pts;
- pds = kcalloc(BITS_TO_LONGS(GEN8_LEGACY_PDPES), sizeof(unsigned long), GFP_KERNEL);
+ pds = kcalloc(BITS_TO_LONGS(pdpes), sizeof(unsigned long), GFP_KERNEL);
if (!pds)
return -ENOMEM;
- pts = kcalloc(GEN8_LEGACY_PDPES, sizeof(unsigned long *), GFP_KERNEL);
+ pts = kcalloc(pdpes, sizeof(unsigned long *), GFP_KERNEL);
if (!pts) {
kfree(pds);
return -ENOMEM;
}
- for (i = 0; i < GEN8_LEGACY_PDPES; i++) {
+ for (i = 0; i < pdpes; i++) {
pts[i] = kcalloc(BITS_TO_LONGS(I915_PDES),
sizeof(unsigned long), GFP_KERNEL);
if (!pts[i])
@@ -826,7 +878,7 @@ int __must_check alloc_gen8_temp_bitmaps(unsigned long **new_pds,
return 0;
err_out:
- free_gen8_temp_bitmaps(pds, pts);
+ free_gen8_temp_bitmaps(pds, pts, pdpes);
return -ENOMEM;
}
@@ -842,6 +894,7 @@ static int gen8_alloc_va_range(struct i915_address_space *vm,
const uint64_t orig_length = length;
uint64_t temp;
uint32_t pdpe;
+ size_t pdpes = I915_PDPES_PER_PDP(dev);
int ret;
/* Wrap is never okay since we can only represent 48b, and we don't
@@ -850,7 +903,7 @@ static int gen8_alloc_va_range(struct i915_address_space *vm,
if (WARN_ON(start + length < start))
return -ERANGE;
- ret = alloc_gen8_temp_bitmaps(&new_page_dirs, &new_page_tables);
+ ret = alloc_gen8_temp_bitmaps(&new_page_dirs, &new_page_tables, pdpes);
if (ret)
return ret;
@@ -858,7 +911,7 @@ static int gen8_alloc_va_range(struct i915_address_space *vm,
ret = gen8_ppgtt_alloc_page_directories(ppgtt, &ppgtt->pdp, start, length,
new_page_dirs);
if (ret) {
- free_gen8_temp_bitmaps(new_page_dirs, new_page_tables);
+ free_gen8_temp_bitmaps(new_page_dirs, new_page_tables, pdpes);
return ret;
}
@@ -914,7 +967,7 @@ static int gen8_alloc_va_range(struct i915_address_space *vm,
set_bit(pdpe, ppgtt->pdp.used_pdpes);
}
- free_gen8_temp_bitmaps(new_page_dirs, new_page_tables);
+ free_gen8_temp_bitmaps(new_page_dirs, new_page_tables, pdpes);
return 0;
err_out:
@@ -923,10 +976,10 @@ err_out:
unmap_and_free_pt(ppgtt->pdp.page_directory[pdpe]->page_table[temp], vm->dev);
}
- for_each_set_bit(pdpe, new_page_dirs, GEN8_LEGACY_PDPES)
+ for_each_set_bit(pdpe, new_page_dirs, pdpes)
unmap_and_free_pd(ppgtt->pdp.page_directory[pdpe], vm->dev);
- free_gen8_temp_bitmaps(new_page_dirs, new_page_tables);
+ free_gen8_temp_bitmaps(new_page_dirs, new_page_tables, pdpes);
return ret;
}
@@ -950,8 +1003,22 @@ static int gen8_ppgtt_init(struct i915_hw_ppgtt *ppgtt)
gen8_initialize_pt(&ppgtt->base, ppgtt->scratch_pt);
gen8_initialize_pd(&ppgtt->base, ppgtt->scratch_pd);
+ if (!USES_FULL_48BIT_PPGTT(ppgtt->base.dev)) {
+ int ret = __pdp_init(&ppgtt->pdp, false);
+
+ if (ret) {
+ unmap_and_free_pd(ppgtt->scratch_pd, ppgtt->base.dev);
+ unmap_and_free_pt(ppgtt->scratch_pt, ppgtt->base.dev);
+ return ret;
+ }
+
+ ppgtt->base.total = 1ULL << 32;
+ } else {
+ ppgtt->base.total = 1ULL << 48;
+ return -EPERM; /* Not yet implemented */
+ }
+
ppgtt->base.start = 0;
- ppgtt->base.total = 1ULL << 32;
ppgtt->base.cleanup = gen8_ppgtt_cleanup;
ppgtt->base.allocate_va_range = gen8_alloc_va_range;
ppgtt->base.insert_entries = gen8_ppgtt_insert_entries;
diff --git a/drivers/gpu/drm/i915/i915_gem_gtt.h b/drivers/gpu/drm/i915/i915_gem_gtt.h
index 15476e0..a01cc34 100644
--- a/drivers/gpu/drm/i915/i915_gem_gtt.h
+++ b/drivers/gpu/drm/i915/i915_gem_gtt.h
@@ -88,9 +88,17 @@ typedef uint64_t gen8_pde_t;
* PDPE | PDE | PTE | offset
* The difference as compared to normal x86 3 level page table is the PDPEs are
* programmed via register.
+ *
+ * GEN8 48b legacy style address is defined as a 4 level page table:
+ * 47:39 | 38:30 | 29:21 | 20:12 | 11:0
+ * PML4E | PDPE | PDE | PTE | offset
*/
+#define GEN8_PML4ES_PER_PML4 512
+#define GEN8_PML4E_SHIFT 39
#define GEN8_PDPE_SHIFT 30
-#define GEN8_PDPE_MASK 0x3
+/* NB: GEN8_PDPE_MASK is untrue for 32b platforms, but it has no impact on 32b page
+ * tables */
+#define GEN8_PDPE_MASK 0x1ff
#define GEN8_PDE_SHIFT 21
#define GEN8_PDE_MASK 0x1ff
#define GEN8_PTE_SHIFT 12
@@ -98,6 +106,13 @@ typedef uint64_t gen8_pde_t;
#define GEN8_LEGACY_PDPES 4
#define GEN8_PTES I915_PTES(sizeof(gen8_pte_t))
+#ifdef CONFIG_64BIT
+# define I915_PDPES_PER_PDP(dev) (USES_FULL_48BIT_PPGTT(dev) ?\
+ GEN8_PML4ES_PER_PML4 : GEN8_LEGACY_PDPES)
+#else
+# define I915_PDPES_PER_PDP GEN8_LEGACY_PDPES
+#endif
+
#define PPAT_UNCACHED_INDEX (_PAGE_PWT | _PAGE_PCD)
#define PPAT_CACHED_PDE_INDEX 0 /* WB LLC */
#define PPAT_CACHED_INDEX _PAGE_PAT /* WB LLCeLLC */
@@ -224,9 +239,17 @@ struct i915_page_directory {
};
struct i915_page_directory_pointer {
- /* struct page *page; */
- DECLARE_BITMAP(used_pdpes, GEN8_LEGACY_PDPES);
- struct i915_page_directory *page_directory[GEN8_LEGACY_PDPES];
+ struct page *page;
+ dma_addr_t daddr;
+ unsigned long *used_pdpes;
+ struct i915_page_directory **page_directory;
+};
+
+struct i915_pml4 {
+ struct page *page;
+ dma_addr_t daddr;
+ DECLARE_BITMAP(used_pml4es, GEN8_PML4ES_PER_PML4);
+ struct i915_page_directory_pointer *pdps[GEN8_PML4ES_PER_PML4];
};
struct i915_address_space {
@@ -325,8 +348,9 @@ struct i915_hw_ppgtt {
struct drm_mm_node node;
unsigned long pd_dirty_rings;
union {
- struct i915_page_directory_pointer pdp;
- struct i915_page_directory pd;
+ struct i915_pml4 pml4; /* GEN8+ & 64b PPGTT */
+ struct i915_page_directory_pointer pdp; /* GEN8+ */
+ struct i915_page_directory pd; /* GEN6-7 */
};
struct i915_page_table *scratch_pt;
@@ -423,14 +447,17 @@ static inline uint32_t gen6_pde_index(uint32_t addr)
temp = min(temp, length), \
start += temp, length -= temp)
-#define gen8_for_each_pdpe(pd, pdp, start, length, temp, iter) \
- for (iter = gen8_pdpe_index(start); \
- pd = (pdp)->page_directory[iter], length > 0 && iter < GEN8_LEGACY_PDPES; \
+#define gen8_for_each_pdpe_e(pd, pdp, start, length, temp, iter, b) \
+ for (iter = gen8_pdpe_index(start); \
+ pd = (pdp)->page_directory[iter], length > 0 && (iter < b); \
iter++, \
temp = ALIGN(start+1, 1 << GEN8_PDPE_SHIFT) - start, \
temp = min(temp, length), \
start += temp, length -= temp)
+#define gen8_for_each_pdpe(pd, pdp, start, length, temp, iter) \
+ gen8_for_each_pdpe_e(pd, pdp, start, length, temp, iter, I915_PDPES_PER_PDP(dev))
+
static inline uint32_t gen8_pte_index(uint64_t address)
{
return i915_pte_index(address, GEN8_PDE_SHIFT);
--
2.4.0
_______________________________________________
Intel-gfx mailing list
Intel-gfx@xxxxxxxxxxxxxxxxxxxxx
http://lists.freedesktop.org/mailman/listinfo/intel-gfx
