From: Lai Jiangshan <jiangshan.ljs@xxxxxxxxxxxx>
Level expansion occurs when mmu->shadow_root_level > mmu->root_level.
There are several cases that can cuase level expansion:
shadow mmu (shadow paging for 32 bit guest):
case1: gCR0_PG=1,gEFER_LMA=0,gCR4_PSE=0
shadow nested NPT (for 32bit L1 hypervisor):
case2: gCR0_PG=1,gEFER_LMA=0,gCR4_PSE=0,hEFER_LMA=0
case3: gCR0_PG=1,gEFER_LMA=0,hEFER_LMA=1
shadow nested NPT (for 64bit L1 hypervisor):
case4: gEFER_LMA=1,gCR4_LA57=0,hEFER_LMA=1,hCR4_LA57=1
When level expansion occurs (32bit guest, case1-3), special roots are
often used. But case4 is not using special roots. It uses shadow page
without fully aware of the specialty. It might work accidentally:
1) The root_page (root_sp->spt) is allocated with level = 5,
and root_sp->spt[0] is allocated with the same gfn and the
same role except role.level = 4. Luckly that they are
different shadow pages.
2) FNAME(walk_addr_generic) sets walker->table_gfn[4] and
walker->pt_access[4], which are normally unused when
mmu->shadow_root_level == mmu->root_level == 4, so that
FNAME(fetch) can use them to allocate shadow page for
root_sp->spt[0] and link them when shadow_root_level == 5.
But it has problems.
If the guest switches from gCR4_LA57=0 to gCR4_LA57=1 (or vice verse)
and uses the same gfn as the root for the nNPT before and after
switching gCR4_LA57. The host (hCR4_LA57=1) wold use the same root_sp
for guest even guest switches gCR4_LA57. The guest will see unexpected
page mapped and L2 can hurts L1. It is lucky the the problem can't
hurt L0.
The root_sp should be like role.direct=1 sometimes: its contents are
not backed by gptes, root_sp->gfns is meaningless. For a normal high
level sp, sp->gfns is often unused and kept zero, but it could be
relevant and meaningful when sp->gfns is used because they are back
by concret gptes. For expanded root_sp described before, root_sp
is just a portal to contribute root_sp->spt[0], and root_sp should not
have root_sp->gfns and root_sp->spt[0] should not be dropped if gpte[0]
of the root gfn is changed.
This patch adds role.glevel to address the two problems.
With the new role.glevel, passthrough sp can be created for expanded
shadow pagetable: 0 < role.glevel < role.level.
An alternative way to fix the problem of case4 is that: also using the
special root pml5_root for it. But it would required to change many
other places because it is assumption that special roots is only used
for 32bit guests.
This patch also paves the way to use passthrough shadow page for
case1-3, but that requires the special handling or PAE paging, so the
extensive usage of it is in later patches.
Signed-off-by: Lai Jiangshan <jiangshan.ljs@xxxxxxxxxxxx>
---
Documentation/virt/kvm/mmu.rst | 7 +++++++
arch/x86/include/asm/kvm_host.h | 5 +++--
arch/x86/kvm/mmu/mmu.c | 21 +++++++++++++++++----
arch/x86/kvm/mmu/paging_tmpl.h | 1 +
4 files changed, 28 insertions(+), 6 deletions(-)
diff --git a/Documentation/virt/kvm/mmu.rst b/Documentation/virt/kvm/mmu.rst
index 5b1ebad24c77..dee0e96d694a 100644
--- a/Documentation/virt/kvm/mmu.rst
+++ b/Documentation/virt/kvm/mmu.rst
@@ -202,6 +202,13 @@ Shadow pages contain the following information:
Is 1 if the MMU instance cannot use A/D bits. EPT did not have A/D
bits before Haswell; shadow EPT page tables also cannot use A/D bits
if the L1 hypervisor does not enable them.
+ role.glevel:
+ The level in guest pagetable if the sp is indirect. Is 0 if the sp
+ is direct without corresponding guest pagetable, like TDP or !CR0.PG.
+ When role.level > guest paging level, indirect sp is created on the
+ top with role.glevel = guest paging level and acks as passthrough sp
+ and its contents are specially installed rather than the translations
+ of the corresponding guest pagetable.
gfn:
Either the guest page table containing the translations shadowed by this
page, or the base page frame for linear translations. See role.direct.
diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h
index 9694dd5e6ccc..67e1bccaf472 100644
--- a/arch/x86/include/asm/kvm_host.h
+++ b/arch/x86/include/asm/kvm_host.h
@@ -314,7 +314,7 @@ struct kvm_kernel_irq_routing_entry;
* cr0_wp=0, therefore these three bits only give rise to 5 possibilities.
*
* Therefore, the maximum number of possible upper-level shadow pages for a
- * single gfn is a bit less than 2^13.
+ * single gfn is a bit less than 2^15.
*/
union kvm_mmu_page_role {
u32 word;
@@ -331,7 +331,8 @@ union kvm_mmu_page_role {
unsigned smap_andnot_wp:1;
unsigned ad_disabled:1;
unsigned guest_mode:1;
- unsigned :6;
+ unsigned glevel:4;
+ unsigned :2;
/*
* This is left at the top of the word so that
diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c
index 02eae110cbe1..d53037df8177 100644
--- a/arch/x86/kvm/mmu/mmu.c
+++ b/arch/x86/kvm/mmu/mmu.c
@@ -737,8 +737,12 @@ static void mmu_free_pte_list_desc(struct pte_list_desc *pte_list_desc)
static gfn_t kvm_mmu_page_get_gfn(struct kvm_mmu_page *sp, int index)
{
- if (!sp->role.direct)
+ if (!sp->role.direct) {
+ if (unlikely(sp->role.glevel < sp->role.level))
+ return sp->gfn;
+
return sp->gfns[index];
+ }
return sp->gfn + (index << ((sp->role.level - 1) * PT64_LEVEL_BITS));
}
@@ -746,6 +750,11 @@ static gfn_t kvm_mmu_page_get_gfn(struct kvm_mmu_page *sp, int index)
static void kvm_mmu_page_set_gfn(struct kvm_mmu_page *sp, int index, gfn_t gfn)
{
if (!sp->role.direct) {
+ if (unlikely(sp->role.glevel < sp->role.level)) {
+ WARN_ON_ONCE(gfn != sp->gfn);
+ return;
+ }
+
sp->gfns[index] = gfn;
return;
}
@@ -1674,8 +1683,7 @@ static void kvm_mmu_free_page(struct kvm_mmu_page *sp)
hlist_del(&sp->hash_link);
list_del(&sp->link);
free_page((unsigned long)sp->spt);
- if (!sp->role.direct)
- free_page((unsigned long)sp->gfns);
+ free_page((unsigned long)sp->gfns);
kmem_cache_free(mmu_page_header_cache, sp);
}
@@ -1713,7 +1721,7 @@ static struct kvm_mmu_page *kvm_mmu_alloc_page(struct kvm_vcpu *vcpu, gfn_t gfn,
sp = kvm_mmu_memory_cache_alloc(&vcpu->arch.mmu_page_header_cache);
sp->spt = kvm_mmu_memory_cache_alloc(&vcpu->arch.mmu_shadow_page_cache);
- if (!role.direct)
+ if (role.glevel == role.level)
sp->gfns = kvm_mmu_memory_cache_alloc(&vcpu->arch.mmu_gfn_array_cache);
set_page_private(virt_to_page(sp->spt), (unsigned long)sp);
@@ -2054,6 +2062,8 @@ static struct kvm_mmu_page *kvm_mmu_get_page(struct kvm_vcpu *vcpu,
quadrant &= (1 << ((PT32_PT_BITS - PT64_PT_BITS) * level)) - 1;
role.quadrant = quadrant;
}
+ if (level < role.glevel)
+ role.glevel = level;
sp_list = &vcpu->kvm->arch.mmu_page_hash[kvm_page_table_hashfn(gfn)];
for_each_valid_sp(vcpu->kvm, sp, sp_list) {
@@ -4817,6 +4827,7 @@ kvm_calc_shadow_root_page_role_common(struct kvm_vcpu *vcpu,
role.base.smep_andnot_wp = role.ext.cr4_smep && !____is_cr0_wp(regs);
role.base.smap_andnot_wp = role.ext.cr4_smap && !____is_cr0_wp(regs);
role.base.has_4_byte_gpte = ____is_cr0_pg(regs) && !____is_cr4_pae(regs);
+ role.base.glevel = role_regs_to_root_level(regs);
return role;
}
@@ -5312,6 +5323,8 @@ static void kvm_mmu_pte_write(struct kvm_vcpu *vcpu, gpa_t gpa,
++vcpu->kvm->stat.mmu_pte_write;
for_each_gfn_indirect_valid_sp(vcpu->kvm, sp, gfn) {
+ if (sp->role.glevel < sp->role.level)
+ continue;
if (detect_write_misaligned(sp, gpa, bytes) ||
detect_write_flooding(sp)) {
kvm_mmu_prepare_zap_page(vcpu->kvm, sp, &invalid_list);
diff --git a/arch/x86/kvm/mmu/paging_tmpl.h b/arch/x86/kvm/mmu/paging_tmpl.h
index 8621188b46df..67489a060eba 100644
--- a/arch/x86/kvm/mmu/paging_tmpl.h
+++ b/arch/x86/kvm/mmu/paging_tmpl.h
@@ -1042,6 +1042,7 @@ static int FNAME(sync_page)(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp)
.level = 0xf,
.access = 0x7,
.quadrant = 0x3,
+ .glevel = 0xf,
};
/*
--
2.19.1.6.gb485710b
