From: David Stevens <stevensd@xxxxxxxxxxxx>
Avoid converting pfns returned by follow_fault_pfn to struct pages to
transiently take a reference. The reference was originally taken to
match the reference taken by gup. However, pfns returned by
follow_fault_pfn may not have a struct page set up for reference
counting.
Signed-off-by: David Stevens <stevensd@xxxxxxxxxxxx>
---
arch/arm64/kvm/mmu.c | 43 +++++++++++++++++++++++--------------------
1 file changed, 23 insertions(+), 20 deletions(-)
diff --git a/arch/arm64/kvm/mmu.c b/arch/arm64/kvm/mmu.c
index 896b3644b36f..a741972cb75f 100644
--- a/arch/arm64/kvm/mmu.c
+++ b/arch/arm64/kvm/mmu.c
@@ -779,17 +779,17 @@ static bool fault_supports_stage2_huge_mapping(struct kvm_memory_slot *memslot,
*/
static unsigned long
transparent_hugepage_adjust(struct kvm_memory_slot *memslot,
- unsigned long hva, kvm_pfn_t *pfnp,
+ unsigned long hva, struct kvm_pfn_page *pfnpgp,
phys_addr_t *ipap)
{
- kvm_pfn_t pfn = *pfnp;
+ kvm_pfn_t pfn = pfnpgp->pfn;
/*
* Make sure the adjustment is done only for THP pages. Also make
* sure that the HVA and IPA are sufficiently aligned and that the
* block map is contained within the memslot.
*/
- if (kvm_is_transparent_hugepage(pfn) &&
+ if (pfnpgp->page && kvm_is_transparent_hugepage(pfn) &&
fault_supports_stage2_huge_mapping(memslot, hva, PMD_SIZE)) {
/*
* The address we faulted on is backed by a transparent huge
@@ -810,10 +810,11 @@ transparent_hugepage_adjust(struct kvm_memory_slot *memslot,
* page accordingly.
*/
*ipap &= PMD_MASK;
- kvm_release_pfn_clean(pfn);
+ put_page(pfnpgp->page);
pfn &= ~(PTRS_PER_PMD - 1);
- kvm_get_pfn(pfn);
- *pfnp = pfn;
+ pfnpgp->pfn = pfn;
+ pfnpgp->page = pfn_to_page(pfnpgp->pfn);
+ get_page(pfnpgp->page);
return PMD_SIZE;
}
@@ -836,7 +837,7 @@ static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa,
struct vm_area_struct *vma;
short vma_shift;
gfn_t gfn;
- kvm_pfn_t pfn;
+ struct kvm_pfn_page pfnpg;
bool logging_active = memslot_is_logging(memslot);
unsigned long fault_level = kvm_vcpu_trap_get_fault_level(vcpu);
unsigned long vma_pagesize, fault_granule;
@@ -933,17 +934,16 @@ static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa,
*/
smp_rmb();
- pfn = kvm_pfn_page_unwrap(__gfn_to_pfn_memslot(memslot, gfn, false,
- NULL, write_fault,
- &writable, NULL));
- if (pfn == KVM_PFN_ERR_HWPOISON) {
+ pfnpg = __gfn_to_pfn_memslot(memslot, gfn, false, NULL,
+ write_fault, &writable, NULL);
+ if (pfnpg.pfn == KVM_PFN_ERR_HWPOISON) {
kvm_send_hwpoison_signal(hva, vma_shift);
return 0;
}
- if (is_error_noslot_pfn(pfn))
+ if (is_error_noslot_pfn(pfnpg.pfn))
return -EFAULT;
- if (kvm_is_device_pfn(pfn)) {
+ if (kvm_is_device_pfn(pfnpg.pfn)) {
device = true;
force_pte = true;
} else if (logging_active && !write_fault) {
@@ -968,16 +968,16 @@ static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa,
*/
if (vma_pagesize == PAGE_SIZE && !force_pte)
vma_pagesize = transparent_hugepage_adjust(memslot, hva,
- &pfn, &fault_ipa);
+ &pfnpg, &fault_ipa);
if (writable)
prot |= KVM_PGTABLE_PROT_W;
if (fault_status != FSC_PERM && !device)
- clean_dcache_guest_page(pfn, vma_pagesize);
+ clean_dcache_guest_page(pfnpg.pfn, vma_pagesize);
if (exec_fault) {
prot |= KVM_PGTABLE_PROT_X;
- invalidate_icache_guest_page(pfn, vma_pagesize);
+ invalidate_icache_guest_page(pfnpg.pfn, vma_pagesize);
}
if (device)
@@ -994,20 +994,23 @@ static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa,
ret = kvm_pgtable_stage2_relax_perms(pgt, fault_ipa, prot);
} else {
ret = kvm_pgtable_stage2_map(pgt, fault_ipa, vma_pagesize,
- __pfn_to_phys(pfn), prot,
+ __pfn_to_phys(pfnpg.pfn), prot,
memcache);
}
/* Mark the page dirty only if the fault is handled successfully */
if (writable && !ret) {
- kvm_set_pfn_dirty(pfn);
+ if (pfnpg.page)
+ kvm_set_pfn_dirty(pfnpg.pfn);
mark_page_dirty_in_slot(kvm, memslot, gfn);
}
out_unlock:
spin_unlock(&kvm->mmu_lock);
- kvm_set_pfn_accessed(pfn);
- kvm_release_pfn_clean(pfn);
+ if (pfnpg.page) {
+ kvm_set_pfn_accessed(pfnpg.pfn);
+ put_page(pfnpg.page);
+ }
return ret != -EAGAIN ? ret : 0;
}
--
2.32.0.93.g670b81a890-goog
