From: Quentin Perret <qperret@xxxxxxxxxx>
Indexed by IPA, so we can efficiently lookup.
Signed-off-by: Quentin Perret <qperret@xxxxxxxxxx>
Signed-off-by: Fuad Tabba <tabba@xxxxxxxxxx>
---
arch/arm64/include/asm/kvm_host.h | 5 +++--
arch/arm64/kvm/mmu.c | 30 ++++++++++++++++++++++++++----
arch/arm64/kvm/pkvm.c | 12 +++++++-----
3 files changed, 36 insertions(+), 11 deletions(-)
diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h
index 2777b0fe1b12..55de71791233 100644
--- a/arch/arm64/include/asm/kvm_host.h
+++ b/arch/arm64/include/asm/kvm_host.h
@@ -207,8 +207,9 @@ struct kvm_smccc_features {
};
struct kvm_pinned_page {
- struct list_head link;
+ struct rb_node node;
struct page *page;
+ u64 ipa;
};
typedef unsigned int pkvm_handle_t;
@@ -216,7 +217,7 @@ typedef unsigned int pkvm_handle_t;
struct kvm_protected_vm {
pkvm_handle_t handle;
struct kvm_hyp_memcache teardown_mc;
- struct list_head pinned_pages;
+ struct rb_root pinned_pages;
bool enabled;
};
diff --git a/arch/arm64/kvm/mmu.c b/arch/arm64/kvm/mmu.c
index ac088dc198e6..f796e092a921 100644
--- a/arch/arm64/kvm/mmu.c
+++ b/arch/arm64/kvm/mmu.c
@@ -337,6 +337,7 @@ static void unmap_stage2_range(struct kvm_s2_mmu *mmu, phys_addr_t start, u64 si
static void pkvm_stage2_flush(struct kvm *kvm)
{
struct kvm_pinned_page *ppage;
+ struct rb_node *node;
/*
* Contrary to stage2_apply_range(), we don't need to check
@@ -344,7 +345,8 @@ static void pkvm_stage2_flush(struct kvm *kvm)
* from a vcpu thread, and the list is only ever freed on VM
* destroy (which only occurs when all vcpu are gone).
*/
- list_for_each_entry(ppage, &kvm->arch.pkvm.pinned_pages, link) {
+ for (node = rb_first(&kvm->arch.pkvm.pinned_pages); node; node = rb_next(node)) {
+ ppage = rb_entry(node, struct kvm_pinned_page, node);
__clean_dcache_guest_page(page_address(ppage->page), PAGE_SIZE);
cond_resched_rwlock_write(&kvm->mmu_lock);
}
@@ -913,7 +915,7 @@ int kvm_init_stage2_mmu(struct kvm *kvm, struct kvm_s2_mmu *mmu, unsigned long t
mmfr0 = read_sanitised_ftr_reg(SYS_ID_AA64MMFR0_EL1);
mmfr1 = read_sanitised_ftr_reg(SYS_ID_AA64MMFR1_EL1);
mmu->vtcr = kvm_get_vtcr(mmfr0, mmfr1, phys_shift);
- INIT_LIST_HEAD(&kvm->arch.pkvm.pinned_pages);
+ kvm->arch.pkvm.pinned_pages = RB_ROOT;
mmu->arch = &kvm->arch;
if (is_protected_kvm_enabled())
@@ -1412,6 +1414,26 @@ static int pkvm_host_map_guest(u64 pfn, u64 gfn)
return (ret == -EPERM) ? -EAGAIN : ret;
}
+static int cmp_ppages(struct rb_node *node, const struct rb_node *parent)
+{
+ struct kvm_pinned_page *a = container_of(node, struct kvm_pinned_page, node);
+ struct kvm_pinned_page *b = container_of(parent, struct kvm_pinned_page, node);
+
+ if (a->ipa < b->ipa)
+ return -1;
+ if (a->ipa > b->ipa)
+ return 1;
+ return 0;
+}
+
+static int insert_ppage(struct kvm *kvm, struct kvm_pinned_page *ppage)
+{
+ if (rb_find_add(&ppage->node, &kvm->arch.pkvm.pinned_pages, cmp_ppages))
+ return -EEXIST;
+
+ return 0;
+}
+
static int pkvm_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa,
struct kvm_memory_slot *memslot)
{
@@ -1479,8 +1501,8 @@ static int pkvm_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa,
}
ppage->page = page;
- INIT_LIST_HEAD(&ppage->link);
- list_add(&ppage->link, &kvm->arch.pkvm.pinned_pages);
+ ppage->ipa = fault_ipa;
+ WARN_ON(insert_ppage(kvm, ppage));
write_unlock(&kvm->mmu_lock);
return 0;
diff --git a/arch/arm64/kvm/pkvm.c b/arch/arm64/kvm/pkvm.c
index 10a619b257c4..11355980e18d 100644
--- a/arch/arm64/kvm/pkvm.c
+++ b/arch/arm64/kvm/pkvm.c
@@ -246,9 +246,9 @@ static bool pkvm_teardown_vm(struct kvm *host_kvm)
void pkvm_destroy_hyp_vm(struct kvm *host_kvm)
{
- struct kvm_pinned_page *ppage, *tmp;
+ struct kvm_pinned_page *ppage;
struct mm_struct *mm = current->mm;
- struct list_head *ppages;
+ struct rb_node *node;
unsigned long pages = 0;
if (!pkvm_teardown_vm(host_kvm))
@@ -256,14 +256,16 @@ void pkvm_destroy_hyp_vm(struct kvm *host_kvm)
free_hyp_memcache(&host_kvm->arch.pkvm.teardown_mc);
- ppages = &host_kvm->arch.pkvm.pinned_pages;
- list_for_each_entry_safe(ppage, tmp, ppages, link) {
+ node = rb_first(&host_kvm->arch.pkvm.pinned_pages);
+ while (node) {
+ ppage = rb_entry(node, struct kvm_pinned_page, node);
WARN_ON(kvm_call_hyp_nvhe(__pkvm_host_reclaim_page,
page_to_pfn(ppage->page)));
cond_resched();
unpin_user_pages_dirty_lock(&ppage->page, 1, true);
- list_del(&ppage->link);
+ node = rb_next(node);
+ rb_erase(&ppage->node, &host_kvm->arch.pkvm.pinned_pages);
kfree(ppage);
pages++;
}
--
2.44.0.rc1.240.g4c46232300-goog
