From: Xiaoyao Li <xiaoyao.li@xxxxxxxxx>
Allow large page level AUG and REMOVE for TDX pages.
Signed-off-by: Xiaoyao Li <xiaoyao.li@xxxxxxxxx>
Signed-off-by: Isaku Yamahata <isaku.yamahata@xxxxxxxxx>
---
arch/x86/kvm/vmx/tdx.c | 70 ++++++++++++++++++++++--------------------
1 file changed, 36 insertions(+), 34 deletions(-)
diff --git a/arch/x86/kvm/vmx/tdx.c b/arch/x86/kvm/vmx/tdx.c
index 72672b2c30a1..992cf3ed02f2 100644
--- a/arch/x86/kvm/vmx/tdx.c
+++ b/arch/x86/kvm/vmx/tdx.c
@@ -1435,11 +1435,12 @@ static void tdx_measure_page(struct kvm_tdx *kvm_tdx, hpa_t gpa, int size)
}
}
-static void tdx_unpin(struct kvm *kvm, kvm_pfn_t pfn)
+static void tdx_unpin(struct kvm *kvm, kvm_pfn_t pfn, int level)
{
- struct page *page = pfn_to_page(pfn);
+ int i;
- put_page(page);
+ for (i = 0; i < KVM_PAGES_PER_HPAGE(level); i++)
+ put_page(pfn_to_page(pfn + i));
}
static int tdx_sept_page_aug(struct kvm *kvm, gfn_t gfn,
@@ -1456,7 +1457,7 @@ static int tdx_sept_page_aug(struct kvm *kvm, gfn_t gfn,
err = tdh_mem_page_aug(kvm_tdx->tdr_pa, gpa, tdx_level, hpa, &out);
if (unlikely(err == TDX_ERROR_SEPT_BUSY)) {
- tdx_unpin(kvm, pfn);
+ tdx_unpin(kvm, pfn, level);
return -EAGAIN;
}
if (unlikely(err == (TDX_EPT_ENTRY_NOT_FREE | TDX_OPERAND_ID_RCX))) {
@@ -1472,7 +1473,7 @@ static int tdx_sept_page_aug(struct kvm *kvm, gfn_t gfn,
&tmpout);
if (KVM_BUG_ON(tmp, kvm)) {
pr_tdx_error(TDH_MEM_SEPT_RD, tmp, &tmpout);
- tdx_unpin(kvm, pfn);
+ tdx_unpin(kvm, pfn, level);
return -EIO;
}
pr_debug_ratelimited("gfn 0x%llx pg_level %d pfn 0x%llx entry 0x%llx level_stat 0x%llx\n",
@@ -1483,7 +1484,7 @@ static int tdx_sept_page_aug(struct kvm *kvm, gfn_t gfn,
if (level_state.level == tdx_level &&
level_state.state == TDX_SEPT_PENDING &&
entry.leaf && entry.pfn == pfn && entry.sve) {
- tdx_unpin(kvm, pfn);
+ tdx_unpin(kvm, pfn, level);
WARN_ON_ONCE(!(to_kvm_tdx(kvm)->attributes &
TDX_TD_ATTR_SEPT_VE_DISABLE));
return -EAGAIN;
@@ -1491,7 +1492,7 @@ static int tdx_sept_page_aug(struct kvm *kvm, gfn_t gfn,
}
if (KVM_BUG_ON(err, kvm)) {
pr_tdx_error(TDH_MEM_PAGE_AUG, err, &out);
- tdx_unpin(kvm, pfn);
+ tdx_unpin(kvm, pfn, level);
return -EIO;
}
@@ -1527,7 +1528,7 @@ static int tdx_sept_page_add(struct kvm *kvm, gfn_t gfn,
* always uses vcpu 0's page table and protected by vcpu->mutex).
*/
if (KVM_BUG_ON(kvm_tdx->source_pa == INVALID_PAGE, kvm)) {
- tdx_unpin(kvm, pfn);
+ tdx_unpin(kvm, pfn, level);
return -EINVAL;
}
@@ -1545,7 +1546,7 @@ static int tdx_sept_page_add(struct kvm *kvm, gfn_t gfn,
} while (unlikely(err == TDX_ERROR_SEPT_BUSY));
if (KVM_BUG_ON(err, kvm)) {
pr_tdx_error(TDH_MEM_PAGE_ADD, err, &out);
- tdx_unpin(kvm, pfn);
+ tdx_unpin(kvm, pfn, level);
return -EIO;
} else if (measure)
tdx_measure_page(kvm_tdx, gpa, KVM_HPAGE_SIZE(level));
@@ -1558,10 +1559,7 @@ static int tdx_sept_set_private_spte(struct kvm *kvm, gfn_t gfn,
enum pg_level level, kvm_pfn_t pfn)
{
struct kvm_tdx *kvm_tdx = to_kvm_tdx(kvm);
-
- /* TODO: handle large pages. */
- if (KVM_BUG_ON(level != PG_LEVEL_4K, kvm))
- return -EINVAL;
+ int i;
/*
* Because restricted mem doesn't support page migration with
@@ -1571,7 +1569,8 @@ static int tdx_sept_set_private_spte(struct kvm *kvm, gfn_t gfn,
* TODO: Once restricted mem introduces callback on page migration,
* implement it and remove get_page/put_page().
*/
- get_page(pfn_to_page(pfn));
+ for (i = 0; i < KVM_PAGES_PER_HPAGE(level); i++)
+ get_page(pfn_to_page(pfn + i));
if (likely(is_td_finalized(kvm_tdx)))
return tdx_sept_page_aug(kvm, gfn, level, pfn);
@@ -1588,11 +1587,9 @@ static int tdx_sept_drop_private_spte(struct kvm *kvm, gfn_t gfn,
gpa_t gpa = gfn_to_gpa(gfn);
hpa_t hpa = pfn_to_hpa(pfn);
hpa_t hpa_with_hkid;
+ int r = 0;
u64 err;
-
- /* TODO: handle large pages. */
- if (KVM_BUG_ON(level != PG_LEVEL_4K, kvm))
- return -EINVAL;
+ int i;
if (unlikely(!is_hkid_assigned(kvm_tdx))) {
/*
@@ -1602,7 +1599,7 @@ static int tdx_sept_drop_private_spte(struct kvm *kvm, gfn_t gfn,
err = tdx_reclaim_page(hpa, level);
if (KVM_BUG_ON(err, kvm))
return -EIO;
- tdx_unpin(kvm, pfn);
+ tdx_unpin(kvm, pfn, level);
return 0;
}
@@ -1619,22 +1616,27 @@ static int tdx_sept_drop_private_spte(struct kvm *kvm, gfn_t gfn,
return -EIO;
}
- hpa_with_hkid = set_hkid_to_hpa(hpa, (u16)kvm_tdx->hkid);
- do {
- /*
- * TDX_OPERAND_BUSY can happen on locking PAMT entry. Because
- * this page was removed above, other thread shouldn't be
- * repeatedly operating on this page. Just retry loop.
- */
- err = tdh_phymem_page_wbinvd(hpa_with_hkid);
- } while (unlikely(err == (TDX_OPERAND_BUSY | TDX_OPERAND_ID_RCX)));
- if (KVM_BUG_ON(err, kvm)) {
- pr_tdx_error(TDH_PHYMEM_PAGE_WBINVD, err, NULL);
- return -EIO;
+ for (i = 0; i < KVM_PAGES_PER_HPAGE(level); i++) {
+ hpa_with_hkid = set_hkid_to_hpa(hpa, (u16)kvm_tdx->hkid);
+ do {
+ /*
+ * TDX_OPERAND_BUSY can happen on locking PAMT entry.
+ * Because this page was removed above, other thread
+ * shouldn't be repeatedly operating on this page.
+ * Simple retry should work.
+ */
+ err = tdh_phymem_page_wbinvd(hpa_with_hkid);
+ } while (unlikely(err == (TDX_OPERAND_BUSY | TDX_OPERAND_ID_RCX)));
+ if (KVM_BUG_ON(err, kvm)) {
+ pr_tdx_error(TDH_PHYMEM_PAGE_WBINVD, err, NULL);
+ r = -EIO;
+ } else {
+ tdx_clear_page(hpa, PAGE_SIZE);
+ tdx_unpin(kvm, pfn + i, PG_LEVEL_4K);
+ }
+ hpa += PAGE_SIZE;
}
- tdx_clear_page(hpa, PAGE_SIZE);
- tdx_unpin(kvm, pfn);
- return 0;
+ return r;
}
static int tdx_sept_link_private_spt(struct kvm *kvm, gfn_t gfn,
--
2.25.1
