Define handle_loadiwkey() VM-Exit handler, which fetch the IWKey guest's
setting and do it onbehalf. Note LOADIWKEY needs CR4.KeyLocker set.
Trap guest write on MSRs of IA32_COPY_LOCAL_TO_PLATFORM and
IA32_COPY_PLATFORM_TO_LOCAL_TO_PLATFORM, emulate IWKey save and restore
operations.
Trap guest read on MSRs of IA32_COPY_STATUS and IA32_IWKEYBACKUP_STATUS,
return their shadow values stored in kvm_vcpu_arch and kvm_arch.
Also, guest CPUID.0x19:EBX[0] is dynamic, its status changes as CR4.KL
changes.
On each VM-Entry, we need to resotre vCPU's IWKey, stored in kvm_vcpu_arch.
Signed-off-by: Robert Hoo <robert.hu@xxxxxxxxxxxxxxx>
---
arch/x86/include/asm/kvm_host.h | 24 ++++-
arch/x86/kvm/cpuid.c | 11 +++
arch/x86/kvm/cpuid.h | 2 +
arch/x86/kvm/vmx/vmx.c | 189 ++++++++++++++++++++++++++++++++++++++++
arch/x86/kvm/x86.c | 4 +-
arch/x86/kvm/x86.h | 2 +
6 files changed, 229 insertions(+), 3 deletions(-)
diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h
index 7e5f33a..dc09142 100644
--- a/arch/x86/include/asm/kvm_host.h
+++ b/arch/x86/include/asm/kvm_host.h
@@ -100,7 +100,7 @@
| X86_CR4_PGE | X86_CR4_PCE | X86_CR4_OSFXSR | X86_CR4_PCIDE \
| X86_CR4_OSXSAVE | X86_CR4_SMEP | X86_CR4_FSGSBASE \
| X86_CR4_OSXMMEXCPT | X86_CR4_LA57 | X86_CR4_VMXE \
- | X86_CR4_SMAP | X86_CR4_PKE | X86_CR4_UMIP))
+ | X86_CR4_SMAP | X86_CR4_PKE | X86_CR4_UMIP | X86_CR4_KEYLOCKER))
#define CR8_RESERVED_BITS (~(unsigned long)X86_CR8_TPR)
@@ -520,6 +520,19 @@ struct kvm_vcpu_hv {
cpumask_t tlb_flush;
};
+#if defined(CONFIG_ARCH_SUPPORTS_INT128) && defined(CONFIG_CC_HAS_INT128)
+typedef unsigned __int128 u128;
+#else
+typedef struct {
+ u64 reg64[2];
+} u128;
+#endif
+
+struct iwkey {
+ u128 encryption_key[2]; /* 256bit encryption key */
+ u128 integrity_key; /* 128bit integration key */
+};
+
struct kvm_vcpu_arch {
/*
* rip and regs accesses must go through
@@ -805,6 +818,11 @@ struct kvm_vcpu_arch {
*/
bool enforce;
} pv_cpuid;
+
+ /* Intel KeyLocker */
+ bool iwkey_loaded;
+ struct iwkey iwkey;
+ u32 msr_ia32_copy_status;
};
struct kvm_lpage_info {
@@ -931,6 +949,10 @@ struct kvm_arch {
bool apic_access_page_done;
unsigned long apicv_inhibit_reasons;
+ bool iwkey_backup_valid;
+ u32 msr_ia32_iwkey_backup_status;
+ struct iwkey iwkey_backup;
+
gpa_t wall_clock;
bool mwait_in_guest;
diff --git a/arch/x86/kvm/cpuid.c b/arch/x86/kvm/cpuid.c
index 83637a2..2fbf4af 100644
--- a/arch/x86/kvm/cpuid.c
+++ b/arch/x86/kvm/cpuid.c
@@ -133,6 +133,12 @@ void kvm_update_cpuid_runtime(struct kvm_vcpu *vcpu)
cpuid_entry_has(best, X86_FEATURE_XSAVEC)))
best->ebx = xstate_required_size(vcpu->arch.xcr0, true);
+ /* update CPUID.0x19.EBX[0], depends on CR4.KL */
+ best = kvm_find_cpuid_entry(vcpu, 0x19, 0);
+ if (best)
+ cpuid_entry_change(best, X86_FEATURE_KL_INS_ENABLED,
+ kvm_read_cr4_bits(vcpu, X86_CR4_KEYLOCKER));
+
best = kvm_find_cpuid_entry(vcpu, KVM_CPUID_FEATURES, 0);
if (kvm_hlt_in_guest(vcpu->kvm) && best &&
(best->eax & (1 << KVM_FEATURE_PV_UNHALT)))
@@ -407,6 +413,11 @@ void kvm_set_cpu_caps(void)
if (cpuid_ecx(7) & F(LA57))
kvm_cpu_cap_set(X86_FEATURE_LA57);
+ /* At present, host and guest can only exclusively use KeyLocker */
+ if (!boot_cpu_has(X86_FEATURE_KEYLOCKER) && (cpuid_ecx(0x7) &
+ feature_bit(KEYLOCKER)))
+ kvm_cpu_cap_set(X86_FEATURE_KEYLOCKER);
+
/*
* PKU not yet implemented for shadow paging and requires OSPKE
* to be set on the host. Clear it if that is not the case
diff --git a/arch/x86/kvm/cpuid.h b/arch/x86/kvm/cpuid.h
index f7a6e8f..639c647 100644
--- a/arch/x86/kvm/cpuid.h
+++ b/arch/x86/kvm/cpuid.h
@@ -63,6 +63,8 @@ struct cpuid_reg {
[CPUID_8000_0007_EBX] = {0x80000007, 0, CPUID_EBX},
[CPUID_7_EDX] = { 7, 0, CPUID_EDX},
[CPUID_7_1_EAX] = { 7, 1, CPUID_EAX},
+ [CPUID_19_EBX] = { 0x19, 0, CPUID_EBX},
+ [CPUID_19_ECX] = { 0x19, 0, CPUID_ECX},
};
/*
diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c
index d01bbb4..6be6d87 100644
--- a/arch/x86/kvm/vmx/vmx.c
+++ b/arch/x86/kvm/vmx/vmx.c
@@ -47,6 +47,7 @@
#include <asm/spec-ctrl.h>
#include <asm/virtext.h>
#include <asm/vmx.h>
+#include <asm/keylocker.h>
#include "capabilities.h"
#include "cpuid.h"
@@ -1192,6 +1193,106 @@ void vmx_set_host_fs_gs(struct vmcs_host_state *host, u16 fs_sel, u16 gs_sel,
}
}
+static int get_xmm(int index, u128 *mem_ptr)
+{
+ int ret = 0;
+
+ asm ("cli");
+ switch (index) {
+ case 0:
+ asm ("movdqu %%xmm0, %0" : : "m"(*mem_ptr));
+ break;
+ case 1:
+ asm ("movdqu %%xmm1, %0" : : "m"(*mem_ptr));
+ break;
+ case 2:
+ asm ("movdqu %%xmm2, %0" : : "m"(*mem_ptr));
+ break;
+ case 3:
+ asm ("movdqu %%xmm3, %0" : : "m"(*mem_ptr));
+ break;
+ case 4:
+ asm ("movdqu %%xmm4, %0" : : "m"(*mem_ptr));
+ break;
+ case 5:
+ asm ("movdqu %%xmm5, %0" : : "m"(*mem_ptr));
+ break;
+ case 6:
+ asm ("movdqu %%xmm6, %0" : : "m"(*mem_ptr));
+ break;
+ case 7:
+ asm ("movdqu %%xmm7, %0" : : "m"(*mem_ptr));
+ break;
+#ifdef CONFIG_X86_64
+ case 8:
+ asm ("movdqu %%xmm8, %0" : : "m"(*mem_ptr));
+ break;
+ case 9:
+ asm ("movdqu %%xmm9, %0" : : "m"(*mem_ptr));
+ break;
+ case 10:
+ asm ("movdqu %%xmm10, %0" : : "m"(*mem_ptr));
+ break;
+ case 11:
+ asm ("movdqu %%xmm11, %0" : : "m"(*mem_ptr));
+ break;
+ case 12:
+ asm ("movdqu %%xmm12, %0" : : "m"(*mem_ptr));
+ break;
+ case 13:
+ asm ("movdqu %%xmm13, %0" : : "m"(*mem_ptr));
+ break;
+ case 14:
+ asm ("movdqu %%xmm14, %0" : : "m"(*mem_ptr));
+ break;
+ case 15:
+ asm ("movdqu %%xmm15, %0" : : "m"(*mem_ptr));
+ break;
+#endif
+ default:
+ pr_err_once("xmm index exceeds");
+ ret = -1;
+ break;
+ }
+ asm ("sti");
+
+ return ret;
+}
+
+static void vmx_load_guest_iwkey(struct kvm_vcpu *vcpu)
+{
+ u128 xmm[3] = {0};
+
+ if (vcpu->arch.iwkey_loaded) {
+ bool clear_cr4 = false;
+ /* Save origin %xmm */
+ get_xmm(0, &xmm[0]);
+ get_xmm(1, &xmm[1]);
+ get_xmm(2, &xmm[2]);
+
+ asm ("movdqu %0, %%xmm0;"
+ "movdqu %1, %%xmm1;"
+ "movdqu %2, %%xmm2;"
+ : : "m"(vcpu->arch.iwkey.integrity_key),
+ "m"(vcpu->arch.iwkey.encryption_key[0]),
+ "m"(vcpu->arch.iwkey.encryption_key[1]));
+ if (!(cr4_read_shadow() & X86_CR4_KEYLOCKER)) {
+ cr4_set_bits(X86_CR4_KEYLOCKER);
+ clear_cr4 = true;
+ }
+ asm volatile(LOADIWKEY : : "a" (0x0));
+ if (clear_cr4)
+ cr4_clear_bits(X86_CR4_KEYLOCKER);
+ /* restore %xmm */
+ asm ("movdqu %0, %%xmm0;"
+ "movdqu %1, %%xmm1;"
+ "movdqu %2, %%xmm2;"
+ : : "m"(xmm[0]),
+ "m"(xmm[1]),
+ "m"(xmm[2]));
+ }
+}
+
void vmx_prepare_switch_to_guest(struct kvm_vcpu *vcpu)
{
struct vcpu_vmx *vmx = to_vmx(vcpu);
@@ -1260,6 +1361,9 @@ void vmx_prepare_switch_to_guest(struct kvm_vcpu *vcpu)
#endif
vmx_set_host_fs_gs(host_state, fs_sel, gs_sel, fs_base, gs_base);
+
+ vmx_load_guest_iwkey(vcpu);
+
vmx->guest_state_loaded = true;
}
@@ -1925,6 +2029,19 @@ static int vmx_get_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
!guest_cpuid_has(vcpu, X86_FEATURE_RDTSCP))
return 1;
goto find_uret_msr;
+ case MSR_IA32_COPY_STATUS:
+ if (!guest_cpuid_has(vcpu, X86_FEATURE_KEYLOCKER))
+ return 1;
+
+ msr_info->data = vcpu->arch.msr_ia32_copy_status;
+ break;
+
+ case MSR_IA32_IWKEYBACKUP_STATUS:
+ if (!guest_cpuid_has(vcpu, X86_FEATURE_KEYLOCKER))
+ return 1;
+
+ msr_info->data = vcpu->kvm->arch.msr_ia32_iwkey_backup_status;
+ break;
default:
find_uret_msr:
msr = vmx_find_uret_msr(vmx, msr_info->index);
@@ -2189,6 +2306,36 @@ static int vmx_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
else
vmx->pt_desc.guest.addr_a[index / 2] = data;
break;
+ case MSR_IA32_COPY_LOCAL_TO_PLATFORM:
+ if (msr_info->data != 1)
+ return 1;
+
+ if (!guest_cpuid_has(vcpu, X86_FEATURE_KEYLOCKER))
+ return 1;
+
+ if (!vcpu->arch.iwkey_loaded)
+ return 1;
+
+ if (!vcpu->kvm->arch.iwkey_backup_valid) {
+ vcpu->kvm->arch.iwkey_backup = vcpu->arch.iwkey;
+ vcpu->kvm->arch.iwkey_backup_valid = true;
+ vcpu->kvm->arch.msr_ia32_iwkey_backup_status = 0x9;
+ }
+ vcpu->arch.msr_ia32_copy_status = 1;
+ break;
+
+ case MSR_IA32_COPY_PLATFORM_TO_LOCAL:
+ if (msr_info->data != 1)
+ return 1;
+
+ if (!guest_cpuid_has(vcpu, X86_FEATURE_KEYLOCKER))
+ return 1;
+ if (!vcpu->kvm->arch.iwkey_backup_valid)
+ return 1;
+ vcpu->arch.iwkey = vcpu->kvm->arch.iwkey_backup;
+ vcpu->arch.msr_ia32_copy_status = 1;
+ break;
+
case MSR_TSC_AUX:
if (!msr_info->host_initiated &&
!guest_cpuid_has(vcpu, X86_FEATURE_RDTSCP))
@@ -5659,6 +5806,47 @@ static int handle_encls(struct kvm_vcpu *vcpu)
return 1;
}
+static int handle_loadiwkey(struct kvm_vcpu *vcpu)
+{
+ u128 xmm[3] = {0};
+ u32 vmx_instruction_info;
+ int reg1, reg2;
+ int r;
+
+ if (!guest_cpuid_has(vcpu, X86_FEATURE_KEYLOCKER)) {
+ kvm_queue_exception(vcpu, UD_VECTOR);
+ return 1;
+ }
+
+ vmx_instruction_info = vmcs_read32(VMX_INSTRUCTION_INFO);
+ reg1 = (vmx_instruction_info & 0x78) >> 3;
+ reg2 = (vmx_instruction_info >> 28) & 0xf;
+
+ /*
+ * vmx instruction info on Current TGL is broken.
+ * Before the microcode fix, we hardcode XMM1 & XMM2 holding
+ * IWKey.encryption_key.
+ */
+ reg1 = 1;
+ reg2 = 2;
+ r = get_xmm(0, &xmm[0]);
+ if (r)
+ return 0;
+ r = get_xmm(reg1, &xmm[1]);
+ if (r)
+ return 0;
+ r = get_xmm(reg2, &xmm[2]);
+ if (r)
+ return 0;
+
+ vcpu->arch.iwkey.integrity_key = xmm[0];
+ vcpu->arch.iwkey.encryption_key[0] = xmm[1];
+ vcpu->arch.iwkey.encryption_key[1] = xmm[2];
+ vcpu->arch.iwkey_loaded = true;
+
+ return kvm_skip_emulated_instruction(vcpu);
+}
+
/*
* The exit handlers return 1 if the exit was handled fully and guest execution
* may resume. Otherwise they set the kvm_run parameter to indicate what needs
@@ -5715,6 +5903,7 @@ static int (*kvm_vmx_exit_handlers[])(struct kvm_vcpu *vcpu) = {
[EXIT_REASON_VMFUNC] = handle_vmx_instruction,
[EXIT_REASON_PREEMPTION_TIMER] = handle_preemption_timer,
[EXIT_REASON_ENCLS] = handle_encls,
+ [EXIT_REASON_LOADIWKEY] = handle_loadiwkey,
};
static const int kvm_vmx_max_exit_handlers =
diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c
index e545a8a..fbc839a 100644
--- a/arch/x86/kvm/x86.c
+++ b/arch/x86/kvm/x86.c
@@ -1013,7 +1013,7 @@ int kvm_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
(!(cr4 & X86_CR4_PCIDE) && (old_cr4 & X86_CR4_PCIDE)))
kvm_mmu_reset_context(vcpu);
- if ((cr4 ^ old_cr4) & (X86_CR4_OSXSAVE | X86_CR4_PKE))
+ if ((cr4 ^ old_cr4) & (X86_CR4_OSXSAVE | X86_CR4_PKE | X86_CR4_KEYLOCKER))
kvm_update_cpuid_runtime(vcpu);
return 0;
@@ -9598,7 +9598,7 @@ static int __set_sregs(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs)
mmu_reset_needed |= kvm_read_cr4(vcpu) != sregs->cr4;
cpuid_update_needed |= ((kvm_read_cr4(vcpu) ^ sregs->cr4) &
- (X86_CR4_OSXSAVE | X86_CR4_PKE));
+ (X86_CR4_OSXSAVE | X86_CR4_PKE | X86_CR4_KEYLOCKER));
kvm_x86_ops.set_cr4(vcpu, sregs->cr4);
if (cpuid_update_needed)
kvm_update_cpuid_runtime(vcpu);
diff --git a/arch/x86/kvm/x86.h b/arch/x86/kvm/x86.h
index e7ca622..0e6b826 100644
--- a/arch/x86/kvm/x86.h
+++ b/arch/x86/kvm/x86.h
@@ -404,6 +404,8 @@ int kvm_handle_memory_failure(struct kvm_vcpu *vcpu, int r,
__reserved_bits |= X86_CR4_UMIP; \
if (!__cpu_has(__c, X86_FEATURE_VMX)) \
__reserved_bits |= X86_CR4_VMXE; \
+ if (!__cpu_has(__c, X86_FEATURE_KEYLOCKER)) \
+ __reserved_bits |= X86_CR4_KEYLOCKER; \
__reserved_bits; \
})
--
1.8.3.1
