Allow the pending and the injected exceptions to co-exist
when they are raised.
Add 'kvm_deliver_pending_exception' function which 'merges' the pending
and injected exception or delivers a VM exit with both for a case when
the L1 intercepts the pending exception.
The later is done by vendor code using new nested callback
'deliver_exception_as_vmexit'
The kvm_deliver_pending_exception is called after each VM exit,
and prior to VM entry which ensures that during userspace VM exits,
only injected exception can be in a raised state.
Signed-off-by: Maxim Levitsky <mlevitsk@xxxxxxxxxx>
---
arch/x86/include/asm/kvm_host.h | 9 ++
arch/x86/kvm/svm/nested.c | 27 ++--
arch/x86/kvm/svm/svm.c | 2 +-
arch/x86/kvm/vmx/nested.c | 58 ++++----
arch/x86/kvm/vmx/vmx.c | 2 +-
arch/x86/kvm/x86.c | 233 ++++++++++++++++++--------------
6 files changed, 181 insertions(+), 150 deletions(-)
diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h
index 3b2fd276e8d5..a9b9cd030d9a 100644
--- a/arch/x86/include/asm/kvm_host.h
+++ b/arch/x86/include/asm/kvm_host.h
@@ -1346,6 +1346,15 @@ struct kvm_x86_ops {
struct kvm_x86_nested_ops {
int (*check_events)(struct kvm_vcpu *vcpu);
+
+ /*
+ * Deliver a pending exception as a VM exit if the L1 intercepts it.
+ * Returns -EBUSY if L1 does intercept the exception but,
+ * it is not possible to deliver it right now.
+ * (for example when nested run is pending)
+ */
+ int (*deliver_exception_as_vmexit)(struct kvm_vcpu *vcpu);
+
bool (*hv_timer_pending)(struct kvm_vcpu *vcpu);
void (*triple_fault)(struct kvm_vcpu *vcpu);
int (*get_state)(struct kvm_vcpu *vcpu,
diff --git a/arch/x86/kvm/svm/nested.c b/arch/x86/kvm/svm/nested.c
index 7adad9b6dcad..ff745d59ffcf 100644
--- a/arch/x86/kvm/svm/nested.c
+++ b/arch/x86/kvm/svm/nested.c
@@ -1061,21 +1061,6 @@ static int svm_check_nested_events(struct kvm_vcpu *vcpu)
return 0;
}
- if (vcpu->arch.pending_exception.valid) {
- /*
- * Only a pending nested run can block a pending exception.
- * Otherwise an injected NMI/interrupt should either be
- * lost or delivered to the nested hypervisor in the EXITINTINFO
- * vmcb field, while delivering the pending exception.
- */
- if (svm->nested.nested_run_pending)
- return -EBUSY;
- if (!nested_exit_on_exception(svm))
- return 0;
- nested_svm_inject_exception_vmexit(svm);
- return 0;
- }
-
if (vcpu->arch.smi_pending && !svm_smi_blocked(vcpu)) {
if (block_nested_events)
return -EBUSY;
@@ -1107,6 +1092,17 @@ static int svm_check_nested_events(struct kvm_vcpu *vcpu)
return 0;
}
+int svm_deliver_nested_exception_as_vmexit(struct kvm_vcpu *vcpu)
+{
+ struct vcpu_svm *svm = to_svm(vcpu);
+
+ if (svm->nested.nested_run_pending)
+ return -EBUSY;
+ if (nested_exit_on_exception(svm))
+ nested_svm_inject_exception_vmexit(svm);
+ return 0;
+}
+
int nested_svm_exit_special(struct vcpu_svm *svm)
{
u32 exit_code = svm->vmcb->control.exit_code;
@@ -1321,6 +1317,7 @@ static int svm_set_nested_state(struct kvm_vcpu *vcpu,
struct kvm_x86_nested_ops svm_nested_ops = {
.check_events = svm_check_nested_events,
.triple_fault = nested_svm_triple_fault,
+ .deliver_exception_as_vmexit = svm_deliver_nested_exception_as_vmexit,
.get_nested_state_pages = svm_get_nested_state_pages,
.get_state = svm_get_nested_state,
.set_state = svm_set_nested_state,
diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c
index 90b541138c5a..b89e48574c39 100644
--- a/arch/x86/kvm/svm/svm.c
+++ b/arch/x86/kvm/svm/svm.c
@@ -363,7 +363,7 @@ static void svm_queue_exception(struct kvm_vcpu *vcpu)
bool has_error_code = vcpu->arch.injected_exception.has_error_code;
u32 error_code = vcpu->arch.injected_exception.error_code;
- kvm_deliver_exception_payload(vcpu);
+ WARN_ON_ONCE(vcpu->arch.pending_exception.valid);
if (nr == BP_VECTOR && !nrips) {
unsigned long rip, old_rip = kvm_rip_read(vcpu);
diff --git a/arch/x86/kvm/vmx/nested.c b/arch/x86/kvm/vmx/nested.c
index 5d54fecff9a7..1c09b132c55c 100644
--- a/arch/x86/kvm/vmx/nested.c
+++ b/arch/x86/kvm/vmx/nested.c
@@ -3768,7 +3768,6 @@ static bool nested_vmx_preemption_timer_pending(struct kvm_vcpu *vcpu)
static int vmx_check_nested_events(struct kvm_vcpu *vcpu)
{
struct vcpu_vmx *vmx = to_vmx(vcpu);
- unsigned long exit_qual;
bool block_nested_events =
vmx->nested.nested_run_pending || kvm_event_needs_reinjection(vcpu);
bool mtf_pending = vmx->nested.mtf_pending;
@@ -3804,41 +3803,15 @@ static int vmx_check_nested_events(struct kvm_vcpu *vcpu)
return 0;
}
- /*
- * Process any exceptions that are not debug traps before MTF.
- *
- * Note that only a pending nested run can block a pending exception.
- * Otherwise an injected NMI/interrupt should either be
- * lost or delivered to the nested hypervisor in the IDT_VECTORING_INFO,
- * while delivering the pending exception.
- */
-
- if (vcpu->arch.pending_exception.valid && !vmx_pending_dbg_trap(vcpu)) {
- if (vmx->nested.nested_run_pending)
- return -EBUSY;
- if (!nested_vmx_check_exception(vcpu, &exit_qual))
- goto no_vmexit;
- nested_vmx_inject_exception_vmexit(vcpu, exit_qual);
- return 0;
- }
-
if (mtf_pending) {
if (block_nested_events)
return -EBUSY;
+
nested_vmx_update_pending_dbg(vcpu);
nested_vmx_vmexit(vcpu, EXIT_REASON_MONITOR_TRAP_FLAG, 0, 0);
return 0;
}
- if (vcpu->arch.pending_exception.valid) {
- if (vmx->nested.nested_run_pending)
- return -EBUSY;
- if (!nested_vmx_check_exception(vcpu, &exit_qual))
- goto no_vmexit;
- nested_vmx_inject_exception_vmexit(vcpu, exit_qual);
- return 0;
- }
-
if (nested_vmx_preemption_timer_pending(vcpu)) {
if (block_nested_events)
return -EBUSY;
@@ -3884,6 +3857,34 @@ static int vmx_check_nested_events(struct kvm_vcpu *vcpu)
return 0;
}
+static int nested_vmx_deliver_exception_as_vmexit(struct kvm_vcpu *vcpu)
+{
+ struct vcpu_vmx *vmx = to_vmx(vcpu);
+ unsigned long exit_qual;
+
+ if (vmx->nested.nested_run_pending)
+ return -EBUSY;
+
+ if (vmx->nested.mtf_pending && vmx_pending_dbg_trap(vcpu)) {
+ /*
+ * A pending monitor trap takes precedence over pending
+ * debug exception which is 'stashed' into
+ * 'GUEST_PENDING_DBG_EXCEPTIONS'
+ */
+
+ nested_vmx_update_pending_dbg(vcpu);
+ vmx->nested.mtf_pending = false;
+ nested_vmx_vmexit(vcpu, EXIT_REASON_MONITOR_TRAP_FLAG, 0, 0);
+ return 0;
+ }
+ if (vcpu->arch.pending_exception.valid) {
+ if (nested_vmx_check_exception(vcpu, &exit_qual))
+ nested_vmx_inject_exception_vmexit(vcpu, exit_qual);
+ return 0;
+ }
+ return 0;
+}
+
static u32 vmx_get_preemption_timer_value(struct kvm_vcpu *vcpu)
{
ktime_t remaining =
@@ -6603,6 +6604,7 @@ __init int nested_vmx_hardware_setup(int (*exit_handlers[])(struct kvm_vcpu *))
struct kvm_x86_nested_ops vmx_nested_ops = {
.check_events = vmx_check_nested_events,
+ .deliver_exception_as_vmexit = nested_vmx_deliver_exception_as_vmexit,
.hv_timer_pending = nested_vmx_preemption_timer_pending,
.triple_fault = nested_vmx_triple_fault,
.get_state = vmx_get_nested_state,
diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c
index a9b241d2b271..fc6bc40d47b0 100644
--- a/arch/x86/kvm/vmx/vmx.c
+++ b/arch/x86/kvm/vmx/vmx.c
@@ -1682,7 +1682,7 @@ static void vmx_queue_exception(struct kvm_vcpu *vcpu)
u32 error_code = vcpu->arch.injected_exception.error_code;
u32 intr_info = nr | INTR_INFO_VALID_MASK;
- kvm_deliver_exception_payload(vcpu);
+ WARN_ON_ONCE(vcpu->arch.pending_exception.valid);
if (has_error_code) {
vmcs_write32(VM_ENTRY_EXCEPTION_ERROR_CODE, error_code);
diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c
index 493d87b0c2d5..a363204f37be 100644
--- a/arch/x86/kvm/x86.c
+++ b/arch/x86/kvm/x86.c
@@ -535,86 +535,30 @@ void kvm_deliver_exception_payload(struct kvm_vcpu *vcpu)
EXPORT_SYMBOL_GPL(kvm_deliver_exception_payload);
static void kvm_multiple_exception(struct kvm_vcpu *vcpu,
- unsigned nr, bool has_error, u32 error_code,
- bool has_payload, unsigned long payload, bool reinject)
+ unsigned int nr, bool has_error, u32 error_code,
+ bool has_payload, unsigned long payload,
+ bool reinject)
{
- u32 prev_nr;
- int class1, class2;
-
+ struct kvm_queued_exception *exc;
kvm_make_request(KVM_REQ_EVENT, vcpu);
- if (!vcpu->arch.pending_exception.valid && !vcpu->arch.injected_exception.valid) {
- queue:
- if (reinject) {
- /*
- * On vmentry, vcpu->arch.exception.pending is only
- * true if an event injection was blocked by
- * nested_run_pending. In that case, however,
- * vcpu_enter_guest requests an immediate exit,
- * and the guest shouldn't proceed far enough to
- * need reinjection.
- */
- WARN_ON_ONCE(vcpu->arch.pending_exception.valid);
- if (WARN_ON_ONCE(has_payload)) {
- /*
- * A reinjected event has already
- * delivered its payload.
- */
- has_payload = false;
- payload = 0;
- }
-
- vcpu->arch.injected_exception.valid = true;
- vcpu->arch.injected_exception.has_error_code = has_error;
- vcpu->arch.injected_exception.nr = nr;
- vcpu->arch.injected_exception.error_code = error_code;
+ WARN_ON_ONCE(vcpu->arch.pending_exception.valid);
+ WARN_ON_ONCE(reinject && vcpu->arch.injected_exception.valid);
- } else {
- vcpu->arch.pending_exception.valid = true;
- vcpu->arch.injected_exception.valid = false;
- vcpu->arch.pending_exception.has_error_code = has_error;
- vcpu->arch.pending_exception.nr = nr;
- vcpu->arch.pending_exception.error_code = error_code;
- }
-
- vcpu->arch.exception_payload.valid = has_payload;
- vcpu->arch.exception_payload.value = payload;
- if (!is_guest_mode(vcpu))
- kvm_deliver_exception_payload(vcpu);
- return;
- }
-
- /* to check exception */
- prev_nr = vcpu->arch.injected_exception.nr;
- if (prev_nr == DF_VECTOR) {
- /* triple fault -> shutdown */
- kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);
- return;
- }
- class1 = exception_class(prev_nr);
- class2 = exception_class(nr);
- if ((class1 == EXCPT_CONTRIBUTORY && class2 == EXCPT_CONTRIBUTORY)
- || (class1 == EXCPT_PF && class2 != EXCPT_BENIGN)) {
- /*
- * Generate double fault per SDM Table 5-5. Set
- * exception.pending = true so that the double fault
- * can trigger a nested vmexit.
- */
- vcpu->arch.pending_exception.valid = true;
- vcpu->arch.injected_exception.valid = false;
- vcpu->arch.pending_exception.has_error_code = true;
- vcpu->arch.pending_exception.nr = DF_VECTOR;
- vcpu->arch.pending_exception.error_code = 0;
+ exc = reinject ? &vcpu->arch.injected_exception :
+ &vcpu->arch.pending_exception;
+ exc->valid = true;
+ exc->nr = nr;
+ exc->has_error_code = has_error;
+ exc->error_code = error_code;
- vcpu->arch.exception_payload.valid = false;
- vcpu->arch.exception_payload.value = 0;
- } else
- /* replace previous exception with a new one in a hope
- that instruction re-execution will regenerate lost
- exception */
- goto queue;
+ // re-injected exception has its payload already delivered
+ WARN_ON_ONCE(reinject && has_payload);
+ vcpu->arch.exception_payload.valid = has_payload;
+ vcpu->arch.exception_payload.value = payload;
}
+
void kvm_queue_exception(struct kvm_vcpu *vcpu, unsigned nr)
{
kvm_multiple_exception(vcpu, nr, false, 0, false, 0, false);
@@ -641,6 +585,95 @@ static void kvm_queue_exception_e_p(struct kvm_vcpu *vcpu, unsigned nr,
true, payload, false);
}
+static int kvm_do_deliver_pending_exception(struct kvm_vcpu *vcpu)
+{
+ int class1, class2, ret;
+
+ /* try to deliver current pending exception as VM exit */
+ if (is_guest_mode(vcpu)) {
+ ret = kvm_x86_ops.nested_ops->deliver_exception_as_vmexit(vcpu);
+ if (ret || !vcpu->arch.pending_exception.valid)
+ return ret;
+ }
+
+ /* No injected exception, so just deliver the payload and inject it */
+ if (!vcpu->arch.injected_exception.valid) {
+ trace_kvm_inj_exception(vcpu->arch.pending_exception.nr,
+ vcpu->arch.pending_exception.has_error_code,
+ vcpu->arch.pending_exception.error_code);
+queue:
+ /* Intel SDM 17.3.1.1 */
+ if (exception_type(vcpu->arch.pending_exception.nr) == EXCPT_FAULT)
+ __kvm_set_rflags(vcpu, kvm_get_rflags(vcpu) |
+ X86_EFLAGS_RF);
+
+ kvm_deliver_exception_payload(vcpu);
+
+ /* Intel SDM 17.2.4
+ * The processor clears the GD flag upon entering to the
+ * debug exception handler, to allow the handler access
+ * to the debug registers.
+ */
+ if (vcpu->arch.pending_exception.nr == DB_VECTOR) {
+ if (vcpu->arch.dr7 & DR7_GD) {
+ vcpu->arch.dr7 &= ~DR7_GD;
+ kvm_update_dr7(vcpu);
+ }
+ }
+
+ if (vcpu->arch.pending_exception.error_code && !is_protmode(vcpu))
+ vcpu->arch.pending_exception.error_code = false;
+
+ vcpu->arch.injected_exception = vcpu->arch.pending_exception;
+ vcpu->arch.pending_exception.valid = false;
+ return 0;
+ }
+
+ /* Convert a pending exception and an injected #DF to a triple fault*/
+ if (vcpu->arch.injected_exception.nr == DF_VECTOR) {
+ /* triple fault -> shutdown */
+ vcpu->arch.injected_exception.valid = false;
+ vcpu->arch.pending_exception.valid = false;
+ kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);
+ return 0;
+ }
+
+ class1 = exception_class(vcpu->arch.injected_exception.nr);
+ class2 = exception_class(vcpu->arch.pending_exception.nr);
+
+ if ((class1 == EXCPT_CONTRIBUTORY && class2 == EXCPT_CONTRIBUTORY)
+ || (class1 == EXCPT_PF && class2 != EXCPT_BENIGN)) {
+
+ /* Generate double fault per SDM Table 5-5. */
+ vcpu->arch.injected_exception.valid = false;
+ vcpu->arch.pending_exception.valid = true;
+ vcpu->arch.pending_exception.has_error_code = true;
+ vcpu->arch.pending_exception.nr = DF_VECTOR;
+ vcpu->arch.pending_exception.error_code = 0;
+ vcpu->arch.exception_payload.valid = false;
+ } else
+ /* Drop the injected exception and replace it with the pending one*/
+ goto queue;
+
+ return 0;
+}
+
+static int kvm_deliver_pending_exception(struct kvm_vcpu *vcpu)
+{
+ int ret = 0;
+
+ if (!vcpu->arch.pending_exception.valid)
+ return ret;
+
+ ret = kvm_do_deliver_pending_exception(vcpu);
+
+ if (ret || !vcpu->arch.pending_exception.valid)
+ return ret;
+
+ WARN_ON_ONCE(vcpu->arch.pending_exception.nr != DF_VECTOR);
+ return kvm_do_deliver_pending_exception(vcpu);
+}
+
int kvm_complete_insn_gp(struct kvm_vcpu *vcpu, int err)
{
if (err)
@@ -4297,6 +4330,12 @@ static void kvm_vcpu_ioctl_x86_get_vcpu_events(struct kvm_vcpu *vcpu,
vcpu->arch.pending_exception.valid && vcpu->arch.exception_payload.valid)
kvm_deliver_exception_payload(vcpu);
+ /*
+ * Currently we merge the pending and the injected exceptions
+ * after each VM exit, which can fail only when nested run is pending,
+ * in which case only injected (from us or L1) exception is possible.
+ */
+
WARN_ON_ONCE(vcpu->arch.pending_exception.valid &&
vcpu->arch.injected_exception.valid);
@@ -8401,8 +8440,6 @@ int kvm_check_nested_events(struct kvm_vcpu *vcpu)
static void kvm_inject_exception(struct kvm_vcpu *vcpu)
{
- if (vcpu->arch.injected_exception.error_code && !is_protmode(vcpu))
- vcpu->arch.injected_exception.error_code = false;
static_call(kvm_x86_queue_exception)(vcpu);
}
@@ -8411,8 +8448,13 @@ static void inject_pending_event(struct kvm_vcpu *vcpu, bool *req_immediate_exit
int r;
bool can_inject = true;
- /* try to reinject previous events if any */
+ r = kvm_deliver_pending_exception(vcpu);
+ if (r < 0)
+ goto busy;
+
+ WARN_ON_ONCE(vcpu->arch.pending_exception.valid);
+ /* try to reinject previous events if any */
if (vcpu->arch.injected_exception.valid) {
kvm_inject_exception(vcpu);
can_inject = false;
@@ -8431,7 +8473,7 @@ static void inject_pending_event(struct kvm_vcpu *vcpu, bool *req_immediate_exit
* serviced prior to recognizing any new events in order to
* fully complete the previous instruction.
*/
- else if (!vcpu->arch.pending_exception.valid) {
+ else {
if (vcpu->arch.nmi_injected) {
static_call(kvm_x86_set_nmi)(vcpu);
can_inject = false;
@@ -8441,9 +8483,6 @@ static void inject_pending_event(struct kvm_vcpu *vcpu, bool *req_immediate_exit
}
}
- WARN_ON_ONCE(vcpu->arch.pending_exception.valid &&
- vcpu->arch.injected_exception.valid);
-
/*
* Call check_nested_events() even if we reinjected a previous event
* in order for caller to determine if it should require immediate-exit
@@ -8456,30 +8495,6 @@ static void inject_pending_event(struct kvm_vcpu *vcpu, bool *req_immediate_exit
goto busy;
}
- /* try to inject new event if pending */
- if (vcpu->arch.pending_exception.valid) {
- trace_kvm_inj_exception(vcpu->arch.pending_exception.nr,
- vcpu->arch.pending_exception.has_error_code,
- vcpu->arch.pending_exception.error_code);
-
- vcpu->arch.injected_exception = vcpu->arch.pending_exception;
- vcpu->arch.pending_exception.valid = false;
-
- if (exception_type(vcpu->arch.injected_exception.nr) == EXCPT_FAULT)
- __kvm_set_rflags(vcpu, kvm_get_rflags(vcpu) |
- X86_EFLAGS_RF);
-
- if (vcpu->arch.injected_exception.nr == DB_VECTOR) {
- kvm_deliver_exception_payload(vcpu);
- if (vcpu->arch.dr7 & DR7_GD) {
- vcpu->arch.dr7 &= ~DR7_GD;
- kvm_update_dr7(vcpu);
- }
- }
-
- kvm_inject_exception(vcpu);
- can_inject = false;
- }
/*
* Finally, inject interrupt events. If an event cannot be injected
@@ -9270,6 +9285,14 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu)
kvm_lapic_sync_from_vapic(vcpu);
r = static_call(kvm_x86_handle_exit)(vcpu, exit_fastpath);
+
+ /*
+ * Deliver the pending exception so that the state of having a pending
+ * and an injected exception is not visible to the userspace.
+ */
+
+ kvm_deliver_pending_exception(vcpu);
+
return r;
cancel_injection:
@@ -11014,7 +11037,7 @@ static inline bool kvm_vcpu_has_events(struct kvm_vcpu *vcpu)
if (vcpu->arch.pv.pv_unhalted)
return true;
- if (vcpu->arch.pending_exception.valid)
+ if (vcpu->arch.pending_exception.valid || vcpu->arch.injected_exception.valid)
return true;
if (kvm_test_request(KVM_REQ_NMI, vcpu) ||
--
2.26.2
