Hi:
> Jiri Denemark <jdenemar@xxxxxxxxxx>
> So could you also provide
> (1) the output of "virsh hypervisor-cpu-baseline" and
> (2) the <cpu> element from the domain XML before it is started
> (3) and once it is running on the host from which migration fails?
sorry when I checked the vm with your theory(qemu will enable vmx
automatically) in my last post, I wrongly checked a vm which is still
using RHEL 9.3 config. RHEL 9.3 config has "vmx" feature but not other
"vmx-blabla." sub features. When the vm config has "vmx" feature,
"virsh dumpxml" a running vm will bring all these "vmx-blabla" sub
feature lines at RHEL 9.4.
but if I delete the "vmx" feature from the vm config, then qemu
won't enable any "vmx" related feature. "vmx" doesn't exist in guest
vm or "virsh dumpxml". and migration works as expected. xml are the
same when running at E3-1280-V3 and I3-9100F. the only difference
between shutdown/running xml is 'check="partial' and 'check="full"'.
> virsh dumpxml my-vm | xmllint --xpath '/domain/cpu' -
<cpu mode="custom" match="exact" check="full">
<model fallback="forbid">Haswell-noTSX-IBRS</model>
<vendor>Intel</vendor>
<topology sockets="1" dies="1" clusters="1" cores="2" threads="1"/>
<maxphysaddr mode="passthrough" limit="39"/>
<feature policy="require" name="vme"/>
<feature policy="require" name="ss"/>
<feature policy="require" name="pdcm"/>
<feature policy="require" name="f16c"/>
<feature policy="require" name="rdrand"/>
<feature policy="require" name="hypervisor"/>
<feature policy="require" name="arat"/>
<feature policy="require" name="tsc_adjust"/>
<feature policy="require" name="umip"/>
<feature policy="require" name="md-clear"/>
<feature policy="require" name="stibp"/>
<feature policy="require" name="flush-l1d"/>
<feature policy="require" name="arch-capabilities"/>
<feature policy="require" name="ssbd"/>
<feature policy="require" name="xsaveopt"/>
<feature policy="require" name="pdpe1gb"/>
<feature policy="require" name="abm"/>
<feature policy="require" name="ibpb"/>
<feature policy="require" name="ibrs"/>
<feature policy="require" name="amd-stibp"/>
<feature policy="require" name="amd-ssbd"/>
<feature policy="require" name="pschange-mc-no"/>
</cpu>
Next test I add "vmx" feature to vm config(so it will look lik RHEL
9.3 config). the xml are the same at both cpu. live migration failed
at both direction and error message are the same: "guest CPU doesn't
match specification: missing features:
vmx-apicv-register,vmx-apicv-vid,vmx-posted-intr". we can see these
features were disabled at running xml.
shutdown xml below:
<cpu mode="custom" match="exact" check="partial">
<model fallback="forbid">Haswell-noTSX-IBRS</model>
<vendor>Intel</vendor>
<topology sockets="1" dies="1" clusters="1" cores="1" threads="1"/>
<maxphysaddr mode="passthrough" limit="39"/>
<feature policy="require" name="vmx"/>
<feature policy="require" name="vme"/>
<feature policy="require" name="ss"/>
<feature policy="require" name="pdcm"/>
<feature policy="require" name="f16c"/>
<feature policy="require" name="rdrand"/>
<feature policy="require" name="hypervisor"/>
<feature policy="require" name="arat"/>
<feature policy="require" name="tsc_adjust"/>
<feature policy="require" name="umip"/>
<feature policy="require" name="md-clear"/>
<feature policy="require" name="stibp"/>
<feature policy="require" name="flush-l1d"/>
<feature policy="require" name="arch-capabilities"/>
<feature policy="require" name="ssbd"/>
<feature policy="require" name="xsaveopt"/>
<feature policy="require" name="pdpe1gb"/>
<feature policy="require" name="abm"/>
<feature policy="require" name="ibpb"/>
<feature policy="require" name="ibrs"/>
<feature policy="require" name="amd-stibp"/>
<feature policy="require" name="amd-ssbd"/>
<feature policy="require" name="pschange-mc-no"/>
</cpu>
running xml below:
<cpu mode="custom" match="exact" check="full">
<model fallback="forbid">Haswell-noTSX-IBRS</model>
<vendor>Intel</vendor>
<topology sockets="1" dies="1" clusters="1" cores="1" threads="1"/>
<maxphysaddr mode="passthrough" limit="39"/>
<feature policy="require" name="vmx"/>
<feature policy="require" name="vme"/>
<feature policy="require" name="ss"/>
<feature policy="require" name="pdcm"/>
<feature policy="require" name="f16c"/>
<feature policy="require" name="rdrand"/>
<feature policy="require" name="hypervisor"/>
<feature policy="require" name="arat"/>
<feature policy="require" name="tsc_adjust"/>
<feature policy="require" name="umip"/>
<feature policy="require" name="md-clear"/>
<feature policy="require" name="stibp"/>
<feature policy="require" name="flush-l1d"/>
<feature policy="require" name="arch-capabilities"/>
<feature policy="require" name="ssbd"/>
<feature policy="require" name="xsaveopt"/>
<feature policy="require" name="pdpe1gb"/>
<feature policy="require" name="abm"/>
<feature policy="require" name="ibpb"/>
<feature policy="require" name="ibrs"/>
<feature policy="require" name="amd-stibp"/>
<feature policy="require" name="amd-ssbd"/>
<feature policy="require" name="pschange-mc-no"/>
<feature policy="require" name="vmx-ins-outs"/>
<feature policy="require" name="vmx-true-ctls"/>
<feature policy="require" name="vmx-store-lma"/>
<feature policy="require" name="vmx-activity-hlt"/>
<feature policy="require" name="vmx-vmwrite-vmexit-fields"/>
<feature policy="require" name="vmx-apicv-xapic"/>
<feature policy="require" name="vmx-ept"/>
<feature policy="require" name="vmx-desc-exit"/>
<feature policy="require" name="vmx-rdtscp-exit"/>
<feature policy="require" name="vmx-apicv-x2apic"/>
<feature policy="require" name="vmx-vpid"/>
<feature policy="require" name="vmx-wbinvd-exit"/>
<feature policy="require" name="vmx-unrestricted-guest"/>
<feature policy="disable" name="vmx-apicv-register"/>
<feature policy="disable" name="vmx-apicv-vid"/>
<feature policy="require" name="vmx-rdrand-exit"/>
<feature policy="require" name="vmx-invpcid-exit"/>
<feature policy="require" name="vmx-vmfunc"/>
<feature policy="require" name="vmx-shadow-vmcs"/>
<feature policy="require" name="vmx-invvpid"/>
<feature policy="require" name="vmx-invvpid-single-addr"/>
<feature policy="require" name="vmx-invvpid-all-context"/>
<feature policy="require" name="vmx-ept-execonly"/>
<feature policy="require" name="vmx-page-walk-4"/>
<feature policy="require" name="vmx-ept-2mb"/>
<feature policy="require" name="vmx-ept-1gb"/>
<feature policy="require" name="vmx-invept"/>
<feature policy="require" name="vmx-eptad"/>
<feature policy="require" name="vmx-invept-single-context"/>
<feature policy="require" name="vmx-invept-all-context"/>
<feature policy="require" name="vmx-intr-exit"/>
<feature policy="require" name="vmx-nmi-exit"/>
<feature policy="require" name="vmx-vnmi"/>
<feature policy="require" name="vmx-preemption-timer"/>
<feature policy="disable" name="vmx-posted-intr"/>
<feature policy="require" name="vmx-vintr-pending"/>
<feature policy="require" name="vmx-tsc-offset"/>
<feature policy="require" name="vmx-hlt-exit"/>
<feature policy="require" name="vmx-invlpg-exit"/>
<feature policy="require" name="vmx-mwait-exit"/>
<feature policy="require" name="vmx-rdpmc-exit"/>
<feature policy="require" name="vmx-rdtsc-exit"/>
<feature policy="require" name="vmx-cr3-load-noexit"/>
<feature policy="require" name="vmx-cr3-store-noexit"/>
<feature policy="require" name="vmx-cr8-load-exit"/>
<feature policy="require" name="vmx-cr8-store-exit"/>
<feature policy="require" name="vmx-flexpriority"/>
<feature policy="require" name="vmx-vnmi-pending"/>
<feature policy="require" name="vmx-movdr-exit"/>
<feature policy="require" name="vmx-io-exit"/>
<feature policy="require" name="vmx-io-bitmap"/>
<feature policy="require" name="vmx-mtf"/>
<feature policy="require" name="vmx-msr-bitmap"/>
<feature policy="require" name="vmx-monitor-exit"/>
<feature policy="require" name="vmx-pause-exit"/>
<feature policy="require" name="vmx-secondary-ctls"/>
<feature policy="require" name="vmx-exit-nosave-debugctl"/>
<feature policy="require" name="vmx-exit-ack-intr"/>
<feature policy="require" name="vmx-exit-save-pat"/>
<feature policy="require" name="vmx-exit-load-pat"/>
<feature policy="require" name="vmx-exit-save-efer"/>
<feature policy="require" name="vmx-exit-load-efer"/>
<feature policy="require" name="vmx-exit-save-preemption-timer"/>
<feature policy="require" name="vmx-entry-noload-debugctl"/>
<feature policy="require" name="vmx-entry-ia32e-mode"/>
<feature policy="require" name="vmx-entry-load-pat"/>
<feature policy="require" name="vmx-entry-load-efer"/>
<feature policy="require" name="vmx-eptp-switching"/>
</cpu>
the final test I try to use "hypervisor-cpu-baseline --migratable "
result. again the xml are the same at both cpu. the difference between
shutdown xml/running xml is the last three lines. failed cpu features
were disabled at running xml automatically
.
shutdown xml below:
<cpu mode="custom" match="exact" check="partial">
<model fallback="forbid">Haswell-noTSX-IBRS</model>
<vendor>Intel</vendor>
<topology sockets="1" dies="1" clusters="1" cores="1" threads="1"/>
<maxphysaddr mode="passthrough" limit="39"/>
<feature policy="require" name="vme"/>
<feature policy="require" name="ss"/>
<feature policy="require" name="vmx"/>
<feature policy="require" name="pdcm"/>
<feature policy="require" name="f16c"/>
<feature policy="require" name="rdrand"/>
<feature policy="require" name="hypervisor"/>
<feature policy="require" name="arat"/>
<feature policy="require" name="tsc_adjust"/>
<feature policy="require" name="umip"/>
<feature policy="require" name="md-clear"/>
<feature policy="require" name="stibp"/>
<feature policy="require" name="flush-l1d"/>
<feature policy="require" name="arch-capabilities"/>
<feature policy="require" name="ssbd"/>
<feature policy="require" name="xsaveopt"/>
<feature policy="require" name="pdpe1gb"/>
<feature policy="require" name="abm"/>
<feature policy="require" name="ibpb"/>
<feature policy="require" name="ibrs"/>
<feature policy="require" name="amd-stibp"/>
<feature policy="require" name="amd-ssbd"/>
<feature policy="require" name="pschange-mc-no"/>
<feature policy="require" name="vmx-ins-outs"/>
<feature policy="require" name="vmx-true-ctls"/>
<feature policy="require" name="vmx-store-lma"/>
<feature policy="require" name="vmx-activity-hlt"/>
<feature policy="require" name="vmx-activity-wait-sipi"/>
<feature policy="require" name="vmx-vmwrite-vmexit-fields"/>
<feature policy="require" name="vmx-apicv-xapic"/>
<feature policy="require" name="vmx-ept"/>
<feature policy="require" name="vmx-desc-exit"/>
<feature policy="require" name="vmx-rdtscp-exit"/>
<feature policy="require" name="vmx-apicv-x2apic"/>
<feature policy="require" name="vmx-vpid"/>
<feature policy="require" name="vmx-wbinvd-exit"/>
<feature policy="require" name="vmx-unrestricted-guest"/>
<feature policy="require" name="vmx-rdrand-exit"/>
<feature policy="require" name="vmx-invpcid-exit"/>
<feature policy="require" name="vmx-vmfunc"/>
<feature policy="require" name="vmx-shadow-vmcs"/>
<feature policy="require" name="vmx-pml"/>
<feature policy="require" name="vmx-invvpid"/>
<feature policy="require" name="vmx-invvpid-single-addr"/>
<feature policy="require" name="vmx-invvpid-all-context"/>
<feature policy="require" name="vmx-ept-execonly"/>
<feature policy="require" name="vmx-page-walk-4"/>
<feature policy="require" name="vmx-ept-2mb"/>
<feature policy="require" name="vmx-ept-1gb"/>
<feature policy="require" name="vmx-invept"/>
<feature policy="require" name="vmx-eptad"/>
<feature policy="require" name="vmx-invept-single-context"/>
<feature policy="require" name="vmx-invept-all-context"/>
<feature policy="require" name="vmx-intr-exit"/>
<feature policy="require" name="vmx-nmi-exit"/>
<feature policy="require" name="vmx-vnmi"/>
<feature policy="require" name="vmx-preemption-timer"/>
<feature policy="require" name="vmx-vintr-pending"/>
<feature policy="require" name="vmx-tsc-offset"/>
<feature policy="require" name="vmx-hlt-exit"/>
<feature policy="require" name="vmx-invlpg-exit"/>
<feature policy="require" name="vmx-mwait-exit"/>
<feature policy="require" name="vmx-rdpmc-exit"/>
<feature policy="require" name="vmx-rdtsc-exit"/>
<feature policy="require" name="vmx-cr3-load-noexit"/>
<feature policy="require" name="vmx-cr3-store-noexit"/>
<feature policy="require" name="vmx-cr8-load-exit"/>
<feature policy="require" name="vmx-cr8-store-exit"/>
<feature policy="require" name="vmx-flexpriority"/>
<feature policy="require" name="vmx-vnmi-pending"/>
<feature policy="require" name="vmx-movdr-exit"/>
<feature policy="require" name="vmx-io-exit"/>
<feature policy="require" name="vmx-io-bitmap"/>
<feature policy="require" name="vmx-mtf"/>
<feature policy="require" name="vmx-msr-bitmap"/>
<feature policy="require" name="vmx-monitor-exit"/>
<feature policy="require" name="vmx-pause-exit"/>
<feature policy="require" name="vmx-secondary-ctls"/>
<feature policy="require" name="vmx-exit-nosave-debugctl"/>
<feature policy="require" name="vmx-exit-load-perf-global-ctrl"/>
<feature policy="require" name="vmx-exit-ack-intr"/>
<feature policy="require" name="vmx-exit-save-pat"/>
<feature policy="require" name="vmx-exit-load-pat"/>
<feature policy="require" name="vmx-exit-save-efer"/>
<feature policy="require" name="vmx-exit-load-efer"/>
<feature policy="require" name="vmx-exit-save-preemption-timer"/>
<feature policy="require" name="vmx-entry-noload-debugctl"/>
<feature policy="require" name="vmx-entry-ia32e-mode"/>
<feature policy="require" name="vmx-entry-load-perf-global-ctrl"/>
<feature policy="require" name="vmx-entry-load-pat"/>
<feature policy="require" name="vmx-entry-load-efer"/>
<feature policy="require" name="vmx-eptp-switching"/>
</cpu>
running xml below:
<cpu mode="custom" match="exact" check="full">
<model fallback="forbid">Haswell-noTSX-IBRS</model>
<vendor>Intel</vendor>
<topology sockets="1" dies="1" clusters="1" cores="1" threads="1"/>
<maxphysaddr mode="passthrough" limit="39"/>
<feature policy="require" name="vme"/>
<feature policy="require" name="ss"/>
<feature policy="require" name="vmx"/>
<feature policy="require" name="pdcm"/>
<feature policy="require" name="f16c"/>
<feature policy="require" name="rdrand"/>
<feature policy="require" name="hypervisor"/>
<feature policy="require" name="arat"/>
<feature policy="require" name="tsc_adjust"/>
<feature policy="require" name="umip"/>
<feature policy="require" name="md-clear"/>
<feature policy="require" name="stibp"/>
<feature policy="require" name="flush-l1d"/>
<feature policy="require" name="arch-capabilities"/>
<feature policy="require" name="ssbd"/>
<feature policy="require" name="xsaveopt"/>
<feature policy="require" name="pdpe1gb"/>
<feature policy="require" name="abm"/>
<feature policy="require" name="ibpb"/>
<feature policy="require" name="ibrs"/>
<feature policy="require" name="amd-stibp"/>
<feature policy="require" name="amd-ssbd"/>
<feature policy="require" name="pschange-mc-no"/>
<feature policy="require" name="vmx-ins-outs"/>
<feature policy="require" name="vmx-true-ctls"/>
<feature policy="require" name="vmx-store-lma"/>
<feature policy="require" name="vmx-activity-hlt"/>
<feature policy="require" name="vmx-activity-wait-sipi"/>
<feature policy="require" name="vmx-vmwrite-vmexit-fields"/>
<feature policy="require" name="vmx-apicv-xapic"/>
<feature policy="require" name="vmx-ept"/>
<feature policy="require" name="vmx-desc-exit"/>
<feature policy="require" name="vmx-rdtscp-exit"/>
<feature policy="require" name="vmx-apicv-x2apic"/>
<feature policy="require" name="vmx-vpid"/>
<feature policy="require" name="vmx-wbinvd-exit"/>
<feature policy="require" name="vmx-unrestricted-guest"/>
<feature policy="require" name="vmx-rdrand-exit"/>
<feature policy="require" name="vmx-invpcid-exit"/>
<feature policy="require" name="vmx-vmfunc"/>
<feature policy="require" name="vmx-shadow-vmcs"/>
<feature policy="require" name="vmx-pml"/>
<feature policy="require" name="vmx-invvpid"/>
<feature policy="require" name="vmx-invvpid-single-addr"/>
<feature policy="require" name="vmx-invvpid-all-context"/>
<feature policy="require" name="vmx-ept-execonly"/>
<feature policy="require" name="vmx-page-walk-4"/>
<feature policy="require" name="vmx-ept-2mb"/>
<feature policy="require" name="vmx-ept-1gb"/>
<feature policy="require" name="vmx-invept"/>
<feature policy="require" name="vmx-eptad"/>
<feature policy="require" name="vmx-invept-single-context"/>
<feature policy="require" name="vmx-invept-all-context"/>
<feature policy="require" name="vmx-intr-exit"/>
<feature policy="require" name="vmx-nmi-exit"/>
<feature policy="require" name="vmx-vnmi"/>
<feature policy="require" name="vmx-preemption-timer"/>
<feature policy="require" name="vmx-vintr-pending"/>
<feature policy="require" name="vmx-tsc-offset"/>
<feature policy="require" name="vmx-hlt-exit"/>
<feature policy="require" name="vmx-invlpg-exit"/>
<feature policy="require" name="vmx-mwait-exit"/>
<feature policy="require" name="vmx-rdpmc-exit"/>
<feature policy="require" name="vmx-rdtsc-exit"/>
<feature policy="require" name="vmx-cr3-load-noexit"/>
<feature policy="require" name="vmx-cr3-store-noexit"/>
<feature policy="require" name="vmx-cr8-load-exit"/>
<feature policy="require" name="vmx-cr8-store-exit"/>
<feature policy="require" name="vmx-flexpriority"/>
<feature policy="require" name="vmx-vnmi-pending"/>
<feature policy="require" name="vmx-movdr-exit"/>
<feature policy="require" name="vmx-io-exit"/>
<feature policy="require" name="vmx-io-bitmap"/>
<feature policy="require" name="vmx-mtf"/>
<feature policy="require" name="vmx-msr-bitmap"/>
<feature policy="require" name="vmx-monitor-exit"/>
<feature policy="require" name="vmx-pause-exit"/>
<feature policy="require" name="vmx-secondary-ctls"/>
<feature policy="require" name="vmx-exit-nosave-debugctl"/>
<feature policy="require" name="vmx-exit-load-perf-global-ctrl"/>
<feature policy="require" name="vmx-exit-ack-intr"/>
<feature policy="require" name="vmx-exit-save-pat"/>
<feature policy="require" name="vmx-exit-load-pat"/>
<feature policy="require" name="vmx-exit-save-efer"/>
<feature policy="require" name="vmx-exit-load-efer"/>
<feature policy="require" name="vmx-exit-save-preemption-timer"/>
<feature policy="require" name="vmx-entry-noload-debugctl"/>
<feature policy="require" name="vmx-entry-ia32e-mode"/>
<feature policy="require" name="vmx-entry-load-perf-global-ctrl"/>
<feature policy="require" name="vmx-entry-load-pat"/>
<feature policy="require" name="vmx-entry-load-efer"/>
<feature policy="require" name="vmx-eptp-switching"/>
<feature policy="disable" name="vmx-apicv-register"/>
<feature policy="disable" name="vmx-apicv-vid"/>
<feature policy="disable" name="vmx-posted-intr"/>
</cpu>
