Lines Matching refs:vmx
15 #include "vmx.h"
189 struct vcpu_vmx *vmx = to_vmx(vcpu);
195 if (vmx->nested.current_vmptr == -1ull && !vmx->nested.hv_evmcs)
205 pr_debug_ratelimited("kvm: nested vmx abort, indicator %d\n", indicator);
218 static void vmx_disable_shadow_vmcs(struct vcpu_vmx *vmx)
220 secondary_exec_controls_clearbit(vmx, SECONDARY_EXEC_SHADOW_VMCS);
222 vmx->nested.need_vmcs12_to_shadow_sync = false;
227 struct vcpu_vmx *vmx = to_vmx(vcpu);
229 if (!vmx->nested.hv_evmcs)
232 kvm_vcpu_unmap(vcpu, &vmx->nested.hv_evmcs_map, true);
233 vmx->nested.hv_evmcs_vmptr = 0;
234 vmx->nested.hv_evmcs = NULL;
237 static void vmx_sync_vmcs_host_state(struct vcpu_vmx *vmx,
242 if (unlikely(!vmx->guest_state_loaded))
246 dest = &vmx->loaded_vmcs->host_state;
258 struct vcpu_vmx *vmx = to_vmx(vcpu);
262 if (WARN_ON_ONCE(vmx->loaded_vmcs == vmcs))
266 prev = vmx->loaded_vmcs;
267 vmx->loaded_vmcs = vmcs;
269 vmx_sync_vmcs_host_state(vmx, prev);
276 * Free whatever needs to be freed from vmx->nested when L1 goes down, or
281 struct vcpu_vmx *vmx = to_vmx(vcpu);
283 if (WARN_ON_ONCE(vmx->loaded_vmcs != &vmx->vmcs01))
284 vmx_switch_vmcs(vcpu, &vmx->vmcs01);
286 if (!vmx->nested.vmxon && !vmx->nested.smm.vmxon)
291 vmx->nested.vmxon = false;
292 vmx->nested.smm.vmxon = false;
293 free_vpid(vmx->nested.vpid02);
294 vmx->nested.posted_intr_nv = -1;
295 vmx->nested.current_vmptr = -1ull;
297 vmx_disable_shadow_vmcs(vmx);
298 vmcs_clear(vmx->vmcs01.shadow_vmcs);
299 free_vmcs(vmx->vmcs01.shadow_vmcs);
300 vmx->vmcs01.shadow_vmcs = NULL;
302 kfree(vmx->nested.cached_vmcs12);
303 vmx->nested.cached_vmcs12 = NULL;
304 kfree(vmx->nested.cached_shadow_vmcs12);
305 vmx->nested.cached_shadow_vmcs12 = NULL;
307 if (vmx->nested.apic_access_page) {
308 kvm_release_page_clean(vmx->nested.apic_access_page);
309 vmx->nested.apic_access_page = NULL;
311 kvm_vcpu_unmap(vcpu, &vmx->nested.virtual_apic_map, true);
312 kvm_vcpu_unmap(vcpu, &vmx->nested.pi_desc_map, true);
313 vmx->nested.pi_desc = NULL;
319 free_loaded_vmcs(&vmx->nested.vmcs02);
337 struct vcpu_vmx *vmx = to_vmx(vcpu);
341 if (vmx->nested.pml_full) {
343 vmx->nested.pml_full = false;
685 struct vcpu_vmx *vmx = to_vmx(vcpu);
691 kvm_write_guest(vmx->vcpu.kvm, vmcs12->vmcs_link_pointer,
881 struct vcpu_vmx *vmx = to_vmx(vcpu);
882 u64 vmx_misc = vmx_control_msr(vmx->nested.msrs.misc_low,
883 vmx->nested.msrs.misc_high);
937 struct vcpu_vmx *vmx = to_vmx(vcpu);
945 int i = vmx_find_loadstore_msr_slot(&vmx->msr_autostore.guest,
949 u64 val = vmx->msr_autostore.guest.val[i].value;
1035 struct vcpu_vmx *vmx = to_vmx(vcpu);
1036 struct vmx_msrs *autostore = &vmx->msr_autostore.guest;
1178 * with different VPID (L1 entries are tagged with vmx->vpid
1179 * while L2 entries are tagged with vmx->nested.vpid02).
1193 struct vcpu_vmx *vmx = to_vmx(vcpu);
1227 vmcs12->virtual_processor_id != vmx->nested.last_vpid) {
1228 vmx->nested.last_vpid = vmcs12->virtual_processor_id;
1241 static int vmx_restore_vmx_basic(struct vcpu_vmx *vmx, u64 data)
1267 vmx->nested.msrs.basic = data;
1301 vmx_restore_control_msr(struct vcpu_vmx *vmx, u32 msr_index, u64 data)
1318 vmx_get_control_msr(&vmx->nested.msrs, msr_index, &lowp, &highp);
1324 static int vmx_restore_vmx_misc(struct vcpu_vmx *vmx, u64 data)
1338 if ((vmx->nested.msrs.pinbased_ctls_high &
1353 vmx->nested.msrs.misc_low = data;
1354 vmx->nested.msrs.misc_high = data >> 32;
1359 static int vmx_restore_vmx_ept_vpid_cap(struct vcpu_vmx *vmx, u64 data)
1368 vmx->nested.msrs.ept_caps = data;
1369 vmx->nested.msrs.vpid_caps = data >> 32;
1385 static int vmx_restore_fixed0_msr(struct vcpu_vmx *vmx, u32 msr_index, u64 data)
1396 *vmx_get_fixed0_msr(&vmx->nested.msrs, msr_index) = data;
1407 struct vcpu_vmx *vmx = to_vmx(vcpu);
1413 if (vmx->nested.vmxon)
1418 return vmx_restore_vmx_basic(vmx, data);
1438 return vmx_restore_control_msr(vmx, msr_index, data);
1440 return vmx_restore_vmx_misc(vmx, data);
1443 return vmx_restore_fixed0_msr(vmx, msr_index, data);
1452 return vmx_restore_vmx_ept_vpid_cap(vmx, data);
1454 vmx->nested.msrs.vmcs_enum = data;
1459 vmx->nested.msrs.vmfunc_controls = data;
1555 static void copy_shadow_to_vmcs12(struct vcpu_vmx *vmx)
1557 struct vmcs *shadow_vmcs = vmx->vmcs01.shadow_vmcs;
1558 struct vmcs12 *vmcs12 = get_vmcs12(&vmx->vcpu);
1577 vmcs_load(vmx->loaded_vmcs->vmcs);
1582 static void copy_vmcs12_to_shadow(struct vcpu_vmx *vmx)
1592 struct vmcs *shadow_vmcs = vmx->vmcs01.shadow_vmcs;
1593 struct vmcs12 *vmcs12 = get_vmcs12(&vmx->vcpu);
1613 vmcs_load(vmx->loaded_vmcs->vmcs);
1616 static int copy_enlightened_to_vmcs12(struct vcpu_vmx *vmx)
1618 struct vmcs12 *vmcs12 = vmx->nested.cached_vmcs12;
1619 struct hv_enlightened_vmcs *evmcs = vmx->nested.hv_evmcs;
1832 static int copy_vmcs12_to_enlightened(struct vcpu_vmx *vmx)
1834 struct vmcs12 *vmcs12 = vmx->nested.cached_vmcs12;
1835 struct hv_enlightened_vmcs *evmcs = vmx->nested.hv_evmcs;
2002 struct vcpu_vmx *vmx = to_vmx(vcpu);
2006 if (likely(!vmx->nested.enlightened_vmcs_enabled))
2012 if (unlikely(!vmx->nested.hv_evmcs ||
2013 evmcs_gpa != vmx->nested.hv_evmcs_vmptr)) {
2014 if (!vmx->nested.hv_evmcs)
2015 vmx->nested.current_vmptr = -1ull;
2020 &vmx->nested.hv_evmcs_map))
2023 vmx->nested.hv_evmcs = vmx->nested.hv_evmcs_map.hva;
2047 if ((vmx->nested.hv_evmcs->revision_id != KVM_EVMCS_VERSION) &&
2048 (vmx->nested.hv_evmcs->revision_id != VMCS12_REVISION)) {
2053 vmx->nested.dirty_vmcs12 = true;
2054 vmx->nested.hv_evmcs_vmptr = evmcs_gpa;
2076 vmx->nested.hv_evmcs->hv_clean_fields &=
2084 struct vcpu_vmx *vmx = to_vmx(vcpu);
2086 if (vmx->nested.hv_evmcs) {
2087 copy_vmcs12_to_enlightened(vmx);
2089 vmx->nested.hv_evmcs->hv_clean_fields |=
2092 copy_vmcs12_to_shadow(vmx);
2095 vmx->nested.need_vmcs12_to_shadow_sync = false;
2100 struct vcpu_vmx *vmx =
2103 vmx->nested.preemption_timer_expired = true;
2104 kvm_make_request(KVM_REQ_EVENT, &vmx->vcpu);
2105 kvm_vcpu_kick(&vmx->vcpu);
2112 struct vcpu_vmx *vmx = to_vmx(vcpu);
2118 if (!vmx->nested.has_preemption_timer_deadline) {
2119 vmx->nested.preemption_timer_deadline =
2121 vmx->nested.has_preemption_timer_deadline = true;
2123 return vmx->nested.preemption_timer_deadline - l1_scaled_tsc;
2129 struct vcpu_vmx *vmx = to_vmx(vcpu);
2136 vmx_preemption_timer_fn(&vmx->nested.preemption_timer);
2146 hrtimer_start(&vmx->nested.preemption_timer,
2151 static u64 nested_vmx_calc_efer(struct vcpu_vmx *vmx, struct vmcs12 *vmcs12)
2153 if (vmx->nested.nested_run_pending &&
2157 return vmx->vcpu.arch.efer | (EFER_LMA | EFER_LME);
2159 return vmx->vcpu.arch.efer & ~(EFER_LMA | EFER_LME);
2162 static void prepare_vmcs02_constant_state(struct vcpu_vmx *vmx)
2170 if (vmx->nested.vmcs02_initialized)
2172 vmx->nested.vmcs02_initialized = true;
2181 construct_eptp(&vmx->vcpu, 0, PT64_ROOT_4LEVEL));
2191 vmcs_write64(MSR_BITMAP, __pa(vmx->nested.vmcs02.msr_bitmap));
2201 vmcs_write64(PML_ADDRESS, page_to_phys(vmx->pml_pg));
2213 vmcs_write64(VM_EXIT_MSR_STORE_ADDR, __pa(vmx->msr_autostore.guest.val));
2214 vmcs_write64(VM_EXIT_MSR_LOAD_ADDR, __pa(vmx->msr_autoload.host.val));
2215 vmcs_write64(VM_ENTRY_MSR_LOAD_ADDR, __pa(vmx->msr_autoload.guest.val));
2217 vmx_set_constant_host_state(vmx);
2220 static void prepare_vmcs02_early_rare(struct vcpu_vmx *vmx,
2223 prepare_vmcs02_constant_state(vmx);
2228 if (nested_cpu_has_vpid(vmcs12) && vmx->nested.vpid02)
2229 vmcs_write16(VIRTUAL_PROCESSOR_ID, vmx->nested.vpid02);
2231 vmcs_write16(VIRTUAL_PROCESSOR_ID, vmx->vpid);
2235 static void prepare_vmcs02_early(struct vcpu_vmx *vmx, struct loaded_vmcs *vmcs01,
2239 u64 guest_efer = nested_vmx_calc_efer(vmx, vmcs12);
2241 if (vmx->nested.dirty_vmcs12 || vmx->nested.hv_evmcs)
2242 prepare_vmcs02_early_rare(vmx, vmcs12);
2252 vmx->nested.pi_pending = false;
2254 vmx->nested.posted_intr_nv = vmcs12->posted_intr_nv;
2257 pin_controls_set(vmx, exec_control);
2268 vmx->nested.l1_tpr_threshold = -1;
2291 exec_control |= exec_controls_get(vmx) & CPU_BASED_USE_MSR_BITMAPS;
2293 exec_controls_set(vmx, exec_control);
2338 secondary_exec_controls_set(vmx, exec_control);
2363 vm_entry_controls_set(vmx, exec_control);
2377 vm_exit_controls_set(vmx, exec_control);
2382 if (vmx->nested.nested_run_pending) {
2391 vmx->loaded_vmcs->nmi_known_unmasked =
2398 static void prepare_vmcs02_rare(struct vcpu_vmx *vmx, struct vmcs12 *vmcs12)
2400 struct hv_enlightened_vmcs *hv_evmcs = vmx->nested.hv_evmcs;
2441 vmx->segment_cache.bitmask = 0;
2463 if (kvm_mpx_supported() && vmx->nested.nested_run_pending &&
2484 if (vmx_need_pf_intercept(&vmx->vcpu)) {
2507 prepare_vmx_msr_autostore_list(&vmx->vcpu, MSR_IA32_TSC);
2509 vmcs_write32(VM_EXIT_MSR_STORE_COUNT, vmx->msr_autostore.guest.nr);
2510 vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, vmx->msr_autoload.host.nr);
2511 vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, vmx->msr_autoload.guest.nr);
2513 set_cr4_guest_host_mask(vmx);
2530 struct vcpu_vmx *vmx = to_vmx(vcpu);
2531 struct hv_enlightened_vmcs *hv_evmcs = vmx->nested.hv_evmcs;
2534 if (vmx->nested.dirty_vmcs12 || hv_evmcs) {
2535 prepare_vmcs02_rare(vmx, vmcs12);
2536 vmx->nested.dirty_vmcs12 = false;
2543 if (vmx->nested.nested_run_pending &&
2549 vmcs_write64(GUEST_IA32_DEBUGCTL, vmx->nested.vmcs01_debugctl);
2551 if (kvm_mpx_supported() && (!vmx->nested.nested_run_pending ||
2553 vmcs_write64(GUEST_BNDCFGS, vmx->nested.vmcs01_guest_bndcfgs);
2564 if (vmx->nested.nested_run_pending &&
2569 vmcs_write64(GUEST_IA32_PAT, vmx->vcpu.arch.pat);
2575 decache_tsc_multiplier(vmx);
2596 vcpu->arch.efer = nested_vmx_calc_efer(vmx, vmcs12);
2664 struct vcpu_vmx *vmx = to_vmx(vcpu);
2670 if (CC(!(vmx->nested.msrs.ept_caps & VMX_EPTP_UC_BIT)))
2674 if (CC(!(vmx->nested.msrs.ept_caps & VMX_EPTP_WB_BIT)))
2684 if (CC(!(vmx->nested.msrs.ept_caps & VMX_EPT_PAGE_WALK_5_BIT)))
2688 if (CC(!(vmx->nested.msrs.ept_caps & VMX_EPT_PAGE_WALK_4_BIT)))
2701 if (CC(!(vmx->nested.msrs.ept_caps & VMX_EPT_AD_BIT)))
2714 struct vcpu_vmx *vmx = to_vmx(vcpu);
2717 vmx->nested.msrs.pinbased_ctls_low,
2718 vmx->nested.msrs.pinbased_ctls_high)) ||
2720 vmx->nested.msrs.procbased_ctls_low,
2721 vmx->nested.msrs.procbased_ctls_high)))
2726 vmx->nested.msrs.secondary_ctls_low,
2727 vmx->nested.msrs.secondary_ctls_high)))
2754 ~vmx->nested.msrs.vmfunc_controls))
2773 struct vcpu_vmx *vmx = to_vmx(vcpu);
2776 vmx->nested.msrs.exit_ctls_low,
2777 vmx->nested.msrs.exit_ctls_high)) ||
2790 struct vcpu_vmx *vmx = to_vmx(vcpu);
2793 vmx->nested.msrs.entry_ctls_low,
2794 vmx->nested.msrs.entry_ctls_high)))
3065 struct vcpu_vmx *vmx = to_vmx(vcpu);
3072 if (vmx->msr_autoload.host.nr)
3074 if (vmx->msr_autoload.guest.nr)
3090 if (unlikely(cr3 != vmx->loaded_vmcs->host_state.cr3)) {
3092 vmx->loaded_vmcs->host_state.cr3 = cr3;
3096 if (unlikely(cr4 != vmx->loaded_vmcs->host_state.cr4)) {
3098 vmx->loaded_vmcs->host_state.cr4 = cr4;
3101 vm_fail = __vmx_vcpu_run(vmx, (unsigned long *)&vcpu->arch.regs,
3102 __vmx_vcpu_run_flags(vmx));
3104 if (vmx->msr_autoload.host.nr)
3105 vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, vmx->msr_autoload.host.nr);
3106 if (vmx->msr_autoload.guest.nr)
3107 vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, vmx->msr_autoload.guest.nr);
3143 struct vcpu_vmx *vmx = to_vmx(vcpu);
3150 if (vmx->nested.enlightened_vmcs_enabled && !vmx->nested.hv_evmcs) {
3165 struct vcpu_vmx *vmx = to_vmx(vcpu);
3177 if (vmx->nested.apic_access_page) { /* shouldn't happen */
3178 kvm_release_page_clean(vmx->nested.apic_access_page);
3179 vmx->nested.apic_access_page = NULL;
3183 vmx->nested.apic_access_page = page;
3184 hpa = page_to_phys(vmx->nested.apic_access_page);
3198 map = &vmx->nested.virtual_apic_map;
3213 exec_controls_clearbit(vmx, CPU_BASED_TPR_SHADOW);
3224 map = &vmx->nested.pi_desc_map;
3227 vmx->nested.pi_desc =
3235 exec_controls_setbit(vmx, CPU_BASED_USE_MSR_BITMAPS);
3237 exec_controls_clearbit(vmx, CPU_BASED_USE_MSR_BITMAPS);
3264 struct vcpu_vmx *vmx = to_vmx(vcpu);
3270 if (WARN_ON_ONCE(vmx->nested.pml_full))
3282 vmx->nested.pml_full = true;
3344 struct vcpu_vmx *vmx = to_vmx(vcpu);
3357 evaluate_pending_interrupts = exec_controls_get(vmx) &
3362 if (!vmx->nested.nested_run_pending ||
3364 vmx->nested.vmcs01_debugctl = vmcs_read64(GUEST_IA32_DEBUGCTL);
3366 (!vmx->nested.nested_run_pending ||
3368 vmx->nested.vmcs01_guest_bndcfgs = vmcs_read64(GUEST_BNDCFGS);
3389 vmx_switch_vmcs(vcpu, &vmx->nested.vmcs02);
3391 prepare_vmcs02_early(vmx, &vmx->vmcs01, vmcs12);
3395 vmx_switch_vmcs(vcpu, &vmx->vmcs01);
3400 vmx_switch_vmcs(vcpu, &vmx->vmcs01);
3464 vmx->nested.preemption_timer_expired = false;
3489 vmx_switch_vmcs(vcpu, &vmx->vmcs01);
3496 if (enable_shadow_vmcs || vmx->nested.hv_evmcs)
3497 vmx->nested.need_vmcs12_to_shadow_sync = true;
3509 struct vcpu_vmx *vmx = to_vmx(vcpu);
3524 if (CC(!vmx->nested.hv_evmcs && vmx->nested.current_vmptr == -1ull))
3538 if (vmx->nested.hv_evmcs) {
3539 copy_enlightened_to_vmcs12(vmx);
3543 copy_shadow_to_vmcs12(vmx);
3577 vmx->nested.nested_run_pending = 1;
3578 vmx->nested.has_preemption_timer_deadline = false;
3585 kvm_apic_has_interrupt(vcpu) == vmx->nested.posted_intr_nv) {
3586 vmx->nested.pi_pending = true;
3588 kvm_apic_clear_irr(vcpu, vmx->nested.posted_intr_nv);
3592 vmx->vcpu.arch.l1tf_flush_l1d = true;
3616 vmx->nested.nested_run_pending = 0;
3622 vmx->nested.nested_run_pending = 0;
3758 struct vcpu_vmx *vmx = to_vmx(vcpu);
3763 if (!vmx->nested.pi_desc || !vmx->nested.pi_pending)
3766 vmx->nested.pi_pending = false;
3767 if (!pi_test_and_clear_on(vmx->nested.pi_desc))
3770 max_irr = find_last_bit((unsigned long *)vmx->nested.pi_desc->pir, 256);
3772 vapic_page = vmx->nested.virtual_apic_map.hva;
3776 __kvm_apic_update_irr(vmx->nested.pi_desc->pir,
3857 struct vcpu_vmx *vmx = to_vmx(vcpu);
3860 vmx->nested.nested_run_pending || kvm_event_needs_reinjection(vcpu);
3861 bool mtf_pending = vmx->nested.mtf_pending;
3869 vmx->nested.mtf_pending = false;
4019 struct vcpu_vmx *vmx = to_vmx(vcpu);
4060 vmx->nested.need_sync_vmcs02_to_vmcs12_rare = false;
4066 struct vcpu_vmx *vmx = to_vmx(vcpu);
4069 if (!vmx->nested.need_sync_vmcs02_to_vmcs12_rare)
4073 WARN_ON_ONCE(vmx->loaded_vmcs != &vmx->vmcs01);
4076 vmx->loaded_vmcs = &vmx->nested.vmcs02;
4077 vmx_vcpu_load_vmcs(vcpu, cpu, &vmx->vmcs01);
4081 vmx->loaded_vmcs = &vmx->vmcs01;
4082 vmx_vcpu_load_vmcs(vcpu, cpu, &vmx->nested.vmcs02);
4094 struct vcpu_vmx *vmx = to_vmx(vcpu);
4096 if (vmx->nested.hv_evmcs)
4099 vmx->nested.need_sync_vmcs02_to_vmcs12_rare = !vmx->nested.hv_evmcs;
4121 !vmx->nested.nested_run_pending)
4348 static inline u64 nested_vmx_get_vmcs01_guest_efer(struct vcpu_vmx *vmx)
4353 if (vm_entry_controls_get(vmx) & VM_ENTRY_LOAD_IA32_EFER)
4359 for (i = 0; i < vmx->msr_autoload.guest.nr; ++i) {
4360 if (vmx->msr_autoload.guest.val[i].index == MSR_EFER)
4361 return vmx->msr_autoload.guest.val[i].value;
4364 efer_msr = vmx_find_uret_msr(vmx, MSR_EFER);
4374 struct vcpu_vmx *vmx = to_vmx(vcpu);
4398 vmx_set_efer(vcpu, nested_vmx_get_vmcs01_guest_efer(vmx));
4487 struct vcpu_vmx *vmx = to_vmx(vcpu);
4491 WARN_ON_ONCE(vmx->nested.nested_run_pending);
4508 * VCPU_EXREG_PDPTR will be clobbered in arch/x86/kvm/vmx/vmx.h between
4523 if (likely(!vmx->fail)) {
4563 vmx_switch_vmcs(vcpu, &vmx->vmcs01);
4577 vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, vmx->msr_autoload.host.nr);
4578 vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, vmx->msr_autoload.guest.nr);
4580 if (vmx->nested.l1_tpr_threshold != -1)
4581 vmcs_write32(TPR_THRESHOLD, vmx->nested.l1_tpr_threshold);
4584 decache_tsc_multiplier(vmx);
4586 if (vmx->nested.change_vmcs01_virtual_apic_mode) {
4587 vmx->nested.change_vmcs01_virtual_apic_mode = false;
4592 if (vmx->nested.apic_access_page) {
4593 kvm_release_page_clean(vmx->nested.apic_access_page);
4594 vmx->nested.apic_access_page = NULL;
4596 kvm_vcpu_unmap(vcpu, &vmx->nested.virtual_apic_map, true);
4597 kvm_vcpu_unmap(vcpu, &vmx->nested.pi_desc_map, true);
4598 vmx->nested.pi_desc = NULL;
4600 if (vmx->nested.reload_vmcs01_apic_access_page) {
4601 vmx->nested.reload_vmcs01_apic_access_page = false;
4606 (enable_shadow_vmcs || vmx->nested.hv_evmcs))
4607 vmx->nested.need_vmcs12_to_shadow_sync = true;
4612 if (likely(!vmx->fail)) {
4651 vmx->fail = 0;
4655 * Decode the memory-address operand of a vmx instruction, as recorded on an
4786 struct vcpu_vmx *vmx;
4791 vmx = to_vmx(vcpu);
4793 vmx->nested.msrs.entry_ctls_high |=
4795 vmx->nested.msrs.exit_ctls_high |=
4798 vmx->nested.msrs.entry_ctls_high &=
4800 vmx->nested.msrs.exit_ctls_high &=
4835 struct vcpu_vmx *vmx = to_vmx(vcpu);
4836 struct loaded_vmcs *loaded_vmcs = vmx->loaded_vmcs;
4844 WARN_ON(loaded_vmcs == &vmx->vmcs01 && loaded_vmcs->shadow_vmcs);
4856 struct vcpu_vmx *vmx = to_vmx(vcpu);
4859 r = alloc_loaded_vmcs(&vmx->nested.vmcs02);
4863 vmx->nested.cached_vmcs12 = kzalloc(VMCS12_SIZE, GFP_KERNEL_ACCOUNT);
4864 if (!vmx->nested.cached_vmcs12)
4867 vmx->nested.cached_shadow_vmcs12 = kzalloc(VMCS12_SIZE, GFP_KERNEL_ACCOUNT);
4868 if (!vmx->nested.cached_shadow_vmcs12)
4874 hrtimer_init(&vmx->nested.preemption_timer, CLOCK_MONOTONIC,
4876 vmx->nested.preemption_timer.function = vmx_preemption_timer_fn;
4878 vmx->nested.vpid02 = allocate_vpid();
4880 vmx->nested.vmcs02_initialized = false;
4881 vmx->nested.vmxon = true;
4884 vmx->pt_desc.guest.ctl = 0;
4891 kfree(vmx->nested.cached_shadow_vmcs12);
4894 kfree(vmx->nested.cached_vmcs12);
4897 free_loaded_vmcs(&vmx->nested.vmcs02);
4916 struct vcpu_vmx *vmx = to_vmx(vcpu);
4956 if (vmx->nested.vmxon)
4970 if ((vmx->msr_ia32_feature_control & VMXON_NEEDED_FEATURES)
4994 vmx->nested.vmxon_ptr = vmptr;
5004 struct vcpu_vmx *vmx = to_vmx(vcpu);
5006 if (vmx->nested.current_vmptr == -1ull)
5014 copy_shadow_to_vmcs12(vmx);
5015 vmx_disable_shadow_vmcs(vmx);
5017 vmx->nested.posted_intr_nv = -1;
5021 vmx->nested.current_vmptr >> PAGE_SHIFT,
5022 vmx->nested.cached_vmcs12, 0, VMCS12_SIZE);
5026 vmx->nested.current_vmptr = -1ull;
5046 struct vcpu_vmx *vmx = to_vmx(vcpu);
5061 if (vmptr == vmx->nested.vmxon_ptr)
5072 * vmx->nested.hv_evmcs but this shouldn't be a problem.
5074 if (likely(!vmx->nested.enlightened_vmcs_enabled ||
5076 if (vmptr == vmx->nested.current_vmptr)
5107 struct vcpu_vmx *vmx = to_vmx(vcpu);
5122 if (vmx->nested.current_vmptr == -1ull ||
5191 struct vcpu_vmx *vmx = to_vmx(vcpu);
5214 if (vmx->nested.current_vmptr == -1ull ||
5278 vmcs_load(vmx->vmcs01.shadow_vmcs);
5282 vmcs_clear(vmx->vmcs01.shadow_vmcs);
5283 vmcs_load(vmx->loaded_vmcs->vmcs);
5286 vmx->nested.dirty_vmcs12 = true;
5292 static void set_current_vmptr(struct vcpu_vmx *vmx, gpa_t vmptr)
5294 vmx->nested.current_vmptr = vmptr;
5296 secondary_exec_controls_setbit(vmx, SECONDARY_EXEC_SHADOW_VMCS);
5298 __pa(vmx->vmcs01.shadow_vmcs));
5299 vmx->nested.need_vmcs12_to_shadow_sync = true;
5301 vmx->nested.dirty_vmcs12 = true;
5307 struct vcpu_vmx *vmx = to_vmx(vcpu);
5320 if (vmptr == vmx->nested.vmxon_ptr)
5324 if (vmx->nested.hv_evmcs)
5327 if (vmx->nested.current_vmptr != vmptr) {
5358 memcpy(vmx->nested.cached_vmcs12, new_vmcs12, VMCS12_SIZE);
5361 set_current_vmptr(vmx, vmptr);
5406 struct vcpu_vmx *vmx = to_vmx(vcpu);
5417 if (!(vmx->nested.msrs.secondary_ctls_high &
5419 !(vmx->nested.msrs.ept_caps & VMX_EPT_INVEPT_BIT)) {
5430 types = (vmx->nested.msrs.ept_caps >> VMX_EPT_EXTENT_SHIFT) & 6;
5485 struct vcpu_vmx *vmx = to_vmx(vcpu);
5497 if (!(vmx->nested.msrs.secondary_ctls_high &
5499 !(vmx->nested.msrs.vpid_caps & VMX_VPID_INVVPID_BIT)) {
5510 types = (vmx->nested.msrs.vpid_caps &
5517 /* according to the intel vmx instruction reference, the memory
5607 struct vcpu_vmx *vmx = to_vmx(vcpu);
5641 nested_vmx_vmexit(vcpu, vmx->exit_reason.full,
6050 struct vcpu_vmx *vmx = to_vmx(vcpu);
6051 union vmx_exit_reason exit_reason = vmx->exit_reason;
6055 WARN_ON_ONCE(vmx->nested.nested_run_pending);
6061 if (unlikely(vmx->fail)) {
6104 struct vcpu_vmx *vmx;
6110 .hdr.vmx.flags = 0,
6111 .hdr.vmx.vmxon_pa = -1ull,
6112 .hdr.vmx.vmcs12_pa = -1ull,
6113 .hdr.vmx.preemption_timer_deadline = 0,
6116 &user_kvm_nested_state->data.vmx[0];
6121 vmx = to_vmx(vcpu);
6125 (vmx->nested.vmxon || vmx->nested.smm.vmxon)) {
6126 kvm_state.hdr.vmx.vmxon_pa = vmx->nested.vmxon_ptr;
6127 kvm_state.hdr.vmx.vmcs12_pa = vmx->nested.current_vmptr;
6132 if (vmx->nested.hv_evmcs)
6141 if (vmx->nested.smm.vmxon)
6142 kvm_state.hdr.vmx.smm.flags |= KVM_STATE_NESTED_SMM_VMXON;
6144 if (vmx->nested.smm.guest_mode)
6145 kvm_state.hdr.vmx.smm.flags |= KVM_STATE_NESTED_SMM_GUEST_MODE;
6150 if (vmx->nested.nested_run_pending)
6153 if (vmx->nested.mtf_pending)
6157 vmx->nested.has_preemption_timer_deadline) {
6158 kvm_state.hdr.vmx.flags |=
6160 kvm_state.hdr.vmx.preemption_timer_deadline =
6161 vmx->nested.preemption_timer_deadline;
6187 if (!vmx->nested.need_vmcs12_to_shadow_sync) {
6188 if (vmx->nested.hv_evmcs)
6189 copy_enlightened_to_vmcs12(vmx);
6191 copy_shadow_to_vmcs12(vmx);
6231 struct vcpu_vmx *vmx = to_vmx(vcpu);
6235 &user_kvm_nested_state->data.vmx[0];
6241 if (kvm_state->hdr.vmx.vmxon_pa == -1ull) {
6242 if (kvm_state->hdr.vmx.smm.flags)
6245 if (kvm_state->hdr.vmx.vmcs12_pa != -1ull)
6263 if (!page_address_valid(vcpu, kvm_state->hdr.vmx.vmxon_pa))
6267 if ((kvm_state->hdr.vmx.smm.flags & KVM_STATE_NESTED_SMM_GUEST_MODE) &&
6271 if (kvm_state->hdr.vmx.smm.flags &
6275 if (kvm_state->hdr.vmx.flags & ~KVM_STATE_VMX_PREEMPTION_TIMER_DEADLINE)
6286 : kvm_state->hdr.vmx.smm.flags)
6289 if ((kvm_state->hdr.vmx.smm.flags & KVM_STATE_NESTED_SMM_GUEST_MODE) &&
6290 !(kvm_state->hdr.vmx.smm.flags & KVM_STATE_NESTED_SMM_VMXON))
6294 (!nested_vmx_allowed(vcpu) || !vmx->nested.enlightened_vmcs_enabled))
6299 if (kvm_state->hdr.vmx.vmxon_pa == -1ull)
6302 vmx->nested.vmxon_ptr = kvm_state->hdr.vmx.vmxon_pa;
6312 (kvm_state->hdr.vmx.vmcs12_pa != -1ull))
6318 if (kvm_state->hdr.vmx.vmcs12_pa != -1ull) {
6319 if (kvm_state->hdr.vmx.vmcs12_pa == kvm_state->hdr.vmx.vmxon_pa ||
6320 !page_address_valid(vcpu, kvm_state->hdr.vmx.vmcs12_pa))
6323 set_current_vmptr(vmx, kvm_state->hdr.vmx.vmcs12_pa);
6336 if (kvm_state->hdr.vmx.smm.flags & KVM_STATE_NESTED_SMM_VMXON) {
6337 vmx->nested.smm.vmxon = true;
6338 vmx->nested.vmxon = false;
6340 if (kvm_state->hdr.vmx.smm.flags & KVM_STATE_NESTED_SMM_GUEST_MODE)
6341 vmx->nested.smm.guest_mode = true;
6354 vmx->nested.nested_run_pending =
6357 vmx->nested.mtf_pending =
6382 vmx->nested.has_preemption_timer_deadline = false;
6383 if (kvm_state->hdr.vmx.flags & KVM_STATE_VMX_PREEMPTION_TIMER_DEADLINE) {
6384 vmx->nested.has_preemption_timer_deadline = true;
6385 vmx->nested.preemption_timer_deadline =
6386 kvm_state->hdr.vmx.preemption_timer_deadline;
6394 vmx->nested.dirty_vmcs12 = true;
6402 vmx->nested.nested_run_pending = 0;