Lines Matching refs:vcpu
103 ((struct kvm_vcpu *)(ctxt)->vcpu)
125 static void update_cr8_intercept(struct kvm_vcpu *vcpu);
126 static void process_nmi(struct kvm_vcpu *vcpu);
127 static void __kvm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags);
128 static void store_regs(struct kvm_vcpu *vcpu);
129 static int sync_regs(struct kvm_vcpu *vcpu);
130 static int kvm_vcpu_do_singlestep(struct kvm_vcpu *vcpu);
132 static int __set_sregs2(struct kvm_vcpu *vcpu, struct kvm_sregs2 *sregs2);
133 static void __get_sregs2(struct kvm_vcpu *vcpu, struct kvm_sregs2 *sregs2);
355 static inline void kvm_async_pf_hash_reset(struct kvm_vcpu *vcpu)
359 vcpu->arch.apf.gfns[i] = ~0;
474 u64 kvm_get_apic_base(struct kvm_vcpu *vcpu)
476 return vcpu->arch.apic_base;
479 enum lapic_mode kvm_get_apic_mode(struct kvm_vcpu *vcpu)
481 return kvm_apic_mode(kvm_get_apic_base(vcpu));
485 int kvm_set_apic_base(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
487 enum lapic_mode old_mode = kvm_get_apic_mode(vcpu);
489 u64 reserved_bits = kvm_vcpu_reserved_gpa_bits_raw(vcpu) | 0x2ff |
490 (guest_cpuid_has(vcpu, X86_FEATURE_X2APIC) ? 0 : X2APIC_ENABLE);
501 kvm_lapic_set_base(vcpu, msr_info->data);
502 kvm_recalculate_apic_map(vcpu->kvm);
573 void kvm_deliver_exception_payload(struct kvm_vcpu *vcpu,
586 vcpu->arch.dr6 &= ~DR_TRAP_BITS;
603 vcpu->arch.dr6 |= DR6_ACTIVE_LOW;
604 vcpu->arch.dr6 |= ex->payload;
605 vcpu->arch.dr6 ^= ex->payload & DR6_ACTIVE_LOW;
613 vcpu->arch.dr6 &= ~BIT(12);
616 vcpu->arch.cr2 = ex->payload;
625 static void kvm_queue_exception_vmexit(struct kvm_vcpu *vcpu, unsigned int vector,
629 struct kvm_queued_exception *ex = &vcpu->arch.exception_vmexit;
641 static void kvm_leave_nested(struct kvm_vcpu *vcpu)
643 kvm_x86_ops.nested_ops->leave_nested(vcpu);
646 static void kvm_multiple_exception(struct kvm_vcpu *vcpu,
653 kvm_make_request(KVM_REQ_EVENT, vcpu);
662 if (!reinject && is_guest_mode(vcpu) &&
663 kvm_x86_ops.nested_ops->is_exception_vmexit(vcpu, nr, error_code)) {
664 kvm_queue_exception_vmexit(vcpu, nr, has_error, error_code,
669 if (!vcpu->arch.exception.pending && !vcpu->arch.exception.injected) {
679 WARN_ON_ONCE(kvm_is_exception_pending(vcpu));
680 vcpu->arch.exception.injected = true;
690 vcpu->arch.exception.pending = true;
691 vcpu->arch.exception.injected = false;
693 vcpu->arch.exception.has_error_code = has_error;
694 vcpu->arch.exception.vector = nr;
695 vcpu->arch.exception.error_code = error_code;
696 vcpu->arch.exception.has_payload = has_payload;
697 vcpu->arch.exception.payload = payload;
698 if (!is_guest_mode(vcpu))
699 kvm_deliver_exception_payload(vcpu,
700 &vcpu->arch.exception);
705 prev_nr = vcpu->arch.exception.vector;
708 kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);
719 vcpu->arch.exception.injected = false;
720 vcpu->arch.exception.pending = false;
722 kvm_queue_exception_e(vcpu, DF_VECTOR, 0);
731 void kvm_queue_exception(struct kvm_vcpu *vcpu, unsigned nr)
733 kvm_multiple_exception(vcpu, nr, false, 0, false, 0, false);
737 void kvm_requeue_exception(struct kvm_vcpu *vcpu, unsigned nr)
739 kvm_multiple_exception(vcpu, nr, false, 0, false, 0, true);
743 void kvm_queue_exception_p(struct kvm_vcpu *vcpu, unsigned nr,
746 kvm_multiple_exception(vcpu, nr, false, 0, true, payload, false);
750 static void kvm_queue_exception_e_p(struct kvm_vcpu *vcpu, unsigned nr,
753 kvm_multiple_exception(vcpu, nr, true, error_code,
757 int kvm_complete_insn_gp(struct kvm_vcpu *vcpu, int err)
760 kvm_inject_gp(vcpu, 0);
762 return kvm_skip_emulated_instruction(vcpu);
768 static int complete_emulated_insn_gp(struct kvm_vcpu *vcpu, int err)
771 kvm_inject_gp(vcpu, 0);
775 return kvm_emulate_instruction(vcpu, EMULTYPE_NO_DECODE | EMULTYPE_SKIP |
779 void kvm_inject_page_fault(struct kvm_vcpu *vcpu, struct x86_exception *fault)
781 ++vcpu->stat.pf_guest;
787 if (is_guest_mode(vcpu) && fault->async_page_fault)
788 kvm_queue_exception_vmexit(vcpu, PF_VECTOR,
792 kvm_queue_exception_e_p(vcpu, PF_VECTOR, fault->error_code,
796 void kvm_inject_emulated_page_fault(struct kvm_vcpu *vcpu,
802 fault_mmu = fault->nested_page_fault ? vcpu->arch.mmu :
803 vcpu->arch.walk_mmu;
811 kvm_mmu_invalidate_addr(vcpu, fault_mmu, fault->address,
814 fault_mmu->inject_page_fault(vcpu, fault);
818 void kvm_inject_nmi(struct kvm_vcpu *vcpu)
820 atomic_inc(&vcpu->arch.nmi_queued);
821 kvm_make_request(KVM_REQ_NMI, vcpu);
824 void kvm_queue_exception_e(struct kvm_vcpu *vcpu, unsigned nr, u32 error_code)
826 kvm_multiple_exception(vcpu, nr, true, error_code, false, 0, false);
830 void kvm_requeue_exception_e(struct kvm_vcpu *vcpu, unsigned nr, u32 error_code)
832 kvm_multiple_exception(vcpu, nr, true, error_code, false, 0, true);
840 bool kvm_require_cpl(struct kvm_vcpu *vcpu, int required_cpl)
842 if (static_call(kvm_x86_get_cpl)(vcpu) <= required_cpl)
844 kvm_queue_exception_e(vcpu, GP_VECTOR, 0);
848 bool kvm_require_dr(struct kvm_vcpu *vcpu, int dr)
850 if ((dr != 4 && dr != 5) || !kvm_is_cr4_bit_set(vcpu, X86_CR4_DE))
853 kvm_queue_exception(vcpu, UD_VECTOR);
858 static inline u64 pdptr_rsvd_bits(struct kvm_vcpu *vcpu)
860 return vcpu->arch.reserved_gpa_bits | rsvd_bits(5, 8) | rsvd_bits(1, 2);
866 int load_pdptrs(struct kvm_vcpu *vcpu, unsigned long cr3)
868 struct kvm_mmu *mmu = vcpu->arch.walk_mmu;
879 real_gpa = kvm_translate_gpa(vcpu, mmu, gfn_to_gpa(pdpt_gfn),
885 ret = kvm_vcpu_read_guest_page(vcpu, gpa_to_gfn(real_gpa), pdpte,
892 (pdpte[i] & pdptr_rsvd_bits(vcpu))) {
902 kvm_mmu_free_roots(vcpu->kvm, mmu, KVM_MMU_ROOT_CURRENT);
905 kvm_register_mark_dirty(vcpu, VCPU_EXREG_PDPTR);
906 kvm_make_request(KVM_REQ_LOAD_MMU_PGD, vcpu);
907 vcpu->arch.pdptrs_from_userspace = false;
913 static bool kvm_is_valid_cr0(struct kvm_vcpu *vcpu, unsigned long cr0)
926 return static_call(kvm_x86_is_valid_cr0)(vcpu, cr0);
929 void kvm_post_set_cr0(struct kvm_vcpu *vcpu, unsigned long old_cr0, unsigned long cr0)
944 kvm_init_mmu(vcpu);
950 kvm_clear_async_pf_completion_queue(vcpu);
951 kvm_async_pf_hash_reset(vcpu);
958 kvm_make_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu);
962 kvm_mmu_reset_context(vcpu);
965 kvm_arch_has_noncoherent_dma(vcpu->kvm) &&
966 !kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_CD_NW_CLEARED))
967 kvm_zap_gfn_range(vcpu->kvm, 0, ~0ULL);
971 int kvm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0)
973 unsigned long old_cr0 = kvm_read_cr0(vcpu);
975 if (!kvm_is_valid_cr0(vcpu, cr0))
984 if ((vcpu->arch.efer & EFER_LME) && !is_paging(vcpu) &&
988 if (!is_pae(vcpu))
990 static_call(kvm_x86_get_cs_db_l_bits)(vcpu, &cs_db, &cs_l);
995 if (!(vcpu->arch.efer & EFER_LME) && (cr0 & X86_CR0_PG) &&
996 is_pae(vcpu) && ((cr0 ^ old_cr0) & X86_CR0_PDPTR_BITS) &&
997 !load_pdptrs(vcpu, kvm_read_cr3(vcpu)))
1001 (is_64_bit_mode(vcpu) || kvm_is_cr4_bit_set(vcpu, X86_CR4_PCIDE)))
1004 static_call(kvm_x86_set_cr0)(vcpu, cr0);
1006 kvm_post_set_cr0(vcpu, old_cr0, cr0);
1012 void kvm_lmsw(struct kvm_vcpu *vcpu, unsigned long msw)
1014 (void)kvm_set_cr0(vcpu, kvm_read_cr0_bits(vcpu, ~0x0eul) | (msw & 0x0f));
1018 void kvm_load_guest_xsave_state(struct kvm_vcpu *vcpu)
1020 if (vcpu->arch.guest_state_protected)
1023 if (kvm_is_cr4_bit_set(vcpu, X86_CR4_OSXSAVE)) {
1025 if (vcpu->arch.xcr0 != host_xcr0)
1026 xsetbv(XCR_XFEATURE_ENABLED_MASK, vcpu->arch.xcr0);
1028 if (guest_can_use(vcpu, X86_FEATURE_XSAVES) &&
1029 vcpu->arch.ia32_xss != host_xss)
1030 wrmsrl(MSR_IA32_XSS, vcpu->arch.ia32_xss);
1034 vcpu->arch.pkru != vcpu->arch.host_pkru &&
1035 ((vcpu->arch.xcr0 & XFEATURE_MASK_PKRU) ||
1036 kvm_is_cr4_bit_set(vcpu, X86_CR4_PKE)))
1037 write_pkru(vcpu->arch.pkru);
1041 void kvm_load_host_xsave_state(struct kvm_vcpu *vcpu)
1043 if (vcpu->arch.guest_state_protected)
1047 ((vcpu->arch.xcr0 & XFEATURE_MASK_PKRU) ||
1048 kvm_is_cr4_bit_set(vcpu, X86_CR4_PKE))) {
1049 vcpu->arch.pkru = rdpkru();
1050 if (vcpu->arch.pkru != vcpu->arch.host_pkru)
1051 write_pkru(vcpu->arch.host_pkru);
1054 if (kvm_is_cr4_bit_set(vcpu, X86_CR4_OSXSAVE)) {
1056 if (vcpu->arch.xcr0 != host_xcr0)
1059 if (guest_can_use(vcpu, X86_FEATURE_XSAVES) &&
1060 vcpu->arch.ia32_xss != host_xss)
1068 static inline u64 kvm_guest_supported_xfd(struct kvm_vcpu *vcpu)
1070 return vcpu->arch.guest_supported_xcr0 & XFEATURE_MASK_USER_DYNAMIC;
1074 static int __kvm_set_xcr(struct kvm_vcpu *vcpu, u32 index, u64 xcr)
1077 u64 old_xcr0 = vcpu->arch.xcr0;
1093 valid_bits = vcpu->arch.guest_supported_xcr0 | XFEATURE_MASK_FP;
1112 vcpu->arch.xcr0 = xcr0;
1115 kvm_update_cpuid_runtime(vcpu);
1119 int kvm_emulate_xsetbv(struct kvm_vcpu *vcpu)
1122 if (static_call(kvm_x86_get_cpl)(vcpu) != 0 ||
1123 __kvm_set_xcr(vcpu, kvm_rcx_read(vcpu), kvm_read_edx_eax(vcpu))) {
1124 kvm_inject_gp(vcpu, 0);
1128 return kvm_skip_emulated_instruction(vcpu);
1132 bool __kvm_is_valid_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
1137 if (cr4 & vcpu->arch.cr4_guest_rsvd_bits)
1144 static bool kvm_is_valid_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
1146 return __kvm_is_valid_cr4(vcpu, cr4) &&
1147 static_call(kvm_x86_is_valid_cr4)(vcpu, cr4);
1150 void kvm_post_set_cr4(struct kvm_vcpu *vcpu, unsigned long old_cr4, unsigned long cr4)
1153 kvm_mmu_reset_context(vcpu);
1165 kvm_mmu_unload(vcpu);
1177 kvm_make_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu);
1187 kvm_make_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu);
1192 int kvm_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
1194 unsigned long old_cr4 = kvm_read_cr4(vcpu);
1196 if (!kvm_is_valid_cr4(vcpu, cr4))
1199 if (is_long_mode(vcpu)) {
1204 } else if (is_paging(vcpu) && (cr4 & X86_CR4_PAE)
1206 && !load_pdptrs(vcpu, kvm_read_cr3(vcpu)))
1211 if ((kvm_read_cr3(vcpu) & X86_CR3_PCID_MASK) || !is_long_mode(vcpu))
1215 static_call(kvm_x86_set_cr4)(vcpu, cr4);
1217 kvm_post_set_cr4(vcpu, old_cr4, cr4);
1223 static void kvm_invalidate_pcid(struct kvm_vcpu *vcpu, unsigned long pcid)
1225 struct kvm_mmu *mmu = vcpu->arch.mmu;
1237 kvm_make_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu);
1246 if (kvm_get_active_pcid(vcpu) == pcid) {
1247 kvm_make_request(KVM_REQ_MMU_SYNC, vcpu);
1248 kvm_make_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu);
1256 if (!kvm_is_cr4_bit_set(vcpu, X86_CR4_PCIDE))
1260 if (kvm_get_pcid(vcpu, mmu->prev_roots[i].pgd) == pcid)
1263 kvm_mmu_free_roots(vcpu->kvm, mmu, roots_to_free);
1266 int kvm_set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3)
1271 if (kvm_is_cr4_bit_set(vcpu, X86_CR4_PCIDE)) {
1279 if (cr3 == kvm_read_cr3(vcpu) && !is_pae_paging(vcpu))
1287 if (kvm_vcpu_is_illegal_gpa(vcpu, cr3))
1290 if (is_pae_paging(vcpu) && !load_pdptrs(vcpu, cr3))
1293 if (cr3 != kvm_read_cr3(vcpu))
1294 kvm_mmu_new_pgd(vcpu, cr3);
1296 vcpu->arch.cr3 = cr3;
1297 kvm_register_mark_dirty(vcpu, VCPU_EXREG_CR3);
1309 kvm_invalidate_pcid(vcpu, pcid);
1315 int kvm_set_cr8(struct kvm_vcpu *vcpu, unsigned long cr8)
1319 if (lapic_in_kernel(vcpu))
1320 kvm_lapic_set_tpr(vcpu, cr8);
1322 vcpu->arch.cr8 = cr8;
1327 unsigned long kvm_get_cr8(struct kvm_vcpu *vcpu)
1329 if (lapic_in_kernel(vcpu))
1330 return kvm_lapic_get_cr8(vcpu);
1332 return vcpu->arch.cr8;
1336 static void kvm_update_dr0123(struct kvm_vcpu *vcpu)
1340 if (!(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP)) {
1342 vcpu->arch.eff_db[i] = vcpu->arch.db[i];
1346 void kvm_update_dr7(struct kvm_vcpu *vcpu)
1350 if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP)
1351 dr7 = vcpu->arch.guest_debug_dr7;
1353 dr7 = vcpu->arch.dr7;
1354 static_call(kvm_x86_set_dr7)(vcpu, dr7);
1355 vcpu->arch.switch_db_regs &= ~KVM_DEBUGREG_BP_ENABLED;
1357 vcpu->arch.switch_db_regs |= KVM_DEBUGREG_BP_ENABLED;
1361 static u64 kvm_dr6_fixed(struct kvm_vcpu *vcpu)
1365 if (!guest_cpuid_has(vcpu, X86_FEATURE_RTM))
1368 if (!guest_cpuid_has(vcpu, X86_FEATURE_BUS_LOCK_DETECT))
1373 int kvm_set_dr(struct kvm_vcpu *vcpu, int dr, unsigned long val)
1375 size_t size = ARRAY_SIZE(vcpu->arch.db);
1379 vcpu->arch.db[array_index_nospec(dr, size)] = val;
1380 if (!(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP))
1381 vcpu->arch.eff_db[dr] = val;
1387 vcpu->arch.dr6 = (val & DR6_VOLATILE) | kvm_dr6_fixed(vcpu);
1393 vcpu->arch.dr7 = (val & DR7_VOLATILE) | DR7_FIXED_1;
1394 kvm_update_dr7(vcpu);
1402 void kvm_get_dr(struct kvm_vcpu *vcpu, int dr, unsigned long *val)
1404 size_t size = ARRAY_SIZE(vcpu->arch.db);
1408 *val = vcpu->arch.db[array_index_nospec(dr, size)];
1412 *val = vcpu->arch.dr6;
1416 *val = vcpu->arch.dr7;
1422 int kvm_emulate_rdpmc(struct kvm_vcpu *vcpu)
1424 u32 ecx = kvm_rcx_read(vcpu);
1427 if (kvm_pmu_rdpmc(vcpu, ecx, &data)) {
1428 kvm_inject_gp(vcpu, 0);
1432 kvm_rax_write(vcpu, (u32)data);
1433 kvm_rdx_write(vcpu, data >> 32);
1434 return kvm_skip_emulated_instruction(vcpu);
1702 static int do_get_msr_feature(struct kvm_vcpu *vcpu, unsigned index, u64 *data)
1725 static bool __kvm_valid_efer(struct kvm_vcpu *vcpu, u64 efer)
1727 if (efer & EFER_AUTOIBRS && !guest_cpuid_has(vcpu, X86_FEATURE_AUTOIBRS))
1730 if (efer & EFER_FFXSR && !guest_cpuid_has(vcpu, X86_FEATURE_FXSR_OPT))
1733 if (efer & EFER_SVME && !guest_cpuid_has(vcpu, X86_FEATURE_SVM))
1737 !guest_cpuid_has(vcpu, X86_FEATURE_LM))
1740 if (efer & EFER_NX && !guest_cpuid_has(vcpu, X86_FEATURE_NX))
1746 bool kvm_valid_efer(struct kvm_vcpu *vcpu, u64 efer)
1751 return __kvm_valid_efer(vcpu, efer);
1755 static int set_efer(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
1757 u64 old_efer = vcpu->arch.efer;
1765 if (!__kvm_valid_efer(vcpu, efer))
1768 if (is_paging(vcpu) &&
1769 (vcpu->arch.efer & EFER_LME) != (efer & EFER_LME))
1774 efer |= vcpu->arch.efer & EFER_LMA;
1776 r = static_call(kvm_x86_set_efer)(vcpu, efer);
1783 kvm_mmu_reset_context(vcpu);
1794 bool kvm_msr_allowed(struct kvm_vcpu *vcpu, u32 index, u32 type)
1798 struct kvm *kvm = vcpu->kvm;
1843 static int __kvm_set_msr(struct kvm_vcpu *vcpu, u32 index, u64 data,
1854 if (is_noncanonical_address(data, vcpu))
1871 data = __canonical_address(data, vcpu_virt_addr_bits(vcpu));
1878 !guest_cpuid_has(vcpu, X86_FEATURE_RDTSCP) &&
1879 !guest_cpuid_has(vcpu, X86_FEATURE_RDPID))
1891 if (guest_cpuid_is_intel(vcpu) && (data >> 32) != 0)
1902 return static_call(kvm_x86_set_msr)(vcpu, &msr);
1905 static int kvm_set_msr_ignored_check(struct kvm_vcpu *vcpu,
1908 int ret = __kvm_set_msr(vcpu, index, data, host_initiated);
1923 int __kvm_get_msr(struct kvm_vcpu *vcpu, u32 index, u64 *data,
1935 !guest_cpuid_has(vcpu, X86_FEATURE_RDTSCP) &&
1936 !guest_cpuid_has(vcpu, X86_FEATURE_RDPID))
1944 ret = static_call(kvm_x86_get_msr)(vcpu, &msr);
1950 static int kvm_get_msr_ignored_check(struct kvm_vcpu *vcpu,
1953 int ret = __kvm_get_msr(vcpu, index, data, host_initiated);
1965 static int kvm_get_msr_with_filter(struct kvm_vcpu *vcpu, u32 index, u64 *data)
1967 if (!kvm_msr_allowed(vcpu, index, KVM_MSR_FILTER_READ))
1969 return kvm_get_msr_ignored_check(vcpu, index, data, false);
1972 static int kvm_set_msr_with_filter(struct kvm_vcpu *vcpu, u32 index, u64 data)
1974 if (!kvm_msr_allowed(vcpu, index, KVM_MSR_FILTER_WRITE))
1976 return kvm_set_msr_ignored_check(vcpu, index, data, false);
1979 int kvm_get_msr(struct kvm_vcpu *vcpu, u32 index, u64 *data)
1981 return kvm_get_msr_ignored_check(vcpu, index, data, false);
1985 int kvm_set_msr(struct kvm_vcpu *vcpu, u32 index, u64 data)
1987 return kvm_set_msr_ignored_check(vcpu, index, data, false);
1991 static void complete_userspace_rdmsr(struct kvm_vcpu *vcpu)
1993 if (!vcpu->run->msr.error) {
1994 kvm_rax_write(vcpu, (u32)vcpu->run->msr.data);
1995 kvm_rdx_write(vcpu, vcpu->run->msr.data >> 32);
1999 static int complete_emulated_msr_access(struct kvm_vcpu *vcpu)
2001 return complete_emulated_insn_gp(vcpu, vcpu->run->msr.error);
2004 static int complete_emulated_rdmsr(struct kvm_vcpu *vcpu)
2006 complete_userspace_rdmsr(vcpu);
2007 return complete_emulated_msr_access(vcpu);
2010 static int complete_fast_msr_access(struct kvm_vcpu *vcpu)
2012 return static_call(kvm_x86_complete_emulated_msr)(vcpu, vcpu->run->msr.error);
2015 static int complete_fast_rdmsr(struct kvm_vcpu *vcpu)
2017 complete_userspace_rdmsr(vcpu);
2018 return complete_fast_msr_access(vcpu);
2033 static int kvm_msr_user_space(struct kvm_vcpu *vcpu, u32 index,
2035 int (*completion)(struct kvm_vcpu *vcpu),
2041 if (!(vcpu->kvm->arch.user_space_msr_mask & msr_reason))
2044 vcpu->run->exit_reason = exit_reason;
2045 vcpu->run->msr.error = 0;
2046 memset(vcpu->run->msr.pad, 0, sizeof(vcpu->run->msr.pad));
2047 vcpu->run->msr.reason = msr_reason;
2048 vcpu->run->msr.index = index;
2049 vcpu->run->msr.data = data;
2050 vcpu->arch.complete_userspace_io = completion;
2055 int kvm_emulate_rdmsr(struct kvm_vcpu *vcpu)
2057 u32 ecx = kvm_rcx_read(vcpu);
2061 r = kvm_get_msr_with_filter(vcpu, ecx, &data);
2066 kvm_rax_write(vcpu, data & -1u);
2067 kvm_rdx_write(vcpu, (data >> 32) & -1u);
2070 if (kvm_msr_user_space(vcpu, ecx, KVM_EXIT_X86_RDMSR, 0,
2076 return static_call(kvm_x86_complete_emulated_msr)(vcpu, r);
2080 int kvm_emulate_wrmsr(struct kvm_vcpu *vcpu)
2082 u32 ecx = kvm_rcx_read(vcpu);
2083 u64 data = kvm_read_edx_eax(vcpu);
2086 r = kvm_set_msr_with_filter(vcpu, ecx, data);
2092 if (kvm_msr_user_space(vcpu, ecx, KVM_EXIT_X86_WRMSR, data,
2101 return static_call(kvm_x86_complete_emulated_msr)(vcpu, r);
2105 int kvm_emulate_as_nop(struct kvm_vcpu *vcpu)
2107 return kvm_skip_emulated_instruction(vcpu);
2110 int kvm_emulate_invd(struct kvm_vcpu *vcpu)
2113 return kvm_emulate_as_nop(vcpu);
2117 int kvm_handle_invalid_op(struct kvm_vcpu *vcpu)
2119 kvm_queue_exception(vcpu, UD_VECTOR);
2125 static int kvm_emulate_monitor_mwait(struct kvm_vcpu *vcpu, const char *insn)
2127 if (!kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_MWAIT_NEVER_UD_FAULTS) &&
2128 !guest_cpuid_has(vcpu, X86_FEATURE_MWAIT))
2129 return kvm_handle_invalid_op(vcpu);
2132 return kvm_emulate_as_nop(vcpu);
2134 int kvm_emulate_mwait(struct kvm_vcpu *vcpu)
2136 return kvm_emulate_monitor_mwait(vcpu, "MWAIT");
2140 int kvm_emulate_monitor(struct kvm_vcpu *vcpu)
2142 return kvm_emulate_monitor_mwait(vcpu, "MONITOR");
2146 static inline bool kvm_vcpu_exit_request(struct kvm_vcpu *vcpu)
2149 return vcpu->mode == EXITING_GUEST_MODE || kvm_request_pending(vcpu) ||
2160 static int handle_fastpath_set_x2apic_icr_irqoff(struct kvm_vcpu *vcpu, u64 data)
2162 if (!lapic_in_kernel(vcpu) || !apic_x2apic_mode(vcpu->arch.apic))
2169 return kvm_x2apic_icr_write(vcpu->arch.apic, data);
2174 static int handle_fastpath_set_tscdeadline(struct kvm_vcpu *vcpu, u64 data)
2176 if (!kvm_can_use_hv_timer(vcpu))
2179 kvm_set_lapic_tscdeadline_msr(vcpu, data);
2183 fastpath_t handle_fastpath_set_msr_irqoff(struct kvm_vcpu *vcpu)
2185 u32 msr = kvm_rcx_read(vcpu);
2189 kvm_vcpu_srcu_read_lock(vcpu);
2193 data = kvm_read_edx_eax(vcpu);
2194 if (!handle_fastpath_set_x2apic_icr_irqoff(vcpu, data)) {
2195 kvm_skip_emulated_instruction(vcpu);
2200 data = kvm_read_edx_eax(vcpu);
2201 if (!handle_fastpath_set_tscdeadline(vcpu, data)) {
2202 kvm_skip_emulated_instruction(vcpu);
2213 kvm_vcpu_srcu_read_unlock(vcpu);
2222 static int do_get_msr(struct kvm_vcpu *vcpu, unsigned index, u64 *data)
2224 return kvm_get_msr_ignored_check(vcpu, index, data, true);
2227 static int do_set_msr(struct kvm_vcpu *vcpu, unsigned index, u64 *data)
2238 if (kvm_vcpu_has_run(vcpu) && kvm_is_immutable_feature_msr(index)) {
2239 if (do_get_msr(vcpu, index, &val) || *data != val)
2245 return kvm_set_msr_ignored_check(vcpu, index, *data, true);
2360 static void kvm_write_system_time(struct kvm_vcpu *vcpu, gpa_t system_time,
2363 struct kvm_arch *ka = &vcpu->kvm->arch;
2365 if (vcpu->vcpu_id == 0 && !host_initiated) {
2367 kvm_make_request(KVM_REQ_MASTERCLOCK_UPDATE, vcpu);
2372 vcpu->arch.time = system_time;
2373 kvm_make_request(KVM_REQ_GLOBAL_CLOCK_UPDATE, vcpu);
2377 kvm_gpc_activate(&vcpu->arch.pv_time, system_time & ~1ULL,
2380 kvm_gpc_deactivate(&vcpu->arch.pv_time);
2433 static void kvm_vcpu_write_tsc_multiplier(struct kvm_vcpu *vcpu, u64 l1_multiplier);
2435 static int set_tsc_khz(struct kvm_vcpu *vcpu, u32 user_tsc_khz, bool scale)
2441 kvm_vcpu_write_tsc_multiplier(vcpu, kvm_caps.default_tsc_scaling_ratio);
2448 vcpu->arch.tsc_catchup = 1;
2449 vcpu->arch.tsc_always_catchup = 1;
2467 kvm_vcpu_write_tsc_multiplier(vcpu, ratio);
2471 static int kvm_set_tsc_khz(struct kvm_vcpu *vcpu, u32 user_tsc_khz)
2479 kvm_vcpu_write_tsc_multiplier(vcpu, kvm_caps.default_tsc_scaling_ratio);
2485 &vcpu->arch.virtual_tsc_shift,
2486 &vcpu->arch.virtual_tsc_mult);
2487 vcpu->arch.virtual_tsc_khz = user_tsc_khz;
2502 return set_tsc_khz(vcpu, user_tsc_khz, use_scaling);
2505 static u64 compute_guest_tsc(struct kvm_vcpu *vcpu, s64 kernel_ns)
2507 u64 tsc = pvclock_scale_delta(kernel_ns-vcpu->arch.this_tsc_nsec,
2508 vcpu->arch.virtual_tsc_mult,
2509 vcpu->arch.virtual_tsc_shift);
2510 tsc += vcpu->arch.this_tsc_write;
2521 static void kvm_track_tsc_matching(struct kvm_vcpu *vcpu)
2525 struct kvm_arch *ka = &vcpu->kvm->arch;
2529 atomic_read(&vcpu->kvm->online_vcpus));
2541 kvm_make_request(KVM_REQ_MASTERCLOCK_UPDATE, vcpu);
2543 trace_kvm_track_tsc(vcpu->vcpu_id, ka->nr_vcpus_matched_tsc,
2544 atomic_read(&vcpu->kvm->online_vcpus),
2574 static u64 kvm_compute_l1_tsc_offset(struct kvm_vcpu *vcpu, u64 target_tsc)
2578 tsc = kvm_scale_tsc(rdtsc(), vcpu->arch.l1_tsc_scaling_ratio);
2583 u64 kvm_read_l1_tsc(struct kvm_vcpu *vcpu, u64 host_tsc)
2585 return vcpu->arch.l1_tsc_offset +
2586 kvm_scale_tsc(host_tsc, vcpu->arch.l1_tsc_scaling_ratio);
2615 static void kvm_vcpu_write_tsc_offset(struct kvm_vcpu *vcpu, u64 l1_offset)
2617 trace_kvm_write_tsc_offset(vcpu->vcpu_id,
2618 vcpu->arch.l1_tsc_offset,
2621 vcpu->arch.l1_tsc_offset = l1_offset;
2628 if (is_guest_mode(vcpu))
2629 vcpu->arch.tsc_offset = kvm_calc_nested_tsc_offset(
2631 static_call(kvm_x86_get_l2_tsc_offset)(vcpu),
2632 static_call(kvm_x86_get_l2_tsc_multiplier)(vcpu));
2634 vcpu->arch.tsc_offset = l1_offset;
2636 static_call(kvm_x86_write_tsc_offset)(vcpu);
2639 static void kvm_vcpu_write_tsc_multiplier(struct kvm_vcpu *vcpu, u64 l1_multiplier)
2641 vcpu->arch.l1_tsc_scaling_ratio = l1_multiplier;
2644 if (is_guest_mode(vcpu))
2645 vcpu->arch.tsc_scaling_ratio = kvm_calc_nested_tsc_multiplier(
2647 static_call(kvm_x86_get_l2_tsc_multiplier)(vcpu));
2649 vcpu->arch.tsc_scaling_ratio = l1_multiplier;
2652 static_call(kvm_x86_write_tsc_multiplier)(vcpu);
2670 * offset for the vcpu and tracks the TSC matching generation that the vcpu
2673 static void __kvm_synchronize_tsc(struct kvm_vcpu *vcpu, u64 offset, u64 tsc,
2676 struct kvm *kvm = vcpu->kvm;
2686 kvm->arch.last_tsc_khz = vcpu->arch.virtual_tsc_khz;
2689 vcpu->arch.last_guest_tsc = tsc;
2691 kvm_vcpu_write_tsc_offset(vcpu, offset);
2708 } else if (vcpu->arch.this_tsc_generation != kvm->arch.cur_tsc_generation) {
2713 vcpu->arch.this_tsc_generation = kvm->arch.cur_tsc_generation;
2714 vcpu->arch.this_tsc_nsec = kvm->arch.cur_tsc_nsec;
2715 vcpu->arch.this_tsc_write = kvm->arch.cur_tsc_write;
2717 kvm_track_tsc_matching(vcpu);
2720 static void kvm_synchronize_tsc(struct kvm_vcpu *vcpu, u64 data)
2722 struct kvm *kvm = vcpu->kvm;
2729 offset = kvm_compute_l1_tsc_offset(vcpu, data);
2733 if (vcpu->arch.virtual_tsc_khz) {
2736 * detection of vcpu initialization -- need to sync
2743 nsec_to_cycles(vcpu, elapsed);
2744 u64 tsc_hz = vcpu->arch.virtual_tsc_khz * 1000LL;
2762 vcpu->arch.virtual_tsc_khz == kvm->arch.last_tsc_khz) {
2766 u64 delta = nsec_to_cycles(vcpu, elapsed);
2768 offset = kvm_compute_l1_tsc_offset(vcpu, data);
2773 __kvm_synchronize_tsc(vcpu, offset, data, ns, matched);
2777 static inline void adjust_tsc_offset_guest(struct kvm_vcpu *vcpu,
2780 u64 tsc_offset = vcpu->arch.l1_tsc_offset;
2781 kvm_vcpu_write_tsc_offset(vcpu, tsc_offset + adjustment);
2784 static inline void adjust_tsc_offset_host(struct kvm_vcpu *vcpu, s64 adjustment)
2786 if (vcpu->arch.l1_tsc_scaling_ratio != kvm_caps.default_tsc_scaling_ratio)
2789 vcpu->arch.l1_tsc_scaling_ratio);
2790 adjust_tsc_offset_guest(vcpu, adjustment);
3008 struct kvm_vcpu *vcpu;
3013 kvm_for_each_vcpu(i, vcpu, kvm)
3014 kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);
3017 kvm_for_each_vcpu(i, vcpu, kvm)
3018 kvm_clear_request(KVM_REQ_MCLOCK_INPROGRESS, vcpu);
3104 struct kvm_vcpu_arch *vcpu = &v->arch;
3127 guest_hv_clock->version = vcpu->hv_clock.version = (guest_hv_clock->version + 1) | 1;
3131 vcpu->hv_clock.flags |= (guest_hv_clock->flags & PVCLOCK_GUEST_STOPPED);
3133 if (vcpu->pvclock_set_guest_stopped_request) {
3134 vcpu->hv_clock.flags |= PVCLOCK_GUEST_STOPPED;
3135 vcpu->pvclock_set_guest_stopped_request = false;
3138 memcpy(guest_hv_clock, &vcpu->hv_clock, sizeof(*guest_hv_clock));
3141 guest_hv_clock->version = ++vcpu->hv_clock.version;
3146 trace_kvm_pvclock_update(v->vcpu_id, &vcpu->hv_clock);
3153 struct kvm_vcpu_arch *vcpu = &v->arch;
3201 if (vcpu->tsc_catchup) {
3217 if (unlikely(vcpu->hw_tsc_khz != tgt_tsc_khz)) {
3219 &vcpu->hv_clock.tsc_shift,
3220 &vcpu->hv_clock.tsc_to_system_mul);
3221 vcpu->hw_tsc_khz = tgt_tsc_khz;
3225 vcpu->hv_clock.tsc_timestamp = tsc_timestamp;
3226 vcpu->hv_clock.system_time = kernel_ns + v->kvm->arch.kvmclock_offset;
3227 vcpu->last_guest_tsc = tsc_timestamp;
3234 vcpu->hv_clock.flags = pvclock_flags;
3236 if (vcpu->pv_time.active)
3237 kvm_setup_guest_pvclock(v, &vcpu->pv_time, 0);
3238 if (vcpu->xen.vcpu_info_cache.active)
3239 kvm_setup_guest_pvclock(v, &vcpu->xen.vcpu_info_cache,
3241 if (vcpu->xen.vcpu_time_info_cache.active)
3242 kvm_setup_guest_pvclock(v, &vcpu->xen.vcpu_time_info_cache, 0);
3243 kvm_hv_setup_tsc_page(v->kvm, &vcpu->hv_clock);
3248 * kvmclock updates which are isolated to a given vcpu, such as
3249 * vcpu->cpu migration, should not allow system_timestamp from
3251 * correction applies to one vcpu's system_timestamp but not
3257 * The time for a remote vcpu to update its kvmclock is bound
3270 struct kvm_vcpu *vcpu;
3272 kvm_for_each_vcpu(i, vcpu, kvm) {
3273 kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);
3274 kvm_vcpu_kick(vcpu);
3317 static bool can_set_mci_status(struct kvm_vcpu *vcpu)
3320 if (guest_cpuid_is_amd_or_hygon(vcpu))
3321 return !!(vcpu->arch.msr_hwcr & BIT_ULL(18));
3326 static int set_msr_mce(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
3328 u64 mcg_cap = vcpu->arch.mcg_cap;
3336 vcpu->arch.mcg_status = data;
3344 vcpu->arch.mcg_ctl = data;
3358 vcpu->arch.mci_ctl2_banks[offset] = data;
3385 data != 0 && !can_set_mci_status(vcpu))
3390 vcpu->arch.mce_banks[offset] = data;
3398 static inline bool kvm_pv_async_pf_enabled(struct kvm_vcpu *vcpu)
3402 return (vcpu->arch.apf.msr_en_val & mask) == mask;
3405 static int kvm_pv_enable_async_pf(struct kvm_vcpu *vcpu, u64 data)
3413 if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF_VMEXIT) &&
3417 if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF_INT) &&
3421 if (!lapic_in_kernel(vcpu))
3424 vcpu->arch.apf.msr_en_val = data;
3426 if (!kvm_pv_async_pf_enabled(vcpu)) {
3427 kvm_clear_async_pf_completion_queue(vcpu);
3428 kvm_async_pf_hash_reset(vcpu);
3432 if (kvm_gfn_to_hva_cache_init(vcpu->kvm, &vcpu->arch.apf.data, gpa,
3436 vcpu->arch.apf.send_user_only = !(data & KVM_ASYNC_PF_SEND_ALWAYS);
3437 vcpu->arch.apf.delivery_as_pf_vmexit = data & KVM_ASYNC_PF_DELIVERY_AS_PF_VMEXIT;
3439 kvm_async_pf_wakeup_all(vcpu);
3444 static int kvm_pv_enable_async_pf_int(struct kvm_vcpu *vcpu, u64 data)
3450 if (!lapic_in_kernel(vcpu))
3453 vcpu->arch.apf.msr_int_val = data;
3455 vcpu->arch.apf.vec = data & KVM_ASYNC_PF_VEC_MASK;
3460 static void kvmclock_reset(struct kvm_vcpu *vcpu)
3462 kvm_gpc_deactivate(&vcpu->arch.pv_time);
3463 vcpu->arch.time = 0;
3466 static void kvm_vcpu_flush_tlb_all(struct kvm_vcpu *vcpu)
3468 ++vcpu->stat.tlb_flush;
3469 static_call(kvm_x86_flush_tlb_all)(vcpu);
3472 kvm_clear_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu);
3475 static void kvm_vcpu_flush_tlb_guest(struct kvm_vcpu *vcpu)
3477 ++vcpu->stat.tlb_flush;
3486 kvm_mmu_sync_roots(vcpu);
3487 kvm_mmu_sync_prev_roots(vcpu);
3490 static_call(kvm_x86_flush_tlb_guest)(vcpu);
3496 kvm_hv_vcpu_purge_flush_tlb(vcpu);
3500 static inline void kvm_vcpu_flush_tlb_current(struct kvm_vcpu *vcpu)
3502 ++vcpu->stat.tlb_flush;
3503 static_call(kvm_x86_flush_tlb_current)(vcpu);
3512 void kvm_service_local_tlb_flush_requests(struct kvm_vcpu *vcpu)
3514 if (kvm_check_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu))
3515 kvm_vcpu_flush_tlb_current(vcpu);
3517 if (kvm_check_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu))
3518 kvm_vcpu_flush_tlb_guest(vcpu);
3522 static void record_steal_time(struct kvm_vcpu *vcpu)
3524 struct gfn_to_hva_cache *ghc = &vcpu->arch.st.cache;
3527 gpa_t gpa = vcpu->arch.st.msr_val & KVM_STEAL_VALID_BITS;
3531 if (kvm_xen_msr_enabled(vcpu->kvm)) {
3532 kvm_xen_runstate_set_running(vcpu);
3536 if (!(vcpu->arch.st.msr_val & KVM_MSR_ENABLED))
3539 if (WARN_ON_ONCE(current->mm != vcpu->kvm->mm))
3542 slots = kvm_memslots(vcpu->kvm);
3550 if (kvm_gfn_to_hva_cache_init(vcpu->kvm, ghc, gpa, sizeof(*st)) ||
3560 if (guest_pv_has(vcpu, KVM_FEATURE_PV_TLB_FLUSH)) {
3579 vcpu->arch.st.preempted = 0;
3581 trace_kvm_pv_tlb_flush(vcpu->vcpu_id,
3584 kvm_vcpu_flush_tlb_guest(vcpu);
3593 vcpu->arch.st.preempted = 0;
3607 vcpu->arch.st.last_steal;
3608 vcpu->arch.st.last_steal = current->sched_info.run_delay;
3617 mark_page_dirty_in_slot(vcpu->kvm, ghc->memslot, gpa_to_gfn(ghc->gpa));
3632 int kvm_set_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
3637 if (msr && msr == vcpu->kvm->arch.xen_hvm_config.msr)
3638 return kvm_xen_write_hypercall_page(vcpu, data);
3653 vcpu->arch.microcode_version = data;
3658 vcpu->arch.arch_capabilities = data;
3671 if (vcpu->arch.perf_capabilities == data)
3674 vcpu->arch.perf_capabilities = data;
3675 kvm_pmu_refresh(vcpu);
3678 if (!msr_info->host_initiated && !guest_has_pred_cmd_msr(vcpu))
3690 !guest_cpuid_has(vcpu, X86_FEATURE_FLUSH_L1D))
3701 return set_efer(vcpu, msr_info);
3709 vcpu->arch.msr_hwcr = data;
3711 kvm_pr_unimpl_wrmsr(vcpu, msr, data);
3717 kvm_pr_unimpl_wrmsr(vcpu, msr, data);
3725 vcpu->arch.pat = data;
3729 return kvm_mtrr_set_msr(vcpu, msr, data);
3731 return kvm_set_apic_base(vcpu, msr_info);
3733 return kvm_x2apic_msr_write(vcpu, msr, data);
3735 kvm_set_lapic_tscdeadline_msr(vcpu, data);
3738 if (guest_cpuid_has(vcpu, X86_FEATURE_TSC_ADJUST)) {
3740 s64 adj = data - vcpu->arch.ia32_tsc_adjust_msr;
3741 adjust_tsc_offset_guest(vcpu, adj);
3745 kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);
3747 vcpu->arch.ia32_tsc_adjust_msr = data;
3751 u64 old_val = vcpu->arch.ia32_misc_enable_msr;
3763 if (!kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_MISC_ENABLE_NO_MWAIT) &&
3765 if (!guest_cpuid_has(vcpu, X86_FEATURE_XMM3))
3767 vcpu->arch.ia32_misc_enable_msr = data;
3768 kvm_update_cpuid_runtime(vcpu);
3770 vcpu->arch.ia32_misc_enable_msr = data;
3777 vcpu->arch.smbase = data;
3780 vcpu->arch.msr_ia32_power_ctl = data;
3784 kvm_synchronize_tsc(vcpu, data);
3786 u64 adj = kvm_compute_l1_tsc_offset(vcpu, data) - vcpu->arch.l1_tsc_offset;
3787 adjust_tsc_offset_guest(vcpu, adj);
3788 vcpu->arch.ia32_tsc_adjust_msr += adj;
3793 !guest_cpuid_has(vcpu, X86_FEATURE_XSAVES))
3802 vcpu->arch.ia32_xss = data;
3803 kvm_update_cpuid_runtime(vcpu);
3808 vcpu->arch.smi_count = data;
3811 if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE2))
3814 vcpu->kvm->arch.wall_clock = data;
3815 kvm_write_wall_clock(vcpu->kvm, data, 0);
3818 if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE))
3821 vcpu->kvm->arch.wall_clock = data;
3822 kvm_write_wall_clock(vcpu->kvm, data, 0);
3825 if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE2))
3828 kvm_write_system_time(vcpu, data, false, msr_info->host_initiated);
3831 if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE))
3834 kvm_write_system_time(vcpu, data, true, msr_info->host_initiated);
3837 if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF))
3840 if (kvm_pv_enable_async_pf(vcpu, data))
3844 if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF_INT))
3847 if (kvm_pv_enable_async_pf_int(vcpu, data))
3851 if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF_INT))
3854 vcpu->arch.apf.pageready_pending = false;
3855 kvm_check_async_pf_completion(vcpu);
3859 if (!guest_pv_has(vcpu, KVM_FEATURE_STEAL_TIME))
3868 vcpu->arch.st.msr_val = data;
3873 kvm_make_request(KVM_REQ_STEAL_UPDATE, vcpu);
3877 if (!guest_pv_has(vcpu, KVM_FEATURE_PV_EOI))
3880 if (kvm_lapic_set_pv_eoi(vcpu, data, sizeof(u8)))
3885 if (!guest_pv_has(vcpu, KVM_FEATURE_POLL_CONTROL))
3892 vcpu->arch.msr_kvm_poll_control = data;
3899 return set_msr_mce(vcpu, msr_info);
3905 if (kvm_pmu_is_valid_msr(vcpu, msr))
3906 return kvm_pmu_set_msr(vcpu, msr_info);
3909 kvm_pr_unimpl_wrmsr(vcpu, msr, data);
3931 return kvm_hv_set_msr_common(vcpu, msr, data,
3937 kvm_pr_unimpl_wrmsr(vcpu, msr, data);
3940 if (!guest_cpuid_has(vcpu, X86_FEATURE_OSVW))
3942 vcpu->arch.osvw.length = data;
3945 if (!guest_cpuid_has(vcpu, X86_FEATURE_OSVW))
3947 vcpu->arch.osvw.status = data;
3952 cpuid_fault_enabled(vcpu)))
3954 vcpu->arch.msr_platform_info = data;
3959 !supports_cpuid_fault(vcpu)))
3961 vcpu->arch.msr_misc_features_enables = data;
3966 !guest_cpuid_has(vcpu, X86_FEATURE_XFD))
3969 if (data & ~kvm_guest_supported_xfd(vcpu))
3972 fpu_update_guest_xfd(&vcpu->arch.guest_fpu, data);
3976 !guest_cpuid_has(vcpu, X86_FEATURE_XFD))
3979 if (data & ~kvm_guest_supported_xfd(vcpu))
3982 vcpu->arch.guest_fpu.xfd_err = data;
3986 if (kvm_pmu_is_valid_msr(vcpu, msr))
3987 return kvm_pmu_set_msr(vcpu, msr_info);
4003 static int get_msr_mce(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata, bool host)
4006 u64 mcg_cap = vcpu->arch.mcg_cap;
4016 data = vcpu->arch.mcg_cap;
4021 data = vcpu->arch.mcg_ctl;
4024 data = vcpu->arch.mcg_status;
4035 data = vcpu->arch.mci_ctl2_banks[offset];
4044 data = vcpu->arch.mce_banks[offset];
4053 int kvm_get_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
4091 if (kvm_pmu_is_valid_msr(vcpu, msr_info->index))
4092 return kvm_pmu_get_msr(vcpu, msr_info);
4096 msr_info->data = vcpu->arch.microcode_version;
4100 !guest_cpuid_has(vcpu, X86_FEATURE_ARCH_CAPABILITIES))
4102 msr_info->data = vcpu->arch.arch_capabilities;
4106 !guest_cpuid_has(vcpu, X86_FEATURE_PDCM))
4108 msr_info->data = vcpu->arch.perf_capabilities;
4111 msr_info->data = vcpu->arch.msr_ia32_power_ctl;
4126 offset = vcpu->arch.l1_tsc_offset;
4127 ratio = vcpu->arch.l1_tsc_scaling_ratio;
4129 offset = vcpu->arch.tsc_offset;
4130 ratio = vcpu->arch.tsc_scaling_ratio;
4137 msr_info->data = vcpu->arch.pat;
4142 return kvm_mtrr_get_msr(vcpu, msr_info->index, &msr_info->data);
4161 msr_info->data = kvm_get_apic_base(vcpu);
4164 return kvm_x2apic_msr_read(vcpu, msr_info->index, &msr_info->data);
4166 msr_info->data = kvm_get_lapic_tscdeadline_msr(vcpu);
4169 msr_info->data = (u64)vcpu->arch.ia32_tsc_adjust_msr;
4172 msr_info->data = vcpu->arch.ia32_misc_enable_msr;
4177 msr_info->data = vcpu->arch.smbase;
4180 msr_info->data = vcpu->arch.smi_count;
4189 msr_info->data = vcpu->arch.efer;
4192 if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE))
4195 msr_info->data = vcpu->kvm->arch.wall_clock;
4198 if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE2))
4201 msr_info->data = vcpu->kvm->arch.wall_clock;
4204 if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE))
4207 msr_info->data = vcpu->arch.time;
4210 if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE2))
4213 msr_info->data = vcpu->arch.time;
4216 if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF))
4219 msr_info->data = vcpu->arch.apf.msr_en_val;
4222 if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF_INT))
4225 msr_info->data = vcpu->arch.apf.msr_int_val;
4228 if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF_INT))
4234 if (!guest_pv_has(vcpu, KVM_FEATURE_STEAL_TIME))
4237 msr_info->data = vcpu->arch.st.msr_val;
4240 if (!guest_pv_has(vcpu, KVM_FEATURE_PV_EOI))
4243 msr_info->data = vcpu->arch.pv_eoi.msr_val;
4246 if (!guest_pv_has(vcpu, KVM_FEATURE_POLL_CONTROL))
4249 msr_info->data = vcpu->arch.msr_kvm_poll_control;
4258 return get_msr_mce(vcpu, msr_info->index, &msr_info->data,
4262 !guest_cpuid_has(vcpu, X86_FEATURE_XSAVES))
4264 msr_info->data = vcpu->arch.ia32_xss;
4288 return kvm_hv_get_msr_common(vcpu,
4305 if (!guest_cpuid_has(vcpu, X86_FEATURE_OSVW))
4307 msr_info->data = vcpu->arch.osvw.length;
4310 if (!guest_cpuid_has(vcpu, X86_FEATURE_OSVW))
4312 msr_info->data = vcpu->arch.osvw.status;
4316 !vcpu->kvm->arch.guest_can_read_msr_platform_info)
4318 msr_info->data = vcpu->arch.msr_platform_info;
4321 msr_info->data = vcpu->arch.msr_misc_features_enables;
4324 msr_info->data = vcpu->arch.msr_hwcr;
4329 !guest_cpuid_has(vcpu, X86_FEATURE_XFD))
4332 msr_info->data = vcpu->arch.guest_fpu.fpstate->xfd;
4336 !guest_cpuid_has(vcpu, X86_FEATURE_XFD))
4339 msr_info->data = vcpu->arch.guest_fpu.xfd_err;
4343 if (kvm_pmu_is_valid_msr(vcpu, msr_info->index))
4344 return kvm_pmu_get_msr(vcpu, msr_info);
4367 static int __msr_io(struct kvm_vcpu *vcpu, struct kvm_msrs *msrs,
4369 int (*do_msr)(struct kvm_vcpu *vcpu,
4375 if (do_msr(vcpu, entries[i].index, &entries[i].data))
4386 static int msr_io(struct kvm_vcpu *vcpu, struct kvm_msrs __user *user_msrs,
4387 int (*do_msr)(struct kvm_vcpu *vcpu,
4411 r = __msr_io(vcpu, &msrs, entries, do_msr);
4428 static int kvm_ioctl_get_supported_hv_cpuid(struct kvm_vcpu *vcpu,
4438 r = kvm_get_hv_cpuid(vcpu, &cpuid, cpuid_arg->entries);
4804 static bool need_emulate_wbinvd(struct kvm_vcpu *vcpu)
4806 return kvm_arch_has_noncoherent_dma(vcpu->kvm);
4809 void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
4812 if (need_emulate_wbinvd(vcpu)) {
4814 cpumask_set_cpu(cpu, vcpu->arch.wbinvd_dirty_mask);
4815 else if (vcpu->cpu != -1 && vcpu->cpu != cpu)
4816 smp_call_function_single(vcpu->cpu,
4820 static_call(kvm_x86_vcpu_load)(vcpu, cpu);
4823 vcpu->arch.host_pkru = read_pkru();
4826 if (unlikely(vcpu->arch.tsc_offset_adjustment)) {
4827 adjust_tsc_offset_host(vcpu, vcpu->arch.tsc_offset_adjustment);
4828 vcpu->arch.tsc_offset_adjustment = 0;
4829 kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);
4832 if (unlikely(vcpu->cpu != cpu) || kvm_check_tsc_unstable()) {
4833 s64 tsc_delta = !vcpu->arch.last_host_tsc ? 0 :
4834 rdtsc() - vcpu->arch.last_host_tsc;
4839 u64 offset = kvm_compute_l1_tsc_offset(vcpu,
4840 vcpu->arch.last_guest_tsc);
4841 kvm_vcpu_write_tsc_offset(vcpu, offset);
4842 vcpu->arch.tsc_catchup = 1;
4845 if (kvm_lapic_hv_timer_in_use(vcpu))
4846 kvm_lapic_restart_hv_timer(vcpu);
4850 * kvmclock on vcpu->cpu migration
4852 if (!vcpu->kvm->arch.use_master_clock || vcpu->cpu == -1)
4853 kvm_make_request(KVM_REQ_GLOBAL_CLOCK_UPDATE, vcpu);
4854 if (vcpu->cpu != cpu)
4855 kvm_make_request(KVM_REQ_MIGRATE_TIMER, vcpu);
4856 vcpu->cpu = cpu;
4859 kvm_make_request(KVM_REQ_STEAL_UPDATE, vcpu);
4862 static void kvm_steal_time_set_preempted(struct kvm_vcpu *vcpu)
4864 struct gfn_to_hva_cache *ghc = &vcpu->arch.st.cache;
4868 gpa_t gpa = vcpu->arch.st.msr_val & KVM_STEAL_VALID_BITS;
4877 if (!vcpu->arch.at_instruction_boundary) {
4878 vcpu->stat.preemption_other++;
4882 vcpu->stat.preemption_reported++;
4883 if (!(vcpu->arch.st.msr_val & KVM_MSR_ENABLED))
4886 if (vcpu->arch.st.preempted)
4890 if (unlikely(current->mm != vcpu->kvm->mm))
4893 slots = kvm_memslots(vcpu->kvm);
4904 vcpu->arch.st.preempted = KVM_VCPU_PREEMPTED;
4906 mark_page_dirty_in_slot(vcpu->kvm, ghc->memslot, gpa_to_gfn(ghc->gpa));
4909 void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
4913 if (vcpu->preempted) {
4914 if (!vcpu->arch.guest_state_protected)
4915 vcpu->arch.preempted_in_kernel = !static_call(kvm_x86_get_cpl)(vcpu);
4921 idx = srcu_read_lock(&vcpu->kvm->srcu);
4922 if (kvm_xen_msr_enabled(vcpu->kvm))
4923 kvm_xen_runstate_set_preempted(vcpu);
4925 kvm_steal_time_set_preempted(vcpu);
4926 srcu_read_unlock(&vcpu->kvm->srcu, idx);
4929 static_call(kvm_x86_vcpu_put)(vcpu);
4930 vcpu->arch.last_host_tsc = rdtsc();
4933 static int kvm_vcpu_ioctl_get_lapic(struct kvm_vcpu *vcpu,
4936 static_call_cond(kvm_x86_sync_pir_to_irr)(vcpu);
4938 return kvm_apic_get_state(vcpu, s);
4941 static int kvm_vcpu_ioctl_set_lapic(struct kvm_vcpu *vcpu,
4946 r = kvm_apic_set_state(vcpu, s);
4949 update_cr8_intercept(vcpu);
4954 static int kvm_cpu_accept_dm_intr(struct kvm_vcpu *vcpu)
4962 if (kvm_cpu_has_extint(vcpu))
4966 return (!lapic_in_kernel(vcpu) ||
4967 kvm_apic_accept_pic_intr(vcpu));
4970 static int kvm_vcpu_ready_for_interrupt_injection(struct kvm_vcpu *vcpu)
4979 return (kvm_arch_interrupt_allowed(vcpu) &&
4980 kvm_cpu_accept_dm_intr(vcpu) &&
4981 !kvm_event_needs_reinjection(vcpu) &&
4982 !kvm_is_exception_pending(vcpu));
4985 static int kvm_vcpu_ioctl_interrupt(struct kvm_vcpu *vcpu,
4991 if (!irqchip_in_kernel(vcpu->kvm)) {
4992 kvm_queue_interrupt(vcpu, irq->irq, false);
4993 kvm_make_request(KVM_REQ_EVENT, vcpu);
5001 if (pic_in_kernel(vcpu->kvm))
5004 if (vcpu->arch.pending_external_vector != -1)
5007 vcpu->arch.pending_external_vector = irq->irq;
5008 kvm_make_request(KVM_REQ_EVENT, vcpu);
5012 static int kvm_vcpu_ioctl_nmi(struct kvm_vcpu *vcpu)
5014 kvm_inject_nmi(vcpu);
5019 static int vcpu_ioctl_tpr_access_reporting(struct kvm_vcpu *vcpu,
5024 vcpu->arch.tpr_access_reporting = !!tac->enabled;
5028 static int kvm_vcpu_ioctl_x86_setup_mce(struct kvm_vcpu *vcpu,
5040 vcpu->arch.mcg_cap = mcg_cap;
5043 vcpu->arch.mcg_ctl = ~(u64)0;
5046 vcpu->arch.mce_banks[bank*4] = ~(u64)0;
5048 vcpu->arch.mci_ctl2_banks[bank] = 0;
5051 kvm_apic_after_set_mcg_cap(vcpu);
5053 static_call(kvm_x86_setup_mce)(vcpu);
5075 static int kvm_vcpu_x86_set_ucna(struct kvm_vcpu *vcpu, struct kvm_x86_mce *mce, u64* banks)
5077 u64 mcg_cap = vcpu->arch.mcg_cap;
5082 vcpu->arch.mcg_status = mce->mcg_status;
5085 !(vcpu->arch.mci_ctl2_banks[mce->bank] & MCI_CTL2_CMCI_EN))
5088 if (lapic_in_kernel(vcpu))
5089 kvm_apic_local_deliver(vcpu->arch.apic, APIC_LVTCMCI);
5094 static int kvm_vcpu_ioctl_x86_set_mce(struct kvm_vcpu *vcpu,
5097 u64 mcg_cap = vcpu->arch.mcg_cap;
5099 u64 *banks = vcpu->arch.mce_banks;
5107 return kvm_vcpu_x86_set_ucna(vcpu, mce, banks);
5114 vcpu->arch.mcg_ctl != ~(u64)0)
5123 if ((vcpu->arch.mcg_status & MCG_STATUS_MCIP) ||
5124 !kvm_is_cr4_bit_set(vcpu, X86_CR4_MCE)) {
5125 kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);
5132 vcpu->arch.mcg_status = mce->mcg_status;
5134 kvm_queue_exception(vcpu, MC_VECTOR);
5147 static void kvm_vcpu_ioctl_x86_get_vcpu_events(struct kvm_vcpu *vcpu,
5152 process_nmi(vcpu);
5155 if (kvm_check_request(KVM_REQ_SMI, vcpu))
5156 process_smi(vcpu);
5166 if (vcpu->arch.exception_vmexit.pending &&
5167 !vcpu->arch.exception.pending &&
5168 !vcpu->arch.exception.injected)
5169 ex = &vcpu->arch.exception_vmexit;
5171 ex = &vcpu->arch.exception;
5181 if (!vcpu->kvm->arch.exception_payload_enabled &&
5183 kvm_deliver_exception_payload(vcpu, ex);
5201 if (!vcpu->kvm->arch.exception_payload_enabled)
5211 vcpu->arch.interrupt.injected && !vcpu->arch.interrupt.soft;
5212 events->interrupt.nr = vcpu->arch.interrupt.nr;
5213 events->interrupt.shadow = static_call(kvm_x86_get_interrupt_shadow)(vcpu);
5215 events->nmi.injected = vcpu->arch.nmi_injected;
5216 events->nmi.pending = kvm_get_nr_pending_nmis(vcpu);
5217 events->nmi.masked = static_call(kvm_x86_get_nmi_mask)(vcpu);
5222 events->smi.smm = is_smm(vcpu);
5223 events->smi.pending = vcpu->arch.smi_pending;
5225 !!(vcpu->arch.hflags & HF_SMM_INSIDE_NMI_MASK);
5227 events->smi.latched_init = kvm_lapic_latched_init(vcpu);
5232 if (vcpu->kvm->arch.exception_payload_enabled)
5234 if (vcpu->kvm->arch.triple_fault_event) {
5235 events->triple_fault.pending = kvm_test_request(KVM_REQ_TRIPLE_FAULT, vcpu);
5240 static int kvm_vcpu_ioctl_x86_set_vcpu_events(struct kvm_vcpu *vcpu,
5252 if (!vcpu->kvm->arch.exception_payload_enabled)
5270 vcpu->arch.mp_state == KVM_MP_STATE_INIT_RECEIVED)
5273 process_nmi(vcpu);
5283 vcpu->arch.exception_from_userspace = events->exception.pending;
5285 vcpu->arch.exception_vmexit.pending = false;
5287 vcpu->arch.exception.injected = events->exception.injected;
5288 vcpu->arch.exception.pending = events->exception.pending;
5289 vcpu->arch.exception.vector = events->exception.nr;
5290 vcpu->arch.exception.has_error_code = events->exception.has_error_code;
5291 vcpu->arch.exception.error_code = events->exception.error_code;
5292 vcpu->arch.exception.has_payload = events->exception_has_payload;
5293 vcpu->arch.exception.payload = events->exception_payload;
5295 vcpu->arch.interrupt.injected = events->interrupt.injected;
5296 vcpu->arch.interrupt.nr = events->interrupt.nr;
5297 vcpu->arch.interrupt.soft = events->interrupt.soft;
5299 static_call(kvm_x86_set_interrupt_shadow)(vcpu,
5302 vcpu->arch.nmi_injected = events->nmi.injected;
5304 vcpu->arch.nmi_pending = 0;
5305 atomic_set(&vcpu->arch.nmi_queued, events->nmi.pending);
5307 kvm_make_request(KVM_REQ_NMI, vcpu);
5309 static_call(kvm_x86_set_nmi_mask)(vcpu, events->nmi.masked);
5312 lapic_in_kernel(vcpu))
5313 vcpu->arch.apic->sipi_vector = events->sipi_vector;
5317 if (!!(vcpu->arch.hflags & HF_SMM_MASK) != events->smi.smm) {
5318 kvm_leave_nested(vcpu);
5319 kvm_smm_changed(vcpu, events->smi.smm);
5322 vcpu->arch.smi_pending = events->smi.pending;
5326 vcpu->arch.hflags |= HF_SMM_INSIDE_NMI_MASK;
5328 vcpu->arch.hflags &= ~HF_SMM_INSIDE_NMI_MASK;
5337 if (lapic_in_kernel(vcpu)) {
5339 set_bit(KVM_APIC_INIT, &vcpu->arch.apic->pending_events);
5341 clear_bit(KVM_APIC_INIT, &vcpu->arch.apic->pending_events);
5346 if (!vcpu->kvm->arch.triple_fault_event)
5349 kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);
5351 kvm_clear_request(KVM_REQ_TRIPLE_FAULT, vcpu);
5354 kvm_make_request(KVM_REQ_EVENT, vcpu);
5359 static void kvm_vcpu_ioctl_x86_get_debugregs(struct kvm_vcpu *vcpu,
5365 memcpy(dbgregs->db, vcpu->arch.db, sizeof(vcpu->arch.db));
5366 kvm_get_dr(vcpu, 6, &val);
5368 dbgregs->dr7 = vcpu->arch.dr7;
5371 static int kvm_vcpu_ioctl_x86_set_debugregs(struct kvm_vcpu *vcpu,
5382 memcpy(vcpu->arch.db, dbgregs->db, sizeof(vcpu->arch.db));
5383 kvm_update_dr0123(vcpu);
5384 vcpu->arch.dr6 = dbgregs->dr6;
5385 vcpu->arch.dr7 = dbgregs->dr7;
5386 kvm_update_dr7(vcpu);
5392 static void kvm_vcpu_ioctl_x86_get_xsave2(struct kvm_vcpu *vcpu,
5407 u64 supported_xcr0 = vcpu->arch.guest_supported_xcr0 |
5410 if (fpstate_is_confidential(&vcpu->arch.guest_fpu))
5413 fpu_copy_guest_fpstate_to_uabi(&vcpu->arch.guest_fpu, state, size,
5414 supported_xcr0, vcpu->arch.pkru);
5417 static void kvm_vcpu_ioctl_x86_get_xsave(struct kvm_vcpu *vcpu,
5420 return kvm_vcpu_ioctl_x86_get_xsave2(vcpu, (void *)guest_xsave->region,
5424 static int kvm_vcpu_ioctl_x86_set_xsave(struct kvm_vcpu *vcpu,
5427 if (fpstate_is_confidential(&vcpu->arch.guest_fpu))
5430 return fpu_copy_uabi_to_guest_fpstate(&vcpu->arch.guest_fpu,
5433 &vcpu->arch.pkru);
5436 static void kvm_vcpu_ioctl_x86_get_xcrs(struct kvm_vcpu *vcpu,
5447 guest_xcrs->xcrs[0].value = vcpu->arch.xcr0;
5450 static int kvm_vcpu_ioctl_x86_set_xcrs(struct kvm_vcpu *vcpu,
5464 r = __kvm_set_xcr(vcpu, XCR_XFEATURE_ENABLED_MASK,
5479 static int kvm_set_guest_paused(struct kvm_vcpu *vcpu)
5481 if (!vcpu->arch.pv_time.active)
5483 vcpu->arch.pvclock_set_guest_stopped_request = true;
5484 kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);
5488 static int kvm_arch_tsc_has_attr(struct kvm_vcpu *vcpu,
5504 static int kvm_arch_tsc_get_attr(struct kvm_vcpu *vcpu,
5516 if (put_user(vcpu->arch.l1_tsc_offset, uaddr))
5527 static int kvm_arch_tsc_set_attr(struct kvm_vcpu *vcpu,
5531 struct kvm *kvm = vcpu->kvm;
5549 matched = (vcpu->arch.virtual_tsc_khz &&
5550 kvm->arch.last_tsc_khz == vcpu->arch.virtual_tsc_khz &&
5553 tsc = kvm_scale_tsc(rdtsc(), vcpu->arch.l1_tsc_scaling_ratio) + offset;
5556 __kvm_synchronize_tsc(vcpu, offset, tsc, ns, matched);
5569 static int kvm_vcpu_ioctl_device_attr(struct kvm_vcpu *vcpu,
5584 r = kvm_arch_tsc_has_attr(vcpu, &attr);
5587 r = kvm_arch_tsc_get_attr(vcpu, &attr);
5590 r = kvm_arch_tsc_set_attr(vcpu, &attr);
5597 static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu,
5614 if (!irqchip_in_kernel(vcpu->kvm))
5616 return kvm_hv_activate_synic(vcpu, cap->cap ==
5621 r = kvm_x86_ops.nested_ops->enable_evmcs(vcpu, &vmcs_version);
5633 return static_call(kvm_x86_enable_l2_tlb_flush)(vcpu);
5636 return kvm_hv_set_enforce_cpuid(vcpu, cap->args[0]);
5639 vcpu->arch.pv_cpuid.enforce = cap->args[0];
5640 if (vcpu->arch.pv_cpuid.enforce)
5641 kvm_update_pv_runtime(vcpu);
5652 struct kvm_vcpu *vcpu = filp->private_data;
5663 vcpu_load(vcpu);
5669 if (!lapic_in_kernel(vcpu))
5677 r = kvm_vcpu_ioctl_get_lapic(vcpu, u.lapic);
5688 if (!lapic_in_kernel(vcpu))
5696 r = kvm_vcpu_ioctl_set_lapic(vcpu, u.lapic);
5705 r = kvm_vcpu_ioctl_interrupt(vcpu, &irq);
5709 r = kvm_vcpu_ioctl_nmi(vcpu);
5713 r = kvm_inject_smi(vcpu);
5723 r = kvm_vcpu_ioctl_set_cpuid(vcpu, &cpuid, cpuid_arg->entries);
5733 r = kvm_vcpu_ioctl_set_cpuid2(vcpu, &cpuid,
5744 r = kvm_vcpu_ioctl_get_cpuid2(vcpu, &cpuid,
5755 int idx = srcu_read_lock(&vcpu->kvm->srcu);
5756 r = msr_io(vcpu, argp, do_get_msr, 1);
5757 srcu_read_unlock(&vcpu->kvm->srcu, idx);
5761 int idx = srcu_read_lock(&vcpu->kvm->srcu);
5762 r = msr_io(vcpu, argp, do_set_msr, 0);
5763 srcu_read_unlock(&vcpu->kvm->srcu, idx);
5772 r = vcpu_ioctl_tpr_access_reporting(vcpu, &tac);
5786 if (!lapic_in_kernel(vcpu))
5791 idx = srcu_read_lock(&vcpu->kvm->srcu);
5792 r = kvm_lapic_set_vapic_addr(vcpu, va.vapic_addr);
5793 srcu_read_unlock(&vcpu->kvm->srcu, idx);
5802 r = kvm_vcpu_ioctl_x86_setup_mce(vcpu, mcg_cap);
5811 r = kvm_vcpu_ioctl_x86_set_mce(vcpu, &mce);
5817 kvm_vcpu_ioctl_x86_get_vcpu_events(vcpu, &events);
5832 r = kvm_vcpu_ioctl_x86_set_vcpu_events(vcpu, &events);
5838 kvm_vcpu_ioctl_x86_get_debugregs(vcpu, &dbgregs);
5855 r = kvm_vcpu_ioctl_x86_set_debugregs(vcpu, &dbgregs);
5860 if (vcpu->arch.guest_fpu.uabi_size > sizeof(struct kvm_xsave))
5868 kvm_vcpu_ioctl_x86_get_xsave(vcpu, u.xsave);
5877 int size = vcpu->arch.guest_fpu.uabi_size;
5885 r = kvm_vcpu_ioctl_x86_set_xsave(vcpu, u.xsave);
5890 int size = vcpu->arch.guest_fpu.uabi_size;
5897 kvm_vcpu_ioctl_x86_get_xsave2(vcpu, u.buffer, size);
5913 kvm_vcpu_ioctl_x86_get_xcrs(vcpu, u.xcrs);
5929 r = kvm_vcpu_ioctl_x86_set_xcrs(vcpu, u.xcrs);
5945 if (!kvm_set_tsc_khz(vcpu, user_tsc_khz))
5951 r = vcpu->arch.virtual_tsc_khz;
5955 r = kvm_set_guest_paused(vcpu);
5964 r = kvm_vcpu_ioctl_enable_cap(vcpu, &cap);
5980 r = kvm_x86_ops.nested_ops->get_state(vcpu, user_kvm_nested_state,
6024 idx = srcu_read_lock(&vcpu->kvm->srcu);
6025 r = kvm_x86_ops.nested_ops->set_state(vcpu, user_kvm_nested_state, &kvm_state);
6026 srcu_read_unlock(&vcpu->kvm->srcu, idx);
6030 r = kvm_ioctl_get_supported_hv_cpuid(vcpu, argp);
6039 r = kvm_xen_vcpu_get_attr(vcpu, &xva);
6050 r = kvm_xen_vcpu_set_attr(vcpu, &xva);
6059 __get_sregs2(vcpu, u.sregs2);
6073 r = __set_sregs2(vcpu, u.sregs2);
6079 r = kvm_vcpu_ioctl_device_attr(vcpu, ioctl, argp);
6087 vcpu_put(vcpu);
6091 vm_fault_t kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf)
6265 struct kvm_vcpu *vcpu;
6268 kvm_for_each_vcpu(i, vcpu, kvm)
6269 kvm_vcpu_kick(vcpu);
6683 struct kvm_vcpu *vcpu;
6688 kvm_for_each_vcpu(i, vcpu, kvm) {
6689 if (!vcpu->arch.pv_time.active)
6692 ret = kvm_set_guest_paused(vcpu);
6695 vcpu->vcpu_id, ret);
7245 static int vcpu_mmio_write(struct kvm_vcpu *vcpu, gpa_t addr, int len,
7253 if (!(lapic_in_kernel(vcpu) &&
7254 !kvm_iodevice_write(vcpu, &vcpu->arch.apic->dev, addr, n, v))
7255 && kvm_io_bus_write(vcpu, KVM_MMIO_BUS, addr, n, v))
7266 static int vcpu_mmio_read(struct kvm_vcpu *vcpu, gpa_t addr, int len, void *v)
7273 if (!(lapic_in_kernel(vcpu) &&
7274 !kvm_iodevice_read(vcpu, &vcpu->arch.apic->dev,
7276 && kvm_io_bus_read(vcpu, KVM_MMIO_BUS, addr, n, v))
7288 void kvm_set_segment(struct kvm_vcpu *vcpu,
7291 static_call(kvm_x86_set_segment)(vcpu, var, seg);
7294 void kvm_get_segment(struct kvm_vcpu *vcpu,
7297 static_call(kvm_x86_get_segment)(vcpu, var, seg);
7300 gpa_t translate_nested_gpa(struct kvm_vcpu *vcpu, gpa_t gpa, u64 access,
7303 struct kvm_mmu *mmu = vcpu->arch.mmu;
7306 BUG_ON(!mmu_is_nested(vcpu));
7310 t_gpa = mmu->gva_to_gpa(vcpu, mmu, gpa, access, exception);
7315 gpa_t kvm_mmu_gva_to_gpa_read(struct kvm_vcpu *vcpu, gva_t gva,
7318 struct kvm_mmu *mmu = vcpu->arch.walk_mmu;
7320 u64 access = (static_call(kvm_x86_get_cpl)(vcpu) == 3) ? PFERR_USER_MASK : 0;
7321 return mmu->gva_to_gpa(vcpu, mmu, gva, access, exception);
7325 gpa_t kvm_mmu_gva_to_gpa_write(struct kvm_vcpu *vcpu, gva_t gva,
7328 struct kvm_mmu *mmu = vcpu->arch.walk_mmu;
7330 u64 access = (static_call(kvm_x86_get_cpl)(vcpu) == 3) ? PFERR_USER_MASK : 0;
7332 return mmu->gva_to_gpa(vcpu, mmu, gva, access, exception);
7337 gpa_t kvm_mmu_gva_to_gpa_system(struct kvm_vcpu *vcpu, gva_t gva,
7340 struct kvm_mmu *mmu = vcpu->arch.walk_mmu;
7342 return mmu->gva_to_gpa(vcpu, mmu, gva, 0, exception);
7346 struct kvm_vcpu *vcpu, u64 access,
7349 struct kvm_mmu *mmu = vcpu->arch.walk_mmu;
7354 gpa_t gpa = mmu->gva_to_gpa(vcpu, mmu, addr, access, exception);
7361 ret = kvm_vcpu_read_guest_page(vcpu, gpa >> PAGE_SHIFT, data,
7381 struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
7382 struct kvm_mmu *mmu = vcpu->arch.walk_mmu;
7383 u64 access = (static_call(kvm_x86_get_cpl)(vcpu) == 3) ? PFERR_USER_MASK : 0;
7388 gpa_t gpa = mmu->gva_to_gpa(vcpu, mmu, addr, access|PFERR_FETCH_MASK,
7396 ret = kvm_vcpu_read_guest_page(vcpu, gpa >> PAGE_SHIFT, val,
7404 int kvm_read_guest_virt(struct kvm_vcpu *vcpu,
7408 u64 access = (static_call(kvm_x86_get_cpl)(vcpu) == 3) ? PFERR_USER_MASK : 0;
7417 return kvm_read_guest_virt_helper(addr, val, bytes, vcpu, access,
7426 struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
7431 else if (static_call(kvm_x86_get_cpl)(vcpu) == 3)
7434 return kvm_read_guest_virt_helper(addr, val, bytes, vcpu, access, exception);
7438 struct kvm_vcpu *vcpu, u64 access,
7441 struct kvm_mmu *mmu = vcpu->arch.walk_mmu;
7446 gpa_t gpa = mmu->gva_to_gpa(vcpu, mmu, addr, access, exception);
7453 ret = kvm_vcpu_write_guest(vcpu, gpa, data, towrite);
7471 struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
7476 else if (static_call(kvm_x86_get_cpl)(vcpu) == 3)
7479 return kvm_write_guest_virt_helper(addr, val, bytes, vcpu,
7483 int kvm_write_guest_virt_system(struct kvm_vcpu *vcpu, gva_t addr, void *val,
7487 vcpu->arch.l1tf_flush_l1d = true;
7489 return kvm_write_guest_virt_helper(addr, val, bytes, vcpu,
7494 static int kvm_can_emulate_insn(struct kvm_vcpu *vcpu, int emul_type,
7497 return static_call(kvm_x86_can_emulate_instruction)(vcpu, emul_type,
7501 int handle_ud(struct kvm_vcpu *vcpu)
7509 if (unlikely(!kvm_can_emulate_insn(vcpu, emul_type, NULL, 0)))
7513 kvm_read_guest_virt(vcpu, kvm_get_linear_rip(vcpu),
7517 kvm_set_rflags(vcpu, kvm_get_rflags(vcpu) & ~X86_EFLAGS_RF);
7518 kvm_rip_write(vcpu, kvm_rip_read(vcpu) + sizeof(sig));
7522 return kvm_emulate_instruction(vcpu, emul_type);
7526 static int vcpu_is_mmio_gpa(struct kvm_vcpu *vcpu, unsigned long gva,
7533 if (vcpu_match_mmio_gpa(vcpu, gpa)) {
7541 static int vcpu_mmio_gva_to_gpa(struct kvm_vcpu *vcpu, unsigned long gva,
7545 struct kvm_mmu *mmu = vcpu->arch.walk_mmu;
7546 u64 access = ((static_call(kvm_x86_get_cpl)(vcpu) == 3) ? PFERR_USER_MASK : 0)
7554 if (vcpu_match_mmio_gva(vcpu, gva) && (!is_paging(vcpu) ||
7555 !permission_fault(vcpu, vcpu->arch.walk_mmu,
7556 vcpu->arch.mmio_access, 0, access))) {
7557 *gpa = vcpu->arch.mmio_gfn << PAGE_SHIFT |
7563 *gpa = mmu->gva_to_gpa(vcpu, mmu, gva, access, exception);
7568 return vcpu_is_mmio_gpa(vcpu, gva, *gpa, write);
7571 int emulator_write_phys(struct kvm_vcpu *vcpu, gpa_t gpa,
7576 ret = kvm_vcpu_write_guest(vcpu, gpa, val, bytes);
7579 kvm_page_track_write(vcpu, gpa, val, bytes);
7584 int (*read_write_prepare)(struct kvm_vcpu *vcpu, void *val,
7586 int (*read_write_emulate)(struct kvm_vcpu *vcpu, gpa_t gpa,
7588 int (*read_write_mmio)(struct kvm_vcpu *vcpu, gpa_t gpa,
7590 int (*read_write_exit_mmio)(struct kvm_vcpu *vcpu, gpa_t gpa,
7595 static int read_prepare(struct kvm_vcpu *vcpu, void *val, int bytes)
7597 if (vcpu->mmio_read_completed) {
7599 vcpu->mmio_fragments[0].gpa, val);
7600 vcpu->mmio_read_completed = 0;
7607 static int read_emulate(struct kvm_vcpu *vcpu, gpa_t gpa,
7610 return !kvm_vcpu_read_guest(vcpu, gpa, val, bytes);
7613 static int write_emulate(struct kvm_vcpu *vcpu, gpa_t gpa,
7616 return emulator_write_phys(vcpu, gpa, val, bytes);
7619 static int write_mmio(struct kvm_vcpu *vcpu, gpa_t gpa, int bytes, void *val)
7622 return vcpu_mmio_write(vcpu, gpa, bytes, val);
7625 static int read_exit_mmio(struct kvm_vcpu *vcpu, gpa_t gpa,
7632 static int write_exit_mmio(struct kvm_vcpu *vcpu, gpa_t gpa,
7635 struct kvm_mmio_fragment *frag = &vcpu->mmio_fragments[0];
7637 memcpy(vcpu->run->mmio.data, frag->data, min(8u, frag->len));
7658 struct kvm_vcpu *vcpu,
7665 struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt;
7677 ret = vcpu_is_mmio_gpa(vcpu, addr, gpa, write);
7679 ret = vcpu_mmio_gva_to_gpa(vcpu, addr, &gpa, exception, write);
7684 if (!ret && ops->read_write_emulate(vcpu, gpa, val, bytes))
7690 handled = ops->read_write_mmio(vcpu, gpa, bytes, val);
7698 WARN_ON(vcpu->mmio_nr_fragments >= KVM_MAX_MMIO_FRAGMENTS);
7699 frag = &vcpu->mmio_fragments[vcpu->mmio_nr_fragments++];
7712 struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
7717 ops->read_write_prepare(vcpu, val, bytes))
7720 vcpu->mmio_nr_fragments = 0;
7728 vcpu, ops);
7740 vcpu, ops);
7744 if (!vcpu->mmio_nr_fragments)
7747 gpa = vcpu->mmio_fragments[0].gpa;
7749 vcpu->mmio_needed = 1;
7750 vcpu->mmio_cur_fragment = 0;
7752 vcpu->run->mmio.len = min(8u, vcpu->mmio_fragments[0].len);
7753 vcpu->run->mmio.is_write = vcpu->mmio_is_write = ops->write;
7754 vcpu->run->exit_reason = KVM_EXIT_MMIO;
7755 vcpu->run->mmio.phys_addr = gpa;
7757 return ops->read_write_exit_mmio(vcpu, gpa, val, bytes);
7790 struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
7800 gpa = kvm_mmu_gva_to_gpa_write(vcpu, addr, NULL);
7818 hva = kvm_vcpu_gfn_to_hva(vcpu, gpa_to_gfn(gpa));
7846 kvm_page_track_write(vcpu, gpa, new, bytes);
7856 static int emulator_pio_in_out(struct kvm_vcpu *vcpu, int size,
7863 WARN_ON_ONCE(vcpu->arch.pio.count);
7866 r = kvm_io_bus_read(vcpu, KVM_PIO_BUS, port, size, data);
7868 r = kvm_io_bus_write(vcpu, KVM_PIO_BUS, port, size, data);
7888 vcpu->arch.pio.port = port;
7889 vcpu->arch.pio.in = in;
7890 vcpu->arch.pio.count = count;
7891 vcpu->arch.pio.size = size;
7894 memset(vcpu->arch.pio_data, 0, size * count);
7896 memcpy(vcpu->arch.pio_data, data, size * count);
7898 vcpu->run->exit_reason = KVM_EXIT_IO;
7899 vcpu->run->io.direction = in ? KVM_EXIT_IO_IN : KVM_EXIT_IO_OUT;
7900 vcpu->run->io.size = size;
7901 vcpu->run->io.data_offset = KVM_PIO_PAGE_OFFSET * PAGE_SIZE;
7902 vcpu->run->io.count = count;
7903 vcpu->run->io.port = port;
7907 static int emulator_pio_in(struct kvm_vcpu *vcpu, int size,
7910 int r = emulator_pio_in_out(vcpu, size, port, val, count, true);
7917 static void complete_emulator_pio_in(struct kvm_vcpu *vcpu, void *val)
7919 int size = vcpu->arch.pio.size;
7920 unsigned int count = vcpu->arch.pio.count;
7921 memcpy(val, vcpu->arch.pio_data, size * count);
7922 trace_kvm_pio(KVM_PIO_IN, vcpu->arch.pio.port, size, count, vcpu->arch.pio_data);
7923 vcpu->arch.pio.count = 0;
7930 struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
7931 if (vcpu->arch.pio.count) {
7939 complete_emulator_pio_in(vcpu, val);
7943 return emulator_pio_in(vcpu, size, port, val, count);
7946 static int emulator_pio_out(struct kvm_vcpu *vcpu, int size,
7951 return emulator_pio_in_out(vcpu, size, port, (void *)val, count, false);
7961 static unsigned long get_segment_base(struct kvm_vcpu *vcpu, int seg)
7963 return static_call(kvm_x86_get_segment_base)(vcpu, seg);
7971 static int kvm_emulate_wbinvd_noskip(struct kvm_vcpu *vcpu)
7973 if (!need_emulate_wbinvd(vcpu))
7979 cpumask_set_cpu(cpu, vcpu->arch.wbinvd_dirty_mask);
7980 on_each_cpu_mask(vcpu->arch.wbinvd_dirty_mask,
7983 cpumask_clear(vcpu->arch.wbinvd_dirty_mask);
7989 int kvm_emulate_wbinvd(struct kvm_vcpu *vcpu)
7991 kvm_emulate_wbinvd_noskip(vcpu);
7992 return kvm_skip_emulated_instruction(vcpu);
8023 struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
8028 value = kvm_read_cr0(vcpu);
8031 value = vcpu->arch.cr2;
8034 value = kvm_read_cr3(vcpu);
8037 value = kvm_read_cr4(vcpu);
8040 value = kvm_get_cr8(vcpu);
8052 struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
8057 res = kvm_set_cr0(vcpu, mk_cr_64(kvm_read_cr0(vcpu), val));
8060 vcpu->arch.cr2 = val;
8063 res = kvm_set_cr3(vcpu, val);
8066 res = kvm_set_cr4(vcpu, mk_cr_64(kvm_read_cr4(vcpu), val));
8069 res = kvm_set_cr8(vcpu, val);
8150 struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
8172 kvm_set_segment(vcpu, &var, seg);
8179 struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
8182 r = kvm_get_msr_with_filter(vcpu, msr_index, pdata);
8187 if (kvm_msr_user_space(vcpu, msr_index, KVM_EXIT_X86_RDMSR, 0,
8202 struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
8205 r = kvm_set_msr_with_filter(vcpu, msr_index, data);
8210 if (kvm_msr_user_space(vcpu, msr_index, KVM_EXIT_X86_WRMSR, data,
8374 static void toggle_interruptibility(struct kvm_vcpu *vcpu, u32 mask)
8376 u32 int_shadow = static_call(kvm_x86_get_interrupt_shadow)(vcpu);
8387 static_call(kvm_x86_set_interrupt_shadow)(vcpu, mask);
8389 kvm_make_request(KVM_REQ_EVENT, vcpu);
8393 static void inject_emulated_exception(struct kvm_vcpu *vcpu)
8395 struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt;
8398 kvm_inject_emulated_page_fault(vcpu, &ctxt->exception);
8400 kvm_queue_exception_e(vcpu, ctxt->exception.vector,
8403 kvm_queue_exception(vcpu, ctxt->exception.vector);
8406 static struct x86_emulate_ctxt *alloc_emulate_ctxt(struct kvm_vcpu *vcpu)
8412 pr_err("failed to allocate vcpu's emulator\n");
8416 ctxt->vcpu = vcpu;
8418 vcpu->arch.emulate_ctxt = ctxt;
8423 static void init_emulate_ctxt(struct kvm_vcpu *vcpu)
8425 struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt;
8428 static_call(kvm_x86_get_cs_db_l_bits)(vcpu, &cs_db, &cs_l);
8431 ctxt->eflags = kvm_get_rflags(vcpu);
8434 ctxt->eip = kvm_rip_read(vcpu);
8435 ctxt->mode = (!is_protmode(vcpu)) ? X86EMUL_MODE_REAL :
8437 (cs_l && is_long_mode(vcpu)) ? X86EMUL_MODE_PROT64 :
8446 vcpu->arch.emulate_regs_need_sync_from_vcpu = false;
8449 void kvm_inject_realmode_interrupt(struct kvm_vcpu *vcpu, int irq, int inc_eip)
8451 struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt;
8454 init_emulate_ctxt(vcpu);
8462 kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);
8465 kvm_rip_write(vcpu, ctxt->eip);
8466 kvm_set_rflags(vcpu, ctxt->eflags);
8471 static void prepare_emulation_failure_exit(struct kvm_vcpu *vcpu, u64 *data,
8474 struct kvm_run *run = vcpu->run;
8484 static_call(kvm_x86_get_exit_info)(vcpu, (u32 *)&info[0], &info[1],
8522 static void prepare_emulation_ctxt_failure_exit(struct kvm_vcpu *vcpu)
8524 struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt;
8526 prepare_emulation_failure_exit(vcpu, NULL, 0, ctxt->fetch.data,
8530 void __kvm_prepare_emulation_failure_exit(struct kvm_vcpu *vcpu, u64 *data,
8533 prepare_emulation_failure_exit(vcpu, data, ndata, NULL, 0);
8537 void kvm_prepare_emulation_failure_exit(struct kvm_vcpu *vcpu)
8539 __kvm_prepare_emulation_failure_exit(vcpu, NULL, 0);
8543 static int handle_emulation_failure(struct kvm_vcpu *vcpu, int emulation_type)
8545 struct kvm *kvm = vcpu->kvm;
8547 ++vcpu->stat.insn_emulation_fail;
8548 trace_kvm_emulate_insn_failed(vcpu);
8551 kvm_queue_exception_e(vcpu, GP_VECTOR, 0);
8557 prepare_emulation_ctxt_failure_exit(vcpu);
8561 kvm_queue_exception(vcpu, UD_VECTOR);
8563 if (!is_guest_mode(vcpu) && static_call(kvm_x86_get_cpl)(vcpu) == 0) {
8564 prepare_emulation_ctxt_failure_exit(vcpu);
8571 static bool reexecute_instruction(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa,
8580 if (WARN_ON_ONCE(is_guest_mode(vcpu)) ||
8584 if (!vcpu->arch.mmu->root_role.direct) {
8589 gpa = kvm_mmu_gva_to_gpa_write(vcpu, cr2_or_gpa, NULL);
8605 pfn = gfn_to_pfn(vcpu->kvm, gpa_to_gfn(gpa));
8617 if (vcpu->arch.mmu->root_role.direct) {
8620 write_lock(&vcpu->kvm->mmu_lock);
8621 indirect_shadow_pages = vcpu->kvm->arch.indirect_shadow_pages;
8622 write_unlock(&vcpu->kvm->mmu_lock);
8625 kvm_mmu_unprotect_page(vcpu->kvm, gpa_to_gfn(gpa));
8635 kvm_mmu_unprotect_page(vcpu->kvm, gpa_to_gfn(gpa));
8648 struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
8651 last_retry_eip = vcpu->arch.last_retry_eip;
8652 last_retry_addr = vcpu->arch.last_retry_addr;
8667 vcpu->arch.last_retry_eip = vcpu->arch.last_retry_addr = 0;
8672 if (WARN_ON_ONCE(is_guest_mode(vcpu)) ||
8682 vcpu->arch.last_retry_eip = ctxt->eip;
8683 vcpu->arch.last_retry_addr = cr2_or_gpa;
8685 if (!vcpu->arch.mmu->root_role.direct)
8686 gpa = kvm_mmu_gva_to_gpa_write(vcpu, cr2_or_gpa, NULL);
8688 kvm_mmu_unprotect_page(vcpu->kvm, gpa_to_gfn(gpa));
8693 static int complete_emulated_mmio(struct kvm_vcpu *vcpu);
8694 static int complete_emulated_pio(struct kvm_vcpu *vcpu);
8711 static int kvm_vcpu_do_singlestep(struct kvm_vcpu *vcpu)
8713 struct kvm_run *kvm_run = vcpu->run;
8715 if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP) {
8717 kvm_run->debug.arch.pc = kvm_get_linear_rip(vcpu);
8722 kvm_queue_exception_p(vcpu, DB_VECTOR, DR6_BS);
8726 int kvm_skip_emulated_instruction(struct kvm_vcpu *vcpu)
8728 unsigned long rflags = static_call(kvm_x86_get_rflags)(vcpu);
8731 r = static_call(kvm_x86_skip_emulated_instruction)(vcpu);
8735 kvm_pmu_trigger_event(vcpu, PERF_COUNT_HW_INSTRUCTIONS);
8746 r = kvm_vcpu_do_singlestep(vcpu);
8751 static bool kvm_is_code_breakpoint_inhibited(struct kvm_vcpu *vcpu)
8755 if (kvm_get_rflags(vcpu) & X86_EFLAGS_RF)
8763 shadow = static_call(kvm_x86_get_interrupt_shadow)(vcpu);
8765 guest_cpuid_is_intel(vcpu);
8768 static bool kvm_vcpu_check_code_breakpoint(struct kvm_vcpu *vcpu,
8791 if (unlikely(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) &&
8792 (vcpu->arch.guest_debug_dr7 & DR7_BP_EN_MASK)) {
8793 struct kvm_run *kvm_run = vcpu->run;
8794 unsigned long eip = kvm_get_linear_rip(vcpu);
8796 vcpu->arch.guest_debug_dr7,
8797 vcpu->arch.eff_db);
8809 if (unlikely(vcpu->arch.dr7 & DR7_BP_EN_MASK) &&
8810 !kvm_is_code_breakpoint_inhibited(vcpu)) {
8811 unsigned long eip = kvm_get_linear_rip(vcpu);
8813 vcpu->arch.dr7,
8814 vcpu->arch.db);
8817 kvm_queue_exception_p(vcpu, DB_VECTOR, dr6);
8867 int x86_decode_emulated_instruction(struct kvm_vcpu *vcpu, int emulation_type,
8870 struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt;
8873 init_emulate_ctxt(vcpu);
8877 trace_kvm_emulate_insn_start(vcpu);
8878 ++vcpu->stat.insn_emulation;
8884 int x86_emulate_instruction(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa,
8888 struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt;
8891 if (unlikely(!kvm_can_emulate_insn(vcpu, emulation_type, insn, insn_len)))
8894 vcpu->arch.l1tf_flush_l1d = true;
8897 kvm_clear_exception_queue(vcpu);
8904 if (kvm_vcpu_check_code_breakpoint(vcpu, emulation_type, &r))
8907 r = x86_decode_emulated_instruction(vcpu, emulation_type,
8912 kvm_queue_exception(vcpu, UD_VECTOR);
8915 if (reexecute_instruction(vcpu, cr2_or_gpa,
8927 inject_emulated_exception(vcpu);
8930 return handle_emulation_failure(vcpu, emulation_type);
8936 kvm_queue_exception_e(vcpu, GP_VECTOR, 0);
8957 kvm_rip_write(vcpu, ctxt->eip);
8959 kvm_set_rflags(vcpu, ctxt->eflags & ~X86_EFLAGS_RF);
8968 if (vcpu->arch.emulate_regs_need_sync_from_vcpu) {
8969 vcpu->arch.emulate_regs_need_sync_from_vcpu = false;
8979 if (vcpu->arch.mmu->root_role.direct) {
8994 if (reexecute_instruction(vcpu, cr2_or_gpa, emulation_type))
8997 return handle_emulation_failure(vcpu, emulation_type);
9001 WARN_ON_ONCE(vcpu->mmio_needed && !vcpu->mmio_is_write);
9002 vcpu->mmio_needed = false;
9004 inject_emulated_exception(vcpu);
9005 } else if (vcpu->arch.pio.count) {
9006 if (!vcpu->arch.pio.in) {
9008 vcpu->arch.pio.count = 0;
9011 vcpu->arch.complete_userspace_io = complete_emulated_pio;
9014 } else if (vcpu->mmio_needed) {
9015 ++vcpu->stat.mmio_exits;
9017 if (!vcpu->mmio_is_write)
9020 vcpu->arch.complete_userspace_io = complete_emulated_mmio;
9021 } else if (vcpu->arch.complete_userspace_io) {
9031 unsigned long rflags = static_call(kvm_x86_get_rflags)(vcpu);
9032 toggle_interruptibility(vcpu, ctxt->interruptibility);
9033 vcpu->arch.emulate_regs_need_sync_to_vcpu = false;
9042 kvm_pmu_trigger_event(vcpu, PERF_COUNT_HW_INSTRUCTIONS);
9044 kvm_pmu_trigger_event(vcpu, PERF_COUNT_HW_BRANCH_INSTRUCTIONS);
9045 kvm_rip_write(vcpu, ctxt->eip);
9046 if (r && (ctxt->tf || (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP)))
9047 r = kvm_vcpu_do_singlestep(vcpu);
9048 static_call_cond(kvm_x86_update_emulated_instruction)(vcpu);
9049 __kvm_set_rflags(vcpu, ctxt->eflags);
9059 kvm_make_request(KVM_REQ_EVENT, vcpu);
9061 vcpu->arch.emulate_regs_need_sync_to_vcpu = true;
9066 int kvm_emulate_instruction(struct kvm_vcpu *vcpu, int emulation_type)
9068 return x86_emulate_instruction(vcpu, 0, emulation_type, NULL, 0);
9072 int kvm_emulate_instruction_from_buffer(struct kvm_vcpu *vcpu,
9075 return x86_emulate_instruction(vcpu, 0, 0, insn, insn_len);
9079 static int complete_fast_pio_out_port_0x7e(struct kvm_vcpu *vcpu)
9081 vcpu->arch.pio.count = 0;
9085 static int complete_fast_pio_out(struct kvm_vcpu *vcpu)
9087 vcpu->arch.pio.count = 0;
9089 if (unlikely(!kvm_is_linear_rip(vcpu, vcpu->arch.pio.linear_rip)))
9092 return kvm_skip_emulated_instruction(vcpu);
9095 static int kvm_fast_pio_out(struct kvm_vcpu *vcpu, int size,
9098 unsigned long val = kvm_rax_read(vcpu);
9099 int ret = emulator_pio_out(vcpu, size, port, &val, 1);
9109 kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_OUT_7E_INC_RIP)) {
9110 vcpu->arch.complete_userspace_io =
9112 kvm_skip_emulated_instruction(vcpu);
9114 vcpu->arch.pio.linear_rip = kvm_get_linear_rip(vcpu);
9115 vcpu->arch.complete_userspace_io = complete_fast_pio_out;
9120 static int complete_fast_pio_in(struct kvm_vcpu *vcpu)
9125 BUG_ON(vcpu->arch.pio.count != 1);
9127 if (unlikely(!kvm_is_linear_rip(vcpu, vcpu->arch.pio.linear_rip))) {
9128 vcpu->arch.pio.count = 0;
9133 val = (vcpu->arch.pio.size < 4) ? kvm_rax_read(vcpu) : 0;
9135 complete_emulator_pio_in(vcpu, &val);
9136 kvm_rax_write(vcpu, val);
9138 return kvm_skip_emulated_instruction(vcpu);
9141 static int kvm_fast_pio_in(struct kvm_vcpu *vcpu, int size,
9148 val = (size < 4) ? kvm_rax_read(vcpu) : 0;
9150 ret = emulator_pio_in(vcpu, size, port, &val, 1);
9152 kvm_rax_write(vcpu, val);
9156 vcpu->arch.pio.linear_rip = kvm_get_linear_rip(vcpu);
9157 vcpu->arch.complete_userspace_io = complete_fast_pio_in;
9162 int kvm_fast_pio(struct kvm_vcpu *vcpu, int size, unsigned short port, int in)
9167 ret = kvm_fast_pio_in(vcpu, size, port);
9169 ret = kvm_fast_pio_out(vcpu, size, port);
9170 return ret && kvm_skip_emulated_instruction(vcpu);
9229 struct kvm_vcpu *vcpu;
9276 kvm_for_each_vcpu(i, vcpu, kvm) {
9277 if (vcpu->cpu != cpu)
9279 kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);
9280 if (vcpu->cpu != raw_smp_processor_id())
9360 struct kvm_vcpu *vcpu;
9365 kvm_for_each_vcpu(i, vcpu, kvm)
9366 kvm_make_request(KVM_REQ_MASTERCLOCK_UPDATE, vcpu);
9636 static int __kvm_emulate_halt(struct kvm_vcpu *vcpu, int state, int reason)
9645 ++vcpu->stat.halt_exits;
9646 if (lapic_in_kernel(vcpu)) {
9647 vcpu->arch.mp_state = state;
9650 vcpu->run->exit_reason = reason;
9655 int kvm_emulate_halt_noskip(struct kvm_vcpu *vcpu)
9657 return __kvm_emulate_halt(vcpu, KVM_MP_STATE_HALTED, KVM_EXIT_HLT);
9661 int kvm_emulate_halt(struct kvm_vcpu *vcpu)
9663 int ret = kvm_skip_emulated_instruction(vcpu);
9668 return kvm_emulate_halt_noskip(vcpu) && ret;
9672 int kvm_emulate_ap_reset_hold(struct kvm_vcpu *vcpu)
9674 int ret = kvm_skip_emulated_instruction(vcpu);
9676 return __kvm_emulate_halt(vcpu, KVM_MP_STATE_AP_RESET_HOLD,
9682 static int kvm_pv_clock_pairing(struct kvm_vcpu *vcpu, gpa_t paddr,
9697 if (vcpu->arch.tsc_always_catchup)
9705 clock_pairing.tsc = kvm_read_l1_tsc(vcpu, cycle);
9710 if (kvm_write_guest(vcpu->kvm, paddr, &clock_pairing,
9719 * kvm_pv_kick_cpu_op: Kick a vcpu.
9721 * @apicid - apicid of vcpu to be kicked.
9745 bool kvm_vcpu_apicv_activated(struct kvm_vcpu *vcpu)
9747 ulong vm_reasons = READ_ONCE(vcpu->kvm->arch.apicv_inhibit_reasons);
9748 ulong vcpu_reasons = static_call(kvm_x86_vcpu_get_apicv_inhibit_reasons)(vcpu);
9778 static void kvm_sched_yield(struct kvm_vcpu *vcpu, unsigned long dest_id)
9783 vcpu->stat.directed_yield_attempted++;
9789 map = rcu_dereference(vcpu->kvm->arch.apic_map);
9792 target = map->phys_map[dest_id]->vcpu;
9800 if (vcpu == target)
9806 vcpu->stat.directed_yield_successful++;
9812 static int complete_hypercall_exit(struct kvm_vcpu *vcpu)
9814 u64 ret = vcpu->run->hypercall.ret;
9816 if (!is_64_bit_mode(vcpu))
9818 kvm_rax_write(vcpu, ret);
9819 ++vcpu->stat.hypercalls;
9820 return kvm_skip_emulated_instruction(vcpu);
9823 int kvm_emulate_hypercall(struct kvm_vcpu *vcpu)
9828 if (kvm_xen_hypercall_enabled(vcpu->kvm))
9829 return kvm_xen_hypercall(vcpu);
9831 if (kvm_hv_hypercall_enabled(vcpu))
9832 return kvm_hv_hypercall(vcpu);
9834 nr = kvm_rax_read(vcpu);
9835 a0 = kvm_rbx_read(vcpu);
9836 a1 = kvm_rcx_read(vcpu);
9837 a2 = kvm_rdx_read(vcpu);
9838 a3 = kvm_rsi_read(vcpu);
9842 op_64_bit = is_64_bit_hypercall(vcpu);
9851 if (static_call(kvm_x86_get_cpl)(vcpu) != 0) {
9863 if (!guest_pv_has(vcpu, KVM_FEATURE_PV_UNHALT))
9866 kvm_pv_kick_cpu_op(vcpu->kvm, a1);
9867 kvm_sched_yield(vcpu, a1);
9872 ret = kvm_pv_clock_pairing(vcpu, a0, a1);
9876 if (!guest_pv_has(vcpu, KVM_FEATURE_PV_SEND_IPI))
9879 ret = kvm_pv_send_ipi(vcpu->kvm, a0, a1, a2, a3, op_64_bit);
9882 if (!guest_pv_has(vcpu, KVM_FEATURE_PV_SCHED_YIELD))
9885 kvm_sched_yield(vcpu, a0);
9892 if (!(vcpu->kvm->arch.hypercall_exit_enabled & (1 << KVM_HC_MAP_GPA_RANGE)))
9901 vcpu->run->exit_reason = KVM_EXIT_HYPERCALL;
9902 vcpu->run->hypercall.nr = KVM_HC_MAP_GPA_RANGE;
9903 vcpu->run->hypercall.args[0] = gpa;
9904 vcpu->run->hypercall.args[1] = npages;
9905 vcpu->run->hypercall.args[2] = attrs;
9906 vcpu->run->hypercall.flags = 0;
9908 vcpu->run->hypercall.flags |= KVM_EXIT_HYPERCALL_LONG_MODE;
9910 WARN_ON_ONCE(vcpu->run->hypercall.flags & KVM_EXIT_HYPERCALL_MBZ);
9911 vcpu->arch.complete_userspace_io = complete_hypercall_exit;
9921 kvm_rax_write(vcpu, ret);
9923 ++vcpu->stat.hypercalls;
9924 return kvm_skip_emulated_instruction(vcpu);
9930 struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
9932 unsigned long rip = kvm_rip_read(vcpu);
9938 if (!kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_FIX_HYPERCALL_INSN)) {
9945 static_call(kvm_x86_patch_hypercall)(vcpu, instruction);
9951 static int dm_request_for_irq_injection(struct kvm_vcpu *vcpu)
9953 return vcpu->run->request_interrupt_window &&
9954 likely(!pic_in_kernel(vcpu->kvm));
9958 static void post_kvm_run_save(struct kvm_vcpu *vcpu)
9960 struct kvm_run *kvm_run = vcpu->run;
9962 kvm_run->if_flag = static_call(kvm_x86_get_if_flag)(vcpu);
9963 kvm_run->cr8 = kvm_get_cr8(vcpu);
9964 kvm_run->apic_base = kvm_get_apic_base(vcpu);
9967 pic_in_kernel(vcpu->kvm) ||
9968 kvm_vcpu_ready_for_interrupt_injection(vcpu);
9970 if (is_smm(vcpu))
9974 static void update_cr8_intercept(struct kvm_vcpu *vcpu)
9981 if (!lapic_in_kernel(vcpu))
9984 if (vcpu->arch.apic->apicv_active)
9987 if (!vcpu->arch.apic->vapic_addr)
9988 max_irr = kvm_lapic_find_highest_irr(vcpu);
9995 tpr = kvm_lapic_get_cr8(vcpu);
9997 static_call(kvm_x86_update_cr8_intercept)(vcpu, tpr, max_irr);
10001 int kvm_check_nested_events(struct kvm_vcpu *vcpu)
10003 if (kvm_test_request(KVM_REQ_TRIPLE_FAULT, vcpu)) {
10004 kvm_x86_ops.nested_ops->triple_fault(vcpu);
10008 return kvm_x86_ops.nested_ops->check_events(vcpu);
10011 static void kvm_inject_exception(struct kvm_vcpu *vcpu)
10020 vcpu->arch.exception.has_error_code &= is_protmode(vcpu);
10022 trace_kvm_inj_exception(vcpu->arch.exception.vector,
10023 vcpu->arch.exception.has_error_code,
10024 vcpu->arch.exception.error_code,
10025 vcpu->arch.exception.injected);
10027 static_call(kvm_x86_inject_exception)(vcpu);
10069 static int kvm_check_and_inject_events(struct kvm_vcpu *vcpu,
10080 if (is_guest_mode(vcpu))
10081 r = kvm_check_nested_events(vcpu);
10108 if (vcpu->arch.exception.injected)
10109 kvm_inject_exception(vcpu);
10110 else if (kvm_is_exception_pending(vcpu))
10112 else if (vcpu->arch.nmi_injected)
10113 static_call(kvm_x86_inject_nmi)(vcpu);
10114 else if (vcpu->arch.interrupt.injected)
10115 static_call(kvm_x86_inject_irq)(vcpu, true);
10122 WARN_ON_ONCE(vcpu->arch.exception.injected &&
10123 vcpu->arch.exception.pending);
10142 WARN_ON_ONCE(vcpu->arch.exception_vmexit.injected ||
10143 vcpu->arch.exception_vmexit.pending);
10150 can_inject = !kvm_event_needs_reinjection(vcpu);
10152 if (vcpu->arch.exception.pending) {
10163 if (exception_type(vcpu->arch.exception.vector) == EXCPT_FAULT)
10164 __kvm_set_rflags(vcpu, kvm_get_rflags(vcpu) |
10167 if (vcpu->arch.exception.vector == DB_VECTOR) {
10168 kvm_deliver_exception_payload(vcpu, &vcpu->arch.exception);
10169 if (vcpu->arch.dr7 & DR7_GD) {
10170 vcpu->arch.dr7 &= ~DR7_GD;
10171 kvm_update_dr7(vcpu);
10175 kvm_inject_exception(vcpu);
10177 vcpu->arch.exception.pending = false;
10178 vcpu->arch.exception.injected = true;
10184 if (vcpu->guest_debug & KVM_GUESTDBG_BLOCKIRQ)
10199 if (vcpu->arch.smi_pending) {
10200 r = can_inject ? static_call(kvm_x86_smi_allowed)(vcpu, true) : -EBUSY;
10204 vcpu->arch.smi_pending = false;
10205 ++vcpu->arch.smi_count;
10206 enter_smm(vcpu);
10209 static_call(kvm_x86_enable_smi_window)(vcpu);
10213 if (vcpu->arch.nmi_pending) {
10214 r = can_inject ? static_call(kvm_x86_nmi_allowed)(vcpu, true) : -EBUSY;
10218 --vcpu->arch.nmi_pending;
10219 vcpu->arch.nmi_injected = true;
10220 static_call(kvm_x86_inject_nmi)(vcpu);
10222 WARN_ON(static_call(kvm_x86_nmi_allowed)(vcpu, true) < 0);
10224 if (vcpu->arch.nmi_pending)
10225 static_call(kvm_x86_enable_nmi_window)(vcpu);
10228 if (kvm_cpu_has_injectable_intr(vcpu)) {
10229 r = can_inject ? static_call(kvm_x86_interrupt_allowed)(vcpu, true) : -EBUSY;
10233 int irq = kvm_cpu_get_interrupt(vcpu);
10236 kvm_queue_interrupt(vcpu, irq, false);
10237 static_call(kvm_x86_inject_irq)(vcpu, false);
10238 WARN_ON(static_call(kvm_x86_interrupt_allowed)(vcpu, true) < 0);
10241 if (kvm_cpu_has_injectable_intr(vcpu))
10242 static_call(kvm_x86_enable_irq_window)(vcpu);
10245 if (is_guest_mode(vcpu) &&
10247 kvm_x86_ops.nested_ops->has_events(vcpu))
10262 WARN_ON_ONCE(vcpu->arch.exception.pending ||
10263 vcpu->arch.exception_vmexit.pending);
10274 static void process_nmi(struct kvm_vcpu *vcpu)
10288 if (static_call(kvm_x86_get_nmi_mask)(vcpu) || vcpu->arch.nmi_injected)
10295 * tracked in vcpu->arch.nmi_pending.
10297 if (static_call(kvm_x86_is_vnmi_pending)(vcpu))
10300 vcpu->arch.nmi_pending += atomic_xchg(&vcpu->arch.nmi_queued, 0);
10301 vcpu->arch.nmi_pending = min(vcpu->arch.nmi_pending, limit);
10303 if (vcpu->arch.nmi_pending &&
10304 (static_call(kvm_x86_set_vnmi_pending)(vcpu)))
10305 vcpu->arch.nmi_pending--;
10307 if (vcpu->arch.nmi_pending)
10308 kvm_make_request(KVM_REQ_EVENT, vcpu);
10312 int kvm_get_nr_pending_nmis(struct kvm_vcpu *vcpu)
10314 return vcpu->arch.nmi_pending +
10315 static_call(kvm_x86_is_vnmi_pending)(vcpu);
10329 void __kvm_vcpu_update_apicv(struct kvm_vcpu *vcpu)
10331 struct kvm_lapic *apic = vcpu->arch.apic;
10334 if (!lapic_in_kernel(vcpu))
10337 down_read(&vcpu->kvm->arch.apicv_update_lock);
10341 activate = kvm_vcpu_apicv_activated(vcpu) &&
10342 (kvm_get_apic_mode(vcpu) != LAPIC_MODE_DISABLED);
10348 kvm_apic_update_apicv(vcpu);
10349 static_call(kvm_x86_refresh_apicv_exec_ctrl)(vcpu);
10358 kvm_make_request(KVM_REQ_EVENT, vcpu);
10362 up_read(&vcpu->kvm->arch.apicv_update_lock);
10366 static void kvm_vcpu_update_apicv(struct kvm_vcpu *vcpu)
10368 if (!lapic_in_kernel(vcpu))
10382 if (apic_x2apic_mode(vcpu->arch.apic) &&
10384 kvm_inhibit_apic_access_page(vcpu);
10386 __kvm_vcpu_update_apicv(vcpu);
10442 static void vcpu_scan_ioapic(struct kvm_vcpu *vcpu)
10444 if (!kvm_apic_present(vcpu))
10447 bitmap_zero(vcpu->arch.ioapic_handled_vectors, 256);
10449 if (irqchip_split(vcpu->kvm))
10450 kvm_scan_ioapic_routes(vcpu, vcpu->arch.ioapic_handled_vectors);
10452 static_call_cond(kvm_x86_sync_pir_to_irr)(vcpu);
10453 if (ioapic_in_kernel(vcpu->kvm))
10454 kvm_ioapic_scan_entry(vcpu, vcpu->arch.ioapic_handled_vectors);
10457 if (is_guest_mode(vcpu))
10458 vcpu->arch.load_eoi_exitmap_pending = true;
10460 kvm_make_request(KVM_REQ_LOAD_EOI_EXITMAP, vcpu);
10463 static void vcpu_load_eoi_exitmap(struct kvm_vcpu *vcpu)
10467 if (!kvm_apic_hw_enabled(vcpu->arch.apic))
10470 if (to_hv_vcpu(vcpu)) {
10472 vcpu->arch.ioapic_handled_vectors,
10473 to_hv_synic(vcpu)->vec_bitmap, 256);
10474 static_call_cond(kvm_x86_load_eoi_exitmap)(vcpu, eoi_exit_bitmap);
10479 vcpu, (u64 *)vcpu->arch.ioapic_handled_vectors);
10487 static void kvm_vcpu_reload_apic_access_page(struct kvm_vcpu *vcpu)
10489 if (!lapic_in_kernel(vcpu))
10492 static_call_cond(kvm_x86_set_apic_access_page_addr)(vcpu);
10495 void __kvm_request_immediate_exit(struct kvm_vcpu *vcpu)
10497 smp_send_reschedule(vcpu->cpu);
10507 static int vcpu_enter_guest(struct kvm_vcpu *vcpu)
10511 dm_request_for_irq_injection(vcpu) &&
10512 kvm_cpu_accept_dm_intr(vcpu);
10517 if (kvm_request_pending(vcpu)) {
10518 if (kvm_check_request(KVM_REQ_VM_DEAD, vcpu)) {
10523 if (kvm_dirty_ring_check_request(vcpu)) {
10528 if (kvm_check_request(KVM_REQ_GET_NESTED_STATE_PAGES, vcpu)) {
10529 if (unlikely(!kvm_x86_ops.nested_ops->get_nested_state_pages(vcpu))) {
10534 if (kvm_check_request(KVM_REQ_MMU_FREE_OBSOLETE_ROOTS, vcpu))
10535 kvm_mmu_free_obsolete_roots(vcpu);
10536 if (kvm_check_request(KVM_REQ_MIGRATE_TIMER, vcpu))
10537 __kvm_migrate_timers(vcpu);
10538 if (kvm_check_request(KVM_REQ_MASTERCLOCK_UPDATE, vcpu))
10539 kvm_update_masterclock(vcpu->kvm);
10540 if (kvm_check_request(KVM_REQ_GLOBAL_CLOCK_UPDATE, vcpu))
10541 kvm_gen_kvmclock_update(vcpu);
10542 if (kvm_check_request(KVM_REQ_CLOCK_UPDATE, vcpu)) {
10543 r = kvm_guest_time_update(vcpu);
10547 if (kvm_check_request(KVM_REQ_MMU_SYNC, vcpu))
10548 kvm_mmu_sync_roots(vcpu);
10549 if (kvm_check_request(KVM_REQ_LOAD_MMU_PGD, vcpu))
10550 kvm_mmu_load_pgd(vcpu);
10557 if (kvm_check_request(KVM_REQ_TLB_FLUSH, vcpu))
10558 kvm_vcpu_flush_tlb_all(vcpu);
10560 kvm_service_local_tlb_flush_requests(vcpu);
10568 if (kvm_check_request(KVM_REQ_HV_TLB_FLUSH, vcpu) &&
10569 kvm_hv_vcpu_flush_tlb(vcpu))
10570 kvm_vcpu_flush_tlb_guest(vcpu);
10572 if (kvm_check_request(KVM_REQ_REPORT_TPR_ACCESS, vcpu)) {
10573 vcpu->run->exit_reason = KVM_EXIT_TPR_ACCESS;
10577 if (kvm_test_request(KVM_REQ_TRIPLE_FAULT, vcpu)) {
10578 if (is_guest_mode(vcpu))
10579 kvm_x86_ops.nested_ops->triple_fault(vcpu);
10581 if (kvm_check_request(KVM_REQ_TRIPLE_FAULT, vcpu)) {
10582 vcpu->run->exit_reason = KVM_EXIT_SHUTDOWN;
10583 vcpu->mmio_needed = 0;
10588 if (kvm_check_request(KVM_REQ_APF_HALT, vcpu)) {
10590 vcpu->arch.apf.halted = true;
10594 if (kvm_check_request(KVM_REQ_STEAL_UPDATE, vcpu))
10595 record_steal_time(vcpu);
10597 if (kvm_check_request(KVM_REQ_SMI, vcpu))
10598 process_smi(vcpu);
10600 if (kvm_check_request(KVM_REQ_NMI, vcpu))
10601 process_nmi(vcpu);
10602 if (kvm_check_request(KVM_REQ_PMU, vcpu))
10603 kvm_pmu_handle_event(vcpu);
10604 if (kvm_check_request(KVM_REQ_PMI, vcpu))
10605 kvm_pmu_deliver_pmi(vcpu);
10606 if (kvm_check_request(KVM_REQ_IOAPIC_EOI_EXIT, vcpu)) {
10607 BUG_ON(vcpu->arch.pending_ioapic_eoi > 255);
10608 if (test_bit(vcpu->arch.pending_ioapic_eoi,
10609 vcpu->arch.ioapic_handled_vectors)) {
10610 vcpu->run->exit_reason = KVM_EXIT_IOAPIC_EOI;
10611 vcpu->run->eoi.vector =
10612 vcpu->arch.pending_ioapic_eoi;
10617 if (kvm_check_request(KVM_REQ_SCAN_IOAPIC, vcpu))
10618 vcpu_scan_ioapic(vcpu);
10619 if (kvm_check_request(KVM_REQ_LOAD_EOI_EXITMAP, vcpu))
10620 vcpu_load_eoi_exitmap(vcpu);
10621 if (kvm_check_request(KVM_REQ_APIC_PAGE_RELOAD, vcpu))
10622 kvm_vcpu_reload_apic_access_page(vcpu);
10623 if (kvm_check_request(KVM_REQ_HV_CRASH, vcpu)) {
10624 vcpu->run->exit_reason = KVM_EXIT_SYSTEM_EVENT;
10625 vcpu->run->system_event.type = KVM_SYSTEM_EVENT_CRASH;
10626 vcpu->run->system_event.ndata = 0;
10630 if (kvm_check_request(KVM_REQ_HV_RESET, vcpu)) {
10631 vcpu->run->exit_reason = KVM_EXIT_SYSTEM_EVENT;
10632 vcpu->run->system_event.type = KVM_SYSTEM_EVENT_RESET;
10633 vcpu->run->system_event.ndata = 0;
10637 if (kvm_check_request(KVM_REQ_HV_EXIT, vcpu)) {
10638 struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu);
10640 vcpu->run->exit_reason = KVM_EXIT_HYPERV;
10641 vcpu->run->hyperv = hv_vcpu->exit;
10651 if (kvm_check_request(KVM_REQ_HV_STIMER, vcpu))
10652 kvm_hv_process_stimers(vcpu);
10653 if (kvm_check_request(KVM_REQ_APICV_UPDATE, vcpu))
10654 kvm_vcpu_update_apicv(vcpu);
10655 if (kvm_check_request(KVM_REQ_APF_READY, vcpu))
10656 kvm_check_async_pf_completion(vcpu);
10657 if (kvm_check_request(KVM_REQ_MSR_FILTER_CHANGED, vcpu))
10658 static_call(kvm_x86_msr_filter_changed)(vcpu);
10660 if (kvm_check_request(KVM_REQ_UPDATE_CPU_DIRTY_LOGGING, vcpu))
10661 static_call(kvm_x86_update_cpu_dirty_logging)(vcpu);
10664 if (kvm_check_request(KVM_REQ_EVENT, vcpu) || req_int_win ||
10665 kvm_xen_has_interrupt(vcpu)) {
10666 ++vcpu->stat.req_event;
10667 r = kvm_apic_accept_events(vcpu);
10672 if (vcpu->arch.mp_state == KVM_MP_STATE_INIT_RECEIVED) {
10677 r = kvm_check_and_inject_events(vcpu, &req_immediate_exit);
10683 static_call(kvm_x86_enable_irq_window)(vcpu);
10685 if (kvm_lapic_enabled(vcpu)) {
10686 update_cr8_intercept(vcpu);
10687 kvm_lapic_sync_to_vapic(vcpu);
10691 r = kvm_mmu_reload(vcpu);
10698 static_call(kvm_x86_prepare_switch_to_guest)(vcpu);
10707 /* Store vcpu->apicv_active before vcpu->mode. */
10708 smp_store_release(&vcpu->mode, IN_GUEST_MODE);
10710 kvm_vcpu_srcu_read_unlock(vcpu);
10733 if (kvm_lapic_enabled(vcpu))
10734 static_call_cond(kvm_x86_sync_pir_to_irr)(vcpu);
10736 if (kvm_vcpu_exit_request(vcpu)) {
10737 vcpu->mode = OUTSIDE_GUEST_MODE;
10741 kvm_vcpu_srcu_read_lock(vcpu);
10747 kvm_make_request(KVM_REQ_EVENT, vcpu);
10748 static_call(kvm_x86_request_immediate_exit)(vcpu);
10755 if (vcpu->arch.guest_fpu.xfd_err)
10756 wrmsrl(MSR_IA32_XFD_ERR, vcpu->arch.guest_fpu.xfd_err);
10758 if (unlikely(vcpu->arch.switch_db_regs)) {
10760 set_debugreg(vcpu->arch.eff_db[0], 0);
10761 set_debugreg(vcpu->arch.eff_db[1], 1);
10762 set_debugreg(vcpu->arch.eff_db[2], 2);
10763 set_debugreg(vcpu->arch.eff_db[3], 3);
10777 WARN_ON_ONCE((kvm_vcpu_apicv_activated(vcpu) != kvm_vcpu_apicv_active(vcpu)) &&
10778 (kvm_get_apic_mode(vcpu) != LAPIC_MODE_DISABLED));
10780 exit_fastpath = static_call(kvm_x86_vcpu_run)(vcpu);
10784 if (kvm_lapic_enabled(vcpu))
10785 static_call_cond(kvm_x86_sync_pir_to_irr)(vcpu);
10787 if (unlikely(kvm_vcpu_exit_request(vcpu))) {
10793 ++vcpu->stat.exits;
10802 if (unlikely(vcpu->arch.switch_db_regs & KVM_DEBUGREG_WONT_EXIT)) {
10803 WARN_ON(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP);
10804 static_call(kvm_x86_sync_dirty_debug_regs)(vcpu);
10805 kvm_update_dr0123(vcpu);
10806 kvm_update_dr7(vcpu);
10819 vcpu->arch.last_vmentry_cpu = vcpu->cpu;
10820 vcpu->arch.last_guest_tsc = kvm_read_l1_tsc(vcpu, rdtsc());
10822 vcpu->mode = OUTSIDE_GUEST_MODE;
10830 if (vcpu->arch.xfd_no_write_intercept)
10833 static_call(kvm_x86_handle_exit_irqoff)(vcpu);
10835 if (vcpu->arch.guest_fpu.xfd_err)
10845 kvm_before_interrupt(vcpu, KVM_HANDLING_IRQ);
10847 ++vcpu->stat.exits;
10849 kvm_after_interrupt(vcpu);
10863 kvm_vcpu_srcu_read_lock(vcpu);
10869 unsigned long rip = kvm_rip_read(vcpu);
10873 if (unlikely(vcpu->arch.tsc_always_catchup))
10874 kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);
10876 if (vcpu->arch.apic_attention)
10877 kvm_lapic_sync_from_vapic(vcpu);
10879 r = static_call(kvm_x86_handle_exit)(vcpu, exit_fastpath);
10884 kvm_make_request(KVM_REQ_EVENT, vcpu);
10885 static_call(kvm_x86_cancel_injection)(vcpu);
10886 if (unlikely(vcpu->arch.apic_attention))
10887 kvm_lapic_sync_from_vapic(vcpu);
10893 static inline int vcpu_block(struct kvm_vcpu *vcpu)
10897 if (!kvm_arch_vcpu_runnable(vcpu)) {
10905 hv_timer = kvm_lapic_hv_timer_in_use(vcpu);
10907 kvm_lapic_switch_to_sw_timer(vcpu);
10909 kvm_vcpu_srcu_read_unlock(vcpu);
10910 if (vcpu->arch.mp_state == KVM_MP_STATE_HALTED)
10911 kvm_vcpu_halt(vcpu);
10913 kvm_vcpu_block(vcpu);
10914 kvm_vcpu_srcu_read_lock(vcpu);
10917 kvm_lapic_switch_to_hv_timer(vcpu);
10924 if (!kvm_arch_vcpu_runnable(vcpu))
10934 if (is_guest_mode(vcpu)) {
10935 if (kvm_check_nested_events(vcpu) < 0)
10939 if (kvm_apic_accept_events(vcpu) < 0)
10941 switch(vcpu->arch.mp_state) {
10944 vcpu->arch.pv.pv_unhalted = false;
10945 vcpu->arch.mp_state =
10949 vcpu->arch.apf.halted = false;
10960 static inline bool kvm_vcpu_running(struct kvm_vcpu *vcpu)
10962 return (vcpu->arch.mp_state == KVM_MP_STATE_RUNNABLE &&
10963 !vcpu->arch.apf.halted);
10967 static int vcpu_run(struct kvm_vcpu *vcpu)
10971 vcpu->arch.l1tf_flush_l1d = true;
10980 vcpu->arch.at_instruction_boundary = false;
10981 if (kvm_vcpu_running(vcpu)) {
10982 r = vcpu_enter_guest(vcpu);
10984 r = vcpu_block(vcpu);
10990 kvm_clear_request(KVM_REQ_UNBLOCK, vcpu);
10991 if (kvm_xen_has_pending_events(vcpu))
10992 kvm_xen_inject_pending_events(vcpu);
10994 if (kvm_cpu_has_pending_timer(vcpu))
10995 kvm_inject_pending_timer_irqs(vcpu);
10997 if (dm_request_for_irq_injection(vcpu) &&
10998 kvm_vcpu_ready_for_interrupt_injection(vcpu)) {
11000 vcpu->run->exit_reason = KVM_EXIT_IRQ_WINDOW_OPEN;
11001 ++vcpu->stat.request_irq_exits;
11006 kvm_vcpu_srcu_read_unlock(vcpu);
11007 r = xfer_to_guest_mode_handle_work(vcpu);
11008 kvm_vcpu_srcu_read_lock(vcpu);
11017 static inline int complete_emulated_io(struct kvm_vcpu *vcpu)
11019 return kvm_emulate_instruction(vcpu, EMULTYPE_NO_DECODE);
11022 static int complete_emulated_pio(struct kvm_vcpu *vcpu)
11024 BUG_ON(!vcpu->arch.pio.count);
11026 return complete_emulated_io(vcpu);
11047 static int complete_emulated_mmio(struct kvm_vcpu *vcpu)
11049 struct kvm_run *run = vcpu->run;
11053 BUG_ON(!vcpu->mmio_needed);
11056 frag = &vcpu->mmio_fragments[vcpu->mmio_cur_fragment];
11058 if (!vcpu->mmio_is_write)
11064 vcpu->mmio_cur_fragment++;
11072 if (vcpu->mmio_cur_fragment >= vcpu->mmio_nr_fragments) {
11073 vcpu->mmio_needed = 0;
11076 if (vcpu->mmio_is_write)
11078 vcpu->mmio_read_completed = 1;
11079 return complete_emulated_io(vcpu);
11084 if (vcpu->mmio_is_write)
11087 run->mmio.is_write = vcpu->mmio_is_write;
11088 vcpu->arch.complete_userspace_io = complete_emulated_mmio;
11093 static void kvm_load_guest_fpu(struct kvm_vcpu *vcpu)
11096 fpu_swap_kvm_fpstate(&vcpu->arch.guest_fpu, true);
11101 static void kvm_put_guest_fpu(struct kvm_vcpu *vcpu)
11103 fpu_swap_kvm_fpstate(&vcpu->arch.guest_fpu, false);
11104 ++vcpu->stat.fpu_reload;
11108 int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu)
11110 struct kvm_queued_exception *ex = &vcpu->arch.exception;
11111 struct kvm_run *kvm_run = vcpu->run;
11114 vcpu_load(vcpu);
11115 kvm_sigset_activate(vcpu);
11117 kvm_load_guest_fpu(vcpu);
11119 kvm_vcpu_srcu_read_lock(vcpu);
11120 if (unlikely(vcpu->arch.mp_state == KVM_MP_STATE_UNINITIALIZED)) {
11136 kvm_vcpu_srcu_read_unlock(vcpu);
11137 kvm_vcpu_block(vcpu);
11138 kvm_vcpu_srcu_read_lock(vcpu);
11140 if (kvm_apic_accept_events(vcpu) < 0) {
11148 ++vcpu->stat.signal_exits;
11160 r = sync_regs(vcpu);
11166 if (!lapic_in_kernel(vcpu)) {
11167 if (kvm_set_cr8(vcpu, kvm_run->cr8) != 0) {
11177 if (vcpu->arch.exception_from_userspace && is_guest_mode(vcpu) &&
11178 kvm_x86_ops.nested_ops->is_exception_vmexit(vcpu, ex->vector,
11180 kvm_queue_exception_vmexit(vcpu, ex->vector,
11186 vcpu->arch.exception_from_userspace = false;
11188 if (unlikely(vcpu->arch.complete_userspace_io)) {
11189 int (*cui)(struct kvm_vcpu *) = vcpu->arch.complete_userspace_io;
11190 vcpu->arch.complete_userspace_io = NULL;
11191 r = cui(vcpu);
11195 WARN_ON_ONCE(vcpu->arch.pio.count);
11196 WARN_ON_ONCE(vcpu->mmio_needed);
11204 r = static_call(kvm_x86_vcpu_pre_run)(vcpu);
11208 r = vcpu_run(vcpu);
11211 kvm_put_guest_fpu(vcpu);
11213 store_regs(vcpu);
11214 post_kvm_run_save(vcpu);
11215 kvm_vcpu_srcu_read_unlock(vcpu);
11217 kvm_sigset_deactivate(vcpu);
11218 vcpu_put(vcpu);
11222 static void __get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
11224 if (vcpu->arch.emulate_regs_need_sync_to_vcpu) {
11228 * back from emulation context to vcpu. Userspace shouldn't do
11232 emulator_writeback_register_cache(vcpu->arch.emulate_ctxt);
11233 vcpu->arch.emulate_regs_need_sync_to_vcpu = false;
11235 regs->rax = kvm_rax_read(vcpu);
11236 regs->rbx = kvm_rbx_read(vcpu);
11237 regs->rcx = kvm_rcx_read(vcpu);
11238 regs->rdx = kvm_rdx_read(vcpu);
11239 regs->rsi = kvm_rsi_read(vcpu);
11240 regs->rdi = kvm_rdi_read(vcpu);
11241 regs->rsp = kvm_rsp_read(vcpu);
11242 regs->rbp = kvm_rbp_read(vcpu);
11244 regs->r8 = kvm_r8_read(vcpu);
11245 regs->r9 = kvm_r9_read(vcpu);
11246 regs->r10 = kvm_r10_read(vcpu);
11247 regs->r11 = kvm_r11_read(vcpu);
11248 regs->r12 = kvm_r12_read(vcpu);
11249 regs->r13 = kvm_r13_read(vcpu);
11250 regs->r14 = kvm_r14_read(vcpu);
11251 regs->r15 = kvm_r15_read(vcpu);
11254 regs->rip = kvm_rip_read(vcpu);
11255 regs->rflags = kvm_get_rflags(vcpu);
11258 int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
11260 vcpu_load(vcpu);
11261 __get_regs(vcpu, regs);
11262 vcpu_put(vcpu);
11266 static void __set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
11268 vcpu->arch.emulate_regs_need_sync_from_vcpu = true;
11269 vcpu->arch.emulate_regs_need_sync_to_vcpu = false;
11271 kvm_rax_write(vcpu, regs->rax);
11272 kvm_rbx_write(vcpu, regs->rbx);
11273 kvm_rcx_write(vcpu, regs->rcx);
11274 kvm_rdx_write(vcpu, regs->rdx);
11275 kvm_rsi_write(vcpu, regs->rsi);
11276 kvm_rdi_write(vcpu, regs->rdi);
11277 kvm_rsp_write(vcpu, regs->rsp);
11278 kvm_rbp_write(vcpu, regs->rbp);
11280 kvm_r8_write(vcpu, regs->r8);
11281 kvm_r9_write(vcpu, regs->r9);
11282 kvm_r10_write(vcpu, regs->r10);
11283 kvm_r11_write(vcpu, regs->r11);
11284 kvm_r12_write(vcpu, regs->r12);
11285 kvm_r13_write(vcpu, regs->r13);
11286 kvm_r14_write(vcpu, regs->r14);
11287 kvm_r15_write(vcpu, regs->r15);
11290 kvm_rip_write(vcpu, regs->rip);
11291 kvm_set_rflags(vcpu, regs->rflags | X86_EFLAGS_FIXED);
11293 vcpu->arch.exception.pending = false;
11294 vcpu->arch.exception_vmexit.pending = false;
11296 kvm_make_request(KVM_REQ_EVENT, vcpu);
11299 int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
11301 vcpu_load(vcpu);
11302 __set_regs(vcpu, regs);
11303 vcpu_put(vcpu);
11307 static void __get_sregs_common(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs)
11311 if (vcpu->arch.guest_state_protected)
11314 kvm_get_segment(vcpu, &sregs->cs, VCPU_SREG_CS);
11315 kvm_get_segment(vcpu, &sregs->ds, VCPU_SREG_DS);
11316 kvm_get_segment(vcpu, &sregs->es, VCPU_SREG_ES);
11317 kvm_get_segment(vcpu, &sregs->fs, VCPU_SREG_FS);
11318 kvm_get_segment(vcpu, &sregs->gs, VCPU_SREG_GS);
11319 kvm_get_segment(vcpu, &sregs->ss, VCPU_SREG_SS);
11321 kvm_get_segment(vcpu, &sregs->tr, VCPU_SREG_TR);
11322 kvm_get_segment(vcpu, &sregs->ldt, VCPU_SREG_LDTR);
11324 static_call(kvm_x86_get_idt)(vcpu, &dt);
11327 static_call(kvm_x86_get_gdt)(vcpu, &dt);
11331 sregs->cr2 = vcpu->arch.cr2;
11332 sregs->cr3 = kvm_read_cr3(vcpu);
11335 sregs->cr0 = kvm_read_cr0(vcpu);
11336 sregs->cr4 = kvm_read_cr4(vcpu);
11337 sregs->cr8 = kvm_get_cr8(vcpu);
11338 sregs->efer = vcpu->arch.efer;
11339 sregs->apic_base = kvm_get_apic_base(vcpu);
11342 static void __get_sregs(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs)
11344 __get_sregs_common(vcpu, sregs);
11346 if (vcpu->arch.guest_state_protected)
11349 if (vcpu->arch.interrupt.injected && !vcpu->arch.interrupt.soft)
11350 set_bit(vcpu->arch.interrupt.nr,
11354 static void __get_sregs2(struct kvm_vcpu *vcpu, struct kvm_sregs2 *sregs2)
11358 __get_sregs_common(vcpu, (struct kvm_sregs *)sregs2);
11360 if (vcpu->arch.guest_state_protected)
11363 if (is_pae_paging(vcpu)) {
11365 sregs2->pdptrs[i] = kvm_pdptr_read(vcpu, i);
11370 int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
11373 vcpu_load(vcpu);
11374 __get_sregs(vcpu, sregs);
11375 vcpu_put(vcpu);
11379 int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
11384 vcpu_load(vcpu);
11386 kvm_load_guest_fpu(vcpu);
11388 r = kvm_apic_accept_events(vcpu);
11393 if ((vcpu->arch.mp_state == KVM_MP_STATE_HALTED ||
11394 vcpu->arch.mp_state == KVM_MP_STATE_AP_RESET_HOLD) &&
11395 vcpu->arch.pv.pv_unhalted)
11398 mp_state->mp_state = vcpu->arch.mp_state;
11402 kvm_put_guest_fpu(vcpu);
11403 vcpu_put(vcpu);
11407 int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
11412 vcpu_load(vcpu);
11420 if (!lapic_in_kernel(vcpu))
11437 if ((!kvm_apic_init_sipi_allowed(vcpu) || vcpu->arch.smi_pending) &&
11443 vcpu->arch.mp_state = KVM_MP_STATE_INIT_RECEIVED;
11444 set_bit(KVM_APIC_SIPI, &vcpu->arch.apic->pending_events);
11446 vcpu->arch.mp_state = mp_state->mp_state;
11447 kvm_make_request(KVM_REQ_EVENT, vcpu);
11451 vcpu_put(vcpu);
11455 int kvm_task_switch(struct kvm_vcpu *vcpu, u16 tss_selector, int idt_index,
11458 struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt;
11461 init_emulate_ctxt(vcpu);
11466 vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
11467 vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_EMULATION;
11468 vcpu->run->internal.ndata = 0;
11472 kvm_rip_write(vcpu, ctxt->eip);
11473 kvm_set_rflags(vcpu, ctxt->eflags);
11478 static bool kvm_is_valid_sregs(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs)
11488 if (kvm_vcpu_is_illegal_gpa(vcpu, sregs->cr3))
11499 return kvm_is_valid_cr4(vcpu, sregs->cr4) &&
11500 kvm_is_valid_cr0(vcpu, sregs->cr0);
11503 static int __set_sregs_common(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs,
11510 if (!kvm_is_valid_sregs(vcpu, sregs))
11515 if (kvm_set_apic_base(vcpu, &apic_base_msr))
11518 if (vcpu->arch.guest_state_protected)
11523 static_call(kvm_x86_set_idt)(vcpu, &dt);
11526 static_call(kvm_x86_set_gdt)(vcpu, &dt);
11528 vcpu->arch.cr2 = sregs->cr2;
11529 *mmu_reset_needed |= kvm_read_cr3(vcpu) != sregs->cr3;
11530 vcpu->arch.cr3 = sregs->cr3;
11531 kvm_register_mark_dirty(vcpu, VCPU_EXREG_CR3);
11532 static_call_cond(kvm_x86_post_set_cr3)(vcpu, sregs->cr3);
11534 kvm_set_cr8(vcpu, sregs->cr8);
11536 *mmu_reset_needed |= vcpu->arch.efer != sregs->efer;
11537 static_call(kvm_x86_set_efer)(vcpu, sregs->efer);
11539 *mmu_reset_needed |= kvm_read_cr0(vcpu) != sregs->cr0;
11540 static_call(kvm_x86_set_cr0)(vcpu, sregs->cr0);
11541 vcpu->arch.cr0 = sregs->cr0;
11543 *mmu_reset_needed |= kvm_read_cr4(vcpu) != sregs->cr4;
11544 static_call(kvm_x86_set_cr4)(vcpu, sregs->cr4);
11547 idx = srcu_read_lock(&vcpu->kvm->srcu);
11548 if (is_pae_paging(vcpu)) {
11549 load_pdptrs(vcpu, kvm_read_cr3(vcpu));
11552 srcu_read_unlock(&vcpu->kvm->srcu, idx);
11555 kvm_set_segment(vcpu, &sregs->cs, VCPU_SREG_CS);
11556 kvm_set_segment(vcpu, &sregs->ds, VCPU_SREG_DS);
11557 kvm_set_segment(vcpu, &sregs->es, VCPU_SREG_ES);
11558 kvm_set_segment(vcpu, &sregs->fs, VCPU_SREG_FS);
11559 kvm_set_segment(vcpu, &sregs->gs, VCPU_SREG_GS);
11560 kvm_set_segment(vcpu, &sregs->ss, VCPU_SREG_SS);
11562 kvm_set_segment(vcpu, &sregs->tr, VCPU_SREG_TR);
11563 kvm_set_segment(vcpu, &sregs->ldt, VCPU_SREG_LDTR);
11565 update_cr8_intercept(vcpu);
11567 /* Older userspace won't unhalt the vcpu on reset. */
11568 if (kvm_vcpu_is_bsp(vcpu) && kvm_rip_read(vcpu) == 0xfff0 &&
11570 !is_protmode(vcpu))
11571 vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE;
11576 static int __set_sregs(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs)
11580 int ret = __set_sregs_common(vcpu, sregs, &mmu_reset_needed, true);
11586 kvm_mmu_reset_context(vcpu);
11593 kvm_queue_interrupt(vcpu, pending_vec, false);
11595 kvm_make_request(KVM_REQ_EVENT, vcpu);
11600 static int __set_sregs2(struct kvm_vcpu *vcpu, struct kvm_sregs2 *sregs2)
11611 if (valid_pdptrs && (!pae || vcpu->arch.guest_state_protected))
11614 ret = __set_sregs_common(vcpu, (struct kvm_sregs *)sregs2,
11621 kvm_pdptr_write(vcpu, i, sregs2->pdptrs[i]);
11623 kvm_register_mark_dirty(vcpu, VCPU_EXREG_PDPTR);
11625 vcpu->arch.pdptrs_from_userspace = true;
11628 kvm_mmu_reset_context(vcpu);
11632 int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
11637 vcpu_load(vcpu);
11638 ret = __set_sregs(vcpu, sregs);
11639 vcpu_put(vcpu);
11646 struct kvm_vcpu *vcpu;
11654 kvm_for_each_vcpu(i, vcpu, kvm) {
11655 if (vcpu->guest_debug & KVM_GUESTDBG_BLOCKIRQ) {
11664 int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
11670 if (vcpu->arch.guest_state_protected)
11673 vcpu_load(vcpu);
11677 if (kvm_is_exception_pending(vcpu))
11680 kvm_queue_exception(vcpu, DB_VECTOR);
11682 kvm_queue_exception(vcpu, BP_VECTOR);
11689 rflags = kvm_get_rflags(vcpu);
11691 vcpu->guest_debug = dbg->control;
11692 if (!(vcpu->guest_debug & KVM_GUESTDBG_ENABLE))
11693 vcpu->guest_debug = 0;
11695 if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) {
11697 vcpu->arch.eff_db[i] = dbg->arch.debugreg[i];
11698 vcpu->arch.guest_debug_dr7 = dbg->arch.debugreg[7];
11701 vcpu->arch.eff_db[i] = vcpu->arch.db[i];
11703 kvm_update_dr7(vcpu);
11705 if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP)
11706 vcpu->arch.singlestep_rip = kvm_get_linear_rip(vcpu);
11712 kvm_set_rflags(vcpu, rflags);
11714 static_call(kvm_x86_update_exception_bitmap)(vcpu);
11716 kvm_arch_vcpu_guestdbg_update_apicv_inhibit(vcpu->kvm);
11721 vcpu_put(vcpu);
11728 int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,
11735 vcpu_load(vcpu);
11737 idx = srcu_read_lock(&vcpu->kvm->srcu);
11738 gpa = kvm_mmu_gva_to_gpa_system(vcpu, vaddr, NULL);
11739 srcu_read_unlock(&vcpu->kvm->srcu, idx);
11745 vcpu_put(vcpu);
11749 int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
11753 if (fpstate_is_confidential(&vcpu->arch.guest_fpu))
11756 vcpu_load(vcpu);
11758 fxsave = &vcpu->arch.guest_fpu.fpstate->regs.fxsave;
11768 vcpu_put(vcpu);
11772 int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
11776 if (fpstate_is_confidential(&vcpu->arch.guest_fpu))
11779 vcpu_load(vcpu);
11781 fxsave = &vcpu->arch.guest_fpu.fpstate->regs.fxsave;
11792 vcpu_put(vcpu);
11796 static void store_regs(struct kvm_vcpu *vcpu)
11800 if (vcpu->run->kvm_valid_regs & KVM_SYNC_X86_REGS)
11801 __get_regs(vcpu, &vcpu->run->s.regs.regs);
11803 if (vcpu->run->kvm_valid_regs & KVM_SYNC_X86_SREGS)
11804 __get_sregs(vcpu, &vcpu->run->s.regs.sregs);
11806 if (vcpu->run->kvm_valid_regs & KVM_SYNC_X86_EVENTS)
11808 vcpu, &vcpu->run->s.regs.events);
11811 static int sync_regs(struct kvm_vcpu *vcpu)
11813 if (vcpu->run->kvm_dirty_regs & KVM_SYNC_X86_REGS) {
11814 __set_regs(vcpu, &vcpu->run->s.regs.regs);
11815 vcpu->run->kvm_dirty_regs &= ~KVM_SYNC_X86_REGS;
11818 if (vcpu->run->kvm_dirty_regs & KVM_SYNC_X86_SREGS) {
11819 struct kvm_sregs sregs = vcpu->run->s.regs.sregs;
11821 if (__set_sregs(vcpu, &sregs))
11824 vcpu->run->kvm_dirty_regs &= ~KVM_SYNC_X86_SREGS;
11827 if (vcpu->run->kvm_dirty_regs & KVM_SYNC_X86_EVENTS) {
11828 struct kvm_vcpu_events events = vcpu->run->s.regs.events;
11830 if (kvm_vcpu_ioctl_x86_set_vcpu_events(vcpu, &events))
11833 vcpu->run->kvm_dirty_regs &= ~KVM_SYNC_X86_EVENTS;
11854 int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu)
11859 vcpu->arch.last_vmentry_cpu = -1;
11860 vcpu->arch.regs_avail = ~0;
11861 vcpu->arch.regs_dirty = ~0;
11863 kvm_gpc_init(&vcpu->arch.pv_time, vcpu->kvm, vcpu, KVM_HOST_USES_PFN);
11865 if (!irqchip_in_kernel(vcpu->kvm) || kvm_vcpu_is_reset_bsp(vcpu))
11866 vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE;
11868 vcpu->arch.mp_state = KVM_MP_STATE_UNINITIALIZED;
11870 r = kvm_mmu_create(vcpu);
11874 if (irqchip_in_kernel(vcpu->kvm)) {
11875 r = kvm_create_lapic(vcpu, lapic_timer_advance_ns);
11890 vcpu->arch.apic->apicv_active = true;
11891 kvm_make_request(KVM_REQ_APICV_UPDATE, vcpu);
11901 vcpu->arch.pio_data = page_address(page);
11903 vcpu->arch.mce_banks = kcalloc(KVM_MAX_MCE_BANKS * 4, sizeof(u64),
11905 vcpu->arch.mci_ctl2_banks = kcalloc(KVM_MAX_MCE_BANKS, sizeof(u64),
11907 if (!vcpu->arch.mce_banks || !vcpu->arch.mci_ctl2_banks)
11909 vcpu->arch.mcg_cap = KVM_MAX_MCE_BANKS;
11911 if (!zalloc_cpumask_var(&vcpu->arch.wbinvd_dirty_mask,
11915 if (!alloc_emulate_ctxt(vcpu))
11918 if (!fpu_alloc_guest_fpstate(&vcpu->arch.guest_fpu)) {
11919 pr_err("failed to allocate vcpu's fpu\n");
11923 vcpu->arch.maxphyaddr = cpuid_query_maxphyaddr(vcpu);
11924 vcpu->arch.reserved_gpa_bits = kvm_vcpu_reserved_gpa_bits_raw(vcpu);
11926 vcpu->arch.pat = MSR_IA32_CR_PAT_DEFAULT;
11928 kvm_async_pf_hash_reset(vcpu);
11930 vcpu->arch.perf_capabilities = kvm_caps.supported_perf_cap;
11931 kvm_pmu_init(vcpu);
11933 vcpu->arch.pending_external_vector = -1;
11934 vcpu->arch.preempted_in_kernel = false;
11937 vcpu->arch.hv_root_tdp = INVALID_PAGE;
11940 r = static_call(kvm_x86_vcpu_create)(vcpu);
11944 vcpu->arch.arch_capabilities = kvm_get_arch_capabilities();
11945 vcpu->arch.msr_platform_info = MSR_PLATFORM_INFO_CPUID_FAULT;
11946 kvm_xen_init_vcpu(vcpu);
11947 kvm_vcpu_mtrr_init(vcpu);
11948 vcpu_load(vcpu);
11949 kvm_set_tsc_khz(vcpu, vcpu->kvm->arch.default_tsc_khz);
11950 kvm_vcpu_reset(vcpu, false);
11951 kvm_init_mmu(vcpu);
11952 vcpu_put(vcpu);
11956 fpu_free_guest_fpstate(&vcpu->arch.guest_fpu);
11958 kmem_cache_free(x86_emulator_cache, vcpu->arch.emulate_ctxt);
11960 free_cpumask_var(vcpu->arch.wbinvd_dirty_mask);
11962 kfree(vcpu->arch.mce_banks);
11963 kfree(vcpu->arch.mci_ctl2_banks);
11964 free_page((unsigned long)vcpu->arch.pio_data);
11966 kvm_free_lapic(vcpu);
11968 kvm_mmu_destroy(vcpu);
11972 void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)
11974 struct kvm *kvm = vcpu->kvm;
11976 if (mutex_lock_killable(&vcpu->mutex))
11978 vcpu_load(vcpu);
11979 kvm_synchronize_tsc(vcpu, 0);
11980 vcpu_put(vcpu);
11983 vcpu->arch.msr_kvm_poll_control = 1;
11985 mutex_unlock(&vcpu->mutex);
11987 if (kvmclock_periodic_sync && vcpu->vcpu_idx == 0)
11992 void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
11996 kvmclock_reset(vcpu);
11998 static_call(kvm_x86_vcpu_free)(vcpu);
12000 kmem_cache_free(x86_emulator_cache, vcpu->arch.emulate_ctxt);
12001 free_cpumask_var(vcpu->arch.wbinvd_dirty_mask);
12002 fpu_free_guest_fpstate(&vcpu->arch.guest_fpu);
12004 kvm_xen_destroy_vcpu(vcpu);
12005 kvm_hv_vcpu_uninit(vcpu);
12006 kvm_pmu_destroy(vcpu);
12007 kfree(vcpu->arch.mce_banks);
12008 kfree(vcpu->arch.mci_ctl2_banks);
12009 kvm_free_lapic(vcpu);
12010 idx = srcu_read_lock(&vcpu->kvm->srcu);
12011 kvm_mmu_destroy(vcpu);
12012 srcu_read_unlock(&vcpu->kvm->srcu, idx);
12013 free_page((unsigned long)vcpu->arch.pio_data);
12014 kvfree(vcpu->arch.cpuid_entries);
12015 if (!lapic_in_kernel(vcpu))
12019 void kvm_vcpu_reset(struct kvm_vcpu *vcpu, bool init_event)
12022 unsigned long old_cr0 = kvm_read_cr0(vcpu);
12033 (old_cr0 || kvm_read_cr3(vcpu) || kvm_read_cr4(vcpu)));
12041 if (is_guest_mode(vcpu))
12042 kvm_leave_nested(vcpu);
12044 kvm_lapic_reset(vcpu, init_event);
12046 WARN_ON_ONCE(is_guest_mode(vcpu) || is_smm(vcpu));
12047 vcpu->arch.hflags = 0;
12049 vcpu->arch.smi_pending = 0;
12050 vcpu->arch.smi_count = 0;
12051 atomic_set(&vcpu->arch.nmi_queued, 0);
12052 vcpu->arch.nmi_pending = 0;
12053 vcpu->arch.nmi_injected = false;
12054 kvm_clear_interrupt_queue(vcpu);
12055 kvm_clear_exception_queue(vcpu);
12057 memset(vcpu->arch.db, 0, sizeof(vcpu->arch.db));
12058 kvm_update_dr0123(vcpu);
12059 vcpu->arch.dr6 = DR6_ACTIVE_LOW;
12060 vcpu->arch.dr7 = DR7_FIXED_1;
12061 kvm_update_dr7(vcpu);
12063 vcpu->arch.cr2 = 0;
12065 kvm_make_request(KVM_REQ_EVENT, vcpu);
12066 vcpu->arch.apf.msr_en_val = 0;
12067 vcpu->arch.apf.msr_int_val = 0;
12068 vcpu->arch.st.msr_val = 0;
12070 kvmclock_reset(vcpu);
12072 kvm_clear_async_pf_completion_queue(vcpu);
12073 kvm_async_pf_hash_reset(vcpu);
12074 vcpu->arch.apf.halted = false;
12076 if (vcpu->arch.guest_fpu.fpstate && kvm_mpx_supported()) {
12077 struct fpstate *fpstate = vcpu->arch.guest_fpu.fpstate;
12084 kvm_put_guest_fpu(vcpu);
12090 kvm_load_guest_fpu(vcpu);
12094 kvm_pmu_reset(vcpu);
12095 vcpu->arch.smbase = 0x30000;
12097 vcpu->arch.msr_misc_features_enables = 0;
12098 vcpu->arch.ia32_misc_enable_msr = MSR_IA32_MISC_ENABLE_PEBS_UNAVAIL |
12101 __kvm_set_xcr(vcpu, 0, XFEATURE_MASK_FP);
12102 __kvm_set_msr(vcpu, MSR_IA32_XSS, 0, true);
12106 memset(vcpu->arch.regs, 0, sizeof(vcpu->arch.regs));
12107 kvm_register_mark_dirty(vcpu, VCPU_REGS_RSP);
12116 cpuid_0x1 = kvm_find_cpuid_entry(vcpu, 1);
12117 kvm_rdx_write(vcpu, cpuid_0x1 ? cpuid_0x1->eax : 0x600);
12119 static_call(kvm_x86_vcpu_reset)(vcpu, init_event);
12121 kvm_set_rflags(vcpu, X86_EFLAGS_FIXED);
12122 kvm_rip_write(vcpu, 0xfff0);
12124 vcpu->arch.cr3 = 0;
12125 kvm_register_mark_dirty(vcpu, VCPU_EXREG_CR3);
12138 static_call(kvm_x86_set_cr0)(vcpu, new_cr0);
12139 static_call(kvm_x86_set_cr4)(vcpu, 0);
12140 static_call(kvm_x86_set_efer)(vcpu, 0);
12141 static_call(kvm_x86_update_exception_bitmap)(vcpu);
12152 kvm_make_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu);
12153 kvm_mmu_reset_context(vcpu);
12166 kvm_make_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu);
12170 void kvm_vcpu_deliver_sipi_vector(struct kvm_vcpu *vcpu, u8 vector)
12174 kvm_get_segment(vcpu, &cs, VCPU_SREG_CS);
12177 kvm_set_segment(vcpu, &cs, VCPU_SREG_CS);
12178 kvm_rip_write(vcpu, 0);
12185 struct kvm_vcpu *vcpu;
12205 kvm_for_each_vcpu(i, vcpu, kvm) {
12206 if (!stable && vcpu->cpu == smp_processor_id())
12207 kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);
12208 if (stable && vcpu->arch.last_host_tsc > local_tsc) {
12210 if (vcpu->arch.last_host_tsc > max_tsc)
12211 max_tsc = vcpu->arch.last_host_tsc;
12258 kvm_for_each_vcpu(i, vcpu, kvm) {
12259 vcpu->arch.tsc_offset_adjustment += delta_cyc;
12260 vcpu->arch.last_host_tsc = local_tsc;
12261 kvm_make_request(KVM_REQ_MASTERCLOCK_UPDATE, vcpu);
12284 bool kvm_vcpu_is_reset_bsp(struct kvm_vcpu *vcpu)
12286 return vcpu->kvm->arch.bsp_vcpu_id == vcpu->vcpu_id;
12289 bool kvm_vcpu_is_bsp(struct kvm_vcpu *vcpu)
12291 return (vcpu->arch.apic_base & MSR_IA32_APICBASE_BSP) != 0;
12297 void kvm_arch_sched_in(struct kvm_vcpu *vcpu, int cpu)
12299 struct kvm_pmu *pmu = vcpu_to_pmu(vcpu);
12301 vcpu->arch.l1tf_flush_l1d = true;
12304 kvm_make_request(KVM_REQ_PMU, vcpu);
12306 static_call(kvm_x86_sched_in)(vcpu, cpu);
12383 static void kvm_unload_vcpu_mmu(struct kvm_vcpu *vcpu)
12385 vcpu_load(vcpu);
12386 kvm_mmu_unload(vcpu);
12387 vcpu_put(vcpu);
12393 struct kvm_vcpu *vcpu;
12395 kvm_for_each_vcpu(i, vcpu, kvm) {
12396 kvm_clear_async_pf_completion_queue(vcpu);
12397 kvm_unload_vcpu_mmu(vcpu);
12631 struct kvm_vcpu *vcpu;
12641 kvm_for_each_vcpu(i, vcpu, kvm)
12642 kvm_vcpu_kick(vcpu);
12830 static inline bool kvm_guest_apic_has_interrupt(struct kvm_vcpu *vcpu)
12832 return (is_guest_mode(vcpu) &&
12833 static_call(kvm_x86_guest_apic_has_interrupt)(vcpu));
12836 static inline bool kvm_vcpu_has_events(struct kvm_vcpu *vcpu)
12838 if (!list_empty_careful(&vcpu->async_pf.done))
12841 if (kvm_apic_has_pending_init_or_sipi(vcpu) &&
12842 kvm_apic_init_sipi_allowed(vcpu))
12845 if (vcpu->arch.pv.pv_unhalted)
12848 if (kvm_is_exception_pending(vcpu))
12851 if (kvm_test_request(KVM_REQ_NMI, vcpu) ||
12852 (vcpu->arch.nmi_pending &&
12853 static_call(kvm_x86_nmi_allowed)(vcpu, false)))
12857 if (kvm_test_request(KVM_REQ_SMI, vcpu) ||
12858 (vcpu->arch.smi_pending &&
12859 static_call(kvm_x86_smi_allowed)(vcpu, false)))
12863 if (kvm_test_request(KVM_REQ_PMI, vcpu))
12866 if (kvm_arch_interrupt_allowed(vcpu) &&
12867 (kvm_cpu_has_interrupt(vcpu) ||
12868 kvm_guest_apic_has_interrupt(vcpu)))
12871 if (kvm_hv_has_stimer_pending(vcpu))
12874 if (is_guest_mode(vcpu) &&
12876 kvm_x86_ops.nested_ops->has_events(vcpu))
12879 if (kvm_xen_has_pending_events(vcpu))
12885 int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu)
12887 return kvm_vcpu_running(vcpu) || kvm_vcpu_has_events(vcpu);
12890 bool kvm_arch_dy_has_pending_interrupt(struct kvm_vcpu *vcpu)
12892 if (kvm_vcpu_apicv_active(vcpu) &&
12893 static_call(kvm_x86_dy_apicv_has_pending_interrupt)(vcpu))
12899 bool kvm_arch_dy_runnable(struct kvm_vcpu *vcpu)
12901 if (READ_ONCE(vcpu->arch.pv.pv_unhalted))
12904 if (kvm_test_request(KVM_REQ_NMI, vcpu) ||
12906 kvm_test_request(KVM_REQ_SMI, vcpu) ||
12908 kvm_test_request(KVM_REQ_EVENT, vcpu))
12911 return kvm_arch_dy_has_pending_interrupt(vcpu);
12914 bool kvm_arch_vcpu_in_kernel(struct kvm_vcpu *vcpu)
12916 if (vcpu->arch.guest_state_protected)
12919 return vcpu->arch.preempted_in_kernel;
12922 unsigned long kvm_arch_vcpu_get_ip(struct kvm_vcpu *vcpu)
12924 return kvm_rip_read(vcpu);
12927 int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu)
12929 return kvm_vcpu_exiting_guest_mode(vcpu) == IN_GUEST_MODE;
12932 int kvm_arch_interrupt_allowed(struct kvm_vcpu *vcpu)
12934 return static_call(kvm_x86_interrupt_allowed)(vcpu, false);
12937 unsigned long kvm_get_linear_rip(struct kvm_vcpu *vcpu)
12940 if (vcpu->arch.guest_state_protected)
12943 if (is_64_bit_mode(vcpu))
12944 return kvm_rip_read(vcpu);
12945 return (u32)(get_segment_base(vcpu, VCPU_SREG_CS) +
12946 kvm_rip_read(vcpu));
12950 bool kvm_is_linear_rip(struct kvm_vcpu *vcpu, unsigned long linear_rip)
12952 return kvm_get_linear_rip(vcpu) == linear_rip;
12956 unsigned long kvm_get_rflags(struct kvm_vcpu *vcpu)
12960 rflags = static_call(kvm_x86_get_rflags)(vcpu);
12961 if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP)
12967 static void __kvm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags)
12969 if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP &&
12970 kvm_is_linear_rip(vcpu, vcpu->arch.singlestep_rip))
12972 static_call(kvm_x86_set_rflags)(vcpu, rflags);
12975 void kvm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags)
12977 __kvm_set_rflags(vcpu, rflags);
12978 kvm_make_request(KVM_REQ_EVENT, vcpu);
12994 static void kvm_add_async_pf_gfn(struct kvm_vcpu *vcpu, gfn_t gfn)
12998 while (vcpu->arch.apf.gfns[key] != ~0)
13001 vcpu->arch.apf.gfns[key] = gfn;
13004 static u32 kvm_async_pf_gfn_slot(struct kvm_vcpu *vcpu, gfn_t gfn)
13010 (vcpu->arch.apf.gfns[key] != gfn &&
13011 vcpu->arch.apf.gfns[key] != ~0); i++)
13017 bool kvm_find_async_pf_gfn(struct kvm_vcpu *vcpu, gfn_t gfn)
13019 return vcpu->arch.apf.gfns[kvm_async_pf_gfn_slot(vcpu, gfn)] == gfn;
13022 static void kvm_del_async_pf_gfn(struct kvm_vcpu *vcpu, gfn_t gfn)
13026 i = j = kvm_async_pf_gfn_slot(vcpu, gfn);
13028 if (WARN_ON_ONCE(vcpu->arch.apf.gfns[i] != gfn))
13032 vcpu->arch.apf.gfns[i] = ~0;
13035 if (vcpu->arch.apf.gfns[j] == ~0)
13037 k = kvm_async_pf_hash_fn(vcpu->arch.apf.gfns[j]);
13044 vcpu->arch.apf.gfns[i] = vcpu->arch.apf.gfns[j];
13049 static inline int apf_put_user_notpresent(struct kvm_vcpu *vcpu)
13053 return kvm_write_guest_cached(vcpu->kvm, &vcpu->arch.apf.data, &reason,
13057 static inline int apf_put_user_ready(struct kvm_vcpu *vcpu, u32 token)
13061 return kvm_write_guest_offset_cached(vcpu->kvm, &vcpu->arch.apf.data,
13065 static inline bool apf_pageready_slot_free(struct kvm_vcpu *vcpu)
13070 if (kvm_read_guest_offset_cached(vcpu->kvm, &vcpu->arch.apf.data,
13077 static bool kvm_can_deliver_async_pf(struct kvm_vcpu *vcpu)
13080 if (!kvm_pv_async_pf_enabled(vcpu))
13083 if (vcpu->arch.apf.send_user_only &&
13084 static_call(kvm_x86_get_cpl)(vcpu) == 0)
13087 if (is_guest_mode(vcpu)) {
13092 return vcpu->arch.apf.delivery_as_pf_vmexit;
13099 return is_paging(vcpu);
13103 bool kvm_can_do_async_pf(struct kvm_vcpu *vcpu)
13105 if (unlikely(!lapic_in_kernel(vcpu) ||
13106 kvm_event_needs_reinjection(vcpu) ||
13107 kvm_is_exception_pending(vcpu)))
13110 if (kvm_hlt_in_guest(vcpu->kvm) && !kvm_can_deliver_async_pf(vcpu))
13117 return kvm_arch_interrupt_allowed(vcpu);
13120 bool kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu,
13126 kvm_add_async_pf_gfn(vcpu, work->arch.gfn);
13128 if (kvm_can_deliver_async_pf(vcpu) &&
13129 !apf_put_user_notpresent(vcpu)) {
13136 kvm_inject_page_fault(vcpu, &fault);
13147 kvm_make_request(KVM_REQ_APF_HALT, vcpu);
13152 void kvm_arch_async_page_present(struct kvm_vcpu *vcpu,
13157 .vector = vcpu->arch.apf.vec
13163 kvm_del_async_pf_gfn(vcpu, work->arch.gfn);
13167 kvm_pv_async_pf_enabled(vcpu) &&
13168 !apf_put_user_ready(vcpu, work->arch.token)) {
13169 vcpu->arch.apf.pageready_pending = true;
13170 kvm_apic_set_irq(vcpu, &irq, NULL);
13173 vcpu->arch.apf.halted = false;
13174 vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE;
13177 void kvm_arch_async_page_present_queued(struct kvm_vcpu *vcpu)
13179 kvm_make_request(KVM_REQ_APF_READY, vcpu);
13180 if (!vcpu->arch.apf.pageready_pending)
13181 kvm_vcpu_kick(vcpu);
13184 bool kvm_arch_can_dequeue_async_page_present(struct kvm_vcpu *vcpu)
13186 if (!kvm_pv_async_pf_enabled(vcpu))
13189 return kvm_lapic_enabled(vcpu) && apf_pageready_slot_free(vcpu);
13296 bool kvm_arch_no_poll(struct kvm_vcpu *vcpu)
13298 return (vcpu->arch.msr_kvm_poll_control & 1) == 0;
13329 void kvm_fixup_and_inject_pf_error(struct kvm_vcpu *vcpu, gva_t gva, u16 error_code)
13331 struct kvm_mmu *mmu = vcpu->arch.walk_mmu;
13337 mmu->gva_to_gpa(vcpu, mmu, gva, access, &fault) != INVALID_GPA) {
13339 * If vcpu->arch.walk_mmu->gva_to_gpa succeeded, the page
13350 vcpu->arch.walk_mmu->inject_page_fault(vcpu, &fault);
13359 int kvm_handle_memory_failure(struct kvm_vcpu *vcpu, int r,
13363 if (KVM_BUG_ON(!e, vcpu->kvm))
13366 kvm_inject_emulated_page_fault(vcpu, e);
13377 kvm_prepare_emulation_failure_exit(vcpu);
13383 int kvm_handle_invpcid(struct kvm_vcpu *vcpu, unsigned long type, gva_t gva)
13393 r = kvm_read_guest_virt(vcpu, gva, &operand, sizeof(operand), &e);
13395 return kvm_handle_memory_failure(vcpu, r, &e);
13398 kvm_inject_gp(vcpu, 0);
13402 pcid_enabled = kvm_is_cr4_bit_set(vcpu, X86_CR4_PCIDE);
13407 is_noncanonical_address(operand.gla, vcpu)) {
13408 kvm_inject_gp(vcpu, 0);
13411 kvm_mmu_invpcid_gva(vcpu, operand.gla, operand.pcid);
13412 return kvm_skip_emulated_instruction(vcpu);
13416 kvm_inject_gp(vcpu, 0);
13420 kvm_invalidate_pcid(vcpu, operand.pcid);
13421 return kvm_skip_emulated_instruction(vcpu);
13433 kvm_make_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu);
13434 return kvm_skip_emulated_instruction(vcpu);
13437 kvm_inject_gp(vcpu, 0);
13443 static int complete_sev_es_emulated_mmio(struct kvm_vcpu *vcpu)
13445 struct kvm_run *run = vcpu->run;
13449 BUG_ON(!vcpu->mmio_needed);
13452 frag = &vcpu->mmio_fragments[vcpu->mmio_cur_fragment];
13454 if (!vcpu->mmio_is_write)
13460 vcpu->mmio_cur_fragment++;
13468 if (vcpu->mmio_cur_fragment >= vcpu->mmio_nr_fragments) {
13469 vcpu->mmio_needed = 0;
13479 run->mmio.is_write = vcpu->mmio_is_write;
13484 vcpu->arch.complete_userspace_io = complete_sev_es_emulated_mmio;
13489 int kvm_sev_es_mmio_write(struct kvm_vcpu *vcpu, gpa_t gpa, unsigned int bytes,
13498 handled = write_emultor.read_write_mmio(vcpu, gpa, bytes, data);
13507 frag = vcpu->mmio_fragments;
13508 vcpu->mmio_nr_fragments = 1;
13513 vcpu->mmio_needed = 1;
13514 vcpu->mmio_cur_fragment = 0;
13516 vcpu->run->mmio.phys_addr = gpa;
13517 vcpu->run->mmio.len = min(8u, frag->len);
13518 vcpu->run->mmio.is_write = 1;
13519 memcpy(vcpu->run->mmio.data, frag->data, min(8u, frag->len));
13520 vcpu->run->exit_reason = KVM_EXIT_MMIO;
13522 vcpu->arch.complete_userspace_io = complete_sev_es_emulated_mmio;
13528 int kvm_sev_es_mmio_read(struct kvm_vcpu *vcpu, gpa_t gpa, unsigned int bytes,
13537 handled = read_emultor.read_write_mmio(vcpu, gpa, bytes, data);
13546 frag = vcpu->mmio_fragments;
13547 vcpu->mmio_nr_fragments = 1;
13552 vcpu->mmio_needed = 1;
13553 vcpu->mmio_cur_fragment = 0;
13555 vcpu->run->mmio.phys_addr = gpa;
13556 vcpu->run->mmio.len = min(8u, frag->len);
13557 vcpu->run->mmio.is_write = 0;
13558 vcpu->run->exit_reason = KVM_EXIT_MMIO;
13560 vcpu->arch.complete_userspace_io = complete_sev_es_emulated_mmio;
13566 static void advance_sev_es_emulated_pio(struct kvm_vcpu *vcpu, unsigned count, int size)
13568 vcpu->arch.sev_pio_count -= count;
13569 vcpu->arch.sev_pio_data += count * size;
13572 static int kvm_sev_es_outs(struct kvm_vcpu *vcpu, unsigned int size,
13575 static int complete_sev_es_emulated_outs(struct kvm_vcpu *vcpu)
13577 int size = vcpu->arch.pio.size;
13578 int port = vcpu->arch.pio.port;
13580 vcpu->arch.pio.count = 0;
13581 if (vcpu->arch.sev_pio_count)
13582 return kvm_sev_es_outs(vcpu, size, port);
13586 static int kvm_sev_es_outs(struct kvm_vcpu *vcpu, unsigned int size,
13591 min_t(unsigned int, PAGE_SIZE / size, vcpu->arch.sev_pio_count);
13592 int ret = emulator_pio_out(vcpu, size, port, vcpu->arch.sev_pio_data, count);
13595 advance_sev_es_emulated_pio(vcpu, count, size);
13600 if (!vcpu->arch.sev_pio_count)
13604 vcpu->arch.complete_userspace_io = complete_sev_es_emulated_outs;
13608 static int kvm_sev_es_ins(struct kvm_vcpu *vcpu, unsigned int size,
13611 static int complete_sev_es_emulated_ins(struct kvm_vcpu *vcpu)
13613 unsigned count = vcpu->arch.pio.count;
13614 int size = vcpu->arch.pio.size;
13615 int port = vcpu->arch.pio.port;
13617 complete_emulator_pio_in(vcpu, vcpu->arch.sev_pio_data);
13618 advance_sev_es_emulated_pio(vcpu, count, size);
13619 if (vcpu->arch.sev_pio_count)
13620 return kvm_sev_es_ins(vcpu, size, port);
13624 static int kvm_sev_es_ins(struct kvm_vcpu *vcpu, unsigned int size,
13629 min_t(unsigned int, PAGE_SIZE / size, vcpu->arch.sev_pio_count);
13630 if (!emulator_pio_in(vcpu, size, port, vcpu->arch.sev_pio_data, count))
13634 advance_sev_es_emulated_pio(vcpu, count, size);
13635 if (!vcpu->arch.sev_pio_count)
13639 vcpu->arch.complete_userspace_io = complete_sev_es_emulated_ins;
13643 int kvm_sev_es_string_io(struct kvm_vcpu *vcpu, unsigned int size,
13647 vcpu->arch.sev_pio_data = data;
13648 vcpu->arch.sev_pio_count = count;
13649 return in ? kvm_sev_es_ins(vcpu, size, port)
13650 : kvm_sev_es_outs(vcpu, size, port);