Lines Matching refs:vcpu
88 ((struct kvm_vcpu *)(ctxt)->vcpu)
106 static void update_cr8_intercept(struct kvm_vcpu *vcpu);
107 static void process_nmi(struct kvm_vcpu *vcpu);
108 static void process_smi(struct kvm_vcpu *vcpu);
109 static void enter_smm(struct kvm_vcpu *vcpu);
110 static void __kvm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags);
111 static void store_regs(struct kvm_vcpu *vcpu);
112 static int sync_regs(struct kvm_vcpu *vcpu);
262 static bool kvm_msr_ignored_check(struct kvm_vcpu *vcpu, u32 msr,
293 static inline void kvm_async_pf_hash_reset(struct kvm_vcpu *vcpu)
297 vcpu->arch.apf.gfns[i] = ~0;
398 u64 kvm_get_apic_base(struct kvm_vcpu *vcpu)
400 return vcpu->arch.apic_base;
404 enum lapic_mode kvm_get_apic_mode(struct kvm_vcpu *vcpu)
406 return kvm_apic_mode(kvm_get_apic_base(vcpu));
410 int kvm_set_apic_base(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
412 enum lapic_mode old_mode = kvm_get_apic_mode(vcpu);
414 u64 reserved_bits = ((~0ULL) << cpuid_maxphyaddr(vcpu)) | 0x2ff |
415 (guest_cpuid_has(vcpu, X86_FEATURE_X2APIC) ? 0 : X2APIC_ENABLE);
426 kvm_lapic_set_base(vcpu, msr_info->data);
427 kvm_recalculate_apic_map(vcpu->kvm);
492 void kvm_deliver_exception_payload(struct kvm_vcpu *vcpu)
494 unsigned nr = vcpu->arch.exception.nr;
495 bool has_payload = vcpu->arch.exception.has_payload;
496 unsigned long payload = vcpu->arch.exception.payload;
508 vcpu->arch.dr6 &= ~DR_TRAP_BITS;
512 vcpu->arch.dr6 |= DR6_RTM;
513 vcpu->arch.dr6 |= payload;
522 vcpu->arch.dr6 ^= payload & DR6_RTM;
530 vcpu->arch.dr6 &= ~BIT(12);
533 vcpu->arch.cr2 = payload;
537 vcpu->arch.exception.has_payload = false;
538 vcpu->arch.exception.payload = 0;
542 static void kvm_multiple_exception(struct kvm_vcpu *vcpu,
549 kvm_make_request(KVM_REQ_EVENT, vcpu);
551 if (!vcpu->arch.exception.pending && !vcpu->arch.exception.injected) {
555 * On vmentry, vcpu->arch.exception.pending is only
562 WARN_ON_ONCE(vcpu->arch.exception.pending);
563 vcpu->arch.exception.injected = true;
573 vcpu->arch.exception.pending = true;
574 vcpu->arch.exception.injected = false;
576 vcpu->arch.exception.has_error_code = has_error;
577 vcpu->arch.exception.nr = nr;
578 vcpu->arch.exception.error_code = error_code;
579 vcpu->arch.exception.has_payload = has_payload;
580 vcpu->arch.exception.payload = payload;
581 if (!is_guest_mode(vcpu))
582 kvm_deliver_exception_payload(vcpu);
587 prev_nr = vcpu->arch.exception.nr;
590 kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);
602 vcpu->arch.exception.pending = true;
603 vcpu->arch.exception.injected = false;
604 vcpu->arch.exception.has_error_code = true;
605 vcpu->arch.exception.nr = DF_VECTOR;
606 vcpu->arch.exception.error_code = 0;
607 vcpu->arch.exception.has_payload = false;
608 vcpu->arch.exception.payload = 0;
616 void kvm_queue_exception(struct kvm_vcpu *vcpu, unsigned nr)
618 kvm_multiple_exception(vcpu, nr, false, 0, false, 0, false);
622 void kvm_requeue_exception(struct kvm_vcpu *vcpu, unsigned nr)
624 kvm_multiple_exception(vcpu, nr, false, 0, false, 0, true);
628 void kvm_queue_exception_p(struct kvm_vcpu *vcpu, unsigned nr,
631 kvm_multiple_exception(vcpu, nr, false, 0, true, payload, false);
635 static void kvm_queue_exception_e_p(struct kvm_vcpu *vcpu, unsigned nr,
638 kvm_multiple_exception(vcpu, nr, true, error_code,
642 int kvm_complete_insn_gp(struct kvm_vcpu *vcpu, int err)
645 kvm_inject_gp(vcpu, 0);
647 return kvm_skip_emulated_instruction(vcpu);
653 void kvm_inject_page_fault(struct kvm_vcpu *vcpu, struct x86_exception *fault)
655 ++vcpu->stat.pf_guest;
656 vcpu->arch.exception.nested_apf =
657 is_guest_mode(vcpu) && fault->async_page_fault;
658 if (vcpu->arch.exception.nested_apf) {
659 vcpu->arch.apf.nested_apf_token = fault->address;
660 kvm_queue_exception_e(vcpu, PF_VECTOR, fault->error_code);
662 kvm_queue_exception_e_p(vcpu, PF_VECTOR, fault->error_code,
668 bool kvm_inject_emulated_page_fault(struct kvm_vcpu *vcpu,
674 fault_mmu = fault->nested_page_fault ? vcpu->arch.mmu :
675 vcpu->arch.walk_mmu;
683 kvm_mmu_invalidate_gva(vcpu, fault_mmu, fault->address,
686 fault_mmu->inject_page_fault(vcpu, fault);
691 void kvm_inject_nmi(struct kvm_vcpu *vcpu)
693 atomic_inc(&vcpu->arch.nmi_queued);
694 kvm_make_request(KVM_REQ_NMI, vcpu);
698 void kvm_queue_exception_e(struct kvm_vcpu *vcpu, unsigned nr, u32 error_code)
700 kvm_multiple_exception(vcpu, nr, true, error_code, false, 0, false);
704 void kvm_requeue_exception_e(struct kvm_vcpu *vcpu, unsigned nr, u32 error_code)
706 kvm_multiple_exception(vcpu, nr, true, error_code, false, 0, true);
714 bool kvm_require_cpl(struct kvm_vcpu *vcpu, int required_cpl)
716 if (kvm_x86_ops.get_cpl(vcpu) <= required_cpl)
718 kvm_queue_exception_e(vcpu, GP_VECTOR, 0);
723 bool kvm_require_dr(struct kvm_vcpu *vcpu, int dr)
725 if ((dr != 4 && dr != 5) || !kvm_read_cr4_bits(vcpu, X86_CR4_DE))
728 kvm_queue_exception(vcpu, UD_VECTOR);
738 int kvm_read_guest_page_mmu(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu,
747 real_gfn = mmu->translate_gpa(vcpu, ngpa, access, &exception);
753 return kvm_vcpu_read_guest_page(vcpu, real_gfn, data, offset, len);
757 static int kvm_read_nested_guest_page(struct kvm_vcpu *vcpu, gfn_t gfn,
760 return kvm_read_guest_page_mmu(vcpu, vcpu->arch.walk_mmu, gfn,
764 static inline u64 pdptr_rsvd_bits(struct kvm_vcpu *vcpu)
766 return rsvd_bits(cpuid_maxphyaddr(vcpu), 63) | rsvd_bits(5, 8) |
773 int load_pdptrs(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu, unsigned long cr3)
781 ret = kvm_read_guest_page_mmu(vcpu, mmu, pdpt_gfn, pdpte,
790 (pdpte[i] & pdptr_rsvd_bits(vcpu))) {
798 kvm_register_mark_dirty(vcpu, VCPU_EXREG_PDPTR);
806 bool pdptrs_changed(struct kvm_vcpu *vcpu)
808 u64 pdpte[ARRAY_SIZE(vcpu->arch.walk_mmu->pdptrs)];
813 if (!is_pae_paging(vcpu))
816 if (!kvm_register_is_available(vcpu, VCPU_EXREG_PDPTR))
819 gfn = (kvm_read_cr3(vcpu) & 0xffffffe0ul) >> PAGE_SHIFT;
820 offset = (kvm_read_cr3(vcpu) & 0xffffffe0ul) & (PAGE_SIZE - 1);
821 r = kvm_read_nested_guest_page(vcpu, gfn, pdpte, offset, sizeof(pdpte),
826 return memcmp(pdpte, vcpu->arch.walk_mmu->pdptrs, sizeof(pdpte)) != 0;
830 int kvm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0)
832 unsigned long old_cr0 = kvm_read_cr0(vcpu);
852 if ((vcpu->arch.efer & EFER_LME) && !is_paging(vcpu) &&
856 if (!is_pae(vcpu))
858 kvm_x86_ops.get_cs_db_l_bits(vcpu, &cs_db, &cs_l);
863 if (!(vcpu->arch.efer & EFER_LME) && (cr0 & X86_CR0_PG) &&
864 is_pae(vcpu) && ((cr0 ^ old_cr0) & pdptr_bits) &&
865 !load_pdptrs(vcpu, vcpu->arch.walk_mmu, kvm_read_cr3(vcpu)))
868 if (!(cr0 & X86_CR0_PG) && kvm_read_cr4_bits(vcpu, X86_CR4_PCIDE))
871 kvm_x86_ops.set_cr0(vcpu, cr0);
874 kvm_clear_async_pf_completion_queue(vcpu);
875 kvm_async_pf_hash_reset(vcpu);
879 kvm_mmu_reset_context(vcpu);
882 kvm_arch_has_noncoherent_dma(vcpu->kvm) &&
883 !kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_CD_NW_CLEARED))
884 kvm_zap_gfn_range(vcpu->kvm, 0, ~0ULL);
890 void kvm_lmsw(struct kvm_vcpu *vcpu, unsigned long msw)
892 (void)kvm_set_cr0(vcpu, kvm_read_cr0_bits(vcpu, ~0x0eul) | (msw & 0x0f));
896 void kvm_load_guest_xsave_state(struct kvm_vcpu *vcpu)
898 if (kvm_read_cr4_bits(vcpu, X86_CR4_OSXSAVE)) {
900 if (vcpu->arch.xcr0 != host_xcr0)
901 xsetbv(XCR_XFEATURE_ENABLED_MASK, vcpu->arch.xcr0);
903 if (vcpu->arch.xsaves_enabled &&
904 vcpu->arch.ia32_xss != host_xss)
905 wrmsrl(MSR_IA32_XSS, vcpu->arch.ia32_xss);
909 (kvm_read_cr4_bits(vcpu, X86_CR4_PKE) ||
910 (vcpu->arch.xcr0 & XFEATURE_MASK_PKRU)) &&
911 vcpu->arch.pkru != vcpu->arch.host_pkru)
912 __write_pkru(vcpu->arch.pkru);
916 void kvm_load_host_xsave_state(struct kvm_vcpu *vcpu)
919 (kvm_read_cr4_bits(vcpu, X86_CR4_PKE) ||
920 (vcpu->arch.xcr0 & XFEATURE_MASK_PKRU))) {
921 vcpu->arch.pkru = rdpkru();
922 if (vcpu->arch.pkru != vcpu->arch.host_pkru)
923 __write_pkru(vcpu->arch.host_pkru);
926 if (kvm_read_cr4_bits(vcpu, X86_CR4_OSXSAVE)) {
928 if (vcpu->arch.xcr0 != host_xcr0)
931 if (vcpu->arch.xsaves_enabled &&
932 vcpu->arch.ia32_xss != host_xss)
939 static int __kvm_set_xcr(struct kvm_vcpu *vcpu, u32 index, u64 xcr)
942 u64 old_xcr0 = vcpu->arch.xcr0;
958 valid_bits = vcpu->arch.guest_supported_xcr0 | XFEATURE_MASK_FP;
972 vcpu->arch.xcr0 = xcr0;
975 kvm_update_cpuid_runtime(vcpu);
979 int kvm_set_xcr(struct kvm_vcpu *vcpu, u32 index, u64 xcr)
981 if (kvm_x86_ops.get_cpl(vcpu) != 0 ||
982 __kvm_set_xcr(vcpu, index, xcr)) {
983 kvm_inject_gp(vcpu, 0);
990 int kvm_valid_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
995 if (cr4 & vcpu->arch.cr4_guest_rsvd_bits)
998 if (!kvm_x86_ops.is_valid_cr4(vcpu, cr4))
1005 int kvm_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
1007 unsigned long old_cr4 = kvm_read_cr4(vcpu);
1012 if (kvm_valid_cr4(vcpu, cr4))
1015 if (is_long_mode(vcpu)) {
1020 } else if (is_paging(vcpu) && (cr4 & X86_CR4_PAE)
1022 && !load_pdptrs(vcpu, vcpu->arch.walk_mmu,
1023 kvm_read_cr3(vcpu)))
1027 if (!guest_cpuid_has(vcpu, X86_FEATURE_PCID))
1031 if ((kvm_read_cr3(vcpu) & X86_CR3_PCID_MASK) || !is_long_mode(vcpu))
1035 kvm_x86_ops.set_cr4(vcpu, cr4);
1039 kvm_mmu_reset_context(vcpu);
1042 kvm_update_cpuid_runtime(vcpu);
1048 int kvm_set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3)
1052 bool pcid_enabled = kvm_read_cr4_bits(vcpu, X86_CR4_PCIDE);
1060 if (cr3 == kvm_read_cr3(vcpu) && !pdptrs_changed(vcpu)) {
1062 kvm_mmu_sync_roots(vcpu);
1063 kvm_make_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu);
1068 if (is_long_mode(vcpu) &&
1069 (cr3 & vcpu->arch.cr3_lm_rsvd_bits))
1071 else if (is_pae_paging(vcpu) &&
1072 !load_pdptrs(vcpu, vcpu->arch.walk_mmu, cr3))
1075 kvm_mmu_new_pgd(vcpu, cr3, skip_tlb_flush, skip_tlb_flush);
1076 vcpu->arch.cr3 = cr3;
1077 kvm_register_mark_available(vcpu, VCPU_EXREG_CR3);
1083 int kvm_set_cr8(struct kvm_vcpu *vcpu, unsigned long cr8)
1087 if (lapic_in_kernel(vcpu))
1088 kvm_lapic_set_tpr(vcpu, cr8);
1090 vcpu->arch.cr8 = cr8;
1095 unsigned long kvm_get_cr8(struct kvm_vcpu *vcpu)
1097 if (lapic_in_kernel(vcpu))
1098 return kvm_lapic_get_cr8(vcpu);
1100 return vcpu->arch.cr8;
1104 static void kvm_update_dr0123(struct kvm_vcpu *vcpu)
1108 if (!(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP)) {
1110 vcpu->arch.eff_db[i] = vcpu->arch.db[i];
1111 vcpu->arch.switch_db_regs |= KVM_DEBUGREG_RELOAD;
1115 void kvm_update_dr7(struct kvm_vcpu *vcpu)
1119 if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP)
1120 dr7 = vcpu->arch.guest_debug_dr7;
1122 dr7 = vcpu->arch.dr7;
1123 kvm_x86_ops.set_dr7(vcpu, dr7);
1124 vcpu->arch.switch_db_regs &= ~KVM_DEBUGREG_BP_ENABLED;
1126 vcpu->arch.switch_db_regs |= KVM_DEBUGREG_BP_ENABLED;
1130 static u64 kvm_dr6_fixed(struct kvm_vcpu *vcpu)
1134 if (!guest_cpuid_has(vcpu, X86_FEATURE_RTM))
1139 static int __kvm_set_dr(struct kvm_vcpu *vcpu, int dr, unsigned long val)
1141 size_t size = ARRAY_SIZE(vcpu->arch.db);
1145 vcpu->arch.db[array_index_nospec(dr, size)] = val;
1146 if (!(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP))
1147 vcpu->arch.eff_db[dr] = val;
1153 vcpu->arch.dr6 = (val & DR6_VOLATILE) | kvm_dr6_fixed(vcpu);
1159 vcpu->arch.dr7 = (val & DR7_VOLATILE) | DR7_FIXED_1;
1160 kvm_update_dr7(vcpu);
1167 int kvm_set_dr(struct kvm_vcpu *vcpu, int dr, unsigned long val)
1169 if (__kvm_set_dr(vcpu, dr, val)) {
1170 kvm_inject_gp(vcpu, 0);
1177 int kvm_get_dr(struct kvm_vcpu *vcpu, int dr, unsigned long *val)
1179 size_t size = ARRAY_SIZE(vcpu->arch.db);
1183 *val = vcpu->arch.db[array_index_nospec(dr, size)];
1187 *val = vcpu->arch.dr6;
1191 *val = vcpu->arch.dr7;
1198 bool kvm_rdpmc(struct kvm_vcpu *vcpu)
1200 u32 ecx = kvm_rcx_read(vcpu);
1204 err = kvm_pmu_rdpmc(vcpu, ecx, &data);
1207 kvm_rax_write(vcpu, (u32)data);
1208 kvm_rdx_write(vcpu, data >> 32);
1473 static int do_get_msr_feature(struct kvm_vcpu *vcpu, unsigned index, u64 *data)
1484 if (kvm_msr_ignored_check(vcpu, index, 0, false))
1496 static bool __kvm_valid_efer(struct kvm_vcpu *vcpu, u64 efer)
1498 if (efer & EFER_FFXSR && !guest_cpuid_has(vcpu, X86_FEATURE_FXSR_OPT))
1501 if (efer & EFER_SVME && !guest_cpuid_has(vcpu, X86_FEATURE_SVM))
1505 !guest_cpuid_has(vcpu, X86_FEATURE_LM))
1508 if (efer & EFER_NX && !guest_cpuid_has(vcpu, X86_FEATURE_NX))
1514 bool kvm_valid_efer(struct kvm_vcpu *vcpu, u64 efer)
1519 return __kvm_valid_efer(vcpu, efer);
1523 static int set_efer(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
1525 u64 old_efer = vcpu->arch.efer;
1533 if (!__kvm_valid_efer(vcpu, efer))
1536 if (is_paging(vcpu) &&
1537 (vcpu->arch.efer & EFER_LME) != (efer & EFER_LME))
1542 efer |= vcpu->arch.efer & EFER_LMA;
1544 r = kvm_x86_ops.set_efer(vcpu, efer);
1552 kvm_mmu_reset_context(vcpu);
1563 bool kvm_msr_allowed(struct kvm_vcpu *vcpu, u32 index, u32 type)
1567 struct kvm *kvm = vcpu->kvm;
1599 ++vcpu->stat.exits;
1615 static int __kvm_set_msr(struct kvm_vcpu *vcpu, u32 index, u64 data,
1620 if (!host_initiated && !kvm_msr_allowed(vcpu, index, KVM_MSR_FILTER_WRITE))
1629 if (is_noncanonical_address(data, vcpu))
1646 data = __canonical_address(data, vcpu_virt_addr_bits(vcpu));
1653 return kvm_x86_ops.set_msr(vcpu, &msr);
1656 static int kvm_set_msr_ignored_check(struct kvm_vcpu *vcpu,
1659 int ret = __kvm_set_msr(vcpu, index, data, host_initiated);
1662 if (kvm_msr_ignored_check(vcpu, index, data, true))
1674 int __kvm_get_msr(struct kvm_vcpu *vcpu, u32 index, u64 *data,
1680 if (!host_initiated && !kvm_msr_allowed(vcpu, index, KVM_MSR_FILTER_READ))
1686 ret = kvm_x86_ops.get_msr(vcpu, &msr);
1692 static int kvm_get_msr_ignored_check(struct kvm_vcpu *vcpu,
1695 int ret = __kvm_get_msr(vcpu, index, data, host_initiated);
1700 if (kvm_msr_ignored_check(vcpu, index, 0, false))
1707 int kvm_get_msr(struct kvm_vcpu *vcpu, u32 index, u64 *data)
1709 return kvm_get_msr_ignored_check(vcpu, index, data, false);
1713 int kvm_set_msr(struct kvm_vcpu *vcpu, u32 index, u64 data)
1715 return kvm_set_msr_ignored_check(vcpu, index, data, false);
1719 static int complete_emulated_msr(struct kvm_vcpu *vcpu, bool is_read)
1721 if (vcpu->run->msr.error) {
1722 kvm_inject_gp(vcpu, 0);
1725 kvm_rax_write(vcpu, (u32)vcpu->run->msr.data);
1726 kvm_rdx_write(vcpu, vcpu->run->msr.data >> 32);
1729 return kvm_skip_emulated_instruction(vcpu);
1732 static int complete_emulated_rdmsr(struct kvm_vcpu *vcpu)
1734 return complete_emulated_msr(vcpu, true);
1737 static int complete_emulated_wrmsr(struct kvm_vcpu *vcpu)
1739 return complete_emulated_msr(vcpu, false);
1754 static int kvm_msr_user_space(struct kvm_vcpu *vcpu, u32 index,
1756 int (*completion)(struct kvm_vcpu *vcpu),
1762 if (!(vcpu->kvm->arch.user_space_msr_mask & msr_reason))
1765 vcpu->run->exit_reason = exit_reason;
1766 vcpu->run->msr.error = 0;
1767 memset(vcpu->run->msr.pad, 0, sizeof(vcpu->run->msr.pad));
1768 vcpu->run->msr.reason = msr_reason;
1769 vcpu->run->msr.index = index;
1770 vcpu->run->msr.data = data;
1771 vcpu->arch.complete_userspace_io = completion;
1776 static int kvm_get_msr_user_space(struct kvm_vcpu *vcpu, u32 index, int r)
1778 return kvm_msr_user_space(vcpu, index, KVM_EXIT_X86_RDMSR, 0,
1782 static int kvm_set_msr_user_space(struct kvm_vcpu *vcpu, u32 index, u64 data, int r)
1784 return kvm_msr_user_space(vcpu, index, KVM_EXIT_X86_WRMSR, data,
1788 int kvm_emulate_rdmsr(struct kvm_vcpu *vcpu)
1790 u32 ecx = kvm_rcx_read(vcpu);
1794 r = kvm_get_msr(vcpu, ecx, &data);
1797 if (r && kvm_get_msr_user_space(vcpu, ecx, r)) {
1805 kvm_inject_gp(vcpu, 0);
1811 kvm_rax_write(vcpu, data & -1u);
1812 kvm_rdx_write(vcpu, (data >> 32) & -1u);
1813 return kvm_skip_emulated_instruction(vcpu);
1817 int kvm_emulate_wrmsr(struct kvm_vcpu *vcpu)
1819 u32 ecx = kvm_rcx_read(vcpu);
1820 u64 data = kvm_read_edx_eax(vcpu);
1823 r = kvm_set_msr(vcpu, ecx, data);
1826 if (r && kvm_set_msr_user_space(vcpu, ecx, data, r))
1837 kvm_inject_gp(vcpu, 0);
1842 return kvm_skip_emulated_instruction(vcpu);
1846 bool kvm_vcpu_exit_request(struct kvm_vcpu *vcpu)
1848 return vcpu->mode == EXITING_GUEST_MODE || kvm_request_pending(vcpu) ||
1860 static int handle_fastpath_set_x2apic_icr_irqoff(struct kvm_vcpu *vcpu, u64 data)
1862 if (!lapic_in_kernel(vcpu) || !apic_x2apic_mode(vcpu->arch.apic))
1871 kvm_apic_send_ipi(vcpu->arch.apic, (u32)data, (u32)(data >> 32));
1872 kvm_lapic_set_reg(vcpu->arch.apic, APIC_ICR2, (u32)(data >> 32));
1873 kvm_lapic_set_reg(vcpu->arch.apic, APIC_ICR, (u32)data);
1881 static int handle_fastpath_set_tscdeadline(struct kvm_vcpu *vcpu, u64 data)
1883 if (!kvm_can_use_hv_timer(vcpu))
1886 kvm_set_lapic_tscdeadline_msr(vcpu, data);
1890 fastpath_t handle_fastpath_set_msr_irqoff(struct kvm_vcpu *vcpu)
1892 u32 msr = kvm_rcx_read(vcpu);
1898 data = kvm_read_edx_eax(vcpu);
1899 if (!handle_fastpath_set_x2apic_icr_irqoff(vcpu, data)) {
1900 kvm_skip_emulated_instruction(vcpu);
1905 data = kvm_read_edx_eax(vcpu);
1906 if (!handle_fastpath_set_tscdeadline(vcpu, data)) {
1907 kvm_skip_emulated_instruction(vcpu);
1925 static int do_get_msr(struct kvm_vcpu *vcpu, unsigned index, u64 *data)
1927 return kvm_get_msr_ignored_check(vcpu, index, data, true);
1930 static int do_set_msr(struct kvm_vcpu *vcpu, unsigned index, u64 *data)
1932 return kvm_set_msr_ignored_check(vcpu, index, *data, true);
2042 static void kvm_write_system_time(struct kvm_vcpu *vcpu, gpa_t system_time,
2045 struct kvm_arch *ka = &vcpu->kvm->arch;
2047 if (vcpu->vcpu_id == 0 && !host_initiated) {
2049 kvm_make_request(KVM_REQ_MASTERCLOCK_UPDATE, vcpu);
2054 vcpu->arch.time = system_time;
2055 kvm_make_request(KVM_REQ_GLOBAL_CLOCK_UPDATE, vcpu);
2058 vcpu->arch.pv_time_enabled = false;
2062 if (!kvm_gfn_to_hva_cache_init(vcpu->kvm,
2063 &vcpu->arch.pv_time, system_time & ~1ULL,
2065 vcpu->arch.pv_time_enabled = true;
2118 static int set_tsc_khz(struct kvm_vcpu *vcpu, u32 user_tsc_khz, bool scale)
2124 vcpu->arch.tsc_scaling_ratio = kvm_default_tsc_scaling_ratio;
2131 vcpu->arch.tsc_catchup = 1;
2132 vcpu->arch.tsc_always_catchup = 1;
2150 vcpu->arch.tsc_scaling_ratio = ratio;
2154 static int kvm_set_tsc_khz(struct kvm_vcpu *vcpu, u32 user_tsc_khz)
2162 vcpu->arch.tsc_scaling_ratio = kvm_default_tsc_scaling_ratio;
2168 &vcpu->arch.virtual_tsc_shift,
2169 &vcpu->arch.virtual_tsc_mult);
2170 vcpu->arch.virtual_tsc_khz = user_tsc_khz;
2184 return set_tsc_khz(vcpu, user_tsc_khz, use_scaling);
2187 static u64 compute_guest_tsc(struct kvm_vcpu *vcpu, s64 kernel_ns)
2189 u64 tsc = pvclock_scale_delta(kernel_ns-vcpu->arch.this_tsc_nsec,
2190 vcpu->arch.virtual_tsc_mult,
2191 vcpu->arch.virtual_tsc_shift);
2192 tsc += vcpu->arch.this_tsc_write;
2201 static void kvm_track_tsc_matching(struct kvm_vcpu *vcpu)
2205 struct kvm_arch *ka = &vcpu->kvm->arch;
2209 atomic_read(&vcpu->kvm->online_vcpus));
2221 kvm_make_request(KVM_REQ_MASTERCLOCK_UPDATE, vcpu);
2223 trace_kvm_track_tsc(vcpu->vcpu_id, ka->nr_vcpus_matched_tsc,
2224 atomic_read(&vcpu->kvm->online_vcpus),
2244 u64 kvm_scale_tsc(struct kvm_vcpu *vcpu, u64 tsc)
2247 u64 ratio = vcpu->arch.tsc_scaling_ratio;
2256 static u64 kvm_compute_tsc_offset(struct kvm_vcpu *vcpu, u64 target_tsc)
2260 tsc = kvm_scale_tsc(vcpu, rdtsc());
2265 u64 kvm_read_l1_tsc(struct kvm_vcpu *vcpu, u64 host_tsc)
2267 return vcpu->arch.l1_tsc_offset + kvm_scale_tsc(vcpu, host_tsc);
2271 static void kvm_vcpu_write_tsc_offset(struct kvm_vcpu *vcpu, u64 offset)
2273 vcpu->arch.l1_tsc_offset = offset;
2274 vcpu->arch.tsc_offset = kvm_x86_ops.write_l1_tsc_offset(vcpu, offset);
2290 static void kvm_synchronize_tsc(struct kvm_vcpu *vcpu, u64 data)
2292 struct kvm *kvm = vcpu->kvm;
2300 offset = kvm_compute_tsc_offset(vcpu, data);
2304 if (vcpu->arch.virtual_tsc_khz) {
2307 * detection of vcpu initialization -- need to sync
2314 nsec_to_cycles(vcpu, elapsed);
2315 u64 tsc_hz = vcpu->arch.virtual_tsc_khz * 1000LL;
2333 vcpu->arch.virtual_tsc_khz == kvm->arch.last_tsc_khz) {
2337 u64 delta = nsec_to_cycles(vcpu, elapsed);
2339 offset = kvm_compute_tsc_offset(vcpu, data);
2342 already_matched = (vcpu->arch.this_tsc_generation == kvm->arch.cur_tsc_generation);
2366 kvm->arch.last_tsc_khz = vcpu->arch.virtual_tsc_khz;
2368 vcpu->arch.last_guest_tsc = data;
2371 vcpu->arch.this_tsc_generation = kvm->arch.cur_tsc_generation;
2372 vcpu->arch.this_tsc_nsec = kvm->arch.cur_tsc_nsec;
2373 vcpu->arch.this_tsc_write = kvm->arch.cur_tsc_write;
2375 kvm_vcpu_write_tsc_offset(vcpu, offset);
2385 kvm_track_tsc_matching(vcpu);
2389 static inline void adjust_tsc_offset_guest(struct kvm_vcpu *vcpu,
2392 u64 tsc_offset = vcpu->arch.l1_tsc_offset;
2393 kvm_vcpu_write_tsc_offset(vcpu, tsc_offset + adjustment);
2396 static inline void adjust_tsc_offset_host(struct kvm_vcpu *vcpu, s64 adjustment)
2398 if (vcpu->arch.tsc_scaling_ratio != kvm_default_tsc_scaling_ratio)
2400 adjustment = kvm_scale_tsc(vcpu, (u64) adjustment);
2401 adjust_tsc_offset_guest(vcpu, adjustment);
2606 struct kvm_vcpu *vcpu;
2614 kvm_for_each_vcpu(i, vcpu, kvm)
2615 kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);
2618 kvm_for_each_vcpu(i, vcpu, kvm)
2619 kvm_clear_request(KVM_REQ_MCLOCK_INPROGRESS, vcpu);
2659 struct kvm_vcpu_arch *vcpu = &v->arch;
2662 if (unlikely(kvm_read_guest_cached(v->kvm, &vcpu->pv_time,
2677 * and third write. The vcpu->pv_time cache is still valid, because the
2685 vcpu->hv_clock.version = guest_hv_clock.version + 1;
2686 kvm_write_guest_cached(v->kvm, &vcpu->pv_time,
2687 &vcpu->hv_clock,
2688 sizeof(vcpu->hv_clock.version));
2693 vcpu->hv_clock.flags |= (guest_hv_clock.flags & PVCLOCK_GUEST_STOPPED);
2695 if (vcpu->pvclock_set_guest_stopped_request) {
2696 vcpu->hv_clock.flags |= PVCLOCK_GUEST_STOPPED;
2697 vcpu->pvclock_set_guest_stopped_request = false;
2700 trace_kvm_pvclock_update(v->vcpu_id, &vcpu->hv_clock);
2702 kvm_write_guest_cached(v->kvm, &vcpu->pv_time,
2703 &vcpu->hv_clock,
2704 sizeof(vcpu->hv_clock));
2708 vcpu->hv_clock.version++;
2709 kvm_write_guest_cached(v->kvm, &vcpu->pv_time,
2710 &vcpu->hv_clock,
2711 sizeof(vcpu->hv_clock.version));
2717 struct kvm_vcpu_arch *vcpu = &v->arch;
2764 if (vcpu->tsc_catchup) {
2779 if (unlikely(vcpu->hw_tsc_khz != tgt_tsc_khz)) {
2781 &vcpu->hv_clock.tsc_shift,
2782 &vcpu->hv_clock.tsc_to_system_mul);
2783 vcpu->hw_tsc_khz = tgt_tsc_khz;
2786 vcpu->hv_clock.tsc_timestamp = tsc_timestamp;
2787 vcpu->hv_clock.system_time = kernel_ns + v->kvm->arch.kvmclock_offset;
2788 vcpu->last_guest_tsc = tsc_timestamp;
2795 vcpu->hv_clock.flags = pvclock_flags;
2797 if (vcpu->pv_time_enabled)
2800 kvm_hv_setup_tsc_page(v->kvm, &vcpu->hv_clock);
2805 * kvmclock updates which are isolated to a given vcpu, such as
2806 * vcpu->cpu migration, should not allow system_timestamp from
2808 * correction applies to one vcpu's system_timestamp but not
2814 * The time for a remote vcpu to update its kvmclock is bound
2827 struct kvm_vcpu *vcpu;
2829 kvm_for_each_vcpu(i, vcpu, kvm) {
2830 kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);
2831 kvm_vcpu_kick(vcpu);
2864 static bool can_set_mci_status(struct kvm_vcpu *vcpu)
2867 if (guest_cpuid_is_amd_or_hygon(vcpu))
2868 return !!(vcpu->arch.msr_hwcr & BIT_ULL(18));
2873 static int set_msr_mce(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
2875 u64 mcg_cap = vcpu->arch.mcg_cap;
2882 vcpu->arch.mcg_status = data;
2890 vcpu->arch.mcg_ctl = data;
2914 if (!can_set_mci_status(vcpu))
2918 vcpu->arch.mce_banks[offset] = data;
2926 static int xen_hvm_config(struct kvm_vcpu *vcpu, u64 data)
2928 struct kvm *kvm = vcpu->kvm;
2929 int lm = is_long_mode(vcpu);
2945 if (kvm_vcpu_write_guest(vcpu, page_addr, page, PAGE_SIZE)) {
2952 static inline bool kvm_pv_async_pf_enabled(struct kvm_vcpu *vcpu)
2956 return (vcpu->arch.apf.msr_en_val & mask) == mask;
2959 static int kvm_pv_enable_async_pf(struct kvm_vcpu *vcpu, u64 data)
2967 if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF_VMEXIT) &&
2971 if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF_INT) &&
2975 if (!lapic_in_kernel(vcpu))
2978 vcpu->arch.apf.msr_en_val = data;
2980 if (!kvm_pv_async_pf_enabled(vcpu)) {
2981 kvm_clear_async_pf_completion_queue(vcpu);
2982 kvm_async_pf_hash_reset(vcpu);
2986 if (kvm_gfn_to_hva_cache_init(vcpu->kvm, &vcpu->arch.apf.data, gpa,
2990 vcpu->arch.apf.send_user_only = !(data & KVM_ASYNC_PF_SEND_ALWAYS);
2991 vcpu->arch.apf.delivery_as_pf_vmexit = data & KVM_ASYNC_PF_DELIVERY_AS_PF_VMEXIT;
2993 kvm_async_pf_wakeup_all(vcpu);
2998 static int kvm_pv_enable_async_pf_int(struct kvm_vcpu *vcpu, u64 data)
3004 if (!lapic_in_kernel(vcpu))
3007 vcpu->arch.apf.msr_int_val = data;
3009 vcpu->arch.apf.vec = data & KVM_ASYNC_PF_VEC_MASK;
3014 static void kvmclock_reset(struct kvm_vcpu *vcpu)
3016 vcpu->arch.pv_time_enabled = false;
3017 vcpu->arch.time = 0;
3020 static void kvm_vcpu_flush_tlb_all(struct kvm_vcpu *vcpu)
3022 ++vcpu->stat.tlb_flush;
3023 kvm_x86_ops.tlb_flush_all(vcpu);
3026 static void kvm_vcpu_flush_tlb_guest(struct kvm_vcpu *vcpu)
3028 ++vcpu->stat.tlb_flush;
3029 kvm_x86_ops.tlb_flush_guest(vcpu);
3032 static void record_steal_time(struct kvm_vcpu *vcpu)
3034 struct gfn_to_hva_cache *ghc = &vcpu->arch.st.cache;
3037 gpa_t gpa = vcpu->arch.st.msr_val & KVM_STEAL_VALID_BITS;
3041 if (!(vcpu->arch.st.msr_val & KVM_MSR_ENABLED))
3044 if (WARN_ON_ONCE(current->mm != vcpu->kvm->mm))
3047 slots = kvm_memslots(vcpu->kvm);
3055 if (kvm_gfn_to_hva_cache_init(vcpu->kvm, ghc, gpa, sizeof(*st)) ||
3065 if (guest_pv_has(vcpu, KVM_FEATURE_PV_TLB_FLUSH)) {
3084 vcpu->arch.st.preempted = 0;
3086 trace_kvm_pv_tlb_flush(vcpu->vcpu_id,
3089 kvm_vcpu_flush_tlb_guest(vcpu);
3098 vcpu->arch.st.preempted = 0;
3112 vcpu->arch.st.last_steal;
3113 vcpu->arch.st.last_steal = current->sched_info.run_delay;
3125 int kvm_set_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
3144 vcpu->arch.microcode_version = data;
3149 vcpu->arch.arch_capabilities = data;
3161 vcpu->arch.perf_capabilities = data;
3166 return set_efer(vcpu, msr_info);
3174 vcpu->arch.msr_hwcr = data;
3176 vcpu_unimpl(vcpu, "unimplemented HWCR wrmsr: 0x%llx\n",
3183 vcpu_unimpl(vcpu, "unimplemented MMIO_CONF_BASE wrmsr: "
3197 vcpu_unimpl(vcpu, "%s: MSR_IA32_DEBUGCTLMSR 0x%llx, nop\n",
3201 return kvm_mtrr_set_msr(vcpu, msr, data);
3203 return kvm_set_apic_base(vcpu, msr_info);
3205 return kvm_x2apic_msr_write(vcpu, msr, data);
3207 kvm_set_lapic_tscdeadline_msr(vcpu, data);
3210 if (guest_cpuid_has(vcpu, X86_FEATURE_TSC_ADJUST)) {
3212 s64 adj = data - vcpu->arch.ia32_tsc_adjust_msr;
3213 adjust_tsc_offset_guest(vcpu, adj);
3217 kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);
3219 vcpu->arch.ia32_tsc_adjust_msr = data;
3223 if (!kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_MISC_ENABLE_NO_MWAIT) &&
3224 ((vcpu->arch.ia32_misc_enable_msr ^ data) & MSR_IA32_MISC_ENABLE_MWAIT)) {
3225 if (!guest_cpuid_has(vcpu, X86_FEATURE_XMM3))
3227 vcpu->arch.ia32_misc_enable_msr = data;
3228 kvm_update_cpuid_runtime(vcpu);
3230 vcpu->arch.ia32_misc_enable_msr = data;
3236 vcpu->arch.smbase = data;
3239 vcpu->arch.msr_ia32_power_ctl = data;
3243 kvm_synchronize_tsc(vcpu, data);
3245 u64 adj = kvm_compute_tsc_offset(vcpu, data) - vcpu->arch.l1_tsc_offset;
3246 adjust_tsc_offset_guest(vcpu, adj);
3247 vcpu->arch.ia32_tsc_adjust_msr += adj;
3252 !guest_cpuid_has(vcpu, X86_FEATURE_XSAVES))
3261 vcpu->arch.ia32_xss = data;
3262 kvm_update_cpuid_runtime(vcpu);
3267 vcpu->arch.smi_count = data;
3270 if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE2))
3273 kvm_write_wall_clock(vcpu->kvm, data);
3276 if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE))
3279 kvm_write_wall_clock(vcpu->kvm, data);
3282 if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE2))
3285 kvm_write_system_time(vcpu, data, false, msr_info->host_initiated);
3288 if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE))
3291 kvm_write_system_time(vcpu, data, true, msr_info->host_initiated);
3294 if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF))
3297 if (kvm_pv_enable_async_pf(vcpu, data))
3301 if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF_INT))
3304 if (kvm_pv_enable_async_pf_int(vcpu, data))
3308 if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF_INT))
3311 vcpu->arch.apf.pageready_pending = false;
3312 kvm_check_async_pf_completion(vcpu);
3316 if (!guest_pv_has(vcpu, KVM_FEATURE_STEAL_TIME))
3325 vcpu->arch.st.msr_val = data;
3330 kvm_make_request(KVM_REQ_STEAL_UPDATE, vcpu);
3334 if (!guest_pv_has(vcpu, KVM_FEATURE_PV_EOI))
3337 if (kvm_lapic_enable_pv_eoi(vcpu, data, sizeof(u8)))
3342 if (!guest_pv_has(vcpu, KVM_FEATURE_POLL_CONTROL))
3349 vcpu->arch.msr_kvm_poll_control = data;
3355 return set_msr_mce(vcpu, msr_info);
3363 if (kvm_pmu_is_valid_msr(vcpu, msr))
3364 return kvm_pmu_set_msr(vcpu, msr_info);
3367 vcpu_unimpl(vcpu, "disabled perfctr wrmsr: "
3389 return kvm_hv_set_msr_common(vcpu, msr, data,
3396 vcpu_unimpl(vcpu, "ignored wrmsr: 0x%x data 0x%llx\n",
3400 if (!guest_cpuid_has(vcpu, X86_FEATURE_OSVW))
3402 vcpu->arch.osvw.length = data;
3405 if (!guest_cpuid_has(vcpu, X86_FEATURE_OSVW))
3407 vcpu->arch.osvw.status = data;
3412 cpuid_fault_enabled(vcpu)))
3414 vcpu->arch.msr_platform_info = data;
3419 !supports_cpuid_fault(vcpu)))
3421 vcpu->arch.msr_misc_features_enables = data;
3424 if (msr && (msr == vcpu->kvm->arch.xen_hvm_config.msr))
3425 return xen_hvm_config(vcpu, data);
3426 if (kvm_pmu_is_valid_msr(vcpu, msr))
3427 return kvm_pmu_set_msr(vcpu, msr_info);
3434 static int get_msr_mce(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata, bool host)
3437 u64 mcg_cap = vcpu->arch.mcg_cap;
3446 data = vcpu->arch.mcg_cap;
3451 data = vcpu->arch.mcg_ctl;
3454 data = vcpu->arch.mcg_status;
3463 data = vcpu->arch.mce_banks[offset];
3472 int kvm_get_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
3512 if (kvm_pmu_is_valid_msr(vcpu, msr_info->index))
3513 return kvm_pmu_get_msr(vcpu, msr_info);
3517 msr_info->data = vcpu->arch.microcode_version;
3521 !guest_cpuid_has(vcpu, X86_FEATURE_ARCH_CAPABILITIES))
3523 msr_info->data = vcpu->arch.arch_capabilities;
3527 !guest_cpuid_has(vcpu, X86_FEATURE_PDCM))
3529 msr_info->data = vcpu->arch.perf_capabilities;
3532 msr_info->data = vcpu->arch.msr_ia32_power_ctl;
3544 u64 tsc_offset = msr_info->host_initiated ? vcpu->arch.l1_tsc_offset :
3545 vcpu->arch.tsc_offset;
3547 msr_info->data = kvm_scale_tsc(vcpu, rdtsc()) + tsc_offset;
3552 return kvm_mtrr_get_msr(vcpu, msr_info->index, &msr_info->data);
3571 msr_info->data = kvm_get_apic_base(vcpu);
3574 return kvm_x2apic_msr_read(vcpu, msr_info->index, &msr_info->data);
3576 msr_info->data = kvm_get_lapic_tscdeadline_msr(vcpu);
3579 msr_info->data = (u64)vcpu->arch.ia32_tsc_adjust_msr;
3582 msr_info->data = vcpu->arch.ia32_misc_enable_msr;
3587 msr_info->data = vcpu->arch.smbase;
3590 msr_info->data = vcpu->arch.smi_count;
3599 msr_info->data = vcpu->arch.efer;
3602 if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE))
3605 msr_info->data = vcpu->kvm->arch.wall_clock;
3608 if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE2))
3611 msr_info->data = vcpu->kvm->arch.wall_clock;
3614 if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE))
3617 msr_info->data = vcpu->arch.time;
3620 if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE2))
3623 msr_info->data = vcpu->arch.time;
3626 if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF))
3629 msr_info->data = vcpu->arch.apf.msr_en_val;
3632 if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF_INT))
3635 msr_info->data = vcpu->arch.apf.msr_int_val;
3638 if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF_INT))
3644 if (!guest_pv_has(vcpu, KVM_FEATURE_STEAL_TIME))
3647 msr_info->data = vcpu->arch.st.msr_val;
3650 if (!guest_pv_has(vcpu, KVM_FEATURE_PV_EOI))
3653 msr_info->data = vcpu->arch.pv_eoi.msr_val;
3656 if (!guest_pv_has(vcpu, KVM_FEATURE_POLL_CONTROL))
3659 msr_info->data = vcpu->arch.msr_kvm_poll_control;
3667 return get_msr_mce(vcpu, msr_info->index, &msr_info->data,
3671 !guest_cpuid_has(vcpu, X86_FEATURE_XSAVES))
3673 msr_info->data = vcpu->arch.ia32_xss;
3696 return kvm_hv_get_msr_common(vcpu,
3713 if (!guest_cpuid_has(vcpu, X86_FEATURE_OSVW))
3715 msr_info->data = vcpu->arch.osvw.length;
3718 if (!guest_cpuid_has(vcpu, X86_FEATURE_OSVW))
3720 msr_info->data = vcpu->arch.osvw.status;
3724 !vcpu->kvm->arch.guest_can_read_msr_platform_info)
3726 msr_info->data = vcpu->arch.msr_platform_info;
3729 msr_info->data = vcpu->arch.msr_misc_features_enables;
3732 msr_info->data = vcpu->arch.msr_hwcr;
3735 if (kvm_pmu_is_valid_msr(vcpu, msr_info->index))
3736 return kvm_pmu_get_msr(vcpu, msr_info);
3748 static int __msr_io(struct kvm_vcpu *vcpu, struct kvm_msrs *msrs,
3750 int (*do_msr)(struct kvm_vcpu *vcpu,
3756 if (do_msr(vcpu, entries[i].index, &entries[i].data))
3767 static int msr_io(struct kvm_vcpu *vcpu, struct kvm_msrs __user *user_msrs,
3768 int (*do_msr)(struct kvm_vcpu *vcpu,
3792 r = n = __msr_io(vcpu, &msrs, entries, do_msr);
4049 static bool need_emulate_wbinvd(struct kvm_vcpu *vcpu)
4051 return kvm_arch_has_noncoherent_dma(vcpu->kvm);
4054 void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
4057 if (need_emulate_wbinvd(vcpu)) {
4059 cpumask_set_cpu(cpu, vcpu->arch.wbinvd_dirty_mask);
4060 else if (vcpu->cpu != -1 && vcpu->cpu != cpu)
4061 smp_call_function_single(vcpu->cpu,
4065 kvm_x86_ops.vcpu_load(vcpu, cpu);
4068 vcpu->arch.host_pkru = read_pkru();
4071 if (unlikely(vcpu->arch.tsc_offset_adjustment)) {
4072 adjust_tsc_offset_host(vcpu, vcpu->arch.tsc_offset_adjustment);
4073 vcpu->arch.tsc_offset_adjustment = 0;
4074 kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);
4077 if (unlikely(vcpu->cpu != cpu) || kvm_check_tsc_unstable()) {
4078 s64 tsc_delta = !vcpu->arch.last_host_tsc ? 0 :
4079 rdtsc() - vcpu->arch.last_host_tsc;
4084 u64 offset = kvm_compute_tsc_offset(vcpu,
4085 vcpu->arch.last_guest_tsc);
4086 kvm_vcpu_write_tsc_offset(vcpu, offset);
4087 vcpu->arch.tsc_catchup = 1;
4090 if (kvm_lapic_hv_timer_in_use(vcpu))
4091 kvm_lapic_restart_hv_timer(vcpu);
4095 * kvmclock on vcpu->cpu migration
4097 if (!vcpu->kvm->arch.use_master_clock || vcpu->cpu == -1)
4098 kvm_make_request(KVM_REQ_GLOBAL_CLOCK_UPDATE, vcpu);
4099 if (vcpu->cpu != cpu)
4100 kvm_make_request(KVM_REQ_MIGRATE_TIMER, vcpu);
4101 vcpu->cpu = cpu;
4104 kvm_make_request(KVM_REQ_STEAL_UPDATE, vcpu);
4107 static void kvm_steal_time_set_preempted(struct kvm_vcpu *vcpu)
4109 struct gfn_to_hva_cache *ghc = &vcpu->arch.st.cache;
4113 gpa_t gpa = vcpu->arch.st.msr_val & KVM_STEAL_VALID_BITS;
4122 if (!vcpu->arch.at_instruction_boundary) {
4123 vcpu->stat.preemption_other++;
4127 vcpu->stat.preemption_reported++;
4128 if (!(vcpu->arch.st.msr_val & KVM_MSR_ENABLED))
4131 if (vcpu->arch.st.preempted)
4135 if (unlikely(current->mm != vcpu->kvm->mm))
4138 slots = kvm_memslots(vcpu->kvm);
4149 vcpu->arch.st.preempted = KVM_VCPU_PREEMPTED;
4154 void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
4158 if (vcpu->preempted) {
4159 vcpu->arch.preempted_in_kernel = !kvm_x86_ops.get_cpl(vcpu);
4165 idx = srcu_read_lock(&vcpu->kvm->srcu);
4166 kvm_steal_time_set_preempted(vcpu);
4167 srcu_read_unlock(&vcpu->kvm->srcu, idx);
4170 kvm_x86_ops.vcpu_put(vcpu);
4171 vcpu->arch.last_host_tsc = rdtsc();
4180 static int kvm_vcpu_ioctl_get_lapic(struct kvm_vcpu *vcpu,
4183 if (vcpu->arch.apicv_active)
4184 kvm_x86_ops.sync_pir_to_irr(vcpu);
4186 return kvm_apic_get_state(vcpu, s);
4189 static int kvm_vcpu_ioctl_set_lapic(struct kvm_vcpu *vcpu,
4194 r = kvm_apic_set_state(vcpu, s);
4197 update_cr8_intercept(vcpu);
4202 static int kvm_cpu_accept_dm_intr(struct kvm_vcpu *vcpu)
4210 if (kvm_cpu_has_extint(vcpu))
4214 return (!lapic_in_kernel(vcpu) ||
4215 kvm_apic_accept_pic_intr(vcpu));
4218 static int kvm_vcpu_ready_for_interrupt_injection(struct kvm_vcpu *vcpu)
4227 return (kvm_arch_interrupt_allowed(vcpu) &&
4228 kvm_cpu_accept_dm_intr(vcpu) &&
4229 !kvm_event_needs_reinjection(vcpu) &&
4230 !vcpu->arch.exception.pending);
4233 static int kvm_vcpu_ioctl_interrupt(struct kvm_vcpu *vcpu,
4239 if (!irqchip_in_kernel(vcpu->kvm)) {
4240 kvm_queue_interrupt(vcpu, irq->irq, false);
4241 kvm_make_request(KVM_REQ_EVENT, vcpu);
4249 if (pic_in_kernel(vcpu->kvm))
4252 if (vcpu->arch.pending_external_vector != -1)
4255 vcpu->arch.pending_external_vector = irq->irq;
4256 kvm_make_request(KVM_REQ_EVENT, vcpu);
4260 static int kvm_vcpu_ioctl_nmi(struct kvm_vcpu *vcpu)
4262 kvm_inject_nmi(vcpu);
4267 static int kvm_vcpu_ioctl_smi(struct kvm_vcpu *vcpu)
4269 kvm_make_request(KVM_REQ_SMI, vcpu);
4274 static int vcpu_ioctl_tpr_access_reporting(struct kvm_vcpu *vcpu,
4279 vcpu->arch.tpr_access_reporting = !!tac->enabled;
4283 static int kvm_vcpu_ioctl_x86_setup_mce(struct kvm_vcpu *vcpu,
4295 vcpu->arch.mcg_cap = mcg_cap;
4298 vcpu->arch.mcg_ctl = ~(u64)0;
4301 vcpu->arch.mce_banks[bank*4] = ~(u64)0;
4303 kvm_x86_ops.setup_mce(vcpu);
4308 static int kvm_vcpu_ioctl_x86_set_mce(struct kvm_vcpu *vcpu,
4311 u64 mcg_cap = vcpu->arch.mcg_cap;
4313 u64 *banks = vcpu->arch.mce_banks;
4322 vcpu->arch.mcg_ctl != ~(u64)0)
4332 if ((vcpu->arch.mcg_status & MCG_STATUS_MCIP) ||
4333 !kvm_read_cr4_bits(vcpu, X86_CR4_MCE)) {
4334 kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);
4341 vcpu->arch.mcg_status = mce->mcg_status;
4343 kvm_queue_exception(vcpu, MC_VECTOR);
4356 static void kvm_vcpu_ioctl_x86_get_vcpu_events(struct kvm_vcpu *vcpu,
4359 process_nmi(vcpu);
4361 if (kvm_check_request(KVM_REQ_SMI, vcpu))
4362 process_smi(vcpu);
4375 if (!vcpu->kvm->arch.exception_payload_enabled &&
4376 vcpu->arch.exception.pending && vcpu->arch.exception.has_payload)
4377 kvm_deliver_exception_payload(vcpu);
4385 if (kvm_exception_is_soft(vcpu->arch.exception.nr)) {
4389 events->exception.injected = vcpu->arch.exception.injected;
4390 events->exception.pending = vcpu->arch.exception.pending;
4396 if (!vcpu->kvm->arch.exception_payload_enabled)
4398 vcpu->arch.exception.pending;
4400 events->exception.nr = vcpu->arch.exception.nr;
4401 events->exception.has_error_code = vcpu->arch.exception.has_error_code;
4402 events->exception.error_code = vcpu->arch.exception.error_code;
4403 events->exception_has_payload = vcpu->arch.exception.has_payload;
4404 events->exception_payload = vcpu->arch.exception.payload;
4407 vcpu->arch.interrupt.injected && !vcpu->arch.interrupt.soft;
4408 events->interrupt.nr = vcpu->arch.interrupt.nr;
4410 events->interrupt.shadow = kvm_x86_ops.get_interrupt_shadow(vcpu);
4412 events->nmi.injected = vcpu->arch.nmi_injected;
4413 events->nmi.pending = vcpu->arch.nmi_pending != 0;
4414 events->nmi.masked = kvm_x86_ops.get_nmi_mask(vcpu);
4419 events->smi.smm = is_smm(vcpu);
4420 events->smi.pending = vcpu->arch.smi_pending;
4422 !!(vcpu->arch.hflags & HF_SMM_INSIDE_NMI_MASK);
4423 events->smi.latched_init = kvm_lapic_latched_init(vcpu);
4428 if (vcpu->kvm->arch.exception_payload_enabled)
4434 static void kvm_smm_changed(struct kvm_vcpu *vcpu);
4436 static int kvm_vcpu_ioctl_x86_set_vcpu_events(struct kvm_vcpu *vcpu,
4447 if (!vcpu->kvm->arch.exception_payload_enabled)
4465 vcpu->arch.mp_state == KVM_MP_STATE_INIT_RECEIVED)
4468 process_nmi(vcpu);
4469 vcpu->arch.exception.injected = events->exception.injected;
4470 vcpu->arch.exception.pending = events->exception.pending;
4471 vcpu->arch.exception.nr = events->exception.nr;
4472 vcpu->arch.exception.has_error_code = events->exception.has_error_code;
4473 vcpu->arch.exception.error_code = events->exception.error_code;
4474 vcpu->arch.exception.has_payload = events->exception_has_payload;
4475 vcpu->arch.exception.payload = events->exception_payload;
4477 vcpu->arch.interrupt.injected = events->interrupt.injected;
4478 vcpu->arch.interrupt.nr = events->interrupt.nr;
4479 vcpu->arch.interrupt.soft = events->interrupt.soft;
4481 kvm_x86_ops.set_interrupt_shadow(vcpu,
4484 vcpu->arch.nmi_injected = events->nmi.injected;
4486 vcpu->arch.nmi_pending = events->nmi.pending;
4487 kvm_x86_ops.set_nmi_mask(vcpu, events->nmi.masked);
4490 lapic_in_kernel(vcpu))
4491 vcpu->arch.apic->sipi_vector = events->sipi_vector;
4494 if (!!(vcpu->arch.hflags & HF_SMM_MASK) != events->smi.smm) {
4496 vcpu->arch.hflags |= HF_SMM_MASK;
4498 vcpu->arch.hflags &= ~HF_SMM_MASK;
4500 kvm_x86_ops.nested_ops->leave_nested(vcpu);
4501 kvm_smm_changed(vcpu);
4504 vcpu->arch.smi_pending = events->smi.pending;
4508 vcpu->arch.hflags |= HF_SMM_INSIDE_NMI_MASK;
4510 vcpu->arch.hflags &= ~HF_SMM_INSIDE_NMI_MASK;
4513 if (lapic_in_kernel(vcpu)) {
4515 set_bit(KVM_APIC_INIT, &vcpu->arch.apic->pending_events);
4517 clear_bit(KVM_APIC_INIT, &vcpu->arch.apic->pending_events);
4521 kvm_make_request(KVM_REQ_EVENT, vcpu);
4526 static void kvm_vcpu_ioctl_x86_get_debugregs(struct kvm_vcpu *vcpu,
4532 memcpy(dbgregs->db, vcpu->arch.db, sizeof(vcpu->arch.db));
4533 kvm_get_dr(vcpu, 6, &val);
4535 dbgregs->dr7 = vcpu->arch.dr7;
4538 static int kvm_vcpu_ioctl_x86_set_debugregs(struct kvm_vcpu *vcpu,
4549 memcpy(vcpu->arch.db, dbgregs->db, sizeof(vcpu->arch.db));
4550 kvm_update_dr0123(vcpu);
4551 vcpu->arch.dr6 = dbgregs->dr6;
4552 vcpu->arch.dr7 = dbgregs->dr7;
4553 kvm_update_dr7(vcpu);
4560 static void fill_xsave(u8 *dest, struct kvm_vcpu *vcpu)
4562 struct xregs_state *xsave = &vcpu->arch.guest_fpu->state.xsave;
4573 xstate_bv &= vcpu->arch.guest_supported_xcr0 | XFEATURE_MASK_FPSSE;
4591 memcpy(dest + offset, &vcpu->arch.pkru,
4592 sizeof(vcpu->arch.pkru));
4602 static void load_xsave(struct kvm_vcpu *vcpu, u8 *src)
4604 struct xregs_state *xsave = &vcpu->arch.guest_fpu->state.xsave;
4634 memcpy(&vcpu->arch.pkru, src + offset,
4635 sizeof(vcpu->arch.pkru));
4644 static void kvm_vcpu_ioctl_x86_get_xsave(struct kvm_vcpu *vcpu,
4649 fill_xsave((u8 *) guest_xsave->region, vcpu);
4652 &vcpu->arch.guest_fpu->state.fxsave,
4661 static int kvm_vcpu_ioctl_x86_set_xsave(struct kvm_vcpu *vcpu,
4676 load_xsave(vcpu, (u8 *)guest_xsave->region);
4681 memcpy(&vcpu->arch.guest_fpu->state.fxsave,
4687 static void kvm_vcpu_ioctl_x86_get_xcrs(struct kvm_vcpu *vcpu,
4698 guest_xcrs->xcrs[0].value = vcpu->arch.xcr0;
4701 static int kvm_vcpu_ioctl_x86_set_xcrs(struct kvm_vcpu *vcpu,
4715 r = __kvm_set_xcr(vcpu, XCR_XFEATURE_ENABLED_MASK,
4730 static int kvm_set_guest_paused(struct kvm_vcpu *vcpu)
4732 if (!vcpu->arch.pv_time_enabled)
4734 vcpu->arch.pvclock_set_guest_stopped_request = true;
4735 kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);
4739 static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu,
4756 if (!irqchip_in_kernel(vcpu->kvm))
4758 return kvm_hv_activate_synic(vcpu, cap->cap ==
4763 r = kvm_x86_ops.nested_ops->enable_evmcs(vcpu, &vmcs_version);
4775 return kvm_x86_ops.enable_direct_tlbflush(vcpu);
4778 vcpu->arch.pv_cpuid.enforce = cap->args[0];
4779 if (vcpu->arch.pv_cpuid.enforce)
4780 kvm_update_pv_runtime(vcpu);
4792 struct kvm_vcpu *vcpu = filp->private_data;
4802 vcpu_load(vcpu);
4808 if (!lapic_in_kernel(vcpu))
4816 r = kvm_vcpu_ioctl_get_lapic(vcpu, u.lapic);
4827 if (!lapic_in_kernel(vcpu))
4835 r = kvm_vcpu_ioctl_set_lapic(vcpu, u.lapic);
4844 r = kvm_vcpu_ioctl_interrupt(vcpu, &irq);
4848 r = kvm_vcpu_ioctl_nmi(vcpu);
4852 r = kvm_vcpu_ioctl_smi(vcpu);
4862 r = kvm_vcpu_ioctl_set_cpuid(vcpu, &cpuid, cpuid_arg->entries);
4872 r = kvm_vcpu_ioctl_set_cpuid2(vcpu, &cpuid,
4883 r = kvm_vcpu_ioctl_get_cpuid2(vcpu, &cpuid,
4894 int idx = srcu_read_lock(&vcpu->kvm->srcu);
4895 r = msr_io(vcpu, argp, do_get_msr, 1);
4896 srcu_read_unlock(&vcpu->kvm->srcu, idx);
4900 int idx = srcu_read_lock(&vcpu->kvm->srcu);
4901 r = msr_io(vcpu, argp, do_set_msr, 0);
4902 srcu_read_unlock(&vcpu->kvm->srcu, idx);
4911 r = vcpu_ioctl_tpr_access_reporting(vcpu, &tac);
4925 if (!lapic_in_kernel(vcpu))
4930 idx = srcu_read_lock(&vcpu->kvm->srcu);
4931 r = kvm_lapic_set_vapic_addr(vcpu, va.vapic_addr);
4932 srcu_read_unlock(&vcpu->kvm->srcu, idx);
4941 r = kvm_vcpu_ioctl_x86_setup_mce(vcpu, mcg_cap);
4950 r = kvm_vcpu_ioctl_x86_set_mce(vcpu, &mce);
4956 kvm_vcpu_ioctl_x86_get_vcpu_events(vcpu, &events);
4971 r = kvm_vcpu_ioctl_x86_set_vcpu_events(vcpu, &events);
4977 kvm_vcpu_ioctl_x86_get_debugregs(vcpu, &dbgregs);
4994 r = kvm_vcpu_ioctl_x86_set_debugregs(vcpu, &dbgregs);
5003 kvm_vcpu_ioctl_x86_get_xsave(vcpu, u.xsave);
5018 r = kvm_vcpu_ioctl_x86_set_xsave(vcpu, u.xsave);
5027 kvm_vcpu_ioctl_x86_get_xcrs(vcpu, u.xcrs);
5043 r = kvm_vcpu_ioctl_x86_set_xcrs(vcpu, u.xcrs);
5059 if (!kvm_set_tsc_khz(vcpu, user_tsc_khz))
5065 r = vcpu->arch.virtual_tsc_khz;
5069 r = kvm_set_guest_paused(vcpu);
5078 r = kvm_vcpu_ioctl_enable_cap(vcpu, &cap);
5094 r = kvm_x86_ops.nested_ops->get_state(vcpu, user_kvm_nested_state,
5138 idx = srcu_read_lock(&vcpu->kvm->srcu);
5139 r = kvm_x86_ops.nested_ops->set_state(vcpu, user_kvm_nested_state, &kvm_state);
5140 srcu_read_unlock(&vcpu->kvm->srcu, idx);
5151 r = kvm_vcpu_ioctl_get_hv_cpuid(vcpu, &cpuid,
5168 vcpu_put(vcpu);
5172 vm_fault_t kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf)
6041 static int vcpu_mmio_write(struct kvm_vcpu *vcpu, gpa_t addr, int len,
6049 if (!(lapic_in_kernel(vcpu) &&
6050 !kvm_iodevice_write(vcpu, &vcpu->arch.apic->dev, addr, n, v))
6051 && kvm_io_bus_write(vcpu, KVM_MMIO_BUS, addr, n, v))
6062 static int vcpu_mmio_read(struct kvm_vcpu *vcpu, gpa_t addr, int len, void *v)
6069 if (!(lapic_in_kernel(vcpu) &&
6070 !kvm_iodevice_read(vcpu, &vcpu->arch.apic->dev,
6072 && kvm_io_bus_read(vcpu, KVM_MMIO_BUS, addr, n, v))
6084 static void kvm_set_segment(struct kvm_vcpu *vcpu,
6087 kvm_x86_ops.set_segment(vcpu, var, seg);
6090 void kvm_get_segment(struct kvm_vcpu *vcpu,
6093 kvm_x86_ops.get_segment(vcpu, var, seg);
6096 gpa_t translate_nested_gpa(struct kvm_vcpu *vcpu, gpa_t gpa, u32 access,
6101 BUG_ON(!mmu_is_nested(vcpu));
6105 t_gpa = vcpu->arch.mmu->gva_to_gpa(vcpu, gpa, access, exception);
6110 gpa_t kvm_mmu_gva_to_gpa_read(struct kvm_vcpu *vcpu, gva_t gva,
6113 u32 access = (kvm_x86_ops.get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0;
6114 return vcpu->arch.walk_mmu->gva_to_gpa(vcpu, gva, access, exception);
6117 gpa_t kvm_mmu_gva_to_gpa_fetch(struct kvm_vcpu *vcpu, gva_t gva,
6120 u32 access = (kvm_x86_ops.get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0;
6122 return vcpu->arch.walk_mmu->gva_to_gpa(vcpu, gva, access, exception);
6125 gpa_t kvm_mmu_gva_to_gpa_write(struct kvm_vcpu *vcpu, gva_t gva,
6128 u32 access = (kvm_x86_ops.get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0;
6130 return vcpu->arch.walk_mmu->gva_to_gpa(vcpu, gva, access, exception);
6134 gpa_t kvm_mmu_gva_to_gpa_system(struct kvm_vcpu *vcpu, gva_t gva,
6137 return vcpu->arch.walk_mmu->gva_to_gpa(vcpu, gva, 0, exception);
6141 struct kvm_vcpu *vcpu, u32 access,
6148 gpa_t gpa = vcpu->arch.walk_mmu->gva_to_gpa(vcpu, addr, access,
6156 ret = kvm_vcpu_read_guest_page(vcpu, gpa >> PAGE_SHIFT, data,
6176 struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
6177 u32 access = (kvm_x86_ops.get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0;
6182 gpa_t gpa = vcpu->arch.walk_mmu->gva_to_gpa(vcpu, addr, access|PFERR_FETCH_MASK,
6190 ret = kvm_vcpu_read_guest_page(vcpu, gpa >> PAGE_SHIFT, val,
6198 int kvm_read_guest_virt(struct kvm_vcpu *vcpu,
6202 u32 access = (kvm_x86_ops.get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0;
6211 return kvm_read_guest_virt_helper(addr, val, bytes, vcpu, access,
6220 struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
6223 if (!system && kvm_x86_ops.get_cpl(vcpu) == 3)
6226 return kvm_read_guest_virt_helper(addr, val, bytes, vcpu, access, exception);
6232 struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
6233 int r = kvm_vcpu_read_guest(vcpu, addr, val, bytes);
6239 struct kvm_vcpu *vcpu, u32 access,
6246 gpa_t gpa = vcpu->arch.walk_mmu->gva_to_gpa(vcpu, addr,
6255 ret = kvm_vcpu_write_guest(vcpu, gpa, data, towrite);
6273 struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
6276 if (!system && kvm_x86_ops.get_cpl(vcpu) == 3)
6279 return kvm_write_guest_virt_helper(addr, val, bytes, vcpu,
6283 int kvm_write_guest_virt_system(struct kvm_vcpu *vcpu, gva_t addr, void *val,
6287 vcpu->arch.l1tf_flush_l1d = true;
6289 return kvm_write_guest_virt_helper(addr, val, bytes, vcpu,
6294 int handle_ud(struct kvm_vcpu *vcpu)
6301 if (unlikely(!kvm_x86_ops.can_emulate_instruction(vcpu, NULL, 0)))
6305 kvm_read_guest_virt(vcpu, kvm_get_linear_rip(vcpu),
6308 kvm_rip_write(vcpu, kvm_rip_read(vcpu) + sizeof(sig));
6312 return kvm_emulate_instruction(vcpu, emul_type);
6316 static int vcpu_is_mmio_gpa(struct kvm_vcpu *vcpu, unsigned long gva,
6323 if (vcpu_match_mmio_gpa(vcpu, gpa)) {
6331 static int vcpu_mmio_gva_to_gpa(struct kvm_vcpu *vcpu, unsigned long gva,
6335 u32 access = ((kvm_x86_ops.get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0)
6343 if (vcpu_match_mmio_gva(vcpu, gva)
6344 && !permission_fault(vcpu, vcpu->arch.walk_mmu,
6345 vcpu->arch.mmio_access, 0, access)) {
6346 *gpa = vcpu->arch.mmio_gfn << PAGE_SHIFT |
6352 *gpa = vcpu->arch.walk_mmu->gva_to_gpa(vcpu, gva, access, exception);
6357 return vcpu_is_mmio_gpa(vcpu, gva, *gpa, write);
6360 int emulator_write_phys(struct kvm_vcpu *vcpu, gpa_t gpa,
6365 ret = kvm_vcpu_write_guest(vcpu, gpa, val, bytes);
6368 kvm_page_track_write(vcpu, gpa, val, bytes);
6373 int (*read_write_prepare)(struct kvm_vcpu *vcpu, void *val,
6375 int (*read_write_emulate)(struct kvm_vcpu *vcpu, gpa_t gpa,
6377 int (*read_write_mmio)(struct kvm_vcpu *vcpu, gpa_t gpa,
6379 int (*read_write_exit_mmio)(struct kvm_vcpu *vcpu, gpa_t gpa,
6384 static int read_prepare(struct kvm_vcpu *vcpu, void *val, int bytes)
6386 if (vcpu->mmio_read_completed) {
6388 vcpu->mmio_fragments[0].gpa, val);
6389 vcpu->mmio_read_completed = 0;
6396 static int read_emulate(struct kvm_vcpu *vcpu, gpa_t gpa,
6399 return !kvm_vcpu_read_guest(vcpu, gpa, val, bytes);
6402 static int write_emulate(struct kvm_vcpu *vcpu, gpa_t gpa,
6405 return emulator_write_phys(vcpu, gpa, val, bytes);
6408 static int write_mmio(struct kvm_vcpu *vcpu, gpa_t gpa, int bytes, void *val)
6411 return vcpu_mmio_write(vcpu, gpa, bytes, val);
6414 static int read_exit_mmio(struct kvm_vcpu *vcpu, gpa_t gpa,
6421 static int write_exit_mmio(struct kvm_vcpu *vcpu, gpa_t gpa,
6424 struct kvm_mmio_fragment *frag = &vcpu->mmio_fragments[0];
6426 memcpy(vcpu->run->mmio.data, frag->data, min(8u, frag->len));
6447 struct kvm_vcpu *vcpu,
6454 struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt;
6466 ret = vcpu_is_mmio_gpa(vcpu, addr, gpa, write);
6468 ret = vcpu_mmio_gva_to_gpa(vcpu, addr, &gpa, exception, write);
6473 if (!ret && ops->read_write_emulate(vcpu, gpa, val, bytes))
6479 handled = ops->read_write_mmio(vcpu, gpa, bytes, val);
6487 WARN_ON(vcpu->mmio_nr_fragments >= KVM_MAX_MMIO_FRAGMENTS);
6488 frag = &vcpu->mmio_fragments[vcpu->mmio_nr_fragments++];
6501 struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
6506 ops->read_write_prepare(vcpu, val, bytes))
6509 vcpu->mmio_nr_fragments = 0;
6517 vcpu, ops);
6529 vcpu, ops);
6533 if (!vcpu->mmio_nr_fragments)
6536 gpa = vcpu->mmio_fragments[0].gpa;
6538 vcpu->mmio_needed = 1;
6539 vcpu->mmio_cur_fragment = 0;
6541 vcpu->run->mmio.len = min(8u, vcpu->mmio_fragments[0].len);
6542 vcpu->run->mmio.is_write = vcpu->mmio_is_write = ops->write;
6543 vcpu->run->exit_reason = KVM_EXIT_MMIO;
6544 vcpu->run->mmio.phys_addr = gpa;
6546 return ops->read_write_exit_mmio(vcpu, gpa, val, bytes);
6587 struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
6597 gpa = kvm_mmu_gva_to_gpa_write(vcpu, addr, NULL);
6615 if (kvm_vcpu_map(vcpu, gpa_to_gfn(gpa), &map))
6637 kvm_vcpu_unmap(vcpu, &map, true);
6642 kvm_page_track_write(vcpu, gpa, new, bytes);
6652 static int kernel_pio(struct kvm_vcpu *vcpu, void *pd)
6656 for (i = 0; i < vcpu->arch.pio.count; i++) {
6657 if (vcpu->arch.pio.in)
6658 r = kvm_io_bus_read(vcpu, KVM_PIO_BUS, vcpu->arch.pio.port,
6659 vcpu->arch.pio.size, pd);
6661 r = kvm_io_bus_write(vcpu, KVM_PIO_BUS,
6662 vcpu->arch.pio.port, vcpu->arch.pio.size,
6666 pd += vcpu->arch.pio.size;
6671 static int emulator_pio_in_out(struct kvm_vcpu *vcpu, int size,
6675 vcpu->arch.pio.port = port;
6676 vcpu->arch.pio.in = in;
6677 vcpu->arch.pio.count = count;
6678 vcpu->arch.pio.size = size;
6680 if (!kernel_pio(vcpu, vcpu->arch.pio_data)) {
6681 vcpu->arch.pio.count = 0;
6685 vcpu->run->exit_reason = KVM_EXIT_IO;
6686 vcpu->run->io.direction = in ? KVM_EXIT_IO_IN : KVM_EXIT_IO_OUT;
6687 vcpu->run->io.size = size;
6688 vcpu->run->io.data_offset = KVM_PIO_PAGE_OFFSET * PAGE_SIZE;
6689 vcpu->run->io.count = count;
6690 vcpu->run->io.port = port;
6695 static int emulator_pio_in(struct kvm_vcpu *vcpu, int size,
6700 if (vcpu->arch.pio.count)
6703 memset(vcpu->arch.pio_data, 0, size * count);
6705 ret = emulator_pio_in_out(vcpu, size, port, val, count, true);
6708 memcpy(val, vcpu->arch.pio_data, size * count);
6709 trace_kvm_pio(KVM_PIO_IN, port, size, count, vcpu->arch.pio_data);
6710 vcpu->arch.pio.count = 0;
6725 static int emulator_pio_out(struct kvm_vcpu *vcpu, int size,
6729 memcpy(vcpu->arch.pio_data, val, size * count);
6730 trace_kvm_pio(KVM_PIO_OUT, port, size, count, vcpu->arch.pio_data);
6731 return emulator_pio_in_out(vcpu, size, port, (void *)val, count, false);
6741 static unsigned long get_segment_base(struct kvm_vcpu *vcpu, int seg)
6743 return kvm_x86_ops.get_segment_base(vcpu, seg);
6751 static int kvm_emulate_wbinvd_noskip(struct kvm_vcpu *vcpu)
6753 if (!need_emulate_wbinvd(vcpu))
6759 cpumask_set_cpu(cpu, vcpu->arch.wbinvd_dirty_mask);
6760 smp_call_function_many(vcpu->arch.wbinvd_dirty_mask,
6763 cpumask_clear(vcpu->arch.wbinvd_dirty_mask);
6769 int kvm_emulate_wbinvd(struct kvm_vcpu *vcpu)
6771 kvm_emulate_wbinvd_noskip(vcpu);
6772 return kvm_skip_emulated_instruction(vcpu);
6803 struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
6808 value = kvm_read_cr0(vcpu);
6811 value = vcpu->arch.cr2;
6814 value = kvm_read_cr3(vcpu);
6817 value = kvm_read_cr4(vcpu);
6820 value = kvm_get_cr8(vcpu);
6832 struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
6837 res = kvm_set_cr0(vcpu, mk_cr_64(kvm_read_cr0(vcpu), val));
6840 vcpu->arch.cr2 = val;
6843 res = kvm_set_cr3(vcpu, val);
6846 res = kvm_set_cr4(vcpu, mk_cr_64(kvm_read_cr4(vcpu), val));
6849 res = kvm_set_cr8(vcpu, val);
6930 struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
6952 kvm_set_segment(vcpu, &var, seg);
6959 struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
6962 r = kvm_get_msr(vcpu, msr_index, pdata);
6964 if (r && kvm_get_msr_user_space(vcpu, msr_index, r)) {
6975 struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
6978 r = kvm_set_msr(vcpu, msr_index, data);
6980 if (r && kvm_set_msr_user_space(vcpu, msr_index, data, r)) {
6990 struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
6992 return vcpu->arch.smbase;
6997 struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
6999 vcpu->arch.smbase = smbase;
7076 struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
7078 vcpu->arch.hflags = emul_flags;
7079 kvm_mmu_reset_context(vcpu);
7146 static void toggle_interruptibility(struct kvm_vcpu *vcpu, u32 mask)
7148 u32 int_shadow = kvm_x86_ops.get_interrupt_shadow(vcpu);
7159 kvm_x86_ops.set_interrupt_shadow(vcpu, mask);
7161 kvm_make_request(KVM_REQ_EVENT, vcpu);
7165 static bool inject_emulated_exception(struct kvm_vcpu *vcpu)
7167 struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt;
7169 return kvm_inject_emulated_page_fault(vcpu, &ctxt->exception);
7172 kvm_queue_exception_e(vcpu, ctxt->exception.vector,
7175 kvm_queue_exception(vcpu, ctxt->exception.vector);
7179 static struct x86_emulate_ctxt *alloc_emulate_ctxt(struct kvm_vcpu *vcpu)
7185 pr_err("kvm: failed to allocate vcpu's emulator\n");
7189 ctxt->vcpu = vcpu;
7191 vcpu->arch.emulate_ctxt = ctxt;
7196 static void init_emulate_ctxt(struct kvm_vcpu *vcpu)
7198 struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt;
7201 kvm_x86_ops.get_cs_db_l_bits(vcpu, &cs_db, &cs_l);
7204 ctxt->eflags = kvm_get_rflags(vcpu);
7207 ctxt->eip = kvm_rip_read(vcpu);
7208 ctxt->mode = (!is_protmode(vcpu)) ? X86EMUL_MODE_REAL :
7210 (cs_l && is_long_mode(vcpu)) ? X86EMUL_MODE_PROT64 :
7223 vcpu->arch.emulate_regs_need_sync_from_vcpu = false;
7226 void kvm_inject_realmode_interrupt(struct kvm_vcpu *vcpu, int irq, int inc_eip)
7228 struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt;
7231 init_emulate_ctxt(vcpu);
7239 kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);
7242 kvm_rip_write(vcpu, ctxt->eip);
7243 kvm_set_rflags(vcpu, ctxt->eflags);
7248 static int handle_emulation_failure(struct kvm_vcpu *vcpu, int emulation_type)
7250 ++vcpu->stat.insn_emulation_fail;
7251 trace_kvm_emulate_insn_failed(vcpu);
7254 kvm_queue_exception_e(vcpu, GP_VECTOR, 0);
7259 vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
7260 vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_EMULATION;
7261 vcpu->run->internal.ndata = 0;
7265 kvm_queue_exception(vcpu, UD_VECTOR);
7267 if (!is_guest_mode(vcpu) && kvm_x86_ops.get_cpl(vcpu) == 0) {
7268 vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
7269 vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_EMULATION;
7270 vcpu->run->internal.ndata = 0;
7277 static bool reexecute_instruction(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa,
7287 if (WARN_ON_ONCE(is_guest_mode(vcpu)) ||
7291 if (!vcpu->arch.mmu->direct_map) {
7296 gpa = kvm_mmu_gva_to_gpa_write(vcpu, cr2_or_gpa, NULL);
7312 pfn = gfn_to_pfn(vcpu->kvm, gpa_to_gfn(gpa));
7324 if (vcpu->arch.mmu->direct_map) {
7327 spin_lock(&vcpu->kvm->mmu_lock);
7328 indirect_shadow_pages = vcpu->kvm->arch.indirect_shadow_pages;
7329 spin_unlock(&vcpu->kvm->mmu_lock);
7332 kvm_mmu_unprotect_page(vcpu->kvm, gpa_to_gfn(gpa));
7342 kvm_mmu_unprotect_page(vcpu->kvm, gpa_to_gfn(gpa));
7355 struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
7358 last_retry_eip = vcpu->arch.last_retry_eip;
7359 last_retry_addr = vcpu->arch.last_retry_addr;
7374 vcpu->arch.last_retry_eip = vcpu->arch.last_retry_addr = 0;
7379 if (WARN_ON_ONCE(is_guest_mode(vcpu)) ||
7389 vcpu->arch.last_retry_eip = ctxt->eip;
7390 vcpu->arch.last_retry_addr = cr2_or_gpa;
7392 if (!vcpu->arch.mmu->direct_map)
7393 gpa = kvm_mmu_gva_to_gpa_write(vcpu, cr2_or_gpa, NULL);
7395 kvm_mmu_unprotect_page(vcpu->kvm, gpa_to_gfn(gpa));
7400 static int complete_emulated_mmio(struct kvm_vcpu *vcpu);
7401 static int complete_emulated_pio(struct kvm_vcpu *vcpu);
7403 static void kvm_smm_changed(struct kvm_vcpu *vcpu)
7405 if (!(vcpu->arch.hflags & HF_SMM_MASK)) {
7407 trace_kvm_enter_smm(vcpu->vcpu_id, vcpu->arch.smbase, false);
7410 kvm_make_request(KVM_REQ_EVENT, vcpu);
7413 kvm_mmu_reset_context(vcpu);
7431 static int kvm_vcpu_do_singlestep(struct kvm_vcpu *vcpu)
7433 struct kvm_run *kvm_run = vcpu->run;
7435 if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP) {
7437 kvm_run->debug.arch.pc = kvm_get_linear_rip(vcpu);
7442 kvm_queue_exception_p(vcpu, DB_VECTOR, DR6_BS);
7446 int kvm_skip_emulated_instruction(struct kvm_vcpu *vcpu)
7448 unsigned long rflags = kvm_x86_ops.get_rflags(vcpu);
7451 r = kvm_x86_ops.skip_emulated_instruction(vcpu);
7464 r = kvm_vcpu_do_singlestep(vcpu);
7469 static bool kvm_vcpu_check_code_breakpoint(struct kvm_vcpu *vcpu, int *r)
7471 if (unlikely(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) &&
7472 (vcpu->arch.guest_debug_dr7 & DR7_BP_EN_MASK)) {
7473 struct kvm_run *kvm_run = vcpu->run;
7474 unsigned long eip = kvm_get_linear_rip(vcpu);
7476 vcpu->arch.guest_debug_dr7,
7477 vcpu->arch.eff_db);
7489 if (unlikely(vcpu->arch.dr7 & DR7_BP_EN_MASK) &&
7490 !(kvm_get_rflags(vcpu) & X86_EFLAGS_RF)) {
7491 unsigned long eip = kvm_get_linear_rip(vcpu);
7493 vcpu->arch.dr7,
7494 vcpu->arch.db);
7497 kvm_queue_exception_p(vcpu, DB_VECTOR, dr6);
7547 int x86_decode_emulated_instruction(struct kvm_vcpu *vcpu, int emulation_type,
7550 struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt;
7553 init_emulate_ctxt(vcpu);
7559 trace_kvm_emulate_insn_start(vcpu);
7560 ++vcpu->stat.insn_emulation;
7566 int x86_emulate_instruction(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa,
7570 struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt;
7574 if (unlikely(!kvm_x86_ops.can_emulate_instruction(vcpu, insn, insn_len)))
7577 vcpu->arch.l1tf_flush_l1d = true;
7583 write_fault_to_spt = vcpu->arch.write_fault_to_shadow_pgtable;
7584 vcpu->arch.write_fault_to_shadow_pgtable = false;
7587 kvm_clear_exception_queue(vcpu);
7595 kvm_vcpu_check_code_breakpoint(vcpu, &r))
7598 r = x86_decode_emulated_instruction(vcpu, emulation_type,
7603 kvm_queue_exception(vcpu, UD_VECTOR);
7606 if (reexecute_instruction(vcpu, cr2_or_gpa,
7619 inject_emulated_exception(vcpu);
7622 return handle_emulation_failure(vcpu, emulation_type);
7628 kvm_queue_exception_e(vcpu, GP_VECTOR, 0);
7638 kvm_rip_write(vcpu, ctxt->_eip);
7640 kvm_set_rflags(vcpu, ctxt->eflags & ~X86_EFLAGS_RF);
7649 if (vcpu->arch.emulate_regs_need_sync_from_vcpu) {
7650 vcpu->arch.emulate_regs_need_sync_from_vcpu = false;
7660 if (vcpu->arch.mmu->direct_map) {
7675 if (reexecute_instruction(vcpu, cr2_or_gpa, write_fault_to_spt,
7679 return handle_emulation_failure(vcpu, emulation_type);
7684 if (inject_emulated_exception(vcpu))
7686 } else if (vcpu->arch.pio.count) {
7687 if (!vcpu->arch.pio.in) {
7689 vcpu->arch.pio.count = 0;
7692 vcpu->arch.complete_userspace_io = complete_emulated_pio;
7695 } else if (vcpu->mmio_needed) {
7696 ++vcpu->stat.mmio_exits;
7698 if (!vcpu->mmio_is_write)
7701 vcpu->arch.complete_userspace_io = complete_emulated_mmio;
7708 unsigned long rflags = kvm_x86_ops.get_rflags(vcpu);
7709 toggle_interruptibility(vcpu, ctxt->interruptibility);
7710 vcpu->arch.emulate_regs_need_sync_to_vcpu = false;
7719 kvm_rip_write(vcpu, ctxt->eip);
7720 if (r && (ctxt->tf || (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP)))
7721 r = kvm_vcpu_do_singlestep(vcpu);
7723 kvm_x86_ops.update_emulated_instruction(vcpu);
7724 __kvm_set_rflags(vcpu, ctxt->eflags);
7734 kvm_make_request(KVM_REQ_EVENT, vcpu);
7736 vcpu->arch.emulate_regs_need_sync_to_vcpu = true;
7741 int kvm_emulate_instruction(struct kvm_vcpu *vcpu, int emulation_type)
7743 return x86_emulate_instruction(vcpu, 0, emulation_type, NULL, 0);
7747 int kvm_emulate_instruction_from_buffer(struct kvm_vcpu *vcpu,
7750 return x86_emulate_instruction(vcpu, 0, 0, insn, insn_len);
7754 static int complete_fast_pio_out_port_0x7e(struct kvm_vcpu *vcpu)
7756 vcpu->arch.pio.count = 0;
7760 static int complete_fast_pio_out(struct kvm_vcpu *vcpu)
7762 vcpu->arch.pio.count = 0;
7764 if (unlikely(!kvm_is_linear_rip(vcpu, vcpu->arch.pio.linear_rip)))
7767 return kvm_skip_emulated_instruction(vcpu);
7770 static int kvm_fast_pio_out(struct kvm_vcpu *vcpu, int size,
7773 unsigned long val = kvm_rax_read(vcpu);
7774 int ret = emulator_pio_out(vcpu, size, port, &val, 1);
7784 kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_OUT_7E_INC_RIP)) {
7785 vcpu->arch.complete_userspace_io =
7787 kvm_skip_emulated_instruction(vcpu);
7789 vcpu->arch.pio.linear_rip = kvm_get_linear_rip(vcpu);
7790 vcpu->arch.complete_userspace_io = complete_fast_pio_out;
7795 static int complete_fast_pio_in(struct kvm_vcpu *vcpu)
7800 BUG_ON(vcpu->arch.pio.count != 1);
7802 if (unlikely(!kvm_is_linear_rip(vcpu, vcpu->arch.pio.linear_rip))) {
7803 vcpu->arch.pio.count = 0;
7808 val = (vcpu->arch.pio.size < 4) ? kvm_rax_read(vcpu) : 0;
7811 * Since vcpu->arch.pio.count == 1 let emulator_pio_in perform
7814 emulator_pio_in(vcpu, vcpu->arch.pio.size, vcpu->arch.pio.port, &val, 1);
7815 kvm_rax_write(vcpu, val);
7817 return kvm_skip_emulated_instruction(vcpu);
7820 static int kvm_fast_pio_in(struct kvm_vcpu *vcpu, int size,
7827 val = (size < 4) ? kvm_rax_read(vcpu) : 0;
7829 ret = emulator_pio_in(vcpu, size, port, &val, 1);
7831 kvm_rax_write(vcpu, val);
7835 vcpu->arch.pio.linear_rip = kvm_get_linear_rip(vcpu);
7836 vcpu->arch.complete_userspace_io = complete_fast_pio_in;
7841 int kvm_fast_pio(struct kvm_vcpu *vcpu, int size, unsigned short port, int in)
7846 ret = kvm_fast_pio_in(vcpu, size, port);
7848 ret = kvm_fast_pio_out(vcpu, size, port);
7849 return ret && kvm_skip_emulated_instruction(vcpu);
7877 struct kvm_vcpu *vcpu;
7898 kvm_for_each_vcpu(cpu, vcpu, kvm)
7899 kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);
7901 kvm_for_each_vcpu(cpu, vcpu, kvm)
7902 kvm_clear_request(KVM_REQ_MCLOCK_INPROGRESS, vcpu);
7913 struct kvm_vcpu *vcpu;
7959 kvm_for_each_vcpu(i, vcpu, kvm) {
7960 if (vcpu->cpu != cpu)
7962 kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);
7963 if (vcpu->cpu != raw_smp_processor_id())
8069 struct kvm_vcpu *vcpu = __this_cpu_read(current_vcpu);
8071 kvm_make_request(KVM_REQ_PMI, vcpu);
8073 (unsigned long *)&vcpu->arch.pmu.global_status);
8088 struct kvm_vcpu *vcpu;
8093 kvm_for_each_vcpu(i, vcpu, kvm)
8094 kvm_make_request(KVM_REQ_MASTERCLOCK_UPDATE, vcpu);
8260 int kvm_vcpu_halt(struct kvm_vcpu *vcpu)
8262 ++vcpu->stat.halt_exits;
8263 if (lapic_in_kernel(vcpu)) {
8264 vcpu->arch.mp_state = KVM_MP_STATE_HALTED;
8267 vcpu->run->exit_reason = KVM_EXIT_HLT;
8273 int kvm_emulate_halt(struct kvm_vcpu *vcpu)
8275 int ret = kvm_skip_emulated_instruction(vcpu);
8280 return kvm_vcpu_halt(vcpu) && ret;
8285 static int kvm_pv_clock_pairing(struct kvm_vcpu *vcpu, gpa_t paddr,
8301 clock_pairing.tsc = kvm_read_l1_tsc(vcpu, cycle);
8306 if (kvm_write_guest(vcpu->kvm, paddr, &clock_pairing,
8315 * kvm_pv_kick_cpu_op: Kick a vcpu.
8317 * @apicid - apicid of vcpu to be kicked.
8361 target = map->phys_map[dest_id]->vcpu;
8369 int kvm_emulate_hypercall(struct kvm_vcpu *vcpu)
8374 if (kvm_hv_hypercall_enabled(vcpu->kvm))
8375 return kvm_hv_hypercall(vcpu);
8377 nr = kvm_rax_read(vcpu);
8378 a0 = kvm_rbx_read(vcpu);
8379 a1 = kvm_rcx_read(vcpu);
8380 a2 = kvm_rdx_read(vcpu);
8381 a3 = kvm_rsi_read(vcpu);
8385 op_64_bit = is_64_bit_mode(vcpu);
8394 if (kvm_x86_ops.get_cpl(vcpu) != 0) {
8406 if (!guest_pv_has(vcpu, KVM_FEATURE_PV_UNHALT))
8409 kvm_pv_kick_cpu_op(vcpu->kvm, a0, a1);
8410 kvm_sched_yield(vcpu->kvm, a1);
8415 ret = kvm_pv_clock_pairing(vcpu, a0, a1);
8419 if (!guest_pv_has(vcpu, KVM_FEATURE_PV_SEND_IPI))
8422 ret = kvm_pv_send_ipi(vcpu->kvm, a0, a1, a2, a3, op_64_bit);
8425 if (!guest_pv_has(vcpu, KVM_FEATURE_PV_SCHED_YIELD))
8428 kvm_sched_yield(vcpu->kvm, a0);
8438 kvm_rax_write(vcpu, ret);
8440 ++vcpu->stat.hypercalls;
8441 return kvm_skip_emulated_instruction(vcpu);
8447 struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
8449 unsigned long rip = kvm_rip_read(vcpu);
8451 kvm_x86_ops.patch_hypercall(vcpu, instruction);
8457 static int dm_request_for_irq_injection(struct kvm_vcpu *vcpu)
8459 return vcpu->run->request_interrupt_window &&
8460 likely(!pic_in_kernel(vcpu->kvm));
8463 static void post_kvm_run_save(struct kvm_vcpu *vcpu)
8465 struct kvm_run *kvm_run = vcpu->run;
8467 kvm_run->if_flag = (kvm_get_rflags(vcpu) & X86_EFLAGS_IF) != 0;
8468 kvm_run->flags = is_smm(vcpu) ? KVM_RUN_X86_SMM : 0;
8469 kvm_run->cr8 = kvm_get_cr8(vcpu);
8470 kvm_run->apic_base = kvm_get_apic_base(vcpu);
8472 pic_in_kernel(vcpu->kvm) ||
8473 kvm_vcpu_ready_for_interrupt_injection(vcpu);
8476 static void update_cr8_intercept(struct kvm_vcpu *vcpu)
8483 if (!lapic_in_kernel(vcpu))
8486 if (vcpu->arch.apicv_active)
8489 if (!vcpu->arch.apic->vapic_addr)
8490 max_irr = kvm_lapic_find_highest_irr(vcpu);
8497 tpr = kvm_lapic_get_cr8(vcpu);
8499 kvm_x86_ops.update_cr8_intercept(vcpu, tpr, max_irr);
8502 static void kvm_inject_exception(struct kvm_vcpu *vcpu)
8504 trace_kvm_inj_exception(vcpu->arch.exception.nr,
8505 vcpu->arch.exception.has_error_code,
8506 vcpu->arch.exception.error_code,
8507 vcpu->arch.exception.injected);
8509 if (vcpu->arch.exception.error_code && !is_protmode(vcpu))
8510 vcpu->arch.exception.error_code = false;
8511 kvm_x86_ops.queue_exception(vcpu);
8514 static void inject_pending_event(struct kvm_vcpu *vcpu, bool *req_immediate_exit)
8521 if (vcpu->arch.exception.injected) {
8522 kvm_inject_exception(vcpu);
8539 else if (!vcpu->arch.exception.pending) {
8540 if (vcpu->arch.nmi_injected) {
8541 kvm_x86_ops.set_nmi(vcpu);
8543 } else if (vcpu->arch.interrupt.injected) {
8544 kvm_x86_ops.set_irq(vcpu);
8549 WARN_ON_ONCE(vcpu->arch.exception.injected &&
8550 vcpu->arch.exception.pending);
8558 if (is_guest_mode(vcpu)) {
8559 r = kvm_x86_ops.nested_ops->check_events(vcpu);
8565 if (vcpu->arch.exception.pending) {
8576 if (exception_type(vcpu->arch.exception.nr) == EXCPT_FAULT)
8577 __kvm_set_rflags(vcpu, kvm_get_rflags(vcpu) |
8580 if (vcpu->arch.exception.nr == DB_VECTOR) {
8581 kvm_deliver_exception_payload(vcpu);
8582 if (vcpu->arch.dr7 & DR7_GD) {
8583 vcpu->arch.dr7 &= ~DR7_GD;
8584 kvm_update_dr7(vcpu);
8588 kvm_inject_exception(vcpu);
8590 vcpu->arch.exception.pending = false;
8591 vcpu->arch.exception.injected = true;
8607 if (vcpu->arch.smi_pending) {
8608 r = can_inject ? kvm_x86_ops.smi_allowed(vcpu, true) : -EBUSY;
8612 vcpu->arch.smi_pending = false;
8613 ++vcpu->arch.smi_count;
8614 enter_smm(vcpu);
8617 kvm_x86_ops.enable_smi_window(vcpu);
8620 if (vcpu->arch.nmi_pending) {
8621 r = can_inject ? kvm_x86_ops.nmi_allowed(vcpu, true) : -EBUSY;
8625 --vcpu->arch.nmi_pending;
8626 vcpu->arch.nmi_injected = true;
8627 kvm_x86_ops.set_nmi(vcpu);
8629 WARN_ON(kvm_x86_ops.nmi_allowed(vcpu, true) < 0);
8631 if (vcpu->arch.nmi_pending)
8632 kvm_x86_ops.enable_nmi_window(vcpu);
8635 if (kvm_cpu_has_injectable_intr(vcpu)) {
8636 r = can_inject ? kvm_x86_ops.interrupt_allowed(vcpu, true) : -EBUSY;
8640 kvm_queue_interrupt(vcpu, kvm_cpu_get_interrupt(vcpu), false);
8641 kvm_x86_ops.set_irq(vcpu);
8642 WARN_ON(kvm_x86_ops.interrupt_allowed(vcpu, true) < 0);
8644 if (kvm_cpu_has_injectable_intr(vcpu))
8645 kvm_x86_ops.enable_irq_window(vcpu);
8648 if (is_guest_mode(vcpu) &&
8650 kvm_x86_ops.nested_ops->hv_timer_pending(vcpu))
8653 WARN_ON(vcpu->arch.exception.pending);
8661 static void process_nmi(struct kvm_vcpu *vcpu)
8670 if (kvm_x86_ops.get_nmi_mask(vcpu) || vcpu->arch.nmi_injected)
8673 vcpu->arch.nmi_pending += atomic_xchg(&vcpu->arch.nmi_queued, 0);
8674 vcpu->arch.nmi_pending = min(vcpu->arch.nmi_pending, limit);
8675 kvm_make_request(KVM_REQ_EVENT, vcpu);
8692 static void enter_smm_save_seg_32(struct kvm_vcpu *vcpu, char *buf, int n)
8697 kvm_get_segment(vcpu, &seg, n);
8711 static void enter_smm_save_seg_64(struct kvm_vcpu *vcpu, char *buf, int n)
8717 kvm_get_segment(vcpu, &seg, n);
8728 static void enter_smm_save_state_32(struct kvm_vcpu *vcpu, char *buf)
8735 put_smstate(u32, buf, 0x7ffc, kvm_read_cr0(vcpu));
8736 put_smstate(u32, buf, 0x7ff8, kvm_read_cr3(vcpu));
8737 put_smstate(u32, buf, 0x7ff4, kvm_get_rflags(vcpu));
8738 put_smstate(u32, buf, 0x7ff0, kvm_rip_read(vcpu));
8741 put_smstate(u32, buf, 0x7fd0 + i * 4, kvm_register_read(vcpu, i));
8743 kvm_get_dr(vcpu, 6, &val);
8745 kvm_get_dr(vcpu, 7, &val);
8748 kvm_get_segment(vcpu, &seg, VCPU_SREG_TR);
8754 kvm_get_segment(vcpu, &seg, VCPU_SREG_LDTR);
8760 kvm_x86_ops.get_gdt(vcpu, &dt);
8764 kvm_x86_ops.get_idt(vcpu, &dt);
8769 enter_smm_save_seg_32(vcpu, buf, i);
8771 put_smstate(u32, buf, 0x7f14, kvm_read_cr4(vcpu));
8775 put_smstate(u32, buf, 0x7ef8, vcpu->arch.smbase);
8779 static void enter_smm_save_state_64(struct kvm_vcpu *vcpu, char *buf)
8787 put_smstate(u64, buf, 0x7ff8 - i * 8, kvm_register_read(vcpu, i));
8789 put_smstate(u64, buf, 0x7f78, kvm_rip_read(vcpu));
8790 put_smstate(u32, buf, 0x7f70, kvm_get_rflags(vcpu));
8792 kvm_get_dr(vcpu, 6, &val);
8794 kvm_get_dr(vcpu, 7, &val);
8797 put_smstate(u64, buf, 0x7f58, kvm_read_cr0(vcpu));
8798 put_smstate(u64, buf, 0x7f50, kvm_read_cr3(vcpu));
8799 put_smstate(u64, buf, 0x7f48, kvm_read_cr4(vcpu));
8801 put_smstate(u32, buf, 0x7f00, vcpu->arch.smbase);
8806 put_smstate(u64, buf, 0x7ed0, vcpu->arch.efer);
8808 kvm_get_segment(vcpu, &seg, VCPU_SREG_TR);
8814 kvm_x86_ops.get_idt(vcpu, &dt);
8818 kvm_get_segment(vcpu, &seg, VCPU_SREG_LDTR);
8824 kvm_x86_ops.get_gdt(vcpu, &dt);
8829 enter_smm_save_seg_64(vcpu, buf, i);
8833 static void enter_smm(struct kvm_vcpu *vcpu)
8840 trace_kvm_enter_smm(vcpu->vcpu_id, vcpu->arch.smbase, true);
8843 if (guest_cpuid_has(vcpu, X86_FEATURE_LM))
8844 enter_smm_save_state_64(vcpu, buf);
8847 enter_smm_save_state_32(vcpu, buf);
8854 kvm_x86_ops.pre_enter_smm(vcpu, buf);
8856 vcpu->arch.hflags |= HF_SMM_MASK;
8857 kvm_vcpu_write_guest(vcpu, vcpu->arch.smbase + 0xfe00, buf, sizeof(buf));
8859 if (kvm_x86_ops.get_nmi_mask(vcpu))
8860 vcpu->arch.hflags |= HF_SMM_INSIDE_NMI_MASK;
8862 kvm_x86_ops.set_nmi_mask(vcpu, true);
8864 kvm_set_rflags(vcpu, X86_EFLAGS_FIXED);
8865 kvm_rip_write(vcpu, 0x8000);
8867 cr0 = vcpu->arch.cr0 & ~(X86_CR0_PE | X86_CR0_EM | X86_CR0_TS | X86_CR0_PG);
8868 kvm_x86_ops.set_cr0(vcpu, cr0);
8869 vcpu->arch.cr0 = cr0;
8871 kvm_x86_ops.set_cr4(vcpu, 0);
8875 kvm_x86_ops.set_idt(vcpu, &dt);
8877 __kvm_set_dr(vcpu, 7, DR7_FIXED_1);
8879 cs.selector = (vcpu->arch.smbase >> 4) & 0xffff;
8880 cs.base = vcpu->arch.smbase;
8897 kvm_set_segment(vcpu, &cs, VCPU_SREG_CS);
8898 kvm_set_segment(vcpu, &ds, VCPU_SREG_DS);
8899 kvm_set_segment(vcpu, &ds, VCPU_SREG_ES);
8900 kvm_set_segment(vcpu, &ds, VCPU_SREG_FS);
8901 kvm_set_segment(vcpu, &ds, VCPU_SREG_GS);
8902 kvm_set_segment(vcpu, &ds, VCPU_SREG_SS);
8905 if (guest_cpuid_has(vcpu, X86_FEATURE_LM))
8906 kvm_x86_ops.set_efer(vcpu, 0);
8909 kvm_update_cpuid_runtime(vcpu);
8910 kvm_mmu_reset_context(vcpu);
8913 static void process_smi(struct kvm_vcpu *vcpu)
8915 vcpu->arch.smi_pending = true;
8916 kvm_make_request(KVM_REQ_EVENT, vcpu);
8937 void kvm_vcpu_update_apicv(struct kvm_vcpu *vcpu)
8939 if (!lapic_in_kernel(vcpu))
8942 vcpu->arch.apicv_active = kvm_apicv_activated(vcpu->kvm);
8943 kvm_apic_update_apicv(vcpu);
8944 kvm_x86_ops.refresh_apicv_exec_ctrl(vcpu);
8985 * while update the calling vcpu immediately instead of
8996 static void vcpu_scan_ioapic(struct kvm_vcpu *vcpu)
8998 if (!kvm_apic_present(vcpu))
9001 bitmap_zero(vcpu->arch.ioapic_handled_vectors, 256);
9003 if (irqchip_split(vcpu->kvm))
9004 kvm_scan_ioapic_routes(vcpu, vcpu->arch.ioapic_handled_vectors);
9006 if (vcpu->arch.apicv_active)
9007 kvm_x86_ops.sync_pir_to_irr(vcpu);
9008 if (ioapic_in_kernel(vcpu->kvm))
9009 kvm_ioapic_scan_entry(vcpu, vcpu->arch.ioapic_handled_vectors);
9012 if (is_guest_mode(vcpu))
9013 vcpu->arch.load_eoi_exitmap_pending = true;
9015 kvm_make_request(KVM_REQ_LOAD_EOI_EXITMAP, vcpu);
9018 static void vcpu_load_eoi_exitmap(struct kvm_vcpu *vcpu)
9022 if (!kvm_apic_hw_enabled(vcpu->arch.apic))
9025 bitmap_or((ulong *)eoi_exit_bitmap, vcpu->arch.ioapic_handled_vectors,
9026 vcpu_to_synic(vcpu)->vec_bitmap, 256);
9027 kvm_x86_ops.load_eoi_exitmap(vcpu, eoi_exit_bitmap);
9050 void kvm_vcpu_reload_apic_access_page(struct kvm_vcpu *vcpu)
9052 if (!lapic_in_kernel(vcpu))
9058 kvm_x86_ops.set_apic_access_page_addr(vcpu);
9061 void __kvm_request_immediate_exit(struct kvm_vcpu *vcpu)
9063 smp_send_reschedule(vcpu->cpu);
9072 static int vcpu_enter_guest(struct kvm_vcpu *vcpu)
9076 dm_request_for_irq_injection(vcpu) &&
9077 kvm_cpu_accept_dm_intr(vcpu);
9082 if (kvm_request_pending(vcpu)) {
9083 if (kvm_check_request(KVM_REQ_GET_NESTED_STATE_PAGES, vcpu)) {
9084 if (unlikely(!kvm_x86_ops.nested_ops->get_nested_state_pages(vcpu))) {
9089 if (kvm_check_request(KVM_REQ_MMU_RELOAD, vcpu))
9090 kvm_mmu_unload(vcpu);
9091 if (kvm_check_request(KVM_REQ_MIGRATE_TIMER, vcpu))
9092 __kvm_migrate_timers(vcpu);
9093 if (kvm_check_request(KVM_REQ_MASTERCLOCK_UPDATE, vcpu))
9094 kvm_gen_update_masterclock(vcpu->kvm);
9095 if (kvm_check_request(KVM_REQ_GLOBAL_CLOCK_UPDATE, vcpu))
9096 kvm_gen_kvmclock_update(vcpu);
9097 if (kvm_check_request(KVM_REQ_CLOCK_UPDATE, vcpu)) {
9098 r = kvm_guest_time_update(vcpu);
9102 if (kvm_check_request(KVM_REQ_MMU_SYNC, vcpu))
9103 kvm_mmu_sync_roots(vcpu);
9104 if (kvm_check_request(KVM_REQ_LOAD_MMU_PGD, vcpu))
9105 kvm_mmu_load_pgd(vcpu);
9106 if (kvm_check_request(KVM_REQ_TLB_FLUSH, vcpu)) {
9107 kvm_vcpu_flush_tlb_all(vcpu);
9110 kvm_clear_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu);
9112 if (kvm_check_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu))
9113 kvm_vcpu_flush_tlb_current(vcpu);
9114 if (kvm_check_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu))
9115 kvm_vcpu_flush_tlb_guest(vcpu);
9117 if (kvm_check_request(KVM_REQ_REPORT_TPR_ACCESS, vcpu)) {
9118 vcpu->run->exit_reason = KVM_EXIT_TPR_ACCESS;
9122 if (kvm_check_request(KVM_REQ_TRIPLE_FAULT, vcpu)) {
9123 vcpu->run->exit_reason = KVM_EXIT_SHUTDOWN;
9124 vcpu->mmio_needed = 0;
9128 if (kvm_check_request(KVM_REQ_APF_HALT, vcpu)) {
9130 vcpu->arch.apf.halted = true;
9134 if (kvm_check_request(KVM_REQ_STEAL_UPDATE, vcpu))
9135 record_steal_time(vcpu);
9136 if (kvm_check_request(KVM_REQ_SMI, vcpu))
9137 process_smi(vcpu);
9138 if (kvm_check_request(KVM_REQ_NMI, vcpu))
9139 process_nmi(vcpu);
9140 if (kvm_check_request(KVM_REQ_PMU, vcpu))
9141 kvm_pmu_handle_event(vcpu);
9142 if (kvm_check_request(KVM_REQ_PMI, vcpu))
9143 kvm_pmu_deliver_pmi(vcpu);
9144 if (kvm_check_request(KVM_REQ_IOAPIC_EOI_EXIT, vcpu)) {
9145 BUG_ON(vcpu->arch.pending_ioapic_eoi > 255);
9146 if (test_bit(vcpu->arch.pending_ioapic_eoi,
9147 vcpu->arch.ioapic_handled_vectors)) {
9148 vcpu->run->exit_reason = KVM_EXIT_IOAPIC_EOI;
9149 vcpu->run->eoi.vector =
9150 vcpu->arch.pending_ioapic_eoi;
9155 if (kvm_check_request(KVM_REQ_SCAN_IOAPIC, vcpu))
9156 vcpu_scan_ioapic(vcpu);
9157 if (kvm_check_request(KVM_REQ_LOAD_EOI_EXITMAP, vcpu))
9158 vcpu_load_eoi_exitmap(vcpu);
9159 if (kvm_check_request(KVM_REQ_APIC_PAGE_RELOAD, vcpu))
9160 kvm_vcpu_reload_apic_access_page(vcpu);
9161 if (kvm_check_request(KVM_REQ_HV_CRASH, vcpu)) {
9162 vcpu->run->exit_reason = KVM_EXIT_SYSTEM_EVENT;
9163 vcpu->run->system_event.type = KVM_SYSTEM_EVENT_CRASH;
9167 if (kvm_check_request(KVM_REQ_HV_RESET, vcpu)) {
9168 vcpu->run->exit_reason = KVM_EXIT_SYSTEM_EVENT;
9169 vcpu->run->system_event.type = KVM_SYSTEM_EVENT_RESET;
9173 if (kvm_check_request(KVM_REQ_HV_EXIT, vcpu)) {
9174 vcpu->run->exit_reason = KVM_EXIT_HYPERV;
9175 vcpu->run->hyperv = vcpu->arch.hyperv.exit;
9185 if (kvm_check_request(KVM_REQ_HV_STIMER, vcpu))
9186 kvm_hv_process_stimers(vcpu);
9187 if (kvm_check_request(KVM_REQ_APICV_UPDATE, vcpu))
9188 kvm_vcpu_update_apicv(vcpu);
9189 if (kvm_check_request(KVM_REQ_APF_READY, vcpu))
9190 kvm_check_async_pf_completion(vcpu);
9191 if (kvm_check_request(KVM_REQ_MSR_FILTER_CHANGED, vcpu))
9192 kvm_x86_ops.msr_filter_changed(vcpu);
9195 if (kvm_check_request(KVM_REQ_EVENT, vcpu) || req_int_win) {
9196 ++vcpu->stat.req_event;
9197 kvm_apic_accept_events(vcpu);
9198 if (vcpu->arch.mp_state == KVM_MP_STATE_INIT_RECEIVED) {
9203 inject_pending_event(vcpu, &req_immediate_exit);
9205 kvm_x86_ops.enable_irq_window(vcpu);
9207 if (kvm_lapic_enabled(vcpu)) {
9208 update_cr8_intercept(vcpu);
9209 kvm_lapic_sync_to_vapic(vcpu);
9213 r = kvm_mmu_reload(vcpu);
9220 kvm_x86_ops.prepare_guest_switch(vcpu);
9228 vcpu->mode = IN_GUEST_MODE;
9230 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
9250 if (kvm_lapic_enabled(vcpu) && vcpu->arch.apicv_active)
9251 kvm_x86_ops.sync_pir_to_irr(vcpu);
9253 if (kvm_vcpu_exit_request(vcpu)) {
9254 vcpu->mode = OUTSIDE_GUEST_MODE;
9258 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
9264 kvm_make_request(KVM_REQ_EVENT, vcpu);
9265 kvm_x86_ops.request_immediate_exit(vcpu);
9268 trace_kvm_entry(vcpu);
9274 if (unlikely(vcpu->arch.switch_db_regs)) {
9276 set_debugreg(vcpu->arch.eff_db[0], 0);
9277 set_debugreg(vcpu->arch.eff_db[1], 1);
9278 set_debugreg(vcpu->arch.eff_db[2], 2);
9279 set_debugreg(vcpu->arch.eff_db[3], 3);
9280 set_debugreg(vcpu->arch.dr6, 6);
9281 vcpu->arch.switch_db_regs &= ~KVM_DEBUGREG_RELOAD;
9286 exit_fastpath = kvm_x86_ops.run(vcpu);
9294 if (unlikely(vcpu->arch.switch_db_regs & KVM_DEBUGREG_WONT_EXIT)) {
9295 WARN_ON(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP);
9296 kvm_x86_ops.sync_dirty_debug_regs(vcpu);
9297 kvm_update_dr0123(vcpu);
9298 kvm_update_dr7(vcpu);
9299 vcpu->arch.switch_db_regs &= ~KVM_DEBUGREG_RELOAD;
9312 vcpu->arch.last_vmentry_cpu = vcpu->cpu;
9313 vcpu->arch.last_guest_tsc = kvm_read_l1_tsc(vcpu, rdtsc());
9315 vcpu->mode = OUTSIDE_GUEST_MODE;
9318 kvm_x86_ops.handle_exit_irqoff(vcpu);
9327 kvm_before_interrupt(vcpu);
9329 ++vcpu->stat.exits;
9331 kvm_after_interrupt(vcpu);
9342 if (lapic_in_kernel(vcpu)) {
9343 s64 delta = vcpu->arch.apic->lapic_timer.advance_expire_delta;
9345 trace_kvm_wait_lapic_expire(vcpu->vcpu_id, delta);
9346 vcpu->arch.apic->lapic_timer.advance_expire_delta = S64_MIN;
9353 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
9359 unsigned long rip = kvm_rip_read(vcpu);
9363 if (unlikely(vcpu->arch.tsc_always_catchup))
9364 kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);
9366 if (vcpu->arch.apic_attention)
9367 kvm_lapic_sync_from_vapic(vcpu);
9369 r = kvm_x86_ops.handle_exit(vcpu, exit_fastpath);
9374 kvm_make_request(KVM_REQ_EVENT, vcpu);
9375 kvm_x86_ops.cancel_injection(vcpu);
9376 if (unlikely(vcpu->arch.apic_attention))
9377 kvm_lapic_sync_from_vapic(vcpu);
9382 static inline int vcpu_block(struct kvm *kvm, struct kvm_vcpu *vcpu)
9384 if (!kvm_arch_vcpu_runnable(vcpu) &&
9385 (!kvm_x86_ops.pre_block || kvm_x86_ops.pre_block(vcpu) == 0)) {
9386 srcu_read_unlock(&kvm->srcu, vcpu->srcu_idx);
9387 kvm_vcpu_block(vcpu);
9388 vcpu->srcu_idx = srcu_read_lock(&kvm->srcu);
9391 kvm_x86_ops.post_block(vcpu);
9393 if (!kvm_check_request(KVM_REQ_UNHALT, vcpu))
9397 kvm_apic_accept_events(vcpu);
9398 switch(vcpu->arch.mp_state) {
9400 vcpu->arch.pv.pv_unhalted = false;
9401 vcpu->arch.mp_state =
9405 vcpu->arch.apf.halted = false;
9415 static inline bool kvm_vcpu_running(struct kvm_vcpu *vcpu)
9417 if (is_guest_mode(vcpu))
9418 kvm_x86_ops.nested_ops->check_events(vcpu);
9420 return (vcpu->arch.mp_state == KVM_MP_STATE_RUNNABLE &&
9421 !vcpu->arch.apf.halted);
9424 static int vcpu_run(struct kvm_vcpu *vcpu)
9427 struct kvm *kvm = vcpu->kvm;
9429 vcpu->srcu_idx = srcu_read_lock(&kvm->srcu);
9430 vcpu->arch.l1tf_flush_l1d = true;
9439 vcpu->arch.at_instruction_boundary = false;
9440 if (kvm_vcpu_running(vcpu)) {
9441 r = vcpu_enter_guest(vcpu);
9443 r = vcpu_block(kvm, vcpu);
9449 kvm_clear_request(KVM_REQ_PENDING_TIMER, vcpu);
9450 if (kvm_cpu_has_pending_timer(vcpu))
9451 kvm_inject_pending_timer_irqs(vcpu);
9453 if (dm_request_for_irq_injection(vcpu) &&
9454 kvm_vcpu_ready_for_interrupt_injection(vcpu)) {
9456 vcpu->run->exit_reason = KVM_EXIT_IRQ_WINDOW_OPEN;
9457 ++vcpu->stat.request_irq_exits;
9462 srcu_read_unlock(&kvm->srcu, vcpu->srcu_idx);
9463 r = xfer_to_guest_mode_handle_work(vcpu);
9466 vcpu->srcu_idx = srcu_read_lock(&kvm->srcu);
9470 srcu_read_unlock(&kvm->srcu, vcpu->srcu_idx);
9475 static inline int complete_emulated_io(struct kvm_vcpu *vcpu)
9479 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
9480 r = kvm_emulate_instruction(vcpu, EMULTYPE_NO_DECODE);
9481 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
9485 static int complete_emulated_pio(struct kvm_vcpu *vcpu)
9487 BUG_ON(!vcpu->arch.pio.count);
9489 return complete_emulated_io(vcpu);
9510 static int complete_emulated_mmio(struct kvm_vcpu *vcpu)
9512 struct kvm_run *run = vcpu->run;
9516 BUG_ON(!vcpu->mmio_needed);
9519 frag = &vcpu->mmio_fragments[vcpu->mmio_cur_fragment];
9521 if (!vcpu->mmio_is_write)
9527 vcpu->mmio_cur_fragment++;
9535 if (vcpu->mmio_cur_fragment >= vcpu->mmio_nr_fragments) {
9536 vcpu->mmio_needed = 0;
9539 if (vcpu->mmio_is_write)
9541 vcpu->mmio_read_completed = 1;
9542 return complete_emulated_io(vcpu);
9547 if (vcpu->mmio_is_write)
9550 run->mmio.is_write = vcpu->mmio_is_write;
9551 vcpu->arch.complete_userspace_io = complete_emulated_mmio;
9569 static void kvm_load_guest_fpu(struct kvm_vcpu *vcpu)
9573 kvm_save_current_fpu(vcpu->arch.user_fpu);
9576 __copy_kernel_to_fpregs(&vcpu->arch.guest_fpu->state,
9586 static void kvm_put_guest_fpu(struct kvm_vcpu *vcpu)
9590 kvm_save_current_fpu(vcpu->arch.guest_fpu);
9592 copy_kernel_to_fpregs(&vcpu->arch.user_fpu->state);
9597 ++vcpu->stat.fpu_reload;
9601 int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu)
9603 struct kvm_run *kvm_run = vcpu->run;
9606 vcpu_load(vcpu);
9607 kvm_sigset_activate(vcpu);
9608 kvm_load_guest_fpu(vcpu);
9610 if (unlikely(vcpu->arch.mp_state == KVM_MP_STATE_UNINITIALIZED)) {
9615 kvm_vcpu_block(vcpu);
9616 kvm_apic_accept_events(vcpu);
9617 kvm_clear_request(KVM_REQ_UNHALT, vcpu);
9622 ++vcpu->stat.signal_exits;
9633 r = sync_regs(vcpu);
9639 if (!lapic_in_kernel(vcpu)) {
9640 if (kvm_set_cr8(vcpu, kvm_run->cr8) != 0) {
9646 if (unlikely(vcpu->arch.complete_userspace_io)) {
9647 int (*cui)(struct kvm_vcpu *) = vcpu->arch.complete_userspace_io;
9648 vcpu->arch.complete_userspace_io = NULL;
9649 r = cui(vcpu);
9653 WARN_ON(vcpu->arch.pio.count || vcpu->mmio_needed);
9658 r = vcpu_run(vcpu);
9661 kvm_put_guest_fpu(vcpu);
9663 store_regs(vcpu);
9664 post_kvm_run_save(vcpu);
9665 kvm_sigset_deactivate(vcpu);
9667 vcpu_put(vcpu);
9671 static void __get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
9673 if (vcpu->arch.emulate_regs_need_sync_to_vcpu) {
9677 * back from emulation context to vcpu. Userspace shouldn't do
9681 emulator_writeback_register_cache(vcpu->arch.emulate_ctxt);
9682 vcpu->arch.emulate_regs_need_sync_to_vcpu = false;
9684 regs->rax = kvm_rax_read(vcpu);
9685 regs->rbx = kvm_rbx_read(vcpu);
9686 regs->rcx = kvm_rcx_read(vcpu);
9687 regs->rdx = kvm_rdx_read(vcpu);
9688 regs->rsi = kvm_rsi_read(vcpu);
9689 regs->rdi = kvm_rdi_read(vcpu);
9690 regs->rsp = kvm_rsp_read(vcpu);
9691 regs->rbp = kvm_rbp_read(vcpu);
9693 regs->r8 = kvm_r8_read(vcpu);
9694 regs->r9 = kvm_r9_read(vcpu);
9695 regs->r10 = kvm_r10_read(vcpu);
9696 regs->r11 = kvm_r11_read(vcpu);
9697 regs->r12 = kvm_r12_read(vcpu);
9698 regs->r13 = kvm_r13_read(vcpu);
9699 regs->r14 = kvm_r14_read(vcpu);
9700 regs->r15 = kvm_r15_read(vcpu);
9703 regs->rip = kvm_rip_read(vcpu);
9704 regs->rflags = kvm_get_rflags(vcpu);
9707 int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
9709 vcpu_load(vcpu);
9710 __get_regs(vcpu, regs);
9711 vcpu_put(vcpu);
9715 static void __set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
9717 vcpu->arch.emulate_regs_need_sync_from_vcpu = true;
9718 vcpu->arch.emulate_regs_need_sync_to_vcpu = false;
9720 kvm_rax_write(vcpu, regs->rax);
9721 kvm_rbx_write(vcpu, regs->rbx);
9722 kvm_rcx_write(vcpu, regs->rcx);
9723 kvm_rdx_write(vcpu, regs->rdx);
9724 kvm_rsi_write(vcpu, regs->rsi);
9725 kvm_rdi_write(vcpu, regs->rdi);
9726 kvm_rsp_write(vcpu, regs->rsp);
9727 kvm_rbp_write(vcpu, regs->rbp);
9729 kvm_r8_write(vcpu, regs->r8);
9730 kvm_r9_write(vcpu, regs->r9);
9731 kvm_r10_write(vcpu, regs->r10);
9732 kvm_r11_write(vcpu, regs->r11);
9733 kvm_r12_write(vcpu, regs->r12);
9734 kvm_r13_write(vcpu, regs->r13);
9735 kvm_r14_write(vcpu, regs->r14);
9736 kvm_r15_write(vcpu, regs->r15);
9739 kvm_rip_write(vcpu, regs->rip);
9740 kvm_set_rflags(vcpu, regs->rflags | X86_EFLAGS_FIXED);
9742 vcpu->arch.exception.pending = false;
9744 kvm_make_request(KVM_REQ_EVENT, vcpu);
9747 int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
9749 vcpu_load(vcpu);
9750 __set_regs(vcpu, regs);
9751 vcpu_put(vcpu);
9755 void kvm_get_cs_db_l_bits(struct kvm_vcpu *vcpu, int *db, int *l)
9759 kvm_get_segment(vcpu, &cs, VCPU_SREG_CS);
9765 static void __get_sregs(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs)
9769 kvm_get_segment(vcpu, &sregs->cs, VCPU_SREG_CS);
9770 kvm_get_segment(vcpu, &sregs->ds, VCPU_SREG_DS);
9771 kvm_get_segment(vcpu, &sregs->es, VCPU_SREG_ES);
9772 kvm_get_segment(vcpu, &sregs->fs, VCPU_SREG_FS);
9773 kvm_get_segment(vcpu, &sregs->gs, VCPU_SREG_GS);
9774 kvm_get_segment(vcpu, &sregs->ss, VCPU_SREG_SS);
9776 kvm_get_segment(vcpu, &sregs->tr, VCPU_SREG_TR);
9777 kvm_get_segment(vcpu, &sregs->ldt, VCPU_SREG_LDTR);
9779 kvm_x86_ops.get_idt(vcpu, &dt);
9782 kvm_x86_ops.get_gdt(vcpu, &dt);
9786 sregs->cr0 = kvm_read_cr0(vcpu);
9787 sregs->cr2 = vcpu->arch.cr2;
9788 sregs->cr3 = kvm_read_cr3(vcpu);
9789 sregs->cr4 = kvm_read_cr4(vcpu);
9790 sregs->cr8 = kvm_get_cr8(vcpu);
9791 sregs->efer = vcpu->arch.efer;
9792 sregs->apic_base = kvm_get_apic_base(vcpu);
9796 if (vcpu->arch.interrupt.injected && !vcpu->arch.interrupt.soft)
9797 set_bit(vcpu->arch.interrupt.nr,
9801 int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
9804 vcpu_load(vcpu);
9805 __get_sregs(vcpu, sregs);
9806 vcpu_put(vcpu);
9810 int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
9813 vcpu_load(vcpu);
9815 kvm_load_guest_fpu(vcpu);
9817 kvm_apic_accept_events(vcpu);
9818 if (vcpu->arch.mp_state == KVM_MP_STATE_HALTED &&
9819 vcpu->arch.pv.pv_unhalted)
9822 mp_state->mp_state = vcpu->arch.mp_state;
9825 kvm_put_guest_fpu(vcpu);
9826 vcpu_put(vcpu);
9830 int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
9835 vcpu_load(vcpu);
9837 if (!lapic_in_kernel(vcpu) &&
9846 if ((kvm_vcpu_latch_init(vcpu) || vcpu->arch.smi_pending) &&
9852 vcpu->arch.mp_state = KVM_MP_STATE_INIT_RECEIVED;
9853 set_bit(KVM_APIC_SIPI, &vcpu->arch.apic->pending_events);
9855 vcpu->arch.mp_state = mp_state->mp_state;
9856 kvm_make_request(KVM_REQ_EVENT, vcpu);
9860 vcpu_put(vcpu);
9864 int kvm_task_switch(struct kvm_vcpu *vcpu, u16 tss_selector, int idt_index,
9867 struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt;
9870 init_emulate_ctxt(vcpu);
9875 vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
9876 vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_EMULATION;
9877 vcpu->run->internal.ndata = 0;
9881 kvm_rip_write(vcpu, ctxt->eip);
9882 kvm_set_rflags(vcpu, ctxt->eflags);
9887 static int kvm_valid_sregs(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs)
9898 if (sregs->cr3 & vcpu->arch.cr3_lm_rsvd_bits)
9909 return kvm_valid_cr4(vcpu, sregs->cr4);
9912 static int __set_sregs(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs)
9921 if (kvm_valid_sregs(vcpu, sregs))
9926 if (kvm_set_apic_base(vcpu, &apic_base_msr))
9931 kvm_x86_ops.set_idt(vcpu, &dt);
9934 kvm_x86_ops.set_gdt(vcpu, &dt);
9936 vcpu->arch.cr2 = sregs->cr2;
9937 mmu_reset_needed |= kvm_read_cr3(vcpu) != sregs->cr3;
9938 vcpu->arch.cr3 = sregs->cr3;
9939 kvm_register_mark_available(vcpu, VCPU_EXREG_CR3);
9941 kvm_set_cr8(vcpu, sregs->cr8);
9943 mmu_reset_needed |= vcpu->arch.efer != sregs->efer;
9944 kvm_x86_ops.set_efer(vcpu, sregs->efer);
9946 mmu_reset_needed |= kvm_read_cr0(vcpu) != sregs->cr0;
9947 kvm_x86_ops.set_cr0(vcpu, sregs->cr0);
9948 vcpu->arch.cr0 = sregs->cr0;
9950 mmu_reset_needed |= kvm_read_cr4(vcpu) != sregs->cr4;
9951 cpuid_update_needed |= ((kvm_read_cr4(vcpu) ^ sregs->cr4) &
9953 kvm_x86_ops.set_cr4(vcpu, sregs->cr4);
9955 kvm_update_cpuid_runtime(vcpu);
9957 idx = srcu_read_lock(&vcpu->kvm->srcu);
9958 if (is_pae_paging(vcpu)) {
9959 load_pdptrs(vcpu, vcpu->arch.walk_mmu, kvm_read_cr3(vcpu));
9962 srcu_read_unlock(&vcpu->kvm->srcu, idx);
9965 kvm_mmu_reset_context(vcpu);
9971 kvm_queue_interrupt(vcpu, pending_vec, false);
9975 kvm_set_segment(vcpu, &sregs->cs, VCPU_SREG_CS);
9976 kvm_set_segment(vcpu, &sregs->ds, VCPU_SREG_DS);
9977 kvm_set_segment(vcpu, &sregs->es, VCPU_SREG_ES);
9978 kvm_set_segment(vcpu, &sregs->fs, VCPU_SREG_FS);
9979 kvm_set_segment(vcpu, &sregs->gs, VCPU_SREG_GS);
9980 kvm_set_segment(vcpu, &sregs->ss, VCPU_SREG_SS);
9982 kvm_set_segment(vcpu, &sregs->tr, VCPU_SREG_TR);
9983 kvm_set_segment(vcpu, &sregs->ldt, VCPU_SREG_LDTR);
9985 update_cr8_intercept(vcpu);
9987 /* Older userspace won't unhalt the vcpu on reset. */
9988 if (kvm_vcpu_is_bsp(vcpu) && kvm_rip_read(vcpu) == 0xfff0 &&
9990 !is_protmode(vcpu))
9991 vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE;
9993 kvm_make_request(KVM_REQ_EVENT, vcpu);
10000 int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
10005 vcpu_load(vcpu);
10006 ret = __set_sregs(vcpu, sregs);
10007 vcpu_put(vcpu);
10011 int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
10017 vcpu_load(vcpu);
10021 if (vcpu->arch.exception.pending)
10024 kvm_queue_exception(vcpu, DB_VECTOR);
10026 kvm_queue_exception(vcpu, BP_VECTOR);
10033 rflags = kvm_get_rflags(vcpu);
10035 vcpu->guest_debug = dbg->control;
10036 if (!(vcpu->guest_debug & KVM_GUESTDBG_ENABLE))
10037 vcpu->guest_debug = 0;
10039 if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) {
10041 vcpu->arch.eff_db[i] = dbg->arch.debugreg[i];
10042 vcpu->arch.guest_debug_dr7 = dbg->arch.debugreg[7];
10045 vcpu->arch.eff_db[i] = vcpu->arch.db[i];
10047 kvm_update_dr7(vcpu);
10049 if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP)
10050 vcpu->arch.singlestep_rip = kvm_rip_read(vcpu) +
10051 get_segment_base(vcpu, VCPU_SREG_CS);
10057 kvm_set_rflags(vcpu, rflags);
10059 kvm_x86_ops.update_exception_bitmap(vcpu);
10064 vcpu_put(vcpu);
10071 int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,
10078 vcpu_load(vcpu);
10080 idx = srcu_read_lock(&vcpu->kvm->srcu);
10081 gpa = kvm_mmu_gva_to_gpa_system(vcpu, vaddr, NULL);
10082 srcu_read_unlock(&vcpu->kvm->srcu, idx);
10088 vcpu_put(vcpu);
10092 int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
10096 vcpu_load(vcpu);
10098 fxsave = &vcpu->arch.guest_fpu->state.fxsave;
10108 vcpu_put(vcpu);
10112 int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
10116 vcpu_load(vcpu);
10118 fxsave = &vcpu->arch.guest_fpu->state.fxsave;
10129 vcpu_put(vcpu);
10133 static void store_regs(struct kvm_vcpu *vcpu)
10137 if (vcpu->run->kvm_valid_regs & KVM_SYNC_X86_REGS)
10138 __get_regs(vcpu, &vcpu->run->s.regs.regs);
10140 if (vcpu->run->kvm_valid_regs & KVM_SYNC_X86_SREGS)
10141 __get_sregs(vcpu, &vcpu->run->s.regs.sregs);
10143 if (vcpu->run->kvm_valid_regs & KVM_SYNC_X86_EVENTS)
10145 vcpu, &vcpu->run->s.regs.events);
10148 static int sync_regs(struct kvm_vcpu *vcpu)
10150 if (vcpu->run->kvm_dirty_regs & ~KVM_SYNC_X86_VALID_FIELDS)
10153 if (vcpu->run->kvm_dirty_regs & KVM_SYNC_X86_REGS) {
10154 __set_regs(vcpu, &vcpu->run->s.regs.regs);
10155 vcpu->run->kvm_dirty_regs &= ~KVM_SYNC_X86_REGS;
10157 if (vcpu->run->kvm_dirty_regs & KVM_SYNC_X86_SREGS) {
10158 if (__set_sregs(vcpu, &vcpu->run->s.regs.sregs))
10160 vcpu->run->kvm_dirty_regs &= ~KVM_SYNC_X86_SREGS;
10162 if (vcpu->run->kvm_dirty_regs & KVM_SYNC_X86_EVENTS) {
10164 vcpu, &vcpu->run->s.regs.events))
10166 vcpu->run->kvm_dirty_regs &= ~KVM_SYNC_X86_EVENTS;
10172 static void fx_init(struct kvm_vcpu *vcpu)
10174 fpstate_init(&vcpu->arch.guest_fpu->state);
10176 vcpu->arch.guest_fpu->state.xsave.header.xcomp_bv =
10182 vcpu->arch.xcr0 = XFEATURE_MASK_FP;
10184 vcpu->arch.cr0 |= X86_CR0_ET;
10196 int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu)
10201 if (!irqchip_in_kernel(vcpu->kvm) || kvm_vcpu_is_reset_bsp(vcpu))
10202 vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE;
10204 vcpu->arch.mp_state = KVM_MP_STATE_UNINITIALIZED;
10206 kvm_set_tsc_khz(vcpu, max_tsc_khz);
10208 r = kvm_mmu_create(vcpu);
10212 if (irqchip_in_kernel(vcpu->kvm)) {
10213 r = kvm_create_lapic(vcpu, lapic_timer_advance_ns);
10216 if (kvm_apicv_activated(vcpu->kvm))
10217 vcpu->arch.apicv_active = true;
10226 vcpu->arch.pio_data = page_address(page);
10228 vcpu->arch.mce_banks = kzalloc(KVM_MAX_MCE_BANKS * sizeof(u64) * 4,
10230 if (!vcpu->arch.mce_banks)
10232 vcpu->arch.mcg_cap = KVM_MAX_MCE_BANKS;
10234 if (!zalloc_cpumask_var(&vcpu->arch.wbinvd_dirty_mask,
10238 if (!alloc_emulate_ctxt(vcpu))
10241 vcpu->arch.user_fpu = kmem_cache_zalloc(x86_fpu_cache,
10243 if (!vcpu->arch.user_fpu) {
10248 vcpu->arch.guest_fpu = kmem_cache_zalloc(x86_fpu_cache,
10250 if (!vcpu->arch.guest_fpu) {
10251 pr_err("kvm: failed to allocate vcpu's fpu\n");
10254 fx_init(vcpu);
10256 vcpu->arch.maxphyaddr = cpuid_query_maxphyaddr(vcpu);
10257 vcpu->arch.cr3_lm_rsvd_bits = rsvd_bits(cpuid_maxphyaddr(vcpu), 63);
10259 vcpu->arch.pat = MSR_IA32_CR_PAT_DEFAULT;
10261 kvm_async_pf_hash_reset(vcpu);
10262 kvm_pmu_init(vcpu);
10264 vcpu->arch.pending_external_vector = -1;
10265 vcpu->arch.preempted_in_kernel = false;
10267 kvm_hv_vcpu_init(vcpu);
10269 r = kvm_x86_ops.vcpu_create(vcpu);
10273 vcpu->arch.arch_capabilities = kvm_get_arch_capabilities();
10274 vcpu->arch.msr_platform_info = MSR_PLATFORM_INFO_CPUID_FAULT;
10275 kvm_vcpu_mtrr_init(vcpu);
10276 vcpu_load(vcpu);
10277 kvm_vcpu_reset(vcpu, false);
10278 kvm_init_mmu(vcpu, false);
10279 vcpu_put(vcpu);
10283 kmem_cache_free(x86_fpu_cache, vcpu->arch.guest_fpu);
10285 kmem_cache_free(x86_fpu_cache, vcpu->arch.user_fpu);
10287 kmem_cache_free(x86_emulator_cache, vcpu->arch.emulate_ctxt);
10289 free_cpumask_var(vcpu->arch.wbinvd_dirty_mask);
10291 kfree(vcpu->arch.mce_banks);
10293 free_page((unsigned long)vcpu->arch.pio_data);
10295 kvm_free_lapic(vcpu);
10297 kvm_mmu_destroy(vcpu);
10301 void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)
10303 struct kvm *kvm = vcpu->kvm;
10305 kvm_hv_vcpu_postcreate(vcpu);
10307 if (mutex_lock_killable(&vcpu->mutex))
10309 vcpu_load(vcpu);
10310 kvm_synchronize_tsc(vcpu, 0);
10311 vcpu_put(vcpu);
10314 vcpu->arch.msr_kvm_poll_control = 1;
10316 mutex_unlock(&vcpu->mutex);
10318 if (kvmclock_periodic_sync && vcpu->vcpu_idx == 0)
10323 void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
10327 kvmclock_reset(vcpu);
10329 kvm_x86_ops.vcpu_free(vcpu);
10331 kmem_cache_free(x86_emulator_cache, vcpu->arch.emulate_ctxt);
10332 free_cpumask_var(vcpu->arch.wbinvd_dirty_mask);
10333 kmem_cache_free(x86_fpu_cache, vcpu->arch.user_fpu);
10334 kmem_cache_free(x86_fpu_cache, vcpu->arch.guest_fpu);
10336 kvm_hv_vcpu_uninit(vcpu);
10337 kvm_pmu_destroy(vcpu);
10338 kfree(vcpu->arch.mce_banks);
10339 kvm_free_lapic(vcpu);
10340 idx = srcu_read_lock(&vcpu->kvm->srcu);
10341 kvm_mmu_destroy(vcpu);
10342 srcu_read_unlock(&vcpu->kvm->srcu, idx);
10343 free_page((unsigned long)vcpu->arch.pio_data);
10344 kvfree(vcpu->arch.cpuid_entries);
10345 if (!lapic_in_kernel(vcpu))
10349 void kvm_vcpu_reset(struct kvm_vcpu *vcpu, bool init_event)
10351 kvm_lapic_reset(vcpu, init_event);
10353 vcpu->arch.hflags = 0;
10355 vcpu->arch.smi_pending = 0;
10356 vcpu->arch.smi_count = 0;
10357 atomic_set(&vcpu->arch.nmi_queued, 0);
10358 vcpu->arch.nmi_pending = 0;
10359 vcpu->arch.nmi_injected = false;
10360 kvm_clear_interrupt_queue(vcpu);
10361 kvm_clear_exception_queue(vcpu);
10363 memset(vcpu->arch.db, 0, sizeof(vcpu->arch.db));
10364 kvm_update_dr0123(vcpu);
10365 vcpu->arch.dr6 = DR6_INIT;
10366 vcpu->arch.dr7 = DR7_FIXED_1;
10367 kvm_update_dr7(vcpu);
10369 vcpu->arch.cr2 = 0;
10371 kvm_make_request(KVM_REQ_EVENT, vcpu);
10372 vcpu->arch.apf.msr_en_val = 0;
10373 vcpu->arch.apf.msr_int_val = 0;
10374 vcpu->arch.st.msr_val = 0;
10376 kvmclock_reset(vcpu);
10378 kvm_clear_async_pf_completion_queue(vcpu);
10379 kvm_async_pf_hash_reset(vcpu);
10380 vcpu->arch.apf.halted = false;
10390 kvm_put_guest_fpu(vcpu);
10391 mpx_state_buffer = get_xsave_addr(&vcpu->arch.guest_fpu->state.xsave,
10395 mpx_state_buffer = get_xsave_addr(&vcpu->arch.guest_fpu->state.xsave,
10400 kvm_load_guest_fpu(vcpu);
10404 kvm_pmu_reset(vcpu);
10405 vcpu->arch.smbase = 0x30000;
10407 vcpu->arch.msr_misc_features_enables = 0;
10409 vcpu->arch.xcr0 = XFEATURE_MASK_FP;
10412 memset(vcpu->arch.regs, 0, sizeof(vcpu->arch.regs));
10413 vcpu->arch.regs_avail = ~0;
10414 vcpu->arch.regs_dirty = ~0;
10416 vcpu->arch.ia32_xss = 0;
10418 kvm_x86_ops.vcpu_reset(vcpu, init_event);
10421 void kvm_vcpu_deliver_sipi_vector(struct kvm_vcpu *vcpu, u8 vector)
10425 kvm_get_segment(vcpu, &cs, VCPU_SREG_CS);
10428 kvm_set_segment(vcpu, &cs, VCPU_SREG_CS);
10429 kvm_rip_write(vcpu, 0);
10435 struct kvm_vcpu *vcpu;
10450 kvm_for_each_vcpu(i, vcpu, kvm) {
10451 if (!stable && vcpu->cpu == smp_processor_id())
10452 kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);
10453 if (stable && vcpu->arch.last_host_tsc > local_tsc) {
10455 if (vcpu->arch.last_host_tsc > max_tsc)
10456 max_tsc = vcpu->arch.last_host_tsc;
10503 kvm_for_each_vcpu(i, vcpu, kvm) {
10504 vcpu->arch.tsc_offset_adjustment += delta_cyc;
10505 vcpu->arch.last_host_tsc = local_tsc;
10506 kvm_make_request(KVM_REQ_MASTERCLOCK_UPDATE, vcpu);
10589 bool kvm_vcpu_is_reset_bsp(struct kvm_vcpu *vcpu)
10591 return vcpu->kvm->arch.bsp_vcpu_id == vcpu->vcpu_id;
10595 bool kvm_vcpu_is_bsp(struct kvm_vcpu *vcpu)
10597 return (vcpu->arch.apic_base & MSR_IA32_APICBASE_BSP) != 0;
10603 void kvm_arch_sched_in(struct kvm_vcpu *vcpu, int cpu)
10605 struct kvm_pmu *pmu = vcpu_to_pmu(vcpu);
10607 vcpu->arch.l1tf_flush_l1d = true;
10610 kvm_make_request(KVM_REQ_PMU, vcpu);
10612 kvm_x86_ops.sched_in(vcpu, cpu);
10669 static void kvm_unload_vcpu_mmu(struct kvm_vcpu *vcpu)
10671 vcpu_load(vcpu);
10672 kvm_mmu_unload(vcpu);
10673 vcpu_put(vcpu);
10679 struct kvm_vcpu *vcpu;
10684 kvm_for_each_vcpu(i, vcpu, kvm) {
10685 kvm_clear_async_pf_completion_queue(vcpu);
10686 kvm_unload_vcpu_mmu(vcpu);
10688 kvm_for_each_vcpu(i, vcpu, kvm)
10689 kvm_vcpu_destroy(vcpu);
10882 struct kvm_vcpu *vcpu;
10892 kvm_for_each_vcpu(i, vcpu, kvm)
10893 kvm_vcpu_kick(vcpu);
11023 static inline bool kvm_guest_apic_has_interrupt(struct kvm_vcpu *vcpu)
11025 return (is_guest_mode(vcpu) &&
11027 kvm_x86_ops.guest_apic_has_interrupt(vcpu));
11030 static inline bool kvm_vcpu_has_events(struct kvm_vcpu *vcpu)
11032 if (!list_empty_careful(&vcpu->async_pf.done))
11035 if (kvm_apic_has_events(vcpu))
11038 if (vcpu->arch.pv.pv_unhalted)
11041 if (vcpu->arch.exception.pending)
11044 if (kvm_test_request(KVM_REQ_NMI, vcpu) ||
11045 (vcpu->arch.nmi_pending &&
11046 kvm_x86_ops.nmi_allowed(vcpu, false)))
11049 if (kvm_test_request(KVM_REQ_SMI, vcpu) ||
11050 (vcpu->arch.smi_pending &&
11051 kvm_x86_ops.smi_allowed(vcpu, false)))
11054 if (kvm_arch_interrupt_allowed(vcpu) &&
11055 (kvm_cpu_has_interrupt(vcpu) ||
11056 kvm_guest_apic_has_interrupt(vcpu)))
11059 if (kvm_hv_has_stimer_pending(vcpu))
11062 if (is_guest_mode(vcpu) &&
11064 kvm_x86_ops.nested_ops->hv_timer_pending(vcpu))
11070 int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu)
11072 return kvm_vcpu_running(vcpu) || kvm_vcpu_has_events(vcpu);
11075 bool kvm_arch_dy_runnable(struct kvm_vcpu *vcpu)
11077 if (READ_ONCE(vcpu->arch.pv.pv_unhalted))
11080 if (kvm_test_request(KVM_REQ_NMI, vcpu) ||
11081 kvm_test_request(KVM_REQ_SMI, vcpu) ||
11082 kvm_test_request(KVM_REQ_EVENT, vcpu))
11085 if (vcpu->arch.apicv_active && kvm_x86_ops.dy_apicv_has_pending_interrupt(vcpu))
11091 bool kvm_arch_vcpu_in_kernel(struct kvm_vcpu *vcpu)
11093 return vcpu->arch.preempted_in_kernel;
11096 int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu)
11098 return kvm_vcpu_exiting_guest_mode(vcpu) == IN_GUEST_MODE;
11101 int kvm_arch_interrupt_allowed(struct kvm_vcpu *vcpu)
11103 return kvm_x86_ops.interrupt_allowed(vcpu, false);
11106 unsigned long kvm_get_linear_rip(struct kvm_vcpu *vcpu)
11108 if (is_64_bit_mode(vcpu))
11109 return kvm_rip_read(vcpu);
11110 return (u32)(get_segment_base(vcpu, VCPU_SREG_CS) +
11111 kvm_rip_read(vcpu));
11115 bool kvm_is_linear_rip(struct kvm_vcpu *vcpu, unsigned long linear_rip)
11117 return kvm_get_linear_rip(vcpu) == linear_rip;
11121 unsigned long kvm_get_rflags(struct kvm_vcpu *vcpu)
11125 rflags = kvm_x86_ops.get_rflags(vcpu);
11126 if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP)
11132 static void __kvm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags)
11134 if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP &&
11135 kvm_is_linear_rip(vcpu, vcpu->arch.singlestep_rip))
11137 kvm_x86_ops.set_rflags(vcpu, rflags);
11140 void kvm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags)
11142 __kvm_set_rflags(vcpu, rflags);
11143 kvm_make_request(KVM_REQ_EVENT, vcpu);
11147 void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu, struct kvm_async_pf *work)
11151 if ((vcpu->arch.mmu->direct_map != work->arch.direct_map) ||
11155 r = kvm_mmu_reload(vcpu);
11159 if (!vcpu->arch.mmu->direct_map &&
11160 work->arch.cr3 != vcpu->arch.mmu->get_guest_pgd(vcpu))
11163 kvm_mmu_do_page_fault(vcpu, work->cr2_or_gpa, 0, true);
11178 static void kvm_add_async_pf_gfn(struct kvm_vcpu *vcpu, gfn_t gfn)
11182 while (vcpu->arch.apf.gfns[key] != ~0)
11185 vcpu->arch.apf.gfns[key] = gfn;
11188 static u32 kvm_async_pf_gfn_slot(struct kvm_vcpu *vcpu, gfn_t gfn)
11194 (vcpu->arch.apf.gfns[key] != gfn &&
11195 vcpu->arch.apf.gfns[key] != ~0); i++)
11201 bool kvm_find_async_pf_gfn(struct kvm_vcpu *vcpu, gfn_t gfn)
11203 return vcpu->arch.apf.gfns[kvm_async_pf_gfn_slot(vcpu, gfn)] == gfn;
11206 static void kvm_del_async_pf_gfn(struct kvm_vcpu *vcpu, gfn_t gfn)
11210 i = j = kvm_async_pf_gfn_slot(vcpu, gfn);
11212 if (WARN_ON_ONCE(vcpu->arch.apf.gfns[i] != gfn))
11216 vcpu->arch.apf.gfns[i] = ~0;
11219 if (vcpu->arch.apf.gfns[j] == ~0)
11221 k = kvm_async_pf_hash_fn(vcpu->arch.apf.gfns[j]);
11228 vcpu->arch.apf.gfns[i] = vcpu->arch.apf.gfns[j];
11233 static inline int apf_put_user_notpresent(struct kvm_vcpu *vcpu)
11237 return kvm_write_guest_cached(vcpu->kvm, &vcpu->arch.apf.data, &reason,
11241 static inline int apf_put_user_ready(struct kvm_vcpu *vcpu, u32 token)
11245 return kvm_write_guest_offset_cached(vcpu->kvm, &vcpu->arch.apf.data,
11249 static inline bool apf_pageready_slot_free(struct kvm_vcpu *vcpu)
11254 if (kvm_read_guest_offset_cached(vcpu->kvm, &vcpu->arch.apf.data,
11261 static bool kvm_can_deliver_async_pf(struct kvm_vcpu *vcpu)
11263 if (!vcpu->arch.apf.delivery_as_pf_vmexit && is_guest_mode(vcpu))
11266 if (!kvm_pv_async_pf_enabled(vcpu) ||
11267 (vcpu->arch.apf.send_user_only && kvm_x86_ops.get_cpl(vcpu) == 0))
11273 bool kvm_can_do_async_pf(struct kvm_vcpu *vcpu)
11275 if (unlikely(!lapic_in_kernel(vcpu) ||
11276 kvm_event_needs_reinjection(vcpu) ||
11277 vcpu->arch.exception.pending))
11280 if (kvm_hlt_in_guest(vcpu->kvm) && !kvm_can_deliver_async_pf(vcpu))
11287 return kvm_arch_interrupt_allowed(vcpu);
11290 bool kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu,
11296 kvm_add_async_pf_gfn(vcpu, work->arch.gfn);
11298 if (kvm_can_deliver_async_pf(vcpu) &&
11299 !apf_put_user_notpresent(vcpu)) {
11306 kvm_inject_page_fault(vcpu, &fault);
11317 kvm_make_request(KVM_REQ_APF_HALT, vcpu);
11322 void kvm_arch_async_page_present(struct kvm_vcpu *vcpu,
11327 .vector = vcpu->arch.apf.vec
11333 kvm_del_async_pf_gfn(vcpu, work->arch.gfn);
11337 kvm_pv_async_pf_enabled(vcpu) &&
11338 !apf_put_user_ready(vcpu, work->arch.token)) {
11339 vcpu->arch.apf.pageready_pending = true;
11340 kvm_apic_set_irq(vcpu, &irq, NULL);
11343 vcpu->arch.apf.halted = false;
11344 vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE;
11347 void kvm_arch_async_page_present_queued(struct kvm_vcpu *vcpu)
11349 kvm_make_request(KVM_REQ_APF_READY, vcpu);
11350 if (!vcpu->arch.apf.pageready_pending)
11351 kvm_vcpu_kick(vcpu);
11354 bool kvm_arch_can_dequeue_async_page_present(struct kvm_vcpu *vcpu)
11356 if (!kvm_pv_async_pf_enabled(vcpu))
11359 return kvm_lapic_enabled(vcpu) && apf_pageready_slot_free(vcpu);
11456 bool kvm_arch_no_poll(struct kvm_vcpu *vcpu)
11458 return (vcpu->arch.msr_kvm_poll_control & 1) == 0;
11489 void kvm_fixup_and_inject_pf_error(struct kvm_vcpu *vcpu, gva_t gva, u16 error_code)
11496 vcpu->arch.walk_mmu->gva_to_gpa(vcpu, gva, access, &fault) != UNMAPPED_GVA) {
11498 * If vcpu->arch.walk_mmu->gva_to_gpa succeeded, the page
11508 vcpu->arch.walk_mmu->inject_page_fault(vcpu, &fault);
11517 int kvm_handle_memory_failure(struct kvm_vcpu *vcpu, int r,
11521 kvm_inject_emulated_page_fault(vcpu, e);
11532 vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
11533 vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_EMULATION;
11534 vcpu->run->internal.ndata = 0;
11540 int kvm_handle_invpcid(struct kvm_vcpu *vcpu, unsigned long type, gva_t gva)
11552 r = kvm_read_guest_virt(vcpu, gva, &operand, sizeof(operand), &e);
11554 return kvm_handle_memory_failure(vcpu, r, &e);
11557 kvm_inject_gp(vcpu, 0);
11561 pcid_enabled = kvm_read_cr4_bits(vcpu, X86_CR4_PCIDE);
11566 is_noncanonical_address(operand.gla, vcpu)) {
11567 kvm_inject_gp(vcpu, 0);
11570 kvm_mmu_invpcid_gva(vcpu, operand.gla, operand.pcid);
11571 return kvm_skip_emulated_instruction(vcpu);
11575 kvm_inject_gp(vcpu, 0);
11579 if (kvm_get_active_pcid(vcpu) == operand.pcid) {
11580 kvm_mmu_sync_roots(vcpu);
11581 kvm_make_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu);
11585 if (kvm_get_pcid(vcpu, vcpu->arch.mmu->prev_roots[i].pgd)
11589 kvm_mmu_free_roots(vcpu, vcpu->arch.mmu, roots_to_free);
11596 return kvm_skip_emulated_instruction(vcpu);
11608 kvm_make_request(KVM_REQ_MMU_RELOAD, vcpu);
11609 return kvm_skip_emulated_instruction(vcpu);