Lines Matching refs:arch
297 vcpu->arch.apf.gfns[i] = ~0;
400 return vcpu->arch.apic_base;
494 unsigned nr = vcpu->arch.exception.nr;
495 bool has_payload = vcpu->arch.exception.has_payload;
496 unsigned long payload = vcpu->arch.exception.payload;
508 vcpu->arch.dr6 &= ~DR_TRAP_BITS;
512 vcpu->arch.dr6 |= DR6_RTM;
513 vcpu->arch.dr6 |= payload;
522 vcpu->arch.dr6 ^= payload & DR6_RTM;
530 vcpu->arch.dr6 &= ~BIT(12);
533 vcpu->arch.cr2 = payload;
537 vcpu->arch.exception.has_payload = false;
538 vcpu->arch.exception.payload = 0;
551 if (!vcpu->arch.exception.pending && !vcpu->arch.exception.injected) {
555 * On vmentry, vcpu->arch.exception.pending is only
562 WARN_ON_ONCE(vcpu->arch.exception.pending);
563 vcpu->arch.exception.injected = true;
573 vcpu->arch.exception.pending = true;
574 vcpu->arch.exception.injected = false;
576 vcpu->arch.exception.has_error_code = has_error;
577 vcpu->arch.exception.nr = nr;
578 vcpu->arch.exception.error_code = error_code;
579 vcpu->arch.exception.has_payload = has_payload;
580 vcpu->arch.exception.payload = payload;
587 prev_nr = vcpu->arch.exception.nr;
602 vcpu->arch.exception.pending = true;
603 vcpu->arch.exception.injected = false;
604 vcpu->arch.exception.has_error_code = true;
605 vcpu->arch.exception.nr = DF_VECTOR;
606 vcpu->arch.exception.error_code = 0;
607 vcpu->arch.exception.has_payload = false;
608 vcpu->arch.exception.payload = 0;
656 vcpu->arch.exception.nested_apf =
658 if (vcpu->arch.exception.nested_apf) {
659 vcpu->arch.apf.nested_apf_token = fault->address;
674 fault_mmu = fault->nested_page_fault ? vcpu->arch.mmu :
675 vcpu->arch.walk_mmu;
693 atomic_inc(&vcpu->arch.nmi_queued);
760 return kvm_read_guest_page_mmu(vcpu, vcpu->arch.walk_mmu, gfn,
808 u64 pdpte[ARRAY_SIZE(vcpu->arch.walk_mmu->pdptrs)];
826 return memcmp(pdpte, vcpu->arch.walk_mmu->pdptrs, sizeof(pdpte)) != 0;
852 if ((vcpu->arch.efer & EFER_LME) && !is_paging(vcpu) &&
863 if (!(vcpu->arch.efer & EFER_LME) && (cr0 & X86_CR0_PG) &&
865 !load_pdptrs(vcpu, vcpu->arch.walk_mmu, kvm_read_cr3(vcpu)))
900 if (vcpu->arch.xcr0 != host_xcr0)
901 xsetbv(XCR_XFEATURE_ENABLED_MASK, vcpu->arch.xcr0);
903 if (vcpu->arch.xsaves_enabled &&
904 vcpu->arch.ia32_xss != host_xss)
905 wrmsrl(MSR_IA32_XSS, vcpu->arch.ia32_xss);
910 (vcpu->arch.xcr0 & XFEATURE_MASK_PKRU)) &&
911 vcpu->arch.pkru != vcpu->arch.host_pkru)
912 __write_pkru(vcpu->arch.pkru);
920 (vcpu->arch.xcr0 & XFEATURE_MASK_PKRU))) {
921 vcpu->arch.pkru = rdpkru();
922 if (vcpu->arch.pkru != vcpu->arch.host_pkru)
923 __write_pkru(vcpu->arch.host_pkru);
928 if (vcpu->arch.xcr0 != host_xcr0)
931 if (vcpu->arch.xsaves_enabled &&
932 vcpu->arch.ia32_xss != host_xss)
942 u64 old_xcr0 = vcpu->arch.xcr0;
958 valid_bits = vcpu->arch.guest_supported_xcr0 | XFEATURE_MASK_FP;
972 vcpu->arch.xcr0 = xcr0;
995 if (cr4 & vcpu->arch.cr4_guest_rsvd_bits)
1022 && !load_pdptrs(vcpu, vcpu->arch.walk_mmu,
1069 (cr3 & vcpu->arch.cr3_lm_rsvd_bits))
1072 !load_pdptrs(vcpu, vcpu->arch.walk_mmu, cr3))
1076 vcpu->arch.cr3 = cr3;
1090 vcpu->arch.cr8 = cr8;
1100 return vcpu->arch.cr8;
1110 vcpu->arch.eff_db[i] = vcpu->arch.db[i];
1111 vcpu->arch.switch_db_regs |= KVM_DEBUGREG_RELOAD;
1120 dr7 = vcpu->arch.guest_debug_dr7;
1122 dr7 = vcpu->arch.dr7;
1124 vcpu->arch.switch_db_regs &= ~KVM_DEBUGREG_BP_ENABLED;
1126 vcpu->arch.switch_db_regs |= KVM_DEBUGREG_BP_ENABLED;
1141 size_t size = ARRAY_SIZE(vcpu->arch.db);
1145 vcpu->arch.db[array_index_nospec(dr, size)] = val;
1147 vcpu->arch.eff_db[dr] = val;
1153 vcpu->arch.dr6 = (val & DR6_VOLATILE) | kvm_dr6_fixed(vcpu);
1159 vcpu->arch.dr7 = (val & DR7_VOLATILE) | DR7_FIXED_1;
1179 size_t size = ARRAY_SIZE(vcpu->arch.db);
1183 *val = vcpu->arch.db[array_index_nospec(dr, size)];
1187 *val = vcpu->arch.dr6;
1191 *val = vcpu->arch.dr7;
1318 * by arch/x86/kvm/vmx/nested.c based on CPUID or other MSRs.
1525 u64 old_efer = vcpu->arch.efer;
1537 (vcpu->arch.efer & EFER_LME) != (efer & EFER_LME))
1542 efer |= vcpu->arch.efer & EFER_LMA;
1578 msr_filter = srcu_dereference(kvm->arch.msr_filter, &kvm->srcu);
1762 if (!(vcpu->kvm->arch.user_space_msr_mask & msr_reason))
1771 vcpu->arch.complete_userspace_io = completion;
1862 if (!lapic_in_kernel(vcpu) || !apic_x2apic_mode(vcpu->arch.apic))
1871 kvm_apic_send_ipi(vcpu->arch.apic, (u32)data, (u32)(data >> 32));
1872 kvm_lapic_set_reg(vcpu->arch.apic, APIC_ICR2, (u32)(data >> 32));
1873 kvm_lapic_set_reg(vcpu->arch.apic, APIC_ICR, (u32)data);
2008 kvm->arch.wall_clock = wall_clock;
2045 struct kvm_arch *ka = &vcpu->kvm->arch;
2054 vcpu->arch.time = system_time;
2058 vcpu->arch.pv_time_enabled = false;
2063 &vcpu->arch.pv_time, system_time & ~1ULL,
2065 vcpu->arch.pv_time_enabled = true;
2124 vcpu->arch.tsc_scaling_ratio = kvm_default_tsc_scaling_ratio;
2131 vcpu->arch.tsc_catchup = 1;
2132 vcpu->arch.tsc_always_catchup = 1;
2150 vcpu->arch.tsc_scaling_ratio = ratio;
2162 vcpu->arch.tsc_scaling_ratio = kvm_default_tsc_scaling_ratio;
2168 &vcpu->arch.virtual_tsc_shift,
2169 &vcpu->arch.virtual_tsc_mult);
2170 vcpu->arch.virtual_tsc_khz = user_tsc_khz;
2189 u64 tsc = pvclock_scale_delta(kernel_ns-vcpu->arch.this_tsc_nsec,
2190 vcpu->arch.virtual_tsc_mult,
2191 vcpu->arch.virtual_tsc_shift);
2192 tsc += vcpu->arch.this_tsc_write;
2205 struct kvm_arch *ka = &vcpu->kvm->arch;
2247 u64 ratio = vcpu->arch.tsc_scaling_ratio;
2267 return vcpu->arch.l1_tsc_offset + kvm_scale_tsc(vcpu, host_tsc);
2273 vcpu->arch.l1_tsc_offset = offset;
2274 vcpu->arch.tsc_offset = kvm_x86_ops.write_l1_tsc_offset(vcpu, offset);
2299 raw_spin_lock_irqsave(&kvm->arch.tsc_write_lock, flags);
2302 elapsed = ns - kvm->arch.last_tsc_nsec;
2304 if (vcpu->arch.virtual_tsc_khz) {
2313 u64 tsc_exp = kvm->arch.last_tsc_write +
2315 u64 tsc_hz = vcpu->arch.virtual_tsc_khz * 1000LL;
2333 vcpu->arch.virtual_tsc_khz == kvm->arch.last_tsc_khz) {
2335 offset = kvm->arch.cur_tsc_offset;
2342 already_matched = (vcpu->arch.this_tsc_generation == kvm->arch.cur_tsc_generation);
2351 * These values are tracked in kvm->arch.cur_xxx variables.
2353 kvm->arch.cur_tsc_generation++;
2354 kvm->arch.cur_tsc_nsec = ns;
2355 kvm->arch.cur_tsc_write = data;
2356 kvm->arch.cur_tsc_offset = offset;
2364 kvm->arch.last_tsc_nsec = ns;
2365 kvm->arch.last_tsc_write = data;
2366 kvm->arch.last_tsc_khz = vcpu->arch.virtual_tsc_khz;
2368 vcpu->arch.last_guest_tsc = data;
2371 vcpu->arch.this_tsc_generation = kvm->arch.cur_tsc_generation;
2372 vcpu->arch.this_tsc_nsec = kvm->arch.cur_tsc_nsec;
2373 vcpu->arch.this_tsc_write = kvm->arch.cur_tsc_write;
2376 raw_spin_unlock_irqrestore(&kvm->arch.tsc_write_lock, flags);
2378 spin_lock(&kvm->arch.pvclock_gtod_sync_lock);
2380 kvm->arch.nr_vcpus_matched_tsc = 0;
2382 kvm->arch.nr_vcpus_matched_tsc++;
2386 spin_unlock(&kvm->arch.pvclock_gtod_sync_lock);
2392 u64 tsc_offset = vcpu->arch.l1_tsc_offset;
2398 if (vcpu->arch.tsc_scaling_ratio != kvm_default_tsc_scaling_ratio)
2569 struct kvm_arch *ka = &kvm->arch;
2607 struct kvm_arch *ka = &kvm->arch;
2627 struct kvm_arch *ka = &kvm->arch;
2659 struct kvm_vcpu_arch *vcpu = &v->arch;
2717 struct kvm_vcpu_arch *vcpu = &v->arch;
2718 struct kvm_arch *ka = &v->kvm->arch;
2787 vcpu->hv_clock.system_time = kernel_ns + v->kvm->arch.kvmclock_offset;
2826 struct kvm *kvm = container_of(ka, struct kvm, arch);
2840 schedule_delayed_work(&kvm->arch.kvmclock_update_work,
2851 struct kvm *kvm = container_of(ka, struct kvm, arch);
2856 schedule_delayed_work(&kvm->arch.kvmclock_update_work, 0);
2857 schedule_delayed_work(&kvm->arch.kvmclock_sync_work,
2868 return !!(vcpu->arch.msr_hwcr & BIT_ULL(18));
2875 u64 mcg_cap = vcpu->arch.mcg_cap;
2882 vcpu->arch.mcg_status = data;
2890 vcpu->arch.mcg_ctl = data;
2918 vcpu->arch.mce_banks[offset] = data;
2930 u8 *blob_addr = lm ? (u8 *)(long)kvm->arch.xen_hvm_config.blob_addr_64
2931 : (u8 *)(long)kvm->arch.xen_hvm_config.blob_addr_32;
2932 u8 blob_size = lm ? kvm->arch.xen_hvm_config.blob_size_64
2933 : kvm->arch.xen_hvm_config.blob_size_32;
2956 return (vcpu->arch.apf.msr_en_val & mask) == mask;
2978 vcpu->arch.apf.msr_en_val = data;
2986 if (kvm_gfn_to_hva_cache_init(vcpu->kvm, &vcpu->arch.apf.data, gpa,
2990 vcpu->arch.apf.send_user_only = !(data & KVM_ASYNC_PF_SEND_ALWAYS);
2991 vcpu->arch.apf.delivery_as_pf_vmexit = data & KVM_ASYNC_PF_DELIVERY_AS_PF_VMEXIT;
3007 vcpu->arch.apf.msr_int_val = data;
3009 vcpu->arch.apf.vec = data & KVM_ASYNC_PF_VEC_MASK;
3016 vcpu->arch.pv_time_enabled = false;
3017 vcpu->arch.time = 0;
3034 struct gfn_to_hva_cache *ghc = &vcpu->arch.st.cache;
3037 gpa_t gpa = vcpu->arch.st.msr_val & KVM_STEAL_VALID_BITS;
3041 if (!(vcpu->arch.st.msr_val & KVM_MSR_ENABLED))
3084 vcpu->arch.st.preempted = 0;
3098 vcpu->arch.st.preempted = 0;
3112 vcpu->arch.st.last_steal;
3113 vcpu->arch.st.last_steal = current->sched_info.run_delay;
3144 vcpu->arch.microcode_version = data;
3149 vcpu->arch.arch_capabilities = data;
3161 vcpu->arch.perf_capabilities = data;
3174 vcpu->arch.msr_hwcr = data;
3212 s64 adj = data - vcpu->arch.ia32_tsc_adjust_msr;
3219 vcpu->arch.ia32_tsc_adjust_msr = data;
3224 ((vcpu->arch.ia32_misc_enable_msr ^ data) & MSR_IA32_MISC_ENABLE_MWAIT)) {
3227 vcpu->arch.ia32_misc_enable_msr = data;
3230 vcpu->arch.ia32_misc_enable_msr = data;
3236 vcpu->arch.smbase = data;
3239 vcpu->arch.msr_ia32_power_ctl = data;
3245 u64 adj = kvm_compute_tsc_offset(vcpu, data) - vcpu->arch.l1_tsc_offset;
3247 vcpu->arch.ia32_tsc_adjust_msr += adj;
3261 vcpu->arch.ia32_xss = data;
3267 vcpu->arch.smi_count = data;
3311 vcpu->arch.apf.pageready_pending = false;
3325 vcpu->arch.st.msr_val = data;
3349 vcpu->arch.msr_kvm_poll_control = data;
3402 vcpu->arch.osvw.length = data;
3407 vcpu->arch.osvw.status = data;
3414 vcpu->arch.msr_platform_info = data;
3421 vcpu->arch.msr_misc_features_enables = data;
3424 if (msr && (msr == vcpu->kvm->arch.xen_hvm_config.msr))
3437 u64 mcg_cap = vcpu->arch.mcg_cap;
3446 data = vcpu->arch.mcg_cap;
3451 data = vcpu->arch.mcg_ctl;
3454 data = vcpu->arch.mcg_status;
3463 data = vcpu->arch.mce_banks[offset];
3517 msr_info->data = vcpu->arch.microcode_version;
3523 msr_info->data = vcpu->arch.arch_capabilities;
3529 msr_info->data = vcpu->arch.perf_capabilities;
3532 msr_info->data = vcpu->arch.msr_ia32_power_ctl;
3544 u64 tsc_offset = msr_info->host_initiated ? vcpu->arch.l1_tsc_offset :
3545 vcpu->arch.tsc_offset;
3579 msr_info->data = (u64)vcpu->arch.ia32_tsc_adjust_msr;
3582 msr_info->data = vcpu->arch.ia32_misc_enable_msr;
3587 msr_info->data = vcpu->arch.smbase;
3590 msr_info->data = vcpu->arch.smi_count;
3599 msr_info->data = vcpu->arch.efer;
3605 msr_info->data = vcpu->kvm->arch.wall_clock;
3611 msr_info->data = vcpu->kvm->arch.wall_clock;
3617 msr_info->data = vcpu->arch.time;
3623 msr_info->data = vcpu->arch.time;
3629 msr_info->data = vcpu->arch.apf.msr_en_val;
3635 msr_info->data = vcpu->arch.apf.msr_int_val;
3647 msr_info->data = vcpu->arch.st.msr_val;
3653 msr_info->data = vcpu->arch.pv_eoi.msr_val;
3659 msr_info->data = vcpu->arch.msr_kvm_poll_control;
3673 msr_info->data = vcpu->arch.ia32_xss;
3715 msr_info->data = vcpu->arch.osvw.length;
3720 msr_info->data = vcpu->arch.osvw.status;
3724 !vcpu->kvm->arch.guest_can_read_msr_platform_info)
3726 msr_info->data = vcpu->arch.msr_platform_info;
3729 msr_info->data = vcpu->arch.msr_misc_features_enables;
3732 msr_info->data = vcpu->arch.msr_hwcr;
4059 cpumask_set_cpu(cpu, vcpu->arch.wbinvd_dirty_mask);
4068 vcpu->arch.host_pkru = read_pkru();
4071 if (unlikely(vcpu->arch.tsc_offset_adjustment)) {
4072 adjust_tsc_offset_host(vcpu, vcpu->arch.tsc_offset_adjustment);
4073 vcpu->arch.tsc_offset_adjustment = 0;
4078 s64 tsc_delta = !vcpu->arch.last_host_tsc ? 0 :
4079 rdtsc() - vcpu->arch.last_host_tsc;
4085 vcpu->arch.last_guest_tsc);
4087 vcpu->arch.tsc_catchup = 1;
4097 if (!vcpu->kvm->arch.use_master_clock || vcpu->cpu == -1)
4109 struct gfn_to_hva_cache *ghc = &vcpu->arch.st.cache;
4113 gpa_t gpa = vcpu->arch.st.msr_val & KVM_STEAL_VALID_BITS;
4122 if (!vcpu->arch.at_instruction_boundary) {
4128 if (!(vcpu->arch.st.msr_val & KVM_MSR_ENABLED))
4131 if (vcpu->arch.st.preempted)
4149 vcpu->arch.st.preempted = KVM_VCPU_PREEMPTED;
4159 vcpu->arch.preempted_in_kernel = !kvm_x86_ops.get_cpl(vcpu);
4171 vcpu->arch.last_host_tsc = rdtsc();
4183 if (vcpu->arch.apicv_active)
4230 !vcpu->arch.exception.pending);
4252 if (vcpu->arch.pending_external_vector != -1)
4255 vcpu->arch.pending_external_vector = irq->irq;
4279 vcpu->arch.tpr_access_reporting = !!tac->enabled;
4295 vcpu->arch.mcg_cap = mcg_cap;
4298 vcpu->arch.mcg_ctl = ~(u64)0;
4301 vcpu->arch.mce_banks[bank*4] = ~(u64)0;
4311 u64 mcg_cap = vcpu->arch.mcg_cap;
4313 u64 *banks = vcpu->arch.mce_banks;
4322 vcpu->arch.mcg_ctl != ~(u64)0)
4332 if ((vcpu->arch.mcg_status & MCG_STATUS_MCIP) ||
4341 vcpu->arch.mcg_status = mce->mcg_status;
4375 if (!vcpu->kvm->arch.exception_payload_enabled &&
4376 vcpu->arch.exception.pending && vcpu->arch.exception.has_payload)
4385 if (kvm_exception_is_soft(vcpu->arch.exception.nr)) {
4389 events->exception.injected = vcpu->arch.exception.injected;
4390 events->exception.pending = vcpu->arch.exception.pending;
4396 if (!vcpu->kvm->arch.exception_payload_enabled)
4398 vcpu->arch.exception.pending;
4400 events->exception.nr = vcpu->arch.exception.nr;
4401 events->exception.has_error_code = vcpu->arch.exception.has_error_code;
4402 events->exception.error_code = vcpu->arch.exception.error_code;
4403 events->exception_has_payload = vcpu->arch.exception.has_payload;
4404 events->exception_payload = vcpu->arch.exception.payload;
4407 vcpu->arch.interrupt.injected && !vcpu->arch.interrupt.soft;
4408 events->interrupt.nr = vcpu->arch.interrupt.nr;
4412 events->nmi.injected = vcpu->arch.nmi_injected;
4413 events->nmi.pending = vcpu->arch.nmi_pending != 0;
4420 events->smi.pending = vcpu->arch.smi_pending;
4422 !!(vcpu->arch.hflags & HF_SMM_INSIDE_NMI_MASK);
4428 if (vcpu->kvm->arch.exception_payload_enabled)
4447 if (!vcpu->kvm->arch.exception_payload_enabled)
4465 vcpu->arch.mp_state == KVM_MP_STATE_INIT_RECEIVED)
4469 vcpu->arch.exception.injected = events->exception.injected;
4470 vcpu->arch.exception.pending = events->exception.pending;
4471 vcpu->arch.exception.nr = events->exception.nr;
4472 vcpu->arch.exception.has_error_code = events->exception.has_error_code;
4473 vcpu->arch.exception.error_code = events->exception.error_code;
4474 vcpu->arch.exception.has_payload = events->exception_has_payload;
4475 vcpu->arch.exception.payload = events->exception_payload;
4477 vcpu->arch.interrupt.injected = events->interrupt.injected;
4478 vcpu->arch.interrupt.nr = events->interrupt.nr;
4479 vcpu->arch.interrupt.soft = events->interrupt.soft;
4484 vcpu->arch.nmi_injected = events->nmi.injected;
4486 vcpu->arch.nmi_pending = events->nmi.pending;
4491 vcpu->arch.apic->sipi_vector = events->sipi_vector;
4494 if (!!(vcpu->arch.hflags & HF_SMM_MASK) != events->smi.smm) {
4496 vcpu->arch.hflags |= HF_SMM_MASK;
4498 vcpu->arch.hflags &= ~HF_SMM_MASK;
4504 vcpu->arch.smi_pending = events->smi.pending;
4508 vcpu->arch.hflags |= HF_SMM_INSIDE_NMI_MASK;
4510 vcpu->arch.hflags &= ~HF_SMM_INSIDE_NMI_MASK;
4515 set_bit(KVM_APIC_INIT, &vcpu->arch.apic->pending_events);
4517 clear_bit(KVM_APIC_INIT, &vcpu->arch.apic->pending_events);
4532 memcpy(dbgregs->db, vcpu->arch.db, sizeof(vcpu->arch.db));
4535 dbgregs->dr7 = vcpu->arch.dr7;
4549 memcpy(vcpu->arch.db, dbgregs->db, sizeof(vcpu->arch.db));
4551 vcpu->arch.dr6 = dbgregs->dr6;
4552 vcpu->arch.dr7 = dbgregs->dr7;
4562 struct xregs_state *xsave = &vcpu->arch.guest_fpu->state.xsave;
4573 xstate_bv &= vcpu->arch.guest_supported_xcr0 | XFEATURE_MASK_FPSSE;
4591 memcpy(dest + offset, &vcpu->arch.pkru,
4592 sizeof(vcpu->arch.pkru));
4604 struct xregs_state *xsave = &vcpu->arch.guest_fpu->state.xsave;
4634 memcpy(&vcpu->arch.pkru, src + offset,
4635 sizeof(vcpu->arch.pkru));
4652 &vcpu->arch.guest_fpu->state.fxsave,
4681 memcpy(&vcpu->arch.guest_fpu->state.fxsave,
4698 guest_xcrs->xcrs[0].value = vcpu->arch.xcr0;
4732 if (!vcpu->arch.pv_time_enabled)
4734 vcpu->arch.pvclock_set_guest_stopped_request = true;
4778 vcpu->arch.pv_cpuid.enforce = cap->args[0];
4779 if (vcpu->arch.pv_cpuid.enforce)
5065 r = vcpu->arch.virtual_tsc_khz;
5202 kvm->arch.n_requested_mmu_pages = kvm_nr_mmu_pages;
5210 return kvm->arch.n_max_mmu_pages;
5215 struct kvm_pic *pic = kvm->arch.vpic;
5240 struct kvm_pic *pic = kvm->arch.vpic;
5270 struct kvm_kpit_state *kps = &kvm->arch.vpit->pit_state;
5283 struct kvm_pit *pit = kvm->arch.vpit;
5295 mutex_lock(&kvm->arch.vpit->pit_state.lock);
5296 memcpy(ps->channels, &kvm->arch.vpit->pit_state.channels,
5298 ps->flags = kvm->arch.vpit->pit_state.flags;
5299 mutex_unlock(&kvm->arch.vpit->pit_state.lock);
5309 struct kvm_pit *pit = kvm->arch.vpit;
5329 struct kvm_pit *pit = kvm->arch.vpit;
5373 kvm->arch.disabled_quirks = cap->args[0];
5391 kvm->arch.irqchip_mode = KVM_IRQCHIP_SPLIT;
5392 kvm->arch.nr_reserved_ioapic_pins = cap->args[0];
5404 kvm->arch.x2apic_format = true;
5406 kvm->arch.x2apic_broadcast_quirk_disabled = true;
5417 kvm->arch.mwait_in_guest = true;
5419 kvm->arch.hlt_in_guest = true;
5421 kvm->arch.pause_in_guest = true;
5423 kvm->arch.cstate_in_guest = true;
5427 kvm->arch.guest_can_read_msr_platform_info = cap->args[0];
5431 kvm->arch.exception_payload_enabled = cap->args[0];
5440 kvm->arch.user_space_msr_mask = cap->args[0];
5555 old_filter = srcu_dereference_check(kvm->arch.msr_filter, &kvm->srcu, 1);
5557 rcu_assign_pointer(kvm->arch.msr_filter, new_filter);
5695 kvm->arch.irqchip_mode = KVM_IRQCHIP_KERNEL;
5711 if (kvm->arch.vpit)
5714 kvm->arch.vpit = kvm_create_pit(kvm, u.pit_config.flags);
5715 if (kvm->arch.vpit)
5767 if (!kvm->arch.vpit)
5784 if (!kvm->arch.vpit)
5793 if (!kvm->arch.vpit)
5810 if (!kvm->arch.vpit)
5823 if (!kvm->arch.vpit)
5834 kvm->arch.bsp_vcpu_id = arg;
5845 memcpy(&kvm->arch.xen_hvm_config, &xhc, sizeof(xhc));
5869 kvm->arch.kvmclock_offset += user_ns.clock - now_ns;
5879 user_ns.flags = kvm->arch.use_master_clock ? KVM_CLOCK_TSC_STABLE : 0;
6050 !kvm_iodevice_write(vcpu, &vcpu->arch.apic->dev, addr, n, v))
6070 !kvm_iodevice_read(vcpu, &vcpu->arch.apic->dev,
6105 t_gpa = vcpu->arch.mmu->gva_to_gpa(vcpu, gpa, access, exception);
6114 return vcpu->arch.walk_mmu->gva_to_gpa(vcpu, gva, access, exception);
6122 return vcpu->arch.walk_mmu->gva_to_gpa(vcpu, gva, access, exception);
6130 return vcpu->arch.walk_mmu->gva_to_gpa(vcpu, gva, access, exception);
6137 return vcpu->arch.walk_mmu->gva_to_gpa(vcpu, gva, 0, exception);
6148 gpa_t gpa = vcpu->arch.walk_mmu->gva_to_gpa(vcpu, addr, access,
6182 gpa_t gpa = vcpu->arch.walk_mmu->gva_to_gpa(vcpu, addr, access|PFERR_FETCH_MASK,
6246 gpa_t gpa = vcpu->arch.walk_mmu->gva_to_gpa(vcpu, addr,
6287 vcpu->arch.l1tf_flush_l1d = true;
6344 && !permission_fault(vcpu, vcpu->arch.walk_mmu,
6345 vcpu->arch.mmio_access, 0, access)) {
6346 *gpa = vcpu->arch.mmio_gfn << PAGE_SHIFT |
6352 *gpa = vcpu->arch.walk_mmu->gva_to_gpa(vcpu, gva, access, exception);
6454 struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt;
6656 for (i = 0; i < vcpu->arch.pio.count; i++) {
6657 if (vcpu->arch.pio.in)
6658 r = kvm_io_bus_read(vcpu, KVM_PIO_BUS, vcpu->arch.pio.port,
6659 vcpu->arch.pio.size, pd);
6662 vcpu->arch.pio.port, vcpu->arch.pio.size,
6666 pd += vcpu->arch.pio.size;
6675 vcpu->arch.pio.port = port;
6676 vcpu->arch.pio.in = in;
6677 vcpu->arch.pio.count = count;
6678 vcpu->arch.pio.size = size;
6680 if (!kernel_pio(vcpu, vcpu->arch.pio_data)) {
6681 vcpu->arch.pio.count = 0;
6700 if (vcpu->arch.pio.count)
6703 memset(vcpu->arch.pio_data, 0, size * count);
6708 memcpy(val, vcpu->arch.pio_data, size * count);
6709 trace_kvm_pio(KVM_PIO_IN, port, size, count, vcpu->arch.pio_data);
6710 vcpu->arch.pio.count = 0;
6729 memcpy(vcpu->arch.pio_data, val, size * count);
6730 trace_kvm_pio(KVM_PIO_OUT, port, size, count, vcpu->arch.pio_data);
6759 cpumask_set_cpu(cpu, vcpu->arch.wbinvd_dirty_mask);
6760 smp_call_function_many(vcpu->arch.wbinvd_dirty_mask,
6763 cpumask_clear(vcpu->arch.wbinvd_dirty_mask);
6811 value = vcpu->arch.cr2;
6840 vcpu->arch.cr2 = val;
6992 return vcpu->arch.smbase;
6999 vcpu->arch.smbase = smbase;
7016 emul_to_vcpu(ctxt)->arch.halt_request = 1;
7071 return emul_to_vcpu(ctxt)->arch.hflags;
7078 vcpu->arch.hflags = emul_flags;
7167 struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt;
7191 vcpu->arch.emulate_ctxt = ctxt;
7198 struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt;
7223 vcpu->arch.emulate_regs_need_sync_from_vcpu = false;
7228 struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt;
7291 if (!vcpu->arch.mmu->direct_map) {
7324 if (vcpu->arch.mmu->direct_map) {
7328 indirect_shadow_pages = vcpu->kvm->arch.indirect_shadow_pages;
7358 last_retry_eip = vcpu->arch.last_retry_eip;
7359 last_retry_addr = vcpu->arch.last_retry_addr;
7374 vcpu->arch.last_retry_eip = vcpu->arch.last_retry_addr = 0;
7389 vcpu->arch.last_retry_eip = ctxt->eip;
7390 vcpu->arch.last_retry_addr = cr2_or_gpa;
7392 if (!vcpu->arch.mmu->direct_map)
7405 if (!(vcpu->arch.hflags & HF_SMM_MASK)) {
7407 trace_kvm_enter_smm(vcpu->vcpu_id, vcpu->arch.smbase, false);
7436 kvm_run->debug.arch.dr6 = DR6_BS | DR6_FIXED_1 | DR6_RTM;
7437 kvm_run->debug.arch.pc = kvm_get_linear_rip(vcpu);
7438 kvm_run->debug.arch.exception = DB_VECTOR;
7472 (vcpu->arch.guest_debug_dr7 & DR7_BP_EN_MASK)) {
7476 vcpu->arch.guest_debug_dr7,
7477 vcpu->arch.eff_db);
7480 kvm_run->debug.arch.dr6 = dr6 | DR6_FIXED_1 | DR6_RTM;
7481 kvm_run->debug.arch.pc = eip;
7482 kvm_run->debug.arch.exception = DB_VECTOR;
7489 if (unlikely(vcpu->arch.dr7 & DR7_BP_EN_MASK) &&
7493 vcpu->arch.dr7,
7494 vcpu->arch.db);
7550 struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt;
7570 struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt;
7577 vcpu->arch.l1tf_flush_l1d = true;
7583 write_fault_to_spt = vcpu->arch.write_fault_to_shadow_pgtable;
7584 vcpu->arch.write_fault_to_shadow_pgtable = false;
7649 if (vcpu->arch.emulate_regs_need_sync_from_vcpu) {
7650 vcpu->arch.emulate_regs_need_sync_from_vcpu = false;
7660 if (vcpu->arch.mmu->direct_map) {
7686 } else if (vcpu->arch.pio.count) {
7687 if (!vcpu->arch.pio.in) {
7689 vcpu->arch.pio.count = 0;
7692 vcpu->arch.complete_userspace_io = complete_emulated_pio;
7701 vcpu->arch.complete_userspace_io = complete_emulated_mmio;
7710 vcpu->arch.emulate_regs_need_sync_to_vcpu = false;
7736 vcpu->arch.emulate_regs_need_sync_to_vcpu = true;
7756 vcpu->arch.pio.count = 0;
7762 vcpu->arch.pio.count = 0;
7764 if (unlikely(!kvm_is_linear_rip(vcpu, vcpu->arch.pio.linear_rip)))
7785 vcpu->arch.complete_userspace_io =
7789 vcpu->arch.pio.linear_rip = kvm_get_linear_rip(vcpu);
7790 vcpu->arch.complete_userspace_io = complete_fast_pio_out;
7799 /* We should only ever be called with arch.pio.count equal to 1 */
7800 BUG_ON(vcpu->arch.pio.count != 1);
7802 if (unlikely(!kvm_is_linear_rip(vcpu, vcpu->arch.pio.linear_rip))) {
7803 vcpu->arch.pio.count = 0;
7808 val = (vcpu->arch.pio.size < 4) ? kvm_rax_read(vcpu) : 0;
7811 * Since vcpu->arch.pio.count == 1 let emulator_pio_in perform
7814 emulator_pio_in(vcpu, vcpu->arch.pio.size, vcpu->arch.pio.port, &val, 1);
7835 vcpu->arch.pio.linear_rip = kvm_get_linear_rip(vcpu);
7836 vcpu->arch.complete_userspace_io = complete_fast_pio_in;
7892 struct kvm_arch *ka = &kvm->arch;
8073 (unsigned long *)&vcpu->arch.pmu.global_status);
8264 vcpu->arch.mp_state = KVM_MP_STATE_HALTED;
8337 return (READ_ONCE(kvm->arch.apicv_inhibit_reasons) == 0);
8345 &kvm->arch.apicv_inhibit_reasons);
8348 &kvm->arch.apicv_inhibit_reasons);
8358 map = rcu_dereference(kvm->arch.apic_map);
8486 if (vcpu->arch.apicv_active)
8489 if (!vcpu->arch.apic->vapic_addr)
8504 trace_kvm_inj_exception(vcpu->arch.exception.nr,
8505 vcpu->arch.exception.has_error_code,
8506 vcpu->arch.exception.error_code,
8507 vcpu->arch.exception.injected);
8509 if (vcpu->arch.exception.error_code && !is_protmode(vcpu))
8510 vcpu->arch.exception.error_code = false;
8521 if (vcpu->arch.exception.injected) {
8539 else if (!vcpu->arch.exception.pending) {
8540 if (vcpu->arch.nmi_injected) {
8543 } else if (vcpu->arch.interrupt.injected) {
8549 WARN_ON_ONCE(vcpu->arch.exception.injected &&
8550 vcpu->arch.exception.pending);
8565 if (vcpu->arch.exception.pending) {
8576 if (exception_type(vcpu->arch.exception.nr) == EXCPT_FAULT)
8580 if (vcpu->arch.exception.nr == DB_VECTOR) {
8582 if (vcpu->arch.dr7 & DR7_GD) {
8583 vcpu->arch.dr7 &= ~DR7_GD;
8590 vcpu->arch.exception.pending = false;
8591 vcpu->arch.exception.injected = true;
8607 if (vcpu->arch.smi_pending) {
8612 vcpu->arch.smi_pending = false;
8613 ++vcpu->arch.smi_count;
8620 if (vcpu->arch.nmi_pending) {
8625 --vcpu->arch.nmi_pending;
8626 vcpu->arch.nmi_injected = true;
8631 if (vcpu->arch.nmi_pending)
8653 WARN_ON(vcpu->arch.exception.pending);
8670 if (kvm_x86_ops.get_nmi_mask(vcpu) || vcpu->arch.nmi_injected)
8673 vcpu->arch.nmi_pending += atomic_xchg(&vcpu->arch.nmi_queued, 0);
8674 vcpu->arch.nmi_pending = min(vcpu->arch.nmi_pending, limit);
8775 put_smstate(u32, buf, 0x7ef8, vcpu->arch.smbase);
8801 put_smstate(u32, buf, 0x7f00, vcpu->arch.smbase);
8806 put_smstate(u64, buf, 0x7ed0, vcpu->arch.efer);
8840 trace_kvm_enter_smm(vcpu->vcpu_id, vcpu->arch.smbase, true);
8856 vcpu->arch.hflags |= HF_SMM_MASK;
8857 kvm_vcpu_write_guest(vcpu, vcpu->arch.smbase + 0xfe00, buf, sizeof(buf));
8860 vcpu->arch.hflags |= HF_SMM_INSIDE_NMI_MASK;
8867 cr0 = vcpu->arch.cr0 & ~(X86_CR0_PE | X86_CR0_EM | X86_CR0_TS | X86_CR0_PG);
8869 vcpu->arch.cr0 = cr0;
8879 cs.selector = (vcpu->arch.smbase >> 4) & 0xffff;
8880 cs.base = vcpu->arch.smbase;
8915 vcpu->arch.smi_pending = true;
8942 vcpu->arch.apicv_active = kvm_apicv_activated(vcpu->kvm);
8964 old = READ_ONCE(kvm->arch.apicv_inhibit_reasons);
8973 old = cmpxchg(&kvm->arch.apicv_inhibit_reasons, expected, new);
9001 bitmap_zero(vcpu->arch.ioapic_handled_vectors, 256);
9004 kvm_scan_ioapic_routes(vcpu, vcpu->arch.ioapic_handled_vectors);
9006 if (vcpu->arch.apicv_active)
9009 kvm_ioapic_scan_entry(vcpu, vcpu->arch.ioapic_handled_vectors);
9013 vcpu->arch.load_eoi_exitmap_pending = true;
9022 if (!kvm_apic_hw_enabled(vcpu->arch.apic))
9025 bitmap_or((ulong *)eoi_exit_bitmap, vcpu->arch.ioapic_handled_vectors,
9130 vcpu->arch.apf.halted = true;
9145 BUG_ON(vcpu->arch.pending_ioapic_eoi > 255);
9146 if (test_bit(vcpu->arch.pending_ioapic_eoi,
9147 vcpu->arch.ioapic_handled_vectors)) {
9150 vcpu->arch.pending_ioapic_eoi;
9175 vcpu->run->hyperv = vcpu->arch.hyperv.exit;
9198 if (vcpu->arch.mp_state == KVM_MP_STATE_INIT_RECEIVED) {
9250 if (kvm_lapic_enabled(vcpu) && vcpu->arch.apicv_active)
9274 if (unlikely(vcpu->arch.switch_db_regs)) {
9276 set_debugreg(vcpu->arch.eff_db[0], 0);
9277 set_debugreg(vcpu->arch.eff_db[1], 1);
9278 set_debugreg(vcpu->arch.eff_db[2], 2);
9279 set_debugreg(vcpu->arch.eff_db[3], 3);
9280 set_debugreg(vcpu->arch.dr6, 6);
9281 vcpu->arch.switch_db_regs &= ~KVM_DEBUGREG_RELOAD;
9294 if (unlikely(vcpu->arch.switch_db_regs & KVM_DEBUGREG_WONT_EXIT)) {
9299 vcpu->arch.switch_db_regs &= ~KVM_DEBUGREG_RELOAD;
9312 vcpu->arch.last_vmentry_cpu = vcpu->cpu;
9313 vcpu->arch.last_guest_tsc = kvm_read_l1_tsc(vcpu, rdtsc());
9343 s64 delta = vcpu->arch.apic->lapic_timer.advance_expire_delta;
9346 vcpu->arch.apic->lapic_timer.advance_expire_delta = S64_MIN;
9363 if (unlikely(vcpu->arch.tsc_always_catchup))
9366 if (vcpu->arch.apic_attention)
9376 if (unlikely(vcpu->arch.apic_attention))
9398 switch(vcpu->arch.mp_state) {
9400 vcpu->arch.pv.pv_unhalted = false;
9401 vcpu->arch.mp_state =
9405 vcpu->arch.apf.halted = false;
9420 return (vcpu->arch.mp_state == KVM_MP_STATE_RUNNABLE &&
9421 !vcpu->arch.apf.halted);
9430 vcpu->arch.l1tf_flush_l1d = true;
9439 vcpu->arch.at_instruction_boundary = false;
9487 BUG_ON(!vcpu->arch.pio.count);
9551 vcpu->arch.complete_userspace_io = complete_emulated_mmio;
9573 kvm_save_current_fpu(vcpu->arch.user_fpu);
9576 __copy_kernel_to_fpregs(&vcpu->arch.guest_fpu->state,
9590 kvm_save_current_fpu(vcpu->arch.guest_fpu);
9592 copy_kernel_to_fpregs(&vcpu->arch.user_fpu->state);
9610 if (unlikely(vcpu->arch.mp_state == KVM_MP_STATE_UNINITIALIZED)) {
9646 if (unlikely(vcpu->arch.complete_userspace_io)) {
9647 int (*cui)(struct kvm_vcpu *) = vcpu->arch.complete_userspace_io;
9648 vcpu->arch.complete_userspace_io = NULL;
9653 WARN_ON(vcpu->arch.pio.count || vcpu->mmio_needed);
9673 if (vcpu->arch.emulate_regs_need_sync_to_vcpu) {
9681 emulator_writeback_register_cache(vcpu->arch.emulate_ctxt);
9682 vcpu->arch.emulate_regs_need_sync_to_vcpu = false;
9717 vcpu->arch.emulate_regs_need_sync_from_vcpu = true;
9718 vcpu->arch.emulate_regs_need_sync_to_vcpu = false;
9742 vcpu->arch.exception.pending = false;
9787 sregs->cr2 = vcpu->arch.cr2;
9791 sregs->efer = vcpu->arch.efer;
9796 if (vcpu->arch.interrupt.injected && !vcpu->arch.interrupt.soft)
9797 set_bit(vcpu->arch.interrupt.nr,
9818 if (vcpu->arch.mp_state == KVM_MP_STATE_HALTED &&
9819 vcpu->arch.pv.pv_unhalted)
9822 mp_state->mp_state = vcpu->arch.mp_state;
9846 if ((kvm_vcpu_latch_init(vcpu) || vcpu->arch.smi_pending) &&
9852 vcpu->arch.mp_state = KVM_MP_STATE_INIT_RECEIVED;
9853 set_bit(KVM_APIC_SIPI, &vcpu->arch.apic->pending_events);
9855 vcpu->arch.mp_state = mp_state->mp_state;
9867 struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt;
9898 if (sregs->cr3 & vcpu->arch.cr3_lm_rsvd_bits)
9936 vcpu->arch.cr2 = sregs->cr2;
9938 vcpu->arch.cr3 = sregs->cr3;
9943 mmu_reset_needed |= vcpu->arch.efer != sregs->efer;
9948 vcpu->arch.cr0 = sregs->cr0;
9959 load_pdptrs(vcpu, vcpu->arch.walk_mmu, kvm_read_cr3(vcpu));
9991 vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE;
10021 if (vcpu->arch.exception.pending)
10041 vcpu->arch.eff_db[i] = dbg->arch.debugreg[i];
10042 vcpu->arch.guest_debug_dr7 = dbg->arch.debugreg[7];
10045 vcpu->arch.eff_db[i] = vcpu->arch.db[i];
10050 vcpu->arch.singlestep_rip = kvm_rip_read(vcpu) +
10098 fxsave = &vcpu->arch.guest_fpu->state.fxsave;
10118 fxsave = &vcpu->arch.guest_fpu->state.fxsave;
10174 fpstate_init(&vcpu->arch.guest_fpu->state);
10176 vcpu->arch.guest_fpu->state.xsave.header.xcomp_bv =
10182 vcpu->arch.xcr0 = XFEATURE_MASK_FP;
10184 vcpu->arch.cr0 |= X86_CR0_ET;
10202 vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE;
10204 vcpu->arch.mp_state = KVM_MP_STATE_UNINITIALIZED;
10217 vcpu->arch.apicv_active = true;
10226 vcpu->arch.pio_data = page_address(page);
10228 vcpu->arch.mce_banks = kzalloc(KVM_MAX_MCE_BANKS * sizeof(u64) * 4,
10230 if (!vcpu->arch.mce_banks)
10232 vcpu->arch.mcg_cap = KVM_MAX_MCE_BANKS;
10234 if (!zalloc_cpumask_var(&vcpu->arch.wbinvd_dirty_mask,
10241 vcpu->arch.user_fpu = kmem_cache_zalloc(x86_fpu_cache,
10243 if (!vcpu->arch.user_fpu) {
10248 vcpu->arch.guest_fpu = kmem_cache_zalloc(x86_fpu_cache,
10250 if (!vcpu->arch.guest_fpu) {
10256 vcpu->arch.maxphyaddr = cpuid_query_maxphyaddr(vcpu);
10257 vcpu->arch.cr3_lm_rsvd_bits = rsvd_bits(cpuid_maxphyaddr(vcpu), 63);
10259 vcpu->arch.pat = MSR_IA32_CR_PAT_DEFAULT;
10264 vcpu->arch.pending_external_vector = -1;
10265 vcpu->arch.preempted_in_kernel = false;
10273 vcpu->arch.arch_capabilities = kvm_get_arch_capabilities();
10274 vcpu->arch.msr_platform_info = MSR_PLATFORM_INFO_CPUID_FAULT;
10283 kmem_cache_free(x86_fpu_cache, vcpu->arch.guest_fpu);
10285 kmem_cache_free(x86_fpu_cache, vcpu->arch.user_fpu);
10287 kmem_cache_free(x86_emulator_cache, vcpu->arch.emulate_ctxt);
10289 free_cpumask_var(vcpu->arch.wbinvd_dirty_mask);
10291 kfree(vcpu->arch.mce_banks);
10293 free_page((unsigned long)vcpu->arch.pio_data);
10314 vcpu->arch.msr_kvm_poll_control = 1;
10319 schedule_delayed_work(&kvm->arch.kvmclock_sync_work,
10331 kmem_cache_free(x86_emulator_cache, vcpu->arch.emulate_ctxt);
10332 free_cpumask_var(vcpu->arch.wbinvd_dirty_mask);
10333 kmem_cache_free(x86_fpu_cache, vcpu->arch.user_fpu);
10334 kmem_cache_free(x86_fpu_cache, vcpu->arch.guest_fpu);
10338 kfree(vcpu->arch.mce_banks);
10343 free_page((unsigned long)vcpu->arch.pio_data);
10344 kvfree(vcpu->arch.cpuid_entries);
10353 vcpu->arch.hflags = 0;
10355 vcpu->arch.smi_pending = 0;
10356 vcpu->arch.smi_count = 0;
10357 atomic_set(&vcpu->arch.nmi_queued, 0);
10358 vcpu->arch.nmi_pending = 0;
10359 vcpu->arch.nmi_injected = false;
10363 memset(vcpu->arch.db, 0, sizeof(vcpu->arch.db));
10365 vcpu->arch.dr6 = DR6_INIT;
10366 vcpu->arch.dr7 = DR7_FIXED_1;
10369 vcpu->arch.cr2 = 0;
10372 vcpu->arch.apf.msr_en_val = 0;
10373 vcpu->arch.apf.msr_int_val = 0;
10374 vcpu->arch.st.msr_val = 0;
10380 vcpu->arch.apf.halted = false;
10391 mpx_state_buffer = get_xsave_addr(&vcpu->arch.guest_fpu->state.xsave,
10395 mpx_state_buffer = get_xsave_addr(&vcpu->arch.guest_fpu->state.xsave,
10405 vcpu->arch.smbase = 0x30000;
10407 vcpu->arch.msr_misc_features_enables = 0;
10409 vcpu->arch.xcr0 = XFEATURE_MASK_FP;
10412 memset(vcpu->arch.regs, 0, sizeof(vcpu->arch.regs));
10413 vcpu->arch.regs_avail = ~0;
10414 vcpu->arch.regs_dirty = ~0;
10416 vcpu->arch.ia32_xss = 0;
10453 if (stable && vcpu->arch.last_host_tsc > local_tsc) {
10455 if (vcpu->arch.last_host_tsc > max_tsc)
10456 max_tsc = vcpu->arch.last_host_tsc;
10502 kvm->arch.backwards_tsc_observed = true;
10504 vcpu->arch.tsc_offset_adjustment += delta_cyc;
10505 vcpu->arch.last_host_tsc = local_tsc;
10515 kvm->arch.last_tsc_nsec = 0;
10516 kvm->arch.last_tsc_write = 0;
10591 return vcpu->kvm->arch.bsp_vcpu_id == vcpu->vcpu_id;
10597 return (vcpu->arch.apic_base & MSR_IA32_APICBASE_BSP) != 0;
10607 vcpu->arch.l1tf_flush_l1d = true;
10617 kfree(kvm->arch.hyperv.hv_pa_pg);
10633 INIT_HLIST_HEAD(&kvm->arch.mask_notifier_list);
10634 INIT_LIST_HEAD(&kvm->arch.active_mmu_pages);
10635 INIT_LIST_HEAD(&kvm->arch.zapped_obsolete_pages);
10636 INIT_LIST_HEAD(&kvm->arch.lpage_disallowed_mmu_pages);
10637 INIT_LIST_HEAD(&kvm->arch.assigned_dev_head);
10638 atomic_set(&kvm->arch.noncoherent_dma_count, 0);
10641 set_bit(KVM_USERSPACE_IRQ_SOURCE_ID, &kvm->arch.irq_sources_bitmap);
10644 &kvm->arch.irq_sources_bitmap);
10646 raw_spin_lock_init(&kvm->arch.tsc_write_lock);
10647 mutex_init(&kvm->arch.apic_map_lock);
10648 spin_lock_init(&kvm->arch.pvclock_gtod_sync_lock);
10650 kvm->arch.kvmclock_offset = -get_kvmclock_base_ns();
10653 kvm->arch.guest_can_read_msr_platform_info = true;
10655 INIT_DELAYED_WORK(&kvm->arch.kvmclock_update_work, kvmclock_update_fn);
10656 INIT_DELAYED_WORK(&kvm->arch.kvmclock_sync_work, kvmclock_sync_fn);
10701 cancel_delayed_work_sync(&kvm->arch.kvmclock_sync_work);
10702 cancel_delayed_work_sync(&kvm->arch.kvmclock_update_work);
10781 kvm_free_msr_filter(srcu_dereference_check(kvm->arch.msr_filter, &kvm->srcu, 1));
10785 kvfree(rcu_dereference_check(kvm->arch.apic_map, 1));
10786 kfree(srcu_dereference_check(kvm->arch.pmu_event_filter, &kvm->srcu, 1));
10797 kvfree(slot->arch.rmap[i]);
10798 slot->arch.rmap[i] = NULL;
10803 kvfree(slot->arch.lpage_info[i - 1]);
10804 slot->arch.lpage_info[i - 1] = NULL;
10820 memset(&slot->arch, 0, sizeof(slot->arch));
10831 slot->arch.rmap[i] =
10832 __vcalloc(lpages, sizeof(*slot->arch.rmap[i]),
10834 if (!slot->arch.rmap[i])
10843 slot->arch.lpage_info[i - 1] = linfo;
10869 kvfree(slot->arch.rmap[i]);
10870 slot->arch.rmap[i] = NULL;
10874 kvfree(slot->arch.lpage_info[i - 1]);
10875 slot->arch.lpage_info[i - 1] = NULL;
10998 if (!kvm->arch.n_requested_mmu_pages)
11038 if (vcpu->arch.pv.pv_unhalted)
11041 if (vcpu->arch.exception.pending)
11045 (vcpu->arch.nmi_pending &&
11050 (vcpu->arch.smi_pending &&
11077 if (READ_ONCE(vcpu->arch.pv.pv_unhalted))
11085 if (vcpu->arch.apicv_active && kvm_x86_ops.dy_apicv_has_pending_interrupt(vcpu))
11093 return vcpu->arch.preempted_in_kernel;
11135 kvm_is_linear_rip(vcpu, vcpu->arch.singlestep_rip))
11151 if ((vcpu->arch.mmu->direct_map != work->arch.direct_map) ||
11159 if (!vcpu->arch.mmu->direct_map &&
11160 work->arch.cr3 != vcpu->arch.mmu->get_guest_pgd(vcpu))
11182 while (vcpu->arch.apf.gfns[key] != ~0)
11185 vcpu->arch.apf.gfns[key] = gfn;
11194 (vcpu->arch.apf.gfns[key] != gfn &&
11195 vcpu->arch.apf.gfns[key] != ~0); i++)
11203 return vcpu->arch.apf.gfns[kvm_async_pf_gfn_slot(vcpu, gfn)] == gfn;
11212 if (WARN_ON_ONCE(vcpu->arch.apf.gfns[i] != gfn))
11216 vcpu->arch.apf.gfns[i] = ~0;
11219 if (vcpu->arch.apf.gfns[j] == ~0)
11221 k = kvm_async_pf_hash_fn(vcpu->arch.apf.gfns[j]);
11228 vcpu->arch.apf.gfns[i] = vcpu->arch.apf.gfns[j];
11237 return kvm_write_guest_cached(vcpu->kvm, &vcpu->arch.apf.data, &reason,
11245 return kvm_write_guest_offset_cached(vcpu->kvm, &vcpu->arch.apf.data,
11254 if (kvm_read_guest_offset_cached(vcpu->kvm, &vcpu->arch.apf.data,
11263 if (!vcpu->arch.apf.delivery_as_pf_vmexit && is_guest_mode(vcpu))
11267 (vcpu->arch.apf.send_user_only && kvm_x86_ops.get_cpl(vcpu) == 0))
11277 vcpu->arch.exception.pending))
11295 trace_kvm_async_pf_not_present(work->arch.token, work->cr2_or_gpa);
11296 kvm_add_async_pf_gfn(vcpu, work->arch.gfn);
11304 fault.address = work->arch.token;
11327 .vector = vcpu->arch.apf.vec
11331 work->arch.token = ~0; /* broadcast wakeup */
11333 kvm_del_async_pf_gfn(vcpu, work->arch.gfn);
11334 trace_kvm_async_pf_ready(work->arch.token, work->cr2_or_gpa);
11338 !apf_put_user_ready(vcpu, work->arch.token)) {
11339 vcpu->arch.apf.pageready_pending = true;
11343 vcpu->arch.apf.halted = false;
11344 vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE;
11350 if (!vcpu->arch.apf.pageready_pending)
11364 atomic_inc(&kvm->arch.assigned_device_count);
11370 atomic_dec(&kvm->arch.assigned_device_count);
11376 return arch_atomic_read(&kvm->arch.assigned_device_count);
11382 atomic_inc(&kvm->arch.noncoherent_dma_count);
11388 atomic_dec(&kvm->arch.noncoherent_dma_count);
11394 return atomic_read(&kvm->arch.noncoherent_dma_count);
11458 return (vcpu->arch.msr_kvm_poll_control & 1) == 0;
11496 vcpu->arch.walk_mmu->gva_to_gpa(vcpu, gva, access, &fault) != UNMAPPED_GVA) {
11498 * If vcpu->arch.walk_mmu->gva_to_gpa succeeded, the page
11508 vcpu->arch.walk_mmu->inject_page_fault(vcpu, &fault);
11585 if (kvm_get_pcid(vcpu, vcpu->arch.mmu->prev_roots[i].pgd)
11589 kvm_mmu_free_roots(vcpu, vcpu->arch.mmu, roots_to_free);