162306a36Sopenharmony_ci// SPDX-License-Identifier: GPL-2.0-only
262306a36Sopenharmony_ci/*
362306a36Sopenharmony_ci * Kernel-based Virtual Machine driver for Linux
462306a36Sopenharmony_ci *
562306a36Sopenharmony_ci * AMD SVM support
662306a36Sopenharmony_ci *
762306a36Sopenharmony_ci * Copyright (C) 2006 Qumranet, Inc.
862306a36Sopenharmony_ci * Copyright 2010 Red Hat, Inc. and/or its affiliates.
962306a36Sopenharmony_ci *
1062306a36Sopenharmony_ci * Authors:
1162306a36Sopenharmony_ci *   Yaniv Kamay  <yaniv@qumranet.com>
1262306a36Sopenharmony_ci *   Avi Kivity   <avi@qumranet.com>
1362306a36Sopenharmony_ci */
1462306a36Sopenharmony_ci
1562306a36Sopenharmony_ci#ifndef __SVM_SVM_H
1662306a36Sopenharmony_ci#define __SVM_SVM_H
1762306a36Sopenharmony_ci
1862306a36Sopenharmony_ci#include <linux/kvm_types.h>
1962306a36Sopenharmony_ci#include <linux/kvm_host.h>
2062306a36Sopenharmony_ci#include <linux/bits.h>
2162306a36Sopenharmony_ci
2262306a36Sopenharmony_ci#include <asm/svm.h>
2362306a36Sopenharmony_ci#include <asm/sev-common.h>
2462306a36Sopenharmony_ci
2562306a36Sopenharmony_ci#include "cpuid.h"
2662306a36Sopenharmony_ci#include "kvm_cache_regs.h"
2762306a36Sopenharmony_ci
2862306a36Sopenharmony_ci#define __sme_page_pa(x) __sme_set(page_to_pfn(x) << PAGE_SHIFT)
2962306a36Sopenharmony_ci
3062306a36Sopenharmony_ci#define	IOPM_SIZE PAGE_SIZE * 3
3162306a36Sopenharmony_ci#define	MSRPM_SIZE PAGE_SIZE * 2
3262306a36Sopenharmony_ci
3362306a36Sopenharmony_ci#define MAX_DIRECT_ACCESS_MSRS	46
3462306a36Sopenharmony_ci#define MSRPM_OFFSETS	32
3562306a36Sopenharmony_ciextern u32 msrpm_offsets[MSRPM_OFFSETS] __read_mostly;
3662306a36Sopenharmony_ciextern bool npt_enabled;
3762306a36Sopenharmony_ciextern int nrips;
3862306a36Sopenharmony_ciextern int vgif;
3962306a36Sopenharmony_ciextern bool intercept_smi;
4062306a36Sopenharmony_ciextern bool x2avic_enabled;
4162306a36Sopenharmony_ciextern bool vnmi;
4262306a36Sopenharmony_ci
4362306a36Sopenharmony_ci/*
4462306a36Sopenharmony_ci * Clean bits in VMCB.
4562306a36Sopenharmony_ci * VMCB_ALL_CLEAN_MASK might also need to
4662306a36Sopenharmony_ci * be updated if this enum is modified.
4762306a36Sopenharmony_ci */
4862306a36Sopenharmony_cienum {
4962306a36Sopenharmony_ci	VMCB_INTERCEPTS, /* Intercept vectors, TSC offset,
5062306a36Sopenharmony_ci			    pause filter count */
5162306a36Sopenharmony_ci	VMCB_PERM_MAP,   /* IOPM Base and MSRPM Base */
5262306a36Sopenharmony_ci	VMCB_ASID,	 /* ASID */
5362306a36Sopenharmony_ci	VMCB_INTR,	 /* int_ctl, int_vector */
5462306a36Sopenharmony_ci	VMCB_NPT,        /* npt_en, nCR3, gPAT */
5562306a36Sopenharmony_ci	VMCB_CR,	 /* CR0, CR3, CR4, EFER */
5662306a36Sopenharmony_ci	VMCB_DR,         /* DR6, DR7 */
5762306a36Sopenharmony_ci	VMCB_DT,         /* GDT, IDT */
5862306a36Sopenharmony_ci	VMCB_SEG,        /* CS, DS, SS, ES, CPL */
5962306a36Sopenharmony_ci	VMCB_CR2,        /* CR2 only */
6062306a36Sopenharmony_ci	VMCB_LBR,        /* DBGCTL, BR_FROM, BR_TO, LAST_EX_FROM, LAST_EX_TO */
6162306a36Sopenharmony_ci	VMCB_AVIC,       /* AVIC APIC_BAR, AVIC APIC_BACKING_PAGE,
6262306a36Sopenharmony_ci			  * AVIC PHYSICAL_TABLE pointer,
6362306a36Sopenharmony_ci			  * AVIC LOGICAL_TABLE pointer
6462306a36Sopenharmony_ci			  */
6562306a36Sopenharmony_ci	VMCB_SW = 31,    /* Reserved for hypervisor/software use */
6662306a36Sopenharmony_ci};
6762306a36Sopenharmony_ci
6862306a36Sopenharmony_ci#define VMCB_ALL_CLEAN_MASK (					\
6962306a36Sopenharmony_ci	(1U << VMCB_INTERCEPTS) | (1U << VMCB_PERM_MAP) |	\
7062306a36Sopenharmony_ci	(1U << VMCB_ASID) | (1U << VMCB_INTR) |			\
7162306a36Sopenharmony_ci	(1U << VMCB_NPT) | (1U << VMCB_CR) | (1U << VMCB_DR) |	\
7262306a36Sopenharmony_ci	(1U << VMCB_DT) | (1U << VMCB_SEG) | (1U << VMCB_CR2) |	\
7362306a36Sopenharmony_ci	(1U << VMCB_LBR) | (1U << VMCB_AVIC) |			\
7462306a36Sopenharmony_ci	(1U << VMCB_SW))
7562306a36Sopenharmony_ci
7662306a36Sopenharmony_ci/* TPR and CR2 are always written before VMRUN */
7762306a36Sopenharmony_ci#define VMCB_ALWAYS_DIRTY_MASK	((1U << VMCB_INTR) | (1U << VMCB_CR2))
7862306a36Sopenharmony_ci
7962306a36Sopenharmony_cistruct kvm_sev_info {
8062306a36Sopenharmony_ci	bool active;		/* SEV enabled guest */
8162306a36Sopenharmony_ci	bool es_active;		/* SEV-ES enabled guest */
8262306a36Sopenharmony_ci	unsigned int asid;	/* ASID used for this guest */
8362306a36Sopenharmony_ci	unsigned int handle;	/* SEV firmware handle */
8462306a36Sopenharmony_ci	int fd;			/* SEV device fd */
8562306a36Sopenharmony_ci	unsigned long pages_locked; /* Number of pages locked */
8662306a36Sopenharmony_ci	struct list_head regions_list;  /* List of registered regions */
8762306a36Sopenharmony_ci	u64 ap_jump_table;	/* SEV-ES AP Jump Table address */
8862306a36Sopenharmony_ci	struct kvm *enc_context_owner; /* Owner of copied encryption context */
8962306a36Sopenharmony_ci	struct list_head mirror_vms; /* List of VMs mirroring */
9062306a36Sopenharmony_ci	struct list_head mirror_entry; /* Use as a list entry of mirrors */
9162306a36Sopenharmony_ci	struct misc_cg *misc_cg; /* For misc cgroup accounting */
9262306a36Sopenharmony_ci	atomic_t migration_in_progress;
9362306a36Sopenharmony_ci};
9462306a36Sopenharmony_ci
9562306a36Sopenharmony_cistruct kvm_svm {
9662306a36Sopenharmony_ci	struct kvm kvm;
9762306a36Sopenharmony_ci
9862306a36Sopenharmony_ci	/* Struct members for AVIC */
9962306a36Sopenharmony_ci	u32 avic_vm_id;
10062306a36Sopenharmony_ci	struct page *avic_logical_id_table_page;
10162306a36Sopenharmony_ci	struct page *avic_physical_id_table_page;
10262306a36Sopenharmony_ci	struct hlist_node hnode;
10362306a36Sopenharmony_ci
10462306a36Sopenharmony_ci	struct kvm_sev_info sev_info;
10562306a36Sopenharmony_ci};
10662306a36Sopenharmony_ci
10762306a36Sopenharmony_cistruct kvm_vcpu;
10862306a36Sopenharmony_ci
10962306a36Sopenharmony_cistruct kvm_vmcb_info {
11062306a36Sopenharmony_ci	struct vmcb *ptr;
11162306a36Sopenharmony_ci	unsigned long pa;
11262306a36Sopenharmony_ci	int cpu;
11362306a36Sopenharmony_ci	uint64_t asid_generation;
11462306a36Sopenharmony_ci};
11562306a36Sopenharmony_ci
11662306a36Sopenharmony_cistruct vmcb_save_area_cached {
11762306a36Sopenharmony_ci	u64 efer;
11862306a36Sopenharmony_ci	u64 cr4;
11962306a36Sopenharmony_ci	u64 cr3;
12062306a36Sopenharmony_ci	u64 cr0;
12162306a36Sopenharmony_ci	u64 dr7;
12262306a36Sopenharmony_ci	u64 dr6;
12362306a36Sopenharmony_ci};
12462306a36Sopenharmony_ci
12562306a36Sopenharmony_cistruct vmcb_ctrl_area_cached {
12662306a36Sopenharmony_ci	u32 intercepts[MAX_INTERCEPT];
12762306a36Sopenharmony_ci	u16 pause_filter_thresh;
12862306a36Sopenharmony_ci	u16 pause_filter_count;
12962306a36Sopenharmony_ci	u64 iopm_base_pa;
13062306a36Sopenharmony_ci	u64 msrpm_base_pa;
13162306a36Sopenharmony_ci	u64 tsc_offset;
13262306a36Sopenharmony_ci	u32 asid;
13362306a36Sopenharmony_ci	u8 tlb_ctl;
13462306a36Sopenharmony_ci	u32 int_ctl;
13562306a36Sopenharmony_ci	u32 int_vector;
13662306a36Sopenharmony_ci	u32 int_state;
13762306a36Sopenharmony_ci	u32 exit_code;
13862306a36Sopenharmony_ci	u32 exit_code_hi;
13962306a36Sopenharmony_ci	u64 exit_info_1;
14062306a36Sopenharmony_ci	u64 exit_info_2;
14162306a36Sopenharmony_ci	u32 exit_int_info;
14262306a36Sopenharmony_ci	u32 exit_int_info_err;
14362306a36Sopenharmony_ci	u64 nested_ctl;
14462306a36Sopenharmony_ci	u32 event_inj;
14562306a36Sopenharmony_ci	u32 event_inj_err;
14662306a36Sopenharmony_ci	u64 next_rip;
14762306a36Sopenharmony_ci	u64 nested_cr3;
14862306a36Sopenharmony_ci	u64 virt_ext;
14962306a36Sopenharmony_ci	u32 clean;
15062306a36Sopenharmony_ci	union {
15162306a36Sopenharmony_ci		struct hv_vmcb_enlightenments hv_enlightenments;
15262306a36Sopenharmony_ci		u8 reserved_sw[32];
15362306a36Sopenharmony_ci	};
15462306a36Sopenharmony_ci};
15562306a36Sopenharmony_ci
15662306a36Sopenharmony_cistruct svm_nested_state {
15762306a36Sopenharmony_ci	struct kvm_vmcb_info vmcb02;
15862306a36Sopenharmony_ci	u64 hsave_msr;
15962306a36Sopenharmony_ci	u64 vm_cr_msr;
16062306a36Sopenharmony_ci	u64 vmcb12_gpa;
16162306a36Sopenharmony_ci	u64 last_vmcb12_gpa;
16262306a36Sopenharmony_ci
16362306a36Sopenharmony_ci	/* These are the merged vectors */
16462306a36Sopenharmony_ci	u32 *msrpm;
16562306a36Sopenharmony_ci
16662306a36Sopenharmony_ci	/* A VMRUN has started but has not yet been performed, so
16762306a36Sopenharmony_ci	 * we cannot inject a nested vmexit yet.  */
16862306a36Sopenharmony_ci	bool nested_run_pending;
16962306a36Sopenharmony_ci
17062306a36Sopenharmony_ci	/* cache for control fields of the guest */
17162306a36Sopenharmony_ci	struct vmcb_ctrl_area_cached ctl;
17262306a36Sopenharmony_ci
17362306a36Sopenharmony_ci	/*
17462306a36Sopenharmony_ci	 * Note: this struct is not kept up-to-date while L2 runs; it is only
17562306a36Sopenharmony_ci	 * valid within nested_svm_vmrun.
17662306a36Sopenharmony_ci	 */
17762306a36Sopenharmony_ci	struct vmcb_save_area_cached save;
17862306a36Sopenharmony_ci
17962306a36Sopenharmony_ci	bool initialized;
18062306a36Sopenharmony_ci
18162306a36Sopenharmony_ci	/*
18262306a36Sopenharmony_ci	 * Indicates whether MSR bitmap for L2 needs to be rebuilt due to
18362306a36Sopenharmony_ci	 * changes in MSR bitmap for L1 or switching to a different L2. Note,
18462306a36Sopenharmony_ci	 * this flag can only be used reliably in conjunction with a paravirt L1
18562306a36Sopenharmony_ci	 * which informs L0 whether any changes to MSR bitmap for L2 were done
18662306a36Sopenharmony_ci	 * on its side.
18762306a36Sopenharmony_ci	 */
18862306a36Sopenharmony_ci	bool force_msr_bitmap_recalc;
18962306a36Sopenharmony_ci};
19062306a36Sopenharmony_ci
19162306a36Sopenharmony_cistruct vcpu_sev_es_state {
19262306a36Sopenharmony_ci	/* SEV-ES support */
19362306a36Sopenharmony_ci	struct sev_es_save_area *vmsa;
19462306a36Sopenharmony_ci	struct ghcb *ghcb;
19562306a36Sopenharmony_ci	u8 valid_bitmap[16];
19662306a36Sopenharmony_ci	struct kvm_host_map ghcb_map;
19762306a36Sopenharmony_ci	bool received_first_sipi;
19862306a36Sopenharmony_ci
19962306a36Sopenharmony_ci	/* SEV-ES scratch area support */
20062306a36Sopenharmony_ci	u64 sw_scratch;
20162306a36Sopenharmony_ci	void *ghcb_sa;
20262306a36Sopenharmony_ci	u32 ghcb_sa_len;
20362306a36Sopenharmony_ci	bool ghcb_sa_sync;
20462306a36Sopenharmony_ci	bool ghcb_sa_free;
20562306a36Sopenharmony_ci};
20662306a36Sopenharmony_ci
20762306a36Sopenharmony_cistruct vcpu_svm {
20862306a36Sopenharmony_ci	struct kvm_vcpu vcpu;
20962306a36Sopenharmony_ci	/* vmcb always points at current_vmcb->ptr, it's purely a shorthand. */
21062306a36Sopenharmony_ci	struct vmcb *vmcb;
21162306a36Sopenharmony_ci	struct kvm_vmcb_info vmcb01;
21262306a36Sopenharmony_ci	struct kvm_vmcb_info *current_vmcb;
21362306a36Sopenharmony_ci	u32 asid;
21462306a36Sopenharmony_ci	u32 sysenter_esp_hi;
21562306a36Sopenharmony_ci	u32 sysenter_eip_hi;
21662306a36Sopenharmony_ci	uint64_t tsc_aux;
21762306a36Sopenharmony_ci
21862306a36Sopenharmony_ci	u64 msr_decfg;
21962306a36Sopenharmony_ci
22062306a36Sopenharmony_ci	u64 next_rip;
22162306a36Sopenharmony_ci
22262306a36Sopenharmony_ci	u64 spec_ctrl;
22362306a36Sopenharmony_ci
22462306a36Sopenharmony_ci	u64 tsc_ratio_msr;
22562306a36Sopenharmony_ci	/*
22662306a36Sopenharmony_ci	 * Contains guest-controlled bits of VIRT_SPEC_CTRL, which will be
22762306a36Sopenharmony_ci	 * translated into the appropriate L2_CFG bits on the host to
22862306a36Sopenharmony_ci	 * perform speculative control.
22962306a36Sopenharmony_ci	 */
23062306a36Sopenharmony_ci	u64 virt_spec_ctrl;
23162306a36Sopenharmony_ci
23262306a36Sopenharmony_ci	u32 *msrpm;
23362306a36Sopenharmony_ci
23462306a36Sopenharmony_ci	ulong nmi_iret_rip;
23562306a36Sopenharmony_ci
23662306a36Sopenharmony_ci	struct svm_nested_state nested;
23762306a36Sopenharmony_ci
23862306a36Sopenharmony_ci	/* NMI mask value, used when vNMI is not enabled */
23962306a36Sopenharmony_ci	bool nmi_masked;
24062306a36Sopenharmony_ci
24162306a36Sopenharmony_ci	/*
24262306a36Sopenharmony_ci	 * True when NMIs are still masked but guest IRET was just intercepted
24362306a36Sopenharmony_ci	 * and KVM is waiting for RIP to change, which will signal that the
24462306a36Sopenharmony_ci	 * intercepted IRET was retired and thus NMI can be unmasked.
24562306a36Sopenharmony_ci	 */
24662306a36Sopenharmony_ci	bool awaiting_iret_completion;
24762306a36Sopenharmony_ci
24862306a36Sopenharmony_ci	/*
24962306a36Sopenharmony_ci	 * Set when KVM is awaiting IRET completion and needs to inject NMIs as
25062306a36Sopenharmony_ci	 * soon as the IRET completes (e.g. NMI is pending injection).  KVM
25162306a36Sopenharmony_ci	 * temporarily steals RFLAGS.TF to single-step the guest in this case
25262306a36Sopenharmony_ci	 * in order to regain control as soon as the NMI-blocking condition
25362306a36Sopenharmony_ci	 * goes away.
25462306a36Sopenharmony_ci	 */
25562306a36Sopenharmony_ci	bool nmi_singlestep;
25662306a36Sopenharmony_ci	u64 nmi_singlestep_guest_rflags;
25762306a36Sopenharmony_ci
25862306a36Sopenharmony_ci	bool nmi_l1_to_l2;
25962306a36Sopenharmony_ci
26062306a36Sopenharmony_ci	unsigned long soft_int_csbase;
26162306a36Sopenharmony_ci	unsigned long soft_int_old_rip;
26262306a36Sopenharmony_ci	unsigned long soft_int_next_rip;
26362306a36Sopenharmony_ci	bool soft_int_injected;
26462306a36Sopenharmony_ci
26562306a36Sopenharmony_ci	u32 ldr_reg;
26662306a36Sopenharmony_ci	u32 dfr_reg;
26762306a36Sopenharmony_ci	struct page *avic_backing_page;
26862306a36Sopenharmony_ci	u64 *avic_physical_id_cache;
26962306a36Sopenharmony_ci
27062306a36Sopenharmony_ci	/*
27162306a36Sopenharmony_ci	 * Per-vcpu list of struct amd_svm_iommu_ir:
27262306a36Sopenharmony_ci	 * This is used mainly to store interrupt remapping information used
27362306a36Sopenharmony_ci	 * when update the vcpu affinity. This avoids the need to scan for
27462306a36Sopenharmony_ci	 * IRTE and try to match ga_tag in the IOMMU driver.
27562306a36Sopenharmony_ci	 */
27662306a36Sopenharmony_ci	struct list_head ir_list;
27762306a36Sopenharmony_ci	spinlock_t ir_list_lock;
27862306a36Sopenharmony_ci
27962306a36Sopenharmony_ci	/* Save desired MSR intercept (read: pass-through) state */
28062306a36Sopenharmony_ci	struct {
28162306a36Sopenharmony_ci		DECLARE_BITMAP(read, MAX_DIRECT_ACCESS_MSRS);
28262306a36Sopenharmony_ci		DECLARE_BITMAP(write, MAX_DIRECT_ACCESS_MSRS);
28362306a36Sopenharmony_ci	} shadow_msr_intercept;
28462306a36Sopenharmony_ci
28562306a36Sopenharmony_ci	struct vcpu_sev_es_state sev_es;
28662306a36Sopenharmony_ci
28762306a36Sopenharmony_ci	bool guest_state_loaded;
28862306a36Sopenharmony_ci
28962306a36Sopenharmony_ci	bool x2avic_msrs_intercepted;
29062306a36Sopenharmony_ci
29162306a36Sopenharmony_ci	/* Guest GIF value, used when vGIF is not enabled */
29262306a36Sopenharmony_ci	bool guest_gif;
29362306a36Sopenharmony_ci};
29462306a36Sopenharmony_ci
29562306a36Sopenharmony_cistruct svm_cpu_data {
29662306a36Sopenharmony_ci	u64 asid_generation;
29762306a36Sopenharmony_ci	u32 max_asid;
29862306a36Sopenharmony_ci	u32 next_asid;
29962306a36Sopenharmony_ci	u32 min_asid;
30062306a36Sopenharmony_ci
30162306a36Sopenharmony_ci	struct page *save_area;
30262306a36Sopenharmony_ci	unsigned long save_area_pa;
30362306a36Sopenharmony_ci
30462306a36Sopenharmony_ci	struct vmcb *current_vmcb;
30562306a36Sopenharmony_ci
30662306a36Sopenharmony_ci	/* index = sev_asid, value = vmcb pointer */
30762306a36Sopenharmony_ci	struct vmcb **sev_vmcbs;
30862306a36Sopenharmony_ci};
30962306a36Sopenharmony_ci
31062306a36Sopenharmony_ciDECLARE_PER_CPU(struct svm_cpu_data, svm_data);
31162306a36Sopenharmony_ci
31262306a36Sopenharmony_civoid recalc_intercepts(struct vcpu_svm *svm);
31362306a36Sopenharmony_ci
31462306a36Sopenharmony_cistatic __always_inline struct kvm_svm *to_kvm_svm(struct kvm *kvm)
31562306a36Sopenharmony_ci{
31662306a36Sopenharmony_ci	return container_of(kvm, struct kvm_svm, kvm);
31762306a36Sopenharmony_ci}
31862306a36Sopenharmony_ci
31962306a36Sopenharmony_cistatic __always_inline bool sev_guest(struct kvm *kvm)
32062306a36Sopenharmony_ci{
32162306a36Sopenharmony_ci#ifdef CONFIG_KVM_AMD_SEV
32262306a36Sopenharmony_ci	struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
32362306a36Sopenharmony_ci
32462306a36Sopenharmony_ci	return sev->active;
32562306a36Sopenharmony_ci#else
32662306a36Sopenharmony_ci	return false;
32762306a36Sopenharmony_ci#endif
32862306a36Sopenharmony_ci}
32962306a36Sopenharmony_ci
33062306a36Sopenharmony_cistatic __always_inline bool sev_es_guest(struct kvm *kvm)
33162306a36Sopenharmony_ci{
33262306a36Sopenharmony_ci#ifdef CONFIG_KVM_AMD_SEV
33362306a36Sopenharmony_ci	struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
33462306a36Sopenharmony_ci
33562306a36Sopenharmony_ci	return sev->es_active && !WARN_ON_ONCE(!sev->active);
33662306a36Sopenharmony_ci#else
33762306a36Sopenharmony_ci	return false;
33862306a36Sopenharmony_ci#endif
33962306a36Sopenharmony_ci}
34062306a36Sopenharmony_ci
34162306a36Sopenharmony_cistatic inline void vmcb_mark_all_dirty(struct vmcb *vmcb)
34262306a36Sopenharmony_ci{
34362306a36Sopenharmony_ci	vmcb->control.clean = 0;
34462306a36Sopenharmony_ci}
34562306a36Sopenharmony_ci
34662306a36Sopenharmony_cistatic inline void vmcb_mark_all_clean(struct vmcb *vmcb)
34762306a36Sopenharmony_ci{
34862306a36Sopenharmony_ci	vmcb->control.clean = VMCB_ALL_CLEAN_MASK
34962306a36Sopenharmony_ci			       & ~VMCB_ALWAYS_DIRTY_MASK;
35062306a36Sopenharmony_ci}
35162306a36Sopenharmony_ci
35262306a36Sopenharmony_cistatic inline void vmcb_mark_dirty(struct vmcb *vmcb, int bit)
35362306a36Sopenharmony_ci{
35462306a36Sopenharmony_ci	vmcb->control.clean &= ~(1 << bit);
35562306a36Sopenharmony_ci}
35662306a36Sopenharmony_ci
35762306a36Sopenharmony_cistatic inline bool vmcb_is_dirty(struct vmcb *vmcb, int bit)
35862306a36Sopenharmony_ci{
35962306a36Sopenharmony_ci        return !test_bit(bit, (unsigned long *)&vmcb->control.clean);
36062306a36Sopenharmony_ci}
36162306a36Sopenharmony_ci
36262306a36Sopenharmony_cistatic __always_inline struct vcpu_svm *to_svm(struct kvm_vcpu *vcpu)
36362306a36Sopenharmony_ci{
36462306a36Sopenharmony_ci	return container_of(vcpu, struct vcpu_svm, vcpu);
36562306a36Sopenharmony_ci}
36662306a36Sopenharmony_ci
36762306a36Sopenharmony_ci/*
36862306a36Sopenharmony_ci * Only the PDPTRs are loaded on demand into the shadow MMU.  All other
36962306a36Sopenharmony_ci * fields are synchronized on VM-Exit, because accessing the VMCB is cheap.
37062306a36Sopenharmony_ci *
37162306a36Sopenharmony_ci * CR3 might be out of date in the VMCB but it is not marked dirty; instead,
37262306a36Sopenharmony_ci * KVM_REQ_LOAD_MMU_PGD is always requested when the cached vcpu->arch.cr3
37362306a36Sopenharmony_ci * is changed.  svm_load_mmu_pgd() then syncs the new CR3 value into the VMCB.
37462306a36Sopenharmony_ci */
37562306a36Sopenharmony_ci#define SVM_REGS_LAZY_LOAD_SET	(1 << VCPU_EXREG_PDPTR)
37662306a36Sopenharmony_ci
37762306a36Sopenharmony_cistatic inline void vmcb_set_intercept(struct vmcb_control_area *control, u32 bit)
37862306a36Sopenharmony_ci{
37962306a36Sopenharmony_ci	WARN_ON_ONCE(bit >= 32 * MAX_INTERCEPT);
38062306a36Sopenharmony_ci	__set_bit(bit, (unsigned long *)&control->intercepts);
38162306a36Sopenharmony_ci}
38262306a36Sopenharmony_ci
38362306a36Sopenharmony_cistatic inline void vmcb_clr_intercept(struct vmcb_control_area *control, u32 bit)
38462306a36Sopenharmony_ci{
38562306a36Sopenharmony_ci	WARN_ON_ONCE(bit >= 32 * MAX_INTERCEPT);
38662306a36Sopenharmony_ci	__clear_bit(bit, (unsigned long *)&control->intercepts);
38762306a36Sopenharmony_ci}
38862306a36Sopenharmony_ci
38962306a36Sopenharmony_cistatic inline bool vmcb_is_intercept(struct vmcb_control_area *control, u32 bit)
39062306a36Sopenharmony_ci{
39162306a36Sopenharmony_ci	WARN_ON_ONCE(bit >= 32 * MAX_INTERCEPT);
39262306a36Sopenharmony_ci	return test_bit(bit, (unsigned long *)&control->intercepts);
39362306a36Sopenharmony_ci}
39462306a36Sopenharmony_ci
39562306a36Sopenharmony_cistatic inline bool vmcb12_is_intercept(struct vmcb_ctrl_area_cached *control, u32 bit)
39662306a36Sopenharmony_ci{
39762306a36Sopenharmony_ci	WARN_ON_ONCE(bit >= 32 * MAX_INTERCEPT);
39862306a36Sopenharmony_ci	return test_bit(bit, (unsigned long *)&control->intercepts);
39962306a36Sopenharmony_ci}
40062306a36Sopenharmony_ci
40162306a36Sopenharmony_cistatic inline void set_exception_intercept(struct vcpu_svm *svm, u32 bit)
40262306a36Sopenharmony_ci{
40362306a36Sopenharmony_ci	struct vmcb *vmcb = svm->vmcb01.ptr;
40462306a36Sopenharmony_ci
40562306a36Sopenharmony_ci	WARN_ON_ONCE(bit >= 32);
40662306a36Sopenharmony_ci	vmcb_set_intercept(&vmcb->control, INTERCEPT_EXCEPTION_OFFSET + bit);
40762306a36Sopenharmony_ci
40862306a36Sopenharmony_ci	recalc_intercepts(svm);
40962306a36Sopenharmony_ci}
41062306a36Sopenharmony_ci
41162306a36Sopenharmony_cistatic inline void clr_exception_intercept(struct vcpu_svm *svm, u32 bit)
41262306a36Sopenharmony_ci{
41362306a36Sopenharmony_ci	struct vmcb *vmcb = svm->vmcb01.ptr;
41462306a36Sopenharmony_ci
41562306a36Sopenharmony_ci	WARN_ON_ONCE(bit >= 32);
41662306a36Sopenharmony_ci	vmcb_clr_intercept(&vmcb->control, INTERCEPT_EXCEPTION_OFFSET + bit);
41762306a36Sopenharmony_ci
41862306a36Sopenharmony_ci	recalc_intercepts(svm);
41962306a36Sopenharmony_ci}
42062306a36Sopenharmony_ci
42162306a36Sopenharmony_cistatic inline void svm_set_intercept(struct vcpu_svm *svm, int bit)
42262306a36Sopenharmony_ci{
42362306a36Sopenharmony_ci	struct vmcb *vmcb = svm->vmcb01.ptr;
42462306a36Sopenharmony_ci
42562306a36Sopenharmony_ci	vmcb_set_intercept(&vmcb->control, bit);
42662306a36Sopenharmony_ci
42762306a36Sopenharmony_ci	recalc_intercepts(svm);
42862306a36Sopenharmony_ci}
42962306a36Sopenharmony_ci
43062306a36Sopenharmony_cistatic inline void svm_clr_intercept(struct vcpu_svm *svm, int bit)
43162306a36Sopenharmony_ci{
43262306a36Sopenharmony_ci	struct vmcb *vmcb = svm->vmcb01.ptr;
43362306a36Sopenharmony_ci
43462306a36Sopenharmony_ci	vmcb_clr_intercept(&vmcb->control, bit);
43562306a36Sopenharmony_ci
43662306a36Sopenharmony_ci	recalc_intercepts(svm);
43762306a36Sopenharmony_ci}
43862306a36Sopenharmony_ci
43962306a36Sopenharmony_cistatic inline bool svm_is_intercept(struct vcpu_svm *svm, int bit)
44062306a36Sopenharmony_ci{
44162306a36Sopenharmony_ci	return vmcb_is_intercept(&svm->vmcb->control, bit);
44262306a36Sopenharmony_ci}
44362306a36Sopenharmony_ci
44462306a36Sopenharmony_cistatic inline bool nested_vgif_enabled(struct vcpu_svm *svm)
44562306a36Sopenharmony_ci{
44662306a36Sopenharmony_ci	return guest_can_use(&svm->vcpu, X86_FEATURE_VGIF) &&
44762306a36Sopenharmony_ci	       (svm->nested.ctl.int_ctl & V_GIF_ENABLE_MASK);
44862306a36Sopenharmony_ci}
44962306a36Sopenharmony_ci
45062306a36Sopenharmony_cistatic inline struct vmcb *get_vgif_vmcb(struct vcpu_svm *svm)
45162306a36Sopenharmony_ci{
45262306a36Sopenharmony_ci	if (!vgif)
45362306a36Sopenharmony_ci		return NULL;
45462306a36Sopenharmony_ci
45562306a36Sopenharmony_ci	if (is_guest_mode(&svm->vcpu) && !nested_vgif_enabled(svm))
45662306a36Sopenharmony_ci		return svm->nested.vmcb02.ptr;
45762306a36Sopenharmony_ci	else
45862306a36Sopenharmony_ci		return svm->vmcb01.ptr;
45962306a36Sopenharmony_ci}
46062306a36Sopenharmony_ci
46162306a36Sopenharmony_cistatic inline void enable_gif(struct vcpu_svm *svm)
46262306a36Sopenharmony_ci{
46362306a36Sopenharmony_ci	struct vmcb *vmcb = get_vgif_vmcb(svm);
46462306a36Sopenharmony_ci
46562306a36Sopenharmony_ci	if (vmcb)
46662306a36Sopenharmony_ci		vmcb->control.int_ctl |= V_GIF_MASK;
46762306a36Sopenharmony_ci	else
46862306a36Sopenharmony_ci		svm->guest_gif = true;
46962306a36Sopenharmony_ci}
47062306a36Sopenharmony_ci
47162306a36Sopenharmony_cistatic inline void disable_gif(struct vcpu_svm *svm)
47262306a36Sopenharmony_ci{
47362306a36Sopenharmony_ci	struct vmcb *vmcb = get_vgif_vmcb(svm);
47462306a36Sopenharmony_ci
47562306a36Sopenharmony_ci	if (vmcb)
47662306a36Sopenharmony_ci		vmcb->control.int_ctl &= ~V_GIF_MASK;
47762306a36Sopenharmony_ci	else
47862306a36Sopenharmony_ci		svm->guest_gif = false;
47962306a36Sopenharmony_ci}
48062306a36Sopenharmony_ci
48162306a36Sopenharmony_cistatic inline bool gif_set(struct vcpu_svm *svm)
48262306a36Sopenharmony_ci{
48362306a36Sopenharmony_ci	struct vmcb *vmcb = get_vgif_vmcb(svm);
48462306a36Sopenharmony_ci
48562306a36Sopenharmony_ci	if (vmcb)
48662306a36Sopenharmony_ci		return !!(vmcb->control.int_ctl & V_GIF_MASK);
48762306a36Sopenharmony_ci	else
48862306a36Sopenharmony_ci		return svm->guest_gif;
48962306a36Sopenharmony_ci}
49062306a36Sopenharmony_ci
49162306a36Sopenharmony_cistatic inline bool nested_npt_enabled(struct vcpu_svm *svm)
49262306a36Sopenharmony_ci{
49362306a36Sopenharmony_ci	return svm->nested.ctl.nested_ctl & SVM_NESTED_CTL_NP_ENABLE;
49462306a36Sopenharmony_ci}
49562306a36Sopenharmony_ci
49662306a36Sopenharmony_cistatic inline bool nested_vnmi_enabled(struct vcpu_svm *svm)
49762306a36Sopenharmony_ci{
49862306a36Sopenharmony_ci	return guest_can_use(&svm->vcpu, X86_FEATURE_VNMI) &&
49962306a36Sopenharmony_ci	       (svm->nested.ctl.int_ctl & V_NMI_ENABLE_MASK);
50062306a36Sopenharmony_ci}
50162306a36Sopenharmony_ci
50262306a36Sopenharmony_cistatic inline bool is_x2apic_msrpm_offset(u32 offset)
50362306a36Sopenharmony_ci{
50462306a36Sopenharmony_ci	/* 4 msrs per u8, and 4 u8 in u32 */
50562306a36Sopenharmony_ci	u32 msr = offset * 16;
50662306a36Sopenharmony_ci
50762306a36Sopenharmony_ci	return (msr >= APIC_BASE_MSR) &&
50862306a36Sopenharmony_ci	       (msr < (APIC_BASE_MSR + 0x100));
50962306a36Sopenharmony_ci}
51062306a36Sopenharmony_ci
51162306a36Sopenharmony_cistatic inline struct vmcb *get_vnmi_vmcb_l1(struct vcpu_svm *svm)
51262306a36Sopenharmony_ci{
51362306a36Sopenharmony_ci	if (!vnmi)
51462306a36Sopenharmony_ci		return NULL;
51562306a36Sopenharmony_ci
51662306a36Sopenharmony_ci	if (is_guest_mode(&svm->vcpu))
51762306a36Sopenharmony_ci		return NULL;
51862306a36Sopenharmony_ci	else
51962306a36Sopenharmony_ci		return svm->vmcb01.ptr;
52062306a36Sopenharmony_ci}
52162306a36Sopenharmony_ci
52262306a36Sopenharmony_cistatic inline bool is_vnmi_enabled(struct vcpu_svm *svm)
52362306a36Sopenharmony_ci{
52462306a36Sopenharmony_ci	struct vmcb *vmcb = get_vnmi_vmcb_l1(svm);
52562306a36Sopenharmony_ci
52662306a36Sopenharmony_ci	if (vmcb)
52762306a36Sopenharmony_ci		return !!(vmcb->control.int_ctl & V_NMI_ENABLE_MASK);
52862306a36Sopenharmony_ci	else
52962306a36Sopenharmony_ci		return false;
53062306a36Sopenharmony_ci}
53162306a36Sopenharmony_ci
53262306a36Sopenharmony_ci/* svm.c */
53362306a36Sopenharmony_ci#define MSR_INVALID				0xffffffffU
53462306a36Sopenharmony_ci
53562306a36Sopenharmony_ci#define DEBUGCTL_RESERVED_BITS (~(0x3fULL))
53662306a36Sopenharmony_ci
53762306a36Sopenharmony_ciextern bool dump_invalid_vmcb;
53862306a36Sopenharmony_ci
53962306a36Sopenharmony_ciu32 svm_msrpm_offset(u32 msr);
54062306a36Sopenharmony_ciu32 *svm_vcpu_alloc_msrpm(void);
54162306a36Sopenharmony_civoid svm_vcpu_init_msrpm(struct kvm_vcpu *vcpu, u32 *msrpm);
54262306a36Sopenharmony_civoid svm_vcpu_free_msrpm(u32 *msrpm);
54362306a36Sopenharmony_civoid svm_copy_lbrs(struct vmcb *to_vmcb, struct vmcb *from_vmcb);
54462306a36Sopenharmony_civoid svm_update_lbrv(struct kvm_vcpu *vcpu);
54562306a36Sopenharmony_ci
54662306a36Sopenharmony_ciint svm_set_efer(struct kvm_vcpu *vcpu, u64 efer);
54762306a36Sopenharmony_civoid svm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0);
54862306a36Sopenharmony_civoid svm_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4);
54962306a36Sopenharmony_civoid disable_nmi_singlestep(struct vcpu_svm *svm);
55062306a36Sopenharmony_cibool svm_smi_blocked(struct kvm_vcpu *vcpu);
55162306a36Sopenharmony_cibool svm_nmi_blocked(struct kvm_vcpu *vcpu);
55262306a36Sopenharmony_cibool svm_interrupt_blocked(struct kvm_vcpu *vcpu);
55362306a36Sopenharmony_civoid svm_set_gif(struct vcpu_svm *svm, bool value);
55462306a36Sopenharmony_ciint svm_invoke_exit_handler(struct kvm_vcpu *vcpu, u64 exit_code);
55562306a36Sopenharmony_civoid set_msr_interception(struct kvm_vcpu *vcpu, u32 *msrpm, u32 msr,
55662306a36Sopenharmony_ci			  int read, int write);
55762306a36Sopenharmony_civoid svm_set_x2apic_msr_interception(struct vcpu_svm *svm, bool disable);
55862306a36Sopenharmony_civoid svm_complete_interrupt_delivery(struct kvm_vcpu *vcpu, int delivery_mode,
55962306a36Sopenharmony_ci				     int trig_mode, int vec);
56062306a36Sopenharmony_ci
56162306a36Sopenharmony_ci/* nested.c */
56262306a36Sopenharmony_ci
56362306a36Sopenharmony_ci#define NESTED_EXIT_HOST	0	/* Exit handled on host level */
56462306a36Sopenharmony_ci#define NESTED_EXIT_DONE	1	/* Exit caused nested vmexit  */
56562306a36Sopenharmony_ci#define NESTED_EXIT_CONTINUE	2	/* Further checks needed      */
56662306a36Sopenharmony_ci
56762306a36Sopenharmony_cistatic inline bool nested_svm_virtualize_tpr(struct kvm_vcpu *vcpu)
56862306a36Sopenharmony_ci{
56962306a36Sopenharmony_ci	struct vcpu_svm *svm = to_svm(vcpu);
57062306a36Sopenharmony_ci
57162306a36Sopenharmony_ci	return is_guest_mode(vcpu) && (svm->nested.ctl.int_ctl & V_INTR_MASKING_MASK);
57262306a36Sopenharmony_ci}
57362306a36Sopenharmony_ci
57462306a36Sopenharmony_cistatic inline bool nested_exit_on_smi(struct vcpu_svm *svm)
57562306a36Sopenharmony_ci{
57662306a36Sopenharmony_ci	return vmcb12_is_intercept(&svm->nested.ctl, INTERCEPT_SMI);
57762306a36Sopenharmony_ci}
57862306a36Sopenharmony_ci
57962306a36Sopenharmony_cistatic inline bool nested_exit_on_intr(struct vcpu_svm *svm)
58062306a36Sopenharmony_ci{
58162306a36Sopenharmony_ci	return vmcb12_is_intercept(&svm->nested.ctl, INTERCEPT_INTR);
58262306a36Sopenharmony_ci}
58362306a36Sopenharmony_ci
58462306a36Sopenharmony_cistatic inline bool nested_exit_on_nmi(struct vcpu_svm *svm)
58562306a36Sopenharmony_ci{
58662306a36Sopenharmony_ci	return vmcb12_is_intercept(&svm->nested.ctl, INTERCEPT_NMI);
58762306a36Sopenharmony_ci}
58862306a36Sopenharmony_ci
58962306a36Sopenharmony_ciint enter_svm_guest_mode(struct kvm_vcpu *vcpu,
59062306a36Sopenharmony_ci			 u64 vmcb_gpa, struct vmcb *vmcb12, bool from_vmrun);
59162306a36Sopenharmony_civoid svm_leave_nested(struct kvm_vcpu *vcpu);
59262306a36Sopenharmony_civoid svm_free_nested(struct vcpu_svm *svm);
59362306a36Sopenharmony_ciint svm_allocate_nested(struct vcpu_svm *svm);
59462306a36Sopenharmony_ciint nested_svm_vmrun(struct kvm_vcpu *vcpu);
59562306a36Sopenharmony_civoid svm_copy_vmrun_state(struct vmcb_save_area *to_save,
59662306a36Sopenharmony_ci			  struct vmcb_save_area *from_save);
59762306a36Sopenharmony_civoid svm_copy_vmloadsave_state(struct vmcb *to_vmcb, struct vmcb *from_vmcb);
59862306a36Sopenharmony_ciint nested_svm_vmexit(struct vcpu_svm *svm);
59962306a36Sopenharmony_ci
60062306a36Sopenharmony_cistatic inline int nested_svm_simple_vmexit(struct vcpu_svm *svm, u32 exit_code)
60162306a36Sopenharmony_ci{
60262306a36Sopenharmony_ci	svm->vmcb->control.exit_code   = exit_code;
60362306a36Sopenharmony_ci	svm->vmcb->control.exit_info_1 = 0;
60462306a36Sopenharmony_ci	svm->vmcb->control.exit_info_2 = 0;
60562306a36Sopenharmony_ci	return nested_svm_vmexit(svm);
60662306a36Sopenharmony_ci}
60762306a36Sopenharmony_ci
60862306a36Sopenharmony_ciint nested_svm_exit_handled(struct vcpu_svm *svm);
60962306a36Sopenharmony_ciint nested_svm_check_permissions(struct kvm_vcpu *vcpu);
61062306a36Sopenharmony_ciint nested_svm_check_exception(struct vcpu_svm *svm, unsigned nr,
61162306a36Sopenharmony_ci			       bool has_error_code, u32 error_code);
61262306a36Sopenharmony_ciint nested_svm_exit_special(struct vcpu_svm *svm);
61362306a36Sopenharmony_civoid nested_svm_update_tsc_ratio_msr(struct kvm_vcpu *vcpu);
61462306a36Sopenharmony_civoid svm_write_tsc_multiplier(struct kvm_vcpu *vcpu);
61562306a36Sopenharmony_civoid nested_copy_vmcb_control_to_cache(struct vcpu_svm *svm,
61662306a36Sopenharmony_ci				       struct vmcb_control_area *control);
61762306a36Sopenharmony_civoid nested_copy_vmcb_save_to_cache(struct vcpu_svm *svm,
61862306a36Sopenharmony_ci				    struct vmcb_save_area *save);
61962306a36Sopenharmony_civoid nested_sync_control_from_vmcb02(struct vcpu_svm *svm);
62062306a36Sopenharmony_civoid nested_vmcb02_compute_g_pat(struct vcpu_svm *svm);
62162306a36Sopenharmony_civoid svm_switch_vmcb(struct vcpu_svm *svm, struct kvm_vmcb_info *target_vmcb);
62262306a36Sopenharmony_ci
62362306a36Sopenharmony_ciextern struct kvm_x86_nested_ops svm_nested_ops;
62462306a36Sopenharmony_ci
62562306a36Sopenharmony_ci/* avic.c */
62662306a36Sopenharmony_ci#define AVIC_REQUIRED_APICV_INHIBITS			\
62762306a36Sopenharmony_ci(							\
62862306a36Sopenharmony_ci	BIT(APICV_INHIBIT_REASON_DISABLE) |		\
62962306a36Sopenharmony_ci	BIT(APICV_INHIBIT_REASON_ABSENT) |		\
63062306a36Sopenharmony_ci	BIT(APICV_INHIBIT_REASON_HYPERV) |		\
63162306a36Sopenharmony_ci	BIT(APICV_INHIBIT_REASON_NESTED) |		\
63262306a36Sopenharmony_ci	BIT(APICV_INHIBIT_REASON_IRQWIN) |		\
63362306a36Sopenharmony_ci	BIT(APICV_INHIBIT_REASON_PIT_REINJ) |		\
63462306a36Sopenharmony_ci	BIT(APICV_INHIBIT_REASON_BLOCKIRQ) |		\
63562306a36Sopenharmony_ci	BIT(APICV_INHIBIT_REASON_SEV)      |		\
63662306a36Sopenharmony_ci	BIT(APICV_INHIBIT_REASON_PHYSICAL_ID_ALIASED) |	\
63762306a36Sopenharmony_ci	BIT(APICV_INHIBIT_REASON_APIC_ID_MODIFIED) |	\
63862306a36Sopenharmony_ci	BIT(APICV_INHIBIT_REASON_APIC_BASE_MODIFIED) |	\
63962306a36Sopenharmony_ci	BIT(APICV_INHIBIT_REASON_LOGICAL_ID_ALIASED)	\
64062306a36Sopenharmony_ci)
64162306a36Sopenharmony_ci
64262306a36Sopenharmony_cibool avic_hardware_setup(void);
64362306a36Sopenharmony_ciint avic_ga_log_notifier(u32 ga_tag);
64462306a36Sopenharmony_civoid avic_vm_destroy(struct kvm *kvm);
64562306a36Sopenharmony_ciint avic_vm_init(struct kvm *kvm);
64662306a36Sopenharmony_civoid avic_init_vmcb(struct vcpu_svm *svm, struct vmcb *vmcb);
64762306a36Sopenharmony_ciint avic_incomplete_ipi_interception(struct kvm_vcpu *vcpu);
64862306a36Sopenharmony_ciint avic_unaccelerated_access_interception(struct kvm_vcpu *vcpu);
64962306a36Sopenharmony_ciint avic_init_vcpu(struct vcpu_svm *svm);
65062306a36Sopenharmony_civoid avic_vcpu_load(struct kvm_vcpu *vcpu, int cpu);
65162306a36Sopenharmony_civoid avic_vcpu_put(struct kvm_vcpu *vcpu);
65262306a36Sopenharmony_civoid avic_apicv_post_state_restore(struct kvm_vcpu *vcpu);
65362306a36Sopenharmony_civoid avic_refresh_apicv_exec_ctrl(struct kvm_vcpu *vcpu);
65462306a36Sopenharmony_ciint avic_pi_update_irte(struct kvm *kvm, unsigned int host_irq,
65562306a36Sopenharmony_ci			uint32_t guest_irq, bool set);
65662306a36Sopenharmony_civoid avic_vcpu_blocking(struct kvm_vcpu *vcpu);
65762306a36Sopenharmony_civoid avic_vcpu_unblocking(struct kvm_vcpu *vcpu);
65862306a36Sopenharmony_civoid avic_ring_doorbell(struct kvm_vcpu *vcpu);
65962306a36Sopenharmony_ciunsigned long avic_vcpu_get_apicv_inhibit_reasons(struct kvm_vcpu *vcpu);
66062306a36Sopenharmony_civoid avic_refresh_virtual_apic_mode(struct kvm_vcpu *vcpu);
66162306a36Sopenharmony_ci
66262306a36Sopenharmony_ci
66362306a36Sopenharmony_ci/* sev.c */
66462306a36Sopenharmony_ci
66562306a36Sopenharmony_ci#define GHCB_VERSION_MAX	1ULL
66662306a36Sopenharmony_ci#define GHCB_VERSION_MIN	1ULL
66762306a36Sopenharmony_ci
66862306a36Sopenharmony_ci
66962306a36Sopenharmony_ciextern unsigned int max_sev_asid;
67062306a36Sopenharmony_ci
67162306a36Sopenharmony_civoid sev_vm_destroy(struct kvm *kvm);
67262306a36Sopenharmony_ciint sev_mem_enc_ioctl(struct kvm *kvm, void __user *argp);
67362306a36Sopenharmony_ciint sev_mem_enc_register_region(struct kvm *kvm,
67462306a36Sopenharmony_ci				struct kvm_enc_region *range);
67562306a36Sopenharmony_ciint sev_mem_enc_unregister_region(struct kvm *kvm,
67662306a36Sopenharmony_ci				  struct kvm_enc_region *range);
67762306a36Sopenharmony_ciint sev_vm_copy_enc_context_from(struct kvm *kvm, unsigned int source_fd);
67862306a36Sopenharmony_ciint sev_vm_move_enc_context_from(struct kvm *kvm, unsigned int source_fd);
67962306a36Sopenharmony_civoid sev_guest_memory_reclaimed(struct kvm *kvm);
68062306a36Sopenharmony_ci
68162306a36Sopenharmony_civoid pre_sev_run(struct vcpu_svm *svm, int cpu);
68262306a36Sopenharmony_civoid __init sev_set_cpu_caps(void);
68362306a36Sopenharmony_civoid __init sev_hardware_setup(void);
68462306a36Sopenharmony_civoid sev_hardware_unsetup(void);
68562306a36Sopenharmony_ciint sev_cpu_init(struct svm_cpu_data *sd);
68662306a36Sopenharmony_civoid sev_init_vmcb(struct vcpu_svm *svm);
68762306a36Sopenharmony_civoid sev_vcpu_after_set_cpuid(struct vcpu_svm *svm);
68862306a36Sopenharmony_civoid sev_free_vcpu(struct kvm_vcpu *vcpu);
68962306a36Sopenharmony_ciint sev_handle_vmgexit(struct kvm_vcpu *vcpu);
69062306a36Sopenharmony_ciint sev_es_string_io(struct vcpu_svm *svm, int size, unsigned int port, int in);
69162306a36Sopenharmony_civoid sev_es_vcpu_reset(struct vcpu_svm *svm);
69262306a36Sopenharmony_civoid sev_vcpu_deliver_sipi_vector(struct kvm_vcpu *vcpu, u8 vector);
69362306a36Sopenharmony_civoid sev_es_prepare_switch_to_guest(struct sev_es_save_area *hostsa);
69462306a36Sopenharmony_civoid sev_es_unmap_ghcb(struct vcpu_svm *svm);
69562306a36Sopenharmony_ci
69662306a36Sopenharmony_ci/* vmenter.S */
69762306a36Sopenharmony_ci
69862306a36Sopenharmony_civoid __svm_sev_es_vcpu_run(struct vcpu_svm *svm, bool spec_ctrl_intercepted);
69962306a36Sopenharmony_civoid __svm_vcpu_run(struct vcpu_svm *svm, bool spec_ctrl_intercepted);
70062306a36Sopenharmony_ci
70162306a36Sopenharmony_ci#define DEFINE_KVM_GHCB_ACCESSORS(field)						\
70262306a36Sopenharmony_ci	static __always_inline bool kvm_ghcb_##field##_is_valid(const struct vcpu_svm *svm) \
70362306a36Sopenharmony_ci	{									\
70462306a36Sopenharmony_ci		return test_bit(GHCB_BITMAP_IDX(field),				\
70562306a36Sopenharmony_ci				(unsigned long *)&svm->sev_es.valid_bitmap);	\
70662306a36Sopenharmony_ci	}									\
70762306a36Sopenharmony_ci										\
70862306a36Sopenharmony_ci	static __always_inline u64 kvm_ghcb_get_##field##_if_valid(struct vcpu_svm *svm, struct ghcb *ghcb) \
70962306a36Sopenharmony_ci	{									\
71062306a36Sopenharmony_ci		return kvm_ghcb_##field##_is_valid(svm) ? ghcb->save.field : 0;	\
71162306a36Sopenharmony_ci	}									\
71262306a36Sopenharmony_ci
71362306a36Sopenharmony_ciDEFINE_KVM_GHCB_ACCESSORS(cpl)
71462306a36Sopenharmony_ciDEFINE_KVM_GHCB_ACCESSORS(rax)
71562306a36Sopenharmony_ciDEFINE_KVM_GHCB_ACCESSORS(rcx)
71662306a36Sopenharmony_ciDEFINE_KVM_GHCB_ACCESSORS(rdx)
71762306a36Sopenharmony_ciDEFINE_KVM_GHCB_ACCESSORS(rbx)
71862306a36Sopenharmony_ciDEFINE_KVM_GHCB_ACCESSORS(rsi)
71962306a36Sopenharmony_ciDEFINE_KVM_GHCB_ACCESSORS(sw_exit_code)
72062306a36Sopenharmony_ciDEFINE_KVM_GHCB_ACCESSORS(sw_exit_info_1)
72162306a36Sopenharmony_ciDEFINE_KVM_GHCB_ACCESSORS(sw_exit_info_2)
72262306a36Sopenharmony_ciDEFINE_KVM_GHCB_ACCESSORS(sw_scratch)
72362306a36Sopenharmony_ciDEFINE_KVM_GHCB_ACCESSORS(xcr0)
72462306a36Sopenharmony_ci
72562306a36Sopenharmony_ci#endif
726