xref: /kernel/linux/linux-5.10/arch/x86/kvm/mmu/spte.h (revision 8c2ecf20) 
1// SPDX-License-Identifier: GPL-2.0-only
2
3#ifndef KVM_X86_MMU_SPTE_H
4#define KVM_X86_MMU_SPTE_H
5
6#include "mmu_internal.h"
7
8#define PT_FIRST_AVAIL_BITS_SHIFT 10
9#define PT64_SECOND_AVAIL_BITS_SHIFT 54
10
11/*
12 * The mask used to denote special SPTEs, which can be either MMIO SPTEs or
13 * Access Tracking SPTEs.
14 */
15#define SPTE_SPECIAL_MASK (3ULL << 52)
16#define SPTE_AD_ENABLED_MASK (0ULL << 52)
17#define SPTE_AD_DISABLED_MASK (1ULL << 52)
18#define SPTE_AD_WRPROT_ONLY_MASK (2ULL << 52)
19#define SPTE_MMIO_MASK (3ULL << 52)
20
21#ifdef CONFIG_DYNAMIC_PHYSICAL_MASK
22#define PT64_BASE_ADDR_MASK (physical_mask & ~(u64)(PAGE_SIZE-1))
23#else
24#define PT64_BASE_ADDR_MASK (((1ULL << 52) - 1) & ~(u64)(PAGE_SIZE-1))
25#endif
26
27#define PT64_PERM_MASK (PT_PRESENT_MASK | PT_WRITABLE_MASK | shadow_user_mask \
28			| shadow_x_mask | shadow_nx_mask | shadow_me_mask)
29
30#define ACC_EXEC_MASK    1
31#define ACC_WRITE_MASK   PT_WRITABLE_MASK
32#define ACC_USER_MASK    PT_USER_MASK
33#define ACC_ALL          (ACC_EXEC_MASK | ACC_WRITE_MASK | ACC_USER_MASK)
34
35/* The mask for the R/X bits in EPT PTEs */
36#define PT64_EPT_READABLE_MASK			0x1ull
37#define PT64_EPT_EXECUTABLE_MASK		0x4ull
38
39#define PT64_LEVEL_BITS 9
40
41#define PT64_LEVEL_SHIFT(level) \
42		(PAGE_SHIFT + (level - 1) * PT64_LEVEL_BITS)
43
44#define PT64_INDEX(address, level)\
45	(((address) >> PT64_LEVEL_SHIFT(level)) & ((1 << PT64_LEVEL_BITS) - 1))
46#define SHADOW_PT_INDEX(addr, level) PT64_INDEX(addr, level)
47
48
49#define SPTE_HOST_WRITEABLE	(1ULL << PT_FIRST_AVAIL_BITS_SHIFT)
50#define SPTE_MMU_WRITEABLE	(1ULL << (PT_FIRST_AVAIL_BITS_SHIFT + 1))
51
52/*
53 * Due to limited space in PTEs, the MMIO generation is a 18 bit subset of
54 * the memslots generation and is derived as follows:
55 *
56 * Bits 0-8 of the MMIO generation are propagated to spte bits 3-11
57 * Bits 9-17 of the MMIO generation are propagated to spte bits 54-62
58 *
59 * The KVM_MEMSLOT_GEN_UPDATE_IN_PROGRESS flag is intentionally not included in
60 * the MMIO generation number, as doing so would require stealing a bit from
61 * the "real" generation number and thus effectively halve the maximum number
62 * of MMIO generations that can be handled before encountering a wrap (which
63 * requires a full MMU zap).  The flag is instead explicitly queried when
64 * checking for MMIO spte cache hits.
65 */
66
67#define MMIO_SPTE_GEN_LOW_START		3
68#define MMIO_SPTE_GEN_LOW_END		11
69
70#define MMIO_SPTE_GEN_HIGH_START	PT64_SECOND_AVAIL_BITS_SHIFT
71#define MMIO_SPTE_GEN_HIGH_END		62
72
73#define MMIO_SPTE_GEN_LOW_MASK		GENMASK_ULL(MMIO_SPTE_GEN_LOW_END, \
74						    MMIO_SPTE_GEN_LOW_START)
75#define MMIO_SPTE_GEN_HIGH_MASK		GENMASK_ULL(MMIO_SPTE_GEN_HIGH_END, \
76						    MMIO_SPTE_GEN_HIGH_START)
77
78#define MMIO_SPTE_GEN_LOW_BITS		(MMIO_SPTE_GEN_LOW_END - MMIO_SPTE_GEN_LOW_START + 1)
79#define MMIO_SPTE_GEN_HIGH_BITS		(MMIO_SPTE_GEN_HIGH_END - MMIO_SPTE_GEN_HIGH_START + 1)
80
81/* remember to adjust the comment above as well if you change these */
82static_assert(MMIO_SPTE_GEN_LOW_BITS == 9 && MMIO_SPTE_GEN_HIGH_BITS == 9);
83
84#define MMIO_SPTE_GEN_LOW_SHIFT		(MMIO_SPTE_GEN_LOW_START - 0)
85#define MMIO_SPTE_GEN_HIGH_SHIFT	(MMIO_SPTE_GEN_HIGH_START - MMIO_SPTE_GEN_LOW_BITS)
86
87#define MMIO_SPTE_GEN_MASK		GENMASK_ULL(MMIO_SPTE_GEN_LOW_BITS + MMIO_SPTE_GEN_HIGH_BITS - 1, 0)
88
89extern u64 __read_mostly shadow_nx_mask;
90extern u64 __read_mostly shadow_x_mask; /* mutual exclusive with nx_mask */
91extern u64 __read_mostly shadow_user_mask;
92extern u64 __read_mostly shadow_accessed_mask;
93extern u64 __read_mostly shadow_dirty_mask;
94extern u64 __read_mostly shadow_mmio_value;
95extern u64 __read_mostly shadow_mmio_access_mask;
96extern u64 __read_mostly shadow_present_mask;
97extern u64 __read_mostly shadow_me_mask;
98
99/*
100 * SPTEs used by MMUs without A/D bits are marked with SPTE_AD_DISABLED_MASK;
101 * shadow_acc_track_mask is the set of bits to be cleared in non-accessed
102 * pages.
103 */
104extern u64 __read_mostly shadow_acc_track_mask;
105
106/*
107 * This mask must be set on all non-zero Non-Present or Reserved SPTEs in order
108 * to guard against L1TF attacks.
109 */
110extern u64 __read_mostly shadow_nonpresent_or_rsvd_mask;
111
112/*
113 * The number of high-order 1 bits to use in the mask above.
114 */
115#define SHADOW_NONPRESENT_OR_RSVD_MASK_LEN 5
116
117/*
118 * The mask/shift to use for saving the original R/X bits when marking the PTE
119 * as not-present for access tracking purposes. We do not save the W bit as the
120 * PTEs being access tracked also need to be dirty tracked, so the W bit will be
121 * restored only when a write is attempted to the page.
122 */
123#define SHADOW_ACC_TRACK_SAVED_BITS_MASK (PT64_EPT_READABLE_MASK | \
124					  PT64_EPT_EXECUTABLE_MASK)
125#define SHADOW_ACC_TRACK_SAVED_BITS_SHIFT PT64_SECOND_AVAIL_BITS_SHIFT
126
127/*
128 * In some cases, we need to preserve the GFN of a non-present or reserved
129 * SPTE when we usurp the upper five bits of the physical address space to
130 * defend against L1TF, e.g. for MMIO SPTEs.  To preserve the GFN, we'll
131 * shift bits of the GFN that overlap with shadow_nonpresent_or_rsvd_mask
132 * left into the reserved bits, i.e. the GFN in the SPTE will be split into
133 * high and low parts.  This mask covers the lower bits of the GFN.
134 */
135extern u64 __read_mostly shadow_nonpresent_or_rsvd_lower_gfn_mask;
136
137/*
138 * The number of non-reserved physical address bits irrespective of features
139 * that repurpose legal bits, e.g. MKTME.
140 */
141extern u8 __read_mostly shadow_phys_bits;
142
143static inline bool is_mmio_spte(u64 spte)
144{
145	return (spte & SPTE_SPECIAL_MASK) == SPTE_MMIO_MASK;
146}
147
148static inline bool sp_ad_disabled(struct kvm_mmu_page *sp)
149{
150	return sp->role.ad_disabled;
151}
152
153static inline bool spte_ad_enabled(u64 spte)
154{
155	MMU_WARN_ON(is_mmio_spte(spte));
156	return (spte & SPTE_SPECIAL_MASK) != SPTE_AD_DISABLED_MASK;
157}
158
159static inline bool spte_ad_need_write_protect(u64 spte)
160{
161	MMU_WARN_ON(is_mmio_spte(spte));
162	return (spte & SPTE_SPECIAL_MASK) != SPTE_AD_ENABLED_MASK;
163}
164
165static inline u64 spte_shadow_accessed_mask(u64 spte)
166{
167	MMU_WARN_ON(is_mmio_spte(spte));
168	return spte_ad_enabled(spte) ? shadow_accessed_mask : 0;
169}
170
171static inline u64 spte_shadow_dirty_mask(u64 spte)
172{
173	MMU_WARN_ON(is_mmio_spte(spte));
174	return spte_ad_enabled(spte) ? shadow_dirty_mask : 0;
175}
176
177static inline bool is_access_track_spte(u64 spte)
178{
179	return !spte_ad_enabled(spte) && (spte & shadow_acc_track_mask) == 0;
180}
181
182static inline int is_shadow_present_pte(u64 pte)
183{
184	return (pte != 0) && !is_mmio_spte(pte);
185}
186
187static inline int is_large_pte(u64 pte)
188{
189	return pte & PT_PAGE_SIZE_MASK;
190}
191
192static inline int is_last_spte(u64 pte, int level)
193{
194	if (level == PG_LEVEL_4K)
195		return 1;
196	if (is_large_pte(pte))
197		return 1;
198	return 0;
199}
200
201static inline bool is_executable_pte(u64 spte)
202{
203	return (spte & (shadow_x_mask | shadow_nx_mask)) == shadow_x_mask;
204}
205
206static inline kvm_pfn_t spte_to_pfn(u64 pte)
207{
208	return (pte & PT64_BASE_ADDR_MASK) >> PAGE_SHIFT;
209}
210
211static inline bool is_accessed_spte(u64 spte)
212{
213	u64 accessed_mask = spte_shadow_accessed_mask(spte);
214
215	return accessed_mask ? spte & accessed_mask
216			     : !is_access_track_spte(spte);
217}
218
219static inline bool is_dirty_spte(u64 spte)
220{
221	u64 dirty_mask = spte_shadow_dirty_mask(spte);
222
223	return dirty_mask ? spte & dirty_mask : spte & PT_WRITABLE_MASK;
224}
225
226static inline bool spte_can_locklessly_be_made_writable(u64 spte)
227{
228	return (spte & (SPTE_HOST_WRITEABLE | SPTE_MMU_WRITEABLE)) ==
229		(SPTE_HOST_WRITEABLE | SPTE_MMU_WRITEABLE);
230}
231
232static inline u64 get_mmio_spte_generation(u64 spte)
233{
234	u64 gen;
235
236	gen = (spte & MMIO_SPTE_GEN_LOW_MASK) >> MMIO_SPTE_GEN_LOW_SHIFT;
237	gen |= (spte & MMIO_SPTE_GEN_HIGH_MASK) >> MMIO_SPTE_GEN_HIGH_SHIFT;
238	return gen;
239}
240
241/* Bits which may be returned by set_spte() */
242#define SET_SPTE_WRITE_PROTECTED_PT    BIT(0)
243#define SET_SPTE_NEED_REMOTE_TLB_FLUSH BIT(1)
244#define SET_SPTE_SPURIOUS              BIT(2)
245
246int make_spte(struct kvm_vcpu *vcpu, unsigned int pte_access, int level,
247		     gfn_t gfn, kvm_pfn_t pfn, u64 old_spte, bool speculative,
248		     bool can_unsync, bool host_writable, bool ad_disabled,
249		     u64 *new_spte);
250u64 make_nonleaf_spte(u64 *child_pt, bool ad_disabled);
251u64 make_mmio_spte(struct kvm_vcpu *vcpu, u64 gfn, unsigned int access);
252u64 mark_spte_for_access_track(u64 spte);
253u64 kvm_mmu_changed_pte_notifier_make_spte(u64 old_spte, kvm_pfn_t new_pfn);
254
255void kvm_mmu_reset_all_pte_masks(void);
256
257#endif
258