Lines Matching refs:sptep
150 u64 *sptep;
169 ({ spte = mmu_spte_get_lockless(_walker.sptep); 1; }); \
176 static void mmu_spte_set(u64 *sptep, u64 spte);
217 static void mark_mmio_spte(struct kvm_vcpu *vcpu, u64 *sptep, u64 gfn,
222 trace_mark_mmio_spte(sptep, gfn, mask);
223 mmu_spte_set(sptep, mask);
241 static bool set_mmio_spte(struct kvm_vcpu *vcpu, u64 *sptep, gfn_t gfn,
245 mark_mmio_spte(vcpu, sptep, gfn, access);
291 static void __set_spte(u64 *sptep, u64 spte)
293 WRITE_ONCE(*sptep, spte);
296 static void __update_clear_spte_fast(u64 *sptep, u64 spte)
298 WRITE_ONCE(*sptep, spte);
301 static u64 __update_clear_spte_slow(u64 *sptep, u64 spte)
303 return xchg(sptep, spte);
306 static u64 __get_spte_lockless(u64 *sptep)
308 return READ_ONCE(*sptep);
319 static void count_spte_clear(u64 *sptep, u64 spte)
321 struct kvm_mmu_page *sp = sptep_to_sp(sptep);
331 static void __set_spte(u64 *sptep, u64 spte)
335 ssptep = (union split_spte *)sptep;
350 static void __update_clear_spte_fast(u64 *sptep, u64 spte)
354 ssptep = (union split_spte *)sptep;
366 count_spte_clear(sptep, spte);
369 static u64 __update_clear_spte_slow(u64 *sptep, u64 spte)
373 ssptep = (union split_spte *)sptep;
380 count_spte_clear(sptep, spte);
403 static u64 __get_spte_lockless(u64 *sptep)
405 struct kvm_mmu_page *sp = sptep_to_sp(sptep);
406 union split_spte spte, *orig = (union split_spte *)sptep;
452 * Set the sptep from nonpresent to present.
453 * Note: the sptep being assigned *must* be either not present
457 static void mmu_spte_set(u64 *sptep, u64 new_spte)
459 WARN_ON(is_shadow_present_pte(*sptep));
460 __set_spte(sptep, new_spte);
467 static u64 mmu_spte_update_no_track(u64 *sptep, u64 new_spte)
469 u64 old_spte = *sptep;
474 mmu_spte_set(sptep, new_spte);
479 __update_clear_spte_fast(sptep, new_spte);
481 old_spte = __update_clear_spte_slow(sptep, new_spte);
499 static bool mmu_spte_update(u64 *sptep, u64 new_spte)
502 u64 old_spte = mmu_spte_update_no_track(sptep, new_spte);
536 * It sets the sptep from present to nonpresent, and track the
537 * state bits, it is used to clear the last level sptep.
540 static int mmu_spte_clear_track_bits(u64 *sptep)
543 u64 old_spte = *sptep;
546 __update_clear_spte_fast(sptep, 0ull);
548 old_spte = __update_clear_spte_slow(sptep, 0ull);
573 * Directly clear spte without caring the state bits of sptep,
576 static void mmu_spte_clear_no_track(u64 *sptep)
578 __update_clear_spte_fast(sptep, 0ull);
581 static u64 mmu_spte_get_lockless(u64 *sptep)
583 return __get_spte_lockless(sptep);
605 static bool mmu_spte_age(u64 *sptep)
607 u64 spte = mmu_spte_get_lockless(sptep);
614 (unsigned long *)sptep);
624 mmu_spte_update_no_track(sptep, spte);
931 static void pte_list_remove(struct kvm_rmap_head *rmap_head, u64 *sptep)
933 mmu_spte_clear_track_bits(sptep);
934 __pte_list_remove(sptep, rmap_head);
994 struct pte_list_desc *desc; /* holds the sptep if not NULL */
995 int pos; /* index of the sptep */
1003 * Returns sptep if found, NULL otherwise.
1008 u64 *sptep;
1015 sptep = (u64 *)rmap_head->val;
1021 sptep = iter->desc->sptes[iter->pos];
1023 BUG_ON(!is_shadow_present_pte(*sptep));
1024 return sptep;
1030 * Returns sptep if found, NULL otherwise.
1034 u64 *sptep;
1039 sptep = iter->desc->sptes[iter->pos];
1040 if (sptep)
1049 sptep = iter->desc->sptes[iter->pos];
1056 BUG_ON(!is_shadow_present_pte(*sptep));
1057 return sptep;
1064 static void drop_spte(struct kvm *kvm, u64 *sptep)
1066 if (mmu_spte_clear_track_bits(sptep))
1067 rmap_remove(kvm, sptep);
1071 static bool __drop_large_spte(struct kvm *kvm, u64 *sptep)
1073 if (is_large_pte(*sptep)) {
1074 WARN_ON(sptep_to_sp(sptep)->role.level == PG_LEVEL_4K);
1075 drop_spte(kvm, sptep);
1083 static void drop_large_spte(struct kvm_vcpu *vcpu, u64 *sptep)
1085 if (__drop_large_spte(vcpu->kvm, sptep)) {
1086 struct kvm_mmu_page *sp = sptep_to_sp(sptep);
1094 * Write-protect on the specified @sptep, @pt_protect indicates whether
1106 static bool spte_write_protect(u64 *sptep, bool pt_protect)
1108 u64 spte = *sptep;
1114 rmap_printk("rmap_write_protect: spte %p %llx\n", sptep, *sptep);
1120 return mmu_spte_update(sptep, spte);
1127 u64 *sptep;
1131 for_each_rmap_spte(rmap_head, &iter, sptep)
1132 flush |= spte_write_protect(sptep, pt_protect);
1137 static bool spte_clear_dirty(u64 *sptep)
1139 u64 spte = *sptep;
1141 rmap_printk("rmap_clear_dirty: spte %p %llx\n", sptep, *sptep);
1145 return mmu_spte_update(sptep, spte);
1148 static bool spte_wrprot_for_clear_dirty(u64 *sptep)
1151 (unsigned long *)sptep);
1152 if (was_writable && !spte_ad_enabled(*sptep))
1153 kvm_set_pfn_dirty(spte_to_pfn(*sptep));
1166 u64 *sptep;
1170 for_each_rmap_spte(rmap_head, &iter, sptep)
1171 if (spte_ad_need_write_protect(*sptep))
1172 flush |= spte_wrprot_for_clear_dirty(sptep);
1174 flush |= spte_clear_dirty(sptep);
1179 static bool spte_set_dirty(u64 *sptep)
1181 u64 spte = *sptep;
1183 rmap_printk("rmap_set_dirty: spte %p %llx\n", sptep, *sptep);
1192 return mmu_spte_update(sptep, spte);
1197 u64 *sptep;
1201 for_each_rmap_spte(rmap_head, &iter, sptep)
1202 if (spte_ad_enabled(*sptep))
1203 flush |= spte_set_dirty(sptep);
1317 u64 *sptep;
1321 while ((sptep = rmap_get_first(rmap_head, &iter))) {
1322 rmap_printk("%s: spte %p %llx.\n", __func__, sptep, *sptep);
1324 pte_list_remove(rmap_head, sptep);
1342 u64 *sptep;
1353 for_each_rmap_spte(rmap_head, &iter, sptep) {
1355 sptep, *sptep, gfn, level);
1360 pte_list_remove(rmap_head, sptep);
1364 *sptep, new_pfn);
1366 mmu_spte_clear_track_bits(sptep);
1367 mmu_spte_set(sptep, new_spte);
1534 u64 *sptep;
1538 for_each_rmap_spte(rmap_head, &iter, sptep)
1539 young |= mmu_spte_age(sptep);
1549 u64 *sptep;
1552 for_each_rmap_spte(rmap_head, &iter, sptep)
1553 if (is_accessed_spte(*sptep))
1686 u64 *sptep;
1689 for_each_rmap_spte(&sp->parent_ptes, &iter, sptep) {
1690 mark_unsync(sptep);
2157 iterator->sptep = ((u64 *)__va(iterator->shadow_addr)) + iterator->index;
2175 __shadow_walk_next(iterator, *iterator->sptep);
2178 static void link_shadow_page(struct kvm_vcpu *vcpu, u64 *sptep,
2187 mmu_spte_set(sptep, spte);
2189 mmu_page_add_parent_pte(vcpu, sp, sptep);
2192 mark_unsync(sptep);
2195 static void validate_direct_spte(struct kvm_vcpu *vcpu, u64 *sptep,
2198 if (is_shadow_present_pte(*sptep) && !is_large_pte(*sptep)) {
2208 child = to_shadow_page(*sptep & PT64_BASE_ADDR_MASK);
2212 drop_parent_pte(child, sptep);
2265 u64 *sptep;
2268 while ((sptep = rmap_get_first(&sp->parent_ptes, &iter)))
2269 drop_parent_pte(sp, sptep);
2560 static int set_spte(struct kvm_vcpu *vcpu, u64 *sptep,
2569 if (set_mmio_spte(vcpu, sptep, gfn, pfn, pte_access))
2572 sp = sptep_to_sp(sptep);
2574 ret = make_spte(vcpu, pte_access, level, gfn, pfn, *sptep, speculative,
2580 if (*sptep == spte)
2582 else if (mmu_spte_update(sptep, spte))
2587 static int mmu_set_spte(struct kvm_vcpu *vcpu, u64 *sptep,
2599 *sptep, write_fault, gfn);
2601 if (is_shadow_present_pte(*sptep)) {
2606 if (level > PG_LEVEL_4K && !is_large_pte(*sptep)) {
2608 u64 pte = *sptep;
2611 drop_parent_pte(child, sptep);
2613 } else if (pfn != spte_to_pfn(*sptep)) {
2615 spte_to_pfn(*sptep), pfn);
2616 drop_spte(vcpu->kvm, sptep);
2622 set_spte_ret = set_spte(vcpu, sptep, pte_access, level, gfn, pfn,
2634 if (unlikely(is_mmio_spte(*sptep)))
2646 pgprintk("%s: setting spte %llx\n", __func__, *sptep);
2647 trace_kvm_mmu_set_spte(level, gfn, sptep);
2648 if (!was_rmapped && is_large_pte(*sptep))
2651 if (is_shadow_present_pte(*sptep)) {
2653 rmap_count = rmap_add(vcpu, sptep, gfn);
2655 rmap_recycle(vcpu, sptep, gfn);
2703 struct kvm_mmu_page *sp, u64 *sptep)
2710 i = (sptep - sp->spt) & ~(PTE_PREFETCH_NUM - 1);
2714 if (is_shadow_present_pte(*spte) || spte == sptep) {
2725 static void direct_pte_prefetch(struct kvm_vcpu *vcpu, u64 *sptep)
2729 sp = sptep_to_sp(sptep);
2742 __direct_pte_prefetch(vcpu, sp, sptep);
2877 disallowed_hugepage_adjust(*it.sptep, gfn, it.level,
2884 drop_large_spte(vcpu, it.sptep);
2885 if (!is_shadow_present_pte(*it.sptep)) {
2889 link_shadow_page(vcpu, it.sptep, sp);
2896 ret = mmu_set_spte(vcpu, it.sptep, ACC_ALL,
2902 direct_pte_prefetch(vcpu, it.sptep);
2986 u64 *sptep, u64 old_spte, u64 new_spte)
3004 if (cmpxchg64(sptep, old_spte, new_spte) != old_spte)
3012 gfn = kvm_mmu_page_get_gfn(sp, sptep - sp->spt);
3055 sp = sptep_to_sp(iterator.sptep);
3113 if (fast_pf_fix_direct_spte(vcpu, sp, iterator.sptep, spte,
3127 trace_fast_page_fault(vcpu, cr2_or_gpa, error_code, iterator.sptep,
3504 spte = mmu_spte_get_lockless(iterator.sptep);
3518 static bool get_mmio_spte(struct kvm_vcpu *vcpu, u64 addr, u64 *sptep)
3526 *sptep = 0ull;
3536 *sptep = 0ull;
3563 *sptep = sptes[leaf - 1];
3629 clear_sp_write_flooding_count(iterator.sptep);
3942 static bool sync_mmio_spte(struct kvm_vcpu *vcpu, u64 *sptep, gfn_t gfn,
3945 if (unlikely(is_mmio_spte(*sptep))) {
3946 if (gfn != get_mmio_spte_gfn(*sptep)) {
3947 mmu_spte_clear_no_track(sptep);
3952 mark_mmio_spte(vcpu, sptep, gfn, access);
5574 u64 *sptep;
5581 for_each_rmap_spte(rmap_head, &iter, sptep) {
5582 sp = sptep_to_sp(sptep);
5583 pfn = spte_to_pfn(*sptep);
5595 pte_list_remove(rmap_head, sptep);