| /kernel/linux/linux-6.6/arch/x86/kvm/mmu/ |
| H A D | tdp_iter.h | 17 static inline u64 kvm_tdp_mmu_read_spte(tdp_ptep_t sptep) in kvm_tdp_mmu_read_spte() argument
19 return READ_ONCE(*rcu_dereference(sptep)); in kvm_tdp_mmu_read_spte()
22 static inline u64 kvm_tdp_mmu_write_spte_atomic(tdp_ptep_t sptep, u64 new_spte) in kvm_tdp_mmu_write_spte_atomic() argument
24 return xchg(rcu_dereference(sptep), new_spte); in kvm_tdp_mmu_write_spte_atomic()
27 static inline void __kvm_tdp_mmu_write_spte(tdp_ptep_t sptep, u64 new_spte) in __kvm_tdp_mmu_write_spte() argument
29 WRITE_ONCE(*rcu_dereference(sptep), new_spte); in __kvm_tdp_mmu_write_spte()
51 static inline u64 kvm_tdp_mmu_write_spte(tdp_ptep_t sptep, u64 old_spte, in kvm_tdp_mmu_write_spte() argument
55 return kvm_tdp_mmu_write_spte_atomic(sptep, new_spte); in kvm_tdp_mmu_write_spte()
57 __kvm_tdp_mmu_write_spte(sptep, new_spte); in kvm_tdp_mmu_write_spte()
61 static inline u64 tdp_mmu_clear_spte_bits(tdp_ptep_t sptep, u6 argument
93 tdp_ptep_t sptep; global() member
[all...] |
| H A D | mmu.c | 158 u64 *sptep; member
177 ({ spte = mmu_spte_get_lockless(_walker.sptep); 1; }); \
184 static void mmu_spte_set(u64 *sptep, u64 spte);
288 static void kvm_flush_remote_tlbs_sptep(struct kvm *kvm, u64 *sptep) in kvm_flush_remote_tlbs_sptep() argument
290 struct kvm_mmu_page *sp = sptep_to_sp(sptep); in kvm_flush_remote_tlbs_sptep()
291 gfn_t gfn = kvm_mmu_page_get_gfn(sp, spte_index(sptep)); in kvm_flush_remote_tlbs_sptep()
296 static void mark_mmio_spte(struct kvm_vcpu *vcpu, u64 *sptep, u64 gfn, in mark_mmio_spte() argument
301 trace_mark_mmio_spte(sptep, gfn, spte); in mark_mmio_spte()
302 mmu_spte_set(sptep, spte); in mark_mmio_spte()
341 static void __set_spte(u64 *sptep, u6 argument
346 __update_clear_spte_fast(u64 *sptep, u64 spte) __update_clear_spte_fast() argument
351 __update_clear_spte_slow(u64 *sptep, u64 spte) __update_clear_spte_slow() argument
356 __get_spte_lockless(u64 *sptep) __get_spte_lockless() argument
369 count_spte_clear(u64 *sptep, u64 spte) count_spte_clear() argument
381 __set_spte(u64 *sptep, u64 spte) __set_spte() argument
400 __update_clear_spte_fast(u64 *sptep, u64 spte) __update_clear_spte_fast() argument
419 __update_clear_spte_slow(u64 *sptep, u64 spte) __update_clear_spte_slow() argument
453 __get_spte_lockless(u64 *sptep) __get_spte_lockless() argument
483 mmu_spte_set(u64 *sptep, u64 new_spte) mmu_spte_set() argument
493 mmu_spte_update_no_track(u64 *sptep, u64 new_spte) mmu_spte_update_no_track() argument
524 mmu_spte_update(u64 *sptep, u64 new_spte) mmu_spte_update() argument
565 mmu_spte_clear_track_bits(struct kvm *kvm, u64 *sptep) mmu_spte_clear_track_bits() argument
608 mmu_spte_clear_no_track(u64 *sptep) mmu_spte_clear_no_track() argument
613 mmu_spte_get_lockless(u64 *sptep) mmu_spte_get_lockless() argument
619 mmu_spte_age(u64 *sptep) mmu_spte_age() argument
1031 kvm_zap_one_rmap_spte(struct kvm *kvm, struct kvm_rmap_head *rmap_head, u64 *sptep) kvm_zap_one_rmap_spte() argument
1133 u64 *sptep; rmap_get_first() local
1159 u64 *sptep; rmap_get_next() local
1189 drop_spte(struct kvm *kvm, u64 *sptep) drop_spte() argument
1197 drop_large_spte(struct kvm *kvm, u64 *sptep, bool flush) drop_large_spte() argument
1223 spte_write_protect(u64 *sptep, bool pt_protect) spte_write_protect() argument
1241 u64 *sptep; rmap_write_protect() local
1251 spte_clear_dirty(u64 *sptep) spte_clear_dirty() argument
1260 spte_wrprot_for_clear_dirty(u64 *sptep) spte_wrprot_for_clear_dirty() argument
1279 u64 *sptep; __rmap_clear_dirty() local
1455 u64 *sptep; kvm_set_pte_rmap() local
1610 u64 *sptep; kvm_age_rmap() local
1624 u64 *sptep; kvm_test_age_rmap() local
1773 u64 *sptep; kvm_mmu_mark_parents_unsync() local
2301 kvm_mmu_child_role(u64 *sptep, bool direct, unsigned int access) kvm_mmu_child_role() argument
2347 kvm_mmu_get_child_sp(struct kvm_vcpu *vcpu, u64 *sptep, gfn_t gfn, bool direct, unsigned int access) kvm_mmu_get_child_sp() argument
2423 __link_shadow_page(struct kvm *kvm, struct kvm_mmu_memory_cache *cache, u64 *sptep, struct kvm_mmu_page *sp, bool flush) __link_shadow_page() argument
2458 link_shadow_page(struct kvm_vcpu *vcpu, u64 *sptep, struct kvm_mmu_page *sp) link_shadow_page() argument
2464 validate_direct_spte(struct kvm_vcpu *vcpu, u64 *sptep, unsigned direct_access) validate_direct_spte() argument
2532 u64 *sptep; kvm_mmu_unlink_parents() local
2900 mmu_set_spte(struct kvm_vcpu *vcpu, struct kvm_memory_slot *slot, u64 *sptep, unsigned int pte_access, gfn_t gfn, kvm_pfn_t pfn, struct kvm_page_fault *fault) mmu_set_spte() argument
2999 __direct_pte_prefetch(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp, u64 *sptep) __direct_pte_prefetch() argument
3024 direct_pte_prefetch(struct kvm_vcpu *vcpu, u64 *sptep) direct_pte_prefetch() argument
3361 fast_pf_fix_direct_spte(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault, u64 *sptep, u64 old_spte, u64 new_spte) fast_pf_fix_direct_spte() argument
3411 u64 *sptep = NULL; fast_pf_get_last_sptep() local
3429 u64 *sptep = NULL; fast_page_fault() local
4097 get_mmio_spte(struct kvm_vcpu *vcpu, u64 addr, u64 *sptep) get_mmio_spte() argument
4665 sync_mmio_spte(struct kvm_vcpu *vcpu, u64 *sptep, gfn_t gfn, unsigned int access) sync_mmio_spte() argument
6377 u64 *sptep, spte; shadow_mmu_split_huge_page() local
6572 u64 *sptep; kvm_mmu_zap_collapsible_spte() local
[all...] |
| H A D | mmutrace.h | 212 TP_PROTO(u64 *sptep, gfn_t gfn, u64 spte),
213 TP_ARGS(sptep, gfn, spte),
216 __field(void *, sptep)
223 __entry->sptep = sptep;
229 TP_printk("sptep:%p gfn %llx access %x gen %x", __entry->sptep,
257 u64 *sptep, u64 old_spte, int ret),
258 TP_ARGS(vcpu, fault, sptep, old_spte, ret),
264 __field(u64 *, sptep)
[all...] |
| H A D | tdp_iter.c | 14 iter->sptep = iter->pt_path[iter->level - 1] + in tdp_iter_refresh_sptep()
16 iter->old_spte = kvm_tdp_mmu_read_spte(iter->sptep); in tdp_iter_refresh_sptep()
89 iter->old_spte = kvm_tdp_mmu_read_spte(iter->sptep); in try_step_down()
122 iter->sptep++; in try_step_side()
123 iter->old_spte = kvm_tdp_mmu_read_spte(iter->sptep); in try_step_side()
|
| H A D | tdp_mmu.c | 191 static void tdp_mmu_init_sp(struct kvm_mmu_page *sp, tdp_ptep_t sptep, in tdp_mmu_init_sp() argument
200 sp->ptep = sptep; in tdp_mmu_init_sp()
212 parent_sp = sptep_to_sp(rcu_dereference(iter->sptep)); in tdp_mmu_init_child_sp()
217 tdp_mmu_init_sp(child_sp, iter->sptep, iter->gfn, role); in tdp_mmu_init_child_sp()
332 tdp_ptep_t sptep = pt + i; in handle_removed_pt() local
346 old_spte = kvm_tdp_mmu_write_spte_atomic(sptep, REMOVED_SPTE); in handle_removed_pt()
361 old_spte = kvm_tdp_mmu_read_spte(sptep); in handle_removed_pt()
393 old_spte = kvm_tdp_mmu_write_spte(sptep, old_spte, in handle_removed_pt()
532 u64 *sptep = rcu_dereference(iter->sptep); in tdp_mmu_set_spte_atomic() local
603 tdp_mmu_set_spte(struct kvm *kvm, int as_id, tdp_ptep_t sptep, u64 old_spte, u64 new_spte, gfn_t gfn, int level) tdp_mmu_set_spte() argument
1799 tdp_ptep_t sptep = NULL; kvm_tdp_mmu_fast_pf_get_last_sptep() local
[all...] |
| H A D | paging_tmpl.h | 585 u64 *sptep) in pte_prefetch()
592 sp = sptep_to_sp(sptep); in pte_prefetch()
605 return __direct_pte_prefetch(vcpu, sp, sptep); in pte_prefetch()
607 i = spte_index(sptep) & ~(PTE_PREFETCH_NUM - 1); in pte_prefetch()
611 if (spte == sptep) in pte_prefetch()
668 clear_sp_write_flooding_count(it.sptep); in fetch()
674 sp = kvm_mmu_get_child_sp(vcpu, it.sptep, table_gfn, in fetch()
707 link_shadow_page(vcpu, it.sptep, sp); in fetch()
729 disallowed_hugepage_adjust(fault, *it.sptep, it.level); in fetch()
735 validate_direct_spte(vcpu, it.sptep, direct_acces in fetch()
584 pte_prefetch(struct kvm_vcpu *vcpu, struct guest_walker *gw, u64 *sptep) pte_prefetch() argument
907 u64 *sptep, spte; sync_spte() local
[all...] |
| H A D | spte.h | 208 static inline int spte_index(u64 *sptep) in spte_index() argument
210 return ((unsigned long)sptep / sizeof(*sptep)) & (SPTE_ENT_PER_PAGE - 1); in spte_index()
235 static inline struct kvm_mmu_page *sptep_to_sp(u64 *sptep) in sptep_to_sp() argument
237 return to_shadow_page(__pa(sptep)); in sptep_to_sp()
|
| /kernel/linux/linux-5.10/arch/x86/kvm/mmu/ |
| H A D | mmu_audit.c | 32 typedef void (*inspect_spte_fn) (struct kvm_vcpu *vcpu, u64 *sptep, int level);
93 static void audit_mappings(struct kvm_vcpu *vcpu, u64 *sptep, int level) in audit_mappings() argument
100 sp = sptep_to_sp(sptep); in audit_mappings()
110 if (!is_shadow_present_pte(*sptep) || !is_last_spte(*sptep, level)) in audit_mappings()
113 gfn = kvm_mmu_page_get_gfn(sp, sptep - sp->spt); in audit_mappings()
120 if ((*sptep & PT64_BASE_ADDR_MASK) != hpa) in audit_mappings()
123 hpa, *sptep); in audit_mappings()
126 static void inspect_spte_has_rmap(struct kvm *kvm, u64 *sptep) in inspect_spte_has_rmap() argument
135 rev_sp = sptep_to_sp(sptep); in inspect_spte_has_rmap()
160 audit_sptes_have_rmaps(struct kvm_vcpu *vcpu, u64 *sptep, int level) audit_sptes_have_rmaps() argument
166 audit_spte_after_sync(struct kvm_vcpu *vcpu, u64 *sptep, int level) audit_spte_after_sync() argument
193 u64 *sptep; audit_write_protection() local
224 audit_spte(struct kvm_vcpu *vcpu, u64 *sptep, int level) audit_spte() argument
[all...] |
| H A D | mmu.c | 150 u64 *sptep; member
169 ({ spte = mmu_spte_get_lockless(_walker.sptep); 1; }); \
176 static void mmu_spte_set(u64 *sptep, u64 spte);
217 static void mark_mmio_spte(struct kvm_vcpu *vcpu, u64 *sptep, u64 gfn, in mark_mmio_spte() argument
222 trace_mark_mmio_spte(sptep, gfn, mask); in mark_mmio_spte()
223 mmu_spte_set(sptep, mask); in mark_mmio_spte()
241 static bool set_mmio_spte(struct kvm_vcpu *vcpu, u64 *sptep, gfn_t gfn, in set_mmio_spte() argument
245 mark_mmio_spte(vcpu, sptep, gfn, access); in set_mmio_spte()
291 static void __set_spte(u64 *sptep, u64 spte) in __set_spte() argument
293 WRITE_ONCE(*sptep, spt in __set_spte()
296 __update_clear_spte_fast(u64 *sptep, u64 spte) __update_clear_spte_fast() argument
301 __update_clear_spte_slow(u64 *sptep, u64 spte) __update_clear_spte_slow() argument
306 __get_spte_lockless(u64 *sptep) __get_spte_lockless() argument
319 count_spte_clear(u64 *sptep, u64 spte) count_spte_clear() argument
331 __set_spte(u64 *sptep, u64 spte) __set_spte() argument
350 __update_clear_spte_fast(u64 *sptep, u64 spte) __update_clear_spte_fast() argument
369 __update_clear_spte_slow(u64 *sptep, u64 spte) __update_clear_spte_slow() argument
403 __get_spte_lockless(u64 *sptep) __get_spte_lockless() argument
457 mmu_spte_set(u64 *sptep, u64 new_spte) mmu_spte_set() argument
467 mmu_spte_update_no_track(u64 *sptep, u64 new_spte) mmu_spte_update_no_track() argument
499 mmu_spte_update(u64 *sptep, u64 new_spte) mmu_spte_update() argument
540 mmu_spte_clear_track_bits(u64 *sptep) mmu_spte_clear_track_bits() argument
576 mmu_spte_clear_no_track(u64 *sptep) mmu_spte_clear_no_track() argument
581 mmu_spte_get_lockless(u64 *sptep) mmu_spte_get_lockless() argument
605 mmu_spte_age(u64 *sptep) mmu_spte_age() argument
931 pte_list_remove(struct kvm_rmap_head *rmap_head, u64 *sptep) pte_list_remove() argument
1008 u64 *sptep; rmap_get_first() local
1034 u64 *sptep; rmap_get_next() local
1064 drop_spte(struct kvm *kvm, u64 *sptep) drop_spte() argument
1071 __drop_large_spte(struct kvm *kvm, u64 *sptep) __drop_large_spte() argument
1083 drop_large_spte(struct kvm_vcpu *vcpu, u64 *sptep) drop_large_spte() argument
1106 spte_write_protect(u64 *sptep, bool pt_protect) spte_write_protect() argument
1127 u64 *sptep; __rmap_write_protect() local
1137 spte_clear_dirty(u64 *sptep) spte_clear_dirty() argument
1148 spte_wrprot_for_clear_dirty(u64 *sptep) spte_wrprot_for_clear_dirty() argument
1166 u64 *sptep; __rmap_clear_dirty() local
1179 spte_set_dirty(u64 *sptep) spte_set_dirty() argument
1197 u64 *sptep; __rmap_set_dirty() local
1317 u64 *sptep; kvm_zap_rmapp() local
1342 u64 *sptep; kvm_set_pte_rmapp() local
1534 u64 *sptep; kvm_age_rmapp() local
1549 u64 *sptep; kvm_test_age_rmapp() local
1686 u64 *sptep; kvm_mmu_mark_parents_unsync() local
2178 link_shadow_page(struct kvm_vcpu *vcpu, u64 *sptep, struct kvm_mmu_page *sp) link_shadow_page() argument
2195 validate_direct_spte(struct kvm_vcpu *vcpu, u64 *sptep, unsigned direct_access) validate_direct_spte() argument
2265 u64 *sptep; kvm_mmu_unlink_parents() local
2560 set_spte(struct kvm_vcpu *vcpu, u64 *sptep, unsigned int pte_access, int level, gfn_t gfn, kvm_pfn_t pfn, bool speculative, bool can_unsync, bool host_writable) set_spte() argument
2587 mmu_set_spte(struct kvm_vcpu *vcpu, u64 *sptep, unsigned int pte_access, bool write_fault, int level, gfn_t gfn, kvm_pfn_t pfn, bool speculative, bool host_writable) mmu_set_spte() argument
2702 __direct_pte_prefetch(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp, u64 *sptep) __direct_pte_prefetch() argument
2725 direct_pte_prefetch(struct kvm_vcpu *vcpu, u64 *sptep) direct_pte_prefetch() argument
2985 fast_pf_fix_direct_spte(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp, u64 *sptep, u64 old_spte, u64 new_spte) fast_pf_fix_direct_spte() argument
3518 get_mmio_spte(struct kvm_vcpu *vcpu, u64 addr, u64 *sptep) get_mmio_spte() argument
3942 sync_mmio_spte(struct kvm_vcpu *vcpu, u64 *sptep, gfn_t gfn, unsigned int access, int *nr_present) sync_mmio_spte() argument
5574 u64 *sptep; kvm_mmu_zap_collapsible_spte() local
[all...] |
| H A D | mmutrace.h | 205 TP_PROTO(u64 *sptep, gfn_t gfn, u64 spte),
206 TP_ARGS(sptep, gfn, spte),
209 __field(void *, sptep)
216 __entry->sptep = sptep;
222 TP_printk("sptep:%p gfn %llx access %x gen %x", __entry->sptep,
250 u64 *sptep, u64 old_spte, int ret),
251 TP_ARGS(vcpu, cr2_or_gpa, error_code, sptep, old_spte, ret),
257 __field(u64 *, sptep)
[all...] |
| H A D | tdp_iter.c | 13 iter->sptep = iter->pt_path[iter->level - 1] + in tdp_iter_refresh_sptep()
15 iter->old_spte = READ_ONCE(*iter->sptep); in tdp_iter_refresh_sptep()
78 iter->old_spte = READ_ONCE(*iter->sptep); in try_step_down()
111 iter->sptep++; in try_step_side()
112 iter->old_spte = READ_ONCE(*iter->sptep); in try_step_side()
|
| H A D | paging_tmpl.h | 591 u64 *sptep) in pte_prefetch()
598 sp = sptep_to_sp(sptep); in pte_prefetch()
604 return __direct_pte_prefetch(vcpu, sp, sptep); in pte_prefetch()
606 i = (sptep - sp->spt) & ~(PTE_PREFETCH_NUM - 1); in pte_prefetch()
610 if (spte == sptep) in pte_prefetch()
663 clear_sp_write_flooding_count(it.sptep); in fetch()
664 drop_large_spte(vcpu, it.sptep); in fetch()
667 if (!is_shadow_present_pte(*it.sptep)) { in fetch()
682 link_shadow_page(vcpu, it.sptep, sp); in fetch()
691 clear_sp_write_flooding_count(it.sptep); in fetch()
590 pte_prefetch(struct kvm_vcpu *vcpu, struct guest_walker *gw, u64 *sptep) pte_prefetch() argument
907 u64 *sptep; invlpg() local
[all...] |
| H A D | mmu_internal.h | 71 static inline struct kvm_mmu_page *sptep_to_sp(u64 *sptep) in sptep_to_sp() argument
73 return to_shadow_page(__pa(sptep)); in sptep_to_sp()
|
| H A D | tdp_iter.h | 28 u64 *sptep; member
37 /* A snapshot of the value at sptep */
|
| H A D | tdp_mmu.c | 360 WRITE_ONCE(*iter->sptep, new_spte); in __tdp_mmu_set_spte()
535 trace_mark_mmio_spte(iter->sptep, iter->gfn, new_spte); in tdp_mmu_map_handle_target_level()
562 trace_kvm_mmu_set_spte(iter->level, iter->gfn, iter->sptep); in tdp_mmu_map_handle_target_level()
625 iter.old_spte = READ_ONCE(*iter.sptep); in kvm_tdp_mmu_map()
|
| /kernel/linux/linux-5.10/arch/s390/mm/ |
| H A D | pgtable.c | 649 pte_t *sptep, pte_t *tptep, pte_t pte) in ptep_shadow_pte()
657 spgste = pgste_get_lock(sptep); in ptep_shadow_pte()
658 spte = *sptep; in ptep_shadow_pte()
671 pgste_set_unlock(sptep, spgste); in ptep_shadow_pte()
648 ptep_shadow_pte(struct mm_struct *mm, unsigned long saddr, pte_t *sptep, pte_t *tptep, pte_t pte) ptep_shadow_pte() argument
|
| H A D | gmap.c | 2112 pte_t *sptep, *tptep; in gmap_shadow_page() local
2136 sptep = gmap_pte_op_walk(parent, paddr, &ptl); in gmap_shadow_page()
2137 if (sptep) { in gmap_shadow_page()
2147 rc = ptep_shadow_pte(sg->mm, saddr, sptep, tptep, pte); in gmap_shadow_page()
|
| /kernel/linux/linux-6.6/arch/s390/mm/ |
| H A D | pgtable.c | 688 pte_t *sptep, pte_t *tptep, pte_t pte) in ptep_shadow_pte()
696 spgste = pgste_get_lock(sptep); in ptep_shadow_pte()
697 spte = *sptep; in ptep_shadow_pte()
710 pgste_set_unlock(sptep, spgste); in ptep_shadow_pte()
687 ptep_shadow_pte(struct mm_struct *mm, unsigned long saddr, pte_t *sptep, pte_t *tptep, pte_t pte) ptep_shadow_pte() argument
|
| H A D | gmap.c | 2141 pte_t *sptep, *tptep; in gmap_shadow_page() local
2165 sptep = gmap_pte_op_walk(parent, paddr, &ptl); in gmap_shadow_page()
2166 if (sptep) { in gmap_shadow_page()
2172 gmap_pte_op_end(sptep, ptl); in gmap_shadow_page()
2176 rc = ptep_shadow_pte(sg->mm, saddr, sptep, tptep, pte); in gmap_shadow_page()
2183 gmap_pte_op_end(sptep, ptl); in gmap_shadow_page()
|
| /kernel/linux/linux-5.10/arch/s390/include/asm/ |
| H A D | pgtable.h | 1165 pte_t *sptep, pte_t *tptep, pte_t pte);
|
| /kernel/linux/linux-6.6/arch/s390/include/asm/ |
| H A D | pgtable.h | 1287 pte_t *sptep, pte_t *tptep, pte_t pte);
|