Lines Matching defs:root
37 * for zapping and thus puts the TDP MMU's reference to each root, i.e.
76 void kvm_tdp_mmu_put_root(struct kvm *kvm, struct kvm_mmu_page *root,
81 if (!refcount_dec_and_test(&root->tdp_mmu_root_count))
85 * The TDP MMU itself holds a reference to each root until the root is
87 * put for a valid root.
89 KVM_BUG_ON(!is_tdp_mmu_page(root) || !root->role.invalid, kvm);
92 list_del_rcu(&root->link);
94 call_rcu(&root->rcu_head, tdp_mmu_free_sp_rcu_callback);
98 * Returns the next root after @prev_root (or the first root if @prev_root is
99 * NULL). A reference to the returned root is acquired, and the reference to
144 * recent root. (Unless keeping a live reference is desirable.)
147 * mode. In the unlikely event that this thread must free a root, the lock
224 struct kvm_mmu_page *root;
229 * Check for an existing root before allocating a new one. Note, the
230 * role check prevents consuming an invalid root.
232 for_each_tdp_mmu_root(kvm, root, kvm_mmu_role_as_id(role)) {
233 if (root->role.word == role.word &&
234 kvm_tdp_mmu_get_root(root))
238 root = tdp_mmu_alloc_sp(vcpu);
239 tdp_mmu_init_sp(root, NULL, 0, role);
245 * the TDP MMU itself, which is held until the root is invalidated and
248 refcount_set(&root->tdp_mmu_root_count, 2);
251 list_add_rcu(&root->link, &kvm->arch.tdp_mmu_roots);
255 return __pa(root->spt);
643 for_each_tdp_pte(_iter, root_to_sp(_mmu->root.hpa), _start, _end)
655 * from the paging structure root.
701 static void __tdp_mmu_zap_root(struct kvm *kvm, struct kvm_mmu_page *root,
709 for_each_tdp_pte_min_level(iter, root, zap_level, start, end) {
727 static void tdp_mmu_zap_root(struct kvm *kvm, struct kvm_mmu_page *root,
732 * The root must have an elevated refcount so that it's reachable via
736 * callback. Dropping mmu_lock with an unreachable root would result
741 WARN_ON_ONCE(!refcount_read(&root->tdp_mmu_root_count));
757 __tdp_mmu_zap_root(kvm, root, shared, PG_LEVEL_1G);
758 __tdp_mmu_zap_root(kvm, root, shared, root->role.level);
768 * This helper intentionally doesn't allow zapping a root shadow page,
791 static bool tdp_mmu_zap_leafs(struct kvm *kvm, struct kvm_mmu_page *root,
802 for_each_tdp_pte_min_level(iter, root, PG_LEVEL_4K, start, end) {
833 struct kvm_mmu_page *root;
835 for_each_tdp_mmu_root_yield_safe(kvm, root, false)
836 flush = tdp_mmu_zap_leafs(kvm, root, start, end, true, flush);
843 struct kvm_mmu_page *root;
847 * before returning to the caller. Zap directly even if the root is
857 for_each_tdp_mmu_root_yield_safe(kvm, root, false)
858 tdp_mmu_zap_root(kvm, root, false);
867 struct kvm_mmu_page *root;
871 for_each_tdp_mmu_root_yield_safe(kvm, root, true) {
872 if (!root->tdp_mmu_scheduled_root_to_zap)
875 root->tdp_mmu_scheduled_root_to_zap = false;
876 KVM_BUG_ON(!root->role.invalid, kvm);
880 * flush when allocating a new root (see kvm_mmu_load()), and
887 tdp_mmu_zap_root(kvm, root, true);
890 * The referenced needs to be put *after* zapping the root, as
891 * the root must be reachable by mmu_notifiers while it's being
894 kvm_tdp_mmu_put_root(kvm, root, true);
901 * Mark each TDP MMU root as invalid to prevent vCPUs from reusing a root that
912 struct kvm_mmu_page *root;
915 * mmu_lock must be held for write to ensure that a root doesn't become
916 * invalid while there are active readers (invalidating a root while
934 list_for_each_entry(root, &kvm->arch.tdp_mmu_roots, link) {
939 * root alive after its been zapped.
941 if (!root->role.invalid) {
942 root->tdp_mmu_scheduled_root_to_zap = true;
943 root->role.invalid = true;
1126 struct kvm_mmu_page *root;
1128 __for_each_tdp_mmu_root_yield_safe(kvm, root, range->slot->as_id, false, false)
1129 flush = tdp_mmu_zap_leafs(kvm, root, range->start, range->end,
1142 struct kvm_mmu_page *root;
1150 for_each_tdp_mmu_root(kvm, root, range->slot->as_id) {
1153 tdp_root_for_each_leaf_pte(iter, root, range->start, range->end)
1271 static bool wrprot_gfn_range(struct kvm *kvm, struct kvm_mmu_page *root,
1282 for_each_tdp_pte_min_level(iter, root, min_level, start, end) {
1312 struct kvm_mmu_page *root;
1317 for_each_valid_tdp_mmu_root_yield_safe(kvm, root, slot->as_id, true)
1318 spte_set |= wrprot_gfn_range(kvm, root, slot->base_gfn,
1422 struct kvm_mmu_page *root,
1443 for_each_tdp_pte_min_level(iter, root, target_level + 1, start, end) {
1495 struct kvm_mmu_page *root;
1500 for_each_valid_tdp_mmu_root_yield_safe(kvm, root, slot->as_id, shared) {
1501 r = tdp_mmu_split_huge_pages_root(kvm, root, start, end, target_level, shared);
1503 kvm_tdp_mmu_put_root(kvm, root, shared);
1516 static bool clear_dirty_gfn_range(struct kvm *kvm, struct kvm_mmu_page *root,
1525 tdp_root_for_each_leaf_pte(iter, root, start, end) {
1559 struct kvm_mmu_page *root;
1564 for_each_valid_tdp_mmu_root_yield_safe(kvm, root, slot->as_id, true)
1565 spte_set |= clear_dirty_gfn_range(kvm, root, slot->base_gfn,
1578 static void clear_dirty_pt_masked(struct kvm *kvm, struct kvm_mmu_page *root,
1589 tdp_root_for_each_leaf_pte(iter, root, gfn + __ffs(mask),
1631 struct kvm_mmu_page *root;
1633 for_each_tdp_mmu_root(kvm, root, slot->as_id)
1634 clear_dirty_pt_masked(kvm, root, gfn, mask, wrprot);
1638 struct kvm_mmu_page *root,
1648 for_each_tdp_pte_min_level(iter, root, PG_LEVEL_2M, start, end) {
1695 struct kvm_mmu_page *root;
1699 for_each_valid_tdp_mmu_root_yield_safe(kvm, root, slot->as_id, true)
1700 zap_collapsible_spte_range(kvm, root, slot);
1708 static bool write_protect_gfn(struct kvm *kvm, struct kvm_mmu_page *root,
1719 for_each_tdp_pte_min_level(iter, root, min_level, gfn, gfn + 1) {
1748 struct kvm_mmu_page *root;
1752 for_each_tdp_mmu_root(kvm, root, slot->as_id)
1753 spte_set |= write_protect_gfn(kvm, root, gfn, min_level);