Lines Matching defs:mm_ops
138 static kvm_pte_t *kvm_pte_follow(kvm_pte_t pte, struct kvm_pgtable_mm_ops *mm_ops)
140 return mm_ops->phys_to_virt(kvm_pte_to_phys(pte));
148 static kvm_pte_t kvm_init_table_pte(kvm_pte_t *childp, struct kvm_pgtable_mm_ops *mm_ops)
150 kvm_pte_t pte = kvm_phys_to_pte(mm_ops->virt_to_phys(childp));
206 struct kvm_pgtable_mm_ops *mm_ops, kvm_pteref_t pgtable, u32 level);
209 struct kvm_pgtable_mm_ops *mm_ops,
218 .mm_ops = mm_ops,
259 childp = (kvm_pteref_t)kvm_pte_follow(ctx.old, mm_ops);
260 ret = __kvm_pgtable_walk(data, mm_ops, childp, level + 1);
275 struct kvm_pgtable_mm_ops *mm_ops, kvm_pteref_t pgtable, u32 level)
289 ret = __kvm_pgtable_visit(data, mm_ops, pteref, level);
312 ret = __kvm_pgtable_walk(data, pgt->mm_ops, pteref, pgt->start_level);
452 ctx->mm_ops->get_page(ctx->ptep);
465 struct kvm_pgtable_mm_ops *mm_ops = ctx->mm_ops;
473 childp = (kvm_pte_t *)mm_ops->zalloc_page(NULL);
477 new = kvm_init_table_pte(childp, mm_ops);
478 mm_ops->get_page(ctx->ptep);
513 struct kvm_pgtable_mm_ops *mm_ops = ctx->mm_ops;
519 childp = kvm_pte_follow(ctx->old, mm_ops);
521 if (mm_ops->page_count(childp) != 1)
539 mm_ops->put_page(ctx->ptep);
542 mm_ops->put_page(childp);
556 if (!pgt->mm_ops->page_count)
564 struct kvm_pgtable_mm_ops *mm_ops)
568 pgt->pgd = (kvm_pteref_t)mm_ops->zalloc_page(NULL);
574 pgt->mm_ops = mm_ops;
584 struct kvm_pgtable_mm_ops *mm_ops = ctx->mm_ops;
589 mm_ops->put_page(ctx->ptep);
592 mm_ops->put_page(kvm_pte_follow(ctx->old, mm_ops));
605 pgt->mm_ops->put_page(kvm_dereference_pteref(&walker, pgt->pgd));
789 struct kvm_pgtable_mm_ops *mm_ops = ctx->mm_ops;
817 mm_ops->put_page(ctx->ptep);
824 struct kvm_pgtable_mm_ops *mm_ops = ctx->mm_ops;
829 mm_ops->get_page(ctx->ptep);
849 struct kvm_pgtable_mm_ops *mm_ops)
866 mm_ops->put_page(ctx->ptep);
918 struct kvm_pgtable_mm_ops *mm_ops = ctx->mm_ops;
941 if (!kvm_pgtable_walk_skip_cmo(ctx) && mm_ops->dcache_clean_inval_poc &&
943 mm_ops->dcache_clean_inval_poc(kvm_pte_follow(new, mm_ops),
946 if (!kvm_pgtable_walk_skip_cmo(ctx) && mm_ops->icache_inval_pou &&
948 mm_ops->icache_inval_pou(kvm_pte_follow(new, mm_ops), granule);
958 struct kvm_pgtable_mm_ops *mm_ops = ctx->mm_ops;
959 kvm_pte_t *childp = kvm_pte_follow(ctx->old, mm_ops);
969 mm_ops->free_unlinked_table(childp, ctx->level);
976 struct kvm_pgtable_mm_ops *mm_ops = ctx->mm_ops;
990 childp = mm_ops->zalloc_page(data->memcache);
995 mm_ops->put_page(childp);
1004 new = kvm_init_table_pte(childp, mm_ops);
1095 struct kvm_pgtable_mm_ops *mm_ops = ctx->mm_ops;
1102 mm_ops->put_page(ctx->ptep);
1108 childp = kvm_pte_follow(ctx->old, mm_ops);
1110 if (mm_ops->page_count(childp) != 1)
1121 stage2_unmap_put_pte(ctx, mmu, mm_ops);
1123 if (need_flush && mm_ops->dcache_clean_inval_poc)
1124 mm_ops->dcache_clean_inval_poc(kvm_pte_follow(ctx->old, mm_ops),
1128 mm_ops->put_page(childp);
1162 struct kvm_pgtable_mm_ops *mm_ops = ctx->mm_ops;
1182 if (mm_ops->icache_inval_pou &&
1184 mm_ops->icache_inval_pou(kvm_pte_follow(pte, mm_ops),
1326 struct kvm_pgtable_mm_ops *mm_ops = pgt->mm_ops;
1331 if (mm_ops->dcache_clean_inval_poc)
1332 mm_ops->dcache_clean_inval_poc(kvm_pte_follow(ctx->old, mm_ops),
1379 struct kvm_pgtable_mm_ops *mm_ops = pgt->mm_ops;
1390 pgtable = mm_ops->zalloc_page(mc);
1394 ret = __kvm_pgtable_walk(&data, mm_ops, (kvm_pteref_t)pgtable,
1397 kvm_pgtable_stage2_free_unlinked(mm_ops, pgtable, level);
1398 mm_ops->put_page(pgtable);
1429 struct kvm_pgtable_mm_ops *mm_ops = ctx->mm_ops;
1480 kvm_pgtable_stage2_free_unlinked(mm_ops, childp, level);
1481 mm_ops->put_page(childp);
1490 new = kvm_init_table_pte(childp, mm_ops);
1509 struct kvm_pgtable_mm_ops *mm_ops,
1520 pgt->pgd = (kvm_pteref_t)mm_ops->zalloc_pages_exact(pgd_sz);
1526 pgt->mm_ops = mm_ops;
1548 struct kvm_pgtable_mm_ops *mm_ops = ctx->mm_ops;
1553 mm_ops->put_page(ctx->ptep);
1556 mm_ops->put_page(kvm_pte_follow(ctx->old, mm_ops));
1572 pgt->mm_ops->free_pages_exact(kvm_dereference_pteref(&walker, pgt->pgd), pgd_sz);
1576 void kvm_pgtable_stage2_free_unlinked(struct kvm_pgtable_mm_ops *mm_ops, void *pgtable, u32 level)
1596 WARN_ON(__kvm_pgtable_walk(&data, mm_ops, ptep, level + 1));
1598 WARN_ON(mm_ops->page_count(pgtable) != 1);
1599 mm_ops->put_page(pgtable);