Lines Matching defs:spt
658 struct intel_vgpu_ppgtt_spt *spt,
663 struct intel_gvt *gvt = spt->vgpu->gvt;
673 spt->guest_page.gfn << I915_GTT_PAGE_SHIFT,
674 spt->vgpu);
679 spt->guest_page.pde_ips : false);
681 gvt_vdbg_mm("read ppgtt entry, spt type %d, entry type %d, index %lu, value %llx\n",
687 struct intel_vgpu_ppgtt_spt *spt,
692 struct intel_gvt *gvt = spt->vgpu->gvt;
698 gvt_vdbg_mm("set ppgtt entry, spt type %d, entry type %d, index %lu, value %llx\n",
702 spt->guest_page.gfn << I915_GTT_PAGE_SHIFT,
703 spt->vgpu);
706 #define ppgtt_get_guest_entry(spt, e, index) \
707 ppgtt_spt_get_entry(spt, NULL, \
708 spt->guest_page.type, e, index, true)
710 #define ppgtt_set_guest_entry(spt, e, index) \
711 ppgtt_spt_set_entry(spt, NULL, \
712 spt->guest_page.type, e, index, true)
714 #define ppgtt_get_shadow_entry(spt, e, index) \
715 ppgtt_spt_get_entry(spt, spt->shadow_page.vaddr, \
716 spt->shadow_page.type, e, index, false)
718 #define ppgtt_set_shadow_entry(spt, e, index) \
719 ppgtt_spt_set_entry(spt, spt->shadow_page.vaddr, \
720 spt->shadow_page.type, e, index, false)
724 struct intel_vgpu_ppgtt_spt *spt;
726 spt = kzalloc(sizeof(*spt), gfp_mask);
727 if (!spt)
730 spt->shadow_page.page = alloc_page(gfp_mask);
731 if (!spt->shadow_page.page) {
732 kfree(spt);
735 return spt;
738 static void free_spt(struct intel_vgpu_ppgtt_spt *spt)
740 __free_page(spt->shadow_page.page);
741 kfree(spt);
747 static void ppgtt_free_spt(struct intel_vgpu_ppgtt_spt *spt)
749 struct device *kdev = &spt->vgpu->gvt->gt->i915->drm.pdev->dev;
751 trace_spt_free(spt->vgpu->id, spt, spt->guest_page.type);
753 dma_unmap_page(kdev, spt->shadow_page.mfn << I915_GTT_PAGE_SHIFT, 4096,
756 radix_tree_delete(&spt->vgpu->gtt.spt_tree, spt->shadow_page.mfn);
758 if (spt->guest_page.gfn) {
759 if (spt->guest_page.oos_page)
760 detach_oos_page(spt->vgpu, spt->guest_page.oos_page);
762 intel_vgpu_unregister_page_track(spt->vgpu, spt->guest_page.gfn);
765 list_del_init(&spt->post_shadow_list);
766 free_spt(spt);
771 struct intel_vgpu_ppgtt_spt *spt, *spn;
778 spt = radix_tree_deref_slot(slot);
779 list_move(&spt->post_shadow_list, &all_spt);
783 list_for_each_entry_safe(spt, spn, &all_spt, post_shadow_list)
784 ppgtt_free_spt(spt);
788 struct intel_vgpu_ppgtt_spt *spt,
795 struct intel_vgpu_ppgtt_spt *spt = page_track->priv_data;
802 ret = ppgtt_handle_guest_write_page_table_bytes(spt, gpa, data, bytes);
808 /* Find a spt by guest gfn. */
821 /* Find the spt by shadow page mfn. */
835 struct intel_vgpu_ppgtt_spt *spt = NULL;
840 spt = alloc_spt(GFP_KERNEL | __GFP_ZERO);
841 if (!spt) {
849 spt->vgpu = vgpu;
850 atomic_set(&spt->refcount, 1);
851 INIT_LIST_HEAD(&spt->post_shadow_list);
856 spt->shadow_page.type = type;
857 daddr = dma_map_page(kdev, spt->shadow_page.page,
864 spt->shadow_page.vaddr = page_address(spt->shadow_page.page);
865 spt->shadow_page.mfn = daddr >> I915_GTT_PAGE_SHIFT;
867 ret = radix_tree_insert(&vgpu->gtt.spt_tree, spt->shadow_page.mfn, spt);
871 return spt;
876 free_spt(spt);
885 struct intel_vgpu_ppgtt_spt *spt;
888 spt = ppgtt_alloc_spt(vgpu, type);
889 if (IS_ERR(spt))
890 return spt;
896 ppgtt_write_protection_handler, spt);
898 ppgtt_free_spt(spt);
902 spt->guest_page.type = type;
903 spt->guest_page.gfn = gfn;
904 spt->guest_page.pde_ips = guest_pde_ips;
906 trace_spt_alloc(vgpu->id, spt, type, spt->shadow_page.mfn, gfn);
908 return spt;
911 #define pt_entry_size_shift(spt) \
912 ((spt)->vgpu->gvt->device_info.gtt_entry_size_shift)
914 #define pt_entries(spt) \
915 (I915_GTT_PAGE_SIZE >> pt_entry_size_shift(spt))
917 #define for_each_present_guest_entry(spt, e, i) \
918 for (i = 0; i < pt_entries(spt); \
919 i += spt->guest_page.pde_ips ? GTT_64K_PTE_STRIDE : 1) \
920 if (!ppgtt_get_guest_entry(spt, e, i) && \
921 spt->vgpu->gvt->gtt.pte_ops->test_present(e))
923 #define for_each_present_shadow_entry(spt, e, i) \
924 for (i = 0; i < pt_entries(spt); \
925 i += spt->shadow_page.pde_ips ? GTT_64K_PTE_STRIDE : 1) \
926 if (!ppgtt_get_shadow_entry(spt, e, i) && \
927 spt->vgpu->gvt->gtt.pte_ops->test_present(e))
929 #define for_each_shadow_entry(spt, e, i) \
930 for (i = 0; i < pt_entries(spt); \
931 i += (spt->shadow_page.pde_ips ? GTT_64K_PTE_STRIDE : 1)) \
932 if (!ppgtt_get_shadow_entry(spt, e, i))
934 static inline void ppgtt_get_spt(struct intel_vgpu_ppgtt_spt *spt)
936 int v = atomic_read(&spt->refcount);
938 trace_spt_refcount(spt->vgpu->id, "inc", spt, v, (v + 1));
939 atomic_inc(&spt->refcount);
942 static inline int ppgtt_put_spt(struct intel_vgpu_ppgtt_spt *spt)
944 int v = atomic_read(&spt->refcount);
946 trace_spt_refcount(spt->vgpu->id, "dec", spt, v, (v - 1));
947 return atomic_dec_return(&spt->refcount);
950 static int ppgtt_invalidate_spt(struct intel_vgpu_ppgtt_spt *spt);
989 static inline void ppgtt_invalidate_pte(struct intel_vgpu_ppgtt_spt *spt,
992 struct intel_vgpu *vgpu = spt->vgpu;
998 type = spt->shadow_page.type;
1007 static int ppgtt_invalidate_spt(struct intel_vgpu_ppgtt_spt *spt)
1009 struct intel_vgpu *vgpu = spt->vgpu;
1014 trace_spt_change(spt->vgpu->id, "die", spt,
1015 spt->guest_page.gfn, spt->shadow_page.type);
1017 if (ppgtt_put_spt(spt) > 0)
1020 for_each_present_shadow_entry(spt, &e, index) {
1024 ppgtt_invalidate_pte(spt, &e);
1041 spt->vgpu, &e);
1050 trace_spt_change(spt->vgpu->id, "release", spt,
1051 spt->guest_page.gfn, spt->shadow_page.type);
1052 ppgtt_free_spt(spt);
1056 spt, e.val64, e.type);
1076 static int ppgtt_populate_spt(struct intel_vgpu_ppgtt_spt *spt);
1082 struct intel_vgpu_ppgtt_spt *spt = NULL;
1091 spt = intel_vgpu_find_spt_by_gfn(vgpu, ops->get_pfn(we));
1092 if (spt) {
1093 ppgtt_get_spt(spt);
1095 if (ips != spt->guest_page.pde_ips) {
1096 spt->guest_page.pde_ips = ips;
1099 clear_page(spt->shadow_page.vaddr);
1100 ret = ppgtt_populate_spt(spt);
1102 ppgtt_put_spt(spt);
1114 spt = ppgtt_alloc_spt_gfn(vgpu, type, ops->get_pfn(we), ips);
1115 if (IS_ERR(spt)) {
1116 ret = PTR_ERR(spt);
1120 ret = intel_vgpu_enable_page_track(vgpu, spt->guest_page.gfn);
1124 ret = ppgtt_populate_spt(spt);
1128 trace_spt_change(vgpu->id, "new", spt, spt->guest_page.gfn,
1129 spt->shadow_page.type);
1131 return spt;
1134 ppgtt_free_spt(spt);
1135 spt = NULL;
1138 spt, we->val64, we->type);
1182 struct intel_vgpu_ppgtt_spt *spt, unsigned long index,
1222 ppgtt_set_shadow_entry(spt, se, index);
1230 /* Release the new allocated spt. */
1238 struct intel_vgpu_ppgtt_spt *spt, unsigned long index,
1263 ppgtt_set_shadow_entry(spt, &entry, index + i);
1269 struct intel_vgpu_ppgtt_spt *spt, unsigned long index,
1294 return split_64KB_gtt_entry(vgpu, spt, index, &se);
1299 return split_2MB_gtt_entry(vgpu, spt, index, &se);
1318 ppgtt_set_shadow_entry(spt, &se, index);
1322 static int ppgtt_populate_spt(struct intel_vgpu_ppgtt_spt *spt)
1324 struct intel_vgpu *vgpu = spt->vgpu;
1332 trace_spt_change(spt->vgpu->id, "born", spt,
1333 spt->guest_page.gfn, spt->shadow_page.type);
1335 for_each_present_guest_entry(spt, &ge, i) {
1342 ppgtt_get_shadow_entry(spt, &se, i);
1344 ppgtt_set_shadow_entry(spt, &se, i);
1349 ppgtt_set_shadow_entry(spt, &se, i);
1353 ret = ppgtt_populate_shadow_entry(vgpu, spt, i, &ge);
1361 spt, ge.val64, ge.type);
1365 static int ppgtt_handle_guest_entry_removal(struct intel_vgpu_ppgtt_spt *spt,
1368 struct intel_vgpu *vgpu = spt->vgpu;
1372 trace_spt_guest_change(spt->vgpu->id, "remove", spt,
1373 spt->shadow_page.type, se->val64, index);
1382 vgpu->gtt.scratch_pt[spt->shadow_page.type].page_mfn)
1400 ppgtt_invalidate_pte(spt, se);
1406 spt, se->val64, se->type);
1410 static int ppgtt_handle_guest_entry_add(struct intel_vgpu_ppgtt_spt *spt,
1413 struct intel_vgpu *vgpu = spt->vgpu;
1418 trace_spt_guest_change(spt->vgpu->id, "add", spt, spt->shadow_page.type,
1430 ppgtt_get_shadow_entry(spt, &m, index);
1432 ppgtt_set_shadow_entry(spt, &m, index);
1434 ret = ppgtt_populate_shadow_entry(vgpu, spt, index, we);
1440 gvt_vgpu_err("fail: spt %p guest entry 0x%llx type %d\n",
1441 spt, we->val64, we->type);
1451 struct intel_vgpu_ppgtt_spt *spt = oos_page->spt;
1457 spt, spt->guest_page.type);
1459 old.type = new.type = get_entry_type(spt->guest_page.type);
1466 spt->guest_page.gfn << PAGE_SHIFT, vgpu);
1469 && !test_and_clear_bit(index, spt->post_shadow_bitmap))
1473 spt, spt->guest_page.type,
1476 ret = ppgtt_populate_shadow_entry(vgpu, spt, index, &new);
1483 spt->guest_page.write_cnt = 0;
1484 list_del_init(&spt->post_shadow_list);
1492 struct intel_vgpu_ppgtt_spt *spt = oos_page->spt;
1495 spt, spt->guest_page.type);
1497 spt->guest_page.write_cnt = 0;
1498 spt->guest_page.oos_page = NULL;
1499 oos_page->spt = NULL;
1508 struct intel_vgpu_ppgtt_spt *spt)
1510 struct intel_gvt *gvt = spt->vgpu->gvt;
1513 ret = intel_gvt_hypervisor_read_gpa(spt->vgpu,
1514 spt->guest_page.gfn << I915_GTT_PAGE_SHIFT,
1519 oos_page->spt = spt;
1520 spt->guest_page.oos_page = oos_page;
1524 trace_oos_change(spt->vgpu->id, "attach", oos_page->id,
1525 spt, spt->guest_page.type);
1529 static int ppgtt_set_guest_page_sync(struct intel_vgpu_ppgtt_spt *spt)
1531 struct intel_vgpu_oos_page *oos_page = spt->guest_page.oos_page;
1534 ret = intel_vgpu_enable_page_track(spt->vgpu, spt->guest_page.gfn);
1538 trace_oos_change(spt->vgpu->id, "set page sync", oos_page->id,
1539 spt, spt->guest_page.type);
1542 return sync_oos_page(spt->vgpu, oos_page);
1545 static int ppgtt_allocate_oos_page(struct intel_vgpu_ppgtt_spt *spt)
1547 struct intel_gvt *gvt = spt->vgpu->gvt;
1549 struct intel_vgpu_oos_page *oos_page = spt->guest_page.oos_page;
1557 ret = ppgtt_set_guest_page_sync(oos_page->spt);
1560 ret = detach_oos_page(spt->vgpu, oos_page);
1566 return attach_oos_page(oos_page, spt);
1569 static int ppgtt_set_guest_page_oos(struct intel_vgpu_ppgtt_spt *spt)
1571 struct intel_vgpu_oos_page *oos_page = spt->guest_page.oos_page;
1576 trace_oos_change(spt->vgpu->id, "set page out of sync", oos_page->id,
1577 spt, spt->guest_page.type);
1579 list_add_tail(&oos_page->vm_list, &spt->vgpu->gtt.oos_page_list_head);
1580 return intel_vgpu_disable_page_track(spt->vgpu, spt->guest_page.gfn);
1605 ret = ppgtt_set_guest_page_sync(oos_page->spt);
1616 struct intel_vgpu_ppgtt_spt *spt,
1619 struct intel_vgpu *vgpu = spt->vgpu;
1620 int type = spt->shadow_page.type;
1633 ppgtt_get_shadow_entry(spt, &old_se, index);
1636 ret = ppgtt_handle_guest_entry_add(spt, we, index);
1641 ret = ppgtt_handle_guest_entry_removal(spt, &old_se, index);
1654 ppgtt_set_shadow_entry(spt, &old_se, index + i);
1661 ppgtt_set_shadow_entry(spt, &old_se, index);
1665 ppgtt_set_shadow_entry(spt, &old_se, index);
1672 spt, we->val64, we->type);
1678 static inline bool can_do_out_of_sync(struct intel_vgpu_ppgtt_spt *spt)
1681 && gtt_type_is_pte_pt(spt->guest_page.type)
1682 && spt->guest_page.write_cnt >= 2;
1685 static void ppgtt_set_post_shadow(struct intel_vgpu_ppgtt_spt *spt,
1688 set_bit(index, spt->post_shadow_bitmap);
1689 if (!list_empty(&spt->post_shadow_list))
1692 list_add_tail(&spt->post_shadow_list,
1693 &spt->vgpu->gtt.post_shadow_list_head);
1709 struct intel_vgpu_ppgtt_spt *spt;
1715 spt = container_of(pos, struct intel_vgpu_ppgtt_spt,
1718 for_each_set_bit(index, spt->post_shadow_bitmap,
1720 ppgtt_get_guest_entry(spt, &ge, index);
1722 ret = ppgtt_handle_guest_write_page_table(spt,
1726 clear_bit(index, spt->post_shadow_bitmap);
1728 list_del_init(&spt->post_shadow_list);
1734 struct intel_vgpu_ppgtt_spt *spt,
1737 struct intel_vgpu *vgpu = spt->vgpu;
1746 ppgtt_get_guest_entry(spt, &we, index);
1761 ret = ppgtt_handle_guest_write_page_table(spt, &we, index);
1765 if (!test_bit(index, spt->post_shadow_bitmap)) {
1766 int type = spt->shadow_page.type;
1768 ppgtt_get_shadow_entry(spt, &se, index);
1769 ret = ppgtt_handle_guest_entry_removal(spt, &se, index);
1773 ppgtt_set_shadow_entry(spt, &se, index);
1775 ppgtt_set_post_shadow(spt, index);
1781 spt->guest_page.write_cnt++;
1783 if (spt->guest_page.oos_page)
1784 ops->set_entry(spt->guest_page.oos_page->mem, &we, index,
1787 if (can_do_out_of_sync(spt)) {
1788 if (!spt->guest_page.oos_page)
1789 ppgtt_allocate_oos_page(spt);
1791 ret = ppgtt_set_guest_page_oos(spt);
1834 struct intel_vgpu_ppgtt_spt *spt;
1852 spt = ppgtt_populate_spt_by_guest_entry(vgpu, &ge);
1853 if (IS_ERR(spt)) {
1855 ret = PTR_ERR(spt);
1858 ppgtt_generate_shadow_entry(&se, spt, &ge);
2553 gvt_err("Why we still has spt not freed?\n");