Lines Matching refs:gmap
23 #include <asm/gmap.h>
31 * @limit: maximum address of the gmap address space
35 static struct gmap *gmap_alloc(unsigned long limit)
37 struct gmap *gmap;
59 gmap = kzalloc(sizeof(struct gmap), GFP_KERNEL);
60 if (!gmap)
62 INIT_LIST_HEAD(&gmap->crst_list);
63 INIT_LIST_HEAD(&gmap->children);
64 INIT_LIST_HEAD(&gmap->pt_list);
65 INIT_RADIX_TREE(&gmap->guest_to_host, GFP_KERNEL);
66 INIT_RADIX_TREE(&gmap->host_to_guest, GFP_ATOMIC);
67 INIT_RADIX_TREE(&gmap->host_to_rmap, GFP_ATOMIC);
68 spin_lock_init(&gmap->guest_table_lock);
69 spin_lock_init(&gmap->shadow_lock);
70 refcount_set(&gmap->ref_count, 1);
75 list_add(&page->lru, &gmap->crst_list);
78 gmap->table = table;
79 gmap->asce = atype | _ASCE_TABLE_LENGTH |
81 gmap->asce_end = limit;
82 return gmap;
85 kfree(gmap);
93 * @limit: maximum size of the gmap address space
97 struct gmap *gmap_create(struct mm_struct *mm, unsigned long limit)
99 struct gmap *gmap;
102 gmap = gmap_alloc(limit);
103 if (!gmap)
105 gmap->mm = mm;
107 list_add_rcu(&gmap->list, &mm->context.gmap_list);
109 gmap_asce = gmap->asce;
114 return gmap;
118 static void gmap_flush_tlb(struct gmap *gmap)
121 __tlb_flush_idte(gmap->asce);
179 * @gmap: pointer to the guest address space structure
181 * No locks required. There are no references to this gmap anymore.
183 static void gmap_free(struct gmap *gmap)
188 if (!(gmap_is_shadow(gmap) && gmap->removed))
189 gmap_flush_tlb(gmap);
191 list_for_each_entry_safe(page, next, &gmap->crst_list, lru)
193 gmap_radix_tree_free(&gmap->guest_to_host);
194 gmap_radix_tree_free(&gmap->host_to_guest);
196 /* Free additional data for a shadow gmap */
197 if (gmap_is_shadow(gmap)) {
199 list_for_each_entry_safe(page, next, &gmap->pt_list, lru)
201 gmap_rmap_radix_tree_free(&gmap->host_to_rmap);
203 gmap_put(gmap->parent);
206 kfree(gmap);
211 * @gmap: pointer to the guest address space structure
213 * Returns the gmap pointer
215 struct gmap *gmap_get(struct gmap *gmap)
217 refcount_inc(&gmap->ref_count);
218 return gmap;
224 * @gmap: pointer to the guest address space structure
228 void gmap_put(struct gmap *gmap)
230 if (refcount_dec_and_test(&gmap->ref_count))
231 gmap_free(gmap);
237 * @gmap: pointer to the guest address space structure
239 void gmap_remove(struct gmap *gmap)
241 struct gmap *sg, *next;
244 /* Remove all shadow gmaps linked to this gmap */
245 if (!list_empty(&gmap->children)) {
246 spin_lock(&gmap->shadow_lock);
247 list_for_each_entry_safe(sg, next, &gmap->children, list) {
251 spin_unlock(&gmap->shadow_lock);
253 /* Remove gmap from the pre-mm list */
254 spin_lock(&gmap->mm->context.lock);
255 list_del_rcu(&gmap->list);
256 if (list_empty(&gmap->mm->context.gmap_list))
258 else if (list_is_singular(&gmap->mm->context.gmap_list))
259 gmap_asce = list_first_entry(&gmap->mm->context.gmap_list,
260 struct gmap, list)->asce;
263 WRITE_ONCE(gmap->mm->context.gmap_asce, gmap_asce);
264 spin_unlock(&gmap->mm->context.lock);
267 gmap_put(gmap);
273 * @gmap: pointer to the guest address space structure
275 void gmap_enable(struct gmap *gmap)
277 S390_lowcore.gmap = (unsigned long) gmap;
283 * @gmap: pointer to the guest address space structure
285 void gmap_disable(struct gmap *gmap)
287 S390_lowcore.gmap = 0UL;
292 * gmap_get_enabled - get a pointer to the currently enabled gmap
294 * Returns a pointer to the currently enabled gmap. 0 if none is enabled.
296 struct gmap *gmap_get_enabled(void)
298 return (struct gmap *) S390_lowcore.gmap;
305 static int gmap_alloc_table(struct gmap *gmap, unsigned long *table,
311 /* since we dont free the gmap table until gmap_free we can unlock */
317 spin_lock(&gmap->guest_table_lock);
319 list_add(&page->lru, &gmap->crst_list);
325 spin_unlock(&gmap->guest_table_lock);
351 * @gmap: pointer to the guest address space structure
356 static int __gmap_unlink_by_vmaddr(struct gmap *gmap, unsigned long vmaddr)
361 BUG_ON(gmap_is_shadow(gmap));
362 spin_lock(&gmap->guest_table_lock);
363 entry = radix_tree_delete(&gmap->host_to_guest, vmaddr >> PMD_SHIFT);
368 spin_unlock(&gmap->guest_table_lock);
374 * @gmap: pointer to the guest address space structure
379 static int __gmap_unmap_by_gaddr(struct gmap *gmap, unsigned long gaddr)
383 vmaddr = (unsigned long) radix_tree_delete(&gmap->guest_to_host,
385 return vmaddr ? __gmap_unlink_by_vmaddr(gmap, vmaddr) : 0;
390 * @gmap: pointer to the guest address space structure
396 int gmap_unmap_segment(struct gmap *gmap, unsigned long to, unsigned long len)
401 BUG_ON(gmap_is_shadow(gmap));
408 mmap_write_lock(gmap->mm);
410 flush |= __gmap_unmap_by_gaddr(gmap, to + off);
411 mmap_write_unlock(gmap->mm);
413 gmap_flush_tlb(gmap);
420 * @gmap: pointer to the guest address space structure
427 int gmap_map_segment(struct gmap *gmap, unsigned long from,
433 BUG_ON(gmap_is_shadow(gmap));
437 from + len - 1 > TASK_SIZE_MAX || to + len - 1 > gmap->asce_end)
441 mmap_write_lock(gmap->mm);
444 flush |= __gmap_unmap_by_gaddr(gmap, to + off);
446 if (radix_tree_insert(&gmap->guest_to_host,
451 mmap_write_unlock(gmap->mm);
453 gmap_flush_tlb(gmap);
456 gmap_unmap_segment(gmap, to, len);
463 * @gmap: pointer to guest mapping meta data structure
474 unsigned long __gmap_translate(struct gmap *gmap, unsigned long gaddr)
479 radix_tree_lookup(&gmap->guest_to_host, gaddr >> PMD_SHIFT);
480 /* Note: guest_to_host is empty for a shadow gmap */
487 * @gmap: pointer to guest mapping meta data structure
494 unsigned long gmap_translate(struct gmap *gmap, unsigned long gaddr)
498 mmap_read_lock(gmap->mm);
499 rc = __gmap_translate(gmap, gaddr);
500 mmap_read_unlock(gmap->mm);
506 * gmap_unlink - disconnect a page table from the gmap shadow tables
507 * @gmap: pointer to guest mapping meta data structure
514 struct gmap *gmap;
518 list_for_each_entry_rcu(gmap, &mm->context.gmap_list, list) {
519 flush = __gmap_unlink_by_vmaddr(gmap, vmaddr);
521 gmap_flush_tlb(gmap);
526 static void gmap_pmdp_xchg(struct gmap *gmap, pmd_t *old, pmd_t new,
531 * @gmap: pointer to guest mapping meta data structure
540 int __gmap_link(struct gmap *gmap, unsigned long gaddr, unsigned long vmaddr)
552 BUG_ON(gmap_is_shadow(gmap));
553 /* Create higher level tables in the gmap page table */
554 table = gmap->table;
555 if ((gmap->asce & _ASCE_TYPE_MASK) >= _ASCE_TYPE_REGION1) {
558 gmap_alloc_table(gmap, table, _REGION2_ENTRY_EMPTY,
563 if ((gmap->asce & _ASCE_TYPE_MASK) >= _ASCE_TYPE_REGION2) {
566 gmap_alloc_table(gmap, table, _REGION3_ENTRY_EMPTY,
571 if ((gmap->asce & _ASCE_TYPE_MASK) >= _ASCE_TYPE_REGION3) {
574 gmap_alloc_table(gmap, table, _SEGMENT_ENTRY_EMPTY,
581 mm = gmap->mm;
594 if (pmd_large(*pmd) && !gmap->mm->context.allow_gmap_hpage_1m)
596 /* Link gmap segment table entry location to page table. */
601 spin_lock(&gmap->guest_table_lock);
603 rc = radix_tree_insert(&gmap->host_to_guest,
619 gmap_pmdp_xchg(gmap, (pmd_t *)table, __pmd(unprot), gaddr);
621 spin_unlock(&gmap->guest_table_lock);
629 * @gmap: pointer to guest mapping meta data structure
636 int gmap_fault(struct gmap *gmap, unsigned long gaddr,
643 mmap_read_lock(gmap->mm);
647 vmaddr = __gmap_translate(gmap, gaddr);
652 if (fixup_user_fault(gmap->mm, vmaddr, fault_flags,
664 rc = __gmap_link(gmap, gaddr, vmaddr);
666 mmap_read_unlock(gmap->mm);
674 void __gmap_zap(struct gmap *gmap, unsigned long gaddr)
681 vmaddr = (unsigned long) radix_tree_lookup(&gmap->guest_to_host,
686 ptep = get_locked_pte(gmap->mm, vmaddr, &ptl);
688 ptep_zap_unused(gmap->mm, vmaddr, ptep, 0);
695 void gmap_discard(struct gmap *gmap, unsigned long from, unsigned long to)
700 mmap_read_lock(gmap->mm);
705 radix_tree_lookup(&gmap->guest_to_host,
711 vma = find_vma(gmap->mm, vmaddr);
723 mmap_read_unlock(gmap->mm);
732 * @nb: pointer to the gmap notifier block
744 * @nb: pointer to the gmap notifier block
757 * @gmap: pointer to guest mapping meta data structure
761 static void gmap_call_notifier(struct gmap *gmap, unsigned long start,
767 nb->notifier_call(gmap, start, end);
771 * gmap_table_walk - walk the gmap page tables
772 * @gmap: pointer to guest mapping meta data structure
783 * Returns NULL if the gmap page tables could not be walked to the
788 static inline unsigned long *gmap_table_walk(struct gmap *gmap,
791 const int asce_type = gmap->asce & _ASCE_TYPE_MASK;
792 unsigned long *table = gmap->table;
794 if (gmap_is_shadow(gmap) && gmap->removed)
842 * gmap_pte_op_walk - walk the gmap page table, get the page table lock
844 * @gmap: pointer to guest mapping meta data structure
850 static pte_t *gmap_pte_op_walk(struct gmap *gmap, unsigned long gaddr,
855 BUG_ON(gmap_is_shadow(gmap));
856 /* Walk the gmap page table, lock and get pte pointer */
857 table = gmap_table_walk(gmap, gaddr, 1); /* get segment pointer */
860 return pte_alloc_map_lock(gmap->mm, (pmd_t *) table, gaddr, ptl);
864 * gmap_pte_op_fixup - force a page in and connect the gmap page table
865 * @gmap: pointer to guest mapping meta data structure
872 * up or connecting the gmap page table.
874 static int gmap_pte_op_fixup(struct gmap *gmap, unsigned long gaddr,
877 struct mm_struct *mm = gmap->mm;
881 BUG_ON(gmap_is_shadow(gmap));
889 return __gmap_link(gmap, gaddr, vmaddr);
903 * gmap_pmd_op_walk - walk the gmap tables, get the guest table lock
905 * @gmap: pointer to guest mapping meta data structure
910 static inline pmd_t *gmap_pmd_op_walk(struct gmap *gmap, unsigned long gaddr)
914 BUG_ON(gmap_is_shadow(gmap));
915 pmdp = (pmd_t *) gmap_table_walk(gmap, gaddr, 1);
920 if (!gmap->mm->context.allow_gmap_hpage_1m)
923 spin_lock(&gmap->guest_table_lock);
925 spin_unlock(&gmap->guest_table_lock);
931 spin_unlock(&gmap->guest_table_lock);
937 * @gmap: pointer to the guest mapping meta data structure
940 static inline void gmap_pmd_op_end(struct gmap *gmap, pmd_t *pmdp)
943 spin_unlock(&gmap->guest_table_lock);
960 static int gmap_protect_pmd(struct gmap *gmap, unsigned long gaddr,
973 gmap_pmdp_xchg(gmap, pmdp, new, gaddr);
979 gmap_pmdp_xchg(gmap, pmdp, new, gaddr);
994 * @gmap: pointer to guest mapping meta data structure
1005 static int gmap_protect_pte(struct gmap *gmap, unsigned long gaddr,
1016 ptep = pte_alloc_map_lock(gmap->mm, pmdp, gaddr, &ptl);
1023 rc = ptep_force_prot(gmap->mm, gaddr, ptep, prot, pbits);
1030 * @gmap: pointer to guest mapping meta data structure
1041 static int gmap_protect_range(struct gmap *gmap, unsigned long gaddr,
1048 BUG_ON(gmap_is_shadow(gmap));
1051 pmdp = gmap_pmd_op_walk(gmap, gaddr);
1054 rc = gmap_protect_pte(gmap, gaddr, pmdp, prot,
1061 rc = gmap_protect_pmd(gmap, gaddr, pmdp, prot,
1069 gmap_pmd_op_end(gmap, pmdp);
1075 /* -EAGAIN, fixup of userspace mm and gmap */
1076 vmaddr = __gmap_translate(gmap, gaddr);
1079 rc = gmap_pte_op_fixup(gmap, gaddr, vmaddr, prot);
1090 * @gmap: pointer to guest mapping meta data structure
1095 * Returns 0 if for each page in the given range a gmap mapping exists,
1097 * If the gmap mapping is missing for one or more pages -EFAULT is
1101 int gmap_mprotect_notify(struct gmap *gmap, unsigned long gaddr,
1106 if ((gaddr & ~PAGE_MASK) || (len & ~PAGE_MASK) || gmap_is_shadow(gmap))
1110 mmap_read_lock(gmap->mm);
1111 rc = gmap_protect_range(gmap, gaddr, len, prot, GMAP_NOTIFY_MPROT);
1112 mmap_read_unlock(gmap->mm);
1120 * @gmap: pointer to guest mapping meta data structure
1125 * if reading using the virtual address failed. -EINVAL if called on a gmap
1128 * Called with gmap->mm->mmap_lock in read.
1130 int gmap_read_table(struct gmap *gmap, unsigned long gaddr, unsigned long *val)
1137 if (gmap_is_shadow(gmap))
1142 ptep = gmap_pte_op_walk(gmap, gaddr, &ptl);
1157 vmaddr = __gmap_translate(gmap, gaddr);
1162 rc = gmap_pte_op_fixup(gmap, gaddr, vmaddr, PROT_READ);
1178 static inline void gmap_insert_rmap(struct gmap *sg, unsigned long vmaddr,
1199 * @raddr: rmap address in the shadow gmap
1206 static int gmap_protect_rmap(struct gmap *sg, unsigned long raddr,
1209 struct gmap *parent;
1287 static void gmap_unshadow_page(struct gmap *sg, unsigned long raddr)
1307 static void __gmap_unshadow_pgt(struct gmap *sg, unsigned long raddr,
1324 static void gmap_unshadow_pgt(struct gmap *sg, unsigned long raddr)
1353 static void __gmap_unshadow_sgt(struct gmap *sg, unsigned long raddr,
1381 static void gmap_unshadow_sgt(struct gmap *sg, unsigned long raddr)
1410 static void __gmap_unshadow_r3t(struct gmap *sg, unsigned long raddr,
1438 static void gmap_unshadow_r3t(struct gmap *sg, unsigned long raddr)
1467 static void __gmap_unshadow_r2t(struct gmap *sg, unsigned long raddr,
1495 static void gmap_unshadow_r2t(struct gmap *sg, unsigned long raddr)
1524 static void __gmap_unshadow_r1t(struct gmap *sg, unsigned long raddr,
1554 static void gmap_unshadow(struct gmap *sg)
1583 * @parent: pointer to the parent gmap
1587 * Returns the pointer to a gmap if a shadow table with the given asce is
1591 static struct gmap *gmap_find_shadow(struct gmap *parent, unsigned long asce,
1594 struct gmap *sg;
1615 * Returns 1 if the gmap shadow is still valid and matches the given
1617 * caller has to request a new shadow gmap in this case.
1620 int gmap_shadow_valid(struct gmap *sg, unsigned long asce, int edat_level)
1630 * @parent: pointer to the parent gmap
1641 * parent gmap table could not be protected.
1643 struct gmap *gmap_shadow(struct gmap *parent, unsigned long asce,
1646 struct gmap *sg, *new;
1657 /* Create a new shadow gmap */
1678 /* only allow one real-space gmap shadow */
1720 * @saddr: faulting address in the shadow gmap
1721 * @r2t: parent gmap address of the region 2 table to get shadowed
1725 * four pages of the source table are made read-only in the parent gmap
1731 * -EFAULT if an address in the parent gmap could not be resolved.
1735 int gmap_shadow_r2t(struct gmap *sg, unsigned long saddr, unsigned long r2t,
1780 /* Make r2t read-only in parent gmap page table */
1809 * @saddr: faulting address in the shadow gmap
1810 * @r3t: parent gmap address of the region 3 table to get shadowed
1815 * -EFAULT if an address in the parent gmap could not be resolved.
1819 int gmap_shadow_r3t(struct gmap *sg, unsigned long saddr, unsigned long r3t,
1864 /* Make r3t read-only in parent gmap page table */
1893 * @saddr: faulting address in the shadow gmap
1894 * @sgt: parent gmap address of the segment table to get shadowed
1899 * -EFAULT if an address in the parent gmap could not be resolved.
1903 int gmap_shadow_sgt(struct gmap *sg, unsigned long saddr, unsigned long sgt,
1948 /* Make sgt read-only in parent gmap page table */
1978 * @pgt: parent gmap address of the page table to get shadowed
1987 int gmap_shadow_pgt_lookup(struct gmap *sg, unsigned long saddr,
2017 * @saddr: faulting address in the shadow gmap
2018 * @pgt: parent gmap address of the page table to get shadowed
2023 * -EFAULT if an address in the parent gmap could not be resolved and
2025 * Called with gmap->mm->mmap_lock in read
2027 int gmap_shadow_pgt(struct gmap *sg, unsigned long saddr, unsigned long pgt,
2069 /* Make pgt read-only in parent gmap page table (not the pgste) */
2097 * @saddr: faulting address in the shadow gmap
2098 * @pte: pte in parent gmap address space to get shadowed
2102 * -EFAULT if an address in the parent gmap could not be resolved.
2106 int gmap_shadow_page(struct gmap *sg, unsigned long saddr, pte_t pte)
2108 struct gmap *parent;
2170 * gmap_shadow_notify - handle notifications for shadow gmap
2174 static void gmap_shadow_notify(struct gmap *sg, unsigned long vmaddr,
2241 struct gmap *gmap, *sg, *next;
2246 list_for_each_entry_rcu(gmap, &mm->context.gmap_list, list) {
2247 spin_lock(&gmap->guest_table_lock);
2248 table = radix_tree_lookup(&gmap->host_to_guest,
2252 spin_unlock(&gmap->guest_table_lock);
2256 if (!list_empty(&gmap->children) && (bits & PGSTE_VSIE_BIT)) {
2257 spin_lock(&gmap->shadow_lock);
2259 &gmap->children, list)
2261 spin_unlock(&gmap->shadow_lock);
2264 gmap_call_notifier(gmap, gaddr, gaddr + PAGE_SIZE - 1);
2270 static void pmdp_notify_gmap(struct gmap *gmap, pmd_t *pmdp,
2274 gmap_call_notifier(gmap, gaddr, gaddr + HPAGE_SIZE - 1);
2278 * gmap_pmdp_xchg - exchange a gmap pmd with another
2279 * @gmap: pointer to the guest address space structure
2287 static void gmap_pmdp_xchg(struct gmap *gmap, pmd_t *pmdp, pmd_t new,
2291 pmdp_notify_gmap(gmap, pmdp, gaddr);
2294 __pmdp_idte(gaddr, (pmd_t *)pmdp, IDTE_GUEST_ASCE, gmap->asce,
2307 struct gmap *gmap;
2311 list_for_each_entry_rcu(gmap, &mm->context.gmap_list, list) {
2312 spin_lock(&gmap->guest_table_lock);
2313 pmdp = (pmd_t *)radix_tree_delete(&gmap->host_to_guest,
2317 pmdp_notify_gmap(gmap, pmdp, gaddr);
2324 spin_unlock(&gmap->guest_table_lock);
2360 struct gmap *gmap;
2364 list_for_each_entry_rcu(gmap, &mm->context.gmap_list, list) {
2365 spin_lock(&gmap->guest_table_lock);
2366 entry = radix_tree_delete(&gmap->host_to_guest,
2371 pmdp_notify_gmap(gmap, pmdp, gaddr);
2376 gmap->asce, IDTE_LOCAL);
2381 spin_unlock(&gmap->guest_table_lock);
2395 struct gmap *gmap;
2399 list_for_each_entry_rcu(gmap, &mm->context.gmap_list, list) {
2400 spin_lock(&gmap->guest_table_lock);
2401 entry = radix_tree_delete(&gmap->host_to_guest,
2406 pmdp_notify_gmap(gmap, pmdp, gaddr);
2411 gmap->asce, IDTE_GLOBAL);
2418 spin_unlock(&gmap->guest_table_lock);
2426 * @gmap: pointer to guest address space
2433 static bool gmap_test_and_clear_dirty_pmd(struct gmap *gmap, pmd_t *pmdp,
2446 gmap_protect_pmd(gmap, gaddr, pmdp, PROT_READ, 0);
2452 * @gmap: pointer to guest address space
2460 void gmap_sync_dirty_log_pmd(struct gmap *gmap, unsigned long bitmap[4],
2468 pmdp = gmap_pmd_op_walk(gmap, gaddr);
2473 if (gmap_test_and_clear_dirty_pmd(gmap, pmdp, gaddr))
2477 ptep = pte_alloc_map_lock(gmap->mm, pmdp, vmaddr, &ptl);
2480 if (ptep_test_and_clear_uc(gmap->mm, vmaddr, ptep))
2485 gmap_pmd_op_end(gmap, pmdp);
2727 * list of page tables of the gmap.
2728 * @gmap: the gmap whose table is to be removed
2731 * gmap (the CRST list). This list is used at tear down time to free all
2742 void s390_unlist_old_asce(struct gmap *gmap)
2746 old = virt_to_page(gmap->table);
2747 spin_lock(&gmap->guest_table_lock);
2764 spin_unlock(&gmap->guest_table_lock);
2769 * s390_replace_asce - Try to replace the current ASCE of a gmap with a copy
2770 * @gmap: the gmap whose ASCE needs to be replaced
2774 * In any case, the old ASCE is always removed from the gmap CRST list.
2778 int s390_replace_asce(struct gmap *gmap)
2784 s390_unlist_old_asce(gmap);
2791 memcpy(table, gmap->table, 1UL << (CRST_ALLOC_ORDER + PAGE_SHIFT));
2798 spin_lock(&gmap->guest_table_lock);
2799 list_add(&page->lru, &gmap->crst_list);
2800 spin_unlock(&gmap->guest_table_lock);
2803 asce = (gmap->asce & ~_ASCE_ORIGIN) | __pa(table);
2804 WRITE_ONCE(gmap->asce, asce);
2805 WRITE_ONCE(gmap->mm->context.gmap_asce, asce);
2806 WRITE_ONCE(gmap->table, table);