Lines Matching refs:sg
241 struct gmap *sg, *next;
247 list_for_each_entry_safe(sg, next, &gmap->children, list) {
248 list_del(&sg->list);
249 gmap_put(sg);
957 * Expected to be called with sg->mm->mmap_lock in read and
1003 * Expected to be called with sg->mm->mmap_lock in read
1039 * Called with sg->mm->mmap_lock in read.
1172 * @sg: pointer to the shadow guest address space structure
1176 * Called with the sg->guest_table_lock
1178 static inline void gmap_insert_rmap(struct gmap *sg, unsigned long vmaddr,
1183 BUG_ON(!gmap_is_shadow(sg));
1184 slot = radix_tree_lookup_slot(&sg->host_to_rmap, vmaddr >> PAGE_SHIFT);
1187 &sg->guest_table_lock);
1188 radix_tree_replace_slot(&sg->host_to_rmap, slot, rmap);
1191 radix_tree_insert(&sg->host_to_rmap, vmaddr >> PAGE_SHIFT,
1198 * @sg: pointer to the shadow guest address space structure
1206 static int gmap_protect_rmap(struct gmap *sg, unsigned long raddr,
1216 BUG_ON(!gmap_is_shadow(sg));
1217 parent = sg->parent;
1234 spin_lock(&sg->guest_table_lock);
1238 gmap_insert_rmap(sg, vmaddr, rmap);
1239 spin_unlock(&sg->guest_table_lock);
1282 * @sg: pointer to the shadow guest address space structure
1285 * Called with the sg->guest_table_lock
1287 static void gmap_unshadow_page(struct gmap *sg, unsigned long raddr)
1291 BUG_ON(!gmap_is_shadow(sg));
1292 table = gmap_table_walk(sg, raddr, 0); /* get page table pointer */
1295 gmap_call_notifier(sg, raddr, raddr + _PAGE_SIZE - 1);
1296 ptep_unshadow_pte(sg->mm, raddr, (pte_t *) table);
1301 * @sg: pointer to the shadow guest address space structure
1305 * Called with the sg->guest_table_lock
1307 static void __gmap_unshadow_pgt(struct gmap *sg, unsigned long raddr,
1312 BUG_ON(!gmap_is_shadow(sg));
1319 * @sg: pointer to the shadow guest address space structure
1322 * Called with the sg->guest_table_lock
1324 static void gmap_unshadow_pgt(struct gmap *sg, unsigned long raddr)
1329 BUG_ON(!gmap_is_shadow(sg));
1330 ste = gmap_table_walk(sg, raddr, 1); /* get segment pointer */
1333 gmap_call_notifier(sg, raddr, raddr + _SEGMENT_SIZE - 1);
1338 __gmap_unshadow_pgt(sg, raddr, pgt);
1347 * @sg: pointer to the shadow guest address space structure
1351 * Called with the sg->guest_table_lock
1353 static void __gmap_unshadow_sgt(struct gmap *sg, unsigned long raddr,
1360 BUG_ON(!gmap_is_shadow(sg));
1366 __gmap_unshadow_pgt(sg, raddr, pgt);
1376 * @sg: pointer to the shadow guest address space structure
1381 static void gmap_unshadow_sgt(struct gmap *sg, unsigned long raddr)
1386 BUG_ON(!gmap_is_shadow(sg));
1387 r3e = gmap_table_walk(sg, raddr, 2); /* get region-3 pointer */
1390 gmap_call_notifier(sg, raddr, raddr + _REGION3_SIZE - 1);
1395 __gmap_unshadow_sgt(sg, raddr, sgt);
1404 * @sg: pointer to the shadow guest address space structure
1408 * Called with the sg->guest_table_lock
1410 static void __gmap_unshadow_r3t(struct gmap *sg, unsigned long raddr,
1417 BUG_ON(!gmap_is_shadow(sg));
1423 __gmap_unshadow_sgt(sg, raddr, sgt);
1433 * @sg: pointer to the shadow guest address space structure
1436 * Called with the sg->guest_table_lock
1438 static void gmap_unshadow_r3t(struct gmap *sg, unsigned long raddr)
1443 BUG_ON(!gmap_is_shadow(sg));
1444 r2e = gmap_table_walk(sg, raddr, 3); /* get region-2 pointer */
1447 gmap_call_notifier(sg, raddr, raddr + _REGION2_SIZE - 1);
1452 __gmap_unshadow_r3t(sg, raddr, r3t);
1461 * @sg: pointer to the shadow guest address space structure
1465 * Called with the sg->guest_table_lock
1467 static void __gmap_unshadow_r2t(struct gmap *sg, unsigned long raddr,
1474 BUG_ON(!gmap_is_shadow(sg));
1480 __gmap_unshadow_r3t(sg, raddr, r3t);
1490 * @sg: pointer to the shadow guest address space structure
1493 * Called with the sg->guest_table_lock
1495 static void gmap_unshadow_r2t(struct gmap *sg, unsigned long raddr)
1500 BUG_ON(!gmap_is_shadow(sg));
1501 r1e = gmap_table_walk(sg, raddr, 4); /* get region-1 pointer */
1504 gmap_call_notifier(sg, raddr, raddr + _REGION1_SIZE - 1);
1509 __gmap_unshadow_r2t(sg, raddr, r2t);
1518 * @sg: pointer to the shadow guest address space structure
1524 static void __gmap_unshadow_r1t(struct gmap *sg, unsigned long raddr,
1531 BUG_ON(!gmap_is_shadow(sg));
1537 __gmap_unshadow_r2t(sg, raddr, r2t);
1550 * @sg: pointer to the shadow guest address space structure
1552 * Called with sg->guest_table_lock
1554 static void gmap_unshadow(struct gmap *sg)
1558 BUG_ON(!gmap_is_shadow(sg));
1559 if (sg->removed)
1561 sg->removed = 1;
1562 gmap_call_notifier(sg, 0, -1UL);
1563 gmap_flush_tlb(sg);
1564 table = (unsigned long *)(sg->asce & _ASCE_ORIGIN);
1565 switch (sg->asce & _ASCE_TYPE_MASK) {
1567 __gmap_unshadow_r1t(sg, 0, table);
1570 __gmap_unshadow_r2t(sg, 0, table);
1573 __gmap_unshadow_r3t(sg, 0, table);
1576 __gmap_unshadow_sgt(sg, 0, table);
1594 struct gmap *sg;
1596 list_for_each_entry(sg, &parent->children, list) {
1597 if (sg->orig_asce != asce || sg->edat_level != edat_level ||
1598 sg->removed)
1600 if (!sg->initialized)
1602 refcount_inc(&sg->ref_count);
1603 return sg;
1611 * @sg: pointer to the shadow guest address space structure
1620 int gmap_shadow_valid(struct gmap *sg, unsigned long asce, int edat_level)
1622 if (sg->removed)
1624 return sg->orig_asce == asce && sg->edat_level == edat_level;
1646 struct gmap *sg, *new;
1653 sg = gmap_find_shadow(parent, asce, edat_level);
1655 if (sg)
1656 return sg;
1671 sg = gmap_find_shadow(parent, asce, edat_level);
1672 if (sg) {
1675 return sg;
1679 list_for_each_entry(sg, &parent->children, list) {
1680 if (sg->orig_asce & _ASCE_REAL_SPACE) {
1681 spin_lock(&sg->guest_table_lock);
1682 gmap_unshadow(sg);
1683 spin_unlock(&sg->guest_table_lock);
1684 list_del(&sg->list);
1685 gmap_put(sg);
1719 * @sg: pointer to the shadow guest address space structure
1733 * Called with sg->mm->mmap_lock in read.
1735 int gmap_shadow_r2t(struct gmap *sg, unsigned long saddr, unsigned long r2t,
1743 BUG_ON(!gmap_is_shadow(sg));
1753 spin_lock(&sg->guest_table_lock);
1754 table = gmap_table_walk(sg, saddr, 4); /* get region-1 pointer */
1770 if (sg->edat_level >= 1)
1772 list_add(&page->lru, &sg->crst_list);
1776 spin_unlock(&sg->guest_table_lock);
1779 spin_unlock(&sg->guest_table_lock);
1785 rc = gmap_protect_rmap(sg, raddr, origin + offset, len);
1786 spin_lock(&sg->guest_table_lock);
1788 table = gmap_table_walk(sg, saddr, 4);
1795 gmap_unshadow_r2t(sg, raddr);
1797 spin_unlock(&sg->guest_table_lock);
1800 spin_unlock(&sg->guest_table_lock);
1808 * @sg: pointer to the shadow guest address space structure
1817 * Called with sg->mm->mmap_lock in read.
1819 int gmap_shadow_r3t(struct gmap *sg, unsigned long saddr, unsigned long r3t,
1827 BUG_ON(!gmap_is_shadow(sg));
1837 spin_lock(&sg->guest_table_lock);
1838 table = gmap_table_walk(sg, saddr, 3); /* get region-2 pointer */
1854 if (sg->edat_level >= 1)
1856 list_add(&page->lru, &sg->crst_list);
1860 spin_unlock(&sg->guest_table_lock);
1863 spin_unlock(&sg->guest_table_lock);
1869 rc = gmap_protect_rmap(sg, raddr, origin + offset, len);
1870 spin_lock(&sg->guest_table_lock);
1872 table = gmap_table_walk(sg, saddr, 3);
1879 gmap_unshadow_r3t(sg, raddr);
1881 spin_unlock(&sg->guest_table_lock);
1884 spin_unlock(&sg->guest_table_lock);
1892 * @sg: pointer to the shadow guest address space structure
1901 * Called with sg->mm->mmap_lock in read.
1903 int gmap_shadow_sgt(struct gmap *sg, unsigned long saddr, unsigned long sgt,
1911 BUG_ON(!gmap_is_shadow(sg) || (sgt & _REGION3_ENTRY_LARGE));
1921 spin_lock(&sg->guest_table_lock);
1922 table = gmap_table_walk(sg, saddr, 2); /* get region-3 pointer */
1938 if (sg->edat_level >= 1)
1940 list_add(&page->lru, &sg->crst_list);
1944 spin_unlock(&sg->guest_table_lock);
1947 spin_unlock(&sg->guest_table_lock);
1953 rc = gmap_protect_rmap(sg, raddr, origin + offset, len);
1954 spin_lock(&sg->guest_table_lock);
1956 table = gmap_table_walk(sg, saddr, 2);
1963 gmap_unshadow_sgt(sg, raddr);
1965 spin_unlock(&sg->guest_table_lock);
1968 spin_unlock(&sg->guest_table_lock);
1976 * @sg: pointer to the shadow guest address space structure
1985 * Called with sg->mm->mmap_lock in read.
1987 int gmap_shadow_pgt_lookup(struct gmap *sg, unsigned long saddr,
1995 BUG_ON(!gmap_is_shadow(sg));
1996 spin_lock(&sg->guest_table_lock);
1997 table = gmap_table_walk(sg, saddr, 1); /* get segment pointer */
2008 spin_unlock(&sg->guest_table_lock);
2016 * @sg: pointer to the shadow guest address space structure
2027 int gmap_shadow_pgt(struct gmap *sg, unsigned long saddr, unsigned long pgt,
2035 BUG_ON(!gmap_is_shadow(sg) || (pgt & _SEGMENT_ENTRY_LARGE));
2037 page = page_table_alloc_pgste(sg->mm);
2045 spin_lock(&sg->guest_table_lock);
2046 table = gmap_table_walk(sg, saddr, 1); /* get segment pointer */
2061 list_add(&page->lru, &sg->pt_list);
2065 spin_unlock(&sg->guest_table_lock);
2068 spin_unlock(&sg->guest_table_lock);
2072 rc = gmap_protect_rmap(sg, raddr, origin, PAGE_SIZE);
2073 spin_lock(&sg->guest_table_lock);
2075 table = gmap_table_walk(sg, saddr, 1);
2082 gmap_unshadow_pgt(sg, raddr);
2084 spin_unlock(&sg->guest_table_lock);
2087 spin_unlock(&sg->guest_table_lock);
2096 * @sg: pointer to the shadow guest address space structure
2104 * Called with sg->mm->mmap_lock in read.
2106 int gmap_shadow_page(struct gmap *sg, unsigned long saddr, pte_t pte)
2116 BUG_ON(!gmap_is_shadow(sg));
2117 parent = sg->parent;
2138 spin_lock(&sg->guest_table_lock);
2140 tptep = (pte_t *) gmap_table_walk(sg, saddr, 0);
2142 spin_unlock(&sg->guest_table_lock);
2147 rc = ptep_shadow_pte(sg->mm, saddr, sptep, tptep, pte);
2150 gmap_insert_rmap(sg, vmaddr, rmap);
2155 spin_unlock(&sg->guest_table_lock);
2172 * Called with sg->parent->shadow_lock.
2174 static void gmap_shadow_notify(struct gmap *sg, unsigned long vmaddr,
2180 BUG_ON(!gmap_is_shadow(sg));
2182 spin_lock(&sg->guest_table_lock);
2183 if (sg->removed) {
2184 spin_unlock(&sg->guest_table_lock);
2188 start = sg->orig_asce & _ASCE_ORIGIN;
2189 end = start + ((sg->orig_asce & _ASCE_TABLE_LENGTH) + 1) * PAGE_SIZE;
2190 if (!(sg->orig_asce & _ASCE_REAL_SPACE) && gaddr >= start &&
2193 gmap_unshadow(sg);
2194 spin_unlock(&sg->guest_table_lock);
2195 list_del(&sg->list);
2196 gmap_put(sg);
2200 head = radix_tree_delete(&sg->host_to_rmap, vmaddr >> PAGE_SHIFT);
2206 gmap_unshadow_r2t(sg, raddr);
2209 gmap_unshadow_r3t(sg, raddr);
2212 gmap_unshadow_sgt(sg, raddr);
2215 gmap_unshadow_pgt(sg, raddr);
2218 gmap_unshadow_page(sg, raddr);
2223 spin_unlock(&sg->guest_table_lock);
2241 struct gmap *gmap, *sg, *next;
2258 list_for_each_entry_safe(sg, next,
2260 gmap_shadow_notify(sg, vmaddr, gaddr);