Lines Matching refs:page
46 * Defrag is invoked by khugepaged hugepage allocations and by page faults
63 struct page *huge_zero_page __read_mostly;
90 static struct page *get_huge_zero_page(void)
92 struct page *zero_page;
127 struct page *mm_get_huge_zero_page(struct mm_struct *mm)
150 /* we can free zero page only if last reference remains */
158 struct page *zero_page = xchg(&huge_zero_page, NULL);
400 * we use page->mapping and page->index in second tail page
487 static inline struct deferred_split *get_deferred_split_queue(struct page *page)
489 struct mem_cgroup *memcg = compound_head(page)->mem_cgroup;
490 struct pglist_data *pgdat = NODE_DATA(page_to_nid(page));
498 static inline struct deferred_split *get_deferred_split_queue(struct page *page)
500 struct pglist_data *pgdat = NODE_DATA(page_to_nid(page));
506 void prep_transhuge_page(struct page *page)
509 * we use page->mapping and page->indexlru in second tail page
513 INIT_LIST_HEAD(page_deferred_list(page));
514 set_compound_page_dtor(page, TRANSHUGE_PAGE_DTOR);
517 bool is_transparent_hugepage(struct page *page)
519 if (!PageCompound(page))
522 page = compound_head(page);
523 return is_huge_zero_page(page) ||
524 page[1].compound_dtor == TRANSHUGE_PAGE_DTOR;
582 struct page *page, gfp_t gfp)
589 VM_BUG_ON_PAGE(!PageCompound(page), page);
591 if (mem_cgroup_charge(page, vma->vm_mm, gfp)) {
592 put_page(page);
597 cgroup_throttle_swaprate(page, gfp);
605 clear_huge_page(page, vmf->address, HPAGE_PMD_NR);
611 __SetPageUptodate(page);
623 /* Deliver the page fault to userland */
628 put_page(page);
635 entry = mk_huge_pmd(page, vma->vm_page_prot);
637 page_add_new_anon_rmap(page, vma, haddr, true);
638 lru_cache_add_inactive_or_unevictable(page, vma);
654 put_page(page);
694 /* Caller must hold page table lock. */
697 struct page *zero_page)
715 struct page *page;
728 struct page *zero_page;
763 page = alloc_hugepage_vma(gfp, vma, haddr, HPAGE_PMD_ORDER);
764 if (unlikely(!page)) {
768 prep_transhuge_page(page);
769 return __do_huge_pmd_anonymous_page(vmf, page, gfp);
823 * @pgprot: page protection to use
914 * @pgprot: page protection to use
964 struct page *follow_devmap_pmd(struct vm_area_struct *vma, unsigned long addr,
969 struct page *page;
997 * caller will manage the page reference count.
1006 page = pfn_to_page(pfn);
1007 if (!try_grab_page(page, flags))
1008 page = ERR_PTR(-ENOMEM);
1010 return page;
1018 struct page *src_page;
1068 * When page table lock is held, the huge zero pmd should not be
1069 * under splitting since we don't split the page itself, only pmd to
1070 * a page table.
1074 * get_huge_zero_page() will never allocate a new page here,
1075 * since we already have a zero page to copy. It just takes a
1086 * If this page is a potentially pinned page, split and retry the fault
1087 * with smaller page size. Normally this should not happen because the
1090 * random page during the coming copy-on-write.
1136 struct page *follow_devmap_pud(struct vm_area_struct *vma, unsigned long addr,
1141 struct page *page;
1163 * caller will manage the page reference count.
1174 page = pfn_to_page(pfn);
1175 if (!try_grab_page(page, flags))
1176 page = ERR_PTR(-ENOMEM);
1178 return page;
1199 * When page table lock is held, the huge zero pud should not be
1200 * under splitting since we don't split the page itself, only pud to
1201 * a page table.
1274 struct page *page;
1290 page = pmd_page(orig_pmd);
1291 VM_BUG_ON_PAGE(!PageCompound(page) || !PageHead(page), page);
1293 /* Lock page for reuse_swap_page() */
1294 if (!trylock_page(page)) {
1295 get_page(page);
1297 lock_page(page);
1301 unlock_page(page);
1302 put_page(page);
1305 put_page(page);
1309 * We can only reuse the page if nobody else maps the huge page or it's
1312 if (reuse_swap_page(page, NULL)) {
1318 unlock_page(page);
1323 unlock_page(page);
1340 struct page *follow_trans_huge_pmd(struct vm_area_struct *vma,
1346 struct page *page = NULL;
1353 /* Avoid dumping huge zero page */
1361 page = pmd_page(*pmd);
1362 VM_BUG_ON_PAGE(!PageHead(page) && !is_zone_device_page(page), page);
1364 if (!try_grab_page(page, flags))
1377 * In most cases the pmd is the only mapping of the page as we
1381 * The only scenario when we have the page shared here is if we
1387 * We can expect PageDoubleMap() to be stable under page lock:
1389 * requires page to be locked.
1392 if (PageAnon(page) && compound_mapcount(page) != 1)
1394 if (PageDoubleMap(page) || !page->mapping)
1396 if (!trylock_page(page))
1398 if (page->mapping && !PageDoubleMap(page))
1399 mlock_vma_page(page);
1400 unlock_page(page);
1403 page += (addr & ~HPAGE_PMD_MASK) >> PAGE_SHIFT;
1404 VM_BUG_ON_PAGE(!PageCompound(page) && !is_zone_device_page(page), page);
1407 return page;
1410 /* NUMA hinting page fault entry point for trans huge pmds */
1415 struct page *page;
1431 * check_same as the page may no longer be mapped.
1434 page = pmd_page(*vmf->pmd);
1435 if (!get_page_unless_zero(page))
1438 put_and_wait_on_page_locked(page);
1442 page = pmd_page(pmd);
1443 BUG_ON(is_huge_zero_page(page));
1444 page_nid = page_to_nid(page);
1445 last_cpupid = page_cpupid_last(page);
1457 * Acquire the page lock to serialise THP migrations but avoid dropping
1460 page_locked = trylock_page(page);
1461 target_nid = mpol_misplaced(page, vma, haddr);
1463 /* If the page was locked, there are no parallel migrations */
1471 if (!get_page_unless_zero(page))
1474 put_and_wait_on_page_locked(page);
1482 get_page(page);
1484 anon_vma = page_lock_anon_vma_read(page);
1489 unlock_page(page);
1490 put_page(page);
1497 put_page(page);
1505 * modifying the page we're about to migrate.
1510 * We are not sure a pending tlb flush here is for a huge page
1529 * Migrate the THP to the requested node, returns with page unlocked
1535 vmf->pmd, pmd, vmf->address, page, target_nid);
1544 BUG_ON(!PageLocked(page));
1552 unlock_page(page);
1568 * Return true if we do MADV_FREE successfully on entire pmd page.
1576 struct page *page;
1596 page = pmd_page(orig_pmd);
1598 * If other processes are mapping this page, we couldn't discard
1599 * the page unless they all do MADV_FREE so let's skip the page.
1601 if (total_mapcount(page) != 1)
1604 if (!trylock_page(page))
1612 get_page(page);
1614 split_huge_page(page);
1615 unlock_page(page);
1616 put_page(page);
1620 if (PageDirty(page))
1621 ClearPageDirty(page);
1622 unlock_page(page);
1633 mark_page_lazyfree(page);
1681 struct page *page = NULL;
1685 page = pmd_page(orig_pmd);
1686 page_remove_rmap(page, true);
1687 VM_BUG_ON_PAGE(page_mapcount(page) < 0, page);
1688 VM_BUG_ON_PAGE(!PageHead(page), page);
1694 page = migration_entry_to_page(entry);
1699 if (PageAnon(page)) {
1705 add_mm_counter(tlb->mm, mm_counter_file(page), -HPAGE_PMD_NR);
1710 tlb_remove_page_size(tlb, page, HPAGE_PMD_SIZE);
1722 * PTE page table if new_pmd is on different PMD page table.
1838 * Avoid trapping faults against the zero page. The read-only
1840 * local/remote hits to the zero page are not interesting.
1894 * Returns page table lock pointer if a given pmd maps a thp, NULL otherwise.
1896 * Note that if it returns page table lock pointer, this routine returns without
1897 * unlocking page table lock. So callers must unlock it.
1913 * Note that if it returns true, this routine returns without unlocking page
1946 /* No zero page support yet */
2003 * replacing a zero pmd write protected page with a zero pte write
2004 * protected page.
2032 struct page *page;
2050 * We are going to unmap this huge page. So
2061 page = migration_entry_to_page(entry);
2063 page = pmd_page(old_pmd);
2064 if (!PageDirty(page) && pmd_dirty(old_pmd))
2065 set_page_dirty(page);
2066 if (!PageReferenced(page) && pmd_young(old_pmd))
2067 SetPageReferenced(page);
2068 page_remove_rmap(page, true);
2069 put_page(page);
2071 add_mm_counter(mm, mm_counter_file(page), -HPAGE_PMD_NR);
2081 * We are going from a zero huge page write protected to zero
2082 * small page also write protected so it does not seems useful
2093 * free), userland could trigger a small page size TLB miss on the
2097 * 383 on page 105. Intel should be safe but is also warns that it's
2115 page = migration_entry_to_page(entry);
2121 page = pmd_page(old_pmd);
2123 SetPageDirty(page);
2129 VM_BUG_ON_PAGE(!page_count(page), page);
2130 page_ref_add(page, HPAGE_PMD_NR - 1);
2148 swp_entry = make_migration_entry(page + i, write);
2155 entry = mk_pte(page + i, READ_ONCE(vma->vm_page_prot));
2170 atomic_inc(&page[i]._mapcount);
2179 if (compound_mapcount(page) > 1 &&
2180 !TestSetPageDoubleMap(page)) {
2182 atomic_inc(&page[i]._mapcount);
2185 lock_page_memcg(page);
2186 if (atomic_add_negative(-1, compound_mapcount_ptr(page))) {
2188 __dec_lruvec_page_state(page, NR_ANON_THPS);
2189 if (TestClearPageDoubleMap(page)) {
2192 atomic_dec(&page[i]._mapcount);
2195 unlock_page_memcg(page);
2203 page_remove_rmap(page + i, false);
2204 put_page(page + i);
2210 unsigned long address, bool freeze, struct page *page)
2224 * If caller asks to setup a migration entries, we need a page to check
2225 * pmd against. Otherwise we can end up replacing wrong page.
2227 VM_BUG_ON(freeze && !page);
2228 if (page) {
2229 VM_WARN_ON_ONCE(!PageLocked(page));
2230 if (page != pmd_page(*pmd))
2236 if (!page) {
2237 page = pmd_page(*pmd);
2239 * An anonymous page must be locked, to ensure that a
2242 * and page lock must not be taken when zap_pmd_range()
2245 if (PageAnon(page)) {
2246 if (unlikely(!trylock_page(page))) {
2247 get_page(page);
2250 lock_page(page);
2253 unlock_page(page);
2254 put_page(page);
2255 page = NULL;
2258 put_page(page);
2263 if (PageMlocked(page))
2264 clear_page_mlock(page);
2271 unlock_page(page);
2276 * 2) __split_huge_zero_page_pmd() read only zero page and any write
2277 * fault will trigger a flush_notify before pointing to a new page
2279 * page in the meantime)
2280 * 3) Split a huge pmd into pte pointing to the same page. No need
2289 bool freeze, struct page *page)
2310 __split_huge_pmd(vma, pmd, address, freeze, page);
2354 static void unmap_page(struct page *page)
2359 VM_BUG_ON_PAGE(!PageHead(page), page);
2361 if (PageAnon(page))
2364 try_to_unmap(page, ttu_flags);
2366 VM_WARN_ON_ONCE_PAGE(page_mapped(page), page);
2369 static void remap_page(struct page *page, unsigned int nr)
2372 if (PageTransHuge(page)) {
2373 remove_migration_ptes(page, page, true);
2376 remove_migration_ptes(page + i, page + i, true);
2380 static void __split_huge_page_tail(struct page *head, int tail,
2383 struct page *page_tail = head + tail;
2388 * Clone page flags before unfreezing refcount.
2409 /* ->mapping in first tail page is compound_mapcount */
2415 /* Page flags must be visible before we make the page non-compound. */
2419 * Clear PageTail before unfreezing page refcount.
2426 /* Finally unfreeze refcount. Additional reference from page cache. */
2445 static void __split_huge_page(struct page *page, struct list_head *list,
2448 struct page *head = compound_head(page);
2471 /* Some pages can be beyond i_size: drop them from page cache */
2478 } else if (!PageAnon(page)) {
2501 /* Additional pin to page cache */
2517 struct page *subpage = head + i;
2518 if (subpage == page)
2524 * like if add_to_swap() is running on a lru page that
2533 int total_mapcount(struct page *page)
2537 VM_BUG_ON_PAGE(PageTail(page), page);
2539 if (likely(!PageCompound(page)))
2540 return atomic_read(&page->_mapcount) + 1;
2542 compound = compound_mapcount(page);
2543 nr = compound_nr(page);
2544 if (PageHuge(page))
2548 ret += atomic_read(&page[i]._mapcount) + 1;
2550 if (!PageAnon(page))
2552 if (PageDoubleMap(page))
2561 * reuse the page and change the mapping to read-write instead of
2578 * need full accuracy to avoid breaking page pinning, because
2581 int page_trans_huge_mapcount(struct page *page, int *total_mapcount)
2586 VM_BUG_ON_PAGE(PageHuge(page), page);
2588 if (likely(!PageTransCompound(page))) {
2589 mapcount = atomic_read(&page->_mapcount) + 1;
2595 page = compound_head(page);
2598 for (i = 0; i < thp_nr_pages(page); i++) {
2599 mapcount = atomic_read(&page[i]._mapcount) + 1;
2603 if (PageDoubleMap(page)) {
2605 _total_mapcount -= thp_nr_pages(page);
2607 mapcount = compound_mapcount(page);
2615 /* Racy check whether the huge page can be split */
2616 bool can_split_huge_page(struct page *page, int *pextra_pins)
2620 /* Additional pins from page cache */
2621 if (PageAnon(page))
2622 extra_pins = PageSwapCache(page) ? thp_nr_pages(page) : 0;
2624 extra_pins = thp_nr_pages(page);
2627 return total_mapcount(page) == page_count(page) - extra_pins - 1;
2631 * This function splits huge page into normal pages. @page can point to any
2632 * subpage of huge page to split. Split doesn't change the position of @page.
2634 * Only caller must hold pin on the @page, otherwise split fails with -EBUSY.
2635 * The huge page must be locked.
2639 * Both head page and tail pages will inherit mapping, flags, and so on from
2642 * GUP pin and PG_locked transferred to @page. Rest subpages can be freed if
2646 * Returns -EBUSY if the page is pinned or if anon_vma disappeared from under
2649 int split_huge_page_to_list(struct page *page, struct list_head *list)
2651 struct page *head = compound_head(page);
2699 * which cannot be nested inside the page tree lock. So note
2701 * head page lock is good enough to serialize the trimming.
2707 * Racy check if we can split the page, before unmap_page() will
2724 * Check if the head page is present in page cache.
2747 __split_huge_page(page, list, end, flags);
2771 void free_transhuge_page(struct page *page)
2773 struct deferred_split *ds_queue = get_deferred_split_queue(page);
2777 if (!list_empty(page_deferred_list(page))) {
2779 list_del(page_deferred_list(page));
2782 free_compound_page(page);
2785 void deferred_split_huge_page(struct page *page)
2787 struct deferred_split *ds_queue = get_deferred_split_queue(page);
2789 struct mem_cgroup *memcg = compound_head(page)->mem_cgroup;
2793 VM_BUG_ON_PAGE(!PageTransHuge(page), page);
2796 * The try_to_unmap() in page reclaim path might reach here too,
2798 * And, if page reclaim is already handling the same page, it is
2801 * Check PageSwapCache to determine if the page is being
2802 * handled by page reclaim since THP swap would add the page into
2805 if (PageSwapCache(page))
2808 if (!list_empty(page_deferred_list(page)))
2812 if (list_empty(page_deferred_list(page))) {
2814 list_add_tail(page_deferred_list(page), &ds_queue->split_queue);
2818 memcg_set_shrinker_bit(memcg, page_to_nid(page),
2845 struct page *page;
2856 page = list_entry((void *)pos, struct page, mapping);
2857 page = compound_head(page);
2858 if (get_page_unless_zero(page)) {
2859 list_move(page_deferred_list(page), &list);
2862 list_del_init(page_deferred_list(page));
2871 page = list_entry((void *)pos, struct page, mapping);
2872 if (!trylock_page(page))
2874 /* split_huge_page() removes page from list on success */
2875 if (!split_huge_page(page))
2877 unlock_page(page);
2879 put_page(page);
2887 * Stop shrinker if we didn't split any page, but the queue is empty.
2907 struct page *page;
2920 page = pfn_to_page(pfn);
2921 if (!get_page_unless_zero(page))
2924 if (zone != page_zone(page))
2927 if (!PageHead(page) || PageHuge(page) || !PageLRU(page))
2931 lock_page(page);
2932 if (!split_huge_page(page))
2934 unlock_page(page);
2936 put_page(page);
2958 struct page *page)
2973 set_page_dirty(page);
2974 entry = make_migration_entry(page, pmd_write(pmdval));
2979 page_remove_rmap(page, true);
2980 put_page(page);
2983 void remove_migration_pmd(struct page_vma_mapped_walk *pvmw, struct page *new)