Lines Matching refs:page
31 static void hpage_pincount_add(struct page *page, int refs)
33 VM_BUG_ON_PAGE(!hpage_pincount_available(page), page);
34 VM_BUG_ON_PAGE(page != compound_head(page), page);
36 atomic_add(refs, compound_pincount_ptr(page));
39 static void hpage_pincount_sub(struct page *page, int refs)
41 VM_BUG_ON_PAGE(!hpage_pincount_available(page), page);
42 VM_BUG_ON_PAGE(page != compound_head(page), page);
44 atomic_sub(refs, compound_pincount_ptr(page));
48 static void put_page_refs(struct page *page, int refs)
51 if (VM_WARN_ON_ONCE_PAGE(page_ref_count(page) < refs, page))
60 page_ref_sub(page, refs - 1);
61 put_page(page);
65 * Return the compound head page with ref appropriately incremented,
68 static inline struct page *try_get_compound_head(struct page *page, int refs)
70 struct page *head = compound_head(page);
78 * At this point we have a stable reference to the head page; but it
80 * increment, the compound page was split, in which case we'd end up
81 * holding a reference on a page that has nothing to do with the page
83 * So now that the head page is stable, recheck that the pages still
86 if (unlikely(compound_head(page) != head)) {
95 * try_grab_compound_head() - attempt to elevate a page's refcount, by a
99 * FOLL_PIN or FOLL_GET behavior, when incrementing the page's refcount.
105 * FOLL_GET: page's refcount will be incremented by 1.
106 * FOLL_PIN: page's refcount will be incremented by GUP_PIN_COUNTING_BIAS.
108 * Return: head page (with refcount appropriately incremented) for success, or
113 static __maybe_unused struct page *try_grab_compound_head(struct page *page,
118 return try_get_compound_head(page, refs);
127 is_migrate_cma_page(page))
131 * CAUTION: Don't use compound_head() on the page before this
134 page = try_get_compound_head(page, refs);
135 if (!page)
139 * When pinning a compound page of order > 1 (which is what
143 * However, be sure to *also* increment the normal page refcount
144 * field at least once, so that the page really is pinned.
146 if (hpage_pincount_available(page))
147 hpage_pincount_add(page, refs);
149 page_ref_add(page, refs * (GUP_PIN_COUNTING_BIAS - 1));
151 mod_node_page_state(page_pgdat(page), NR_FOLL_PIN_ACQUIRED,
154 return page;
161 static void put_compound_head(struct page *page, int refs, unsigned int flags)
164 mod_node_page_state(page_pgdat(page), NR_FOLL_PIN_RELEASED,
167 if (hpage_pincount_available(page))
168 hpage_pincount_sub(page, refs);
173 put_page_refs(page, refs);
177 * try_grab_page() - elevate a page's refcount by a flag-dependent amount
182 * FOLL_PIN or FOLL_GET behavior, when incrementing the page's refcount.
184 * @page: pointer to page to be grabbed
190 * FOLL_GET: page's refcount will be incremented by 1.
191 * FOLL_PIN: page's refcount will be incremented by GUP_PIN_COUNTING_BIAS.
195 * FOLL_PIN was set, but the page could not be grabbed.
197 bool __must_check try_grab_page(struct page *page, unsigned int flags)
202 return try_get_page(page);
206 page = compound_head(page);
208 if (WARN_ON_ONCE(page_ref_count(page) <= 0))
211 if (hpage_pincount_available(page))
212 hpage_pincount_add(page, 1);
219 * *also* increment the normal page refcount field at least
220 * once, so that the page really is pinned.
222 page_ref_add(page, refs);
224 mod_node_page_state(page_pgdat(page), NR_FOLL_PIN_ACQUIRED, 1);
231 * unpin_user_page() - release a dma-pinned page
232 * @page: pointer to page to be released
239 void unpin_user_page(struct page *page)
241 put_compound_head(compound_head(page), 1, FOLL_PIN);
251 * "gup-pinned page" refers to a page that has had one of the get_user_pages()
252 * variants called on that page.
254 * For each page in the @pages array, make that page (or its head page, if a
255 * compound page) dirty, if @make_dirty is true, and if the page was previously
267 void unpin_user_pages_dirty_lock(struct page **pages, unsigned long npages,
274 * physically contiguous and part of the same compound page, then a
275 * single operation to the head page should suffice.
284 struct page *page = compound_head(pages[index]);
290 * 1) This code sees the page as already dirty, so it
294 * However, now the page is going to get written back,
297 * on to call TestClearPageDirty(), and write the page
300 * 2) This code sees the page as clean, so it calls
301 * set_page_dirty(). The page stays dirty, despite being
305 if (!PageDirty(page))
306 set_page_dirty_lock(page);
307 unpin_user_page(page);
317 * For each page in the @pages array, release the page using unpin_user_page().
321 void unpin_user_pages(struct page **pages, unsigned long npages)
334 * physically contiguous and part of the same compound page, then a
335 * single operation to the head page should suffice.
343 static struct page *no_page_table(struct vm_area_struct *vma,
349 * page tables. Return error instead of NULL to skip handle_mm_fault,
363 /* No page to get reference */
380 /* Proper page table entry exists, but no corresponding struct page */
394 static struct page *follow_page_pte(struct vm_area_struct *vma,
399 struct page *page;
414 page = follow_huge_pmd_pte(vma, address, flags);
415 if (page)
416 return page;
429 * KSM's break_ksm() relies upon recognizing a ksm page
451 page = vm_normal_page(vma, address, pte);
452 if (!page && pte_devmap(pte) && (flags & (FOLL_GET | FOLL_PIN))) {
460 page = pte_page(pte);
463 } else if (unlikely(!page)) {
466 page = ERR_PTR(-EFAULT);
471 page = pte_page(pte);
474 page = ERR_PTR(ret);
479 if (flags & FOLL_SPLIT && PageTransCompound(page)) {
480 get_page(page);
482 lock_page(page);
483 ret = split_huge_page(page);
484 unlock_page(page);
485 put_page(page);
492 if (unlikely(!try_grab_page(page, flags))) {
493 page = ERR_PTR(-ENOMEM);
497 * We need to make the page accessible if and only if we are going
502 ret = arch_make_page_accessible(page);
504 unpin_user_page(page);
505 page = ERR_PTR(ret);
511 !pte_dirty(pte) && !PageDirty(page))
512 set_page_dirty(page);
518 mark_page_accessed(page);
522 if (PageTransCompound(page))
530 * If the page is already locked, we don't need to
532 * when it attempts to reclaim the page.
534 if (page->mapping && trylock_page(page)) {
537 * Because we lock page here, and migration is
538 * blocked by the pte's page reference, and we
539 * know the page is still mapped, we don't even
540 * need to check for file-cache page truncation.
542 mlock_vma_page(page);
543 unlock_page(page);
548 return page;
556 static struct page *follow_pmd_mask(struct vm_area_struct *vma,
563 struct page *page;
575 page = follow_huge_pmd_pte(vma, address, flags);
576 if (page)
577 return page;
581 page = follow_huge_pd(vma, address,
584 if (page)
585 return page;
607 page = follow_devmap_pmd(vma, address, pmd, flags, &ctx->pgmap);
609 if (page)
610 return page;
637 page = pmd_page(*pmd);
638 if (is_huge_zero_page(page)) {
645 if (unlikely(!try_get_page(page))) {
650 lock_page(page);
651 ret = split_huge_page(page);
652 unlock_page(page);
653 put_page(page);
665 page = follow_trans_huge_pmd(vma, address, pmd, flags);
668 return page;
671 static struct page *follow_pud_mask(struct vm_area_struct *vma,
678 struct page *page;
685 page = follow_huge_pud(mm, address, pud, flags);
686 if (page)
687 return page;
691 page = follow_huge_pd(vma, address,
694 if (page)
695 return page;
700 page = follow_devmap_pud(vma, address, pud, flags, &ctx->pgmap);
702 if (page)
703 return page;
711 static struct page *follow_p4d_mask(struct vm_area_struct *vma,
717 struct page *page;
727 page = follow_huge_pd(vma, address,
730 if (page)
731 return page;
738 * follow_page_mask - look up a page descriptor from a user-virtual address
750 * On output, the @ctx->page_mask is set according to the size of the page.
752 * Return: the mapped (struct page *), %NULL if no mapping exists, or
754 * by a page descriptor (see also vm_normal_page()).
756 static struct page *follow_page_mask(struct vm_area_struct *vma,
761 struct page *page;
767 page = follow_huge_addr(mm, address, flags & FOLL_WRITE);
768 if (!IS_ERR(page)) {
770 return page;
779 page = follow_huge_pgd(mm, address, pgd, flags);
780 if (page)
781 return page;
785 page = follow_huge_pd(vma, address,
788 if (page)
789 return page;
796 struct page *follow_page(struct vm_area_struct *vma, unsigned long address,
800 struct page *page;
802 page = follow_page_mask(vma, address, foll_flags, &ctx);
805 return page;
810 struct page **page)
842 if (!page)
844 *page = vm_normal_page(*vma, address, *pte);
845 if (!*page) {
848 *page = pte_page(*pte);
850 if (unlikely(!try_grab_page(*page, gup_flags))) {
909 * can thus safely do subsequent page lookups as if they were reads.
911 * userspace may also be wanting to write to the gotten user page,
912 * which a read fault here might prevent (a readonly page might get
976 * @vmas: array of pointers to vmas corresponding to each page.
995 * __get_user_pages walks a process's page tables and takes a reference to
996 * each struct page that each user address corresponds to at a given
997 * instant. That is, it takes the page that would be accessed if a user
1000 * This does not guarantee that the page exists in the user mappings when
1002 * page there in some cases (eg. if mmapped pagecache has been invalidated
1003 * and subsequently re faulted). However it does guarantee that the page
1004 * won't be freed completely. And mostly callers simply care that the page
1009 * If @gup_flags & FOLL_WRITE == 0, the page must not be written to. If
1010 * the page is written to, set_page_dirty (or set_page_dirty_lock, as
1011 * appropriate) must be called after the page is finished with, and
1029 unsigned int gup_flags, struct page **pages,
1052 struct page *page;
1101 page = follow_page_mask(vma, start, foll_flags, &ctx);
1102 if (!page) {
1118 } else if (PTR_ERR(page) == -EEXIST) {
1120 * Proper page table entry exists, but no corresponding
1121 * struct page.
1124 } else if (IS_ERR(page)) {
1125 ret = PTR_ERR(page);
1129 pages[i] = page;
1130 flush_anon_page(vma, page, start);
1131 flush_dcache_page(page);
1176 * fixup_user_fault() - manually resolve a user page fault
1194 * get_user_pages() only guarantees to update these in the struct page.
1197 * access permission to the page because they are maintained in software. On
1256 struct page **pages,
1501 unsigned long nr_pages, struct page **pages,
1545 * get_dump_page() - pin user page in memory while writing it to core dump
1548 * Returns struct page pointer of user page pinned for dump,
1559 struct page *get_dump_page(unsigned long addr)
1562 struct page *page;
1568 ret = __get_user_pages_locked(mm, addr, 1, &page, NULL, &locked,
1572 return (ret == 1) ? page : NULL;
1600 struct page **pages,
1608 struct page *prev_head, *head;
1624 * If we get a page from the CMA zone, since we are going to
1695 struct page **pages,
1710 struct page **pages,
1763 struct page **pages,
1794 unsigned int gup_flags, struct page **pages,
1830 * @vmas: array of pointers to vmas corresponding to each page.
1850 * get_user_pages_remote walks a process's page tables and takes a reference
1851 * to each struct page that each user address corresponds to at a given
1852 * instant. That is, it takes the page that would be accessed if a user
1855 * This does not guarantee that the page exists in the user mappings when
1857 * page there in some cases (eg. if mmapped pagecache has been invalidated
1858 * and subsequently re faulted). However it does guarantee that the page
1859 * won't be freed completely. And mostly callers simply care that the page
1864 * If gup_flags & FOLL_WRITE == 0, the page must not be written to. If the page
1866 * be called after the page is finished with, and before put_page is called.
1883 unsigned int gup_flags, struct page **pages,
1897 unsigned int gup_flags, struct page **pages,
1905 unsigned int gup_flags, struct page **pages,
1920 * @vmas: array of pointers to vmas corresponding to each page.
1929 unsigned int gup_flags, struct page **pages,
1967 * We can leverage the VM_FAULT_RETRY functionality in the page fault
1973 unsigned int gup_flags, struct page **pages,
2013 struct page **pages, unsigned int gup_flags)
2040 * get_user_pages_fast attempts to pin user pages by walking the page
2042 * protected from page table pages being freed from under it, and should
2046 * rely on IPIs from the TLB flushing code blocking before the page table
2050 * Another way to achieve this is to batch up page table containing pages
2060 * free pages containing page tables or TLB flushing requires IPI broadcast.
2081 * completely different present page without a TLB flush in between; something
2129 struct page **pages)
2132 struct page *page = pages[--(*nr)];
2134 ClearPageReferenced(page);
2136 unpin_user_page(page);
2138 put_page(page);
2147 * To pin the page, fast-gup needs to do below in order:
2148 * (1) pin the page (by prefetching pte), then (2) check pte not changed.
2151 * with fast-gup, we need to do (1) clear pte, then (2) check whether page
2157 * walking a pgtable page that is being freed (pte is still valid but pmd
2164 struct page **pages, int *nr)
2173 struct page *head, *page;
2198 page = pte_page(pte);
2200 head = try_grab_compound_head(page, 1, flags);
2210 VM_BUG_ON_PAGE(compound_head(page) != head, page);
2213 * We need to make the page accessible if and only if we are
2219 ret = arch_make_page_accessible(page);
2221 unpin_user_page(page);
2225 SetPageReferenced(page);
2226 pages[*nr] = page;
2246 * For a futex to be placed on a THP tail page, get_futex_key requires a
2252 struct page **pages, int *nr)
2261 struct page **pages, int *nr)
2267 struct page *page = pfn_to_page(pfn);
2274 SetPageReferenced(page);
2275 pages[*nr] = page;
2276 if (unlikely(!try_grab_page(page, flags))) {
2291 struct page **pages, int *nr)
2309 struct page **pages, int *nr)
2327 struct page **pages, int *nr)
2335 struct page **pages, int *nr)
2342 static int record_subpages(struct page *page, unsigned long addr,
2343 unsigned long end, struct page **pages)
2348 pages[nr++] = page++;
2363 struct page **pages, int *nr)
2366 struct page *head, *page;
2383 page = head + ((addr & (sz-1)) >> PAGE_SHIFT);
2384 refs = record_subpages(page, addr, end, pages + *nr);
2402 struct page **pages, int *nr)
2420 struct page **pages, int *nr)
2428 struct page **pages, int *nr)
2430 struct page *head, *page;
2443 page = pmd_page(orig) + ((addr & ~PMD_MASK) >> PAGE_SHIFT);
2444 refs = record_subpages(page, addr, end, pages + *nr);
2462 struct page **pages, int *nr)
2464 struct page *head, *page;
2477 page = pud_page(orig) + ((addr & ~PUD_MASK) >> PAGE_SHIFT);
2478 refs = record_subpages(page, addr, end, pages + *nr);
2496 struct page **pages, int *nr)
2499 struct page *head, *page;
2506 page = pgd_page(orig) + ((addr & ~PGDIR_MASK) >> PAGE_SHIFT);
2507 refs = record_subpages(page, addr, end, pages + *nr);
2524 unsigned int flags, struct page **pages, int *nr)
2567 unsigned int flags, struct page **pages, int *nr)
2595 unsigned int flags, struct page **pages, int *nr)
2620 unsigned int flags, struct page **pages, int *nr)
2646 unsigned int flags, struct page **pages, int *nr)
2663 unsigned int gup_flags, struct page **pages)
2688 struct page **pages)
2708 * With interrupts disabled, we block page table pages from being freed
2735 struct page **pages)
2797 * access can get ambiguous page results. If you call this function without
2801 unsigned int gup_flags, struct page **pages)
2806 * because gup fast is always a "pin with a +1 page refcount" request.
2846 unsigned int gup_flags, struct page **pages)
2854 * FOLL_GET, because gup fast is always a "pin with a +1 page refcount"
2879 unsigned int gup_flags, struct page **pages)
2897 unsigned int gup_flags, struct page **pages)
2936 * @vmas: array of pointers to vmas corresponding to each page.
2951 unsigned int gup_flags, struct page **pages,
2973 * @vmas: array of pointers to vmas corresponding to each page.
2983 unsigned int gup_flags, struct page **pages,
3002 struct page **pages, unsigned int gup_flags)
3019 unsigned int gup_flags, struct page **pages,