Lines Matching refs:pages
33 static inline void sanity_check_pinned_pages(struct page **pages,
40 * We only pin anonymous pages if they are exclusive. Once pinned, we
44 * We'd like to verify that our pinned anonymous pages are still mapped
51 for (; npages; npages--, pages++) {
52 struct page *page = *pages;
269 * that such pages can be separately tracked and uniquely handled. In
337 * unpin_user_pages_dirty_lock() - release and optionally dirty gup-pinned pages
338 * @pages: array of pages to be maybe marked dirty, and definitely released.
339 * @npages: number of pages in the @pages array.
340 * @make_dirty: whether to mark the pages dirty
345 * For each page in the @pages array, make that page (or its head page, if a
347 * listed as clean. In any case, releases all pages using unpin_user_page(),
358 void unpin_user_pages_dirty_lock(struct page **pages, unsigned long npages,
366 unpin_user_pages(pages, npages);
370 sanity_check_pinned_pages(pages, npages);
372 folio = gup_folio_next(pages, npages, i, &nr);
408 * @npages: number of consecutive pages to release.
409 * @make_dirty: whether to mark the pages dirty
411 * "gup-pinned page range" refers to a range of pages that has had one of the
415 * its head pages, if a compound page) dirty, if @make_dirty is true, and if the
443 static void unpin_user_pages_lockless(struct page **pages, unsigned long npages)
451 * fork() and some anonymous pages might now actually be shared --
455 folio = gup_folio_next(pages, npages, i, &nr);
461 * unpin_user_pages() - release an array of gup-pinned pages.
462 * @pages: array of pages to be marked dirty and released.
463 * @npages: number of pages in the @pages array.
465 * For each page in the @pages array, release the page using unpin_user_page().
469 void unpin_user_pages(struct page **pages, unsigned long npages)
476 * If this WARN_ON() fires, then the system *might* be leaking pages (by
483 sanity_check_pinned_pages(pages, npages);
485 folio = gup_folio_next(pages, npages, i, &nr);
508 * has touched so far, we don't want to allocate unnecessary pages or
606 * We only care about anon pages in can_follow_write_pte() and don't
617 * Only return device mapping pages in the FOLL_GET or FOLL_PIN
628 /* Avoid special (like zero) pages in core dumps */
795 * When getting pages from ZONE_DEVICE memory, the @ctx->pgmap caches
849 * to fail on PROT_NONE-mapped pages.
869 /* user gate pages are read-only */
1066 * Anon pages in shared mappings are surprising: now
1131 * __get_user_pages() - pin user pages in memory
1134 * @nr_pages: number of pages from start to pin
1136 * @pages: array that receives pointers to the pages pinned.
1138 * only intends to ensure the pages are faulted in.
1141 * Returns either number of pages pinned (which may be less than the
1145 * -- If nr_pages is >0, but no pages were pinned, returns -errno.
1146 * -- If nr_pages is >0, and some pages were pinned, returns the number of
1147 * pages pinned. Again, this may be less than nr_pages.
1150 * The caller is responsible for releasing returned @pages, via put_page().
1186 unsigned int gup_flags, struct page **pages,
1198 VM_BUG_ON(!!pages != !!(gup_flags & (FOLL_GET | FOLL_PIN)));
1211 pages ? &page : NULL);
1228 * If we have a pending SIGKILL, don't keep faulting pages and
1257 * struct page. If the caller expects **pages to be
1261 if (pages) {
1274 if (pages) {
1286 * pages.
1311 pages[i + j] = subpage;
1467 struct page **pages,
1492 * is to set FOLL_GET if the caller wants pages[] filled in (but has
1496 * FOLL_PIN always expects pages to be non-null, but no need to assert
1499 if (pages && !(flags & FOLL_PIN))
1504 ret = __get_user_pages(mm, start, nr_pages, flags, pages,
1535 * For the prefault case (!pages) we only update counts.
1537 if (likely(pages))
1538 pages += ret;
1569 pages, locked);
1585 if (likely(pages))
1586 pages++;
1602 * populate_vma_page_range() - populate a range of pages in the vma.
1608 * This takes care of mlocking the pages too if VM_LOCKED is set.
1610 * Return either number of pages pinned in the vma, or a negative error
1676 * This takes care of mlocking the pages, too, if VM_LOCKED is set.
1684 * Returns either number of processed pages in the vma, or a negative error
1731 * __mm_populate - populate and/or mlock pages within a range of address space.
1749 * We want to fault in pages for [nstart; end) address range.
1771 * Now fault in a range of pages. populate_vma_page_range()
1772 * double checks the vma flags, so that it won't mlock pages
1792 unsigned long nr_pages, struct page **pages,
1832 if (pages) {
1833 pages[i] = virt_to_page((void *)start);
1834 if (pages[i])
1835 get_page(pages[i]);
1921 * already know that some or all of the pages in the address range aren't in
1926 * Note that we don't pin or otherwise hold the pages referenced that we fault
2026 * Returns the number of collected pages. Return value is always >= 0.
2031 struct page **pages)
2038 struct folio *folio = page_folio(pages[i]);
2075 * Unpins all pages and migrates device coherent pages and movable_page_list.
2076 * Returns -EAGAIN if all pages were successfully migrated or -errno for failure
2082 struct page **pages)
2088 struct folio *folio = page_folio(pages[i]);
2095 pages[i] = NULL;
2108 * We can't migrate pages with unexpected references, so drop
2110 * Migrating pages have been added to movable_page_list after
2114 unpin_user_page(pages[i]);
2115 pages[i] = NULL;
2138 if (pages[i])
2139 unpin_user_page(pages[i]);
2146 * Check whether all pages are *allowed* to be pinned. Rather confusingly, all
2147 * pages in the range are required to be pinned via FOLL_PIN, before calling
2150 * If any pages in the range are not allowed to be pinned, then this routine
2151 * will migrate those pages away, unpin all the pages in the range and return
2158 * If everything is OK and all pages in the range are allowed to be pinned, then
2159 * this routine leaves all pages pinned and returns zero for success.
2162 struct page **pages)
2168 nr_pages, pages);
2173 pages);
2177 struct page **pages)
2190 struct page **pages,
2198 return __get_user_pages_locked(mm, start, nr_pages, pages,
2204 pages, locked,
2212 rc = check_and_migrate_movable_pages(nr_pinned_pages, pages);
2222 static bool is_valid_gup_args(struct page **pages, int *locked,
2257 if (WARN_ON_ONCE((gup_flags & (FOLL_GET | FOLL_PIN)) && !pages))
2271 * get_user_pages_remote() - pin user pages in memory
2274 * @nr_pages: number of pages from start to pin
2276 * @pages: array that receives pointers to the pages pinned.
2278 * only intends to ensure the pages are faulted in.
2283 * Returns either number of pages pinned (which may be less than the
2287 * -- If nr_pages is >0, but no pages were pinned, returns -errno.
2288 * -- If nr_pages is >0, and some pages were pinned, returns the number of
2289 * pages pinned. Again, this may be less than nr_pages.
2291 * The caller is responsible for releasing returned @pages, via put_page().
2315 * via the user virtual addresses. The pages may be submitted for
2328 unsigned int gup_flags, struct page **pages,
2333 if (!is_valid_gup_args(pages, locked, &gup_flags,
2337 return __get_user_pages_locked(mm, start, nr_pages, pages,
2346 unsigned int gup_flags, struct page **pages,
2354 * get_user_pages() - pin user pages in memory
2356 * @nr_pages: number of pages from start to pin
2358 * @pages: array that receives pointers to the pages pinned.
2360 * only intends to ensure the pages are faulted in.
2368 unsigned int gup_flags, struct page **pages)
2372 if (!is_valid_gup_args(pages, NULL, &gup_flags, FOLL_TOUCH))
2375 return __get_user_pages_locked(current->mm, start, nr_pages, pages,
2384 * get_user_pages(mm, ..., pages, NULL);
2389 * get_user_pages_unlocked(mm, ..., pages);
2396 struct page **pages, unsigned int gup_flags)
2400 if (!is_valid_gup_args(pages, NULL, &gup_flags,
2404 return __get_user_pages_locked(current->mm, start, nr_pages, pages,
2412 * get_user_pages_fast attempts to pin user pages by walking the page
2414 * protected from page table pages being freed from under it, and should
2419 * pages are freed. This is unsuitable for architectures that do not need
2422 * Another way to achieve this is to batch up page table containing pages
2424 * pages. Disabling interrupts will allow the fast_gup walker to both block
2432 * free pages containing page tables or TLB flushing requires IPI broadcast.
2522 struct page **pages)
2525 struct page *page = pages[--(*nr)];
2557 struct page **pages, int *nr)
2572 * Always fallback to ordinary GUP on PROT_NONE-mapped pages:
2573 * pte_access_permitted() better should reject these pages
2590 undo_dev_pagemap(nr, nr_start, flags, pages);
2638 pages[*nr] = page;
2658 * get_user_pages_fast_only implementation that can pin pages. Thus it's still
2663 struct page **pages, int *nr)
2672 struct page **pages, int *nr)
2682 undo_dev_pagemap(nr, nr_start, flags, pages);
2687 undo_dev_pagemap(nr, nr_start, flags, pages);
2692 pages[*nr] = page;
2694 undo_dev_pagemap(nr, nr_start, flags, pages);
2707 struct page **pages, int *nr)
2713 if (!__gup_device_huge(fault_pfn, addr, end, flags, pages, nr))
2717 undo_dev_pagemap(nr, nr_start, flags, pages);
2725 struct page **pages, int *nr)
2731 if (!__gup_device_huge(fault_pfn, addr, end, flags, pages, nr))
2735 undo_dev_pagemap(nr, nr_start, flags, pages);
2743 struct page **pages, int *nr)
2751 struct page **pages, int *nr)
2759 unsigned long end, struct page **pages)
2764 pages[nr] = nth_page(page, nr);
2779 struct page **pages, int *nr)
2800 refs = record_subpages(page, addr, end, pages + *nr);
2828 struct page **pages, int *nr)
2837 if (!gup_hugepte(ptep, sz, addr, end, flags, pages, nr))
2846 struct page **pages, int *nr)
2854 struct page **pages, int *nr)
2867 pages, nr);
2871 refs = record_subpages(page, addr, end, pages + *nr);
2898 struct page **pages, int *nr)
2911 pages, nr);
2915 refs = record_subpages(page, addr, end, pages + *nr);
2943 struct page **pages, int *nr)
2955 refs = record_subpages(page, addr, end, pages + *nr);
2982 unsigned int flags, struct page **pages, int *nr)
3002 pages, nr))
3011 PMD_SHIFT, next, flags, pages, nr))
3013 } else if (!gup_pte_range(pmd, pmdp, addr, next, flags, pages, nr))
3021 unsigned int flags, struct page **pages, int *nr)
3035 pages, nr))
3039 PUD_SHIFT, next, flags, pages, nr))
3041 } else if (!gup_pmd_range(pudp, pud, addr, next, flags, pages, nr))
3049 unsigned int flags, struct page **pages, int *nr)
3064 P4D_SHIFT, next, flags, pages, nr))
3066 } else if (!gup_pud_range(p4dp, p4d, addr, next, flags, pages, nr))
3074 unsigned int flags, struct page **pages, int *nr)
3088 pages, nr))
3092 PGDIR_SHIFT, next, flags, pages, nr))
3094 } else if (!gup_p4d_range(pgdp, pgd, addr, next, flags, pages, nr))
3100 unsigned int flags, struct page **pages, int *nr)
3119 struct page **pages)
3139 * With interrupts disabled, we block page table pages from being freed
3147 gup_pgd_range(start, end, gup_flags, pages, &nr_pinned);
3151 * When pinning pages for DMA there could be a concurrent write protect
3156 unpin_user_pages_lockless(pages, nr_pinned);
3159 sanity_check_pinned_pages(pages, nr_pinned);
3168 struct page **pages)
3196 nr_pinned = lockless_pages_from_mm(start, end, gup_flags, pages);
3200 /* Slow path: try to get the remaining pages with get_user_pages */
3202 pages += nr_pinned;
3204 pages, &locked,
3208 * The caller has to unpin the pages we already pinned so
3219 * get_user_pages_fast_only() - pin user pages in memory
3221 * @nr_pages: number of pages from start to pin
3223 * @pages: array that receives pointers to the pages pinned.
3230 * pages pinned.
3237 unsigned int gup_flags, struct page **pages)
3246 if (!is_valid_gup_args(pages, NULL, &gup_flags,
3250 return internal_get_user_pages_fast(start, nr_pages, gup_flags, pages);
3255 * get_user_pages_fast() - pin user pages in memory
3257 * @nr_pages: number of pages from start to pin
3259 * @pages: array that receives pointers to the pages pinned.
3262 * Attempt to pin user pages in memory without taking mm->mmap_lock.
3266 * Returns number of pages pinned. This may be fewer than the number requested.
3267 * If nr_pages is 0 or negative, returns 0. If no pages were pinned, returns
3271 unsigned int gup_flags, struct page **pages)
3279 if (!is_valid_gup_args(pages, NULL, &gup_flags, FOLL_GET))
3281 return internal_get_user_pages_fast(start, nr_pages, gup_flags, pages);
3286 * pin_user_pages_fast() - pin user pages in memory without taking locks
3289 * @nr_pages: number of pages from start to pin
3291 * @pages: array that receives pointers to the pages pinned.
3298 * FOLL_PIN means that the pages must be released via unpin_user_page(). Please
3301 * Note that if a zero_page is amongst the returned pages, it will not have
3305 unsigned int gup_flags, struct page **pages)
3307 if (!is_valid_gup_args(pages, NULL, &gup_flags, FOLL_PIN))
3309 return internal_get_user_pages_fast(start, nr_pages, gup_flags, pages);
3314 * pin_user_pages_remote() - pin pages of a remote process
3318 * @nr_pages: number of pages from start to pin
3320 * @pages: array that receives pointers to the pages pinned.
3330 * FOLL_PIN means that the pages must be released via unpin_user_page(). Please
3333 * Note that if a zero_page is amongst the returned pages, it will not have
3338 unsigned int gup_flags, struct page **pages,
3343 if (!is_valid_gup_args(pages, locked, &gup_flags,
3346 return __gup_longterm_locked(mm, start, nr_pages, pages,
3353 * pin_user_pages() - pin user pages in memory for use by other devices
3356 * @nr_pages: number of pages from start to pin
3358 * @pages: array that receives pointers to the pages pinned.
3364 * FOLL_PIN means that the pages must be released via unpin_user_page(). Please
3367 * Note that if a zero_page is amongst the returned pages, it will not have
3371 unsigned int gup_flags, struct page **pages)
3375 if (!is_valid_gup_args(pages, NULL, &gup_flags, FOLL_PIN))
3378 pages, &locked, gup_flags);
3387 * Note that if a zero_page is amongst the returned pages, it will not have
3391 struct page **pages, unsigned int gup_flags)
3395 if (!is_valid_gup_args(pages, NULL, &gup_flags,
3399 return __gup_longterm_locked(current->mm, start, nr_pages, pages,