Lines Matching refs:page
33 * 05.04.94 - Multi-page memory management added for v1.1.
39 * Aug/Sep 2004 Changed to four level page tables (Andi Kleen)
92 #warning Unfortunate NUMA and NUMA Balancing config, growing page-frame for last_cpupid.
100 struct page *mem_map;
193 /* sync counter once per 64 page faults */
328 * This function frees user-level page tables of a process.
379 * We add page table cache pages with PAGE_SIZE,
435 * Ensure all pte setup (eg. pte page lock and page clearing) are
437 * put into page tables.
439 * The other side of the story is the pointer chasing in the page
440 * table walking code (when walking the page table without locking;
444 * seen in-order. See the alpha page table accessors for the
445 * smp_rmb() barriers in page table walking code.
504 pte_t pte, struct page *page)
526 pr_alert("BUG: Bad page map: %lu messages suppressed\n",
538 pr_alert("BUG: Bad page map in process %s pte:%08llx pmd:%08llx\n",
541 if (page)
542 dump_page(page, "bad pte");
555 * vm_normal_page -- This function gets the "struct page" associated with a pte.
557 * "Special" mappings do not wish to be associated with a "struct page" (either
559 * case, NULL is returned here. "Normal" mappings do have a struct page.
588 * page" backing, however the difference is that _all_ pages with a struct
589 * page (that is, those where pfn_valid is true) are refcounted and considered
596 struct page *vm_normal_page(struct vm_area_struct *vma, unsigned long addr,
644 * NOTE! We still have PageReserved() pages in the page tables.
652 struct page *vm_normal_page_pmd(struct vm_area_struct *vma, unsigned long addr,
685 * NOTE! We still have PageReserved() pages in the page tables.
694 * copy one vm_area from one task to the other. Assumes the page tables
706 struct page *page;
723 page = migration_entry_to_page(entry);
725 rss[mm_counter(page)]++;
742 page = device_private_entry_to_page(entry);
753 get_page(page);
754 rss[mm_counter(page)]++;
755 page_dup_rmap(page, false);
780 * Copy a present and normal page if necessary.
785 * the page refcount and re-use the pte the traditional
790 * copy rather than just a reference to the same page),
794 * And if we need a pre-allocated page but don't yet have
796 * code know so that it can do so outside the page table
802 struct page **prealloc, pte_t pte, struct page *page)
805 struct page *new_page;
811 * What we want to do is to check whether this page may
814 * the page immediately so that we'll always guarantee
815 * the pinned page won't be randomly replaced in the
818 * The page pinning checks are just "has this mm ever
820 * the page count. That might give false positives for
825 if (likely(!page_maybe_dma_pinned(page)))
832 * a copy of a non-anonymous page of that vma to the
834 * Return 1 if the page is not an anonymous page.
836 if (!PageAnon(page))
844 * We have a prealloc page, all good! Take it
845 * over and copy the page & arm it.
848 copy_user_highpage(new_page, page, addr, src_vma);
854 /* All done, just insert the new page copy in the child */
865 * Copy one pte. Returns 0 if succeeded, or -EAGAIN if one preallocated page
871 struct page **prealloc)
876 struct page *page;
878 page = vm_normal_page(src_vma, addr, pte);
879 if (page) {
883 addr, rss, prealloc, pte, page);
887 get_page(page);
888 page_dup_rmap(page, false);
889 rss[mm_counter(page)]++;
916 static inline struct page *
920 struct page *new_page;
948 struct page *prealloc = NULL;
995 * If we need a pre-allocated page for this pte, drop the
1002 * pre-alloc page cannot be reused by next time so as
1004 * will allocate page according to address). This
1154 * Don't copy ptes where a page fault will fill them correctly.
1260 struct page *page;
1262 page = vm_normal_page(vma, addr, ptent);
1264 page = NULL;
1265 if (unlikely(details) && page) {
1272 details->check_mapping != page_rmapping(page))
1278 if (unlikely(!page))
1282 if (!PageAnon(page)) {
1285 set_page_dirty(page);
1289 mark_page_accessed(page);
1291 rss[mm_counter(page)]--;
1292 page_remove_rmap(page, false);
1293 if (unlikely(page_mapcount(page) < 0))
1294 print_bad_pte(vma, addr, ptent, page);
1295 if (unlikely(__tlb_remove_page(tlb, page))) {
1305 struct page *page = device_private_entry_to_page(entry);
1314 page_rmapping(page))
1319 rss[mm_counter(page)]--;
1320 page_remove_rmap(page, false);
1321 put_page(page);
1326 /* Genuine swap entry, hence a private anon page */
1331 struct page *page;
1333 page = migration_entry_to_page(entry);
1335 details->check_mapping != page_rmapping(page))
1337 rss[mm_counter(page)]--;
1402 * trans huge page faults running, and if the pmd is
1667 static int validate_page_before_insert(struct page *page)
1669 if (PageAnon(page) || PageSlab(page) || page_has_type(page))
1671 flush_dcache_page(page);
1676 unsigned long addr, struct page *page, pgprot_t prot)
1681 get_page(page);
1682 inc_mm_counter_fast(mm, mm_counter_file(page));
1683 page_add_file_rmap(page, false);
1684 set_pte_at(mm, addr, pte, mk_pte(page, prot));
1689 * This is the old fallback for page remapping.
1696 struct page *page, pgprot_t prot)
1703 retval = validate_page_before_insert(page);
1710 retval = insert_page_into_pte_locked(mm, pte, addr, page, prot);
1718 unsigned long addr, struct page *page, pgprot_t prot)
1722 if (!page_count(page))
1724 err = validate_page_before_insert(page);
1727 return insert_page_into_pte_locked(mm, pte, addr, page, prot);
1734 struct page **pages, unsigned long *num, pgprot_t prot)
1804 struct page **pages, unsigned long *num)
1816 /* Defer page refcount checking till we're about to map that page. */
1834 * vm_insert_page - insert single page into user vma
1836 * @addr: target user address of this page
1837 * @page: source kernel page
1842 * The page has to be a nice clean _individual_ kernel allocation.
1843 * If you allocate a compound page, you need to have marked it as
1844 * such (__GFP_COMP), or manually just split the page up yourself
1848 * took an arbitrary page protection parameter. This doesn't allow
1853 * The page does not need to be reserved.
1858 * function from other places, for example from page-fault handler.
1863 struct page *page)
1867 if (!page_count(page))
1874 return insert_page(vma, addr, page, vma->vm_page_prot);
1882 * @num: number of pages in page array
1889 static int __vm_map_pages(struct vm_area_struct *vma, struct page **pages,
1918 * @num: number of pages in page array
1923 * If we fail to insert any page into the vma, the function will return
1932 int vm_map_pages(struct vm_area_struct *vma, struct page **pages,
1943 * @num: number of pages in page array
1952 int vm_map_pages_zero(struct vm_area_struct *vma, struct page **pages,
1974 * mapped PFN is a writeable COW page. In the mkwrite
2015 * @addr: target user address of this page
2017 * @pgprot: pgprot flags for the inserted page
2020 * to override pgprot on a per-page basis.
2064 * @addr: target user address of this page
2121 * refcount the page if pfn_valid is true (hence insert_page rather
2123 * without pte special, it would there be refcounted as a normal page.
2127 struct page *page;
2134 page = pfn_to_page(pfn_t_to_pfn(pfn));
2135 err = insert_page(vma, addr, page, pgprot);
2151 * @addr: target user address of this page
2153 * @pgprot: pgprot flags for the inserted page
2156 * to override pgprot on a per-page basis.
2163 * This is ensured by core vm only modifying these page table entries using
2166 * Also when new page-table entries are created, this is only done using the
2168 * except for page-table entries that point to anonymous pages as the result
2301 * @addr: target page aligned user address to start at
2302 * @pfn: page frame number of kernel physical memory address
2304 * @prot: page protection flags for this mapping
2329 * raw PFN mappings, and do not have a "struct page" associated
2395 * You *really* shouldn't map things that aren't page-aligned,
2580 * Scan a region of virtual memory, filling in page tables as necessary
2581 * and calling a provided function on each leaf page table.
2592 * each leaf page table where it exists.
2594 * Unlike apply_to_page_range, this does _not_ fill in page tables
2605 * handle_pte_fault chooses page fault handler according to an entry which was
2628 static inline bool cow_user_page(struct page *dst, struct page *src,
2645 * If the source page was a PFN mapping, we don't have
2646 * a "struct page" for it. We do a best-effort copy by
2655 * take a double page fault, so mark it accessed here.
2678 * This really shouldn't fail, because the page is there
2679 * in the page tables. But it might just be unreadable,
2687 /* Re-validate under PTL if the page is still mapped */
2698 * The same page can be mapped back since last copy attempt.
2738 * Notify the address space that the page is about to become writable so that
2739 * it can prohibit this or wait for the page to get into an appropriate state.
2746 struct page *page = vmf->page;
2761 lock_page(page);
2762 if (!page->mapping) {
2763 unlock_page(page);
2768 VM_BUG_ON_PAGE(!PageLocked(page), page);
2773 * Handle dirtying of a page in shared file mapping on a write fault.
2775 * The function expects the page to be locked and unlocks it.
2781 struct page *page = vmf->page;
2785 dirtied = set_page_dirty(page);
2786 VM_BUG_ON_PAGE(PageAnon(page), page);
2788 * Take a local copy of the address_space - page.mapping may be zeroed
2793 mapping = page_rmapping(page);
2794 unlock_page(page);
2800 * Throttle page dirtying rate down to writeback speed.
2803 * set page.mapping but still dirty their pages
2823 * Handle write page faults for pages that can be reused in the current vma
2826 * or due to us being the last reference standing to the page. In either
2827 * case, all we need to do here is to mark the page as writable and update
2834 struct page *page = vmf->page;
2841 if (page)
2842 page_cpupid_xchg_last(page, (1 << LAST_CPUPID_SHIFT) - 1);
2854 * Handle the case of a page which we actually need to copy to a new page.
2856 * Called with mmap_lock locked and the old page referenced, but
2861 * - Allocate a page, copy the content of the old page to the new one.
2863 * - Take the PTL. If the pte changed, bail out and release the allocated page
2864 * - If the pte is still the way we remember it, update the page table and all
2865 * relevant references. This includes dropping the reference the page-table
2866 * held to the old page, as well as updating the rmap.
2867 * - In any case, unlock the PTL and drop the reference we took to the old page.
2873 struct page *old_page = vmf->page;
2874 struct page *new_page = NULL;
2945 pr_info("set wp new page %lx purgeable\n", page_to_pfn(new_page));
2952 * mmu page tables (such as kvm shadow page tables), we want the
2953 * new page to be mapped directly into the secondary page table.
2960 * Only after switching the pte to the new page may
2963 * before the pte is switched to the new page, and
2964 * "reuse" the old page writing into it while our pte
2975 * no process can access the old page before the
2976 * decremented mapcount is visible. And the old page
2979 * old page will be flushed before it can be reused.
2984 /* Free the old page.. */
3003 * keep the mlocked page.
3023 * finish_mkwrite_fault - finish page fault for a shared mapping, making PTE
3024 * writeable once the page is prepared
3028 * This function handles all that is needed to finish a write page fault in a
3029 * shared mapping due to PTE being read-only once the mapped page is prepared.
3032 * The function expects the page to be locked or other protection against
3044 * We might have raced with another page fault while we released the
3054 vmf->address, vmf->page))) {
3064 * Handle write page faults for VM_MIXEDMAP or VM_PFNMAP for a VM_SHARED
3091 get_page(vmf->page);
3100 put_page(vmf->page);
3105 unlock_page(vmf->page);
3106 put_page(vmf->page);
3111 vmf->page))){
3113 put_page(vmf->page);
3118 lock_page(vmf->page);
3121 put_page(vmf->page);
3128 * to a shared page. It is done by copying the page to a new address
3129 * and decrementing the shared-page counter for the old page.
3132 * done by the caller (the low-level page fault routine in most cases).
3136 * We also mark the page dirty at this point even though the page will
3162 vmf->page = vm_normal_page(vma, vmf->address, vmf->orig_pte);
3163 if (!vmf->page) {
3183 if (PageAnon(vmf->page)) {
3184 struct page *page = vmf->page;
3186 /* PageKsm() doesn't necessarily raise the page refcount */
3187 if (PageKsm(page) || page_count(page) != 1)
3189 if (!trylock_page(page))
3191 if (PageKsm(page) || page_mapcount(page) != 1 || page_count(page) != 1) {
3192 unlock_page(page);
3197 * page count reference, and the page is locked,
3200 unlock_page(page);
3203 vmf->page))){
3218 get_page(vmf->page);
3257 * unmap_mapping_page() - Unmap single page from processes.
3258 * @page: The locked page to be unmapped.
3260 * Unmap this page from any userspace process which still has it mmaped.
3263 * truncation or invalidation holds the lock on a page, it may find that
3264 * the page has been remapped again: and then uses unmap_mapping_page()
3267 void unmap_mapping_page(struct page *page)
3269 struct address_space *mapping = page->mapping;
3272 VM_BUG_ON(!PageLocked(page));
3273 VM_BUG_ON(PageTail(page));
3276 details.first_index = page->index;
3277 details.last_index = page->index + thp_nr_pages(page) - 1;
3278 details.single_page = page;
3289 * @start: Index of first page to be unmapped.
3295 * a file is being truncated, but not when invalidating pages from the page
3321 * @holebegin: byte in first page to unmap, relative to the start of
3324 * must keep the partial page. In contrast, we must get rid of
3361 struct page *page = NULL, *swapcache;
3378 vmf->page = device_private_entry_to_page(entry);
3387 * Get a page reference while we know the page can't be
3390 get_page(vmf->page);
3392 vmf->page->pgmap->ops->migrate_to_ram(vmf);
3393 put_page(vmf->page);
3405 page = lookup_swap_cache(entry, vma, vmf->address);
3406 swapcache = page;
3408 if (!page) {
3414 page = alloc_page_vma(GFP_HIGHUSER_MOVABLE, vma,
3416 if (page) {
3419 __SetPageLocked(page);
3420 __SetPageSwapBacked(page);
3421 set_page_private(page, entry.val);
3424 SetPageSwapCache(page);
3425 err = mem_cgroup_charge(page, vma->vm_mm,
3427 ClearPageSwapCache(page);
3435 workingset_refault(page, shadow);
3437 lru_cache_add(page);
3438 swap_readpage(page, true);
3441 page = swapin_readahead(entry, GFP_HIGHUSER_MOVABLE,
3443 swapcache = page;
3446 if (!page) {
3459 /* Had to read the page from swap area: Major fault */
3463 } else if (PageHWPoison(page)) {
3473 locked = lock_page_or_retry(page, vma->vm_mm, vmf->flags);
3483 * release the swapcache from under us. The page pin, and pte_same
3485 * swapcache, we need to check that the page's swap has not changed.
3487 if (unlikely((!PageSwapCache(page) ||
3488 page_private(page) != entry.val)) && swapcache)
3491 page = ksm_might_need_to_copy(page, vma, vmf->address);
3492 if (unlikely(!page)) {
3494 page = swapcache;
3498 cgroup_throttle_swaprate(page, GFP_KERNEL);
3508 if (unlikely(!PageUptodate(page))) {
3514 * The page isn't present yet, go ahead with the fault.
3518 * while the page is counted on swap but not yet in mapcount i.e.
3523 vmf->address, page))){
3530 pte = mk_pte(page, vma->vm_page_prot);
3531 if ((vmf->flags & FAULT_FLAG_WRITE) && reuse_swap_page(page, NULL)) {
3537 flush_icache_page(vma, page);
3549 if (unlikely(page != swapcache && swapcache)) {
3550 page_add_new_anon_rmap(page, vma, vmf->address, false);
3551 lru_cache_add_inactive_or_unevictable(page, vma);
3553 do_page_add_anon_rmap(page, vma, vmf->address, exclusive);
3557 if (mem_cgroup_swap_full(page) ||
3558 (vma->vm_flags & VM_LOCKED) || PageMlocked(page))
3559 try_to_free_swap(page);
3560 unlock_page(page);
3561 if (page != swapcache && swapcache) {
3590 unlock_page(page);
3592 put_page(page);
3593 if (page != swapcache && swapcache) {
3608 struct page *page;
3633 /* use extra page table for userexpte */
3645 /* Use the zero-page for reads */
3660 /* Deliver the page fault to userland, check inside PT lock */
3668 /* Allocate our own private page. */
3671 page = alloc_zeroed_user_highpage_movable(vma, vmf->address);
3672 if (!page)
3675 if (mem_cgroup_charge(page, vma->vm_mm, GFP_KERNEL))
3677 cgroup_throttle_swaprate(page, GFP_KERNEL);
3681 * preceding stores to the page contents become visible before
3684 __SetPageUptodate(page);
3686 entry = mk_pte(page, vma->vm_page_prot);
3702 /* Deliver the page fault to userland, check inside PT lock */
3705 put_page(page);
3710 page_add_new_anon_rmap(page, vma, vmf->address, false);
3712 SetPagePurgeable(page);
3714 lru_cache_add_inactive_or_unevictable(page, vma);
3720 xpm_integrity_update_hook(vma, vmf->flags, page);
3731 put_page(page);
3734 put_page(page);
3776 if (unlikely(PageHWPoison(vmf->page))) {
3777 struct page *page = vmf->page;
3780 if (page_mapped(page))
3781 unmap_mapping_pages(page_mapping(page),
3782 page->index, 1, false);
3783 /* Retry if a clean page was removed from the cache. */
3784 if (invalidate_inode_page(page))
3786 unlock_page(page);
3788 put_page(page);
3789 vmf->page = NULL;
3794 lock_page(vmf->page);
3796 VM_BUG_ON_PAGE(!PageLocked(vmf->page), vmf->page);
3848 * At this point we know that our vmf->pmd points to a page of ptes
3875 static vm_fault_t do_set_pmd(struct vm_fault *vmf, struct page *page)
3887 page = compound_head(page);
3888 if (compound_order(page) != HPAGE_PMD_ORDER)
3907 flush_icache_page(vma, page + i);
3909 entry = mk_huge_pmd(page, vma->vm_page_prot);
3913 add_mm_counter(vma->vm_mm, mm_counter_file(page), HPAGE_PMD_NR);
3914 page_add_file_rmap(page, true);
3933 static vm_fault_t do_set_pmd(struct vm_fault *vmf, struct page *page)
3941 * alloc_set_pte - setup new PTE entry for given page and add reverse page
3942 * mapping. If needed, the function allocates page table or use pre-allocated.
3945 * @page: page to map
3950 * Target users are page handler itself and implementations of
3955 vm_fault_t alloc_set_pte(struct vm_fault *vmf, struct page *page)
3962 if (pmd_none(*vmf->pmd) && PageTransCompound(page)) {
3963 ret = do_set_pmd(vmf, page);
3982 vmf->address, page)))
3985 flush_icache_page(vma, page);
3986 entry = mk_pte(page, vma->vm_page_prot);
3990 /* copy-on-write page */
3993 page_add_new_anon_rmap(page, vma, vmf->address, false);
3994 lru_cache_add_inactive_or_unevictable(page, vma);
3996 inc_mm_counter_fast(vma->vm_mm, mm_counter_file(page));
3997 page_add_file_rmap(page, false);
4001 /* no need to invalidate: a not-present page won't be cached */
4009 * finish_fault - finish page fault once we have prepared the page to fault
4013 * This function handles all that is needed to finish a page fault once the
4014 * page to fault in is prepared. It handles locking of PTEs, inserts PTE for
4015 * given page, adds reverse page mapping, handles memcg charges and LRU
4018 * The function expects the page to be locked and on success it consumes a
4019 * reference of a page being mapped (for the PTE which maps it).
4025 struct page *page;
4028 /* Did we COW the page? */
4031 page = vmf->cow_page;
4033 page = vmf->page;
4037 * page
4042 ret = alloc_set_pte(vmf, page);
4059 * fault_around_bytes must be rounded down to the nearest page order as it's
4089 * It uses vm_ops->map_pages() to map the pages, which skips the page if it's
4092 * This function is called with the page table lock taken. In the split ptlock
4093 * case the page table lock only protects only those entries which belong to
4094 * the page table corresponding to the fault address.
4104 * fault_around_bytes rounded down to the machine page size
4105 * (and therefore to page order). This way it's easier to guarantee
4106 * that we don't cross page table boundaries.
4124 * end_pgoff is either the end of the page table, the end of
4142 /* Huge page is mapped? Page fault is solved */
4148 /* ->map_pages() haven't done anything useful. Cold page cache? */
4152 /* check if the page fault is solved */
4170 * if page by the offset is not ready to be mapped (cold cache or
4184 unlock_page(vmf->page);
4186 put_page(vmf->page);
4214 copy_user_highpage(vmf->cow_page, vmf->page, vmf->address, vma);
4218 unlock_page(vmf->page);
4219 put_page(vmf->page);
4238 * Check if the backing address space wants to know that the page is
4242 unlock_page(vmf->page);
4246 put_page(vmf->page);
4254 unlock_page(vmf->page);
4255 put_page(vmf->page);
4321 static int numa_migrate_prep(struct page *page, struct vm_area_struct *vma,
4325 get_page(page);
4333 return mpol_misplaced(page, vma, addr);
4339 struct page *page = NULL;
4372 page = vm_normal_page(vma, vmf->address, pte);
4373 if (!page) {
4379 if (PageCompound(page)) {
4396 * Flag if the page is shared between multiple address spaces. This
4399 if (page_mapcount(page) > 1 && (vma->vm_flags & VM_SHARED))
4402 last_cpupid = page_cpupid_last(page);
4403 page_nid = page_to_nid(page);
4404 target_nid = numa_migrate_prep(page, vma, vmf->address, page_nid,
4408 put_page(page);
4413 migrated = migrate_misplaced_page(page, vma, target_nid);
4496 * PowerPC hashed page tables that act as extended TLBs).
4511 * want to allocate huge page, and if we expose page table
4574 /* Skip spurious TLB flush for retried page fault */
4580 * This still avoids useless tlb flushes for .text page faults
4649 /* Huge pud page fault raced with pmd_alloc? */
4687 * mm_account_fault - Do page fault accountings
4691 * the task who triggered this page fault.
4696 * This will take care of most of the page fault accountings. Meanwhile, it
4699 * still be in per-arch page fault handlers at the entry of page fault.
4712 * So this is not a "this many hardware page faults" counter. We
4803 * Allocate p4d page table.
4826 * Allocate page upper directory.
4850 * Allocate page middle directory.
5057 * given task for page fault accounting.
5073 struct page *page = NULL;
5076 gup_flags, &page, &vma, NULL);
5101 maddr = kmap(page);
5103 copy_to_user_page(vma, page, addr,
5105 set_page_dirty_lock(page);
5107 copy_from_user_page(vma, page, addr,
5110 kunmap(page);
5111 put_page(page);
5143 * Do not walk the page table directly, use get_user_pages
5220 * Process all subpages of the specified huge page with the specified
5237 /* If target subpage in first half of huge page */
5240 /* Process subpages at the end of huge page */
5246 /* If target subpage in second half of huge page */
5249 /* Process subpages at the begin of huge page */
5270 static void clear_gigantic_page(struct page *page,
5275 struct page *p = page;
5279 i++, p = mem_map_next(p, page, i)) {
5287 struct page *page = arg;
5289 clear_user_highpage(page + idx, addr);
5292 void clear_huge_page(struct page *page,
5299 clear_gigantic_page(page, addr, pages_per_huge_page);
5303 process_huge_page(addr_hint, pages_per_huge_page, clear_subpage, page);
5306 static void copy_user_gigantic_page(struct page *dst, struct page *src,
5312 struct page *dst_base = dst;
5313 struct page *src_base = src;
5326 struct page *dst;
5327 struct page *src;
5339 void copy_user_huge_page(struct page *dst, struct page *src,
5360 long copy_huge_page_from_user(struct page *dst_page,
5369 struct page *subpage = dst_page;
5403 page_ptl_cachep = kmem_cache_create("page->ptl", sizeof(spinlock_t), 0,
5407 bool ptlock_alloc(struct page *page)
5414 page->ptl = ptl;
5418 void ptlock_free(struct page *page)
5420 kmem_cache_free(page_ptl_cachep, page->ptl);