Lines Matching defs:page
14 * Per cpu hot/cold page lists, bulk allocation, Martin J. Bligh, Sept 2002
52 #include <linux/page-isolation.h>
92 * Skip free page reporting notification for the (possibly merged) page.
93 * This does not hinder free page reporting from grabbing the page,
95 * the free page reporting infrastructure about a newly freed page. For
96 * example, used when temporarily pulling a page from a freelist and
102 * Place the (possibly merged) page to the tail of the freelist. Will ignore
103 * page shuffling (relevant code - e.g., memory onlining - is expected to
108 * (memory onlining) or untouched pages (page isolation, free page
223 * A cached value of the page's pageblock's migratetype, used when the page is
226 * Also the migratetype set in the page does not necessarily match the pcplist
227 * index, e.g. page might have MIGRATE_CMA set but be on a pcplist with any
230 static inline int get_pcppage_migratetype(struct page *page)
232 return page->index;
235 static inline void set_pcppage_migratetype(struct page *page, int migratetype)
237 page->index = migratetype;
282 static void __free_pages_ok(struct page *page, unsigned int order,
414 * KASAN memory tracking as the poison will be properly inserted at page
419 static inline void kasan_free_nondeferred_pages(struct page *page, int order)
422 kasan_free_pages(page, order);
425 /* Returns true if the struct page for the pfn is uninitialised */
487 static inline unsigned long *get_pageblock_bitmap(struct page *page,
493 return page_zone(page)->pageblock_flags;
497 static inline int pfn_to_bitidx(struct page *page, unsigned long pfn)
502 pfn = pfn - round_down(page_zone(page)->zone_start_pfn, pageblock_nr_pages);
509 * @page: The page within the block of interest
510 * @pfn: The target page frame number
516 unsigned long __get_pfnblock_flags_mask(struct page *page,
524 bitmap = get_pageblock_bitmap(page, pfn);
525 bitidx = pfn_to_bitidx(page, pfn);
533 unsigned long get_pfnblock_flags_mask(struct page *page, unsigned long pfn,
536 return __get_pfnblock_flags_mask(page, pfn, mask);
539 static __always_inline int get_pfnblock_migratetype(struct page *page, unsigned long pfn)
541 return __get_pfnblock_flags_mask(page, pfn, MIGRATETYPE_MASK);
546 * @page: The page within the block of interest
548 * @pfn: The target page frame number
551 void set_pfnblock_flags_mask(struct page *page, unsigned long flags,
562 bitmap = get_pageblock_bitmap(page, pfn);
563 bitidx = pfn_to_bitidx(page, pfn);
567 VM_BUG_ON_PAGE(!zone_spans_pfn(page_zone(page), pfn), page);
581 void set_pageblock_migratetype(struct page *page, int migratetype)
587 set_pfnblock_flags_mask(page, (unsigned long)migratetype,
588 page_to_pfn(page), MIGRATETYPE_MASK);
592 static int page_outside_zone_boundaries(struct zone *zone, struct page *page)
596 unsigned long pfn = page_to_pfn(page);
608 pr_err("page 0x%lx outside node %d zone %s [ 0x%lx - 0x%lx ]\n",
615 static int page_is_consistent(struct zone *zone, struct page *page)
617 if (!pfn_valid_within(page_to_pfn(page)))
619 if (zone != page_zone(page))
627 static int __maybe_unused bad_range(struct zone *zone, struct page *page)
629 if (page_outside_zone_boundaries(zone, page))
631 if (!page_is_consistent(zone, page))
637 static inline int __maybe_unused bad_range(struct zone *zone, struct page *page)
643 static void bad_page(struct page *page, const char *reason)
660 "BUG: Bad page state: %lu messages suppressed\n",
669 pr_alert("BUG: Bad page state in process %s pfn:%05lx\n",
670 current->comm, page_to_pfn(page));
671 __dump_page(page, reason);
672 dump_page_owner(page);
678 page_mapcount_reset(page); /* remove PageBuddy */
685 * The first PAGE_SIZE page is called the "head page" and have PG_head set.
688 * in bit 0 of page->compound_head. The rest of bits is pointer to head page.
690 * The first tail page's ->compound_dtor holds the offset in array of compound
691 * page destructors. See compound_page_dtors.
693 * The first tail page's ->compound_order holds the order of allocation.
697 void free_compound_page(struct page *page)
699 mem_cgroup_uncharge(page);
700 __free_pages_ok(page, compound_order(page), FPI_NONE);
703 void prep_compound_page(struct page *page, unsigned int order)
708 __SetPageHead(page);
710 struct page *p = page + i;
713 set_compound_head(p, page);
716 set_compound_page_dtor(page, COMPOUND_PAGE_DTOR);
717 set_compound_order(page, order);
718 atomic_set(compound_mapcount_ptr(page), -1);
719 if (hpage_pincount_available(page))
720 atomic_set(compound_pincount_ptr(page), 0);
767 static inline bool set_page_guard(struct zone *zone, struct page *page,
776 __SetPageGuard(page);
777 INIT_LIST_HEAD(&page->lru);
778 set_page_private(page, order);
785 static inline void clear_page_guard(struct zone *zone, struct page *page,
791 __ClearPageGuard(page);
793 set_page_private(page, 0);
798 static inline bool set_page_guard(struct zone *zone, struct page *page,
800 static inline void clear_page_guard(struct zone *zone, struct page *page,
804 static inline void set_buddy_order(struct page *page, unsigned int order)
806 set_page_private(page, order);
807 __SetPageBuddy(page);
811 * This function checks whether a page is free && is the buddy
812 * we can coalesce a page and its buddy if
815 * (c) a page and its buddy have the same order &&
816 * (d) a page and its buddy are in the same zone.
818 * For recording whether a page is in the buddy system, we set PageBuddy.
821 * For recording page's order, we use page_private(page).
823 static inline bool page_is_buddy(struct page *page, struct page *buddy,
836 if (page_zone_id(page) != page_zone_id(buddy))
851 !capc->page &&
856 compaction_capture(struct capture_control *capc, struct page *page,
871 * have trouble finding a high-order free page.
876 capc->page = page;
887 compaction_capture(struct capture_control *capc, struct page *page,
895 static inline void add_to_free_list(struct page *page, struct zone *zone,
900 list_add(&page->lru, &area->free_list[migratetype]);
905 static inline void add_to_free_list_tail(struct page *page, struct zone *zone,
910 list_add_tail(&page->lru, &area->free_list[migratetype]);
919 static inline void move_to_free_list(struct page *page, struct zone *zone,
924 list_move_tail(&page->lru, &area->free_list[migratetype]);
927 static inline void del_page_from_free_list(struct page *page, struct zone *zone,
930 /* clear reported state and update reported page count */
931 if (page_reported(page))
932 __ClearPageReported(page);
934 list_del(&page->lru);
935 __ClearPageBuddy(page);
936 set_page_private(page, 0);
941 * If this is not the largest possible page, check if the buddy
944 * that is happening, add the free page to the tail of the list
946 * as a higher order page
950 struct page *page, unsigned int order)
952 struct page *higher_page, *higher_buddy;
962 higher_page = page + (combined_pfn - pfn);
984 * Page's order is recorded in page_private(page) field.
994 static inline void __free_one_page(struct page *page,
1003 struct page *buddy;
1009 VM_BUG_ON_PAGE(page->flags & PAGE_FLAGS_CHECK_AT_PREP, page);
1015 VM_BUG_ON_PAGE(pfn & ((1 << order) - 1), page);
1016 VM_BUG_ON_PAGE(bad_range(zone, page), page);
1020 if (compaction_capture(capc, page, order, migratetype)) {
1026 buddy = page + (buddy_pfn - pfn);
1030 if (!page_is_buddy(page, buddy, order))
1033 * Our buddy is free or it is CONFIG_DEBUG_PAGEALLOC guard page,
1041 page = page + (combined_pfn - pfn);
1058 buddy = page + (buddy_pfn - pfn);
1071 set_buddy_order(page, order);
1078 to_tail = buddy_merge_likely(pfn, buddy_pfn, page, order);
1081 add_to_free_list_tail(page, zone, order, migratetype);
1083 add_to_free_list(page, zone, order, migratetype);
1085 /* Notify page reporting subsystem of freed page */
1091 * A bad page could be due to a number of fields. Instead of multiple branches,
1095 static inline bool page_expected_state(struct page *page,
1098 if (unlikely(atomic_read(&page->_mapcount) != -1))
1101 if (unlikely((unsigned long)page->mapping |
1102 page_ref_count(page) |
1104 (unsigned long)page->mem_cgroup |
1106 (page->flags & check_flags)))
1112 static const char *page_bad_reason(struct page *page, unsigned long flags)
1116 if (unlikely(atomic_read(&page->_mapcount) != -1))
1118 if (unlikely(page->mapping != NULL))
1120 if (unlikely(page_ref_count(page) != 0))
1122 if (unlikely(page->flags & flags)) {
1129 if (unlikely(page->mem_cgroup))
1130 bad_reason = "page still charged to cgroup";
1135 static void check_free_page_bad(struct page *page)
1137 bad_page(page,
1138 page_bad_reason(page, PAGE_FLAGS_CHECK_AT_FREE));
1141 static inline int check_free_page(struct page *page)
1143 if (likely(page_expected_state(page, PAGE_FLAGS_CHECK_AT_FREE)))
1147 check_free_page_bad(page);
1151 static int free_tail_pages_check(struct page *head_page, struct page *page)
1156 * We rely page->lru.next never has bit 0 set, unless the page
1165 switch (page - head_page) {
1167 /* the first tail page: ->mapping may be compound_mapcount() */
1168 if (unlikely(compound_mapcount(page))) {
1169 bad_page(page, "nonzero compound_mapcount");
1175 * the second tail page: ->mapping is
1180 if (page->mapping != TAIL_MAPPING) {
1181 bad_page(page, "corrupted mapping in tail page");
1186 if (unlikely(!PageTail(page))) {
1187 bad_page(page, "PageTail not set");
1190 if (unlikely(compound_head(page) != head_page)) {
1191 bad_page(page, "compound_head not consistent");
1196 page->mapping = NULL;
1197 clear_compound_head(page);
1201 static void kernel_init_free_pages(struct page *page, int numpages)
1208 clear_highpage(page + i);
1212 static __always_inline bool free_pages_prepare(struct page *page,
1217 VM_BUG_ON_PAGE(PageTail(page), page);
1219 trace_mm_page_free(page, order);
1221 if (unlikely(PageHWPoison(page)) && !order) {
1224 * Untie memcg state and reset page's owner
1226 if (memcg_kmem_enabled() && PageKmemcg(page))
1227 __memcg_kmem_uncharge_page(page, order);
1228 reset_page_owner(page, order);
1233 * Check tail pages before head page information is cleared to
1237 bool compound = PageCompound(page);
1240 VM_BUG_ON_PAGE(compound && compound_order(page) != order, page);
1243 ClearPageDoubleMap(page);
1246 bad += free_tail_pages_check(page, page + i);
1247 if (unlikely(check_free_page(page + i))) {
1251 (page + i)->flags &= ~PAGE_FLAGS_CHECK_AT_PREP;
1254 if (PageMappingFlags(page))
1255 page->mapping = NULL;
1256 if (memcg_kmem_enabled() && PageKmemcg(page))
1257 __memcg_kmem_uncharge_page(page, order);
1259 bad += check_free_page(page);
1263 page_cpupid_reset_last(page);
1264 page->flags &= ~PAGE_FLAGS_CHECK_AT_PREP;
1265 reset_page_owner(page, order);
1267 if (!PageHighMem(page)) {
1268 debug_check_no_locks_freed(page_address(page),
1270 debug_check_no_obj_freed(page_address(page),
1274 kernel_init_free_pages(page, 1 << order);
1276 kernel_poison_pages(page, 1 << order, 0);
1278 * arch_free_page() can make the page's contents inaccessible. s390
1279 * does this. So nothing which can access the page's contents should
1282 arch_free_page(page, order);
1285 kernel_map_pages(page, 1 << order, 0);
1287 kasan_free_nondeferred_pages(page, order);
1298 static bool free_pcp_prepare(struct page *page)
1300 return free_pages_prepare(page, 0, true);
1303 static bool bulkfree_pcp_prepare(struct page *page)
1306 return check_free_page(page);
1317 static bool free_pcp_prepare(struct page *page)
1320 return free_pages_prepare(page, 0, true);
1322 return free_pages_prepare(page, 0, false);
1325 static bool bulkfree_pcp_prepare(struct page *page)
1327 return check_free_page(page);
1331 static inline void prefetch_buddy(struct page *page)
1333 unsigned long pfn = page_to_pfn(page);
1335 struct page *buddy = page + (buddy_pfn - pfn);
1358 struct page *page, *tmp;
1388 page = list_last_entry(list, struct page, lru);
1390 list_del(&page->lru);
1393 if (bulkfree_pcp_prepare(page))
1396 list_add_tail(&page->lru, &head);
1399 * We are going to put the page back to the global
1408 prefetch_buddy(page);
1417 * page->lru.next will not point to original list.
1419 list_for_each_entry_safe(page, tmp, &head, lru) {
1420 int mt = get_pcppage_migratetype(page);
1421 /* MIGRATE_ISOLATE page should not go to pcplists */
1422 VM_BUG_ON_PAGE(is_migrate_isolate(mt), page);
1425 mt = get_pageblock_migratetype(page);
1427 __free_one_page(page, page_to_pfn(page), zone, 0, mt, FPI_NONE);
1428 trace_mm_page_pcpu_drain(page, 0, mt);
1434 struct page *page, unsigned long pfn,
1441 migratetype = get_pfnblock_migratetype(page, pfn);
1443 __free_one_page(page, pfn, zone, order, migratetype, fpi_flags);
1447 static void __meminit __init_single_page(struct page *page, unsigned long pfn,
1450 mm_zero_struct_page(page);
1451 set_page_links(page, zone, nid, pfn);
1452 init_page_count(page);
1453 page_mapcount_reset(page);
1454 page_cpupid_reset_last(page);
1455 page_kasan_tag_reset(page);
1457 INIT_LIST_HEAD(&page->lru);
1461 set_page_address(page, __va(pfn << PAGE_SHIFT));
1495 * sent to the buddy page allocator.
1504 struct page *page = pfn_to_page(start_pfn);
1509 INIT_LIST_HEAD(&page->lru);
1513 * page is not visible yet so nobody should
1516 __SetPageReserved(page);
1521 static void __free_pages_ok(struct page *page, unsigned int order,
1526 unsigned long pfn = page_to_pfn(page);
1528 if (!free_pages_prepare(page, order, true))
1531 migratetype = get_pfnblock_migratetype(page, pfn);
1534 free_one_page(page_zone(page), page, pfn, order, migratetype,
1539 void __free_pages_core(struct page *page, unsigned int order)
1542 struct page *p = page;
1559 atomic_long_add(nr_pages, &page_zone(page)->managed_pages);
1565 __free_pages_ok(page, order, FPI_TO_TAIL);
1612 void __init memblock_free_pages(struct page *page, unsigned long pfn,
1617 __free_pages_core(page, order);
1627 * Return struct page pointer of start_pfn, or NULL if checks were not passed.
1634 * the first and last page of a pageblock and avoid checking each individual
1635 * page in a pageblock.
1637 struct page *__pageblock_pfn_to_page(unsigned long start_pfn,
1640 struct page *start_page;
1641 struct page *end_page;
1696 struct page *page;
1702 page = pfn_to_page(pfn);
1707 set_pageblock_migratetype(page, MIGRATE_MOVABLE);
1708 __free_pages_core(page, pageblock_order);
1712 for (i = 0; i < nr_pages; i++, page++, pfn++) {
1714 set_pageblock_migratetype(page, MIGRATE_MOVABLE);
1715 __free_pages_core(page, 0);
1730 * Returns true if page needs to be initialized or freed to buddy allocator.
1736 * Then, we check if a current large page is valid by only checking the validity
1774 * Initialize struct pages. We minimize pfn page lookups and scheduler checks
1786 struct page *page = NULL;
1790 page = NULL;
1792 } else if (!page || !(pfn & nr_pgmask)) {
1793 page = pfn_to_page(pfn);
1795 page++;
1797 __init_single_page(page, pfn, zid, nid);
1835 * struct page, then free to buddy allocator, because while we are
1837 * page in __free_one_page()).
1840 * broken along max page order boundaries. This way we will not cause
1841 * any issues with the buddy page computation.
1852 /* First we loop through and initialize the page values */
1946 * pre-grown prior to start of deferred page initialization.
1996 * PAGES_PER_SECTION * sizeof(struct page) bytes.
2107 * on-demand struct page initialization.
2127 void __init init_cma_reserved_pageblock(struct page *page)
2130 struct page *p = page;
2137 set_pageblock_migratetype(page, MIGRATE_CMA);
2141 p = page;
2148 set_page_refcounted(page);
2149 __free_pages(page, pageblock_order);
2152 adjust_managed_page_count(page, pageblock_nr_pages);
2170 static inline void expand(struct zone *zone, struct page *page,
2178 VM_BUG_ON_PAGE(bad_range(zone, &page[size]), &page[size]);
2181 * Mark as guard pages (or page), that will allow to
2183 * Corresponding page table entries will not be touched,
2186 if (set_page_guard(zone, &page[size], high, migratetype))
2189 add_to_free_list(&page[size], zone, high, migratetype);
2190 set_buddy_order(&page[size], high);
2194 static void check_new_page_bad(struct page *page)
2196 if (unlikely(page->flags & __PG_HWPOISON)) {
2198 page_mapcount_reset(page); /* remove PageBuddy */
2202 bad_page(page,
2203 page_bad_reason(page, PAGE_FLAGS_CHECK_AT_PREP));
2207 * This page is about to be returned from the page allocator
2209 static inline int check_new_page(struct page *page)
2211 if (likely(page_expected_state(page,
2215 check_new_page_bad(page);
2231 static inline bool check_pcp_refill(struct page *page)
2234 return check_new_page(page);
2239 static inline bool check_new_pcp(struct page *page)
2241 return check_new_page(page);
2249 static inline bool check_pcp_refill(struct page *page)
2251 return check_new_page(page);
2253 static inline bool check_new_pcp(struct page *page)
2256 return check_new_page(page);
2262 static bool check_new_pages(struct page *page, unsigned int order)
2266 struct page *p = page + i;
2275 inline void post_alloc_hook(struct page *page, unsigned int order,
2278 set_page_private(page, 0);
2279 set_page_refcounted(page);
2281 arch_alloc_page(page, order);
2283 kernel_map_pages(page, 1 << order, 1);
2284 kasan_alloc_pages(page, order);
2285 kernel_poison_pages(page, 1 << order, 1);
2286 set_page_owner(page, order, gfp_flags);
2289 static void prep_new_page(struct page *page, unsigned int order, gfp_t gfp_flags,
2292 post_alloc_hook(page, order, gfp_flags);
2295 kernel_init_free_pages(page, 1 << order);
2298 prep_compound_page(page, order);
2301 * page is set pfmemalloc when ALLOC_NO_WATERMARKS was necessary to
2302 * allocate the page. The expectation is that the caller is taking
2303 * steps that will free more memory. The caller should avoid the page
2307 set_page_pfmemalloc(page);
2309 clear_page_pfmemalloc(page);
2314 * the smallest available page from the freelists
2317 struct page *__rmqueue_smallest(struct zone *zone, unsigned int order,
2322 struct page *page;
2324 /* Find a page of the appropriate size in the preferred list */
2327 page = get_page_from_free_area(area, migratetype);
2328 if (!page)
2330 del_page_from_free_list(page, zone, current_order);
2331 expand(zone, page, order, current_order, migratetype);
2332 set_pcppage_migratetype(page, migratetype);
2333 return page;
2357 static __always_inline struct page *__rmqueue_cma_fallback(struct zone *zone,
2363 static inline struct page *__rmqueue_cma_fallback(struct zone *zone,
2376 struct page *page;
2387 page = pfn_to_page(pfn);
2388 if (!PageBuddy(page)) {
2395 (PageLRU(page) || __PageMovable(page)))
2402 VM_BUG_ON_PAGE(page_to_nid(page) != zone_to_nid(zone), page);
2403 VM_BUG_ON_PAGE(page_zone(page) != zone, page);
2405 order = buddy_order(page);
2406 move_to_free_list(page, zone, order, migratetype);
2414 int move_freepages_block(struct zone *zone, struct page *page,
2422 pfn = page_to_pfn(page);
2436 static void change_pageblock_range(struct page *pageblock_page,
2452 * If we are stealing a relatively large buddy page, it is likely there will
2454 * reclaimable and unmovable allocations, we steal regardless of page size,
2525 static void steal_suitable_fallback(struct zone *zone, struct page *page,
2528 unsigned int current_order = buddy_order(page);
2532 old_block_type = get_pageblock_migratetype(page);
2543 change_pageblock_range(page, current_order, start_type);
2559 free_pages = move_freepages_block(zone, page, start_type,
2593 set_pageblock_migratetype(page, start_type);
2598 move_to_free_list(page, zone, current_order, start_type);
2640 * there are no empty page blocks that contain a page with a suitable order
2642 static void reserve_highatomic_pageblock(struct page *page, struct zone *zone,
2663 mt = get_pageblock_migratetype(page);
2667 set_pageblock_migratetype(page, MIGRATE_HIGHATOMIC);
2668 move_freepages_block(zone, page, MIGRATE_HIGHATOMIC, NULL);
2691 struct page *page;
2709 page = get_page_from_free_area(area, MIGRATE_HIGHATOMIC);
2710 if (!page)
2714 * In page freeing path, migratetype change is racy so
2720 if (is_migrate_highatomic_page(page)) {
2742 set_pageblock_migratetype(page, ac->migratetype);
2743 ret = move_freepages_block(zone, page, ac->migratetype,
2757 * Try finding a free buddy page on the fallback list and put it on the free
2773 struct page *page;
2786 * Find the largest available free page in the other list. This roughly
2801 * steal and split the smallest available page instead of the
2802 * largest available page, because even if the next movable
2827 * when looking for the largest page.
2832 page = get_page_from_free_area(area, fallback_mt);
2834 steal_suitable_fallback(zone, page, alloc_flags, start_migratetype,
2837 trace_mm_page_alloc_extfrag(page, order, current_order,
2844 static __always_inline struct page *
2848 struct page *page = NULL;
2850 page = __rmqueue_smallest(zone, order, migratetype);
2852 if (unlikely(!page) && is_migrate_cma(migratetype)) {
2855 page = __rmqueue_smallest(zone, order, migratetype);
2858 if (unlikely(!page) &&
2862 return page;
2869 static __always_inline struct page *
2873 struct page *page;
2876 page = __rmqueue_with_cma_reuse(zone, order, migratetype, alloc_flags);
2889 page = __rmqueue_cma_fallback(zone, order);
2890 if (page)
2895 page = __rmqueue_smallest(zone, order, migratetype);
2896 if (unlikely(!page)) {
2898 page = __rmqueue_cma_fallback(zone, order);
2900 if (!page && __rmqueue_fallback(zone, order, migratetype,
2905 if (page)
2906 trace_mm_page_alloc_zone_locked(page, order, migratetype);
2907 return page;
2923 struct page *page = __rmqueue(zone, order, migratetype,
2925 if (unlikely(page == NULL))
2928 if (unlikely(check_pcp_refill(page)))
2933 * physical page order. The page is added to the tail of
2935 * is ordered by page number under some conditions. This is
2937 * head, thus also in the physical page order. This is useful
2941 list_add_tail(&page->lru, list);
2943 if (is_migrate_cma(get_pcppage_migratetype(page)))
3145 struct page *page;
3155 page = pfn_to_page(pfn);
3162 if (page_zone(page) != zone)
3165 if (!swsusp_page_is_forbidden(page))
3166 swsusp_unset_page_free(page);
3170 list_for_each_entry(page,
3174 pfn = page_to_pfn(page);
3188 static bool free_unref_page_prepare(struct page *page, unsigned long pfn)
3192 if (!free_pcp_prepare(page))
3195 migratetype = get_pfnblock_migratetype(page, pfn);
3196 set_pcppage_migratetype(page, migratetype);
3200 static void free_unref_page_commit(struct page *page, unsigned long pfn)
3202 struct zone *zone = page_zone(page);
3206 migratetype = get_pcppage_migratetype(page);
3214 * excessively into the page allocator
3218 free_one_page(zone, page, pfn, 0, migratetype,
3226 list_add(&page->lru, &pcp->lists[migratetype]);
3235 * Free a 0-order page
3237 void free_unref_page(struct page *page)
3240 unsigned long pfn = page_to_pfn(page);
3242 if (!free_unref_page_prepare(page, pfn))
3246 free_unref_page_commit(page, pfn);
3255 struct page *page, *next;
3260 list_for_each_entry_safe(page, next, list, lru) {
3261 pfn = page_to_pfn(page);
3262 if (!free_unref_page_prepare(page, pfn))
3263 list_del(&page->lru);
3264 set_page_private(page, pfn);
3268 list_for_each_entry_safe(page, next, list, lru) {
3269 unsigned long pfn = page_private(page);
3271 set_page_private(page, 0);
3272 trace_mm_page_free_batched(page);
3273 free_unref_page_commit(page, pfn);
3289 * split_page takes a non-compound higher-order page, and splits it into
3290 * n (1<<order) sub-pages: page[0..n]
3291 * Each sub-page must be freed individually.
3296 void split_page(struct page *page, unsigned int order)
3300 VM_BUG_ON_PAGE(PageCompound(page), page);
3301 VM_BUG_ON_PAGE(!page_count(page), page);
3304 set_page_refcounted(page + i);
3305 split_page_owner(page, 1 << order);
3306 split_page_memcg(page, 1 << order);
3310 int __isolate_free_page(struct page *page, unsigned int order)
3316 BUG_ON(!PageBuddy(page));
3318 zone = page_zone(page);
3319 mt = get_pageblock_migratetype(page);
3323 * Obey watermarks as if the page was being allocated. We can
3325 * watermark, because we already know our high-order page
3335 /* Remove page from free list */
3337 del_page_from_free_list(page, zone, order);
3340 * Set the pageblock if the isolated page is at least half of a
3344 struct page *endpage = page + (1 << order) - 1;
3345 for (; page < endpage; page += pageblock_nr_pages) {
3346 int mt = get_pageblock_migratetype(page);
3349 set_pageblock_migratetype(page,
3359 * __putback_isolated_page - Return a now-isolated page back where we got it
3360 * @page: Page that was isolated
3361 * @order: Order of the isolated page
3362 * @mt: The page's pageblock's migratetype
3364 * This function is meant to return a page pulled from the free lists via
3367 void __putback_isolated_page(struct page *page, unsigned int order, int mt)
3369 struct zone *zone = page_zone(page);
3374 /* Return isolated page to tail of freelist. */
3375 __free_one_page(page, page_to_pfn(page), zone, order, mt,
3406 /* Remove page from the per-cpu list, caller must protect the list */
3407 static struct page *__rmqueue_pcplist(struct zone *zone, int migratetype,
3412 struct page *page;
3423 page = list_first_entry(list, struct page, lru);
3424 list_del(&page->lru);
3426 } while (check_new_pcp(page));
3428 return page;
3431 /* Lock and remove page from the per-cpu list */
3432 static struct page *rmqueue_pcplist(struct zone *preferred_zone,
3438 struct page *page;
3444 page = __rmqueue_pcplist(zone, migratetype, alloc_flags, pcp, list);
3445 if (page) {
3446 __count_zid_vm_events(PGALLOC, page_zonenum(page), 1);
3450 return page;
3454 * Allocate a page from the given zone. Use pcplists for order-0 allocations.
3457 struct page *rmqueue(struct zone *preferred_zone,
3463 struct page *page;
3473 page = rmqueue_pcplist(preferred_zone, zone, gfp_flags,
3481 * allocate greater than order-1 page units with __GFP_NOFAIL.
3487 page = NULL;
3495 page = __rmqueue_smallest(zone, order, MIGRATE_HIGHATOMIC);
3496 if (page)
3497 trace_mm_page_alloc_zone_locked(page, order, migratetype);
3499 if (!page)
3500 page = __rmqueue(zone, order, migratetype, alloc_flags);
3501 } while (page && check_new_pages(page, order));
3503 if (!page)
3506 get_pcppage_migratetype(page));
3508 __count_zid_vm_events(PGALLOC, page_zonenum(page), 1 << order);
3519 VM_BUG_ON_PAGE(page && bad_range(zone, page), page);
3520 return page;
3627 * one free page of a suitable size. Checking now avoids taking the zone lock
3660 * even if a suitable page happened to be free.
3669 /* For a high-order request, check at least one suitable page is free */
3826 * a page.
3828 static struct page *
3846 struct page *page;
3854 * When allocating a page cache page for writing, we
3942 page = rmqueue(ac->preferred_zoneref->zone, zone, order,
3944 if (page) {
3945 prep_new_page(page, order, gfp_mask, alloc_flags);
3952 reserve_highatomic_pageblock(page, zone, order);
3954 return page;
4022 static inline struct page *
4027 struct page *page;
4029 page = get_page_from_freelist(gfp_mask, order,
4035 if (!page)
4036 page = get_page_from_freelist(gfp_mask, order,
4039 return page;
4042 static inline struct page *
4053 struct page *page;
4074 page = get_page_from_freelist((gfp_mask | __GFP_HARDWALL) &
4077 if (page)
4120 page = __alloc_pages_cpuset_fallback(gfp_mask, order,
4125 return page;
4136 static struct page *
4141 struct page *page = NULL;
4152 prio, &page);
4163 /* Prep a captured page if available */
4164 if (page)
4165 prep_new_page(page, order, gfp_mask, alloc_flags);
4167 /* Try get a page from the freelist if available */
4168 if (!page)
4169 page = get_page_from_freelist(gfp_mask, order, alloc_flags, ac);
4171 if (page) {
4172 struct zone *zone = page_zone(page);
4177 return page;
4269 static inline struct page *
4381 /* Perform direct synchronous page reclaim */
4410 static inline struct page *
4415 struct page *page = NULL;
4423 page = get_page_from_freelist(gfp_mask, order, alloc_flags, ac);
4430 if (!page && !drained) {
4443 return page;
4476 * The caller may dip into page reserves a bit more if the caller
4680 static inline struct page *
4686 struct page *page = NULL;
4740 page = get_page_from_freelist(gfp_mask, order, alloc_flags, ac);
4741 if (page)
4757 page = __alloc_pages_direct_compact(gfp_mask, order,
4761 if (page)
4766 * includes some THP page fault allocations
4820 page = get_page_from_freelist(gfp_mask, order, alloc_flags, ac);
4821 if (page)
4836 page = __alloc_pages_direct_reclaim(gfp_mask, order, alloc_flags, ac,
4841 if (page)
4845 page = __alloc_pages_direct_compact(gfp_mask, order, alloc_flags, ac,
4847 if (page)
4887 page = __alloc_pages_may_oom(gfp_mask, order, ac, &did_some_progress);
4888 if (page)
4945 page = __alloc_pages_cpuset_fallback(gfp_mask, order, ALLOC_HARDER, ac);
4946 if (page)
4954 "page allocation failure: order:%u", order);
4956 return page;
5013 struct page *
5017 struct page *page;
5043 page = get_page_from_freelist(alloc_mask, order, alloc_flags, &ac);
5044 if (likely(page))
5062 page = __alloc_pages_slowpath(alloc_mask, order, &ac);
5065 if (memcg_kmem_enabled() && (gfp_mask & __GFP_ACCOUNT) && page &&
5066 unlikely(__memcg_kmem_charge_page(page, gfp_mask, order) != 0)) {
5067 __free_pages(page, order);
5068 page = NULL;
5071 trace_mm_page_alloc(page, order, alloc_mask, ac.migratetype);
5073 return page;
5084 struct page *page;
5086 page = alloc_pages(gfp_mask & ~__GFP_HIGHMEM, order);
5087 if (!page)
5089 return (unsigned long) page_address(page);
5099 static inline void free_the_page(struct page *page, unsigned int order)
5102 free_unref_page(page);
5104 __free_pages_ok(page, order, FPI_NONE);
5107 void __free_pages(struct page *page, unsigned int order)
5110 int head = PageHead(page);
5112 if (put_page_testzero(page))
5113 free_the_page(page, order);
5116 free_the_page(page + (1 << order), order);
5133 * within a 0 or higher order page. Multiple fragments within that page
5134 * are individually refcounted, in the page's reference counter.
5137 * page fragments. This is used by the network stack and network device
5141 static struct page *__page_frag_cache_refill(struct page_frag_cache *nc,
5144 struct page *page = NULL;
5150 page = alloc_pages_node(NUMA_NO_NODE, gfp_mask,
5152 nc->size = page ? PAGE_FRAG_CACHE_MAX_SIZE : PAGE_SIZE;
5154 if (unlikely(!page))
5155 page = alloc_pages_node(NUMA_NO_NODE, gfp, 0);
5157 nc->va = page ? page_address(page) : NULL;
5160 if (likely(page)) {
5163 struct page *newpage = page;
5172 mod_zone_page_state(page_zone(page), NR_SKB_PAGES, (long)deta);
5176 return page;
5179 void __page_frag_cache_drain(struct page *page, unsigned int count)
5181 VM_BUG_ON_PAGE(page_ref_count(page) == 0, page);
5183 if (page_ref_sub_and_test(page, count)) {
5185 if (likely(page)) {
5186 unsigned int deta = 1U << compound_order(page);
5188 mod_zone_page_state(page_zone(page), NR_SKB_PAGES, -(long)deta);
5191 free_the_page(page, compound_order(page));
5200 struct page *page;
5205 page = __page_frag_cache_refill(nc, gfp_mask);
5206 if (!page)
5213 /* Even if we own the page, we do not use atomic_set().
5216 page_ref_add(page, PAGE_FRAG_CACHE_MAX_SIZE);
5218 /* reset page count bias and offset to start of new frag */
5219 nc->pfmemalloc = page_is_pfmemalloc(page);
5226 page = virt_to_page(nc->va);
5228 if (!page_ref_sub_and_test(page, nc->pagecnt_bias))
5232 free_the_page(page, compound_order(page));
5240 /* OK, page count is 0, we can safely set it */
5241 set_page_count(page, PAGE_FRAG_CACHE_MAX_SIZE + 1);
5243 /* reset page count bias and offset to start of new frag */
5252 * We don't release the cache page because
5268 * Frees a page fragment allocated out of either a compound or order 0 page.
5272 struct page *page = virt_to_head_page(addr);
5274 if (unlikely(put_page_testzero(page))) {
5276 if (likely(page)) {
5277 unsigned int deta = 1U << compound_order(page);
5279 mod_zone_page_state(page_zone(page), NR_SKB_PAGES, -(long)deta);
5282 free_the_page(page, compound_order(page));
5346 struct page *p;
5452 * Not all the page cache can be freed, otherwise the system will
5453 * start swapping. Assume at least half of the page cache, or the
6065 * to prevent any IRQ handler from calling into the page allocator
6130 * F.e. the percpu allocator needs the page allocator which
6217 struct page *page;
6227 * ZONE_DEVICE page initialization until after we have released
6252 page = pfn_to_page(pfn);
6253 __init_single_page(page, pfn, zone, nid);
6255 __SetPageReserved(page);
6263 set_pageblock_migratetype(page, migratetype);
6297 struct page *page = pfn_to_page(pfn);
6299 __init_single_page(page, pfn, zone_idx, nid);
6302 * Mark page reserved as it will need to wait for onlining
6308 __SetPageReserved(page);
6312 * and zone_device_data. It is a bug if a ZONE_DEVICE page is
6315 page->pgmap = pgmap;
6316 page->zone_device_data = NULL;
6329 set_pageblock_migratetype(page, MIGRATE_MOVABLE);
6364 * - zone and node links point to zone and node that span the page if the
6493 * of pages of one half of the possible page colors
6505 * of contiguous memory as there's no hardware page translation to
6680 * get_pfn_range_for_nid - Return the start and end page frames for a node
6685 * It returns the start and end page frame of a node based on information
6823 * absent_pages_in_range - Return number of page frames in holes within a range
6835 /* Return the number of page frames in holes in a zone on a node */
6987 * Assume the largest contiguous order of interest is a huge page.
7017 * populated regions may not be naturally aligned on page boundary.
7024 return PAGE_ALIGN(pages * sizeof(struct page)) >> PAGE_SHIFT;
7190 struct page *map;
7199 size = (end - start) * sizeof(struct page);
7611 * Using the page ranges provided by memblock_set_node(), the size of each
7764 void adjust_managed_page_count(struct page *page, long count)
7766 atomic_long_add(count, &page_zone(page)->managed_pages);
7769 if (PageHighMem(page))
7783 struct page *page = virt_to_page(pos);
7793 direct_map_addr = page_address(page);
7797 free_reserved_page(page);
7808 void free_highmem_page(struct page *page)
7810 __free_reserved_page(page);
7812 atomic_long_inc(&page_zone(page)->managed_pages);
8039 * deltas control async page reclaim, and so should
8461 * Returns a page without holding a reference. If the caller wants to
8462 * dereference that page (e.g., dumping), it has to make sure that it
8466 struct page *has_unmovable_pages(struct zone *zone, struct page *page,
8470 unsigned long pfn = page_to_pfn(page);
8473 if (is_migrate_cma_page(page)) {
8482 return page;
8489 page = pfn_to_page(pfn + iter);
8497 if (PageReserved(page))
8498 return page;
8512 * handle each tail page individually in migration.
8514 if (PageHuge(page) || PageTransCompound(page)) {
8515 struct page *head = compound_head(page);
8518 if (PageHuge(page)) {
8520 return page;
8522 return page;
8525 skip_pages = compound_nr(head) - (page - head);
8531 * We can't use page_count without pin a page
8532 * because another CPU can free compound page.
8534 * because their page->_refcount is zero at all time.
8536 if (!page_ref_count(page)) {
8537 if (PageBuddy(page))
8538 iter += (1 << buddy_order(page)) - 1;
8543 * The HWPoisoned page may be not in buddy system, and
8546 if ((flags & MEMORY_OFFLINE) && PageHWPoison(page))
8559 if ((flags & MEMORY_OFFLINE) && PageOffline(page))
8562 if (__PageMovable(page) || PageLRU(page))
8570 return page;
8680 * have different sizes, and due to the way page allocator
8682 * that page allocator won't try to merge buddies from
8689 * range back to page allocator as MIGRATE_ISOLATE.
8691 * When this is done, we take the pages in range from page
8693 * page allocator will never consider using them.
8698 * put back to page allocator so that buddy can use them.
8707 * In case of -EBUSY, we'd like to know which page causes problem.
8709 * which will report the busy page.
8724 * more, all pages in [start, end) are free in page allocator.
8726 * [start, end) (that is remove them from page allocator).
8730 * page allocator holds, ie. they can be part of higher order
8754 * outer_start page could be small order buddy page and
8755 * it doesn't include start page. Adjust outer_start
8756 * in this case to report failed page properly
8804 struct page *page;
8807 page = pfn_to_online_page(i);
8808 if (!page)
8811 if (page_zone(page) != z)
8814 if (PageReserved(page))
8817 if (page_count(page) > 0)
8820 if (PageHuge(page))
8846 * The allocated memory is always aligned to a page boundary. If nr_pages is a
8851 * __free_page() on each allocated page.
8855 struct page *alloc_contig_pages(unsigned long nr_pages, gfp_t gfp_mask,
8898 struct page *page = pfn_to_page(pfn);
8900 count += page_count(page) != 1;
8901 __free_page(page);
8909 * page high values need to be recalulated.
8945 struct page *page;
8954 page = pfn_to_page(pfn);
8956 * The HWPoisoned page may be not in buddy system, and
8959 if (unlikely(!PageBuddy(page) && PageHWPoison(page))) {
8967 if (PageOffline(page)) {
8968 BUG_ON(page_count(page));
8969 BUG_ON(PageBuddy(page));
8974 BUG_ON(page_count(page));
8975 BUG_ON(!PageBuddy(page));
8976 order = buddy_order(page);
8977 del_page_from_free_list(page, zone, order);
8984 bool is_free_buddy_page(struct page *page)
8986 struct zone *zone = page_zone(page);
8987 unsigned long pfn = page_to_pfn(page);
8993 struct page *page_head = page - (pfn & ((1 << order) - 1));
9005 * Break down a higher-order page in sub-pages, and keep our target out of
9008 static void break_down_buddy_pages(struct zone *zone, struct page *page,
9009 struct page *target, int low, int high,
9013 struct page *current_buddy, *next_page;
9019 if (target >= &page[size]) {
9020 next_page = page + size;
9021 current_buddy = page;
9023 next_page = page;
9024 current_buddy = page + size;
9026 page = next_page;
9039 * Take a page that will be marked as poisoned off the buddy allocator.
9041 bool take_page_off_buddy(struct page *page)
9043 struct zone *zone = page_zone(page);
9044 unsigned long pfn = page_to_pfn(page);
9051 struct page *page_head = page - (pfn & ((1 << order) - 1));
9060 break_down_buddy_pages(zone, page_head, page, 0,