Lines Matching refs:memcg
134 * memcg which the event belongs to.
136 struct mem_cgroup *memcg;
150 int (*register_event)(struct mem_cgroup *memcg,
157 void (*unregister_event)(struct mem_cgroup *memcg,
169 static void mem_cgroup_threshold(struct mem_cgroup *memcg);
170 static void mem_cgroup_oom_notify(struct mem_cgroup *memcg);
241 struct vmpressure *memcg_to_vmpressure(struct mem_cgroup *memcg)
243 if (!memcg)
244 memcg = root_mem_cgroup;
245 return &memcg->vmpressure;
259 struct mem_cgroup *memcg;
273 * 3) CPU1: a process from another memcg is allocating something,
289 memcg = obj_cgroup_memcg(objcg);
291 __memcg_kmem_uncharge(memcg, nr_pages);
293 mem_cgroup_put(memcg);
319 static void memcg_reparent_objcgs(struct mem_cgroup *memcg,
324 objcg = rcu_replace_pointer(memcg->objcg, NULL, true);
329 xchg(&objcg->memcg, parent);
334 list_for_each_entry(iter, &memcg->objcg_list, list) {
336 xchg(&iter->memcg, parent);
337 css_put(&memcg->css);
339 list_splice(&memcg->objcg_list, &parent->objcg_list);
406 static int memcg_expand_one_shrinker_map(struct mem_cgroup *memcg,
416 mem_cgroup_nodeinfo(memcg, nid)->shrinker_map, true);
417 /* Not yet online memcg */
429 rcu_assign_pointer(memcg->nodeinfo[nid]->shrinker_map, new);
436 static void memcg_free_shrinker_maps(struct mem_cgroup *memcg)
442 if (mem_cgroup_is_root(memcg))
446 pn = mem_cgroup_nodeinfo(memcg, nid);
454 static int memcg_alloc_shrinker_maps(struct mem_cgroup *memcg)
459 if (mem_cgroup_is_root(memcg))
467 memcg_free_shrinker_maps(memcg);
471 rcu_assign_pointer(memcg->nodeinfo[nid]->shrinker_map, map);
481 struct mem_cgroup *memcg;
492 for_each_mem_cgroup(memcg) {
493 if (mem_cgroup_is_root(memcg))
495 ret = memcg_expand_one_shrinker_map(memcg, size, old_size);
497 mem_cgroup_iter_break(NULL, memcg);
508 void memcg_set_shrinker_bit(struct mem_cgroup *memcg, int nid, int shrinker_id)
510 if (shrinker_id >= 0 && memcg && !mem_cgroup_is_root(memcg)) {
514 map = rcu_dereference(memcg->nodeinfo[nid]->shrinker_map);
523 * mem_cgroup_css_from_page - css of the memcg associated with a page
526 * If memcg is bound to the default hierarchy, css of the memcg associated
530 * If memcg is bound to a traditional hierarchy, the css of root_mem_cgroup
535 struct mem_cgroup *memcg;
537 memcg = page->mem_cgroup;
539 if (!memcg || !cgroup_subsys_on_dfl(memory_cgrp_subsys))
540 memcg = root_mem_cgroup;
542 return &memcg->css;
546 * page_cgroup_ino - return inode number of the memcg a page is charged to
560 struct mem_cgroup *memcg;
564 memcg = page->mem_cgroup;
567 * The lowest bit set means that memcg isn't a valid
568 * memcg pointer, but a obj_cgroups pointer.
572 if ((unsigned long) memcg & 0x1UL)
573 memcg = NULL;
575 while (memcg && !(memcg->css.flags & CSS_ONLINE))
576 memcg = parent_mem_cgroup(memcg);
577 if (memcg)
578 ino = cgroup_ino(memcg->css.cgroup);
584 mem_cgroup_page_nodeinfo(struct mem_cgroup *memcg, struct page *page)
588 return memcg->nodeinfo[nid];
668 static unsigned long soft_limit_excess(struct mem_cgroup *memcg)
671 struct mem_cgroup_per_node *mz = mem_cgroup_nodeinfo(memcg, 0);
677 unsigned long nr_pages = page_counter_read(&memcg->memory);
679 unsigned long soft_limit = READ_ONCE(memcg->soft_limit);
688 static void mem_cgroup_update_tree(struct mem_cgroup *memcg, struct page *page)
701 for (; memcg; memcg = parent_mem_cgroup(memcg)) {
702 mz = mem_cgroup_page_nodeinfo(memcg, page);
703 excess = soft_limit_excess(memcg);
725 static void mem_cgroup_remove_from_trees(struct mem_cgroup *memcg)
732 mz = mem_cgroup_nodeinfo(memcg, nid);
757 if (!soft_limit_excess(mz->memcg) ||
758 !css_tryget(&mz->memcg->css))
777 * @memcg: the memory cgroup
781 void __mod_memcg_state(struct mem_cgroup *memcg, int idx, int val)
791 x = val + __this_cpu_read(memcg->vmstats_percpu->stat[idx]);
799 __this_cpu_add(memcg->vmstats_local->stat[idx], x);
800 for (mi = memcg; mi; mi = parent_mem_cgroup(mi))
804 __this_cpu_write(memcg->vmstats_percpu->stat[idx], x);
812 parent = parent_mem_cgroup(pn->memcg);
822 struct mem_cgroup *memcg;
826 memcg = pn->memcg;
828 /* Update memcg */
829 __mod_memcg_state(memcg, idx, val);
865 /* Update memcg and lruvec */
878 struct mem_cgroup *memcg;
882 memcg = mem_cgroup_from_obj(p);
885 * Untracked pages have no memcg, no lruvec. Update only the
886 * node. If we reparent the slab objects to the root memcg,
887 * when we free the slab object, we need to update the per-memcg
888 * vmstats to keep it correct for the root memcg.
890 if (!memcg) {
893 lruvec = mem_cgroup_lruvec(memcg, pgdat);
901 struct mem_cgroup *memcg;
904 memcg = mem_cgroup_from_obj(p);
905 if (memcg)
906 mod_memcg_state(memcg, idx, val);
912 * @memcg: the memory cgroup
916 void __count_memcg_events(struct mem_cgroup *memcg, enum vm_event_item idx,
924 if (!memcg)
928 x = count + __this_cpu_read(memcg->vmstats_percpu->events[idx]);
936 __this_cpu_add(memcg->vmstats_local->events[idx], x);
937 for (mi = memcg; mi; mi = parent_mem_cgroup(mi))
941 __this_cpu_write(memcg->vmstats_percpu->events[idx], x);
944 static unsigned long memcg_events(struct mem_cgroup *memcg, int event)
946 return atomic_long_read(&memcg->vmevents[event]);
949 static unsigned long memcg_events_local(struct mem_cgroup *memcg, int event)
955 x += per_cpu(memcg->vmstats_local->events[event], cpu);
959 static void mem_cgroup_charge_statistics(struct mem_cgroup *memcg,
965 __count_memcg_events(memcg, PGPGIN, 1);
967 __count_memcg_events(memcg, PGPGOUT, 1);
971 __this_cpu_add(memcg->vmstats_percpu->nr_page_events, nr_pages);
974 static bool mem_cgroup_event_ratelimit(struct mem_cgroup *memcg,
979 val = __this_cpu_read(memcg->vmstats_percpu->nr_page_events);
980 next = __this_cpu_read(memcg->vmstats_percpu->targets[target]);
993 __this_cpu_write(memcg->vmstats_percpu->targets[target], next);
1003 static void memcg_check_events(struct mem_cgroup *memcg, struct page *page)
1006 if (unlikely(mem_cgroup_event_ratelimit(memcg,
1010 do_softlimit = mem_cgroup_event_ratelimit(memcg,
1012 mem_cgroup_threshold(memcg);
1014 mem_cgroup_update_tree(memcg, page);
1033 * get_mem_cgroup_from_mm: Obtain a reference on given mm_struct's memcg.
1034 * @mm: mm from which memcg should be extracted. It can be NULL.
1036 * Obtain a reference on mm->memcg and returns it if successful. Otherwise
1042 struct mem_cgroup *memcg;
1055 memcg = root_mem_cgroup;
1057 memcg = mem_cgroup_from_task(rcu_dereference(mm->owner));
1058 if (unlikely(!memcg))
1059 memcg = root_mem_cgroup;
1061 } while (!css_tryget(&memcg->css));
1063 return memcg;
1068 * get_mem_cgroup_from_page: Obtain a reference on given page's memcg.
1069 * @page: page from which memcg should be extracted.
1071 * Obtain a reference on page->memcg and returns it if successful. Otherwise
1076 struct mem_cgroup *memcg = page->mem_cgroup;
1082 /* Page should not get uncharged and freed memcg under us. */
1083 if (!memcg || WARN_ON_ONCE(!css_tryget(&memcg->css)))
1084 memcg = root_mem_cgroup;
1086 return memcg;
1100 struct mem_cgroup *memcg;
1103 memcg = active_memcg();
1104 /* remote memcg must hold a ref. */
1105 if (memcg && WARN_ON_ONCE(!css_tryget(&memcg->css)))
1106 memcg = root_mem_cgroup;
1109 return memcg;
1114 /* Allow remote memcg charging from any context. */
1126 * If active memcg is set, do not fallback to current->mm->memcg.
1142 * @prev: previously returned memcg, NULL on first invocation
1162 struct mem_cgroup *memcg = NULL;
1229 memcg = mem_cgroup_from_css(css);
1237 memcg = NULL;
1246 (void)cmpxchg(&iter->position, pos, memcg);
1251 if (!memcg)
1263 return memcg;
1296 struct mem_cgroup *memcg = dead_memcg;
1300 __invalidate_reclaim_iterators(memcg, dead_memcg);
1301 last = memcg;
1302 } while ((memcg = parent_mem_cgroup(memcg)));
1317 * @memcg: hierarchy root
1321 * This function iterates over tasks attached to @memcg or to any of its
1328 int mem_cgroup_scan_tasks(struct mem_cgroup *memcg,
1334 BUG_ON(memcg == root_mem_cgroup);
1336 for_each_mem_cgroup_tree(iter, memcg) {
1345 mem_cgroup_iter_break(memcg, iter);
1363 struct mem_cgroup *memcg;
1378 memcg = page->mem_cgroup;
1383 if (!memcg)
1384 memcg = root_mem_cgroup;
1386 mz = mem_cgroup_page_nodeinfo(memcg, page);
1444 * @memcg: the memory cgroup
1449 static unsigned long mem_cgroup_margin(struct mem_cgroup *memcg)
1455 count = page_counter_read(&memcg->memory);
1456 limit = READ_ONCE(memcg->memory.max);
1461 count = page_counter_read(&memcg->memsw);
1462 limit = READ_ONCE(memcg->memsw.max);
1479 static bool mem_cgroup_under_move(struct mem_cgroup *memcg)
1494 ret = mem_cgroup_is_descendant(from, memcg) ||
1495 mem_cgroup_is_descendant(to, memcg);
1501 static bool mem_cgroup_wait_acct_move(struct mem_cgroup *memcg)
1504 if (mem_cgroup_under_move(memcg)) {
1581 static char *memory_stat_format(struct mem_cgroup *memcg)
1604 size = memcg_page_state(memcg, memory_stats[i].idx);
1609 size = memcg_page_state(memcg, NR_SLAB_RECLAIMABLE_B) +
1610 memcg_page_state(memcg, NR_SLAB_UNRECLAIMABLE_B);
1618 memcg_events(memcg, PGFAULT));
1620 memcg_events(memcg, PGMAJFAULT));
1622 memcg_events(memcg, PGREFILL));
1624 memcg_events(memcg, PGSCAN_KSWAPD) +
1625 memcg_events(memcg, PGSCAN_DIRECT));
1627 memcg_events(memcg, PGSTEAL_KSWAPD) +
1628 memcg_events(memcg, PGSTEAL_DIRECT));
1630 memcg_events(memcg, PGACTIVATE));
1632 memcg_events(memcg, PGDEACTIVATE));
1634 memcg_events(memcg, PGLAZYFREE));
1636 memcg_events(memcg, PGLAZYFREED));
1640 memcg_events(memcg, THP_FAULT_ALLOC));
1642 memcg_events(memcg, THP_COLLAPSE_ALLOC));
1655 * @memcg: The memory cgroup that went over limit
1658 * NOTE: @memcg and @p's mem_cgroup can be different when hierarchy is
1661 void mem_cgroup_print_oom_context(struct mem_cgroup *memcg, struct task_struct *p)
1665 if (memcg) {
1667 pr_cont_cgroup_path(memcg->css.cgroup);
1680 * @memcg: The memory cgroup that went over limit
1682 void mem_cgroup_print_oom_meminfo(struct mem_cgroup *memcg)
1687 K((u64)page_counter_read(&memcg->memory)),
1688 K((u64)READ_ONCE(memcg->memory.max)), memcg->memory.failcnt);
1691 K((u64)page_counter_read(&memcg->swap)),
1692 K((u64)READ_ONCE(memcg->swap.max)), memcg->swap.failcnt);
1695 K((u64)page_counter_read(&memcg->memsw)),
1696 K((u64)memcg->memsw.max), memcg->memsw.failcnt);
1698 K((u64)page_counter_read(&memcg->kmem)),
1699 K((u64)memcg->kmem.max), memcg->kmem.failcnt);
1703 pr_cont_cgroup_path(memcg->css.cgroup);
1705 buf = memory_stat_format(memcg);
1713 * Return the memory (and swap, if configured) limit for a memcg.
1715 unsigned long mem_cgroup_get_max(struct mem_cgroup *memcg)
1717 unsigned long max = READ_ONCE(memcg->memory.max);
1720 if (mem_cgroup_swappiness(memcg))
1721 max += min(READ_ONCE(memcg->swap.max),
1724 if (mem_cgroup_swappiness(memcg)) {
1726 unsigned long swap = READ_ONCE(memcg->memsw.max) - max;
1734 unsigned long mem_cgroup_size(struct mem_cgroup *memcg)
1736 return page_counter_read(&memcg->memory);
1739 static bool mem_cgroup_out_of_memory(struct mem_cgroup *memcg, gfp_t gfp_mask,
1745 .memcg = memcg,
1754 if (mem_cgroup_margin(memcg) >= (1 << order))
1830 static bool mem_cgroup_oom_trylock(struct mem_cgroup *memcg)
1836 for_each_mem_cgroup_tree(iter, memcg) {
1843 mem_cgroup_iter_break(memcg, iter);
1854 for_each_mem_cgroup_tree(iter, memcg) {
1856 mem_cgroup_iter_break(memcg, iter);
1869 static void mem_cgroup_oom_unlock(struct mem_cgroup *memcg)
1875 for_each_mem_cgroup_tree(iter, memcg)
1880 static void mem_cgroup_mark_under_oom(struct mem_cgroup *memcg)
1885 for_each_mem_cgroup_tree(iter, memcg)
1890 static void mem_cgroup_unmark_under_oom(struct mem_cgroup *memcg)
1895 * Be careful about under_oom underflows becase a child memcg
1899 for_each_mem_cgroup_tree(iter, memcg)
1908 struct mem_cgroup *memcg;
1920 oom_wait_memcg = oom_wait_info->memcg;
1928 static void memcg_oom_recover(struct mem_cgroup *memcg)
1938 if (memcg && memcg->under_oom)
1939 __wake_up(&memcg_oom_waitq, TASK_NORMAL, 0, memcg);
1949 static enum oom_status mem_cgroup_oom(struct mem_cgroup *memcg, gfp_t mask, int order)
1957 memcg_memory_event(memcg, MEMCG_OOM);
1977 if (memcg->oom_kill_disable) {
1980 css_get(&memcg->css);
1981 current->memcg_in_oom = memcg;
1988 mem_cgroup_mark_under_oom(memcg);
1990 locked = mem_cgroup_oom_trylock(memcg);
1993 mem_cgroup_oom_notify(memcg);
1995 mem_cgroup_unmark_under_oom(memcg);
1996 if (mem_cgroup_out_of_memory(memcg, mask, order))
2002 mem_cgroup_oom_unlock(memcg);
2008 * mem_cgroup_oom_synchronize - complete memcg OOM handling
2011 * This has to be called at the end of a page fault if the memcg OOM
2021 * Returns %true if an ongoing memcg OOM situation was detected and
2026 struct mem_cgroup *memcg = current->memcg_in_oom;
2031 if (!memcg)
2037 owait.memcg = memcg;
2044 mem_cgroup_mark_under_oom(memcg);
2046 locked = mem_cgroup_oom_trylock(memcg);
2049 mem_cgroup_oom_notify(memcg);
2051 if (locked && !memcg->oom_kill_disable) {
2052 mem_cgroup_unmark_under_oom(memcg);
2054 mem_cgroup_out_of_memory(memcg, current->memcg_oom_gfp_mask,
2058 mem_cgroup_unmark_under_oom(memcg);
2063 mem_cgroup_oom_unlock(memcg);
2069 memcg_oom_recover(memcg);
2073 css_put(&memcg->css);
2080 * @oom_domain: memcg in case of memcg OOM, NULL in case of system-wide OOM
2085 * Caller has to call mem_cgroup_put() on the returned non-NULL memcg.
2091 struct mem_cgroup *memcg;
2101 memcg = mem_cgroup_from_task(victim);
2102 if (memcg == root_mem_cgroup)
2110 if (unlikely(!mem_cgroup_is_descendant(memcg, oom_domain)))
2118 for (; memcg; memcg = parent_mem_cgroup(memcg)) {
2119 if (memcg->oom_group)
2120 oom_group = memcg;
2122 if (memcg == oom_domain)
2134 void mem_cgroup_print_oom_group(struct mem_cgroup *memcg)
2137 pr_cont_cgroup_path(memcg->css.cgroup);
2148 * It ensures lifetime of the returned memcg. Caller is responsible
2155 struct mem_cgroup *memcg;
2160 * path can get away without acquiring the memcg->move_lock
2163 * The RCU lock also protects the memcg from being freed when
2174 memcg = head->mem_cgroup;
2175 if (unlikely(!memcg))
2178 if (atomic_read(&memcg->moving_account) <= 0)
2179 return memcg;
2181 spin_lock_irqsave(&memcg->move_lock, flags);
2182 if (memcg != head->mem_cgroup) {
2183 spin_unlock_irqrestore(&memcg->move_lock, flags);
2192 memcg->move_lock_task = current;
2193 memcg->move_lock_flags = flags;
2195 return memcg;
2200 * __unlock_page_memcg - unlock and unpin a memcg
2201 * @memcg: the memcg
2203 * Unlock and unpin a memcg returned by lock_page_memcg().
2205 void __unlock_page_memcg(struct mem_cgroup *memcg)
2207 if (memcg && memcg->move_lock_task == current) {
2208 unsigned long flags = memcg->move_lock_flags;
2210 memcg->move_lock_task = NULL;
2211 memcg->move_lock_flags = 0;
2213 spin_unlock_irqrestore(&memcg->move_lock, flags);
2265 * @memcg: memcg to consume from.
2268 * The charges will only happen if @memcg matches the current cpu's memcg
2274 static bool consume_stock(struct mem_cgroup *memcg, unsigned int nr_pages)
2286 if (memcg == stock->cached && stock->nr_pages >= nr_pages) {
2340 static void refill_stock(struct mem_cgroup *memcg, unsigned int nr_pages)
2348 if (stock->cached != memcg) { /* reset if necessary */
2350 css_get(&memcg->css);
2351 stock->cached = memcg;
2381 struct mem_cgroup *memcg;
2385 memcg = stock->cached;
2386 if (memcg && stock->nr_pages &&
2387 mem_cgroup_is_descendant(memcg, root_memcg))
2408 struct mem_cgroup *memcg, *mi;
2413 for_each_mem_cgroup(memcg) {
2420 x = this_cpu_xchg(memcg->vmstats_percpu->stat[i], 0);
2422 for (mi = memcg; mi; mi = parent_mem_cgroup(mi))
2423 atomic_long_add(x, &memcg->vmstats[i]);
2431 pn = mem_cgroup_nodeinfo(memcg, nid);
2443 x = this_cpu_xchg(memcg->vmstats_percpu->events[i], 0);
2445 for (mi = memcg; mi; mi = parent_mem_cgroup(mi))
2446 atomic_long_add(x, &memcg->vmevents[i]);
2453 static unsigned long reclaim_high(struct mem_cgroup *memcg,
2462 if (page_counter_read(&memcg->memory) <=
2463 READ_ONCE(memcg->memory.high))
2466 memcg_memory_event(memcg, MEMCG_HIGH);
2469 nr_reclaimed += try_to_free_mem_cgroup_pages(memcg, nr_pages,
2472 } while ((memcg = parent_mem_cgroup(memcg)) &&
2473 !mem_cgroup_is_root(memcg));
2480 struct mem_cgroup *memcg;
2482 memcg = container_of(work, struct mem_cgroup, high_work);
2483 reclaim_high(memcg, MEMCG_CHARGE_BATCH, GFP_KERNEL);
2557 static u64 mem_find_max_overage(struct mem_cgroup *memcg)
2562 overage = calculate_overage(page_counter_read(&memcg->memory),
2563 READ_ONCE(memcg->memory.high));
2565 } while ((memcg = parent_mem_cgroup(memcg)) &&
2566 !mem_cgroup_is_root(memcg));
2571 static u64 swap_find_max_overage(struct mem_cgroup *memcg)
2576 overage = calculate_overage(page_counter_read(&memcg->swap),
2577 READ_ONCE(memcg->swap.high));
2579 memcg_memory_event(memcg, MEMCG_SWAP_HIGH);
2581 } while ((memcg = parent_mem_cgroup(memcg)) &&
2582 !mem_cgroup_is_root(memcg));
2591 static unsigned long calculate_high_delay(struct mem_cgroup *memcg,
2604 * memcg in question makes it clear that it has no intention of stopping
2634 struct mem_cgroup *memcg;
2640 memcg = get_mem_cgroup_from_mm(current->mm);
2653 nr_reclaimed = reclaim_high(memcg,
2661 penalty_jiffies = calculate_high_delay(memcg, nr_pages,
2662 mem_find_max_overage(memcg));
2664 penalty_jiffies += calculate_high_delay(memcg, nr_pages,
2665 swap_find_max_overage(memcg));
2675 * Don't sleep if the amount of jiffies this memcg owes us is so low
2703 css_put(&memcg->css);
2706 static int try_charge(struct mem_cgroup *memcg, gfp_t gfp_mask,
2720 if (mem_cgroup_is_root(memcg))
2723 if (consume_stock(memcg, nr_pages))
2727 page_counter_try_charge(&memcg->memsw, batch, &counter)) {
2728 if (page_counter_try_charge(&memcg->memory, batch, &counter))
2731 page_counter_uncharge(&memcg->memsw, batch);
2817 * keep retrying as long as the memcg oom killer is able to make
2837 page_counter_charge(&memcg->memory, nr_pages);
2839 page_counter_charge(&memcg->memsw, nr_pages);
2845 refill_stock(memcg, batch - nr_pages);
2851 * GFP_KERNEL can consistently be used during reclaim. @memcg is
2859 mem_high = page_counter_read(&memcg->memory) >
2860 READ_ONCE(memcg->memory.high);
2861 swap_high = page_counter_read(&memcg->swap) >
2862 READ_ONCE(memcg->swap.high);
2867 schedule_work(&memcg->high_work);
2887 } while ((memcg = parent_mem_cgroup(memcg)));
2893 static void cancel_charge(struct mem_cgroup *memcg, unsigned int nr_pages)
2895 if (mem_cgroup_is_root(memcg))
2898 page_counter_uncharge(&memcg->memory, nr_pages);
2900 page_counter_uncharge(&memcg->memsw, nr_pages);
2904 static void commit_charge(struct page *page, struct mem_cgroup *memcg)
2915 page->mem_cgroup = memcg;
2951 * The caller must ensure the memcg lifetime, e.g. by taking rcu_read_lock(),
2969 * from a valid memcg pointer to objcg vector or back.
2998 struct mem_cgroup *memcg;
3005 memcg = active_memcg();
3007 memcg = mem_cgroup_from_task(current);
3009 for (; memcg != root_mem_cgroup; memcg = parent_mem_cgroup(memcg)) {
3010 objcg = rcu_dereference(memcg->objcg);
3064 * __memcg_kmem_charge: charge a number of kernel pages to a memcg
3065 * @memcg: memory cgroup to charge
3071 int __memcg_kmem_charge(struct mem_cgroup *memcg, gfp_t gfp,
3077 ret = try_charge(memcg, gfp, nr_pages);
3082 !page_counter_try_charge(&memcg->kmem, nr_pages, &counter)) {
3090 page_counter_charge(&memcg->kmem, nr_pages);
3093 cancel_charge(memcg, nr_pages);
3100 * __memcg_kmem_uncharge: uncharge a number of kernel pages from a memcg
3101 * @memcg: memcg to uncharge
3104 void __memcg_kmem_uncharge(struct mem_cgroup *memcg, unsigned int nr_pages)
3107 page_counter_uncharge(&memcg->kmem, nr_pages);
3109 refill_stock(memcg, nr_pages);
3122 struct mem_cgroup *memcg;
3125 memcg = get_mem_cgroup_from_current();
3126 if (memcg && !mem_cgroup_is_root(memcg)) {
3127 ret = __memcg_kmem_charge(memcg, gfp, 1 << order);
3129 page->mem_cgroup = memcg;
3133 css_put(&memcg->css);
3145 struct mem_cgroup *memcg = page->mem_cgroup;
3148 if (!memcg)
3151 VM_BUG_ON_PAGE(mem_cgroup_is_root(memcg), page);
3152 __memcg_kmem_uncharge(memcg, nr_pages);
3154 css_put(&memcg->css);
3192 struct mem_cgroup *memcg;
3196 memcg = obj_cgroup_memcg(old);
3197 if (unlikely(!css_tryget(&memcg->css)))
3201 __memcg_kmem_uncharge(memcg, nr_pages);
3202 css_put(&memcg->css);
3206 * The leftover is flushed to the centralized per-memcg value.
3226 struct mem_cgroup *memcg;
3229 memcg = obj_cgroup_memcg(stock->cached_objcg);
3230 if (memcg && mem_cgroup_is_descendant(memcg, root_memcg))
3261 struct mem_cgroup *memcg;
3269 * In theory, memcg->nr_charged_bytes can have enough
3271 * flushing memcg->nr_charged_bytes requires two atomic
3272 * operations, and memcg->nr_charged_bytes can't be big,
3274 * memcg->nr_charged_bytes will be flushed in
3280 memcg = obj_cgroup_memcg(objcg);
3281 if (unlikely(!css_tryget(&memcg->css)))
3291 ret = __memcg_kmem_charge(memcg, gfp, nr_pages);
3295 css_put(&memcg->css);
3311 struct mem_cgroup *memcg = head->mem_cgroup;
3315 if (mem_cgroup_disabled() || !memcg)
3319 head[i].mem_cgroup = memcg;
3323 css_get_many(&memcg->css, nr - 1);
3366 static int mem_cgroup_resize_max(struct mem_cgroup *memcg,
3373 struct page_counter *counter = memsw ? &memcg->memsw : &memcg->memory;
3386 limits_invariant = memsw ? max >= READ_ONCE(memcg->memory.max) :
3387 max <= memcg->memsw.max;
3402 drain_all_stock(memcg);
3407 if (!try_to_free_mem_cgroup_pages(memcg, 1,
3415 memcg_oom_recover(memcg);
3459 reclaimed = mem_cgroup_soft_reclaim(mz->memcg, pgdat,
3474 excess = soft_limit_excess(mz->memcg);
3486 css_put(&mz->memcg->css);
3499 css_put(&next_mz->memcg->css);
3504 * Test whether @memcg has children, dead or alive. Note that this
3505 * function doesn't care whether @memcg has use_hierarchy enabled and
3509 static inline bool memcg_has_children(struct mem_cgroup *memcg)
3514 ret = css_next_child(NULL, &memcg->css);
3520 * Reclaims as many pages from the given memcg as possible.
3522 * Caller is responsible for holding css reference for memcg.
3524 static int mem_cgroup_force_empty(struct mem_cgroup *memcg)
3531 drain_all_stock(memcg);
3534 while (nr_retries && page_counter_read(&memcg->memory)) {
3540 progress = try_to_free_mem_cgroup_pages(memcg, 1,
3557 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
3559 if (mem_cgroup_is_root(memcg))
3561 return mem_cgroup_force_empty(memcg) ?: nbytes;
3574 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
3575 struct mem_cgroup *parent_memcg = mem_cgroup_from_css(memcg->css.parent);
3577 if (memcg->use_hierarchy == val)
3590 if (!memcg_has_children(memcg))
3591 memcg->use_hierarchy = val;
3600 static unsigned long mem_cgroup_usage(struct mem_cgroup *memcg, bool swap)
3604 if (mem_cgroup_is_root(memcg)) {
3605 val = memcg_page_state(memcg, NR_FILE_PAGES) +
3606 memcg_page_state(memcg, NR_ANON_MAPPED);
3608 val += memcg_page_state(memcg, MEMCG_SWAP);
3611 val = page_counter_read(&memcg->memory);
3613 val = page_counter_read(&memcg->memsw);
3629 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
3634 counter = &memcg->memory;
3637 counter = &memcg->memsw;
3640 counter = &memcg->kmem;
3643 counter = &memcg->tcpmem;
3651 if (counter == &memcg->memory)
3652 return (u64)mem_cgroup_usage(memcg, false) * PAGE_SIZE;
3653 if (counter == &memcg->memsw)
3654 return (u64)mem_cgroup_usage(memcg, true) * PAGE_SIZE;
3663 return (u64)memcg->soft_limit * PAGE_SIZE;
3669 static void memcg_flush_percpu_vmstats(struct mem_cgroup *memcg)
3677 stat[i] += per_cpu(memcg->vmstats_percpu->stat[i], cpu);
3679 for (mi = memcg; mi; mi = parent_mem_cgroup(mi))
3684 struct mem_cgroup_per_node *pn = memcg->nodeinfo[node];
3701 static void memcg_flush_percpu_vmevents(struct mem_cgroup *memcg)
3712 events[i] += per_cpu(memcg->vmstats_percpu->events[i],
3715 for (mi = memcg; mi; mi = parent_mem_cgroup(mi))
3721 static int memcg_online_kmem(struct mem_cgroup *memcg)
3729 BUG_ON(memcg->kmemcg_id >= 0);
3730 BUG_ON(memcg->kmem_state);
3741 objcg->memcg = memcg;
3742 rcu_assign_pointer(memcg->objcg, objcg);
3752 memcg->kmemcg_id = memcg_id;
3753 memcg->kmem_state = KMEM_ONLINE;
3758 static void memcg_offline_kmem(struct mem_cgroup *memcg)
3764 if (memcg->kmem_state != KMEM_ONLINE)
3767 memcg->kmem_state = KMEM_ALLOCATED;
3769 parent = parent_mem_cgroup(memcg);
3773 memcg_reparent_objcgs(memcg, parent);
3775 kmemcg_id = memcg->kmemcg_id;
3787 css_for_each_descendant_pre(css, &memcg->css) {
3791 if (!memcg->use_hierarchy)
3801 static void memcg_free_kmem(struct mem_cgroup *memcg)
3804 if (unlikely(memcg->kmem_state == KMEM_ONLINE))
3805 memcg_offline_kmem(memcg);
3808 static int memcg_online_kmem(struct mem_cgroup *memcg)
3812 static void memcg_offline_kmem(struct mem_cgroup *memcg)
3815 static void memcg_free_kmem(struct mem_cgroup *memcg)
3820 static int memcg_update_kmem_max(struct mem_cgroup *memcg,
3826 ret = page_counter_set_max(&memcg->kmem, max);
3831 static int memcg_update_tcp_max(struct mem_cgroup *memcg, unsigned long max)
3837 ret = page_counter_set_max(&memcg->tcpmem, max);
3841 if (!memcg->tcpmem_active) {
3847 * belonging to this memcg until that flag is up.
3859 memcg->tcpmem_active = true;
3873 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
3884 if (mem_cgroup_is_root(memcg)) { /* Can't set limit on root */
3890 ret = mem_cgroup_resize_max(memcg, nr_pages, false);
3893 ret = mem_cgroup_resize_max(memcg, nr_pages, true);
3899 ret = memcg_update_kmem_max(memcg, nr_pages);
3902 ret = memcg_update_tcp_max(memcg, nr_pages);
3907 memcg->soft_limit = nr_pages;
3917 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
3922 counter = &memcg->memory;
3925 counter = &memcg->memsw;
3928 counter = &memcg->kmem;
3931 counter = &memcg->tcpmem;
3961 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
3976 memcg->move_charge_at_immigrate = val;
3993 static unsigned long mem_cgroup_node_nr_lru_pages(struct mem_cgroup *memcg,
3996 struct lruvec *lruvec = mem_cgroup_lruvec(memcg, NODE_DATA(nid));
4013 static unsigned long mem_cgroup_nr_lru_pages(struct mem_cgroup *memcg,
4024 nr += memcg_page_state(memcg, NR_LRU_BASE + lru);
4026 nr += memcg_page_state_local(memcg, NR_LRU_BASE + lru);
4046 struct mem_cgroup *memcg = mem_cgroup_from_seq(m);
4050 mem_cgroup_nr_lru_pages(memcg, stat->lru_mask,
4054 mem_cgroup_node_nr_lru_pages(memcg, nid,
4062 mem_cgroup_nr_lru_pages(memcg, stat->lru_mask,
4066 mem_cgroup_node_nr_lru_pages(memcg, nid,
4111 struct mem_cgroup *memcg = mem_cgroup_from_seq(m);
4123 nr = memcg_page_state_local(memcg, memcg1_stats[i]);
4133 memcg_events_local(memcg, memcg1_events[i]));
4141 memcg_page_state_local(memcg, NR_LRU_BASE + i) *
4147 for (mi = memcg; mi; mi = parent_mem_cgroup(mi)) {
4162 nr = memcg_page_state(memcg, memcg1_stats[i]);
4174 (u64)memcg_events(memcg, memcg1_events[i]));
4182 (u64)memcg_page_state(memcg, NR_LRU_BASE + i) *
4194 mz = mem_cgroup_nodeinfo(memcg, pgdat->node_id);
4213 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
4215 return mem_cgroup_swappiness(memcg);
4221 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
4227 memcg->swappiness = val;
4234 static void __mem_cgroup_threshold(struct mem_cgroup *memcg, bool swap)
4242 t = rcu_dereference(memcg->thresholds.primary);
4244 t = rcu_dereference(memcg->memsw_thresholds.primary);
4249 usage = mem_cgroup_usage(memcg, swap);
4285 static void mem_cgroup_threshold(struct mem_cgroup *memcg)
4287 while (memcg) {
4288 __mem_cgroup_threshold(memcg, false);
4290 __mem_cgroup_threshold(memcg, true);
4292 memcg = parent_mem_cgroup(memcg);
4310 static int mem_cgroup_oom_notify_cb(struct mem_cgroup *memcg)
4316 list_for_each_entry(ev, &memcg->oom_notify, list)
4323 static void mem_cgroup_oom_notify(struct mem_cgroup *memcg)
4327 for_each_mem_cgroup_tree(iter, memcg)
4331 static int __mem_cgroup_usage_register_event(struct mem_cgroup *memcg,
4344 mutex_lock(&memcg->thresholds_lock);
4347 thresholds = &memcg->thresholds;
4348 usage = mem_cgroup_usage(memcg, false);
4350 thresholds = &memcg->memsw_thresholds;
4351 usage = mem_cgroup_usage(memcg, true);
4357 __mem_cgroup_threshold(memcg, type == _MEMSWAP);
4406 mutex_unlock(&memcg->thresholds_lock);
4411 static int mem_cgroup_usage_register_event(struct mem_cgroup *memcg,
4414 return __mem_cgroup_usage_register_event(memcg, eventfd, args, _MEM);
4417 static int memsw_cgroup_usage_register_event(struct mem_cgroup *memcg,
4420 return __mem_cgroup_usage_register_event(memcg, eventfd, args, _MEMSWAP);
4423 static void __mem_cgroup_usage_unregister_event(struct mem_cgroup *memcg,
4431 mutex_lock(&memcg->thresholds_lock);
4434 thresholds = &memcg->thresholds;
4435 usage = mem_cgroup_usage(memcg, false);
4437 thresholds = &memcg->memsw_thresholds;
4438 usage = mem_cgroup_usage(memcg, true);
4446 __mem_cgroup_threshold(memcg, type == _MEMSWAP);
4505 mutex_unlock(&memcg->thresholds_lock);
4508 static void mem_cgroup_usage_unregister_event(struct mem_cgroup *memcg,
4511 return __mem_cgroup_usage_unregister_event(memcg, eventfd, _MEM);
4514 static void memsw_cgroup_usage_unregister_event(struct mem_cgroup *memcg,
4517 return __mem_cgroup_usage_unregister_event(memcg, eventfd, _MEMSWAP);
4520 static int mem_cgroup_oom_register_event(struct mem_cgroup *memcg,
4532 list_add(&event->list, &memcg->oom_notify);
4535 if (memcg->under_oom)
4542 static void mem_cgroup_oom_unregister_event(struct mem_cgroup *memcg,
4549 list_for_each_entry_safe(ev, tmp, &memcg->oom_notify, list) {
4561 struct mem_cgroup *memcg = mem_cgroup_from_seq(sf);
4563 seq_printf(sf, "oom_kill_disable %d\n", memcg->oom_kill_disable);
4564 seq_printf(sf, "under_oom %d\n", (bool)memcg->under_oom);
4566 atomic_long_read(&memcg->memory_events[MEMCG_OOM_KILL]));
4573 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
4579 memcg->oom_kill_disable = val;
4581 memcg_oom_recover(memcg);
4590 static int memcg_wb_domain_init(struct mem_cgroup *memcg, gfp_t gfp)
4592 return wb_domain_init(&memcg->cgwb_domain, gfp);
4595 static void memcg_wb_domain_exit(struct mem_cgroup *memcg)
4597 wb_domain_exit(&memcg->cgwb_domain);
4600 static void memcg_wb_domain_size_changed(struct mem_cgroup *memcg)
4602 wb_domain_size_changed(&memcg->cgwb_domain);
4607 struct mem_cgroup *memcg = mem_cgroup_from_css(wb->memcg_css);
4609 if (!memcg->css.parent)
4612 return &memcg->cgwb_domain;
4619 static unsigned long memcg_exact_page_state(struct mem_cgroup *memcg, int idx)
4621 long x = atomic_long_read(&memcg->vmstats[idx]);
4625 x += per_cpu_ptr(memcg->vmstats_percpu, cpu)->stat[idx];
4632 * mem_cgroup_wb_stats - retrieve writeback related stats from its memcg
4635 * @pheadroom: out parameter for number of allocatable pages according to memcg
4640 * @wb's memcg. File, dirty and writeback are self-explanatory. Headroom
4643 * A memcg's headroom is "min(max, high) - used". In the hierarchy, the
4653 struct mem_cgroup *memcg = mem_cgroup_from_css(wb->memcg_css);
4656 *pdirty = memcg_exact_page_state(memcg, NR_FILE_DIRTY);
4658 *pwriteback = memcg_exact_page_state(memcg, NR_WRITEBACK);
4659 *pfilepages = memcg_exact_page_state(memcg, NR_INACTIVE_FILE) +
4660 memcg_exact_page_state(memcg, NR_ACTIVE_FILE);
4663 while ((parent = parent_mem_cgroup(memcg))) {
4664 unsigned long ceiling = min(READ_ONCE(memcg->memory.max),
4665 READ_ONCE(memcg->memory.high));
4666 unsigned long used = page_counter_read(&memcg->memory);
4669 memcg = parent;
4676 * There's an inherent mismatch between memcg and writeback. The former
4698 * granularities between memcg and writeback in either direction. However,
4704 * page - a page whose memcg and writeback ownerships don't match - is
4706 * bdi_writeback on the page owning memcg. When balance_dirty_pages()
4707 * decides that the memcg needs to sleep due to high dirty ratio, it calls
4720 struct mem_cgroup *memcg = page->mem_cgroup;
4735 frn = &memcg->cgwb_frn[i];
4762 frn = &memcg->cgwb_frn[oldest];
4772 struct mem_cgroup *memcg = mem_cgroup_from_css(wb->memcg_css);
4778 struct memcg_cgwb_frn *frn = &memcg->cgwb_frn[i];
4799 static int memcg_wb_domain_init(struct mem_cgroup *memcg, gfp_t gfp)
4804 static void memcg_wb_domain_exit(struct mem_cgroup *memcg)
4808 static void memcg_wb_domain_size_changed(struct mem_cgroup *memcg)
4836 struct mem_cgroup *memcg = event->memcg;
4840 event->unregister_event(memcg, event->eventfd);
4847 css_put(&memcg->css);
4860 struct mem_cgroup *memcg = event->memcg;
4873 spin_lock(&memcg->event_list_lock);
4882 spin_unlock(&memcg->event_list_lock);
4910 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
4940 event->memcg = memcg;
5022 ret = event->register_event(memcg, event->eventfd, buf);
5028 spin_lock(&memcg->event_list_lock);
5029 list_add(&event->list, &memcg->event_list);
5030 spin_unlock(&memcg->event_list_lock);
5183 * Swap-out records and page cache shadow entries need to store memcg
5195 * Maintain a private 16-bit ID space for memcg, and allow the ID to
5220 static void mem_cgroup_id_remove(struct mem_cgroup *memcg)
5222 if (memcg->id.id > 0) {
5224 idr_remove(&mem_cgroup_idr, memcg->id.id);
5227 memcg->id.id = 0;
5231 static void __maybe_unused mem_cgroup_id_get_many(struct mem_cgroup *memcg,
5234 refcount_add(n, &memcg->id.ref);
5237 static void mem_cgroup_id_put_many(struct mem_cgroup *memcg, unsigned int n)
5239 if (refcount_sub_and_test(n, &memcg->id.ref)) {
5240 mem_cgroup_id_remove(memcg);
5243 css_put(&memcg->css);
5247 static inline void mem_cgroup_id_put(struct mem_cgroup *memcg)
5249 mem_cgroup_id_put_many(memcg, 1);
5253 * mem_cgroup_from_id - look up a memcg from a memcg id
5254 * @id: the memcg id to look up
5268 static int alloc_mem_cgroup_per_node_info(struct mem_cgroup *memcg, int node)
5305 pn->memcg = memcg;
5307 memcg->nodeinfo[node] = pn;
5311 static void free_mem_cgroup_per_node_info(struct mem_cgroup *memcg, int node)
5313 struct mem_cgroup_per_node *pn = memcg->nodeinfo[node];
5323 static void __mem_cgroup_free(struct mem_cgroup *memcg)
5328 free_mem_cgroup_per_node_info(memcg, node);
5329 free_percpu(memcg->vmstats_percpu);
5330 free_percpu(memcg->vmstats_local);
5331 kfree(memcg);
5334 static void mem_cgroup_free(struct mem_cgroup *memcg)
5336 memcg_wb_domain_exit(memcg);
5341 memcg_flush_percpu_vmstats(memcg);
5342 memcg_flush_percpu_vmevents(memcg);
5343 __mem_cgroup_free(memcg);
5348 struct mem_cgroup *memcg;
5357 memcg = kzalloc(size, GFP_KERNEL);
5358 if (!memcg)
5361 memcg->id.id = mem_cgroup_alloc_id();
5362 if (memcg->id.id < 0) {
5363 error = memcg->id.id;
5367 memcg->vmstats_local = alloc_percpu_gfp(struct memcg_vmstats_percpu,
5369 if (!memcg->vmstats_local)
5372 memcg->vmstats_percpu = alloc_percpu_gfp(struct memcg_vmstats_percpu,
5374 if (!memcg->vmstats_percpu)
5378 if (alloc_mem_cgroup_per_node_info(memcg, node))
5381 if (memcg_wb_domain_init(memcg, GFP_KERNEL))
5384 INIT_WORK(&memcg->high_work, high_work_func);
5385 INIT_LIST_HEAD(&memcg->oom_notify);
5386 mutex_init(&memcg->thresholds_lock);
5387 spin_lock_init(&memcg->move_lock);
5388 vmpressure_init(&memcg->vmpressure);
5389 INIT_LIST_HEAD(&memcg->event_list);
5390 spin_lock_init(&memcg->event_list_lock);
5391 memcg->socket_pressure = jiffies;
5393 memcg->kmemcg_id = -1;
5394 INIT_LIST_HEAD(&memcg->objcg_list);
5397 INIT_LIST_HEAD(&memcg->cgwb_list);
5399 memcg->cgwb_frn[i].done =
5403 spin_lock_init(&memcg->deferred_split_queue.split_queue_lock);
5404 INIT_LIST_HEAD(&memcg->deferred_split_queue.split_queue);
5405 memcg->deferred_split_queue.split_queue_len = 0;
5416 INIT_LIST_HEAD(&memcg->score_node);
5418 idr_replace(&mem_cgroup_idr, memcg, memcg->id.id);
5419 return memcg;
5421 mem_cgroup_id_remove(memcg);
5422 __mem_cgroup_free(memcg);
5430 struct mem_cgroup *memcg, *old_memcg;
5434 memcg = mem_cgroup_alloc();
5436 if (IS_ERR(memcg))
5437 return ERR_CAST(memcg);
5440 atomic64_set(&memcg->memcg_reclaimed.app_score, 300);
5443 atomic_set(&memcg->memcg_reclaimed.ub_zram2ufs_ratio, 10);
5444 atomic_set(&memcg->memcg_reclaimed.ub_mem2zram_ratio, 60);
5445 atomic_set(&memcg->memcg_reclaimed.refault_threshold, 50);
5447 page_counter_set_high(&memcg->memory, PAGE_COUNTER_MAX);
5448 memcg->soft_limit = PAGE_COUNTER_MAX;
5449 page_counter_set_high(&memcg->swap, PAGE_COUNTER_MAX);
5451 memcg->swappiness = mem_cgroup_swappiness(parent);
5452 memcg->oom_kill_disable = parent->oom_kill_disable;
5455 page_counter_init(&memcg->memory, NULL);
5456 page_counter_init(&memcg->swap, NULL);
5457 page_counter_init(&memcg->kmem, NULL);
5458 page_counter_init(&memcg->tcpmem, NULL);
5460 memcg->use_hierarchy = true;
5461 page_counter_init(&memcg->memory, &parent->memory);
5462 page_counter_init(&memcg->swap, &parent->swap);
5463 page_counter_init(&memcg->kmem, &parent->kmem);
5464 page_counter_init(&memcg->tcpmem, &parent->tcpmem);
5466 page_counter_init(&memcg->memory, &root_mem_cgroup->memory);
5467 page_counter_init(&memcg->swap, &root_mem_cgroup->swap);
5468 page_counter_init(&memcg->kmem, &root_mem_cgroup->kmem);
5469 page_counter_init(&memcg->tcpmem, &root_mem_cgroup->tcpmem);
5481 root_mem_cgroup = memcg;
5482 return &memcg->css;
5485 error = memcg_online_kmem(memcg);
5492 return &memcg->css;
5494 mem_cgroup_id_remove(memcg);
5495 mem_cgroup_free(memcg);
5501 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
5504 * A memcg must be visible for memcg_expand_shrinker_maps()
5508 if (memcg_alloc_shrinker_maps(memcg)) {
5509 mem_cgroup_id_remove(memcg);
5514 memcg_app_score_update(memcg);
5518 /* Online state pins memcg ID, memcg ID pins CSS */
5519 refcount_set(&memcg->id.ref, 1);
5526 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
5533 list_del_init(&memcg->score_node);
5543 spin_lock(&memcg->event_list_lock);
5544 list_for_each_entry_safe(event, tmp, &memcg->event_list, list) {
5548 spin_unlock(&memcg->event_list_lock);
5550 page_counter_set_min(&memcg->memory, 0);
5551 page_counter_set_low(&memcg->memory, 0);
5553 memcg_offline_kmem(memcg);
5554 wb_memcg_offline(memcg);
5556 drain_all_stock(memcg);
5558 mem_cgroup_id_put(memcg);
5563 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
5565 invalidate_reclaim_iterators(memcg);
5570 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
5575 wb_wait_for_completion(&memcg->cgwb_frn[i].done);
5580 if (!cgroup_subsys_on_dfl(memory_cgrp_subsys) && memcg->tcpmem_active)
5583 vmpressure_cleanup(&memcg->vmpressure);
5584 cancel_work_sync(&memcg->high_work);
5585 mem_cgroup_remove_from_trees(memcg);
5586 memcg_free_shrinker_maps(memcg);
5587 memcg_free_kmem(memcg);
5588 mem_cgroup_free(memcg);
5597 * but the memcg is pinned through dependency. The memcg should stop
5606 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
5608 page_counter_set_max(&memcg->memory, PAGE_COUNTER_MAX);
5609 page_counter_set_max(&memcg->swap, PAGE_COUNTER_MAX);
5610 page_counter_set_max(&memcg->kmem, PAGE_COUNTER_MAX);
5611 page_counter_set_max(&memcg->tcpmem, PAGE_COUNTER_MAX);
5612 page_counter_set_min(&memcg->memory, 0);
5613 page_counter_set_low(&memcg->memory, 0);
5614 page_counter_set_high(&memcg->memory, PAGE_COUNTER_MAX);
5615 memcg->soft_limit = PAGE_COUNTER_MAX;
5616 page_counter_set_high(&memcg->swap, PAGE_COUNTER_MAX);
5617 memcg_wb_domain_size_changed(memcg);
5822 * All state has been migrated, let's switch to the new memcg.
5829 * Note that lock_page_memcg is a memcg lock, not a page lock,
5831 * new memcg that isn't locked, the above state can change
6089 struct mem_cgroup *memcg = NULL; /* unneeded init to make gcc happy */
6110 memcg = mem_cgroup_from_css(css);
6120 move_flags = READ_ONCE(memcg->move_charge_at_immigrate);
6126 VM_BUG_ON(from == memcg);
6142 mc.to = memcg;
6227 * memcg. There should be somebody mapping the head.
6283 * Signal lock_page_memcg() to take the memcg's move_lock
6284 * while we're moving its pages to another memcg. Then wait
6343 * on for the root memcg is enough.
6364 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
6366 return (u64)page_counter_read(&memcg->memory) * PAGE_SIZE;
6378 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
6387 page_counter_set_min(&memcg->memory, min);
6401 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
6410 page_counter_set_low(&memcg->memory, low);
6424 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
6435 page_counter_set_high(&memcg->memory, high);
6438 unsigned long nr_pages = page_counter_read(&memcg->memory);
6448 drain_all_stock(memcg);
6453 reclaimed = try_to_free_mem_cgroup_pages(memcg, nr_pages - high,
6460 memcg_wb_domain_size_changed(memcg);
6473 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
6484 xchg(&memcg->memory.max, max);
6487 unsigned long nr_pages = page_counter_read(&memcg->memory);
6496 drain_all_stock(memcg);
6502 if (!try_to_free_mem_cgroup_pages(memcg, nr_pages - max,
6508 memcg_memory_event(memcg, MEMCG_OOM);
6509 if (!mem_cgroup_out_of_memory(memcg, GFP_KERNEL, 0))
6513 memcg_wb_domain_size_changed(memcg);
6529 struct mem_cgroup *memcg = mem_cgroup_from_seq(m);
6531 __memory_events_show(m, memcg->memory_events);
6537 struct mem_cgroup *memcg = mem_cgroup_from_seq(m);
6539 __memory_events_show(m, memcg->memory_events_local);
6545 struct mem_cgroup *memcg = mem_cgroup_from_seq(m);
6548 buf = memory_stat_format(memcg);
6560 struct mem_cgroup *memcg = mem_cgroup_from_seq(m);
6573 lruvec = mem_cgroup_lruvec(memcg, NODE_DATA(nid));
6587 struct mem_cgroup *memcg = mem_cgroup_from_seq(m);
6589 seq_printf(m, "%d\n", memcg->oom_group);
6597 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
6611 memcg->oom_group = oom_group;
6812 * @memcg: the memory cgroup to check
6818 struct mem_cgroup *memcg)
6836 if (memcg == root)
6839 usage = page_counter_read(&memcg->memory);
6843 parent = parent_mem_cgroup(memcg);
6844 /* No parent means a non-hierarchical mode on v1 memcg */
6849 memcg->memory.emin = READ_ONCE(memcg->memory.min);
6850 memcg->memory.elow = READ_ONCE(memcg->memory.low);
6856 WRITE_ONCE(memcg->memory.emin, effective_protection(usage, parent_usage,
6857 READ_ONCE(memcg->memory.min),
6861 WRITE_ONCE(memcg->memory.elow, effective_protection(usage, parent_usage,
6862 READ_ONCE(memcg->memory.low),
6873 * Try to charge @page to the memcg that @mm belongs to, reclaiming
6881 struct mem_cgroup *memcg = NULL;
6904 memcg = mem_cgroup_from_id(id);
6905 if (memcg && !css_tryget_online(&memcg->css))
6906 memcg = NULL;
6910 if (!memcg)
6911 memcg = get_mem_cgroup_from_mm(mm);
6913 ret = try_charge(memcg, gfp_mask, nr_pages);
6917 css_get(&memcg->css);
6918 commit_charge(page, memcg);
6921 mem_cgroup_charge_statistics(memcg, page, nr_pages);
6922 memcg_check_events(memcg, page);
6948 css_put(&memcg->css);
6954 struct mem_cgroup *memcg;
6970 if (!mem_cgroup_is_root(ug->memcg)) {
6971 page_counter_uncharge(&ug->memcg->memory, ug->nr_pages);
6973 page_counter_uncharge(&ug->memcg->memsw, ug->nr_pages);
6975 page_counter_uncharge(&ug->memcg->kmem, ug->nr_kmem);
6976 memcg_oom_recover(ug->memcg);
6980 __count_memcg_events(ug->memcg, PGPGOUT, ug->pgpgout);
6981 __this_cpu_add(ug->memcg->vmstats_percpu->nr_page_events, ug->nr_pages);
6982 memcg_check_events(ug->memcg, ug->dummy_page);
6986 css_put(&ug->memcg->css);
7004 if (ug->memcg != page->mem_cgroup) {
7005 if (ug->memcg) {
7009 ug->memcg = page->mem_cgroup;
7012 css_get(&ug->memcg->css);
7027 css_put(&ug->memcg->css);
7051 if (ug.memcg)
7105 struct mem_cgroup *memcg;
7123 memcg = oldpage->mem_cgroup;
7124 if (!memcg)
7130 page_counter_charge(&memcg->memory, nr_pages);
7132 page_counter_charge(&memcg->memsw, nr_pages);
7134 css_get(&memcg->css);
7135 commit_charge(newpage, memcg);
7138 mem_cgroup_charge_statistics(memcg, newpage, nr_pages);
7139 memcg_check_events(memcg, newpage);
7148 struct mem_cgroup *memcg;
7153 /* Do not associate the sock with unrelated interrupted task's memcg. */
7158 memcg = mem_cgroup_from_task(current);
7159 if (memcg == root_mem_cgroup)
7161 if (!cgroup_subsys_on_dfl(memory_cgrp_subsys) && !memcg->tcpmem_active)
7163 if (css_tryget(&memcg->css))
7164 sk->sk_memcg = memcg;
7177 * @memcg: memcg to charge
7180 * Charges @nr_pages to @memcg. Returns %true if the charge fit within
7181 * @memcg's configured limit, %false if the charge had to be forced.
7183 bool mem_cgroup_charge_skmem(struct mem_cgroup *memcg, unsigned int nr_pages)
7190 if (page_counter_try_charge(&memcg->tcpmem, nr_pages, &fail)) {
7191 memcg->tcpmem_pressure = 0;
7194 page_counter_charge(&memcg->tcpmem, nr_pages);
7195 memcg->tcpmem_pressure = 1;
7203 mod_memcg_state(memcg, MEMCG_SOCK, nr_pages);
7205 if (try_charge(memcg, gfp_mask, nr_pages) == 0)
7208 try_charge(memcg, gfp_mask|__GFP_NOFAIL, nr_pages);
7214 * @memcg: memcg to uncharge
7217 void mem_cgroup_uncharge_skmem(struct mem_cgroup *memcg, unsigned int nr_pages)
7220 page_counter_uncharge(&memcg->tcpmem, nr_pages);
7224 mod_memcg_state(memcg, MEMCG_SOCK, -nr_pages);
7226 refill_stock(memcg, nr_pages);
7283 static struct mem_cgroup *mem_cgroup_id_get_online(struct mem_cgroup *memcg)
7285 while (!refcount_inc_not_zero(&memcg->id.ref)) {
7290 if (WARN_ON_ONCE(memcg == root_mem_cgroup)) {
7294 memcg = parent_mem_cgroup(memcg);
7295 if (!memcg)
7296 memcg = root_mem_cgroup;
7298 return memcg;
7310 struct mem_cgroup *memcg, *swap_memcg;
7320 memcg = page->mem_cgroup;
7323 if (!memcg)
7327 * In case the memcg owning these pages has been offlined and doesn't
7331 swap_memcg = mem_cgroup_id_get_online(memcg);
7343 if (!mem_cgroup_is_root(memcg))
7344 page_counter_uncharge(&memcg->memory, nr_entries);
7346 if (!cgroup_memory_noswap && memcg != swap_memcg) {
7349 page_counter_uncharge(&memcg->memsw, nr_entries);
7359 mem_cgroup_charge_statistics(memcg, page, -nr_entries);
7360 memcg_check_events(memcg, page);
7362 css_put(&memcg->css);
7370 * Try to charge @page's memcg for the swap space at @entry.
7378 struct mem_cgroup *memcg;
7384 memcg = page->mem_cgroup;
7387 if (!memcg)
7391 memcg_memory_event(memcg, MEMCG_SWAP_FAIL);
7395 memcg = mem_cgroup_id_get_online(memcg);
7397 if (!cgroup_memory_noswap && !mem_cgroup_is_root(memcg) &&
7398 !page_counter_try_charge(&memcg->swap, nr_pages, &counter)) {
7399 memcg_memory_event(memcg, MEMCG_SWAP_MAX);
7400 memcg_memory_event(memcg, MEMCG_SWAP_FAIL);
7401 mem_cgroup_id_put(memcg);
7407 mem_cgroup_id_get_many(memcg, nr_pages - 1);
7408 oldid = swap_cgroup_record(entry, mem_cgroup_id(memcg), nr_pages);
7410 mod_memcg_state(memcg, MEMCG_SWAP, nr_pages);
7422 struct mem_cgroup *memcg;
7427 memcg = mem_cgroup_from_id(id);
7428 if (memcg) {
7429 if (!cgroup_memory_noswap && !mem_cgroup_is_root(memcg)) {
7431 page_counter_uncharge(&memcg->swap, nr_pages);
7433 page_counter_uncharge(&memcg->memsw, nr_pages);
7435 mod_memcg_state(memcg, MEMCG_SWAP, -nr_pages);
7436 mem_cgroup_id_put_many(memcg, nr_pages);
7441 long mem_cgroup_get_nr_swap_pages(struct mem_cgroup *memcg)
7447 for (; memcg != root_mem_cgroup; memcg = parent_mem_cgroup(memcg))
7449 READ_ONCE(memcg->swap.max) -
7450 page_counter_read(&memcg->swap));
7456 struct mem_cgroup *memcg;
7465 memcg = page->mem_cgroup;
7466 if (!memcg)
7469 for (; memcg != root_mem_cgroup; memcg = parent_mem_cgroup(memcg)) {
7470 unsigned long usage = page_counter_read(&memcg->swap);
7472 if (usage * 2 >= READ_ONCE(memcg->swap.high) ||
7473 usage * 2 >= READ_ONCE(memcg->swap.max))
7493 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
7495 return (u64)page_counter_read(&memcg->swap) * PAGE_SIZE;
7507 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
7516 page_counter_set_high(&memcg->swap, high);
7530 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
7539 xchg(&memcg->swap.max, max);
7546 struct mem_cgroup *memcg = mem_cgroup_from_seq(m);
7549 atomic_long_read(&memcg->memory_events[MEMCG_SWAP_HIGH]));
7551 atomic_long_read(&memcg->memory_events[MEMCG_SWAP_MAX]));
7553 atomic_long_read(&memcg->memory_events[MEMCG_SWAP_FAIL]));
7615 * remains set to false even when memcg is disabled via "cgroup_disable=memory"