Lines Matching refs:memcg
24 * Per memcg lru locking
137 * memcg which the event belongs to.
139 struct mem_cgroup *memcg;
153 int (*register_event)(struct mem_cgroup *memcg,
160 void (*unregister_event)(struct mem_cgroup *memcg,
172 static void mem_cgroup_threshold(struct mem_cgroup *memcg);
173 static void mem_cgroup_oom_notify(struct mem_cgroup *memcg);
241 struct vmpressure *memcg_to_vmpressure(struct mem_cgroup *memcg)
243 if (!memcg)
244 memcg = root_mem_cgroup;
245 return &memcg->vmpressure;
280 * 3) CPU1: a process from another memcg is allocating something,
325 static void memcg_reparent_objcgs(struct mem_cgroup *memcg,
330 objcg = rcu_replace_pointer(memcg->objcg, NULL, true);
335 list_add(&objcg->list, &memcg->objcg_list);
337 list_for_each_entry(iter, &memcg->objcg_list, list)
338 WRITE_ONCE(iter->memcg, parent);
340 list_splice(&memcg->objcg_list, &parent->objcg_list);
361 * mem_cgroup_css_from_folio - css of the memcg associated with a folio
364 * If memcg is bound to the default hierarchy, css of the memcg associated
368 * If memcg is bound to a traditional hierarchy, the css of root_mem_cgroup
373 struct mem_cgroup *memcg = folio_memcg(folio);
375 if (!memcg || !cgroup_subsys_on_dfl(memory_cgrp_subsys))
376 memcg = root_mem_cgroup;
378 return &memcg->css;
382 * page_cgroup_ino - return inode number of the memcg a page is charged to
396 struct mem_cgroup *memcg;
401 memcg = folio_memcg_check(page_folio(page));
403 while (memcg && !(memcg->css.flags & CSS_ONLINE))
404 memcg = parent_mem_cgroup(memcg);
405 if (memcg)
406 ino = cgroup_ino(memcg->css.cgroup);
469 static unsigned long soft_limit_excess(struct mem_cgroup *memcg)
472 struct mem_cgroup_per_node *mz = mem_cgroup_nodeinfo(memcg, 0);
478 unsigned long nr_pages = page_counter_read(&memcg->memory);
480 unsigned long soft_limit = READ_ONCE(memcg->soft_limit);
489 static void mem_cgroup_update_tree(struct mem_cgroup *memcg, int nid)
496 if (soft_limit_excess(memcg))
497 lru_gen_soft_reclaim(memcg, nid);
508 for (; memcg; memcg = parent_mem_cgroup(memcg)) {
509 mz = memcg->nodeinfo[nid];
510 excess = soft_limit_excess(memcg);
532 static void mem_cgroup_remove_from_trees(struct mem_cgroup *memcg)
539 mz = memcg->nodeinfo[nid];
564 if (!soft_limit_excess(mz->memcg) ||
565 !css_tryget(&mz->memcg->css))
583 * memcg and lruvec stats flushing
628 static inline void memcg_rstat_updated(struct mem_cgroup *memcg, int val)
635 cgroup_rstat_updated(memcg->css.cgroup, smp_processor_id());
656 * from memcg flushers (e.g. reclaim, refault, etc).
692 /* Subset of vm_event_item to report for memcg event stats */
763 unsigned long memcg_page_state(struct mem_cgroup *memcg, int idx)
765 long x = READ_ONCE(memcg->vmstats->state[idx]);
775 * @memcg: the memory cgroup
779 void __mod_memcg_state(struct mem_cgroup *memcg, int idx, int val)
784 __this_cpu_add(memcg->vmstats_percpu->state[idx], val);
785 memcg_rstat_updated(memcg, val);
789 static unsigned long memcg_page_state_local(struct mem_cgroup *memcg, int idx)
791 long x = READ_ONCE(memcg->vmstats->state_local[idx]);
804 struct mem_cgroup *memcg;
807 memcg = pn->memcg;
830 /* Update memcg */
831 __this_cpu_add(memcg->vmstats_percpu->state[idx], val);
836 memcg_rstat_updated(memcg, val);
856 /* Update memcg and lruvec */
870 struct mem_cgroup *memcg;
882 memcg = page_memcg(head);
883 /* Untracked pages have no memcg, no lruvec. Update only the node */
884 if (!memcg) {
890 lruvec = mem_cgroup_lruvec(memcg, pgdat);
899 struct mem_cgroup *memcg;
903 memcg = mem_cgroup_from_slab_obj(p);
906 * Untracked pages have no memcg, no lruvec. Update only the
907 * node. If we reparent the slab objects to the root memcg,
908 * when we free the slab object, we need to update the per-memcg
909 * vmstats to keep it correct for the root memcg.
911 if (!memcg) {
914 lruvec = mem_cgroup_lruvec(memcg, pgdat);
922 * @memcg: the memory cgroup
926 void __count_memcg_events(struct mem_cgroup *memcg, enum vm_event_item idx,
934 if (!memcg)
939 __this_cpu_add(memcg->vmstats_percpu->events[index], count);
940 memcg_rstat_updated(memcg, count);
944 static unsigned long memcg_events(struct mem_cgroup *memcg, int event)
950 return READ_ONCE(memcg->vmstats->events[index]);
953 static unsigned long memcg_events_local(struct mem_cgroup *memcg, int event)
960 return READ_ONCE(memcg->vmstats->events_local[index]);
963 static void mem_cgroup_charge_statistics(struct mem_cgroup *memcg,
968 __count_memcg_events(memcg, PGPGIN, 1);
970 __count_memcg_events(memcg, PGPGOUT, 1);
974 __this_cpu_add(memcg->vmstats_percpu->nr_page_events, nr_pages);
977 static bool mem_cgroup_event_ratelimit(struct mem_cgroup *memcg,
982 val = __this_cpu_read(memcg->vmstats_percpu->nr_page_events);
983 next = __this_cpu_read(memcg->vmstats_percpu->targets[target]);
996 __this_cpu_write(memcg->vmstats_percpu->targets[target], next);
1006 static void memcg_check_events(struct mem_cgroup *memcg, int nid)
1012 if (unlikely(mem_cgroup_event_ratelimit(memcg,
1016 do_softlimit = mem_cgroup_event_ratelimit(memcg,
1018 mem_cgroup_threshold(memcg);
1020 mem_cgroup_update_tree(memcg, nid);
1047 * get_mem_cgroup_from_mm: Obtain a reference on given mm_struct's memcg.
1048 * @mm: mm from which memcg should be extracted. It can be NULL.
1050 * Obtain a reference on mm->memcg and returns it if successful. If mm
1051 * is NULL, then the memcg is chosen as follows:
1052 * 1) The active memcg, if set.
1053 * 2) current->mm->memcg, if available
1054 * 3) root memcg
1059 struct mem_cgroup *memcg;
1069 * No need to css_get on root memcg as the reference
1074 memcg = active_memcg();
1075 if (unlikely(memcg)) {
1076 /* remote memcg must hold a ref */
1077 css_get(&memcg->css);
1078 return memcg;
1087 memcg = mem_cgroup_from_task(rcu_dereference(mm->owner));
1088 if (unlikely(!memcg))
1089 memcg = root_mem_cgroup;
1090 } while (!css_tryget(&memcg->css));
1092 return memcg;
1098 /* Allow remote memcg charging from any context. */
1112 * @prev: previously returned memcg, NULL on first invocation
1132 struct mem_cgroup *memcg = NULL;
1199 memcg = mem_cgroup_from_css(css);
1210 (void)cmpxchg(&iter->position, pos, memcg);
1215 if (!memcg)
1224 return memcg;
1257 struct mem_cgroup *memcg = dead_memcg;
1261 __invalidate_reclaim_iterators(memcg, dead_memcg);
1262 last = memcg;
1263 } while ((memcg = parent_mem_cgroup(memcg)));
1278 * @memcg: hierarchy root
1282 * This function iterates over tasks attached to @memcg or to any of its
1289 void mem_cgroup_scan_tasks(struct mem_cgroup *memcg,
1295 BUG_ON(mem_cgroup_is_root(memcg));
1297 for_each_mem_cgroup_tree(iter, memcg) {
1306 mem_cgroup_iter_break(memcg, iter);
1315 struct mem_cgroup *memcg;
1320 memcg = folio_memcg(folio);
1322 if (!memcg)
1325 VM_BUG_ON_FOLIO(lruvec_memcg(lruvec) != memcg, folio);
1444 * @memcg: the memory cgroup
1449 static unsigned long mem_cgroup_margin(struct mem_cgroup *memcg)
1455 count = page_counter_read(&memcg->memory);
1456 limit = READ_ONCE(memcg->memory.max);
1461 count = page_counter_read(&memcg->memsw);
1462 limit = READ_ONCE(memcg->memsw.max);
1479 static bool mem_cgroup_under_move(struct mem_cgroup *memcg)
1494 ret = mem_cgroup_is_descendant(from, memcg) ||
1495 mem_cgroup_is_descendant(to, memcg);
1501 static bool mem_cgroup_wait_acct_move(struct mem_cgroup *memcg)
1504 if (mem_cgroup_under_move(memcg)) {
1589 static inline unsigned long memcg_page_state_output(struct mem_cgroup *memcg,
1592 return memcg_page_state(memcg, item) * memcg_page_state_unit(item);
1595 static void memcg_stat_format(struct mem_cgroup *memcg, struct seq_buf *s)
1614 size = memcg_page_state_output(memcg, memory_stats[i].idx);
1618 size += memcg_page_state_output(memcg,
1626 memcg_events(memcg, PGSCAN_KSWAPD) +
1627 memcg_events(memcg, PGSCAN_DIRECT) +
1628 memcg_events(memcg, PGSCAN_KHUGEPAGED));
1630 memcg_events(memcg, PGSTEAL_KSWAPD) +
1631 memcg_events(memcg, PGSTEAL_DIRECT) +
1632 memcg_events(memcg, PGSTEAL_KHUGEPAGED));
1641 memcg_events(memcg, memcg_vm_event_stat[i]));
1648 static void memcg1_stat_format(struct mem_cgroup *memcg, struct seq_buf *s);
1650 static void memory_stat_format(struct mem_cgroup *memcg, struct seq_buf *s)
1653 memcg_stat_format(memcg, s);
1655 memcg1_stat_format(memcg, s);
1662 * @memcg: The memory cgroup that went over limit
1665 * NOTE: @memcg and @p's mem_cgroup can be different when hierarchy is
1668 void mem_cgroup_print_oom_context(struct mem_cgroup *memcg, struct task_struct *p)
1672 if (memcg) {
1674 pr_cont_cgroup_path(memcg->css.cgroup);
1687 * @memcg: The memory cgroup that went over limit
1689 void mem_cgroup_print_oom_meminfo(struct mem_cgroup *memcg)
1698 K((u64)page_counter_read(&memcg->memory)),
1699 K((u64)READ_ONCE(memcg->memory.max)), memcg->memory.failcnt);
1702 K((u64)page_counter_read(&memcg->swap)),
1703 K((u64)READ_ONCE(memcg->swap.max)), memcg->swap.failcnt);
1706 K((u64)page_counter_read(&memcg->memsw)),
1707 K((u64)memcg->memsw.max), memcg->memsw.failcnt);
1709 K((u64)page_counter_read(&memcg->kmem)),
1710 K((u64)memcg->kmem.max), memcg->kmem.failcnt);
1714 pr_cont_cgroup_path(memcg->css.cgroup);
1717 memory_stat_format(memcg, &s);
1722 * Return the memory (and swap, if configured) limit for a memcg.
1724 unsigned long mem_cgroup_get_max(struct mem_cgroup *memcg)
1726 unsigned long max = READ_ONCE(memcg->memory.max);
1729 if (mem_cgroup_swappiness(memcg)) {
1731 unsigned long swap = READ_ONCE(memcg->memsw.max) - max;
1736 if (mem_cgroup_swappiness(memcg))
1737 max += min(READ_ONCE(memcg->swap.max),
1743 unsigned long mem_cgroup_size(struct mem_cgroup *memcg)
1745 return page_counter_read(&memcg->memory);
1748 static bool mem_cgroup_out_of_memory(struct mem_cgroup *memcg, gfp_t gfp_mask,
1754 .memcg = memcg,
1763 if (mem_cgroup_margin(memcg) >= (1 << order))
1839 static bool mem_cgroup_oom_trylock(struct mem_cgroup *memcg)
1845 for_each_mem_cgroup_tree(iter, memcg) {
1852 mem_cgroup_iter_break(memcg, iter);
1863 for_each_mem_cgroup_tree(iter, memcg) {
1865 mem_cgroup_iter_break(memcg, iter);
1878 static void mem_cgroup_oom_unlock(struct mem_cgroup *memcg)
1884 for_each_mem_cgroup_tree(iter, memcg)
1889 static void mem_cgroup_mark_under_oom(struct mem_cgroup *memcg)
1894 for_each_mem_cgroup_tree(iter, memcg)
1899 static void mem_cgroup_unmark_under_oom(struct mem_cgroup *memcg)
1904 * Be careful about under_oom underflows because a child memcg
1908 for_each_mem_cgroup_tree(iter, memcg)
1917 struct mem_cgroup *memcg;
1929 oom_wait_memcg = oom_wait_info->memcg;
1937 static void memcg_oom_recover(struct mem_cgroup *memcg)
1947 if (memcg && memcg->under_oom)
1948 __wake_up(&memcg_oom_waitq, TASK_NORMAL, 0, memcg);
1955 static bool mem_cgroup_oom(struct mem_cgroup *memcg, gfp_t mask, int order)
1962 memcg_memory_event(memcg, MEMCG_OOM);
1982 if (READ_ONCE(memcg->oom_kill_disable)) {
1984 css_get(&memcg->css);
1985 current->memcg_in_oom = memcg;
1992 mem_cgroup_mark_under_oom(memcg);
1994 locked = mem_cgroup_oom_trylock(memcg);
1997 mem_cgroup_oom_notify(memcg);
1999 mem_cgroup_unmark_under_oom(memcg);
2000 ret = mem_cgroup_out_of_memory(memcg, mask, order);
2003 mem_cgroup_oom_unlock(memcg);
2009 * mem_cgroup_oom_synchronize - complete memcg OOM handling
2012 * This has to be called at the end of a page fault if the memcg OOM
2022 * Returns %true if an ongoing memcg OOM situation was detected and
2027 struct mem_cgroup *memcg = current->memcg_in_oom;
2032 if (!memcg)
2038 owait.memcg = memcg;
2045 mem_cgroup_mark_under_oom(memcg);
2047 locked = mem_cgroup_oom_trylock(memcg);
2050 mem_cgroup_oom_notify(memcg);
2053 mem_cgroup_unmark_under_oom(memcg);
2057 mem_cgroup_oom_unlock(memcg);
2060 css_put(&memcg->css);
2067 * @oom_domain: memcg in case of memcg OOM, NULL in case of system-wide OOM
2072 * Caller has to call mem_cgroup_put() on the returned non-NULL memcg.
2078 struct mem_cgroup *memcg;
2088 memcg = mem_cgroup_from_task(victim);
2089 if (mem_cgroup_is_root(memcg))
2097 if (unlikely(!mem_cgroup_is_descendant(memcg, oom_domain)))
2105 for (; memcg; memcg = parent_mem_cgroup(memcg)) {
2106 if (READ_ONCE(memcg->oom_group))
2107 oom_group = memcg;
2109 if (memcg == oom_domain)
2121 void mem_cgroup_print_oom_group(struct mem_cgroup *memcg)
2124 pr_cont_cgroup_path(memcg->css.cgroup);
2129 * folio_memcg_lock - Bind a folio to its memcg.
2135 * It ensures lifetime of the bound memcg. The caller is responsible
2140 struct mem_cgroup *memcg;
2145 * path can get away without acquiring the memcg->move_lock
2153 memcg = folio_memcg(folio);
2154 if (unlikely(!memcg))
2159 might_lock(&memcg->move_lock);
2163 if (atomic_read(&memcg->moving_account) <= 0)
2166 spin_lock_irqsave(&memcg->move_lock, flags);
2167 if (memcg != folio_memcg(folio)) {
2168 spin_unlock_irqrestore(&memcg->move_lock, flags);
2178 memcg->move_lock_task = current;
2179 memcg->move_lock_flags = flags;
2182 static void __folio_memcg_unlock(struct mem_cgroup *memcg)
2184 if (memcg && memcg->move_lock_task == current) {
2185 unsigned long flags = memcg->move_lock_flags;
2187 memcg->move_lock_task = NULL;
2188 memcg->move_lock_flags = 0;
2190 spin_unlock_irqrestore(&memcg->move_lock, flags);
2197 * folio_memcg_unlock - Release the binding between a folio and its memcg.
2201 * not change the accounting of this folio to its memcg, but it does
2235 static void memcg_account_kmem(struct mem_cgroup *memcg, int nr_pages);
2247 static void memcg_account_kmem(struct mem_cgroup *memcg, int nr_pages)
2254 * @memcg: memcg to consume from.
2257 * The charges will only happen if @memcg matches the current cpu's memcg
2263 static bool consume_stock(struct mem_cgroup *memcg, unsigned int nr_pages)
2275 if (memcg == READ_ONCE(stock->cached) && stock->nr_pages >= nr_pages) {
2333 static void __refill_stock(struct mem_cgroup *memcg, unsigned int nr_pages)
2338 if (READ_ONCE(stock->cached) != memcg) { /* reset if necessary */
2340 css_get(&memcg->css);
2341 WRITE_ONCE(stock->cached, memcg);
2349 static void refill_stock(struct mem_cgroup *memcg, unsigned int nr_pages)
2354 __refill_stock(memcg, nr_pages);
2379 struct mem_cgroup *memcg;
2383 memcg = READ_ONCE(stock->cached);
2384 if (memcg && stock->nr_pages &&
2385 mem_cgroup_is_descendant(memcg, root_memcg))
2413 static unsigned long reclaim_high(struct mem_cgroup *memcg,
2422 if (page_counter_read(&memcg->memory) <=
2423 READ_ONCE(memcg->memory.high))
2426 memcg_memory_event(memcg, MEMCG_HIGH);
2429 nr_reclaimed += try_to_free_mem_cgroup_pages(memcg, nr_pages,
2433 } while ((memcg = parent_mem_cgroup(memcg)) &&
2434 !mem_cgroup_is_root(memcg));
2441 struct mem_cgroup *memcg;
2443 memcg = container_of(work, struct mem_cgroup, high_work);
2444 reclaim_high(memcg, MEMCG_CHARGE_BATCH, GFP_KERNEL);
2518 static u64 mem_find_max_overage(struct mem_cgroup *memcg)
2523 overage = calculate_overage(page_counter_read(&memcg->memory),
2524 READ_ONCE(memcg->memory.high));
2526 } while ((memcg = parent_mem_cgroup(memcg)) &&
2527 !mem_cgroup_is_root(memcg));
2532 static u64 swap_find_max_overage(struct mem_cgroup *memcg)
2537 overage = calculate_overage(page_counter_read(&memcg->swap),
2538 READ_ONCE(memcg->swap.high));
2540 memcg_memory_event(memcg, MEMCG_SWAP_HIGH);
2542 } while ((memcg = parent_mem_cgroup(memcg)) &&
2543 !mem_cgroup_is_root(memcg));
2552 static unsigned long calculate_high_delay(struct mem_cgroup *memcg,
2565 * memcg in question makes it clear that it has no intention of stopping
2595 struct mem_cgroup *memcg;
2601 memcg = get_mem_cgroup_from_mm(current->mm);
2614 nr_reclaimed = reclaim_high(memcg,
2622 penalty_jiffies = calculate_high_delay(memcg, nr_pages,
2623 mem_find_max_overage(memcg));
2625 penalty_jiffies += calculate_high_delay(memcg, nr_pages,
2626 swap_find_max_overage(memcg));
2636 * Don't sleep if the amount of jiffies this memcg owes us is so low
2664 css_put(&memcg->css);
2667 static int try_charge_memcg(struct mem_cgroup *memcg, gfp_t gfp_mask,
2682 if (consume_stock(memcg, nr_pages))
2686 page_counter_try_charge(&memcg->memsw, batch, &counter)) {
2687 if (page_counter_try_charge(&memcg->memory, batch, &counter))
2690 page_counter_uncharge(&memcg->memsw, batch);
2765 * keep retrying as long as the memcg oom killer is able to make
2797 page_counter_charge(&memcg->memory, nr_pages);
2799 page_counter_charge(&memcg->memsw, nr_pages);
2805 refill_stock(memcg, batch - nr_pages);
2811 * GFP_KERNEL can consistently be used during reclaim. @memcg is
2819 mem_high = page_counter_read(&memcg->memory) >
2820 READ_ONCE(memcg->memory.high);
2821 swap_high = page_counter_read(&memcg->swap) >
2822 READ_ONCE(memcg->swap.high);
2827 schedule_work(&memcg->high_work);
2847 } while ((memcg = parent_mem_cgroup(memcg)));
2857 static inline int try_charge(struct mem_cgroup *memcg, gfp_t gfp_mask,
2860 if (mem_cgroup_is_root(memcg))
2863 return try_charge_memcg(memcg, gfp_mask, nr_pages);
2866 static inline void cancel_charge(struct mem_cgroup *memcg, unsigned int nr_pages)
2868 if (mem_cgroup_is_root(memcg))
2871 page_counter_uncharge(&memcg->memory, nr_pages);
2873 page_counter_uncharge(&memcg->memsw, nr_pages);
2876 static void commit_charge(struct folio *folio, struct mem_cgroup *memcg)
2880 * Any of the following ensures page's memcg stability:
2888 folio->memcg_data = (unsigned long)memcg;
2908 struct mem_cgroup *memcg;
2912 memcg = obj_cgroup_memcg(objcg);
2913 lruvec = mem_cgroup_lruvec(memcg, pgdat);
2999 * The caller must ensure the memcg lifetime, e.g. by taking rcu_read_lock(),
3024 * The caller must ensure the memcg lifetime, e.g. by taking rcu_read_lock(),
3035 static struct obj_cgroup *__get_obj_cgroup_from_memcg(struct mem_cgroup *memcg)
3039 for (; !mem_cgroup_is_root(memcg); memcg = parent_mem_cgroup(memcg)) {
3040 objcg = rcu_dereference(memcg->objcg);
3051 struct mem_cgroup *memcg;
3058 memcg = active_memcg();
3060 memcg = mem_cgroup_from_task(current);
3061 objcg = __get_obj_cgroup_from_memcg(memcg);
3077 struct mem_cgroup *memcg;
3080 memcg = __folio_memcg(folio);
3081 if (memcg)
3082 objcg = __get_obj_cgroup_from_memcg(memcg);
3090 static void memcg_account_kmem(struct mem_cgroup *memcg, int nr_pages)
3092 mod_memcg_state(memcg, MEMCG_KMEM, nr_pages);
3095 page_counter_charge(&memcg->kmem, nr_pages);
3097 page_counter_uncharge(&memcg->kmem, -nr_pages);
3110 struct mem_cgroup *memcg;
3112 memcg = get_mem_cgroup_from_objcg(objcg);
3114 memcg_account_kmem(memcg, -nr_pages);
3115 refill_stock(memcg, nr_pages);
3117 css_put(&memcg->css);
3131 struct mem_cgroup *memcg;
3134 memcg = get_mem_cgroup_from_objcg(objcg);
3136 ret = try_charge_memcg(memcg, gfp, nr_pages);
3140 memcg_account_kmem(memcg, nr_pages);
3142 css_put(&memcg->css);
3290 struct mem_cgroup *memcg;
3292 memcg = get_mem_cgroup_from_objcg(old);
3294 memcg_account_kmem(memcg, -nr_pages);
3295 __refill_stock(memcg, nr_pages);
3297 css_put(&memcg->css);
3301 * The leftover is flushed to the centralized per-memcg value.
3345 struct mem_cgroup *memcg;
3348 memcg = obj_cgroup_memcg(objcg);
3349 if (memcg && mem_cgroup_is_descendant(memcg, root_memcg))
3404 * performance bottleneck if all tasks of the same memcg are
3447 struct mem_cgroup *memcg = folio_memcg(folio);
3450 if (mem_cgroup_disabled() || !memcg)
3459 css_get_many(&memcg->css, nr - 1);
3502 static int mem_cgroup_resize_max(struct mem_cgroup *memcg,
3509 struct page_counter *counter = memsw ? &memcg->memsw : &memcg->memory;
3522 limits_invariant = memsw ? max >= READ_ONCE(memcg->memory.max) :
3523 max <= memcg->memsw.max;
3538 drain_all_stock(memcg);
3543 if (!try_to_free_mem_cgroup_pages(memcg, 1, GFP_KERNEL,
3551 memcg_oom_recover(memcg);
3596 reclaimed = mem_cgroup_soft_reclaim(mz->memcg, pgdat,
3609 excess = soft_limit_excess(mz->memcg);
3621 css_put(&mz->memcg->css);
3634 css_put(&next_mz->memcg->css);
3639 * Reclaims as many pages from the given memcg as possible.
3641 * Caller is responsible for holding css reference for memcg.
3643 static int mem_cgroup_force_empty(struct mem_cgroup *memcg)
3650 drain_all_stock(memcg);
3653 while (nr_retries && page_counter_read(&memcg->memory)) {
3657 if (!try_to_free_mem_cgroup_pages(memcg, 1, GFP_KERNEL,
3669 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
3671 if (mem_cgroup_is_root(memcg))
3673 return mem_cgroup_force_empty(memcg) ?: nbytes;
3695 static unsigned long mem_cgroup_usage(struct mem_cgroup *memcg, bool swap)
3699 if (mem_cgroup_is_root(memcg)) {
3710 val = page_counter_read(&memcg->memory);
3712 val = page_counter_read(&memcg->memsw);
3728 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
3733 counter = &memcg->memory;
3736 counter = &memcg->memsw;
3739 counter = &memcg->kmem;
3742 counter = &memcg->tcpmem;
3750 if (counter == &memcg->memory)
3751 return (u64)mem_cgroup_usage(memcg, false) * PAGE_SIZE;
3752 if (counter == &memcg->memsw)
3753 return (u64)mem_cgroup_usage(memcg, true) * PAGE_SIZE;
3762 return (u64)READ_ONCE(memcg->soft_limit) * PAGE_SIZE;
3779 static int memcg_online_kmem(struct mem_cgroup *memcg)
3786 if (unlikely(mem_cgroup_is_root(memcg)))
3793 objcg->memcg = memcg;
3794 rcu_assign_pointer(memcg->objcg, objcg);
3798 memcg->kmemcg_id = memcg->id.id;
3803 static void memcg_offline_kmem(struct mem_cgroup *memcg)
3810 if (unlikely(mem_cgroup_is_root(memcg)))
3813 parent = parent_mem_cgroup(memcg);
3817 memcg_reparent_objcgs(memcg, parent);
3825 memcg_reparent_list_lrus(memcg, parent);
3828 static int memcg_online_kmem(struct mem_cgroup *memcg)
3832 static void memcg_offline_kmem(struct mem_cgroup *memcg)
3837 static int memcg_update_tcp_max(struct mem_cgroup *memcg, unsigned long max)
3843 ret = page_counter_set_max(&memcg->tcpmem, max);
3847 if (!memcg->tcpmem_active) {
3853 * belonging to this memcg until that flag is up.
3865 memcg->tcpmem_active = true;
3879 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
3890 if (mem_cgroup_is_root(memcg)) { /* Can't set limit on root */
3896 ret = mem_cgroup_resize_max(memcg, nr_pages, false);
3899 ret = mem_cgroup_resize_max(memcg, nr_pages, true);
3909 ret = memcg_update_tcp_max(memcg, nr_pages);
3917 WRITE_ONCE(memcg->soft_limit, nr_pages);
3928 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
3933 counter = &memcg->memory;
3936 counter = &memcg->memsw;
3939 counter = &memcg->kmem;
3942 counter = &memcg->tcpmem;
3972 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
3987 memcg->move_charge_at_immigrate = val;
4004 static unsigned long mem_cgroup_node_nr_lru_pages(struct mem_cgroup *memcg,
4007 struct lruvec *lruvec = mem_cgroup_lruvec(memcg, NODE_DATA(nid));
4024 static unsigned long mem_cgroup_nr_lru_pages(struct mem_cgroup *memcg,
4035 nr += memcg_page_state(memcg, NR_LRU_BASE + lru);
4037 nr += memcg_page_state_local(memcg, NR_LRU_BASE + lru);
4057 struct mem_cgroup *memcg = mem_cgroup_from_seq(m);
4063 mem_cgroup_nr_lru_pages(memcg, stat->lru_mask,
4067 mem_cgroup_node_nr_lru_pages(memcg, nid,
4075 mem_cgroup_nr_lru_pages(memcg, stat->lru_mask,
4079 mem_cgroup_node_nr_lru_pages(memcg, nid,
4126 static void memcg1_stat_format(struct mem_cgroup *memcg, struct seq_buf *s)
4141 nr = memcg_page_state_local(memcg, memcg1_stats[i]);
4148 memcg_events_local(memcg, memcg1_events[i]));
4152 memcg_page_state_local(memcg, NR_LRU_BASE + i) *
4157 for (mi = memcg; mi; mi = parent_mem_cgroup(mi)) {
4172 nr = memcg_page_state(memcg, memcg1_stats[i]);
4180 (u64)memcg_events(memcg, memcg1_events[i]));
4184 (u64)memcg_page_state(memcg, NR_LRU_BASE + i) *
4195 mz = memcg->nodeinfo[pgdat->node_id];
4209 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
4211 return mem_cgroup_swappiness(memcg);
4217 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
4222 if (!mem_cgroup_is_root(memcg))
4223 WRITE_ONCE(memcg->swappiness, val);
4230 static void __mem_cgroup_threshold(struct mem_cgroup *memcg, bool swap)
4238 t = rcu_dereference(memcg->thresholds.primary);
4240 t = rcu_dereference(memcg->memsw_thresholds.primary);
4245 usage = mem_cgroup_usage(memcg, swap);
4281 static void mem_cgroup_threshold(struct mem_cgroup *memcg)
4283 while (memcg) {
4284 __mem_cgroup_threshold(memcg, false);
4286 __mem_cgroup_threshold(memcg, true);
4288 memcg = parent_mem_cgroup(memcg);
4306 static int mem_cgroup_oom_notify_cb(struct mem_cgroup *memcg)
4312 list_for_each_entry(ev, &memcg->oom_notify, list)
4319 static void mem_cgroup_oom_notify(struct mem_cgroup *memcg)
4323 for_each_mem_cgroup_tree(iter, memcg)
4327 static int __mem_cgroup_usage_register_event(struct mem_cgroup *memcg,
4340 mutex_lock(&memcg->thresholds_lock);
4343 thresholds = &memcg->thresholds;
4344 usage = mem_cgroup_usage(memcg, false);
4346 thresholds = &memcg->memsw_thresholds;
4347 usage = mem_cgroup_usage(memcg, true);
4353 __mem_cgroup_threshold(memcg, type == _MEMSWAP);
4402 mutex_unlock(&memcg->thresholds_lock);
4407 static int mem_cgroup_usage_register_event(struct mem_cgroup *memcg,
4410 return __mem_cgroup_usage_register_event(memcg, eventfd, args, _MEM);
4413 static int memsw_cgroup_usage_register_event(struct mem_cgroup *memcg,
4416 return __mem_cgroup_usage_register_event(memcg, eventfd, args, _MEMSWAP);
4419 static void __mem_cgroup_usage_unregister_event(struct mem_cgroup *memcg,
4427 mutex_lock(&memcg->thresholds_lock);
4430 thresholds = &memcg->thresholds;
4431 usage = mem_cgroup_usage(memcg, false);
4433 thresholds = &memcg->memsw_thresholds;
4434 usage = mem_cgroup_usage(memcg, true);
4442 __mem_cgroup_threshold(memcg, type == _MEMSWAP);
4501 mutex_unlock(&memcg->thresholds_lock);
4504 static void mem_cgroup_usage_unregister_event(struct mem_cgroup *memcg,
4507 return __mem_cgroup_usage_unregister_event(memcg, eventfd, _MEM);
4510 static void memsw_cgroup_usage_unregister_event(struct mem_cgroup *memcg,
4513 return __mem_cgroup_usage_unregister_event(memcg, eventfd, _MEMSWAP);
4516 static int mem_cgroup_oom_register_event(struct mem_cgroup *memcg,
4528 list_add(&event->list, &memcg->oom_notify);
4531 if (memcg->under_oom)
4538 static void mem_cgroup_oom_unregister_event(struct mem_cgroup *memcg,
4545 list_for_each_entry_safe(ev, tmp, &memcg->oom_notify, list) {
4557 struct mem_cgroup *memcg = mem_cgroup_from_seq(sf);
4559 seq_printf(sf, "oom_kill_disable %d\n", READ_ONCE(memcg->oom_kill_disable));
4560 seq_printf(sf, "under_oom %d\n", (bool)memcg->under_oom);
4562 atomic_long_read(&memcg->memory_events[MEMCG_OOM_KILL]));
4569 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
4572 if (mem_cgroup_is_root(memcg) || !((val == 0) || (val == 1)))
4575 WRITE_ONCE(memcg->oom_kill_disable, val);
4577 memcg_oom_recover(memcg);
4586 static int memcg_wb_domain_init(struct mem_cgroup *memcg, gfp_t gfp)
4588 return wb_domain_init(&memcg->cgwb_domain, gfp);
4591 static void memcg_wb_domain_exit(struct mem_cgroup *memcg)
4593 wb_domain_exit(&memcg->cgwb_domain);
4596 static void memcg_wb_domain_size_changed(struct mem_cgroup *memcg)
4598 wb_domain_size_changed(&memcg->cgwb_domain);
4603 struct mem_cgroup *memcg = mem_cgroup_from_css(wb->memcg_css);
4605 if (!memcg->css.parent)
4608 return &memcg->cgwb_domain;
4612 * mem_cgroup_wb_stats - retrieve writeback related stats from its memcg
4615 * @pheadroom: out parameter for number of allocatable pages according to memcg
4620 * @wb's memcg. File, dirty and writeback are self-explanatory. Headroom
4623 * A memcg's headroom is "min(max, high) - used". In the hierarchy, the
4633 struct mem_cgroup *memcg = mem_cgroup_from_css(wb->memcg_css);
4638 *pdirty = memcg_page_state(memcg, NR_FILE_DIRTY);
4639 *pwriteback = memcg_page_state(memcg, NR_WRITEBACK);
4640 *pfilepages = memcg_page_state(memcg, NR_INACTIVE_FILE) +
4641 memcg_page_state(memcg, NR_ACTIVE_FILE);
4644 while ((parent = parent_mem_cgroup(memcg))) {
4645 unsigned long ceiling = min(READ_ONCE(memcg->memory.max),
4646 READ_ONCE(memcg->memory.high));
4647 unsigned long used = page_counter_read(&memcg->memory);
4650 memcg = parent;
4657 * There's an inherent mismatch between memcg and writeback. The former
4679 * granularities between memcg and writeback in either direction. However,
4685 * page - a page whose memcg and writeback ownerships don't match - is
4687 * bdi_writeback on the page owning memcg. When balance_dirty_pages()
4688 * decides that the memcg needs to sleep due to high dirty ratio, it calls
4701 struct mem_cgroup *memcg = folio_memcg(folio);
4716 frn = &memcg->cgwb_frn[i];
4743 frn = &memcg->cgwb_frn[oldest];
4753 struct mem_cgroup *memcg = mem_cgroup_from_css(wb->memcg_css);
4759 struct memcg_cgwb_frn *frn = &memcg->cgwb_frn[i];
4780 static int memcg_wb_domain_init(struct mem_cgroup *memcg, gfp_t gfp)
4785 static void memcg_wb_domain_exit(struct mem_cgroup *memcg)
4789 static void memcg_wb_domain_size_changed(struct mem_cgroup *memcg)
4817 struct mem_cgroup *memcg = event->memcg;
4821 event->unregister_event(memcg, event->eventfd);
4828 css_put(&memcg->css);
4841 struct mem_cgroup *memcg = event->memcg;
4854 spin_lock(&memcg->event_list_lock);
4863 spin_unlock(&memcg->event_list_lock);
4891 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
4921 event->memcg = memcg;
5003 ret = event->register_event(memcg, event->eventfd, buf);
5009 spin_lock_irq(&memcg->event_list_lock);
5010 list_add(&event->list, &memcg->event_list);
5011 spin_unlock_irq(&memcg->event_list_lock);
5177 * Swap-out records and page cache shadow entries need to store memcg
5189 * Maintain a private 16-bit ID space for memcg, and allow the ID to
5201 static void mem_cgroup_id_remove(struct mem_cgroup *memcg)
5203 if (memcg->id.id > 0) {
5204 idr_remove(&mem_cgroup_idr, memcg->id.id);
5205 memcg->id.id = 0;
5209 static void __maybe_unused mem_cgroup_id_get_many(struct mem_cgroup *memcg,
5212 refcount_add(n, &memcg->id.ref);
5215 static void mem_cgroup_id_put_many(struct mem_cgroup *memcg, unsigned int n)
5217 if (refcount_sub_and_test(n, &memcg->id.ref)) {
5218 mem_cgroup_id_remove(memcg);
5221 css_put(&memcg->css);
5225 static inline void mem_cgroup_id_put(struct mem_cgroup *memcg)
5227 mem_cgroup_id_put_many(memcg, 1);
5231 * mem_cgroup_from_id - look up a memcg from a memcg id
5232 * @id: the memcg id to look up
5251 struct mem_cgroup *memcg;
5259 memcg = container_of(css, struct mem_cgroup, css);
5261 memcg = ERR_PTR(-ENOENT);
5265 return memcg;
5269 static int alloc_mem_cgroup_per_node_info(struct mem_cgroup *memcg, int node)
5288 pn->memcg = memcg;
5290 memcg->nodeinfo[node] = pn;
5294 static void free_mem_cgroup_per_node_info(struct mem_cgroup *memcg, int node)
5296 struct mem_cgroup_per_node *pn = memcg->nodeinfo[node];
5305 static void __mem_cgroup_free(struct mem_cgroup *memcg)
5310 free_mem_cgroup_per_node_info(memcg, node);
5311 kfree(memcg->vmstats);
5312 free_percpu(memcg->vmstats_percpu);
5313 kfree(memcg);
5316 static void mem_cgroup_free(struct mem_cgroup *memcg)
5318 lru_gen_exit_memcg(memcg);
5319 memcg_wb_domain_exit(memcg);
5320 __mem_cgroup_free(memcg);
5325 struct mem_cgroup *memcg;
5330 memcg = kzalloc(struct_size(memcg, nodeinfo, nr_node_ids), GFP_KERNEL);
5331 if (!memcg)
5334 memcg->id.id = idr_alloc(&mem_cgroup_idr, NULL,
5336 if (memcg->id.id < 0) {
5337 error = memcg->id.id;
5341 memcg->vmstats = kzalloc(sizeof(struct memcg_vmstats), GFP_KERNEL);
5342 if (!memcg->vmstats)
5345 memcg->vmstats_percpu = alloc_percpu_gfp(struct memcg_vmstats_percpu,
5347 if (!memcg->vmstats_percpu)
5351 if (alloc_mem_cgroup_per_node_info(memcg, node))
5354 if (memcg_wb_domain_init(memcg, GFP_KERNEL))
5357 INIT_WORK(&memcg->high_work, high_work_func);
5358 INIT_LIST_HEAD(&memcg->oom_notify);
5359 mutex_init(&memcg->thresholds_lock);
5360 spin_lock_init(&memcg->move_lock);
5361 vmpressure_init(&memcg->vmpressure);
5362 INIT_LIST_HEAD(&memcg->event_list);
5363 spin_lock_init(&memcg->event_list_lock);
5364 memcg->socket_pressure = jiffies;
5366 memcg->kmemcg_id = -1;
5367 INIT_LIST_HEAD(&memcg->objcg_list);
5370 INIT_LIST_HEAD(&memcg->cgwb_list);
5372 memcg->cgwb_frn[i].done =
5376 spin_lock_init(&memcg->deferred_split_queue.split_queue_lock);
5377 INIT_LIST_HEAD(&memcg->deferred_split_queue.split_queue);
5378 memcg->deferred_split_queue.split_queue_len = 0;
5389 INIT_LIST_HEAD(&memcg->score_node);
5392 lru_gen_init_memcg(memcg);
5393 return memcg;
5395 mem_cgroup_id_remove(memcg);
5396 __mem_cgroup_free(memcg);
5404 struct mem_cgroup *memcg, *old_memcg;
5407 memcg = mem_cgroup_alloc();
5409 if (IS_ERR(memcg))
5410 return ERR_CAST(memcg);
5413 atomic64_set(&memcg->memcg_reclaimed.app_score, 300);
5416 atomic_set(&memcg->memcg_reclaimed.ub_zram2ufs_ratio, 10);
5417 atomic_set(&memcg->memcg_reclaimed.ub_mem2zram_ratio, 60);
5418 atomic_set(&memcg->memcg_reclaimed.refault_threshold, 50);
5420 page_counter_set_high(&memcg->memory, PAGE_COUNTER_MAX);
5421 WRITE_ONCE(memcg->soft_limit, PAGE_COUNTER_MAX);
5423 memcg->zswap_max = PAGE_COUNTER_MAX;
5425 page_counter_set_high(&memcg->swap, PAGE_COUNTER_MAX);
5427 WRITE_ONCE(memcg->swappiness, mem_cgroup_swappiness(parent));
5428 WRITE_ONCE(memcg->oom_kill_disable, READ_ONCE(parent->oom_kill_disable));
5430 page_counter_init(&memcg->memory, &parent->memory);
5431 page_counter_init(&memcg->swap, &parent->swap);
5432 page_counter_init(&memcg->kmem, &parent->kmem);
5433 page_counter_init(&memcg->tcpmem, &parent->tcpmem);
5436 page_counter_init(&memcg->memory, NULL);
5437 page_counter_init(&memcg->swap, NULL);
5438 page_counter_init(&memcg->kmem, NULL);
5439 page_counter_init(&memcg->tcpmem, NULL);
5441 root_mem_cgroup = memcg;
5442 return &memcg->css;
5453 return &memcg->css;
5458 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
5460 if (memcg_online_kmem(memcg))
5464 * A memcg must be visible for expand_shrinker_info()
5468 if (alloc_shrinker_info(memcg))
5471 if (unlikely(mem_cgroup_is_root(memcg)))
5474 lru_gen_online_memcg(memcg);
5477 memcg_app_score_update(memcg);
5481 /* Online state pins memcg ID, memcg ID pins CSS */
5482 refcount_set(&memcg->id.ref, 1);
5495 idr_replace(&mem_cgroup_idr, memcg, memcg->id.id);
5499 memcg_offline_kmem(memcg);
5501 mem_cgroup_id_remove(memcg);
5507 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
5514 list_del_init(&memcg->score_node);
5524 spin_lock_irq(&memcg->event_list_lock);
5525 list_for_each_entry_safe(event, tmp, &memcg->event_list, list) {
5529 spin_unlock_irq(&memcg->event_list_lock);
5531 page_counter_set_min(&memcg->memory, 0);
5532 page_counter_set_low(&memcg->memory, 0);
5534 memcg_offline_kmem(memcg);
5535 reparent_shrinker_deferred(memcg);
5536 wb_memcg_offline(memcg);
5537 lru_gen_offline_memcg(memcg);
5539 drain_all_stock(memcg);
5541 mem_cgroup_id_put(memcg);
5546 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
5548 invalidate_reclaim_iterators(memcg);
5549 lru_gen_release_memcg(memcg);
5554 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
5559 wb_wait_for_completion(&memcg->cgwb_frn[i].done);
5564 if (!cgroup_subsys_on_dfl(memory_cgrp_subsys) && memcg->tcpmem_active)
5572 vmpressure_cleanup(&memcg->vmpressure);
5573 cancel_work_sync(&memcg->high_work);
5574 mem_cgroup_remove_from_trees(memcg);
5575 free_shrinker_info(memcg);
5576 mem_cgroup_free(memcg);
5585 * but the memcg is pinned through dependency. The memcg should stop
5594 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
5596 page_counter_set_max(&memcg->memory, PAGE_COUNTER_MAX);
5597 page_counter_set_max(&memcg->swap, PAGE_COUNTER_MAX);
5598 page_counter_set_max(&memcg->kmem, PAGE_COUNTER_MAX);
5599 page_counter_set_max(&memcg->tcpmem, PAGE_COUNTER_MAX);
5600 page_counter_set_min(&memcg->memory, 0);
5601 page_counter_set_low(&memcg->memory, 0);
5602 page_counter_set_high(&memcg->memory, PAGE_COUNTER_MAX);
5603 WRITE_ONCE(memcg->soft_limit, PAGE_COUNTER_MAX);
5604 page_counter_set_high(&memcg->swap, PAGE_COUNTER_MAX);
5605 memcg_wb_domain_size_changed(memcg);
5610 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
5611 struct mem_cgroup *parent = parent_mem_cgroup(memcg);
5616 statc = per_cpu_ptr(memcg->vmstats_percpu, cpu);
5624 delta = memcg->vmstats->state_pending[i];
5626 memcg->vmstats->state_pending[i] = 0;
5639 memcg->vmstats->state_local[i] += delta_cpu;
5642 memcg->vmstats->state[i] += delta;
5649 delta = memcg->vmstats->events_pending[i];
5651 memcg->vmstats->events_pending[i] = 0;
5662 memcg->vmstats->events_local[i] += delta_cpu;
5665 memcg->vmstats->events[i] += delta;
5672 struct mem_cgroup_per_node *pn = memcg->nodeinfo[nid];
5909 * All state has been migrated, let's switch to the new memcg.
5911 * It is safe to change page's memcg here because the page
5916 * Note that folio_memcg_lock is a memcg lock, not a page lock,
5918 * new memcg that isn't locked, the above state can change
6206 struct mem_cgroup *memcg = NULL; /* unneeded init to make gcc happy */
6227 memcg = mem_cgroup_from_css(css);
6237 move_flags = READ_ONCE(memcg->move_charge_at_immigrate);
6243 VM_BUG_ON(from == memcg);
6259 mc.to = memcg;
6346 * memcg. There should be somebody mapping the head.
6404 * Signal folio_memcg_lock() to take the memcg's move_lock
6405 * while we're moving its pages to another memcg. Then wait
6489 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
6491 return (u64)page_counter_read(&memcg->memory) * PAGE_SIZE;
6497 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
6499 return (u64)memcg->memory.watermark * PAGE_SIZE;
6511 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
6520 page_counter_set_min(&memcg->memory, min);
6534 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
6543 page_counter_set_low(&memcg->memory, low);
6557 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
6568 page_counter_set_high(&memcg->memory, high);
6571 unsigned long nr_pages = page_counter_read(&memcg->memory);
6581 drain_all_stock(memcg);
6586 reclaimed = try_to_free_mem_cgroup_pages(memcg, nr_pages - high,
6593 memcg_wb_domain_size_changed(memcg);
6606 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
6617 xchg(&memcg->memory.max, max);
6620 unsigned long nr_pages = page_counter_read(&memcg->memory);
6629 drain_all_stock(memcg);
6635 if (!try_to_free_mem_cgroup_pages(memcg, nr_pages - max,
6641 memcg_memory_event(memcg, MEMCG_OOM);
6642 if (!mem_cgroup_out_of_memory(memcg, GFP_KERNEL, 0))
6646 memcg_wb_domain_size_changed(memcg);
6664 struct mem_cgroup *memcg = mem_cgroup_from_seq(m);
6666 __memory_events_show(m, memcg->memory_events);
6672 struct mem_cgroup *memcg = mem_cgroup_from_seq(m);
6674 __memory_events_show(m, memcg->memory_events_local);
6680 struct mem_cgroup *memcg = mem_cgroup_from_seq(m);
6687 memory_stat_format(memcg, &s);
6706 struct mem_cgroup *memcg = mem_cgroup_from_seq(m);
6721 lruvec = mem_cgroup_lruvec(memcg, NODE_DATA(nid));
6735 struct mem_cgroup *memcg = mem_cgroup_from_seq(m);
6737 seq_printf(m, "%d\n", READ_ONCE(memcg->oom_group));
6745 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
6759 WRITE_ONCE(memcg->oom_group, oom_group);
6767 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
6793 reclaimed = try_to_free_mem_cgroup_pages(memcg,
7013 * @memcg: the memory cgroup to check
7019 struct mem_cgroup *memcg)
7037 if (memcg == root)
7040 usage = page_counter_read(&memcg->memory);
7044 parent = parent_mem_cgroup(memcg);
7047 memcg->memory.emin = READ_ONCE(memcg->memory.min);
7048 memcg->memory.elow = READ_ONCE(memcg->memory.low);
7054 WRITE_ONCE(memcg->memory.emin, effective_protection(usage, parent_usage,
7055 READ_ONCE(memcg->memory.min),
7059 WRITE_ONCE(memcg->memory.elow, effective_protection(usage, parent_usage,
7060 READ_ONCE(memcg->memory.low),
7065 static int charge_memcg(struct folio *folio, struct mem_cgroup *memcg,
7071 ret = try_charge(memcg, gfp, nr_pages);
7075 css_get(&memcg->css);
7076 commit_charge(folio, memcg);
7079 mem_cgroup_charge_statistics(memcg, nr_pages);
7080 memcg_check_events(memcg, folio_nid(folio));
7088 struct mem_cgroup *memcg;
7091 memcg = get_mem_cgroup_from_mm(mm);
7092 ret = charge_memcg(folio, memcg, gfp);
7093 css_put(&memcg->css);
7113 struct mem_cgroup *memcg;
7122 memcg = mem_cgroup_from_id(id);
7123 if (!memcg || !css_tryget_online(&memcg->css))
7124 memcg = get_mem_cgroup_from_mm(mm);
7127 ret = charge_memcg(folio, memcg, gfp);
7129 css_put(&memcg->css);
7167 struct mem_cgroup *memcg;
7184 page_counter_uncharge(&ug->memcg->memory, ug->nr_memory);
7186 page_counter_uncharge(&ug->memcg->memsw, ug->nr_memory);
7188 memcg_account_kmem(ug->memcg, -ug->nr_kmem);
7189 memcg_oom_recover(ug->memcg);
7193 __count_memcg_events(ug->memcg, PGPGOUT, ug->pgpgout);
7194 __this_cpu_add(ug->memcg->vmstats_percpu->nr_page_events, ug->nr_memory);
7195 memcg_check_events(ug->memcg, ug->nid);
7199 css_put(&ug->memcg->css);
7205 struct mem_cgroup *memcg;
7212 * folio memcg or objcg at this point, we have fully
7219 * kmem pages do not hold memcg references anymore.
7221 memcg = get_mem_cgroup_from_objcg(objcg);
7223 memcg = __folio_memcg(folio);
7226 if (!memcg)
7229 if (ug->memcg != memcg) {
7230 if (ug->memcg) {
7234 ug->memcg = memcg;
7238 css_get(&memcg->css);
7251 if (!mem_cgroup_is_root(memcg))
7258 css_put(&memcg->css);
7289 if (ug.memcg)
7305 struct mem_cgroup *memcg;
7321 memcg = folio_memcg(old);
7322 VM_WARN_ON_ONCE_FOLIO(!memcg, old);
7323 if (!memcg)
7327 if (!mem_cgroup_is_root(memcg)) {
7328 page_counter_charge(&memcg->memory, nr_pages);
7330 page_counter_charge(&memcg->memsw, nr_pages);
7333 css_get(&memcg->css);
7334 commit_charge(new, memcg);
7337 mem_cgroup_charge_statistics(memcg, nr_pages);
7338 memcg_check_events(memcg, folio_nid(new));
7347 struct mem_cgroup *memcg;
7352 /* Do not associate the sock with unrelated interrupted task's memcg. */
7357 memcg = mem_cgroup_from_task(current);
7358 if (mem_cgroup_is_root(memcg))
7360 if (!cgroup_subsys_on_dfl(memory_cgrp_subsys) && !memcg->tcpmem_active)
7362 if (css_tryget(&memcg->css))
7363 sk->sk_memcg = memcg;
7376 * @memcg: memcg to charge
7380 * Charges @nr_pages to @memcg. Returns %true if the charge fit within
7381 * @memcg's configured limit, %false if it doesn't.
7383 bool mem_cgroup_charge_skmem(struct mem_cgroup *memcg, unsigned int nr_pages,
7389 if (page_counter_try_charge(&memcg->tcpmem, nr_pages, &fail)) {
7390 memcg->tcpmem_pressure = 0;
7393 memcg->tcpmem_pressure = 1;
7395 page_counter_charge(&memcg->tcpmem, nr_pages);
7401 if (try_charge(memcg, gfp_mask, nr_pages) == 0) {
7402 mod_memcg_state(memcg, MEMCG_SOCK, nr_pages);
7411 * @memcg: memcg to uncharge
7414 void mem_cgroup_uncharge_skmem(struct mem_cgroup *memcg, unsigned int nr_pages)
7417 page_counter_uncharge(&memcg->tcpmem, nr_pages);
7421 mod_memcg_state(memcg, MEMCG_SOCK, -nr_pages);
7423 refill_stock(memcg, nr_pages);
7460 * used for per-memcg-per-cpu caching of per-node statistics. In order
7489 static struct mem_cgroup *mem_cgroup_id_get_online(struct mem_cgroup *memcg)
7491 while (!refcount_inc_not_zero(&memcg->id.ref)) {
7496 if (WARN_ON_ONCE(mem_cgroup_is_root(memcg))) {
7500 memcg = parent_mem_cgroup(memcg);
7501 if (!memcg)
7502 memcg = root_mem_cgroup;
7504 return memcg;
7516 struct mem_cgroup *memcg, *swap_memcg;
7529 memcg = folio_memcg(folio);
7531 VM_WARN_ON_ONCE_FOLIO(!memcg, folio);
7532 if (!memcg)
7536 * In case the memcg owning these pages has been offlined and doesn't
7540 swap_memcg = mem_cgroup_id_get_online(memcg);
7552 if (!mem_cgroup_is_root(memcg))
7553 page_counter_uncharge(&memcg->memory, nr_entries);
7555 if (memcg != swap_memcg) {
7558 page_counter_uncharge(&memcg->memsw, nr_entries);
7568 mem_cgroup_charge_statistics(memcg, -nr_entries);
7570 memcg_check_events(memcg, folio_nid(folio));
7572 css_put(&memcg->css);
7580 * Try to charge @folio's memcg for the swap space at @entry.
7588 struct mem_cgroup *memcg;
7594 memcg = folio_memcg(folio);
7596 VM_WARN_ON_ONCE_FOLIO(!memcg, folio);
7597 if (!memcg)
7601 memcg_memory_event(memcg, MEMCG_SWAP_FAIL);
7605 memcg = mem_cgroup_id_get_online(memcg);
7607 if (!mem_cgroup_is_root(memcg) &&
7608 !page_counter_try_charge(&memcg->swap, nr_pages, &counter)) {
7609 memcg_memory_event(memcg, MEMCG_SWAP_MAX);
7610 memcg_memory_event(memcg, MEMCG_SWAP_FAIL);
7611 mem_cgroup_id_put(memcg);
7617 mem_cgroup_id_get_many(memcg, nr_pages - 1);
7618 oldid = swap_cgroup_record(entry, mem_cgroup_id(memcg), nr_pages);
7620 mod_memcg_state(memcg, MEMCG_SWAP, nr_pages);
7632 struct mem_cgroup *memcg;
7637 memcg = mem_cgroup_from_id(id);
7638 if (memcg) {
7639 if (!mem_cgroup_is_root(memcg)) {
7641 page_counter_uncharge(&memcg->memsw, nr_pages);
7643 page_counter_uncharge(&memcg->swap, nr_pages);
7645 mod_memcg_state(memcg, MEMCG_SWAP, -nr_pages);
7646 mem_cgroup_id_put_many(memcg, nr_pages);
7651 long mem_cgroup_get_nr_swap_pages(struct mem_cgroup *memcg)
7657 for (; !mem_cgroup_is_root(memcg); memcg = parent_mem_cgroup(memcg))
7659 READ_ONCE(memcg->swap.max) -
7660 page_counter_read(&memcg->swap));
7666 struct mem_cgroup *memcg;
7675 memcg = folio_memcg(folio);
7676 if (!memcg)
7679 for (; !mem_cgroup_is_root(memcg); memcg = parent_mem_cgroup(memcg)) {
7680 unsigned long usage = page_counter_read(&memcg->swap);
7682 if (usage * 2 >= READ_ONCE(memcg->swap.high) ||
7683 usage * 2 >= READ_ONCE(memcg->swap.max))
7706 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
7708 return (u64)page_counter_read(&memcg->swap) * PAGE_SIZE;
7714 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
7716 return (u64)memcg->swap.watermark * PAGE_SIZE;
7728 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
7737 page_counter_set_high(&memcg->swap, high);
7751 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
7760 xchg(&memcg->swap.max, max);
7767 struct mem_cgroup *memcg = mem_cgroup_from_seq(m);
7770 atomic_long_read(&memcg->memory_events[MEMCG_SWAP_HIGH]));
7772 atomic_long_read(&memcg->memory_events[MEMCG_SWAP_MAX]));
7774 atomic_long_read(&memcg->memory_events[MEMCG_SWAP_FAIL]));
7853 struct mem_cgroup *memcg, *original_memcg;
7860 for (memcg = original_memcg; !mem_cgroup_is_root(memcg);
7861 memcg = parent_mem_cgroup(memcg)) {
7862 unsigned long max = READ_ONCE(memcg->zswap_max);
7872 cgroup_rstat_flush(memcg->css.cgroup);
7873 pages = memcg_page_state(memcg, MEMCG_ZSWAP_B) / PAGE_SIZE;
7893 struct mem_cgroup *memcg;
7905 memcg = obj_cgroup_memcg(objcg);
7906 mod_memcg_state(memcg, MEMCG_ZSWAP_B, size);
7907 mod_memcg_state(memcg, MEMCG_ZSWAPPED, 1);
7920 struct mem_cgroup *memcg;
7928 memcg = obj_cgroup_memcg(objcg);
7929 mod_memcg_state(memcg, MEMCG_ZSWAP_B, -size);
7930 mod_memcg_state(memcg, MEMCG_ZSWAPPED, -1);
7950 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
7959 xchg(&memcg->zswap_max, max);