Lines Matching refs:mem_cgroup
75 struct mem_cgroup *root_mem_cgroup __read_mostly;
78 DEFINE_PER_CPU(struct mem_cgroup *, int_active_memcg);
136 struct mem_cgroup *memcg;
150 int (*register_event)(struct mem_cgroup *memcg,
157 void (*unregister_event)(struct mem_cgroup *memcg,
169 static void mem_cgroup_threshold(struct mem_cgroup *memcg);
170 static void mem_cgroup_oom_notify(struct mem_cgroup *memcg);
184 struct mem_cgroup *from;
185 struct mem_cgroup *to;
241 struct vmpressure *memcg_to_vmpressure(struct mem_cgroup *memcg)
250 return &container_of(vmpr, struct mem_cgroup, vmpressure)->css;
259 struct mem_cgroup *memcg;
319 static void memcg_reparent_objcgs(struct mem_cgroup *memcg,
320 struct mem_cgroup *parent)
406 static int memcg_expand_one_shrinker_map(struct mem_cgroup *memcg,
436 static void memcg_free_shrinker_maps(struct mem_cgroup *memcg)
454 static int memcg_alloc_shrinker_maps(struct mem_cgroup *memcg)
481 struct mem_cgroup *memcg;
508 void memcg_set_shrinker_bit(struct mem_cgroup *memcg, int nid, int shrinker_id)
535 struct mem_cgroup *memcg;
537 memcg = page->mem_cgroup;
560 struct mem_cgroup *memcg;
564 memcg = page->mem_cgroup;
584 mem_cgroup_page_nodeinfo(struct mem_cgroup *memcg, struct page *page)
668 static unsigned long soft_limit_excess(struct mem_cgroup *memcg)
688 static void mem_cgroup_update_tree(struct mem_cgroup *memcg, struct page *page)
725 static void mem_cgroup_remove_from_trees(struct mem_cgroup *memcg)
781 void __mod_memcg_state(struct mem_cgroup *memcg, int idx, int val)
793 struct mem_cgroup *mi;
810 struct mem_cgroup *parent;
822 struct mem_cgroup *memcg;
878 struct mem_cgroup *memcg;
901 struct mem_cgroup *memcg;
916 void __count_memcg_events(struct mem_cgroup *memcg, enum vm_event_item idx,
930 struct mem_cgroup *mi;
944 static unsigned long memcg_events(struct mem_cgroup *memcg, int event)
949 static unsigned long memcg_events_local(struct mem_cgroup *memcg, int event)
959 static void mem_cgroup_charge_statistics(struct mem_cgroup *memcg,
974 static bool mem_cgroup_event_ratelimit(struct mem_cgroup *memcg,
1003 static void memcg_check_events(struct mem_cgroup *memcg, struct page *page)
1018 struct mem_cgroup *mem_cgroup_from_task(struct task_struct *p)
1037 * root_mem_cgroup is returned. However if mem_cgroup is disabled, NULL is
1040 struct mem_cgroup *get_mem_cgroup_from_mm(struct mm_struct *mm)
1042 struct mem_cgroup *memcg;
1074 struct mem_cgroup *get_mem_cgroup_from_page(struct page *page)
1076 struct mem_cgroup *memcg = page->mem_cgroup;
1090 static __always_inline struct mem_cgroup *active_memcg(void)
1098 static __always_inline struct mem_cgroup *get_active_memcg(void)
1100 struct mem_cgroup *memcg;
1128 static __always_inline struct mem_cgroup *get_mem_cgroup_from_current(void)
1156 struct mem_cgroup *mem_cgroup_iter(struct mem_cgroup *root,
1157 struct mem_cgroup *prev,
1162 struct mem_cgroup *memcg = NULL;
1163 struct mem_cgroup *pos = NULL;
1271 void mem_cgroup_iter_break(struct mem_cgroup *root,
1272 struct mem_cgroup *prev)
1280 static void __invalidate_reclaim_iterators(struct mem_cgroup *from,
1281 struct mem_cgroup *dead_memcg)
1294 static void invalidate_reclaim_iterators(struct mem_cgroup *dead_memcg)
1296 struct mem_cgroup *memcg = dead_memcg;
1297 struct mem_cgroup *last;
1328 int mem_cgroup_scan_tasks(struct mem_cgroup *memcg,
1331 struct mem_cgroup *iter;
1357 * This function relies on page->mem_cgroup being stable - see the
1363 struct mem_cgroup *memcg;
1378 memcg = page->mem_cgroup;
1390 * Since a node can be onlined after the mem_cgroup was created,
1401 * @lruvec: mem_cgroup per zone lru vector
1449 static unsigned long mem_cgroup_margin(struct mem_cgroup *memcg)
1479 static bool mem_cgroup_under_move(struct mem_cgroup *memcg)
1481 struct mem_cgroup *from;
1482 struct mem_cgroup *to;
1501 static bool mem_cgroup_wait_acct_move(struct mem_cgroup *memcg)
1581 static char *memory_stat_format(struct mem_cgroup *memcg)
1658 * NOTE: @memcg and @p's mem_cgroup can be different when hierarchy is
1661 void mem_cgroup_print_oom_context(struct mem_cgroup *memcg, struct task_struct *p)
1682 void mem_cgroup_print_oom_meminfo(struct mem_cgroup *memcg)
1715 unsigned long mem_cgroup_get_max(struct mem_cgroup *memcg)
1734 unsigned long mem_cgroup_size(struct mem_cgroup *memcg)
1739 static bool mem_cgroup_out_of_memory(struct mem_cgroup *memcg, gfp_t gfp_mask,
1768 static int mem_cgroup_soft_reclaim(struct mem_cgroup *root_memcg,
1773 struct mem_cgroup *victim = NULL;
1830 static bool mem_cgroup_oom_trylock(struct mem_cgroup *memcg)
1832 struct mem_cgroup *iter, *failed = NULL;
1869 static void mem_cgroup_oom_unlock(struct mem_cgroup *memcg)
1871 struct mem_cgroup *iter;
1880 static void mem_cgroup_mark_under_oom(struct mem_cgroup *memcg)
1882 struct mem_cgroup *iter;
1890 static void mem_cgroup_unmark_under_oom(struct mem_cgroup *memcg)
1892 struct mem_cgroup *iter;
1908 struct mem_cgroup *memcg;
1915 struct mem_cgroup *wake_memcg = (struct mem_cgroup *)arg;
1916 struct mem_cgroup *oom_wait_memcg;
1928 static void memcg_oom_recover(struct mem_cgroup *memcg)
1949 static enum oom_status mem_cgroup_oom(struct mem_cgroup *memcg, gfp_t mask, int order)
2026 struct mem_cgroup *memcg = current->memcg_in_oom;
2087 struct mem_cgroup *mem_cgroup_get_oom_group(struct task_struct *victim,
2088 struct mem_cgroup *oom_domain)
2090 struct mem_cgroup *oom_group = NULL;
2091 struct mem_cgroup *memcg;
2134 void mem_cgroup_print_oom_group(struct mem_cgroup *memcg)
2142 * lock_page_memcg - lock a page->mem_cgroup binding
2152 struct mem_cgroup *lock_page_memcg(struct page *page)
2155 struct mem_cgroup *memcg;
2174 memcg = head->mem_cgroup;
2182 if (memcg != head->mem_cgroup) {
2205 void __unlock_page_memcg(struct mem_cgroup *memcg)
2220 * unlock_page_memcg - unlock a page->mem_cgroup binding
2227 __unlock_page_memcg(head->mem_cgroup);
2232 struct mem_cgroup *cached; /* this never be root cgroup */
2250 struct mem_cgroup *root_memcg);
2257 struct mem_cgroup *root_memcg)
2274 static bool consume_stock(struct mem_cgroup *memcg, unsigned int nr_pages)
2301 struct mem_cgroup *old = stock->cached;
2340 static void refill_stock(struct mem_cgroup *memcg, unsigned int nr_pages)
2365 static void drain_all_stock(struct mem_cgroup *root_memcg)
2381 struct mem_cgroup *memcg;
2408 struct mem_cgroup *memcg, *mi;
2453 static unsigned long reclaim_high(struct mem_cgroup *memcg,
2480 struct mem_cgroup *memcg;
2482 memcg = container_of(work, struct mem_cgroup, high_work);
2557 static u64 mem_find_max_overage(struct mem_cgroup *memcg)
2571 static u64 swap_find_max_overage(struct mem_cgroup *memcg)
2591 static unsigned long calculate_high_delay(struct mem_cgroup *memcg,
2634 struct mem_cgroup *memcg;
2706 static int try_charge(struct mem_cgroup *memcg, gfp_t gfp_mask,
2711 struct mem_cgroup *mem_over_limit;
2893 static void cancel_charge(struct mem_cgroup *memcg, unsigned int nr_pages)
2904 static void commit_charge(struct page *page, struct mem_cgroup *memcg)
2906 VM_BUG_ON_PAGE(page->mem_cgroup, page);
2908 * Any of the following ensures page->mem_cgroup stability:
2915 page->mem_cgroup = memcg;
2954 struct mem_cgroup *mem_cgroup_from_obj(void *p)
2964 * If page->mem_cgroup is set, it's either a simple mem_cgroup pointer
2967 * The page->mem_cgroup pointer can be asynchronously changed
2971 if (!page->mem_cgroup)
2991 /* All other pages use page->mem_cgroup */
2992 return page->mem_cgroup;
2998 struct mem_cgroup *memcg;
3071 int __memcg_kmem_charge(struct mem_cgroup *memcg, gfp_t gfp,
3104 void __memcg_kmem_uncharge(struct mem_cgroup *memcg, unsigned int nr_pages)
3122 struct mem_cgroup *memcg;
3129 page->mem_cgroup = memcg;
3145 struct mem_cgroup *memcg = page->mem_cgroup;
3153 page->mem_cgroup = NULL;
3192 struct mem_cgroup *memcg;
3224 struct mem_cgroup *root_memcg)
3226 struct mem_cgroup *memcg;
3261 struct mem_cgroup *memcg;
3307 * Because head->mem_cgroup is not set on tails, set it now.
3311 struct mem_cgroup *memcg = head->mem_cgroup;
3319 head[i].mem_cgroup = memcg;
3330 * @from: mem_cgroup which the entry is moved from
3331 * @to: mem_cgroup which the entry is moved to
3334 * as the mem_cgroup's id of @from.
3342 struct mem_cgroup *from, struct mem_cgroup *to)
3358 struct mem_cgroup *from, struct mem_cgroup *to)
3366 static int mem_cgroup_resize_max(struct mem_cgroup *memcg,
3446 * This loop can run a while, specially if mem_cgroup's continuously
3509 static inline bool memcg_has_children(struct mem_cgroup *memcg)
3524 static int mem_cgroup_force_empty(struct mem_cgroup *memcg)
3557 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
3574 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
3575 struct mem_cgroup *parent_memcg = mem_cgroup_from_css(memcg->css.parent);
3600 static unsigned long mem_cgroup_usage(struct mem_cgroup *memcg, bool swap)
3629 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
3669 static void memcg_flush_percpu_vmstats(struct mem_cgroup *memcg)
3672 struct mem_cgroup *mi;
3701 static void memcg_flush_percpu_vmevents(struct mem_cgroup *memcg)
3704 struct mem_cgroup *mi;
3721 static int memcg_online_kmem(struct mem_cgroup *memcg)
3758 static void memcg_offline_kmem(struct mem_cgroup *memcg)
3761 struct mem_cgroup *parent, *child;
3801 static void memcg_free_kmem(struct mem_cgroup *memcg)
3808 static int memcg_online_kmem(struct mem_cgroup *memcg)
3812 static void memcg_offline_kmem(struct mem_cgroup *memcg)
3815 static void memcg_free_kmem(struct mem_cgroup *memcg)
3820 static int memcg_update_kmem_max(struct mem_cgroup *memcg,
3831 static int memcg_update_tcp_max(struct mem_cgroup *memcg, unsigned long max)
3873 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
3917 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
3961 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
3993 static unsigned long mem_cgroup_node_nr_lru_pages(struct mem_cgroup *memcg,
4013 static unsigned long mem_cgroup_nr_lru_pages(struct mem_cgroup *memcg,
4046 struct mem_cgroup *memcg = mem_cgroup_from_seq(m);
4111 struct mem_cgroup *memcg = mem_cgroup_from_seq(m);
4113 struct mem_cgroup *mi;
4213 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
4221 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
4234 static void __mem_cgroup_threshold(struct mem_cgroup *memcg, bool swap)
4285 static void mem_cgroup_threshold(struct mem_cgroup *memcg)
4310 static int mem_cgroup_oom_notify_cb(struct mem_cgroup *memcg)
4323 static void mem_cgroup_oom_notify(struct mem_cgroup *memcg)
4325 struct mem_cgroup *iter;
4331 static int __mem_cgroup_usage_register_event(struct mem_cgroup *memcg,
4411 static int mem_cgroup_usage_register_event(struct mem_cgroup *memcg,
4417 static int memsw_cgroup_usage_register_event(struct mem_cgroup *memcg,
4423 static void __mem_cgroup_usage_unregister_event(struct mem_cgroup *memcg,
4508 static void mem_cgroup_usage_unregister_event(struct mem_cgroup *memcg,
4514 static void memsw_cgroup_usage_unregister_event(struct mem_cgroup *memcg,
4520 static int mem_cgroup_oom_register_event(struct mem_cgroup *memcg,
4542 static void mem_cgroup_oom_unregister_event(struct mem_cgroup *memcg,
4561 struct mem_cgroup *memcg = mem_cgroup_from_seq(sf);
4573 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
4590 static int memcg_wb_domain_init(struct mem_cgroup *memcg, gfp_t gfp)
4595 static void memcg_wb_domain_exit(struct mem_cgroup *memcg)
4600 static void memcg_wb_domain_size_changed(struct mem_cgroup *memcg)
4607 struct mem_cgroup *memcg = mem_cgroup_from_css(wb->memcg_css);
4619 static unsigned long memcg_exact_page_state(struct mem_cgroup *memcg, int idx)
4653 struct mem_cgroup *memcg = mem_cgroup_from_css(wb->memcg_css);
4654 struct mem_cgroup *parent;
4720 struct mem_cgroup *memcg = page->mem_cgroup;
4772 struct mem_cgroup *memcg = mem_cgroup_from_css(wb->memcg_css);
4799 static int memcg_wb_domain_init(struct mem_cgroup *memcg, gfp_t gfp)
4804 static void memcg_wb_domain_exit(struct mem_cgroup *memcg)
4808 static void memcg_wb_domain_size_changed(struct mem_cgroup *memcg)
4836 struct mem_cgroup *memcg = event->memcg;
4860 struct mem_cgroup *memcg = event->memcg;
4910 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
5220 static void mem_cgroup_id_remove(struct mem_cgroup *memcg)
5231 static void __maybe_unused mem_cgroup_id_get_many(struct mem_cgroup *memcg,
5237 static void mem_cgroup_id_put_many(struct mem_cgroup *memcg, unsigned int n)
5247 static inline void mem_cgroup_id_put(struct mem_cgroup *memcg)
5258 struct mem_cgroup *mem_cgroup_from_id(unsigned short id)
5268 static int alloc_mem_cgroup_per_node_info(struct mem_cgroup *memcg, int node)
5311 static void free_mem_cgroup_per_node_info(struct mem_cgroup *memcg, int node)
5323 static void __mem_cgroup_free(struct mem_cgroup *memcg)
5334 static void mem_cgroup_free(struct mem_cgroup *memcg)
5346 static struct mem_cgroup *mem_cgroup_alloc(void)
5348 struct mem_cgroup *memcg;
5354 size = sizeof(struct mem_cgroup);
5429 struct mem_cgroup *parent = mem_cgroup_from_css(parent_css);
5430 struct mem_cgroup *memcg, *old_memcg;
5501 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
5526 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
5563 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
5570 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
5592 * mem_cgroup_css_reset - reset the states of a mem_cgroup
5595 * Reset the states of the mem_cgroup associated with @css. This is
5606 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
5740 * @from: mem_cgroup which the page is moved from.
5741 * @to: mem_cgroup which the page is moved to. @from != @to.
5750 struct mem_cgroup *from,
5751 struct mem_cgroup *to)
5764 * page->mem_cgroup of its source page while we change it.
5771 if (page->mem_cgroup != from)
5824 * It is safe to change page->mem_cgroup here because the page
5827 * that would rely on a stable page->mem_cgroup.
5830 * to save space. As soon as we switch page->mem_cgroup to a
5839 page->mem_cgroup = to;
5905 if (page->mem_cgroup == mc.from) {
5949 if (page->mem_cgroup == mc.from) {
6029 struct mem_cgroup *from = mc.from;
6030 struct mem_cgroup *to = mc.to;
6089 struct mem_cgroup *memcg = NULL; /* unneeded init to make gcc happy */
6090 struct mem_cgroup *from;
6364 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
6378 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
6401 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
6424 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
6473 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
6529 struct mem_cgroup *memcg = mem_cgroup_from_seq(m);
6537 struct mem_cgroup *memcg = mem_cgroup_from_seq(m);
6545 struct mem_cgroup *memcg = mem_cgroup_from_seq(m);
6560 struct mem_cgroup *memcg = mem_cgroup_from_seq(m);
6587 struct mem_cgroup *memcg = mem_cgroup_from_seq(m);
6597 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
6649 .file_offset = offsetof(struct mem_cgroup, events_file),
6655 .file_offset = offsetof(struct mem_cgroup, events_local_file),
6817 void mem_cgroup_calculate_protection(struct mem_cgroup *root,
6818 struct mem_cgroup *memcg)
6821 struct mem_cgroup *parent;
6881 struct mem_cgroup *memcg = NULL;
6894 * already charged pages, too. page->mem_cgroup is protected
6899 if (compound_head(page)->mem_cgroup)
6954 struct mem_cgroup *memcg;
6995 if (!page->mem_cgroup)
7000 * page->mem_cgroup at this point, we have fully
7004 if (ug->memcg != page->mem_cgroup) {
7009 ug->memcg = page->mem_cgroup;
7026 page->mem_cgroup = NULL;
7069 if (!page->mem_cgroup)
7105 struct mem_cgroup *memcg;
7119 if (newpage->mem_cgroup)
7123 memcg = oldpage->mem_cgroup;
7148 struct mem_cgroup *memcg;
7183 bool mem_cgroup_charge_skmem(struct mem_cgroup *memcg, unsigned int nr_pages)
7217 void mem_cgroup_uncharge_skmem(struct mem_cgroup *memcg, unsigned int nr_pages)
7252 * basically everything that doesn't depend on a specific mem_cgroup structure
7283 static struct mem_cgroup *mem_cgroup_id_get_online(struct mem_cgroup *memcg)
7310 struct mem_cgroup *memcg, *swap_memcg;
7320 memcg = page->mem_cgroup;
7341 page->mem_cgroup = NULL;
7378 struct mem_cgroup *memcg;
7384 memcg = page->mem_cgroup;
7422 struct mem_cgroup *memcg;
7441 long mem_cgroup_get_nr_swap_pages(struct mem_cgroup *memcg)
7456 struct mem_cgroup *memcg;
7465 memcg = page->mem_cgroup;
7493 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
7507 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
7530 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
7546 struct mem_cgroup *memcg = mem_cgroup_from_seq(m);
7579 .file_offset = offsetof(struct mem_cgroup, swap_events_file),