Lines Matching defs:objcg
261 static void obj_cgroup_uncharge_pages(struct obj_cgroup *objcg,
266 struct obj_cgroup *objcg = container_of(ref, struct obj_cgroup, refcnt);
273 * objcg->nr_charged_bytes can't have an arbitrary byte value.
277 * 1) CPU0: objcg == stock->cached_objcg
282 * objcg->nr_charged_bytes = PAGE_SIZE - 92
286 * 92 bytes are added to objcg->nr_charged_bytes
291 nr_bytes = atomic_read(&objcg->nr_charged_bytes);
296 obj_cgroup_uncharge_pages(objcg, nr_pages);
299 list_del(&objcg->list);
303 kfree_rcu(objcg, rcu);
308 struct obj_cgroup *objcg;
311 objcg = kzalloc(sizeof(struct obj_cgroup), GFP_KERNEL);
312 if (!objcg)
315 ret = percpu_ref_init(&objcg->refcnt, obj_cgroup_release, 0,
318 kfree(objcg);
321 INIT_LIST_HEAD(&objcg->list);
322 return objcg;
328 struct obj_cgroup *objcg, *iter;
330 objcg = rcu_replace_pointer(memcg->objcg, NULL, true);
334 /* 1) Ready to reparent active objcg. */
335 list_add(&objcg->list, &memcg->objcg_list);
336 /* 2) Reparent active objcg and already reparented objcgs to parent. */
344 percpu_ref_kill(&objcg->refcnt);
2893 * The allocated objcg pointers array is not accounted directly.
2904 static inline void mod_objcg_mlstate(struct obj_cgroup *objcg,
2912 memcg = obj_cgroup_memcg(objcg);
2943 * objcg vector should be reused.
3037 struct obj_cgroup *objcg = NULL;
3040 objcg = rcu_dereference(memcg->objcg);
3041 if (objcg && obj_cgroup_tryget(objcg))
3043 objcg = NULL;
3045 return objcg;
3050 struct obj_cgroup *objcg = NULL;
3061 objcg = __get_obj_cgroup_from_memcg(memcg);
3063 return objcg;
3068 struct obj_cgroup *objcg;
3074 objcg = __folio_objcg(folio);
3075 obj_cgroup_get(objcg);
3082 objcg = __get_obj_cgroup_from_memcg(memcg);
3084 objcg = NULL;
3087 return objcg;
3103 * obj_cgroup_uncharge_pages: uncharge a number of kernel pages from a objcg
3104 * @objcg: object cgroup to uncharge
3107 static void obj_cgroup_uncharge_pages(struct obj_cgroup *objcg,
3112 memcg = get_mem_cgroup_from_objcg(objcg);
3121 * obj_cgroup_charge_pages: charge a number of kernel pages to a objcg
3122 * @objcg: object cgroup to charge
3128 static int obj_cgroup_charge_pages(struct obj_cgroup *objcg, gfp_t gfp,
3134 memcg = get_mem_cgroup_from_objcg(objcg);
3157 struct obj_cgroup *objcg;
3160 objcg = get_obj_cgroup_from_current();
3161 if (objcg) {
3162 ret = obj_cgroup_charge_pages(objcg, gfp, 1 << order);
3164 page->memcg_data = (unsigned long)objcg |
3168 obj_cgroup_put(objcg);
3181 struct obj_cgroup *objcg;
3187 objcg = __folio_objcg(folio);
3188 obj_cgroup_uncharge_pages(objcg, nr_pages);
3190 obj_cgroup_put(objcg);
3193 void mod_objcg_state(struct obj_cgroup *objcg, struct pglist_data *pgdat,
3209 if (READ_ONCE(stock->cached_objcg) != objcg) {
3211 obj_cgroup_get(objcg);
3212 stock->nr_bytes = atomic_read(&objcg->nr_charged_bytes)
3213 ? atomic_xchg(&objcg->nr_charged_bytes, 0) : 0;
3214 WRITE_ONCE(stock->cached_objcg, objcg);
3221 mod_objcg_mlstate(objcg, oldpg, NR_SLAB_RECLAIMABLE_B,
3226 mod_objcg_mlstate(objcg, oldpg, NR_SLAB_UNRECLAIMABLE_B,
3252 mod_objcg_mlstate(objcg, pgdat, idx, nr);
3259 static bool consume_obj_stock(struct obj_cgroup *objcg, unsigned int nr_bytes)
3268 if (objcg == READ_ONCE(stock->cached_objcg) && stock->nr_bytes >= nr_bytes) {
3344 struct obj_cgroup *objcg = READ_ONCE(stock->cached_objcg);
3347 if (objcg) {
3348 memcg = obj_cgroup_memcg(objcg);
3356 static void refill_obj_stock(struct obj_cgroup *objcg, unsigned int nr_bytes,
3367 if (READ_ONCE(stock->cached_objcg) != objcg) { /* reset if necessary */
3369 obj_cgroup_get(objcg);
3370 WRITE_ONCE(stock->cached_objcg, objcg);
3371 stock->nr_bytes = atomic_read(&objcg->nr_charged_bytes)
3372 ? atomic_xchg(&objcg->nr_charged_bytes, 0) : 0;
3373 allow_uncharge = true; /* Allow uncharge when objcg changes */
3387 obj_cgroup_uncharge_pages(objcg, nr_pages);
3390 int obj_cgroup_charge(struct obj_cgroup *objcg, gfp_t gfp, size_t size)
3395 if (consume_obj_stock(objcg, size))
3399 * In theory, objcg->nr_charged_bytes can have enough
3401 * flushing objcg->nr_charged_bytes requires two atomic
3402 * operations, and objcg->nr_charged_bytes can't be big.
3403 * The shared objcg->nr_charged_bytes can also become a
3407 * objcg->nr_charged_bytes later on when objcg changes.
3427 ret = obj_cgroup_charge_pages(objcg, gfp, nr_pages);
3429 refill_obj_stock(objcg, PAGE_SIZE - nr_bytes, false);
3434 void obj_cgroup_uncharge(struct obj_cgroup *objcg, size_t size)
3436 refill_obj_stock(objcg, size, true);
3781 struct obj_cgroup *objcg;
3789 objcg = obj_cgroup_alloc();
3790 if (!objcg)
3793 objcg->memcg = memcg;
3794 rcu_assign_pointer(memcg->objcg, objcg);
7206 struct obj_cgroup *objcg;
7212 * folio memcg or objcg at this point, we have fully
7216 objcg = __folio_objcg(folio);
7221 memcg = get_mem_cgroup_from_objcg(objcg);
7248 obj_cgroup_put(objcg);
7841 * @objcg: the object cgroup
7851 bool obj_cgroup_may_zswap(struct obj_cgroup *objcg)
7859 original_memcg = get_mem_cgroup_from_objcg(objcg);
7885 * @objcg: the object cgroup
7891 void obj_cgroup_charge_zswap(struct obj_cgroup *objcg, size_t size)
7901 if (obj_cgroup_charge(objcg, GFP_KERNEL, size))
7905 memcg = obj_cgroup_memcg(objcg);
7913 * @objcg: the object cgroup
7918 void obj_cgroup_uncharge_zswap(struct obj_cgroup *objcg, size_t size)
7925 obj_cgroup_uncharge(objcg, size);
7928 memcg = obj_cgroup_memcg(objcg);