Lines Matching defs:folio
373 static inline bool folio_memcg_kmem(struct folio *folio);
388 * __folio_memcg - Get the memory cgroup associated with a non-kmem folio
389 * @folio: Pointer to the folio.
391 * Returns a pointer to the memory cgroup associated with the folio,
392 * or NULL. This function assumes that the folio is known to have a
397 static inline struct mem_cgroup *__folio_memcg(struct folio *folio)
399 unsigned long memcg_data = folio->memcg_data;
401 VM_BUG_ON_FOLIO(folio_test_slab(folio), folio);
402 VM_BUG_ON_FOLIO(memcg_data & MEMCG_DATA_OBJCGS, folio);
403 VM_BUG_ON_FOLIO(memcg_data & MEMCG_DATA_KMEM, folio);
409 * __folio_objcg - get the object cgroup associated with a kmem folio.
410 * @folio: Pointer to the folio.
412 * Returns a pointer to the object cgroup associated with the folio,
413 * or NULL. This function assumes that the folio is known to have a
418 static inline struct obj_cgroup *__folio_objcg(struct folio *folio)
420 unsigned long memcg_data = folio->memcg_data;
422 VM_BUG_ON_FOLIO(folio_test_slab(folio), folio);
423 VM_BUG_ON_FOLIO(memcg_data & MEMCG_DATA_OBJCGS, folio);
424 VM_BUG_ON_FOLIO(!(memcg_data & MEMCG_DATA_KMEM), folio);
430 * folio_memcg - Get the memory cgroup associated with a folio.
431 * @folio: Pointer to the folio.
433 * Returns a pointer to the memory cgroup associated with the folio,
434 * or NULL. This function assumes that the folio is known to have a
438 * For a non-kmem folio any of the following ensures folio and memcg binding
441 * - the folio lock
447 * For a kmem folio a caller should hold an rcu read lock to protect memcg
448 * associated with a kmem folio from being released.
450 static inline struct mem_cgroup *folio_memcg(struct folio *folio)
452 if (folio_memcg_kmem(folio))
453 return obj_cgroup_memcg(__folio_objcg(folio));
454 return __folio_memcg(folio);
463 * folio_memcg_rcu - Locklessly get the memory cgroup associated with a folio.
464 * @folio: Pointer to the folio.
466 * This function assumes that the folio is known to have a
470 * Return: A pointer to the memory cgroup associated with the folio,
473 static inline struct mem_cgroup *folio_memcg_rcu(struct folio *folio)
475 unsigned long memcg_data = READ_ONCE(folio->memcg_data);
477 VM_BUG_ON_FOLIO(folio_test_slab(folio), folio);
491 * folio_memcg_check - Get the memory cgroup associated with a folio.
492 * @folio: Pointer to the folio.
494 * Returns a pointer to the memory cgroup associated with the folio,
495 * or NULL. This function unlike folio_memcg() can take any folio
496 * as an argument. It has to be used in cases when it's not known if a folio
500 * For a non-kmem folio any of the following ensures folio and memcg binding
503 * - the folio lock
509 * For a kmem folio a caller should hold an rcu read lock to protect memcg
510 * associated with a kmem folio from being released.
512 static inline struct mem_cgroup *folio_memcg_check(struct folio *folio)
515 * Because folio->memcg_data might be changed asynchronously
518 unsigned long memcg_data = READ_ONCE(folio->memcg_data);
537 return folio_memcg_check((struct folio *)page);
556 * folio_memcg_kmem - Check if the folio has the memcg_kmem flag set.
557 * @folio: Pointer to the folio.
559 * Checks if the folio has MemcgKmem flag set. The caller must ensure
560 * that the folio has an associated memory cgroup. It's not safe to call
563 static inline bool folio_memcg_kmem(struct folio *folio)
565 VM_BUG_ON_PGFLAGS(PageTail(&folio->page), &folio->page);
566 VM_BUG_ON_FOLIO(folio->memcg_data & MEMCG_DATA_OBJCGS, folio);
567 return folio->memcg_data & MEMCG_DATA_KMEM;
572 static inline bool folio_memcg_kmem(struct folio *folio)
679 int __mem_cgroup_charge(struct folio *folio, struct mm_struct *mm, gfp_t gfp);
682 * mem_cgroup_charge - Charge a newly allocated folio to a cgroup.
683 * @folio: Folio to charge.
687 * Try to charge @folio to the memcg that @mm belongs to, reclaiming
695 static inline int mem_cgroup_charge(struct folio *folio, struct mm_struct *mm,
700 return __mem_cgroup_charge(folio, mm, gfp);
703 int mem_cgroup_swapin_charge_folio(struct folio *folio, struct mm_struct *mm,
707 void __mem_cgroup_uncharge(struct folio *folio);
710 * mem_cgroup_uncharge - Uncharge a folio.
711 * @folio: Folio to uncharge.
713 * Uncharge a folio previously charged with mem_cgroup_charge().
715 static inline void mem_cgroup_uncharge(struct folio *folio)
719 __mem_cgroup_uncharge(folio);
730 void mem_cgroup_migrate(struct folio *old, struct folio *new);
775 * folio_lruvec - return lruvec for isolating/putting an LRU folio
776 * @folio: Pointer to the folio.
778 * This function relies on folio->mem_cgroup being stable.
780 static inline struct lruvec *folio_lruvec(struct folio *folio)
782 struct mem_cgroup *memcg = folio_memcg(folio);
784 VM_WARN_ON_ONCE_FOLIO(!memcg && !mem_cgroup_disabled(), folio);
785 return mem_cgroup_lruvec(memcg, folio_pgdat(folio));
792 struct lruvec *folio_lruvec_lock(struct folio *folio);
793 struct lruvec *folio_lruvec_lock_irq(struct folio *folio);
794 struct lruvec *folio_lruvec_lock_irqsave(struct folio *folio,
798 void lruvec_memcg_debug(struct lruvec *lruvec, struct folio *folio);
801 void lruvec_memcg_debug(struct lruvec *lruvec, struct folio *folio)
929 struct cgroup_subsys_state *mem_cgroup_css_from_folio(struct folio *folio);
985 void folio_memcg_lock(struct folio *folio);
986 void folio_memcg_unlock(struct folio *folio);
1141 static inline void count_memcg_folio_events(struct folio *folio,
1144 struct mem_cgroup *memcg = folio_memcg(folio);
1215 static inline struct mem_cgroup *folio_memcg(struct folio *folio)
1225 static inline struct mem_cgroup *folio_memcg_rcu(struct folio *folio)
1231 static inline struct mem_cgroup *folio_memcg_check(struct folio *folio)
1241 static inline bool folio_memcg_kmem(struct folio *folio)
1301 static inline int mem_cgroup_charge(struct folio *folio,
1307 static inline int mem_cgroup_swapin_charge_folio(struct folio *folio,
1317 static inline void mem_cgroup_uncharge(struct folio *folio)
1325 static inline void mem_cgroup_migrate(struct folio *old, struct folio *new)
1335 static inline struct lruvec *folio_lruvec(struct folio *folio)
1337 struct pglist_data *pgdat = folio_pgdat(folio);
1342 void lruvec_memcg_debug(struct lruvec *lruvec, struct folio *folio)
1381 static inline struct lruvec *folio_lruvec_lock(struct folio *folio)
1383 struct pglist_data *pgdat = folio_pgdat(folio);
1389 static inline struct lruvec *folio_lruvec_lock_irq(struct folio *folio)
1391 struct pglist_data *pgdat = folio_pgdat(folio);
1397 static inline struct lruvec *folio_lruvec_lock_irqsave(struct folio *folio,
1400 struct pglist_data *pgdat = folio_pgdat(folio);
1490 static inline void folio_memcg_lock(struct folio *folio)
1494 static inline void folio_memcg_unlock(struct folio *folio)
1622 static inline void count_memcg_folio_events(struct folio *folio,
1685 static inline bool folio_matches_lruvec(struct folio *folio,
1688 return lruvec_pgdat(lruvec) == folio_pgdat(folio) &&
1689 lruvec_memcg(lruvec) == folio_memcg(folio);
1693 static inline struct lruvec *folio_lruvec_relock_irq(struct folio *folio,
1697 if (folio_matches_lruvec(folio, locked_lruvec))
1703 return folio_lruvec_lock_irq(folio);
1707 static inline struct lruvec *folio_lruvec_relock_irqsave(struct folio *folio,
1711 if (folio_matches_lruvec(folio, locked_lruvec))
1717 return folio_lruvec_lock_irqsave(folio, flags);
1727 void mem_cgroup_track_foreign_dirty_slowpath(struct folio *folio,
1730 static inline void mem_cgroup_track_foreign_dirty(struct folio *folio,
1738 memcg = folio_memcg(folio);
1740 mem_cgroup_track_foreign_dirty_slowpath(folio, wb);
1760 static inline void mem_cgroup_track_foreign_dirty(struct folio *folio,
1816 struct obj_cgroup *get_obj_cgroup_from_folio(struct folio *folio);
1900 static inline struct obj_cgroup *get_obj_cgroup_from_folio(struct folio *folio)