Lines Matching defs:folio
55 * An mlocked folio [folio_test_mlocked(folio)] is unevictable. As such, it
61 static struct lruvec *__mlock_folio(struct folio *folio, struct lruvec *lruvec)
64 if (!folio_test_clear_lru(folio))
67 lruvec = folio_lruvec_relock_irq(folio, lruvec);
69 if (unlikely(folio_evictable(folio))) {
73 * folio be unevictable? I'm not sure, but move it now if so.
75 if (folio_test_unevictable(folio)) {
76 lruvec_del_folio(lruvec, folio);
77 folio_clear_unevictable(folio);
78 lruvec_add_folio(lruvec, folio);
81 folio_nr_pages(folio));
86 if (folio_test_unevictable(folio)) {
87 if (folio_test_mlocked(folio))
88 folio->mlock_count++;
92 lruvec_del_folio(lruvec, folio);
93 folio_clear_active(folio);
94 folio_set_unevictable(folio);
95 folio->mlock_count = !!folio_test_mlocked(folio);
96 lruvec_add_folio(lruvec, folio);
97 __count_vm_events(UNEVICTABLE_PGCULLED, folio_nr_pages(folio));
99 folio_set_lru(folio);
103 static struct lruvec *__mlock_new_folio(struct folio *folio, struct lruvec *lruvec)
105 VM_BUG_ON_FOLIO(folio_test_lru(folio), folio);
107 lruvec = folio_lruvec_relock_irq(folio, lruvec);
110 if (unlikely(folio_evictable(folio)))
113 folio_set_unevictable(folio);
114 folio->mlock_count = !!folio_test_mlocked(folio);
115 __count_vm_events(UNEVICTABLE_PGCULLED, folio_nr_pages(folio));
117 lruvec_add_folio(lruvec, folio);
118 folio_set_lru(folio);
122 static struct lruvec *__munlock_folio(struct folio *folio, struct lruvec *lruvec)
124 int nr_pages = folio_nr_pages(folio);
127 if (!folio_test_clear_lru(folio))
131 lruvec = folio_lruvec_relock_irq(folio, lruvec);
133 if (folio_test_unevictable(folio)) {
135 if (folio->mlock_count)
136 folio->mlock_count--;
137 if (folio->mlock_count)
143 if (folio_test_clear_mlocked(folio)) {
144 __zone_stat_mod_folio(folio, NR_MLOCK, -nr_pages);
145 if (isolated || !folio_test_unevictable(folio))
152 if (isolated && folio_test_unevictable(folio) && folio_evictable(folio)) {
153 lruvec_del_folio(lruvec, folio);
154 folio_clear_unevictable(folio);
155 lruvec_add_folio(lruvec, folio);
160 folio_set_lru(folio);
165 * Flags held in the low bits of a struct folio pointer on the mlock_fbatch.
169 static inline struct folio *mlock_lru(struct folio *folio)
171 return (struct folio *)((unsigned long)folio + LRU_FOLIO);
174 static inline struct folio *mlock_new(struct folio *folio)
176 return (struct folio *)((unsigned long)folio + NEW_FOLIO);
181 * make use of such folio pointer flags in future, but for now just keep it for
182 * mlock. We could use three separate folio batches instead, but one feels
183 * better (munlocking a full folio batch does not need to drain mlocking folio
190 struct folio *folio;
194 folio = fbatch->folios[i];
195 mlock = (unsigned long)folio & (LRU_FOLIO | NEW_FOLIO);
196 folio = (struct folio *)((unsigned long)folio - mlock);
197 fbatch->folios[i] = folio;
200 lruvec = __mlock_folio(folio, lruvec);
202 lruvec = __mlock_new_folio(folio, lruvec);
204 lruvec = __munlock_folio(folio, lruvec);
240 * mlock_folio - mlock a folio already on (or temporarily off) LRU
241 * @folio: folio to be mlocked.
243 void mlock_folio(struct folio *folio)
250 if (!folio_test_set_mlocked(folio)) {
251 int nr_pages = folio_nr_pages(folio);
253 zone_stat_mod_folio(folio, NR_MLOCK, nr_pages);
257 folio_get(folio);
258 if (!folio_batch_add(fbatch, mlock_lru(folio)) ||
259 folio_test_large(folio) || lru_cache_disabled())
265 * mlock_new_folio - mlock a newly allocated folio not yet on LRU
266 * @folio: folio to be mlocked, either normal or a THP head.
268 void mlock_new_folio(struct folio *folio)
271 int nr_pages = folio_nr_pages(folio);
275 folio_set_mlocked(folio);
277 zone_stat_mod_folio(folio, NR_MLOCK, nr_pages);
280 folio_get(folio);
281 if (!folio_batch_add(fbatch, mlock_new(folio)) ||
282 folio_test_large(folio) || lru_cache_disabled())
288 * munlock_folio - munlock a folio
289 * @folio: folio to be munlocked, either normal or a THP head.
291 void munlock_folio(struct folio *folio)
298 * folio_test_clear_mlocked(folio) must be left to __munlock_folio(),
299 * which will check whether the folio is multiply mlocked.
301 folio_get(folio);
302 if (!folio_batch_add(fbatch, folio) ||
303 folio_test_large(folio) || lru_cache_disabled())
316 struct folio *folio;
324 folio = page_folio(pmd_page(*pmd));
326 mlock_folio(folio);
328 munlock_folio(folio);
341 folio = vm_normal_folio(vma, addr, ptent);
342 if (!folio || folio_is_zone_device(folio))
344 if (folio_test_large(folio))
347 mlock_folio(folio);
349 munlock_folio(folio);