Lines Matching defs:lrugen
3223 unsigned long max_seq = READ_ONCE((lruvec)->lrugen.max_seq)
3227 READ_ONCE((lruvec)->lrugen.min_seq[LRU_GEN_ANON]), \
3228 READ_ONCE((lruvec)->lrugen.min_seq[LRU_GEN_FILE]), \
3276 return lruvec->lrugen.max_seq - lruvec->lrugen.min_seq[type] + 1;
3666 struct lru_gen_folio *lrugen = &lruvec->lrugen;
3667 int hist = lru_hist_from_seq(lrugen->min_seq[type]);
3669 pos->refaulted = lrugen->avg_refaulted[type][tier] +
3670 atomic_long_read(&lrugen->refaulted[hist][type][tier]);
3671 pos->total = lrugen->avg_total[type][tier] +
3672 atomic_long_read(&lrugen->evicted[hist][type][tier]);
3674 pos->total += lrugen->protected[hist][type][tier - 1];
3681 struct lru_gen_folio *lrugen = &lruvec->lrugen;
3683 unsigned long seq = carryover ? lrugen->min_seq[type] : lrugen->max_seq + 1;
3696 sum = lrugen->avg_refaulted[type][tier] +
3697 atomic_long_read(&lrugen->refaulted[hist][type][tier]);
3698 WRITE_ONCE(lrugen->avg_refaulted[type][tier], sum / 2);
3700 sum = lrugen->avg_total[type][tier] +
3701 atomic_long_read(&lrugen->evicted[hist][type][tier]);
3703 sum += lrugen->protected[hist][type][tier - 1];
3704 WRITE_ONCE(lrugen->avg_total[type][tier], sum / 2);
3708 atomic_long_set(&lrugen->refaulted[hist][type][tier], 0);
3709 atomic_long_set(&lrugen->evicted[hist][type][tier], 0);
3711 WRITE_ONCE(lrugen->protected[hist][type][tier - 1], 0);
3758 struct lru_gen_folio *lrugen = &lruvec->lrugen;
3759 int new_gen, old_gen = lru_gen_from_seq(lrugen->min_seq[type]);
3803 struct lru_gen_folio *lrugen = &lruvec->lrugen;
3815 WRITE_ONCE(lrugen->nr_pages[gen][type][zone],
3816 lrugen->nr_pages[gen][type][zone] + delta);
4322 struct lru_gen_folio *lrugen = &lruvec->lrugen;
4323 int new_gen, old_gen = lru_gen_from_seq(lrugen->min_seq[type]);
4330 struct list_head *head = &lrugen->folios[old_gen][type][zone];
4341 list_move_tail(&folio->lru, &lrugen->folios[new_gen][type][zone]);
4349 WRITE_ONCE(lrugen->min_seq[type], lrugen->min_seq[type] + 1);
4358 struct lru_gen_folio *lrugen = &lruvec->lrugen;
4365 while (min_seq[type] + MIN_NR_GENS <= lrugen->max_seq) {
4369 if (!list_empty(&lrugen->folios[gen][type][zone]))
4382 min_seq[LRU_GEN_FILE] = max(min_seq[LRU_GEN_ANON], lrugen->min_seq[LRU_GEN_FILE]);
4386 if (min_seq[type] == lrugen->min_seq[type])
4390 WRITE_ONCE(lrugen->min_seq[type], min_seq[type]);
4401 struct lru_gen_folio *lrugen = &lruvec->lrugen;
4427 prev = lru_gen_from_seq(lrugen->max_seq - 1);
4428 next = lru_gen_from_seq(lrugen->max_seq + 1);
4433 long delta = lrugen->nr_pages[prev][type][zone] -
4434 lrugen->nr_pages[next][type][zone];
4447 WRITE_ONCE(lrugen->timestamps[next], jiffies);
4449 smp_store_release(&lrugen->max_seq, lrugen->max_seq + 1);
4460 struct lru_gen_folio *lrugen = &lruvec->lrugen;
4462 VM_WARN_ON_ONCE(max_seq > READ_ONCE(lrugen->max_seq));
4513 struct lru_gen_folio *lrugen = &lruvec->lrugen;
4525 total += max(READ_ONCE(lrugen->nr_pages[gen][type][zone]), 0L);
4543 birth = READ_ONCE(lruvec->lrugen.timestamps[gen]);
4727 return READ_ONCE(lruvec->lrugen.seg);
4740 VM_WARN_ON_ONCE(hlist_nulls_unhashed(&lruvec->lrugen.list));
4743 new = old = lruvec->lrugen.gen;
4757 WRITE_ONCE(lruvec->lrugen.seg, seg);
4758 WRITE_ONCE(lruvec->lrugen.gen, new);
4760 hlist_nulls_del_rcu(&lruvec->lrugen.list);
4763 hlist_nulls_add_head_rcu(&lruvec->lrugen.list, &pgdat->memcg_lru.fifo[new][bin]);
4765 hlist_nulls_add_tail_rcu(&lruvec->lrugen.list, &pgdat->memcg_lru.fifo[new][bin]);
4788 VM_WARN_ON_ONCE(!hlist_nulls_unhashed(&lruvec->lrugen.list));
4792 lruvec->lrugen.gen = gen;
4794 hlist_nulls_add_tail_rcu(&lruvec->lrugen.list, &pgdat->memcg_lru.fifo[gen][bin]);
4823 if (hlist_nulls_unhashed(&lruvec->lrugen.list))
4826 gen = lruvec->lrugen.gen;
4828 hlist_nulls_del_init_rcu(&lruvec->lrugen.list);
4870 struct lru_gen_folio *lrugen = &lruvec->lrugen;
4894 if (gen != lru_gen_from_seq(lrugen->min_seq[type])) {
4895 list_move(&folio->lru, &lrugen->folios[gen][type][zone]);
4901 int hist = lru_hist_from_seq(lrugen->min_seq[type]);
4904 list_move_tail(&folio->lru, &lrugen->folios[gen][type][zone]);
4906 WRITE_ONCE(lrugen->protected[hist][type][tier - 1],
4907 lrugen->protected[hist][type][tier - 1] + delta);
4914 list_move_tail(&folio->lru, &lrugen->folios[gen][type][zone]);
4922 list_move(&folio->lru, &lrugen->folios[gen][type][zone]);
4973 struct lru_gen_folio *lrugen = &lruvec->lrugen;
4981 gen = lru_gen_from_seq(lrugen->min_seq[type]);
4987 struct list_head *head = &lrugen->folios[gen][type][zone];
5228 struct lru_gen_folio *lrugen = &lruvec->lrugen;
5247 size += max(READ_ONCE(lrugen->nr_pages[gen][type][zone]), 0L);
5430 struct lru_gen_folio *lrugen;
5442 hlist_nulls_for_each_entry_rcu(lrugen, pos, &pgdat->memcg_lru.fifo[gen][bin], list) {
5451 if (gen != READ_ONCE(lrugen->gen))
5454 lruvec = container_of(lrugen, struct lruvec, lrugen);
5603 struct lru_gen_folio *lrugen = &lruvec->lrugen;
5605 if (lrugen->enabled) {
5616 if (!list_empty(&lrugen->folios[gen][type][zone]))
5661 struct list_head *head = &lruvec->lrugen.folios[gen][type][zone];
5715 lruvec->lrugen.enabled = enabled;
5878 struct lru_gen_folio *lrugen = &lruvec->lrugen;
5888 n[0] = READ_ONCE(lrugen->avg_refaulted[type][tier]);
5889 n[1] = READ_ONCE(lrugen->avg_total[type][tier]);
5892 n[0] = atomic_long_read(&lrugen->refaulted[hist][type][tier]);
5893 n[1] = atomic_long_read(&lrugen->evicted[hist][type][tier]);
5895 n[2] = READ_ONCE(lrugen->protected[hist][type][tier - 1]);
5928 struct lru_gen_folio *lrugen = &lruvec->lrugen;
5956 unsigned long birth = READ_ONCE(lruvec->lrugen.timestamps[gen]);
5965 size += max(READ_ONCE(lrugen->nr_pages[gen][type][zone]), 0L);
6182 struct lru_gen_folio *lrugen = &lruvec->lrugen;
6184 lrugen->max_seq = MIN_NR_GENS + 1;
6185 lrugen->enabled = lru_gen_enabled();
6188 lrugen->timestamps[i] = jiffies;
6191 INIT_LIST_HEAD(&lrugen->folios[gen][type][zone]);
6226 VM_WARN_ON_ONCE(memchr_inv(lruvec->lrugen.nr_pages, 0,
6227 sizeof(lruvec->lrugen.nr_pages)));
6229 lruvec->lrugen.list.next = LIST_POISON1;