Home
last modified time | relevance | path

Searched refs:rwb (Results 1 - 14 of 14) sorted by relevance

/kernel/linux/linux-5.10/block/
H A Dblk-wbt.c78 static inline bool rwb_enabled(struct rq_wb *rwb) in rwb_enabled() argument
80 return rwb && rwb->enable_state != WBT_STATE_OFF_DEFAULT && in rwb_enabled()
81 rwb->wb_normal != 0; in rwb_enabled()
84 static void wb_timestamp(struct rq_wb *rwb, unsigned long *var) in wb_timestamp() argument
86 if (rwb_enabled(rwb)) { in wb_timestamp()
98 static bool wb_recent_wait(struct rq_wb *rwb) in wb_recent_wait() argument
100 struct bdi_writeback *wb = &rwb->rqos.q->backing_dev_info->wb; in wb_recent_wait()
105 static inline struct rq_wait *get_rq_wait(struct rq_wb *rwb, in get_rq_wait() argument
109 return &rwb in get_rq_wait()
116 rwb_wake_all(struct rq_wb *rwb) rwb_wake_all() argument
128 wbt_rqw_done(struct rq_wb *rwb, struct rq_wait *rqw, enum wbt_flags wb_acct) wbt_rqw_done() argument
172 struct rq_wb *rwb = RQWB(rqos); __wbt_done() local
188 struct rq_wb *rwb = RQWB(rqos); wbt_done() local
217 rwb_sync_issue_lat(struct rq_wb *rwb) rwb_sync_issue_lat() argument
235 latency_exceeded(struct rq_wb *rwb, struct blk_rq_stat *stat) latency_exceeded() argument
288 rwb_trace_step(struct rq_wb *rwb, const char *msg) rwb_trace_step() argument
297 calc_wb_limits(struct rq_wb *rwb) calc_wb_limits() argument
310 scale_up(struct rq_wb *rwb) scale_up() argument
320 scale_down(struct rq_wb *rwb, bool hard_throttle) scale_down() argument
329 rwb_arm_timer(struct rq_wb *rwb) rwb_arm_timer() argument
355 struct rq_wb *rwb = cb->data; wb_timer_fn() local
409 wbt_update_limits(struct rq_wb *rwb) wbt_update_limits() argument
441 close_io(struct rq_wb *rwb) close_io() argument
451 get_limit(struct rq_wb *rwb, unsigned long rw) get_limit() argument
488 struct rq_wb *rwb; global() member
509 __wbt_wait(struct rq_wb *rwb, enum wbt_flags wb_acct, unsigned long rw) __wbt_wait() argument
522 wbt_should_throttle(struct rq_wb *rwb, struct bio *bio) wbt_should_throttle() argument
540 bio_to_wbt_flags(struct rq_wb *rwb, struct bio *bio) bio_to_wbt_flags() argument
561 struct rq_wb *rwb = RQWB(rqos); wbt_cleanup() local
574 struct rq_wb *rwb = RQWB(rqos); wbt_wait() local
592 struct rq_wb *rwb = RQWB(rqos); wbt_track() local
598 struct rq_wb *rwb = RQWB(rqos); wbt_issue() local
618 struct rq_wb *rwb = RQWB(rqos); wbt_requeue() local
690 struct rq_wb *rwb = RQWB(rqos); wbt_exit() local
704 struct rq_wb *rwb; wbt_disable_default() local
719 struct rq_wb *rwb = RQWB(rqos); wbt_curr_win_nsec_show() local
728 struct rq_wb *rwb = RQWB(rqos); wbt_enabled_show() local
745 struct rq_wb *rwb = RQWB(rqos); wbt_inflight_show() local
757 struct rq_wb *rwb = RQWB(rqos); wbt_min_lat_nsec_show() local
766 struct rq_wb *rwb = RQWB(rqos); wbt_unknown_cnt_show() local
775 struct rq_wb *rwb = RQWB(rqos); wbt_normal_show() local
784 struct rq_wb *rwb = RQWB(rqos); wbt_background_show() local
819 struct rq_wb *rwb; wbt_init() local
[all...]
H A Dblk-wbt.h78 static inline unsigned int wbt_inflight(struct rq_wb *rwb) in wbt_inflight() argument
83 ret += atomic_read(&rwb->rq_wait[i].inflight); in wbt_inflight()
/kernel/linux/linux-6.6/block/
H A Dblk-wbt.c146 static inline bool rwb_enabled(struct rq_wb *rwb) in rwb_enabled() argument
148 return rwb && rwb->enable_state != WBT_STATE_OFF_DEFAULT && in rwb_enabled()
149 rwb->enable_state != WBT_STATE_OFF_MANUAL; in rwb_enabled()
152 static void wb_timestamp(struct rq_wb *rwb, unsigned long *var) in wb_timestamp() argument
154 if (rwb_enabled(rwb)) { in wb_timestamp()
166 static bool wb_recent_wait(struct rq_wb *rwb) in wb_recent_wait() argument
168 struct backing_dev_info *bdi = rwb->rqos.disk->bdi; in wb_recent_wait()
173 static inline struct rq_wait *get_rq_wait(struct rq_wb *rwb, in get_rq_wait() argument
177 return &rwb in get_rq_wait()
184 rwb_wake_all(struct rq_wb *rwb) rwb_wake_all() argument
196 wbt_rqw_done(struct rq_wb *rwb, struct rq_wait *rqw, enum wbt_flags wb_acct) wbt_rqw_done() argument
231 struct rq_wb *rwb = RQWB(rqos); __wbt_done() local
247 struct rq_wb *rwb = RQWB(rqos); wbt_done() local
276 rwb_sync_issue_lat(struct rq_wb *rwb) rwb_sync_issue_lat() argument
287 wbt_inflight(struct rq_wb *rwb) wbt_inflight() argument
304 latency_exceeded(struct rq_wb *rwb, struct blk_rq_stat *stat) latency_exceeded() argument
357 rwb_trace_step(struct rq_wb *rwb, const char *msg) rwb_trace_step() argument
366 calc_wb_limits(struct rq_wb *rwb) calc_wb_limits() argument
379 scale_up(struct rq_wb *rwb) scale_up() argument
389 scale_down(struct rq_wb *rwb, bool hard_throttle) scale_down() argument
398 rwb_arm_timer(struct rq_wb *rwb) rwb_arm_timer() argument
424 struct rq_wb *rwb = cb->data; wb_timer_fn() local
480 wbt_update_limits(struct rq_wb *rwb) wbt_update_limits() argument
524 close_io(struct rq_wb *rwb) close_io() argument
534 get_limit(struct rq_wb *rwb, blk_opf_t opf) get_limit() argument
564 struct rq_wb *rwb; global() member
585 __wbt_wait(struct rq_wb *rwb, enum wbt_flags wb_acct, blk_opf_t opf) __wbt_wait() argument
616 bio_to_wbt_flags(struct rq_wb *rwb, struct bio *bio) bio_to_wbt_flags() argument
637 struct rq_wb *rwb = RQWB(rqos); wbt_cleanup() local
649 struct rq_wb *rwb = RQWB(rqos); wbt_wait() local
667 struct rq_wb *rwb = RQWB(rqos); wbt_track() local
673 struct rq_wb *rwb = RQWB(rqos); wbt_issue() local
693 struct rq_wb *rwb = RQWB(rqos); wbt_requeue() local
772 struct rq_wb *rwb = RQWB(rqos); wbt_exit() local
785 struct rq_wb *rwb; wbt_disable_default() local
800 struct rq_wb *rwb = RQWB(rqos); wbt_curr_win_nsec_show() local
809 struct rq_wb *rwb = RQWB(rqos); wbt_enabled_show() local
826 struct rq_wb *rwb = RQWB(rqos); wbt_inflight_show() local
838 struct rq_wb *rwb = RQWB(rqos); wbt_min_lat_nsec_show() local
847 struct rq_wb *rwb = RQWB(rqos); wbt_unknown_cnt_show() local
856 struct rq_wb *rwb = RQWB(rqos); wbt_normal_show() local
865 struct rq_wb *rwb = RQWB(rqos); wbt_background_show() local
901 struct rq_wb *rwb; wbt_init() local
[all...]
/kernel/linux/linux-6.6/kernel/locking/
H A Drwbase_rt.c53 static __always_inline int rwbase_read_trylock(struct rwbase_rt *rwb) in rwbase_read_trylock() argument
61 for (r = atomic_read(&rwb->readers); r < 0;) { in rwbase_read_trylock()
62 if (likely(atomic_try_cmpxchg_acquire(&rwb->readers, &r, r + 1))) in rwbase_read_trylock()
68 static int __sched __rwbase_read_lock(struct rwbase_rt *rwb, in __rwbase_read_lock() argument
71 struct rt_mutex_base *rtm = &rwb->rtmutex; in __rwbase_read_lock()
106 trace_contention_begin(rwb, LCB_F_RT | LCB_F_READ); in __rwbase_read_lock()
122 atomic_inc(&rwb->readers); in __rwbase_read_lock()
127 trace_contention_end(rwb, ret); in __rwbase_read_lock()
131 static __always_inline int rwbase_read_lock(struct rwbase_rt *rwb, in rwbase_read_lock() argument
134 if (rwbase_read_trylock(rwb)) in rwbase_read_lock()
140 __rwbase_read_unlock(struct rwbase_rt *rwb, unsigned int state) __rwbase_read_unlock() argument
164 rwbase_read_unlock(struct rwbase_rt *rwb, unsigned int state) rwbase_read_unlock() argument
177 __rwbase_write_unlock(struct rwbase_rt *rwb, int bias, unsigned long flags) __rwbase_write_unlock() argument
191 rwbase_write_unlock(struct rwbase_rt *rwb) rwbase_write_unlock() argument
200 rwbase_write_downgrade(struct rwbase_rt *rwb) rwbase_write_downgrade() argument
210 __rwbase_write_trylock(struct rwbase_rt *rwb) __rwbase_write_trylock() argument
227 rwbase_write_lock(struct rwbase_rt *rwb, unsigned int state) rwbase_write_lock() argument
272 rwbase_write_trylock(struct rwbase_rt *rwb) rwbase_write_trylock() argument
[all...]
/kernel/linux/linux-6.6/include/linux/
H A Drwbase_rt.h29 static __always_inline bool rw_base_is_locked(struct rwbase_rt *rwb) in rw_base_is_locked() argument
31 return atomic_read(&rwb->readers) != READER_BIAS; in rw_base_is_locked()
34 static __always_inline bool rw_base_is_contended(struct rwbase_rt *rwb) in rw_base_is_contended() argument
36 return atomic_read(&rwb->readers) > 0; in rw_base_is_contended()
/kernel/linux/linux-5.10/drivers/lightnvm/
H A Dpblk-cache.c38 ret = pblk_rb_may_write_user(&pblk->rwb, bio, nr_entries, &bpos); in pblk_write_to_cache()
64 pos = pblk_rb_wrap_pos(&pblk->rwb, bpos + i); in pblk_write_to_cache()
65 pblk_rb_write_entry_user(&pblk->rwb, data, w_ctx, pos); in pblk_write_to_cache()
103 if (!pblk_rb_may_write_gc(&pblk->rwb, gc_rq->secs_to_gc, &bpos)) { in pblk_write_gc_to_cache()
117 pos = pblk_rb_wrap_pos(&pblk->rwb, bpos + valid_entries); in pblk_write_gc_to_cache()
118 pblk_rb_write_entry_gc(&pblk->rwb, data, w_ctx, gc_rq->line, in pblk_write_gc_to_cache()
H A Dpblk-write.c26 struct pblk_rb *rwb = &pblk->rwb; in pblk_end_w_bio() local
35 w_ctx = pblk_rb_w_ctx(rwb, pos); in pblk_end_w_bio()
44 atomic_dec(&rwb->inflight_flush_point); in pblk_end_w_bio()
60 ret = pblk_rb_sync_advance(&pblk->rwb, c_ctx->nr_valid); in pblk_end_w_bio()
88 pos = pblk_rb_sync_init(&pblk->rwb, &flags); in pblk_complete_write()
104 pblk_rb_sync_end(&pblk->rwb, &flags); in pblk_complete_write()
150 struct pblk_rb *rb = &pblk->rwb; in pblk_prepare_resubmit()
593 secs_avail = pblk_rb_read_count(&pblk->rwb); in pblk_submit_write()
597 secs_to_flush = pblk_rb_flush_point_count(&pblk->rwb); in pblk_submit_write()
[all...]
H A Dpblk-rb.c75 struct pblk *pblk = container_of(rb, struct pblk, rwb); in pblk_rb_init()
239 struct pblk *pblk = container_of(rb, struct pblk, rwb); in __pblk_rb_update_l2p()
340 struct pblk *pblk = container_of(rb, struct pblk, rwb); in pblk_rb_write_entry_user()
364 struct pblk *pblk = container_of(rb, struct pblk, rwb); in pblk_rb_write_entry_gc()
454 struct pblk *pblk = container_of(rb, struct pblk, rwb); in pblk_rb_flush()
476 struct pblk *pblk = container_of(rb, struct pblk, rwb); in pblk_rb_may_write_flush()
479 if (pblk_rb_flush_point_set(&pblk->rwb, bio, mem)) in pblk_rb_may_write_flush()
497 struct pblk *pblk = container_of(rb, struct pblk, rwb); in pblk_rb_may_write_user()
524 struct pblk *pblk = container_of(rb, struct pblk, rwb); in pblk_rb_may_write_gc()
554 struct pblk *pblk = container_of(rb, struct pblk, rwb); in pblk_rb_read_to_bio()
[all...]
H A Dpblk-init.c176 if (pblk_rb_tear_down_check(&pblk->rwb)) in pblk_rwb_free()
179 pblk_rb_free(&pblk->rwb); in pblk_rwb_free()
198 return pblk_rb_init(&pblk->rwb, buffer_size, threshold, geo->csecs); in pblk_rwb_init()
1095 WARN(pblk_rb_read_count(&pblk->rwb), in pblk_writer_stop()
1098 WARN(pblk_rb_sync_count(&pblk->rwb), in pblk_writer_stop()
1122 pblk_rb_sync_l2p(&pblk->rwb); in pblk_tear_down()
1260 pblk->rwb.nr_entries); in pblk_init()
H A Dpblk-map.c77 w_ctx = pblk_rb_w_ctx(&pblk->rwb, sentry + i); in pblk_map_page_data()
H A Dpblk-core.c379 unsigned int secs_avail = pblk_rb_read_count(&pblk->rwb); in pblk_write_should_kick()
397 pblk_rb_flush(&pblk->rwb); in pblk_flush_writer()
399 if (!pblk_rb_sync_count(&pblk->rwb)) in pblk_flush_writer()
1973 BUG_ON(pblk_rb_pos_oob(&pblk->rwb, pblk_addr_to_cacheline(ppa))); in pblk_update_map_cache()
1988 BUG_ON(pblk_rb_pos_oob(&pblk->rwb, pblk_addr_to_cacheline(ppa_new))); in pblk_update_map_gc()
H A Dpblk-read.c37 return pblk_rb_copy_to_bio(&pblk->rwb, bio, lba, ppa); in pblk_read_from_cache()
H A Dpblk-sysfs.c110 return pblk_rb_sysfs(&pblk->rwb, page); in pblk_sysfs_write_buffer()
H A Dpblk.h608 struct pblk_rb rwb; member

Completed in 14 milliseconds