Lines Matching defs:rwb
78 static inline bool rwb_enabled(struct rq_wb *rwb)
80 return rwb && rwb->enable_state != WBT_STATE_OFF_DEFAULT &&
81 rwb->wb_normal != 0;
84 static void wb_timestamp(struct rq_wb *rwb, unsigned long *var)
86 if (rwb_enabled(rwb)) {
98 static bool wb_recent_wait(struct rq_wb *rwb)
100 struct bdi_writeback *wb = &rwb->rqos.q->backing_dev_info->wb;
105 static inline struct rq_wait *get_rq_wait(struct rq_wb *rwb,
109 return &rwb->rq_wait[WBT_RWQ_KSWAPD];
111 return &rwb->rq_wait[WBT_RWQ_DISCARD];
113 return &rwb->rq_wait[WBT_RWQ_BG];
116 static void rwb_wake_all(struct rq_wb *rwb)
121 struct rq_wait *rqw = &rwb->rq_wait[i];
128 static void wbt_rqw_done(struct rq_wb *rwb, struct rq_wait *rqw,
139 if (unlikely(!rwb_enabled(rwb))) {
140 rwb_wake_all(rwb);
150 limit = rwb->wb_background;
151 else if (rwb->wc && !wb_recent_wait(rwb))
154 limit = rwb->wb_normal;
165 if (!inflight || diff >= rwb->wb_background / 2)
172 struct rq_wb *rwb = RQWB(rqos);
178 rqw = get_rq_wait(rwb, wb_acct);
179 wbt_rqw_done(rwb, rqw, wb_acct);
188 struct rq_wb *rwb = RQWB(rqos);
191 if (rwb->sync_cookie == rq) {
192 rwb->sync_issue = 0;
193 rwb->sync_cookie = NULL;
197 wb_timestamp(rwb, &rwb->last_comp);
199 WARN_ON_ONCE(rq == rwb->sync_cookie);
217 static u64 rwb_sync_issue_lat(struct rq_wb *rwb)
219 u64 now, issue = READ_ONCE(rwb->sync_issue);
221 if (!issue || !rwb->sync_cookie)
235 static int latency_exceeded(struct rq_wb *rwb, struct blk_rq_stat *stat)
237 struct backing_dev_info *bdi = rwb->rqos.q->backing_dev_info;
238 struct rq_depth *rqd = &rwb->rq_depth;
250 thislat = rwb_sync_issue_lat(rwb);
251 if (thislat > rwb->cur_win_nsec ||
252 (thislat > rwb->min_lat_nsec && !stat[READ].nr_samples)) {
267 if (stat[WRITE].nr_samples || wb_recent_wait(rwb) ||
268 wbt_inflight(rwb))
276 if (stat[READ].min > rwb->min_lat_nsec) {
288 static void rwb_trace_step(struct rq_wb *rwb, const char *msg)
290 struct backing_dev_info *bdi = rwb->rqos.q->backing_dev_info;
291 struct rq_depth *rqd = &rwb->rq_depth;
293 trace_wbt_step(bdi, msg, rqd->scale_step, rwb->cur_win_nsec,
294 rwb->wb_background, rwb->wb_normal, rqd->max_depth);
297 static void calc_wb_limits(struct rq_wb *rwb)
299 if (rwb->min_lat_nsec == 0) {
300 rwb->wb_normal = rwb->wb_background = 0;
301 } else if (rwb->rq_depth.max_depth <= 2) {
302 rwb->wb_normal = rwb->rq_depth.max_depth;
303 rwb->wb_background = 1;
305 rwb->wb_normal = (rwb->rq_depth.max_depth + 1) / 2;
306 rwb->wb_background = (rwb->rq_depth.max_depth + 3) / 4;
310 static void scale_up(struct rq_wb *rwb)
312 if (!rq_depth_scale_up(&rwb->rq_depth))
314 calc_wb_limits(rwb);
315 rwb->unknown_cnt = 0;
316 rwb_wake_all(rwb);
317 rwb_trace_step(rwb, tracepoint_string("scale up"));
320 static void scale_down(struct rq_wb *rwb, bool hard_throttle)
322 if (!rq_depth_scale_down(&rwb->rq_depth, hard_throttle))
324 calc_wb_limits(rwb);
325 rwb->unknown_cnt = 0;
326 rwb_trace_step(rwb, tracepoint_string("scale down"));
329 static void rwb_arm_timer(struct rq_wb *rwb)
331 struct rq_depth *rqd = &rwb->rq_depth;
340 rwb->cur_win_nsec = div_u64(rwb->win_nsec << 4,
347 rwb->cur_win_nsec = rwb->win_nsec;
350 blk_stat_activate_nsecs(rwb->cb, rwb->cur_win_nsec);
355 struct rq_wb *rwb = cb->data;
356 struct rq_depth *rqd = &rwb->rq_depth;
357 unsigned int inflight = wbt_inflight(rwb);
360 status = latency_exceeded(rwb, cb->stat);
362 trace_wbt_timer(rwb->rqos.q->backing_dev_info, status, rqd->scale_step,
372 scale_down(rwb, true);
375 scale_up(rwb);
383 scale_up(rwb);
386 if (++rwb->unknown_cnt < RWB_UNKNOWN_BUMP)
394 scale_up(rwb);
396 scale_down(rwb, false);
406 rwb_arm_timer(rwb);
409 static void wbt_update_limits(struct rq_wb *rwb)
411 struct rq_depth *rqd = &rwb->rq_depth;
417 calc_wb_limits(rwb);
419 rwb_wake_all(rwb);
441 static bool close_io(struct rq_wb *rwb)
445 return time_before(now, rwb->last_issue + HZ / 10) ||
446 time_before(now, rwb->last_comp + HZ / 10);
451 static inline unsigned int get_limit(struct rq_wb *rwb, unsigned long rw)
459 if (!rwb_enabled(rwb))
463 return rwb->wb_background;
473 if ((rw & REQ_HIPRIO) || wb_recent_wait(rwb) || current_is_kswapd())
474 limit = rwb->rq_depth.max_depth;
475 else if ((rw & REQ_BACKGROUND) || close_io(rwb)) {
480 limit = rwb->wb_background;
482 limit = rwb->wb_normal;
488 struct rq_wb *rwb;
496 return rq_wait_inc_below(rqw, get_limit(data->rwb, data->rw));
502 wbt_rqw_done(data->rwb, rqw, data->wb_acct);
509 static void __wbt_wait(struct rq_wb *rwb, enum wbt_flags wb_acct,
512 struct rq_wait *rqw = get_rq_wait(rwb, wb_acct);
514 .rwb = rwb,
522 static inline bool wbt_should_throttle(struct rq_wb *rwb, struct bio *bio)
540 static enum wbt_flags bio_to_wbt_flags(struct rq_wb *rwb, struct bio *bio)
544 if (!rwb_enabled(rwb))
549 } else if (wbt_should_throttle(rwb, bio)) {
561 struct rq_wb *rwb = RQWB(rqos);
562 enum wbt_flags flags = bio_to_wbt_flags(rwb, bio);
574 struct rq_wb *rwb = RQWB(rqos);
577 flags = bio_to_wbt_flags(rwb, bio);
580 wb_timestamp(rwb, &rwb->last_issue);
584 __wbt_wait(rwb, flags, bio->bi_opf);
586 if (!blk_stat_is_active(rwb->cb))
587 rwb_arm_timer(rwb);
592 struct rq_wb *rwb = RQWB(rqos);
593 rq->wbt_flags |= bio_to_wbt_flags(rwb, bio);
598 struct rq_wb *rwb = RQWB(rqos);
600 if (!rwb_enabled(rwb))
610 if (wbt_is_read(rq) && !rwb->sync_issue) {
611 rwb->sync_cookie = rq;
612 rwb->sync_issue = rq->io_start_time_ns;
618 struct rq_wb *rwb = RQWB(rqos);
619 if (!rwb_enabled(rwb))
621 if (rq == rwb->sync_cookie) {
622 rwb->sync_issue = 0;
623 rwb->sync_cookie = NULL;
690 struct rq_wb *rwb = RQWB(rqos);
693 blk_stat_remove_callback(q, rwb->cb);
694 blk_stat_free_callback(rwb->cb);
695 kfree(rwb);
704 struct rq_wb *rwb;
707 rwb = RQWB(rqos);
708 if (rwb->enable_state == WBT_STATE_ON_DEFAULT) {
709 blk_stat_deactivate(rwb->cb);
710 rwb->enable_state = WBT_STATE_OFF_DEFAULT;
719 struct rq_wb *rwb = RQWB(rqos);
721 seq_printf(m, "%llu\n", rwb->cur_win_nsec);
728 struct rq_wb *rwb = RQWB(rqos);
730 seq_printf(m, "%d\n", rwb->enable_state);
745 struct rq_wb *rwb = RQWB(rqos);
750 atomic_read(&rwb->rq_wait[i].inflight));
757 struct rq_wb *rwb = RQWB(rqos);
759 seq_printf(m, "%lu\n", rwb->min_lat_nsec);
766 struct rq_wb *rwb = RQWB(rqos);
768 seq_printf(m, "%u\n", rwb->unknown_cnt);
775 struct rq_wb *rwb = RQWB(rqos);
777 seq_printf(m, "%u\n", rwb->wb_normal);
784 struct rq_wb *rwb = RQWB(rqos);
786 seq_printf(m, "%u\n", rwb->wb_background);
819 struct rq_wb *rwb;
822 rwb = kzalloc(sizeof(*rwb), GFP_KERNEL);
823 if (!rwb)
826 rwb->cb = blk_stat_alloc_callback(wb_timer_fn, wbt_data_dir, 2, rwb);
827 if (!rwb->cb) {
828 kfree(rwb);
833 rq_wait_init(&rwb->rq_wait[i]);
835 rwb->rqos.id = RQ_QOS_WBT;
836 rwb->rqos.ops = &wbt_rqos_ops;
837 rwb->rqos.q = q;
838 rwb->last_comp = rwb->last_issue = jiffies;
839 rwb->win_nsec = RWB_WINDOW_NSEC;
840 rwb->enable_state = WBT_STATE_ON_DEFAULT;
841 rwb->wc = test_bit(QUEUE_FLAG_WC, &q->queue_flags);
842 rwb->rq_depth.default_depth = RWB_DEF_DEPTH;
843 rwb->min_lat_nsec = wbt_default_latency_nsec(q);
845 wbt_queue_depth_changed(&rwb->rqos);
848 * Assign rwb and add the stats callback.
850 rq_qos_add(q, &rwb->rqos);
851 blk_stat_add_callback(q, rwb->cb);