Lines Matching defs:per_prio
87 struct dd_per_prio per_prio[DD_PRIO_COUNT];
117 deadline_rb_root(struct dd_per_prio *per_prio, struct request *rq)
119 return &per_prio->sort_list[rq_data_dir(rq)];
163 static inline struct request *deadline_from_pos(struct dd_per_prio *per_prio,
166 struct rb_node *node = per_prio->sort_list[data_dir].rb_node;
194 deadline_add_rq_rb(struct dd_per_prio *per_prio, struct request *rq)
196 struct rb_root *root = deadline_rb_root(per_prio, rq);
202 deadline_del_rq_rb(struct dd_per_prio *per_prio, struct request *rq)
204 elv_rb_del(deadline_rb_root(per_prio, rq), rq);
211 struct dd_per_prio *per_prio,
220 deadline_del_rq_rb(per_prio, rq);
233 struct dd_per_prio *per_prio = &dd->per_prio[prio];
239 elv_rb_del(deadline_rb_root(per_prio, req), req);
240 deadline_add_rq_rb(per_prio, req);
256 dd->per_prio[prio].stats.merged++;
273 deadline_remove_request(q, &dd->per_prio[prio], next);
280 deadline_move_request(struct deadline_data *dd, struct dd_per_prio *per_prio,
286 deadline_remove_request(rq->q, per_prio, rq);
292 const struct io_stats_per_prio *stats = &dd->per_prio[prio].stats;
303 static inline bool deadline_check_fifo(struct dd_per_prio *per_prio,
306 struct request *rq = rq_entry_fifo(per_prio->fifo_list[data_dir].next);
346 deadline_fifo_request(struct deadline_data *dd, struct dd_per_prio *per_prio,
352 if (list_empty(&per_prio->fifo_list[data_dir]))
355 rq = rq_entry_fifo(per_prio->fifo_list[data_dir].next);
367 list_for_each_entry_safe(rq, next, &per_prio->fifo_list[DD_WRITE],
370 rb_rq = deadline_from_pos(per_prio, data_dir, blk_rq_pos(rq));
390 deadline_next_request(struct deadline_data *dd, struct dd_per_prio *per_prio,
396 rq = deadline_from_pos(per_prio, data_dir,
397 per_prio->latest_pos[data_dir]);
444 struct dd_per_prio *per_prio,
454 if (!list_empty(&per_prio->dispatch)) {
455 rq = list_first_entry(&per_prio->dispatch, struct request,
467 rq = deadline_next_request(dd, per_prio, dd->last_dir);
479 if (!list_empty(&per_prio->fifo_list[DD_READ])) {
480 BUG_ON(RB_EMPTY_ROOT(&per_prio->sort_list[DD_READ]));
482 if (deadline_fifo_request(dd, per_prio, DD_WRITE) &&
495 if (!list_empty(&per_prio->fifo_list[DD_WRITE])) {
497 BUG_ON(RB_EMPTY_ROOT(&per_prio->sort_list[DD_WRITE]));
512 next_rq = deadline_next_request(dd, per_prio, data_dir);
513 if (deadline_check_fifo(per_prio, data_dir) || !next_rq) {
519 rq = deadline_fifo_request(dd, per_prio, data_dir);
546 deadline_move_request(dd, per_prio, rq);
550 dd->per_prio[prio].latest_pos[data_dir] = blk_rq_pos(rq);
551 dd->per_prio[prio].stats.dispatched++;
579 rq = __dd_dispatch_request(dd, &dd->per_prio[prio],
613 rq = __dd_dispatch_request(dd, &dd->per_prio[prio], now);
669 struct dd_per_prio *per_prio = &dd->per_prio[prio];
670 const struct io_stats_per_prio *stats = &per_prio->stats;
673 WARN_ON_ONCE(!list_empty(&per_prio->fifo_list[DD_READ]));
674 WARN_ON_ONCE(!list_empty(&per_prio->fifo_list[DD_WRITE]));
710 struct dd_per_prio *per_prio = &dd->per_prio[prio];
712 INIT_LIST_HEAD(&per_prio->dispatch);
713 INIT_LIST_HEAD(&per_prio->fifo_list[DD_READ]);
714 INIT_LIST_HEAD(&per_prio->fifo_list[DD_WRITE]);
715 per_prio->sort_list[DD_READ] = RB_ROOT;
716 per_prio->sort_list[DD_WRITE] = RB_ROOT;
749 struct dd_per_prio *per_prio = &dd->per_prio[prio];
756 __rq = elv_rb_find(&per_prio->sort_list[bio_data_dir(bio)], sector);
803 struct dd_per_prio *per_prio;
815 per_prio = &dd->per_prio[prio];
817 per_prio->stats.inserted++;
827 list_add(&rq->queuelist, &per_prio->dispatch);
832 deadline_add_rq_rb(per_prio, rq);
844 insert_before = &per_prio->fifo_list[data_dir];
897 if (!list_empty_careful(&dd->per_prio[p].fifo_list[DD_WRITE]))
925 struct dd_per_prio *per_prio = &dd->per_prio[prio];
935 atomic_inc(&per_prio->stats.completed);
949 static bool dd_has_work_for_prio(struct dd_per_prio *per_prio)
951 return !list_empty_careful(&per_prio->dispatch) ||
952 !list_empty_careful(&per_prio->fifo_list[DD_READ]) ||
953 !list_empty_careful(&per_prio->fifo_list[DD_WRITE]);
962 if (dd_has_work_for_prio(&dd->per_prio[prio]))
1042 struct dd_per_prio *per_prio = &dd->per_prio[prio]; \
1045 return seq_list_start(&per_prio->fifo_list[data_dir], *pos); \
1053 struct dd_per_prio *per_prio = &dd->per_prio[prio]; \
1055 return seq_list_next(v, &per_prio->fifo_list[data_dir], pos); \
1079 struct dd_per_prio *per_prio = &dd->per_prio[prio]; \
1082 rq = deadline_from_pos(per_prio, data_dir, \
1083 per_prio->latest_pos[data_dir]); \
1144 const struct io_stats_per_prio *stats = &dd->per_prio[prio].stats;
1176 struct dd_per_prio *per_prio = &dd->per_prio[prio]; \
1179 return seq_list_start(&per_prio->dispatch, *pos); \
1187 struct dd_per_prio *per_prio = &dd->per_prio[prio]; \
1189 return seq_list_next(v, &per_prio->dispatch, pos); \