Lines Matching refs:acct
111 struct io_wq_acct acct[IO_WQ_ACCT_NR];
139 struct io_wq_acct *acct,
157 return &wq->acct[bound ? IO_WQ_ACCT_BOUND : IO_WQ_ACCT_UNBOUND];
189 struct io_wq_acct *acct = io_wq_get_acct(worker);
192 atomic_dec(&acct->nr_running);
194 acct->nr_workers--;
245 static inline bool __io_acct_run_queue(struct io_wq_acct *acct)
247 return !test_bit(IO_ACCT_STALLED_BIT, &acct->flags) &&
248 !wq_list_empty(&acct->work_list);
252 * If there's work to do, returns true with acct->lock acquired. If not,
255 static inline bool io_acct_run_queue(struct io_wq_acct *acct)
256 __acquires(&acct->lock)
258 raw_spin_lock(&acct->lock);
259 if (__io_acct_run_queue(acct))
262 raw_spin_unlock(&acct->lock);
271 struct io_wq_acct *acct)
285 if (io_wq_get_acct(worker) != acct) {
306 static bool io_wq_create_worker(struct io_wq *wq, struct io_wq_acct *acct)
312 if (unlikely(!acct->max_workers))
316 if (acct->nr_workers >= acct->max_workers) {
320 acct->nr_workers++;
322 atomic_inc(&acct->nr_running);
324 return create_io_worker(wq, acct->index);
329 struct io_wq_acct *acct = io_wq_get_acct(worker);
331 atomic_inc(&acct->nr_running);
339 struct io_wq_acct *acct;
344 acct = &wq->acct[worker->create_index];
347 if (acct->nr_workers < acct->max_workers) {
348 acct->nr_workers++;
355 atomic_dec(&acct->nr_running);
363 struct io_wq_acct *acct,
385 worker->create_index = acct->index;
403 atomic_dec(&acct->nr_running);
410 struct io_wq_acct *acct = io_wq_get_acct(worker);
416 if (!atomic_dec_and_test(&acct->nr_running))
418 if (!io_acct_run_queue(acct))
421 raw_spin_unlock(&acct->lock);
422 atomic_inc(&acct->nr_running);
424 io_queue_worker_create(worker, acct, create_worker_cb);
475 static struct io_wq_work *io_get_next_work(struct io_wq_acct *acct,
477 __must_hold(acct->lock)
484 wq_list_for_each(node, prev, &acct->work_list) {
491 wq_list_del(&acct->work_list, node, prev);
502 wq_list_cut(&acct->work_list, &tail->list, prev);
518 set_bit(IO_ACCT_STALLED_BIT, &acct->flags);
519 raw_spin_unlock(&acct->lock);
521 raw_spin_lock(&acct->lock);
523 clear_bit(IO_ACCT_STALLED_BIT, &acct->flags);
547 * Called with acct->lock held, drops it before returning
549 static void io_worker_handle_work(struct io_wq_acct *acct,
551 __releases(&acct->lock)
566 work = io_get_next_work(acct, worker);
567 raw_spin_unlock(&acct->lock);
613 clear_bit(IO_ACCT_STALLED_BIT, &acct->flags);
620 if (!__io_acct_run_queue(acct))
622 raw_spin_lock(&acct->lock);
629 struct io_wq_acct *acct = io_wq_get_acct(worker);
646 * the acct->lock held. If not, it will drop it.
648 while (io_acct_run_queue(acct))
649 io_worker_handle_work(acct, worker);
656 if (last_timeout && (exit_mask || acct->nr_workers > 1)) {
657 acct->nr_workers--;
682 if (test_bit(IO_WQ_BIT_EXIT, &wq->state) && io_acct_run_queue(acct))
683 io_worker_handle_work(acct, worker);
780 struct io_wq_acct *acct = io_wq_get_acct(worker);
782 atomic_dec(&acct->nr_running);
784 acct->nr_workers--;
785 if (!acct->nr_workers) {
792 while (io_acct_cancel_pending_work(wq, acct, &match))
810 struct io_wq_acct *acct = io_wq_get_acct(worker);
812 if (!io_queue_worker_create(worker, acct, create_worker_cont))
818 struct io_wq_acct *acct = &wq->acct[index];
827 atomic_dec(&acct->nr_running);
829 acct->nr_workers--;
900 struct io_wq_acct *acct = io_work_get_acct(wq, work);
906 wq_list_add_tail(&work->list, &acct->work_list);
916 wq_list_add_after(&work->list, &tail->list, &acct->work_list);
926 struct io_wq_acct *acct = io_work_get_acct(wq, work);
941 raw_spin_lock(&acct->lock);
943 clear_bit(IO_ACCT_STALLED_BIT, &acct->flags);
944 raw_spin_unlock(&acct->lock);
947 do_create = !io_wq_activate_free_worker(wq, acct);
951 !atomic_read(&acct->nr_running))) {
954 did_create = io_wq_create_worker(wq, acct);
959 if (acct->nr_workers) {
970 io_acct_cancel_pending_work(wq, acct, &match);
1020 struct io_wq_acct *acct = io_work_get_acct(wq, work);
1032 wq_list_del(&acct->work_list, &work->list, prev);
1036 struct io_wq_acct *acct,
1042 raw_spin_lock(&acct->lock);
1043 wq_list_for_each(node, prev, &acct->work_list) {
1048 raw_spin_unlock(&acct->lock);
1054 raw_spin_unlock(&acct->lock);
1065 struct io_wq_acct *acct = io_get_acct(wq, i == 0);
1067 if (io_acct_cancel_pending_work(wq, acct, match)) {
1132 struct io_wq_acct *acct = &wq->acct[i];
1134 if (test_and_clear_bit(IO_ACCT_STALLED_BIT, &acct->flags))
1135 io_wq_activate_free_worker(wq, acct);
1165 wq->acct[IO_WQ_ACCT_BOUND].max_workers = bounded;
1166 wq->acct[IO_WQ_ACCT_UNBOUND].max_workers =
1171 struct io_wq_acct *acct = &wq->acct[i];
1173 acct->index = i;
1174 atomic_set(&acct->nr_running, 0);
1175 INIT_WQ_LIST(&acct->work_list);
1176 raw_spin_lock_init(&acct->lock);
1338 struct io_wq_acct *acct;
1358 acct = &wq->acct[i];
1359 prev[i] = max_t(int, acct->max_workers, prev[i]);
1361 acct->max_workers = new_count[i];