Lines Matching refs:sbq
384 static unsigned int sbq_calc_wake_batch(struct sbitmap_queue *sbq,
404 * Each word can be limited to sbq->min_shallow_depth bits.
406 shallow_depth = min(1U << sbq->sb.shift, sbq->min_shallow_depth);
407 depth = ((depth >> sbq->sb.shift) * shallow_depth +
408 min(depth & ((1U << sbq->sb.shift) - 1), shallow_depth));
415 int sbitmap_queue_init_node(struct sbitmap_queue *sbq, unsigned int depth,
421 ret = sbitmap_init_node(&sbq->sb, depth, shift, flags, node,
426 sbq->min_shallow_depth = UINT_MAX;
427 sbq->wake_batch = sbq_calc_wake_batch(sbq, depth);
428 atomic_set(&sbq->wake_index, 0);
429 atomic_set(&sbq->ws_active, 0);
430 atomic_set(&sbq->completion_cnt, 0);
431 atomic_set(&sbq->wakeup_cnt, 0);
433 sbq->ws = kzalloc_node(SBQ_WAIT_QUEUES * sizeof(*sbq->ws), flags, node);
434 if (!sbq->ws) {
435 sbitmap_free(&sbq->sb);
440 init_waitqueue_head(&sbq->ws[i].wait);
446 static void sbitmap_queue_update_wake_batch(struct sbitmap_queue *sbq,
451 wake_batch = sbq_calc_wake_batch(sbq, depth);
452 if (sbq->wake_batch != wake_batch)
453 WRITE_ONCE(sbq->wake_batch, wake_batch);
456 void sbitmap_queue_recalculate_wake_batch(struct sbitmap_queue *sbq,
460 unsigned int depth = (sbq->sb.depth + users - 1) / users;
465 WRITE_ONCE(sbq->wake_batch, wake_batch);
469 void sbitmap_queue_resize(struct sbitmap_queue *sbq, unsigned int depth)
471 sbitmap_queue_update_wake_batch(sbq, depth);
472 sbitmap_resize(&sbq->sb, depth);
476 int __sbitmap_queue_get(struct sbitmap_queue *sbq)
478 return sbitmap_get(&sbq->sb);
482 unsigned long __sbitmap_queue_get_batch(struct sbitmap_queue *sbq, int nr_tags,
485 struct sbitmap *sb = &sbq->sb;
534 int sbitmap_queue_get_shallow(struct sbitmap_queue *sbq,
537 WARN_ON_ONCE(shallow_depth < sbq->min_shallow_depth);
539 return sbitmap_get_shallow(&sbq->sb, shallow_depth);
543 void sbitmap_queue_min_shallow_depth(struct sbitmap_queue *sbq,
546 sbq->min_shallow_depth = min_shallow_depth;
547 sbitmap_queue_update_wake_batch(sbq, sbq->sb.depth);
551 static void __sbitmap_queue_wake_up(struct sbitmap_queue *sbq, int nr)
555 if (!atomic_read(&sbq->ws_active))
558 wake_index = atomic_read(&sbq->wake_index);
560 struct sbq_wait_state *ws = &sbq->ws[wake_index];
578 if (wake_index != atomic_read(&sbq->wake_index))
579 atomic_set(&sbq->wake_index, wake_index);
582 void sbitmap_queue_wake_up(struct sbitmap_queue *sbq, int nr)
584 unsigned int wake_batch = READ_ONCE(sbq->wake_batch);
587 if (!atomic_read(&sbq->ws_active))
590 atomic_add(nr, &sbq->completion_cnt);
591 wakeups = atomic_read(&sbq->wakeup_cnt);
594 if (atomic_read(&sbq->completion_cnt) - wakeups < wake_batch)
596 } while (!atomic_try_cmpxchg(&sbq->wakeup_cnt,
599 __sbitmap_queue_wake_up(sbq, wake_batch);
609 void sbitmap_queue_clear_batch(struct sbitmap_queue *sbq, int offset,
612 struct sbitmap *sb = &sbq->sb;
638 sbitmap_queue_wake_up(sbq, nr_tags);
639 sbitmap_update_cpu_hint(&sbq->sb, raw_smp_processor_id(),
643 void sbitmap_queue_clear(struct sbitmap_queue *sbq, unsigned int nr,
657 sbitmap_deferred_clear_bit(&sbq->sb, nr);
666 sbitmap_queue_wake_up(sbq, 1);
667 sbitmap_update_cpu_hint(&sbq->sb, cpu, nr);
671 void sbitmap_queue_wake_all(struct sbitmap_queue *sbq)
680 wake_index = atomic_read(&sbq->wake_index);
682 struct sbq_wait_state *ws = &sbq->ws[wake_index];
692 void sbitmap_queue_show(struct sbitmap_queue *sbq, struct seq_file *m)
697 sbitmap_show(&sbq->sb, m);
705 seq_printf(m, "%u", *per_cpu_ptr(sbq->sb.alloc_hint, i));
709 seq_printf(m, "wake_batch=%u\n", sbq->wake_batch);
710 seq_printf(m, "wake_index=%d\n", atomic_read(&sbq->wake_index));
711 seq_printf(m, "ws_active=%d\n", atomic_read(&sbq->ws_active));
715 struct sbq_wait_state *ws = &sbq->ws[i];
721 seq_printf(m, "round_robin=%d\n", sbq->sb.round_robin);
722 seq_printf(m, "min_shallow_depth=%u\n", sbq->min_shallow_depth);
726 void sbitmap_add_wait_queue(struct sbitmap_queue *sbq,
730 if (!sbq_wait->sbq) {
731 sbq_wait->sbq = sbq;
732 atomic_inc(&sbq->ws_active);
741 if (sbq_wait->sbq) {
742 atomic_dec(&sbq_wait->sbq->ws_active);
743 sbq_wait->sbq = NULL;
748 void sbitmap_prepare_to_wait(struct sbitmap_queue *sbq,
752 if (!sbq_wait->sbq) {
753 atomic_inc(&sbq->ws_active);
754 sbq_wait->sbq = sbq;
760 void sbitmap_finish_wait(struct sbitmap_queue *sbq, struct sbq_wait_state *ws,
764 if (sbq_wait->sbq) {
765 atomic_dec(&sbq->ws_active);
766 sbq_wait->sbq = NULL;