| /kernel/linux/linux-5.10/lib/ |
| H A D | sbitmap.c | 324 static unsigned int sbq_calc_wake_batch(struct sbitmap_queue *sbq, in sbq_calc_wake_batch() argument
344 * Each word can be limited to sbq->min_shallow_depth bits. in sbq_calc_wake_batch()
346 shallow_depth = min(1U << sbq->sb.shift, sbq->min_shallow_depth); in sbq_calc_wake_batch()
347 depth = ((depth >> sbq->sb.shift) * shallow_depth + in sbq_calc_wake_batch()
348 min(depth & ((1U << sbq->sb.shift) - 1), shallow_depth)); in sbq_calc_wake_batch()
355 int sbitmap_queue_init_node(struct sbitmap_queue *sbq, unsigned int depth, in sbitmap_queue_init_node() argument
361 ret = sbitmap_init_node(&sbq->sb, depth, shift, flags, node); in sbitmap_queue_init_node()
365 sbq->alloc_hint = alloc_percpu_gfp(unsigned int, flags); in sbitmap_queue_init_node()
366 if (!sbq in sbitmap_queue_init_node()
398 sbitmap_queue_update_wake_batch(struct sbitmap_queue *sbq, unsigned int depth) sbitmap_queue_update_wake_batch() argument
417 sbitmap_queue_resize(struct sbitmap_queue *sbq, unsigned int depth) sbitmap_queue_resize() argument
424 __sbitmap_queue_get(struct sbitmap_queue *sbq) __sbitmap_queue_get() argument
452 __sbitmap_queue_get_shallow(struct sbitmap_queue *sbq, unsigned int shallow_depth) __sbitmap_queue_get_shallow() argument
483 sbitmap_queue_min_shallow_depth(struct sbitmap_queue *sbq, unsigned int min_shallow_depth) sbitmap_queue_min_shallow_depth() argument
491 sbq_wake_ptr(struct sbitmap_queue *sbq) sbq_wake_ptr() argument
514 __sbq_wake_up(struct sbitmap_queue *sbq) __sbq_wake_up() argument
555 sbitmap_queue_wake_up(struct sbitmap_queue *sbq) sbitmap_queue_wake_up() argument
562 sbitmap_queue_clear(struct sbitmap_queue *sbq, unsigned int nr, unsigned int cpu) sbitmap_queue_clear() argument
592 sbitmap_queue_wake_all(struct sbitmap_queue *sbq) sbitmap_queue_wake_all() argument
613 sbitmap_queue_show(struct sbitmap_queue *sbq, struct seq_file *m) sbitmap_queue_show() argument
649 sbitmap_add_wait_queue(struct sbitmap_queue *sbq, struct sbq_wait_state *ws, struct sbq_wait *sbq_wait) sbitmap_add_wait_queue() argument
671 sbitmap_prepare_to_wait(struct sbitmap_queue *sbq, struct sbq_wait_state *ws, struct sbq_wait *sbq_wait, int state) sbitmap_prepare_to_wait() argument
683 sbitmap_finish_wait(struct sbitmap_queue *sbq, struct sbq_wait_state *ws, struct sbq_wait *sbq_wait) sbitmap_finish_wait() argument
[all...] |
| /kernel/linux/linux-6.6/lib/ |
| H A D | sbitmap.c | 384 static unsigned int sbq_calc_wake_batch(struct sbitmap_queue *sbq, in sbq_calc_wake_batch() argument
404 * Each word can be limited to sbq->min_shallow_depth bits. in sbq_calc_wake_batch()
406 shallow_depth = min(1U << sbq->sb.shift, sbq->min_shallow_depth); in sbq_calc_wake_batch()
407 depth = ((depth >> sbq->sb.shift) * shallow_depth + in sbq_calc_wake_batch()
408 min(depth & ((1U << sbq->sb.shift) - 1), shallow_depth)); in sbq_calc_wake_batch()
415 int sbitmap_queue_init_node(struct sbitmap_queue *sbq, unsigned int depth, in sbitmap_queue_init_node() argument
421 ret = sbitmap_init_node(&sbq->sb, depth, shift, flags, node, in sbitmap_queue_init_node()
426 sbq->min_shallow_depth = UINT_MAX; in sbitmap_queue_init_node()
427 sbq in sbitmap_queue_init_node()
446 sbitmap_queue_update_wake_batch(struct sbitmap_queue *sbq, unsigned int depth) sbitmap_queue_update_wake_batch() argument
456 sbitmap_queue_recalculate_wake_batch(struct sbitmap_queue *sbq, unsigned int users) sbitmap_queue_recalculate_wake_batch() argument
469 sbitmap_queue_resize(struct sbitmap_queue *sbq, unsigned int depth) sbitmap_queue_resize() argument
476 __sbitmap_queue_get(struct sbitmap_queue *sbq) __sbitmap_queue_get() argument
482 __sbitmap_queue_get_batch(struct sbitmap_queue *sbq, int nr_tags, unsigned int *offset) __sbitmap_queue_get_batch() argument
534 sbitmap_queue_get_shallow(struct sbitmap_queue *sbq, unsigned int shallow_depth) sbitmap_queue_get_shallow() argument
543 sbitmap_queue_min_shallow_depth(struct sbitmap_queue *sbq, unsigned int min_shallow_depth) sbitmap_queue_min_shallow_depth() argument
551 __sbitmap_queue_wake_up(struct sbitmap_queue *sbq, int nr) __sbitmap_queue_wake_up() argument
582 sbitmap_queue_wake_up(struct sbitmap_queue *sbq, int nr) sbitmap_queue_wake_up() argument
609 sbitmap_queue_clear_batch(struct sbitmap_queue *sbq, int offset, int *tags, int nr_tags) sbitmap_queue_clear_batch() argument
643 sbitmap_queue_clear(struct sbitmap_queue *sbq, unsigned int nr, unsigned int cpu) sbitmap_queue_clear() argument
671 sbitmap_queue_wake_all(struct sbitmap_queue *sbq) sbitmap_queue_wake_all() argument
692 sbitmap_queue_show(struct sbitmap_queue *sbq, struct seq_file *m) sbitmap_queue_show() argument
726 sbitmap_add_wait_queue(struct sbitmap_queue *sbq, struct sbq_wait_state *ws, struct sbq_wait *sbq_wait) sbitmap_add_wait_queue() argument
748 sbitmap_prepare_to_wait(struct sbitmap_queue *sbq, struct sbq_wait_state *ws, struct sbq_wait *sbq_wait, int state) sbitmap_prepare_to_wait() argument
760 sbitmap_finish_wait(struct sbitmap_queue *sbq, struct sbq_wait_state *ws, struct sbq_wait *sbq_wait) sbitmap_finish_wait() argument
[all...] |
| /kernel/linux/linux-5.10/include/linux/ |
| H A D | sbitmap.h | 357 * @sbq: Bitmap queue to initialize.
366 int sbitmap_queue_init_node(struct sbitmap_queue *sbq, unsigned int depth,
372 * @sbq: Bitmap queue to free.
374 static inline void sbitmap_queue_free(struct sbitmap_queue *sbq) in sbitmap_queue_free() argument
376 kfree(sbq->ws); in sbitmap_queue_free()
377 free_percpu(sbq->alloc_hint); in sbitmap_queue_free()
378 sbitmap_free(&sbq->sb); in sbitmap_queue_free()
383 * @sbq: Bitmap queue to resize.
390 void sbitmap_queue_resize(struct sbitmap_queue *sbq, unsigned int depth);
395 * @sbq
426 sbitmap_queue_get(struct sbitmap_queue *sbq, unsigned int *cpu) sbitmap_queue_get() argument
451 sbitmap_queue_get_shallow(struct sbitmap_queue *sbq, unsigned int *cpu, unsigned int shallow_depth) sbitmap_queue_get_shallow() argument
508 sbq_wait_ptr(struct sbitmap_queue *sbq, atomic_t *wait_index) sbq_wait_ptr() argument
543 struct sbitmap_queue *sbq; /* if set, sbq_wait is accounted */ global() member
[all...] |
| /kernel/linux/linux-6.6/include/linux/ |
| H A D | sbitmap.h | 404 * @sbq: Bitmap queue to initialize.
413 int sbitmap_queue_init_node(struct sbitmap_queue *sbq, unsigned int depth,
419 * @sbq: Bitmap queue to free.
421 static inline void sbitmap_queue_free(struct sbitmap_queue *sbq) in sbitmap_queue_free() argument
423 kfree(sbq->ws); in sbitmap_queue_free()
424 sbitmap_free(&sbq->sb); in sbitmap_queue_free()
429 * @sbq: Bitmap queue to recalculate wake batch.
435 void sbitmap_queue_recalculate_wake_batch(struct sbitmap_queue *sbq,
440 * @sbq: Bitmap queue to resize.
447 void sbitmap_queue_resize(struct sbitmap_queue *sbq, unsigne
496 sbitmap_queue_get(struct sbitmap_queue *sbq, unsigned int *cpu) sbitmap_queue_get() argument
563 sbq_wait_ptr(struct sbitmap_queue *sbq, atomic_t *wait_index) sbq_wait_ptr() argument
599 struct sbitmap_queue *sbq; /* if set, sbq_wait is accounted */ global() member
[all...] |
| /kernel/linux/linux-5.10/drivers/dma/idxd/ |
| H A D | submit.c | 28 struct sbitmap_queue *sbq; in idxd_alloc_desc() local
33 sbq = &wq->sbq; in idxd_alloc_desc()
34 idx = sbitmap_queue_get(sbq, &cpu); in idxd_alloc_desc()
42 ws = &sbq->ws[0]; in idxd_alloc_desc()
44 sbitmap_prepare_to_wait(sbq, ws, &wait, TASK_INTERRUPTIBLE); in idxd_alloc_desc()
47 idx = sbitmap_queue_get(sbq, &cpu); in idxd_alloc_desc()
53 sbitmap_finish_wait(sbq, ws, &wait); in idxd_alloc_desc()
65 sbitmap_queue_clear(&wq->sbq, desc->id, cpu); in idxd_free_desc()
|
| H A D | idxd.h | 123 struct sbitmap_queue sbq; member
|
| H A D | device.c | 157 rc = sbitmap_queue_init_node(&wq->sbq, num_descs, -1, false, GFP_KERNEL, in idxd_wq_alloc_resources()
195 sbitmap_queue_free(&wq->sbq); in idxd_wq_free_resources()
|
| /kernel/linux/linux-6.6/drivers/dma/idxd/ |
| H A D | submit.c | 33 struct sbitmap_queue *sbq; in idxd_alloc_desc() local
38 sbq = &wq->sbq; in idxd_alloc_desc()
39 idx = sbitmap_queue_get(sbq, &cpu); in idxd_alloc_desc()
47 ws = &sbq->ws[0]; in idxd_alloc_desc()
49 sbitmap_prepare_to_wait(sbq, ws, &wait, TASK_INTERRUPTIBLE); in idxd_alloc_desc()
52 idx = sbitmap_queue_get(sbq, &cpu); in idxd_alloc_desc()
58 sbitmap_finish_wait(sbq, ws, &wait); in idxd_alloc_desc()
70 sbitmap_queue_clear(&wq->sbq, desc->id, cpu); in idxd_free_desc()
|
| H A D | idxd.h | 221 struct sbitmap_queue sbq; member
|
| H A D | device.c | 135 rc = sbitmap_queue_init_node(&wq->sbq, num_descs, -1, false, GFP_KERNEL, in idxd_wq_alloc_resources()
175 sbitmap_queue_free(&wq->sbq); in idxd_wq_free_resources()
|
| /kernel/linux/linux-6.6/drivers/net/ethernet/fungible/funcore/ |
| H A D | fun_dev.c | 339 struct sbitmap_queue *sbq = &fdev->admin_sbq; in fun_wait_for_tag() local
340 struct sbq_wait_state *ws = &sbq->ws[0]; in fun_wait_for_tag()
345 sbitmap_prepare_to_wait(sbq, ws, &wait, TASK_UNINTERRUPTIBLE); in fun_wait_for_tag()
350 tag = sbitmap_queue_get(sbq, cpup); in fun_wait_for_tag()
356 sbitmap_finish_wait(sbq, ws, &wait); in fun_wait_for_tag()
|
| /kernel/linux/linux-5.10/drivers/staging/qlge/ |
| H A D | qlge_main.c | 1001 [QLGE_SB] = "sbq",
1016 "ring %u sbq: getting new skb for index %d.\n", in qlge_refill_sb()
1149 sbq_fail = !!qlge_refill_bq(&rx_ring->sbq, gfp); in ql_update_buffer_queues()
1154 * sbq: 1 for header + 1 for data in ql_update_buffer_queues()
1158 if ((sbq_fail && QLGE_BQ_HW_OWNED(&rx_ring->sbq) < 2) || in ql_update_buffer_queues()
1559 struct qlge_bq_desc *sbq_desc = qlge_get_curr_buf(&rx_ring->sbq); in ql_process_mac_rx_skb()
1694 sbq_desc = qlge_get_curr_buf(&rx_ring->sbq); in ql_build_rx_skb()
1724 sbq_desc = qlge_get_curr_buf(&rx_ring->sbq); in ql_build_rx_skb()
1734 sbq_desc = qlge_get_curr_buf(&rx_ring->sbq); in ql_build_rx_skb()
1807 sbq_desc = qlge_get_curr_buf(&rx_ring->sbq); in ql_build_rx_skb()
[all...] |
| H A D | qlge_dbg.c | 1775 netdev_err(qdev->ndev, "rx_ring->sbq.base = %p\n", rx_ring->sbq.base); in ql_dump_rx_ring()
1776 netdev_err(qdev->ndev, "rx_ring->sbq.base_dma = %llx\n", in ql_dump_rx_ring()
1777 (unsigned long long)rx_ring->sbq.base_dma); in ql_dump_rx_ring()
1778 netdev_err(qdev->ndev, "rx_ring->sbq.base_indirect = %p\n", in ql_dump_rx_ring()
1779 rx_ring->sbq.base_indirect); in ql_dump_rx_ring()
1780 netdev_err(qdev->ndev, "rx_ring->sbq.base_indirect_dma = %llx\n", in ql_dump_rx_ring()
1781 (unsigned long long)rx_ring->sbq.base_indirect_dma); in ql_dump_rx_ring()
1782 netdev_err(qdev->ndev, "rx_ring->sbq = %p\n", rx_ring->sbq in ql_dump_rx_ring()
[all...] |
| H A D | qlge.h | 36 /* Use the same len for sbq and lbq. Note that it seems like the device might
1441 offsetof(struct rx_ring, sbq) : \
1481 struct qlge_bq sbq; member
|
| /kernel/linux/linux-6.6/drivers/staging/qlge/ |
| H A D | qlge_main.c | 997 [QLGE_SB] = "sbq",
1012 "ring %u sbq: getting new skb for index %d.\n", in qlge_refill_sb()
1145 sbq_fail = !!qlge_refill_bq(&rx_ring->sbq, gfp); in qlge_update_buffer_queues()
1150 * sbq: 1 for header + 1 for data in qlge_update_buffer_queues()
1154 if ((sbq_fail && QLGE_BQ_HW_OWNED(&rx_ring->sbq) < 2) || in qlge_update_buffer_queues()
1554 struct qlge_bq_desc *sbq_desc = qlge_get_curr_buf(&rx_ring->sbq); in qlge_process_mac_rx_skb()
1689 sbq_desc = qlge_get_curr_buf(&rx_ring->sbq); in qlge_build_rx_skb()
1719 sbq_desc = qlge_get_curr_buf(&rx_ring->sbq); in qlge_build_rx_skb()
1729 sbq_desc = qlge_get_curr_buf(&rx_ring->sbq); in qlge_build_rx_skb()
1802 sbq_desc = qlge_get_curr_buf(&rx_ring->sbq); in qlge_build_rx_skb()
[all...] |
| H A D | qlge.h | 36 /* Use the same len for sbq and lbq. Note that it seems like the device might
1441 offsetof(struct rx_ring, sbq) : \
1481 struct qlge_bq sbq; member
|
| /kernel/linux/linux-5.10/drivers/target/iscsi/ |
| H A D | iscsi_target_util.c | 147 struct sbitmap_queue *sbq; in iscsit_wait_for_tag() local
152 sbq = &se_sess->sess_tag_pool; in iscsit_wait_for_tag()
153 ws = &sbq->ws[0]; in iscsit_wait_for_tag()
155 sbitmap_prepare_to_wait(sbq, ws, &wait, state); in iscsit_wait_for_tag()
158 tag = sbitmap_queue_get(sbq, cpup); in iscsit_wait_for_tag()
164 sbitmap_finish_wait(sbq, ws, &wait); in iscsit_wait_for_tag()
|
| /kernel/linux/linux-6.6/drivers/net/ethernet/intel/ice/ |
| H A D | ice_controlq.c | 62 struct ice_ctl_q_info *cq = &hw->sbq; in ice_sb_init_regs()
622 cq = &hw->sbq; in ice_init_ctrlq()
679 return &hw->sbq; in ice_get_sbq()
701 cq = &hw->sbq; in ice_shutdown_ctrlq()
813 ice_init_ctrlq_locks(&hw->sbq); in ice_create_all_ctrlq()
847 ice_destroy_ctrlq_locks(&hw->sbq); in ice_destroy_all_ctrlq()
|
| H A D | ice_type.h | 870 struct ice_ctl_q_info sbq; member
|
| H A D | ice_main.c | 1436 cq = &hw->sbq; in __ice_clean_ctrlq()
1639 if (ice_ctrlq_pending(hw, &hw->sbq)) in ice_clean_sbq_subtask()
2409 hw->sbq.num_rq_entries = ICE_SBQ_LEN; in ice_set_ctrlq_len()
2410 hw->sbq.num_sq_entries = ICE_SBQ_LEN; in ice_set_ctrlq_len()
2411 hw->sbq.rq_buf_size = ICE_SBQ_MAX_BUF_LEN; in ice_set_ctrlq_len()
2412 hw->sbq.sq_buf_size = ICE_SBQ_MAX_BUF_LEN; in ice_set_ctrlq_len()
|
| /kernel/linux/linux-6.6/drivers/target/iscsi/ |
| H A D | iscsi_target_util.c | 130 struct sbitmap_queue *sbq; in iscsit_wait_for_tag() local
135 sbq = &se_sess->sess_tag_pool; in iscsit_wait_for_tag()
136 ws = &sbq->ws[0]; in iscsit_wait_for_tag()
138 sbitmap_prepare_to_wait(sbq, ws, &wait, state); in iscsit_wait_for_tag()
141 tag = sbitmap_queue_get(sbq, cpup); in iscsit_wait_for_tag()
147 sbitmap_finish_wait(sbq, ws, &wait); in iscsit_wait_for_tag()
|
| /kernel/linux/linux-6.6/block/ |
| H A D | blk-mq.c | 1815 struct sbitmap_queue *sbq; in blk_mq_dispatch_wake() local
1818 sbq = &hctx->tags->bitmap_tags; in blk_mq_dispatch_wake()
1819 atomic_dec(&sbq->ws_active); in blk_mq_dispatch_wake()
1836 struct sbitmap_queue *sbq; in blk_mq_mark_tag_wait() local
1861 sbq = &hctx->tags->breserved_tags; in blk_mq_mark_tag_wait()
1863 sbq = &hctx->tags->bitmap_tags; in blk_mq_mark_tag_wait()
1864 wq = &bt_wait_ptr(sbq, hctx)->wait; in blk_mq_mark_tag_wait()
1874 atomic_inc(&sbq->ws_active); in blk_mq_mark_tag_wait()
1911 atomic_dec(&sbq->ws_active); in blk_mq_mark_tag_wait()
|
| H A D | kyber-iosched.c | 496 khd->domain_wait[i].sbq = NULL; in kyber_init_hctx()
|
| /kernel/linux/linux-5.10/block/ |
| H A D | blk-mq.c | 1149 struct sbitmap_queue *sbq; in blk_mq_dispatch_wake() local
1152 sbq = hctx->tags->bitmap_tags; in blk_mq_dispatch_wake()
1153 atomic_dec(&sbq->ws_active); in blk_mq_dispatch_wake()
1170 struct sbitmap_queue *sbq = hctx->tags->bitmap_tags; in blk_mq_mark_tag_wait() local
1193 wq = &bt_wait_ptr(sbq, hctx)->wait; in blk_mq_mark_tag_wait()
1203 atomic_inc(&sbq->ws_active); in blk_mq_mark_tag_wait()
1240 atomic_dec(&sbq->ws_active); in blk_mq_mark_tag_wait()
|
| H A D | kyber-iosched.c | 490 khd->domain_wait[i].sbq = NULL; in kyber_init_hctx()
|