Lines Matching refs:hash
35 IO_ACCT_STALLED_BIT = 0, /* stalled on hash */
116 struct io_wq_hash *hash;
449 static bool io_wait_on_hash(struct io_wqe *wqe, unsigned int hash)
454 spin_lock_irq(&wq->hash->wait.lock);
456 __add_wait_queue(&wq->hash->wait, &wqe->wait);
457 if (!test_bit(hash, &wq->hash->map)) {
463 spin_unlock_irq(&wq->hash->wait.lock);
477 unsigned int hash;
487 hash = io_get_work_hash(work);
488 /* all items with this hash lie in [work, tail] */
489 tail = wqe->hash_tail[hash];
492 if (!test_and_set_bit(hash, &wqe->wq->hash->map)) {
493 wqe->hash_tail[hash] = NULL;
498 stall_hash = hash;
499 /* fast forward to a next hash, for-each will fix up @prev */
516 if (wq_has_sleeper(&wqe->wq->hash->wait))
517 wake_up(&wqe->wq->hash->wait);
580 unsigned int hash = io_get_work_hash(work);
599 if (hash != -1U && !next_hashed) {
600 /* serialize hash clear with wake_up() */
601 spin_lock_irq(&wq->hash->wait.lock);
602 clear_bit(hash, &wq->hash->map);
604 spin_unlock_irq(&wq->hash->wait.lock);
605 if (wq_has_sleeper(&wq->hash->wait))
606 wake_up(&wq->hash->wait);
891 unsigned int hash;
900 hash = io_get_work_hash(work);
901 tail = wqe->hash_tail[hash];
902 wqe->hash_tail[hash] = work;
972 * Work items that hash to the same value will not be done in parallel.
1007 unsigned int hash = io_get_work_hash(work);
1010 if (io_wq_is_hashed(work) && work == wqe->hash_tail[hash]) {
1013 if (prev_work && io_get_work_hash(prev_work) == hash)
1014 wqe->hash_tail[hash] = prev_work;
1016 wqe->hash_tail[hash] = NULL;
1150 refcount_inc(&data->hash->refs);
1151 wq->hash = data->hash;
1193 io_wq_put_hash(data->hash);
1259 spin_lock_irq(&wq->hash->wait.lock);
1261 spin_unlock_irq(&wq->hash->wait.lock);
1283 io_wq_put_hash(wq->hash);