Lines Matching refs:hwq
159 struct hwq *hwq = get_hwq(afu, cmd->hwq_index);
161 spin_lock_irqsave(&hwq->hsq_slock, lock_flags);
163 spin_unlock_irqrestore(&hwq->hsq_slock, lock_flags);
186 * @hwq: Hardware queue to flush.
191 static void flush_pending_cmds(struct hwq *hwq)
193 struct cxlflash_cfg *cfg = hwq->afu->parent;
198 list_for_each_entry_safe(cmd, tmp, &hwq->pending_cmds, list) {
226 * @hwq: Hardware queue owning the context to be reset.
235 static int context_reset(struct hwq *hwq, __be64 __iomem *reset_reg)
237 struct cxlflash_cfg *cfg = hwq->afu->parent;
244 dev_dbg(dev, "%s: hwq=%p\n", __func__, hwq);
246 spin_lock_irqsave(&hwq->hsq_slock, lock_flags);
261 flush_pending_cmds(hwq);
263 spin_unlock_irqrestore(&hwq->hsq_slock, lock_flags);
272 * @hwq: Hardware queue owning the context to be reset.
276 static int context_reset_ioarrin(struct hwq *hwq)
278 return context_reset(hwq, &hwq->host_map->ioarrin);
283 * @hwq: Hardware queue owning the context to be reset.
287 static int context_reset_sq(struct hwq *hwq)
289 return context_reset(hwq, &hwq->host_map->sq_ctx_reset);
304 struct hwq *hwq = get_hwq(afu, cmd->hwq_index);
313 spin_lock_irqsave(&hwq->hsq_slock, lock_flags);
314 if (--hwq->room < 0) {
315 room = readq_be(&hwq->host_map->cmd_room);
320 hwq->room = 0;
324 hwq->room = room - 1;
327 list_add(&cmd->list, &hwq->pending_cmds);
328 writeq_be((u64)&cmd->rcb, &hwq->host_map->ioarrin);
330 spin_unlock_irqrestore(&hwq->hsq_slock, lock_flags);
348 struct hwq *hwq = get_hwq(afu, cmd->hwq_index);
353 newval = atomic_dec_if_positive(&hwq->hsq_credits);
361 spin_lock_irqsave(&hwq->hsq_slock, lock_flags);
363 *hwq->hsq_curr = cmd->rcb;
364 if (hwq->hsq_curr < hwq->hsq_end)
365 hwq->hsq_curr++;
367 hwq->hsq_curr = hwq->hsq_start;
369 list_add(&cmd->list, &hwq->pending_cmds);
370 writeq_be((u64)hwq->hsq_curr, &hwq->host_map->sq_tail);
372 spin_unlock_irqrestore(&hwq->hsq_slock, lock_flags);
376 cmd->rcb.data_ea, cmd->rcb.ioasa, rc, hwq->hsq_curr,
377 readq_be(&hwq->host_map->sq_head),
378 readq_be(&hwq->host_map->sq_tail));
426 u32 hwq = 0;
433 hwq = afu->hwq_rr_count++ % afu->num_hwqs;
437 hwq = blk_mq_unique_tag_to_hwq(tag);
440 hwq = smp_processor_id() % afu->num_hwqs;
446 return hwq;
464 struct hwq *hwq = get_hwq(afu, PRIMARY_HWQ);
492 cmd->hwq_index = hwq->index;
494 cmd->rcb.ctx_id = hwq->ctx_hndl;
533 spin_lock_irqsave(&hwq->hsq_slock, lock_flags);
535 spin_unlock_irqrestore(&hwq->hsq_slock, lock_flags);
568 struct hwq *hwq = get_hwq(afu, hwq_index);
622 cmd->rcb.ctx_id = hwq->ctx_hndl;
693 struct hwq *hwq;
706 hwq = get_hwq(afu, i);
708 irq_poll_disable(&hwq->irqpoll);
732 struct hwq *hwq;
739 hwq = get_hwq(afu, index);
741 if (!hwq->ctx_cookie) {
750 cfg->ops->unmap_afu_irq(hwq->ctx_cookie, 3, hwq);
753 cfg->ops->unmap_afu_irq(hwq->ctx_cookie, 2, hwq);
756 cfg->ops->unmap_afu_irq(hwq->ctx_cookie, 1, hwq);
759 cfg->ops->free_afu_irqs(hwq->ctx_cookie);
778 struct hwq *hwq;
786 hwq = get_hwq(afu, index);
788 if (!hwq->ctx_cookie) {
793 WARN_ON(cfg->ops->stop_context(hwq->ctx_cookie));
795 WARN_ON(cfg->ops->release_context(hwq->ctx_cookie));
796 hwq->ctx_cookie = NULL;
798 spin_lock_irqsave(&hwq->hrrq_slock, lock_flags);
799 hwq->hrrq_online = false;
800 spin_unlock_irqrestore(&hwq->hrrq_slock, lock_flags);
802 spin_lock_irqsave(&hwq->hsq_slock, lock_flags);
803 flush_pending_cmds(hwq);
804 spin_unlock_irqrestore(&hwq->hsq_slock, lock_flags);
1265 struct hwq *hwq = get_hwq(afu, PRIMARY_HWQ);
1278 reg = ((u64) (((hwq->ctx_hndl << 8) | SISL_MSI_ASYNC_ERROR)) << 40);
1315 hwq = get_hwq(afu, i);
1317 reg = readq_be(&hwq->host_map->ctx_ctrl);
1320 writeq_be(reg, &hwq->host_map->ctx_ctrl);
1321 writeq_be(SISL_ISTATUS_MASK, &hwq->host_map->intr_mask);
1334 struct hwq *hwq = (struct hwq *)data;
1335 struct cxlflash_cfg *cfg = hwq->afu->parent;
1340 reg = readq_be(&hwq->host_map->intr_status);
1352 writeq_be(reg_unmasked, &hwq->host_map->intr_clear);
1368 static int process_hrrq(struct hwq *hwq, struct list_head *doneq, int budget)
1370 struct afu *afu = hwq->afu;
1374 bool toggle = hwq->toggle;
1377 *hrrq_start = hwq->hrrq_start,
1378 *hrrq_end = hwq->hrrq_end,
1379 *hrrq_curr = hwq->hrrq_curr;
1408 atomic_inc(&hwq->hsq_credits);
1415 hwq->hrrq_curr = hrrq_curr;
1416 hwq->toggle = toggle;
1446 struct hwq *hwq = container_of(irqpoll, struct hwq, irqpoll);
1451 spin_lock_irqsave(&hwq->hrrq_slock, hrrq_flags);
1453 num_entries = process_hrrq(hwq, &doneq, budget);
1457 spin_unlock_irqrestore(&hwq->hrrq_slock, hrrq_flags);
1472 struct hwq *hwq = (struct hwq *)data;
1473 struct afu *afu = hwq->afu;
1478 spin_lock_irqsave(&hwq->hrrq_slock, hrrq_flags);
1481 if (!hwq->hrrq_online) {
1482 spin_unlock_irqrestore(&hwq->hrrq_slock, hrrq_flags);
1487 irq_poll_sched(&hwq->irqpoll);
1488 spin_unlock_irqrestore(&hwq->hrrq_slock, hrrq_flags);
1492 num_entries = process_hrrq(hwq, &doneq, -1);
1493 spin_unlock_irqrestore(&hwq->hrrq_slock, hrrq_flags);
1540 struct hwq *hwq = (struct hwq *)data;
1541 struct afu *afu = hwq->afu;
1734 struct hwq *hwq;
1747 /* Copy frequently used fields into hwq */
1749 hwq = get_hwq(afu, i);
1750 cookie = hwq->ctx_cookie;
1752 hwq->ctx_hndl = (u16) cfg->ops->process_element(cookie);
1753 hwq->host_map = &afu->afu_map->hosts[hwq->ctx_hndl].host;
1754 hwq->ctrl_map = &afu->afu_map->ctrls[hwq->ctx_hndl].ctrl;
1757 writeq_be(SISL_ENDIAN_CTRL, &hwq->host_map->endian_ctrl);
1769 struct hwq *hwq;
1787 hwq = get_hwq(afu, i);
1788 hmap = hwq->host_map;
1790 writeq_be((u64) hwq->hrrq_start, &hmap->rrq_start);
1791 writeq_be((u64) hwq->hrrq_end, &hmap->rrq_end);
1792 hwq->hrrq_online = true;
1795 writeq_be((u64)hwq->hsq_start, &hmap->sq_start);
1796 writeq_be((u64)hwq->hsq_end, &hmap->sq_end);
1841 hwq = get_hwq(afu, i);
1842 ctx = hwq->ctx_cookie;
1844 for (j = 0; j < hwq->num_irqs; j++) {
1846 writeq_be(reg, &hwq->ctrl_map->lisn_ea[j]);
1849 reg = hwq->ctx_hndl;
1851 &hwq->ctrl_map->lisn_pasid[0]);
1853 &hwq->ctrl_map->lisn_pasid[1]);
1861 hwq = get_hwq(afu, i);
1863 (void)readq_be(&hwq->ctrl_map->mbox_r); /* unlock ctx_cap */
1867 &hwq->ctrl_map->ctx_cap);
1876 hwq = get_hwq(afu, PRIMARY_HWQ);
1877 reg = readq_be(&hwq->host_map->ctx_ctrl);
1895 struct hwq *hwq;
1903 hwq = get_hwq(afu, i);
1906 memset(&hwq->rrq_entry, 0, sizeof(hwq->rrq_entry));
1909 hwq->hrrq_start = &hwq->rrq_entry[0];
1910 hwq->hrrq_end = &hwq->rrq_entry[NUM_RRQ_ENTRY - 1];
1911 hwq->hrrq_curr = hwq->hrrq_start;
1912 hwq->toggle = 1;
1915 spin_lock_init(&hwq->hrrq_slock);
1916 spin_lock_init(&hwq->hsq_slock);
1920 memset(&hwq->sq, 0, sizeof(hwq->sq));
1921 hwq->hsq_start = &hwq->sq[0];
1922 hwq->hsq_end = &hwq->sq[NUM_SQ_ENTRY - 1];
1923 hwq->hsq_curr = hwq->hsq_start;
1925 atomic_set(&hwq->hsq_credits, NUM_SQ_ENTRY - 1);
1930 irq_poll_init(&hwq->irqpoll, afu->irqpoll_weight,
1944 * @hwq: Hardware queue to initialize.
1949 struct hwq *hwq)
1952 void *ctx = hwq->ctx_cookie;
1955 bool is_primary_hwq = (hwq->index == PRIMARY_HWQ);
1956 int num_irqs = hwq->num_irqs;
1966 rc = cfg->ops->map_afu_irq(ctx, 1, cxlflash_sync_err_irq, hwq,
1974 rc = cfg->ops->map_afu_irq(ctx, 2, cxlflash_rrq_irq, hwq,
1986 rc = cfg->ops->map_afu_irq(ctx, 3, cxlflash_async_err_irq, hwq,
2008 struct hwq *hwq = get_hwq(cfg->afu, index);
2013 hwq->afu = cfg->afu;
2014 hwq->index = index;
2015 INIT_LIST_HEAD(&hwq->pending_cmds);
2029 WARN_ON(hwq->ctx_cookie);
2030 hwq->ctx_cookie = ctx;
2031 hwq->num_irqs = num_irqs;
2046 level = init_intr(cfg, hwq);
2053 rc = cfg->ops->start_context(hwq->ctx_cookie);
2068 hwq->ctx_cookie = NULL;
2120 struct hwq *hwq;
2137 hwq = get_hwq(afu, PRIMARY_HWQ);
2138 afu->afu_map = cfg->ops->psa_map(hwq->ctx_cookie);
2179 hwq = get_hwq(afu, i);
2181 hwq->room = readq_be(&hwq->host_map->cmd_room);
2304 struct hwq *hwq = get_hwq(afu, PRIMARY_HWQ);
2333 cmd->hwq_index = hwq->index;
2334 cmd->rcb.ctx_id = hwq->ctx_hndl;
2348 rc = afu->context_reset(hwq);
2351 spin_lock_irqsave(&hwq->hsq_slock, lock_flags);
2353 spin_unlock_irqrestore(&hwq->hsq_slock, lock_flags);
2432 struct hwq *hwq = get_hwq(afu, cmd->hwq_index);
2451 rc = afu->context_reset(hwq);
2909 struct hwq *hwq;
2931 hwq = get_hwq(afu, i);
2933 irq_poll_disable(&hwq->irqpoll);
2941 hwq = get_hwq(afu, i);
2943 irq_poll_init(&hwq->irqpoll, weight, cxlflash_irqpoll);