Lines Matching refs:qp
96 struct siw_qp *qp;
105 qp = sk_to_qp(sk);
107 if (likely(!qp->rx_stream.rx_suspend &&
108 down_read_trylock(&qp->state_lock))) {
109 read_descriptor_t rd_desc = { .arg.data = qp, .count = 1 };
111 if (likely(qp->attrs.state == SIW_QP_STATE_RTS))
120 up_read(&qp->state_lock);
122 siw_dbg_qp(qp, "unable to process RX, suspend: %d\n",
123 qp->rx_stream.rx_suspend);
129 void siw_qp_llp_close(struct siw_qp *qp)
131 siw_dbg_qp(qp, "enter llp close, state = %s\n",
132 siw_qp_state_to_string[qp->attrs.state]);
134 down_write(&qp->state_lock);
136 qp->rx_stream.rx_suspend = 1;
137 qp->tx_ctx.tx_suspend = 1;
138 qp->attrs.sk = NULL;
140 switch (qp->attrs.state) {
145 qp->attrs.state = SIW_QP_STATE_ERROR;
154 if (tx_wqe(qp)->wr_status == SIW_WR_IDLE)
155 qp->attrs.state = SIW_QP_STATE_ERROR;
157 qp->attrs.state = SIW_QP_STATE_IDLE;
161 siw_dbg_qp(qp, "llp close: no state transition needed: %s\n",
162 siw_qp_state_to_string[qp->attrs.state]);
165 siw_sq_flush(qp);
166 siw_rq_flush(qp);
171 if (qp->cep) {
172 siw_cep_put(qp->cep);
173 qp->cep = NULL;
176 up_write(&qp->state_lock);
178 siw_dbg_qp(qp, "llp close exit: state %s\n",
179 siw_qp_state_to_string[qp->attrs.state]);
197 (void)siw_sq_start(cep->qp);
203 static int siw_qp_readq_init(struct siw_qp *qp, int irq_size, int orq_size)
207 qp->irq = vcalloc(irq_size, sizeof(struct siw_sqe));
208 if (!qp->irq) {
209 qp->attrs.irq_size = 0;
215 qp->orq = vcalloc(orq_size, sizeof(struct siw_sqe));
216 if (!qp->orq) {
217 qp->attrs.orq_size = 0;
218 qp->attrs.irq_size = 0;
219 vfree(qp->irq);
223 qp->attrs.irq_size = irq_size;
224 qp->attrs.orq_size = orq_size;
225 siw_dbg_qp(qp, "ORD %d, IRD %d\n", orq_size, irq_size);
229 static int siw_qp_enable_crc(struct siw_qp *qp)
231 struct siw_rx_stream *c_rx = &qp->rx_stream;
232 struct siw_iwarp_tx *c_tx = &qp->tx_ctx;
263 int siw_qp_mpa_rts(struct siw_qp *qp, enum mpa_v2_ctrl ctrl)
265 struct siw_wqe *wqe = tx_wqe(qp);
269 spin_lock_irqsave(&qp->sq_lock, flags);
272 spin_unlock_irqrestore(&qp->sq_lock, flags);
298 spin_lock(&qp->orq_lock);
300 if (qp->attrs.orq_size)
301 rreq = orq_get_free(qp);
304 qp->orq_put++;
308 spin_unlock(&qp->orq_lock);
315 spin_unlock_irqrestore(&qp->sq_lock, flags);
318 rv = siw_sq_start(qp);
367 void siw_init_terminate(struct siw_qp *qp, enum term_elayer layer, u8 etype,
370 if (!qp->term_info.valid) {
371 memset(&qp->term_info, 0, sizeof(qp->term_info));
372 qp->term_info.layer = layer;
373 qp->term_info.etype = etype;
374 qp->term_info.ecode = ecode;
375 qp->term_info.in_tx = in_tx;
376 qp->term_info.valid = 1;
378 siw_dbg_qp(qp, "init TERM: layer %d, type %d, code %d, in tx %s\n",
390 void siw_send_terminate(struct siw_qp *qp)
396 struct socket *s = qp->attrs.sk;
397 struct siw_rx_stream *srx = &qp->rx_stream;
402 if (!qp->term_info.valid)
405 qp->term_info.valid = 0;
407 if (tx_wqe(qp)->wr_status == SIW_WR_INPROGRESS) {
408 siw_dbg_qp(qp, "cannot send TERMINATE: op %d in progress\n",
409 tx_type(tx_wqe(qp)));
412 if (!s && qp->cep)
414 s = qp->cep->sock;
417 siw_dbg_qp(qp, "cannot send TERMINATE: not connected\n");
432 if ((qp->term_info.layer == TERM_ERROR_LAYER_DDP) ||
433 ((qp->term_info.layer == TERM_ERROR_LAYER_RDMAP) &&
434 (qp->term_info.etype != RDMAP_ETYPE_CATASTROPHIC))) {
444 __rdmap_term_set_layer(term, qp->term_info.layer);
445 __rdmap_term_set_etype(term, qp->term_info.etype);
446 __rdmap_term_set_ecode(term, qp->term_info.ecode);
448 switch (qp->term_info.layer) {
450 if (qp->term_info.etype == RDMAP_ETYPE_CATASTROPHIC)
454 if (qp->term_info.etype == RDMAP_ETYPE_REMOTE_PROTECTION) {
463 if (qp->term_info.in_tx) {
465 struct siw_wqe *wqe = tx_wqe(qp);
515 if ((qp->term_info.ecode == RDMAP_ECODE_VERSION) ||
516 (qp->term_info.ecode == RDMAP_ECODE_OPCODE))
543 if (((qp->term_info.etype == DDP_ETYPE_TAGGED_BUF) &&
544 (qp->term_info.ecode == DDP_ECODE_T_VERSION)) ||
545 ((qp->term_info.etype == DDP_ETYPE_UNTAGGED_BUF) &&
546 (qp->term_info.ecode == DDP_ECODE_UT_VERSION)))
586 if (qp->tx_ctx.mpa_crc_hd) {
587 crypto_shash_init(qp->tx_ctx.mpa_crc_hd);
588 if (crypto_shash_update(qp->tx_ctx.mpa_crc_hd,
594 if (crypto_shash_update(qp->tx_ctx.mpa_crc_hd,
599 crypto_shash_final(qp->tx_ctx.mpa_crc_hd, (u8 *)&crc);
603 siw_dbg_qp(qp, "sent TERM: %s, layer %d, type %d, code %d (%d bytes)\n",
615 static void siw_qp_modify_nonstate(struct siw_qp *qp,
621 qp->attrs.flags |= SIW_RDMA_BIND_ENABLED;
623 qp->attrs.flags &= ~SIW_RDMA_BIND_ENABLED;
626 qp->attrs.flags |= SIW_RDMA_WRITE_ENABLED;
628 qp->attrs.flags &= ~SIW_RDMA_WRITE_ENABLED;
631 qp->attrs.flags |= SIW_RDMA_READ_ENABLED;
633 qp->attrs.flags &= ~SIW_RDMA_READ_ENABLED;
637 static int siw_qp_nextstate_from_idle(struct siw_qp *qp,
646 rv = siw_qp_enable_crc(qp);
651 siw_dbg_qp(qp, "no socket\n");
656 siw_dbg_qp(qp, "no MPA\n");
663 qp->tx_ctx.ddp_msn[RDMAP_UNTAGGED_QN_SEND] = 0;
664 qp->tx_ctx.ddp_msn[RDMAP_UNTAGGED_QN_RDMA_READ] = 0;
665 qp->tx_ctx.ddp_msn[RDMAP_UNTAGGED_QN_TERMINATE] = 0;
670 qp->rx_stream.ddp_msn[RDMAP_UNTAGGED_QN_SEND] = 1;
671 qp->rx_stream.ddp_msn[RDMAP_UNTAGGED_QN_RDMA_READ] = 1;
672 qp->rx_stream.ddp_msn[RDMAP_UNTAGGED_QN_TERMINATE] = 1;
678 rv = siw_qp_readq_init(qp, attrs->irq_size,
683 qp->attrs.sk = attrs->sk;
684 qp->attrs.state = SIW_QP_STATE_RTS;
686 siw_dbg_qp(qp, "enter RTS: crc=%s, ord=%u, ird=%u\n",
688 qp->attrs.orq_size, qp->attrs.irq_size);
692 siw_rq_flush(qp);
693 qp->attrs.state = SIW_QP_STATE_ERROR;
694 if (qp->cep) {
695 siw_cep_put(qp->cep);
696 qp->cep = NULL;
706 static int siw_qp_nextstate_from_rts(struct siw_qp *qp,
721 if (tx_wqe(qp)->wr_status == SIW_WR_IDLE) {
722 qp->attrs.state = SIW_QP_STATE_CLOSING;
724 qp->attrs.state = SIW_QP_STATE_ERROR;
725 siw_sq_flush(qp);
727 siw_rq_flush(qp);
733 qp->attrs.state = SIW_QP_STATE_TERMINATE;
735 siw_init_terminate(qp, TERM_ERROR_LAYER_RDMAP,
755 siw_sq_flush(qp);
756 siw_rq_flush(qp);
757 qp->attrs.state = SIW_QP_STATE_ERROR;
767 static void siw_qp_nextstate_from_term(struct siw_qp *qp,
772 siw_rq_flush(qp);
773 qp->attrs.state = SIW_QP_STATE_ERROR;
775 if (tx_wqe(qp)->wr_status != SIW_WR_IDLE)
776 siw_sq_flush(qp);
784 static int siw_qp_nextstate_from_close(struct siw_qp *qp,
791 WARN_ON(tx_wqe(qp)->wr_status != SIW_WR_IDLE);
792 qp->attrs.state = SIW_QP_STATE_IDLE;
807 qp->attrs.state = SIW_QP_STATE_ERROR;
809 if (tx_wqe(qp)->wr_status != SIW_WR_IDLE)
810 siw_sq_flush(qp);
812 siw_rq_flush(qp);
816 siw_dbg_qp(qp, "state transition undefined: %s => %s\n",
817 siw_qp_state_to_string[qp->attrs.state],
826 * Caller must hold qp->state_lock
828 int siw_qp_modify(struct siw_qp *qp, struct siw_qp_attrs *attrs,
836 siw_dbg_qp(qp, "state: %s => %s\n",
837 siw_qp_state_to_string[qp->attrs.state],
841 siw_qp_modify_nonstate(qp, attrs, mask);
846 switch (qp->attrs.state) {
849 rv = siw_qp_nextstate_from_idle(qp, attrs, mask);
853 drop_conn = siw_qp_nextstate_from_rts(qp, attrs);
857 siw_qp_nextstate_from_term(qp, attrs);
861 siw_qp_nextstate_from_close(qp, attrs);
867 siw_qp_cm_drop(qp, 0);
884 static int siw_activate_tx_from_sq(struct siw_qp *qp)
887 struct siw_wqe *wqe = tx_wqe(qp);
890 sqe = sq_get_next(qp);
923 siw_dbg_qp(qp, "cannot fence read\n");
927 spin_lock(&qp->orq_lock);
929 if (qp->attrs.orq_size && !siw_orq_empty(qp)) {
930 qp->tx_ctx.orq_fence = 1;
933 spin_unlock(&qp->orq_lock);
939 if (unlikely(!qp->attrs.orq_size)) {
946 spin_lock(&qp->orq_lock);
948 rreq = orq_get_free(qp);
955 qp->orq_put++;
957 qp->tx_ctx.orq_fence = 1;
960 spin_unlock(&qp->orq_lock);
965 qp->sq_get++;
968 siw_dbg_qp(qp, "error %d\n", rv);
977 * the active IRQ will not be served after qp->irq_burst, if the
980 int siw_activate_tx(struct siw_qp *qp)
983 struct siw_wqe *wqe = tx_wqe(qp);
985 if (!qp->attrs.irq_size)
986 return siw_activate_tx_from_sq(qp);
988 irqe = &qp->irq[qp->irq_get % qp->attrs.irq_size];
991 return siw_activate_tx_from_sq(qp);
997 if (sq_get_next(qp) && ++qp->irq_burst >= SIW_IRQ_MAXBURST_SQ_ACTIVE) {
998 qp->irq_burst = 0;
999 return siw_activate_tx_from_sq(qp);
1025 qp->irq_get++;
1063 int siw_sqe_complete(struct siw_qp *qp, struct siw_sqe *sqe, u32 bytes,
1066 struct siw_cq *cq = qp->scq;
1090 cqe->base_qp = &qp->base_qp;
1092 cqe->qp_id = qp_id(qp);
1121 int siw_rqe_complete(struct siw_qp *qp, struct siw_rqe *rqe, u32 bytes,
1124 struct siw_cq *cq = qp->rcq;
1148 cqe->base_qp = &qp->base_qp;
1154 cqe->qp_id = qp_id(qp);
1191 void siw_sq_flush(struct siw_qp *qp)
1194 struct siw_wqe *wqe = tx_wqe(qp);
1200 while (qp->attrs.orq_size) {
1201 sqe = &qp->orq[qp->orq_get % qp->attrs.orq_size];
1205 if (siw_sqe_complete(qp, sqe, 0, SIW_WC_WR_FLUSH_ERR) != 0)
1209 qp->orq_get++;
1215 siw_dbg_qp(qp, "flush current SQE, type %d, status %d\n",
1228 siw_sqe_complete(qp, &wqe->sqe, wqe->bytes,
1236 while (qp->attrs.sq_size) {
1237 sqe = &qp->sendq[qp->sq_get % qp->attrs.sq_size];
1242 if (siw_sqe_complete(qp, sqe, 0, SIW_WC_WR_FLUSH_ERR) != 0)
1250 qp->sq_get++;
1253 siw_qp_event(qp, IB_EVENT_SQ_DRAINED);
1267 void siw_rq_flush(struct siw_qp *qp)
1269 struct siw_wqe *wqe = &qp->rx_untagged.wqe_active;
1275 siw_dbg_qp(qp, "flush current rqe, type %d, status %d\n",
1281 siw_rqe_complete(qp, &wqe->rqe, wqe->bytes,
1286 siw_sqe_complete(qp, &wqe->sqe, 0, SIW_WC_WR_FLUSH_ERR);
1290 wqe = &qp->rx_tagged.wqe_active;
1299 while (qp->attrs.rq_size) {
1301 &qp->recvq[qp->rq_get % qp->attrs.rq_size];
1306 if (siw_rqe_complete(qp, rqe, 0, 0, SIW_WC_WR_FLUSH_ERR) != 0)
1310 qp->rq_get++;
1314 int siw_qp_add(struct siw_device *sdev, struct siw_qp *qp)
1316 int rv = xa_alloc(&sdev->qp_xa, &qp->base_qp.qp_num, qp, xa_limit_32b,
1320 kref_init(&qp->ref);
1321 qp->sdev = sdev;
1322 siw_dbg_qp(qp, "new QP\n");
1329 struct siw_qp *found, *qp = container_of(ref, struct siw_qp, ref);
1330 struct siw_device *sdev = qp->sdev;
1333 if (qp->cep)
1334 siw_cep_put(qp->cep);
1336 found = xa_erase(&sdev->qp_xa, qp_id(qp));
1337 WARN_ON(found != qp);
1339 list_del(&qp->devq);
1342 vfree(qp->sendq);
1343 vfree(qp->recvq);
1344 vfree(qp->irq);
1345 vfree(qp->orq);
1347 siw_put_tx_cpu(qp->tx_cpu);
1348 complete(&qp->qp_free);