Lines Matching refs:qp
11 #include "qp.h"
16 struct rvt_ack_entry *find_prev_entry(struct rvt_qp *qp, u32 psn, u8 *prev,
18 __must_hold(&qp->s_lock)
24 for (i = qp->r_head_ack_queue; ; i = p) {
25 if (i == qp->s_tail_ack_queue)
30 p = rvt_size_atomic(ib_to_rvt(qp->ibqp.device));
31 if (p == qp->r_head_ack_queue) {
35 e = &qp->s_ack_queue[p];
41 if (p == qp->s_tail_ack_queue &&
59 * @qp: a pointer to the QP
67 static int make_rc_ack(struct hfi1_ibdev *dev, struct rvt_qp *qp,
75 u32 bth1 = qp->remote_qpn | (HFI1_CAP_IS_KSET(OPFN) << IB_BTHE_E_SHIFT);
77 u32 pmtu = qp->pmtu;
78 struct hfi1_qp_priv *qpriv = qp->priv;
81 u8 next = qp->s_tail_ack_queue;
84 trace_hfi1_rsp_make_rc_ack(qp, 0);
85 lockdep_assert_held(&qp->s_lock);
87 if (!(ib_rvt_state_ops[qp->state] & RVT_PROCESS_RECV_OK))
97 switch (qp->s_ack_state) {
100 e = &qp->s_ack_queue[qp->s_tail_ack_queue];
115 e = &qp->s_ack_queue[qp->s_tail_ack_queue];
117 qp->s_acked_ack_queue == qp->s_tail_ack_queue)
118 qp->s_acked_ack_queue = next;
119 qp->s_tail_ack_queue = next;
120 trace_hfi1_rsp_make_rc_ack(qp, e->psn);
125 if (qp->r_head_ack_queue == qp->s_tail_ack_queue) {
126 if (qp->s_flags & RVT_S_ACK_PENDING)
131 e = &qp->s_ack_queue[qp->s_tail_ack_queue];
134 hfi1_tid_rdma_ack_interlock(qp, e)) {
147 if (qp->s_acked_ack_queue ==
148 qp->s_tail_ack_queue)
149 qp->s_acked_ack_queue =
150 qp->r_head_ack_queue;
151 qp->s_tail_ack_queue = qp->r_head_ack_queue;
158 qp->s_ack_rdma_sge.sge = e->rdma_sge;
159 qp->s_ack_rdma_sge.num_sge = 1;
160 ps->s_txreq->ss = &qp->s_ack_rdma_sge;
163 qp->s_ack_state = OP(RDMA_READ_RESPONSE_FIRST);
165 qp->s_ack_state = OP(RDMA_READ_RESPONSE_ONLY);
168 ohdr->u.aeth = rvt_compute_aeth(qp);
170 qp->s_ack_rdma_psn = e->psn;
171 bth2 = mask_psn(qp->s_ack_rdma_psn++);
184 qp->s_ack_state = TID_OP(WRITE_RESP);
185 qp->s_ack_rdma_psn = mask_psn(e->psn + req->cur_seg);
196 if (qp->s_acked_ack_queue ==
197 qp->s_tail_ack_queue)
198 qp->s_acked_ack_queue =
199 qp->r_head_ack_queue;
200 qp->s_tail_ack_queue = qp->r_head_ack_queue;
207 qp->s_ack_rdma_sge.sge = e->rdma_sge;
208 qp->s_ack_rdma_sge.num_sge = 1;
209 qp->s_ack_state = TID_OP(READ_RESP);
215 qp->s_ack_state = OP(ATOMIC_ACKNOWLEDGE);
216 ohdr->u.at.aeth = rvt_compute_aeth(qp);
222 trace_hfi1_tid_write_rsp_make_rc_ack(qp);
223 bth0 = qp->s_ack_state << 24;
227 qp->s_ack_state = OP(RDMA_READ_RESPONSE_MIDDLE);
230 ps->s_txreq->ss = &qp->s_ack_rdma_sge;
231 ps->s_txreq->mr = qp->s_ack_rdma_sge.sge.mr;
234 len = qp->s_ack_rdma_sge.sge.sge_length;
239 ohdr->u.aeth = rvt_compute_aeth(qp);
241 qp->s_ack_state = OP(RDMA_READ_RESPONSE_LAST);
242 e = &qp->s_ack_queue[qp->s_tail_ack_queue];
245 bth0 = qp->s_ack_state << 24;
246 bth2 = mask_psn(qp->s_ack_rdma_psn++);
270 e = &qp->s_ack_queue[qp->s_tail_ack_queue];
279 qp->s_tail_ack_queue == qpriv->r_tid_alloc &&
285 bth2 = mask_psn(qp->s_ack_rdma_psn);
286 hdrlen = hfi1_build_tid_rdma_write_resp(qp, e, ohdr, &bth1,
293 bth0 = qp->s_ack_state << 24;
294 qp->s_ack_rdma_psn++;
295 trace_hfi1_tid_req_make_rc_ack_write(qp, 0, e->opcode, e->psn,
302 qp->s_ack_state = OP(ATOMIC_ACKNOWLEDGE);
307 e = &qp->s_ack_queue[qp->s_tail_ack_queue];
308 ps->s_txreq->ss = &qp->s_ack_rdma_sge;
309 delta = hfi1_build_tid_rdma_read_resp(qp, e, ohdr, &bth0,
318 * Increment qp->s_tail_ack_queue through s_ack_state
321 qp->s_ack_state = OP(RDMA_READ_RESPONSE_LAST);
335 qp->s_ack_state = OP(SEND_ONLY);
337 if (qp->s_nak_state)
339 cpu_to_be32((qp->r_msn & IB_MSN_MASK) |
340 (qp->s_nak_state <<
343 ohdr->u.aeth = rvt_compute_aeth(qp);
347 bth2 = mask_psn(qp->s_ack_psn);
348 qp->s_flags &= ~RVT_S_ACK_PENDING;
352 qp->s_rdma_ack_cnt++;
356 hfi1_make_ruc_header(qp, ohdr, bth0, bth1, bth2, middle, ps);
359 spin_unlock_irqrestore(&qp->s_lock, ps->flags);
360 spin_lock_irqsave(&qp->r_lock, ps->flags);
361 spin_lock(&qp->s_lock);
362 rvt_error_qp(qp, IB_WC_WR_FLUSH_ERR);
363 spin_unlock(&qp->s_lock);
364 spin_unlock_irqrestore(&qp->r_lock, ps->flags);
365 spin_lock_irqsave(&qp->s_lock, ps->flags);
367 qp->s_ack_state = OP(ACKNOWLEDGE);
373 qp->s_flags &= ~(RVT_S_RESP_PENDING
381 * @qp: a pointer to the QP
388 int hfi1_make_rc_req(struct rvt_qp *qp, struct hfi1_pkt_state *ps)
390 struct hfi1_qp_priv *priv = qp->priv;
391 struct hfi1_ibdev *dev = to_idev(qp->ibqp.device);
401 u32 bth1 = qp->remote_qpn | (HFI1_CAP_IS_KSET(OPFN) << IB_BTHE_E_SHIFT);
402 u32 pmtu = qp->pmtu;
409 trace_hfi1_sender_make_rc_req(qp);
410 lockdep_assert_held(&qp->s_lock);
411 ps->s_txreq = get_txreq(ps->dev, qp);
418 if (rdma_ah_get_ah_flags(&qp->remote_ah_attr) & IB_AH_GRH)
425 if ((rdma_ah_get_ah_flags(&qp->remote_ah_attr) & IB_AH_GRH) &&
426 (hfi1_check_mcast(rdma_ah_get_dlid(&qp->remote_ah_attr))))
433 if ((qp->s_flags & RVT_S_RESP_PENDING) &&
434 make_rc_ack(dev, qp, ohdr, ps))
437 if (!(ib_rvt_state_ops[qp->state] & RVT_PROCESS_SEND_OK)) {
438 if (!(ib_rvt_state_ops[qp->state] & RVT_FLUSH_SEND))
441 if (qp->s_last == READ_ONCE(qp->s_head))
445 qp->s_flags |= RVT_S_WAIT_DMA;
448 clear_ahg(qp);
449 wqe = rvt_get_swqe_ptr(qp, qp->s_last);
450 hfi1_trdma_send_complete(qp, wqe, qp->s_last != qp->s_acked ?
456 if (qp->s_flags & (RVT_S_WAIT_RNR | RVT_S_WAIT_ACK | HFI1_S_WAIT_HALT))
459 if (cmp_psn(qp->s_psn, qp->s_sending_hpsn) <= 0) {
460 if (cmp_psn(qp->s_sending_psn, qp->s_sending_hpsn) <= 0) {
461 qp->s_flags |= RVT_S_WAIT_PSN;
464 qp->s_sending_psn = qp->s_psn;
465 qp->s_sending_hpsn = qp->s_psn - 1;
469 wqe = rvt_get_swqe_ptr(qp, qp->s_cur);
471 switch (qp->s_state) {
473 if (!(ib_rvt_state_ops[qp->state] & RVT_PROCESS_NEXT_SEND_OK))
483 if (qp->s_cur == qp->s_tail) {
485 if (qp->s_tail == READ_ONCE(qp->s_head)) {
486 clear_ahg(qp);
496 qp->s_num_rd_atomic &&
498 priv->pending_tid_r_segs < qp->s_num_rd_atomic)) {
499 qp->s_flags |= RVT_S_WAIT_FENCE;
511 if (qp->s_last != qp->s_cur)
513 if (++qp->s_cur == qp->s_size)
514 qp->s_cur = 0;
515 if (++qp->s_tail == qp->s_size)
516 qp->s_tail = 0;
520 qp,
524 rvt_send_complete(qp, wqe,
528 atomic_dec(&qp->local_ops_pending);
533 qp->s_psn = wqe->psn;
541 ss = &qp->s_sge;
542 bth2 = mask_psn(qp->s_psn);
549 hfi1_tid_rdma_wqe_interlock(qp, wqe))
557 if (!rvt_rc_credit_avail(qp, wqe))
560 qp->s_state = OP(SEND_FIRST);
565 qp->s_state = OP(SEND_ONLY);
567 qp->s_state = OP(SEND_ONLY_WITH_IMMEDIATE);
572 qp->s_state = OP(SEND_ONLY_WITH_INVALIDATE);
581 if (++qp->s_cur == qp->s_size)
582 qp->s_cur = 0;
586 if (newreq && !(qp->s_flags & RVT_S_UNLIMITED_CREDIT))
587 qp->s_lsn++;
591 if (!rvt_rc_credit_avail(qp, wqe))
602 qp->s_state = OP(RDMA_WRITE_FIRST);
607 qp->s_state = OP(RDMA_WRITE_ONLY);
609 qp->s_state =
618 if (++qp->s_cur == qp->s_size)
619 qp->s_cur = 0;
631 if (!(qp->s_flags & RVT_S_UNLIMITED_CREDIT))
632 qp->s_lsn++;
635 hwords += hfi1_build_tid_rdma_write_req(qp, wqe, ohdr,
640 priv->s_tid_cur = qp->s_cur;
642 priv->s_tid_tail = qp->s_cur;
649 __w = rvt_get_swqe_ptr(qp, priv->s_tid_cur);
682 priv->s_tid_tail = qp->s_cur;
686 priv->s_tid_cur = qp->s_cur;
698 if (priv->s_tid_tail == qp->s_cur &&
704 priv->s_tid_head = qp->s_cur;
720 trace_hfi1_tid_write_sender_make_req(qp, newreq);
721 trace_hfi1_tid_req_make_req_write(qp, newreq,
725 if (++qp->s_cur == qp->s_size)
726 qp->s_cur = 0;
734 if (qp->s_num_rd_atomic >=
735 qp->s_max_rd_atomic) {
736 qp->s_flags |= RVT_S_WAIT_RDMAR;
739 qp->s_num_rd_atomic++;
740 if (newreq && !(qp->s_flags & RVT_S_UNLIMITED_CREDIT))
741 qp->s_lsn++;
748 qp->s_state = OP(RDMA_READ_REQUEST);
753 if (++qp->s_cur == qp->s_size)
754 qp->s_cur = 0;
758 trace_hfi1_tid_read_sender_make_req(qp, newreq);
761 trace_hfi1_tid_req_make_req_read(qp, newreq,
765 delta = cmp_psn(qp->s_psn, wqe->psn);
772 * but the qp->s_state is set to OP(RDMA_READ_REQUEST)
777 if (qp->s_num_rd_atomic >= qp->s_max_rd_atomic) {
778 qp->s_flags |= RVT_S_WAIT_RDMAR;
792 qp->s_sge.sge = wqe->sg_list[0];
793 qp->s_sge.sg_list = wqe->sg_list + 1;
794 qp->s_sge.num_sge = wqe->wr.num_sge;
795 qp->s_sge.total_len = wqe->length;
796 qp->s_len = wqe->length;
810 req->s_next_psn = qp->s_psn;
814 delta = hfi1_build_tid_rdma_read_req(qp, wqe, ohdr,
821 if (newreq && !(qp->s_flags & RVT_S_UNLIMITED_CREDIT))
822 qp->s_lsn++;
827 ++qp->s_cur == qp->s_size)
828 qp->s_cur = 0;
837 if (qp->s_num_rd_atomic >=
838 qp->s_max_rd_atomic) {
839 qp->s_flags |= RVT_S_WAIT_RDMAR;
842 qp->s_num_rd_atomic++;
845 if (newreq && !(qp->s_flags & RVT_S_UNLIMITED_CREDIT))
846 qp->s_lsn++;
849 qp->s_state = OP(COMPARE_SWAP);
855 qp->s_state = OP(FETCH_ADD);
868 if (++qp->s_cur == qp->s_size)
869 qp->s_cur = 0;
876 qp->s_sge.sge = wqe->sg_list[0];
877 qp->s_sge.sg_list = wqe->sg_list + 1;
878 qp->s_sge.num_sge = wqe->wr.num_sge;
879 qp->s_sge.total_len = wqe->length;
880 qp->s_len = wqe->length;
883 qp->s_tail++;
884 if (qp->s_tail >= qp->s_size)
885 qp->s_tail = 0;
889 qp->s_psn = wqe->lpsn + 1;
891 qp->s_psn = req->s_next_psn;
893 qp->s_psn++;
898 * qp->s_state is normally set to the opcode of the
906 qp->s_len = restart_sge(&qp->s_sge, wqe, qp->s_psn, pmtu);
909 qp->s_state = OP(SEND_MIDDLE);
912 bth2 = mask_psn(qp->s_psn++);
913 ss = &qp->s_sge;
914 len = qp->s_len;
921 qp->s_state = OP(SEND_LAST);
923 qp->s_state = OP(SEND_LAST_WITH_IMMEDIATE);
928 qp->s_state = OP(SEND_LAST_WITH_INVALIDATE);
936 qp->s_cur++;
937 if (qp->s_cur >= qp->s_size)
938 qp->s_cur = 0;
943 * qp->s_state is normally set to the opcode of the
951 qp->s_len = restart_sge(&qp->s_sge, wqe, qp->s_psn, pmtu);
954 qp->s_state = OP(RDMA_WRITE_MIDDLE);
957 bth2 = mask_psn(qp->s_psn++);
958 ss = &qp->s_sge;
959 len = qp->s_len;
966 qp->s_state = OP(RDMA_WRITE_LAST);
968 qp->s_state = OP(RDMA_WRITE_LAST_WITH_IMMEDIATE);
976 qp->s_cur++;
977 if (qp->s_cur >= qp->s_size)
978 qp->s_cur = 0;
983 * qp->s_state is normally set to the opcode of the
991 len = (delta_psn(qp->s_psn, wqe->psn)) * pmtu;
998 qp->s_state = OP(RDMA_READ_REQUEST);
1000 bth2 = mask_psn(qp->s_psn) | IB_BTH_REQ_ACK;
1001 qp->s_psn = wqe->lpsn + 1;
1004 qp->s_cur++;
1005 if (qp->s_cur == qp->s_size)
1006 qp->s_cur = 0;
1019 req->comp_seg = delta_psn(qp->s_psn, wqe->psn);
1023 bth2 = mask_psn(qp->s_psn);
1024 hwords += hfi1_build_tid_rdma_write_req(qp, wqe, ohdr, &bth1,
1026 qp->s_psn = wqe->lpsn + 1;
1028 qp->s_state = TID_OP(WRITE_REQ);
1030 priv->s_tid_cur = qp->s_cur;
1031 if (++qp->s_cur == qp->s_size)
1032 qp->s_cur = 0;
1033 trace_hfi1_tid_req_make_req_write(qp, 0, wqe->wr.opcode,
1044 * Back down. The field qp->s_psn has been set to the psn with
1048 req->cur_seg = delta_psn(qp->s_psn, wqe->psn) / priv->pkts_ps;
1057 hfi1_tid_rdma_restart_req(qp, wqe, &bth2);
1064 hfi1_kern_clear_hw_flow(priv->rcd, qp);
1066 hfi1_trdma_send_complete(qp, wqe, IB_WC_LOC_QP_OP_ERR);
1085 ++qp->s_cur == qp->s_size)
1086 qp->s_cur = 0;
1087 qp->s_psn = req->s_next_psn;
1088 trace_hfi1_tid_req_make_req_read(qp, 0, wqe->wr.opcode,
1093 delta = cmp_psn(qp->s_psn, wqe->psn);
1096 * of a new request, we need to change the qp->s_state so that
1100 qp->s_cur == qp->s_tail) {
1101 qp->s_state = OP(RDMA_READ_REQUEST);
1102 if (delta == 0 || qp->s_cur == qp->s_tail)
1109 if (qp->s_num_rd_atomic >= qp->s_max_rd_atomic) {
1110 qp->s_flags |= RVT_S_WAIT_RDMAR;
1118 delta = hfi1_build_tid_rdma_read_req(qp, wqe, ohdr, &bth1,
1128 ++qp->s_cur == qp->s_size)
1129 qp->s_cur = 0;
1130 qp->s_psn = req->s_next_psn;
1131 trace_hfi1_tid_req_make_req_read(qp, 0, wqe->wr.opcode,
1135 qp->s_sending_hpsn = bth2;
1140 if (qp->s_flags & RVT_S_SEND_ONE) {
1141 qp->s_flags &= ~RVT_S_SEND_ONE;
1142 qp->s_flags |= RVT_S_WAIT_ACK;
1145 qp->s_len -= len;
1151 qp,
1153 bth0 | (qp->s_state << 24),
1170 qp->s_flags &= ~RVT_S_BUSY;
1180 static inline void hfi1_make_bth_aeth(struct rvt_qp *qp,
1184 if (qp->r_nak_state)
1185 ohdr->u.aeth = cpu_to_be32((qp->r_msn & IB_MSN_MASK) |
1186 (qp->r_nak_state <<
1189 ohdr->u.aeth = rvt_compute_aeth(qp);
1192 ohdr->bth[1] = cpu_to_be32(bth1 | qp->remote_qpn);
1193 ohdr->bth[2] = cpu_to_be32(mask_psn(qp->r_ack_psn));
1198 struct rvt_qp *qp = packet->qp;
1202 spin_lock_irqsave(&qp->s_lock, flags);
1203 if (!(ib_rvt_state_ops[qp->state] & RVT_PROCESS_RECV_OK))
1207 qp->s_flags |= RVT_S_ACK_PENDING | RVT_S_RESP_PENDING;
1208 qp->s_nak_state = qp->r_nak_state;
1209 qp->s_ack_psn = qp->r_ack_psn;
1211 qp->s_flags |= RVT_S_ECN;
1214 hfi1_schedule_send(qp);
1216 spin_unlock_irqrestore(&qp->s_lock, flags);
1225 struct rvt_qp *qp = packet->qp;
1239 if (unlikely(rdma_ah_get_ah_flags(&qp->remote_ah_attr) & IB_AH_GRH)) {
1241 rdma_ah_read_grh(&qp->remote_ah_attr),
1250 pkey = hfi1_get_pkey(ibp, qp->s_pkey_index);
1253 (rdma_ah_get_sl(&qp->remote_ah_attr) & IB_SL_MASK) <<
1257 opa_get_lid(rdma_ah_get_dlid(&qp->remote_ah_attr), 9B),
1258 ppd->lid | rdma_ah_get_path_bits(&qp->remote_ah_attr));
1261 if (qp->s_mig_state == IB_MIG_MIGRATED)
1269 hfi1_make_bth_aeth(qp, ohdr, bth0, bth1);
1278 struct rvt_qp *qp = packet->qp;
1296 if (unlikely(rdma_ah_get_ah_flags(&qp->remote_ah_attr) & IB_AH_GRH) &&
1297 hfi1_check_mcast(rdma_ah_get_dlid(&qp->remote_ah_attr))) {
1299 rdma_ah_read_grh(&qp->remote_ah_attr),
1307 pkey = hfi1_get_pkey(ibp, qp->s_pkey_index);
1313 (rdma_ah_get_path_bits(&qp->remote_ah_attr) &
1315 opa_get_lid(rdma_ah_get_dlid(&qp->remote_ah_attr),
1320 if (qp->s_mig_state == IB_MIG_MIGRATED)
1322 hfi1_make_bth_aeth(qp, ohdr, bth0, bth1);
1347 struct rvt_qp *qp = packet->qp;
1349 struct hfi1_qp_priv *priv = qp->priv;
1351 u8 sc5 = ibp->sl_to_sc[rdma_ah_get_sl(&qp->remote_ah_attr)];
1360 qp->r_adefered = 0;
1363 if (qp->s_flags & RVT_S_RESP_PENDING) {
1369 if (qp->s_rdma_ack_cnt) {
1383 pbc = create_pbc(ppd, pbc_flags, qp->srate_mbps,
1396 trace_ack_output_ibhdr(dd_from_ibdev(qp->ibqp.device),
1408 * update_num_rd_atomic - update the qp->s_num_rd_atomic
1409 * @qp: the QP
1413 * This is called from reset_psn() to update qp->s_num_rd_atomic
1417 static void update_num_rd_atomic(struct rvt_qp *qp, u32 psn,
1425 qp->s_num_rd_atomic++;
1428 struct hfi1_qp_priv *priv = qp->priv;
1436 qp->s_num_rd_atomic += req->ack_pending;
1437 trace_hfi1_tid_req_update_num_rd_atomic(qp, 0,
1444 qp->s_num_rd_atomic += req->total_segs;
1451 * @qp: the QP
1458 static void reset_psn(struct rvt_qp *qp, u32 psn)
1460 u32 n = qp->s_acked;
1461 struct rvt_swqe *wqe = rvt_get_swqe_ptr(qp, n);
1463 struct hfi1_qp_priv *priv = qp->priv;
1465 lockdep_assert_held(&qp->s_lock);
1466 qp->s_cur = n;
1469 qp->s_num_rd_atomic = 0;
1476 qp->s_state = OP(SEND_LAST);
1479 update_num_rd_atomic(qp, psn, wqe);
1485 if (++n == qp->s_size)
1487 if (n == qp->s_tail)
1489 wqe = rvt_get_swqe_ptr(qp, n);
1493 wqe = rvt_get_swqe_ptr(qp, qp->s_cur);
1496 qp->s_cur = n;
1502 qp->s_state = OP(SEND_LAST);
1506 update_num_rd_atomic(qp, psn, wqe);
1518 qp->s_state = OP(RDMA_READ_RESPONSE_FIRST);
1523 qp->s_state = OP(RDMA_READ_RESPONSE_LAST);
1527 qp->s_state = TID_OP(WRITE_RESP);
1531 qp->s_state = OP(RDMA_READ_RESPONSE_MIDDLE);
1535 qp->s_state = TID_OP(READ_RESP);
1543 qp->s_state = OP(SEND_LAST);
1547 qp->s_psn = psn;
1553 if ((cmp_psn(qp->s_psn, qp->s_sending_hpsn) <= 0) &&
1554 (cmp_psn(qp->s_sending_psn, qp->s_sending_hpsn) <= 0))
1555 qp->s_flags |= RVT_S_WAIT_PSN;
1556 qp->s_flags &= ~HFI1_S_AHG_VALID;
1557 trace_hfi1_sender_reset_psn(qp);
1564 void hfi1_restart_rc(struct rvt_qp *qp, u32 psn, int wait)
1566 struct hfi1_qp_priv *priv = qp->priv;
1567 struct rvt_swqe *wqe = rvt_get_swqe_ptr(qp, qp->s_acked);
1570 lockdep_assert_held(&qp->r_lock);
1571 lockdep_assert_held(&qp->s_lock);
1572 trace_hfi1_sender_restart_rc(qp);
1573 if (qp->s_retry == 0) {
1574 if (qp->s_mig_state == IB_MIG_ARMED) {
1575 hfi1_migrate_qp(qp);
1576 qp->s_retry = qp->s_retry_cnt;
1577 } else if (qp->s_last == qp->s_acked) {
1584 to_iport(qp->ibqp.device, qp->port_num);
1590 opfn_conn_reply(qp, priv->opfn.curr);
1591 wqe = do_rc_completion(qp, wqe, ibp);
1592 qp->s_flags &= ~RVT_S_WAIT_ACK;
1594 trace_hfi1_tid_write_sender_restart_rc(qp, 0);
1600 hfi1_kern_clear_hw_flow(priv->rcd, qp);
1603 hfi1_trdma_send_complete(qp, wqe,
1605 rvt_error_qp(qp, IB_WC_WR_FLUSH_ERR);
1612 qp->s_retry--;
1615 ibp = to_iport(qp->ibqp.device, qp->port_num);
1620 ibp->rvp.n_rc_resends += delta_psn(qp->s_psn, psn);
1622 qp->s_flags &= ~(RVT_S_WAIT_FENCE | RVT_S_WAIT_RDMAR |
1626 qp->s_flags |= RVT_S_SEND_ONE;
1627 reset_psn(qp, psn);
1631 * Set qp->s_sending_psn to the next PSN after the given one.
1635 static void reset_sending_psn(struct rvt_qp *qp, u32 psn)
1638 u32 n = qp->s_last;
1640 lockdep_assert_held(&qp->s_lock);
1643 wqe = rvt_get_swqe_ptr(qp, n);
1648 qp->s_sending_psn = wqe->lpsn + 1;
1650 qp->s_sending_psn = psn + 1;
1653 if (++n == qp->s_size)
1655 if (n == qp->s_tail)
1662 * @qp: the QP
1672 void hfi1_rc_verbs_aborted(struct rvt_qp *qp, struct hfi1_opa_header *opah)
1687 qp->s_flags |= RVT_S_SEND_ONE;
1693 void hfi1_rc_send_complete(struct rvt_qp *qp, struct hfi1_opa_header *opah)
1696 struct hfi1_qp_priv *priv = qp->priv;
1702 lockdep_assert_held(&qp->s_lock);
1703 if (!(ib_rvt_state_ops[qp->state] & RVT_SEND_OR_FLUSH_OR_RECV_OK))
1712 WARN_ON(!qp->s_rdma_ack_cnt);
1713 qp->s_rdma_ack_cnt--;
1725 reset_sending_psn(qp, psn);
1740 wqe = rvt_get_swqe_ptr(qp, tail);
1744 tail = qp->s_size - 1;
1749 head = qp->s_tail;
1750 tail = qp->s_acked;
1760 !(qp->s_flags &
1762 (ib_rvt_state_ops[qp->state] & RVT_PROCESS_RECV_OK)) {
1764 rvt_add_retry_timer_ext(qp, priv->timeout_shift);
1766 rvt_add_retry_timer(qp);
1775 (ib_rvt_state_ops[qp->state] & RVT_PROCESS_RECV_OK)) {
1781 wqe = rvt_get_swqe_ptr(qp, qp->s_acked);
1785 hfi1_add_tid_retry_timer(qp);
1788 while (qp->s_last != qp->s_acked) {
1789 wqe = rvt_get_swqe_ptr(qp, qp->s_last);
1790 if (cmp_psn(wqe->lpsn, qp->s_sending_psn) >= 0 &&
1791 cmp_psn(qp->s_sending_psn, qp->s_sending_hpsn) <= 0)
1793 trdma_clean_swqe(qp, wqe);
1794 trace_hfi1_qp_send_completion(qp, wqe, qp->s_last);
1795 rvt_qp_complete_swqe(qp,
1804 trace_hfi1_sendcomplete(qp, psn);
1805 if (qp->s_flags & RVT_S_WAIT_PSN &&
1806 cmp_psn(qp->s_sending_psn, qp->s_sending_hpsn) > 0) {
1807 qp->s_flags &= ~RVT_S_WAIT_PSN;
1808 qp->s_sending_psn = qp->s_psn;
1809 qp->s_sending_hpsn = qp->s_psn - 1;
1810 hfi1_schedule_send(qp);
1814 static inline void update_last_psn(struct rvt_qp *qp, u32 psn)
1816 qp->s_last_psn = psn;
1824 struct rvt_swqe *do_rc_completion(struct rvt_qp *qp,
1828 struct hfi1_qp_priv *priv = qp->priv;
1830 lockdep_assert_held(&qp->s_lock);
1836 trace_hfi1_rc_completion(qp, wqe->lpsn);
1837 if (cmp_psn(wqe->lpsn, qp->s_sending_psn) < 0 ||
1838 cmp_psn(qp->s_sending_psn, qp->s_sending_hpsn) > 0) {
1839 trdma_clean_swqe(qp, wqe);
1840 trace_hfi1_qp_send_completion(qp, wqe, qp->s_last);
1841 rvt_qp_complete_swqe(qp,
1855 u8 sl = rdma_ah_get_sl(&qp->remote_ah_attr);
1860 engine = qp_to_sdma_engine(qp, sc5);
1865 qp->s_retry = qp->s_retry_cnt;
1875 update_last_psn(qp, wqe->lpsn);
1882 if (qp->s_acked == qp->s_cur) {
1883 if (++qp->s_cur >= qp->s_size)
1884 qp->s_cur = 0;
1885 qp->s_acked = qp->s_cur;
1886 wqe = rvt_get_swqe_ptr(qp, qp->s_cur);
1887 if (qp->s_acked != qp->s_tail) {
1888 qp->s_state = OP(SEND_LAST);
1889 qp->s_psn = wqe->psn;
1892 if (++qp->s_acked >= qp->s_size)
1893 qp->s_acked = 0;
1894 if (qp->state == IB_QPS_SQD && qp->s_acked == qp->s_cur)
1895 qp->s_draining = 0;
1896 wqe = rvt_get_swqe_ptr(qp, qp->s_acked);
1900 hfi1_schedule_send(qp);
1905 static void set_restart_qp(struct rvt_qp *qp, struct hfi1_ctxtdata *rcd)
1908 if (!(qp->r_flags & RVT_R_RDMAR_SEQ)) {
1909 qp->r_flags |= RVT_R_RDMAR_SEQ;
1910 hfi1_restart_rc(qp, qp->s_last_psn + 1, 0);
1911 if (list_empty(&qp->rspwait)) {
1912 qp->r_flags |= RVT_R_RSP_SEND;
1913 rvt_get_qp(qp);
1914 list_add_tail(&qp->rspwait, &rcd->qp_wait_list);
1920 * update_qp_retry_state - Update qp retry state.
1921 * @qp: the QP
1926 * This function is called to update the qp retry state upon
1927 * receiving a TID WRITE RESP after the qp is scheduled to retry
1930 static void update_qp_retry_state(struct rvt_qp *qp, u32 psn, u32 spsn,
1933 struct hfi1_qp_priv *qpriv = qp->priv;
1935 qp->s_psn = psn + 1;
1943 qp->s_cur = qpriv->s_tid_cur + 1;
1944 if (qp->s_cur >= qp->s_size)
1945 qp->s_cur = 0;
1946 qp->s_state = TID_OP(WRITE_REQ);
1948 qp->s_cur = qpriv->s_tid_cur;
1949 qp->s_state = TID_OP(WRITE_RESP);
1955 * @qp: the QP the ACK came in on
1964 int do_rc_ack(struct rvt_qp *qp, u32 aeth, u32 psn, int opcode,
1969 struct hfi1_qp_priv *qpriv = qp->priv;
1976 lockdep_assert_held(&qp->s_lock);
1986 wqe = rvt_get_swqe_ptr(qp, qp->s_acked);
2023 (delta_psn(psn, qp->s_last_psn) != 1))) {
2024 set_restart_qp(qp, rcd);
2037 opfn_conn_reply(qp, val);
2039 if (qp->s_num_rd_atomic &&
2043 qp->s_num_rd_atomic--;
2045 if ((qp->s_flags & RVT_S_WAIT_FENCE) &&
2046 !qp->s_num_rd_atomic) {
2047 qp->s_flags &= ~(RVT_S_WAIT_FENCE |
2049 hfi1_schedule_send(qp);
2050 } else if (qp->s_flags & RVT_S_WAIT_RDMAR) {
2051 qp->s_flags &= ~(RVT_S_WAIT_RDMAR |
2053 hfi1_schedule_send(qp);
2064 wqe = do_rc_completion(qp, wqe, ibp);
2065 if (qp->s_acked == qp->s_tail)
2069 trace_hfi1_rc_ack_do(qp, aeth, psn, wqe);
2070 trace_hfi1_sender_do_rc_ack(qp);
2076 rvt_mod_retry_timer_ext(qp,
2079 rvt_stop_rc_timers(qp);
2080 } else if (qp->s_acked != qp->s_tail) {
2084 __w = rvt_get_swqe_ptr(qp, qpriv->s_tid_cur);
2104 if (cmp_psn(psn, qp->s_last_psn + 1)) {
2105 set_restart_qp(qp, rcd);
2112 if (qp->s_cur != qp->s_tail &&
2113 cmp_psn(qp->s_psn, psn) <= 0)
2114 update_qp_retry_state(qp, psn,
2118 rvt_mod_retry_timer(qp);
2120 rvt_stop_rc_timers(qp);
2126 rvt_mod_retry_timer(qp);
2132 if (cmp_psn(qp->s_psn, psn) <= 0)
2133 reset_psn(qp, psn + 1);
2137 rvt_stop_rc_timers(qp);
2138 if (cmp_psn(qp->s_psn, psn) <= 0) {
2139 qp->s_state = OP(SEND_LAST);
2140 qp->s_psn = psn + 1;
2143 if (qp->s_flags & RVT_S_WAIT_ACK) {
2144 qp->s_flags &= ~RVT_S_WAIT_ACK;
2145 hfi1_schedule_send(qp);
2147 rvt_get_credit(qp, aeth);
2148 qp->s_rnr_retry = qp->s_rnr_retry_cnt;
2149 qp->s_retry = qp->s_retry_cnt;
2159 update_last_psn(qp, psn);
2164 if (qp->s_acked == qp->s_tail)
2166 if (qp->s_flags & RVT_S_WAIT_RNR)
2168 rdi = ib_to_rvt(qp->ibqp.device);
2171 if (qp->s_rnr_retry == 0) {
2175 if (qp->s_rnr_retry_cnt < 7 && qp->s_rnr_retry_cnt > 0)
2176 qp->s_rnr_retry--;
2186 reset_psn(qp, qp->s_last_psn + 1);
2188 update_last_psn(qp, psn - 1);
2189 reset_psn(qp, psn);
2192 ibp->rvp.n_rc_resends += delta_psn(qp->s_psn, psn);
2193 qp->s_flags &= ~(RVT_S_WAIT_SSN_CREDIT | RVT_S_WAIT_ACK);
2194 rvt_stop_rc_timers(qp);
2195 rvt_add_rnr_timer(qp, aeth);
2199 if (qp->s_acked == qp->s_tail)
2202 update_last_psn(qp, psn - 1);
2213 hfi1_restart_rc(qp, psn, 0);
2214 hfi1_schedule_send(qp);
2231 if (qp->s_last == qp->s_acked) {
2233 hfi1_kern_read_tid_flow_free(qp);
2235 hfi1_trdma_send_complete(qp, wqe, status);
2236 rvt_error_qp(qp, IB_WC_WR_FLUSH_ERR);
2244 qp->s_retry = qp->s_retry_cnt;
2245 qp->s_rnr_retry = qp->s_rnr_retry_cnt;
2255 rvt_stop_rc_timers(qp);
2263 static void rdma_seq_err(struct rvt_qp *qp, struct hfi1_ibport *ibp, u32 psn,
2268 lockdep_assert_held(&qp->s_lock);
2270 rvt_stop_rc_timers(qp);
2272 wqe = rvt_get_swqe_ptr(qp, qp->s_acked);
2281 wqe = do_rc_completion(qp, wqe, ibp);
2285 qp->r_flags |= RVT_R_RDMAR_SEQ;
2286 hfi1_restart_rc(qp, qp->s_last_psn + 1, 0);
2287 if (list_empty(&qp->rspwait)) {
2288 qp->r_flags |= RVT_R_RSP_SEND;
2289 rvt_get_qp(qp);
2290 list_add_tail(&qp->rspwait, &rcd->qp_wait_list);
2307 struct rvt_qp *qp = packet->qp;
2317 u32 pmtu = qp->pmtu;
2323 spin_lock_irqsave(&qp->s_lock, flags);
2324 trace_hfi1_ack(qp, psn);
2327 if (cmp_psn(psn, READ_ONCE(qp->s_next_psn)) >= 0)
2331 diff = cmp_psn(psn, qp->s_last_psn);
2337 rvt_get_credit(qp, aeth);
2346 if (qp->r_flags & RVT_R_RDMAR_SEQ) {
2347 if (cmp_psn(psn, qp->s_last_psn + 1) != 0)
2349 qp->r_flags &= ~RVT_R_RDMAR_SEQ;
2352 if (unlikely(qp->s_acked == qp->s_tail))
2354 wqe = rvt_get_swqe_ptr(qp, qp->s_acked);
2366 if (!do_rc_ack(qp, aeth, psn, opcode, val, rcd) ||
2369 wqe = rvt_get_swqe_ptr(qp, qp->s_acked);
2377 qp->s_rdma_read_len = restart_sge(&qp->s_rdma_read_sge,
2383 if (unlikely(cmp_psn(psn, qp->s_last_psn + 1)))
2390 if (unlikely(pmtu >= qp->s_rdma_read_len))
2395 * 4.096 usec. * (1 << qp->timeout)
2397 rvt_mod_retry_timer(qp);
2398 if (qp->s_flags & RVT_S_WAIT_ACK) {
2399 qp->s_flags &= ~RVT_S_WAIT_ACK;
2400 hfi1_schedule_send(qp);
2404 qp->s_retry = qp->s_retry_cnt;
2410 qp->s_rdma_read_len -= pmtu;
2411 update_last_psn(qp, psn);
2412 spin_unlock_irqrestore(&qp->s_lock, flags);
2413 rvt_copy_sge(qp, &qp->s_rdma_read_sge,
2419 if (!do_rc_ack(qp, aeth, psn, opcode, 0, rcd))
2432 wqe = rvt_get_swqe_ptr(qp, qp->s_acked);
2433 qp->s_rdma_read_len = restart_sge(&qp->s_rdma_read_sge,
2439 if (unlikely(cmp_psn(psn, qp->s_last_psn + 1)))
2451 if (unlikely(tlen != qp->s_rdma_read_len))
2454 rvt_copy_sge(qp, &qp->s_rdma_read_sge,
2456 WARN_ON(qp->s_rdma_read_sge.num_sge);
2457 (void)do_rc_ack(qp, aeth, psn,
2468 rdma_seq_err(qp, ibp, psn, rcd);
2474 if (qp->s_last == qp->s_acked) {
2475 rvt_send_complete(qp, wqe, status);
2476 rvt_error_qp(qp, IB_WC_WR_FLUSH_ERR);
2479 spin_unlock_irqrestore(&qp->s_lock, flags);
2484 static inline void rc_cancel_ack(struct rvt_qp *qp)
2486 qp->r_adefered = 0;
2487 if (list_empty(&qp->rspwait))
2489 list_del_init(&qp->rspwait);
2490 qp->r_flags &= ~RVT_R_RSP_NAK;
2491 rvt_put_qp(qp);
2498 * @qp: the QP for this packet
2511 struct rvt_qp *qp, u32 opcode, u32 psn,
2521 trace_hfi1_rcv_error(qp, psn);
2528 if (!qp->r_nak_state) {
2530 qp->r_nak_state = IB_NAK_PSN_ERROR;
2532 qp->r_ack_psn = qp->r_psn;
2538 rc_defered_ack(rcd, qp);
2563 spin_lock_irqsave(&qp->s_lock, flags);
2565 e = find_prev_entry(qp, psn, &prev, &mra, &old_req);
2588 offset = delta_psn(psn, e->psn) * qp->pmtu;
2598 ok = rvt_rkey_ok(qp, &e->rdma_sge, len, vaddr, rkey,
2610 if (qp->s_acked_ack_queue == qp->s_tail_ack_queue)
2611 qp->s_acked_ack_queue = prev;
2612 qp->s_tail_ack_queue = prev;
2625 if (qp->s_tail_ack_queue == qp->s_acked_ack_queue)
2626 qp->s_acked_ack_queue = prev;
2627 qp->s_tail_ack_queue = prev;
2642 if (mra == qp->r_head_ack_queue) {
2643 spin_unlock_irqrestore(&qp->s_lock, flags);
2644 qp->r_nak_state = 0;
2645 qp->r_ack_psn = qp->r_psn - 1;
2653 if (qp->s_tail_ack_queue == qp->s_acked_ack_queue)
2654 qp->s_acked_ack_queue = mra;
2655 qp->s_tail_ack_queue = mra;
2658 qp->s_ack_state = OP(ACKNOWLEDGE);
2659 qp->s_flags |= RVT_S_RESP_PENDING;
2660 qp->r_nak_state = 0;
2661 hfi1_schedule_send(qp);
2664 spin_unlock_irqrestore(&qp->s_lock, flags);
2768 struct rvt_qp *qp = packet->qp;
2769 struct hfi1_qp_priv *qpriv = qp->priv;
2777 u32 pmtu = qp->pmtu;
2786 lockdep_assert_held(&qp->r_lock);
2791 fecn = process_ecn(qp, packet);
2792 opfn_trigger_conn_request(qp, be32_to_cpu(ohdr->bth[1]));
2807 diff = delta_psn(psn, qp->r_psn);
2809 if (rc_rcv_error(ohdr, data, qp, opcode, psn, diff, rcd))
2815 switch (qp->r_state) {
2850 if (qp->state == IB_QPS_RTR && !(qp->r_flags & RVT_R_COMM_EST))
2851 rvt_comm_est(qp);
2856 ret = rvt_get_rwqe(qp, false);
2861 qp->r_rcv_len = 0;
2874 qp->r_rcv_len += pmtu;
2875 if (unlikely(qp->r_rcv_len > qp->r_len))
2877 rvt_copy_sge(qp, &qp->r_sge, data, pmtu, true, false);
2882 ret = rvt_get_rwqe(qp, true);
2892 ret = rvt_get_rwqe(qp, false);
2897 qp->r_rcv_len = 0;
2911 if (rvt_invalidate_rkey(qp, rkey))
2917 copy_last = rvt_is_user_qp(qp);
2930 wc.byte_len = tlen + qp->r_rcv_len;
2931 if (unlikely(wc.byte_len > qp->r_len))
2933 rvt_copy_sge(qp, &qp->r_sge, data, tlen, true, copy_last);
2934 rvt_put_ss(&qp->r_sge);
2935 qp->r_msn++;
2936 if (!__test_and_clear_bit(RVT_R_WRID_VALID, &qp->r_aflags))
2938 wc.wr_id = qp->r_wr_id;
2945 wc.qp = &qp->ibqp;
2946 wc.src_qp = qp->remote_qpn;
2947 wc.slid = rdma_ah_get_dlid(&qp->remote_ah_attr) & U16_MAX;
2959 wc.sl = rdma_ah_get_sl(&qp->remote_ah_attr);
2966 rvt_recv_cq(qp, &wc, ib_bth_is_solicited(ohdr));
2970 copy_last = rvt_is_user_qp(qp);
2974 if (unlikely(!(qp->qp_access_flags & IB_ACCESS_REMOTE_WRITE)))
2978 qp->r_len = be32_to_cpu(reth->length);
2979 qp->r_rcv_len = 0;
2980 qp->r_sge.sg_list = NULL;
2981 if (qp->r_len != 0) {
2987 ok = rvt_rkey_ok(qp, &qp->r_sge.sge, qp->r_len, vaddr,
2991 qp->r_sge.num_sge = 1;
2993 qp->r_sge.num_sge = 0;
2994 qp->r_sge.sge.mr = NULL;
2995 qp->r_sge.sge.vaddr = NULL;
2996 qp->r_sge.sge.length = 0;
2997 qp->r_sge.sge.sge_length = 0;
3003 ret = rvt_get_rwqe(qp, true);
3008 rvt_put_ss(&qp->r_sge);
3020 if (unlikely(!(qp->qp_access_flags & IB_ACCESS_REMOTE_READ)))
3022 next = qp->r_head_ack_queue + 1;
3024 if (next > rvt_size_atomic(ib_to_rvt(qp->ibqp.device)))
3026 spin_lock_irqsave(&qp->s_lock, flags);
3027 if (unlikely(next == qp->s_acked_ack_queue)) {
3028 if (!qp->s_ack_queue[next].sent)
3030 update_ack_queue(qp, next);
3032 e = &qp->s_ack_queue[qp->r_head_ack_queue];
3042 ok = rvt_rkey_ok(qp, &e->rdma_sge, len, vaddr,
3050 qp->r_psn += rvt_div_mtu(qp, len - 1);
3060 e->lpsn = qp->r_psn;
3066 qp->r_msn++;
3067 qp->r_psn++;
3068 qp->r_state = opcode;
3069 qp->r_nak_state = 0;
3070 qp->r_head_ack_queue = next;
3071 qpriv->r_tid_alloc = qp->r_head_ack_queue;
3074 qp->s_flags |= RVT_S_RESP_PENDING;
3076 qp->s_flags |= RVT_S_ECN;
3077 hfi1_schedule_send(qp);
3079 spin_unlock_irqrestore(&qp->s_lock, flags);
3095 if (unlikely(!(qp->qp_access_flags & IB_ACCESS_REMOTE_ATOMIC) &&
3098 next = qp->r_head_ack_queue + 1;
3099 if (next > rvt_size_atomic(ib_to_rvt(qp->ibqp.device)))
3101 spin_lock_irqsave(&qp->s_lock, flags);
3102 if (unlikely(next == qp->s_acked_ack_queue)) {
3103 if (!qp->s_ack_queue[next].sent)
3105 update_ack_queue(qp, next);
3107 e = &qp->s_ack_queue[qp->r_head_ack_queue];
3111 opfn_conn_response(qp, e, ateth);
3118 if (unlikely(!rvt_rkey_ok(qp, &qp->r_sge.sge, sizeof(u64),
3123 maddr = (atomic64_t *)qp->r_sge.sge.vaddr;
3127 (u64)cmpxchg((u64 *)qp->r_sge.sge.vaddr,
3130 rvt_put_mr(qp->r_sge.sge.mr);
3131 qp->r_sge.num_sge = 0;
3137 qp->r_msn++;
3138 qp->r_psn++;
3139 qp->r_state = opcode;
3140 qp->r_nak_state = 0;
3141 qp->r_head_ack_queue = next;
3142 qpriv->r_tid_alloc = qp->r_head_ack_queue;
3145 qp->s_flags |= RVT_S_RESP_PENDING;
3147 qp->s_flags |= RVT_S_ECN;
3148 hfi1_schedule_send(qp);
3150 spin_unlock_irqrestore(&qp->s_lock, flags);
3158 qp->r_psn++;
3159 qp->r_state = opcode;
3160 qp->r_ack_psn = psn;
3161 qp->r_nak_state = 0;
3165 qp->r_adefered >= HFI1_PSN_CREDIT) {
3166 rc_cancel_ack(qp);
3169 qp->r_adefered++;
3170 rc_defered_ack(rcd, qp);
3175 qp->r_nak_state = qp->r_min_rnr_timer | IB_RNR_NAK;
3176 qp->r_ack_psn = qp->r_psn;
3178 rc_defered_ack(rcd, qp);
3182 rvt_rc_error(qp, IB_WC_LOC_QP_OP_ERR);
3183 qp->r_nak_state = IB_NAK_REMOTE_OPERATIONAL_ERROR;
3184 qp->r_ack_psn = qp->r_psn;
3186 rc_defered_ack(rcd, qp);
3190 spin_unlock_irqrestore(&qp->s_lock, flags);
3192 rvt_rc_error(qp, IB_WC_LOC_QP_OP_ERR);
3193 qp->r_nak_state = IB_NAK_INVALID_REQUEST;
3194 qp->r_ack_psn = qp->r_psn;
3196 rc_defered_ack(rcd, qp);
3200 spin_unlock_irqrestore(&qp->s_lock, flags);
3202 rvt_rc_error(qp, IB_WC_LOC_PROT_ERR);
3203 qp->r_nak_state = IB_NAK_REMOTE_ACCESS_ERROR;
3204 qp->r_ack_psn = qp->r_psn;
3212 struct rvt_qp *qp)
3227 diff = delta_psn(psn, qp->r_psn);
3228 if (!qp->r_nak_state && diff >= 0) {
3230 qp->r_nak_state = IB_NAK_PSN_ERROR;
3232 qp->r_ack_psn = qp->r_psn;
3241 rc_defered_ack(rcd, qp);