Lines Matching refs:qp
54 * @qp: a pointer to the QP
62 static int qib_make_rc_ack(struct qib_ibdev *dev, struct rvt_qp *qp,
72 if (!(ib_rvt_state_ops[qp->state] & RVT_PROCESS_RECV_OK))
78 switch (qp->s_ack_state) {
81 e = &qp->s_ack_queue[qp->s_tail_ack_queue];
93 if (++qp->s_tail_ack_queue > QIB_MAX_RDMA_ATOMIC)
94 qp->s_tail_ack_queue = 0;
99 if (qp->r_head_ack_queue == qp->s_tail_ack_queue) {
100 if (qp->s_flags & RVT_S_ACK_PENDING)
105 e = &qp->s_ack_queue[qp->s_tail_ack_queue];
115 qp->s_tail_ack_queue = qp->r_head_ack_queue;
119 qp->s_rdma_mr = e->rdma_sge.mr;
120 if (qp->s_rdma_mr)
121 rvt_get_mr(qp->s_rdma_mr);
122 qp->s_ack_rdma_sge.sge = e->rdma_sge;
123 qp->s_ack_rdma_sge.num_sge = 1;
124 qp->s_cur_sge = &qp->s_ack_rdma_sge;
127 qp->s_ack_state = OP(RDMA_READ_RESPONSE_FIRST);
129 qp->s_ack_state = OP(RDMA_READ_RESPONSE_ONLY);
132 ohdr->u.aeth = rvt_compute_aeth(qp);
134 qp->s_ack_rdma_psn = e->psn;
135 bth2 = qp->s_ack_rdma_psn++ & QIB_PSN_MASK;
138 qp->s_cur_sge = NULL;
140 qp->s_ack_state = OP(ATOMIC_ACKNOWLEDGE);
141 ohdr->u.at.aeth = rvt_compute_aeth(qp);
147 bth0 = qp->s_ack_state << 24;
151 qp->s_ack_state = OP(RDMA_READ_RESPONSE_MIDDLE);
154 qp->s_cur_sge = &qp->s_ack_rdma_sge;
155 qp->s_rdma_mr = qp->s_ack_rdma_sge.sge.mr;
156 if (qp->s_rdma_mr)
157 rvt_get_mr(qp->s_rdma_mr);
158 len = qp->s_ack_rdma_sge.sge.sge_length;
162 ohdr->u.aeth = rvt_compute_aeth(qp);
164 qp->s_ack_state = OP(RDMA_READ_RESPONSE_LAST);
165 e = &qp->s_ack_queue[qp->s_tail_ack_queue];
168 bth0 = qp->s_ack_state << 24;
169 bth2 = qp->s_ack_rdma_psn++ & QIB_PSN_MASK;
180 qp->s_ack_state = OP(SEND_ONLY);
181 qp->s_flags &= ~RVT_S_ACK_PENDING;
182 qp->s_cur_sge = NULL;
183 if (qp->s_nak_state)
185 cpu_to_be32((qp->r_msn & IB_MSN_MASK) |
186 (qp->s_nak_state <<
189 ohdr->u.aeth = rvt_compute_aeth(qp);
193 bth2 = qp->s_ack_psn & QIB_PSN_MASK;
195 qp->s_rdma_ack_cnt++;
196 qp->s_hdrwords = hwords;
197 qp->s_cur_size = len;
198 qib_make_ruc_header(qp, ohdr, bth0, bth2);
202 qp->s_ack_state = OP(ACKNOWLEDGE);
203 qp->s_flags &= ~(RVT_S_RESP_PENDING | RVT_S_ACK_PENDING);
209 * @qp: a pointer to the QP
215 int qib_make_rc_req(struct rvt_qp *qp, unsigned long *flags)
217 struct qib_qp_priv *priv = qp->priv;
218 struct qib_ibdev *dev = to_idev(qp->ibqp.device);
226 u32 pmtu = qp->pmtu;
232 if (rdma_ah_get_ah_flags(&qp->remote_ah_attr) & IB_AH_GRH)
236 if ((qp->s_flags & RVT_S_RESP_PENDING) &&
237 qib_make_rc_ack(dev, qp, ohdr, pmtu))
240 if (!(ib_rvt_state_ops[qp->state] & RVT_PROCESS_SEND_OK)) {
241 if (!(ib_rvt_state_ops[qp->state] & RVT_FLUSH_SEND))
244 if (qp->s_last == READ_ONCE(qp->s_head))
248 qp->s_flags |= RVT_S_WAIT_DMA;
251 wqe = rvt_get_swqe_ptr(qp, qp->s_last);
252 rvt_send_complete(qp, wqe, qp->s_last != qp->s_acked ?
258 if (qp->s_flags & (RVT_S_WAIT_RNR | RVT_S_WAIT_ACK))
261 if (qib_cmp24(qp->s_psn, qp->s_sending_hpsn) <= 0) {
262 if (qib_cmp24(qp->s_sending_psn, qp->s_sending_hpsn) <= 0) {
263 qp->s_flags |= RVT_S_WAIT_PSN;
266 qp->s_sending_psn = qp->s_psn;
267 qp->s_sending_hpsn = qp->s_psn - 1;
275 wqe = rvt_get_swqe_ptr(qp, qp->s_cur);
276 switch (qp->s_state) {
278 if (!(ib_rvt_state_ops[qp->state] & RVT_PROCESS_NEXT_SEND_OK))
288 if (qp->s_cur == qp->s_tail) {
290 if (qp->s_tail == READ_ONCE(qp->s_head))
297 qp->s_num_rd_atomic) {
298 qp->s_flags |= RVT_S_WAIT_FENCE;
302 qp->s_psn = wqe->psn;
310 ss = &qp->s_sge;
311 bth2 = qp->s_psn & QIB_PSN_MASK;
316 if (!rvt_rc_credit_avail(qp, wqe))
319 qp->s_state = OP(SEND_FIRST);
324 qp->s_state = OP(SEND_ONLY);
326 qp->s_state = OP(SEND_ONLY_WITH_IMMEDIATE);
334 if (++qp->s_cur == qp->s_size)
335 qp->s_cur = 0;
339 if (newreq && !(qp->s_flags & RVT_S_UNLIMITED_CREDIT))
340 qp->s_lsn++;
344 if (!rvt_rc_credit_avail(qp, wqe))
354 qp->s_state = OP(RDMA_WRITE_FIRST);
359 qp->s_state = OP(RDMA_WRITE_ONLY);
361 qp->s_state = OP(RDMA_WRITE_ONLY_WITH_IMMEDIATE);
370 if (++qp->s_cur == qp->s_size)
371 qp->s_cur = 0;
380 if (qp->s_num_rd_atomic >=
381 qp->s_max_rd_atomic) {
382 qp->s_flags |= RVT_S_WAIT_RDMAR;
385 qp->s_num_rd_atomic++;
386 if (!(qp->s_flags & RVT_S_UNLIMITED_CREDIT))
387 qp->s_lsn++;
395 qp->s_state = OP(RDMA_READ_REQUEST);
400 if (++qp->s_cur == qp->s_size)
401 qp->s_cur = 0;
411 if (qp->s_num_rd_atomic >=
412 qp->s_max_rd_atomic) {
413 qp->s_flags |= RVT_S_WAIT_RDMAR;
416 qp->s_num_rd_atomic++;
417 if (!(qp->s_flags & RVT_S_UNLIMITED_CREDIT))
418 qp->s_lsn++;
421 qp->s_state = OP(COMPARE_SWAP);
427 qp->s_state = OP(FETCH_ADD);
440 if (++qp->s_cur == qp->s_size)
441 qp->s_cur = 0;
447 qp->s_sge.sge = wqe->sg_list[0];
448 qp->s_sge.sg_list = wqe->sg_list + 1;
449 qp->s_sge.num_sge = wqe->wr.num_sge;
450 qp->s_sge.total_len = wqe->length;
451 qp->s_len = wqe->length;
453 qp->s_tail++;
454 if (qp->s_tail >= qp->s_size)
455 qp->s_tail = 0;
458 qp->s_psn = wqe->lpsn + 1;
460 qp->s_psn++;
465 * qp->s_state is normally set to the opcode of the
473 qp->s_len = restart_sge(&qp->s_sge, wqe, qp->s_psn, pmtu);
476 qp->s_state = OP(SEND_MIDDLE);
479 bth2 = qp->s_psn++ & QIB_PSN_MASK;
480 ss = &qp->s_sge;
481 len = qp->s_len;
487 qp->s_state = OP(SEND_LAST);
489 qp->s_state = OP(SEND_LAST_WITH_IMMEDIATE);
497 qp->s_cur++;
498 if (qp->s_cur >= qp->s_size)
499 qp->s_cur = 0;
504 * qp->s_state is normally set to the opcode of the
512 qp->s_len = restart_sge(&qp->s_sge, wqe, qp->s_psn, pmtu);
515 qp->s_state = OP(RDMA_WRITE_MIDDLE);
518 bth2 = qp->s_psn++ & QIB_PSN_MASK;
519 ss = &qp->s_sge;
520 len = qp->s_len;
526 qp->s_state = OP(RDMA_WRITE_LAST);
528 qp->s_state = OP(RDMA_WRITE_LAST_WITH_IMMEDIATE);
536 qp->s_cur++;
537 if (qp->s_cur >= qp->s_size)
538 qp->s_cur = 0;
543 * qp->s_state is normally set to the opcode of the
551 len = ((qp->s_psn - wqe->psn) & QIB_PSN_MASK) * pmtu;
557 qp->s_state = OP(RDMA_READ_REQUEST);
559 bth2 = (qp->s_psn & QIB_PSN_MASK) | IB_BTH_REQ_ACK;
560 qp->s_psn = wqe->lpsn + 1;
563 qp->s_cur++;
564 if (qp->s_cur == qp->s_size)
565 qp->s_cur = 0;
568 qp->s_sending_hpsn = bth2;
572 if (qp->s_flags & RVT_S_SEND_ONE) {
573 qp->s_flags &= ~RVT_S_SEND_ONE;
574 qp->s_flags |= RVT_S_WAIT_ACK;
577 qp->s_len -= len;
578 qp->s_hdrwords = hwords;
579 qp->s_cur_sge = ss;
580 qp->s_cur_size = len;
581 qib_make_ruc_header(qp, ohdr, bth0 | (qp->s_state << 24), bth2);
585 qp->s_flags &= ~RVT_S_BUSY;
591 * @qp: a pointer to the QP
597 void qib_send_rc_ack(struct rvt_qp *qp)
599 struct qib_devdata *dd = dd_from_ibdev(qp->ibqp.device);
600 struct qib_ibport *ibp = to_iport(qp->ibqp.device, qp->port_num);
613 spin_lock_irqsave(&qp->s_lock, flags);
615 if (!(ib_rvt_state_ops[qp->state] & RVT_PROCESS_RECV_OK))
619 if ((qp->s_flags & RVT_S_RESP_PENDING) || qp->s_rdma_ack_cnt)
627 if (unlikely(rdma_ah_get_ah_flags(&qp->remote_ah_attr) &
630 rdma_ah_read_grh(&qp->remote_ah_attr),
636 bth0 = qib_get_pkey(ibp, qp->s_pkey_index) | (OP(ACKNOWLEDGE) << 24);
637 if (qp->s_mig_state == IB_MIG_MIGRATED)
639 if (qp->r_nak_state)
640 ohdr->u.aeth = cpu_to_be32((qp->r_msn & IB_MSN_MASK) |
641 (qp->r_nak_state <<
644 ohdr->u.aeth = rvt_compute_aeth(qp);
645 lrh0 |= ibp->sl_to_vl[rdma_ah_get_sl(&qp->remote_ah_attr)] << 12 |
646 rdma_ah_get_sl(&qp->remote_ah_attr) << 4;
648 hdr.lrh[1] = cpu_to_be16(rdma_ah_get_dlid(&qp->remote_ah_attr));
651 rdma_ah_get_path_bits(&qp->remote_ah_attr));
653 ohdr->bth[1] = cpu_to_be32(qp->remote_qpn);
654 ohdr->bth[2] = cpu_to_be32(qp->r_ack_psn & QIB_PSN_MASK);
656 spin_unlock_irqrestore(&qp->s_lock, flags);
663 qp->s_srate, lrh0 >> 12);
676 spin_lock_irqsave(&qp->s_lock, flags);
711 if (ib_rvt_state_ops[qp->state] & RVT_PROCESS_RECV_OK) {
713 qp->s_flags |= RVT_S_ACK_PENDING | RVT_S_RESP_PENDING;
714 qp->s_nak_state = qp->r_nak_state;
715 qp->s_ack_psn = qp->r_ack_psn;
718 qib_schedule_send(qp);
721 spin_unlock_irqrestore(&qp->s_lock, flags);
728 * @qp: the QP
735 static void reset_psn(struct rvt_qp *qp, u32 psn)
737 u32 n = qp->s_acked;
738 struct rvt_swqe *wqe = rvt_get_swqe_ptr(qp, n);
741 qp->s_cur = n;
748 qp->s_state = OP(SEND_LAST);
757 if (++n == qp->s_size)
759 if (n == qp->s_tail)
761 wqe = rvt_get_swqe_ptr(qp, n);
765 qp->s_cur = n;
771 qp->s_state = OP(SEND_LAST);
785 qp->s_state = OP(RDMA_READ_RESPONSE_FIRST);
790 qp->s_state = OP(RDMA_READ_RESPONSE_LAST);
794 qp->s_state = OP(RDMA_READ_RESPONSE_MIDDLE);
802 qp->s_state = OP(SEND_LAST);
805 qp->s_psn = psn;
811 if ((qib_cmp24(qp->s_psn, qp->s_sending_hpsn) <= 0) &&
812 (qib_cmp24(qp->s_sending_psn, qp->s_sending_hpsn) <= 0))
813 qp->s_flags |= RVT_S_WAIT_PSN;
820 void qib_restart_rc(struct rvt_qp *qp, u32 psn, int wait)
822 struct rvt_swqe *wqe = rvt_get_swqe_ptr(qp, qp->s_acked);
825 if (qp->s_retry == 0) {
826 if (qp->s_mig_state == IB_MIG_ARMED) {
827 qib_migrate_qp(qp);
828 qp->s_retry = qp->s_retry_cnt;
829 } else if (qp->s_last == qp->s_acked) {
830 rvt_send_complete(qp, wqe, IB_WC_RETRY_EXC_ERR);
831 rvt_error_qp(qp, IB_WC_WR_FLUSH_ERR);
836 qp->s_retry--;
838 ibp = to_iport(qp->ibqp.device, qp->port_num);
842 ibp->rvp.n_rc_resends += (qp->s_psn - psn) & QIB_PSN_MASK;
844 qp->s_flags &= ~(RVT_S_WAIT_FENCE | RVT_S_WAIT_RDMAR |
848 qp->s_flags |= RVT_S_SEND_ONE;
849 reset_psn(qp, psn);
853 * Set qp->s_sending_psn to the next PSN after the given one.
856 static void reset_sending_psn(struct rvt_qp *qp, u32 psn)
859 u32 n = qp->s_last;
863 wqe = rvt_get_swqe_ptr(qp, n);
866 qp->s_sending_psn = wqe->lpsn + 1;
868 qp->s_sending_psn = psn + 1;
871 if (++n == qp->s_size)
873 if (n == qp->s_tail)
881 void qib_rc_send_complete(struct rvt_qp *qp, struct ib_header *hdr)
888 if (!(ib_rvt_state_ops[qp->state] & RVT_SEND_OR_FLUSH_OR_RECV_OK))
900 WARN_ON(!qp->s_rdma_ack_cnt);
901 qp->s_rdma_ack_cnt--;
906 reset_sending_psn(qp, psn);
912 if ((psn & IB_BTH_REQ_ACK) && qp->s_acked != qp->s_tail &&
913 !(qp->s_flags & (RVT_S_TIMER | RVT_S_WAIT_RNR | RVT_S_WAIT_PSN)) &&
914 (ib_rvt_state_ops[qp->state] & RVT_PROCESS_RECV_OK))
915 rvt_add_retry_timer(qp);
917 while (qp->s_last != qp->s_acked) {
918 wqe = rvt_get_swqe_ptr(qp, qp->s_last);
919 if (qib_cmp24(wqe->lpsn, qp->s_sending_psn) >= 0 &&
920 qib_cmp24(qp->s_sending_psn, qp->s_sending_hpsn) <= 0)
922 rvt_qp_complete_swqe(qp,
931 if (qp->s_flags & RVT_S_WAIT_PSN &&
932 qib_cmp24(qp->s_sending_psn, qp->s_sending_hpsn) > 0) {
933 qp->s_flags &= ~RVT_S_WAIT_PSN;
934 qp->s_sending_psn = qp->s_psn;
935 qp->s_sending_hpsn = qp->s_psn - 1;
936 qib_schedule_send(qp);
940 static inline void update_last_psn(struct rvt_qp *qp, u32 psn)
942 qp->s_last_psn = psn;
950 static struct rvt_swqe *do_rc_completion(struct rvt_qp *qp,
959 if (qib_cmp24(wqe->lpsn, qp->s_sending_psn) < 0 ||
960 qib_cmp24(qp->s_sending_psn, qp->s_sending_hpsn) > 0)
961 rvt_qp_complete_swqe(qp,
968 qp->s_retry = qp->s_retry_cnt;
969 update_last_psn(qp, wqe->lpsn);
976 if (qp->s_acked == qp->s_cur) {
977 if (++qp->s_cur >= qp->s_size)
978 qp->s_cur = 0;
979 qp->s_acked = qp->s_cur;
980 wqe = rvt_get_swqe_ptr(qp, qp->s_cur);
981 if (qp->s_acked != qp->s_tail) {
982 qp->s_state = OP(SEND_LAST);
983 qp->s_psn = wqe->psn;
986 if (++qp->s_acked >= qp->s_size)
987 qp->s_acked = 0;
988 if (qp->state == IB_QPS_SQD && qp->s_acked == qp->s_cur)
989 qp->s_draining = 0;
990 wqe = rvt_get_swqe_ptr(qp, qp->s_acked);
997 * @qp: the QP the ACK came in on
1006 static int do_rc_ack(struct rvt_qp *qp, u32 aeth, u32 psn, int opcode,
1025 wqe = rvt_get_swqe_ptr(qp, qp->s_acked);
1026 ibp = to_iport(qp->ibqp.device, qp->port_num);
1060 if (!(qp->r_flags & RVT_R_RDMAR_SEQ)) {
1061 qp->r_flags |= RVT_R_RDMAR_SEQ;
1062 qib_restart_rc(qp, qp->s_last_psn + 1, 0);
1063 if (list_empty(&qp->rspwait)) {
1064 qp->r_flags |= RVT_R_RSP_SEND;
1065 rvt_get_qp(qp);
1066 list_add_tail(&qp->rspwait,
1081 if (qp->s_num_rd_atomic &&
1085 qp->s_num_rd_atomic--;
1087 if ((qp->s_flags & RVT_S_WAIT_FENCE) &&
1088 !qp->s_num_rd_atomic) {
1089 qp->s_flags &= ~(RVT_S_WAIT_FENCE |
1091 qib_schedule_send(qp);
1092 } else if (qp->s_flags & RVT_S_WAIT_RDMAR) {
1093 qp->s_flags &= ~(RVT_S_WAIT_RDMAR |
1095 qib_schedule_send(qp);
1098 wqe = do_rc_completion(qp, wqe, ibp);
1099 if (qp->s_acked == qp->s_tail)
1106 if (qp->s_acked != qp->s_tail) {
1111 rvt_mod_retry_timer(qp);
1116 if (qib_cmp24(qp->s_psn, psn) <= 0)
1117 reset_psn(qp, psn + 1);
1120 rvt_stop_rc_timers(qp);
1121 if (qib_cmp24(qp->s_psn, psn) <= 0) {
1122 qp->s_state = OP(SEND_LAST);
1123 qp->s_psn = psn + 1;
1126 if (qp->s_flags & RVT_S_WAIT_ACK) {
1127 qp->s_flags &= ~RVT_S_WAIT_ACK;
1128 qib_schedule_send(qp);
1130 rvt_get_credit(qp, aeth);
1131 qp->s_rnr_retry = qp->s_rnr_retry_cnt;
1132 qp->s_retry = qp->s_retry_cnt;
1133 update_last_psn(qp, psn);
1138 if (qp->s_acked == qp->s_tail)
1140 if (qp->s_flags & RVT_S_WAIT_RNR)
1142 if (qp->s_rnr_retry == 0) {
1146 if (qp->s_rnr_retry_cnt < 7)
1147 qp->s_rnr_retry--;
1150 update_last_psn(qp, psn - 1);
1152 ibp->rvp.n_rc_resends += (qp->s_psn - psn) & QIB_PSN_MASK;
1154 reset_psn(qp, psn);
1156 qp->s_flags &= ~(RVT_S_WAIT_SSN_CREDIT | RVT_S_WAIT_ACK);
1157 rvt_stop_rc_timers(qp);
1158 rvt_add_rnr_timer(qp, aeth);
1162 if (qp->s_acked == qp->s_tail)
1165 update_last_psn(qp, psn - 1);
1176 qib_restart_rc(qp, psn, 0);
1177 qib_schedule_send(qp);
1194 if (qp->s_last == qp->s_acked) {
1195 rvt_send_complete(qp, wqe, status);
1196 rvt_error_qp(qp, IB_WC_WR_FLUSH_ERR);
1204 qp->s_retry = qp->s_retry_cnt;
1205 qp->s_rnr_retry = qp->s_rnr_retry_cnt;
1215 rvt_stop_rc_timers(qp);
1223 static void rdma_seq_err(struct rvt_qp *qp, struct qib_ibport *ibp, u32 psn,
1229 rvt_stop_rc_timers(qp);
1231 wqe = rvt_get_swqe_ptr(qp, qp->s_acked);
1238 wqe = do_rc_completion(qp, wqe, ibp);
1242 qp->r_flags |= RVT_R_RDMAR_SEQ;
1243 qib_restart_rc(qp, qp->s_last_psn + 1, 0);
1244 if (list_empty(&qp->rspwait)) {
1245 qp->r_flags |= RVT_R_RSP_SEND;
1246 rvt_get_qp(qp);
1247 list_add_tail(&qp->rspwait, &rcd->qp_wait_list);
1257 * @qp: the QP for this packet
1270 struct rvt_qp *qp,
1289 if ((qib_cmp24(psn, qp->s_sending_psn) >= 0) &&
1290 (qib_cmp24(qp->s_sending_psn, qp->s_sending_hpsn) <= 0)) {
1296 if (!(qp->s_flags & RVT_S_BUSY)) {
1307 spin_lock_irqsave(&qp->s_lock, flags);
1308 if (!(ib_rvt_state_ops[qp->state] & RVT_PROCESS_RECV_OK))
1312 if (qib_cmp24(psn, READ_ONCE(qp->s_next_psn)) >= 0)
1316 diff = qib_cmp24(psn, qp->s_last_psn);
1322 rvt_get_credit(qp, aeth);
1331 if (qp->r_flags & RVT_R_RDMAR_SEQ) {
1332 if (qib_cmp24(psn, qp->s_last_psn + 1) != 0)
1334 qp->r_flags &= ~RVT_R_RDMAR_SEQ;
1337 if (unlikely(qp->s_acked == qp->s_tail))
1339 wqe = rvt_get_swqe_ptr(qp, qp->s_acked);
1351 if (!do_rc_ack(qp, aeth, psn, opcode, val, rcd) ||
1355 wqe = rvt_get_swqe_ptr(qp, qp->s_acked);
1363 qp->s_rdma_read_len = restart_sge(&qp->s_rdma_read_sge,
1369 if (unlikely(qib_cmp24(psn, qp->s_last_psn + 1)))
1376 if (unlikely(pmtu >= qp->s_rdma_read_len))
1381 * 4.096 usec. * (1 << qp->timeout)
1383 rvt_mod_retry_timer(qp);
1384 if (qp->s_flags & RVT_S_WAIT_ACK) {
1385 qp->s_flags &= ~RVT_S_WAIT_ACK;
1386 qib_schedule_send(qp);
1390 qp->s_retry = qp->s_retry_cnt;
1396 qp->s_rdma_read_len -= pmtu;
1397 update_last_psn(qp, psn);
1398 spin_unlock_irqrestore(&qp->s_lock, flags);
1399 rvt_copy_sge(qp, &qp->s_rdma_read_sge,
1405 if (!do_rc_ack(qp, aeth, psn, opcode, 0, rcd))
1421 wqe = rvt_get_swqe_ptr(qp, qp->s_acked);
1422 qp->s_rdma_read_len = restart_sge(&qp->s_rdma_read_sge,
1428 if (unlikely(qib_cmp24(psn, qp->s_last_psn + 1)))
1443 if (unlikely(tlen != qp->s_rdma_read_len))
1446 rvt_copy_sge(qp, &qp->s_rdma_read_sge,
1448 WARN_ON(qp->s_rdma_read_sge.num_sge);
1449 (void) do_rc_ack(qp, aeth, psn,
1459 rdma_seq_err(qp, ibp, psn, rcd);
1465 if (qp->s_last == qp->s_acked) {
1466 rvt_send_complete(qp, wqe, status);
1467 rvt_error_qp(qp, IB_WC_WR_FLUSH_ERR);
1470 spin_unlock_irqrestore(&qp->s_lock, flags);
1479 * @qp: the QP for this packet
1492 struct rvt_qp *qp,
1498 struct qib_ibport *ibp = to_iport(qp->ibqp.device, qp->port_num);
1510 if (!qp->r_nak_state) {
1512 qp->r_nak_state = IB_NAK_PSN_ERROR;
1514 qp->r_ack_psn = qp->r_psn;
1520 if (list_empty(&qp->rspwait)) {
1521 qp->r_flags |= RVT_R_RSP_NAK;
1522 rvt_get_qp(qp);
1523 list_add_tail(&qp->rspwait, &rcd->qp_wait_list);
1549 spin_lock_irqsave(&qp->s_lock, flags);
1551 for (i = qp->r_head_ack_queue; ; i = prev) {
1552 if (i == qp->s_tail_ack_queue)
1558 if (prev == qp->r_head_ack_queue) {
1562 e = &qp->s_ack_queue[prev];
1568 if (prev == qp->s_tail_ack_queue &&
1596 qp->pmtu;
1609 ok = rvt_rkey_ok(qp, &e->rdma_sge, len, vaddr, rkey,
1621 qp->s_tail_ack_queue = prev;
1634 qp->s_tail_ack_queue = prev;
1649 if (i == qp->r_head_ack_queue) {
1650 spin_unlock_irqrestore(&qp->s_lock, flags);
1651 qp->r_nak_state = 0;
1652 qp->r_ack_psn = qp->r_psn - 1;
1660 if (!(qp->s_flags & RVT_S_RESP_PENDING)) {
1661 spin_unlock_irqrestore(&qp->s_lock, flags);
1662 qp->r_nak_state = 0;
1663 qp->r_ack_psn = qp->s_ack_queue[i].psn - 1;
1670 qp->s_tail_ack_queue = i;
1673 qp->s_ack_state = OP(ACKNOWLEDGE);
1674 qp->s_flags |= RVT_S_RESP_PENDING;
1675 qp->r_nak_state = 0;
1676 qib_schedule_send(qp);
1679 spin_unlock_irqrestore(&qp->s_lock, flags);
1687 static inline void qib_update_ack_queue(struct rvt_qp *qp, unsigned n)
1694 qp->s_tail_ack_queue = next;
1695 qp->s_ack_state = OP(ACKNOWLEDGE);
1705 * @qp: the QP for this packet
1712 int has_grh, void *data, u32 tlen, struct rvt_qp *qp)
1721 u32 pmtu = qp->pmtu;
1737 if (qib_ruc_check_hdr(ibp, hdr, has_grh, qp, opcode))
1751 qib_rc_rcv_resp(ibp, ohdr, data, tlen, qp, opcode, psn,
1757 diff = qib_cmp24(psn, qp->r_psn);
1759 if (qib_rc_rcv_error(ohdr, data, qp, opcode, psn, diff, rcd))
1765 switch (qp->r_state) {
1798 if (qp->state == IB_QPS_RTR && !(qp->r_flags & RVT_R_COMM_EST))
1799 rvt_comm_est(qp);
1804 ret = rvt_get_rwqe(qp, false);
1809 qp->r_rcv_len = 0;
1817 qp->r_rcv_len += pmtu;
1818 if (unlikely(qp->r_rcv_len > qp->r_len))
1820 rvt_copy_sge(qp, &qp->r_sge, data, pmtu, true, false);
1825 ret = rvt_get_rwqe(qp, true);
1834 ret = rvt_get_rwqe(qp, false);
1839 qp->r_rcv_len = 0;
1863 wc.byte_len = tlen + qp->r_rcv_len;
1864 if (unlikely(wc.byte_len > qp->r_len))
1866 rvt_copy_sge(qp, &qp->r_sge, data, tlen, true, false);
1867 rvt_put_ss(&qp->r_sge);
1868 qp->r_msn++;
1869 if (!test_and_clear_bit(RVT_R_WRID_VALID, &qp->r_aflags))
1871 wc.wr_id = qp->r_wr_id;
1878 wc.qp = &qp->ibqp;
1879 wc.src_qp = qp->remote_qpn;
1880 wc.slid = rdma_ah_get_dlid(&qp->remote_ah_attr);
1881 wc.sl = rdma_ah_get_sl(&qp->remote_ah_attr);
1888 rvt_recv_cq(qp, &wc, ib_bth_is_solicited(ohdr));
1894 if (unlikely(!(qp->qp_access_flags & IB_ACCESS_REMOTE_WRITE)))
1899 qp->r_len = be32_to_cpu(reth->length);
1900 qp->r_rcv_len = 0;
1901 qp->r_sge.sg_list = NULL;
1902 if (qp->r_len != 0) {
1908 ok = rvt_rkey_ok(qp, &qp->r_sge.sge, qp->r_len, vaddr,
1912 qp->r_sge.num_sge = 1;
1914 qp->r_sge.num_sge = 0;
1915 qp->r_sge.sge.mr = NULL;
1916 qp->r_sge.sge.vaddr = NULL;
1917 qp->r_sge.sge.length = 0;
1918 qp->r_sge.sge.sge_length = 0;
1924 ret = rvt_get_rwqe(qp, true);
1928 rvt_put_ss(&qp->r_sge);
1941 if (unlikely(!(qp->qp_access_flags & IB_ACCESS_REMOTE_READ)))
1943 next = qp->r_head_ack_queue + 1;
1947 spin_lock_irqsave(&qp->s_lock, flags);
1948 if (unlikely(next == qp->s_tail_ack_queue)) {
1949 if (!qp->s_ack_queue[next].sent)
1951 qib_update_ack_queue(qp, next);
1953 e = &qp->s_ack_queue[qp->r_head_ack_queue];
1966 ok = rvt_rkey_ok(qp, &e->rdma_sge, len, vaddr,
1974 qp->r_psn += rvt_div_mtu(qp, len - 1);
1984 e->lpsn = qp->r_psn;
1990 qp->r_msn++;
1991 qp->r_psn++;
1992 qp->r_state = opcode;
1993 qp->r_nak_state = 0;
1994 qp->r_head_ack_queue = next;
1997 qp->s_flags |= RVT_S_RESP_PENDING;
1998 qib_schedule_send(qp);
2013 if (unlikely(!(qp->qp_access_flags & IB_ACCESS_REMOTE_ATOMIC)))
2015 next = qp->r_head_ack_queue + 1;
2018 spin_lock_irqsave(&qp->s_lock, flags);
2019 if (unlikely(next == qp->s_tail_ack_queue)) {
2020 if (!qp->s_ack_queue[next].sent)
2022 qib_update_ack_queue(qp, next);
2024 e = &qp->s_ack_queue[qp->r_head_ack_queue];
2035 if (unlikely(!rvt_rkey_ok(qp, &qp->r_sge.sge, sizeof(u64),
2040 maddr = (atomic64_t *) qp->r_sge.sge.vaddr;
2044 (u64) cmpxchg((u64 *) qp->r_sge.sge.vaddr,
2047 rvt_put_mr(qp->r_sge.sge.mr);
2048 qp->r_sge.num_sge = 0;
2053 qp->r_msn++;
2054 qp->r_psn++;
2055 qp->r_state = opcode;
2056 qp->r_nak_state = 0;
2057 qp->r_head_ack_queue = next;
2060 qp->s_flags |= RVT_S_RESP_PENDING;
2061 qib_schedule_send(qp);
2070 qp->r_psn++;
2071 qp->r_state = opcode;
2072 qp->r_ack_psn = psn;
2073 qp->r_nak_state = 0;
2080 qp->r_nak_state = IB_RNR_NAK | qp->r_min_rnr_timer;
2081 qp->r_ack_psn = qp->r_psn;
2083 if (list_empty(&qp->rspwait)) {
2084 qp->r_flags |= RVT_R_RSP_NAK;
2085 rvt_get_qp(qp);
2086 list_add_tail(&qp->rspwait, &rcd->qp_wait_list);
2091 rvt_rc_error(qp, IB_WC_LOC_QP_OP_ERR);
2092 qp->r_nak_state = IB_NAK_REMOTE_OPERATIONAL_ERROR;
2093 qp->r_ack_psn = qp->r_psn;
2095 if (list_empty(&qp->rspwait)) {
2096 qp->r_flags |= RVT_R_RSP_NAK;
2097 rvt_get_qp(qp);
2098 list_add_tail(&qp->rspwait, &rcd->qp_wait_list);
2103 spin_unlock_irqrestore(&qp->s_lock, flags);
2105 rvt_rc_error(qp, IB_WC_LOC_QP_OP_ERR);
2106 qp->r_nak_state = IB_NAK_INVALID_REQUEST;
2107 qp->r_ack_psn = qp->r_psn;
2109 if (list_empty(&qp->rspwait)) {
2110 qp->r_flags |= RVT_R_RSP_NAK;
2111 rvt_get_qp(qp);
2112 list_add_tail(&qp->rspwait, &rcd->qp_wait_list);
2117 spin_unlock_irqrestore(&qp->s_lock, flags);
2119 rvt_rc_error(qp, IB_WC_LOC_PROT_ERR);
2120 qp->r_nak_state = IB_NAK_REMOTE_ACCESS_ERROR;
2121 qp->r_ack_psn = qp->r_psn;
2123 qib_send_rc_ack(qp);
2127 spin_unlock_irqrestore(&qp->s_lock, flags);