Lines Matching refs:wqe
435 struct rvt_swqe *wqe;
490 wqe = rvt_get_swqe_ptr(qp, qp->s_last);
491 hfi1_trdma_send_complete(qp, wqe, qp->s_last != qp->s_acked ?
510 wqe = rvt_get_swqe_ptr(qp, qp->s_cur);
536 if ((wqe->wr.send_flags & IB_SEND_FENCE) &&
538 (wqe->wr.opcode != IB_WR_TID_RDMA_READ ||
547 if (wqe->wr.opcode == IB_WR_REG_MR ||
548 wqe->wr.opcode == IB_WR_LOCAL_INV) {
558 if (!(wqe->wr.send_flags &
562 wqe->wr.ex.invalidate_rkey);
565 rvt_send_complete(qp, wqe,
574 qp->s_psn = wqe->psn;
581 len = wqe->length;
590 hfi1_tid_rdma_wqe_interlock(qp, wqe))
593 switch (wqe->wr.opcode) {
598 if (!rvt_rc_credit_avail(qp, wqe))
605 if (wqe->wr.opcode == IB_WR_SEND) {
607 } else if (wqe->wr.opcode == IB_WR_SEND_WITH_IMM) {
610 ohdr->u.imm_data = wqe->wr.ex.imm_data;
616 wqe->wr.ex.invalidate_rkey);
619 if (wqe->wr.send_flags & IB_SEND_SOLICITED)
632 if (!rvt_rc_credit_avail(qp, wqe))
636 wqe->rdma_wr.remote_addr,
639 cpu_to_be32(wqe->rdma_wr.rkey);
647 if (wqe->wr.opcode == IB_WR_RDMA_WRITE) {
653 ohdr->u.rc.imm_data = wqe->wr.ex.imm_data;
655 if (wqe->wr.send_flags & IB_SEND_SOLICITED)
676 hwords += hfi1_build_tid_rdma_write_req(qp, wqe, ohdr,
743 req = wqe_to_tid_req(wqe);
751 req->comp_seg = delta_psn(bth2, wqe->psn);
758 delta_psn(wqe->lpsn, bth2) + 1;
763 wqe->wr.opcode,
764 wqe->psn, wqe->lpsn,
784 wqe->rdma_wr.remote_addr,
787 cpu_to_be32(wqe->rdma_wr.rkey);
800 wpriv = wqe->priv;
801 req = wqe_to_tid_req(wqe);
803 wqe->wr.opcode,
804 wqe->psn, wqe->lpsn,
806 delta = cmp_psn(qp->s_psn, wqe->psn);
833 qp->s_sge.sge = wqe->sg_list[0];
834 qp->s_sge.sg_list = wqe->sg_list + 1;
835 qp->s_sge.num_sge = wqe->wr.num_sge;
836 qp->s_sge.total_len = wqe->length;
837 qp->s_len = wqe->length;
854 wqe->length - req->seg_len * req->cur_seg);
855 delta = hfi1_build_tid_rdma_read_req(qp, wqe, ohdr,
888 if (wqe->wr.opcode == IB_WR_ATOMIC_CMP_AND_SWP ||
889 wqe->wr.opcode == IB_WR_OPFN) {
891 put_ib_ateth_swap(wqe->atomic_wr.swap,
893 put_ib_ateth_compare(wqe->atomic_wr.compare_add,
897 put_ib_ateth_swap(wqe->atomic_wr.compare_add,
901 put_ib_ateth_vaddr(wqe->atomic_wr.remote_addr,
904 wqe->atomic_wr.rkey);
916 if (wqe->wr.opcode != IB_WR_TID_RDMA_READ) {
917 qp->s_sge.sge = wqe->sg_list[0];
918 qp->s_sge.sg_list = wqe->sg_list + 1;
919 qp->s_sge.num_sge = wqe->wr.num_sge;
920 qp->s_sge.total_len = wqe->length;
921 qp->s_len = wqe->length;
928 if (wqe->wr.opcode == IB_WR_RDMA_READ ||
929 wqe->wr.opcode == IB_WR_TID_RDMA_WRITE)
930 qp->s_psn = wqe->lpsn + 1;
931 else if (wqe->wr.opcode == IB_WR_TID_RDMA_READ)
947 qp->s_len = restart_sge(&qp->s_sge, wqe, qp->s_psn, pmtu);
961 if (wqe->wr.opcode == IB_WR_SEND) {
963 } else if (wqe->wr.opcode == IB_WR_SEND_WITH_IMM) {
966 ohdr->u.imm_data = wqe->wr.ex.imm_data;
971 ohdr->u.ieth = cpu_to_be32(wqe->wr.ex.invalidate_rkey);
974 if (wqe->wr.send_flags & IB_SEND_SOLICITED)
992 qp->s_len = restart_sge(&qp->s_sge, wqe, qp->s_psn, pmtu);
1006 if (wqe->wr.opcode == IB_WR_RDMA_WRITE) {
1011 ohdr->u.imm_data = wqe->wr.ex.imm_data;
1013 if (wqe->wr.send_flags & IB_SEND_SOLICITED)
1032 len = (delta_psn(qp->s_psn, wqe->psn)) * pmtu;
1034 wqe->rdma_wr.remote_addr + len,
1037 cpu_to_be32(wqe->rdma_wr.rkey);
1038 ohdr->u.rc.reth.length = cpu_to_be32(wqe->length - len);
1042 qp->s_psn = wqe->lpsn + 1;
1056 req = wqe_to_tid_req(wqe);
1060 req->comp_seg = delta_psn(qp->s_psn, wqe->psn);
1061 len = wqe->length - (req->comp_seg * remote->max_len);
1065 hwords += hfi1_build_tid_rdma_write_req(qp, wqe, ohdr, &bth1,
1067 qp->s_psn = wqe->lpsn + 1;
1070 priv->pending_tid_w_resp += delta_psn(wqe->lpsn, bth2) + 1;
1074 trace_hfi1_tid_req_make_req_write(qp, 0, wqe->wr.opcode,
1075 wqe->psn, wqe->lpsn, req);
1079 if (wqe->wr.opcode != IB_WR_TID_RDMA_READ)
1082 req = wqe_to_tid_req(wqe);
1083 wpriv = wqe->priv;
1089 req->cur_seg = delta_psn(qp->s_psn, wqe->psn) / priv->pkts_ps;
1098 hfi1_tid_rdma_restart_req(qp, wqe, &bth2);
1107 hfi1_trdma_send_complete(qp, wqe, IB_WC_LOC_QP_OP_ERR);
1112 wqe->length - req->seg_len * req->cur_seg);
1116 delta = hfi1_build_tid_rdma_read_packet(wqe, ohdr, &bth1,
1129 trace_hfi1_tid_req_make_req_read(qp, 0, wqe->wr.opcode,
1130 wqe->psn, wqe->lpsn, req);
1133 req = wqe_to_tid_req(wqe);
1134 delta = cmp_psn(qp->s_psn, wqe->psn);
1140 if (wqe->wr.opcode != IB_WR_TID_RDMA_READ || delta == 0 ||
1155 wpriv = wqe->priv;
1158 wqe->length - req->seg_len * req->cur_seg);
1159 delta = hfi1_build_tid_rdma_read_req(qp, wqe, ohdr, &bth1,
1172 trace_hfi1_tid_req_make_req_read(qp, 0, wqe->wr.opcode,
1173 wqe->psn, wqe->lpsn, req);
1177 delta = delta_psn(bth2, wqe->psn);
1179 wqe->wr.opcode != IB_WR_TID_RDMA_WRITE)
1453 * @wqe: the wqe
1456 * for the current wqe.
1460 struct rvt_swqe *wqe)
1462 u32 opcode = wqe->wr.opcode;
1469 struct tid_rdma_request *req = wqe_to_tid_req(wqe);
1472 if (cmp_psn(psn, wqe->lpsn) <= 0) {
1475 cur_seg = (psn - wqe->psn) / priv->pkts_ps;
1480 wqe->wr.opcode,
1481 wqe->psn,
1482 wqe->lpsn,
1503 struct rvt_swqe *wqe = rvt_get_swqe_ptr(qp, n);
1517 if (cmp_psn(psn, wqe->psn) <= 0) {
1521 update_num_rd_atomic(qp, psn, wqe);
1531 wqe = rvt_get_swqe_ptr(qp, n);
1532 diff = cmp_psn(psn, wqe->psn);
1534 /* Point wqe back to the previous one*/
1535 wqe = rvt_get_swqe_ptr(qp, qp->s_cur);
1548 update_num_rd_atomic(qp, psn, wqe);
1550 opcode = wqe->wr.opcode;
1609 struct rvt_swqe *wqe = rvt_get_swqe_ptr(qp, qp->s_acked);
1624 if (wqe->wr.opcode == IB_WR_OPFN) {
1633 wqe = do_rc_completion(qp, wqe, ibp);
1637 if (wqe->wr.opcode == IB_WR_TID_RDMA_READ) {
1640 req = wqe_to_tid_req(wqe);
1645 hfi1_trdma_send_complete(qp, wqe,
1658 if (wqe->wr.opcode == IB_WR_RDMA_READ ||
1659 wqe->wr.opcode == IB_WR_TID_RDMA_READ)
1679 struct rvt_swqe *wqe;
1685 wqe = rvt_get_swqe_ptr(qp, n);
1686 if (cmp_psn(psn, wqe->lpsn) <= 0) {
1687 if (wqe->wr.opcode == IB_WR_RDMA_READ ||
1688 wqe->wr.opcode == IB_WR_TID_RDMA_READ ||
1689 wqe->wr.opcode == IB_WR_TID_RDMA_WRITE)
1690 qp->s_sending_psn = wqe->lpsn + 1;
1739 struct rvt_swqe *wqe;
1782 wqe = rvt_get_swqe_ptr(qp, tail);
1783 req = wqe_to_tid_req(wqe);
1823 wqe = rvt_get_swqe_ptr(qp, qp->s_acked);
1824 req = wqe_to_tid_req(wqe);
1825 if (wqe->wr.opcode == IB_WR_TID_RDMA_WRITE &&
1831 wqe = rvt_get_swqe_ptr(qp, qp->s_last);
1832 if (cmp_psn(wqe->lpsn, qp->s_sending_psn) >= 0 &&
1835 trdma_clean_swqe(qp, wqe);
1836 trace_hfi1_qp_send_completion(qp, wqe, qp->s_last);
1838 wqe,
1839 ib_hfi1_wc_opcode[wqe->wr.opcode],
1867 struct rvt_swqe *wqe,
1878 trace_hfi1_rc_completion(qp, wqe->lpsn);
1879 if (cmp_psn(wqe->lpsn, qp->s_sending_psn) < 0 ||
1881 trdma_clean_swqe(qp, wqe);
1882 trace_hfi1_qp_send_completion(qp, wqe, qp->s_last);
1884 wqe,
1885 ib_hfi1_wc_opcode[wqe->wr.opcode],
1916 if (wqe->wr.opcode != IB_WR_TID_RDMA_WRITE)
1917 update_last_psn(qp, wqe->lpsn);
1928 wqe = rvt_get_swqe_ptr(qp, qp->s_cur);
1931 qp->s_psn = wqe->psn;
1938 wqe = rvt_get_swqe_ptr(qp, qp->s_acked);
1944 return wqe;
2012 struct rvt_swqe *wqe;
2028 wqe = rvt_get_swqe_ptr(qp, qp->s_acked);
2035 while ((diff = delta_psn(ack_psn, wqe->lpsn)) >= 0) {
2042 if (wqe->wr.opcode == IB_WR_RDMA_READ &&
2057 if ((wqe->wr.opcode == IB_WR_RDMA_READ &&
2059 (wqe->wr.opcode == IB_WR_TID_RDMA_READ &&
2061 ((wqe->wr.opcode == IB_WR_ATOMIC_CMP_AND_SWP ||
2062 wqe->wr.opcode == IB_WR_ATOMIC_FETCH_AND_ADD) &&
2064 (wqe->wr.opcode == IB_WR_TID_RDMA_WRITE &&
2073 if (wqe->wr.opcode == IB_WR_ATOMIC_CMP_AND_SWP ||
2074 wqe->wr.opcode == IB_WR_ATOMIC_FETCH_AND_ADD) {
2075 u64 *vaddr = wqe->sg_list[0].vaddr;
2078 if (wqe->wr.opcode == IB_WR_OPFN)
2082 (wqe->wr.opcode == IB_WR_RDMA_READ ||
2083 wqe->wr.opcode == IB_WR_ATOMIC_CMP_AND_SWP ||
2084 wqe->wr.opcode == IB_WR_ATOMIC_FETCH_AND_ADD)) {
2103 if (wqe->wr.opcode == IB_WR_TID_RDMA_WRITE)
2106 wqe = do_rc_completion(qp, wqe, ibp);
2111 trace_hfi1_rc_ack_do(qp, aeth, psn, wqe);
2116 if (wqe->wr.opcode == IB_WR_TID_RDMA_READ) {
2117 if (wqe_to_tid_req(wqe)->ack_pending)
2197 if (wqe->wr.opcode == IB_WR_TID_RDMA_WRITE &&
2199 cmp_psn(psn, wqe->psn) >= 0)
2211 if (!(rdi->post_parms[wqe->wr.opcode].flags &
2227 if (wqe->wr.opcode == IB_WR_TID_RDMA_WRITE) {
2274 if (wqe->wr.opcode == IB_WR_TID_RDMA_READ)
2277 hfi1_trdma_send_complete(qp, wqe, status);
2308 struct rvt_swqe *wqe;
2314 wqe = rvt_get_swqe_ptr(qp, qp->s_acked);
2316 while (cmp_psn(psn, wqe->lpsn) > 0) {
2317 if (wqe->wr.opcode == IB_WR_RDMA_READ ||
2318 wqe->wr.opcode == IB_WR_TID_RDMA_READ ||
2319 wqe->wr.opcode == IB_WR_TID_RDMA_WRITE ||
2320 wqe->wr.opcode == IB_WR_ATOMIC_CMP_AND_SWP ||
2321 wqe->wr.opcode == IB_WR_ATOMIC_FETCH_AND_ADD)
2323 wqe = do_rc_completion(qp, wqe, ibp);
2352 struct rvt_swqe *wqe;
2396 wqe = rvt_get_swqe_ptr(qp, qp->s_acked);
2411 wqe = rvt_get_swqe_ptr(qp, qp->s_acked);
2412 if (unlikely(wqe->wr.opcode != IB_WR_RDMA_READ))
2420 wqe, psn, pmtu);
2427 if (unlikely(wqe->wr.opcode != IB_WR_RDMA_READ))
2474 wqe = rvt_get_swqe_ptr(qp, qp->s_acked);
2476 wqe, psn, pmtu);
2483 if (unlikely(wqe->wr.opcode != IB_WR_RDMA_READ))
2517 rvt_send_complete(qp, wqe, status);