Lines Matching refs:send
151 struct rds_ib_send_work *send,
157 switch (send->s_wr.opcode) {
159 if (send->s_op) {
160 rm = container_of(send->s_op, struct rds_message, data);
161 rds_ib_send_unmap_data(ic, send->s_op, wc_status);
166 if (send->s_op) {
167 rm = container_of(send->s_op, struct rds_message, rdma);
168 rds_ib_send_unmap_rdma(ic, send->s_op, wc_status);
173 if (send->s_op) {
174 rm = container_of(send->s_op, struct rds_message, atomic);
175 rds_ib_send_unmap_atomic(ic, send->s_op, wc_status);
181 __func__, send->s_wr.opcode);
185 send->s_wr.opcode = 0xdead;
192 struct rds_ib_send_work *send;
195 for (i = 0, send = ic->i_sends; i < ic->i_send_ring.w_nr; i++, send++) {
198 send->s_op = NULL;
200 send->s_wr.wr_id = i;
201 send->s_wr.sg_list = send->s_sge;
202 send->s_wr.ex.imm_data = 0;
204 sge = &send->s_sge[0];
210 send->s_sge[1].lkey = ic->i_pd->local_dma_lkey;
216 struct rds_ib_send_work *send;
219 for (i = 0, send = ic->i_sends; i < ic->i_send_ring.w_nr; i++, send++) {
220 if (send->s_op && send->s_wr.opcode != 0xdead)
221 rds_ib_send_unmap_op(ic, send, IB_WC_WR_FLUSH_ERR);
239 * operations performed in the send path. As the sender allocs and potentially
247 struct rds_ib_send_work *send;
272 send = &ic->i_sends[oldest];
273 if (send->s_wr.send_flags & IB_SEND_SIGNALED)
276 rm = rds_ib_send_unmap_op(ic, send, wc->status);
278 if (time_after(jiffies, send->s_queued + HZ / 2))
281 if (send->s_op) {
282 if (send->s_op == rm->m_final_op) {
289 send->s_op = NULL;
305 rds_ib_conn_error(conn, "send completion on <%pI6c,%pI6c,%d> had status %u (%s), vendor err 0x%x, disconnecting and reconnecting\n",
317 * - send credits: this tells us how many WRs we're allowed
328 * exhausted their send credits, and are unable to send new credits
329 * to the peer. We achieve this by requiring that we send at least
335 * The RDS send code is essentially single-threaded; rds_send_xmit
336 * sets RDS_IN_XMIT to ensure exclusive access to the send ring.
340 * In the send path, we need to update the counters for send credits
346 * Spinlocks shared between the send and the receive path are bad,
375 /* The last credit must be used to send a credit update. */
393 * the posted regardless of whether any send credits are
439 /* Decide whether to send an update to the peer now.
440 * If we would send a credit update for every single buffer we
442 * consumes buffer, we refill the ring, send ACK to remote
445 * Performance pretty much depends on how often we send
456 struct rds_ib_send_work *send,
466 send->s_wr.send_flags |= IB_SEND_SIGNALED;
479 * once we send the final fragment.
490 struct rds_ib_send_work *send = NULL;
510 /* Do not send cong updates to IB loopback */
595 * sticking the header into the send ring. Which is why we
620 send = &ic->i_sends[pos];
621 first = send;
629 send->s_wr.send_flags = send_flags;
630 send->s_wr.opcode = IB_WR_SEND;
631 send->s_wr.num_sge = 1;
632 send->s_wr.next = NULL;
633 send->s_queued = jiffies;
634 send->s_op = NULL;
636 send->s_sge[0].addr = ic->i_send_hdrs_dma[pos];
638 send->s_sge[0].length = sizeof(struct rds_header);
639 send->s_sge[0].lkey = ic->i_pd->local_dma_lkey;
654 send->s_wr.num_sge = 2;
656 send->s_sge[1].addr = sg_dma_address(scat);
657 send->s_sge[1].addr += rm->data.op_dmaoff;
658 send->s_sge[1].length = len;
659 send->s_sge[1].lkey = ic->i_pd->local_dma_lkey;
670 rds_ib_set_wr_signal_state(ic, send, false);
676 rds_ib_set_wr_signal_state(ic, send, true);
677 send->s_wr.send_flags |= IB_SEND_SOLICITED;
680 if (send->s_wr.send_flags & IB_SEND_SIGNALED)
683 rdsdebug("send %p wr %p num_sge %u next %p\n", send,
684 &send->s_wr, send->s_wr.num_sge, send->s_wr.next);
701 prev->s_wr.next = &send->s_wr;
702 prev = send;
705 send = &ic->i_sends[pos];
716 /* if we finished the message then send completion owns it */
770 struct rds_ib_send_work *send = NULL;
784 /* address of send request in ring */
785 send = &ic->i_sends[pos];
786 send->s_queued = jiffies;
789 send->s_atomic_wr.wr.opcode = IB_WR_MASKED_ATOMIC_CMP_AND_SWP;
790 send->s_atomic_wr.compare_add = op->op_m_cswp.compare;
791 send->s_atomic_wr.swap = op->op_m_cswp.swap;
792 send->s_atomic_wr.compare_add_mask = op->op_m_cswp.compare_mask;
793 send->s_atomic_wr.swap_mask = op->op_m_cswp.swap_mask;
795 send->s_atomic_wr.wr.opcode = IB_WR_MASKED_ATOMIC_FETCH_AND_ADD;
796 send->s_atomic_wr.compare_add = op->op_m_fadd.add;
797 send->s_atomic_wr.swap = 0;
798 send->s_atomic_wr.compare_add_mask = op->op_m_fadd.nocarry_mask;
799 send->s_atomic_wr.swap_mask = 0;
801 send->s_wr.send_flags = 0;
802 nr_sig = rds_ib_set_wr_signal_state(ic, send, op->op_notify);
803 send->s_atomic_wr.wr.num_sge = 1;
804 send->s_atomic_wr.wr.next = NULL;
805 send->s_atomic_wr.remote_addr = op->op_remote_addr;
806 send->s_atomic_wr.rkey = op->op_rkey;
807 send->s_op = op;
808 rds_message_addref(container_of(send->s_op, struct rds_message, atomic));
821 send->s_sge[0].addr = sg_dma_address(op->op_sg);
822 send->s_sge[0].length = sg_dma_len(op->op_sg);
823 send->s_sge[0].lkey = ic->i_pd->local_dma_lkey;
826 send->s_sge[0].addr, send->s_sge[0].length);
831 failed_wr = &send->s_atomic_wr.wr;
832 ret = ib_post_send(ic->i_cm_id->qp, &send->s_atomic_wr.wr, &failed_wr);
833 rdsdebug("ic %p send %p (wr %p) ret %d wr %p\n", ic,
834 send, &send->s_atomic_wr, ret, failed_wr);
835 BUG_ON(failed_wr != &send->s_atomic_wr.wr);
844 if (unlikely(failed_wr != &send->s_atomic_wr.wr)) {
846 BUG_ON(failed_wr != &send->s_atomic_wr.wr);
856 struct rds_ib_send_work *send = NULL;
899 * be enough work requests to send the entire message.
911 send = &ic->i_sends[pos];
912 first = send;
919 send->s_wr.send_flags = 0;
920 send->s_queued = jiffies;
921 send->s_op = NULL;
924 nr_sig += rds_ib_set_wr_signal_state(ic, send,
927 send->s_wr.opcode = op->op_write ? IB_WR_RDMA_WRITE : IB_WR_RDMA_READ;
928 send->s_rdma_wr.remote_addr = remote_addr;
929 send->s_rdma_wr.rkey = op->op_rkey;
932 send->s_rdma_wr.wr.num_sge = max_sge;
935 send->s_rdma_wr.wr.num_sge = num_sge;
938 send->s_rdma_wr.wr.next = NULL;
941 prev->s_rdma_wr.wr.next = &send->s_rdma_wr.wr;
943 for (j = 0; j < send->s_rdma_wr.wr.num_sge &&
947 send->s_sge[j].addr = sg_dma_address(scat);
948 send->s_sge[j].lkey = ic->i_pd->local_dma_lkey;
950 send->s_sge[j].addr = odp_addr;
951 send->s_sge[j].lkey = odp_lkey;
953 send->s_sge[j].length = len;
963 rdsdebug("send %p wr %p num_sge %u next %p\n", send,
964 &send->s_rdma_wr.wr,
965 send->s_rdma_wr.wr.num_sge,
966 send->s_rdma_wr.wr.next);
968 prev = send;
969 if (++send == &ic->i_sends[ic->i_send_ring.w_nr])
970 send = ic->i_sends;
1016 * to send previously (due to flow control). Try again. */