Lines Matching refs:ic
73 static void rds_ib_send_unmap_data(struct rds_ib_connection *ic,
78 ib_dma_unmap_sg(ic->i_cm_id->device,
83 static void rds_ib_send_unmap_rdma(struct rds_ib_connection *ic,
88 ib_dma_unmap_sg(ic->i_cm_id->device,
123 static void rds_ib_send_unmap_atomic(struct rds_ib_connection *ic,
129 ib_dma_unmap_sg(ic->i_cm_id->device, op->op_sg, 1,
150 static struct rds_message *rds_ib_send_unmap_op(struct rds_ib_connection *ic,
161 rds_ib_send_unmap_data(ic, send->s_op, wc_status);
168 rds_ib_send_unmap_rdma(ic, send->s_op, wc_status);
175 rds_ib_send_unmap_atomic(ic, send->s_op, wc_status);
190 void rds_ib_send_init_ring(struct rds_ib_connection *ic)
195 for (i = 0, send = ic->i_sends; i < ic->i_send_ring.w_nr; i++, send++) {
205 sge->addr = ic->i_send_hdrs_dma[i];
208 sge->lkey = ic->i_pd->local_dma_lkey;
210 send->s_sge[1].lkey = ic->i_pd->local_dma_lkey;
214 void rds_ib_send_clear_ring(struct rds_ib_connection *ic)
219 for (i = 0, send = ic->i_sends; i < ic->i_send_ring.w_nr; i++, send++) {
221 rds_ib_send_unmap_op(ic, send, IB_WC_WR_FLUSH_ERR);
229 static void rds_ib_sub_signaled(struct rds_ib_connection *ic, int nr)
231 if ((atomic_sub_return(nr, &ic->i_signaled_sends) == 0) &&
234 BUG_ON(atomic_read(&ic->i_signaled_sends) < 0);
243 void rds_ib_send_cqe_handler(struct rds_ib_connection *ic, struct ib_wc *wc)
246 struct rds_connection *conn = ic->conn;
261 if (time_after(jiffies, ic->i_ack_queued + HZ / 2))
263 rds_ib_ack_send_complete(ic);
267 oldest = rds_ib_ring_oldest(&ic->i_send_ring);
269 completed = rds_ib_ring_completed(&ic->i_send_ring, wc->wr_id, oldest);
272 send = &ic->i_sends[oldest];
276 rm = rds_ib_send_unmap_op(ic, send, wc->status);
292 oldest = (oldest + 1) % ic->i_send_ring.w_nr;
295 rds_ib_ring_free(&ic->i_send_ring, completed);
296 rds_ib_sub_signaled(ic, nr_sig);
356 int rds_ib_send_grab_credits(struct rds_ib_connection *ic,
363 if (!ic->i_flowctl)
368 oldval = newval = atomic_read(&ic->i_credits);
380 struct rds_connection *conn = ic->i_cm_id->context;
402 if (atomic_cmpxchg(&ic->i_credits, oldval, newval) != oldval)
411 struct rds_ib_connection *ic = conn->c_transport_data;
418 IB_GET_SEND_CREDITS(atomic_read(&ic->i_credits)),
421 atomic_add(IB_SET_SEND_CREDITS(credits), &ic->i_credits);
432 struct rds_ib_connection *ic = conn->c_transport_data;
437 atomic_add(IB_SET_POST_CREDITS(posted), &ic->i_credits);
451 if (IB_GET_POST_CREDITS(atomic_read(&ic->i_credits)) >= 16)
452 set_bit(IB_ACK_REQUESTED, &ic->i_ack_flags);
455 static inline int rds_ib_set_wr_signal_state(struct rds_ib_connection *ic,
464 if (ic->i_unsignaled_wrs-- == 0 || notify) {
465 ic->i_unsignaled_wrs = rds_ib_sysctl_max_unsig_wrs;
488 struct rds_ib_connection *ic = conn->c_transport_data;
489 struct ib_device *dev = ic->i_cm_id->device;
525 work_alloc = rds_ib_ring_alloc(&ic->i_send_ring, i, &pos);
533 if (ic->i_flowctl) {
534 credit_alloc = rds_ib_send_grab_credits(ic, work_alloc, &posted, 0, RDS_MAX_ADV_CREDIT);
537 rds_ib_ring_unalloc(&ic->i_send_ring, work_alloc - credit_alloc);
550 if (!ic->i_data_op) {
556 rdsdebug("ic %p mapping rm %p: %d\n", ic, rm, rm->data.op_count);
559 rds_ib_ring_unalloc(&ic->i_send_ring, work_alloc);
570 ic->i_data_op = &rm->data;
597 rm->m_inc.i_hdr.h_ack = cpu_to_be64(rds_ib_piggyb_ack(ic));
603 if (ic->i_flowctl) {
604 rds_ib_send_grab_credits(ic, 0, &posted, 1, RDS_MAX_ADV_CREDIT - adv_credits);
620 send = &ic->i_sends[pos];
623 scat = &ic->i_data_op->op_sg[rm->data.op_dmasg];
636 send->s_sge[0].addr = ic->i_send_hdrs_dma[pos];
639 send->s_sge[0].lkey = ic->i_pd->local_dma_lkey;
641 ib_dma_sync_single_for_cpu(ic->rds_ibdev->dev,
642 ic->i_send_hdrs_dma[pos],
645 memcpy(ic->i_send_hdrs[pos], &rm->m_inc.i_hdr,
659 send->s_sge[1].lkey = ic->i_pd->local_dma_lkey;
670 rds_ib_set_wr_signal_state(ic, send, false);
675 if (ic->i_flowctl && flow_controlled && i == (work_alloc - 1)) {
676 rds_ib_set_wr_signal_state(ic, send, true);
686 if (ic->i_flowctl && adv_credits) {
687 struct rds_header *hdr = ic->i_send_hdrs[pos];
695 ib_dma_sync_single_for_device(ic->rds_ibdev->dev,
696 ic->i_send_hdrs_dma[pos],
704 pos = (pos + 1) % ic->i_send_ring.w_nr;
705 send = &ic->i_sends[pos];
718 prev->s_op = ic->i_data_op;
721 nr_sig += rds_ib_set_wr_signal_state(ic, prev, true);
722 ic->i_data_op = NULL;
727 rds_ib_ring_unalloc(&ic->i_send_ring, work_alloc - i);
730 if (ic->i_flowctl && i < credit_alloc)
734 atomic_add(nr_sig, &ic->i_signaled_sends);
738 ret = ib_post_send(ic->i_cm_id->qp, &first->s_wr, &failed_wr);
739 rdsdebug("ic %p first %p (wr %p) ret %d wr %p\n", ic,
745 rds_ib_ring_unalloc(&ic->i_send_ring, work_alloc);
746 rds_ib_sub_signaled(ic, nr_sig);
748 ic->i_data_op = prev->s_op;
752 rds_ib_conn_error(ic->conn, "ib_post_send failed\n");
769 struct rds_ib_connection *ic = conn->c_transport_data;
777 work_alloc = rds_ib_ring_alloc(&ic->i_send_ring, 1, &pos);
785 send = &ic->i_sends[pos];
802 nr_sig = rds_ib_set_wr_signal_state(ic, send, op->op_notify);
811 ret = ib_dma_map_sg(ic->i_cm_id->device, op->op_sg, 1, DMA_FROM_DEVICE);
812 rdsdebug("ic %p mapping atomic op %p. mapped %d pg\n", ic, op, ret);
814 rds_ib_ring_unalloc(&ic->i_send_ring, work_alloc);
823 send->s_sge[0].lkey = ic->i_pd->local_dma_lkey;
829 atomic_add(nr_sig, &ic->i_signaled_sends);
832 ret = ib_post_send(ic->i_cm_id->qp, &send->s_atomic_wr.wr, &failed_wr);
833 rdsdebug("ic %p send %p (wr %p) ret %d wr %p\n", ic,
839 rds_ib_ring_unalloc(&ic->i_send_ring, work_alloc);
840 rds_ib_sub_signaled(ic, nr_sig);
855 struct rds_ib_connection *ic = conn->c_transport_data;
863 u32 max_sge = ic->rds_ibdev->max_sge;
879 ib_dma_map_sg(ic->i_cm_id->device, op->op_sg,
883 rdsdebug("ic %p mapping op %p: %d\n", ic, op,
903 work_alloc = rds_ib_ring_alloc(&ic->i_send_ring, i, &pos);
905 rds_ib_ring_unalloc(&ic->i_send_ring, work_alloc);
911 send = &ic->i_sends[pos];
924 nr_sig += rds_ib_set_wr_signal_state(ic, send,
948 send->s_sge[j].lkey = ic->i_pd->local_dma_lkey;
956 rdsdebug("ic %p sent %d remote_addr %llu\n", ic, sent, remote_addr);
969 if (++send == &ic->i_sends[ic->i_send_ring.w_nr])
970 send = ic->i_sends;
980 rds_ib_ring_unalloc(&ic->i_send_ring, work_alloc - i);
985 atomic_add(nr_sig, &ic->i_signaled_sends);
988 ret = ib_post_send(ic->i_cm_id->qp, &first->s_rdma_wr.wr, &failed_wr);
989 rdsdebug("ic %p first %p (wr %p) ret %d wr %p\n", ic,
995 rds_ib_ring_unalloc(&ic->i_send_ring, work_alloc);
996 rds_ib_sub_signaled(ic, nr_sig);
1013 struct rds_ib_connection *ic = conn->c_transport_data;
1017 rds_ib_attempt_ack(ic);