Lines Matching refs:ic
59 struct rds_ib_connection *ic = conn->c_transport_data;
63 ic->i_flowctl = 1;
66 ic->i_flowctl = 0;
85 rds_ib_tune_rnr(struct rds_ib_connection *ic, struct ib_qp_attr *attr)
90 ret = ib_modify_qp(ic->i_cm_id->qp, attr, IB_QP_MIN_RNR_TIMER);
101 struct rds_ib_connection *ic = conn->c_transport_data;
150 ic->i_active_side ? "Active" : "Passive",
154 ic->i_flowctl ? ", flow control" : "");
157 ic->i_sl = ic->i_cm_id->route.path_rec->sl;
159 atomic_set(&ic->i_cq_quiesce, 0);
165 rds_ib_send_init_ring(ic);
166 rds_ib_recv_init_ring(ic);
172 rds_ib_tune_rnr(ic, &qp_attr);
175 err = ib_modify_qp(ic->i_cm_id->qp, &qp_attr, IB_QP_STATE);
180 err = rds_ib_update_ipaddr(ic->rds_ibdev, &conn->c_laddr);
205 struct rds_ib_connection *ic = conn->c_transport_data;
206 struct rds_ib_device *rds_ibdev = ic->rds_ibdev;
229 cpu_to_be64(rds_ib_piggyb_ack(ic));
244 cpu_to_be64(rds_ib_piggyb_ack(ic));
252 if (ic->i_flowctl) {
256 (atomic_read(&ic->i_credits));
262 &ic->i_credits);
284 struct rds_ib_connection *ic = conn->c_transport_data;
290 tasklet_schedule(&ic->i_recv_tasklet);
293 static void poll_scq(struct rds_ib_connection *ic, struct ib_cq *cq,
306 if (wc->wr_id <= ic->i_send_ring.w_nr ||
308 rds_ib_send_cqe_handler(ic, wc);
310 rds_ib_mr_cqe_handler(ic, wc);
318 struct rds_ib_connection *ic = (struct rds_ib_connection *)data;
319 struct rds_connection *conn = ic->conn;
324 if (atomic_read(&ic->i_cq_quiesce))
327 poll_scq(ic, ic->i_send_cq, ic->i_send_wc);
328 ib_req_notify_cq(ic->i_send_cq, IB_CQ_NEXT_COMP);
329 poll_scq(ic, ic->i_send_cq, ic->i_send_wc);
334 rds_send_xmit(&ic->conn->c_path[0]);
337 static void poll_rcq(struct rds_ib_connection *ic, struct ib_cq *cq,
351 rds_ib_recv_cqe_handler(ic, wc, ack_state);
358 struct rds_ib_connection *ic = (struct rds_ib_connection *)data;
359 struct rds_connection *conn = ic->conn;
360 struct rds_ib_device *rds_ibdev = ic->rds_ibdev;
369 if (atomic_read(&ic->i_cq_quiesce))
373 poll_rcq(ic, ic->i_recv_cq, ic->i_recv_wc, &state);
374 ib_req_notify_cq(ic->i_recv_cq, IB_CQ_SOLICITED);
375 poll_rcq(ic, ic->i_recv_cq, ic->i_recv_wc, &state);
378 rds_ib_set_ack(ic, state.ack_next, state.ack_required);
379 if (state.ack_recv_valid && state.ack_recv > ic->i_ack_recv) {
381 ic->i_ack_recv = state.ack_recv;
385 rds_ib_attempt_ack(ic);
391 struct rds_ib_connection *ic = conn->c_transport_data;
393 rdsdebug("conn %p ic %p event %u (%s)\n", conn, ic, event->event,
398 rdma_notify(ic->i_cm_id, IB_EVENT_COMM_EST);
412 struct rds_ib_connection *ic = conn->c_transport_data;
418 tasklet_schedule(&ic->i_send_tasklet);
538 struct rds_ib_connection *ic = conn->c_transport_data;
539 struct ib_device *dev = ic->i_cm_id->device;
565 if (ic->i_send_ring.w_nr != max_wrs)
566 rds_ib_ring_resize(&ic->i_send_ring, max_wrs);
570 if (ic->i_recv_ring.w_nr != max_wrs)
571 rds_ib_ring_resize(&ic->i_recv_ring, max_wrs);
574 ic->i_pd = rds_ibdev->pd;
576 ic->i_scq_vector = ibdev_get_unused_vector(rds_ibdev);
577 cq_attr.cqe = ic->i_send_ring.w_nr + fr_queue_space + 1;
578 cq_attr.comp_vector = ic->i_scq_vector;
579 ic->i_send_cq = ib_create_cq(dev, rds_ib_cq_comp_handler_send,
582 if (IS_ERR(ic->i_send_cq)) {
583 ret = PTR_ERR(ic->i_send_cq);
584 ic->i_send_cq = NULL;
585 ibdev_put_vector(rds_ibdev, ic->i_scq_vector);
590 ic->i_rcq_vector = ibdev_get_unused_vector(rds_ibdev);
591 cq_attr.cqe = ic->i_recv_ring.w_nr;
592 cq_attr.comp_vector = ic->i_rcq_vector;
593 ic->i_recv_cq = ib_create_cq(dev, rds_ib_cq_comp_handler_recv,
596 if (IS_ERR(ic->i_recv_cq)) {
597 ret = PTR_ERR(ic->i_recv_cq);
598 ic->i_recv_cq = NULL;
599 ibdev_put_vector(rds_ibdev, ic->i_rcq_vector);
604 ret = ib_req_notify_cq(ic->i_send_cq, IB_CQ_NEXT_COMP);
610 ret = ib_req_notify_cq(ic->i_recv_cq, IB_CQ_SOLICITED);
621 attr.cap.max_send_wr = ic->i_send_ring.w_nr + fr_queue_space + 1;
622 attr.cap.max_recv_wr = ic->i_recv_ring.w_nr + 1;
627 attr.send_cq = ic->i_send_cq;
628 attr.recv_cq = ic->i_recv_cq;
634 ret = rdma_create_qp(ic->i_cm_id, ic->i_pd, &attr);
640 ic->i_send_hdrs = rds_dma_hdrs_alloc(rds_ibdev, &ic->i_send_hdrs_dma,
641 ic->i_send_ring.w_nr,
643 if (!ic->i_send_hdrs) {
649 ic->i_recv_hdrs = rds_dma_hdrs_alloc(rds_ibdev, &ic->i_recv_hdrs_dma,
650 ic->i_recv_ring.w_nr,
652 if (!ic->i_recv_hdrs) {
658 ic->i_ack = rds_dma_hdr_alloc(rds_ibdev->dev, &ic->i_ack_dma,
660 if (!ic->i_ack) {
666 ic->i_sends = vzalloc_node(array_size(sizeof(struct rds_ib_send_work),
667 ic->i_send_ring.w_nr),
669 if (!ic->i_sends) {
675 ic->i_recvs = vzalloc_node(array_size(sizeof(struct rds_ib_recv_work),
676 ic->i_recv_ring.w_nr),
678 if (!ic->i_recvs) {
684 rds_ib_recv_init_ack(ic);
686 rdsdebug("conn %p pd %p cq %p %p\n", conn, ic->i_pd,
687 ic->i_send_cq, ic->i_recv_cq);
692 vfree(ic->i_sends);
695 rds_dma_hdr_free(rds_ibdev->dev, ic->i_ack, ic->i_ack_dma,
697 ic->i_ack = NULL;
700 rds_dma_hdrs_free(rds_ibdev, ic->i_recv_hdrs, ic->i_recv_hdrs_dma,
701 ic->i_recv_ring.w_nr, DMA_FROM_DEVICE);
702 ic->i_recv_hdrs = NULL;
703 ic->i_recv_hdrs_dma = NULL;
706 rds_dma_hdrs_free(rds_ibdev, ic->i_send_hdrs, ic->i_send_hdrs_dma,
707 ic->i_send_ring.w_nr, DMA_TO_DEVICE);
708 ic->i_send_hdrs = NULL;
709 ic->i_send_hdrs_dma = NULL;
712 rdma_destroy_qp(ic->i_cm_id);
714 ib_destroy_cq(ic->i_recv_cq);
715 ic->i_recv_cq = NULL;
717 ib_destroy_cq(ic->i_send_cq);
718 ic->i_send_cq = NULL;
823 struct rds_ib_connection *ic = NULL;
919 ic = conn->c_transport_data;
931 BUG_ON(ic->i_cm_id);
933 ic->i_cm_id = cm_id;
967 struct rds_ib_connection *ic = conn->c_transport_data;
975 ic->i_flowctl = rds_ib_sysctl_flow_control; /* advertise flow control */
996 if (ic->i_cm_id == cm_id)
999 ic->i_active_side = true;
1008 struct rds_ib_connection *ic;
1011 ic = conn->c_transport_data;
1021 ic->i_cm_id = rdma_create_id(&init_net, handler, conn,
1023 if (IS_ERR(ic->i_cm_id)) {
1024 ret = PTR_ERR(ic->i_cm_id);
1025 ic->i_cm_id = NULL;
1030 rdsdebug("created cm id %p for conn %p\n", ic->i_cm_id, conn);
1060 ret = rdma_resolve_addr(ic->i_cm_id, (struct sockaddr *)&src,
1064 rdsdebug("addr resolve failed for cm id %p: %d\n", ic->i_cm_id,
1066 rdma_destroy_id(ic->i_cm_id);
1067 ic->i_cm_id = NULL;
1082 struct rds_ib_connection *ic = conn->c_transport_data;
1085 rdsdebug("cm %p pd %p cq %p %p qp %p\n", ic->i_cm_id,
1086 ic->i_pd, ic->i_send_cq, ic->i_recv_cq,
1087 ic->i_cm_id ? ic->i_cm_id->qp : NULL);
1089 if (ic->i_cm_id) {
1090 rdsdebug("disconnecting cm %p\n", ic->i_cm_id);
1091 err = rdma_disconnect(ic->i_cm_id);
1097 ic->i_cm_id, err);
1115 rds_ib_ring_empty(&ic->i_recv_ring) &&
1116 (atomic_read(&ic->i_signaled_sends) == 0) &&
1117 (atomic_read(&ic->i_fastreg_inuse_count) == 0) &&
1118 (atomic_read(&ic->i_fastreg_wrs) == RDS_IB_DEFAULT_FR_WR));
1119 tasklet_kill(&ic->i_send_tasklet);
1120 tasklet_kill(&ic->i_recv_tasklet);
1122 atomic_set(&ic->i_cq_quiesce, 1);
1125 if (ic->i_cm_id->qp)
1126 rdma_destroy_qp(ic->i_cm_id);
1127 if (ic->i_send_cq) {
1128 if (ic->rds_ibdev)
1129 ibdev_put_vector(ic->rds_ibdev, ic->i_scq_vector);
1130 ib_destroy_cq(ic->i_send_cq);
1133 if (ic->i_recv_cq) {
1134 if (ic->rds_ibdev)
1135 ibdev_put_vector(ic->rds_ibdev, ic->i_rcq_vector);
1136 ib_destroy_cq(ic->i_recv_cq);
1139 if (ic->rds_ibdev) {
1141 if (ic->i_send_hdrs) {
1142 rds_dma_hdrs_free(ic->rds_ibdev,
1143 ic->i_send_hdrs,
1144 ic->i_send_hdrs_dma,
1145 ic->i_send_ring.w_nr,
1147 ic->i_send_hdrs = NULL;
1148 ic->i_send_hdrs_dma = NULL;
1151 if (ic->i_recv_hdrs) {
1152 rds_dma_hdrs_free(ic->rds_ibdev,
1153 ic->i_recv_hdrs,
1154 ic->i_recv_hdrs_dma,
1155 ic->i_recv_ring.w_nr,
1157 ic->i_recv_hdrs = NULL;
1158 ic->i_recv_hdrs_dma = NULL;
1161 if (ic->i_ack) {
1162 rds_dma_hdr_free(ic->rds_ibdev->dev, ic->i_ack,
1163 ic->i_ack_dma, DMA_TO_DEVICE);
1164 ic->i_ack = NULL;
1167 WARN_ON(ic->i_send_hdrs);
1168 WARN_ON(ic->i_send_hdrs_dma);
1169 WARN_ON(ic->i_recv_hdrs);
1170 WARN_ON(ic->i_recv_hdrs_dma);
1171 WARN_ON(ic->i_ack);
1174 if (ic->i_sends)
1175 rds_ib_send_clear_ring(ic);
1176 if (ic->i_recvs)
1177 rds_ib_recv_clear_ring(ic);
1179 rdma_destroy_id(ic->i_cm_id);
1184 if (ic->rds_ibdev)
1185 rds_ib_remove_conn(ic->rds_ibdev, conn);
1187 ic->i_cm_id = NULL;
1188 ic->i_pd = NULL;
1189 ic->i_send_cq = NULL;
1190 ic->i_recv_cq = NULL;
1192 BUG_ON(ic->rds_ibdev);
1195 if (ic->i_data_op) {
1198 rm = container_of(ic->i_data_op, struct rds_message, data);
1200 ic->i_data_op = NULL;
1204 clear_bit(IB_ACK_IN_FLIGHT, &ic->i_ack_flags);
1206 atomic64_set(&ic->i_ack_next, 0);
1208 ic->i_ack_next = 0;
1210 ic->i_ack_recv = 0;
1213 ic->i_flowctl = 0;
1214 atomic_set(&ic->i_credits, 0);
1217 rds_ib_ring_init(&ic->i_send_ring, ic->i_send_ring.w_nr);
1218 rds_ib_ring_init(&ic->i_recv_ring, ic->i_recv_ring.w_nr);
1220 if (ic->i_ibinc) {
1221 rds_inc_put(&ic->i_ibinc->ii_inc);
1222 ic->i_ibinc = NULL;
1225 vfree(ic->i_sends);
1226 ic->i_sends = NULL;
1227 vfree(ic->i_recvs);
1228 ic->i_recvs = NULL;
1229 ic->i_active_side = false;
1234 struct rds_ib_connection *ic;
1239 ic = kzalloc(sizeof(struct rds_ib_connection), gfp);
1240 if (!ic)
1243 ret = rds_ib_recv_alloc_caches(ic, gfp);
1245 kfree(ic);
1249 INIT_LIST_HEAD(&ic->ib_node);
1250 tasklet_init(&ic->i_send_tasklet, rds_ib_tasklet_fn_send,
1251 (unsigned long)ic);
1252 tasklet_init(&ic->i_recv_tasklet, rds_ib_tasklet_fn_recv,
1253 (unsigned long)ic);
1254 mutex_init(&ic->i_recv_mutex);
1256 spin_lock_init(&ic->i_ack_lock);
1258 atomic_set(&ic->i_signaled_sends, 0);
1259 atomic_set(&ic->i_fastreg_wrs, RDS_IB_DEFAULT_FR_WR);
1265 rds_ib_ring_init(&ic->i_send_ring, 0);
1266 rds_ib_ring_init(&ic->i_recv_ring, 0);
1268 ic->conn = conn;
1269 conn->c_transport_data = ic;
1272 list_add_tail(&ic->ib_node, &ib_nodev_conns);
1276 rdsdebug("conn %p conn ic %p\n", conn, conn->c_transport_data);
1285 struct rds_ib_connection *ic = arg;
1288 rdsdebug("ic %p\n", ic);
1295 lock_ptr = ic->rds_ibdev ? &ic->rds_ibdev->spinlock : &ib_nodev_conns_lock;
1298 list_del(&ic->ib_node);
1301 rds_ib_recv_free_caches(ic);
1303 kfree(ic);