Lines Matching refs:sess
70 static bool __rtrs_srv_change_state(struct rtrs_srv_sess *sess,
76 lockdep_assert_held(&sess->state_lock);
77 old_state = sess->state;
111 sess->state = new_state;
116 static bool rtrs_srv_change_state_get_old(struct rtrs_srv_sess *sess,
122 spin_lock_irq(&sess->state_lock);
123 *old_state = sess->state;
124 changed = __rtrs_srv_change_state(sess, new_state);
125 spin_unlock_irq(&sess->state_lock);
130 static bool rtrs_srv_change_state(struct rtrs_srv_sess *sess,
135 return rtrs_srv_change_state_get_old(sess, new_state, &old_state);
145 static void rtrs_srv_free_ops_ids(struct rtrs_srv_sess *sess)
147 struct rtrs_srv *srv = sess->srv;
150 WARN_ON(atomic_read(&sess->ids_inflight));
151 if (sess->ops_ids) {
153 free_id(sess->ops_ids[i]);
154 kfree(sess->ops_ids);
155 sess->ops_ids = NULL;
165 static int rtrs_srv_alloc_ops_ids(struct rtrs_srv_sess *sess)
167 struct rtrs_srv *srv = sess->srv;
171 sess->ops_ids = kcalloc(srv->queue_depth, sizeof(*sess->ops_ids),
173 if (!sess->ops_ids)
181 sess->ops_ids[i] = id;
183 init_waitqueue_head(&sess->ids_waitq);
184 atomic_set(&sess->ids_inflight, 0);
189 rtrs_srv_free_ops_ids(sess);
193 static inline void rtrs_srv_get_ops_ids(struct rtrs_srv_sess *sess)
195 atomic_inc(&sess->ids_inflight);
198 static inline void rtrs_srv_put_ops_ids(struct rtrs_srv_sess *sess)
200 if (atomic_dec_and_test(&sess->ids_inflight))
201 wake_up(&sess->ids_waitq);
204 static void rtrs_srv_wait_ops_ids(struct rtrs_srv_sess *sess)
206 wait_event(sess->ids_waitq, !atomic_read(&sess->ids_inflight));
213 struct rtrs_sess *s = con->c.sess;
214 struct rtrs_srv_sess *sess = to_srv_sess(s);
219 close_sess(sess);
230 struct rtrs_sess *s = id->con->c.sess;
231 struct rtrs_srv_sess *sess = to_srv_sess(s);
232 dma_addr_t dma_addr = sess->dma_addr[id->msg_id];
234 struct rtrs_srv *srv = sess->srv;
267 plist->lkey = sess->s.dev->ib_pd->local_dma_lkey;
318 srv_mr = &sess->mrs[id->msg_id];
334 list.lkey = sess->s.dev->ib_pd->local_dma_lkey;
338 ib_dma_sync_single_for_device(sess->s.dev->ib_dev,
351 ib_dma_sync_single_for_device(sess->s.dev->ib_dev, dma_addr,
375 struct rtrs_sess *s = con->c.sess;
376 struct rtrs_srv_sess *sess = to_srv_sess(s);
380 struct rtrs_srv *srv = sess->srv;
437 srv_mr = &sess->mrs[id->msg_id];
454 list.lkey = sess->s.dev->ib_pd->local_dma_lkey;
458 ib_dma_sync_single_for_device(sess->s.dev->ib_dev,
479 void close_sess(struct rtrs_srv_sess *sess)
483 if (rtrs_srv_change_state_get_old(sess, RTRS_SRV_CLOSING,
485 queue_work(rtrs_wq, &sess->close_work);
486 WARN_ON(sess->state != RTRS_SRV_CLOSING);
518 struct rtrs_srv_sess *sess;
527 s = con->c.sess;
528 sess = to_srv_sess(s);
532 if (unlikely(sess->state != RTRS_SRV_CONNECTED)) {
534 "Sending I/O response failed, session is disconnected, sess state %s\n",
535 rtrs_srv_state_str(sess->state));
539 struct rtrs_srv_mr *mr = &sess->mrs[id->msg_id];
560 close_sess(sess);
563 rtrs_srv_put_ops_ids(sess);
579 static void unmap_cont_bufs(struct rtrs_srv_sess *sess)
583 for (i = 0; i < sess->mrs_num; i++) {
586 srv_mr = &sess->mrs[i];
587 rtrs_iu_free(srv_mr->iu, sess->s.dev->ib_dev, 1);
589 ib_dma_unmap_sg(sess->s.dev->ib_dev, srv_mr->sgt.sgl,
593 kfree(sess->mrs);
596 static int map_cont_bufs(struct rtrs_srv_sess *sess)
598 struct rtrs_srv *srv = sess->srv;
599 struct rtrs_sess *ss = &sess->s;
616 sess->s.dev->ib_dev->attrs.max_fast_reg_page_list_len;
621 sess->mrs = kcalloc(mrs_num, sizeof(*sess->mrs), GFP_KERNEL);
622 if (!sess->mrs)
625 sess->mrs_num = mrs_num;
628 struct rtrs_srv_mr *srv_mr = &sess->mrs[mri];
647 nr = ib_dma_map_sg(sess->s.dev->ib_dev, sgt->sgl,
653 mr = ib_alloc_mr(sess->s.dev->ib_pd, IB_MR_TYPE_MEM_REG,
669 GFP_KERNEL, sess->s.dev->ib_dev,
679 sess->dma_addr[chunks + i] = sg_dma_address(s);
687 srv_mr = &sess->mrs[mri];
690 rtrs_iu_free(srv_mr->iu, sess->s.dev->ib_dev, 1);
694 ib_dma_unmap_sg(sess->s.dev->ib_dev, sgt->sgl,
699 kfree(sess->mrs);
705 sess->mem_bits = (MAX_IMM_PAYL_BITS - chunk_bits);
712 close_sess(to_srv_sess(c->sess));
715 static void rtrs_srv_init_hb(struct rtrs_srv_sess *sess)
717 rtrs_init_hb(&sess->s, &io_comp_cqe,
724 static void rtrs_srv_start_hb(struct rtrs_srv_sess *sess)
726 rtrs_start_hb(&sess->s);
729 static void rtrs_srv_stop_hb(struct rtrs_srv_sess *sess)
731 rtrs_stop_hb(&sess->s);
737 struct rtrs_sess *s = con->c.sess;
738 struct rtrs_srv_sess *sess = to_srv_sess(s);
742 rtrs_iu_free(iu, sess->s.dev->ib_dev, 1);
747 close_sess(sess);
753 static void rtrs_srv_sess_up(struct rtrs_srv_sess *sess)
755 struct rtrs_srv *srv = sess->srv;
766 sess->established = true;
769 static void rtrs_srv_sess_down(struct rtrs_srv_sess *sess)
771 struct rtrs_srv *srv = sess->srv;
774 if (!sess->established)
777 sess->established = false;
785 static int post_recv_sess(struct rtrs_srv_sess *sess);
790 struct rtrs_sess *s = con->c.sess;
791 struct rtrs_srv_sess *sess = to_srv_sess(s);
799 err = post_recv_sess(sess);
804 rwr = kcalloc(sess->mrs_num, sizeof(*rwr), GFP_KERNEL);
807 strlcpy(sess->s.sessname, msg->sessname, sizeof(sess->s.sessname));
810 tx_sz += sizeof(rsp->desc[0]) * sess->mrs_num;
811 tx_iu = rtrs_iu_alloc(1, tx_sz, GFP_KERNEL, sess->s.dev->ib_dev,
820 rsp->sg_cnt = cpu_to_le16(sess->mrs_num);
822 for (mri = 0; mri < sess->mrs_num; mri++) {
823 struct ib_mr *mr = sess->mrs[mri].mr;
844 err = rtrs_srv_create_sess_files(sess);
847 kobject_get(&sess->kobj);
848 get_device(&sess->srv->dev);
849 rtrs_srv_change_state(sess, RTRS_SRV_CONNECTED);
850 rtrs_srv_start_hb(sess);
858 rtrs_srv_sess_up(sess);
860 ib_dma_sync_single_for_device(sess->s.dev->ib_dev, tx_iu->dma_addr,
868 rtrs_iu_free(tx_iu, sess->s.dev->ib_dev, 1);
879 struct rtrs_sess *s = con->c.sess;
880 struct rtrs_srv_sess *sess = to_srv_sess(s);
900 ib_dma_sync_single_for_cpu(sess->s.dev->ib_dev, iu->dma_addr,
913 rtrs_iu_free(iu, sess->s.dev->ib_dev, 1);
916 close_sess(sess);
922 struct rtrs_sess *s = con->c.sess;
923 struct rtrs_srv_sess *sess = to_srv_sess(s);
928 GFP_KERNEL, sess->s.dev->ib_dev,
936 rtrs_iu_free(rx_iu, sess->s.dev->ib_dev, 1);
956 static int post_recv_sess(struct rtrs_srv_sess *sess)
958 struct rtrs_srv *srv = sess->srv;
959 struct rtrs_sess *s = &sess->s;
963 for (cid = 0; cid < sess->s.con_num; cid++) {
969 err = post_recv_io(to_srv_con(sess->s.con[cid]), q_size);
983 struct rtrs_sess *s = con->c.sess;
984 struct rtrs_srv_sess *sess = to_srv_sess(s);
985 struct rtrs_srv *srv = sess->srv;
993 if (unlikely(sess->state != RTRS_SRV_CONNECTED)) {
995 "Processing read request failed, session is disconnected, sess state %s\n",
996 rtrs_srv_state_str(sess->state));
1004 rtrs_srv_get_ops_ids(sess);
1005 rtrs_srv_update_rdma_stats(sess->stats, off, READ);
1006 id = sess->ops_ids[buf_id];
1032 close_sess(sess);
1034 rtrs_srv_put_ops_ids(sess);
1041 struct rtrs_sess *s = con->c.sess;
1042 struct rtrs_srv_sess *sess = to_srv_sess(s);
1043 struct rtrs_srv *srv = sess->srv;
1051 if (unlikely(sess->state != RTRS_SRV_CONNECTED)) {
1053 "Processing write request failed, session is disconnected, sess state %s\n",
1054 rtrs_srv_state_str(sess->state));
1057 rtrs_srv_get_ops_ids(sess);
1058 rtrs_srv_update_rdma_stats(sess->stats, off, WRITE);
1059 id = sess->ops_ids[buf_id];
1084 close_sess(sess);
1086 rtrs_srv_put_ops_ids(sess);
1092 struct rtrs_sess *s = con->c.sess;
1093 struct rtrs_srv_sess *sess = to_srv_sess(s);
1097 ib_dma_sync_single_for_cpu(sess->s.dev->ib_dev, sess->dma_addr[id],
1119 close_sess(sess);
1127 struct rtrs_sess *s = con->c.sess;
1128 struct rtrs_srv_sess *sess = to_srv_sess(s);
1129 struct rtrs_srv *srv = sess->srv;
1136 close_sess(sess);
1184 struct rtrs_sess *s = con->c.sess;
1185 struct rtrs_srv_sess *sess = to_srv_sess(s);
1186 struct rtrs_srv *srv = sess->srv;
1196 close_sess(sess);
1212 close_sess(sess);
1221 msg_id = imm_payload >> sess->mem_bits;
1222 off = imm_payload & ((1 << sess->mem_bits) - 1);
1227 close_sess(sess);
1231 struct rtrs_srv_mr *mr = &sess->mrs[msg_id];
1239 close_sess(sess);
1248 rtrs_send_hb_ack(&sess->s);
1251 sess->s.hb_missed_cnt = 0;
1281 struct rtrs_srv_sess *sess;
1285 list_for_each_entry(sess, &srv->paths_list, s.entry) {
1286 if (sess->state != RTRS_SRV_CONNECTED)
1288 strlcpy(sessname, sess->s.sessname,
1289 min_t(size_t, sizeof(sess->s.sessname), len));
1309 static int find_next_bit_ring(struct rtrs_srv_sess *sess)
1311 struct ib_device *ib_dev = sess->s.dev->ib_dev;
1314 v = cpumask_next(sess->cur_cq_vector, &cq_affinity_mask);
1320 static int rtrs_srv_get_next_cq_vector(struct rtrs_srv_sess *sess)
1322 sess->cur_cq_vector = find_next_bit_ring(sess);
1324 return sess->cur_cq_vector;
1427 struct rtrs_srv_sess *sess)
1429 list_add_tail(&sess->s.entry, &srv->paths_list);
1434 static void del_path_from_srv(struct rtrs_srv_sess *sess)
1436 struct rtrs_srv *srv = sess->srv;
1442 list_del(&sess->s.entry);
1475 struct rtrs_srv_sess *sess;
1477 list_for_each_entry(sess, &srv->paths_list, s.entry)
1478 if (!sockaddr_cmp((struct sockaddr *)&sess->s.dst_addr,
1480 !sockaddr_cmp((struct sockaddr *)&sess->s.src_addr,
1487 static void free_sess(struct rtrs_srv_sess *sess)
1489 if (sess->kobj.state_in_sysfs) {
1490 kobject_del(&sess->kobj);
1491 kobject_put(&sess->kobj);
1493 kfree(sess->stats);
1494 kfree(sess);
1500 struct rtrs_srv_sess *sess;
1504 sess = container_of(work, typeof(*sess), close_work);
1506 rtrs_srv_destroy_sess_files(sess);
1507 rtrs_srv_stop_hb(sess);
1509 for (i = 0; i < sess->s.con_num; i++) {
1510 if (!sess->s.con[i])
1512 con = to_srv_con(sess->s.con[i]);
1517 rtrs_srv_wait_ops_ids(sess);
1520 rtrs_srv_sess_down(sess);
1522 unmap_cont_bufs(sess);
1523 rtrs_srv_free_ops_ids(sess);
1525 for (i = 0; i < sess->s.con_num; i++) {
1526 if (!sess->s.con[i])
1528 con = to_srv_con(sess->s.con[i]);
1533 rtrs_ib_dev_put(sess->s.dev);
1535 del_path_from_srv(sess);
1536 put_srv(sess->srv);
1537 sess->srv = NULL;
1538 rtrs_srv_change_state(sess, RTRS_SRV_CLOSED);
1540 kfree(sess->dma_addr);
1541 kfree(sess->s.con);
1542 free_sess(sess);
1545 static int rtrs_rdma_do_accept(struct rtrs_srv_sess *sess,
1548 struct rtrs_srv *srv = sess->srv;
1599 struct rtrs_srv_sess *sess;
1601 list_for_each_entry(sess, &srv->paths_list, s.entry) {
1602 if (uuid_equal(&sess->s.uuid, sess_uuid))
1603 return sess;
1609 static int create_con(struct rtrs_srv_sess *sess,
1613 struct rtrs_srv *srv = sess->srv;
1614 struct rtrs_sess *s = &sess->s;
1629 con->c.sess = &sess->s;
1648 wr_limit = sess->s.dev->ib_dev->attrs.max_qp_wr / 3;
1669 cq_vector = rtrs_srv_get_next_cq_vector(sess);
1672 err = rtrs_cq_qp_create(&sess->s, &con->c, 1, cq_vector, cq_size,
1684 WARN_ON(sess->s.con[cid]);
1685 sess->s.con[cid] = &con->c;
1710 struct rtrs_srv_sess *sess;
1722 sess = kzalloc(sizeof(*sess), GFP_KERNEL);
1723 if (!sess)
1726 sess->stats = kzalloc(sizeof(*sess->stats), GFP_KERNEL);
1727 if (!sess->stats)
1730 sess->stats->sess = sess;
1732 sess->dma_addr = kcalloc(srv->queue_depth, sizeof(*sess->dma_addr),
1734 if (!sess->dma_addr)
1737 sess->s.con = kcalloc(con_num, sizeof(*sess->s.con), GFP_KERNEL);
1738 if (!sess->s.con)
1741 sess->state = RTRS_SRV_CONNECTING;
1742 sess->srv = srv;
1743 sess->cur_cq_vector = -1;
1744 sess->s.dst_addr = cm_id->route.addr.dst_addr;
1745 sess->s.src_addr = cm_id->route.addr.src_addr;
1746 sess->s.con_num = con_num;
1747 sess->s.recon_cnt = recon_cnt;
1748 uuid_copy(&sess->s.uuid, uuid);
1749 spin_lock_init(&sess->state_lock);
1750 INIT_WORK(&sess->close_work, rtrs_srv_close_work);
1751 rtrs_srv_init_hb(sess);
1753 sess->s.dev = rtrs_ib_dev_find_or_add(cm_id->device, &dev_pd);
1754 if (!sess->s.dev) {
1758 err = map_cont_bufs(sess);
1762 err = rtrs_srv_alloc_ops_ids(sess);
1766 __add_path_to_srv(srv, sess);
1768 return sess;
1771 unmap_cont_bufs(sess);
1773 rtrs_ib_dev_put(sess->s.dev);
1775 kfree(sess->s.con);
1777 kfree(sess->dma_addr);
1779 kfree(sess->stats);
1781 kfree(sess);
1791 struct rtrs_srv_sess *sess;
1831 sess = __find_sess(srv, &msg->sess_uuid);
1832 if (sess) {
1833 struct rtrs_sess *s = &sess->s;
1838 if (sess->state != RTRS_SRV_CONNECTING) {
1840 rtrs_srv_state_str(sess->state));
1860 sess = __alloc_sess(srv, cm_id, con_num, recon_cnt,
1862 if (IS_ERR(sess)) {
1865 err = PTR_ERR(sess);
1869 err = create_con(sess, cm_id, cid);
1879 err = rtrs_rdma_do_accept(sess, cm_id);
1903 close_sess(sess);
1911 struct rtrs_srv_sess *sess = NULL;
1917 s = c->sess;
1918 sess = to_srv_sess(s);
1937 close_sess(sess);
1942 close_sess(sess);
1945 close_sess(sess);
2155 struct rtrs_srv_sess *sess;
2158 list_for_each_entry(sess, &srv->paths_list, s.entry)
2159 close_sess(sess);