1// SPDX-License-Identifier: GPL-2.0 2/* 3 * Shared Memory Communications over RDMA (SMC-R) and RoCE 4 * 5 * Work Requests exploiting Infiniband API 6 * 7 * Work requests (WR) of type ib_post_send or ib_post_recv respectively 8 * are submitted to either RC SQ or RC RQ respectively 9 * (reliably connected send/receive queue) 10 * and become work queue entries (WQEs). 11 * While an SQ WR/WQE is pending, we track it until transmission completion. 12 * Through a send or receive completion queue (CQ) respectively, 13 * we get completion queue entries (CQEs) [aka work completions (WCs)]. 14 * Since the CQ callback is called from IRQ context, we split work by using 15 * bottom halves implemented by tasklets. 16 * 17 * SMC uses this to exchange LLC (link layer control) 18 * and CDC (connection data control) messages. 19 * 20 * Copyright IBM Corp. 2016 21 * 22 * Author(s): Steffen Maier <maier@linux.vnet.ibm.com> 23 */ 24 25#include <linux/atomic.h> 26#include <linux/hashtable.h> 27#include <linux/wait.h> 28#include <rdma/ib_verbs.h> 29#include <asm/div64.h> 30 31#include "smc.h" 32#include "smc_wr.h" 33 34#define SMC_WR_MAX_POLL_CQE 10 /* max. # of compl. queue elements in 1 poll */ 35 36#define SMC_WR_RX_HASH_BITS 4 37static DEFINE_HASHTABLE(smc_wr_rx_hash, SMC_WR_RX_HASH_BITS); 38static DEFINE_SPINLOCK(smc_wr_rx_hash_lock); 39 40struct smc_wr_tx_pend { /* control data for a pending send request */ 41 u64 wr_id; /* work request id sent */ 42 smc_wr_tx_handler handler; 43 enum ib_wc_status wc_status; /* CQE status */ 44 struct smc_link *link; 45 u32 idx; 46 struct smc_wr_tx_pend_priv priv; 47 u8 compl_requested; 48}; 49 50/******************************** send queue *********************************/ 51 52/*------------------------------- completion --------------------------------*/ 53 54/* returns true if at least one tx work request is pending on the given link */ 55static inline bool smc_wr_is_tx_pend(struct smc_link *link) 56{ 57 if (find_first_bit(link->wr_tx_mask, link->wr_tx_cnt) != 58 link->wr_tx_cnt) { 59 return true; 60 } 61 return false; 62} 63 64/* wait till all pending tx work requests on the given link are completed */ 65void smc_wr_tx_wait_no_pending_sends(struct smc_link *link) 66{ 67 wait_event(link->wr_tx_wait, !smc_wr_is_tx_pend(link)); 68} 69 70static inline int smc_wr_tx_find_pending_index(struct smc_link *link, u64 wr_id) 71{ 72 u32 i; 73 74 for (i = 0; i < link->wr_tx_cnt; i++) { 75 if (link->wr_tx_pends[i].wr_id == wr_id) 76 return i; 77 } 78 return link->wr_tx_cnt; 79} 80 81static inline void smc_wr_tx_process_cqe(struct ib_wc *wc) 82{ 83 struct smc_wr_tx_pend pnd_snd; 84 struct smc_link *link; 85 u32 pnd_snd_idx; 86 87 link = wc->qp->qp_context; 88 89 if (wc->opcode == IB_WC_REG_MR) { 90 if (wc->status) 91 link->wr_reg_state = FAILED; 92 else 93 link->wr_reg_state = CONFIRMED; 94 smc_wr_wakeup_reg_wait(link); 95 return; 96 } 97 98 pnd_snd_idx = smc_wr_tx_find_pending_index(link, wc->wr_id); 99 if (pnd_snd_idx == link->wr_tx_cnt) 100 return; 101 link->wr_tx_pends[pnd_snd_idx].wc_status = wc->status; 102 if (link->wr_tx_pends[pnd_snd_idx].compl_requested) 103 complete(&link->wr_tx_compl[pnd_snd_idx]); 104 memcpy(&pnd_snd, &link->wr_tx_pends[pnd_snd_idx], sizeof(pnd_snd)); 105 /* clear the full struct smc_wr_tx_pend including .priv */ 106 memset(&link->wr_tx_pends[pnd_snd_idx], 0, 107 sizeof(link->wr_tx_pends[pnd_snd_idx])); 108 memset(&link->wr_tx_bufs[pnd_snd_idx], 0, 109 sizeof(link->wr_tx_bufs[pnd_snd_idx])); 110 if (!test_and_clear_bit(pnd_snd_idx, link->wr_tx_mask)) 111 return; 112 if (wc->status) { 113 /* terminate link */ 114 smcr_link_down_cond_sched(link); 115 } 116 if (pnd_snd.handler) 117 pnd_snd.handler(&pnd_snd.priv, link, wc->status); 118 wake_up(&link->wr_tx_wait); 119} 120 121static void smc_wr_tx_tasklet_fn(unsigned long data) 122{ 123 struct smc_ib_device *dev = (struct smc_ib_device *)data; 124 struct ib_wc wc[SMC_WR_MAX_POLL_CQE]; 125 int i = 0, rc; 126 int polled = 0; 127 128again: 129 polled++; 130 do { 131 memset(&wc, 0, sizeof(wc)); 132 rc = ib_poll_cq(dev->roce_cq_send, SMC_WR_MAX_POLL_CQE, wc); 133 if (polled == 1) { 134 ib_req_notify_cq(dev->roce_cq_send, 135 IB_CQ_NEXT_COMP | 136 IB_CQ_REPORT_MISSED_EVENTS); 137 } 138 if (!rc) 139 break; 140 for (i = 0; i < rc; i++) 141 smc_wr_tx_process_cqe(&wc[i]); 142 } while (rc > 0); 143 if (polled == 1) 144 goto again; 145} 146 147void smc_wr_tx_cq_handler(struct ib_cq *ib_cq, void *cq_context) 148{ 149 struct smc_ib_device *dev = (struct smc_ib_device *)cq_context; 150 151 tasklet_schedule(&dev->send_tasklet); 152} 153 154/*---------------------------- request submission ---------------------------*/ 155 156static inline int smc_wr_tx_get_free_slot_index(struct smc_link *link, u32 *idx) 157{ 158 *idx = link->wr_tx_cnt; 159 if (!smc_link_sendable(link)) 160 return -ENOLINK; 161 for_each_clear_bit(*idx, link->wr_tx_mask, link->wr_tx_cnt) { 162 if (!test_and_set_bit(*idx, link->wr_tx_mask)) 163 return 0; 164 } 165 *idx = link->wr_tx_cnt; 166 return -EBUSY; 167} 168 169/** 170 * smc_wr_tx_get_free_slot() - returns buffer for message assembly, 171 * and sets info for pending transmit tracking 172 * @link: Pointer to smc_link used to later send the message. 173 * @handler: Send completion handler function pointer. 174 * @wr_buf: Out value returns pointer to message buffer. 175 * @wr_rdma_buf: Out value returns pointer to rdma work request. 176 * @wr_pend_priv: Out value returns pointer serving as handler context. 177 * 178 * Return: 0 on success, or -errno on error. 179 */ 180int smc_wr_tx_get_free_slot(struct smc_link *link, 181 smc_wr_tx_handler handler, 182 struct smc_wr_buf **wr_buf, 183 struct smc_rdma_wr **wr_rdma_buf, 184 struct smc_wr_tx_pend_priv **wr_pend_priv) 185{ 186 struct smc_link_group *lgr = smc_get_lgr(link); 187 struct smc_wr_tx_pend *wr_pend; 188 u32 idx = link->wr_tx_cnt; 189 struct ib_send_wr *wr_ib; 190 u64 wr_id; 191 int rc; 192 193 *wr_buf = NULL; 194 *wr_pend_priv = NULL; 195 if (in_softirq() || lgr->terminating) { 196 rc = smc_wr_tx_get_free_slot_index(link, &idx); 197 if (rc) 198 return rc; 199 } else { 200 rc = wait_event_interruptible_timeout( 201 link->wr_tx_wait, 202 !smc_link_sendable(link) || 203 lgr->terminating || 204 (smc_wr_tx_get_free_slot_index(link, &idx) != -EBUSY), 205 SMC_WR_TX_WAIT_FREE_SLOT_TIME); 206 if (!rc) { 207 /* timeout - terminate link */ 208 smcr_link_down_cond_sched(link); 209 return -EPIPE; 210 } 211 if (idx == link->wr_tx_cnt) 212 return -EPIPE; 213 } 214 wr_id = smc_wr_tx_get_next_wr_id(link); 215 wr_pend = &link->wr_tx_pends[idx]; 216 wr_pend->wr_id = wr_id; 217 wr_pend->handler = handler; 218 wr_pend->link = link; 219 wr_pend->idx = idx; 220 wr_ib = &link->wr_tx_ibs[idx]; 221 wr_ib->wr_id = wr_id; 222 *wr_buf = &link->wr_tx_bufs[idx]; 223 if (wr_rdma_buf) 224 *wr_rdma_buf = &link->wr_tx_rdmas[idx]; 225 *wr_pend_priv = &wr_pend->priv; 226 return 0; 227} 228 229int smc_wr_tx_put_slot(struct smc_link *link, 230 struct smc_wr_tx_pend_priv *wr_pend_priv) 231{ 232 struct smc_wr_tx_pend *pend; 233 234 pend = container_of(wr_pend_priv, struct smc_wr_tx_pend, priv); 235 if (pend->idx < link->wr_tx_cnt) { 236 u32 idx = pend->idx; 237 238 /* clear the full struct smc_wr_tx_pend including .priv */ 239 memset(&link->wr_tx_pends[idx], 0, 240 sizeof(link->wr_tx_pends[idx])); 241 memset(&link->wr_tx_bufs[idx], 0, 242 sizeof(link->wr_tx_bufs[idx])); 243 test_and_clear_bit(idx, link->wr_tx_mask); 244 wake_up(&link->wr_tx_wait); 245 return 1; 246 } 247 248 return 0; 249} 250 251/* Send prepared WR slot via ib_post_send. 252 * @priv: pointer to smc_wr_tx_pend_priv identifying prepared message buffer 253 */ 254int smc_wr_tx_send(struct smc_link *link, struct smc_wr_tx_pend_priv *priv) 255{ 256 struct smc_wr_tx_pend *pend; 257 int rc; 258 259 ib_req_notify_cq(link->smcibdev->roce_cq_send, 260 IB_CQ_NEXT_COMP | IB_CQ_REPORT_MISSED_EVENTS); 261 pend = container_of(priv, struct smc_wr_tx_pend, priv); 262 rc = ib_post_send(link->roce_qp, &link->wr_tx_ibs[pend->idx], NULL); 263 if (rc) { 264 smc_wr_tx_put_slot(link, priv); 265 smcr_link_down_cond_sched(link); 266 } 267 return rc; 268} 269 270/* Send prepared WR slot via ib_post_send and wait for send completion 271 * notification. 272 * @priv: pointer to smc_wr_tx_pend_priv identifying prepared message buffer 273 */ 274int smc_wr_tx_send_wait(struct smc_link *link, struct smc_wr_tx_pend_priv *priv, 275 unsigned long timeout) 276{ 277 struct smc_wr_tx_pend *pend; 278 u32 pnd_idx; 279 int rc; 280 281 pend = container_of(priv, struct smc_wr_tx_pend, priv); 282 pend->compl_requested = 1; 283 pnd_idx = pend->idx; 284 init_completion(&link->wr_tx_compl[pnd_idx]); 285 286 rc = smc_wr_tx_send(link, priv); 287 if (rc) 288 return rc; 289 /* wait for completion by smc_wr_tx_process_cqe() */ 290 rc = wait_for_completion_interruptible_timeout( 291 &link->wr_tx_compl[pnd_idx], timeout); 292 if (rc <= 0) 293 rc = -ENODATA; 294 if (rc > 0) 295 rc = 0; 296 return rc; 297} 298 299/* Register a memory region and wait for result. */ 300int smc_wr_reg_send(struct smc_link *link, struct ib_mr *mr) 301{ 302 int rc; 303 304 ib_req_notify_cq(link->smcibdev->roce_cq_send, 305 IB_CQ_NEXT_COMP | IB_CQ_REPORT_MISSED_EVENTS); 306 link->wr_reg_state = POSTED; 307 link->wr_reg.wr.wr_id = (u64)(uintptr_t)mr; 308 link->wr_reg.mr = mr; 309 link->wr_reg.key = mr->rkey; 310 rc = ib_post_send(link->roce_qp, &link->wr_reg.wr, NULL); 311 if (rc) 312 return rc; 313 314 atomic_inc(&link->wr_reg_refcnt); 315 rc = wait_event_interruptible_timeout(link->wr_reg_wait, 316 (link->wr_reg_state != POSTED), 317 SMC_WR_REG_MR_WAIT_TIME); 318 if (atomic_dec_and_test(&link->wr_reg_refcnt)) 319 wake_up_all(&link->wr_reg_wait); 320 if (!rc) { 321 /* timeout - terminate link */ 322 smcr_link_down_cond_sched(link); 323 return -EPIPE; 324 } 325 if (rc == -ERESTARTSYS) 326 return -EINTR; 327 switch (link->wr_reg_state) { 328 case CONFIRMED: 329 rc = 0; 330 break; 331 case FAILED: 332 rc = -EIO; 333 break; 334 case POSTED: 335 rc = -EPIPE; 336 break; 337 } 338 return rc; 339} 340 341/****************************** receive queue ********************************/ 342 343int smc_wr_rx_register_handler(struct smc_wr_rx_handler *handler) 344{ 345 struct smc_wr_rx_handler *h_iter; 346 int rc = 0; 347 348 spin_lock(&smc_wr_rx_hash_lock); 349 hash_for_each_possible(smc_wr_rx_hash, h_iter, list, handler->type) { 350 if (h_iter->type == handler->type) { 351 rc = -EEXIST; 352 goto out_unlock; 353 } 354 } 355 hash_add(smc_wr_rx_hash, &handler->list, handler->type); 356out_unlock: 357 spin_unlock(&smc_wr_rx_hash_lock); 358 return rc; 359} 360 361/* Demultiplex a received work request based on the message type to its handler. 362 * Relies on smc_wr_rx_hash having been completely filled before any IB WRs, 363 * and not being modified any more afterwards so we don't need to lock it. 364 */ 365static inline void smc_wr_rx_demultiplex(struct ib_wc *wc) 366{ 367 struct smc_link *link = (struct smc_link *)wc->qp->qp_context; 368 struct smc_wr_rx_handler *handler; 369 struct smc_wr_rx_hdr *wr_rx; 370 u64 temp_wr_id; 371 u32 index; 372 373 if (wc->byte_len < sizeof(*wr_rx)) 374 return; /* short message */ 375 temp_wr_id = wc->wr_id; 376 index = do_div(temp_wr_id, link->wr_rx_cnt); 377 wr_rx = (struct smc_wr_rx_hdr *)&link->wr_rx_bufs[index]; 378 hash_for_each_possible(smc_wr_rx_hash, handler, list, wr_rx->type) { 379 if (handler->type == wr_rx->type) 380 handler->handler(wc, wr_rx); 381 } 382} 383 384static inline void smc_wr_rx_process_cqes(struct ib_wc wc[], int num) 385{ 386 struct smc_link *link; 387 int i; 388 389 for (i = 0; i < num; i++) { 390 link = wc[i].qp->qp_context; 391 if (wc[i].status == IB_WC_SUCCESS) { 392 link->wr_rx_tstamp = jiffies; 393 smc_wr_rx_demultiplex(&wc[i]); 394 smc_wr_rx_post(link); /* refill WR RX */ 395 } else { 396 /* handle status errors */ 397 switch (wc[i].status) { 398 case IB_WC_RETRY_EXC_ERR: 399 case IB_WC_RNR_RETRY_EXC_ERR: 400 case IB_WC_WR_FLUSH_ERR: 401 smcr_link_down_cond_sched(link); 402 break; 403 default: 404 smc_wr_rx_post(link); /* refill WR RX */ 405 break; 406 } 407 } 408 } 409} 410 411static void smc_wr_rx_tasklet_fn(unsigned long data) 412{ 413 struct smc_ib_device *dev = (struct smc_ib_device *)data; 414 struct ib_wc wc[SMC_WR_MAX_POLL_CQE]; 415 int polled = 0; 416 int rc; 417 418again: 419 polled++; 420 do { 421 memset(&wc, 0, sizeof(wc)); 422 rc = ib_poll_cq(dev->roce_cq_recv, SMC_WR_MAX_POLL_CQE, wc); 423 if (polled == 1) { 424 ib_req_notify_cq(dev->roce_cq_recv, 425 IB_CQ_SOLICITED_MASK 426 | IB_CQ_REPORT_MISSED_EVENTS); 427 } 428 if (!rc) 429 break; 430 smc_wr_rx_process_cqes(&wc[0], rc); 431 } while (rc > 0); 432 if (polled == 1) 433 goto again; 434} 435 436void smc_wr_rx_cq_handler(struct ib_cq *ib_cq, void *cq_context) 437{ 438 struct smc_ib_device *dev = (struct smc_ib_device *)cq_context; 439 440 tasklet_schedule(&dev->recv_tasklet); 441} 442 443int smc_wr_rx_post_init(struct smc_link *link) 444{ 445 u32 i; 446 int rc = 0; 447 448 for (i = 0; i < link->wr_rx_cnt; i++) 449 rc = smc_wr_rx_post(link); 450 return rc; 451} 452 453/***************************** init, exit, misc ******************************/ 454 455void smc_wr_remember_qp_attr(struct smc_link *lnk) 456{ 457 struct ib_qp_attr *attr = &lnk->qp_attr; 458 struct ib_qp_init_attr init_attr; 459 460 memset(attr, 0, sizeof(*attr)); 461 memset(&init_attr, 0, sizeof(init_attr)); 462 ib_query_qp(lnk->roce_qp, attr, 463 IB_QP_STATE | 464 IB_QP_CUR_STATE | 465 IB_QP_PKEY_INDEX | 466 IB_QP_PORT | 467 IB_QP_QKEY | 468 IB_QP_AV | 469 IB_QP_PATH_MTU | 470 IB_QP_TIMEOUT | 471 IB_QP_RETRY_CNT | 472 IB_QP_RNR_RETRY | 473 IB_QP_RQ_PSN | 474 IB_QP_ALT_PATH | 475 IB_QP_MIN_RNR_TIMER | 476 IB_QP_SQ_PSN | 477 IB_QP_PATH_MIG_STATE | 478 IB_QP_CAP | 479 IB_QP_DEST_QPN, 480 &init_attr); 481 482 lnk->wr_tx_cnt = min_t(size_t, SMC_WR_BUF_CNT, 483 lnk->qp_attr.cap.max_send_wr); 484 lnk->wr_rx_cnt = min_t(size_t, SMC_WR_BUF_CNT * 3, 485 lnk->qp_attr.cap.max_recv_wr); 486} 487 488static void smc_wr_init_sge(struct smc_link *lnk) 489{ 490 u32 i; 491 492 for (i = 0; i < lnk->wr_tx_cnt; i++) { 493 lnk->wr_tx_sges[i].addr = 494 lnk->wr_tx_dma_addr + i * SMC_WR_BUF_SIZE; 495 lnk->wr_tx_sges[i].length = SMC_WR_TX_SIZE; 496 lnk->wr_tx_sges[i].lkey = lnk->roce_pd->local_dma_lkey; 497 lnk->wr_tx_rdma_sges[i].tx_rdma_sge[0].wr_tx_rdma_sge[0].lkey = 498 lnk->roce_pd->local_dma_lkey; 499 lnk->wr_tx_rdma_sges[i].tx_rdma_sge[0].wr_tx_rdma_sge[1].lkey = 500 lnk->roce_pd->local_dma_lkey; 501 lnk->wr_tx_rdma_sges[i].tx_rdma_sge[1].wr_tx_rdma_sge[0].lkey = 502 lnk->roce_pd->local_dma_lkey; 503 lnk->wr_tx_rdma_sges[i].tx_rdma_sge[1].wr_tx_rdma_sge[1].lkey = 504 lnk->roce_pd->local_dma_lkey; 505 lnk->wr_tx_ibs[i].next = NULL; 506 lnk->wr_tx_ibs[i].sg_list = &lnk->wr_tx_sges[i]; 507 lnk->wr_tx_ibs[i].num_sge = 1; 508 lnk->wr_tx_ibs[i].opcode = IB_WR_SEND; 509 lnk->wr_tx_ibs[i].send_flags = 510 IB_SEND_SIGNALED | IB_SEND_SOLICITED; 511 lnk->wr_tx_rdmas[i].wr_tx_rdma[0].wr.opcode = IB_WR_RDMA_WRITE; 512 lnk->wr_tx_rdmas[i].wr_tx_rdma[1].wr.opcode = IB_WR_RDMA_WRITE; 513 lnk->wr_tx_rdmas[i].wr_tx_rdma[0].wr.sg_list = 514 lnk->wr_tx_rdma_sges[i].tx_rdma_sge[0].wr_tx_rdma_sge; 515 lnk->wr_tx_rdmas[i].wr_tx_rdma[1].wr.sg_list = 516 lnk->wr_tx_rdma_sges[i].tx_rdma_sge[1].wr_tx_rdma_sge; 517 } 518 for (i = 0; i < lnk->wr_rx_cnt; i++) { 519 lnk->wr_rx_sges[i].addr = 520 lnk->wr_rx_dma_addr + i * SMC_WR_BUF_SIZE; 521 lnk->wr_rx_sges[i].length = SMC_WR_BUF_SIZE; 522 lnk->wr_rx_sges[i].lkey = lnk->roce_pd->local_dma_lkey; 523 lnk->wr_rx_ibs[i].next = NULL; 524 lnk->wr_rx_ibs[i].sg_list = &lnk->wr_rx_sges[i]; 525 lnk->wr_rx_ibs[i].num_sge = 1; 526 } 527 lnk->wr_reg.wr.next = NULL; 528 lnk->wr_reg.wr.num_sge = 0; 529 lnk->wr_reg.wr.send_flags = IB_SEND_SIGNALED; 530 lnk->wr_reg.wr.opcode = IB_WR_REG_MR; 531 lnk->wr_reg.access = IB_ACCESS_LOCAL_WRITE | IB_ACCESS_REMOTE_WRITE; 532} 533 534void smc_wr_free_link(struct smc_link *lnk) 535{ 536 struct ib_device *ibdev; 537 538 if (!lnk->smcibdev) 539 return; 540 ibdev = lnk->smcibdev->ibdev; 541 542 smc_wr_wakeup_reg_wait(lnk); 543 smc_wr_wakeup_tx_wait(lnk); 544 545 smc_wr_tx_wait_no_pending_sends(lnk); 546 wait_event(lnk->wr_reg_wait, (!atomic_read(&lnk->wr_reg_refcnt))); 547 wait_event(lnk->wr_tx_wait, (!atomic_read(&lnk->wr_tx_refcnt))); 548 549 if (lnk->wr_rx_dma_addr) { 550 ib_dma_unmap_single(ibdev, lnk->wr_rx_dma_addr, 551 SMC_WR_BUF_SIZE * lnk->wr_rx_cnt, 552 DMA_FROM_DEVICE); 553 lnk->wr_rx_dma_addr = 0; 554 } 555 if (lnk->wr_tx_dma_addr) { 556 ib_dma_unmap_single(ibdev, lnk->wr_tx_dma_addr, 557 SMC_WR_BUF_SIZE * lnk->wr_tx_cnt, 558 DMA_TO_DEVICE); 559 lnk->wr_tx_dma_addr = 0; 560 } 561} 562 563void smc_wr_free_link_mem(struct smc_link *lnk) 564{ 565 kfree(lnk->wr_tx_compl); 566 lnk->wr_tx_compl = NULL; 567 kfree(lnk->wr_tx_pends); 568 lnk->wr_tx_pends = NULL; 569 kfree(lnk->wr_tx_mask); 570 lnk->wr_tx_mask = NULL; 571 kfree(lnk->wr_tx_sges); 572 lnk->wr_tx_sges = NULL; 573 kfree(lnk->wr_tx_rdma_sges); 574 lnk->wr_tx_rdma_sges = NULL; 575 kfree(lnk->wr_rx_sges); 576 lnk->wr_rx_sges = NULL; 577 kfree(lnk->wr_tx_rdmas); 578 lnk->wr_tx_rdmas = NULL; 579 kfree(lnk->wr_rx_ibs); 580 lnk->wr_rx_ibs = NULL; 581 kfree(lnk->wr_tx_ibs); 582 lnk->wr_tx_ibs = NULL; 583 kfree(lnk->wr_tx_bufs); 584 lnk->wr_tx_bufs = NULL; 585 kfree(lnk->wr_rx_bufs); 586 lnk->wr_rx_bufs = NULL; 587} 588 589int smc_wr_alloc_link_mem(struct smc_link *link) 590{ 591 /* allocate link related memory */ 592 link->wr_tx_bufs = kcalloc(SMC_WR_BUF_CNT, SMC_WR_BUF_SIZE, GFP_KERNEL); 593 if (!link->wr_tx_bufs) 594 goto no_mem; 595 link->wr_rx_bufs = kcalloc(SMC_WR_BUF_CNT * 3, SMC_WR_BUF_SIZE, 596 GFP_KERNEL); 597 if (!link->wr_rx_bufs) 598 goto no_mem_wr_tx_bufs; 599 link->wr_tx_ibs = kcalloc(SMC_WR_BUF_CNT, sizeof(link->wr_tx_ibs[0]), 600 GFP_KERNEL); 601 if (!link->wr_tx_ibs) 602 goto no_mem_wr_rx_bufs; 603 link->wr_rx_ibs = kcalloc(SMC_WR_BUF_CNT * 3, 604 sizeof(link->wr_rx_ibs[0]), 605 GFP_KERNEL); 606 if (!link->wr_rx_ibs) 607 goto no_mem_wr_tx_ibs; 608 link->wr_tx_rdmas = kcalloc(SMC_WR_BUF_CNT, 609 sizeof(link->wr_tx_rdmas[0]), 610 GFP_KERNEL); 611 if (!link->wr_tx_rdmas) 612 goto no_mem_wr_rx_ibs; 613 link->wr_tx_rdma_sges = kcalloc(SMC_WR_BUF_CNT, 614 sizeof(link->wr_tx_rdma_sges[0]), 615 GFP_KERNEL); 616 if (!link->wr_tx_rdma_sges) 617 goto no_mem_wr_tx_rdmas; 618 link->wr_tx_sges = kcalloc(SMC_WR_BUF_CNT, sizeof(link->wr_tx_sges[0]), 619 GFP_KERNEL); 620 if (!link->wr_tx_sges) 621 goto no_mem_wr_tx_rdma_sges; 622 link->wr_rx_sges = kcalloc(SMC_WR_BUF_CNT * 3, 623 sizeof(link->wr_rx_sges[0]), 624 GFP_KERNEL); 625 if (!link->wr_rx_sges) 626 goto no_mem_wr_tx_sges; 627 link->wr_tx_mask = kcalloc(BITS_TO_LONGS(SMC_WR_BUF_CNT), 628 sizeof(*link->wr_tx_mask), 629 GFP_KERNEL); 630 if (!link->wr_tx_mask) 631 goto no_mem_wr_rx_sges; 632 link->wr_tx_pends = kcalloc(SMC_WR_BUF_CNT, 633 sizeof(link->wr_tx_pends[0]), 634 GFP_KERNEL); 635 if (!link->wr_tx_pends) 636 goto no_mem_wr_tx_mask; 637 link->wr_tx_compl = kcalloc(SMC_WR_BUF_CNT, 638 sizeof(link->wr_tx_compl[0]), 639 GFP_KERNEL); 640 if (!link->wr_tx_compl) 641 goto no_mem_wr_tx_pends; 642 return 0; 643 644no_mem_wr_tx_pends: 645 kfree(link->wr_tx_pends); 646no_mem_wr_tx_mask: 647 kfree(link->wr_tx_mask); 648no_mem_wr_rx_sges: 649 kfree(link->wr_rx_sges); 650no_mem_wr_tx_sges: 651 kfree(link->wr_tx_sges); 652no_mem_wr_tx_rdma_sges: 653 kfree(link->wr_tx_rdma_sges); 654no_mem_wr_tx_rdmas: 655 kfree(link->wr_tx_rdmas); 656no_mem_wr_rx_ibs: 657 kfree(link->wr_rx_ibs); 658no_mem_wr_tx_ibs: 659 kfree(link->wr_tx_ibs); 660no_mem_wr_rx_bufs: 661 kfree(link->wr_rx_bufs); 662no_mem_wr_tx_bufs: 663 kfree(link->wr_tx_bufs); 664no_mem: 665 return -ENOMEM; 666} 667 668void smc_wr_remove_dev(struct smc_ib_device *smcibdev) 669{ 670 tasklet_kill(&smcibdev->recv_tasklet); 671 tasklet_kill(&smcibdev->send_tasklet); 672} 673 674void smc_wr_add_dev(struct smc_ib_device *smcibdev) 675{ 676 tasklet_init(&smcibdev->recv_tasklet, smc_wr_rx_tasklet_fn, 677 (unsigned long)smcibdev); 678 tasklet_init(&smcibdev->send_tasklet, smc_wr_tx_tasklet_fn, 679 (unsigned long)smcibdev); 680} 681 682int smc_wr_create_link(struct smc_link *lnk) 683{ 684 struct ib_device *ibdev = lnk->smcibdev->ibdev; 685 int rc = 0; 686 687 smc_wr_tx_set_wr_id(&lnk->wr_tx_id, 0); 688 lnk->wr_rx_id = 0; 689 lnk->wr_rx_dma_addr = ib_dma_map_single( 690 ibdev, lnk->wr_rx_bufs, SMC_WR_BUF_SIZE * lnk->wr_rx_cnt, 691 DMA_FROM_DEVICE); 692 if (ib_dma_mapping_error(ibdev, lnk->wr_rx_dma_addr)) { 693 lnk->wr_rx_dma_addr = 0; 694 rc = -EIO; 695 goto out; 696 } 697 lnk->wr_tx_dma_addr = ib_dma_map_single( 698 ibdev, lnk->wr_tx_bufs, SMC_WR_BUF_SIZE * lnk->wr_tx_cnt, 699 DMA_TO_DEVICE); 700 if (ib_dma_mapping_error(ibdev, lnk->wr_tx_dma_addr)) { 701 rc = -EIO; 702 goto dma_unmap; 703 } 704 smc_wr_init_sge(lnk); 705 memset(lnk->wr_tx_mask, 0, 706 BITS_TO_LONGS(SMC_WR_BUF_CNT) * sizeof(*lnk->wr_tx_mask)); 707 init_waitqueue_head(&lnk->wr_tx_wait); 708 atomic_set(&lnk->wr_tx_refcnt, 0); 709 init_waitqueue_head(&lnk->wr_reg_wait); 710 atomic_set(&lnk->wr_reg_refcnt, 0); 711 return rc; 712 713dma_unmap: 714 ib_dma_unmap_single(ibdev, lnk->wr_rx_dma_addr, 715 SMC_WR_BUF_SIZE * lnk->wr_rx_cnt, 716 DMA_FROM_DEVICE); 717 lnk->wr_rx_dma_addr = 0; 718out: 719 return rc; 720} 721