1// SPDX-License-Identifier: GPL-2.0 2/* 3 * Shared Memory Communications over RDMA (SMC-R) and RoCE 4 * 5 * Link Layer Control (LLC) 6 * 7 * Copyright IBM Corp. 2016 8 * 9 * Author(s): Klaus Wacker <Klaus.Wacker@de.ibm.com> 10 * Ursula Braun <ubraun@linux.vnet.ibm.com> 11 */ 12 13#include <net/tcp.h> 14#include <rdma/ib_verbs.h> 15 16#include "smc.h" 17#include "smc_core.h" 18#include "smc_clc.h" 19#include "smc_llc.h" 20#include "smc_pnet.h" 21 22#define SMC_LLC_DATA_LEN 40 23 24struct smc_llc_hdr { 25 struct smc_wr_rx_hdr common; 26 u8 length; /* 44 */ 27#if defined(__BIG_ENDIAN_BITFIELD) 28 u8 reserved:4, 29 add_link_rej_rsn:4; 30#elif defined(__LITTLE_ENDIAN_BITFIELD) 31 u8 add_link_rej_rsn:4, 32 reserved:4; 33#endif 34 u8 flags; 35}; 36 37#define SMC_LLC_FLAG_NO_RMBE_EYEC 0x03 38 39struct smc_llc_msg_confirm_link { /* type 0x01 */ 40 struct smc_llc_hdr hd; 41 u8 sender_mac[ETH_ALEN]; 42 u8 sender_gid[SMC_GID_SIZE]; 43 u8 sender_qp_num[3]; 44 u8 link_num; 45 u8 link_uid[SMC_LGR_ID_SIZE]; 46 u8 max_links; 47 u8 reserved[9]; 48}; 49 50#define SMC_LLC_FLAG_ADD_LNK_REJ 0x40 51#define SMC_LLC_REJ_RSN_NO_ALT_PATH 1 52 53#define SMC_LLC_ADD_LNK_MAX_LINKS 2 54 55struct smc_llc_msg_add_link { /* type 0x02 */ 56 struct smc_llc_hdr hd; 57 u8 sender_mac[ETH_ALEN]; 58 u8 reserved2[2]; 59 u8 sender_gid[SMC_GID_SIZE]; 60 u8 sender_qp_num[3]; 61 u8 link_num; 62#if defined(__BIG_ENDIAN_BITFIELD) 63 u8 reserved3 : 4, 64 qp_mtu : 4; 65#elif defined(__LITTLE_ENDIAN_BITFIELD) 66 u8 qp_mtu : 4, 67 reserved3 : 4; 68#endif 69 u8 initial_psn[3]; 70 u8 reserved[8]; 71}; 72 73struct smc_llc_msg_add_link_cont_rt { 74 __be32 rmb_key; 75 __be32 rmb_key_new; 76 __be64 rmb_vaddr_new; 77}; 78 79#define SMC_LLC_RKEYS_PER_CONT_MSG 2 80 81struct smc_llc_msg_add_link_cont { /* type 0x03 */ 82 struct smc_llc_hdr hd; 83 u8 link_num; 84 u8 num_rkeys; 85 u8 reserved2[2]; 86 struct smc_llc_msg_add_link_cont_rt rt[SMC_LLC_RKEYS_PER_CONT_MSG]; 87 u8 reserved[4]; 88} __packed; /* format defined in RFC7609 */ 89 90#define SMC_LLC_FLAG_DEL_LINK_ALL 0x40 91#define SMC_LLC_FLAG_DEL_LINK_ORDERLY 0x20 92 93struct smc_llc_msg_del_link { /* type 0x04 */ 94 struct smc_llc_hdr hd; 95 u8 link_num; 96 __be32 reason; 97 u8 reserved[35]; 98} __packed; /* format defined in RFC7609 */ 99 100struct smc_llc_msg_test_link { /* type 0x07 */ 101 struct smc_llc_hdr hd; 102 u8 user_data[16]; 103 u8 reserved[24]; 104}; 105 106struct smc_rmb_rtoken { 107 union { 108 u8 num_rkeys; /* first rtoken byte of CONFIRM LINK msg */ 109 /* is actually the num of rtokens, first */ 110 /* rtoken is always for the current link */ 111 u8 link_id; /* link id of the rtoken */ 112 }; 113 __be32 rmb_key; 114 __be64 rmb_vaddr; 115} __packed; /* format defined in RFC7609 */ 116 117#define SMC_LLC_RKEYS_PER_MSG 3 118 119struct smc_llc_msg_confirm_rkey { /* type 0x06 */ 120 struct smc_llc_hdr hd; 121 struct smc_rmb_rtoken rtoken[SMC_LLC_RKEYS_PER_MSG]; 122 u8 reserved; 123}; 124 125#define SMC_LLC_DEL_RKEY_MAX 8 126#define SMC_LLC_FLAG_RKEY_RETRY 0x10 127#define SMC_LLC_FLAG_RKEY_NEG 0x20 128 129struct smc_llc_msg_delete_rkey { /* type 0x09 */ 130 struct smc_llc_hdr hd; 131 u8 num_rkeys; 132 u8 err_mask; 133 u8 reserved[2]; 134 __be32 rkey[8]; 135 u8 reserved2[4]; 136}; 137 138union smc_llc_msg { 139 struct smc_llc_msg_confirm_link confirm_link; 140 struct smc_llc_msg_add_link add_link; 141 struct smc_llc_msg_add_link_cont add_link_cont; 142 struct smc_llc_msg_del_link delete_link; 143 144 struct smc_llc_msg_confirm_rkey confirm_rkey; 145 struct smc_llc_msg_delete_rkey delete_rkey; 146 147 struct smc_llc_msg_test_link test_link; 148 struct { 149 struct smc_llc_hdr hdr; 150 u8 data[SMC_LLC_DATA_LEN]; 151 } raw; 152}; 153 154#define SMC_LLC_FLAG_RESP 0x80 155 156struct smc_llc_qentry { 157 struct list_head list; 158 struct smc_link *link; 159 union smc_llc_msg msg; 160}; 161 162static void smc_llc_enqueue(struct smc_link *link, union smc_llc_msg *llc); 163 164struct smc_llc_qentry *smc_llc_flow_qentry_clr(struct smc_llc_flow *flow) 165{ 166 struct smc_llc_qentry *qentry = flow->qentry; 167 168 flow->qentry = NULL; 169 return qentry; 170} 171 172void smc_llc_flow_qentry_del(struct smc_llc_flow *flow) 173{ 174 struct smc_llc_qentry *qentry; 175 176 if (flow->qentry) { 177 qentry = flow->qentry; 178 flow->qentry = NULL; 179 kfree(qentry); 180 } 181} 182 183static inline void smc_llc_flow_qentry_set(struct smc_llc_flow *flow, 184 struct smc_llc_qentry *qentry) 185{ 186 flow->qentry = qentry; 187} 188 189static void smc_llc_flow_parallel(struct smc_link_group *lgr, u8 flow_type, 190 struct smc_llc_qentry *qentry) 191{ 192 u8 msg_type = qentry->msg.raw.hdr.common.type; 193 194 if ((msg_type == SMC_LLC_ADD_LINK || msg_type == SMC_LLC_DELETE_LINK) && 195 flow_type != msg_type && !lgr->delayed_event) { 196 lgr->delayed_event = qentry; 197 return; 198 } 199 /* drop parallel or already-in-progress llc requests */ 200 if (flow_type != msg_type) 201 pr_warn_once("smc: SMC-R lg %*phN dropped parallel " 202 "LLC msg: msg %d flow %d role %d\n", 203 SMC_LGR_ID_SIZE, &lgr->id, 204 qentry->msg.raw.hdr.common.type, 205 flow_type, lgr->role); 206 kfree(qentry); 207} 208 209/* try to start a new llc flow, initiated by an incoming llc msg */ 210static bool smc_llc_flow_start(struct smc_llc_flow *flow, 211 struct smc_llc_qentry *qentry) 212{ 213 struct smc_link_group *lgr = qentry->link->lgr; 214 215 spin_lock_bh(&lgr->llc_flow_lock); 216 if (flow->type) { 217 /* a flow is already active */ 218 smc_llc_flow_parallel(lgr, flow->type, qentry); 219 spin_unlock_bh(&lgr->llc_flow_lock); 220 return false; 221 } 222 switch (qentry->msg.raw.hdr.common.type) { 223 case SMC_LLC_ADD_LINK: 224 flow->type = SMC_LLC_FLOW_ADD_LINK; 225 break; 226 case SMC_LLC_DELETE_LINK: 227 flow->type = SMC_LLC_FLOW_DEL_LINK; 228 break; 229 case SMC_LLC_CONFIRM_RKEY: 230 case SMC_LLC_DELETE_RKEY: 231 flow->type = SMC_LLC_FLOW_RKEY; 232 break; 233 default: 234 flow->type = SMC_LLC_FLOW_NONE; 235 } 236 smc_llc_flow_qentry_set(flow, qentry); 237 spin_unlock_bh(&lgr->llc_flow_lock); 238 return true; 239} 240 241/* start a new local llc flow, wait till current flow finished */ 242int smc_llc_flow_initiate(struct smc_link_group *lgr, 243 enum smc_llc_flowtype type) 244{ 245 enum smc_llc_flowtype allowed_remote = SMC_LLC_FLOW_NONE; 246 int rc; 247 248 /* all flows except confirm_rkey and delete_rkey are exclusive, 249 * confirm/delete rkey flows can run concurrently (local and remote) 250 */ 251 if (type == SMC_LLC_FLOW_RKEY) 252 allowed_remote = SMC_LLC_FLOW_RKEY; 253again: 254 if (list_empty(&lgr->list)) 255 return -ENODEV; 256 spin_lock_bh(&lgr->llc_flow_lock); 257 if (lgr->llc_flow_lcl.type == SMC_LLC_FLOW_NONE && 258 (lgr->llc_flow_rmt.type == SMC_LLC_FLOW_NONE || 259 lgr->llc_flow_rmt.type == allowed_remote)) { 260 lgr->llc_flow_lcl.type = type; 261 spin_unlock_bh(&lgr->llc_flow_lock); 262 return 0; 263 } 264 spin_unlock_bh(&lgr->llc_flow_lock); 265 rc = wait_event_timeout(lgr->llc_flow_waiter, (list_empty(&lgr->list) || 266 (lgr->llc_flow_lcl.type == SMC_LLC_FLOW_NONE && 267 (lgr->llc_flow_rmt.type == SMC_LLC_FLOW_NONE || 268 lgr->llc_flow_rmt.type == allowed_remote))), 269 SMC_LLC_WAIT_TIME * 10); 270 if (!rc) 271 return -ETIMEDOUT; 272 goto again; 273} 274 275/* finish the current llc flow */ 276void smc_llc_flow_stop(struct smc_link_group *lgr, struct smc_llc_flow *flow) 277{ 278 spin_lock_bh(&lgr->llc_flow_lock); 279 memset(flow, 0, sizeof(*flow)); 280 flow->type = SMC_LLC_FLOW_NONE; 281 spin_unlock_bh(&lgr->llc_flow_lock); 282 if (!list_empty(&lgr->list) && lgr->delayed_event && 283 flow == &lgr->llc_flow_lcl) 284 schedule_work(&lgr->llc_event_work); 285 else 286 wake_up(&lgr->llc_flow_waiter); 287} 288 289/* lnk is optional and used for early wakeup when link goes down, useful in 290 * cases where we wait for a response on the link after we sent a request 291 */ 292struct smc_llc_qentry *smc_llc_wait(struct smc_link_group *lgr, 293 struct smc_link *lnk, 294 int time_out, u8 exp_msg) 295{ 296 struct smc_llc_flow *flow = &lgr->llc_flow_lcl; 297 u8 rcv_msg; 298 299 wait_event_timeout(lgr->llc_msg_waiter, 300 (flow->qentry || 301 (lnk && !smc_link_usable(lnk)) || 302 list_empty(&lgr->list)), 303 time_out); 304 if (!flow->qentry || 305 (lnk && !smc_link_usable(lnk)) || list_empty(&lgr->list)) { 306 smc_llc_flow_qentry_del(flow); 307 goto out; 308 } 309 rcv_msg = flow->qentry->msg.raw.hdr.common.type; 310 if (exp_msg && rcv_msg != exp_msg) { 311 if (exp_msg == SMC_LLC_ADD_LINK && 312 rcv_msg == SMC_LLC_DELETE_LINK) { 313 /* flow_start will delay the unexpected msg */ 314 smc_llc_flow_start(&lgr->llc_flow_lcl, 315 smc_llc_flow_qentry_clr(flow)); 316 return NULL; 317 } 318 pr_warn_once("smc: SMC-R lg %*phN dropped unexpected LLC msg: " 319 "msg %d exp %d flow %d role %d flags %x\n", 320 SMC_LGR_ID_SIZE, &lgr->id, rcv_msg, exp_msg, 321 flow->type, lgr->role, 322 flow->qentry->msg.raw.hdr.flags); 323 smc_llc_flow_qentry_del(flow); 324 } 325out: 326 return flow->qentry; 327} 328 329/********************************** send *************************************/ 330 331struct smc_llc_tx_pend { 332}; 333 334/* handler for send/transmission completion of an LLC msg */ 335static void smc_llc_tx_handler(struct smc_wr_tx_pend_priv *pend, 336 struct smc_link *link, 337 enum ib_wc_status wc_status) 338{ 339 /* future work: handle wc_status error for recovery and failover */ 340} 341 342/** 343 * smc_llc_add_pending_send() - add LLC control message to pending WQE transmits 344 * @link: Pointer to SMC link used for sending LLC control message. 345 * @wr_buf: Out variable returning pointer to work request payload buffer. 346 * @pend: Out variable returning pointer to private pending WR tracking. 347 * It's the context the transmit complete handler will get. 348 * 349 * Reserves and pre-fills an entry for a pending work request send/tx. 350 * Used by mid-level smc_llc_send_msg() to prepare for later actual send/tx. 351 * Can sleep due to smc_get_ctrl_buf (if not in softirq context). 352 * 353 * Return: 0 on success, otherwise an error value. 354 */ 355static int smc_llc_add_pending_send(struct smc_link *link, 356 struct smc_wr_buf **wr_buf, 357 struct smc_wr_tx_pend_priv **pend) 358{ 359 int rc; 360 361 rc = smc_wr_tx_get_free_slot(link, smc_llc_tx_handler, wr_buf, NULL, 362 pend); 363 if (rc < 0) 364 return rc; 365 BUILD_BUG_ON_MSG( 366 sizeof(union smc_llc_msg) > SMC_WR_BUF_SIZE, 367 "must increase SMC_WR_BUF_SIZE to at least sizeof(struct smc_llc_msg)"); 368 BUILD_BUG_ON_MSG( 369 sizeof(union smc_llc_msg) != SMC_WR_TX_SIZE, 370 "must adapt SMC_WR_TX_SIZE to sizeof(struct smc_llc_msg); if not all smc_wr upper layer protocols use the same message size any more, must start to set link->wr_tx_sges[i].length on each individual smc_wr_tx_send()"); 371 BUILD_BUG_ON_MSG( 372 sizeof(struct smc_llc_tx_pend) > SMC_WR_TX_PEND_PRIV_SIZE, 373 "must increase SMC_WR_TX_PEND_PRIV_SIZE to at least sizeof(struct smc_llc_tx_pend)"); 374 return 0; 375} 376 377/* high-level API to send LLC confirm link */ 378int smc_llc_send_confirm_link(struct smc_link *link, 379 enum smc_llc_reqresp reqresp) 380{ 381 struct smc_llc_msg_confirm_link *confllc; 382 struct smc_wr_tx_pend_priv *pend; 383 struct smc_wr_buf *wr_buf; 384 int rc; 385 386 if (!smc_wr_tx_link_hold(link)) 387 return -ENOLINK; 388 rc = smc_llc_add_pending_send(link, &wr_buf, &pend); 389 if (rc) 390 goto put_out; 391 confllc = (struct smc_llc_msg_confirm_link *)wr_buf; 392 memset(confllc, 0, sizeof(*confllc)); 393 confllc->hd.common.type = SMC_LLC_CONFIRM_LINK; 394 confllc->hd.length = sizeof(struct smc_llc_msg_confirm_link); 395 confllc->hd.flags |= SMC_LLC_FLAG_NO_RMBE_EYEC; 396 if (reqresp == SMC_LLC_RESP) 397 confllc->hd.flags |= SMC_LLC_FLAG_RESP; 398 memcpy(confllc->sender_mac, link->smcibdev->mac[link->ibport - 1], 399 ETH_ALEN); 400 memcpy(confllc->sender_gid, link->gid, SMC_GID_SIZE); 401 hton24(confllc->sender_qp_num, link->roce_qp->qp_num); 402 confllc->link_num = link->link_id; 403 memcpy(confllc->link_uid, link->link_uid, SMC_LGR_ID_SIZE); 404 confllc->max_links = SMC_LLC_ADD_LNK_MAX_LINKS; 405 /* send llc message */ 406 rc = smc_wr_tx_send(link, pend); 407put_out: 408 smc_wr_tx_link_put(link); 409 return rc; 410} 411 412/* send LLC confirm rkey request */ 413static int smc_llc_send_confirm_rkey(struct smc_link *send_link, 414 struct smc_buf_desc *rmb_desc) 415{ 416 struct smc_llc_msg_confirm_rkey *rkeyllc; 417 struct smc_wr_tx_pend_priv *pend; 418 struct smc_wr_buf *wr_buf; 419 struct smc_link *link; 420 int i, rc, rtok_ix; 421 422 if (!smc_wr_tx_link_hold(send_link)) 423 return -ENOLINK; 424 rc = smc_llc_add_pending_send(send_link, &wr_buf, &pend); 425 if (rc) 426 goto put_out; 427 rkeyllc = (struct smc_llc_msg_confirm_rkey *)wr_buf; 428 memset(rkeyllc, 0, sizeof(*rkeyllc)); 429 rkeyllc->hd.common.type = SMC_LLC_CONFIRM_RKEY; 430 rkeyllc->hd.length = sizeof(struct smc_llc_msg_confirm_rkey); 431 432 rtok_ix = 1; 433 for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) { 434 link = &send_link->lgr->lnk[i]; 435 if (smc_link_active(link) && link != send_link) { 436 rkeyllc->rtoken[rtok_ix].link_id = link->link_id; 437 rkeyllc->rtoken[rtok_ix].rmb_key = 438 htonl(rmb_desc->mr_rx[link->link_idx]->rkey); 439 rkeyllc->rtoken[rtok_ix].rmb_vaddr = cpu_to_be64( 440 (u64)sg_dma_address( 441 rmb_desc->sgt[link->link_idx].sgl)); 442 rtok_ix++; 443 } 444 } 445 /* rkey of send_link is in rtoken[0] */ 446 rkeyllc->rtoken[0].num_rkeys = rtok_ix - 1; 447 rkeyllc->rtoken[0].rmb_key = 448 htonl(rmb_desc->mr_rx[send_link->link_idx]->rkey); 449 rkeyllc->rtoken[0].rmb_vaddr = cpu_to_be64( 450 (u64)sg_dma_address(rmb_desc->sgt[send_link->link_idx].sgl)); 451 /* send llc message */ 452 rc = smc_wr_tx_send(send_link, pend); 453put_out: 454 smc_wr_tx_link_put(send_link); 455 return rc; 456} 457 458/* send LLC delete rkey request */ 459static int smc_llc_send_delete_rkey(struct smc_link *link, 460 struct smc_buf_desc *rmb_desc) 461{ 462 struct smc_llc_msg_delete_rkey *rkeyllc; 463 struct smc_wr_tx_pend_priv *pend; 464 struct smc_wr_buf *wr_buf; 465 int rc; 466 467 if (!smc_wr_tx_link_hold(link)) 468 return -ENOLINK; 469 rc = smc_llc_add_pending_send(link, &wr_buf, &pend); 470 if (rc) 471 goto put_out; 472 rkeyllc = (struct smc_llc_msg_delete_rkey *)wr_buf; 473 memset(rkeyllc, 0, sizeof(*rkeyllc)); 474 rkeyllc->hd.common.type = SMC_LLC_DELETE_RKEY; 475 rkeyllc->hd.length = sizeof(struct smc_llc_msg_delete_rkey); 476 rkeyllc->num_rkeys = 1; 477 rkeyllc->rkey[0] = htonl(rmb_desc->mr_rx[link->link_idx]->rkey); 478 /* send llc message */ 479 rc = smc_wr_tx_send(link, pend); 480put_out: 481 smc_wr_tx_link_put(link); 482 return rc; 483} 484 485/* send ADD LINK request or response */ 486int smc_llc_send_add_link(struct smc_link *link, u8 mac[], u8 gid[], 487 struct smc_link *link_new, 488 enum smc_llc_reqresp reqresp) 489{ 490 struct smc_llc_msg_add_link *addllc; 491 struct smc_wr_tx_pend_priv *pend; 492 struct smc_wr_buf *wr_buf; 493 int rc; 494 495 if (!smc_wr_tx_link_hold(link)) 496 return -ENOLINK; 497 rc = smc_llc_add_pending_send(link, &wr_buf, &pend); 498 if (rc) 499 goto put_out; 500 addllc = (struct smc_llc_msg_add_link *)wr_buf; 501 502 memset(addllc, 0, sizeof(*addllc)); 503 addllc->hd.common.type = SMC_LLC_ADD_LINK; 504 addllc->hd.length = sizeof(struct smc_llc_msg_add_link); 505 if (reqresp == SMC_LLC_RESP) 506 addllc->hd.flags |= SMC_LLC_FLAG_RESP; 507 memcpy(addllc->sender_mac, mac, ETH_ALEN); 508 memcpy(addllc->sender_gid, gid, SMC_GID_SIZE); 509 if (link_new) { 510 addllc->link_num = link_new->link_id; 511 hton24(addllc->sender_qp_num, link_new->roce_qp->qp_num); 512 hton24(addllc->initial_psn, link_new->psn_initial); 513 if (reqresp == SMC_LLC_REQ) 514 addllc->qp_mtu = link_new->path_mtu; 515 else 516 addllc->qp_mtu = min(link_new->path_mtu, 517 link_new->peer_mtu); 518 } 519 /* send llc message */ 520 rc = smc_wr_tx_send(link, pend); 521put_out: 522 smc_wr_tx_link_put(link); 523 return rc; 524} 525 526/* send DELETE LINK request or response */ 527int smc_llc_send_delete_link(struct smc_link *link, u8 link_del_id, 528 enum smc_llc_reqresp reqresp, bool orderly, 529 u32 reason) 530{ 531 struct smc_llc_msg_del_link *delllc; 532 struct smc_wr_tx_pend_priv *pend; 533 struct smc_wr_buf *wr_buf; 534 int rc; 535 536 if (!smc_wr_tx_link_hold(link)) 537 return -ENOLINK; 538 rc = smc_llc_add_pending_send(link, &wr_buf, &pend); 539 if (rc) 540 goto put_out; 541 delllc = (struct smc_llc_msg_del_link *)wr_buf; 542 543 memset(delllc, 0, sizeof(*delllc)); 544 delllc->hd.common.type = SMC_LLC_DELETE_LINK; 545 delllc->hd.length = sizeof(struct smc_llc_msg_del_link); 546 if (reqresp == SMC_LLC_RESP) 547 delllc->hd.flags |= SMC_LLC_FLAG_RESP; 548 if (orderly) 549 delllc->hd.flags |= SMC_LLC_FLAG_DEL_LINK_ORDERLY; 550 if (link_del_id) 551 delllc->link_num = link_del_id; 552 else 553 delllc->hd.flags |= SMC_LLC_FLAG_DEL_LINK_ALL; 554 delllc->reason = htonl(reason); 555 /* send llc message */ 556 rc = smc_wr_tx_send(link, pend); 557put_out: 558 smc_wr_tx_link_put(link); 559 return rc; 560} 561 562/* send LLC test link request */ 563static int smc_llc_send_test_link(struct smc_link *link, u8 user_data[16]) 564{ 565 struct smc_llc_msg_test_link *testllc; 566 struct smc_wr_tx_pend_priv *pend; 567 struct smc_wr_buf *wr_buf; 568 int rc; 569 570 if (!smc_wr_tx_link_hold(link)) 571 return -ENOLINK; 572 rc = smc_llc_add_pending_send(link, &wr_buf, &pend); 573 if (rc) 574 goto put_out; 575 testllc = (struct smc_llc_msg_test_link *)wr_buf; 576 memset(testllc, 0, sizeof(*testllc)); 577 testllc->hd.common.type = SMC_LLC_TEST_LINK; 578 testllc->hd.length = sizeof(struct smc_llc_msg_test_link); 579 memcpy(testllc->user_data, user_data, sizeof(testllc->user_data)); 580 /* send llc message */ 581 rc = smc_wr_tx_send(link, pend); 582put_out: 583 smc_wr_tx_link_put(link); 584 return rc; 585} 586 587/* schedule an llc send on link, may wait for buffers */ 588static int smc_llc_send_message(struct smc_link *link, void *llcbuf) 589{ 590 struct smc_wr_tx_pend_priv *pend; 591 struct smc_wr_buf *wr_buf; 592 int rc; 593 594 if (!smc_wr_tx_link_hold(link)) 595 return -ENOLINK; 596 rc = smc_llc_add_pending_send(link, &wr_buf, &pend); 597 if (rc) 598 goto put_out; 599 memcpy(wr_buf, llcbuf, sizeof(union smc_llc_msg)); 600 rc = smc_wr_tx_send(link, pend); 601put_out: 602 smc_wr_tx_link_put(link); 603 return rc; 604} 605 606/* schedule an llc send on link, may wait for buffers, 607 * and wait for send completion notification. 608 * @return 0 on success 609 */ 610static int smc_llc_send_message_wait(struct smc_link *link, void *llcbuf) 611{ 612 struct smc_wr_tx_pend_priv *pend; 613 struct smc_wr_buf *wr_buf; 614 int rc; 615 616 if (!smc_wr_tx_link_hold(link)) 617 return -ENOLINK; 618 rc = smc_llc_add_pending_send(link, &wr_buf, &pend); 619 if (rc) 620 goto put_out; 621 memcpy(wr_buf, llcbuf, sizeof(union smc_llc_msg)); 622 rc = smc_wr_tx_send_wait(link, pend, SMC_LLC_WAIT_TIME); 623put_out: 624 smc_wr_tx_link_put(link); 625 return rc; 626} 627 628/********************************* receive ***********************************/ 629 630static int smc_llc_alloc_alt_link(struct smc_link_group *lgr, 631 enum smc_lgr_type lgr_new_t) 632{ 633 int i; 634 635 if (lgr->type == SMC_LGR_SYMMETRIC || 636 (lgr->type != SMC_LGR_SINGLE && 637 (lgr_new_t == SMC_LGR_ASYMMETRIC_LOCAL || 638 lgr_new_t == SMC_LGR_ASYMMETRIC_PEER))) 639 return -EMLINK; 640 641 if (lgr_new_t == SMC_LGR_ASYMMETRIC_LOCAL || 642 lgr_new_t == SMC_LGR_ASYMMETRIC_PEER) { 643 for (i = SMC_LINKS_PER_LGR_MAX - 1; i >= 0; i--) 644 if (lgr->lnk[i].state == SMC_LNK_UNUSED) 645 return i; 646 } else { 647 for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) 648 if (lgr->lnk[i].state == SMC_LNK_UNUSED) 649 return i; 650 } 651 return -EMLINK; 652} 653 654/* return first buffer from any of the next buf lists */ 655static struct smc_buf_desc *_smc_llc_get_next_rmb(struct smc_link_group *lgr, 656 int *buf_lst) 657{ 658 struct smc_buf_desc *buf_pos; 659 660 while (*buf_lst < SMC_RMBE_SIZES) { 661 buf_pos = list_first_entry_or_null(&lgr->rmbs[*buf_lst], 662 struct smc_buf_desc, list); 663 if (buf_pos) 664 return buf_pos; 665 (*buf_lst)++; 666 } 667 return NULL; 668} 669 670/* return next rmb from buffer lists */ 671static struct smc_buf_desc *smc_llc_get_next_rmb(struct smc_link_group *lgr, 672 int *buf_lst, 673 struct smc_buf_desc *buf_pos) 674{ 675 struct smc_buf_desc *buf_next; 676 677 if (!buf_pos || list_is_last(&buf_pos->list, &lgr->rmbs[*buf_lst])) { 678 (*buf_lst)++; 679 return _smc_llc_get_next_rmb(lgr, buf_lst); 680 } 681 buf_next = list_next_entry(buf_pos, list); 682 return buf_next; 683} 684 685static struct smc_buf_desc *smc_llc_get_first_rmb(struct smc_link_group *lgr, 686 int *buf_lst) 687{ 688 *buf_lst = 0; 689 return smc_llc_get_next_rmb(lgr, buf_lst, NULL); 690} 691 692/* send one add_link_continue msg */ 693static int smc_llc_add_link_cont(struct smc_link *link, 694 struct smc_link *link_new, u8 *num_rkeys_todo, 695 int *buf_lst, struct smc_buf_desc **buf_pos) 696{ 697 struct smc_llc_msg_add_link_cont *addc_llc; 698 struct smc_link_group *lgr = link->lgr; 699 int prim_lnk_idx, lnk_idx, i, rc; 700 struct smc_wr_tx_pend_priv *pend; 701 struct smc_wr_buf *wr_buf; 702 struct smc_buf_desc *rmb; 703 u8 n; 704 705 if (!smc_wr_tx_link_hold(link)) 706 return -ENOLINK; 707 rc = smc_llc_add_pending_send(link, &wr_buf, &pend); 708 if (rc) 709 goto put_out; 710 addc_llc = (struct smc_llc_msg_add_link_cont *)wr_buf; 711 memset(addc_llc, 0, sizeof(*addc_llc)); 712 713 prim_lnk_idx = link->link_idx; 714 lnk_idx = link_new->link_idx; 715 addc_llc->link_num = link_new->link_id; 716 addc_llc->num_rkeys = *num_rkeys_todo; 717 n = *num_rkeys_todo; 718 for (i = 0; i < min_t(u8, n, SMC_LLC_RKEYS_PER_CONT_MSG); i++) { 719 while (*buf_pos && !(*buf_pos)->used) 720 *buf_pos = smc_llc_get_next_rmb(lgr, buf_lst, *buf_pos); 721 if (!*buf_pos) { 722 addc_llc->num_rkeys = addc_llc->num_rkeys - 723 *num_rkeys_todo; 724 *num_rkeys_todo = 0; 725 break; 726 } 727 rmb = *buf_pos; 728 729 addc_llc->rt[i].rmb_key = htonl(rmb->mr_rx[prim_lnk_idx]->rkey); 730 addc_llc->rt[i].rmb_key_new = htonl(rmb->mr_rx[lnk_idx]->rkey); 731 addc_llc->rt[i].rmb_vaddr_new = 732 cpu_to_be64((u64)sg_dma_address(rmb->sgt[lnk_idx].sgl)); 733 734 (*num_rkeys_todo)--; 735 *buf_pos = smc_llc_get_next_rmb(lgr, buf_lst, *buf_pos); 736 } 737 addc_llc->hd.common.type = SMC_LLC_ADD_LINK_CONT; 738 addc_llc->hd.length = sizeof(struct smc_llc_msg_add_link_cont); 739 if (lgr->role == SMC_CLNT) 740 addc_llc->hd.flags |= SMC_LLC_FLAG_RESP; 741 rc = smc_wr_tx_send(link, pend); 742put_out: 743 smc_wr_tx_link_put(link); 744 return rc; 745} 746 747static int smc_llc_cli_rkey_exchange(struct smc_link *link, 748 struct smc_link *link_new) 749{ 750 struct smc_llc_msg_add_link_cont *addc_llc; 751 struct smc_link_group *lgr = link->lgr; 752 u8 max, num_rkeys_send, num_rkeys_recv; 753 struct smc_llc_qentry *qentry; 754 struct smc_buf_desc *buf_pos; 755 int buf_lst; 756 int rc = 0; 757 int i; 758 759 mutex_lock(&lgr->rmbs_lock); 760 num_rkeys_send = lgr->conns_num; 761 buf_pos = smc_llc_get_first_rmb(lgr, &buf_lst); 762 do { 763 qentry = smc_llc_wait(lgr, NULL, SMC_LLC_WAIT_TIME, 764 SMC_LLC_ADD_LINK_CONT); 765 if (!qentry) { 766 rc = -ETIMEDOUT; 767 break; 768 } 769 addc_llc = &qentry->msg.add_link_cont; 770 num_rkeys_recv = addc_llc->num_rkeys; 771 max = min_t(u8, num_rkeys_recv, SMC_LLC_RKEYS_PER_CONT_MSG); 772 for (i = 0; i < max; i++) { 773 smc_rtoken_set(lgr, link->link_idx, link_new->link_idx, 774 addc_llc->rt[i].rmb_key, 775 addc_llc->rt[i].rmb_vaddr_new, 776 addc_llc->rt[i].rmb_key_new); 777 num_rkeys_recv--; 778 } 779 smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); 780 rc = smc_llc_add_link_cont(link, link_new, &num_rkeys_send, 781 &buf_lst, &buf_pos); 782 if (rc) 783 break; 784 } while (num_rkeys_send || num_rkeys_recv); 785 786 mutex_unlock(&lgr->rmbs_lock); 787 return rc; 788} 789 790/* prepare and send an add link reject response */ 791static int smc_llc_cli_add_link_reject(struct smc_llc_qentry *qentry) 792{ 793 qentry->msg.raw.hdr.flags |= SMC_LLC_FLAG_RESP; 794 qentry->msg.raw.hdr.flags |= SMC_LLC_FLAG_ADD_LNK_REJ; 795 qentry->msg.raw.hdr.add_link_rej_rsn = SMC_LLC_REJ_RSN_NO_ALT_PATH; 796 return smc_llc_send_message(qentry->link, &qentry->msg); 797} 798 799static int smc_llc_cli_conf_link(struct smc_link *link, 800 struct smc_init_info *ini, 801 struct smc_link *link_new, 802 enum smc_lgr_type lgr_new_t) 803{ 804 struct smc_link_group *lgr = link->lgr; 805 struct smc_llc_qentry *qentry = NULL; 806 int rc = 0; 807 808 /* receive CONFIRM LINK request over RoCE fabric */ 809 qentry = smc_llc_wait(lgr, NULL, SMC_LLC_WAIT_FIRST_TIME, 0); 810 if (!qentry) { 811 rc = smc_llc_send_delete_link(link, link_new->link_id, 812 SMC_LLC_REQ, false, 813 SMC_LLC_DEL_LOST_PATH); 814 return -ENOLINK; 815 } 816 if (qentry->msg.raw.hdr.common.type != SMC_LLC_CONFIRM_LINK) { 817 /* received DELETE_LINK instead */ 818 qentry->msg.raw.hdr.flags |= SMC_LLC_FLAG_RESP; 819 smc_llc_send_message(link, &qentry->msg); 820 smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); 821 return -ENOLINK; 822 } 823 smc_llc_save_peer_uid(qentry); 824 smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); 825 826 rc = smc_ib_modify_qp_rts(link_new); 827 if (rc) { 828 smc_llc_send_delete_link(link, link_new->link_id, SMC_LLC_REQ, 829 false, SMC_LLC_DEL_LOST_PATH); 830 return -ENOLINK; 831 } 832 smc_wr_remember_qp_attr(link_new); 833 834 rc = smcr_buf_reg_lgr(link_new); 835 if (rc) { 836 smc_llc_send_delete_link(link, link_new->link_id, SMC_LLC_REQ, 837 false, SMC_LLC_DEL_LOST_PATH); 838 return -ENOLINK; 839 } 840 841 /* send CONFIRM LINK response over RoCE fabric */ 842 rc = smc_llc_send_confirm_link(link_new, SMC_LLC_RESP); 843 if (rc) { 844 smc_llc_send_delete_link(link, link_new->link_id, SMC_LLC_REQ, 845 false, SMC_LLC_DEL_LOST_PATH); 846 return -ENOLINK; 847 } 848 smc_llc_link_active(link_new); 849 if (lgr_new_t == SMC_LGR_ASYMMETRIC_LOCAL || 850 lgr_new_t == SMC_LGR_ASYMMETRIC_PEER) 851 smcr_lgr_set_type_asym(lgr, lgr_new_t, link_new->link_idx); 852 else 853 smcr_lgr_set_type(lgr, lgr_new_t); 854 return 0; 855} 856 857static void smc_llc_save_add_link_info(struct smc_link *link, 858 struct smc_llc_msg_add_link *add_llc) 859{ 860 link->peer_qpn = ntoh24(add_llc->sender_qp_num); 861 memcpy(link->peer_gid, add_llc->sender_gid, SMC_GID_SIZE); 862 memcpy(link->peer_mac, add_llc->sender_mac, ETH_ALEN); 863 link->peer_psn = ntoh24(add_llc->initial_psn); 864 link->peer_mtu = add_llc->qp_mtu; 865} 866 867/* as an SMC client, process an add link request */ 868int smc_llc_cli_add_link(struct smc_link *link, struct smc_llc_qentry *qentry) 869{ 870 struct smc_llc_msg_add_link *llc = &qentry->msg.add_link; 871 enum smc_lgr_type lgr_new_t = SMC_LGR_SYMMETRIC; 872 struct smc_link_group *lgr = smc_get_lgr(link); 873 struct smc_link *lnk_new = NULL; 874 struct smc_init_info ini; 875 int lnk_idx, rc = 0; 876 877 if (!llc->qp_mtu) 878 goto out_reject; 879 880 ini.vlan_id = lgr->vlan_id; 881 smc_pnet_find_alt_roce(lgr, &ini, link->smcibdev); 882 if (!memcmp(llc->sender_gid, link->peer_gid, SMC_GID_SIZE) && 883 !memcmp(llc->sender_mac, link->peer_mac, ETH_ALEN)) { 884 if (!ini.ib_dev) 885 goto out_reject; 886 lgr_new_t = SMC_LGR_ASYMMETRIC_PEER; 887 } 888 if (!ini.ib_dev) { 889 lgr_new_t = SMC_LGR_ASYMMETRIC_LOCAL; 890 ini.ib_dev = link->smcibdev; 891 ini.ib_port = link->ibport; 892 } 893 lnk_idx = smc_llc_alloc_alt_link(lgr, lgr_new_t); 894 if (lnk_idx < 0) 895 goto out_reject; 896 lnk_new = &lgr->lnk[lnk_idx]; 897 rc = smcr_link_init(lgr, lnk_new, lnk_idx, &ini); 898 if (rc) 899 goto out_reject; 900 smc_llc_save_add_link_info(lnk_new, llc); 901 lnk_new->link_id = llc->link_num; /* SMC server assigns link id */ 902 smc_llc_link_set_uid(lnk_new); 903 904 rc = smc_ib_ready_link(lnk_new); 905 if (rc) 906 goto out_clear_lnk; 907 908 rc = smcr_buf_map_lgr(lnk_new); 909 if (rc) 910 goto out_clear_lnk; 911 912 rc = smc_llc_send_add_link(link, 913 lnk_new->smcibdev->mac[ini.ib_port - 1], 914 lnk_new->gid, lnk_new, SMC_LLC_RESP); 915 if (rc) 916 goto out_clear_lnk; 917 rc = smc_llc_cli_rkey_exchange(link, lnk_new); 918 if (rc) { 919 rc = 0; 920 goto out_clear_lnk; 921 } 922 rc = smc_llc_cli_conf_link(link, &ini, lnk_new, lgr_new_t); 923 if (!rc) 924 goto out; 925out_clear_lnk: 926 lnk_new->state = SMC_LNK_INACTIVE; 927 smcr_link_clear(lnk_new, false); 928out_reject: 929 smc_llc_cli_add_link_reject(qentry); 930out: 931 kfree(qentry); 932 return rc; 933} 934 935/* as an SMC client, invite server to start the add_link processing */ 936static void smc_llc_cli_add_link_invite(struct smc_link *link, 937 struct smc_llc_qentry *qentry) 938{ 939 struct smc_link_group *lgr = smc_get_lgr(link); 940 struct smc_init_info ini; 941 942 if (lgr->type == SMC_LGR_SYMMETRIC || 943 lgr->type == SMC_LGR_ASYMMETRIC_PEER) 944 goto out; 945 946 ini.vlan_id = lgr->vlan_id; 947 smc_pnet_find_alt_roce(lgr, &ini, link->smcibdev); 948 if (!ini.ib_dev) 949 goto out; 950 951 smc_llc_send_add_link(link, ini.ib_dev->mac[ini.ib_port - 1], 952 ini.ib_gid, NULL, SMC_LLC_REQ); 953out: 954 kfree(qentry); 955} 956 957static bool smc_llc_is_empty_llc_message(union smc_llc_msg *llc) 958{ 959 int i; 960 961 for (i = 0; i < ARRAY_SIZE(llc->raw.data); i++) 962 if (llc->raw.data[i]) 963 return false; 964 return true; 965} 966 967static bool smc_llc_is_local_add_link(union smc_llc_msg *llc) 968{ 969 if (llc->raw.hdr.common.type == SMC_LLC_ADD_LINK && 970 smc_llc_is_empty_llc_message(llc)) 971 return true; 972 return false; 973} 974 975static void smc_llc_process_cli_add_link(struct smc_link_group *lgr) 976{ 977 struct smc_llc_qentry *qentry; 978 979 qentry = smc_llc_flow_qentry_clr(&lgr->llc_flow_lcl); 980 981 mutex_lock(&lgr->llc_conf_mutex); 982 if (smc_llc_is_local_add_link(&qentry->msg)) 983 smc_llc_cli_add_link_invite(qentry->link, qentry); 984 else 985 smc_llc_cli_add_link(qentry->link, qentry); 986 mutex_unlock(&lgr->llc_conf_mutex); 987} 988 989static int smc_llc_active_link_count(struct smc_link_group *lgr) 990{ 991 int i, link_count = 0; 992 993 for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) { 994 if (!smc_link_active(&lgr->lnk[i])) 995 continue; 996 link_count++; 997 } 998 return link_count; 999} 1000 1001/* find the asymmetric link when 3 links are established */ 1002static struct smc_link *smc_llc_find_asym_link(struct smc_link_group *lgr) 1003{ 1004 int asym_idx = -ENOENT; 1005 int i, j, k; 1006 bool found; 1007 1008 /* determine asymmetric link */ 1009 found = false; 1010 for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) { 1011 for (j = i + 1; j < SMC_LINKS_PER_LGR_MAX; j++) { 1012 if (!smc_link_usable(&lgr->lnk[i]) || 1013 !smc_link_usable(&lgr->lnk[j])) 1014 continue; 1015 if (!memcmp(lgr->lnk[i].gid, lgr->lnk[j].gid, 1016 SMC_GID_SIZE)) { 1017 found = true; /* asym_lnk is i or j */ 1018 break; 1019 } 1020 } 1021 if (found) 1022 break; 1023 } 1024 if (!found) 1025 goto out; /* no asymmetric link */ 1026 for (k = 0; k < SMC_LINKS_PER_LGR_MAX; k++) { 1027 if (!smc_link_usable(&lgr->lnk[k])) 1028 continue; 1029 if (k != i && 1030 !memcmp(lgr->lnk[i].peer_gid, lgr->lnk[k].peer_gid, 1031 SMC_GID_SIZE)) { 1032 asym_idx = i; 1033 break; 1034 } 1035 if (k != j && 1036 !memcmp(lgr->lnk[j].peer_gid, lgr->lnk[k].peer_gid, 1037 SMC_GID_SIZE)) { 1038 asym_idx = j; 1039 break; 1040 } 1041 } 1042out: 1043 return (asym_idx < 0) ? NULL : &lgr->lnk[asym_idx]; 1044} 1045 1046static void smc_llc_delete_asym_link(struct smc_link_group *lgr) 1047{ 1048 struct smc_link *lnk_new = NULL, *lnk_asym; 1049 struct smc_llc_qentry *qentry; 1050 int rc; 1051 1052 lnk_asym = smc_llc_find_asym_link(lgr); 1053 if (!lnk_asym) 1054 return; /* no asymmetric link */ 1055 if (!smc_link_downing(&lnk_asym->state)) 1056 return; 1057 lnk_new = smc_switch_conns(lgr, lnk_asym, false); 1058 smc_wr_tx_wait_no_pending_sends(lnk_asym); 1059 if (!lnk_new) 1060 goto out_free; 1061 /* change flow type from ADD_LINK into DEL_LINK */ 1062 lgr->llc_flow_lcl.type = SMC_LLC_FLOW_DEL_LINK; 1063 rc = smc_llc_send_delete_link(lnk_new, lnk_asym->link_id, SMC_LLC_REQ, 1064 true, SMC_LLC_DEL_NO_ASYM_NEEDED); 1065 if (rc) { 1066 smcr_link_down_cond(lnk_new); 1067 goto out_free; 1068 } 1069 qentry = smc_llc_wait(lgr, lnk_new, SMC_LLC_WAIT_TIME, 1070 SMC_LLC_DELETE_LINK); 1071 if (!qentry) { 1072 smcr_link_down_cond(lnk_new); 1073 goto out_free; 1074 } 1075 smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); 1076out_free: 1077 smcr_link_clear(lnk_asym, true); 1078} 1079 1080static int smc_llc_srv_rkey_exchange(struct smc_link *link, 1081 struct smc_link *link_new) 1082{ 1083 struct smc_llc_msg_add_link_cont *addc_llc; 1084 struct smc_link_group *lgr = link->lgr; 1085 u8 max, num_rkeys_send, num_rkeys_recv; 1086 struct smc_llc_qentry *qentry = NULL; 1087 struct smc_buf_desc *buf_pos; 1088 int buf_lst; 1089 int rc = 0; 1090 int i; 1091 1092 mutex_lock(&lgr->rmbs_lock); 1093 num_rkeys_send = lgr->conns_num; 1094 buf_pos = smc_llc_get_first_rmb(lgr, &buf_lst); 1095 do { 1096 smc_llc_add_link_cont(link, link_new, &num_rkeys_send, 1097 &buf_lst, &buf_pos); 1098 qentry = smc_llc_wait(lgr, link, SMC_LLC_WAIT_TIME, 1099 SMC_LLC_ADD_LINK_CONT); 1100 if (!qentry) { 1101 rc = -ETIMEDOUT; 1102 goto out; 1103 } 1104 addc_llc = &qentry->msg.add_link_cont; 1105 num_rkeys_recv = addc_llc->num_rkeys; 1106 max = min_t(u8, num_rkeys_recv, SMC_LLC_RKEYS_PER_CONT_MSG); 1107 for (i = 0; i < max; i++) { 1108 smc_rtoken_set(lgr, link->link_idx, link_new->link_idx, 1109 addc_llc->rt[i].rmb_key, 1110 addc_llc->rt[i].rmb_vaddr_new, 1111 addc_llc->rt[i].rmb_key_new); 1112 num_rkeys_recv--; 1113 } 1114 smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); 1115 } while (num_rkeys_send || num_rkeys_recv); 1116out: 1117 mutex_unlock(&lgr->rmbs_lock); 1118 return rc; 1119} 1120 1121static int smc_llc_srv_conf_link(struct smc_link *link, 1122 struct smc_link *link_new, 1123 enum smc_lgr_type lgr_new_t) 1124{ 1125 struct smc_link_group *lgr = link->lgr; 1126 struct smc_llc_qentry *qentry = NULL; 1127 int rc; 1128 1129 /* send CONFIRM LINK request over the RoCE fabric */ 1130 rc = smc_llc_send_confirm_link(link_new, SMC_LLC_REQ); 1131 if (rc) 1132 return -ENOLINK; 1133 /* receive CONFIRM LINK response over the RoCE fabric */ 1134 qentry = smc_llc_wait(lgr, link, SMC_LLC_WAIT_FIRST_TIME, 0); 1135 if (!qentry || 1136 qentry->msg.raw.hdr.common.type != SMC_LLC_CONFIRM_LINK) { 1137 /* send DELETE LINK */ 1138 smc_llc_send_delete_link(link, link_new->link_id, SMC_LLC_REQ, 1139 false, SMC_LLC_DEL_LOST_PATH); 1140 if (qentry) 1141 smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); 1142 return -ENOLINK; 1143 } 1144 smc_llc_save_peer_uid(qentry); 1145 smc_llc_link_active(link_new); 1146 if (lgr_new_t == SMC_LGR_ASYMMETRIC_LOCAL || 1147 lgr_new_t == SMC_LGR_ASYMMETRIC_PEER) 1148 smcr_lgr_set_type_asym(lgr, lgr_new_t, link_new->link_idx); 1149 else 1150 smcr_lgr_set_type(lgr, lgr_new_t); 1151 smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); 1152 return 0; 1153} 1154 1155int smc_llc_srv_add_link(struct smc_link *link) 1156{ 1157 enum smc_lgr_type lgr_new_t = SMC_LGR_SYMMETRIC; 1158 struct smc_link_group *lgr = link->lgr; 1159 struct smc_llc_msg_add_link *add_llc; 1160 struct smc_llc_qentry *qentry = NULL; 1161 struct smc_link *link_new; 1162 struct smc_init_info ini; 1163 int lnk_idx, rc = 0; 1164 1165 /* ignore client add link recommendation, start new flow */ 1166 ini.vlan_id = lgr->vlan_id; 1167 smc_pnet_find_alt_roce(lgr, &ini, link->smcibdev); 1168 if (!ini.ib_dev) { 1169 lgr_new_t = SMC_LGR_ASYMMETRIC_LOCAL; 1170 ini.ib_dev = link->smcibdev; 1171 ini.ib_port = link->ibport; 1172 } 1173 lnk_idx = smc_llc_alloc_alt_link(lgr, lgr_new_t); 1174 if (lnk_idx < 0) 1175 return 0; 1176 1177 rc = smcr_link_init(lgr, &lgr->lnk[lnk_idx], lnk_idx, &ini); 1178 if (rc) 1179 return rc; 1180 link_new = &lgr->lnk[lnk_idx]; 1181 rc = smc_llc_send_add_link(link, 1182 link_new->smcibdev->mac[ini.ib_port - 1], 1183 link_new->gid, link_new, SMC_LLC_REQ); 1184 if (rc) 1185 goto out_err; 1186 /* receive ADD LINK response over the RoCE fabric */ 1187 qentry = smc_llc_wait(lgr, link, SMC_LLC_WAIT_TIME, SMC_LLC_ADD_LINK); 1188 if (!qentry) { 1189 rc = -ETIMEDOUT; 1190 goto out_err; 1191 } 1192 add_llc = &qentry->msg.add_link; 1193 if (add_llc->hd.flags & SMC_LLC_FLAG_ADD_LNK_REJ) { 1194 smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); 1195 rc = -ENOLINK; 1196 goto out_err; 1197 } 1198 if (lgr->type == SMC_LGR_SINGLE && 1199 (!memcmp(add_llc->sender_gid, link->peer_gid, SMC_GID_SIZE) && 1200 !memcmp(add_llc->sender_mac, link->peer_mac, ETH_ALEN))) { 1201 lgr_new_t = SMC_LGR_ASYMMETRIC_PEER; 1202 } 1203 smc_llc_save_add_link_info(link_new, add_llc); 1204 smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); 1205 1206 rc = smc_ib_ready_link(link_new); 1207 if (rc) 1208 goto out_err; 1209 rc = smcr_buf_map_lgr(link_new); 1210 if (rc) 1211 goto out_err; 1212 rc = smcr_buf_reg_lgr(link_new); 1213 if (rc) 1214 goto out_err; 1215 rc = smc_llc_srv_rkey_exchange(link, link_new); 1216 if (rc) 1217 goto out_err; 1218 rc = smc_llc_srv_conf_link(link, link_new, lgr_new_t); 1219 if (rc) 1220 goto out_err; 1221 return 0; 1222out_err: 1223 link_new->state = SMC_LNK_INACTIVE; 1224 smcr_link_clear(link_new, false); 1225 return rc; 1226} 1227 1228static void smc_llc_process_srv_add_link(struct smc_link_group *lgr) 1229{ 1230 struct smc_link *link = lgr->llc_flow_lcl.qentry->link; 1231 int rc; 1232 1233 smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); 1234 1235 mutex_lock(&lgr->llc_conf_mutex); 1236 rc = smc_llc_srv_add_link(link); 1237 if (!rc && lgr->type == SMC_LGR_SYMMETRIC) { 1238 /* delete any asymmetric link */ 1239 smc_llc_delete_asym_link(lgr); 1240 } 1241 mutex_unlock(&lgr->llc_conf_mutex); 1242} 1243 1244/* enqueue a local add_link req to trigger a new add_link flow */ 1245void smc_llc_add_link_local(struct smc_link *link) 1246{ 1247 struct smc_llc_msg_add_link add_llc = {}; 1248 1249 add_llc.hd.length = sizeof(add_llc); 1250 add_llc.hd.common.type = SMC_LLC_ADD_LINK; 1251 /* no dev and port needed */ 1252 smc_llc_enqueue(link, (union smc_llc_msg *)&add_llc); 1253} 1254 1255/* worker to process an add link message */ 1256static void smc_llc_add_link_work(struct work_struct *work) 1257{ 1258 struct smc_link_group *lgr = container_of(work, struct smc_link_group, 1259 llc_add_link_work); 1260 1261 if (list_empty(&lgr->list)) { 1262 /* link group is terminating */ 1263 smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); 1264 goto out; 1265 } 1266 1267 if (lgr->role == SMC_CLNT) 1268 smc_llc_process_cli_add_link(lgr); 1269 else 1270 smc_llc_process_srv_add_link(lgr); 1271out: 1272 smc_llc_flow_stop(lgr, &lgr->llc_flow_lcl); 1273} 1274 1275/* enqueue a local del_link msg to trigger a new del_link flow, 1276 * called only for role SMC_SERV 1277 */ 1278void smc_llc_srv_delete_link_local(struct smc_link *link, u8 del_link_id) 1279{ 1280 struct smc_llc_msg_del_link del_llc = {}; 1281 1282 del_llc.hd.length = sizeof(del_llc); 1283 del_llc.hd.common.type = SMC_LLC_DELETE_LINK; 1284 del_llc.link_num = del_link_id; 1285 del_llc.reason = htonl(SMC_LLC_DEL_LOST_PATH); 1286 del_llc.hd.flags |= SMC_LLC_FLAG_DEL_LINK_ORDERLY; 1287 smc_llc_enqueue(link, (union smc_llc_msg *)&del_llc); 1288} 1289 1290static void smc_llc_process_cli_delete_link(struct smc_link_group *lgr) 1291{ 1292 struct smc_link *lnk_del = NULL, *lnk_asym, *lnk; 1293 struct smc_llc_msg_del_link *del_llc; 1294 struct smc_llc_qentry *qentry; 1295 int active_links; 1296 int lnk_idx; 1297 1298 qentry = smc_llc_flow_qentry_clr(&lgr->llc_flow_lcl); 1299 lnk = qentry->link; 1300 del_llc = &qentry->msg.delete_link; 1301 1302 if (del_llc->hd.flags & SMC_LLC_FLAG_DEL_LINK_ALL) { 1303 smc_lgr_terminate_sched(lgr); 1304 goto out; 1305 } 1306 mutex_lock(&lgr->llc_conf_mutex); 1307 /* delete single link */ 1308 for (lnk_idx = 0; lnk_idx < SMC_LINKS_PER_LGR_MAX; lnk_idx++) { 1309 if (lgr->lnk[lnk_idx].link_id != del_llc->link_num) 1310 continue; 1311 lnk_del = &lgr->lnk[lnk_idx]; 1312 break; 1313 } 1314 del_llc->hd.flags |= SMC_LLC_FLAG_RESP; 1315 if (!lnk_del) { 1316 /* link was not found */ 1317 del_llc->reason = htonl(SMC_LLC_DEL_NOLNK); 1318 smc_llc_send_message(lnk, &qentry->msg); 1319 goto out_unlock; 1320 } 1321 lnk_asym = smc_llc_find_asym_link(lgr); 1322 1323 del_llc->reason = 0; 1324 smc_llc_send_message(lnk, &qentry->msg); /* response */ 1325 1326 if (smc_link_downing(&lnk_del->state)) 1327 smc_switch_conns(lgr, lnk_del, false); 1328 smcr_link_clear(lnk_del, true); 1329 1330 active_links = smc_llc_active_link_count(lgr); 1331 if (lnk_del == lnk_asym) { 1332 /* expected deletion of asym link, don't change lgr state */ 1333 } else if (active_links == 1) { 1334 smcr_lgr_set_type(lgr, SMC_LGR_SINGLE); 1335 } else if (!active_links) { 1336 smcr_lgr_set_type(lgr, SMC_LGR_NONE); 1337 smc_lgr_terminate_sched(lgr); 1338 } 1339out_unlock: 1340 mutex_unlock(&lgr->llc_conf_mutex); 1341out: 1342 kfree(qentry); 1343} 1344 1345/* try to send a DELETE LINK ALL request on any active link, 1346 * waiting for send completion 1347 */ 1348void smc_llc_send_link_delete_all(struct smc_link_group *lgr, bool ord, u32 rsn) 1349{ 1350 struct smc_llc_msg_del_link delllc = {}; 1351 int i; 1352 1353 delllc.hd.common.type = SMC_LLC_DELETE_LINK; 1354 delllc.hd.length = sizeof(delllc); 1355 if (ord) 1356 delllc.hd.flags |= SMC_LLC_FLAG_DEL_LINK_ORDERLY; 1357 delllc.hd.flags |= SMC_LLC_FLAG_DEL_LINK_ALL; 1358 delllc.reason = htonl(rsn); 1359 1360 for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) { 1361 if (!smc_link_sendable(&lgr->lnk[i])) 1362 continue; 1363 if (!smc_llc_send_message_wait(&lgr->lnk[i], &delllc)) 1364 break; 1365 } 1366} 1367 1368static void smc_llc_process_srv_delete_link(struct smc_link_group *lgr) 1369{ 1370 struct smc_llc_msg_del_link *del_llc; 1371 struct smc_link *lnk, *lnk_del; 1372 struct smc_llc_qentry *qentry; 1373 int active_links; 1374 int i; 1375 1376 mutex_lock(&lgr->llc_conf_mutex); 1377 qentry = smc_llc_flow_qentry_clr(&lgr->llc_flow_lcl); 1378 lnk = qentry->link; 1379 del_llc = &qentry->msg.delete_link; 1380 1381 if (qentry->msg.delete_link.hd.flags & SMC_LLC_FLAG_DEL_LINK_ALL) { 1382 /* delete entire lgr */ 1383 smc_llc_send_link_delete_all(lgr, true, ntohl( 1384 qentry->msg.delete_link.reason)); 1385 smc_lgr_terminate_sched(lgr); 1386 goto out; 1387 } 1388 /* delete single link */ 1389 lnk_del = NULL; 1390 for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) { 1391 if (lgr->lnk[i].link_id == del_llc->link_num) { 1392 lnk_del = &lgr->lnk[i]; 1393 break; 1394 } 1395 } 1396 if (!lnk_del) 1397 goto out; /* asymmetric link already deleted */ 1398 1399 if (smc_link_downing(&lnk_del->state)) { 1400 if (smc_switch_conns(lgr, lnk_del, false)) 1401 smc_wr_tx_wait_no_pending_sends(lnk_del); 1402 } 1403 if (!list_empty(&lgr->list)) { 1404 /* qentry is either a request from peer (send it back to 1405 * initiate the DELETE_LINK processing), or a locally 1406 * enqueued DELETE_LINK request (forward it) 1407 */ 1408 if (!smc_llc_send_message(lnk, &qentry->msg)) { 1409 struct smc_llc_qentry *qentry2; 1410 1411 qentry2 = smc_llc_wait(lgr, lnk, SMC_LLC_WAIT_TIME, 1412 SMC_LLC_DELETE_LINK); 1413 if (qentry2) 1414 smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); 1415 } 1416 } 1417 smcr_link_clear(lnk_del, true); 1418 1419 active_links = smc_llc_active_link_count(lgr); 1420 if (active_links == 1) { 1421 smcr_lgr_set_type(lgr, SMC_LGR_SINGLE); 1422 } else if (!active_links) { 1423 smcr_lgr_set_type(lgr, SMC_LGR_NONE); 1424 smc_lgr_terminate_sched(lgr); 1425 } 1426 1427 if (lgr->type == SMC_LGR_SINGLE && !list_empty(&lgr->list)) { 1428 /* trigger setup of asymm alt link */ 1429 smc_llc_add_link_local(lnk); 1430 } 1431out: 1432 mutex_unlock(&lgr->llc_conf_mutex); 1433 kfree(qentry); 1434} 1435 1436static void smc_llc_delete_link_work(struct work_struct *work) 1437{ 1438 struct smc_link_group *lgr = container_of(work, struct smc_link_group, 1439 llc_del_link_work); 1440 1441 if (list_empty(&lgr->list)) { 1442 /* link group is terminating */ 1443 smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); 1444 goto out; 1445 } 1446 1447 if (lgr->role == SMC_CLNT) 1448 smc_llc_process_cli_delete_link(lgr); 1449 else 1450 smc_llc_process_srv_delete_link(lgr); 1451out: 1452 smc_llc_flow_stop(lgr, &lgr->llc_flow_lcl); 1453} 1454 1455/* process a confirm_rkey request from peer, remote flow */ 1456static void smc_llc_rmt_conf_rkey(struct smc_link_group *lgr) 1457{ 1458 struct smc_llc_msg_confirm_rkey *llc; 1459 struct smc_llc_qentry *qentry; 1460 struct smc_link *link; 1461 int num_entries; 1462 int rk_idx; 1463 int i; 1464 1465 qentry = lgr->llc_flow_rmt.qentry; 1466 llc = &qentry->msg.confirm_rkey; 1467 link = qentry->link; 1468 1469 num_entries = llc->rtoken[0].num_rkeys; 1470 /* first rkey entry is for receiving link */ 1471 rk_idx = smc_rtoken_add(link, 1472 llc->rtoken[0].rmb_vaddr, 1473 llc->rtoken[0].rmb_key); 1474 if (rk_idx < 0) 1475 goto out_err; 1476 1477 for (i = 1; i <= min_t(u8, num_entries, SMC_LLC_RKEYS_PER_MSG - 1); i++) 1478 smc_rtoken_set2(lgr, rk_idx, llc->rtoken[i].link_id, 1479 llc->rtoken[i].rmb_vaddr, 1480 llc->rtoken[i].rmb_key); 1481 /* max links is 3 so there is no need to support conf_rkey_cont msgs */ 1482 goto out; 1483out_err: 1484 llc->hd.flags |= SMC_LLC_FLAG_RKEY_NEG; 1485 llc->hd.flags |= SMC_LLC_FLAG_RKEY_RETRY; 1486out: 1487 llc->hd.flags |= SMC_LLC_FLAG_RESP; 1488 smc_llc_send_message(link, &qentry->msg); 1489 smc_llc_flow_qentry_del(&lgr->llc_flow_rmt); 1490} 1491 1492/* process a delete_rkey request from peer, remote flow */ 1493static void smc_llc_rmt_delete_rkey(struct smc_link_group *lgr) 1494{ 1495 struct smc_llc_msg_delete_rkey *llc; 1496 struct smc_llc_qentry *qentry; 1497 struct smc_link *link; 1498 u8 err_mask = 0; 1499 int i, max; 1500 1501 qentry = lgr->llc_flow_rmt.qentry; 1502 llc = &qentry->msg.delete_rkey; 1503 link = qentry->link; 1504 1505 max = min_t(u8, llc->num_rkeys, SMC_LLC_DEL_RKEY_MAX); 1506 for (i = 0; i < max; i++) { 1507 if (smc_rtoken_delete(link, llc->rkey[i])) 1508 err_mask |= 1 << (SMC_LLC_DEL_RKEY_MAX - 1 - i); 1509 } 1510 if (err_mask) { 1511 llc->hd.flags |= SMC_LLC_FLAG_RKEY_NEG; 1512 llc->err_mask = err_mask; 1513 } 1514 llc->hd.flags |= SMC_LLC_FLAG_RESP; 1515 smc_llc_send_message(link, &qentry->msg); 1516 smc_llc_flow_qentry_del(&lgr->llc_flow_rmt); 1517} 1518 1519static void smc_llc_protocol_violation(struct smc_link_group *lgr, u8 type) 1520{ 1521 pr_warn_ratelimited("smc: SMC-R lg %*phN LLC protocol violation: " 1522 "llc_type %d\n", SMC_LGR_ID_SIZE, &lgr->id, type); 1523 smc_llc_set_termination_rsn(lgr, SMC_LLC_DEL_PROT_VIOL); 1524 smc_lgr_terminate_sched(lgr); 1525} 1526 1527/* flush the llc event queue */ 1528static void smc_llc_event_flush(struct smc_link_group *lgr) 1529{ 1530 struct smc_llc_qentry *qentry, *q; 1531 1532 spin_lock_bh(&lgr->llc_event_q_lock); 1533 list_for_each_entry_safe(qentry, q, &lgr->llc_event_q, list) { 1534 list_del_init(&qentry->list); 1535 kfree(qentry); 1536 } 1537 spin_unlock_bh(&lgr->llc_event_q_lock); 1538} 1539 1540static void smc_llc_event_handler(struct smc_llc_qentry *qentry) 1541{ 1542 union smc_llc_msg *llc = &qentry->msg; 1543 struct smc_link *link = qentry->link; 1544 struct smc_link_group *lgr = link->lgr; 1545 1546 if (!smc_link_usable(link)) 1547 goto out; 1548 1549 switch (llc->raw.hdr.common.type) { 1550 case SMC_LLC_TEST_LINK: 1551 llc->test_link.hd.flags |= SMC_LLC_FLAG_RESP; 1552 smc_llc_send_message(link, llc); 1553 break; 1554 case SMC_LLC_ADD_LINK: 1555 if (list_empty(&lgr->list)) 1556 goto out; /* lgr is terminating */ 1557 if (lgr->role == SMC_CLNT) { 1558 if (smc_llc_is_local_add_link(llc)) { 1559 if (lgr->llc_flow_lcl.type == 1560 SMC_LLC_FLOW_ADD_LINK) 1561 break; /* add_link in progress */ 1562 if (smc_llc_flow_start(&lgr->llc_flow_lcl, 1563 qentry)) { 1564 schedule_work(&lgr->llc_add_link_work); 1565 } 1566 return; 1567 } 1568 if (lgr->llc_flow_lcl.type == SMC_LLC_FLOW_ADD_LINK && 1569 !lgr->llc_flow_lcl.qentry) { 1570 /* a flow is waiting for this message */ 1571 smc_llc_flow_qentry_set(&lgr->llc_flow_lcl, 1572 qentry); 1573 wake_up(&lgr->llc_msg_waiter); 1574 } else if (smc_llc_flow_start(&lgr->llc_flow_lcl, 1575 qentry)) { 1576 schedule_work(&lgr->llc_add_link_work); 1577 } 1578 } else if (smc_llc_flow_start(&lgr->llc_flow_lcl, qentry)) { 1579 /* as smc server, handle client suggestion */ 1580 schedule_work(&lgr->llc_add_link_work); 1581 } 1582 return; 1583 case SMC_LLC_CONFIRM_LINK: 1584 case SMC_LLC_ADD_LINK_CONT: 1585 if (lgr->llc_flow_lcl.type != SMC_LLC_FLOW_NONE) { 1586 /* a flow is waiting for this message */ 1587 smc_llc_flow_qentry_set(&lgr->llc_flow_lcl, qentry); 1588 wake_up(&lgr->llc_msg_waiter); 1589 return; 1590 } 1591 break; 1592 case SMC_LLC_DELETE_LINK: 1593 if (lgr->llc_flow_lcl.type == SMC_LLC_FLOW_ADD_LINK && 1594 !lgr->llc_flow_lcl.qentry) { 1595 /* DEL LINK REQ during ADD LINK SEQ */ 1596 smc_llc_flow_qentry_set(&lgr->llc_flow_lcl, qentry); 1597 wake_up(&lgr->llc_msg_waiter); 1598 } else if (smc_llc_flow_start(&lgr->llc_flow_lcl, qentry)) { 1599 schedule_work(&lgr->llc_del_link_work); 1600 } 1601 return; 1602 case SMC_LLC_CONFIRM_RKEY: 1603 /* new request from remote, assign to remote flow */ 1604 if (smc_llc_flow_start(&lgr->llc_flow_rmt, qentry)) { 1605 /* process here, does not wait for more llc msgs */ 1606 smc_llc_rmt_conf_rkey(lgr); 1607 smc_llc_flow_stop(lgr, &lgr->llc_flow_rmt); 1608 } 1609 return; 1610 case SMC_LLC_CONFIRM_RKEY_CONT: 1611 /* not used because max links is 3, and 3 rkeys fit into 1612 * one CONFIRM_RKEY message 1613 */ 1614 break; 1615 case SMC_LLC_DELETE_RKEY: 1616 /* new request from remote, assign to remote flow */ 1617 if (smc_llc_flow_start(&lgr->llc_flow_rmt, qentry)) { 1618 /* process here, does not wait for more llc msgs */ 1619 smc_llc_rmt_delete_rkey(lgr); 1620 smc_llc_flow_stop(lgr, &lgr->llc_flow_rmt); 1621 } 1622 return; 1623 default: 1624 smc_llc_protocol_violation(lgr, llc->raw.hdr.common.type); 1625 break; 1626 } 1627out: 1628 kfree(qentry); 1629} 1630 1631/* worker to process llc messages on the event queue */ 1632static void smc_llc_event_work(struct work_struct *work) 1633{ 1634 struct smc_link_group *lgr = container_of(work, struct smc_link_group, 1635 llc_event_work); 1636 struct smc_llc_qentry *qentry; 1637 1638 if (!lgr->llc_flow_lcl.type && lgr->delayed_event) { 1639 qentry = lgr->delayed_event; 1640 lgr->delayed_event = NULL; 1641 if (smc_link_usable(qentry->link)) 1642 smc_llc_event_handler(qentry); 1643 else 1644 kfree(qentry); 1645 } 1646 1647again: 1648 spin_lock_bh(&lgr->llc_event_q_lock); 1649 if (!list_empty(&lgr->llc_event_q)) { 1650 qentry = list_first_entry(&lgr->llc_event_q, 1651 struct smc_llc_qentry, list); 1652 list_del_init(&qentry->list); 1653 spin_unlock_bh(&lgr->llc_event_q_lock); 1654 smc_llc_event_handler(qentry); 1655 goto again; 1656 } 1657 spin_unlock_bh(&lgr->llc_event_q_lock); 1658} 1659 1660/* process llc responses in tasklet context */ 1661static void smc_llc_rx_response(struct smc_link *link, 1662 struct smc_llc_qentry *qentry) 1663{ 1664 enum smc_llc_flowtype flowtype = link->lgr->llc_flow_lcl.type; 1665 struct smc_llc_flow *flow = &link->lgr->llc_flow_lcl; 1666 u8 llc_type = qentry->msg.raw.hdr.common.type; 1667 1668 switch (llc_type) { 1669 case SMC_LLC_TEST_LINK: 1670 if (smc_link_active(link)) 1671 complete(&link->llc_testlink_resp); 1672 break; 1673 case SMC_LLC_ADD_LINK: 1674 case SMC_LLC_ADD_LINK_CONT: 1675 case SMC_LLC_CONFIRM_LINK: 1676 if (flowtype != SMC_LLC_FLOW_ADD_LINK || flow->qentry) 1677 break; /* drop out-of-flow response */ 1678 goto assign; 1679 case SMC_LLC_DELETE_LINK: 1680 if (flowtype != SMC_LLC_FLOW_DEL_LINK || flow->qentry) 1681 break; /* drop out-of-flow response */ 1682 goto assign; 1683 case SMC_LLC_CONFIRM_RKEY: 1684 case SMC_LLC_DELETE_RKEY: 1685 if (flowtype != SMC_LLC_FLOW_RKEY || flow->qentry) 1686 break; /* drop out-of-flow response */ 1687 goto assign; 1688 case SMC_LLC_CONFIRM_RKEY_CONT: 1689 /* not used because max links is 3 */ 1690 break; 1691 default: 1692 smc_llc_protocol_violation(link->lgr, llc_type); 1693 break; 1694 } 1695 kfree(qentry); 1696 return; 1697assign: 1698 /* assign responses to the local flow, we requested them */ 1699 smc_llc_flow_qentry_set(&link->lgr->llc_flow_lcl, qentry); 1700 wake_up(&link->lgr->llc_msg_waiter); 1701} 1702 1703static void smc_llc_enqueue(struct smc_link *link, union smc_llc_msg *llc) 1704{ 1705 struct smc_link_group *lgr = link->lgr; 1706 struct smc_llc_qentry *qentry; 1707 unsigned long flags; 1708 1709 qentry = kmalloc(sizeof(*qentry), GFP_ATOMIC); 1710 if (!qentry) 1711 return; 1712 qentry->link = link; 1713 INIT_LIST_HEAD(&qentry->list); 1714 memcpy(&qentry->msg, llc, sizeof(union smc_llc_msg)); 1715 1716 /* process responses immediately */ 1717 if (llc->raw.hdr.flags & SMC_LLC_FLAG_RESP) { 1718 smc_llc_rx_response(link, qentry); 1719 return; 1720 } 1721 1722 /* add requests to event queue */ 1723 spin_lock_irqsave(&lgr->llc_event_q_lock, flags); 1724 list_add_tail(&qentry->list, &lgr->llc_event_q); 1725 spin_unlock_irqrestore(&lgr->llc_event_q_lock, flags); 1726 queue_work(system_highpri_wq, &lgr->llc_event_work); 1727} 1728 1729/* copy received msg and add it to the event queue */ 1730static void smc_llc_rx_handler(struct ib_wc *wc, void *buf) 1731{ 1732 struct smc_link *link = (struct smc_link *)wc->qp->qp_context; 1733 union smc_llc_msg *llc = buf; 1734 1735 if (wc->byte_len < sizeof(*llc)) 1736 return; /* short message */ 1737 if (llc->raw.hdr.length != sizeof(*llc)) 1738 return; /* invalid message */ 1739 1740 smc_llc_enqueue(link, llc); 1741} 1742 1743/***************************** worker, utils *********************************/ 1744 1745static void smc_llc_testlink_work(struct work_struct *work) 1746{ 1747 struct smc_link *link = container_of(to_delayed_work(work), 1748 struct smc_link, llc_testlink_wrk); 1749 unsigned long next_interval; 1750 unsigned long expire_time; 1751 u8 user_data[16] = { 0 }; 1752 int rc; 1753 1754 if (!smc_link_active(link)) 1755 return; /* don't reschedule worker */ 1756 expire_time = link->wr_rx_tstamp + link->llc_testlink_time; 1757 if (time_is_after_jiffies(expire_time)) { 1758 next_interval = expire_time - jiffies; 1759 goto out; 1760 } 1761 reinit_completion(&link->llc_testlink_resp); 1762 smc_llc_send_test_link(link, user_data); 1763 /* receive TEST LINK response over RoCE fabric */ 1764 rc = wait_for_completion_interruptible_timeout(&link->llc_testlink_resp, 1765 SMC_LLC_WAIT_TIME); 1766 if (!smc_link_active(link)) 1767 return; /* link state changed */ 1768 if (rc <= 0) { 1769 smcr_link_down_cond_sched(link); 1770 return; 1771 } 1772 next_interval = link->llc_testlink_time; 1773out: 1774 schedule_delayed_work(&link->llc_testlink_wrk, next_interval); 1775} 1776 1777void smc_llc_lgr_init(struct smc_link_group *lgr, struct smc_sock *smc) 1778{ 1779 struct net *net = sock_net(smc->clcsock->sk); 1780 1781 INIT_WORK(&lgr->llc_event_work, smc_llc_event_work); 1782 INIT_WORK(&lgr->llc_add_link_work, smc_llc_add_link_work); 1783 INIT_WORK(&lgr->llc_del_link_work, smc_llc_delete_link_work); 1784 INIT_LIST_HEAD(&lgr->llc_event_q); 1785 spin_lock_init(&lgr->llc_event_q_lock); 1786 spin_lock_init(&lgr->llc_flow_lock); 1787 init_waitqueue_head(&lgr->llc_flow_waiter); 1788 init_waitqueue_head(&lgr->llc_msg_waiter); 1789 mutex_init(&lgr->llc_conf_mutex); 1790 lgr->llc_testlink_time = READ_ONCE(net->ipv4.sysctl_tcp_keepalive_time); 1791} 1792 1793/* called after lgr was removed from lgr_list */ 1794void smc_llc_lgr_clear(struct smc_link_group *lgr) 1795{ 1796 smc_llc_event_flush(lgr); 1797 wake_up_all(&lgr->llc_flow_waiter); 1798 wake_up_all(&lgr->llc_msg_waiter); 1799 cancel_work_sync(&lgr->llc_event_work); 1800 cancel_work_sync(&lgr->llc_add_link_work); 1801 cancel_work_sync(&lgr->llc_del_link_work); 1802 if (lgr->delayed_event) { 1803 kfree(lgr->delayed_event); 1804 lgr->delayed_event = NULL; 1805 } 1806} 1807 1808int smc_llc_link_init(struct smc_link *link) 1809{ 1810 init_completion(&link->llc_testlink_resp); 1811 INIT_DELAYED_WORK(&link->llc_testlink_wrk, smc_llc_testlink_work); 1812 return 0; 1813} 1814 1815void smc_llc_link_active(struct smc_link *link) 1816{ 1817 pr_warn_ratelimited("smc: SMC-R lg %*phN link added: id %*phN, " 1818 "peerid %*phN, ibdev %s, ibport %d\n", 1819 SMC_LGR_ID_SIZE, &link->lgr->id, 1820 SMC_LGR_ID_SIZE, &link->link_uid, 1821 SMC_LGR_ID_SIZE, &link->peer_link_uid, 1822 link->smcibdev->ibdev->name, link->ibport); 1823 link->state = SMC_LNK_ACTIVE; 1824 if (link->lgr->llc_testlink_time) { 1825 link->llc_testlink_time = link->lgr->llc_testlink_time; 1826 schedule_delayed_work(&link->llc_testlink_wrk, 1827 link->llc_testlink_time); 1828 } 1829} 1830 1831/* called in worker context */ 1832void smc_llc_link_clear(struct smc_link *link, bool log) 1833{ 1834 if (log) 1835 pr_warn_ratelimited("smc: SMC-R lg %*phN link removed: id %*phN" 1836 ", peerid %*phN, ibdev %s, ibport %d\n", 1837 SMC_LGR_ID_SIZE, &link->lgr->id, 1838 SMC_LGR_ID_SIZE, &link->link_uid, 1839 SMC_LGR_ID_SIZE, &link->peer_link_uid, 1840 link->smcibdev->ibdev->name, link->ibport); 1841 complete(&link->llc_testlink_resp); 1842 cancel_delayed_work_sync(&link->llc_testlink_wrk); 1843} 1844 1845/* register a new rtoken at the remote peer (for all links) */ 1846int smc_llc_do_confirm_rkey(struct smc_link *send_link, 1847 struct smc_buf_desc *rmb_desc) 1848{ 1849 struct smc_link_group *lgr = send_link->lgr; 1850 struct smc_llc_qentry *qentry = NULL; 1851 int rc = 0; 1852 1853 rc = smc_llc_send_confirm_rkey(send_link, rmb_desc); 1854 if (rc) 1855 goto out; 1856 /* receive CONFIRM RKEY response from server over RoCE fabric */ 1857 qentry = smc_llc_wait(lgr, send_link, SMC_LLC_WAIT_TIME, 1858 SMC_LLC_CONFIRM_RKEY); 1859 if (!qentry || (qentry->msg.raw.hdr.flags & SMC_LLC_FLAG_RKEY_NEG)) 1860 rc = -EFAULT; 1861out: 1862 if (qentry) 1863 smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); 1864 return rc; 1865} 1866 1867/* unregister an rtoken at the remote peer */ 1868int smc_llc_do_delete_rkey(struct smc_link_group *lgr, 1869 struct smc_buf_desc *rmb_desc) 1870{ 1871 struct smc_llc_qentry *qentry = NULL; 1872 struct smc_link *send_link; 1873 int rc = 0; 1874 1875 send_link = smc_llc_usable_link(lgr); 1876 if (!send_link) 1877 return -ENOLINK; 1878 1879 /* protected by llc_flow control */ 1880 rc = smc_llc_send_delete_rkey(send_link, rmb_desc); 1881 if (rc) 1882 goto out; 1883 /* receive DELETE RKEY response from server over RoCE fabric */ 1884 qentry = smc_llc_wait(lgr, send_link, SMC_LLC_WAIT_TIME, 1885 SMC_LLC_DELETE_RKEY); 1886 if (!qentry || (qentry->msg.raw.hdr.flags & SMC_LLC_FLAG_RKEY_NEG)) 1887 rc = -EFAULT; 1888out: 1889 if (qentry) 1890 smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); 1891 return rc; 1892} 1893 1894void smc_llc_link_set_uid(struct smc_link *link) 1895{ 1896 __be32 link_uid; 1897 1898 link_uid = htonl(*((u32 *)link->lgr->id) + link->link_id); 1899 memcpy(link->link_uid, &link_uid, SMC_LGR_ID_SIZE); 1900} 1901 1902/* save peers link user id, used for debug purposes */ 1903void smc_llc_save_peer_uid(struct smc_llc_qentry *qentry) 1904{ 1905 memcpy(qentry->link->peer_link_uid, qentry->msg.confirm_link.link_uid, 1906 SMC_LGR_ID_SIZE); 1907} 1908 1909/* evaluate confirm link request or response */ 1910int smc_llc_eval_conf_link(struct smc_llc_qentry *qentry, 1911 enum smc_llc_reqresp type) 1912{ 1913 if (type == SMC_LLC_REQ) { /* SMC server assigns link_id */ 1914 qentry->link->link_id = qentry->msg.confirm_link.link_num; 1915 smc_llc_link_set_uid(qentry->link); 1916 } 1917 if (!(qentry->msg.raw.hdr.flags & SMC_LLC_FLAG_NO_RMBE_EYEC)) 1918 return -ENOTSUPP; 1919 return 0; 1920} 1921 1922/***************************** init, exit, misc ******************************/ 1923 1924static struct smc_wr_rx_handler smc_llc_rx_handlers[] = { 1925 { 1926 .handler = smc_llc_rx_handler, 1927 .type = SMC_LLC_CONFIRM_LINK 1928 }, 1929 { 1930 .handler = smc_llc_rx_handler, 1931 .type = SMC_LLC_TEST_LINK 1932 }, 1933 { 1934 .handler = smc_llc_rx_handler, 1935 .type = SMC_LLC_ADD_LINK 1936 }, 1937 { 1938 .handler = smc_llc_rx_handler, 1939 .type = SMC_LLC_ADD_LINK_CONT 1940 }, 1941 { 1942 .handler = smc_llc_rx_handler, 1943 .type = SMC_LLC_DELETE_LINK 1944 }, 1945 { 1946 .handler = smc_llc_rx_handler, 1947 .type = SMC_LLC_CONFIRM_RKEY 1948 }, 1949 { 1950 .handler = smc_llc_rx_handler, 1951 .type = SMC_LLC_CONFIRM_RKEY_CONT 1952 }, 1953 { 1954 .handler = smc_llc_rx_handler, 1955 .type = SMC_LLC_DELETE_RKEY 1956 }, 1957 { 1958 .handler = NULL, 1959 } 1960}; 1961 1962int __init smc_llc_init(void) 1963{ 1964 struct smc_wr_rx_handler *handler; 1965 int rc = 0; 1966 1967 for (handler = smc_llc_rx_handlers; handler->handler; handler++) { 1968 INIT_HLIST_NODE(&handler->list); 1969 rc = smc_wr_rx_register_handler(handler); 1970 if (rc) 1971 break; 1972 } 1973 return rc; 1974} 1975