1// SPDX-License-Identifier: GPL-2.0-only 2/* 3 * Copyright (c) 2016 Chelsio Communications, Inc. 4 */ 5 6#include <linux/module.h> 7#include <linux/list.h> 8#include <linux/workqueue.h> 9#include <linux/skbuff.h> 10#include <linux/timer.h> 11#include <linux/notifier.h> 12#include <linux/inetdevice.h> 13#include <linux/ip.h> 14#include <linux/tcp.h> 15#include <linux/if_vlan.h> 16 17#include <net/neighbour.h> 18#include <net/netevent.h> 19#include <net/route.h> 20#include <net/tcp.h> 21#include <net/ip6_route.h> 22#include <net/addrconf.h> 23 24#include <libcxgb_cm.h> 25#include "cxgbit.h" 26#include "clip_tbl.h" 27 28static void cxgbit_init_wr_wait(struct cxgbit_wr_wait *wr_waitp) 29{ 30 wr_waitp->ret = 0; 31 reinit_completion(&wr_waitp->completion); 32} 33 34static void 35cxgbit_wake_up(struct cxgbit_wr_wait *wr_waitp, const char *func, u8 ret) 36{ 37 if (ret == CPL_ERR_NONE) 38 wr_waitp->ret = 0; 39 else 40 wr_waitp->ret = -EIO; 41 42 if (wr_waitp->ret) 43 pr_err("%s: err:%u", func, ret); 44 45 complete(&wr_waitp->completion); 46} 47 48static int 49cxgbit_wait_for_reply(struct cxgbit_device *cdev, 50 struct cxgbit_wr_wait *wr_waitp, u32 tid, u32 timeout, 51 const char *func) 52{ 53 int ret; 54 55 if (!test_bit(CDEV_STATE_UP, &cdev->flags)) { 56 wr_waitp->ret = -EIO; 57 goto out; 58 } 59 60 ret = wait_for_completion_timeout(&wr_waitp->completion, timeout * HZ); 61 if (!ret) { 62 pr_info("%s - Device %s not responding tid %u\n", 63 func, pci_name(cdev->lldi.pdev), tid); 64 wr_waitp->ret = -ETIMEDOUT; 65 } 66out: 67 if (wr_waitp->ret) 68 pr_info("%s: FW reply %d tid %u\n", 69 pci_name(cdev->lldi.pdev), wr_waitp->ret, tid); 70 return wr_waitp->ret; 71} 72 73static int cxgbit_np_hashfn(const struct cxgbit_np *cnp) 74{ 75 return ((unsigned long)cnp >> 10) & (NP_INFO_HASH_SIZE - 1); 76} 77 78static struct np_info * 79cxgbit_np_hash_add(struct cxgbit_device *cdev, struct cxgbit_np *cnp, 80 unsigned int stid) 81{ 82 struct np_info *p = kzalloc(sizeof(*p), GFP_KERNEL); 83 84 if (p) { 85 int bucket = cxgbit_np_hashfn(cnp); 86 87 p->cnp = cnp; 88 p->stid = stid; 89 spin_lock(&cdev->np_lock); 90 p->next = cdev->np_hash_tab[bucket]; 91 cdev->np_hash_tab[bucket] = p; 92 spin_unlock(&cdev->np_lock); 93 } 94 95 return p; 96} 97 98static int 99cxgbit_np_hash_find(struct cxgbit_device *cdev, struct cxgbit_np *cnp) 100{ 101 int stid = -1, bucket = cxgbit_np_hashfn(cnp); 102 struct np_info *p; 103 104 spin_lock(&cdev->np_lock); 105 for (p = cdev->np_hash_tab[bucket]; p; p = p->next) { 106 if (p->cnp == cnp) { 107 stid = p->stid; 108 break; 109 } 110 } 111 spin_unlock(&cdev->np_lock); 112 113 return stid; 114} 115 116static int cxgbit_np_hash_del(struct cxgbit_device *cdev, struct cxgbit_np *cnp) 117{ 118 int stid = -1, bucket = cxgbit_np_hashfn(cnp); 119 struct np_info *p, **prev = &cdev->np_hash_tab[bucket]; 120 121 spin_lock(&cdev->np_lock); 122 for (p = *prev; p; prev = &p->next, p = p->next) { 123 if (p->cnp == cnp) { 124 stid = p->stid; 125 *prev = p->next; 126 kfree(p); 127 break; 128 } 129 } 130 spin_unlock(&cdev->np_lock); 131 132 return stid; 133} 134 135void _cxgbit_free_cnp(struct kref *kref) 136{ 137 struct cxgbit_np *cnp; 138 139 cnp = container_of(kref, struct cxgbit_np, kref); 140 kfree(cnp); 141} 142 143static int 144cxgbit_create_server6(struct cxgbit_device *cdev, unsigned int stid, 145 struct cxgbit_np *cnp) 146{ 147 struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *) 148 &cnp->com.local_addr; 149 int addr_type; 150 int ret; 151 152 pr_debug("%s: dev = %s; stid = %u; sin6_port = %u\n", 153 __func__, cdev->lldi.ports[0]->name, stid, sin6->sin6_port); 154 155 addr_type = ipv6_addr_type((const struct in6_addr *) 156 &sin6->sin6_addr); 157 if (addr_type != IPV6_ADDR_ANY) { 158 ret = cxgb4_clip_get(cdev->lldi.ports[0], 159 (const u32 *)&sin6->sin6_addr.s6_addr, 1); 160 if (ret) { 161 pr_err("Unable to find clip table entry. laddr %pI6. Error:%d.\n", 162 sin6->sin6_addr.s6_addr, ret); 163 return -ENOMEM; 164 } 165 } 166 167 cxgbit_get_cnp(cnp); 168 cxgbit_init_wr_wait(&cnp->com.wr_wait); 169 170 ret = cxgb4_create_server6(cdev->lldi.ports[0], 171 stid, &sin6->sin6_addr, 172 sin6->sin6_port, 173 cdev->lldi.rxq_ids[0]); 174 if (!ret) 175 ret = cxgbit_wait_for_reply(cdev, &cnp->com.wr_wait, 176 0, 10, __func__); 177 else if (ret > 0) 178 ret = net_xmit_errno(ret); 179 else 180 cxgbit_put_cnp(cnp); 181 182 if (ret) { 183 if (ret != -ETIMEDOUT) 184 cxgb4_clip_release(cdev->lldi.ports[0], 185 (const u32 *)&sin6->sin6_addr.s6_addr, 1); 186 187 pr_err("create server6 err %d stid %d laddr %pI6 lport %d\n", 188 ret, stid, sin6->sin6_addr.s6_addr, 189 ntohs(sin6->sin6_port)); 190 } 191 192 return ret; 193} 194 195static int 196cxgbit_create_server4(struct cxgbit_device *cdev, unsigned int stid, 197 struct cxgbit_np *cnp) 198{ 199 struct sockaddr_in *sin = (struct sockaddr_in *) 200 &cnp->com.local_addr; 201 int ret; 202 203 pr_debug("%s: dev = %s; stid = %u; sin_port = %u\n", 204 __func__, cdev->lldi.ports[0]->name, stid, sin->sin_port); 205 206 cxgbit_get_cnp(cnp); 207 cxgbit_init_wr_wait(&cnp->com.wr_wait); 208 209 ret = cxgb4_create_server(cdev->lldi.ports[0], 210 stid, sin->sin_addr.s_addr, 211 sin->sin_port, 0, 212 cdev->lldi.rxq_ids[0]); 213 if (!ret) 214 ret = cxgbit_wait_for_reply(cdev, 215 &cnp->com.wr_wait, 216 0, 10, __func__); 217 else if (ret > 0) 218 ret = net_xmit_errno(ret); 219 else 220 cxgbit_put_cnp(cnp); 221 222 if (ret) 223 pr_err("create server failed err %d stid %d laddr %pI4 lport %d\n", 224 ret, stid, &sin->sin_addr, ntohs(sin->sin_port)); 225 return ret; 226} 227 228struct cxgbit_device *cxgbit_find_device(struct net_device *ndev, u8 *port_id) 229{ 230 struct cxgbit_device *cdev; 231 u8 i; 232 233 list_for_each_entry(cdev, &cdev_list_head, list) { 234 struct cxgb4_lld_info *lldi = &cdev->lldi; 235 236 for (i = 0; i < lldi->nports; i++) { 237 if (lldi->ports[i] == ndev) { 238 if (port_id) 239 *port_id = i; 240 return cdev; 241 } 242 } 243 } 244 245 return NULL; 246} 247 248static struct net_device *cxgbit_get_real_dev(struct net_device *ndev) 249{ 250 if (ndev->priv_flags & IFF_BONDING) { 251 pr_err("Bond devices are not supported. Interface:%s\n", 252 ndev->name); 253 return NULL; 254 } 255 256 if (is_vlan_dev(ndev)) 257 return vlan_dev_real_dev(ndev); 258 259 return ndev; 260} 261 262static struct net_device *cxgbit_ipv4_netdev(__be32 saddr) 263{ 264 struct net_device *ndev; 265 266 ndev = __ip_dev_find(&init_net, saddr, false); 267 if (!ndev) 268 return NULL; 269 270 return cxgbit_get_real_dev(ndev); 271} 272 273static struct net_device *cxgbit_ipv6_netdev(struct in6_addr *addr6) 274{ 275 struct net_device *ndev = NULL; 276 bool found = false; 277 278 if (IS_ENABLED(CONFIG_IPV6)) { 279 for_each_netdev_rcu(&init_net, ndev) 280 if (ipv6_chk_addr(&init_net, addr6, ndev, 1)) { 281 found = true; 282 break; 283 } 284 } 285 if (!found) 286 return NULL; 287 return cxgbit_get_real_dev(ndev); 288} 289 290static struct cxgbit_device *cxgbit_find_np_cdev(struct cxgbit_np *cnp) 291{ 292 struct sockaddr_storage *sockaddr = &cnp->com.local_addr; 293 int ss_family = sockaddr->ss_family; 294 struct net_device *ndev = NULL; 295 struct cxgbit_device *cdev = NULL; 296 297 rcu_read_lock(); 298 if (ss_family == AF_INET) { 299 struct sockaddr_in *sin; 300 301 sin = (struct sockaddr_in *)sockaddr; 302 ndev = cxgbit_ipv4_netdev(sin->sin_addr.s_addr); 303 } else if (ss_family == AF_INET6) { 304 struct sockaddr_in6 *sin6; 305 306 sin6 = (struct sockaddr_in6 *)sockaddr; 307 ndev = cxgbit_ipv6_netdev(&sin6->sin6_addr); 308 } 309 if (!ndev) 310 goto out; 311 312 cdev = cxgbit_find_device(ndev, NULL); 313out: 314 rcu_read_unlock(); 315 return cdev; 316} 317 318static bool cxgbit_inaddr_any(struct cxgbit_np *cnp) 319{ 320 struct sockaddr_storage *sockaddr = &cnp->com.local_addr; 321 int ss_family = sockaddr->ss_family; 322 int addr_type; 323 324 if (ss_family == AF_INET) { 325 struct sockaddr_in *sin; 326 327 sin = (struct sockaddr_in *)sockaddr; 328 if (sin->sin_addr.s_addr == htonl(INADDR_ANY)) 329 return true; 330 } else if (ss_family == AF_INET6) { 331 struct sockaddr_in6 *sin6; 332 333 sin6 = (struct sockaddr_in6 *)sockaddr; 334 addr_type = ipv6_addr_type((const struct in6_addr *) 335 &sin6->sin6_addr); 336 if (addr_type == IPV6_ADDR_ANY) 337 return true; 338 } 339 return false; 340} 341 342static int 343__cxgbit_setup_cdev_np(struct cxgbit_device *cdev, struct cxgbit_np *cnp) 344{ 345 int stid, ret; 346 int ss_family = cnp->com.local_addr.ss_family; 347 348 if (!test_bit(CDEV_STATE_UP, &cdev->flags)) 349 return -EINVAL; 350 351 stid = cxgb4_alloc_stid(cdev->lldi.tids, ss_family, cnp); 352 if (stid < 0) 353 return -EINVAL; 354 355 if (!cxgbit_np_hash_add(cdev, cnp, stid)) { 356 cxgb4_free_stid(cdev->lldi.tids, stid, ss_family); 357 return -EINVAL; 358 } 359 360 if (ss_family == AF_INET) 361 ret = cxgbit_create_server4(cdev, stid, cnp); 362 else 363 ret = cxgbit_create_server6(cdev, stid, cnp); 364 365 if (ret) { 366 if (ret != -ETIMEDOUT) 367 cxgb4_free_stid(cdev->lldi.tids, stid, 368 ss_family); 369 cxgbit_np_hash_del(cdev, cnp); 370 return ret; 371 } 372 return ret; 373} 374 375static int cxgbit_setup_cdev_np(struct cxgbit_np *cnp) 376{ 377 struct cxgbit_device *cdev; 378 int ret = -1; 379 380 mutex_lock(&cdev_list_lock); 381 cdev = cxgbit_find_np_cdev(cnp); 382 if (!cdev) 383 goto out; 384 385 if (cxgbit_np_hash_find(cdev, cnp) >= 0) 386 goto out; 387 388 if (__cxgbit_setup_cdev_np(cdev, cnp)) 389 goto out; 390 391 cnp->com.cdev = cdev; 392 ret = 0; 393out: 394 mutex_unlock(&cdev_list_lock); 395 return ret; 396} 397 398static int cxgbit_setup_all_np(struct cxgbit_np *cnp) 399{ 400 struct cxgbit_device *cdev; 401 int ret; 402 u32 count = 0; 403 404 mutex_lock(&cdev_list_lock); 405 list_for_each_entry(cdev, &cdev_list_head, list) { 406 if (cxgbit_np_hash_find(cdev, cnp) >= 0) { 407 mutex_unlock(&cdev_list_lock); 408 return -1; 409 } 410 } 411 412 list_for_each_entry(cdev, &cdev_list_head, list) { 413 ret = __cxgbit_setup_cdev_np(cdev, cnp); 414 if (ret == -ETIMEDOUT) 415 break; 416 if (ret != 0) 417 continue; 418 count++; 419 } 420 mutex_unlock(&cdev_list_lock); 421 422 return count ? 0 : -1; 423} 424 425int cxgbit_setup_np(struct iscsi_np *np, struct sockaddr_storage *ksockaddr) 426{ 427 struct cxgbit_np *cnp; 428 int ret; 429 430 if ((ksockaddr->ss_family != AF_INET) && 431 (ksockaddr->ss_family != AF_INET6)) 432 return -EINVAL; 433 434 cnp = kzalloc(sizeof(*cnp), GFP_KERNEL); 435 if (!cnp) 436 return -ENOMEM; 437 438 init_waitqueue_head(&cnp->accept_wait); 439 init_completion(&cnp->com.wr_wait.completion); 440 init_completion(&cnp->accept_comp); 441 INIT_LIST_HEAD(&cnp->np_accept_list); 442 spin_lock_init(&cnp->np_accept_lock); 443 kref_init(&cnp->kref); 444 memcpy(&np->np_sockaddr, ksockaddr, 445 sizeof(struct sockaddr_storage)); 446 memcpy(&cnp->com.local_addr, &np->np_sockaddr, 447 sizeof(cnp->com.local_addr)); 448 449 cnp->np = np; 450 cnp->com.cdev = NULL; 451 452 if (cxgbit_inaddr_any(cnp)) 453 ret = cxgbit_setup_all_np(cnp); 454 else 455 ret = cxgbit_setup_cdev_np(cnp); 456 457 if (ret) { 458 cxgbit_put_cnp(cnp); 459 return -EINVAL; 460 } 461 462 np->np_context = cnp; 463 cnp->com.state = CSK_STATE_LISTEN; 464 return 0; 465} 466 467static void 468cxgbit_set_conn_info(struct iscsi_np *np, struct iscsit_conn *conn, 469 struct cxgbit_sock *csk) 470{ 471 conn->login_family = np->np_sockaddr.ss_family; 472 conn->login_sockaddr = csk->com.remote_addr; 473 conn->local_sockaddr = csk->com.local_addr; 474} 475 476int cxgbit_accept_np(struct iscsi_np *np, struct iscsit_conn *conn) 477{ 478 struct cxgbit_np *cnp = np->np_context; 479 struct cxgbit_sock *csk; 480 int ret = 0; 481 482accept_wait: 483 ret = wait_for_completion_interruptible(&cnp->accept_comp); 484 if (ret) 485 return -ENODEV; 486 487 spin_lock_bh(&np->np_thread_lock); 488 if (np->np_thread_state >= ISCSI_NP_THREAD_RESET) { 489 spin_unlock_bh(&np->np_thread_lock); 490 /** 491 * No point in stalling here when np_thread 492 * is in state RESET/SHUTDOWN/EXIT - bail 493 **/ 494 return -ENODEV; 495 } 496 spin_unlock_bh(&np->np_thread_lock); 497 498 spin_lock_bh(&cnp->np_accept_lock); 499 if (list_empty(&cnp->np_accept_list)) { 500 spin_unlock_bh(&cnp->np_accept_lock); 501 goto accept_wait; 502 } 503 504 csk = list_first_entry(&cnp->np_accept_list, 505 struct cxgbit_sock, 506 accept_node); 507 508 list_del_init(&csk->accept_node); 509 spin_unlock_bh(&cnp->np_accept_lock); 510 conn->context = csk; 511 csk->conn = conn; 512 513 cxgbit_set_conn_info(np, conn, csk); 514 return 0; 515} 516 517static int 518__cxgbit_free_cdev_np(struct cxgbit_device *cdev, struct cxgbit_np *cnp) 519{ 520 int stid, ret; 521 bool ipv6 = false; 522 523 stid = cxgbit_np_hash_del(cdev, cnp); 524 if (stid < 0) 525 return -EINVAL; 526 if (!test_bit(CDEV_STATE_UP, &cdev->flags)) 527 return -EINVAL; 528 529 if (cnp->np->np_sockaddr.ss_family == AF_INET6) 530 ipv6 = true; 531 532 cxgbit_get_cnp(cnp); 533 cxgbit_init_wr_wait(&cnp->com.wr_wait); 534 ret = cxgb4_remove_server(cdev->lldi.ports[0], stid, 535 cdev->lldi.rxq_ids[0], ipv6); 536 537 if (ret > 0) 538 ret = net_xmit_errno(ret); 539 540 if (ret) { 541 cxgbit_put_cnp(cnp); 542 return ret; 543 } 544 545 ret = cxgbit_wait_for_reply(cdev, &cnp->com.wr_wait, 546 0, 10, __func__); 547 if (ret == -ETIMEDOUT) 548 return ret; 549 550 if (ipv6 && cnp->com.cdev) { 551 struct sockaddr_in6 *sin6; 552 553 sin6 = (struct sockaddr_in6 *)&cnp->com.local_addr; 554 cxgb4_clip_release(cdev->lldi.ports[0], 555 (const u32 *)&sin6->sin6_addr.s6_addr, 556 1); 557 } 558 559 cxgb4_free_stid(cdev->lldi.tids, stid, 560 cnp->com.local_addr.ss_family); 561 return 0; 562} 563 564static void cxgbit_free_all_np(struct cxgbit_np *cnp) 565{ 566 struct cxgbit_device *cdev; 567 int ret; 568 569 mutex_lock(&cdev_list_lock); 570 list_for_each_entry(cdev, &cdev_list_head, list) { 571 ret = __cxgbit_free_cdev_np(cdev, cnp); 572 if (ret == -ETIMEDOUT) 573 break; 574 } 575 mutex_unlock(&cdev_list_lock); 576} 577 578static void cxgbit_free_cdev_np(struct cxgbit_np *cnp) 579{ 580 struct cxgbit_device *cdev; 581 bool found = false; 582 583 mutex_lock(&cdev_list_lock); 584 list_for_each_entry(cdev, &cdev_list_head, list) { 585 if (cdev == cnp->com.cdev) { 586 found = true; 587 break; 588 } 589 } 590 if (!found) 591 goto out; 592 593 __cxgbit_free_cdev_np(cdev, cnp); 594out: 595 mutex_unlock(&cdev_list_lock); 596} 597 598static void __cxgbit_free_conn(struct cxgbit_sock *csk); 599 600void cxgbit_free_np(struct iscsi_np *np) 601{ 602 struct cxgbit_np *cnp = np->np_context; 603 struct cxgbit_sock *csk, *tmp; 604 605 cnp->com.state = CSK_STATE_DEAD; 606 if (cnp->com.cdev) 607 cxgbit_free_cdev_np(cnp); 608 else 609 cxgbit_free_all_np(cnp); 610 611 spin_lock_bh(&cnp->np_accept_lock); 612 list_for_each_entry_safe(csk, tmp, &cnp->np_accept_list, accept_node) { 613 list_del_init(&csk->accept_node); 614 __cxgbit_free_conn(csk); 615 } 616 spin_unlock_bh(&cnp->np_accept_lock); 617 618 np->np_context = NULL; 619 cxgbit_put_cnp(cnp); 620} 621 622static void cxgbit_send_halfclose(struct cxgbit_sock *csk) 623{ 624 struct sk_buff *skb; 625 u32 len = roundup(sizeof(struct cpl_close_con_req), 16); 626 627 skb = alloc_skb(len, GFP_ATOMIC); 628 if (!skb) 629 return; 630 631 cxgb_mk_close_con_req(skb, len, csk->tid, csk->txq_idx, 632 NULL, NULL); 633 634 cxgbit_skcb_flags(skb) |= SKCBF_TX_FLAG_COMPL; 635 __skb_queue_tail(&csk->txq, skb); 636 cxgbit_push_tx_frames(csk); 637} 638 639static void cxgbit_arp_failure_discard(void *handle, struct sk_buff *skb) 640{ 641 struct cxgbit_sock *csk = handle; 642 643 pr_debug("%s cxgbit_device %p\n", __func__, handle); 644 kfree_skb(skb); 645 cxgbit_put_csk(csk); 646} 647 648static void cxgbit_abort_arp_failure(void *handle, struct sk_buff *skb) 649{ 650 struct cxgbit_device *cdev = handle; 651 struct cpl_abort_req *req = cplhdr(skb); 652 653 pr_debug("%s cdev %p\n", __func__, cdev); 654 req->cmd = CPL_ABORT_NO_RST; 655 cxgbit_ofld_send(cdev, skb); 656} 657 658static int cxgbit_send_abort_req(struct cxgbit_sock *csk) 659{ 660 struct sk_buff *skb; 661 u32 len = roundup(sizeof(struct cpl_abort_req), 16); 662 663 pr_debug("%s: csk %p tid %u; state %d\n", 664 __func__, csk, csk->tid, csk->com.state); 665 666 __skb_queue_purge(&csk->txq); 667 668 if (!test_and_set_bit(CSK_TX_DATA_SENT, &csk->com.flags)) 669 cxgbit_send_tx_flowc_wr(csk); 670 671 skb = __skb_dequeue(&csk->skbq); 672 cxgb_mk_abort_req(skb, len, csk->tid, csk->txq_idx, 673 csk->com.cdev, cxgbit_abort_arp_failure); 674 675 return cxgbit_l2t_send(csk->com.cdev, skb, csk->l2t); 676} 677 678static void 679__cxgbit_abort_conn(struct cxgbit_sock *csk, struct sk_buff *skb) 680{ 681 __kfree_skb(skb); 682 683 if (csk->com.state != CSK_STATE_ESTABLISHED) 684 goto no_abort; 685 686 set_bit(CSK_ABORT_RPL_WAIT, &csk->com.flags); 687 csk->com.state = CSK_STATE_ABORTING; 688 689 cxgbit_send_abort_req(csk); 690 691 return; 692 693no_abort: 694 cxgbit_wake_up(&csk->com.wr_wait, __func__, CPL_ERR_NONE); 695 cxgbit_put_csk(csk); 696} 697 698void cxgbit_abort_conn(struct cxgbit_sock *csk) 699{ 700 struct sk_buff *skb = alloc_skb(0, GFP_KERNEL | __GFP_NOFAIL); 701 702 cxgbit_get_csk(csk); 703 cxgbit_init_wr_wait(&csk->com.wr_wait); 704 705 spin_lock_bh(&csk->lock); 706 if (csk->lock_owner) { 707 cxgbit_skcb_rx_backlog_fn(skb) = __cxgbit_abort_conn; 708 __skb_queue_tail(&csk->backlogq, skb); 709 } else { 710 __cxgbit_abort_conn(csk, skb); 711 } 712 spin_unlock_bh(&csk->lock); 713 714 cxgbit_wait_for_reply(csk->com.cdev, &csk->com.wr_wait, 715 csk->tid, 600, __func__); 716} 717 718static void __cxgbit_free_conn(struct cxgbit_sock *csk) 719{ 720 struct iscsit_conn *conn = csk->conn; 721 bool release = false; 722 723 pr_debug("%s: state %d\n", 724 __func__, csk->com.state); 725 726 spin_lock_bh(&csk->lock); 727 switch (csk->com.state) { 728 case CSK_STATE_ESTABLISHED: 729 if (conn && (conn->conn_state == TARG_CONN_STATE_IN_LOGOUT)) { 730 csk->com.state = CSK_STATE_CLOSING; 731 cxgbit_send_halfclose(csk); 732 } else { 733 csk->com.state = CSK_STATE_ABORTING; 734 cxgbit_send_abort_req(csk); 735 } 736 break; 737 case CSK_STATE_CLOSING: 738 csk->com.state = CSK_STATE_MORIBUND; 739 cxgbit_send_halfclose(csk); 740 break; 741 case CSK_STATE_DEAD: 742 release = true; 743 break; 744 default: 745 pr_err("%s: csk %p; state %d\n", 746 __func__, csk, csk->com.state); 747 } 748 spin_unlock_bh(&csk->lock); 749 750 if (release) 751 cxgbit_put_csk(csk); 752} 753 754void cxgbit_free_conn(struct iscsit_conn *conn) 755{ 756 __cxgbit_free_conn(conn->context); 757} 758 759static void cxgbit_set_emss(struct cxgbit_sock *csk, u16 opt) 760{ 761 csk->emss = csk->com.cdev->lldi.mtus[TCPOPT_MSS_G(opt)] - 762 ((csk->com.remote_addr.ss_family == AF_INET) ? 763 sizeof(struct iphdr) : sizeof(struct ipv6hdr)) - 764 sizeof(struct tcphdr); 765 csk->mss = csk->emss; 766 if (TCPOPT_TSTAMP_G(opt)) 767 csk->emss -= round_up(TCPOLEN_TIMESTAMP, 4); 768 if (csk->emss < 128) 769 csk->emss = 128; 770 if (csk->emss & 7) 771 pr_info("Warning: misaligned mtu idx %u mss %u emss=%u\n", 772 TCPOPT_MSS_G(opt), csk->mss, csk->emss); 773 pr_debug("%s mss_idx %u mss %u emss=%u\n", __func__, TCPOPT_MSS_G(opt), 774 csk->mss, csk->emss); 775} 776 777static void cxgbit_free_skb(struct cxgbit_sock *csk) 778{ 779 struct sk_buff *skb; 780 781 __skb_queue_purge(&csk->txq); 782 __skb_queue_purge(&csk->rxq); 783 __skb_queue_purge(&csk->backlogq); 784 __skb_queue_purge(&csk->ppodq); 785 __skb_queue_purge(&csk->skbq); 786 787 while ((skb = cxgbit_sock_dequeue_wr(csk))) 788 kfree_skb(skb); 789 790 __kfree_skb(csk->lro_hskb); 791} 792 793void _cxgbit_free_csk(struct kref *kref) 794{ 795 struct cxgbit_sock *csk; 796 struct cxgbit_device *cdev; 797 798 csk = container_of(kref, struct cxgbit_sock, kref); 799 800 pr_debug("%s csk %p state %d\n", __func__, csk, csk->com.state); 801 802 if (csk->com.local_addr.ss_family == AF_INET6) { 803 struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *) 804 &csk->com.local_addr; 805 cxgb4_clip_release(csk->com.cdev->lldi.ports[0], 806 (const u32 *) 807 &sin6->sin6_addr.s6_addr, 1); 808 } 809 810 cxgb4_remove_tid(csk->com.cdev->lldi.tids, 0, csk->tid, 811 csk->com.local_addr.ss_family); 812 dst_release(csk->dst); 813 cxgb4_l2t_release(csk->l2t); 814 815 cdev = csk->com.cdev; 816 spin_lock_bh(&cdev->cskq.lock); 817 list_del(&csk->list); 818 spin_unlock_bh(&cdev->cskq.lock); 819 820 cxgbit_free_skb(csk); 821 cxgbit_put_cnp(csk->cnp); 822 cxgbit_put_cdev(cdev); 823 824 kfree(csk); 825} 826 827static void cxgbit_set_tcp_window(struct cxgbit_sock *csk, struct port_info *pi) 828{ 829 unsigned int linkspeed; 830 u8 scale; 831 832 linkspeed = pi->link_cfg.speed; 833 scale = linkspeed / SPEED_10000; 834 835#define CXGBIT_10G_RCV_WIN (256 * 1024) 836 csk->rcv_win = CXGBIT_10G_RCV_WIN; 837 if (scale) 838 csk->rcv_win *= scale; 839 csk->rcv_win = min(csk->rcv_win, RCV_BUFSIZ_M << 10); 840 841#define CXGBIT_10G_SND_WIN (256 * 1024) 842 csk->snd_win = CXGBIT_10G_SND_WIN; 843 if (scale) 844 csk->snd_win *= scale; 845 csk->snd_win = min(csk->snd_win, 512U * 1024); 846 847 pr_debug("%s snd_win %d rcv_win %d\n", 848 __func__, csk->snd_win, csk->rcv_win); 849} 850 851#ifdef CONFIG_CHELSIO_T4_DCB 852static u8 cxgbit_get_iscsi_dcb_state(struct net_device *ndev) 853{ 854 return ndev->dcbnl_ops->getstate(ndev); 855} 856 857static int cxgbit_select_priority(int pri_mask) 858{ 859 if (!pri_mask) 860 return 0; 861 862 return (ffs(pri_mask) - 1); 863} 864 865static u8 cxgbit_get_iscsi_dcb_priority(struct net_device *ndev, u16 local_port) 866{ 867 int ret; 868 u8 caps; 869 870 struct dcb_app iscsi_dcb_app = { 871 .protocol = local_port 872 }; 873 874 ret = (int)ndev->dcbnl_ops->getcap(ndev, DCB_CAP_ATTR_DCBX, &caps); 875 876 if (ret) 877 return 0; 878 879 if (caps & DCB_CAP_DCBX_VER_IEEE) { 880 iscsi_dcb_app.selector = IEEE_8021QAZ_APP_SEL_STREAM; 881 ret = dcb_ieee_getapp_mask(ndev, &iscsi_dcb_app); 882 if (!ret) { 883 iscsi_dcb_app.selector = IEEE_8021QAZ_APP_SEL_ANY; 884 ret = dcb_ieee_getapp_mask(ndev, &iscsi_dcb_app); 885 } 886 } else if (caps & DCB_CAP_DCBX_VER_CEE) { 887 iscsi_dcb_app.selector = DCB_APP_IDTYPE_PORTNUM; 888 889 ret = dcb_getapp(ndev, &iscsi_dcb_app); 890 } 891 892 pr_info("iSCSI priority is set to %u\n", cxgbit_select_priority(ret)); 893 894 return cxgbit_select_priority(ret); 895} 896#endif 897 898static int 899cxgbit_offload_init(struct cxgbit_sock *csk, int iptype, __u8 *peer_ip, 900 u16 local_port, struct dst_entry *dst, 901 struct cxgbit_device *cdev) 902{ 903 struct neighbour *n; 904 int ret, step; 905 struct net_device *ndev; 906 u16 rxq_idx, port_id; 907#ifdef CONFIG_CHELSIO_T4_DCB 908 u8 priority = 0; 909#endif 910 911 n = dst_neigh_lookup(dst, peer_ip); 912 if (!n) 913 return -ENODEV; 914 915 rcu_read_lock(); 916 if (!(n->nud_state & NUD_VALID)) 917 neigh_event_send(n, NULL); 918 919 ret = -ENOMEM; 920 if (n->dev->flags & IFF_LOOPBACK) { 921 if (iptype == 4) 922 ndev = cxgbit_ipv4_netdev(*(__be32 *)peer_ip); 923 else if (IS_ENABLED(CONFIG_IPV6)) 924 ndev = cxgbit_ipv6_netdev((struct in6_addr *)peer_ip); 925 else 926 ndev = NULL; 927 928 if (!ndev) { 929 ret = -ENODEV; 930 goto out; 931 } 932 933 csk->l2t = cxgb4_l2t_get(cdev->lldi.l2t, 934 n, ndev, 0); 935 if (!csk->l2t) 936 goto out; 937 csk->mtu = ndev->mtu; 938 csk->tx_chan = cxgb4_port_chan(ndev); 939 csk->smac_idx = 940 ((struct port_info *)netdev_priv(ndev))->smt_idx; 941 step = cdev->lldi.ntxq / 942 cdev->lldi.nchan; 943 csk->txq_idx = cxgb4_port_idx(ndev) * step; 944 step = cdev->lldi.nrxq / 945 cdev->lldi.nchan; 946 csk->ctrlq_idx = cxgb4_port_idx(ndev); 947 csk->rss_qid = cdev->lldi.rxq_ids[ 948 cxgb4_port_idx(ndev) * step]; 949 csk->port_id = cxgb4_port_idx(ndev); 950 cxgbit_set_tcp_window(csk, 951 (struct port_info *)netdev_priv(ndev)); 952 } else { 953 ndev = cxgbit_get_real_dev(n->dev); 954 if (!ndev) { 955 ret = -ENODEV; 956 goto out; 957 } 958 959#ifdef CONFIG_CHELSIO_T4_DCB 960 if (cxgbit_get_iscsi_dcb_state(ndev)) 961 priority = cxgbit_get_iscsi_dcb_priority(ndev, 962 local_port); 963 964 csk->dcb_priority = priority; 965 966 csk->l2t = cxgb4_l2t_get(cdev->lldi.l2t, n, ndev, priority); 967#else 968 csk->l2t = cxgb4_l2t_get(cdev->lldi.l2t, n, ndev, 0); 969#endif 970 if (!csk->l2t) 971 goto out; 972 port_id = cxgb4_port_idx(ndev); 973 csk->mtu = dst_mtu(dst); 974 csk->tx_chan = cxgb4_port_chan(ndev); 975 csk->smac_idx = 976 ((struct port_info *)netdev_priv(ndev))->smt_idx; 977 step = cdev->lldi.ntxq / 978 cdev->lldi.nports; 979 csk->txq_idx = (port_id * step) + 980 (cdev->selectq[port_id][0]++ % step); 981 csk->ctrlq_idx = cxgb4_port_idx(ndev); 982 step = cdev->lldi.nrxq / 983 cdev->lldi.nports; 984 rxq_idx = (port_id * step) + 985 (cdev->selectq[port_id][1]++ % step); 986 csk->rss_qid = cdev->lldi.rxq_ids[rxq_idx]; 987 csk->port_id = port_id; 988 cxgbit_set_tcp_window(csk, 989 (struct port_info *)netdev_priv(ndev)); 990 } 991 ret = 0; 992out: 993 rcu_read_unlock(); 994 neigh_release(n); 995 return ret; 996} 997 998int cxgbit_ofld_send(struct cxgbit_device *cdev, struct sk_buff *skb) 999{ 1000 int ret = 0; 1001 1002 if (!test_bit(CDEV_STATE_UP, &cdev->flags)) { 1003 kfree_skb(skb); 1004 pr_err("%s - device not up - dropping\n", __func__); 1005 return -EIO; 1006 } 1007 1008 ret = cxgb4_ofld_send(cdev->lldi.ports[0], skb); 1009 if (ret < 0) 1010 kfree_skb(skb); 1011 return ret < 0 ? ret : 0; 1012} 1013 1014static void cxgbit_release_tid(struct cxgbit_device *cdev, u32 tid) 1015{ 1016 u32 len = roundup(sizeof(struct cpl_tid_release), 16); 1017 struct sk_buff *skb; 1018 1019 skb = alloc_skb(len, GFP_ATOMIC); 1020 if (!skb) 1021 return; 1022 1023 cxgb_mk_tid_release(skb, len, tid, 0); 1024 cxgbit_ofld_send(cdev, skb); 1025} 1026 1027int 1028cxgbit_l2t_send(struct cxgbit_device *cdev, struct sk_buff *skb, 1029 struct l2t_entry *l2e) 1030{ 1031 int ret = 0; 1032 1033 if (!test_bit(CDEV_STATE_UP, &cdev->flags)) { 1034 kfree_skb(skb); 1035 pr_err("%s - device not up - dropping\n", __func__); 1036 return -EIO; 1037 } 1038 1039 ret = cxgb4_l2t_send(cdev->lldi.ports[0], skb, l2e); 1040 if (ret < 0) 1041 kfree_skb(skb); 1042 return ret < 0 ? ret : 0; 1043} 1044 1045static void cxgbit_send_rx_credits(struct cxgbit_sock *csk, struct sk_buff *skb) 1046{ 1047 if (csk->com.state != CSK_STATE_ESTABLISHED) { 1048 __kfree_skb(skb); 1049 return; 1050 } 1051 1052 cxgbit_ofld_send(csk->com.cdev, skb); 1053} 1054 1055/* 1056 * CPL connection rx data ack: host -> 1057 * Send RX credits through an RX_DATA_ACK CPL message. 1058 * Returns the number of credits sent. 1059 */ 1060int cxgbit_rx_data_ack(struct cxgbit_sock *csk) 1061{ 1062 struct sk_buff *skb; 1063 u32 len = roundup(sizeof(struct cpl_rx_data_ack), 16); 1064 u32 credit_dack; 1065 1066 skb = alloc_skb(len, GFP_KERNEL); 1067 if (!skb) 1068 return -1; 1069 1070 credit_dack = RX_DACK_CHANGE_F | RX_DACK_MODE_V(3) | 1071 RX_CREDITS_V(csk->rx_credits); 1072 1073 cxgb_mk_rx_data_ack(skb, len, csk->tid, csk->ctrlq_idx, 1074 credit_dack); 1075 1076 csk->rx_credits = 0; 1077 1078 spin_lock_bh(&csk->lock); 1079 if (csk->lock_owner) { 1080 cxgbit_skcb_rx_backlog_fn(skb) = cxgbit_send_rx_credits; 1081 __skb_queue_tail(&csk->backlogq, skb); 1082 spin_unlock_bh(&csk->lock); 1083 return 0; 1084 } 1085 1086 cxgbit_send_rx_credits(csk, skb); 1087 spin_unlock_bh(&csk->lock); 1088 1089 return 0; 1090} 1091 1092#define FLOWC_WR_NPARAMS_MIN 9 1093#define FLOWC_WR_NPARAMS_MAX 11 1094static int cxgbit_alloc_csk_skb(struct cxgbit_sock *csk) 1095{ 1096 struct sk_buff *skb; 1097 u32 len, flowclen; 1098 u8 i; 1099 1100 flowclen = offsetof(struct fw_flowc_wr, 1101 mnemval[FLOWC_WR_NPARAMS_MAX]); 1102 1103 len = max_t(u32, sizeof(struct cpl_abort_req), 1104 sizeof(struct cpl_abort_rpl)); 1105 1106 len = max(len, flowclen); 1107 len = roundup(len, 16); 1108 1109 for (i = 0; i < 3; i++) { 1110 skb = alloc_skb(len, GFP_ATOMIC); 1111 if (!skb) 1112 goto out; 1113 __skb_queue_tail(&csk->skbq, skb); 1114 } 1115 1116 skb = alloc_skb(LRO_SKB_MIN_HEADROOM, GFP_ATOMIC); 1117 if (!skb) 1118 goto out; 1119 1120 memset(skb->data, 0, LRO_SKB_MIN_HEADROOM); 1121 csk->lro_hskb = skb; 1122 1123 return 0; 1124out: 1125 __skb_queue_purge(&csk->skbq); 1126 return -ENOMEM; 1127} 1128 1129static void 1130cxgbit_pass_accept_rpl(struct cxgbit_sock *csk, struct cpl_pass_accept_req *req) 1131{ 1132 struct sk_buff *skb; 1133 const struct tcphdr *tcph; 1134 struct cpl_t5_pass_accept_rpl *rpl5; 1135 struct cxgb4_lld_info *lldi = &csk->com.cdev->lldi; 1136 unsigned int len = roundup(sizeof(*rpl5), 16); 1137 unsigned int mtu_idx; 1138 u64 opt0; 1139 u32 opt2, hlen; 1140 u32 wscale; 1141 u32 win; 1142 1143 pr_debug("%s csk %p tid %u\n", __func__, csk, csk->tid); 1144 1145 skb = alloc_skb(len, GFP_ATOMIC); 1146 if (!skb) { 1147 cxgbit_put_csk(csk); 1148 return; 1149 } 1150 1151 rpl5 = __skb_put_zero(skb, len); 1152 1153 INIT_TP_WR(rpl5, csk->tid); 1154 OPCODE_TID(rpl5) = cpu_to_be32(MK_OPCODE_TID(CPL_PASS_ACCEPT_RPL, 1155 csk->tid)); 1156 cxgb_best_mtu(csk->com.cdev->lldi.mtus, csk->mtu, &mtu_idx, 1157 req->tcpopt.tstamp, 1158 (csk->com.remote_addr.ss_family == AF_INET) ? 0 : 1); 1159 wscale = cxgb_compute_wscale(csk->rcv_win); 1160 /* 1161 * Specify the largest window that will fit in opt0. The 1162 * remainder will be specified in the rx_data_ack. 1163 */ 1164 win = csk->rcv_win >> 10; 1165 if (win > RCV_BUFSIZ_M) 1166 win = RCV_BUFSIZ_M; 1167 opt0 = TCAM_BYPASS_F | 1168 WND_SCALE_V(wscale) | 1169 MSS_IDX_V(mtu_idx) | 1170 L2T_IDX_V(csk->l2t->idx) | 1171 TX_CHAN_V(csk->tx_chan) | 1172 SMAC_SEL_V(csk->smac_idx) | 1173 DSCP_V(csk->tos >> 2) | 1174 ULP_MODE_V(ULP_MODE_ISCSI) | 1175 RCV_BUFSIZ_V(win); 1176 1177 opt2 = RX_CHANNEL_V(0) | 1178 RSS_QUEUE_VALID_F | RSS_QUEUE_V(csk->rss_qid); 1179 1180 if (!is_t5(lldi->adapter_type)) 1181 opt2 |= RX_FC_DISABLE_F; 1182 1183 if (req->tcpopt.tstamp) 1184 opt2 |= TSTAMPS_EN_F; 1185 if (req->tcpopt.sack) 1186 opt2 |= SACK_EN_F; 1187 if (wscale) 1188 opt2 |= WND_SCALE_EN_F; 1189 1190 hlen = ntohl(req->hdr_len); 1191 1192 if (is_t5(lldi->adapter_type)) 1193 tcph = (struct tcphdr *)((u8 *)(req + 1) + 1194 ETH_HDR_LEN_G(hlen) + IP_HDR_LEN_G(hlen)); 1195 else 1196 tcph = (struct tcphdr *)((u8 *)(req + 1) + 1197 T6_ETH_HDR_LEN_G(hlen) + T6_IP_HDR_LEN_G(hlen)); 1198 1199 if (tcph->ece && tcph->cwr) 1200 opt2 |= CCTRL_ECN_V(1); 1201 1202 opt2 |= CONG_CNTRL_V(CONG_ALG_NEWRENO); 1203 1204 opt2 |= T5_ISS_F; 1205 rpl5->iss = cpu_to_be32((get_random_u32() & ~7UL) - 1); 1206 1207 opt2 |= T5_OPT_2_VALID_F; 1208 1209 rpl5->opt0 = cpu_to_be64(opt0); 1210 rpl5->opt2 = cpu_to_be32(opt2); 1211 set_wr_txq(skb, CPL_PRIORITY_SETUP, csk->ctrlq_idx); 1212 t4_set_arp_err_handler(skb, csk, cxgbit_arp_failure_discard); 1213 cxgbit_l2t_send(csk->com.cdev, skb, csk->l2t); 1214} 1215 1216static void 1217cxgbit_pass_accept_req(struct cxgbit_device *cdev, struct sk_buff *skb) 1218{ 1219 struct cxgbit_sock *csk = NULL; 1220 struct cxgbit_np *cnp; 1221 struct cpl_pass_accept_req *req = cplhdr(skb); 1222 unsigned int stid = PASS_OPEN_TID_G(ntohl(req->tos_stid)); 1223 struct tid_info *t = cdev->lldi.tids; 1224 unsigned int tid = GET_TID(req); 1225 u16 peer_mss = ntohs(req->tcpopt.mss); 1226 unsigned short hdrs; 1227 1228 struct dst_entry *dst; 1229 __u8 local_ip[16], peer_ip[16]; 1230 __be16 local_port, peer_port; 1231 int ret; 1232 int iptype; 1233 1234 pr_debug("%s: cdev = %p; stid = %u; tid = %u\n", 1235 __func__, cdev, stid, tid); 1236 1237 cnp = lookup_stid(t, stid); 1238 if (!cnp) { 1239 pr_err("%s connect request on invalid stid %d\n", 1240 __func__, stid); 1241 goto rel_skb; 1242 } 1243 1244 if (cnp->com.state != CSK_STATE_LISTEN) { 1245 pr_err("%s - listening parent not in CSK_STATE_LISTEN\n", 1246 __func__); 1247 goto reject; 1248 } 1249 1250 csk = lookup_tid(t, tid); 1251 if (csk) { 1252 pr_err("%s csk not null tid %u\n", 1253 __func__, tid); 1254 goto rel_skb; 1255 } 1256 1257 cxgb_get_4tuple(req, cdev->lldi.adapter_type, &iptype, local_ip, 1258 peer_ip, &local_port, &peer_port); 1259 1260 /* Find output route */ 1261 if (iptype == 4) { 1262 pr_debug("%s parent sock %p tid %u laddr %pI4 raddr %pI4 " 1263 "lport %d rport %d peer_mss %d\n" 1264 , __func__, cnp, tid, 1265 local_ip, peer_ip, ntohs(local_port), 1266 ntohs(peer_port), peer_mss); 1267 dst = cxgb_find_route(&cdev->lldi, cxgbit_get_real_dev, 1268 *(__be32 *)local_ip, 1269 *(__be32 *)peer_ip, 1270 local_port, peer_port, 1271 PASS_OPEN_TOS_G(ntohl(req->tos_stid))); 1272 } else { 1273 pr_debug("%s parent sock %p tid %u laddr %pI6 raddr %pI6 " 1274 "lport %d rport %d peer_mss %d\n" 1275 , __func__, cnp, tid, 1276 local_ip, peer_ip, ntohs(local_port), 1277 ntohs(peer_port), peer_mss); 1278 dst = cxgb_find_route6(&cdev->lldi, cxgbit_get_real_dev, 1279 local_ip, peer_ip, 1280 local_port, peer_port, 1281 PASS_OPEN_TOS_G(ntohl(req->tos_stid)), 1282 ((struct sockaddr_in6 *) 1283 &cnp->com.local_addr)->sin6_scope_id); 1284 } 1285 if (!dst) { 1286 pr_err("%s - failed to find dst entry!\n", 1287 __func__); 1288 goto reject; 1289 } 1290 1291 csk = kzalloc(sizeof(*csk), GFP_ATOMIC); 1292 if (!csk) { 1293 dst_release(dst); 1294 goto rel_skb; 1295 } 1296 1297 ret = cxgbit_offload_init(csk, iptype, peer_ip, ntohs(local_port), 1298 dst, cdev); 1299 if (ret) { 1300 pr_err("%s - failed to allocate l2t entry!\n", 1301 __func__); 1302 dst_release(dst); 1303 kfree(csk); 1304 goto reject; 1305 } 1306 1307 kref_init(&csk->kref); 1308 init_completion(&csk->com.wr_wait.completion); 1309 1310 INIT_LIST_HEAD(&csk->accept_node); 1311 1312 hdrs = (iptype == 4 ? sizeof(struct iphdr) : sizeof(struct ipv6hdr)) + 1313 sizeof(struct tcphdr) + (req->tcpopt.tstamp ? 12 : 0); 1314 if (peer_mss && csk->mtu > (peer_mss + hdrs)) 1315 csk->mtu = peer_mss + hdrs; 1316 1317 csk->com.state = CSK_STATE_CONNECTING; 1318 csk->com.cdev = cdev; 1319 csk->cnp = cnp; 1320 csk->tos = PASS_OPEN_TOS_G(ntohl(req->tos_stid)); 1321 csk->dst = dst; 1322 csk->tid = tid; 1323 csk->wr_cred = cdev->lldi.wr_cred - 1324 DIV_ROUND_UP(sizeof(struct cpl_abort_req), 16); 1325 csk->wr_max_cred = csk->wr_cred; 1326 csk->wr_una_cred = 0; 1327 1328 if (iptype == 4) { 1329 struct sockaddr_in *sin = (struct sockaddr_in *) 1330 &csk->com.local_addr; 1331 sin->sin_family = AF_INET; 1332 sin->sin_port = local_port; 1333 sin->sin_addr.s_addr = *(__be32 *)local_ip; 1334 1335 sin = (struct sockaddr_in *)&csk->com.remote_addr; 1336 sin->sin_family = AF_INET; 1337 sin->sin_port = peer_port; 1338 sin->sin_addr.s_addr = *(__be32 *)peer_ip; 1339 } else { 1340 struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *) 1341 &csk->com.local_addr; 1342 1343 sin6->sin6_family = PF_INET6; 1344 sin6->sin6_port = local_port; 1345 memcpy(sin6->sin6_addr.s6_addr, local_ip, 16); 1346 cxgb4_clip_get(cdev->lldi.ports[0], 1347 (const u32 *)&sin6->sin6_addr.s6_addr, 1348 1); 1349 1350 sin6 = (struct sockaddr_in6 *)&csk->com.remote_addr; 1351 sin6->sin6_family = PF_INET6; 1352 sin6->sin6_port = peer_port; 1353 memcpy(sin6->sin6_addr.s6_addr, peer_ip, 16); 1354 } 1355 1356 skb_queue_head_init(&csk->rxq); 1357 skb_queue_head_init(&csk->txq); 1358 skb_queue_head_init(&csk->ppodq); 1359 skb_queue_head_init(&csk->backlogq); 1360 skb_queue_head_init(&csk->skbq); 1361 cxgbit_sock_reset_wr_list(csk); 1362 spin_lock_init(&csk->lock); 1363 init_waitqueue_head(&csk->waitq); 1364 csk->lock_owner = false; 1365 1366 if (cxgbit_alloc_csk_skb(csk)) { 1367 dst_release(dst); 1368 kfree(csk); 1369 goto rel_skb; 1370 } 1371 1372 cxgbit_get_cnp(cnp); 1373 cxgbit_get_cdev(cdev); 1374 1375 spin_lock(&cdev->cskq.lock); 1376 list_add_tail(&csk->list, &cdev->cskq.list); 1377 spin_unlock(&cdev->cskq.lock); 1378 cxgb4_insert_tid(t, csk, tid, csk->com.local_addr.ss_family); 1379 cxgbit_pass_accept_rpl(csk, req); 1380 goto rel_skb; 1381 1382reject: 1383 cxgbit_release_tid(cdev, tid); 1384rel_skb: 1385 __kfree_skb(skb); 1386} 1387 1388static u32 1389cxgbit_tx_flowc_wr_credits(struct cxgbit_sock *csk, u32 *nparamsp, 1390 u32 *flowclenp) 1391{ 1392 u32 nparams, flowclen16, flowclen; 1393 1394 nparams = FLOWC_WR_NPARAMS_MIN; 1395 1396 if (csk->snd_wscale) 1397 nparams++; 1398 1399#ifdef CONFIG_CHELSIO_T4_DCB 1400 nparams++; 1401#endif 1402 flowclen = offsetof(struct fw_flowc_wr, mnemval[nparams]); 1403 flowclen16 = DIV_ROUND_UP(flowclen, 16); 1404 flowclen = flowclen16 * 16; 1405 /* 1406 * Return the number of 16-byte credits used by the flowc request. 1407 * Pass back the nparams and actual flowc length if requested. 1408 */ 1409 if (nparamsp) 1410 *nparamsp = nparams; 1411 if (flowclenp) 1412 *flowclenp = flowclen; 1413 return flowclen16; 1414} 1415 1416u32 cxgbit_send_tx_flowc_wr(struct cxgbit_sock *csk) 1417{ 1418 struct cxgbit_device *cdev = csk->com.cdev; 1419 struct fw_flowc_wr *flowc; 1420 u32 nparams, flowclen16, flowclen; 1421 struct sk_buff *skb; 1422 u8 index; 1423 1424#ifdef CONFIG_CHELSIO_T4_DCB 1425 u16 vlan = ((struct l2t_entry *)csk->l2t)->vlan; 1426#endif 1427 1428 flowclen16 = cxgbit_tx_flowc_wr_credits(csk, &nparams, &flowclen); 1429 1430 skb = __skb_dequeue(&csk->skbq); 1431 flowc = __skb_put_zero(skb, flowclen); 1432 1433 flowc->op_to_nparams = cpu_to_be32(FW_WR_OP_V(FW_FLOWC_WR) | 1434 FW_FLOWC_WR_NPARAMS_V(nparams)); 1435 flowc->flowid_len16 = cpu_to_be32(FW_WR_LEN16_V(flowclen16) | 1436 FW_WR_FLOWID_V(csk->tid)); 1437 flowc->mnemval[0].mnemonic = FW_FLOWC_MNEM_PFNVFN; 1438 flowc->mnemval[0].val = cpu_to_be32(FW_PFVF_CMD_PFN_V 1439 (csk->com.cdev->lldi.pf)); 1440 flowc->mnemval[1].mnemonic = FW_FLOWC_MNEM_CH; 1441 flowc->mnemval[1].val = cpu_to_be32(csk->tx_chan); 1442 flowc->mnemval[2].mnemonic = FW_FLOWC_MNEM_PORT; 1443 flowc->mnemval[2].val = cpu_to_be32(csk->tx_chan); 1444 flowc->mnemval[3].mnemonic = FW_FLOWC_MNEM_IQID; 1445 flowc->mnemval[3].val = cpu_to_be32(csk->rss_qid); 1446 flowc->mnemval[4].mnemonic = FW_FLOWC_MNEM_SNDNXT; 1447 flowc->mnemval[4].val = cpu_to_be32(csk->snd_nxt); 1448 flowc->mnemval[5].mnemonic = FW_FLOWC_MNEM_RCVNXT; 1449 flowc->mnemval[5].val = cpu_to_be32(csk->rcv_nxt); 1450 flowc->mnemval[6].mnemonic = FW_FLOWC_MNEM_SNDBUF; 1451 flowc->mnemval[6].val = cpu_to_be32(csk->snd_win); 1452 flowc->mnemval[7].mnemonic = FW_FLOWC_MNEM_MSS; 1453 flowc->mnemval[7].val = cpu_to_be32(csk->emss); 1454 1455 flowc->mnemval[8].mnemonic = FW_FLOWC_MNEM_TXDATAPLEN_MAX; 1456 if (test_bit(CDEV_ISO_ENABLE, &cdev->flags)) 1457 flowc->mnemval[8].val = cpu_to_be32(CXGBIT_MAX_ISO_PAYLOAD); 1458 else 1459 flowc->mnemval[8].val = cpu_to_be32(16384); 1460 1461 index = 9; 1462 1463 if (csk->snd_wscale) { 1464 flowc->mnemval[index].mnemonic = FW_FLOWC_MNEM_RCV_SCALE; 1465 flowc->mnemval[index].val = cpu_to_be32(csk->snd_wscale); 1466 index++; 1467 } 1468 1469#ifdef CONFIG_CHELSIO_T4_DCB 1470 flowc->mnemval[index].mnemonic = FW_FLOWC_MNEM_DCBPRIO; 1471 if (vlan == VLAN_NONE) { 1472 pr_warn("csk %u without VLAN Tag on DCB Link\n", csk->tid); 1473 flowc->mnemval[index].val = cpu_to_be32(0); 1474 } else 1475 flowc->mnemval[index].val = cpu_to_be32( 1476 (vlan & VLAN_PRIO_MASK) >> VLAN_PRIO_SHIFT); 1477#endif 1478 1479 pr_debug("%s: csk %p; tx_chan = %u; rss_qid = %u; snd_seq = %u;" 1480 " rcv_seq = %u; snd_win = %u; emss = %u\n", 1481 __func__, csk, csk->tx_chan, csk->rss_qid, csk->snd_nxt, 1482 csk->rcv_nxt, csk->snd_win, csk->emss); 1483 set_wr_txq(skb, CPL_PRIORITY_DATA, csk->txq_idx); 1484 cxgbit_ofld_send(csk->com.cdev, skb); 1485 return flowclen16; 1486} 1487 1488static int 1489cxgbit_send_tcb_skb(struct cxgbit_sock *csk, struct sk_buff *skb) 1490{ 1491 spin_lock_bh(&csk->lock); 1492 if (unlikely(csk->com.state != CSK_STATE_ESTABLISHED)) { 1493 spin_unlock_bh(&csk->lock); 1494 pr_err("%s: csk 0x%p, tid %u, state %u\n", 1495 __func__, csk, csk->tid, csk->com.state); 1496 __kfree_skb(skb); 1497 return -1; 1498 } 1499 1500 cxgbit_get_csk(csk); 1501 cxgbit_init_wr_wait(&csk->com.wr_wait); 1502 cxgbit_ofld_send(csk->com.cdev, skb); 1503 spin_unlock_bh(&csk->lock); 1504 1505 return 0; 1506} 1507 1508int cxgbit_setup_conn_digest(struct cxgbit_sock *csk) 1509{ 1510 struct sk_buff *skb; 1511 struct cpl_set_tcb_field *req; 1512 u8 hcrc = csk->submode & CXGBIT_SUBMODE_HCRC; 1513 u8 dcrc = csk->submode & CXGBIT_SUBMODE_DCRC; 1514 unsigned int len = roundup(sizeof(*req), 16); 1515 int ret; 1516 1517 skb = alloc_skb(len, GFP_KERNEL); 1518 if (!skb) 1519 return -ENOMEM; 1520 1521 /* set up ulp submode */ 1522 req = __skb_put_zero(skb, len); 1523 1524 INIT_TP_WR(req, csk->tid); 1525 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, csk->tid)); 1526 req->reply_ctrl = htons(NO_REPLY_V(0) | QUEUENO_V(csk->rss_qid)); 1527 req->word_cookie = htons(0); 1528 req->mask = cpu_to_be64(0x3 << 4); 1529 req->val = cpu_to_be64(((hcrc ? ULP_CRC_HEADER : 0) | 1530 (dcrc ? ULP_CRC_DATA : 0)) << 4); 1531 set_wr_txq(skb, CPL_PRIORITY_CONTROL, csk->ctrlq_idx); 1532 1533 if (cxgbit_send_tcb_skb(csk, skb)) 1534 return -1; 1535 1536 ret = cxgbit_wait_for_reply(csk->com.cdev, 1537 &csk->com.wr_wait, 1538 csk->tid, 5, __func__); 1539 if (ret) 1540 return -1; 1541 1542 return 0; 1543} 1544 1545int cxgbit_setup_conn_pgidx(struct cxgbit_sock *csk, u32 pg_idx) 1546{ 1547 struct sk_buff *skb; 1548 struct cpl_set_tcb_field *req; 1549 unsigned int len = roundup(sizeof(*req), 16); 1550 int ret; 1551 1552 skb = alloc_skb(len, GFP_KERNEL); 1553 if (!skb) 1554 return -ENOMEM; 1555 1556 req = __skb_put_zero(skb, len); 1557 1558 INIT_TP_WR(req, csk->tid); 1559 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, csk->tid)); 1560 req->reply_ctrl = htons(NO_REPLY_V(0) | QUEUENO_V(csk->rss_qid)); 1561 req->word_cookie = htons(0); 1562 req->mask = cpu_to_be64(0x3 << 8); 1563 req->val = cpu_to_be64(pg_idx << 8); 1564 set_wr_txq(skb, CPL_PRIORITY_CONTROL, csk->ctrlq_idx); 1565 1566 if (cxgbit_send_tcb_skb(csk, skb)) 1567 return -1; 1568 1569 ret = cxgbit_wait_for_reply(csk->com.cdev, 1570 &csk->com.wr_wait, 1571 csk->tid, 5, __func__); 1572 if (ret) 1573 return -1; 1574 1575 return 0; 1576} 1577 1578static void 1579cxgbit_pass_open_rpl(struct cxgbit_device *cdev, struct sk_buff *skb) 1580{ 1581 struct cpl_pass_open_rpl *rpl = cplhdr(skb); 1582 struct tid_info *t = cdev->lldi.tids; 1583 unsigned int stid = GET_TID(rpl); 1584 struct cxgbit_np *cnp = lookup_stid(t, stid); 1585 1586 pr_debug("%s: cnp = %p; stid = %u; status = %d\n", 1587 __func__, cnp, stid, rpl->status); 1588 1589 if (!cnp) { 1590 pr_info("%s stid %d lookup failure\n", __func__, stid); 1591 goto rel_skb; 1592 } 1593 1594 cxgbit_wake_up(&cnp->com.wr_wait, __func__, rpl->status); 1595 cxgbit_put_cnp(cnp); 1596rel_skb: 1597 __kfree_skb(skb); 1598} 1599 1600static void 1601cxgbit_close_listsrv_rpl(struct cxgbit_device *cdev, struct sk_buff *skb) 1602{ 1603 struct cpl_close_listsvr_rpl *rpl = cplhdr(skb); 1604 struct tid_info *t = cdev->lldi.tids; 1605 unsigned int stid = GET_TID(rpl); 1606 struct cxgbit_np *cnp = lookup_stid(t, stid); 1607 1608 pr_debug("%s: cnp = %p; stid = %u; status = %d\n", 1609 __func__, cnp, stid, rpl->status); 1610 1611 if (!cnp) { 1612 pr_info("%s stid %d lookup failure\n", __func__, stid); 1613 goto rel_skb; 1614 } 1615 1616 cxgbit_wake_up(&cnp->com.wr_wait, __func__, rpl->status); 1617 cxgbit_put_cnp(cnp); 1618rel_skb: 1619 __kfree_skb(skb); 1620} 1621 1622static void 1623cxgbit_pass_establish(struct cxgbit_device *cdev, struct sk_buff *skb) 1624{ 1625 struct cpl_pass_establish *req = cplhdr(skb); 1626 struct tid_info *t = cdev->lldi.tids; 1627 unsigned int tid = GET_TID(req); 1628 struct cxgbit_sock *csk; 1629 struct cxgbit_np *cnp; 1630 u16 tcp_opt = be16_to_cpu(req->tcp_opt); 1631 u32 snd_isn = be32_to_cpu(req->snd_isn); 1632 u32 rcv_isn = be32_to_cpu(req->rcv_isn); 1633 1634 csk = lookup_tid(t, tid); 1635 if (unlikely(!csk)) { 1636 pr_err("can't find connection for tid %u.\n", tid); 1637 goto rel_skb; 1638 } 1639 cnp = csk->cnp; 1640 1641 pr_debug("%s: csk %p; tid %u; cnp %p\n", 1642 __func__, csk, tid, cnp); 1643 1644 csk->write_seq = snd_isn; 1645 csk->snd_una = snd_isn; 1646 csk->snd_nxt = snd_isn; 1647 1648 csk->rcv_nxt = rcv_isn; 1649 1650 csk->snd_wscale = TCPOPT_SND_WSCALE_G(tcp_opt); 1651 cxgbit_set_emss(csk, tcp_opt); 1652 dst_confirm(csk->dst); 1653 csk->com.state = CSK_STATE_ESTABLISHED; 1654 spin_lock_bh(&cnp->np_accept_lock); 1655 list_add_tail(&csk->accept_node, &cnp->np_accept_list); 1656 spin_unlock_bh(&cnp->np_accept_lock); 1657 complete(&cnp->accept_comp); 1658rel_skb: 1659 __kfree_skb(skb); 1660} 1661 1662static void cxgbit_queue_rx_skb(struct cxgbit_sock *csk, struct sk_buff *skb) 1663{ 1664 cxgbit_skcb_flags(skb) = 0; 1665 spin_lock_bh(&csk->rxq.lock); 1666 __skb_queue_tail(&csk->rxq, skb); 1667 spin_unlock_bh(&csk->rxq.lock); 1668 wake_up(&csk->waitq); 1669} 1670 1671static void cxgbit_peer_close(struct cxgbit_sock *csk, struct sk_buff *skb) 1672{ 1673 pr_debug("%s: csk %p; tid %u; state %d\n", 1674 __func__, csk, csk->tid, csk->com.state); 1675 1676 switch (csk->com.state) { 1677 case CSK_STATE_ESTABLISHED: 1678 csk->com.state = CSK_STATE_CLOSING; 1679 cxgbit_queue_rx_skb(csk, skb); 1680 return; 1681 case CSK_STATE_CLOSING: 1682 /* simultaneous close */ 1683 csk->com.state = CSK_STATE_MORIBUND; 1684 break; 1685 case CSK_STATE_MORIBUND: 1686 csk->com.state = CSK_STATE_DEAD; 1687 cxgbit_put_csk(csk); 1688 break; 1689 case CSK_STATE_ABORTING: 1690 break; 1691 default: 1692 pr_info("%s: cpl_peer_close in bad state %d\n", 1693 __func__, csk->com.state); 1694 } 1695 1696 __kfree_skb(skb); 1697} 1698 1699static void cxgbit_close_con_rpl(struct cxgbit_sock *csk, struct sk_buff *skb) 1700{ 1701 pr_debug("%s: csk %p; tid %u; state %d\n", 1702 __func__, csk, csk->tid, csk->com.state); 1703 1704 switch (csk->com.state) { 1705 case CSK_STATE_CLOSING: 1706 csk->com.state = CSK_STATE_MORIBUND; 1707 break; 1708 case CSK_STATE_MORIBUND: 1709 csk->com.state = CSK_STATE_DEAD; 1710 cxgbit_put_csk(csk); 1711 break; 1712 case CSK_STATE_ABORTING: 1713 case CSK_STATE_DEAD: 1714 break; 1715 default: 1716 pr_info("%s: cpl_close_con_rpl in bad state %d\n", 1717 __func__, csk->com.state); 1718 } 1719 1720 __kfree_skb(skb); 1721} 1722 1723static void cxgbit_abort_req_rss(struct cxgbit_sock *csk, struct sk_buff *skb) 1724{ 1725 struct cpl_abort_req_rss *hdr = cplhdr(skb); 1726 unsigned int tid = GET_TID(hdr); 1727 struct sk_buff *rpl_skb; 1728 bool release = false; 1729 bool wakeup_thread = false; 1730 u32 len = roundup(sizeof(struct cpl_abort_rpl), 16); 1731 1732 pr_debug("%s: csk %p; tid %u; state %d\n", 1733 __func__, csk, tid, csk->com.state); 1734 1735 if (cxgb_is_neg_adv(hdr->status)) { 1736 pr_err("%s: got neg advise %d on tid %u\n", 1737 __func__, hdr->status, tid); 1738 goto rel_skb; 1739 } 1740 1741 switch (csk->com.state) { 1742 case CSK_STATE_CONNECTING: 1743 case CSK_STATE_MORIBUND: 1744 csk->com.state = CSK_STATE_DEAD; 1745 release = true; 1746 break; 1747 case CSK_STATE_ESTABLISHED: 1748 csk->com.state = CSK_STATE_DEAD; 1749 wakeup_thread = true; 1750 break; 1751 case CSK_STATE_CLOSING: 1752 csk->com.state = CSK_STATE_DEAD; 1753 if (!csk->conn) 1754 release = true; 1755 break; 1756 case CSK_STATE_ABORTING: 1757 break; 1758 default: 1759 pr_info("%s: cpl_abort_req_rss in bad state %d\n", 1760 __func__, csk->com.state); 1761 csk->com.state = CSK_STATE_DEAD; 1762 } 1763 1764 __skb_queue_purge(&csk->txq); 1765 1766 if (!test_and_set_bit(CSK_TX_DATA_SENT, &csk->com.flags)) 1767 cxgbit_send_tx_flowc_wr(csk); 1768 1769 rpl_skb = __skb_dequeue(&csk->skbq); 1770 1771 cxgb_mk_abort_rpl(rpl_skb, len, csk->tid, csk->txq_idx); 1772 cxgbit_ofld_send(csk->com.cdev, rpl_skb); 1773 1774 if (wakeup_thread) { 1775 cxgbit_queue_rx_skb(csk, skb); 1776 return; 1777 } 1778 1779 if (release) 1780 cxgbit_put_csk(csk); 1781rel_skb: 1782 __kfree_skb(skb); 1783} 1784 1785static void cxgbit_abort_rpl_rss(struct cxgbit_sock *csk, struct sk_buff *skb) 1786{ 1787 struct cpl_abort_rpl_rss *rpl = cplhdr(skb); 1788 1789 pr_debug("%s: csk %p; tid %u; state %d\n", 1790 __func__, csk, csk->tid, csk->com.state); 1791 1792 switch (csk->com.state) { 1793 case CSK_STATE_ABORTING: 1794 csk->com.state = CSK_STATE_DEAD; 1795 if (test_bit(CSK_ABORT_RPL_WAIT, &csk->com.flags)) 1796 cxgbit_wake_up(&csk->com.wr_wait, __func__, 1797 rpl->status); 1798 cxgbit_put_csk(csk); 1799 break; 1800 default: 1801 pr_info("%s: cpl_abort_rpl_rss in state %d\n", 1802 __func__, csk->com.state); 1803 } 1804 1805 __kfree_skb(skb); 1806} 1807 1808static bool cxgbit_credit_err(const struct cxgbit_sock *csk) 1809{ 1810 const struct sk_buff *skb = csk->wr_pending_head; 1811 u32 credit = 0; 1812 1813 if (unlikely(csk->wr_cred > csk->wr_max_cred)) { 1814 pr_err("csk 0x%p, tid %u, credit %u > %u\n", 1815 csk, csk->tid, csk->wr_cred, csk->wr_max_cred); 1816 return true; 1817 } 1818 1819 while (skb) { 1820 credit += (__force u32)skb->csum; 1821 skb = cxgbit_skcb_tx_wr_next(skb); 1822 } 1823 1824 if (unlikely((csk->wr_cred + credit) != csk->wr_max_cred)) { 1825 pr_err("csk 0x%p, tid %u, credit %u + %u != %u.\n", 1826 csk, csk->tid, csk->wr_cred, 1827 credit, csk->wr_max_cred); 1828 1829 return true; 1830 } 1831 1832 return false; 1833} 1834 1835static void cxgbit_fw4_ack(struct cxgbit_sock *csk, struct sk_buff *skb) 1836{ 1837 struct cpl_fw4_ack *rpl = (struct cpl_fw4_ack *)cplhdr(skb); 1838 u32 credits = rpl->credits; 1839 u32 snd_una = ntohl(rpl->snd_una); 1840 1841 csk->wr_cred += credits; 1842 if (csk->wr_una_cred > (csk->wr_max_cred - csk->wr_cred)) 1843 csk->wr_una_cred = csk->wr_max_cred - csk->wr_cred; 1844 1845 while (credits) { 1846 struct sk_buff *p = cxgbit_sock_peek_wr(csk); 1847 u32 csum; 1848 1849 if (unlikely(!p)) { 1850 pr_err("csk 0x%p,%u, cr %u,%u+%u, empty.\n", 1851 csk, csk->tid, credits, 1852 csk->wr_cred, csk->wr_una_cred); 1853 break; 1854 } 1855 1856 csum = (__force u32)p->csum; 1857 if (unlikely(credits < csum)) { 1858 pr_warn("csk 0x%p,%u, cr %u,%u+%u, < %u.\n", 1859 csk, csk->tid, 1860 credits, csk->wr_cred, csk->wr_una_cred, 1861 csum); 1862 p->csum = (__force __wsum)(csum - credits); 1863 break; 1864 } 1865 1866 cxgbit_sock_dequeue_wr(csk); 1867 credits -= csum; 1868 kfree_skb(p); 1869 } 1870 1871 if (unlikely(cxgbit_credit_err(csk))) { 1872 cxgbit_queue_rx_skb(csk, skb); 1873 return; 1874 } 1875 1876 if (rpl->seq_vld & CPL_FW4_ACK_FLAGS_SEQVAL) { 1877 if (unlikely(before(snd_una, csk->snd_una))) { 1878 pr_warn("csk 0x%p,%u, snd_una %u/%u.", 1879 csk, csk->tid, snd_una, 1880 csk->snd_una); 1881 goto rel_skb; 1882 } 1883 1884 if (csk->snd_una != snd_una) { 1885 csk->snd_una = snd_una; 1886 dst_confirm(csk->dst); 1887 } 1888 } 1889 1890 if (skb_queue_len(&csk->txq)) 1891 cxgbit_push_tx_frames(csk); 1892 1893rel_skb: 1894 __kfree_skb(skb); 1895} 1896 1897static void cxgbit_set_tcb_rpl(struct cxgbit_device *cdev, struct sk_buff *skb) 1898{ 1899 struct cxgbit_sock *csk; 1900 struct cpl_set_tcb_rpl *rpl = (struct cpl_set_tcb_rpl *)skb->data; 1901 unsigned int tid = GET_TID(rpl); 1902 struct cxgb4_lld_info *lldi = &cdev->lldi; 1903 struct tid_info *t = lldi->tids; 1904 1905 csk = lookup_tid(t, tid); 1906 if (unlikely(!csk)) { 1907 pr_err("can't find connection for tid %u.\n", tid); 1908 goto rel_skb; 1909 } else { 1910 cxgbit_wake_up(&csk->com.wr_wait, __func__, rpl->status); 1911 } 1912 1913 cxgbit_put_csk(csk); 1914rel_skb: 1915 __kfree_skb(skb); 1916} 1917 1918static void cxgbit_rx_data(struct cxgbit_device *cdev, struct sk_buff *skb) 1919{ 1920 struct cxgbit_sock *csk; 1921 struct cpl_rx_data *cpl = cplhdr(skb); 1922 unsigned int tid = GET_TID(cpl); 1923 struct cxgb4_lld_info *lldi = &cdev->lldi; 1924 struct tid_info *t = lldi->tids; 1925 1926 csk = lookup_tid(t, tid); 1927 if (unlikely(!csk)) { 1928 pr_err("can't find conn. for tid %u.\n", tid); 1929 goto rel_skb; 1930 } 1931 1932 cxgbit_queue_rx_skb(csk, skb); 1933 return; 1934rel_skb: 1935 __kfree_skb(skb); 1936} 1937 1938static void 1939__cxgbit_process_rx_cpl(struct cxgbit_sock *csk, struct sk_buff *skb) 1940{ 1941 spin_lock(&csk->lock); 1942 if (csk->lock_owner) { 1943 __skb_queue_tail(&csk->backlogq, skb); 1944 spin_unlock(&csk->lock); 1945 return; 1946 } 1947 1948 cxgbit_skcb_rx_backlog_fn(skb)(csk, skb); 1949 spin_unlock(&csk->lock); 1950} 1951 1952static void cxgbit_process_rx_cpl(struct cxgbit_sock *csk, struct sk_buff *skb) 1953{ 1954 cxgbit_get_csk(csk); 1955 __cxgbit_process_rx_cpl(csk, skb); 1956 cxgbit_put_csk(csk); 1957} 1958 1959static void cxgbit_rx_cpl(struct cxgbit_device *cdev, struct sk_buff *skb) 1960{ 1961 struct cxgbit_sock *csk; 1962 struct cpl_tx_data *cpl = cplhdr(skb); 1963 struct cxgb4_lld_info *lldi = &cdev->lldi; 1964 struct tid_info *t = lldi->tids; 1965 unsigned int tid = GET_TID(cpl); 1966 u8 opcode = cxgbit_skcb_rx_opcode(skb); 1967 bool ref = true; 1968 1969 switch (opcode) { 1970 case CPL_FW4_ACK: 1971 cxgbit_skcb_rx_backlog_fn(skb) = cxgbit_fw4_ack; 1972 ref = false; 1973 break; 1974 case CPL_PEER_CLOSE: 1975 cxgbit_skcb_rx_backlog_fn(skb) = cxgbit_peer_close; 1976 break; 1977 case CPL_CLOSE_CON_RPL: 1978 cxgbit_skcb_rx_backlog_fn(skb) = cxgbit_close_con_rpl; 1979 break; 1980 case CPL_ABORT_REQ_RSS: 1981 cxgbit_skcb_rx_backlog_fn(skb) = cxgbit_abort_req_rss; 1982 break; 1983 case CPL_ABORT_RPL_RSS: 1984 cxgbit_skcb_rx_backlog_fn(skb) = cxgbit_abort_rpl_rss; 1985 break; 1986 default: 1987 goto rel_skb; 1988 } 1989 1990 csk = lookup_tid(t, tid); 1991 if (unlikely(!csk)) { 1992 pr_err("can't find conn. for tid %u.\n", tid); 1993 goto rel_skb; 1994 } 1995 1996 if (ref) 1997 cxgbit_process_rx_cpl(csk, skb); 1998 else 1999 __cxgbit_process_rx_cpl(csk, skb); 2000 2001 return; 2002rel_skb: 2003 __kfree_skb(skb); 2004} 2005 2006cxgbit_cplhandler_func cxgbit_cplhandlers[NUM_CPL_CMDS] = { 2007 [CPL_PASS_OPEN_RPL] = cxgbit_pass_open_rpl, 2008 [CPL_CLOSE_LISTSRV_RPL] = cxgbit_close_listsrv_rpl, 2009 [CPL_PASS_ACCEPT_REQ] = cxgbit_pass_accept_req, 2010 [CPL_PASS_ESTABLISH] = cxgbit_pass_establish, 2011 [CPL_SET_TCB_RPL] = cxgbit_set_tcb_rpl, 2012 [CPL_RX_DATA] = cxgbit_rx_data, 2013 [CPL_FW4_ACK] = cxgbit_rx_cpl, 2014 [CPL_PEER_CLOSE] = cxgbit_rx_cpl, 2015 [CPL_CLOSE_CON_RPL] = cxgbit_rx_cpl, 2016 [CPL_ABORT_REQ_RSS] = cxgbit_rx_cpl, 2017 [CPL_ABORT_RPL_RSS] = cxgbit_rx_cpl, 2018}; 2019