1// SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB 2/* Copyright (c) 2015 - 2021 Intel Corporation */ 3#include "main.h" 4 5/** 6 * irdma_arp_table -manage arp table 7 * @rf: RDMA PCI function 8 * @ip_addr: ip address for device 9 * @ipv4: IPv4 flag 10 * @mac_addr: mac address ptr 11 * @action: modify, delete or add 12 */ 13int irdma_arp_table(struct irdma_pci_f *rf, u32 *ip_addr, bool ipv4, 14 const u8 *mac_addr, u32 action) 15{ 16 unsigned long flags; 17 int arp_index; 18 u32 ip[4] = {}; 19 20 if (ipv4) 21 ip[0] = *ip_addr; 22 else 23 memcpy(ip, ip_addr, sizeof(ip)); 24 25 spin_lock_irqsave(&rf->arp_lock, flags); 26 for (arp_index = 0; (u32)arp_index < rf->arp_table_size; arp_index++) { 27 if (!memcmp(rf->arp_table[arp_index].ip_addr, ip, sizeof(ip))) 28 break; 29 } 30 31 switch (action) { 32 case IRDMA_ARP_ADD: 33 if (arp_index != rf->arp_table_size) { 34 arp_index = -1; 35 break; 36 } 37 38 arp_index = 0; 39 if (irdma_alloc_rsrc(rf, rf->allocated_arps, rf->arp_table_size, 40 (u32 *)&arp_index, &rf->next_arp_index)) { 41 arp_index = -1; 42 break; 43 } 44 45 memcpy(rf->arp_table[arp_index].ip_addr, ip, 46 sizeof(rf->arp_table[arp_index].ip_addr)); 47 ether_addr_copy(rf->arp_table[arp_index].mac_addr, mac_addr); 48 break; 49 case IRDMA_ARP_RESOLVE: 50 if (arp_index == rf->arp_table_size) 51 arp_index = -1; 52 break; 53 case IRDMA_ARP_DELETE: 54 if (arp_index == rf->arp_table_size) { 55 arp_index = -1; 56 break; 57 } 58 59 memset(rf->arp_table[arp_index].ip_addr, 0, 60 sizeof(rf->arp_table[arp_index].ip_addr)); 61 eth_zero_addr(rf->arp_table[arp_index].mac_addr); 62 irdma_free_rsrc(rf, rf->allocated_arps, arp_index); 63 break; 64 default: 65 arp_index = -1; 66 break; 67 } 68 69 spin_unlock_irqrestore(&rf->arp_lock, flags); 70 return arp_index; 71} 72 73/** 74 * irdma_add_arp - add a new arp entry if needed 75 * @rf: RDMA function 76 * @ip: IP address 77 * @ipv4: IPv4 flag 78 * @mac: MAC address 79 */ 80int irdma_add_arp(struct irdma_pci_f *rf, u32 *ip, bool ipv4, const u8 *mac) 81{ 82 int arpidx; 83 84 arpidx = irdma_arp_table(rf, &ip[0], ipv4, NULL, IRDMA_ARP_RESOLVE); 85 if (arpidx >= 0) { 86 if (ether_addr_equal(rf->arp_table[arpidx].mac_addr, mac)) 87 return arpidx; 88 89 irdma_manage_arp_cache(rf, rf->arp_table[arpidx].mac_addr, ip, 90 ipv4, IRDMA_ARP_DELETE); 91 } 92 93 irdma_manage_arp_cache(rf, mac, ip, ipv4, IRDMA_ARP_ADD); 94 95 return irdma_arp_table(rf, ip, ipv4, NULL, IRDMA_ARP_RESOLVE); 96} 97 98/** 99 * wr32 - write 32 bits to hw register 100 * @hw: hardware information including registers 101 * @reg: register offset 102 * @val: value to write to register 103 */ 104inline void wr32(struct irdma_hw *hw, u32 reg, u32 val) 105{ 106 writel(val, hw->hw_addr + reg); 107} 108 109/** 110 * rd32 - read a 32 bit hw register 111 * @hw: hardware information including registers 112 * @reg: register offset 113 * 114 * Return value of register content 115 */ 116inline u32 rd32(struct irdma_hw *hw, u32 reg) 117{ 118 return readl(hw->hw_addr + reg); 119} 120 121/** 122 * rd64 - read a 64 bit hw register 123 * @hw: hardware information including registers 124 * @reg: register offset 125 * 126 * Return value of register content 127 */ 128inline u64 rd64(struct irdma_hw *hw, u32 reg) 129{ 130 return readq(hw->hw_addr + reg); 131} 132 133static void irdma_gid_change_event(struct ib_device *ibdev) 134{ 135 struct ib_event ib_event; 136 137 ib_event.event = IB_EVENT_GID_CHANGE; 138 ib_event.device = ibdev; 139 ib_event.element.port_num = 1; 140 ib_dispatch_event(&ib_event); 141} 142 143/** 144 * irdma_inetaddr_event - system notifier for ipv4 addr events 145 * @notifier: not used 146 * @event: event for notifier 147 * @ptr: if address 148 */ 149int irdma_inetaddr_event(struct notifier_block *notifier, unsigned long event, 150 void *ptr) 151{ 152 struct in_ifaddr *ifa = ptr; 153 struct net_device *real_dev, *netdev = ifa->ifa_dev->dev; 154 struct irdma_device *iwdev; 155 struct ib_device *ibdev; 156 u32 local_ipaddr; 157 158 real_dev = rdma_vlan_dev_real_dev(netdev); 159 if (!real_dev) 160 real_dev = netdev; 161 162 ibdev = ib_device_get_by_netdev(real_dev, RDMA_DRIVER_IRDMA); 163 if (!ibdev) 164 return NOTIFY_DONE; 165 166 iwdev = to_iwdev(ibdev); 167 local_ipaddr = ntohl(ifa->ifa_address); 168 ibdev_dbg(&iwdev->ibdev, 169 "DEV: netdev %p event %lu local_ip=%pI4 MAC=%pM\n", real_dev, 170 event, &local_ipaddr, real_dev->dev_addr); 171 switch (event) { 172 case NETDEV_DOWN: 173 irdma_manage_arp_cache(iwdev->rf, real_dev->dev_addr, 174 &local_ipaddr, true, IRDMA_ARP_DELETE); 175 irdma_if_notify(iwdev, real_dev, &local_ipaddr, true, false); 176 irdma_gid_change_event(&iwdev->ibdev); 177 break; 178 case NETDEV_UP: 179 case NETDEV_CHANGEADDR: 180 irdma_add_arp(iwdev->rf, &local_ipaddr, true, real_dev->dev_addr); 181 irdma_if_notify(iwdev, real_dev, &local_ipaddr, true, true); 182 irdma_gid_change_event(&iwdev->ibdev); 183 break; 184 default: 185 break; 186 } 187 188 ib_device_put(ibdev); 189 190 return NOTIFY_DONE; 191} 192 193/** 194 * irdma_inet6addr_event - system notifier for ipv6 addr events 195 * @notifier: not used 196 * @event: event for notifier 197 * @ptr: if address 198 */ 199int irdma_inet6addr_event(struct notifier_block *notifier, unsigned long event, 200 void *ptr) 201{ 202 struct inet6_ifaddr *ifa = ptr; 203 struct net_device *real_dev, *netdev = ifa->idev->dev; 204 struct irdma_device *iwdev; 205 struct ib_device *ibdev; 206 u32 local_ipaddr6[4]; 207 208 real_dev = rdma_vlan_dev_real_dev(netdev); 209 if (!real_dev) 210 real_dev = netdev; 211 212 ibdev = ib_device_get_by_netdev(real_dev, RDMA_DRIVER_IRDMA); 213 if (!ibdev) 214 return NOTIFY_DONE; 215 216 iwdev = to_iwdev(ibdev); 217 irdma_copy_ip_ntohl(local_ipaddr6, ifa->addr.in6_u.u6_addr32); 218 ibdev_dbg(&iwdev->ibdev, 219 "DEV: netdev %p event %lu local_ip=%pI6 MAC=%pM\n", real_dev, 220 event, local_ipaddr6, real_dev->dev_addr); 221 switch (event) { 222 case NETDEV_DOWN: 223 irdma_manage_arp_cache(iwdev->rf, real_dev->dev_addr, 224 local_ipaddr6, false, IRDMA_ARP_DELETE); 225 irdma_if_notify(iwdev, real_dev, local_ipaddr6, false, false); 226 irdma_gid_change_event(&iwdev->ibdev); 227 break; 228 case NETDEV_UP: 229 case NETDEV_CHANGEADDR: 230 irdma_add_arp(iwdev->rf, local_ipaddr6, false, 231 real_dev->dev_addr); 232 irdma_if_notify(iwdev, real_dev, local_ipaddr6, false, true); 233 irdma_gid_change_event(&iwdev->ibdev); 234 break; 235 default: 236 break; 237 } 238 239 ib_device_put(ibdev); 240 241 return NOTIFY_DONE; 242} 243 244/** 245 * irdma_net_event - system notifier for net events 246 * @notifier: not used 247 * @event: event for notifier 248 * @ptr: neighbor 249 */ 250int irdma_net_event(struct notifier_block *notifier, unsigned long event, 251 void *ptr) 252{ 253 struct neighbour *neigh = ptr; 254 struct net_device *real_dev, *netdev = (struct net_device *)neigh->dev; 255 struct irdma_device *iwdev; 256 struct ib_device *ibdev; 257 __be32 *p; 258 u32 local_ipaddr[4] = {}; 259 bool ipv4 = true; 260 261 switch (event) { 262 case NETEVENT_NEIGH_UPDATE: 263 real_dev = rdma_vlan_dev_real_dev(netdev); 264 if (!real_dev) 265 real_dev = netdev; 266 ibdev = ib_device_get_by_netdev(real_dev, RDMA_DRIVER_IRDMA); 267 if (!ibdev) 268 return NOTIFY_DONE; 269 270 iwdev = to_iwdev(ibdev); 271 p = (__be32 *)neigh->primary_key; 272 if (neigh->tbl->family == AF_INET6) { 273 ipv4 = false; 274 irdma_copy_ip_ntohl(local_ipaddr, p); 275 } else { 276 local_ipaddr[0] = ntohl(*p); 277 } 278 279 ibdev_dbg(&iwdev->ibdev, 280 "DEV: netdev %p state %d local_ip=%pI4 MAC=%pM\n", 281 iwdev->netdev, neigh->nud_state, local_ipaddr, 282 neigh->ha); 283 284 if (neigh->nud_state & NUD_VALID) 285 irdma_add_arp(iwdev->rf, local_ipaddr, ipv4, neigh->ha); 286 287 else 288 irdma_manage_arp_cache(iwdev->rf, neigh->ha, 289 local_ipaddr, ipv4, 290 IRDMA_ARP_DELETE); 291 ib_device_put(ibdev); 292 break; 293 default: 294 break; 295 } 296 297 return NOTIFY_DONE; 298} 299 300/** 301 * irdma_netdevice_event - system notifier for netdev events 302 * @notifier: not used 303 * @event: event for notifier 304 * @ptr: netdev 305 */ 306int irdma_netdevice_event(struct notifier_block *notifier, unsigned long event, 307 void *ptr) 308{ 309 struct irdma_device *iwdev; 310 struct ib_device *ibdev; 311 struct net_device *netdev = netdev_notifier_info_to_dev(ptr); 312 313 ibdev = ib_device_get_by_netdev(netdev, RDMA_DRIVER_IRDMA); 314 if (!ibdev) 315 return NOTIFY_DONE; 316 317 iwdev = to_iwdev(ibdev); 318 iwdev->iw_status = 1; 319 switch (event) { 320 case NETDEV_DOWN: 321 iwdev->iw_status = 0; 322 fallthrough; 323 case NETDEV_UP: 324 irdma_port_ibevent(iwdev); 325 break; 326 default: 327 break; 328 } 329 ib_device_put(ibdev); 330 331 return NOTIFY_DONE; 332} 333 334/** 335 * irdma_add_ipv6_addr - add ipv6 address to the hw arp table 336 * @iwdev: irdma device 337 */ 338static void irdma_add_ipv6_addr(struct irdma_device *iwdev) 339{ 340 struct net_device *ip_dev; 341 struct inet6_dev *idev; 342 struct inet6_ifaddr *ifp, *tmp; 343 u32 local_ipaddr6[4]; 344 345 rcu_read_lock(); 346 for_each_netdev_rcu (&init_net, ip_dev) { 347 if (((rdma_vlan_dev_vlan_id(ip_dev) < 0xFFFF && 348 rdma_vlan_dev_real_dev(ip_dev) == iwdev->netdev) || 349 ip_dev == iwdev->netdev) && 350 (READ_ONCE(ip_dev->flags) & IFF_UP)) { 351 idev = __in6_dev_get(ip_dev); 352 if (!idev) { 353 ibdev_err(&iwdev->ibdev, "ipv6 inet device not found\n"); 354 break; 355 } 356 list_for_each_entry_safe (ifp, tmp, &idev->addr_list, 357 if_list) { 358 ibdev_dbg(&iwdev->ibdev, 359 "INIT: IP=%pI6, vlan_id=%d, MAC=%pM\n", 360 &ifp->addr, 361 rdma_vlan_dev_vlan_id(ip_dev), 362 ip_dev->dev_addr); 363 364 irdma_copy_ip_ntohl(local_ipaddr6, 365 ifp->addr.in6_u.u6_addr32); 366 irdma_manage_arp_cache(iwdev->rf, 367 ip_dev->dev_addr, 368 local_ipaddr6, false, 369 IRDMA_ARP_ADD); 370 } 371 } 372 } 373 rcu_read_unlock(); 374} 375 376/** 377 * irdma_add_ipv4_addr - add ipv4 address to the hw arp table 378 * @iwdev: irdma device 379 */ 380static void irdma_add_ipv4_addr(struct irdma_device *iwdev) 381{ 382 struct net_device *dev; 383 struct in_device *idev; 384 u32 ip_addr; 385 386 rcu_read_lock(); 387 for_each_netdev_rcu (&init_net, dev) { 388 if (((rdma_vlan_dev_vlan_id(dev) < 0xFFFF && 389 rdma_vlan_dev_real_dev(dev) == iwdev->netdev) || 390 dev == iwdev->netdev) && (READ_ONCE(dev->flags) & IFF_UP)) { 391 const struct in_ifaddr *ifa; 392 393 idev = __in_dev_get_rcu(dev); 394 if (!idev) 395 continue; 396 397 in_dev_for_each_ifa_rcu(ifa, idev) { 398 ibdev_dbg(&iwdev->ibdev, "CM: IP=%pI4, vlan_id=%d, MAC=%pM\n", 399 &ifa->ifa_address, rdma_vlan_dev_vlan_id(dev), 400 dev->dev_addr); 401 402 ip_addr = ntohl(ifa->ifa_address); 403 irdma_manage_arp_cache(iwdev->rf, dev->dev_addr, 404 &ip_addr, true, 405 IRDMA_ARP_ADD); 406 } 407 } 408 } 409 rcu_read_unlock(); 410} 411 412/** 413 * irdma_add_ip - add ip addresses 414 * @iwdev: irdma device 415 * 416 * Add ipv4/ipv6 addresses to the arp cache 417 */ 418void irdma_add_ip(struct irdma_device *iwdev) 419{ 420 irdma_add_ipv4_addr(iwdev); 421 irdma_add_ipv6_addr(iwdev); 422} 423 424/** 425 * irdma_alloc_and_get_cqp_request - get cqp struct 426 * @cqp: device cqp ptr 427 * @wait: cqp to be used in wait mode 428 */ 429struct irdma_cqp_request *irdma_alloc_and_get_cqp_request(struct irdma_cqp *cqp, 430 bool wait) 431{ 432 struct irdma_cqp_request *cqp_request = NULL; 433 unsigned long flags; 434 435 spin_lock_irqsave(&cqp->req_lock, flags); 436 if (!list_empty(&cqp->cqp_avail_reqs)) { 437 cqp_request = list_first_entry(&cqp->cqp_avail_reqs, 438 struct irdma_cqp_request, list); 439 list_del_init(&cqp_request->list); 440 } 441 spin_unlock_irqrestore(&cqp->req_lock, flags); 442 if (!cqp_request) { 443 cqp_request = kzalloc(sizeof(*cqp_request), GFP_ATOMIC); 444 if (cqp_request) { 445 cqp_request->dynamic = true; 446 if (wait) 447 init_waitqueue_head(&cqp_request->waitq); 448 } 449 } 450 if (!cqp_request) { 451 ibdev_dbg(to_ibdev(cqp->sc_cqp.dev), "ERR: CQP Request Fail: No Memory"); 452 return NULL; 453 } 454 455 cqp_request->waiting = wait; 456 refcount_set(&cqp_request->refcnt, 1); 457 memset(&cqp_request->compl_info, 0, sizeof(cqp_request->compl_info)); 458 459 return cqp_request; 460} 461 462/** 463 * irdma_get_cqp_request - increase refcount for cqp_request 464 * @cqp_request: pointer to cqp_request instance 465 */ 466static inline void irdma_get_cqp_request(struct irdma_cqp_request *cqp_request) 467{ 468 refcount_inc(&cqp_request->refcnt); 469} 470 471/** 472 * irdma_free_cqp_request - free cqp request 473 * @cqp: cqp ptr 474 * @cqp_request: to be put back in cqp list 475 */ 476void irdma_free_cqp_request(struct irdma_cqp *cqp, 477 struct irdma_cqp_request *cqp_request) 478{ 479 unsigned long flags; 480 481 if (cqp_request->dynamic) { 482 kfree(cqp_request); 483 } else { 484 WRITE_ONCE(cqp_request->request_done, false); 485 cqp_request->callback_fcn = NULL; 486 cqp_request->waiting = false; 487 488 spin_lock_irqsave(&cqp->req_lock, flags); 489 list_add_tail(&cqp_request->list, &cqp->cqp_avail_reqs); 490 spin_unlock_irqrestore(&cqp->req_lock, flags); 491 } 492 wake_up(&cqp->remove_wq); 493} 494 495/** 496 * irdma_put_cqp_request - dec ref count and free if 0 497 * @cqp: cqp ptr 498 * @cqp_request: to be put back in cqp list 499 */ 500void irdma_put_cqp_request(struct irdma_cqp *cqp, 501 struct irdma_cqp_request *cqp_request) 502{ 503 if (refcount_dec_and_test(&cqp_request->refcnt)) 504 irdma_free_cqp_request(cqp, cqp_request); 505} 506 507/** 508 * irdma_free_pending_cqp_request -free pending cqp request objs 509 * @cqp: cqp ptr 510 * @cqp_request: to be put back in cqp list 511 */ 512static void 513irdma_free_pending_cqp_request(struct irdma_cqp *cqp, 514 struct irdma_cqp_request *cqp_request) 515{ 516 if (cqp_request->waiting) { 517 cqp_request->compl_info.error = true; 518 WRITE_ONCE(cqp_request->request_done, true); 519 wake_up(&cqp_request->waitq); 520 } 521 wait_event_timeout(cqp->remove_wq, 522 refcount_read(&cqp_request->refcnt) == 1, 1000); 523 irdma_put_cqp_request(cqp, cqp_request); 524} 525 526/** 527 * irdma_cleanup_pending_cqp_op - clean-up cqp with no 528 * completions 529 * @rf: RDMA PCI function 530 */ 531void irdma_cleanup_pending_cqp_op(struct irdma_pci_f *rf) 532{ 533 struct irdma_sc_dev *dev = &rf->sc_dev; 534 struct irdma_cqp *cqp = &rf->cqp; 535 struct irdma_cqp_request *cqp_request = NULL; 536 struct cqp_cmds_info *pcmdinfo = NULL; 537 u32 i, pending_work, wqe_idx; 538 539 pending_work = IRDMA_RING_USED_QUANTA(cqp->sc_cqp.sq_ring); 540 wqe_idx = IRDMA_RING_CURRENT_TAIL(cqp->sc_cqp.sq_ring); 541 for (i = 0; i < pending_work; i++) { 542 cqp_request = (struct irdma_cqp_request *)(unsigned long) 543 cqp->scratch_array[wqe_idx]; 544 if (cqp_request) 545 irdma_free_pending_cqp_request(cqp, cqp_request); 546 wqe_idx = (wqe_idx + 1) % IRDMA_RING_SIZE(cqp->sc_cqp.sq_ring); 547 } 548 549 while (!list_empty(&dev->cqp_cmd_head)) { 550 pcmdinfo = irdma_remove_cqp_head(dev); 551 cqp_request = 552 container_of(pcmdinfo, struct irdma_cqp_request, info); 553 if (cqp_request) 554 irdma_free_pending_cqp_request(cqp, cqp_request); 555 } 556} 557 558/** 559 * irdma_wait_event - wait for completion 560 * @rf: RDMA PCI function 561 * @cqp_request: cqp request to wait 562 */ 563static int irdma_wait_event(struct irdma_pci_f *rf, 564 struct irdma_cqp_request *cqp_request) 565{ 566 struct irdma_cqp_timeout cqp_timeout = {}; 567 bool cqp_error = false; 568 int err_code = 0; 569 570 cqp_timeout.compl_cqp_cmds = atomic64_read(&rf->sc_dev.cqp->completed_ops); 571 do { 572 irdma_cqp_ce_handler(rf, &rf->ccq.sc_cq); 573 if (wait_event_timeout(cqp_request->waitq, 574 READ_ONCE(cqp_request->request_done), 575 msecs_to_jiffies(CQP_COMPL_WAIT_TIME_MS))) 576 break; 577 578 irdma_check_cqp_progress(&cqp_timeout, &rf->sc_dev); 579 580 if (cqp_timeout.count < CQP_TIMEOUT_THRESHOLD) 581 continue; 582 583 if (!rf->reset) { 584 rf->reset = true; 585 rf->gen_ops.request_reset(rf); 586 } 587 return -ETIMEDOUT; 588 } while (1); 589 590 cqp_error = cqp_request->compl_info.error; 591 if (cqp_error) { 592 err_code = -EIO; 593 if (cqp_request->compl_info.maj_err_code == 0xFFFF) { 594 if (cqp_request->compl_info.min_err_code == 0x8002) 595 err_code = -EBUSY; 596 else if (cqp_request->compl_info.min_err_code == 0x8029) { 597 if (!rf->reset) { 598 rf->reset = true; 599 rf->gen_ops.request_reset(rf); 600 } 601 } 602 } 603 } 604 605 return err_code; 606} 607 608static const char *const irdma_cqp_cmd_names[IRDMA_MAX_CQP_OPS] = { 609 [IRDMA_OP_CEQ_DESTROY] = "Destroy CEQ Cmd", 610 [IRDMA_OP_AEQ_DESTROY] = "Destroy AEQ Cmd", 611 [IRDMA_OP_DELETE_ARP_CACHE_ENTRY] = "Delete ARP Cache Cmd", 612 [IRDMA_OP_MANAGE_APBVT_ENTRY] = "Manage APBV Table Entry Cmd", 613 [IRDMA_OP_CEQ_CREATE] = "CEQ Create Cmd", 614 [IRDMA_OP_AEQ_CREATE] = "AEQ Destroy Cmd", 615 [IRDMA_OP_MANAGE_QHASH_TABLE_ENTRY] = "Manage Quad Hash Table Entry Cmd", 616 [IRDMA_OP_QP_MODIFY] = "Modify QP Cmd", 617 [IRDMA_OP_QP_UPLOAD_CONTEXT] = "Upload Context Cmd", 618 [IRDMA_OP_CQ_CREATE] = "Create CQ Cmd", 619 [IRDMA_OP_CQ_DESTROY] = "Destroy CQ Cmd", 620 [IRDMA_OP_QP_CREATE] = "Create QP Cmd", 621 [IRDMA_OP_QP_DESTROY] = "Destroy QP Cmd", 622 [IRDMA_OP_ALLOC_STAG] = "Allocate STag Cmd", 623 [IRDMA_OP_MR_REG_NON_SHARED] = "Register Non-Shared MR Cmd", 624 [IRDMA_OP_DEALLOC_STAG] = "Deallocate STag Cmd", 625 [IRDMA_OP_MW_ALLOC] = "Allocate Memory Window Cmd", 626 [IRDMA_OP_QP_FLUSH_WQES] = "Flush QP Cmd", 627 [IRDMA_OP_ADD_ARP_CACHE_ENTRY] = "Add ARP Cache Cmd", 628 [IRDMA_OP_MANAGE_PUSH_PAGE] = "Manage Push Page Cmd", 629 [IRDMA_OP_UPDATE_PE_SDS] = "Update PE SDs Cmd", 630 [IRDMA_OP_MANAGE_HMC_PM_FUNC_TABLE] = "Manage HMC PM Function Table Cmd", 631 [IRDMA_OP_SUSPEND] = "Suspend QP Cmd", 632 [IRDMA_OP_RESUME] = "Resume QP Cmd", 633 [IRDMA_OP_MANAGE_VF_PBLE_BP] = "Manage VF PBLE Backing Pages Cmd", 634 [IRDMA_OP_QUERY_FPM_VAL] = "Query FPM Values Cmd", 635 [IRDMA_OP_COMMIT_FPM_VAL] = "Commit FPM Values Cmd", 636 [IRDMA_OP_AH_CREATE] = "Create Address Handle Cmd", 637 [IRDMA_OP_AH_MODIFY] = "Modify Address Handle Cmd", 638 [IRDMA_OP_AH_DESTROY] = "Destroy Address Handle Cmd", 639 [IRDMA_OP_MC_CREATE] = "Create Multicast Group Cmd", 640 [IRDMA_OP_MC_DESTROY] = "Destroy Multicast Group Cmd", 641 [IRDMA_OP_MC_MODIFY] = "Modify Multicast Group Cmd", 642 [IRDMA_OP_STATS_ALLOCATE] = "Add Statistics Instance Cmd", 643 [IRDMA_OP_STATS_FREE] = "Free Statistics Instance Cmd", 644 [IRDMA_OP_STATS_GATHER] = "Gather Statistics Cmd", 645 [IRDMA_OP_WS_ADD_NODE] = "Add Work Scheduler Node Cmd", 646 [IRDMA_OP_WS_MODIFY_NODE] = "Modify Work Scheduler Node Cmd", 647 [IRDMA_OP_WS_DELETE_NODE] = "Delete Work Scheduler Node Cmd", 648 [IRDMA_OP_SET_UP_MAP] = "Set UP-UP Mapping Cmd", 649 [IRDMA_OP_GEN_AE] = "Generate AE Cmd", 650 [IRDMA_OP_QUERY_RDMA_FEATURES] = "RDMA Get Features Cmd", 651 [IRDMA_OP_ALLOC_LOCAL_MAC_ENTRY] = "Allocate Local MAC Entry Cmd", 652 [IRDMA_OP_ADD_LOCAL_MAC_ENTRY] = "Add Local MAC Entry Cmd", 653 [IRDMA_OP_DELETE_LOCAL_MAC_ENTRY] = "Delete Local MAC Entry Cmd", 654 [IRDMA_OP_CQ_MODIFY] = "CQ Modify Cmd", 655}; 656 657static const struct irdma_cqp_err_info irdma_noncrit_err_list[] = { 658 {0xffff, 0x8002, "Invalid State"}, 659 {0xffff, 0x8006, "Flush No Wqe Pending"}, 660 {0xffff, 0x8007, "Modify QP Bad Close"}, 661 {0xffff, 0x8009, "LLP Closed"}, 662 {0xffff, 0x800a, "Reset Not Sent"} 663}; 664 665/** 666 * irdma_cqp_crit_err - check if CQP error is critical 667 * @dev: pointer to dev structure 668 * @cqp_cmd: code for last CQP operation 669 * @maj_err_code: major error code 670 * @min_err_code: minot error code 671 */ 672bool irdma_cqp_crit_err(struct irdma_sc_dev *dev, u8 cqp_cmd, 673 u16 maj_err_code, u16 min_err_code) 674{ 675 int i; 676 677 for (i = 0; i < ARRAY_SIZE(irdma_noncrit_err_list); ++i) { 678 if (maj_err_code == irdma_noncrit_err_list[i].maj && 679 min_err_code == irdma_noncrit_err_list[i].min) { 680 ibdev_dbg(to_ibdev(dev), 681 "CQP: [%s Error][%s] maj=0x%x min=0x%x\n", 682 irdma_noncrit_err_list[i].desc, 683 irdma_cqp_cmd_names[cqp_cmd], maj_err_code, 684 min_err_code); 685 return false; 686 } 687 } 688 return true; 689} 690 691/** 692 * irdma_handle_cqp_op - process cqp command 693 * @rf: RDMA PCI function 694 * @cqp_request: cqp request to process 695 */ 696int irdma_handle_cqp_op(struct irdma_pci_f *rf, 697 struct irdma_cqp_request *cqp_request) 698{ 699 struct irdma_sc_dev *dev = &rf->sc_dev; 700 struct cqp_cmds_info *info = &cqp_request->info; 701 int status; 702 bool put_cqp_request = true; 703 704 if (rf->reset) 705 return -EBUSY; 706 707 irdma_get_cqp_request(cqp_request); 708 status = irdma_process_cqp_cmd(dev, info); 709 if (status) 710 goto err; 711 712 if (cqp_request->waiting) { 713 put_cqp_request = false; 714 status = irdma_wait_event(rf, cqp_request); 715 if (status) 716 goto err; 717 } 718 719 return 0; 720 721err: 722 if (irdma_cqp_crit_err(dev, info->cqp_cmd, 723 cqp_request->compl_info.maj_err_code, 724 cqp_request->compl_info.min_err_code)) 725 ibdev_err(&rf->iwdev->ibdev, 726 "[%s Error][op_code=%d] status=%d waiting=%d completion_err=%d maj=0x%x min=0x%x\n", 727 irdma_cqp_cmd_names[info->cqp_cmd], info->cqp_cmd, status, cqp_request->waiting, 728 cqp_request->compl_info.error, cqp_request->compl_info.maj_err_code, 729 cqp_request->compl_info.min_err_code); 730 731 if (put_cqp_request) 732 irdma_put_cqp_request(&rf->cqp, cqp_request); 733 734 return status; 735} 736 737void irdma_qp_add_ref(struct ib_qp *ibqp) 738{ 739 struct irdma_qp *iwqp = (struct irdma_qp *)ibqp; 740 741 refcount_inc(&iwqp->refcnt); 742} 743 744void irdma_qp_rem_ref(struct ib_qp *ibqp) 745{ 746 struct irdma_qp *iwqp = to_iwqp(ibqp); 747 struct irdma_device *iwdev = iwqp->iwdev; 748 u32 qp_num; 749 unsigned long flags; 750 751 spin_lock_irqsave(&iwdev->rf->qptable_lock, flags); 752 if (!refcount_dec_and_test(&iwqp->refcnt)) { 753 spin_unlock_irqrestore(&iwdev->rf->qptable_lock, flags); 754 return; 755 } 756 757 qp_num = iwqp->ibqp.qp_num; 758 iwdev->rf->qp_table[qp_num] = NULL; 759 spin_unlock_irqrestore(&iwdev->rf->qptable_lock, flags); 760 complete(&iwqp->free_qp); 761} 762 763void irdma_cq_add_ref(struct ib_cq *ibcq) 764{ 765 struct irdma_cq *iwcq = to_iwcq(ibcq); 766 767 refcount_inc(&iwcq->refcnt); 768} 769 770void irdma_cq_rem_ref(struct ib_cq *ibcq) 771{ 772 struct ib_device *ibdev = ibcq->device; 773 struct irdma_device *iwdev = to_iwdev(ibdev); 774 struct irdma_cq *iwcq = to_iwcq(ibcq); 775 unsigned long flags; 776 777 spin_lock_irqsave(&iwdev->rf->cqtable_lock, flags); 778 if (!refcount_dec_and_test(&iwcq->refcnt)) { 779 spin_unlock_irqrestore(&iwdev->rf->cqtable_lock, flags); 780 return; 781 } 782 783 iwdev->rf->cq_table[iwcq->cq_num] = NULL; 784 spin_unlock_irqrestore(&iwdev->rf->cqtable_lock, flags); 785 complete(&iwcq->free_cq); 786} 787 788struct ib_device *to_ibdev(struct irdma_sc_dev *dev) 789{ 790 return &(container_of(dev, struct irdma_pci_f, sc_dev))->iwdev->ibdev; 791} 792 793/** 794 * irdma_get_qp - get qp address 795 * @device: iwarp device 796 * @qpn: qp number 797 */ 798struct ib_qp *irdma_get_qp(struct ib_device *device, int qpn) 799{ 800 struct irdma_device *iwdev = to_iwdev(device); 801 802 if (qpn < IW_FIRST_QPN || qpn >= iwdev->rf->max_qp) 803 return NULL; 804 805 return &iwdev->rf->qp_table[qpn]->ibqp; 806} 807 808/** 809 * irdma_remove_cqp_head - return head entry and remove 810 * @dev: device 811 */ 812void *irdma_remove_cqp_head(struct irdma_sc_dev *dev) 813{ 814 struct list_head *entry; 815 struct list_head *list = &dev->cqp_cmd_head; 816 817 if (list_empty(list)) 818 return NULL; 819 820 entry = list->next; 821 list_del(entry); 822 823 return entry; 824} 825 826/** 827 * irdma_cqp_sds_cmd - create cqp command for sd 828 * @dev: hardware control device structure 829 * @sdinfo: information for sd cqp 830 * 831 */ 832int irdma_cqp_sds_cmd(struct irdma_sc_dev *dev, 833 struct irdma_update_sds_info *sdinfo) 834{ 835 struct irdma_cqp_request *cqp_request; 836 struct cqp_cmds_info *cqp_info; 837 struct irdma_pci_f *rf = dev_to_rf(dev); 838 int status; 839 840 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true); 841 if (!cqp_request) 842 return -ENOMEM; 843 844 cqp_info = &cqp_request->info; 845 memcpy(&cqp_info->in.u.update_pe_sds.info, sdinfo, 846 sizeof(cqp_info->in.u.update_pe_sds.info)); 847 cqp_info->cqp_cmd = IRDMA_OP_UPDATE_PE_SDS; 848 cqp_info->post_sq = 1; 849 cqp_info->in.u.update_pe_sds.dev = dev; 850 cqp_info->in.u.update_pe_sds.scratch = (uintptr_t)cqp_request; 851 852 status = irdma_handle_cqp_op(rf, cqp_request); 853 irdma_put_cqp_request(&rf->cqp, cqp_request); 854 855 return status; 856} 857 858/** 859 * irdma_cqp_qp_suspend_resume - cqp command for suspend/resume 860 * @qp: hardware control qp 861 * @op: suspend or resume 862 */ 863int irdma_cqp_qp_suspend_resume(struct irdma_sc_qp *qp, u8 op) 864{ 865 struct irdma_sc_dev *dev = qp->dev; 866 struct irdma_cqp_request *cqp_request; 867 struct irdma_sc_cqp *cqp = dev->cqp; 868 struct cqp_cmds_info *cqp_info; 869 struct irdma_pci_f *rf = dev_to_rf(dev); 870 int status; 871 872 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, false); 873 if (!cqp_request) 874 return -ENOMEM; 875 876 cqp_info = &cqp_request->info; 877 cqp_info->cqp_cmd = op; 878 cqp_info->in.u.suspend_resume.cqp = cqp; 879 cqp_info->in.u.suspend_resume.qp = qp; 880 cqp_info->in.u.suspend_resume.scratch = (uintptr_t)cqp_request; 881 882 status = irdma_handle_cqp_op(rf, cqp_request); 883 irdma_put_cqp_request(&rf->cqp, cqp_request); 884 885 return status; 886} 887 888/** 889 * irdma_term_modify_qp - modify qp for term message 890 * @qp: hardware control qp 891 * @next_state: qp's next state 892 * @term: terminate code 893 * @term_len: length 894 */ 895void irdma_term_modify_qp(struct irdma_sc_qp *qp, u8 next_state, u8 term, 896 u8 term_len) 897{ 898 struct irdma_qp *iwqp; 899 900 iwqp = qp->qp_uk.back_qp; 901 irdma_next_iw_state(iwqp, next_state, 0, term, term_len); 902}; 903 904/** 905 * irdma_terminate_done - after terminate is completed 906 * @qp: hardware control qp 907 * @timeout_occurred: indicates if terminate timer expired 908 */ 909void irdma_terminate_done(struct irdma_sc_qp *qp, int timeout_occurred) 910{ 911 struct irdma_qp *iwqp; 912 u8 hte = 0; 913 bool first_time; 914 unsigned long flags; 915 916 iwqp = qp->qp_uk.back_qp; 917 spin_lock_irqsave(&iwqp->lock, flags); 918 if (iwqp->hte_added) { 919 iwqp->hte_added = 0; 920 hte = 1; 921 } 922 first_time = !(qp->term_flags & IRDMA_TERM_DONE); 923 qp->term_flags |= IRDMA_TERM_DONE; 924 spin_unlock_irqrestore(&iwqp->lock, flags); 925 if (first_time) { 926 if (!timeout_occurred) 927 irdma_terminate_del_timer(qp); 928 929 irdma_next_iw_state(iwqp, IRDMA_QP_STATE_ERROR, hte, 0, 0); 930 irdma_cm_disconn(iwqp); 931 } 932} 933 934static void irdma_terminate_timeout(struct timer_list *t) 935{ 936 struct irdma_qp *iwqp = from_timer(iwqp, t, terminate_timer); 937 struct irdma_sc_qp *qp = &iwqp->sc_qp; 938 939 irdma_terminate_done(qp, 1); 940 irdma_qp_rem_ref(&iwqp->ibqp); 941} 942 943/** 944 * irdma_terminate_start_timer - start terminate timeout 945 * @qp: hardware control qp 946 */ 947void irdma_terminate_start_timer(struct irdma_sc_qp *qp) 948{ 949 struct irdma_qp *iwqp; 950 951 iwqp = qp->qp_uk.back_qp; 952 irdma_qp_add_ref(&iwqp->ibqp); 953 timer_setup(&iwqp->terminate_timer, irdma_terminate_timeout, 0); 954 iwqp->terminate_timer.expires = jiffies + HZ; 955 956 add_timer(&iwqp->terminate_timer); 957} 958 959/** 960 * irdma_terminate_del_timer - delete terminate timeout 961 * @qp: hardware control qp 962 */ 963void irdma_terminate_del_timer(struct irdma_sc_qp *qp) 964{ 965 struct irdma_qp *iwqp; 966 int ret; 967 968 iwqp = qp->qp_uk.back_qp; 969 ret = del_timer(&iwqp->terminate_timer); 970 if (ret) 971 irdma_qp_rem_ref(&iwqp->ibqp); 972} 973 974/** 975 * irdma_cqp_query_fpm_val_cmd - send cqp command for fpm 976 * @dev: function device struct 977 * @val_mem: buffer for fpm 978 * @hmc_fn_id: function id for fpm 979 */ 980int irdma_cqp_query_fpm_val_cmd(struct irdma_sc_dev *dev, 981 struct irdma_dma_mem *val_mem, u8 hmc_fn_id) 982{ 983 struct irdma_cqp_request *cqp_request; 984 struct cqp_cmds_info *cqp_info; 985 struct irdma_pci_f *rf = dev_to_rf(dev); 986 int status; 987 988 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true); 989 if (!cqp_request) 990 return -ENOMEM; 991 992 cqp_info = &cqp_request->info; 993 cqp_request->param = NULL; 994 cqp_info->in.u.query_fpm_val.cqp = dev->cqp; 995 cqp_info->in.u.query_fpm_val.fpm_val_pa = val_mem->pa; 996 cqp_info->in.u.query_fpm_val.fpm_val_va = val_mem->va; 997 cqp_info->in.u.query_fpm_val.hmc_fn_id = hmc_fn_id; 998 cqp_info->cqp_cmd = IRDMA_OP_QUERY_FPM_VAL; 999 cqp_info->post_sq = 1; 1000 cqp_info->in.u.query_fpm_val.scratch = (uintptr_t)cqp_request; 1001 1002 status = irdma_handle_cqp_op(rf, cqp_request); 1003 irdma_put_cqp_request(&rf->cqp, cqp_request); 1004 1005 return status; 1006} 1007 1008/** 1009 * irdma_cqp_commit_fpm_val_cmd - commit fpm values in hw 1010 * @dev: hardware control device structure 1011 * @val_mem: buffer with fpm values 1012 * @hmc_fn_id: function id for fpm 1013 */ 1014int irdma_cqp_commit_fpm_val_cmd(struct irdma_sc_dev *dev, 1015 struct irdma_dma_mem *val_mem, u8 hmc_fn_id) 1016{ 1017 struct irdma_cqp_request *cqp_request; 1018 struct cqp_cmds_info *cqp_info; 1019 struct irdma_pci_f *rf = dev_to_rf(dev); 1020 int status; 1021 1022 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true); 1023 if (!cqp_request) 1024 return -ENOMEM; 1025 1026 cqp_info = &cqp_request->info; 1027 cqp_request->param = NULL; 1028 cqp_info->in.u.commit_fpm_val.cqp = dev->cqp; 1029 cqp_info->in.u.commit_fpm_val.fpm_val_pa = val_mem->pa; 1030 cqp_info->in.u.commit_fpm_val.fpm_val_va = val_mem->va; 1031 cqp_info->in.u.commit_fpm_val.hmc_fn_id = hmc_fn_id; 1032 cqp_info->cqp_cmd = IRDMA_OP_COMMIT_FPM_VAL; 1033 cqp_info->post_sq = 1; 1034 cqp_info->in.u.commit_fpm_val.scratch = (uintptr_t)cqp_request; 1035 1036 status = irdma_handle_cqp_op(rf, cqp_request); 1037 irdma_put_cqp_request(&rf->cqp, cqp_request); 1038 1039 return status; 1040} 1041 1042/** 1043 * irdma_cqp_cq_create_cmd - create a cq for the cqp 1044 * @dev: device pointer 1045 * @cq: pointer to created cq 1046 */ 1047int irdma_cqp_cq_create_cmd(struct irdma_sc_dev *dev, struct irdma_sc_cq *cq) 1048{ 1049 struct irdma_pci_f *rf = dev_to_rf(dev); 1050 struct irdma_cqp *iwcqp = &rf->cqp; 1051 struct irdma_cqp_request *cqp_request; 1052 struct cqp_cmds_info *cqp_info; 1053 int status; 1054 1055 cqp_request = irdma_alloc_and_get_cqp_request(iwcqp, true); 1056 if (!cqp_request) 1057 return -ENOMEM; 1058 1059 cqp_info = &cqp_request->info; 1060 cqp_info->cqp_cmd = IRDMA_OP_CQ_CREATE; 1061 cqp_info->post_sq = 1; 1062 cqp_info->in.u.cq_create.cq = cq; 1063 cqp_info->in.u.cq_create.scratch = (uintptr_t)cqp_request; 1064 1065 status = irdma_handle_cqp_op(rf, cqp_request); 1066 irdma_put_cqp_request(iwcqp, cqp_request); 1067 1068 return status; 1069} 1070 1071/** 1072 * irdma_cqp_qp_create_cmd - create a qp for the cqp 1073 * @dev: device pointer 1074 * @qp: pointer to created qp 1075 */ 1076int irdma_cqp_qp_create_cmd(struct irdma_sc_dev *dev, struct irdma_sc_qp *qp) 1077{ 1078 struct irdma_pci_f *rf = dev_to_rf(dev); 1079 struct irdma_cqp *iwcqp = &rf->cqp; 1080 struct irdma_cqp_request *cqp_request; 1081 struct cqp_cmds_info *cqp_info; 1082 struct irdma_create_qp_info *qp_info; 1083 int status; 1084 1085 cqp_request = irdma_alloc_and_get_cqp_request(iwcqp, true); 1086 if (!cqp_request) 1087 return -ENOMEM; 1088 1089 cqp_info = &cqp_request->info; 1090 qp_info = &cqp_request->info.in.u.qp_create.info; 1091 memset(qp_info, 0, sizeof(*qp_info)); 1092 qp_info->cq_num_valid = true; 1093 qp_info->next_iwarp_state = IRDMA_QP_STATE_RTS; 1094 cqp_info->cqp_cmd = IRDMA_OP_QP_CREATE; 1095 cqp_info->post_sq = 1; 1096 cqp_info->in.u.qp_create.qp = qp; 1097 cqp_info->in.u.qp_create.scratch = (uintptr_t)cqp_request; 1098 1099 status = irdma_handle_cqp_op(rf, cqp_request); 1100 irdma_put_cqp_request(iwcqp, cqp_request); 1101 1102 return status; 1103} 1104 1105/** 1106 * irdma_dealloc_push_page - free a push page for qp 1107 * @rf: RDMA PCI function 1108 * @qp: hardware control qp 1109 */ 1110static void irdma_dealloc_push_page(struct irdma_pci_f *rf, 1111 struct irdma_sc_qp *qp) 1112{ 1113 struct irdma_cqp_request *cqp_request; 1114 struct cqp_cmds_info *cqp_info; 1115 int status; 1116 1117 if (qp->push_idx == IRDMA_INVALID_PUSH_PAGE_INDEX) 1118 return; 1119 1120 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, false); 1121 if (!cqp_request) 1122 return; 1123 1124 cqp_info = &cqp_request->info; 1125 cqp_info->cqp_cmd = IRDMA_OP_MANAGE_PUSH_PAGE; 1126 cqp_info->post_sq = 1; 1127 cqp_info->in.u.manage_push_page.info.push_idx = qp->push_idx; 1128 cqp_info->in.u.manage_push_page.info.qs_handle = qp->qs_handle; 1129 cqp_info->in.u.manage_push_page.info.free_page = 1; 1130 cqp_info->in.u.manage_push_page.info.push_page_type = 0; 1131 cqp_info->in.u.manage_push_page.cqp = &rf->cqp.sc_cqp; 1132 cqp_info->in.u.manage_push_page.scratch = (uintptr_t)cqp_request; 1133 status = irdma_handle_cqp_op(rf, cqp_request); 1134 if (!status) 1135 qp->push_idx = IRDMA_INVALID_PUSH_PAGE_INDEX; 1136 irdma_put_cqp_request(&rf->cqp, cqp_request); 1137} 1138 1139/** 1140 * irdma_free_qp_rsrc - free up memory resources for qp 1141 * @iwqp: qp ptr (user or kernel) 1142 */ 1143void irdma_free_qp_rsrc(struct irdma_qp *iwqp) 1144{ 1145 struct irdma_device *iwdev = iwqp->iwdev; 1146 struct irdma_pci_f *rf = iwdev->rf; 1147 u32 qp_num = iwqp->ibqp.qp_num; 1148 1149 irdma_ieq_cleanup_qp(iwdev->vsi.ieq, &iwqp->sc_qp); 1150 irdma_dealloc_push_page(rf, &iwqp->sc_qp); 1151 if (iwqp->sc_qp.vsi) { 1152 irdma_qp_rem_qos(&iwqp->sc_qp); 1153 iwqp->sc_qp.dev->ws_remove(iwqp->sc_qp.vsi, 1154 iwqp->sc_qp.user_pri); 1155 } 1156 1157 if (qp_num > 2) 1158 irdma_free_rsrc(rf, rf->allocated_qps, qp_num); 1159 dma_free_coherent(rf->sc_dev.hw->device, iwqp->q2_ctx_mem.size, 1160 iwqp->q2_ctx_mem.va, iwqp->q2_ctx_mem.pa); 1161 iwqp->q2_ctx_mem.va = NULL; 1162 dma_free_coherent(rf->sc_dev.hw->device, iwqp->kqp.dma_mem.size, 1163 iwqp->kqp.dma_mem.va, iwqp->kqp.dma_mem.pa); 1164 iwqp->kqp.dma_mem.va = NULL; 1165 kfree(iwqp->kqp.sq_wrid_mem); 1166 kfree(iwqp->kqp.rq_wrid_mem); 1167} 1168 1169/** 1170 * irdma_cq_wq_destroy - send cq destroy cqp 1171 * @rf: RDMA PCI function 1172 * @cq: hardware control cq 1173 */ 1174void irdma_cq_wq_destroy(struct irdma_pci_f *rf, struct irdma_sc_cq *cq) 1175{ 1176 struct irdma_cqp_request *cqp_request; 1177 struct cqp_cmds_info *cqp_info; 1178 1179 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true); 1180 if (!cqp_request) 1181 return; 1182 1183 cqp_info = &cqp_request->info; 1184 cqp_info->cqp_cmd = IRDMA_OP_CQ_DESTROY; 1185 cqp_info->post_sq = 1; 1186 cqp_info->in.u.cq_destroy.cq = cq; 1187 cqp_info->in.u.cq_destroy.scratch = (uintptr_t)cqp_request; 1188 1189 irdma_handle_cqp_op(rf, cqp_request); 1190 irdma_put_cqp_request(&rf->cqp, cqp_request); 1191} 1192 1193/** 1194 * irdma_hw_modify_qp_callback - handle state for modifyQPs that don't wait 1195 * @cqp_request: modify QP completion 1196 */ 1197static void irdma_hw_modify_qp_callback(struct irdma_cqp_request *cqp_request) 1198{ 1199 struct cqp_cmds_info *cqp_info; 1200 struct irdma_qp *iwqp; 1201 1202 cqp_info = &cqp_request->info; 1203 iwqp = cqp_info->in.u.qp_modify.qp->qp_uk.back_qp; 1204 atomic_dec(&iwqp->hw_mod_qp_pend); 1205 wake_up(&iwqp->mod_qp_waitq); 1206} 1207 1208/** 1209 * irdma_hw_modify_qp - setup cqp for modify qp 1210 * @iwdev: RDMA device 1211 * @iwqp: qp ptr (user or kernel) 1212 * @info: info for modify qp 1213 * @wait: flag to wait or not for modify qp completion 1214 */ 1215int irdma_hw_modify_qp(struct irdma_device *iwdev, struct irdma_qp *iwqp, 1216 struct irdma_modify_qp_info *info, bool wait) 1217{ 1218 int status; 1219 struct irdma_pci_f *rf = iwdev->rf; 1220 struct irdma_cqp_request *cqp_request; 1221 struct cqp_cmds_info *cqp_info; 1222 struct irdma_modify_qp_info *m_info; 1223 1224 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, wait); 1225 if (!cqp_request) 1226 return -ENOMEM; 1227 1228 if (!wait) { 1229 cqp_request->callback_fcn = irdma_hw_modify_qp_callback; 1230 atomic_inc(&iwqp->hw_mod_qp_pend); 1231 } 1232 cqp_info = &cqp_request->info; 1233 m_info = &cqp_info->in.u.qp_modify.info; 1234 memcpy(m_info, info, sizeof(*m_info)); 1235 cqp_info->cqp_cmd = IRDMA_OP_QP_MODIFY; 1236 cqp_info->post_sq = 1; 1237 cqp_info->in.u.qp_modify.qp = &iwqp->sc_qp; 1238 cqp_info->in.u.qp_modify.scratch = (uintptr_t)cqp_request; 1239 status = irdma_handle_cqp_op(rf, cqp_request); 1240 irdma_put_cqp_request(&rf->cqp, cqp_request); 1241 if (status) { 1242 if (rdma_protocol_roce(&iwdev->ibdev, 1)) 1243 return status; 1244 1245 switch (m_info->next_iwarp_state) { 1246 struct irdma_gen_ae_info ae_info; 1247 1248 case IRDMA_QP_STATE_RTS: 1249 case IRDMA_QP_STATE_IDLE: 1250 case IRDMA_QP_STATE_TERMINATE: 1251 case IRDMA_QP_STATE_CLOSING: 1252 if (info->curr_iwarp_state == IRDMA_QP_STATE_IDLE) 1253 irdma_send_reset(iwqp->cm_node); 1254 else 1255 iwqp->sc_qp.term_flags = IRDMA_TERM_DONE; 1256 if (!wait) { 1257 ae_info.ae_code = IRDMA_AE_BAD_CLOSE; 1258 ae_info.ae_src = 0; 1259 irdma_gen_ae(rf, &iwqp->sc_qp, &ae_info, false); 1260 } else { 1261 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, 1262 wait); 1263 if (!cqp_request) 1264 return -ENOMEM; 1265 1266 cqp_info = &cqp_request->info; 1267 m_info = &cqp_info->in.u.qp_modify.info; 1268 memcpy(m_info, info, sizeof(*m_info)); 1269 cqp_info->cqp_cmd = IRDMA_OP_QP_MODIFY; 1270 cqp_info->post_sq = 1; 1271 cqp_info->in.u.qp_modify.qp = &iwqp->sc_qp; 1272 cqp_info->in.u.qp_modify.scratch = (uintptr_t)cqp_request; 1273 m_info->next_iwarp_state = IRDMA_QP_STATE_ERROR; 1274 m_info->reset_tcp_conn = true; 1275 irdma_handle_cqp_op(rf, cqp_request); 1276 irdma_put_cqp_request(&rf->cqp, cqp_request); 1277 } 1278 break; 1279 case IRDMA_QP_STATE_ERROR: 1280 default: 1281 break; 1282 } 1283 } 1284 1285 return status; 1286} 1287 1288/** 1289 * irdma_cqp_cq_destroy_cmd - destroy the cqp cq 1290 * @dev: device pointer 1291 * @cq: pointer to cq 1292 */ 1293void irdma_cqp_cq_destroy_cmd(struct irdma_sc_dev *dev, struct irdma_sc_cq *cq) 1294{ 1295 struct irdma_pci_f *rf = dev_to_rf(dev); 1296 1297 irdma_cq_wq_destroy(rf, cq); 1298} 1299 1300/** 1301 * irdma_cqp_qp_destroy_cmd - destroy the cqp 1302 * @dev: device pointer 1303 * @qp: pointer to qp 1304 */ 1305int irdma_cqp_qp_destroy_cmd(struct irdma_sc_dev *dev, struct irdma_sc_qp *qp) 1306{ 1307 struct irdma_pci_f *rf = dev_to_rf(dev); 1308 struct irdma_cqp *iwcqp = &rf->cqp; 1309 struct irdma_cqp_request *cqp_request; 1310 struct cqp_cmds_info *cqp_info; 1311 int status; 1312 1313 cqp_request = irdma_alloc_and_get_cqp_request(iwcqp, true); 1314 if (!cqp_request) 1315 return -ENOMEM; 1316 1317 cqp_info = &cqp_request->info; 1318 memset(cqp_info, 0, sizeof(*cqp_info)); 1319 cqp_info->cqp_cmd = IRDMA_OP_QP_DESTROY; 1320 cqp_info->post_sq = 1; 1321 cqp_info->in.u.qp_destroy.qp = qp; 1322 cqp_info->in.u.qp_destroy.scratch = (uintptr_t)cqp_request; 1323 cqp_info->in.u.qp_destroy.remove_hash_idx = true; 1324 1325 status = irdma_handle_cqp_op(rf, cqp_request); 1326 irdma_put_cqp_request(&rf->cqp, cqp_request); 1327 1328 return status; 1329} 1330 1331/** 1332 * irdma_ieq_mpa_crc_ae - generate AE for crc error 1333 * @dev: hardware control device structure 1334 * @qp: hardware control qp 1335 */ 1336void irdma_ieq_mpa_crc_ae(struct irdma_sc_dev *dev, struct irdma_sc_qp *qp) 1337{ 1338 struct irdma_gen_ae_info info = {}; 1339 struct irdma_pci_f *rf = dev_to_rf(dev); 1340 1341 ibdev_dbg(&rf->iwdev->ibdev, "AEQ: Generate MPA CRC AE\n"); 1342 info.ae_code = IRDMA_AE_LLP_RECEIVED_MPA_CRC_ERROR; 1343 info.ae_src = IRDMA_AE_SOURCE_RQ; 1344 irdma_gen_ae(rf, qp, &info, false); 1345} 1346 1347/** 1348 * irdma_init_hash_desc - initialize hash for crc calculation 1349 * @desc: cryption type 1350 */ 1351int irdma_init_hash_desc(struct shash_desc **desc) 1352{ 1353 struct crypto_shash *tfm; 1354 struct shash_desc *tdesc; 1355 1356 tfm = crypto_alloc_shash("crc32c", 0, 0); 1357 if (IS_ERR(tfm)) 1358 return -EINVAL; 1359 1360 tdesc = kzalloc(sizeof(*tdesc) + crypto_shash_descsize(tfm), 1361 GFP_KERNEL); 1362 if (!tdesc) { 1363 crypto_free_shash(tfm); 1364 return -EINVAL; 1365 } 1366 1367 tdesc->tfm = tfm; 1368 *desc = tdesc; 1369 1370 return 0; 1371} 1372 1373/** 1374 * irdma_free_hash_desc - free hash desc 1375 * @desc: to be freed 1376 */ 1377void irdma_free_hash_desc(struct shash_desc *desc) 1378{ 1379 if (desc) { 1380 crypto_free_shash(desc->tfm); 1381 kfree(desc); 1382 } 1383} 1384 1385/** 1386 * irdma_ieq_check_mpacrc - check if mpa crc is OK 1387 * @desc: desc for hash 1388 * @addr: address of buffer for crc 1389 * @len: length of buffer 1390 * @val: value to be compared 1391 */ 1392int irdma_ieq_check_mpacrc(struct shash_desc *desc, void *addr, u32 len, 1393 u32 val) 1394{ 1395 u32 crc = 0; 1396 int ret; 1397 int ret_code = 0; 1398 1399 crypto_shash_init(desc); 1400 ret = crypto_shash_update(desc, addr, len); 1401 if (!ret) 1402 crypto_shash_final(desc, (u8 *)&crc); 1403 if (crc != val) 1404 ret_code = -EINVAL; 1405 1406 return ret_code; 1407} 1408 1409/** 1410 * irdma_ieq_get_qp - get qp based on quad in puda buffer 1411 * @dev: hardware control device structure 1412 * @buf: receive puda buffer on exception q 1413 */ 1414struct irdma_sc_qp *irdma_ieq_get_qp(struct irdma_sc_dev *dev, 1415 struct irdma_puda_buf *buf) 1416{ 1417 struct irdma_qp *iwqp; 1418 struct irdma_cm_node *cm_node; 1419 struct irdma_device *iwdev = buf->vsi->back_vsi; 1420 u32 loc_addr[4] = {}; 1421 u32 rem_addr[4] = {}; 1422 u16 loc_port, rem_port; 1423 struct ipv6hdr *ip6h; 1424 struct iphdr *iph = (struct iphdr *)buf->iph; 1425 struct tcphdr *tcph = (struct tcphdr *)buf->tcph; 1426 1427 if (iph->version == 4) { 1428 loc_addr[0] = ntohl(iph->daddr); 1429 rem_addr[0] = ntohl(iph->saddr); 1430 } else { 1431 ip6h = (struct ipv6hdr *)buf->iph; 1432 irdma_copy_ip_ntohl(loc_addr, ip6h->daddr.in6_u.u6_addr32); 1433 irdma_copy_ip_ntohl(rem_addr, ip6h->saddr.in6_u.u6_addr32); 1434 } 1435 loc_port = ntohs(tcph->dest); 1436 rem_port = ntohs(tcph->source); 1437 cm_node = irdma_find_node(&iwdev->cm_core, rem_port, rem_addr, loc_port, 1438 loc_addr, buf->vlan_valid ? buf->vlan_id : 0xFFFF); 1439 if (!cm_node) 1440 return NULL; 1441 1442 iwqp = cm_node->iwqp; 1443 irdma_rem_ref_cm_node(cm_node); 1444 1445 return &iwqp->sc_qp; 1446} 1447 1448/** 1449 * irdma_send_ieq_ack - ACKs for duplicate or OOO partials FPDUs 1450 * @qp: qp ptr 1451 */ 1452void irdma_send_ieq_ack(struct irdma_sc_qp *qp) 1453{ 1454 struct irdma_cm_node *cm_node = ((struct irdma_qp *)qp->qp_uk.back_qp)->cm_node; 1455 struct irdma_puda_buf *buf = qp->pfpdu.lastrcv_buf; 1456 struct tcphdr *tcph = (struct tcphdr *)buf->tcph; 1457 1458 cm_node->tcp_cntxt.rcv_nxt = qp->pfpdu.nextseqnum; 1459 cm_node->tcp_cntxt.loc_seq_num = ntohl(tcph->ack_seq); 1460 1461 irdma_send_ack(cm_node); 1462} 1463 1464/** 1465 * irdma_puda_ieq_get_ah_info - get AH info from IEQ buffer 1466 * @qp: qp pointer 1467 * @ah_info: AH info pointer 1468 */ 1469void irdma_puda_ieq_get_ah_info(struct irdma_sc_qp *qp, 1470 struct irdma_ah_info *ah_info) 1471{ 1472 struct irdma_puda_buf *buf = qp->pfpdu.ah_buf; 1473 struct iphdr *iph; 1474 struct ipv6hdr *ip6h; 1475 1476 memset(ah_info, 0, sizeof(*ah_info)); 1477 ah_info->do_lpbk = true; 1478 ah_info->vlan_tag = buf->vlan_id; 1479 ah_info->insert_vlan_tag = buf->vlan_valid; 1480 ah_info->ipv4_valid = buf->ipv4; 1481 ah_info->vsi = qp->vsi; 1482 1483 if (buf->smac_valid) 1484 ether_addr_copy(ah_info->mac_addr, buf->smac); 1485 1486 if (buf->ipv4) { 1487 ah_info->ipv4_valid = true; 1488 iph = (struct iphdr *)buf->iph; 1489 ah_info->hop_ttl = iph->ttl; 1490 ah_info->tc_tos = iph->tos; 1491 ah_info->dest_ip_addr[0] = ntohl(iph->daddr); 1492 ah_info->src_ip_addr[0] = ntohl(iph->saddr); 1493 } else { 1494 ip6h = (struct ipv6hdr *)buf->iph; 1495 ah_info->hop_ttl = ip6h->hop_limit; 1496 ah_info->tc_tos = ip6h->priority; 1497 irdma_copy_ip_ntohl(ah_info->dest_ip_addr, 1498 ip6h->daddr.in6_u.u6_addr32); 1499 irdma_copy_ip_ntohl(ah_info->src_ip_addr, 1500 ip6h->saddr.in6_u.u6_addr32); 1501 } 1502 1503 ah_info->dst_arpindex = irdma_arp_table(dev_to_rf(qp->dev), 1504 ah_info->dest_ip_addr, 1505 ah_info->ipv4_valid, 1506 NULL, IRDMA_ARP_RESOLVE); 1507} 1508 1509/** 1510 * irdma_gen1_ieq_update_tcpip_info - update tcpip in the buffer 1511 * @buf: puda to update 1512 * @len: length of buffer 1513 * @seqnum: seq number for tcp 1514 */ 1515static void irdma_gen1_ieq_update_tcpip_info(struct irdma_puda_buf *buf, 1516 u16 len, u32 seqnum) 1517{ 1518 struct tcphdr *tcph; 1519 struct iphdr *iph; 1520 u16 iphlen; 1521 u16 pktsize; 1522 u8 *addr = buf->mem.va; 1523 1524 iphlen = (buf->ipv4) ? 20 : 40; 1525 iph = (struct iphdr *)(addr + buf->maclen); 1526 tcph = (struct tcphdr *)(addr + buf->maclen + iphlen); 1527 pktsize = len + buf->tcphlen + iphlen; 1528 iph->tot_len = htons(pktsize); 1529 tcph->seq = htonl(seqnum); 1530} 1531 1532/** 1533 * irdma_ieq_update_tcpip_info - update tcpip in the buffer 1534 * @buf: puda to update 1535 * @len: length of buffer 1536 * @seqnum: seq number for tcp 1537 */ 1538void irdma_ieq_update_tcpip_info(struct irdma_puda_buf *buf, u16 len, 1539 u32 seqnum) 1540{ 1541 struct tcphdr *tcph; 1542 u8 *addr; 1543 1544 if (buf->vsi->dev->hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_1) 1545 return irdma_gen1_ieq_update_tcpip_info(buf, len, seqnum); 1546 1547 addr = buf->mem.va; 1548 tcph = (struct tcphdr *)addr; 1549 tcph->seq = htonl(seqnum); 1550} 1551 1552/** 1553 * irdma_gen1_puda_get_tcpip_info - get tcpip info from puda 1554 * buffer 1555 * @info: to get information 1556 * @buf: puda buffer 1557 */ 1558static int irdma_gen1_puda_get_tcpip_info(struct irdma_puda_cmpl_info *info, 1559 struct irdma_puda_buf *buf) 1560{ 1561 struct iphdr *iph; 1562 struct ipv6hdr *ip6h; 1563 struct tcphdr *tcph; 1564 u16 iphlen; 1565 u16 pkt_len; 1566 u8 *mem = buf->mem.va; 1567 struct ethhdr *ethh = buf->mem.va; 1568 1569 if (ethh->h_proto == htons(0x8100)) { 1570 info->vlan_valid = true; 1571 buf->vlan_id = ntohs(((struct vlan_ethhdr *)ethh)->h_vlan_TCI) & 1572 VLAN_VID_MASK; 1573 } 1574 1575 buf->maclen = (info->vlan_valid) ? 18 : 14; 1576 iphlen = (info->l3proto) ? 40 : 20; 1577 buf->ipv4 = (info->l3proto) ? false : true; 1578 buf->iph = mem + buf->maclen; 1579 iph = (struct iphdr *)buf->iph; 1580 buf->tcph = buf->iph + iphlen; 1581 tcph = (struct tcphdr *)buf->tcph; 1582 1583 if (buf->ipv4) { 1584 pkt_len = ntohs(iph->tot_len); 1585 } else { 1586 ip6h = (struct ipv6hdr *)buf->iph; 1587 pkt_len = ntohs(ip6h->payload_len) + iphlen; 1588 } 1589 1590 buf->totallen = pkt_len + buf->maclen; 1591 1592 if (info->payload_len < buf->totallen) { 1593 ibdev_dbg(to_ibdev(buf->vsi->dev), 1594 "ERR: payload_len = 0x%x totallen expected0x%x\n", 1595 info->payload_len, buf->totallen); 1596 return -EINVAL; 1597 } 1598 1599 buf->tcphlen = tcph->doff << 2; 1600 buf->datalen = pkt_len - iphlen - buf->tcphlen; 1601 buf->data = buf->datalen ? buf->tcph + buf->tcphlen : NULL; 1602 buf->hdrlen = buf->maclen + iphlen + buf->tcphlen; 1603 buf->seqnum = ntohl(tcph->seq); 1604 1605 return 0; 1606} 1607 1608/** 1609 * irdma_puda_get_tcpip_info - get tcpip info from puda buffer 1610 * @info: to get information 1611 * @buf: puda buffer 1612 */ 1613int irdma_puda_get_tcpip_info(struct irdma_puda_cmpl_info *info, 1614 struct irdma_puda_buf *buf) 1615{ 1616 struct tcphdr *tcph; 1617 u32 pkt_len; 1618 u8 *mem; 1619 1620 if (buf->vsi->dev->hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_1) 1621 return irdma_gen1_puda_get_tcpip_info(info, buf); 1622 1623 mem = buf->mem.va; 1624 buf->vlan_valid = info->vlan_valid; 1625 if (info->vlan_valid) 1626 buf->vlan_id = info->vlan; 1627 1628 buf->ipv4 = info->ipv4; 1629 if (buf->ipv4) 1630 buf->iph = mem + IRDMA_IPV4_PAD; 1631 else 1632 buf->iph = mem; 1633 1634 buf->tcph = mem + IRDMA_TCP_OFFSET; 1635 tcph = (struct tcphdr *)buf->tcph; 1636 pkt_len = info->payload_len; 1637 buf->totallen = pkt_len; 1638 buf->tcphlen = tcph->doff << 2; 1639 buf->datalen = pkt_len - IRDMA_TCP_OFFSET - buf->tcphlen; 1640 buf->data = buf->datalen ? buf->tcph + buf->tcphlen : NULL; 1641 buf->hdrlen = IRDMA_TCP_OFFSET + buf->tcphlen; 1642 buf->seqnum = ntohl(tcph->seq); 1643 1644 if (info->smac_valid) { 1645 ether_addr_copy(buf->smac, info->smac); 1646 buf->smac_valid = true; 1647 } 1648 1649 return 0; 1650} 1651 1652/** 1653 * irdma_hw_stats_timeout - Stats timer-handler which updates all HW stats 1654 * @t: timer_list pointer 1655 */ 1656static void irdma_hw_stats_timeout(struct timer_list *t) 1657{ 1658 struct irdma_vsi_pestat *pf_devstat = 1659 from_timer(pf_devstat, t, stats_timer); 1660 struct irdma_sc_vsi *sc_vsi = pf_devstat->vsi; 1661 1662 if (sc_vsi->dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2) 1663 irdma_cqp_gather_stats_cmd(sc_vsi->dev, sc_vsi->pestat, false); 1664 else 1665 irdma_cqp_gather_stats_gen1(sc_vsi->dev, sc_vsi->pestat); 1666 1667 mod_timer(&pf_devstat->stats_timer, 1668 jiffies + msecs_to_jiffies(STATS_TIMER_DELAY)); 1669} 1670 1671/** 1672 * irdma_hw_stats_start_timer - Start periodic stats timer 1673 * @vsi: vsi structure pointer 1674 */ 1675void irdma_hw_stats_start_timer(struct irdma_sc_vsi *vsi) 1676{ 1677 struct irdma_vsi_pestat *devstat = vsi->pestat; 1678 1679 timer_setup(&devstat->stats_timer, irdma_hw_stats_timeout, 0); 1680 mod_timer(&devstat->stats_timer, 1681 jiffies + msecs_to_jiffies(STATS_TIMER_DELAY)); 1682} 1683 1684/** 1685 * irdma_hw_stats_stop_timer - Delete periodic stats timer 1686 * @vsi: pointer to vsi structure 1687 */ 1688void irdma_hw_stats_stop_timer(struct irdma_sc_vsi *vsi) 1689{ 1690 struct irdma_vsi_pestat *devstat = vsi->pestat; 1691 1692 del_timer_sync(&devstat->stats_timer); 1693} 1694 1695/** 1696 * irdma_process_stats - Checking for wrap and update stats 1697 * @pestat: stats structure pointer 1698 */ 1699static inline void irdma_process_stats(struct irdma_vsi_pestat *pestat) 1700{ 1701 sc_vsi_update_stats(pestat->vsi); 1702} 1703 1704/** 1705 * irdma_cqp_gather_stats_gen1 - Gather stats 1706 * @dev: pointer to device structure 1707 * @pestat: statistics structure 1708 */ 1709void irdma_cqp_gather_stats_gen1(struct irdma_sc_dev *dev, 1710 struct irdma_vsi_pestat *pestat) 1711{ 1712 struct irdma_gather_stats *gather_stats = 1713 pestat->gather_info.gather_stats_va; 1714 const struct irdma_hw_stat_map *map = dev->hw_stats_map; 1715 u16 max_stats_idx = dev->hw_attrs.max_stat_idx; 1716 u32 stats_inst_offset_32; 1717 u32 stats_inst_offset_64; 1718 u64 new_val; 1719 u16 i; 1720 1721 stats_inst_offset_32 = (pestat->gather_info.use_stats_inst) ? 1722 pestat->gather_info.stats_inst_index : 1723 pestat->hw->hmc.hmc_fn_id; 1724 stats_inst_offset_32 *= 4; 1725 stats_inst_offset_64 = stats_inst_offset_32 * 2; 1726 1727 for (i = 0; i < max_stats_idx; i++) { 1728 if (map[i].bitmask <= IRDMA_MAX_STATS_32) 1729 new_val = rd32(dev->hw, 1730 dev->hw_stats_regs[i] + stats_inst_offset_32); 1731 else 1732 new_val = rd64(dev->hw, 1733 dev->hw_stats_regs[i] + stats_inst_offset_64); 1734 gather_stats->val[map[i].byteoff / sizeof(u64)] = new_val; 1735 } 1736 1737 irdma_process_stats(pestat); 1738} 1739 1740/** 1741 * irdma_process_cqp_stats - Checking for wrap and update stats 1742 * @cqp_request: cqp_request structure pointer 1743 */ 1744static void irdma_process_cqp_stats(struct irdma_cqp_request *cqp_request) 1745{ 1746 struct irdma_vsi_pestat *pestat = cqp_request->param; 1747 1748 irdma_process_stats(pestat); 1749} 1750 1751/** 1752 * irdma_cqp_gather_stats_cmd - Gather stats 1753 * @dev: pointer to device structure 1754 * @pestat: pointer to stats info 1755 * @wait: flag to wait or not wait for stats 1756 */ 1757int irdma_cqp_gather_stats_cmd(struct irdma_sc_dev *dev, 1758 struct irdma_vsi_pestat *pestat, bool wait) 1759 1760{ 1761 struct irdma_pci_f *rf = dev_to_rf(dev); 1762 struct irdma_cqp *iwcqp = &rf->cqp; 1763 struct irdma_cqp_request *cqp_request; 1764 struct cqp_cmds_info *cqp_info; 1765 int status; 1766 1767 cqp_request = irdma_alloc_and_get_cqp_request(iwcqp, wait); 1768 if (!cqp_request) 1769 return -ENOMEM; 1770 1771 cqp_info = &cqp_request->info; 1772 memset(cqp_info, 0, sizeof(*cqp_info)); 1773 cqp_info->cqp_cmd = IRDMA_OP_STATS_GATHER; 1774 cqp_info->post_sq = 1; 1775 cqp_info->in.u.stats_gather.info = pestat->gather_info; 1776 cqp_info->in.u.stats_gather.scratch = (uintptr_t)cqp_request; 1777 cqp_info->in.u.stats_gather.cqp = &rf->cqp.sc_cqp; 1778 cqp_request->param = pestat; 1779 if (!wait) 1780 cqp_request->callback_fcn = irdma_process_cqp_stats; 1781 status = irdma_handle_cqp_op(rf, cqp_request); 1782 if (wait) 1783 irdma_process_stats(pestat); 1784 irdma_put_cqp_request(&rf->cqp, cqp_request); 1785 1786 return status; 1787} 1788 1789/** 1790 * irdma_cqp_stats_inst_cmd - Allocate/free stats instance 1791 * @vsi: pointer to vsi structure 1792 * @cmd: command to allocate or free 1793 * @stats_info: pointer to allocate stats info 1794 */ 1795int irdma_cqp_stats_inst_cmd(struct irdma_sc_vsi *vsi, u8 cmd, 1796 struct irdma_stats_inst_info *stats_info) 1797{ 1798 struct irdma_pci_f *rf = dev_to_rf(vsi->dev); 1799 struct irdma_cqp *iwcqp = &rf->cqp; 1800 struct irdma_cqp_request *cqp_request; 1801 struct cqp_cmds_info *cqp_info; 1802 int status; 1803 bool wait = false; 1804 1805 if (cmd == IRDMA_OP_STATS_ALLOCATE) 1806 wait = true; 1807 cqp_request = irdma_alloc_and_get_cqp_request(iwcqp, wait); 1808 if (!cqp_request) 1809 return -ENOMEM; 1810 1811 cqp_info = &cqp_request->info; 1812 memset(cqp_info, 0, sizeof(*cqp_info)); 1813 cqp_info->cqp_cmd = cmd; 1814 cqp_info->post_sq = 1; 1815 cqp_info->in.u.stats_manage.info = *stats_info; 1816 cqp_info->in.u.stats_manage.scratch = (uintptr_t)cqp_request; 1817 cqp_info->in.u.stats_manage.cqp = &rf->cqp.sc_cqp; 1818 status = irdma_handle_cqp_op(rf, cqp_request); 1819 if (wait) 1820 stats_info->stats_idx = cqp_request->compl_info.op_ret_val; 1821 irdma_put_cqp_request(iwcqp, cqp_request); 1822 1823 return status; 1824} 1825 1826/** 1827 * irdma_cqp_ceq_cmd - Create/Destroy CEQ's after CEQ 0 1828 * @dev: pointer to device info 1829 * @sc_ceq: pointer to ceq structure 1830 * @op: Create or Destroy 1831 */ 1832int irdma_cqp_ceq_cmd(struct irdma_sc_dev *dev, struct irdma_sc_ceq *sc_ceq, 1833 u8 op) 1834{ 1835 struct irdma_cqp_request *cqp_request; 1836 struct cqp_cmds_info *cqp_info; 1837 struct irdma_pci_f *rf = dev_to_rf(dev); 1838 int status; 1839 1840 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true); 1841 if (!cqp_request) 1842 return -ENOMEM; 1843 1844 cqp_info = &cqp_request->info; 1845 cqp_info->post_sq = 1; 1846 cqp_info->cqp_cmd = op; 1847 cqp_info->in.u.ceq_create.ceq = sc_ceq; 1848 cqp_info->in.u.ceq_create.scratch = (uintptr_t)cqp_request; 1849 1850 status = irdma_handle_cqp_op(rf, cqp_request); 1851 irdma_put_cqp_request(&rf->cqp, cqp_request); 1852 1853 return status; 1854} 1855 1856/** 1857 * irdma_cqp_aeq_cmd - Create/Destroy AEQ 1858 * @dev: pointer to device info 1859 * @sc_aeq: pointer to aeq structure 1860 * @op: Create or Destroy 1861 */ 1862int irdma_cqp_aeq_cmd(struct irdma_sc_dev *dev, struct irdma_sc_aeq *sc_aeq, 1863 u8 op) 1864{ 1865 struct irdma_cqp_request *cqp_request; 1866 struct cqp_cmds_info *cqp_info; 1867 struct irdma_pci_f *rf = dev_to_rf(dev); 1868 int status; 1869 1870 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true); 1871 if (!cqp_request) 1872 return -ENOMEM; 1873 1874 cqp_info = &cqp_request->info; 1875 cqp_info->post_sq = 1; 1876 cqp_info->cqp_cmd = op; 1877 cqp_info->in.u.aeq_create.aeq = sc_aeq; 1878 cqp_info->in.u.aeq_create.scratch = (uintptr_t)cqp_request; 1879 1880 status = irdma_handle_cqp_op(rf, cqp_request); 1881 irdma_put_cqp_request(&rf->cqp, cqp_request); 1882 1883 return status; 1884} 1885 1886/** 1887 * irdma_cqp_ws_node_cmd - Add/modify/delete ws node 1888 * @dev: pointer to device structure 1889 * @cmd: Add, modify or delete 1890 * @node_info: pointer to ws node info 1891 */ 1892int irdma_cqp_ws_node_cmd(struct irdma_sc_dev *dev, u8 cmd, 1893 struct irdma_ws_node_info *node_info) 1894{ 1895 struct irdma_pci_f *rf = dev_to_rf(dev); 1896 struct irdma_cqp *iwcqp = &rf->cqp; 1897 struct irdma_sc_cqp *cqp = &iwcqp->sc_cqp; 1898 struct irdma_cqp_request *cqp_request; 1899 struct cqp_cmds_info *cqp_info; 1900 int status; 1901 bool poll; 1902 1903 if (!rf->sc_dev.ceq_valid) 1904 poll = true; 1905 else 1906 poll = false; 1907 1908 cqp_request = irdma_alloc_and_get_cqp_request(iwcqp, !poll); 1909 if (!cqp_request) 1910 return -ENOMEM; 1911 1912 cqp_info = &cqp_request->info; 1913 memset(cqp_info, 0, sizeof(*cqp_info)); 1914 cqp_info->cqp_cmd = cmd; 1915 cqp_info->post_sq = 1; 1916 cqp_info->in.u.ws_node.info = *node_info; 1917 cqp_info->in.u.ws_node.cqp = cqp; 1918 cqp_info->in.u.ws_node.scratch = (uintptr_t)cqp_request; 1919 status = irdma_handle_cqp_op(rf, cqp_request); 1920 if (status) 1921 goto exit; 1922 1923 if (poll) { 1924 struct irdma_ccq_cqe_info compl_info; 1925 1926 status = irdma_sc_poll_for_cqp_op_done(cqp, IRDMA_CQP_OP_WORK_SCHED_NODE, 1927 &compl_info); 1928 node_info->qs_handle = compl_info.op_ret_val; 1929 ibdev_dbg(&rf->iwdev->ibdev, "DCB: opcode=%d, compl_info.retval=%d\n", 1930 compl_info.op_code, compl_info.op_ret_val); 1931 } else { 1932 node_info->qs_handle = cqp_request->compl_info.op_ret_val; 1933 } 1934 1935exit: 1936 irdma_put_cqp_request(&rf->cqp, cqp_request); 1937 1938 return status; 1939} 1940 1941/** 1942 * irdma_ah_cqp_op - perform an AH cqp operation 1943 * @rf: RDMA PCI function 1944 * @sc_ah: address handle 1945 * @cmd: AH operation 1946 * @wait: wait if true 1947 * @callback_fcn: Callback function on CQP op completion 1948 * @cb_param: parameter for callback function 1949 * 1950 * returns errno 1951 */ 1952int irdma_ah_cqp_op(struct irdma_pci_f *rf, struct irdma_sc_ah *sc_ah, u8 cmd, 1953 bool wait, 1954 void (*callback_fcn)(struct irdma_cqp_request *), 1955 void *cb_param) 1956{ 1957 struct irdma_cqp_request *cqp_request; 1958 struct cqp_cmds_info *cqp_info; 1959 int status; 1960 1961 if (cmd != IRDMA_OP_AH_CREATE && cmd != IRDMA_OP_AH_DESTROY) 1962 return -EINVAL; 1963 1964 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, wait); 1965 if (!cqp_request) 1966 return -ENOMEM; 1967 1968 cqp_info = &cqp_request->info; 1969 cqp_info->cqp_cmd = cmd; 1970 cqp_info->post_sq = 1; 1971 if (cmd == IRDMA_OP_AH_CREATE) { 1972 cqp_info->in.u.ah_create.info = sc_ah->ah_info; 1973 cqp_info->in.u.ah_create.scratch = (uintptr_t)cqp_request; 1974 cqp_info->in.u.ah_create.cqp = &rf->cqp.sc_cqp; 1975 } else if (cmd == IRDMA_OP_AH_DESTROY) { 1976 cqp_info->in.u.ah_destroy.info = sc_ah->ah_info; 1977 cqp_info->in.u.ah_destroy.scratch = (uintptr_t)cqp_request; 1978 cqp_info->in.u.ah_destroy.cqp = &rf->cqp.sc_cqp; 1979 } 1980 1981 if (!wait) { 1982 cqp_request->callback_fcn = callback_fcn; 1983 cqp_request->param = cb_param; 1984 } 1985 status = irdma_handle_cqp_op(rf, cqp_request); 1986 irdma_put_cqp_request(&rf->cqp, cqp_request); 1987 1988 if (status) 1989 return -ENOMEM; 1990 1991 if (wait) 1992 sc_ah->ah_info.ah_valid = (cmd == IRDMA_OP_AH_CREATE); 1993 1994 return 0; 1995} 1996 1997/** 1998 * irdma_ieq_ah_cb - callback after creation of AH for IEQ 1999 * @cqp_request: pointer to cqp_request of create AH 2000 */ 2001static void irdma_ieq_ah_cb(struct irdma_cqp_request *cqp_request) 2002{ 2003 struct irdma_sc_qp *qp = cqp_request->param; 2004 struct irdma_sc_ah *sc_ah = qp->pfpdu.ah; 2005 unsigned long flags; 2006 2007 spin_lock_irqsave(&qp->pfpdu.lock, flags); 2008 if (!cqp_request->compl_info.op_ret_val) { 2009 sc_ah->ah_info.ah_valid = true; 2010 irdma_ieq_process_fpdus(qp, qp->vsi->ieq); 2011 } else { 2012 sc_ah->ah_info.ah_valid = false; 2013 irdma_ieq_cleanup_qp(qp->vsi->ieq, qp); 2014 } 2015 spin_unlock_irqrestore(&qp->pfpdu.lock, flags); 2016} 2017 2018/** 2019 * irdma_ilq_ah_cb - callback after creation of AH for ILQ 2020 * @cqp_request: pointer to cqp_request of create AH 2021 */ 2022static void irdma_ilq_ah_cb(struct irdma_cqp_request *cqp_request) 2023{ 2024 struct irdma_cm_node *cm_node = cqp_request->param; 2025 struct irdma_sc_ah *sc_ah = cm_node->ah; 2026 2027 sc_ah->ah_info.ah_valid = !cqp_request->compl_info.op_ret_val; 2028 irdma_add_conn_est_qh(cm_node); 2029} 2030 2031/** 2032 * irdma_puda_create_ah - create AH for ILQ/IEQ qp's 2033 * @dev: device pointer 2034 * @ah_info: Address handle info 2035 * @wait: When true will wait for operation to complete 2036 * @type: ILQ/IEQ 2037 * @cb_param: Callback param when not waiting 2038 * @ah_ret: Returned pointer to address handle if created 2039 * 2040 */ 2041int irdma_puda_create_ah(struct irdma_sc_dev *dev, 2042 struct irdma_ah_info *ah_info, bool wait, 2043 enum puda_rsrc_type type, void *cb_param, 2044 struct irdma_sc_ah **ah_ret) 2045{ 2046 struct irdma_sc_ah *ah; 2047 struct irdma_pci_f *rf = dev_to_rf(dev); 2048 int err; 2049 2050 ah = kzalloc(sizeof(*ah), GFP_ATOMIC); 2051 *ah_ret = ah; 2052 if (!ah) 2053 return -ENOMEM; 2054 2055 err = irdma_alloc_rsrc(rf, rf->allocated_ahs, rf->max_ah, 2056 &ah_info->ah_idx, &rf->next_ah); 2057 if (err) 2058 goto err_free; 2059 2060 ah->dev = dev; 2061 ah->ah_info = *ah_info; 2062 2063 if (type == IRDMA_PUDA_RSRC_TYPE_ILQ) 2064 err = irdma_ah_cqp_op(rf, ah, IRDMA_OP_AH_CREATE, wait, 2065 irdma_ilq_ah_cb, cb_param); 2066 else 2067 err = irdma_ah_cqp_op(rf, ah, IRDMA_OP_AH_CREATE, wait, 2068 irdma_ieq_ah_cb, cb_param); 2069 2070 if (err) 2071 goto error; 2072 return 0; 2073 2074error: 2075 irdma_free_rsrc(rf, rf->allocated_ahs, ah->ah_info.ah_idx); 2076err_free: 2077 kfree(ah); 2078 *ah_ret = NULL; 2079 return -ENOMEM; 2080} 2081 2082/** 2083 * irdma_puda_free_ah - free a puda address handle 2084 * @dev: device pointer 2085 * @ah: The address handle to free 2086 */ 2087void irdma_puda_free_ah(struct irdma_sc_dev *dev, struct irdma_sc_ah *ah) 2088{ 2089 struct irdma_pci_f *rf = dev_to_rf(dev); 2090 2091 if (!ah) 2092 return; 2093 2094 if (ah->ah_info.ah_valid) { 2095 irdma_ah_cqp_op(rf, ah, IRDMA_OP_AH_DESTROY, false, NULL, NULL); 2096 irdma_free_rsrc(rf, rf->allocated_ahs, ah->ah_info.ah_idx); 2097 } 2098 2099 kfree(ah); 2100} 2101 2102/** 2103 * irdma_gsi_ud_qp_ah_cb - callback after creation of AH for GSI/ID QP 2104 * @cqp_request: pointer to cqp_request of create AH 2105 */ 2106void irdma_gsi_ud_qp_ah_cb(struct irdma_cqp_request *cqp_request) 2107{ 2108 struct irdma_sc_ah *sc_ah = cqp_request->param; 2109 2110 if (!cqp_request->compl_info.op_ret_val) 2111 sc_ah->ah_info.ah_valid = true; 2112 else 2113 sc_ah->ah_info.ah_valid = false; 2114} 2115 2116/** 2117 * irdma_prm_add_pble_mem - add moemory to pble resources 2118 * @pprm: pble resource manager 2119 * @pchunk: chunk of memory to add 2120 */ 2121int irdma_prm_add_pble_mem(struct irdma_pble_prm *pprm, 2122 struct irdma_chunk *pchunk) 2123{ 2124 u64 sizeofbitmap; 2125 2126 if (pchunk->size & 0xfff) 2127 return -EINVAL; 2128 2129 sizeofbitmap = (u64)pchunk->size >> pprm->pble_shift; 2130 2131 pchunk->bitmapbuf = bitmap_zalloc(sizeofbitmap, GFP_KERNEL); 2132 if (!pchunk->bitmapbuf) 2133 return -ENOMEM; 2134 2135 pchunk->sizeofbitmap = sizeofbitmap; 2136 /* each pble is 8 bytes hence shift by 3 */ 2137 pprm->total_pble_alloc += pchunk->size >> 3; 2138 pprm->free_pble_cnt += pchunk->size >> 3; 2139 2140 return 0; 2141} 2142 2143/** 2144 * irdma_prm_get_pbles - get pble's from prm 2145 * @pprm: pble resource manager 2146 * @chunkinfo: nformation about chunk where pble's were acquired 2147 * @mem_size: size of pble memory needed 2148 * @vaddr: returns virtual address of pble memory 2149 * @fpm_addr: returns fpm address of pble memory 2150 */ 2151int irdma_prm_get_pbles(struct irdma_pble_prm *pprm, 2152 struct irdma_pble_chunkinfo *chunkinfo, u64 mem_size, 2153 u64 **vaddr, u64 *fpm_addr) 2154{ 2155 u64 bits_needed; 2156 u64 bit_idx = PBLE_INVALID_IDX; 2157 struct irdma_chunk *pchunk = NULL; 2158 struct list_head *chunk_entry = pprm->clist.next; 2159 u32 offset; 2160 unsigned long flags; 2161 *vaddr = NULL; 2162 *fpm_addr = 0; 2163 2164 bits_needed = DIV_ROUND_UP_ULL(mem_size, BIT_ULL(pprm->pble_shift)); 2165 2166 spin_lock_irqsave(&pprm->prm_lock, flags); 2167 while (chunk_entry != &pprm->clist) { 2168 pchunk = (struct irdma_chunk *)chunk_entry; 2169 bit_idx = bitmap_find_next_zero_area(pchunk->bitmapbuf, 2170 pchunk->sizeofbitmap, 0, 2171 bits_needed, 0); 2172 if (bit_idx < pchunk->sizeofbitmap) 2173 break; 2174 2175 /* list.next used macro */ 2176 chunk_entry = pchunk->list.next; 2177 } 2178 2179 if (!pchunk || bit_idx >= pchunk->sizeofbitmap) { 2180 spin_unlock_irqrestore(&pprm->prm_lock, flags); 2181 return -ENOMEM; 2182 } 2183 2184 bitmap_set(pchunk->bitmapbuf, bit_idx, bits_needed); 2185 offset = bit_idx << pprm->pble_shift; 2186 *vaddr = pchunk->vaddr + offset; 2187 *fpm_addr = pchunk->fpm_addr + offset; 2188 2189 chunkinfo->pchunk = pchunk; 2190 chunkinfo->bit_idx = bit_idx; 2191 chunkinfo->bits_used = bits_needed; 2192 /* 3 is sizeof pble divide */ 2193 pprm->free_pble_cnt -= chunkinfo->bits_used << (pprm->pble_shift - 3); 2194 spin_unlock_irqrestore(&pprm->prm_lock, flags); 2195 2196 return 0; 2197} 2198 2199/** 2200 * irdma_prm_return_pbles - return pbles back to prm 2201 * @pprm: pble resource manager 2202 * @chunkinfo: chunk where pble's were acquired and to be freed 2203 */ 2204void irdma_prm_return_pbles(struct irdma_pble_prm *pprm, 2205 struct irdma_pble_chunkinfo *chunkinfo) 2206{ 2207 unsigned long flags; 2208 2209 spin_lock_irqsave(&pprm->prm_lock, flags); 2210 pprm->free_pble_cnt += chunkinfo->bits_used << (pprm->pble_shift - 3); 2211 bitmap_clear(chunkinfo->pchunk->bitmapbuf, chunkinfo->bit_idx, 2212 chunkinfo->bits_used); 2213 spin_unlock_irqrestore(&pprm->prm_lock, flags); 2214} 2215 2216int irdma_map_vm_page_list(struct irdma_hw *hw, void *va, dma_addr_t *pg_dma, 2217 u32 pg_cnt) 2218{ 2219 struct page *vm_page; 2220 int i; 2221 u8 *addr; 2222 2223 addr = (u8 *)(uintptr_t)va; 2224 for (i = 0; i < pg_cnt; i++) { 2225 vm_page = vmalloc_to_page(addr); 2226 if (!vm_page) 2227 goto err; 2228 2229 pg_dma[i] = dma_map_page(hw->device, vm_page, 0, PAGE_SIZE, 2230 DMA_BIDIRECTIONAL); 2231 if (dma_mapping_error(hw->device, pg_dma[i])) 2232 goto err; 2233 2234 addr += PAGE_SIZE; 2235 } 2236 2237 return 0; 2238 2239err: 2240 irdma_unmap_vm_page_list(hw, pg_dma, i); 2241 return -ENOMEM; 2242} 2243 2244void irdma_unmap_vm_page_list(struct irdma_hw *hw, dma_addr_t *pg_dma, u32 pg_cnt) 2245{ 2246 int i; 2247 2248 for (i = 0; i < pg_cnt; i++) 2249 dma_unmap_page(hw->device, pg_dma[i], PAGE_SIZE, DMA_BIDIRECTIONAL); 2250} 2251 2252/** 2253 * irdma_pble_free_paged_mem - free virtual paged memory 2254 * @chunk: chunk to free with paged memory 2255 */ 2256void irdma_pble_free_paged_mem(struct irdma_chunk *chunk) 2257{ 2258 if (!chunk->pg_cnt) 2259 goto done; 2260 2261 irdma_unmap_vm_page_list(chunk->dev->hw, chunk->dmainfo.dmaaddrs, 2262 chunk->pg_cnt); 2263 2264done: 2265 kfree(chunk->dmainfo.dmaaddrs); 2266 chunk->dmainfo.dmaaddrs = NULL; 2267 vfree(chunk->vaddr); 2268 chunk->vaddr = NULL; 2269 chunk->type = 0; 2270} 2271 2272/** 2273 * irdma_pble_get_paged_mem -allocate paged memory for pbles 2274 * @chunk: chunk to add for paged memory 2275 * @pg_cnt: number of pages needed 2276 */ 2277int irdma_pble_get_paged_mem(struct irdma_chunk *chunk, u32 pg_cnt) 2278{ 2279 u32 size; 2280 void *va; 2281 2282 chunk->dmainfo.dmaaddrs = kzalloc(pg_cnt << 3, GFP_KERNEL); 2283 if (!chunk->dmainfo.dmaaddrs) 2284 return -ENOMEM; 2285 2286 size = PAGE_SIZE * pg_cnt; 2287 va = vmalloc(size); 2288 if (!va) 2289 goto err; 2290 2291 if (irdma_map_vm_page_list(chunk->dev->hw, va, chunk->dmainfo.dmaaddrs, 2292 pg_cnt)) { 2293 vfree(va); 2294 goto err; 2295 } 2296 chunk->vaddr = va; 2297 chunk->size = size; 2298 chunk->pg_cnt = pg_cnt; 2299 chunk->type = PBLE_SD_PAGED; 2300 2301 return 0; 2302err: 2303 kfree(chunk->dmainfo.dmaaddrs); 2304 chunk->dmainfo.dmaaddrs = NULL; 2305 2306 return -ENOMEM; 2307} 2308 2309/** 2310 * irdma_alloc_ws_node_id - Allocate a tx scheduler node ID 2311 * @dev: device pointer 2312 */ 2313u16 irdma_alloc_ws_node_id(struct irdma_sc_dev *dev) 2314{ 2315 struct irdma_pci_f *rf = dev_to_rf(dev); 2316 u32 next = 1; 2317 u32 node_id; 2318 2319 if (irdma_alloc_rsrc(rf, rf->allocated_ws_nodes, rf->max_ws_node_id, 2320 &node_id, &next)) 2321 return IRDMA_WS_NODE_INVALID; 2322 2323 return (u16)node_id; 2324} 2325 2326/** 2327 * irdma_free_ws_node_id - Free a tx scheduler node ID 2328 * @dev: device pointer 2329 * @node_id: Work scheduler node ID 2330 */ 2331void irdma_free_ws_node_id(struct irdma_sc_dev *dev, u16 node_id) 2332{ 2333 struct irdma_pci_f *rf = dev_to_rf(dev); 2334 2335 irdma_free_rsrc(rf, rf->allocated_ws_nodes, (u32)node_id); 2336} 2337 2338/** 2339 * irdma_modify_qp_to_err - Modify a QP to error 2340 * @sc_qp: qp structure 2341 */ 2342void irdma_modify_qp_to_err(struct irdma_sc_qp *sc_qp) 2343{ 2344 struct irdma_qp *qp = sc_qp->qp_uk.back_qp; 2345 struct ib_qp_attr attr; 2346 2347 if (qp->iwdev->rf->reset) 2348 return; 2349 attr.qp_state = IB_QPS_ERR; 2350 2351 if (rdma_protocol_roce(qp->ibqp.device, 1)) 2352 irdma_modify_qp_roce(&qp->ibqp, &attr, IB_QP_STATE, NULL); 2353 else 2354 irdma_modify_qp(&qp->ibqp, &attr, IB_QP_STATE, NULL); 2355} 2356 2357void irdma_ib_qp_event(struct irdma_qp *iwqp, enum irdma_qp_event_type event) 2358{ 2359 struct ib_event ibevent; 2360 2361 if (!iwqp->ibqp.event_handler) 2362 return; 2363 2364 switch (event) { 2365 case IRDMA_QP_EVENT_CATASTROPHIC: 2366 ibevent.event = IB_EVENT_QP_FATAL; 2367 break; 2368 case IRDMA_QP_EVENT_ACCESS_ERR: 2369 ibevent.event = IB_EVENT_QP_ACCESS_ERR; 2370 break; 2371 case IRDMA_QP_EVENT_REQ_ERR: 2372 ibevent.event = IB_EVENT_QP_REQ_ERR; 2373 break; 2374 } 2375 ibevent.device = iwqp->ibqp.device; 2376 ibevent.element.qp = &iwqp->ibqp; 2377 iwqp->ibqp.event_handler(&ibevent, iwqp->ibqp.qp_context); 2378} 2379 2380bool irdma_cq_empty(struct irdma_cq *iwcq) 2381{ 2382 struct irdma_cq_uk *ukcq; 2383 u64 qword3; 2384 __le64 *cqe; 2385 u8 polarity; 2386 2387 ukcq = &iwcq->sc_cq.cq_uk; 2388 cqe = IRDMA_GET_CURRENT_CQ_ELEM(ukcq); 2389 get_64bit_val(cqe, 24, &qword3); 2390 polarity = (u8)FIELD_GET(IRDMA_CQ_VALID, qword3); 2391 2392 return polarity != ukcq->polarity; 2393} 2394 2395void irdma_remove_cmpls_list(struct irdma_cq *iwcq) 2396{ 2397 struct irdma_cmpl_gen *cmpl_node; 2398 struct list_head *tmp_node, *list_node; 2399 2400 list_for_each_safe (list_node, tmp_node, &iwcq->cmpl_generated) { 2401 cmpl_node = list_entry(list_node, struct irdma_cmpl_gen, list); 2402 list_del(&cmpl_node->list); 2403 kfree(cmpl_node); 2404 } 2405} 2406 2407int irdma_generated_cmpls(struct irdma_cq *iwcq, struct irdma_cq_poll_info *cq_poll_info) 2408{ 2409 struct irdma_cmpl_gen *cmpl; 2410 2411 if (list_empty(&iwcq->cmpl_generated)) 2412 return -ENOENT; 2413 cmpl = list_first_entry_or_null(&iwcq->cmpl_generated, struct irdma_cmpl_gen, list); 2414 list_del(&cmpl->list); 2415 memcpy(cq_poll_info, &cmpl->cpi, sizeof(*cq_poll_info)); 2416 kfree(cmpl); 2417 2418 ibdev_dbg(iwcq->ibcq.device, 2419 "VERBS: %s: Poll artificially generated completion for QP 0x%X, op %u, wr_id=0x%llx\n", 2420 __func__, cq_poll_info->qp_id, cq_poll_info->op_type, 2421 cq_poll_info->wr_id); 2422 2423 return 0; 2424} 2425 2426/** 2427 * irdma_set_cpi_common_values - fill in values for polling info struct 2428 * @cpi: resulting structure of cq_poll_info type 2429 * @qp: QPair 2430 * @qp_num: id of the QP 2431 */ 2432static void irdma_set_cpi_common_values(struct irdma_cq_poll_info *cpi, 2433 struct irdma_qp_uk *qp, u32 qp_num) 2434{ 2435 cpi->comp_status = IRDMA_COMPL_STATUS_FLUSHED; 2436 cpi->error = true; 2437 cpi->major_err = IRDMA_FLUSH_MAJOR_ERR; 2438 cpi->minor_err = FLUSH_GENERAL_ERR; 2439 cpi->qp_handle = (irdma_qp_handle)(uintptr_t)qp; 2440 cpi->qp_id = qp_num; 2441} 2442 2443static inline void irdma_comp_handler(struct irdma_cq *cq) 2444{ 2445 if (!cq->ibcq.comp_handler) 2446 return; 2447 if (atomic_cmpxchg(&cq->armed, 1, 0)) 2448 cq->ibcq.comp_handler(&cq->ibcq, cq->ibcq.cq_context); 2449} 2450 2451void irdma_generate_flush_completions(struct irdma_qp *iwqp) 2452{ 2453 struct irdma_qp_uk *qp = &iwqp->sc_qp.qp_uk; 2454 struct irdma_ring *sq_ring = &qp->sq_ring; 2455 struct irdma_ring *rq_ring = &qp->rq_ring; 2456 struct irdma_cmpl_gen *cmpl; 2457 __le64 *sw_wqe; 2458 u64 wqe_qword; 2459 u32 wqe_idx; 2460 bool compl_generated = false; 2461 unsigned long flags1; 2462 2463 spin_lock_irqsave(&iwqp->iwscq->lock, flags1); 2464 if (irdma_cq_empty(iwqp->iwscq)) { 2465 unsigned long flags2; 2466 2467 spin_lock_irqsave(&iwqp->lock, flags2); 2468 while (IRDMA_RING_MORE_WORK(*sq_ring)) { 2469 cmpl = kzalloc(sizeof(*cmpl), GFP_ATOMIC); 2470 if (!cmpl) { 2471 spin_unlock_irqrestore(&iwqp->lock, flags2); 2472 spin_unlock_irqrestore(&iwqp->iwscq->lock, flags1); 2473 return; 2474 } 2475 2476 wqe_idx = sq_ring->tail; 2477 irdma_set_cpi_common_values(&cmpl->cpi, qp, qp->qp_id); 2478 2479 cmpl->cpi.wr_id = qp->sq_wrtrk_array[wqe_idx].wrid; 2480 sw_wqe = qp->sq_base[wqe_idx].elem; 2481 get_64bit_val(sw_wqe, 24, &wqe_qword); 2482 cmpl->cpi.op_type = (u8)FIELD_GET(IRDMAQPSQ_OPCODE, IRDMAQPSQ_OPCODE); 2483 cmpl->cpi.q_type = IRDMA_CQE_QTYPE_SQ; 2484 /* remove the SQ WR by moving SQ tail*/ 2485 IRDMA_RING_SET_TAIL(*sq_ring, 2486 sq_ring->tail + qp->sq_wrtrk_array[sq_ring->tail].quanta); 2487 if (cmpl->cpi.op_type == IRDMAQP_OP_NOP) { 2488 kfree(cmpl); 2489 continue; 2490 } 2491 ibdev_dbg(iwqp->iwscq->ibcq.device, 2492 "DEV: %s: adding wr_id = 0x%llx SQ Completion to list qp_id=%d\n", 2493 __func__, cmpl->cpi.wr_id, qp->qp_id); 2494 list_add_tail(&cmpl->list, &iwqp->iwscq->cmpl_generated); 2495 compl_generated = true; 2496 } 2497 spin_unlock_irqrestore(&iwqp->lock, flags2); 2498 spin_unlock_irqrestore(&iwqp->iwscq->lock, flags1); 2499 if (compl_generated) 2500 irdma_comp_handler(iwqp->iwscq); 2501 } else { 2502 spin_unlock_irqrestore(&iwqp->iwscq->lock, flags1); 2503 mod_delayed_work(iwqp->iwdev->cleanup_wq, &iwqp->dwork_flush, 2504 msecs_to_jiffies(IRDMA_FLUSH_DELAY_MS)); 2505 } 2506 2507 spin_lock_irqsave(&iwqp->iwrcq->lock, flags1); 2508 if (irdma_cq_empty(iwqp->iwrcq)) { 2509 unsigned long flags2; 2510 2511 spin_lock_irqsave(&iwqp->lock, flags2); 2512 while (IRDMA_RING_MORE_WORK(*rq_ring)) { 2513 cmpl = kzalloc(sizeof(*cmpl), GFP_ATOMIC); 2514 if (!cmpl) { 2515 spin_unlock_irqrestore(&iwqp->lock, flags2); 2516 spin_unlock_irqrestore(&iwqp->iwrcq->lock, flags1); 2517 return; 2518 } 2519 2520 wqe_idx = rq_ring->tail; 2521 irdma_set_cpi_common_values(&cmpl->cpi, qp, qp->qp_id); 2522 2523 cmpl->cpi.wr_id = qp->rq_wrid_array[wqe_idx]; 2524 cmpl->cpi.op_type = IRDMA_OP_TYPE_REC; 2525 cmpl->cpi.q_type = IRDMA_CQE_QTYPE_RQ; 2526 /* remove the RQ WR by moving RQ tail */ 2527 IRDMA_RING_SET_TAIL(*rq_ring, rq_ring->tail + 1); 2528 ibdev_dbg(iwqp->iwrcq->ibcq.device, 2529 "DEV: %s: adding wr_id = 0x%llx RQ Completion to list qp_id=%d, wqe_idx=%d\n", 2530 __func__, cmpl->cpi.wr_id, qp->qp_id, 2531 wqe_idx); 2532 list_add_tail(&cmpl->list, &iwqp->iwrcq->cmpl_generated); 2533 2534 compl_generated = true; 2535 } 2536 spin_unlock_irqrestore(&iwqp->lock, flags2); 2537 spin_unlock_irqrestore(&iwqp->iwrcq->lock, flags1); 2538 if (compl_generated) 2539 irdma_comp_handler(iwqp->iwrcq); 2540 } else { 2541 spin_unlock_irqrestore(&iwqp->iwrcq->lock, flags1); 2542 mod_delayed_work(iwqp->iwdev->cleanup_wq, &iwqp->dwork_flush, 2543 msecs_to_jiffies(IRDMA_FLUSH_DELAY_MS)); 2544 } 2545} 2546