1// SPDX-License-Identifier: GPL-2.0 2/* Copyright(c) 2013 - 2019 Intel Corporation. */ 3 4#include <linux/types.h> 5#include <linux/module.h> 6#include <net/ipv6.h> 7#include <net/ip.h> 8#include <net/tcp.h> 9#include <linux/if_macvlan.h> 10#include <linux/prefetch.h> 11 12#include "fm10k.h" 13 14#define DRV_SUMMARY "Intel(R) Ethernet Switch Host Interface Driver" 15char fm10k_driver_name[] = "fm10k"; 16static const char fm10k_driver_string[] = DRV_SUMMARY; 17static const char fm10k_copyright[] = 18 "Copyright(c) 2013 - 2019 Intel Corporation."; 19 20MODULE_AUTHOR("Intel Corporation, <linux.nics@intel.com>"); 21MODULE_DESCRIPTION(DRV_SUMMARY); 22MODULE_LICENSE("GPL v2"); 23 24/* single workqueue for entire fm10k driver */ 25struct workqueue_struct *fm10k_workqueue; 26 27/** 28 * fm10k_init_module - Driver Registration Routine 29 * 30 * fm10k_init_module is the first routine called when the driver is 31 * loaded. All it does is register with the PCI subsystem. 32 **/ 33static int __init fm10k_init_module(void) 34{ 35 int ret; 36 37 pr_info("%s\n", fm10k_driver_string); 38 pr_info("%s\n", fm10k_copyright); 39 40 /* create driver workqueue */ 41 fm10k_workqueue = alloc_workqueue("%s", WQ_MEM_RECLAIM, 0, 42 fm10k_driver_name); 43 if (!fm10k_workqueue) 44 return -ENOMEM; 45 46 fm10k_dbg_init(); 47 48 ret = fm10k_register_pci_driver(); 49 if (ret) { 50 fm10k_dbg_exit(); 51 destroy_workqueue(fm10k_workqueue); 52 } 53 54 return ret; 55} 56module_init(fm10k_init_module); 57 58/** 59 * fm10k_exit_module - Driver Exit Cleanup Routine 60 * 61 * fm10k_exit_module is called just before the driver is removed 62 * from memory. 63 **/ 64static void __exit fm10k_exit_module(void) 65{ 66 fm10k_unregister_pci_driver(); 67 68 fm10k_dbg_exit(); 69 70 /* destroy driver workqueue */ 71 destroy_workqueue(fm10k_workqueue); 72} 73module_exit(fm10k_exit_module); 74 75static bool fm10k_alloc_mapped_page(struct fm10k_ring *rx_ring, 76 struct fm10k_rx_buffer *bi) 77{ 78 struct page *page = bi->page; 79 dma_addr_t dma; 80 81 /* Only page will be NULL if buffer was consumed */ 82 if (likely(page)) 83 return true; 84 85 /* alloc new page for storage */ 86 page = dev_alloc_page(); 87 if (unlikely(!page)) { 88 rx_ring->rx_stats.alloc_failed++; 89 return false; 90 } 91 92 /* map page for use */ 93 dma = dma_map_page(rx_ring->dev, page, 0, PAGE_SIZE, DMA_FROM_DEVICE); 94 95 /* if mapping failed free memory back to system since 96 * there isn't much point in holding memory we can't use 97 */ 98 if (dma_mapping_error(rx_ring->dev, dma)) { 99 __free_page(page); 100 101 rx_ring->rx_stats.alloc_failed++; 102 return false; 103 } 104 105 bi->dma = dma; 106 bi->page = page; 107 bi->page_offset = 0; 108 109 return true; 110} 111 112/** 113 * fm10k_alloc_rx_buffers - Replace used receive buffers 114 * @rx_ring: ring to place buffers on 115 * @cleaned_count: number of buffers to replace 116 **/ 117void fm10k_alloc_rx_buffers(struct fm10k_ring *rx_ring, u16 cleaned_count) 118{ 119 union fm10k_rx_desc *rx_desc; 120 struct fm10k_rx_buffer *bi; 121 u16 i = rx_ring->next_to_use; 122 123 /* nothing to do */ 124 if (!cleaned_count) 125 return; 126 127 rx_desc = FM10K_RX_DESC(rx_ring, i); 128 bi = &rx_ring->rx_buffer[i]; 129 i -= rx_ring->count; 130 131 do { 132 if (!fm10k_alloc_mapped_page(rx_ring, bi)) 133 break; 134 135 /* Refresh the desc even if buffer_addrs didn't change 136 * because each write-back erases this info. 137 */ 138 rx_desc->q.pkt_addr = cpu_to_le64(bi->dma + bi->page_offset); 139 140 rx_desc++; 141 bi++; 142 i++; 143 if (unlikely(!i)) { 144 rx_desc = FM10K_RX_DESC(rx_ring, 0); 145 bi = rx_ring->rx_buffer; 146 i -= rx_ring->count; 147 } 148 149 /* clear the status bits for the next_to_use descriptor */ 150 rx_desc->d.staterr = 0; 151 152 cleaned_count--; 153 } while (cleaned_count); 154 155 i += rx_ring->count; 156 157 if (rx_ring->next_to_use != i) { 158 /* record the next descriptor to use */ 159 rx_ring->next_to_use = i; 160 161 /* update next to alloc since we have filled the ring */ 162 rx_ring->next_to_alloc = i; 163 164 /* Force memory writes to complete before letting h/w 165 * know there are new descriptors to fetch. (Only 166 * applicable for weak-ordered memory model archs, 167 * such as IA-64). 168 */ 169 wmb(); 170 171 /* notify hardware of new descriptors */ 172 writel(i, rx_ring->tail); 173 } 174} 175 176/** 177 * fm10k_reuse_rx_page - page flip buffer and store it back on the ring 178 * @rx_ring: rx descriptor ring to store buffers on 179 * @old_buff: donor buffer to have page reused 180 * 181 * Synchronizes page for reuse by the interface 182 **/ 183static void fm10k_reuse_rx_page(struct fm10k_ring *rx_ring, 184 struct fm10k_rx_buffer *old_buff) 185{ 186 struct fm10k_rx_buffer *new_buff; 187 u16 nta = rx_ring->next_to_alloc; 188 189 new_buff = &rx_ring->rx_buffer[nta]; 190 191 /* update, and store next to alloc */ 192 nta++; 193 rx_ring->next_to_alloc = (nta < rx_ring->count) ? nta : 0; 194 195 /* transfer page from old buffer to new buffer */ 196 *new_buff = *old_buff; 197 198 /* sync the buffer for use by the device */ 199 dma_sync_single_range_for_device(rx_ring->dev, old_buff->dma, 200 old_buff->page_offset, 201 FM10K_RX_BUFSZ, 202 DMA_FROM_DEVICE); 203} 204 205static inline bool fm10k_page_is_reserved(struct page *page) 206{ 207 return (page_to_nid(page) != numa_mem_id()) || page_is_pfmemalloc(page); 208} 209 210static bool fm10k_can_reuse_rx_page(struct fm10k_rx_buffer *rx_buffer, 211 struct page *page, 212 unsigned int __maybe_unused truesize) 213{ 214 /* avoid re-using remote pages */ 215 if (unlikely(fm10k_page_is_reserved(page))) 216 return false; 217 218#if (PAGE_SIZE < 8192) 219 /* if we are only owner of page we can reuse it */ 220 if (unlikely(page_count(page) != 1)) 221 return false; 222 223 /* flip page offset to other buffer */ 224 rx_buffer->page_offset ^= FM10K_RX_BUFSZ; 225#else 226 /* move offset up to the next cache line */ 227 rx_buffer->page_offset += truesize; 228 229 if (rx_buffer->page_offset > (PAGE_SIZE - FM10K_RX_BUFSZ)) 230 return false; 231#endif 232 233 /* Even if we own the page, we are not allowed to use atomic_set() 234 * This would break get_page_unless_zero() users. 235 */ 236 page_ref_inc(page); 237 238 return true; 239} 240 241/** 242 * fm10k_add_rx_frag - Add contents of Rx buffer to sk_buff 243 * @rx_buffer: buffer containing page to add 244 * @size: packet size from rx_desc 245 * @rx_desc: descriptor containing length of buffer written by hardware 246 * @skb: sk_buff to place the data into 247 * 248 * This function will add the data contained in rx_buffer->page to the skb. 249 * This is done either through a direct copy if the data in the buffer is 250 * less than the skb header size, otherwise it will just attach the page as 251 * a frag to the skb. 252 * 253 * The function will then update the page offset if necessary and return 254 * true if the buffer can be reused by the interface. 255 **/ 256static bool fm10k_add_rx_frag(struct fm10k_rx_buffer *rx_buffer, 257 unsigned int size, 258 union fm10k_rx_desc *rx_desc, 259 struct sk_buff *skb) 260{ 261 struct page *page = rx_buffer->page; 262 unsigned char *va = page_address(page) + rx_buffer->page_offset; 263#if (PAGE_SIZE < 8192) 264 unsigned int truesize = FM10K_RX_BUFSZ; 265#else 266 unsigned int truesize = ALIGN(size, 512); 267#endif 268 unsigned int pull_len; 269 270 if (unlikely(skb_is_nonlinear(skb))) 271 goto add_tail_frag; 272 273 if (likely(size <= FM10K_RX_HDR_LEN)) { 274 memcpy(__skb_put(skb, size), va, ALIGN(size, sizeof(long))); 275 276 /* page is not reserved, we can reuse buffer as-is */ 277 if (likely(!fm10k_page_is_reserved(page))) 278 return true; 279 280 /* this page cannot be reused so discard it */ 281 __free_page(page); 282 return false; 283 } 284 285 /* we need the header to contain the greater of either ETH_HLEN or 286 * 60 bytes if the skb->len is less than 60 for skb_pad. 287 */ 288 pull_len = eth_get_headlen(skb->dev, va, FM10K_RX_HDR_LEN); 289 290 /* align pull length to size of long to optimize memcpy performance */ 291 memcpy(__skb_put(skb, pull_len), va, ALIGN(pull_len, sizeof(long))); 292 293 /* update all of the pointers */ 294 va += pull_len; 295 size -= pull_len; 296 297add_tail_frag: 298 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, page, 299 (unsigned long)va & ~PAGE_MASK, size, truesize); 300 301 return fm10k_can_reuse_rx_page(rx_buffer, page, truesize); 302} 303 304static struct sk_buff *fm10k_fetch_rx_buffer(struct fm10k_ring *rx_ring, 305 union fm10k_rx_desc *rx_desc, 306 struct sk_buff *skb) 307{ 308 unsigned int size = le16_to_cpu(rx_desc->w.length); 309 struct fm10k_rx_buffer *rx_buffer; 310 struct page *page; 311 312 rx_buffer = &rx_ring->rx_buffer[rx_ring->next_to_clean]; 313 page = rx_buffer->page; 314 prefetchw(page); 315 316 if (likely(!skb)) { 317 void *page_addr = page_address(page) + 318 rx_buffer->page_offset; 319 320 /* prefetch first cache line of first page */ 321 net_prefetch(page_addr); 322 323 /* allocate a skb to store the frags */ 324 skb = napi_alloc_skb(&rx_ring->q_vector->napi, 325 FM10K_RX_HDR_LEN); 326 if (unlikely(!skb)) { 327 rx_ring->rx_stats.alloc_failed++; 328 return NULL; 329 } 330 331 /* we will be copying header into skb->data in 332 * pskb_may_pull so it is in our interest to prefetch 333 * it now to avoid a possible cache miss 334 */ 335 prefetchw(skb->data); 336 } 337 338 /* we are reusing so sync this buffer for CPU use */ 339 dma_sync_single_range_for_cpu(rx_ring->dev, 340 rx_buffer->dma, 341 rx_buffer->page_offset, 342 size, 343 DMA_FROM_DEVICE); 344 345 /* pull page into skb */ 346 if (fm10k_add_rx_frag(rx_buffer, size, rx_desc, skb)) { 347 /* hand second half of page back to the ring */ 348 fm10k_reuse_rx_page(rx_ring, rx_buffer); 349 } else { 350 /* we are not reusing the buffer so unmap it */ 351 dma_unmap_page(rx_ring->dev, rx_buffer->dma, 352 PAGE_SIZE, DMA_FROM_DEVICE); 353 } 354 355 /* clear contents of rx_buffer */ 356 rx_buffer->page = NULL; 357 358 return skb; 359} 360 361static inline void fm10k_rx_checksum(struct fm10k_ring *ring, 362 union fm10k_rx_desc *rx_desc, 363 struct sk_buff *skb) 364{ 365 skb_checksum_none_assert(skb); 366 367 /* Rx checksum disabled via ethtool */ 368 if (!(ring->netdev->features & NETIF_F_RXCSUM)) 369 return; 370 371 /* TCP/UDP checksum error bit is set */ 372 if (fm10k_test_staterr(rx_desc, 373 FM10K_RXD_STATUS_L4E | 374 FM10K_RXD_STATUS_L4E2 | 375 FM10K_RXD_STATUS_IPE | 376 FM10K_RXD_STATUS_IPE2)) { 377 ring->rx_stats.csum_err++; 378 return; 379 } 380 381 /* It must be a TCP or UDP packet with a valid checksum */ 382 if (fm10k_test_staterr(rx_desc, FM10K_RXD_STATUS_L4CS2)) 383 skb->encapsulation = true; 384 else if (!fm10k_test_staterr(rx_desc, FM10K_RXD_STATUS_L4CS)) 385 return; 386 387 skb->ip_summed = CHECKSUM_UNNECESSARY; 388 389 ring->rx_stats.csum_good++; 390} 391 392#define FM10K_RSS_L4_TYPES_MASK \ 393 (BIT(FM10K_RSSTYPE_IPV4_TCP) | \ 394 BIT(FM10K_RSSTYPE_IPV4_UDP) | \ 395 BIT(FM10K_RSSTYPE_IPV6_TCP) | \ 396 BIT(FM10K_RSSTYPE_IPV6_UDP)) 397 398static inline void fm10k_rx_hash(struct fm10k_ring *ring, 399 union fm10k_rx_desc *rx_desc, 400 struct sk_buff *skb) 401{ 402 u16 rss_type; 403 404 if (!(ring->netdev->features & NETIF_F_RXHASH)) 405 return; 406 407 rss_type = le16_to_cpu(rx_desc->w.pkt_info) & FM10K_RXD_RSSTYPE_MASK; 408 if (!rss_type) 409 return; 410 411 skb_set_hash(skb, le32_to_cpu(rx_desc->d.rss), 412 (BIT(rss_type) & FM10K_RSS_L4_TYPES_MASK) ? 413 PKT_HASH_TYPE_L4 : PKT_HASH_TYPE_L3); 414} 415 416static void fm10k_type_trans(struct fm10k_ring *rx_ring, 417 union fm10k_rx_desc __maybe_unused *rx_desc, 418 struct sk_buff *skb) 419{ 420 struct net_device *dev = rx_ring->netdev; 421 struct fm10k_l2_accel *l2_accel = rcu_dereference_bh(rx_ring->l2_accel); 422 423 /* check to see if DGLORT belongs to a MACVLAN */ 424 if (l2_accel) { 425 u16 idx = le16_to_cpu(FM10K_CB(skb)->fi.w.dglort) - 1; 426 427 idx -= l2_accel->dglort; 428 if (idx < l2_accel->size && l2_accel->macvlan[idx]) 429 dev = l2_accel->macvlan[idx]; 430 else 431 l2_accel = NULL; 432 } 433 434 /* Record Rx queue, or update macvlan statistics */ 435 if (!l2_accel) 436 skb_record_rx_queue(skb, rx_ring->queue_index); 437 else 438 macvlan_count_rx(netdev_priv(dev), skb->len + ETH_HLEN, true, 439 false); 440 441 skb->protocol = eth_type_trans(skb, dev); 442} 443 444/** 445 * fm10k_process_skb_fields - Populate skb header fields from Rx descriptor 446 * @rx_ring: rx descriptor ring packet is being transacted on 447 * @rx_desc: pointer to the EOP Rx descriptor 448 * @skb: pointer to current skb being populated 449 * 450 * This function checks the ring, descriptor, and packet information in 451 * order to populate the hash, checksum, VLAN, timestamp, protocol, and 452 * other fields within the skb. 453 **/ 454static unsigned int fm10k_process_skb_fields(struct fm10k_ring *rx_ring, 455 union fm10k_rx_desc *rx_desc, 456 struct sk_buff *skb) 457{ 458 unsigned int len = skb->len; 459 460 fm10k_rx_hash(rx_ring, rx_desc, skb); 461 462 fm10k_rx_checksum(rx_ring, rx_desc, skb); 463 464 FM10K_CB(skb)->tstamp = rx_desc->q.timestamp; 465 466 FM10K_CB(skb)->fi.w.vlan = rx_desc->w.vlan; 467 468 FM10K_CB(skb)->fi.d.glort = rx_desc->d.glort; 469 470 if (rx_desc->w.vlan) { 471 u16 vid = le16_to_cpu(rx_desc->w.vlan); 472 473 if ((vid & VLAN_VID_MASK) != rx_ring->vid) 474 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vid); 475 else if (vid & VLAN_PRIO_MASK) 476 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), 477 vid & VLAN_PRIO_MASK); 478 } 479 480 fm10k_type_trans(rx_ring, rx_desc, skb); 481 482 return len; 483} 484 485/** 486 * fm10k_is_non_eop - process handling of non-EOP buffers 487 * @rx_ring: Rx ring being processed 488 * @rx_desc: Rx descriptor for current buffer 489 * 490 * This function updates next to clean. If the buffer is an EOP buffer 491 * this function exits returning false, otherwise it will place the 492 * sk_buff in the next buffer to be chained and return true indicating 493 * that this is in fact a non-EOP buffer. 494 **/ 495static bool fm10k_is_non_eop(struct fm10k_ring *rx_ring, 496 union fm10k_rx_desc *rx_desc) 497{ 498 u32 ntc = rx_ring->next_to_clean + 1; 499 500 /* fetch, update, and store next to clean */ 501 ntc = (ntc < rx_ring->count) ? ntc : 0; 502 rx_ring->next_to_clean = ntc; 503 504 prefetch(FM10K_RX_DESC(rx_ring, ntc)); 505 506 if (likely(fm10k_test_staterr(rx_desc, FM10K_RXD_STATUS_EOP))) 507 return false; 508 509 return true; 510} 511 512/** 513 * fm10k_cleanup_headers - Correct corrupted or empty headers 514 * @rx_ring: rx descriptor ring packet is being transacted on 515 * @rx_desc: pointer to the EOP Rx descriptor 516 * @skb: pointer to current skb being fixed 517 * 518 * Address the case where we are pulling data in on pages only 519 * and as such no data is present in the skb header. 520 * 521 * In addition if skb is not at least 60 bytes we need to pad it so that 522 * it is large enough to qualify as a valid Ethernet frame. 523 * 524 * Returns true if an error was encountered and skb was freed. 525 **/ 526static bool fm10k_cleanup_headers(struct fm10k_ring *rx_ring, 527 union fm10k_rx_desc *rx_desc, 528 struct sk_buff *skb) 529{ 530 if (unlikely((fm10k_test_staterr(rx_desc, 531 FM10K_RXD_STATUS_RXE)))) { 532#define FM10K_TEST_RXD_BIT(rxd, bit) \ 533 ((rxd)->w.csum_err & cpu_to_le16(bit)) 534 if (FM10K_TEST_RXD_BIT(rx_desc, FM10K_RXD_ERR_SWITCH_ERROR)) 535 rx_ring->rx_stats.switch_errors++; 536 if (FM10K_TEST_RXD_BIT(rx_desc, FM10K_RXD_ERR_NO_DESCRIPTOR)) 537 rx_ring->rx_stats.drops++; 538 if (FM10K_TEST_RXD_BIT(rx_desc, FM10K_RXD_ERR_PP_ERROR)) 539 rx_ring->rx_stats.pp_errors++; 540 if (FM10K_TEST_RXD_BIT(rx_desc, FM10K_RXD_ERR_SWITCH_READY)) 541 rx_ring->rx_stats.link_errors++; 542 if (FM10K_TEST_RXD_BIT(rx_desc, FM10K_RXD_ERR_TOO_BIG)) 543 rx_ring->rx_stats.length_errors++; 544 dev_kfree_skb_any(skb); 545 rx_ring->rx_stats.errors++; 546 return true; 547 } 548 549 /* if eth_skb_pad returns an error the skb was freed */ 550 if (eth_skb_pad(skb)) 551 return true; 552 553 return false; 554} 555 556/** 557 * fm10k_receive_skb - helper function to handle rx indications 558 * @q_vector: structure containing interrupt and ring information 559 * @skb: packet to send up 560 **/ 561static void fm10k_receive_skb(struct fm10k_q_vector *q_vector, 562 struct sk_buff *skb) 563{ 564 napi_gro_receive(&q_vector->napi, skb); 565} 566 567static int fm10k_clean_rx_irq(struct fm10k_q_vector *q_vector, 568 struct fm10k_ring *rx_ring, 569 int budget) 570{ 571 struct sk_buff *skb = rx_ring->skb; 572 unsigned int total_bytes = 0, total_packets = 0; 573 u16 cleaned_count = fm10k_desc_unused(rx_ring); 574 575 while (likely(total_packets < budget)) { 576 union fm10k_rx_desc *rx_desc; 577 578 /* return some buffers to hardware, one at a time is too slow */ 579 if (cleaned_count >= FM10K_RX_BUFFER_WRITE) { 580 fm10k_alloc_rx_buffers(rx_ring, cleaned_count); 581 cleaned_count = 0; 582 } 583 584 rx_desc = FM10K_RX_DESC(rx_ring, rx_ring->next_to_clean); 585 586 if (!rx_desc->d.staterr) 587 break; 588 589 /* This memory barrier is needed to keep us from reading 590 * any other fields out of the rx_desc until we know the 591 * descriptor has been written back 592 */ 593 dma_rmb(); 594 595 /* retrieve a buffer from the ring */ 596 skb = fm10k_fetch_rx_buffer(rx_ring, rx_desc, skb); 597 598 /* exit if we failed to retrieve a buffer */ 599 if (!skb) 600 break; 601 602 cleaned_count++; 603 604 /* fetch next buffer in frame if non-eop */ 605 if (fm10k_is_non_eop(rx_ring, rx_desc)) 606 continue; 607 608 /* verify the packet layout is correct */ 609 if (fm10k_cleanup_headers(rx_ring, rx_desc, skb)) { 610 skb = NULL; 611 continue; 612 } 613 614 /* populate checksum, timestamp, VLAN, and protocol */ 615 total_bytes += fm10k_process_skb_fields(rx_ring, rx_desc, skb); 616 617 fm10k_receive_skb(q_vector, skb); 618 619 /* reset skb pointer */ 620 skb = NULL; 621 622 /* update budget accounting */ 623 total_packets++; 624 } 625 626 /* place incomplete frames back on ring for completion */ 627 rx_ring->skb = skb; 628 629 u64_stats_update_begin(&rx_ring->syncp); 630 rx_ring->stats.packets += total_packets; 631 rx_ring->stats.bytes += total_bytes; 632 u64_stats_update_end(&rx_ring->syncp); 633 q_vector->rx.total_packets += total_packets; 634 q_vector->rx.total_bytes += total_bytes; 635 636 return total_packets; 637} 638 639#define VXLAN_HLEN (sizeof(struct udphdr) + 8) 640static struct ethhdr *fm10k_port_is_vxlan(struct sk_buff *skb) 641{ 642 struct fm10k_intfc *interface = netdev_priv(skb->dev); 643 644 if (interface->vxlan_port != udp_hdr(skb)->dest) 645 return NULL; 646 647 /* return offset of udp_hdr plus 8 bytes for VXLAN header */ 648 return (struct ethhdr *)(skb_transport_header(skb) + VXLAN_HLEN); 649} 650 651#define FM10K_NVGRE_RESERVED0_FLAGS htons(0x9FFF) 652#define NVGRE_TNI htons(0x2000) 653struct fm10k_nvgre_hdr { 654 __be16 flags; 655 __be16 proto; 656 __be32 tni; 657}; 658 659static struct ethhdr *fm10k_gre_is_nvgre(struct sk_buff *skb) 660{ 661 struct fm10k_nvgre_hdr *nvgre_hdr; 662 int hlen = ip_hdrlen(skb); 663 664 /* currently only IPv4 is supported due to hlen above */ 665 if (vlan_get_protocol(skb) != htons(ETH_P_IP)) 666 return NULL; 667 668 /* our transport header should be NVGRE */ 669 nvgre_hdr = (struct fm10k_nvgre_hdr *)(skb_network_header(skb) + hlen); 670 671 /* verify all reserved flags are 0 */ 672 if (nvgre_hdr->flags & FM10K_NVGRE_RESERVED0_FLAGS) 673 return NULL; 674 675 /* report start of ethernet header */ 676 if (nvgre_hdr->flags & NVGRE_TNI) 677 return (struct ethhdr *)(nvgre_hdr + 1); 678 679 return (struct ethhdr *)(&nvgre_hdr->tni); 680} 681 682__be16 fm10k_tx_encap_offload(struct sk_buff *skb) 683{ 684 u8 l4_hdr = 0, inner_l4_hdr = 0, inner_l4_hlen; 685 struct ethhdr *eth_hdr; 686 687 if (skb->inner_protocol_type != ENCAP_TYPE_ETHER || 688 skb->inner_protocol != htons(ETH_P_TEB)) 689 return 0; 690 691 switch (vlan_get_protocol(skb)) { 692 case htons(ETH_P_IP): 693 l4_hdr = ip_hdr(skb)->protocol; 694 break; 695 case htons(ETH_P_IPV6): 696 l4_hdr = ipv6_hdr(skb)->nexthdr; 697 break; 698 default: 699 return 0; 700 } 701 702 switch (l4_hdr) { 703 case IPPROTO_UDP: 704 eth_hdr = fm10k_port_is_vxlan(skb); 705 break; 706 case IPPROTO_GRE: 707 eth_hdr = fm10k_gre_is_nvgre(skb); 708 break; 709 default: 710 return 0; 711 } 712 713 if (!eth_hdr) 714 return 0; 715 716 switch (eth_hdr->h_proto) { 717 case htons(ETH_P_IP): 718 inner_l4_hdr = inner_ip_hdr(skb)->protocol; 719 break; 720 case htons(ETH_P_IPV6): 721 inner_l4_hdr = inner_ipv6_hdr(skb)->nexthdr; 722 break; 723 default: 724 return 0; 725 } 726 727 switch (inner_l4_hdr) { 728 case IPPROTO_TCP: 729 inner_l4_hlen = inner_tcp_hdrlen(skb); 730 break; 731 case IPPROTO_UDP: 732 inner_l4_hlen = 8; 733 break; 734 default: 735 return 0; 736 } 737 738 /* The hardware allows tunnel offloads only if the combined inner and 739 * outer header is 184 bytes or less 740 */ 741 if (skb_inner_transport_header(skb) + inner_l4_hlen - 742 skb_mac_header(skb) > FM10K_TUNNEL_HEADER_LENGTH) 743 return 0; 744 745 return eth_hdr->h_proto; 746} 747 748static int fm10k_tso(struct fm10k_ring *tx_ring, 749 struct fm10k_tx_buffer *first) 750{ 751 struct sk_buff *skb = first->skb; 752 struct fm10k_tx_desc *tx_desc; 753 unsigned char *th; 754 u8 hdrlen; 755 756 if (skb->ip_summed != CHECKSUM_PARTIAL) 757 return 0; 758 759 if (!skb_is_gso(skb)) 760 return 0; 761 762 /* compute header lengths */ 763 if (skb->encapsulation) { 764 if (!fm10k_tx_encap_offload(skb)) 765 goto err_vxlan; 766 th = skb_inner_transport_header(skb); 767 } else { 768 th = skb_transport_header(skb); 769 } 770 771 /* compute offset from SOF to transport header and add header len */ 772 hdrlen = (th - skb->data) + (((struct tcphdr *)th)->doff << 2); 773 774 first->tx_flags |= FM10K_TX_FLAGS_CSUM; 775 776 /* update gso size and bytecount with header size */ 777 first->gso_segs = skb_shinfo(skb)->gso_segs; 778 first->bytecount += (first->gso_segs - 1) * hdrlen; 779 780 /* populate Tx descriptor header size and mss */ 781 tx_desc = FM10K_TX_DESC(tx_ring, tx_ring->next_to_use); 782 tx_desc->hdrlen = hdrlen; 783 tx_desc->mss = cpu_to_le16(skb_shinfo(skb)->gso_size); 784 785 return 1; 786 787err_vxlan: 788 tx_ring->netdev->features &= ~NETIF_F_GSO_UDP_TUNNEL; 789 if (net_ratelimit()) 790 netdev_err(tx_ring->netdev, 791 "TSO requested for unsupported tunnel, disabling offload\n"); 792 return -1; 793} 794 795static void fm10k_tx_csum(struct fm10k_ring *tx_ring, 796 struct fm10k_tx_buffer *first) 797{ 798 struct sk_buff *skb = first->skb; 799 struct fm10k_tx_desc *tx_desc; 800 union { 801 struct iphdr *ipv4; 802 struct ipv6hdr *ipv6; 803 u8 *raw; 804 } network_hdr; 805 u8 *transport_hdr; 806 __be16 frag_off; 807 __be16 protocol; 808 u8 l4_hdr = 0; 809 810 if (skb->ip_summed != CHECKSUM_PARTIAL) 811 goto no_csum; 812 813 if (skb->encapsulation) { 814 protocol = fm10k_tx_encap_offload(skb); 815 if (!protocol) { 816 if (skb_checksum_help(skb)) { 817 dev_warn(tx_ring->dev, 818 "failed to offload encap csum!\n"); 819 tx_ring->tx_stats.csum_err++; 820 } 821 goto no_csum; 822 } 823 network_hdr.raw = skb_inner_network_header(skb); 824 transport_hdr = skb_inner_transport_header(skb); 825 } else { 826 protocol = vlan_get_protocol(skb); 827 network_hdr.raw = skb_network_header(skb); 828 transport_hdr = skb_transport_header(skb); 829 } 830 831 switch (protocol) { 832 case htons(ETH_P_IP): 833 l4_hdr = network_hdr.ipv4->protocol; 834 break; 835 case htons(ETH_P_IPV6): 836 l4_hdr = network_hdr.ipv6->nexthdr; 837 if (likely((transport_hdr - network_hdr.raw) == 838 sizeof(struct ipv6hdr))) 839 break; 840 ipv6_skip_exthdr(skb, network_hdr.raw - skb->data + 841 sizeof(struct ipv6hdr), 842 &l4_hdr, &frag_off); 843 if (unlikely(frag_off)) 844 l4_hdr = NEXTHDR_FRAGMENT; 845 break; 846 default: 847 break; 848 } 849 850 switch (l4_hdr) { 851 case IPPROTO_TCP: 852 case IPPROTO_UDP: 853 break; 854 case IPPROTO_GRE: 855 if (skb->encapsulation) 856 break; 857 fallthrough; 858 default: 859 if (unlikely(net_ratelimit())) { 860 dev_warn(tx_ring->dev, 861 "partial checksum, version=%d l4 proto=%x\n", 862 protocol, l4_hdr); 863 } 864 skb_checksum_help(skb); 865 tx_ring->tx_stats.csum_err++; 866 goto no_csum; 867 } 868 869 /* update TX checksum flag */ 870 first->tx_flags |= FM10K_TX_FLAGS_CSUM; 871 tx_ring->tx_stats.csum_good++; 872 873no_csum: 874 /* populate Tx descriptor header size and mss */ 875 tx_desc = FM10K_TX_DESC(tx_ring, tx_ring->next_to_use); 876 tx_desc->hdrlen = 0; 877 tx_desc->mss = 0; 878} 879 880#define FM10K_SET_FLAG(_input, _flag, _result) \ 881 ((_flag <= _result) ? \ 882 ((u32)(_input & _flag) * (_result / _flag)) : \ 883 ((u32)(_input & _flag) / (_flag / _result))) 884 885static u8 fm10k_tx_desc_flags(struct sk_buff *skb, u32 tx_flags) 886{ 887 /* set type for advanced descriptor with frame checksum insertion */ 888 u32 desc_flags = 0; 889 890 /* set checksum offload bits */ 891 desc_flags |= FM10K_SET_FLAG(tx_flags, FM10K_TX_FLAGS_CSUM, 892 FM10K_TXD_FLAG_CSUM); 893 894 return desc_flags; 895} 896 897static bool fm10k_tx_desc_push(struct fm10k_ring *tx_ring, 898 struct fm10k_tx_desc *tx_desc, u16 i, 899 dma_addr_t dma, unsigned int size, u8 desc_flags) 900{ 901 /* set RS and INT for last frame in a cache line */ 902 if ((++i & (FM10K_TXD_WB_FIFO_SIZE - 1)) == 0) 903 desc_flags |= FM10K_TXD_FLAG_RS | FM10K_TXD_FLAG_INT; 904 905 /* record values to descriptor */ 906 tx_desc->buffer_addr = cpu_to_le64(dma); 907 tx_desc->flags = desc_flags; 908 tx_desc->buflen = cpu_to_le16(size); 909 910 /* return true if we just wrapped the ring */ 911 return i == tx_ring->count; 912} 913 914static int __fm10k_maybe_stop_tx(struct fm10k_ring *tx_ring, u16 size) 915{ 916 netif_stop_subqueue(tx_ring->netdev, tx_ring->queue_index); 917 918 /* Memory barrier before checking head and tail */ 919 smp_mb(); 920 921 /* Check again in a case another CPU has just made room available */ 922 if (likely(fm10k_desc_unused(tx_ring) < size)) 923 return -EBUSY; 924 925 /* A reprieve! - use start_queue because it doesn't call schedule */ 926 netif_start_subqueue(tx_ring->netdev, tx_ring->queue_index); 927 ++tx_ring->tx_stats.restart_queue; 928 return 0; 929} 930 931static inline int fm10k_maybe_stop_tx(struct fm10k_ring *tx_ring, u16 size) 932{ 933 if (likely(fm10k_desc_unused(tx_ring) >= size)) 934 return 0; 935 return __fm10k_maybe_stop_tx(tx_ring, size); 936} 937 938static void fm10k_tx_map(struct fm10k_ring *tx_ring, 939 struct fm10k_tx_buffer *first) 940{ 941 struct sk_buff *skb = first->skb; 942 struct fm10k_tx_buffer *tx_buffer; 943 struct fm10k_tx_desc *tx_desc; 944 skb_frag_t *frag; 945 unsigned char *data; 946 dma_addr_t dma; 947 unsigned int data_len, size; 948 u32 tx_flags = first->tx_flags; 949 u16 i = tx_ring->next_to_use; 950 u8 flags = fm10k_tx_desc_flags(skb, tx_flags); 951 952 tx_desc = FM10K_TX_DESC(tx_ring, i); 953 954 /* add HW VLAN tag */ 955 if (skb_vlan_tag_present(skb)) 956 tx_desc->vlan = cpu_to_le16(skb_vlan_tag_get(skb)); 957 else 958 tx_desc->vlan = 0; 959 960 size = skb_headlen(skb); 961 data = skb->data; 962 963 dma = dma_map_single(tx_ring->dev, data, size, DMA_TO_DEVICE); 964 965 data_len = skb->data_len; 966 tx_buffer = first; 967 968 for (frag = &skb_shinfo(skb)->frags[0];; frag++) { 969 if (dma_mapping_error(tx_ring->dev, dma)) 970 goto dma_error; 971 972 /* record length, and DMA address */ 973 dma_unmap_len_set(tx_buffer, len, size); 974 dma_unmap_addr_set(tx_buffer, dma, dma); 975 976 while (unlikely(size > FM10K_MAX_DATA_PER_TXD)) { 977 if (fm10k_tx_desc_push(tx_ring, tx_desc++, i++, dma, 978 FM10K_MAX_DATA_PER_TXD, flags)) { 979 tx_desc = FM10K_TX_DESC(tx_ring, 0); 980 i = 0; 981 } 982 983 dma += FM10K_MAX_DATA_PER_TXD; 984 size -= FM10K_MAX_DATA_PER_TXD; 985 } 986 987 if (likely(!data_len)) 988 break; 989 990 if (fm10k_tx_desc_push(tx_ring, tx_desc++, i++, 991 dma, size, flags)) { 992 tx_desc = FM10K_TX_DESC(tx_ring, 0); 993 i = 0; 994 } 995 996 size = skb_frag_size(frag); 997 data_len -= size; 998 999 dma = skb_frag_dma_map(tx_ring->dev, frag, 0, size, 1000 DMA_TO_DEVICE); 1001 1002 tx_buffer = &tx_ring->tx_buffer[i]; 1003 } 1004 1005 /* write last descriptor with LAST bit set */ 1006 flags |= FM10K_TXD_FLAG_LAST; 1007 1008 if (fm10k_tx_desc_push(tx_ring, tx_desc, i++, dma, size, flags)) 1009 i = 0; 1010 1011 /* record bytecount for BQL */ 1012 netdev_tx_sent_queue(txring_txq(tx_ring), first->bytecount); 1013 1014 /* record SW timestamp if HW timestamp is not available */ 1015 skb_tx_timestamp(first->skb); 1016 1017 /* Force memory writes to complete before letting h/w know there 1018 * are new descriptors to fetch. (Only applicable for weak-ordered 1019 * memory model archs, such as IA-64). 1020 * 1021 * We also need this memory barrier to make certain all of the 1022 * status bits have been updated before next_to_watch is written. 1023 */ 1024 wmb(); 1025 1026 /* set next_to_watch value indicating a packet is present */ 1027 first->next_to_watch = tx_desc; 1028 1029 tx_ring->next_to_use = i; 1030 1031 /* Make sure there is space in the ring for the next send. */ 1032 fm10k_maybe_stop_tx(tx_ring, DESC_NEEDED); 1033 1034 /* notify HW of packet */ 1035 if (netif_xmit_stopped(txring_txq(tx_ring)) || !netdev_xmit_more()) { 1036 writel(i, tx_ring->tail); 1037 } 1038 1039 return; 1040dma_error: 1041 dev_err(tx_ring->dev, "TX DMA map failed\n"); 1042 1043 /* clear dma mappings for failed tx_buffer map */ 1044 for (;;) { 1045 tx_buffer = &tx_ring->tx_buffer[i]; 1046 fm10k_unmap_and_free_tx_resource(tx_ring, tx_buffer); 1047 if (tx_buffer == first) 1048 break; 1049 if (i == 0) 1050 i = tx_ring->count; 1051 i--; 1052 } 1053 1054 tx_ring->next_to_use = i; 1055} 1056 1057netdev_tx_t fm10k_xmit_frame_ring(struct sk_buff *skb, 1058 struct fm10k_ring *tx_ring) 1059{ 1060 u16 count = TXD_USE_COUNT(skb_headlen(skb)); 1061 struct fm10k_tx_buffer *first; 1062 unsigned short f; 1063 u32 tx_flags = 0; 1064 int tso; 1065 1066 /* need: 1 descriptor per page * PAGE_SIZE/FM10K_MAX_DATA_PER_TXD, 1067 * + 1 desc for skb_headlen/FM10K_MAX_DATA_PER_TXD, 1068 * + 2 desc gap to keep tail from touching head 1069 * otherwise try next time 1070 */ 1071 for (f = 0; f < skb_shinfo(skb)->nr_frags; f++) { 1072 skb_frag_t *frag = &skb_shinfo(skb)->frags[f]; 1073 1074 count += TXD_USE_COUNT(skb_frag_size(frag)); 1075 } 1076 1077 if (fm10k_maybe_stop_tx(tx_ring, count + 3)) { 1078 tx_ring->tx_stats.tx_busy++; 1079 return NETDEV_TX_BUSY; 1080 } 1081 1082 /* record the location of the first descriptor for this packet */ 1083 first = &tx_ring->tx_buffer[tx_ring->next_to_use]; 1084 first->skb = skb; 1085 first->bytecount = max_t(unsigned int, skb->len, ETH_ZLEN); 1086 first->gso_segs = 1; 1087 1088 /* record initial flags and protocol */ 1089 first->tx_flags = tx_flags; 1090 1091 tso = fm10k_tso(tx_ring, first); 1092 if (tso < 0) 1093 goto out_drop; 1094 else if (!tso) 1095 fm10k_tx_csum(tx_ring, first); 1096 1097 fm10k_tx_map(tx_ring, first); 1098 1099 return NETDEV_TX_OK; 1100 1101out_drop: 1102 dev_kfree_skb_any(first->skb); 1103 first->skb = NULL; 1104 1105 return NETDEV_TX_OK; 1106} 1107 1108static u64 fm10k_get_tx_completed(struct fm10k_ring *ring) 1109{ 1110 return ring->stats.packets; 1111} 1112 1113/** 1114 * fm10k_get_tx_pending - how many Tx descriptors not processed 1115 * @ring: the ring structure 1116 * @in_sw: is tx_pending being checked in SW or in HW? 1117 */ 1118u64 fm10k_get_tx_pending(struct fm10k_ring *ring, bool in_sw) 1119{ 1120 struct fm10k_intfc *interface = ring->q_vector->interface; 1121 struct fm10k_hw *hw = &interface->hw; 1122 u32 head, tail; 1123 1124 if (likely(in_sw)) { 1125 head = ring->next_to_clean; 1126 tail = ring->next_to_use; 1127 } else { 1128 head = fm10k_read_reg(hw, FM10K_TDH(ring->reg_idx)); 1129 tail = fm10k_read_reg(hw, FM10K_TDT(ring->reg_idx)); 1130 } 1131 1132 return ((head <= tail) ? tail : tail + ring->count) - head; 1133} 1134 1135bool fm10k_check_tx_hang(struct fm10k_ring *tx_ring) 1136{ 1137 u32 tx_done = fm10k_get_tx_completed(tx_ring); 1138 u32 tx_done_old = tx_ring->tx_stats.tx_done_old; 1139 u32 tx_pending = fm10k_get_tx_pending(tx_ring, true); 1140 1141 clear_check_for_tx_hang(tx_ring); 1142 1143 /* Check for a hung queue, but be thorough. This verifies 1144 * that a transmit has been completed since the previous 1145 * check AND there is at least one packet pending. By 1146 * requiring this to fail twice we avoid races with 1147 * clearing the ARMED bit and conditions where we 1148 * run the check_tx_hang logic with a transmit completion 1149 * pending but without time to complete it yet. 1150 */ 1151 if (!tx_pending || (tx_done_old != tx_done)) { 1152 /* update completed stats and continue */ 1153 tx_ring->tx_stats.tx_done_old = tx_done; 1154 /* reset the countdown */ 1155 clear_bit(__FM10K_HANG_CHECK_ARMED, tx_ring->state); 1156 1157 return false; 1158 } 1159 1160 /* make sure it is true for two checks in a row */ 1161 return test_and_set_bit(__FM10K_HANG_CHECK_ARMED, tx_ring->state); 1162} 1163 1164/** 1165 * fm10k_tx_timeout_reset - initiate reset due to Tx timeout 1166 * @interface: driver private struct 1167 **/ 1168void fm10k_tx_timeout_reset(struct fm10k_intfc *interface) 1169{ 1170 /* Do the reset outside of interrupt context */ 1171 if (!test_bit(__FM10K_DOWN, interface->state)) { 1172 interface->tx_timeout_count++; 1173 set_bit(FM10K_FLAG_RESET_REQUESTED, interface->flags); 1174 fm10k_service_event_schedule(interface); 1175 } 1176} 1177 1178/** 1179 * fm10k_clean_tx_irq - Reclaim resources after transmit completes 1180 * @q_vector: structure containing interrupt and ring information 1181 * @tx_ring: tx ring to clean 1182 * @napi_budget: Used to determine if we are in netpoll 1183 **/ 1184static bool fm10k_clean_tx_irq(struct fm10k_q_vector *q_vector, 1185 struct fm10k_ring *tx_ring, int napi_budget) 1186{ 1187 struct fm10k_intfc *interface = q_vector->interface; 1188 struct fm10k_tx_buffer *tx_buffer; 1189 struct fm10k_tx_desc *tx_desc; 1190 unsigned int total_bytes = 0, total_packets = 0; 1191 unsigned int budget = q_vector->tx.work_limit; 1192 unsigned int i = tx_ring->next_to_clean; 1193 1194 if (test_bit(__FM10K_DOWN, interface->state)) 1195 return true; 1196 1197 tx_buffer = &tx_ring->tx_buffer[i]; 1198 tx_desc = FM10K_TX_DESC(tx_ring, i); 1199 i -= tx_ring->count; 1200 1201 do { 1202 struct fm10k_tx_desc *eop_desc = tx_buffer->next_to_watch; 1203 1204 /* if next_to_watch is not set then there is no work pending */ 1205 if (!eop_desc) 1206 break; 1207 1208 /* prevent any other reads prior to eop_desc */ 1209 smp_rmb(); 1210 1211 /* if DD is not set pending work has not been completed */ 1212 if (!(eop_desc->flags & FM10K_TXD_FLAG_DONE)) 1213 break; 1214 1215 /* clear next_to_watch to prevent false hangs */ 1216 tx_buffer->next_to_watch = NULL; 1217 1218 /* update the statistics for this packet */ 1219 total_bytes += tx_buffer->bytecount; 1220 total_packets += tx_buffer->gso_segs; 1221 1222 /* free the skb */ 1223 napi_consume_skb(tx_buffer->skb, napi_budget); 1224 1225 /* unmap skb header data */ 1226 dma_unmap_single(tx_ring->dev, 1227 dma_unmap_addr(tx_buffer, dma), 1228 dma_unmap_len(tx_buffer, len), 1229 DMA_TO_DEVICE); 1230 1231 /* clear tx_buffer data */ 1232 tx_buffer->skb = NULL; 1233 dma_unmap_len_set(tx_buffer, len, 0); 1234 1235 /* unmap remaining buffers */ 1236 while (tx_desc != eop_desc) { 1237 tx_buffer++; 1238 tx_desc++; 1239 i++; 1240 if (unlikely(!i)) { 1241 i -= tx_ring->count; 1242 tx_buffer = tx_ring->tx_buffer; 1243 tx_desc = FM10K_TX_DESC(tx_ring, 0); 1244 } 1245 1246 /* unmap any remaining paged data */ 1247 if (dma_unmap_len(tx_buffer, len)) { 1248 dma_unmap_page(tx_ring->dev, 1249 dma_unmap_addr(tx_buffer, dma), 1250 dma_unmap_len(tx_buffer, len), 1251 DMA_TO_DEVICE); 1252 dma_unmap_len_set(tx_buffer, len, 0); 1253 } 1254 } 1255 1256 /* move us one more past the eop_desc for start of next pkt */ 1257 tx_buffer++; 1258 tx_desc++; 1259 i++; 1260 if (unlikely(!i)) { 1261 i -= tx_ring->count; 1262 tx_buffer = tx_ring->tx_buffer; 1263 tx_desc = FM10K_TX_DESC(tx_ring, 0); 1264 } 1265 1266 /* issue prefetch for next Tx descriptor */ 1267 prefetch(tx_desc); 1268 1269 /* update budget accounting */ 1270 budget--; 1271 } while (likely(budget)); 1272 1273 i += tx_ring->count; 1274 tx_ring->next_to_clean = i; 1275 u64_stats_update_begin(&tx_ring->syncp); 1276 tx_ring->stats.bytes += total_bytes; 1277 tx_ring->stats.packets += total_packets; 1278 u64_stats_update_end(&tx_ring->syncp); 1279 q_vector->tx.total_bytes += total_bytes; 1280 q_vector->tx.total_packets += total_packets; 1281 1282 if (check_for_tx_hang(tx_ring) && fm10k_check_tx_hang(tx_ring)) { 1283 /* schedule immediate reset if we believe we hung */ 1284 struct fm10k_hw *hw = &interface->hw; 1285 1286 netif_err(interface, drv, tx_ring->netdev, 1287 "Detected Tx Unit Hang\n" 1288 " Tx Queue <%d>\n" 1289 " TDH, TDT <%x>, <%x>\n" 1290 " next_to_use <%x>\n" 1291 " next_to_clean <%x>\n", 1292 tx_ring->queue_index, 1293 fm10k_read_reg(hw, FM10K_TDH(tx_ring->reg_idx)), 1294 fm10k_read_reg(hw, FM10K_TDT(tx_ring->reg_idx)), 1295 tx_ring->next_to_use, i); 1296 1297 netif_stop_subqueue(tx_ring->netdev, 1298 tx_ring->queue_index); 1299 1300 netif_info(interface, probe, tx_ring->netdev, 1301 "tx hang %d detected on queue %d, resetting interface\n", 1302 interface->tx_timeout_count + 1, 1303 tx_ring->queue_index); 1304 1305 fm10k_tx_timeout_reset(interface); 1306 1307 /* the netdev is about to reset, no point in enabling stuff */ 1308 return true; 1309 } 1310 1311 /* notify netdev of completed buffers */ 1312 netdev_tx_completed_queue(txring_txq(tx_ring), 1313 total_packets, total_bytes); 1314 1315#define TX_WAKE_THRESHOLD min_t(u16, FM10K_MIN_TXD - 1, DESC_NEEDED * 2) 1316 if (unlikely(total_packets && netif_carrier_ok(tx_ring->netdev) && 1317 (fm10k_desc_unused(tx_ring) >= TX_WAKE_THRESHOLD))) { 1318 /* Make sure that anybody stopping the queue after this 1319 * sees the new next_to_clean. 1320 */ 1321 smp_mb(); 1322 if (__netif_subqueue_stopped(tx_ring->netdev, 1323 tx_ring->queue_index) && 1324 !test_bit(__FM10K_DOWN, interface->state)) { 1325 netif_wake_subqueue(tx_ring->netdev, 1326 tx_ring->queue_index); 1327 ++tx_ring->tx_stats.restart_queue; 1328 } 1329 } 1330 1331 return !!budget; 1332} 1333 1334/** 1335 * fm10k_update_itr - update the dynamic ITR value based on packet size 1336 * 1337 * Stores a new ITR value based on strictly on packet size. The 1338 * divisors and thresholds used by this function were determined based 1339 * on theoretical maximum wire speed and testing data, in order to 1340 * minimize response time while increasing bulk throughput. 1341 * 1342 * @ring_container: Container for rings to have ITR updated 1343 **/ 1344static void fm10k_update_itr(struct fm10k_ring_container *ring_container) 1345{ 1346 unsigned int avg_wire_size, packets, itr_round; 1347 1348 /* Only update ITR if we are using adaptive setting */ 1349 if (!ITR_IS_ADAPTIVE(ring_container->itr)) 1350 goto clear_counts; 1351 1352 packets = ring_container->total_packets; 1353 if (!packets) 1354 goto clear_counts; 1355 1356 avg_wire_size = ring_container->total_bytes / packets; 1357 1358 /* The following is a crude approximation of: 1359 * wmem_default / (size + overhead) = desired_pkts_per_int 1360 * rate / bits_per_byte / (size + ethernet overhead) = pkt_rate 1361 * (desired_pkt_rate / pkt_rate) * usecs_per_sec = ITR value 1362 * 1363 * Assuming wmem_default is 212992 and overhead is 640 bytes per 1364 * packet, (256 skb, 64 headroom, 320 shared info), we can reduce the 1365 * formula down to 1366 * 1367 * (34 * (size + 24)) / (size + 640) = ITR 1368 * 1369 * We first do some math on the packet size and then finally bitshift 1370 * by 8 after rounding up. We also have to account for PCIe link speed 1371 * difference as ITR scales based on this. 1372 */ 1373 if (avg_wire_size <= 360) { 1374 /* Start at 250K ints/sec and gradually drop to 77K ints/sec */ 1375 avg_wire_size *= 8; 1376 avg_wire_size += 376; 1377 } else if (avg_wire_size <= 1152) { 1378 /* 77K ints/sec to 45K ints/sec */ 1379 avg_wire_size *= 3; 1380 avg_wire_size += 2176; 1381 } else if (avg_wire_size <= 1920) { 1382 /* 45K ints/sec to 38K ints/sec */ 1383 avg_wire_size += 4480; 1384 } else { 1385 /* plateau at a limit of 38K ints/sec */ 1386 avg_wire_size = 6656; 1387 } 1388 1389 /* Perform final bitshift for division after rounding up to ensure 1390 * that the calculation will never get below a 1. The bit shift 1391 * accounts for changes in the ITR due to PCIe link speed. 1392 */ 1393 itr_round = READ_ONCE(ring_container->itr_scale) + 8; 1394 avg_wire_size += BIT(itr_round) - 1; 1395 avg_wire_size >>= itr_round; 1396 1397 /* write back value and retain adaptive flag */ 1398 ring_container->itr = avg_wire_size | FM10K_ITR_ADAPTIVE; 1399 1400clear_counts: 1401 ring_container->total_bytes = 0; 1402 ring_container->total_packets = 0; 1403} 1404 1405static void fm10k_qv_enable(struct fm10k_q_vector *q_vector) 1406{ 1407 /* Enable auto-mask and clear the current mask */ 1408 u32 itr = FM10K_ITR_ENABLE; 1409 1410 /* Update Tx ITR */ 1411 fm10k_update_itr(&q_vector->tx); 1412 1413 /* Update Rx ITR */ 1414 fm10k_update_itr(&q_vector->rx); 1415 1416 /* Store Tx itr in timer slot 0 */ 1417 itr |= (q_vector->tx.itr & FM10K_ITR_MAX); 1418 1419 /* Shift Rx itr to timer slot 1 */ 1420 itr |= (q_vector->rx.itr & FM10K_ITR_MAX) << FM10K_ITR_INTERVAL1_SHIFT; 1421 1422 /* Write the final value to the ITR register */ 1423 writel(itr, q_vector->itr); 1424} 1425 1426static int fm10k_poll(struct napi_struct *napi, int budget) 1427{ 1428 struct fm10k_q_vector *q_vector = 1429 container_of(napi, struct fm10k_q_vector, napi); 1430 struct fm10k_ring *ring; 1431 int per_ring_budget, work_done = 0; 1432 bool clean_complete = true; 1433 1434 fm10k_for_each_ring(ring, q_vector->tx) { 1435 if (!fm10k_clean_tx_irq(q_vector, ring, budget)) 1436 clean_complete = false; 1437 } 1438 1439 /* Handle case where we are called by netpoll with a budget of 0 */ 1440 if (budget <= 0) 1441 return budget; 1442 1443 /* attempt to distribute budget to each queue fairly, but don't 1444 * allow the budget to go below 1 because we'll exit polling 1445 */ 1446 if (q_vector->rx.count > 1) 1447 per_ring_budget = max(budget / q_vector->rx.count, 1); 1448 else 1449 per_ring_budget = budget; 1450 1451 fm10k_for_each_ring(ring, q_vector->rx) { 1452 int work = fm10k_clean_rx_irq(q_vector, ring, per_ring_budget); 1453 1454 work_done += work; 1455 if (work >= per_ring_budget) 1456 clean_complete = false; 1457 } 1458 1459 /* If all work not completed, return budget and keep polling */ 1460 if (!clean_complete) 1461 return budget; 1462 1463 /* Exit the polling mode, but don't re-enable interrupts if stack might 1464 * poll us due to busy-polling 1465 */ 1466 if (likely(napi_complete_done(napi, work_done))) 1467 fm10k_qv_enable(q_vector); 1468 1469 return min(work_done, budget - 1); 1470} 1471 1472/** 1473 * fm10k_set_qos_queues: Allocate queues for a QOS-enabled device 1474 * @interface: board private structure to initialize 1475 * 1476 * When QoS (Quality of Service) is enabled, allocate queues for 1477 * each traffic class. If multiqueue isn't available,then abort QoS 1478 * initialization. 1479 * 1480 * This function handles all combinations of Qos and RSS. 1481 * 1482 **/ 1483static bool fm10k_set_qos_queues(struct fm10k_intfc *interface) 1484{ 1485 struct net_device *dev = interface->netdev; 1486 struct fm10k_ring_feature *f; 1487 int rss_i, i; 1488 int pcs; 1489 1490 /* Map queue offset and counts onto allocated tx queues */ 1491 pcs = netdev_get_num_tc(dev); 1492 1493 if (pcs <= 1) 1494 return false; 1495 1496 /* set QoS mask and indices */ 1497 f = &interface->ring_feature[RING_F_QOS]; 1498 f->indices = pcs; 1499 f->mask = BIT(fls(pcs - 1)) - 1; 1500 1501 /* determine the upper limit for our current DCB mode */ 1502 rss_i = interface->hw.mac.max_queues / pcs; 1503 rss_i = BIT(fls(rss_i) - 1); 1504 1505 /* set RSS mask and indices */ 1506 f = &interface->ring_feature[RING_F_RSS]; 1507 rss_i = min_t(u16, rss_i, f->limit); 1508 f->indices = rss_i; 1509 f->mask = BIT(fls(rss_i - 1)) - 1; 1510 1511 /* configure pause class to queue mapping */ 1512 for (i = 0; i < pcs; i++) 1513 netdev_set_tc_queue(dev, i, rss_i, rss_i * i); 1514 1515 interface->num_rx_queues = rss_i * pcs; 1516 interface->num_tx_queues = rss_i * pcs; 1517 1518 return true; 1519} 1520 1521/** 1522 * fm10k_set_rss_queues: Allocate queues for RSS 1523 * @interface: board private structure to initialize 1524 * 1525 * This is our "base" multiqueue mode. RSS (Receive Side Scaling) will try 1526 * to allocate one Rx queue per CPU, and if available, one Tx queue per CPU. 1527 * 1528 **/ 1529static bool fm10k_set_rss_queues(struct fm10k_intfc *interface) 1530{ 1531 struct fm10k_ring_feature *f; 1532 u16 rss_i; 1533 1534 f = &interface->ring_feature[RING_F_RSS]; 1535 rss_i = min_t(u16, interface->hw.mac.max_queues, f->limit); 1536 1537 /* record indices and power of 2 mask for RSS */ 1538 f->indices = rss_i; 1539 f->mask = BIT(fls(rss_i - 1)) - 1; 1540 1541 interface->num_rx_queues = rss_i; 1542 interface->num_tx_queues = rss_i; 1543 1544 return true; 1545} 1546 1547/** 1548 * fm10k_set_num_queues: Allocate queues for device, feature dependent 1549 * @interface: board private structure to initialize 1550 * 1551 * This is the top level queue allocation routine. The order here is very 1552 * important, starting with the "most" number of features turned on at once, 1553 * and ending with the smallest set of features. This way large combinations 1554 * can be allocated if they're turned on, and smaller combinations are the 1555 * fall through conditions. 1556 * 1557 **/ 1558static void fm10k_set_num_queues(struct fm10k_intfc *interface) 1559{ 1560 /* Attempt to setup QoS and RSS first */ 1561 if (fm10k_set_qos_queues(interface)) 1562 return; 1563 1564 /* If we don't have QoS, just fallback to only RSS. */ 1565 fm10k_set_rss_queues(interface); 1566} 1567 1568/** 1569 * fm10k_reset_num_queues - Reset the number of queues to zero 1570 * @interface: board private structure 1571 * 1572 * This function should be called whenever we need to reset the number of 1573 * queues after an error condition. 1574 */ 1575static void fm10k_reset_num_queues(struct fm10k_intfc *interface) 1576{ 1577 interface->num_tx_queues = 0; 1578 interface->num_rx_queues = 0; 1579 interface->num_q_vectors = 0; 1580} 1581 1582/** 1583 * fm10k_alloc_q_vector - Allocate memory for a single interrupt vector 1584 * @interface: board private structure to initialize 1585 * @v_count: q_vectors allocated on interface, used for ring interleaving 1586 * @v_idx: index of vector in interface struct 1587 * @txr_count: total number of Tx rings to allocate 1588 * @txr_idx: index of first Tx ring to allocate 1589 * @rxr_count: total number of Rx rings to allocate 1590 * @rxr_idx: index of first Rx ring to allocate 1591 * 1592 * We allocate one q_vector. If allocation fails we return -ENOMEM. 1593 **/ 1594static int fm10k_alloc_q_vector(struct fm10k_intfc *interface, 1595 unsigned int v_count, unsigned int v_idx, 1596 unsigned int txr_count, unsigned int txr_idx, 1597 unsigned int rxr_count, unsigned int rxr_idx) 1598{ 1599 struct fm10k_q_vector *q_vector; 1600 struct fm10k_ring *ring; 1601 int ring_count; 1602 1603 ring_count = txr_count + rxr_count; 1604 1605 /* allocate q_vector and rings */ 1606 q_vector = kzalloc(struct_size(q_vector, ring, ring_count), GFP_KERNEL); 1607 if (!q_vector) 1608 return -ENOMEM; 1609 1610 /* initialize NAPI */ 1611 netif_napi_add(interface->netdev, &q_vector->napi, 1612 fm10k_poll, NAPI_POLL_WEIGHT); 1613 1614 /* tie q_vector and interface together */ 1615 interface->q_vector[v_idx] = q_vector; 1616 q_vector->interface = interface; 1617 q_vector->v_idx = v_idx; 1618 1619 /* initialize pointer to rings */ 1620 ring = q_vector->ring; 1621 1622 /* save Tx ring container info */ 1623 q_vector->tx.ring = ring; 1624 q_vector->tx.work_limit = FM10K_DEFAULT_TX_WORK; 1625 q_vector->tx.itr = interface->tx_itr; 1626 q_vector->tx.itr_scale = interface->hw.mac.itr_scale; 1627 q_vector->tx.count = txr_count; 1628 1629 while (txr_count) { 1630 /* assign generic ring traits */ 1631 ring->dev = &interface->pdev->dev; 1632 ring->netdev = interface->netdev; 1633 1634 /* configure backlink on ring */ 1635 ring->q_vector = q_vector; 1636 1637 /* apply Tx specific ring traits */ 1638 ring->count = interface->tx_ring_count; 1639 ring->queue_index = txr_idx; 1640 1641 /* assign ring to interface */ 1642 interface->tx_ring[txr_idx] = ring; 1643 1644 /* update count and index */ 1645 txr_count--; 1646 txr_idx += v_count; 1647 1648 /* push pointer to next ring */ 1649 ring++; 1650 } 1651 1652 /* save Rx ring container info */ 1653 q_vector->rx.ring = ring; 1654 q_vector->rx.itr = interface->rx_itr; 1655 q_vector->rx.itr_scale = interface->hw.mac.itr_scale; 1656 q_vector->rx.count = rxr_count; 1657 1658 while (rxr_count) { 1659 /* assign generic ring traits */ 1660 ring->dev = &interface->pdev->dev; 1661 ring->netdev = interface->netdev; 1662 rcu_assign_pointer(ring->l2_accel, interface->l2_accel); 1663 1664 /* configure backlink on ring */ 1665 ring->q_vector = q_vector; 1666 1667 /* apply Rx specific ring traits */ 1668 ring->count = interface->rx_ring_count; 1669 ring->queue_index = rxr_idx; 1670 1671 /* assign ring to interface */ 1672 interface->rx_ring[rxr_idx] = ring; 1673 1674 /* update count and index */ 1675 rxr_count--; 1676 rxr_idx += v_count; 1677 1678 /* push pointer to next ring */ 1679 ring++; 1680 } 1681 1682 fm10k_dbg_q_vector_init(q_vector); 1683 1684 return 0; 1685} 1686 1687/** 1688 * fm10k_free_q_vector - Free memory allocated for specific interrupt vector 1689 * @interface: board private structure to initialize 1690 * @v_idx: Index of vector to be freed 1691 * 1692 * This function frees the memory allocated to the q_vector. In addition if 1693 * NAPI is enabled it will delete any references to the NAPI struct prior 1694 * to freeing the q_vector. 1695 **/ 1696static void fm10k_free_q_vector(struct fm10k_intfc *interface, int v_idx) 1697{ 1698 struct fm10k_q_vector *q_vector = interface->q_vector[v_idx]; 1699 struct fm10k_ring *ring; 1700 1701 fm10k_dbg_q_vector_exit(q_vector); 1702 1703 fm10k_for_each_ring(ring, q_vector->tx) 1704 interface->tx_ring[ring->queue_index] = NULL; 1705 1706 fm10k_for_each_ring(ring, q_vector->rx) 1707 interface->rx_ring[ring->queue_index] = NULL; 1708 1709 interface->q_vector[v_idx] = NULL; 1710 netif_napi_del(&q_vector->napi); 1711 kfree_rcu(q_vector, rcu); 1712} 1713 1714/** 1715 * fm10k_alloc_q_vectors - Allocate memory for interrupt vectors 1716 * @interface: board private structure to initialize 1717 * 1718 * We allocate one q_vector per queue interrupt. If allocation fails we 1719 * return -ENOMEM. 1720 **/ 1721static int fm10k_alloc_q_vectors(struct fm10k_intfc *interface) 1722{ 1723 unsigned int q_vectors = interface->num_q_vectors; 1724 unsigned int rxr_remaining = interface->num_rx_queues; 1725 unsigned int txr_remaining = interface->num_tx_queues; 1726 unsigned int rxr_idx = 0, txr_idx = 0, v_idx = 0; 1727 int err; 1728 1729 if (q_vectors >= (rxr_remaining + txr_remaining)) { 1730 for (; rxr_remaining; v_idx++) { 1731 err = fm10k_alloc_q_vector(interface, q_vectors, v_idx, 1732 0, 0, 1, rxr_idx); 1733 if (err) 1734 goto err_out; 1735 1736 /* update counts and index */ 1737 rxr_remaining--; 1738 rxr_idx++; 1739 } 1740 } 1741 1742 for (; v_idx < q_vectors; v_idx++) { 1743 int rqpv = DIV_ROUND_UP(rxr_remaining, q_vectors - v_idx); 1744 int tqpv = DIV_ROUND_UP(txr_remaining, q_vectors - v_idx); 1745 1746 err = fm10k_alloc_q_vector(interface, q_vectors, v_idx, 1747 tqpv, txr_idx, 1748 rqpv, rxr_idx); 1749 1750 if (err) 1751 goto err_out; 1752 1753 /* update counts and index */ 1754 rxr_remaining -= rqpv; 1755 txr_remaining -= tqpv; 1756 rxr_idx++; 1757 txr_idx++; 1758 } 1759 1760 return 0; 1761 1762err_out: 1763 fm10k_reset_num_queues(interface); 1764 1765 while (v_idx--) 1766 fm10k_free_q_vector(interface, v_idx); 1767 1768 return -ENOMEM; 1769} 1770 1771/** 1772 * fm10k_free_q_vectors - Free memory allocated for interrupt vectors 1773 * @interface: board private structure to initialize 1774 * 1775 * This function frees the memory allocated to the q_vectors. In addition if 1776 * NAPI is enabled it will delete any references to the NAPI struct prior 1777 * to freeing the q_vector. 1778 **/ 1779static void fm10k_free_q_vectors(struct fm10k_intfc *interface) 1780{ 1781 int v_idx = interface->num_q_vectors; 1782 1783 fm10k_reset_num_queues(interface); 1784 1785 while (v_idx--) 1786 fm10k_free_q_vector(interface, v_idx); 1787} 1788 1789/** 1790 * f10k_reset_msix_capability - reset MSI-X capability 1791 * @interface: board private structure to initialize 1792 * 1793 * Reset the MSI-X capability back to its starting state 1794 **/ 1795static void fm10k_reset_msix_capability(struct fm10k_intfc *interface) 1796{ 1797 pci_disable_msix(interface->pdev); 1798 kfree(interface->msix_entries); 1799 interface->msix_entries = NULL; 1800} 1801 1802/** 1803 * f10k_init_msix_capability - configure MSI-X capability 1804 * @interface: board private structure to initialize 1805 * 1806 * Attempt to configure the interrupts using the best available 1807 * capabilities of the hardware and the kernel. 1808 **/ 1809static int fm10k_init_msix_capability(struct fm10k_intfc *interface) 1810{ 1811 struct fm10k_hw *hw = &interface->hw; 1812 int v_budget, vector; 1813 1814 /* It's easy to be greedy for MSI-X vectors, but it really 1815 * doesn't do us much good if we have a lot more vectors 1816 * than CPU's. So let's be conservative and only ask for 1817 * (roughly) the same number of vectors as there are CPU's. 1818 * the default is to use pairs of vectors 1819 */ 1820 v_budget = max(interface->num_rx_queues, interface->num_tx_queues); 1821 v_budget = min_t(u16, v_budget, num_online_cpus()); 1822 1823 /* account for vectors not related to queues */ 1824 v_budget += NON_Q_VECTORS; 1825 1826 /* At the same time, hardware can only support a maximum of 1827 * hw.mac->max_msix_vectors vectors. With features 1828 * such as RSS and VMDq, we can easily surpass the number of Rx and Tx 1829 * descriptor queues supported by our device. Thus, we cap it off in 1830 * those rare cases where the cpu count also exceeds our vector limit. 1831 */ 1832 v_budget = min_t(int, v_budget, hw->mac.max_msix_vectors); 1833 1834 /* A failure in MSI-X entry allocation is fatal. */ 1835 interface->msix_entries = kcalloc(v_budget, sizeof(struct msix_entry), 1836 GFP_KERNEL); 1837 if (!interface->msix_entries) 1838 return -ENOMEM; 1839 1840 /* populate entry values */ 1841 for (vector = 0; vector < v_budget; vector++) 1842 interface->msix_entries[vector].entry = vector; 1843 1844 /* Attempt to enable MSI-X with requested value */ 1845 v_budget = pci_enable_msix_range(interface->pdev, 1846 interface->msix_entries, 1847 MIN_MSIX_COUNT(hw), 1848 v_budget); 1849 if (v_budget < 0) { 1850 kfree(interface->msix_entries); 1851 interface->msix_entries = NULL; 1852 return v_budget; 1853 } 1854 1855 /* record the number of queues available for q_vectors */ 1856 interface->num_q_vectors = v_budget - NON_Q_VECTORS; 1857 1858 return 0; 1859} 1860 1861/** 1862 * fm10k_cache_ring_qos - Descriptor ring to register mapping for QoS 1863 * @interface: Interface structure continaining rings and devices 1864 * 1865 * Cache the descriptor ring offsets for Qos 1866 **/ 1867static bool fm10k_cache_ring_qos(struct fm10k_intfc *interface) 1868{ 1869 struct net_device *dev = interface->netdev; 1870 int pc, offset, rss_i, i; 1871 u16 pc_stride = interface->ring_feature[RING_F_QOS].mask + 1; 1872 u8 num_pcs = netdev_get_num_tc(dev); 1873 1874 if (num_pcs <= 1) 1875 return false; 1876 1877 rss_i = interface->ring_feature[RING_F_RSS].indices; 1878 1879 for (pc = 0, offset = 0; pc < num_pcs; pc++, offset += rss_i) { 1880 int q_idx = pc; 1881 1882 for (i = 0; i < rss_i; i++) { 1883 interface->tx_ring[offset + i]->reg_idx = q_idx; 1884 interface->tx_ring[offset + i]->qos_pc = pc; 1885 interface->rx_ring[offset + i]->reg_idx = q_idx; 1886 interface->rx_ring[offset + i]->qos_pc = pc; 1887 q_idx += pc_stride; 1888 } 1889 } 1890 1891 return true; 1892} 1893 1894/** 1895 * fm10k_cache_ring_rss - Descriptor ring to register mapping for RSS 1896 * @interface: Interface structure continaining rings and devices 1897 * 1898 * Cache the descriptor ring offsets for RSS 1899 **/ 1900static void fm10k_cache_ring_rss(struct fm10k_intfc *interface) 1901{ 1902 int i; 1903 1904 for (i = 0; i < interface->num_rx_queues; i++) 1905 interface->rx_ring[i]->reg_idx = i; 1906 1907 for (i = 0; i < interface->num_tx_queues; i++) 1908 interface->tx_ring[i]->reg_idx = i; 1909} 1910 1911/** 1912 * fm10k_assign_rings - Map rings to network devices 1913 * @interface: Interface structure containing rings and devices 1914 * 1915 * This function is meant to go though and configure both the network 1916 * devices so that they contain rings, and configure the rings so that 1917 * they function with their network devices. 1918 **/ 1919static void fm10k_assign_rings(struct fm10k_intfc *interface) 1920{ 1921 if (fm10k_cache_ring_qos(interface)) 1922 return; 1923 1924 fm10k_cache_ring_rss(interface); 1925} 1926 1927static void fm10k_init_reta(struct fm10k_intfc *interface) 1928{ 1929 u16 i, rss_i = interface->ring_feature[RING_F_RSS].indices; 1930 u32 reta; 1931 1932 /* If the Rx flow indirection table has been configured manually, we 1933 * need to maintain it when possible. 1934 */ 1935 if (netif_is_rxfh_configured(interface->netdev)) { 1936 for (i = FM10K_RETA_SIZE; i--;) { 1937 reta = interface->reta[i]; 1938 if ((((reta << 24) >> 24) < rss_i) && 1939 (((reta << 16) >> 24) < rss_i) && 1940 (((reta << 8) >> 24) < rss_i) && 1941 (((reta) >> 24) < rss_i)) 1942 continue; 1943 1944 /* this should never happen */ 1945 dev_err(&interface->pdev->dev, 1946 "RSS indirection table assigned flows out of queue bounds. Reconfiguring.\n"); 1947 goto repopulate_reta; 1948 } 1949 1950 /* do nothing if all of the elements are in bounds */ 1951 return; 1952 } 1953 1954repopulate_reta: 1955 fm10k_write_reta(interface, NULL); 1956} 1957 1958/** 1959 * fm10k_init_queueing_scheme - Determine proper queueing scheme 1960 * @interface: board private structure to initialize 1961 * 1962 * We determine which queueing scheme to use based on... 1963 * - Hardware queue count (num_*_queues) 1964 * - defined by miscellaneous hardware support/features (RSS, etc.) 1965 **/ 1966int fm10k_init_queueing_scheme(struct fm10k_intfc *interface) 1967{ 1968 int err; 1969 1970 /* Number of supported queues */ 1971 fm10k_set_num_queues(interface); 1972 1973 /* Configure MSI-X capability */ 1974 err = fm10k_init_msix_capability(interface); 1975 if (err) { 1976 dev_err(&interface->pdev->dev, 1977 "Unable to initialize MSI-X capability\n"); 1978 goto err_init_msix; 1979 } 1980 1981 /* Allocate memory for queues */ 1982 err = fm10k_alloc_q_vectors(interface); 1983 if (err) { 1984 dev_err(&interface->pdev->dev, 1985 "Unable to allocate queue vectors\n"); 1986 goto err_alloc_q_vectors; 1987 } 1988 1989 /* Map rings to devices, and map devices to physical queues */ 1990 fm10k_assign_rings(interface); 1991 1992 /* Initialize RSS redirection table */ 1993 fm10k_init_reta(interface); 1994 1995 return 0; 1996 1997err_alloc_q_vectors: 1998 fm10k_reset_msix_capability(interface); 1999err_init_msix: 2000 fm10k_reset_num_queues(interface); 2001 return err; 2002} 2003 2004/** 2005 * fm10k_clear_queueing_scheme - Clear the current queueing scheme settings 2006 * @interface: board private structure to clear queueing scheme on 2007 * 2008 * We go through and clear queueing specific resources and reset the structure 2009 * to pre-load conditions 2010 **/ 2011void fm10k_clear_queueing_scheme(struct fm10k_intfc *interface) 2012{ 2013 fm10k_free_q_vectors(interface); 2014 fm10k_reset_msix_capability(interface); 2015} 2016