1/************************************************************************* 2 * myri10ge.c: Myricom Myri-10G Ethernet driver. 3 * 4 * Copyright (C) 2005 - 2011 Myricom, Inc. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 3. Neither the name of Myricom, Inc. nor the names of its contributors 16 * may be used to endorse or promote products derived from this software 17 * without specific prior written permission. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 20 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 22 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 23 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 * 31 * 32 * If the eeprom on your board is not recent enough, you will need to get a 33 * newer firmware image at: 34 * http://www.myri.com/scs/download-Myri10GE.html 35 * 36 * Contact Information: 37 * <help@myri.com> 38 * Myricom, Inc., 325N Santa Anita Avenue, Arcadia, CA 91006 39 *************************************************************************/ 40 41#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 42 43#include <linux/tcp.h> 44#include <linux/netdevice.h> 45#include <linux/skbuff.h> 46#include <linux/string.h> 47#include <linux/module.h> 48#include <linux/pci.h> 49#include <linux/dma-mapping.h> 50#include <linux/etherdevice.h> 51#include <linux/if_ether.h> 52#include <linux/if_vlan.h> 53#include <linux/dca.h> 54#include <linux/ip.h> 55#include <linux/inet.h> 56#include <linux/in.h> 57#include <linux/ethtool.h> 58#include <linux/firmware.h> 59#include <linux/delay.h> 60#include <linux/timer.h> 61#include <linux/vmalloc.h> 62#include <linux/crc32.h> 63#include <linux/moduleparam.h> 64#include <linux/io.h> 65#include <linux/log2.h> 66#include <linux/slab.h> 67#include <linux/prefetch.h> 68#include <net/checksum.h> 69#include <net/ip.h> 70#include <net/tcp.h> 71#include <asm/byteorder.h> 72#include <asm/processor.h> 73 74#include "myri10ge_mcp.h" 75#include "myri10ge_mcp_gen_header.h" 76 77#define MYRI10GE_VERSION_STR "1.5.3-1.534" 78 79MODULE_DESCRIPTION("Myricom 10G driver (10GbE)"); 80MODULE_AUTHOR("Maintainer: help@myri.com"); 81MODULE_VERSION(MYRI10GE_VERSION_STR); 82MODULE_LICENSE("Dual BSD/GPL"); 83 84#define MYRI10GE_MAX_ETHER_MTU 9014 85 86#define MYRI10GE_ETH_STOPPED 0 87#define MYRI10GE_ETH_STOPPING 1 88#define MYRI10GE_ETH_STARTING 2 89#define MYRI10GE_ETH_RUNNING 3 90#define MYRI10GE_ETH_OPEN_FAILED 4 91 92#define MYRI10GE_EEPROM_STRINGS_SIZE 256 93#define MYRI10GE_MAX_SEND_DESC_TSO ((65536 / 2048) * 2) 94 95#define MYRI10GE_NO_CONFIRM_DATA htonl(0xffffffff) 96#define MYRI10GE_NO_RESPONSE_RESULT 0xffffffff 97 98#define MYRI10GE_ALLOC_ORDER 0 99#define MYRI10GE_ALLOC_SIZE ((1 << MYRI10GE_ALLOC_ORDER) * PAGE_SIZE) 100#define MYRI10GE_MAX_FRAGS_PER_FRAME (MYRI10GE_MAX_ETHER_MTU/MYRI10GE_ALLOC_SIZE + 1) 101 102#define MYRI10GE_MAX_SLICES 32 103 104struct myri10ge_rx_buffer_state { 105 struct page *page; 106 int page_offset; 107 DEFINE_DMA_UNMAP_ADDR(bus); 108 DEFINE_DMA_UNMAP_LEN(len); 109}; 110 111struct myri10ge_tx_buffer_state { 112 struct sk_buff *skb; 113 int last; 114 DEFINE_DMA_UNMAP_ADDR(bus); 115 DEFINE_DMA_UNMAP_LEN(len); 116}; 117 118struct myri10ge_cmd { 119 u32 data0; 120 u32 data1; 121 u32 data2; 122}; 123 124struct myri10ge_rx_buf { 125 struct mcp_kreq_ether_recv __iomem *lanai; /* lanai ptr for recv ring */ 126 struct mcp_kreq_ether_recv *shadow; /* host shadow of recv ring */ 127 struct myri10ge_rx_buffer_state *info; 128 struct page *page; 129 dma_addr_t bus; 130 int page_offset; 131 int cnt; 132 int fill_cnt; 133 int alloc_fail; 134 int mask; /* number of rx slots -1 */ 135 int watchdog_needed; 136}; 137 138struct myri10ge_tx_buf { 139 struct mcp_kreq_ether_send __iomem *lanai; /* lanai ptr for sendq */ 140 __be32 __iomem *send_go; /* "go" doorbell ptr */ 141 __be32 __iomem *send_stop; /* "stop" doorbell ptr */ 142 struct mcp_kreq_ether_send *req_list; /* host shadow of sendq */ 143 char *req_bytes; 144 struct myri10ge_tx_buffer_state *info; 145 int mask; /* number of transmit slots -1 */ 146 int req ____cacheline_aligned; /* transmit slots submitted */ 147 int pkt_start; /* packets started */ 148 int stop_queue; 149 int linearized; 150 int done ____cacheline_aligned; /* transmit slots completed */ 151 int pkt_done; /* packets completed */ 152 int wake_queue; 153 int queue_active; 154}; 155 156struct myri10ge_rx_done { 157 struct mcp_slot *entry; 158 dma_addr_t bus; 159 int cnt; 160 int idx; 161}; 162 163struct myri10ge_slice_netstats { 164 unsigned long rx_packets; 165 unsigned long tx_packets; 166 unsigned long rx_bytes; 167 unsigned long tx_bytes; 168 unsigned long rx_dropped; 169 unsigned long tx_dropped; 170}; 171 172struct myri10ge_slice_state { 173 struct myri10ge_tx_buf tx; /* transmit ring */ 174 struct myri10ge_rx_buf rx_small; 175 struct myri10ge_rx_buf rx_big; 176 struct myri10ge_rx_done rx_done; 177 struct net_device *dev; 178 struct napi_struct napi; 179 struct myri10ge_priv *mgp; 180 struct myri10ge_slice_netstats stats; 181 __be32 __iomem *irq_claim; 182 struct mcp_irq_data *fw_stats; 183 dma_addr_t fw_stats_bus; 184 int watchdog_tx_done; 185 int watchdog_tx_req; 186 int watchdog_rx_done; 187 int stuck; 188#ifdef CONFIG_MYRI10GE_DCA 189 int cached_dca_tag; 190 int cpu; 191 __be32 __iomem *dca_tag; 192#endif 193 char irq_desc[32]; 194}; 195 196struct myri10ge_priv { 197 struct myri10ge_slice_state *ss; 198 int tx_boundary; /* boundary transmits cannot cross */ 199 int num_slices; 200 int running; /* running? */ 201 int small_bytes; 202 int big_bytes; 203 int max_intr_slots; 204 struct net_device *dev; 205 u8 __iomem *sram; 206 int sram_size; 207 unsigned long board_span; 208 unsigned long iomem_base; 209 __be32 __iomem *irq_deassert; 210 char *mac_addr_string; 211 struct mcp_cmd_response *cmd; 212 dma_addr_t cmd_bus; 213 struct pci_dev *pdev; 214 int msi_enabled; 215 int msix_enabled; 216 struct msix_entry *msix_vectors; 217#ifdef CONFIG_MYRI10GE_DCA 218 int dca_enabled; 219 int relaxed_order; 220#endif 221 u32 link_state; 222 unsigned int rdma_tags_available; 223 int intr_coal_delay; 224 __be32 __iomem *intr_coal_delay_ptr; 225 int wc_cookie; 226 int down_cnt; 227 wait_queue_head_t down_wq; 228 struct work_struct watchdog_work; 229 struct timer_list watchdog_timer; 230 int watchdog_resets; 231 int watchdog_pause; 232 int pause; 233 bool fw_name_allocated; 234 char *fw_name; 235 char eeprom_strings[MYRI10GE_EEPROM_STRINGS_SIZE]; 236 char *product_code_string; 237 char fw_version[128]; 238 int fw_ver_major; 239 int fw_ver_minor; 240 int fw_ver_tiny; 241 int adopted_rx_filter_bug; 242 u8 mac_addr[ETH_ALEN]; /* eeprom mac address */ 243 unsigned long serial_number; 244 int vendor_specific_offset; 245 int fw_multicast_support; 246 u32 features; 247 u32 max_tso6; 248 u32 read_dma; 249 u32 write_dma; 250 u32 read_write_dma; 251 u32 link_changes; 252 u32 msg_enable; 253 unsigned int board_number; 254 int rebooted; 255}; 256 257static char *myri10ge_fw_unaligned = "myri10ge_ethp_z8e.dat"; 258static char *myri10ge_fw_aligned = "myri10ge_eth_z8e.dat"; 259static char *myri10ge_fw_rss_unaligned = "myri10ge_rss_ethp_z8e.dat"; 260static char *myri10ge_fw_rss_aligned = "myri10ge_rss_eth_z8e.dat"; 261MODULE_FIRMWARE("myri10ge_ethp_z8e.dat"); 262MODULE_FIRMWARE("myri10ge_eth_z8e.dat"); 263MODULE_FIRMWARE("myri10ge_rss_ethp_z8e.dat"); 264MODULE_FIRMWARE("myri10ge_rss_eth_z8e.dat"); 265 266/* Careful: must be accessed under kernel_param_lock() */ 267static char *myri10ge_fw_name = NULL; 268module_param(myri10ge_fw_name, charp, 0644); 269MODULE_PARM_DESC(myri10ge_fw_name, "Firmware image name"); 270 271#define MYRI10GE_MAX_BOARDS 8 272static char *myri10ge_fw_names[MYRI10GE_MAX_BOARDS] = 273 {[0 ... (MYRI10GE_MAX_BOARDS - 1)] = NULL }; 274module_param_array_named(myri10ge_fw_names, myri10ge_fw_names, charp, NULL, 275 0444); 276MODULE_PARM_DESC(myri10ge_fw_names, "Firmware image names per board"); 277 278static int myri10ge_ecrc_enable = 1; 279module_param(myri10ge_ecrc_enable, int, 0444); 280MODULE_PARM_DESC(myri10ge_ecrc_enable, "Enable Extended CRC on PCI-E"); 281 282static int myri10ge_small_bytes = -1; /* -1 == auto */ 283module_param(myri10ge_small_bytes, int, 0644); 284MODULE_PARM_DESC(myri10ge_small_bytes, "Threshold of small packets"); 285 286static int myri10ge_msi = 1; /* enable msi by default */ 287module_param(myri10ge_msi, int, 0644); 288MODULE_PARM_DESC(myri10ge_msi, "Enable Message Signalled Interrupts"); 289 290static int myri10ge_intr_coal_delay = 75; 291module_param(myri10ge_intr_coal_delay, int, 0444); 292MODULE_PARM_DESC(myri10ge_intr_coal_delay, "Interrupt coalescing delay"); 293 294static int myri10ge_flow_control = 1; 295module_param(myri10ge_flow_control, int, 0444); 296MODULE_PARM_DESC(myri10ge_flow_control, "Pause parameter"); 297 298static int myri10ge_deassert_wait = 1; 299module_param(myri10ge_deassert_wait, int, 0644); 300MODULE_PARM_DESC(myri10ge_deassert_wait, 301 "Wait when deasserting legacy interrupts"); 302 303static int myri10ge_force_firmware = 0; 304module_param(myri10ge_force_firmware, int, 0444); 305MODULE_PARM_DESC(myri10ge_force_firmware, 306 "Force firmware to assume aligned completions"); 307 308static int myri10ge_initial_mtu = MYRI10GE_MAX_ETHER_MTU - ETH_HLEN; 309module_param(myri10ge_initial_mtu, int, 0444); 310MODULE_PARM_DESC(myri10ge_initial_mtu, "Initial MTU"); 311 312static int myri10ge_napi_weight = 64; 313module_param(myri10ge_napi_weight, int, 0444); 314MODULE_PARM_DESC(myri10ge_napi_weight, "Set NAPI weight"); 315 316static int myri10ge_watchdog_timeout = 1; 317module_param(myri10ge_watchdog_timeout, int, 0444); 318MODULE_PARM_DESC(myri10ge_watchdog_timeout, "Set watchdog timeout"); 319 320static int myri10ge_max_irq_loops = 1048576; 321module_param(myri10ge_max_irq_loops, int, 0444); 322MODULE_PARM_DESC(myri10ge_max_irq_loops, 323 "Set stuck legacy IRQ detection threshold"); 324 325#define MYRI10GE_MSG_DEFAULT NETIF_MSG_LINK 326 327static int myri10ge_debug = -1; /* defaults above */ 328module_param(myri10ge_debug, int, 0); 329MODULE_PARM_DESC(myri10ge_debug, "Debug level (0=none,...,16=all)"); 330 331static int myri10ge_fill_thresh = 256; 332module_param(myri10ge_fill_thresh, int, 0644); 333MODULE_PARM_DESC(myri10ge_fill_thresh, "Number of empty rx slots allowed"); 334 335static int myri10ge_reset_recover = 1; 336 337static int myri10ge_max_slices = 1; 338module_param(myri10ge_max_slices, int, 0444); 339MODULE_PARM_DESC(myri10ge_max_slices, "Max tx/rx queues"); 340 341static int myri10ge_rss_hash = MXGEFW_RSS_HASH_TYPE_SRC_DST_PORT; 342module_param(myri10ge_rss_hash, int, 0444); 343MODULE_PARM_DESC(myri10ge_rss_hash, "Type of RSS hashing to do"); 344 345static int myri10ge_dca = 1; 346module_param(myri10ge_dca, int, 0444); 347MODULE_PARM_DESC(myri10ge_dca, "Enable DCA if possible"); 348 349#define MYRI10GE_FW_OFFSET 1024*1024 350#define MYRI10GE_HIGHPART_TO_U32(X) \ 351(sizeof (X) == 8) ? ((u32)((u64)(X) >> 32)) : (0) 352#define MYRI10GE_LOWPART_TO_U32(X) ((u32)(X)) 353 354#define myri10ge_pio_copy(to,from,size) __iowrite64_copy(to,from,size/8) 355 356static void myri10ge_set_multicast_list(struct net_device *dev); 357static netdev_tx_t myri10ge_sw_tso(struct sk_buff *skb, 358 struct net_device *dev); 359 360static inline void put_be32(__be32 val, __be32 __iomem * p) 361{ 362 __raw_writel((__force __u32) val, (__force void __iomem *)p); 363} 364 365static void myri10ge_get_stats(struct net_device *dev, 366 struct rtnl_link_stats64 *stats); 367 368static void set_fw_name(struct myri10ge_priv *mgp, char *name, bool allocated) 369{ 370 if (mgp->fw_name_allocated) 371 kfree(mgp->fw_name); 372 mgp->fw_name = name; 373 mgp->fw_name_allocated = allocated; 374} 375 376static int 377myri10ge_send_cmd(struct myri10ge_priv *mgp, u32 cmd, 378 struct myri10ge_cmd *data, int atomic) 379{ 380 struct mcp_cmd *buf; 381 char buf_bytes[sizeof(*buf) + 8]; 382 struct mcp_cmd_response *response = mgp->cmd; 383 char __iomem *cmd_addr = mgp->sram + MXGEFW_ETH_CMD; 384 u32 dma_low, dma_high, result, value; 385 int sleep_total = 0; 386 387 /* ensure buf is aligned to 8 bytes */ 388 buf = (struct mcp_cmd *)ALIGN((unsigned long)buf_bytes, 8); 389 390 buf->data0 = htonl(data->data0); 391 buf->data1 = htonl(data->data1); 392 buf->data2 = htonl(data->data2); 393 buf->cmd = htonl(cmd); 394 dma_low = MYRI10GE_LOWPART_TO_U32(mgp->cmd_bus); 395 dma_high = MYRI10GE_HIGHPART_TO_U32(mgp->cmd_bus); 396 397 buf->response_addr.low = htonl(dma_low); 398 buf->response_addr.high = htonl(dma_high); 399 response->result = htonl(MYRI10GE_NO_RESPONSE_RESULT); 400 mb(); 401 myri10ge_pio_copy(cmd_addr, buf, sizeof(*buf)); 402 403 /* wait up to 15ms. Longest command is the DMA benchmark, 404 * which is capped at 5ms, but runs from a timeout handler 405 * that runs every 7.8ms. So a 15ms timeout leaves us with 406 * a 2.2ms margin 407 */ 408 if (atomic) { 409 /* if atomic is set, do not sleep, 410 * and try to get the completion quickly 411 * (1ms will be enough for those commands) */ 412 for (sleep_total = 0; 413 sleep_total < 1000 && 414 response->result == htonl(MYRI10GE_NO_RESPONSE_RESULT); 415 sleep_total += 10) { 416 udelay(10); 417 mb(); 418 } 419 } else { 420 /* use msleep for most command */ 421 for (sleep_total = 0; 422 sleep_total < 15 && 423 response->result == htonl(MYRI10GE_NO_RESPONSE_RESULT); 424 sleep_total++) 425 msleep(1); 426 } 427 428 result = ntohl(response->result); 429 value = ntohl(response->data); 430 if (result != MYRI10GE_NO_RESPONSE_RESULT) { 431 if (result == 0) { 432 data->data0 = value; 433 return 0; 434 } else if (result == MXGEFW_CMD_UNKNOWN) { 435 return -ENOSYS; 436 } else if (result == MXGEFW_CMD_ERROR_UNALIGNED) { 437 return -E2BIG; 438 } else if (result == MXGEFW_CMD_ERROR_RANGE && 439 cmd == MXGEFW_CMD_ENABLE_RSS_QUEUES && 440 (data-> 441 data1 & MXGEFW_SLICE_ENABLE_MULTIPLE_TX_QUEUES) != 442 0) { 443 return -ERANGE; 444 } else { 445 dev_err(&mgp->pdev->dev, 446 "command %d failed, result = %d\n", 447 cmd, result); 448 return -ENXIO; 449 } 450 } 451 452 dev_err(&mgp->pdev->dev, "command %d timed out, result = %d\n", 453 cmd, result); 454 return -EAGAIN; 455} 456 457/* 458 * The eeprom strings on the lanaiX have the format 459 * SN=x\0 460 * MAC=x:x:x:x:x:x\0 461 * PT:ddd mmm xx xx:xx:xx xx\0 462 * PV:ddd mmm xx xx:xx:xx xx\0 463 */ 464static int myri10ge_read_mac_addr(struct myri10ge_priv *mgp) 465{ 466 char *ptr, *limit; 467 int i; 468 469 ptr = mgp->eeprom_strings; 470 limit = mgp->eeprom_strings + MYRI10GE_EEPROM_STRINGS_SIZE; 471 472 while (*ptr != '\0' && ptr < limit) { 473 if (memcmp(ptr, "MAC=", 4) == 0) { 474 ptr += 4; 475 mgp->mac_addr_string = ptr; 476 for (i = 0; i < 6; i++) { 477 if ((ptr + 2) > limit) 478 goto abort; 479 mgp->mac_addr[i] = 480 simple_strtoul(ptr, &ptr, 16); 481 ptr += 1; 482 } 483 } 484 if (memcmp(ptr, "PC=", 3) == 0) { 485 ptr += 3; 486 mgp->product_code_string = ptr; 487 } 488 if (memcmp((const void *)ptr, "SN=", 3) == 0) { 489 ptr += 3; 490 mgp->serial_number = simple_strtoul(ptr, &ptr, 10); 491 } 492 while (ptr < limit && *ptr++) ; 493 } 494 495 return 0; 496 497abort: 498 dev_err(&mgp->pdev->dev, "failed to parse eeprom_strings\n"); 499 return -ENXIO; 500} 501 502/* 503 * Enable or disable periodic RDMAs from the host to make certain 504 * chipsets resend dropped PCIe messages 505 */ 506 507static void myri10ge_dummy_rdma(struct myri10ge_priv *mgp, int enable) 508{ 509 char __iomem *submit; 510 __be32 buf[16] __attribute__ ((__aligned__(8))); 511 u32 dma_low, dma_high; 512 int i; 513 514 /* clear confirmation addr */ 515 mgp->cmd->data = 0; 516 mb(); 517 518 /* send a rdma command to the PCIe engine, and wait for the 519 * response in the confirmation address. The firmware should 520 * write a -1 there to indicate it is alive and well 521 */ 522 dma_low = MYRI10GE_LOWPART_TO_U32(mgp->cmd_bus); 523 dma_high = MYRI10GE_HIGHPART_TO_U32(mgp->cmd_bus); 524 525 buf[0] = htonl(dma_high); /* confirm addr MSW */ 526 buf[1] = htonl(dma_low); /* confirm addr LSW */ 527 buf[2] = MYRI10GE_NO_CONFIRM_DATA; /* confirm data */ 528 buf[3] = htonl(dma_high); /* dummy addr MSW */ 529 buf[4] = htonl(dma_low); /* dummy addr LSW */ 530 buf[5] = htonl(enable); /* enable? */ 531 532 submit = mgp->sram + MXGEFW_BOOT_DUMMY_RDMA; 533 534 myri10ge_pio_copy(submit, &buf, sizeof(buf)); 535 for (i = 0; mgp->cmd->data != MYRI10GE_NO_CONFIRM_DATA && i < 20; i++) 536 msleep(1); 537 if (mgp->cmd->data != MYRI10GE_NO_CONFIRM_DATA) 538 dev_err(&mgp->pdev->dev, "dummy rdma %s failed\n", 539 (enable ? "enable" : "disable")); 540} 541 542static int 543myri10ge_validate_firmware(struct myri10ge_priv *mgp, 544 struct mcp_gen_header *hdr) 545{ 546 struct device *dev = &mgp->pdev->dev; 547 548 /* check firmware type */ 549 if (ntohl(hdr->mcp_type) != MCP_TYPE_ETH) { 550 dev_err(dev, "Bad firmware type: 0x%x\n", ntohl(hdr->mcp_type)); 551 return -EINVAL; 552 } 553 554 /* save firmware version for ethtool */ 555 strncpy(mgp->fw_version, hdr->version, sizeof(mgp->fw_version)); 556 mgp->fw_version[sizeof(mgp->fw_version) - 1] = '\0'; 557 558 sscanf(mgp->fw_version, "%d.%d.%d", &mgp->fw_ver_major, 559 &mgp->fw_ver_minor, &mgp->fw_ver_tiny); 560 561 if (!(mgp->fw_ver_major == MXGEFW_VERSION_MAJOR && 562 mgp->fw_ver_minor == MXGEFW_VERSION_MINOR)) { 563 dev_err(dev, "Found firmware version %s\n", mgp->fw_version); 564 dev_err(dev, "Driver needs %d.%d\n", MXGEFW_VERSION_MAJOR, 565 MXGEFW_VERSION_MINOR); 566 return -EINVAL; 567 } 568 return 0; 569} 570 571static int myri10ge_load_hotplug_firmware(struct myri10ge_priv *mgp, u32 * size) 572{ 573 unsigned crc, reread_crc; 574 const struct firmware *fw; 575 struct device *dev = &mgp->pdev->dev; 576 unsigned char *fw_readback; 577 struct mcp_gen_header *hdr; 578 size_t hdr_offset; 579 int status; 580 unsigned i; 581 582 if ((status = request_firmware(&fw, mgp->fw_name, dev)) < 0) { 583 dev_err(dev, "Unable to load %s firmware image via hotplug\n", 584 mgp->fw_name); 585 status = -EINVAL; 586 goto abort_with_nothing; 587 } 588 589 /* check size */ 590 591 if (fw->size >= mgp->sram_size - MYRI10GE_FW_OFFSET || 592 fw->size < MCP_HEADER_PTR_OFFSET + 4) { 593 dev_err(dev, "Firmware size invalid:%d\n", (int)fw->size); 594 status = -EINVAL; 595 goto abort_with_fw; 596 } 597 598 /* check id */ 599 hdr_offset = ntohl(*(__be32 *) (fw->data + MCP_HEADER_PTR_OFFSET)); 600 if ((hdr_offset & 3) || hdr_offset + sizeof(*hdr) > fw->size) { 601 dev_err(dev, "Bad firmware file\n"); 602 status = -EINVAL; 603 goto abort_with_fw; 604 } 605 hdr = (void *)(fw->data + hdr_offset); 606 607 status = myri10ge_validate_firmware(mgp, hdr); 608 if (status != 0) 609 goto abort_with_fw; 610 611 crc = crc32(~0, fw->data, fw->size); 612 for (i = 0; i < fw->size; i += 256) { 613 myri10ge_pio_copy(mgp->sram + MYRI10GE_FW_OFFSET + i, 614 fw->data + i, 615 min(256U, (unsigned)(fw->size - i))); 616 mb(); 617 readb(mgp->sram); 618 } 619 fw_readback = vmalloc(fw->size); 620 if (!fw_readback) { 621 status = -ENOMEM; 622 goto abort_with_fw; 623 } 624 /* corruption checking is good for parity recovery and buggy chipset */ 625 memcpy_fromio(fw_readback, mgp->sram + MYRI10GE_FW_OFFSET, fw->size); 626 reread_crc = crc32(~0, fw_readback, fw->size); 627 vfree(fw_readback); 628 if (crc != reread_crc) { 629 dev_err(dev, "CRC failed(fw-len=%u), got 0x%x (expect 0x%x)\n", 630 (unsigned)fw->size, reread_crc, crc); 631 status = -EIO; 632 goto abort_with_fw; 633 } 634 *size = (u32) fw->size; 635 636abort_with_fw: 637 release_firmware(fw); 638 639abort_with_nothing: 640 return status; 641} 642 643static int myri10ge_adopt_running_firmware(struct myri10ge_priv *mgp) 644{ 645 struct mcp_gen_header *hdr; 646 struct device *dev = &mgp->pdev->dev; 647 const size_t bytes = sizeof(struct mcp_gen_header); 648 size_t hdr_offset; 649 int status; 650 651 /* find running firmware header */ 652 hdr_offset = swab32(readl(mgp->sram + MCP_HEADER_PTR_OFFSET)); 653 654 if ((hdr_offset & 3) || hdr_offset + sizeof(*hdr) > mgp->sram_size) { 655 dev_err(dev, "Running firmware has bad header offset (%d)\n", 656 (int)hdr_offset); 657 return -EIO; 658 } 659 660 /* copy header of running firmware from SRAM to host memory to 661 * validate firmware */ 662 hdr = kmalloc(bytes, GFP_KERNEL); 663 if (hdr == NULL) 664 return -ENOMEM; 665 666 memcpy_fromio(hdr, mgp->sram + hdr_offset, bytes); 667 status = myri10ge_validate_firmware(mgp, hdr); 668 kfree(hdr); 669 670 /* check to see if adopted firmware has bug where adopting 671 * it will cause broadcasts to be filtered unless the NIC 672 * is kept in ALLMULTI mode */ 673 if (mgp->fw_ver_major == 1 && mgp->fw_ver_minor == 4 && 674 mgp->fw_ver_tiny >= 4 && mgp->fw_ver_tiny <= 11) { 675 mgp->adopted_rx_filter_bug = 1; 676 dev_warn(dev, "Adopting fw %d.%d.%d: " 677 "working around rx filter bug\n", 678 mgp->fw_ver_major, mgp->fw_ver_minor, 679 mgp->fw_ver_tiny); 680 } 681 return status; 682} 683 684static int myri10ge_get_firmware_capabilities(struct myri10ge_priv *mgp) 685{ 686 struct myri10ge_cmd cmd; 687 int status; 688 689 /* probe for IPv6 TSO support */ 690 mgp->features = NETIF_F_SG | NETIF_F_HW_CSUM | NETIF_F_TSO; 691 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_MAX_TSO6_HDR_SIZE, 692 &cmd, 0); 693 if (status == 0) { 694 mgp->max_tso6 = cmd.data0; 695 mgp->features |= NETIF_F_TSO6; 696 } 697 698 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_RX_RING_SIZE, &cmd, 0); 699 if (status != 0) { 700 dev_err(&mgp->pdev->dev, 701 "failed MXGEFW_CMD_GET_RX_RING_SIZE\n"); 702 return -ENXIO; 703 } 704 705 mgp->max_intr_slots = 2 * (cmd.data0 / sizeof(struct mcp_dma_addr)); 706 707 return 0; 708} 709 710static int myri10ge_load_firmware(struct myri10ge_priv *mgp, int adopt) 711{ 712 char __iomem *submit; 713 __be32 buf[16] __attribute__ ((__aligned__(8))); 714 u32 dma_low, dma_high, size; 715 int status, i; 716 717 size = 0; 718 status = myri10ge_load_hotplug_firmware(mgp, &size); 719 if (status) { 720 if (!adopt) 721 return status; 722 dev_warn(&mgp->pdev->dev, "hotplug firmware loading failed\n"); 723 724 /* Do not attempt to adopt firmware if there 725 * was a bad crc */ 726 if (status == -EIO) 727 return status; 728 729 status = myri10ge_adopt_running_firmware(mgp); 730 if (status != 0) { 731 dev_err(&mgp->pdev->dev, 732 "failed to adopt running firmware\n"); 733 return status; 734 } 735 dev_info(&mgp->pdev->dev, 736 "Successfully adopted running firmware\n"); 737 if (mgp->tx_boundary == 4096) { 738 dev_warn(&mgp->pdev->dev, 739 "Using firmware currently running on NIC" 740 ". For optimal\n"); 741 dev_warn(&mgp->pdev->dev, 742 "performance consider loading optimized " 743 "firmware\n"); 744 dev_warn(&mgp->pdev->dev, "via hotplug\n"); 745 } 746 747 set_fw_name(mgp, "adopted", false); 748 mgp->tx_boundary = 2048; 749 myri10ge_dummy_rdma(mgp, 1); 750 status = myri10ge_get_firmware_capabilities(mgp); 751 return status; 752 } 753 754 /* clear confirmation addr */ 755 mgp->cmd->data = 0; 756 mb(); 757 758 /* send a reload command to the bootstrap MCP, and wait for the 759 * response in the confirmation address. The firmware should 760 * write a -1 there to indicate it is alive and well 761 */ 762 dma_low = MYRI10GE_LOWPART_TO_U32(mgp->cmd_bus); 763 dma_high = MYRI10GE_HIGHPART_TO_U32(mgp->cmd_bus); 764 765 buf[0] = htonl(dma_high); /* confirm addr MSW */ 766 buf[1] = htonl(dma_low); /* confirm addr LSW */ 767 buf[2] = MYRI10GE_NO_CONFIRM_DATA; /* confirm data */ 768 769 /* FIX: All newest firmware should un-protect the bottom of 770 * the sram before handoff. However, the very first interfaces 771 * do not. Therefore the handoff copy must skip the first 8 bytes 772 */ 773 buf[3] = htonl(MYRI10GE_FW_OFFSET + 8); /* where the code starts */ 774 buf[4] = htonl(size - 8); /* length of code */ 775 buf[5] = htonl(8); /* where to copy to */ 776 buf[6] = htonl(0); /* where to jump to */ 777 778 submit = mgp->sram + MXGEFW_BOOT_HANDOFF; 779 780 myri10ge_pio_copy(submit, &buf, sizeof(buf)); 781 mb(); 782 msleep(1); 783 mb(); 784 i = 0; 785 while (mgp->cmd->data != MYRI10GE_NO_CONFIRM_DATA && i < 9) { 786 msleep(1 << i); 787 i++; 788 } 789 if (mgp->cmd->data != MYRI10GE_NO_CONFIRM_DATA) { 790 dev_err(&mgp->pdev->dev, "handoff failed\n"); 791 return -ENXIO; 792 } 793 myri10ge_dummy_rdma(mgp, 1); 794 status = myri10ge_get_firmware_capabilities(mgp); 795 796 return status; 797} 798 799static int myri10ge_update_mac_address(struct myri10ge_priv *mgp, u8 * addr) 800{ 801 struct myri10ge_cmd cmd; 802 int status; 803 804 cmd.data0 = ((addr[0] << 24) | (addr[1] << 16) 805 | (addr[2] << 8) | addr[3]); 806 807 cmd.data1 = ((addr[4] << 8) | (addr[5])); 808 809 status = myri10ge_send_cmd(mgp, MXGEFW_SET_MAC_ADDRESS, &cmd, 0); 810 return status; 811} 812 813static int myri10ge_change_pause(struct myri10ge_priv *mgp, int pause) 814{ 815 struct myri10ge_cmd cmd; 816 int status, ctl; 817 818 ctl = pause ? MXGEFW_ENABLE_FLOW_CONTROL : MXGEFW_DISABLE_FLOW_CONTROL; 819 status = myri10ge_send_cmd(mgp, ctl, &cmd, 0); 820 821 if (status) { 822 netdev_err(mgp->dev, "Failed to set flow control mode\n"); 823 return status; 824 } 825 mgp->pause = pause; 826 return 0; 827} 828 829static void 830myri10ge_change_promisc(struct myri10ge_priv *mgp, int promisc, int atomic) 831{ 832 struct myri10ge_cmd cmd; 833 int status, ctl; 834 835 ctl = promisc ? MXGEFW_ENABLE_PROMISC : MXGEFW_DISABLE_PROMISC; 836 status = myri10ge_send_cmd(mgp, ctl, &cmd, atomic); 837 if (status) 838 netdev_err(mgp->dev, "Failed to set promisc mode\n"); 839} 840 841static int myri10ge_dma_test(struct myri10ge_priv *mgp, int test_type) 842{ 843 struct myri10ge_cmd cmd; 844 int status; 845 u32 len; 846 struct page *dmatest_page; 847 dma_addr_t dmatest_bus; 848 char *test = " "; 849 850 dmatest_page = alloc_page(GFP_KERNEL); 851 if (!dmatest_page) 852 return -ENOMEM; 853 dmatest_bus = pci_map_page(mgp->pdev, dmatest_page, 0, PAGE_SIZE, 854 DMA_BIDIRECTIONAL); 855 if (unlikely(pci_dma_mapping_error(mgp->pdev, dmatest_bus))) { 856 __free_page(dmatest_page); 857 return -ENOMEM; 858 } 859 860 /* Run a small DMA test. 861 * The magic multipliers to the length tell the firmware 862 * to do DMA read, write, or read+write tests. The 863 * results are returned in cmd.data0. The upper 16 864 * bits or the return is the number of transfers completed. 865 * The lower 16 bits is the time in 0.5us ticks that the 866 * transfers took to complete. 867 */ 868 869 len = mgp->tx_boundary; 870 871 cmd.data0 = MYRI10GE_LOWPART_TO_U32(dmatest_bus); 872 cmd.data1 = MYRI10GE_HIGHPART_TO_U32(dmatest_bus); 873 cmd.data2 = len * 0x10000; 874 status = myri10ge_send_cmd(mgp, test_type, &cmd, 0); 875 if (status != 0) { 876 test = "read"; 877 goto abort; 878 } 879 mgp->read_dma = ((cmd.data0 >> 16) * len * 2) / (cmd.data0 & 0xffff); 880 cmd.data0 = MYRI10GE_LOWPART_TO_U32(dmatest_bus); 881 cmd.data1 = MYRI10GE_HIGHPART_TO_U32(dmatest_bus); 882 cmd.data2 = len * 0x1; 883 status = myri10ge_send_cmd(mgp, test_type, &cmd, 0); 884 if (status != 0) { 885 test = "write"; 886 goto abort; 887 } 888 mgp->write_dma = ((cmd.data0 >> 16) * len * 2) / (cmd.data0 & 0xffff); 889 890 cmd.data0 = MYRI10GE_LOWPART_TO_U32(dmatest_bus); 891 cmd.data1 = MYRI10GE_HIGHPART_TO_U32(dmatest_bus); 892 cmd.data2 = len * 0x10001; 893 status = myri10ge_send_cmd(mgp, test_type, &cmd, 0); 894 if (status != 0) { 895 test = "read/write"; 896 goto abort; 897 } 898 mgp->read_write_dma = ((cmd.data0 >> 16) * len * 2 * 2) / 899 (cmd.data0 & 0xffff); 900 901abort: 902 pci_unmap_page(mgp->pdev, dmatest_bus, PAGE_SIZE, DMA_BIDIRECTIONAL); 903 put_page(dmatest_page); 904 905 if (status != 0 && test_type != MXGEFW_CMD_UNALIGNED_TEST) 906 dev_warn(&mgp->pdev->dev, "DMA %s benchmark failed: %d\n", 907 test, status); 908 909 return status; 910} 911 912static int myri10ge_reset(struct myri10ge_priv *mgp) 913{ 914 struct myri10ge_cmd cmd; 915 struct myri10ge_slice_state *ss; 916 int i, status; 917 size_t bytes; 918#ifdef CONFIG_MYRI10GE_DCA 919 unsigned long dca_tag_off; 920#endif 921 922 /* try to send a reset command to the card to see if it 923 * is alive */ 924 memset(&cmd, 0, sizeof(cmd)); 925 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_RESET, &cmd, 0); 926 if (status != 0) { 927 dev_err(&mgp->pdev->dev, "failed reset\n"); 928 return -ENXIO; 929 } 930 931 (void)myri10ge_dma_test(mgp, MXGEFW_DMA_TEST); 932 /* 933 * Use non-ndis mcp_slot (eg, 4 bytes total, 934 * no toeplitz hash value returned. Older firmware will 935 * not understand this command, but will use the correct 936 * sized mcp_slot, so we ignore error returns 937 */ 938 cmd.data0 = MXGEFW_RSS_MCP_SLOT_TYPE_MIN; 939 (void)myri10ge_send_cmd(mgp, MXGEFW_CMD_SET_RSS_MCP_SLOT_TYPE, &cmd, 0); 940 941 /* Now exchange information about interrupts */ 942 943 bytes = mgp->max_intr_slots * sizeof(*mgp->ss[0].rx_done.entry); 944 cmd.data0 = (u32) bytes; 945 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_SET_INTRQ_SIZE, &cmd, 0); 946 947 /* 948 * Even though we already know how many slices are supported 949 * via myri10ge_probe_slices() MXGEFW_CMD_GET_MAX_RSS_QUEUES 950 * has magic side effects, and must be called after a reset. 951 * It must be called prior to calling any RSS related cmds, 952 * including assigning an interrupt queue for anything but 953 * slice 0. It must also be called *after* 954 * MXGEFW_CMD_SET_INTRQ_SIZE, since the intrq size is used by 955 * the firmware to compute offsets. 956 */ 957 958 if (mgp->num_slices > 1) { 959 960 /* ask the maximum number of slices it supports */ 961 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_MAX_RSS_QUEUES, 962 &cmd, 0); 963 if (status != 0) { 964 dev_err(&mgp->pdev->dev, 965 "failed to get number of slices\n"); 966 } 967 968 /* 969 * MXGEFW_CMD_ENABLE_RSS_QUEUES must be called prior 970 * to setting up the interrupt queue DMA 971 */ 972 973 cmd.data0 = mgp->num_slices; 974 cmd.data1 = MXGEFW_SLICE_INTR_MODE_ONE_PER_SLICE; 975 if (mgp->dev->real_num_tx_queues > 1) 976 cmd.data1 |= MXGEFW_SLICE_ENABLE_MULTIPLE_TX_QUEUES; 977 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_ENABLE_RSS_QUEUES, 978 &cmd, 0); 979 980 /* Firmware older than 1.4.32 only supports multiple 981 * RX queues, so if we get an error, first retry using a 982 * single TX queue before giving up */ 983 if (status != 0 && mgp->dev->real_num_tx_queues > 1) { 984 netif_set_real_num_tx_queues(mgp->dev, 1); 985 cmd.data0 = mgp->num_slices; 986 cmd.data1 = MXGEFW_SLICE_INTR_MODE_ONE_PER_SLICE; 987 status = myri10ge_send_cmd(mgp, 988 MXGEFW_CMD_ENABLE_RSS_QUEUES, 989 &cmd, 0); 990 } 991 992 if (status != 0) { 993 dev_err(&mgp->pdev->dev, 994 "failed to set number of slices\n"); 995 996 return status; 997 } 998 } 999 for (i = 0; i < mgp->num_slices; i++) { 1000 ss = &mgp->ss[i]; 1001 cmd.data0 = MYRI10GE_LOWPART_TO_U32(ss->rx_done.bus); 1002 cmd.data1 = MYRI10GE_HIGHPART_TO_U32(ss->rx_done.bus); 1003 cmd.data2 = i; 1004 status |= myri10ge_send_cmd(mgp, MXGEFW_CMD_SET_INTRQ_DMA, 1005 &cmd, 0); 1006 } 1007 1008 status |= 1009 myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_IRQ_ACK_OFFSET, &cmd, 0); 1010 for (i = 0; i < mgp->num_slices; i++) { 1011 ss = &mgp->ss[i]; 1012 ss->irq_claim = 1013 (__iomem __be32 *) (mgp->sram + cmd.data0 + 8 * i); 1014 } 1015 status |= myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_IRQ_DEASSERT_OFFSET, 1016 &cmd, 0); 1017 mgp->irq_deassert = (__iomem __be32 *) (mgp->sram + cmd.data0); 1018 1019 status |= myri10ge_send_cmd 1020 (mgp, MXGEFW_CMD_GET_INTR_COAL_DELAY_OFFSET, &cmd, 0); 1021 mgp->intr_coal_delay_ptr = (__iomem __be32 *) (mgp->sram + cmd.data0); 1022 if (status != 0) { 1023 dev_err(&mgp->pdev->dev, "failed set interrupt parameters\n"); 1024 return status; 1025 } 1026 put_be32(htonl(mgp->intr_coal_delay), mgp->intr_coal_delay_ptr); 1027 1028#ifdef CONFIG_MYRI10GE_DCA 1029 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_DCA_OFFSET, &cmd, 0); 1030 dca_tag_off = cmd.data0; 1031 for (i = 0; i < mgp->num_slices; i++) { 1032 ss = &mgp->ss[i]; 1033 if (status == 0) { 1034 ss->dca_tag = (__iomem __be32 *) 1035 (mgp->sram + dca_tag_off + 4 * i); 1036 } else { 1037 ss->dca_tag = NULL; 1038 } 1039 } 1040#endif /* CONFIG_MYRI10GE_DCA */ 1041 1042 /* reset mcp/driver shared state back to 0 */ 1043 1044 mgp->link_changes = 0; 1045 for (i = 0; i < mgp->num_slices; i++) { 1046 ss = &mgp->ss[i]; 1047 1048 memset(ss->rx_done.entry, 0, bytes); 1049 ss->tx.req = 0; 1050 ss->tx.done = 0; 1051 ss->tx.pkt_start = 0; 1052 ss->tx.pkt_done = 0; 1053 ss->rx_big.cnt = 0; 1054 ss->rx_small.cnt = 0; 1055 ss->rx_done.idx = 0; 1056 ss->rx_done.cnt = 0; 1057 ss->tx.wake_queue = 0; 1058 ss->tx.stop_queue = 0; 1059 } 1060 1061 status = myri10ge_update_mac_address(mgp, mgp->dev->dev_addr); 1062 myri10ge_change_pause(mgp, mgp->pause); 1063 myri10ge_set_multicast_list(mgp->dev); 1064 return status; 1065} 1066 1067#ifdef CONFIG_MYRI10GE_DCA 1068static int myri10ge_toggle_relaxed(struct pci_dev *pdev, int on) 1069{ 1070 int ret; 1071 u16 ctl; 1072 1073 pcie_capability_read_word(pdev, PCI_EXP_DEVCTL, &ctl); 1074 1075 ret = (ctl & PCI_EXP_DEVCTL_RELAX_EN) >> 4; 1076 if (ret != on) { 1077 ctl &= ~PCI_EXP_DEVCTL_RELAX_EN; 1078 ctl |= (on << 4); 1079 pcie_capability_write_word(pdev, PCI_EXP_DEVCTL, ctl); 1080 } 1081 return ret; 1082} 1083 1084static void 1085myri10ge_write_dca(struct myri10ge_slice_state *ss, int cpu, int tag) 1086{ 1087 ss->cached_dca_tag = tag; 1088 put_be32(htonl(tag), ss->dca_tag); 1089} 1090 1091static inline void myri10ge_update_dca(struct myri10ge_slice_state *ss) 1092{ 1093 int cpu = get_cpu(); 1094 int tag; 1095 1096 if (cpu != ss->cpu) { 1097 tag = dca3_get_tag(&ss->mgp->pdev->dev, cpu); 1098 if (ss->cached_dca_tag != tag) 1099 myri10ge_write_dca(ss, cpu, tag); 1100 ss->cpu = cpu; 1101 } 1102 put_cpu(); 1103} 1104 1105static void myri10ge_setup_dca(struct myri10ge_priv *mgp) 1106{ 1107 int err, i; 1108 struct pci_dev *pdev = mgp->pdev; 1109 1110 if (mgp->ss[0].dca_tag == NULL || mgp->dca_enabled) 1111 return; 1112 if (!myri10ge_dca) { 1113 dev_err(&pdev->dev, "dca disabled by administrator\n"); 1114 return; 1115 } 1116 err = dca_add_requester(&pdev->dev); 1117 if (err) { 1118 if (err != -ENODEV) 1119 dev_err(&pdev->dev, 1120 "dca_add_requester() failed, err=%d\n", err); 1121 return; 1122 } 1123 mgp->relaxed_order = myri10ge_toggle_relaxed(pdev, 0); 1124 mgp->dca_enabled = 1; 1125 for (i = 0; i < mgp->num_slices; i++) { 1126 mgp->ss[i].cpu = -1; 1127 mgp->ss[i].cached_dca_tag = -1; 1128 myri10ge_update_dca(&mgp->ss[i]); 1129 } 1130} 1131 1132static void myri10ge_teardown_dca(struct myri10ge_priv *mgp) 1133{ 1134 struct pci_dev *pdev = mgp->pdev; 1135 1136 if (!mgp->dca_enabled) 1137 return; 1138 mgp->dca_enabled = 0; 1139 if (mgp->relaxed_order) 1140 myri10ge_toggle_relaxed(pdev, 1); 1141 dca_remove_requester(&pdev->dev); 1142} 1143 1144static int myri10ge_notify_dca_device(struct device *dev, void *data) 1145{ 1146 struct myri10ge_priv *mgp; 1147 unsigned long event; 1148 1149 mgp = dev_get_drvdata(dev); 1150 event = *(unsigned long *)data; 1151 1152 if (event == DCA_PROVIDER_ADD) 1153 myri10ge_setup_dca(mgp); 1154 else if (event == DCA_PROVIDER_REMOVE) 1155 myri10ge_teardown_dca(mgp); 1156 return 0; 1157} 1158#endif /* CONFIG_MYRI10GE_DCA */ 1159 1160static inline void 1161myri10ge_submit_8rx(struct mcp_kreq_ether_recv __iomem * dst, 1162 struct mcp_kreq_ether_recv *src) 1163{ 1164 __be32 low; 1165 1166 low = src->addr_low; 1167 src->addr_low = htonl(DMA_BIT_MASK(32)); 1168 myri10ge_pio_copy(dst, src, 4 * sizeof(*src)); 1169 mb(); 1170 myri10ge_pio_copy(dst + 4, src + 4, 4 * sizeof(*src)); 1171 mb(); 1172 src->addr_low = low; 1173 put_be32(low, &dst->addr_low); 1174 mb(); 1175} 1176 1177static void 1178myri10ge_alloc_rx_pages(struct myri10ge_priv *mgp, struct myri10ge_rx_buf *rx, 1179 int bytes, int watchdog) 1180{ 1181 struct page *page; 1182 dma_addr_t bus; 1183 int idx; 1184#if MYRI10GE_ALLOC_SIZE > 4096 1185 int end_offset; 1186#endif 1187 1188 if (unlikely(rx->watchdog_needed && !watchdog)) 1189 return; 1190 1191 /* try to refill entire ring */ 1192 while (rx->fill_cnt != (rx->cnt + rx->mask + 1)) { 1193 idx = rx->fill_cnt & rx->mask; 1194 if (rx->page_offset + bytes <= MYRI10GE_ALLOC_SIZE) { 1195 /* we can use part of previous page */ 1196 get_page(rx->page); 1197 } else { 1198 /* we need a new page */ 1199 page = 1200 alloc_pages(GFP_ATOMIC | __GFP_COMP, 1201 MYRI10GE_ALLOC_ORDER); 1202 if (unlikely(page == NULL)) { 1203 if (rx->fill_cnt - rx->cnt < 16) 1204 rx->watchdog_needed = 1; 1205 return; 1206 } 1207 1208 bus = pci_map_page(mgp->pdev, page, 0, 1209 MYRI10GE_ALLOC_SIZE, 1210 PCI_DMA_FROMDEVICE); 1211 if (unlikely(pci_dma_mapping_error(mgp->pdev, bus))) { 1212 __free_pages(page, MYRI10GE_ALLOC_ORDER); 1213 if (rx->fill_cnt - rx->cnt < 16) 1214 rx->watchdog_needed = 1; 1215 return; 1216 } 1217 1218 rx->page = page; 1219 rx->page_offset = 0; 1220 rx->bus = bus; 1221 1222 } 1223 rx->info[idx].page = rx->page; 1224 rx->info[idx].page_offset = rx->page_offset; 1225 /* note that this is the address of the start of the 1226 * page */ 1227 dma_unmap_addr_set(&rx->info[idx], bus, rx->bus); 1228 rx->shadow[idx].addr_low = 1229 htonl(MYRI10GE_LOWPART_TO_U32(rx->bus) + rx->page_offset); 1230 rx->shadow[idx].addr_high = 1231 htonl(MYRI10GE_HIGHPART_TO_U32(rx->bus)); 1232 1233 /* start next packet on a cacheline boundary */ 1234 rx->page_offset += SKB_DATA_ALIGN(bytes); 1235 1236#if MYRI10GE_ALLOC_SIZE > 4096 1237 /* don't cross a 4KB boundary */ 1238 end_offset = rx->page_offset + bytes - 1; 1239 if ((unsigned)(rx->page_offset ^ end_offset) > 4095) 1240 rx->page_offset = end_offset & ~4095; 1241#endif 1242 rx->fill_cnt++; 1243 1244 /* copy 8 descriptors to the firmware at a time */ 1245 if ((idx & 7) == 7) { 1246 myri10ge_submit_8rx(&rx->lanai[idx - 7], 1247 &rx->shadow[idx - 7]); 1248 } 1249 } 1250} 1251 1252static inline void 1253myri10ge_unmap_rx_page(struct pci_dev *pdev, 1254 struct myri10ge_rx_buffer_state *info, int bytes) 1255{ 1256 /* unmap the recvd page if we're the only or last user of it */ 1257 if (bytes >= MYRI10GE_ALLOC_SIZE / 2 || 1258 (info->page_offset + 2 * bytes) > MYRI10GE_ALLOC_SIZE) { 1259 pci_unmap_page(pdev, (dma_unmap_addr(info, bus) 1260 & ~(MYRI10GE_ALLOC_SIZE - 1)), 1261 MYRI10GE_ALLOC_SIZE, PCI_DMA_FROMDEVICE); 1262 } 1263} 1264 1265/* 1266 * GRO does not support acceleration of tagged vlan frames, and 1267 * this NIC does not support vlan tag offload, so we must pop 1268 * the tag ourselves to be able to achieve GRO performance that 1269 * is comparable to LRO. 1270 */ 1271 1272static inline void 1273myri10ge_vlan_rx(struct net_device *dev, void *addr, struct sk_buff *skb) 1274{ 1275 u8 *va; 1276 struct vlan_ethhdr *veh; 1277 skb_frag_t *frag; 1278 __wsum vsum; 1279 1280 va = addr; 1281 va += MXGEFW_PAD; 1282 veh = (struct vlan_ethhdr *)va; 1283 if ((dev->features & NETIF_F_HW_VLAN_CTAG_RX) == 1284 NETIF_F_HW_VLAN_CTAG_RX && 1285 veh->h_vlan_proto == htons(ETH_P_8021Q)) { 1286 /* fixup csum if needed */ 1287 if (skb->ip_summed == CHECKSUM_COMPLETE) { 1288 vsum = csum_partial(va + ETH_HLEN, VLAN_HLEN, 0); 1289 skb->csum = csum_sub(skb->csum, vsum); 1290 } 1291 /* pop tag */ 1292 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), ntohs(veh->h_vlan_TCI)); 1293 memmove(va + VLAN_HLEN, va, 2 * ETH_ALEN); 1294 skb->len -= VLAN_HLEN; 1295 skb->data_len -= VLAN_HLEN; 1296 frag = skb_shinfo(skb)->frags; 1297 skb_frag_off_add(frag, VLAN_HLEN); 1298 skb_frag_size_sub(frag, VLAN_HLEN); 1299 } 1300} 1301 1302#define MYRI10GE_HLEN 64 /* Bytes to copy from page to skb linear memory */ 1303 1304static inline int 1305myri10ge_rx_done(struct myri10ge_slice_state *ss, int len, __wsum csum) 1306{ 1307 struct myri10ge_priv *mgp = ss->mgp; 1308 struct sk_buff *skb; 1309 skb_frag_t *rx_frags; 1310 struct myri10ge_rx_buf *rx; 1311 int i, idx, remainder, bytes; 1312 struct pci_dev *pdev = mgp->pdev; 1313 struct net_device *dev = mgp->dev; 1314 u8 *va; 1315 1316 if (len <= mgp->small_bytes) { 1317 rx = &ss->rx_small; 1318 bytes = mgp->small_bytes; 1319 } else { 1320 rx = &ss->rx_big; 1321 bytes = mgp->big_bytes; 1322 } 1323 1324 len += MXGEFW_PAD; 1325 idx = rx->cnt & rx->mask; 1326 va = page_address(rx->info[idx].page) + rx->info[idx].page_offset; 1327 prefetch(va); 1328 1329 skb = napi_get_frags(&ss->napi); 1330 if (unlikely(skb == NULL)) { 1331 ss->stats.rx_dropped++; 1332 for (i = 0, remainder = len; remainder > 0; i++) { 1333 myri10ge_unmap_rx_page(pdev, &rx->info[idx], bytes); 1334 put_page(rx->info[idx].page); 1335 rx->cnt++; 1336 idx = rx->cnt & rx->mask; 1337 remainder -= MYRI10GE_ALLOC_SIZE; 1338 } 1339 return 0; 1340 } 1341 rx_frags = skb_shinfo(skb)->frags; 1342 /* Fill skb_frag_t(s) with data from our receive */ 1343 for (i = 0, remainder = len; remainder > 0; i++) { 1344 myri10ge_unmap_rx_page(pdev, &rx->info[idx], bytes); 1345 skb_fill_page_desc(skb, i, rx->info[idx].page, 1346 rx->info[idx].page_offset, 1347 remainder < MYRI10GE_ALLOC_SIZE ? 1348 remainder : MYRI10GE_ALLOC_SIZE); 1349 rx->cnt++; 1350 idx = rx->cnt & rx->mask; 1351 remainder -= MYRI10GE_ALLOC_SIZE; 1352 } 1353 1354 /* remove padding */ 1355 skb_frag_off_add(&rx_frags[0], MXGEFW_PAD); 1356 skb_frag_size_sub(&rx_frags[0], MXGEFW_PAD); 1357 len -= MXGEFW_PAD; 1358 1359 skb->len = len; 1360 skb->data_len = len; 1361 skb->truesize += len; 1362 if (dev->features & NETIF_F_RXCSUM) { 1363 skb->ip_summed = CHECKSUM_COMPLETE; 1364 skb->csum = csum; 1365 } 1366 myri10ge_vlan_rx(mgp->dev, va, skb); 1367 skb_record_rx_queue(skb, ss - &mgp->ss[0]); 1368 1369 napi_gro_frags(&ss->napi); 1370 1371 return 1; 1372} 1373 1374static inline void 1375myri10ge_tx_done(struct myri10ge_slice_state *ss, int mcp_index) 1376{ 1377 struct pci_dev *pdev = ss->mgp->pdev; 1378 struct myri10ge_tx_buf *tx = &ss->tx; 1379 struct netdev_queue *dev_queue; 1380 struct sk_buff *skb; 1381 int idx, len; 1382 1383 while (tx->pkt_done != mcp_index) { 1384 idx = tx->done & tx->mask; 1385 skb = tx->info[idx].skb; 1386 1387 /* Mark as free */ 1388 tx->info[idx].skb = NULL; 1389 if (tx->info[idx].last) { 1390 tx->pkt_done++; 1391 tx->info[idx].last = 0; 1392 } 1393 tx->done++; 1394 len = dma_unmap_len(&tx->info[idx], len); 1395 dma_unmap_len_set(&tx->info[idx], len, 0); 1396 if (skb) { 1397 ss->stats.tx_bytes += skb->len; 1398 ss->stats.tx_packets++; 1399 dev_consume_skb_irq(skb); 1400 if (len) 1401 pci_unmap_single(pdev, 1402 dma_unmap_addr(&tx->info[idx], 1403 bus), len, 1404 PCI_DMA_TODEVICE); 1405 } else { 1406 if (len) 1407 pci_unmap_page(pdev, 1408 dma_unmap_addr(&tx->info[idx], 1409 bus), len, 1410 PCI_DMA_TODEVICE); 1411 } 1412 } 1413 1414 dev_queue = netdev_get_tx_queue(ss->dev, ss - ss->mgp->ss); 1415 /* 1416 * Make a minimal effort to prevent the NIC from polling an 1417 * idle tx queue. If we can't get the lock we leave the queue 1418 * active. In this case, either a thread was about to start 1419 * using the queue anyway, or we lost a race and the NIC will 1420 * waste some of its resources polling an inactive queue for a 1421 * while. 1422 */ 1423 1424 if ((ss->mgp->dev->real_num_tx_queues > 1) && 1425 __netif_tx_trylock(dev_queue)) { 1426 if (tx->req == tx->done) { 1427 tx->queue_active = 0; 1428 put_be32(htonl(1), tx->send_stop); 1429 mb(); 1430 } 1431 __netif_tx_unlock(dev_queue); 1432 } 1433 1434 /* start the queue if we've stopped it */ 1435 if (netif_tx_queue_stopped(dev_queue) && 1436 tx->req - tx->done < (tx->mask >> 1) && 1437 ss->mgp->running == MYRI10GE_ETH_RUNNING) { 1438 tx->wake_queue++; 1439 netif_tx_wake_queue(dev_queue); 1440 } 1441} 1442 1443static inline int 1444myri10ge_clean_rx_done(struct myri10ge_slice_state *ss, int budget) 1445{ 1446 struct myri10ge_rx_done *rx_done = &ss->rx_done; 1447 struct myri10ge_priv *mgp = ss->mgp; 1448 unsigned long rx_bytes = 0; 1449 unsigned long rx_packets = 0; 1450 unsigned long rx_ok; 1451 int idx = rx_done->idx; 1452 int cnt = rx_done->cnt; 1453 int work_done = 0; 1454 u16 length; 1455 __wsum checksum; 1456 1457 while (rx_done->entry[idx].length != 0 && work_done < budget) { 1458 length = ntohs(rx_done->entry[idx].length); 1459 rx_done->entry[idx].length = 0; 1460 checksum = csum_unfold(rx_done->entry[idx].checksum); 1461 rx_ok = myri10ge_rx_done(ss, length, checksum); 1462 rx_packets += rx_ok; 1463 rx_bytes += rx_ok * (unsigned long)length; 1464 cnt++; 1465 idx = cnt & (mgp->max_intr_slots - 1); 1466 work_done++; 1467 } 1468 rx_done->idx = idx; 1469 rx_done->cnt = cnt; 1470 ss->stats.rx_packets += rx_packets; 1471 ss->stats.rx_bytes += rx_bytes; 1472 1473 /* restock receive rings if needed */ 1474 if (ss->rx_small.fill_cnt - ss->rx_small.cnt < myri10ge_fill_thresh) 1475 myri10ge_alloc_rx_pages(mgp, &ss->rx_small, 1476 mgp->small_bytes + MXGEFW_PAD, 0); 1477 if (ss->rx_big.fill_cnt - ss->rx_big.cnt < myri10ge_fill_thresh) 1478 myri10ge_alloc_rx_pages(mgp, &ss->rx_big, mgp->big_bytes, 0); 1479 1480 return work_done; 1481} 1482 1483static inline void myri10ge_check_statblock(struct myri10ge_priv *mgp) 1484{ 1485 struct mcp_irq_data *stats = mgp->ss[0].fw_stats; 1486 1487 if (unlikely(stats->stats_updated)) { 1488 unsigned link_up = ntohl(stats->link_up); 1489 if (mgp->link_state != link_up) { 1490 mgp->link_state = link_up; 1491 1492 if (mgp->link_state == MXGEFW_LINK_UP) { 1493 netif_info(mgp, link, mgp->dev, "link up\n"); 1494 netif_carrier_on(mgp->dev); 1495 mgp->link_changes++; 1496 } else { 1497 netif_info(mgp, link, mgp->dev, "link %s\n", 1498 (link_up == MXGEFW_LINK_MYRINET ? 1499 "mismatch (Myrinet detected)" : 1500 "down")); 1501 netif_carrier_off(mgp->dev); 1502 mgp->link_changes++; 1503 } 1504 } 1505 if (mgp->rdma_tags_available != 1506 ntohl(stats->rdma_tags_available)) { 1507 mgp->rdma_tags_available = 1508 ntohl(stats->rdma_tags_available); 1509 netdev_warn(mgp->dev, "RDMA timed out! %d tags left\n", 1510 mgp->rdma_tags_available); 1511 } 1512 mgp->down_cnt += stats->link_down; 1513 if (stats->link_down) 1514 wake_up(&mgp->down_wq); 1515 } 1516} 1517 1518static int myri10ge_poll(struct napi_struct *napi, int budget) 1519{ 1520 struct myri10ge_slice_state *ss = 1521 container_of(napi, struct myri10ge_slice_state, napi); 1522 int work_done; 1523 1524#ifdef CONFIG_MYRI10GE_DCA 1525 if (ss->mgp->dca_enabled) 1526 myri10ge_update_dca(ss); 1527#endif 1528 /* process as many rx events as NAPI will allow */ 1529 work_done = myri10ge_clean_rx_done(ss, budget); 1530 1531 if (work_done < budget) { 1532 napi_complete_done(napi, work_done); 1533 put_be32(htonl(3), ss->irq_claim); 1534 } 1535 return work_done; 1536} 1537 1538static irqreturn_t myri10ge_intr(int irq, void *arg) 1539{ 1540 struct myri10ge_slice_state *ss = arg; 1541 struct myri10ge_priv *mgp = ss->mgp; 1542 struct mcp_irq_data *stats = ss->fw_stats; 1543 struct myri10ge_tx_buf *tx = &ss->tx; 1544 u32 send_done_count; 1545 int i; 1546 1547 /* an interrupt on a non-zero receive-only slice is implicitly 1548 * valid since MSI-X irqs are not shared */ 1549 if ((mgp->dev->real_num_tx_queues == 1) && (ss != mgp->ss)) { 1550 napi_schedule(&ss->napi); 1551 return IRQ_HANDLED; 1552 } 1553 1554 /* make sure it is our IRQ, and that the DMA has finished */ 1555 if (unlikely(!stats->valid)) 1556 return IRQ_NONE; 1557 1558 /* low bit indicates receives are present, so schedule 1559 * napi poll handler */ 1560 if (stats->valid & 1) 1561 napi_schedule(&ss->napi); 1562 1563 if (!mgp->msi_enabled && !mgp->msix_enabled) { 1564 put_be32(0, mgp->irq_deassert); 1565 if (!myri10ge_deassert_wait) 1566 stats->valid = 0; 1567 mb(); 1568 } else 1569 stats->valid = 0; 1570 1571 /* Wait for IRQ line to go low, if using INTx */ 1572 i = 0; 1573 while (1) { 1574 i++; 1575 /* check for transmit completes and receives */ 1576 send_done_count = ntohl(stats->send_done_count); 1577 if (send_done_count != tx->pkt_done) 1578 myri10ge_tx_done(ss, (int)send_done_count); 1579 if (unlikely(i > myri10ge_max_irq_loops)) { 1580 netdev_warn(mgp->dev, "irq stuck?\n"); 1581 stats->valid = 0; 1582 schedule_work(&mgp->watchdog_work); 1583 } 1584 if (likely(stats->valid == 0)) 1585 break; 1586 cpu_relax(); 1587 barrier(); 1588 } 1589 1590 /* Only slice 0 updates stats */ 1591 if (ss == mgp->ss) 1592 myri10ge_check_statblock(mgp); 1593 1594 put_be32(htonl(3), ss->irq_claim + 1); 1595 return IRQ_HANDLED; 1596} 1597 1598static int 1599myri10ge_get_link_ksettings(struct net_device *netdev, 1600 struct ethtool_link_ksettings *cmd) 1601{ 1602 struct myri10ge_priv *mgp = netdev_priv(netdev); 1603 char *ptr; 1604 int i; 1605 1606 cmd->base.autoneg = AUTONEG_DISABLE; 1607 cmd->base.speed = SPEED_10000; 1608 cmd->base.duplex = DUPLEX_FULL; 1609 1610 /* 1611 * parse the product code to deterimine the interface type 1612 * (CX4, XFP, Quad Ribbon Fiber) by looking at the character 1613 * after the 3rd dash in the driver's cached copy of the 1614 * EEPROM's product code string. 1615 */ 1616 ptr = mgp->product_code_string; 1617 if (ptr == NULL) { 1618 netdev_err(netdev, "Missing product code\n"); 1619 return 0; 1620 } 1621 for (i = 0; i < 3; i++, ptr++) { 1622 ptr = strchr(ptr, '-'); 1623 if (ptr == NULL) { 1624 netdev_err(netdev, "Invalid product code %s\n", 1625 mgp->product_code_string); 1626 return 0; 1627 } 1628 } 1629 if (*ptr == '2') 1630 ptr++; 1631 if (*ptr == 'R' || *ptr == 'Q' || *ptr == 'S') { 1632 /* We've found either an XFP, quad ribbon fiber, or SFP+ */ 1633 cmd->base.port = PORT_FIBRE; 1634 ethtool_link_ksettings_add_link_mode(cmd, supported, FIBRE); 1635 ethtool_link_ksettings_add_link_mode(cmd, advertising, FIBRE); 1636 } else { 1637 cmd->base.port = PORT_OTHER; 1638 } 1639 1640 return 0; 1641} 1642 1643static void 1644myri10ge_get_drvinfo(struct net_device *netdev, struct ethtool_drvinfo *info) 1645{ 1646 struct myri10ge_priv *mgp = netdev_priv(netdev); 1647 1648 strlcpy(info->driver, "myri10ge", sizeof(info->driver)); 1649 strlcpy(info->version, MYRI10GE_VERSION_STR, sizeof(info->version)); 1650 strlcpy(info->fw_version, mgp->fw_version, sizeof(info->fw_version)); 1651 strlcpy(info->bus_info, pci_name(mgp->pdev), sizeof(info->bus_info)); 1652} 1653 1654static int 1655myri10ge_get_coalesce(struct net_device *netdev, struct ethtool_coalesce *coal) 1656{ 1657 struct myri10ge_priv *mgp = netdev_priv(netdev); 1658 1659 coal->rx_coalesce_usecs = mgp->intr_coal_delay; 1660 return 0; 1661} 1662 1663static int 1664myri10ge_set_coalesce(struct net_device *netdev, struct ethtool_coalesce *coal) 1665{ 1666 struct myri10ge_priv *mgp = netdev_priv(netdev); 1667 1668 mgp->intr_coal_delay = coal->rx_coalesce_usecs; 1669 put_be32(htonl(mgp->intr_coal_delay), mgp->intr_coal_delay_ptr); 1670 return 0; 1671} 1672 1673static void 1674myri10ge_get_pauseparam(struct net_device *netdev, 1675 struct ethtool_pauseparam *pause) 1676{ 1677 struct myri10ge_priv *mgp = netdev_priv(netdev); 1678 1679 pause->autoneg = 0; 1680 pause->rx_pause = mgp->pause; 1681 pause->tx_pause = mgp->pause; 1682} 1683 1684static int 1685myri10ge_set_pauseparam(struct net_device *netdev, 1686 struct ethtool_pauseparam *pause) 1687{ 1688 struct myri10ge_priv *mgp = netdev_priv(netdev); 1689 1690 if (pause->tx_pause != mgp->pause) 1691 return myri10ge_change_pause(mgp, pause->tx_pause); 1692 if (pause->rx_pause != mgp->pause) 1693 return myri10ge_change_pause(mgp, pause->rx_pause); 1694 if (pause->autoneg != 0) 1695 return -EINVAL; 1696 return 0; 1697} 1698 1699static void 1700myri10ge_get_ringparam(struct net_device *netdev, 1701 struct ethtool_ringparam *ring) 1702{ 1703 struct myri10ge_priv *mgp = netdev_priv(netdev); 1704 1705 ring->rx_mini_max_pending = mgp->ss[0].rx_small.mask + 1; 1706 ring->rx_max_pending = mgp->ss[0].rx_big.mask + 1; 1707 ring->rx_jumbo_max_pending = 0; 1708 ring->tx_max_pending = mgp->ss[0].tx.mask + 1; 1709 ring->rx_mini_pending = ring->rx_mini_max_pending; 1710 ring->rx_pending = ring->rx_max_pending; 1711 ring->rx_jumbo_pending = ring->rx_jumbo_max_pending; 1712 ring->tx_pending = ring->tx_max_pending; 1713} 1714 1715static const char myri10ge_gstrings_main_stats[][ETH_GSTRING_LEN] = { 1716 "rx_packets", "tx_packets", "rx_bytes", "tx_bytes", "rx_errors", 1717 "tx_errors", "rx_dropped", "tx_dropped", "multicast", "collisions", 1718 "rx_length_errors", "rx_over_errors", "rx_crc_errors", 1719 "rx_frame_errors", "rx_fifo_errors", "rx_missed_errors", 1720 "tx_aborted_errors", "tx_carrier_errors", "tx_fifo_errors", 1721 "tx_heartbeat_errors", "tx_window_errors", 1722 /* device-specific stats */ 1723 "tx_boundary", "irq", "MSI", "MSIX", 1724 "read_dma_bw_MBs", "write_dma_bw_MBs", "read_write_dma_bw_MBs", 1725 "serial_number", "watchdog_resets", 1726#ifdef CONFIG_MYRI10GE_DCA 1727 "dca_capable_firmware", "dca_device_present", 1728#endif 1729 "link_changes", "link_up", "dropped_link_overflow", 1730 "dropped_link_error_or_filtered", 1731 "dropped_pause", "dropped_bad_phy", "dropped_bad_crc32", 1732 "dropped_unicast_filtered", "dropped_multicast_filtered", 1733 "dropped_runt", "dropped_overrun", "dropped_no_small_buffer", 1734 "dropped_no_big_buffer" 1735}; 1736 1737static const char myri10ge_gstrings_slice_stats[][ETH_GSTRING_LEN] = { 1738 "----------- slice ---------", 1739 "tx_pkt_start", "tx_pkt_done", "tx_req", "tx_done", 1740 "rx_small_cnt", "rx_big_cnt", 1741 "wake_queue", "stop_queue", "tx_linearized", 1742}; 1743 1744#define MYRI10GE_NET_STATS_LEN 21 1745#define MYRI10GE_MAIN_STATS_LEN ARRAY_SIZE(myri10ge_gstrings_main_stats) 1746#define MYRI10GE_SLICE_STATS_LEN ARRAY_SIZE(myri10ge_gstrings_slice_stats) 1747 1748static void 1749myri10ge_get_strings(struct net_device *netdev, u32 stringset, u8 * data) 1750{ 1751 struct myri10ge_priv *mgp = netdev_priv(netdev); 1752 int i; 1753 1754 switch (stringset) { 1755 case ETH_SS_STATS: 1756 memcpy(data, *myri10ge_gstrings_main_stats, 1757 sizeof(myri10ge_gstrings_main_stats)); 1758 data += sizeof(myri10ge_gstrings_main_stats); 1759 for (i = 0; i < mgp->num_slices; i++) { 1760 memcpy(data, *myri10ge_gstrings_slice_stats, 1761 sizeof(myri10ge_gstrings_slice_stats)); 1762 data += sizeof(myri10ge_gstrings_slice_stats); 1763 } 1764 break; 1765 } 1766} 1767 1768static int myri10ge_get_sset_count(struct net_device *netdev, int sset) 1769{ 1770 struct myri10ge_priv *mgp = netdev_priv(netdev); 1771 1772 switch (sset) { 1773 case ETH_SS_STATS: 1774 return MYRI10GE_MAIN_STATS_LEN + 1775 mgp->num_slices * MYRI10GE_SLICE_STATS_LEN; 1776 default: 1777 return -EOPNOTSUPP; 1778 } 1779} 1780 1781static void 1782myri10ge_get_ethtool_stats(struct net_device *netdev, 1783 struct ethtool_stats *stats, u64 * data) 1784{ 1785 struct myri10ge_priv *mgp = netdev_priv(netdev); 1786 struct myri10ge_slice_state *ss; 1787 struct rtnl_link_stats64 link_stats; 1788 int slice; 1789 int i; 1790 1791 /* force stats update */ 1792 memset(&link_stats, 0, sizeof(link_stats)); 1793 (void)myri10ge_get_stats(netdev, &link_stats); 1794 for (i = 0; i < MYRI10GE_NET_STATS_LEN; i++) 1795 data[i] = ((u64 *)&link_stats)[i]; 1796 1797 data[i++] = (unsigned int)mgp->tx_boundary; 1798 data[i++] = (unsigned int)mgp->pdev->irq; 1799 data[i++] = (unsigned int)mgp->msi_enabled; 1800 data[i++] = (unsigned int)mgp->msix_enabled; 1801 data[i++] = (unsigned int)mgp->read_dma; 1802 data[i++] = (unsigned int)mgp->write_dma; 1803 data[i++] = (unsigned int)mgp->read_write_dma; 1804 data[i++] = (unsigned int)mgp->serial_number; 1805 data[i++] = (unsigned int)mgp->watchdog_resets; 1806#ifdef CONFIG_MYRI10GE_DCA 1807 data[i++] = (unsigned int)(mgp->ss[0].dca_tag != NULL); 1808 data[i++] = (unsigned int)(mgp->dca_enabled); 1809#endif 1810 data[i++] = (unsigned int)mgp->link_changes; 1811 1812 /* firmware stats are useful only in the first slice */ 1813 ss = &mgp->ss[0]; 1814 data[i++] = (unsigned int)ntohl(ss->fw_stats->link_up); 1815 data[i++] = (unsigned int)ntohl(ss->fw_stats->dropped_link_overflow); 1816 data[i++] = 1817 (unsigned int)ntohl(ss->fw_stats->dropped_link_error_or_filtered); 1818 data[i++] = (unsigned int)ntohl(ss->fw_stats->dropped_pause); 1819 data[i++] = (unsigned int)ntohl(ss->fw_stats->dropped_bad_phy); 1820 data[i++] = (unsigned int)ntohl(ss->fw_stats->dropped_bad_crc32); 1821 data[i++] = (unsigned int)ntohl(ss->fw_stats->dropped_unicast_filtered); 1822 data[i++] = 1823 (unsigned int)ntohl(ss->fw_stats->dropped_multicast_filtered); 1824 data[i++] = (unsigned int)ntohl(ss->fw_stats->dropped_runt); 1825 data[i++] = (unsigned int)ntohl(ss->fw_stats->dropped_overrun); 1826 data[i++] = (unsigned int)ntohl(ss->fw_stats->dropped_no_small_buffer); 1827 data[i++] = (unsigned int)ntohl(ss->fw_stats->dropped_no_big_buffer); 1828 1829 for (slice = 0; slice < mgp->num_slices; slice++) { 1830 ss = &mgp->ss[slice]; 1831 data[i++] = slice; 1832 data[i++] = (unsigned int)ss->tx.pkt_start; 1833 data[i++] = (unsigned int)ss->tx.pkt_done; 1834 data[i++] = (unsigned int)ss->tx.req; 1835 data[i++] = (unsigned int)ss->tx.done; 1836 data[i++] = (unsigned int)ss->rx_small.cnt; 1837 data[i++] = (unsigned int)ss->rx_big.cnt; 1838 data[i++] = (unsigned int)ss->tx.wake_queue; 1839 data[i++] = (unsigned int)ss->tx.stop_queue; 1840 data[i++] = (unsigned int)ss->tx.linearized; 1841 } 1842} 1843 1844static void myri10ge_set_msglevel(struct net_device *netdev, u32 value) 1845{ 1846 struct myri10ge_priv *mgp = netdev_priv(netdev); 1847 mgp->msg_enable = value; 1848} 1849 1850static u32 myri10ge_get_msglevel(struct net_device *netdev) 1851{ 1852 struct myri10ge_priv *mgp = netdev_priv(netdev); 1853 return mgp->msg_enable; 1854} 1855 1856/* 1857 * Use a low-level command to change the LED behavior. Rather than 1858 * blinking (which is the normal case), when identify is used, the 1859 * yellow LED turns solid. 1860 */ 1861static int myri10ge_led(struct myri10ge_priv *mgp, int on) 1862{ 1863 struct mcp_gen_header *hdr; 1864 struct device *dev = &mgp->pdev->dev; 1865 size_t hdr_off, pattern_off, hdr_len; 1866 u32 pattern = 0xfffffffe; 1867 1868 /* find running firmware header */ 1869 hdr_off = swab32(readl(mgp->sram + MCP_HEADER_PTR_OFFSET)); 1870 if ((hdr_off & 3) || hdr_off + sizeof(*hdr) > mgp->sram_size) { 1871 dev_err(dev, "Running firmware has bad header offset (%d)\n", 1872 (int)hdr_off); 1873 return -EIO; 1874 } 1875 hdr_len = swab32(readl(mgp->sram + hdr_off + 1876 offsetof(struct mcp_gen_header, header_length))); 1877 pattern_off = hdr_off + offsetof(struct mcp_gen_header, led_pattern); 1878 if (pattern_off >= (hdr_len + hdr_off)) { 1879 dev_info(dev, "Firmware does not support LED identification\n"); 1880 return -EINVAL; 1881 } 1882 if (!on) 1883 pattern = swab32(readl(mgp->sram + pattern_off + 4)); 1884 writel(swab32(pattern), mgp->sram + pattern_off); 1885 return 0; 1886} 1887 1888static int 1889myri10ge_phys_id(struct net_device *netdev, enum ethtool_phys_id_state state) 1890{ 1891 struct myri10ge_priv *mgp = netdev_priv(netdev); 1892 int rc; 1893 1894 switch (state) { 1895 case ETHTOOL_ID_ACTIVE: 1896 rc = myri10ge_led(mgp, 1); 1897 break; 1898 1899 case ETHTOOL_ID_INACTIVE: 1900 rc = myri10ge_led(mgp, 0); 1901 break; 1902 1903 default: 1904 rc = -EINVAL; 1905 } 1906 1907 return rc; 1908} 1909 1910static const struct ethtool_ops myri10ge_ethtool_ops = { 1911 .supported_coalesce_params = ETHTOOL_COALESCE_RX_USECS, 1912 .get_drvinfo = myri10ge_get_drvinfo, 1913 .get_coalesce = myri10ge_get_coalesce, 1914 .set_coalesce = myri10ge_set_coalesce, 1915 .get_pauseparam = myri10ge_get_pauseparam, 1916 .set_pauseparam = myri10ge_set_pauseparam, 1917 .get_ringparam = myri10ge_get_ringparam, 1918 .get_link = ethtool_op_get_link, 1919 .get_strings = myri10ge_get_strings, 1920 .get_sset_count = myri10ge_get_sset_count, 1921 .get_ethtool_stats = myri10ge_get_ethtool_stats, 1922 .set_msglevel = myri10ge_set_msglevel, 1923 .get_msglevel = myri10ge_get_msglevel, 1924 .set_phys_id = myri10ge_phys_id, 1925 .get_link_ksettings = myri10ge_get_link_ksettings, 1926}; 1927 1928static int myri10ge_allocate_rings(struct myri10ge_slice_state *ss) 1929{ 1930 struct myri10ge_priv *mgp = ss->mgp; 1931 struct myri10ge_cmd cmd; 1932 struct net_device *dev = mgp->dev; 1933 int tx_ring_size, rx_ring_size; 1934 int tx_ring_entries, rx_ring_entries; 1935 int i, slice, status; 1936 size_t bytes; 1937 1938 /* get ring sizes */ 1939 slice = ss - mgp->ss; 1940 cmd.data0 = slice; 1941 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_SEND_RING_SIZE, &cmd, 0); 1942 tx_ring_size = cmd.data0; 1943 cmd.data0 = slice; 1944 status |= myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_RX_RING_SIZE, &cmd, 0); 1945 if (status != 0) 1946 return status; 1947 rx_ring_size = cmd.data0; 1948 1949 tx_ring_entries = tx_ring_size / sizeof(struct mcp_kreq_ether_send); 1950 rx_ring_entries = rx_ring_size / sizeof(struct mcp_dma_addr); 1951 ss->tx.mask = tx_ring_entries - 1; 1952 ss->rx_small.mask = ss->rx_big.mask = rx_ring_entries - 1; 1953 1954 status = -ENOMEM; 1955 1956 /* allocate the host shadow rings */ 1957 1958 bytes = 8 + (MYRI10GE_MAX_SEND_DESC_TSO + 4) 1959 * sizeof(*ss->tx.req_list); 1960 ss->tx.req_bytes = kzalloc(bytes, GFP_KERNEL); 1961 if (ss->tx.req_bytes == NULL) 1962 goto abort_with_nothing; 1963 1964 /* ensure req_list entries are aligned to 8 bytes */ 1965 ss->tx.req_list = (struct mcp_kreq_ether_send *) 1966 ALIGN((unsigned long)ss->tx.req_bytes, 8); 1967 ss->tx.queue_active = 0; 1968 1969 bytes = rx_ring_entries * sizeof(*ss->rx_small.shadow); 1970 ss->rx_small.shadow = kzalloc(bytes, GFP_KERNEL); 1971 if (ss->rx_small.shadow == NULL) 1972 goto abort_with_tx_req_bytes; 1973 1974 bytes = rx_ring_entries * sizeof(*ss->rx_big.shadow); 1975 ss->rx_big.shadow = kzalloc(bytes, GFP_KERNEL); 1976 if (ss->rx_big.shadow == NULL) 1977 goto abort_with_rx_small_shadow; 1978 1979 /* allocate the host info rings */ 1980 1981 bytes = tx_ring_entries * sizeof(*ss->tx.info); 1982 ss->tx.info = kzalloc(bytes, GFP_KERNEL); 1983 if (ss->tx.info == NULL) 1984 goto abort_with_rx_big_shadow; 1985 1986 bytes = rx_ring_entries * sizeof(*ss->rx_small.info); 1987 ss->rx_small.info = kzalloc(bytes, GFP_KERNEL); 1988 if (ss->rx_small.info == NULL) 1989 goto abort_with_tx_info; 1990 1991 bytes = rx_ring_entries * sizeof(*ss->rx_big.info); 1992 ss->rx_big.info = kzalloc(bytes, GFP_KERNEL); 1993 if (ss->rx_big.info == NULL) 1994 goto abort_with_rx_small_info; 1995 1996 /* Fill the receive rings */ 1997 ss->rx_big.cnt = 0; 1998 ss->rx_small.cnt = 0; 1999 ss->rx_big.fill_cnt = 0; 2000 ss->rx_small.fill_cnt = 0; 2001 ss->rx_small.page_offset = MYRI10GE_ALLOC_SIZE; 2002 ss->rx_big.page_offset = MYRI10GE_ALLOC_SIZE; 2003 ss->rx_small.watchdog_needed = 0; 2004 ss->rx_big.watchdog_needed = 0; 2005 if (mgp->small_bytes == 0) { 2006 ss->rx_small.fill_cnt = ss->rx_small.mask + 1; 2007 } else { 2008 myri10ge_alloc_rx_pages(mgp, &ss->rx_small, 2009 mgp->small_bytes + MXGEFW_PAD, 0); 2010 } 2011 2012 if (ss->rx_small.fill_cnt < ss->rx_small.mask + 1) { 2013 netdev_err(dev, "slice-%d: alloced only %d small bufs\n", 2014 slice, ss->rx_small.fill_cnt); 2015 goto abort_with_rx_small_ring; 2016 } 2017 2018 myri10ge_alloc_rx_pages(mgp, &ss->rx_big, mgp->big_bytes, 0); 2019 if (ss->rx_big.fill_cnt < ss->rx_big.mask + 1) { 2020 netdev_err(dev, "slice-%d: alloced only %d big bufs\n", 2021 slice, ss->rx_big.fill_cnt); 2022 goto abort_with_rx_big_ring; 2023 } 2024 2025 return 0; 2026 2027abort_with_rx_big_ring: 2028 for (i = ss->rx_big.cnt; i < ss->rx_big.fill_cnt; i++) { 2029 int idx = i & ss->rx_big.mask; 2030 myri10ge_unmap_rx_page(mgp->pdev, &ss->rx_big.info[idx], 2031 mgp->big_bytes); 2032 put_page(ss->rx_big.info[idx].page); 2033 } 2034 2035abort_with_rx_small_ring: 2036 if (mgp->small_bytes == 0) 2037 ss->rx_small.fill_cnt = ss->rx_small.cnt; 2038 for (i = ss->rx_small.cnt; i < ss->rx_small.fill_cnt; i++) { 2039 int idx = i & ss->rx_small.mask; 2040 myri10ge_unmap_rx_page(mgp->pdev, &ss->rx_small.info[idx], 2041 mgp->small_bytes + MXGEFW_PAD); 2042 put_page(ss->rx_small.info[idx].page); 2043 } 2044 2045 kfree(ss->rx_big.info); 2046 2047abort_with_rx_small_info: 2048 kfree(ss->rx_small.info); 2049 2050abort_with_tx_info: 2051 kfree(ss->tx.info); 2052 2053abort_with_rx_big_shadow: 2054 kfree(ss->rx_big.shadow); 2055 2056abort_with_rx_small_shadow: 2057 kfree(ss->rx_small.shadow); 2058 2059abort_with_tx_req_bytes: 2060 kfree(ss->tx.req_bytes); 2061 ss->tx.req_bytes = NULL; 2062 ss->tx.req_list = NULL; 2063 2064abort_with_nothing: 2065 return status; 2066} 2067 2068static void myri10ge_free_rings(struct myri10ge_slice_state *ss) 2069{ 2070 struct myri10ge_priv *mgp = ss->mgp; 2071 struct sk_buff *skb; 2072 struct myri10ge_tx_buf *tx; 2073 int i, len, idx; 2074 2075 /* If not allocated, skip it */ 2076 if (ss->tx.req_list == NULL) 2077 return; 2078 2079 for (i = ss->rx_big.cnt; i < ss->rx_big.fill_cnt; i++) { 2080 idx = i & ss->rx_big.mask; 2081 if (i == ss->rx_big.fill_cnt - 1) 2082 ss->rx_big.info[idx].page_offset = MYRI10GE_ALLOC_SIZE; 2083 myri10ge_unmap_rx_page(mgp->pdev, &ss->rx_big.info[idx], 2084 mgp->big_bytes); 2085 put_page(ss->rx_big.info[idx].page); 2086 } 2087 2088 if (mgp->small_bytes == 0) 2089 ss->rx_small.fill_cnt = ss->rx_small.cnt; 2090 for (i = ss->rx_small.cnt; i < ss->rx_small.fill_cnt; i++) { 2091 idx = i & ss->rx_small.mask; 2092 if (i == ss->rx_small.fill_cnt - 1) 2093 ss->rx_small.info[idx].page_offset = 2094 MYRI10GE_ALLOC_SIZE; 2095 myri10ge_unmap_rx_page(mgp->pdev, &ss->rx_small.info[idx], 2096 mgp->small_bytes + MXGEFW_PAD); 2097 put_page(ss->rx_small.info[idx].page); 2098 } 2099 tx = &ss->tx; 2100 while (tx->done != tx->req) { 2101 idx = tx->done & tx->mask; 2102 skb = tx->info[idx].skb; 2103 2104 /* Mark as free */ 2105 tx->info[idx].skb = NULL; 2106 tx->done++; 2107 len = dma_unmap_len(&tx->info[idx], len); 2108 dma_unmap_len_set(&tx->info[idx], len, 0); 2109 if (skb) { 2110 ss->stats.tx_dropped++; 2111 dev_kfree_skb_any(skb); 2112 if (len) 2113 pci_unmap_single(mgp->pdev, 2114 dma_unmap_addr(&tx->info[idx], 2115 bus), len, 2116 PCI_DMA_TODEVICE); 2117 } else { 2118 if (len) 2119 pci_unmap_page(mgp->pdev, 2120 dma_unmap_addr(&tx->info[idx], 2121 bus), len, 2122 PCI_DMA_TODEVICE); 2123 } 2124 } 2125 kfree(ss->rx_big.info); 2126 2127 kfree(ss->rx_small.info); 2128 2129 kfree(ss->tx.info); 2130 2131 kfree(ss->rx_big.shadow); 2132 2133 kfree(ss->rx_small.shadow); 2134 2135 kfree(ss->tx.req_bytes); 2136 ss->tx.req_bytes = NULL; 2137 ss->tx.req_list = NULL; 2138} 2139 2140static int myri10ge_request_irq(struct myri10ge_priv *mgp) 2141{ 2142 struct pci_dev *pdev = mgp->pdev; 2143 struct myri10ge_slice_state *ss; 2144 struct net_device *netdev = mgp->dev; 2145 int i; 2146 int status; 2147 2148 mgp->msi_enabled = 0; 2149 mgp->msix_enabled = 0; 2150 status = 0; 2151 if (myri10ge_msi) { 2152 if (mgp->num_slices > 1) { 2153 status = pci_enable_msix_range(pdev, mgp->msix_vectors, 2154 mgp->num_slices, mgp->num_slices); 2155 if (status < 0) { 2156 dev_err(&pdev->dev, 2157 "Error %d setting up MSI-X\n", status); 2158 return status; 2159 } 2160 mgp->msix_enabled = 1; 2161 } 2162 if (mgp->msix_enabled == 0) { 2163 status = pci_enable_msi(pdev); 2164 if (status != 0) { 2165 dev_err(&pdev->dev, 2166 "Error %d setting up MSI; falling back to xPIC\n", 2167 status); 2168 } else { 2169 mgp->msi_enabled = 1; 2170 } 2171 } 2172 } 2173 if (mgp->msix_enabled) { 2174 for (i = 0; i < mgp->num_slices; i++) { 2175 ss = &mgp->ss[i]; 2176 snprintf(ss->irq_desc, sizeof(ss->irq_desc), 2177 "%s:slice-%d", netdev->name, i); 2178 status = request_irq(mgp->msix_vectors[i].vector, 2179 myri10ge_intr, 0, ss->irq_desc, 2180 ss); 2181 if (status != 0) { 2182 dev_err(&pdev->dev, 2183 "slice %d failed to allocate IRQ\n", i); 2184 i--; 2185 while (i >= 0) { 2186 free_irq(mgp->msix_vectors[i].vector, 2187 &mgp->ss[i]); 2188 i--; 2189 } 2190 pci_disable_msix(pdev); 2191 return status; 2192 } 2193 } 2194 } else { 2195 status = request_irq(pdev->irq, myri10ge_intr, IRQF_SHARED, 2196 mgp->dev->name, &mgp->ss[0]); 2197 if (status != 0) { 2198 dev_err(&pdev->dev, "failed to allocate IRQ\n"); 2199 if (mgp->msi_enabled) 2200 pci_disable_msi(pdev); 2201 } 2202 } 2203 return status; 2204} 2205 2206static void myri10ge_free_irq(struct myri10ge_priv *mgp) 2207{ 2208 struct pci_dev *pdev = mgp->pdev; 2209 int i; 2210 2211 if (mgp->msix_enabled) { 2212 for (i = 0; i < mgp->num_slices; i++) 2213 free_irq(mgp->msix_vectors[i].vector, &mgp->ss[i]); 2214 } else { 2215 free_irq(pdev->irq, &mgp->ss[0]); 2216 } 2217 if (mgp->msi_enabled) 2218 pci_disable_msi(pdev); 2219 if (mgp->msix_enabled) 2220 pci_disable_msix(pdev); 2221} 2222 2223static int myri10ge_get_txrx(struct myri10ge_priv *mgp, int slice) 2224{ 2225 struct myri10ge_cmd cmd; 2226 struct myri10ge_slice_state *ss; 2227 int status; 2228 2229 ss = &mgp->ss[slice]; 2230 status = 0; 2231 if (slice == 0 || (mgp->dev->real_num_tx_queues > 1)) { 2232 cmd.data0 = slice; 2233 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_SEND_OFFSET, 2234 &cmd, 0); 2235 ss->tx.lanai = (struct mcp_kreq_ether_send __iomem *) 2236 (mgp->sram + cmd.data0); 2237 } 2238 cmd.data0 = slice; 2239 status |= myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_SMALL_RX_OFFSET, 2240 &cmd, 0); 2241 ss->rx_small.lanai = (struct mcp_kreq_ether_recv __iomem *) 2242 (mgp->sram + cmd.data0); 2243 2244 cmd.data0 = slice; 2245 status |= myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_BIG_RX_OFFSET, &cmd, 0); 2246 ss->rx_big.lanai = (struct mcp_kreq_ether_recv __iomem *) 2247 (mgp->sram + cmd.data0); 2248 2249 ss->tx.send_go = (__iomem __be32 *) 2250 (mgp->sram + MXGEFW_ETH_SEND_GO + 64 * slice); 2251 ss->tx.send_stop = (__iomem __be32 *) 2252 (mgp->sram + MXGEFW_ETH_SEND_STOP + 64 * slice); 2253 return status; 2254 2255} 2256 2257static int myri10ge_set_stats(struct myri10ge_priv *mgp, int slice) 2258{ 2259 struct myri10ge_cmd cmd; 2260 struct myri10ge_slice_state *ss; 2261 int status; 2262 2263 ss = &mgp->ss[slice]; 2264 cmd.data0 = MYRI10GE_LOWPART_TO_U32(ss->fw_stats_bus); 2265 cmd.data1 = MYRI10GE_HIGHPART_TO_U32(ss->fw_stats_bus); 2266 cmd.data2 = sizeof(struct mcp_irq_data) | (slice << 16); 2267 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_SET_STATS_DMA_V2, &cmd, 0); 2268 if (status == -ENOSYS) { 2269 dma_addr_t bus = ss->fw_stats_bus; 2270 if (slice != 0) 2271 return -EINVAL; 2272 bus += offsetof(struct mcp_irq_data, send_done_count); 2273 cmd.data0 = MYRI10GE_LOWPART_TO_U32(bus); 2274 cmd.data1 = MYRI10GE_HIGHPART_TO_U32(bus); 2275 status = myri10ge_send_cmd(mgp, 2276 MXGEFW_CMD_SET_STATS_DMA_OBSOLETE, 2277 &cmd, 0); 2278 /* Firmware cannot support multicast without STATS_DMA_V2 */ 2279 mgp->fw_multicast_support = 0; 2280 } else { 2281 mgp->fw_multicast_support = 1; 2282 } 2283 return 0; 2284} 2285 2286static int myri10ge_open(struct net_device *dev) 2287{ 2288 struct myri10ge_slice_state *ss; 2289 struct myri10ge_priv *mgp = netdev_priv(dev); 2290 struct myri10ge_cmd cmd; 2291 int i, status, big_pow2, slice; 2292 u8 __iomem *itable; 2293 2294 if (mgp->running != MYRI10GE_ETH_STOPPED) 2295 return -EBUSY; 2296 2297 mgp->running = MYRI10GE_ETH_STARTING; 2298 status = myri10ge_reset(mgp); 2299 if (status != 0) { 2300 netdev_err(dev, "failed reset\n"); 2301 goto abort_with_nothing; 2302 } 2303 2304 if (mgp->num_slices > 1) { 2305 cmd.data0 = mgp->num_slices; 2306 cmd.data1 = MXGEFW_SLICE_INTR_MODE_ONE_PER_SLICE; 2307 if (mgp->dev->real_num_tx_queues > 1) 2308 cmd.data1 |= MXGEFW_SLICE_ENABLE_MULTIPLE_TX_QUEUES; 2309 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_ENABLE_RSS_QUEUES, 2310 &cmd, 0); 2311 if (status != 0) { 2312 netdev_err(dev, "failed to set number of slices\n"); 2313 goto abort_with_nothing; 2314 } 2315 /* setup the indirection table */ 2316 cmd.data0 = mgp->num_slices; 2317 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_SET_RSS_TABLE_SIZE, 2318 &cmd, 0); 2319 2320 status |= myri10ge_send_cmd(mgp, 2321 MXGEFW_CMD_GET_RSS_TABLE_OFFSET, 2322 &cmd, 0); 2323 if (status != 0) { 2324 netdev_err(dev, "failed to setup rss tables\n"); 2325 goto abort_with_nothing; 2326 } 2327 2328 /* just enable an identity mapping */ 2329 itable = mgp->sram + cmd.data0; 2330 for (i = 0; i < mgp->num_slices; i++) 2331 __raw_writeb(i, &itable[i]); 2332 2333 cmd.data0 = 1; 2334 cmd.data1 = myri10ge_rss_hash; 2335 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_SET_RSS_ENABLE, 2336 &cmd, 0); 2337 if (status != 0) { 2338 netdev_err(dev, "failed to enable slices\n"); 2339 goto abort_with_nothing; 2340 } 2341 } 2342 2343 status = myri10ge_request_irq(mgp); 2344 if (status != 0) 2345 goto abort_with_nothing; 2346 2347 /* decide what small buffer size to use. For good TCP rx 2348 * performance, it is important to not receive 1514 byte 2349 * frames into jumbo buffers, as it confuses the socket buffer 2350 * accounting code, leading to drops and erratic performance. 2351 */ 2352 2353 if (dev->mtu <= ETH_DATA_LEN) 2354 /* enough for a TCP header */ 2355 mgp->small_bytes = (128 > SMP_CACHE_BYTES) 2356 ? (128 - MXGEFW_PAD) 2357 : (SMP_CACHE_BYTES - MXGEFW_PAD); 2358 else 2359 /* enough for a vlan encapsulated ETH_DATA_LEN frame */ 2360 mgp->small_bytes = VLAN_ETH_FRAME_LEN; 2361 2362 /* Override the small buffer size? */ 2363 if (myri10ge_small_bytes >= 0) 2364 mgp->small_bytes = myri10ge_small_bytes; 2365 2366 /* Firmware needs the big buff size as a power of 2. Lie and 2367 * tell him the buffer is larger, because we only use 1 2368 * buffer/pkt, and the mtu will prevent overruns. 2369 */ 2370 big_pow2 = dev->mtu + ETH_HLEN + VLAN_HLEN + MXGEFW_PAD; 2371 if (big_pow2 < MYRI10GE_ALLOC_SIZE / 2) { 2372 while (!is_power_of_2(big_pow2)) 2373 big_pow2++; 2374 mgp->big_bytes = dev->mtu + ETH_HLEN + VLAN_HLEN + MXGEFW_PAD; 2375 } else { 2376 big_pow2 = MYRI10GE_ALLOC_SIZE; 2377 mgp->big_bytes = big_pow2; 2378 } 2379 2380 /* setup the per-slice data structures */ 2381 for (slice = 0; slice < mgp->num_slices; slice++) { 2382 ss = &mgp->ss[slice]; 2383 2384 status = myri10ge_get_txrx(mgp, slice); 2385 if (status != 0) { 2386 netdev_err(dev, "failed to get ring sizes or locations\n"); 2387 goto abort_with_rings; 2388 } 2389 status = myri10ge_allocate_rings(ss); 2390 if (status != 0) 2391 goto abort_with_rings; 2392 2393 /* only firmware which supports multiple TX queues 2394 * supports setting up the tx stats on non-zero 2395 * slices */ 2396 if (slice == 0 || mgp->dev->real_num_tx_queues > 1) 2397 status = myri10ge_set_stats(mgp, slice); 2398 if (status) { 2399 netdev_err(dev, "Couldn't set stats DMA\n"); 2400 goto abort_with_rings; 2401 } 2402 2403 /* must happen prior to any irq */ 2404 napi_enable(&(ss)->napi); 2405 } 2406 2407 /* now give firmware buffers sizes, and MTU */ 2408 cmd.data0 = dev->mtu + ETH_HLEN + VLAN_HLEN; 2409 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_SET_MTU, &cmd, 0); 2410 cmd.data0 = mgp->small_bytes; 2411 status |= 2412 myri10ge_send_cmd(mgp, MXGEFW_CMD_SET_SMALL_BUFFER_SIZE, &cmd, 0); 2413 cmd.data0 = big_pow2; 2414 status |= 2415 myri10ge_send_cmd(mgp, MXGEFW_CMD_SET_BIG_BUFFER_SIZE, &cmd, 0); 2416 if (status) { 2417 netdev_err(dev, "Couldn't set buffer sizes\n"); 2418 goto abort_with_rings; 2419 } 2420 2421 /* 2422 * Set Linux style TSO mode; this is needed only on newer 2423 * firmware versions. Older versions default to Linux 2424 * style TSO 2425 */ 2426 cmd.data0 = 0; 2427 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_SET_TSO_MODE, &cmd, 0); 2428 if (status && status != -ENOSYS) { 2429 netdev_err(dev, "Couldn't set TSO mode\n"); 2430 goto abort_with_rings; 2431 } 2432 2433 mgp->link_state = ~0U; 2434 mgp->rdma_tags_available = 15; 2435 2436 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_ETHERNET_UP, &cmd, 0); 2437 if (status) { 2438 netdev_err(dev, "Couldn't bring up link\n"); 2439 goto abort_with_rings; 2440 } 2441 2442 mgp->running = MYRI10GE_ETH_RUNNING; 2443 mgp->watchdog_timer.expires = jiffies + myri10ge_watchdog_timeout * HZ; 2444 add_timer(&mgp->watchdog_timer); 2445 netif_tx_wake_all_queues(dev); 2446 2447 return 0; 2448 2449abort_with_rings: 2450 while (slice) { 2451 slice--; 2452 napi_disable(&mgp->ss[slice].napi); 2453 } 2454 for (i = 0; i < mgp->num_slices; i++) 2455 myri10ge_free_rings(&mgp->ss[i]); 2456 2457 myri10ge_free_irq(mgp); 2458 2459abort_with_nothing: 2460 mgp->running = MYRI10GE_ETH_STOPPED; 2461 return -ENOMEM; 2462} 2463 2464static int myri10ge_close(struct net_device *dev) 2465{ 2466 struct myri10ge_priv *mgp = netdev_priv(dev); 2467 struct myri10ge_cmd cmd; 2468 int status, old_down_cnt; 2469 int i; 2470 2471 if (mgp->running != MYRI10GE_ETH_RUNNING) 2472 return 0; 2473 2474 if (mgp->ss[0].tx.req_bytes == NULL) 2475 return 0; 2476 2477 del_timer_sync(&mgp->watchdog_timer); 2478 mgp->running = MYRI10GE_ETH_STOPPING; 2479 for (i = 0; i < mgp->num_slices; i++) 2480 napi_disable(&mgp->ss[i].napi); 2481 2482 netif_carrier_off(dev); 2483 2484 netif_tx_stop_all_queues(dev); 2485 if (mgp->rebooted == 0) { 2486 old_down_cnt = mgp->down_cnt; 2487 mb(); 2488 status = 2489 myri10ge_send_cmd(mgp, MXGEFW_CMD_ETHERNET_DOWN, &cmd, 0); 2490 if (status) 2491 netdev_err(dev, "Couldn't bring down link\n"); 2492 2493 wait_event_timeout(mgp->down_wq, old_down_cnt != mgp->down_cnt, 2494 HZ); 2495 if (old_down_cnt == mgp->down_cnt) 2496 netdev_err(dev, "never got down irq\n"); 2497 } 2498 netif_tx_disable(dev); 2499 myri10ge_free_irq(mgp); 2500 for (i = 0; i < mgp->num_slices; i++) 2501 myri10ge_free_rings(&mgp->ss[i]); 2502 2503 mgp->running = MYRI10GE_ETH_STOPPED; 2504 return 0; 2505} 2506 2507/* copy an array of struct mcp_kreq_ether_send's to the mcp. Copy 2508 * backwards one at a time and handle ring wraps */ 2509 2510static inline void 2511myri10ge_submit_req_backwards(struct myri10ge_tx_buf *tx, 2512 struct mcp_kreq_ether_send *src, int cnt) 2513{ 2514 int idx, starting_slot; 2515 starting_slot = tx->req; 2516 while (cnt > 1) { 2517 cnt--; 2518 idx = (starting_slot + cnt) & tx->mask; 2519 myri10ge_pio_copy(&tx->lanai[idx], &src[cnt], sizeof(*src)); 2520 mb(); 2521 } 2522} 2523 2524/* 2525 * copy an array of struct mcp_kreq_ether_send's to the mcp. Copy 2526 * at most 32 bytes at a time, so as to avoid involving the software 2527 * pio handler in the nic. We re-write the first segment's flags 2528 * to mark them valid only after writing the entire chain. 2529 */ 2530 2531static inline void 2532myri10ge_submit_req(struct myri10ge_tx_buf *tx, struct mcp_kreq_ether_send *src, 2533 int cnt) 2534{ 2535 int idx, i; 2536 struct mcp_kreq_ether_send __iomem *dstp, *dst; 2537 struct mcp_kreq_ether_send *srcp; 2538 u8 last_flags; 2539 2540 idx = tx->req & tx->mask; 2541 2542 last_flags = src->flags; 2543 src->flags = 0; 2544 mb(); 2545 dst = dstp = &tx->lanai[idx]; 2546 srcp = src; 2547 2548 if ((idx + cnt) < tx->mask) { 2549 for (i = 0; i < (cnt - 1); i += 2) { 2550 myri10ge_pio_copy(dstp, srcp, 2 * sizeof(*src)); 2551 mb(); /* force write every 32 bytes */ 2552 srcp += 2; 2553 dstp += 2; 2554 } 2555 } else { 2556 /* submit all but the first request, and ensure 2557 * that it is submitted below */ 2558 myri10ge_submit_req_backwards(tx, src, cnt); 2559 i = 0; 2560 } 2561 if (i < cnt) { 2562 /* submit the first request */ 2563 myri10ge_pio_copy(dstp, srcp, sizeof(*src)); 2564 mb(); /* barrier before setting valid flag */ 2565 } 2566 2567 /* re-write the last 32-bits with the valid flags */ 2568 src->flags = last_flags; 2569 put_be32(*((__be32 *) src + 3), (__be32 __iomem *) dst + 3); 2570 tx->req += cnt; 2571 mb(); 2572} 2573 2574static void myri10ge_unmap_tx_dma(struct myri10ge_priv *mgp, 2575 struct myri10ge_tx_buf *tx, int idx) 2576{ 2577 unsigned int len; 2578 int last_idx; 2579 2580 /* Free any DMA resources we've alloced and clear out the skb slot */ 2581 last_idx = (idx + 1) & tx->mask; 2582 idx = tx->req & tx->mask; 2583 do { 2584 len = dma_unmap_len(&tx->info[idx], len); 2585 if (len) { 2586 if (tx->info[idx].skb != NULL) 2587 pci_unmap_single(mgp->pdev, 2588 dma_unmap_addr(&tx->info[idx], 2589 bus), len, 2590 PCI_DMA_TODEVICE); 2591 else 2592 pci_unmap_page(mgp->pdev, 2593 dma_unmap_addr(&tx->info[idx], 2594 bus), len, 2595 PCI_DMA_TODEVICE); 2596 dma_unmap_len_set(&tx->info[idx], len, 0); 2597 tx->info[idx].skb = NULL; 2598 } 2599 idx = (idx + 1) & tx->mask; 2600 } while (idx != last_idx); 2601} 2602 2603/* 2604 * Transmit a packet. We need to split the packet so that a single 2605 * segment does not cross myri10ge->tx_boundary, so this makes segment 2606 * counting tricky. So rather than try to count segments up front, we 2607 * just give up if there are too few segments to hold a reasonably 2608 * fragmented packet currently available. If we run 2609 * out of segments while preparing a packet for DMA, we just linearize 2610 * it and try again. 2611 */ 2612 2613static netdev_tx_t myri10ge_xmit(struct sk_buff *skb, 2614 struct net_device *dev) 2615{ 2616 struct myri10ge_priv *mgp = netdev_priv(dev); 2617 struct myri10ge_slice_state *ss; 2618 struct mcp_kreq_ether_send *req; 2619 struct myri10ge_tx_buf *tx; 2620 skb_frag_t *frag; 2621 struct netdev_queue *netdev_queue; 2622 dma_addr_t bus; 2623 u32 low; 2624 __be32 high_swapped; 2625 unsigned int len; 2626 int idx, avail, frag_cnt, frag_idx, count, mss, max_segments; 2627 u16 pseudo_hdr_offset, cksum_offset, queue; 2628 int cum_len, seglen, boundary, rdma_count; 2629 u8 flags, odd_flag; 2630 2631 queue = skb_get_queue_mapping(skb); 2632 ss = &mgp->ss[queue]; 2633 netdev_queue = netdev_get_tx_queue(mgp->dev, queue); 2634 tx = &ss->tx; 2635 2636again: 2637 req = tx->req_list; 2638 avail = tx->mask - 1 - (tx->req - tx->done); 2639 2640 mss = 0; 2641 max_segments = MXGEFW_MAX_SEND_DESC; 2642 2643 if (skb_is_gso(skb)) { 2644 mss = skb_shinfo(skb)->gso_size; 2645 max_segments = MYRI10GE_MAX_SEND_DESC_TSO; 2646 } 2647 2648 if ((unlikely(avail < max_segments))) { 2649 /* we are out of transmit resources */ 2650 tx->stop_queue++; 2651 netif_tx_stop_queue(netdev_queue); 2652 return NETDEV_TX_BUSY; 2653 } 2654 2655 /* Setup checksum offloading, if needed */ 2656 cksum_offset = 0; 2657 pseudo_hdr_offset = 0; 2658 odd_flag = 0; 2659 flags = (MXGEFW_FLAGS_NO_TSO | MXGEFW_FLAGS_FIRST); 2660 if (likely(skb->ip_summed == CHECKSUM_PARTIAL)) { 2661 cksum_offset = skb_checksum_start_offset(skb); 2662 pseudo_hdr_offset = cksum_offset + skb->csum_offset; 2663 /* If the headers are excessively large, then we must 2664 * fall back to a software checksum */ 2665 if (unlikely(!mss && (cksum_offset > 255 || 2666 pseudo_hdr_offset > 127))) { 2667 if (skb_checksum_help(skb)) 2668 goto drop; 2669 cksum_offset = 0; 2670 pseudo_hdr_offset = 0; 2671 } else { 2672 odd_flag = MXGEFW_FLAGS_ALIGN_ODD; 2673 flags |= MXGEFW_FLAGS_CKSUM; 2674 } 2675 } 2676 2677 cum_len = 0; 2678 2679 if (mss) { /* TSO */ 2680 /* this removes any CKSUM flag from before */ 2681 flags = (MXGEFW_FLAGS_TSO_HDR | MXGEFW_FLAGS_FIRST); 2682 2683 /* negative cum_len signifies to the 2684 * send loop that we are still in the 2685 * header portion of the TSO packet. 2686 * TSO header can be at most 1KB long */ 2687 cum_len = -(skb_transport_offset(skb) + tcp_hdrlen(skb)); 2688 2689 /* for IPv6 TSO, the checksum offset stores the 2690 * TCP header length, to save the firmware from 2691 * the need to parse the headers */ 2692 if (skb_is_gso_v6(skb)) { 2693 cksum_offset = tcp_hdrlen(skb); 2694 /* Can only handle headers <= max_tso6 long */ 2695 if (unlikely(-cum_len > mgp->max_tso6)) 2696 return myri10ge_sw_tso(skb, dev); 2697 } 2698 /* for TSO, pseudo_hdr_offset holds mss. 2699 * The firmware figures out where to put 2700 * the checksum by parsing the header. */ 2701 pseudo_hdr_offset = mss; 2702 } else 2703 /* Mark small packets, and pad out tiny packets */ 2704 if (skb->len <= MXGEFW_SEND_SMALL_SIZE) { 2705 flags |= MXGEFW_FLAGS_SMALL; 2706 2707 /* pad frames to at least ETH_ZLEN bytes */ 2708 if (eth_skb_pad(skb)) { 2709 /* The packet is gone, so we must 2710 * return 0 */ 2711 ss->stats.tx_dropped += 1; 2712 return NETDEV_TX_OK; 2713 } 2714 } 2715 2716 /* map the skb for DMA */ 2717 len = skb_headlen(skb); 2718 bus = pci_map_single(mgp->pdev, skb->data, len, PCI_DMA_TODEVICE); 2719 if (unlikely(pci_dma_mapping_error(mgp->pdev, bus))) 2720 goto drop; 2721 2722 idx = tx->req & tx->mask; 2723 tx->info[idx].skb = skb; 2724 dma_unmap_addr_set(&tx->info[idx], bus, bus); 2725 dma_unmap_len_set(&tx->info[idx], len, len); 2726 2727 frag_cnt = skb_shinfo(skb)->nr_frags; 2728 frag_idx = 0; 2729 count = 0; 2730 rdma_count = 0; 2731 2732 /* "rdma_count" is the number of RDMAs belonging to the 2733 * current packet BEFORE the current send request. For 2734 * non-TSO packets, this is equal to "count". 2735 * For TSO packets, rdma_count needs to be reset 2736 * to 0 after a segment cut. 2737 * 2738 * The rdma_count field of the send request is 2739 * the number of RDMAs of the packet starting at 2740 * that request. For TSO send requests with one ore more cuts 2741 * in the middle, this is the number of RDMAs starting 2742 * after the last cut in the request. All previous 2743 * segments before the last cut implicitly have 1 RDMA. 2744 * 2745 * Since the number of RDMAs is not known beforehand, 2746 * it must be filled-in retroactively - after each 2747 * segmentation cut or at the end of the entire packet. 2748 */ 2749 2750 while (1) { 2751 /* Break the SKB or Fragment up into pieces which 2752 * do not cross mgp->tx_boundary */ 2753 low = MYRI10GE_LOWPART_TO_U32(bus); 2754 high_swapped = htonl(MYRI10GE_HIGHPART_TO_U32(bus)); 2755 while (len) { 2756 u8 flags_next; 2757 int cum_len_next; 2758 2759 if (unlikely(count == max_segments)) 2760 goto abort_linearize; 2761 2762 boundary = 2763 (low + mgp->tx_boundary) & ~(mgp->tx_boundary - 1); 2764 seglen = boundary - low; 2765 if (seglen > len) 2766 seglen = len; 2767 flags_next = flags & ~MXGEFW_FLAGS_FIRST; 2768 cum_len_next = cum_len + seglen; 2769 if (mss) { /* TSO */ 2770 (req - rdma_count)->rdma_count = rdma_count + 1; 2771 2772 if (likely(cum_len >= 0)) { /* payload */ 2773 int next_is_first, chop; 2774 2775 chop = (cum_len_next > mss); 2776 cum_len_next = cum_len_next % mss; 2777 next_is_first = (cum_len_next == 0); 2778 flags |= chop * MXGEFW_FLAGS_TSO_CHOP; 2779 flags_next |= next_is_first * 2780 MXGEFW_FLAGS_FIRST; 2781 rdma_count |= -(chop | next_is_first); 2782 rdma_count += chop & ~next_is_first; 2783 } else if (likely(cum_len_next >= 0)) { /* header ends */ 2784 int small; 2785 2786 rdma_count = -1; 2787 cum_len_next = 0; 2788 seglen = -cum_len; 2789 small = (mss <= MXGEFW_SEND_SMALL_SIZE); 2790 flags_next = MXGEFW_FLAGS_TSO_PLD | 2791 MXGEFW_FLAGS_FIRST | 2792 (small * MXGEFW_FLAGS_SMALL); 2793 } 2794 } 2795 req->addr_high = high_swapped; 2796 req->addr_low = htonl(low); 2797 req->pseudo_hdr_offset = htons(pseudo_hdr_offset); 2798 req->pad = 0; /* complete solid 16-byte block; does this matter? */ 2799 req->rdma_count = 1; 2800 req->length = htons(seglen); 2801 req->cksum_offset = cksum_offset; 2802 req->flags = flags | ((cum_len & 1) * odd_flag); 2803 2804 low += seglen; 2805 len -= seglen; 2806 cum_len = cum_len_next; 2807 flags = flags_next; 2808 req++; 2809 count++; 2810 rdma_count++; 2811 if (cksum_offset != 0 && !(mss && skb_is_gso_v6(skb))) { 2812 if (unlikely(cksum_offset > seglen)) 2813 cksum_offset -= seglen; 2814 else 2815 cksum_offset = 0; 2816 } 2817 } 2818 if (frag_idx == frag_cnt) 2819 break; 2820 2821 /* map next fragment for DMA */ 2822 frag = &skb_shinfo(skb)->frags[frag_idx]; 2823 frag_idx++; 2824 len = skb_frag_size(frag); 2825 bus = skb_frag_dma_map(&mgp->pdev->dev, frag, 0, len, 2826 DMA_TO_DEVICE); 2827 if (unlikely(pci_dma_mapping_error(mgp->pdev, bus))) { 2828 myri10ge_unmap_tx_dma(mgp, tx, idx); 2829 goto drop; 2830 } 2831 idx = (count + tx->req) & tx->mask; 2832 dma_unmap_addr_set(&tx->info[idx], bus, bus); 2833 dma_unmap_len_set(&tx->info[idx], len, len); 2834 } 2835 2836 (req - rdma_count)->rdma_count = rdma_count; 2837 if (mss) 2838 do { 2839 req--; 2840 req->flags |= MXGEFW_FLAGS_TSO_LAST; 2841 } while (!(req->flags & (MXGEFW_FLAGS_TSO_CHOP | 2842 MXGEFW_FLAGS_FIRST))); 2843 idx = ((count - 1) + tx->req) & tx->mask; 2844 tx->info[idx].last = 1; 2845 myri10ge_submit_req(tx, tx->req_list, count); 2846 /* if using multiple tx queues, make sure NIC polls the 2847 * current slice */ 2848 if ((mgp->dev->real_num_tx_queues > 1) && tx->queue_active == 0) { 2849 tx->queue_active = 1; 2850 put_be32(htonl(1), tx->send_go); 2851 mb(); 2852 } 2853 tx->pkt_start++; 2854 if ((avail - count) < MXGEFW_MAX_SEND_DESC) { 2855 tx->stop_queue++; 2856 netif_tx_stop_queue(netdev_queue); 2857 } 2858 return NETDEV_TX_OK; 2859 2860abort_linearize: 2861 myri10ge_unmap_tx_dma(mgp, tx, idx); 2862 2863 if (skb_is_gso(skb)) { 2864 netdev_err(mgp->dev, "TSO but wanted to linearize?!?!?\n"); 2865 goto drop; 2866 } 2867 2868 if (skb_linearize(skb)) 2869 goto drop; 2870 2871 tx->linearized++; 2872 goto again; 2873 2874drop: 2875 dev_kfree_skb_any(skb); 2876 ss->stats.tx_dropped += 1; 2877 return NETDEV_TX_OK; 2878 2879} 2880 2881static netdev_tx_t myri10ge_sw_tso(struct sk_buff *skb, 2882 struct net_device *dev) 2883{ 2884 struct sk_buff *segs, *curr, *next; 2885 struct myri10ge_priv *mgp = netdev_priv(dev); 2886 struct myri10ge_slice_state *ss; 2887 netdev_tx_t status; 2888 2889 segs = skb_gso_segment(skb, dev->features & ~NETIF_F_TSO6); 2890 if (IS_ERR(segs)) 2891 goto drop; 2892 2893 skb_list_walk_safe(segs, curr, next) { 2894 skb_mark_not_on_list(curr); 2895 status = myri10ge_xmit(curr, dev); 2896 if (status != 0) { 2897 dev_kfree_skb_any(curr); 2898 skb_list_walk_safe(next, curr, next) { 2899 curr->next = NULL; 2900 dev_kfree_skb_any(curr); 2901 } 2902 goto drop; 2903 } 2904 } 2905 dev_kfree_skb_any(skb); 2906 return NETDEV_TX_OK; 2907 2908drop: 2909 ss = &mgp->ss[skb_get_queue_mapping(skb)]; 2910 dev_kfree_skb_any(skb); 2911 ss->stats.tx_dropped += 1; 2912 return NETDEV_TX_OK; 2913} 2914 2915static void myri10ge_get_stats(struct net_device *dev, 2916 struct rtnl_link_stats64 *stats) 2917{ 2918 const struct myri10ge_priv *mgp = netdev_priv(dev); 2919 const struct myri10ge_slice_netstats *slice_stats; 2920 int i; 2921 2922 for (i = 0; i < mgp->num_slices; i++) { 2923 slice_stats = &mgp->ss[i].stats; 2924 stats->rx_packets += slice_stats->rx_packets; 2925 stats->tx_packets += slice_stats->tx_packets; 2926 stats->rx_bytes += slice_stats->rx_bytes; 2927 stats->tx_bytes += slice_stats->tx_bytes; 2928 stats->rx_dropped += slice_stats->rx_dropped; 2929 stats->tx_dropped += slice_stats->tx_dropped; 2930 } 2931} 2932 2933static void myri10ge_set_multicast_list(struct net_device *dev) 2934{ 2935 struct myri10ge_priv *mgp = netdev_priv(dev); 2936 struct myri10ge_cmd cmd; 2937 struct netdev_hw_addr *ha; 2938 __be32 data[2] = { 0, 0 }; 2939 int err; 2940 2941 /* can be called from atomic contexts, 2942 * pass 1 to force atomicity in myri10ge_send_cmd() */ 2943 myri10ge_change_promisc(mgp, dev->flags & IFF_PROMISC, 1); 2944 2945 /* This firmware is known to not support multicast */ 2946 if (!mgp->fw_multicast_support) 2947 return; 2948 2949 /* Disable multicast filtering */ 2950 2951 err = myri10ge_send_cmd(mgp, MXGEFW_ENABLE_ALLMULTI, &cmd, 1); 2952 if (err != 0) { 2953 netdev_err(dev, "Failed MXGEFW_ENABLE_ALLMULTI, error status: %d\n", 2954 err); 2955 goto abort; 2956 } 2957 2958 if ((dev->flags & IFF_ALLMULTI) || mgp->adopted_rx_filter_bug) { 2959 /* request to disable multicast filtering, so quit here */ 2960 return; 2961 } 2962 2963 /* Flush the filters */ 2964 2965 err = myri10ge_send_cmd(mgp, MXGEFW_LEAVE_ALL_MULTICAST_GROUPS, 2966 &cmd, 1); 2967 if (err != 0) { 2968 netdev_err(dev, "Failed MXGEFW_LEAVE_ALL_MULTICAST_GROUPS, error status: %d\n", 2969 err); 2970 goto abort; 2971 } 2972 2973 /* Walk the multicast list, and add each address */ 2974 netdev_for_each_mc_addr(ha, dev) { 2975 memcpy(data, &ha->addr, ETH_ALEN); 2976 cmd.data0 = ntohl(data[0]); 2977 cmd.data1 = ntohl(data[1]); 2978 err = myri10ge_send_cmd(mgp, MXGEFW_JOIN_MULTICAST_GROUP, 2979 &cmd, 1); 2980 2981 if (err != 0) { 2982 netdev_err(dev, "Failed MXGEFW_JOIN_MULTICAST_GROUP, error status:%d %pM\n", 2983 err, ha->addr); 2984 goto abort; 2985 } 2986 } 2987 /* Enable multicast filtering */ 2988 err = myri10ge_send_cmd(mgp, MXGEFW_DISABLE_ALLMULTI, &cmd, 1); 2989 if (err != 0) { 2990 netdev_err(dev, "Failed MXGEFW_DISABLE_ALLMULTI, error status: %d\n", 2991 err); 2992 goto abort; 2993 } 2994 2995 return; 2996 2997abort: 2998 return; 2999} 3000 3001static int myri10ge_set_mac_address(struct net_device *dev, void *addr) 3002{ 3003 struct sockaddr *sa = addr; 3004 struct myri10ge_priv *mgp = netdev_priv(dev); 3005 int status; 3006 3007 if (!is_valid_ether_addr(sa->sa_data)) 3008 return -EADDRNOTAVAIL; 3009 3010 status = myri10ge_update_mac_address(mgp, sa->sa_data); 3011 if (status != 0) { 3012 netdev_err(dev, "changing mac address failed with %d\n", 3013 status); 3014 return status; 3015 } 3016 3017 /* change the dev structure */ 3018 memcpy(dev->dev_addr, sa->sa_data, ETH_ALEN); 3019 return 0; 3020} 3021 3022static int myri10ge_change_mtu(struct net_device *dev, int new_mtu) 3023{ 3024 struct myri10ge_priv *mgp = netdev_priv(dev); 3025 3026 netdev_info(dev, "changing mtu from %d to %d\n", dev->mtu, new_mtu); 3027 if (mgp->running) { 3028 /* if we change the mtu on an active device, we must 3029 * reset the device so the firmware sees the change */ 3030 myri10ge_close(dev); 3031 dev->mtu = new_mtu; 3032 myri10ge_open(dev); 3033 } else 3034 dev->mtu = new_mtu; 3035 3036 return 0; 3037} 3038 3039/* 3040 * Enable ECRC to align PCI-E Completion packets on an 8-byte boundary. 3041 * Only do it if the bridge is a root port since we don't want to disturb 3042 * any other device, except if forced with myri10ge_ecrc_enable > 1. 3043 */ 3044 3045static void myri10ge_enable_ecrc(struct myri10ge_priv *mgp) 3046{ 3047 struct pci_dev *bridge = mgp->pdev->bus->self; 3048 struct device *dev = &mgp->pdev->dev; 3049 int cap; 3050 unsigned err_cap; 3051 int ret; 3052 3053 if (!myri10ge_ecrc_enable || !bridge) 3054 return; 3055 3056 /* check that the bridge is a root port */ 3057 if (pci_pcie_type(bridge) != PCI_EXP_TYPE_ROOT_PORT) { 3058 if (myri10ge_ecrc_enable > 1) { 3059 struct pci_dev *prev_bridge, *old_bridge = bridge; 3060 3061 /* Walk the hierarchy up to the root port 3062 * where ECRC has to be enabled */ 3063 do { 3064 prev_bridge = bridge; 3065 bridge = bridge->bus->self; 3066 if (!bridge || prev_bridge == bridge) { 3067 dev_err(dev, 3068 "Failed to find root port" 3069 " to force ECRC\n"); 3070 return; 3071 } 3072 } while (pci_pcie_type(bridge) != 3073 PCI_EXP_TYPE_ROOT_PORT); 3074 3075 dev_info(dev, 3076 "Forcing ECRC on non-root port %s" 3077 " (enabling on root port %s)\n", 3078 pci_name(old_bridge), pci_name(bridge)); 3079 } else { 3080 dev_err(dev, 3081 "Not enabling ECRC on non-root port %s\n", 3082 pci_name(bridge)); 3083 return; 3084 } 3085 } 3086 3087 cap = pci_find_ext_capability(bridge, PCI_EXT_CAP_ID_ERR); 3088 if (!cap) 3089 return; 3090 3091 ret = pci_read_config_dword(bridge, cap + PCI_ERR_CAP, &err_cap); 3092 if (ret) { 3093 dev_err(dev, "failed reading ext-conf-space of %s\n", 3094 pci_name(bridge)); 3095 dev_err(dev, "\t pci=nommconf in use? " 3096 "or buggy/incomplete/absent ACPI MCFG attr?\n"); 3097 return; 3098 } 3099 if (!(err_cap & PCI_ERR_CAP_ECRC_GENC)) 3100 return; 3101 3102 err_cap |= PCI_ERR_CAP_ECRC_GENE; 3103 pci_write_config_dword(bridge, cap + PCI_ERR_CAP, err_cap); 3104 dev_info(dev, "Enabled ECRC on upstream bridge %s\n", pci_name(bridge)); 3105} 3106 3107/* 3108 * The Lanai Z8E PCI-E interface achieves higher Read-DMA throughput 3109 * when the PCI-E Completion packets are aligned on an 8-byte 3110 * boundary. Some PCI-E chip sets always align Completion packets; on 3111 * the ones that do not, the alignment can be enforced by enabling 3112 * ECRC generation (if supported). 3113 * 3114 * When PCI-E Completion packets are not aligned, it is actually more 3115 * efficient to limit Read-DMA transactions to 2KB, rather than 4KB. 3116 * 3117 * If the driver can neither enable ECRC nor verify that it has 3118 * already been enabled, then it must use a firmware image which works 3119 * around unaligned completion packets (myri10ge_rss_ethp_z8e.dat), and it 3120 * should also ensure that it never gives the device a Read-DMA which is 3121 * larger than 2KB by setting the tx_boundary to 2KB. If ECRC is 3122 * enabled, then the driver should use the aligned (myri10ge_rss_eth_z8e.dat) 3123 * firmware image, and set tx_boundary to 4KB. 3124 */ 3125 3126static void myri10ge_firmware_probe(struct myri10ge_priv *mgp) 3127{ 3128 struct pci_dev *pdev = mgp->pdev; 3129 struct device *dev = &pdev->dev; 3130 int status; 3131 3132 mgp->tx_boundary = 4096; 3133 /* 3134 * Verify the max read request size was set to 4KB 3135 * before trying the test with 4KB. 3136 */ 3137 status = pcie_get_readrq(pdev); 3138 if (status < 0) { 3139 dev_err(dev, "Couldn't read max read req size: %d\n", status); 3140 goto abort; 3141 } 3142 if (status != 4096) { 3143 dev_warn(dev, "Max Read Request size != 4096 (%d)\n", status); 3144 mgp->tx_boundary = 2048; 3145 } 3146 /* 3147 * load the optimized firmware (which assumes aligned PCIe 3148 * completions) in order to see if it works on this host. 3149 */ 3150 set_fw_name(mgp, myri10ge_fw_aligned, false); 3151 status = myri10ge_load_firmware(mgp, 1); 3152 if (status != 0) { 3153 goto abort; 3154 } 3155 3156 /* 3157 * Enable ECRC if possible 3158 */ 3159 myri10ge_enable_ecrc(mgp); 3160 3161 /* 3162 * Run a DMA test which watches for unaligned completions and 3163 * aborts on the first one seen. 3164 */ 3165 3166 status = myri10ge_dma_test(mgp, MXGEFW_CMD_UNALIGNED_TEST); 3167 if (status == 0) 3168 return; /* keep the aligned firmware */ 3169 3170 if (status != -E2BIG) 3171 dev_warn(dev, "DMA test failed: %d\n", status); 3172 if (status == -ENOSYS) 3173 dev_warn(dev, "Falling back to ethp! " 3174 "Please install up to date fw\n"); 3175abort: 3176 /* fall back to using the unaligned firmware */ 3177 mgp->tx_boundary = 2048; 3178 set_fw_name(mgp, myri10ge_fw_unaligned, false); 3179} 3180 3181static void myri10ge_select_firmware(struct myri10ge_priv *mgp) 3182{ 3183 int overridden = 0; 3184 3185 if (myri10ge_force_firmware == 0) { 3186 int link_width; 3187 u16 lnk; 3188 3189 pcie_capability_read_word(mgp->pdev, PCI_EXP_LNKSTA, &lnk); 3190 link_width = (lnk >> 4) & 0x3f; 3191 3192 /* Check to see if Link is less than 8 or if the 3193 * upstream bridge is known to provide aligned 3194 * completions */ 3195 if (link_width < 8) { 3196 dev_info(&mgp->pdev->dev, "PCIE x%d Link\n", 3197 link_width); 3198 mgp->tx_boundary = 4096; 3199 set_fw_name(mgp, myri10ge_fw_aligned, false); 3200 } else { 3201 myri10ge_firmware_probe(mgp); 3202 } 3203 } else { 3204 if (myri10ge_force_firmware == 1) { 3205 dev_info(&mgp->pdev->dev, 3206 "Assuming aligned completions (forced)\n"); 3207 mgp->tx_boundary = 4096; 3208 set_fw_name(mgp, myri10ge_fw_aligned, false); 3209 } else { 3210 dev_info(&mgp->pdev->dev, 3211 "Assuming unaligned completions (forced)\n"); 3212 mgp->tx_boundary = 2048; 3213 set_fw_name(mgp, myri10ge_fw_unaligned, false); 3214 } 3215 } 3216 3217 kernel_param_lock(THIS_MODULE); 3218 if (myri10ge_fw_name != NULL) { 3219 char *fw_name = kstrdup(myri10ge_fw_name, GFP_KERNEL); 3220 if (fw_name) { 3221 overridden = 1; 3222 set_fw_name(mgp, fw_name, true); 3223 } 3224 } 3225 kernel_param_unlock(THIS_MODULE); 3226 3227 if (mgp->board_number < MYRI10GE_MAX_BOARDS && 3228 myri10ge_fw_names[mgp->board_number] != NULL && 3229 strlen(myri10ge_fw_names[mgp->board_number])) { 3230 set_fw_name(mgp, myri10ge_fw_names[mgp->board_number], false); 3231 overridden = 1; 3232 } 3233 if (overridden) 3234 dev_info(&mgp->pdev->dev, "overriding firmware to %s\n", 3235 mgp->fw_name); 3236} 3237 3238static void myri10ge_mask_surprise_down(struct pci_dev *pdev) 3239{ 3240 struct pci_dev *bridge = pdev->bus->self; 3241 int cap; 3242 u32 mask; 3243 3244 if (bridge == NULL) 3245 return; 3246 3247 cap = pci_find_ext_capability(bridge, PCI_EXT_CAP_ID_ERR); 3248 if (cap) { 3249 /* a sram parity error can cause a surprise link 3250 * down; since we expect and can recover from sram 3251 * parity errors, mask surprise link down events */ 3252 pci_read_config_dword(bridge, cap + PCI_ERR_UNCOR_MASK, &mask); 3253 mask |= 0x20; 3254 pci_write_config_dword(bridge, cap + PCI_ERR_UNCOR_MASK, mask); 3255 } 3256} 3257 3258static int __maybe_unused myri10ge_suspend(struct device *dev) 3259{ 3260 struct myri10ge_priv *mgp; 3261 struct net_device *netdev; 3262 3263 mgp = dev_get_drvdata(dev); 3264 if (mgp == NULL) 3265 return -EINVAL; 3266 netdev = mgp->dev; 3267 3268 netif_device_detach(netdev); 3269 if (netif_running(netdev)) { 3270 netdev_info(netdev, "closing\n"); 3271 rtnl_lock(); 3272 myri10ge_close(netdev); 3273 rtnl_unlock(); 3274 } 3275 myri10ge_dummy_rdma(mgp, 0); 3276 3277 return 0; 3278} 3279 3280static int __maybe_unused myri10ge_resume(struct device *dev) 3281{ 3282 struct pci_dev *pdev = to_pci_dev(dev); 3283 struct myri10ge_priv *mgp; 3284 struct net_device *netdev; 3285 int status; 3286 u16 vendor; 3287 3288 mgp = pci_get_drvdata(pdev); 3289 if (mgp == NULL) 3290 return -EINVAL; 3291 netdev = mgp->dev; 3292 msleep(5); /* give card time to respond */ 3293 pci_read_config_word(mgp->pdev, PCI_VENDOR_ID, &vendor); 3294 if (vendor == 0xffff) { 3295 netdev_err(mgp->dev, "device disappeared!\n"); 3296 return -EIO; 3297 } 3298 3299 myri10ge_reset(mgp); 3300 myri10ge_dummy_rdma(mgp, 1); 3301 3302 if (netif_running(netdev)) { 3303 rtnl_lock(); 3304 status = myri10ge_open(netdev); 3305 rtnl_unlock(); 3306 if (status != 0) 3307 goto abort_with_enabled; 3308 3309 } 3310 netif_device_attach(netdev); 3311 3312 return 0; 3313 3314abort_with_enabled: 3315 return -EIO; 3316} 3317 3318static u32 myri10ge_read_reboot(struct myri10ge_priv *mgp) 3319{ 3320 struct pci_dev *pdev = mgp->pdev; 3321 int vs = mgp->vendor_specific_offset; 3322 u32 reboot; 3323 3324 /*enter read32 mode */ 3325 pci_write_config_byte(pdev, vs + 0x10, 0x3); 3326 3327 /*read REBOOT_STATUS (0xfffffff0) */ 3328 pci_write_config_dword(pdev, vs + 0x18, 0xfffffff0); 3329 pci_read_config_dword(pdev, vs + 0x14, &reboot); 3330 return reboot; 3331} 3332 3333static void 3334myri10ge_check_slice(struct myri10ge_slice_state *ss, int *reset_needed, 3335 int *busy_slice_cnt, u32 rx_pause_cnt) 3336{ 3337 struct myri10ge_priv *mgp = ss->mgp; 3338 int slice = ss - mgp->ss; 3339 3340 if (ss->tx.req != ss->tx.done && 3341 ss->tx.done == ss->watchdog_tx_done && 3342 ss->watchdog_tx_req != ss->watchdog_tx_done) { 3343 /* nic seems like it might be stuck.. */ 3344 if (rx_pause_cnt != mgp->watchdog_pause) { 3345 if (net_ratelimit()) 3346 netdev_warn(mgp->dev, "slice %d: TX paused, " 3347 "check link partner\n", slice); 3348 } else { 3349 netdev_warn(mgp->dev, 3350 "slice %d: TX stuck %d %d %d %d %d %d\n", 3351 slice, ss->tx.queue_active, ss->tx.req, 3352 ss->tx.done, ss->tx.pkt_start, 3353 ss->tx.pkt_done, 3354 (int)ntohl(mgp->ss[slice].fw_stats-> 3355 send_done_count)); 3356 *reset_needed = 1; 3357 ss->stuck = 1; 3358 } 3359 } 3360 if (ss->watchdog_tx_done != ss->tx.done || 3361 ss->watchdog_rx_done != ss->rx_done.cnt) { 3362 *busy_slice_cnt += 1; 3363 } 3364 ss->watchdog_tx_done = ss->tx.done; 3365 ss->watchdog_tx_req = ss->tx.req; 3366 ss->watchdog_rx_done = ss->rx_done.cnt; 3367} 3368 3369/* 3370 * This watchdog is used to check whether the board has suffered 3371 * from a parity error and needs to be recovered. 3372 */ 3373static void myri10ge_watchdog(struct work_struct *work) 3374{ 3375 struct myri10ge_priv *mgp = 3376 container_of(work, struct myri10ge_priv, watchdog_work); 3377 struct myri10ge_slice_state *ss; 3378 u32 reboot, rx_pause_cnt; 3379 int status, rebooted; 3380 int i; 3381 int reset_needed = 0; 3382 int busy_slice_cnt = 0; 3383 u16 cmd, vendor; 3384 3385 mgp->watchdog_resets++; 3386 pci_read_config_word(mgp->pdev, PCI_COMMAND, &cmd); 3387 rebooted = 0; 3388 if ((cmd & PCI_COMMAND_MASTER) == 0) { 3389 /* Bus master DMA disabled? Check to see 3390 * if the card rebooted due to a parity error 3391 * For now, just report it */ 3392 reboot = myri10ge_read_reboot(mgp); 3393 netdev_err(mgp->dev, "NIC rebooted (0x%x),%s resetting\n", 3394 reboot, myri10ge_reset_recover ? "" : " not"); 3395 if (myri10ge_reset_recover == 0) 3396 return; 3397 rtnl_lock(); 3398 mgp->rebooted = 1; 3399 rebooted = 1; 3400 myri10ge_close(mgp->dev); 3401 myri10ge_reset_recover--; 3402 mgp->rebooted = 0; 3403 /* 3404 * A rebooted nic will come back with config space as 3405 * it was after power was applied to PCIe bus. 3406 * Attempt to restore config space which was saved 3407 * when the driver was loaded, or the last time the 3408 * nic was resumed from power saving mode. 3409 */ 3410 pci_restore_state(mgp->pdev); 3411 3412 /* save state again for accounting reasons */ 3413 pci_save_state(mgp->pdev); 3414 3415 } else { 3416 /* if we get back -1's from our slot, perhaps somebody 3417 * powered off our card. Don't try to reset it in 3418 * this case */ 3419 if (cmd == 0xffff) { 3420 pci_read_config_word(mgp->pdev, PCI_VENDOR_ID, &vendor); 3421 if (vendor == 0xffff) { 3422 netdev_err(mgp->dev, "device disappeared!\n"); 3423 return; 3424 } 3425 } 3426 /* Perhaps it is a software error. See if stuck slice 3427 * has recovered, reset if not */ 3428 rx_pause_cnt = ntohl(mgp->ss[0].fw_stats->dropped_pause); 3429 for (i = 0; i < mgp->num_slices; i++) { 3430 ss = mgp->ss; 3431 if (ss->stuck) { 3432 myri10ge_check_slice(ss, &reset_needed, 3433 &busy_slice_cnt, 3434 rx_pause_cnt); 3435 ss->stuck = 0; 3436 } 3437 } 3438 if (!reset_needed) { 3439 netdev_dbg(mgp->dev, "not resetting\n"); 3440 return; 3441 } 3442 3443 netdev_err(mgp->dev, "device timeout, resetting\n"); 3444 } 3445 3446 if (!rebooted) { 3447 rtnl_lock(); 3448 myri10ge_close(mgp->dev); 3449 } 3450 status = myri10ge_load_firmware(mgp, 1); 3451 if (status != 0) 3452 netdev_err(mgp->dev, "failed to load firmware\n"); 3453 else 3454 myri10ge_open(mgp->dev); 3455 rtnl_unlock(); 3456} 3457 3458/* 3459 * We use our own timer routine rather than relying upon 3460 * netdev->tx_timeout because we have a very large hardware transmit 3461 * queue. Due to the large queue, the netdev->tx_timeout function 3462 * cannot detect a NIC with a parity error in a timely fashion if the 3463 * NIC is lightly loaded. 3464 */ 3465static void myri10ge_watchdog_timer(struct timer_list *t) 3466{ 3467 struct myri10ge_priv *mgp; 3468 struct myri10ge_slice_state *ss; 3469 int i, reset_needed, busy_slice_cnt; 3470 u32 rx_pause_cnt; 3471 u16 cmd; 3472 3473 mgp = from_timer(mgp, t, watchdog_timer); 3474 3475 rx_pause_cnt = ntohl(mgp->ss[0].fw_stats->dropped_pause); 3476 busy_slice_cnt = 0; 3477 for (i = 0, reset_needed = 0; 3478 i < mgp->num_slices && reset_needed == 0; ++i) { 3479 3480 ss = &mgp->ss[i]; 3481 if (ss->rx_small.watchdog_needed) { 3482 myri10ge_alloc_rx_pages(mgp, &ss->rx_small, 3483 mgp->small_bytes + MXGEFW_PAD, 3484 1); 3485 if (ss->rx_small.fill_cnt - ss->rx_small.cnt >= 3486 myri10ge_fill_thresh) 3487 ss->rx_small.watchdog_needed = 0; 3488 } 3489 if (ss->rx_big.watchdog_needed) { 3490 myri10ge_alloc_rx_pages(mgp, &ss->rx_big, 3491 mgp->big_bytes, 1); 3492 if (ss->rx_big.fill_cnt - ss->rx_big.cnt >= 3493 myri10ge_fill_thresh) 3494 ss->rx_big.watchdog_needed = 0; 3495 } 3496 myri10ge_check_slice(ss, &reset_needed, &busy_slice_cnt, 3497 rx_pause_cnt); 3498 } 3499 /* if we've sent or received no traffic, poll the NIC to 3500 * ensure it is still there. Otherwise, we risk not noticing 3501 * an error in a timely fashion */ 3502 if (busy_slice_cnt == 0) { 3503 pci_read_config_word(mgp->pdev, PCI_COMMAND, &cmd); 3504 if ((cmd & PCI_COMMAND_MASTER) == 0) { 3505 reset_needed = 1; 3506 } 3507 } 3508 mgp->watchdog_pause = rx_pause_cnt; 3509 3510 if (reset_needed) { 3511 schedule_work(&mgp->watchdog_work); 3512 } else { 3513 /* rearm timer */ 3514 mod_timer(&mgp->watchdog_timer, 3515 jiffies + myri10ge_watchdog_timeout * HZ); 3516 } 3517} 3518 3519static void myri10ge_free_slices(struct myri10ge_priv *mgp) 3520{ 3521 struct myri10ge_slice_state *ss; 3522 struct pci_dev *pdev = mgp->pdev; 3523 size_t bytes; 3524 int i; 3525 3526 if (mgp->ss == NULL) 3527 return; 3528 3529 for (i = 0; i < mgp->num_slices; i++) { 3530 ss = &mgp->ss[i]; 3531 if (ss->rx_done.entry != NULL) { 3532 bytes = mgp->max_intr_slots * 3533 sizeof(*ss->rx_done.entry); 3534 dma_free_coherent(&pdev->dev, bytes, 3535 ss->rx_done.entry, ss->rx_done.bus); 3536 ss->rx_done.entry = NULL; 3537 } 3538 if (ss->fw_stats != NULL) { 3539 bytes = sizeof(*ss->fw_stats); 3540 dma_free_coherent(&pdev->dev, bytes, 3541 ss->fw_stats, ss->fw_stats_bus); 3542 ss->fw_stats = NULL; 3543 } 3544 __netif_napi_del(&ss->napi); 3545 } 3546 /* Wait till napi structs are no longer used, and then free ss. */ 3547 synchronize_net(); 3548 kfree(mgp->ss); 3549 mgp->ss = NULL; 3550} 3551 3552static int myri10ge_alloc_slices(struct myri10ge_priv *mgp) 3553{ 3554 struct myri10ge_slice_state *ss; 3555 struct pci_dev *pdev = mgp->pdev; 3556 size_t bytes; 3557 int i; 3558 3559 bytes = sizeof(*mgp->ss) * mgp->num_slices; 3560 mgp->ss = kzalloc(bytes, GFP_KERNEL); 3561 if (mgp->ss == NULL) { 3562 return -ENOMEM; 3563 } 3564 3565 for (i = 0; i < mgp->num_slices; i++) { 3566 ss = &mgp->ss[i]; 3567 bytes = mgp->max_intr_slots * sizeof(*ss->rx_done.entry); 3568 ss->rx_done.entry = dma_alloc_coherent(&pdev->dev, bytes, 3569 &ss->rx_done.bus, 3570 GFP_KERNEL); 3571 if (ss->rx_done.entry == NULL) 3572 goto abort; 3573 bytes = sizeof(*ss->fw_stats); 3574 ss->fw_stats = dma_alloc_coherent(&pdev->dev, bytes, 3575 &ss->fw_stats_bus, 3576 GFP_KERNEL); 3577 if (ss->fw_stats == NULL) 3578 goto abort; 3579 ss->mgp = mgp; 3580 ss->dev = mgp->dev; 3581 netif_napi_add(ss->dev, &ss->napi, myri10ge_poll, 3582 myri10ge_napi_weight); 3583 } 3584 return 0; 3585abort: 3586 myri10ge_free_slices(mgp); 3587 return -ENOMEM; 3588} 3589 3590/* 3591 * This function determines the number of slices supported. 3592 * The number slices is the minimum of the number of CPUS, 3593 * the number of MSI-X irqs supported, the number of slices 3594 * supported by the firmware 3595 */ 3596static void myri10ge_probe_slices(struct myri10ge_priv *mgp) 3597{ 3598 struct myri10ge_cmd cmd; 3599 struct pci_dev *pdev = mgp->pdev; 3600 char *old_fw; 3601 bool old_allocated; 3602 int i, status, ncpus; 3603 3604 mgp->num_slices = 1; 3605 ncpus = netif_get_num_default_rss_queues(); 3606 3607 if (myri10ge_max_slices == 1 || !pdev->msix_cap || 3608 (myri10ge_max_slices == -1 && ncpus < 2)) 3609 return; 3610 3611 /* try to load the slice aware rss firmware */ 3612 old_fw = mgp->fw_name; 3613 old_allocated = mgp->fw_name_allocated; 3614 /* don't free old_fw if we override it. */ 3615 mgp->fw_name_allocated = false; 3616 3617 if (myri10ge_fw_name != NULL) { 3618 dev_info(&mgp->pdev->dev, "overriding rss firmware to %s\n", 3619 myri10ge_fw_name); 3620 set_fw_name(mgp, myri10ge_fw_name, false); 3621 } else if (old_fw == myri10ge_fw_aligned) 3622 set_fw_name(mgp, myri10ge_fw_rss_aligned, false); 3623 else 3624 set_fw_name(mgp, myri10ge_fw_rss_unaligned, false); 3625 status = myri10ge_load_firmware(mgp, 0); 3626 if (status != 0) { 3627 dev_info(&pdev->dev, "Rss firmware not found\n"); 3628 if (old_allocated) 3629 kfree(old_fw); 3630 return; 3631 } 3632 3633 /* hit the board with a reset to ensure it is alive */ 3634 memset(&cmd, 0, sizeof(cmd)); 3635 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_RESET, &cmd, 0); 3636 if (status != 0) { 3637 dev_err(&mgp->pdev->dev, "failed reset\n"); 3638 goto abort_with_fw; 3639 } 3640 3641 mgp->max_intr_slots = cmd.data0 / sizeof(struct mcp_slot); 3642 3643 /* tell it the size of the interrupt queues */ 3644 cmd.data0 = mgp->max_intr_slots * sizeof(struct mcp_slot); 3645 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_SET_INTRQ_SIZE, &cmd, 0); 3646 if (status != 0) { 3647 dev_err(&mgp->pdev->dev, "failed MXGEFW_CMD_SET_INTRQ_SIZE\n"); 3648 goto abort_with_fw; 3649 } 3650 3651 /* ask the maximum number of slices it supports */ 3652 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_MAX_RSS_QUEUES, &cmd, 0); 3653 if (status != 0) 3654 goto abort_with_fw; 3655 else 3656 mgp->num_slices = cmd.data0; 3657 3658 /* Only allow multiple slices if MSI-X is usable */ 3659 if (!myri10ge_msi) { 3660 goto abort_with_fw; 3661 } 3662 3663 /* if the admin did not specify a limit to how many 3664 * slices we should use, cap it automatically to the 3665 * number of CPUs currently online */ 3666 if (myri10ge_max_slices == -1) 3667 myri10ge_max_slices = ncpus; 3668 3669 if (mgp->num_slices > myri10ge_max_slices) 3670 mgp->num_slices = myri10ge_max_slices; 3671 3672 /* Now try to allocate as many MSI-X vectors as we have 3673 * slices. We give up on MSI-X if we can only get a single 3674 * vector. */ 3675 3676 mgp->msix_vectors = kcalloc(mgp->num_slices, sizeof(*mgp->msix_vectors), 3677 GFP_KERNEL); 3678 if (mgp->msix_vectors == NULL) 3679 goto no_msix; 3680 for (i = 0; i < mgp->num_slices; i++) { 3681 mgp->msix_vectors[i].entry = i; 3682 } 3683 3684 while (mgp->num_slices > 1) { 3685 mgp->num_slices = rounddown_pow_of_two(mgp->num_slices); 3686 if (mgp->num_slices == 1) 3687 goto no_msix; 3688 status = pci_enable_msix_range(pdev, 3689 mgp->msix_vectors, 3690 mgp->num_slices, 3691 mgp->num_slices); 3692 if (status < 0) 3693 goto no_msix; 3694 3695 pci_disable_msix(pdev); 3696 3697 if (status == mgp->num_slices) { 3698 if (old_allocated) 3699 kfree(old_fw); 3700 return; 3701 } else { 3702 mgp->num_slices = status; 3703 } 3704 } 3705 3706no_msix: 3707 if (mgp->msix_vectors != NULL) { 3708 kfree(mgp->msix_vectors); 3709 mgp->msix_vectors = NULL; 3710 } 3711 3712abort_with_fw: 3713 mgp->num_slices = 1; 3714 set_fw_name(mgp, old_fw, old_allocated); 3715 myri10ge_load_firmware(mgp, 0); 3716} 3717 3718static const struct net_device_ops myri10ge_netdev_ops = { 3719 .ndo_open = myri10ge_open, 3720 .ndo_stop = myri10ge_close, 3721 .ndo_start_xmit = myri10ge_xmit, 3722 .ndo_get_stats64 = myri10ge_get_stats, 3723 .ndo_validate_addr = eth_validate_addr, 3724 .ndo_change_mtu = myri10ge_change_mtu, 3725 .ndo_set_rx_mode = myri10ge_set_multicast_list, 3726 .ndo_set_mac_address = myri10ge_set_mac_address, 3727}; 3728 3729static int myri10ge_probe(struct pci_dev *pdev, const struct pci_device_id *ent) 3730{ 3731 struct net_device *netdev; 3732 struct myri10ge_priv *mgp; 3733 struct device *dev = &pdev->dev; 3734 int i; 3735 int status = -ENXIO; 3736 int dac_enabled; 3737 unsigned hdr_offset, ss_offset; 3738 static int board_number; 3739 3740 netdev = alloc_etherdev_mq(sizeof(*mgp), MYRI10GE_MAX_SLICES); 3741 if (netdev == NULL) 3742 return -ENOMEM; 3743 3744 SET_NETDEV_DEV(netdev, &pdev->dev); 3745 3746 mgp = netdev_priv(netdev); 3747 mgp->dev = netdev; 3748 mgp->pdev = pdev; 3749 mgp->pause = myri10ge_flow_control; 3750 mgp->intr_coal_delay = myri10ge_intr_coal_delay; 3751 mgp->msg_enable = netif_msg_init(myri10ge_debug, MYRI10GE_MSG_DEFAULT); 3752 mgp->board_number = board_number; 3753 init_waitqueue_head(&mgp->down_wq); 3754 3755 if (pci_enable_device(pdev)) { 3756 dev_err(&pdev->dev, "pci_enable_device call failed\n"); 3757 status = -ENODEV; 3758 goto abort_with_netdev; 3759 } 3760 3761 /* Find the vendor-specific cap so we can check 3762 * the reboot register later on */ 3763 mgp->vendor_specific_offset 3764 = pci_find_capability(pdev, PCI_CAP_ID_VNDR); 3765 3766 /* Set our max read request to 4KB */ 3767 status = pcie_set_readrq(pdev, 4096); 3768 if (status != 0) { 3769 dev_err(&pdev->dev, "Error %d writing PCI_EXP_DEVCTL\n", 3770 status); 3771 goto abort_with_enabled; 3772 } 3773 3774 myri10ge_mask_surprise_down(pdev); 3775 pci_set_master(pdev); 3776 dac_enabled = 1; 3777 status = pci_set_dma_mask(pdev, DMA_BIT_MASK(64)); 3778 if (status != 0) { 3779 dac_enabled = 0; 3780 dev_err(&pdev->dev, 3781 "64-bit pci address mask was refused, " 3782 "trying 32-bit\n"); 3783 status = pci_set_dma_mask(pdev, DMA_BIT_MASK(32)); 3784 } 3785 if (status != 0) { 3786 dev_err(&pdev->dev, "Error %d setting DMA mask\n", status); 3787 goto abort_with_enabled; 3788 } 3789 (void)pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64)); 3790 mgp->cmd = dma_alloc_coherent(&pdev->dev, sizeof(*mgp->cmd), 3791 &mgp->cmd_bus, GFP_KERNEL); 3792 if (!mgp->cmd) { 3793 status = -ENOMEM; 3794 goto abort_with_enabled; 3795 } 3796 3797 mgp->board_span = pci_resource_len(pdev, 0); 3798 mgp->iomem_base = pci_resource_start(pdev, 0); 3799 mgp->wc_cookie = arch_phys_wc_add(mgp->iomem_base, mgp->board_span); 3800 mgp->sram = ioremap_wc(mgp->iomem_base, mgp->board_span); 3801 if (mgp->sram == NULL) { 3802 dev_err(&pdev->dev, "ioremap failed for %ld bytes at 0x%lx\n", 3803 mgp->board_span, mgp->iomem_base); 3804 status = -ENXIO; 3805 goto abort_with_mtrr; 3806 } 3807 hdr_offset = 3808 swab32(readl(mgp->sram + MCP_HEADER_PTR_OFFSET)) & 0xffffc; 3809 ss_offset = hdr_offset + offsetof(struct mcp_gen_header, string_specs); 3810 mgp->sram_size = swab32(readl(mgp->sram + ss_offset)); 3811 if (mgp->sram_size > mgp->board_span || 3812 mgp->sram_size <= MYRI10GE_FW_OFFSET) { 3813 dev_err(&pdev->dev, 3814 "invalid sram_size %dB or board span %ldB\n", 3815 mgp->sram_size, mgp->board_span); 3816 status = -EINVAL; 3817 goto abort_with_ioremap; 3818 } 3819 memcpy_fromio(mgp->eeprom_strings, 3820 mgp->sram + mgp->sram_size, MYRI10GE_EEPROM_STRINGS_SIZE); 3821 memset(mgp->eeprom_strings + MYRI10GE_EEPROM_STRINGS_SIZE - 2, 0, 2); 3822 status = myri10ge_read_mac_addr(mgp); 3823 if (status) 3824 goto abort_with_ioremap; 3825 3826 for (i = 0; i < ETH_ALEN; i++) 3827 netdev->dev_addr[i] = mgp->mac_addr[i]; 3828 3829 myri10ge_select_firmware(mgp); 3830 3831 status = myri10ge_load_firmware(mgp, 1); 3832 if (status != 0) { 3833 dev_err(&pdev->dev, "failed to load firmware\n"); 3834 goto abort_with_ioremap; 3835 } 3836 myri10ge_probe_slices(mgp); 3837 status = myri10ge_alloc_slices(mgp); 3838 if (status != 0) { 3839 dev_err(&pdev->dev, "failed to alloc slice state\n"); 3840 goto abort_with_firmware; 3841 } 3842 netif_set_real_num_tx_queues(netdev, mgp->num_slices); 3843 netif_set_real_num_rx_queues(netdev, mgp->num_slices); 3844 status = myri10ge_reset(mgp); 3845 if (status != 0) { 3846 dev_err(&pdev->dev, "failed reset\n"); 3847 goto abort_with_slices; 3848 } 3849#ifdef CONFIG_MYRI10GE_DCA 3850 myri10ge_setup_dca(mgp); 3851#endif 3852 pci_set_drvdata(pdev, mgp); 3853 3854 /* MTU range: 68 - 9000 */ 3855 netdev->min_mtu = ETH_MIN_MTU; 3856 netdev->max_mtu = MYRI10GE_MAX_ETHER_MTU - ETH_HLEN; 3857 3858 if (myri10ge_initial_mtu > netdev->max_mtu) 3859 myri10ge_initial_mtu = netdev->max_mtu; 3860 if (myri10ge_initial_mtu < netdev->min_mtu) 3861 myri10ge_initial_mtu = netdev->min_mtu; 3862 3863 netdev->mtu = myri10ge_initial_mtu; 3864 3865 netdev->netdev_ops = &myri10ge_netdev_ops; 3866 netdev->hw_features = mgp->features | NETIF_F_RXCSUM; 3867 3868 /* fake NETIF_F_HW_VLAN_CTAG_RX for good GRO performance */ 3869 netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_RX; 3870 3871 netdev->features = netdev->hw_features; 3872 3873 if (dac_enabled) 3874 netdev->features |= NETIF_F_HIGHDMA; 3875 3876 netdev->vlan_features |= mgp->features; 3877 if (mgp->fw_ver_tiny < 37) 3878 netdev->vlan_features &= ~NETIF_F_TSO6; 3879 if (mgp->fw_ver_tiny < 32) 3880 netdev->vlan_features &= ~NETIF_F_TSO; 3881 3882 /* make sure we can get an irq, and that MSI can be 3883 * setup (if available). */ 3884 status = myri10ge_request_irq(mgp); 3885 if (status != 0) 3886 goto abort_with_slices; 3887 myri10ge_free_irq(mgp); 3888 3889 /* Save configuration space to be restored if the 3890 * nic resets due to a parity error */ 3891 pci_save_state(pdev); 3892 3893 /* Setup the watchdog timer */ 3894 timer_setup(&mgp->watchdog_timer, myri10ge_watchdog_timer, 0); 3895 3896 netdev->ethtool_ops = &myri10ge_ethtool_ops; 3897 INIT_WORK(&mgp->watchdog_work, myri10ge_watchdog); 3898 status = register_netdev(netdev); 3899 if (status != 0) { 3900 dev_err(&pdev->dev, "register_netdev failed: %d\n", status); 3901 goto abort_with_state; 3902 } 3903 if (mgp->msix_enabled) 3904 dev_info(dev, "%d MSI-X IRQs, tx bndry %d, fw %s, MTRR %s, WC Enabled\n", 3905 mgp->num_slices, mgp->tx_boundary, mgp->fw_name, 3906 (mgp->wc_cookie > 0 ? "Enabled" : "Disabled")); 3907 else 3908 dev_info(dev, "%s IRQ %d, tx bndry %d, fw %s, MTRR %s, WC Enabled\n", 3909 mgp->msi_enabled ? "MSI" : "xPIC", 3910 pdev->irq, mgp->tx_boundary, mgp->fw_name, 3911 (mgp->wc_cookie > 0 ? "Enabled" : "Disabled")); 3912 3913 board_number++; 3914 return 0; 3915 3916abort_with_state: 3917 pci_restore_state(pdev); 3918 3919abort_with_slices: 3920 myri10ge_free_slices(mgp); 3921 3922abort_with_firmware: 3923 kfree(mgp->msix_vectors); 3924 myri10ge_dummy_rdma(mgp, 0); 3925 3926abort_with_ioremap: 3927 if (mgp->mac_addr_string != NULL) 3928 dev_err(&pdev->dev, 3929 "myri10ge_probe() failed: MAC=%s, SN=%ld\n", 3930 mgp->mac_addr_string, mgp->serial_number); 3931 iounmap(mgp->sram); 3932 3933abort_with_mtrr: 3934 arch_phys_wc_del(mgp->wc_cookie); 3935 dma_free_coherent(&pdev->dev, sizeof(*mgp->cmd), 3936 mgp->cmd, mgp->cmd_bus); 3937 3938abort_with_enabled: 3939 pci_disable_device(pdev); 3940 3941abort_with_netdev: 3942 set_fw_name(mgp, NULL, false); 3943 free_netdev(netdev); 3944 return status; 3945} 3946 3947/* 3948 * myri10ge_remove 3949 * 3950 * Does what is necessary to shutdown one Myrinet device. Called 3951 * once for each Myrinet card by the kernel when a module is 3952 * unloaded. 3953 */ 3954static void myri10ge_remove(struct pci_dev *pdev) 3955{ 3956 struct myri10ge_priv *mgp; 3957 struct net_device *netdev; 3958 3959 mgp = pci_get_drvdata(pdev); 3960 if (mgp == NULL) 3961 return; 3962 3963 cancel_work_sync(&mgp->watchdog_work); 3964 netdev = mgp->dev; 3965 unregister_netdev(netdev); 3966 3967#ifdef CONFIG_MYRI10GE_DCA 3968 myri10ge_teardown_dca(mgp); 3969#endif 3970 myri10ge_dummy_rdma(mgp, 0); 3971 3972 /* avoid a memory leak */ 3973 pci_restore_state(pdev); 3974 3975 iounmap(mgp->sram); 3976 arch_phys_wc_del(mgp->wc_cookie); 3977 myri10ge_free_slices(mgp); 3978 kfree(mgp->msix_vectors); 3979 dma_free_coherent(&pdev->dev, sizeof(*mgp->cmd), 3980 mgp->cmd, mgp->cmd_bus); 3981 3982 set_fw_name(mgp, NULL, false); 3983 free_netdev(netdev); 3984 pci_disable_device(pdev); 3985} 3986 3987#define PCI_DEVICE_ID_MYRICOM_MYRI10GE_Z8E 0x0008 3988#define PCI_DEVICE_ID_MYRICOM_MYRI10GE_Z8E_9 0x0009 3989 3990static const struct pci_device_id myri10ge_pci_tbl[] = { 3991 {PCI_DEVICE(PCI_VENDOR_ID_MYRICOM, PCI_DEVICE_ID_MYRICOM_MYRI10GE_Z8E)}, 3992 {PCI_DEVICE 3993 (PCI_VENDOR_ID_MYRICOM, PCI_DEVICE_ID_MYRICOM_MYRI10GE_Z8E_9)}, 3994 {0}, 3995}; 3996 3997MODULE_DEVICE_TABLE(pci, myri10ge_pci_tbl); 3998 3999static SIMPLE_DEV_PM_OPS(myri10ge_pm_ops, myri10ge_suspend, myri10ge_resume); 4000 4001static struct pci_driver myri10ge_driver = { 4002 .name = "myri10ge", 4003 .probe = myri10ge_probe, 4004 .remove = myri10ge_remove, 4005 .id_table = myri10ge_pci_tbl, 4006 .driver.pm = &myri10ge_pm_ops, 4007}; 4008 4009#ifdef CONFIG_MYRI10GE_DCA 4010static int 4011myri10ge_notify_dca(struct notifier_block *nb, unsigned long event, void *p) 4012{ 4013 int err = driver_for_each_device(&myri10ge_driver.driver, 4014 NULL, &event, 4015 myri10ge_notify_dca_device); 4016 4017 if (err) 4018 return NOTIFY_BAD; 4019 return NOTIFY_DONE; 4020} 4021 4022static struct notifier_block myri10ge_dca_notifier = { 4023 .notifier_call = myri10ge_notify_dca, 4024 .next = NULL, 4025 .priority = 0, 4026}; 4027#endif /* CONFIG_MYRI10GE_DCA */ 4028 4029static __init int myri10ge_init_module(void) 4030{ 4031 pr_info("Version %s\n", MYRI10GE_VERSION_STR); 4032 4033 if (myri10ge_rss_hash > MXGEFW_RSS_HASH_TYPE_MAX) { 4034 pr_err("Illegal rssh hash type %d, defaulting to source port\n", 4035 myri10ge_rss_hash); 4036 myri10ge_rss_hash = MXGEFW_RSS_HASH_TYPE_SRC_PORT; 4037 } 4038#ifdef CONFIG_MYRI10GE_DCA 4039 dca_register_notify(&myri10ge_dca_notifier); 4040#endif 4041 if (myri10ge_max_slices > MYRI10GE_MAX_SLICES) 4042 myri10ge_max_slices = MYRI10GE_MAX_SLICES; 4043 4044 return pci_register_driver(&myri10ge_driver); 4045} 4046 4047module_init(myri10ge_init_module); 4048 4049static __exit void myri10ge_cleanup_module(void) 4050{ 4051#ifdef CONFIG_MYRI10GE_DCA 4052 dca_unregister_notify(&myri10ge_dca_notifier); 4053#endif 4054 pci_unregister_driver(&myri10ge_driver); 4055} 4056 4057module_exit(myri10ge_cleanup_module); 4058