1// SPDX-License-Identifier: GPL-2.0 2/* Copyright(c) 1999 - 2018 Intel Corporation. */ 3 4#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 5 6#include <linux/module.h> 7#include <linux/types.h> 8#include <linux/init.h> 9#include <linux/pci.h> 10#include <linux/vmalloc.h> 11#include <linux/pagemap.h> 12#include <linux/delay.h> 13#include <linux/netdevice.h> 14#include <linux/interrupt.h> 15#include <linux/tcp.h> 16#include <linux/ipv6.h> 17#include <linux/slab.h> 18#include <net/checksum.h> 19#include <net/ip6_checksum.h> 20#include <linux/ethtool.h> 21#include <linux/if_vlan.h> 22#include <linux/cpu.h> 23#include <linux/smp.h> 24#include <linux/pm_qos.h> 25#include <linux/pm_runtime.h> 26#include <linux/aer.h> 27#include <linux/prefetch.h> 28 29#include "e1000.h" 30 31char e1000e_driver_name[] = "e1000e"; 32 33#define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV|NETIF_MSG_PROBE|NETIF_MSG_LINK) 34static int debug = -1; 35module_param(debug, int, 0); 36MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)"); 37 38static const struct e1000_info *e1000_info_tbl[] = { 39 [board_82571] = &e1000_82571_info, 40 [board_82572] = &e1000_82572_info, 41 [board_82573] = &e1000_82573_info, 42 [board_82574] = &e1000_82574_info, 43 [board_82583] = &e1000_82583_info, 44 [board_80003es2lan] = &e1000_es2_info, 45 [board_ich8lan] = &e1000_ich8_info, 46 [board_ich9lan] = &e1000_ich9_info, 47 [board_ich10lan] = &e1000_ich10_info, 48 [board_pchlan] = &e1000_pch_info, 49 [board_pch2lan] = &e1000_pch2_info, 50 [board_pch_lpt] = &e1000_pch_lpt_info, 51 [board_pch_spt] = &e1000_pch_spt_info, 52 [board_pch_cnp] = &e1000_pch_cnp_info, 53 [board_pch_tgp] = &e1000_pch_tgp_info, 54}; 55 56struct e1000_reg_info { 57 u32 ofs; 58 char *name; 59}; 60 61static const struct e1000_reg_info e1000_reg_info_tbl[] = { 62 /* General Registers */ 63 {E1000_CTRL, "CTRL"}, 64 {E1000_STATUS, "STATUS"}, 65 {E1000_CTRL_EXT, "CTRL_EXT"}, 66 67 /* Interrupt Registers */ 68 {E1000_ICR, "ICR"}, 69 70 /* Rx Registers */ 71 {E1000_RCTL, "RCTL"}, 72 {E1000_RDLEN(0), "RDLEN"}, 73 {E1000_RDH(0), "RDH"}, 74 {E1000_RDT(0), "RDT"}, 75 {E1000_RDTR, "RDTR"}, 76 {E1000_RXDCTL(0), "RXDCTL"}, 77 {E1000_ERT, "ERT"}, 78 {E1000_RDBAL(0), "RDBAL"}, 79 {E1000_RDBAH(0), "RDBAH"}, 80 {E1000_RDFH, "RDFH"}, 81 {E1000_RDFT, "RDFT"}, 82 {E1000_RDFHS, "RDFHS"}, 83 {E1000_RDFTS, "RDFTS"}, 84 {E1000_RDFPC, "RDFPC"}, 85 86 /* Tx Registers */ 87 {E1000_TCTL, "TCTL"}, 88 {E1000_TDBAL(0), "TDBAL"}, 89 {E1000_TDBAH(0), "TDBAH"}, 90 {E1000_TDLEN(0), "TDLEN"}, 91 {E1000_TDH(0), "TDH"}, 92 {E1000_TDT(0), "TDT"}, 93 {E1000_TIDV, "TIDV"}, 94 {E1000_TXDCTL(0), "TXDCTL"}, 95 {E1000_TADV, "TADV"}, 96 {E1000_TARC(0), "TARC"}, 97 {E1000_TDFH, "TDFH"}, 98 {E1000_TDFT, "TDFT"}, 99 {E1000_TDFHS, "TDFHS"}, 100 {E1000_TDFTS, "TDFTS"}, 101 {E1000_TDFPC, "TDFPC"}, 102 103 /* List Terminator */ 104 {0, NULL} 105}; 106 107/** 108 * __ew32_prepare - prepare to write to MAC CSR register on certain parts 109 * @hw: pointer to the HW structure 110 * 111 * When updating the MAC CSR registers, the Manageability Engine (ME) could 112 * be accessing the registers at the same time. Normally, this is handled in 113 * h/w by an arbiter but on some parts there is a bug that acknowledges Host 114 * accesses later than it should which could result in the register to have 115 * an incorrect value. Workaround this by checking the FWSM register which 116 * has bit 24 set while ME is accessing MAC CSR registers, wait if it is set 117 * and try again a number of times. 118 **/ 119static void __ew32_prepare(struct e1000_hw *hw) 120{ 121 s32 i = E1000_ICH_FWSM_PCIM2PCI_COUNT; 122 123 while ((er32(FWSM) & E1000_ICH_FWSM_PCIM2PCI) && --i) 124 udelay(50); 125} 126 127void __ew32(struct e1000_hw *hw, unsigned long reg, u32 val) 128{ 129 if (hw->adapter->flags2 & FLAG2_PCIM2PCI_ARBITER_WA) 130 __ew32_prepare(hw); 131 132 writel(val, hw->hw_addr + reg); 133} 134 135/** 136 * e1000_regdump - register printout routine 137 * @hw: pointer to the HW structure 138 * @reginfo: pointer to the register info table 139 **/ 140static void e1000_regdump(struct e1000_hw *hw, struct e1000_reg_info *reginfo) 141{ 142 int n = 0; 143 char rname[16]; 144 u32 regs[8]; 145 146 switch (reginfo->ofs) { 147 case E1000_RXDCTL(0): 148 for (n = 0; n < 2; n++) 149 regs[n] = __er32(hw, E1000_RXDCTL(n)); 150 break; 151 case E1000_TXDCTL(0): 152 for (n = 0; n < 2; n++) 153 regs[n] = __er32(hw, E1000_TXDCTL(n)); 154 break; 155 case E1000_TARC(0): 156 for (n = 0; n < 2; n++) 157 regs[n] = __er32(hw, E1000_TARC(n)); 158 break; 159 default: 160 pr_info("%-15s %08x\n", 161 reginfo->name, __er32(hw, reginfo->ofs)); 162 return; 163 } 164 165 snprintf(rname, 16, "%s%s", reginfo->name, "[0-1]"); 166 pr_info("%-15s %08x %08x\n", rname, regs[0], regs[1]); 167} 168 169static void e1000e_dump_ps_pages(struct e1000_adapter *adapter, 170 struct e1000_buffer *bi) 171{ 172 int i; 173 struct e1000_ps_page *ps_page; 174 175 for (i = 0; i < adapter->rx_ps_pages; i++) { 176 ps_page = &bi->ps_pages[i]; 177 178 if (ps_page->page) { 179 pr_info("packet dump for ps_page %d:\n", i); 180 print_hex_dump(KERN_INFO, "", DUMP_PREFIX_ADDRESS, 181 16, 1, page_address(ps_page->page), 182 PAGE_SIZE, true); 183 } 184 } 185} 186 187/** 188 * e1000e_dump - Print registers, Tx-ring and Rx-ring 189 * @adapter: board private structure 190 **/ 191static void e1000e_dump(struct e1000_adapter *adapter) 192{ 193 struct net_device *netdev = adapter->netdev; 194 struct e1000_hw *hw = &adapter->hw; 195 struct e1000_reg_info *reginfo; 196 struct e1000_ring *tx_ring = adapter->tx_ring; 197 struct e1000_tx_desc *tx_desc; 198 struct my_u0 { 199 __le64 a; 200 __le64 b; 201 } *u0; 202 struct e1000_buffer *buffer_info; 203 struct e1000_ring *rx_ring = adapter->rx_ring; 204 union e1000_rx_desc_packet_split *rx_desc_ps; 205 union e1000_rx_desc_extended *rx_desc; 206 struct my_u1 { 207 __le64 a; 208 __le64 b; 209 __le64 c; 210 __le64 d; 211 } *u1; 212 u32 staterr; 213 int i = 0; 214 215 if (!netif_msg_hw(adapter)) 216 return; 217 218 /* Print netdevice Info */ 219 if (netdev) { 220 dev_info(&adapter->pdev->dev, "Net device Info\n"); 221 pr_info("Device Name state trans_start\n"); 222 pr_info("%-15s %016lX %016lX\n", netdev->name, 223 netdev->state, dev_trans_start(netdev)); 224 } 225 226 /* Print Registers */ 227 dev_info(&adapter->pdev->dev, "Register Dump\n"); 228 pr_info(" Register Name Value\n"); 229 for (reginfo = (struct e1000_reg_info *)e1000_reg_info_tbl; 230 reginfo->name; reginfo++) { 231 e1000_regdump(hw, reginfo); 232 } 233 234 /* Print Tx Ring Summary */ 235 if (!netdev || !netif_running(netdev)) 236 return; 237 238 dev_info(&adapter->pdev->dev, "Tx Ring Summary\n"); 239 pr_info("Queue [NTU] [NTC] [bi(ntc)->dma ] leng ntw timestamp\n"); 240 buffer_info = &tx_ring->buffer_info[tx_ring->next_to_clean]; 241 pr_info(" %5d %5X %5X %016llX %04X %3X %016llX\n", 242 0, tx_ring->next_to_use, tx_ring->next_to_clean, 243 (unsigned long long)buffer_info->dma, 244 buffer_info->length, 245 buffer_info->next_to_watch, 246 (unsigned long long)buffer_info->time_stamp); 247 248 /* Print Tx Ring */ 249 if (!netif_msg_tx_done(adapter)) 250 goto rx_ring_summary; 251 252 dev_info(&adapter->pdev->dev, "Tx Ring Dump\n"); 253 254 /* Transmit Descriptor Formats - DEXT[29] is 0 (Legacy) or 1 (Extended) 255 * 256 * Legacy Transmit Descriptor 257 * +--------------------------------------------------------------+ 258 * 0 | Buffer Address [63:0] (Reserved on Write Back) | 259 * +--------------------------------------------------------------+ 260 * 8 | Special | CSS | Status | CMD | CSO | Length | 261 * +--------------------------------------------------------------+ 262 * 63 48 47 36 35 32 31 24 23 16 15 0 263 * 264 * Extended Context Descriptor (DTYP=0x0) for TSO or checksum offload 265 * 63 48 47 40 39 32 31 16 15 8 7 0 266 * +----------------------------------------------------------------+ 267 * 0 | TUCSE | TUCS0 | TUCSS | IPCSE | IPCS0 | IPCSS | 268 * +----------------------------------------------------------------+ 269 * 8 | MSS | HDRLEN | RSV | STA | TUCMD | DTYP | PAYLEN | 270 * +----------------------------------------------------------------+ 271 * 63 48 47 40 39 36 35 32 31 24 23 20 19 0 272 * 273 * Extended Data Descriptor (DTYP=0x1) 274 * +----------------------------------------------------------------+ 275 * 0 | Buffer Address [63:0] | 276 * +----------------------------------------------------------------+ 277 * 8 | VLAN tag | POPTS | Rsvd | Status | Command | DTYP | DTALEN | 278 * +----------------------------------------------------------------+ 279 * 63 48 47 40 39 36 35 32 31 24 23 20 19 0 280 */ 281 pr_info("Tl[desc] [address 63:0 ] [SpeCssSCmCsLen] [bi->dma ] leng ntw timestamp bi->skb <-- Legacy format\n"); 282 pr_info("Tc[desc] [Ce CoCsIpceCoS] [MssHlRSCm0Plen] [bi->dma ] leng ntw timestamp bi->skb <-- Ext Context format\n"); 283 pr_info("Td[desc] [address 63:0 ] [VlaPoRSCm1Dlen] [bi->dma ] leng ntw timestamp bi->skb <-- Ext Data format\n"); 284 for (i = 0; tx_ring->desc && (i < tx_ring->count); i++) { 285 const char *next_desc; 286 tx_desc = E1000_TX_DESC(*tx_ring, i); 287 buffer_info = &tx_ring->buffer_info[i]; 288 u0 = (struct my_u0 *)tx_desc; 289 if (i == tx_ring->next_to_use && i == tx_ring->next_to_clean) 290 next_desc = " NTC/U"; 291 else if (i == tx_ring->next_to_use) 292 next_desc = " NTU"; 293 else if (i == tx_ring->next_to_clean) 294 next_desc = " NTC"; 295 else 296 next_desc = ""; 297 pr_info("T%c[0x%03X] %016llX %016llX %016llX %04X %3X %016llX %p%s\n", 298 (!(le64_to_cpu(u0->b) & BIT(29)) ? 'l' : 299 ((le64_to_cpu(u0->b) & BIT(20)) ? 'd' : 'c')), 300 i, 301 (unsigned long long)le64_to_cpu(u0->a), 302 (unsigned long long)le64_to_cpu(u0->b), 303 (unsigned long long)buffer_info->dma, 304 buffer_info->length, buffer_info->next_to_watch, 305 (unsigned long long)buffer_info->time_stamp, 306 buffer_info->skb, next_desc); 307 308 if (netif_msg_pktdata(adapter) && buffer_info->skb) 309 print_hex_dump(KERN_INFO, "", DUMP_PREFIX_ADDRESS, 310 16, 1, buffer_info->skb->data, 311 buffer_info->skb->len, true); 312 } 313 314 /* Print Rx Ring Summary */ 315rx_ring_summary: 316 dev_info(&adapter->pdev->dev, "Rx Ring Summary\n"); 317 pr_info("Queue [NTU] [NTC]\n"); 318 pr_info(" %5d %5X %5X\n", 319 0, rx_ring->next_to_use, rx_ring->next_to_clean); 320 321 /* Print Rx Ring */ 322 if (!netif_msg_rx_status(adapter)) 323 return; 324 325 dev_info(&adapter->pdev->dev, "Rx Ring Dump\n"); 326 switch (adapter->rx_ps_pages) { 327 case 1: 328 case 2: 329 case 3: 330 /* [Extended] Packet Split Receive Descriptor Format 331 * 332 * +-----------------------------------------------------+ 333 * 0 | Buffer Address 0 [63:0] | 334 * +-----------------------------------------------------+ 335 * 8 | Buffer Address 1 [63:0] | 336 * +-----------------------------------------------------+ 337 * 16 | Buffer Address 2 [63:0] | 338 * +-----------------------------------------------------+ 339 * 24 | Buffer Address 3 [63:0] | 340 * +-----------------------------------------------------+ 341 */ 342 pr_info("R [desc] [buffer 0 63:0 ] [buffer 1 63:0 ] [buffer 2 63:0 ] [buffer 3 63:0 ] [bi->dma ] [bi->skb] <-- Ext Pkt Split format\n"); 343 /* [Extended] Receive Descriptor (Write-Back) Format 344 * 345 * 63 48 47 32 31 13 12 8 7 4 3 0 346 * +------------------------------------------------------+ 347 * 0 | Packet | IP | Rsvd | MRQ | Rsvd | MRQ RSS | 348 * | Checksum | Ident | | Queue | | Type | 349 * +------------------------------------------------------+ 350 * 8 | VLAN Tag | Length | Extended Error | Extended Status | 351 * +------------------------------------------------------+ 352 * 63 48 47 32 31 20 19 0 353 */ 354 pr_info("RWB[desc] [ck ipid mrqhsh] [vl l0 ee es] [ l3 l2 l1 hs] [reserved ] ---------------- [bi->skb] <-- Ext Rx Write-Back format\n"); 355 for (i = 0; i < rx_ring->count; i++) { 356 const char *next_desc; 357 buffer_info = &rx_ring->buffer_info[i]; 358 rx_desc_ps = E1000_RX_DESC_PS(*rx_ring, i); 359 u1 = (struct my_u1 *)rx_desc_ps; 360 staterr = 361 le32_to_cpu(rx_desc_ps->wb.middle.status_error); 362 363 if (i == rx_ring->next_to_use) 364 next_desc = " NTU"; 365 else if (i == rx_ring->next_to_clean) 366 next_desc = " NTC"; 367 else 368 next_desc = ""; 369 370 if (staterr & E1000_RXD_STAT_DD) { 371 /* Descriptor Done */ 372 pr_info("%s[0x%03X] %016llX %016llX %016llX %016llX ---------------- %p%s\n", 373 "RWB", i, 374 (unsigned long long)le64_to_cpu(u1->a), 375 (unsigned long long)le64_to_cpu(u1->b), 376 (unsigned long long)le64_to_cpu(u1->c), 377 (unsigned long long)le64_to_cpu(u1->d), 378 buffer_info->skb, next_desc); 379 } else { 380 pr_info("%s[0x%03X] %016llX %016llX %016llX %016llX %016llX %p%s\n", 381 "R ", i, 382 (unsigned long long)le64_to_cpu(u1->a), 383 (unsigned long long)le64_to_cpu(u1->b), 384 (unsigned long long)le64_to_cpu(u1->c), 385 (unsigned long long)le64_to_cpu(u1->d), 386 (unsigned long long)buffer_info->dma, 387 buffer_info->skb, next_desc); 388 389 if (netif_msg_pktdata(adapter)) 390 e1000e_dump_ps_pages(adapter, 391 buffer_info); 392 } 393 } 394 break; 395 default: 396 case 0: 397 /* Extended Receive Descriptor (Read) Format 398 * 399 * +-----------------------------------------------------+ 400 * 0 | Buffer Address [63:0] | 401 * +-----------------------------------------------------+ 402 * 8 | Reserved | 403 * +-----------------------------------------------------+ 404 */ 405 pr_info("R [desc] [buf addr 63:0 ] [reserved 63:0 ] [bi->dma ] [bi->skb] <-- Ext (Read) format\n"); 406 /* Extended Receive Descriptor (Write-Back) Format 407 * 408 * 63 48 47 32 31 24 23 4 3 0 409 * +------------------------------------------------------+ 410 * | RSS Hash | | | | 411 * 0 +-------------------+ Rsvd | Reserved | MRQ RSS | 412 * | Packet | IP | | | Type | 413 * | Checksum | Ident | | | | 414 * +------------------------------------------------------+ 415 * 8 | VLAN Tag | Length | Extended Error | Extended Status | 416 * +------------------------------------------------------+ 417 * 63 48 47 32 31 20 19 0 418 */ 419 pr_info("RWB[desc] [cs ipid mrq] [vt ln xe xs] [bi->skb] <-- Ext (Write-Back) format\n"); 420 421 for (i = 0; i < rx_ring->count; i++) { 422 const char *next_desc; 423 424 buffer_info = &rx_ring->buffer_info[i]; 425 rx_desc = E1000_RX_DESC_EXT(*rx_ring, i); 426 u1 = (struct my_u1 *)rx_desc; 427 staterr = le32_to_cpu(rx_desc->wb.upper.status_error); 428 429 if (i == rx_ring->next_to_use) 430 next_desc = " NTU"; 431 else if (i == rx_ring->next_to_clean) 432 next_desc = " NTC"; 433 else 434 next_desc = ""; 435 436 if (staterr & E1000_RXD_STAT_DD) { 437 /* Descriptor Done */ 438 pr_info("%s[0x%03X] %016llX %016llX ---------------- %p%s\n", 439 "RWB", i, 440 (unsigned long long)le64_to_cpu(u1->a), 441 (unsigned long long)le64_to_cpu(u1->b), 442 buffer_info->skb, next_desc); 443 } else { 444 pr_info("%s[0x%03X] %016llX %016llX %016llX %p%s\n", 445 "R ", i, 446 (unsigned long long)le64_to_cpu(u1->a), 447 (unsigned long long)le64_to_cpu(u1->b), 448 (unsigned long long)buffer_info->dma, 449 buffer_info->skb, next_desc); 450 451 if (netif_msg_pktdata(adapter) && 452 buffer_info->skb) 453 print_hex_dump(KERN_INFO, "", 454 DUMP_PREFIX_ADDRESS, 16, 455 1, 456 buffer_info->skb->data, 457 adapter->rx_buffer_len, 458 true); 459 } 460 } 461 } 462} 463 464/** 465 * e1000_desc_unused - calculate if we have unused descriptors 466 * @ring: pointer to ring struct to perform calculation on 467 **/ 468static int e1000_desc_unused(struct e1000_ring *ring) 469{ 470 if (ring->next_to_clean > ring->next_to_use) 471 return ring->next_to_clean - ring->next_to_use - 1; 472 473 return ring->count + ring->next_to_clean - ring->next_to_use - 1; 474} 475 476/** 477 * e1000e_systim_to_hwtstamp - convert system time value to hw time stamp 478 * @adapter: board private structure 479 * @hwtstamps: time stamp structure to update 480 * @systim: unsigned 64bit system time value. 481 * 482 * Convert the system time value stored in the RX/TXSTMP registers into a 483 * hwtstamp which can be used by the upper level time stamping functions. 484 * 485 * The 'systim_lock' spinlock is used to protect the consistency of the 486 * system time value. This is needed because reading the 64 bit time 487 * value involves reading two 32 bit registers. The first read latches the 488 * value. 489 **/ 490static void e1000e_systim_to_hwtstamp(struct e1000_adapter *adapter, 491 struct skb_shared_hwtstamps *hwtstamps, 492 u64 systim) 493{ 494 u64 ns; 495 unsigned long flags; 496 497 spin_lock_irqsave(&adapter->systim_lock, flags); 498 ns = timecounter_cyc2time(&adapter->tc, systim); 499 spin_unlock_irqrestore(&adapter->systim_lock, flags); 500 501 memset(hwtstamps, 0, sizeof(*hwtstamps)); 502 hwtstamps->hwtstamp = ns_to_ktime(ns); 503} 504 505/** 506 * e1000e_rx_hwtstamp - utility function which checks for Rx time stamp 507 * @adapter: board private structure 508 * @status: descriptor extended error and status field 509 * @skb: particular skb to include time stamp 510 * 511 * If the time stamp is valid, convert it into the timecounter ns value 512 * and store that result into the shhwtstamps structure which is passed 513 * up the network stack. 514 **/ 515static void e1000e_rx_hwtstamp(struct e1000_adapter *adapter, u32 status, 516 struct sk_buff *skb) 517{ 518 struct e1000_hw *hw = &adapter->hw; 519 u64 rxstmp; 520 521 if (!(adapter->flags & FLAG_HAS_HW_TIMESTAMP) || 522 !(status & E1000_RXDEXT_STATERR_TST) || 523 !(er32(TSYNCRXCTL) & E1000_TSYNCRXCTL_VALID)) 524 return; 525 526 /* The Rx time stamp registers contain the time stamp. No other 527 * received packet will be time stamped until the Rx time stamp 528 * registers are read. Because only one packet can be time stamped 529 * at a time, the register values must belong to this packet and 530 * therefore none of the other additional attributes need to be 531 * compared. 532 */ 533 rxstmp = (u64)er32(RXSTMPL); 534 rxstmp |= (u64)er32(RXSTMPH) << 32; 535 e1000e_systim_to_hwtstamp(adapter, skb_hwtstamps(skb), rxstmp); 536 537 adapter->flags2 &= ~FLAG2_CHECK_RX_HWTSTAMP; 538} 539 540/** 541 * e1000_receive_skb - helper function to handle Rx indications 542 * @adapter: board private structure 543 * @netdev: pointer to netdev struct 544 * @staterr: descriptor extended error and status field as written by hardware 545 * @vlan: descriptor vlan field as written by hardware (no le/be conversion) 546 * @skb: pointer to sk_buff to be indicated to stack 547 **/ 548static void e1000_receive_skb(struct e1000_adapter *adapter, 549 struct net_device *netdev, struct sk_buff *skb, 550 u32 staterr, __le16 vlan) 551{ 552 u16 tag = le16_to_cpu(vlan); 553 554 e1000e_rx_hwtstamp(adapter, staterr, skb); 555 556 skb->protocol = eth_type_trans(skb, netdev); 557 558 if (staterr & E1000_RXD_STAT_VP) 559 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), tag); 560 561 napi_gro_receive(&adapter->napi, skb); 562} 563 564/** 565 * e1000_rx_checksum - Receive Checksum Offload 566 * @adapter: board private structure 567 * @status_err: receive descriptor status and error fields 568 * @skb: socket buffer with received data 569 **/ 570static void e1000_rx_checksum(struct e1000_adapter *adapter, u32 status_err, 571 struct sk_buff *skb) 572{ 573 u16 status = (u16)status_err; 574 u8 errors = (u8)(status_err >> 24); 575 576 skb_checksum_none_assert(skb); 577 578 /* Rx checksum disabled */ 579 if (!(adapter->netdev->features & NETIF_F_RXCSUM)) 580 return; 581 582 /* Ignore Checksum bit is set */ 583 if (status & E1000_RXD_STAT_IXSM) 584 return; 585 586 /* TCP/UDP checksum error bit or IP checksum error bit is set */ 587 if (errors & (E1000_RXD_ERR_TCPE | E1000_RXD_ERR_IPE)) { 588 /* let the stack verify checksum errors */ 589 adapter->hw_csum_err++; 590 return; 591 } 592 593 /* TCP/UDP Checksum has not been calculated */ 594 if (!(status & (E1000_RXD_STAT_TCPCS | E1000_RXD_STAT_UDPCS))) 595 return; 596 597 /* It must be a TCP or UDP packet with a valid checksum */ 598 skb->ip_summed = CHECKSUM_UNNECESSARY; 599 adapter->hw_csum_good++; 600} 601 602static void e1000e_update_rdt_wa(struct e1000_ring *rx_ring, unsigned int i) 603{ 604 struct e1000_adapter *adapter = rx_ring->adapter; 605 struct e1000_hw *hw = &adapter->hw; 606 607 __ew32_prepare(hw); 608 writel(i, rx_ring->tail); 609 610 if (unlikely(i != readl(rx_ring->tail))) { 611 u32 rctl = er32(RCTL); 612 613 ew32(RCTL, rctl & ~E1000_RCTL_EN); 614 e_err("ME firmware caused invalid RDT - resetting\n"); 615 schedule_work(&adapter->reset_task); 616 } 617} 618 619static void e1000e_update_tdt_wa(struct e1000_ring *tx_ring, unsigned int i) 620{ 621 struct e1000_adapter *adapter = tx_ring->adapter; 622 struct e1000_hw *hw = &adapter->hw; 623 624 __ew32_prepare(hw); 625 writel(i, tx_ring->tail); 626 627 if (unlikely(i != readl(tx_ring->tail))) { 628 u32 tctl = er32(TCTL); 629 630 ew32(TCTL, tctl & ~E1000_TCTL_EN); 631 e_err("ME firmware caused invalid TDT - resetting\n"); 632 schedule_work(&adapter->reset_task); 633 } 634} 635 636/** 637 * e1000_alloc_rx_buffers - Replace used receive buffers 638 * @rx_ring: Rx descriptor ring 639 * @cleaned_count: number to reallocate 640 * @gfp: flags for allocation 641 **/ 642static void e1000_alloc_rx_buffers(struct e1000_ring *rx_ring, 643 int cleaned_count, gfp_t gfp) 644{ 645 struct e1000_adapter *adapter = rx_ring->adapter; 646 struct net_device *netdev = adapter->netdev; 647 struct pci_dev *pdev = adapter->pdev; 648 union e1000_rx_desc_extended *rx_desc; 649 struct e1000_buffer *buffer_info; 650 struct sk_buff *skb; 651 unsigned int i; 652 unsigned int bufsz = adapter->rx_buffer_len; 653 654 i = rx_ring->next_to_use; 655 buffer_info = &rx_ring->buffer_info[i]; 656 657 while (cleaned_count--) { 658 skb = buffer_info->skb; 659 if (skb) { 660 skb_trim(skb, 0); 661 goto map_skb; 662 } 663 664 skb = __netdev_alloc_skb_ip_align(netdev, bufsz, gfp); 665 if (!skb) { 666 /* Better luck next round */ 667 adapter->alloc_rx_buff_failed++; 668 break; 669 } 670 671 buffer_info->skb = skb; 672map_skb: 673 buffer_info->dma = dma_map_single(&pdev->dev, skb->data, 674 adapter->rx_buffer_len, 675 DMA_FROM_DEVICE); 676 if (dma_mapping_error(&pdev->dev, buffer_info->dma)) { 677 dev_err(&pdev->dev, "Rx DMA map failed\n"); 678 adapter->rx_dma_failed++; 679 break; 680 } 681 682 rx_desc = E1000_RX_DESC_EXT(*rx_ring, i); 683 rx_desc->read.buffer_addr = cpu_to_le64(buffer_info->dma); 684 685 if (unlikely(!(i & (E1000_RX_BUFFER_WRITE - 1)))) { 686 /* Force memory writes to complete before letting h/w 687 * know there are new descriptors to fetch. (Only 688 * applicable for weak-ordered memory model archs, 689 * such as IA-64). 690 */ 691 wmb(); 692 if (adapter->flags2 & FLAG2_PCIM2PCI_ARBITER_WA) 693 e1000e_update_rdt_wa(rx_ring, i); 694 else 695 writel(i, rx_ring->tail); 696 } 697 i++; 698 if (i == rx_ring->count) 699 i = 0; 700 buffer_info = &rx_ring->buffer_info[i]; 701 } 702 703 rx_ring->next_to_use = i; 704} 705 706/** 707 * e1000_alloc_rx_buffers_ps - Replace used receive buffers; packet split 708 * @rx_ring: Rx descriptor ring 709 * @cleaned_count: number to reallocate 710 * @gfp: flags for allocation 711 **/ 712static void e1000_alloc_rx_buffers_ps(struct e1000_ring *rx_ring, 713 int cleaned_count, gfp_t gfp) 714{ 715 struct e1000_adapter *adapter = rx_ring->adapter; 716 struct net_device *netdev = adapter->netdev; 717 struct pci_dev *pdev = adapter->pdev; 718 union e1000_rx_desc_packet_split *rx_desc; 719 struct e1000_buffer *buffer_info; 720 struct e1000_ps_page *ps_page; 721 struct sk_buff *skb; 722 unsigned int i, j; 723 724 i = rx_ring->next_to_use; 725 buffer_info = &rx_ring->buffer_info[i]; 726 727 while (cleaned_count--) { 728 rx_desc = E1000_RX_DESC_PS(*rx_ring, i); 729 730 for (j = 0; j < PS_PAGE_BUFFERS; j++) { 731 ps_page = &buffer_info->ps_pages[j]; 732 if (j >= adapter->rx_ps_pages) { 733 /* all unused desc entries get hw null ptr */ 734 rx_desc->read.buffer_addr[j + 1] = 735 ~cpu_to_le64(0); 736 continue; 737 } 738 if (!ps_page->page) { 739 ps_page->page = alloc_page(gfp); 740 if (!ps_page->page) { 741 adapter->alloc_rx_buff_failed++; 742 goto no_buffers; 743 } 744 ps_page->dma = dma_map_page(&pdev->dev, 745 ps_page->page, 746 0, PAGE_SIZE, 747 DMA_FROM_DEVICE); 748 if (dma_mapping_error(&pdev->dev, 749 ps_page->dma)) { 750 dev_err(&adapter->pdev->dev, 751 "Rx DMA page map failed\n"); 752 adapter->rx_dma_failed++; 753 goto no_buffers; 754 } 755 } 756 /* Refresh the desc even if buffer_addrs 757 * didn't change because each write-back 758 * erases this info. 759 */ 760 rx_desc->read.buffer_addr[j + 1] = 761 cpu_to_le64(ps_page->dma); 762 } 763 764 skb = __netdev_alloc_skb_ip_align(netdev, adapter->rx_ps_bsize0, 765 gfp); 766 767 if (!skb) { 768 adapter->alloc_rx_buff_failed++; 769 break; 770 } 771 772 buffer_info->skb = skb; 773 buffer_info->dma = dma_map_single(&pdev->dev, skb->data, 774 adapter->rx_ps_bsize0, 775 DMA_FROM_DEVICE); 776 if (dma_mapping_error(&pdev->dev, buffer_info->dma)) { 777 dev_err(&pdev->dev, "Rx DMA map failed\n"); 778 adapter->rx_dma_failed++; 779 /* cleanup skb */ 780 dev_kfree_skb_any(skb); 781 buffer_info->skb = NULL; 782 break; 783 } 784 785 rx_desc->read.buffer_addr[0] = cpu_to_le64(buffer_info->dma); 786 787 if (unlikely(!(i & (E1000_RX_BUFFER_WRITE - 1)))) { 788 /* Force memory writes to complete before letting h/w 789 * know there are new descriptors to fetch. (Only 790 * applicable for weak-ordered memory model archs, 791 * such as IA-64). 792 */ 793 wmb(); 794 if (adapter->flags2 & FLAG2_PCIM2PCI_ARBITER_WA) 795 e1000e_update_rdt_wa(rx_ring, i << 1); 796 else 797 writel(i << 1, rx_ring->tail); 798 } 799 800 i++; 801 if (i == rx_ring->count) 802 i = 0; 803 buffer_info = &rx_ring->buffer_info[i]; 804 } 805 806no_buffers: 807 rx_ring->next_to_use = i; 808} 809 810/** 811 * e1000_alloc_jumbo_rx_buffers - Replace used jumbo receive buffers 812 * @rx_ring: Rx descriptor ring 813 * @cleaned_count: number of buffers to allocate this pass 814 * @gfp: flags for allocation 815 **/ 816 817static void e1000_alloc_jumbo_rx_buffers(struct e1000_ring *rx_ring, 818 int cleaned_count, gfp_t gfp) 819{ 820 struct e1000_adapter *adapter = rx_ring->adapter; 821 struct net_device *netdev = adapter->netdev; 822 struct pci_dev *pdev = adapter->pdev; 823 union e1000_rx_desc_extended *rx_desc; 824 struct e1000_buffer *buffer_info; 825 struct sk_buff *skb; 826 unsigned int i; 827 unsigned int bufsz = 256 - 16; /* for skb_reserve */ 828 829 i = rx_ring->next_to_use; 830 buffer_info = &rx_ring->buffer_info[i]; 831 832 while (cleaned_count--) { 833 skb = buffer_info->skb; 834 if (skb) { 835 skb_trim(skb, 0); 836 goto check_page; 837 } 838 839 skb = __netdev_alloc_skb_ip_align(netdev, bufsz, gfp); 840 if (unlikely(!skb)) { 841 /* Better luck next round */ 842 adapter->alloc_rx_buff_failed++; 843 break; 844 } 845 846 buffer_info->skb = skb; 847check_page: 848 /* allocate a new page if necessary */ 849 if (!buffer_info->page) { 850 buffer_info->page = alloc_page(gfp); 851 if (unlikely(!buffer_info->page)) { 852 adapter->alloc_rx_buff_failed++; 853 break; 854 } 855 } 856 857 if (!buffer_info->dma) { 858 buffer_info->dma = dma_map_page(&pdev->dev, 859 buffer_info->page, 0, 860 PAGE_SIZE, 861 DMA_FROM_DEVICE); 862 if (dma_mapping_error(&pdev->dev, buffer_info->dma)) { 863 adapter->alloc_rx_buff_failed++; 864 break; 865 } 866 } 867 868 rx_desc = E1000_RX_DESC_EXT(*rx_ring, i); 869 rx_desc->read.buffer_addr = cpu_to_le64(buffer_info->dma); 870 871 if (unlikely(++i == rx_ring->count)) 872 i = 0; 873 buffer_info = &rx_ring->buffer_info[i]; 874 } 875 876 if (likely(rx_ring->next_to_use != i)) { 877 rx_ring->next_to_use = i; 878 if (unlikely(i-- == 0)) 879 i = (rx_ring->count - 1); 880 881 /* Force memory writes to complete before letting h/w 882 * know there are new descriptors to fetch. (Only 883 * applicable for weak-ordered memory model archs, 884 * such as IA-64). 885 */ 886 wmb(); 887 if (adapter->flags2 & FLAG2_PCIM2PCI_ARBITER_WA) 888 e1000e_update_rdt_wa(rx_ring, i); 889 else 890 writel(i, rx_ring->tail); 891 } 892} 893 894static inline void e1000_rx_hash(struct net_device *netdev, __le32 rss, 895 struct sk_buff *skb) 896{ 897 if (netdev->features & NETIF_F_RXHASH) 898 skb_set_hash(skb, le32_to_cpu(rss), PKT_HASH_TYPE_L3); 899} 900 901/** 902 * e1000_clean_rx_irq - Send received data up the network stack 903 * @rx_ring: Rx descriptor ring 904 * @work_done: output parameter for indicating completed work 905 * @work_to_do: how many packets we can clean 906 * 907 * the return value indicates whether actual cleaning was done, there 908 * is no guarantee that everything was cleaned 909 **/ 910static bool e1000_clean_rx_irq(struct e1000_ring *rx_ring, int *work_done, 911 int work_to_do) 912{ 913 struct e1000_adapter *adapter = rx_ring->adapter; 914 struct net_device *netdev = adapter->netdev; 915 struct pci_dev *pdev = adapter->pdev; 916 struct e1000_hw *hw = &adapter->hw; 917 union e1000_rx_desc_extended *rx_desc, *next_rxd; 918 struct e1000_buffer *buffer_info, *next_buffer; 919 u32 length, staterr; 920 unsigned int i; 921 int cleaned_count = 0; 922 bool cleaned = false; 923 unsigned int total_rx_bytes = 0, total_rx_packets = 0; 924 925 i = rx_ring->next_to_clean; 926 rx_desc = E1000_RX_DESC_EXT(*rx_ring, i); 927 staterr = le32_to_cpu(rx_desc->wb.upper.status_error); 928 buffer_info = &rx_ring->buffer_info[i]; 929 930 while (staterr & E1000_RXD_STAT_DD) { 931 struct sk_buff *skb; 932 933 if (*work_done >= work_to_do) 934 break; 935 (*work_done)++; 936 dma_rmb(); /* read descriptor and rx_buffer_info after status DD */ 937 938 skb = buffer_info->skb; 939 buffer_info->skb = NULL; 940 941 prefetch(skb->data - NET_IP_ALIGN); 942 943 i++; 944 if (i == rx_ring->count) 945 i = 0; 946 next_rxd = E1000_RX_DESC_EXT(*rx_ring, i); 947 prefetch(next_rxd); 948 949 next_buffer = &rx_ring->buffer_info[i]; 950 951 cleaned = true; 952 cleaned_count++; 953 dma_unmap_single(&pdev->dev, buffer_info->dma, 954 adapter->rx_buffer_len, DMA_FROM_DEVICE); 955 buffer_info->dma = 0; 956 957 length = le16_to_cpu(rx_desc->wb.upper.length); 958 959 /* !EOP means multiple descriptors were used to store a single 960 * packet, if that's the case we need to toss it. In fact, we 961 * need to toss every packet with the EOP bit clear and the 962 * next frame that _does_ have the EOP bit set, as it is by 963 * definition only a frame fragment 964 */ 965 if (unlikely(!(staterr & E1000_RXD_STAT_EOP))) 966 adapter->flags2 |= FLAG2_IS_DISCARDING; 967 968 if (adapter->flags2 & FLAG2_IS_DISCARDING) { 969 /* All receives must fit into a single buffer */ 970 e_dbg("Receive packet consumed multiple buffers\n"); 971 /* recycle */ 972 buffer_info->skb = skb; 973 if (staterr & E1000_RXD_STAT_EOP) 974 adapter->flags2 &= ~FLAG2_IS_DISCARDING; 975 goto next_desc; 976 } 977 978 if (unlikely((staterr & E1000_RXDEXT_ERR_FRAME_ERR_MASK) && 979 !(netdev->features & NETIF_F_RXALL))) { 980 /* recycle */ 981 buffer_info->skb = skb; 982 goto next_desc; 983 } 984 985 /* adjust length to remove Ethernet CRC */ 986 if (!(adapter->flags2 & FLAG2_CRC_STRIPPING)) { 987 /* If configured to store CRC, don't subtract FCS, 988 * but keep the FCS bytes out of the total_rx_bytes 989 * counter 990 */ 991 if (netdev->features & NETIF_F_RXFCS) 992 total_rx_bytes -= 4; 993 else 994 length -= 4; 995 } 996 997 total_rx_bytes += length; 998 total_rx_packets++; 999 1000 /* code added for copybreak, this should improve 1001 * performance for small packets with large amounts 1002 * of reassembly being done in the stack 1003 */ 1004 if (length < copybreak) { 1005 struct sk_buff *new_skb = 1006 napi_alloc_skb(&adapter->napi, length); 1007 if (new_skb) { 1008 skb_copy_to_linear_data_offset(new_skb, 1009 -NET_IP_ALIGN, 1010 (skb->data - 1011 NET_IP_ALIGN), 1012 (length + 1013 NET_IP_ALIGN)); 1014 /* save the skb in buffer_info as good */ 1015 buffer_info->skb = skb; 1016 skb = new_skb; 1017 } 1018 /* else just continue with the old one */ 1019 } 1020 /* end copybreak code */ 1021 skb_put(skb, length); 1022 1023 /* Receive Checksum Offload */ 1024 e1000_rx_checksum(adapter, staterr, skb); 1025 1026 e1000_rx_hash(netdev, rx_desc->wb.lower.hi_dword.rss, skb); 1027 1028 e1000_receive_skb(adapter, netdev, skb, staterr, 1029 rx_desc->wb.upper.vlan); 1030 1031next_desc: 1032 rx_desc->wb.upper.status_error &= cpu_to_le32(~0xFF); 1033 1034 /* return some buffers to hardware, one at a time is too slow */ 1035 if (cleaned_count >= E1000_RX_BUFFER_WRITE) { 1036 adapter->alloc_rx_buf(rx_ring, cleaned_count, 1037 GFP_ATOMIC); 1038 cleaned_count = 0; 1039 } 1040 1041 /* use prefetched values */ 1042 rx_desc = next_rxd; 1043 buffer_info = next_buffer; 1044 1045 staterr = le32_to_cpu(rx_desc->wb.upper.status_error); 1046 } 1047 rx_ring->next_to_clean = i; 1048 1049 cleaned_count = e1000_desc_unused(rx_ring); 1050 if (cleaned_count) 1051 adapter->alloc_rx_buf(rx_ring, cleaned_count, GFP_ATOMIC); 1052 1053 adapter->total_rx_bytes += total_rx_bytes; 1054 adapter->total_rx_packets += total_rx_packets; 1055 return cleaned; 1056} 1057 1058static void e1000_put_txbuf(struct e1000_ring *tx_ring, 1059 struct e1000_buffer *buffer_info, 1060 bool drop) 1061{ 1062 struct e1000_adapter *adapter = tx_ring->adapter; 1063 1064 if (buffer_info->dma) { 1065 if (buffer_info->mapped_as_page) 1066 dma_unmap_page(&adapter->pdev->dev, buffer_info->dma, 1067 buffer_info->length, DMA_TO_DEVICE); 1068 else 1069 dma_unmap_single(&adapter->pdev->dev, buffer_info->dma, 1070 buffer_info->length, DMA_TO_DEVICE); 1071 buffer_info->dma = 0; 1072 } 1073 if (buffer_info->skb) { 1074 if (drop) 1075 dev_kfree_skb_any(buffer_info->skb); 1076 else 1077 dev_consume_skb_any(buffer_info->skb); 1078 buffer_info->skb = NULL; 1079 } 1080 buffer_info->time_stamp = 0; 1081} 1082 1083static void e1000_print_hw_hang(struct work_struct *work) 1084{ 1085 struct e1000_adapter *adapter = container_of(work, 1086 struct e1000_adapter, 1087 print_hang_task); 1088 struct net_device *netdev = adapter->netdev; 1089 struct e1000_ring *tx_ring = adapter->tx_ring; 1090 unsigned int i = tx_ring->next_to_clean; 1091 unsigned int eop = tx_ring->buffer_info[i].next_to_watch; 1092 struct e1000_tx_desc *eop_desc = E1000_TX_DESC(*tx_ring, eop); 1093 struct e1000_hw *hw = &adapter->hw; 1094 u16 phy_status, phy_1000t_status, phy_ext_status; 1095 u16 pci_status; 1096 1097 if (test_bit(__E1000_DOWN, &adapter->state)) 1098 return; 1099 1100 if (!adapter->tx_hang_recheck && (adapter->flags2 & FLAG2_DMA_BURST)) { 1101 /* May be block on write-back, flush and detect again 1102 * flush pending descriptor writebacks to memory 1103 */ 1104 ew32(TIDV, adapter->tx_int_delay | E1000_TIDV_FPD); 1105 /* execute the writes immediately */ 1106 e1e_flush(); 1107 /* Due to rare timing issues, write to TIDV again to ensure 1108 * the write is successful 1109 */ 1110 ew32(TIDV, adapter->tx_int_delay | E1000_TIDV_FPD); 1111 /* execute the writes immediately */ 1112 e1e_flush(); 1113 adapter->tx_hang_recheck = true; 1114 return; 1115 } 1116 adapter->tx_hang_recheck = false; 1117 1118 if (er32(TDH(0)) == er32(TDT(0))) { 1119 e_dbg("false hang detected, ignoring\n"); 1120 return; 1121 } 1122 1123 /* Real hang detected */ 1124 netif_stop_queue(netdev); 1125 1126 e1e_rphy(hw, MII_BMSR, &phy_status); 1127 e1e_rphy(hw, MII_STAT1000, &phy_1000t_status); 1128 e1e_rphy(hw, MII_ESTATUS, &phy_ext_status); 1129 1130 pci_read_config_word(adapter->pdev, PCI_STATUS, &pci_status); 1131 1132 /* detected Hardware unit hang */ 1133 e_err("Detected Hardware Unit Hang:\n" 1134 " TDH <%x>\n" 1135 " TDT <%x>\n" 1136 " next_to_use <%x>\n" 1137 " next_to_clean <%x>\n" 1138 "buffer_info[next_to_clean]:\n" 1139 " time_stamp <%lx>\n" 1140 " next_to_watch <%x>\n" 1141 " jiffies <%lx>\n" 1142 " next_to_watch.status <%x>\n" 1143 "MAC Status <%x>\n" 1144 "PHY Status <%x>\n" 1145 "PHY 1000BASE-T Status <%x>\n" 1146 "PHY Extended Status <%x>\n" 1147 "PCI Status <%x>\n", 1148 readl(tx_ring->head), readl(tx_ring->tail), tx_ring->next_to_use, 1149 tx_ring->next_to_clean, tx_ring->buffer_info[eop].time_stamp, 1150 eop, jiffies, eop_desc->upper.fields.status, er32(STATUS), 1151 phy_status, phy_1000t_status, phy_ext_status, pci_status); 1152 1153 e1000e_dump(adapter); 1154 1155 /* Suggest workaround for known h/w issue */ 1156 if ((hw->mac.type == e1000_pchlan) && (er32(CTRL) & E1000_CTRL_TFCE)) 1157 e_err("Try turning off Tx pause (flow control) via ethtool\n"); 1158} 1159 1160/** 1161 * e1000e_tx_hwtstamp_work - check for Tx time stamp 1162 * @work: pointer to work struct 1163 * 1164 * This work function polls the TSYNCTXCTL valid bit to determine when a 1165 * timestamp has been taken for the current stored skb. The timestamp must 1166 * be for this skb because only one such packet is allowed in the queue. 1167 */ 1168static void e1000e_tx_hwtstamp_work(struct work_struct *work) 1169{ 1170 struct e1000_adapter *adapter = container_of(work, struct e1000_adapter, 1171 tx_hwtstamp_work); 1172 struct e1000_hw *hw = &adapter->hw; 1173 1174 if (er32(TSYNCTXCTL) & E1000_TSYNCTXCTL_VALID) { 1175 struct sk_buff *skb = adapter->tx_hwtstamp_skb; 1176 struct skb_shared_hwtstamps shhwtstamps; 1177 u64 txstmp; 1178 1179 txstmp = er32(TXSTMPL); 1180 txstmp |= (u64)er32(TXSTMPH) << 32; 1181 1182 e1000e_systim_to_hwtstamp(adapter, &shhwtstamps, txstmp); 1183 1184 /* Clear the global tx_hwtstamp_skb pointer and force writes 1185 * prior to notifying the stack of a Tx timestamp. 1186 */ 1187 adapter->tx_hwtstamp_skb = NULL; 1188 wmb(); /* force write prior to skb_tstamp_tx */ 1189 1190 skb_tstamp_tx(skb, &shhwtstamps); 1191 dev_consume_skb_any(skb); 1192 } else if (time_after(jiffies, adapter->tx_hwtstamp_start 1193 + adapter->tx_timeout_factor * HZ)) { 1194 dev_kfree_skb_any(adapter->tx_hwtstamp_skb); 1195 adapter->tx_hwtstamp_skb = NULL; 1196 adapter->tx_hwtstamp_timeouts++; 1197 e_warn("clearing Tx timestamp hang\n"); 1198 } else { 1199 /* reschedule to check later */ 1200 schedule_work(&adapter->tx_hwtstamp_work); 1201 } 1202} 1203 1204/** 1205 * e1000_clean_tx_irq - Reclaim resources after transmit completes 1206 * @tx_ring: Tx descriptor ring 1207 * 1208 * the return value indicates whether actual cleaning was done, there 1209 * is no guarantee that everything was cleaned 1210 **/ 1211static bool e1000_clean_tx_irq(struct e1000_ring *tx_ring) 1212{ 1213 struct e1000_adapter *adapter = tx_ring->adapter; 1214 struct net_device *netdev = adapter->netdev; 1215 struct e1000_hw *hw = &adapter->hw; 1216 struct e1000_tx_desc *tx_desc, *eop_desc; 1217 struct e1000_buffer *buffer_info; 1218 unsigned int i, eop; 1219 unsigned int count = 0; 1220 unsigned int total_tx_bytes = 0, total_tx_packets = 0; 1221 unsigned int bytes_compl = 0, pkts_compl = 0; 1222 1223 i = tx_ring->next_to_clean; 1224 eop = tx_ring->buffer_info[i].next_to_watch; 1225 eop_desc = E1000_TX_DESC(*tx_ring, eop); 1226 1227 while ((eop_desc->upper.data & cpu_to_le32(E1000_TXD_STAT_DD)) && 1228 (count < tx_ring->count)) { 1229 bool cleaned = false; 1230 1231 dma_rmb(); /* read buffer_info after eop_desc */ 1232 for (; !cleaned; count++) { 1233 tx_desc = E1000_TX_DESC(*tx_ring, i); 1234 buffer_info = &tx_ring->buffer_info[i]; 1235 cleaned = (i == eop); 1236 1237 if (cleaned) { 1238 total_tx_packets += buffer_info->segs; 1239 total_tx_bytes += buffer_info->bytecount; 1240 if (buffer_info->skb) { 1241 bytes_compl += buffer_info->skb->len; 1242 pkts_compl++; 1243 } 1244 } 1245 1246 e1000_put_txbuf(tx_ring, buffer_info, false); 1247 tx_desc->upper.data = 0; 1248 1249 i++; 1250 if (i == tx_ring->count) 1251 i = 0; 1252 } 1253 1254 if (i == tx_ring->next_to_use) 1255 break; 1256 eop = tx_ring->buffer_info[i].next_to_watch; 1257 eop_desc = E1000_TX_DESC(*tx_ring, eop); 1258 } 1259 1260 tx_ring->next_to_clean = i; 1261 1262 netdev_completed_queue(netdev, pkts_compl, bytes_compl); 1263 1264#define TX_WAKE_THRESHOLD 32 1265 if (count && netif_carrier_ok(netdev) && 1266 e1000_desc_unused(tx_ring) >= TX_WAKE_THRESHOLD) { 1267 /* Make sure that anybody stopping the queue after this 1268 * sees the new next_to_clean. 1269 */ 1270 smp_mb(); 1271 1272 if (netif_queue_stopped(netdev) && 1273 !(test_bit(__E1000_DOWN, &adapter->state))) { 1274 netif_wake_queue(netdev); 1275 ++adapter->restart_queue; 1276 } 1277 } 1278 1279 if (adapter->detect_tx_hung) { 1280 /* Detect a transmit hang in hardware, this serializes the 1281 * check with the clearing of time_stamp and movement of i 1282 */ 1283 adapter->detect_tx_hung = false; 1284 if (tx_ring->buffer_info[i].time_stamp && 1285 time_after(jiffies, tx_ring->buffer_info[i].time_stamp 1286 + (adapter->tx_timeout_factor * HZ)) && 1287 !(er32(STATUS) & E1000_STATUS_TXOFF)) 1288 schedule_work(&adapter->print_hang_task); 1289 else 1290 adapter->tx_hang_recheck = false; 1291 } 1292 adapter->total_tx_bytes += total_tx_bytes; 1293 adapter->total_tx_packets += total_tx_packets; 1294 return count < tx_ring->count; 1295} 1296 1297/** 1298 * e1000_clean_rx_irq_ps - Send received data up the network stack; packet split 1299 * @rx_ring: Rx descriptor ring 1300 * @work_done: output parameter for indicating completed work 1301 * @work_to_do: how many packets we can clean 1302 * 1303 * the return value indicates whether actual cleaning was done, there 1304 * is no guarantee that everything was cleaned 1305 **/ 1306static bool e1000_clean_rx_irq_ps(struct e1000_ring *rx_ring, int *work_done, 1307 int work_to_do) 1308{ 1309 struct e1000_adapter *adapter = rx_ring->adapter; 1310 struct e1000_hw *hw = &adapter->hw; 1311 union e1000_rx_desc_packet_split *rx_desc, *next_rxd; 1312 struct net_device *netdev = adapter->netdev; 1313 struct pci_dev *pdev = adapter->pdev; 1314 struct e1000_buffer *buffer_info, *next_buffer; 1315 struct e1000_ps_page *ps_page; 1316 struct sk_buff *skb; 1317 unsigned int i, j; 1318 u32 length, staterr; 1319 int cleaned_count = 0; 1320 bool cleaned = false; 1321 unsigned int total_rx_bytes = 0, total_rx_packets = 0; 1322 1323 i = rx_ring->next_to_clean; 1324 rx_desc = E1000_RX_DESC_PS(*rx_ring, i); 1325 staterr = le32_to_cpu(rx_desc->wb.middle.status_error); 1326 buffer_info = &rx_ring->buffer_info[i]; 1327 1328 while (staterr & E1000_RXD_STAT_DD) { 1329 if (*work_done >= work_to_do) 1330 break; 1331 (*work_done)++; 1332 skb = buffer_info->skb; 1333 dma_rmb(); /* read descriptor and rx_buffer_info after status DD */ 1334 1335 /* in the packet split case this is header only */ 1336 prefetch(skb->data - NET_IP_ALIGN); 1337 1338 i++; 1339 if (i == rx_ring->count) 1340 i = 0; 1341 next_rxd = E1000_RX_DESC_PS(*rx_ring, i); 1342 prefetch(next_rxd); 1343 1344 next_buffer = &rx_ring->buffer_info[i]; 1345 1346 cleaned = true; 1347 cleaned_count++; 1348 dma_unmap_single(&pdev->dev, buffer_info->dma, 1349 adapter->rx_ps_bsize0, DMA_FROM_DEVICE); 1350 buffer_info->dma = 0; 1351 1352 /* see !EOP comment in other Rx routine */ 1353 if (!(staterr & E1000_RXD_STAT_EOP)) 1354 adapter->flags2 |= FLAG2_IS_DISCARDING; 1355 1356 if (adapter->flags2 & FLAG2_IS_DISCARDING) { 1357 e_dbg("Packet Split buffers didn't pick up the full packet\n"); 1358 dev_kfree_skb_irq(skb); 1359 if (staterr & E1000_RXD_STAT_EOP) 1360 adapter->flags2 &= ~FLAG2_IS_DISCARDING; 1361 goto next_desc; 1362 } 1363 1364 if (unlikely((staterr & E1000_RXDEXT_ERR_FRAME_ERR_MASK) && 1365 !(netdev->features & NETIF_F_RXALL))) { 1366 dev_kfree_skb_irq(skb); 1367 goto next_desc; 1368 } 1369 1370 length = le16_to_cpu(rx_desc->wb.middle.length0); 1371 1372 if (!length) { 1373 e_dbg("Last part of the packet spanning multiple descriptors\n"); 1374 dev_kfree_skb_irq(skb); 1375 goto next_desc; 1376 } 1377 1378 /* Good Receive */ 1379 skb_put(skb, length); 1380 1381 { 1382 /* this looks ugly, but it seems compiler issues make 1383 * it more efficient than reusing j 1384 */ 1385 int l1 = le16_to_cpu(rx_desc->wb.upper.length[0]); 1386 1387 /* page alloc/put takes too long and effects small 1388 * packet throughput, so unsplit small packets and 1389 * save the alloc/put only valid in softirq (napi) 1390 * context to call kmap_* 1391 */ 1392 if (l1 && (l1 <= copybreak) && 1393 ((length + l1) <= adapter->rx_ps_bsize0)) { 1394 u8 *vaddr; 1395 1396 ps_page = &buffer_info->ps_pages[0]; 1397 1398 /* there is no documentation about how to call 1399 * kmap_atomic, so we can't hold the mapping 1400 * very long 1401 */ 1402 dma_sync_single_for_cpu(&pdev->dev, 1403 ps_page->dma, 1404 PAGE_SIZE, 1405 DMA_FROM_DEVICE); 1406 vaddr = kmap_atomic(ps_page->page); 1407 memcpy(skb_tail_pointer(skb), vaddr, l1); 1408 kunmap_atomic(vaddr); 1409 dma_sync_single_for_device(&pdev->dev, 1410 ps_page->dma, 1411 PAGE_SIZE, 1412 DMA_FROM_DEVICE); 1413 1414 /* remove the CRC */ 1415 if (!(adapter->flags2 & FLAG2_CRC_STRIPPING)) { 1416 if (!(netdev->features & NETIF_F_RXFCS)) 1417 l1 -= 4; 1418 } 1419 1420 skb_put(skb, l1); 1421 goto copydone; 1422 } /* if */ 1423 } 1424 1425 for (j = 0; j < PS_PAGE_BUFFERS; j++) { 1426 length = le16_to_cpu(rx_desc->wb.upper.length[j]); 1427 if (!length) 1428 break; 1429 1430 ps_page = &buffer_info->ps_pages[j]; 1431 dma_unmap_page(&pdev->dev, ps_page->dma, PAGE_SIZE, 1432 DMA_FROM_DEVICE); 1433 ps_page->dma = 0; 1434 skb_fill_page_desc(skb, j, ps_page->page, 0, length); 1435 ps_page->page = NULL; 1436 skb->len += length; 1437 skb->data_len += length; 1438 skb->truesize += PAGE_SIZE; 1439 } 1440 1441 /* strip the ethernet crc, problem is we're using pages now so 1442 * this whole operation can get a little cpu intensive 1443 */ 1444 if (!(adapter->flags2 & FLAG2_CRC_STRIPPING)) { 1445 if (!(netdev->features & NETIF_F_RXFCS)) 1446 pskb_trim(skb, skb->len - 4); 1447 } 1448 1449copydone: 1450 total_rx_bytes += skb->len; 1451 total_rx_packets++; 1452 1453 e1000_rx_checksum(adapter, staterr, skb); 1454 1455 e1000_rx_hash(netdev, rx_desc->wb.lower.hi_dword.rss, skb); 1456 1457 if (rx_desc->wb.upper.header_status & 1458 cpu_to_le16(E1000_RXDPS_HDRSTAT_HDRSP)) 1459 adapter->rx_hdr_split++; 1460 1461 e1000_receive_skb(adapter, netdev, skb, staterr, 1462 rx_desc->wb.middle.vlan); 1463 1464next_desc: 1465 rx_desc->wb.middle.status_error &= cpu_to_le32(~0xFF); 1466 buffer_info->skb = NULL; 1467 1468 /* return some buffers to hardware, one at a time is too slow */ 1469 if (cleaned_count >= E1000_RX_BUFFER_WRITE) { 1470 adapter->alloc_rx_buf(rx_ring, cleaned_count, 1471 GFP_ATOMIC); 1472 cleaned_count = 0; 1473 } 1474 1475 /* use prefetched values */ 1476 rx_desc = next_rxd; 1477 buffer_info = next_buffer; 1478 1479 staterr = le32_to_cpu(rx_desc->wb.middle.status_error); 1480 } 1481 rx_ring->next_to_clean = i; 1482 1483 cleaned_count = e1000_desc_unused(rx_ring); 1484 if (cleaned_count) 1485 adapter->alloc_rx_buf(rx_ring, cleaned_count, GFP_ATOMIC); 1486 1487 adapter->total_rx_bytes += total_rx_bytes; 1488 adapter->total_rx_packets += total_rx_packets; 1489 return cleaned; 1490} 1491 1492static void e1000_consume_page(struct e1000_buffer *bi, struct sk_buff *skb, 1493 u16 length) 1494{ 1495 bi->page = NULL; 1496 skb->len += length; 1497 skb->data_len += length; 1498 skb->truesize += PAGE_SIZE; 1499} 1500 1501/** 1502 * e1000_clean_jumbo_rx_irq - Send received data up the network stack; legacy 1503 * @rx_ring: Rx descriptor ring 1504 * @work_done: output parameter for indicating completed work 1505 * @work_to_do: how many packets we can clean 1506 * 1507 * the return value indicates whether actual cleaning was done, there 1508 * is no guarantee that everything was cleaned 1509 **/ 1510static bool e1000_clean_jumbo_rx_irq(struct e1000_ring *rx_ring, int *work_done, 1511 int work_to_do) 1512{ 1513 struct e1000_adapter *adapter = rx_ring->adapter; 1514 struct net_device *netdev = adapter->netdev; 1515 struct pci_dev *pdev = adapter->pdev; 1516 union e1000_rx_desc_extended *rx_desc, *next_rxd; 1517 struct e1000_buffer *buffer_info, *next_buffer; 1518 u32 length, staterr; 1519 unsigned int i; 1520 int cleaned_count = 0; 1521 bool cleaned = false; 1522 unsigned int total_rx_bytes = 0, total_rx_packets = 0; 1523 struct skb_shared_info *shinfo; 1524 1525 i = rx_ring->next_to_clean; 1526 rx_desc = E1000_RX_DESC_EXT(*rx_ring, i); 1527 staterr = le32_to_cpu(rx_desc->wb.upper.status_error); 1528 buffer_info = &rx_ring->buffer_info[i]; 1529 1530 while (staterr & E1000_RXD_STAT_DD) { 1531 struct sk_buff *skb; 1532 1533 if (*work_done >= work_to_do) 1534 break; 1535 (*work_done)++; 1536 dma_rmb(); /* read descriptor and rx_buffer_info after status DD */ 1537 1538 skb = buffer_info->skb; 1539 buffer_info->skb = NULL; 1540 1541 ++i; 1542 if (i == rx_ring->count) 1543 i = 0; 1544 next_rxd = E1000_RX_DESC_EXT(*rx_ring, i); 1545 prefetch(next_rxd); 1546 1547 next_buffer = &rx_ring->buffer_info[i]; 1548 1549 cleaned = true; 1550 cleaned_count++; 1551 dma_unmap_page(&pdev->dev, buffer_info->dma, PAGE_SIZE, 1552 DMA_FROM_DEVICE); 1553 buffer_info->dma = 0; 1554 1555 length = le16_to_cpu(rx_desc->wb.upper.length); 1556 1557 /* errors is only valid for DD + EOP descriptors */ 1558 if (unlikely((staterr & E1000_RXD_STAT_EOP) && 1559 ((staterr & E1000_RXDEXT_ERR_FRAME_ERR_MASK) && 1560 !(netdev->features & NETIF_F_RXALL)))) { 1561 /* recycle both page and skb */ 1562 buffer_info->skb = skb; 1563 /* an error means any chain goes out the window too */ 1564 if (rx_ring->rx_skb_top) 1565 dev_kfree_skb_irq(rx_ring->rx_skb_top); 1566 rx_ring->rx_skb_top = NULL; 1567 goto next_desc; 1568 } 1569#define rxtop (rx_ring->rx_skb_top) 1570 if (!(staterr & E1000_RXD_STAT_EOP)) { 1571 /* this descriptor is only the beginning (or middle) */ 1572 if (!rxtop) { 1573 /* this is the beginning of a chain */ 1574 rxtop = skb; 1575 skb_fill_page_desc(rxtop, 0, buffer_info->page, 1576 0, length); 1577 } else { 1578 /* this is the middle of a chain */ 1579 shinfo = skb_shinfo(rxtop); 1580 skb_fill_page_desc(rxtop, shinfo->nr_frags, 1581 buffer_info->page, 0, 1582 length); 1583 /* re-use the skb, only consumed the page */ 1584 buffer_info->skb = skb; 1585 } 1586 e1000_consume_page(buffer_info, rxtop, length); 1587 goto next_desc; 1588 } else { 1589 if (rxtop) { 1590 /* end of the chain */ 1591 shinfo = skb_shinfo(rxtop); 1592 skb_fill_page_desc(rxtop, shinfo->nr_frags, 1593 buffer_info->page, 0, 1594 length); 1595 /* re-use the current skb, we only consumed the 1596 * page 1597 */ 1598 buffer_info->skb = skb; 1599 skb = rxtop; 1600 rxtop = NULL; 1601 e1000_consume_page(buffer_info, skb, length); 1602 } else { 1603 /* no chain, got EOP, this buf is the packet 1604 * copybreak to save the put_page/alloc_page 1605 */ 1606 if (length <= copybreak && 1607 skb_tailroom(skb) >= length) { 1608 u8 *vaddr; 1609 vaddr = kmap_atomic(buffer_info->page); 1610 memcpy(skb_tail_pointer(skb), vaddr, 1611 length); 1612 kunmap_atomic(vaddr); 1613 /* re-use the page, so don't erase 1614 * buffer_info->page 1615 */ 1616 skb_put(skb, length); 1617 } else { 1618 skb_fill_page_desc(skb, 0, 1619 buffer_info->page, 0, 1620 length); 1621 e1000_consume_page(buffer_info, skb, 1622 length); 1623 } 1624 } 1625 } 1626 1627 /* Receive Checksum Offload */ 1628 e1000_rx_checksum(adapter, staterr, skb); 1629 1630 e1000_rx_hash(netdev, rx_desc->wb.lower.hi_dword.rss, skb); 1631 1632 /* probably a little skewed due to removing CRC */ 1633 total_rx_bytes += skb->len; 1634 total_rx_packets++; 1635 1636 /* eth type trans needs skb->data to point to something */ 1637 if (!pskb_may_pull(skb, ETH_HLEN)) { 1638 e_err("pskb_may_pull failed.\n"); 1639 dev_kfree_skb_irq(skb); 1640 goto next_desc; 1641 } 1642 1643 e1000_receive_skb(adapter, netdev, skb, staterr, 1644 rx_desc->wb.upper.vlan); 1645 1646next_desc: 1647 rx_desc->wb.upper.status_error &= cpu_to_le32(~0xFF); 1648 1649 /* return some buffers to hardware, one at a time is too slow */ 1650 if (unlikely(cleaned_count >= E1000_RX_BUFFER_WRITE)) { 1651 adapter->alloc_rx_buf(rx_ring, cleaned_count, 1652 GFP_ATOMIC); 1653 cleaned_count = 0; 1654 } 1655 1656 /* use prefetched values */ 1657 rx_desc = next_rxd; 1658 buffer_info = next_buffer; 1659 1660 staterr = le32_to_cpu(rx_desc->wb.upper.status_error); 1661 } 1662 rx_ring->next_to_clean = i; 1663 1664 cleaned_count = e1000_desc_unused(rx_ring); 1665 if (cleaned_count) 1666 adapter->alloc_rx_buf(rx_ring, cleaned_count, GFP_ATOMIC); 1667 1668 adapter->total_rx_bytes += total_rx_bytes; 1669 adapter->total_rx_packets += total_rx_packets; 1670 return cleaned; 1671} 1672 1673/** 1674 * e1000_clean_rx_ring - Free Rx Buffers per Queue 1675 * @rx_ring: Rx descriptor ring 1676 **/ 1677static void e1000_clean_rx_ring(struct e1000_ring *rx_ring) 1678{ 1679 struct e1000_adapter *adapter = rx_ring->adapter; 1680 struct e1000_buffer *buffer_info; 1681 struct e1000_ps_page *ps_page; 1682 struct pci_dev *pdev = adapter->pdev; 1683 unsigned int i, j; 1684 1685 /* Free all the Rx ring sk_buffs */ 1686 for (i = 0; i < rx_ring->count; i++) { 1687 buffer_info = &rx_ring->buffer_info[i]; 1688 if (buffer_info->dma) { 1689 if (adapter->clean_rx == e1000_clean_rx_irq) 1690 dma_unmap_single(&pdev->dev, buffer_info->dma, 1691 adapter->rx_buffer_len, 1692 DMA_FROM_DEVICE); 1693 else if (adapter->clean_rx == e1000_clean_jumbo_rx_irq) 1694 dma_unmap_page(&pdev->dev, buffer_info->dma, 1695 PAGE_SIZE, DMA_FROM_DEVICE); 1696 else if (adapter->clean_rx == e1000_clean_rx_irq_ps) 1697 dma_unmap_single(&pdev->dev, buffer_info->dma, 1698 adapter->rx_ps_bsize0, 1699 DMA_FROM_DEVICE); 1700 buffer_info->dma = 0; 1701 } 1702 1703 if (buffer_info->page) { 1704 put_page(buffer_info->page); 1705 buffer_info->page = NULL; 1706 } 1707 1708 if (buffer_info->skb) { 1709 dev_kfree_skb(buffer_info->skb); 1710 buffer_info->skb = NULL; 1711 } 1712 1713 for (j = 0; j < PS_PAGE_BUFFERS; j++) { 1714 ps_page = &buffer_info->ps_pages[j]; 1715 if (!ps_page->page) 1716 break; 1717 dma_unmap_page(&pdev->dev, ps_page->dma, PAGE_SIZE, 1718 DMA_FROM_DEVICE); 1719 ps_page->dma = 0; 1720 put_page(ps_page->page); 1721 ps_page->page = NULL; 1722 } 1723 } 1724 1725 /* there also may be some cached data from a chained receive */ 1726 if (rx_ring->rx_skb_top) { 1727 dev_kfree_skb(rx_ring->rx_skb_top); 1728 rx_ring->rx_skb_top = NULL; 1729 } 1730 1731 /* Zero out the descriptor ring */ 1732 memset(rx_ring->desc, 0, rx_ring->size); 1733 1734 rx_ring->next_to_clean = 0; 1735 rx_ring->next_to_use = 0; 1736 adapter->flags2 &= ~FLAG2_IS_DISCARDING; 1737} 1738 1739static void e1000e_downshift_workaround(struct work_struct *work) 1740{ 1741 struct e1000_adapter *adapter = container_of(work, 1742 struct e1000_adapter, 1743 downshift_task); 1744 1745 if (test_bit(__E1000_DOWN, &adapter->state)) 1746 return; 1747 1748 e1000e_gig_downshift_workaround_ich8lan(&adapter->hw); 1749} 1750 1751/** 1752 * e1000_intr_msi - Interrupt Handler 1753 * @irq: interrupt number 1754 * @data: pointer to a network interface device structure 1755 **/ 1756static irqreturn_t e1000_intr_msi(int __always_unused irq, void *data) 1757{ 1758 struct net_device *netdev = data; 1759 struct e1000_adapter *adapter = netdev_priv(netdev); 1760 struct e1000_hw *hw = &adapter->hw; 1761 u32 icr = er32(ICR); 1762 1763 /* read ICR disables interrupts using IAM */ 1764 if (icr & E1000_ICR_LSC) { 1765 hw->mac.get_link_status = true; 1766 /* ICH8 workaround-- Call gig speed drop workaround on cable 1767 * disconnect (LSC) before accessing any PHY registers 1768 */ 1769 if ((adapter->flags & FLAG_LSC_GIG_SPEED_DROP) && 1770 (!(er32(STATUS) & E1000_STATUS_LU))) 1771 schedule_work(&adapter->downshift_task); 1772 1773 /* 80003ES2LAN workaround-- For packet buffer work-around on 1774 * link down event; disable receives here in the ISR and reset 1775 * adapter in watchdog 1776 */ 1777 if (netif_carrier_ok(netdev) && 1778 adapter->flags & FLAG_RX_NEEDS_RESTART) { 1779 /* disable receives */ 1780 u32 rctl = er32(RCTL); 1781 1782 ew32(RCTL, rctl & ~E1000_RCTL_EN); 1783 adapter->flags |= FLAG_RESTART_NOW; 1784 } 1785 /* guard against interrupt when we're going down */ 1786 if (!test_bit(__E1000_DOWN, &adapter->state)) 1787 mod_timer(&adapter->watchdog_timer, jiffies + 1); 1788 } 1789 1790 /* Reset on uncorrectable ECC error */ 1791 if ((icr & E1000_ICR_ECCER) && (hw->mac.type >= e1000_pch_lpt)) { 1792 u32 pbeccsts = er32(PBECCSTS); 1793 1794 adapter->corr_errors += 1795 pbeccsts & E1000_PBECCSTS_CORR_ERR_CNT_MASK; 1796 adapter->uncorr_errors += 1797 (pbeccsts & E1000_PBECCSTS_UNCORR_ERR_CNT_MASK) >> 1798 E1000_PBECCSTS_UNCORR_ERR_CNT_SHIFT; 1799 1800 /* Do the reset outside of interrupt context */ 1801 schedule_work(&adapter->reset_task); 1802 1803 /* return immediately since reset is imminent */ 1804 return IRQ_HANDLED; 1805 } 1806 1807 if (napi_schedule_prep(&adapter->napi)) { 1808 adapter->total_tx_bytes = 0; 1809 adapter->total_tx_packets = 0; 1810 adapter->total_rx_bytes = 0; 1811 adapter->total_rx_packets = 0; 1812 __napi_schedule(&adapter->napi); 1813 } 1814 1815 return IRQ_HANDLED; 1816} 1817 1818/** 1819 * e1000_intr - Interrupt Handler 1820 * @irq: interrupt number 1821 * @data: pointer to a network interface device structure 1822 **/ 1823static irqreturn_t e1000_intr(int __always_unused irq, void *data) 1824{ 1825 struct net_device *netdev = data; 1826 struct e1000_adapter *adapter = netdev_priv(netdev); 1827 struct e1000_hw *hw = &adapter->hw; 1828 u32 rctl, icr = er32(ICR); 1829 1830 if (!icr || test_bit(__E1000_DOWN, &adapter->state)) 1831 return IRQ_NONE; /* Not our interrupt */ 1832 1833 /* IMS will not auto-mask if INT_ASSERTED is not set, and if it is 1834 * not set, then the adapter didn't send an interrupt 1835 */ 1836 if (!(icr & E1000_ICR_INT_ASSERTED)) 1837 return IRQ_NONE; 1838 1839 /* Interrupt Auto-Mask...upon reading ICR, 1840 * interrupts are masked. No need for the 1841 * IMC write 1842 */ 1843 1844 if (icr & E1000_ICR_LSC) { 1845 hw->mac.get_link_status = true; 1846 /* ICH8 workaround-- Call gig speed drop workaround on cable 1847 * disconnect (LSC) before accessing any PHY registers 1848 */ 1849 if ((adapter->flags & FLAG_LSC_GIG_SPEED_DROP) && 1850 (!(er32(STATUS) & E1000_STATUS_LU))) 1851 schedule_work(&adapter->downshift_task); 1852 1853 /* 80003ES2LAN workaround-- 1854 * For packet buffer work-around on link down event; 1855 * disable receives here in the ISR and 1856 * reset adapter in watchdog 1857 */ 1858 if (netif_carrier_ok(netdev) && 1859 (adapter->flags & FLAG_RX_NEEDS_RESTART)) { 1860 /* disable receives */ 1861 rctl = er32(RCTL); 1862 ew32(RCTL, rctl & ~E1000_RCTL_EN); 1863 adapter->flags |= FLAG_RESTART_NOW; 1864 } 1865 /* guard against interrupt when we're going down */ 1866 if (!test_bit(__E1000_DOWN, &adapter->state)) 1867 mod_timer(&adapter->watchdog_timer, jiffies + 1); 1868 } 1869 1870 /* Reset on uncorrectable ECC error */ 1871 if ((icr & E1000_ICR_ECCER) && (hw->mac.type >= e1000_pch_lpt)) { 1872 u32 pbeccsts = er32(PBECCSTS); 1873 1874 adapter->corr_errors += 1875 pbeccsts & E1000_PBECCSTS_CORR_ERR_CNT_MASK; 1876 adapter->uncorr_errors += 1877 (pbeccsts & E1000_PBECCSTS_UNCORR_ERR_CNT_MASK) >> 1878 E1000_PBECCSTS_UNCORR_ERR_CNT_SHIFT; 1879 1880 /* Do the reset outside of interrupt context */ 1881 schedule_work(&adapter->reset_task); 1882 1883 /* return immediately since reset is imminent */ 1884 return IRQ_HANDLED; 1885 } 1886 1887 if (napi_schedule_prep(&adapter->napi)) { 1888 adapter->total_tx_bytes = 0; 1889 adapter->total_tx_packets = 0; 1890 adapter->total_rx_bytes = 0; 1891 adapter->total_rx_packets = 0; 1892 __napi_schedule(&adapter->napi); 1893 } 1894 1895 return IRQ_HANDLED; 1896} 1897 1898static irqreturn_t e1000_msix_other(int __always_unused irq, void *data) 1899{ 1900 struct net_device *netdev = data; 1901 struct e1000_adapter *adapter = netdev_priv(netdev); 1902 struct e1000_hw *hw = &adapter->hw; 1903 u32 icr = er32(ICR); 1904 1905 if (icr & adapter->eiac_mask) 1906 ew32(ICS, (icr & adapter->eiac_mask)); 1907 1908 if (icr & E1000_ICR_LSC) { 1909 hw->mac.get_link_status = true; 1910 /* guard against interrupt when we're going down */ 1911 if (!test_bit(__E1000_DOWN, &adapter->state)) 1912 mod_timer(&adapter->watchdog_timer, jiffies + 1); 1913 } 1914 1915 if (!test_bit(__E1000_DOWN, &adapter->state)) 1916 ew32(IMS, E1000_IMS_OTHER | IMS_OTHER_MASK); 1917 1918 return IRQ_HANDLED; 1919} 1920 1921static irqreturn_t e1000_intr_msix_tx(int __always_unused irq, void *data) 1922{ 1923 struct net_device *netdev = data; 1924 struct e1000_adapter *adapter = netdev_priv(netdev); 1925 struct e1000_hw *hw = &adapter->hw; 1926 struct e1000_ring *tx_ring = adapter->tx_ring; 1927 1928 adapter->total_tx_bytes = 0; 1929 adapter->total_tx_packets = 0; 1930 1931 if (!e1000_clean_tx_irq(tx_ring)) 1932 /* Ring was not completely cleaned, so fire another interrupt */ 1933 ew32(ICS, tx_ring->ims_val); 1934 1935 if (!test_bit(__E1000_DOWN, &adapter->state)) 1936 ew32(IMS, adapter->tx_ring->ims_val); 1937 1938 return IRQ_HANDLED; 1939} 1940 1941static irqreturn_t e1000_intr_msix_rx(int __always_unused irq, void *data) 1942{ 1943 struct net_device *netdev = data; 1944 struct e1000_adapter *adapter = netdev_priv(netdev); 1945 struct e1000_ring *rx_ring = adapter->rx_ring; 1946 1947 /* Write the ITR value calculated at the end of the 1948 * previous interrupt. 1949 */ 1950 if (rx_ring->set_itr) { 1951 u32 itr = rx_ring->itr_val ? 1952 1000000000 / (rx_ring->itr_val * 256) : 0; 1953 1954 writel(itr, rx_ring->itr_register); 1955 rx_ring->set_itr = 0; 1956 } 1957 1958 if (napi_schedule_prep(&adapter->napi)) { 1959 adapter->total_rx_bytes = 0; 1960 adapter->total_rx_packets = 0; 1961 __napi_schedule(&adapter->napi); 1962 } 1963 return IRQ_HANDLED; 1964} 1965 1966/** 1967 * e1000_configure_msix - Configure MSI-X hardware 1968 * @adapter: board private structure 1969 * 1970 * e1000_configure_msix sets up the hardware to properly 1971 * generate MSI-X interrupts. 1972 **/ 1973static void e1000_configure_msix(struct e1000_adapter *adapter) 1974{ 1975 struct e1000_hw *hw = &adapter->hw; 1976 struct e1000_ring *rx_ring = adapter->rx_ring; 1977 struct e1000_ring *tx_ring = adapter->tx_ring; 1978 int vector = 0; 1979 u32 ctrl_ext, ivar = 0; 1980 1981 adapter->eiac_mask = 0; 1982 1983 /* Workaround issue with spurious interrupts on 82574 in MSI-X mode */ 1984 if (hw->mac.type == e1000_82574) { 1985 u32 rfctl = er32(RFCTL); 1986 1987 rfctl |= E1000_RFCTL_ACK_DIS; 1988 ew32(RFCTL, rfctl); 1989 } 1990 1991 /* Configure Rx vector */ 1992 rx_ring->ims_val = E1000_IMS_RXQ0; 1993 adapter->eiac_mask |= rx_ring->ims_val; 1994 if (rx_ring->itr_val) 1995 writel(1000000000 / (rx_ring->itr_val * 256), 1996 rx_ring->itr_register); 1997 else 1998 writel(1, rx_ring->itr_register); 1999 ivar = E1000_IVAR_INT_ALLOC_VALID | vector; 2000 2001 /* Configure Tx vector */ 2002 tx_ring->ims_val = E1000_IMS_TXQ0; 2003 vector++; 2004 if (tx_ring->itr_val) 2005 writel(1000000000 / (tx_ring->itr_val * 256), 2006 tx_ring->itr_register); 2007 else 2008 writel(1, tx_ring->itr_register); 2009 adapter->eiac_mask |= tx_ring->ims_val; 2010 ivar |= ((E1000_IVAR_INT_ALLOC_VALID | vector) << 8); 2011 2012 /* set vector for Other Causes, e.g. link changes */ 2013 vector++; 2014 ivar |= ((E1000_IVAR_INT_ALLOC_VALID | vector) << 16); 2015 if (rx_ring->itr_val) 2016 writel(1000000000 / (rx_ring->itr_val * 256), 2017 hw->hw_addr + E1000_EITR_82574(vector)); 2018 else 2019 writel(1, hw->hw_addr + E1000_EITR_82574(vector)); 2020 2021 /* Cause Tx interrupts on every write back */ 2022 ivar |= BIT(31); 2023 2024 ew32(IVAR, ivar); 2025 2026 /* enable MSI-X PBA support */ 2027 ctrl_ext = er32(CTRL_EXT) & ~E1000_CTRL_EXT_IAME; 2028 ctrl_ext |= E1000_CTRL_EXT_PBA_CLR | E1000_CTRL_EXT_EIAME; 2029 ew32(CTRL_EXT, ctrl_ext); 2030 e1e_flush(); 2031} 2032 2033void e1000e_reset_interrupt_capability(struct e1000_adapter *adapter) 2034{ 2035 if (adapter->msix_entries) { 2036 pci_disable_msix(adapter->pdev); 2037 kfree(adapter->msix_entries); 2038 adapter->msix_entries = NULL; 2039 } else if (adapter->flags & FLAG_MSI_ENABLED) { 2040 pci_disable_msi(adapter->pdev); 2041 adapter->flags &= ~FLAG_MSI_ENABLED; 2042 } 2043} 2044 2045/** 2046 * e1000e_set_interrupt_capability - set MSI or MSI-X if supported 2047 * @adapter: board private structure 2048 * 2049 * Attempt to configure interrupts using the best available 2050 * capabilities of the hardware and kernel. 2051 **/ 2052void e1000e_set_interrupt_capability(struct e1000_adapter *adapter) 2053{ 2054 int err; 2055 int i; 2056 2057 switch (adapter->int_mode) { 2058 case E1000E_INT_MODE_MSIX: 2059 if (adapter->flags & FLAG_HAS_MSIX) { 2060 adapter->num_vectors = 3; /* RxQ0, TxQ0 and other */ 2061 adapter->msix_entries = kcalloc(adapter->num_vectors, 2062 sizeof(struct 2063 msix_entry), 2064 GFP_KERNEL); 2065 if (adapter->msix_entries) { 2066 struct e1000_adapter *a = adapter; 2067 2068 for (i = 0; i < adapter->num_vectors; i++) 2069 adapter->msix_entries[i].entry = i; 2070 2071 err = pci_enable_msix_range(a->pdev, 2072 a->msix_entries, 2073 a->num_vectors, 2074 a->num_vectors); 2075 if (err > 0) 2076 return; 2077 } 2078 /* MSI-X failed, so fall through and try MSI */ 2079 e_err("Failed to initialize MSI-X interrupts. Falling back to MSI interrupts.\n"); 2080 e1000e_reset_interrupt_capability(adapter); 2081 } 2082 adapter->int_mode = E1000E_INT_MODE_MSI; 2083 fallthrough; 2084 case E1000E_INT_MODE_MSI: 2085 if (!pci_enable_msi(adapter->pdev)) { 2086 adapter->flags |= FLAG_MSI_ENABLED; 2087 } else { 2088 adapter->int_mode = E1000E_INT_MODE_LEGACY; 2089 e_err("Failed to initialize MSI interrupts. Falling back to legacy interrupts.\n"); 2090 } 2091 fallthrough; 2092 case E1000E_INT_MODE_LEGACY: 2093 /* Don't do anything; this is the system default */ 2094 break; 2095 } 2096 2097 /* store the number of vectors being used */ 2098 adapter->num_vectors = 1; 2099} 2100 2101/** 2102 * e1000_request_msix - Initialize MSI-X interrupts 2103 * @adapter: board private structure 2104 * 2105 * e1000_request_msix allocates MSI-X vectors and requests interrupts from the 2106 * kernel. 2107 **/ 2108static int e1000_request_msix(struct e1000_adapter *adapter) 2109{ 2110 struct net_device *netdev = adapter->netdev; 2111 int err = 0, vector = 0; 2112 2113 if (strlen(netdev->name) < (IFNAMSIZ - 5)) 2114 snprintf(adapter->rx_ring->name, 2115 sizeof(adapter->rx_ring->name) - 1, 2116 "%.14s-rx-0", netdev->name); 2117 else 2118 memcpy(adapter->rx_ring->name, netdev->name, IFNAMSIZ); 2119 err = request_irq(adapter->msix_entries[vector].vector, 2120 e1000_intr_msix_rx, 0, adapter->rx_ring->name, 2121 netdev); 2122 if (err) 2123 return err; 2124 adapter->rx_ring->itr_register = adapter->hw.hw_addr + 2125 E1000_EITR_82574(vector); 2126 adapter->rx_ring->itr_val = adapter->itr; 2127 vector++; 2128 2129 if (strlen(netdev->name) < (IFNAMSIZ - 5)) 2130 snprintf(adapter->tx_ring->name, 2131 sizeof(adapter->tx_ring->name) - 1, 2132 "%.14s-tx-0", netdev->name); 2133 else 2134 memcpy(adapter->tx_ring->name, netdev->name, IFNAMSIZ); 2135 err = request_irq(adapter->msix_entries[vector].vector, 2136 e1000_intr_msix_tx, 0, adapter->tx_ring->name, 2137 netdev); 2138 if (err) 2139 return err; 2140 adapter->tx_ring->itr_register = adapter->hw.hw_addr + 2141 E1000_EITR_82574(vector); 2142 adapter->tx_ring->itr_val = adapter->itr; 2143 vector++; 2144 2145 err = request_irq(adapter->msix_entries[vector].vector, 2146 e1000_msix_other, 0, netdev->name, netdev); 2147 if (err) 2148 return err; 2149 2150 e1000_configure_msix(adapter); 2151 2152 return 0; 2153} 2154 2155/** 2156 * e1000_request_irq - initialize interrupts 2157 * @adapter: board private structure 2158 * 2159 * Attempts to configure interrupts using the best available 2160 * capabilities of the hardware and kernel. 2161 **/ 2162static int e1000_request_irq(struct e1000_adapter *adapter) 2163{ 2164 struct net_device *netdev = adapter->netdev; 2165 int err; 2166 2167 if (adapter->msix_entries) { 2168 err = e1000_request_msix(adapter); 2169 if (!err) 2170 return err; 2171 /* fall back to MSI */ 2172 e1000e_reset_interrupt_capability(adapter); 2173 adapter->int_mode = E1000E_INT_MODE_MSI; 2174 e1000e_set_interrupt_capability(adapter); 2175 } 2176 if (adapter->flags & FLAG_MSI_ENABLED) { 2177 err = request_irq(adapter->pdev->irq, e1000_intr_msi, 0, 2178 netdev->name, netdev); 2179 if (!err) 2180 return err; 2181 2182 /* fall back to legacy interrupt */ 2183 e1000e_reset_interrupt_capability(adapter); 2184 adapter->int_mode = E1000E_INT_MODE_LEGACY; 2185 } 2186 2187 err = request_irq(adapter->pdev->irq, e1000_intr, IRQF_SHARED, 2188 netdev->name, netdev); 2189 if (err) 2190 e_err("Unable to allocate interrupt, Error: %d\n", err); 2191 2192 return err; 2193} 2194 2195static void e1000_free_irq(struct e1000_adapter *adapter) 2196{ 2197 struct net_device *netdev = adapter->netdev; 2198 2199 if (adapter->msix_entries) { 2200 int vector = 0; 2201 2202 free_irq(adapter->msix_entries[vector].vector, netdev); 2203 vector++; 2204 2205 free_irq(adapter->msix_entries[vector].vector, netdev); 2206 vector++; 2207 2208 /* Other Causes interrupt vector */ 2209 free_irq(adapter->msix_entries[vector].vector, netdev); 2210 return; 2211 } 2212 2213 free_irq(adapter->pdev->irq, netdev); 2214} 2215 2216/** 2217 * e1000_irq_disable - Mask off interrupt generation on the NIC 2218 * @adapter: board private structure 2219 **/ 2220static void e1000_irq_disable(struct e1000_adapter *adapter) 2221{ 2222 struct e1000_hw *hw = &adapter->hw; 2223 2224 ew32(IMC, ~0); 2225 if (adapter->msix_entries) 2226 ew32(EIAC_82574, 0); 2227 e1e_flush(); 2228 2229 if (adapter->msix_entries) { 2230 int i; 2231 2232 for (i = 0; i < adapter->num_vectors; i++) 2233 synchronize_irq(adapter->msix_entries[i].vector); 2234 } else { 2235 synchronize_irq(adapter->pdev->irq); 2236 } 2237} 2238 2239/** 2240 * e1000_irq_enable - Enable default interrupt generation settings 2241 * @adapter: board private structure 2242 **/ 2243static void e1000_irq_enable(struct e1000_adapter *adapter) 2244{ 2245 struct e1000_hw *hw = &adapter->hw; 2246 2247 if (adapter->msix_entries) { 2248 ew32(EIAC_82574, adapter->eiac_mask & E1000_EIAC_MASK_82574); 2249 ew32(IMS, adapter->eiac_mask | E1000_IMS_OTHER | 2250 IMS_OTHER_MASK); 2251 } else if (hw->mac.type >= e1000_pch_lpt) { 2252 ew32(IMS, IMS_ENABLE_MASK | E1000_IMS_ECCER); 2253 } else { 2254 ew32(IMS, IMS_ENABLE_MASK); 2255 } 2256 e1e_flush(); 2257} 2258 2259/** 2260 * e1000e_get_hw_control - get control of the h/w from f/w 2261 * @adapter: address of board private structure 2262 * 2263 * e1000e_get_hw_control sets {CTRL_EXT|SWSM}:DRV_LOAD bit. 2264 * For ASF and Pass Through versions of f/w this means that 2265 * the driver is loaded. For AMT version (only with 82573) 2266 * of the f/w this means that the network i/f is open. 2267 **/ 2268void e1000e_get_hw_control(struct e1000_adapter *adapter) 2269{ 2270 struct e1000_hw *hw = &adapter->hw; 2271 u32 ctrl_ext; 2272 u32 swsm; 2273 2274 /* Let firmware know the driver has taken over */ 2275 if (adapter->flags & FLAG_HAS_SWSM_ON_LOAD) { 2276 swsm = er32(SWSM); 2277 ew32(SWSM, swsm | E1000_SWSM_DRV_LOAD); 2278 } else if (adapter->flags & FLAG_HAS_CTRLEXT_ON_LOAD) { 2279 ctrl_ext = er32(CTRL_EXT); 2280 ew32(CTRL_EXT, ctrl_ext | E1000_CTRL_EXT_DRV_LOAD); 2281 } 2282} 2283 2284/** 2285 * e1000e_release_hw_control - release control of the h/w to f/w 2286 * @adapter: address of board private structure 2287 * 2288 * e1000e_release_hw_control resets {CTRL_EXT|SWSM}:DRV_LOAD bit. 2289 * For ASF and Pass Through versions of f/w this means that the 2290 * driver is no longer loaded. For AMT version (only with 82573) i 2291 * of the f/w this means that the network i/f is closed. 2292 * 2293 **/ 2294void e1000e_release_hw_control(struct e1000_adapter *adapter) 2295{ 2296 struct e1000_hw *hw = &adapter->hw; 2297 u32 ctrl_ext; 2298 u32 swsm; 2299 2300 /* Let firmware taken over control of h/w */ 2301 if (adapter->flags & FLAG_HAS_SWSM_ON_LOAD) { 2302 swsm = er32(SWSM); 2303 ew32(SWSM, swsm & ~E1000_SWSM_DRV_LOAD); 2304 } else if (adapter->flags & FLAG_HAS_CTRLEXT_ON_LOAD) { 2305 ctrl_ext = er32(CTRL_EXT); 2306 ew32(CTRL_EXT, ctrl_ext & ~E1000_CTRL_EXT_DRV_LOAD); 2307 } 2308} 2309 2310/** 2311 * e1000_alloc_ring_dma - allocate memory for a ring structure 2312 * @adapter: board private structure 2313 * @ring: ring struct for which to allocate dma 2314 **/ 2315static int e1000_alloc_ring_dma(struct e1000_adapter *adapter, 2316 struct e1000_ring *ring) 2317{ 2318 struct pci_dev *pdev = adapter->pdev; 2319 2320 ring->desc = dma_alloc_coherent(&pdev->dev, ring->size, &ring->dma, 2321 GFP_KERNEL); 2322 if (!ring->desc) 2323 return -ENOMEM; 2324 2325 return 0; 2326} 2327 2328/** 2329 * e1000e_setup_tx_resources - allocate Tx resources (Descriptors) 2330 * @tx_ring: Tx descriptor ring 2331 * 2332 * Return 0 on success, negative on failure 2333 **/ 2334int e1000e_setup_tx_resources(struct e1000_ring *tx_ring) 2335{ 2336 struct e1000_adapter *adapter = tx_ring->adapter; 2337 int err = -ENOMEM, size; 2338 2339 size = sizeof(struct e1000_buffer) * tx_ring->count; 2340 tx_ring->buffer_info = vzalloc(size); 2341 if (!tx_ring->buffer_info) 2342 goto err; 2343 2344 /* round up to nearest 4K */ 2345 tx_ring->size = tx_ring->count * sizeof(struct e1000_tx_desc); 2346 tx_ring->size = ALIGN(tx_ring->size, 4096); 2347 2348 err = e1000_alloc_ring_dma(adapter, tx_ring); 2349 if (err) 2350 goto err; 2351 2352 tx_ring->next_to_use = 0; 2353 tx_ring->next_to_clean = 0; 2354 2355 return 0; 2356err: 2357 vfree(tx_ring->buffer_info); 2358 e_err("Unable to allocate memory for the transmit descriptor ring\n"); 2359 return err; 2360} 2361 2362/** 2363 * e1000e_setup_rx_resources - allocate Rx resources (Descriptors) 2364 * @rx_ring: Rx descriptor ring 2365 * 2366 * Returns 0 on success, negative on failure 2367 **/ 2368int e1000e_setup_rx_resources(struct e1000_ring *rx_ring) 2369{ 2370 struct e1000_adapter *adapter = rx_ring->adapter; 2371 struct e1000_buffer *buffer_info; 2372 int i, size, desc_len, err = -ENOMEM; 2373 2374 size = sizeof(struct e1000_buffer) * rx_ring->count; 2375 rx_ring->buffer_info = vzalloc(size); 2376 if (!rx_ring->buffer_info) 2377 goto err; 2378 2379 for (i = 0; i < rx_ring->count; i++) { 2380 buffer_info = &rx_ring->buffer_info[i]; 2381 buffer_info->ps_pages = kcalloc(PS_PAGE_BUFFERS, 2382 sizeof(struct e1000_ps_page), 2383 GFP_KERNEL); 2384 if (!buffer_info->ps_pages) 2385 goto err_pages; 2386 } 2387 2388 desc_len = sizeof(union e1000_rx_desc_packet_split); 2389 2390 /* Round up to nearest 4K */ 2391 rx_ring->size = rx_ring->count * desc_len; 2392 rx_ring->size = ALIGN(rx_ring->size, 4096); 2393 2394 err = e1000_alloc_ring_dma(adapter, rx_ring); 2395 if (err) 2396 goto err_pages; 2397 2398 rx_ring->next_to_clean = 0; 2399 rx_ring->next_to_use = 0; 2400 rx_ring->rx_skb_top = NULL; 2401 2402 return 0; 2403 2404err_pages: 2405 for (i = 0; i < rx_ring->count; i++) { 2406 buffer_info = &rx_ring->buffer_info[i]; 2407 kfree(buffer_info->ps_pages); 2408 } 2409err: 2410 vfree(rx_ring->buffer_info); 2411 e_err("Unable to allocate memory for the receive descriptor ring\n"); 2412 return err; 2413} 2414 2415/** 2416 * e1000_clean_tx_ring - Free Tx Buffers 2417 * @tx_ring: Tx descriptor ring 2418 **/ 2419static void e1000_clean_tx_ring(struct e1000_ring *tx_ring) 2420{ 2421 struct e1000_adapter *adapter = tx_ring->adapter; 2422 struct e1000_buffer *buffer_info; 2423 unsigned long size; 2424 unsigned int i; 2425 2426 for (i = 0; i < tx_ring->count; i++) { 2427 buffer_info = &tx_ring->buffer_info[i]; 2428 e1000_put_txbuf(tx_ring, buffer_info, false); 2429 } 2430 2431 netdev_reset_queue(adapter->netdev); 2432 size = sizeof(struct e1000_buffer) * tx_ring->count; 2433 memset(tx_ring->buffer_info, 0, size); 2434 2435 memset(tx_ring->desc, 0, tx_ring->size); 2436 2437 tx_ring->next_to_use = 0; 2438 tx_ring->next_to_clean = 0; 2439} 2440 2441/** 2442 * e1000e_free_tx_resources - Free Tx Resources per Queue 2443 * @tx_ring: Tx descriptor ring 2444 * 2445 * Free all transmit software resources 2446 **/ 2447void e1000e_free_tx_resources(struct e1000_ring *tx_ring) 2448{ 2449 struct e1000_adapter *adapter = tx_ring->adapter; 2450 struct pci_dev *pdev = adapter->pdev; 2451 2452 e1000_clean_tx_ring(tx_ring); 2453 2454 vfree(tx_ring->buffer_info); 2455 tx_ring->buffer_info = NULL; 2456 2457 dma_free_coherent(&pdev->dev, tx_ring->size, tx_ring->desc, 2458 tx_ring->dma); 2459 tx_ring->desc = NULL; 2460} 2461 2462/** 2463 * e1000e_free_rx_resources - Free Rx Resources 2464 * @rx_ring: Rx descriptor ring 2465 * 2466 * Free all receive software resources 2467 **/ 2468void e1000e_free_rx_resources(struct e1000_ring *rx_ring) 2469{ 2470 struct e1000_adapter *adapter = rx_ring->adapter; 2471 struct pci_dev *pdev = adapter->pdev; 2472 int i; 2473 2474 e1000_clean_rx_ring(rx_ring); 2475 2476 for (i = 0; i < rx_ring->count; i++) 2477 kfree(rx_ring->buffer_info[i].ps_pages); 2478 2479 vfree(rx_ring->buffer_info); 2480 rx_ring->buffer_info = NULL; 2481 2482 dma_free_coherent(&pdev->dev, rx_ring->size, rx_ring->desc, 2483 rx_ring->dma); 2484 rx_ring->desc = NULL; 2485} 2486 2487/** 2488 * e1000_update_itr - update the dynamic ITR value based on statistics 2489 * @itr_setting: current adapter->itr 2490 * @packets: the number of packets during this measurement interval 2491 * @bytes: the number of bytes during this measurement interval 2492 * 2493 * Stores a new ITR value based on packets and byte 2494 * counts during the last interrupt. The advantage of per interrupt 2495 * computation is faster updates and more accurate ITR for the current 2496 * traffic pattern. Constants in this function were computed 2497 * based on theoretical maximum wire speed and thresholds were set based 2498 * on testing data as well as attempting to minimize response time 2499 * while increasing bulk throughput. This functionality is controlled 2500 * by the InterruptThrottleRate module parameter. 2501 **/ 2502static unsigned int e1000_update_itr(u16 itr_setting, int packets, int bytes) 2503{ 2504 unsigned int retval = itr_setting; 2505 2506 if (packets == 0) 2507 return itr_setting; 2508 2509 switch (itr_setting) { 2510 case lowest_latency: 2511 /* handle TSO and jumbo frames */ 2512 if (bytes / packets > 8000) 2513 retval = bulk_latency; 2514 else if ((packets < 5) && (bytes > 512)) 2515 retval = low_latency; 2516 break; 2517 case low_latency: /* 50 usec aka 20000 ints/s */ 2518 if (bytes > 10000) { 2519 /* this if handles the TSO accounting */ 2520 if (bytes / packets > 8000) 2521 retval = bulk_latency; 2522 else if ((packets < 10) || ((bytes / packets) > 1200)) 2523 retval = bulk_latency; 2524 else if ((packets > 35)) 2525 retval = lowest_latency; 2526 } else if (bytes / packets > 2000) { 2527 retval = bulk_latency; 2528 } else if (packets <= 2 && bytes < 512) { 2529 retval = lowest_latency; 2530 } 2531 break; 2532 case bulk_latency: /* 250 usec aka 4000 ints/s */ 2533 if (bytes > 25000) { 2534 if (packets > 35) 2535 retval = low_latency; 2536 } else if (bytes < 6000) { 2537 retval = low_latency; 2538 } 2539 break; 2540 } 2541 2542 return retval; 2543} 2544 2545static void e1000_set_itr(struct e1000_adapter *adapter) 2546{ 2547 u16 current_itr; 2548 u32 new_itr = adapter->itr; 2549 2550 /* for non-gigabit speeds, just fix the interrupt rate at 4000 */ 2551 if (adapter->link_speed != SPEED_1000) { 2552 current_itr = 0; 2553 new_itr = 4000; 2554 goto set_itr_now; 2555 } 2556 2557 if (adapter->flags2 & FLAG2_DISABLE_AIM) { 2558 new_itr = 0; 2559 goto set_itr_now; 2560 } 2561 2562 adapter->tx_itr = e1000_update_itr(adapter->tx_itr, 2563 adapter->total_tx_packets, 2564 adapter->total_tx_bytes); 2565 /* conservative mode (itr 3) eliminates the lowest_latency setting */ 2566 if (adapter->itr_setting == 3 && adapter->tx_itr == lowest_latency) 2567 adapter->tx_itr = low_latency; 2568 2569 adapter->rx_itr = e1000_update_itr(adapter->rx_itr, 2570 adapter->total_rx_packets, 2571 adapter->total_rx_bytes); 2572 /* conservative mode (itr 3) eliminates the lowest_latency setting */ 2573 if (adapter->itr_setting == 3 && adapter->rx_itr == lowest_latency) 2574 adapter->rx_itr = low_latency; 2575 2576 current_itr = max(adapter->rx_itr, adapter->tx_itr); 2577 2578 /* counts and packets in update_itr are dependent on these numbers */ 2579 switch (current_itr) { 2580 case lowest_latency: 2581 new_itr = 70000; 2582 break; 2583 case low_latency: 2584 new_itr = 20000; /* aka hwitr = ~200 */ 2585 break; 2586 case bulk_latency: 2587 new_itr = 4000; 2588 break; 2589 default: 2590 break; 2591 } 2592 2593set_itr_now: 2594 if (new_itr != adapter->itr) { 2595 /* this attempts to bias the interrupt rate towards Bulk 2596 * by adding intermediate steps when interrupt rate is 2597 * increasing 2598 */ 2599 new_itr = new_itr > adapter->itr ? 2600 min(adapter->itr + (new_itr >> 2), new_itr) : new_itr; 2601 adapter->itr = new_itr; 2602 adapter->rx_ring->itr_val = new_itr; 2603 if (adapter->msix_entries) 2604 adapter->rx_ring->set_itr = 1; 2605 else 2606 e1000e_write_itr(adapter, new_itr); 2607 } 2608} 2609 2610/** 2611 * e1000e_write_itr - write the ITR value to the appropriate registers 2612 * @adapter: address of board private structure 2613 * @itr: new ITR value to program 2614 * 2615 * e1000e_write_itr determines if the adapter is in MSI-X mode 2616 * and, if so, writes the EITR registers with the ITR value. 2617 * Otherwise, it writes the ITR value into the ITR register. 2618 **/ 2619void e1000e_write_itr(struct e1000_adapter *adapter, u32 itr) 2620{ 2621 struct e1000_hw *hw = &adapter->hw; 2622 u32 new_itr = itr ? 1000000000 / (itr * 256) : 0; 2623 2624 if (adapter->msix_entries) { 2625 int vector; 2626 2627 for (vector = 0; vector < adapter->num_vectors; vector++) 2628 writel(new_itr, hw->hw_addr + E1000_EITR_82574(vector)); 2629 } else { 2630 ew32(ITR, new_itr); 2631 } 2632} 2633 2634/** 2635 * e1000_alloc_queues - Allocate memory for all rings 2636 * @adapter: board private structure to initialize 2637 **/ 2638static int e1000_alloc_queues(struct e1000_adapter *adapter) 2639{ 2640 int size = sizeof(struct e1000_ring); 2641 2642 adapter->tx_ring = kzalloc(size, GFP_KERNEL); 2643 if (!adapter->tx_ring) 2644 goto err; 2645 adapter->tx_ring->count = adapter->tx_ring_count; 2646 adapter->tx_ring->adapter = adapter; 2647 2648 adapter->rx_ring = kzalloc(size, GFP_KERNEL); 2649 if (!adapter->rx_ring) 2650 goto err; 2651 adapter->rx_ring->count = adapter->rx_ring_count; 2652 adapter->rx_ring->adapter = adapter; 2653 2654 return 0; 2655err: 2656 e_err("Unable to allocate memory for queues\n"); 2657 kfree(adapter->rx_ring); 2658 kfree(adapter->tx_ring); 2659 return -ENOMEM; 2660} 2661 2662/** 2663 * e1000e_poll - NAPI Rx polling callback 2664 * @napi: struct associated with this polling callback 2665 * @budget: number of packets driver is allowed to process this poll 2666 **/ 2667static int e1000e_poll(struct napi_struct *napi, int budget) 2668{ 2669 struct e1000_adapter *adapter = container_of(napi, struct e1000_adapter, 2670 napi); 2671 struct e1000_hw *hw = &adapter->hw; 2672 struct net_device *poll_dev = adapter->netdev; 2673 int tx_cleaned = 1, work_done = 0; 2674 2675 adapter = netdev_priv(poll_dev); 2676 2677 if (!adapter->msix_entries || 2678 (adapter->rx_ring->ims_val & adapter->tx_ring->ims_val)) 2679 tx_cleaned = e1000_clean_tx_irq(adapter->tx_ring); 2680 2681 adapter->clean_rx(adapter->rx_ring, &work_done, budget); 2682 2683 if (!tx_cleaned || work_done == budget) 2684 return budget; 2685 2686 /* Exit the polling mode, but don't re-enable interrupts if stack might 2687 * poll us due to busy-polling 2688 */ 2689 if (likely(napi_complete_done(napi, work_done))) { 2690 if (adapter->itr_setting & 3) 2691 e1000_set_itr(adapter); 2692 if (!test_bit(__E1000_DOWN, &adapter->state)) { 2693 if (adapter->msix_entries) 2694 ew32(IMS, adapter->rx_ring->ims_val); 2695 else 2696 e1000_irq_enable(adapter); 2697 } 2698 } 2699 2700 return work_done; 2701} 2702 2703static int e1000_vlan_rx_add_vid(struct net_device *netdev, 2704 __always_unused __be16 proto, u16 vid) 2705{ 2706 struct e1000_adapter *adapter = netdev_priv(netdev); 2707 struct e1000_hw *hw = &adapter->hw; 2708 u32 vfta, index; 2709 2710 /* don't update vlan cookie if already programmed */ 2711 if ((adapter->hw.mng_cookie.status & 2712 E1000_MNG_DHCP_COOKIE_STATUS_VLAN) && 2713 (vid == adapter->mng_vlan_id)) 2714 return 0; 2715 2716 /* add VID to filter table */ 2717 if (adapter->flags & FLAG_HAS_HW_VLAN_FILTER) { 2718 index = (vid >> 5) & 0x7F; 2719 vfta = E1000_READ_REG_ARRAY(hw, E1000_VFTA, index); 2720 vfta |= BIT((vid & 0x1F)); 2721 hw->mac.ops.write_vfta(hw, index, vfta); 2722 } 2723 2724 set_bit(vid, adapter->active_vlans); 2725 2726 return 0; 2727} 2728 2729static int e1000_vlan_rx_kill_vid(struct net_device *netdev, 2730 __always_unused __be16 proto, u16 vid) 2731{ 2732 struct e1000_adapter *adapter = netdev_priv(netdev); 2733 struct e1000_hw *hw = &adapter->hw; 2734 u32 vfta, index; 2735 2736 if ((adapter->hw.mng_cookie.status & 2737 E1000_MNG_DHCP_COOKIE_STATUS_VLAN) && 2738 (vid == adapter->mng_vlan_id)) { 2739 /* release control to f/w */ 2740 e1000e_release_hw_control(adapter); 2741 return 0; 2742 } 2743 2744 /* remove VID from filter table */ 2745 if (adapter->flags & FLAG_HAS_HW_VLAN_FILTER) { 2746 index = (vid >> 5) & 0x7F; 2747 vfta = E1000_READ_REG_ARRAY(hw, E1000_VFTA, index); 2748 vfta &= ~BIT((vid & 0x1F)); 2749 hw->mac.ops.write_vfta(hw, index, vfta); 2750 } 2751 2752 clear_bit(vid, adapter->active_vlans); 2753 2754 return 0; 2755} 2756 2757/** 2758 * e1000e_vlan_filter_disable - helper to disable hw VLAN filtering 2759 * @adapter: board private structure to initialize 2760 **/ 2761static void e1000e_vlan_filter_disable(struct e1000_adapter *adapter) 2762{ 2763 struct net_device *netdev = adapter->netdev; 2764 struct e1000_hw *hw = &adapter->hw; 2765 u32 rctl; 2766 2767 if (adapter->flags & FLAG_HAS_HW_VLAN_FILTER) { 2768 /* disable VLAN receive filtering */ 2769 rctl = er32(RCTL); 2770 rctl &= ~(E1000_RCTL_VFE | E1000_RCTL_CFIEN); 2771 ew32(RCTL, rctl); 2772 2773 if (adapter->mng_vlan_id != (u16)E1000_MNG_VLAN_NONE) { 2774 e1000_vlan_rx_kill_vid(netdev, htons(ETH_P_8021Q), 2775 adapter->mng_vlan_id); 2776 adapter->mng_vlan_id = E1000_MNG_VLAN_NONE; 2777 } 2778 } 2779} 2780 2781/** 2782 * e1000e_vlan_filter_enable - helper to enable HW VLAN filtering 2783 * @adapter: board private structure to initialize 2784 **/ 2785static void e1000e_vlan_filter_enable(struct e1000_adapter *adapter) 2786{ 2787 struct e1000_hw *hw = &adapter->hw; 2788 u32 rctl; 2789 2790 if (adapter->flags & FLAG_HAS_HW_VLAN_FILTER) { 2791 /* enable VLAN receive filtering */ 2792 rctl = er32(RCTL); 2793 rctl |= E1000_RCTL_VFE; 2794 rctl &= ~E1000_RCTL_CFIEN; 2795 ew32(RCTL, rctl); 2796 } 2797} 2798 2799/** 2800 * e1000e_vlan_strip_disable - helper to disable HW VLAN stripping 2801 * @adapter: board private structure to initialize 2802 **/ 2803static void e1000e_vlan_strip_disable(struct e1000_adapter *adapter) 2804{ 2805 struct e1000_hw *hw = &adapter->hw; 2806 u32 ctrl; 2807 2808 /* disable VLAN tag insert/strip */ 2809 ctrl = er32(CTRL); 2810 ctrl &= ~E1000_CTRL_VME; 2811 ew32(CTRL, ctrl); 2812} 2813 2814/** 2815 * e1000e_vlan_strip_enable - helper to enable HW VLAN stripping 2816 * @adapter: board private structure to initialize 2817 **/ 2818static void e1000e_vlan_strip_enable(struct e1000_adapter *adapter) 2819{ 2820 struct e1000_hw *hw = &adapter->hw; 2821 u32 ctrl; 2822 2823 /* enable VLAN tag insert/strip */ 2824 ctrl = er32(CTRL); 2825 ctrl |= E1000_CTRL_VME; 2826 ew32(CTRL, ctrl); 2827} 2828 2829static void e1000_update_mng_vlan(struct e1000_adapter *adapter) 2830{ 2831 struct net_device *netdev = adapter->netdev; 2832 u16 vid = adapter->hw.mng_cookie.vlan_id; 2833 u16 old_vid = adapter->mng_vlan_id; 2834 2835 if (adapter->hw.mng_cookie.status & E1000_MNG_DHCP_COOKIE_STATUS_VLAN) { 2836 e1000_vlan_rx_add_vid(netdev, htons(ETH_P_8021Q), vid); 2837 adapter->mng_vlan_id = vid; 2838 } 2839 2840 if ((old_vid != (u16)E1000_MNG_VLAN_NONE) && (vid != old_vid)) 2841 e1000_vlan_rx_kill_vid(netdev, htons(ETH_P_8021Q), old_vid); 2842} 2843 2844static void e1000_restore_vlan(struct e1000_adapter *adapter) 2845{ 2846 u16 vid; 2847 2848 e1000_vlan_rx_add_vid(adapter->netdev, htons(ETH_P_8021Q), 0); 2849 2850 for_each_set_bit(vid, adapter->active_vlans, VLAN_N_VID) 2851 e1000_vlan_rx_add_vid(adapter->netdev, htons(ETH_P_8021Q), vid); 2852} 2853 2854static void e1000_init_manageability_pt(struct e1000_adapter *adapter) 2855{ 2856 struct e1000_hw *hw = &adapter->hw; 2857 u32 manc, manc2h, mdef, i, j; 2858 2859 if (!(adapter->flags & FLAG_MNG_PT_ENABLED)) 2860 return; 2861 2862 manc = er32(MANC); 2863 2864 /* enable receiving management packets to the host. this will probably 2865 * generate destination unreachable messages from the host OS, but 2866 * the packets will be handled on SMBUS 2867 */ 2868 manc |= E1000_MANC_EN_MNG2HOST; 2869 manc2h = er32(MANC2H); 2870 2871 switch (hw->mac.type) { 2872 default: 2873 manc2h |= (E1000_MANC2H_PORT_623 | E1000_MANC2H_PORT_664); 2874 break; 2875 case e1000_82574: 2876 case e1000_82583: 2877 /* Check if IPMI pass-through decision filter already exists; 2878 * if so, enable it. 2879 */ 2880 for (i = 0, j = 0; i < 8; i++) { 2881 mdef = er32(MDEF(i)); 2882 2883 /* Ignore filters with anything other than IPMI ports */ 2884 if (mdef & ~(E1000_MDEF_PORT_623 | E1000_MDEF_PORT_664)) 2885 continue; 2886 2887 /* Enable this decision filter in MANC2H */ 2888 if (mdef) 2889 manc2h |= BIT(i); 2890 2891 j |= mdef; 2892 } 2893 2894 if (j == (E1000_MDEF_PORT_623 | E1000_MDEF_PORT_664)) 2895 break; 2896 2897 /* Create new decision filter in an empty filter */ 2898 for (i = 0, j = 0; i < 8; i++) 2899 if (er32(MDEF(i)) == 0) { 2900 ew32(MDEF(i), (E1000_MDEF_PORT_623 | 2901 E1000_MDEF_PORT_664)); 2902 manc2h |= BIT(1); 2903 j++; 2904 break; 2905 } 2906 2907 if (!j) 2908 e_warn("Unable to create IPMI pass-through filter\n"); 2909 break; 2910 } 2911 2912 ew32(MANC2H, manc2h); 2913 ew32(MANC, manc); 2914} 2915 2916/** 2917 * e1000_configure_tx - Configure Transmit Unit after Reset 2918 * @adapter: board private structure 2919 * 2920 * Configure the Tx unit of the MAC after a reset. 2921 **/ 2922static void e1000_configure_tx(struct e1000_adapter *adapter) 2923{ 2924 struct e1000_hw *hw = &adapter->hw; 2925 struct e1000_ring *tx_ring = adapter->tx_ring; 2926 u64 tdba; 2927 u32 tdlen, tctl, tarc; 2928 2929 /* Setup the HW Tx Head and Tail descriptor pointers */ 2930 tdba = tx_ring->dma; 2931 tdlen = tx_ring->count * sizeof(struct e1000_tx_desc); 2932 ew32(TDBAL(0), (tdba & DMA_BIT_MASK(32))); 2933 ew32(TDBAH(0), (tdba >> 32)); 2934 ew32(TDLEN(0), tdlen); 2935 ew32(TDH(0), 0); 2936 ew32(TDT(0), 0); 2937 tx_ring->head = adapter->hw.hw_addr + E1000_TDH(0); 2938 tx_ring->tail = adapter->hw.hw_addr + E1000_TDT(0); 2939 2940 writel(0, tx_ring->head); 2941 if (adapter->flags2 & FLAG2_PCIM2PCI_ARBITER_WA) 2942 e1000e_update_tdt_wa(tx_ring, 0); 2943 else 2944 writel(0, tx_ring->tail); 2945 2946 /* Set the Tx Interrupt Delay register */ 2947 ew32(TIDV, adapter->tx_int_delay); 2948 /* Tx irq moderation */ 2949 ew32(TADV, adapter->tx_abs_int_delay); 2950 2951 if (adapter->flags2 & FLAG2_DMA_BURST) { 2952 u32 txdctl = er32(TXDCTL(0)); 2953 2954 txdctl &= ~(E1000_TXDCTL_PTHRESH | E1000_TXDCTL_HTHRESH | 2955 E1000_TXDCTL_WTHRESH); 2956 /* set up some performance related parameters to encourage the 2957 * hardware to use the bus more efficiently in bursts, depends 2958 * on the tx_int_delay to be enabled, 2959 * wthresh = 1 ==> burst write is disabled to avoid Tx stalls 2960 * hthresh = 1 ==> prefetch when one or more available 2961 * pthresh = 0x1f ==> prefetch if internal cache 31 or less 2962 * BEWARE: this seems to work but should be considered first if 2963 * there are Tx hangs or other Tx related bugs 2964 */ 2965 txdctl |= E1000_TXDCTL_DMA_BURST_ENABLE; 2966 ew32(TXDCTL(0), txdctl); 2967 } 2968 /* erratum work around: set txdctl the same for both queues */ 2969 ew32(TXDCTL(1), er32(TXDCTL(0))); 2970 2971 /* Program the Transmit Control Register */ 2972 tctl = er32(TCTL); 2973 tctl &= ~E1000_TCTL_CT; 2974 tctl |= E1000_TCTL_PSP | E1000_TCTL_RTLC | 2975 (E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT); 2976 2977 if (adapter->flags & FLAG_TARC_SPEED_MODE_BIT) { 2978 tarc = er32(TARC(0)); 2979 /* set the speed mode bit, we'll clear it if we're not at 2980 * gigabit link later 2981 */ 2982#define SPEED_MODE_BIT BIT(21) 2983 tarc |= SPEED_MODE_BIT; 2984 ew32(TARC(0), tarc); 2985 } 2986 2987 /* errata: program both queues to unweighted RR */ 2988 if (adapter->flags & FLAG_TARC_SET_BIT_ZERO) { 2989 tarc = er32(TARC(0)); 2990 tarc |= 1; 2991 ew32(TARC(0), tarc); 2992 tarc = er32(TARC(1)); 2993 tarc |= 1; 2994 ew32(TARC(1), tarc); 2995 } 2996 2997 /* Setup Transmit Descriptor Settings for eop descriptor */ 2998 adapter->txd_cmd = E1000_TXD_CMD_EOP | E1000_TXD_CMD_IFCS; 2999 3000 /* only set IDE if we are delaying interrupts using the timers */ 3001 if (adapter->tx_int_delay) 3002 adapter->txd_cmd |= E1000_TXD_CMD_IDE; 3003 3004 /* enable Report Status bit */ 3005 adapter->txd_cmd |= E1000_TXD_CMD_RS; 3006 3007 ew32(TCTL, tctl); 3008 3009 hw->mac.ops.config_collision_dist(hw); 3010 3011 /* SPT and KBL Si errata workaround to avoid data corruption */ 3012 if (hw->mac.type == e1000_pch_spt) { 3013 u32 reg_val; 3014 3015 reg_val = er32(IOSFPC); 3016 reg_val |= E1000_RCTL_RDMTS_HEX; 3017 ew32(IOSFPC, reg_val); 3018 3019 reg_val = er32(TARC(0)); 3020 /* SPT and KBL Si errata workaround to avoid Tx hang. 3021 * Dropping the number of outstanding requests from 3022 * 3 to 2 in order to avoid a buffer overrun. 3023 */ 3024 reg_val &= ~E1000_TARC0_CB_MULTIQ_3_REQ; 3025 reg_val |= E1000_TARC0_CB_MULTIQ_2_REQ; 3026 ew32(TARC(0), reg_val); 3027 } 3028} 3029 3030#define PAGE_USE_COUNT(S) (((S) >> PAGE_SHIFT) + \ 3031 (((S) & (PAGE_SIZE - 1)) ? 1 : 0)) 3032 3033/** 3034 * e1000_setup_rctl - configure the receive control registers 3035 * @adapter: Board private structure 3036 **/ 3037static void e1000_setup_rctl(struct e1000_adapter *adapter) 3038{ 3039 struct e1000_hw *hw = &adapter->hw; 3040 u32 rctl, rfctl; 3041 u32 pages = 0; 3042 3043 /* Workaround Si errata on PCHx - configure jumbo frame flow. 3044 * If jumbo frames not set, program related MAC/PHY registers 3045 * to h/w defaults 3046 */ 3047 if (hw->mac.type >= e1000_pch2lan) { 3048 s32 ret_val; 3049 3050 if (adapter->netdev->mtu > ETH_DATA_LEN) 3051 ret_val = e1000_lv_jumbo_workaround_ich8lan(hw, true); 3052 else 3053 ret_val = e1000_lv_jumbo_workaround_ich8lan(hw, false); 3054 3055 if (ret_val) 3056 e_dbg("failed to enable|disable jumbo frame workaround mode\n"); 3057 } 3058 3059 /* Program MC offset vector base */ 3060 rctl = er32(RCTL); 3061 rctl &= ~(3 << E1000_RCTL_MO_SHIFT); 3062 rctl |= E1000_RCTL_EN | E1000_RCTL_BAM | 3063 E1000_RCTL_LBM_NO | E1000_RCTL_RDMTS_HALF | 3064 (adapter->hw.mac.mc_filter_type << E1000_RCTL_MO_SHIFT); 3065 3066 /* Do not Store bad packets */ 3067 rctl &= ~E1000_RCTL_SBP; 3068 3069 /* Enable Long Packet receive */ 3070 if (adapter->netdev->mtu <= ETH_DATA_LEN) 3071 rctl &= ~E1000_RCTL_LPE; 3072 else 3073 rctl |= E1000_RCTL_LPE; 3074 3075 /* Some systems expect that the CRC is included in SMBUS traffic. The 3076 * hardware strips the CRC before sending to both SMBUS (BMC) and to 3077 * host memory when this is enabled 3078 */ 3079 if (adapter->flags2 & FLAG2_CRC_STRIPPING) 3080 rctl |= E1000_RCTL_SECRC; 3081 3082 /* Workaround Si errata on 82577 PHY - configure IPG for jumbos */ 3083 if ((hw->phy.type == e1000_phy_82577) && (rctl & E1000_RCTL_LPE)) { 3084 u16 phy_data; 3085 3086 e1e_rphy(hw, PHY_REG(770, 26), &phy_data); 3087 phy_data &= 0xfff8; 3088 phy_data |= BIT(2); 3089 e1e_wphy(hw, PHY_REG(770, 26), phy_data); 3090 3091 e1e_rphy(hw, 22, &phy_data); 3092 phy_data &= 0x0fff; 3093 phy_data |= BIT(14); 3094 e1e_wphy(hw, 0x10, 0x2823); 3095 e1e_wphy(hw, 0x11, 0x0003); 3096 e1e_wphy(hw, 22, phy_data); 3097 } 3098 3099 /* Setup buffer sizes */ 3100 rctl &= ~E1000_RCTL_SZ_4096; 3101 rctl |= E1000_RCTL_BSEX; 3102 switch (adapter->rx_buffer_len) { 3103 case 2048: 3104 default: 3105 rctl |= E1000_RCTL_SZ_2048; 3106 rctl &= ~E1000_RCTL_BSEX; 3107 break; 3108 case 4096: 3109 rctl |= E1000_RCTL_SZ_4096; 3110 break; 3111 case 8192: 3112 rctl |= E1000_RCTL_SZ_8192; 3113 break; 3114 case 16384: 3115 rctl |= E1000_RCTL_SZ_16384; 3116 break; 3117 } 3118 3119 /* Enable Extended Status in all Receive Descriptors */ 3120 rfctl = er32(RFCTL); 3121 rfctl |= E1000_RFCTL_EXTEN; 3122 ew32(RFCTL, rfctl); 3123 3124 /* 82571 and greater support packet-split where the protocol 3125 * header is placed in skb->data and the packet data is 3126 * placed in pages hanging off of skb_shinfo(skb)->nr_frags. 3127 * In the case of a non-split, skb->data is linearly filled, 3128 * followed by the page buffers. Therefore, skb->data is 3129 * sized to hold the largest protocol header. 3130 * 3131 * allocations using alloc_page take too long for regular MTU 3132 * so only enable packet split for jumbo frames 3133 * 3134 * Using pages when the page size is greater than 16k wastes 3135 * a lot of memory, since we allocate 3 pages at all times 3136 * per packet. 3137 */ 3138 pages = PAGE_USE_COUNT(adapter->netdev->mtu); 3139 if ((pages <= 3) && (PAGE_SIZE <= 16384) && (rctl & E1000_RCTL_LPE)) 3140 adapter->rx_ps_pages = pages; 3141 else 3142 adapter->rx_ps_pages = 0; 3143 3144 if (adapter->rx_ps_pages) { 3145 u32 psrctl = 0; 3146 3147 /* Enable Packet split descriptors */ 3148 rctl |= E1000_RCTL_DTYP_PS; 3149 3150 psrctl |= adapter->rx_ps_bsize0 >> E1000_PSRCTL_BSIZE0_SHIFT; 3151 3152 switch (adapter->rx_ps_pages) { 3153 case 3: 3154 psrctl |= PAGE_SIZE << E1000_PSRCTL_BSIZE3_SHIFT; 3155 fallthrough; 3156 case 2: 3157 psrctl |= PAGE_SIZE << E1000_PSRCTL_BSIZE2_SHIFT; 3158 fallthrough; 3159 case 1: 3160 psrctl |= PAGE_SIZE >> E1000_PSRCTL_BSIZE1_SHIFT; 3161 break; 3162 } 3163 3164 ew32(PSRCTL, psrctl); 3165 } 3166 3167 /* This is useful for sniffing bad packets. */ 3168 if (adapter->netdev->features & NETIF_F_RXALL) { 3169 /* UPE and MPE will be handled by normal PROMISC logic 3170 * in e1000e_set_rx_mode 3171 */ 3172 rctl |= (E1000_RCTL_SBP | /* Receive bad packets */ 3173 E1000_RCTL_BAM | /* RX All Bcast Pkts */ 3174 E1000_RCTL_PMCF); /* RX All MAC Ctrl Pkts */ 3175 3176 rctl &= ~(E1000_RCTL_VFE | /* Disable VLAN filter */ 3177 E1000_RCTL_DPF | /* Allow filtered pause */ 3178 E1000_RCTL_CFIEN); /* Dis VLAN CFIEN Filter */ 3179 /* Do not mess with E1000_CTRL_VME, it affects transmit as well, 3180 * and that breaks VLANs. 3181 */ 3182 } 3183 3184 ew32(RCTL, rctl); 3185 /* just started the receive unit, no need to restart */ 3186 adapter->flags &= ~FLAG_RESTART_NOW; 3187} 3188 3189/** 3190 * e1000_configure_rx - Configure Receive Unit after Reset 3191 * @adapter: board private structure 3192 * 3193 * Configure the Rx unit of the MAC after a reset. 3194 **/ 3195static void e1000_configure_rx(struct e1000_adapter *adapter) 3196{ 3197 struct e1000_hw *hw = &adapter->hw; 3198 struct e1000_ring *rx_ring = adapter->rx_ring; 3199 u64 rdba; 3200 u32 rdlen, rctl, rxcsum, ctrl_ext; 3201 3202 if (adapter->rx_ps_pages) { 3203 /* this is a 32 byte descriptor */ 3204 rdlen = rx_ring->count * 3205 sizeof(union e1000_rx_desc_packet_split); 3206 adapter->clean_rx = e1000_clean_rx_irq_ps; 3207 adapter->alloc_rx_buf = e1000_alloc_rx_buffers_ps; 3208 } else if (adapter->netdev->mtu > ETH_FRAME_LEN + ETH_FCS_LEN) { 3209 rdlen = rx_ring->count * sizeof(union e1000_rx_desc_extended); 3210 adapter->clean_rx = e1000_clean_jumbo_rx_irq; 3211 adapter->alloc_rx_buf = e1000_alloc_jumbo_rx_buffers; 3212 } else { 3213 rdlen = rx_ring->count * sizeof(union e1000_rx_desc_extended); 3214 adapter->clean_rx = e1000_clean_rx_irq; 3215 adapter->alloc_rx_buf = e1000_alloc_rx_buffers; 3216 } 3217 3218 /* disable receives while setting up the descriptors */ 3219 rctl = er32(RCTL); 3220 if (!(adapter->flags2 & FLAG2_NO_DISABLE_RX)) 3221 ew32(RCTL, rctl & ~E1000_RCTL_EN); 3222 e1e_flush(); 3223 usleep_range(10000, 11000); 3224 3225 if (adapter->flags2 & FLAG2_DMA_BURST) { 3226 /* set the writeback threshold (only takes effect if the RDTR 3227 * is set). set GRAN=1 and write back up to 0x4 worth, and 3228 * enable prefetching of 0x20 Rx descriptors 3229 * granularity = 01 3230 * wthresh = 04, 3231 * hthresh = 04, 3232 * pthresh = 0x20 3233 */ 3234 ew32(RXDCTL(0), E1000_RXDCTL_DMA_BURST_ENABLE); 3235 ew32(RXDCTL(1), E1000_RXDCTL_DMA_BURST_ENABLE); 3236 } 3237 3238 /* set the Receive Delay Timer Register */ 3239 ew32(RDTR, adapter->rx_int_delay); 3240 3241 /* irq moderation */ 3242 ew32(RADV, adapter->rx_abs_int_delay); 3243 if ((adapter->itr_setting != 0) && (adapter->itr != 0)) 3244 e1000e_write_itr(adapter, adapter->itr); 3245 3246 ctrl_ext = er32(CTRL_EXT); 3247 /* Auto-Mask interrupts upon ICR access */ 3248 ctrl_ext |= E1000_CTRL_EXT_IAME; 3249 ew32(IAM, 0xffffffff); 3250 ew32(CTRL_EXT, ctrl_ext); 3251 e1e_flush(); 3252 3253 /* Setup the HW Rx Head and Tail Descriptor Pointers and 3254 * the Base and Length of the Rx Descriptor Ring 3255 */ 3256 rdba = rx_ring->dma; 3257 ew32(RDBAL(0), (rdba & DMA_BIT_MASK(32))); 3258 ew32(RDBAH(0), (rdba >> 32)); 3259 ew32(RDLEN(0), rdlen); 3260 ew32(RDH(0), 0); 3261 ew32(RDT(0), 0); 3262 rx_ring->head = adapter->hw.hw_addr + E1000_RDH(0); 3263 rx_ring->tail = adapter->hw.hw_addr + E1000_RDT(0); 3264 3265 writel(0, rx_ring->head); 3266 if (adapter->flags2 & FLAG2_PCIM2PCI_ARBITER_WA) 3267 e1000e_update_rdt_wa(rx_ring, 0); 3268 else 3269 writel(0, rx_ring->tail); 3270 3271 /* Enable Receive Checksum Offload for TCP and UDP */ 3272 rxcsum = er32(RXCSUM); 3273 if (adapter->netdev->features & NETIF_F_RXCSUM) 3274 rxcsum |= E1000_RXCSUM_TUOFL; 3275 else 3276 rxcsum &= ~E1000_RXCSUM_TUOFL; 3277 ew32(RXCSUM, rxcsum); 3278 3279 /* With jumbo frames, excessive C-state transition latencies result 3280 * in dropped transactions. 3281 */ 3282 if (adapter->netdev->mtu > ETH_DATA_LEN) { 3283 u32 lat = 3284 ((er32(PBA) & E1000_PBA_RXA_MASK) * 1024 - 3285 adapter->max_frame_size) * 8 / 1000; 3286 3287 if (adapter->flags & FLAG_IS_ICH) { 3288 u32 rxdctl = er32(RXDCTL(0)); 3289 3290 ew32(RXDCTL(0), rxdctl | 0x3 | BIT(8)); 3291 } 3292 3293 dev_info(&adapter->pdev->dev, 3294 "Some CPU C-states have been disabled in order to enable jumbo frames\n"); 3295 cpu_latency_qos_update_request(&adapter->pm_qos_req, lat); 3296 } else { 3297 cpu_latency_qos_update_request(&adapter->pm_qos_req, 3298 PM_QOS_DEFAULT_VALUE); 3299 } 3300 3301 /* Enable Receives */ 3302 ew32(RCTL, rctl); 3303} 3304 3305/** 3306 * e1000e_write_mc_addr_list - write multicast addresses to MTA 3307 * @netdev: network interface device structure 3308 * 3309 * Writes multicast address list to the MTA hash table. 3310 * Returns: -ENOMEM on failure 3311 * 0 on no addresses written 3312 * X on writing X addresses to MTA 3313 */ 3314static int e1000e_write_mc_addr_list(struct net_device *netdev) 3315{ 3316 struct e1000_adapter *adapter = netdev_priv(netdev); 3317 struct e1000_hw *hw = &adapter->hw; 3318 struct netdev_hw_addr *ha; 3319 u8 *mta_list; 3320 int i; 3321 3322 if (netdev_mc_empty(netdev)) { 3323 /* nothing to program, so clear mc list */ 3324 hw->mac.ops.update_mc_addr_list(hw, NULL, 0); 3325 return 0; 3326 } 3327 3328 mta_list = kcalloc(netdev_mc_count(netdev), ETH_ALEN, GFP_ATOMIC); 3329 if (!mta_list) 3330 return -ENOMEM; 3331 3332 /* update_mc_addr_list expects a packed array of only addresses. */ 3333 i = 0; 3334 netdev_for_each_mc_addr(ha, netdev) 3335 memcpy(mta_list + (i++ * ETH_ALEN), ha->addr, ETH_ALEN); 3336 3337 hw->mac.ops.update_mc_addr_list(hw, mta_list, i); 3338 kfree(mta_list); 3339 3340 return netdev_mc_count(netdev); 3341} 3342 3343/** 3344 * e1000e_write_uc_addr_list - write unicast addresses to RAR table 3345 * @netdev: network interface device structure 3346 * 3347 * Writes unicast address list to the RAR table. 3348 * Returns: -ENOMEM on failure/insufficient address space 3349 * 0 on no addresses written 3350 * X on writing X addresses to the RAR table 3351 **/ 3352static int e1000e_write_uc_addr_list(struct net_device *netdev) 3353{ 3354 struct e1000_adapter *adapter = netdev_priv(netdev); 3355 struct e1000_hw *hw = &adapter->hw; 3356 unsigned int rar_entries; 3357 int count = 0; 3358 3359 rar_entries = hw->mac.ops.rar_get_count(hw); 3360 3361 /* save a rar entry for our hardware address */ 3362 rar_entries--; 3363 3364 /* save a rar entry for the LAA workaround */ 3365 if (adapter->flags & FLAG_RESET_OVERWRITES_LAA) 3366 rar_entries--; 3367 3368 /* return ENOMEM indicating insufficient memory for addresses */ 3369 if (netdev_uc_count(netdev) > rar_entries) 3370 return -ENOMEM; 3371 3372 if (!netdev_uc_empty(netdev) && rar_entries) { 3373 struct netdev_hw_addr *ha; 3374 3375 /* write the addresses in reverse order to avoid write 3376 * combining 3377 */ 3378 netdev_for_each_uc_addr(ha, netdev) { 3379 int ret_val; 3380 3381 if (!rar_entries) 3382 break; 3383 ret_val = hw->mac.ops.rar_set(hw, ha->addr, rar_entries--); 3384 if (ret_val < 0) 3385 return -ENOMEM; 3386 count++; 3387 } 3388 } 3389 3390 /* zero out the remaining RAR entries not used above */ 3391 for (; rar_entries > 0; rar_entries--) { 3392 ew32(RAH(rar_entries), 0); 3393 ew32(RAL(rar_entries), 0); 3394 } 3395 e1e_flush(); 3396 3397 return count; 3398} 3399 3400/** 3401 * e1000e_set_rx_mode - secondary unicast, Multicast and Promiscuous mode set 3402 * @netdev: network interface device structure 3403 * 3404 * The ndo_set_rx_mode entry point is called whenever the unicast or multicast 3405 * address list or the network interface flags are updated. This routine is 3406 * responsible for configuring the hardware for proper unicast, multicast, 3407 * promiscuous mode, and all-multi behavior. 3408 **/ 3409static void e1000e_set_rx_mode(struct net_device *netdev) 3410{ 3411 struct e1000_adapter *adapter = netdev_priv(netdev); 3412 struct e1000_hw *hw = &adapter->hw; 3413 u32 rctl; 3414 3415 if (pm_runtime_suspended(netdev->dev.parent)) 3416 return; 3417 3418 /* Check for Promiscuous and All Multicast modes */ 3419 rctl = er32(RCTL); 3420 3421 /* clear the affected bits */ 3422 rctl &= ~(E1000_RCTL_UPE | E1000_RCTL_MPE); 3423 3424 if (netdev->flags & IFF_PROMISC) { 3425 rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE); 3426 /* Do not hardware filter VLANs in promisc mode */ 3427 e1000e_vlan_filter_disable(adapter); 3428 } else { 3429 int count; 3430 3431 if (netdev->flags & IFF_ALLMULTI) { 3432 rctl |= E1000_RCTL_MPE; 3433 } else { 3434 /* Write addresses to the MTA, if the attempt fails 3435 * then we should just turn on promiscuous mode so 3436 * that we can at least receive multicast traffic 3437 */ 3438 count = e1000e_write_mc_addr_list(netdev); 3439 if (count < 0) 3440 rctl |= E1000_RCTL_MPE; 3441 } 3442 e1000e_vlan_filter_enable(adapter); 3443 /* Write addresses to available RAR registers, if there is not 3444 * sufficient space to store all the addresses then enable 3445 * unicast promiscuous mode 3446 */ 3447 count = e1000e_write_uc_addr_list(netdev); 3448 if (count < 0) 3449 rctl |= E1000_RCTL_UPE; 3450 } 3451 3452 ew32(RCTL, rctl); 3453 3454 if (netdev->features & NETIF_F_HW_VLAN_CTAG_RX) 3455 e1000e_vlan_strip_enable(adapter); 3456 else 3457 e1000e_vlan_strip_disable(adapter); 3458} 3459 3460static void e1000e_setup_rss_hash(struct e1000_adapter *adapter) 3461{ 3462 struct e1000_hw *hw = &adapter->hw; 3463 u32 mrqc, rxcsum; 3464 u32 rss_key[10]; 3465 int i; 3466 3467 netdev_rss_key_fill(rss_key, sizeof(rss_key)); 3468 for (i = 0; i < 10; i++) 3469 ew32(RSSRK(i), rss_key[i]); 3470 3471 /* Direct all traffic to queue 0 */ 3472 for (i = 0; i < 32; i++) 3473 ew32(RETA(i), 0); 3474 3475 /* Disable raw packet checksumming so that RSS hash is placed in 3476 * descriptor on writeback. 3477 */ 3478 rxcsum = er32(RXCSUM); 3479 rxcsum |= E1000_RXCSUM_PCSD; 3480 3481 ew32(RXCSUM, rxcsum); 3482 3483 mrqc = (E1000_MRQC_RSS_FIELD_IPV4 | 3484 E1000_MRQC_RSS_FIELD_IPV4_TCP | 3485 E1000_MRQC_RSS_FIELD_IPV6 | 3486 E1000_MRQC_RSS_FIELD_IPV6_TCP | 3487 E1000_MRQC_RSS_FIELD_IPV6_TCP_EX); 3488 3489 ew32(MRQC, mrqc); 3490} 3491 3492/** 3493 * e1000e_get_base_timinca - get default SYSTIM time increment attributes 3494 * @adapter: board private structure 3495 * @timinca: pointer to returned time increment attributes 3496 * 3497 * Get attributes for incrementing the System Time Register SYSTIML/H at 3498 * the default base frequency, and set the cyclecounter shift value. 3499 **/ 3500s32 e1000e_get_base_timinca(struct e1000_adapter *adapter, u32 *timinca) 3501{ 3502 struct e1000_hw *hw = &adapter->hw; 3503 u32 incvalue, incperiod, shift; 3504 3505 /* Make sure clock is enabled on I217/I218/I219 before checking 3506 * the frequency 3507 */ 3508 if ((hw->mac.type >= e1000_pch_lpt) && 3509 !(er32(TSYNCTXCTL) & E1000_TSYNCTXCTL_ENABLED) && 3510 !(er32(TSYNCRXCTL) & E1000_TSYNCRXCTL_ENABLED)) { 3511 u32 fextnvm7 = er32(FEXTNVM7); 3512 3513 if (!(fextnvm7 & BIT(0))) { 3514 ew32(FEXTNVM7, fextnvm7 | BIT(0)); 3515 e1e_flush(); 3516 } 3517 } 3518 3519 switch (hw->mac.type) { 3520 case e1000_pch2lan: 3521 /* Stable 96MHz frequency */ 3522 incperiod = INCPERIOD_96MHZ; 3523 incvalue = INCVALUE_96MHZ; 3524 shift = INCVALUE_SHIFT_96MHZ; 3525 adapter->cc.shift = shift + INCPERIOD_SHIFT_96MHZ; 3526 break; 3527 case e1000_pch_lpt: 3528 if (er32(TSYNCRXCTL) & E1000_TSYNCRXCTL_SYSCFI) { 3529 /* Stable 96MHz frequency */ 3530 incperiod = INCPERIOD_96MHZ; 3531 incvalue = INCVALUE_96MHZ; 3532 shift = INCVALUE_SHIFT_96MHZ; 3533 adapter->cc.shift = shift + INCPERIOD_SHIFT_96MHZ; 3534 } else { 3535 /* Stable 25MHz frequency */ 3536 incperiod = INCPERIOD_25MHZ; 3537 incvalue = INCVALUE_25MHZ; 3538 shift = INCVALUE_SHIFT_25MHZ; 3539 adapter->cc.shift = shift; 3540 } 3541 break; 3542 case e1000_pch_spt: 3543 /* Stable 24MHz frequency */ 3544 incperiod = INCPERIOD_24MHZ; 3545 incvalue = INCVALUE_24MHZ; 3546 shift = INCVALUE_SHIFT_24MHZ; 3547 adapter->cc.shift = shift; 3548 break; 3549 case e1000_pch_cnp: 3550 case e1000_pch_tgp: 3551 case e1000_pch_adp: 3552 case e1000_pch_mtp: 3553 if (er32(TSYNCRXCTL) & E1000_TSYNCRXCTL_SYSCFI) { 3554 /* Stable 24MHz frequency */ 3555 incperiod = INCPERIOD_24MHZ; 3556 incvalue = INCVALUE_24MHZ; 3557 shift = INCVALUE_SHIFT_24MHZ; 3558 adapter->cc.shift = shift; 3559 } else { 3560 /* Stable 38400KHz frequency */ 3561 incperiod = INCPERIOD_38400KHZ; 3562 incvalue = INCVALUE_38400KHZ; 3563 shift = INCVALUE_SHIFT_38400KHZ; 3564 adapter->cc.shift = shift; 3565 } 3566 break; 3567 case e1000_82574: 3568 case e1000_82583: 3569 /* Stable 25MHz frequency */ 3570 incperiod = INCPERIOD_25MHZ; 3571 incvalue = INCVALUE_25MHZ; 3572 shift = INCVALUE_SHIFT_25MHZ; 3573 adapter->cc.shift = shift; 3574 break; 3575 default: 3576 return -EINVAL; 3577 } 3578 3579 *timinca = ((incperiod << E1000_TIMINCA_INCPERIOD_SHIFT) | 3580 ((incvalue << shift) & E1000_TIMINCA_INCVALUE_MASK)); 3581 3582 return 0; 3583} 3584 3585/** 3586 * e1000e_config_hwtstamp - configure the hwtstamp registers and enable/disable 3587 * @adapter: board private structure 3588 * @config: timestamp configuration 3589 * 3590 * Outgoing time stamping can be enabled and disabled. Play nice and 3591 * disable it when requested, although it shouldn't cause any overhead 3592 * when no packet needs it. At most one packet in the queue may be 3593 * marked for time stamping, otherwise it would be impossible to tell 3594 * for sure to which packet the hardware time stamp belongs. 3595 * 3596 * Incoming time stamping has to be configured via the hardware filters. 3597 * Not all combinations are supported, in particular event type has to be 3598 * specified. Matching the kind of event packet is not supported, with the 3599 * exception of "all V2 events regardless of level 2 or 4". 3600 **/ 3601static int e1000e_config_hwtstamp(struct e1000_adapter *adapter, 3602 struct hwtstamp_config *config) 3603{ 3604 struct e1000_hw *hw = &adapter->hw; 3605 u32 tsync_tx_ctl = E1000_TSYNCTXCTL_ENABLED; 3606 u32 tsync_rx_ctl = E1000_TSYNCRXCTL_ENABLED; 3607 u32 rxmtrl = 0; 3608 u16 rxudp = 0; 3609 bool is_l4 = false; 3610 bool is_l2 = false; 3611 u32 regval; 3612 3613 if (!(adapter->flags & FLAG_HAS_HW_TIMESTAMP)) 3614 return -EINVAL; 3615 3616 /* flags reserved for future extensions - must be zero */ 3617 if (config->flags) 3618 return -EINVAL; 3619 3620 switch (config->tx_type) { 3621 case HWTSTAMP_TX_OFF: 3622 tsync_tx_ctl = 0; 3623 break; 3624 case HWTSTAMP_TX_ON: 3625 break; 3626 default: 3627 return -ERANGE; 3628 } 3629 3630 switch (config->rx_filter) { 3631 case HWTSTAMP_FILTER_NONE: 3632 tsync_rx_ctl = 0; 3633 break; 3634 case HWTSTAMP_FILTER_PTP_V1_L4_SYNC: 3635 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L4_V1; 3636 rxmtrl = E1000_RXMTRL_PTP_V1_SYNC_MESSAGE; 3637 is_l4 = true; 3638 break; 3639 case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ: 3640 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L4_V1; 3641 rxmtrl = E1000_RXMTRL_PTP_V1_DELAY_REQ_MESSAGE; 3642 is_l4 = true; 3643 break; 3644 case HWTSTAMP_FILTER_PTP_V2_L2_SYNC: 3645 /* Also time stamps V2 L2 Path Delay Request/Response */ 3646 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L2_V2; 3647 rxmtrl = E1000_RXMTRL_PTP_V2_SYNC_MESSAGE; 3648 is_l2 = true; 3649 break; 3650 case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ: 3651 /* Also time stamps V2 L2 Path Delay Request/Response. */ 3652 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L2_V2; 3653 rxmtrl = E1000_RXMTRL_PTP_V2_DELAY_REQ_MESSAGE; 3654 is_l2 = true; 3655 break; 3656 case HWTSTAMP_FILTER_PTP_V2_L4_SYNC: 3657 /* Hardware cannot filter just V2 L4 Sync messages */ 3658 fallthrough; 3659 case HWTSTAMP_FILTER_PTP_V2_SYNC: 3660 /* Also time stamps V2 Path Delay Request/Response. */ 3661 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L2_L4_V2; 3662 rxmtrl = E1000_RXMTRL_PTP_V2_SYNC_MESSAGE; 3663 is_l2 = true; 3664 is_l4 = true; 3665 break; 3666 case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ: 3667 /* Hardware cannot filter just V2 L4 Delay Request messages */ 3668 fallthrough; 3669 case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ: 3670 /* Also time stamps V2 Path Delay Request/Response. */ 3671 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L2_L4_V2; 3672 rxmtrl = E1000_RXMTRL_PTP_V2_DELAY_REQ_MESSAGE; 3673 is_l2 = true; 3674 is_l4 = true; 3675 break; 3676 case HWTSTAMP_FILTER_PTP_V2_L4_EVENT: 3677 case HWTSTAMP_FILTER_PTP_V2_L2_EVENT: 3678 /* Hardware cannot filter just V2 L4 or L2 Event messages */ 3679 fallthrough; 3680 case HWTSTAMP_FILTER_PTP_V2_EVENT: 3681 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_EVENT_V2; 3682 config->rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT; 3683 is_l2 = true; 3684 is_l4 = true; 3685 break; 3686 case HWTSTAMP_FILTER_PTP_V1_L4_EVENT: 3687 /* For V1, the hardware can only filter Sync messages or 3688 * Delay Request messages but not both so fall-through to 3689 * time stamp all packets. 3690 */ 3691 fallthrough; 3692 case HWTSTAMP_FILTER_NTP_ALL: 3693 case HWTSTAMP_FILTER_ALL: 3694 is_l2 = true; 3695 is_l4 = true; 3696 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_ALL; 3697 config->rx_filter = HWTSTAMP_FILTER_ALL; 3698 break; 3699 default: 3700 return -ERANGE; 3701 } 3702 3703 adapter->hwtstamp_config = *config; 3704 3705 /* enable/disable Tx h/w time stamping */ 3706 regval = er32(TSYNCTXCTL); 3707 regval &= ~E1000_TSYNCTXCTL_ENABLED; 3708 regval |= tsync_tx_ctl; 3709 ew32(TSYNCTXCTL, regval); 3710 if ((er32(TSYNCTXCTL) & E1000_TSYNCTXCTL_ENABLED) != 3711 (regval & E1000_TSYNCTXCTL_ENABLED)) { 3712 e_err("Timesync Tx Control register not set as expected\n"); 3713 return -EAGAIN; 3714 } 3715 3716 /* enable/disable Rx h/w time stamping */ 3717 regval = er32(TSYNCRXCTL); 3718 regval &= ~(E1000_TSYNCRXCTL_ENABLED | E1000_TSYNCRXCTL_TYPE_MASK); 3719 regval |= tsync_rx_ctl; 3720 ew32(TSYNCRXCTL, regval); 3721 if ((er32(TSYNCRXCTL) & (E1000_TSYNCRXCTL_ENABLED | 3722 E1000_TSYNCRXCTL_TYPE_MASK)) != 3723 (regval & (E1000_TSYNCRXCTL_ENABLED | 3724 E1000_TSYNCRXCTL_TYPE_MASK))) { 3725 e_err("Timesync Rx Control register not set as expected\n"); 3726 return -EAGAIN; 3727 } 3728 3729 /* L2: define ethertype filter for time stamped packets */ 3730 if (is_l2) 3731 rxmtrl |= ETH_P_1588; 3732 3733 /* define which PTP packets get time stamped */ 3734 ew32(RXMTRL, rxmtrl); 3735 3736 /* Filter by destination port */ 3737 if (is_l4) { 3738 rxudp = PTP_EV_PORT; 3739 cpu_to_be16s(&rxudp); 3740 } 3741 ew32(RXUDP, rxudp); 3742 3743 e1e_flush(); 3744 3745 /* Clear TSYNCRXCTL_VALID & TSYNCTXCTL_VALID bit */ 3746 er32(RXSTMPH); 3747 er32(TXSTMPH); 3748 3749 return 0; 3750} 3751 3752/** 3753 * e1000_configure - configure the hardware for Rx and Tx 3754 * @adapter: private board structure 3755 **/ 3756static void e1000_configure(struct e1000_adapter *adapter) 3757{ 3758 struct e1000_ring *rx_ring = adapter->rx_ring; 3759 3760 e1000e_set_rx_mode(adapter->netdev); 3761 3762 e1000_restore_vlan(adapter); 3763 e1000_init_manageability_pt(adapter); 3764 3765 e1000_configure_tx(adapter); 3766 3767 if (adapter->netdev->features & NETIF_F_RXHASH) 3768 e1000e_setup_rss_hash(adapter); 3769 e1000_setup_rctl(adapter); 3770 e1000_configure_rx(adapter); 3771 adapter->alloc_rx_buf(rx_ring, e1000_desc_unused(rx_ring), GFP_KERNEL); 3772} 3773 3774/** 3775 * e1000e_power_up_phy - restore link in case the phy was powered down 3776 * @adapter: address of board private structure 3777 * 3778 * The phy may be powered down to save power and turn off link when the 3779 * driver is unloaded and wake on lan is not enabled (among others) 3780 * *** this routine MUST be followed by a call to e1000e_reset *** 3781 **/ 3782void e1000e_power_up_phy(struct e1000_adapter *adapter) 3783{ 3784 if (adapter->hw.phy.ops.power_up) 3785 adapter->hw.phy.ops.power_up(&adapter->hw); 3786 3787 adapter->hw.mac.ops.setup_link(&adapter->hw); 3788} 3789 3790/** 3791 * e1000_power_down_phy - Power down the PHY 3792 * @adapter: board private structure 3793 * 3794 * Power down the PHY so no link is implied when interface is down. 3795 * The PHY cannot be powered down if management or WoL is active. 3796 */ 3797static void e1000_power_down_phy(struct e1000_adapter *adapter) 3798{ 3799 if (adapter->hw.phy.ops.power_down) 3800 adapter->hw.phy.ops.power_down(&adapter->hw); 3801} 3802 3803/** 3804 * e1000_flush_tx_ring - remove all descriptors from the tx_ring 3805 * @adapter: board private structure 3806 * 3807 * We want to clear all pending descriptors from the TX ring. 3808 * zeroing happens when the HW reads the regs. We assign the ring itself as 3809 * the data of the next descriptor. We don't care about the data we are about 3810 * to reset the HW. 3811 */ 3812static void e1000_flush_tx_ring(struct e1000_adapter *adapter) 3813{ 3814 struct e1000_hw *hw = &adapter->hw; 3815 struct e1000_ring *tx_ring = adapter->tx_ring; 3816 struct e1000_tx_desc *tx_desc = NULL; 3817 u32 tdt, tctl, txd_lower = E1000_TXD_CMD_IFCS; 3818 u16 size = 512; 3819 3820 tctl = er32(TCTL); 3821 ew32(TCTL, tctl | E1000_TCTL_EN); 3822 tdt = er32(TDT(0)); 3823 BUG_ON(tdt != tx_ring->next_to_use); 3824 tx_desc = E1000_TX_DESC(*tx_ring, tx_ring->next_to_use); 3825 tx_desc->buffer_addr = cpu_to_le64(tx_ring->dma); 3826 3827 tx_desc->lower.data = cpu_to_le32(txd_lower | size); 3828 tx_desc->upper.data = 0; 3829 /* flush descriptors to memory before notifying the HW */ 3830 wmb(); 3831 tx_ring->next_to_use++; 3832 if (tx_ring->next_to_use == tx_ring->count) 3833 tx_ring->next_to_use = 0; 3834 ew32(TDT(0), tx_ring->next_to_use); 3835 usleep_range(200, 250); 3836} 3837 3838/** 3839 * e1000_flush_rx_ring - remove all descriptors from the rx_ring 3840 * @adapter: board private structure 3841 * 3842 * Mark all descriptors in the RX ring as consumed and disable the rx ring 3843 */ 3844static void e1000_flush_rx_ring(struct e1000_adapter *adapter) 3845{ 3846 u32 rctl, rxdctl; 3847 struct e1000_hw *hw = &adapter->hw; 3848 3849 rctl = er32(RCTL); 3850 ew32(RCTL, rctl & ~E1000_RCTL_EN); 3851 e1e_flush(); 3852 usleep_range(100, 150); 3853 3854 rxdctl = er32(RXDCTL(0)); 3855 /* zero the lower 14 bits (prefetch and host thresholds) */ 3856 rxdctl &= 0xffffc000; 3857 3858 /* update thresholds: prefetch threshold to 31, host threshold to 1 3859 * and make sure the granularity is "descriptors" and not "cache lines" 3860 */ 3861 rxdctl |= (0x1F | BIT(8) | E1000_RXDCTL_THRESH_UNIT_DESC); 3862 3863 ew32(RXDCTL(0), rxdctl); 3864 /* momentarily enable the RX ring for the changes to take effect */ 3865 ew32(RCTL, rctl | E1000_RCTL_EN); 3866 e1e_flush(); 3867 usleep_range(100, 150); 3868 ew32(RCTL, rctl & ~E1000_RCTL_EN); 3869} 3870 3871/** 3872 * e1000_flush_desc_rings - remove all descriptors from the descriptor rings 3873 * @adapter: board private structure 3874 * 3875 * In i219, the descriptor rings must be emptied before resetting the HW 3876 * or before changing the device state to D3 during runtime (runtime PM). 3877 * 3878 * Failure to do this will cause the HW to enter a unit hang state which can 3879 * only be released by PCI reset on the device 3880 * 3881 */ 3882 3883static void e1000_flush_desc_rings(struct e1000_adapter *adapter) 3884{ 3885 u16 hang_state; 3886 u32 fext_nvm11, tdlen; 3887 struct e1000_hw *hw = &adapter->hw; 3888 3889 /* First, disable MULR fix in FEXTNVM11 */ 3890 fext_nvm11 = er32(FEXTNVM11); 3891 fext_nvm11 |= E1000_FEXTNVM11_DISABLE_MULR_FIX; 3892 ew32(FEXTNVM11, fext_nvm11); 3893 /* do nothing if we're not in faulty state, or if the queue is empty */ 3894 tdlen = er32(TDLEN(0)); 3895 pci_read_config_word(adapter->pdev, PCICFG_DESC_RING_STATUS, 3896 &hang_state); 3897 if (!(hang_state & FLUSH_DESC_REQUIRED) || !tdlen) 3898 return; 3899 e1000_flush_tx_ring(adapter); 3900 /* recheck, maybe the fault is caused by the rx ring */ 3901 pci_read_config_word(adapter->pdev, PCICFG_DESC_RING_STATUS, 3902 &hang_state); 3903 if (hang_state & FLUSH_DESC_REQUIRED) 3904 e1000_flush_rx_ring(adapter); 3905} 3906 3907/** 3908 * e1000e_systim_reset - reset the timesync registers after a hardware reset 3909 * @adapter: board private structure 3910 * 3911 * When the MAC is reset, all hardware bits for timesync will be reset to the 3912 * default values. This function will restore the settings last in place. 3913 * Since the clock SYSTIME registers are reset, we will simply restore the 3914 * cyclecounter to the kernel real clock time. 3915 **/ 3916static void e1000e_systim_reset(struct e1000_adapter *adapter) 3917{ 3918 struct ptp_clock_info *info = &adapter->ptp_clock_info; 3919 struct e1000_hw *hw = &adapter->hw; 3920 unsigned long flags; 3921 u32 timinca; 3922 s32 ret_val; 3923 3924 if (!(adapter->flags & FLAG_HAS_HW_TIMESTAMP)) 3925 return; 3926 3927 if (info->adjfreq) { 3928 /* restore the previous ptp frequency delta */ 3929 ret_val = info->adjfreq(info, adapter->ptp_delta); 3930 } else { 3931 /* set the default base frequency if no adjustment possible */ 3932 ret_val = e1000e_get_base_timinca(adapter, &timinca); 3933 if (!ret_val) 3934 ew32(TIMINCA, timinca); 3935 } 3936 3937 if (ret_val) { 3938 dev_warn(&adapter->pdev->dev, 3939 "Failed to restore TIMINCA clock rate delta: %d\n", 3940 ret_val); 3941 return; 3942 } 3943 3944 /* reset the systim ns time counter */ 3945 spin_lock_irqsave(&adapter->systim_lock, flags); 3946 timecounter_init(&adapter->tc, &adapter->cc, 3947 ktime_to_ns(ktime_get_real())); 3948 spin_unlock_irqrestore(&adapter->systim_lock, flags); 3949 3950 /* restore the previous hwtstamp configuration settings */ 3951 e1000e_config_hwtstamp(adapter, &adapter->hwtstamp_config); 3952} 3953 3954/** 3955 * e1000e_reset - bring the hardware into a known good state 3956 * @adapter: board private structure 3957 * 3958 * This function boots the hardware and enables some settings that 3959 * require a configuration cycle of the hardware - those cannot be 3960 * set/changed during runtime. After reset the device needs to be 3961 * properly configured for Rx, Tx etc. 3962 */ 3963void e1000e_reset(struct e1000_adapter *adapter) 3964{ 3965 struct e1000_mac_info *mac = &adapter->hw.mac; 3966 struct e1000_fc_info *fc = &adapter->hw.fc; 3967 struct e1000_hw *hw = &adapter->hw; 3968 u32 tx_space, min_tx_space, min_rx_space; 3969 u32 pba = adapter->pba; 3970 u16 hwm; 3971 3972 /* reset Packet Buffer Allocation to default */ 3973 ew32(PBA, pba); 3974 3975 if (adapter->max_frame_size > (VLAN_ETH_FRAME_LEN + ETH_FCS_LEN)) { 3976 /* To maintain wire speed transmits, the Tx FIFO should be 3977 * large enough to accommodate two full transmit packets, 3978 * rounded up to the next 1KB and expressed in KB. Likewise, 3979 * the Rx FIFO should be large enough to accommodate at least 3980 * one full receive packet and is similarly rounded up and 3981 * expressed in KB. 3982 */ 3983 pba = er32(PBA); 3984 /* upper 16 bits has Tx packet buffer allocation size in KB */ 3985 tx_space = pba >> 16; 3986 /* lower 16 bits has Rx packet buffer allocation size in KB */ 3987 pba &= 0xffff; 3988 /* the Tx fifo also stores 16 bytes of information about the Tx 3989 * but don't include ethernet FCS because hardware appends it 3990 */ 3991 min_tx_space = (adapter->max_frame_size + 3992 sizeof(struct e1000_tx_desc) - ETH_FCS_LEN) * 2; 3993 min_tx_space = ALIGN(min_tx_space, 1024); 3994 min_tx_space >>= 10; 3995 /* software strips receive CRC, so leave room for it */ 3996 min_rx_space = adapter->max_frame_size; 3997 min_rx_space = ALIGN(min_rx_space, 1024); 3998 min_rx_space >>= 10; 3999 4000 /* If current Tx allocation is less than the min Tx FIFO size, 4001 * and the min Tx FIFO size is less than the current Rx FIFO 4002 * allocation, take space away from current Rx allocation 4003 */ 4004 if ((tx_space < min_tx_space) && 4005 ((min_tx_space - tx_space) < pba)) { 4006 pba -= min_tx_space - tx_space; 4007 4008 /* if short on Rx space, Rx wins and must trump Tx 4009 * adjustment 4010 */ 4011 if (pba < min_rx_space) 4012 pba = min_rx_space; 4013 } 4014 4015 ew32(PBA, pba); 4016 } 4017 4018 /* flow control settings 4019 * 4020 * The high water mark must be low enough to fit one full frame 4021 * (or the size used for early receive) above it in the Rx FIFO. 4022 * Set it to the lower of: 4023 * - 90% of the Rx FIFO size, and 4024 * - the full Rx FIFO size minus one full frame 4025 */ 4026 if (adapter->flags & FLAG_DISABLE_FC_PAUSE_TIME) 4027 fc->pause_time = 0xFFFF; 4028 else 4029 fc->pause_time = E1000_FC_PAUSE_TIME; 4030 fc->send_xon = true; 4031 fc->current_mode = fc->requested_mode; 4032 4033 switch (hw->mac.type) { 4034 case e1000_ich9lan: 4035 case e1000_ich10lan: 4036 if (adapter->netdev->mtu > ETH_DATA_LEN) { 4037 pba = 14; 4038 ew32(PBA, pba); 4039 fc->high_water = 0x2800; 4040 fc->low_water = fc->high_water - 8; 4041 break; 4042 } 4043 fallthrough; 4044 default: 4045 hwm = min(((pba << 10) * 9 / 10), 4046 ((pba << 10) - adapter->max_frame_size)); 4047 4048 fc->high_water = hwm & E1000_FCRTH_RTH; /* 8-byte granularity */ 4049 fc->low_water = fc->high_water - 8; 4050 break; 4051 case e1000_pchlan: 4052 /* Workaround PCH LOM adapter hangs with certain network 4053 * loads. If hangs persist, try disabling Tx flow control. 4054 */ 4055 if (adapter->netdev->mtu > ETH_DATA_LEN) { 4056 fc->high_water = 0x3500; 4057 fc->low_water = 0x1500; 4058 } else { 4059 fc->high_water = 0x5000; 4060 fc->low_water = 0x3000; 4061 } 4062 fc->refresh_time = 0x1000; 4063 break; 4064 case e1000_pch2lan: 4065 case e1000_pch_lpt: 4066 case e1000_pch_spt: 4067 case e1000_pch_cnp: 4068 case e1000_pch_tgp: 4069 case e1000_pch_adp: 4070 case e1000_pch_mtp: 4071 fc->refresh_time = 0xFFFF; 4072 fc->pause_time = 0xFFFF; 4073 4074 if (adapter->netdev->mtu <= ETH_DATA_LEN) { 4075 fc->high_water = 0x05C20; 4076 fc->low_water = 0x05048; 4077 break; 4078 } 4079 4080 pba = 14; 4081 ew32(PBA, pba); 4082 fc->high_water = ((pba << 10) * 9 / 10) & E1000_FCRTH_RTH; 4083 fc->low_water = ((pba << 10) * 8 / 10) & E1000_FCRTL_RTL; 4084 break; 4085 } 4086 4087 /* Alignment of Tx data is on an arbitrary byte boundary with the 4088 * maximum size per Tx descriptor limited only to the transmit 4089 * allocation of the packet buffer minus 96 bytes with an upper 4090 * limit of 24KB due to receive synchronization limitations. 4091 */ 4092 adapter->tx_fifo_limit = min_t(u32, ((er32(PBA) >> 16) << 10) - 96, 4093 24 << 10); 4094 4095 /* Disable Adaptive Interrupt Moderation if 2 full packets cannot 4096 * fit in receive buffer. 4097 */ 4098 if (adapter->itr_setting & 0x3) { 4099 if ((adapter->max_frame_size * 2) > (pba << 10)) { 4100 if (!(adapter->flags2 & FLAG2_DISABLE_AIM)) { 4101 dev_info(&adapter->pdev->dev, 4102 "Interrupt Throttle Rate off\n"); 4103 adapter->flags2 |= FLAG2_DISABLE_AIM; 4104 e1000e_write_itr(adapter, 0); 4105 } 4106 } else if (adapter->flags2 & FLAG2_DISABLE_AIM) { 4107 dev_info(&adapter->pdev->dev, 4108 "Interrupt Throttle Rate on\n"); 4109 adapter->flags2 &= ~FLAG2_DISABLE_AIM; 4110 adapter->itr = 20000; 4111 e1000e_write_itr(adapter, adapter->itr); 4112 } 4113 } 4114 4115 if (hw->mac.type >= e1000_pch_spt) 4116 e1000_flush_desc_rings(adapter); 4117 /* Allow time for pending master requests to run */ 4118 mac->ops.reset_hw(hw); 4119 4120 /* For parts with AMT enabled, let the firmware know 4121 * that the network interface is in control 4122 */ 4123 if (adapter->flags & FLAG_HAS_AMT) 4124 e1000e_get_hw_control(adapter); 4125 4126 ew32(WUC, 0); 4127 4128 if (mac->ops.init_hw(hw)) 4129 e_err("Hardware Error\n"); 4130 4131 e1000_update_mng_vlan(adapter); 4132 4133 /* Enable h/w to recognize an 802.1Q VLAN Ethernet packet */ 4134 ew32(VET, ETH_P_8021Q); 4135 4136 e1000e_reset_adaptive(hw); 4137 4138 /* restore systim and hwtstamp settings */ 4139 e1000e_systim_reset(adapter); 4140 4141 /* Set EEE advertisement as appropriate */ 4142 if (adapter->flags2 & FLAG2_HAS_EEE) { 4143 s32 ret_val; 4144 u16 adv_addr; 4145 4146 switch (hw->phy.type) { 4147 case e1000_phy_82579: 4148 adv_addr = I82579_EEE_ADVERTISEMENT; 4149 break; 4150 case e1000_phy_i217: 4151 adv_addr = I217_EEE_ADVERTISEMENT; 4152 break; 4153 default: 4154 dev_err(&adapter->pdev->dev, 4155 "Invalid PHY type setting EEE advertisement\n"); 4156 return; 4157 } 4158 4159 ret_val = hw->phy.ops.acquire(hw); 4160 if (ret_val) { 4161 dev_err(&adapter->pdev->dev, 4162 "EEE advertisement - unable to acquire PHY\n"); 4163 return; 4164 } 4165 4166 e1000_write_emi_reg_locked(hw, adv_addr, 4167 hw->dev_spec.ich8lan.eee_disable ? 4168 0 : adapter->eee_advert); 4169 4170 hw->phy.ops.release(hw); 4171 } 4172 4173 if (!netif_running(adapter->netdev) && 4174 !test_bit(__E1000_TESTING, &adapter->state)) 4175 e1000_power_down_phy(adapter); 4176 4177 e1000_get_phy_info(hw); 4178 4179 if ((adapter->flags & FLAG_HAS_SMART_POWER_DOWN) && 4180 !(adapter->flags & FLAG_SMART_POWER_DOWN)) { 4181 u16 phy_data = 0; 4182 /* speed up time to link by disabling smart power down, ignore 4183 * the return value of this function because there is nothing 4184 * different we would do if it failed 4185 */ 4186 e1e_rphy(hw, IGP02E1000_PHY_POWER_MGMT, &phy_data); 4187 phy_data &= ~IGP02E1000_PM_SPD; 4188 e1e_wphy(hw, IGP02E1000_PHY_POWER_MGMT, phy_data); 4189 } 4190 if (hw->mac.type >= e1000_pch_spt && adapter->int_mode == 0) { 4191 u32 reg; 4192 4193 /* Fextnvm7 @ 0xe4[2] = 1 */ 4194 reg = er32(FEXTNVM7); 4195 reg |= E1000_FEXTNVM7_SIDE_CLK_UNGATE; 4196 ew32(FEXTNVM7, reg); 4197 /* Fextnvm9 @ 0x5bb4[13:12] = 11 */ 4198 reg = er32(FEXTNVM9); 4199 reg |= E1000_FEXTNVM9_IOSFSB_CLKGATE_DIS | 4200 E1000_FEXTNVM9_IOSFSB_CLKREQ_DIS; 4201 ew32(FEXTNVM9, reg); 4202 } 4203 4204} 4205 4206/** 4207 * e1000e_trigger_lsc - trigger an LSC interrupt 4208 * @adapter: 4209 * 4210 * Fire a link status change interrupt to start the watchdog. 4211 **/ 4212static void e1000e_trigger_lsc(struct e1000_adapter *adapter) 4213{ 4214 struct e1000_hw *hw = &adapter->hw; 4215 4216 if (adapter->msix_entries) 4217 ew32(ICS, E1000_ICS_LSC | E1000_ICS_OTHER); 4218 else 4219 ew32(ICS, E1000_ICS_LSC); 4220} 4221 4222void e1000e_up(struct e1000_adapter *adapter) 4223{ 4224 /* hardware has been reset, we need to reload some things */ 4225 e1000_configure(adapter); 4226 4227 clear_bit(__E1000_DOWN, &adapter->state); 4228 4229 if (adapter->msix_entries) 4230 e1000_configure_msix(adapter); 4231 e1000_irq_enable(adapter); 4232 4233 /* Tx queue started by watchdog timer when link is up */ 4234 4235 e1000e_trigger_lsc(adapter); 4236} 4237 4238static void e1000e_flush_descriptors(struct e1000_adapter *adapter) 4239{ 4240 struct e1000_hw *hw = &adapter->hw; 4241 4242 if (!(adapter->flags2 & FLAG2_DMA_BURST)) 4243 return; 4244 4245 /* flush pending descriptor writebacks to memory */ 4246 ew32(TIDV, adapter->tx_int_delay | E1000_TIDV_FPD); 4247 ew32(RDTR, adapter->rx_int_delay | E1000_RDTR_FPD); 4248 4249 /* execute the writes immediately */ 4250 e1e_flush(); 4251 4252 /* due to rare timing issues, write to TIDV/RDTR again to ensure the 4253 * write is successful 4254 */ 4255 ew32(TIDV, adapter->tx_int_delay | E1000_TIDV_FPD); 4256 ew32(RDTR, adapter->rx_int_delay | E1000_RDTR_FPD); 4257 4258 /* execute the writes immediately */ 4259 e1e_flush(); 4260} 4261 4262static void e1000e_update_stats(struct e1000_adapter *adapter); 4263 4264/** 4265 * e1000e_down - quiesce the device and optionally reset the hardware 4266 * @adapter: board private structure 4267 * @reset: boolean flag to reset the hardware or not 4268 */ 4269void e1000e_down(struct e1000_adapter *adapter, bool reset) 4270{ 4271 struct net_device *netdev = adapter->netdev; 4272 struct e1000_hw *hw = &adapter->hw; 4273 u32 tctl, rctl; 4274 4275 /* signal that we're down so the interrupt handler does not 4276 * reschedule our watchdog timer 4277 */ 4278 set_bit(__E1000_DOWN, &adapter->state); 4279 4280 netif_carrier_off(netdev); 4281 4282 /* disable receives in the hardware */ 4283 rctl = er32(RCTL); 4284 if (!(adapter->flags2 & FLAG2_NO_DISABLE_RX)) 4285 ew32(RCTL, rctl & ~E1000_RCTL_EN); 4286 /* flush and sleep below */ 4287 4288 netif_stop_queue(netdev); 4289 4290 /* disable transmits in the hardware */ 4291 tctl = er32(TCTL); 4292 tctl &= ~E1000_TCTL_EN; 4293 ew32(TCTL, tctl); 4294 4295 /* flush both disables and wait for them to finish */ 4296 e1e_flush(); 4297 usleep_range(10000, 11000); 4298 4299 e1000_irq_disable(adapter); 4300 4301 napi_synchronize(&adapter->napi); 4302 4303 del_timer_sync(&adapter->watchdog_timer); 4304 del_timer_sync(&adapter->phy_info_timer); 4305 4306 spin_lock(&adapter->stats64_lock); 4307 e1000e_update_stats(adapter); 4308 spin_unlock(&adapter->stats64_lock); 4309 4310 e1000e_flush_descriptors(adapter); 4311 4312 adapter->link_speed = 0; 4313 adapter->link_duplex = 0; 4314 4315 /* Disable Si errata workaround on PCHx for jumbo frame flow */ 4316 if ((hw->mac.type >= e1000_pch2lan) && 4317 (adapter->netdev->mtu > ETH_DATA_LEN) && 4318 e1000_lv_jumbo_workaround_ich8lan(hw, false)) 4319 e_dbg("failed to disable jumbo frame workaround mode\n"); 4320 4321 if (!pci_channel_offline(adapter->pdev)) { 4322 if (reset) 4323 e1000e_reset(adapter); 4324 else if (hw->mac.type >= e1000_pch_spt) 4325 e1000_flush_desc_rings(adapter); 4326 } 4327 e1000_clean_tx_ring(adapter->tx_ring); 4328 e1000_clean_rx_ring(adapter->rx_ring); 4329} 4330 4331void e1000e_reinit_locked(struct e1000_adapter *adapter) 4332{ 4333 might_sleep(); 4334 while (test_and_set_bit(__E1000_RESETTING, &adapter->state)) 4335 usleep_range(1000, 1100); 4336 e1000e_down(adapter, true); 4337 e1000e_up(adapter); 4338 clear_bit(__E1000_RESETTING, &adapter->state); 4339} 4340 4341/** 4342 * e1000e_sanitize_systim - sanitize raw cycle counter reads 4343 * @hw: pointer to the HW structure 4344 * @systim: PHC time value read, sanitized and returned 4345 * @sts: structure to hold system time before and after reading SYSTIML, 4346 * may be NULL 4347 * 4348 * Errata for 82574/82583 possible bad bits read from SYSTIMH/L: 4349 * check to see that the time is incrementing at a reasonable 4350 * rate and is a multiple of incvalue. 4351 **/ 4352static u64 e1000e_sanitize_systim(struct e1000_hw *hw, u64 systim, 4353 struct ptp_system_timestamp *sts) 4354{ 4355 u64 time_delta, rem, temp; 4356 u64 systim_next; 4357 u32 incvalue; 4358 int i; 4359 4360 incvalue = er32(TIMINCA) & E1000_TIMINCA_INCVALUE_MASK; 4361 for (i = 0; i < E1000_MAX_82574_SYSTIM_REREADS; i++) { 4362 /* latch SYSTIMH on read of SYSTIML */ 4363 ptp_read_system_prets(sts); 4364 systim_next = (u64)er32(SYSTIML); 4365 ptp_read_system_postts(sts); 4366 systim_next |= (u64)er32(SYSTIMH) << 32; 4367 4368 time_delta = systim_next - systim; 4369 temp = time_delta; 4370 /* VMWare users have seen incvalue of zero, don't div / 0 */ 4371 rem = incvalue ? do_div(temp, incvalue) : (time_delta != 0); 4372 4373 systim = systim_next; 4374 4375 if ((time_delta < E1000_82574_SYSTIM_EPSILON) && (rem == 0)) 4376 break; 4377 } 4378 4379 return systim; 4380} 4381 4382/** 4383 * e1000e_read_systim - read SYSTIM register 4384 * @adapter: board private structure 4385 * @sts: structure which will contain system time before and after reading 4386 * SYSTIML, may be NULL 4387 **/ 4388u64 e1000e_read_systim(struct e1000_adapter *adapter, 4389 struct ptp_system_timestamp *sts) 4390{ 4391 struct e1000_hw *hw = &adapter->hw; 4392 u32 systimel, systimel_2, systimeh; 4393 u64 systim; 4394 /* SYSTIMH latching upon SYSTIML read does not work well. 4395 * This means that if SYSTIML overflows after we read it but before 4396 * we read SYSTIMH, the value of SYSTIMH has been incremented and we 4397 * will experience a huge non linear increment in the systime value 4398 * to fix that we test for overflow and if true, we re-read systime. 4399 */ 4400 ptp_read_system_prets(sts); 4401 systimel = er32(SYSTIML); 4402 ptp_read_system_postts(sts); 4403 systimeh = er32(SYSTIMH); 4404 /* Is systimel is so large that overflow is possible? */ 4405 if (systimel >= (u32)0xffffffff - E1000_TIMINCA_INCVALUE_MASK) { 4406 ptp_read_system_prets(sts); 4407 systimel_2 = er32(SYSTIML); 4408 ptp_read_system_postts(sts); 4409 if (systimel > systimel_2) { 4410 /* There was an overflow, read again SYSTIMH, and use 4411 * systimel_2 4412 */ 4413 systimeh = er32(SYSTIMH); 4414 systimel = systimel_2; 4415 } 4416 } 4417 systim = (u64)systimel; 4418 systim |= (u64)systimeh << 32; 4419 4420 if (adapter->flags2 & FLAG2_CHECK_SYSTIM_OVERFLOW) 4421 systim = e1000e_sanitize_systim(hw, systim, sts); 4422 4423 return systim; 4424} 4425 4426/** 4427 * e1000e_cyclecounter_read - read raw cycle counter (used by time counter) 4428 * @cc: cyclecounter structure 4429 **/ 4430static u64 e1000e_cyclecounter_read(const struct cyclecounter *cc) 4431{ 4432 struct e1000_adapter *adapter = container_of(cc, struct e1000_adapter, 4433 cc); 4434 4435 return e1000e_read_systim(adapter, NULL); 4436} 4437 4438/** 4439 * e1000_sw_init - Initialize general software structures (struct e1000_adapter) 4440 * @adapter: board private structure to initialize 4441 * 4442 * e1000_sw_init initializes the Adapter private data structure. 4443 * Fields are initialized based on PCI device information and 4444 * OS network device settings (MTU size). 4445 **/ 4446static int e1000_sw_init(struct e1000_adapter *adapter) 4447{ 4448 struct net_device *netdev = adapter->netdev; 4449 4450 adapter->rx_buffer_len = VLAN_ETH_FRAME_LEN + ETH_FCS_LEN; 4451 adapter->rx_ps_bsize0 = 128; 4452 adapter->max_frame_size = netdev->mtu + VLAN_ETH_HLEN + ETH_FCS_LEN; 4453 adapter->min_frame_size = ETH_ZLEN + ETH_FCS_LEN; 4454 adapter->tx_ring_count = E1000_DEFAULT_TXD; 4455 adapter->rx_ring_count = E1000_DEFAULT_RXD; 4456 4457 spin_lock_init(&adapter->stats64_lock); 4458 4459 e1000e_set_interrupt_capability(adapter); 4460 4461 if (e1000_alloc_queues(adapter)) 4462 return -ENOMEM; 4463 4464 /* Setup hardware time stamping cyclecounter */ 4465 if (adapter->flags & FLAG_HAS_HW_TIMESTAMP) { 4466 adapter->cc.read = e1000e_cyclecounter_read; 4467 adapter->cc.mask = CYCLECOUNTER_MASK(64); 4468 adapter->cc.mult = 1; 4469 /* cc.shift set in e1000e_get_base_tininca() */ 4470 4471 spin_lock_init(&adapter->systim_lock); 4472 INIT_WORK(&adapter->tx_hwtstamp_work, e1000e_tx_hwtstamp_work); 4473 } 4474 4475 /* Explicitly disable IRQ since the NIC can be in any state. */ 4476 e1000_irq_disable(adapter); 4477 4478 set_bit(__E1000_DOWN, &adapter->state); 4479 return 0; 4480} 4481 4482/** 4483 * e1000_intr_msi_test - Interrupt Handler 4484 * @irq: interrupt number 4485 * @data: pointer to a network interface device structure 4486 **/ 4487static irqreturn_t e1000_intr_msi_test(int __always_unused irq, void *data) 4488{ 4489 struct net_device *netdev = data; 4490 struct e1000_adapter *adapter = netdev_priv(netdev); 4491 struct e1000_hw *hw = &adapter->hw; 4492 u32 icr = er32(ICR); 4493 4494 e_dbg("icr is %08X\n", icr); 4495 if (icr & E1000_ICR_RXSEQ) { 4496 adapter->flags &= ~FLAG_MSI_TEST_FAILED; 4497 /* Force memory writes to complete before acknowledging the 4498 * interrupt is handled. 4499 */ 4500 wmb(); 4501 } 4502 4503 return IRQ_HANDLED; 4504} 4505 4506/** 4507 * e1000_test_msi_interrupt - Returns 0 for successful test 4508 * @adapter: board private struct 4509 * 4510 * code flow taken from tg3.c 4511 **/ 4512static int e1000_test_msi_interrupt(struct e1000_adapter *adapter) 4513{ 4514 struct net_device *netdev = adapter->netdev; 4515 struct e1000_hw *hw = &adapter->hw; 4516 int err; 4517 4518 /* poll_enable hasn't been called yet, so don't need disable */ 4519 /* clear any pending events */ 4520 er32(ICR); 4521 4522 /* free the real vector and request a test handler */ 4523 e1000_free_irq(adapter); 4524 e1000e_reset_interrupt_capability(adapter); 4525 4526 /* Assume that the test fails, if it succeeds then the test 4527 * MSI irq handler will unset this flag 4528 */ 4529 adapter->flags |= FLAG_MSI_TEST_FAILED; 4530 4531 err = pci_enable_msi(adapter->pdev); 4532 if (err) 4533 goto msi_test_failed; 4534 4535 err = request_irq(adapter->pdev->irq, e1000_intr_msi_test, 0, 4536 netdev->name, netdev); 4537 if (err) { 4538 pci_disable_msi(adapter->pdev); 4539 goto msi_test_failed; 4540 } 4541 4542 /* Force memory writes to complete before enabling and firing an 4543 * interrupt. 4544 */ 4545 wmb(); 4546 4547 e1000_irq_enable(adapter); 4548 4549 /* fire an unusual interrupt on the test handler */ 4550 ew32(ICS, E1000_ICS_RXSEQ); 4551 e1e_flush(); 4552 msleep(100); 4553 4554 e1000_irq_disable(adapter); 4555 4556 rmb(); /* read flags after interrupt has been fired */ 4557 4558 if (adapter->flags & FLAG_MSI_TEST_FAILED) { 4559 adapter->int_mode = E1000E_INT_MODE_LEGACY; 4560 e_info("MSI interrupt test failed, using legacy interrupt.\n"); 4561 } else { 4562 e_dbg("MSI interrupt test succeeded!\n"); 4563 } 4564 4565 free_irq(adapter->pdev->irq, netdev); 4566 pci_disable_msi(adapter->pdev); 4567 4568msi_test_failed: 4569 e1000e_set_interrupt_capability(adapter); 4570 return e1000_request_irq(adapter); 4571} 4572 4573/** 4574 * e1000_test_msi - Returns 0 if MSI test succeeds or INTx mode is restored 4575 * @adapter: board private struct 4576 * 4577 * code flow taken from tg3.c, called with e1000 interrupts disabled. 4578 **/ 4579static int e1000_test_msi(struct e1000_adapter *adapter) 4580{ 4581 int err; 4582 u16 pci_cmd; 4583 4584 if (!(adapter->flags & FLAG_MSI_ENABLED)) 4585 return 0; 4586 4587 /* disable SERR in case the MSI write causes a master abort */ 4588 pci_read_config_word(adapter->pdev, PCI_COMMAND, &pci_cmd); 4589 if (pci_cmd & PCI_COMMAND_SERR) 4590 pci_write_config_word(adapter->pdev, PCI_COMMAND, 4591 pci_cmd & ~PCI_COMMAND_SERR); 4592 4593 err = e1000_test_msi_interrupt(adapter); 4594 4595 /* re-enable SERR */ 4596 if (pci_cmd & PCI_COMMAND_SERR) { 4597 pci_read_config_word(adapter->pdev, PCI_COMMAND, &pci_cmd); 4598 pci_cmd |= PCI_COMMAND_SERR; 4599 pci_write_config_word(adapter->pdev, PCI_COMMAND, pci_cmd); 4600 } 4601 4602 return err; 4603} 4604 4605/** 4606 * e1000e_open - Called when a network interface is made active 4607 * @netdev: network interface device structure 4608 * 4609 * Returns 0 on success, negative value on failure 4610 * 4611 * The open entry point is called when a network interface is made 4612 * active by the system (IFF_UP). At this point all resources needed 4613 * for transmit and receive operations are allocated, the interrupt 4614 * handler is registered with the OS, the watchdog timer is started, 4615 * and the stack is notified that the interface is ready. 4616 **/ 4617int e1000e_open(struct net_device *netdev) 4618{ 4619 struct e1000_adapter *adapter = netdev_priv(netdev); 4620 struct e1000_hw *hw = &adapter->hw; 4621 struct pci_dev *pdev = adapter->pdev; 4622 int err; 4623 4624 /* disallow open during test */ 4625 if (test_bit(__E1000_TESTING, &adapter->state)) 4626 return -EBUSY; 4627 4628 pm_runtime_get_sync(&pdev->dev); 4629 4630 netif_carrier_off(netdev); 4631 netif_stop_queue(netdev); 4632 4633 /* allocate transmit descriptors */ 4634 err = e1000e_setup_tx_resources(adapter->tx_ring); 4635 if (err) 4636 goto err_setup_tx; 4637 4638 /* allocate receive descriptors */ 4639 err = e1000e_setup_rx_resources(adapter->rx_ring); 4640 if (err) 4641 goto err_setup_rx; 4642 4643 /* If AMT is enabled, let the firmware know that the network 4644 * interface is now open and reset the part to a known state. 4645 */ 4646 if (adapter->flags & FLAG_HAS_AMT) { 4647 e1000e_get_hw_control(adapter); 4648 e1000e_reset(adapter); 4649 } 4650 4651 e1000e_power_up_phy(adapter); 4652 4653 adapter->mng_vlan_id = E1000_MNG_VLAN_NONE; 4654 if ((adapter->hw.mng_cookie.status & E1000_MNG_DHCP_COOKIE_STATUS_VLAN)) 4655 e1000_update_mng_vlan(adapter); 4656 4657 /* DMA latency requirement to workaround jumbo issue */ 4658 cpu_latency_qos_add_request(&adapter->pm_qos_req, PM_QOS_DEFAULT_VALUE); 4659 4660 /* before we allocate an interrupt, we must be ready to handle it. 4661 * Setting DEBUG_SHIRQ in the kernel makes it fire an interrupt 4662 * as soon as we call pci_request_irq, so we have to setup our 4663 * clean_rx handler before we do so. 4664 */ 4665 e1000_configure(adapter); 4666 4667 err = e1000_request_irq(adapter); 4668 if (err) 4669 goto err_req_irq; 4670 4671 /* Work around PCIe errata with MSI interrupts causing some chipsets to 4672 * ignore e1000e MSI messages, which means we need to test our MSI 4673 * interrupt now 4674 */ 4675 if (adapter->int_mode != E1000E_INT_MODE_LEGACY) { 4676 err = e1000_test_msi(adapter); 4677 if (err) { 4678 e_err("Interrupt allocation failed\n"); 4679 goto err_req_irq; 4680 } 4681 } 4682 4683 /* From here on the code is the same as e1000e_up() */ 4684 clear_bit(__E1000_DOWN, &adapter->state); 4685 4686 napi_enable(&adapter->napi); 4687 4688 e1000_irq_enable(adapter); 4689 4690 adapter->tx_hang_recheck = false; 4691 4692 hw->mac.get_link_status = true; 4693 pm_runtime_put(&pdev->dev); 4694 4695 e1000e_trigger_lsc(adapter); 4696 4697 return 0; 4698 4699err_req_irq: 4700 cpu_latency_qos_remove_request(&adapter->pm_qos_req); 4701 e1000e_release_hw_control(adapter); 4702 e1000_power_down_phy(adapter); 4703 e1000e_free_rx_resources(adapter->rx_ring); 4704err_setup_rx: 4705 e1000e_free_tx_resources(adapter->tx_ring); 4706err_setup_tx: 4707 e1000e_reset(adapter); 4708 pm_runtime_put_sync(&pdev->dev); 4709 4710 return err; 4711} 4712 4713/** 4714 * e1000e_close - Disables a network interface 4715 * @netdev: network interface device structure 4716 * 4717 * Returns 0, this is not allowed to fail 4718 * 4719 * The close entry point is called when an interface is de-activated 4720 * by the OS. The hardware is still under the drivers control, but 4721 * needs to be disabled. A global MAC reset is issued to stop the 4722 * hardware, and all transmit and receive resources are freed. 4723 **/ 4724int e1000e_close(struct net_device *netdev) 4725{ 4726 struct e1000_adapter *adapter = netdev_priv(netdev); 4727 struct pci_dev *pdev = adapter->pdev; 4728 int count = E1000_CHECK_RESET_COUNT; 4729 4730 while (test_bit(__E1000_RESETTING, &adapter->state) && count--) 4731 usleep_range(10000, 11000); 4732 4733 WARN_ON(test_bit(__E1000_RESETTING, &adapter->state)); 4734 4735 pm_runtime_get_sync(&pdev->dev); 4736 4737 if (netif_device_present(netdev)) { 4738 e1000e_down(adapter, true); 4739 e1000_free_irq(adapter); 4740 4741 /* Link status message must follow this format */ 4742 netdev_info(netdev, "NIC Link is Down\n"); 4743 } 4744 4745 napi_disable(&adapter->napi); 4746 4747 e1000e_free_tx_resources(adapter->tx_ring); 4748 e1000e_free_rx_resources(adapter->rx_ring); 4749 4750 /* kill manageability vlan ID if supported, but not if a vlan with 4751 * the same ID is registered on the host OS (let 8021q kill it) 4752 */ 4753 if (adapter->hw.mng_cookie.status & E1000_MNG_DHCP_COOKIE_STATUS_VLAN) 4754 e1000_vlan_rx_kill_vid(netdev, htons(ETH_P_8021Q), 4755 adapter->mng_vlan_id); 4756 4757 /* If AMT is enabled, let the firmware know that the network 4758 * interface is now closed 4759 */ 4760 if ((adapter->flags & FLAG_HAS_AMT) && 4761 !test_bit(__E1000_TESTING, &adapter->state)) 4762 e1000e_release_hw_control(adapter); 4763 4764 cpu_latency_qos_remove_request(&adapter->pm_qos_req); 4765 4766 pm_runtime_put_sync(&pdev->dev); 4767 4768 return 0; 4769} 4770 4771/** 4772 * e1000_set_mac - Change the Ethernet Address of the NIC 4773 * @netdev: network interface device structure 4774 * @p: pointer to an address structure 4775 * 4776 * Returns 0 on success, negative on failure 4777 **/ 4778static int e1000_set_mac(struct net_device *netdev, void *p) 4779{ 4780 struct e1000_adapter *adapter = netdev_priv(netdev); 4781 struct e1000_hw *hw = &adapter->hw; 4782 struct sockaddr *addr = p; 4783 4784 if (!is_valid_ether_addr(addr->sa_data)) 4785 return -EADDRNOTAVAIL; 4786 4787 memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len); 4788 memcpy(adapter->hw.mac.addr, addr->sa_data, netdev->addr_len); 4789 4790 hw->mac.ops.rar_set(&adapter->hw, adapter->hw.mac.addr, 0); 4791 4792 if (adapter->flags & FLAG_RESET_OVERWRITES_LAA) { 4793 /* activate the work around */ 4794 e1000e_set_laa_state_82571(&adapter->hw, 1); 4795 4796 /* Hold a copy of the LAA in RAR[14] This is done so that 4797 * between the time RAR[0] gets clobbered and the time it 4798 * gets fixed (in e1000_watchdog), the actual LAA is in one 4799 * of the RARs and no incoming packets directed to this port 4800 * are dropped. Eventually the LAA will be in RAR[0] and 4801 * RAR[14] 4802 */ 4803 hw->mac.ops.rar_set(&adapter->hw, adapter->hw.mac.addr, 4804 adapter->hw.mac.rar_entry_count - 1); 4805 } 4806 4807 return 0; 4808} 4809 4810/** 4811 * e1000e_update_phy_task - work thread to update phy 4812 * @work: pointer to our work struct 4813 * 4814 * this worker thread exists because we must acquire a 4815 * semaphore to read the phy, which we could msleep while 4816 * waiting for it, and we can't msleep in a timer. 4817 **/ 4818static void e1000e_update_phy_task(struct work_struct *work) 4819{ 4820 struct e1000_adapter *adapter = container_of(work, 4821 struct e1000_adapter, 4822 update_phy_task); 4823 struct e1000_hw *hw = &adapter->hw; 4824 4825 if (test_bit(__E1000_DOWN, &adapter->state)) 4826 return; 4827 4828 e1000_get_phy_info(hw); 4829 4830 /* Enable EEE on 82579 after link up */ 4831 if (hw->phy.type >= e1000_phy_82579) 4832 e1000_set_eee_pchlan(hw); 4833} 4834 4835/** 4836 * e1000_update_phy_info - timre call-back to update PHY info 4837 * @t: pointer to timer_list containing private info adapter 4838 * 4839 * Need to wait a few seconds after link up to get diagnostic information from 4840 * the phy 4841 **/ 4842static void e1000_update_phy_info(struct timer_list *t) 4843{ 4844 struct e1000_adapter *adapter = from_timer(adapter, t, phy_info_timer); 4845 4846 if (test_bit(__E1000_DOWN, &adapter->state)) 4847 return; 4848 4849 schedule_work(&adapter->update_phy_task); 4850} 4851 4852/** 4853 * e1000e_update_phy_stats - Update the PHY statistics counters 4854 * @adapter: board private structure 4855 * 4856 * Read/clear the upper 16-bit PHY registers and read/accumulate lower 4857 **/ 4858static void e1000e_update_phy_stats(struct e1000_adapter *adapter) 4859{ 4860 struct e1000_hw *hw = &adapter->hw; 4861 s32 ret_val; 4862 u16 phy_data; 4863 4864 ret_val = hw->phy.ops.acquire(hw); 4865 if (ret_val) 4866 return; 4867 4868 /* A page set is expensive so check if already on desired page. 4869 * If not, set to the page with the PHY status registers. 4870 */ 4871 hw->phy.addr = 1; 4872 ret_val = e1000e_read_phy_reg_mdic(hw, IGP01E1000_PHY_PAGE_SELECT, 4873 &phy_data); 4874 if (ret_val) 4875 goto release; 4876 if (phy_data != (HV_STATS_PAGE << IGP_PAGE_SHIFT)) { 4877 ret_val = hw->phy.ops.set_page(hw, 4878 HV_STATS_PAGE << IGP_PAGE_SHIFT); 4879 if (ret_val) 4880 goto release; 4881 } 4882 4883 /* Single Collision Count */ 4884 hw->phy.ops.read_reg_page(hw, HV_SCC_UPPER, &phy_data); 4885 ret_val = hw->phy.ops.read_reg_page(hw, HV_SCC_LOWER, &phy_data); 4886 if (!ret_val) 4887 adapter->stats.scc += phy_data; 4888 4889 /* Excessive Collision Count */ 4890 hw->phy.ops.read_reg_page(hw, HV_ECOL_UPPER, &phy_data); 4891 ret_val = hw->phy.ops.read_reg_page(hw, HV_ECOL_LOWER, &phy_data); 4892 if (!ret_val) 4893 adapter->stats.ecol += phy_data; 4894 4895 /* Multiple Collision Count */ 4896 hw->phy.ops.read_reg_page(hw, HV_MCC_UPPER, &phy_data); 4897 ret_val = hw->phy.ops.read_reg_page(hw, HV_MCC_LOWER, &phy_data); 4898 if (!ret_val) 4899 adapter->stats.mcc += phy_data; 4900 4901 /* Late Collision Count */ 4902 hw->phy.ops.read_reg_page(hw, HV_LATECOL_UPPER, &phy_data); 4903 ret_val = hw->phy.ops.read_reg_page(hw, HV_LATECOL_LOWER, &phy_data); 4904 if (!ret_val) 4905 adapter->stats.latecol += phy_data; 4906 4907 /* Collision Count - also used for adaptive IFS */ 4908 hw->phy.ops.read_reg_page(hw, HV_COLC_UPPER, &phy_data); 4909 ret_val = hw->phy.ops.read_reg_page(hw, HV_COLC_LOWER, &phy_data); 4910 if (!ret_val) 4911 hw->mac.collision_delta = phy_data; 4912 4913 /* Defer Count */ 4914 hw->phy.ops.read_reg_page(hw, HV_DC_UPPER, &phy_data); 4915 ret_val = hw->phy.ops.read_reg_page(hw, HV_DC_LOWER, &phy_data); 4916 if (!ret_val) 4917 adapter->stats.dc += phy_data; 4918 4919 /* Transmit with no CRS */ 4920 hw->phy.ops.read_reg_page(hw, HV_TNCRS_UPPER, &phy_data); 4921 ret_val = hw->phy.ops.read_reg_page(hw, HV_TNCRS_LOWER, &phy_data); 4922 if (!ret_val) 4923 adapter->stats.tncrs += phy_data; 4924 4925release: 4926 hw->phy.ops.release(hw); 4927} 4928 4929/** 4930 * e1000e_update_stats - Update the board statistics counters 4931 * @adapter: board private structure 4932 **/ 4933static void e1000e_update_stats(struct e1000_adapter *adapter) 4934{ 4935 struct net_device *netdev = adapter->netdev; 4936 struct e1000_hw *hw = &adapter->hw; 4937 struct pci_dev *pdev = adapter->pdev; 4938 4939 /* Prevent stats update while adapter is being reset, or if the pci 4940 * connection is down. 4941 */ 4942 if (adapter->link_speed == 0) 4943 return; 4944 if (pci_channel_offline(pdev)) 4945 return; 4946 4947 adapter->stats.crcerrs += er32(CRCERRS); 4948 adapter->stats.gprc += er32(GPRC); 4949 adapter->stats.gorc += er32(GORCL); 4950 er32(GORCH); /* Clear gorc */ 4951 adapter->stats.bprc += er32(BPRC); 4952 adapter->stats.mprc += er32(MPRC); 4953 adapter->stats.roc += er32(ROC); 4954 4955 adapter->stats.mpc += er32(MPC); 4956 4957 /* Half-duplex statistics */ 4958 if (adapter->link_duplex == HALF_DUPLEX) { 4959 if (adapter->flags2 & FLAG2_HAS_PHY_STATS) { 4960 e1000e_update_phy_stats(adapter); 4961 } else { 4962 adapter->stats.scc += er32(SCC); 4963 adapter->stats.ecol += er32(ECOL); 4964 adapter->stats.mcc += er32(MCC); 4965 adapter->stats.latecol += er32(LATECOL); 4966 adapter->stats.dc += er32(DC); 4967 4968 hw->mac.collision_delta = er32(COLC); 4969 4970 if ((hw->mac.type != e1000_82574) && 4971 (hw->mac.type != e1000_82583)) 4972 adapter->stats.tncrs += er32(TNCRS); 4973 } 4974 adapter->stats.colc += hw->mac.collision_delta; 4975 } 4976 4977 adapter->stats.xonrxc += er32(XONRXC); 4978 adapter->stats.xontxc += er32(XONTXC); 4979 adapter->stats.xoffrxc += er32(XOFFRXC); 4980 adapter->stats.xofftxc += er32(XOFFTXC); 4981 adapter->stats.gptc += er32(GPTC); 4982 adapter->stats.gotc += er32(GOTCL); 4983 er32(GOTCH); /* Clear gotc */ 4984 adapter->stats.rnbc += er32(RNBC); 4985 adapter->stats.ruc += er32(RUC); 4986 4987 adapter->stats.mptc += er32(MPTC); 4988 adapter->stats.bptc += er32(BPTC); 4989 4990 /* used for adaptive IFS */ 4991 4992 hw->mac.tx_packet_delta = er32(TPT); 4993 adapter->stats.tpt += hw->mac.tx_packet_delta; 4994 4995 adapter->stats.algnerrc += er32(ALGNERRC); 4996 adapter->stats.rxerrc += er32(RXERRC); 4997 adapter->stats.cexterr += er32(CEXTERR); 4998 adapter->stats.tsctc += er32(TSCTC); 4999 adapter->stats.tsctfc += er32(TSCTFC); 5000 5001 /* Fill out the OS statistics structure */ 5002 netdev->stats.multicast = adapter->stats.mprc; 5003 netdev->stats.collisions = adapter->stats.colc; 5004 5005 /* Rx Errors */ 5006 5007 /* RLEC on some newer hardware can be incorrect so build 5008 * our own version based on RUC and ROC 5009 */ 5010 netdev->stats.rx_errors = adapter->stats.rxerrc + 5011 adapter->stats.crcerrs + adapter->stats.algnerrc + 5012 adapter->stats.ruc + adapter->stats.roc + adapter->stats.cexterr; 5013 netdev->stats.rx_length_errors = adapter->stats.ruc + 5014 adapter->stats.roc; 5015 netdev->stats.rx_crc_errors = adapter->stats.crcerrs; 5016 netdev->stats.rx_frame_errors = adapter->stats.algnerrc; 5017 netdev->stats.rx_missed_errors = adapter->stats.mpc; 5018 5019 /* Tx Errors */ 5020 netdev->stats.tx_errors = adapter->stats.ecol + adapter->stats.latecol; 5021 netdev->stats.tx_aborted_errors = adapter->stats.ecol; 5022 netdev->stats.tx_window_errors = adapter->stats.latecol; 5023 netdev->stats.tx_carrier_errors = adapter->stats.tncrs; 5024 5025 /* Tx Dropped needs to be maintained elsewhere */ 5026 5027 /* Management Stats */ 5028 adapter->stats.mgptc += er32(MGTPTC); 5029 adapter->stats.mgprc += er32(MGTPRC); 5030 adapter->stats.mgpdc += er32(MGTPDC); 5031 5032 /* Correctable ECC Errors */ 5033 if (hw->mac.type >= e1000_pch_lpt) { 5034 u32 pbeccsts = er32(PBECCSTS); 5035 5036 adapter->corr_errors += 5037 pbeccsts & E1000_PBECCSTS_CORR_ERR_CNT_MASK; 5038 adapter->uncorr_errors += 5039 (pbeccsts & E1000_PBECCSTS_UNCORR_ERR_CNT_MASK) >> 5040 E1000_PBECCSTS_UNCORR_ERR_CNT_SHIFT; 5041 } 5042} 5043 5044/** 5045 * e1000_phy_read_status - Update the PHY register status snapshot 5046 * @adapter: board private structure 5047 **/ 5048static void e1000_phy_read_status(struct e1000_adapter *adapter) 5049{ 5050 struct e1000_hw *hw = &adapter->hw; 5051 struct e1000_phy_regs *phy = &adapter->phy_regs; 5052 5053 if (!pm_runtime_suspended((&adapter->pdev->dev)->parent) && 5054 (er32(STATUS) & E1000_STATUS_LU) && 5055 (adapter->hw.phy.media_type == e1000_media_type_copper)) { 5056 int ret_val; 5057 5058 ret_val = e1e_rphy(hw, MII_BMCR, &phy->bmcr); 5059 ret_val |= e1e_rphy(hw, MII_BMSR, &phy->bmsr); 5060 ret_val |= e1e_rphy(hw, MII_ADVERTISE, &phy->advertise); 5061 ret_val |= e1e_rphy(hw, MII_LPA, &phy->lpa); 5062 ret_val |= e1e_rphy(hw, MII_EXPANSION, &phy->expansion); 5063 ret_val |= e1e_rphy(hw, MII_CTRL1000, &phy->ctrl1000); 5064 ret_val |= e1e_rphy(hw, MII_STAT1000, &phy->stat1000); 5065 ret_val |= e1e_rphy(hw, MII_ESTATUS, &phy->estatus); 5066 if (ret_val) 5067 e_warn("Error reading PHY register\n"); 5068 } else { 5069 /* Do not read PHY registers if link is not up 5070 * Set values to typical power-on defaults 5071 */ 5072 phy->bmcr = (BMCR_SPEED1000 | BMCR_ANENABLE | BMCR_FULLDPLX); 5073 phy->bmsr = (BMSR_100FULL | BMSR_100HALF | BMSR_10FULL | 5074 BMSR_10HALF | BMSR_ESTATEN | BMSR_ANEGCAPABLE | 5075 BMSR_ERCAP); 5076 phy->advertise = (ADVERTISE_PAUSE_ASYM | ADVERTISE_PAUSE_CAP | 5077 ADVERTISE_ALL | ADVERTISE_CSMA); 5078 phy->lpa = 0; 5079 phy->expansion = EXPANSION_ENABLENPAGE; 5080 phy->ctrl1000 = ADVERTISE_1000FULL; 5081 phy->stat1000 = 0; 5082 phy->estatus = (ESTATUS_1000_TFULL | ESTATUS_1000_THALF); 5083 } 5084} 5085 5086static void e1000_print_link_info(struct e1000_adapter *adapter) 5087{ 5088 struct e1000_hw *hw = &adapter->hw; 5089 u32 ctrl = er32(CTRL); 5090 5091 /* Link status message must follow this format for user tools */ 5092 netdev_info(adapter->netdev, 5093 "NIC Link is Up %d Mbps %s Duplex, Flow Control: %s\n", 5094 adapter->link_speed, 5095 adapter->link_duplex == FULL_DUPLEX ? "Full" : "Half", 5096 (ctrl & E1000_CTRL_TFCE) && (ctrl & E1000_CTRL_RFCE) ? "Rx/Tx" : 5097 (ctrl & E1000_CTRL_RFCE) ? "Rx" : 5098 (ctrl & E1000_CTRL_TFCE) ? "Tx" : "None"); 5099} 5100 5101static bool e1000e_has_link(struct e1000_adapter *adapter) 5102{ 5103 struct e1000_hw *hw = &adapter->hw; 5104 bool link_active = false; 5105 s32 ret_val = 0; 5106 5107 /* get_link_status is set on LSC (link status) interrupt or 5108 * Rx sequence error interrupt. get_link_status will stay 5109 * true until the check_for_link establishes link 5110 * for copper adapters ONLY 5111 */ 5112 switch (hw->phy.media_type) { 5113 case e1000_media_type_copper: 5114 if (hw->mac.get_link_status) { 5115 ret_val = hw->mac.ops.check_for_link(hw); 5116 link_active = !hw->mac.get_link_status; 5117 } else { 5118 link_active = true; 5119 } 5120 break; 5121 case e1000_media_type_fiber: 5122 ret_val = hw->mac.ops.check_for_link(hw); 5123 link_active = !!(er32(STATUS) & E1000_STATUS_LU); 5124 break; 5125 case e1000_media_type_internal_serdes: 5126 ret_val = hw->mac.ops.check_for_link(hw); 5127 link_active = hw->mac.serdes_has_link; 5128 break; 5129 default: 5130 case e1000_media_type_unknown: 5131 break; 5132 } 5133 5134 if ((ret_val == -E1000_ERR_PHY) && (hw->phy.type == e1000_phy_igp_3) && 5135 (er32(CTRL) & E1000_PHY_CTRL_GBE_DISABLE)) { 5136 /* See e1000_kmrn_lock_loss_workaround_ich8lan() */ 5137 e_info("Gigabit has been disabled, downgrading speed\n"); 5138 } 5139 5140 return link_active; 5141} 5142 5143static void e1000e_enable_receives(struct e1000_adapter *adapter) 5144{ 5145 /* make sure the receive unit is started */ 5146 if ((adapter->flags & FLAG_RX_NEEDS_RESTART) && 5147 (adapter->flags & FLAG_RESTART_NOW)) { 5148 struct e1000_hw *hw = &adapter->hw; 5149 u32 rctl = er32(RCTL); 5150 5151 ew32(RCTL, rctl | E1000_RCTL_EN); 5152 adapter->flags &= ~FLAG_RESTART_NOW; 5153 } 5154} 5155 5156static void e1000e_check_82574_phy_workaround(struct e1000_adapter *adapter) 5157{ 5158 struct e1000_hw *hw = &adapter->hw; 5159 5160 /* With 82574 controllers, PHY needs to be checked periodically 5161 * for hung state and reset, if two calls return true 5162 */ 5163 if (e1000_check_phy_82574(hw)) 5164 adapter->phy_hang_count++; 5165 else 5166 adapter->phy_hang_count = 0; 5167 5168 if (adapter->phy_hang_count > 1) { 5169 adapter->phy_hang_count = 0; 5170 e_dbg("PHY appears hung - resetting\n"); 5171 schedule_work(&adapter->reset_task); 5172 } 5173} 5174 5175/** 5176 * e1000_watchdog - Timer Call-back 5177 * @t: pointer to timer_list containing private info adapter 5178 **/ 5179static void e1000_watchdog(struct timer_list *t) 5180{ 5181 struct e1000_adapter *adapter = from_timer(adapter, t, watchdog_timer); 5182 5183 /* Do the rest outside of interrupt context */ 5184 schedule_work(&adapter->watchdog_task); 5185 5186 /* TODO: make this use queue_delayed_work() */ 5187} 5188 5189static void e1000_watchdog_task(struct work_struct *work) 5190{ 5191 struct e1000_adapter *adapter = container_of(work, 5192 struct e1000_adapter, 5193 watchdog_task); 5194 struct net_device *netdev = adapter->netdev; 5195 struct e1000_mac_info *mac = &adapter->hw.mac; 5196 struct e1000_phy_info *phy = &adapter->hw.phy; 5197 struct e1000_ring *tx_ring = adapter->tx_ring; 5198 u32 dmoff_exit_timeout = 100, tries = 0; 5199 struct e1000_hw *hw = &adapter->hw; 5200 u32 link, tctl, pcim_state; 5201 5202 if (test_bit(__E1000_DOWN, &adapter->state)) 5203 return; 5204 5205 link = e1000e_has_link(adapter); 5206 if ((netif_carrier_ok(netdev)) && link) { 5207 /* Cancel scheduled suspend requests. */ 5208 pm_runtime_resume(netdev->dev.parent); 5209 5210 e1000e_enable_receives(adapter); 5211 goto link_up; 5212 } 5213 5214 if ((e1000e_enable_tx_pkt_filtering(hw)) && 5215 (adapter->mng_vlan_id != adapter->hw.mng_cookie.vlan_id)) 5216 e1000_update_mng_vlan(adapter); 5217 5218 if (link) { 5219 if (!netif_carrier_ok(netdev)) { 5220 bool txb2b = true; 5221 5222 /* Cancel scheduled suspend requests. */ 5223 pm_runtime_resume(netdev->dev.parent); 5224 5225 /* Checking if MAC is in DMoff state*/ 5226 if (er32(FWSM) & E1000_ICH_FWSM_FW_VALID) { 5227 pcim_state = er32(STATUS); 5228 while (pcim_state & E1000_STATUS_PCIM_STATE) { 5229 if (tries++ == dmoff_exit_timeout) { 5230 e_dbg("Error in exiting dmoff\n"); 5231 break; 5232 } 5233 usleep_range(10000, 20000); 5234 pcim_state = er32(STATUS); 5235 5236 /* Checking if MAC exited DMoff state */ 5237 if (!(pcim_state & E1000_STATUS_PCIM_STATE)) 5238 e1000_phy_hw_reset(&adapter->hw); 5239 } 5240 } 5241 5242 /* update snapshot of PHY registers on LSC */ 5243 e1000_phy_read_status(adapter); 5244 mac->ops.get_link_up_info(&adapter->hw, 5245 &adapter->link_speed, 5246 &adapter->link_duplex); 5247 e1000_print_link_info(adapter); 5248 5249 /* check if SmartSpeed worked */ 5250 e1000e_check_downshift(hw); 5251 if (phy->speed_downgraded) 5252 netdev_warn(netdev, 5253 "Link Speed was downgraded by SmartSpeed\n"); 5254 5255 /* On supported PHYs, check for duplex mismatch only 5256 * if link has autonegotiated at 10/100 half 5257 */ 5258 if ((hw->phy.type == e1000_phy_igp_3 || 5259 hw->phy.type == e1000_phy_bm) && 5260 hw->mac.autoneg && 5261 (adapter->link_speed == SPEED_10 || 5262 adapter->link_speed == SPEED_100) && 5263 (adapter->link_duplex == HALF_DUPLEX)) { 5264 u16 autoneg_exp; 5265 5266 e1e_rphy(hw, MII_EXPANSION, &autoneg_exp); 5267 5268 if (!(autoneg_exp & EXPANSION_NWAY)) 5269 e_info("Autonegotiated half duplex but link partner cannot autoneg. Try forcing full duplex if link gets many collisions.\n"); 5270 } 5271 5272 /* adjust timeout factor according to speed/duplex */ 5273 adapter->tx_timeout_factor = 1; 5274 switch (adapter->link_speed) { 5275 case SPEED_10: 5276 txb2b = false; 5277 adapter->tx_timeout_factor = 16; 5278 break; 5279 case SPEED_100: 5280 txb2b = false; 5281 adapter->tx_timeout_factor = 10; 5282 break; 5283 } 5284 5285 /* workaround: re-program speed mode bit after 5286 * link-up event 5287 */ 5288 if ((adapter->flags & FLAG_TARC_SPEED_MODE_BIT) && 5289 !txb2b) { 5290 u32 tarc0; 5291 5292 tarc0 = er32(TARC(0)); 5293 tarc0 &= ~SPEED_MODE_BIT; 5294 ew32(TARC(0), tarc0); 5295 } 5296 5297 /* enable transmits in the hardware, need to do this 5298 * after setting TARC(0) 5299 */ 5300 tctl = er32(TCTL); 5301 tctl |= E1000_TCTL_EN; 5302 ew32(TCTL, tctl); 5303 5304 /* Perform any post-link-up configuration before 5305 * reporting link up. 5306 */ 5307 if (phy->ops.cfg_on_link_up) 5308 phy->ops.cfg_on_link_up(hw); 5309 5310 netif_wake_queue(netdev); 5311 netif_carrier_on(netdev); 5312 5313 if (!test_bit(__E1000_DOWN, &adapter->state)) 5314 mod_timer(&adapter->phy_info_timer, 5315 round_jiffies(jiffies + 2 * HZ)); 5316 } 5317 } else { 5318 if (netif_carrier_ok(netdev)) { 5319 adapter->link_speed = 0; 5320 adapter->link_duplex = 0; 5321 /* Link status message must follow this format */ 5322 netdev_info(netdev, "NIC Link is Down\n"); 5323 netif_carrier_off(netdev); 5324 netif_stop_queue(netdev); 5325 if (!test_bit(__E1000_DOWN, &adapter->state)) 5326 mod_timer(&adapter->phy_info_timer, 5327 round_jiffies(jiffies + 2 * HZ)); 5328 5329 /* 8000ES2LAN requires a Rx packet buffer work-around 5330 * on link down event; reset the controller to flush 5331 * the Rx packet buffer. 5332 */ 5333 if (adapter->flags & FLAG_RX_NEEDS_RESTART) 5334 adapter->flags |= FLAG_RESTART_NOW; 5335 else 5336 pm_schedule_suspend(netdev->dev.parent, 5337 LINK_TIMEOUT); 5338 } 5339 } 5340 5341link_up: 5342 spin_lock(&adapter->stats64_lock); 5343 e1000e_update_stats(adapter); 5344 5345 mac->tx_packet_delta = adapter->stats.tpt - adapter->tpt_old; 5346 adapter->tpt_old = adapter->stats.tpt; 5347 mac->collision_delta = adapter->stats.colc - adapter->colc_old; 5348 adapter->colc_old = adapter->stats.colc; 5349 5350 adapter->gorc = adapter->stats.gorc - adapter->gorc_old; 5351 adapter->gorc_old = adapter->stats.gorc; 5352 adapter->gotc = adapter->stats.gotc - adapter->gotc_old; 5353 adapter->gotc_old = adapter->stats.gotc; 5354 spin_unlock(&adapter->stats64_lock); 5355 5356 /* If the link is lost the controller stops DMA, but 5357 * if there is queued Tx work it cannot be done. So 5358 * reset the controller to flush the Tx packet buffers. 5359 */ 5360 if (!netif_carrier_ok(netdev) && 5361 (e1000_desc_unused(tx_ring) + 1 < tx_ring->count)) 5362 adapter->flags |= FLAG_RESTART_NOW; 5363 5364 /* If reset is necessary, do it outside of interrupt context. */ 5365 if (adapter->flags & FLAG_RESTART_NOW) { 5366 schedule_work(&adapter->reset_task); 5367 /* return immediately since reset is imminent */ 5368 return; 5369 } 5370 5371 e1000e_update_adaptive(&adapter->hw); 5372 5373 /* Simple mode for Interrupt Throttle Rate (ITR) */ 5374 if (adapter->itr_setting == 4) { 5375 /* Symmetric Tx/Rx gets a reduced ITR=2000; 5376 * Total asymmetrical Tx or Rx gets ITR=8000; 5377 * everyone else is between 2000-8000. 5378 */ 5379 u32 goc = (adapter->gotc + adapter->gorc) / 10000; 5380 u32 dif = (adapter->gotc > adapter->gorc ? 5381 adapter->gotc - adapter->gorc : 5382 adapter->gorc - adapter->gotc) / 10000; 5383 u32 itr = goc > 0 ? (dif * 6000 / goc + 2000) : 8000; 5384 5385 e1000e_write_itr(adapter, itr); 5386 } 5387 5388 /* Cause software interrupt to ensure Rx ring is cleaned */ 5389 if (adapter->msix_entries) 5390 ew32(ICS, adapter->rx_ring->ims_val); 5391 else 5392 ew32(ICS, E1000_ICS_RXDMT0); 5393 5394 /* flush pending descriptors to memory before detecting Tx hang */ 5395 e1000e_flush_descriptors(adapter); 5396 5397 /* Force detection of hung controller every watchdog period */ 5398 adapter->detect_tx_hung = true; 5399 5400 /* With 82571 controllers, LAA may be overwritten due to controller 5401 * reset from the other port. Set the appropriate LAA in RAR[0] 5402 */ 5403 if (e1000e_get_laa_state_82571(hw)) 5404 hw->mac.ops.rar_set(hw, adapter->hw.mac.addr, 0); 5405 5406 if (adapter->flags2 & FLAG2_CHECK_PHY_HANG) 5407 e1000e_check_82574_phy_workaround(adapter); 5408 5409 /* Clear valid timestamp stuck in RXSTMPL/H due to a Rx error */ 5410 if (adapter->hwtstamp_config.rx_filter != HWTSTAMP_FILTER_NONE) { 5411 if ((adapter->flags2 & FLAG2_CHECK_RX_HWTSTAMP) && 5412 (er32(TSYNCRXCTL) & E1000_TSYNCRXCTL_VALID)) { 5413 er32(RXSTMPH); 5414 adapter->rx_hwtstamp_cleared++; 5415 } else { 5416 adapter->flags2 |= FLAG2_CHECK_RX_HWTSTAMP; 5417 } 5418 } 5419 5420 /* Reset the timer */ 5421 if (!test_bit(__E1000_DOWN, &adapter->state)) 5422 mod_timer(&adapter->watchdog_timer, 5423 round_jiffies(jiffies + 2 * HZ)); 5424} 5425 5426#define E1000_TX_FLAGS_CSUM 0x00000001 5427#define E1000_TX_FLAGS_VLAN 0x00000002 5428#define E1000_TX_FLAGS_TSO 0x00000004 5429#define E1000_TX_FLAGS_IPV4 0x00000008 5430#define E1000_TX_FLAGS_NO_FCS 0x00000010 5431#define E1000_TX_FLAGS_HWTSTAMP 0x00000020 5432#define E1000_TX_FLAGS_VLAN_MASK 0xffff0000 5433#define E1000_TX_FLAGS_VLAN_SHIFT 16 5434 5435static int e1000_tso(struct e1000_ring *tx_ring, struct sk_buff *skb, 5436 __be16 protocol) 5437{ 5438 struct e1000_context_desc *context_desc; 5439 struct e1000_buffer *buffer_info; 5440 unsigned int i; 5441 u32 cmd_length = 0; 5442 u16 ipcse = 0, mss; 5443 u8 ipcss, ipcso, tucss, tucso, hdr_len; 5444 int err; 5445 5446 if (!skb_is_gso(skb)) 5447 return 0; 5448 5449 err = skb_cow_head(skb, 0); 5450 if (err < 0) 5451 return err; 5452 5453 hdr_len = skb_transport_offset(skb) + tcp_hdrlen(skb); 5454 mss = skb_shinfo(skb)->gso_size; 5455 if (protocol == htons(ETH_P_IP)) { 5456 struct iphdr *iph = ip_hdr(skb); 5457 iph->tot_len = 0; 5458 iph->check = 0; 5459 tcp_hdr(skb)->check = ~csum_tcpudp_magic(iph->saddr, iph->daddr, 5460 0, IPPROTO_TCP, 0); 5461 cmd_length = E1000_TXD_CMD_IP; 5462 ipcse = skb_transport_offset(skb) - 1; 5463 } else if (skb_is_gso_v6(skb)) { 5464 tcp_v6_gso_csum_prep(skb); 5465 ipcse = 0; 5466 } 5467 ipcss = skb_network_offset(skb); 5468 ipcso = (void *)&(ip_hdr(skb)->check) - (void *)skb->data; 5469 tucss = skb_transport_offset(skb); 5470 tucso = (void *)&(tcp_hdr(skb)->check) - (void *)skb->data; 5471 5472 cmd_length |= (E1000_TXD_CMD_DEXT | E1000_TXD_CMD_TSE | 5473 E1000_TXD_CMD_TCP | (skb->len - (hdr_len))); 5474 5475 i = tx_ring->next_to_use; 5476 context_desc = E1000_CONTEXT_DESC(*tx_ring, i); 5477 buffer_info = &tx_ring->buffer_info[i]; 5478 5479 context_desc->lower_setup.ip_fields.ipcss = ipcss; 5480 context_desc->lower_setup.ip_fields.ipcso = ipcso; 5481 context_desc->lower_setup.ip_fields.ipcse = cpu_to_le16(ipcse); 5482 context_desc->upper_setup.tcp_fields.tucss = tucss; 5483 context_desc->upper_setup.tcp_fields.tucso = tucso; 5484 context_desc->upper_setup.tcp_fields.tucse = 0; 5485 context_desc->tcp_seg_setup.fields.mss = cpu_to_le16(mss); 5486 context_desc->tcp_seg_setup.fields.hdr_len = hdr_len; 5487 context_desc->cmd_and_length = cpu_to_le32(cmd_length); 5488 5489 buffer_info->time_stamp = jiffies; 5490 buffer_info->next_to_watch = i; 5491 5492 i++; 5493 if (i == tx_ring->count) 5494 i = 0; 5495 tx_ring->next_to_use = i; 5496 5497 return 1; 5498} 5499 5500static bool e1000_tx_csum(struct e1000_ring *tx_ring, struct sk_buff *skb, 5501 __be16 protocol) 5502{ 5503 struct e1000_adapter *adapter = tx_ring->adapter; 5504 struct e1000_context_desc *context_desc; 5505 struct e1000_buffer *buffer_info; 5506 unsigned int i; 5507 u8 css; 5508 u32 cmd_len = E1000_TXD_CMD_DEXT; 5509 5510 if (skb->ip_summed != CHECKSUM_PARTIAL) 5511 return false; 5512 5513 switch (protocol) { 5514 case cpu_to_be16(ETH_P_IP): 5515 if (ip_hdr(skb)->protocol == IPPROTO_TCP) 5516 cmd_len |= E1000_TXD_CMD_TCP; 5517 break; 5518 case cpu_to_be16(ETH_P_IPV6): 5519 /* XXX not handling all IPV6 headers */ 5520 if (ipv6_hdr(skb)->nexthdr == IPPROTO_TCP) 5521 cmd_len |= E1000_TXD_CMD_TCP; 5522 break; 5523 default: 5524 if (unlikely(net_ratelimit())) 5525 e_warn("checksum_partial proto=%x!\n", 5526 be16_to_cpu(protocol)); 5527 break; 5528 } 5529 5530 css = skb_checksum_start_offset(skb); 5531 5532 i = tx_ring->next_to_use; 5533 buffer_info = &tx_ring->buffer_info[i]; 5534 context_desc = E1000_CONTEXT_DESC(*tx_ring, i); 5535 5536 context_desc->lower_setup.ip_config = 0; 5537 context_desc->upper_setup.tcp_fields.tucss = css; 5538 context_desc->upper_setup.tcp_fields.tucso = css + skb->csum_offset; 5539 context_desc->upper_setup.tcp_fields.tucse = 0; 5540 context_desc->tcp_seg_setup.data = 0; 5541 context_desc->cmd_and_length = cpu_to_le32(cmd_len); 5542 5543 buffer_info->time_stamp = jiffies; 5544 buffer_info->next_to_watch = i; 5545 5546 i++; 5547 if (i == tx_ring->count) 5548 i = 0; 5549 tx_ring->next_to_use = i; 5550 5551 return true; 5552} 5553 5554static int e1000_tx_map(struct e1000_ring *tx_ring, struct sk_buff *skb, 5555 unsigned int first, unsigned int max_per_txd, 5556 unsigned int nr_frags) 5557{ 5558 struct e1000_adapter *adapter = tx_ring->adapter; 5559 struct pci_dev *pdev = adapter->pdev; 5560 struct e1000_buffer *buffer_info; 5561 unsigned int len = skb_headlen(skb); 5562 unsigned int offset = 0, size, count = 0, i; 5563 unsigned int f, bytecount, segs; 5564 5565 i = tx_ring->next_to_use; 5566 5567 while (len) { 5568 buffer_info = &tx_ring->buffer_info[i]; 5569 size = min(len, max_per_txd); 5570 5571 buffer_info->length = size; 5572 buffer_info->time_stamp = jiffies; 5573 buffer_info->next_to_watch = i; 5574 buffer_info->dma = dma_map_single(&pdev->dev, 5575 skb->data + offset, 5576 size, DMA_TO_DEVICE); 5577 buffer_info->mapped_as_page = false; 5578 if (dma_mapping_error(&pdev->dev, buffer_info->dma)) 5579 goto dma_error; 5580 5581 len -= size; 5582 offset += size; 5583 count++; 5584 5585 if (len) { 5586 i++; 5587 if (i == tx_ring->count) 5588 i = 0; 5589 } 5590 } 5591 5592 for (f = 0; f < nr_frags; f++) { 5593 const skb_frag_t *frag = &skb_shinfo(skb)->frags[f]; 5594 5595 len = skb_frag_size(frag); 5596 offset = 0; 5597 5598 while (len) { 5599 i++; 5600 if (i == tx_ring->count) 5601 i = 0; 5602 5603 buffer_info = &tx_ring->buffer_info[i]; 5604 size = min(len, max_per_txd); 5605 5606 buffer_info->length = size; 5607 buffer_info->time_stamp = jiffies; 5608 buffer_info->next_to_watch = i; 5609 buffer_info->dma = skb_frag_dma_map(&pdev->dev, frag, 5610 offset, size, 5611 DMA_TO_DEVICE); 5612 buffer_info->mapped_as_page = true; 5613 if (dma_mapping_error(&pdev->dev, buffer_info->dma)) 5614 goto dma_error; 5615 5616 len -= size; 5617 offset += size; 5618 count++; 5619 } 5620 } 5621 5622 segs = skb_shinfo(skb)->gso_segs ? : 1; 5623 /* multiply data chunks by size of headers */ 5624 bytecount = ((segs - 1) * skb_headlen(skb)) + skb->len; 5625 5626 tx_ring->buffer_info[i].skb = skb; 5627 tx_ring->buffer_info[i].segs = segs; 5628 tx_ring->buffer_info[i].bytecount = bytecount; 5629 tx_ring->buffer_info[first].next_to_watch = i; 5630 5631 return count; 5632 5633dma_error: 5634 dev_err(&pdev->dev, "Tx DMA map failed\n"); 5635 buffer_info->dma = 0; 5636 if (count) 5637 count--; 5638 5639 while (count--) { 5640 if (i == 0) 5641 i += tx_ring->count; 5642 i--; 5643 buffer_info = &tx_ring->buffer_info[i]; 5644 e1000_put_txbuf(tx_ring, buffer_info, true); 5645 } 5646 5647 return 0; 5648} 5649 5650static void e1000_tx_queue(struct e1000_ring *tx_ring, int tx_flags, int count) 5651{ 5652 struct e1000_adapter *adapter = tx_ring->adapter; 5653 struct e1000_tx_desc *tx_desc = NULL; 5654 struct e1000_buffer *buffer_info; 5655 u32 txd_upper = 0, txd_lower = E1000_TXD_CMD_IFCS; 5656 unsigned int i; 5657 5658 if (tx_flags & E1000_TX_FLAGS_TSO) { 5659 txd_lower |= E1000_TXD_CMD_DEXT | E1000_TXD_DTYP_D | 5660 E1000_TXD_CMD_TSE; 5661 txd_upper |= E1000_TXD_POPTS_TXSM << 8; 5662 5663 if (tx_flags & E1000_TX_FLAGS_IPV4) 5664 txd_upper |= E1000_TXD_POPTS_IXSM << 8; 5665 } 5666 5667 if (tx_flags & E1000_TX_FLAGS_CSUM) { 5668 txd_lower |= E1000_TXD_CMD_DEXT | E1000_TXD_DTYP_D; 5669 txd_upper |= E1000_TXD_POPTS_TXSM << 8; 5670 } 5671 5672 if (tx_flags & E1000_TX_FLAGS_VLAN) { 5673 txd_lower |= E1000_TXD_CMD_VLE; 5674 txd_upper |= (tx_flags & E1000_TX_FLAGS_VLAN_MASK); 5675 } 5676 5677 if (unlikely(tx_flags & E1000_TX_FLAGS_NO_FCS)) 5678 txd_lower &= ~(E1000_TXD_CMD_IFCS); 5679 5680 if (unlikely(tx_flags & E1000_TX_FLAGS_HWTSTAMP)) { 5681 txd_lower |= E1000_TXD_CMD_DEXT | E1000_TXD_DTYP_D; 5682 txd_upper |= E1000_TXD_EXTCMD_TSTAMP; 5683 } 5684 5685 i = tx_ring->next_to_use; 5686 5687 do { 5688 buffer_info = &tx_ring->buffer_info[i]; 5689 tx_desc = E1000_TX_DESC(*tx_ring, i); 5690 tx_desc->buffer_addr = cpu_to_le64(buffer_info->dma); 5691 tx_desc->lower.data = cpu_to_le32(txd_lower | 5692 buffer_info->length); 5693 tx_desc->upper.data = cpu_to_le32(txd_upper); 5694 5695 i++; 5696 if (i == tx_ring->count) 5697 i = 0; 5698 } while (--count > 0); 5699 5700 tx_desc->lower.data |= cpu_to_le32(adapter->txd_cmd); 5701 5702 /* txd_cmd re-enables FCS, so we'll re-disable it here as desired. */ 5703 if (unlikely(tx_flags & E1000_TX_FLAGS_NO_FCS)) 5704 tx_desc->lower.data &= ~(cpu_to_le32(E1000_TXD_CMD_IFCS)); 5705 5706 /* Force memory writes to complete before letting h/w 5707 * know there are new descriptors to fetch. (Only 5708 * applicable for weak-ordered memory model archs, 5709 * such as IA-64). 5710 */ 5711 wmb(); 5712 5713 tx_ring->next_to_use = i; 5714} 5715 5716#define MINIMUM_DHCP_PACKET_SIZE 282 5717static int e1000_transfer_dhcp_info(struct e1000_adapter *adapter, 5718 struct sk_buff *skb) 5719{ 5720 struct e1000_hw *hw = &adapter->hw; 5721 u16 length, offset; 5722 5723 if (skb_vlan_tag_present(skb) && 5724 !((skb_vlan_tag_get(skb) == adapter->hw.mng_cookie.vlan_id) && 5725 (adapter->hw.mng_cookie.status & 5726 E1000_MNG_DHCP_COOKIE_STATUS_VLAN))) 5727 return 0; 5728 5729 if (skb->len <= MINIMUM_DHCP_PACKET_SIZE) 5730 return 0; 5731 5732 if (((struct ethhdr *)skb->data)->h_proto != htons(ETH_P_IP)) 5733 return 0; 5734 5735 { 5736 const struct iphdr *ip = (struct iphdr *)((u8 *)skb->data + 14); 5737 struct udphdr *udp; 5738 5739 if (ip->protocol != IPPROTO_UDP) 5740 return 0; 5741 5742 udp = (struct udphdr *)((u8 *)ip + (ip->ihl << 2)); 5743 if (ntohs(udp->dest) != 67) 5744 return 0; 5745 5746 offset = (u8 *)udp + 8 - skb->data; 5747 length = skb->len - offset; 5748 return e1000e_mng_write_dhcp_info(hw, (u8 *)udp + 8, length); 5749 } 5750 5751 return 0; 5752} 5753 5754static int __e1000_maybe_stop_tx(struct e1000_ring *tx_ring, int size) 5755{ 5756 struct e1000_adapter *adapter = tx_ring->adapter; 5757 5758 netif_stop_queue(adapter->netdev); 5759 /* Herbert's original patch had: 5760 * smp_mb__after_netif_stop_queue(); 5761 * but since that doesn't exist yet, just open code it. 5762 */ 5763 smp_mb(); 5764 5765 /* We need to check again in a case another CPU has just 5766 * made room available. 5767 */ 5768 if (e1000_desc_unused(tx_ring) < size) 5769 return -EBUSY; 5770 5771 /* A reprieve! */ 5772 netif_start_queue(adapter->netdev); 5773 ++adapter->restart_queue; 5774 return 0; 5775} 5776 5777static int e1000_maybe_stop_tx(struct e1000_ring *tx_ring, int size) 5778{ 5779 BUG_ON(size > tx_ring->count); 5780 5781 if (e1000_desc_unused(tx_ring) >= size) 5782 return 0; 5783 return __e1000_maybe_stop_tx(tx_ring, size); 5784} 5785 5786static netdev_tx_t e1000_xmit_frame(struct sk_buff *skb, 5787 struct net_device *netdev) 5788{ 5789 struct e1000_adapter *adapter = netdev_priv(netdev); 5790 struct e1000_ring *tx_ring = adapter->tx_ring; 5791 unsigned int first; 5792 unsigned int tx_flags = 0; 5793 unsigned int len = skb_headlen(skb); 5794 unsigned int nr_frags; 5795 unsigned int mss; 5796 int count = 0; 5797 int tso; 5798 unsigned int f; 5799 __be16 protocol = vlan_get_protocol(skb); 5800 5801 if (test_bit(__E1000_DOWN, &adapter->state)) { 5802 dev_kfree_skb_any(skb); 5803 return NETDEV_TX_OK; 5804 } 5805 5806 if (skb->len <= 0) { 5807 dev_kfree_skb_any(skb); 5808 return NETDEV_TX_OK; 5809 } 5810 5811 /* The minimum packet size with TCTL.PSP set is 17 bytes so 5812 * pad skb in order to meet this minimum size requirement 5813 */ 5814 if (skb_put_padto(skb, 17)) 5815 return NETDEV_TX_OK; 5816 5817 mss = skb_shinfo(skb)->gso_size; 5818 if (mss) { 5819 u8 hdr_len; 5820 5821 /* TSO Workaround for 82571/2/3 Controllers -- if skb->data 5822 * points to just header, pull a few bytes of payload from 5823 * frags into skb->data 5824 */ 5825 hdr_len = skb_transport_offset(skb) + tcp_hdrlen(skb); 5826 /* we do this workaround for ES2LAN, but it is un-necessary, 5827 * avoiding it could save a lot of cycles 5828 */ 5829 if (skb->data_len && (hdr_len == len)) { 5830 unsigned int pull_size; 5831 5832 pull_size = min_t(unsigned int, 4, skb->data_len); 5833 if (!__pskb_pull_tail(skb, pull_size)) { 5834 e_err("__pskb_pull_tail failed.\n"); 5835 dev_kfree_skb_any(skb); 5836 return NETDEV_TX_OK; 5837 } 5838 len = skb_headlen(skb); 5839 } 5840 } 5841 5842 /* reserve a descriptor for the offload context */ 5843 if ((mss) || (skb->ip_summed == CHECKSUM_PARTIAL)) 5844 count++; 5845 count++; 5846 5847 count += DIV_ROUND_UP(len, adapter->tx_fifo_limit); 5848 5849 nr_frags = skb_shinfo(skb)->nr_frags; 5850 for (f = 0; f < nr_frags; f++) 5851 count += DIV_ROUND_UP(skb_frag_size(&skb_shinfo(skb)->frags[f]), 5852 adapter->tx_fifo_limit); 5853 5854 if (adapter->hw.mac.tx_pkt_filtering) 5855 e1000_transfer_dhcp_info(adapter, skb); 5856 5857 /* need: count + 2 desc gap to keep tail from touching 5858 * head, otherwise try next time 5859 */ 5860 if (e1000_maybe_stop_tx(tx_ring, count + 2)) 5861 return NETDEV_TX_BUSY; 5862 5863 if (skb_vlan_tag_present(skb)) { 5864 tx_flags |= E1000_TX_FLAGS_VLAN; 5865 tx_flags |= (skb_vlan_tag_get(skb) << 5866 E1000_TX_FLAGS_VLAN_SHIFT); 5867 } 5868 5869 first = tx_ring->next_to_use; 5870 5871 tso = e1000_tso(tx_ring, skb, protocol); 5872 if (tso < 0) { 5873 dev_kfree_skb_any(skb); 5874 return NETDEV_TX_OK; 5875 } 5876 5877 if (tso) 5878 tx_flags |= E1000_TX_FLAGS_TSO; 5879 else if (e1000_tx_csum(tx_ring, skb, protocol)) 5880 tx_flags |= E1000_TX_FLAGS_CSUM; 5881 5882 /* Old method was to assume IPv4 packet by default if TSO was enabled. 5883 * 82571 hardware supports TSO capabilities for IPv6 as well... 5884 * no longer assume, we must. 5885 */ 5886 if (protocol == htons(ETH_P_IP)) 5887 tx_flags |= E1000_TX_FLAGS_IPV4; 5888 5889 if (unlikely(skb->no_fcs)) 5890 tx_flags |= E1000_TX_FLAGS_NO_FCS; 5891 5892 /* if count is 0 then mapping error has occurred */ 5893 count = e1000_tx_map(tx_ring, skb, first, adapter->tx_fifo_limit, 5894 nr_frags); 5895 if (count) { 5896 if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) && 5897 (adapter->flags & FLAG_HAS_HW_TIMESTAMP)) { 5898 if (!adapter->tx_hwtstamp_skb) { 5899 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; 5900 tx_flags |= E1000_TX_FLAGS_HWTSTAMP; 5901 adapter->tx_hwtstamp_skb = skb_get(skb); 5902 adapter->tx_hwtstamp_start = jiffies; 5903 schedule_work(&adapter->tx_hwtstamp_work); 5904 } else { 5905 adapter->tx_hwtstamp_skipped++; 5906 } 5907 } 5908 5909 skb_tx_timestamp(skb); 5910 5911 netdev_sent_queue(netdev, skb->len); 5912 e1000_tx_queue(tx_ring, tx_flags, count); 5913 /* Make sure there is space in the ring for the next send. */ 5914 e1000_maybe_stop_tx(tx_ring, 5915 ((MAX_SKB_FRAGS + 1) * 5916 DIV_ROUND_UP(PAGE_SIZE, 5917 adapter->tx_fifo_limit) + 4)); 5918 5919 if (!netdev_xmit_more() || 5920 netif_xmit_stopped(netdev_get_tx_queue(netdev, 0))) { 5921 if (adapter->flags2 & FLAG2_PCIM2PCI_ARBITER_WA) 5922 e1000e_update_tdt_wa(tx_ring, 5923 tx_ring->next_to_use); 5924 else 5925 writel(tx_ring->next_to_use, tx_ring->tail); 5926 } 5927 } else { 5928 dev_kfree_skb_any(skb); 5929 tx_ring->buffer_info[first].time_stamp = 0; 5930 tx_ring->next_to_use = first; 5931 } 5932 5933 return NETDEV_TX_OK; 5934} 5935 5936/** 5937 * e1000_tx_timeout - Respond to a Tx Hang 5938 * @netdev: network interface device structure 5939 * @txqueue: index of the hung queue (unused) 5940 **/ 5941static void e1000_tx_timeout(struct net_device *netdev, unsigned int __always_unused txqueue) 5942{ 5943 struct e1000_adapter *adapter = netdev_priv(netdev); 5944 5945 /* Do the reset outside of interrupt context */ 5946 adapter->tx_timeout_count++; 5947 schedule_work(&adapter->reset_task); 5948} 5949 5950static void e1000_reset_task(struct work_struct *work) 5951{ 5952 struct e1000_adapter *adapter; 5953 adapter = container_of(work, struct e1000_adapter, reset_task); 5954 5955 rtnl_lock(); 5956 /* don't run the task if already down */ 5957 if (test_bit(__E1000_DOWN, &adapter->state)) { 5958 rtnl_unlock(); 5959 return; 5960 } 5961 5962 if (!(adapter->flags & FLAG_RESTART_NOW)) { 5963 e1000e_dump(adapter); 5964 e_err("Reset adapter unexpectedly\n"); 5965 } 5966 e1000e_reinit_locked(adapter); 5967 rtnl_unlock(); 5968} 5969 5970/** 5971 * e1000_get_stats64 - Get System Network Statistics 5972 * @netdev: network interface device structure 5973 * @stats: rtnl_link_stats64 pointer 5974 * 5975 * Returns the address of the device statistics structure. 5976 **/ 5977void e1000e_get_stats64(struct net_device *netdev, 5978 struct rtnl_link_stats64 *stats) 5979{ 5980 struct e1000_adapter *adapter = netdev_priv(netdev); 5981 5982 spin_lock(&adapter->stats64_lock); 5983 e1000e_update_stats(adapter); 5984 /* Fill out the OS statistics structure */ 5985 stats->rx_bytes = adapter->stats.gorc; 5986 stats->rx_packets = adapter->stats.gprc; 5987 stats->tx_bytes = adapter->stats.gotc; 5988 stats->tx_packets = adapter->stats.gptc; 5989 stats->multicast = adapter->stats.mprc; 5990 stats->collisions = adapter->stats.colc; 5991 5992 /* Rx Errors */ 5993 5994 /* RLEC on some newer hardware can be incorrect so build 5995 * our own version based on RUC and ROC 5996 */ 5997 stats->rx_errors = adapter->stats.rxerrc + 5998 adapter->stats.crcerrs + adapter->stats.algnerrc + 5999 adapter->stats.ruc + adapter->stats.roc + adapter->stats.cexterr; 6000 stats->rx_length_errors = adapter->stats.ruc + adapter->stats.roc; 6001 stats->rx_crc_errors = adapter->stats.crcerrs; 6002 stats->rx_frame_errors = adapter->stats.algnerrc; 6003 stats->rx_missed_errors = adapter->stats.mpc; 6004 6005 /* Tx Errors */ 6006 stats->tx_errors = adapter->stats.ecol + adapter->stats.latecol; 6007 stats->tx_aborted_errors = adapter->stats.ecol; 6008 stats->tx_window_errors = adapter->stats.latecol; 6009 stats->tx_carrier_errors = adapter->stats.tncrs; 6010 6011 /* Tx Dropped needs to be maintained elsewhere */ 6012 6013 spin_unlock(&adapter->stats64_lock); 6014} 6015 6016/** 6017 * e1000_change_mtu - Change the Maximum Transfer Unit 6018 * @netdev: network interface device structure 6019 * @new_mtu: new value for maximum frame size 6020 * 6021 * Returns 0 on success, negative on failure 6022 **/ 6023static int e1000_change_mtu(struct net_device *netdev, int new_mtu) 6024{ 6025 struct e1000_adapter *adapter = netdev_priv(netdev); 6026 int max_frame = new_mtu + VLAN_ETH_HLEN + ETH_FCS_LEN; 6027 6028 /* Jumbo frame support */ 6029 if ((new_mtu > ETH_DATA_LEN) && 6030 !(adapter->flags & FLAG_HAS_JUMBO_FRAMES)) { 6031 e_err("Jumbo Frames not supported.\n"); 6032 return -EINVAL; 6033 } 6034 6035 /* Jumbo frame workaround on 82579 and newer requires CRC be stripped */ 6036 if ((adapter->hw.mac.type >= e1000_pch2lan) && 6037 !(adapter->flags2 & FLAG2_CRC_STRIPPING) && 6038 (new_mtu > ETH_DATA_LEN)) { 6039 e_err("Jumbo Frames not supported on this device when CRC stripping is disabled.\n"); 6040 return -EINVAL; 6041 } 6042 6043 while (test_and_set_bit(__E1000_RESETTING, &adapter->state)) 6044 usleep_range(1000, 1100); 6045 /* e1000e_down -> e1000e_reset dependent on max_frame_size & mtu */ 6046 adapter->max_frame_size = max_frame; 6047 netdev_dbg(netdev, "changing MTU from %d to %d\n", 6048 netdev->mtu, new_mtu); 6049 netdev->mtu = new_mtu; 6050 6051 pm_runtime_get_sync(netdev->dev.parent); 6052 6053 if (netif_running(netdev)) 6054 e1000e_down(adapter, true); 6055 6056 /* NOTE: netdev_alloc_skb reserves 16 bytes, and typically NET_IP_ALIGN 6057 * means we reserve 2 more, this pushes us to allocate from the next 6058 * larger slab size. 6059 * i.e. RXBUFFER_2048 --> size-4096 slab 6060 * However with the new *_jumbo_rx* routines, jumbo receives will use 6061 * fragmented skbs 6062 */ 6063 6064 if (max_frame <= 2048) 6065 adapter->rx_buffer_len = 2048; 6066 else 6067 adapter->rx_buffer_len = 4096; 6068 6069 /* adjust allocation if LPE protects us, and we aren't using SBP */ 6070 if (max_frame <= (VLAN_ETH_FRAME_LEN + ETH_FCS_LEN)) 6071 adapter->rx_buffer_len = VLAN_ETH_FRAME_LEN + ETH_FCS_LEN; 6072 6073 if (netif_running(netdev)) 6074 e1000e_up(adapter); 6075 else 6076 e1000e_reset(adapter); 6077 6078 pm_runtime_put_sync(netdev->dev.parent); 6079 6080 clear_bit(__E1000_RESETTING, &adapter->state); 6081 6082 return 0; 6083} 6084 6085static int e1000_mii_ioctl(struct net_device *netdev, struct ifreq *ifr, 6086 int cmd) 6087{ 6088 struct e1000_adapter *adapter = netdev_priv(netdev); 6089 struct mii_ioctl_data *data = if_mii(ifr); 6090 6091 if (adapter->hw.phy.media_type != e1000_media_type_copper) 6092 return -EOPNOTSUPP; 6093 6094 switch (cmd) { 6095 case SIOCGMIIPHY: 6096 data->phy_id = adapter->hw.phy.addr; 6097 break; 6098 case SIOCGMIIREG: 6099 e1000_phy_read_status(adapter); 6100 6101 switch (data->reg_num & 0x1F) { 6102 case MII_BMCR: 6103 data->val_out = adapter->phy_regs.bmcr; 6104 break; 6105 case MII_BMSR: 6106 data->val_out = adapter->phy_regs.bmsr; 6107 break; 6108 case MII_PHYSID1: 6109 data->val_out = (adapter->hw.phy.id >> 16); 6110 break; 6111 case MII_PHYSID2: 6112 data->val_out = (adapter->hw.phy.id & 0xFFFF); 6113 break; 6114 case MII_ADVERTISE: 6115 data->val_out = adapter->phy_regs.advertise; 6116 break; 6117 case MII_LPA: 6118 data->val_out = adapter->phy_regs.lpa; 6119 break; 6120 case MII_EXPANSION: 6121 data->val_out = adapter->phy_regs.expansion; 6122 break; 6123 case MII_CTRL1000: 6124 data->val_out = adapter->phy_regs.ctrl1000; 6125 break; 6126 case MII_STAT1000: 6127 data->val_out = adapter->phy_regs.stat1000; 6128 break; 6129 case MII_ESTATUS: 6130 data->val_out = adapter->phy_regs.estatus; 6131 break; 6132 default: 6133 return -EIO; 6134 } 6135 break; 6136 case SIOCSMIIREG: 6137 default: 6138 return -EOPNOTSUPP; 6139 } 6140 return 0; 6141} 6142 6143/** 6144 * e1000e_hwtstamp_ioctl - control hardware time stamping 6145 * @netdev: network interface device structure 6146 * @ifr: interface request 6147 * 6148 * Outgoing time stamping can be enabled and disabled. Play nice and 6149 * disable it when requested, although it shouldn't cause any overhead 6150 * when no packet needs it. At most one packet in the queue may be 6151 * marked for time stamping, otherwise it would be impossible to tell 6152 * for sure to which packet the hardware time stamp belongs. 6153 * 6154 * Incoming time stamping has to be configured via the hardware filters. 6155 * Not all combinations are supported, in particular event type has to be 6156 * specified. Matching the kind of event packet is not supported, with the 6157 * exception of "all V2 events regardless of level 2 or 4". 6158 **/ 6159static int e1000e_hwtstamp_set(struct net_device *netdev, struct ifreq *ifr) 6160{ 6161 struct e1000_adapter *adapter = netdev_priv(netdev); 6162 struct hwtstamp_config config; 6163 int ret_val; 6164 6165 if (copy_from_user(&config, ifr->ifr_data, sizeof(config))) 6166 return -EFAULT; 6167 6168 ret_val = e1000e_config_hwtstamp(adapter, &config); 6169 if (ret_val) 6170 return ret_val; 6171 6172 switch (config.rx_filter) { 6173 case HWTSTAMP_FILTER_PTP_V2_L4_SYNC: 6174 case HWTSTAMP_FILTER_PTP_V2_L2_SYNC: 6175 case HWTSTAMP_FILTER_PTP_V2_SYNC: 6176 case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ: 6177 case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ: 6178 case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ: 6179 /* With V2 type filters which specify a Sync or Delay Request, 6180 * Path Delay Request/Response messages are also time stamped 6181 * by hardware so notify the caller the requested packets plus 6182 * some others are time stamped. 6183 */ 6184 config.rx_filter = HWTSTAMP_FILTER_SOME; 6185 break; 6186 default: 6187 break; 6188 } 6189 6190 return copy_to_user(ifr->ifr_data, &config, 6191 sizeof(config)) ? -EFAULT : 0; 6192} 6193 6194static int e1000e_hwtstamp_get(struct net_device *netdev, struct ifreq *ifr) 6195{ 6196 struct e1000_adapter *adapter = netdev_priv(netdev); 6197 6198 return copy_to_user(ifr->ifr_data, &adapter->hwtstamp_config, 6199 sizeof(adapter->hwtstamp_config)) ? -EFAULT : 0; 6200} 6201 6202static int e1000_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd) 6203{ 6204 switch (cmd) { 6205 case SIOCGMIIPHY: 6206 case SIOCGMIIREG: 6207 case SIOCSMIIREG: 6208 return e1000_mii_ioctl(netdev, ifr, cmd); 6209 case SIOCSHWTSTAMP: 6210 return e1000e_hwtstamp_set(netdev, ifr); 6211 case SIOCGHWTSTAMP: 6212 return e1000e_hwtstamp_get(netdev, ifr); 6213 default: 6214 return -EOPNOTSUPP; 6215 } 6216} 6217 6218static int e1000_init_phy_wakeup(struct e1000_adapter *adapter, u32 wufc) 6219{ 6220 struct e1000_hw *hw = &adapter->hw; 6221 u32 i, mac_reg, wuc; 6222 u16 phy_reg, wuc_enable; 6223 int retval; 6224 6225 /* copy MAC RARs to PHY RARs */ 6226 e1000_copy_rx_addrs_to_phy_ich8lan(hw); 6227 6228 retval = hw->phy.ops.acquire(hw); 6229 if (retval) { 6230 e_err("Could not acquire PHY\n"); 6231 return retval; 6232 } 6233 6234 /* Enable access to wakeup registers on and set page to BM_WUC_PAGE */ 6235 retval = e1000_enable_phy_wakeup_reg_access_bm(hw, &wuc_enable); 6236 if (retval) 6237 goto release; 6238 6239 /* copy MAC MTA to PHY MTA - only needed for pchlan */ 6240 for (i = 0; i < adapter->hw.mac.mta_reg_count; i++) { 6241 mac_reg = E1000_READ_REG_ARRAY(hw, E1000_MTA, i); 6242 hw->phy.ops.write_reg_page(hw, BM_MTA(i), 6243 (u16)(mac_reg & 0xFFFF)); 6244 hw->phy.ops.write_reg_page(hw, BM_MTA(i) + 1, 6245 (u16)((mac_reg >> 16) & 0xFFFF)); 6246 } 6247 6248 /* configure PHY Rx Control register */ 6249 hw->phy.ops.read_reg_page(&adapter->hw, BM_RCTL, &phy_reg); 6250 mac_reg = er32(RCTL); 6251 if (mac_reg & E1000_RCTL_UPE) 6252 phy_reg |= BM_RCTL_UPE; 6253 if (mac_reg & E1000_RCTL_MPE) 6254 phy_reg |= BM_RCTL_MPE; 6255 phy_reg &= ~(BM_RCTL_MO_MASK); 6256 if (mac_reg & E1000_RCTL_MO_3) 6257 phy_reg |= (((mac_reg & E1000_RCTL_MO_3) >> E1000_RCTL_MO_SHIFT) 6258 << BM_RCTL_MO_SHIFT); 6259 if (mac_reg & E1000_RCTL_BAM) 6260 phy_reg |= BM_RCTL_BAM; 6261 if (mac_reg & E1000_RCTL_PMCF) 6262 phy_reg |= BM_RCTL_PMCF; 6263 mac_reg = er32(CTRL); 6264 if (mac_reg & E1000_CTRL_RFCE) 6265 phy_reg |= BM_RCTL_RFCE; 6266 hw->phy.ops.write_reg_page(&adapter->hw, BM_RCTL, phy_reg); 6267 6268 wuc = E1000_WUC_PME_EN; 6269 if (wufc & (E1000_WUFC_MAG | E1000_WUFC_LNKC)) 6270 wuc |= E1000_WUC_APME; 6271 6272 /* enable PHY wakeup in MAC register */ 6273 ew32(WUFC, wufc); 6274 ew32(WUC, (E1000_WUC_PHY_WAKE | E1000_WUC_APMPME | 6275 E1000_WUC_PME_STATUS | wuc)); 6276 6277 /* configure and enable PHY wakeup in PHY registers */ 6278 hw->phy.ops.write_reg_page(&adapter->hw, BM_WUFC, wufc); 6279 hw->phy.ops.write_reg_page(&adapter->hw, BM_WUC, wuc); 6280 6281 /* activate PHY wakeup */ 6282 wuc_enable |= BM_WUC_ENABLE_BIT | BM_WUC_HOST_WU_BIT; 6283 retval = e1000_disable_phy_wakeup_reg_access_bm(hw, &wuc_enable); 6284 if (retval) 6285 e_err("Could not set PHY Host Wakeup bit\n"); 6286release: 6287 hw->phy.ops.release(hw); 6288 6289 return retval; 6290} 6291 6292static void e1000e_flush_lpic(struct pci_dev *pdev) 6293{ 6294 struct net_device *netdev = pci_get_drvdata(pdev); 6295 struct e1000_adapter *adapter = netdev_priv(netdev); 6296 struct e1000_hw *hw = &adapter->hw; 6297 u32 ret_val; 6298 6299 pm_runtime_get_sync(netdev->dev.parent); 6300 6301 ret_val = hw->phy.ops.acquire(hw); 6302 if (ret_val) 6303 goto fl_out; 6304 6305 pr_info("EEE TX LPI TIMER: %08X\n", 6306 er32(LPIC) >> E1000_LPIC_LPIET_SHIFT); 6307 6308 hw->phy.ops.release(hw); 6309 6310fl_out: 6311 pm_runtime_put_sync(netdev->dev.parent); 6312} 6313 6314/* S0ix implementation */ 6315static void e1000e_s0ix_entry_flow(struct e1000_adapter *adapter) 6316{ 6317 struct e1000_hw *hw = &adapter->hw; 6318 u32 mac_data; 6319 u16 phy_data; 6320 6321 /* Disable the periodic inband message, 6322 * don't request PCIe clock in K1 page770_17[10:9] = 10b 6323 */ 6324 e1e_rphy(hw, HV_PM_CTRL, &phy_data); 6325 phy_data &= ~HV_PM_CTRL_K1_CLK_REQ; 6326 phy_data |= BIT(10); 6327 e1e_wphy(hw, HV_PM_CTRL, phy_data); 6328 6329 /* Make sure we don't exit K1 every time a new packet arrives 6330 * 772_29[5] = 1 CS_Mode_Stay_In_K1 6331 */ 6332 e1e_rphy(hw, I217_CGFREG, &phy_data); 6333 phy_data |= BIT(5); 6334 e1e_wphy(hw, I217_CGFREG, phy_data); 6335 6336 /* Change the MAC/PHY interface to SMBus 6337 * Force the SMBus in PHY page769_23[0] = 1 6338 * Force the SMBus in MAC CTRL_EXT[11] = 1 6339 */ 6340 e1e_rphy(hw, CV_SMB_CTRL, &phy_data); 6341 phy_data |= CV_SMB_CTRL_FORCE_SMBUS; 6342 e1e_wphy(hw, CV_SMB_CTRL, phy_data); 6343 mac_data = er32(CTRL_EXT); 6344 mac_data |= E1000_CTRL_EXT_FORCE_SMBUS; 6345 ew32(CTRL_EXT, mac_data); 6346 6347 /* DFT control: PHY bit: page769_20[0] = 1 6348 * Gate PPW via EXTCNF_CTRL - set 0x0F00[7] = 1 6349 */ 6350 e1e_rphy(hw, I82579_DFT_CTRL, &phy_data); 6351 phy_data |= BIT(0); 6352 e1e_wphy(hw, I82579_DFT_CTRL, phy_data); 6353 6354 mac_data = er32(EXTCNF_CTRL); 6355 mac_data |= E1000_EXTCNF_CTRL_GATE_PHY_CFG; 6356 ew32(EXTCNF_CTRL, mac_data); 6357 6358 /* Check MAC Tx/Rx packet buffer pointers. 6359 * Reset MAC Tx/Rx packet buffer pointers to suppress any 6360 * pending traffic indication that would prevent power gating. 6361 */ 6362 mac_data = er32(TDFH); 6363 if (mac_data) 6364 ew32(TDFH, 0); 6365 mac_data = er32(TDFT); 6366 if (mac_data) 6367 ew32(TDFT, 0); 6368 mac_data = er32(TDFHS); 6369 if (mac_data) 6370 ew32(TDFHS, 0); 6371 mac_data = er32(TDFTS); 6372 if (mac_data) 6373 ew32(TDFTS, 0); 6374 mac_data = er32(TDFPC); 6375 if (mac_data) 6376 ew32(TDFPC, 0); 6377 mac_data = er32(RDFH); 6378 if (mac_data) 6379 ew32(RDFH, 0); 6380 mac_data = er32(RDFT); 6381 if (mac_data) 6382 ew32(RDFT, 0); 6383 mac_data = er32(RDFHS); 6384 if (mac_data) 6385 ew32(RDFHS, 0); 6386 mac_data = er32(RDFTS); 6387 if (mac_data) 6388 ew32(RDFTS, 0); 6389 mac_data = er32(RDFPC); 6390 if (mac_data) 6391 ew32(RDFPC, 0); 6392 6393 /* Enable the Dynamic Power Gating in the MAC */ 6394 mac_data = er32(FEXTNVM7); 6395 mac_data |= BIT(22); 6396 ew32(FEXTNVM7, mac_data); 6397 6398 /* Disable the time synchronization clock */ 6399 mac_data = er32(FEXTNVM7); 6400 mac_data |= BIT(31); 6401 mac_data &= ~BIT(0); 6402 ew32(FEXTNVM7, mac_data); 6403 6404 /* Dynamic Power Gating Enable */ 6405 mac_data = er32(CTRL_EXT); 6406 mac_data |= BIT(3); 6407 ew32(CTRL_EXT, mac_data); 6408 6409 /* Disable disconnected cable conditioning for Power Gating */ 6410 mac_data = er32(DPGFR); 6411 mac_data |= BIT(2); 6412 ew32(DPGFR, mac_data); 6413 6414 /* Don't wake from dynamic Power Gating with clock request */ 6415 mac_data = er32(FEXTNVM12); 6416 mac_data |= BIT(12); 6417 ew32(FEXTNVM12, mac_data); 6418 6419 /* Ungate PGCB clock */ 6420 mac_data = er32(FEXTNVM9); 6421 mac_data &= ~BIT(28); 6422 ew32(FEXTNVM9, mac_data); 6423 6424 /* Enable K1 off to enable mPHY Power Gating */ 6425 mac_data = er32(FEXTNVM6); 6426 mac_data |= BIT(31); 6427 ew32(FEXTNVM6, mac_data); 6428 6429 /* Enable mPHY power gating for any link and speed */ 6430 mac_data = er32(FEXTNVM8); 6431 mac_data |= BIT(9); 6432 ew32(FEXTNVM8, mac_data); 6433 6434 /* Enable the Dynamic Clock Gating in the DMA and MAC */ 6435 mac_data = er32(CTRL_EXT); 6436 mac_data |= E1000_CTRL_EXT_DMA_DYN_CLK_EN; 6437 ew32(CTRL_EXT, mac_data); 6438 6439 /* No MAC DPG gating SLP_S0 in modern standby 6440 * Switch the logic of the lanphypc to use PMC counter 6441 */ 6442 mac_data = er32(FEXTNVM5); 6443 mac_data |= BIT(7); 6444 ew32(FEXTNVM5, mac_data); 6445} 6446 6447static void e1000e_s0ix_exit_flow(struct e1000_adapter *adapter) 6448{ 6449 struct e1000_hw *hw = &adapter->hw; 6450 u32 mac_data; 6451 u16 phy_data; 6452 6453 /* Disable the Dynamic Power Gating in the MAC */ 6454 mac_data = er32(FEXTNVM7); 6455 mac_data &= 0xFFBFFFFF; 6456 ew32(FEXTNVM7, mac_data); 6457 6458 /* Enable the time synchronization clock */ 6459 mac_data = er32(FEXTNVM7); 6460 mac_data |= BIT(0); 6461 ew32(FEXTNVM7, mac_data); 6462 6463 /* Disable mPHY power gating for any link and speed */ 6464 mac_data = er32(FEXTNVM8); 6465 mac_data &= ~BIT(9); 6466 ew32(FEXTNVM8, mac_data); 6467 6468 /* Disable K1 off */ 6469 mac_data = er32(FEXTNVM6); 6470 mac_data &= ~BIT(31); 6471 ew32(FEXTNVM6, mac_data); 6472 6473 /* Disable Ungate PGCB clock */ 6474 mac_data = er32(FEXTNVM9); 6475 mac_data |= BIT(28); 6476 ew32(FEXTNVM9, mac_data); 6477 6478 /* Cancel not waking from dynamic 6479 * Power Gating with clock request 6480 */ 6481 mac_data = er32(FEXTNVM12); 6482 mac_data &= ~BIT(12); 6483 ew32(FEXTNVM12, mac_data); 6484 6485 /* Cancel disable disconnected cable conditioning 6486 * for Power Gating 6487 */ 6488 mac_data = er32(DPGFR); 6489 mac_data &= ~BIT(2); 6490 ew32(DPGFR, mac_data); 6491 6492 /* Disable Dynamic Power Gating */ 6493 mac_data = er32(CTRL_EXT); 6494 mac_data &= 0xFFFFFFF7; 6495 ew32(CTRL_EXT, mac_data); 6496 6497 /* Disable the Dynamic Clock Gating in the DMA and MAC */ 6498 mac_data = er32(CTRL_EXT); 6499 mac_data &= 0xFFF7FFFF; 6500 ew32(CTRL_EXT, mac_data); 6501 6502 /* Revert the lanphypc logic to use the internal Gbe counter 6503 * and not the PMC counter 6504 */ 6505 mac_data = er32(FEXTNVM5); 6506 mac_data &= 0xFFFFFF7F; 6507 ew32(FEXTNVM5, mac_data); 6508 6509 /* Enable the periodic inband message, 6510 * Request PCIe clock in K1 page770_17[10:9] =01b 6511 */ 6512 e1e_rphy(hw, HV_PM_CTRL, &phy_data); 6513 phy_data &= 0xFBFF; 6514 phy_data |= HV_PM_CTRL_K1_CLK_REQ; 6515 e1e_wphy(hw, HV_PM_CTRL, phy_data); 6516 6517 /* Return back configuration 6518 * 772_29[5] = 0 CS_Mode_Stay_In_K1 6519 */ 6520 e1e_rphy(hw, I217_CGFREG, &phy_data); 6521 phy_data &= 0xFFDF; 6522 e1e_wphy(hw, I217_CGFREG, phy_data); 6523 6524 /* Change the MAC/PHY interface to Kumeran 6525 * Unforce the SMBus in PHY page769_23[0] = 0 6526 * Unforce the SMBus in MAC CTRL_EXT[11] = 0 6527 */ 6528 e1e_rphy(hw, CV_SMB_CTRL, &phy_data); 6529 phy_data &= ~CV_SMB_CTRL_FORCE_SMBUS; 6530 e1e_wphy(hw, CV_SMB_CTRL, phy_data); 6531 mac_data = er32(CTRL_EXT); 6532 mac_data &= ~E1000_CTRL_EXT_FORCE_SMBUS; 6533 ew32(CTRL_EXT, mac_data); 6534} 6535 6536static int e1000e_pm_freeze(struct device *dev) 6537{ 6538 struct net_device *netdev = dev_get_drvdata(dev); 6539 struct e1000_adapter *adapter = netdev_priv(netdev); 6540 bool present; 6541 6542 rtnl_lock(); 6543 6544 present = netif_device_present(netdev); 6545 netif_device_detach(netdev); 6546 6547 if (present && netif_running(netdev)) { 6548 int count = E1000_CHECK_RESET_COUNT; 6549 6550 while (test_bit(__E1000_RESETTING, &adapter->state) && count--) 6551 usleep_range(10000, 11000); 6552 6553 WARN_ON(test_bit(__E1000_RESETTING, &adapter->state)); 6554 6555 /* Quiesce the device without resetting the hardware */ 6556 e1000e_down(adapter, false); 6557 e1000_free_irq(adapter); 6558 } 6559 rtnl_unlock(); 6560 6561 e1000e_reset_interrupt_capability(adapter); 6562 6563 /* Allow time for pending master requests to run */ 6564 e1000e_disable_pcie_master(&adapter->hw); 6565 6566 return 0; 6567} 6568 6569static int __e1000_shutdown(struct pci_dev *pdev, bool runtime) 6570{ 6571 struct net_device *netdev = pci_get_drvdata(pdev); 6572 struct e1000_adapter *adapter = netdev_priv(netdev); 6573 struct e1000_hw *hw = &adapter->hw; 6574 u32 ctrl, ctrl_ext, rctl, status, wufc; 6575 int retval = 0; 6576 6577 /* Runtime suspend should only enable wakeup for link changes */ 6578 if (runtime) 6579 wufc = E1000_WUFC_LNKC; 6580 else if (device_may_wakeup(&pdev->dev)) 6581 wufc = adapter->wol; 6582 else 6583 wufc = 0; 6584 6585 status = er32(STATUS); 6586 if (status & E1000_STATUS_LU) 6587 wufc &= ~E1000_WUFC_LNKC; 6588 6589 if (wufc) { 6590 e1000_setup_rctl(adapter); 6591 e1000e_set_rx_mode(netdev); 6592 6593 /* turn on all-multi mode if wake on multicast is enabled */ 6594 if (wufc & E1000_WUFC_MC) { 6595 rctl = er32(RCTL); 6596 rctl |= E1000_RCTL_MPE; 6597 ew32(RCTL, rctl); 6598 } 6599 6600 ctrl = er32(CTRL); 6601 ctrl |= E1000_CTRL_ADVD3WUC; 6602 if (!(adapter->flags2 & FLAG2_HAS_PHY_WAKEUP)) 6603 ctrl |= E1000_CTRL_EN_PHY_PWR_MGMT; 6604 ew32(CTRL, ctrl); 6605 6606 if (adapter->hw.phy.media_type == e1000_media_type_fiber || 6607 adapter->hw.phy.media_type == 6608 e1000_media_type_internal_serdes) { 6609 /* keep the laser running in D3 */ 6610 ctrl_ext = er32(CTRL_EXT); 6611 ctrl_ext |= E1000_CTRL_EXT_SDP3_DATA; 6612 ew32(CTRL_EXT, ctrl_ext); 6613 } 6614 6615 if (!runtime) 6616 e1000e_power_up_phy(adapter); 6617 6618 if (adapter->flags & FLAG_IS_ICH) 6619 e1000_suspend_workarounds_ich8lan(&adapter->hw); 6620 6621 if (adapter->flags2 & FLAG2_HAS_PHY_WAKEUP) { 6622 /* enable wakeup by the PHY */ 6623 retval = e1000_init_phy_wakeup(adapter, wufc); 6624 if (retval) 6625 return retval; 6626 } else { 6627 /* enable wakeup by the MAC */ 6628 ew32(WUFC, wufc); 6629 ew32(WUC, E1000_WUC_PME_EN); 6630 } 6631 } else { 6632 ew32(WUC, 0); 6633 ew32(WUFC, 0); 6634 6635 e1000_power_down_phy(adapter); 6636 } 6637 6638 if (adapter->hw.phy.type == e1000_phy_igp_3) { 6639 e1000e_igp3_phy_powerdown_workaround_ich8lan(&adapter->hw); 6640 } else if (hw->mac.type >= e1000_pch_lpt) { 6641 if (wufc && !(wufc & (E1000_WUFC_EX | E1000_WUFC_MC | E1000_WUFC_BC))) 6642 /* ULP does not support wake from unicast, multicast 6643 * or broadcast. 6644 */ 6645 retval = e1000_enable_ulp_lpt_lp(hw, !runtime); 6646 6647 if (retval) 6648 return retval; 6649 } 6650 6651 /* Ensure that the appropriate bits are set in LPI_CTRL 6652 * for EEE in Sx 6653 */ 6654 if ((hw->phy.type >= e1000_phy_i217) && 6655 adapter->eee_advert && hw->dev_spec.ich8lan.eee_lp_ability) { 6656 u16 lpi_ctrl = 0; 6657 6658 retval = hw->phy.ops.acquire(hw); 6659 if (!retval) { 6660 retval = e1e_rphy_locked(hw, I82579_LPI_CTRL, 6661 &lpi_ctrl); 6662 if (!retval) { 6663 if (adapter->eee_advert & 6664 hw->dev_spec.ich8lan.eee_lp_ability & 6665 I82579_EEE_100_SUPPORTED) 6666 lpi_ctrl |= I82579_LPI_CTRL_100_ENABLE; 6667 if (adapter->eee_advert & 6668 hw->dev_spec.ich8lan.eee_lp_ability & 6669 I82579_EEE_1000_SUPPORTED) 6670 lpi_ctrl |= I82579_LPI_CTRL_1000_ENABLE; 6671 6672 retval = e1e_wphy_locked(hw, I82579_LPI_CTRL, 6673 lpi_ctrl); 6674 } 6675 } 6676 hw->phy.ops.release(hw); 6677 } 6678 6679 /* Release control of h/w to f/w. If f/w is AMT enabled, this 6680 * would have already happened in close and is redundant. 6681 */ 6682 e1000e_release_hw_control(adapter); 6683 6684 pci_clear_master(pdev); 6685 6686 /* The pci-e switch on some quad port adapters will report a 6687 * correctable error when the MAC transitions from D0 to D3. To 6688 * prevent this we need to mask off the correctable errors on the 6689 * downstream port of the pci-e switch. 6690 * 6691 * We don't have the associated upstream bridge while assigning 6692 * the PCI device into guest. For example, the KVM on power is 6693 * one of the cases. 6694 */ 6695 if (adapter->flags & FLAG_IS_QUAD_PORT) { 6696 struct pci_dev *us_dev = pdev->bus->self; 6697 u16 devctl; 6698 6699 if (!us_dev) 6700 return 0; 6701 6702 pcie_capability_read_word(us_dev, PCI_EXP_DEVCTL, &devctl); 6703 pcie_capability_write_word(us_dev, PCI_EXP_DEVCTL, 6704 (devctl & ~PCI_EXP_DEVCTL_CERE)); 6705 6706 pci_save_state(pdev); 6707 pci_prepare_to_sleep(pdev); 6708 6709 pcie_capability_write_word(us_dev, PCI_EXP_DEVCTL, devctl); 6710 } 6711 6712 return 0; 6713} 6714 6715/** 6716 * __e1000e_disable_aspm - Disable ASPM states 6717 * @pdev: pointer to PCI device struct 6718 * @state: bit-mask of ASPM states to disable 6719 * @locked: indication if this context holds pci_bus_sem locked. 6720 * 6721 * Some devices *must* have certain ASPM states disabled per hardware errata. 6722 **/ 6723static void __e1000e_disable_aspm(struct pci_dev *pdev, u16 state, int locked) 6724{ 6725 struct pci_dev *parent = pdev->bus->self; 6726 u16 aspm_dis_mask = 0; 6727 u16 pdev_aspmc, parent_aspmc; 6728 6729 switch (state) { 6730 case PCIE_LINK_STATE_L0S: 6731 case PCIE_LINK_STATE_L0S | PCIE_LINK_STATE_L1: 6732 aspm_dis_mask |= PCI_EXP_LNKCTL_ASPM_L0S; 6733 fallthrough; /* can't have L1 without L0s */ 6734 case PCIE_LINK_STATE_L1: 6735 aspm_dis_mask |= PCI_EXP_LNKCTL_ASPM_L1; 6736 break; 6737 default: 6738 return; 6739 } 6740 6741 pcie_capability_read_word(pdev, PCI_EXP_LNKCTL, &pdev_aspmc); 6742 pdev_aspmc &= PCI_EXP_LNKCTL_ASPMC; 6743 6744 if (parent) { 6745 pcie_capability_read_word(parent, PCI_EXP_LNKCTL, 6746 &parent_aspmc); 6747 parent_aspmc &= PCI_EXP_LNKCTL_ASPMC; 6748 } 6749 6750 /* Nothing to do if the ASPM states to be disabled already are */ 6751 if (!(pdev_aspmc & aspm_dis_mask) && 6752 (!parent || !(parent_aspmc & aspm_dis_mask))) 6753 return; 6754 6755 dev_info(&pdev->dev, "Disabling ASPM %s %s\n", 6756 (aspm_dis_mask & pdev_aspmc & PCI_EXP_LNKCTL_ASPM_L0S) ? 6757 "L0s" : "", 6758 (aspm_dis_mask & pdev_aspmc & PCI_EXP_LNKCTL_ASPM_L1) ? 6759 "L1" : ""); 6760 6761#ifdef CONFIG_PCIEASPM 6762 if (locked) 6763 pci_disable_link_state_locked(pdev, state); 6764 else 6765 pci_disable_link_state(pdev, state); 6766 6767 /* Double-check ASPM control. If not disabled by the above, the 6768 * BIOS is preventing that from happening (or CONFIG_PCIEASPM is 6769 * not enabled); override by writing PCI config space directly. 6770 */ 6771 pcie_capability_read_word(pdev, PCI_EXP_LNKCTL, &pdev_aspmc); 6772 pdev_aspmc &= PCI_EXP_LNKCTL_ASPMC; 6773 6774 if (!(aspm_dis_mask & pdev_aspmc)) 6775 return; 6776#endif 6777 6778 /* Both device and parent should have the same ASPM setting. 6779 * Disable ASPM in downstream component first and then upstream. 6780 */ 6781 pcie_capability_clear_word(pdev, PCI_EXP_LNKCTL, aspm_dis_mask); 6782 6783 if (parent) 6784 pcie_capability_clear_word(parent, PCI_EXP_LNKCTL, 6785 aspm_dis_mask); 6786} 6787 6788/** 6789 * e1000e_disable_aspm - Disable ASPM states. 6790 * @pdev: pointer to PCI device struct 6791 * @state: bit-mask of ASPM states to disable 6792 * 6793 * This function acquires the pci_bus_sem! 6794 * Some devices *must* have certain ASPM states disabled per hardware errata. 6795 **/ 6796static void e1000e_disable_aspm(struct pci_dev *pdev, u16 state) 6797{ 6798 __e1000e_disable_aspm(pdev, state, 0); 6799} 6800 6801/** 6802 * e1000e_disable_aspm_locked Disable ASPM states. 6803 * @pdev: pointer to PCI device struct 6804 * @state: bit-mask of ASPM states to disable 6805 * 6806 * This function must be called with pci_bus_sem acquired! 6807 * Some devices *must* have certain ASPM states disabled per hardware errata. 6808 **/ 6809static void e1000e_disable_aspm_locked(struct pci_dev *pdev, u16 state) 6810{ 6811 __e1000e_disable_aspm(pdev, state, 1); 6812} 6813 6814static int e1000e_pm_thaw(struct device *dev) 6815{ 6816 struct net_device *netdev = dev_get_drvdata(dev); 6817 struct e1000_adapter *adapter = netdev_priv(netdev); 6818 int rc = 0; 6819 6820 e1000e_set_interrupt_capability(adapter); 6821 6822 rtnl_lock(); 6823 if (netif_running(netdev)) { 6824 rc = e1000_request_irq(adapter); 6825 if (rc) 6826 goto err_irq; 6827 6828 e1000e_up(adapter); 6829 } 6830 6831 netif_device_attach(netdev); 6832err_irq: 6833 rtnl_unlock(); 6834 6835 return rc; 6836} 6837 6838static int __e1000_resume(struct pci_dev *pdev) 6839{ 6840 struct net_device *netdev = pci_get_drvdata(pdev); 6841 struct e1000_adapter *adapter = netdev_priv(netdev); 6842 struct e1000_hw *hw = &adapter->hw; 6843 u16 aspm_disable_flag = 0; 6844 6845 if (adapter->flags2 & FLAG2_DISABLE_ASPM_L0S) 6846 aspm_disable_flag = PCIE_LINK_STATE_L0S; 6847 if (adapter->flags2 & FLAG2_DISABLE_ASPM_L1) 6848 aspm_disable_flag |= PCIE_LINK_STATE_L1; 6849 if (aspm_disable_flag) 6850 e1000e_disable_aspm(pdev, aspm_disable_flag); 6851 6852 pci_set_master(pdev); 6853 6854 if (hw->mac.type >= e1000_pch2lan) 6855 e1000_resume_workarounds_pchlan(&adapter->hw); 6856 6857 e1000e_power_up_phy(adapter); 6858 6859 /* report the system wakeup cause from S3/S4 */ 6860 if (adapter->flags2 & FLAG2_HAS_PHY_WAKEUP) { 6861 u16 phy_data; 6862 6863 e1e_rphy(&adapter->hw, BM_WUS, &phy_data); 6864 if (phy_data) { 6865 e_info("PHY Wakeup cause - %s\n", 6866 phy_data & E1000_WUS_EX ? "Unicast Packet" : 6867 phy_data & E1000_WUS_MC ? "Multicast Packet" : 6868 phy_data & E1000_WUS_BC ? "Broadcast Packet" : 6869 phy_data & E1000_WUS_MAG ? "Magic Packet" : 6870 phy_data & E1000_WUS_LNKC ? 6871 "Link Status Change" : "other"); 6872 } 6873 e1e_wphy(&adapter->hw, BM_WUS, ~0); 6874 } else { 6875 u32 wus = er32(WUS); 6876 6877 if (wus) { 6878 e_info("MAC Wakeup cause - %s\n", 6879 wus & E1000_WUS_EX ? "Unicast Packet" : 6880 wus & E1000_WUS_MC ? "Multicast Packet" : 6881 wus & E1000_WUS_BC ? "Broadcast Packet" : 6882 wus & E1000_WUS_MAG ? "Magic Packet" : 6883 wus & E1000_WUS_LNKC ? "Link Status Change" : 6884 "other"); 6885 } 6886 ew32(WUS, ~0); 6887 } 6888 6889 e1000e_reset(adapter); 6890 6891 e1000_init_manageability_pt(adapter); 6892 6893 /* If the controller has AMT, do not set DRV_LOAD until the interface 6894 * is up. For all other cases, let the f/w know that the h/w is now 6895 * under the control of the driver. 6896 */ 6897 if (!(adapter->flags & FLAG_HAS_AMT)) 6898 e1000e_get_hw_control(adapter); 6899 6900 return 0; 6901} 6902 6903static __maybe_unused int e1000e_pm_suspend(struct device *dev) 6904{ 6905 struct net_device *netdev = pci_get_drvdata(to_pci_dev(dev)); 6906 struct e1000_adapter *adapter = netdev_priv(netdev); 6907 struct pci_dev *pdev = to_pci_dev(dev); 6908 int rc; 6909 6910 e1000e_flush_lpic(pdev); 6911 6912 e1000e_pm_freeze(dev); 6913 6914 rc = __e1000_shutdown(pdev, false); 6915 if (rc) { 6916 e1000e_pm_thaw(dev); 6917 } else { 6918 /* Introduce S0ix implementation */ 6919 if (adapter->flags2 & FLAG2_ENABLE_S0IX_FLOWS) 6920 e1000e_s0ix_entry_flow(adapter); 6921 } 6922 6923 return rc; 6924} 6925 6926static __maybe_unused int e1000e_pm_resume(struct device *dev) 6927{ 6928 struct net_device *netdev = pci_get_drvdata(to_pci_dev(dev)); 6929 struct e1000_adapter *adapter = netdev_priv(netdev); 6930 struct pci_dev *pdev = to_pci_dev(dev); 6931 int rc; 6932 6933 /* Introduce S0ix implementation */ 6934 if (adapter->flags2 & FLAG2_ENABLE_S0IX_FLOWS) 6935 e1000e_s0ix_exit_flow(adapter); 6936 6937 rc = __e1000_resume(pdev); 6938 if (rc) 6939 return rc; 6940 6941 return e1000e_pm_thaw(dev); 6942} 6943 6944static __maybe_unused int e1000e_pm_runtime_idle(struct device *dev) 6945{ 6946 struct net_device *netdev = dev_get_drvdata(dev); 6947 struct e1000_adapter *adapter = netdev_priv(netdev); 6948 u16 eee_lp; 6949 6950 eee_lp = adapter->hw.dev_spec.ich8lan.eee_lp_ability; 6951 6952 if (!e1000e_has_link(adapter)) { 6953 adapter->hw.dev_spec.ich8lan.eee_lp_ability = eee_lp; 6954 pm_schedule_suspend(dev, 5 * MSEC_PER_SEC); 6955 } 6956 6957 return -EBUSY; 6958} 6959 6960static __maybe_unused int e1000e_pm_runtime_resume(struct device *dev) 6961{ 6962 struct pci_dev *pdev = to_pci_dev(dev); 6963 struct net_device *netdev = pci_get_drvdata(pdev); 6964 struct e1000_adapter *adapter = netdev_priv(netdev); 6965 int rc; 6966 6967 rc = __e1000_resume(pdev); 6968 if (rc) 6969 return rc; 6970 6971 if (netdev->flags & IFF_UP) 6972 e1000e_up(adapter); 6973 6974 return rc; 6975} 6976 6977static __maybe_unused int e1000e_pm_runtime_suspend(struct device *dev) 6978{ 6979 struct pci_dev *pdev = to_pci_dev(dev); 6980 struct net_device *netdev = pci_get_drvdata(pdev); 6981 struct e1000_adapter *adapter = netdev_priv(netdev); 6982 6983 if (netdev->flags & IFF_UP) { 6984 int count = E1000_CHECK_RESET_COUNT; 6985 6986 while (test_bit(__E1000_RESETTING, &adapter->state) && count--) 6987 usleep_range(10000, 11000); 6988 6989 WARN_ON(test_bit(__E1000_RESETTING, &adapter->state)); 6990 6991 /* Down the device without resetting the hardware */ 6992 e1000e_down(adapter, false); 6993 } 6994 6995 if (__e1000_shutdown(pdev, true)) { 6996 e1000e_pm_runtime_resume(dev); 6997 return -EBUSY; 6998 } 6999 7000 return 0; 7001} 7002 7003static void e1000_shutdown(struct pci_dev *pdev) 7004{ 7005 e1000e_flush_lpic(pdev); 7006 7007 e1000e_pm_freeze(&pdev->dev); 7008 7009 __e1000_shutdown(pdev, false); 7010} 7011 7012#ifdef CONFIG_NET_POLL_CONTROLLER 7013 7014static irqreturn_t e1000_intr_msix(int __always_unused irq, void *data) 7015{ 7016 struct net_device *netdev = data; 7017 struct e1000_adapter *adapter = netdev_priv(netdev); 7018 7019 if (adapter->msix_entries) { 7020 int vector, msix_irq; 7021 7022 vector = 0; 7023 msix_irq = adapter->msix_entries[vector].vector; 7024 if (disable_hardirq(msix_irq)) 7025 e1000_intr_msix_rx(msix_irq, netdev); 7026 enable_irq(msix_irq); 7027 7028 vector++; 7029 msix_irq = adapter->msix_entries[vector].vector; 7030 if (disable_hardirq(msix_irq)) 7031 e1000_intr_msix_tx(msix_irq, netdev); 7032 enable_irq(msix_irq); 7033 7034 vector++; 7035 msix_irq = adapter->msix_entries[vector].vector; 7036 if (disable_hardirq(msix_irq)) 7037 e1000_msix_other(msix_irq, netdev); 7038 enable_irq(msix_irq); 7039 } 7040 7041 return IRQ_HANDLED; 7042} 7043 7044/** 7045 * e1000_netpoll 7046 * @netdev: network interface device structure 7047 * 7048 * Polling 'interrupt' - used by things like netconsole to send skbs 7049 * without having to re-enable interrupts. It's not called while 7050 * the interrupt routine is executing. 7051 */ 7052static void e1000_netpoll(struct net_device *netdev) 7053{ 7054 struct e1000_adapter *adapter = netdev_priv(netdev); 7055 7056 switch (adapter->int_mode) { 7057 case E1000E_INT_MODE_MSIX: 7058 e1000_intr_msix(adapter->pdev->irq, netdev); 7059 break; 7060 case E1000E_INT_MODE_MSI: 7061 if (disable_hardirq(adapter->pdev->irq)) 7062 e1000_intr_msi(adapter->pdev->irq, netdev); 7063 enable_irq(adapter->pdev->irq); 7064 break; 7065 default: /* E1000E_INT_MODE_LEGACY */ 7066 if (disable_hardirq(adapter->pdev->irq)) 7067 e1000_intr(adapter->pdev->irq, netdev); 7068 enable_irq(adapter->pdev->irq); 7069 break; 7070 } 7071} 7072#endif 7073 7074/** 7075 * e1000_io_error_detected - called when PCI error is detected 7076 * @pdev: Pointer to PCI device 7077 * @state: The current pci connection state 7078 * 7079 * This function is called after a PCI bus error affecting 7080 * this device has been detected. 7081 */ 7082static pci_ers_result_t e1000_io_error_detected(struct pci_dev *pdev, 7083 pci_channel_state_t state) 7084{ 7085 e1000e_pm_freeze(&pdev->dev); 7086 7087 if (state == pci_channel_io_perm_failure) 7088 return PCI_ERS_RESULT_DISCONNECT; 7089 7090 pci_disable_device(pdev); 7091 7092 /* Request a slot slot reset. */ 7093 return PCI_ERS_RESULT_NEED_RESET; 7094} 7095 7096/** 7097 * e1000_io_slot_reset - called after the pci bus has been reset. 7098 * @pdev: Pointer to PCI device 7099 * 7100 * Restart the card from scratch, as if from a cold-boot. Implementation 7101 * resembles the first-half of the e1000e_pm_resume routine. 7102 */ 7103static pci_ers_result_t e1000_io_slot_reset(struct pci_dev *pdev) 7104{ 7105 struct net_device *netdev = pci_get_drvdata(pdev); 7106 struct e1000_adapter *adapter = netdev_priv(netdev); 7107 struct e1000_hw *hw = &adapter->hw; 7108 u16 aspm_disable_flag = 0; 7109 int err; 7110 pci_ers_result_t result; 7111 7112 if (adapter->flags2 & FLAG2_DISABLE_ASPM_L0S) 7113 aspm_disable_flag = PCIE_LINK_STATE_L0S; 7114 if (adapter->flags2 & FLAG2_DISABLE_ASPM_L1) 7115 aspm_disable_flag |= PCIE_LINK_STATE_L1; 7116 if (aspm_disable_flag) 7117 e1000e_disable_aspm_locked(pdev, aspm_disable_flag); 7118 7119 err = pci_enable_device_mem(pdev); 7120 if (err) { 7121 dev_err(&pdev->dev, 7122 "Cannot re-enable PCI device after reset.\n"); 7123 result = PCI_ERS_RESULT_DISCONNECT; 7124 } else { 7125 pdev->state_saved = true; 7126 pci_restore_state(pdev); 7127 pci_set_master(pdev); 7128 7129 pci_enable_wake(pdev, PCI_D3hot, 0); 7130 pci_enable_wake(pdev, PCI_D3cold, 0); 7131 7132 e1000e_reset(adapter); 7133 ew32(WUS, ~0); 7134 result = PCI_ERS_RESULT_RECOVERED; 7135 } 7136 7137 return result; 7138} 7139 7140/** 7141 * e1000_io_resume - called when traffic can start flowing again. 7142 * @pdev: Pointer to PCI device 7143 * 7144 * This callback is called when the error recovery driver tells us that 7145 * its OK to resume normal operation. Implementation resembles the 7146 * second-half of the e1000e_pm_resume routine. 7147 */ 7148static void e1000_io_resume(struct pci_dev *pdev) 7149{ 7150 struct net_device *netdev = pci_get_drvdata(pdev); 7151 struct e1000_adapter *adapter = netdev_priv(netdev); 7152 7153 e1000_init_manageability_pt(adapter); 7154 7155 e1000e_pm_thaw(&pdev->dev); 7156 7157 /* If the controller has AMT, do not set DRV_LOAD until the interface 7158 * is up. For all other cases, let the f/w know that the h/w is now 7159 * under the control of the driver. 7160 */ 7161 if (!(adapter->flags & FLAG_HAS_AMT)) 7162 e1000e_get_hw_control(adapter); 7163} 7164 7165static void e1000_print_device_info(struct e1000_adapter *adapter) 7166{ 7167 struct e1000_hw *hw = &adapter->hw; 7168 struct net_device *netdev = adapter->netdev; 7169 u32 ret_val; 7170 u8 pba_str[E1000_PBANUM_LENGTH]; 7171 7172 /* print bus type/speed/width info */ 7173 e_info("(PCI Express:2.5GT/s:%s) %pM\n", 7174 /* bus width */ 7175 ((hw->bus.width == e1000_bus_width_pcie_x4) ? "Width x4" : 7176 "Width x1"), 7177 /* MAC address */ 7178 netdev->dev_addr); 7179 e_info("Intel(R) PRO/%s Network Connection\n", 7180 (hw->phy.type == e1000_phy_ife) ? "10/100" : "1000"); 7181 ret_val = e1000_read_pba_string_generic(hw, pba_str, 7182 E1000_PBANUM_LENGTH); 7183 if (ret_val) 7184 strlcpy((char *)pba_str, "Unknown", sizeof(pba_str)); 7185 e_info("MAC: %d, PHY: %d, PBA No: %s\n", 7186 hw->mac.type, hw->phy.type, pba_str); 7187} 7188 7189static void e1000_eeprom_checks(struct e1000_adapter *adapter) 7190{ 7191 struct e1000_hw *hw = &adapter->hw; 7192 int ret_val; 7193 u16 buf = 0; 7194 7195 if (hw->mac.type != e1000_82573) 7196 return; 7197 7198 ret_val = e1000_read_nvm(hw, NVM_INIT_CONTROL2_REG, 1, &buf); 7199 le16_to_cpus(&buf); 7200 if (!ret_val && (!(buf & BIT(0)))) { 7201 /* Deep Smart Power Down (DSPD) */ 7202 dev_warn(&adapter->pdev->dev, 7203 "Warning: detected DSPD enabled in EEPROM\n"); 7204 } 7205} 7206 7207static netdev_features_t e1000_fix_features(struct net_device *netdev, 7208 netdev_features_t features) 7209{ 7210 struct e1000_adapter *adapter = netdev_priv(netdev); 7211 struct e1000_hw *hw = &adapter->hw; 7212 7213 /* Jumbo frame workaround on 82579 and newer requires CRC be stripped */ 7214 if ((hw->mac.type >= e1000_pch2lan) && (netdev->mtu > ETH_DATA_LEN)) 7215 features &= ~NETIF_F_RXFCS; 7216 7217 /* Since there is no support for separate Rx/Tx vlan accel 7218 * enable/disable make sure Tx flag is always in same state as Rx. 7219 */ 7220 if (features & NETIF_F_HW_VLAN_CTAG_RX) 7221 features |= NETIF_F_HW_VLAN_CTAG_TX; 7222 else 7223 features &= ~NETIF_F_HW_VLAN_CTAG_TX; 7224 7225 return features; 7226} 7227 7228static int e1000_set_features(struct net_device *netdev, 7229 netdev_features_t features) 7230{ 7231 struct e1000_adapter *adapter = netdev_priv(netdev); 7232 netdev_features_t changed = features ^ netdev->features; 7233 7234 if (changed & (NETIF_F_TSO | NETIF_F_TSO6)) 7235 adapter->flags |= FLAG_TSO_FORCE; 7236 7237 if (!(changed & (NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_CTAG_TX | 7238 NETIF_F_RXCSUM | NETIF_F_RXHASH | NETIF_F_RXFCS | 7239 NETIF_F_RXALL))) 7240 return 0; 7241 7242 if (changed & NETIF_F_RXFCS) { 7243 if (features & NETIF_F_RXFCS) { 7244 adapter->flags2 &= ~FLAG2_CRC_STRIPPING; 7245 } else { 7246 /* We need to take it back to defaults, which might mean 7247 * stripping is still disabled at the adapter level. 7248 */ 7249 if (adapter->flags2 & FLAG2_DFLT_CRC_STRIPPING) 7250 adapter->flags2 |= FLAG2_CRC_STRIPPING; 7251 else 7252 adapter->flags2 &= ~FLAG2_CRC_STRIPPING; 7253 } 7254 } 7255 7256 netdev->features = features; 7257 7258 if (netif_running(netdev)) 7259 e1000e_reinit_locked(adapter); 7260 else 7261 e1000e_reset(adapter); 7262 7263 return 1; 7264} 7265 7266static const struct net_device_ops e1000e_netdev_ops = { 7267 .ndo_open = e1000e_open, 7268 .ndo_stop = e1000e_close, 7269 .ndo_start_xmit = e1000_xmit_frame, 7270 .ndo_get_stats64 = e1000e_get_stats64, 7271 .ndo_set_rx_mode = e1000e_set_rx_mode, 7272 .ndo_set_mac_address = e1000_set_mac, 7273 .ndo_change_mtu = e1000_change_mtu, 7274 .ndo_do_ioctl = e1000_ioctl, 7275 .ndo_tx_timeout = e1000_tx_timeout, 7276 .ndo_validate_addr = eth_validate_addr, 7277 7278 .ndo_vlan_rx_add_vid = e1000_vlan_rx_add_vid, 7279 .ndo_vlan_rx_kill_vid = e1000_vlan_rx_kill_vid, 7280#ifdef CONFIG_NET_POLL_CONTROLLER 7281 .ndo_poll_controller = e1000_netpoll, 7282#endif 7283 .ndo_set_features = e1000_set_features, 7284 .ndo_fix_features = e1000_fix_features, 7285 .ndo_features_check = passthru_features_check, 7286}; 7287 7288/** 7289 * e1000_probe - Device Initialization Routine 7290 * @pdev: PCI device information struct 7291 * @ent: entry in e1000_pci_tbl 7292 * 7293 * Returns 0 on success, negative on failure 7294 * 7295 * e1000_probe initializes an adapter identified by a pci_dev structure. 7296 * The OS initialization, configuring of the adapter private structure, 7297 * and a hardware reset occur. 7298 **/ 7299static int e1000_probe(struct pci_dev *pdev, const struct pci_device_id *ent) 7300{ 7301 struct net_device *netdev; 7302 struct e1000_adapter *adapter; 7303 struct e1000_hw *hw; 7304 const struct e1000_info *ei = e1000_info_tbl[ent->driver_data]; 7305 resource_size_t mmio_start, mmio_len; 7306 resource_size_t flash_start, flash_len; 7307 static int cards_found; 7308 u16 aspm_disable_flag = 0; 7309 int bars, i, err, pci_using_dac; 7310 u16 eeprom_data = 0; 7311 u16 eeprom_apme_mask = E1000_EEPROM_APME; 7312 s32 ret_val = 0; 7313 7314 if (ei->flags2 & FLAG2_DISABLE_ASPM_L0S) 7315 aspm_disable_flag = PCIE_LINK_STATE_L0S; 7316 if (ei->flags2 & FLAG2_DISABLE_ASPM_L1) 7317 aspm_disable_flag |= PCIE_LINK_STATE_L1; 7318 if (aspm_disable_flag) 7319 e1000e_disable_aspm(pdev, aspm_disable_flag); 7320 7321 err = pci_enable_device_mem(pdev); 7322 if (err) 7323 return err; 7324 7325 pci_using_dac = 0; 7326 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); 7327 if (!err) { 7328 pci_using_dac = 1; 7329 } else { 7330 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); 7331 if (err) { 7332 dev_err(&pdev->dev, 7333 "No usable DMA configuration, aborting\n"); 7334 goto err_dma; 7335 } 7336 } 7337 7338 bars = pci_select_bars(pdev, IORESOURCE_MEM); 7339 err = pci_request_selected_regions_exclusive(pdev, bars, 7340 e1000e_driver_name); 7341 if (err) 7342 goto err_pci_reg; 7343 7344 /* AER (Advanced Error Reporting) hooks */ 7345 pci_enable_pcie_error_reporting(pdev); 7346 7347 pci_set_master(pdev); 7348 /* PCI config space info */ 7349 err = pci_save_state(pdev); 7350 if (err) 7351 goto err_alloc_etherdev; 7352 7353 err = -ENOMEM; 7354 netdev = alloc_etherdev(sizeof(struct e1000_adapter)); 7355 if (!netdev) 7356 goto err_alloc_etherdev; 7357 7358 SET_NETDEV_DEV(netdev, &pdev->dev); 7359 7360 netdev->irq = pdev->irq; 7361 7362 pci_set_drvdata(pdev, netdev); 7363 adapter = netdev_priv(netdev); 7364 hw = &adapter->hw; 7365 adapter->netdev = netdev; 7366 adapter->pdev = pdev; 7367 adapter->ei = ei; 7368 adapter->pba = ei->pba; 7369 adapter->flags = ei->flags; 7370 adapter->flags2 = ei->flags2; 7371 adapter->hw.adapter = adapter; 7372 adapter->hw.mac.type = ei->mac; 7373 adapter->max_hw_frame_size = ei->max_hw_frame_size; 7374 adapter->msg_enable = netif_msg_init(debug, DEFAULT_MSG_ENABLE); 7375 7376 mmio_start = pci_resource_start(pdev, 0); 7377 mmio_len = pci_resource_len(pdev, 0); 7378 7379 err = -EIO; 7380 adapter->hw.hw_addr = ioremap(mmio_start, mmio_len); 7381 if (!adapter->hw.hw_addr) 7382 goto err_ioremap; 7383 7384 if ((adapter->flags & FLAG_HAS_FLASH) && 7385 (pci_resource_flags(pdev, 1) & IORESOURCE_MEM) && 7386 (hw->mac.type < e1000_pch_spt)) { 7387 flash_start = pci_resource_start(pdev, 1); 7388 flash_len = pci_resource_len(pdev, 1); 7389 adapter->hw.flash_address = ioremap(flash_start, flash_len); 7390 if (!adapter->hw.flash_address) 7391 goto err_flashmap; 7392 } 7393 7394 /* Set default EEE advertisement */ 7395 if (adapter->flags2 & FLAG2_HAS_EEE) 7396 adapter->eee_advert = MDIO_EEE_100TX | MDIO_EEE_1000T; 7397 7398 /* construct the net_device struct */ 7399 netdev->netdev_ops = &e1000e_netdev_ops; 7400 e1000e_set_ethtool_ops(netdev); 7401 netdev->watchdog_timeo = 5 * HZ; 7402 netif_napi_add(netdev, &adapter->napi, e1000e_poll, 64); 7403 strlcpy(netdev->name, pci_name(pdev), sizeof(netdev->name)); 7404 7405 netdev->mem_start = mmio_start; 7406 netdev->mem_end = mmio_start + mmio_len; 7407 7408 adapter->bd_number = cards_found++; 7409 7410 e1000e_check_options(adapter); 7411 7412 /* setup adapter struct */ 7413 err = e1000_sw_init(adapter); 7414 if (err) 7415 goto err_sw_init; 7416 7417 memcpy(&hw->mac.ops, ei->mac_ops, sizeof(hw->mac.ops)); 7418 memcpy(&hw->nvm.ops, ei->nvm_ops, sizeof(hw->nvm.ops)); 7419 memcpy(&hw->phy.ops, ei->phy_ops, sizeof(hw->phy.ops)); 7420 7421 err = ei->get_variants(adapter); 7422 if (err) 7423 goto err_hw_init; 7424 7425 if ((adapter->flags & FLAG_IS_ICH) && 7426 (adapter->flags & FLAG_READ_ONLY_NVM) && 7427 (hw->mac.type < e1000_pch_spt)) 7428 e1000e_write_protect_nvm_ich8lan(&adapter->hw); 7429 7430 hw->mac.ops.get_bus_info(&adapter->hw); 7431 7432 adapter->hw.phy.autoneg_wait_to_complete = 0; 7433 7434 /* Copper options */ 7435 if (adapter->hw.phy.media_type == e1000_media_type_copper) { 7436 adapter->hw.phy.mdix = AUTO_ALL_MODES; 7437 adapter->hw.phy.disable_polarity_correction = 0; 7438 adapter->hw.phy.ms_type = e1000_ms_hw_default; 7439 } 7440 7441 if (hw->phy.ops.check_reset_block && hw->phy.ops.check_reset_block(hw)) 7442 dev_info(&pdev->dev, 7443 "PHY reset is blocked due to SOL/IDER session.\n"); 7444 7445 /* Set initial default active device features */ 7446 netdev->features = (NETIF_F_SG | 7447 NETIF_F_HW_VLAN_CTAG_RX | 7448 NETIF_F_HW_VLAN_CTAG_TX | 7449 NETIF_F_TSO | 7450 NETIF_F_TSO6 | 7451 NETIF_F_RXHASH | 7452 NETIF_F_RXCSUM | 7453 NETIF_F_HW_CSUM); 7454 7455 /* disable TSO for pcie and 10/100 speeds to avoid 7456 * some hardware issues and for i219 to fix transfer 7457 * speed being capped at 60% 7458 */ 7459 if (!(adapter->flags & FLAG_TSO_FORCE)) { 7460 switch (adapter->link_speed) { 7461 case SPEED_10: 7462 case SPEED_100: 7463 e_info("10/100 speed: disabling TSO\n"); 7464 netdev->features &= ~NETIF_F_TSO; 7465 netdev->features &= ~NETIF_F_TSO6; 7466 break; 7467 case SPEED_1000: 7468 netdev->features |= NETIF_F_TSO; 7469 netdev->features |= NETIF_F_TSO6; 7470 break; 7471 default: 7472 /* oops */ 7473 break; 7474 } 7475 if (hw->mac.type == e1000_pch_spt) { 7476 netdev->features &= ~NETIF_F_TSO; 7477 netdev->features &= ~NETIF_F_TSO6; 7478 } 7479 } 7480 7481 /* Set user-changeable features (subset of all device features) */ 7482 netdev->hw_features = netdev->features; 7483 netdev->hw_features |= NETIF_F_RXFCS; 7484 netdev->priv_flags |= IFF_SUPP_NOFCS; 7485 netdev->hw_features |= NETIF_F_RXALL; 7486 7487 if (adapter->flags & FLAG_HAS_HW_VLAN_FILTER) 7488 netdev->features |= NETIF_F_HW_VLAN_CTAG_FILTER; 7489 7490 netdev->vlan_features |= (NETIF_F_SG | 7491 NETIF_F_TSO | 7492 NETIF_F_TSO6 | 7493 NETIF_F_HW_CSUM); 7494 7495 netdev->priv_flags |= IFF_UNICAST_FLT; 7496 7497 if (pci_using_dac) { 7498 netdev->features |= NETIF_F_HIGHDMA; 7499 netdev->vlan_features |= NETIF_F_HIGHDMA; 7500 } 7501 7502 /* MTU range: 68 - max_hw_frame_size */ 7503 netdev->min_mtu = ETH_MIN_MTU; 7504 netdev->max_mtu = adapter->max_hw_frame_size - 7505 (VLAN_ETH_HLEN + ETH_FCS_LEN); 7506 7507 if (e1000e_enable_mng_pass_thru(&adapter->hw)) 7508 adapter->flags |= FLAG_MNG_PT_ENABLED; 7509 7510 /* before reading the NVM, reset the controller to 7511 * put the device in a known good starting state 7512 */ 7513 adapter->hw.mac.ops.reset_hw(&adapter->hw); 7514 7515 /* systems with ASPM and others may see the checksum fail on the first 7516 * attempt. Let's give it a few tries 7517 */ 7518 for (i = 0;; i++) { 7519 if (e1000_validate_nvm_checksum(&adapter->hw) >= 0) 7520 break; 7521 if (i == 2) { 7522 dev_err(&pdev->dev, "The NVM Checksum Is Not Valid\n"); 7523 err = -EIO; 7524 goto err_eeprom; 7525 } 7526 } 7527 7528 e1000_eeprom_checks(adapter); 7529 7530 /* copy the MAC address */ 7531 if (e1000e_read_mac_addr(&adapter->hw)) 7532 dev_err(&pdev->dev, 7533 "NVM Read Error while reading MAC address\n"); 7534 7535 memcpy(netdev->dev_addr, adapter->hw.mac.addr, netdev->addr_len); 7536 7537 if (!is_valid_ether_addr(netdev->dev_addr)) { 7538 dev_err(&pdev->dev, "Invalid MAC Address: %pM\n", 7539 netdev->dev_addr); 7540 err = -EIO; 7541 goto err_eeprom; 7542 } 7543 7544 timer_setup(&adapter->watchdog_timer, e1000_watchdog, 0); 7545 timer_setup(&adapter->phy_info_timer, e1000_update_phy_info, 0); 7546 7547 INIT_WORK(&adapter->reset_task, e1000_reset_task); 7548 INIT_WORK(&adapter->watchdog_task, e1000_watchdog_task); 7549 INIT_WORK(&adapter->downshift_task, e1000e_downshift_workaround); 7550 INIT_WORK(&adapter->update_phy_task, e1000e_update_phy_task); 7551 INIT_WORK(&adapter->print_hang_task, e1000_print_hw_hang); 7552 7553 /* Initialize link parameters. User can change them with ethtool */ 7554 adapter->hw.mac.autoneg = 1; 7555 adapter->fc_autoneg = true; 7556 adapter->hw.fc.requested_mode = e1000_fc_default; 7557 adapter->hw.fc.current_mode = e1000_fc_default; 7558 adapter->hw.phy.autoneg_advertised = 0x2f; 7559 7560 /* Initial Wake on LAN setting - If APM wake is enabled in 7561 * the EEPROM, enable the ACPI Magic Packet filter 7562 */ 7563 if (adapter->flags & FLAG_APME_IN_WUC) { 7564 /* APME bit in EEPROM is mapped to WUC.APME */ 7565 eeprom_data = er32(WUC); 7566 eeprom_apme_mask = E1000_WUC_APME; 7567 if ((hw->mac.type > e1000_ich10lan) && 7568 (eeprom_data & E1000_WUC_PHY_WAKE)) 7569 adapter->flags2 |= FLAG2_HAS_PHY_WAKEUP; 7570 } else if (adapter->flags & FLAG_APME_IN_CTRL3) { 7571 if (adapter->flags & FLAG_APME_CHECK_PORT_B && 7572 (adapter->hw.bus.func == 1)) 7573 ret_val = e1000_read_nvm(&adapter->hw, 7574 NVM_INIT_CONTROL3_PORT_B, 7575 1, &eeprom_data); 7576 else 7577 ret_val = e1000_read_nvm(&adapter->hw, 7578 NVM_INIT_CONTROL3_PORT_A, 7579 1, &eeprom_data); 7580 } 7581 7582 /* fetch WoL from EEPROM */ 7583 if (ret_val) 7584 e_dbg("NVM read error getting WoL initial values: %d\n", ret_val); 7585 else if (eeprom_data & eeprom_apme_mask) 7586 adapter->eeprom_wol |= E1000_WUFC_MAG; 7587 7588 /* now that we have the eeprom settings, apply the special cases 7589 * where the eeprom may be wrong or the board simply won't support 7590 * wake on lan on a particular port 7591 */ 7592 if (!(adapter->flags & FLAG_HAS_WOL)) 7593 adapter->eeprom_wol = 0; 7594 7595 /* initialize the wol settings based on the eeprom settings */ 7596 adapter->wol = adapter->eeprom_wol; 7597 7598 /* make sure adapter isn't asleep if manageability is enabled */ 7599 if (adapter->wol || (adapter->flags & FLAG_MNG_PT_ENABLED) || 7600 (hw->mac.ops.check_mng_mode(hw))) 7601 device_wakeup_enable(&pdev->dev); 7602 7603 /* save off EEPROM version number */ 7604 ret_val = e1000_read_nvm(&adapter->hw, 5, 1, &adapter->eeprom_vers); 7605 7606 if (ret_val) { 7607 e_dbg("NVM read error getting EEPROM version: %d\n", ret_val); 7608 adapter->eeprom_vers = 0; 7609 } 7610 7611 /* init PTP hardware clock */ 7612 e1000e_ptp_init(adapter); 7613 7614 /* reset the hardware with the new settings */ 7615 e1000e_reset(adapter); 7616 7617 /* If the controller has AMT, do not set DRV_LOAD until the interface 7618 * is up. For all other cases, let the f/w know that the h/w is now 7619 * under the control of the driver. 7620 */ 7621 if (!(adapter->flags & FLAG_HAS_AMT)) 7622 e1000e_get_hw_control(adapter); 7623 7624 if (hw->mac.type >= e1000_pch_cnp) 7625 adapter->flags2 |= FLAG2_ENABLE_S0IX_FLOWS; 7626 7627 strlcpy(netdev->name, "eth%d", sizeof(netdev->name)); 7628 err = register_netdev(netdev); 7629 if (err) 7630 goto err_register; 7631 7632 /* carrier off reporting is important to ethtool even BEFORE open */ 7633 netif_carrier_off(netdev); 7634 7635 e1000_print_device_info(adapter); 7636 7637 dev_pm_set_driver_flags(&pdev->dev, DPM_FLAG_NO_DIRECT_COMPLETE); 7638 7639 if (pci_dev_run_wake(pdev) && hw->mac.type < e1000_pch_cnp) 7640 pm_runtime_put_noidle(&pdev->dev); 7641 7642 return 0; 7643 7644err_register: 7645 if (!(adapter->flags & FLAG_HAS_AMT)) 7646 e1000e_release_hw_control(adapter); 7647err_eeprom: 7648 if (hw->phy.ops.check_reset_block && !hw->phy.ops.check_reset_block(hw)) 7649 e1000_phy_hw_reset(&adapter->hw); 7650err_hw_init: 7651 kfree(adapter->tx_ring); 7652 kfree(adapter->rx_ring); 7653err_sw_init: 7654 if ((adapter->hw.flash_address) && (hw->mac.type < e1000_pch_spt)) 7655 iounmap(adapter->hw.flash_address); 7656 e1000e_reset_interrupt_capability(adapter); 7657err_flashmap: 7658 iounmap(adapter->hw.hw_addr); 7659err_ioremap: 7660 free_netdev(netdev); 7661err_alloc_etherdev: 7662 pci_disable_pcie_error_reporting(pdev); 7663 pci_release_mem_regions(pdev); 7664err_pci_reg: 7665err_dma: 7666 pci_disable_device(pdev); 7667 return err; 7668} 7669 7670/** 7671 * e1000_remove - Device Removal Routine 7672 * @pdev: PCI device information struct 7673 * 7674 * e1000_remove is called by the PCI subsystem to alert the driver 7675 * that it should release a PCI device. The could be caused by a 7676 * Hot-Plug event, or because the driver is going to be removed from 7677 * memory. 7678 **/ 7679static void e1000_remove(struct pci_dev *pdev) 7680{ 7681 struct net_device *netdev = pci_get_drvdata(pdev); 7682 struct e1000_adapter *adapter = netdev_priv(netdev); 7683 7684 e1000e_ptp_remove(adapter); 7685 7686 /* The timers may be rescheduled, so explicitly disable them 7687 * from being rescheduled. 7688 */ 7689 set_bit(__E1000_DOWN, &adapter->state); 7690 del_timer_sync(&adapter->watchdog_timer); 7691 del_timer_sync(&adapter->phy_info_timer); 7692 7693 cancel_work_sync(&adapter->reset_task); 7694 cancel_work_sync(&adapter->watchdog_task); 7695 cancel_work_sync(&adapter->downshift_task); 7696 cancel_work_sync(&adapter->update_phy_task); 7697 cancel_work_sync(&adapter->print_hang_task); 7698 7699 if (adapter->flags & FLAG_HAS_HW_TIMESTAMP) { 7700 cancel_work_sync(&adapter->tx_hwtstamp_work); 7701 if (adapter->tx_hwtstamp_skb) { 7702 dev_consume_skb_any(adapter->tx_hwtstamp_skb); 7703 adapter->tx_hwtstamp_skb = NULL; 7704 } 7705 } 7706 7707 unregister_netdev(netdev); 7708 7709 if (pci_dev_run_wake(pdev)) 7710 pm_runtime_get_noresume(&pdev->dev); 7711 7712 /* Release control of h/w to f/w. If f/w is AMT enabled, this 7713 * would have already happened in close and is redundant. 7714 */ 7715 e1000e_release_hw_control(adapter); 7716 7717 e1000e_reset_interrupt_capability(adapter); 7718 kfree(adapter->tx_ring); 7719 kfree(adapter->rx_ring); 7720 7721 iounmap(adapter->hw.hw_addr); 7722 if ((adapter->hw.flash_address) && 7723 (adapter->hw.mac.type < e1000_pch_spt)) 7724 iounmap(adapter->hw.flash_address); 7725 pci_release_mem_regions(pdev); 7726 7727 free_netdev(netdev); 7728 7729 /* AER disable */ 7730 pci_disable_pcie_error_reporting(pdev); 7731 7732 pci_disable_device(pdev); 7733} 7734 7735/* PCI Error Recovery (ERS) */ 7736static const struct pci_error_handlers e1000_err_handler = { 7737 .error_detected = e1000_io_error_detected, 7738 .slot_reset = e1000_io_slot_reset, 7739 .resume = e1000_io_resume, 7740}; 7741 7742static const struct pci_device_id e1000_pci_tbl[] = { 7743 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82571EB_COPPER), board_82571 }, 7744 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82571EB_FIBER), board_82571 }, 7745 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82571EB_QUAD_COPPER), board_82571 }, 7746 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82571EB_QUAD_COPPER_LP), 7747 board_82571 }, 7748 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82571EB_QUAD_FIBER), board_82571 }, 7749 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82571EB_SERDES), board_82571 }, 7750 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82571EB_SERDES_DUAL), board_82571 }, 7751 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82571EB_SERDES_QUAD), board_82571 }, 7752 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82571PT_QUAD_COPPER), board_82571 }, 7753 7754 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82572EI), board_82572 }, 7755 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82572EI_COPPER), board_82572 }, 7756 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82572EI_FIBER), board_82572 }, 7757 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82572EI_SERDES), board_82572 }, 7758 7759 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82573E), board_82573 }, 7760 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82573E_IAMT), board_82573 }, 7761 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82573L), board_82573 }, 7762 7763 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82574L), board_82574 }, 7764 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82574LA), board_82574 }, 7765 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82583V), board_82583 }, 7766 7767 { PCI_VDEVICE(INTEL, E1000_DEV_ID_80003ES2LAN_COPPER_DPT), 7768 board_80003es2lan }, 7769 { PCI_VDEVICE(INTEL, E1000_DEV_ID_80003ES2LAN_COPPER_SPT), 7770 board_80003es2lan }, 7771 { PCI_VDEVICE(INTEL, E1000_DEV_ID_80003ES2LAN_SERDES_DPT), 7772 board_80003es2lan }, 7773 { PCI_VDEVICE(INTEL, E1000_DEV_ID_80003ES2LAN_SERDES_SPT), 7774 board_80003es2lan }, 7775 7776 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH8_IFE), board_ich8lan }, 7777 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH8_IFE_G), board_ich8lan }, 7778 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH8_IFE_GT), board_ich8lan }, 7779 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH8_IGP_AMT), board_ich8lan }, 7780 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH8_IGP_C), board_ich8lan }, 7781 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH8_IGP_M), board_ich8lan }, 7782 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH8_IGP_M_AMT), board_ich8lan }, 7783 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH8_82567V_3), board_ich8lan }, 7784 7785 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH9_IFE), board_ich9lan }, 7786 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH9_IFE_G), board_ich9lan }, 7787 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH9_IFE_GT), board_ich9lan }, 7788 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH9_IGP_AMT), board_ich9lan }, 7789 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH9_IGP_C), board_ich9lan }, 7790 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH9_BM), board_ich9lan }, 7791 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH9_IGP_M), board_ich9lan }, 7792 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH9_IGP_M_AMT), board_ich9lan }, 7793 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH9_IGP_M_V), board_ich9lan }, 7794 7795 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH10_R_BM_LM), board_ich9lan }, 7796 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH10_R_BM_LF), board_ich9lan }, 7797 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH10_R_BM_V), board_ich9lan }, 7798 7799 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH10_D_BM_LM), board_ich10lan }, 7800 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH10_D_BM_LF), board_ich10lan }, 7801 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH10_D_BM_V), board_ich10lan }, 7802 7803 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_M_HV_LM), board_pchlan }, 7804 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_M_HV_LC), board_pchlan }, 7805 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_D_HV_DM), board_pchlan }, 7806 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_D_HV_DC), board_pchlan }, 7807 7808 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH2_LV_LM), board_pch2lan }, 7809 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH2_LV_V), board_pch2lan }, 7810 7811 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_LPT_I217_LM), board_pch_lpt }, 7812 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_LPT_I217_V), board_pch_lpt }, 7813 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_LPTLP_I218_LM), board_pch_lpt }, 7814 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_LPTLP_I218_V), board_pch_lpt }, 7815 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_I218_LM2), board_pch_lpt }, 7816 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_I218_V2), board_pch_lpt }, 7817 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_I218_LM3), board_pch_lpt }, 7818 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_I218_V3), board_pch_lpt }, 7819 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_SPT_I219_LM), board_pch_spt }, 7820 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_SPT_I219_V), board_pch_spt }, 7821 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_SPT_I219_LM2), board_pch_spt }, 7822 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_SPT_I219_V2), board_pch_spt }, 7823 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_LBG_I219_LM3), board_pch_spt }, 7824 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_SPT_I219_LM4), board_pch_spt }, 7825 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_SPT_I219_V4), board_pch_spt }, 7826 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_SPT_I219_LM5), board_pch_spt }, 7827 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_SPT_I219_V5), board_pch_spt }, 7828 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_CNP_I219_LM6), board_pch_cnp }, 7829 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_CNP_I219_V6), board_pch_cnp }, 7830 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_CNP_I219_LM7), board_pch_cnp }, 7831 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_CNP_I219_V7), board_pch_cnp }, 7832 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_ICP_I219_LM8), board_pch_cnp }, 7833 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_ICP_I219_V8), board_pch_cnp }, 7834 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_ICP_I219_LM9), board_pch_cnp }, 7835 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_ICP_I219_V9), board_pch_cnp }, 7836 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_CMP_I219_LM10), board_pch_cnp }, 7837 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_CMP_I219_V10), board_pch_cnp }, 7838 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_CMP_I219_LM11), board_pch_cnp }, 7839 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_CMP_I219_V11), board_pch_cnp }, 7840 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_CMP_I219_LM12), board_pch_spt }, 7841 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_CMP_I219_V12), board_pch_spt }, 7842 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_TGP_I219_LM13), board_pch_tgp }, 7843 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_TGP_I219_V13), board_pch_tgp }, 7844 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_TGP_I219_LM14), board_pch_tgp }, 7845 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_TGP_I219_V14), board_pch_tgp }, 7846 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_TGP_I219_LM15), board_pch_tgp }, 7847 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_TGP_I219_V15), board_pch_tgp }, 7848 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_ADP_I219_LM16), board_pch_tgp }, 7849 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_ADP_I219_V16), board_pch_tgp }, 7850 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_ADP_I219_LM17), board_pch_tgp }, 7851 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_ADP_I219_V17), board_pch_tgp }, 7852 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_MTP_I219_LM18), board_pch_tgp }, 7853 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_MTP_I219_V18), board_pch_tgp }, 7854 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_MTP_I219_LM19), board_pch_tgp }, 7855 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_MTP_I219_V19), board_pch_tgp }, 7856 7857 { 0, 0, 0, 0, 0, 0, 0 } /* terminate list */ 7858}; 7859MODULE_DEVICE_TABLE(pci, e1000_pci_tbl); 7860 7861static const struct dev_pm_ops e1000_pm_ops = { 7862#ifdef CONFIG_PM_SLEEP 7863 .suspend = e1000e_pm_suspend, 7864 .resume = e1000e_pm_resume, 7865 .freeze = e1000e_pm_freeze, 7866 .thaw = e1000e_pm_thaw, 7867 .poweroff = e1000e_pm_suspend, 7868 .restore = e1000e_pm_resume, 7869#endif 7870 SET_RUNTIME_PM_OPS(e1000e_pm_runtime_suspend, e1000e_pm_runtime_resume, 7871 e1000e_pm_runtime_idle) 7872}; 7873 7874/* PCI Device API Driver */ 7875static struct pci_driver e1000_driver = { 7876 .name = e1000e_driver_name, 7877 .id_table = e1000_pci_tbl, 7878 .probe = e1000_probe, 7879 .remove = e1000_remove, 7880 .driver = { 7881 .pm = &e1000_pm_ops, 7882 }, 7883 .shutdown = e1000_shutdown, 7884 .err_handler = &e1000_err_handler 7885}; 7886 7887/** 7888 * e1000_init_module - Driver Registration Routine 7889 * 7890 * e1000_init_module is the first routine called when the driver is 7891 * loaded. All it does is register with the PCI subsystem. 7892 **/ 7893static int __init e1000_init_module(void) 7894{ 7895 pr_info("Intel(R) PRO/1000 Network Driver\n"); 7896 pr_info("Copyright(c) 1999 - 2015 Intel Corporation.\n"); 7897 7898 return pci_register_driver(&e1000_driver); 7899} 7900module_init(e1000_init_module); 7901 7902/** 7903 * e1000_exit_module - Driver Exit Cleanup Routine 7904 * 7905 * e1000_exit_module is called just before the driver is removed 7906 * from memory. 7907 **/ 7908static void __exit e1000_exit_module(void) 7909{ 7910 pci_unregister_driver(&e1000_driver); 7911} 7912module_exit(e1000_exit_module); 7913 7914MODULE_AUTHOR("Intel Corporation, <linux.nics@intel.com>"); 7915MODULE_DESCRIPTION("Intel(R) PRO/1000 Network Driver"); 7916MODULE_LICENSE("GPL v2"); 7917 7918/* netdev.c */ 7919