1// SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause) 2/* Copyright 2017-2019 NXP */ 3 4#include "enetc.h" 5#include <linux/tcp.h> 6#include <linux/udp.h> 7#include <linux/vmalloc.h> 8#include <net/pkt_sched.h> 9 10/* ENETC overhead: optional extension BD + 1 BD gap */ 11#define ENETC_TXBDS_NEEDED(val) ((val) + 2) 12/* max # of chained Tx BDs is 15, including head and extension BD */ 13#define ENETC_MAX_SKB_FRAGS 13 14#define ENETC_TXBDS_MAX_NEEDED ENETC_TXBDS_NEEDED(ENETC_MAX_SKB_FRAGS + 1) 15 16static int enetc_map_tx_buffs(struct enetc_bdr *tx_ring, struct sk_buff *skb, 17 int active_offloads); 18 19netdev_tx_t enetc_xmit(struct sk_buff *skb, struct net_device *ndev) 20{ 21 struct enetc_ndev_priv *priv = netdev_priv(ndev); 22 struct enetc_bdr *tx_ring; 23 int count; 24 25 tx_ring = priv->tx_ring[skb->queue_mapping]; 26 27 if (unlikely(skb_shinfo(skb)->nr_frags > ENETC_MAX_SKB_FRAGS)) 28 if (unlikely(skb_linearize(skb))) 29 goto drop_packet_err; 30 31 count = skb_shinfo(skb)->nr_frags + 1; /* fragments + head */ 32 if (enetc_bd_unused(tx_ring) < ENETC_TXBDS_NEEDED(count)) { 33 netif_stop_subqueue(ndev, tx_ring->index); 34 return NETDEV_TX_BUSY; 35 } 36 37 enetc_lock_mdio(); 38 count = enetc_map_tx_buffs(tx_ring, skb, priv->active_offloads); 39 enetc_unlock_mdio(); 40 41 if (unlikely(!count)) 42 goto drop_packet_err; 43 44 if (enetc_bd_unused(tx_ring) < ENETC_TXBDS_MAX_NEEDED) 45 netif_stop_subqueue(ndev, tx_ring->index); 46 47 return NETDEV_TX_OK; 48 49drop_packet_err: 50 dev_kfree_skb_any(skb); 51 return NETDEV_TX_OK; 52} 53 54static bool enetc_tx_csum(struct sk_buff *skb, union enetc_tx_bd *txbd) 55{ 56 int l3_start, l3_hsize; 57 u16 l3_flags, l4_flags; 58 59 if (skb->ip_summed != CHECKSUM_PARTIAL) 60 return false; 61 62 switch (skb->csum_offset) { 63 case offsetof(struct tcphdr, check): 64 l4_flags = ENETC_TXBD_L4_TCP; 65 break; 66 case offsetof(struct udphdr, check): 67 l4_flags = ENETC_TXBD_L4_UDP; 68 break; 69 default: 70 skb_checksum_help(skb); 71 return false; 72 } 73 74 l3_start = skb_network_offset(skb); 75 l3_hsize = skb_network_header_len(skb); 76 77 l3_flags = 0; 78 if (skb->protocol == htons(ETH_P_IPV6)) 79 l3_flags = ENETC_TXBD_L3_IPV6; 80 81 /* write BD fields */ 82 txbd->l3_csoff = enetc_txbd_l3_csoff(l3_start, l3_hsize, l3_flags); 83 txbd->l4_csoff = l4_flags; 84 85 return true; 86} 87 88static void enetc_unmap_tx_buff(struct enetc_bdr *tx_ring, 89 struct enetc_tx_swbd *tx_swbd) 90{ 91 if (tx_swbd->is_dma_page) 92 dma_unmap_page(tx_ring->dev, tx_swbd->dma, 93 tx_swbd->len, DMA_TO_DEVICE); 94 else 95 dma_unmap_single(tx_ring->dev, tx_swbd->dma, 96 tx_swbd->len, DMA_TO_DEVICE); 97 tx_swbd->dma = 0; 98} 99 100static void enetc_free_tx_skb(struct enetc_bdr *tx_ring, 101 struct enetc_tx_swbd *tx_swbd) 102{ 103 if (tx_swbd->dma) 104 enetc_unmap_tx_buff(tx_ring, tx_swbd); 105 106 if (tx_swbd->skb) { 107 dev_kfree_skb_any(tx_swbd->skb); 108 tx_swbd->skb = NULL; 109 } 110} 111 112static int enetc_map_tx_buffs(struct enetc_bdr *tx_ring, struct sk_buff *skb, 113 int active_offloads) 114{ 115 struct enetc_tx_swbd *tx_swbd; 116 skb_frag_t *frag; 117 int len = skb_headlen(skb); 118 union enetc_tx_bd temp_bd; 119 union enetc_tx_bd *txbd; 120 bool do_vlan, do_tstamp; 121 int i, count = 0; 122 unsigned int f; 123 dma_addr_t dma; 124 u8 flags = 0; 125 126 i = tx_ring->next_to_use; 127 txbd = ENETC_TXBD(*tx_ring, i); 128 prefetchw(txbd); 129 130 dma = dma_map_single(tx_ring->dev, skb->data, len, DMA_TO_DEVICE); 131 if (unlikely(dma_mapping_error(tx_ring->dev, dma))) 132 goto dma_err; 133 134 temp_bd.addr = cpu_to_le64(dma); 135 temp_bd.buf_len = cpu_to_le16(len); 136 temp_bd.lstatus = 0; 137 138 tx_swbd = &tx_ring->tx_swbd[i]; 139 tx_swbd->dma = dma; 140 tx_swbd->len = len; 141 tx_swbd->is_dma_page = 0; 142 count++; 143 144 do_vlan = skb_vlan_tag_present(skb); 145 do_tstamp = (active_offloads & ENETC_F_TX_TSTAMP) && 146 (skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP); 147 tx_swbd->do_tstamp = do_tstamp; 148 tx_swbd->check_wb = tx_swbd->do_tstamp; 149 150 if (do_vlan || do_tstamp) 151 flags |= ENETC_TXBD_FLAGS_EX; 152 153 if (enetc_tx_csum(skb, &temp_bd)) 154 flags |= ENETC_TXBD_FLAGS_CSUM | ENETC_TXBD_FLAGS_L4CS; 155 else if (tx_ring->tsd_enable) 156 flags |= ENETC_TXBD_FLAGS_TSE | ENETC_TXBD_FLAGS_TXSTART; 157 158 /* first BD needs frm_len and offload flags set */ 159 temp_bd.frm_len = cpu_to_le16(skb->len); 160 temp_bd.flags = flags; 161 162 if (flags & ENETC_TXBD_FLAGS_TSE) { 163 u32 temp; 164 165 temp = (skb->skb_mstamp_ns >> 5 & ENETC_TXBD_TXSTART_MASK) 166 | (flags << ENETC_TXBD_FLAGS_OFFSET); 167 temp_bd.txstart = cpu_to_le32(temp); 168 } 169 170 if (flags & ENETC_TXBD_FLAGS_EX) { 171 u8 e_flags = 0; 172 *txbd = temp_bd; 173 enetc_clear_tx_bd(&temp_bd); 174 175 /* add extension BD for VLAN and/or timestamping */ 176 flags = 0; 177 tx_swbd++; 178 txbd++; 179 i++; 180 if (unlikely(i == tx_ring->bd_count)) { 181 i = 0; 182 tx_swbd = tx_ring->tx_swbd; 183 txbd = ENETC_TXBD(*tx_ring, 0); 184 } 185 prefetchw(txbd); 186 187 if (do_vlan) { 188 temp_bd.ext.vid = cpu_to_le16(skb_vlan_tag_get(skb)); 189 temp_bd.ext.tpid = 0; /* < C-TAG */ 190 e_flags |= ENETC_TXBD_E_FLAGS_VLAN_INS; 191 } 192 193 if (do_tstamp) { 194 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; 195 e_flags |= ENETC_TXBD_E_FLAGS_TWO_STEP_PTP; 196 } 197 198 temp_bd.ext.e_flags = e_flags; 199 count++; 200 } 201 202 frag = &skb_shinfo(skb)->frags[0]; 203 for (f = 0; f < skb_shinfo(skb)->nr_frags; f++, frag++) { 204 len = skb_frag_size(frag); 205 dma = skb_frag_dma_map(tx_ring->dev, frag, 0, len, 206 DMA_TO_DEVICE); 207 if (dma_mapping_error(tx_ring->dev, dma)) 208 goto dma_err; 209 210 *txbd = temp_bd; 211 enetc_clear_tx_bd(&temp_bd); 212 213 flags = 0; 214 tx_swbd++; 215 txbd++; 216 i++; 217 if (unlikely(i == tx_ring->bd_count)) { 218 i = 0; 219 tx_swbd = tx_ring->tx_swbd; 220 txbd = ENETC_TXBD(*tx_ring, 0); 221 } 222 prefetchw(txbd); 223 224 temp_bd.addr = cpu_to_le64(dma); 225 temp_bd.buf_len = cpu_to_le16(len); 226 227 tx_swbd->dma = dma; 228 tx_swbd->len = len; 229 tx_swbd->is_dma_page = 1; 230 count++; 231 } 232 233 /* last BD needs 'F' bit set */ 234 flags |= ENETC_TXBD_FLAGS_F; 235 temp_bd.flags = flags; 236 *txbd = temp_bd; 237 238 tx_ring->tx_swbd[i].skb = skb; 239 240 enetc_bdr_idx_inc(tx_ring, &i); 241 tx_ring->next_to_use = i; 242 243 skb_tx_timestamp(skb); 244 245 /* let H/W know BD ring has been updated */ 246 enetc_wr_reg_hot(tx_ring->tpir, i); /* includes wmb() */ 247 248 return count; 249 250dma_err: 251 dev_err(tx_ring->dev, "DMA map error"); 252 253 do { 254 tx_swbd = &tx_ring->tx_swbd[i]; 255 enetc_free_tx_skb(tx_ring, tx_swbd); 256 if (i == 0) 257 i = tx_ring->bd_count; 258 i--; 259 } while (count--); 260 261 return 0; 262} 263 264static irqreturn_t enetc_msix(int irq, void *data) 265{ 266 struct enetc_int_vector *v = data; 267 int i; 268 269 enetc_lock_mdio(); 270 271 /* disable interrupts */ 272 enetc_wr_reg_hot(v->rbier, 0); 273 enetc_wr_reg_hot(v->ricr1, v->rx_ictt); 274 275 for_each_set_bit(i, &v->tx_rings_map, ENETC_MAX_NUM_TXQS) 276 enetc_wr_reg_hot(v->tbier_base + ENETC_BDR_OFF(i), 0); 277 278 enetc_unlock_mdio(); 279 280 napi_schedule(&v->napi); 281 282 return IRQ_HANDLED; 283} 284 285static bool enetc_clean_tx_ring(struct enetc_bdr *tx_ring, int napi_budget); 286static int enetc_clean_rx_ring(struct enetc_bdr *rx_ring, 287 struct napi_struct *napi, int work_limit); 288 289static void enetc_rx_dim_work(struct work_struct *w) 290{ 291 struct dim *dim = container_of(w, struct dim, work); 292 struct dim_cq_moder moder = 293 net_dim_get_rx_moderation(dim->mode, dim->profile_ix); 294 struct enetc_int_vector *v = 295 container_of(dim, struct enetc_int_vector, rx_dim); 296 297 v->rx_ictt = enetc_usecs_to_cycles(moder.usec); 298 dim->state = DIM_START_MEASURE; 299} 300 301static void enetc_rx_net_dim(struct enetc_int_vector *v) 302{ 303 struct dim_sample dim_sample = {}; 304 305 v->comp_cnt++; 306 307 if (!v->rx_napi_work) 308 return; 309 310 dim_update_sample(v->comp_cnt, 311 v->rx_ring.stats.packets, 312 v->rx_ring.stats.bytes, 313 &dim_sample); 314 net_dim(&v->rx_dim, dim_sample); 315} 316 317static int enetc_poll(struct napi_struct *napi, int budget) 318{ 319 struct enetc_int_vector 320 *v = container_of(napi, struct enetc_int_vector, napi); 321 bool complete = true; 322 int work_done; 323 int i; 324 325 enetc_lock_mdio(); 326 327 for (i = 0; i < v->count_tx_rings; i++) 328 if (!enetc_clean_tx_ring(&v->tx_ring[i], budget)) 329 complete = false; 330 331 work_done = enetc_clean_rx_ring(&v->rx_ring, napi, budget); 332 if (work_done == budget) 333 complete = false; 334 if (work_done) 335 v->rx_napi_work = true; 336 337 if (!complete) { 338 enetc_unlock_mdio(); 339 return budget; 340 } 341 342 napi_complete_done(napi, work_done); 343 344 if (likely(v->rx_dim_en)) 345 enetc_rx_net_dim(v); 346 347 v->rx_napi_work = false; 348 349 /* enable interrupts */ 350 enetc_wr_reg_hot(v->rbier, ENETC_RBIER_RXTIE); 351 352 for_each_set_bit(i, &v->tx_rings_map, ENETC_MAX_NUM_TXQS) 353 enetc_wr_reg_hot(v->tbier_base + ENETC_BDR_OFF(i), 354 ENETC_TBIER_TXTIE); 355 356 enetc_unlock_mdio(); 357 358 return work_done; 359} 360 361static int enetc_bd_ready_count(struct enetc_bdr *tx_ring, int ci) 362{ 363 int pi = enetc_rd_reg_hot(tx_ring->tcir) & ENETC_TBCIR_IDX_MASK; 364 365 return pi >= ci ? pi - ci : tx_ring->bd_count - ci + pi; 366} 367 368static void enetc_get_tx_tstamp(struct enetc_hw *hw, union enetc_tx_bd *txbd, 369 u64 *tstamp) 370{ 371 u32 lo, hi, tstamp_lo; 372 373 lo = enetc_rd_hot(hw, ENETC_SICTR0); 374 hi = enetc_rd_hot(hw, ENETC_SICTR1); 375 tstamp_lo = le32_to_cpu(txbd->wb.tstamp); 376 if (lo <= tstamp_lo) 377 hi -= 1; 378 *tstamp = (u64)hi << 32 | tstamp_lo; 379} 380 381static void enetc_tstamp_tx(struct sk_buff *skb, u64 tstamp) 382{ 383 struct skb_shared_hwtstamps shhwtstamps; 384 385 if (skb_shinfo(skb)->tx_flags & SKBTX_IN_PROGRESS) { 386 memset(&shhwtstamps, 0, sizeof(shhwtstamps)); 387 shhwtstamps.hwtstamp = ns_to_ktime(tstamp); 388 skb_txtime_consumed(skb); 389 skb_tstamp_tx(skb, &shhwtstamps); 390 } 391} 392 393static bool enetc_clean_tx_ring(struct enetc_bdr *tx_ring, int napi_budget) 394{ 395 struct net_device *ndev = tx_ring->ndev; 396 int tx_frm_cnt = 0, tx_byte_cnt = 0; 397 struct enetc_tx_swbd *tx_swbd; 398 int i, bds_to_clean; 399 bool do_tstamp; 400 u64 tstamp = 0; 401 402 i = tx_ring->next_to_clean; 403 tx_swbd = &tx_ring->tx_swbd[i]; 404 405 bds_to_clean = enetc_bd_ready_count(tx_ring, i); 406 407 do_tstamp = false; 408 409 while (bds_to_clean && tx_frm_cnt < ENETC_DEFAULT_TX_WORK) { 410 bool is_eof = !!tx_swbd->skb; 411 412 if (unlikely(tx_swbd->check_wb)) { 413 struct enetc_ndev_priv *priv = netdev_priv(ndev); 414 union enetc_tx_bd *txbd; 415 416 txbd = ENETC_TXBD(*tx_ring, i); 417 418 if (txbd->flags & ENETC_TXBD_FLAGS_W && 419 tx_swbd->do_tstamp) { 420 enetc_get_tx_tstamp(&priv->si->hw, txbd, 421 &tstamp); 422 do_tstamp = true; 423 } 424 } 425 426 if (likely(tx_swbd->dma)) 427 enetc_unmap_tx_buff(tx_ring, tx_swbd); 428 429 if (is_eof) { 430 if (unlikely(do_tstamp)) { 431 enetc_tstamp_tx(tx_swbd->skb, tstamp); 432 do_tstamp = false; 433 } 434 napi_consume_skb(tx_swbd->skb, napi_budget); 435 tx_swbd->skb = NULL; 436 } 437 438 tx_byte_cnt += tx_swbd->len; 439 440 bds_to_clean--; 441 tx_swbd++; 442 i++; 443 if (unlikely(i == tx_ring->bd_count)) { 444 i = 0; 445 tx_swbd = tx_ring->tx_swbd; 446 } 447 448 /* BD iteration loop end */ 449 if (is_eof) { 450 tx_frm_cnt++; 451 /* re-arm interrupt source */ 452 enetc_wr_reg_hot(tx_ring->idr, BIT(tx_ring->index) | 453 BIT(16 + tx_ring->index)); 454 } 455 456 if (unlikely(!bds_to_clean)) 457 bds_to_clean = enetc_bd_ready_count(tx_ring, i); 458 } 459 460 tx_ring->next_to_clean = i; 461 tx_ring->stats.packets += tx_frm_cnt; 462 tx_ring->stats.bytes += tx_byte_cnt; 463 464 if (unlikely(tx_frm_cnt && netif_carrier_ok(ndev) && 465 __netif_subqueue_stopped(ndev, tx_ring->index) && 466 (enetc_bd_unused(tx_ring) >= ENETC_TXBDS_MAX_NEEDED))) { 467 netif_wake_subqueue(ndev, tx_ring->index); 468 } 469 470 return tx_frm_cnt != ENETC_DEFAULT_TX_WORK; 471} 472 473static bool enetc_new_page(struct enetc_bdr *rx_ring, 474 struct enetc_rx_swbd *rx_swbd) 475{ 476 struct page *page; 477 dma_addr_t addr; 478 479 page = dev_alloc_page(); 480 if (unlikely(!page)) 481 return false; 482 483 addr = dma_map_page(rx_ring->dev, page, 0, PAGE_SIZE, DMA_FROM_DEVICE); 484 if (unlikely(dma_mapping_error(rx_ring->dev, addr))) { 485 __free_page(page); 486 487 return false; 488 } 489 490 rx_swbd->dma = addr; 491 rx_swbd->page = page; 492 rx_swbd->page_offset = ENETC_RXB_PAD; 493 494 return true; 495} 496 497static int enetc_refill_rx_ring(struct enetc_bdr *rx_ring, const int buff_cnt) 498{ 499 struct enetc_rx_swbd *rx_swbd; 500 union enetc_rx_bd *rxbd; 501 int i, j; 502 503 i = rx_ring->next_to_use; 504 rx_swbd = &rx_ring->rx_swbd[i]; 505 rxbd = enetc_rxbd(rx_ring, i); 506 507 for (j = 0; j < buff_cnt; j++) { 508 /* try reuse page */ 509 if (unlikely(!rx_swbd->page)) { 510 if (unlikely(!enetc_new_page(rx_ring, rx_swbd))) { 511 rx_ring->stats.rx_alloc_errs++; 512 break; 513 } 514 } 515 516 /* update RxBD */ 517 rxbd->w.addr = cpu_to_le64(rx_swbd->dma + 518 rx_swbd->page_offset); 519 /* clear 'R" as well */ 520 rxbd->r.lstatus = 0; 521 522 rxbd = enetc_rxbd_next(rx_ring, rxbd, i); 523 rx_swbd++; 524 i++; 525 if (unlikely(i == rx_ring->bd_count)) { 526 i = 0; 527 rx_swbd = rx_ring->rx_swbd; 528 } 529 } 530 531 if (likely(j)) { 532 rx_ring->next_to_alloc = i; /* keep track from page reuse */ 533 rx_ring->next_to_use = i; 534 } 535 536 return j; 537} 538 539#ifdef CONFIG_FSL_ENETC_PTP_CLOCK 540static void enetc_get_rx_tstamp(struct net_device *ndev, 541 union enetc_rx_bd *rxbd, 542 struct sk_buff *skb) 543{ 544 struct skb_shared_hwtstamps *shhwtstamps = skb_hwtstamps(skb); 545 struct enetc_ndev_priv *priv = netdev_priv(ndev); 546 struct enetc_hw *hw = &priv->si->hw; 547 u32 lo, hi, tstamp_lo; 548 u64 tstamp; 549 550 if (le16_to_cpu(rxbd->r.flags) & ENETC_RXBD_FLAG_TSTMP) { 551 lo = enetc_rd_reg_hot(hw->reg + ENETC_SICTR0); 552 hi = enetc_rd_reg_hot(hw->reg + ENETC_SICTR1); 553 rxbd = enetc_rxbd_ext(rxbd); 554 tstamp_lo = le32_to_cpu(rxbd->ext.tstamp); 555 if (lo <= tstamp_lo) 556 hi -= 1; 557 558 tstamp = (u64)hi << 32 | tstamp_lo; 559 memset(shhwtstamps, 0, sizeof(*shhwtstamps)); 560 shhwtstamps->hwtstamp = ns_to_ktime(tstamp); 561 } 562} 563#endif 564 565static void enetc_get_offloads(struct enetc_bdr *rx_ring, 566 union enetc_rx_bd *rxbd, struct sk_buff *skb) 567{ 568 struct enetc_ndev_priv *priv = netdev_priv(rx_ring->ndev); 569 570 /* TODO: hashing */ 571 if (rx_ring->ndev->features & NETIF_F_RXCSUM) { 572 u16 inet_csum = le16_to_cpu(rxbd->r.inet_csum); 573 574 skb->csum = csum_unfold((__force __sum16)~htons(inet_csum)); 575 skb->ip_summed = CHECKSUM_COMPLETE; 576 } 577 578 if (le16_to_cpu(rxbd->r.flags) & ENETC_RXBD_FLAG_VLAN) { 579 __be16 tpid = 0; 580 581 switch (le16_to_cpu(rxbd->r.flags) & ENETC_RXBD_FLAG_TPID) { 582 case 0: 583 tpid = htons(ETH_P_8021Q); 584 break; 585 case 1: 586 tpid = htons(ETH_P_8021AD); 587 break; 588 case 2: 589 tpid = htons(enetc_port_rd(&priv->si->hw, 590 ENETC_PCVLANR1)); 591 break; 592 case 3: 593 tpid = htons(enetc_port_rd(&priv->si->hw, 594 ENETC_PCVLANR2)); 595 break; 596 default: 597 break; 598 } 599 600 __vlan_hwaccel_put_tag(skb, tpid, le16_to_cpu(rxbd->r.vlan_opt)); 601 } 602 603#ifdef CONFIG_FSL_ENETC_PTP_CLOCK 604 if (priv->active_offloads & ENETC_F_RX_TSTAMP) 605 enetc_get_rx_tstamp(rx_ring->ndev, rxbd, skb); 606#endif 607} 608 609static void enetc_process_skb(struct enetc_bdr *rx_ring, 610 struct sk_buff *skb) 611{ 612 skb_record_rx_queue(skb, rx_ring->index); 613 skb->protocol = eth_type_trans(skb, rx_ring->ndev); 614} 615 616static bool enetc_page_reusable(struct page *page) 617{ 618 return (!page_is_pfmemalloc(page) && page_ref_count(page) == 1); 619} 620 621static void enetc_reuse_page(struct enetc_bdr *rx_ring, 622 struct enetc_rx_swbd *old) 623{ 624 struct enetc_rx_swbd *new; 625 626 new = &rx_ring->rx_swbd[rx_ring->next_to_alloc]; 627 628 /* next buf that may reuse a page */ 629 enetc_bdr_idx_inc(rx_ring, &rx_ring->next_to_alloc); 630 631 /* copy page reference */ 632 *new = *old; 633} 634 635static struct enetc_rx_swbd *enetc_get_rx_buff(struct enetc_bdr *rx_ring, 636 int i, u16 size) 637{ 638 struct enetc_rx_swbd *rx_swbd = &rx_ring->rx_swbd[i]; 639 640 dma_sync_single_range_for_cpu(rx_ring->dev, rx_swbd->dma, 641 rx_swbd->page_offset, 642 size, DMA_FROM_DEVICE); 643 return rx_swbd; 644} 645 646static void enetc_put_rx_buff(struct enetc_bdr *rx_ring, 647 struct enetc_rx_swbd *rx_swbd) 648{ 649 if (likely(enetc_page_reusable(rx_swbd->page))) { 650 rx_swbd->page_offset ^= ENETC_RXB_TRUESIZE; 651 page_ref_inc(rx_swbd->page); 652 653 enetc_reuse_page(rx_ring, rx_swbd); 654 655 /* sync for use by the device */ 656 dma_sync_single_range_for_device(rx_ring->dev, rx_swbd->dma, 657 rx_swbd->page_offset, 658 ENETC_RXB_DMA_SIZE, 659 DMA_FROM_DEVICE); 660 } else { 661 dma_unmap_page(rx_ring->dev, rx_swbd->dma, 662 PAGE_SIZE, DMA_FROM_DEVICE); 663 } 664 665 rx_swbd->page = NULL; 666} 667 668static struct sk_buff *enetc_map_rx_buff_to_skb(struct enetc_bdr *rx_ring, 669 int i, u16 size) 670{ 671 struct enetc_rx_swbd *rx_swbd = enetc_get_rx_buff(rx_ring, i, size); 672 struct sk_buff *skb; 673 void *ba; 674 675 ba = page_address(rx_swbd->page) + rx_swbd->page_offset; 676 skb = build_skb(ba - ENETC_RXB_PAD, ENETC_RXB_TRUESIZE); 677 if (unlikely(!skb)) { 678 rx_ring->stats.rx_alloc_errs++; 679 return NULL; 680 } 681 682 skb_reserve(skb, ENETC_RXB_PAD); 683 __skb_put(skb, size); 684 685 enetc_put_rx_buff(rx_ring, rx_swbd); 686 687 return skb; 688} 689 690static void enetc_add_rx_buff_to_skb(struct enetc_bdr *rx_ring, int i, 691 u16 size, struct sk_buff *skb) 692{ 693 struct enetc_rx_swbd *rx_swbd = enetc_get_rx_buff(rx_ring, i, size); 694 695 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_swbd->page, 696 rx_swbd->page_offset, size, ENETC_RXB_TRUESIZE); 697 698 enetc_put_rx_buff(rx_ring, rx_swbd); 699} 700 701#define ENETC_RXBD_BUNDLE 16 /* # of BDs to update at once */ 702 703static int enetc_clean_rx_ring(struct enetc_bdr *rx_ring, 704 struct napi_struct *napi, int work_limit) 705{ 706 int rx_frm_cnt = 0, rx_byte_cnt = 0; 707 int cleaned_cnt, i; 708 709 cleaned_cnt = enetc_bd_unused(rx_ring); 710 /* next descriptor to process */ 711 i = rx_ring->next_to_clean; 712 713 while (likely(rx_frm_cnt < work_limit)) { 714 union enetc_rx_bd *rxbd; 715 struct sk_buff *skb; 716 u32 bd_status; 717 u16 size; 718 719 if (cleaned_cnt >= ENETC_RXBD_BUNDLE) { 720 int count = enetc_refill_rx_ring(rx_ring, cleaned_cnt); 721 722 /* update ENETC's consumer index */ 723 enetc_wr_reg_hot(rx_ring->rcir, rx_ring->next_to_use); 724 cleaned_cnt -= count; 725 } 726 727 rxbd = enetc_rxbd(rx_ring, i); 728 bd_status = le32_to_cpu(rxbd->r.lstatus); 729 if (!bd_status) 730 break; 731 732 enetc_wr_reg_hot(rx_ring->idr, BIT(rx_ring->index)); 733 dma_rmb(); /* for reading other rxbd fields */ 734 size = le16_to_cpu(rxbd->r.buf_len); 735 skb = enetc_map_rx_buff_to_skb(rx_ring, i, size); 736 if (!skb) 737 break; 738 739 enetc_get_offloads(rx_ring, rxbd, skb); 740 741 cleaned_cnt++; 742 743 rxbd = enetc_rxbd_next(rx_ring, rxbd, i); 744 if (unlikely(++i == rx_ring->bd_count)) 745 i = 0; 746 747 if (unlikely(bd_status & 748 ENETC_RXBD_LSTATUS(ENETC_RXBD_ERR_MASK))) { 749 dev_kfree_skb(skb); 750 while (!(bd_status & ENETC_RXBD_LSTATUS_F)) { 751 dma_rmb(); 752 bd_status = le32_to_cpu(rxbd->r.lstatus); 753 754 rxbd = enetc_rxbd_next(rx_ring, rxbd, i); 755 if (unlikely(++i == rx_ring->bd_count)) 756 i = 0; 757 } 758 759 rx_ring->ndev->stats.rx_dropped++; 760 rx_ring->ndev->stats.rx_errors++; 761 762 break; 763 } 764 765 /* not last BD in frame? */ 766 while (!(bd_status & ENETC_RXBD_LSTATUS_F)) { 767 bd_status = le32_to_cpu(rxbd->r.lstatus); 768 size = ENETC_RXB_DMA_SIZE; 769 770 if (bd_status & ENETC_RXBD_LSTATUS_F) { 771 dma_rmb(); 772 size = le16_to_cpu(rxbd->r.buf_len); 773 } 774 775 enetc_add_rx_buff_to_skb(rx_ring, i, size, skb); 776 777 cleaned_cnt++; 778 779 rxbd = enetc_rxbd_next(rx_ring, rxbd, i); 780 if (unlikely(++i == rx_ring->bd_count)) 781 i = 0; 782 } 783 784 rx_byte_cnt += skb->len; 785 786 enetc_process_skb(rx_ring, skb); 787 788 napi_gro_receive(napi, skb); 789 790 rx_frm_cnt++; 791 } 792 793 rx_ring->next_to_clean = i; 794 795 rx_ring->stats.packets += rx_frm_cnt; 796 rx_ring->stats.bytes += rx_byte_cnt; 797 798 return rx_frm_cnt; 799} 800 801/* Probing and Init */ 802#define ENETC_MAX_RFS_SIZE 64 803void enetc_get_si_caps(struct enetc_si *si) 804{ 805 struct enetc_hw *hw = &si->hw; 806 u32 val; 807 808 /* find out how many of various resources we have to work with */ 809 val = enetc_rd(hw, ENETC_SICAPR0); 810 si->num_rx_rings = (val >> 16) & 0xff; 811 si->num_tx_rings = val & 0xff; 812 813 val = enetc_rd(hw, ENETC_SIRFSCAPR); 814 si->num_fs_entries = ENETC_SIRFSCAPR_GET_NUM_RFS(val); 815 si->num_fs_entries = min(si->num_fs_entries, ENETC_MAX_RFS_SIZE); 816 817 si->num_rss = 0; 818 val = enetc_rd(hw, ENETC_SIPCAPR0); 819 if (val & ENETC_SIPCAPR0_RSS) { 820 u32 rss; 821 822 rss = enetc_rd(hw, ENETC_SIRSSCAPR); 823 si->num_rss = ENETC_SIRSSCAPR_GET_NUM_RSS(rss); 824 } 825 826 if (val & ENETC_SIPCAPR0_QBV) 827 si->hw_features |= ENETC_SI_F_QBV; 828 829 if (val & ENETC_SIPCAPR0_PSFP) 830 si->hw_features |= ENETC_SI_F_PSFP; 831} 832 833static int enetc_dma_alloc_bdr(struct enetc_bdr *r, size_t bd_size) 834{ 835 r->bd_base = dma_alloc_coherent(r->dev, r->bd_count * bd_size, 836 &r->bd_dma_base, GFP_KERNEL); 837 if (!r->bd_base) 838 return -ENOMEM; 839 840 /* h/w requires 128B alignment */ 841 if (!IS_ALIGNED(r->bd_dma_base, 128)) { 842 dma_free_coherent(r->dev, r->bd_count * bd_size, r->bd_base, 843 r->bd_dma_base); 844 return -EINVAL; 845 } 846 847 return 0; 848} 849 850static int enetc_alloc_txbdr(struct enetc_bdr *txr) 851{ 852 int err; 853 854 txr->tx_swbd = vzalloc(txr->bd_count * sizeof(struct enetc_tx_swbd)); 855 if (!txr->tx_swbd) 856 return -ENOMEM; 857 858 err = enetc_dma_alloc_bdr(txr, sizeof(union enetc_tx_bd)); 859 if (err) { 860 vfree(txr->tx_swbd); 861 return err; 862 } 863 864 txr->next_to_clean = 0; 865 txr->next_to_use = 0; 866 867 return 0; 868} 869 870static void enetc_free_txbdr(struct enetc_bdr *txr) 871{ 872 int size, i; 873 874 for (i = 0; i < txr->bd_count; i++) 875 enetc_free_tx_skb(txr, &txr->tx_swbd[i]); 876 877 size = txr->bd_count * sizeof(union enetc_tx_bd); 878 879 dma_free_coherent(txr->dev, size, txr->bd_base, txr->bd_dma_base); 880 txr->bd_base = NULL; 881 882 vfree(txr->tx_swbd); 883 txr->tx_swbd = NULL; 884} 885 886static int enetc_alloc_tx_resources(struct enetc_ndev_priv *priv) 887{ 888 int i, err; 889 890 for (i = 0; i < priv->num_tx_rings; i++) { 891 err = enetc_alloc_txbdr(priv->tx_ring[i]); 892 893 if (err) 894 goto fail; 895 } 896 897 return 0; 898 899fail: 900 while (i-- > 0) 901 enetc_free_txbdr(priv->tx_ring[i]); 902 903 return err; 904} 905 906static void enetc_free_tx_resources(struct enetc_ndev_priv *priv) 907{ 908 int i; 909 910 for (i = 0; i < priv->num_tx_rings; i++) 911 enetc_free_txbdr(priv->tx_ring[i]); 912} 913 914static int enetc_alloc_rxbdr(struct enetc_bdr *rxr, bool extended) 915{ 916 size_t size = sizeof(union enetc_rx_bd); 917 int err; 918 919 rxr->rx_swbd = vzalloc(rxr->bd_count * sizeof(struct enetc_rx_swbd)); 920 if (!rxr->rx_swbd) 921 return -ENOMEM; 922 923 if (extended) 924 size *= 2; 925 926 err = enetc_dma_alloc_bdr(rxr, size); 927 if (err) { 928 vfree(rxr->rx_swbd); 929 return err; 930 } 931 932 rxr->next_to_clean = 0; 933 rxr->next_to_use = 0; 934 rxr->next_to_alloc = 0; 935 rxr->ext_en = extended; 936 937 return 0; 938} 939 940static void enetc_free_rxbdr(struct enetc_bdr *rxr) 941{ 942 int size; 943 944 size = rxr->bd_count * sizeof(union enetc_rx_bd); 945 946 dma_free_coherent(rxr->dev, size, rxr->bd_base, rxr->bd_dma_base); 947 rxr->bd_base = NULL; 948 949 vfree(rxr->rx_swbd); 950 rxr->rx_swbd = NULL; 951} 952 953static int enetc_alloc_rx_resources(struct enetc_ndev_priv *priv) 954{ 955 bool extended = !!(priv->active_offloads & ENETC_F_RX_TSTAMP); 956 int i, err; 957 958 for (i = 0; i < priv->num_rx_rings; i++) { 959 err = enetc_alloc_rxbdr(priv->rx_ring[i], extended); 960 961 if (err) 962 goto fail; 963 } 964 965 return 0; 966 967fail: 968 while (i-- > 0) 969 enetc_free_rxbdr(priv->rx_ring[i]); 970 971 return err; 972} 973 974static void enetc_free_rx_resources(struct enetc_ndev_priv *priv) 975{ 976 int i; 977 978 for (i = 0; i < priv->num_rx_rings; i++) 979 enetc_free_rxbdr(priv->rx_ring[i]); 980} 981 982static void enetc_free_tx_ring(struct enetc_bdr *tx_ring) 983{ 984 int i; 985 986 if (!tx_ring->tx_swbd) 987 return; 988 989 for (i = 0; i < tx_ring->bd_count; i++) { 990 struct enetc_tx_swbd *tx_swbd = &tx_ring->tx_swbd[i]; 991 992 enetc_free_tx_skb(tx_ring, tx_swbd); 993 } 994 995 tx_ring->next_to_clean = 0; 996 tx_ring->next_to_use = 0; 997} 998 999static void enetc_free_rx_ring(struct enetc_bdr *rx_ring) 1000{ 1001 int i; 1002 1003 if (!rx_ring->rx_swbd) 1004 return; 1005 1006 for (i = 0; i < rx_ring->bd_count; i++) { 1007 struct enetc_rx_swbd *rx_swbd = &rx_ring->rx_swbd[i]; 1008 1009 if (!rx_swbd->page) 1010 continue; 1011 1012 dma_unmap_page(rx_ring->dev, rx_swbd->dma, 1013 PAGE_SIZE, DMA_FROM_DEVICE); 1014 __free_page(rx_swbd->page); 1015 rx_swbd->page = NULL; 1016 } 1017 1018 rx_ring->next_to_clean = 0; 1019 rx_ring->next_to_use = 0; 1020 rx_ring->next_to_alloc = 0; 1021} 1022 1023static void enetc_free_rxtx_rings(struct enetc_ndev_priv *priv) 1024{ 1025 int i; 1026 1027 for (i = 0; i < priv->num_rx_rings; i++) 1028 enetc_free_rx_ring(priv->rx_ring[i]); 1029 1030 for (i = 0; i < priv->num_tx_rings; i++) 1031 enetc_free_tx_ring(priv->tx_ring[i]); 1032} 1033 1034int enetc_alloc_cbdr(struct device *dev, struct enetc_cbdr *cbdr) 1035{ 1036 int size = cbdr->bd_count * sizeof(struct enetc_cbd); 1037 1038 cbdr->bd_base = dma_alloc_coherent(dev, size, &cbdr->bd_dma_base, 1039 GFP_KERNEL); 1040 if (!cbdr->bd_base) 1041 return -ENOMEM; 1042 1043 /* h/w requires 128B alignment */ 1044 if (!IS_ALIGNED(cbdr->bd_dma_base, 128)) { 1045 dma_free_coherent(dev, size, cbdr->bd_base, cbdr->bd_dma_base); 1046 return -EINVAL; 1047 } 1048 1049 cbdr->next_to_clean = 0; 1050 cbdr->next_to_use = 0; 1051 1052 return 0; 1053} 1054 1055void enetc_free_cbdr(struct device *dev, struct enetc_cbdr *cbdr) 1056{ 1057 int size = cbdr->bd_count * sizeof(struct enetc_cbd); 1058 1059 dma_free_coherent(dev, size, cbdr->bd_base, cbdr->bd_dma_base); 1060 cbdr->bd_base = NULL; 1061} 1062 1063void enetc_setup_cbdr(struct enetc_hw *hw, struct enetc_cbdr *cbdr) 1064{ 1065 /* set CBDR cache attributes */ 1066 enetc_wr(hw, ENETC_SICAR2, 1067 ENETC_SICAR_RD_COHERENT | ENETC_SICAR_WR_COHERENT); 1068 1069 enetc_wr(hw, ENETC_SICBDRBAR0, lower_32_bits(cbdr->bd_dma_base)); 1070 enetc_wr(hw, ENETC_SICBDRBAR1, upper_32_bits(cbdr->bd_dma_base)); 1071 enetc_wr(hw, ENETC_SICBDRLENR, ENETC_RTBLENR_LEN(cbdr->bd_count)); 1072 1073 enetc_wr(hw, ENETC_SICBDRPIR, 0); 1074 enetc_wr(hw, ENETC_SICBDRCIR, 0); 1075 1076 /* enable ring */ 1077 enetc_wr(hw, ENETC_SICBDRMR, BIT(31)); 1078 1079 cbdr->pir = hw->reg + ENETC_SICBDRPIR; 1080 cbdr->cir = hw->reg + ENETC_SICBDRCIR; 1081} 1082 1083void enetc_clear_cbdr(struct enetc_hw *hw) 1084{ 1085 enetc_wr(hw, ENETC_SICBDRMR, 0); 1086} 1087 1088static int enetc_setup_default_rss_table(struct enetc_si *si, int num_groups) 1089{ 1090 int *rss_table; 1091 int i; 1092 1093 rss_table = kmalloc_array(si->num_rss, sizeof(*rss_table), GFP_KERNEL); 1094 if (!rss_table) 1095 return -ENOMEM; 1096 1097 /* Set up RSS table defaults */ 1098 for (i = 0; i < si->num_rss; i++) 1099 rss_table[i] = i % num_groups; 1100 1101 enetc_set_rss_table(si, rss_table, si->num_rss); 1102 1103 kfree(rss_table); 1104 1105 return 0; 1106} 1107 1108int enetc_configure_si(struct enetc_ndev_priv *priv) 1109{ 1110 struct enetc_si *si = priv->si; 1111 struct enetc_hw *hw = &si->hw; 1112 int err; 1113 1114 /* set SI cache attributes */ 1115 enetc_wr(hw, ENETC_SICAR0, 1116 ENETC_SICAR_RD_COHERENT | ENETC_SICAR_WR_COHERENT); 1117 enetc_wr(hw, ENETC_SICAR1, ENETC_SICAR_MSI); 1118 /* enable SI */ 1119 enetc_wr(hw, ENETC_SIMR, ENETC_SIMR_EN); 1120 1121 if (si->num_rss) { 1122 err = enetc_setup_default_rss_table(si, priv->num_rx_rings); 1123 if (err) 1124 return err; 1125 } 1126 1127 return 0; 1128} 1129 1130void enetc_init_si_rings_params(struct enetc_ndev_priv *priv) 1131{ 1132 struct enetc_si *si = priv->si; 1133 int cpus = num_online_cpus(); 1134 1135 priv->tx_bd_count = ENETC_TX_RING_DEFAULT_SIZE; 1136 priv->rx_bd_count = ENETC_RX_RING_DEFAULT_SIZE; 1137 1138 /* Enable all available TX rings in order to configure as many 1139 * priorities as possible, when needed. 1140 * TODO: Make # of TX rings run-time configurable 1141 */ 1142 priv->num_rx_rings = min_t(int, cpus, si->num_rx_rings); 1143 priv->num_tx_rings = si->num_tx_rings; 1144 priv->bdr_int_num = cpus; 1145 priv->ic_mode = ENETC_IC_RX_ADAPTIVE | ENETC_IC_TX_MANUAL; 1146 priv->tx_ictt = ENETC_TXIC_TIMETHR; 1147 1148 /* SI specific */ 1149 si->cbd_ring.bd_count = ENETC_CBDR_DEFAULT_SIZE; 1150} 1151 1152int enetc_alloc_si_resources(struct enetc_ndev_priv *priv) 1153{ 1154 struct enetc_si *si = priv->si; 1155 int err; 1156 1157 err = enetc_alloc_cbdr(priv->dev, &si->cbd_ring); 1158 if (err) 1159 return err; 1160 1161 enetc_setup_cbdr(&si->hw, &si->cbd_ring); 1162 1163 priv->cls_rules = kcalloc(si->num_fs_entries, sizeof(*priv->cls_rules), 1164 GFP_KERNEL); 1165 if (!priv->cls_rules) { 1166 err = -ENOMEM; 1167 goto err_alloc_cls; 1168 } 1169 1170 return 0; 1171 1172err_alloc_cls: 1173 enetc_clear_cbdr(&si->hw); 1174 enetc_free_cbdr(priv->dev, &si->cbd_ring); 1175 1176 return err; 1177} 1178 1179void enetc_free_si_resources(struct enetc_ndev_priv *priv) 1180{ 1181 struct enetc_si *si = priv->si; 1182 1183 enetc_clear_cbdr(&si->hw); 1184 enetc_free_cbdr(priv->dev, &si->cbd_ring); 1185 1186 kfree(priv->cls_rules); 1187} 1188 1189static void enetc_setup_txbdr(struct enetc_hw *hw, struct enetc_bdr *tx_ring) 1190{ 1191 int idx = tx_ring->index; 1192 u32 tbmr; 1193 1194 enetc_txbdr_wr(hw, idx, ENETC_TBBAR0, 1195 lower_32_bits(tx_ring->bd_dma_base)); 1196 1197 enetc_txbdr_wr(hw, idx, ENETC_TBBAR1, 1198 upper_32_bits(tx_ring->bd_dma_base)); 1199 1200 WARN_ON(!IS_ALIGNED(tx_ring->bd_count, 64)); /* multiple of 64 */ 1201 enetc_txbdr_wr(hw, idx, ENETC_TBLENR, 1202 ENETC_RTBLENR_LEN(tx_ring->bd_count)); 1203 1204 /* clearing PI/CI registers for Tx not supported, adjust sw indexes */ 1205 tx_ring->next_to_use = enetc_txbdr_rd(hw, idx, ENETC_TBPIR); 1206 tx_ring->next_to_clean = enetc_txbdr_rd(hw, idx, ENETC_TBCIR); 1207 1208 /* enable Tx ints by setting pkt thr to 1 */ 1209 enetc_txbdr_wr(hw, idx, ENETC_TBICR0, ENETC_TBICR0_ICEN | 0x1); 1210 1211 tbmr = ENETC_TBMR_EN | ENETC_TBMR_SET_PRIO(tx_ring->prio); 1212 if (tx_ring->ndev->features & NETIF_F_HW_VLAN_CTAG_TX) 1213 tbmr |= ENETC_TBMR_VIH; 1214 1215 /* enable ring */ 1216 enetc_txbdr_wr(hw, idx, ENETC_TBMR, tbmr); 1217 1218 tx_ring->tpir = hw->reg + ENETC_BDR(TX, idx, ENETC_TBPIR); 1219 tx_ring->tcir = hw->reg + ENETC_BDR(TX, idx, ENETC_TBCIR); 1220 tx_ring->idr = hw->reg + ENETC_SITXIDR; 1221} 1222 1223static void enetc_setup_rxbdr(struct enetc_hw *hw, struct enetc_bdr *rx_ring) 1224{ 1225 int idx = rx_ring->index; 1226 u32 rbmr; 1227 1228 enetc_rxbdr_wr(hw, idx, ENETC_RBBAR0, 1229 lower_32_bits(rx_ring->bd_dma_base)); 1230 1231 enetc_rxbdr_wr(hw, idx, ENETC_RBBAR1, 1232 upper_32_bits(rx_ring->bd_dma_base)); 1233 1234 WARN_ON(!IS_ALIGNED(rx_ring->bd_count, 64)); /* multiple of 64 */ 1235 enetc_rxbdr_wr(hw, idx, ENETC_RBLENR, 1236 ENETC_RTBLENR_LEN(rx_ring->bd_count)); 1237 1238 enetc_rxbdr_wr(hw, idx, ENETC_RBBSR, ENETC_RXB_DMA_SIZE); 1239 1240 /* Also prepare the consumer index in case page allocation never 1241 * succeeds. In that case, hardware will never advance producer index 1242 * to match consumer index, and will drop all frames. 1243 */ 1244 enetc_rxbdr_wr(hw, idx, ENETC_RBPIR, 0); 1245 enetc_rxbdr_wr(hw, idx, ENETC_RBCIR, 1); 1246 1247 /* enable Rx ints by setting pkt thr to 1 */ 1248 enetc_rxbdr_wr(hw, idx, ENETC_RBICR0, ENETC_RBICR0_ICEN | 0x1); 1249 1250 rbmr = ENETC_RBMR_EN; 1251 1252 if (rx_ring->ext_en) 1253 rbmr |= ENETC_RBMR_BDS; 1254 1255 if (rx_ring->ndev->features & NETIF_F_HW_VLAN_CTAG_RX) 1256 rbmr |= ENETC_RBMR_VTE; 1257 1258 rx_ring->rcir = hw->reg + ENETC_BDR(RX, idx, ENETC_RBCIR); 1259 rx_ring->idr = hw->reg + ENETC_SIRXIDR; 1260 1261 enetc_refill_rx_ring(rx_ring, enetc_bd_unused(rx_ring)); 1262 /* update ENETC's consumer index */ 1263 enetc_rxbdr_wr(hw, idx, ENETC_RBCIR, rx_ring->next_to_use); 1264 1265 /* enable ring */ 1266 enetc_rxbdr_wr(hw, idx, ENETC_RBMR, rbmr); 1267} 1268 1269static void enetc_setup_bdrs(struct enetc_ndev_priv *priv) 1270{ 1271 struct enetc_hw *hw = &priv->si->hw; 1272 int i; 1273 1274 for (i = 0; i < priv->num_tx_rings; i++) 1275 enetc_setup_txbdr(hw, priv->tx_ring[i]); 1276 1277 for (i = 0; i < priv->num_rx_rings; i++) 1278 enetc_setup_rxbdr(hw, priv->rx_ring[i]); 1279} 1280 1281static void enetc_clear_rxbdr(struct enetc_hw *hw, struct enetc_bdr *rx_ring) 1282{ 1283 int idx = rx_ring->index; 1284 1285 /* disable EN bit on ring */ 1286 enetc_rxbdr_wr(hw, idx, ENETC_RBMR, 0); 1287} 1288 1289static void enetc_clear_txbdr(struct enetc_hw *hw, struct enetc_bdr *tx_ring) 1290{ 1291 int delay = 8, timeout = 100; 1292 int idx = tx_ring->index; 1293 1294 /* disable EN bit on ring */ 1295 enetc_txbdr_wr(hw, idx, ENETC_TBMR, 0); 1296 1297 /* wait for busy to clear */ 1298 while (delay < timeout && 1299 enetc_txbdr_rd(hw, idx, ENETC_TBSR) & ENETC_TBSR_BUSY) { 1300 msleep(delay); 1301 delay *= 2; 1302 } 1303 1304 if (delay >= timeout) 1305 netdev_warn(tx_ring->ndev, "timeout for tx ring #%d clear\n", 1306 idx); 1307} 1308 1309static void enetc_clear_bdrs(struct enetc_ndev_priv *priv) 1310{ 1311 struct enetc_hw *hw = &priv->si->hw; 1312 int i; 1313 1314 for (i = 0; i < priv->num_tx_rings; i++) 1315 enetc_clear_txbdr(hw, priv->tx_ring[i]); 1316 1317 for (i = 0; i < priv->num_rx_rings; i++) 1318 enetc_clear_rxbdr(hw, priv->rx_ring[i]); 1319 1320 udelay(1); 1321} 1322 1323static int enetc_setup_irqs(struct enetc_ndev_priv *priv) 1324{ 1325 struct pci_dev *pdev = priv->si->pdev; 1326 struct enetc_hw *hw = &priv->si->hw; 1327 int i, j, err; 1328 1329 for (i = 0; i < priv->bdr_int_num; i++) { 1330 int irq = pci_irq_vector(pdev, ENETC_BDR_INT_BASE_IDX + i); 1331 struct enetc_int_vector *v = priv->int_vector[i]; 1332 int entry = ENETC_BDR_INT_BASE_IDX + i; 1333 1334 snprintf(v->name, sizeof(v->name), "%s-rxtx%d", 1335 priv->ndev->name, i); 1336 err = request_irq(irq, enetc_msix, 0, v->name, v); 1337 if (err) { 1338 dev_err(priv->dev, "request_irq() failed!\n"); 1339 goto irq_err; 1340 } 1341 disable_irq(irq); 1342 1343 v->tbier_base = hw->reg + ENETC_BDR(TX, 0, ENETC_TBIER); 1344 v->rbier = hw->reg + ENETC_BDR(RX, i, ENETC_RBIER); 1345 v->ricr1 = hw->reg + ENETC_BDR(RX, i, ENETC_RBICR1); 1346 1347 enetc_wr(hw, ENETC_SIMSIRRV(i), entry); 1348 1349 for (j = 0; j < v->count_tx_rings; j++) { 1350 int idx = v->tx_ring[j].index; 1351 1352 enetc_wr(hw, ENETC_SIMSITRV(idx), entry); 1353 } 1354 irq_set_affinity_hint(irq, get_cpu_mask(i % num_online_cpus())); 1355 } 1356 1357 return 0; 1358 1359irq_err: 1360 while (i--) { 1361 int irq = pci_irq_vector(pdev, ENETC_BDR_INT_BASE_IDX + i); 1362 1363 irq_set_affinity_hint(irq, NULL); 1364 free_irq(irq, priv->int_vector[i]); 1365 } 1366 1367 return err; 1368} 1369 1370static void enetc_free_irqs(struct enetc_ndev_priv *priv) 1371{ 1372 struct pci_dev *pdev = priv->si->pdev; 1373 int i; 1374 1375 for (i = 0; i < priv->bdr_int_num; i++) { 1376 int irq = pci_irq_vector(pdev, ENETC_BDR_INT_BASE_IDX + i); 1377 1378 irq_set_affinity_hint(irq, NULL); 1379 free_irq(irq, priv->int_vector[i]); 1380 } 1381} 1382 1383static void enetc_setup_interrupts(struct enetc_ndev_priv *priv) 1384{ 1385 struct enetc_hw *hw = &priv->si->hw; 1386 u32 icpt, ictt; 1387 int i; 1388 1389 /* enable Tx & Rx event indication */ 1390 if (priv->ic_mode & 1391 (ENETC_IC_RX_MANUAL | ENETC_IC_RX_ADAPTIVE)) { 1392 icpt = ENETC_RBICR0_SET_ICPT(ENETC_RXIC_PKTTHR); 1393 /* init to non-0 minimum, will be adjusted later */ 1394 ictt = 0x1; 1395 } else { 1396 icpt = 0x1; /* enable Rx ints by setting pkt thr to 1 */ 1397 ictt = 0; 1398 } 1399 1400 for (i = 0; i < priv->num_rx_rings; i++) { 1401 enetc_rxbdr_wr(hw, i, ENETC_RBICR1, ictt); 1402 enetc_rxbdr_wr(hw, i, ENETC_RBICR0, ENETC_RBICR0_ICEN | icpt); 1403 enetc_rxbdr_wr(hw, i, ENETC_RBIER, ENETC_RBIER_RXTIE); 1404 } 1405 1406 if (priv->ic_mode & ENETC_IC_TX_MANUAL) 1407 icpt = ENETC_TBICR0_SET_ICPT(ENETC_TXIC_PKTTHR); 1408 else 1409 icpt = 0x1; /* enable Tx ints by setting pkt thr to 1 */ 1410 1411 for (i = 0; i < priv->num_tx_rings; i++) { 1412 enetc_txbdr_wr(hw, i, ENETC_TBICR1, priv->tx_ictt); 1413 enetc_txbdr_wr(hw, i, ENETC_TBICR0, ENETC_TBICR0_ICEN | icpt); 1414 enetc_txbdr_wr(hw, i, ENETC_TBIER, ENETC_TBIER_TXTIE); 1415 } 1416} 1417 1418static void enetc_clear_interrupts(struct enetc_ndev_priv *priv) 1419{ 1420 struct enetc_hw *hw = &priv->si->hw; 1421 int i; 1422 1423 for (i = 0; i < priv->num_tx_rings; i++) 1424 enetc_txbdr_wr(hw, i, ENETC_TBIER, 0); 1425 1426 for (i = 0; i < priv->num_rx_rings; i++) 1427 enetc_rxbdr_wr(hw, i, ENETC_RBIER, 0); 1428} 1429 1430static int enetc_phylink_connect(struct net_device *ndev) 1431{ 1432 struct enetc_ndev_priv *priv = netdev_priv(ndev); 1433 struct ethtool_eee edata; 1434 int err; 1435 1436 if (!priv->phylink) 1437 return 0; /* phy-less mode */ 1438 1439 err = phylink_of_phy_connect(priv->phylink, priv->dev->of_node, 0); 1440 if (err) { 1441 dev_err(&ndev->dev, "could not attach to PHY\n"); 1442 return err; 1443 } 1444 1445 /* disable EEE autoneg, until ENETC driver supports it */ 1446 memset(&edata, 0, sizeof(struct ethtool_eee)); 1447 phylink_ethtool_set_eee(priv->phylink, &edata); 1448 1449 return 0; 1450} 1451 1452void enetc_start(struct net_device *ndev) 1453{ 1454 struct enetc_ndev_priv *priv = netdev_priv(ndev); 1455 int i; 1456 1457 enetc_setup_interrupts(priv); 1458 1459 for (i = 0; i < priv->bdr_int_num; i++) { 1460 int irq = pci_irq_vector(priv->si->pdev, 1461 ENETC_BDR_INT_BASE_IDX + i); 1462 1463 napi_enable(&priv->int_vector[i]->napi); 1464 enable_irq(irq); 1465 } 1466 1467 if (priv->phylink) 1468 phylink_start(priv->phylink); 1469 else 1470 netif_carrier_on(ndev); 1471 1472 netif_tx_start_all_queues(ndev); 1473} 1474 1475int enetc_open(struct net_device *ndev) 1476{ 1477 struct enetc_ndev_priv *priv = netdev_priv(ndev); 1478 int err; 1479 1480 err = enetc_setup_irqs(priv); 1481 if (err) 1482 return err; 1483 1484 err = enetc_phylink_connect(ndev); 1485 if (err) 1486 goto err_phy_connect; 1487 1488 err = enetc_alloc_tx_resources(priv); 1489 if (err) 1490 goto err_alloc_tx; 1491 1492 err = enetc_alloc_rx_resources(priv); 1493 if (err) 1494 goto err_alloc_rx; 1495 1496 err = netif_set_real_num_tx_queues(ndev, priv->num_tx_rings); 1497 if (err) 1498 goto err_set_queues; 1499 1500 err = netif_set_real_num_rx_queues(ndev, priv->num_rx_rings); 1501 if (err) 1502 goto err_set_queues; 1503 1504 enetc_setup_bdrs(priv); 1505 enetc_start(ndev); 1506 1507 return 0; 1508 1509err_set_queues: 1510 enetc_free_rx_resources(priv); 1511err_alloc_rx: 1512 enetc_free_tx_resources(priv); 1513err_alloc_tx: 1514 if (priv->phylink) 1515 phylink_disconnect_phy(priv->phylink); 1516err_phy_connect: 1517 enetc_free_irqs(priv); 1518 1519 return err; 1520} 1521 1522void enetc_stop(struct net_device *ndev) 1523{ 1524 struct enetc_ndev_priv *priv = netdev_priv(ndev); 1525 int i; 1526 1527 netif_tx_stop_all_queues(ndev); 1528 1529 for (i = 0; i < priv->bdr_int_num; i++) { 1530 int irq = pci_irq_vector(priv->si->pdev, 1531 ENETC_BDR_INT_BASE_IDX + i); 1532 1533 disable_irq(irq); 1534 napi_synchronize(&priv->int_vector[i]->napi); 1535 napi_disable(&priv->int_vector[i]->napi); 1536 } 1537 1538 if (priv->phylink) 1539 phylink_stop(priv->phylink); 1540 else 1541 netif_carrier_off(ndev); 1542 1543 enetc_clear_interrupts(priv); 1544} 1545 1546int enetc_close(struct net_device *ndev) 1547{ 1548 struct enetc_ndev_priv *priv = netdev_priv(ndev); 1549 1550 enetc_stop(ndev); 1551 enetc_clear_bdrs(priv); 1552 1553 if (priv->phylink) 1554 phylink_disconnect_phy(priv->phylink); 1555 enetc_free_rxtx_rings(priv); 1556 enetc_free_rx_resources(priv); 1557 enetc_free_tx_resources(priv); 1558 enetc_free_irqs(priv); 1559 1560 return 0; 1561} 1562 1563static int enetc_setup_tc_mqprio(struct net_device *ndev, void *type_data) 1564{ 1565 struct enetc_ndev_priv *priv = netdev_priv(ndev); 1566 struct tc_mqprio_qopt *mqprio = type_data; 1567 struct enetc_hw *hw = &priv->si->hw; 1568 struct enetc_bdr *tx_ring; 1569 u8 num_tc; 1570 int i; 1571 1572 mqprio->hw = TC_MQPRIO_HW_OFFLOAD_TCS; 1573 num_tc = mqprio->num_tc; 1574 1575 if (!num_tc) { 1576 netdev_reset_tc(ndev); 1577 netif_set_real_num_tx_queues(ndev, priv->num_tx_rings); 1578 1579 /* Reset all ring priorities to 0 */ 1580 for (i = 0; i < priv->num_tx_rings; i++) { 1581 tx_ring = priv->tx_ring[i]; 1582 tx_ring->prio = 0; 1583 enetc_set_bdr_prio(hw, tx_ring->index, tx_ring->prio); 1584 } 1585 1586 return 0; 1587 } 1588 1589 /* Check if we have enough BD rings available to accommodate all TCs */ 1590 if (num_tc > priv->num_tx_rings) { 1591 netdev_err(ndev, "Max %d traffic classes supported\n", 1592 priv->num_tx_rings); 1593 return -EINVAL; 1594 } 1595 1596 /* For the moment, we use only one BD ring per TC. 1597 * 1598 * Configure num_tc BD rings with increasing priorities. 1599 */ 1600 for (i = 0; i < num_tc; i++) { 1601 tx_ring = priv->tx_ring[i]; 1602 tx_ring->prio = i; 1603 enetc_set_bdr_prio(hw, tx_ring->index, tx_ring->prio); 1604 } 1605 1606 /* Reset the number of netdev queues based on the TC count */ 1607 netif_set_real_num_tx_queues(ndev, num_tc); 1608 1609 netdev_set_num_tc(ndev, num_tc); 1610 1611 /* Each TC is associated with one netdev queue */ 1612 for (i = 0; i < num_tc; i++) 1613 netdev_set_tc_queue(ndev, i, 1, i); 1614 1615 return 0; 1616} 1617 1618int enetc_setup_tc(struct net_device *ndev, enum tc_setup_type type, 1619 void *type_data) 1620{ 1621 switch (type) { 1622 case TC_SETUP_QDISC_MQPRIO: 1623 return enetc_setup_tc_mqprio(ndev, type_data); 1624 case TC_SETUP_QDISC_TAPRIO: 1625 return enetc_setup_tc_taprio(ndev, type_data); 1626 case TC_SETUP_QDISC_CBS: 1627 return enetc_setup_tc_cbs(ndev, type_data); 1628 case TC_SETUP_QDISC_ETF: 1629 return enetc_setup_tc_txtime(ndev, type_data); 1630 case TC_SETUP_BLOCK: 1631 return enetc_setup_tc_psfp(ndev, type_data); 1632 default: 1633 return -EOPNOTSUPP; 1634 } 1635} 1636 1637struct net_device_stats *enetc_get_stats(struct net_device *ndev) 1638{ 1639 struct enetc_ndev_priv *priv = netdev_priv(ndev); 1640 struct net_device_stats *stats = &ndev->stats; 1641 unsigned long packets = 0, bytes = 0; 1642 int i; 1643 1644 for (i = 0; i < priv->num_rx_rings; i++) { 1645 packets += priv->rx_ring[i]->stats.packets; 1646 bytes += priv->rx_ring[i]->stats.bytes; 1647 } 1648 1649 stats->rx_packets = packets; 1650 stats->rx_bytes = bytes; 1651 bytes = 0; 1652 packets = 0; 1653 1654 for (i = 0; i < priv->num_tx_rings; i++) { 1655 packets += priv->tx_ring[i]->stats.packets; 1656 bytes += priv->tx_ring[i]->stats.bytes; 1657 } 1658 1659 stats->tx_packets = packets; 1660 stats->tx_bytes = bytes; 1661 1662 return stats; 1663} 1664 1665static int enetc_set_rss(struct net_device *ndev, int en) 1666{ 1667 struct enetc_ndev_priv *priv = netdev_priv(ndev); 1668 struct enetc_hw *hw = &priv->si->hw; 1669 u32 reg; 1670 1671 enetc_wr(hw, ENETC_SIRBGCR, priv->num_rx_rings); 1672 1673 reg = enetc_rd(hw, ENETC_SIMR); 1674 reg &= ~ENETC_SIMR_RSSE; 1675 reg |= (en) ? ENETC_SIMR_RSSE : 0; 1676 enetc_wr(hw, ENETC_SIMR, reg); 1677 1678 return 0; 1679} 1680 1681static void enetc_enable_rxvlan(struct net_device *ndev, bool en) 1682{ 1683 struct enetc_ndev_priv *priv = netdev_priv(ndev); 1684 struct enetc_hw *hw = &priv->si->hw; 1685 int i; 1686 1687 for (i = 0; i < priv->num_rx_rings; i++) 1688 enetc_bdr_enable_rxvlan(hw, i, en); 1689} 1690 1691static void enetc_enable_txvlan(struct net_device *ndev, bool en) 1692{ 1693 struct enetc_ndev_priv *priv = netdev_priv(ndev); 1694 struct enetc_hw *hw = &priv->si->hw; 1695 int i; 1696 1697 for (i = 0; i < priv->num_tx_rings; i++) 1698 enetc_bdr_enable_txvlan(hw, i, en); 1699} 1700 1701void enetc_set_features(struct net_device *ndev, netdev_features_t features) 1702{ 1703 netdev_features_t changed = ndev->features ^ features; 1704 1705 if (changed & NETIF_F_RXHASH) 1706 enetc_set_rss(ndev, !!(features & NETIF_F_RXHASH)); 1707 1708 if (changed & NETIF_F_HW_VLAN_CTAG_RX) 1709 enetc_enable_rxvlan(ndev, 1710 !!(features & NETIF_F_HW_VLAN_CTAG_RX)); 1711 1712 if (changed & NETIF_F_HW_VLAN_CTAG_TX) 1713 enetc_enable_txvlan(ndev, 1714 !!(features & NETIF_F_HW_VLAN_CTAG_TX)); 1715} 1716 1717#ifdef CONFIG_FSL_ENETC_PTP_CLOCK 1718static int enetc_hwtstamp_set(struct net_device *ndev, struct ifreq *ifr) 1719{ 1720 struct enetc_ndev_priv *priv = netdev_priv(ndev); 1721 struct hwtstamp_config config; 1722 int ao; 1723 1724 if (copy_from_user(&config, ifr->ifr_data, sizeof(config))) 1725 return -EFAULT; 1726 1727 switch (config.tx_type) { 1728 case HWTSTAMP_TX_OFF: 1729 priv->active_offloads &= ~ENETC_F_TX_TSTAMP; 1730 break; 1731 case HWTSTAMP_TX_ON: 1732 priv->active_offloads |= ENETC_F_TX_TSTAMP; 1733 break; 1734 default: 1735 return -ERANGE; 1736 } 1737 1738 ao = priv->active_offloads; 1739 switch (config.rx_filter) { 1740 case HWTSTAMP_FILTER_NONE: 1741 priv->active_offloads &= ~ENETC_F_RX_TSTAMP; 1742 break; 1743 default: 1744 priv->active_offloads |= ENETC_F_RX_TSTAMP; 1745 config.rx_filter = HWTSTAMP_FILTER_ALL; 1746 } 1747 1748 if (netif_running(ndev) && ao != priv->active_offloads) { 1749 enetc_close(ndev); 1750 enetc_open(ndev); 1751 } 1752 1753 return copy_to_user(ifr->ifr_data, &config, sizeof(config)) ? 1754 -EFAULT : 0; 1755} 1756 1757static int enetc_hwtstamp_get(struct net_device *ndev, struct ifreq *ifr) 1758{ 1759 struct enetc_ndev_priv *priv = netdev_priv(ndev); 1760 struct hwtstamp_config config; 1761 1762 config.flags = 0; 1763 1764 if (priv->active_offloads & ENETC_F_TX_TSTAMP) 1765 config.tx_type = HWTSTAMP_TX_ON; 1766 else 1767 config.tx_type = HWTSTAMP_TX_OFF; 1768 1769 config.rx_filter = (priv->active_offloads & ENETC_F_RX_TSTAMP) ? 1770 HWTSTAMP_FILTER_ALL : HWTSTAMP_FILTER_NONE; 1771 1772 return copy_to_user(ifr->ifr_data, &config, sizeof(config)) ? 1773 -EFAULT : 0; 1774} 1775#endif 1776 1777int enetc_ioctl(struct net_device *ndev, struct ifreq *rq, int cmd) 1778{ 1779 struct enetc_ndev_priv *priv = netdev_priv(ndev); 1780#ifdef CONFIG_FSL_ENETC_PTP_CLOCK 1781 if (cmd == SIOCSHWTSTAMP) 1782 return enetc_hwtstamp_set(ndev, rq); 1783 if (cmd == SIOCGHWTSTAMP) 1784 return enetc_hwtstamp_get(ndev, rq); 1785#endif 1786 1787 if (!priv->phylink) 1788 return -EOPNOTSUPP; 1789 1790 return phylink_mii_ioctl(priv->phylink, rq, cmd); 1791} 1792 1793int enetc_alloc_msix(struct enetc_ndev_priv *priv) 1794{ 1795 struct pci_dev *pdev = priv->si->pdev; 1796 int v_tx_rings; 1797 int i, n, err, nvec; 1798 1799 nvec = ENETC_BDR_INT_BASE_IDX + priv->bdr_int_num; 1800 /* allocate MSIX for both messaging and Rx/Tx interrupts */ 1801 n = pci_alloc_irq_vectors(pdev, nvec, nvec, PCI_IRQ_MSIX); 1802 1803 if (n < 0) 1804 return n; 1805 1806 if (n != nvec) 1807 return -EPERM; 1808 1809 /* # of tx rings per int vector */ 1810 v_tx_rings = priv->num_tx_rings / priv->bdr_int_num; 1811 1812 for (i = 0; i < priv->bdr_int_num; i++) { 1813 struct enetc_int_vector *v; 1814 struct enetc_bdr *bdr; 1815 int j; 1816 1817 v = kzalloc(struct_size(v, tx_ring, v_tx_rings), GFP_KERNEL); 1818 if (!v) { 1819 err = -ENOMEM; 1820 goto fail; 1821 } 1822 1823 priv->int_vector[i] = v; 1824 1825 /* init defaults for adaptive IC */ 1826 if (priv->ic_mode & ENETC_IC_RX_ADAPTIVE) { 1827 v->rx_ictt = 0x1; 1828 v->rx_dim_en = true; 1829 } 1830 INIT_WORK(&v->rx_dim.work, enetc_rx_dim_work); 1831 netif_napi_add(priv->ndev, &v->napi, enetc_poll, 1832 NAPI_POLL_WEIGHT); 1833 v->count_tx_rings = v_tx_rings; 1834 1835 for (j = 0; j < v_tx_rings; j++) { 1836 int idx; 1837 1838 /* default tx ring mapping policy */ 1839 if (priv->bdr_int_num == ENETC_MAX_BDR_INT) 1840 idx = 2 * j + i; /* 2 CPUs */ 1841 else 1842 idx = j + i * v_tx_rings; /* default */ 1843 1844 __set_bit(idx, &v->tx_rings_map); 1845 bdr = &v->tx_ring[j]; 1846 bdr->index = idx; 1847 bdr->ndev = priv->ndev; 1848 bdr->dev = priv->dev; 1849 bdr->bd_count = priv->tx_bd_count; 1850 priv->tx_ring[idx] = bdr; 1851 } 1852 1853 bdr = &v->rx_ring; 1854 bdr->index = i; 1855 bdr->ndev = priv->ndev; 1856 bdr->dev = priv->dev; 1857 bdr->bd_count = priv->rx_bd_count; 1858 priv->rx_ring[i] = bdr; 1859 } 1860 1861 return 0; 1862 1863fail: 1864 while (i--) { 1865 netif_napi_del(&priv->int_vector[i]->napi); 1866 cancel_work_sync(&priv->int_vector[i]->rx_dim.work); 1867 kfree(priv->int_vector[i]); 1868 } 1869 1870 pci_free_irq_vectors(pdev); 1871 1872 return err; 1873} 1874 1875void enetc_free_msix(struct enetc_ndev_priv *priv) 1876{ 1877 int i; 1878 1879 for (i = 0; i < priv->bdr_int_num; i++) { 1880 struct enetc_int_vector *v = priv->int_vector[i]; 1881 1882 netif_napi_del(&v->napi); 1883 cancel_work_sync(&v->rx_dim.work); 1884 } 1885 1886 for (i = 0; i < priv->num_rx_rings; i++) 1887 priv->rx_ring[i] = NULL; 1888 1889 for (i = 0; i < priv->num_tx_rings; i++) 1890 priv->tx_ring[i] = NULL; 1891 1892 for (i = 0; i < priv->bdr_int_num; i++) { 1893 kfree(priv->int_vector[i]); 1894 priv->int_vector[i] = NULL; 1895 } 1896 1897 /* disable all MSIX for this device */ 1898 pci_free_irq_vectors(priv->si->pdev); 1899} 1900 1901static void enetc_kfree_si(struct enetc_si *si) 1902{ 1903 char *p = (char *)si - si->pad; 1904 1905 kfree(p); 1906} 1907 1908static void enetc_detect_errata(struct enetc_si *si) 1909{ 1910 if (si->pdev->revision == ENETC_REV1) 1911 si->errata = ENETC_ERR_TXCSUM | ENETC_ERR_VLAN_ISOL | 1912 ENETC_ERR_UCMCSWP; 1913} 1914 1915int enetc_pci_probe(struct pci_dev *pdev, const char *name, int sizeof_priv) 1916{ 1917 struct enetc_si *si, *p; 1918 struct enetc_hw *hw; 1919 size_t alloc_size; 1920 int err, len; 1921 1922 pcie_flr(pdev); 1923 err = pci_enable_device_mem(pdev); 1924 if (err) { 1925 dev_err(&pdev->dev, "device enable failed\n"); 1926 return err; 1927 } 1928 1929 /* set up for high or low dma */ 1930 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); 1931 if (err) { 1932 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); 1933 if (err) { 1934 dev_err(&pdev->dev, 1935 "DMA configuration failed: 0x%x\n", err); 1936 goto err_dma; 1937 } 1938 } 1939 1940 err = pci_request_mem_regions(pdev, name); 1941 if (err) { 1942 dev_err(&pdev->dev, "pci_request_regions failed err=%d\n", err); 1943 goto err_pci_mem_reg; 1944 } 1945 1946 pci_set_master(pdev); 1947 1948 alloc_size = sizeof(struct enetc_si); 1949 if (sizeof_priv) { 1950 /* align priv to 32B */ 1951 alloc_size = ALIGN(alloc_size, ENETC_SI_ALIGN); 1952 alloc_size += sizeof_priv; 1953 } 1954 /* force 32B alignment for enetc_si */ 1955 alloc_size += ENETC_SI_ALIGN - 1; 1956 1957 p = kzalloc(alloc_size, GFP_KERNEL); 1958 if (!p) { 1959 err = -ENOMEM; 1960 goto err_alloc_si; 1961 } 1962 1963 si = PTR_ALIGN(p, ENETC_SI_ALIGN); 1964 si->pad = (char *)si - (char *)p; 1965 1966 pci_set_drvdata(pdev, si); 1967 si->pdev = pdev; 1968 hw = &si->hw; 1969 1970 len = pci_resource_len(pdev, ENETC_BAR_REGS); 1971 hw->reg = ioremap(pci_resource_start(pdev, ENETC_BAR_REGS), len); 1972 if (!hw->reg) { 1973 err = -ENXIO; 1974 dev_err(&pdev->dev, "ioremap() failed\n"); 1975 goto err_ioremap; 1976 } 1977 if (len > ENETC_PORT_BASE) 1978 hw->port = hw->reg + ENETC_PORT_BASE; 1979 if (len > ENETC_GLOBAL_BASE) 1980 hw->global = hw->reg + ENETC_GLOBAL_BASE; 1981 1982 enetc_detect_errata(si); 1983 1984 return 0; 1985 1986err_ioremap: 1987 enetc_kfree_si(si); 1988err_alloc_si: 1989 pci_release_mem_regions(pdev); 1990err_pci_mem_reg: 1991err_dma: 1992 pci_disable_device(pdev); 1993 1994 return err; 1995} 1996 1997void enetc_pci_remove(struct pci_dev *pdev) 1998{ 1999 struct enetc_si *si = pci_get_drvdata(pdev); 2000 struct enetc_hw *hw = &si->hw; 2001 2002 iounmap(hw->reg); 2003 enetc_kfree_si(si); 2004 pci_release_mem_regions(pdev); 2005 pci_disable_device(pdev); 2006} 2007