1// SPDX-License-Identifier: GPL-2.0 2/* Copyright(c) 2017 - 2019 Pensando Systems, Inc */ 3 4#include <linux/ip.h> 5#include <linux/ipv6.h> 6#include <linux/if_vlan.h> 7#include <net/ip6_checksum.h> 8 9#include "ionic.h" 10#include "ionic_lif.h" 11#include "ionic_txrx.h" 12 13static void ionic_rx_clean(struct ionic_queue *q, 14 struct ionic_desc_info *desc_info, 15 struct ionic_cq_info *cq_info, 16 void *cb_arg); 17 18static bool ionic_rx_service(struct ionic_cq *cq, struct ionic_cq_info *cq_info); 19 20static bool ionic_tx_service(struct ionic_cq *cq, struct ionic_cq_info *cq_info); 21 22static inline void ionic_txq_post(struct ionic_queue *q, bool ring_dbell, 23 ionic_desc_cb cb_func, void *cb_arg) 24{ 25 DEBUG_STATS_TXQ_POST(q, ring_dbell); 26 27 ionic_q_post(q, ring_dbell, cb_func, cb_arg); 28} 29 30static inline void ionic_rxq_post(struct ionic_queue *q, bool ring_dbell, 31 ionic_desc_cb cb_func, void *cb_arg) 32{ 33 ionic_q_post(q, ring_dbell, cb_func, cb_arg); 34 35 DEBUG_STATS_RX_BUFF_CNT(q); 36} 37 38static inline struct netdev_queue *q_to_ndq(struct ionic_queue *q) 39{ 40 return netdev_get_tx_queue(q->lif->netdev, q->index); 41} 42 43static struct sk_buff *ionic_rx_skb_alloc(struct ionic_queue *q, 44 unsigned int len, bool frags) 45{ 46 struct ionic_lif *lif = q->lif; 47 struct ionic_rx_stats *stats; 48 struct net_device *netdev; 49 struct sk_buff *skb; 50 51 netdev = lif->netdev; 52 stats = &q->lif->rxqstats[q->index]; 53 54 if (frags) 55 skb = napi_get_frags(&q_to_qcq(q)->napi); 56 else 57 skb = netdev_alloc_skb_ip_align(netdev, len); 58 59 if (unlikely(!skb)) { 60 net_warn_ratelimited("%s: SKB alloc failed on %s!\n", 61 netdev->name, q->name); 62 stats->alloc_err++; 63 return NULL; 64 } 65 66 return skb; 67} 68 69static struct sk_buff *ionic_rx_frags(struct ionic_queue *q, 70 struct ionic_desc_info *desc_info, 71 struct ionic_cq_info *cq_info) 72{ 73 struct ionic_rxq_comp *comp = cq_info->cq_desc; 74 struct device *dev = q->lif->ionic->dev; 75 struct ionic_page_info *page_info; 76 struct sk_buff *skb; 77 unsigned int i; 78 u16 frag_len; 79 u16 len; 80 81 page_info = &desc_info->pages[0]; 82 len = le16_to_cpu(comp->len); 83 84 prefetch(page_address(page_info->page) + NET_IP_ALIGN); 85 86 skb = ionic_rx_skb_alloc(q, len, true); 87 if (unlikely(!skb)) 88 return NULL; 89 90 i = comp->num_sg_elems + 1; 91 do { 92 if (unlikely(!page_info->page)) { 93 struct napi_struct *napi = &q_to_qcq(q)->napi; 94 95 napi->skb = NULL; 96 dev_kfree_skb(skb); 97 return NULL; 98 } 99 100 frag_len = min(len, (u16)PAGE_SIZE); 101 len -= frag_len; 102 103 dma_unmap_page(dev, dma_unmap_addr(page_info, dma_addr), 104 PAGE_SIZE, DMA_FROM_DEVICE); 105 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, 106 page_info->page, 0, frag_len, PAGE_SIZE); 107 page_info->page = NULL; 108 page_info++; 109 i--; 110 } while (i > 0); 111 112 return skb; 113} 114 115static struct sk_buff *ionic_rx_copybreak(struct ionic_queue *q, 116 struct ionic_desc_info *desc_info, 117 struct ionic_cq_info *cq_info) 118{ 119 struct ionic_rxq_comp *comp = cq_info->cq_desc; 120 struct device *dev = q->lif->ionic->dev; 121 struct ionic_page_info *page_info; 122 struct sk_buff *skb; 123 u16 len; 124 125 page_info = &desc_info->pages[0]; 126 len = le16_to_cpu(comp->len); 127 128 skb = ionic_rx_skb_alloc(q, len, false); 129 if (unlikely(!skb)) 130 return NULL; 131 132 if (unlikely(!page_info->page)) { 133 dev_kfree_skb(skb); 134 return NULL; 135 } 136 137 dma_sync_single_for_cpu(dev, dma_unmap_addr(page_info, dma_addr), 138 len, DMA_FROM_DEVICE); 139 skb_copy_to_linear_data(skb, page_address(page_info->page), len); 140 dma_sync_single_for_device(dev, dma_unmap_addr(page_info, dma_addr), 141 len, DMA_FROM_DEVICE); 142 143 skb_put(skb, len); 144 skb->protocol = eth_type_trans(skb, q->lif->netdev); 145 146 return skb; 147} 148 149static void ionic_rx_clean(struct ionic_queue *q, 150 struct ionic_desc_info *desc_info, 151 struct ionic_cq_info *cq_info, 152 void *cb_arg) 153{ 154 struct ionic_rxq_comp *comp = cq_info->cq_desc; 155 struct ionic_qcq *qcq = q_to_qcq(q); 156 struct ionic_rx_stats *stats; 157 struct net_device *netdev; 158 struct sk_buff *skb; 159 160 stats = q_to_rx_stats(q); 161 netdev = q->lif->netdev; 162 163 if (comp->status) { 164 stats->dropped++; 165 return; 166 } 167 168 stats->pkts++; 169 stats->bytes += le16_to_cpu(comp->len); 170 171 if (le16_to_cpu(comp->len) <= q->lif->rx_copybreak) 172 skb = ionic_rx_copybreak(q, desc_info, cq_info); 173 else 174 skb = ionic_rx_frags(q, desc_info, cq_info); 175 176 if (unlikely(!skb)) { 177 stats->dropped++; 178 return; 179 } 180 181 skb_record_rx_queue(skb, q->index); 182 183 if (likely(netdev->features & NETIF_F_RXHASH)) { 184 switch (comp->pkt_type_color & IONIC_RXQ_COMP_PKT_TYPE_MASK) { 185 case IONIC_PKT_TYPE_IPV4: 186 case IONIC_PKT_TYPE_IPV6: 187 skb_set_hash(skb, le32_to_cpu(comp->rss_hash), 188 PKT_HASH_TYPE_L3); 189 break; 190 case IONIC_PKT_TYPE_IPV4_TCP: 191 case IONIC_PKT_TYPE_IPV6_TCP: 192 case IONIC_PKT_TYPE_IPV4_UDP: 193 case IONIC_PKT_TYPE_IPV6_UDP: 194 skb_set_hash(skb, le32_to_cpu(comp->rss_hash), 195 PKT_HASH_TYPE_L4); 196 break; 197 } 198 } 199 200 if (likely(netdev->features & NETIF_F_RXCSUM) && 201 (comp->csum_flags & IONIC_RXQ_COMP_CSUM_F_CALC)) { 202 skb->ip_summed = CHECKSUM_COMPLETE; 203 skb->csum = (__force __wsum)le16_to_cpu(comp->csum); 204 stats->csum_complete++; 205 } else { 206 stats->csum_none++; 207 } 208 209 if (unlikely((comp->csum_flags & IONIC_RXQ_COMP_CSUM_F_TCP_BAD) || 210 (comp->csum_flags & IONIC_RXQ_COMP_CSUM_F_UDP_BAD) || 211 (comp->csum_flags & IONIC_RXQ_COMP_CSUM_F_IP_BAD))) 212 stats->csum_error++; 213 214 if (likely(netdev->features & NETIF_F_HW_VLAN_CTAG_RX) && 215 (comp->csum_flags & IONIC_RXQ_COMP_CSUM_F_VLAN)) { 216 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), 217 le16_to_cpu(comp->vlan_tci)); 218 stats->vlan_stripped++; 219 } 220 221 if (le16_to_cpu(comp->len) <= q->lif->rx_copybreak) 222 napi_gro_receive(&qcq->napi, skb); 223 else 224 napi_gro_frags(&qcq->napi); 225} 226 227static bool ionic_rx_service(struct ionic_cq *cq, struct ionic_cq_info *cq_info) 228{ 229 struct ionic_rxq_comp *comp = cq_info->cq_desc; 230 struct ionic_queue *q = cq->bound_q; 231 struct ionic_desc_info *desc_info; 232 233 if (!color_match(comp->pkt_type_color, cq->done_color)) 234 return false; 235 236 /* check for empty queue */ 237 if (q->tail_idx == q->head_idx) 238 return false; 239 240 if (q->tail_idx != le16_to_cpu(comp->comp_index)) 241 return false; 242 243 desc_info = &q->info[q->tail_idx]; 244 q->tail_idx = (q->tail_idx + 1) & (q->num_descs - 1); 245 246 /* clean the related q entry, only one per qc completion */ 247 ionic_rx_clean(q, desc_info, cq_info, desc_info->cb_arg); 248 249 desc_info->cb = NULL; 250 desc_info->cb_arg = NULL; 251 252 return true; 253} 254 255static int ionic_rx_page_alloc(struct ionic_queue *q, 256 struct ionic_page_info *page_info) 257{ 258 struct ionic_lif *lif = q->lif; 259 struct ionic_rx_stats *stats; 260 struct net_device *netdev; 261 struct device *dev; 262 263 netdev = lif->netdev; 264 dev = lif->ionic->dev; 265 stats = q_to_rx_stats(q); 266 267 if (unlikely(!page_info)) { 268 net_err_ratelimited("%s: %s invalid page_info in alloc\n", 269 netdev->name, q->name); 270 return -EINVAL; 271 } 272 273 page_info->page = dev_alloc_page(); 274 if (unlikely(!page_info->page)) { 275 net_err_ratelimited("%s: %s page alloc failed\n", 276 netdev->name, q->name); 277 stats->alloc_err++; 278 return -ENOMEM; 279 } 280 281 page_info->dma_addr = dma_map_page(dev, page_info->page, 0, PAGE_SIZE, 282 DMA_FROM_DEVICE); 283 if (unlikely(dma_mapping_error(dev, page_info->dma_addr))) { 284 put_page(page_info->page); 285 page_info->dma_addr = 0; 286 page_info->page = NULL; 287 net_err_ratelimited("%s: %s dma map failed\n", 288 netdev->name, q->name); 289 stats->dma_map_err++; 290 return -EIO; 291 } 292 293 return 0; 294} 295 296static void ionic_rx_page_free(struct ionic_queue *q, 297 struct ionic_page_info *page_info) 298{ 299 struct ionic_lif *lif = q->lif; 300 struct net_device *netdev; 301 struct device *dev; 302 303 netdev = lif->netdev; 304 dev = lif->ionic->dev; 305 306 if (unlikely(!page_info)) { 307 net_err_ratelimited("%s: %s invalid page_info in free\n", 308 netdev->name, q->name); 309 return; 310 } 311 312 if (unlikely(!page_info->page)) { 313 net_err_ratelimited("%s: %s invalid page in free\n", 314 netdev->name, q->name); 315 return; 316 } 317 318 dma_unmap_page(dev, page_info->dma_addr, PAGE_SIZE, DMA_FROM_DEVICE); 319 320 put_page(page_info->page); 321 page_info->dma_addr = 0; 322 page_info->page = NULL; 323} 324 325void ionic_rx_fill(struct ionic_queue *q) 326{ 327 struct net_device *netdev = q->lif->netdev; 328 struct ionic_desc_info *desc_info; 329 struct ionic_page_info *page_info; 330 struct ionic_rxq_sg_desc *sg_desc; 331 struct ionic_rxq_sg_elem *sg_elem; 332 struct ionic_rxq_desc *desc; 333 unsigned int remain_len; 334 unsigned int seg_len; 335 unsigned int nfrags; 336 unsigned int i, j; 337 unsigned int len; 338 339 len = netdev->mtu + ETH_HLEN + VLAN_HLEN; 340 nfrags = round_up(len, PAGE_SIZE) / PAGE_SIZE; 341 342 for (i = ionic_q_space_avail(q); i; i--) { 343 remain_len = len; 344 desc_info = &q->info[q->head_idx]; 345 desc = desc_info->desc; 346 sg_desc = desc_info->sg_desc; 347 page_info = &desc_info->pages[0]; 348 349 if (page_info->page) { /* recycle the buffer */ 350 ionic_rxq_post(q, false, ionic_rx_clean, NULL); 351 continue; 352 } 353 354 /* fill main descriptor - pages[0] */ 355 desc->opcode = (nfrags > 1) ? IONIC_RXQ_DESC_OPCODE_SG : 356 IONIC_RXQ_DESC_OPCODE_SIMPLE; 357 desc_info->npages = nfrags; 358 if (unlikely(ionic_rx_page_alloc(q, page_info))) { 359 desc->addr = 0; 360 desc->len = 0; 361 return; 362 } 363 desc->addr = cpu_to_le64(page_info->dma_addr); 364 seg_len = min_t(unsigned int, PAGE_SIZE, len); 365 desc->len = cpu_to_le16(seg_len); 366 remain_len -= seg_len; 367 page_info++; 368 369 /* fill sg descriptors - pages[1..n] */ 370 for (j = 0; j < nfrags - 1; j++) { 371 if (page_info->page) /* recycle the sg buffer */ 372 continue; 373 374 sg_elem = &sg_desc->elems[j]; 375 if (unlikely(ionic_rx_page_alloc(q, page_info))) { 376 sg_elem->addr = 0; 377 sg_elem->len = 0; 378 return; 379 } 380 sg_elem->addr = cpu_to_le64(page_info->dma_addr); 381 seg_len = min_t(unsigned int, PAGE_SIZE, remain_len); 382 sg_elem->len = cpu_to_le16(seg_len); 383 remain_len -= seg_len; 384 page_info++; 385 } 386 387 ionic_rxq_post(q, false, ionic_rx_clean, NULL); 388 } 389 390 ionic_dbell_ring(q->lif->kern_dbpage, q->hw_type, 391 q->dbval | q->head_idx); 392} 393 394static void ionic_rx_fill_cb(void *arg) 395{ 396 ionic_rx_fill(arg); 397} 398 399void ionic_rx_empty(struct ionic_queue *q) 400{ 401 struct ionic_desc_info *desc_info; 402 struct ionic_page_info *page_info; 403 unsigned int i, j; 404 405 for (i = 0; i < q->num_descs; i++) { 406 desc_info = &q->info[i]; 407 for (j = 0; j < IONIC_RX_MAX_SG_ELEMS + 1; j++) { 408 page_info = &desc_info->pages[j]; 409 if (page_info->page) 410 ionic_rx_page_free(q, page_info); 411 } 412 413 desc_info->npages = 0; 414 desc_info->cb = NULL; 415 desc_info->cb_arg = NULL; 416 } 417} 418 419static void ionic_dim_update(struct ionic_qcq *qcq, int napi_mode) 420{ 421 struct dim_sample dim_sample; 422 struct ionic_lif *lif; 423 unsigned int qi; 424 u64 pkts, bytes; 425 426 if (!qcq->intr.dim_coal_hw) 427 return; 428 429 lif = qcq->q.lif; 430 qi = qcq->cq.bound_q->index; 431 432 switch (napi_mode) { 433 case IONIC_LIF_F_TX_DIM_INTR: 434 pkts = lif->txqstats[qi].pkts; 435 bytes = lif->txqstats[qi].bytes; 436 break; 437 case IONIC_LIF_F_RX_DIM_INTR: 438 pkts = lif->rxqstats[qi].pkts; 439 bytes = lif->rxqstats[qi].bytes; 440 break; 441 default: 442 pkts = lif->txqstats[qi].pkts + lif->rxqstats[qi].pkts; 443 bytes = lif->txqstats[qi].bytes + lif->rxqstats[qi].bytes; 444 break; 445 } 446 447 dim_update_sample(qcq->cq.bound_intr->rearm_count, 448 pkts, bytes, &dim_sample); 449 450 net_dim(&qcq->dim, dim_sample); 451} 452 453int ionic_tx_napi(struct napi_struct *napi, int budget) 454{ 455 struct ionic_qcq *qcq = napi_to_qcq(napi); 456 struct ionic_cq *cq = napi_to_cq(napi); 457 struct ionic_dev *idev; 458 struct ionic_lif *lif; 459 u32 work_done = 0; 460 u32 flags = 0; 461 462 lif = cq->bound_q->lif; 463 idev = &lif->ionic->idev; 464 465 work_done = ionic_cq_service(cq, budget, 466 ionic_tx_service, NULL, NULL); 467 468 if (work_done < budget && napi_complete_done(napi, work_done)) { 469 ionic_dim_update(qcq, IONIC_LIF_F_TX_DIM_INTR); 470 flags |= IONIC_INTR_CRED_UNMASK; 471 cq->bound_intr->rearm_count++; 472 } 473 474 if (work_done || flags) { 475 flags |= IONIC_INTR_CRED_RESET_COALESCE; 476 ionic_intr_credits(idev->intr_ctrl, 477 cq->bound_intr->index, 478 work_done, flags); 479 } 480 481 DEBUG_STATS_NAPI_POLL(qcq, work_done); 482 483 return work_done; 484} 485 486int ionic_rx_napi(struct napi_struct *napi, int budget) 487{ 488 struct ionic_qcq *qcq = napi_to_qcq(napi); 489 struct ionic_cq *cq = napi_to_cq(napi); 490 struct ionic_dev *idev; 491 struct ionic_lif *lif; 492 u32 work_done = 0; 493 u32 flags = 0; 494 495 lif = cq->bound_q->lif; 496 idev = &lif->ionic->idev; 497 498 work_done = ionic_cq_service(cq, budget, 499 ionic_rx_service, NULL, NULL); 500 501 if (work_done) 502 ionic_rx_fill(cq->bound_q); 503 504 if (work_done < budget && napi_complete_done(napi, work_done)) { 505 ionic_dim_update(qcq, IONIC_LIF_F_RX_DIM_INTR); 506 flags |= IONIC_INTR_CRED_UNMASK; 507 cq->bound_intr->rearm_count++; 508 } 509 510 if (work_done || flags) { 511 flags |= IONIC_INTR_CRED_RESET_COALESCE; 512 ionic_intr_credits(idev->intr_ctrl, 513 cq->bound_intr->index, 514 work_done, flags); 515 } 516 517 DEBUG_STATS_NAPI_POLL(qcq, work_done); 518 519 return work_done; 520} 521 522int ionic_txrx_napi(struct napi_struct *napi, int budget) 523{ 524 struct ionic_qcq *qcq = napi_to_qcq(napi); 525 struct ionic_cq *rxcq = napi_to_cq(napi); 526 unsigned int qi = rxcq->bound_q->index; 527 struct ionic_dev *idev; 528 struct ionic_lif *lif; 529 struct ionic_cq *txcq; 530 u32 rx_work_done = 0; 531 u32 tx_work_done = 0; 532 u32 flags = 0; 533 534 lif = rxcq->bound_q->lif; 535 idev = &lif->ionic->idev; 536 txcq = &lif->txqcqs[qi]->cq; 537 538 tx_work_done = ionic_cq_service(txcq, lif->tx_budget, 539 ionic_tx_service, NULL, NULL); 540 541 rx_work_done = ionic_cq_service(rxcq, budget, 542 ionic_rx_service, NULL, NULL); 543 if (rx_work_done) 544 ionic_rx_fill_cb(rxcq->bound_q); 545 546 if (rx_work_done < budget && napi_complete_done(napi, rx_work_done)) { 547 ionic_dim_update(qcq, 0); 548 flags |= IONIC_INTR_CRED_UNMASK; 549 rxcq->bound_intr->rearm_count++; 550 } 551 552 if (rx_work_done || flags) { 553 flags |= IONIC_INTR_CRED_RESET_COALESCE; 554 ionic_intr_credits(idev->intr_ctrl, rxcq->bound_intr->index, 555 tx_work_done + rx_work_done, flags); 556 } 557 558 DEBUG_STATS_NAPI_POLL(qcq, rx_work_done); 559 DEBUG_STATS_NAPI_POLL(qcq, tx_work_done); 560 561 return rx_work_done; 562} 563 564static dma_addr_t ionic_tx_map_single(struct ionic_queue *q, 565 void *data, size_t len) 566{ 567 struct ionic_tx_stats *stats = q_to_tx_stats(q); 568 struct device *dev = q->lif->ionic->dev; 569 dma_addr_t dma_addr; 570 571 dma_addr = dma_map_single(dev, data, len, DMA_TO_DEVICE); 572 if (dma_mapping_error(dev, dma_addr)) { 573 net_warn_ratelimited("%s: DMA single map failed on %s!\n", 574 q->lif->netdev->name, q->name); 575 stats->dma_map_err++; 576 return 0; 577 } 578 return dma_addr; 579} 580 581static dma_addr_t ionic_tx_map_frag(struct ionic_queue *q, 582 const skb_frag_t *frag, 583 size_t offset, size_t len) 584{ 585 struct ionic_tx_stats *stats = q_to_tx_stats(q); 586 struct device *dev = q->lif->ionic->dev; 587 dma_addr_t dma_addr; 588 589 dma_addr = skb_frag_dma_map(dev, frag, offset, len, DMA_TO_DEVICE); 590 if (dma_mapping_error(dev, dma_addr)) { 591 net_warn_ratelimited("%s: DMA frag map failed on %s!\n", 592 q->lif->netdev->name, q->name); 593 stats->dma_map_err++; 594 } 595 return dma_addr; 596} 597 598static void ionic_tx_clean(struct ionic_queue *q, 599 struct ionic_desc_info *desc_info, 600 struct ionic_cq_info *cq_info, 601 void *cb_arg) 602{ 603 struct ionic_txq_sg_desc *sg_desc = desc_info->sg_desc; 604 struct ionic_txq_sg_elem *elem = sg_desc->elems; 605 struct ionic_tx_stats *stats = q_to_tx_stats(q); 606 struct ionic_txq_desc *desc = desc_info->desc; 607 struct device *dev = q->lif->ionic->dev; 608 u8 opcode, flags, nsge; 609 u16 queue_index; 610 unsigned int i; 611 u64 addr; 612 613 decode_txq_desc_cmd(le64_to_cpu(desc->cmd), 614 &opcode, &flags, &nsge, &addr); 615 616 /* use unmap_single only if either this is not TSO, 617 * or this is first descriptor of a TSO 618 */ 619 if (opcode != IONIC_TXQ_DESC_OPCODE_TSO || 620 flags & IONIC_TXQ_DESC_FLAG_TSO_SOT) 621 dma_unmap_single(dev, (dma_addr_t)addr, 622 le16_to_cpu(desc->len), DMA_TO_DEVICE); 623 else 624 dma_unmap_page(dev, (dma_addr_t)addr, 625 le16_to_cpu(desc->len), DMA_TO_DEVICE); 626 627 for (i = 0; i < nsge; i++, elem++) 628 dma_unmap_page(dev, (dma_addr_t)le64_to_cpu(elem->addr), 629 le16_to_cpu(elem->len), DMA_TO_DEVICE); 630 631 if (cb_arg) { 632 struct sk_buff *skb = cb_arg; 633 u32 len = skb->len; 634 635 queue_index = skb_get_queue_mapping(skb); 636 if (unlikely(__netif_subqueue_stopped(q->lif->netdev, 637 queue_index))) { 638 netif_wake_subqueue(q->lif->netdev, queue_index); 639 q->wake++; 640 } 641 dev_kfree_skb_any(skb); 642 stats->clean++; 643 netdev_tx_completed_queue(q_to_ndq(q), 1, len); 644 } 645} 646 647static bool ionic_tx_service(struct ionic_cq *cq, struct ionic_cq_info *cq_info) 648{ 649 struct ionic_txq_comp *comp = cq_info->cq_desc; 650 struct ionic_queue *q = cq->bound_q; 651 struct ionic_desc_info *desc_info; 652 u16 index; 653 654 if (!color_match(comp->color, cq->done_color)) 655 return false; 656 657 /* clean the related q entries, there could be 658 * several q entries completed for each cq completion 659 */ 660 do { 661 desc_info = &q->info[q->tail_idx]; 662 index = q->tail_idx; 663 q->tail_idx = (q->tail_idx + 1) & (q->num_descs - 1); 664 ionic_tx_clean(q, desc_info, cq_info, desc_info->cb_arg); 665 desc_info->cb = NULL; 666 desc_info->cb_arg = NULL; 667 } while (index != le16_to_cpu(comp->comp_index)); 668 669 return true; 670} 671 672void ionic_tx_flush(struct ionic_cq *cq) 673{ 674 struct ionic_dev *idev = &cq->lif->ionic->idev; 675 u32 work_done; 676 677 work_done = ionic_cq_service(cq, cq->num_descs, 678 ionic_tx_service, NULL, NULL); 679 if (work_done) 680 ionic_intr_credits(idev->intr_ctrl, cq->bound_intr->index, 681 work_done, IONIC_INTR_CRED_RESET_COALESCE); 682} 683 684void ionic_tx_empty(struct ionic_queue *q) 685{ 686 struct ionic_desc_info *desc_info; 687 688 /* walk the not completed tx entries, if any */ 689 while (q->head_idx != q->tail_idx) { 690 desc_info = &q->info[q->tail_idx]; 691 q->tail_idx = (q->tail_idx + 1) & (q->num_descs - 1); 692 ionic_tx_clean(q, desc_info, NULL, desc_info->cb_arg); 693 desc_info->cb = NULL; 694 desc_info->cb_arg = NULL; 695 } 696} 697 698static int ionic_tx_tcp_inner_pseudo_csum(struct sk_buff *skb) 699{ 700 int err; 701 702 err = skb_cow_head(skb, 0); 703 if (err) 704 return err; 705 706 if (skb->protocol == cpu_to_be16(ETH_P_IP)) { 707 inner_ip_hdr(skb)->check = 0; 708 inner_tcp_hdr(skb)->check = 709 ~csum_tcpudp_magic(inner_ip_hdr(skb)->saddr, 710 inner_ip_hdr(skb)->daddr, 711 0, IPPROTO_TCP, 0); 712 } else if (skb->protocol == cpu_to_be16(ETH_P_IPV6)) { 713 inner_tcp_hdr(skb)->check = 714 ~csum_ipv6_magic(&inner_ipv6_hdr(skb)->saddr, 715 &inner_ipv6_hdr(skb)->daddr, 716 0, IPPROTO_TCP, 0); 717 } 718 719 return 0; 720} 721 722static int ionic_tx_tcp_pseudo_csum(struct sk_buff *skb) 723{ 724 int err; 725 726 err = skb_cow_head(skb, 0); 727 if (err) 728 return err; 729 730 if (skb->protocol == cpu_to_be16(ETH_P_IP)) { 731 ip_hdr(skb)->check = 0; 732 tcp_hdr(skb)->check = 733 ~csum_tcpudp_magic(ip_hdr(skb)->saddr, 734 ip_hdr(skb)->daddr, 735 0, IPPROTO_TCP, 0); 736 } else if (skb->protocol == cpu_to_be16(ETH_P_IPV6)) { 737 tcp_v6_gso_csum_prep(skb); 738 } 739 740 return 0; 741} 742 743static void ionic_tx_tso_post(struct ionic_queue *q, struct ionic_txq_desc *desc, 744 struct sk_buff *skb, 745 dma_addr_t addr, u8 nsge, u16 len, 746 unsigned int hdrlen, unsigned int mss, 747 bool outer_csum, 748 u16 vlan_tci, bool has_vlan, 749 bool start, bool done) 750{ 751 u8 flags = 0; 752 u64 cmd; 753 754 flags |= has_vlan ? IONIC_TXQ_DESC_FLAG_VLAN : 0; 755 flags |= outer_csum ? IONIC_TXQ_DESC_FLAG_ENCAP : 0; 756 flags |= start ? IONIC_TXQ_DESC_FLAG_TSO_SOT : 0; 757 flags |= done ? IONIC_TXQ_DESC_FLAG_TSO_EOT : 0; 758 759 cmd = encode_txq_desc_cmd(IONIC_TXQ_DESC_OPCODE_TSO, flags, nsge, addr); 760 desc->cmd = cpu_to_le64(cmd); 761 desc->len = cpu_to_le16(len); 762 desc->vlan_tci = cpu_to_le16(vlan_tci); 763 desc->hdr_len = cpu_to_le16(hdrlen); 764 desc->mss = cpu_to_le16(mss); 765 766 if (done) { 767 skb_tx_timestamp(skb); 768 netdev_tx_sent_queue(q_to_ndq(q), skb->len); 769 ionic_txq_post(q, !netdev_xmit_more(), ionic_tx_clean, skb); 770 } else { 771 ionic_txq_post(q, false, ionic_tx_clean, NULL); 772 } 773} 774 775static struct ionic_txq_desc *ionic_tx_tso_next(struct ionic_queue *q, 776 struct ionic_txq_sg_elem **elem) 777{ 778 struct ionic_txq_sg_desc *sg_desc = q->info[q->head_idx].txq_sg_desc; 779 struct ionic_txq_desc *desc = q->info[q->head_idx].txq_desc; 780 781 *elem = sg_desc->elems; 782 return desc; 783} 784 785static int ionic_tx_tso(struct ionic_queue *q, struct sk_buff *skb) 786{ 787 struct ionic_tx_stats *stats = q_to_tx_stats(q); 788 struct ionic_desc_info *rewind_desc_info; 789 struct device *dev = q->lif->ionic->dev; 790 struct ionic_txq_sg_elem *elem; 791 struct ionic_txq_desc *desc; 792 unsigned int frag_left = 0; 793 unsigned int offset = 0; 794 u16 abort = q->head_idx; 795 unsigned int len_left; 796 dma_addr_t desc_addr; 797 unsigned int hdrlen; 798 unsigned int nfrags; 799 unsigned int seglen; 800 u64 total_bytes = 0; 801 u64 total_pkts = 0; 802 u16 rewind = abort; 803 unsigned int left; 804 unsigned int len; 805 unsigned int mss; 806 skb_frag_t *frag; 807 bool start, done; 808 bool outer_csum; 809 dma_addr_t addr; 810 bool has_vlan; 811 u16 desc_len; 812 u8 desc_nsge; 813 u16 vlan_tci; 814 bool encap; 815 int err; 816 817 mss = skb_shinfo(skb)->gso_size; 818 nfrags = skb_shinfo(skb)->nr_frags; 819 len_left = skb->len - skb_headlen(skb); 820 outer_csum = (skb_shinfo(skb)->gso_type & SKB_GSO_GRE_CSUM) || 821 (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_TUNNEL_CSUM); 822 has_vlan = !!skb_vlan_tag_present(skb); 823 vlan_tci = skb_vlan_tag_get(skb); 824 encap = skb->encapsulation; 825 826 /* Preload inner-most TCP csum field with IP pseudo hdr 827 * calculated with IP length set to zero. HW will later 828 * add in length to each TCP segment resulting from the TSO. 829 */ 830 831 if (encap) 832 err = ionic_tx_tcp_inner_pseudo_csum(skb); 833 else 834 err = ionic_tx_tcp_pseudo_csum(skb); 835 if (err) 836 return err; 837 838 if (encap) 839 hdrlen = skb_inner_transport_header(skb) - skb->data + 840 inner_tcp_hdrlen(skb); 841 else 842 hdrlen = skb_transport_offset(skb) + tcp_hdrlen(skb); 843 844 seglen = hdrlen + mss; 845 left = skb_headlen(skb); 846 847 desc = ionic_tx_tso_next(q, &elem); 848 start = true; 849 850 /* Chop skb->data up into desc segments */ 851 852 while (left > 0) { 853 len = min(seglen, left); 854 frag_left = seglen - len; 855 desc_addr = ionic_tx_map_single(q, skb->data + offset, len); 856 if (dma_mapping_error(dev, desc_addr)) 857 goto err_out_abort; 858 desc_len = len; 859 desc_nsge = 0; 860 left -= len; 861 offset += len; 862 if (nfrags > 0 && frag_left > 0) 863 continue; 864 done = (nfrags == 0 && left == 0); 865 ionic_tx_tso_post(q, desc, skb, 866 desc_addr, desc_nsge, desc_len, 867 hdrlen, mss, 868 outer_csum, 869 vlan_tci, has_vlan, 870 start, done); 871 total_pkts++; 872 total_bytes += start ? len : len + hdrlen; 873 desc = ionic_tx_tso_next(q, &elem); 874 start = false; 875 seglen = mss; 876 } 877 878 /* Chop skb frags into desc segments */ 879 880 for (frag = skb_shinfo(skb)->frags; len_left; frag++) { 881 offset = 0; 882 left = skb_frag_size(frag); 883 len_left -= left; 884 nfrags--; 885 stats->frags++; 886 887 while (left > 0) { 888 if (frag_left > 0) { 889 len = min(frag_left, left); 890 frag_left -= len; 891 addr = ionic_tx_map_frag(q, frag, offset, len); 892 if (dma_mapping_error(dev, addr)) 893 goto err_out_abort; 894 elem->addr = cpu_to_le64(addr); 895 elem->len = cpu_to_le16(len); 896 elem++; 897 desc_nsge++; 898 left -= len; 899 offset += len; 900 if (nfrags > 0 && frag_left > 0) 901 continue; 902 done = (nfrags == 0 && left == 0); 903 ionic_tx_tso_post(q, desc, skb, desc_addr, 904 desc_nsge, desc_len, 905 hdrlen, mss, outer_csum, 906 vlan_tci, has_vlan, 907 start, done); 908 total_pkts++; 909 total_bytes += start ? len : len + hdrlen; 910 desc = ionic_tx_tso_next(q, &elem); 911 start = false; 912 } else { 913 len = min(mss, left); 914 frag_left = mss - len; 915 desc_addr = ionic_tx_map_frag(q, frag, 916 offset, len); 917 if (dma_mapping_error(dev, desc_addr)) 918 goto err_out_abort; 919 desc_len = len; 920 desc_nsge = 0; 921 left -= len; 922 offset += len; 923 if (nfrags > 0 && frag_left > 0) 924 continue; 925 done = (nfrags == 0 && left == 0); 926 ionic_tx_tso_post(q, desc, skb, desc_addr, 927 desc_nsge, desc_len, 928 hdrlen, mss, outer_csum, 929 vlan_tci, has_vlan, 930 start, done); 931 total_pkts++; 932 total_bytes += start ? len : len + hdrlen; 933 desc = ionic_tx_tso_next(q, &elem); 934 start = false; 935 } 936 } 937 } 938 939 stats->pkts += total_pkts; 940 stats->bytes += total_bytes; 941 stats->tso++; 942 stats->tso_bytes += total_bytes; 943 944 return 0; 945 946err_out_abort: 947 while (rewind != q->head_idx) { 948 rewind_desc_info = &q->info[rewind]; 949 ionic_tx_clean(q, rewind_desc_info, NULL, NULL); 950 rewind = (rewind + 1) & (q->num_descs - 1); 951 } 952 q->head_idx = abort; 953 954 return -ENOMEM; 955} 956 957static int ionic_tx_calc_csum(struct ionic_queue *q, struct sk_buff *skb) 958{ 959 struct ionic_txq_desc *desc = q->info[q->head_idx].txq_desc; 960 struct ionic_tx_stats *stats = q_to_tx_stats(q); 961 struct device *dev = q->lif->ionic->dev; 962 dma_addr_t dma_addr; 963 bool has_vlan; 964 u8 flags = 0; 965 bool encap; 966 u64 cmd; 967 968 has_vlan = !!skb_vlan_tag_present(skb); 969 encap = skb->encapsulation; 970 971 dma_addr = ionic_tx_map_single(q, skb->data, skb_headlen(skb)); 972 if (dma_mapping_error(dev, dma_addr)) 973 return -ENOMEM; 974 975 flags |= has_vlan ? IONIC_TXQ_DESC_FLAG_VLAN : 0; 976 flags |= encap ? IONIC_TXQ_DESC_FLAG_ENCAP : 0; 977 978 cmd = encode_txq_desc_cmd(IONIC_TXQ_DESC_OPCODE_CSUM_PARTIAL, 979 flags, skb_shinfo(skb)->nr_frags, dma_addr); 980 desc->cmd = cpu_to_le64(cmd); 981 desc->len = cpu_to_le16(skb_headlen(skb)); 982 desc->csum_start = cpu_to_le16(skb_checksum_start_offset(skb)); 983 desc->csum_offset = cpu_to_le16(skb->csum_offset); 984 if (has_vlan) { 985 desc->vlan_tci = cpu_to_le16(skb_vlan_tag_get(skb)); 986 stats->vlan_inserted++; 987 } 988 989 if (skb_csum_is_sctp(skb)) 990 stats->crc32_csum++; 991 else 992 stats->csum++; 993 994 return 0; 995} 996 997static int ionic_tx_calc_no_csum(struct ionic_queue *q, struct sk_buff *skb) 998{ 999 struct ionic_txq_desc *desc = q->info[q->head_idx].txq_desc; 1000 struct ionic_tx_stats *stats = q_to_tx_stats(q); 1001 struct device *dev = q->lif->ionic->dev; 1002 dma_addr_t dma_addr; 1003 bool has_vlan; 1004 u8 flags = 0; 1005 bool encap; 1006 u64 cmd; 1007 1008 has_vlan = !!skb_vlan_tag_present(skb); 1009 encap = skb->encapsulation; 1010 1011 dma_addr = ionic_tx_map_single(q, skb->data, skb_headlen(skb)); 1012 if (dma_mapping_error(dev, dma_addr)) 1013 return -ENOMEM; 1014 1015 flags |= has_vlan ? IONIC_TXQ_DESC_FLAG_VLAN : 0; 1016 flags |= encap ? IONIC_TXQ_DESC_FLAG_ENCAP : 0; 1017 1018 cmd = encode_txq_desc_cmd(IONIC_TXQ_DESC_OPCODE_CSUM_NONE, 1019 flags, skb_shinfo(skb)->nr_frags, dma_addr); 1020 desc->cmd = cpu_to_le64(cmd); 1021 desc->len = cpu_to_le16(skb_headlen(skb)); 1022 if (has_vlan) { 1023 desc->vlan_tci = cpu_to_le16(skb_vlan_tag_get(skb)); 1024 stats->vlan_inserted++; 1025 } 1026 1027 stats->csum_none++; 1028 1029 return 0; 1030} 1031 1032static int ionic_tx_skb_frags(struct ionic_queue *q, struct sk_buff *skb) 1033{ 1034 struct ionic_txq_sg_desc *sg_desc = q->info[q->head_idx].txq_sg_desc; 1035 unsigned int len_left = skb->len - skb_headlen(skb); 1036 struct ionic_txq_sg_elem *elem = sg_desc->elems; 1037 struct ionic_tx_stats *stats = q_to_tx_stats(q); 1038 struct device *dev = q->lif->ionic->dev; 1039 dma_addr_t dma_addr; 1040 skb_frag_t *frag; 1041 u16 len; 1042 1043 for (frag = skb_shinfo(skb)->frags; len_left; frag++, elem++) { 1044 len = skb_frag_size(frag); 1045 elem->len = cpu_to_le16(len); 1046 dma_addr = ionic_tx_map_frag(q, frag, 0, len); 1047 if (dma_mapping_error(dev, dma_addr)) 1048 return -ENOMEM; 1049 elem->addr = cpu_to_le64(dma_addr); 1050 len_left -= len; 1051 stats->frags++; 1052 } 1053 1054 return 0; 1055} 1056 1057static int ionic_tx(struct ionic_queue *q, struct sk_buff *skb) 1058{ 1059 struct ionic_tx_stats *stats = q_to_tx_stats(q); 1060 int err; 1061 1062 /* set up the initial descriptor */ 1063 if (skb->ip_summed == CHECKSUM_PARTIAL) 1064 err = ionic_tx_calc_csum(q, skb); 1065 else 1066 err = ionic_tx_calc_no_csum(q, skb); 1067 if (err) 1068 return err; 1069 1070 /* add frags */ 1071 err = ionic_tx_skb_frags(q, skb); 1072 if (err) 1073 return err; 1074 1075 skb_tx_timestamp(skb); 1076 stats->pkts++; 1077 stats->bytes += skb->len; 1078 1079 netdev_tx_sent_queue(q_to_ndq(q), skb->len); 1080 ionic_txq_post(q, !netdev_xmit_more(), ionic_tx_clean, skb); 1081 1082 return 0; 1083} 1084 1085static int ionic_tx_descs_needed(struct ionic_queue *q, struct sk_buff *skb) 1086{ 1087 int sg_elems = q->lif->qtype_info[IONIC_QTYPE_TXQ].max_sg_elems; 1088 struct ionic_tx_stats *stats = q_to_tx_stats(q); 1089 int ndescs; 1090 int err; 1091 1092 /* Each desc is mss long max, so a descriptor for each gso_seg */ 1093 if (skb_is_gso(skb)) 1094 ndescs = skb_shinfo(skb)->gso_segs; 1095 else 1096 ndescs = 1; 1097 1098 if (skb_shinfo(skb)->nr_frags <= sg_elems) 1099 return ndescs; 1100 1101 /* Too many frags, so linearize */ 1102 err = skb_linearize(skb); 1103 if (err) 1104 return err; 1105 1106 stats->linearize++; 1107 1108 return ndescs; 1109} 1110 1111static int ionic_maybe_stop_tx(struct ionic_queue *q, int ndescs) 1112{ 1113 int stopped = 0; 1114 1115 if (unlikely(!ionic_q_has_space(q, ndescs))) { 1116 netif_stop_subqueue(q->lif->netdev, q->index); 1117 q->stop++; 1118 stopped = 1; 1119 1120 /* Might race with ionic_tx_clean, check again */ 1121 smp_rmb(); 1122 if (ionic_q_has_space(q, ndescs)) { 1123 netif_wake_subqueue(q->lif->netdev, q->index); 1124 stopped = 0; 1125 } 1126 } 1127 1128 return stopped; 1129} 1130 1131netdev_tx_t ionic_start_xmit(struct sk_buff *skb, struct net_device *netdev) 1132{ 1133 u16 queue_index = skb_get_queue_mapping(skb); 1134 struct ionic_lif *lif = netdev_priv(netdev); 1135 struct ionic_queue *q; 1136 int ndescs; 1137 int err; 1138 1139 if (unlikely(!test_bit(IONIC_LIF_F_UP, lif->state))) { 1140 dev_kfree_skb(skb); 1141 return NETDEV_TX_OK; 1142 } 1143 1144 if (unlikely(queue_index >= lif->nxqs)) 1145 queue_index = 0; 1146 q = &lif->txqcqs[queue_index]->q; 1147 1148 ndescs = ionic_tx_descs_needed(q, skb); 1149 if (ndescs < 0) 1150 goto err_out_drop; 1151 1152 if (unlikely(ionic_maybe_stop_tx(q, ndescs))) 1153 return NETDEV_TX_BUSY; 1154 1155 if (skb_is_gso(skb)) 1156 err = ionic_tx_tso(q, skb); 1157 else 1158 err = ionic_tx(q, skb); 1159 1160 if (err) 1161 goto err_out_drop; 1162 1163 /* Stop the queue if there aren't descriptors for the next packet. 1164 * Since our SG lists per descriptor take care of most of the possible 1165 * fragmentation, we don't need to have many descriptors available. 1166 */ 1167 ionic_maybe_stop_tx(q, 4); 1168 1169 return NETDEV_TX_OK; 1170 1171err_out_drop: 1172 q->stop++; 1173 q->drop++; 1174 dev_kfree_skb(skb); 1175 return NETDEV_TX_OK; 1176} 1177