1// SPDX-License-Identifier: GPL-2.0 2/* Copyright (c) 2019, Intel Corporation. */ 3 4#include <linux/bpf_trace.h> 5#include <net/xdp_sock_drv.h> 6#include <net/xdp.h> 7#include "ice.h" 8#include "ice_base.h" 9#include "ice_type.h" 10#include "ice_xsk.h" 11#include "ice_txrx.h" 12#include "ice_txrx_lib.h" 13#include "ice_lib.h" 14 15static struct xdp_buff **ice_xdp_buf(struct ice_rx_ring *rx_ring, u32 idx) 16{ 17 return &rx_ring->xdp_buf[idx]; 18} 19 20/** 21 * ice_qp_reset_stats - Resets all stats for rings of given index 22 * @vsi: VSI that contains rings of interest 23 * @q_idx: ring index in array 24 */ 25static void ice_qp_reset_stats(struct ice_vsi *vsi, u16 q_idx) 26{ 27 struct ice_vsi_stats *vsi_stat; 28 struct ice_pf *pf; 29 30 pf = vsi->back; 31 if (!pf->vsi_stats) 32 return; 33 34 vsi_stat = pf->vsi_stats[vsi->idx]; 35 if (!vsi_stat) 36 return; 37 38 memset(&vsi_stat->rx_ring_stats[q_idx]->rx_stats, 0, 39 sizeof(vsi_stat->rx_ring_stats[q_idx]->rx_stats)); 40 memset(&vsi_stat->tx_ring_stats[q_idx]->stats, 0, 41 sizeof(vsi_stat->tx_ring_stats[q_idx]->stats)); 42 if (ice_is_xdp_ena_vsi(vsi)) 43 memset(&vsi->xdp_rings[q_idx]->ring_stats->stats, 0, 44 sizeof(vsi->xdp_rings[q_idx]->ring_stats->stats)); 45} 46 47/** 48 * ice_qp_clean_rings - Cleans all the rings of a given index 49 * @vsi: VSI that contains rings of interest 50 * @q_idx: ring index in array 51 */ 52static void ice_qp_clean_rings(struct ice_vsi *vsi, u16 q_idx) 53{ 54 ice_clean_tx_ring(vsi->tx_rings[q_idx]); 55 if (ice_is_xdp_ena_vsi(vsi)) { 56 synchronize_rcu(); 57 ice_clean_tx_ring(vsi->xdp_rings[q_idx]); 58 } 59 ice_clean_rx_ring(vsi->rx_rings[q_idx]); 60} 61 62/** 63 * ice_qvec_toggle_napi - Enables/disables NAPI for a given q_vector 64 * @vsi: VSI that has netdev 65 * @q_vector: q_vector that has NAPI context 66 * @enable: true for enable, false for disable 67 */ 68static void 69ice_qvec_toggle_napi(struct ice_vsi *vsi, struct ice_q_vector *q_vector, 70 bool enable) 71{ 72 if (!vsi->netdev || !q_vector) 73 return; 74 75 if (enable) 76 napi_enable(&q_vector->napi); 77 else 78 napi_disable(&q_vector->napi); 79} 80 81/** 82 * ice_qvec_dis_irq - Mask off queue interrupt generation on given ring 83 * @vsi: the VSI that contains queue vector being un-configured 84 * @rx_ring: Rx ring that will have its IRQ disabled 85 * @q_vector: queue vector 86 */ 87static void 88ice_qvec_dis_irq(struct ice_vsi *vsi, struct ice_rx_ring *rx_ring, 89 struct ice_q_vector *q_vector) 90{ 91 struct ice_pf *pf = vsi->back; 92 struct ice_hw *hw = &pf->hw; 93 u16 reg; 94 u32 val; 95 96 /* QINT_TQCTL is being cleared in ice_vsi_stop_tx_ring, so handle 97 * here only QINT_RQCTL 98 */ 99 reg = rx_ring->reg_idx; 100 val = rd32(hw, QINT_RQCTL(reg)); 101 val &= ~QINT_RQCTL_CAUSE_ENA_M; 102 wr32(hw, QINT_RQCTL(reg), val); 103 104 if (q_vector) { 105 wr32(hw, GLINT_DYN_CTL(q_vector->reg_idx), 0); 106 ice_flush(hw); 107 synchronize_irq(q_vector->irq.virq); 108 } 109} 110 111/** 112 * ice_qvec_cfg_msix - Enable IRQ for given queue vector 113 * @vsi: the VSI that contains queue vector 114 * @q_vector: queue vector 115 */ 116static void 117ice_qvec_cfg_msix(struct ice_vsi *vsi, struct ice_q_vector *q_vector) 118{ 119 u16 reg_idx = q_vector->reg_idx; 120 struct ice_pf *pf = vsi->back; 121 struct ice_hw *hw = &pf->hw; 122 struct ice_tx_ring *tx_ring; 123 struct ice_rx_ring *rx_ring; 124 125 ice_cfg_itr(hw, q_vector); 126 127 ice_for_each_tx_ring(tx_ring, q_vector->tx) 128 ice_cfg_txq_interrupt(vsi, tx_ring->reg_idx, reg_idx, 129 q_vector->tx.itr_idx); 130 131 ice_for_each_rx_ring(rx_ring, q_vector->rx) 132 ice_cfg_rxq_interrupt(vsi, rx_ring->reg_idx, reg_idx, 133 q_vector->rx.itr_idx); 134 135 ice_flush(hw); 136} 137 138/** 139 * ice_qvec_ena_irq - Enable IRQ for given queue vector 140 * @vsi: the VSI that contains queue vector 141 * @q_vector: queue vector 142 */ 143static void ice_qvec_ena_irq(struct ice_vsi *vsi, struct ice_q_vector *q_vector) 144{ 145 struct ice_pf *pf = vsi->back; 146 struct ice_hw *hw = &pf->hw; 147 148 ice_irq_dynamic_ena(hw, vsi, q_vector); 149 150 ice_flush(hw); 151} 152 153/** 154 * ice_qp_dis - Disables a queue pair 155 * @vsi: VSI of interest 156 * @q_idx: ring index in array 157 * 158 * Returns 0 on success, negative on failure. 159 */ 160static int ice_qp_dis(struct ice_vsi *vsi, u16 q_idx) 161{ 162 struct ice_txq_meta txq_meta = { }; 163 struct ice_q_vector *q_vector; 164 struct ice_tx_ring *tx_ring; 165 struct ice_rx_ring *rx_ring; 166 int timeout = 50; 167 int err; 168 169 if (q_idx >= vsi->num_rxq || q_idx >= vsi->num_txq) 170 return -EINVAL; 171 172 tx_ring = vsi->tx_rings[q_idx]; 173 rx_ring = vsi->rx_rings[q_idx]; 174 q_vector = rx_ring->q_vector; 175 176 while (test_and_set_bit(ICE_CFG_BUSY, vsi->state)) { 177 timeout--; 178 if (!timeout) 179 return -EBUSY; 180 usleep_range(1000, 2000); 181 } 182 183 ice_qvec_dis_irq(vsi, rx_ring, q_vector); 184 ice_qvec_toggle_napi(vsi, q_vector, false); 185 186 netif_tx_stop_queue(netdev_get_tx_queue(vsi->netdev, q_idx)); 187 188 ice_fill_txq_meta(vsi, tx_ring, &txq_meta); 189 err = ice_vsi_stop_tx_ring(vsi, ICE_NO_RESET, 0, tx_ring, &txq_meta); 190 if (err) 191 return err; 192 if (ice_is_xdp_ena_vsi(vsi)) { 193 struct ice_tx_ring *xdp_ring = vsi->xdp_rings[q_idx]; 194 195 memset(&txq_meta, 0, sizeof(txq_meta)); 196 ice_fill_txq_meta(vsi, xdp_ring, &txq_meta); 197 err = ice_vsi_stop_tx_ring(vsi, ICE_NO_RESET, 0, xdp_ring, 198 &txq_meta); 199 if (err) 200 return err; 201 } 202 err = ice_vsi_ctrl_one_rx_ring(vsi, false, q_idx, true); 203 if (err) 204 return err; 205 206 ice_qp_clean_rings(vsi, q_idx); 207 ice_qp_reset_stats(vsi, q_idx); 208 209 return 0; 210} 211 212/** 213 * ice_qp_ena - Enables a queue pair 214 * @vsi: VSI of interest 215 * @q_idx: ring index in array 216 * 217 * Returns 0 on success, negative on failure. 218 */ 219static int ice_qp_ena(struct ice_vsi *vsi, u16 q_idx) 220{ 221 struct ice_aqc_add_tx_qgrp *qg_buf; 222 struct ice_q_vector *q_vector; 223 struct ice_tx_ring *tx_ring; 224 struct ice_rx_ring *rx_ring; 225 u16 size; 226 int err; 227 228 if (q_idx >= vsi->num_rxq || q_idx >= vsi->num_txq) 229 return -EINVAL; 230 231 size = struct_size(qg_buf, txqs, 1); 232 qg_buf = kzalloc(size, GFP_KERNEL); 233 if (!qg_buf) 234 return -ENOMEM; 235 236 qg_buf->num_txqs = 1; 237 238 tx_ring = vsi->tx_rings[q_idx]; 239 rx_ring = vsi->rx_rings[q_idx]; 240 q_vector = rx_ring->q_vector; 241 242 err = ice_vsi_cfg_txq(vsi, tx_ring, qg_buf); 243 if (err) 244 goto free_buf; 245 246 if (ice_is_xdp_ena_vsi(vsi)) { 247 struct ice_tx_ring *xdp_ring = vsi->xdp_rings[q_idx]; 248 249 memset(qg_buf, 0, size); 250 qg_buf->num_txqs = 1; 251 err = ice_vsi_cfg_txq(vsi, xdp_ring, qg_buf); 252 if (err) 253 goto free_buf; 254 ice_set_ring_xdp(xdp_ring); 255 ice_tx_xsk_pool(vsi, q_idx); 256 } 257 258 err = ice_vsi_cfg_rxq(rx_ring); 259 if (err) 260 goto free_buf; 261 262 ice_qvec_cfg_msix(vsi, q_vector); 263 264 err = ice_vsi_ctrl_one_rx_ring(vsi, true, q_idx, true); 265 if (err) 266 goto free_buf; 267 268 ice_qvec_toggle_napi(vsi, q_vector, true); 269 ice_qvec_ena_irq(vsi, q_vector); 270 271 netif_tx_start_queue(netdev_get_tx_queue(vsi->netdev, q_idx)); 272 clear_bit(ICE_CFG_BUSY, vsi->state); 273free_buf: 274 kfree(qg_buf); 275 return err; 276} 277 278/** 279 * ice_xsk_pool_disable - disable a buffer pool region 280 * @vsi: Current VSI 281 * @qid: queue ID 282 * 283 * Returns 0 on success, negative on failure 284 */ 285static int ice_xsk_pool_disable(struct ice_vsi *vsi, u16 qid) 286{ 287 struct xsk_buff_pool *pool = xsk_get_pool_from_qid(vsi->netdev, qid); 288 289 if (!pool) 290 return -EINVAL; 291 292 clear_bit(qid, vsi->af_xdp_zc_qps); 293 xsk_pool_dma_unmap(pool, ICE_RX_DMA_ATTR); 294 295 return 0; 296} 297 298/** 299 * ice_xsk_pool_enable - enable a buffer pool region 300 * @vsi: Current VSI 301 * @pool: pointer to a requested buffer pool region 302 * @qid: queue ID 303 * 304 * Returns 0 on success, negative on failure 305 */ 306static int 307ice_xsk_pool_enable(struct ice_vsi *vsi, struct xsk_buff_pool *pool, u16 qid) 308{ 309 int err; 310 311 if (vsi->type != ICE_VSI_PF) 312 return -EINVAL; 313 314 if (qid >= vsi->netdev->real_num_rx_queues || 315 qid >= vsi->netdev->real_num_tx_queues) 316 return -EINVAL; 317 318 err = xsk_pool_dma_map(pool, ice_pf_to_dev(vsi->back), 319 ICE_RX_DMA_ATTR); 320 if (err) 321 return err; 322 323 set_bit(qid, vsi->af_xdp_zc_qps); 324 325 return 0; 326} 327 328/** 329 * ice_realloc_rx_xdp_bufs - reallocate for either XSK or normal buffer 330 * @rx_ring: Rx ring 331 * @pool_present: is pool for XSK present 332 * 333 * Try allocating memory and return ENOMEM, if failed to allocate. 334 * If allocation was successful, substitute buffer with allocated one. 335 * Returns 0 on success, negative on failure 336 */ 337static int 338ice_realloc_rx_xdp_bufs(struct ice_rx_ring *rx_ring, bool pool_present) 339{ 340 size_t elem_size = pool_present ? sizeof(*rx_ring->xdp_buf) : 341 sizeof(*rx_ring->rx_buf); 342 void *sw_ring = kcalloc(rx_ring->count, elem_size, GFP_KERNEL); 343 344 if (!sw_ring) 345 return -ENOMEM; 346 347 if (pool_present) { 348 kfree(rx_ring->rx_buf); 349 rx_ring->rx_buf = NULL; 350 rx_ring->xdp_buf = sw_ring; 351 } else { 352 kfree(rx_ring->xdp_buf); 353 rx_ring->xdp_buf = NULL; 354 rx_ring->rx_buf = sw_ring; 355 } 356 357 return 0; 358} 359 360/** 361 * ice_realloc_zc_buf - reallocate XDP ZC queue pairs 362 * @vsi: Current VSI 363 * @zc: is zero copy set 364 * 365 * Reallocate buffer for rx_rings that might be used by XSK. 366 * XDP requires more memory, than rx_buf provides. 367 * Returns 0 on success, negative on failure 368 */ 369int ice_realloc_zc_buf(struct ice_vsi *vsi, bool zc) 370{ 371 struct ice_rx_ring *rx_ring; 372 unsigned long q; 373 374 for_each_set_bit(q, vsi->af_xdp_zc_qps, 375 max_t(int, vsi->alloc_txq, vsi->alloc_rxq)) { 376 rx_ring = vsi->rx_rings[q]; 377 if (ice_realloc_rx_xdp_bufs(rx_ring, zc)) 378 return -ENOMEM; 379 } 380 381 return 0; 382} 383 384/** 385 * ice_xsk_pool_setup - enable/disable a buffer pool region depending on its state 386 * @vsi: Current VSI 387 * @pool: buffer pool to enable/associate to a ring, NULL to disable 388 * @qid: queue ID 389 * 390 * Returns 0 on success, negative on failure 391 */ 392int ice_xsk_pool_setup(struct ice_vsi *vsi, struct xsk_buff_pool *pool, u16 qid) 393{ 394 bool if_running, pool_present = !!pool; 395 int ret = 0, pool_failure = 0; 396 397 if (qid >= vsi->num_rxq || qid >= vsi->num_txq) { 398 netdev_err(vsi->netdev, "Please use queue id in scope of combined queues count\n"); 399 pool_failure = -EINVAL; 400 goto failure; 401 } 402 403 if_running = netif_running(vsi->netdev) && ice_is_xdp_ena_vsi(vsi); 404 405 if (if_running) { 406 struct ice_rx_ring *rx_ring = vsi->rx_rings[qid]; 407 408 ret = ice_qp_dis(vsi, qid); 409 if (ret) { 410 netdev_err(vsi->netdev, "ice_qp_dis error = %d\n", ret); 411 goto xsk_pool_if_up; 412 } 413 414 ret = ice_realloc_rx_xdp_bufs(rx_ring, pool_present); 415 if (ret) 416 goto xsk_pool_if_up; 417 } 418 419 pool_failure = pool_present ? ice_xsk_pool_enable(vsi, pool, qid) : 420 ice_xsk_pool_disable(vsi, qid); 421 422xsk_pool_if_up: 423 if (if_running) { 424 ret = ice_qp_ena(vsi, qid); 425 if (!ret && pool_present) 426 napi_schedule(&vsi->rx_rings[qid]->xdp_ring->q_vector->napi); 427 else if (ret) 428 netdev_err(vsi->netdev, "ice_qp_ena error = %d\n", ret); 429 } 430 431failure: 432 if (pool_failure) { 433 netdev_err(vsi->netdev, "Could not %sable buffer pool, error = %d\n", 434 pool_present ? "en" : "dis", pool_failure); 435 return pool_failure; 436 } 437 438 return ret; 439} 440 441/** 442 * ice_fill_rx_descs - pick buffers from XSK buffer pool and use it 443 * @pool: XSK Buffer pool to pull the buffers from 444 * @xdp: SW ring of xdp_buff that will hold the buffers 445 * @rx_desc: Pointer to Rx descriptors that will be filled 446 * @count: The number of buffers to allocate 447 * 448 * This function allocates a number of Rx buffers from the fill ring 449 * or the internal recycle mechanism and places them on the Rx ring. 450 * 451 * Note that ring wrap should be handled by caller of this function. 452 * 453 * Returns the amount of allocated Rx descriptors 454 */ 455static u16 ice_fill_rx_descs(struct xsk_buff_pool *pool, struct xdp_buff **xdp, 456 union ice_32b_rx_flex_desc *rx_desc, u16 count) 457{ 458 dma_addr_t dma; 459 u16 buffs; 460 int i; 461 462 buffs = xsk_buff_alloc_batch(pool, xdp, count); 463 for (i = 0; i < buffs; i++) { 464 dma = xsk_buff_xdp_get_dma(*xdp); 465 rx_desc->read.pkt_addr = cpu_to_le64(dma); 466 rx_desc->wb.status_error0 = 0; 467 468 rx_desc++; 469 xdp++; 470 } 471 472 return buffs; 473} 474 475/** 476 * __ice_alloc_rx_bufs_zc - allocate a number of Rx buffers 477 * @rx_ring: Rx ring 478 * @count: The number of buffers to allocate 479 * 480 * Place the @count of descriptors onto Rx ring. Handle the ring wrap 481 * for case where space from next_to_use up to the end of ring is less 482 * than @count. Finally do a tail bump. 483 * 484 * Returns true if all allocations were successful, false if any fail. 485 */ 486static bool __ice_alloc_rx_bufs_zc(struct ice_rx_ring *rx_ring, u16 count) 487{ 488 u32 nb_buffs_extra = 0, nb_buffs = 0; 489 union ice_32b_rx_flex_desc *rx_desc; 490 u16 ntu = rx_ring->next_to_use; 491 u16 total_count = count; 492 struct xdp_buff **xdp; 493 494 rx_desc = ICE_RX_DESC(rx_ring, ntu); 495 xdp = ice_xdp_buf(rx_ring, ntu); 496 497 if (ntu + count >= rx_ring->count) { 498 nb_buffs_extra = ice_fill_rx_descs(rx_ring->xsk_pool, xdp, 499 rx_desc, 500 rx_ring->count - ntu); 501 if (nb_buffs_extra != rx_ring->count - ntu) { 502 ntu += nb_buffs_extra; 503 goto exit; 504 } 505 rx_desc = ICE_RX_DESC(rx_ring, 0); 506 xdp = ice_xdp_buf(rx_ring, 0); 507 ntu = 0; 508 count -= nb_buffs_extra; 509 ice_release_rx_desc(rx_ring, 0); 510 } 511 512 nb_buffs = ice_fill_rx_descs(rx_ring->xsk_pool, xdp, rx_desc, count); 513 514 ntu += nb_buffs; 515 if (ntu == rx_ring->count) 516 ntu = 0; 517 518exit: 519 if (rx_ring->next_to_use != ntu) 520 ice_release_rx_desc(rx_ring, ntu); 521 522 return total_count == (nb_buffs_extra + nb_buffs); 523} 524 525/** 526 * ice_alloc_rx_bufs_zc - allocate a number of Rx buffers 527 * @rx_ring: Rx ring 528 * @count: The number of buffers to allocate 529 * 530 * Wrapper for internal allocation routine; figure out how many tail 531 * bumps should take place based on the given threshold 532 * 533 * Returns true if all calls to internal alloc routine succeeded 534 */ 535bool ice_alloc_rx_bufs_zc(struct ice_rx_ring *rx_ring, u16 count) 536{ 537 u16 rx_thresh = ICE_RING_QUARTER(rx_ring); 538 u16 leftover, i, tail_bumps; 539 540 tail_bumps = count / rx_thresh; 541 leftover = count - (tail_bumps * rx_thresh); 542 543 for (i = 0; i < tail_bumps; i++) 544 if (!__ice_alloc_rx_bufs_zc(rx_ring, rx_thresh)) 545 return false; 546 return __ice_alloc_rx_bufs_zc(rx_ring, leftover); 547} 548 549/** 550 * ice_construct_skb_zc - Create an sk_buff from zero-copy buffer 551 * @rx_ring: Rx ring 552 * @xdp: Pointer to XDP buffer 553 * 554 * This function allocates a new skb from a zero-copy Rx buffer. 555 * 556 * Returns the skb on success, NULL on failure. 557 */ 558static struct sk_buff * 559ice_construct_skb_zc(struct ice_rx_ring *rx_ring, struct xdp_buff *xdp) 560{ 561 unsigned int totalsize = xdp->data_end - xdp->data_meta; 562 unsigned int metasize = xdp->data - xdp->data_meta; 563 struct skb_shared_info *sinfo = NULL; 564 struct sk_buff *skb; 565 u32 nr_frags = 0; 566 567 if (unlikely(xdp_buff_has_frags(xdp))) { 568 sinfo = xdp_get_shared_info_from_buff(xdp); 569 nr_frags = sinfo->nr_frags; 570 } 571 net_prefetch(xdp->data_meta); 572 573 skb = __napi_alloc_skb(&rx_ring->q_vector->napi, totalsize, 574 GFP_ATOMIC | __GFP_NOWARN); 575 if (unlikely(!skb)) 576 return NULL; 577 578 memcpy(__skb_put(skb, totalsize), xdp->data_meta, 579 ALIGN(totalsize, sizeof(long))); 580 581 if (metasize) { 582 skb_metadata_set(skb, metasize); 583 __skb_pull(skb, metasize); 584 } 585 586 if (likely(!xdp_buff_has_frags(xdp))) 587 goto out; 588 589 for (int i = 0; i < nr_frags; i++) { 590 struct skb_shared_info *skinfo = skb_shinfo(skb); 591 skb_frag_t *frag = &sinfo->frags[i]; 592 struct page *page; 593 void *addr; 594 595 page = dev_alloc_page(); 596 if (!page) { 597 dev_kfree_skb(skb); 598 return NULL; 599 } 600 addr = page_to_virt(page); 601 602 memcpy(addr, skb_frag_page(frag), skb_frag_size(frag)); 603 604 __skb_fill_page_desc_noacc(skinfo, skinfo->nr_frags++, 605 addr, 0, skb_frag_size(frag)); 606 } 607 608out: 609 xsk_buff_free(xdp); 610 return skb; 611} 612 613/** 614 * ice_clean_xdp_irq_zc - produce AF_XDP descriptors to CQ 615 * @xdp_ring: XDP Tx ring 616 */ 617static u32 ice_clean_xdp_irq_zc(struct ice_tx_ring *xdp_ring) 618{ 619 u16 ntc = xdp_ring->next_to_clean; 620 struct ice_tx_desc *tx_desc; 621 u16 cnt = xdp_ring->count; 622 struct ice_tx_buf *tx_buf; 623 u16 completed_frames = 0; 624 u16 xsk_frames = 0; 625 u16 last_rs; 626 int i; 627 628 last_rs = xdp_ring->next_to_use ? xdp_ring->next_to_use - 1 : cnt - 1; 629 tx_desc = ICE_TX_DESC(xdp_ring, last_rs); 630 if (tx_desc->cmd_type_offset_bsz & 631 cpu_to_le64(ICE_TX_DESC_DTYPE_DESC_DONE)) { 632 if (last_rs >= ntc) 633 completed_frames = last_rs - ntc + 1; 634 else 635 completed_frames = last_rs + cnt - ntc + 1; 636 } 637 638 if (!completed_frames) 639 return 0; 640 641 if (likely(!xdp_ring->xdp_tx_active)) { 642 xsk_frames = completed_frames; 643 goto skip; 644 } 645 646 ntc = xdp_ring->next_to_clean; 647 for (i = 0; i < completed_frames; i++) { 648 tx_buf = &xdp_ring->tx_buf[ntc]; 649 650 if (tx_buf->type == ICE_TX_BUF_XSK_TX) { 651 tx_buf->type = ICE_TX_BUF_EMPTY; 652 xsk_buff_free(tx_buf->xdp); 653 xdp_ring->xdp_tx_active--; 654 } else { 655 xsk_frames++; 656 } 657 658 ntc++; 659 if (ntc >= xdp_ring->count) 660 ntc = 0; 661 } 662skip: 663 tx_desc->cmd_type_offset_bsz = 0; 664 xdp_ring->next_to_clean += completed_frames; 665 if (xdp_ring->next_to_clean >= cnt) 666 xdp_ring->next_to_clean -= cnt; 667 if (xsk_frames) 668 xsk_tx_completed(xdp_ring->xsk_pool, xsk_frames); 669 670 return completed_frames; 671} 672 673/** 674 * ice_xmit_xdp_tx_zc - AF_XDP ZC handler for XDP_TX 675 * @xdp: XDP buffer to xmit 676 * @xdp_ring: XDP ring to produce descriptor onto 677 * 678 * note that this function works directly on xdp_buff, no need to convert 679 * it to xdp_frame. xdp_buff pointer is stored to ice_tx_buf so that cleaning 680 * side will be able to xsk_buff_free() it. 681 * 682 * Returns ICE_XDP_TX for successfully produced desc, ICE_XDP_CONSUMED if there 683 * was not enough space on XDP ring 684 */ 685static int ice_xmit_xdp_tx_zc(struct xdp_buff *xdp, 686 struct ice_tx_ring *xdp_ring) 687{ 688 struct skb_shared_info *sinfo = NULL; 689 u32 size = xdp->data_end - xdp->data; 690 u32 ntu = xdp_ring->next_to_use; 691 struct ice_tx_desc *tx_desc; 692 struct ice_tx_buf *tx_buf; 693 struct xdp_buff *head; 694 u32 nr_frags = 0; 695 u32 free_space; 696 u32 frag = 0; 697 698 free_space = ICE_DESC_UNUSED(xdp_ring); 699 if (free_space < ICE_RING_QUARTER(xdp_ring)) 700 free_space += ice_clean_xdp_irq_zc(xdp_ring); 701 702 if (unlikely(!free_space)) 703 goto busy; 704 705 if (unlikely(xdp_buff_has_frags(xdp))) { 706 sinfo = xdp_get_shared_info_from_buff(xdp); 707 nr_frags = sinfo->nr_frags; 708 if (free_space < nr_frags + 1) 709 goto busy; 710 } 711 712 tx_desc = ICE_TX_DESC(xdp_ring, ntu); 713 tx_buf = &xdp_ring->tx_buf[ntu]; 714 head = xdp; 715 716 for (;;) { 717 dma_addr_t dma; 718 719 dma = xsk_buff_xdp_get_dma(xdp); 720 xsk_buff_raw_dma_sync_for_device(xdp_ring->xsk_pool, dma, size); 721 722 tx_buf->xdp = xdp; 723 tx_buf->type = ICE_TX_BUF_XSK_TX; 724 tx_desc->buf_addr = cpu_to_le64(dma); 725 tx_desc->cmd_type_offset_bsz = ice_build_ctob(0, 0, size, 0); 726 /* account for each xdp_buff from xsk_buff_pool */ 727 xdp_ring->xdp_tx_active++; 728 729 if (++ntu == xdp_ring->count) 730 ntu = 0; 731 732 if (frag == nr_frags) 733 break; 734 735 tx_desc = ICE_TX_DESC(xdp_ring, ntu); 736 tx_buf = &xdp_ring->tx_buf[ntu]; 737 738 xdp = xsk_buff_get_frag(head); 739 size = skb_frag_size(&sinfo->frags[frag]); 740 frag++; 741 } 742 743 xdp_ring->next_to_use = ntu; 744 /* update last descriptor from a frame with EOP */ 745 tx_desc->cmd_type_offset_bsz |= 746 cpu_to_le64(ICE_TX_DESC_CMD_EOP << ICE_TXD_QW1_CMD_S); 747 748 return ICE_XDP_TX; 749 750busy: 751 xdp_ring->ring_stats->tx_stats.tx_busy++; 752 753 return ICE_XDP_CONSUMED; 754} 755 756/** 757 * ice_run_xdp_zc - Executes an XDP program in zero-copy path 758 * @rx_ring: Rx ring 759 * @xdp: xdp_buff used as input to the XDP program 760 * @xdp_prog: XDP program to run 761 * @xdp_ring: ring to be used for XDP_TX action 762 * 763 * Returns any of ICE_XDP_{PASS, CONSUMED, TX, REDIR} 764 */ 765static int 766ice_run_xdp_zc(struct ice_rx_ring *rx_ring, struct xdp_buff *xdp, 767 struct bpf_prog *xdp_prog, struct ice_tx_ring *xdp_ring) 768{ 769 int err, result = ICE_XDP_PASS; 770 u32 act; 771 772 act = bpf_prog_run_xdp(xdp_prog, xdp); 773 774 if (likely(act == XDP_REDIRECT)) { 775 err = xdp_do_redirect(rx_ring->netdev, xdp, xdp_prog); 776 if (!err) 777 return ICE_XDP_REDIR; 778 if (xsk_uses_need_wakeup(rx_ring->xsk_pool) && err == -ENOBUFS) 779 result = ICE_XDP_EXIT; 780 else 781 result = ICE_XDP_CONSUMED; 782 goto out_failure; 783 } 784 785 switch (act) { 786 case XDP_PASS: 787 break; 788 case XDP_TX: 789 result = ice_xmit_xdp_tx_zc(xdp, xdp_ring); 790 if (result == ICE_XDP_CONSUMED) 791 goto out_failure; 792 break; 793 case XDP_DROP: 794 result = ICE_XDP_CONSUMED; 795 break; 796 default: 797 bpf_warn_invalid_xdp_action(rx_ring->netdev, xdp_prog, act); 798 fallthrough; 799 case XDP_ABORTED: 800 result = ICE_XDP_CONSUMED; 801out_failure: 802 trace_xdp_exception(rx_ring->netdev, xdp_prog, act); 803 break; 804 } 805 806 return result; 807} 808 809static int 810ice_add_xsk_frag(struct ice_rx_ring *rx_ring, struct xdp_buff *first, 811 struct xdp_buff *xdp, const unsigned int size) 812{ 813 struct skb_shared_info *sinfo = xdp_get_shared_info_from_buff(first); 814 815 if (!size) 816 return 0; 817 818 if (!xdp_buff_has_frags(first)) { 819 sinfo->nr_frags = 0; 820 sinfo->xdp_frags_size = 0; 821 xdp_buff_set_frags_flag(first); 822 } 823 824 if (unlikely(sinfo->nr_frags == MAX_SKB_FRAGS)) { 825 xsk_buff_free(first); 826 return -ENOMEM; 827 } 828 829 __skb_fill_page_desc_noacc(sinfo, sinfo->nr_frags++, 830 virt_to_page(xdp->data_hard_start), 831 XDP_PACKET_HEADROOM, size); 832 sinfo->xdp_frags_size += size; 833 xsk_buff_add_frag(xdp); 834 835 return 0; 836} 837 838/** 839 * ice_clean_rx_irq_zc - consumes packets from the hardware ring 840 * @rx_ring: AF_XDP Rx ring 841 * @budget: NAPI budget 842 * 843 * Returns number of processed packets on success, remaining budget on failure. 844 */ 845int ice_clean_rx_irq_zc(struct ice_rx_ring *rx_ring, int budget) 846{ 847 unsigned int total_rx_bytes = 0, total_rx_packets = 0; 848 struct xsk_buff_pool *xsk_pool = rx_ring->xsk_pool; 849 u32 ntc = rx_ring->next_to_clean; 850 u32 ntu = rx_ring->next_to_use; 851 struct xdp_buff *first = NULL; 852 struct ice_tx_ring *xdp_ring; 853 unsigned int xdp_xmit = 0; 854 struct bpf_prog *xdp_prog; 855 u32 cnt = rx_ring->count; 856 bool failure = false; 857 int entries_to_alloc; 858 859 /* ZC patch is enabled only when XDP program is set, 860 * so here it can not be NULL 861 */ 862 xdp_prog = READ_ONCE(rx_ring->xdp_prog); 863 xdp_ring = rx_ring->xdp_ring; 864 865 if (ntc != rx_ring->first_desc) 866 first = *ice_xdp_buf(rx_ring, rx_ring->first_desc); 867 868 while (likely(total_rx_packets < (unsigned int)budget)) { 869 union ice_32b_rx_flex_desc *rx_desc; 870 unsigned int size, xdp_res = 0; 871 struct xdp_buff *xdp; 872 struct sk_buff *skb; 873 u16 stat_err_bits; 874 u16 vlan_tag = 0; 875 u16 rx_ptype; 876 877 rx_desc = ICE_RX_DESC(rx_ring, ntc); 878 879 stat_err_bits = BIT(ICE_RX_FLEX_DESC_STATUS0_DD_S); 880 if (!ice_test_staterr(rx_desc->wb.status_error0, stat_err_bits)) 881 break; 882 883 /* This memory barrier is needed to keep us from reading 884 * any other fields out of the rx_desc until we have 885 * verified the descriptor has been written back. 886 */ 887 dma_rmb(); 888 889 if (unlikely(ntc == ntu)) 890 break; 891 892 xdp = *ice_xdp_buf(rx_ring, ntc); 893 894 size = le16_to_cpu(rx_desc->wb.pkt_len) & 895 ICE_RX_FLX_DESC_PKT_LEN_M; 896 897 xsk_buff_set_size(xdp, size); 898 xsk_buff_dma_sync_for_cpu(xdp, xsk_pool); 899 900 if (!first) { 901 first = xdp; 902 } else if (ice_add_xsk_frag(rx_ring, first, xdp, size)) { 903 break; 904 } 905 906 if (++ntc == cnt) 907 ntc = 0; 908 909 if (ice_is_non_eop(rx_ring, rx_desc)) 910 continue; 911 912 xdp_res = ice_run_xdp_zc(rx_ring, first, xdp_prog, xdp_ring); 913 if (likely(xdp_res & (ICE_XDP_TX | ICE_XDP_REDIR))) { 914 xdp_xmit |= xdp_res; 915 } else if (xdp_res == ICE_XDP_EXIT) { 916 failure = true; 917 first = NULL; 918 rx_ring->first_desc = ntc; 919 break; 920 } else if (xdp_res == ICE_XDP_CONSUMED) { 921 xsk_buff_free(first); 922 } else if (xdp_res == ICE_XDP_PASS) { 923 goto construct_skb; 924 } 925 926 total_rx_bytes += xdp_get_buff_len(first); 927 total_rx_packets++; 928 929 first = NULL; 930 rx_ring->first_desc = ntc; 931 continue; 932 933construct_skb: 934 /* XDP_PASS path */ 935 skb = ice_construct_skb_zc(rx_ring, first); 936 if (!skb) { 937 rx_ring->ring_stats->rx_stats.alloc_buf_failed++; 938 break; 939 } 940 941 first = NULL; 942 rx_ring->first_desc = ntc; 943 944 if (eth_skb_pad(skb)) { 945 skb = NULL; 946 continue; 947 } 948 949 total_rx_bytes += skb->len; 950 total_rx_packets++; 951 952 vlan_tag = ice_get_vlan_tag_from_rx_desc(rx_desc); 953 954 rx_ptype = le16_to_cpu(rx_desc->wb.ptype_flex_flags0) & 955 ICE_RX_FLEX_DESC_PTYPE_M; 956 957 ice_process_skb_fields(rx_ring, rx_desc, skb, rx_ptype); 958 ice_receive_skb(rx_ring, skb, vlan_tag); 959 } 960 961 rx_ring->next_to_clean = ntc; 962 entries_to_alloc = ICE_RX_DESC_UNUSED(rx_ring); 963 if (entries_to_alloc > ICE_RING_QUARTER(rx_ring)) 964 failure |= !ice_alloc_rx_bufs_zc(rx_ring, entries_to_alloc); 965 966 ice_finalize_xdp_rx(xdp_ring, xdp_xmit, 0); 967 ice_update_rx_ring_stats(rx_ring, total_rx_packets, total_rx_bytes); 968 969 if (xsk_uses_need_wakeup(xsk_pool)) { 970 /* ntu could have changed when allocating entries above, so 971 * use rx_ring value instead of stack based one 972 */ 973 if (failure || ntc == rx_ring->next_to_use) 974 xsk_set_rx_need_wakeup(xsk_pool); 975 else 976 xsk_clear_rx_need_wakeup(xsk_pool); 977 978 return (int)total_rx_packets; 979 } 980 981 return failure ? budget : (int)total_rx_packets; 982} 983 984/** 985 * ice_xmit_pkt - produce a single HW Tx descriptor out of AF_XDP descriptor 986 * @xdp_ring: XDP ring to produce the HW Tx descriptor on 987 * @desc: AF_XDP descriptor to pull the DMA address and length from 988 * @total_bytes: bytes accumulator that will be used for stats update 989 */ 990static void ice_xmit_pkt(struct ice_tx_ring *xdp_ring, struct xdp_desc *desc, 991 unsigned int *total_bytes) 992{ 993 struct ice_tx_desc *tx_desc; 994 dma_addr_t dma; 995 996 dma = xsk_buff_raw_get_dma(xdp_ring->xsk_pool, desc->addr); 997 xsk_buff_raw_dma_sync_for_device(xdp_ring->xsk_pool, dma, desc->len); 998 999 tx_desc = ICE_TX_DESC(xdp_ring, xdp_ring->next_to_use++); 1000 tx_desc->buf_addr = cpu_to_le64(dma); 1001 tx_desc->cmd_type_offset_bsz = ice_build_ctob(xsk_is_eop_desc(desc), 1002 0, desc->len, 0); 1003 1004 *total_bytes += desc->len; 1005} 1006 1007/** 1008 * ice_xmit_pkt_batch - produce a batch of HW Tx descriptors out of AF_XDP descriptors 1009 * @xdp_ring: XDP ring to produce the HW Tx descriptors on 1010 * @descs: AF_XDP descriptors to pull the DMA addresses and lengths from 1011 * @total_bytes: bytes accumulator that will be used for stats update 1012 */ 1013static void ice_xmit_pkt_batch(struct ice_tx_ring *xdp_ring, struct xdp_desc *descs, 1014 unsigned int *total_bytes) 1015{ 1016 u16 ntu = xdp_ring->next_to_use; 1017 struct ice_tx_desc *tx_desc; 1018 u32 i; 1019 1020 loop_unrolled_for(i = 0; i < PKTS_PER_BATCH; i++) { 1021 dma_addr_t dma; 1022 1023 dma = xsk_buff_raw_get_dma(xdp_ring->xsk_pool, descs[i].addr); 1024 xsk_buff_raw_dma_sync_for_device(xdp_ring->xsk_pool, dma, descs[i].len); 1025 1026 tx_desc = ICE_TX_DESC(xdp_ring, ntu++); 1027 tx_desc->buf_addr = cpu_to_le64(dma); 1028 tx_desc->cmd_type_offset_bsz = ice_build_ctob(xsk_is_eop_desc(&descs[i]), 1029 0, descs[i].len, 0); 1030 1031 *total_bytes += descs[i].len; 1032 } 1033 1034 xdp_ring->next_to_use = ntu; 1035} 1036 1037/** 1038 * ice_fill_tx_hw_ring - produce the number of Tx descriptors onto ring 1039 * @xdp_ring: XDP ring to produce the HW Tx descriptors on 1040 * @descs: AF_XDP descriptors to pull the DMA addresses and lengths from 1041 * @nb_pkts: count of packets to be send 1042 * @total_bytes: bytes accumulator that will be used for stats update 1043 */ 1044static void ice_fill_tx_hw_ring(struct ice_tx_ring *xdp_ring, struct xdp_desc *descs, 1045 u32 nb_pkts, unsigned int *total_bytes) 1046{ 1047 u32 batched, leftover, i; 1048 1049 batched = ALIGN_DOWN(nb_pkts, PKTS_PER_BATCH); 1050 leftover = nb_pkts & (PKTS_PER_BATCH - 1); 1051 for (i = 0; i < batched; i += PKTS_PER_BATCH) 1052 ice_xmit_pkt_batch(xdp_ring, &descs[i], total_bytes); 1053 for (; i < batched + leftover; i++) 1054 ice_xmit_pkt(xdp_ring, &descs[i], total_bytes); 1055} 1056 1057/** 1058 * ice_xmit_zc - take entries from XSK Tx ring and place them onto HW Tx ring 1059 * @xdp_ring: XDP ring to produce the HW Tx descriptors on 1060 * 1061 * Returns true if there is no more work that needs to be done, false otherwise 1062 */ 1063bool ice_xmit_zc(struct ice_tx_ring *xdp_ring) 1064{ 1065 struct xdp_desc *descs = xdp_ring->xsk_pool->tx_descs; 1066 u32 nb_pkts, nb_processed = 0; 1067 unsigned int total_bytes = 0; 1068 int budget; 1069 1070 ice_clean_xdp_irq_zc(xdp_ring); 1071 1072 budget = ICE_DESC_UNUSED(xdp_ring); 1073 budget = min_t(u16, budget, ICE_RING_QUARTER(xdp_ring)); 1074 1075 nb_pkts = xsk_tx_peek_release_desc_batch(xdp_ring->xsk_pool, budget); 1076 if (!nb_pkts) 1077 return true; 1078 1079 if (xdp_ring->next_to_use + nb_pkts >= xdp_ring->count) { 1080 nb_processed = xdp_ring->count - xdp_ring->next_to_use; 1081 ice_fill_tx_hw_ring(xdp_ring, descs, nb_processed, &total_bytes); 1082 xdp_ring->next_to_use = 0; 1083 } 1084 1085 ice_fill_tx_hw_ring(xdp_ring, &descs[nb_processed], nb_pkts - nb_processed, 1086 &total_bytes); 1087 1088 ice_set_rs_bit(xdp_ring); 1089 ice_xdp_ring_update_tail(xdp_ring); 1090 ice_update_tx_ring_stats(xdp_ring, nb_pkts, total_bytes); 1091 1092 if (xsk_uses_need_wakeup(xdp_ring->xsk_pool)) 1093 xsk_set_tx_need_wakeup(xdp_ring->xsk_pool); 1094 1095 return nb_pkts < budget; 1096} 1097 1098/** 1099 * ice_xsk_wakeup - Implements ndo_xsk_wakeup 1100 * @netdev: net_device 1101 * @queue_id: queue to wake up 1102 * @flags: ignored in our case, since we have Rx and Tx in the same NAPI 1103 * 1104 * Returns negative on error, zero otherwise. 1105 */ 1106int 1107ice_xsk_wakeup(struct net_device *netdev, u32 queue_id, 1108 u32 __always_unused flags) 1109{ 1110 struct ice_netdev_priv *np = netdev_priv(netdev); 1111 struct ice_q_vector *q_vector; 1112 struct ice_vsi *vsi = np->vsi; 1113 struct ice_tx_ring *ring; 1114 1115 if (test_bit(ICE_VSI_DOWN, vsi->state)) 1116 return -ENETDOWN; 1117 1118 if (!ice_is_xdp_ena_vsi(vsi)) 1119 return -EINVAL; 1120 1121 if (queue_id >= vsi->num_txq || queue_id >= vsi->num_rxq) 1122 return -EINVAL; 1123 1124 ring = vsi->rx_rings[queue_id]->xdp_ring; 1125 1126 if (!ring->xsk_pool) 1127 return -EINVAL; 1128 1129 /* The idea here is that if NAPI is running, mark a miss, so 1130 * it will run again. If not, trigger an interrupt and 1131 * schedule the NAPI from interrupt context. If NAPI would be 1132 * scheduled here, the interrupt affinity would not be 1133 * honored. 1134 */ 1135 q_vector = ring->q_vector; 1136 if (!napi_if_scheduled_mark_missed(&q_vector->napi)) 1137 ice_trigger_sw_intr(&vsi->back->hw, q_vector); 1138 1139 return 0; 1140} 1141 1142/** 1143 * ice_xsk_any_rx_ring_ena - Checks if Rx rings have AF_XDP buff pool attached 1144 * @vsi: VSI to be checked 1145 * 1146 * Returns true if any of the Rx rings has an AF_XDP buff pool attached 1147 */ 1148bool ice_xsk_any_rx_ring_ena(struct ice_vsi *vsi) 1149{ 1150 int i; 1151 1152 ice_for_each_rxq(vsi, i) { 1153 if (xsk_get_pool_from_qid(vsi->netdev, i)) 1154 return true; 1155 } 1156 1157 return false; 1158} 1159 1160/** 1161 * ice_xsk_clean_rx_ring - clean buffer pool queues connected to a given Rx ring 1162 * @rx_ring: ring to be cleaned 1163 */ 1164void ice_xsk_clean_rx_ring(struct ice_rx_ring *rx_ring) 1165{ 1166 u16 ntc = rx_ring->next_to_clean; 1167 u16 ntu = rx_ring->next_to_use; 1168 1169 while (ntc != ntu) { 1170 struct xdp_buff *xdp = *ice_xdp_buf(rx_ring, ntc); 1171 1172 xsk_buff_free(xdp); 1173 ntc++; 1174 if (ntc >= rx_ring->count) 1175 ntc = 0; 1176 } 1177} 1178 1179/** 1180 * ice_xsk_clean_xdp_ring - Clean the XDP Tx ring and its buffer pool queues 1181 * @xdp_ring: XDP_Tx ring 1182 */ 1183void ice_xsk_clean_xdp_ring(struct ice_tx_ring *xdp_ring) 1184{ 1185 u16 ntc = xdp_ring->next_to_clean, ntu = xdp_ring->next_to_use; 1186 u32 xsk_frames = 0; 1187 1188 while (ntc != ntu) { 1189 struct ice_tx_buf *tx_buf = &xdp_ring->tx_buf[ntc]; 1190 1191 if (tx_buf->type == ICE_TX_BUF_XSK_TX) { 1192 tx_buf->type = ICE_TX_BUF_EMPTY; 1193 xsk_buff_free(tx_buf->xdp); 1194 } else { 1195 xsk_frames++; 1196 } 1197 1198 ntc++; 1199 if (ntc >= xdp_ring->count) 1200 ntc = 0; 1201 } 1202 1203 if (xsk_frames) 1204 xsk_tx_completed(xdp_ring->xsk_pool, xsk_frames); 1205} 1206