1// SPDX-License-Identifier: GPL-2.0-only 2/**************************************************************************** 3 * Driver for Solarflare network controllers and boards 4 * Copyright 2005-2006 Fen Systems Ltd. 5 * Copyright 2005-2013 Solarflare Communications Inc. 6 */ 7 8#include <linux/pci.h> 9#include <linux/tcp.h> 10#include <linux/ip.h> 11#include <linux/in.h> 12#include <linux/ipv6.h> 13#include <linux/slab.h> 14#include <net/ipv6.h> 15#include <linux/if_ether.h> 16#include <linux/highmem.h> 17#include <linux/cache.h> 18#include "net_driver.h" 19#include "efx.h" 20#include "io.h" 21#include "nic.h" 22#include "tx.h" 23#include "tx_common.h" 24#include "workarounds.h" 25#include "ef10_regs.h" 26 27#ifdef EFX_USE_PIO 28 29#define EFX_PIOBUF_SIZE_DEF ALIGN(256, L1_CACHE_BYTES) 30unsigned int efx_piobuf_size __read_mostly = EFX_PIOBUF_SIZE_DEF; 31 32#endif /* EFX_USE_PIO */ 33 34static inline u8 *efx_tx_get_copy_buffer(struct efx_tx_queue *tx_queue, 35 struct efx_tx_buffer *buffer) 36{ 37 unsigned int index = efx_tx_queue_get_insert_index(tx_queue); 38 struct efx_buffer *page_buf = 39 &tx_queue->cb_page[index >> (PAGE_SHIFT - EFX_TX_CB_ORDER)]; 40 unsigned int offset = 41 ((index << EFX_TX_CB_ORDER) + NET_IP_ALIGN) & (PAGE_SIZE - 1); 42 43 if (unlikely(!page_buf->addr) && 44 efx_nic_alloc_buffer(tx_queue->efx, page_buf, PAGE_SIZE, 45 GFP_ATOMIC)) 46 return NULL; 47 buffer->dma_addr = page_buf->dma_addr + offset; 48 buffer->unmap_len = 0; 49 return (u8 *)page_buf->addr + offset; 50} 51 52u8 *efx_tx_get_copy_buffer_limited(struct efx_tx_queue *tx_queue, 53 struct efx_tx_buffer *buffer, size_t len) 54{ 55 if (len > EFX_TX_CB_SIZE) 56 return NULL; 57 return efx_tx_get_copy_buffer(tx_queue, buffer); 58} 59 60static void efx_tx_maybe_stop_queue(struct efx_tx_queue *txq1) 61{ 62 /* We need to consider all queues that the net core sees as one */ 63 struct efx_nic *efx = txq1->efx; 64 struct efx_tx_queue *txq2; 65 unsigned int fill_level; 66 67 fill_level = efx_channel_tx_old_fill_level(txq1->channel); 68 if (likely(fill_level < efx->txq_stop_thresh)) 69 return; 70 71 /* We used the stale old_read_count above, which gives us a 72 * pessimistic estimate of the fill level (which may even 73 * validly be >= efx->txq_entries). Now try again using 74 * read_count (more likely to be a cache miss). 75 * 76 * If we read read_count and then conditionally stop the 77 * queue, it is possible for the completion path to race with 78 * us and complete all outstanding descriptors in the middle, 79 * after which there will be no more completions to wake it. 80 * Therefore we stop the queue first, then read read_count 81 * (with a memory barrier to ensure the ordering), then 82 * restart the queue if the fill level turns out to be low 83 * enough. 84 */ 85 netif_tx_stop_queue(txq1->core_txq); 86 smp_mb(); 87 efx_for_each_channel_tx_queue(txq2, txq1->channel) 88 txq2->old_read_count = READ_ONCE(txq2->read_count); 89 90 fill_level = efx_channel_tx_old_fill_level(txq1->channel); 91 EFX_WARN_ON_ONCE_PARANOID(fill_level >= efx->txq_entries); 92 if (likely(fill_level < efx->txq_stop_thresh)) { 93 smp_mb(); 94 if (likely(!efx->loopback_selftest)) 95 netif_tx_start_queue(txq1->core_txq); 96 } 97} 98 99static int efx_enqueue_skb_copy(struct efx_tx_queue *tx_queue, 100 struct sk_buff *skb) 101{ 102 unsigned int copy_len = skb->len; 103 struct efx_tx_buffer *buffer; 104 u8 *copy_buffer; 105 int rc; 106 107 EFX_WARN_ON_ONCE_PARANOID(copy_len > EFX_TX_CB_SIZE); 108 109 buffer = efx_tx_queue_get_insert_buffer(tx_queue); 110 111 copy_buffer = efx_tx_get_copy_buffer(tx_queue, buffer); 112 if (unlikely(!copy_buffer)) 113 return -ENOMEM; 114 115 rc = skb_copy_bits(skb, 0, copy_buffer, copy_len); 116 EFX_WARN_ON_PARANOID(rc); 117 buffer->len = copy_len; 118 119 buffer->skb = skb; 120 buffer->flags = EFX_TX_BUF_SKB; 121 122 ++tx_queue->insert_count; 123 return rc; 124} 125 126#ifdef EFX_USE_PIO 127 128struct efx_short_copy_buffer { 129 int used; 130 u8 buf[L1_CACHE_BYTES]; 131}; 132 133/* Copy to PIO, respecting that writes to PIO buffers must be dword aligned. 134 * Advances piobuf pointer. Leaves additional data in the copy buffer. 135 */ 136static void efx_memcpy_toio_aligned(struct efx_nic *efx, u8 __iomem **piobuf, 137 u8 *data, int len, 138 struct efx_short_copy_buffer *copy_buf) 139{ 140 int block_len = len & ~(sizeof(copy_buf->buf) - 1); 141 142 __iowrite64_copy(*piobuf, data, block_len >> 3); 143 *piobuf += block_len; 144 len -= block_len; 145 146 if (len) { 147 data += block_len; 148 BUG_ON(copy_buf->used); 149 BUG_ON(len > sizeof(copy_buf->buf)); 150 memcpy(copy_buf->buf, data, len); 151 copy_buf->used = len; 152 } 153} 154 155/* Copy to PIO, respecting dword alignment, popping data from copy buffer first. 156 * Advances piobuf pointer. Leaves additional data in the copy buffer. 157 */ 158static void efx_memcpy_toio_aligned_cb(struct efx_nic *efx, u8 __iomem **piobuf, 159 u8 *data, int len, 160 struct efx_short_copy_buffer *copy_buf) 161{ 162 if (copy_buf->used) { 163 /* if the copy buffer is partially full, fill it up and write */ 164 int copy_to_buf = 165 min_t(int, sizeof(copy_buf->buf) - copy_buf->used, len); 166 167 memcpy(copy_buf->buf + copy_buf->used, data, copy_to_buf); 168 copy_buf->used += copy_to_buf; 169 170 /* if we didn't fill it up then we're done for now */ 171 if (copy_buf->used < sizeof(copy_buf->buf)) 172 return; 173 174 __iowrite64_copy(*piobuf, copy_buf->buf, 175 sizeof(copy_buf->buf) >> 3); 176 *piobuf += sizeof(copy_buf->buf); 177 data += copy_to_buf; 178 len -= copy_to_buf; 179 copy_buf->used = 0; 180 } 181 182 efx_memcpy_toio_aligned(efx, piobuf, data, len, copy_buf); 183} 184 185static void efx_flush_copy_buffer(struct efx_nic *efx, u8 __iomem *piobuf, 186 struct efx_short_copy_buffer *copy_buf) 187{ 188 /* if there's anything in it, write the whole buffer, including junk */ 189 if (copy_buf->used) 190 __iowrite64_copy(piobuf, copy_buf->buf, 191 sizeof(copy_buf->buf) >> 3); 192} 193 194/* Traverse skb structure and copy fragments in to PIO buffer. 195 * Advances piobuf pointer. 196 */ 197static void efx_skb_copy_bits_to_pio(struct efx_nic *efx, struct sk_buff *skb, 198 u8 __iomem **piobuf, 199 struct efx_short_copy_buffer *copy_buf) 200{ 201 int i; 202 203 efx_memcpy_toio_aligned(efx, piobuf, skb->data, skb_headlen(skb), 204 copy_buf); 205 206 for (i = 0; i < skb_shinfo(skb)->nr_frags; ++i) { 207 skb_frag_t *f = &skb_shinfo(skb)->frags[i]; 208 u8 *vaddr; 209 210 vaddr = kmap_atomic(skb_frag_page(f)); 211 212 efx_memcpy_toio_aligned_cb(efx, piobuf, vaddr + skb_frag_off(f), 213 skb_frag_size(f), copy_buf); 214 kunmap_atomic(vaddr); 215 } 216 217 EFX_WARN_ON_ONCE_PARANOID(skb_shinfo(skb)->frag_list); 218} 219 220static int efx_enqueue_skb_pio(struct efx_tx_queue *tx_queue, 221 struct sk_buff *skb) 222{ 223 struct efx_tx_buffer *buffer = 224 efx_tx_queue_get_insert_buffer(tx_queue); 225 u8 __iomem *piobuf = tx_queue->piobuf; 226 227 /* Copy to PIO buffer. Ensure the writes are padded to the end 228 * of a cache line, as this is required for write-combining to be 229 * effective on at least x86. 230 */ 231 232 if (skb_shinfo(skb)->nr_frags) { 233 /* The size of the copy buffer will ensure all writes 234 * are the size of a cache line. 235 */ 236 struct efx_short_copy_buffer copy_buf; 237 238 copy_buf.used = 0; 239 240 efx_skb_copy_bits_to_pio(tx_queue->efx, skb, 241 &piobuf, ©_buf); 242 efx_flush_copy_buffer(tx_queue->efx, piobuf, ©_buf); 243 } else { 244 /* Pad the write to the size of a cache line. 245 * We can do this because we know the skb_shared_info struct is 246 * after the source, and the destination buffer is big enough. 247 */ 248 BUILD_BUG_ON(L1_CACHE_BYTES > 249 SKB_DATA_ALIGN(sizeof(struct skb_shared_info))); 250 __iowrite64_copy(tx_queue->piobuf, skb->data, 251 ALIGN(skb->len, L1_CACHE_BYTES) >> 3); 252 } 253 254 buffer->skb = skb; 255 buffer->flags = EFX_TX_BUF_SKB | EFX_TX_BUF_OPTION; 256 257 EFX_POPULATE_QWORD_5(buffer->option, 258 ESF_DZ_TX_DESC_IS_OPT, 1, 259 ESF_DZ_TX_OPTION_TYPE, ESE_DZ_TX_OPTION_DESC_PIO, 260 ESF_DZ_TX_PIO_CONT, 0, 261 ESF_DZ_TX_PIO_BYTE_CNT, skb->len, 262 ESF_DZ_TX_PIO_BUF_ADDR, 263 tx_queue->piobuf_offset); 264 ++tx_queue->insert_count; 265 return 0; 266} 267 268/* Decide whether we can use TX PIO, ie. write packet data directly into 269 * a buffer on the device. This can reduce latency at the expense of 270 * throughput, so we only do this if both hardware and software TX rings 271 * are empty, including all queues for the channel. This also ensures that 272 * only one packet at a time can be using the PIO buffer. If the xmit_more 273 * flag is set then we don't use this - there'll be another packet along 274 * shortly and we want to hold off the doorbell. 275 */ 276static bool efx_tx_may_pio(struct efx_tx_queue *tx_queue) 277{ 278 struct efx_channel *channel = tx_queue->channel; 279 280 if (!tx_queue->piobuf) 281 return false; 282 283 EFX_WARN_ON_ONCE_PARANOID(!channel->efx->type->option_descriptors); 284 285 efx_for_each_channel_tx_queue(tx_queue, channel) 286 if (!efx_nic_tx_is_empty(tx_queue, tx_queue->packet_write_count)) 287 return false; 288 289 return true; 290} 291#endif /* EFX_USE_PIO */ 292 293/* Send any pending traffic for a channel. xmit_more is shared across all 294 * queues for a channel, so we must check all of them. 295 */ 296static void efx_tx_send_pending(struct efx_channel *channel) 297{ 298 struct efx_tx_queue *q; 299 300 efx_for_each_channel_tx_queue(q, channel) { 301 if (q->xmit_pending) 302 efx_nic_push_buffers(q); 303 } 304} 305 306/* 307 * Add a socket buffer to a TX queue 308 * 309 * This maps all fragments of a socket buffer for DMA and adds them to 310 * the TX queue. The queue's insert pointer will be incremented by 311 * the number of fragments in the socket buffer. 312 * 313 * If any DMA mapping fails, any mapped fragments will be unmapped, 314 * the queue's insert pointer will be restored to its original value. 315 * 316 * This function is split out from efx_hard_start_xmit to allow the 317 * loopback test to direct packets via specific TX queues. 318 * 319 * Returns NETDEV_TX_OK. 320 * You must hold netif_tx_lock() to call this function. 321 */ 322netdev_tx_t __efx_enqueue_skb(struct efx_tx_queue *tx_queue, struct sk_buff *skb) 323{ 324 unsigned int old_insert_count = tx_queue->insert_count; 325 bool xmit_more = netdev_xmit_more(); 326 bool data_mapped = false; 327 unsigned int segments; 328 unsigned int skb_len; 329 int rc; 330 331 skb_len = skb->len; 332 segments = skb_is_gso(skb) ? skb_shinfo(skb)->gso_segs : 0; 333 if (segments == 1) 334 segments = 0; /* Don't use TSO for a single segment. */ 335 336 /* Handle TSO first - it's *possible* (although unlikely) that we might 337 * be passed a packet to segment that's smaller than the copybreak/PIO 338 * size limit. 339 */ 340 if (segments) { 341 switch (tx_queue->tso_version) { 342 case 1: 343 rc = efx_enqueue_skb_tso(tx_queue, skb, &data_mapped); 344 break; 345 case 2: 346 rc = efx_ef10_tx_tso_desc(tx_queue, skb, &data_mapped); 347 break; 348 case 0: /* No TSO on this queue, SW fallback needed */ 349 default: 350 rc = -EINVAL; 351 break; 352 } 353 if (rc == -EINVAL) { 354 rc = efx_tx_tso_fallback(tx_queue, skb); 355 tx_queue->tso_fallbacks++; 356 if (rc == 0) 357 return 0; 358 } 359 if (rc) 360 goto err; 361#ifdef EFX_USE_PIO 362 } else if (skb_len <= efx_piobuf_size && !xmit_more && 363 efx_tx_may_pio(tx_queue)) { 364 /* Use PIO for short packets with an empty queue. */ 365 if (efx_enqueue_skb_pio(tx_queue, skb)) 366 goto err; 367 tx_queue->pio_packets++; 368 data_mapped = true; 369#endif 370 } else if (skb->data_len && skb_len <= EFX_TX_CB_SIZE) { 371 /* Pad short packets or coalesce short fragmented packets. */ 372 if (efx_enqueue_skb_copy(tx_queue, skb)) 373 goto err; 374 tx_queue->cb_packets++; 375 data_mapped = true; 376 } 377 378 /* Map for DMA and create descriptors if we haven't done so already. */ 379 if (!data_mapped && (efx_tx_map_data(tx_queue, skb, segments))) 380 goto err; 381 382 efx_tx_maybe_stop_queue(tx_queue); 383 384 tx_queue->xmit_pending = true; 385 386 /* Pass off to hardware */ 387 if (__netdev_tx_sent_queue(tx_queue->core_txq, skb_len, xmit_more)) 388 efx_tx_send_pending(tx_queue->channel); 389 390 if (segments) { 391 tx_queue->tso_bursts++; 392 tx_queue->tso_packets += segments; 393 tx_queue->tx_packets += segments; 394 } else { 395 tx_queue->tx_packets++; 396 } 397 398 return NETDEV_TX_OK; 399 400 401err: 402 efx_enqueue_unwind(tx_queue, old_insert_count); 403 dev_kfree_skb_any(skb); 404 405 /* If we're not expecting another transmit and we had something to push 406 * on this queue or a partner queue then we need to push here to get the 407 * previous packets out. 408 */ 409 if (!xmit_more) 410 efx_tx_send_pending(tx_queue->channel); 411 412 return NETDEV_TX_OK; 413} 414 415static void efx_xdp_return_frames(int n, struct xdp_frame **xdpfs) 416{ 417 int i; 418 419 for (i = 0; i < n; i++) 420 xdp_return_frame_rx_napi(xdpfs[i]); 421} 422 423/* Transmit a packet from an XDP buffer 424 * 425 * Returns number of packets sent on success, error code otherwise. 426 * Runs in NAPI context, either in our poll (for XDP TX) or a different NIC 427 * (for XDP redirect). 428 */ 429int efx_xdp_tx_buffers(struct efx_nic *efx, int n, struct xdp_frame **xdpfs, 430 bool flush) 431{ 432 struct efx_tx_buffer *tx_buffer; 433 struct efx_tx_queue *tx_queue; 434 struct xdp_frame *xdpf; 435 dma_addr_t dma_addr; 436 unsigned int len; 437 int space; 438 int cpu; 439 int i; 440 441 cpu = raw_smp_processor_id(); 442 443 if (!efx->xdp_tx_queue_count || 444 unlikely(cpu >= efx->xdp_tx_queue_count)) 445 return -EINVAL; 446 447 tx_queue = efx->xdp_tx_queues[cpu]; 448 if (unlikely(!tx_queue)) 449 return -EINVAL; 450 451 if (unlikely(n && !xdpfs)) 452 return -EINVAL; 453 454 if (!n) 455 return 0; 456 457 /* Check for available space. We should never need multiple 458 * descriptors per frame. 459 */ 460 space = efx->txq_entries + 461 tx_queue->read_count - tx_queue->insert_count; 462 463 for (i = 0; i < n; i++) { 464 xdpf = xdpfs[i]; 465 466 if (i >= space) 467 break; 468 469 /* We'll want a descriptor for this tx. */ 470 prefetchw(__efx_tx_queue_get_insert_buffer(tx_queue)); 471 472 len = xdpf->len; 473 474 /* Map for DMA. */ 475 dma_addr = dma_map_single(&efx->pci_dev->dev, 476 xdpf->data, len, 477 DMA_TO_DEVICE); 478 if (dma_mapping_error(&efx->pci_dev->dev, dma_addr)) 479 break; 480 481 /* Create descriptor and set up for unmapping DMA. */ 482 tx_buffer = efx_tx_map_chunk(tx_queue, dma_addr, len); 483 tx_buffer->xdpf = xdpf; 484 tx_buffer->flags = EFX_TX_BUF_XDP | 485 EFX_TX_BUF_MAP_SINGLE; 486 tx_buffer->dma_offset = 0; 487 tx_buffer->unmap_len = len; 488 tx_queue->tx_packets++; 489 } 490 491 /* Pass mapped frames to hardware. */ 492 if (flush && i > 0) 493 efx_nic_push_buffers(tx_queue); 494 495 if (i == 0) 496 return -EIO; 497 498 efx_xdp_return_frames(n - i, xdpfs + i); 499 500 return i; 501} 502 503/* Initiate a packet transmission. We use one channel per CPU 504 * (sharing when we have more CPUs than channels). 505 * 506 * Context: non-blocking. 507 * Should always return NETDEV_TX_OK and consume the skb. 508 */ 509netdev_tx_t efx_hard_start_xmit(struct sk_buff *skb, 510 struct net_device *net_dev) 511{ 512 struct efx_nic *efx = netdev_priv(net_dev); 513 struct efx_tx_queue *tx_queue; 514 unsigned index, type; 515 516 EFX_WARN_ON_PARANOID(!netif_device_present(net_dev)); 517 518 index = skb_get_queue_mapping(skb); 519 type = efx_tx_csum_type_skb(skb); 520 if (index >= efx->n_tx_channels) { 521 index -= efx->n_tx_channels; 522 type |= EFX_TXQ_TYPE_HIGHPRI; 523 } 524 525 /* PTP "event" packet */ 526 if (unlikely(efx_xmit_with_hwtstamp(skb)) && 527 unlikely(efx_ptp_is_ptp_tx(efx, skb))) { 528 /* There may be existing transmits on the channel that are 529 * waiting for this packet to trigger the doorbell write. 530 * We need to send the packets at this point. 531 */ 532 efx_tx_send_pending(efx_get_tx_channel(efx, index)); 533 return efx_ptp_tx(efx, skb); 534 } 535 536 tx_queue = efx_get_tx_queue(efx, index, type); 537 if (WARN_ON_ONCE(!tx_queue)) { 538 /* We don't have a TXQ of the right type. 539 * This should never happen, as we don't advertise offload 540 * features unless we can support them. 541 */ 542 dev_kfree_skb_any(skb); 543 /* If we're not expecting another transmit and we had something to push 544 * on this queue or a partner queue then we need to push here to get the 545 * previous packets out. 546 */ 547 if (!netdev_xmit_more()) 548 efx_tx_send_pending(efx_get_tx_channel(efx, index)); 549 return NETDEV_TX_OK; 550 } 551 552 return __efx_enqueue_skb(tx_queue, skb); 553} 554 555void efx_xmit_done_single(struct efx_tx_queue *tx_queue) 556{ 557 unsigned int pkts_compl = 0, bytes_compl = 0; 558 unsigned int read_ptr; 559 bool finished = false; 560 561 read_ptr = tx_queue->read_count & tx_queue->ptr_mask; 562 563 while (!finished) { 564 struct efx_tx_buffer *buffer = &tx_queue->buffer[read_ptr]; 565 566 if (!efx_tx_buffer_in_use(buffer)) { 567 struct efx_nic *efx = tx_queue->efx; 568 569 netif_err(efx, hw, efx->net_dev, 570 "TX queue %d spurious single TX completion\n", 571 tx_queue->queue); 572 efx_schedule_reset(efx, RESET_TYPE_TX_SKIP); 573 return; 574 } 575 576 /* Need to check the flag before dequeueing. */ 577 if (buffer->flags & EFX_TX_BUF_SKB) 578 finished = true; 579 efx_dequeue_buffer(tx_queue, buffer, &pkts_compl, &bytes_compl); 580 581 ++tx_queue->read_count; 582 read_ptr = tx_queue->read_count & tx_queue->ptr_mask; 583 } 584 585 tx_queue->pkts_compl += pkts_compl; 586 tx_queue->bytes_compl += bytes_compl; 587 588 EFX_WARN_ON_PARANOID(pkts_compl != 1); 589 590 efx_xmit_done_check_empty(tx_queue); 591} 592 593void efx_init_tx_queue_core_txq(struct efx_tx_queue *tx_queue) 594{ 595 struct efx_nic *efx = tx_queue->efx; 596 597 /* Must be inverse of queue lookup in efx_hard_start_xmit() */ 598 tx_queue->core_txq = 599 netdev_get_tx_queue(efx->net_dev, 600 tx_queue->channel->channel + 601 ((tx_queue->type & EFX_TXQ_TYPE_HIGHPRI) ? 602 efx->n_tx_channels : 0)); 603} 604 605int efx_setup_tc(struct net_device *net_dev, enum tc_setup_type type, 606 void *type_data) 607{ 608 struct efx_nic *efx = netdev_priv(net_dev); 609 struct tc_mqprio_qopt *mqprio = type_data; 610 unsigned tc, num_tc; 611 612 if (type != TC_SETUP_QDISC_MQPRIO) 613 return -EOPNOTSUPP; 614 615 /* Only Siena supported highpri queues */ 616 if (efx_nic_rev(efx) > EFX_REV_SIENA_A0) 617 return -EOPNOTSUPP; 618 619 num_tc = mqprio->num_tc; 620 621 if (num_tc > EFX_MAX_TX_TC) 622 return -EINVAL; 623 624 mqprio->hw = TC_MQPRIO_HW_OFFLOAD_TCS; 625 626 if (num_tc == net_dev->num_tc) 627 return 0; 628 629 for (tc = 0; tc < num_tc; tc++) { 630 net_dev->tc_to_txq[tc].offset = tc * efx->n_tx_channels; 631 net_dev->tc_to_txq[tc].count = efx->n_tx_channels; 632 } 633 634 net_dev->num_tc = num_tc; 635 636 return netif_set_real_num_tx_queues(net_dev, 637 max_t(int, num_tc, 1) * 638 efx->n_tx_channels); 639} 640