1// SPDX-License-Identifier: GPL-2.0-only 2/**************************************************************************** 3 * Driver for Solarflare network controllers and boards 4 * Copyright 2018 Solarflare Communications Inc. 5 * Copyright 2019-2020 Xilinx Inc. 6 * 7 * This program is free software; you can redistribute it and/or modify it 8 * under the terms of the GNU General Public License version 2 as published 9 * by the Free Software Foundation, incorporated herein by reference. 10 */ 11 12#include <net/ip6_checksum.h> 13 14#include "net_driver.h" 15#include "tx_common.h" 16#include "nic_common.h" 17#include "mcdi_functions.h" 18#include "ef100_regs.h" 19#include "io.h" 20#include "ef100_tx.h" 21#include "ef100_nic.h" 22 23int ef100_tx_probe(struct efx_tx_queue *tx_queue) 24{ 25 /* Allocate an extra descriptor for the QMDA status completion entry */ 26 return efx_nic_alloc_buffer(tx_queue->efx, &tx_queue->txd.buf, 27 (tx_queue->ptr_mask + 2) * 28 sizeof(efx_oword_t), 29 GFP_KERNEL); 30} 31 32void ef100_tx_init(struct efx_tx_queue *tx_queue) 33{ 34 /* must be the inverse of lookup in efx_get_tx_channel */ 35 tx_queue->core_txq = 36 netdev_get_tx_queue(tx_queue->efx->net_dev, 37 tx_queue->channel->channel - 38 tx_queue->efx->tx_channel_offset); 39 40 /* This value is purely documentational; as EF100 never passes through 41 * the switch statement in tx.c:__efx_enqueue_skb(), that switch does 42 * not handle case 3. EF100's TSOv3 descriptors are generated by 43 * ef100_make_tso_desc(). 44 * Meanwhile, all efx_mcdi_tx_init() cares about is that it's not 2. 45 */ 46 tx_queue->tso_version = 3; 47 if (efx_mcdi_tx_init(tx_queue)) 48 netdev_WARN(tx_queue->efx->net_dev, 49 "failed to initialise TXQ %d\n", tx_queue->queue); 50} 51 52static bool ef100_tx_can_tso(struct efx_tx_queue *tx_queue, struct sk_buff *skb) 53{ 54 struct efx_nic *efx = tx_queue->efx; 55 struct ef100_nic_data *nic_data; 56 struct efx_tx_buffer *buffer; 57 struct tcphdr *tcphdr; 58 struct iphdr *iphdr; 59 size_t header_len; 60 u32 mss; 61 62 nic_data = efx->nic_data; 63 64 if (!skb_is_gso_tcp(skb)) 65 return false; 66 if (!(efx->net_dev->features & NETIF_F_TSO)) 67 return false; 68 69 mss = skb_shinfo(skb)->gso_size; 70 if (unlikely(mss < 4)) { 71 WARN_ONCE(1, "MSS of %u is too small for TSO\n", mss); 72 return false; 73 } 74 75 header_len = efx_tx_tso_header_length(skb); 76 if (header_len > nic_data->tso_max_hdr_len) 77 return false; 78 79 if (skb_shinfo(skb)->gso_segs > nic_data->tso_max_payload_num_segs) { 80 /* net_dev->gso_max_segs should've caught this */ 81 WARN_ON_ONCE(1); 82 return false; 83 } 84 85 if (skb->data_len / mss > nic_data->tso_max_frames) 86 return false; 87 88 /* net_dev->gso_max_size should've caught this */ 89 if (WARN_ON_ONCE(skb->data_len > nic_data->tso_max_payload_len)) 90 return false; 91 92 /* Reserve an empty buffer for the TSO V3 descriptor. 93 * Convey the length of the header since we already know it. 94 */ 95 buffer = efx_tx_queue_get_insert_buffer(tx_queue); 96 buffer->flags = EFX_TX_BUF_TSO_V3 | EFX_TX_BUF_CONT; 97 buffer->len = header_len; 98 buffer->unmap_len = 0; 99 buffer->skb = skb; 100 ++tx_queue->insert_count; 101 102 /* Adjust the TCP checksum to exclude the total length, since we set 103 * ED_INNER_IP_LEN in the descriptor. 104 */ 105 tcphdr = tcp_hdr(skb); 106 if (skb_is_gso_v6(skb)) { 107 tcphdr->check = ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr, 108 &ipv6_hdr(skb)->daddr, 109 0, IPPROTO_TCP, 0); 110 } else { 111 iphdr = ip_hdr(skb); 112 tcphdr->check = ~csum_tcpudp_magic(iphdr->saddr, iphdr->daddr, 113 0, IPPROTO_TCP, 0); 114 } 115 return true; 116} 117 118static efx_oword_t *ef100_tx_desc(struct efx_tx_queue *tx_queue, unsigned int index) 119{ 120 if (likely(tx_queue->txd.buf.addr)) 121 return ((efx_oword_t *)tx_queue->txd.buf.addr) + index; 122 else 123 return NULL; 124} 125 126static void ef100_notify_tx_desc(struct efx_tx_queue *tx_queue) 127{ 128 unsigned int write_ptr; 129 efx_dword_t reg; 130 131 tx_queue->xmit_pending = false; 132 133 if (unlikely(tx_queue->notify_count == tx_queue->write_count)) 134 return; 135 136 write_ptr = tx_queue->write_count & tx_queue->ptr_mask; 137 /* The write pointer goes into the high word */ 138 EFX_POPULATE_DWORD_1(reg, ERF_GZ_TX_RING_PIDX, write_ptr); 139 efx_writed_page(tx_queue->efx, ®, 140 ER_GZ_TX_RING_DOORBELL, tx_queue->queue); 141 tx_queue->notify_count = tx_queue->write_count; 142} 143 144static void ef100_tx_push_buffers(struct efx_tx_queue *tx_queue) 145{ 146 ef100_notify_tx_desc(tx_queue); 147 ++tx_queue->pushes; 148} 149 150static void ef100_set_tx_csum_partial(const struct sk_buff *skb, 151 struct efx_tx_buffer *buffer, efx_oword_t *txd) 152{ 153 efx_oword_t csum; 154 int csum_start; 155 156 if (!skb || skb->ip_summed != CHECKSUM_PARTIAL) 157 return; 158 159 /* skb->csum_start has the offset from head, but we need the offset 160 * from data. 161 */ 162 csum_start = skb_checksum_start_offset(skb); 163 EFX_POPULATE_OWORD_3(csum, 164 ESF_GZ_TX_SEND_CSO_PARTIAL_EN, 1, 165 ESF_GZ_TX_SEND_CSO_PARTIAL_START_W, 166 csum_start >> 1, 167 ESF_GZ_TX_SEND_CSO_PARTIAL_CSUM_W, 168 skb->csum_offset >> 1); 169 EFX_OR_OWORD(*txd, *txd, csum); 170} 171 172static void ef100_set_tx_hw_vlan(const struct sk_buff *skb, efx_oword_t *txd) 173{ 174 u16 vlan_tci = skb_vlan_tag_get(skb); 175 efx_oword_t vlan; 176 177 EFX_POPULATE_OWORD_2(vlan, 178 ESF_GZ_TX_SEND_VLAN_INSERT_EN, 1, 179 ESF_GZ_TX_SEND_VLAN_INSERT_TCI, vlan_tci); 180 EFX_OR_OWORD(*txd, *txd, vlan); 181} 182 183static void ef100_make_send_desc(struct efx_nic *efx, 184 const struct sk_buff *skb, 185 struct efx_tx_buffer *buffer, efx_oword_t *txd, 186 unsigned int segment_count) 187{ 188 /* TX send descriptor */ 189 EFX_POPULATE_OWORD_3(*txd, 190 ESF_GZ_TX_SEND_NUM_SEGS, segment_count, 191 ESF_GZ_TX_SEND_LEN, buffer->len, 192 ESF_GZ_TX_SEND_ADDR, buffer->dma_addr); 193 194 if (likely(efx->net_dev->features & NETIF_F_HW_CSUM)) 195 ef100_set_tx_csum_partial(skb, buffer, txd); 196 if (efx->net_dev->features & NETIF_F_HW_VLAN_CTAG_TX && 197 skb && skb_vlan_tag_present(skb)) 198 ef100_set_tx_hw_vlan(skb, txd); 199} 200 201static void ef100_make_tso_desc(struct efx_nic *efx, 202 const struct sk_buff *skb, 203 struct efx_tx_buffer *buffer, efx_oword_t *txd, 204 unsigned int segment_count) 205{ 206 u32 mangleid = (efx->net_dev->features & NETIF_F_TSO_MANGLEID) || 207 skb_shinfo(skb)->gso_type & SKB_GSO_TCP_FIXEDID ? 208 ESE_GZ_TX_DESC_IP4_ID_NO_OP : 209 ESE_GZ_TX_DESC_IP4_ID_INC_MOD16; 210 u16 vlan_enable = efx->net_dev->features & NETIF_F_HW_VLAN_CTAG_TX ? 211 skb_vlan_tag_present(skb) : 0; 212 unsigned int len, ip_offset, tcp_offset, payload_segs; 213 u16 vlan_tci = skb_vlan_tag_get(skb); 214 u32 mss = skb_shinfo(skb)->gso_size; 215 216 len = skb->len - buffer->len; 217 /* We use 1 for the TSO descriptor and 1 for the header */ 218 payload_segs = segment_count - 2; 219 ip_offset = skb_network_offset(skb); 220 tcp_offset = skb_transport_offset(skb); 221 222 EFX_POPULATE_OWORD_13(*txd, 223 ESF_GZ_TX_DESC_TYPE, ESE_GZ_TX_DESC_TYPE_TSO, 224 ESF_GZ_TX_TSO_MSS, mss, 225 ESF_GZ_TX_TSO_HDR_NUM_SEGS, 1, 226 ESF_GZ_TX_TSO_PAYLOAD_NUM_SEGS, payload_segs, 227 ESF_GZ_TX_TSO_HDR_LEN_W, buffer->len >> 1, 228 ESF_GZ_TX_TSO_PAYLOAD_LEN, len, 229 ESF_GZ_TX_TSO_CSO_INNER_L4, 1, 230 ESF_GZ_TX_TSO_INNER_L3_OFF_W, ip_offset >> 1, 231 ESF_GZ_TX_TSO_INNER_L4_OFF_W, tcp_offset >> 1, 232 ESF_GZ_TX_TSO_ED_INNER_IP4_ID, mangleid, 233 ESF_GZ_TX_TSO_ED_INNER_IP_LEN, 1, 234 ESF_GZ_TX_TSO_VLAN_INSERT_EN, vlan_enable, 235 ESF_GZ_TX_TSO_VLAN_INSERT_TCI, vlan_tci 236 ); 237} 238 239static void ef100_tx_make_descriptors(struct efx_tx_queue *tx_queue, 240 const struct sk_buff *skb, 241 unsigned int segment_count) 242{ 243 unsigned int old_write_count = tx_queue->write_count; 244 unsigned int new_write_count = old_write_count; 245 struct efx_tx_buffer *buffer; 246 unsigned int next_desc_type; 247 unsigned int write_ptr; 248 efx_oword_t *txd; 249 unsigned int nr_descs = tx_queue->insert_count - old_write_count; 250 251 if (unlikely(nr_descs == 0)) 252 return; 253 254 if (segment_count) 255 next_desc_type = ESE_GZ_TX_DESC_TYPE_TSO; 256 else 257 next_desc_type = ESE_GZ_TX_DESC_TYPE_SEND; 258 259 /* if it's a raw write (such as XDP) then always SEND single frames */ 260 if (!skb) 261 nr_descs = 1; 262 263 do { 264 write_ptr = new_write_count & tx_queue->ptr_mask; 265 buffer = &tx_queue->buffer[write_ptr]; 266 txd = ef100_tx_desc(tx_queue, write_ptr); 267 ++new_write_count; 268 269 /* Create TX descriptor ring entry */ 270 tx_queue->packet_write_count = new_write_count; 271 272 switch (next_desc_type) { 273 case ESE_GZ_TX_DESC_TYPE_SEND: 274 ef100_make_send_desc(tx_queue->efx, skb, 275 buffer, txd, nr_descs); 276 break; 277 case ESE_GZ_TX_DESC_TYPE_TSO: 278 /* TX TSO descriptor */ 279 WARN_ON_ONCE(!(buffer->flags & EFX_TX_BUF_TSO_V3)); 280 ef100_make_tso_desc(tx_queue->efx, skb, 281 buffer, txd, nr_descs); 282 break; 283 default: 284 /* TX segment descriptor */ 285 EFX_POPULATE_OWORD_3(*txd, 286 ESF_GZ_TX_DESC_TYPE, ESE_GZ_TX_DESC_TYPE_SEG, 287 ESF_GZ_TX_SEG_LEN, buffer->len, 288 ESF_GZ_TX_SEG_ADDR, buffer->dma_addr); 289 } 290 /* if it's a raw write (such as XDP) then always SEND */ 291 next_desc_type = skb ? ESE_GZ_TX_DESC_TYPE_SEG : 292 ESE_GZ_TX_DESC_TYPE_SEND; 293 294 } while (new_write_count != tx_queue->insert_count); 295 296 wmb(); /* Ensure descriptors are written before they are fetched */ 297 298 tx_queue->write_count = new_write_count; 299 300 /* The write_count above must be updated before reading 301 * channel->holdoff_doorbell to avoid a race with the 302 * completion path, so ensure these operations are not 303 * re-ordered. This also flushes the update of write_count 304 * back into the cache. 305 */ 306 smp_mb(); 307} 308 309void ef100_tx_write(struct efx_tx_queue *tx_queue) 310{ 311 ef100_tx_make_descriptors(tx_queue, NULL, 0); 312 ef100_tx_push_buffers(tx_queue); 313} 314 315void ef100_ev_tx(struct efx_channel *channel, const efx_qword_t *p_event) 316{ 317 unsigned int tx_done = 318 EFX_QWORD_FIELD(*p_event, ESF_GZ_EV_TXCMPL_NUM_DESC); 319 unsigned int qlabel = 320 EFX_QWORD_FIELD(*p_event, ESF_GZ_EV_TXCMPL_Q_LABEL); 321 struct efx_tx_queue *tx_queue = 322 efx_channel_get_tx_queue(channel, qlabel); 323 unsigned int tx_index = (tx_queue->read_count + tx_done - 1) & 324 tx_queue->ptr_mask; 325 326 efx_xmit_done(tx_queue, tx_index); 327} 328 329/* Add a socket buffer to a TX queue 330 * 331 * You must hold netif_tx_lock() to call this function. 332 * 333 * Returns 0 on success, error code otherwise. In case of an error this 334 * function will free the SKB. 335 */ 336netdev_tx_t ef100_enqueue_skb(struct efx_tx_queue *tx_queue, 337 struct sk_buff *skb) 338{ 339 unsigned int old_insert_count = tx_queue->insert_count; 340 struct efx_nic *efx = tx_queue->efx; 341 bool xmit_more = netdev_xmit_more(); 342 unsigned int fill_level; 343 unsigned int segments; 344 int rc; 345 346 if (!tx_queue->buffer || !tx_queue->ptr_mask) { 347 netif_stop_queue(efx->net_dev); 348 dev_kfree_skb_any(skb); 349 return -ENODEV; 350 } 351 352 segments = skb_is_gso(skb) ? skb_shinfo(skb)->gso_segs : 0; 353 if (segments == 1) 354 segments = 0; /* Don't use TSO/GSO for a single segment. */ 355 if (segments && !ef100_tx_can_tso(tx_queue, skb)) { 356 rc = efx_tx_tso_fallback(tx_queue, skb); 357 tx_queue->tso_fallbacks++; 358 if (rc) 359 goto err; 360 else 361 return 0; 362 } 363 364 /* Map for DMA and create descriptors */ 365 rc = efx_tx_map_data(tx_queue, skb, segments); 366 if (rc) 367 goto err; 368 ef100_tx_make_descriptors(tx_queue, skb, segments); 369 370 fill_level = efx_channel_tx_old_fill_level(tx_queue->channel); 371 if (fill_level > efx->txq_stop_thresh) { 372 struct efx_tx_queue *txq2; 373 374 netif_tx_stop_queue(tx_queue->core_txq); 375 /* Re-read after a memory barrier in case we've raced with 376 * the completion path. Otherwise there's a danger we'll never 377 * restart the queue if all completions have just happened. 378 */ 379 smp_mb(); 380 efx_for_each_channel_tx_queue(txq2, tx_queue->channel) 381 txq2->old_read_count = READ_ONCE(txq2->read_count); 382 fill_level = efx_channel_tx_old_fill_level(tx_queue->channel); 383 if (fill_level < efx->txq_stop_thresh) 384 netif_tx_start_queue(tx_queue->core_txq); 385 } 386 387 tx_queue->xmit_pending = true; 388 389 /* If xmit_more then we don't need to push the doorbell, unless there 390 * are 256 descriptors already queued in which case we have to push to 391 * ensure we never push more than 256 at once. 392 */ 393 if (__netdev_tx_sent_queue(tx_queue->core_txq, skb->len, xmit_more) || 394 tx_queue->write_count - tx_queue->notify_count > 255) 395 ef100_tx_push_buffers(tx_queue); 396 397 if (segments) { 398 tx_queue->tso_bursts++; 399 tx_queue->tso_packets += segments; 400 tx_queue->tx_packets += segments; 401 } else { 402 tx_queue->tx_packets++; 403 } 404 return 0; 405 406err: 407 efx_enqueue_unwind(tx_queue, old_insert_count); 408 if (!IS_ERR_OR_NULL(skb)) 409 dev_kfree_skb_any(skb); 410 411 /* If we're not expecting another transmit and we had something to push 412 * on this queue then we need to push here to get the previous packets 413 * out. We only enter this branch from before the xmit_more handling 414 * above, so xmit_pending still refers to the old state. 415 */ 416 if (tx_queue->xmit_pending && !xmit_more) 417 ef100_tx_push_buffers(tx_queue); 418 return rc; 419} 420