1/****************************************************************************** 2 * 3 * This file is provided under a dual BSD/GPLv2 license. When using or 4 * redistributing this file, you may do so under either license. 5 * 6 * GPL LICENSE SUMMARY 7 * 8 * Copyright(c) 2020 Intel Corporation 9 * 10 * This program is free software; you can redistribute it and/or modify 11 * it under the terms of version 2 of the GNU General Public License as 12 * published by the Free Software Foundation. 13 * 14 * This program is distributed in the hope that it will be useful, but 15 * WITHOUT ANY WARRANTY; without even the implied warranty of 16 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 17 * General Public License for more details. 18 * 19 * BSD LICENSE 20 * 21 * Copyright(c) 2020 Intel Corporation 22 * All rights reserved. 23 * 24 * Redistribution and use in source and binary forms, with or without 25 * modification, are permitted provided that the following conditions 26 * are met: 27 * 28 * * Redistributions of source code must retain the above copyright 29 * notice, this list of conditions and the following disclaimer. 30 * * Redistributions in binary form must reproduce the above copyright 31 * notice, this list of conditions and the following disclaimer in 32 * the documentation and/or other materials provided with the 33 * distribution. 34 * * Neither the name Intel Corporation nor the names of its 35 * contributors may be used to endorse or promote products derived 36 * from this software without specific prior written permission. 37 * 38 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 39 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 40 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 41 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 42 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 43 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 44 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 45 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 46 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 47 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 48 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 49 * 50 *****************************************************************************/ 51#include <net/tso.h> 52#include <linux/tcp.h> 53 54#include "iwl-debug.h" 55#include "iwl-io.h" 56#include "fw/api/tx.h" 57#include "queue/tx.h" 58#include "iwl-fh.h" 59#include "iwl-scd.h" 60#include <linux/dmapool.h> 61 62/* 63 * iwl_txq_gen2_tx_stop - Stop all Tx DMA channels 64 */ 65void iwl_txq_gen2_tx_stop(struct iwl_trans *trans) 66{ 67 int txq_id; 68 69 /* 70 * This function can be called before the op_mode disabled the 71 * queues. This happens when we have an rfkill interrupt. 72 * Since we stop Tx altogether - mark the queues as stopped. 73 */ 74 memset(trans->txqs.queue_stopped, 0, 75 sizeof(trans->txqs.queue_stopped)); 76 memset(trans->txqs.queue_used, 0, sizeof(trans->txqs.queue_used)); 77 78 /* Unmap DMA from host system and free skb's */ 79 for (txq_id = 0; txq_id < ARRAY_SIZE(trans->txqs.txq); txq_id++) { 80 if (!trans->txqs.txq[txq_id]) 81 continue; 82 iwl_txq_gen2_unmap(trans, txq_id); 83 } 84} 85 86/* 87 * iwl_txq_update_byte_tbl - Set up entry in Tx byte-count array 88 */ 89static void iwl_pcie_gen2_update_byte_tbl(struct iwl_trans *trans, 90 struct iwl_txq *txq, u16 byte_cnt, 91 int num_tbs) 92{ 93 int idx = iwl_txq_get_cmd_index(txq, txq->write_ptr); 94 u8 filled_tfd_size, num_fetch_chunks; 95 u16 len = byte_cnt; 96 __le16 bc_ent; 97 98 if (WARN(idx >= txq->n_window, "%d >= %d\n", idx, txq->n_window)) 99 return; 100 101 filled_tfd_size = offsetof(struct iwl_tfh_tfd, tbs) + 102 num_tbs * sizeof(struct iwl_tfh_tb); 103 /* 104 * filled_tfd_size contains the number of filled bytes in the TFD. 105 * Dividing it by 64 will give the number of chunks to fetch 106 * to SRAM- 0 for one chunk, 1 for 2 and so on. 107 * If, for example, TFD contains only 3 TBs then 32 bytes 108 * of the TFD are used, and only one chunk of 64 bytes should 109 * be fetched 110 */ 111 num_fetch_chunks = DIV_ROUND_UP(filled_tfd_size, 64) - 1; 112 113 if (trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_AX210) { 114 struct iwl_gen3_bc_tbl *scd_bc_tbl_gen3 = txq->bc_tbl.addr; 115 116 /* Starting from AX210, the HW expects bytes */ 117 WARN_ON(trans->txqs.bc_table_dword); 118 WARN_ON(len > 0x3FFF); 119 bc_ent = cpu_to_le16(len | (num_fetch_chunks << 14)); 120 scd_bc_tbl_gen3->tfd_offset[idx] = bc_ent; 121 } else { 122 struct iwlagn_scd_bc_tbl *scd_bc_tbl = txq->bc_tbl.addr; 123 124 /* Before AX210, the HW expects DW */ 125 WARN_ON(!trans->txqs.bc_table_dword); 126 len = DIV_ROUND_UP(len, 4); 127 WARN_ON(len > 0xFFF); 128 bc_ent = cpu_to_le16(len | (num_fetch_chunks << 12)); 129 scd_bc_tbl->tfd_offset[idx] = bc_ent; 130 } 131} 132 133/* 134 * iwl_txq_inc_wr_ptr - Send new write index to hardware 135 */ 136void iwl_txq_inc_wr_ptr(struct iwl_trans *trans, struct iwl_txq *txq) 137{ 138 lockdep_assert_held(&txq->lock); 139 140 IWL_DEBUG_TX(trans, "Q:%d WR: 0x%x\n", txq->id, txq->write_ptr); 141 142 /* 143 * if not in power-save mode, uCode will never sleep when we're 144 * trying to tx (during RFKILL, we're not trying to tx). 145 */ 146 iwl_write32(trans, HBUS_TARG_WRPTR, txq->write_ptr | (txq->id << 16)); 147} 148 149static u8 iwl_txq_gen2_get_num_tbs(struct iwl_trans *trans, 150 struct iwl_tfh_tfd *tfd) 151{ 152 return le16_to_cpu(tfd->num_tbs) & 0x1f; 153} 154 155void iwl_txq_gen2_tfd_unmap(struct iwl_trans *trans, struct iwl_cmd_meta *meta, 156 struct iwl_tfh_tfd *tfd) 157{ 158 int i, num_tbs; 159 160 /* Sanity check on number of chunks */ 161 num_tbs = iwl_txq_gen2_get_num_tbs(trans, tfd); 162 163 if (num_tbs > trans->txqs.tfd.max_tbs) { 164 IWL_ERR(trans, "Too many chunks: %i\n", num_tbs); 165 return; 166 } 167 168 /* first TB is never freed - it's the bidirectional DMA data */ 169 for (i = 1; i < num_tbs; i++) { 170 if (meta->tbs & BIT(i)) 171 dma_unmap_page(trans->dev, 172 le64_to_cpu(tfd->tbs[i].addr), 173 le16_to_cpu(tfd->tbs[i].tb_len), 174 DMA_TO_DEVICE); 175 else 176 dma_unmap_single(trans->dev, 177 le64_to_cpu(tfd->tbs[i].addr), 178 le16_to_cpu(tfd->tbs[i].tb_len), 179 DMA_TO_DEVICE); 180 } 181 182 tfd->num_tbs = 0; 183} 184 185void iwl_txq_gen2_free_tfd(struct iwl_trans *trans, struct iwl_txq *txq) 186{ 187 /* rd_ptr is bounded by TFD_QUEUE_SIZE_MAX and 188 * idx is bounded by n_window 189 */ 190 int idx = iwl_txq_get_cmd_index(txq, txq->read_ptr); 191 192 lockdep_assert_held(&txq->lock); 193 194 iwl_txq_gen2_tfd_unmap(trans, &txq->entries[idx].meta, 195 iwl_txq_get_tfd(trans, txq, idx)); 196 197 /* free SKB */ 198 if (txq->entries) { 199 struct sk_buff *skb; 200 201 skb = txq->entries[idx].skb; 202 203 /* Can be called from irqs-disabled context 204 * If skb is not NULL, it means that the whole queue is being 205 * freed and that the queue is not empty - free the skb 206 */ 207 if (skb) { 208 iwl_op_mode_free_skb(trans->op_mode, skb); 209 txq->entries[idx].skb = NULL; 210 } 211 } 212} 213 214int iwl_txq_gen2_set_tb(struct iwl_trans *trans, struct iwl_tfh_tfd *tfd, 215 dma_addr_t addr, u16 len) 216{ 217 int idx = iwl_txq_gen2_get_num_tbs(trans, tfd); 218 struct iwl_tfh_tb *tb; 219 220 /* 221 * Only WARN here so we know about the issue, but we mess up our 222 * unmap path because not every place currently checks for errors 223 * returned from this function - it can only return an error if 224 * there's no more space, and so when we know there is enough we 225 * don't always check ... 226 */ 227 WARN(iwl_txq_crosses_4g_boundary(addr, len), 228 "possible DMA problem with iova:0x%llx, len:%d\n", 229 (unsigned long long)addr, len); 230 231 if (WARN_ON(idx >= IWL_TFH_NUM_TBS)) 232 return -EINVAL; 233 tb = &tfd->tbs[idx]; 234 235 /* Each TFD can point to a maximum max_tbs Tx buffers */ 236 if (le16_to_cpu(tfd->num_tbs) >= trans->txqs.tfd.max_tbs) { 237 IWL_ERR(trans, "Error can not send more than %d chunks\n", 238 trans->txqs.tfd.max_tbs); 239 return -EINVAL; 240 } 241 242 put_unaligned_le64(addr, &tb->addr); 243 tb->tb_len = cpu_to_le16(len); 244 245 tfd->num_tbs = cpu_to_le16(idx + 1); 246 247 return idx; 248} 249 250static struct page *get_workaround_page(struct iwl_trans *trans, 251 struct sk_buff *skb) 252{ 253 struct page **page_ptr; 254 struct page *ret; 255 256 page_ptr = (void *)((u8 *)skb->cb + trans->txqs.page_offs); 257 258 ret = alloc_page(GFP_ATOMIC); 259 if (!ret) 260 return NULL; 261 262 /* set the chaining pointer to the previous page if there */ 263 *(void **)(page_address(ret) + PAGE_SIZE - sizeof(void *)) = *page_ptr; 264 *page_ptr = ret; 265 266 return ret; 267} 268 269/* 270 * Add a TB and if needed apply the FH HW bug workaround; 271 * meta != NULL indicates that it's a page mapping and we 272 * need to dma_unmap_page() and set the meta->tbs bit in 273 * this case. 274 */ 275static int iwl_txq_gen2_set_tb_with_wa(struct iwl_trans *trans, 276 struct sk_buff *skb, 277 struct iwl_tfh_tfd *tfd, 278 dma_addr_t phys, void *virt, 279 u16 len, struct iwl_cmd_meta *meta) 280{ 281 dma_addr_t oldphys = phys; 282 struct page *page; 283 int ret; 284 285 if (unlikely(dma_mapping_error(trans->dev, phys))) 286 return -ENOMEM; 287 288 if (likely(!iwl_txq_crosses_4g_boundary(phys, len))) { 289 ret = iwl_txq_gen2_set_tb(trans, tfd, phys, len); 290 291 if (ret < 0) 292 goto unmap; 293 294 if (meta) 295 meta->tbs |= BIT(ret); 296 297 ret = 0; 298 goto trace; 299 } 300 301 /* 302 * Work around a hardware bug. If (as expressed in the 303 * condition above) the TB ends on a 32-bit boundary, 304 * then the next TB may be accessed with the wrong 305 * address. 306 * To work around it, copy the data elsewhere and make 307 * a new mapping for it so the device will not fail. 308 */ 309 310 if (WARN_ON(len > PAGE_SIZE - sizeof(void *))) { 311 ret = -ENOBUFS; 312 goto unmap; 313 } 314 315 page = get_workaround_page(trans, skb); 316 if (!page) { 317 ret = -ENOMEM; 318 goto unmap; 319 } 320 321 memcpy(page_address(page), virt, len); 322 323 phys = dma_map_single(trans->dev, page_address(page), len, 324 DMA_TO_DEVICE); 325 if (unlikely(dma_mapping_error(trans->dev, phys))) 326 return -ENOMEM; 327 ret = iwl_txq_gen2_set_tb(trans, tfd, phys, len); 328 if (ret < 0) { 329 /* unmap the new allocation as single */ 330 oldphys = phys; 331 meta = NULL; 332 goto unmap; 333 } 334 IWL_WARN(trans, 335 "TB bug workaround: copied %d bytes from 0x%llx to 0x%llx\n", 336 len, (unsigned long long)oldphys, (unsigned long long)phys); 337 338 ret = 0; 339unmap: 340 if (meta) 341 dma_unmap_page(trans->dev, oldphys, len, DMA_TO_DEVICE); 342 else 343 dma_unmap_single(trans->dev, oldphys, len, DMA_TO_DEVICE); 344trace: 345 trace_iwlwifi_dev_tx_tb(trans->dev, skb, virt, phys, len); 346 347 return ret; 348} 349 350#ifdef CONFIG_INET 351struct iwl_tso_hdr_page *get_page_hdr(struct iwl_trans *trans, size_t len, 352 struct sk_buff *skb) 353{ 354 struct iwl_tso_hdr_page *p = this_cpu_ptr(trans->txqs.tso_hdr_page); 355 struct page **page_ptr; 356 357 page_ptr = (void *)((u8 *)skb->cb + trans->txqs.page_offs); 358 359 if (WARN_ON(*page_ptr)) 360 return NULL; 361 362 if (!p->page) 363 goto alloc; 364 365 /* 366 * Check if there's enough room on this page 367 * 368 * Note that we put a page chaining pointer *last* in the 369 * page - we need it somewhere, and if it's there then we 370 * avoid DMA mapping the last bits of the page which may 371 * trigger the 32-bit boundary hardware bug. 372 * 373 * (see also get_workaround_page() in tx-gen2.c) 374 */ 375 if (p->pos + len < (u8 *)page_address(p->page) + PAGE_SIZE - 376 sizeof(void *)) 377 goto out; 378 379 /* We don't have enough room on this page, get a new one. */ 380 __free_page(p->page); 381 382alloc: 383 p->page = alloc_page(GFP_ATOMIC); 384 if (!p->page) 385 return NULL; 386 p->pos = page_address(p->page); 387 /* set the chaining pointer to NULL */ 388 *(void **)(page_address(p->page) + PAGE_SIZE - sizeof(void *)) = NULL; 389out: 390 *page_ptr = p->page; 391 get_page(p->page); 392 return p; 393} 394#endif 395 396static int iwl_txq_gen2_build_amsdu(struct iwl_trans *trans, 397 struct sk_buff *skb, 398 struct iwl_tfh_tfd *tfd, int start_len, 399 u8 hdr_len, 400 struct iwl_device_tx_cmd *dev_cmd) 401{ 402#ifdef CONFIG_INET 403 struct iwl_tx_cmd_gen2 *tx_cmd = (void *)dev_cmd->payload; 404 struct ieee80211_hdr *hdr = (void *)skb->data; 405 unsigned int snap_ip_tcp_hdrlen, ip_hdrlen, total_len, hdr_room; 406 unsigned int mss = skb_shinfo(skb)->gso_size; 407 u16 length, amsdu_pad; 408 u8 *start_hdr; 409 struct iwl_tso_hdr_page *hdr_page; 410 struct tso_t tso; 411 412 trace_iwlwifi_dev_tx(trans->dev, skb, tfd, sizeof(*tfd), 413 &dev_cmd->hdr, start_len, 0); 414 415 ip_hdrlen = skb_transport_header(skb) - skb_network_header(skb); 416 snap_ip_tcp_hdrlen = 8 + ip_hdrlen + tcp_hdrlen(skb); 417 total_len = skb->len - snap_ip_tcp_hdrlen - hdr_len; 418 amsdu_pad = 0; 419 420 /* total amount of header we may need for this A-MSDU */ 421 hdr_room = DIV_ROUND_UP(total_len, mss) * 422 (3 + snap_ip_tcp_hdrlen + sizeof(struct ethhdr)); 423 424 /* Our device supports 9 segments at most, it will fit in 1 page */ 425 hdr_page = get_page_hdr(trans, hdr_room, skb); 426 if (!hdr_page) 427 return -ENOMEM; 428 429 start_hdr = hdr_page->pos; 430 431 /* 432 * Pull the ieee80211 header to be able to use TSO core, 433 * we will restore it for the tx_status flow. 434 */ 435 skb_pull(skb, hdr_len); 436 437 /* 438 * Remove the length of all the headers that we don't actually 439 * have in the MPDU by themselves, but that we duplicate into 440 * all the different MSDUs inside the A-MSDU. 441 */ 442 le16_add_cpu(&tx_cmd->len, -snap_ip_tcp_hdrlen); 443 444 tso_start(skb, &tso); 445 446 while (total_len) { 447 /* this is the data left for this subframe */ 448 unsigned int data_left = min_t(unsigned int, mss, total_len); 449 struct sk_buff *csum_skb = NULL; 450 unsigned int tb_len; 451 dma_addr_t tb_phys; 452 u8 *subf_hdrs_start = hdr_page->pos; 453 454 total_len -= data_left; 455 456 memset(hdr_page->pos, 0, amsdu_pad); 457 hdr_page->pos += amsdu_pad; 458 amsdu_pad = (4 - (sizeof(struct ethhdr) + snap_ip_tcp_hdrlen + 459 data_left)) & 0x3; 460 ether_addr_copy(hdr_page->pos, ieee80211_get_DA(hdr)); 461 hdr_page->pos += ETH_ALEN; 462 ether_addr_copy(hdr_page->pos, ieee80211_get_SA(hdr)); 463 hdr_page->pos += ETH_ALEN; 464 465 length = snap_ip_tcp_hdrlen + data_left; 466 *((__be16 *)hdr_page->pos) = cpu_to_be16(length); 467 hdr_page->pos += sizeof(length); 468 469 /* 470 * This will copy the SNAP as well which will be considered 471 * as MAC header. 472 */ 473 tso_build_hdr(skb, hdr_page->pos, &tso, data_left, !total_len); 474 475 hdr_page->pos += snap_ip_tcp_hdrlen; 476 477 tb_len = hdr_page->pos - start_hdr; 478 tb_phys = dma_map_single(trans->dev, start_hdr, 479 tb_len, DMA_TO_DEVICE); 480 if (unlikely(dma_mapping_error(trans->dev, tb_phys))) { 481 dev_kfree_skb(csum_skb); 482 goto out_err; 483 } 484 /* 485 * No need for _with_wa, this is from the TSO page and 486 * we leave some space at the end of it so can't hit 487 * the buggy scenario. 488 */ 489 iwl_txq_gen2_set_tb(trans, tfd, tb_phys, tb_len); 490 trace_iwlwifi_dev_tx_tb(trans->dev, skb, start_hdr, 491 tb_phys, tb_len); 492 /* add this subframe's headers' length to the tx_cmd */ 493 le16_add_cpu(&tx_cmd->len, hdr_page->pos - subf_hdrs_start); 494 495 /* prepare the start_hdr for the next subframe */ 496 start_hdr = hdr_page->pos; 497 498 /* put the payload */ 499 while (data_left) { 500 int ret; 501 502 tb_len = min_t(unsigned int, tso.size, data_left); 503 tb_phys = dma_map_single(trans->dev, tso.data, 504 tb_len, DMA_TO_DEVICE); 505 ret = iwl_txq_gen2_set_tb_with_wa(trans, skb, tfd, 506 tb_phys, tso.data, 507 tb_len, NULL); 508 if (ret) { 509 dev_kfree_skb(csum_skb); 510 goto out_err; 511 } 512 513 data_left -= tb_len; 514 tso_build_data(skb, &tso, tb_len); 515 } 516 } 517 518 /* re -add the WiFi header */ 519 skb_push(skb, hdr_len); 520 521 return 0; 522 523out_err: 524#endif 525 return -EINVAL; 526} 527 528static struct 529iwl_tfh_tfd *iwl_txq_gen2_build_tx_amsdu(struct iwl_trans *trans, 530 struct iwl_txq *txq, 531 struct iwl_device_tx_cmd *dev_cmd, 532 struct sk_buff *skb, 533 struct iwl_cmd_meta *out_meta, 534 int hdr_len, 535 int tx_cmd_len) 536{ 537 int idx = iwl_txq_get_cmd_index(txq, txq->write_ptr); 538 struct iwl_tfh_tfd *tfd = iwl_txq_get_tfd(trans, txq, idx); 539 dma_addr_t tb_phys; 540 int len; 541 void *tb1_addr; 542 543 tb_phys = iwl_txq_get_first_tb_dma(txq, idx); 544 545 /* 546 * No need for _with_wa, the first TB allocation is aligned up 547 * to a 64-byte boundary and thus can't be at the end or cross 548 * a page boundary (much less a 2^32 boundary). 549 */ 550 iwl_txq_gen2_set_tb(trans, tfd, tb_phys, IWL_FIRST_TB_SIZE); 551 552 /* 553 * The second TB (tb1) points to the remainder of the TX command 554 * and the 802.11 header - dword aligned size 555 * (This calculation modifies the TX command, so do it before the 556 * setup of the first TB) 557 */ 558 len = tx_cmd_len + sizeof(struct iwl_cmd_header) + hdr_len - 559 IWL_FIRST_TB_SIZE; 560 561 /* do not align A-MSDU to dword as the subframe header aligns it */ 562 563 /* map the data for TB1 */ 564 tb1_addr = ((u8 *)&dev_cmd->hdr) + IWL_FIRST_TB_SIZE; 565 tb_phys = dma_map_single(trans->dev, tb1_addr, len, DMA_TO_DEVICE); 566 if (unlikely(dma_mapping_error(trans->dev, tb_phys))) 567 goto out_err; 568 /* 569 * No need for _with_wa(), we ensure (via alignment) that the data 570 * here can never cross or end at a page boundary. 571 */ 572 iwl_txq_gen2_set_tb(trans, tfd, tb_phys, len); 573 574 if (iwl_txq_gen2_build_amsdu(trans, skb, tfd, len + IWL_FIRST_TB_SIZE, 575 hdr_len, dev_cmd)) 576 goto out_err; 577 578 /* building the A-MSDU might have changed this data, memcpy it now */ 579 memcpy(&txq->first_tb_bufs[idx], dev_cmd, IWL_FIRST_TB_SIZE); 580 return tfd; 581 582out_err: 583 iwl_txq_gen2_tfd_unmap(trans, out_meta, tfd); 584 return NULL; 585} 586 587static int iwl_txq_gen2_tx_add_frags(struct iwl_trans *trans, 588 struct sk_buff *skb, 589 struct iwl_tfh_tfd *tfd, 590 struct iwl_cmd_meta *out_meta) 591{ 592 int i; 593 594 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 595 const skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 596 dma_addr_t tb_phys; 597 unsigned int fragsz = skb_frag_size(frag); 598 int ret; 599 600 if (!fragsz) 601 continue; 602 603 tb_phys = skb_frag_dma_map(trans->dev, frag, 0, 604 fragsz, DMA_TO_DEVICE); 605 ret = iwl_txq_gen2_set_tb_with_wa(trans, skb, tfd, tb_phys, 606 skb_frag_address(frag), 607 fragsz, out_meta); 608 if (ret) 609 return ret; 610 } 611 612 return 0; 613} 614 615static struct 616iwl_tfh_tfd *iwl_txq_gen2_build_tx(struct iwl_trans *trans, 617 struct iwl_txq *txq, 618 struct iwl_device_tx_cmd *dev_cmd, 619 struct sk_buff *skb, 620 struct iwl_cmd_meta *out_meta, 621 int hdr_len, 622 int tx_cmd_len, 623 bool pad) 624{ 625 int idx = iwl_txq_get_cmd_index(txq, txq->write_ptr); 626 struct iwl_tfh_tfd *tfd = iwl_txq_get_tfd(trans, txq, idx); 627 dma_addr_t tb_phys; 628 int len, tb1_len, tb2_len; 629 void *tb1_addr; 630 struct sk_buff *frag; 631 632 tb_phys = iwl_txq_get_first_tb_dma(txq, idx); 633 634 /* The first TB points to bi-directional DMA data */ 635 memcpy(&txq->first_tb_bufs[idx], dev_cmd, IWL_FIRST_TB_SIZE); 636 637 /* 638 * No need for _with_wa, the first TB allocation is aligned up 639 * to a 64-byte boundary and thus can't be at the end or cross 640 * a page boundary (much less a 2^32 boundary). 641 */ 642 iwl_txq_gen2_set_tb(trans, tfd, tb_phys, IWL_FIRST_TB_SIZE); 643 644 /* 645 * The second TB (tb1) points to the remainder of the TX command 646 * and the 802.11 header - dword aligned size 647 * (This calculation modifies the TX command, so do it before the 648 * setup of the first TB) 649 */ 650 len = tx_cmd_len + sizeof(struct iwl_cmd_header) + hdr_len - 651 IWL_FIRST_TB_SIZE; 652 653 if (pad) 654 tb1_len = ALIGN(len, 4); 655 else 656 tb1_len = len; 657 658 /* map the data for TB1 */ 659 tb1_addr = ((u8 *)&dev_cmd->hdr) + IWL_FIRST_TB_SIZE; 660 tb_phys = dma_map_single(trans->dev, tb1_addr, tb1_len, DMA_TO_DEVICE); 661 if (unlikely(dma_mapping_error(trans->dev, tb_phys))) 662 goto out_err; 663 /* 664 * No need for _with_wa(), we ensure (via alignment) that the data 665 * here can never cross or end at a page boundary. 666 */ 667 iwl_txq_gen2_set_tb(trans, tfd, tb_phys, tb1_len); 668 trace_iwlwifi_dev_tx(trans->dev, skb, tfd, sizeof(*tfd), &dev_cmd->hdr, 669 IWL_FIRST_TB_SIZE + tb1_len, hdr_len); 670 671 /* set up TFD's third entry to point to remainder of skb's head */ 672 tb2_len = skb_headlen(skb) - hdr_len; 673 674 if (tb2_len > 0) { 675 int ret; 676 677 tb_phys = dma_map_single(trans->dev, skb->data + hdr_len, 678 tb2_len, DMA_TO_DEVICE); 679 ret = iwl_txq_gen2_set_tb_with_wa(trans, skb, tfd, tb_phys, 680 skb->data + hdr_len, tb2_len, 681 NULL); 682 if (ret) 683 goto out_err; 684 } 685 686 if (iwl_txq_gen2_tx_add_frags(trans, skb, tfd, out_meta)) 687 goto out_err; 688 689 skb_walk_frags(skb, frag) { 690 int ret; 691 692 tb_phys = dma_map_single(trans->dev, frag->data, 693 skb_headlen(frag), DMA_TO_DEVICE); 694 ret = iwl_txq_gen2_set_tb_with_wa(trans, skb, tfd, tb_phys, 695 frag->data, 696 skb_headlen(frag), NULL); 697 if (ret) 698 goto out_err; 699 if (iwl_txq_gen2_tx_add_frags(trans, frag, tfd, out_meta)) 700 goto out_err; 701 } 702 703 return tfd; 704 705out_err: 706 iwl_txq_gen2_tfd_unmap(trans, out_meta, tfd); 707 return NULL; 708} 709 710static 711struct iwl_tfh_tfd *iwl_txq_gen2_build_tfd(struct iwl_trans *trans, 712 struct iwl_txq *txq, 713 struct iwl_device_tx_cmd *dev_cmd, 714 struct sk_buff *skb, 715 struct iwl_cmd_meta *out_meta) 716{ 717 struct ieee80211_hdr *hdr = (struct ieee80211_hdr *)skb->data; 718 int idx = iwl_txq_get_cmd_index(txq, txq->write_ptr); 719 struct iwl_tfh_tfd *tfd = iwl_txq_get_tfd(trans, txq, idx); 720 int len, hdr_len; 721 bool amsdu; 722 723 /* There must be data left over for TB1 or this code must be changed */ 724 BUILD_BUG_ON(sizeof(struct iwl_tx_cmd_gen2) < IWL_FIRST_TB_SIZE); 725 726 memset(tfd, 0, sizeof(*tfd)); 727 728 if (trans->trans_cfg->device_family < IWL_DEVICE_FAMILY_AX210) 729 len = sizeof(struct iwl_tx_cmd_gen2); 730 else 731 len = sizeof(struct iwl_tx_cmd_gen3); 732 733 amsdu = ieee80211_is_data_qos(hdr->frame_control) && 734 (*ieee80211_get_qos_ctl(hdr) & 735 IEEE80211_QOS_CTL_A_MSDU_PRESENT); 736 737 hdr_len = ieee80211_hdrlen(hdr->frame_control); 738 739 /* 740 * Only build A-MSDUs here if doing so by GSO, otherwise it may be 741 * an A-MSDU for other reasons, e.g. NAN or an A-MSDU having been 742 * built in the higher layers already. 743 */ 744 if (amsdu && skb_shinfo(skb)->gso_size) 745 return iwl_txq_gen2_build_tx_amsdu(trans, txq, dev_cmd, skb, 746 out_meta, hdr_len, len); 747 return iwl_txq_gen2_build_tx(trans, txq, dev_cmd, skb, out_meta, 748 hdr_len, len, !amsdu); 749} 750 751int iwl_txq_space(struct iwl_trans *trans, const struct iwl_txq *q) 752{ 753 unsigned int max; 754 unsigned int used; 755 756 /* 757 * To avoid ambiguity between empty and completely full queues, there 758 * should always be less than max_tfd_queue_size elements in the queue. 759 * If q->n_window is smaller than max_tfd_queue_size, there is no need 760 * to reserve any queue entries for this purpose. 761 */ 762 if (q->n_window < trans->trans_cfg->base_params->max_tfd_queue_size) 763 max = q->n_window; 764 else 765 max = trans->trans_cfg->base_params->max_tfd_queue_size - 1; 766 767 /* 768 * max_tfd_queue_size is a power of 2, so the following is equivalent to 769 * modulo by max_tfd_queue_size and is well defined. 770 */ 771 used = (q->write_ptr - q->read_ptr) & 772 (trans->trans_cfg->base_params->max_tfd_queue_size - 1); 773 774 if (WARN_ON(used > max)) 775 return 0; 776 777 return max - used; 778} 779 780int iwl_txq_gen2_tx(struct iwl_trans *trans, struct sk_buff *skb, 781 struct iwl_device_tx_cmd *dev_cmd, int txq_id) 782{ 783 struct iwl_cmd_meta *out_meta; 784 struct iwl_txq *txq = trans->txqs.txq[txq_id]; 785 u16 cmd_len; 786 int idx; 787 void *tfd; 788 789 if (WARN_ONCE(txq_id >= IWL_MAX_TVQM_QUEUES, 790 "queue %d out of range", txq_id)) 791 return -EINVAL; 792 793 if (WARN_ONCE(!test_bit(txq_id, trans->txqs.queue_used), 794 "TX on unused queue %d\n", txq_id)) 795 return -EINVAL; 796 797 if (skb_is_nonlinear(skb) && 798 skb_shinfo(skb)->nr_frags > IWL_TRANS_MAX_FRAGS(trans) && 799 __skb_linearize(skb)) 800 return -ENOMEM; 801 802 spin_lock(&txq->lock); 803 804 if (iwl_txq_space(trans, txq) < txq->high_mark) { 805 iwl_txq_stop(trans, txq); 806 807 /* don't put the packet on the ring, if there is no room */ 808 if (unlikely(iwl_txq_space(trans, txq) < 3)) { 809 struct iwl_device_tx_cmd **dev_cmd_ptr; 810 811 dev_cmd_ptr = (void *)((u8 *)skb->cb + 812 trans->txqs.dev_cmd_offs); 813 814 *dev_cmd_ptr = dev_cmd; 815 __skb_queue_tail(&txq->overflow_q, skb); 816 spin_unlock(&txq->lock); 817 return 0; 818 } 819 } 820 821 idx = iwl_txq_get_cmd_index(txq, txq->write_ptr); 822 823 /* Set up driver data for this TFD */ 824 txq->entries[idx].skb = skb; 825 txq->entries[idx].cmd = dev_cmd; 826 827 dev_cmd->hdr.sequence = 828 cpu_to_le16((u16)(QUEUE_TO_SEQ(txq_id) | 829 INDEX_TO_SEQ(idx))); 830 831 /* Set up first empty entry in queue's array of Tx/cmd buffers */ 832 out_meta = &txq->entries[idx].meta; 833 out_meta->flags = 0; 834 835 tfd = iwl_txq_gen2_build_tfd(trans, txq, dev_cmd, skb, out_meta); 836 if (!tfd) { 837 spin_unlock(&txq->lock); 838 return -1; 839 } 840 841 if (trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_AX210) { 842 struct iwl_tx_cmd_gen3 *tx_cmd_gen3 = 843 (void *)dev_cmd->payload; 844 845 cmd_len = le16_to_cpu(tx_cmd_gen3->len); 846 } else { 847 struct iwl_tx_cmd_gen2 *tx_cmd_gen2 = 848 (void *)dev_cmd->payload; 849 850 cmd_len = le16_to_cpu(tx_cmd_gen2->len); 851 } 852 853 /* Set up entry for this TFD in Tx byte-count array */ 854 iwl_pcie_gen2_update_byte_tbl(trans, txq, cmd_len, 855 iwl_txq_gen2_get_num_tbs(trans, tfd)); 856 857 /* start timer if queue currently empty */ 858 if (txq->read_ptr == txq->write_ptr && txq->wd_timeout) 859 mod_timer(&txq->stuck_timer, jiffies + txq->wd_timeout); 860 861 /* Tell device the write index *just past* this latest filled TFD */ 862 txq->write_ptr = iwl_txq_inc_wrap(trans, txq->write_ptr); 863 iwl_txq_inc_wr_ptr(trans, txq); 864 /* 865 * At this point the frame is "transmitted" successfully 866 * and we will get a TX status notification eventually. 867 */ 868 spin_unlock(&txq->lock); 869 return 0; 870} 871 872/*************** HOST COMMAND QUEUE FUNCTIONS *****/ 873 874/* 875 * iwl_txq_gen2_unmap - Unmap any remaining DMA mappings and free skb's 876 */ 877void iwl_txq_gen2_unmap(struct iwl_trans *trans, int txq_id) 878{ 879 struct iwl_txq *txq = trans->txqs.txq[txq_id]; 880 881 spin_lock_bh(&txq->lock); 882 while (txq->write_ptr != txq->read_ptr) { 883 IWL_DEBUG_TX_REPLY(trans, "Q %d Free %d\n", 884 txq_id, txq->read_ptr); 885 886 if (txq_id != trans->txqs.cmd.q_id) { 887 int idx = iwl_txq_get_cmd_index(txq, txq->read_ptr); 888 struct sk_buff *skb = txq->entries[idx].skb; 889 890 if (!WARN_ON_ONCE(!skb)) 891 iwl_txq_free_tso_page(trans, skb); 892 } 893 iwl_txq_gen2_free_tfd(trans, txq); 894 txq->read_ptr = iwl_txq_inc_wrap(trans, txq->read_ptr); 895 } 896 897 while (!skb_queue_empty(&txq->overflow_q)) { 898 struct sk_buff *skb = __skb_dequeue(&txq->overflow_q); 899 900 iwl_op_mode_free_skb(trans->op_mode, skb); 901 } 902 903 spin_unlock_bh(&txq->lock); 904 905 /* just in case - this queue may have been stopped */ 906 iwl_wake_queue(trans, txq); 907} 908 909static void iwl_txq_gen2_free_memory(struct iwl_trans *trans, 910 struct iwl_txq *txq) 911{ 912 struct device *dev = trans->dev; 913 914 /* De-alloc circular buffer of TFDs */ 915 if (txq->tfds) { 916 dma_free_coherent(dev, 917 trans->txqs.tfd.size * txq->n_window, 918 txq->tfds, txq->dma_addr); 919 dma_free_coherent(dev, 920 sizeof(*txq->first_tb_bufs) * txq->n_window, 921 txq->first_tb_bufs, txq->first_tb_dma); 922 } 923 924 kfree(txq->entries); 925 if (txq->bc_tbl.addr) 926 dma_pool_free(trans->txqs.bc_pool, 927 txq->bc_tbl.addr, txq->bc_tbl.dma); 928 kfree(txq); 929} 930 931/* 932 * iwl_pcie_txq_free - Deallocate DMA queue. 933 * @txq: Transmit queue to deallocate. 934 * 935 * Empty queue by removing and destroying all BD's. 936 * Free all buffers. 937 * 0-fill, but do not free "txq" descriptor structure. 938 */ 939static void iwl_txq_gen2_free(struct iwl_trans *trans, int txq_id) 940{ 941 struct iwl_txq *txq; 942 int i; 943 944 if (WARN_ONCE(txq_id >= IWL_MAX_TVQM_QUEUES, 945 "queue %d out of range", txq_id)) 946 return; 947 948 txq = trans->txqs.txq[txq_id]; 949 950 if (WARN_ON(!txq)) 951 return; 952 953 iwl_txq_gen2_unmap(trans, txq_id); 954 955 /* De-alloc array of command/tx buffers */ 956 if (txq_id == trans->txqs.cmd.q_id) 957 for (i = 0; i < txq->n_window; i++) { 958 kfree_sensitive(txq->entries[i].cmd); 959 kfree_sensitive(txq->entries[i].free_buf); 960 } 961 del_timer_sync(&txq->stuck_timer); 962 963 iwl_txq_gen2_free_memory(trans, txq); 964 965 trans->txqs.txq[txq_id] = NULL; 966 967 clear_bit(txq_id, trans->txqs.queue_used); 968} 969 970/* 971 * iwl_queue_init - Initialize queue's high/low-water and read/write indexes 972 */ 973static int iwl_queue_init(struct iwl_txq *q, int slots_num) 974{ 975 q->n_window = slots_num; 976 977 /* slots_num must be power-of-two size, otherwise 978 * iwl_txq_get_cmd_index is broken. */ 979 if (WARN_ON(!is_power_of_2(slots_num))) 980 return -EINVAL; 981 982 q->low_mark = q->n_window / 4; 983 if (q->low_mark < 4) 984 q->low_mark = 4; 985 986 q->high_mark = q->n_window / 8; 987 if (q->high_mark < 2) 988 q->high_mark = 2; 989 990 q->write_ptr = 0; 991 q->read_ptr = 0; 992 993 return 0; 994} 995 996int iwl_txq_init(struct iwl_trans *trans, struct iwl_txq *txq, int slots_num, 997 bool cmd_queue) 998{ 999 int ret; 1000 u32 tfd_queue_max_size = 1001 trans->trans_cfg->base_params->max_tfd_queue_size; 1002 1003 txq->need_update = false; 1004 1005 /* max_tfd_queue_size must be power-of-two size, otherwise 1006 * iwl_txq_inc_wrap and iwl_txq_dec_wrap are broken. */ 1007 if (WARN_ONCE(tfd_queue_max_size & (tfd_queue_max_size - 1), 1008 "Max tfd queue size must be a power of two, but is %d", 1009 tfd_queue_max_size)) 1010 return -EINVAL; 1011 1012 /* Initialize queue's high/low-water marks, and head/tail indexes */ 1013 ret = iwl_queue_init(txq, slots_num); 1014 if (ret) 1015 return ret; 1016 1017 spin_lock_init(&txq->lock); 1018 1019 if (cmd_queue) { 1020 static struct lock_class_key iwl_txq_cmd_queue_lock_class; 1021 1022 lockdep_set_class(&txq->lock, &iwl_txq_cmd_queue_lock_class); 1023 } 1024 1025 __skb_queue_head_init(&txq->overflow_q); 1026 1027 return 0; 1028} 1029 1030void iwl_txq_free_tso_page(struct iwl_trans *trans, struct sk_buff *skb) 1031{ 1032 struct page **page_ptr; 1033 struct page *next; 1034 1035 page_ptr = (void *)((u8 *)skb->cb + trans->txqs.page_offs); 1036 next = *page_ptr; 1037 *page_ptr = NULL; 1038 1039 while (next) { 1040 struct page *tmp = next; 1041 1042 next = *(void **)(page_address(next) + PAGE_SIZE - 1043 sizeof(void *)); 1044 __free_page(tmp); 1045 } 1046} 1047 1048void iwl_txq_log_scd_error(struct iwl_trans *trans, struct iwl_txq *txq) 1049{ 1050 u32 txq_id = txq->id; 1051 u32 status; 1052 bool active; 1053 u8 fifo; 1054 1055 if (trans->trans_cfg->use_tfh) { 1056 IWL_ERR(trans, "Queue %d is stuck %d %d\n", txq_id, 1057 txq->read_ptr, txq->write_ptr); 1058 /* TODO: access new SCD registers and dump them */ 1059 return; 1060 } 1061 1062 status = iwl_read_prph(trans, SCD_QUEUE_STATUS_BITS(txq_id)); 1063 fifo = (status >> SCD_QUEUE_STTS_REG_POS_TXF) & 0x7; 1064 active = !!(status & BIT(SCD_QUEUE_STTS_REG_POS_ACTIVE)); 1065 1066 IWL_ERR(trans, 1067 "Queue %d is %sactive on fifo %d and stuck for %u ms. SW [%d, %d] HW [%d, %d] FH TRB=0x0%x\n", 1068 txq_id, active ? "" : "in", fifo, 1069 jiffies_to_msecs(txq->wd_timeout), 1070 txq->read_ptr, txq->write_ptr, 1071 iwl_read_prph(trans, SCD_QUEUE_RDPTR(txq_id)) & 1072 (trans->trans_cfg->base_params->max_tfd_queue_size - 1), 1073 iwl_read_prph(trans, SCD_QUEUE_WRPTR(txq_id)) & 1074 (trans->trans_cfg->base_params->max_tfd_queue_size - 1), 1075 iwl_read_direct32(trans, FH_TX_TRB_REG(fifo))); 1076} 1077 1078static void iwl_txq_stuck_timer(struct timer_list *t) 1079{ 1080 struct iwl_txq *txq = from_timer(txq, t, stuck_timer); 1081 struct iwl_trans *trans = txq->trans; 1082 1083 spin_lock(&txq->lock); 1084 /* check if triggered erroneously */ 1085 if (txq->read_ptr == txq->write_ptr) { 1086 spin_unlock(&txq->lock); 1087 return; 1088 } 1089 spin_unlock(&txq->lock); 1090 1091 iwl_txq_log_scd_error(trans, txq); 1092 1093 iwl_force_nmi(trans); 1094} 1095 1096int iwl_txq_alloc(struct iwl_trans *trans, struct iwl_txq *txq, int slots_num, 1097 bool cmd_queue) 1098{ 1099 size_t tfd_sz = trans->txqs.tfd.size * 1100 trans->trans_cfg->base_params->max_tfd_queue_size; 1101 size_t tb0_buf_sz; 1102 int i; 1103 1104 if (WARN_ON(txq->entries || txq->tfds)) 1105 return -EINVAL; 1106 1107 if (trans->trans_cfg->use_tfh) 1108 tfd_sz = trans->txqs.tfd.size * slots_num; 1109 1110 timer_setup(&txq->stuck_timer, iwl_txq_stuck_timer, 0); 1111 txq->trans = trans; 1112 1113 txq->n_window = slots_num; 1114 1115 txq->entries = kcalloc(slots_num, 1116 sizeof(struct iwl_pcie_txq_entry), 1117 GFP_KERNEL); 1118 1119 if (!txq->entries) 1120 goto error; 1121 1122 if (cmd_queue) 1123 for (i = 0; i < slots_num; i++) { 1124 txq->entries[i].cmd = 1125 kmalloc(sizeof(struct iwl_device_cmd), 1126 GFP_KERNEL); 1127 if (!txq->entries[i].cmd) 1128 goto error; 1129 } 1130 1131 /* Circular buffer of transmit frame descriptors (TFDs), 1132 * shared with device */ 1133 txq->tfds = dma_alloc_coherent(trans->dev, tfd_sz, 1134 &txq->dma_addr, GFP_KERNEL); 1135 if (!txq->tfds) 1136 goto error; 1137 1138 BUILD_BUG_ON(sizeof(*txq->first_tb_bufs) != IWL_FIRST_TB_SIZE_ALIGN); 1139 1140 tb0_buf_sz = sizeof(*txq->first_tb_bufs) * slots_num; 1141 1142 txq->first_tb_bufs = dma_alloc_coherent(trans->dev, tb0_buf_sz, 1143 &txq->first_tb_dma, 1144 GFP_KERNEL); 1145 if (!txq->first_tb_bufs) 1146 goto err_free_tfds; 1147 1148 return 0; 1149err_free_tfds: 1150 dma_free_coherent(trans->dev, tfd_sz, txq->tfds, txq->dma_addr); 1151 txq->tfds = NULL; 1152error: 1153 if (txq->entries && cmd_queue) 1154 for (i = 0; i < slots_num; i++) 1155 kfree(txq->entries[i].cmd); 1156 kfree(txq->entries); 1157 txq->entries = NULL; 1158 1159 return -ENOMEM; 1160} 1161 1162static int iwl_txq_dyn_alloc_dma(struct iwl_trans *trans, 1163 struct iwl_txq **intxq, int size, 1164 unsigned int timeout) 1165{ 1166 size_t bc_tbl_size, bc_tbl_entries; 1167 struct iwl_txq *txq; 1168 int ret; 1169 1170 WARN_ON(!trans->txqs.bc_tbl_size); 1171 1172 bc_tbl_size = trans->txqs.bc_tbl_size; 1173 bc_tbl_entries = bc_tbl_size / sizeof(u16); 1174 1175 if (WARN_ON(size > bc_tbl_entries)) 1176 return -EINVAL; 1177 1178 txq = kzalloc(sizeof(*txq), GFP_KERNEL); 1179 if (!txq) 1180 return -ENOMEM; 1181 1182 txq->bc_tbl.addr = dma_pool_alloc(trans->txqs.bc_pool, GFP_KERNEL, 1183 &txq->bc_tbl.dma); 1184 if (!txq->bc_tbl.addr) { 1185 IWL_ERR(trans, "Scheduler BC Table allocation failed\n"); 1186 kfree(txq); 1187 return -ENOMEM; 1188 } 1189 1190 ret = iwl_txq_alloc(trans, txq, size, false); 1191 if (ret) { 1192 IWL_ERR(trans, "Tx queue alloc failed\n"); 1193 goto error; 1194 } 1195 ret = iwl_txq_init(trans, txq, size, false); 1196 if (ret) { 1197 IWL_ERR(trans, "Tx queue init failed\n"); 1198 goto error; 1199 } 1200 1201 txq->wd_timeout = msecs_to_jiffies(timeout); 1202 1203 *intxq = txq; 1204 return 0; 1205 1206error: 1207 iwl_txq_gen2_free_memory(trans, txq); 1208 return ret; 1209} 1210 1211static int iwl_txq_alloc_response(struct iwl_trans *trans, struct iwl_txq *txq, 1212 struct iwl_host_cmd *hcmd) 1213{ 1214 struct iwl_tx_queue_cfg_rsp *rsp; 1215 int ret, qid; 1216 u32 wr_ptr; 1217 1218 if (WARN_ON(iwl_rx_packet_payload_len(hcmd->resp_pkt) != 1219 sizeof(*rsp))) { 1220 ret = -EINVAL; 1221 goto error_free_resp; 1222 } 1223 1224 rsp = (void *)hcmd->resp_pkt->data; 1225 qid = le16_to_cpu(rsp->queue_number); 1226 wr_ptr = le16_to_cpu(rsp->write_pointer); 1227 1228 if (qid >= ARRAY_SIZE(trans->txqs.txq)) { 1229 WARN_ONCE(1, "queue index %d unsupported", qid); 1230 ret = -EIO; 1231 goto error_free_resp; 1232 } 1233 1234 if (test_and_set_bit(qid, trans->txqs.queue_used)) { 1235 WARN_ONCE(1, "queue %d already used", qid); 1236 ret = -EIO; 1237 goto error_free_resp; 1238 } 1239 1240 txq->id = qid; 1241 trans->txqs.txq[qid] = txq; 1242 wr_ptr &= (trans->trans_cfg->base_params->max_tfd_queue_size - 1); 1243 1244 /* Place first TFD at index corresponding to start sequence number */ 1245 txq->read_ptr = wr_ptr; 1246 txq->write_ptr = wr_ptr; 1247 1248 IWL_DEBUG_TX_QUEUES(trans, "Activate queue %d\n", qid); 1249 1250 iwl_free_resp(hcmd); 1251 return qid; 1252 1253error_free_resp: 1254 iwl_free_resp(hcmd); 1255 iwl_txq_gen2_free_memory(trans, txq); 1256 return ret; 1257} 1258 1259int iwl_txq_dyn_alloc(struct iwl_trans *trans, __le16 flags, u8 sta_id, u8 tid, 1260 int cmd_id, int size, unsigned int timeout) 1261{ 1262 struct iwl_txq *txq = NULL; 1263 struct iwl_tx_queue_cfg_cmd cmd = { 1264 .flags = flags, 1265 .sta_id = sta_id, 1266 .tid = tid, 1267 }; 1268 struct iwl_host_cmd hcmd = { 1269 .id = cmd_id, 1270 .len = { sizeof(cmd) }, 1271 .data = { &cmd, }, 1272 .flags = CMD_WANT_SKB, 1273 }; 1274 int ret; 1275 1276 ret = iwl_txq_dyn_alloc_dma(trans, &txq, size, timeout); 1277 if (ret) 1278 return ret; 1279 1280 cmd.tfdq_addr = cpu_to_le64(txq->dma_addr); 1281 cmd.byte_cnt_addr = cpu_to_le64(txq->bc_tbl.dma); 1282 cmd.cb_size = cpu_to_le32(TFD_QUEUE_CB_SIZE(size)); 1283 1284 ret = iwl_trans_send_cmd(trans, &hcmd); 1285 if (ret) 1286 goto error; 1287 1288 return iwl_txq_alloc_response(trans, txq, &hcmd); 1289 1290error: 1291 iwl_txq_gen2_free_memory(trans, txq); 1292 return ret; 1293} 1294 1295void iwl_txq_dyn_free(struct iwl_trans *trans, int queue) 1296{ 1297 if (WARN(queue >= IWL_MAX_TVQM_QUEUES, 1298 "queue %d out of range", queue)) 1299 return; 1300 1301 /* 1302 * Upon HW Rfkill - we stop the device, and then stop the queues 1303 * in the op_mode. Just for the sake of the simplicity of the op_mode, 1304 * allow the op_mode to call txq_disable after it already called 1305 * stop_device. 1306 */ 1307 if (!test_and_clear_bit(queue, trans->txqs.queue_used)) { 1308 WARN_ONCE(test_bit(STATUS_DEVICE_ENABLED, &trans->status), 1309 "queue %d not used", queue); 1310 return; 1311 } 1312 1313 iwl_txq_gen2_unmap(trans, queue); 1314 1315 iwl_txq_gen2_free_memory(trans, trans->txqs.txq[queue]); 1316 1317 trans->txqs.txq[queue] = NULL; 1318 1319 IWL_DEBUG_TX_QUEUES(trans, "Deactivate queue %d\n", queue); 1320} 1321 1322void iwl_txq_gen2_tx_free(struct iwl_trans *trans) 1323{ 1324 int i; 1325 1326 memset(trans->txqs.queue_used, 0, sizeof(trans->txqs.queue_used)); 1327 1328 /* Free all TX queues */ 1329 for (i = 0; i < ARRAY_SIZE(trans->txqs.txq); i++) { 1330 if (!trans->txqs.txq[i]) 1331 continue; 1332 1333 iwl_txq_gen2_free(trans, i); 1334 } 1335} 1336 1337int iwl_txq_gen2_init(struct iwl_trans *trans, int txq_id, int queue_size) 1338{ 1339 struct iwl_txq *queue; 1340 int ret; 1341 1342 /* alloc and init the tx queue */ 1343 if (!trans->txqs.txq[txq_id]) { 1344 queue = kzalloc(sizeof(*queue), GFP_KERNEL); 1345 if (!queue) { 1346 IWL_ERR(trans, "Not enough memory for tx queue\n"); 1347 return -ENOMEM; 1348 } 1349 trans->txqs.txq[txq_id] = queue; 1350 ret = iwl_txq_alloc(trans, queue, queue_size, true); 1351 if (ret) { 1352 IWL_ERR(trans, "Tx %d queue init failed\n", txq_id); 1353 goto error; 1354 } 1355 } else { 1356 queue = trans->txqs.txq[txq_id]; 1357 } 1358 1359 ret = iwl_txq_init(trans, queue, queue_size, 1360 (txq_id == trans->txqs.cmd.q_id)); 1361 if (ret) { 1362 IWL_ERR(trans, "Tx %d queue alloc failed\n", txq_id); 1363 goto error; 1364 } 1365 trans->txqs.txq[txq_id]->id = txq_id; 1366 set_bit(txq_id, trans->txqs.queue_used); 1367 1368 return 0; 1369 1370error: 1371 iwl_txq_gen2_tx_free(trans); 1372 return ret; 1373} 1374 1375static inline dma_addr_t iwl_txq_gen1_tfd_tb_get_addr(struct iwl_trans *trans, 1376 void *_tfd, u8 idx) 1377{ 1378 struct iwl_tfd *tfd; 1379 struct iwl_tfd_tb *tb; 1380 dma_addr_t addr; 1381 dma_addr_t hi_len; 1382 1383 if (trans->trans_cfg->use_tfh) { 1384 struct iwl_tfh_tfd *tfd = _tfd; 1385 struct iwl_tfh_tb *tb = &tfd->tbs[idx]; 1386 1387 return (dma_addr_t)(le64_to_cpu(tb->addr)); 1388 } 1389 1390 tfd = _tfd; 1391 tb = &tfd->tbs[idx]; 1392 addr = get_unaligned_le32(&tb->lo); 1393 1394 if (sizeof(dma_addr_t) <= sizeof(u32)) 1395 return addr; 1396 1397 hi_len = le16_to_cpu(tb->hi_n_len) & 0xF; 1398 1399 /* 1400 * shift by 16 twice to avoid warnings on 32-bit 1401 * (where this code never runs anyway due to the 1402 * if statement above) 1403 */ 1404 return addr | ((hi_len << 16) << 16); 1405} 1406 1407void iwl_txq_gen1_tfd_unmap(struct iwl_trans *trans, 1408 struct iwl_cmd_meta *meta, 1409 struct iwl_txq *txq, int index) 1410{ 1411 int i, num_tbs; 1412 void *tfd = iwl_txq_get_tfd(trans, txq, index); 1413 1414 /* Sanity check on number of chunks */ 1415 num_tbs = iwl_txq_gen1_tfd_get_num_tbs(trans, tfd); 1416 1417 if (num_tbs > trans->txqs.tfd.max_tbs) { 1418 IWL_ERR(trans, "Too many chunks: %i\n", num_tbs); 1419 /* @todo issue fatal error, it is quite serious situation */ 1420 return; 1421 } 1422 1423 /* first TB is never freed - it's the bidirectional DMA data */ 1424 1425 for (i = 1; i < num_tbs; i++) { 1426 if (meta->tbs & BIT(i)) 1427 dma_unmap_page(trans->dev, 1428 iwl_txq_gen1_tfd_tb_get_addr(trans, 1429 tfd, i), 1430 iwl_txq_gen1_tfd_tb_get_len(trans, 1431 tfd, i), 1432 DMA_TO_DEVICE); 1433 else 1434 dma_unmap_single(trans->dev, 1435 iwl_txq_gen1_tfd_tb_get_addr(trans, 1436 tfd, i), 1437 iwl_txq_gen1_tfd_tb_get_len(trans, 1438 tfd, i), 1439 DMA_TO_DEVICE); 1440 } 1441 1442 meta->tbs = 0; 1443 1444 if (trans->trans_cfg->use_tfh) { 1445 struct iwl_tfh_tfd *tfd_fh = (void *)tfd; 1446 1447 tfd_fh->num_tbs = 0; 1448 } else { 1449 struct iwl_tfd *tfd_fh = (void *)tfd; 1450 1451 tfd_fh->num_tbs = 0; 1452 } 1453} 1454 1455#define IWL_TX_CRC_SIZE 4 1456#define IWL_TX_DELIMITER_SIZE 4 1457 1458/* 1459 * iwl_txq_gen1_update_byte_cnt_tbl - Set up entry in Tx byte-count array 1460 */ 1461void iwl_txq_gen1_update_byte_cnt_tbl(struct iwl_trans *trans, 1462 struct iwl_txq *txq, u16 byte_cnt, 1463 int num_tbs) 1464{ 1465 struct iwlagn_scd_bc_tbl *scd_bc_tbl; 1466 int write_ptr = txq->write_ptr; 1467 int txq_id = txq->id; 1468 u8 sec_ctl = 0; 1469 u16 len = byte_cnt + IWL_TX_CRC_SIZE + IWL_TX_DELIMITER_SIZE; 1470 __le16 bc_ent; 1471 struct iwl_device_tx_cmd *dev_cmd = txq->entries[txq->write_ptr].cmd; 1472 struct iwl_tx_cmd *tx_cmd = (void *)dev_cmd->payload; 1473 u8 sta_id = tx_cmd->sta_id; 1474 1475 scd_bc_tbl = trans->txqs.scd_bc_tbls.addr; 1476 1477 sec_ctl = tx_cmd->sec_ctl; 1478 1479 switch (sec_ctl & TX_CMD_SEC_MSK) { 1480 case TX_CMD_SEC_CCM: 1481 len += IEEE80211_CCMP_MIC_LEN; 1482 break; 1483 case TX_CMD_SEC_TKIP: 1484 len += IEEE80211_TKIP_ICV_LEN; 1485 break; 1486 case TX_CMD_SEC_WEP: 1487 len += IEEE80211_WEP_IV_LEN + IEEE80211_WEP_ICV_LEN; 1488 break; 1489 } 1490 if (trans->txqs.bc_table_dword) 1491 len = DIV_ROUND_UP(len, 4); 1492 1493 if (WARN_ON(len > 0xFFF || write_ptr >= TFD_QUEUE_SIZE_MAX)) 1494 return; 1495 1496 bc_ent = cpu_to_le16(len | (sta_id << 12)); 1497 1498 scd_bc_tbl[txq_id].tfd_offset[write_ptr] = bc_ent; 1499 1500 if (write_ptr < TFD_QUEUE_SIZE_BC_DUP) 1501 scd_bc_tbl[txq_id].tfd_offset[TFD_QUEUE_SIZE_MAX + write_ptr] = 1502 bc_ent; 1503} 1504 1505void iwl_txq_gen1_inval_byte_cnt_tbl(struct iwl_trans *trans, 1506 struct iwl_txq *txq) 1507{ 1508 struct iwlagn_scd_bc_tbl *scd_bc_tbl = trans->txqs.scd_bc_tbls.addr; 1509 int txq_id = txq->id; 1510 int read_ptr = txq->read_ptr; 1511 u8 sta_id = 0; 1512 __le16 bc_ent; 1513 struct iwl_device_tx_cmd *dev_cmd = txq->entries[read_ptr].cmd; 1514 struct iwl_tx_cmd *tx_cmd = (void *)dev_cmd->payload; 1515 1516 WARN_ON(read_ptr >= TFD_QUEUE_SIZE_MAX); 1517 1518 if (txq_id != trans->txqs.cmd.q_id) 1519 sta_id = tx_cmd->sta_id; 1520 1521 bc_ent = cpu_to_le16(1 | (sta_id << 12)); 1522 1523 scd_bc_tbl[txq_id].tfd_offset[read_ptr] = bc_ent; 1524 1525 if (read_ptr < TFD_QUEUE_SIZE_BC_DUP) 1526 scd_bc_tbl[txq_id].tfd_offset[TFD_QUEUE_SIZE_MAX + read_ptr] = 1527 bc_ent; 1528} 1529