1 // SPDX-License-Identifier: ISC
2 /*
3 * Copyright (C) 2016 Felix Fietkau <nbd@nbd.name>
4 */
5
6 #include <linux/dma-mapping.h>
7 #include "mt76.h"
8 #include "dma.h"
9
10 static struct mt76_txwi_cache *
mt76_alloc_txwi(struct mt76_dev *dev)11 mt76_alloc_txwi(struct mt76_dev *dev)
12 {
13 struct mt76_txwi_cache *t;
14 dma_addr_t addr;
15 u8 *txwi;
16 int size;
17
18 size = L1_CACHE_ALIGN(dev->drv->txwi_size + sizeof(*t));
19 txwi = devm_kzalloc(dev->dev, size, GFP_ATOMIC);
20 if (!txwi)
21 return NULL;
22
23 addr = dma_map_single(dev->dev, txwi, dev->drv->txwi_size,
24 DMA_TO_DEVICE);
25 t = (struct mt76_txwi_cache *)(txwi + dev->drv->txwi_size);
26 t->dma_addr = addr;
27
28 return t;
29 }
30
31 static struct mt76_txwi_cache *
__mt76_get_txwi(struct mt76_dev *dev)32 __mt76_get_txwi(struct mt76_dev *dev)
33 {
34 struct mt76_txwi_cache *t = NULL;
35
36 spin_lock(&dev->lock);
37 if (!list_empty(&dev->txwi_cache)) {
38 t = list_first_entry(&dev->txwi_cache, struct mt76_txwi_cache,
39 list);
40 list_del(&t->list);
41 }
42 spin_unlock(&dev->lock);
43
44 return t;
45 }
46
47 static struct mt76_txwi_cache *
mt76_get_txwi(struct mt76_dev *dev)48 mt76_get_txwi(struct mt76_dev *dev)
49 {
50 struct mt76_txwi_cache *t = __mt76_get_txwi(dev);
51
52 if (t)
53 return t;
54
55 return mt76_alloc_txwi(dev);
56 }
57
58 void
mt76_put_txwi(struct mt76_dev *dev, struct mt76_txwi_cache *t)59 mt76_put_txwi(struct mt76_dev *dev, struct mt76_txwi_cache *t)
60 {
61 if (!t)
62 return;
63
64 spin_lock(&dev->lock);
65 list_add(&t->list, &dev->txwi_cache);
66 spin_unlock(&dev->lock);
67 }
68 EXPORT_SYMBOL_GPL(mt76_put_txwi);
69
70 static void
mt76_free_pending_txwi(struct mt76_dev *dev)71 mt76_free_pending_txwi(struct mt76_dev *dev)
72 {
73 struct mt76_txwi_cache *t;
74
75 local_bh_disable();
76 while ((t = __mt76_get_txwi(dev)) != NULL)
77 dma_unmap_single(dev->dev, t->dma_addr, dev->drv->txwi_size,
78 DMA_TO_DEVICE);
79 local_bh_enable();
80 }
81
82 static int
mt76_dma_alloc_queue(struct mt76_dev *dev, struct mt76_queue *q, int idx, int n_desc, int bufsize, u32 ring_base)83 mt76_dma_alloc_queue(struct mt76_dev *dev, struct mt76_queue *q,
84 int idx, int n_desc, int bufsize,
85 u32 ring_base)
86 {
87 int size;
88 int i;
89
90 spin_lock_init(&q->lock);
91
92 q->regs = dev->mmio.regs + ring_base + idx * MT_RING_SIZE;
93 q->ndesc = n_desc;
94 q->buf_size = bufsize;
95 q->hw_idx = idx;
96
97 size = q->ndesc * sizeof(struct mt76_desc);
98 q->desc = dmam_alloc_coherent(dev->dev, size, &q->desc_dma, GFP_KERNEL);
99 if (!q->desc)
100 return -ENOMEM;
101
102 size = q->ndesc * sizeof(*q->entry);
103 q->entry = devm_kzalloc(dev->dev, size, GFP_KERNEL);
104 if (!q->entry)
105 return -ENOMEM;
106
107 /* clear descriptors */
108 for (i = 0; i < q->ndesc; i++)
109 q->desc[i].ctrl = cpu_to_le32(MT_DMA_CTL_DMA_DONE);
110
111 writel(q->desc_dma, &q->regs->desc_base);
112 writel(0, &q->regs->cpu_idx);
113 writel(0, &q->regs->dma_idx);
114 writel(q->ndesc, &q->regs->ring_size);
115
116 return 0;
117 }
118
119 static int
mt76_dma_add_buf(struct mt76_dev *dev, struct mt76_queue *q, struct mt76_queue_buf *buf, int nbufs, u32 info, struct sk_buff *skb, void *txwi)120 mt76_dma_add_buf(struct mt76_dev *dev, struct mt76_queue *q,
121 struct mt76_queue_buf *buf, int nbufs, u32 info,
122 struct sk_buff *skb, void *txwi)
123 {
124 struct mt76_queue_entry *entry;
125 struct mt76_desc *desc;
126 u32 ctrl;
127 int i, idx = -1;
128
129 if (txwi) {
130 q->entry[q->head].txwi = DMA_DUMMY_DATA;
131 q->entry[q->head].skip_buf0 = true;
132 }
133
134 for (i = 0; i < nbufs; i += 2, buf += 2) {
135 u32 buf0 = buf[0].addr, buf1 = 0;
136
137 idx = q->head;
138 q->head = (q->head + 1) % q->ndesc;
139
140 desc = &q->desc[idx];
141 entry = &q->entry[idx];
142
143 if (buf[0].skip_unmap)
144 entry->skip_buf0 = true;
145 entry->skip_buf1 = i == nbufs - 1;
146
147 entry->dma_addr[0] = buf[0].addr;
148 entry->dma_len[0] = buf[0].len;
149
150 ctrl = FIELD_PREP(MT_DMA_CTL_SD_LEN0, buf[0].len);
151 if (i < nbufs - 1) {
152 entry->dma_addr[1] = buf[1].addr;
153 entry->dma_len[1] = buf[1].len;
154 buf1 = buf[1].addr;
155 ctrl |= FIELD_PREP(MT_DMA_CTL_SD_LEN1, buf[1].len);
156 if (buf[1].skip_unmap)
157 entry->skip_buf1 = true;
158 }
159
160 if (i == nbufs - 1)
161 ctrl |= MT_DMA_CTL_LAST_SEC0;
162 else if (i == nbufs - 2)
163 ctrl |= MT_DMA_CTL_LAST_SEC1;
164
165 WRITE_ONCE(desc->buf0, cpu_to_le32(buf0));
166 WRITE_ONCE(desc->buf1, cpu_to_le32(buf1));
167 WRITE_ONCE(desc->info, cpu_to_le32(info));
168 WRITE_ONCE(desc->ctrl, cpu_to_le32(ctrl));
169
170 q->queued++;
171 }
172
173 q->entry[idx].txwi = txwi;
174 q->entry[idx].skb = skb;
175
176 return idx;
177 }
178
179 static void
mt76_dma_tx_cleanup_idx(struct mt76_dev *dev, struct mt76_queue *q, int idx, struct mt76_queue_entry *prev_e)180 mt76_dma_tx_cleanup_idx(struct mt76_dev *dev, struct mt76_queue *q, int idx,
181 struct mt76_queue_entry *prev_e)
182 {
183 struct mt76_queue_entry *e = &q->entry[idx];
184
185 if (!e->skip_buf0)
186 dma_unmap_single(dev->dev, e->dma_addr[0], e->dma_len[0],
187 DMA_TO_DEVICE);
188
189 if (!e->skip_buf1)
190 dma_unmap_single(dev->dev, e->dma_addr[1], e->dma_len[1],
191 DMA_TO_DEVICE);
192
193 if (e->txwi == DMA_DUMMY_DATA)
194 e->txwi = NULL;
195
196 if (e->skb == DMA_DUMMY_DATA)
197 e->skb = NULL;
198
199 *prev_e = *e;
200 memset(e, 0, sizeof(*e));
201 }
202
203 static void
mt76_dma_sync_idx(struct mt76_dev *dev, struct mt76_queue *q)204 mt76_dma_sync_idx(struct mt76_dev *dev, struct mt76_queue *q)
205 {
206 writel(q->desc_dma, &q->regs->desc_base);
207 writel(q->ndesc, &q->regs->ring_size);
208 q->head = readl(&q->regs->dma_idx);
209 q->tail = q->head;
210 }
211
212 static void
mt76_dma_kick_queue(struct mt76_dev *dev, struct mt76_queue *q)213 mt76_dma_kick_queue(struct mt76_dev *dev, struct mt76_queue *q)
214 {
215 wmb();
216 writel(q->head, &q->regs->cpu_idx);
217 }
218
219 static void
mt76_dma_tx_cleanup(struct mt76_dev *dev, enum mt76_txq_id qid, bool flush)220 mt76_dma_tx_cleanup(struct mt76_dev *dev, enum mt76_txq_id qid, bool flush)
221 {
222 struct mt76_queue *q = dev->q_tx[qid];
223 struct mt76_queue_entry entry;
224 bool wake = false;
225 int last;
226
227 if (!q)
228 return;
229
230 if (flush)
231 last = -1;
232 else
233 last = readl(&q->regs->dma_idx);
234
235 while (q->queued > 0 && q->tail != last) {
236 mt76_dma_tx_cleanup_idx(dev, q, q->tail, &entry);
237 mt76_queue_tx_complete(dev, q, &entry);
238
239 if (entry.txwi) {
240 if (!(dev->drv->drv_flags & MT_DRV_TXWI_NO_FREE))
241 mt76_put_txwi(dev, entry.txwi);
242 wake = !flush;
243 }
244
245 if (!flush && q->tail == last)
246 last = readl(&q->regs->dma_idx);
247
248 }
249
250 if (flush) {
251 spin_lock_bh(&q->lock);
252 mt76_dma_sync_idx(dev, q);
253 mt76_dma_kick_queue(dev, q);
254 spin_unlock_bh(&q->lock);
255 }
256
257 wake = wake && q->stopped &&
258 qid < IEEE80211_NUM_ACS && q->queued < q->ndesc - 8;
259 if (wake)
260 q->stopped = false;
261
262 if (!q->queued)
263 wake_up(&dev->tx_wait);
264
265 if (wake)
266 ieee80211_wake_queue(dev->hw, qid);
267 }
268
269 static void *
mt76_dma_get_buf(struct mt76_dev *dev, struct mt76_queue *q, int idx, int *len, u32 *info, bool *more)270 mt76_dma_get_buf(struct mt76_dev *dev, struct mt76_queue *q, int idx,
271 int *len, u32 *info, bool *more)
272 {
273 struct mt76_queue_entry *e = &q->entry[idx];
274 struct mt76_desc *desc = &q->desc[idx];
275 dma_addr_t buf_addr;
276 void *buf = e->buf;
277 int buf_len = SKB_WITH_OVERHEAD(q->buf_size);
278
279 buf_addr = e->dma_addr[0];
280 if (len) {
281 u32 ctl = le32_to_cpu(READ_ONCE(desc->ctrl));
282 *len = FIELD_GET(MT_DMA_CTL_SD_LEN0, ctl);
283 *more = !(ctl & MT_DMA_CTL_LAST_SEC0);
284 }
285
286 if (info)
287 *info = le32_to_cpu(desc->info);
288
289 dma_unmap_single(dev->dev, buf_addr, buf_len, DMA_FROM_DEVICE);
290 e->buf = NULL;
291
292 return buf;
293 }
294
295 static void *
mt76_dma_dequeue(struct mt76_dev *dev, struct mt76_queue *q, bool flush, int *len, u32 *info, bool *more)296 mt76_dma_dequeue(struct mt76_dev *dev, struct mt76_queue *q, bool flush,
297 int *len, u32 *info, bool *more)
298 {
299 int idx = q->tail;
300
301 *more = false;
302 if (!q->queued)
303 return NULL;
304
305 if (flush)
306 q->desc[idx].ctrl |= cpu_to_le32(MT_DMA_CTL_DMA_DONE);
307 else if (!(q->desc[idx].ctrl & cpu_to_le32(MT_DMA_CTL_DMA_DONE)))
308 return NULL;
309
310 q->tail = (q->tail + 1) % q->ndesc;
311 q->queued--;
312
313 return mt76_dma_get_buf(dev, q, idx, len, info, more);
314 }
315
316 static int
mt76_dma_tx_queue_skb_raw(struct mt76_dev *dev, enum mt76_txq_id qid, struct sk_buff *skb, u32 tx_info)317 mt76_dma_tx_queue_skb_raw(struct mt76_dev *dev, enum mt76_txq_id qid,
318 struct sk_buff *skb, u32 tx_info)
319 {
320 struct mt76_queue *q = dev->q_tx[qid];
321 struct mt76_queue_buf buf = {};
322 dma_addr_t addr;
323
324 if (q->queued + 1 >= q->ndesc - 1)
325 goto error;
326
327 addr = dma_map_single(dev->dev, skb->data, skb->len,
328 DMA_TO_DEVICE);
329 if (unlikely(dma_mapping_error(dev->dev, addr)))
330 goto error;
331
332 buf.addr = addr;
333 buf.len = skb->len;
334
335 spin_lock_bh(&q->lock);
336 mt76_dma_add_buf(dev, q, &buf, 1, tx_info, skb, NULL);
337 mt76_dma_kick_queue(dev, q);
338 spin_unlock_bh(&q->lock);
339
340 return 0;
341
342 error:
343 dev_kfree_skb(skb);
344 return -ENOMEM;
345 }
346
347 static int
mt76_dma_tx_queue_skb(struct mt76_dev *dev, enum mt76_txq_id qid, struct sk_buff *skb, struct mt76_wcid *wcid, struct ieee80211_sta *sta)348 mt76_dma_tx_queue_skb(struct mt76_dev *dev, enum mt76_txq_id qid,
349 struct sk_buff *skb, struct mt76_wcid *wcid,
350 struct ieee80211_sta *sta)
351 {
352 struct mt76_queue *q = dev->q_tx[qid];
353 struct mt76_tx_info tx_info = {
354 .skb = skb,
355 };
356 struct ieee80211_hw *hw;
357 int len, n = 0, ret = -ENOMEM;
358 struct mt76_txwi_cache *t;
359 struct sk_buff *iter;
360 dma_addr_t addr;
361 u8 *txwi;
362
363 t = mt76_get_txwi(dev);
364 if (!t) {
365 hw = mt76_tx_status_get_hw(dev, skb);
366 ieee80211_free_txskb(hw, skb);
367 return -ENOMEM;
368 }
369 txwi = mt76_get_txwi_ptr(dev, t);
370
371 skb->prev = skb->next = NULL;
372 if (dev->drv->drv_flags & MT_DRV_TX_ALIGNED4_SKBS)
373 mt76_insert_hdr_pad(skb);
374
375 len = skb_headlen(skb);
376 addr = dma_map_single(dev->dev, skb->data, len, DMA_TO_DEVICE);
377 if (unlikely(dma_mapping_error(dev->dev, addr)))
378 goto free;
379
380 tx_info.buf[n].addr = t->dma_addr;
381 tx_info.buf[n++].len = dev->drv->txwi_size;
382 tx_info.buf[n].addr = addr;
383 tx_info.buf[n++].len = len;
384
385 skb_walk_frags(skb, iter) {
386 if (n == ARRAY_SIZE(tx_info.buf))
387 goto unmap;
388
389 addr = dma_map_single(dev->dev, iter->data, iter->len,
390 DMA_TO_DEVICE);
391 if (unlikely(dma_mapping_error(dev->dev, addr)))
392 goto unmap;
393
394 tx_info.buf[n].addr = addr;
395 tx_info.buf[n++].len = iter->len;
396 }
397 tx_info.nbuf = n;
398
399 if (q->queued + (tx_info.nbuf + 1) / 2 >= q->ndesc - 1) {
400 ret = -ENOMEM;
401 goto unmap;
402 }
403
404 dma_sync_single_for_cpu(dev->dev, t->dma_addr, dev->drv->txwi_size,
405 DMA_TO_DEVICE);
406 ret = dev->drv->tx_prepare_skb(dev, txwi, qid, wcid, sta, &tx_info);
407 dma_sync_single_for_device(dev->dev, t->dma_addr, dev->drv->txwi_size,
408 DMA_TO_DEVICE);
409 if (ret < 0)
410 goto unmap;
411
412 return mt76_dma_add_buf(dev, q, tx_info.buf, tx_info.nbuf,
413 tx_info.info, tx_info.skb, t);
414
415 unmap:
416 for (n--; n > 0; n--)
417 dma_unmap_single(dev->dev, tx_info.buf[n].addr,
418 tx_info.buf[n].len, DMA_TO_DEVICE);
419
420 free:
421 #ifdef CONFIG_NL80211_TESTMODE
422 /* fix tx_done accounting on queue overflow */
423 if (tx_info.skb == dev->test.tx_skb)
424 dev->test.tx_done--;
425 #endif
426
427 dev_kfree_skb(tx_info.skb);
428 mt76_put_txwi(dev, t);
429 return ret;
430 }
431
432 static int
mt76_dma_rx_fill(struct mt76_dev *dev, struct mt76_queue *q)433 mt76_dma_rx_fill(struct mt76_dev *dev, struct mt76_queue *q)
434 {
435 dma_addr_t addr;
436 void *buf;
437 int frames = 0;
438 int len = SKB_WITH_OVERHEAD(q->buf_size);
439 int offset = q->buf_offset;
440
441 spin_lock_bh(&q->lock);
442
443 while (q->queued < q->ndesc - 1) {
444 struct mt76_queue_buf qbuf;
445
446 buf = page_frag_alloc(&q->rx_page, q->buf_size, GFP_ATOMIC);
447 if (!buf)
448 break;
449
450 addr = dma_map_single(dev->dev, buf, len, DMA_FROM_DEVICE);
451 if (unlikely(dma_mapping_error(dev->dev, addr))) {
452 skb_free_frag(buf);
453 break;
454 }
455
456 qbuf.addr = addr + offset;
457 qbuf.len = len - offset;
458 qbuf.skip_unmap = false;
459 mt76_dma_add_buf(dev, q, &qbuf, 1, 0, buf, NULL);
460 frames++;
461 }
462
463 if (frames)
464 mt76_dma_kick_queue(dev, q);
465
466 spin_unlock_bh(&q->lock);
467
468 return frames;
469 }
470
471 static void
mt76_dma_rx_cleanup(struct mt76_dev *dev, struct mt76_queue *q)472 mt76_dma_rx_cleanup(struct mt76_dev *dev, struct mt76_queue *q)
473 {
474 struct page *page;
475 void *buf;
476 bool more;
477
478 spin_lock_bh(&q->lock);
479
480 do {
481 buf = mt76_dma_dequeue(dev, q, true, NULL, NULL, &more);
482 if (!buf)
483 break;
484
485 skb_free_frag(buf);
486 } while (1);
487
488 if (q->rx_head) {
489 dev_kfree_skb(q->rx_head);
490 q->rx_head = NULL;
491 }
492
493 spin_unlock_bh(&q->lock);
494
495 if (!q->rx_page.va)
496 return;
497
498 page = virt_to_page(q->rx_page.va);
499 __page_frag_cache_drain(page, q->rx_page.pagecnt_bias);
500 memset(&q->rx_page, 0, sizeof(q->rx_page));
501 }
502
503 static void
mt76_dma_rx_reset(struct mt76_dev *dev, enum mt76_rxq_id qid)504 mt76_dma_rx_reset(struct mt76_dev *dev, enum mt76_rxq_id qid)
505 {
506 struct mt76_queue *q = &dev->q_rx[qid];
507 int i;
508
509 for (i = 0; i < q->ndesc; i++)
510 q->desc[i].ctrl = cpu_to_le32(MT_DMA_CTL_DMA_DONE);
511
512 mt76_dma_rx_cleanup(dev, q);
513 mt76_dma_sync_idx(dev, q);
514 mt76_dma_rx_fill(dev, q);
515 }
516
517 static void
mt76_add_fragment(struct mt76_dev *dev, struct mt76_queue *q, void *data, int len, bool more)518 mt76_add_fragment(struct mt76_dev *dev, struct mt76_queue *q, void *data,
519 int len, bool more)
520 {
521 struct sk_buff *skb = q->rx_head;
522 struct skb_shared_info *shinfo = skb_shinfo(skb);
523 int nr_frags = shinfo->nr_frags;
524
525 if (nr_frags < ARRAY_SIZE(shinfo->frags)) {
526 struct page *page = virt_to_head_page(data);
527 int offset = data - page_address(page) + q->buf_offset;
528
529 skb_add_rx_frag(skb, nr_frags, page, offset, len, q->buf_size);
530 } else {
531 skb_free_frag(data);
532 }
533
534 if (more)
535 return;
536
537 q->rx_head = NULL;
538 if (nr_frags < ARRAY_SIZE(shinfo->frags))
539 dev->drv->rx_skb(dev, q - dev->q_rx, skb);
540 else
541 dev_kfree_skb(skb);
542 }
543
544 static int
mt76_dma_rx_process(struct mt76_dev *dev, struct mt76_queue *q, int budget)545 mt76_dma_rx_process(struct mt76_dev *dev, struct mt76_queue *q, int budget)
546 {
547 int len, data_len, done = 0;
548 struct sk_buff *skb;
549 unsigned char *data;
550 bool more;
551
552 while (done < budget) {
553 u32 info;
554
555 data = mt76_dma_dequeue(dev, q, false, &len, &info, &more);
556 if (!data)
557 break;
558
559 if (q->rx_head)
560 data_len = q->buf_size;
561 else
562 data_len = SKB_WITH_OVERHEAD(q->buf_size);
563
564 if (data_len < len + q->buf_offset) {
565 dev_kfree_skb(q->rx_head);
566 q->rx_head = NULL;
567
568 skb_free_frag(data);
569 continue;
570 }
571
572 if (q->rx_head) {
573 mt76_add_fragment(dev, q, data, len, more);
574 continue;
575 }
576
577 skb = build_skb(data, q->buf_size);
578 if (!skb) {
579 skb_free_frag(data);
580 continue;
581 }
582 skb_reserve(skb, q->buf_offset);
583
584 if (q == &dev->q_rx[MT_RXQ_MCU]) {
585 u32 *rxfce = (u32 *)skb->cb;
586 *rxfce = info;
587 }
588
589 __skb_put(skb, len);
590 done++;
591
592 if (more) {
593 q->rx_head = skb;
594 continue;
595 }
596
597 dev->drv->rx_skb(dev, q - dev->q_rx, skb);
598 }
599
600 mt76_dma_rx_fill(dev, q);
601 return done;
602 }
603
604 static int
mt76_dma_rx_poll(struct napi_struct *napi, int budget)605 mt76_dma_rx_poll(struct napi_struct *napi, int budget)
606 {
607 struct mt76_dev *dev;
608 int qid, done = 0, cur;
609
610 dev = container_of(napi->dev, struct mt76_dev, napi_dev);
611 qid = napi - dev->napi;
612
613 local_bh_disable();
614 rcu_read_lock();
615
616 do {
617 cur = mt76_dma_rx_process(dev, &dev->q_rx[qid], budget - done);
618 mt76_rx_poll_complete(dev, qid, napi);
619 done += cur;
620 } while (cur && done < budget);
621
622 rcu_read_unlock();
623 local_bh_enable();
624
625 if (done < budget && napi_complete(napi))
626 dev->drv->rx_poll_complete(dev, qid);
627
628 return done;
629 }
630
631 static int
mt76_dma_init(struct mt76_dev *dev)632 mt76_dma_init(struct mt76_dev *dev)
633 {
634 int i;
635
636 init_dummy_netdev(&dev->napi_dev);
637
638 mt76_for_each_q_rx(dev, i) {
639 netif_napi_add(&dev->napi_dev, &dev->napi[i], mt76_dma_rx_poll,
640 64);
641 mt76_dma_rx_fill(dev, &dev->q_rx[i]);
642 napi_enable(&dev->napi[i]);
643 }
644
645 return 0;
646 }
647
648 static const struct mt76_queue_ops mt76_dma_ops = {
649 .init = mt76_dma_init,
650 .alloc = mt76_dma_alloc_queue,
651 .tx_queue_skb_raw = mt76_dma_tx_queue_skb_raw,
652 .tx_queue_skb = mt76_dma_tx_queue_skb,
653 .tx_cleanup = mt76_dma_tx_cleanup,
654 .rx_reset = mt76_dma_rx_reset,
655 .kick = mt76_dma_kick_queue,
656 };
657
mt76_dma_attach(struct mt76_dev *dev)658 void mt76_dma_attach(struct mt76_dev *dev)
659 {
660 dev->queue_ops = &mt76_dma_ops;
661 }
662 EXPORT_SYMBOL_GPL(mt76_dma_attach);
663
mt76_dma_cleanup(struct mt76_dev *dev)664 void mt76_dma_cleanup(struct mt76_dev *dev)
665 {
666 int i;
667
668 mt76_worker_disable(&dev->tx_worker);
669 netif_napi_del(&dev->tx_napi);
670 for (i = 0; i < ARRAY_SIZE(dev->q_tx); i++)
671 mt76_dma_tx_cleanup(dev, i, true);
672
673 mt76_for_each_q_rx(dev, i) {
674 netif_napi_del(&dev->napi[i]);
675 mt76_dma_rx_cleanup(dev, &dev->q_rx[i]);
676 }
677
678 mt76_free_pending_txwi(dev);
679 }
680 EXPORT_SYMBOL_GPL(mt76_dma_cleanup);
681