1// SPDX-License-Identifier: GPL-2.0 2/* 3 * Copyright (c) 2018-2019 Synopsys, Inc. and/or its affiliates. 4 * Synopsys DesignWare eDMA core driver 5 * 6 * Author: Gustavo Pimentel <gustavo.pimentel@synopsys.com> 7 */ 8 9#include <linux/module.h> 10#include <linux/device.h> 11#include <linux/kernel.h> 12#include <linux/pm_runtime.h> 13#include <linux/dmaengine.h> 14#include <linux/err.h> 15#include <linux/interrupt.h> 16#include <linux/irq.h> 17#include <linux/dma/edma.h> 18#include <linux/dma-mapping.h> 19 20#include "dw-edma-core.h" 21#include "dw-edma-v0-core.h" 22#include "../dmaengine.h" 23#include "../virt-dma.h" 24 25static inline 26struct device *dchan2dev(struct dma_chan *dchan) 27{ 28 return &dchan->dev->device; 29} 30 31static inline 32struct device *chan2dev(struct dw_edma_chan *chan) 33{ 34 return &chan->vc.chan.dev->device; 35} 36 37static inline 38struct dw_edma_desc *vd2dw_edma_desc(struct virt_dma_desc *vd) 39{ 40 return container_of(vd, struct dw_edma_desc, vd); 41} 42 43static struct dw_edma_burst *dw_edma_alloc_burst(struct dw_edma_chunk *chunk) 44{ 45 struct dw_edma_burst *burst; 46 47 burst = kzalloc(sizeof(*burst), GFP_NOWAIT); 48 if (unlikely(!burst)) 49 return NULL; 50 51 INIT_LIST_HEAD(&burst->list); 52 if (chunk->burst) { 53 /* Create and add new element into the linked list */ 54 chunk->bursts_alloc++; 55 list_add_tail(&burst->list, &chunk->burst->list); 56 } else { 57 /* List head */ 58 chunk->bursts_alloc = 0; 59 chunk->burst = burst; 60 } 61 62 return burst; 63} 64 65static struct dw_edma_chunk *dw_edma_alloc_chunk(struct dw_edma_desc *desc) 66{ 67 struct dw_edma_chan *chan = desc->chan; 68 struct dw_edma *dw = chan->chip->dw; 69 struct dw_edma_chunk *chunk; 70 71 chunk = kzalloc(sizeof(*chunk), GFP_NOWAIT); 72 if (unlikely(!chunk)) 73 return NULL; 74 75 INIT_LIST_HEAD(&chunk->list); 76 chunk->chan = chan; 77 /* Toggling change bit (CB) in each chunk, this is a mechanism to 78 * inform the eDMA HW block that this is a new linked list ready 79 * to be consumed. 80 * - Odd chunks originate CB equal to 0 81 * - Even chunks originate CB equal to 1 82 */ 83 chunk->cb = !(desc->chunks_alloc % 2); 84 chunk->ll_region.paddr = dw->ll_region.paddr + chan->ll_off; 85 chunk->ll_region.vaddr = dw->ll_region.vaddr + chan->ll_off; 86 87 if (desc->chunk) { 88 /* Create and add new element into the linked list */ 89 if (!dw_edma_alloc_burst(chunk)) { 90 kfree(chunk); 91 return NULL; 92 } 93 desc->chunks_alloc++; 94 list_add_tail(&chunk->list, &desc->chunk->list); 95 } else { 96 /* List head */ 97 chunk->burst = NULL; 98 desc->chunks_alloc = 0; 99 desc->chunk = chunk; 100 } 101 102 return chunk; 103} 104 105static struct dw_edma_desc *dw_edma_alloc_desc(struct dw_edma_chan *chan) 106{ 107 struct dw_edma_desc *desc; 108 109 desc = kzalloc(sizeof(*desc), GFP_NOWAIT); 110 if (unlikely(!desc)) 111 return NULL; 112 113 desc->chan = chan; 114 if (!dw_edma_alloc_chunk(desc)) { 115 kfree(desc); 116 return NULL; 117 } 118 119 return desc; 120} 121 122static void dw_edma_free_burst(struct dw_edma_chunk *chunk) 123{ 124 struct dw_edma_burst *child, *_next; 125 126 /* Remove all the list elements */ 127 list_for_each_entry_safe(child, _next, &chunk->burst->list, list) { 128 list_del(&child->list); 129 kfree(child); 130 chunk->bursts_alloc--; 131 } 132 133 /* Remove the list head */ 134 kfree(child); 135 chunk->burst = NULL; 136} 137 138static void dw_edma_free_chunk(struct dw_edma_desc *desc) 139{ 140 struct dw_edma_chunk *child, *_next; 141 142 if (!desc->chunk) 143 return; 144 145 /* Remove all the list elements */ 146 list_for_each_entry_safe(child, _next, &desc->chunk->list, list) { 147 dw_edma_free_burst(child); 148 list_del(&child->list); 149 kfree(child); 150 desc->chunks_alloc--; 151 } 152 153 /* Remove the list head */ 154 kfree(child); 155 desc->chunk = NULL; 156} 157 158static void dw_edma_free_desc(struct dw_edma_desc *desc) 159{ 160 dw_edma_free_chunk(desc); 161 kfree(desc); 162} 163 164static void vchan_free_desc(struct virt_dma_desc *vdesc) 165{ 166 dw_edma_free_desc(vd2dw_edma_desc(vdesc)); 167} 168 169static int dw_edma_start_transfer(struct dw_edma_chan *chan) 170{ 171 struct dw_edma_chunk *child; 172 struct dw_edma_desc *desc; 173 struct virt_dma_desc *vd; 174 175 vd = vchan_next_desc(&chan->vc); 176 if (!vd) 177 return 0; 178 179 desc = vd2dw_edma_desc(vd); 180 if (!desc) 181 return 0; 182 183 child = list_first_entry_or_null(&desc->chunk->list, 184 struct dw_edma_chunk, list); 185 if (!child) 186 return 0; 187 188 dw_edma_v0_core_start(child, !desc->xfer_sz); 189 desc->xfer_sz += child->ll_region.sz; 190 dw_edma_free_burst(child); 191 list_del(&child->list); 192 kfree(child); 193 desc->chunks_alloc--; 194 195 return 1; 196} 197 198static int dw_edma_device_config(struct dma_chan *dchan, 199 struct dma_slave_config *config) 200{ 201 struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan); 202 203 memcpy(&chan->config, config, sizeof(*config)); 204 chan->configured = true; 205 206 return 0; 207} 208 209static int dw_edma_device_pause(struct dma_chan *dchan) 210{ 211 struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan); 212 int err = 0; 213 214 if (!chan->configured) 215 err = -EPERM; 216 else if (chan->status != EDMA_ST_BUSY) 217 err = -EPERM; 218 else if (chan->request != EDMA_REQ_NONE) 219 err = -EPERM; 220 else 221 chan->request = EDMA_REQ_PAUSE; 222 223 return err; 224} 225 226static int dw_edma_device_resume(struct dma_chan *dchan) 227{ 228 struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan); 229 int err = 0; 230 231 if (!chan->configured) { 232 err = -EPERM; 233 } else if (chan->status != EDMA_ST_PAUSE) { 234 err = -EPERM; 235 } else if (chan->request != EDMA_REQ_NONE) { 236 err = -EPERM; 237 } else { 238 chan->status = EDMA_ST_BUSY; 239 dw_edma_start_transfer(chan); 240 } 241 242 return err; 243} 244 245static int dw_edma_device_terminate_all(struct dma_chan *dchan) 246{ 247 struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan); 248 int err = 0; 249 LIST_HEAD(head); 250 251 if (!chan->configured) { 252 /* Do nothing */ 253 } else if (chan->status == EDMA_ST_PAUSE) { 254 chan->status = EDMA_ST_IDLE; 255 chan->configured = false; 256 } else if (chan->status == EDMA_ST_IDLE) { 257 chan->configured = false; 258 } else if (dw_edma_v0_core_ch_status(chan) == DMA_COMPLETE) { 259 /* 260 * The channel is in a false BUSY state, probably didn't 261 * receive or lost an interrupt 262 */ 263 chan->status = EDMA_ST_IDLE; 264 chan->configured = false; 265 } else if (chan->request > EDMA_REQ_PAUSE) { 266 err = -EPERM; 267 } else { 268 chan->request = EDMA_REQ_STOP; 269 } 270 271 return err; 272} 273 274static void dw_edma_device_issue_pending(struct dma_chan *dchan) 275{ 276 struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan); 277 unsigned long flags; 278 279 if (!chan->configured) 280 return; 281 282 spin_lock_irqsave(&chan->vc.lock, flags); 283 if (vchan_issue_pending(&chan->vc) && chan->request == EDMA_REQ_NONE && 284 chan->status == EDMA_ST_IDLE) { 285 chan->status = EDMA_ST_BUSY; 286 dw_edma_start_transfer(chan); 287 } 288 spin_unlock_irqrestore(&chan->vc.lock, flags); 289} 290 291static enum dma_status 292dw_edma_device_tx_status(struct dma_chan *dchan, dma_cookie_t cookie, 293 struct dma_tx_state *txstate) 294{ 295 struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan); 296 struct dw_edma_desc *desc; 297 struct virt_dma_desc *vd; 298 unsigned long flags; 299 enum dma_status ret; 300 u32 residue = 0; 301 302 ret = dma_cookie_status(dchan, cookie, txstate); 303 if (ret == DMA_COMPLETE) 304 return ret; 305 306 if (ret == DMA_IN_PROGRESS && chan->status == EDMA_ST_PAUSE) 307 ret = DMA_PAUSED; 308 309 if (!txstate) 310 goto ret_residue; 311 312 spin_lock_irqsave(&chan->vc.lock, flags); 313 vd = vchan_find_desc(&chan->vc, cookie); 314 if (vd) { 315 desc = vd2dw_edma_desc(vd); 316 if (desc) 317 residue = desc->alloc_sz - desc->xfer_sz; 318 } 319 spin_unlock_irqrestore(&chan->vc.lock, flags); 320 321ret_residue: 322 dma_set_residue(txstate, residue); 323 324 return ret; 325} 326 327static struct dma_async_tx_descriptor * 328dw_edma_device_transfer(struct dw_edma_transfer *xfer) 329{ 330 struct dw_edma_chan *chan = dchan2dw_edma_chan(xfer->dchan); 331 enum dma_transfer_direction dir = xfer->direction; 332 phys_addr_t src_addr, dst_addr; 333 struct scatterlist *sg = NULL; 334 struct dw_edma_chunk *chunk; 335 struct dw_edma_burst *burst; 336 struct dw_edma_desc *desc; 337 u32 cnt; 338 int i; 339 340 if (!chan->configured) 341 return NULL; 342 343 switch (chan->config.direction) { 344 case DMA_DEV_TO_MEM: /* local dma */ 345 if (dir == DMA_DEV_TO_MEM && chan->dir == EDMA_DIR_READ) 346 break; 347 return NULL; 348 case DMA_MEM_TO_DEV: /* local dma */ 349 if (dir == DMA_MEM_TO_DEV && chan->dir == EDMA_DIR_WRITE) 350 break; 351 return NULL; 352 default: /* remote dma */ 353 if (dir == DMA_MEM_TO_DEV && chan->dir == EDMA_DIR_READ) 354 break; 355 if (dir == DMA_DEV_TO_MEM && chan->dir == EDMA_DIR_WRITE) 356 break; 357 return NULL; 358 } 359 360 if (xfer->cyclic) { 361 if (!xfer->xfer.cyclic.len || !xfer->xfer.cyclic.cnt) 362 return NULL; 363 } else { 364 if (xfer->xfer.sg.len < 1) 365 return NULL; 366 } 367 368 desc = dw_edma_alloc_desc(chan); 369 if (unlikely(!desc)) 370 goto err_alloc; 371 372 chunk = dw_edma_alloc_chunk(desc); 373 if (unlikely(!chunk)) 374 goto err_alloc; 375 376 src_addr = chan->config.src_addr; 377 dst_addr = chan->config.dst_addr; 378 379 if (xfer->cyclic) { 380 cnt = xfer->xfer.cyclic.cnt; 381 } else { 382 cnt = xfer->xfer.sg.len; 383 sg = xfer->xfer.sg.sgl; 384 } 385 386 for (i = 0; i < cnt; i++) { 387 if (!xfer->cyclic && !sg) 388 break; 389 390 if (chunk->bursts_alloc == chan->ll_max) { 391 chunk = dw_edma_alloc_chunk(desc); 392 if (unlikely(!chunk)) 393 goto err_alloc; 394 } 395 396 burst = dw_edma_alloc_burst(chunk); 397 if (unlikely(!burst)) 398 goto err_alloc; 399 400 if (xfer->cyclic) 401 burst->sz = xfer->xfer.cyclic.len; 402 else 403 burst->sz = sg_dma_len(sg); 404 405 chunk->ll_region.sz += burst->sz; 406 desc->alloc_sz += burst->sz; 407 408 if (dir == DMA_DEV_TO_MEM) { 409 burst->sar = src_addr; 410 if (xfer->cyclic) { 411 burst->dar = xfer->xfer.cyclic.paddr; 412 } else { 413 burst->dar = dst_addr; 414 /* Unlike the typical assumption by other 415 * drivers/IPs the peripheral memory isn't 416 * a FIFO memory, in this case, it's a 417 * linear memory and that why the source 418 * and destination addresses are increased 419 * by the same portion (data length) 420 */ 421 } 422 } else { 423 burst->dar = dst_addr; 424 if (xfer->cyclic) { 425 burst->sar = xfer->xfer.cyclic.paddr; 426 } else { 427 burst->sar = src_addr; 428 /* Unlike the typical assumption by other 429 * drivers/IPs the peripheral memory isn't 430 * a FIFO memory, in this case, it's a 431 * linear memory and that why the source 432 * and destination addresses are increased 433 * by the same portion (data length) 434 */ 435 } 436 } 437 438 if (!xfer->cyclic) { 439 src_addr += sg_dma_len(sg); 440 dst_addr += sg_dma_len(sg); 441 sg = sg_next(sg); 442 } 443 } 444 445 return vchan_tx_prep(&chan->vc, &desc->vd, xfer->flags); 446 447err_alloc: 448 if (desc) 449 dw_edma_free_desc(desc); 450 451 return NULL; 452} 453 454static struct dma_async_tx_descriptor * 455dw_edma_device_prep_slave_sg(struct dma_chan *dchan, struct scatterlist *sgl, 456 unsigned int len, 457 enum dma_transfer_direction direction, 458 unsigned long flags, void *context) 459{ 460 struct dw_edma_transfer xfer; 461 462 xfer.dchan = dchan; 463 xfer.direction = direction; 464 xfer.xfer.sg.sgl = sgl; 465 xfer.xfer.sg.len = len; 466 xfer.flags = flags; 467 xfer.cyclic = false; 468 469 return dw_edma_device_transfer(&xfer); 470} 471 472static struct dma_async_tx_descriptor * 473dw_edma_device_prep_dma_cyclic(struct dma_chan *dchan, dma_addr_t paddr, 474 size_t len, size_t count, 475 enum dma_transfer_direction direction, 476 unsigned long flags) 477{ 478 struct dw_edma_transfer xfer; 479 480 xfer.dchan = dchan; 481 xfer.direction = direction; 482 xfer.xfer.cyclic.paddr = paddr; 483 xfer.xfer.cyclic.len = len; 484 xfer.xfer.cyclic.cnt = count; 485 xfer.flags = flags; 486 xfer.cyclic = true; 487 488 return dw_edma_device_transfer(&xfer); 489} 490 491static void dw_edma_done_interrupt(struct dw_edma_chan *chan) 492{ 493 struct dw_edma_desc *desc; 494 struct virt_dma_desc *vd; 495 unsigned long flags; 496 497 dw_edma_v0_core_clear_done_int(chan); 498 499 spin_lock_irqsave(&chan->vc.lock, flags); 500 vd = vchan_next_desc(&chan->vc); 501 if (vd) { 502 switch (chan->request) { 503 case EDMA_REQ_NONE: 504 desc = vd2dw_edma_desc(vd); 505 if (!desc->chunks_alloc) { 506 list_del(&vd->node); 507 vchan_cookie_complete(vd); 508 } 509 510 /* Continue transferring if there are remaining chunks or issued requests. 511 */ 512 chan->status = dw_edma_start_transfer(chan) ? EDMA_ST_BUSY : EDMA_ST_IDLE; 513 break; 514 515 case EDMA_REQ_STOP: 516 list_del(&vd->node); 517 vchan_cookie_complete(vd); 518 chan->request = EDMA_REQ_NONE; 519 chan->status = EDMA_ST_IDLE; 520 break; 521 522 case EDMA_REQ_PAUSE: 523 chan->request = EDMA_REQ_NONE; 524 chan->status = EDMA_ST_PAUSE; 525 break; 526 527 default: 528 break; 529 } 530 } 531 spin_unlock_irqrestore(&chan->vc.lock, flags); 532} 533 534static void dw_edma_abort_interrupt(struct dw_edma_chan *chan) 535{ 536 struct virt_dma_desc *vd; 537 unsigned long flags; 538 539 dw_edma_v0_core_clear_abort_int(chan); 540 541 spin_lock_irqsave(&chan->vc.lock, flags); 542 vd = vchan_next_desc(&chan->vc); 543 if (vd) { 544 list_del(&vd->node); 545 vchan_cookie_complete(vd); 546 } 547 spin_unlock_irqrestore(&chan->vc.lock, flags); 548 chan->request = EDMA_REQ_NONE; 549 chan->status = EDMA_ST_IDLE; 550} 551 552static irqreturn_t dw_edma_interrupt(int irq, void *data, bool write) 553{ 554 struct dw_edma_irq *dw_irq = data; 555 struct dw_edma *dw = dw_irq->dw; 556 unsigned long total, pos, val; 557 unsigned long off; 558 u32 mask; 559 560 if (write) { 561 total = dw->wr_ch_cnt; 562 off = 0; 563 mask = dw_irq->wr_mask; 564 } else { 565 total = dw->rd_ch_cnt; 566 off = dw->wr_ch_cnt; 567 mask = dw_irq->rd_mask; 568 } 569 570 val = dw_edma_v0_core_status_done_int(dw, write ? 571 EDMA_DIR_WRITE : 572 EDMA_DIR_READ); 573 val &= mask; 574 for_each_set_bit(pos, &val, total) { 575 struct dw_edma_chan *chan = &dw->chan[pos + off]; 576 577 dw_edma_done_interrupt(chan); 578 } 579 580 val = dw_edma_v0_core_status_abort_int(dw, write ? 581 EDMA_DIR_WRITE : 582 EDMA_DIR_READ); 583 val &= mask; 584 for_each_set_bit(pos, &val, total) { 585 struct dw_edma_chan *chan = &dw->chan[pos + off]; 586 587 dw_edma_abort_interrupt(chan); 588 } 589 590 return IRQ_HANDLED; 591} 592 593static inline irqreturn_t dw_edma_interrupt_write(int irq, void *data) 594{ 595 return dw_edma_interrupt(irq, data, true); 596} 597 598static inline irqreturn_t dw_edma_interrupt_read(int irq, void *data) 599{ 600 return dw_edma_interrupt(irq, data, false); 601} 602 603static irqreturn_t dw_edma_interrupt_common(int irq, void *data) 604{ 605 dw_edma_interrupt(irq, data, true); 606 dw_edma_interrupt(irq, data, false); 607 608 return IRQ_HANDLED; 609} 610 611static int dw_edma_alloc_chan_resources(struct dma_chan *dchan) 612{ 613 struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan); 614 615 if (chan->status != EDMA_ST_IDLE) 616 return -EBUSY; 617 618 pm_runtime_get(chan->chip->dev); 619 620 return 0; 621} 622 623static void dw_edma_free_chan_resources(struct dma_chan *dchan) 624{ 625 unsigned long timeout = jiffies + msecs_to_jiffies(5000); 626 struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan); 627 int ret; 628 629 while (time_before(jiffies, timeout)) { 630 ret = dw_edma_device_terminate_all(dchan); 631 if (!ret) 632 break; 633 634 if (time_after_eq(jiffies, timeout)) 635 return; 636 637 cpu_relax(); 638 } 639 640 pm_runtime_put(chan->chip->dev); 641} 642 643static int dw_edma_channel_setup(struct dw_edma_chip *chip, bool write, 644 u32 wr_alloc, u32 rd_alloc) 645{ 646 struct dw_edma_region *dt_region; 647 struct device *dev = chip->dev; 648 struct dw_edma *dw = chip->dw; 649 struct dw_edma_chan *chan; 650 size_t ll_chunk, dt_chunk; 651 struct dw_edma_irq *irq; 652 struct dma_device *dma; 653 u32 i, j, cnt, ch_cnt; 654 u32 alloc, off_alloc; 655 int err = 0; 656 u32 pos; 657 658 ch_cnt = dw->wr_ch_cnt + dw->rd_ch_cnt; 659 ll_chunk = dw->ll_region.sz; 660 dt_chunk = dw->dt_region.sz; 661 662 /* Calculate linked list chunk for each channel */ 663 ll_chunk /= roundup_pow_of_two(ch_cnt); 664 665 /* Calculate linked list chunk for each channel */ 666 dt_chunk /= roundup_pow_of_two(ch_cnt); 667 668 if (write) { 669 i = 0; 670 cnt = dw->wr_ch_cnt; 671 dma = &dw->wr_edma; 672 alloc = wr_alloc; 673 off_alloc = 0; 674 } else { 675 i = dw->wr_ch_cnt; 676 cnt = dw->rd_ch_cnt; 677 dma = &dw->rd_edma; 678 alloc = rd_alloc; 679 off_alloc = wr_alloc; 680 } 681 682 INIT_LIST_HEAD(&dma->channels); 683 for (j = 0; (alloc || dw->nr_irqs == 1) && j < cnt; j++, i++) { 684 chan = &dw->chan[i]; 685 686 dt_region = devm_kzalloc(dev, sizeof(*dt_region), GFP_KERNEL); 687 if (!dt_region) 688 return -ENOMEM; 689 690 chan->vc.chan.private = dt_region; 691 692 chan->chip = chip; 693 chan->id = j; 694 chan->dir = write ? EDMA_DIR_WRITE : EDMA_DIR_READ; 695 chan->configured = false; 696 chan->request = EDMA_REQ_NONE; 697 chan->status = EDMA_ST_IDLE; 698 699 chan->ll_off = (ll_chunk * i); 700 chan->ll_max = (ll_chunk / EDMA_LL_SZ) - 1; 701 702 chan->dt_off = (dt_chunk * i); 703 704 dev_vdbg(dev, "L. List:\tChannel %s[%u] off=0x%.8lx, max_cnt=%u\n", 705 write ? "write" : "read", j, 706 chan->ll_off, chan->ll_max); 707 708 if (dw->nr_irqs == 1) 709 pos = 0; 710 else 711 pos = off_alloc + (j % alloc); 712 713 irq = &dw->irq[pos]; 714 715 if (write) 716 irq->wr_mask |= BIT(j); 717 else 718 irq->rd_mask |= BIT(j); 719 720 irq->dw = dw; 721 memcpy(&chan->msi, &irq->msi, sizeof(chan->msi)); 722 723 dev_vdbg(dev, "MSI:\t\tChannel %s[%u] addr=0x%.8x%.8x, data=0x%.8x\n", 724 write ? "write" : "read", j, 725 chan->msi.address_hi, chan->msi.address_lo, 726 chan->msi.data); 727 728 chan->vc.desc_free = vchan_free_desc; 729 vchan_init(&chan->vc, dma); 730 731 dt_region->paddr = dw->dt_region.paddr + chan->dt_off; 732 dt_region->vaddr = dw->dt_region.vaddr + chan->dt_off; 733 dt_region->sz = dt_chunk; 734 735 dev_vdbg(dev, "Data:\tChannel %s[%u] off=0x%.8lx\n", 736 write ? "write" : "read", j, chan->dt_off); 737 738 dw_edma_v0_core_device_config(chan); 739 } 740 741 /* Set DMA channel capabilities */ 742 dma_cap_zero(dma->cap_mask); 743 dma_cap_set(DMA_SLAVE, dma->cap_mask); 744 dma_cap_set(DMA_CYCLIC, dma->cap_mask); 745 dma_cap_set(DMA_PRIVATE, dma->cap_mask); 746 dma->directions = BIT(write ? DMA_DEV_TO_MEM : DMA_MEM_TO_DEV); 747 dma->src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_4_BYTES); 748 dma->dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_4_BYTES); 749 dma->residue_granularity = DMA_RESIDUE_GRANULARITY_DESCRIPTOR; 750 dma->chancnt = cnt; 751 752 /* Set DMA channel callbacks */ 753 dma->dev = chip->dev; 754 dma->device_alloc_chan_resources = dw_edma_alloc_chan_resources; 755 dma->device_free_chan_resources = dw_edma_free_chan_resources; 756 dma->device_config = dw_edma_device_config; 757 dma->device_pause = dw_edma_device_pause; 758 dma->device_resume = dw_edma_device_resume; 759 dma->device_terminate_all = dw_edma_device_terminate_all; 760 dma->device_issue_pending = dw_edma_device_issue_pending; 761 dma->device_tx_status = dw_edma_device_tx_status; 762 dma->device_prep_slave_sg = dw_edma_device_prep_slave_sg; 763 dma->device_prep_dma_cyclic = dw_edma_device_prep_dma_cyclic; 764 765 dma_set_max_seg_size(dma->dev, U32_MAX); 766 767 /* Register DMA device */ 768 err = dma_async_device_register(dma); 769 770 return err; 771} 772 773static inline void dw_edma_dec_irq_alloc(int *nr_irqs, u32 *alloc, u16 cnt) 774{ 775 if (*nr_irqs && *alloc < cnt) { 776 (*alloc)++; 777 (*nr_irqs)--; 778 } 779} 780 781static inline void dw_edma_add_irq_mask(u32 *mask, u32 alloc, u16 cnt) 782{ 783 while (*mask * alloc < cnt) 784 (*mask)++; 785} 786 787static int dw_edma_irq_request(struct dw_edma_chip *chip, 788 u32 *wr_alloc, u32 *rd_alloc) 789{ 790 struct device *dev = chip->dev; 791 struct dw_edma *dw = chip->dw; 792 u32 wr_mask = 1; 793 u32 rd_mask = 1; 794 int i, err = 0; 795 u32 ch_cnt; 796 int irq; 797 798 ch_cnt = dw->wr_ch_cnt + dw->rd_ch_cnt; 799 800 if (dw->nr_irqs < 1) 801 return -EINVAL; 802 803 if (dw->nr_irqs == 1) { 804 /* Common IRQ shared among all channels */ 805 irq = dw->ops->irq_vector(dev, 0); 806 err = request_irq(irq, dw_edma_interrupt_common, 807 IRQF_SHARED, dw->name, &dw->irq[0]); 808 if (err) { 809 dw->nr_irqs = 0; 810 return err; 811 } 812 813 if (irq_get_msi_desc(irq)) 814 get_cached_msi_msg(irq, &dw->irq[0].msi); 815 } else { 816 /* Distribute IRQs equally among all channels */ 817 int tmp = dw->nr_irqs; 818 819 while (tmp && (*wr_alloc + *rd_alloc) < ch_cnt) { 820 dw_edma_dec_irq_alloc(&tmp, wr_alloc, dw->wr_ch_cnt); 821 dw_edma_dec_irq_alloc(&tmp, rd_alloc, dw->rd_ch_cnt); 822 } 823 824 dw_edma_add_irq_mask(&wr_mask, *wr_alloc, dw->wr_ch_cnt); 825 dw_edma_add_irq_mask(&rd_mask, *rd_alloc, dw->rd_ch_cnt); 826 827 for (i = 0; i < (*wr_alloc + *rd_alloc); i++) { 828 irq = dw->ops->irq_vector(dev, i); 829 err = request_irq(irq, 830 i < *wr_alloc ? 831 dw_edma_interrupt_write : 832 dw_edma_interrupt_read, 833 IRQF_SHARED, dw->name, 834 &dw->irq[i]); 835 if (err) { 836 dw->nr_irqs = i; 837 return err; 838 } 839 840 if (irq_get_msi_desc(irq)) 841 get_cached_msi_msg(irq, &dw->irq[i].msi); 842 } 843 844 dw->nr_irqs = i; 845 } 846 847 return err; 848} 849 850int dw_edma_probe(struct dw_edma_chip *chip) 851{ 852 struct device *dev; 853 struct dw_edma *dw; 854 u32 wr_alloc = 0; 855 u32 rd_alloc = 0; 856 int i, err; 857 858 if (!chip) 859 return -EINVAL; 860 861 dev = chip->dev; 862 if (!dev) 863 return -EINVAL; 864 865 dw = chip->dw; 866 if (!dw || !dw->irq || !dw->ops || !dw->ops->irq_vector) 867 return -EINVAL; 868 869 raw_spin_lock_init(&dw->lock); 870 871 /* Find out how many write channels are supported by hardware */ 872 dw->wr_ch_cnt = dw_edma_v0_core_ch_count(dw, EDMA_DIR_WRITE); 873 if (!dw->wr_ch_cnt) 874 return -EINVAL; 875 876 /* Find out how many read channels are supported by hardware */ 877 dw->rd_ch_cnt = dw_edma_v0_core_ch_count(dw, EDMA_DIR_READ); 878 if (!dw->rd_ch_cnt) 879 return -EINVAL; 880 881 dev_vdbg(dev, "Channels:\twrite=%d, read=%d\n", 882 dw->wr_ch_cnt, dw->rd_ch_cnt); 883 884 /* Allocate channels */ 885 dw->chan = devm_kcalloc(dev, dw->wr_ch_cnt + dw->rd_ch_cnt, 886 sizeof(*dw->chan), GFP_KERNEL); 887 if (!dw->chan) 888 return -ENOMEM; 889 890 snprintf(dw->name, sizeof(dw->name), "dw-edma-core:%d", chip->id); 891 892 /* Disable eDMA, only to establish the ideal initial conditions */ 893 dw_edma_v0_core_off(dw); 894 895 /* Request IRQs */ 896 err = dw_edma_irq_request(chip, &wr_alloc, &rd_alloc); 897 if (err) 898 return err; 899 900 /* Setup write channels */ 901 err = dw_edma_channel_setup(chip, true, wr_alloc, rd_alloc); 902 if (err) 903 goto err_irq_free; 904 905 /* Setup read channels */ 906 err = dw_edma_channel_setup(chip, false, wr_alloc, rd_alloc); 907 if (err) 908 goto err_irq_free; 909 910 /* Power management */ 911 pm_runtime_enable(dev); 912 913 /* Turn debugfs on */ 914 dw_edma_v0_core_debugfs_on(chip); 915 916 return 0; 917 918err_irq_free: 919 for (i = (dw->nr_irqs - 1); i >= 0; i--) 920 free_irq(dw->ops->irq_vector(dev, i), &dw->irq[i]); 921 922 dw->nr_irqs = 0; 923 924 return err; 925} 926EXPORT_SYMBOL_GPL(dw_edma_probe); 927 928int dw_edma_remove(struct dw_edma_chip *chip) 929{ 930 struct dw_edma_chan *chan, *_chan; 931 struct device *dev = chip->dev; 932 struct dw_edma *dw = chip->dw; 933 int i; 934 935 /* Disable eDMA */ 936 dw_edma_v0_core_off(dw); 937 938 /* Free irqs */ 939 for (i = (dw->nr_irqs - 1); i >= 0; i--) 940 free_irq(dw->ops->irq_vector(dev, i), &dw->irq[i]); 941 942 /* Power management */ 943 pm_runtime_disable(dev); 944 945 /* Deregister eDMA device */ 946 dma_async_device_unregister(&dw->wr_edma); 947 list_for_each_entry_safe(chan, _chan, &dw->wr_edma.channels, 948 vc.chan.device_node) { 949 tasklet_kill(&chan->vc.task); 950 list_del(&chan->vc.chan.device_node); 951 } 952 953 dma_async_device_unregister(&dw->rd_edma); 954 list_for_each_entry_safe(chan, _chan, &dw->rd_edma.channels, 955 vc.chan.device_node) { 956 tasklet_kill(&chan->vc.task); 957 list_del(&chan->vc.chan.device_node); 958 } 959 960 /* Turn debugfs off */ 961 dw_edma_v0_core_debugfs_off(); 962 963 return 0; 964} 965EXPORT_SYMBOL_GPL(dw_edma_remove); 966 967MODULE_LICENSE("GPL v2"); 968MODULE_DESCRIPTION("Synopsys DesignWare eDMA controller core driver"); 969MODULE_AUTHOR("Gustavo Pimentel <gustavo.pimentel@synopsys.com>"); 970