1// SPDX-License-Identifier: GPL-2.0-only 2/* 3 * Copyright 2012 Marvell International Ltd. 4 */ 5 6#include <linux/err.h> 7#include <linux/module.h> 8#include <linux/init.h> 9#include <linux/types.h> 10#include <linux/interrupt.h> 11#include <linux/dma-mapping.h> 12#include <linux/slab.h> 13#include <linux/dmaengine.h> 14#include <linux/platform_device.h> 15#include <linux/device.h> 16#include <linux/platform_data/mmp_dma.h> 17#include <linux/dmapool.h> 18#include <linux/of_device.h> 19#include <linux/of_dma.h> 20#include <linux/of.h> 21 22#include "dmaengine.h" 23 24#define DCSR 0x0000 25#define DALGN 0x00a0 26#define DINT 0x00f0 27#define DDADR 0x0200 28#define DSADR(n) (0x0204 + ((n) << 4)) 29#define DTADR(n) (0x0208 + ((n) << 4)) 30#define DCMD 0x020c 31 32#define DCSR_RUN BIT(31) /* Run Bit (read / write) */ 33#define DCSR_NODESC BIT(30) /* No-Descriptor Fetch (read / write) */ 34#define DCSR_STOPIRQEN BIT(29) /* Stop Interrupt Enable (read / write) */ 35#define DCSR_REQPEND BIT(8) /* Request Pending (read-only) */ 36#define DCSR_STOPSTATE BIT(3) /* Stop State (read-only) */ 37#define DCSR_ENDINTR BIT(2) /* End Interrupt (read / write) */ 38#define DCSR_STARTINTR BIT(1) /* Start Interrupt (read / write) */ 39#define DCSR_BUSERR BIT(0) /* Bus Error Interrupt (read / write) */ 40 41#define DCSR_EORIRQEN BIT(28) /* End of Receive Interrupt Enable (R/W) */ 42#define DCSR_EORJMPEN BIT(27) /* Jump to next descriptor on EOR */ 43#define DCSR_EORSTOPEN BIT(26) /* STOP on an EOR */ 44#define DCSR_SETCMPST BIT(25) /* Set Descriptor Compare Status */ 45#define DCSR_CLRCMPST BIT(24) /* Clear Descriptor Compare Status */ 46#define DCSR_CMPST BIT(10) /* The Descriptor Compare Status */ 47#define DCSR_EORINTR BIT(9) /* The end of Receive */ 48 49#define DRCMR(n) ((((n) < 64) ? 0x0100 : 0x1100) + (((n) & 0x3f) << 2)) 50#define DRCMR_MAPVLD BIT(7) /* Map Valid (read / write) */ 51#define DRCMR_CHLNUM 0x1f /* mask for Channel Number (read / write) */ 52 53#define DDADR_DESCADDR 0xfffffff0 /* Address of next descriptor (mask) */ 54#define DDADR_STOP BIT(0) /* Stop (read / write) */ 55 56#define DCMD_INCSRCADDR BIT(31) /* Source Address Increment Setting. */ 57#define DCMD_INCTRGADDR BIT(30) /* Target Address Increment Setting. */ 58#define DCMD_FLOWSRC BIT(29) /* Flow Control by the source. */ 59#define DCMD_FLOWTRG BIT(28) /* Flow Control by the target. */ 60#define DCMD_STARTIRQEN BIT(22) /* Start Interrupt Enable */ 61#define DCMD_ENDIRQEN BIT(21) /* End Interrupt Enable */ 62#define DCMD_ENDIAN BIT(18) /* Device Endian-ness. */ 63#define DCMD_BURST8 (1 << 16) /* 8 byte burst */ 64#define DCMD_BURST16 (2 << 16) /* 16 byte burst */ 65#define DCMD_BURST32 (3 << 16) /* 32 byte burst */ 66#define DCMD_WIDTH1 (1 << 14) /* 1 byte width */ 67#define DCMD_WIDTH2 (2 << 14) /* 2 byte width (HalfWord) */ 68#define DCMD_WIDTH4 (3 << 14) /* 4 byte width (Word) */ 69#define DCMD_LENGTH 0x01fff /* length mask (max = 8K - 1) */ 70 71#define PDMA_MAX_DESC_BYTES DCMD_LENGTH 72 73struct mmp_pdma_desc_hw { 74 u32 ddadr; /* Points to the next descriptor + flags */ 75 u32 dsadr; /* DSADR value for the current transfer */ 76 u32 dtadr; /* DTADR value for the current transfer */ 77 u32 dcmd; /* DCMD value for the current transfer */ 78} __aligned(32); 79 80struct mmp_pdma_desc_sw { 81 struct mmp_pdma_desc_hw desc; 82 struct list_head node; 83 struct list_head tx_list; 84 struct dma_async_tx_descriptor async_tx; 85}; 86 87struct mmp_pdma_phy; 88 89struct mmp_pdma_chan { 90 struct device *dev; 91 struct dma_chan chan; 92 struct dma_async_tx_descriptor desc; 93 struct mmp_pdma_phy *phy; 94 enum dma_transfer_direction dir; 95 struct dma_slave_config slave_config; 96 97 struct mmp_pdma_desc_sw *cyclic_first; /* first desc_sw if channel 98 * is in cyclic mode */ 99 100 /* channel's basic info */ 101 struct tasklet_struct tasklet; 102 u32 dcmd; 103 u32 drcmr; 104 u32 dev_addr; 105 106 /* list for desc */ 107 spinlock_t desc_lock; /* Descriptor list lock */ 108 struct list_head chain_pending; /* Link descriptors queue for pending */ 109 struct list_head chain_running; /* Link descriptors queue for running */ 110 bool idle; /* channel statue machine */ 111 bool byte_align; 112 113 struct dma_pool *desc_pool; /* Descriptors pool */ 114}; 115 116struct mmp_pdma_phy { 117 int idx; 118 void __iomem *base; 119 struct mmp_pdma_chan *vchan; 120}; 121 122struct mmp_pdma_device { 123 int dma_channels; 124 void __iomem *base; 125 struct device *dev; 126 struct dma_device device; 127 struct mmp_pdma_phy *phy; 128 spinlock_t phy_lock; /* protect alloc/free phy channels */ 129}; 130 131#define tx_to_mmp_pdma_desc(tx) \ 132 container_of(tx, struct mmp_pdma_desc_sw, async_tx) 133#define to_mmp_pdma_desc(lh) \ 134 container_of(lh, struct mmp_pdma_desc_sw, node) 135#define to_mmp_pdma_chan(dchan) \ 136 container_of(dchan, struct mmp_pdma_chan, chan) 137#define to_mmp_pdma_dev(dmadev) \ 138 container_of(dmadev, struct mmp_pdma_device, device) 139 140static int mmp_pdma_config_write(struct dma_chan *dchan, 141 struct dma_slave_config *cfg, 142 enum dma_transfer_direction direction); 143 144static void set_desc(struct mmp_pdma_phy *phy, dma_addr_t addr) 145{ 146 u32 reg = (phy->idx << 4) + DDADR; 147 148 writel(addr, phy->base + reg); 149} 150 151static void enable_chan(struct mmp_pdma_phy *phy) 152{ 153 u32 reg, dalgn; 154 155 if (!phy->vchan) 156 return; 157 158 reg = DRCMR(phy->vchan->drcmr); 159 writel(DRCMR_MAPVLD | phy->idx, phy->base + reg); 160 161 dalgn = readl(phy->base + DALGN); 162 if (phy->vchan->byte_align) 163 dalgn |= 1 << phy->idx; 164 else 165 dalgn &= ~(1 << phy->idx); 166 writel(dalgn, phy->base + DALGN); 167 168 reg = (phy->idx << 2) + DCSR; 169 writel(readl(phy->base + reg) | DCSR_RUN, phy->base + reg); 170} 171 172static void disable_chan(struct mmp_pdma_phy *phy) 173{ 174 u32 reg; 175 176 if (!phy) 177 return; 178 179 reg = (phy->idx << 2) + DCSR; 180 writel(readl(phy->base + reg) & ~DCSR_RUN, phy->base + reg); 181} 182 183static int clear_chan_irq(struct mmp_pdma_phy *phy) 184{ 185 u32 dcsr; 186 u32 dint = readl(phy->base + DINT); 187 u32 reg = (phy->idx << 2) + DCSR; 188 189 if (!(dint & BIT(phy->idx))) 190 return -EAGAIN; 191 192 /* clear irq */ 193 dcsr = readl(phy->base + reg); 194 writel(dcsr, phy->base + reg); 195 if ((dcsr & DCSR_BUSERR) && (phy->vchan)) 196 dev_warn(phy->vchan->dev, "DCSR_BUSERR\n"); 197 198 return 0; 199} 200 201static irqreturn_t mmp_pdma_chan_handler(int irq, void *dev_id) 202{ 203 struct mmp_pdma_phy *phy = dev_id; 204 205 if (clear_chan_irq(phy) != 0) 206 return IRQ_NONE; 207 208 tasklet_schedule(&phy->vchan->tasklet); 209 return IRQ_HANDLED; 210} 211 212static irqreturn_t mmp_pdma_int_handler(int irq, void *dev_id) 213{ 214 struct mmp_pdma_device *pdev = dev_id; 215 struct mmp_pdma_phy *phy; 216 u32 dint = readl(pdev->base + DINT); 217 int i, ret; 218 int irq_num = 0; 219 220 while (dint) { 221 i = __ffs(dint); 222 /* only handle interrupts belonging to pdma driver*/ 223 if (i >= pdev->dma_channels) 224 break; 225 dint &= (dint - 1); 226 phy = &pdev->phy[i]; 227 ret = mmp_pdma_chan_handler(irq, phy); 228 if (ret == IRQ_HANDLED) 229 irq_num++; 230 } 231 232 if (irq_num) 233 return IRQ_HANDLED; 234 235 return IRQ_NONE; 236} 237 238/* lookup free phy channel as descending priority */ 239static struct mmp_pdma_phy *lookup_phy(struct mmp_pdma_chan *pchan) 240{ 241 int prio, i; 242 struct mmp_pdma_device *pdev = to_mmp_pdma_dev(pchan->chan.device); 243 struct mmp_pdma_phy *phy, *found = NULL; 244 unsigned long flags; 245 246 /* 247 * dma channel priorities 248 * ch 0 - 3, 16 - 19 <--> (0) 249 * ch 4 - 7, 20 - 23 <--> (1) 250 * ch 8 - 11, 24 - 27 <--> (2) 251 * ch 12 - 15, 28 - 31 <--> (3) 252 */ 253 254 spin_lock_irqsave(&pdev->phy_lock, flags); 255 for (prio = 0; prio <= ((pdev->dma_channels - 1) & 0xf) >> 2; prio++) { 256 for (i = 0; i < pdev->dma_channels; i++) { 257 if (prio != (i & 0xf) >> 2) 258 continue; 259 phy = &pdev->phy[i]; 260 if (!phy->vchan) { 261 phy->vchan = pchan; 262 found = phy; 263 goto out_unlock; 264 } 265 } 266 } 267 268out_unlock: 269 spin_unlock_irqrestore(&pdev->phy_lock, flags); 270 return found; 271} 272 273static void mmp_pdma_free_phy(struct mmp_pdma_chan *pchan) 274{ 275 struct mmp_pdma_device *pdev = to_mmp_pdma_dev(pchan->chan.device); 276 unsigned long flags; 277 u32 reg; 278 279 if (!pchan->phy) 280 return; 281 282 /* clear the channel mapping in DRCMR */ 283 reg = DRCMR(pchan->drcmr); 284 writel(0, pchan->phy->base + reg); 285 286 spin_lock_irqsave(&pdev->phy_lock, flags); 287 pchan->phy->vchan = NULL; 288 pchan->phy = NULL; 289 spin_unlock_irqrestore(&pdev->phy_lock, flags); 290} 291 292/* 293 * start_pending_queue - transfer any pending transactions 294 * pending list ==> running list 295 */ 296static void start_pending_queue(struct mmp_pdma_chan *chan) 297{ 298 struct mmp_pdma_desc_sw *desc; 299 300 /* still in running, irq will start the pending list */ 301 if (!chan->idle) { 302 dev_dbg(chan->dev, "DMA controller still busy\n"); 303 return; 304 } 305 306 if (list_empty(&chan->chain_pending)) { 307 /* chance to re-fetch phy channel with higher prio */ 308 mmp_pdma_free_phy(chan); 309 dev_dbg(chan->dev, "no pending list\n"); 310 return; 311 } 312 313 if (!chan->phy) { 314 chan->phy = lookup_phy(chan); 315 if (!chan->phy) { 316 dev_dbg(chan->dev, "no free dma channel\n"); 317 return; 318 } 319 } 320 321 /* 322 * pending -> running 323 * reintilize pending list 324 */ 325 desc = list_first_entry(&chan->chain_pending, 326 struct mmp_pdma_desc_sw, node); 327 list_splice_tail_init(&chan->chain_pending, &chan->chain_running); 328 329 /* 330 * Program the descriptor's address into the DMA controller, 331 * then start the DMA transaction 332 */ 333 set_desc(chan->phy, desc->async_tx.phys); 334 enable_chan(chan->phy); 335 chan->idle = false; 336} 337 338 339/* desc->tx_list ==> pending list */ 340static dma_cookie_t mmp_pdma_tx_submit(struct dma_async_tx_descriptor *tx) 341{ 342 struct mmp_pdma_chan *chan = to_mmp_pdma_chan(tx->chan); 343 struct mmp_pdma_desc_sw *desc = tx_to_mmp_pdma_desc(tx); 344 struct mmp_pdma_desc_sw *child; 345 unsigned long flags; 346 dma_cookie_t cookie = -EBUSY; 347 348 spin_lock_irqsave(&chan->desc_lock, flags); 349 350 list_for_each_entry(child, &desc->tx_list, node) { 351 cookie = dma_cookie_assign(&child->async_tx); 352 } 353 354 /* softly link to pending list - desc->tx_list ==> pending list */ 355 list_splice_tail_init(&desc->tx_list, &chan->chain_pending); 356 357 spin_unlock_irqrestore(&chan->desc_lock, flags); 358 359 return cookie; 360} 361 362static struct mmp_pdma_desc_sw * 363mmp_pdma_alloc_descriptor(struct mmp_pdma_chan *chan) 364{ 365 struct mmp_pdma_desc_sw *desc; 366 dma_addr_t pdesc; 367 368 desc = dma_pool_zalloc(chan->desc_pool, GFP_ATOMIC, &pdesc); 369 if (!desc) { 370 dev_err(chan->dev, "out of memory for link descriptor\n"); 371 return NULL; 372 } 373 374 INIT_LIST_HEAD(&desc->tx_list); 375 dma_async_tx_descriptor_init(&desc->async_tx, &chan->chan); 376 /* each desc has submit */ 377 desc->async_tx.tx_submit = mmp_pdma_tx_submit; 378 desc->async_tx.phys = pdesc; 379 380 return desc; 381} 382 383/* 384 * mmp_pdma_alloc_chan_resources - Allocate resources for DMA channel. 385 * 386 * This function will create a dma pool for descriptor allocation. 387 * Request irq only when channel is requested 388 * Return - The number of allocated descriptors. 389 */ 390 391static int mmp_pdma_alloc_chan_resources(struct dma_chan *dchan) 392{ 393 struct mmp_pdma_chan *chan = to_mmp_pdma_chan(dchan); 394 395 if (chan->desc_pool) 396 return 1; 397 398 chan->desc_pool = dma_pool_create(dev_name(&dchan->dev->device), 399 chan->dev, 400 sizeof(struct mmp_pdma_desc_sw), 401 __alignof__(struct mmp_pdma_desc_sw), 402 0); 403 if (!chan->desc_pool) { 404 dev_err(chan->dev, "unable to allocate descriptor pool\n"); 405 return -ENOMEM; 406 } 407 408 mmp_pdma_free_phy(chan); 409 chan->idle = true; 410 chan->dev_addr = 0; 411 return 1; 412} 413 414static void mmp_pdma_free_desc_list(struct mmp_pdma_chan *chan, 415 struct list_head *list) 416{ 417 struct mmp_pdma_desc_sw *desc, *_desc; 418 419 list_for_each_entry_safe(desc, _desc, list, node) { 420 list_del(&desc->node); 421 dma_pool_free(chan->desc_pool, desc, desc->async_tx.phys); 422 } 423} 424 425static void mmp_pdma_free_chan_resources(struct dma_chan *dchan) 426{ 427 struct mmp_pdma_chan *chan = to_mmp_pdma_chan(dchan); 428 unsigned long flags; 429 430 spin_lock_irqsave(&chan->desc_lock, flags); 431 mmp_pdma_free_desc_list(chan, &chan->chain_pending); 432 mmp_pdma_free_desc_list(chan, &chan->chain_running); 433 spin_unlock_irqrestore(&chan->desc_lock, flags); 434 435 dma_pool_destroy(chan->desc_pool); 436 chan->desc_pool = NULL; 437 chan->idle = true; 438 chan->dev_addr = 0; 439 mmp_pdma_free_phy(chan); 440 return; 441} 442 443static struct dma_async_tx_descriptor * 444mmp_pdma_prep_memcpy(struct dma_chan *dchan, 445 dma_addr_t dma_dst, dma_addr_t dma_src, 446 size_t len, unsigned long flags) 447{ 448 struct mmp_pdma_chan *chan; 449 struct mmp_pdma_desc_sw *first = NULL, *prev = NULL, *new; 450 size_t copy = 0; 451 452 if (!dchan) 453 return NULL; 454 455 if (!len) 456 return NULL; 457 458 chan = to_mmp_pdma_chan(dchan); 459 chan->byte_align = false; 460 461 if (!chan->dir) { 462 chan->dir = DMA_MEM_TO_MEM; 463 chan->dcmd = DCMD_INCTRGADDR | DCMD_INCSRCADDR; 464 chan->dcmd |= DCMD_BURST32; 465 } 466 467 do { 468 /* Allocate the link descriptor from DMA pool */ 469 new = mmp_pdma_alloc_descriptor(chan); 470 if (!new) { 471 dev_err(chan->dev, "no memory for desc\n"); 472 goto fail; 473 } 474 475 copy = min_t(size_t, len, PDMA_MAX_DESC_BYTES); 476 if (dma_src & 0x7 || dma_dst & 0x7) 477 chan->byte_align = true; 478 479 new->desc.dcmd = chan->dcmd | (DCMD_LENGTH & copy); 480 new->desc.dsadr = dma_src; 481 new->desc.dtadr = dma_dst; 482 483 if (!first) 484 first = new; 485 else 486 prev->desc.ddadr = new->async_tx.phys; 487 488 new->async_tx.cookie = 0; 489 async_tx_ack(&new->async_tx); 490 491 prev = new; 492 len -= copy; 493 494 if (chan->dir == DMA_MEM_TO_DEV) { 495 dma_src += copy; 496 } else if (chan->dir == DMA_DEV_TO_MEM) { 497 dma_dst += copy; 498 } else if (chan->dir == DMA_MEM_TO_MEM) { 499 dma_src += copy; 500 dma_dst += copy; 501 } 502 503 /* Insert the link descriptor to the LD ring */ 504 list_add_tail(&new->node, &first->tx_list); 505 } while (len); 506 507 first->async_tx.flags = flags; /* client is in control of this ack */ 508 first->async_tx.cookie = -EBUSY; 509 510 /* last desc and fire IRQ */ 511 new->desc.ddadr = DDADR_STOP; 512 new->desc.dcmd |= DCMD_ENDIRQEN; 513 514 chan->cyclic_first = NULL; 515 516 return &first->async_tx; 517 518fail: 519 if (first) 520 mmp_pdma_free_desc_list(chan, &first->tx_list); 521 return NULL; 522} 523 524static struct dma_async_tx_descriptor * 525mmp_pdma_prep_slave_sg(struct dma_chan *dchan, struct scatterlist *sgl, 526 unsigned int sg_len, enum dma_transfer_direction dir, 527 unsigned long flags, void *context) 528{ 529 struct mmp_pdma_chan *chan = to_mmp_pdma_chan(dchan); 530 struct mmp_pdma_desc_sw *first = NULL, *prev = NULL, *new = NULL; 531 size_t len, avail; 532 struct scatterlist *sg; 533 dma_addr_t addr; 534 int i; 535 536 if ((sgl == NULL) || (sg_len == 0)) 537 return NULL; 538 539 chan->byte_align = false; 540 541 mmp_pdma_config_write(dchan, &chan->slave_config, dir); 542 543 for_each_sg(sgl, sg, sg_len, i) { 544 addr = sg_dma_address(sg); 545 avail = sg_dma_len(sgl); 546 547 do { 548 len = min_t(size_t, avail, PDMA_MAX_DESC_BYTES); 549 if (addr & 0x7) 550 chan->byte_align = true; 551 552 /* allocate and populate the descriptor */ 553 new = mmp_pdma_alloc_descriptor(chan); 554 if (!new) { 555 dev_err(chan->dev, "no memory for desc\n"); 556 goto fail; 557 } 558 559 new->desc.dcmd = chan->dcmd | (DCMD_LENGTH & len); 560 if (dir == DMA_MEM_TO_DEV) { 561 new->desc.dsadr = addr; 562 new->desc.dtadr = chan->dev_addr; 563 } else { 564 new->desc.dsadr = chan->dev_addr; 565 new->desc.dtadr = addr; 566 } 567 568 if (!first) 569 first = new; 570 else 571 prev->desc.ddadr = new->async_tx.phys; 572 573 new->async_tx.cookie = 0; 574 async_tx_ack(&new->async_tx); 575 prev = new; 576 577 /* Insert the link descriptor to the LD ring */ 578 list_add_tail(&new->node, &first->tx_list); 579 580 /* update metadata */ 581 addr += len; 582 avail -= len; 583 } while (avail); 584 } 585 586 first->async_tx.cookie = -EBUSY; 587 first->async_tx.flags = flags; 588 589 /* last desc and fire IRQ */ 590 new->desc.ddadr = DDADR_STOP; 591 new->desc.dcmd |= DCMD_ENDIRQEN; 592 593 chan->dir = dir; 594 chan->cyclic_first = NULL; 595 596 return &first->async_tx; 597 598fail: 599 if (first) 600 mmp_pdma_free_desc_list(chan, &first->tx_list); 601 return NULL; 602} 603 604static struct dma_async_tx_descriptor * 605mmp_pdma_prep_dma_cyclic(struct dma_chan *dchan, 606 dma_addr_t buf_addr, size_t len, size_t period_len, 607 enum dma_transfer_direction direction, 608 unsigned long flags) 609{ 610 struct mmp_pdma_chan *chan; 611 struct mmp_pdma_desc_sw *first = NULL, *prev = NULL, *new; 612 dma_addr_t dma_src, dma_dst; 613 614 if (!dchan || !len || !period_len) 615 return NULL; 616 617 /* the buffer length must be a multiple of period_len */ 618 if (len % period_len != 0) 619 return NULL; 620 621 if (period_len > PDMA_MAX_DESC_BYTES) 622 return NULL; 623 624 chan = to_mmp_pdma_chan(dchan); 625 mmp_pdma_config_write(dchan, &chan->slave_config, direction); 626 627 switch (direction) { 628 case DMA_MEM_TO_DEV: 629 dma_src = buf_addr; 630 dma_dst = chan->dev_addr; 631 break; 632 case DMA_DEV_TO_MEM: 633 dma_dst = buf_addr; 634 dma_src = chan->dev_addr; 635 break; 636 default: 637 dev_err(chan->dev, "Unsupported direction for cyclic DMA\n"); 638 return NULL; 639 } 640 641 chan->dir = direction; 642 643 do { 644 /* Allocate the link descriptor from DMA pool */ 645 new = mmp_pdma_alloc_descriptor(chan); 646 if (!new) { 647 dev_err(chan->dev, "no memory for desc\n"); 648 goto fail; 649 } 650 651 new->desc.dcmd = (chan->dcmd | DCMD_ENDIRQEN | 652 (DCMD_LENGTH & period_len)); 653 new->desc.dsadr = dma_src; 654 new->desc.dtadr = dma_dst; 655 656 if (!first) 657 first = new; 658 else 659 prev->desc.ddadr = new->async_tx.phys; 660 661 new->async_tx.cookie = 0; 662 async_tx_ack(&new->async_tx); 663 664 prev = new; 665 len -= period_len; 666 667 if (chan->dir == DMA_MEM_TO_DEV) 668 dma_src += period_len; 669 else 670 dma_dst += period_len; 671 672 /* Insert the link descriptor to the LD ring */ 673 list_add_tail(&new->node, &first->tx_list); 674 } while (len); 675 676 first->async_tx.flags = flags; /* client is in control of this ack */ 677 first->async_tx.cookie = -EBUSY; 678 679 /* make the cyclic link */ 680 new->desc.ddadr = first->async_tx.phys; 681 chan->cyclic_first = first; 682 683 return &first->async_tx; 684 685fail: 686 if (first) 687 mmp_pdma_free_desc_list(chan, &first->tx_list); 688 return NULL; 689} 690 691static int mmp_pdma_config_write(struct dma_chan *dchan, 692 struct dma_slave_config *cfg, 693 enum dma_transfer_direction direction) 694{ 695 struct mmp_pdma_chan *chan = to_mmp_pdma_chan(dchan); 696 u32 maxburst = 0, addr = 0; 697 enum dma_slave_buswidth width = DMA_SLAVE_BUSWIDTH_UNDEFINED; 698 699 if (!dchan) 700 return -EINVAL; 701 702 if (direction == DMA_DEV_TO_MEM) { 703 chan->dcmd = DCMD_INCTRGADDR | DCMD_FLOWSRC; 704 maxburst = cfg->src_maxburst; 705 width = cfg->src_addr_width; 706 addr = cfg->src_addr; 707 } else if (direction == DMA_MEM_TO_DEV) { 708 chan->dcmd = DCMD_INCSRCADDR | DCMD_FLOWTRG; 709 maxburst = cfg->dst_maxburst; 710 width = cfg->dst_addr_width; 711 addr = cfg->dst_addr; 712 } 713 714 if (width == DMA_SLAVE_BUSWIDTH_1_BYTE) 715 chan->dcmd |= DCMD_WIDTH1; 716 else if (width == DMA_SLAVE_BUSWIDTH_2_BYTES) 717 chan->dcmd |= DCMD_WIDTH2; 718 else if (width == DMA_SLAVE_BUSWIDTH_4_BYTES) 719 chan->dcmd |= DCMD_WIDTH4; 720 721 if (maxburst == 8) 722 chan->dcmd |= DCMD_BURST8; 723 else if (maxburst == 16) 724 chan->dcmd |= DCMD_BURST16; 725 else if (maxburst == 32) 726 chan->dcmd |= DCMD_BURST32; 727 728 chan->dir = direction; 729 chan->dev_addr = addr; 730 731 return 0; 732} 733 734static int mmp_pdma_config(struct dma_chan *dchan, 735 struct dma_slave_config *cfg) 736{ 737 struct mmp_pdma_chan *chan = to_mmp_pdma_chan(dchan); 738 739 memcpy(&chan->slave_config, cfg, sizeof(*cfg)); 740 return 0; 741} 742 743static int mmp_pdma_terminate_all(struct dma_chan *dchan) 744{ 745 struct mmp_pdma_chan *chan = to_mmp_pdma_chan(dchan); 746 unsigned long flags; 747 748 if (!dchan) 749 return -EINVAL; 750 751 disable_chan(chan->phy); 752 mmp_pdma_free_phy(chan); 753 spin_lock_irqsave(&chan->desc_lock, flags); 754 mmp_pdma_free_desc_list(chan, &chan->chain_pending); 755 mmp_pdma_free_desc_list(chan, &chan->chain_running); 756 spin_unlock_irqrestore(&chan->desc_lock, flags); 757 chan->idle = true; 758 759 return 0; 760} 761 762static unsigned int mmp_pdma_residue(struct mmp_pdma_chan *chan, 763 dma_cookie_t cookie) 764{ 765 struct mmp_pdma_desc_sw *sw; 766 u32 curr, residue = 0; 767 bool passed = false; 768 bool cyclic = chan->cyclic_first != NULL; 769 770 /* 771 * If the channel does not have a phy pointer anymore, it has already 772 * been completed. Therefore, its residue is 0. 773 */ 774 if (!chan->phy) 775 return 0; 776 777 if (chan->dir == DMA_DEV_TO_MEM) 778 curr = readl(chan->phy->base + DTADR(chan->phy->idx)); 779 else 780 curr = readl(chan->phy->base + DSADR(chan->phy->idx)); 781 782 list_for_each_entry(sw, &chan->chain_running, node) { 783 u32 start, end, len; 784 785 if (chan->dir == DMA_DEV_TO_MEM) 786 start = sw->desc.dtadr; 787 else 788 start = sw->desc.dsadr; 789 790 len = sw->desc.dcmd & DCMD_LENGTH; 791 end = start + len; 792 793 /* 794 * 'passed' will be latched once we found the descriptor which 795 * lies inside the boundaries of the curr pointer. All 796 * descriptors that occur in the list _after_ we found that 797 * partially handled descriptor are still to be processed and 798 * are hence added to the residual bytes counter. 799 */ 800 801 if (passed) { 802 residue += len; 803 } else if (curr >= start && curr <= end) { 804 residue += end - curr; 805 passed = true; 806 } 807 808 /* 809 * Descriptors that have the ENDIRQEN bit set mark the end of a 810 * transaction chain, and the cookie assigned with it has been 811 * returned previously from mmp_pdma_tx_submit(). 812 * 813 * In case we have multiple transactions in the running chain, 814 * and the cookie does not match the one the user asked us 815 * about, reset the state variables and start over. 816 * 817 * This logic does not apply to cyclic transactions, where all 818 * descriptors have the ENDIRQEN bit set, and for which we 819 * can't have multiple transactions on one channel anyway. 820 */ 821 if (cyclic || !(sw->desc.dcmd & DCMD_ENDIRQEN)) 822 continue; 823 824 if (sw->async_tx.cookie == cookie) { 825 return residue; 826 } else { 827 residue = 0; 828 passed = false; 829 } 830 } 831 832 /* We should only get here in case of cyclic transactions */ 833 return residue; 834} 835 836static enum dma_status mmp_pdma_tx_status(struct dma_chan *dchan, 837 dma_cookie_t cookie, 838 struct dma_tx_state *txstate) 839{ 840 struct mmp_pdma_chan *chan = to_mmp_pdma_chan(dchan); 841 enum dma_status ret; 842 843 ret = dma_cookie_status(dchan, cookie, txstate); 844 if (likely(ret != DMA_ERROR)) 845 dma_set_residue(txstate, mmp_pdma_residue(chan, cookie)); 846 847 return ret; 848} 849 850/* 851 * mmp_pdma_issue_pending - Issue the DMA start command 852 * pending list ==> running list 853 */ 854static void mmp_pdma_issue_pending(struct dma_chan *dchan) 855{ 856 struct mmp_pdma_chan *chan = to_mmp_pdma_chan(dchan); 857 unsigned long flags; 858 859 spin_lock_irqsave(&chan->desc_lock, flags); 860 start_pending_queue(chan); 861 spin_unlock_irqrestore(&chan->desc_lock, flags); 862} 863 864/* 865 * dma_do_tasklet 866 * Do call back 867 * Start pending list 868 */ 869static void dma_do_tasklet(struct tasklet_struct *t) 870{ 871 struct mmp_pdma_chan *chan = from_tasklet(chan, t, tasklet); 872 struct mmp_pdma_desc_sw *desc, *_desc; 873 LIST_HEAD(chain_cleanup); 874 unsigned long flags; 875 struct dmaengine_desc_callback cb; 876 877 if (chan->cyclic_first) { 878 spin_lock_irqsave(&chan->desc_lock, flags); 879 desc = chan->cyclic_first; 880 dmaengine_desc_get_callback(&desc->async_tx, &cb); 881 spin_unlock_irqrestore(&chan->desc_lock, flags); 882 883 dmaengine_desc_callback_invoke(&cb, NULL); 884 885 return; 886 } 887 888 /* submit pending list; callback for each desc; free desc */ 889 spin_lock_irqsave(&chan->desc_lock, flags); 890 891 list_for_each_entry_safe(desc, _desc, &chan->chain_running, node) { 892 /* 893 * move the descriptors to a temporary list so we can drop 894 * the lock during the entire cleanup operation 895 */ 896 list_move(&desc->node, &chain_cleanup); 897 898 /* 899 * Look for the first list entry which has the ENDIRQEN flag 900 * set. That is the descriptor we got an interrupt for, so 901 * complete that transaction and its cookie. 902 */ 903 if (desc->desc.dcmd & DCMD_ENDIRQEN) { 904 dma_cookie_t cookie = desc->async_tx.cookie; 905 dma_cookie_complete(&desc->async_tx); 906 dev_dbg(chan->dev, "completed_cookie=%d\n", cookie); 907 break; 908 } 909 } 910 911 /* 912 * The hardware is idle and ready for more when the 913 * chain_running list is empty. 914 */ 915 chan->idle = list_empty(&chan->chain_running); 916 917 /* Start any pending transactions automatically */ 918 start_pending_queue(chan); 919 spin_unlock_irqrestore(&chan->desc_lock, flags); 920 921 /* Run the callback for each descriptor, in order */ 922 list_for_each_entry_safe(desc, _desc, &chain_cleanup, node) { 923 struct dma_async_tx_descriptor *txd = &desc->async_tx; 924 925 /* Remove from the list of transactions */ 926 list_del(&desc->node); 927 /* Run the link descriptor callback function */ 928 dmaengine_desc_get_callback(txd, &cb); 929 dmaengine_desc_callback_invoke(&cb, NULL); 930 931 dma_pool_free(chan->desc_pool, desc, txd->phys); 932 } 933} 934 935static int mmp_pdma_remove(struct platform_device *op) 936{ 937 struct mmp_pdma_device *pdev = platform_get_drvdata(op); 938 struct mmp_pdma_phy *phy; 939 int i, irq = 0, irq_num = 0; 940 941 if (op->dev.of_node) 942 of_dma_controller_free(op->dev.of_node); 943 944 for (i = 0; i < pdev->dma_channels; i++) { 945 if (platform_get_irq(op, i) > 0) 946 irq_num++; 947 } 948 949 if (irq_num != pdev->dma_channels) { 950 irq = platform_get_irq(op, 0); 951 devm_free_irq(&op->dev, irq, pdev); 952 } else { 953 for (i = 0; i < pdev->dma_channels; i++) { 954 phy = &pdev->phy[i]; 955 irq = platform_get_irq(op, i); 956 devm_free_irq(&op->dev, irq, phy); 957 } 958 } 959 960 dma_async_device_unregister(&pdev->device); 961 return 0; 962} 963 964static int mmp_pdma_chan_init(struct mmp_pdma_device *pdev, int idx, int irq) 965{ 966 struct mmp_pdma_phy *phy = &pdev->phy[idx]; 967 struct mmp_pdma_chan *chan; 968 int ret; 969 970 chan = devm_kzalloc(pdev->dev, sizeof(*chan), GFP_KERNEL); 971 if (chan == NULL) 972 return -ENOMEM; 973 974 phy->idx = idx; 975 phy->base = pdev->base; 976 977 if (irq) { 978 ret = devm_request_irq(pdev->dev, irq, mmp_pdma_chan_handler, 979 IRQF_SHARED, "pdma", phy); 980 if (ret) { 981 dev_err(pdev->dev, "channel request irq fail!\n"); 982 return ret; 983 } 984 } 985 986 spin_lock_init(&chan->desc_lock); 987 chan->dev = pdev->dev; 988 chan->chan.device = &pdev->device; 989 tasklet_setup(&chan->tasklet, dma_do_tasklet); 990 INIT_LIST_HEAD(&chan->chain_pending); 991 INIT_LIST_HEAD(&chan->chain_running); 992 993 /* register virt channel to dma engine */ 994 list_add_tail(&chan->chan.device_node, &pdev->device.channels); 995 996 return 0; 997} 998 999static const struct of_device_id mmp_pdma_dt_ids[] = { 1000 { .compatible = "marvell,pdma-1.0", }, 1001 {} 1002}; 1003MODULE_DEVICE_TABLE(of, mmp_pdma_dt_ids); 1004 1005static struct dma_chan *mmp_pdma_dma_xlate(struct of_phandle_args *dma_spec, 1006 struct of_dma *ofdma) 1007{ 1008 struct mmp_pdma_device *d = ofdma->of_dma_data; 1009 struct dma_chan *chan; 1010 1011 chan = dma_get_any_slave_channel(&d->device); 1012 if (!chan) 1013 return NULL; 1014 1015 to_mmp_pdma_chan(chan)->drcmr = dma_spec->args[0]; 1016 1017 return chan; 1018} 1019 1020static int mmp_pdma_probe(struct platform_device *op) 1021{ 1022 struct mmp_pdma_device *pdev; 1023 const struct of_device_id *of_id; 1024 struct mmp_dma_platdata *pdata = dev_get_platdata(&op->dev); 1025 int i, ret, irq = 0; 1026 int dma_channels = 0, irq_num = 0; 1027 const enum dma_slave_buswidth widths = 1028 DMA_SLAVE_BUSWIDTH_1_BYTE | DMA_SLAVE_BUSWIDTH_2_BYTES | 1029 DMA_SLAVE_BUSWIDTH_4_BYTES; 1030 1031 pdev = devm_kzalloc(&op->dev, sizeof(*pdev), GFP_KERNEL); 1032 if (!pdev) 1033 return -ENOMEM; 1034 1035 pdev->dev = &op->dev; 1036 1037 spin_lock_init(&pdev->phy_lock); 1038 1039 pdev->base = devm_platform_ioremap_resource(op, 0); 1040 if (IS_ERR(pdev->base)) 1041 return PTR_ERR(pdev->base); 1042 1043 of_id = of_match_device(mmp_pdma_dt_ids, pdev->dev); 1044 if (of_id) { 1045 /* Parse new and deprecated dma-channels properties */ 1046 if (of_property_read_u32(pdev->dev->of_node, "dma-channels", 1047 &dma_channels)) 1048 of_property_read_u32(pdev->dev->of_node, "#dma-channels", 1049 &dma_channels); 1050 } else if (pdata && pdata->dma_channels) { 1051 dma_channels = pdata->dma_channels; 1052 } else { 1053 dma_channels = 32; /* default 32 channel */ 1054 } 1055 pdev->dma_channels = dma_channels; 1056 1057 for (i = 0; i < dma_channels; i++) { 1058 if (platform_get_irq_optional(op, i) > 0) 1059 irq_num++; 1060 } 1061 1062 pdev->phy = devm_kcalloc(pdev->dev, dma_channels, sizeof(*pdev->phy), 1063 GFP_KERNEL); 1064 if (pdev->phy == NULL) 1065 return -ENOMEM; 1066 1067 INIT_LIST_HEAD(&pdev->device.channels); 1068 1069 if (irq_num != dma_channels) { 1070 /* all chan share one irq, demux inside */ 1071 irq = platform_get_irq(op, 0); 1072 ret = devm_request_irq(pdev->dev, irq, mmp_pdma_int_handler, 1073 IRQF_SHARED, "pdma", pdev); 1074 if (ret) 1075 return ret; 1076 } 1077 1078 for (i = 0; i < dma_channels; i++) { 1079 irq = (irq_num != dma_channels) ? 0 : platform_get_irq(op, i); 1080 ret = mmp_pdma_chan_init(pdev, i, irq); 1081 if (ret) 1082 return ret; 1083 } 1084 1085 dma_cap_set(DMA_SLAVE, pdev->device.cap_mask); 1086 dma_cap_set(DMA_MEMCPY, pdev->device.cap_mask); 1087 dma_cap_set(DMA_CYCLIC, pdev->device.cap_mask); 1088 dma_cap_set(DMA_PRIVATE, pdev->device.cap_mask); 1089 pdev->device.dev = &op->dev; 1090 pdev->device.device_alloc_chan_resources = mmp_pdma_alloc_chan_resources; 1091 pdev->device.device_free_chan_resources = mmp_pdma_free_chan_resources; 1092 pdev->device.device_tx_status = mmp_pdma_tx_status; 1093 pdev->device.device_prep_dma_memcpy = mmp_pdma_prep_memcpy; 1094 pdev->device.device_prep_slave_sg = mmp_pdma_prep_slave_sg; 1095 pdev->device.device_prep_dma_cyclic = mmp_pdma_prep_dma_cyclic; 1096 pdev->device.device_issue_pending = mmp_pdma_issue_pending; 1097 pdev->device.device_config = mmp_pdma_config; 1098 pdev->device.device_terminate_all = mmp_pdma_terminate_all; 1099 pdev->device.copy_align = DMAENGINE_ALIGN_8_BYTES; 1100 pdev->device.src_addr_widths = widths; 1101 pdev->device.dst_addr_widths = widths; 1102 pdev->device.directions = BIT(DMA_MEM_TO_DEV) | BIT(DMA_DEV_TO_MEM); 1103 pdev->device.residue_granularity = DMA_RESIDUE_GRANULARITY_DESCRIPTOR; 1104 1105 if (pdev->dev->coherent_dma_mask) 1106 dma_set_mask(pdev->dev, pdev->dev->coherent_dma_mask); 1107 else 1108 dma_set_mask(pdev->dev, DMA_BIT_MASK(64)); 1109 1110 ret = dma_async_device_register(&pdev->device); 1111 if (ret) { 1112 dev_err(pdev->device.dev, "unable to register\n"); 1113 return ret; 1114 } 1115 1116 if (op->dev.of_node) { 1117 /* Device-tree DMA controller registration */ 1118 ret = of_dma_controller_register(op->dev.of_node, 1119 mmp_pdma_dma_xlate, pdev); 1120 if (ret < 0) { 1121 dev_err(&op->dev, "of_dma_controller_register failed\n"); 1122 dma_async_device_unregister(&pdev->device); 1123 return ret; 1124 } 1125 } 1126 1127 platform_set_drvdata(op, pdev); 1128 dev_info(pdev->device.dev, "initialized %d channels\n", dma_channels); 1129 return 0; 1130} 1131 1132static const struct platform_device_id mmp_pdma_id_table[] = { 1133 { "mmp-pdma", }, 1134 { }, 1135}; 1136 1137static struct platform_driver mmp_pdma_driver = { 1138 .driver = { 1139 .name = "mmp-pdma", 1140 .of_match_table = mmp_pdma_dt_ids, 1141 }, 1142 .id_table = mmp_pdma_id_table, 1143 .probe = mmp_pdma_probe, 1144 .remove = mmp_pdma_remove, 1145}; 1146 1147module_platform_driver(mmp_pdma_driver); 1148 1149MODULE_DESCRIPTION("MARVELL MMP Peripheral DMA Driver"); 1150MODULE_AUTHOR("Marvell International Ltd."); 1151MODULE_LICENSE("GPL v2"); 1152