1// SPDX-License-Identifier: GPL-2.0-only 2/* 3 * Copyright 2012 Marvell International Ltd. 4 */ 5 6#include <linux/err.h> 7#include <linux/module.h> 8#include <linux/init.h> 9#include <linux/types.h> 10#include <linux/interrupt.h> 11#include <linux/dma-mapping.h> 12#include <linux/slab.h> 13#include <linux/dmaengine.h> 14#include <linux/platform_device.h> 15#include <linux/device.h> 16#include <linux/platform_data/mmp_dma.h> 17#include <linux/dmapool.h> 18#include <linux/of_device.h> 19#include <linux/of_dma.h> 20#include <linux/of.h> 21#include <linux/dma/mmp-pdma.h> 22 23#include "dmaengine.h" 24 25#define DCSR 0x0000 26#define DALGN 0x00a0 27#define DINT 0x00f0 28#define DDADR 0x0200 29#define DSADR(n) (0x0204 + ((n) << 4)) 30#define DTADR(n) (0x0208 + ((n) << 4)) 31#define DCMD 0x020c 32 33#define DCSR_RUN BIT(31) /* Run Bit (read / write) */ 34#define DCSR_NODESC BIT(30) /* No-Descriptor Fetch (read / write) */ 35#define DCSR_STOPIRQEN BIT(29) /* Stop Interrupt Enable (read / write) */ 36#define DCSR_REQPEND BIT(8) /* Request Pending (read-only) */ 37#define DCSR_STOPSTATE BIT(3) /* Stop State (read-only) */ 38#define DCSR_ENDINTR BIT(2) /* End Interrupt (read / write) */ 39#define DCSR_STARTINTR BIT(1) /* Start Interrupt (read / write) */ 40#define DCSR_BUSERR BIT(0) /* Bus Error Interrupt (read / write) */ 41 42#define DCSR_EORIRQEN BIT(28) /* End of Receive Interrupt Enable (R/W) */ 43#define DCSR_EORJMPEN BIT(27) /* Jump to next descriptor on EOR */ 44#define DCSR_EORSTOPEN BIT(26) /* STOP on an EOR */ 45#define DCSR_SETCMPST BIT(25) /* Set Descriptor Compare Status */ 46#define DCSR_CLRCMPST BIT(24) /* Clear Descriptor Compare Status */ 47#define DCSR_CMPST BIT(10) /* The Descriptor Compare Status */ 48#define DCSR_EORINTR BIT(9) /* The end of Receive */ 49 50#define DRCMR(n) ((((n) < 64) ? 0x0100 : 0x1100) + (((n) & 0x3f) << 2)) 51#define DRCMR_MAPVLD BIT(7) /* Map Valid (read / write) */ 52#define DRCMR_CHLNUM 0x1f /* mask for Channel Number (read / write) */ 53 54#define DDADR_DESCADDR 0xfffffff0 /* Address of next descriptor (mask) */ 55#define DDADR_STOP BIT(0) /* Stop (read / write) */ 56 57#define DCMD_INCSRCADDR BIT(31) /* Source Address Increment Setting. */ 58#define DCMD_INCTRGADDR BIT(30) /* Target Address Increment Setting. */ 59#define DCMD_FLOWSRC BIT(29) /* Flow Control by the source. */ 60#define DCMD_FLOWTRG BIT(28) /* Flow Control by the target. */ 61#define DCMD_STARTIRQEN BIT(22) /* Start Interrupt Enable */ 62#define DCMD_ENDIRQEN BIT(21) /* End Interrupt Enable */ 63#define DCMD_ENDIAN BIT(18) /* Device Endian-ness. */ 64#define DCMD_BURST8 (1 << 16) /* 8 byte burst */ 65#define DCMD_BURST16 (2 << 16) /* 16 byte burst */ 66#define DCMD_BURST32 (3 << 16) /* 32 byte burst */ 67#define DCMD_WIDTH1 (1 << 14) /* 1 byte width */ 68#define DCMD_WIDTH2 (2 << 14) /* 2 byte width (HalfWord) */ 69#define DCMD_WIDTH4 (3 << 14) /* 4 byte width (Word) */ 70#define DCMD_LENGTH 0x01fff /* length mask (max = 8K - 1) */ 71 72#define PDMA_MAX_DESC_BYTES DCMD_LENGTH 73 74struct mmp_pdma_desc_hw { 75 u32 ddadr; /* Points to the next descriptor + flags */ 76 u32 dsadr; /* DSADR value for the current transfer */ 77 u32 dtadr; /* DTADR value for the current transfer */ 78 u32 dcmd; /* DCMD value for the current transfer */ 79} __aligned(32); 80 81struct mmp_pdma_desc_sw { 82 struct mmp_pdma_desc_hw desc; 83 struct list_head node; 84 struct list_head tx_list; 85 struct dma_async_tx_descriptor async_tx; 86}; 87 88struct mmp_pdma_phy; 89 90struct mmp_pdma_chan { 91 struct device *dev; 92 struct dma_chan chan; 93 struct dma_async_tx_descriptor desc; 94 struct mmp_pdma_phy *phy; 95 enum dma_transfer_direction dir; 96 struct dma_slave_config slave_config; 97 98 struct mmp_pdma_desc_sw *cyclic_first; /* first desc_sw if channel 99 * is in cyclic mode */ 100 101 /* channel's basic info */ 102 struct tasklet_struct tasklet; 103 u32 dcmd; 104 u32 drcmr; 105 u32 dev_addr; 106 107 /* list for desc */ 108 spinlock_t desc_lock; /* Descriptor list lock */ 109 struct list_head chain_pending; /* Link descriptors queue for pending */ 110 struct list_head chain_running; /* Link descriptors queue for running */ 111 bool idle; /* channel statue machine */ 112 bool byte_align; 113 114 struct dma_pool *desc_pool; /* Descriptors pool */ 115}; 116 117struct mmp_pdma_phy { 118 int idx; 119 void __iomem *base; 120 struct mmp_pdma_chan *vchan; 121}; 122 123struct mmp_pdma_device { 124 int dma_channels; 125 void __iomem *base; 126 struct device *dev; 127 struct dma_device device; 128 struct mmp_pdma_phy *phy; 129 spinlock_t phy_lock; /* protect alloc/free phy channels */ 130}; 131 132#define tx_to_mmp_pdma_desc(tx) \ 133 container_of(tx, struct mmp_pdma_desc_sw, async_tx) 134#define to_mmp_pdma_desc(lh) \ 135 container_of(lh, struct mmp_pdma_desc_sw, node) 136#define to_mmp_pdma_chan(dchan) \ 137 container_of(dchan, struct mmp_pdma_chan, chan) 138#define to_mmp_pdma_dev(dmadev) \ 139 container_of(dmadev, struct mmp_pdma_device, device) 140 141static int mmp_pdma_config_write(struct dma_chan *dchan, 142 struct dma_slave_config *cfg, 143 enum dma_transfer_direction direction); 144 145static void set_desc(struct mmp_pdma_phy *phy, dma_addr_t addr) 146{ 147 u32 reg = (phy->idx << 4) + DDADR; 148 149 writel(addr, phy->base + reg); 150} 151 152static void enable_chan(struct mmp_pdma_phy *phy) 153{ 154 u32 reg, dalgn; 155 156 if (!phy->vchan) 157 return; 158 159 reg = DRCMR(phy->vchan->drcmr); 160 writel(DRCMR_MAPVLD | phy->idx, phy->base + reg); 161 162 dalgn = readl(phy->base + DALGN); 163 if (phy->vchan->byte_align) 164 dalgn |= 1 << phy->idx; 165 else 166 dalgn &= ~(1 << phy->idx); 167 writel(dalgn, phy->base + DALGN); 168 169 reg = (phy->idx << 2) + DCSR; 170 writel(readl(phy->base + reg) | DCSR_RUN, phy->base + reg); 171} 172 173static void disable_chan(struct mmp_pdma_phy *phy) 174{ 175 u32 reg; 176 177 if (!phy) 178 return; 179 180 reg = (phy->idx << 2) + DCSR; 181 writel(readl(phy->base + reg) & ~DCSR_RUN, phy->base + reg); 182} 183 184static int clear_chan_irq(struct mmp_pdma_phy *phy) 185{ 186 u32 dcsr; 187 u32 dint = readl(phy->base + DINT); 188 u32 reg = (phy->idx << 2) + DCSR; 189 190 if (!(dint & BIT(phy->idx))) 191 return -EAGAIN; 192 193 /* clear irq */ 194 dcsr = readl(phy->base + reg); 195 writel(dcsr, phy->base + reg); 196 if ((dcsr & DCSR_BUSERR) && (phy->vchan)) 197 dev_warn(phy->vchan->dev, "DCSR_BUSERR\n"); 198 199 return 0; 200} 201 202static irqreturn_t mmp_pdma_chan_handler(int irq, void *dev_id) 203{ 204 struct mmp_pdma_phy *phy = dev_id; 205 206 if (clear_chan_irq(phy) != 0) 207 return IRQ_NONE; 208 209 tasklet_schedule(&phy->vchan->tasklet); 210 return IRQ_HANDLED; 211} 212 213static irqreturn_t mmp_pdma_int_handler(int irq, void *dev_id) 214{ 215 struct mmp_pdma_device *pdev = dev_id; 216 struct mmp_pdma_phy *phy; 217 u32 dint = readl(pdev->base + DINT); 218 int i, ret; 219 int irq_num = 0; 220 221 while (dint) { 222 i = __ffs(dint); 223 /* only handle interrupts belonging to pdma driver*/ 224 if (i >= pdev->dma_channels) 225 break; 226 dint &= (dint - 1); 227 phy = &pdev->phy[i]; 228 ret = mmp_pdma_chan_handler(irq, phy); 229 if (ret == IRQ_HANDLED) 230 irq_num++; 231 } 232 233 if (irq_num) 234 return IRQ_HANDLED; 235 236 return IRQ_NONE; 237} 238 239/* lookup free phy channel as descending priority */ 240static struct mmp_pdma_phy *lookup_phy(struct mmp_pdma_chan *pchan) 241{ 242 int prio, i; 243 struct mmp_pdma_device *pdev = to_mmp_pdma_dev(pchan->chan.device); 244 struct mmp_pdma_phy *phy, *found = NULL; 245 unsigned long flags; 246 247 /* 248 * dma channel priorities 249 * ch 0 - 3, 16 - 19 <--> (0) 250 * ch 4 - 7, 20 - 23 <--> (1) 251 * ch 8 - 11, 24 - 27 <--> (2) 252 * ch 12 - 15, 28 - 31 <--> (3) 253 */ 254 255 spin_lock_irqsave(&pdev->phy_lock, flags); 256 for (prio = 0; prio <= ((pdev->dma_channels - 1) & 0xf) >> 2; prio++) { 257 for (i = 0; i < pdev->dma_channels; i++) { 258 if (prio != (i & 0xf) >> 2) 259 continue; 260 phy = &pdev->phy[i]; 261 if (!phy->vchan) { 262 phy->vchan = pchan; 263 found = phy; 264 goto out_unlock; 265 } 266 } 267 } 268 269out_unlock: 270 spin_unlock_irqrestore(&pdev->phy_lock, flags); 271 return found; 272} 273 274static void mmp_pdma_free_phy(struct mmp_pdma_chan *pchan) 275{ 276 struct mmp_pdma_device *pdev = to_mmp_pdma_dev(pchan->chan.device); 277 unsigned long flags; 278 u32 reg; 279 280 if (!pchan->phy) 281 return; 282 283 /* clear the channel mapping in DRCMR */ 284 reg = DRCMR(pchan->drcmr); 285 writel(0, pchan->phy->base + reg); 286 287 spin_lock_irqsave(&pdev->phy_lock, flags); 288 pchan->phy->vchan = NULL; 289 pchan->phy = NULL; 290 spin_unlock_irqrestore(&pdev->phy_lock, flags); 291} 292 293/* 294 * start_pending_queue - transfer any pending transactions 295 * pending list ==> running list 296 */ 297static void start_pending_queue(struct mmp_pdma_chan *chan) 298{ 299 struct mmp_pdma_desc_sw *desc; 300 301 /* still in running, irq will start the pending list */ 302 if (!chan->idle) { 303 dev_dbg(chan->dev, "DMA controller still busy\n"); 304 return; 305 } 306 307 if (list_empty(&chan->chain_pending)) { 308 /* chance to re-fetch phy channel with higher prio */ 309 mmp_pdma_free_phy(chan); 310 dev_dbg(chan->dev, "no pending list\n"); 311 return; 312 } 313 314 if (!chan->phy) { 315 chan->phy = lookup_phy(chan); 316 if (!chan->phy) { 317 dev_dbg(chan->dev, "no free dma channel\n"); 318 return; 319 } 320 } 321 322 /* 323 * pending -> running 324 * reintilize pending list 325 */ 326 desc = list_first_entry(&chan->chain_pending, 327 struct mmp_pdma_desc_sw, node); 328 list_splice_tail_init(&chan->chain_pending, &chan->chain_running); 329 330 /* 331 * Program the descriptor's address into the DMA controller, 332 * then start the DMA transaction 333 */ 334 set_desc(chan->phy, desc->async_tx.phys); 335 enable_chan(chan->phy); 336 chan->idle = false; 337} 338 339 340/* desc->tx_list ==> pending list */ 341static dma_cookie_t mmp_pdma_tx_submit(struct dma_async_tx_descriptor *tx) 342{ 343 struct mmp_pdma_chan *chan = to_mmp_pdma_chan(tx->chan); 344 struct mmp_pdma_desc_sw *desc = tx_to_mmp_pdma_desc(tx); 345 struct mmp_pdma_desc_sw *child; 346 unsigned long flags; 347 dma_cookie_t cookie = -EBUSY; 348 349 spin_lock_irqsave(&chan->desc_lock, flags); 350 351 list_for_each_entry(child, &desc->tx_list, node) { 352 cookie = dma_cookie_assign(&child->async_tx); 353 } 354 355 /* softly link to pending list - desc->tx_list ==> pending list */ 356 list_splice_tail_init(&desc->tx_list, &chan->chain_pending); 357 358 spin_unlock_irqrestore(&chan->desc_lock, flags); 359 360 return cookie; 361} 362 363static struct mmp_pdma_desc_sw * 364mmp_pdma_alloc_descriptor(struct mmp_pdma_chan *chan) 365{ 366 struct mmp_pdma_desc_sw *desc; 367 dma_addr_t pdesc; 368 369 desc = dma_pool_zalloc(chan->desc_pool, GFP_ATOMIC, &pdesc); 370 if (!desc) { 371 dev_err(chan->dev, "out of memory for link descriptor\n"); 372 return NULL; 373 } 374 375 INIT_LIST_HEAD(&desc->tx_list); 376 dma_async_tx_descriptor_init(&desc->async_tx, &chan->chan); 377 /* each desc has submit */ 378 desc->async_tx.tx_submit = mmp_pdma_tx_submit; 379 desc->async_tx.phys = pdesc; 380 381 return desc; 382} 383 384/* 385 * mmp_pdma_alloc_chan_resources - Allocate resources for DMA channel. 386 * 387 * This function will create a dma pool for descriptor allocation. 388 * Request irq only when channel is requested 389 * Return - The number of allocated descriptors. 390 */ 391 392static int mmp_pdma_alloc_chan_resources(struct dma_chan *dchan) 393{ 394 struct mmp_pdma_chan *chan = to_mmp_pdma_chan(dchan); 395 396 if (chan->desc_pool) 397 return 1; 398 399 chan->desc_pool = dma_pool_create(dev_name(&dchan->dev->device), 400 chan->dev, 401 sizeof(struct mmp_pdma_desc_sw), 402 __alignof__(struct mmp_pdma_desc_sw), 403 0); 404 if (!chan->desc_pool) { 405 dev_err(chan->dev, "unable to allocate descriptor pool\n"); 406 return -ENOMEM; 407 } 408 409 mmp_pdma_free_phy(chan); 410 chan->idle = true; 411 chan->dev_addr = 0; 412 return 1; 413} 414 415static void mmp_pdma_free_desc_list(struct mmp_pdma_chan *chan, 416 struct list_head *list) 417{ 418 struct mmp_pdma_desc_sw *desc, *_desc; 419 420 list_for_each_entry_safe(desc, _desc, list, node) { 421 list_del(&desc->node); 422 dma_pool_free(chan->desc_pool, desc, desc->async_tx.phys); 423 } 424} 425 426static void mmp_pdma_free_chan_resources(struct dma_chan *dchan) 427{ 428 struct mmp_pdma_chan *chan = to_mmp_pdma_chan(dchan); 429 unsigned long flags; 430 431 spin_lock_irqsave(&chan->desc_lock, flags); 432 mmp_pdma_free_desc_list(chan, &chan->chain_pending); 433 mmp_pdma_free_desc_list(chan, &chan->chain_running); 434 spin_unlock_irqrestore(&chan->desc_lock, flags); 435 436 dma_pool_destroy(chan->desc_pool); 437 chan->desc_pool = NULL; 438 chan->idle = true; 439 chan->dev_addr = 0; 440 mmp_pdma_free_phy(chan); 441 return; 442} 443 444static struct dma_async_tx_descriptor * 445mmp_pdma_prep_memcpy(struct dma_chan *dchan, 446 dma_addr_t dma_dst, dma_addr_t dma_src, 447 size_t len, unsigned long flags) 448{ 449 struct mmp_pdma_chan *chan; 450 struct mmp_pdma_desc_sw *first = NULL, *prev = NULL, *new; 451 size_t copy = 0; 452 453 if (!dchan) 454 return NULL; 455 456 if (!len) 457 return NULL; 458 459 chan = to_mmp_pdma_chan(dchan); 460 chan->byte_align = false; 461 462 if (!chan->dir) { 463 chan->dir = DMA_MEM_TO_MEM; 464 chan->dcmd = DCMD_INCTRGADDR | DCMD_INCSRCADDR; 465 chan->dcmd |= DCMD_BURST32; 466 } 467 468 do { 469 /* Allocate the link descriptor from DMA pool */ 470 new = mmp_pdma_alloc_descriptor(chan); 471 if (!new) { 472 dev_err(chan->dev, "no memory for desc\n"); 473 goto fail; 474 } 475 476 copy = min_t(size_t, len, PDMA_MAX_DESC_BYTES); 477 if (dma_src & 0x7 || dma_dst & 0x7) 478 chan->byte_align = true; 479 480 new->desc.dcmd = chan->dcmd | (DCMD_LENGTH & copy); 481 new->desc.dsadr = dma_src; 482 new->desc.dtadr = dma_dst; 483 484 if (!first) 485 first = new; 486 else 487 prev->desc.ddadr = new->async_tx.phys; 488 489 new->async_tx.cookie = 0; 490 async_tx_ack(&new->async_tx); 491 492 prev = new; 493 len -= copy; 494 495 if (chan->dir == DMA_MEM_TO_DEV) { 496 dma_src += copy; 497 } else if (chan->dir == DMA_DEV_TO_MEM) { 498 dma_dst += copy; 499 } else if (chan->dir == DMA_MEM_TO_MEM) { 500 dma_src += copy; 501 dma_dst += copy; 502 } 503 504 /* Insert the link descriptor to the LD ring */ 505 list_add_tail(&new->node, &first->tx_list); 506 } while (len); 507 508 first->async_tx.flags = flags; /* client is in control of this ack */ 509 first->async_tx.cookie = -EBUSY; 510 511 /* last desc and fire IRQ */ 512 new->desc.ddadr = DDADR_STOP; 513 new->desc.dcmd |= DCMD_ENDIRQEN; 514 515 chan->cyclic_first = NULL; 516 517 return &first->async_tx; 518 519fail: 520 if (first) 521 mmp_pdma_free_desc_list(chan, &first->tx_list); 522 return NULL; 523} 524 525static struct dma_async_tx_descriptor * 526mmp_pdma_prep_slave_sg(struct dma_chan *dchan, struct scatterlist *sgl, 527 unsigned int sg_len, enum dma_transfer_direction dir, 528 unsigned long flags, void *context) 529{ 530 struct mmp_pdma_chan *chan = to_mmp_pdma_chan(dchan); 531 struct mmp_pdma_desc_sw *first = NULL, *prev = NULL, *new = NULL; 532 size_t len, avail; 533 struct scatterlist *sg; 534 dma_addr_t addr; 535 int i; 536 537 if ((sgl == NULL) || (sg_len == 0)) 538 return NULL; 539 540 chan->byte_align = false; 541 542 mmp_pdma_config_write(dchan, &chan->slave_config, dir); 543 544 for_each_sg(sgl, sg, sg_len, i) { 545 addr = sg_dma_address(sg); 546 avail = sg_dma_len(sgl); 547 548 do { 549 len = min_t(size_t, avail, PDMA_MAX_DESC_BYTES); 550 if (addr & 0x7) 551 chan->byte_align = true; 552 553 /* allocate and populate the descriptor */ 554 new = mmp_pdma_alloc_descriptor(chan); 555 if (!new) { 556 dev_err(chan->dev, "no memory for desc\n"); 557 goto fail; 558 } 559 560 new->desc.dcmd = chan->dcmd | (DCMD_LENGTH & len); 561 if (dir == DMA_MEM_TO_DEV) { 562 new->desc.dsadr = addr; 563 new->desc.dtadr = chan->dev_addr; 564 } else { 565 new->desc.dsadr = chan->dev_addr; 566 new->desc.dtadr = addr; 567 } 568 569 if (!first) 570 first = new; 571 else 572 prev->desc.ddadr = new->async_tx.phys; 573 574 new->async_tx.cookie = 0; 575 async_tx_ack(&new->async_tx); 576 prev = new; 577 578 /* Insert the link descriptor to the LD ring */ 579 list_add_tail(&new->node, &first->tx_list); 580 581 /* update metadata */ 582 addr += len; 583 avail -= len; 584 } while (avail); 585 } 586 587 first->async_tx.cookie = -EBUSY; 588 first->async_tx.flags = flags; 589 590 /* last desc and fire IRQ */ 591 new->desc.ddadr = DDADR_STOP; 592 new->desc.dcmd |= DCMD_ENDIRQEN; 593 594 chan->dir = dir; 595 chan->cyclic_first = NULL; 596 597 return &first->async_tx; 598 599fail: 600 if (first) 601 mmp_pdma_free_desc_list(chan, &first->tx_list); 602 return NULL; 603} 604 605static struct dma_async_tx_descriptor * 606mmp_pdma_prep_dma_cyclic(struct dma_chan *dchan, 607 dma_addr_t buf_addr, size_t len, size_t period_len, 608 enum dma_transfer_direction direction, 609 unsigned long flags) 610{ 611 struct mmp_pdma_chan *chan; 612 struct mmp_pdma_desc_sw *first = NULL, *prev = NULL, *new; 613 dma_addr_t dma_src, dma_dst; 614 615 if (!dchan || !len || !period_len) 616 return NULL; 617 618 /* the buffer length must be a multiple of period_len */ 619 if (len % period_len != 0) 620 return NULL; 621 622 if (period_len > PDMA_MAX_DESC_BYTES) 623 return NULL; 624 625 chan = to_mmp_pdma_chan(dchan); 626 mmp_pdma_config_write(dchan, &chan->slave_config, direction); 627 628 switch (direction) { 629 case DMA_MEM_TO_DEV: 630 dma_src = buf_addr; 631 dma_dst = chan->dev_addr; 632 break; 633 case DMA_DEV_TO_MEM: 634 dma_dst = buf_addr; 635 dma_src = chan->dev_addr; 636 break; 637 default: 638 dev_err(chan->dev, "Unsupported direction for cyclic DMA\n"); 639 return NULL; 640 } 641 642 chan->dir = direction; 643 644 do { 645 /* Allocate the link descriptor from DMA pool */ 646 new = mmp_pdma_alloc_descriptor(chan); 647 if (!new) { 648 dev_err(chan->dev, "no memory for desc\n"); 649 goto fail; 650 } 651 652 new->desc.dcmd = (chan->dcmd | DCMD_ENDIRQEN | 653 (DCMD_LENGTH & period_len)); 654 new->desc.dsadr = dma_src; 655 new->desc.dtadr = dma_dst; 656 657 if (!first) 658 first = new; 659 else 660 prev->desc.ddadr = new->async_tx.phys; 661 662 new->async_tx.cookie = 0; 663 async_tx_ack(&new->async_tx); 664 665 prev = new; 666 len -= period_len; 667 668 if (chan->dir == DMA_MEM_TO_DEV) 669 dma_src += period_len; 670 else 671 dma_dst += period_len; 672 673 /* Insert the link descriptor to the LD ring */ 674 list_add_tail(&new->node, &first->tx_list); 675 } while (len); 676 677 first->async_tx.flags = flags; /* client is in control of this ack */ 678 first->async_tx.cookie = -EBUSY; 679 680 /* make the cyclic link */ 681 new->desc.ddadr = first->async_tx.phys; 682 chan->cyclic_first = first; 683 684 return &first->async_tx; 685 686fail: 687 if (first) 688 mmp_pdma_free_desc_list(chan, &first->tx_list); 689 return NULL; 690} 691 692static int mmp_pdma_config_write(struct dma_chan *dchan, 693 struct dma_slave_config *cfg, 694 enum dma_transfer_direction direction) 695{ 696 struct mmp_pdma_chan *chan = to_mmp_pdma_chan(dchan); 697 u32 maxburst = 0, addr = 0; 698 enum dma_slave_buswidth width = DMA_SLAVE_BUSWIDTH_UNDEFINED; 699 700 if (!dchan) 701 return -EINVAL; 702 703 if (direction == DMA_DEV_TO_MEM) { 704 chan->dcmd = DCMD_INCTRGADDR | DCMD_FLOWSRC; 705 maxburst = cfg->src_maxburst; 706 width = cfg->src_addr_width; 707 addr = cfg->src_addr; 708 } else if (direction == DMA_MEM_TO_DEV) { 709 chan->dcmd = DCMD_INCSRCADDR | DCMD_FLOWTRG; 710 maxburst = cfg->dst_maxburst; 711 width = cfg->dst_addr_width; 712 addr = cfg->dst_addr; 713 } 714 715 if (width == DMA_SLAVE_BUSWIDTH_1_BYTE) 716 chan->dcmd |= DCMD_WIDTH1; 717 else if (width == DMA_SLAVE_BUSWIDTH_2_BYTES) 718 chan->dcmd |= DCMD_WIDTH2; 719 else if (width == DMA_SLAVE_BUSWIDTH_4_BYTES) 720 chan->dcmd |= DCMD_WIDTH4; 721 722 if (maxburst == 8) 723 chan->dcmd |= DCMD_BURST8; 724 else if (maxburst == 16) 725 chan->dcmd |= DCMD_BURST16; 726 else if (maxburst == 32) 727 chan->dcmd |= DCMD_BURST32; 728 729 chan->dir = direction; 730 chan->dev_addr = addr; 731 732 return 0; 733} 734 735static int mmp_pdma_config(struct dma_chan *dchan, 736 struct dma_slave_config *cfg) 737{ 738 struct mmp_pdma_chan *chan = to_mmp_pdma_chan(dchan); 739 740 memcpy(&chan->slave_config, cfg, sizeof(*cfg)); 741 return 0; 742} 743 744static int mmp_pdma_terminate_all(struct dma_chan *dchan) 745{ 746 struct mmp_pdma_chan *chan = to_mmp_pdma_chan(dchan); 747 unsigned long flags; 748 749 if (!dchan) 750 return -EINVAL; 751 752 disable_chan(chan->phy); 753 mmp_pdma_free_phy(chan); 754 spin_lock_irqsave(&chan->desc_lock, flags); 755 mmp_pdma_free_desc_list(chan, &chan->chain_pending); 756 mmp_pdma_free_desc_list(chan, &chan->chain_running); 757 spin_unlock_irqrestore(&chan->desc_lock, flags); 758 chan->idle = true; 759 760 return 0; 761} 762 763static unsigned int mmp_pdma_residue(struct mmp_pdma_chan *chan, 764 dma_cookie_t cookie) 765{ 766 struct mmp_pdma_desc_sw *sw; 767 u32 curr, residue = 0; 768 bool passed = false; 769 bool cyclic = chan->cyclic_first != NULL; 770 771 /* 772 * If the channel does not have a phy pointer anymore, it has already 773 * been completed. Therefore, its residue is 0. 774 */ 775 if (!chan->phy) 776 return 0; 777 778 if (chan->dir == DMA_DEV_TO_MEM) 779 curr = readl(chan->phy->base + DTADR(chan->phy->idx)); 780 else 781 curr = readl(chan->phy->base + DSADR(chan->phy->idx)); 782 783 list_for_each_entry(sw, &chan->chain_running, node) { 784 u32 start, end, len; 785 786 if (chan->dir == DMA_DEV_TO_MEM) 787 start = sw->desc.dtadr; 788 else 789 start = sw->desc.dsadr; 790 791 len = sw->desc.dcmd & DCMD_LENGTH; 792 end = start + len; 793 794 /* 795 * 'passed' will be latched once we found the descriptor which 796 * lies inside the boundaries of the curr pointer. All 797 * descriptors that occur in the list _after_ we found that 798 * partially handled descriptor are still to be processed and 799 * are hence added to the residual bytes counter. 800 */ 801 802 if (passed) { 803 residue += len; 804 } else if (curr >= start && curr <= end) { 805 residue += end - curr; 806 passed = true; 807 } 808 809 /* 810 * Descriptors that have the ENDIRQEN bit set mark the end of a 811 * transaction chain, and the cookie assigned with it has been 812 * returned previously from mmp_pdma_tx_submit(). 813 * 814 * In case we have multiple transactions in the running chain, 815 * and the cookie does not match the one the user asked us 816 * about, reset the state variables and start over. 817 * 818 * This logic does not apply to cyclic transactions, where all 819 * descriptors have the ENDIRQEN bit set, and for which we 820 * can't have multiple transactions on one channel anyway. 821 */ 822 if (cyclic || !(sw->desc.dcmd & DCMD_ENDIRQEN)) 823 continue; 824 825 if (sw->async_tx.cookie == cookie) { 826 return residue; 827 } else { 828 residue = 0; 829 passed = false; 830 } 831 } 832 833 /* We should only get here in case of cyclic transactions */ 834 return residue; 835} 836 837static enum dma_status mmp_pdma_tx_status(struct dma_chan *dchan, 838 dma_cookie_t cookie, 839 struct dma_tx_state *txstate) 840{ 841 struct mmp_pdma_chan *chan = to_mmp_pdma_chan(dchan); 842 enum dma_status ret; 843 844 ret = dma_cookie_status(dchan, cookie, txstate); 845 if (likely(ret != DMA_ERROR)) 846 dma_set_residue(txstate, mmp_pdma_residue(chan, cookie)); 847 848 return ret; 849} 850 851/* 852 * mmp_pdma_issue_pending - Issue the DMA start command 853 * pending list ==> running list 854 */ 855static void mmp_pdma_issue_pending(struct dma_chan *dchan) 856{ 857 struct mmp_pdma_chan *chan = to_mmp_pdma_chan(dchan); 858 unsigned long flags; 859 860 spin_lock_irqsave(&chan->desc_lock, flags); 861 start_pending_queue(chan); 862 spin_unlock_irqrestore(&chan->desc_lock, flags); 863} 864 865/* 866 * dma_do_tasklet 867 * Do call back 868 * Start pending list 869 */ 870static void dma_do_tasklet(struct tasklet_struct *t) 871{ 872 struct mmp_pdma_chan *chan = from_tasklet(chan, t, tasklet); 873 struct mmp_pdma_desc_sw *desc, *_desc; 874 LIST_HEAD(chain_cleanup); 875 unsigned long flags; 876 struct dmaengine_desc_callback cb; 877 878 if (chan->cyclic_first) { 879 spin_lock_irqsave(&chan->desc_lock, flags); 880 desc = chan->cyclic_first; 881 dmaengine_desc_get_callback(&desc->async_tx, &cb); 882 spin_unlock_irqrestore(&chan->desc_lock, flags); 883 884 dmaengine_desc_callback_invoke(&cb, NULL); 885 886 return; 887 } 888 889 /* submit pending list; callback for each desc; free desc */ 890 spin_lock_irqsave(&chan->desc_lock, flags); 891 892 list_for_each_entry_safe(desc, _desc, &chan->chain_running, node) { 893 /* 894 * move the descriptors to a temporary list so we can drop 895 * the lock during the entire cleanup operation 896 */ 897 list_move(&desc->node, &chain_cleanup); 898 899 /* 900 * Look for the first list entry which has the ENDIRQEN flag 901 * set. That is the descriptor we got an interrupt for, so 902 * complete that transaction and its cookie. 903 */ 904 if (desc->desc.dcmd & DCMD_ENDIRQEN) { 905 dma_cookie_t cookie = desc->async_tx.cookie; 906 dma_cookie_complete(&desc->async_tx); 907 dev_dbg(chan->dev, "completed_cookie=%d\n", cookie); 908 break; 909 } 910 } 911 912 /* 913 * The hardware is idle and ready for more when the 914 * chain_running list is empty. 915 */ 916 chan->idle = list_empty(&chan->chain_running); 917 918 /* Start any pending transactions automatically */ 919 start_pending_queue(chan); 920 spin_unlock_irqrestore(&chan->desc_lock, flags); 921 922 /* Run the callback for each descriptor, in order */ 923 list_for_each_entry_safe(desc, _desc, &chain_cleanup, node) { 924 struct dma_async_tx_descriptor *txd = &desc->async_tx; 925 926 /* Remove from the list of transactions */ 927 list_del(&desc->node); 928 /* Run the link descriptor callback function */ 929 dmaengine_desc_get_callback(txd, &cb); 930 dmaengine_desc_callback_invoke(&cb, NULL); 931 932 dma_pool_free(chan->desc_pool, desc, txd->phys); 933 } 934} 935 936static int mmp_pdma_remove(struct platform_device *op) 937{ 938 struct mmp_pdma_device *pdev = platform_get_drvdata(op); 939 struct mmp_pdma_phy *phy; 940 int i, irq = 0, irq_num = 0; 941 942 if (op->dev.of_node) 943 of_dma_controller_free(op->dev.of_node); 944 945 for (i = 0; i < pdev->dma_channels; i++) { 946 if (platform_get_irq(op, i) > 0) 947 irq_num++; 948 } 949 950 if (irq_num != pdev->dma_channels) { 951 irq = platform_get_irq(op, 0); 952 devm_free_irq(&op->dev, irq, pdev); 953 } else { 954 for (i = 0; i < pdev->dma_channels; i++) { 955 phy = &pdev->phy[i]; 956 irq = platform_get_irq(op, i); 957 devm_free_irq(&op->dev, irq, phy); 958 } 959 } 960 961 dma_async_device_unregister(&pdev->device); 962 return 0; 963} 964 965static int mmp_pdma_chan_init(struct mmp_pdma_device *pdev, int idx, int irq) 966{ 967 struct mmp_pdma_phy *phy = &pdev->phy[idx]; 968 struct mmp_pdma_chan *chan; 969 int ret; 970 971 chan = devm_kzalloc(pdev->dev, sizeof(*chan), GFP_KERNEL); 972 if (chan == NULL) 973 return -ENOMEM; 974 975 phy->idx = idx; 976 phy->base = pdev->base; 977 978 if (irq) { 979 ret = devm_request_irq(pdev->dev, irq, mmp_pdma_chan_handler, 980 IRQF_SHARED, "pdma", phy); 981 if (ret) { 982 dev_err(pdev->dev, "channel request irq fail!\n"); 983 return ret; 984 } 985 } 986 987 spin_lock_init(&chan->desc_lock); 988 chan->dev = pdev->dev; 989 chan->chan.device = &pdev->device; 990 tasklet_setup(&chan->tasklet, dma_do_tasklet); 991 INIT_LIST_HEAD(&chan->chain_pending); 992 INIT_LIST_HEAD(&chan->chain_running); 993 994 /* register virt channel to dma engine */ 995 list_add_tail(&chan->chan.device_node, &pdev->device.channels); 996 997 return 0; 998} 999 1000static const struct of_device_id mmp_pdma_dt_ids[] = { 1001 { .compatible = "marvell,pdma-1.0", }, 1002 {} 1003}; 1004MODULE_DEVICE_TABLE(of, mmp_pdma_dt_ids); 1005 1006static struct dma_chan *mmp_pdma_dma_xlate(struct of_phandle_args *dma_spec, 1007 struct of_dma *ofdma) 1008{ 1009 struct mmp_pdma_device *d = ofdma->of_dma_data; 1010 struct dma_chan *chan; 1011 1012 chan = dma_get_any_slave_channel(&d->device); 1013 if (!chan) 1014 return NULL; 1015 1016 to_mmp_pdma_chan(chan)->drcmr = dma_spec->args[0]; 1017 1018 return chan; 1019} 1020 1021static int mmp_pdma_probe(struct platform_device *op) 1022{ 1023 struct mmp_pdma_device *pdev; 1024 const struct of_device_id *of_id; 1025 struct mmp_dma_platdata *pdata = dev_get_platdata(&op->dev); 1026 struct resource *iores; 1027 int i, ret, irq = 0; 1028 int dma_channels = 0, irq_num = 0; 1029 const enum dma_slave_buswidth widths = 1030 DMA_SLAVE_BUSWIDTH_1_BYTE | DMA_SLAVE_BUSWIDTH_2_BYTES | 1031 DMA_SLAVE_BUSWIDTH_4_BYTES; 1032 1033 pdev = devm_kzalloc(&op->dev, sizeof(*pdev), GFP_KERNEL); 1034 if (!pdev) 1035 return -ENOMEM; 1036 1037 pdev->dev = &op->dev; 1038 1039 spin_lock_init(&pdev->phy_lock); 1040 1041 iores = platform_get_resource(op, IORESOURCE_MEM, 0); 1042 pdev->base = devm_ioremap_resource(pdev->dev, iores); 1043 if (IS_ERR(pdev->base)) 1044 return PTR_ERR(pdev->base); 1045 1046 of_id = of_match_device(mmp_pdma_dt_ids, pdev->dev); 1047 if (of_id) 1048 of_property_read_u32(pdev->dev->of_node, "#dma-channels", 1049 &dma_channels); 1050 else if (pdata && pdata->dma_channels) 1051 dma_channels = pdata->dma_channels; 1052 else 1053 dma_channels = 32; /* default 32 channel */ 1054 pdev->dma_channels = dma_channels; 1055 1056 for (i = 0; i < dma_channels; i++) { 1057 if (platform_get_irq_optional(op, i) > 0) 1058 irq_num++; 1059 } 1060 1061 pdev->phy = devm_kcalloc(pdev->dev, dma_channels, sizeof(*pdev->phy), 1062 GFP_KERNEL); 1063 if (pdev->phy == NULL) 1064 return -ENOMEM; 1065 1066 INIT_LIST_HEAD(&pdev->device.channels); 1067 1068 if (irq_num != dma_channels) { 1069 /* all chan share one irq, demux inside */ 1070 irq = platform_get_irq(op, 0); 1071 ret = devm_request_irq(pdev->dev, irq, mmp_pdma_int_handler, 1072 IRQF_SHARED, "pdma", pdev); 1073 if (ret) 1074 return ret; 1075 } 1076 1077 for (i = 0; i < dma_channels; i++) { 1078 irq = (irq_num != dma_channels) ? 0 : platform_get_irq(op, i); 1079 ret = mmp_pdma_chan_init(pdev, i, irq); 1080 if (ret) 1081 return ret; 1082 } 1083 1084 dma_cap_set(DMA_SLAVE, pdev->device.cap_mask); 1085 dma_cap_set(DMA_MEMCPY, pdev->device.cap_mask); 1086 dma_cap_set(DMA_CYCLIC, pdev->device.cap_mask); 1087 dma_cap_set(DMA_PRIVATE, pdev->device.cap_mask); 1088 pdev->device.dev = &op->dev; 1089 pdev->device.device_alloc_chan_resources = mmp_pdma_alloc_chan_resources; 1090 pdev->device.device_free_chan_resources = mmp_pdma_free_chan_resources; 1091 pdev->device.device_tx_status = mmp_pdma_tx_status; 1092 pdev->device.device_prep_dma_memcpy = mmp_pdma_prep_memcpy; 1093 pdev->device.device_prep_slave_sg = mmp_pdma_prep_slave_sg; 1094 pdev->device.device_prep_dma_cyclic = mmp_pdma_prep_dma_cyclic; 1095 pdev->device.device_issue_pending = mmp_pdma_issue_pending; 1096 pdev->device.device_config = mmp_pdma_config; 1097 pdev->device.device_terminate_all = mmp_pdma_terminate_all; 1098 pdev->device.copy_align = DMAENGINE_ALIGN_8_BYTES; 1099 pdev->device.src_addr_widths = widths; 1100 pdev->device.dst_addr_widths = widths; 1101 pdev->device.directions = BIT(DMA_MEM_TO_DEV) | BIT(DMA_DEV_TO_MEM); 1102 pdev->device.residue_granularity = DMA_RESIDUE_GRANULARITY_DESCRIPTOR; 1103 1104 if (pdev->dev->coherent_dma_mask) 1105 dma_set_mask(pdev->dev, pdev->dev->coherent_dma_mask); 1106 else 1107 dma_set_mask(pdev->dev, DMA_BIT_MASK(64)); 1108 1109 ret = dma_async_device_register(&pdev->device); 1110 if (ret) { 1111 dev_err(pdev->device.dev, "unable to register\n"); 1112 return ret; 1113 } 1114 1115 if (op->dev.of_node) { 1116 /* Device-tree DMA controller registration */ 1117 ret = of_dma_controller_register(op->dev.of_node, 1118 mmp_pdma_dma_xlate, pdev); 1119 if (ret < 0) { 1120 dev_err(&op->dev, "of_dma_controller_register failed\n"); 1121 return ret; 1122 } 1123 } 1124 1125 platform_set_drvdata(op, pdev); 1126 dev_info(pdev->device.dev, "initialized %d channels\n", dma_channels); 1127 return 0; 1128} 1129 1130static const struct platform_device_id mmp_pdma_id_table[] = { 1131 { "mmp-pdma", }, 1132 { }, 1133}; 1134 1135static struct platform_driver mmp_pdma_driver = { 1136 .driver = { 1137 .name = "mmp-pdma", 1138 .of_match_table = mmp_pdma_dt_ids, 1139 }, 1140 .id_table = mmp_pdma_id_table, 1141 .probe = mmp_pdma_probe, 1142 .remove = mmp_pdma_remove, 1143}; 1144 1145bool mmp_pdma_filter_fn(struct dma_chan *chan, void *param) 1146{ 1147 struct mmp_pdma_chan *c = to_mmp_pdma_chan(chan); 1148 1149 if (chan->device->dev->driver != &mmp_pdma_driver.driver) 1150 return false; 1151 1152 c->drcmr = *(unsigned int *)param; 1153 1154 return true; 1155} 1156EXPORT_SYMBOL_GPL(mmp_pdma_filter_fn); 1157 1158module_platform_driver(mmp_pdma_driver); 1159 1160MODULE_DESCRIPTION("MARVELL MMP Peripheral DMA Driver"); 1161MODULE_AUTHOR("Marvell International Ltd."); 1162MODULE_LICENSE("GPL v2"); 1163