1// SPDX-License-Identifier: GPL-2.0-only 2/* 3 * Driver for the Atmel Extensible DMA Controller (aka XDMAC on AT91 systems) 4 * 5 * Copyright (C) 2014 Atmel Corporation 6 * 7 * Author: Ludovic Desroches <ludovic.desroches@atmel.com> 8 */ 9 10#include <asm/barrier.h> 11#include <dt-bindings/dma/at91.h> 12#include <linux/clk.h> 13#include <linux/dmaengine.h> 14#include <linux/dmapool.h> 15#include <linux/interrupt.h> 16#include <linux/irq.h> 17#include <linux/kernel.h> 18#include <linux/list.h> 19#include <linux/module.h> 20#include <linux/of_dma.h> 21#include <linux/of_platform.h> 22#include <linux/platform_device.h> 23#include <linux/pm.h> 24 25#include "dmaengine.h" 26 27/* Global registers */ 28#define AT_XDMAC_GTYPE 0x00 /* Global Type Register */ 29#define AT_XDMAC_NB_CH(i) (((i) & 0x1F) + 1) /* Number of Channels Minus One */ 30#define AT_XDMAC_FIFO_SZ(i) (((i) >> 5) & 0x7FF) /* Number of Bytes */ 31#define AT_XDMAC_NB_REQ(i) ((((i) >> 16) & 0x3F) + 1) /* Number of Peripheral Requests Minus One */ 32#define AT_XDMAC_GCFG 0x04 /* Global Configuration Register */ 33#define AT_XDMAC_GWAC 0x08 /* Global Weighted Arbiter Configuration Register */ 34#define AT_XDMAC_GIE 0x0C /* Global Interrupt Enable Register */ 35#define AT_XDMAC_GID 0x10 /* Global Interrupt Disable Register */ 36#define AT_XDMAC_GIM 0x14 /* Global Interrupt Mask Register */ 37#define AT_XDMAC_GIS 0x18 /* Global Interrupt Status Register */ 38#define AT_XDMAC_GE 0x1C /* Global Channel Enable Register */ 39#define AT_XDMAC_GD 0x20 /* Global Channel Disable Register */ 40#define AT_XDMAC_GS 0x24 /* Global Channel Status Register */ 41#define AT_XDMAC_GRS 0x28 /* Global Channel Read Suspend Register */ 42#define AT_XDMAC_GWS 0x2C /* Global Write Suspend Register */ 43#define AT_XDMAC_GRWS 0x30 /* Global Channel Read Write Suspend Register */ 44#define AT_XDMAC_GRWR 0x34 /* Global Channel Read Write Resume Register */ 45#define AT_XDMAC_GSWR 0x38 /* Global Channel Software Request Register */ 46#define AT_XDMAC_GSWS 0x3C /* Global channel Software Request Status Register */ 47#define AT_XDMAC_GSWF 0x40 /* Global Channel Software Flush Request Register */ 48#define AT_XDMAC_VERSION 0xFFC /* XDMAC Version Register */ 49 50/* Channel relative registers offsets */ 51#define AT_XDMAC_CIE 0x00 /* Channel Interrupt Enable Register */ 52#define AT_XDMAC_CIE_BIE BIT(0) /* End of Block Interrupt Enable Bit */ 53#define AT_XDMAC_CIE_LIE BIT(1) /* End of Linked List Interrupt Enable Bit */ 54#define AT_XDMAC_CIE_DIE BIT(2) /* End of Disable Interrupt Enable Bit */ 55#define AT_XDMAC_CIE_FIE BIT(3) /* End of Flush Interrupt Enable Bit */ 56#define AT_XDMAC_CIE_RBEIE BIT(4) /* Read Bus Error Interrupt Enable Bit */ 57#define AT_XDMAC_CIE_WBEIE BIT(5) /* Write Bus Error Interrupt Enable Bit */ 58#define AT_XDMAC_CIE_ROIE BIT(6) /* Request Overflow Interrupt Enable Bit */ 59#define AT_XDMAC_CID 0x04 /* Channel Interrupt Disable Register */ 60#define AT_XDMAC_CID_BID BIT(0) /* End of Block Interrupt Disable Bit */ 61#define AT_XDMAC_CID_LID BIT(1) /* End of Linked List Interrupt Disable Bit */ 62#define AT_XDMAC_CID_DID BIT(2) /* End of Disable Interrupt Disable Bit */ 63#define AT_XDMAC_CID_FID BIT(3) /* End of Flush Interrupt Disable Bit */ 64#define AT_XDMAC_CID_RBEID BIT(4) /* Read Bus Error Interrupt Disable Bit */ 65#define AT_XDMAC_CID_WBEID BIT(5) /* Write Bus Error Interrupt Disable Bit */ 66#define AT_XDMAC_CID_ROID BIT(6) /* Request Overflow Interrupt Disable Bit */ 67#define AT_XDMAC_CIM 0x08 /* Channel Interrupt Mask Register */ 68#define AT_XDMAC_CIM_BIM BIT(0) /* End of Block Interrupt Mask Bit */ 69#define AT_XDMAC_CIM_LIM BIT(1) /* End of Linked List Interrupt Mask Bit */ 70#define AT_XDMAC_CIM_DIM BIT(2) /* End of Disable Interrupt Mask Bit */ 71#define AT_XDMAC_CIM_FIM BIT(3) /* End of Flush Interrupt Mask Bit */ 72#define AT_XDMAC_CIM_RBEIM BIT(4) /* Read Bus Error Interrupt Mask Bit */ 73#define AT_XDMAC_CIM_WBEIM BIT(5) /* Write Bus Error Interrupt Mask Bit */ 74#define AT_XDMAC_CIM_ROIM BIT(6) /* Request Overflow Interrupt Mask Bit */ 75#define AT_XDMAC_CIS 0x0C /* Channel Interrupt Status Register */ 76#define AT_XDMAC_CIS_BIS BIT(0) /* End of Block Interrupt Status Bit */ 77#define AT_XDMAC_CIS_LIS BIT(1) /* End of Linked List Interrupt Status Bit */ 78#define AT_XDMAC_CIS_DIS BIT(2) /* End of Disable Interrupt Status Bit */ 79#define AT_XDMAC_CIS_FIS BIT(3) /* End of Flush Interrupt Status Bit */ 80#define AT_XDMAC_CIS_RBEIS BIT(4) /* Read Bus Error Interrupt Status Bit */ 81#define AT_XDMAC_CIS_WBEIS BIT(5) /* Write Bus Error Interrupt Status Bit */ 82#define AT_XDMAC_CIS_ROIS BIT(6) /* Request Overflow Interrupt Status Bit */ 83#define AT_XDMAC_CSA 0x10 /* Channel Source Address Register */ 84#define AT_XDMAC_CDA 0x14 /* Channel Destination Address Register */ 85#define AT_XDMAC_CNDA 0x18 /* Channel Next Descriptor Address Register */ 86#define AT_XDMAC_CNDA_NDAIF(i) ((i) & 0x1) /* Channel x Next Descriptor Interface */ 87#define AT_XDMAC_CNDA_NDA(i) ((i) & 0xfffffffc) /* Channel x Next Descriptor Address */ 88#define AT_XDMAC_CNDC 0x1C /* Channel Next Descriptor Control Register */ 89#define AT_XDMAC_CNDC_NDE (0x1 << 0) /* Channel x Next Descriptor Enable */ 90#define AT_XDMAC_CNDC_NDSUP (0x1 << 1) /* Channel x Next Descriptor Source Update */ 91#define AT_XDMAC_CNDC_NDDUP (0x1 << 2) /* Channel x Next Descriptor Destination Update */ 92#define AT_XDMAC_CNDC_NDVIEW_MASK GENMASK(28, 27) 93#define AT_XDMAC_CNDC_NDVIEW_NDV0 (0x0 << 3) /* Channel x Next Descriptor View 0 */ 94#define AT_XDMAC_CNDC_NDVIEW_NDV1 (0x1 << 3) /* Channel x Next Descriptor View 1 */ 95#define AT_XDMAC_CNDC_NDVIEW_NDV2 (0x2 << 3) /* Channel x Next Descriptor View 2 */ 96#define AT_XDMAC_CNDC_NDVIEW_NDV3 (0x3 << 3) /* Channel x Next Descriptor View 3 */ 97#define AT_XDMAC_CUBC 0x20 /* Channel Microblock Control Register */ 98#define AT_XDMAC_CBC 0x24 /* Channel Block Control Register */ 99#define AT_XDMAC_CC 0x28 /* Channel Configuration Register */ 100#define AT_XDMAC_CC_TYPE (0x1 << 0) /* Channel Transfer Type */ 101#define AT_XDMAC_CC_TYPE_MEM_TRAN (0x0 << 0) /* Memory to Memory Transfer */ 102#define AT_XDMAC_CC_TYPE_PER_TRAN (0x1 << 0) /* Peripheral to Memory or Memory to Peripheral Transfer */ 103#define AT_XDMAC_CC_MBSIZE_MASK (0x3 << 1) 104#define AT_XDMAC_CC_MBSIZE_SINGLE (0x0 << 1) 105#define AT_XDMAC_CC_MBSIZE_FOUR (0x1 << 1) 106#define AT_XDMAC_CC_MBSIZE_EIGHT (0x2 << 1) 107#define AT_XDMAC_CC_MBSIZE_SIXTEEN (0x3 << 1) 108#define AT_XDMAC_CC_DSYNC (0x1 << 4) /* Channel Synchronization */ 109#define AT_XDMAC_CC_DSYNC_PER2MEM (0x0 << 4) 110#define AT_XDMAC_CC_DSYNC_MEM2PER (0x1 << 4) 111#define AT_XDMAC_CC_PROT (0x1 << 5) /* Channel Protection */ 112#define AT_XDMAC_CC_PROT_SEC (0x0 << 5) 113#define AT_XDMAC_CC_PROT_UNSEC (0x1 << 5) 114#define AT_XDMAC_CC_SWREQ (0x1 << 6) /* Channel Software Request Trigger */ 115#define AT_XDMAC_CC_SWREQ_HWR_CONNECTED (0x0 << 6) 116#define AT_XDMAC_CC_SWREQ_SWR_CONNECTED (0x1 << 6) 117#define AT_XDMAC_CC_MEMSET (0x1 << 7) /* Channel Fill Block of memory */ 118#define AT_XDMAC_CC_MEMSET_NORMAL_MODE (0x0 << 7) 119#define AT_XDMAC_CC_MEMSET_HW_MODE (0x1 << 7) 120#define AT_XDMAC_CC_CSIZE(i) ((0x7 & (i)) << 8) /* Channel Chunk Size */ 121#define AT_XDMAC_CC_DWIDTH_OFFSET 11 122#define AT_XDMAC_CC_DWIDTH_MASK (0x3 << AT_XDMAC_CC_DWIDTH_OFFSET) 123#define AT_XDMAC_CC_DWIDTH(i) ((0x3 & (i)) << AT_XDMAC_CC_DWIDTH_OFFSET) /* Channel Data Width */ 124#define AT_XDMAC_CC_DWIDTH_BYTE 0x0 125#define AT_XDMAC_CC_DWIDTH_HALFWORD 0x1 126#define AT_XDMAC_CC_DWIDTH_WORD 0x2 127#define AT_XDMAC_CC_DWIDTH_DWORD 0x3 128#define AT_XDMAC_CC_SIF(i) ((0x1 & (i)) << 13) /* Channel Source Interface Identifier */ 129#define AT_XDMAC_CC_DIF(i) ((0x1 & (i)) << 14) /* Channel Destination Interface Identifier */ 130#define AT_XDMAC_CC_SAM_MASK (0x3 << 16) /* Channel Source Addressing Mode */ 131#define AT_XDMAC_CC_SAM_FIXED_AM (0x0 << 16) 132#define AT_XDMAC_CC_SAM_INCREMENTED_AM (0x1 << 16) 133#define AT_XDMAC_CC_SAM_UBS_AM (0x2 << 16) 134#define AT_XDMAC_CC_SAM_UBS_DS_AM (0x3 << 16) 135#define AT_XDMAC_CC_DAM_MASK (0x3 << 18) /* Channel Source Addressing Mode */ 136#define AT_XDMAC_CC_DAM_FIXED_AM (0x0 << 18) 137#define AT_XDMAC_CC_DAM_INCREMENTED_AM (0x1 << 18) 138#define AT_XDMAC_CC_DAM_UBS_AM (0x2 << 18) 139#define AT_XDMAC_CC_DAM_UBS_DS_AM (0x3 << 18) 140#define AT_XDMAC_CC_INITD (0x1 << 21) /* Channel Initialization Terminated (read only) */ 141#define AT_XDMAC_CC_INITD_TERMINATED (0x0 << 21) 142#define AT_XDMAC_CC_INITD_IN_PROGRESS (0x1 << 21) 143#define AT_XDMAC_CC_RDIP (0x1 << 22) /* Read in Progress (read only) */ 144#define AT_XDMAC_CC_RDIP_DONE (0x0 << 22) 145#define AT_XDMAC_CC_RDIP_IN_PROGRESS (0x1 << 22) 146#define AT_XDMAC_CC_WRIP (0x1 << 23) /* Write in Progress (read only) */ 147#define AT_XDMAC_CC_WRIP_DONE (0x0 << 23) 148#define AT_XDMAC_CC_WRIP_IN_PROGRESS (0x1 << 23) 149#define AT_XDMAC_CC_PERID(i) ((0x7f & (i)) << 24) /* Channel Peripheral Identifier */ 150#define AT_XDMAC_CDS_MSP 0x2C /* Channel Data Stride Memory Set Pattern */ 151#define AT_XDMAC_CSUS 0x30 /* Channel Source Microblock Stride */ 152#define AT_XDMAC_CDUS 0x34 /* Channel Destination Microblock Stride */ 153 154#define AT_XDMAC_CHAN_REG_BASE 0x50 /* Channel registers base address */ 155 156/* Microblock control members */ 157#define AT_XDMAC_MBR_UBC_UBLEN_MAX 0xFFFFFFUL /* Maximum Microblock Length */ 158#define AT_XDMAC_MBR_UBC_NDE (0x1 << 24) /* Next Descriptor Enable */ 159#define AT_XDMAC_MBR_UBC_NSEN (0x1 << 25) /* Next Descriptor Source Update */ 160#define AT_XDMAC_MBR_UBC_NDEN (0x1 << 26) /* Next Descriptor Destination Update */ 161#define AT_XDMAC_MBR_UBC_NDV0 (0x0 << 27) /* Next Descriptor View 0 */ 162#define AT_XDMAC_MBR_UBC_NDV1 (0x1 << 27) /* Next Descriptor View 1 */ 163#define AT_XDMAC_MBR_UBC_NDV2 (0x2 << 27) /* Next Descriptor View 2 */ 164#define AT_XDMAC_MBR_UBC_NDV3 (0x3 << 27) /* Next Descriptor View 3 */ 165 166#define AT_XDMAC_MAX_CHAN 0x20 167#define AT_XDMAC_MAX_CSIZE 16 /* 16 data */ 168#define AT_XDMAC_MAX_DWIDTH 8 /* 64 bits */ 169#define AT_XDMAC_RESIDUE_MAX_RETRIES 5 170 171#define AT_XDMAC_DMA_BUSWIDTHS\ 172 (BIT(DMA_SLAVE_BUSWIDTH_UNDEFINED) |\ 173 BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) |\ 174 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) |\ 175 BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) |\ 176 BIT(DMA_SLAVE_BUSWIDTH_8_BYTES)) 177 178enum atc_status { 179 AT_XDMAC_CHAN_IS_CYCLIC = 0, 180 AT_XDMAC_CHAN_IS_PAUSED, 181}; 182 183/* ----- Channels ----- */ 184struct at_xdmac_chan { 185 struct dma_chan chan; 186 void __iomem *ch_regs; 187 u32 mask; /* Channel Mask */ 188 u32 cfg; /* Channel Configuration Register */ 189 u8 perid; /* Peripheral ID */ 190 u8 perif; /* Peripheral Interface */ 191 u8 memif; /* Memory Interface */ 192 u32 save_cc; 193 u32 save_cim; 194 u32 save_cnda; 195 u32 save_cndc; 196 u32 irq_status; 197 unsigned long status; 198 struct tasklet_struct tasklet; 199 struct dma_slave_config sconfig; 200 201 spinlock_t lock; 202 203 struct list_head xfers_list; 204 struct list_head free_descs_list; 205}; 206 207 208/* ----- Controller ----- */ 209struct at_xdmac { 210 struct dma_device dma; 211 void __iomem *regs; 212 int irq; 213 struct clk *clk; 214 u32 save_gim; 215 u32 save_gs; 216 struct dma_pool *at_xdmac_desc_pool; 217 struct at_xdmac_chan chan[]; 218}; 219 220 221/* ----- Descriptors ----- */ 222 223/* Linked List Descriptor */ 224struct at_xdmac_lld { 225 u32 mbr_nda; /* Next Descriptor Member */ 226 u32 mbr_ubc; /* Microblock Control Member */ 227 u32 mbr_sa; /* Source Address Member */ 228 u32 mbr_da; /* Destination Address Member */ 229 u32 mbr_cfg; /* Configuration Register */ 230 u32 mbr_bc; /* Block Control Register */ 231 u32 mbr_ds; /* Data Stride Register */ 232 u32 mbr_sus; /* Source Microblock Stride Register */ 233 u32 mbr_dus; /* Destination Microblock Stride Register */ 234}; 235 236/* 64-bit alignment needed to update CNDA and CUBC registers in an atomic way. */ 237struct at_xdmac_desc { 238 struct at_xdmac_lld lld; 239 enum dma_transfer_direction direction; 240 struct dma_async_tx_descriptor tx_dma_desc; 241 struct list_head desc_node; 242 /* Following members are only used by the first descriptor */ 243 bool active_xfer; 244 unsigned int xfer_size; 245 struct list_head descs_list; 246 struct list_head xfer_node; 247} __aligned(sizeof(u64)); 248 249static inline void __iomem *at_xdmac_chan_reg_base(struct at_xdmac *atxdmac, unsigned int chan_nb) 250{ 251 return atxdmac->regs + (AT_XDMAC_CHAN_REG_BASE + chan_nb * 0x40); 252} 253 254#define at_xdmac_read(atxdmac, reg) readl_relaxed((atxdmac)->regs + (reg)) 255#define at_xdmac_write(atxdmac, reg, value) \ 256 writel_relaxed((value), (atxdmac)->regs + (reg)) 257 258#define at_xdmac_chan_read(atchan, reg) readl_relaxed((atchan)->ch_regs + (reg)) 259#define at_xdmac_chan_write(atchan, reg, value) writel_relaxed((value), (atchan)->ch_regs + (reg)) 260 261static inline struct at_xdmac_chan *to_at_xdmac_chan(struct dma_chan *dchan) 262{ 263 return container_of(dchan, struct at_xdmac_chan, chan); 264} 265 266static struct device *chan2dev(struct dma_chan *chan) 267{ 268 return &chan->dev->device; 269} 270 271static inline struct at_xdmac *to_at_xdmac(struct dma_device *ddev) 272{ 273 return container_of(ddev, struct at_xdmac, dma); 274} 275 276static inline struct at_xdmac_desc *txd_to_at_desc(struct dma_async_tx_descriptor *txd) 277{ 278 return container_of(txd, struct at_xdmac_desc, tx_dma_desc); 279} 280 281static inline int at_xdmac_chan_is_cyclic(struct at_xdmac_chan *atchan) 282{ 283 return test_bit(AT_XDMAC_CHAN_IS_CYCLIC, &atchan->status); 284} 285 286static inline int at_xdmac_chan_is_paused(struct at_xdmac_chan *atchan) 287{ 288 return test_bit(AT_XDMAC_CHAN_IS_PAUSED, &atchan->status); 289} 290 291static inline int at_xdmac_csize(u32 maxburst) 292{ 293 int csize; 294 295 csize = ffs(maxburst) - 1; 296 if (csize > 4) 297 csize = -EINVAL; 298 299 return csize; 300}; 301 302static inline bool at_xdmac_chan_is_peripheral_xfer(u32 cfg) 303{ 304 return cfg & AT_XDMAC_CC_TYPE_PER_TRAN; 305} 306 307static inline u8 at_xdmac_get_dwidth(u32 cfg) 308{ 309 return (cfg & AT_XDMAC_CC_DWIDTH_MASK) >> AT_XDMAC_CC_DWIDTH_OFFSET; 310}; 311 312static unsigned int init_nr_desc_per_channel = 64; 313module_param(init_nr_desc_per_channel, uint, 0644); 314MODULE_PARM_DESC(init_nr_desc_per_channel, 315 "initial descriptors per channel (default: 64)"); 316 317 318static bool at_xdmac_chan_is_enabled(struct at_xdmac_chan *atchan) 319{ 320 return at_xdmac_chan_read(atchan, AT_XDMAC_GS) & atchan->mask; 321} 322 323static void at_xdmac_off(struct at_xdmac *atxdmac) 324{ 325 at_xdmac_write(atxdmac, AT_XDMAC_GD, -1L); 326 327 /* Wait that all chans are disabled. */ 328 while (at_xdmac_read(atxdmac, AT_XDMAC_GS)) 329 cpu_relax(); 330 331 at_xdmac_write(atxdmac, AT_XDMAC_GID, -1L); 332} 333 334/* Call with lock hold. */ 335static void at_xdmac_start_xfer(struct at_xdmac_chan *atchan, 336 struct at_xdmac_desc *first) 337{ 338 struct at_xdmac *atxdmac = to_at_xdmac(atchan->chan.device); 339 u32 reg; 340 341 dev_vdbg(chan2dev(&atchan->chan), "%s: desc 0x%p\n", __func__, first); 342 343 /* Set transfer as active to not try to start it again. */ 344 first->active_xfer = true; 345 346 /* Tell xdmac where to get the first descriptor. */ 347 reg = AT_XDMAC_CNDA_NDA(first->tx_dma_desc.phys) 348 | AT_XDMAC_CNDA_NDAIF(atchan->memif); 349 at_xdmac_chan_write(atchan, AT_XDMAC_CNDA, reg); 350 351 /* 352 * When doing non cyclic transfer we need to use the next 353 * descriptor view 2 since some fields of the configuration register 354 * depend on transfer size and src/dest addresses. 355 */ 356 if (at_xdmac_chan_is_cyclic(atchan)) 357 reg = AT_XDMAC_CNDC_NDVIEW_NDV1; 358 else if ((first->lld.mbr_ubc & 359 AT_XDMAC_CNDC_NDVIEW_MASK) == AT_XDMAC_MBR_UBC_NDV3) 360 reg = AT_XDMAC_CNDC_NDVIEW_NDV3; 361 else 362 reg = AT_XDMAC_CNDC_NDVIEW_NDV2; 363 /* 364 * Even if the register will be updated from the configuration in the 365 * descriptor when using view 2 or higher, the PROT bit won't be set 366 * properly. This bit can be modified only by using the channel 367 * configuration register. 368 */ 369 at_xdmac_chan_write(atchan, AT_XDMAC_CC, first->lld.mbr_cfg); 370 371 reg |= AT_XDMAC_CNDC_NDDUP 372 | AT_XDMAC_CNDC_NDSUP 373 | AT_XDMAC_CNDC_NDE; 374 at_xdmac_chan_write(atchan, AT_XDMAC_CNDC, reg); 375 376 dev_vdbg(chan2dev(&atchan->chan), 377 "%s: CC=0x%08x CNDA=0x%08x, CNDC=0x%08x, CSA=0x%08x, CDA=0x%08x, CUBC=0x%08x\n", 378 __func__, at_xdmac_chan_read(atchan, AT_XDMAC_CC), 379 at_xdmac_chan_read(atchan, AT_XDMAC_CNDA), 380 at_xdmac_chan_read(atchan, AT_XDMAC_CNDC), 381 at_xdmac_chan_read(atchan, AT_XDMAC_CSA), 382 at_xdmac_chan_read(atchan, AT_XDMAC_CDA), 383 at_xdmac_chan_read(atchan, AT_XDMAC_CUBC)); 384 385 at_xdmac_chan_write(atchan, AT_XDMAC_CID, 0xffffffff); 386 reg = AT_XDMAC_CIE_RBEIE | AT_XDMAC_CIE_WBEIE; 387 /* 388 * Request Overflow Error is only for peripheral synchronized transfers 389 */ 390 if (at_xdmac_chan_is_peripheral_xfer(first->lld.mbr_cfg)) 391 reg |= AT_XDMAC_CIE_ROIE; 392 393 /* 394 * There is no end of list when doing cyclic dma, we need to get 395 * an interrupt after each periods. 396 */ 397 if (at_xdmac_chan_is_cyclic(atchan)) 398 at_xdmac_chan_write(atchan, AT_XDMAC_CIE, 399 reg | AT_XDMAC_CIE_BIE); 400 else 401 at_xdmac_chan_write(atchan, AT_XDMAC_CIE, 402 reg | AT_XDMAC_CIE_LIE); 403 at_xdmac_write(atxdmac, AT_XDMAC_GIE, atchan->mask); 404 dev_vdbg(chan2dev(&atchan->chan), 405 "%s: enable channel (0x%08x)\n", __func__, atchan->mask); 406 wmb(); 407 at_xdmac_write(atxdmac, AT_XDMAC_GE, atchan->mask); 408 409 dev_vdbg(chan2dev(&atchan->chan), 410 "%s: CC=0x%08x CNDA=0x%08x, CNDC=0x%08x, CSA=0x%08x, CDA=0x%08x, CUBC=0x%08x\n", 411 __func__, at_xdmac_chan_read(atchan, AT_XDMAC_CC), 412 at_xdmac_chan_read(atchan, AT_XDMAC_CNDA), 413 at_xdmac_chan_read(atchan, AT_XDMAC_CNDC), 414 at_xdmac_chan_read(atchan, AT_XDMAC_CSA), 415 at_xdmac_chan_read(atchan, AT_XDMAC_CDA), 416 at_xdmac_chan_read(atchan, AT_XDMAC_CUBC)); 417 418} 419 420static dma_cookie_t at_xdmac_tx_submit(struct dma_async_tx_descriptor *tx) 421{ 422 struct at_xdmac_desc *desc = txd_to_at_desc(tx); 423 struct at_xdmac_chan *atchan = to_at_xdmac_chan(tx->chan); 424 dma_cookie_t cookie; 425 unsigned long irqflags; 426 427 spin_lock_irqsave(&atchan->lock, irqflags); 428 cookie = dma_cookie_assign(tx); 429 430 list_add_tail(&desc->xfer_node, &atchan->xfers_list); 431 spin_unlock_irqrestore(&atchan->lock, irqflags); 432 433 dev_vdbg(chan2dev(tx->chan), "%s: atchan 0x%p, add desc 0x%p to xfers_list\n", 434 __func__, atchan, desc); 435 436 return cookie; 437} 438 439static struct at_xdmac_desc *at_xdmac_alloc_desc(struct dma_chan *chan, 440 gfp_t gfp_flags) 441{ 442 struct at_xdmac_desc *desc; 443 struct at_xdmac *atxdmac = to_at_xdmac(chan->device); 444 dma_addr_t phys; 445 446 desc = dma_pool_zalloc(atxdmac->at_xdmac_desc_pool, gfp_flags, &phys); 447 if (desc) { 448 INIT_LIST_HEAD(&desc->descs_list); 449 dma_async_tx_descriptor_init(&desc->tx_dma_desc, chan); 450 desc->tx_dma_desc.tx_submit = at_xdmac_tx_submit; 451 desc->tx_dma_desc.phys = phys; 452 } 453 454 return desc; 455} 456 457static void at_xdmac_init_used_desc(struct at_xdmac_desc *desc) 458{ 459 memset(&desc->lld, 0, sizeof(desc->lld)); 460 INIT_LIST_HEAD(&desc->descs_list); 461 desc->direction = DMA_TRANS_NONE; 462 desc->xfer_size = 0; 463 desc->active_xfer = false; 464} 465 466/* Call must be protected by lock. */ 467static struct at_xdmac_desc *at_xdmac_get_desc(struct at_xdmac_chan *atchan) 468{ 469 struct at_xdmac_desc *desc; 470 471 if (list_empty(&atchan->free_descs_list)) { 472 desc = at_xdmac_alloc_desc(&atchan->chan, GFP_NOWAIT); 473 } else { 474 desc = list_first_entry(&atchan->free_descs_list, 475 struct at_xdmac_desc, desc_node); 476 list_del(&desc->desc_node); 477 at_xdmac_init_used_desc(desc); 478 } 479 480 return desc; 481} 482 483static void at_xdmac_queue_desc(struct dma_chan *chan, 484 struct at_xdmac_desc *prev, 485 struct at_xdmac_desc *desc) 486{ 487 if (!prev || !desc) 488 return; 489 490 prev->lld.mbr_nda = desc->tx_dma_desc.phys; 491 prev->lld.mbr_ubc |= AT_XDMAC_MBR_UBC_NDE; 492 493 dev_dbg(chan2dev(chan), "%s: chain lld: prev=0x%p, mbr_nda=%pad\n", 494 __func__, prev, &prev->lld.mbr_nda); 495} 496 497static inline void at_xdmac_increment_block_count(struct dma_chan *chan, 498 struct at_xdmac_desc *desc) 499{ 500 if (!desc) 501 return; 502 503 desc->lld.mbr_bc++; 504 505 dev_dbg(chan2dev(chan), 506 "%s: incrementing the block count of the desc 0x%p\n", 507 __func__, desc); 508} 509 510static struct dma_chan *at_xdmac_xlate(struct of_phandle_args *dma_spec, 511 struct of_dma *of_dma) 512{ 513 struct at_xdmac *atxdmac = of_dma->of_dma_data; 514 struct at_xdmac_chan *atchan; 515 struct dma_chan *chan; 516 struct device *dev = atxdmac->dma.dev; 517 518 if (dma_spec->args_count != 1) { 519 dev_err(dev, "dma phandler args: bad number of args\n"); 520 return NULL; 521 } 522 523 chan = dma_get_any_slave_channel(&atxdmac->dma); 524 if (!chan) { 525 dev_err(dev, "can't get a dma channel\n"); 526 return NULL; 527 } 528 529 atchan = to_at_xdmac_chan(chan); 530 atchan->memif = AT91_XDMAC_DT_GET_MEM_IF(dma_spec->args[0]); 531 atchan->perif = AT91_XDMAC_DT_GET_PER_IF(dma_spec->args[0]); 532 atchan->perid = AT91_XDMAC_DT_GET_PERID(dma_spec->args[0]); 533 dev_dbg(dev, "chan dt cfg: memif=%u perif=%u perid=%u\n", 534 atchan->memif, atchan->perif, atchan->perid); 535 536 return chan; 537} 538 539static int at_xdmac_compute_chan_conf(struct dma_chan *chan, 540 enum dma_transfer_direction direction) 541{ 542 struct at_xdmac_chan *atchan = to_at_xdmac_chan(chan); 543 int csize, dwidth; 544 545 if (direction == DMA_DEV_TO_MEM) { 546 atchan->cfg = 547 AT91_XDMAC_DT_PERID(atchan->perid) 548 | AT_XDMAC_CC_DAM_INCREMENTED_AM 549 | AT_XDMAC_CC_SAM_FIXED_AM 550 | AT_XDMAC_CC_DIF(atchan->memif) 551 | AT_XDMAC_CC_SIF(atchan->perif) 552 | AT_XDMAC_CC_SWREQ_HWR_CONNECTED 553 | AT_XDMAC_CC_DSYNC_PER2MEM 554 | AT_XDMAC_CC_MBSIZE_SIXTEEN 555 | AT_XDMAC_CC_TYPE_PER_TRAN; 556 csize = ffs(atchan->sconfig.src_maxburst) - 1; 557 if (csize < 0) { 558 dev_err(chan2dev(chan), "invalid src maxburst value\n"); 559 return -EINVAL; 560 } 561 atchan->cfg |= AT_XDMAC_CC_CSIZE(csize); 562 dwidth = ffs(atchan->sconfig.src_addr_width) - 1; 563 if (dwidth < 0) { 564 dev_err(chan2dev(chan), "invalid src addr width value\n"); 565 return -EINVAL; 566 } 567 atchan->cfg |= AT_XDMAC_CC_DWIDTH(dwidth); 568 } else if (direction == DMA_MEM_TO_DEV) { 569 atchan->cfg = 570 AT91_XDMAC_DT_PERID(atchan->perid) 571 | AT_XDMAC_CC_DAM_FIXED_AM 572 | AT_XDMAC_CC_SAM_INCREMENTED_AM 573 | AT_XDMAC_CC_DIF(atchan->perif) 574 | AT_XDMAC_CC_SIF(atchan->memif) 575 | AT_XDMAC_CC_SWREQ_HWR_CONNECTED 576 | AT_XDMAC_CC_DSYNC_MEM2PER 577 | AT_XDMAC_CC_MBSIZE_SIXTEEN 578 | AT_XDMAC_CC_TYPE_PER_TRAN; 579 csize = ffs(atchan->sconfig.dst_maxburst) - 1; 580 if (csize < 0) { 581 dev_err(chan2dev(chan), "invalid src maxburst value\n"); 582 return -EINVAL; 583 } 584 atchan->cfg |= AT_XDMAC_CC_CSIZE(csize); 585 dwidth = ffs(atchan->sconfig.dst_addr_width) - 1; 586 if (dwidth < 0) { 587 dev_err(chan2dev(chan), "invalid dst addr width value\n"); 588 return -EINVAL; 589 } 590 atchan->cfg |= AT_XDMAC_CC_DWIDTH(dwidth); 591 } 592 593 dev_dbg(chan2dev(chan), "%s: cfg=0x%08x\n", __func__, atchan->cfg); 594 595 return 0; 596} 597 598/* 599 * Only check that maxburst and addr width values are supported by the 600 * the controller but not that the configuration is good to perform the 601 * transfer since we don't know the direction at this stage. 602 */ 603static int at_xdmac_check_slave_config(struct dma_slave_config *sconfig) 604{ 605 if ((sconfig->src_maxburst > AT_XDMAC_MAX_CSIZE) 606 || (sconfig->dst_maxburst > AT_XDMAC_MAX_CSIZE)) 607 return -EINVAL; 608 609 if ((sconfig->src_addr_width > AT_XDMAC_MAX_DWIDTH) 610 || (sconfig->dst_addr_width > AT_XDMAC_MAX_DWIDTH)) 611 return -EINVAL; 612 613 return 0; 614} 615 616static int at_xdmac_set_slave_config(struct dma_chan *chan, 617 struct dma_slave_config *sconfig) 618{ 619 struct at_xdmac_chan *atchan = to_at_xdmac_chan(chan); 620 621 if (at_xdmac_check_slave_config(sconfig)) { 622 dev_err(chan2dev(chan), "invalid slave configuration\n"); 623 return -EINVAL; 624 } 625 626 memcpy(&atchan->sconfig, sconfig, sizeof(atchan->sconfig)); 627 628 return 0; 629} 630 631static struct dma_async_tx_descriptor * 632at_xdmac_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl, 633 unsigned int sg_len, enum dma_transfer_direction direction, 634 unsigned long flags, void *context) 635{ 636 struct at_xdmac_chan *atchan = to_at_xdmac_chan(chan); 637 struct at_xdmac_desc *first = NULL, *prev = NULL; 638 struct scatterlist *sg; 639 int i; 640 unsigned int xfer_size = 0; 641 unsigned long irqflags; 642 struct dma_async_tx_descriptor *ret = NULL; 643 644 if (!sgl) 645 return NULL; 646 647 if (!is_slave_direction(direction)) { 648 dev_err(chan2dev(chan), "invalid DMA direction\n"); 649 return NULL; 650 } 651 652 dev_dbg(chan2dev(chan), "%s: sg_len=%d, dir=%s, flags=0x%lx\n", 653 __func__, sg_len, 654 direction == DMA_MEM_TO_DEV ? "to device" : "from device", 655 flags); 656 657 /* Protect dma_sconfig field that can be modified by set_slave_conf. */ 658 spin_lock_irqsave(&atchan->lock, irqflags); 659 660 if (at_xdmac_compute_chan_conf(chan, direction)) 661 goto spin_unlock; 662 663 /* Prepare descriptors. */ 664 for_each_sg(sgl, sg, sg_len, i) { 665 struct at_xdmac_desc *desc = NULL; 666 u32 len, mem, dwidth, fixed_dwidth; 667 668 len = sg_dma_len(sg); 669 mem = sg_dma_address(sg); 670 if (unlikely(!len)) { 671 dev_err(chan2dev(chan), "sg data length is zero\n"); 672 goto spin_unlock; 673 } 674 dev_dbg(chan2dev(chan), "%s: * sg%d len=%u, mem=0x%08x\n", 675 __func__, i, len, mem); 676 677 desc = at_xdmac_get_desc(atchan); 678 if (!desc) { 679 dev_err(chan2dev(chan), "can't get descriptor\n"); 680 if (first) 681 list_splice_tail_init(&first->descs_list, 682 &atchan->free_descs_list); 683 goto spin_unlock; 684 } 685 686 /* Linked list descriptor setup. */ 687 if (direction == DMA_DEV_TO_MEM) { 688 desc->lld.mbr_sa = atchan->sconfig.src_addr; 689 desc->lld.mbr_da = mem; 690 } else { 691 desc->lld.mbr_sa = mem; 692 desc->lld.mbr_da = atchan->sconfig.dst_addr; 693 } 694 dwidth = at_xdmac_get_dwidth(atchan->cfg); 695 fixed_dwidth = IS_ALIGNED(len, 1 << dwidth) 696 ? dwidth 697 : AT_XDMAC_CC_DWIDTH_BYTE; 698 desc->lld.mbr_ubc = AT_XDMAC_MBR_UBC_NDV2 /* next descriptor view */ 699 | AT_XDMAC_MBR_UBC_NDEN /* next descriptor dst parameter update */ 700 | AT_XDMAC_MBR_UBC_NSEN /* next descriptor src parameter update */ 701 | (len >> fixed_dwidth); /* microblock length */ 702 desc->lld.mbr_cfg = (atchan->cfg & ~AT_XDMAC_CC_DWIDTH_MASK) | 703 AT_XDMAC_CC_DWIDTH(fixed_dwidth); 704 dev_dbg(chan2dev(chan), 705 "%s: lld: mbr_sa=%pad, mbr_da=%pad, mbr_ubc=0x%08x\n", 706 __func__, &desc->lld.mbr_sa, &desc->lld.mbr_da, desc->lld.mbr_ubc); 707 708 /* Chain lld. */ 709 if (prev) 710 at_xdmac_queue_desc(chan, prev, desc); 711 712 prev = desc; 713 if (!first) 714 first = desc; 715 716 dev_dbg(chan2dev(chan), "%s: add desc 0x%p to descs_list 0x%p\n", 717 __func__, desc, first); 718 list_add_tail(&desc->desc_node, &first->descs_list); 719 xfer_size += len; 720 } 721 722 723 first->tx_dma_desc.flags = flags; 724 first->xfer_size = xfer_size; 725 first->direction = direction; 726 ret = &first->tx_dma_desc; 727 728spin_unlock: 729 spin_unlock_irqrestore(&atchan->lock, irqflags); 730 return ret; 731} 732 733static struct dma_async_tx_descriptor * 734at_xdmac_prep_dma_cyclic(struct dma_chan *chan, dma_addr_t buf_addr, 735 size_t buf_len, size_t period_len, 736 enum dma_transfer_direction direction, 737 unsigned long flags) 738{ 739 struct at_xdmac_chan *atchan = to_at_xdmac_chan(chan); 740 struct at_xdmac_desc *first = NULL, *prev = NULL; 741 unsigned int periods = buf_len / period_len; 742 int i; 743 unsigned long irqflags; 744 745 dev_dbg(chan2dev(chan), "%s: buf_addr=%pad, buf_len=%zd, period_len=%zd, dir=%s, flags=0x%lx\n", 746 __func__, &buf_addr, buf_len, period_len, 747 direction == DMA_MEM_TO_DEV ? "mem2per" : "per2mem", flags); 748 749 if (!is_slave_direction(direction)) { 750 dev_err(chan2dev(chan), "invalid DMA direction\n"); 751 return NULL; 752 } 753 754 if (test_and_set_bit(AT_XDMAC_CHAN_IS_CYCLIC, &atchan->status)) { 755 dev_err(chan2dev(chan), "channel currently used\n"); 756 return NULL; 757 } 758 759 if (at_xdmac_compute_chan_conf(chan, direction)) 760 return NULL; 761 762 for (i = 0; i < periods; i++) { 763 struct at_xdmac_desc *desc = NULL; 764 765 spin_lock_irqsave(&atchan->lock, irqflags); 766 desc = at_xdmac_get_desc(atchan); 767 if (!desc) { 768 dev_err(chan2dev(chan), "can't get descriptor\n"); 769 if (first) 770 list_splice_tail_init(&first->descs_list, 771 &atchan->free_descs_list); 772 spin_unlock_irqrestore(&atchan->lock, irqflags); 773 return NULL; 774 } 775 spin_unlock_irqrestore(&atchan->lock, irqflags); 776 dev_dbg(chan2dev(chan), 777 "%s: desc=0x%p, tx_dma_desc.phys=%pad\n", 778 __func__, desc, &desc->tx_dma_desc.phys); 779 780 if (direction == DMA_DEV_TO_MEM) { 781 desc->lld.mbr_sa = atchan->sconfig.src_addr; 782 desc->lld.mbr_da = buf_addr + i * period_len; 783 } else { 784 desc->lld.mbr_sa = buf_addr + i * period_len; 785 desc->lld.mbr_da = atchan->sconfig.dst_addr; 786 } 787 desc->lld.mbr_cfg = atchan->cfg; 788 desc->lld.mbr_ubc = AT_XDMAC_MBR_UBC_NDV1 789 | AT_XDMAC_MBR_UBC_NDEN 790 | AT_XDMAC_MBR_UBC_NSEN 791 | period_len >> at_xdmac_get_dwidth(desc->lld.mbr_cfg); 792 793 dev_dbg(chan2dev(chan), 794 "%s: lld: mbr_sa=%pad, mbr_da=%pad, mbr_ubc=0x%08x\n", 795 __func__, &desc->lld.mbr_sa, &desc->lld.mbr_da, desc->lld.mbr_ubc); 796 797 /* Chain lld. */ 798 if (prev) 799 at_xdmac_queue_desc(chan, prev, desc); 800 801 prev = desc; 802 if (!first) 803 first = desc; 804 805 dev_dbg(chan2dev(chan), "%s: add desc 0x%p to descs_list 0x%p\n", 806 __func__, desc, first); 807 list_add_tail(&desc->desc_node, &first->descs_list); 808 } 809 810 at_xdmac_queue_desc(chan, prev, first); 811 first->tx_dma_desc.flags = flags; 812 first->xfer_size = buf_len; 813 first->direction = direction; 814 815 return &first->tx_dma_desc; 816} 817 818static inline u32 at_xdmac_align_width(struct dma_chan *chan, dma_addr_t addr) 819{ 820 u32 width; 821 822 /* 823 * Check address alignment to select the greater data width we 824 * can use. 825 * 826 * Some XDMAC implementations don't provide dword transfer, in 827 * this case selecting dword has the same behavior as 828 * selecting word transfers. 829 */ 830 if (!(addr & 7)) { 831 width = AT_XDMAC_CC_DWIDTH_DWORD; 832 dev_dbg(chan2dev(chan), "%s: dwidth: double word\n", __func__); 833 } else if (!(addr & 3)) { 834 width = AT_XDMAC_CC_DWIDTH_WORD; 835 dev_dbg(chan2dev(chan), "%s: dwidth: word\n", __func__); 836 } else if (!(addr & 1)) { 837 width = AT_XDMAC_CC_DWIDTH_HALFWORD; 838 dev_dbg(chan2dev(chan), "%s: dwidth: half word\n", __func__); 839 } else { 840 width = AT_XDMAC_CC_DWIDTH_BYTE; 841 dev_dbg(chan2dev(chan), "%s: dwidth: byte\n", __func__); 842 } 843 844 return width; 845} 846 847static struct at_xdmac_desc * 848at_xdmac_interleaved_queue_desc(struct dma_chan *chan, 849 struct at_xdmac_chan *atchan, 850 struct at_xdmac_desc *prev, 851 dma_addr_t src, dma_addr_t dst, 852 struct dma_interleaved_template *xt, 853 struct data_chunk *chunk) 854{ 855 struct at_xdmac_desc *desc; 856 u32 dwidth; 857 unsigned long flags; 858 size_t ublen; 859 /* 860 * WARNING: The channel configuration is set here since there is no 861 * dmaengine_slave_config call in this case. Moreover we don't know the 862 * direction, it involves we can't dynamically set the source and dest 863 * interface so we have to use the same one. Only interface 0 allows EBI 864 * access. Hopefully we can access DDR through both ports (at least on 865 * SAMA5D4x), so we can use the same interface for source and dest, 866 * that solves the fact we don't know the direction. 867 * ERRATA: Even if useless for memory transfers, the PERID has to not 868 * match the one of another channel. If not, it could lead to spurious 869 * flag status. 870 */ 871 u32 chan_cc = AT_XDMAC_CC_PERID(0x3f) 872 | AT_XDMAC_CC_DIF(0) 873 | AT_XDMAC_CC_SIF(0) 874 | AT_XDMAC_CC_MBSIZE_SIXTEEN 875 | AT_XDMAC_CC_TYPE_MEM_TRAN; 876 877 dwidth = at_xdmac_align_width(chan, src | dst | chunk->size); 878 if (chunk->size >= (AT_XDMAC_MBR_UBC_UBLEN_MAX << dwidth)) { 879 dev_dbg(chan2dev(chan), 880 "%s: chunk too big (%zu, max size %lu)...\n", 881 __func__, chunk->size, 882 AT_XDMAC_MBR_UBC_UBLEN_MAX << dwidth); 883 return NULL; 884 } 885 886 if (prev) 887 dev_dbg(chan2dev(chan), 888 "Adding items at the end of desc 0x%p\n", prev); 889 890 if (xt->src_inc) { 891 if (xt->src_sgl) 892 chan_cc |= AT_XDMAC_CC_SAM_UBS_AM; 893 else 894 chan_cc |= AT_XDMAC_CC_SAM_INCREMENTED_AM; 895 } 896 897 if (xt->dst_inc) { 898 if (xt->dst_sgl) 899 chan_cc |= AT_XDMAC_CC_DAM_UBS_AM; 900 else 901 chan_cc |= AT_XDMAC_CC_DAM_INCREMENTED_AM; 902 } 903 904 spin_lock_irqsave(&atchan->lock, flags); 905 desc = at_xdmac_get_desc(atchan); 906 spin_unlock_irqrestore(&atchan->lock, flags); 907 if (!desc) { 908 dev_err(chan2dev(chan), "can't get descriptor\n"); 909 return NULL; 910 } 911 912 chan_cc |= AT_XDMAC_CC_DWIDTH(dwidth); 913 914 ublen = chunk->size >> dwidth; 915 916 desc->lld.mbr_sa = src; 917 desc->lld.mbr_da = dst; 918 desc->lld.mbr_sus = dmaengine_get_src_icg(xt, chunk); 919 desc->lld.mbr_dus = dmaengine_get_dst_icg(xt, chunk); 920 921 desc->lld.mbr_ubc = AT_XDMAC_MBR_UBC_NDV3 922 | AT_XDMAC_MBR_UBC_NDEN 923 | AT_XDMAC_MBR_UBC_NSEN 924 | ublen; 925 desc->lld.mbr_cfg = chan_cc; 926 927 dev_dbg(chan2dev(chan), 928 "%s: lld: mbr_sa=%pad, mbr_da=%pad, mbr_ubc=0x%08x, mbr_cfg=0x%08x\n", 929 __func__, &desc->lld.mbr_sa, &desc->lld.mbr_da, 930 desc->lld.mbr_ubc, desc->lld.mbr_cfg); 931 932 /* Chain lld. */ 933 if (prev) 934 at_xdmac_queue_desc(chan, prev, desc); 935 936 return desc; 937} 938 939static struct dma_async_tx_descriptor * 940at_xdmac_prep_interleaved(struct dma_chan *chan, 941 struct dma_interleaved_template *xt, 942 unsigned long flags) 943{ 944 struct at_xdmac_chan *atchan = to_at_xdmac_chan(chan); 945 struct at_xdmac_desc *prev = NULL, *first = NULL; 946 dma_addr_t dst_addr, src_addr; 947 size_t src_skip = 0, dst_skip = 0, len = 0; 948 struct data_chunk *chunk; 949 int i; 950 951 if (!xt || !xt->numf || (xt->dir != DMA_MEM_TO_MEM)) 952 return NULL; 953 954 /* 955 * TODO: Handle the case where we have to repeat a chain of 956 * descriptors... 957 */ 958 if ((xt->numf > 1) && (xt->frame_size > 1)) 959 return NULL; 960 961 dev_dbg(chan2dev(chan), "%s: src=%pad, dest=%pad, numf=%zu, frame_size=%zu, flags=0x%lx\n", 962 __func__, &xt->src_start, &xt->dst_start, xt->numf, 963 xt->frame_size, flags); 964 965 src_addr = xt->src_start; 966 dst_addr = xt->dst_start; 967 968 if (xt->numf > 1) { 969 first = at_xdmac_interleaved_queue_desc(chan, atchan, 970 NULL, 971 src_addr, dst_addr, 972 xt, xt->sgl); 973 if (!first) 974 return NULL; 975 976 /* Length of the block is (BLEN+1) microblocks. */ 977 for (i = 0; i < xt->numf - 1; i++) 978 at_xdmac_increment_block_count(chan, first); 979 980 dev_dbg(chan2dev(chan), "%s: add desc 0x%p to descs_list 0x%p\n", 981 __func__, first, first); 982 list_add_tail(&first->desc_node, &first->descs_list); 983 } else { 984 for (i = 0; i < xt->frame_size; i++) { 985 size_t src_icg = 0, dst_icg = 0; 986 struct at_xdmac_desc *desc; 987 988 chunk = xt->sgl + i; 989 990 dst_icg = dmaengine_get_dst_icg(xt, chunk); 991 src_icg = dmaengine_get_src_icg(xt, chunk); 992 993 src_skip = chunk->size + src_icg; 994 dst_skip = chunk->size + dst_icg; 995 996 dev_dbg(chan2dev(chan), 997 "%s: chunk size=%zu, src icg=%zu, dst icg=%zu\n", 998 __func__, chunk->size, src_icg, dst_icg); 999 1000 desc = at_xdmac_interleaved_queue_desc(chan, atchan, 1001 prev, 1002 src_addr, dst_addr, 1003 xt, chunk); 1004 if (!desc) { 1005 if (first) 1006 list_splice_tail_init(&first->descs_list, 1007 &atchan->free_descs_list); 1008 return NULL; 1009 } 1010 1011 if (!first) 1012 first = desc; 1013 1014 dev_dbg(chan2dev(chan), "%s: add desc 0x%p to descs_list 0x%p\n", 1015 __func__, desc, first); 1016 list_add_tail(&desc->desc_node, &first->descs_list); 1017 1018 if (xt->src_sgl) 1019 src_addr += src_skip; 1020 1021 if (xt->dst_sgl) 1022 dst_addr += dst_skip; 1023 1024 len += chunk->size; 1025 prev = desc; 1026 } 1027 } 1028 1029 first->tx_dma_desc.cookie = -EBUSY; 1030 first->tx_dma_desc.flags = flags; 1031 first->xfer_size = len; 1032 1033 return &first->tx_dma_desc; 1034} 1035 1036static struct dma_async_tx_descriptor * 1037at_xdmac_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src, 1038 size_t len, unsigned long flags) 1039{ 1040 struct at_xdmac_chan *atchan = to_at_xdmac_chan(chan); 1041 struct at_xdmac_desc *first = NULL, *prev = NULL; 1042 size_t remaining_size = len, xfer_size = 0, ublen; 1043 dma_addr_t src_addr = src, dst_addr = dest; 1044 u32 dwidth; 1045 /* 1046 * WARNING: We don't know the direction, it involves we can't 1047 * dynamically set the source and dest interface so we have to use the 1048 * same one. Only interface 0 allows EBI access. Hopefully we can 1049 * access DDR through both ports (at least on SAMA5D4x), so we can use 1050 * the same interface for source and dest, that solves the fact we 1051 * don't know the direction. 1052 * ERRATA: Even if useless for memory transfers, the PERID has to not 1053 * match the one of another channel. If not, it could lead to spurious 1054 * flag status. 1055 */ 1056 u32 chan_cc = AT_XDMAC_CC_PERID(0x3f) 1057 | AT_XDMAC_CC_DAM_INCREMENTED_AM 1058 | AT_XDMAC_CC_SAM_INCREMENTED_AM 1059 | AT_XDMAC_CC_DIF(0) 1060 | AT_XDMAC_CC_SIF(0) 1061 | AT_XDMAC_CC_MBSIZE_SIXTEEN 1062 | AT_XDMAC_CC_TYPE_MEM_TRAN; 1063 unsigned long irqflags; 1064 1065 dev_dbg(chan2dev(chan), "%s: src=%pad, dest=%pad, len=%zd, flags=0x%lx\n", 1066 __func__, &src, &dest, len, flags); 1067 1068 if (unlikely(!len)) 1069 return NULL; 1070 1071 dwidth = at_xdmac_align_width(chan, src_addr | dst_addr); 1072 1073 /* Prepare descriptors. */ 1074 while (remaining_size) { 1075 struct at_xdmac_desc *desc = NULL; 1076 1077 dev_dbg(chan2dev(chan), "%s: remaining_size=%zu\n", __func__, remaining_size); 1078 1079 spin_lock_irqsave(&atchan->lock, irqflags); 1080 desc = at_xdmac_get_desc(atchan); 1081 spin_unlock_irqrestore(&atchan->lock, irqflags); 1082 if (!desc) { 1083 dev_err(chan2dev(chan), "can't get descriptor\n"); 1084 if (first) 1085 list_splice_tail_init(&first->descs_list, 1086 &atchan->free_descs_list); 1087 return NULL; 1088 } 1089 1090 /* Update src and dest addresses. */ 1091 src_addr += xfer_size; 1092 dst_addr += xfer_size; 1093 1094 if (remaining_size >= AT_XDMAC_MBR_UBC_UBLEN_MAX << dwidth) 1095 xfer_size = AT_XDMAC_MBR_UBC_UBLEN_MAX << dwidth; 1096 else 1097 xfer_size = remaining_size; 1098 1099 dev_dbg(chan2dev(chan), "%s: xfer_size=%zu\n", __func__, xfer_size); 1100 1101 /* Check remaining length and change data width if needed. */ 1102 dwidth = at_xdmac_align_width(chan, 1103 src_addr | dst_addr | xfer_size); 1104 chan_cc &= ~AT_XDMAC_CC_DWIDTH_MASK; 1105 chan_cc |= AT_XDMAC_CC_DWIDTH(dwidth); 1106 1107 ublen = xfer_size >> dwidth; 1108 remaining_size -= xfer_size; 1109 1110 desc->lld.mbr_sa = src_addr; 1111 desc->lld.mbr_da = dst_addr; 1112 desc->lld.mbr_ubc = AT_XDMAC_MBR_UBC_NDV2 1113 | AT_XDMAC_MBR_UBC_NDEN 1114 | AT_XDMAC_MBR_UBC_NSEN 1115 | ublen; 1116 desc->lld.mbr_cfg = chan_cc; 1117 1118 dev_dbg(chan2dev(chan), 1119 "%s: lld: mbr_sa=%pad, mbr_da=%pad, mbr_ubc=0x%08x, mbr_cfg=0x%08x\n", 1120 __func__, &desc->lld.mbr_sa, &desc->lld.mbr_da, desc->lld.mbr_ubc, desc->lld.mbr_cfg); 1121 1122 /* Chain lld. */ 1123 if (prev) 1124 at_xdmac_queue_desc(chan, prev, desc); 1125 1126 prev = desc; 1127 if (!first) 1128 first = desc; 1129 1130 dev_dbg(chan2dev(chan), "%s: add desc 0x%p to descs_list 0x%p\n", 1131 __func__, desc, first); 1132 list_add_tail(&desc->desc_node, &first->descs_list); 1133 } 1134 1135 first->tx_dma_desc.flags = flags; 1136 first->xfer_size = len; 1137 1138 return &first->tx_dma_desc; 1139} 1140 1141static struct at_xdmac_desc *at_xdmac_memset_create_desc(struct dma_chan *chan, 1142 struct at_xdmac_chan *atchan, 1143 dma_addr_t dst_addr, 1144 size_t len, 1145 int value) 1146{ 1147 struct at_xdmac_desc *desc; 1148 unsigned long flags; 1149 size_t ublen; 1150 u32 dwidth; 1151 /* 1152 * WARNING: The channel configuration is set here since there is no 1153 * dmaengine_slave_config call in this case. Moreover we don't know the 1154 * direction, it involves we can't dynamically set the source and dest 1155 * interface so we have to use the same one. Only interface 0 allows EBI 1156 * access. Hopefully we can access DDR through both ports (at least on 1157 * SAMA5D4x), so we can use the same interface for source and dest, 1158 * that solves the fact we don't know the direction. 1159 * ERRATA: Even if useless for memory transfers, the PERID has to not 1160 * match the one of another channel. If not, it could lead to spurious 1161 * flag status. 1162 */ 1163 u32 chan_cc = AT_XDMAC_CC_PERID(0x3f) 1164 | AT_XDMAC_CC_DAM_UBS_AM 1165 | AT_XDMAC_CC_SAM_INCREMENTED_AM 1166 | AT_XDMAC_CC_DIF(0) 1167 | AT_XDMAC_CC_SIF(0) 1168 | AT_XDMAC_CC_MBSIZE_SIXTEEN 1169 | AT_XDMAC_CC_MEMSET_HW_MODE 1170 | AT_XDMAC_CC_TYPE_MEM_TRAN; 1171 1172 dwidth = at_xdmac_align_width(chan, dst_addr); 1173 1174 if (len >= (AT_XDMAC_MBR_UBC_UBLEN_MAX << dwidth)) { 1175 dev_err(chan2dev(chan), 1176 "%s: Transfer too large, aborting...\n", 1177 __func__); 1178 return NULL; 1179 } 1180 1181 spin_lock_irqsave(&atchan->lock, flags); 1182 desc = at_xdmac_get_desc(atchan); 1183 spin_unlock_irqrestore(&atchan->lock, flags); 1184 if (!desc) { 1185 dev_err(chan2dev(chan), "can't get descriptor\n"); 1186 return NULL; 1187 } 1188 1189 chan_cc |= AT_XDMAC_CC_DWIDTH(dwidth); 1190 1191 ublen = len >> dwidth; 1192 1193 desc->lld.mbr_da = dst_addr; 1194 desc->lld.mbr_ds = value; 1195 desc->lld.mbr_ubc = AT_XDMAC_MBR_UBC_NDV3 1196 | AT_XDMAC_MBR_UBC_NDEN 1197 | AT_XDMAC_MBR_UBC_NSEN 1198 | ublen; 1199 desc->lld.mbr_cfg = chan_cc; 1200 1201 dev_dbg(chan2dev(chan), 1202 "%s: lld: mbr_da=%pad, mbr_ds=0x%08x, mbr_ubc=0x%08x, mbr_cfg=0x%08x\n", 1203 __func__, &desc->lld.mbr_da, desc->lld.mbr_ds, desc->lld.mbr_ubc, 1204 desc->lld.mbr_cfg); 1205 1206 return desc; 1207} 1208 1209static struct dma_async_tx_descriptor * 1210at_xdmac_prep_dma_memset(struct dma_chan *chan, dma_addr_t dest, int value, 1211 size_t len, unsigned long flags) 1212{ 1213 struct at_xdmac_chan *atchan = to_at_xdmac_chan(chan); 1214 struct at_xdmac_desc *desc; 1215 1216 dev_dbg(chan2dev(chan), "%s: dest=%pad, len=%zu, pattern=0x%x, flags=0x%lx\n", 1217 __func__, &dest, len, value, flags); 1218 1219 if (unlikely(!len)) 1220 return NULL; 1221 1222 desc = at_xdmac_memset_create_desc(chan, atchan, dest, len, value); 1223 list_add_tail(&desc->desc_node, &desc->descs_list); 1224 1225 desc->tx_dma_desc.cookie = -EBUSY; 1226 desc->tx_dma_desc.flags = flags; 1227 desc->xfer_size = len; 1228 1229 return &desc->tx_dma_desc; 1230} 1231 1232static struct dma_async_tx_descriptor * 1233at_xdmac_prep_dma_memset_sg(struct dma_chan *chan, struct scatterlist *sgl, 1234 unsigned int sg_len, int value, 1235 unsigned long flags) 1236{ 1237 struct at_xdmac_chan *atchan = to_at_xdmac_chan(chan); 1238 struct at_xdmac_desc *desc, *pdesc = NULL, 1239 *ppdesc = NULL, *first = NULL; 1240 struct scatterlist *sg, *psg = NULL, *ppsg = NULL; 1241 size_t stride = 0, pstride = 0, len = 0; 1242 int i; 1243 1244 if (!sgl) 1245 return NULL; 1246 1247 dev_dbg(chan2dev(chan), "%s: sg_len=%d, value=0x%x, flags=0x%lx\n", 1248 __func__, sg_len, value, flags); 1249 1250 /* Prepare descriptors. */ 1251 for_each_sg(sgl, sg, sg_len, i) { 1252 dev_dbg(chan2dev(chan), "%s: dest=%pad, len=%d, pattern=0x%x, flags=0x%lx\n", 1253 __func__, &sg_dma_address(sg), sg_dma_len(sg), 1254 value, flags); 1255 desc = at_xdmac_memset_create_desc(chan, atchan, 1256 sg_dma_address(sg), 1257 sg_dma_len(sg), 1258 value); 1259 if (!desc && first) 1260 list_splice_tail_init(&first->descs_list, 1261 &atchan->free_descs_list); 1262 1263 if (!first) 1264 first = desc; 1265 1266 /* Update our strides */ 1267 pstride = stride; 1268 if (psg) 1269 stride = sg_dma_address(sg) - 1270 (sg_dma_address(psg) + sg_dma_len(psg)); 1271 1272 /* 1273 * The scatterlist API gives us only the address and 1274 * length of each elements. 1275 * 1276 * Unfortunately, we don't have the stride, which we 1277 * will need to compute. 1278 * 1279 * That make us end up in a situation like this one: 1280 * len stride len stride len 1281 * +-------+ +-------+ +-------+ 1282 * | N-2 | | N-1 | | N | 1283 * +-------+ +-------+ +-------+ 1284 * 1285 * We need all these three elements (N-2, N-1 and N) 1286 * to actually take the decision on whether we need to 1287 * queue N-1 or reuse N-2. 1288 * 1289 * We will only consider N if it is the last element. 1290 */ 1291 if (ppdesc && pdesc) { 1292 if ((stride == pstride) && 1293 (sg_dma_len(ppsg) == sg_dma_len(psg))) { 1294 dev_dbg(chan2dev(chan), 1295 "%s: desc 0x%p can be merged with desc 0x%p\n", 1296 __func__, pdesc, ppdesc); 1297 1298 /* 1299 * Increment the block count of the 1300 * N-2 descriptor 1301 */ 1302 at_xdmac_increment_block_count(chan, ppdesc); 1303 ppdesc->lld.mbr_dus = stride; 1304 1305 /* 1306 * Put back the N-1 descriptor in the 1307 * free descriptor list 1308 */ 1309 list_add_tail(&pdesc->desc_node, 1310 &atchan->free_descs_list); 1311 1312 /* 1313 * Make our N-1 descriptor pointer 1314 * point to the N-2 since they were 1315 * actually merged. 1316 */ 1317 pdesc = ppdesc; 1318 1319 /* 1320 * Rule out the case where we don't have 1321 * pstride computed yet (our second sg 1322 * element) 1323 * 1324 * We also want to catch the case where there 1325 * would be a negative stride, 1326 */ 1327 } else if (pstride || 1328 sg_dma_address(sg) < sg_dma_address(psg)) { 1329 /* 1330 * Queue the N-1 descriptor after the 1331 * N-2 1332 */ 1333 at_xdmac_queue_desc(chan, ppdesc, pdesc); 1334 1335 /* 1336 * Add the N-1 descriptor to the list 1337 * of the descriptors used for this 1338 * transfer 1339 */ 1340 list_add_tail(&desc->desc_node, 1341 &first->descs_list); 1342 dev_dbg(chan2dev(chan), 1343 "%s: add desc 0x%p to descs_list 0x%p\n", 1344 __func__, desc, first); 1345 } 1346 } 1347 1348 /* 1349 * If we are the last element, just see if we have the 1350 * same size than the previous element. 1351 * 1352 * If so, we can merge it with the previous descriptor 1353 * since we don't care about the stride anymore. 1354 */ 1355 if ((i == (sg_len - 1)) && 1356 sg_dma_len(psg) == sg_dma_len(sg)) { 1357 dev_dbg(chan2dev(chan), 1358 "%s: desc 0x%p can be merged with desc 0x%p\n", 1359 __func__, desc, pdesc); 1360 1361 /* 1362 * Increment the block count of the N-1 1363 * descriptor 1364 */ 1365 at_xdmac_increment_block_count(chan, pdesc); 1366 pdesc->lld.mbr_dus = stride; 1367 1368 /* 1369 * Put back the N descriptor in the free 1370 * descriptor list 1371 */ 1372 list_add_tail(&desc->desc_node, 1373 &atchan->free_descs_list); 1374 } 1375 1376 /* Update our descriptors */ 1377 ppdesc = pdesc; 1378 pdesc = desc; 1379 1380 /* Update our scatter pointers */ 1381 ppsg = psg; 1382 psg = sg; 1383 1384 len += sg_dma_len(sg); 1385 } 1386 1387 first->tx_dma_desc.cookie = -EBUSY; 1388 first->tx_dma_desc.flags = flags; 1389 first->xfer_size = len; 1390 1391 return &first->tx_dma_desc; 1392} 1393 1394static enum dma_status 1395at_xdmac_tx_status(struct dma_chan *chan, dma_cookie_t cookie, 1396 struct dma_tx_state *txstate) 1397{ 1398 struct at_xdmac_chan *atchan = to_at_xdmac_chan(chan); 1399 struct at_xdmac *atxdmac = to_at_xdmac(atchan->chan.device); 1400 struct at_xdmac_desc *desc, *_desc, *iter; 1401 struct list_head *descs_list; 1402 enum dma_status ret; 1403 int residue, retry; 1404 u32 cur_nda, check_nda, cur_ubc, mask, value; 1405 u8 dwidth = 0; 1406 unsigned long flags; 1407 bool initd; 1408 1409 ret = dma_cookie_status(chan, cookie, txstate); 1410 if (ret == DMA_COMPLETE) 1411 return ret; 1412 1413 if (!txstate) 1414 return ret; 1415 1416 spin_lock_irqsave(&atchan->lock, flags); 1417 1418 desc = list_first_entry(&atchan->xfers_list, struct at_xdmac_desc, xfer_node); 1419 1420 /* 1421 * If the transfer has not been started yet, don't need to compute the 1422 * residue, it's the transfer length. 1423 */ 1424 if (!desc->active_xfer) { 1425 dma_set_residue(txstate, desc->xfer_size); 1426 goto spin_unlock; 1427 } 1428 1429 residue = desc->xfer_size; 1430 /* 1431 * Flush FIFO: only relevant when the transfer is source peripheral 1432 * synchronized. Flush is needed before reading CUBC because data in 1433 * the FIFO are not reported by CUBC. Reporting a residue of the 1434 * transfer length while we have data in FIFO can cause issue. 1435 * Usecase: atmel USART has a timeout which means I have received 1436 * characters but there is no more character received for a while. On 1437 * timeout, it requests the residue. If the data are in the DMA FIFO, 1438 * we will return a residue of the transfer length. It means no data 1439 * received. If an application is waiting for these data, it will hang 1440 * since we won't have another USART timeout without receiving new 1441 * data. 1442 */ 1443 mask = AT_XDMAC_CC_TYPE | AT_XDMAC_CC_DSYNC; 1444 value = AT_XDMAC_CC_TYPE_PER_TRAN | AT_XDMAC_CC_DSYNC_PER2MEM; 1445 if ((desc->lld.mbr_cfg & mask) == value) { 1446 at_xdmac_write(atxdmac, AT_XDMAC_GSWF, atchan->mask); 1447 while (!(at_xdmac_chan_read(atchan, AT_XDMAC_CIS) & AT_XDMAC_CIS_FIS)) 1448 cpu_relax(); 1449 } 1450 1451 /* 1452 * The easiest way to compute the residue should be to pause the DMA 1453 * but doing this can lead to miss some data as some devices don't 1454 * have FIFO. 1455 * We need to read several registers because: 1456 * - DMA is running therefore a descriptor change is possible while 1457 * reading these registers 1458 * - When the block transfer is done, the value of the CUBC register 1459 * is set to its initial value until the fetch of the next descriptor. 1460 * This value will corrupt the residue calculation so we have to skip 1461 * it. 1462 * 1463 * INITD -------- ------------ 1464 * |____________________| 1465 * _______________________ _______________ 1466 * NDA @desc2 \/ @desc3 1467 * _______________________/\_______________ 1468 * __________ ___________ _______________ 1469 * CUBC 0 \/ MAX desc1 \/ MAX desc2 1470 * __________/\___________/\_______________ 1471 * 1472 * Since descriptors are aligned on 64 bits, we can assume that 1473 * the update of NDA and CUBC is atomic. 1474 * Memory barriers are used to ensure the read order of the registers. 1475 * A max number of retries is set because unlikely it could never ends. 1476 */ 1477 for (retry = 0; retry < AT_XDMAC_RESIDUE_MAX_RETRIES; retry++) { 1478 check_nda = at_xdmac_chan_read(atchan, AT_XDMAC_CNDA) & 0xfffffffc; 1479 rmb(); 1480 cur_ubc = at_xdmac_chan_read(atchan, AT_XDMAC_CUBC); 1481 rmb(); 1482 initd = !!(at_xdmac_chan_read(atchan, AT_XDMAC_CC) & AT_XDMAC_CC_INITD); 1483 rmb(); 1484 cur_nda = at_xdmac_chan_read(atchan, AT_XDMAC_CNDA) & 0xfffffffc; 1485 rmb(); 1486 1487 if ((check_nda == cur_nda) && initd) 1488 break; 1489 } 1490 1491 if (unlikely(retry >= AT_XDMAC_RESIDUE_MAX_RETRIES)) { 1492 ret = DMA_ERROR; 1493 goto spin_unlock; 1494 } 1495 1496 /* 1497 * Flush FIFO: only relevant when the transfer is source peripheral 1498 * synchronized. Another flush is needed here because CUBC is updated 1499 * when the controller sends the data write command. It can lead to 1500 * report data that are not written in the memory or the device. The 1501 * FIFO flush ensures that data are really written. 1502 */ 1503 if ((desc->lld.mbr_cfg & mask) == value) { 1504 at_xdmac_write(atxdmac, AT_XDMAC_GSWF, atchan->mask); 1505 while (!(at_xdmac_chan_read(atchan, AT_XDMAC_CIS) & AT_XDMAC_CIS_FIS)) 1506 cpu_relax(); 1507 } 1508 1509 /* 1510 * Remove size of all microblocks already transferred and the current 1511 * one. Then add the remaining size to transfer of the current 1512 * microblock. 1513 */ 1514 descs_list = &desc->descs_list; 1515 list_for_each_entry_safe(iter, _desc, descs_list, desc_node) { 1516 dwidth = at_xdmac_get_dwidth(iter->lld.mbr_cfg); 1517 residue -= (iter->lld.mbr_ubc & 0xffffff) << dwidth; 1518 if ((iter->lld.mbr_nda & 0xfffffffc) == cur_nda) { 1519 desc = iter; 1520 break; 1521 } 1522 } 1523 residue += cur_ubc << dwidth; 1524 1525 dma_set_residue(txstate, residue); 1526 1527 dev_dbg(chan2dev(chan), 1528 "%s: desc=0x%p, tx_dma_desc.phys=%pad, tx_status=%d, cookie=%d, residue=%d\n", 1529 __func__, desc, &desc->tx_dma_desc.phys, ret, cookie, residue); 1530 1531spin_unlock: 1532 spin_unlock_irqrestore(&atchan->lock, flags); 1533 return ret; 1534} 1535 1536static void at_xdmac_advance_work(struct at_xdmac_chan *atchan) 1537{ 1538 struct at_xdmac_desc *desc; 1539 1540 /* 1541 * If channel is enabled, do nothing, advance_work will be triggered 1542 * after the interruption. 1543 */ 1544 if (!at_xdmac_chan_is_enabled(atchan) && !list_empty(&atchan->xfers_list)) { 1545 desc = list_first_entry(&atchan->xfers_list, 1546 struct at_xdmac_desc, 1547 xfer_node); 1548 dev_vdbg(chan2dev(&atchan->chan), "%s: desc 0x%p\n", __func__, desc); 1549 if (!desc->active_xfer) 1550 at_xdmac_start_xfer(atchan, desc); 1551 } 1552} 1553 1554static void at_xdmac_handle_cyclic(struct at_xdmac_chan *atchan) 1555{ 1556 struct at_xdmac_desc *desc; 1557 struct dma_async_tx_descriptor *txd; 1558 1559 spin_lock_irq(&atchan->lock); 1560 if (list_empty(&atchan->xfers_list)) { 1561 spin_unlock_irq(&atchan->lock); 1562 return; 1563 } 1564 desc = list_first_entry(&atchan->xfers_list, struct at_xdmac_desc, 1565 xfer_node); 1566 spin_unlock_irq(&atchan->lock); 1567 txd = &desc->tx_dma_desc; 1568 if (txd->flags & DMA_PREP_INTERRUPT) 1569 dmaengine_desc_get_callback_invoke(txd, NULL); 1570} 1571 1572static void at_xdmac_handle_error(struct at_xdmac_chan *atchan) 1573{ 1574 struct at_xdmac *atxdmac = to_at_xdmac(atchan->chan.device); 1575 struct at_xdmac_desc *bad_desc; 1576 1577 /* 1578 * The descriptor currently at the head of the active list is 1579 * broken. Since we don't have any way to report errors, we'll 1580 * just have to scream loudly and try to continue with other 1581 * descriptors queued (if any). 1582 */ 1583 if (atchan->irq_status & AT_XDMAC_CIS_RBEIS) 1584 dev_err(chan2dev(&atchan->chan), "read bus error!!!"); 1585 if (atchan->irq_status & AT_XDMAC_CIS_WBEIS) 1586 dev_err(chan2dev(&atchan->chan), "write bus error!!!"); 1587 if (atchan->irq_status & AT_XDMAC_CIS_ROIS) 1588 dev_err(chan2dev(&atchan->chan), "request overflow error!!!"); 1589 1590 spin_lock_irq(&atchan->lock); 1591 1592 /* Channel must be disabled first as it's not done automatically */ 1593 at_xdmac_write(atxdmac, AT_XDMAC_GD, atchan->mask); 1594 while (at_xdmac_read(atxdmac, AT_XDMAC_GS) & atchan->mask) 1595 cpu_relax(); 1596 1597 bad_desc = list_first_entry(&atchan->xfers_list, 1598 struct at_xdmac_desc, 1599 xfer_node); 1600 1601 spin_unlock_irq(&atchan->lock); 1602 1603 /* Print bad descriptor's details if needed */ 1604 dev_dbg(chan2dev(&atchan->chan), 1605 "%s: lld: mbr_sa=%pad, mbr_da=%pad, mbr_ubc=0x%08x\n", 1606 __func__, &bad_desc->lld.mbr_sa, &bad_desc->lld.mbr_da, 1607 bad_desc->lld.mbr_ubc); 1608 1609 /* Then continue with usual descriptor management */ 1610} 1611 1612static void at_xdmac_tasklet(struct tasklet_struct *t) 1613{ 1614 struct at_xdmac_chan *atchan = from_tasklet(atchan, t, tasklet); 1615 struct at_xdmac_desc *desc; 1616 u32 error_mask; 1617 1618 dev_dbg(chan2dev(&atchan->chan), "%s: status=0x%08x\n", 1619 __func__, atchan->irq_status); 1620 1621 error_mask = AT_XDMAC_CIS_RBEIS 1622 | AT_XDMAC_CIS_WBEIS 1623 | AT_XDMAC_CIS_ROIS; 1624 1625 if (at_xdmac_chan_is_cyclic(atchan)) { 1626 at_xdmac_handle_cyclic(atchan); 1627 } else if ((atchan->irq_status & AT_XDMAC_CIS_LIS) 1628 || (atchan->irq_status & error_mask)) { 1629 struct dma_async_tx_descriptor *txd; 1630 1631 if (atchan->irq_status & error_mask) 1632 at_xdmac_handle_error(atchan); 1633 1634 spin_lock_irq(&atchan->lock); 1635 desc = list_first_entry(&atchan->xfers_list, 1636 struct at_xdmac_desc, 1637 xfer_node); 1638 dev_vdbg(chan2dev(&atchan->chan), "%s: desc 0x%p\n", __func__, desc); 1639 if (!desc->active_xfer) { 1640 dev_err(chan2dev(&atchan->chan), "Xfer not active: exiting"); 1641 spin_unlock_irq(&atchan->lock); 1642 return; 1643 } 1644 1645 txd = &desc->tx_dma_desc; 1646 dma_cookie_complete(txd); 1647 /* Remove the transfer from the transfer list. */ 1648 list_del(&desc->xfer_node); 1649 spin_unlock_irq(&atchan->lock); 1650 1651 if (txd->flags & DMA_PREP_INTERRUPT) 1652 dmaengine_desc_get_callback_invoke(txd, NULL); 1653 1654 dma_run_dependencies(txd); 1655 1656 spin_lock_irq(&atchan->lock); 1657 /* Move the xfer descriptors into the free descriptors list. */ 1658 list_splice_tail_init(&desc->descs_list, 1659 &atchan->free_descs_list); 1660 at_xdmac_advance_work(atchan); 1661 spin_unlock_irq(&atchan->lock); 1662 } 1663} 1664 1665static irqreturn_t at_xdmac_interrupt(int irq, void *dev_id) 1666{ 1667 struct at_xdmac *atxdmac = (struct at_xdmac *)dev_id; 1668 struct at_xdmac_chan *atchan; 1669 u32 imr, status, pending; 1670 u32 chan_imr, chan_status; 1671 int i, ret = IRQ_NONE; 1672 1673 do { 1674 imr = at_xdmac_read(atxdmac, AT_XDMAC_GIM); 1675 status = at_xdmac_read(atxdmac, AT_XDMAC_GIS); 1676 pending = status & imr; 1677 1678 dev_vdbg(atxdmac->dma.dev, 1679 "%s: status=0x%08x, imr=0x%08x, pending=0x%08x\n", 1680 __func__, status, imr, pending); 1681 1682 if (!pending) 1683 break; 1684 1685 /* We have to find which channel has generated the interrupt. */ 1686 for (i = 0; i < atxdmac->dma.chancnt; i++) { 1687 if (!((1 << i) & pending)) 1688 continue; 1689 1690 atchan = &atxdmac->chan[i]; 1691 chan_imr = at_xdmac_chan_read(atchan, AT_XDMAC_CIM); 1692 chan_status = at_xdmac_chan_read(atchan, AT_XDMAC_CIS); 1693 atchan->irq_status = chan_status & chan_imr; 1694 dev_vdbg(atxdmac->dma.dev, 1695 "%s: chan%d: imr=0x%x, status=0x%x\n", 1696 __func__, i, chan_imr, chan_status); 1697 dev_vdbg(chan2dev(&atchan->chan), 1698 "%s: CC=0x%08x CNDA=0x%08x, CNDC=0x%08x, CSA=0x%08x, CDA=0x%08x, CUBC=0x%08x\n", 1699 __func__, 1700 at_xdmac_chan_read(atchan, AT_XDMAC_CC), 1701 at_xdmac_chan_read(atchan, AT_XDMAC_CNDA), 1702 at_xdmac_chan_read(atchan, AT_XDMAC_CNDC), 1703 at_xdmac_chan_read(atchan, AT_XDMAC_CSA), 1704 at_xdmac_chan_read(atchan, AT_XDMAC_CDA), 1705 at_xdmac_chan_read(atchan, AT_XDMAC_CUBC)); 1706 1707 if (atchan->irq_status & (AT_XDMAC_CIS_RBEIS | AT_XDMAC_CIS_WBEIS)) 1708 at_xdmac_write(atxdmac, AT_XDMAC_GD, atchan->mask); 1709 1710 tasklet_schedule(&atchan->tasklet); 1711 ret = IRQ_HANDLED; 1712 } 1713 1714 } while (pending); 1715 1716 return ret; 1717} 1718 1719static void at_xdmac_issue_pending(struct dma_chan *chan) 1720{ 1721 struct at_xdmac_chan *atchan = to_at_xdmac_chan(chan); 1722 unsigned long flags; 1723 1724 dev_dbg(chan2dev(&atchan->chan), "%s\n", __func__); 1725 1726 spin_lock_irqsave(&atchan->lock, flags); 1727 at_xdmac_advance_work(atchan); 1728 spin_unlock_irqrestore(&atchan->lock, flags); 1729 1730 return; 1731} 1732 1733static int at_xdmac_device_config(struct dma_chan *chan, 1734 struct dma_slave_config *config) 1735{ 1736 struct at_xdmac_chan *atchan = to_at_xdmac_chan(chan); 1737 int ret; 1738 unsigned long flags; 1739 1740 dev_dbg(chan2dev(chan), "%s\n", __func__); 1741 1742 spin_lock_irqsave(&atchan->lock, flags); 1743 ret = at_xdmac_set_slave_config(chan, config); 1744 spin_unlock_irqrestore(&atchan->lock, flags); 1745 1746 return ret; 1747} 1748 1749static int at_xdmac_device_pause(struct dma_chan *chan) 1750{ 1751 struct at_xdmac_chan *atchan = to_at_xdmac_chan(chan); 1752 struct at_xdmac *atxdmac = to_at_xdmac(atchan->chan.device); 1753 unsigned long flags; 1754 1755 dev_dbg(chan2dev(chan), "%s\n", __func__); 1756 1757 if (test_and_set_bit(AT_XDMAC_CHAN_IS_PAUSED, &atchan->status)) 1758 return 0; 1759 1760 spin_lock_irqsave(&atchan->lock, flags); 1761 at_xdmac_write(atxdmac, AT_XDMAC_GRWS, atchan->mask); 1762 while (at_xdmac_chan_read(atchan, AT_XDMAC_CC) 1763 & (AT_XDMAC_CC_WRIP | AT_XDMAC_CC_RDIP)) 1764 cpu_relax(); 1765 spin_unlock_irqrestore(&atchan->lock, flags); 1766 1767 return 0; 1768} 1769 1770static int at_xdmac_device_resume(struct dma_chan *chan) 1771{ 1772 struct at_xdmac_chan *atchan = to_at_xdmac_chan(chan); 1773 struct at_xdmac *atxdmac = to_at_xdmac(atchan->chan.device); 1774 unsigned long flags; 1775 1776 dev_dbg(chan2dev(chan), "%s\n", __func__); 1777 1778 spin_lock_irqsave(&atchan->lock, flags); 1779 if (!at_xdmac_chan_is_paused(atchan)) { 1780 spin_unlock_irqrestore(&atchan->lock, flags); 1781 return 0; 1782 } 1783 1784 at_xdmac_write(atxdmac, AT_XDMAC_GRWR, atchan->mask); 1785 clear_bit(AT_XDMAC_CHAN_IS_PAUSED, &atchan->status); 1786 spin_unlock_irqrestore(&atchan->lock, flags); 1787 1788 return 0; 1789} 1790 1791static int at_xdmac_device_terminate_all(struct dma_chan *chan) 1792{ 1793 struct at_xdmac_desc *desc, *_desc; 1794 struct at_xdmac_chan *atchan = to_at_xdmac_chan(chan); 1795 struct at_xdmac *atxdmac = to_at_xdmac(atchan->chan.device); 1796 unsigned long flags; 1797 1798 dev_dbg(chan2dev(chan), "%s\n", __func__); 1799 1800 spin_lock_irqsave(&atchan->lock, flags); 1801 at_xdmac_write(atxdmac, AT_XDMAC_GD, atchan->mask); 1802 while (at_xdmac_read(atxdmac, AT_XDMAC_GS) & atchan->mask) 1803 cpu_relax(); 1804 1805 /* Cancel all pending transfers. */ 1806 list_for_each_entry_safe(desc, _desc, &atchan->xfers_list, xfer_node) { 1807 list_del(&desc->xfer_node); 1808 list_splice_tail_init(&desc->descs_list, 1809 &atchan->free_descs_list); 1810 } 1811 1812 clear_bit(AT_XDMAC_CHAN_IS_PAUSED, &atchan->status); 1813 clear_bit(AT_XDMAC_CHAN_IS_CYCLIC, &atchan->status); 1814 spin_unlock_irqrestore(&atchan->lock, flags); 1815 1816 return 0; 1817} 1818 1819static int at_xdmac_alloc_chan_resources(struct dma_chan *chan) 1820{ 1821 struct at_xdmac_chan *atchan = to_at_xdmac_chan(chan); 1822 struct at_xdmac_desc *desc; 1823 int i; 1824 1825 if (at_xdmac_chan_is_enabled(atchan)) { 1826 dev_err(chan2dev(chan), 1827 "can't allocate channel resources (channel enabled)\n"); 1828 return -EIO; 1829 } 1830 1831 if (!list_empty(&atchan->free_descs_list)) { 1832 dev_err(chan2dev(chan), 1833 "can't allocate channel resources (channel not free from a previous use)\n"); 1834 return -EIO; 1835 } 1836 1837 for (i = 0; i < init_nr_desc_per_channel; i++) { 1838 desc = at_xdmac_alloc_desc(chan, GFP_KERNEL); 1839 if (!desc) { 1840 if (i == 0) { 1841 dev_warn(chan2dev(chan), 1842 "can't allocate any descriptors\n"); 1843 return -EIO; 1844 } 1845 dev_warn(chan2dev(chan), 1846 "only %d descriptors have been allocated\n", i); 1847 break; 1848 } 1849 list_add_tail(&desc->desc_node, &atchan->free_descs_list); 1850 } 1851 1852 dma_cookie_init(chan); 1853 1854 dev_dbg(chan2dev(chan), "%s: allocated %d descriptors\n", __func__, i); 1855 1856 return i; 1857} 1858 1859static void at_xdmac_free_chan_resources(struct dma_chan *chan) 1860{ 1861 struct at_xdmac_chan *atchan = to_at_xdmac_chan(chan); 1862 struct at_xdmac *atxdmac = to_at_xdmac(chan->device); 1863 struct at_xdmac_desc *desc, *_desc; 1864 1865 list_for_each_entry_safe(desc, _desc, &atchan->free_descs_list, desc_node) { 1866 dev_dbg(chan2dev(chan), "%s: freeing descriptor %p\n", __func__, desc); 1867 list_del(&desc->desc_node); 1868 dma_pool_free(atxdmac->at_xdmac_desc_pool, desc, desc->tx_dma_desc.phys); 1869 } 1870 1871 return; 1872} 1873 1874#ifdef CONFIG_PM 1875static int atmel_xdmac_prepare(struct device *dev) 1876{ 1877 struct at_xdmac *atxdmac = dev_get_drvdata(dev); 1878 struct dma_chan *chan, *_chan; 1879 1880 list_for_each_entry_safe(chan, _chan, &atxdmac->dma.channels, device_node) { 1881 struct at_xdmac_chan *atchan = to_at_xdmac_chan(chan); 1882 1883 /* Wait for transfer completion, except in cyclic case. */ 1884 if (at_xdmac_chan_is_enabled(atchan) && !at_xdmac_chan_is_cyclic(atchan)) 1885 return -EAGAIN; 1886 } 1887 return 0; 1888} 1889#else 1890# define atmel_xdmac_prepare NULL 1891#endif 1892 1893#ifdef CONFIG_PM_SLEEP 1894static int atmel_xdmac_suspend(struct device *dev) 1895{ 1896 struct at_xdmac *atxdmac = dev_get_drvdata(dev); 1897 struct dma_chan *chan, *_chan; 1898 1899 list_for_each_entry_safe(chan, _chan, &atxdmac->dma.channels, device_node) { 1900 struct at_xdmac_chan *atchan = to_at_xdmac_chan(chan); 1901 1902 atchan->save_cc = at_xdmac_chan_read(atchan, AT_XDMAC_CC); 1903 if (at_xdmac_chan_is_cyclic(atchan)) { 1904 if (!at_xdmac_chan_is_paused(atchan)) 1905 at_xdmac_device_pause(chan); 1906 atchan->save_cim = at_xdmac_chan_read(atchan, AT_XDMAC_CIM); 1907 atchan->save_cnda = at_xdmac_chan_read(atchan, AT_XDMAC_CNDA); 1908 atchan->save_cndc = at_xdmac_chan_read(atchan, AT_XDMAC_CNDC); 1909 } 1910 } 1911 atxdmac->save_gim = at_xdmac_read(atxdmac, AT_XDMAC_GIM); 1912 atxdmac->save_gs = at_xdmac_read(atxdmac, AT_XDMAC_GS); 1913 1914 at_xdmac_off(atxdmac); 1915 clk_disable_unprepare(atxdmac->clk); 1916 return 0; 1917} 1918 1919static int atmel_xdmac_resume(struct device *dev) 1920{ 1921 struct at_xdmac *atxdmac = dev_get_drvdata(dev); 1922 struct at_xdmac_chan *atchan; 1923 struct dma_chan *chan, *_chan; 1924 int i; 1925 int ret; 1926 1927 ret = clk_prepare_enable(atxdmac->clk); 1928 if (ret) 1929 return ret; 1930 1931 /* Clear pending interrupts. */ 1932 for (i = 0; i < atxdmac->dma.chancnt; i++) { 1933 atchan = &atxdmac->chan[i]; 1934 while (at_xdmac_chan_read(atchan, AT_XDMAC_CIS)) 1935 cpu_relax(); 1936 } 1937 1938 at_xdmac_write(atxdmac, AT_XDMAC_GIE, atxdmac->save_gim); 1939 list_for_each_entry_safe(chan, _chan, &atxdmac->dma.channels, device_node) { 1940 atchan = to_at_xdmac_chan(chan); 1941 at_xdmac_chan_write(atchan, AT_XDMAC_CC, atchan->save_cc); 1942 if (at_xdmac_chan_is_cyclic(atchan)) { 1943 if (at_xdmac_chan_is_paused(atchan)) 1944 at_xdmac_device_resume(chan); 1945 at_xdmac_chan_write(atchan, AT_XDMAC_CNDA, atchan->save_cnda); 1946 at_xdmac_chan_write(atchan, AT_XDMAC_CNDC, atchan->save_cndc); 1947 at_xdmac_chan_write(atchan, AT_XDMAC_CIE, atchan->save_cim); 1948 wmb(); 1949 if (atxdmac->save_gs & atchan->mask) 1950 at_xdmac_write(atxdmac, AT_XDMAC_GE, atchan->mask); 1951 } 1952 } 1953 return 0; 1954} 1955#endif /* CONFIG_PM_SLEEP */ 1956 1957static int at_xdmac_probe(struct platform_device *pdev) 1958{ 1959 struct at_xdmac *atxdmac; 1960 int irq, size, nr_channels, i, ret; 1961 void __iomem *base; 1962 u32 reg; 1963 1964 irq = platform_get_irq(pdev, 0); 1965 if (irq < 0) 1966 return irq; 1967 1968 base = devm_platform_ioremap_resource(pdev, 0); 1969 if (IS_ERR(base)) 1970 return PTR_ERR(base); 1971 1972 /* 1973 * Read number of xdmac channels, read helper function can't be used 1974 * since atxdmac is not yet allocated and we need to know the number 1975 * of channels to do the allocation. 1976 */ 1977 reg = readl_relaxed(base + AT_XDMAC_GTYPE); 1978 nr_channels = AT_XDMAC_NB_CH(reg); 1979 if (nr_channels > AT_XDMAC_MAX_CHAN) { 1980 dev_err(&pdev->dev, "invalid number of channels (%u)\n", 1981 nr_channels); 1982 return -EINVAL; 1983 } 1984 1985 size = sizeof(*atxdmac); 1986 size += nr_channels * sizeof(struct at_xdmac_chan); 1987 atxdmac = devm_kzalloc(&pdev->dev, size, GFP_KERNEL); 1988 if (!atxdmac) { 1989 dev_err(&pdev->dev, "can't allocate at_xdmac structure\n"); 1990 return -ENOMEM; 1991 } 1992 1993 atxdmac->regs = base; 1994 atxdmac->irq = irq; 1995 1996 atxdmac->clk = devm_clk_get(&pdev->dev, "dma_clk"); 1997 if (IS_ERR(atxdmac->clk)) { 1998 dev_err(&pdev->dev, "can't get dma_clk\n"); 1999 return PTR_ERR(atxdmac->clk); 2000 } 2001 2002 /* Do not use dev res to prevent races with tasklet */ 2003 ret = request_irq(atxdmac->irq, at_xdmac_interrupt, 0, "at_xdmac", atxdmac); 2004 if (ret) { 2005 dev_err(&pdev->dev, "can't request irq\n"); 2006 return ret; 2007 } 2008 2009 ret = clk_prepare_enable(atxdmac->clk); 2010 if (ret) { 2011 dev_err(&pdev->dev, "can't prepare or enable clock\n"); 2012 goto err_free_irq; 2013 } 2014 2015 atxdmac->at_xdmac_desc_pool = 2016 dmam_pool_create(dev_name(&pdev->dev), &pdev->dev, 2017 sizeof(struct at_xdmac_desc), 4, 0); 2018 if (!atxdmac->at_xdmac_desc_pool) { 2019 dev_err(&pdev->dev, "no memory for descriptors dma pool\n"); 2020 ret = -ENOMEM; 2021 goto err_clk_disable; 2022 } 2023 2024 dma_cap_set(DMA_CYCLIC, atxdmac->dma.cap_mask); 2025 dma_cap_set(DMA_INTERLEAVE, atxdmac->dma.cap_mask); 2026 dma_cap_set(DMA_MEMCPY, atxdmac->dma.cap_mask); 2027 dma_cap_set(DMA_MEMSET, atxdmac->dma.cap_mask); 2028 dma_cap_set(DMA_MEMSET_SG, atxdmac->dma.cap_mask); 2029 dma_cap_set(DMA_SLAVE, atxdmac->dma.cap_mask); 2030 /* 2031 * Without DMA_PRIVATE the driver is not able to allocate more than 2032 * one channel, second allocation fails in private_candidate. 2033 */ 2034 dma_cap_set(DMA_PRIVATE, atxdmac->dma.cap_mask); 2035 atxdmac->dma.dev = &pdev->dev; 2036 atxdmac->dma.device_alloc_chan_resources = at_xdmac_alloc_chan_resources; 2037 atxdmac->dma.device_free_chan_resources = at_xdmac_free_chan_resources; 2038 atxdmac->dma.device_tx_status = at_xdmac_tx_status; 2039 atxdmac->dma.device_issue_pending = at_xdmac_issue_pending; 2040 atxdmac->dma.device_prep_dma_cyclic = at_xdmac_prep_dma_cyclic; 2041 atxdmac->dma.device_prep_interleaved_dma = at_xdmac_prep_interleaved; 2042 atxdmac->dma.device_prep_dma_memcpy = at_xdmac_prep_dma_memcpy; 2043 atxdmac->dma.device_prep_dma_memset = at_xdmac_prep_dma_memset; 2044 atxdmac->dma.device_prep_dma_memset_sg = at_xdmac_prep_dma_memset_sg; 2045 atxdmac->dma.device_prep_slave_sg = at_xdmac_prep_slave_sg; 2046 atxdmac->dma.device_config = at_xdmac_device_config; 2047 atxdmac->dma.device_pause = at_xdmac_device_pause; 2048 atxdmac->dma.device_resume = at_xdmac_device_resume; 2049 atxdmac->dma.device_terminate_all = at_xdmac_device_terminate_all; 2050 atxdmac->dma.src_addr_widths = AT_XDMAC_DMA_BUSWIDTHS; 2051 atxdmac->dma.dst_addr_widths = AT_XDMAC_DMA_BUSWIDTHS; 2052 atxdmac->dma.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV); 2053 atxdmac->dma.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST; 2054 2055 /* Disable all chans and interrupts. */ 2056 at_xdmac_off(atxdmac); 2057 2058 /* Init channels. */ 2059 INIT_LIST_HEAD(&atxdmac->dma.channels); 2060 for (i = 0; i < nr_channels; i++) { 2061 struct at_xdmac_chan *atchan = &atxdmac->chan[i]; 2062 2063 atchan->chan.device = &atxdmac->dma; 2064 list_add_tail(&atchan->chan.device_node, 2065 &atxdmac->dma.channels); 2066 2067 atchan->ch_regs = at_xdmac_chan_reg_base(atxdmac, i); 2068 atchan->mask = 1 << i; 2069 2070 spin_lock_init(&atchan->lock); 2071 INIT_LIST_HEAD(&atchan->xfers_list); 2072 INIT_LIST_HEAD(&atchan->free_descs_list); 2073 tasklet_setup(&atchan->tasklet, at_xdmac_tasklet); 2074 2075 /* Clear pending interrupts. */ 2076 while (at_xdmac_chan_read(atchan, AT_XDMAC_CIS)) 2077 cpu_relax(); 2078 } 2079 platform_set_drvdata(pdev, atxdmac); 2080 2081 ret = dma_async_device_register(&atxdmac->dma); 2082 if (ret) { 2083 dev_err(&pdev->dev, "fail to register DMA engine device\n"); 2084 goto err_clk_disable; 2085 } 2086 2087 ret = of_dma_controller_register(pdev->dev.of_node, 2088 at_xdmac_xlate, atxdmac); 2089 if (ret) { 2090 dev_err(&pdev->dev, "could not register of dma controller\n"); 2091 goto err_dma_unregister; 2092 } 2093 2094 dev_info(&pdev->dev, "%d channels, mapped at 0x%p\n", 2095 nr_channels, atxdmac->regs); 2096 2097 return 0; 2098 2099err_dma_unregister: 2100 dma_async_device_unregister(&atxdmac->dma); 2101err_clk_disable: 2102 clk_disable_unprepare(atxdmac->clk); 2103err_free_irq: 2104 free_irq(atxdmac->irq, atxdmac); 2105 return ret; 2106} 2107 2108static int at_xdmac_remove(struct platform_device *pdev) 2109{ 2110 struct at_xdmac *atxdmac = (struct at_xdmac *)platform_get_drvdata(pdev); 2111 int i; 2112 2113 at_xdmac_off(atxdmac); 2114 of_dma_controller_free(pdev->dev.of_node); 2115 dma_async_device_unregister(&atxdmac->dma); 2116 clk_disable_unprepare(atxdmac->clk); 2117 2118 free_irq(atxdmac->irq, atxdmac); 2119 2120 for (i = 0; i < atxdmac->dma.chancnt; i++) { 2121 struct at_xdmac_chan *atchan = &atxdmac->chan[i]; 2122 2123 tasklet_kill(&atchan->tasklet); 2124 at_xdmac_free_chan_resources(&atchan->chan); 2125 } 2126 2127 return 0; 2128} 2129 2130static const struct dev_pm_ops atmel_xdmac_dev_pm_ops = { 2131 .prepare = atmel_xdmac_prepare, 2132 SET_LATE_SYSTEM_SLEEP_PM_OPS(atmel_xdmac_suspend, atmel_xdmac_resume) 2133}; 2134 2135static const struct of_device_id atmel_xdmac_dt_ids[] = { 2136 { 2137 .compatible = "atmel,sama5d4-dma", 2138 }, { 2139 /* sentinel */ 2140 } 2141}; 2142MODULE_DEVICE_TABLE(of, atmel_xdmac_dt_ids); 2143 2144static struct platform_driver at_xdmac_driver = { 2145 .probe = at_xdmac_probe, 2146 .remove = at_xdmac_remove, 2147 .driver = { 2148 .name = "at_xdmac", 2149 .of_match_table = of_match_ptr(atmel_xdmac_dt_ids), 2150 .pm = &atmel_xdmac_dev_pm_ops, 2151 } 2152}; 2153 2154static int __init at_xdmac_init(void) 2155{ 2156 return platform_driver_probe(&at_xdmac_driver, at_xdmac_probe); 2157} 2158subsys_initcall(at_xdmac_init); 2159 2160MODULE_DESCRIPTION("Atmel Extended DMA Controller driver"); 2161MODULE_AUTHOR("Ludovic Desroches <ludovic.desroches@atmel.com>"); 2162MODULE_LICENSE("GPL"); 2163