162306a36Sopenharmony_ci// SPDX-License-Identifier: GPL-2.0-only
262306a36Sopenharmony_ci/*
362306a36Sopenharmony_ci * timb_dma.c timberdale FPGA DMA driver
462306a36Sopenharmony_ci * Copyright (c) 2010 Intel Corporation
562306a36Sopenharmony_ci */
662306a36Sopenharmony_ci
762306a36Sopenharmony_ci/* Supports:
862306a36Sopenharmony_ci * Timberdale FPGA DMA engine
962306a36Sopenharmony_ci */
1062306a36Sopenharmony_ci
1162306a36Sopenharmony_ci#include <linux/dmaengine.h>
1262306a36Sopenharmony_ci#include <linux/dma-mapping.h>
1362306a36Sopenharmony_ci#include <linux/init.h>
1462306a36Sopenharmony_ci#include <linux/interrupt.h>
1562306a36Sopenharmony_ci#include <linux/io.h>
1662306a36Sopenharmony_ci#include <linux/module.h>
1762306a36Sopenharmony_ci#include <linux/platform_device.h>
1862306a36Sopenharmony_ci#include <linux/slab.h>
1962306a36Sopenharmony_ci
2062306a36Sopenharmony_ci#include <linux/timb_dma.h>
2162306a36Sopenharmony_ci
2262306a36Sopenharmony_ci#include "dmaengine.h"
2362306a36Sopenharmony_ci
2462306a36Sopenharmony_ci#define DRIVER_NAME "timb-dma"
2562306a36Sopenharmony_ci
2662306a36Sopenharmony_ci/* Global DMA registers */
2762306a36Sopenharmony_ci#define TIMBDMA_ACR		0x34
2862306a36Sopenharmony_ci#define TIMBDMA_32BIT_ADDR	0x01
2962306a36Sopenharmony_ci
3062306a36Sopenharmony_ci#define TIMBDMA_ISR		0x080000
3162306a36Sopenharmony_ci#define TIMBDMA_IPR		0x080004
3262306a36Sopenharmony_ci#define TIMBDMA_IER		0x080008
3362306a36Sopenharmony_ci
3462306a36Sopenharmony_ci/* Channel specific registers */
3562306a36Sopenharmony_ci/* RX instances base addresses are 0x00, 0x40, 0x80 ...
3662306a36Sopenharmony_ci * TX instances base addresses are 0x18, 0x58, 0x98 ...
3762306a36Sopenharmony_ci */
3862306a36Sopenharmony_ci#define TIMBDMA_INSTANCE_OFFSET		0x40
3962306a36Sopenharmony_ci#define TIMBDMA_INSTANCE_TX_OFFSET	0x18
4062306a36Sopenharmony_ci
4162306a36Sopenharmony_ci/* RX registers, relative the instance base */
4262306a36Sopenharmony_ci#define TIMBDMA_OFFS_RX_DHAR	0x00
4362306a36Sopenharmony_ci#define TIMBDMA_OFFS_RX_DLAR	0x04
4462306a36Sopenharmony_ci#define TIMBDMA_OFFS_RX_LR	0x0C
4562306a36Sopenharmony_ci#define TIMBDMA_OFFS_RX_BLR	0x10
4662306a36Sopenharmony_ci#define TIMBDMA_OFFS_RX_ER	0x14
4762306a36Sopenharmony_ci#define TIMBDMA_RX_EN		0x01
4862306a36Sopenharmony_ci/* bytes per Row, video specific register
4962306a36Sopenharmony_ci * which is placed after the TX registers...
5062306a36Sopenharmony_ci */
5162306a36Sopenharmony_ci#define TIMBDMA_OFFS_RX_BPRR	0x30
5262306a36Sopenharmony_ci
5362306a36Sopenharmony_ci/* TX registers, relative the instance base */
5462306a36Sopenharmony_ci#define TIMBDMA_OFFS_TX_DHAR	0x00
5562306a36Sopenharmony_ci#define TIMBDMA_OFFS_TX_DLAR	0x04
5662306a36Sopenharmony_ci#define TIMBDMA_OFFS_TX_BLR	0x0C
5762306a36Sopenharmony_ci#define TIMBDMA_OFFS_TX_LR	0x14
5862306a36Sopenharmony_ci
5962306a36Sopenharmony_ci
6062306a36Sopenharmony_ci#define TIMB_DMA_DESC_SIZE	8
6162306a36Sopenharmony_ci
6262306a36Sopenharmony_cistruct timb_dma_desc {
6362306a36Sopenharmony_ci	struct list_head		desc_node;
6462306a36Sopenharmony_ci	struct dma_async_tx_descriptor	txd;
6562306a36Sopenharmony_ci	u8				*desc_list;
6662306a36Sopenharmony_ci	unsigned int			desc_list_len;
6762306a36Sopenharmony_ci	bool				interrupt;
6862306a36Sopenharmony_ci};
6962306a36Sopenharmony_ci
7062306a36Sopenharmony_cistruct timb_dma_chan {
7162306a36Sopenharmony_ci	struct dma_chan		chan;
7262306a36Sopenharmony_ci	void __iomem		*membase;
7362306a36Sopenharmony_ci	spinlock_t		lock; /* Used to protect data structures,
7462306a36Sopenharmony_ci					especially the lists and descriptors,
7562306a36Sopenharmony_ci					from races between the tasklet and calls
7662306a36Sopenharmony_ci					from above */
7762306a36Sopenharmony_ci	bool			ongoing;
7862306a36Sopenharmony_ci	struct list_head	active_list;
7962306a36Sopenharmony_ci	struct list_head	queue;
8062306a36Sopenharmony_ci	struct list_head	free_list;
8162306a36Sopenharmony_ci	unsigned int		bytes_per_line;
8262306a36Sopenharmony_ci	enum dma_transfer_direction	direction;
8362306a36Sopenharmony_ci	unsigned int		descs; /* Descriptors to allocate */
8462306a36Sopenharmony_ci	unsigned int		desc_elems; /* number of elems per descriptor */
8562306a36Sopenharmony_ci};
8662306a36Sopenharmony_ci
8762306a36Sopenharmony_cistruct timb_dma {
8862306a36Sopenharmony_ci	struct dma_device	dma;
8962306a36Sopenharmony_ci	void __iomem		*membase;
9062306a36Sopenharmony_ci	struct tasklet_struct	tasklet;
9162306a36Sopenharmony_ci	struct timb_dma_chan	channels[];
9262306a36Sopenharmony_ci};
9362306a36Sopenharmony_ci
9462306a36Sopenharmony_cistatic struct device *chan2dev(struct dma_chan *chan)
9562306a36Sopenharmony_ci{
9662306a36Sopenharmony_ci	return &chan->dev->device;
9762306a36Sopenharmony_ci}
9862306a36Sopenharmony_cistatic struct device *chan2dmadev(struct dma_chan *chan)
9962306a36Sopenharmony_ci{
10062306a36Sopenharmony_ci	return chan2dev(chan)->parent->parent;
10162306a36Sopenharmony_ci}
10262306a36Sopenharmony_ci
10362306a36Sopenharmony_cistatic struct timb_dma *tdchantotd(struct timb_dma_chan *td_chan)
10462306a36Sopenharmony_ci{
10562306a36Sopenharmony_ci	int id = td_chan->chan.chan_id;
10662306a36Sopenharmony_ci	return (struct timb_dma *)((u8 *)td_chan -
10762306a36Sopenharmony_ci		id * sizeof(struct timb_dma_chan) - sizeof(struct timb_dma));
10862306a36Sopenharmony_ci}
10962306a36Sopenharmony_ci
11062306a36Sopenharmony_ci/* Must be called with the spinlock held */
11162306a36Sopenharmony_cistatic void __td_enable_chan_irq(struct timb_dma_chan *td_chan)
11262306a36Sopenharmony_ci{
11362306a36Sopenharmony_ci	int id = td_chan->chan.chan_id;
11462306a36Sopenharmony_ci	struct timb_dma *td = tdchantotd(td_chan);
11562306a36Sopenharmony_ci	u32 ier;
11662306a36Sopenharmony_ci
11762306a36Sopenharmony_ci	/* enable interrupt for this channel */
11862306a36Sopenharmony_ci	ier = ioread32(td->membase + TIMBDMA_IER);
11962306a36Sopenharmony_ci	ier |= 1 << id;
12062306a36Sopenharmony_ci	dev_dbg(chan2dev(&td_chan->chan), "Enabling irq: %d, IER: 0x%x\n", id,
12162306a36Sopenharmony_ci		ier);
12262306a36Sopenharmony_ci	iowrite32(ier, td->membase + TIMBDMA_IER);
12362306a36Sopenharmony_ci}
12462306a36Sopenharmony_ci
12562306a36Sopenharmony_ci/* Should be called with the spinlock held */
12662306a36Sopenharmony_cistatic bool __td_dma_done_ack(struct timb_dma_chan *td_chan)
12762306a36Sopenharmony_ci{
12862306a36Sopenharmony_ci	int id = td_chan->chan.chan_id;
12962306a36Sopenharmony_ci	struct timb_dma *td = (struct timb_dma *)((u8 *)td_chan -
13062306a36Sopenharmony_ci		id * sizeof(struct timb_dma_chan) - sizeof(struct timb_dma));
13162306a36Sopenharmony_ci	u32 isr;
13262306a36Sopenharmony_ci	bool done = false;
13362306a36Sopenharmony_ci
13462306a36Sopenharmony_ci	dev_dbg(chan2dev(&td_chan->chan), "Checking irq: %d, td: %p\n", id, td);
13562306a36Sopenharmony_ci
13662306a36Sopenharmony_ci	isr = ioread32(td->membase + TIMBDMA_ISR) & (1 << id);
13762306a36Sopenharmony_ci	if (isr) {
13862306a36Sopenharmony_ci		iowrite32(isr, td->membase + TIMBDMA_ISR);
13962306a36Sopenharmony_ci		done = true;
14062306a36Sopenharmony_ci	}
14162306a36Sopenharmony_ci
14262306a36Sopenharmony_ci	return done;
14362306a36Sopenharmony_ci}
14462306a36Sopenharmony_ci
14562306a36Sopenharmony_cistatic int td_fill_desc(struct timb_dma_chan *td_chan, u8 *dma_desc,
14662306a36Sopenharmony_ci	struct scatterlist *sg, bool last)
14762306a36Sopenharmony_ci{
14862306a36Sopenharmony_ci	if (sg_dma_len(sg) > USHRT_MAX) {
14962306a36Sopenharmony_ci		dev_err(chan2dev(&td_chan->chan), "Too big sg element\n");
15062306a36Sopenharmony_ci		return -EINVAL;
15162306a36Sopenharmony_ci	}
15262306a36Sopenharmony_ci
15362306a36Sopenharmony_ci	/* length must be word aligned */
15462306a36Sopenharmony_ci	if (sg_dma_len(sg) % sizeof(u32)) {
15562306a36Sopenharmony_ci		dev_err(chan2dev(&td_chan->chan), "Incorrect length: %d\n",
15662306a36Sopenharmony_ci			sg_dma_len(sg));
15762306a36Sopenharmony_ci		return -EINVAL;
15862306a36Sopenharmony_ci	}
15962306a36Sopenharmony_ci
16062306a36Sopenharmony_ci	dev_dbg(chan2dev(&td_chan->chan), "desc: %p, addr: 0x%llx\n",
16162306a36Sopenharmony_ci		dma_desc, (unsigned long long)sg_dma_address(sg));
16262306a36Sopenharmony_ci
16362306a36Sopenharmony_ci	dma_desc[7] = (sg_dma_address(sg) >> 24) & 0xff;
16462306a36Sopenharmony_ci	dma_desc[6] = (sg_dma_address(sg) >> 16) & 0xff;
16562306a36Sopenharmony_ci	dma_desc[5] = (sg_dma_address(sg) >> 8) & 0xff;
16662306a36Sopenharmony_ci	dma_desc[4] = (sg_dma_address(sg) >> 0) & 0xff;
16762306a36Sopenharmony_ci
16862306a36Sopenharmony_ci	dma_desc[3] = (sg_dma_len(sg) >> 8) & 0xff;
16962306a36Sopenharmony_ci	dma_desc[2] = (sg_dma_len(sg) >> 0) & 0xff;
17062306a36Sopenharmony_ci
17162306a36Sopenharmony_ci	dma_desc[1] = 0x00;
17262306a36Sopenharmony_ci	dma_desc[0] = 0x21 | (last ? 0x02 : 0); /* tran, valid */
17362306a36Sopenharmony_ci
17462306a36Sopenharmony_ci	return 0;
17562306a36Sopenharmony_ci}
17662306a36Sopenharmony_ci
17762306a36Sopenharmony_ci/* Must be called with the spinlock held */
17862306a36Sopenharmony_cistatic void __td_start_dma(struct timb_dma_chan *td_chan)
17962306a36Sopenharmony_ci{
18062306a36Sopenharmony_ci	struct timb_dma_desc *td_desc;
18162306a36Sopenharmony_ci
18262306a36Sopenharmony_ci	if (td_chan->ongoing) {
18362306a36Sopenharmony_ci		dev_err(chan2dev(&td_chan->chan),
18462306a36Sopenharmony_ci			"Transfer already ongoing\n");
18562306a36Sopenharmony_ci		return;
18662306a36Sopenharmony_ci	}
18762306a36Sopenharmony_ci
18862306a36Sopenharmony_ci	td_desc = list_entry(td_chan->active_list.next, struct timb_dma_desc,
18962306a36Sopenharmony_ci		desc_node);
19062306a36Sopenharmony_ci
19162306a36Sopenharmony_ci	dev_dbg(chan2dev(&td_chan->chan),
19262306a36Sopenharmony_ci		"td_chan: %p, chan: %d, membase: %p\n",
19362306a36Sopenharmony_ci		td_chan, td_chan->chan.chan_id, td_chan->membase);
19462306a36Sopenharmony_ci
19562306a36Sopenharmony_ci	if (td_chan->direction == DMA_DEV_TO_MEM) {
19662306a36Sopenharmony_ci
19762306a36Sopenharmony_ci		/* descriptor address */
19862306a36Sopenharmony_ci		iowrite32(0, td_chan->membase + TIMBDMA_OFFS_RX_DHAR);
19962306a36Sopenharmony_ci		iowrite32(td_desc->txd.phys, td_chan->membase +
20062306a36Sopenharmony_ci			TIMBDMA_OFFS_RX_DLAR);
20162306a36Sopenharmony_ci		/* Bytes per line */
20262306a36Sopenharmony_ci		iowrite32(td_chan->bytes_per_line, td_chan->membase +
20362306a36Sopenharmony_ci			TIMBDMA_OFFS_RX_BPRR);
20462306a36Sopenharmony_ci		/* enable RX */
20562306a36Sopenharmony_ci		iowrite32(TIMBDMA_RX_EN, td_chan->membase + TIMBDMA_OFFS_RX_ER);
20662306a36Sopenharmony_ci	} else {
20762306a36Sopenharmony_ci		/* address high */
20862306a36Sopenharmony_ci		iowrite32(0, td_chan->membase + TIMBDMA_OFFS_TX_DHAR);
20962306a36Sopenharmony_ci		iowrite32(td_desc->txd.phys, td_chan->membase +
21062306a36Sopenharmony_ci			TIMBDMA_OFFS_TX_DLAR);
21162306a36Sopenharmony_ci	}
21262306a36Sopenharmony_ci
21362306a36Sopenharmony_ci	td_chan->ongoing = true;
21462306a36Sopenharmony_ci
21562306a36Sopenharmony_ci	if (td_desc->interrupt)
21662306a36Sopenharmony_ci		__td_enable_chan_irq(td_chan);
21762306a36Sopenharmony_ci}
21862306a36Sopenharmony_ci
21962306a36Sopenharmony_cistatic void __td_finish(struct timb_dma_chan *td_chan)
22062306a36Sopenharmony_ci{
22162306a36Sopenharmony_ci	struct dmaengine_desc_callback	cb;
22262306a36Sopenharmony_ci	struct dma_async_tx_descriptor	*txd;
22362306a36Sopenharmony_ci	struct timb_dma_desc		*td_desc;
22462306a36Sopenharmony_ci
22562306a36Sopenharmony_ci	/* can happen if the descriptor is canceled */
22662306a36Sopenharmony_ci	if (list_empty(&td_chan->active_list))
22762306a36Sopenharmony_ci		return;
22862306a36Sopenharmony_ci
22962306a36Sopenharmony_ci	td_desc = list_entry(td_chan->active_list.next, struct timb_dma_desc,
23062306a36Sopenharmony_ci		desc_node);
23162306a36Sopenharmony_ci	txd = &td_desc->txd;
23262306a36Sopenharmony_ci
23362306a36Sopenharmony_ci	dev_dbg(chan2dev(&td_chan->chan), "descriptor %u complete\n",
23462306a36Sopenharmony_ci		txd->cookie);
23562306a36Sopenharmony_ci
23662306a36Sopenharmony_ci	/* make sure to stop the transfer */
23762306a36Sopenharmony_ci	if (td_chan->direction == DMA_DEV_TO_MEM)
23862306a36Sopenharmony_ci		iowrite32(0, td_chan->membase + TIMBDMA_OFFS_RX_ER);
23962306a36Sopenharmony_ci/* Currently no support for stopping DMA transfers
24062306a36Sopenharmony_ci	else
24162306a36Sopenharmony_ci		iowrite32(0, td_chan->membase + TIMBDMA_OFFS_TX_DLAR);
24262306a36Sopenharmony_ci*/
24362306a36Sopenharmony_ci	dma_cookie_complete(txd);
24462306a36Sopenharmony_ci	td_chan->ongoing = false;
24562306a36Sopenharmony_ci
24662306a36Sopenharmony_ci	dmaengine_desc_get_callback(txd, &cb);
24762306a36Sopenharmony_ci
24862306a36Sopenharmony_ci	list_move(&td_desc->desc_node, &td_chan->free_list);
24962306a36Sopenharmony_ci
25062306a36Sopenharmony_ci	dma_descriptor_unmap(txd);
25162306a36Sopenharmony_ci	/*
25262306a36Sopenharmony_ci	 * The API requires that no submissions are done from a
25362306a36Sopenharmony_ci	 * callback, so we don't need to drop the lock here
25462306a36Sopenharmony_ci	 */
25562306a36Sopenharmony_ci	dmaengine_desc_callback_invoke(&cb, NULL);
25662306a36Sopenharmony_ci}
25762306a36Sopenharmony_ci
25862306a36Sopenharmony_cistatic u32 __td_ier_mask(struct timb_dma *td)
25962306a36Sopenharmony_ci{
26062306a36Sopenharmony_ci	int i;
26162306a36Sopenharmony_ci	u32 ret = 0;
26262306a36Sopenharmony_ci
26362306a36Sopenharmony_ci	for (i = 0; i < td->dma.chancnt; i++) {
26462306a36Sopenharmony_ci		struct timb_dma_chan *td_chan = td->channels + i;
26562306a36Sopenharmony_ci		if (td_chan->ongoing) {
26662306a36Sopenharmony_ci			struct timb_dma_desc *td_desc =
26762306a36Sopenharmony_ci				list_entry(td_chan->active_list.next,
26862306a36Sopenharmony_ci				struct timb_dma_desc, desc_node);
26962306a36Sopenharmony_ci			if (td_desc->interrupt)
27062306a36Sopenharmony_ci				ret |= 1 << i;
27162306a36Sopenharmony_ci		}
27262306a36Sopenharmony_ci	}
27362306a36Sopenharmony_ci
27462306a36Sopenharmony_ci	return ret;
27562306a36Sopenharmony_ci}
27662306a36Sopenharmony_ci
27762306a36Sopenharmony_cistatic void __td_start_next(struct timb_dma_chan *td_chan)
27862306a36Sopenharmony_ci{
27962306a36Sopenharmony_ci	struct timb_dma_desc *td_desc;
28062306a36Sopenharmony_ci
28162306a36Sopenharmony_ci	BUG_ON(list_empty(&td_chan->queue));
28262306a36Sopenharmony_ci	BUG_ON(td_chan->ongoing);
28362306a36Sopenharmony_ci
28462306a36Sopenharmony_ci	td_desc = list_entry(td_chan->queue.next, struct timb_dma_desc,
28562306a36Sopenharmony_ci		desc_node);
28662306a36Sopenharmony_ci
28762306a36Sopenharmony_ci	dev_dbg(chan2dev(&td_chan->chan), "%s: started %u\n",
28862306a36Sopenharmony_ci		__func__, td_desc->txd.cookie);
28962306a36Sopenharmony_ci
29062306a36Sopenharmony_ci	list_move(&td_desc->desc_node, &td_chan->active_list);
29162306a36Sopenharmony_ci	__td_start_dma(td_chan);
29262306a36Sopenharmony_ci}
29362306a36Sopenharmony_ci
29462306a36Sopenharmony_cistatic dma_cookie_t td_tx_submit(struct dma_async_tx_descriptor *txd)
29562306a36Sopenharmony_ci{
29662306a36Sopenharmony_ci	struct timb_dma_desc *td_desc = container_of(txd, struct timb_dma_desc,
29762306a36Sopenharmony_ci		txd);
29862306a36Sopenharmony_ci	struct timb_dma_chan *td_chan = container_of(txd->chan,
29962306a36Sopenharmony_ci		struct timb_dma_chan, chan);
30062306a36Sopenharmony_ci	dma_cookie_t cookie;
30162306a36Sopenharmony_ci
30262306a36Sopenharmony_ci	spin_lock_bh(&td_chan->lock);
30362306a36Sopenharmony_ci	cookie = dma_cookie_assign(txd);
30462306a36Sopenharmony_ci
30562306a36Sopenharmony_ci	if (list_empty(&td_chan->active_list)) {
30662306a36Sopenharmony_ci		dev_dbg(chan2dev(txd->chan), "%s: started %u\n", __func__,
30762306a36Sopenharmony_ci			txd->cookie);
30862306a36Sopenharmony_ci		list_add_tail(&td_desc->desc_node, &td_chan->active_list);
30962306a36Sopenharmony_ci		__td_start_dma(td_chan);
31062306a36Sopenharmony_ci	} else {
31162306a36Sopenharmony_ci		dev_dbg(chan2dev(txd->chan), "tx_submit: queued %u\n",
31262306a36Sopenharmony_ci			txd->cookie);
31362306a36Sopenharmony_ci
31462306a36Sopenharmony_ci		list_add_tail(&td_desc->desc_node, &td_chan->queue);
31562306a36Sopenharmony_ci	}
31662306a36Sopenharmony_ci
31762306a36Sopenharmony_ci	spin_unlock_bh(&td_chan->lock);
31862306a36Sopenharmony_ci
31962306a36Sopenharmony_ci	return cookie;
32062306a36Sopenharmony_ci}
32162306a36Sopenharmony_ci
32262306a36Sopenharmony_cistatic struct timb_dma_desc *td_alloc_init_desc(struct timb_dma_chan *td_chan)
32362306a36Sopenharmony_ci{
32462306a36Sopenharmony_ci	struct dma_chan *chan = &td_chan->chan;
32562306a36Sopenharmony_ci	struct timb_dma_desc *td_desc;
32662306a36Sopenharmony_ci	int err;
32762306a36Sopenharmony_ci
32862306a36Sopenharmony_ci	td_desc = kzalloc(sizeof(struct timb_dma_desc), GFP_KERNEL);
32962306a36Sopenharmony_ci	if (!td_desc)
33062306a36Sopenharmony_ci		goto out;
33162306a36Sopenharmony_ci
33262306a36Sopenharmony_ci	td_desc->desc_list_len = td_chan->desc_elems * TIMB_DMA_DESC_SIZE;
33362306a36Sopenharmony_ci
33462306a36Sopenharmony_ci	td_desc->desc_list = kzalloc(td_desc->desc_list_len, GFP_KERNEL);
33562306a36Sopenharmony_ci	if (!td_desc->desc_list)
33662306a36Sopenharmony_ci		goto err;
33762306a36Sopenharmony_ci
33862306a36Sopenharmony_ci	dma_async_tx_descriptor_init(&td_desc->txd, chan);
33962306a36Sopenharmony_ci	td_desc->txd.tx_submit = td_tx_submit;
34062306a36Sopenharmony_ci	td_desc->txd.flags = DMA_CTRL_ACK;
34162306a36Sopenharmony_ci
34262306a36Sopenharmony_ci	td_desc->txd.phys = dma_map_single(chan2dmadev(chan),
34362306a36Sopenharmony_ci		td_desc->desc_list, td_desc->desc_list_len, DMA_TO_DEVICE);
34462306a36Sopenharmony_ci
34562306a36Sopenharmony_ci	err = dma_mapping_error(chan2dmadev(chan), td_desc->txd.phys);
34662306a36Sopenharmony_ci	if (err) {
34762306a36Sopenharmony_ci		dev_err(chan2dev(chan), "DMA mapping error: %d\n", err);
34862306a36Sopenharmony_ci		goto err;
34962306a36Sopenharmony_ci	}
35062306a36Sopenharmony_ci
35162306a36Sopenharmony_ci	return td_desc;
35262306a36Sopenharmony_cierr:
35362306a36Sopenharmony_ci	kfree(td_desc->desc_list);
35462306a36Sopenharmony_ci	kfree(td_desc);
35562306a36Sopenharmony_ciout:
35662306a36Sopenharmony_ci	return NULL;
35762306a36Sopenharmony_ci
35862306a36Sopenharmony_ci}
35962306a36Sopenharmony_ci
36062306a36Sopenharmony_cistatic void td_free_desc(struct timb_dma_desc *td_desc)
36162306a36Sopenharmony_ci{
36262306a36Sopenharmony_ci	dev_dbg(chan2dev(td_desc->txd.chan), "Freeing desc: %p\n", td_desc);
36362306a36Sopenharmony_ci	dma_unmap_single(chan2dmadev(td_desc->txd.chan), td_desc->txd.phys,
36462306a36Sopenharmony_ci		td_desc->desc_list_len, DMA_TO_DEVICE);
36562306a36Sopenharmony_ci
36662306a36Sopenharmony_ci	kfree(td_desc->desc_list);
36762306a36Sopenharmony_ci	kfree(td_desc);
36862306a36Sopenharmony_ci}
36962306a36Sopenharmony_ci
37062306a36Sopenharmony_cistatic void td_desc_put(struct timb_dma_chan *td_chan,
37162306a36Sopenharmony_ci	struct timb_dma_desc *td_desc)
37262306a36Sopenharmony_ci{
37362306a36Sopenharmony_ci	dev_dbg(chan2dev(&td_chan->chan), "Putting desc: %p\n", td_desc);
37462306a36Sopenharmony_ci
37562306a36Sopenharmony_ci	spin_lock_bh(&td_chan->lock);
37662306a36Sopenharmony_ci	list_add(&td_desc->desc_node, &td_chan->free_list);
37762306a36Sopenharmony_ci	spin_unlock_bh(&td_chan->lock);
37862306a36Sopenharmony_ci}
37962306a36Sopenharmony_ci
38062306a36Sopenharmony_cistatic struct timb_dma_desc *td_desc_get(struct timb_dma_chan *td_chan)
38162306a36Sopenharmony_ci{
38262306a36Sopenharmony_ci	struct timb_dma_desc *td_desc, *_td_desc;
38362306a36Sopenharmony_ci	struct timb_dma_desc *ret = NULL;
38462306a36Sopenharmony_ci
38562306a36Sopenharmony_ci	spin_lock_bh(&td_chan->lock);
38662306a36Sopenharmony_ci	list_for_each_entry_safe(td_desc, _td_desc, &td_chan->free_list,
38762306a36Sopenharmony_ci		desc_node) {
38862306a36Sopenharmony_ci		if (async_tx_test_ack(&td_desc->txd)) {
38962306a36Sopenharmony_ci			list_del(&td_desc->desc_node);
39062306a36Sopenharmony_ci			ret = td_desc;
39162306a36Sopenharmony_ci			break;
39262306a36Sopenharmony_ci		}
39362306a36Sopenharmony_ci		dev_dbg(chan2dev(&td_chan->chan), "desc %p not ACKed\n",
39462306a36Sopenharmony_ci			td_desc);
39562306a36Sopenharmony_ci	}
39662306a36Sopenharmony_ci	spin_unlock_bh(&td_chan->lock);
39762306a36Sopenharmony_ci
39862306a36Sopenharmony_ci	return ret;
39962306a36Sopenharmony_ci}
40062306a36Sopenharmony_ci
40162306a36Sopenharmony_cistatic int td_alloc_chan_resources(struct dma_chan *chan)
40262306a36Sopenharmony_ci{
40362306a36Sopenharmony_ci	struct timb_dma_chan *td_chan =
40462306a36Sopenharmony_ci		container_of(chan, struct timb_dma_chan, chan);
40562306a36Sopenharmony_ci	int i;
40662306a36Sopenharmony_ci
40762306a36Sopenharmony_ci	dev_dbg(chan2dev(chan), "%s: entry\n", __func__);
40862306a36Sopenharmony_ci
40962306a36Sopenharmony_ci	BUG_ON(!list_empty(&td_chan->free_list));
41062306a36Sopenharmony_ci	for (i = 0; i < td_chan->descs; i++) {
41162306a36Sopenharmony_ci		struct timb_dma_desc *td_desc = td_alloc_init_desc(td_chan);
41262306a36Sopenharmony_ci		if (!td_desc) {
41362306a36Sopenharmony_ci			if (i)
41462306a36Sopenharmony_ci				break;
41562306a36Sopenharmony_ci			else {
41662306a36Sopenharmony_ci				dev_err(chan2dev(chan),
41762306a36Sopenharmony_ci					"Couldn't allocate any descriptors\n");
41862306a36Sopenharmony_ci				return -ENOMEM;
41962306a36Sopenharmony_ci			}
42062306a36Sopenharmony_ci		}
42162306a36Sopenharmony_ci
42262306a36Sopenharmony_ci		td_desc_put(td_chan, td_desc);
42362306a36Sopenharmony_ci	}
42462306a36Sopenharmony_ci
42562306a36Sopenharmony_ci	spin_lock_bh(&td_chan->lock);
42662306a36Sopenharmony_ci	dma_cookie_init(chan);
42762306a36Sopenharmony_ci	spin_unlock_bh(&td_chan->lock);
42862306a36Sopenharmony_ci
42962306a36Sopenharmony_ci	return 0;
43062306a36Sopenharmony_ci}
43162306a36Sopenharmony_ci
43262306a36Sopenharmony_cistatic void td_free_chan_resources(struct dma_chan *chan)
43362306a36Sopenharmony_ci{
43462306a36Sopenharmony_ci	struct timb_dma_chan *td_chan =
43562306a36Sopenharmony_ci		container_of(chan, struct timb_dma_chan, chan);
43662306a36Sopenharmony_ci	struct timb_dma_desc *td_desc, *_td_desc;
43762306a36Sopenharmony_ci	LIST_HEAD(list);
43862306a36Sopenharmony_ci
43962306a36Sopenharmony_ci	dev_dbg(chan2dev(chan), "%s: Entry\n", __func__);
44062306a36Sopenharmony_ci
44162306a36Sopenharmony_ci	/* check that all descriptors are free */
44262306a36Sopenharmony_ci	BUG_ON(!list_empty(&td_chan->active_list));
44362306a36Sopenharmony_ci	BUG_ON(!list_empty(&td_chan->queue));
44462306a36Sopenharmony_ci
44562306a36Sopenharmony_ci	spin_lock_bh(&td_chan->lock);
44662306a36Sopenharmony_ci	list_splice_init(&td_chan->free_list, &list);
44762306a36Sopenharmony_ci	spin_unlock_bh(&td_chan->lock);
44862306a36Sopenharmony_ci
44962306a36Sopenharmony_ci	list_for_each_entry_safe(td_desc, _td_desc, &list, desc_node) {
45062306a36Sopenharmony_ci		dev_dbg(chan2dev(chan), "%s: Freeing desc: %p\n", __func__,
45162306a36Sopenharmony_ci			td_desc);
45262306a36Sopenharmony_ci		td_free_desc(td_desc);
45362306a36Sopenharmony_ci	}
45462306a36Sopenharmony_ci}
45562306a36Sopenharmony_ci
45662306a36Sopenharmony_cistatic enum dma_status td_tx_status(struct dma_chan *chan, dma_cookie_t cookie,
45762306a36Sopenharmony_ci				    struct dma_tx_state *txstate)
45862306a36Sopenharmony_ci{
45962306a36Sopenharmony_ci	enum dma_status ret;
46062306a36Sopenharmony_ci
46162306a36Sopenharmony_ci	dev_dbg(chan2dev(chan), "%s: Entry\n", __func__);
46262306a36Sopenharmony_ci
46362306a36Sopenharmony_ci	ret = dma_cookie_status(chan, cookie, txstate);
46462306a36Sopenharmony_ci
46562306a36Sopenharmony_ci	dev_dbg(chan2dev(chan), "%s: exit, ret: %d\n", 	__func__, ret);
46662306a36Sopenharmony_ci
46762306a36Sopenharmony_ci	return ret;
46862306a36Sopenharmony_ci}
46962306a36Sopenharmony_ci
47062306a36Sopenharmony_cistatic void td_issue_pending(struct dma_chan *chan)
47162306a36Sopenharmony_ci{
47262306a36Sopenharmony_ci	struct timb_dma_chan *td_chan =
47362306a36Sopenharmony_ci		container_of(chan, struct timb_dma_chan, chan);
47462306a36Sopenharmony_ci
47562306a36Sopenharmony_ci	dev_dbg(chan2dev(chan), "%s: Entry\n", __func__);
47662306a36Sopenharmony_ci	spin_lock_bh(&td_chan->lock);
47762306a36Sopenharmony_ci
47862306a36Sopenharmony_ci	if (!list_empty(&td_chan->active_list))
47962306a36Sopenharmony_ci		/* transfer ongoing */
48062306a36Sopenharmony_ci		if (__td_dma_done_ack(td_chan))
48162306a36Sopenharmony_ci			__td_finish(td_chan);
48262306a36Sopenharmony_ci
48362306a36Sopenharmony_ci	if (list_empty(&td_chan->active_list) && !list_empty(&td_chan->queue))
48462306a36Sopenharmony_ci		__td_start_next(td_chan);
48562306a36Sopenharmony_ci
48662306a36Sopenharmony_ci	spin_unlock_bh(&td_chan->lock);
48762306a36Sopenharmony_ci}
48862306a36Sopenharmony_ci
48962306a36Sopenharmony_cistatic struct dma_async_tx_descriptor *td_prep_slave_sg(struct dma_chan *chan,
49062306a36Sopenharmony_ci	struct scatterlist *sgl, unsigned int sg_len,
49162306a36Sopenharmony_ci	enum dma_transfer_direction direction, unsigned long flags,
49262306a36Sopenharmony_ci	void *context)
49362306a36Sopenharmony_ci{
49462306a36Sopenharmony_ci	struct timb_dma_chan *td_chan =
49562306a36Sopenharmony_ci		container_of(chan, struct timb_dma_chan, chan);
49662306a36Sopenharmony_ci	struct timb_dma_desc *td_desc;
49762306a36Sopenharmony_ci	struct scatterlist *sg;
49862306a36Sopenharmony_ci	unsigned int i;
49962306a36Sopenharmony_ci	unsigned int desc_usage = 0;
50062306a36Sopenharmony_ci
50162306a36Sopenharmony_ci	if (!sgl || !sg_len) {
50262306a36Sopenharmony_ci		dev_err(chan2dev(chan), "%s: No SG list\n", __func__);
50362306a36Sopenharmony_ci		return NULL;
50462306a36Sopenharmony_ci	}
50562306a36Sopenharmony_ci
50662306a36Sopenharmony_ci	/* even channels are for RX, odd for TX */
50762306a36Sopenharmony_ci	if (td_chan->direction != direction) {
50862306a36Sopenharmony_ci		dev_err(chan2dev(chan),
50962306a36Sopenharmony_ci			"Requesting channel in wrong direction\n");
51062306a36Sopenharmony_ci		return NULL;
51162306a36Sopenharmony_ci	}
51262306a36Sopenharmony_ci
51362306a36Sopenharmony_ci	td_desc = td_desc_get(td_chan);
51462306a36Sopenharmony_ci	if (!td_desc) {
51562306a36Sopenharmony_ci		dev_err(chan2dev(chan), "Not enough descriptors available\n");
51662306a36Sopenharmony_ci		return NULL;
51762306a36Sopenharmony_ci	}
51862306a36Sopenharmony_ci
51962306a36Sopenharmony_ci	td_desc->interrupt = (flags & DMA_PREP_INTERRUPT) != 0;
52062306a36Sopenharmony_ci
52162306a36Sopenharmony_ci	for_each_sg(sgl, sg, sg_len, i) {
52262306a36Sopenharmony_ci		int err;
52362306a36Sopenharmony_ci		if (desc_usage > td_desc->desc_list_len) {
52462306a36Sopenharmony_ci			dev_err(chan2dev(chan), "No descriptor space\n");
52562306a36Sopenharmony_ci			return NULL;
52662306a36Sopenharmony_ci		}
52762306a36Sopenharmony_ci
52862306a36Sopenharmony_ci		err = td_fill_desc(td_chan, td_desc->desc_list + desc_usage, sg,
52962306a36Sopenharmony_ci			i == (sg_len - 1));
53062306a36Sopenharmony_ci		if (err) {
53162306a36Sopenharmony_ci			dev_err(chan2dev(chan), "Failed to update desc: %d\n",
53262306a36Sopenharmony_ci				err);
53362306a36Sopenharmony_ci			td_desc_put(td_chan, td_desc);
53462306a36Sopenharmony_ci			return NULL;
53562306a36Sopenharmony_ci		}
53662306a36Sopenharmony_ci		desc_usage += TIMB_DMA_DESC_SIZE;
53762306a36Sopenharmony_ci	}
53862306a36Sopenharmony_ci
53962306a36Sopenharmony_ci	dma_sync_single_for_device(chan2dmadev(chan), td_desc->txd.phys,
54062306a36Sopenharmony_ci		td_desc->desc_list_len, DMA_TO_DEVICE);
54162306a36Sopenharmony_ci
54262306a36Sopenharmony_ci	return &td_desc->txd;
54362306a36Sopenharmony_ci}
54462306a36Sopenharmony_ci
54562306a36Sopenharmony_cistatic int td_terminate_all(struct dma_chan *chan)
54662306a36Sopenharmony_ci{
54762306a36Sopenharmony_ci	struct timb_dma_chan *td_chan =
54862306a36Sopenharmony_ci		container_of(chan, struct timb_dma_chan, chan);
54962306a36Sopenharmony_ci	struct timb_dma_desc *td_desc, *_td_desc;
55062306a36Sopenharmony_ci
55162306a36Sopenharmony_ci	dev_dbg(chan2dev(chan), "%s: Entry\n", __func__);
55262306a36Sopenharmony_ci
55362306a36Sopenharmony_ci	/* first the easy part, put the queue into the free list */
55462306a36Sopenharmony_ci	spin_lock_bh(&td_chan->lock);
55562306a36Sopenharmony_ci	list_for_each_entry_safe(td_desc, _td_desc, &td_chan->queue,
55662306a36Sopenharmony_ci		desc_node)
55762306a36Sopenharmony_ci		list_move(&td_desc->desc_node, &td_chan->free_list);
55862306a36Sopenharmony_ci
55962306a36Sopenharmony_ci	/* now tear down the running */
56062306a36Sopenharmony_ci	__td_finish(td_chan);
56162306a36Sopenharmony_ci	spin_unlock_bh(&td_chan->lock);
56262306a36Sopenharmony_ci
56362306a36Sopenharmony_ci	return 0;
56462306a36Sopenharmony_ci}
56562306a36Sopenharmony_ci
56662306a36Sopenharmony_cistatic void td_tasklet(struct tasklet_struct *t)
56762306a36Sopenharmony_ci{
56862306a36Sopenharmony_ci	struct timb_dma *td = from_tasklet(td, t, tasklet);
56962306a36Sopenharmony_ci	u32 isr;
57062306a36Sopenharmony_ci	u32 ipr;
57162306a36Sopenharmony_ci	u32 ier;
57262306a36Sopenharmony_ci	int i;
57362306a36Sopenharmony_ci
57462306a36Sopenharmony_ci	isr = ioread32(td->membase + TIMBDMA_ISR);
57562306a36Sopenharmony_ci	ipr = isr & __td_ier_mask(td);
57662306a36Sopenharmony_ci
57762306a36Sopenharmony_ci	/* ack the interrupts */
57862306a36Sopenharmony_ci	iowrite32(ipr, td->membase + TIMBDMA_ISR);
57962306a36Sopenharmony_ci
58062306a36Sopenharmony_ci	for (i = 0; i < td->dma.chancnt; i++)
58162306a36Sopenharmony_ci		if (ipr & (1 << i)) {
58262306a36Sopenharmony_ci			struct timb_dma_chan *td_chan = td->channels + i;
58362306a36Sopenharmony_ci			spin_lock(&td_chan->lock);
58462306a36Sopenharmony_ci			__td_finish(td_chan);
58562306a36Sopenharmony_ci			if (!list_empty(&td_chan->queue))
58662306a36Sopenharmony_ci				__td_start_next(td_chan);
58762306a36Sopenharmony_ci			spin_unlock(&td_chan->lock);
58862306a36Sopenharmony_ci		}
58962306a36Sopenharmony_ci
59062306a36Sopenharmony_ci	ier = __td_ier_mask(td);
59162306a36Sopenharmony_ci	iowrite32(ier, td->membase + TIMBDMA_IER);
59262306a36Sopenharmony_ci}
59362306a36Sopenharmony_ci
59462306a36Sopenharmony_ci
59562306a36Sopenharmony_cistatic irqreturn_t td_irq(int irq, void *devid)
59662306a36Sopenharmony_ci{
59762306a36Sopenharmony_ci	struct timb_dma *td = devid;
59862306a36Sopenharmony_ci	u32 ipr = ioread32(td->membase + TIMBDMA_IPR);
59962306a36Sopenharmony_ci
60062306a36Sopenharmony_ci	if (ipr) {
60162306a36Sopenharmony_ci		/* disable interrupts, will be re-enabled in tasklet */
60262306a36Sopenharmony_ci		iowrite32(0, td->membase + TIMBDMA_IER);
60362306a36Sopenharmony_ci
60462306a36Sopenharmony_ci		tasklet_schedule(&td->tasklet);
60562306a36Sopenharmony_ci
60662306a36Sopenharmony_ci		return IRQ_HANDLED;
60762306a36Sopenharmony_ci	} else
60862306a36Sopenharmony_ci		return IRQ_NONE;
60962306a36Sopenharmony_ci}
61062306a36Sopenharmony_ci
61162306a36Sopenharmony_ci
61262306a36Sopenharmony_cistatic int td_probe(struct platform_device *pdev)
61362306a36Sopenharmony_ci{
61462306a36Sopenharmony_ci	struct timb_dma_platform_data *pdata = dev_get_platdata(&pdev->dev);
61562306a36Sopenharmony_ci	struct timb_dma *td;
61662306a36Sopenharmony_ci	struct resource *iomem;
61762306a36Sopenharmony_ci	int irq;
61862306a36Sopenharmony_ci	int err;
61962306a36Sopenharmony_ci	int i;
62062306a36Sopenharmony_ci
62162306a36Sopenharmony_ci	if (!pdata) {
62262306a36Sopenharmony_ci		dev_err(&pdev->dev, "No platform data\n");
62362306a36Sopenharmony_ci		return -EINVAL;
62462306a36Sopenharmony_ci	}
62562306a36Sopenharmony_ci
62662306a36Sopenharmony_ci	iomem = platform_get_resource(pdev, IORESOURCE_MEM, 0);
62762306a36Sopenharmony_ci	if (!iomem)
62862306a36Sopenharmony_ci		return -EINVAL;
62962306a36Sopenharmony_ci
63062306a36Sopenharmony_ci	irq = platform_get_irq(pdev, 0);
63162306a36Sopenharmony_ci	if (irq < 0)
63262306a36Sopenharmony_ci		return irq;
63362306a36Sopenharmony_ci
63462306a36Sopenharmony_ci	if (!request_mem_region(iomem->start, resource_size(iomem),
63562306a36Sopenharmony_ci		DRIVER_NAME))
63662306a36Sopenharmony_ci		return -EBUSY;
63762306a36Sopenharmony_ci
63862306a36Sopenharmony_ci	td  = kzalloc(struct_size(td, channels, pdata->nr_channels),
63962306a36Sopenharmony_ci		      GFP_KERNEL);
64062306a36Sopenharmony_ci	if (!td) {
64162306a36Sopenharmony_ci		err = -ENOMEM;
64262306a36Sopenharmony_ci		goto err_release_region;
64362306a36Sopenharmony_ci	}
64462306a36Sopenharmony_ci
64562306a36Sopenharmony_ci	dev_dbg(&pdev->dev, "Allocated TD: %p\n", td);
64662306a36Sopenharmony_ci
64762306a36Sopenharmony_ci	td->membase = ioremap(iomem->start, resource_size(iomem));
64862306a36Sopenharmony_ci	if (!td->membase) {
64962306a36Sopenharmony_ci		dev_err(&pdev->dev, "Failed to remap I/O memory\n");
65062306a36Sopenharmony_ci		err = -ENOMEM;
65162306a36Sopenharmony_ci		goto err_free_mem;
65262306a36Sopenharmony_ci	}
65362306a36Sopenharmony_ci
65462306a36Sopenharmony_ci	/* 32bit addressing */
65562306a36Sopenharmony_ci	iowrite32(TIMBDMA_32BIT_ADDR, td->membase + TIMBDMA_ACR);
65662306a36Sopenharmony_ci
65762306a36Sopenharmony_ci	/* disable and clear any interrupts */
65862306a36Sopenharmony_ci	iowrite32(0x0, td->membase + TIMBDMA_IER);
65962306a36Sopenharmony_ci	iowrite32(0xFFFFFFFF, td->membase + TIMBDMA_ISR);
66062306a36Sopenharmony_ci
66162306a36Sopenharmony_ci	tasklet_setup(&td->tasklet, td_tasklet);
66262306a36Sopenharmony_ci
66362306a36Sopenharmony_ci	err = request_irq(irq, td_irq, IRQF_SHARED, DRIVER_NAME, td);
66462306a36Sopenharmony_ci	if (err) {
66562306a36Sopenharmony_ci		dev_err(&pdev->dev, "Failed to request IRQ\n");
66662306a36Sopenharmony_ci		goto err_tasklet_kill;
66762306a36Sopenharmony_ci	}
66862306a36Sopenharmony_ci
66962306a36Sopenharmony_ci	td->dma.device_alloc_chan_resources	= td_alloc_chan_resources;
67062306a36Sopenharmony_ci	td->dma.device_free_chan_resources	= td_free_chan_resources;
67162306a36Sopenharmony_ci	td->dma.device_tx_status		= td_tx_status;
67262306a36Sopenharmony_ci	td->dma.device_issue_pending		= td_issue_pending;
67362306a36Sopenharmony_ci
67462306a36Sopenharmony_ci	dma_cap_set(DMA_SLAVE, td->dma.cap_mask);
67562306a36Sopenharmony_ci	dma_cap_set(DMA_PRIVATE, td->dma.cap_mask);
67662306a36Sopenharmony_ci	td->dma.device_prep_slave_sg = td_prep_slave_sg;
67762306a36Sopenharmony_ci	td->dma.device_terminate_all = td_terminate_all;
67862306a36Sopenharmony_ci
67962306a36Sopenharmony_ci	td->dma.dev = &pdev->dev;
68062306a36Sopenharmony_ci
68162306a36Sopenharmony_ci	INIT_LIST_HEAD(&td->dma.channels);
68262306a36Sopenharmony_ci
68362306a36Sopenharmony_ci	for (i = 0; i < pdata->nr_channels; i++) {
68462306a36Sopenharmony_ci		struct timb_dma_chan *td_chan = &td->channels[i];
68562306a36Sopenharmony_ci		struct timb_dma_platform_data_channel *pchan =
68662306a36Sopenharmony_ci			pdata->channels + i;
68762306a36Sopenharmony_ci
68862306a36Sopenharmony_ci		/* even channels are RX, odd are TX */
68962306a36Sopenharmony_ci		if ((i % 2) == pchan->rx) {
69062306a36Sopenharmony_ci			dev_err(&pdev->dev, "Wrong channel configuration\n");
69162306a36Sopenharmony_ci			err = -EINVAL;
69262306a36Sopenharmony_ci			goto err_free_irq;
69362306a36Sopenharmony_ci		}
69462306a36Sopenharmony_ci
69562306a36Sopenharmony_ci		td_chan->chan.device = &td->dma;
69662306a36Sopenharmony_ci		dma_cookie_init(&td_chan->chan);
69762306a36Sopenharmony_ci		spin_lock_init(&td_chan->lock);
69862306a36Sopenharmony_ci		INIT_LIST_HEAD(&td_chan->active_list);
69962306a36Sopenharmony_ci		INIT_LIST_HEAD(&td_chan->queue);
70062306a36Sopenharmony_ci		INIT_LIST_HEAD(&td_chan->free_list);
70162306a36Sopenharmony_ci
70262306a36Sopenharmony_ci		td_chan->descs = pchan->descriptors;
70362306a36Sopenharmony_ci		td_chan->desc_elems = pchan->descriptor_elements;
70462306a36Sopenharmony_ci		td_chan->bytes_per_line = pchan->bytes_per_line;
70562306a36Sopenharmony_ci		td_chan->direction = pchan->rx ? DMA_DEV_TO_MEM :
70662306a36Sopenharmony_ci			DMA_MEM_TO_DEV;
70762306a36Sopenharmony_ci
70862306a36Sopenharmony_ci		td_chan->membase = td->membase +
70962306a36Sopenharmony_ci			(i / 2) * TIMBDMA_INSTANCE_OFFSET +
71062306a36Sopenharmony_ci			(pchan->rx ? 0 : TIMBDMA_INSTANCE_TX_OFFSET);
71162306a36Sopenharmony_ci
71262306a36Sopenharmony_ci		dev_dbg(&pdev->dev, "Chan: %d, membase: %p\n",
71362306a36Sopenharmony_ci			i, td_chan->membase);
71462306a36Sopenharmony_ci
71562306a36Sopenharmony_ci		list_add_tail(&td_chan->chan.device_node, &td->dma.channels);
71662306a36Sopenharmony_ci	}
71762306a36Sopenharmony_ci
71862306a36Sopenharmony_ci	err = dma_async_device_register(&td->dma);
71962306a36Sopenharmony_ci	if (err) {
72062306a36Sopenharmony_ci		dev_err(&pdev->dev, "Failed to register async device\n");
72162306a36Sopenharmony_ci		goto err_free_irq;
72262306a36Sopenharmony_ci	}
72362306a36Sopenharmony_ci
72462306a36Sopenharmony_ci	platform_set_drvdata(pdev, td);
72562306a36Sopenharmony_ci
72662306a36Sopenharmony_ci	dev_dbg(&pdev->dev, "Probe result: %d\n", err);
72762306a36Sopenharmony_ci	return err;
72862306a36Sopenharmony_ci
72962306a36Sopenharmony_cierr_free_irq:
73062306a36Sopenharmony_ci	free_irq(irq, td);
73162306a36Sopenharmony_cierr_tasklet_kill:
73262306a36Sopenharmony_ci	tasklet_kill(&td->tasklet);
73362306a36Sopenharmony_ci	iounmap(td->membase);
73462306a36Sopenharmony_cierr_free_mem:
73562306a36Sopenharmony_ci	kfree(td);
73662306a36Sopenharmony_cierr_release_region:
73762306a36Sopenharmony_ci	release_mem_region(iomem->start, resource_size(iomem));
73862306a36Sopenharmony_ci
73962306a36Sopenharmony_ci	return err;
74062306a36Sopenharmony_ci
74162306a36Sopenharmony_ci}
74262306a36Sopenharmony_ci
74362306a36Sopenharmony_cistatic int td_remove(struct platform_device *pdev)
74462306a36Sopenharmony_ci{
74562306a36Sopenharmony_ci	struct timb_dma *td = platform_get_drvdata(pdev);
74662306a36Sopenharmony_ci	struct resource *iomem = platform_get_resource(pdev, IORESOURCE_MEM, 0);
74762306a36Sopenharmony_ci	int irq = platform_get_irq(pdev, 0);
74862306a36Sopenharmony_ci
74962306a36Sopenharmony_ci	dma_async_device_unregister(&td->dma);
75062306a36Sopenharmony_ci	free_irq(irq, td);
75162306a36Sopenharmony_ci	tasklet_kill(&td->tasklet);
75262306a36Sopenharmony_ci	iounmap(td->membase);
75362306a36Sopenharmony_ci	kfree(td);
75462306a36Sopenharmony_ci	release_mem_region(iomem->start, resource_size(iomem));
75562306a36Sopenharmony_ci
75662306a36Sopenharmony_ci	dev_dbg(&pdev->dev, "Removed...\n");
75762306a36Sopenharmony_ci	return 0;
75862306a36Sopenharmony_ci}
75962306a36Sopenharmony_ci
76062306a36Sopenharmony_cistatic struct platform_driver td_driver = {
76162306a36Sopenharmony_ci	.driver = {
76262306a36Sopenharmony_ci		.name	= DRIVER_NAME,
76362306a36Sopenharmony_ci	},
76462306a36Sopenharmony_ci	.probe	= td_probe,
76562306a36Sopenharmony_ci	.remove	= td_remove,
76662306a36Sopenharmony_ci};
76762306a36Sopenharmony_ci
76862306a36Sopenharmony_cimodule_platform_driver(td_driver);
76962306a36Sopenharmony_ci
77062306a36Sopenharmony_ciMODULE_LICENSE("GPL v2");
77162306a36Sopenharmony_ciMODULE_DESCRIPTION("Timberdale DMA controller driver");
77262306a36Sopenharmony_ciMODULE_AUTHOR("Pelagicore AB <info@pelagicore.com>");
77362306a36Sopenharmony_ciMODULE_ALIAS("platform:"DRIVER_NAME);
774