1// SPDX-License-Identifier: GPL-2.0
2/*
3 * K3 NAVSS DMA glue interface
4 *
5 * Copyright (C) 2019 Texas Instruments Incorporated - http://www.ti.com
6 *
7 */
8
9#include <linux/module.h>
10#include <linux/atomic.h>
11#include <linux/delay.h>
12#include <linux/dma-mapping.h>
13#include <linux/io.h>
14#include <linux/init.h>
15#include <linux/of.h>
16#include <linux/platform_device.h>
17#include <linux/soc/ti/k3-ringacc.h>
18#include <linux/dma/ti-cppi5.h>
19#include <linux/dma/k3-udma-glue.h>
20
21#include "k3-udma.h"
22#include "k3-psil-priv.h"
23
24struct k3_udma_glue_common {
25	struct device *dev;
26	struct device chan_dev;
27	struct udma_dev *udmax;
28	const struct udma_tisci_rm *tisci_rm;
29	struct k3_ringacc *ringacc;
30	u32 src_thread;
31	u32 dst_thread;
32
33	u32  hdesc_size;
34	bool epib;
35	u32  psdata_size;
36	u32  swdata_size;
37	u32  atype_asel;
38	struct psil_endpoint_config *ep_config;
39};
40
41struct k3_udma_glue_tx_channel {
42	struct k3_udma_glue_common common;
43
44	struct udma_tchan *udma_tchanx;
45	int udma_tchan_id;
46
47	struct k3_ring *ringtx;
48	struct k3_ring *ringtxcq;
49
50	bool psil_paired;
51
52	int virq;
53
54	atomic_t free_pkts;
55	bool tx_pause_on_err;
56	bool tx_filt_einfo;
57	bool tx_filt_pswords;
58	bool tx_supr_tdpkt;
59
60	int udma_tflow_id;
61};
62
63struct k3_udma_glue_rx_flow {
64	struct udma_rflow *udma_rflow;
65	int udma_rflow_id;
66	struct k3_ring *ringrx;
67	struct k3_ring *ringrxfdq;
68
69	int virq;
70};
71
72struct k3_udma_glue_rx_channel {
73	struct k3_udma_glue_common common;
74
75	struct udma_rchan *udma_rchanx;
76	int udma_rchan_id;
77	bool remote;
78
79	bool psil_paired;
80
81	u32  swdata_size;
82	int  flow_id_base;
83
84	struct k3_udma_glue_rx_flow *flows;
85	u32 flow_num;
86	u32 flows_ready;
87};
88
89static void k3_udma_chan_dev_release(struct device *dev)
90{
91	/* The struct containing the device is devm managed */
92}
93
94static struct class k3_udma_glue_devclass = {
95	.name		= "k3_udma_glue_chan",
96	.dev_release	= k3_udma_chan_dev_release,
97};
98
99#define K3_UDMAX_TDOWN_TIMEOUT_US 1000
100
101static int of_k3_udma_glue_parse(struct device_node *udmax_np,
102				 struct k3_udma_glue_common *common)
103{
104	common->udmax = of_xudma_dev_get(udmax_np, NULL);
105	if (IS_ERR(common->udmax))
106		return PTR_ERR(common->udmax);
107
108	common->ringacc = xudma_get_ringacc(common->udmax);
109	common->tisci_rm = xudma_dev_get_tisci_rm(common->udmax);
110
111	return 0;
112}
113
114static int of_k3_udma_glue_parse_chn(struct device_node *chn_np,
115		const char *name, struct k3_udma_glue_common *common,
116		bool tx_chn)
117{
118	struct of_phandle_args dma_spec;
119	u32 thread_id;
120	int ret = 0;
121	int index;
122
123	if (unlikely(!name))
124		return -EINVAL;
125
126	index = of_property_match_string(chn_np, "dma-names", name);
127	if (index < 0)
128		return index;
129
130	if (of_parse_phandle_with_args(chn_np, "dmas", "#dma-cells", index,
131				       &dma_spec))
132		return -ENOENT;
133
134	ret = of_k3_udma_glue_parse(dma_spec.np, common);
135	if (ret)
136		goto out_put_spec;
137
138	thread_id = dma_spec.args[0];
139	if (dma_spec.args_count == 2) {
140		if (dma_spec.args[1] > 2 && !xudma_is_pktdma(common->udmax)) {
141			dev_err(common->dev, "Invalid channel atype: %u\n",
142				dma_spec.args[1]);
143			ret = -EINVAL;
144			goto out_put_spec;
145		}
146		if (dma_spec.args[1] > 15 && xudma_is_pktdma(common->udmax)) {
147			dev_err(common->dev, "Invalid channel asel: %u\n",
148				dma_spec.args[1]);
149			ret = -EINVAL;
150			goto out_put_spec;
151		}
152
153		common->atype_asel = dma_spec.args[1];
154	}
155
156	if (tx_chn && !(thread_id & K3_PSIL_DST_THREAD_ID_OFFSET)) {
157		ret = -EINVAL;
158		goto out_put_spec;
159	}
160
161	if (!tx_chn && (thread_id & K3_PSIL_DST_THREAD_ID_OFFSET)) {
162		ret = -EINVAL;
163		goto out_put_spec;
164	}
165
166	/* get psil endpoint config */
167	common->ep_config = psil_get_ep_config(thread_id);
168	if (IS_ERR(common->ep_config)) {
169		dev_err(common->dev,
170			"No configuration for psi-l thread 0x%04x\n",
171			thread_id);
172		ret = PTR_ERR(common->ep_config);
173		goto out_put_spec;
174	}
175
176	common->epib = common->ep_config->needs_epib;
177	common->psdata_size = common->ep_config->psd_size;
178
179	if (tx_chn)
180		common->dst_thread = thread_id;
181	else
182		common->src_thread = thread_id;
183
184out_put_spec:
185	of_node_put(dma_spec.np);
186	return ret;
187};
188
189static void k3_udma_glue_dump_tx_chn(struct k3_udma_glue_tx_channel *tx_chn)
190{
191	struct device *dev = tx_chn->common.dev;
192
193	dev_dbg(dev, "dump_tx_chn:\n"
194		"udma_tchan_id: %d\n"
195		"src_thread: %08x\n"
196		"dst_thread: %08x\n",
197		tx_chn->udma_tchan_id,
198		tx_chn->common.src_thread,
199		tx_chn->common.dst_thread);
200}
201
202static void k3_udma_glue_dump_tx_rt_chn(struct k3_udma_glue_tx_channel *chn,
203					char *mark)
204{
205	struct device *dev = chn->common.dev;
206
207	dev_dbg(dev, "=== dump ===> %s\n", mark);
208	dev_dbg(dev, "0x%08X: %08X\n", UDMA_CHAN_RT_CTL_REG,
209		xudma_tchanrt_read(chn->udma_tchanx, UDMA_CHAN_RT_CTL_REG));
210	dev_dbg(dev, "0x%08X: %08X\n", UDMA_CHAN_RT_PEER_RT_EN_REG,
211		xudma_tchanrt_read(chn->udma_tchanx,
212				   UDMA_CHAN_RT_PEER_RT_EN_REG));
213	dev_dbg(dev, "0x%08X: %08X\n", UDMA_CHAN_RT_PCNT_REG,
214		xudma_tchanrt_read(chn->udma_tchanx, UDMA_CHAN_RT_PCNT_REG));
215	dev_dbg(dev, "0x%08X: %08X\n", UDMA_CHAN_RT_BCNT_REG,
216		xudma_tchanrt_read(chn->udma_tchanx, UDMA_CHAN_RT_BCNT_REG));
217	dev_dbg(dev, "0x%08X: %08X\n", UDMA_CHAN_RT_SBCNT_REG,
218		xudma_tchanrt_read(chn->udma_tchanx, UDMA_CHAN_RT_SBCNT_REG));
219}
220
221static int k3_udma_glue_cfg_tx_chn(struct k3_udma_glue_tx_channel *tx_chn)
222{
223	const struct udma_tisci_rm *tisci_rm = tx_chn->common.tisci_rm;
224	struct ti_sci_msg_rm_udmap_tx_ch_cfg req;
225
226	memset(&req, 0, sizeof(req));
227
228	req.valid_params = TI_SCI_MSG_VALUE_RM_UDMAP_CH_PAUSE_ON_ERR_VALID |
229			TI_SCI_MSG_VALUE_RM_UDMAP_CH_TX_FILT_EINFO_VALID |
230			TI_SCI_MSG_VALUE_RM_UDMAP_CH_TX_FILT_PSWORDS_VALID |
231			TI_SCI_MSG_VALUE_RM_UDMAP_CH_CHAN_TYPE_VALID |
232			TI_SCI_MSG_VALUE_RM_UDMAP_CH_TX_SUPR_TDPKT_VALID |
233			TI_SCI_MSG_VALUE_RM_UDMAP_CH_FETCH_SIZE_VALID |
234			TI_SCI_MSG_VALUE_RM_UDMAP_CH_CQ_QNUM_VALID |
235			TI_SCI_MSG_VALUE_RM_UDMAP_CH_ATYPE_VALID;
236	req.nav_id = tisci_rm->tisci_dev_id;
237	req.index = tx_chn->udma_tchan_id;
238	if (tx_chn->tx_pause_on_err)
239		req.tx_pause_on_err = 1;
240	if (tx_chn->tx_filt_einfo)
241		req.tx_filt_einfo = 1;
242	if (tx_chn->tx_filt_pswords)
243		req.tx_filt_pswords = 1;
244	req.tx_chan_type = TI_SCI_RM_UDMAP_CHAN_TYPE_PKT_PBRR;
245	if (tx_chn->tx_supr_tdpkt)
246		req.tx_supr_tdpkt = 1;
247	req.tx_fetch_size = tx_chn->common.hdesc_size >> 2;
248	req.txcq_qnum = k3_ringacc_get_ring_id(tx_chn->ringtxcq);
249	req.tx_atype = tx_chn->common.atype_asel;
250
251	return tisci_rm->tisci_udmap_ops->tx_ch_cfg(tisci_rm->tisci, &req);
252}
253
254struct k3_udma_glue_tx_channel *k3_udma_glue_request_tx_chn(struct device *dev,
255		const char *name, struct k3_udma_glue_tx_channel_cfg *cfg)
256{
257	struct k3_udma_glue_tx_channel *tx_chn;
258	int ret;
259
260	tx_chn = devm_kzalloc(dev, sizeof(*tx_chn), GFP_KERNEL);
261	if (!tx_chn)
262		return ERR_PTR(-ENOMEM);
263
264	tx_chn->common.dev = dev;
265	tx_chn->common.swdata_size = cfg->swdata_size;
266	tx_chn->tx_pause_on_err = cfg->tx_pause_on_err;
267	tx_chn->tx_filt_einfo = cfg->tx_filt_einfo;
268	tx_chn->tx_filt_pswords = cfg->tx_filt_pswords;
269	tx_chn->tx_supr_tdpkt = cfg->tx_supr_tdpkt;
270
271	/* parse of udmap channel */
272	ret = of_k3_udma_glue_parse_chn(dev->of_node, name,
273					&tx_chn->common, true);
274	if (ret)
275		goto err;
276
277	tx_chn->common.hdesc_size = cppi5_hdesc_calc_size(tx_chn->common.epib,
278						tx_chn->common.psdata_size,
279						tx_chn->common.swdata_size);
280
281	if (xudma_is_pktdma(tx_chn->common.udmax))
282		tx_chn->udma_tchan_id = tx_chn->common.ep_config->mapped_channel_id;
283	else
284		tx_chn->udma_tchan_id = -1;
285
286	/* request and cfg UDMAP TX channel */
287	tx_chn->udma_tchanx = xudma_tchan_get(tx_chn->common.udmax,
288					      tx_chn->udma_tchan_id);
289	if (IS_ERR(tx_chn->udma_tchanx)) {
290		ret = PTR_ERR(tx_chn->udma_tchanx);
291		dev_err(dev, "UDMAX tchanx get err %d\n", ret);
292		goto err;
293	}
294	tx_chn->udma_tchan_id = xudma_tchan_get_id(tx_chn->udma_tchanx);
295
296	tx_chn->common.chan_dev.class = &k3_udma_glue_devclass;
297	tx_chn->common.chan_dev.parent = xudma_get_device(tx_chn->common.udmax);
298	dev_set_name(&tx_chn->common.chan_dev, "tchan%d-0x%04x",
299		     tx_chn->udma_tchan_id, tx_chn->common.dst_thread);
300	ret = device_register(&tx_chn->common.chan_dev);
301	if (ret) {
302		dev_err(dev, "Channel Device registration failed %d\n", ret);
303		put_device(&tx_chn->common.chan_dev);
304		tx_chn->common.chan_dev.parent = NULL;
305		goto err;
306	}
307
308	if (xudma_is_pktdma(tx_chn->common.udmax)) {
309		/* prepare the channel device as coherent */
310		tx_chn->common.chan_dev.dma_coherent = true;
311		dma_coerce_mask_and_coherent(&tx_chn->common.chan_dev,
312					     DMA_BIT_MASK(48));
313	}
314
315	atomic_set(&tx_chn->free_pkts, cfg->txcq_cfg.size);
316
317	if (xudma_is_pktdma(tx_chn->common.udmax))
318		tx_chn->udma_tflow_id = tx_chn->common.ep_config->default_flow_id;
319	else
320		tx_chn->udma_tflow_id = tx_chn->udma_tchan_id;
321
322	/* request and cfg rings */
323	ret =  k3_ringacc_request_rings_pair(tx_chn->common.ringacc,
324					     tx_chn->udma_tflow_id, -1,
325					     &tx_chn->ringtx,
326					     &tx_chn->ringtxcq);
327	if (ret) {
328		dev_err(dev, "Failed to get TX/TXCQ rings %d\n", ret);
329		goto err;
330	}
331
332	/* Set the dma_dev for the rings to be configured */
333	cfg->tx_cfg.dma_dev = k3_udma_glue_tx_get_dma_device(tx_chn);
334	cfg->txcq_cfg.dma_dev = cfg->tx_cfg.dma_dev;
335
336	/* Set the ASEL value for DMA rings of PKTDMA */
337	if (xudma_is_pktdma(tx_chn->common.udmax)) {
338		cfg->tx_cfg.asel = tx_chn->common.atype_asel;
339		cfg->txcq_cfg.asel = tx_chn->common.atype_asel;
340	}
341
342	ret = k3_ringacc_ring_cfg(tx_chn->ringtx, &cfg->tx_cfg);
343	if (ret) {
344		dev_err(dev, "Failed to cfg ringtx %d\n", ret);
345		goto err;
346	}
347
348	ret = k3_ringacc_ring_cfg(tx_chn->ringtxcq, &cfg->txcq_cfg);
349	if (ret) {
350		dev_err(dev, "Failed to cfg ringtx %d\n", ret);
351		goto err;
352	}
353
354	/* request and cfg psi-l */
355	tx_chn->common.src_thread =
356			xudma_dev_get_psil_base(tx_chn->common.udmax) +
357			tx_chn->udma_tchan_id;
358
359	ret = k3_udma_glue_cfg_tx_chn(tx_chn);
360	if (ret) {
361		dev_err(dev, "Failed to cfg tchan %d\n", ret);
362		goto err;
363	}
364
365	k3_udma_glue_dump_tx_chn(tx_chn);
366
367	return tx_chn;
368
369err:
370	k3_udma_glue_release_tx_chn(tx_chn);
371	return ERR_PTR(ret);
372}
373EXPORT_SYMBOL_GPL(k3_udma_glue_request_tx_chn);
374
375void k3_udma_glue_release_tx_chn(struct k3_udma_glue_tx_channel *tx_chn)
376{
377	if (tx_chn->psil_paired) {
378		xudma_navss_psil_unpair(tx_chn->common.udmax,
379					tx_chn->common.src_thread,
380					tx_chn->common.dst_thread);
381		tx_chn->psil_paired = false;
382	}
383
384	if (!IS_ERR_OR_NULL(tx_chn->udma_tchanx))
385		xudma_tchan_put(tx_chn->common.udmax,
386				tx_chn->udma_tchanx);
387
388	if (tx_chn->ringtxcq)
389		k3_ringacc_ring_free(tx_chn->ringtxcq);
390
391	if (tx_chn->ringtx)
392		k3_ringacc_ring_free(tx_chn->ringtx);
393
394	if (tx_chn->common.chan_dev.parent) {
395		device_unregister(&tx_chn->common.chan_dev);
396		tx_chn->common.chan_dev.parent = NULL;
397	}
398}
399EXPORT_SYMBOL_GPL(k3_udma_glue_release_tx_chn);
400
401int k3_udma_glue_push_tx_chn(struct k3_udma_glue_tx_channel *tx_chn,
402			     struct cppi5_host_desc_t *desc_tx,
403			     dma_addr_t desc_dma)
404{
405	u32 ringtxcq_id;
406
407	if (!atomic_add_unless(&tx_chn->free_pkts, -1, 0))
408		return -ENOMEM;
409
410	ringtxcq_id = k3_ringacc_get_ring_id(tx_chn->ringtxcq);
411	cppi5_desc_set_retpolicy(&desc_tx->hdr, 0, ringtxcq_id);
412
413	return k3_ringacc_ring_push(tx_chn->ringtx, &desc_dma);
414}
415EXPORT_SYMBOL_GPL(k3_udma_glue_push_tx_chn);
416
417int k3_udma_glue_pop_tx_chn(struct k3_udma_glue_tx_channel *tx_chn,
418			    dma_addr_t *desc_dma)
419{
420	int ret;
421
422	ret = k3_ringacc_ring_pop(tx_chn->ringtxcq, desc_dma);
423	if (!ret)
424		atomic_inc(&tx_chn->free_pkts);
425
426	return ret;
427}
428EXPORT_SYMBOL_GPL(k3_udma_glue_pop_tx_chn);
429
430int k3_udma_glue_enable_tx_chn(struct k3_udma_glue_tx_channel *tx_chn)
431{
432	int ret;
433
434	ret = xudma_navss_psil_pair(tx_chn->common.udmax,
435				    tx_chn->common.src_thread,
436				    tx_chn->common.dst_thread);
437	if (ret) {
438		dev_err(tx_chn->common.dev, "PSI-L request err %d\n", ret);
439		return ret;
440	}
441
442	tx_chn->psil_paired = true;
443
444	xudma_tchanrt_write(tx_chn->udma_tchanx, UDMA_CHAN_RT_PEER_RT_EN_REG,
445			    UDMA_PEER_RT_EN_ENABLE);
446
447	xudma_tchanrt_write(tx_chn->udma_tchanx, UDMA_CHAN_RT_CTL_REG,
448			    UDMA_CHAN_RT_CTL_EN);
449
450	k3_udma_glue_dump_tx_rt_chn(tx_chn, "txchn en");
451	return 0;
452}
453EXPORT_SYMBOL_GPL(k3_udma_glue_enable_tx_chn);
454
455void k3_udma_glue_disable_tx_chn(struct k3_udma_glue_tx_channel *tx_chn)
456{
457	k3_udma_glue_dump_tx_rt_chn(tx_chn, "txchn dis1");
458
459	xudma_tchanrt_write(tx_chn->udma_tchanx, UDMA_CHAN_RT_CTL_REG, 0);
460
461	xudma_tchanrt_write(tx_chn->udma_tchanx,
462			    UDMA_CHAN_RT_PEER_RT_EN_REG, 0);
463	k3_udma_glue_dump_tx_rt_chn(tx_chn, "txchn dis2");
464
465	if (tx_chn->psil_paired) {
466		xudma_navss_psil_unpair(tx_chn->common.udmax,
467					tx_chn->common.src_thread,
468					tx_chn->common.dst_thread);
469		tx_chn->psil_paired = false;
470	}
471}
472EXPORT_SYMBOL_GPL(k3_udma_glue_disable_tx_chn);
473
474void k3_udma_glue_tdown_tx_chn(struct k3_udma_glue_tx_channel *tx_chn,
475			       bool sync)
476{
477	int i = 0;
478	u32 val;
479
480	k3_udma_glue_dump_tx_rt_chn(tx_chn, "txchn tdown1");
481
482	xudma_tchanrt_write(tx_chn->udma_tchanx, UDMA_CHAN_RT_CTL_REG,
483			    UDMA_CHAN_RT_CTL_EN | UDMA_CHAN_RT_CTL_TDOWN);
484
485	val = xudma_tchanrt_read(tx_chn->udma_tchanx, UDMA_CHAN_RT_CTL_REG);
486
487	while (sync && (val & UDMA_CHAN_RT_CTL_EN)) {
488		val = xudma_tchanrt_read(tx_chn->udma_tchanx,
489					 UDMA_CHAN_RT_CTL_REG);
490		udelay(1);
491		if (i > K3_UDMAX_TDOWN_TIMEOUT_US) {
492			dev_err(tx_chn->common.dev, "TX tdown timeout\n");
493			break;
494		}
495		i++;
496	}
497
498	val = xudma_tchanrt_read(tx_chn->udma_tchanx,
499				 UDMA_CHAN_RT_PEER_RT_EN_REG);
500	if (sync && (val & UDMA_PEER_RT_EN_ENABLE))
501		dev_err(tx_chn->common.dev, "TX tdown peer not stopped\n");
502	k3_udma_glue_dump_tx_rt_chn(tx_chn, "txchn tdown2");
503}
504EXPORT_SYMBOL_GPL(k3_udma_glue_tdown_tx_chn);
505
506void k3_udma_glue_reset_tx_chn(struct k3_udma_glue_tx_channel *tx_chn,
507			       void *data,
508			       void (*cleanup)(void *data, dma_addr_t desc_dma))
509{
510	struct device *dev = tx_chn->common.dev;
511	dma_addr_t desc_dma;
512	int occ_tx, i, ret;
513
514	/*
515	 * TXQ reset need to be special way as it is input for udma and its
516	 * state cached by udma, so:
517	 * 1) save TXQ occ
518	 * 2) clean up TXQ and call callback .cleanup() for each desc
519	 * 3) reset TXQ in a special way
520	 */
521	occ_tx = k3_ringacc_ring_get_occ(tx_chn->ringtx);
522	dev_dbg(dev, "TX reset occ_tx %u\n", occ_tx);
523
524	for (i = 0; i < occ_tx; i++) {
525		ret = k3_ringacc_ring_pop(tx_chn->ringtx, &desc_dma);
526		if (ret) {
527			if (ret != -ENODATA)
528				dev_err(dev, "TX reset pop %d\n", ret);
529			break;
530		}
531		cleanup(data, desc_dma);
532	}
533
534	/* reset TXCQ as it is not input for udma - expected to be empty */
535	k3_ringacc_ring_reset(tx_chn->ringtxcq);
536	k3_ringacc_ring_reset_dma(tx_chn->ringtx, occ_tx);
537}
538EXPORT_SYMBOL_GPL(k3_udma_glue_reset_tx_chn);
539
540u32 k3_udma_glue_tx_get_hdesc_size(struct k3_udma_glue_tx_channel *tx_chn)
541{
542	return tx_chn->common.hdesc_size;
543}
544EXPORT_SYMBOL_GPL(k3_udma_glue_tx_get_hdesc_size);
545
546u32 k3_udma_glue_tx_get_txcq_id(struct k3_udma_glue_tx_channel *tx_chn)
547{
548	return k3_ringacc_get_ring_id(tx_chn->ringtxcq);
549}
550EXPORT_SYMBOL_GPL(k3_udma_glue_tx_get_txcq_id);
551
552int k3_udma_glue_tx_get_irq(struct k3_udma_glue_tx_channel *tx_chn)
553{
554	if (xudma_is_pktdma(tx_chn->common.udmax)) {
555		tx_chn->virq = xudma_pktdma_tflow_get_irq(tx_chn->common.udmax,
556							  tx_chn->udma_tflow_id);
557	} else {
558		tx_chn->virq = k3_ringacc_get_ring_irq_num(tx_chn->ringtxcq);
559	}
560
561	if (!tx_chn->virq)
562		return -ENXIO;
563
564	return tx_chn->virq;
565}
566EXPORT_SYMBOL_GPL(k3_udma_glue_tx_get_irq);
567
568struct device *
569	k3_udma_glue_tx_get_dma_device(struct k3_udma_glue_tx_channel *tx_chn)
570{
571	if (xudma_is_pktdma(tx_chn->common.udmax) &&
572	    (tx_chn->common.atype_asel == 14 || tx_chn->common.atype_asel == 15))
573		return &tx_chn->common.chan_dev;
574
575	return xudma_get_device(tx_chn->common.udmax);
576}
577EXPORT_SYMBOL_GPL(k3_udma_glue_tx_get_dma_device);
578
579void k3_udma_glue_tx_dma_to_cppi5_addr(struct k3_udma_glue_tx_channel *tx_chn,
580				       dma_addr_t *addr)
581{
582	if (!xudma_is_pktdma(tx_chn->common.udmax) ||
583	    !tx_chn->common.atype_asel)
584		return;
585
586	*addr |= (u64)tx_chn->common.atype_asel << K3_ADDRESS_ASEL_SHIFT;
587}
588EXPORT_SYMBOL_GPL(k3_udma_glue_tx_dma_to_cppi5_addr);
589
590void k3_udma_glue_tx_cppi5_to_dma_addr(struct k3_udma_glue_tx_channel *tx_chn,
591				       dma_addr_t *addr)
592{
593	if (!xudma_is_pktdma(tx_chn->common.udmax) ||
594	    !tx_chn->common.atype_asel)
595		return;
596
597	*addr &= (u64)GENMASK(K3_ADDRESS_ASEL_SHIFT - 1, 0);
598}
599EXPORT_SYMBOL_GPL(k3_udma_glue_tx_cppi5_to_dma_addr);
600
601static int k3_udma_glue_cfg_rx_chn(struct k3_udma_glue_rx_channel *rx_chn)
602{
603	const struct udma_tisci_rm *tisci_rm = rx_chn->common.tisci_rm;
604	struct ti_sci_msg_rm_udmap_rx_ch_cfg req;
605	int ret;
606
607	memset(&req, 0, sizeof(req));
608
609	req.valid_params = TI_SCI_MSG_VALUE_RM_UDMAP_CH_FETCH_SIZE_VALID |
610			   TI_SCI_MSG_VALUE_RM_UDMAP_CH_CQ_QNUM_VALID |
611			   TI_SCI_MSG_VALUE_RM_UDMAP_CH_CHAN_TYPE_VALID |
612			   TI_SCI_MSG_VALUE_RM_UDMAP_CH_ATYPE_VALID;
613
614	req.nav_id = tisci_rm->tisci_dev_id;
615	req.index = rx_chn->udma_rchan_id;
616	req.rx_fetch_size = rx_chn->common.hdesc_size >> 2;
617	/*
618	 * TODO: we can't support rxcq_qnum/RCHAN[a]_RCQ cfg with current sysfw
619	 * and udmax impl, so just configure it to invalid value.
620	 * req.rxcq_qnum = k3_ringacc_get_ring_id(rx_chn->flows[0].ringrx);
621	 */
622	req.rxcq_qnum = 0xFFFF;
623	if (!xudma_is_pktdma(rx_chn->common.udmax) && rx_chn->flow_num &&
624	    rx_chn->flow_id_base != rx_chn->udma_rchan_id) {
625		/* Default flow + extra ones */
626		req.valid_params |= TI_SCI_MSG_VALUE_RM_UDMAP_CH_RX_FLOWID_START_VALID |
627				    TI_SCI_MSG_VALUE_RM_UDMAP_CH_RX_FLOWID_CNT_VALID;
628		req.flowid_start = rx_chn->flow_id_base;
629		req.flowid_cnt = rx_chn->flow_num;
630	}
631	req.rx_chan_type = TI_SCI_RM_UDMAP_CHAN_TYPE_PKT_PBRR;
632	req.rx_atype = rx_chn->common.atype_asel;
633
634	ret = tisci_rm->tisci_udmap_ops->rx_ch_cfg(tisci_rm->tisci, &req);
635	if (ret)
636		dev_err(rx_chn->common.dev, "rchan%d cfg failed %d\n",
637			rx_chn->udma_rchan_id, ret);
638
639	return ret;
640}
641
642static void k3_udma_glue_release_rx_flow(struct k3_udma_glue_rx_channel *rx_chn,
643					 u32 flow_num)
644{
645	struct k3_udma_glue_rx_flow *flow = &rx_chn->flows[flow_num];
646
647	if (IS_ERR_OR_NULL(flow->udma_rflow))
648		return;
649
650	if (flow->ringrxfdq)
651		k3_ringacc_ring_free(flow->ringrxfdq);
652
653	if (flow->ringrx)
654		k3_ringacc_ring_free(flow->ringrx);
655
656	xudma_rflow_put(rx_chn->common.udmax, flow->udma_rflow);
657	flow->udma_rflow = NULL;
658	rx_chn->flows_ready--;
659}
660
661static int k3_udma_glue_cfg_rx_flow(struct k3_udma_glue_rx_channel *rx_chn,
662				    u32 flow_idx,
663				    struct k3_udma_glue_rx_flow_cfg *flow_cfg)
664{
665	struct k3_udma_glue_rx_flow *flow = &rx_chn->flows[flow_idx];
666	const struct udma_tisci_rm *tisci_rm = rx_chn->common.tisci_rm;
667	struct device *dev = rx_chn->common.dev;
668	struct ti_sci_msg_rm_udmap_flow_cfg req;
669	int rx_ring_id;
670	int rx_ringfdq_id;
671	int ret = 0;
672
673	flow->udma_rflow = xudma_rflow_get(rx_chn->common.udmax,
674					   flow->udma_rflow_id);
675	if (IS_ERR(flow->udma_rflow)) {
676		ret = PTR_ERR(flow->udma_rflow);
677		dev_err(dev, "UDMAX rflow get err %d\n", ret);
678		return ret;
679	}
680
681	if (flow->udma_rflow_id != xudma_rflow_get_id(flow->udma_rflow)) {
682		ret = -ENODEV;
683		goto err_rflow_put;
684	}
685
686	if (xudma_is_pktdma(rx_chn->common.udmax)) {
687		rx_ringfdq_id = flow->udma_rflow_id +
688				xudma_get_rflow_ring_offset(rx_chn->common.udmax);
689		rx_ring_id = 0;
690	} else {
691		rx_ring_id = flow_cfg->ring_rxq_id;
692		rx_ringfdq_id = flow_cfg->ring_rxfdq0_id;
693	}
694
695	/* request and cfg rings */
696	ret =  k3_ringacc_request_rings_pair(rx_chn->common.ringacc,
697					     rx_ringfdq_id, rx_ring_id,
698					     &flow->ringrxfdq,
699					     &flow->ringrx);
700	if (ret) {
701		dev_err(dev, "Failed to get RX/RXFDQ rings %d\n", ret);
702		goto err_rflow_put;
703	}
704
705	/* Set the dma_dev for the rings to be configured */
706	flow_cfg->rx_cfg.dma_dev = k3_udma_glue_rx_get_dma_device(rx_chn);
707	flow_cfg->rxfdq_cfg.dma_dev = flow_cfg->rx_cfg.dma_dev;
708
709	/* Set the ASEL value for DMA rings of PKTDMA */
710	if (xudma_is_pktdma(rx_chn->common.udmax)) {
711		flow_cfg->rx_cfg.asel = rx_chn->common.atype_asel;
712		flow_cfg->rxfdq_cfg.asel = rx_chn->common.atype_asel;
713	}
714
715	ret = k3_ringacc_ring_cfg(flow->ringrx, &flow_cfg->rx_cfg);
716	if (ret) {
717		dev_err(dev, "Failed to cfg ringrx %d\n", ret);
718		goto err_ringrxfdq_free;
719	}
720
721	ret = k3_ringacc_ring_cfg(flow->ringrxfdq, &flow_cfg->rxfdq_cfg);
722	if (ret) {
723		dev_err(dev, "Failed to cfg ringrxfdq %d\n", ret);
724		goto err_ringrxfdq_free;
725	}
726
727	if (rx_chn->remote) {
728		rx_ring_id = TI_SCI_RESOURCE_NULL;
729		rx_ringfdq_id = TI_SCI_RESOURCE_NULL;
730	} else {
731		rx_ring_id = k3_ringacc_get_ring_id(flow->ringrx);
732		rx_ringfdq_id = k3_ringacc_get_ring_id(flow->ringrxfdq);
733	}
734
735	memset(&req, 0, sizeof(req));
736
737	req.valid_params =
738			TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_EINFO_PRESENT_VALID |
739			TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_PSINFO_PRESENT_VALID |
740			TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_ERROR_HANDLING_VALID |
741			TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_DESC_TYPE_VALID |
742			TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_DEST_QNUM_VALID |
743			TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_SRC_TAG_HI_SEL_VALID |
744			TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_SRC_TAG_LO_SEL_VALID |
745			TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_DEST_TAG_HI_SEL_VALID |
746			TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_DEST_TAG_LO_SEL_VALID |
747			TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ0_SZ0_QNUM_VALID |
748			TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ1_QNUM_VALID |
749			TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ2_QNUM_VALID |
750			TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ3_QNUM_VALID;
751	req.nav_id = tisci_rm->tisci_dev_id;
752	req.flow_index = flow->udma_rflow_id;
753	if (rx_chn->common.epib)
754		req.rx_einfo_present = 1;
755	if (rx_chn->common.psdata_size)
756		req.rx_psinfo_present = 1;
757	if (flow_cfg->rx_error_handling)
758		req.rx_error_handling = 1;
759	req.rx_desc_type = 0;
760	req.rx_dest_qnum = rx_ring_id;
761	req.rx_src_tag_hi_sel = 0;
762	req.rx_src_tag_lo_sel = flow_cfg->src_tag_lo_sel;
763	req.rx_dest_tag_hi_sel = 0;
764	req.rx_dest_tag_lo_sel = 0;
765	req.rx_fdq0_sz0_qnum = rx_ringfdq_id;
766	req.rx_fdq1_qnum = rx_ringfdq_id;
767	req.rx_fdq2_qnum = rx_ringfdq_id;
768	req.rx_fdq3_qnum = rx_ringfdq_id;
769
770	ret = tisci_rm->tisci_udmap_ops->rx_flow_cfg(tisci_rm->tisci, &req);
771	if (ret) {
772		dev_err(dev, "flow%d config failed: %d\n", flow->udma_rflow_id,
773			ret);
774		goto err_ringrxfdq_free;
775	}
776
777	rx_chn->flows_ready++;
778	dev_dbg(dev, "flow%d config done. ready:%d\n",
779		flow->udma_rflow_id, rx_chn->flows_ready);
780
781	return 0;
782
783err_ringrxfdq_free:
784	k3_ringacc_ring_free(flow->ringrxfdq);
785	k3_ringacc_ring_free(flow->ringrx);
786
787err_rflow_put:
788	xudma_rflow_put(rx_chn->common.udmax, flow->udma_rflow);
789	flow->udma_rflow = NULL;
790
791	return ret;
792}
793
794static void k3_udma_glue_dump_rx_chn(struct k3_udma_glue_rx_channel *chn)
795{
796	struct device *dev = chn->common.dev;
797
798	dev_dbg(dev, "dump_rx_chn:\n"
799		"udma_rchan_id: %d\n"
800		"src_thread: %08x\n"
801		"dst_thread: %08x\n"
802		"epib: %d\n"
803		"hdesc_size: %u\n"
804		"psdata_size: %u\n"
805		"swdata_size: %u\n"
806		"flow_id_base: %d\n"
807		"flow_num: %d\n",
808		chn->udma_rchan_id,
809		chn->common.src_thread,
810		chn->common.dst_thread,
811		chn->common.epib,
812		chn->common.hdesc_size,
813		chn->common.psdata_size,
814		chn->common.swdata_size,
815		chn->flow_id_base,
816		chn->flow_num);
817}
818
819static void k3_udma_glue_dump_rx_rt_chn(struct k3_udma_glue_rx_channel *chn,
820					char *mark)
821{
822	struct device *dev = chn->common.dev;
823
824	dev_dbg(dev, "=== dump ===> %s\n", mark);
825
826	dev_dbg(dev, "0x%08X: %08X\n", UDMA_CHAN_RT_CTL_REG,
827		xudma_rchanrt_read(chn->udma_rchanx, UDMA_CHAN_RT_CTL_REG));
828	dev_dbg(dev, "0x%08X: %08X\n", UDMA_CHAN_RT_PEER_RT_EN_REG,
829		xudma_rchanrt_read(chn->udma_rchanx,
830				   UDMA_CHAN_RT_PEER_RT_EN_REG));
831	dev_dbg(dev, "0x%08X: %08X\n", UDMA_CHAN_RT_PCNT_REG,
832		xudma_rchanrt_read(chn->udma_rchanx, UDMA_CHAN_RT_PCNT_REG));
833	dev_dbg(dev, "0x%08X: %08X\n", UDMA_CHAN_RT_BCNT_REG,
834		xudma_rchanrt_read(chn->udma_rchanx, UDMA_CHAN_RT_BCNT_REG));
835	dev_dbg(dev, "0x%08X: %08X\n", UDMA_CHAN_RT_SBCNT_REG,
836		xudma_rchanrt_read(chn->udma_rchanx, UDMA_CHAN_RT_SBCNT_REG));
837}
838
839static int
840k3_udma_glue_allocate_rx_flows(struct k3_udma_glue_rx_channel *rx_chn,
841			       struct k3_udma_glue_rx_channel_cfg *cfg)
842{
843	int ret;
844
845	/* default rflow */
846	if (cfg->flow_id_use_rxchan_id)
847		return 0;
848
849	/* not a GP rflows */
850	if (rx_chn->flow_id_base != -1 &&
851	    !xudma_rflow_is_gp(rx_chn->common.udmax, rx_chn->flow_id_base))
852		return 0;
853
854	/* Allocate range of GP rflows */
855	ret = xudma_alloc_gp_rflow_range(rx_chn->common.udmax,
856					 rx_chn->flow_id_base,
857					 rx_chn->flow_num);
858	if (ret < 0) {
859		dev_err(rx_chn->common.dev, "UDMAX reserve_rflow %d cnt:%d err: %d\n",
860			rx_chn->flow_id_base, rx_chn->flow_num, ret);
861		return ret;
862	}
863	rx_chn->flow_id_base = ret;
864
865	return 0;
866}
867
868static struct k3_udma_glue_rx_channel *
869k3_udma_glue_request_rx_chn_priv(struct device *dev, const char *name,
870				 struct k3_udma_glue_rx_channel_cfg *cfg)
871{
872	struct k3_udma_glue_rx_channel *rx_chn;
873	struct psil_endpoint_config *ep_cfg;
874	int ret, i;
875
876	if (cfg->flow_id_num <= 0)
877		return ERR_PTR(-EINVAL);
878
879	if (cfg->flow_id_num != 1 &&
880	    (cfg->def_flow_cfg || cfg->flow_id_use_rxchan_id))
881		return ERR_PTR(-EINVAL);
882
883	rx_chn = devm_kzalloc(dev, sizeof(*rx_chn), GFP_KERNEL);
884	if (!rx_chn)
885		return ERR_PTR(-ENOMEM);
886
887	rx_chn->common.dev = dev;
888	rx_chn->common.swdata_size = cfg->swdata_size;
889	rx_chn->remote = false;
890
891	/* parse of udmap channel */
892	ret = of_k3_udma_glue_parse_chn(dev->of_node, name,
893					&rx_chn->common, false);
894	if (ret)
895		goto err;
896
897	rx_chn->common.hdesc_size = cppi5_hdesc_calc_size(rx_chn->common.epib,
898						rx_chn->common.psdata_size,
899						rx_chn->common.swdata_size);
900
901	ep_cfg = rx_chn->common.ep_config;
902
903	if (xudma_is_pktdma(rx_chn->common.udmax))
904		rx_chn->udma_rchan_id = ep_cfg->mapped_channel_id;
905	else
906		rx_chn->udma_rchan_id = -1;
907
908	/* request and cfg UDMAP RX channel */
909	rx_chn->udma_rchanx = xudma_rchan_get(rx_chn->common.udmax,
910					      rx_chn->udma_rchan_id);
911	if (IS_ERR(rx_chn->udma_rchanx)) {
912		ret = PTR_ERR(rx_chn->udma_rchanx);
913		dev_err(dev, "UDMAX rchanx get err %d\n", ret);
914		goto err;
915	}
916	rx_chn->udma_rchan_id = xudma_rchan_get_id(rx_chn->udma_rchanx);
917
918	rx_chn->common.chan_dev.class = &k3_udma_glue_devclass;
919	rx_chn->common.chan_dev.parent = xudma_get_device(rx_chn->common.udmax);
920	dev_set_name(&rx_chn->common.chan_dev, "rchan%d-0x%04x",
921		     rx_chn->udma_rchan_id, rx_chn->common.src_thread);
922	ret = device_register(&rx_chn->common.chan_dev);
923	if (ret) {
924		dev_err(dev, "Channel Device registration failed %d\n", ret);
925		put_device(&rx_chn->common.chan_dev);
926		rx_chn->common.chan_dev.parent = NULL;
927		goto err;
928	}
929
930	if (xudma_is_pktdma(rx_chn->common.udmax)) {
931		/* prepare the channel device as coherent */
932		rx_chn->common.chan_dev.dma_coherent = true;
933		dma_coerce_mask_and_coherent(&rx_chn->common.chan_dev,
934					     DMA_BIT_MASK(48));
935	}
936
937	if (xudma_is_pktdma(rx_chn->common.udmax)) {
938		int flow_start = cfg->flow_id_base;
939		int flow_end;
940
941		if (flow_start == -1)
942			flow_start = ep_cfg->flow_start;
943
944		flow_end = flow_start + cfg->flow_id_num - 1;
945		if (flow_start < ep_cfg->flow_start ||
946		    flow_end > (ep_cfg->flow_start + ep_cfg->flow_num - 1)) {
947			dev_err(dev, "Invalid flow range requested\n");
948			ret = -EINVAL;
949			goto err;
950		}
951		rx_chn->flow_id_base = flow_start;
952	} else {
953		rx_chn->flow_id_base = cfg->flow_id_base;
954
955		/* Use RX channel id as flow id: target dev can't generate flow_id */
956		if (cfg->flow_id_use_rxchan_id)
957			rx_chn->flow_id_base = rx_chn->udma_rchan_id;
958	}
959
960	rx_chn->flow_num = cfg->flow_id_num;
961
962	rx_chn->flows = devm_kcalloc(dev, rx_chn->flow_num,
963				     sizeof(*rx_chn->flows), GFP_KERNEL);
964	if (!rx_chn->flows) {
965		ret = -ENOMEM;
966		goto err;
967	}
968
969	ret = k3_udma_glue_allocate_rx_flows(rx_chn, cfg);
970	if (ret)
971		goto err;
972
973	for (i = 0; i < rx_chn->flow_num; i++)
974		rx_chn->flows[i].udma_rflow_id = rx_chn->flow_id_base + i;
975
976	/* request and cfg psi-l */
977	rx_chn->common.dst_thread =
978			xudma_dev_get_psil_base(rx_chn->common.udmax) +
979			rx_chn->udma_rchan_id;
980
981	ret = k3_udma_glue_cfg_rx_chn(rx_chn);
982	if (ret) {
983		dev_err(dev, "Failed to cfg rchan %d\n", ret);
984		goto err;
985	}
986
987	/* init default RX flow only if flow_num = 1 */
988	if (cfg->def_flow_cfg) {
989		ret = k3_udma_glue_cfg_rx_flow(rx_chn, 0, cfg->def_flow_cfg);
990		if (ret)
991			goto err;
992	}
993
994	k3_udma_glue_dump_rx_chn(rx_chn);
995
996	return rx_chn;
997
998err:
999	k3_udma_glue_release_rx_chn(rx_chn);
1000	return ERR_PTR(ret);
1001}
1002
1003static struct k3_udma_glue_rx_channel *
1004k3_udma_glue_request_remote_rx_chn(struct device *dev, const char *name,
1005				   struct k3_udma_glue_rx_channel_cfg *cfg)
1006{
1007	struct k3_udma_glue_rx_channel *rx_chn;
1008	int ret, i;
1009
1010	if (cfg->flow_id_num <= 0 ||
1011	    cfg->flow_id_use_rxchan_id ||
1012	    cfg->def_flow_cfg ||
1013	    cfg->flow_id_base < 0)
1014		return ERR_PTR(-EINVAL);
1015
1016	/*
1017	 * Remote RX channel is under control of Remote CPU core, so
1018	 * Linux can only request and manipulate by dedicated RX flows
1019	 */
1020
1021	rx_chn = devm_kzalloc(dev, sizeof(*rx_chn), GFP_KERNEL);
1022	if (!rx_chn)
1023		return ERR_PTR(-ENOMEM);
1024
1025	rx_chn->common.dev = dev;
1026	rx_chn->common.swdata_size = cfg->swdata_size;
1027	rx_chn->remote = true;
1028	rx_chn->udma_rchan_id = -1;
1029	rx_chn->flow_num = cfg->flow_id_num;
1030	rx_chn->flow_id_base = cfg->flow_id_base;
1031	rx_chn->psil_paired = false;
1032
1033	/* parse of udmap channel */
1034	ret = of_k3_udma_glue_parse_chn(dev->of_node, name,
1035					&rx_chn->common, false);
1036	if (ret)
1037		goto err;
1038
1039	rx_chn->common.hdesc_size = cppi5_hdesc_calc_size(rx_chn->common.epib,
1040						rx_chn->common.psdata_size,
1041						rx_chn->common.swdata_size);
1042
1043	rx_chn->flows = devm_kcalloc(dev, rx_chn->flow_num,
1044				     sizeof(*rx_chn->flows), GFP_KERNEL);
1045	if (!rx_chn->flows) {
1046		ret = -ENOMEM;
1047		goto err;
1048	}
1049
1050	rx_chn->common.chan_dev.class = &k3_udma_glue_devclass;
1051	rx_chn->common.chan_dev.parent = xudma_get_device(rx_chn->common.udmax);
1052	dev_set_name(&rx_chn->common.chan_dev, "rchan_remote-0x%04x",
1053		     rx_chn->common.src_thread);
1054	ret = device_register(&rx_chn->common.chan_dev);
1055	if (ret) {
1056		dev_err(dev, "Channel Device registration failed %d\n", ret);
1057		put_device(&rx_chn->common.chan_dev);
1058		rx_chn->common.chan_dev.parent = NULL;
1059		goto err;
1060	}
1061
1062	if (xudma_is_pktdma(rx_chn->common.udmax)) {
1063		/* prepare the channel device as coherent */
1064		rx_chn->common.chan_dev.dma_coherent = true;
1065		dma_coerce_mask_and_coherent(&rx_chn->common.chan_dev,
1066					     DMA_BIT_MASK(48));
1067	}
1068
1069	ret = k3_udma_glue_allocate_rx_flows(rx_chn, cfg);
1070	if (ret)
1071		goto err;
1072
1073	for (i = 0; i < rx_chn->flow_num; i++)
1074		rx_chn->flows[i].udma_rflow_id = rx_chn->flow_id_base + i;
1075
1076	k3_udma_glue_dump_rx_chn(rx_chn);
1077
1078	return rx_chn;
1079
1080err:
1081	k3_udma_glue_release_rx_chn(rx_chn);
1082	return ERR_PTR(ret);
1083}
1084
1085struct k3_udma_glue_rx_channel *
1086k3_udma_glue_request_rx_chn(struct device *dev, const char *name,
1087			    struct k3_udma_glue_rx_channel_cfg *cfg)
1088{
1089	if (cfg->remote)
1090		return k3_udma_glue_request_remote_rx_chn(dev, name, cfg);
1091	else
1092		return k3_udma_glue_request_rx_chn_priv(dev, name, cfg);
1093}
1094EXPORT_SYMBOL_GPL(k3_udma_glue_request_rx_chn);
1095
1096void k3_udma_glue_release_rx_chn(struct k3_udma_glue_rx_channel *rx_chn)
1097{
1098	int i;
1099
1100	if (IS_ERR_OR_NULL(rx_chn->common.udmax))
1101		return;
1102
1103	if (rx_chn->psil_paired) {
1104		xudma_navss_psil_unpair(rx_chn->common.udmax,
1105					rx_chn->common.src_thread,
1106					rx_chn->common.dst_thread);
1107		rx_chn->psil_paired = false;
1108	}
1109
1110	for (i = 0; i < rx_chn->flow_num; i++)
1111		k3_udma_glue_release_rx_flow(rx_chn, i);
1112
1113	if (xudma_rflow_is_gp(rx_chn->common.udmax, rx_chn->flow_id_base))
1114		xudma_free_gp_rflow_range(rx_chn->common.udmax,
1115					  rx_chn->flow_id_base,
1116					  rx_chn->flow_num);
1117
1118	if (!IS_ERR_OR_NULL(rx_chn->udma_rchanx))
1119		xudma_rchan_put(rx_chn->common.udmax,
1120				rx_chn->udma_rchanx);
1121
1122	if (rx_chn->common.chan_dev.parent) {
1123		device_unregister(&rx_chn->common.chan_dev);
1124		rx_chn->common.chan_dev.parent = NULL;
1125	}
1126}
1127EXPORT_SYMBOL_GPL(k3_udma_glue_release_rx_chn);
1128
1129int k3_udma_glue_rx_flow_init(struct k3_udma_glue_rx_channel *rx_chn,
1130			      u32 flow_idx,
1131			      struct k3_udma_glue_rx_flow_cfg *flow_cfg)
1132{
1133	if (flow_idx >= rx_chn->flow_num)
1134		return -EINVAL;
1135
1136	return k3_udma_glue_cfg_rx_flow(rx_chn, flow_idx, flow_cfg);
1137}
1138EXPORT_SYMBOL_GPL(k3_udma_glue_rx_flow_init);
1139
1140u32 k3_udma_glue_rx_flow_get_fdq_id(struct k3_udma_glue_rx_channel *rx_chn,
1141				    u32 flow_idx)
1142{
1143	struct k3_udma_glue_rx_flow *flow;
1144
1145	if (flow_idx >= rx_chn->flow_num)
1146		return -EINVAL;
1147
1148	flow = &rx_chn->flows[flow_idx];
1149
1150	return k3_ringacc_get_ring_id(flow->ringrxfdq);
1151}
1152EXPORT_SYMBOL_GPL(k3_udma_glue_rx_flow_get_fdq_id);
1153
1154u32 k3_udma_glue_rx_get_flow_id_base(struct k3_udma_glue_rx_channel *rx_chn)
1155{
1156	return rx_chn->flow_id_base;
1157}
1158EXPORT_SYMBOL_GPL(k3_udma_glue_rx_get_flow_id_base);
1159
1160int k3_udma_glue_rx_flow_enable(struct k3_udma_glue_rx_channel *rx_chn,
1161				u32 flow_idx)
1162{
1163	struct k3_udma_glue_rx_flow *flow = &rx_chn->flows[flow_idx];
1164	const struct udma_tisci_rm *tisci_rm = rx_chn->common.tisci_rm;
1165	struct device *dev = rx_chn->common.dev;
1166	struct ti_sci_msg_rm_udmap_flow_cfg req;
1167	int rx_ring_id;
1168	int rx_ringfdq_id;
1169	int ret = 0;
1170
1171	if (!rx_chn->remote)
1172		return -EINVAL;
1173
1174	rx_ring_id = k3_ringacc_get_ring_id(flow->ringrx);
1175	rx_ringfdq_id = k3_ringacc_get_ring_id(flow->ringrxfdq);
1176
1177	memset(&req, 0, sizeof(req));
1178
1179	req.valid_params =
1180			TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_DEST_QNUM_VALID |
1181			TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ0_SZ0_QNUM_VALID |
1182			TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ1_QNUM_VALID |
1183			TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ2_QNUM_VALID |
1184			TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ3_QNUM_VALID;
1185	req.nav_id = tisci_rm->tisci_dev_id;
1186	req.flow_index = flow->udma_rflow_id;
1187	req.rx_dest_qnum = rx_ring_id;
1188	req.rx_fdq0_sz0_qnum = rx_ringfdq_id;
1189	req.rx_fdq1_qnum = rx_ringfdq_id;
1190	req.rx_fdq2_qnum = rx_ringfdq_id;
1191	req.rx_fdq3_qnum = rx_ringfdq_id;
1192
1193	ret = tisci_rm->tisci_udmap_ops->rx_flow_cfg(tisci_rm->tisci, &req);
1194	if (ret) {
1195		dev_err(dev, "flow%d enable failed: %d\n", flow->udma_rflow_id,
1196			ret);
1197	}
1198
1199	return ret;
1200}
1201EXPORT_SYMBOL_GPL(k3_udma_glue_rx_flow_enable);
1202
1203int k3_udma_glue_rx_flow_disable(struct k3_udma_glue_rx_channel *rx_chn,
1204				 u32 flow_idx)
1205{
1206	struct k3_udma_glue_rx_flow *flow = &rx_chn->flows[flow_idx];
1207	const struct udma_tisci_rm *tisci_rm = rx_chn->common.tisci_rm;
1208	struct device *dev = rx_chn->common.dev;
1209	struct ti_sci_msg_rm_udmap_flow_cfg req;
1210	int ret = 0;
1211
1212	if (!rx_chn->remote)
1213		return -EINVAL;
1214
1215	memset(&req, 0, sizeof(req));
1216	req.valid_params =
1217			TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_DEST_QNUM_VALID |
1218			TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ0_SZ0_QNUM_VALID |
1219			TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ1_QNUM_VALID |
1220			TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ2_QNUM_VALID |
1221			TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ3_QNUM_VALID;
1222	req.nav_id = tisci_rm->tisci_dev_id;
1223	req.flow_index = flow->udma_rflow_id;
1224	req.rx_dest_qnum = TI_SCI_RESOURCE_NULL;
1225	req.rx_fdq0_sz0_qnum = TI_SCI_RESOURCE_NULL;
1226	req.rx_fdq1_qnum = TI_SCI_RESOURCE_NULL;
1227	req.rx_fdq2_qnum = TI_SCI_RESOURCE_NULL;
1228	req.rx_fdq3_qnum = TI_SCI_RESOURCE_NULL;
1229
1230	ret = tisci_rm->tisci_udmap_ops->rx_flow_cfg(tisci_rm->tisci, &req);
1231	if (ret) {
1232		dev_err(dev, "flow%d disable failed: %d\n", flow->udma_rflow_id,
1233			ret);
1234	}
1235
1236	return ret;
1237}
1238EXPORT_SYMBOL_GPL(k3_udma_glue_rx_flow_disable);
1239
1240int k3_udma_glue_enable_rx_chn(struct k3_udma_glue_rx_channel *rx_chn)
1241{
1242	int ret;
1243
1244	if (rx_chn->remote)
1245		return -EINVAL;
1246
1247	if (rx_chn->flows_ready < rx_chn->flow_num)
1248		return -EINVAL;
1249
1250	ret = xudma_navss_psil_pair(rx_chn->common.udmax,
1251				    rx_chn->common.src_thread,
1252				    rx_chn->common.dst_thread);
1253	if (ret) {
1254		dev_err(rx_chn->common.dev, "PSI-L request err %d\n", ret);
1255		return ret;
1256	}
1257
1258	rx_chn->psil_paired = true;
1259
1260	xudma_rchanrt_write(rx_chn->udma_rchanx, UDMA_CHAN_RT_CTL_REG,
1261			    UDMA_CHAN_RT_CTL_EN);
1262
1263	xudma_rchanrt_write(rx_chn->udma_rchanx, UDMA_CHAN_RT_PEER_RT_EN_REG,
1264			    UDMA_PEER_RT_EN_ENABLE);
1265
1266	k3_udma_glue_dump_rx_rt_chn(rx_chn, "rxrt en");
1267	return 0;
1268}
1269EXPORT_SYMBOL_GPL(k3_udma_glue_enable_rx_chn);
1270
1271void k3_udma_glue_disable_rx_chn(struct k3_udma_glue_rx_channel *rx_chn)
1272{
1273	k3_udma_glue_dump_rx_rt_chn(rx_chn, "rxrt dis1");
1274
1275	xudma_rchanrt_write(rx_chn->udma_rchanx,
1276			    UDMA_CHAN_RT_PEER_RT_EN_REG, 0);
1277	xudma_rchanrt_write(rx_chn->udma_rchanx, UDMA_CHAN_RT_CTL_REG, 0);
1278
1279	k3_udma_glue_dump_rx_rt_chn(rx_chn, "rxrt dis2");
1280
1281	if (rx_chn->psil_paired) {
1282		xudma_navss_psil_unpair(rx_chn->common.udmax,
1283					rx_chn->common.src_thread,
1284					rx_chn->common.dst_thread);
1285		rx_chn->psil_paired = false;
1286	}
1287}
1288EXPORT_SYMBOL_GPL(k3_udma_glue_disable_rx_chn);
1289
1290void k3_udma_glue_tdown_rx_chn(struct k3_udma_glue_rx_channel *rx_chn,
1291			       bool sync)
1292{
1293	int i = 0;
1294	u32 val;
1295
1296	if (rx_chn->remote)
1297		return;
1298
1299	k3_udma_glue_dump_rx_rt_chn(rx_chn, "rxrt tdown1");
1300
1301	xudma_rchanrt_write(rx_chn->udma_rchanx, UDMA_CHAN_RT_PEER_RT_EN_REG,
1302			    UDMA_PEER_RT_EN_ENABLE | UDMA_PEER_RT_EN_TEARDOWN);
1303
1304	val = xudma_rchanrt_read(rx_chn->udma_rchanx, UDMA_CHAN_RT_CTL_REG);
1305
1306	while (sync && (val & UDMA_CHAN_RT_CTL_EN)) {
1307		val = xudma_rchanrt_read(rx_chn->udma_rchanx,
1308					 UDMA_CHAN_RT_CTL_REG);
1309		udelay(1);
1310		if (i > K3_UDMAX_TDOWN_TIMEOUT_US) {
1311			dev_err(rx_chn->common.dev, "RX tdown timeout\n");
1312			break;
1313		}
1314		i++;
1315	}
1316
1317	val = xudma_rchanrt_read(rx_chn->udma_rchanx,
1318				 UDMA_CHAN_RT_PEER_RT_EN_REG);
1319	if (sync && (val & UDMA_PEER_RT_EN_ENABLE))
1320		dev_err(rx_chn->common.dev, "TX tdown peer not stopped\n");
1321	k3_udma_glue_dump_rx_rt_chn(rx_chn, "rxrt tdown2");
1322}
1323EXPORT_SYMBOL_GPL(k3_udma_glue_tdown_rx_chn);
1324
1325void k3_udma_glue_reset_rx_chn(struct k3_udma_glue_rx_channel *rx_chn,
1326		u32 flow_num, void *data,
1327		void (*cleanup)(void *data, dma_addr_t desc_dma), bool skip_fdq)
1328{
1329	struct k3_udma_glue_rx_flow *flow = &rx_chn->flows[flow_num];
1330	struct device *dev = rx_chn->common.dev;
1331	dma_addr_t desc_dma;
1332	int occ_rx, i, ret;
1333
1334	/* reset RXCQ as it is not input for udma - expected to be empty */
1335	occ_rx = k3_ringacc_ring_get_occ(flow->ringrx);
1336	dev_dbg(dev, "RX reset flow %u occ_rx %u\n", flow_num, occ_rx);
1337
1338	/* Skip RX FDQ in case one FDQ is used for the set of flows */
1339	if (skip_fdq)
1340		goto do_reset;
1341
1342	/*
1343	 * RX FDQ reset need to be special way as it is input for udma and its
1344	 * state cached by udma, so:
1345	 * 1) save RX FDQ occ
1346	 * 2) clean up RX FDQ and call callback .cleanup() for each desc
1347	 * 3) reset RX FDQ in a special way
1348	 */
1349	occ_rx = k3_ringacc_ring_get_occ(flow->ringrxfdq);
1350	dev_dbg(dev, "RX reset flow %u occ_rx_fdq %u\n", flow_num, occ_rx);
1351
1352	for (i = 0; i < occ_rx; i++) {
1353		ret = k3_ringacc_ring_pop(flow->ringrxfdq, &desc_dma);
1354		if (ret) {
1355			if (ret != -ENODATA)
1356				dev_err(dev, "RX reset pop %d\n", ret);
1357			break;
1358		}
1359		cleanup(data, desc_dma);
1360	}
1361
1362	k3_ringacc_ring_reset_dma(flow->ringrxfdq, occ_rx);
1363
1364do_reset:
1365	k3_ringacc_ring_reset(flow->ringrx);
1366}
1367EXPORT_SYMBOL_GPL(k3_udma_glue_reset_rx_chn);
1368
1369int k3_udma_glue_push_rx_chn(struct k3_udma_glue_rx_channel *rx_chn,
1370			     u32 flow_num, struct cppi5_host_desc_t *desc_rx,
1371			     dma_addr_t desc_dma)
1372{
1373	struct k3_udma_glue_rx_flow *flow = &rx_chn->flows[flow_num];
1374
1375	return k3_ringacc_ring_push(flow->ringrxfdq, &desc_dma);
1376}
1377EXPORT_SYMBOL_GPL(k3_udma_glue_push_rx_chn);
1378
1379int k3_udma_glue_pop_rx_chn(struct k3_udma_glue_rx_channel *rx_chn,
1380			    u32 flow_num, dma_addr_t *desc_dma)
1381{
1382	struct k3_udma_glue_rx_flow *flow = &rx_chn->flows[flow_num];
1383
1384	return k3_ringacc_ring_pop(flow->ringrx, desc_dma);
1385}
1386EXPORT_SYMBOL_GPL(k3_udma_glue_pop_rx_chn);
1387
1388int k3_udma_glue_rx_get_irq(struct k3_udma_glue_rx_channel *rx_chn,
1389			    u32 flow_num)
1390{
1391	struct k3_udma_glue_rx_flow *flow;
1392
1393	flow = &rx_chn->flows[flow_num];
1394
1395	if (xudma_is_pktdma(rx_chn->common.udmax)) {
1396		flow->virq = xudma_pktdma_rflow_get_irq(rx_chn->common.udmax,
1397							flow->udma_rflow_id);
1398	} else {
1399		flow->virq = k3_ringacc_get_ring_irq_num(flow->ringrx);
1400	}
1401
1402	return flow->virq;
1403}
1404EXPORT_SYMBOL_GPL(k3_udma_glue_rx_get_irq);
1405
1406struct device *
1407	k3_udma_glue_rx_get_dma_device(struct k3_udma_glue_rx_channel *rx_chn)
1408{
1409	if (xudma_is_pktdma(rx_chn->common.udmax) &&
1410	    (rx_chn->common.atype_asel == 14 || rx_chn->common.atype_asel == 15))
1411		return &rx_chn->common.chan_dev;
1412
1413	return xudma_get_device(rx_chn->common.udmax);
1414}
1415EXPORT_SYMBOL_GPL(k3_udma_glue_rx_get_dma_device);
1416
1417void k3_udma_glue_rx_dma_to_cppi5_addr(struct k3_udma_glue_rx_channel *rx_chn,
1418				       dma_addr_t *addr)
1419{
1420	if (!xudma_is_pktdma(rx_chn->common.udmax) ||
1421	    !rx_chn->common.atype_asel)
1422		return;
1423
1424	*addr |= (u64)rx_chn->common.atype_asel << K3_ADDRESS_ASEL_SHIFT;
1425}
1426EXPORT_SYMBOL_GPL(k3_udma_glue_rx_dma_to_cppi5_addr);
1427
1428void k3_udma_glue_rx_cppi5_to_dma_addr(struct k3_udma_glue_rx_channel *rx_chn,
1429				       dma_addr_t *addr)
1430{
1431	if (!xudma_is_pktdma(rx_chn->common.udmax) ||
1432	    !rx_chn->common.atype_asel)
1433		return;
1434
1435	*addr &= (u64)GENMASK(K3_ADDRESS_ASEL_SHIFT - 1, 0);
1436}
1437EXPORT_SYMBOL_GPL(k3_udma_glue_rx_cppi5_to_dma_addr);
1438
1439static int __init k3_udma_glue_class_init(void)
1440{
1441	return class_register(&k3_udma_glue_devclass);
1442}
1443
1444module_init(k3_udma_glue_class_init);
1445MODULE_LICENSE("GPL v2");
1446