1// SPDX-License-Identifier: GPL-2.0
2/*
3 * sl3516-ce-core.c - hardware cryptographic offloader for Storlink SL3516 SoC
4 *
5 * Copyright (C) 2021 Corentin Labbe <clabbe@baylibre.com>
6 *
7 * Core file which registers crypto algorithms supported by the CryptoEngine
8 */
9
10#include <crypto/engine.h>
11#include <crypto/internal/rng.h>
12#include <crypto/internal/skcipher.h>
13#include <linux/clk.h>
14#include <linux/debugfs.h>
15#include <linux/dev_printk.h>
16#include <linux/dma-mapping.h>
17#include <linux/err.h>
18#include <linux/interrupt.h>
19#include <linux/io.h>
20#include <linux/irq.h>
21#include <linux/kernel.h>
22#include <linux/module.h>
23#include <linux/of.h>
24#include <linux/platform_device.h>
25#include <linux/pm_runtime.h>
26#include <linux/reset.h>
27
28#include "sl3516-ce.h"
29
30static int sl3516_ce_desc_init(struct sl3516_ce_dev *ce)
31{
32	const size_t sz = sizeof(struct descriptor) * MAXDESC;
33	int i;
34
35	ce->tx = dma_alloc_coherent(ce->dev, sz, &ce->dtx, GFP_KERNEL);
36	if (!ce->tx)
37		return -ENOMEM;
38	ce->rx = dma_alloc_coherent(ce->dev, sz, &ce->drx, GFP_KERNEL);
39	if (!ce->rx)
40		goto err_rx;
41
42	for (i = 0; i < MAXDESC; i++) {
43		ce->tx[i].frame_ctrl.bits.own = CE_CPU;
44		ce->tx[i].next_desc.next_descriptor = ce->dtx + (i + 1) * sizeof(struct descriptor);
45	}
46	ce->tx[MAXDESC - 1].next_desc.next_descriptor = ce->dtx;
47
48	for (i = 0; i < MAXDESC; i++) {
49		ce->rx[i].frame_ctrl.bits.own = CE_CPU;
50		ce->rx[i].next_desc.next_descriptor = ce->drx + (i + 1) * sizeof(struct descriptor);
51	}
52	ce->rx[MAXDESC - 1].next_desc.next_descriptor = ce->drx;
53
54	ce->pctrl = dma_alloc_coherent(ce->dev, sizeof(struct pkt_control_ecb),
55				       &ce->dctrl, GFP_KERNEL);
56	if (!ce->pctrl)
57		goto err_pctrl;
58
59	return 0;
60err_pctrl:
61	dma_free_coherent(ce->dev, sz, ce->rx, ce->drx);
62err_rx:
63	dma_free_coherent(ce->dev, sz, ce->tx, ce->dtx);
64	return -ENOMEM;
65}
66
67static void sl3516_ce_free_descs(struct sl3516_ce_dev *ce)
68{
69	const size_t sz = sizeof(struct descriptor) * MAXDESC;
70
71	dma_free_coherent(ce->dev, sz, ce->tx, ce->dtx);
72	dma_free_coherent(ce->dev, sz, ce->rx, ce->drx);
73	dma_free_coherent(ce->dev, sizeof(struct pkt_control_ecb), ce->pctrl,
74			  ce->dctrl);
75}
76
77static void start_dma_tx(struct sl3516_ce_dev *ce)
78{
79	u32 v;
80
81	v = TXDMA_CTRL_START | TXDMA_CTRL_CHAIN_MODE | TXDMA_CTRL_CONTINUE | \
82		TXDMA_CTRL_INT_FAIL | TXDMA_CTRL_INT_PERR | TXDMA_CTRL_BURST_UNK;
83
84	writel(v, ce->base + IPSEC_TXDMA_CTRL);
85}
86
87static void start_dma_rx(struct sl3516_ce_dev *ce)
88{
89	u32 v;
90
91	v = RXDMA_CTRL_START | RXDMA_CTRL_CHAIN_MODE | RXDMA_CTRL_CONTINUE | \
92		RXDMA_CTRL_BURST_UNK | RXDMA_CTRL_INT_FINISH | \
93		RXDMA_CTRL_INT_FAIL | RXDMA_CTRL_INT_PERR | \
94		RXDMA_CTRL_INT_EOD | RXDMA_CTRL_INT_EOF;
95
96	writel(v, ce->base + IPSEC_RXDMA_CTRL);
97}
98
99static struct descriptor *get_desc_tx(struct sl3516_ce_dev *ce)
100{
101	struct descriptor *dd;
102
103	dd = &ce->tx[ce->ctx];
104	ce->ctx++;
105	if (ce->ctx >= MAXDESC)
106		ce->ctx = 0;
107	return dd;
108}
109
110static struct descriptor *get_desc_rx(struct sl3516_ce_dev *ce)
111{
112	struct descriptor *rdd;
113
114	rdd = &ce->rx[ce->crx];
115	ce->crx++;
116	if (ce->crx >= MAXDESC)
117		ce->crx = 0;
118	return rdd;
119}
120
121int sl3516_ce_run_task(struct sl3516_ce_dev *ce, struct sl3516_ce_cipher_req_ctx *rctx,
122		       const char *name)
123{
124	struct descriptor *dd, *rdd = NULL;
125	u32 v;
126	int i, err = 0;
127
128	ce->stat_req++;
129
130	reinit_completion(&ce->complete);
131	ce->status = 0;
132
133	for (i = 0; i < rctx->nr_sgd; i++) {
134		dev_dbg(ce->dev, "%s handle DST SG %d/%d len=%d\n", __func__,
135			i, rctx->nr_sgd, rctx->t_dst[i].len);
136		rdd = get_desc_rx(ce);
137		rdd->buf_adr = rctx->t_dst[i].addr;
138		rdd->frame_ctrl.bits.buffer_size = rctx->t_dst[i].len;
139		rdd->frame_ctrl.bits.own = CE_DMA;
140	}
141	rdd->next_desc.bits.eofie = 1;
142
143	for (i = 0; i < rctx->nr_sgs; i++) {
144		dev_dbg(ce->dev, "%s handle SRC SG %d/%d len=%d\n", __func__,
145			i, rctx->nr_sgs, rctx->t_src[i].len);
146		rctx->h->algorithm_len = rctx->t_src[i].len;
147
148		dd = get_desc_tx(ce);
149		dd->frame_ctrl.raw = 0;
150		dd->flag_status.raw = 0;
151		dd->frame_ctrl.bits.buffer_size = rctx->pctrllen;
152		dd->buf_adr = ce->dctrl;
153		dd->flag_status.tx_flag.tqflag = rctx->tqflag;
154		dd->next_desc.bits.eofie = 0;
155		dd->next_desc.bits.dec = 0;
156		dd->next_desc.bits.sof_eof = DESC_FIRST | DESC_LAST;
157		dd->frame_ctrl.bits.own = CE_DMA;
158
159		dd = get_desc_tx(ce);
160		dd->frame_ctrl.raw = 0;
161		dd->flag_status.raw = 0;
162		dd->frame_ctrl.bits.buffer_size = rctx->t_src[i].len;
163		dd->buf_adr = rctx->t_src[i].addr;
164		dd->flag_status.tx_flag.tqflag = 0;
165		dd->next_desc.bits.eofie = 0;
166		dd->next_desc.bits.dec = 0;
167		dd->next_desc.bits.sof_eof = DESC_FIRST | DESC_LAST;
168		dd->frame_ctrl.bits.own = CE_DMA;
169		start_dma_tx(ce);
170		start_dma_rx(ce);
171	}
172	wait_for_completion_interruptible_timeout(&ce->complete,
173						  msecs_to_jiffies(5000));
174	if (ce->status == 0) {
175		dev_err(ce->dev, "DMA timeout for %s\n", name);
176		err = -EFAULT;
177	}
178	v = readl(ce->base + IPSEC_STATUS_REG);
179	if (v & 0xFFF) {
180		dev_err(ce->dev, "IPSEC_STATUS_REG %x\n", v);
181		err = -EFAULT;
182	}
183
184	return err;
185}
186
187static irqreturn_t ce_irq_handler(int irq, void *data)
188{
189	struct sl3516_ce_dev *ce = (struct sl3516_ce_dev *)data;
190	u32 v;
191
192	ce->stat_irq++;
193
194	v = readl(ce->base + IPSEC_DMA_STATUS);
195	writel(v, ce->base + IPSEC_DMA_STATUS);
196
197	if (v & DMA_STATUS_TS_DERR)
198		dev_err(ce->dev, "AHB bus Error While Tx !!!\n");
199	if (v & DMA_STATUS_TS_PERR)
200		dev_err(ce->dev, "Tx Descriptor Protocol Error !!!\n");
201	if (v & DMA_STATUS_RS_DERR)
202		dev_err(ce->dev, "AHB bus Error While Rx !!!\n");
203	if (v & DMA_STATUS_RS_PERR)
204		dev_err(ce->dev, "Rx Descriptor Protocol Error !!!\n");
205
206	if (v & DMA_STATUS_TS_EOFI)
207		ce->stat_irq_tx++;
208	if (v & DMA_STATUS_RS_EOFI) {
209		ce->status = 1;
210		complete(&ce->complete);
211		ce->stat_irq_rx++;
212		return IRQ_HANDLED;
213	}
214
215	return IRQ_HANDLED;
216}
217
218static struct sl3516_ce_alg_template ce_algs[] = {
219{
220	.type = CRYPTO_ALG_TYPE_SKCIPHER,
221	.mode = ECB_AES,
222	.alg.skcipher.base = {
223		.base = {
224			.cra_name = "ecb(aes)",
225			.cra_driver_name = "ecb-aes-sl3516",
226			.cra_priority = 400,
227			.cra_blocksize = AES_BLOCK_SIZE,
228			.cra_flags = CRYPTO_ALG_TYPE_SKCIPHER |
229				CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK,
230			.cra_ctxsize = sizeof(struct sl3516_ce_cipher_tfm_ctx),
231			.cra_module = THIS_MODULE,
232			.cra_alignmask = 0xf,
233			.cra_init = sl3516_ce_cipher_init,
234			.cra_exit = sl3516_ce_cipher_exit,
235		},
236		.min_keysize	= AES_MIN_KEY_SIZE,
237		.max_keysize	= AES_MAX_KEY_SIZE,
238		.setkey		= sl3516_ce_aes_setkey,
239		.encrypt	= sl3516_ce_skencrypt,
240		.decrypt	= sl3516_ce_skdecrypt,
241	},
242	.alg.skcipher.op = {
243		.do_one_request = sl3516_ce_handle_cipher_request,
244	},
245},
246};
247
248static int sl3516_ce_debugfs_show(struct seq_file *seq, void *v)
249{
250	struct sl3516_ce_dev *ce = seq->private;
251	unsigned int i;
252
253	seq_printf(seq, "HWRNG %lu %lu\n",
254		   ce->hwrng_stat_req, ce->hwrng_stat_bytes);
255	seq_printf(seq, "IRQ %lu\n", ce->stat_irq);
256	seq_printf(seq, "IRQ TX %lu\n", ce->stat_irq_tx);
257	seq_printf(seq, "IRQ RX %lu\n", ce->stat_irq_rx);
258	seq_printf(seq, "nreq %lu\n", ce->stat_req);
259	seq_printf(seq, "fallback SG count TX %lu\n", ce->fallback_sg_count_tx);
260	seq_printf(seq, "fallback SG count RX %lu\n", ce->fallback_sg_count_rx);
261	seq_printf(seq, "fallback modulo16 %lu\n", ce->fallback_mod16);
262	seq_printf(seq, "fallback align16 %lu\n", ce->fallback_align16);
263	seq_printf(seq, "fallback not same len %lu\n", ce->fallback_not_same_len);
264
265	for (i = 0; i < ARRAY_SIZE(ce_algs); i++) {
266		if (!ce_algs[i].ce)
267			continue;
268		switch (ce_algs[i].type) {
269		case CRYPTO_ALG_TYPE_SKCIPHER:
270			seq_printf(seq, "%s %s reqs=%lu fallback=%lu\n",
271				   ce_algs[i].alg.skcipher.base.base.cra_driver_name,
272				   ce_algs[i].alg.skcipher.base.base.cra_name,
273				   ce_algs[i].stat_req, ce_algs[i].stat_fb);
274			break;
275		}
276	}
277	return 0;
278}
279
280DEFINE_SHOW_ATTRIBUTE(sl3516_ce_debugfs);
281
282static int sl3516_ce_register_algs(struct sl3516_ce_dev *ce)
283{
284	int err;
285	unsigned int i;
286
287	for (i = 0; i < ARRAY_SIZE(ce_algs); i++) {
288		ce_algs[i].ce = ce;
289		switch (ce_algs[i].type) {
290		case CRYPTO_ALG_TYPE_SKCIPHER:
291			dev_info(ce->dev, "DEBUG: Register %s\n",
292				 ce_algs[i].alg.skcipher.base.base.cra_name);
293			err = crypto_engine_register_skcipher(&ce_algs[i].alg.skcipher);
294			if (err) {
295				dev_err(ce->dev, "Fail to register %s\n",
296					ce_algs[i].alg.skcipher.base.base.cra_name);
297				ce_algs[i].ce = NULL;
298				return err;
299			}
300			break;
301		default:
302			ce_algs[i].ce = NULL;
303			dev_err(ce->dev, "ERROR: tried to register an unknown algo\n");
304		}
305	}
306	return 0;
307}
308
309static void sl3516_ce_unregister_algs(struct sl3516_ce_dev *ce)
310{
311	unsigned int i;
312
313	for (i = 0; i < ARRAY_SIZE(ce_algs); i++) {
314		if (!ce_algs[i].ce)
315			continue;
316		switch (ce_algs[i].type) {
317		case CRYPTO_ALG_TYPE_SKCIPHER:
318			dev_info(ce->dev, "Unregister %d %s\n", i,
319				 ce_algs[i].alg.skcipher.base.base.cra_name);
320			crypto_engine_unregister_skcipher(&ce_algs[i].alg.skcipher);
321			break;
322		}
323	}
324}
325
326static void sl3516_ce_start(struct sl3516_ce_dev *ce)
327{
328	ce->ctx = 0;
329	ce->crx = 0;
330	writel(ce->dtx, ce->base + IPSEC_TXDMA_CURR_DESC);
331	writel(ce->drx, ce->base + IPSEC_RXDMA_CURR_DESC);
332	writel(0, ce->base + IPSEC_DMA_STATUS);
333}
334
335/*
336 * Power management strategy: The device is suspended unless a TFM exists for
337 * one of the algorithms proposed by this driver.
338 */
339static int sl3516_ce_pm_suspend(struct device *dev)
340{
341	struct sl3516_ce_dev *ce = dev_get_drvdata(dev);
342
343	reset_control_assert(ce->reset);
344	clk_disable_unprepare(ce->clks);
345	return 0;
346}
347
348static int sl3516_ce_pm_resume(struct device *dev)
349{
350	struct sl3516_ce_dev *ce = dev_get_drvdata(dev);
351	int err;
352
353	err = clk_prepare_enable(ce->clks);
354	if (err) {
355		dev_err(ce->dev, "Cannot prepare_enable\n");
356		goto error;
357	}
358	err = reset_control_deassert(ce->reset);
359	if (err) {
360		dev_err(ce->dev, "Cannot deassert reset control\n");
361		goto error;
362	}
363
364	sl3516_ce_start(ce);
365
366	return 0;
367error:
368	sl3516_ce_pm_suspend(dev);
369	return err;
370}
371
372static const struct dev_pm_ops sl3516_ce_pm_ops = {
373	SET_RUNTIME_PM_OPS(sl3516_ce_pm_suspend, sl3516_ce_pm_resume, NULL)
374};
375
376static int sl3516_ce_pm_init(struct sl3516_ce_dev *ce)
377{
378	int err;
379
380	pm_runtime_use_autosuspend(ce->dev);
381	pm_runtime_set_autosuspend_delay(ce->dev, 2000);
382
383	err = pm_runtime_set_suspended(ce->dev);
384	if (err)
385		return err;
386	pm_runtime_enable(ce->dev);
387	return err;
388}
389
390static void sl3516_ce_pm_exit(struct sl3516_ce_dev *ce)
391{
392	pm_runtime_disable(ce->dev);
393}
394
395static int sl3516_ce_probe(struct platform_device *pdev)
396{
397	struct sl3516_ce_dev *ce;
398	int err, irq;
399	u32 v;
400
401	ce = devm_kzalloc(&pdev->dev, sizeof(*ce), GFP_KERNEL);
402	if (!ce)
403		return -ENOMEM;
404
405	ce->dev = &pdev->dev;
406	platform_set_drvdata(pdev, ce);
407
408	ce->base = devm_platform_ioremap_resource(pdev, 0);
409	if (IS_ERR(ce->base))
410		return PTR_ERR(ce->base);
411
412	irq = platform_get_irq(pdev, 0);
413	if (irq < 0)
414		return irq;
415
416	err = devm_request_irq(&pdev->dev, irq, ce_irq_handler, 0, "crypto", ce);
417	if (err) {
418		dev_err(ce->dev, "Cannot request Crypto Engine IRQ (err=%d)\n", err);
419		return err;
420	}
421
422	ce->reset = devm_reset_control_get(&pdev->dev, NULL);
423	if (IS_ERR(ce->reset))
424		return dev_err_probe(&pdev->dev, PTR_ERR(ce->reset),
425				     "No reset control found\n");
426	ce->clks = devm_clk_get(ce->dev, NULL);
427	if (IS_ERR(ce->clks)) {
428		err = PTR_ERR(ce->clks);
429		dev_err(ce->dev, "Cannot get clock err=%d\n", err);
430		return err;
431	}
432
433	err = sl3516_ce_desc_init(ce);
434	if (err)
435		return err;
436
437	err = sl3516_ce_pm_init(ce);
438	if (err)
439		goto error_pm;
440
441	init_completion(&ce->complete);
442
443	ce->engine = crypto_engine_alloc_init(ce->dev, true);
444	if (!ce->engine) {
445		dev_err(ce->dev, "Cannot allocate engine\n");
446		err = -ENOMEM;
447		goto error_engine;
448	}
449
450	err = crypto_engine_start(ce->engine);
451	if (err) {
452		dev_err(ce->dev, "Cannot start engine\n");
453		goto error_engine;
454	}
455
456	err = sl3516_ce_register_algs(ce);
457	if (err)
458		goto error_alg;
459
460	err = sl3516_ce_rng_register(ce);
461	if (err)
462		goto error_rng;
463
464	err = pm_runtime_resume_and_get(ce->dev);
465	if (err < 0)
466		goto error_pmuse;
467
468	v = readl(ce->base + IPSEC_ID);
469	dev_info(ce->dev, "SL3516 dev %lx rev %lx\n",
470		 v & GENMASK(31, 4),
471		 v & GENMASK(3, 0));
472	v = readl(ce->base + IPSEC_DMA_DEVICE_ID);
473	dev_info(ce->dev, "SL3516 DMA dev %lx rev %lx\n",
474		 v & GENMASK(15, 4),
475		 v & GENMASK(3, 0));
476
477	pm_runtime_put_sync(ce->dev);
478
479	if (IS_ENABLED(CONFIG_CRYPTO_DEV_SL3516_DEBUG)) {
480		struct dentry *dbgfs_dir __maybe_unused;
481		struct dentry *dbgfs_stats __maybe_unused;
482
483		/* Ignore error of debugfs */
484		dbgfs_dir = debugfs_create_dir("sl3516", NULL);
485		dbgfs_stats = debugfs_create_file("stats", 0444,
486						  dbgfs_dir, ce,
487						  &sl3516_ce_debugfs_fops);
488#ifdef CONFIG_CRYPTO_DEV_SL3516_DEBUG
489		ce->dbgfs_dir = dbgfs_dir;
490		ce->dbgfs_stats = dbgfs_stats;
491#endif
492	}
493
494	return 0;
495error_pmuse:
496	sl3516_ce_rng_unregister(ce);
497error_rng:
498	sl3516_ce_unregister_algs(ce);
499error_alg:
500	crypto_engine_exit(ce->engine);
501error_engine:
502	sl3516_ce_pm_exit(ce);
503error_pm:
504	sl3516_ce_free_descs(ce);
505	return err;
506}
507
508static int sl3516_ce_remove(struct platform_device *pdev)
509{
510	struct sl3516_ce_dev *ce = platform_get_drvdata(pdev);
511
512	sl3516_ce_rng_unregister(ce);
513	sl3516_ce_unregister_algs(ce);
514	crypto_engine_exit(ce->engine);
515	sl3516_ce_pm_exit(ce);
516	sl3516_ce_free_descs(ce);
517
518#ifdef CONFIG_CRYPTO_DEV_SL3516_DEBUG
519	debugfs_remove_recursive(ce->dbgfs_dir);
520#endif
521
522	return 0;
523}
524
525static const struct of_device_id sl3516_ce_crypto_of_match_table[] = {
526	{ .compatible = "cortina,sl3516-crypto"},
527	{}
528};
529MODULE_DEVICE_TABLE(of, sl3516_ce_crypto_of_match_table);
530
531static struct platform_driver sl3516_ce_driver = {
532	.probe		 = sl3516_ce_probe,
533	.remove		 = sl3516_ce_remove,
534	.driver		 = {
535		.name		= "sl3516-crypto",
536		.pm		= &sl3516_ce_pm_ops,
537		.of_match_table	= sl3516_ce_crypto_of_match_table,
538	},
539};
540
541module_platform_driver(sl3516_ce_driver);
542
543MODULE_DESCRIPTION("SL3516 cryptographic offloader");
544MODULE_LICENSE("GPL");
545MODULE_AUTHOR("Corentin Labbe <clabbe@baylibre.com>");
546