162306a36Sopenharmony_ci// SPDX-License-Identifier: GPL-2.0-only
262306a36Sopenharmony_ci/*
362306a36Sopenharmony_ci * Crypto acceleration support for Rockchip RK3288
462306a36Sopenharmony_ci *
562306a36Sopenharmony_ci * Copyright (c) 2015, Fuzhou Rockchip Electronics Co., Ltd
662306a36Sopenharmony_ci *
762306a36Sopenharmony_ci * Author: Zain Wang <zain.wang@rock-chips.com>
862306a36Sopenharmony_ci *
962306a36Sopenharmony_ci * Some ideas are from marvell-cesa.c and s5p-sss.c driver.
1062306a36Sopenharmony_ci */
1162306a36Sopenharmony_ci
1262306a36Sopenharmony_ci#include <crypto/engine.h>
1362306a36Sopenharmony_ci#include <crypto/internal/skcipher.h>
1462306a36Sopenharmony_ci#include <crypto/scatterwalk.h>
1562306a36Sopenharmony_ci#include <linux/device.h>
1662306a36Sopenharmony_ci#include <linux/err.h>
1762306a36Sopenharmony_ci#include <linux/kernel.h>
1862306a36Sopenharmony_ci#include <linux/string.h>
1962306a36Sopenharmony_ci#include "rk3288_crypto.h"
2062306a36Sopenharmony_ci
2162306a36Sopenharmony_ci#define RK_CRYPTO_DEC			BIT(0)
2262306a36Sopenharmony_ci
2362306a36Sopenharmony_cistatic int rk_cipher_need_fallback(struct skcipher_request *req)
2462306a36Sopenharmony_ci{
2562306a36Sopenharmony_ci	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
2662306a36Sopenharmony_ci	struct skcipher_alg *alg = crypto_skcipher_alg(tfm);
2762306a36Sopenharmony_ci	struct rk_crypto_tmp *algt = container_of(alg, struct rk_crypto_tmp, alg.skcipher.base);
2862306a36Sopenharmony_ci	struct scatterlist *sgs, *sgd;
2962306a36Sopenharmony_ci	unsigned int stodo, dtodo, len;
3062306a36Sopenharmony_ci	unsigned int bs = crypto_skcipher_blocksize(tfm);
3162306a36Sopenharmony_ci
3262306a36Sopenharmony_ci	if (!req->cryptlen)
3362306a36Sopenharmony_ci		return true;
3462306a36Sopenharmony_ci
3562306a36Sopenharmony_ci	len = req->cryptlen;
3662306a36Sopenharmony_ci	sgs = req->src;
3762306a36Sopenharmony_ci	sgd = req->dst;
3862306a36Sopenharmony_ci	while (sgs && sgd) {
3962306a36Sopenharmony_ci		if (!IS_ALIGNED(sgs->offset, sizeof(u32))) {
4062306a36Sopenharmony_ci			algt->stat_fb_align++;
4162306a36Sopenharmony_ci			return true;
4262306a36Sopenharmony_ci		}
4362306a36Sopenharmony_ci		if (!IS_ALIGNED(sgd->offset, sizeof(u32))) {
4462306a36Sopenharmony_ci			algt->stat_fb_align++;
4562306a36Sopenharmony_ci			return true;
4662306a36Sopenharmony_ci		}
4762306a36Sopenharmony_ci		stodo = min(len, sgs->length);
4862306a36Sopenharmony_ci		if (stodo % bs) {
4962306a36Sopenharmony_ci			algt->stat_fb_len++;
5062306a36Sopenharmony_ci			return true;
5162306a36Sopenharmony_ci		}
5262306a36Sopenharmony_ci		dtodo = min(len, sgd->length);
5362306a36Sopenharmony_ci		if (dtodo % bs) {
5462306a36Sopenharmony_ci			algt->stat_fb_len++;
5562306a36Sopenharmony_ci			return true;
5662306a36Sopenharmony_ci		}
5762306a36Sopenharmony_ci		if (stodo != dtodo) {
5862306a36Sopenharmony_ci			algt->stat_fb_sgdiff++;
5962306a36Sopenharmony_ci			return true;
6062306a36Sopenharmony_ci		}
6162306a36Sopenharmony_ci		len -= stodo;
6262306a36Sopenharmony_ci		sgs = sg_next(sgs);
6362306a36Sopenharmony_ci		sgd = sg_next(sgd);
6462306a36Sopenharmony_ci	}
6562306a36Sopenharmony_ci	return false;
6662306a36Sopenharmony_ci}
6762306a36Sopenharmony_ci
6862306a36Sopenharmony_cistatic int rk_cipher_fallback(struct skcipher_request *areq)
6962306a36Sopenharmony_ci{
7062306a36Sopenharmony_ci	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq);
7162306a36Sopenharmony_ci	struct rk_cipher_ctx *op = crypto_skcipher_ctx(tfm);
7262306a36Sopenharmony_ci	struct rk_cipher_rctx *rctx = skcipher_request_ctx(areq);
7362306a36Sopenharmony_ci	struct skcipher_alg *alg = crypto_skcipher_alg(tfm);
7462306a36Sopenharmony_ci	struct rk_crypto_tmp *algt = container_of(alg, struct rk_crypto_tmp, alg.skcipher.base);
7562306a36Sopenharmony_ci	int err;
7662306a36Sopenharmony_ci
7762306a36Sopenharmony_ci	algt->stat_fb++;
7862306a36Sopenharmony_ci
7962306a36Sopenharmony_ci	skcipher_request_set_tfm(&rctx->fallback_req, op->fallback_tfm);
8062306a36Sopenharmony_ci	skcipher_request_set_callback(&rctx->fallback_req, areq->base.flags,
8162306a36Sopenharmony_ci				      areq->base.complete, areq->base.data);
8262306a36Sopenharmony_ci	skcipher_request_set_crypt(&rctx->fallback_req, areq->src, areq->dst,
8362306a36Sopenharmony_ci				   areq->cryptlen, areq->iv);
8462306a36Sopenharmony_ci	if (rctx->mode & RK_CRYPTO_DEC)
8562306a36Sopenharmony_ci		err = crypto_skcipher_decrypt(&rctx->fallback_req);
8662306a36Sopenharmony_ci	else
8762306a36Sopenharmony_ci		err = crypto_skcipher_encrypt(&rctx->fallback_req);
8862306a36Sopenharmony_ci	return err;
8962306a36Sopenharmony_ci}
9062306a36Sopenharmony_ci
9162306a36Sopenharmony_cistatic int rk_cipher_handle_req(struct skcipher_request *req)
9262306a36Sopenharmony_ci{
9362306a36Sopenharmony_ci	struct rk_cipher_rctx *rctx = skcipher_request_ctx(req);
9462306a36Sopenharmony_ci	struct rk_crypto_info *rkc;
9562306a36Sopenharmony_ci	struct crypto_engine *engine;
9662306a36Sopenharmony_ci
9762306a36Sopenharmony_ci	if (rk_cipher_need_fallback(req))
9862306a36Sopenharmony_ci		return rk_cipher_fallback(req);
9962306a36Sopenharmony_ci
10062306a36Sopenharmony_ci	rkc = get_rk_crypto();
10162306a36Sopenharmony_ci
10262306a36Sopenharmony_ci	engine = rkc->engine;
10362306a36Sopenharmony_ci	rctx->dev = rkc;
10462306a36Sopenharmony_ci
10562306a36Sopenharmony_ci	return crypto_transfer_skcipher_request_to_engine(engine, req);
10662306a36Sopenharmony_ci}
10762306a36Sopenharmony_ci
10862306a36Sopenharmony_cistatic int rk_aes_setkey(struct crypto_skcipher *cipher,
10962306a36Sopenharmony_ci			 const u8 *key, unsigned int keylen)
11062306a36Sopenharmony_ci{
11162306a36Sopenharmony_ci	struct crypto_tfm *tfm = crypto_skcipher_tfm(cipher);
11262306a36Sopenharmony_ci	struct rk_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
11362306a36Sopenharmony_ci
11462306a36Sopenharmony_ci	if (keylen != AES_KEYSIZE_128 && keylen != AES_KEYSIZE_192 &&
11562306a36Sopenharmony_ci	    keylen != AES_KEYSIZE_256)
11662306a36Sopenharmony_ci		return -EINVAL;
11762306a36Sopenharmony_ci	ctx->keylen = keylen;
11862306a36Sopenharmony_ci	memcpy(ctx->key, key, keylen);
11962306a36Sopenharmony_ci
12062306a36Sopenharmony_ci	return crypto_skcipher_setkey(ctx->fallback_tfm, key, keylen);
12162306a36Sopenharmony_ci}
12262306a36Sopenharmony_ci
12362306a36Sopenharmony_cistatic int rk_des_setkey(struct crypto_skcipher *cipher,
12462306a36Sopenharmony_ci			 const u8 *key, unsigned int keylen)
12562306a36Sopenharmony_ci{
12662306a36Sopenharmony_ci	struct rk_cipher_ctx *ctx = crypto_skcipher_ctx(cipher);
12762306a36Sopenharmony_ci	int err;
12862306a36Sopenharmony_ci
12962306a36Sopenharmony_ci	err = verify_skcipher_des_key(cipher, key);
13062306a36Sopenharmony_ci	if (err)
13162306a36Sopenharmony_ci		return err;
13262306a36Sopenharmony_ci
13362306a36Sopenharmony_ci	ctx->keylen = keylen;
13462306a36Sopenharmony_ci	memcpy(ctx->key, key, keylen);
13562306a36Sopenharmony_ci
13662306a36Sopenharmony_ci	return crypto_skcipher_setkey(ctx->fallback_tfm, key, keylen);
13762306a36Sopenharmony_ci}
13862306a36Sopenharmony_ci
13962306a36Sopenharmony_cistatic int rk_tdes_setkey(struct crypto_skcipher *cipher,
14062306a36Sopenharmony_ci			  const u8 *key, unsigned int keylen)
14162306a36Sopenharmony_ci{
14262306a36Sopenharmony_ci	struct rk_cipher_ctx *ctx = crypto_skcipher_ctx(cipher);
14362306a36Sopenharmony_ci	int err;
14462306a36Sopenharmony_ci
14562306a36Sopenharmony_ci	err = verify_skcipher_des3_key(cipher, key);
14662306a36Sopenharmony_ci	if (err)
14762306a36Sopenharmony_ci		return err;
14862306a36Sopenharmony_ci
14962306a36Sopenharmony_ci	ctx->keylen = keylen;
15062306a36Sopenharmony_ci	memcpy(ctx->key, key, keylen);
15162306a36Sopenharmony_ci
15262306a36Sopenharmony_ci	return crypto_skcipher_setkey(ctx->fallback_tfm, key, keylen);
15362306a36Sopenharmony_ci}
15462306a36Sopenharmony_ci
15562306a36Sopenharmony_cistatic int rk_aes_ecb_encrypt(struct skcipher_request *req)
15662306a36Sopenharmony_ci{
15762306a36Sopenharmony_ci	struct rk_cipher_rctx *rctx = skcipher_request_ctx(req);
15862306a36Sopenharmony_ci
15962306a36Sopenharmony_ci	rctx->mode = RK_CRYPTO_AES_ECB_MODE;
16062306a36Sopenharmony_ci	return rk_cipher_handle_req(req);
16162306a36Sopenharmony_ci}
16262306a36Sopenharmony_ci
16362306a36Sopenharmony_cistatic int rk_aes_ecb_decrypt(struct skcipher_request *req)
16462306a36Sopenharmony_ci{
16562306a36Sopenharmony_ci	struct rk_cipher_rctx *rctx = skcipher_request_ctx(req);
16662306a36Sopenharmony_ci
16762306a36Sopenharmony_ci	rctx->mode = RK_CRYPTO_AES_ECB_MODE | RK_CRYPTO_DEC;
16862306a36Sopenharmony_ci	return rk_cipher_handle_req(req);
16962306a36Sopenharmony_ci}
17062306a36Sopenharmony_ci
17162306a36Sopenharmony_cistatic int rk_aes_cbc_encrypt(struct skcipher_request *req)
17262306a36Sopenharmony_ci{
17362306a36Sopenharmony_ci	struct rk_cipher_rctx *rctx = skcipher_request_ctx(req);
17462306a36Sopenharmony_ci
17562306a36Sopenharmony_ci	rctx->mode = RK_CRYPTO_AES_CBC_MODE;
17662306a36Sopenharmony_ci	return rk_cipher_handle_req(req);
17762306a36Sopenharmony_ci}
17862306a36Sopenharmony_ci
17962306a36Sopenharmony_cistatic int rk_aes_cbc_decrypt(struct skcipher_request *req)
18062306a36Sopenharmony_ci{
18162306a36Sopenharmony_ci	struct rk_cipher_rctx *rctx = skcipher_request_ctx(req);
18262306a36Sopenharmony_ci
18362306a36Sopenharmony_ci	rctx->mode = RK_CRYPTO_AES_CBC_MODE | RK_CRYPTO_DEC;
18462306a36Sopenharmony_ci	return rk_cipher_handle_req(req);
18562306a36Sopenharmony_ci}
18662306a36Sopenharmony_ci
18762306a36Sopenharmony_cistatic int rk_des_ecb_encrypt(struct skcipher_request *req)
18862306a36Sopenharmony_ci{
18962306a36Sopenharmony_ci	struct rk_cipher_rctx *rctx = skcipher_request_ctx(req);
19062306a36Sopenharmony_ci
19162306a36Sopenharmony_ci	rctx->mode = 0;
19262306a36Sopenharmony_ci	return rk_cipher_handle_req(req);
19362306a36Sopenharmony_ci}
19462306a36Sopenharmony_ci
19562306a36Sopenharmony_cistatic int rk_des_ecb_decrypt(struct skcipher_request *req)
19662306a36Sopenharmony_ci{
19762306a36Sopenharmony_ci	struct rk_cipher_rctx *rctx = skcipher_request_ctx(req);
19862306a36Sopenharmony_ci
19962306a36Sopenharmony_ci	rctx->mode = RK_CRYPTO_DEC;
20062306a36Sopenharmony_ci	return rk_cipher_handle_req(req);
20162306a36Sopenharmony_ci}
20262306a36Sopenharmony_ci
20362306a36Sopenharmony_cistatic int rk_des_cbc_encrypt(struct skcipher_request *req)
20462306a36Sopenharmony_ci{
20562306a36Sopenharmony_ci	struct rk_cipher_rctx *rctx = skcipher_request_ctx(req);
20662306a36Sopenharmony_ci
20762306a36Sopenharmony_ci	rctx->mode = RK_CRYPTO_TDES_CHAINMODE_CBC;
20862306a36Sopenharmony_ci	return rk_cipher_handle_req(req);
20962306a36Sopenharmony_ci}
21062306a36Sopenharmony_ci
21162306a36Sopenharmony_cistatic int rk_des_cbc_decrypt(struct skcipher_request *req)
21262306a36Sopenharmony_ci{
21362306a36Sopenharmony_ci	struct rk_cipher_rctx *rctx = skcipher_request_ctx(req);
21462306a36Sopenharmony_ci
21562306a36Sopenharmony_ci	rctx->mode = RK_CRYPTO_TDES_CHAINMODE_CBC | RK_CRYPTO_DEC;
21662306a36Sopenharmony_ci	return rk_cipher_handle_req(req);
21762306a36Sopenharmony_ci}
21862306a36Sopenharmony_ci
21962306a36Sopenharmony_cistatic int rk_des3_ede_ecb_encrypt(struct skcipher_request *req)
22062306a36Sopenharmony_ci{
22162306a36Sopenharmony_ci	struct rk_cipher_rctx *rctx = skcipher_request_ctx(req);
22262306a36Sopenharmony_ci
22362306a36Sopenharmony_ci	rctx->mode = RK_CRYPTO_TDES_SELECT;
22462306a36Sopenharmony_ci	return rk_cipher_handle_req(req);
22562306a36Sopenharmony_ci}
22662306a36Sopenharmony_ci
22762306a36Sopenharmony_cistatic int rk_des3_ede_ecb_decrypt(struct skcipher_request *req)
22862306a36Sopenharmony_ci{
22962306a36Sopenharmony_ci	struct rk_cipher_rctx *rctx = skcipher_request_ctx(req);
23062306a36Sopenharmony_ci
23162306a36Sopenharmony_ci	rctx->mode = RK_CRYPTO_TDES_SELECT | RK_CRYPTO_DEC;
23262306a36Sopenharmony_ci	return rk_cipher_handle_req(req);
23362306a36Sopenharmony_ci}
23462306a36Sopenharmony_ci
23562306a36Sopenharmony_cistatic int rk_des3_ede_cbc_encrypt(struct skcipher_request *req)
23662306a36Sopenharmony_ci{
23762306a36Sopenharmony_ci	struct rk_cipher_rctx *rctx = skcipher_request_ctx(req);
23862306a36Sopenharmony_ci
23962306a36Sopenharmony_ci	rctx->mode = RK_CRYPTO_TDES_SELECT | RK_CRYPTO_TDES_CHAINMODE_CBC;
24062306a36Sopenharmony_ci	return rk_cipher_handle_req(req);
24162306a36Sopenharmony_ci}
24262306a36Sopenharmony_ci
24362306a36Sopenharmony_cistatic int rk_des3_ede_cbc_decrypt(struct skcipher_request *req)
24462306a36Sopenharmony_ci{
24562306a36Sopenharmony_ci	struct rk_cipher_rctx *rctx = skcipher_request_ctx(req);
24662306a36Sopenharmony_ci
24762306a36Sopenharmony_ci	rctx->mode = RK_CRYPTO_TDES_SELECT | RK_CRYPTO_TDES_CHAINMODE_CBC |
24862306a36Sopenharmony_ci		    RK_CRYPTO_DEC;
24962306a36Sopenharmony_ci	return rk_cipher_handle_req(req);
25062306a36Sopenharmony_ci}
25162306a36Sopenharmony_ci
25262306a36Sopenharmony_cistatic void rk_cipher_hw_init(struct rk_crypto_info *dev, struct skcipher_request *req)
25362306a36Sopenharmony_ci{
25462306a36Sopenharmony_ci	struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(req);
25562306a36Sopenharmony_ci	struct crypto_tfm *tfm = crypto_skcipher_tfm(cipher);
25662306a36Sopenharmony_ci	struct rk_cipher_rctx *rctx = skcipher_request_ctx(req);
25762306a36Sopenharmony_ci	struct rk_cipher_ctx *ctx = crypto_skcipher_ctx(cipher);
25862306a36Sopenharmony_ci	u32 block, conf_reg = 0;
25962306a36Sopenharmony_ci
26062306a36Sopenharmony_ci	block = crypto_tfm_alg_blocksize(tfm);
26162306a36Sopenharmony_ci
26262306a36Sopenharmony_ci	if (block == DES_BLOCK_SIZE) {
26362306a36Sopenharmony_ci		rctx->mode |= RK_CRYPTO_TDES_FIFO_MODE |
26462306a36Sopenharmony_ci			     RK_CRYPTO_TDES_BYTESWAP_KEY |
26562306a36Sopenharmony_ci			     RK_CRYPTO_TDES_BYTESWAP_IV;
26662306a36Sopenharmony_ci		CRYPTO_WRITE(dev, RK_CRYPTO_TDES_CTRL, rctx->mode);
26762306a36Sopenharmony_ci		memcpy_toio(dev->reg + RK_CRYPTO_TDES_KEY1_0, ctx->key, ctx->keylen);
26862306a36Sopenharmony_ci		conf_reg = RK_CRYPTO_DESSEL;
26962306a36Sopenharmony_ci	} else {
27062306a36Sopenharmony_ci		rctx->mode |= RK_CRYPTO_AES_FIFO_MODE |
27162306a36Sopenharmony_ci			     RK_CRYPTO_AES_KEY_CHANGE |
27262306a36Sopenharmony_ci			     RK_CRYPTO_AES_BYTESWAP_KEY |
27362306a36Sopenharmony_ci			     RK_CRYPTO_AES_BYTESWAP_IV;
27462306a36Sopenharmony_ci		if (ctx->keylen == AES_KEYSIZE_192)
27562306a36Sopenharmony_ci			rctx->mode |= RK_CRYPTO_AES_192BIT_key;
27662306a36Sopenharmony_ci		else if (ctx->keylen == AES_KEYSIZE_256)
27762306a36Sopenharmony_ci			rctx->mode |= RK_CRYPTO_AES_256BIT_key;
27862306a36Sopenharmony_ci		CRYPTO_WRITE(dev, RK_CRYPTO_AES_CTRL, rctx->mode);
27962306a36Sopenharmony_ci		memcpy_toio(dev->reg + RK_CRYPTO_AES_KEY_0, ctx->key, ctx->keylen);
28062306a36Sopenharmony_ci	}
28162306a36Sopenharmony_ci	conf_reg |= RK_CRYPTO_BYTESWAP_BTFIFO |
28262306a36Sopenharmony_ci		    RK_CRYPTO_BYTESWAP_BRFIFO;
28362306a36Sopenharmony_ci	CRYPTO_WRITE(dev, RK_CRYPTO_CONF, conf_reg);
28462306a36Sopenharmony_ci	CRYPTO_WRITE(dev, RK_CRYPTO_INTENA,
28562306a36Sopenharmony_ci		     RK_CRYPTO_BCDMA_ERR_ENA | RK_CRYPTO_BCDMA_DONE_ENA);
28662306a36Sopenharmony_ci}
28762306a36Sopenharmony_ci
28862306a36Sopenharmony_cistatic void crypto_dma_start(struct rk_crypto_info *dev,
28962306a36Sopenharmony_ci			     struct scatterlist *sgs,
29062306a36Sopenharmony_ci			     struct scatterlist *sgd, unsigned int todo)
29162306a36Sopenharmony_ci{
29262306a36Sopenharmony_ci	CRYPTO_WRITE(dev, RK_CRYPTO_BRDMAS, sg_dma_address(sgs));
29362306a36Sopenharmony_ci	CRYPTO_WRITE(dev, RK_CRYPTO_BRDMAL, todo);
29462306a36Sopenharmony_ci	CRYPTO_WRITE(dev, RK_CRYPTO_BTDMAS, sg_dma_address(sgd));
29562306a36Sopenharmony_ci	CRYPTO_WRITE(dev, RK_CRYPTO_CTRL, RK_CRYPTO_BLOCK_START |
29662306a36Sopenharmony_ci		     _SBF(RK_CRYPTO_BLOCK_START, 16));
29762306a36Sopenharmony_ci}
29862306a36Sopenharmony_ci
29962306a36Sopenharmony_cistatic int rk_cipher_run(struct crypto_engine *engine, void *async_req)
30062306a36Sopenharmony_ci{
30162306a36Sopenharmony_ci	struct skcipher_request *areq = container_of(async_req, struct skcipher_request, base);
30262306a36Sopenharmony_ci	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq);
30362306a36Sopenharmony_ci	struct rk_cipher_rctx *rctx = skcipher_request_ctx(areq);
30462306a36Sopenharmony_ci	struct scatterlist *sgs, *sgd;
30562306a36Sopenharmony_ci	int err = 0;
30662306a36Sopenharmony_ci	int ivsize = crypto_skcipher_ivsize(tfm);
30762306a36Sopenharmony_ci	int offset;
30862306a36Sopenharmony_ci	u8 iv[AES_BLOCK_SIZE];
30962306a36Sopenharmony_ci	u8 biv[AES_BLOCK_SIZE];
31062306a36Sopenharmony_ci	u8 *ivtouse = areq->iv;
31162306a36Sopenharmony_ci	unsigned int len = areq->cryptlen;
31262306a36Sopenharmony_ci	unsigned int todo;
31362306a36Sopenharmony_ci	struct skcipher_alg *alg = crypto_skcipher_alg(tfm);
31462306a36Sopenharmony_ci	struct rk_crypto_tmp *algt = container_of(alg, struct rk_crypto_tmp, alg.skcipher.base);
31562306a36Sopenharmony_ci	struct rk_crypto_info *rkc = rctx->dev;
31662306a36Sopenharmony_ci
31762306a36Sopenharmony_ci	err = pm_runtime_resume_and_get(rkc->dev);
31862306a36Sopenharmony_ci	if (err)
31962306a36Sopenharmony_ci		return err;
32062306a36Sopenharmony_ci
32162306a36Sopenharmony_ci	algt->stat_req++;
32262306a36Sopenharmony_ci	rkc->nreq++;
32362306a36Sopenharmony_ci
32462306a36Sopenharmony_ci	ivsize = crypto_skcipher_ivsize(tfm);
32562306a36Sopenharmony_ci	if (areq->iv && crypto_skcipher_ivsize(tfm) > 0) {
32662306a36Sopenharmony_ci		if (rctx->mode & RK_CRYPTO_DEC) {
32762306a36Sopenharmony_ci			offset = areq->cryptlen - ivsize;
32862306a36Sopenharmony_ci			scatterwalk_map_and_copy(rctx->backup_iv, areq->src,
32962306a36Sopenharmony_ci						 offset, ivsize, 0);
33062306a36Sopenharmony_ci		}
33162306a36Sopenharmony_ci	}
33262306a36Sopenharmony_ci
33362306a36Sopenharmony_ci	sgs = areq->src;
33462306a36Sopenharmony_ci	sgd = areq->dst;
33562306a36Sopenharmony_ci
33662306a36Sopenharmony_ci	while (sgs && sgd && len) {
33762306a36Sopenharmony_ci		if (!sgs->length) {
33862306a36Sopenharmony_ci			sgs = sg_next(sgs);
33962306a36Sopenharmony_ci			sgd = sg_next(sgd);
34062306a36Sopenharmony_ci			continue;
34162306a36Sopenharmony_ci		}
34262306a36Sopenharmony_ci		if (rctx->mode & RK_CRYPTO_DEC) {
34362306a36Sopenharmony_ci			/* we backup last block of source to be used as IV at next step */
34462306a36Sopenharmony_ci			offset = sgs->length - ivsize;
34562306a36Sopenharmony_ci			scatterwalk_map_and_copy(biv, sgs, offset, ivsize, 0);
34662306a36Sopenharmony_ci		}
34762306a36Sopenharmony_ci		if (sgs == sgd) {
34862306a36Sopenharmony_ci			err = dma_map_sg(rkc->dev, sgs, 1, DMA_BIDIRECTIONAL);
34962306a36Sopenharmony_ci			if (err <= 0) {
35062306a36Sopenharmony_ci				err = -EINVAL;
35162306a36Sopenharmony_ci				goto theend_iv;
35262306a36Sopenharmony_ci			}
35362306a36Sopenharmony_ci		} else {
35462306a36Sopenharmony_ci			err = dma_map_sg(rkc->dev, sgs, 1, DMA_TO_DEVICE);
35562306a36Sopenharmony_ci			if (err <= 0) {
35662306a36Sopenharmony_ci				err = -EINVAL;
35762306a36Sopenharmony_ci				goto theend_iv;
35862306a36Sopenharmony_ci			}
35962306a36Sopenharmony_ci			err = dma_map_sg(rkc->dev, sgd, 1, DMA_FROM_DEVICE);
36062306a36Sopenharmony_ci			if (err <= 0) {
36162306a36Sopenharmony_ci				err = -EINVAL;
36262306a36Sopenharmony_ci				goto theend_sgs;
36362306a36Sopenharmony_ci			}
36462306a36Sopenharmony_ci		}
36562306a36Sopenharmony_ci		err = 0;
36662306a36Sopenharmony_ci		rk_cipher_hw_init(rkc, areq);
36762306a36Sopenharmony_ci		if (ivsize) {
36862306a36Sopenharmony_ci			if (ivsize == DES_BLOCK_SIZE)
36962306a36Sopenharmony_ci				memcpy_toio(rkc->reg + RK_CRYPTO_TDES_IV_0, ivtouse, ivsize);
37062306a36Sopenharmony_ci			else
37162306a36Sopenharmony_ci				memcpy_toio(rkc->reg + RK_CRYPTO_AES_IV_0, ivtouse, ivsize);
37262306a36Sopenharmony_ci		}
37362306a36Sopenharmony_ci		reinit_completion(&rkc->complete);
37462306a36Sopenharmony_ci		rkc->status = 0;
37562306a36Sopenharmony_ci
37662306a36Sopenharmony_ci		todo = min(sg_dma_len(sgs), len);
37762306a36Sopenharmony_ci		len -= todo;
37862306a36Sopenharmony_ci		crypto_dma_start(rkc, sgs, sgd, todo / 4);
37962306a36Sopenharmony_ci		wait_for_completion_interruptible_timeout(&rkc->complete,
38062306a36Sopenharmony_ci							  msecs_to_jiffies(2000));
38162306a36Sopenharmony_ci		if (!rkc->status) {
38262306a36Sopenharmony_ci			dev_err(rkc->dev, "DMA timeout\n");
38362306a36Sopenharmony_ci			err = -EFAULT;
38462306a36Sopenharmony_ci			goto theend;
38562306a36Sopenharmony_ci		}
38662306a36Sopenharmony_ci		if (sgs == sgd) {
38762306a36Sopenharmony_ci			dma_unmap_sg(rkc->dev, sgs, 1, DMA_BIDIRECTIONAL);
38862306a36Sopenharmony_ci		} else {
38962306a36Sopenharmony_ci			dma_unmap_sg(rkc->dev, sgs, 1, DMA_TO_DEVICE);
39062306a36Sopenharmony_ci			dma_unmap_sg(rkc->dev, sgd, 1, DMA_FROM_DEVICE);
39162306a36Sopenharmony_ci		}
39262306a36Sopenharmony_ci		if (rctx->mode & RK_CRYPTO_DEC) {
39362306a36Sopenharmony_ci			memcpy(iv, biv, ivsize);
39462306a36Sopenharmony_ci			ivtouse = iv;
39562306a36Sopenharmony_ci		} else {
39662306a36Sopenharmony_ci			offset = sgd->length - ivsize;
39762306a36Sopenharmony_ci			scatterwalk_map_and_copy(iv, sgd, offset, ivsize, 0);
39862306a36Sopenharmony_ci			ivtouse = iv;
39962306a36Sopenharmony_ci		}
40062306a36Sopenharmony_ci		sgs = sg_next(sgs);
40162306a36Sopenharmony_ci		sgd = sg_next(sgd);
40262306a36Sopenharmony_ci	}
40362306a36Sopenharmony_ci
40462306a36Sopenharmony_ci	if (areq->iv && ivsize > 0) {
40562306a36Sopenharmony_ci		offset = areq->cryptlen - ivsize;
40662306a36Sopenharmony_ci		if (rctx->mode & RK_CRYPTO_DEC) {
40762306a36Sopenharmony_ci			memcpy(areq->iv, rctx->backup_iv, ivsize);
40862306a36Sopenharmony_ci			memzero_explicit(rctx->backup_iv, ivsize);
40962306a36Sopenharmony_ci		} else {
41062306a36Sopenharmony_ci			scatterwalk_map_and_copy(areq->iv, areq->dst, offset,
41162306a36Sopenharmony_ci						 ivsize, 0);
41262306a36Sopenharmony_ci		}
41362306a36Sopenharmony_ci	}
41462306a36Sopenharmony_ci
41562306a36Sopenharmony_citheend:
41662306a36Sopenharmony_ci	pm_runtime_put_autosuspend(rkc->dev);
41762306a36Sopenharmony_ci
41862306a36Sopenharmony_ci	local_bh_disable();
41962306a36Sopenharmony_ci	crypto_finalize_skcipher_request(engine, areq, err);
42062306a36Sopenharmony_ci	local_bh_enable();
42162306a36Sopenharmony_ci	return 0;
42262306a36Sopenharmony_ci
42362306a36Sopenharmony_citheend_sgs:
42462306a36Sopenharmony_ci	if (sgs == sgd) {
42562306a36Sopenharmony_ci		dma_unmap_sg(rkc->dev, sgs, 1, DMA_BIDIRECTIONAL);
42662306a36Sopenharmony_ci	} else {
42762306a36Sopenharmony_ci		dma_unmap_sg(rkc->dev, sgs, 1, DMA_TO_DEVICE);
42862306a36Sopenharmony_ci		dma_unmap_sg(rkc->dev, sgd, 1, DMA_FROM_DEVICE);
42962306a36Sopenharmony_ci	}
43062306a36Sopenharmony_citheend_iv:
43162306a36Sopenharmony_ci	return err;
43262306a36Sopenharmony_ci}
43362306a36Sopenharmony_ci
43462306a36Sopenharmony_cistatic int rk_cipher_tfm_init(struct crypto_skcipher *tfm)
43562306a36Sopenharmony_ci{
43662306a36Sopenharmony_ci	struct rk_cipher_ctx *ctx = crypto_skcipher_ctx(tfm);
43762306a36Sopenharmony_ci	const char *name = crypto_tfm_alg_name(&tfm->base);
43862306a36Sopenharmony_ci	struct skcipher_alg *alg = crypto_skcipher_alg(tfm);
43962306a36Sopenharmony_ci	struct rk_crypto_tmp *algt = container_of(alg, struct rk_crypto_tmp, alg.skcipher.base);
44062306a36Sopenharmony_ci
44162306a36Sopenharmony_ci	ctx->fallback_tfm = crypto_alloc_skcipher(name, 0, CRYPTO_ALG_NEED_FALLBACK);
44262306a36Sopenharmony_ci	if (IS_ERR(ctx->fallback_tfm)) {
44362306a36Sopenharmony_ci		dev_err(algt->dev->dev, "ERROR: Cannot allocate fallback for %s %ld\n",
44462306a36Sopenharmony_ci			name, PTR_ERR(ctx->fallback_tfm));
44562306a36Sopenharmony_ci		return PTR_ERR(ctx->fallback_tfm);
44662306a36Sopenharmony_ci	}
44762306a36Sopenharmony_ci
44862306a36Sopenharmony_ci	tfm->reqsize = sizeof(struct rk_cipher_rctx) +
44962306a36Sopenharmony_ci		crypto_skcipher_reqsize(ctx->fallback_tfm);
45062306a36Sopenharmony_ci
45162306a36Sopenharmony_ci	return 0;
45262306a36Sopenharmony_ci}
45362306a36Sopenharmony_ci
45462306a36Sopenharmony_cistatic void rk_cipher_tfm_exit(struct crypto_skcipher *tfm)
45562306a36Sopenharmony_ci{
45662306a36Sopenharmony_ci	struct rk_cipher_ctx *ctx = crypto_skcipher_ctx(tfm);
45762306a36Sopenharmony_ci
45862306a36Sopenharmony_ci	memzero_explicit(ctx->key, ctx->keylen);
45962306a36Sopenharmony_ci	crypto_free_skcipher(ctx->fallback_tfm);
46062306a36Sopenharmony_ci}
46162306a36Sopenharmony_ci
46262306a36Sopenharmony_cistruct rk_crypto_tmp rk_ecb_aes_alg = {
46362306a36Sopenharmony_ci	.type = CRYPTO_ALG_TYPE_SKCIPHER,
46462306a36Sopenharmony_ci	.alg.skcipher.base = {
46562306a36Sopenharmony_ci		.base.cra_name		= "ecb(aes)",
46662306a36Sopenharmony_ci		.base.cra_driver_name	= "ecb-aes-rk",
46762306a36Sopenharmony_ci		.base.cra_priority	= 300,
46862306a36Sopenharmony_ci		.base.cra_flags		= CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK,
46962306a36Sopenharmony_ci		.base.cra_blocksize	= AES_BLOCK_SIZE,
47062306a36Sopenharmony_ci		.base.cra_ctxsize	= sizeof(struct rk_cipher_ctx),
47162306a36Sopenharmony_ci		.base.cra_alignmask	= 0x0f,
47262306a36Sopenharmony_ci		.base.cra_module	= THIS_MODULE,
47362306a36Sopenharmony_ci
47462306a36Sopenharmony_ci		.init			= rk_cipher_tfm_init,
47562306a36Sopenharmony_ci		.exit			= rk_cipher_tfm_exit,
47662306a36Sopenharmony_ci		.min_keysize		= AES_MIN_KEY_SIZE,
47762306a36Sopenharmony_ci		.max_keysize		= AES_MAX_KEY_SIZE,
47862306a36Sopenharmony_ci		.setkey			= rk_aes_setkey,
47962306a36Sopenharmony_ci		.encrypt		= rk_aes_ecb_encrypt,
48062306a36Sopenharmony_ci		.decrypt		= rk_aes_ecb_decrypt,
48162306a36Sopenharmony_ci	},
48262306a36Sopenharmony_ci	.alg.skcipher.op = {
48362306a36Sopenharmony_ci		.do_one_request = rk_cipher_run,
48462306a36Sopenharmony_ci	},
48562306a36Sopenharmony_ci};
48662306a36Sopenharmony_ci
48762306a36Sopenharmony_cistruct rk_crypto_tmp rk_cbc_aes_alg = {
48862306a36Sopenharmony_ci	.type = CRYPTO_ALG_TYPE_SKCIPHER,
48962306a36Sopenharmony_ci	.alg.skcipher.base = {
49062306a36Sopenharmony_ci		.base.cra_name		= "cbc(aes)",
49162306a36Sopenharmony_ci		.base.cra_driver_name	= "cbc-aes-rk",
49262306a36Sopenharmony_ci		.base.cra_priority	= 300,
49362306a36Sopenharmony_ci		.base.cra_flags		= CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK,
49462306a36Sopenharmony_ci		.base.cra_blocksize	= AES_BLOCK_SIZE,
49562306a36Sopenharmony_ci		.base.cra_ctxsize	= sizeof(struct rk_cipher_ctx),
49662306a36Sopenharmony_ci		.base.cra_alignmask	= 0x0f,
49762306a36Sopenharmony_ci		.base.cra_module	= THIS_MODULE,
49862306a36Sopenharmony_ci
49962306a36Sopenharmony_ci		.init			= rk_cipher_tfm_init,
50062306a36Sopenharmony_ci		.exit			= rk_cipher_tfm_exit,
50162306a36Sopenharmony_ci		.min_keysize		= AES_MIN_KEY_SIZE,
50262306a36Sopenharmony_ci		.max_keysize		= AES_MAX_KEY_SIZE,
50362306a36Sopenharmony_ci		.ivsize			= AES_BLOCK_SIZE,
50462306a36Sopenharmony_ci		.setkey			= rk_aes_setkey,
50562306a36Sopenharmony_ci		.encrypt		= rk_aes_cbc_encrypt,
50662306a36Sopenharmony_ci		.decrypt		= rk_aes_cbc_decrypt,
50762306a36Sopenharmony_ci	},
50862306a36Sopenharmony_ci	.alg.skcipher.op = {
50962306a36Sopenharmony_ci		.do_one_request = rk_cipher_run,
51062306a36Sopenharmony_ci	},
51162306a36Sopenharmony_ci};
51262306a36Sopenharmony_ci
51362306a36Sopenharmony_cistruct rk_crypto_tmp rk_ecb_des_alg = {
51462306a36Sopenharmony_ci	.type = CRYPTO_ALG_TYPE_SKCIPHER,
51562306a36Sopenharmony_ci	.alg.skcipher.base = {
51662306a36Sopenharmony_ci		.base.cra_name		= "ecb(des)",
51762306a36Sopenharmony_ci		.base.cra_driver_name	= "ecb-des-rk",
51862306a36Sopenharmony_ci		.base.cra_priority	= 300,
51962306a36Sopenharmony_ci		.base.cra_flags		= CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK,
52062306a36Sopenharmony_ci		.base.cra_blocksize	= DES_BLOCK_SIZE,
52162306a36Sopenharmony_ci		.base.cra_ctxsize	= sizeof(struct rk_cipher_ctx),
52262306a36Sopenharmony_ci		.base.cra_alignmask	= 0x07,
52362306a36Sopenharmony_ci		.base.cra_module	= THIS_MODULE,
52462306a36Sopenharmony_ci
52562306a36Sopenharmony_ci		.init			= rk_cipher_tfm_init,
52662306a36Sopenharmony_ci		.exit			= rk_cipher_tfm_exit,
52762306a36Sopenharmony_ci		.min_keysize		= DES_KEY_SIZE,
52862306a36Sopenharmony_ci		.max_keysize		= DES_KEY_SIZE,
52962306a36Sopenharmony_ci		.setkey			= rk_des_setkey,
53062306a36Sopenharmony_ci		.encrypt		= rk_des_ecb_encrypt,
53162306a36Sopenharmony_ci		.decrypt		= rk_des_ecb_decrypt,
53262306a36Sopenharmony_ci	},
53362306a36Sopenharmony_ci	.alg.skcipher.op = {
53462306a36Sopenharmony_ci		.do_one_request = rk_cipher_run,
53562306a36Sopenharmony_ci	},
53662306a36Sopenharmony_ci};
53762306a36Sopenharmony_ci
53862306a36Sopenharmony_cistruct rk_crypto_tmp rk_cbc_des_alg = {
53962306a36Sopenharmony_ci	.type = CRYPTO_ALG_TYPE_SKCIPHER,
54062306a36Sopenharmony_ci	.alg.skcipher.base = {
54162306a36Sopenharmony_ci		.base.cra_name		= "cbc(des)",
54262306a36Sopenharmony_ci		.base.cra_driver_name	= "cbc-des-rk",
54362306a36Sopenharmony_ci		.base.cra_priority	= 300,
54462306a36Sopenharmony_ci		.base.cra_flags		= CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK,
54562306a36Sopenharmony_ci		.base.cra_blocksize	= DES_BLOCK_SIZE,
54662306a36Sopenharmony_ci		.base.cra_ctxsize	= sizeof(struct rk_cipher_ctx),
54762306a36Sopenharmony_ci		.base.cra_alignmask	= 0x07,
54862306a36Sopenharmony_ci		.base.cra_module	= THIS_MODULE,
54962306a36Sopenharmony_ci
55062306a36Sopenharmony_ci		.init			= rk_cipher_tfm_init,
55162306a36Sopenharmony_ci		.exit			= rk_cipher_tfm_exit,
55262306a36Sopenharmony_ci		.min_keysize		= DES_KEY_SIZE,
55362306a36Sopenharmony_ci		.max_keysize		= DES_KEY_SIZE,
55462306a36Sopenharmony_ci		.ivsize			= DES_BLOCK_SIZE,
55562306a36Sopenharmony_ci		.setkey			= rk_des_setkey,
55662306a36Sopenharmony_ci		.encrypt		= rk_des_cbc_encrypt,
55762306a36Sopenharmony_ci		.decrypt		= rk_des_cbc_decrypt,
55862306a36Sopenharmony_ci	},
55962306a36Sopenharmony_ci	.alg.skcipher.op = {
56062306a36Sopenharmony_ci		.do_one_request = rk_cipher_run,
56162306a36Sopenharmony_ci	},
56262306a36Sopenharmony_ci};
56362306a36Sopenharmony_ci
56462306a36Sopenharmony_cistruct rk_crypto_tmp rk_ecb_des3_ede_alg = {
56562306a36Sopenharmony_ci	.type = CRYPTO_ALG_TYPE_SKCIPHER,
56662306a36Sopenharmony_ci	.alg.skcipher.base = {
56762306a36Sopenharmony_ci		.base.cra_name		= "ecb(des3_ede)",
56862306a36Sopenharmony_ci		.base.cra_driver_name	= "ecb-des3-ede-rk",
56962306a36Sopenharmony_ci		.base.cra_priority	= 300,
57062306a36Sopenharmony_ci		.base.cra_flags		= CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK,
57162306a36Sopenharmony_ci		.base.cra_blocksize	= DES_BLOCK_SIZE,
57262306a36Sopenharmony_ci		.base.cra_ctxsize	= sizeof(struct rk_cipher_ctx),
57362306a36Sopenharmony_ci		.base.cra_alignmask	= 0x07,
57462306a36Sopenharmony_ci		.base.cra_module	= THIS_MODULE,
57562306a36Sopenharmony_ci
57662306a36Sopenharmony_ci		.init			= rk_cipher_tfm_init,
57762306a36Sopenharmony_ci		.exit			= rk_cipher_tfm_exit,
57862306a36Sopenharmony_ci		.min_keysize		= DES3_EDE_KEY_SIZE,
57962306a36Sopenharmony_ci		.max_keysize		= DES3_EDE_KEY_SIZE,
58062306a36Sopenharmony_ci		.setkey			= rk_tdes_setkey,
58162306a36Sopenharmony_ci		.encrypt		= rk_des3_ede_ecb_encrypt,
58262306a36Sopenharmony_ci		.decrypt		= rk_des3_ede_ecb_decrypt,
58362306a36Sopenharmony_ci	},
58462306a36Sopenharmony_ci	.alg.skcipher.op = {
58562306a36Sopenharmony_ci		.do_one_request = rk_cipher_run,
58662306a36Sopenharmony_ci	},
58762306a36Sopenharmony_ci};
58862306a36Sopenharmony_ci
58962306a36Sopenharmony_cistruct rk_crypto_tmp rk_cbc_des3_ede_alg = {
59062306a36Sopenharmony_ci	.type = CRYPTO_ALG_TYPE_SKCIPHER,
59162306a36Sopenharmony_ci	.alg.skcipher.base = {
59262306a36Sopenharmony_ci		.base.cra_name		= "cbc(des3_ede)",
59362306a36Sopenharmony_ci		.base.cra_driver_name	= "cbc-des3-ede-rk",
59462306a36Sopenharmony_ci		.base.cra_priority	= 300,
59562306a36Sopenharmony_ci		.base.cra_flags		= CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK,
59662306a36Sopenharmony_ci		.base.cra_blocksize	= DES_BLOCK_SIZE,
59762306a36Sopenharmony_ci		.base.cra_ctxsize	= sizeof(struct rk_cipher_ctx),
59862306a36Sopenharmony_ci		.base.cra_alignmask	= 0x07,
59962306a36Sopenharmony_ci		.base.cra_module	= THIS_MODULE,
60062306a36Sopenharmony_ci
60162306a36Sopenharmony_ci		.init			= rk_cipher_tfm_init,
60262306a36Sopenharmony_ci		.exit			= rk_cipher_tfm_exit,
60362306a36Sopenharmony_ci		.min_keysize		= DES3_EDE_KEY_SIZE,
60462306a36Sopenharmony_ci		.max_keysize		= DES3_EDE_KEY_SIZE,
60562306a36Sopenharmony_ci		.ivsize			= DES_BLOCK_SIZE,
60662306a36Sopenharmony_ci		.setkey			= rk_tdes_setkey,
60762306a36Sopenharmony_ci		.encrypt		= rk_des3_ede_cbc_encrypt,
60862306a36Sopenharmony_ci		.decrypt		= rk_des3_ede_cbc_decrypt,
60962306a36Sopenharmony_ci	},
61062306a36Sopenharmony_ci	.alg.skcipher.op = {
61162306a36Sopenharmony_ci		.do_one_request = rk_cipher_run,
61262306a36Sopenharmony_ci	},
61362306a36Sopenharmony_ci};
614