162306a36Sopenharmony_ci// SPDX-License-Identifier: GPL-2.0-only 262306a36Sopenharmony_ci/* 362306a36Sopenharmony_ci * Crypto acceleration support for Rockchip RK3288 462306a36Sopenharmony_ci * 562306a36Sopenharmony_ci * Copyright (c) 2015, Fuzhou Rockchip Electronics Co., Ltd 662306a36Sopenharmony_ci * 762306a36Sopenharmony_ci * Author: Zain Wang <zain.wang@rock-chips.com> 862306a36Sopenharmony_ci * 962306a36Sopenharmony_ci * Some ideas are from marvell-cesa.c and s5p-sss.c driver. 1062306a36Sopenharmony_ci */ 1162306a36Sopenharmony_ci 1262306a36Sopenharmony_ci#include <crypto/engine.h> 1362306a36Sopenharmony_ci#include <crypto/internal/skcipher.h> 1462306a36Sopenharmony_ci#include <crypto/scatterwalk.h> 1562306a36Sopenharmony_ci#include <linux/device.h> 1662306a36Sopenharmony_ci#include <linux/err.h> 1762306a36Sopenharmony_ci#include <linux/kernel.h> 1862306a36Sopenharmony_ci#include <linux/string.h> 1962306a36Sopenharmony_ci#include "rk3288_crypto.h" 2062306a36Sopenharmony_ci 2162306a36Sopenharmony_ci#define RK_CRYPTO_DEC BIT(0) 2262306a36Sopenharmony_ci 2362306a36Sopenharmony_cistatic int rk_cipher_need_fallback(struct skcipher_request *req) 2462306a36Sopenharmony_ci{ 2562306a36Sopenharmony_ci struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 2662306a36Sopenharmony_ci struct skcipher_alg *alg = crypto_skcipher_alg(tfm); 2762306a36Sopenharmony_ci struct rk_crypto_tmp *algt = container_of(alg, struct rk_crypto_tmp, alg.skcipher.base); 2862306a36Sopenharmony_ci struct scatterlist *sgs, *sgd; 2962306a36Sopenharmony_ci unsigned int stodo, dtodo, len; 3062306a36Sopenharmony_ci unsigned int bs = crypto_skcipher_blocksize(tfm); 3162306a36Sopenharmony_ci 3262306a36Sopenharmony_ci if (!req->cryptlen) 3362306a36Sopenharmony_ci return true; 3462306a36Sopenharmony_ci 3562306a36Sopenharmony_ci len = req->cryptlen; 3662306a36Sopenharmony_ci sgs = req->src; 3762306a36Sopenharmony_ci sgd = req->dst; 3862306a36Sopenharmony_ci while (sgs && sgd) { 3962306a36Sopenharmony_ci if (!IS_ALIGNED(sgs->offset, sizeof(u32))) { 4062306a36Sopenharmony_ci algt->stat_fb_align++; 4162306a36Sopenharmony_ci return true; 4262306a36Sopenharmony_ci } 4362306a36Sopenharmony_ci if (!IS_ALIGNED(sgd->offset, sizeof(u32))) { 4462306a36Sopenharmony_ci algt->stat_fb_align++; 4562306a36Sopenharmony_ci return true; 4662306a36Sopenharmony_ci } 4762306a36Sopenharmony_ci stodo = min(len, sgs->length); 4862306a36Sopenharmony_ci if (stodo % bs) { 4962306a36Sopenharmony_ci algt->stat_fb_len++; 5062306a36Sopenharmony_ci return true; 5162306a36Sopenharmony_ci } 5262306a36Sopenharmony_ci dtodo = min(len, sgd->length); 5362306a36Sopenharmony_ci if (dtodo % bs) { 5462306a36Sopenharmony_ci algt->stat_fb_len++; 5562306a36Sopenharmony_ci return true; 5662306a36Sopenharmony_ci } 5762306a36Sopenharmony_ci if (stodo != dtodo) { 5862306a36Sopenharmony_ci algt->stat_fb_sgdiff++; 5962306a36Sopenharmony_ci return true; 6062306a36Sopenharmony_ci } 6162306a36Sopenharmony_ci len -= stodo; 6262306a36Sopenharmony_ci sgs = sg_next(sgs); 6362306a36Sopenharmony_ci sgd = sg_next(sgd); 6462306a36Sopenharmony_ci } 6562306a36Sopenharmony_ci return false; 6662306a36Sopenharmony_ci} 6762306a36Sopenharmony_ci 6862306a36Sopenharmony_cistatic int rk_cipher_fallback(struct skcipher_request *areq) 6962306a36Sopenharmony_ci{ 7062306a36Sopenharmony_ci struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq); 7162306a36Sopenharmony_ci struct rk_cipher_ctx *op = crypto_skcipher_ctx(tfm); 7262306a36Sopenharmony_ci struct rk_cipher_rctx *rctx = skcipher_request_ctx(areq); 7362306a36Sopenharmony_ci struct skcipher_alg *alg = crypto_skcipher_alg(tfm); 7462306a36Sopenharmony_ci struct rk_crypto_tmp *algt = container_of(alg, struct rk_crypto_tmp, alg.skcipher.base); 7562306a36Sopenharmony_ci int err; 7662306a36Sopenharmony_ci 7762306a36Sopenharmony_ci algt->stat_fb++; 7862306a36Sopenharmony_ci 7962306a36Sopenharmony_ci skcipher_request_set_tfm(&rctx->fallback_req, op->fallback_tfm); 8062306a36Sopenharmony_ci skcipher_request_set_callback(&rctx->fallback_req, areq->base.flags, 8162306a36Sopenharmony_ci areq->base.complete, areq->base.data); 8262306a36Sopenharmony_ci skcipher_request_set_crypt(&rctx->fallback_req, areq->src, areq->dst, 8362306a36Sopenharmony_ci areq->cryptlen, areq->iv); 8462306a36Sopenharmony_ci if (rctx->mode & RK_CRYPTO_DEC) 8562306a36Sopenharmony_ci err = crypto_skcipher_decrypt(&rctx->fallback_req); 8662306a36Sopenharmony_ci else 8762306a36Sopenharmony_ci err = crypto_skcipher_encrypt(&rctx->fallback_req); 8862306a36Sopenharmony_ci return err; 8962306a36Sopenharmony_ci} 9062306a36Sopenharmony_ci 9162306a36Sopenharmony_cistatic int rk_cipher_handle_req(struct skcipher_request *req) 9262306a36Sopenharmony_ci{ 9362306a36Sopenharmony_ci struct rk_cipher_rctx *rctx = skcipher_request_ctx(req); 9462306a36Sopenharmony_ci struct rk_crypto_info *rkc; 9562306a36Sopenharmony_ci struct crypto_engine *engine; 9662306a36Sopenharmony_ci 9762306a36Sopenharmony_ci if (rk_cipher_need_fallback(req)) 9862306a36Sopenharmony_ci return rk_cipher_fallback(req); 9962306a36Sopenharmony_ci 10062306a36Sopenharmony_ci rkc = get_rk_crypto(); 10162306a36Sopenharmony_ci 10262306a36Sopenharmony_ci engine = rkc->engine; 10362306a36Sopenharmony_ci rctx->dev = rkc; 10462306a36Sopenharmony_ci 10562306a36Sopenharmony_ci return crypto_transfer_skcipher_request_to_engine(engine, req); 10662306a36Sopenharmony_ci} 10762306a36Sopenharmony_ci 10862306a36Sopenharmony_cistatic int rk_aes_setkey(struct crypto_skcipher *cipher, 10962306a36Sopenharmony_ci const u8 *key, unsigned int keylen) 11062306a36Sopenharmony_ci{ 11162306a36Sopenharmony_ci struct crypto_tfm *tfm = crypto_skcipher_tfm(cipher); 11262306a36Sopenharmony_ci struct rk_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 11362306a36Sopenharmony_ci 11462306a36Sopenharmony_ci if (keylen != AES_KEYSIZE_128 && keylen != AES_KEYSIZE_192 && 11562306a36Sopenharmony_ci keylen != AES_KEYSIZE_256) 11662306a36Sopenharmony_ci return -EINVAL; 11762306a36Sopenharmony_ci ctx->keylen = keylen; 11862306a36Sopenharmony_ci memcpy(ctx->key, key, keylen); 11962306a36Sopenharmony_ci 12062306a36Sopenharmony_ci return crypto_skcipher_setkey(ctx->fallback_tfm, key, keylen); 12162306a36Sopenharmony_ci} 12262306a36Sopenharmony_ci 12362306a36Sopenharmony_cistatic int rk_des_setkey(struct crypto_skcipher *cipher, 12462306a36Sopenharmony_ci const u8 *key, unsigned int keylen) 12562306a36Sopenharmony_ci{ 12662306a36Sopenharmony_ci struct rk_cipher_ctx *ctx = crypto_skcipher_ctx(cipher); 12762306a36Sopenharmony_ci int err; 12862306a36Sopenharmony_ci 12962306a36Sopenharmony_ci err = verify_skcipher_des_key(cipher, key); 13062306a36Sopenharmony_ci if (err) 13162306a36Sopenharmony_ci return err; 13262306a36Sopenharmony_ci 13362306a36Sopenharmony_ci ctx->keylen = keylen; 13462306a36Sopenharmony_ci memcpy(ctx->key, key, keylen); 13562306a36Sopenharmony_ci 13662306a36Sopenharmony_ci return crypto_skcipher_setkey(ctx->fallback_tfm, key, keylen); 13762306a36Sopenharmony_ci} 13862306a36Sopenharmony_ci 13962306a36Sopenharmony_cistatic int rk_tdes_setkey(struct crypto_skcipher *cipher, 14062306a36Sopenharmony_ci const u8 *key, unsigned int keylen) 14162306a36Sopenharmony_ci{ 14262306a36Sopenharmony_ci struct rk_cipher_ctx *ctx = crypto_skcipher_ctx(cipher); 14362306a36Sopenharmony_ci int err; 14462306a36Sopenharmony_ci 14562306a36Sopenharmony_ci err = verify_skcipher_des3_key(cipher, key); 14662306a36Sopenharmony_ci if (err) 14762306a36Sopenharmony_ci return err; 14862306a36Sopenharmony_ci 14962306a36Sopenharmony_ci ctx->keylen = keylen; 15062306a36Sopenharmony_ci memcpy(ctx->key, key, keylen); 15162306a36Sopenharmony_ci 15262306a36Sopenharmony_ci return crypto_skcipher_setkey(ctx->fallback_tfm, key, keylen); 15362306a36Sopenharmony_ci} 15462306a36Sopenharmony_ci 15562306a36Sopenharmony_cistatic int rk_aes_ecb_encrypt(struct skcipher_request *req) 15662306a36Sopenharmony_ci{ 15762306a36Sopenharmony_ci struct rk_cipher_rctx *rctx = skcipher_request_ctx(req); 15862306a36Sopenharmony_ci 15962306a36Sopenharmony_ci rctx->mode = RK_CRYPTO_AES_ECB_MODE; 16062306a36Sopenharmony_ci return rk_cipher_handle_req(req); 16162306a36Sopenharmony_ci} 16262306a36Sopenharmony_ci 16362306a36Sopenharmony_cistatic int rk_aes_ecb_decrypt(struct skcipher_request *req) 16462306a36Sopenharmony_ci{ 16562306a36Sopenharmony_ci struct rk_cipher_rctx *rctx = skcipher_request_ctx(req); 16662306a36Sopenharmony_ci 16762306a36Sopenharmony_ci rctx->mode = RK_CRYPTO_AES_ECB_MODE | RK_CRYPTO_DEC; 16862306a36Sopenharmony_ci return rk_cipher_handle_req(req); 16962306a36Sopenharmony_ci} 17062306a36Sopenharmony_ci 17162306a36Sopenharmony_cistatic int rk_aes_cbc_encrypt(struct skcipher_request *req) 17262306a36Sopenharmony_ci{ 17362306a36Sopenharmony_ci struct rk_cipher_rctx *rctx = skcipher_request_ctx(req); 17462306a36Sopenharmony_ci 17562306a36Sopenharmony_ci rctx->mode = RK_CRYPTO_AES_CBC_MODE; 17662306a36Sopenharmony_ci return rk_cipher_handle_req(req); 17762306a36Sopenharmony_ci} 17862306a36Sopenharmony_ci 17962306a36Sopenharmony_cistatic int rk_aes_cbc_decrypt(struct skcipher_request *req) 18062306a36Sopenharmony_ci{ 18162306a36Sopenharmony_ci struct rk_cipher_rctx *rctx = skcipher_request_ctx(req); 18262306a36Sopenharmony_ci 18362306a36Sopenharmony_ci rctx->mode = RK_CRYPTO_AES_CBC_MODE | RK_CRYPTO_DEC; 18462306a36Sopenharmony_ci return rk_cipher_handle_req(req); 18562306a36Sopenharmony_ci} 18662306a36Sopenharmony_ci 18762306a36Sopenharmony_cistatic int rk_des_ecb_encrypt(struct skcipher_request *req) 18862306a36Sopenharmony_ci{ 18962306a36Sopenharmony_ci struct rk_cipher_rctx *rctx = skcipher_request_ctx(req); 19062306a36Sopenharmony_ci 19162306a36Sopenharmony_ci rctx->mode = 0; 19262306a36Sopenharmony_ci return rk_cipher_handle_req(req); 19362306a36Sopenharmony_ci} 19462306a36Sopenharmony_ci 19562306a36Sopenharmony_cistatic int rk_des_ecb_decrypt(struct skcipher_request *req) 19662306a36Sopenharmony_ci{ 19762306a36Sopenharmony_ci struct rk_cipher_rctx *rctx = skcipher_request_ctx(req); 19862306a36Sopenharmony_ci 19962306a36Sopenharmony_ci rctx->mode = RK_CRYPTO_DEC; 20062306a36Sopenharmony_ci return rk_cipher_handle_req(req); 20162306a36Sopenharmony_ci} 20262306a36Sopenharmony_ci 20362306a36Sopenharmony_cistatic int rk_des_cbc_encrypt(struct skcipher_request *req) 20462306a36Sopenharmony_ci{ 20562306a36Sopenharmony_ci struct rk_cipher_rctx *rctx = skcipher_request_ctx(req); 20662306a36Sopenharmony_ci 20762306a36Sopenharmony_ci rctx->mode = RK_CRYPTO_TDES_CHAINMODE_CBC; 20862306a36Sopenharmony_ci return rk_cipher_handle_req(req); 20962306a36Sopenharmony_ci} 21062306a36Sopenharmony_ci 21162306a36Sopenharmony_cistatic int rk_des_cbc_decrypt(struct skcipher_request *req) 21262306a36Sopenharmony_ci{ 21362306a36Sopenharmony_ci struct rk_cipher_rctx *rctx = skcipher_request_ctx(req); 21462306a36Sopenharmony_ci 21562306a36Sopenharmony_ci rctx->mode = RK_CRYPTO_TDES_CHAINMODE_CBC | RK_CRYPTO_DEC; 21662306a36Sopenharmony_ci return rk_cipher_handle_req(req); 21762306a36Sopenharmony_ci} 21862306a36Sopenharmony_ci 21962306a36Sopenharmony_cistatic int rk_des3_ede_ecb_encrypt(struct skcipher_request *req) 22062306a36Sopenharmony_ci{ 22162306a36Sopenharmony_ci struct rk_cipher_rctx *rctx = skcipher_request_ctx(req); 22262306a36Sopenharmony_ci 22362306a36Sopenharmony_ci rctx->mode = RK_CRYPTO_TDES_SELECT; 22462306a36Sopenharmony_ci return rk_cipher_handle_req(req); 22562306a36Sopenharmony_ci} 22662306a36Sopenharmony_ci 22762306a36Sopenharmony_cistatic int rk_des3_ede_ecb_decrypt(struct skcipher_request *req) 22862306a36Sopenharmony_ci{ 22962306a36Sopenharmony_ci struct rk_cipher_rctx *rctx = skcipher_request_ctx(req); 23062306a36Sopenharmony_ci 23162306a36Sopenharmony_ci rctx->mode = RK_CRYPTO_TDES_SELECT | RK_CRYPTO_DEC; 23262306a36Sopenharmony_ci return rk_cipher_handle_req(req); 23362306a36Sopenharmony_ci} 23462306a36Sopenharmony_ci 23562306a36Sopenharmony_cistatic int rk_des3_ede_cbc_encrypt(struct skcipher_request *req) 23662306a36Sopenharmony_ci{ 23762306a36Sopenharmony_ci struct rk_cipher_rctx *rctx = skcipher_request_ctx(req); 23862306a36Sopenharmony_ci 23962306a36Sopenharmony_ci rctx->mode = RK_CRYPTO_TDES_SELECT | RK_CRYPTO_TDES_CHAINMODE_CBC; 24062306a36Sopenharmony_ci return rk_cipher_handle_req(req); 24162306a36Sopenharmony_ci} 24262306a36Sopenharmony_ci 24362306a36Sopenharmony_cistatic int rk_des3_ede_cbc_decrypt(struct skcipher_request *req) 24462306a36Sopenharmony_ci{ 24562306a36Sopenharmony_ci struct rk_cipher_rctx *rctx = skcipher_request_ctx(req); 24662306a36Sopenharmony_ci 24762306a36Sopenharmony_ci rctx->mode = RK_CRYPTO_TDES_SELECT | RK_CRYPTO_TDES_CHAINMODE_CBC | 24862306a36Sopenharmony_ci RK_CRYPTO_DEC; 24962306a36Sopenharmony_ci return rk_cipher_handle_req(req); 25062306a36Sopenharmony_ci} 25162306a36Sopenharmony_ci 25262306a36Sopenharmony_cistatic void rk_cipher_hw_init(struct rk_crypto_info *dev, struct skcipher_request *req) 25362306a36Sopenharmony_ci{ 25462306a36Sopenharmony_ci struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(req); 25562306a36Sopenharmony_ci struct crypto_tfm *tfm = crypto_skcipher_tfm(cipher); 25662306a36Sopenharmony_ci struct rk_cipher_rctx *rctx = skcipher_request_ctx(req); 25762306a36Sopenharmony_ci struct rk_cipher_ctx *ctx = crypto_skcipher_ctx(cipher); 25862306a36Sopenharmony_ci u32 block, conf_reg = 0; 25962306a36Sopenharmony_ci 26062306a36Sopenharmony_ci block = crypto_tfm_alg_blocksize(tfm); 26162306a36Sopenharmony_ci 26262306a36Sopenharmony_ci if (block == DES_BLOCK_SIZE) { 26362306a36Sopenharmony_ci rctx->mode |= RK_CRYPTO_TDES_FIFO_MODE | 26462306a36Sopenharmony_ci RK_CRYPTO_TDES_BYTESWAP_KEY | 26562306a36Sopenharmony_ci RK_CRYPTO_TDES_BYTESWAP_IV; 26662306a36Sopenharmony_ci CRYPTO_WRITE(dev, RK_CRYPTO_TDES_CTRL, rctx->mode); 26762306a36Sopenharmony_ci memcpy_toio(dev->reg + RK_CRYPTO_TDES_KEY1_0, ctx->key, ctx->keylen); 26862306a36Sopenharmony_ci conf_reg = RK_CRYPTO_DESSEL; 26962306a36Sopenharmony_ci } else { 27062306a36Sopenharmony_ci rctx->mode |= RK_CRYPTO_AES_FIFO_MODE | 27162306a36Sopenharmony_ci RK_CRYPTO_AES_KEY_CHANGE | 27262306a36Sopenharmony_ci RK_CRYPTO_AES_BYTESWAP_KEY | 27362306a36Sopenharmony_ci RK_CRYPTO_AES_BYTESWAP_IV; 27462306a36Sopenharmony_ci if (ctx->keylen == AES_KEYSIZE_192) 27562306a36Sopenharmony_ci rctx->mode |= RK_CRYPTO_AES_192BIT_key; 27662306a36Sopenharmony_ci else if (ctx->keylen == AES_KEYSIZE_256) 27762306a36Sopenharmony_ci rctx->mode |= RK_CRYPTO_AES_256BIT_key; 27862306a36Sopenharmony_ci CRYPTO_WRITE(dev, RK_CRYPTO_AES_CTRL, rctx->mode); 27962306a36Sopenharmony_ci memcpy_toio(dev->reg + RK_CRYPTO_AES_KEY_0, ctx->key, ctx->keylen); 28062306a36Sopenharmony_ci } 28162306a36Sopenharmony_ci conf_reg |= RK_CRYPTO_BYTESWAP_BTFIFO | 28262306a36Sopenharmony_ci RK_CRYPTO_BYTESWAP_BRFIFO; 28362306a36Sopenharmony_ci CRYPTO_WRITE(dev, RK_CRYPTO_CONF, conf_reg); 28462306a36Sopenharmony_ci CRYPTO_WRITE(dev, RK_CRYPTO_INTENA, 28562306a36Sopenharmony_ci RK_CRYPTO_BCDMA_ERR_ENA | RK_CRYPTO_BCDMA_DONE_ENA); 28662306a36Sopenharmony_ci} 28762306a36Sopenharmony_ci 28862306a36Sopenharmony_cistatic void crypto_dma_start(struct rk_crypto_info *dev, 28962306a36Sopenharmony_ci struct scatterlist *sgs, 29062306a36Sopenharmony_ci struct scatterlist *sgd, unsigned int todo) 29162306a36Sopenharmony_ci{ 29262306a36Sopenharmony_ci CRYPTO_WRITE(dev, RK_CRYPTO_BRDMAS, sg_dma_address(sgs)); 29362306a36Sopenharmony_ci CRYPTO_WRITE(dev, RK_CRYPTO_BRDMAL, todo); 29462306a36Sopenharmony_ci CRYPTO_WRITE(dev, RK_CRYPTO_BTDMAS, sg_dma_address(sgd)); 29562306a36Sopenharmony_ci CRYPTO_WRITE(dev, RK_CRYPTO_CTRL, RK_CRYPTO_BLOCK_START | 29662306a36Sopenharmony_ci _SBF(RK_CRYPTO_BLOCK_START, 16)); 29762306a36Sopenharmony_ci} 29862306a36Sopenharmony_ci 29962306a36Sopenharmony_cistatic int rk_cipher_run(struct crypto_engine *engine, void *async_req) 30062306a36Sopenharmony_ci{ 30162306a36Sopenharmony_ci struct skcipher_request *areq = container_of(async_req, struct skcipher_request, base); 30262306a36Sopenharmony_ci struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq); 30362306a36Sopenharmony_ci struct rk_cipher_rctx *rctx = skcipher_request_ctx(areq); 30462306a36Sopenharmony_ci struct scatterlist *sgs, *sgd; 30562306a36Sopenharmony_ci int err = 0; 30662306a36Sopenharmony_ci int ivsize = crypto_skcipher_ivsize(tfm); 30762306a36Sopenharmony_ci int offset; 30862306a36Sopenharmony_ci u8 iv[AES_BLOCK_SIZE]; 30962306a36Sopenharmony_ci u8 biv[AES_BLOCK_SIZE]; 31062306a36Sopenharmony_ci u8 *ivtouse = areq->iv; 31162306a36Sopenharmony_ci unsigned int len = areq->cryptlen; 31262306a36Sopenharmony_ci unsigned int todo; 31362306a36Sopenharmony_ci struct skcipher_alg *alg = crypto_skcipher_alg(tfm); 31462306a36Sopenharmony_ci struct rk_crypto_tmp *algt = container_of(alg, struct rk_crypto_tmp, alg.skcipher.base); 31562306a36Sopenharmony_ci struct rk_crypto_info *rkc = rctx->dev; 31662306a36Sopenharmony_ci 31762306a36Sopenharmony_ci err = pm_runtime_resume_and_get(rkc->dev); 31862306a36Sopenharmony_ci if (err) 31962306a36Sopenharmony_ci return err; 32062306a36Sopenharmony_ci 32162306a36Sopenharmony_ci algt->stat_req++; 32262306a36Sopenharmony_ci rkc->nreq++; 32362306a36Sopenharmony_ci 32462306a36Sopenharmony_ci ivsize = crypto_skcipher_ivsize(tfm); 32562306a36Sopenharmony_ci if (areq->iv && crypto_skcipher_ivsize(tfm) > 0) { 32662306a36Sopenharmony_ci if (rctx->mode & RK_CRYPTO_DEC) { 32762306a36Sopenharmony_ci offset = areq->cryptlen - ivsize; 32862306a36Sopenharmony_ci scatterwalk_map_and_copy(rctx->backup_iv, areq->src, 32962306a36Sopenharmony_ci offset, ivsize, 0); 33062306a36Sopenharmony_ci } 33162306a36Sopenharmony_ci } 33262306a36Sopenharmony_ci 33362306a36Sopenharmony_ci sgs = areq->src; 33462306a36Sopenharmony_ci sgd = areq->dst; 33562306a36Sopenharmony_ci 33662306a36Sopenharmony_ci while (sgs && sgd && len) { 33762306a36Sopenharmony_ci if (!sgs->length) { 33862306a36Sopenharmony_ci sgs = sg_next(sgs); 33962306a36Sopenharmony_ci sgd = sg_next(sgd); 34062306a36Sopenharmony_ci continue; 34162306a36Sopenharmony_ci } 34262306a36Sopenharmony_ci if (rctx->mode & RK_CRYPTO_DEC) { 34362306a36Sopenharmony_ci /* we backup last block of source to be used as IV at next step */ 34462306a36Sopenharmony_ci offset = sgs->length - ivsize; 34562306a36Sopenharmony_ci scatterwalk_map_and_copy(biv, sgs, offset, ivsize, 0); 34662306a36Sopenharmony_ci } 34762306a36Sopenharmony_ci if (sgs == sgd) { 34862306a36Sopenharmony_ci err = dma_map_sg(rkc->dev, sgs, 1, DMA_BIDIRECTIONAL); 34962306a36Sopenharmony_ci if (err <= 0) { 35062306a36Sopenharmony_ci err = -EINVAL; 35162306a36Sopenharmony_ci goto theend_iv; 35262306a36Sopenharmony_ci } 35362306a36Sopenharmony_ci } else { 35462306a36Sopenharmony_ci err = dma_map_sg(rkc->dev, sgs, 1, DMA_TO_DEVICE); 35562306a36Sopenharmony_ci if (err <= 0) { 35662306a36Sopenharmony_ci err = -EINVAL; 35762306a36Sopenharmony_ci goto theend_iv; 35862306a36Sopenharmony_ci } 35962306a36Sopenharmony_ci err = dma_map_sg(rkc->dev, sgd, 1, DMA_FROM_DEVICE); 36062306a36Sopenharmony_ci if (err <= 0) { 36162306a36Sopenharmony_ci err = -EINVAL; 36262306a36Sopenharmony_ci goto theend_sgs; 36362306a36Sopenharmony_ci } 36462306a36Sopenharmony_ci } 36562306a36Sopenharmony_ci err = 0; 36662306a36Sopenharmony_ci rk_cipher_hw_init(rkc, areq); 36762306a36Sopenharmony_ci if (ivsize) { 36862306a36Sopenharmony_ci if (ivsize == DES_BLOCK_SIZE) 36962306a36Sopenharmony_ci memcpy_toio(rkc->reg + RK_CRYPTO_TDES_IV_0, ivtouse, ivsize); 37062306a36Sopenharmony_ci else 37162306a36Sopenharmony_ci memcpy_toio(rkc->reg + RK_CRYPTO_AES_IV_0, ivtouse, ivsize); 37262306a36Sopenharmony_ci } 37362306a36Sopenharmony_ci reinit_completion(&rkc->complete); 37462306a36Sopenharmony_ci rkc->status = 0; 37562306a36Sopenharmony_ci 37662306a36Sopenharmony_ci todo = min(sg_dma_len(sgs), len); 37762306a36Sopenharmony_ci len -= todo; 37862306a36Sopenharmony_ci crypto_dma_start(rkc, sgs, sgd, todo / 4); 37962306a36Sopenharmony_ci wait_for_completion_interruptible_timeout(&rkc->complete, 38062306a36Sopenharmony_ci msecs_to_jiffies(2000)); 38162306a36Sopenharmony_ci if (!rkc->status) { 38262306a36Sopenharmony_ci dev_err(rkc->dev, "DMA timeout\n"); 38362306a36Sopenharmony_ci err = -EFAULT; 38462306a36Sopenharmony_ci goto theend; 38562306a36Sopenharmony_ci } 38662306a36Sopenharmony_ci if (sgs == sgd) { 38762306a36Sopenharmony_ci dma_unmap_sg(rkc->dev, sgs, 1, DMA_BIDIRECTIONAL); 38862306a36Sopenharmony_ci } else { 38962306a36Sopenharmony_ci dma_unmap_sg(rkc->dev, sgs, 1, DMA_TO_DEVICE); 39062306a36Sopenharmony_ci dma_unmap_sg(rkc->dev, sgd, 1, DMA_FROM_DEVICE); 39162306a36Sopenharmony_ci } 39262306a36Sopenharmony_ci if (rctx->mode & RK_CRYPTO_DEC) { 39362306a36Sopenharmony_ci memcpy(iv, biv, ivsize); 39462306a36Sopenharmony_ci ivtouse = iv; 39562306a36Sopenharmony_ci } else { 39662306a36Sopenharmony_ci offset = sgd->length - ivsize; 39762306a36Sopenharmony_ci scatterwalk_map_and_copy(iv, sgd, offset, ivsize, 0); 39862306a36Sopenharmony_ci ivtouse = iv; 39962306a36Sopenharmony_ci } 40062306a36Sopenharmony_ci sgs = sg_next(sgs); 40162306a36Sopenharmony_ci sgd = sg_next(sgd); 40262306a36Sopenharmony_ci } 40362306a36Sopenharmony_ci 40462306a36Sopenharmony_ci if (areq->iv && ivsize > 0) { 40562306a36Sopenharmony_ci offset = areq->cryptlen - ivsize; 40662306a36Sopenharmony_ci if (rctx->mode & RK_CRYPTO_DEC) { 40762306a36Sopenharmony_ci memcpy(areq->iv, rctx->backup_iv, ivsize); 40862306a36Sopenharmony_ci memzero_explicit(rctx->backup_iv, ivsize); 40962306a36Sopenharmony_ci } else { 41062306a36Sopenharmony_ci scatterwalk_map_and_copy(areq->iv, areq->dst, offset, 41162306a36Sopenharmony_ci ivsize, 0); 41262306a36Sopenharmony_ci } 41362306a36Sopenharmony_ci } 41462306a36Sopenharmony_ci 41562306a36Sopenharmony_citheend: 41662306a36Sopenharmony_ci pm_runtime_put_autosuspend(rkc->dev); 41762306a36Sopenharmony_ci 41862306a36Sopenharmony_ci local_bh_disable(); 41962306a36Sopenharmony_ci crypto_finalize_skcipher_request(engine, areq, err); 42062306a36Sopenharmony_ci local_bh_enable(); 42162306a36Sopenharmony_ci return 0; 42262306a36Sopenharmony_ci 42362306a36Sopenharmony_citheend_sgs: 42462306a36Sopenharmony_ci if (sgs == sgd) { 42562306a36Sopenharmony_ci dma_unmap_sg(rkc->dev, sgs, 1, DMA_BIDIRECTIONAL); 42662306a36Sopenharmony_ci } else { 42762306a36Sopenharmony_ci dma_unmap_sg(rkc->dev, sgs, 1, DMA_TO_DEVICE); 42862306a36Sopenharmony_ci dma_unmap_sg(rkc->dev, sgd, 1, DMA_FROM_DEVICE); 42962306a36Sopenharmony_ci } 43062306a36Sopenharmony_citheend_iv: 43162306a36Sopenharmony_ci return err; 43262306a36Sopenharmony_ci} 43362306a36Sopenharmony_ci 43462306a36Sopenharmony_cistatic int rk_cipher_tfm_init(struct crypto_skcipher *tfm) 43562306a36Sopenharmony_ci{ 43662306a36Sopenharmony_ci struct rk_cipher_ctx *ctx = crypto_skcipher_ctx(tfm); 43762306a36Sopenharmony_ci const char *name = crypto_tfm_alg_name(&tfm->base); 43862306a36Sopenharmony_ci struct skcipher_alg *alg = crypto_skcipher_alg(tfm); 43962306a36Sopenharmony_ci struct rk_crypto_tmp *algt = container_of(alg, struct rk_crypto_tmp, alg.skcipher.base); 44062306a36Sopenharmony_ci 44162306a36Sopenharmony_ci ctx->fallback_tfm = crypto_alloc_skcipher(name, 0, CRYPTO_ALG_NEED_FALLBACK); 44262306a36Sopenharmony_ci if (IS_ERR(ctx->fallback_tfm)) { 44362306a36Sopenharmony_ci dev_err(algt->dev->dev, "ERROR: Cannot allocate fallback for %s %ld\n", 44462306a36Sopenharmony_ci name, PTR_ERR(ctx->fallback_tfm)); 44562306a36Sopenharmony_ci return PTR_ERR(ctx->fallback_tfm); 44662306a36Sopenharmony_ci } 44762306a36Sopenharmony_ci 44862306a36Sopenharmony_ci tfm->reqsize = sizeof(struct rk_cipher_rctx) + 44962306a36Sopenharmony_ci crypto_skcipher_reqsize(ctx->fallback_tfm); 45062306a36Sopenharmony_ci 45162306a36Sopenharmony_ci return 0; 45262306a36Sopenharmony_ci} 45362306a36Sopenharmony_ci 45462306a36Sopenharmony_cistatic void rk_cipher_tfm_exit(struct crypto_skcipher *tfm) 45562306a36Sopenharmony_ci{ 45662306a36Sopenharmony_ci struct rk_cipher_ctx *ctx = crypto_skcipher_ctx(tfm); 45762306a36Sopenharmony_ci 45862306a36Sopenharmony_ci memzero_explicit(ctx->key, ctx->keylen); 45962306a36Sopenharmony_ci crypto_free_skcipher(ctx->fallback_tfm); 46062306a36Sopenharmony_ci} 46162306a36Sopenharmony_ci 46262306a36Sopenharmony_cistruct rk_crypto_tmp rk_ecb_aes_alg = { 46362306a36Sopenharmony_ci .type = CRYPTO_ALG_TYPE_SKCIPHER, 46462306a36Sopenharmony_ci .alg.skcipher.base = { 46562306a36Sopenharmony_ci .base.cra_name = "ecb(aes)", 46662306a36Sopenharmony_ci .base.cra_driver_name = "ecb-aes-rk", 46762306a36Sopenharmony_ci .base.cra_priority = 300, 46862306a36Sopenharmony_ci .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK, 46962306a36Sopenharmony_ci .base.cra_blocksize = AES_BLOCK_SIZE, 47062306a36Sopenharmony_ci .base.cra_ctxsize = sizeof(struct rk_cipher_ctx), 47162306a36Sopenharmony_ci .base.cra_alignmask = 0x0f, 47262306a36Sopenharmony_ci .base.cra_module = THIS_MODULE, 47362306a36Sopenharmony_ci 47462306a36Sopenharmony_ci .init = rk_cipher_tfm_init, 47562306a36Sopenharmony_ci .exit = rk_cipher_tfm_exit, 47662306a36Sopenharmony_ci .min_keysize = AES_MIN_KEY_SIZE, 47762306a36Sopenharmony_ci .max_keysize = AES_MAX_KEY_SIZE, 47862306a36Sopenharmony_ci .setkey = rk_aes_setkey, 47962306a36Sopenharmony_ci .encrypt = rk_aes_ecb_encrypt, 48062306a36Sopenharmony_ci .decrypt = rk_aes_ecb_decrypt, 48162306a36Sopenharmony_ci }, 48262306a36Sopenharmony_ci .alg.skcipher.op = { 48362306a36Sopenharmony_ci .do_one_request = rk_cipher_run, 48462306a36Sopenharmony_ci }, 48562306a36Sopenharmony_ci}; 48662306a36Sopenharmony_ci 48762306a36Sopenharmony_cistruct rk_crypto_tmp rk_cbc_aes_alg = { 48862306a36Sopenharmony_ci .type = CRYPTO_ALG_TYPE_SKCIPHER, 48962306a36Sopenharmony_ci .alg.skcipher.base = { 49062306a36Sopenharmony_ci .base.cra_name = "cbc(aes)", 49162306a36Sopenharmony_ci .base.cra_driver_name = "cbc-aes-rk", 49262306a36Sopenharmony_ci .base.cra_priority = 300, 49362306a36Sopenharmony_ci .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK, 49462306a36Sopenharmony_ci .base.cra_blocksize = AES_BLOCK_SIZE, 49562306a36Sopenharmony_ci .base.cra_ctxsize = sizeof(struct rk_cipher_ctx), 49662306a36Sopenharmony_ci .base.cra_alignmask = 0x0f, 49762306a36Sopenharmony_ci .base.cra_module = THIS_MODULE, 49862306a36Sopenharmony_ci 49962306a36Sopenharmony_ci .init = rk_cipher_tfm_init, 50062306a36Sopenharmony_ci .exit = rk_cipher_tfm_exit, 50162306a36Sopenharmony_ci .min_keysize = AES_MIN_KEY_SIZE, 50262306a36Sopenharmony_ci .max_keysize = AES_MAX_KEY_SIZE, 50362306a36Sopenharmony_ci .ivsize = AES_BLOCK_SIZE, 50462306a36Sopenharmony_ci .setkey = rk_aes_setkey, 50562306a36Sopenharmony_ci .encrypt = rk_aes_cbc_encrypt, 50662306a36Sopenharmony_ci .decrypt = rk_aes_cbc_decrypt, 50762306a36Sopenharmony_ci }, 50862306a36Sopenharmony_ci .alg.skcipher.op = { 50962306a36Sopenharmony_ci .do_one_request = rk_cipher_run, 51062306a36Sopenharmony_ci }, 51162306a36Sopenharmony_ci}; 51262306a36Sopenharmony_ci 51362306a36Sopenharmony_cistruct rk_crypto_tmp rk_ecb_des_alg = { 51462306a36Sopenharmony_ci .type = CRYPTO_ALG_TYPE_SKCIPHER, 51562306a36Sopenharmony_ci .alg.skcipher.base = { 51662306a36Sopenharmony_ci .base.cra_name = "ecb(des)", 51762306a36Sopenharmony_ci .base.cra_driver_name = "ecb-des-rk", 51862306a36Sopenharmony_ci .base.cra_priority = 300, 51962306a36Sopenharmony_ci .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK, 52062306a36Sopenharmony_ci .base.cra_blocksize = DES_BLOCK_SIZE, 52162306a36Sopenharmony_ci .base.cra_ctxsize = sizeof(struct rk_cipher_ctx), 52262306a36Sopenharmony_ci .base.cra_alignmask = 0x07, 52362306a36Sopenharmony_ci .base.cra_module = THIS_MODULE, 52462306a36Sopenharmony_ci 52562306a36Sopenharmony_ci .init = rk_cipher_tfm_init, 52662306a36Sopenharmony_ci .exit = rk_cipher_tfm_exit, 52762306a36Sopenharmony_ci .min_keysize = DES_KEY_SIZE, 52862306a36Sopenharmony_ci .max_keysize = DES_KEY_SIZE, 52962306a36Sopenharmony_ci .setkey = rk_des_setkey, 53062306a36Sopenharmony_ci .encrypt = rk_des_ecb_encrypt, 53162306a36Sopenharmony_ci .decrypt = rk_des_ecb_decrypt, 53262306a36Sopenharmony_ci }, 53362306a36Sopenharmony_ci .alg.skcipher.op = { 53462306a36Sopenharmony_ci .do_one_request = rk_cipher_run, 53562306a36Sopenharmony_ci }, 53662306a36Sopenharmony_ci}; 53762306a36Sopenharmony_ci 53862306a36Sopenharmony_cistruct rk_crypto_tmp rk_cbc_des_alg = { 53962306a36Sopenharmony_ci .type = CRYPTO_ALG_TYPE_SKCIPHER, 54062306a36Sopenharmony_ci .alg.skcipher.base = { 54162306a36Sopenharmony_ci .base.cra_name = "cbc(des)", 54262306a36Sopenharmony_ci .base.cra_driver_name = "cbc-des-rk", 54362306a36Sopenharmony_ci .base.cra_priority = 300, 54462306a36Sopenharmony_ci .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK, 54562306a36Sopenharmony_ci .base.cra_blocksize = DES_BLOCK_SIZE, 54662306a36Sopenharmony_ci .base.cra_ctxsize = sizeof(struct rk_cipher_ctx), 54762306a36Sopenharmony_ci .base.cra_alignmask = 0x07, 54862306a36Sopenharmony_ci .base.cra_module = THIS_MODULE, 54962306a36Sopenharmony_ci 55062306a36Sopenharmony_ci .init = rk_cipher_tfm_init, 55162306a36Sopenharmony_ci .exit = rk_cipher_tfm_exit, 55262306a36Sopenharmony_ci .min_keysize = DES_KEY_SIZE, 55362306a36Sopenharmony_ci .max_keysize = DES_KEY_SIZE, 55462306a36Sopenharmony_ci .ivsize = DES_BLOCK_SIZE, 55562306a36Sopenharmony_ci .setkey = rk_des_setkey, 55662306a36Sopenharmony_ci .encrypt = rk_des_cbc_encrypt, 55762306a36Sopenharmony_ci .decrypt = rk_des_cbc_decrypt, 55862306a36Sopenharmony_ci }, 55962306a36Sopenharmony_ci .alg.skcipher.op = { 56062306a36Sopenharmony_ci .do_one_request = rk_cipher_run, 56162306a36Sopenharmony_ci }, 56262306a36Sopenharmony_ci}; 56362306a36Sopenharmony_ci 56462306a36Sopenharmony_cistruct rk_crypto_tmp rk_ecb_des3_ede_alg = { 56562306a36Sopenharmony_ci .type = CRYPTO_ALG_TYPE_SKCIPHER, 56662306a36Sopenharmony_ci .alg.skcipher.base = { 56762306a36Sopenharmony_ci .base.cra_name = "ecb(des3_ede)", 56862306a36Sopenharmony_ci .base.cra_driver_name = "ecb-des3-ede-rk", 56962306a36Sopenharmony_ci .base.cra_priority = 300, 57062306a36Sopenharmony_ci .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK, 57162306a36Sopenharmony_ci .base.cra_blocksize = DES_BLOCK_SIZE, 57262306a36Sopenharmony_ci .base.cra_ctxsize = sizeof(struct rk_cipher_ctx), 57362306a36Sopenharmony_ci .base.cra_alignmask = 0x07, 57462306a36Sopenharmony_ci .base.cra_module = THIS_MODULE, 57562306a36Sopenharmony_ci 57662306a36Sopenharmony_ci .init = rk_cipher_tfm_init, 57762306a36Sopenharmony_ci .exit = rk_cipher_tfm_exit, 57862306a36Sopenharmony_ci .min_keysize = DES3_EDE_KEY_SIZE, 57962306a36Sopenharmony_ci .max_keysize = DES3_EDE_KEY_SIZE, 58062306a36Sopenharmony_ci .setkey = rk_tdes_setkey, 58162306a36Sopenharmony_ci .encrypt = rk_des3_ede_ecb_encrypt, 58262306a36Sopenharmony_ci .decrypt = rk_des3_ede_ecb_decrypt, 58362306a36Sopenharmony_ci }, 58462306a36Sopenharmony_ci .alg.skcipher.op = { 58562306a36Sopenharmony_ci .do_one_request = rk_cipher_run, 58662306a36Sopenharmony_ci }, 58762306a36Sopenharmony_ci}; 58862306a36Sopenharmony_ci 58962306a36Sopenharmony_cistruct rk_crypto_tmp rk_cbc_des3_ede_alg = { 59062306a36Sopenharmony_ci .type = CRYPTO_ALG_TYPE_SKCIPHER, 59162306a36Sopenharmony_ci .alg.skcipher.base = { 59262306a36Sopenharmony_ci .base.cra_name = "cbc(des3_ede)", 59362306a36Sopenharmony_ci .base.cra_driver_name = "cbc-des3-ede-rk", 59462306a36Sopenharmony_ci .base.cra_priority = 300, 59562306a36Sopenharmony_ci .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK, 59662306a36Sopenharmony_ci .base.cra_blocksize = DES_BLOCK_SIZE, 59762306a36Sopenharmony_ci .base.cra_ctxsize = sizeof(struct rk_cipher_ctx), 59862306a36Sopenharmony_ci .base.cra_alignmask = 0x07, 59962306a36Sopenharmony_ci .base.cra_module = THIS_MODULE, 60062306a36Sopenharmony_ci 60162306a36Sopenharmony_ci .init = rk_cipher_tfm_init, 60262306a36Sopenharmony_ci .exit = rk_cipher_tfm_exit, 60362306a36Sopenharmony_ci .min_keysize = DES3_EDE_KEY_SIZE, 60462306a36Sopenharmony_ci .max_keysize = DES3_EDE_KEY_SIZE, 60562306a36Sopenharmony_ci .ivsize = DES_BLOCK_SIZE, 60662306a36Sopenharmony_ci .setkey = rk_tdes_setkey, 60762306a36Sopenharmony_ci .encrypt = rk_des3_ede_cbc_encrypt, 60862306a36Sopenharmony_ci .decrypt = rk_des3_ede_cbc_decrypt, 60962306a36Sopenharmony_ci }, 61062306a36Sopenharmony_ci .alg.skcipher.op = { 61162306a36Sopenharmony_ci .do_one_request = rk_cipher_run, 61262306a36Sopenharmony_ci }, 61362306a36Sopenharmony_ci}; 614