1/* SPDX-License-Identifier: GPL-2.0-or-later */ 2/* 3 * SM4 Cipher Algorithm, using ARMv8 NEON 4 * as specified in 5 * https://tools.ietf.org/id/draft-ribose-cfrg-sm4-10.html 6 * 7 * Copyright (C) 2022, Alibaba Group. 8 * Copyright (C) 2022 Tianjia Zhang <tianjia.zhang@linux.alibaba.com> 9 */ 10 11#include <linux/module.h> 12#include <linux/crypto.h> 13#include <linux/kernel.h> 14#include <linux/cpufeature.h> 15#include <asm/neon.h> 16#include <asm/simd.h> 17#include <crypto/internal/simd.h> 18#include <crypto/internal/skcipher.h> 19#include <crypto/sm4.h> 20 21asmlinkage void sm4_neon_crypt(const u32 *rkey, u8 *dst, const u8 *src, 22 unsigned int nblocks); 23asmlinkage void sm4_neon_cbc_dec(const u32 *rkey_dec, u8 *dst, const u8 *src, 24 u8 *iv, unsigned int nblocks); 25asmlinkage void sm4_neon_cfb_dec(const u32 *rkey_enc, u8 *dst, const u8 *src, 26 u8 *iv, unsigned int nblocks); 27asmlinkage void sm4_neon_ctr_crypt(const u32 *rkey_enc, u8 *dst, const u8 *src, 28 u8 *iv, unsigned int nblocks); 29 30static int sm4_setkey(struct crypto_skcipher *tfm, const u8 *key, 31 unsigned int key_len) 32{ 33 struct sm4_ctx *ctx = crypto_skcipher_ctx(tfm); 34 35 return sm4_expandkey(ctx, key, key_len); 36} 37 38static int sm4_ecb_do_crypt(struct skcipher_request *req, const u32 *rkey) 39{ 40 struct skcipher_walk walk; 41 unsigned int nbytes; 42 int err; 43 44 err = skcipher_walk_virt(&walk, req, false); 45 46 while ((nbytes = walk.nbytes) > 0) { 47 const u8 *src = walk.src.virt.addr; 48 u8 *dst = walk.dst.virt.addr; 49 unsigned int nblocks; 50 51 nblocks = nbytes / SM4_BLOCK_SIZE; 52 if (nblocks) { 53 kernel_neon_begin(); 54 55 sm4_neon_crypt(rkey, dst, src, nblocks); 56 57 kernel_neon_end(); 58 } 59 60 err = skcipher_walk_done(&walk, nbytes % SM4_BLOCK_SIZE); 61 } 62 63 return err; 64} 65 66static int sm4_ecb_encrypt(struct skcipher_request *req) 67{ 68 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 69 struct sm4_ctx *ctx = crypto_skcipher_ctx(tfm); 70 71 return sm4_ecb_do_crypt(req, ctx->rkey_enc); 72} 73 74static int sm4_ecb_decrypt(struct skcipher_request *req) 75{ 76 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 77 struct sm4_ctx *ctx = crypto_skcipher_ctx(tfm); 78 79 return sm4_ecb_do_crypt(req, ctx->rkey_dec); 80} 81 82static int sm4_cbc_encrypt(struct skcipher_request *req) 83{ 84 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 85 struct sm4_ctx *ctx = crypto_skcipher_ctx(tfm); 86 struct skcipher_walk walk; 87 unsigned int nbytes; 88 int err; 89 90 err = skcipher_walk_virt(&walk, req, false); 91 92 while ((nbytes = walk.nbytes) > 0) { 93 const u8 *iv = walk.iv; 94 const u8 *src = walk.src.virt.addr; 95 u8 *dst = walk.dst.virt.addr; 96 97 while (nbytes >= SM4_BLOCK_SIZE) { 98 crypto_xor_cpy(dst, src, iv, SM4_BLOCK_SIZE); 99 sm4_crypt_block(ctx->rkey_enc, dst, dst); 100 iv = dst; 101 src += SM4_BLOCK_SIZE; 102 dst += SM4_BLOCK_SIZE; 103 nbytes -= SM4_BLOCK_SIZE; 104 } 105 if (iv != walk.iv) 106 memcpy(walk.iv, iv, SM4_BLOCK_SIZE); 107 108 err = skcipher_walk_done(&walk, nbytes); 109 } 110 111 return err; 112} 113 114static int sm4_cbc_decrypt(struct skcipher_request *req) 115{ 116 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 117 struct sm4_ctx *ctx = crypto_skcipher_ctx(tfm); 118 struct skcipher_walk walk; 119 unsigned int nbytes; 120 int err; 121 122 err = skcipher_walk_virt(&walk, req, false); 123 124 while ((nbytes = walk.nbytes) > 0) { 125 const u8 *src = walk.src.virt.addr; 126 u8 *dst = walk.dst.virt.addr; 127 unsigned int nblocks; 128 129 nblocks = nbytes / SM4_BLOCK_SIZE; 130 if (nblocks) { 131 kernel_neon_begin(); 132 133 sm4_neon_cbc_dec(ctx->rkey_dec, dst, src, 134 walk.iv, nblocks); 135 136 kernel_neon_end(); 137 } 138 139 err = skcipher_walk_done(&walk, nbytes % SM4_BLOCK_SIZE); 140 } 141 142 return err; 143} 144 145static int sm4_cfb_encrypt(struct skcipher_request *req) 146{ 147 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 148 struct sm4_ctx *ctx = crypto_skcipher_ctx(tfm); 149 struct skcipher_walk walk; 150 unsigned int nbytes; 151 int err; 152 153 err = skcipher_walk_virt(&walk, req, false); 154 155 while ((nbytes = walk.nbytes) > 0) { 156 u8 keystream[SM4_BLOCK_SIZE]; 157 const u8 *iv = walk.iv; 158 const u8 *src = walk.src.virt.addr; 159 u8 *dst = walk.dst.virt.addr; 160 161 while (nbytes >= SM4_BLOCK_SIZE) { 162 sm4_crypt_block(ctx->rkey_enc, keystream, iv); 163 crypto_xor_cpy(dst, src, keystream, SM4_BLOCK_SIZE); 164 iv = dst; 165 src += SM4_BLOCK_SIZE; 166 dst += SM4_BLOCK_SIZE; 167 nbytes -= SM4_BLOCK_SIZE; 168 } 169 if (iv != walk.iv) 170 memcpy(walk.iv, iv, SM4_BLOCK_SIZE); 171 172 /* tail */ 173 if (walk.nbytes == walk.total && nbytes > 0) { 174 sm4_crypt_block(ctx->rkey_enc, keystream, walk.iv); 175 crypto_xor_cpy(dst, src, keystream, nbytes); 176 nbytes = 0; 177 } 178 179 err = skcipher_walk_done(&walk, nbytes); 180 } 181 182 return err; 183} 184 185static int sm4_cfb_decrypt(struct skcipher_request *req) 186{ 187 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 188 struct sm4_ctx *ctx = crypto_skcipher_ctx(tfm); 189 struct skcipher_walk walk; 190 unsigned int nbytes; 191 int err; 192 193 err = skcipher_walk_virt(&walk, req, false); 194 195 while ((nbytes = walk.nbytes) > 0) { 196 const u8 *src = walk.src.virt.addr; 197 u8 *dst = walk.dst.virt.addr; 198 unsigned int nblocks; 199 200 nblocks = nbytes / SM4_BLOCK_SIZE; 201 if (nblocks) { 202 kernel_neon_begin(); 203 204 sm4_neon_cfb_dec(ctx->rkey_enc, dst, src, 205 walk.iv, nblocks); 206 207 kernel_neon_end(); 208 209 dst += nblocks * SM4_BLOCK_SIZE; 210 src += nblocks * SM4_BLOCK_SIZE; 211 nbytes -= nblocks * SM4_BLOCK_SIZE; 212 } 213 214 /* tail */ 215 if (walk.nbytes == walk.total && nbytes > 0) { 216 u8 keystream[SM4_BLOCK_SIZE]; 217 218 sm4_crypt_block(ctx->rkey_enc, keystream, walk.iv); 219 crypto_xor_cpy(dst, src, keystream, nbytes); 220 nbytes = 0; 221 } 222 223 err = skcipher_walk_done(&walk, nbytes); 224 } 225 226 return err; 227} 228 229static int sm4_ctr_crypt(struct skcipher_request *req) 230{ 231 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 232 struct sm4_ctx *ctx = crypto_skcipher_ctx(tfm); 233 struct skcipher_walk walk; 234 unsigned int nbytes; 235 int err; 236 237 err = skcipher_walk_virt(&walk, req, false); 238 239 while ((nbytes = walk.nbytes) > 0) { 240 const u8 *src = walk.src.virt.addr; 241 u8 *dst = walk.dst.virt.addr; 242 unsigned int nblocks; 243 244 nblocks = nbytes / SM4_BLOCK_SIZE; 245 if (nblocks) { 246 kernel_neon_begin(); 247 248 sm4_neon_ctr_crypt(ctx->rkey_enc, dst, src, 249 walk.iv, nblocks); 250 251 kernel_neon_end(); 252 253 dst += nblocks * SM4_BLOCK_SIZE; 254 src += nblocks * SM4_BLOCK_SIZE; 255 nbytes -= nblocks * SM4_BLOCK_SIZE; 256 } 257 258 /* tail */ 259 if (walk.nbytes == walk.total && nbytes > 0) { 260 u8 keystream[SM4_BLOCK_SIZE]; 261 262 sm4_crypt_block(ctx->rkey_enc, keystream, walk.iv); 263 crypto_inc(walk.iv, SM4_BLOCK_SIZE); 264 crypto_xor_cpy(dst, src, keystream, nbytes); 265 nbytes = 0; 266 } 267 268 err = skcipher_walk_done(&walk, nbytes); 269 } 270 271 return err; 272} 273 274static struct skcipher_alg sm4_algs[] = { 275 { 276 .base = { 277 .cra_name = "ecb(sm4)", 278 .cra_driver_name = "ecb-sm4-neon", 279 .cra_priority = 200, 280 .cra_blocksize = SM4_BLOCK_SIZE, 281 .cra_ctxsize = sizeof(struct sm4_ctx), 282 .cra_module = THIS_MODULE, 283 }, 284 .min_keysize = SM4_KEY_SIZE, 285 .max_keysize = SM4_KEY_SIZE, 286 .setkey = sm4_setkey, 287 .encrypt = sm4_ecb_encrypt, 288 .decrypt = sm4_ecb_decrypt, 289 }, { 290 .base = { 291 .cra_name = "cbc(sm4)", 292 .cra_driver_name = "cbc-sm4-neon", 293 .cra_priority = 200, 294 .cra_blocksize = SM4_BLOCK_SIZE, 295 .cra_ctxsize = sizeof(struct sm4_ctx), 296 .cra_module = THIS_MODULE, 297 }, 298 .min_keysize = SM4_KEY_SIZE, 299 .max_keysize = SM4_KEY_SIZE, 300 .ivsize = SM4_BLOCK_SIZE, 301 .setkey = sm4_setkey, 302 .encrypt = sm4_cbc_encrypt, 303 .decrypt = sm4_cbc_decrypt, 304 }, { 305 .base = { 306 .cra_name = "cfb(sm4)", 307 .cra_driver_name = "cfb-sm4-neon", 308 .cra_priority = 200, 309 .cra_blocksize = 1, 310 .cra_ctxsize = sizeof(struct sm4_ctx), 311 .cra_module = THIS_MODULE, 312 }, 313 .min_keysize = SM4_KEY_SIZE, 314 .max_keysize = SM4_KEY_SIZE, 315 .ivsize = SM4_BLOCK_SIZE, 316 .chunksize = SM4_BLOCK_SIZE, 317 .setkey = sm4_setkey, 318 .encrypt = sm4_cfb_encrypt, 319 .decrypt = sm4_cfb_decrypt, 320 }, { 321 .base = { 322 .cra_name = "ctr(sm4)", 323 .cra_driver_name = "ctr-sm4-neon", 324 .cra_priority = 200, 325 .cra_blocksize = 1, 326 .cra_ctxsize = sizeof(struct sm4_ctx), 327 .cra_module = THIS_MODULE, 328 }, 329 .min_keysize = SM4_KEY_SIZE, 330 .max_keysize = SM4_KEY_SIZE, 331 .ivsize = SM4_BLOCK_SIZE, 332 .chunksize = SM4_BLOCK_SIZE, 333 .setkey = sm4_setkey, 334 .encrypt = sm4_ctr_crypt, 335 .decrypt = sm4_ctr_crypt, 336 } 337}; 338 339static int __init sm4_init(void) 340{ 341 return crypto_register_skciphers(sm4_algs, ARRAY_SIZE(sm4_algs)); 342} 343 344static void __exit sm4_exit(void) 345{ 346 crypto_unregister_skciphers(sm4_algs, ARRAY_SIZE(sm4_algs)); 347} 348 349module_init(sm4_init); 350module_exit(sm4_exit); 351 352MODULE_DESCRIPTION("SM4 ECB/CBC/CFB/CTR using ARMv8 NEON"); 353MODULE_ALIAS_CRYPTO("sm4-neon"); 354MODULE_ALIAS_CRYPTO("sm4"); 355MODULE_ALIAS_CRYPTO("ecb(sm4)"); 356MODULE_ALIAS_CRYPTO("cbc(sm4)"); 357MODULE_ALIAS_CRYPTO("cfb(sm4)"); 358MODULE_ALIAS_CRYPTO("ctr(sm4)"); 359MODULE_AUTHOR("Tianjia Zhang <tianjia.zhang@linux.alibaba.com>"); 360MODULE_LICENSE("GPL v2"); 361