1// SPDX-License-Identifier: GPL-2.0-or-later 2/* 3 * Glue Code for AVX assembler version of Twofish Cipher 4 * 5 * Copyright (C) 2012 Johannes Goetzfried 6 * <Johannes.Goetzfried@informatik.stud.uni-erlangen.de> 7 * 8 * Copyright © 2013 Jussi Kivilinna <jussi.kivilinna@iki.fi> 9 */ 10 11#include <linux/module.h> 12#include <linux/types.h> 13#include <linux/crypto.h> 14#include <linux/err.h> 15#include <crypto/algapi.h> 16#include <crypto/internal/simd.h> 17#include <crypto/twofish.h> 18#include <crypto/xts.h> 19#include <asm/crypto/glue_helper.h> 20#include <asm/crypto/twofish.h> 21 22#define TWOFISH_PARALLEL_BLOCKS 8 23 24/* 8-way parallel cipher functions */ 25asmlinkage void twofish_ecb_enc_8way(const void *ctx, u8 *dst, const u8 *src); 26asmlinkage void twofish_ecb_dec_8way(const void *ctx, u8 *dst, const u8 *src); 27 28asmlinkage void twofish_cbc_dec_8way(const void *ctx, u8 *dst, const u8 *src); 29asmlinkage void twofish_ctr_8way(const void *ctx, u8 *dst, const u8 *src, 30 le128 *iv); 31 32asmlinkage void twofish_xts_enc_8way(const void *ctx, u8 *dst, const u8 *src, 33 le128 *iv); 34asmlinkage void twofish_xts_dec_8way(const void *ctx, u8 *dst, const u8 *src, 35 le128 *iv); 36 37static int twofish_setkey_skcipher(struct crypto_skcipher *tfm, 38 const u8 *key, unsigned int keylen) 39{ 40 return twofish_setkey(&tfm->base, key, keylen); 41} 42 43static inline void twofish_enc_blk_3way(const void *ctx, u8 *dst, const u8 *src) 44{ 45 __twofish_enc_blk_3way(ctx, dst, src, false); 46} 47 48static void twofish_xts_enc(const void *ctx, u8 *dst, const u8 *src, le128 *iv) 49{ 50 glue_xts_crypt_128bit_one(ctx, dst, src, iv, twofish_enc_blk); 51} 52 53static void twofish_xts_dec(const void *ctx, u8 *dst, const u8 *src, le128 *iv) 54{ 55 glue_xts_crypt_128bit_one(ctx, dst, src, iv, twofish_dec_blk); 56} 57 58struct twofish_xts_ctx { 59 struct twofish_ctx tweak_ctx; 60 struct twofish_ctx crypt_ctx; 61}; 62 63static int xts_twofish_setkey(struct crypto_skcipher *tfm, const u8 *key, 64 unsigned int keylen) 65{ 66 struct twofish_xts_ctx *ctx = crypto_skcipher_ctx(tfm); 67 int err; 68 69 err = xts_verify_key(tfm, key, keylen); 70 if (err) 71 return err; 72 73 /* first half of xts-key is for crypt */ 74 err = __twofish_setkey(&ctx->crypt_ctx, key, keylen / 2); 75 if (err) 76 return err; 77 78 /* second half of xts-key is for tweak */ 79 return __twofish_setkey(&ctx->tweak_ctx, key + keylen / 2, keylen / 2); 80} 81 82static const struct common_glue_ctx twofish_enc = { 83 .num_funcs = 3, 84 .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS, 85 86 .funcs = { { 87 .num_blocks = TWOFISH_PARALLEL_BLOCKS, 88 .fn_u = { .ecb = twofish_ecb_enc_8way } 89 }, { 90 .num_blocks = 3, 91 .fn_u = { .ecb = twofish_enc_blk_3way } 92 }, { 93 .num_blocks = 1, 94 .fn_u = { .ecb = twofish_enc_blk } 95 } } 96}; 97 98static const struct common_glue_ctx twofish_ctr = { 99 .num_funcs = 3, 100 .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS, 101 102 .funcs = { { 103 .num_blocks = TWOFISH_PARALLEL_BLOCKS, 104 .fn_u = { .ctr = twofish_ctr_8way } 105 }, { 106 .num_blocks = 3, 107 .fn_u = { .ctr = twofish_enc_blk_ctr_3way } 108 }, { 109 .num_blocks = 1, 110 .fn_u = { .ctr = twofish_enc_blk_ctr } 111 } } 112}; 113 114static const struct common_glue_ctx twofish_enc_xts = { 115 .num_funcs = 2, 116 .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS, 117 118 .funcs = { { 119 .num_blocks = TWOFISH_PARALLEL_BLOCKS, 120 .fn_u = { .xts = twofish_xts_enc_8way } 121 }, { 122 .num_blocks = 1, 123 .fn_u = { .xts = twofish_xts_enc } 124 } } 125}; 126 127static const struct common_glue_ctx twofish_dec = { 128 .num_funcs = 3, 129 .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS, 130 131 .funcs = { { 132 .num_blocks = TWOFISH_PARALLEL_BLOCKS, 133 .fn_u = { .ecb = twofish_ecb_dec_8way } 134 }, { 135 .num_blocks = 3, 136 .fn_u = { .ecb = twofish_dec_blk_3way } 137 }, { 138 .num_blocks = 1, 139 .fn_u = { .ecb = twofish_dec_blk } 140 } } 141}; 142 143static const struct common_glue_ctx twofish_dec_cbc = { 144 .num_funcs = 3, 145 .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS, 146 147 .funcs = { { 148 .num_blocks = TWOFISH_PARALLEL_BLOCKS, 149 .fn_u = { .cbc = twofish_cbc_dec_8way } 150 }, { 151 .num_blocks = 3, 152 .fn_u = { .cbc = twofish_dec_blk_cbc_3way } 153 }, { 154 .num_blocks = 1, 155 .fn_u = { .cbc = twofish_dec_blk } 156 } } 157}; 158 159static const struct common_glue_ctx twofish_dec_xts = { 160 .num_funcs = 2, 161 .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS, 162 163 .funcs = { { 164 .num_blocks = TWOFISH_PARALLEL_BLOCKS, 165 .fn_u = { .xts = twofish_xts_dec_8way } 166 }, { 167 .num_blocks = 1, 168 .fn_u = { .xts = twofish_xts_dec } 169 } } 170}; 171 172static int ecb_encrypt(struct skcipher_request *req) 173{ 174 return glue_ecb_req_128bit(&twofish_enc, req); 175} 176 177static int ecb_decrypt(struct skcipher_request *req) 178{ 179 return glue_ecb_req_128bit(&twofish_dec, req); 180} 181 182static int cbc_encrypt(struct skcipher_request *req) 183{ 184 return glue_cbc_encrypt_req_128bit(twofish_enc_blk, req); 185} 186 187static int cbc_decrypt(struct skcipher_request *req) 188{ 189 return glue_cbc_decrypt_req_128bit(&twofish_dec_cbc, req); 190} 191 192static int ctr_crypt(struct skcipher_request *req) 193{ 194 return glue_ctr_req_128bit(&twofish_ctr, req); 195} 196 197static int xts_encrypt(struct skcipher_request *req) 198{ 199 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 200 struct twofish_xts_ctx *ctx = crypto_skcipher_ctx(tfm); 201 202 return glue_xts_req_128bit(&twofish_enc_xts, req, twofish_enc_blk, 203 &ctx->tweak_ctx, &ctx->crypt_ctx, false); 204} 205 206static int xts_decrypt(struct skcipher_request *req) 207{ 208 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 209 struct twofish_xts_ctx *ctx = crypto_skcipher_ctx(tfm); 210 211 return glue_xts_req_128bit(&twofish_dec_xts, req, twofish_enc_blk, 212 &ctx->tweak_ctx, &ctx->crypt_ctx, true); 213} 214 215static struct skcipher_alg twofish_algs[] = { 216 { 217 .base.cra_name = "__ecb(twofish)", 218 .base.cra_driver_name = "__ecb-twofish-avx", 219 .base.cra_priority = 400, 220 .base.cra_flags = CRYPTO_ALG_INTERNAL, 221 .base.cra_blocksize = TF_BLOCK_SIZE, 222 .base.cra_ctxsize = sizeof(struct twofish_ctx), 223 .base.cra_module = THIS_MODULE, 224 .min_keysize = TF_MIN_KEY_SIZE, 225 .max_keysize = TF_MAX_KEY_SIZE, 226 .setkey = twofish_setkey_skcipher, 227 .encrypt = ecb_encrypt, 228 .decrypt = ecb_decrypt, 229 }, { 230 .base.cra_name = "__cbc(twofish)", 231 .base.cra_driver_name = "__cbc-twofish-avx", 232 .base.cra_priority = 400, 233 .base.cra_flags = CRYPTO_ALG_INTERNAL, 234 .base.cra_blocksize = TF_BLOCK_SIZE, 235 .base.cra_ctxsize = sizeof(struct twofish_ctx), 236 .base.cra_module = THIS_MODULE, 237 .min_keysize = TF_MIN_KEY_SIZE, 238 .max_keysize = TF_MAX_KEY_SIZE, 239 .ivsize = TF_BLOCK_SIZE, 240 .setkey = twofish_setkey_skcipher, 241 .encrypt = cbc_encrypt, 242 .decrypt = cbc_decrypt, 243 }, { 244 .base.cra_name = "__ctr(twofish)", 245 .base.cra_driver_name = "__ctr-twofish-avx", 246 .base.cra_priority = 400, 247 .base.cra_flags = CRYPTO_ALG_INTERNAL, 248 .base.cra_blocksize = 1, 249 .base.cra_ctxsize = sizeof(struct twofish_ctx), 250 .base.cra_module = THIS_MODULE, 251 .min_keysize = TF_MIN_KEY_SIZE, 252 .max_keysize = TF_MAX_KEY_SIZE, 253 .ivsize = TF_BLOCK_SIZE, 254 .chunksize = TF_BLOCK_SIZE, 255 .setkey = twofish_setkey_skcipher, 256 .encrypt = ctr_crypt, 257 .decrypt = ctr_crypt, 258 }, { 259 .base.cra_name = "__xts(twofish)", 260 .base.cra_driver_name = "__xts-twofish-avx", 261 .base.cra_priority = 400, 262 .base.cra_flags = CRYPTO_ALG_INTERNAL, 263 .base.cra_blocksize = TF_BLOCK_SIZE, 264 .base.cra_ctxsize = sizeof(struct twofish_xts_ctx), 265 .base.cra_module = THIS_MODULE, 266 .min_keysize = 2 * TF_MIN_KEY_SIZE, 267 .max_keysize = 2 * TF_MAX_KEY_SIZE, 268 .ivsize = TF_BLOCK_SIZE, 269 .setkey = xts_twofish_setkey, 270 .encrypt = xts_encrypt, 271 .decrypt = xts_decrypt, 272 }, 273}; 274 275static struct simd_skcipher_alg *twofish_simd_algs[ARRAY_SIZE(twofish_algs)]; 276 277static int __init twofish_init(void) 278{ 279 const char *feature_name; 280 281 if (!cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, &feature_name)) { 282 pr_info("CPU feature '%s' is not supported.\n", feature_name); 283 return -ENODEV; 284 } 285 286 return simd_register_skciphers_compat(twofish_algs, 287 ARRAY_SIZE(twofish_algs), 288 twofish_simd_algs); 289} 290 291static void __exit twofish_exit(void) 292{ 293 simd_unregister_skciphers(twofish_algs, ARRAY_SIZE(twofish_algs), 294 twofish_simd_algs); 295} 296 297module_init(twofish_init); 298module_exit(twofish_exit); 299 300MODULE_DESCRIPTION("Twofish Cipher Algorithm, AVX optimized"); 301MODULE_LICENSE("GPL"); 302MODULE_ALIAS_CRYPTO("twofish"); 303