1// SPDX-License-Identifier: GPL-2.0 2/* 3 * Copyright (C) 2017 Marvell 4 * 5 * Antoine Tenart <antoine.tenart@free-electrons.com> 6 */ 7 8#include <asm/unaligned.h> 9#include <linux/device.h> 10#include <linux/dma-mapping.h> 11#include <linux/dmapool.h> 12#include <crypto/aead.h> 13#include <crypto/aes.h> 14#include <crypto/authenc.h> 15#include <crypto/chacha.h> 16#include <crypto/ctr.h> 17#include <crypto/internal/des.h> 18#include <crypto/gcm.h> 19#include <crypto/ghash.h> 20#include <crypto/poly1305.h> 21#include <crypto/sha.h> 22#include <crypto/sm3.h> 23#include <crypto/sm4.h> 24#include <crypto/xts.h> 25#include <crypto/skcipher.h> 26#include <crypto/internal/aead.h> 27#include <crypto/internal/skcipher.h> 28 29#include "safexcel.h" 30 31enum safexcel_cipher_direction { 32 SAFEXCEL_ENCRYPT, 33 SAFEXCEL_DECRYPT, 34}; 35 36enum safexcel_cipher_alg { 37 SAFEXCEL_DES, 38 SAFEXCEL_3DES, 39 SAFEXCEL_AES, 40 SAFEXCEL_CHACHA20, 41 SAFEXCEL_SM4, 42}; 43 44struct safexcel_cipher_ctx { 45 struct safexcel_context base; 46 struct safexcel_crypto_priv *priv; 47 48 u32 mode; 49 enum safexcel_cipher_alg alg; 50 u8 aead; /* !=0=AEAD, 2=IPSec ESP AEAD, 3=IPsec ESP GMAC */ 51 u8 xcm; /* 0=authenc, 1=GCM, 2 reserved for CCM */ 52 u8 aadskip; 53 u8 blocksz; 54 u32 ivmask; 55 u32 ctrinit; 56 57 __le32 key[16]; 58 u32 nonce; 59 unsigned int key_len, xts; 60 61 /* All the below is AEAD specific */ 62 u32 hash_alg; 63 u32 state_sz; 64 65 struct crypto_cipher *hkaes; 66 struct crypto_aead *fback; 67}; 68 69struct safexcel_cipher_req { 70 enum safexcel_cipher_direction direction; 71 /* Number of result descriptors associated to the request */ 72 unsigned int rdescs; 73 bool needs_inv; 74 int nr_src, nr_dst; 75}; 76 77static int safexcel_skcipher_iv(struct safexcel_cipher_ctx *ctx, u8 *iv, 78 struct safexcel_command_desc *cdesc) 79{ 80 if (ctx->mode == CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD) { 81 cdesc->control_data.options |= EIP197_OPTION_4_TOKEN_IV_CMD; 82 /* 32 bit nonce */ 83 cdesc->control_data.token[0] = ctx->nonce; 84 /* 64 bit IV part */ 85 memcpy(&cdesc->control_data.token[1], iv, 8); 86 /* 32 bit counter, start at 0 or 1 (big endian!) */ 87 cdesc->control_data.token[3] = 88 (__force u32)cpu_to_be32(ctx->ctrinit); 89 return 4; 90 } 91 if (ctx->alg == SAFEXCEL_CHACHA20) { 92 cdesc->control_data.options |= EIP197_OPTION_4_TOKEN_IV_CMD; 93 /* 96 bit nonce part */ 94 memcpy(&cdesc->control_data.token[0], &iv[4], 12); 95 /* 32 bit counter */ 96 cdesc->control_data.token[3] = *(u32 *)iv; 97 return 4; 98 } 99 100 cdesc->control_data.options |= ctx->ivmask; 101 memcpy(cdesc->control_data.token, iv, ctx->blocksz); 102 return ctx->blocksz / sizeof(u32); 103} 104 105static void safexcel_skcipher_token(struct safexcel_cipher_ctx *ctx, u8 *iv, 106 struct safexcel_command_desc *cdesc, 107 struct safexcel_token *atoken, 108 u32 length) 109{ 110 struct safexcel_token *token; 111 int ivlen; 112 113 ivlen = safexcel_skcipher_iv(ctx, iv, cdesc); 114 if (ivlen == 4) { 115 /* No space in cdesc, instruction moves to atoken */ 116 cdesc->additional_cdata_size = 1; 117 token = atoken; 118 } else { 119 /* Everything fits in cdesc */ 120 token = (struct safexcel_token *)(cdesc->control_data.token + 2); 121 /* Need to pad with NOP */ 122 eip197_noop_token(&token[1]); 123 } 124 125 token->opcode = EIP197_TOKEN_OPCODE_DIRECTION; 126 token->packet_length = length; 127 token->stat = EIP197_TOKEN_STAT_LAST_PACKET | 128 EIP197_TOKEN_STAT_LAST_HASH; 129 token->instructions = EIP197_TOKEN_INS_LAST | 130 EIP197_TOKEN_INS_TYPE_CRYPTO | 131 EIP197_TOKEN_INS_TYPE_OUTPUT; 132} 133 134static void safexcel_aead_iv(struct safexcel_cipher_ctx *ctx, u8 *iv, 135 struct safexcel_command_desc *cdesc) 136{ 137 if (ctx->mode == CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD || 138 ctx->aead & EIP197_AEAD_TYPE_IPSEC_ESP) { /* _ESP and _ESP_GMAC */ 139 /* 32 bit nonce */ 140 cdesc->control_data.token[0] = ctx->nonce; 141 /* 64 bit IV part */ 142 memcpy(&cdesc->control_data.token[1], iv, 8); 143 /* 32 bit counter, start at 0 or 1 (big endian!) */ 144 cdesc->control_data.token[3] = 145 (__force u32)cpu_to_be32(ctx->ctrinit); 146 return; 147 } 148 if (ctx->xcm == EIP197_XCM_MODE_GCM || ctx->alg == SAFEXCEL_CHACHA20) { 149 /* 96 bit IV part */ 150 memcpy(&cdesc->control_data.token[0], iv, 12); 151 /* 32 bit counter, start at 0 or 1 (big endian!) */ 152 cdesc->control_data.token[3] = 153 (__force u32)cpu_to_be32(ctx->ctrinit); 154 return; 155 } 156 /* CBC */ 157 memcpy(cdesc->control_data.token, iv, ctx->blocksz); 158} 159 160static void safexcel_aead_token(struct safexcel_cipher_ctx *ctx, u8 *iv, 161 struct safexcel_command_desc *cdesc, 162 struct safexcel_token *atoken, 163 enum safexcel_cipher_direction direction, 164 u32 cryptlen, u32 assoclen, u32 digestsize) 165{ 166 struct safexcel_token *aadref; 167 int atoksize = 2; /* Start with minimum size */ 168 int assocadj = assoclen - ctx->aadskip, aadalign; 169 170 /* Always 4 dwords of embedded IV for AEAD modes */ 171 cdesc->control_data.options |= EIP197_OPTION_4_TOKEN_IV_CMD; 172 173 if (direction == SAFEXCEL_DECRYPT) 174 cryptlen -= digestsize; 175 176 if (unlikely(ctx->xcm == EIP197_XCM_MODE_CCM)) { 177 /* Construct IV block B0 for the CBC-MAC */ 178 u8 *final_iv = (u8 *)cdesc->control_data.token; 179 u8 *cbcmaciv = (u8 *)&atoken[1]; 180 __le32 *aadlen = (__le32 *)&atoken[5]; 181 182 if (ctx->aead == EIP197_AEAD_TYPE_IPSEC_ESP) { 183 /* Length + nonce */ 184 cdesc->control_data.token[0] = ctx->nonce; 185 /* Fixup flags byte */ 186 *(__le32 *)cbcmaciv = 187 cpu_to_le32(ctx->nonce | 188 ((assocadj > 0) << 6) | 189 ((digestsize - 2) << 2)); 190 /* 64 bit IV part */ 191 memcpy(&cdesc->control_data.token[1], iv, 8); 192 memcpy(cbcmaciv + 4, iv, 8); 193 /* Start counter at 0 */ 194 cdesc->control_data.token[3] = 0; 195 /* Message length */ 196 *(__be32 *)(cbcmaciv + 12) = cpu_to_be32(cryptlen); 197 } else { 198 /* Variable length IV part */ 199 memcpy(final_iv, iv, 15 - iv[0]); 200 memcpy(cbcmaciv, iv, 15 - iv[0]); 201 /* Start variable length counter at 0 */ 202 memset(final_iv + 15 - iv[0], 0, iv[0] + 1); 203 memset(cbcmaciv + 15 - iv[0], 0, iv[0] - 1); 204 /* fixup flags byte */ 205 cbcmaciv[0] |= ((assocadj > 0) << 6) | 206 ((digestsize - 2) << 2); 207 /* insert lower 2 bytes of message length */ 208 cbcmaciv[14] = cryptlen >> 8; 209 cbcmaciv[15] = cryptlen & 255; 210 } 211 212 atoken->opcode = EIP197_TOKEN_OPCODE_INSERT; 213 atoken->packet_length = AES_BLOCK_SIZE + 214 ((assocadj > 0) << 1); 215 atoken->stat = 0; 216 atoken->instructions = EIP197_TOKEN_INS_ORIGIN_TOKEN | 217 EIP197_TOKEN_INS_TYPE_HASH; 218 219 if (likely(assocadj)) { 220 *aadlen = cpu_to_le32((assocadj >> 8) | 221 (assocadj & 255) << 8); 222 atoken += 6; 223 atoksize += 7; 224 } else { 225 atoken += 5; 226 atoksize += 6; 227 } 228 229 /* Process AAD data */ 230 aadref = atoken; 231 atoken->opcode = EIP197_TOKEN_OPCODE_DIRECTION; 232 atoken->packet_length = assocadj; 233 atoken->stat = 0; 234 atoken->instructions = EIP197_TOKEN_INS_TYPE_HASH; 235 atoken++; 236 237 /* For CCM only, align AAD data towards hash engine */ 238 atoken->opcode = EIP197_TOKEN_OPCODE_INSERT; 239 aadalign = (assocadj + 2) & 15; 240 atoken->packet_length = assocadj && aadalign ? 241 16 - aadalign : 242 0; 243 if (likely(cryptlen)) { 244 atoken->stat = 0; 245 atoken->instructions = EIP197_TOKEN_INS_TYPE_HASH; 246 } else { 247 atoken->stat = EIP197_TOKEN_STAT_LAST_HASH; 248 atoken->instructions = EIP197_TOKEN_INS_LAST | 249 EIP197_TOKEN_INS_TYPE_HASH; 250 } 251 } else { 252 safexcel_aead_iv(ctx, iv, cdesc); 253 254 /* Process AAD data */ 255 aadref = atoken; 256 atoken->opcode = EIP197_TOKEN_OPCODE_DIRECTION; 257 atoken->packet_length = assocadj; 258 atoken->stat = EIP197_TOKEN_STAT_LAST_HASH; 259 atoken->instructions = EIP197_TOKEN_INS_LAST | 260 EIP197_TOKEN_INS_TYPE_HASH; 261 } 262 atoken++; 263 264 if (ctx->aead == EIP197_AEAD_TYPE_IPSEC_ESP) { 265 /* For ESP mode (and not GMAC), skip over the IV */ 266 atoken->opcode = EIP197_TOKEN_OPCODE_DIRECTION; 267 atoken->packet_length = EIP197_AEAD_IPSEC_IV_SIZE; 268 atoken->stat = 0; 269 atoken->instructions = 0; 270 atoken++; 271 atoksize++; 272 } else if (unlikely(ctx->alg == SAFEXCEL_CHACHA20 && 273 direction == SAFEXCEL_DECRYPT)) { 274 /* Poly-chacha decryption needs a dummy NOP here ... */ 275 atoken->opcode = EIP197_TOKEN_OPCODE_INSERT; 276 atoken->packet_length = 16; /* According to Op Manual */ 277 atoken->stat = 0; 278 atoken->instructions = 0; 279 atoken++; 280 atoksize++; 281 } 282 283 if (ctx->xcm) { 284 /* For GCM and CCM, obtain enc(Y0) */ 285 atoken->opcode = EIP197_TOKEN_OPCODE_INSERT_REMRES; 286 atoken->packet_length = 0; 287 atoken->stat = 0; 288 atoken->instructions = AES_BLOCK_SIZE; 289 atoken++; 290 291 atoken->opcode = EIP197_TOKEN_OPCODE_INSERT; 292 atoken->packet_length = AES_BLOCK_SIZE; 293 atoken->stat = 0; 294 atoken->instructions = EIP197_TOKEN_INS_TYPE_OUTPUT | 295 EIP197_TOKEN_INS_TYPE_CRYPTO; 296 atoken++; 297 atoksize += 2; 298 } 299 300 if (likely(cryptlen || ctx->alg == SAFEXCEL_CHACHA20)) { 301 /* Fixup stat field for AAD direction instruction */ 302 aadref->stat = 0; 303 304 /* Process crypto data */ 305 atoken->opcode = EIP197_TOKEN_OPCODE_DIRECTION; 306 atoken->packet_length = cryptlen; 307 308 if (unlikely(ctx->aead == EIP197_AEAD_TYPE_IPSEC_ESP_GMAC)) { 309 /* Fixup instruction field for AAD dir instruction */ 310 aadref->instructions = EIP197_TOKEN_INS_TYPE_HASH; 311 312 /* Do not send to crypt engine in case of GMAC */ 313 atoken->instructions = EIP197_TOKEN_INS_LAST | 314 EIP197_TOKEN_INS_TYPE_HASH | 315 EIP197_TOKEN_INS_TYPE_OUTPUT; 316 } else { 317 atoken->instructions = EIP197_TOKEN_INS_LAST | 318 EIP197_TOKEN_INS_TYPE_CRYPTO | 319 EIP197_TOKEN_INS_TYPE_HASH | 320 EIP197_TOKEN_INS_TYPE_OUTPUT; 321 } 322 323 cryptlen &= 15; 324 if (unlikely(ctx->xcm == EIP197_XCM_MODE_CCM && cryptlen)) { 325 atoken->stat = 0; 326 /* For CCM only, pad crypto data to the hash engine */ 327 atoken++; 328 atoksize++; 329 atoken->opcode = EIP197_TOKEN_OPCODE_INSERT; 330 atoken->packet_length = 16 - cryptlen; 331 atoken->stat = EIP197_TOKEN_STAT_LAST_HASH; 332 atoken->instructions = EIP197_TOKEN_INS_TYPE_HASH; 333 } else { 334 atoken->stat = EIP197_TOKEN_STAT_LAST_HASH; 335 } 336 atoken++; 337 atoksize++; 338 } 339 340 if (direction == SAFEXCEL_ENCRYPT) { 341 /* Append ICV */ 342 atoken->opcode = EIP197_TOKEN_OPCODE_INSERT; 343 atoken->packet_length = digestsize; 344 atoken->stat = EIP197_TOKEN_STAT_LAST_HASH | 345 EIP197_TOKEN_STAT_LAST_PACKET; 346 atoken->instructions = EIP197_TOKEN_INS_TYPE_OUTPUT | 347 EIP197_TOKEN_INS_INSERT_HASH_DIGEST; 348 } else { 349 /* Extract ICV */ 350 atoken->opcode = EIP197_TOKEN_OPCODE_RETRIEVE; 351 atoken->packet_length = digestsize; 352 atoken->stat = EIP197_TOKEN_STAT_LAST_HASH | 353 EIP197_TOKEN_STAT_LAST_PACKET; 354 atoken->instructions = EIP197_TOKEN_INS_INSERT_HASH_DIGEST; 355 atoken++; 356 atoksize++; 357 358 /* Verify ICV */ 359 atoken->opcode = EIP197_TOKEN_OPCODE_VERIFY; 360 atoken->packet_length = digestsize | 361 EIP197_TOKEN_HASH_RESULT_VERIFY; 362 atoken->stat = EIP197_TOKEN_STAT_LAST_HASH | 363 EIP197_TOKEN_STAT_LAST_PACKET; 364 atoken->instructions = EIP197_TOKEN_INS_TYPE_OUTPUT; 365 } 366 367 /* Fixup length of the token in the command descriptor */ 368 cdesc->additional_cdata_size = atoksize; 369} 370 371static int safexcel_skcipher_aes_setkey(struct crypto_skcipher *ctfm, 372 const u8 *key, unsigned int len) 373{ 374 struct crypto_tfm *tfm = crypto_skcipher_tfm(ctfm); 375 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 376 struct safexcel_crypto_priv *priv = ctx->base.priv; 377 struct crypto_aes_ctx aes; 378 int ret, i; 379 380 ret = aes_expandkey(&aes, key, len); 381 if (ret) 382 return ret; 383 384 if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma) { 385 for (i = 0; i < len / sizeof(u32); i++) { 386 if (le32_to_cpu(ctx->key[i]) != aes.key_enc[i]) { 387 ctx->base.needs_inv = true; 388 break; 389 } 390 } 391 } 392 393 for (i = 0; i < len / sizeof(u32); i++) 394 ctx->key[i] = cpu_to_le32(aes.key_enc[i]); 395 396 ctx->key_len = len; 397 398 memzero_explicit(&aes, sizeof(aes)); 399 return 0; 400} 401 402static int safexcel_aead_setkey(struct crypto_aead *ctfm, const u8 *key, 403 unsigned int len) 404{ 405 struct crypto_tfm *tfm = crypto_aead_tfm(ctfm); 406 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 407 struct safexcel_crypto_priv *priv = ctx->base.priv; 408 struct crypto_authenc_keys keys; 409 struct crypto_aes_ctx aes; 410 int err = -EINVAL, i; 411 const char *alg; 412 413 if (unlikely(crypto_authenc_extractkeys(&keys, key, len))) 414 goto badkey; 415 416 if (ctx->mode == CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD) { 417 /* Must have at least space for the nonce here */ 418 if (unlikely(keys.enckeylen < CTR_RFC3686_NONCE_SIZE)) 419 goto badkey; 420 /* last 4 bytes of key are the nonce! */ 421 ctx->nonce = *(u32 *)(keys.enckey + keys.enckeylen - 422 CTR_RFC3686_NONCE_SIZE); 423 /* exclude the nonce here */ 424 keys.enckeylen -= CTR_RFC3686_NONCE_SIZE; 425 } 426 427 /* Encryption key */ 428 switch (ctx->alg) { 429 case SAFEXCEL_DES: 430 err = verify_aead_des_key(ctfm, keys.enckey, keys.enckeylen); 431 if (unlikely(err)) 432 goto badkey; 433 break; 434 case SAFEXCEL_3DES: 435 err = verify_aead_des3_key(ctfm, keys.enckey, keys.enckeylen); 436 if (unlikely(err)) 437 goto badkey; 438 break; 439 case SAFEXCEL_AES: 440 err = aes_expandkey(&aes, keys.enckey, keys.enckeylen); 441 if (unlikely(err)) 442 goto badkey; 443 break; 444 case SAFEXCEL_SM4: 445 if (unlikely(keys.enckeylen != SM4_KEY_SIZE)) 446 goto badkey; 447 break; 448 default: 449 dev_err(priv->dev, "aead: unsupported cipher algorithm\n"); 450 goto badkey; 451 } 452 453 if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma) { 454 for (i = 0; i < keys.enckeylen / sizeof(u32); i++) { 455 if (le32_to_cpu(ctx->key[i]) != 456 ((u32 *)keys.enckey)[i]) { 457 ctx->base.needs_inv = true; 458 break; 459 } 460 } 461 } 462 463 /* Auth key */ 464 switch (ctx->hash_alg) { 465 case CONTEXT_CONTROL_CRYPTO_ALG_SHA1: 466 alg = "safexcel-sha1"; 467 break; 468 case CONTEXT_CONTROL_CRYPTO_ALG_SHA224: 469 alg = "safexcel-sha224"; 470 break; 471 case CONTEXT_CONTROL_CRYPTO_ALG_SHA256: 472 alg = "safexcel-sha256"; 473 break; 474 case CONTEXT_CONTROL_CRYPTO_ALG_SHA384: 475 alg = "safexcel-sha384"; 476 break; 477 case CONTEXT_CONTROL_CRYPTO_ALG_SHA512: 478 alg = "safexcel-sha512"; 479 break; 480 case CONTEXT_CONTROL_CRYPTO_ALG_SM3: 481 alg = "safexcel-sm3"; 482 break; 483 default: 484 dev_err(priv->dev, "aead: unsupported hash algorithm\n"); 485 goto badkey; 486 } 487 488 if (safexcel_hmac_setkey(&ctx->base, keys.authkey, keys.authkeylen, 489 alg, ctx->state_sz)) 490 goto badkey; 491 492 /* Now copy the keys into the context */ 493 for (i = 0; i < keys.enckeylen / sizeof(u32); i++) 494 ctx->key[i] = cpu_to_le32(((u32 *)keys.enckey)[i]); 495 ctx->key_len = keys.enckeylen; 496 497 memzero_explicit(&keys, sizeof(keys)); 498 return 0; 499 500badkey: 501 memzero_explicit(&keys, sizeof(keys)); 502 return err; 503} 504 505static int safexcel_context_control(struct safexcel_cipher_ctx *ctx, 506 struct crypto_async_request *async, 507 struct safexcel_cipher_req *sreq, 508 struct safexcel_command_desc *cdesc) 509{ 510 struct safexcel_crypto_priv *priv = ctx->base.priv; 511 int ctrl_size = ctx->key_len / sizeof(u32); 512 513 cdesc->control_data.control1 = ctx->mode; 514 515 if (ctx->aead) { 516 /* Take in account the ipad+opad digests */ 517 if (ctx->xcm) { 518 ctrl_size += ctx->state_sz / sizeof(u32); 519 cdesc->control_data.control0 = 520 CONTEXT_CONTROL_KEY_EN | 521 CONTEXT_CONTROL_DIGEST_XCM | 522 ctx->hash_alg | 523 CONTEXT_CONTROL_SIZE(ctrl_size); 524 } else if (ctx->alg == SAFEXCEL_CHACHA20) { 525 /* Chacha20-Poly1305 */ 526 cdesc->control_data.control0 = 527 CONTEXT_CONTROL_KEY_EN | 528 CONTEXT_CONTROL_CRYPTO_ALG_CHACHA20 | 529 (sreq->direction == SAFEXCEL_ENCRYPT ? 530 CONTEXT_CONTROL_TYPE_ENCRYPT_HASH_OUT : 531 CONTEXT_CONTROL_TYPE_HASH_DECRYPT_IN) | 532 ctx->hash_alg | 533 CONTEXT_CONTROL_SIZE(ctrl_size); 534 return 0; 535 } else { 536 ctrl_size += ctx->state_sz / sizeof(u32) * 2; 537 cdesc->control_data.control0 = 538 CONTEXT_CONTROL_KEY_EN | 539 CONTEXT_CONTROL_DIGEST_HMAC | 540 ctx->hash_alg | 541 CONTEXT_CONTROL_SIZE(ctrl_size); 542 } 543 544 if (sreq->direction == SAFEXCEL_ENCRYPT && 545 (ctx->xcm == EIP197_XCM_MODE_CCM || 546 ctx->aead == EIP197_AEAD_TYPE_IPSEC_ESP_GMAC)) 547 cdesc->control_data.control0 |= 548 CONTEXT_CONTROL_TYPE_HASH_ENCRYPT_OUT; 549 else if (sreq->direction == SAFEXCEL_ENCRYPT) 550 cdesc->control_data.control0 |= 551 CONTEXT_CONTROL_TYPE_ENCRYPT_HASH_OUT; 552 else if (ctx->xcm == EIP197_XCM_MODE_CCM) 553 cdesc->control_data.control0 |= 554 CONTEXT_CONTROL_TYPE_DECRYPT_HASH_IN; 555 else 556 cdesc->control_data.control0 |= 557 CONTEXT_CONTROL_TYPE_HASH_DECRYPT_IN; 558 } else { 559 if (sreq->direction == SAFEXCEL_ENCRYPT) 560 cdesc->control_data.control0 = 561 CONTEXT_CONTROL_TYPE_CRYPTO_OUT | 562 CONTEXT_CONTROL_KEY_EN | 563 CONTEXT_CONTROL_SIZE(ctrl_size); 564 else 565 cdesc->control_data.control0 = 566 CONTEXT_CONTROL_TYPE_CRYPTO_IN | 567 CONTEXT_CONTROL_KEY_EN | 568 CONTEXT_CONTROL_SIZE(ctrl_size); 569 } 570 571 if (ctx->alg == SAFEXCEL_DES) { 572 cdesc->control_data.control0 |= 573 CONTEXT_CONTROL_CRYPTO_ALG_DES; 574 } else if (ctx->alg == SAFEXCEL_3DES) { 575 cdesc->control_data.control0 |= 576 CONTEXT_CONTROL_CRYPTO_ALG_3DES; 577 } else if (ctx->alg == SAFEXCEL_AES) { 578 switch (ctx->key_len >> ctx->xts) { 579 case AES_KEYSIZE_128: 580 cdesc->control_data.control0 |= 581 CONTEXT_CONTROL_CRYPTO_ALG_AES128; 582 break; 583 case AES_KEYSIZE_192: 584 cdesc->control_data.control0 |= 585 CONTEXT_CONTROL_CRYPTO_ALG_AES192; 586 break; 587 case AES_KEYSIZE_256: 588 cdesc->control_data.control0 |= 589 CONTEXT_CONTROL_CRYPTO_ALG_AES256; 590 break; 591 default: 592 dev_err(priv->dev, "aes keysize not supported: %u\n", 593 ctx->key_len >> ctx->xts); 594 return -EINVAL; 595 } 596 } else if (ctx->alg == SAFEXCEL_CHACHA20) { 597 cdesc->control_data.control0 |= 598 CONTEXT_CONTROL_CRYPTO_ALG_CHACHA20; 599 } else if (ctx->alg == SAFEXCEL_SM4) { 600 cdesc->control_data.control0 |= 601 CONTEXT_CONTROL_CRYPTO_ALG_SM4; 602 } 603 604 return 0; 605} 606 607static int safexcel_handle_req_result(struct safexcel_crypto_priv *priv, int ring, 608 struct crypto_async_request *async, 609 struct scatterlist *src, 610 struct scatterlist *dst, 611 unsigned int cryptlen, 612 struct safexcel_cipher_req *sreq, 613 bool *should_complete, int *ret) 614{ 615 struct skcipher_request *areq = skcipher_request_cast(async); 616 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(areq); 617 struct safexcel_cipher_ctx *ctx = crypto_skcipher_ctx(skcipher); 618 struct safexcel_result_desc *rdesc; 619 int ndesc = 0; 620 621 *ret = 0; 622 623 if (unlikely(!sreq->rdescs)) 624 return 0; 625 626 while (sreq->rdescs--) { 627 rdesc = safexcel_ring_next_rptr(priv, &priv->ring[ring].rdr); 628 if (IS_ERR(rdesc)) { 629 dev_err(priv->dev, 630 "cipher: result: could not retrieve the result descriptor\n"); 631 *ret = PTR_ERR(rdesc); 632 break; 633 } 634 635 if (likely(!*ret)) 636 *ret = safexcel_rdesc_check_errors(priv, rdesc); 637 638 ndesc++; 639 } 640 641 safexcel_complete(priv, ring); 642 643 if (src == dst) { 644 dma_unmap_sg(priv->dev, src, sreq->nr_src, DMA_BIDIRECTIONAL); 645 } else { 646 dma_unmap_sg(priv->dev, src, sreq->nr_src, DMA_TO_DEVICE); 647 dma_unmap_sg(priv->dev, dst, sreq->nr_dst, DMA_FROM_DEVICE); 648 } 649 650 /* 651 * Update IV in req from last crypto output word for CBC modes 652 */ 653 if ((!ctx->aead) && (ctx->mode == CONTEXT_CONTROL_CRYPTO_MODE_CBC) && 654 (sreq->direction == SAFEXCEL_ENCRYPT)) { 655 /* For encrypt take the last output word */ 656 sg_pcopy_to_buffer(dst, sreq->nr_dst, areq->iv, 657 crypto_skcipher_ivsize(skcipher), 658 (cryptlen - 659 crypto_skcipher_ivsize(skcipher))); 660 } 661 662 *should_complete = true; 663 664 return ndesc; 665} 666 667static int safexcel_send_req(struct crypto_async_request *base, int ring, 668 struct safexcel_cipher_req *sreq, 669 struct scatterlist *src, struct scatterlist *dst, 670 unsigned int cryptlen, unsigned int assoclen, 671 unsigned int digestsize, u8 *iv, int *commands, 672 int *results) 673{ 674 struct skcipher_request *areq = skcipher_request_cast(base); 675 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(areq); 676 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(base->tfm); 677 struct safexcel_crypto_priv *priv = ctx->base.priv; 678 struct safexcel_command_desc *cdesc; 679 struct safexcel_command_desc *first_cdesc = NULL; 680 struct safexcel_result_desc *rdesc, *first_rdesc = NULL; 681 struct scatterlist *sg; 682 unsigned int totlen; 683 unsigned int totlen_src = cryptlen + assoclen; 684 unsigned int totlen_dst = totlen_src; 685 struct safexcel_token *atoken; 686 int n_cdesc = 0, n_rdesc = 0; 687 int queued, i, ret = 0; 688 bool first = true; 689 690 sreq->nr_src = sg_nents_for_len(src, totlen_src); 691 692 if (ctx->aead) { 693 /* 694 * AEAD has auth tag appended to output for encrypt and 695 * removed from the output for decrypt! 696 */ 697 if (sreq->direction == SAFEXCEL_DECRYPT) 698 totlen_dst -= digestsize; 699 else 700 totlen_dst += digestsize; 701 702 memcpy(ctx->base.ctxr->data + ctx->key_len / sizeof(u32), 703 &ctx->base.ipad, ctx->state_sz); 704 if (!ctx->xcm) 705 memcpy(ctx->base.ctxr->data + (ctx->key_len + 706 ctx->state_sz) / sizeof(u32), &ctx->base.opad, 707 ctx->state_sz); 708 } else if ((ctx->mode == CONTEXT_CONTROL_CRYPTO_MODE_CBC) && 709 (sreq->direction == SAFEXCEL_DECRYPT)) { 710 /* 711 * Save IV from last crypto input word for CBC modes in decrypt 712 * direction. Need to do this first in case of inplace operation 713 * as it will be overwritten. 714 */ 715 sg_pcopy_to_buffer(src, sreq->nr_src, areq->iv, 716 crypto_skcipher_ivsize(skcipher), 717 (totlen_src - 718 crypto_skcipher_ivsize(skcipher))); 719 } 720 721 sreq->nr_dst = sg_nents_for_len(dst, totlen_dst); 722 723 /* 724 * Remember actual input length, source buffer length may be 725 * updated in case of inline operation below. 726 */ 727 totlen = totlen_src; 728 queued = totlen_src; 729 730 if (src == dst) { 731 sreq->nr_src = max(sreq->nr_src, sreq->nr_dst); 732 sreq->nr_dst = sreq->nr_src; 733 if (unlikely((totlen_src || totlen_dst) && 734 (sreq->nr_src <= 0))) { 735 dev_err(priv->dev, "In-place buffer not large enough (need %d bytes)!", 736 max(totlen_src, totlen_dst)); 737 return -EINVAL; 738 } 739 dma_map_sg(priv->dev, src, sreq->nr_src, DMA_BIDIRECTIONAL); 740 } else { 741 if (unlikely(totlen_src && (sreq->nr_src <= 0))) { 742 dev_err(priv->dev, "Source buffer not large enough (need %d bytes)!", 743 totlen_src); 744 return -EINVAL; 745 } 746 dma_map_sg(priv->dev, src, sreq->nr_src, DMA_TO_DEVICE); 747 748 if (unlikely(totlen_dst && (sreq->nr_dst <= 0))) { 749 dev_err(priv->dev, "Dest buffer not large enough (need %d bytes)!", 750 totlen_dst); 751 dma_unmap_sg(priv->dev, src, sreq->nr_src, 752 DMA_TO_DEVICE); 753 return -EINVAL; 754 } 755 dma_map_sg(priv->dev, dst, sreq->nr_dst, DMA_FROM_DEVICE); 756 } 757 758 memcpy(ctx->base.ctxr->data, ctx->key, ctx->key_len); 759 760 if (!totlen) { 761 /* 762 * The EIP97 cannot deal with zero length input packets! 763 * So stuff a dummy command descriptor indicating a 1 byte 764 * (dummy) input packet, using the context record as source. 765 */ 766 first_cdesc = safexcel_add_cdesc(priv, ring, 767 1, 1, ctx->base.ctxr_dma, 768 1, 1, ctx->base.ctxr_dma, 769 &atoken); 770 if (IS_ERR(first_cdesc)) { 771 /* No space left in the command descriptor ring */ 772 ret = PTR_ERR(first_cdesc); 773 goto cdesc_rollback; 774 } 775 n_cdesc = 1; 776 goto skip_cdesc; 777 } 778 779 /* command descriptors */ 780 for_each_sg(src, sg, sreq->nr_src, i) { 781 int len = sg_dma_len(sg); 782 783 /* Do not overflow the request */ 784 if (queued < len) 785 len = queued; 786 787 cdesc = safexcel_add_cdesc(priv, ring, !n_cdesc, 788 !(queued - len), 789 sg_dma_address(sg), len, totlen, 790 ctx->base.ctxr_dma, &atoken); 791 if (IS_ERR(cdesc)) { 792 /* No space left in the command descriptor ring */ 793 ret = PTR_ERR(cdesc); 794 goto cdesc_rollback; 795 } 796 797 if (!n_cdesc) 798 first_cdesc = cdesc; 799 800 n_cdesc++; 801 queued -= len; 802 if (!queued) 803 break; 804 } 805skip_cdesc: 806 /* Add context control words and token to first command descriptor */ 807 safexcel_context_control(ctx, base, sreq, first_cdesc); 808 if (ctx->aead) 809 safexcel_aead_token(ctx, iv, first_cdesc, atoken, 810 sreq->direction, cryptlen, 811 assoclen, digestsize); 812 else 813 safexcel_skcipher_token(ctx, iv, first_cdesc, atoken, 814 cryptlen); 815 816 /* result descriptors */ 817 for_each_sg(dst, sg, sreq->nr_dst, i) { 818 bool last = (i == sreq->nr_dst - 1); 819 u32 len = sg_dma_len(sg); 820 821 /* only allow the part of the buffer we know we need */ 822 if (len > totlen_dst) 823 len = totlen_dst; 824 if (unlikely(!len)) 825 break; 826 totlen_dst -= len; 827 828 /* skip over AAD space in buffer - not written */ 829 if (assoclen) { 830 if (assoclen >= len) { 831 assoclen -= len; 832 continue; 833 } 834 rdesc = safexcel_add_rdesc(priv, ring, first, last, 835 sg_dma_address(sg) + 836 assoclen, 837 len - assoclen); 838 assoclen = 0; 839 } else { 840 rdesc = safexcel_add_rdesc(priv, ring, first, last, 841 sg_dma_address(sg), 842 len); 843 } 844 if (IS_ERR(rdesc)) { 845 /* No space left in the result descriptor ring */ 846 ret = PTR_ERR(rdesc); 847 goto rdesc_rollback; 848 } 849 if (first) { 850 first_rdesc = rdesc; 851 first = false; 852 } 853 n_rdesc++; 854 } 855 856 if (unlikely(first)) { 857 /* 858 * Special case: AEAD decrypt with only AAD data. 859 * In this case there is NO output data from the engine, 860 * but the engine still needs a result descriptor! 861 * Create a dummy one just for catching the result token. 862 */ 863 rdesc = safexcel_add_rdesc(priv, ring, true, true, 0, 0); 864 if (IS_ERR(rdesc)) { 865 /* No space left in the result descriptor ring */ 866 ret = PTR_ERR(rdesc); 867 goto rdesc_rollback; 868 } 869 first_rdesc = rdesc; 870 n_rdesc = 1; 871 } 872 873 safexcel_rdr_req_set(priv, ring, first_rdesc, base); 874 875 *commands = n_cdesc; 876 *results = n_rdesc; 877 return 0; 878 879rdesc_rollback: 880 for (i = 0; i < n_rdesc; i++) 881 safexcel_ring_rollback_wptr(priv, &priv->ring[ring].rdr); 882cdesc_rollback: 883 for (i = 0; i < n_cdesc; i++) 884 safexcel_ring_rollback_wptr(priv, &priv->ring[ring].cdr); 885 886 if (src == dst) { 887 dma_unmap_sg(priv->dev, src, sreq->nr_src, DMA_BIDIRECTIONAL); 888 } else { 889 dma_unmap_sg(priv->dev, src, sreq->nr_src, DMA_TO_DEVICE); 890 dma_unmap_sg(priv->dev, dst, sreq->nr_dst, DMA_FROM_DEVICE); 891 } 892 893 return ret; 894} 895 896static int safexcel_handle_inv_result(struct safexcel_crypto_priv *priv, 897 int ring, 898 struct crypto_async_request *base, 899 struct safexcel_cipher_req *sreq, 900 bool *should_complete, int *ret) 901{ 902 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(base->tfm); 903 struct safexcel_result_desc *rdesc; 904 int ndesc = 0, enq_ret; 905 906 *ret = 0; 907 908 if (unlikely(!sreq->rdescs)) 909 return 0; 910 911 while (sreq->rdescs--) { 912 rdesc = safexcel_ring_next_rptr(priv, &priv->ring[ring].rdr); 913 if (IS_ERR(rdesc)) { 914 dev_err(priv->dev, 915 "cipher: invalidate: could not retrieve the result descriptor\n"); 916 *ret = PTR_ERR(rdesc); 917 break; 918 } 919 920 if (likely(!*ret)) 921 *ret = safexcel_rdesc_check_errors(priv, rdesc); 922 923 ndesc++; 924 } 925 926 safexcel_complete(priv, ring); 927 928 if (ctx->base.exit_inv) { 929 dma_pool_free(priv->context_pool, ctx->base.ctxr, 930 ctx->base.ctxr_dma); 931 932 *should_complete = true; 933 934 return ndesc; 935 } 936 937 ring = safexcel_select_ring(priv); 938 ctx->base.ring = ring; 939 940 spin_lock_bh(&priv->ring[ring].queue_lock); 941 enq_ret = crypto_enqueue_request(&priv->ring[ring].queue, base); 942 spin_unlock_bh(&priv->ring[ring].queue_lock); 943 944 if (enq_ret != -EINPROGRESS) 945 *ret = enq_ret; 946 947 queue_work(priv->ring[ring].workqueue, 948 &priv->ring[ring].work_data.work); 949 950 *should_complete = false; 951 952 return ndesc; 953} 954 955static int safexcel_skcipher_handle_result(struct safexcel_crypto_priv *priv, 956 int ring, 957 struct crypto_async_request *async, 958 bool *should_complete, int *ret) 959{ 960 struct skcipher_request *req = skcipher_request_cast(async); 961 struct safexcel_cipher_req *sreq = skcipher_request_ctx(req); 962 int err; 963 964 if (sreq->needs_inv) { 965 sreq->needs_inv = false; 966 err = safexcel_handle_inv_result(priv, ring, async, sreq, 967 should_complete, ret); 968 } else { 969 err = safexcel_handle_req_result(priv, ring, async, req->src, 970 req->dst, req->cryptlen, sreq, 971 should_complete, ret); 972 } 973 974 return err; 975} 976 977static int safexcel_aead_handle_result(struct safexcel_crypto_priv *priv, 978 int ring, 979 struct crypto_async_request *async, 980 bool *should_complete, int *ret) 981{ 982 struct aead_request *req = aead_request_cast(async); 983 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 984 struct safexcel_cipher_req *sreq = aead_request_ctx(req); 985 int err; 986 987 if (sreq->needs_inv) { 988 sreq->needs_inv = false; 989 err = safexcel_handle_inv_result(priv, ring, async, sreq, 990 should_complete, ret); 991 } else { 992 err = safexcel_handle_req_result(priv, ring, async, req->src, 993 req->dst, 994 req->cryptlen + crypto_aead_authsize(tfm), 995 sreq, should_complete, ret); 996 } 997 998 return err; 999} 1000 1001static int safexcel_cipher_send_inv(struct crypto_async_request *base, 1002 int ring, int *commands, int *results) 1003{ 1004 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(base->tfm); 1005 struct safexcel_crypto_priv *priv = ctx->base.priv; 1006 int ret; 1007 1008 ret = safexcel_invalidate_cache(base, priv, ctx->base.ctxr_dma, ring); 1009 if (unlikely(ret)) 1010 return ret; 1011 1012 *commands = 1; 1013 *results = 1; 1014 1015 return 0; 1016} 1017 1018static int safexcel_skcipher_send(struct crypto_async_request *async, int ring, 1019 int *commands, int *results) 1020{ 1021 struct skcipher_request *req = skcipher_request_cast(async); 1022 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(req->base.tfm); 1023 struct safexcel_cipher_req *sreq = skcipher_request_ctx(req); 1024 struct safexcel_crypto_priv *priv = ctx->base.priv; 1025 int ret; 1026 1027 BUG_ON(!(priv->flags & EIP197_TRC_CACHE) && sreq->needs_inv); 1028 1029 if (sreq->needs_inv) { 1030 ret = safexcel_cipher_send_inv(async, ring, commands, results); 1031 } else { 1032 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); 1033 u8 input_iv[AES_BLOCK_SIZE]; 1034 1035 /* 1036 * Save input IV in case of CBC decrypt mode 1037 * Will be overwritten with output IV prior to use! 1038 */ 1039 memcpy(input_iv, req->iv, crypto_skcipher_ivsize(skcipher)); 1040 1041 ret = safexcel_send_req(async, ring, sreq, req->src, 1042 req->dst, req->cryptlen, 0, 0, input_iv, 1043 commands, results); 1044 } 1045 1046 sreq->rdescs = *results; 1047 return ret; 1048} 1049 1050static int safexcel_aead_send(struct crypto_async_request *async, int ring, 1051 int *commands, int *results) 1052{ 1053 struct aead_request *req = aead_request_cast(async); 1054 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1055 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(req->base.tfm); 1056 struct safexcel_cipher_req *sreq = aead_request_ctx(req); 1057 struct safexcel_crypto_priv *priv = ctx->base.priv; 1058 int ret; 1059 1060 BUG_ON(!(priv->flags & EIP197_TRC_CACHE) && sreq->needs_inv); 1061 1062 if (sreq->needs_inv) 1063 ret = safexcel_cipher_send_inv(async, ring, commands, results); 1064 else 1065 ret = safexcel_send_req(async, ring, sreq, req->src, req->dst, 1066 req->cryptlen, req->assoclen, 1067 crypto_aead_authsize(tfm), req->iv, 1068 commands, results); 1069 sreq->rdescs = *results; 1070 return ret; 1071} 1072 1073static int safexcel_cipher_exit_inv(struct crypto_tfm *tfm, 1074 struct crypto_async_request *base, 1075 struct safexcel_cipher_req *sreq, 1076 struct safexcel_inv_result *result) 1077{ 1078 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1079 struct safexcel_crypto_priv *priv = ctx->base.priv; 1080 int ring = ctx->base.ring; 1081 1082 init_completion(&result->completion); 1083 1084 ctx = crypto_tfm_ctx(base->tfm); 1085 ctx->base.exit_inv = true; 1086 sreq->needs_inv = true; 1087 1088 spin_lock_bh(&priv->ring[ring].queue_lock); 1089 crypto_enqueue_request(&priv->ring[ring].queue, base); 1090 spin_unlock_bh(&priv->ring[ring].queue_lock); 1091 1092 queue_work(priv->ring[ring].workqueue, 1093 &priv->ring[ring].work_data.work); 1094 1095 wait_for_completion(&result->completion); 1096 1097 if (result->error) { 1098 dev_warn(priv->dev, 1099 "cipher: sync: invalidate: completion error %d\n", 1100 result->error); 1101 return result->error; 1102 } 1103 1104 return 0; 1105} 1106 1107static int safexcel_skcipher_exit_inv(struct crypto_tfm *tfm) 1108{ 1109 EIP197_REQUEST_ON_STACK(req, skcipher, EIP197_SKCIPHER_REQ_SIZE); 1110 struct safexcel_cipher_req *sreq = skcipher_request_ctx(req); 1111 struct safexcel_inv_result result = {}; 1112 1113 memset(req, 0, sizeof(struct skcipher_request)); 1114 1115 skcipher_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG, 1116 safexcel_inv_complete, &result); 1117 skcipher_request_set_tfm(req, __crypto_skcipher_cast(tfm)); 1118 1119 return safexcel_cipher_exit_inv(tfm, &req->base, sreq, &result); 1120} 1121 1122static int safexcel_aead_exit_inv(struct crypto_tfm *tfm) 1123{ 1124 EIP197_REQUEST_ON_STACK(req, aead, EIP197_AEAD_REQ_SIZE); 1125 struct safexcel_cipher_req *sreq = aead_request_ctx(req); 1126 struct safexcel_inv_result result = {}; 1127 1128 memset(req, 0, sizeof(struct aead_request)); 1129 1130 aead_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG, 1131 safexcel_inv_complete, &result); 1132 aead_request_set_tfm(req, __crypto_aead_cast(tfm)); 1133 1134 return safexcel_cipher_exit_inv(tfm, &req->base, sreq, &result); 1135} 1136 1137static int safexcel_queue_req(struct crypto_async_request *base, 1138 struct safexcel_cipher_req *sreq, 1139 enum safexcel_cipher_direction dir) 1140{ 1141 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(base->tfm); 1142 struct safexcel_crypto_priv *priv = ctx->base.priv; 1143 int ret, ring; 1144 1145 sreq->needs_inv = false; 1146 sreq->direction = dir; 1147 1148 if (ctx->base.ctxr) { 1149 if (priv->flags & EIP197_TRC_CACHE && ctx->base.needs_inv) { 1150 sreq->needs_inv = true; 1151 ctx->base.needs_inv = false; 1152 } 1153 } else { 1154 ctx->base.ring = safexcel_select_ring(priv); 1155 ctx->base.ctxr = dma_pool_zalloc(priv->context_pool, 1156 EIP197_GFP_FLAGS(*base), 1157 &ctx->base.ctxr_dma); 1158 if (!ctx->base.ctxr) 1159 return -ENOMEM; 1160 } 1161 1162 ring = ctx->base.ring; 1163 1164 spin_lock_bh(&priv->ring[ring].queue_lock); 1165 ret = crypto_enqueue_request(&priv->ring[ring].queue, base); 1166 spin_unlock_bh(&priv->ring[ring].queue_lock); 1167 1168 queue_work(priv->ring[ring].workqueue, 1169 &priv->ring[ring].work_data.work); 1170 1171 return ret; 1172} 1173 1174static int safexcel_encrypt(struct skcipher_request *req) 1175{ 1176 return safexcel_queue_req(&req->base, skcipher_request_ctx(req), 1177 SAFEXCEL_ENCRYPT); 1178} 1179 1180static int safexcel_decrypt(struct skcipher_request *req) 1181{ 1182 return safexcel_queue_req(&req->base, skcipher_request_ctx(req), 1183 SAFEXCEL_DECRYPT); 1184} 1185 1186static int safexcel_skcipher_cra_init(struct crypto_tfm *tfm) 1187{ 1188 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1189 struct safexcel_alg_template *tmpl = 1190 container_of(tfm->__crt_alg, struct safexcel_alg_template, 1191 alg.skcipher.base); 1192 1193 crypto_skcipher_set_reqsize(__crypto_skcipher_cast(tfm), 1194 sizeof(struct safexcel_cipher_req)); 1195 1196 ctx->base.priv = tmpl->priv; 1197 1198 ctx->base.send = safexcel_skcipher_send; 1199 ctx->base.handle_result = safexcel_skcipher_handle_result; 1200 ctx->ivmask = EIP197_OPTION_4_TOKEN_IV_CMD; 1201 ctx->ctrinit = 1; 1202 return 0; 1203} 1204 1205static int safexcel_cipher_cra_exit(struct crypto_tfm *tfm) 1206{ 1207 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1208 1209 memzero_explicit(ctx->key, sizeof(ctx->key)); 1210 1211 /* context not allocated, skip invalidation */ 1212 if (!ctx->base.ctxr) 1213 return -ENOMEM; 1214 1215 memzero_explicit(ctx->base.ctxr->data, sizeof(ctx->base.ctxr->data)); 1216 return 0; 1217} 1218 1219static void safexcel_skcipher_cra_exit(struct crypto_tfm *tfm) 1220{ 1221 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1222 struct safexcel_crypto_priv *priv = ctx->base.priv; 1223 int ret; 1224 1225 if (safexcel_cipher_cra_exit(tfm)) 1226 return; 1227 1228 if (priv->flags & EIP197_TRC_CACHE) { 1229 ret = safexcel_skcipher_exit_inv(tfm); 1230 if (ret) 1231 dev_warn(priv->dev, "skcipher: invalidation error %d\n", 1232 ret); 1233 } else { 1234 dma_pool_free(priv->context_pool, ctx->base.ctxr, 1235 ctx->base.ctxr_dma); 1236 } 1237} 1238 1239static void safexcel_aead_cra_exit(struct crypto_tfm *tfm) 1240{ 1241 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1242 struct safexcel_crypto_priv *priv = ctx->base.priv; 1243 int ret; 1244 1245 if (safexcel_cipher_cra_exit(tfm)) 1246 return; 1247 1248 if (priv->flags & EIP197_TRC_CACHE) { 1249 ret = safexcel_aead_exit_inv(tfm); 1250 if (ret) 1251 dev_warn(priv->dev, "aead: invalidation error %d\n", 1252 ret); 1253 } else { 1254 dma_pool_free(priv->context_pool, ctx->base.ctxr, 1255 ctx->base.ctxr_dma); 1256 } 1257} 1258 1259static int safexcel_skcipher_aes_ecb_cra_init(struct crypto_tfm *tfm) 1260{ 1261 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1262 1263 safexcel_skcipher_cra_init(tfm); 1264 ctx->alg = SAFEXCEL_AES; 1265 ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_ECB; 1266 ctx->blocksz = 0; 1267 ctx->ivmask = EIP197_OPTION_2_TOKEN_IV_CMD; 1268 return 0; 1269} 1270 1271struct safexcel_alg_template safexcel_alg_ecb_aes = { 1272 .type = SAFEXCEL_ALG_TYPE_SKCIPHER, 1273 .algo_mask = SAFEXCEL_ALG_AES, 1274 .alg.skcipher = { 1275 .setkey = safexcel_skcipher_aes_setkey, 1276 .encrypt = safexcel_encrypt, 1277 .decrypt = safexcel_decrypt, 1278 .min_keysize = AES_MIN_KEY_SIZE, 1279 .max_keysize = AES_MAX_KEY_SIZE, 1280 .base = { 1281 .cra_name = "ecb(aes)", 1282 .cra_driver_name = "safexcel-ecb-aes", 1283 .cra_priority = SAFEXCEL_CRA_PRIORITY, 1284 .cra_flags = CRYPTO_ALG_ASYNC | 1285 CRYPTO_ALG_ALLOCATES_MEMORY | 1286 CRYPTO_ALG_KERN_DRIVER_ONLY, 1287 .cra_blocksize = AES_BLOCK_SIZE, 1288 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 1289 .cra_alignmask = 0, 1290 .cra_init = safexcel_skcipher_aes_ecb_cra_init, 1291 .cra_exit = safexcel_skcipher_cra_exit, 1292 .cra_module = THIS_MODULE, 1293 }, 1294 }, 1295}; 1296 1297static int safexcel_skcipher_aes_cbc_cra_init(struct crypto_tfm *tfm) 1298{ 1299 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1300 1301 safexcel_skcipher_cra_init(tfm); 1302 ctx->alg = SAFEXCEL_AES; 1303 ctx->blocksz = AES_BLOCK_SIZE; 1304 ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CBC; 1305 return 0; 1306} 1307 1308struct safexcel_alg_template safexcel_alg_cbc_aes = { 1309 .type = SAFEXCEL_ALG_TYPE_SKCIPHER, 1310 .algo_mask = SAFEXCEL_ALG_AES, 1311 .alg.skcipher = { 1312 .setkey = safexcel_skcipher_aes_setkey, 1313 .encrypt = safexcel_encrypt, 1314 .decrypt = safexcel_decrypt, 1315 .min_keysize = AES_MIN_KEY_SIZE, 1316 .max_keysize = AES_MAX_KEY_SIZE, 1317 .ivsize = AES_BLOCK_SIZE, 1318 .base = { 1319 .cra_name = "cbc(aes)", 1320 .cra_driver_name = "safexcel-cbc-aes", 1321 .cra_priority = SAFEXCEL_CRA_PRIORITY, 1322 .cra_flags = CRYPTO_ALG_ASYNC | 1323 CRYPTO_ALG_ALLOCATES_MEMORY | 1324 CRYPTO_ALG_KERN_DRIVER_ONLY, 1325 .cra_blocksize = AES_BLOCK_SIZE, 1326 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 1327 .cra_alignmask = 0, 1328 .cra_init = safexcel_skcipher_aes_cbc_cra_init, 1329 .cra_exit = safexcel_skcipher_cra_exit, 1330 .cra_module = THIS_MODULE, 1331 }, 1332 }, 1333}; 1334 1335static int safexcel_skcipher_aes_cfb_cra_init(struct crypto_tfm *tfm) 1336{ 1337 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1338 1339 safexcel_skcipher_cra_init(tfm); 1340 ctx->alg = SAFEXCEL_AES; 1341 ctx->blocksz = AES_BLOCK_SIZE; 1342 ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CFB; 1343 return 0; 1344} 1345 1346struct safexcel_alg_template safexcel_alg_cfb_aes = { 1347 .type = SAFEXCEL_ALG_TYPE_SKCIPHER, 1348 .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_AES_XFB, 1349 .alg.skcipher = { 1350 .setkey = safexcel_skcipher_aes_setkey, 1351 .encrypt = safexcel_encrypt, 1352 .decrypt = safexcel_decrypt, 1353 .min_keysize = AES_MIN_KEY_SIZE, 1354 .max_keysize = AES_MAX_KEY_SIZE, 1355 .ivsize = AES_BLOCK_SIZE, 1356 .base = { 1357 .cra_name = "cfb(aes)", 1358 .cra_driver_name = "safexcel-cfb-aes", 1359 .cra_priority = SAFEXCEL_CRA_PRIORITY, 1360 .cra_flags = CRYPTO_ALG_ASYNC | 1361 CRYPTO_ALG_ALLOCATES_MEMORY | 1362 CRYPTO_ALG_KERN_DRIVER_ONLY, 1363 .cra_blocksize = 1, 1364 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 1365 .cra_alignmask = 0, 1366 .cra_init = safexcel_skcipher_aes_cfb_cra_init, 1367 .cra_exit = safexcel_skcipher_cra_exit, 1368 .cra_module = THIS_MODULE, 1369 }, 1370 }, 1371}; 1372 1373static int safexcel_skcipher_aes_ofb_cra_init(struct crypto_tfm *tfm) 1374{ 1375 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1376 1377 safexcel_skcipher_cra_init(tfm); 1378 ctx->alg = SAFEXCEL_AES; 1379 ctx->blocksz = AES_BLOCK_SIZE; 1380 ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_OFB; 1381 return 0; 1382} 1383 1384struct safexcel_alg_template safexcel_alg_ofb_aes = { 1385 .type = SAFEXCEL_ALG_TYPE_SKCIPHER, 1386 .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_AES_XFB, 1387 .alg.skcipher = { 1388 .setkey = safexcel_skcipher_aes_setkey, 1389 .encrypt = safexcel_encrypt, 1390 .decrypt = safexcel_decrypt, 1391 .min_keysize = AES_MIN_KEY_SIZE, 1392 .max_keysize = AES_MAX_KEY_SIZE, 1393 .ivsize = AES_BLOCK_SIZE, 1394 .base = { 1395 .cra_name = "ofb(aes)", 1396 .cra_driver_name = "safexcel-ofb-aes", 1397 .cra_priority = SAFEXCEL_CRA_PRIORITY, 1398 .cra_flags = CRYPTO_ALG_ASYNC | 1399 CRYPTO_ALG_ALLOCATES_MEMORY | 1400 CRYPTO_ALG_KERN_DRIVER_ONLY, 1401 .cra_blocksize = 1, 1402 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 1403 .cra_alignmask = 0, 1404 .cra_init = safexcel_skcipher_aes_ofb_cra_init, 1405 .cra_exit = safexcel_skcipher_cra_exit, 1406 .cra_module = THIS_MODULE, 1407 }, 1408 }, 1409}; 1410 1411static int safexcel_skcipher_aesctr_setkey(struct crypto_skcipher *ctfm, 1412 const u8 *key, unsigned int len) 1413{ 1414 struct crypto_tfm *tfm = crypto_skcipher_tfm(ctfm); 1415 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1416 struct safexcel_crypto_priv *priv = ctx->base.priv; 1417 struct crypto_aes_ctx aes; 1418 int ret, i; 1419 unsigned int keylen; 1420 1421 /* last 4 bytes of key are the nonce! */ 1422 ctx->nonce = *(u32 *)(key + len - CTR_RFC3686_NONCE_SIZE); 1423 /* exclude the nonce here */ 1424 keylen = len - CTR_RFC3686_NONCE_SIZE; 1425 ret = aes_expandkey(&aes, key, keylen); 1426 if (ret) 1427 return ret; 1428 1429 if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma) { 1430 for (i = 0; i < keylen / sizeof(u32); i++) { 1431 if (le32_to_cpu(ctx->key[i]) != aes.key_enc[i]) { 1432 ctx->base.needs_inv = true; 1433 break; 1434 } 1435 } 1436 } 1437 1438 for (i = 0; i < keylen / sizeof(u32); i++) 1439 ctx->key[i] = cpu_to_le32(aes.key_enc[i]); 1440 1441 ctx->key_len = keylen; 1442 1443 memzero_explicit(&aes, sizeof(aes)); 1444 return 0; 1445} 1446 1447static int safexcel_skcipher_aes_ctr_cra_init(struct crypto_tfm *tfm) 1448{ 1449 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1450 1451 safexcel_skcipher_cra_init(tfm); 1452 ctx->alg = SAFEXCEL_AES; 1453 ctx->blocksz = AES_BLOCK_SIZE; 1454 ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD; 1455 return 0; 1456} 1457 1458struct safexcel_alg_template safexcel_alg_ctr_aes = { 1459 .type = SAFEXCEL_ALG_TYPE_SKCIPHER, 1460 .algo_mask = SAFEXCEL_ALG_AES, 1461 .alg.skcipher = { 1462 .setkey = safexcel_skcipher_aesctr_setkey, 1463 .encrypt = safexcel_encrypt, 1464 .decrypt = safexcel_decrypt, 1465 /* Add nonce size */ 1466 .min_keysize = AES_MIN_KEY_SIZE + CTR_RFC3686_NONCE_SIZE, 1467 .max_keysize = AES_MAX_KEY_SIZE + CTR_RFC3686_NONCE_SIZE, 1468 .ivsize = CTR_RFC3686_IV_SIZE, 1469 .base = { 1470 .cra_name = "rfc3686(ctr(aes))", 1471 .cra_driver_name = "safexcel-ctr-aes", 1472 .cra_priority = SAFEXCEL_CRA_PRIORITY, 1473 .cra_flags = CRYPTO_ALG_ASYNC | 1474 CRYPTO_ALG_ALLOCATES_MEMORY | 1475 CRYPTO_ALG_KERN_DRIVER_ONLY, 1476 .cra_blocksize = 1, 1477 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 1478 .cra_alignmask = 0, 1479 .cra_init = safexcel_skcipher_aes_ctr_cra_init, 1480 .cra_exit = safexcel_skcipher_cra_exit, 1481 .cra_module = THIS_MODULE, 1482 }, 1483 }, 1484}; 1485 1486static int safexcel_des_setkey(struct crypto_skcipher *ctfm, const u8 *key, 1487 unsigned int len) 1488{ 1489 struct safexcel_cipher_ctx *ctx = crypto_skcipher_ctx(ctfm); 1490 struct safexcel_crypto_priv *priv = ctx->base.priv; 1491 int ret; 1492 1493 ret = verify_skcipher_des_key(ctfm, key); 1494 if (ret) 1495 return ret; 1496 1497 /* if context exits and key changed, need to invalidate it */ 1498 if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma) 1499 if (memcmp(ctx->key, key, len)) 1500 ctx->base.needs_inv = true; 1501 1502 memcpy(ctx->key, key, len); 1503 ctx->key_len = len; 1504 1505 return 0; 1506} 1507 1508static int safexcel_skcipher_des_cbc_cra_init(struct crypto_tfm *tfm) 1509{ 1510 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1511 1512 safexcel_skcipher_cra_init(tfm); 1513 ctx->alg = SAFEXCEL_DES; 1514 ctx->blocksz = DES_BLOCK_SIZE; 1515 ctx->ivmask = EIP197_OPTION_2_TOKEN_IV_CMD; 1516 ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CBC; 1517 return 0; 1518} 1519 1520struct safexcel_alg_template safexcel_alg_cbc_des = { 1521 .type = SAFEXCEL_ALG_TYPE_SKCIPHER, 1522 .algo_mask = SAFEXCEL_ALG_DES, 1523 .alg.skcipher = { 1524 .setkey = safexcel_des_setkey, 1525 .encrypt = safexcel_encrypt, 1526 .decrypt = safexcel_decrypt, 1527 .min_keysize = DES_KEY_SIZE, 1528 .max_keysize = DES_KEY_SIZE, 1529 .ivsize = DES_BLOCK_SIZE, 1530 .base = { 1531 .cra_name = "cbc(des)", 1532 .cra_driver_name = "safexcel-cbc-des", 1533 .cra_priority = SAFEXCEL_CRA_PRIORITY, 1534 .cra_flags = CRYPTO_ALG_ASYNC | 1535 CRYPTO_ALG_ALLOCATES_MEMORY | 1536 CRYPTO_ALG_KERN_DRIVER_ONLY, 1537 .cra_blocksize = DES_BLOCK_SIZE, 1538 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 1539 .cra_alignmask = 0, 1540 .cra_init = safexcel_skcipher_des_cbc_cra_init, 1541 .cra_exit = safexcel_skcipher_cra_exit, 1542 .cra_module = THIS_MODULE, 1543 }, 1544 }, 1545}; 1546 1547static int safexcel_skcipher_des_ecb_cra_init(struct crypto_tfm *tfm) 1548{ 1549 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1550 1551 safexcel_skcipher_cra_init(tfm); 1552 ctx->alg = SAFEXCEL_DES; 1553 ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_ECB; 1554 ctx->blocksz = 0; 1555 ctx->ivmask = EIP197_OPTION_2_TOKEN_IV_CMD; 1556 return 0; 1557} 1558 1559struct safexcel_alg_template safexcel_alg_ecb_des = { 1560 .type = SAFEXCEL_ALG_TYPE_SKCIPHER, 1561 .algo_mask = SAFEXCEL_ALG_DES, 1562 .alg.skcipher = { 1563 .setkey = safexcel_des_setkey, 1564 .encrypt = safexcel_encrypt, 1565 .decrypt = safexcel_decrypt, 1566 .min_keysize = DES_KEY_SIZE, 1567 .max_keysize = DES_KEY_SIZE, 1568 .base = { 1569 .cra_name = "ecb(des)", 1570 .cra_driver_name = "safexcel-ecb-des", 1571 .cra_priority = SAFEXCEL_CRA_PRIORITY, 1572 .cra_flags = CRYPTO_ALG_ASYNC | 1573 CRYPTO_ALG_ALLOCATES_MEMORY | 1574 CRYPTO_ALG_KERN_DRIVER_ONLY, 1575 .cra_blocksize = DES_BLOCK_SIZE, 1576 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 1577 .cra_alignmask = 0, 1578 .cra_init = safexcel_skcipher_des_ecb_cra_init, 1579 .cra_exit = safexcel_skcipher_cra_exit, 1580 .cra_module = THIS_MODULE, 1581 }, 1582 }, 1583}; 1584 1585static int safexcel_des3_ede_setkey(struct crypto_skcipher *ctfm, 1586 const u8 *key, unsigned int len) 1587{ 1588 struct safexcel_cipher_ctx *ctx = crypto_skcipher_ctx(ctfm); 1589 struct safexcel_crypto_priv *priv = ctx->base.priv; 1590 int err; 1591 1592 err = verify_skcipher_des3_key(ctfm, key); 1593 if (err) 1594 return err; 1595 1596 /* if context exits and key changed, need to invalidate it */ 1597 if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma) 1598 if (memcmp(ctx->key, key, len)) 1599 ctx->base.needs_inv = true; 1600 1601 memcpy(ctx->key, key, len); 1602 ctx->key_len = len; 1603 1604 return 0; 1605} 1606 1607static int safexcel_skcipher_des3_cbc_cra_init(struct crypto_tfm *tfm) 1608{ 1609 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1610 1611 safexcel_skcipher_cra_init(tfm); 1612 ctx->alg = SAFEXCEL_3DES; 1613 ctx->blocksz = DES3_EDE_BLOCK_SIZE; 1614 ctx->ivmask = EIP197_OPTION_2_TOKEN_IV_CMD; 1615 ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CBC; 1616 return 0; 1617} 1618 1619struct safexcel_alg_template safexcel_alg_cbc_des3_ede = { 1620 .type = SAFEXCEL_ALG_TYPE_SKCIPHER, 1621 .algo_mask = SAFEXCEL_ALG_DES, 1622 .alg.skcipher = { 1623 .setkey = safexcel_des3_ede_setkey, 1624 .encrypt = safexcel_encrypt, 1625 .decrypt = safexcel_decrypt, 1626 .min_keysize = DES3_EDE_KEY_SIZE, 1627 .max_keysize = DES3_EDE_KEY_SIZE, 1628 .ivsize = DES3_EDE_BLOCK_SIZE, 1629 .base = { 1630 .cra_name = "cbc(des3_ede)", 1631 .cra_driver_name = "safexcel-cbc-des3_ede", 1632 .cra_priority = SAFEXCEL_CRA_PRIORITY, 1633 .cra_flags = CRYPTO_ALG_ASYNC | 1634 CRYPTO_ALG_ALLOCATES_MEMORY | 1635 CRYPTO_ALG_KERN_DRIVER_ONLY, 1636 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 1637 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 1638 .cra_alignmask = 0, 1639 .cra_init = safexcel_skcipher_des3_cbc_cra_init, 1640 .cra_exit = safexcel_skcipher_cra_exit, 1641 .cra_module = THIS_MODULE, 1642 }, 1643 }, 1644}; 1645 1646static int safexcel_skcipher_des3_ecb_cra_init(struct crypto_tfm *tfm) 1647{ 1648 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1649 1650 safexcel_skcipher_cra_init(tfm); 1651 ctx->alg = SAFEXCEL_3DES; 1652 ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_ECB; 1653 ctx->blocksz = 0; 1654 ctx->ivmask = EIP197_OPTION_2_TOKEN_IV_CMD; 1655 return 0; 1656} 1657 1658struct safexcel_alg_template safexcel_alg_ecb_des3_ede = { 1659 .type = SAFEXCEL_ALG_TYPE_SKCIPHER, 1660 .algo_mask = SAFEXCEL_ALG_DES, 1661 .alg.skcipher = { 1662 .setkey = safexcel_des3_ede_setkey, 1663 .encrypt = safexcel_encrypt, 1664 .decrypt = safexcel_decrypt, 1665 .min_keysize = DES3_EDE_KEY_SIZE, 1666 .max_keysize = DES3_EDE_KEY_SIZE, 1667 .base = { 1668 .cra_name = "ecb(des3_ede)", 1669 .cra_driver_name = "safexcel-ecb-des3_ede", 1670 .cra_priority = SAFEXCEL_CRA_PRIORITY, 1671 .cra_flags = CRYPTO_ALG_ASYNC | 1672 CRYPTO_ALG_ALLOCATES_MEMORY | 1673 CRYPTO_ALG_KERN_DRIVER_ONLY, 1674 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 1675 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 1676 .cra_alignmask = 0, 1677 .cra_init = safexcel_skcipher_des3_ecb_cra_init, 1678 .cra_exit = safexcel_skcipher_cra_exit, 1679 .cra_module = THIS_MODULE, 1680 }, 1681 }, 1682}; 1683 1684static int safexcel_aead_encrypt(struct aead_request *req) 1685{ 1686 struct safexcel_cipher_req *creq = aead_request_ctx(req); 1687 1688 return safexcel_queue_req(&req->base, creq, SAFEXCEL_ENCRYPT); 1689} 1690 1691static int safexcel_aead_decrypt(struct aead_request *req) 1692{ 1693 struct safexcel_cipher_req *creq = aead_request_ctx(req); 1694 1695 return safexcel_queue_req(&req->base, creq, SAFEXCEL_DECRYPT); 1696} 1697 1698static int safexcel_aead_cra_init(struct crypto_tfm *tfm) 1699{ 1700 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1701 struct safexcel_alg_template *tmpl = 1702 container_of(tfm->__crt_alg, struct safexcel_alg_template, 1703 alg.aead.base); 1704 1705 crypto_aead_set_reqsize(__crypto_aead_cast(tfm), 1706 sizeof(struct safexcel_cipher_req)); 1707 1708 ctx->base.priv = tmpl->priv; 1709 1710 ctx->alg = SAFEXCEL_AES; /* default */ 1711 ctx->blocksz = AES_BLOCK_SIZE; 1712 ctx->ivmask = EIP197_OPTION_4_TOKEN_IV_CMD; 1713 ctx->ctrinit = 1; 1714 ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CBC; /* default */ 1715 ctx->aead = true; 1716 ctx->base.send = safexcel_aead_send; 1717 ctx->base.handle_result = safexcel_aead_handle_result; 1718 return 0; 1719} 1720 1721static int safexcel_aead_sha1_cra_init(struct crypto_tfm *tfm) 1722{ 1723 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1724 1725 safexcel_aead_cra_init(tfm); 1726 ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA1; 1727 ctx->state_sz = SHA1_DIGEST_SIZE; 1728 return 0; 1729} 1730 1731struct safexcel_alg_template safexcel_alg_authenc_hmac_sha1_cbc_aes = { 1732 .type = SAFEXCEL_ALG_TYPE_AEAD, 1733 .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_SHA1, 1734 .alg.aead = { 1735 .setkey = safexcel_aead_setkey, 1736 .encrypt = safexcel_aead_encrypt, 1737 .decrypt = safexcel_aead_decrypt, 1738 .ivsize = AES_BLOCK_SIZE, 1739 .maxauthsize = SHA1_DIGEST_SIZE, 1740 .base = { 1741 .cra_name = "authenc(hmac(sha1),cbc(aes))", 1742 .cra_driver_name = "safexcel-authenc-hmac-sha1-cbc-aes", 1743 .cra_priority = SAFEXCEL_CRA_PRIORITY, 1744 .cra_flags = CRYPTO_ALG_ASYNC | 1745 CRYPTO_ALG_ALLOCATES_MEMORY | 1746 CRYPTO_ALG_KERN_DRIVER_ONLY, 1747 .cra_blocksize = AES_BLOCK_SIZE, 1748 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 1749 .cra_alignmask = 0, 1750 .cra_init = safexcel_aead_sha1_cra_init, 1751 .cra_exit = safexcel_aead_cra_exit, 1752 .cra_module = THIS_MODULE, 1753 }, 1754 }, 1755}; 1756 1757static int safexcel_aead_sha256_cra_init(struct crypto_tfm *tfm) 1758{ 1759 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1760 1761 safexcel_aead_cra_init(tfm); 1762 ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA256; 1763 ctx->state_sz = SHA256_DIGEST_SIZE; 1764 return 0; 1765} 1766 1767struct safexcel_alg_template safexcel_alg_authenc_hmac_sha256_cbc_aes = { 1768 .type = SAFEXCEL_ALG_TYPE_AEAD, 1769 .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_SHA2_256, 1770 .alg.aead = { 1771 .setkey = safexcel_aead_setkey, 1772 .encrypt = safexcel_aead_encrypt, 1773 .decrypt = safexcel_aead_decrypt, 1774 .ivsize = AES_BLOCK_SIZE, 1775 .maxauthsize = SHA256_DIGEST_SIZE, 1776 .base = { 1777 .cra_name = "authenc(hmac(sha256),cbc(aes))", 1778 .cra_driver_name = "safexcel-authenc-hmac-sha256-cbc-aes", 1779 .cra_priority = SAFEXCEL_CRA_PRIORITY, 1780 .cra_flags = CRYPTO_ALG_ASYNC | 1781 CRYPTO_ALG_ALLOCATES_MEMORY | 1782 CRYPTO_ALG_KERN_DRIVER_ONLY, 1783 .cra_blocksize = AES_BLOCK_SIZE, 1784 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 1785 .cra_alignmask = 0, 1786 .cra_init = safexcel_aead_sha256_cra_init, 1787 .cra_exit = safexcel_aead_cra_exit, 1788 .cra_module = THIS_MODULE, 1789 }, 1790 }, 1791}; 1792 1793static int safexcel_aead_sha224_cra_init(struct crypto_tfm *tfm) 1794{ 1795 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1796 1797 safexcel_aead_cra_init(tfm); 1798 ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA224; 1799 ctx->state_sz = SHA256_DIGEST_SIZE; 1800 return 0; 1801} 1802 1803struct safexcel_alg_template safexcel_alg_authenc_hmac_sha224_cbc_aes = { 1804 .type = SAFEXCEL_ALG_TYPE_AEAD, 1805 .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_SHA2_256, 1806 .alg.aead = { 1807 .setkey = safexcel_aead_setkey, 1808 .encrypt = safexcel_aead_encrypt, 1809 .decrypt = safexcel_aead_decrypt, 1810 .ivsize = AES_BLOCK_SIZE, 1811 .maxauthsize = SHA224_DIGEST_SIZE, 1812 .base = { 1813 .cra_name = "authenc(hmac(sha224),cbc(aes))", 1814 .cra_driver_name = "safexcel-authenc-hmac-sha224-cbc-aes", 1815 .cra_priority = SAFEXCEL_CRA_PRIORITY, 1816 .cra_flags = CRYPTO_ALG_ASYNC | 1817 CRYPTO_ALG_ALLOCATES_MEMORY | 1818 CRYPTO_ALG_KERN_DRIVER_ONLY, 1819 .cra_blocksize = AES_BLOCK_SIZE, 1820 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 1821 .cra_alignmask = 0, 1822 .cra_init = safexcel_aead_sha224_cra_init, 1823 .cra_exit = safexcel_aead_cra_exit, 1824 .cra_module = THIS_MODULE, 1825 }, 1826 }, 1827}; 1828 1829static int safexcel_aead_sha512_cra_init(struct crypto_tfm *tfm) 1830{ 1831 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1832 1833 safexcel_aead_cra_init(tfm); 1834 ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA512; 1835 ctx->state_sz = SHA512_DIGEST_SIZE; 1836 return 0; 1837} 1838 1839struct safexcel_alg_template safexcel_alg_authenc_hmac_sha512_cbc_aes = { 1840 .type = SAFEXCEL_ALG_TYPE_AEAD, 1841 .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_SHA2_512, 1842 .alg.aead = { 1843 .setkey = safexcel_aead_setkey, 1844 .encrypt = safexcel_aead_encrypt, 1845 .decrypt = safexcel_aead_decrypt, 1846 .ivsize = AES_BLOCK_SIZE, 1847 .maxauthsize = SHA512_DIGEST_SIZE, 1848 .base = { 1849 .cra_name = "authenc(hmac(sha512),cbc(aes))", 1850 .cra_driver_name = "safexcel-authenc-hmac-sha512-cbc-aes", 1851 .cra_priority = SAFEXCEL_CRA_PRIORITY, 1852 .cra_flags = CRYPTO_ALG_ASYNC | 1853 CRYPTO_ALG_ALLOCATES_MEMORY | 1854 CRYPTO_ALG_KERN_DRIVER_ONLY, 1855 .cra_blocksize = AES_BLOCK_SIZE, 1856 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 1857 .cra_alignmask = 0, 1858 .cra_init = safexcel_aead_sha512_cra_init, 1859 .cra_exit = safexcel_aead_cra_exit, 1860 .cra_module = THIS_MODULE, 1861 }, 1862 }, 1863}; 1864 1865static int safexcel_aead_sha384_cra_init(struct crypto_tfm *tfm) 1866{ 1867 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1868 1869 safexcel_aead_cra_init(tfm); 1870 ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA384; 1871 ctx->state_sz = SHA512_DIGEST_SIZE; 1872 return 0; 1873} 1874 1875struct safexcel_alg_template safexcel_alg_authenc_hmac_sha384_cbc_aes = { 1876 .type = SAFEXCEL_ALG_TYPE_AEAD, 1877 .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_SHA2_512, 1878 .alg.aead = { 1879 .setkey = safexcel_aead_setkey, 1880 .encrypt = safexcel_aead_encrypt, 1881 .decrypt = safexcel_aead_decrypt, 1882 .ivsize = AES_BLOCK_SIZE, 1883 .maxauthsize = SHA384_DIGEST_SIZE, 1884 .base = { 1885 .cra_name = "authenc(hmac(sha384),cbc(aes))", 1886 .cra_driver_name = "safexcel-authenc-hmac-sha384-cbc-aes", 1887 .cra_priority = SAFEXCEL_CRA_PRIORITY, 1888 .cra_flags = CRYPTO_ALG_ASYNC | 1889 CRYPTO_ALG_ALLOCATES_MEMORY | 1890 CRYPTO_ALG_KERN_DRIVER_ONLY, 1891 .cra_blocksize = AES_BLOCK_SIZE, 1892 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 1893 .cra_alignmask = 0, 1894 .cra_init = safexcel_aead_sha384_cra_init, 1895 .cra_exit = safexcel_aead_cra_exit, 1896 .cra_module = THIS_MODULE, 1897 }, 1898 }, 1899}; 1900 1901static int safexcel_aead_sha1_des3_cra_init(struct crypto_tfm *tfm) 1902{ 1903 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1904 1905 safexcel_aead_sha1_cra_init(tfm); 1906 ctx->alg = SAFEXCEL_3DES; /* override default */ 1907 ctx->blocksz = DES3_EDE_BLOCK_SIZE; 1908 ctx->ivmask = EIP197_OPTION_2_TOKEN_IV_CMD; 1909 return 0; 1910} 1911 1912struct safexcel_alg_template safexcel_alg_authenc_hmac_sha1_cbc_des3_ede = { 1913 .type = SAFEXCEL_ALG_TYPE_AEAD, 1914 .algo_mask = SAFEXCEL_ALG_DES | SAFEXCEL_ALG_SHA1, 1915 .alg.aead = { 1916 .setkey = safexcel_aead_setkey, 1917 .encrypt = safexcel_aead_encrypt, 1918 .decrypt = safexcel_aead_decrypt, 1919 .ivsize = DES3_EDE_BLOCK_SIZE, 1920 .maxauthsize = SHA1_DIGEST_SIZE, 1921 .base = { 1922 .cra_name = "authenc(hmac(sha1),cbc(des3_ede))", 1923 .cra_driver_name = "safexcel-authenc-hmac-sha1-cbc-des3_ede", 1924 .cra_priority = SAFEXCEL_CRA_PRIORITY, 1925 .cra_flags = CRYPTO_ALG_ASYNC | 1926 CRYPTO_ALG_ALLOCATES_MEMORY | 1927 CRYPTO_ALG_KERN_DRIVER_ONLY, 1928 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 1929 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 1930 .cra_alignmask = 0, 1931 .cra_init = safexcel_aead_sha1_des3_cra_init, 1932 .cra_exit = safexcel_aead_cra_exit, 1933 .cra_module = THIS_MODULE, 1934 }, 1935 }, 1936}; 1937 1938static int safexcel_aead_sha256_des3_cra_init(struct crypto_tfm *tfm) 1939{ 1940 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1941 1942 safexcel_aead_sha256_cra_init(tfm); 1943 ctx->alg = SAFEXCEL_3DES; /* override default */ 1944 ctx->blocksz = DES3_EDE_BLOCK_SIZE; 1945 ctx->ivmask = EIP197_OPTION_2_TOKEN_IV_CMD; 1946 return 0; 1947} 1948 1949struct safexcel_alg_template safexcel_alg_authenc_hmac_sha256_cbc_des3_ede = { 1950 .type = SAFEXCEL_ALG_TYPE_AEAD, 1951 .algo_mask = SAFEXCEL_ALG_DES | SAFEXCEL_ALG_SHA2_256, 1952 .alg.aead = { 1953 .setkey = safexcel_aead_setkey, 1954 .encrypt = safexcel_aead_encrypt, 1955 .decrypt = safexcel_aead_decrypt, 1956 .ivsize = DES3_EDE_BLOCK_SIZE, 1957 .maxauthsize = SHA256_DIGEST_SIZE, 1958 .base = { 1959 .cra_name = "authenc(hmac(sha256),cbc(des3_ede))", 1960 .cra_driver_name = "safexcel-authenc-hmac-sha256-cbc-des3_ede", 1961 .cra_priority = SAFEXCEL_CRA_PRIORITY, 1962 .cra_flags = CRYPTO_ALG_ASYNC | 1963 CRYPTO_ALG_ALLOCATES_MEMORY | 1964 CRYPTO_ALG_KERN_DRIVER_ONLY, 1965 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 1966 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 1967 .cra_alignmask = 0, 1968 .cra_init = safexcel_aead_sha256_des3_cra_init, 1969 .cra_exit = safexcel_aead_cra_exit, 1970 .cra_module = THIS_MODULE, 1971 }, 1972 }, 1973}; 1974 1975static int safexcel_aead_sha224_des3_cra_init(struct crypto_tfm *tfm) 1976{ 1977 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1978 1979 safexcel_aead_sha224_cra_init(tfm); 1980 ctx->alg = SAFEXCEL_3DES; /* override default */ 1981 ctx->blocksz = DES3_EDE_BLOCK_SIZE; 1982 ctx->ivmask = EIP197_OPTION_2_TOKEN_IV_CMD; 1983 return 0; 1984} 1985 1986struct safexcel_alg_template safexcel_alg_authenc_hmac_sha224_cbc_des3_ede = { 1987 .type = SAFEXCEL_ALG_TYPE_AEAD, 1988 .algo_mask = SAFEXCEL_ALG_DES | SAFEXCEL_ALG_SHA2_256, 1989 .alg.aead = { 1990 .setkey = safexcel_aead_setkey, 1991 .encrypt = safexcel_aead_encrypt, 1992 .decrypt = safexcel_aead_decrypt, 1993 .ivsize = DES3_EDE_BLOCK_SIZE, 1994 .maxauthsize = SHA224_DIGEST_SIZE, 1995 .base = { 1996 .cra_name = "authenc(hmac(sha224),cbc(des3_ede))", 1997 .cra_driver_name = "safexcel-authenc-hmac-sha224-cbc-des3_ede", 1998 .cra_priority = SAFEXCEL_CRA_PRIORITY, 1999 .cra_flags = CRYPTO_ALG_ASYNC | 2000 CRYPTO_ALG_ALLOCATES_MEMORY | 2001 CRYPTO_ALG_KERN_DRIVER_ONLY, 2002 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2003 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 2004 .cra_alignmask = 0, 2005 .cra_init = safexcel_aead_sha224_des3_cra_init, 2006 .cra_exit = safexcel_aead_cra_exit, 2007 .cra_module = THIS_MODULE, 2008 }, 2009 }, 2010}; 2011 2012static int safexcel_aead_sha512_des3_cra_init(struct crypto_tfm *tfm) 2013{ 2014 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 2015 2016 safexcel_aead_sha512_cra_init(tfm); 2017 ctx->alg = SAFEXCEL_3DES; /* override default */ 2018 ctx->blocksz = DES3_EDE_BLOCK_SIZE; 2019 ctx->ivmask = EIP197_OPTION_2_TOKEN_IV_CMD; 2020 return 0; 2021} 2022 2023struct safexcel_alg_template safexcel_alg_authenc_hmac_sha512_cbc_des3_ede = { 2024 .type = SAFEXCEL_ALG_TYPE_AEAD, 2025 .algo_mask = SAFEXCEL_ALG_DES | SAFEXCEL_ALG_SHA2_512, 2026 .alg.aead = { 2027 .setkey = safexcel_aead_setkey, 2028 .encrypt = safexcel_aead_encrypt, 2029 .decrypt = safexcel_aead_decrypt, 2030 .ivsize = DES3_EDE_BLOCK_SIZE, 2031 .maxauthsize = SHA512_DIGEST_SIZE, 2032 .base = { 2033 .cra_name = "authenc(hmac(sha512),cbc(des3_ede))", 2034 .cra_driver_name = "safexcel-authenc-hmac-sha512-cbc-des3_ede", 2035 .cra_priority = SAFEXCEL_CRA_PRIORITY, 2036 .cra_flags = CRYPTO_ALG_ASYNC | 2037 CRYPTO_ALG_ALLOCATES_MEMORY | 2038 CRYPTO_ALG_KERN_DRIVER_ONLY, 2039 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2040 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 2041 .cra_alignmask = 0, 2042 .cra_init = safexcel_aead_sha512_des3_cra_init, 2043 .cra_exit = safexcel_aead_cra_exit, 2044 .cra_module = THIS_MODULE, 2045 }, 2046 }, 2047}; 2048 2049static int safexcel_aead_sha384_des3_cra_init(struct crypto_tfm *tfm) 2050{ 2051 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 2052 2053 safexcel_aead_sha384_cra_init(tfm); 2054 ctx->alg = SAFEXCEL_3DES; /* override default */ 2055 ctx->blocksz = DES3_EDE_BLOCK_SIZE; 2056 ctx->ivmask = EIP197_OPTION_2_TOKEN_IV_CMD; 2057 return 0; 2058} 2059 2060struct safexcel_alg_template safexcel_alg_authenc_hmac_sha384_cbc_des3_ede = { 2061 .type = SAFEXCEL_ALG_TYPE_AEAD, 2062 .algo_mask = SAFEXCEL_ALG_DES | SAFEXCEL_ALG_SHA2_512, 2063 .alg.aead = { 2064 .setkey = safexcel_aead_setkey, 2065 .encrypt = safexcel_aead_encrypt, 2066 .decrypt = safexcel_aead_decrypt, 2067 .ivsize = DES3_EDE_BLOCK_SIZE, 2068 .maxauthsize = SHA384_DIGEST_SIZE, 2069 .base = { 2070 .cra_name = "authenc(hmac(sha384),cbc(des3_ede))", 2071 .cra_driver_name = "safexcel-authenc-hmac-sha384-cbc-des3_ede", 2072 .cra_priority = SAFEXCEL_CRA_PRIORITY, 2073 .cra_flags = CRYPTO_ALG_ASYNC | 2074 CRYPTO_ALG_ALLOCATES_MEMORY | 2075 CRYPTO_ALG_KERN_DRIVER_ONLY, 2076 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2077 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 2078 .cra_alignmask = 0, 2079 .cra_init = safexcel_aead_sha384_des3_cra_init, 2080 .cra_exit = safexcel_aead_cra_exit, 2081 .cra_module = THIS_MODULE, 2082 }, 2083 }, 2084}; 2085 2086static int safexcel_aead_sha1_des_cra_init(struct crypto_tfm *tfm) 2087{ 2088 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 2089 2090 safexcel_aead_sha1_cra_init(tfm); 2091 ctx->alg = SAFEXCEL_DES; /* override default */ 2092 ctx->blocksz = DES_BLOCK_SIZE; 2093 ctx->ivmask = EIP197_OPTION_2_TOKEN_IV_CMD; 2094 return 0; 2095} 2096 2097struct safexcel_alg_template safexcel_alg_authenc_hmac_sha1_cbc_des = { 2098 .type = SAFEXCEL_ALG_TYPE_AEAD, 2099 .algo_mask = SAFEXCEL_ALG_DES | SAFEXCEL_ALG_SHA1, 2100 .alg.aead = { 2101 .setkey = safexcel_aead_setkey, 2102 .encrypt = safexcel_aead_encrypt, 2103 .decrypt = safexcel_aead_decrypt, 2104 .ivsize = DES_BLOCK_SIZE, 2105 .maxauthsize = SHA1_DIGEST_SIZE, 2106 .base = { 2107 .cra_name = "authenc(hmac(sha1),cbc(des))", 2108 .cra_driver_name = "safexcel-authenc-hmac-sha1-cbc-des", 2109 .cra_priority = SAFEXCEL_CRA_PRIORITY, 2110 .cra_flags = CRYPTO_ALG_ASYNC | 2111 CRYPTO_ALG_ALLOCATES_MEMORY | 2112 CRYPTO_ALG_KERN_DRIVER_ONLY, 2113 .cra_blocksize = DES_BLOCK_SIZE, 2114 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 2115 .cra_alignmask = 0, 2116 .cra_init = safexcel_aead_sha1_des_cra_init, 2117 .cra_exit = safexcel_aead_cra_exit, 2118 .cra_module = THIS_MODULE, 2119 }, 2120 }, 2121}; 2122 2123static int safexcel_aead_sha256_des_cra_init(struct crypto_tfm *tfm) 2124{ 2125 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 2126 2127 safexcel_aead_sha256_cra_init(tfm); 2128 ctx->alg = SAFEXCEL_DES; /* override default */ 2129 ctx->blocksz = DES_BLOCK_SIZE; 2130 ctx->ivmask = EIP197_OPTION_2_TOKEN_IV_CMD; 2131 return 0; 2132} 2133 2134struct safexcel_alg_template safexcel_alg_authenc_hmac_sha256_cbc_des = { 2135 .type = SAFEXCEL_ALG_TYPE_AEAD, 2136 .algo_mask = SAFEXCEL_ALG_DES | SAFEXCEL_ALG_SHA2_256, 2137 .alg.aead = { 2138 .setkey = safexcel_aead_setkey, 2139 .encrypt = safexcel_aead_encrypt, 2140 .decrypt = safexcel_aead_decrypt, 2141 .ivsize = DES_BLOCK_SIZE, 2142 .maxauthsize = SHA256_DIGEST_SIZE, 2143 .base = { 2144 .cra_name = "authenc(hmac(sha256),cbc(des))", 2145 .cra_driver_name = "safexcel-authenc-hmac-sha256-cbc-des", 2146 .cra_priority = SAFEXCEL_CRA_PRIORITY, 2147 .cra_flags = CRYPTO_ALG_ASYNC | 2148 CRYPTO_ALG_ALLOCATES_MEMORY | 2149 CRYPTO_ALG_KERN_DRIVER_ONLY, 2150 .cra_blocksize = DES_BLOCK_SIZE, 2151 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 2152 .cra_alignmask = 0, 2153 .cra_init = safexcel_aead_sha256_des_cra_init, 2154 .cra_exit = safexcel_aead_cra_exit, 2155 .cra_module = THIS_MODULE, 2156 }, 2157 }, 2158}; 2159 2160static int safexcel_aead_sha224_des_cra_init(struct crypto_tfm *tfm) 2161{ 2162 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 2163 2164 safexcel_aead_sha224_cra_init(tfm); 2165 ctx->alg = SAFEXCEL_DES; /* override default */ 2166 ctx->blocksz = DES_BLOCK_SIZE; 2167 ctx->ivmask = EIP197_OPTION_2_TOKEN_IV_CMD; 2168 return 0; 2169} 2170 2171struct safexcel_alg_template safexcel_alg_authenc_hmac_sha224_cbc_des = { 2172 .type = SAFEXCEL_ALG_TYPE_AEAD, 2173 .algo_mask = SAFEXCEL_ALG_DES | SAFEXCEL_ALG_SHA2_256, 2174 .alg.aead = { 2175 .setkey = safexcel_aead_setkey, 2176 .encrypt = safexcel_aead_encrypt, 2177 .decrypt = safexcel_aead_decrypt, 2178 .ivsize = DES_BLOCK_SIZE, 2179 .maxauthsize = SHA224_DIGEST_SIZE, 2180 .base = { 2181 .cra_name = "authenc(hmac(sha224),cbc(des))", 2182 .cra_driver_name = "safexcel-authenc-hmac-sha224-cbc-des", 2183 .cra_priority = SAFEXCEL_CRA_PRIORITY, 2184 .cra_flags = CRYPTO_ALG_ASYNC | 2185 CRYPTO_ALG_ALLOCATES_MEMORY | 2186 CRYPTO_ALG_KERN_DRIVER_ONLY, 2187 .cra_blocksize = DES_BLOCK_SIZE, 2188 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 2189 .cra_alignmask = 0, 2190 .cra_init = safexcel_aead_sha224_des_cra_init, 2191 .cra_exit = safexcel_aead_cra_exit, 2192 .cra_module = THIS_MODULE, 2193 }, 2194 }, 2195}; 2196 2197static int safexcel_aead_sha512_des_cra_init(struct crypto_tfm *tfm) 2198{ 2199 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 2200 2201 safexcel_aead_sha512_cra_init(tfm); 2202 ctx->alg = SAFEXCEL_DES; /* override default */ 2203 ctx->blocksz = DES_BLOCK_SIZE; 2204 ctx->ivmask = EIP197_OPTION_2_TOKEN_IV_CMD; 2205 return 0; 2206} 2207 2208struct safexcel_alg_template safexcel_alg_authenc_hmac_sha512_cbc_des = { 2209 .type = SAFEXCEL_ALG_TYPE_AEAD, 2210 .algo_mask = SAFEXCEL_ALG_DES | SAFEXCEL_ALG_SHA2_512, 2211 .alg.aead = { 2212 .setkey = safexcel_aead_setkey, 2213 .encrypt = safexcel_aead_encrypt, 2214 .decrypt = safexcel_aead_decrypt, 2215 .ivsize = DES_BLOCK_SIZE, 2216 .maxauthsize = SHA512_DIGEST_SIZE, 2217 .base = { 2218 .cra_name = "authenc(hmac(sha512),cbc(des))", 2219 .cra_driver_name = "safexcel-authenc-hmac-sha512-cbc-des", 2220 .cra_priority = SAFEXCEL_CRA_PRIORITY, 2221 .cra_flags = CRYPTO_ALG_ASYNC | 2222 CRYPTO_ALG_ALLOCATES_MEMORY | 2223 CRYPTO_ALG_KERN_DRIVER_ONLY, 2224 .cra_blocksize = DES_BLOCK_SIZE, 2225 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 2226 .cra_alignmask = 0, 2227 .cra_init = safexcel_aead_sha512_des_cra_init, 2228 .cra_exit = safexcel_aead_cra_exit, 2229 .cra_module = THIS_MODULE, 2230 }, 2231 }, 2232}; 2233 2234static int safexcel_aead_sha384_des_cra_init(struct crypto_tfm *tfm) 2235{ 2236 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 2237 2238 safexcel_aead_sha384_cra_init(tfm); 2239 ctx->alg = SAFEXCEL_DES; /* override default */ 2240 ctx->blocksz = DES_BLOCK_SIZE; 2241 ctx->ivmask = EIP197_OPTION_2_TOKEN_IV_CMD; 2242 return 0; 2243} 2244 2245struct safexcel_alg_template safexcel_alg_authenc_hmac_sha384_cbc_des = { 2246 .type = SAFEXCEL_ALG_TYPE_AEAD, 2247 .algo_mask = SAFEXCEL_ALG_DES | SAFEXCEL_ALG_SHA2_512, 2248 .alg.aead = { 2249 .setkey = safexcel_aead_setkey, 2250 .encrypt = safexcel_aead_encrypt, 2251 .decrypt = safexcel_aead_decrypt, 2252 .ivsize = DES_BLOCK_SIZE, 2253 .maxauthsize = SHA384_DIGEST_SIZE, 2254 .base = { 2255 .cra_name = "authenc(hmac(sha384),cbc(des))", 2256 .cra_driver_name = "safexcel-authenc-hmac-sha384-cbc-des", 2257 .cra_priority = SAFEXCEL_CRA_PRIORITY, 2258 .cra_flags = CRYPTO_ALG_ASYNC | 2259 CRYPTO_ALG_ALLOCATES_MEMORY | 2260 CRYPTO_ALG_KERN_DRIVER_ONLY, 2261 .cra_blocksize = DES_BLOCK_SIZE, 2262 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 2263 .cra_alignmask = 0, 2264 .cra_init = safexcel_aead_sha384_des_cra_init, 2265 .cra_exit = safexcel_aead_cra_exit, 2266 .cra_module = THIS_MODULE, 2267 }, 2268 }, 2269}; 2270 2271static int safexcel_aead_sha1_ctr_cra_init(struct crypto_tfm *tfm) 2272{ 2273 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 2274 2275 safexcel_aead_sha1_cra_init(tfm); 2276 ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD; /* override default */ 2277 return 0; 2278} 2279 2280struct safexcel_alg_template safexcel_alg_authenc_hmac_sha1_ctr_aes = { 2281 .type = SAFEXCEL_ALG_TYPE_AEAD, 2282 .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_SHA1, 2283 .alg.aead = { 2284 .setkey = safexcel_aead_setkey, 2285 .encrypt = safexcel_aead_encrypt, 2286 .decrypt = safexcel_aead_decrypt, 2287 .ivsize = CTR_RFC3686_IV_SIZE, 2288 .maxauthsize = SHA1_DIGEST_SIZE, 2289 .base = { 2290 .cra_name = "authenc(hmac(sha1),rfc3686(ctr(aes)))", 2291 .cra_driver_name = "safexcel-authenc-hmac-sha1-ctr-aes", 2292 .cra_priority = SAFEXCEL_CRA_PRIORITY, 2293 .cra_flags = CRYPTO_ALG_ASYNC | 2294 CRYPTO_ALG_ALLOCATES_MEMORY | 2295 CRYPTO_ALG_KERN_DRIVER_ONLY, 2296 .cra_blocksize = 1, 2297 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 2298 .cra_alignmask = 0, 2299 .cra_init = safexcel_aead_sha1_ctr_cra_init, 2300 .cra_exit = safexcel_aead_cra_exit, 2301 .cra_module = THIS_MODULE, 2302 }, 2303 }, 2304}; 2305 2306static int safexcel_aead_sha256_ctr_cra_init(struct crypto_tfm *tfm) 2307{ 2308 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 2309 2310 safexcel_aead_sha256_cra_init(tfm); 2311 ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD; /* override default */ 2312 return 0; 2313} 2314 2315struct safexcel_alg_template safexcel_alg_authenc_hmac_sha256_ctr_aes = { 2316 .type = SAFEXCEL_ALG_TYPE_AEAD, 2317 .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_SHA2_256, 2318 .alg.aead = { 2319 .setkey = safexcel_aead_setkey, 2320 .encrypt = safexcel_aead_encrypt, 2321 .decrypt = safexcel_aead_decrypt, 2322 .ivsize = CTR_RFC3686_IV_SIZE, 2323 .maxauthsize = SHA256_DIGEST_SIZE, 2324 .base = { 2325 .cra_name = "authenc(hmac(sha256),rfc3686(ctr(aes)))", 2326 .cra_driver_name = "safexcel-authenc-hmac-sha256-ctr-aes", 2327 .cra_priority = SAFEXCEL_CRA_PRIORITY, 2328 .cra_flags = CRYPTO_ALG_ASYNC | 2329 CRYPTO_ALG_ALLOCATES_MEMORY | 2330 CRYPTO_ALG_KERN_DRIVER_ONLY, 2331 .cra_blocksize = 1, 2332 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 2333 .cra_alignmask = 0, 2334 .cra_init = safexcel_aead_sha256_ctr_cra_init, 2335 .cra_exit = safexcel_aead_cra_exit, 2336 .cra_module = THIS_MODULE, 2337 }, 2338 }, 2339}; 2340 2341static int safexcel_aead_sha224_ctr_cra_init(struct crypto_tfm *tfm) 2342{ 2343 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 2344 2345 safexcel_aead_sha224_cra_init(tfm); 2346 ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD; /* override default */ 2347 return 0; 2348} 2349 2350struct safexcel_alg_template safexcel_alg_authenc_hmac_sha224_ctr_aes = { 2351 .type = SAFEXCEL_ALG_TYPE_AEAD, 2352 .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_SHA2_256, 2353 .alg.aead = { 2354 .setkey = safexcel_aead_setkey, 2355 .encrypt = safexcel_aead_encrypt, 2356 .decrypt = safexcel_aead_decrypt, 2357 .ivsize = CTR_RFC3686_IV_SIZE, 2358 .maxauthsize = SHA224_DIGEST_SIZE, 2359 .base = { 2360 .cra_name = "authenc(hmac(sha224),rfc3686(ctr(aes)))", 2361 .cra_driver_name = "safexcel-authenc-hmac-sha224-ctr-aes", 2362 .cra_priority = SAFEXCEL_CRA_PRIORITY, 2363 .cra_flags = CRYPTO_ALG_ASYNC | 2364 CRYPTO_ALG_ALLOCATES_MEMORY | 2365 CRYPTO_ALG_KERN_DRIVER_ONLY, 2366 .cra_blocksize = 1, 2367 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 2368 .cra_alignmask = 0, 2369 .cra_init = safexcel_aead_sha224_ctr_cra_init, 2370 .cra_exit = safexcel_aead_cra_exit, 2371 .cra_module = THIS_MODULE, 2372 }, 2373 }, 2374}; 2375 2376static int safexcel_aead_sha512_ctr_cra_init(struct crypto_tfm *tfm) 2377{ 2378 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 2379 2380 safexcel_aead_sha512_cra_init(tfm); 2381 ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD; /* override default */ 2382 return 0; 2383} 2384 2385struct safexcel_alg_template safexcel_alg_authenc_hmac_sha512_ctr_aes = { 2386 .type = SAFEXCEL_ALG_TYPE_AEAD, 2387 .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_SHA2_512, 2388 .alg.aead = { 2389 .setkey = safexcel_aead_setkey, 2390 .encrypt = safexcel_aead_encrypt, 2391 .decrypt = safexcel_aead_decrypt, 2392 .ivsize = CTR_RFC3686_IV_SIZE, 2393 .maxauthsize = SHA512_DIGEST_SIZE, 2394 .base = { 2395 .cra_name = "authenc(hmac(sha512),rfc3686(ctr(aes)))", 2396 .cra_driver_name = "safexcel-authenc-hmac-sha512-ctr-aes", 2397 .cra_priority = SAFEXCEL_CRA_PRIORITY, 2398 .cra_flags = CRYPTO_ALG_ASYNC | 2399 CRYPTO_ALG_ALLOCATES_MEMORY | 2400 CRYPTO_ALG_KERN_DRIVER_ONLY, 2401 .cra_blocksize = 1, 2402 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 2403 .cra_alignmask = 0, 2404 .cra_init = safexcel_aead_sha512_ctr_cra_init, 2405 .cra_exit = safexcel_aead_cra_exit, 2406 .cra_module = THIS_MODULE, 2407 }, 2408 }, 2409}; 2410 2411static int safexcel_aead_sha384_ctr_cra_init(struct crypto_tfm *tfm) 2412{ 2413 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 2414 2415 safexcel_aead_sha384_cra_init(tfm); 2416 ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD; /* override default */ 2417 return 0; 2418} 2419 2420struct safexcel_alg_template safexcel_alg_authenc_hmac_sha384_ctr_aes = { 2421 .type = SAFEXCEL_ALG_TYPE_AEAD, 2422 .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_SHA2_512, 2423 .alg.aead = { 2424 .setkey = safexcel_aead_setkey, 2425 .encrypt = safexcel_aead_encrypt, 2426 .decrypt = safexcel_aead_decrypt, 2427 .ivsize = CTR_RFC3686_IV_SIZE, 2428 .maxauthsize = SHA384_DIGEST_SIZE, 2429 .base = { 2430 .cra_name = "authenc(hmac(sha384),rfc3686(ctr(aes)))", 2431 .cra_driver_name = "safexcel-authenc-hmac-sha384-ctr-aes", 2432 .cra_priority = SAFEXCEL_CRA_PRIORITY, 2433 .cra_flags = CRYPTO_ALG_ASYNC | 2434 CRYPTO_ALG_ALLOCATES_MEMORY | 2435 CRYPTO_ALG_KERN_DRIVER_ONLY, 2436 .cra_blocksize = 1, 2437 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 2438 .cra_alignmask = 0, 2439 .cra_init = safexcel_aead_sha384_ctr_cra_init, 2440 .cra_exit = safexcel_aead_cra_exit, 2441 .cra_module = THIS_MODULE, 2442 }, 2443 }, 2444}; 2445 2446static int safexcel_skcipher_aesxts_setkey(struct crypto_skcipher *ctfm, 2447 const u8 *key, unsigned int len) 2448{ 2449 struct crypto_tfm *tfm = crypto_skcipher_tfm(ctfm); 2450 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 2451 struct safexcel_crypto_priv *priv = ctx->base.priv; 2452 struct crypto_aes_ctx aes; 2453 int ret, i; 2454 unsigned int keylen; 2455 2456 /* Check for illegal XTS keys */ 2457 ret = xts_verify_key(ctfm, key, len); 2458 if (ret) 2459 return ret; 2460 2461 /* Only half of the key data is cipher key */ 2462 keylen = (len >> 1); 2463 ret = aes_expandkey(&aes, key, keylen); 2464 if (ret) 2465 return ret; 2466 2467 if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma) { 2468 for (i = 0; i < keylen / sizeof(u32); i++) { 2469 if (le32_to_cpu(ctx->key[i]) != aes.key_enc[i]) { 2470 ctx->base.needs_inv = true; 2471 break; 2472 } 2473 } 2474 } 2475 2476 for (i = 0; i < keylen / sizeof(u32); i++) 2477 ctx->key[i] = cpu_to_le32(aes.key_enc[i]); 2478 2479 /* The other half is the tweak key */ 2480 ret = aes_expandkey(&aes, (u8 *)(key + keylen), keylen); 2481 if (ret) 2482 return ret; 2483 2484 if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma) { 2485 for (i = 0; i < keylen / sizeof(u32); i++) { 2486 if (le32_to_cpu(ctx->key[i + keylen / sizeof(u32)]) != 2487 aes.key_enc[i]) { 2488 ctx->base.needs_inv = true; 2489 break; 2490 } 2491 } 2492 } 2493 2494 for (i = 0; i < keylen / sizeof(u32); i++) 2495 ctx->key[i + keylen / sizeof(u32)] = 2496 cpu_to_le32(aes.key_enc[i]); 2497 2498 ctx->key_len = keylen << 1; 2499 2500 memzero_explicit(&aes, sizeof(aes)); 2501 return 0; 2502} 2503 2504static int safexcel_skcipher_aes_xts_cra_init(struct crypto_tfm *tfm) 2505{ 2506 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 2507 2508 safexcel_skcipher_cra_init(tfm); 2509 ctx->alg = SAFEXCEL_AES; 2510 ctx->blocksz = AES_BLOCK_SIZE; 2511 ctx->xts = 1; 2512 ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_XTS; 2513 return 0; 2514} 2515 2516static int safexcel_encrypt_xts(struct skcipher_request *req) 2517{ 2518 if (req->cryptlen < XTS_BLOCK_SIZE) 2519 return -EINVAL; 2520 return safexcel_queue_req(&req->base, skcipher_request_ctx(req), 2521 SAFEXCEL_ENCRYPT); 2522} 2523 2524static int safexcel_decrypt_xts(struct skcipher_request *req) 2525{ 2526 if (req->cryptlen < XTS_BLOCK_SIZE) 2527 return -EINVAL; 2528 return safexcel_queue_req(&req->base, skcipher_request_ctx(req), 2529 SAFEXCEL_DECRYPT); 2530} 2531 2532struct safexcel_alg_template safexcel_alg_xts_aes = { 2533 .type = SAFEXCEL_ALG_TYPE_SKCIPHER, 2534 .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_AES_XTS, 2535 .alg.skcipher = { 2536 .setkey = safexcel_skcipher_aesxts_setkey, 2537 .encrypt = safexcel_encrypt_xts, 2538 .decrypt = safexcel_decrypt_xts, 2539 /* XTS actually uses 2 AES keys glued together */ 2540 .min_keysize = AES_MIN_KEY_SIZE * 2, 2541 .max_keysize = AES_MAX_KEY_SIZE * 2, 2542 .ivsize = XTS_BLOCK_SIZE, 2543 .base = { 2544 .cra_name = "xts(aes)", 2545 .cra_driver_name = "safexcel-xts-aes", 2546 .cra_priority = SAFEXCEL_CRA_PRIORITY, 2547 .cra_flags = CRYPTO_ALG_ASYNC | 2548 CRYPTO_ALG_ALLOCATES_MEMORY | 2549 CRYPTO_ALG_KERN_DRIVER_ONLY, 2550 .cra_blocksize = XTS_BLOCK_SIZE, 2551 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 2552 .cra_alignmask = 0, 2553 .cra_init = safexcel_skcipher_aes_xts_cra_init, 2554 .cra_exit = safexcel_skcipher_cra_exit, 2555 .cra_module = THIS_MODULE, 2556 }, 2557 }, 2558}; 2559 2560static int safexcel_aead_gcm_setkey(struct crypto_aead *ctfm, const u8 *key, 2561 unsigned int len) 2562{ 2563 struct crypto_tfm *tfm = crypto_aead_tfm(ctfm); 2564 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 2565 struct safexcel_crypto_priv *priv = ctx->base.priv; 2566 struct crypto_aes_ctx aes; 2567 u32 hashkey[AES_BLOCK_SIZE >> 2]; 2568 int ret, i; 2569 2570 ret = aes_expandkey(&aes, key, len); 2571 if (ret) { 2572 memzero_explicit(&aes, sizeof(aes)); 2573 return ret; 2574 } 2575 2576 if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma) { 2577 for (i = 0; i < len / sizeof(u32); i++) { 2578 if (le32_to_cpu(ctx->key[i]) != aes.key_enc[i]) { 2579 ctx->base.needs_inv = true; 2580 break; 2581 } 2582 } 2583 } 2584 2585 for (i = 0; i < len / sizeof(u32); i++) 2586 ctx->key[i] = cpu_to_le32(aes.key_enc[i]); 2587 2588 ctx->key_len = len; 2589 2590 /* Compute hash key by encrypting zeroes with cipher key */ 2591 crypto_cipher_clear_flags(ctx->hkaes, CRYPTO_TFM_REQ_MASK); 2592 crypto_cipher_set_flags(ctx->hkaes, crypto_aead_get_flags(ctfm) & 2593 CRYPTO_TFM_REQ_MASK); 2594 ret = crypto_cipher_setkey(ctx->hkaes, key, len); 2595 if (ret) 2596 return ret; 2597 2598 memset(hashkey, 0, AES_BLOCK_SIZE); 2599 crypto_cipher_encrypt_one(ctx->hkaes, (u8 *)hashkey, (u8 *)hashkey); 2600 2601 if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma) { 2602 for (i = 0; i < AES_BLOCK_SIZE / sizeof(u32); i++) { 2603 if (be32_to_cpu(ctx->base.ipad.be[i]) != hashkey[i]) { 2604 ctx->base.needs_inv = true; 2605 break; 2606 } 2607 } 2608 } 2609 2610 for (i = 0; i < AES_BLOCK_SIZE / sizeof(u32); i++) 2611 ctx->base.ipad.be[i] = cpu_to_be32(hashkey[i]); 2612 2613 memzero_explicit(hashkey, AES_BLOCK_SIZE); 2614 memzero_explicit(&aes, sizeof(aes)); 2615 return 0; 2616} 2617 2618static int safexcel_aead_gcm_cra_init(struct crypto_tfm *tfm) 2619{ 2620 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 2621 2622 safexcel_aead_cra_init(tfm); 2623 ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_GHASH; 2624 ctx->state_sz = GHASH_BLOCK_SIZE; 2625 ctx->xcm = EIP197_XCM_MODE_GCM; 2626 ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_XCM; /* override default */ 2627 2628 ctx->hkaes = crypto_alloc_cipher("aes", 0, 0); 2629 return PTR_ERR_OR_ZERO(ctx->hkaes); 2630} 2631 2632static void safexcel_aead_gcm_cra_exit(struct crypto_tfm *tfm) 2633{ 2634 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 2635 2636 crypto_free_cipher(ctx->hkaes); 2637 safexcel_aead_cra_exit(tfm); 2638} 2639 2640static int safexcel_aead_gcm_setauthsize(struct crypto_aead *tfm, 2641 unsigned int authsize) 2642{ 2643 return crypto_gcm_check_authsize(authsize); 2644} 2645 2646struct safexcel_alg_template safexcel_alg_gcm = { 2647 .type = SAFEXCEL_ALG_TYPE_AEAD, 2648 .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_GHASH, 2649 .alg.aead = { 2650 .setkey = safexcel_aead_gcm_setkey, 2651 .setauthsize = safexcel_aead_gcm_setauthsize, 2652 .encrypt = safexcel_aead_encrypt, 2653 .decrypt = safexcel_aead_decrypt, 2654 .ivsize = GCM_AES_IV_SIZE, 2655 .maxauthsize = GHASH_DIGEST_SIZE, 2656 .base = { 2657 .cra_name = "gcm(aes)", 2658 .cra_driver_name = "safexcel-gcm-aes", 2659 .cra_priority = SAFEXCEL_CRA_PRIORITY, 2660 .cra_flags = CRYPTO_ALG_ASYNC | 2661 CRYPTO_ALG_ALLOCATES_MEMORY | 2662 CRYPTO_ALG_KERN_DRIVER_ONLY, 2663 .cra_blocksize = 1, 2664 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 2665 .cra_alignmask = 0, 2666 .cra_init = safexcel_aead_gcm_cra_init, 2667 .cra_exit = safexcel_aead_gcm_cra_exit, 2668 .cra_module = THIS_MODULE, 2669 }, 2670 }, 2671}; 2672 2673static int safexcel_aead_ccm_setkey(struct crypto_aead *ctfm, const u8 *key, 2674 unsigned int len) 2675{ 2676 struct crypto_tfm *tfm = crypto_aead_tfm(ctfm); 2677 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 2678 struct safexcel_crypto_priv *priv = ctx->base.priv; 2679 struct crypto_aes_ctx aes; 2680 int ret, i; 2681 2682 ret = aes_expandkey(&aes, key, len); 2683 if (ret) { 2684 memzero_explicit(&aes, sizeof(aes)); 2685 return ret; 2686 } 2687 2688 if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma) { 2689 for (i = 0; i < len / sizeof(u32); i++) { 2690 if (le32_to_cpu(ctx->key[i]) != aes.key_enc[i]) { 2691 ctx->base.needs_inv = true; 2692 break; 2693 } 2694 } 2695 } 2696 2697 for (i = 0; i < len / sizeof(u32); i++) { 2698 ctx->key[i] = cpu_to_le32(aes.key_enc[i]); 2699 ctx->base.ipad.be[i + 2 * AES_BLOCK_SIZE / sizeof(u32)] = 2700 cpu_to_be32(aes.key_enc[i]); 2701 } 2702 2703 ctx->key_len = len; 2704 ctx->state_sz = 2 * AES_BLOCK_SIZE + len; 2705 2706 if (len == AES_KEYSIZE_192) 2707 ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_XCBC192; 2708 else if (len == AES_KEYSIZE_256) 2709 ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_XCBC256; 2710 else 2711 ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_XCBC128; 2712 2713 memzero_explicit(&aes, sizeof(aes)); 2714 return 0; 2715} 2716 2717static int safexcel_aead_ccm_cra_init(struct crypto_tfm *tfm) 2718{ 2719 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 2720 2721 safexcel_aead_cra_init(tfm); 2722 ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_XCBC128; 2723 ctx->state_sz = 3 * AES_BLOCK_SIZE; 2724 ctx->xcm = EIP197_XCM_MODE_CCM; 2725 ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_XCM; /* override default */ 2726 ctx->ctrinit = 0; 2727 return 0; 2728} 2729 2730static int safexcel_aead_ccm_setauthsize(struct crypto_aead *tfm, 2731 unsigned int authsize) 2732{ 2733 /* Borrowed from crypto/ccm.c */ 2734 switch (authsize) { 2735 case 4: 2736 case 6: 2737 case 8: 2738 case 10: 2739 case 12: 2740 case 14: 2741 case 16: 2742 break; 2743 default: 2744 return -EINVAL; 2745 } 2746 2747 return 0; 2748} 2749 2750static int safexcel_ccm_encrypt(struct aead_request *req) 2751{ 2752 struct safexcel_cipher_req *creq = aead_request_ctx(req); 2753 2754 if (req->iv[0] < 1 || req->iv[0] > 7) 2755 return -EINVAL; 2756 2757 return safexcel_queue_req(&req->base, creq, SAFEXCEL_ENCRYPT); 2758} 2759 2760static int safexcel_ccm_decrypt(struct aead_request *req) 2761{ 2762 struct safexcel_cipher_req *creq = aead_request_ctx(req); 2763 2764 if (req->iv[0] < 1 || req->iv[0] > 7) 2765 return -EINVAL; 2766 2767 return safexcel_queue_req(&req->base, creq, SAFEXCEL_DECRYPT); 2768} 2769 2770struct safexcel_alg_template safexcel_alg_ccm = { 2771 .type = SAFEXCEL_ALG_TYPE_AEAD, 2772 .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_CBC_MAC_ALL, 2773 .alg.aead = { 2774 .setkey = safexcel_aead_ccm_setkey, 2775 .setauthsize = safexcel_aead_ccm_setauthsize, 2776 .encrypt = safexcel_ccm_encrypt, 2777 .decrypt = safexcel_ccm_decrypt, 2778 .ivsize = AES_BLOCK_SIZE, 2779 .maxauthsize = AES_BLOCK_SIZE, 2780 .base = { 2781 .cra_name = "ccm(aes)", 2782 .cra_driver_name = "safexcel-ccm-aes", 2783 .cra_priority = SAFEXCEL_CRA_PRIORITY, 2784 .cra_flags = CRYPTO_ALG_ASYNC | 2785 CRYPTO_ALG_ALLOCATES_MEMORY | 2786 CRYPTO_ALG_KERN_DRIVER_ONLY, 2787 .cra_blocksize = 1, 2788 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 2789 .cra_alignmask = 0, 2790 .cra_init = safexcel_aead_ccm_cra_init, 2791 .cra_exit = safexcel_aead_cra_exit, 2792 .cra_module = THIS_MODULE, 2793 }, 2794 }, 2795}; 2796 2797static void safexcel_chacha20_setkey(struct safexcel_cipher_ctx *ctx, 2798 const u8 *key) 2799{ 2800 struct safexcel_crypto_priv *priv = ctx->base.priv; 2801 2802 if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma) 2803 if (memcmp(ctx->key, key, CHACHA_KEY_SIZE)) 2804 ctx->base.needs_inv = true; 2805 2806 memcpy(ctx->key, key, CHACHA_KEY_SIZE); 2807 ctx->key_len = CHACHA_KEY_SIZE; 2808} 2809 2810static int safexcel_skcipher_chacha20_setkey(struct crypto_skcipher *ctfm, 2811 const u8 *key, unsigned int len) 2812{ 2813 struct safexcel_cipher_ctx *ctx = crypto_skcipher_ctx(ctfm); 2814 2815 if (len != CHACHA_KEY_SIZE) 2816 return -EINVAL; 2817 2818 safexcel_chacha20_setkey(ctx, key); 2819 2820 return 0; 2821} 2822 2823static int safexcel_skcipher_chacha20_cra_init(struct crypto_tfm *tfm) 2824{ 2825 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 2826 2827 safexcel_skcipher_cra_init(tfm); 2828 ctx->alg = SAFEXCEL_CHACHA20; 2829 ctx->ctrinit = 0; 2830 ctx->mode = CONTEXT_CONTROL_CHACHA20_MODE_256_32; 2831 return 0; 2832} 2833 2834struct safexcel_alg_template safexcel_alg_chacha20 = { 2835 .type = SAFEXCEL_ALG_TYPE_SKCIPHER, 2836 .algo_mask = SAFEXCEL_ALG_CHACHA20, 2837 .alg.skcipher = { 2838 .setkey = safexcel_skcipher_chacha20_setkey, 2839 .encrypt = safexcel_encrypt, 2840 .decrypt = safexcel_decrypt, 2841 .min_keysize = CHACHA_KEY_SIZE, 2842 .max_keysize = CHACHA_KEY_SIZE, 2843 .ivsize = CHACHA_IV_SIZE, 2844 .base = { 2845 .cra_name = "chacha20", 2846 .cra_driver_name = "safexcel-chacha20", 2847 .cra_priority = SAFEXCEL_CRA_PRIORITY, 2848 .cra_flags = CRYPTO_ALG_ASYNC | 2849 CRYPTO_ALG_ALLOCATES_MEMORY | 2850 CRYPTO_ALG_KERN_DRIVER_ONLY, 2851 .cra_blocksize = 1, 2852 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 2853 .cra_alignmask = 0, 2854 .cra_init = safexcel_skcipher_chacha20_cra_init, 2855 .cra_exit = safexcel_skcipher_cra_exit, 2856 .cra_module = THIS_MODULE, 2857 }, 2858 }, 2859}; 2860 2861static int safexcel_aead_chachapoly_setkey(struct crypto_aead *ctfm, 2862 const u8 *key, unsigned int len) 2863{ 2864 struct safexcel_cipher_ctx *ctx = crypto_aead_ctx(ctfm); 2865 2866 if (ctx->aead == EIP197_AEAD_TYPE_IPSEC_ESP && 2867 len > EIP197_AEAD_IPSEC_NONCE_SIZE) { 2868 /* ESP variant has nonce appended to key */ 2869 len -= EIP197_AEAD_IPSEC_NONCE_SIZE; 2870 ctx->nonce = *(u32 *)(key + len); 2871 } 2872 if (len != CHACHA_KEY_SIZE) 2873 return -EINVAL; 2874 2875 safexcel_chacha20_setkey(ctx, key); 2876 2877 return 0; 2878} 2879 2880static int safexcel_aead_chachapoly_setauthsize(struct crypto_aead *tfm, 2881 unsigned int authsize) 2882{ 2883 if (authsize != POLY1305_DIGEST_SIZE) 2884 return -EINVAL; 2885 return 0; 2886} 2887 2888static int safexcel_aead_chachapoly_crypt(struct aead_request *req, 2889 enum safexcel_cipher_direction dir) 2890{ 2891 struct safexcel_cipher_req *creq = aead_request_ctx(req); 2892 struct crypto_aead *aead = crypto_aead_reqtfm(req); 2893 struct crypto_tfm *tfm = crypto_aead_tfm(aead); 2894 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 2895 struct aead_request *subreq = aead_request_ctx(req); 2896 u32 key[CHACHA_KEY_SIZE / sizeof(u32) + 1]; 2897 int ret = 0; 2898 2899 /* 2900 * Instead of wasting time detecting umpteen silly corner cases, 2901 * just dump all "small" requests to the fallback implementation. 2902 * HW would not be faster on such small requests anyway. 2903 */ 2904 if (likely((ctx->aead != EIP197_AEAD_TYPE_IPSEC_ESP || 2905 req->assoclen >= EIP197_AEAD_IPSEC_IV_SIZE) && 2906 req->cryptlen > POLY1305_DIGEST_SIZE)) { 2907 return safexcel_queue_req(&req->base, creq, dir); 2908 } 2909 2910 /* HW cannot do full (AAD+payload) zero length, use fallback */ 2911 memcpy(key, ctx->key, CHACHA_KEY_SIZE); 2912 if (ctx->aead == EIP197_AEAD_TYPE_IPSEC_ESP) { 2913 /* ESP variant has nonce appended to the key */ 2914 key[CHACHA_KEY_SIZE / sizeof(u32)] = ctx->nonce; 2915 ret = crypto_aead_setkey(ctx->fback, (u8 *)key, 2916 CHACHA_KEY_SIZE + 2917 EIP197_AEAD_IPSEC_NONCE_SIZE); 2918 } else { 2919 ret = crypto_aead_setkey(ctx->fback, (u8 *)key, 2920 CHACHA_KEY_SIZE); 2921 } 2922 if (ret) { 2923 crypto_aead_clear_flags(aead, CRYPTO_TFM_REQ_MASK); 2924 crypto_aead_set_flags(aead, crypto_aead_get_flags(ctx->fback) & 2925 CRYPTO_TFM_REQ_MASK); 2926 return ret; 2927 } 2928 2929 aead_request_set_tfm(subreq, ctx->fback); 2930 aead_request_set_callback(subreq, req->base.flags, req->base.complete, 2931 req->base.data); 2932 aead_request_set_crypt(subreq, req->src, req->dst, req->cryptlen, 2933 req->iv); 2934 aead_request_set_ad(subreq, req->assoclen); 2935 2936 return (dir == SAFEXCEL_ENCRYPT) ? 2937 crypto_aead_encrypt(subreq) : 2938 crypto_aead_decrypt(subreq); 2939} 2940 2941static int safexcel_aead_chachapoly_encrypt(struct aead_request *req) 2942{ 2943 return safexcel_aead_chachapoly_crypt(req, SAFEXCEL_ENCRYPT); 2944} 2945 2946static int safexcel_aead_chachapoly_decrypt(struct aead_request *req) 2947{ 2948 return safexcel_aead_chachapoly_crypt(req, SAFEXCEL_DECRYPT); 2949} 2950 2951static int safexcel_aead_fallback_cra_init(struct crypto_tfm *tfm) 2952{ 2953 struct crypto_aead *aead = __crypto_aead_cast(tfm); 2954 struct aead_alg *alg = crypto_aead_alg(aead); 2955 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 2956 2957 safexcel_aead_cra_init(tfm); 2958 2959 /* Allocate fallback implementation */ 2960 ctx->fback = crypto_alloc_aead(alg->base.cra_name, 0, 2961 CRYPTO_ALG_ASYNC | 2962 CRYPTO_ALG_NEED_FALLBACK); 2963 if (IS_ERR(ctx->fback)) 2964 return PTR_ERR(ctx->fback); 2965 2966 crypto_aead_set_reqsize(aead, max(sizeof(struct safexcel_cipher_req), 2967 sizeof(struct aead_request) + 2968 crypto_aead_reqsize(ctx->fback))); 2969 2970 return 0; 2971} 2972 2973static int safexcel_aead_chachapoly_cra_init(struct crypto_tfm *tfm) 2974{ 2975 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 2976 2977 safexcel_aead_fallback_cra_init(tfm); 2978 ctx->alg = SAFEXCEL_CHACHA20; 2979 ctx->mode = CONTEXT_CONTROL_CHACHA20_MODE_256_32 | 2980 CONTEXT_CONTROL_CHACHA20_MODE_CALC_OTK; 2981 ctx->ctrinit = 0; 2982 ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_POLY1305; 2983 ctx->state_sz = 0; /* Precomputed by HW */ 2984 return 0; 2985} 2986 2987static void safexcel_aead_fallback_cra_exit(struct crypto_tfm *tfm) 2988{ 2989 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 2990 2991 crypto_free_aead(ctx->fback); 2992 safexcel_aead_cra_exit(tfm); 2993} 2994 2995struct safexcel_alg_template safexcel_alg_chachapoly = { 2996 .type = SAFEXCEL_ALG_TYPE_AEAD, 2997 .algo_mask = SAFEXCEL_ALG_CHACHA20 | SAFEXCEL_ALG_POLY1305, 2998 .alg.aead = { 2999 .setkey = safexcel_aead_chachapoly_setkey, 3000 .setauthsize = safexcel_aead_chachapoly_setauthsize, 3001 .encrypt = safexcel_aead_chachapoly_encrypt, 3002 .decrypt = safexcel_aead_chachapoly_decrypt, 3003 .ivsize = CHACHAPOLY_IV_SIZE, 3004 .maxauthsize = POLY1305_DIGEST_SIZE, 3005 .base = { 3006 .cra_name = "rfc7539(chacha20,poly1305)", 3007 .cra_driver_name = "safexcel-chacha20-poly1305", 3008 /* +1 to put it above HW chacha + SW poly */ 3009 .cra_priority = SAFEXCEL_CRA_PRIORITY + 1, 3010 .cra_flags = CRYPTO_ALG_ASYNC | 3011 CRYPTO_ALG_ALLOCATES_MEMORY | 3012 CRYPTO_ALG_KERN_DRIVER_ONLY | 3013 CRYPTO_ALG_NEED_FALLBACK, 3014 .cra_blocksize = 1, 3015 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 3016 .cra_alignmask = 0, 3017 .cra_init = safexcel_aead_chachapoly_cra_init, 3018 .cra_exit = safexcel_aead_fallback_cra_exit, 3019 .cra_module = THIS_MODULE, 3020 }, 3021 }, 3022}; 3023 3024static int safexcel_aead_chachapolyesp_cra_init(struct crypto_tfm *tfm) 3025{ 3026 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 3027 int ret; 3028 3029 ret = safexcel_aead_chachapoly_cra_init(tfm); 3030 ctx->aead = EIP197_AEAD_TYPE_IPSEC_ESP; 3031 ctx->aadskip = EIP197_AEAD_IPSEC_IV_SIZE; 3032 return ret; 3033} 3034 3035struct safexcel_alg_template safexcel_alg_chachapoly_esp = { 3036 .type = SAFEXCEL_ALG_TYPE_AEAD, 3037 .algo_mask = SAFEXCEL_ALG_CHACHA20 | SAFEXCEL_ALG_POLY1305, 3038 .alg.aead = { 3039 .setkey = safexcel_aead_chachapoly_setkey, 3040 .setauthsize = safexcel_aead_chachapoly_setauthsize, 3041 .encrypt = safexcel_aead_chachapoly_encrypt, 3042 .decrypt = safexcel_aead_chachapoly_decrypt, 3043 .ivsize = CHACHAPOLY_IV_SIZE - EIP197_AEAD_IPSEC_NONCE_SIZE, 3044 .maxauthsize = POLY1305_DIGEST_SIZE, 3045 .base = { 3046 .cra_name = "rfc7539esp(chacha20,poly1305)", 3047 .cra_driver_name = "safexcel-chacha20-poly1305-esp", 3048 /* +1 to put it above HW chacha + SW poly */ 3049 .cra_priority = SAFEXCEL_CRA_PRIORITY + 1, 3050 .cra_flags = CRYPTO_ALG_ASYNC | 3051 CRYPTO_ALG_ALLOCATES_MEMORY | 3052 CRYPTO_ALG_KERN_DRIVER_ONLY | 3053 CRYPTO_ALG_NEED_FALLBACK, 3054 .cra_blocksize = 1, 3055 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 3056 .cra_alignmask = 0, 3057 .cra_init = safexcel_aead_chachapolyesp_cra_init, 3058 .cra_exit = safexcel_aead_fallback_cra_exit, 3059 .cra_module = THIS_MODULE, 3060 }, 3061 }, 3062}; 3063 3064static int safexcel_skcipher_sm4_setkey(struct crypto_skcipher *ctfm, 3065 const u8 *key, unsigned int len) 3066{ 3067 struct crypto_tfm *tfm = crypto_skcipher_tfm(ctfm); 3068 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 3069 struct safexcel_crypto_priv *priv = ctx->base.priv; 3070 3071 if (len != SM4_KEY_SIZE) 3072 return -EINVAL; 3073 3074 if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma) 3075 if (memcmp(ctx->key, key, SM4_KEY_SIZE)) 3076 ctx->base.needs_inv = true; 3077 3078 memcpy(ctx->key, key, SM4_KEY_SIZE); 3079 ctx->key_len = SM4_KEY_SIZE; 3080 3081 return 0; 3082} 3083 3084static int safexcel_sm4_blk_encrypt(struct skcipher_request *req) 3085{ 3086 /* Workaround for HW bug: EIP96 4.3 does not report blocksize error */ 3087 if (req->cryptlen & (SM4_BLOCK_SIZE - 1)) 3088 return -EINVAL; 3089 else 3090 return safexcel_queue_req(&req->base, skcipher_request_ctx(req), 3091 SAFEXCEL_ENCRYPT); 3092} 3093 3094static int safexcel_sm4_blk_decrypt(struct skcipher_request *req) 3095{ 3096 /* Workaround for HW bug: EIP96 4.3 does not report blocksize error */ 3097 if (req->cryptlen & (SM4_BLOCK_SIZE - 1)) 3098 return -EINVAL; 3099 else 3100 return safexcel_queue_req(&req->base, skcipher_request_ctx(req), 3101 SAFEXCEL_DECRYPT); 3102} 3103 3104static int safexcel_skcipher_sm4_ecb_cra_init(struct crypto_tfm *tfm) 3105{ 3106 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 3107 3108 safexcel_skcipher_cra_init(tfm); 3109 ctx->alg = SAFEXCEL_SM4; 3110 ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_ECB; 3111 ctx->blocksz = 0; 3112 ctx->ivmask = EIP197_OPTION_2_TOKEN_IV_CMD; 3113 return 0; 3114} 3115 3116struct safexcel_alg_template safexcel_alg_ecb_sm4 = { 3117 .type = SAFEXCEL_ALG_TYPE_SKCIPHER, 3118 .algo_mask = SAFEXCEL_ALG_SM4, 3119 .alg.skcipher = { 3120 .setkey = safexcel_skcipher_sm4_setkey, 3121 .encrypt = safexcel_sm4_blk_encrypt, 3122 .decrypt = safexcel_sm4_blk_decrypt, 3123 .min_keysize = SM4_KEY_SIZE, 3124 .max_keysize = SM4_KEY_SIZE, 3125 .base = { 3126 .cra_name = "ecb(sm4)", 3127 .cra_driver_name = "safexcel-ecb-sm4", 3128 .cra_priority = SAFEXCEL_CRA_PRIORITY, 3129 .cra_flags = CRYPTO_ALG_ASYNC | 3130 CRYPTO_ALG_ALLOCATES_MEMORY | 3131 CRYPTO_ALG_KERN_DRIVER_ONLY, 3132 .cra_blocksize = SM4_BLOCK_SIZE, 3133 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 3134 .cra_alignmask = 0, 3135 .cra_init = safexcel_skcipher_sm4_ecb_cra_init, 3136 .cra_exit = safexcel_skcipher_cra_exit, 3137 .cra_module = THIS_MODULE, 3138 }, 3139 }, 3140}; 3141 3142static int safexcel_skcipher_sm4_cbc_cra_init(struct crypto_tfm *tfm) 3143{ 3144 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 3145 3146 safexcel_skcipher_cra_init(tfm); 3147 ctx->alg = SAFEXCEL_SM4; 3148 ctx->blocksz = SM4_BLOCK_SIZE; 3149 ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CBC; 3150 return 0; 3151} 3152 3153struct safexcel_alg_template safexcel_alg_cbc_sm4 = { 3154 .type = SAFEXCEL_ALG_TYPE_SKCIPHER, 3155 .algo_mask = SAFEXCEL_ALG_SM4, 3156 .alg.skcipher = { 3157 .setkey = safexcel_skcipher_sm4_setkey, 3158 .encrypt = safexcel_sm4_blk_encrypt, 3159 .decrypt = safexcel_sm4_blk_decrypt, 3160 .min_keysize = SM4_KEY_SIZE, 3161 .max_keysize = SM4_KEY_SIZE, 3162 .ivsize = SM4_BLOCK_SIZE, 3163 .base = { 3164 .cra_name = "cbc(sm4)", 3165 .cra_driver_name = "safexcel-cbc-sm4", 3166 .cra_priority = SAFEXCEL_CRA_PRIORITY, 3167 .cra_flags = CRYPTO_ALG_ASYNC | 3168 CRYPTO_ALG_ALLOCATES_MEMORY | 3169 CRYPTO_ALG_KERN_DRIVER_ONLY, 3170 .cra_blocksize = SM4_BLOCK_SIZE, 3171 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 3172 .cra_alignmask = 0, 3173 .cra_init = safexcel_skcipher_sm4_cbc_cra_init, 3174 .cra_exit = safexcel_skcipher_cra_exit, 3175 .cra_module = THIS_MODULE, 3176 }, 3177 }, 3178}; 3179 3180static int safexcel_skcipher_sm4_ofb_cra_init(struct crypto_tfm *tfm) 3181{ 3182 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 3183 3184 safexcel_skcipher_cra_init(tfm); 3185 ctx->alg = SAFEXCEL_SM4; 3186 ctx->blocksz = SM4_BLOCK_SIZE; 3187 ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_OFB; 3188 return 0; 3189} 3190 3191struct safexcel_alg_template safexcel_alg_ofb_sm4 = { 3192 .type = SAFEXCEL_ALG_TYPE_SKCIPHER, 3193 .algo_mask = SAFEXCEL_ALG_SM4 | SAFEXCEL_ALG_AES_XFB, 3194 .alg.skcipher = { 3195 .setkey = safexcel_skcipher_sm4_setkey, 3196 .encrypt = safexcel_encrypt, 3197 .decrypt = safexcel_decrypt, 3198 .min_keysize = SM4_KEY_SIZE, 3199 .max_keysize = SM4_KEY_SIZE, 3200 .ivsize = SM4_BLOCK_SIZE, 3201 .base = { 3202 .cra_name = "ofb(sm4)", 3203 .cra_driver_name = "safexcel-ofb-sm4", 3204 .cra_priority = SAFEXCEL_CRA_PRIORITY, 3205 .cra_flags = CRYPTO_ALG_ASYNC | 3206 CRYPTO_ALG_ALLOCATES_MEMORY | 3207 CRYPTO_ALG_KERN_DRIVER_ONLY, 3208 .cra_blocksize = 1, 3209 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 3210 .cra_alignmask = 0, 3211 .cra_init = safexcel_skcipher_sm4_ofb_cra_init, 3212 .cra_exit = safexcel_skcipher_cra_exit, 3213 .cra_module = THIS_MODULE, 3214 }, 3215 }, 3216}; 3217 3218static int safexcel_skcipher_sm4_cfb_cra_init(struct crypto_tfm *tfm) 3219{ 3220 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 3221 3222 safexcel_skcipher_cra_init(tfm); 3223 ctx->alg = SAFEXCEL_SM4; 3224 ctx->blocksz = SM4_BLOCK_SIZE; 3225 ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CFB; 3226 return 0; 3227} 3228 3229struct safexcel_alg_template safexcel_alg_cfb_sm4 = { 3230 .type = SAFEXCEL_ALG_TYPE_SKCIPHER, 3231 .algo_mask = SAFEXCEL_ALG_SM4 | SAFEXCEL_ALG_AES_XFB, 3232 .alg.skcipher = { 3233 .setkey = safexcel_skcipher_sm4_setkey, 3234 .encrypt = safexcel_encrypt, 3235 .decrypt = safexcel_decrypt, 3236 .min_keysize = SM4_KEY_SIZE, 3237 .max_keysize = SM4_KEY_SIZE, 3238 .ivsize = SM4_BLOCK_SIZE, 3239 .base = { 3240 .cra_name = "cfb(sm4)", 3241 .cra_driver_name = "safexcel-cfb-sm4", 3242 .cra_priority = SAFEXCEL_CRA_PRIORITY, 3243 .cra_flags = CRYPTO_ALG_ASYNC | 3244 CRYPTO_ALG_ALLOCATES_MEMORY | 3245 CRYPTO_ALG_KERN_DRIVER_ONLY, 3246 .cra_blocksize = 1, 3247 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 3248 .cra_alignmask = 0, 3249 .cra_init = safexcel_skcipher_sm4_cfb_cra_init, 3250 .cra_exit = safexcel_skcipher_cra_exit, 3251 .cra_module = THIS_MODULE, 3252 }, 3253 }, 3254}; 3255 3256static int safexcel_skcipher_sm4ctr_setkey(struct crypto_skcipher *ctfm, 3257 const u8 *key, unsigned int len) 3258{ 3259 struct crypto_tfm *tfm = crypto_skcipher_tfm(ctfm); 3260 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 3261 3262 /* last 4 bytes of key are the nonce! */ 3263 ctx->nonce = *(u32 *)(key + len - CTR_RFC3686_NONCE_SIZE); 3264 /* exclude the nonce here */ 3265 len -= CTR_RFC3686_NONCE_SIZE; 3266 3267 return safexcel_skcipher_sm4_setkey(ctfm, key, len); 3268} 3269 3270static int safexcel_skcipher_sm4_ctr_cra_init(struct crypto_tfm *tfm) 3271{ 3272 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 3273 3274 safexcel_skcipher_cra_init(tfm); 3275 ctx->alg = SAFEXCEL_SM4; 3276 ctx->blocksz = SM4_BLOCK_SIZE; 3277 ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD; 3278 return 0; 3279} 3280 3281struct safexcel_alg_template safexcel_alg_ctr_sm4 = { 3282 .type = SAFEXCEL_ALG_TYPE_SKCIPHER, 3283 .algo_mask = SAFEXCEL_ALG_SM4, 3284 .alg.skcipher = { 3285 .setkey = safexcel_skcipher_sm4ctr_setkey, 3286 .encrypt = safexcel_encrypt, 3287 .decrypt = safexcel_decrypt, 3288 /* Add nonce size */ 3289 .min_keysize = SM4_KEY_SIZE + CTR_RFC3686_NONCE_SIZE, 3290 .max_keysize = SM4_KEY_SIZE + CTR_RFC3686_NONCE_SIZE, 3291 .ivsize = CTR_RFC3686_IV_SIZE, 3292 .base = { 3293 .cra_name = "rfc3686(ctr(sm4))", 3294 .cra_driver_name = "safexcel-ctr-sm4", 3295 .cra_priority = SAFEXCEL_CRA_PRIORITY, 3296 .cra_flags = CRYPTO_ALG_ASYNC | 3297 CRYPTO_ALG_ALLOCATES_MEMORY | 3298 CRYPTO_ALG_KERN_DRIVER_ONLY, 3299 .cra_blocksize = 1, 3300 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 3301 .cra_alignmask = 0, 3302 .cra_init = safexcel_skcipher_sm4_ctr_cra_init, 3303 .cra_exit = safexcel_skcipher_cra_exit, 3304 .cra_module = THIS_MODULE, 3305 }, 3306 }, 3307}; 3308 3309static int safexcel_aead_sm4_blk_encrypt(struct aead_request *req) 3310{ 3311 /* Workaround for HW bug: EIP96 4.3 does not report blocksize error */ 3312 if (req->cryptlen & (SM4_BLOCK_SIZE - 1)) 3313 return -EINVAL; 3314 3315 return safexcel_queue_req(&req->base, aead_request_ctx(req), 3316 SAFEXCEL_ENCRYPT); 3317} 3318 3319static int safexcel_aead_sm4_blk_decrypt(struct aead_request *req) 3320{ 3321 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 3322 3323 /* Workaround for HW bug: EIP96 4.3 does not report blocksize error */ 3324 if ((req->cryptlen - crypto_aead_authsize(tfm)) & (SM4_BLOCK_SIZE - 1)) 3325 return -EINVAL; 3326 3327 return safexcel_queue_req(&req->base, aead_request_ctx(req), 3328 SAFEXCEL_DECRYPT); 3329} 3330 3331static int safexcel_aead_sm4cbc_sha1_cra_init(struct crypto_tfm *tfm) 3332{ 3333 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 3334 3335 safexcel_aead_cra_init(tfm); 3336 ctx->alg = SAFEXCEL_SM4; 3337 ctx->blocksz = SM4_BLOCK_SIZE; 3338 ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA1; 3339 ctx->state_sz = SHA1_DIGEST_SIZE; 3340 return 0; 3341} 3342 3343struct safexcel_alg_template safexcel_alg_authenc_hmac_sha1_cbc_sm4 = { 3344 .type = SAFEXCEL_ALG_TYPE_AEAD, 3345 .algo_mask = SAFEXCEL_ALG_SM4 | SAFEXCEL_ALG_SHA1, 3346 .alg.aead = { 3347 .setkey = safexcel_aead_setkey, 3348 .encrypt = safexcel_aead_sm4_blk_encrypt, 3349 .decrypt = safexcel_aead_sm4_blk_decrypt, 3350 .ivsize = SM4_BLOCK_SIZE, 3351 .maxauthsize = SHA1_DIGEST_SIZE, 3352 .base = { 3353 .cra_name = "authenc(hmac(sha1),cbc(sm4))", 3354 .cra_driver_name = "safexcel-authenc-hmac-sha1-cbc-sm4", 3355 .cra_priority = SAFEXCEL_CRA_PRIORITY, 3356 .cra_flags = CRYPTO_ALG_ASYNC | 3357 CRYPTO_ALG_ALLOCATES_MEMORY | 3358 CRYPTO_ALG_KERN_DRIVER_ONLY, 3359 .cra_blocksize = SM4_BLOCK_SIZE, 3360 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 3361 .cra_alignmask = 0, 3362 .cra_init = safexcel_aead_sm4cbc_sha1_cra_init, 3363 .cra_exit = safexcel_aead_cra_exit, 3364 .cra_module = THIS_MODULE, 3365 }, 3366 }, 3367}; 3368 3369static int safexcel_aead_fallback_setkey(struct crypto_aead *ctfm, 3370 const u8 *key, unsigned int len) 3371{ 3372 struct crypto_tfm *tfm = crypto_aead_tfm(ctfm); 3373 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 3374 3375 /* Keep fallback cipher synchronized */ 3376 return crypto_aead_setkey(ctx->fback, (u8 *)key, len) ?: 3377 safexcel_aead_setkey(ctfm, key, len); 3378} 3379 3380static int safexcel_aead_fallback_setauthsize(struct crypto_aead *ctfm, 3381 unsigned int authsize) 3382{ 3383 struct crypto_tfm *tfm = crypto_aead_tfm(ctfm); 3384 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 3385 3386 /* Keep fallback cipher synchronized */ 3387 return crypto_aead_setauthsize(ctx->fback, authsize); 3388} 3389 3390static int safexcel_aead_fallback_crypt(struct aead_request *req, 3391 enum safexcel_cipher_direction dir) 3392{ 3393 struct crypto_aead *aead = crypto_aead_reqtfm(req); 3394 struct crypto_tfm *tfm = crypto_aead_tfm(aead); 3395 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 3396 struct aead_request *subreq = aead_request_ctx(req); 3397 3398 aead_request_set_tfm(subreq, ctx->fback); 3399 aead_request_set_callback(subreq, req->base.flags, req->base.complete, 3400 req->base.data); 3401 aead_request_set_crypt(subreq, req->src, req->dst, req->cryptlen, 3402 req->iv); 3403 aead_request_set_ad(subreq, req->assoclen); 3404 3405 return (dir == SAFEXCEL_ENCRYPT) ? 3406 crypto_aead_encrypt(subreq) : 3407 crypto_aead_decrypt(subreq); 3408} 3409 3410static int safexcel_aead_sm4cbc_sm3_encrypt(struct aead_request *req) 3411{ 3412 struct safexcel_cipher_req *creq = aead_request_ctx(req); 3413 3414 /* Workaround for HW bug: EIP96 4.3 does not report blocksize error */ 3415 if (req->cryptlen & (SM4_BLOCK_SIZE - 1)) 3416 return -EINVAL; 3417 else if (req->cryptlen || req->assoclen) /* If input length > 0 only */ 3418 return safexcel_queue_req(&req->base, creq, SAFEXCEL_ENCRYPT); 3419 3420 /* HW cannot do full (AAD+payload) zero length, use fallback */ 3421 return safexcel_aead_fallback_crypt(req, SAFEXCEL_ENCRYPT); 3422} 3423 3424static int safexcel_aead_sm4cbc_sm3_decrypt(struct aead_request *req) 3425{ 3426 struct safexcel_cipher_req *creq = aead_request_ctx(req); 3427 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 3428 3429 /* Workaround for HW bug: EIP96 4.3 does not report blocksize error */ 3430 if ((req->cryptlen - crypto_aead_authsize(tfm)) & (SM4_BLOCK_SIZE - 1)) 3431 return -EINVAL; 3432 else if (req->cryptlen > crypto_aead_authsize(tfm) || req->assoclen) 3433 /* If input length > 0 only */ 3434 return safexcel_queue_req(&req->base, creq, SAFEXCEL_DECRYPT); 3435 3436 /* HW cannot do full (AAD+payload) zero length, use fallback */ 3437 return safexcel_aead_fallback_crypt(req, SAFEXCEL_DECRYPT); 3438} 3439 3440static int safexcel_aead_sm4cbc_sm3_cra_init(struct crypto_tfm *tfm) 3441{ 3442 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 3443 3444 safexcel_aead_fallback_cra_init(tfm); 3445 ctx->alg = SAFEXCEL_SM4; 3446 ctx->blocksz = SM4_BLOCK_SIZE; 3447 ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_SM3; 3448 ctx->state_sz = SM3_DIGEST_SIZE; 3449 return 0; 3450} 3451 3452struct safexcel_alg_template safexcel_alg_authenc_hmac_sm3_cbc_sm4 = { 3453 .type = SAFEXCEL_ALG_TYPE_AEAD, 3454 .algo_mask = SAFEXCEL_ALG_SM4 | SAFEXCEL_ALG_SM3, 3455 .alg.aead = { 3456 .setkey = safexcel_aead_fallback_setkey, 3457 .setauthsize = safexcel_aead_fallback_setauthsize, 3458 .encrypt = safexcel_aead_sm4cbc_sm3_encrypt, 3459 .decrypt = safexcel_aead_sm4cbc_sm3_decrypt, 3460 .ivsize = SM4_BLOCK_SIZE, 3461 .maxauthsize = SM3_DIGEST_SIZE, 3462 .base = { 3463 .cra_name = "authenc(hmac(sm3),cbc(sm4))", 3464 .cra_driver_name = "safexcel-authenc-hmac-sm3-cbc-sm4", 3465 .cra_priority = SAFEXCEL_CRA_PRIORITY, 3466 .cra_flags = CRYPTO_ALG_ASYNC | 3467 CRYPTO_ALG_ALLOCATES_MEMORY | 3468 CRYPTO_ALG_KERN_DRIVER_ONLY | 3469 CRYPTO_ALG_NEED_FALLBACK, 3470 .cra_blocksize = SM4_BLOCK_SIZE, 3471 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 3472 .cra_alignmask = 0, 3473 .cra_init = safexcel_aead_sm4cbc_sm3_cra_init, 3474 .cra_exit = safexcel_aead_fallback_cra_exit, 3475 .cra_module = THIS_MODULE, 3476 }, 3477 }, 3478}; 3479 3480static int safexcel_aead_sm4ctr_sha1_cra_init(struct crypto_tfm *tfm) 3481{ 3482 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 3483 3484 safexcel_aead_sm4cbc_sha1_cra_init(tfm); 3485 ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD; 3486 return 0; 3487} 3488 3489struct safexcel_alg_template safexcel_alg_authenc_hmac_sha1_ctr_sm4 = { 3490 .type = SAFEXCEL_ALG_TYPE_AEAD, 3491 .algo_mask = SAFEXCEL_ALG_SM4 | SAFEXCEL_ALG_SHA1, 3492 .alg.aead = { 3493 .setkey = safexcel_aead_setkey, 3494 .encrypt = safexcel_aead_encrypt, 3495 .decrypt = safexcel_aead_decrypt, 3496 .ivsize = CTR_RFC3686_IV_SIZE, 3497 .maxauthsize = SHA1_DIGEST_SIZE, 3498 .base = { 3499 .cra_name = "authenc(hmac(sha1),rfc3686(ctr(sm4)))", 3500 .cra_driver_name = "safexcel-authenc-hmac-sha1-ctr-sm4", 3501 .cra_priority = SAFEXCEL_CRA_PRIORITY, 3502 .cra_flags = CRYPTO_ALG_ASYNC | 3503 CRYPTO_ALG_ALLOCATES_MEMORY | 3504 CRYPTO_ALG_KERN_DRIVER_ONLY, 3505 .cra_blocksize = 1, 3506 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 3507 .cra_alignmask = 0, 3508 .cra_init = safexcel_aead_sm4ctr_sha1_cra_init, 3509 .cra_exit = safexcel_aead_cra_exit, 3510 .cra_module = THIS_MODULE, 3511 }, 3512 }, 3513}; 3514 3515static int safexcel_aead_sm4ctr_sm3_cra_init(struct crypto_tfm *tfm) 3516{ 3517 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 3518 3519 safexcel_aead_sm4cbc_sm3_cra_init(tfm); 3520 ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD; 3521 return 0; 3522} 3523 3524struct safexcel_alg_template safexcel_alg_authenc_hmac_sm3_ctr_sm4 = { 3525 .type = SAFEXCEL_ALG_TYPE_AEAD, 3526 .algo_mask = SAFEXCEL_ALG_SM4 | SAFEXCEL_ALG_SM3, 3527 .alg.aead = { 3528 .setkey = safexcel_aead_setkey, 3529 .encrypt = safexcel_aead_encrypt, 3530 .decrypt = safexcel_aead_decrypt, 3531 .ivsize = CTR_RFC3686_IV_SIZE, 3532 .maxauthsize = SM3_DIGEST_SIZE, 3533 .base = { 3534 .cra_name = "authenc(hmac(sm3),rfc3686(ctr(sm4)))", 3535 .cra_driver_name = "safexcel-authenc-hmac-sm3-ctr-sm4", 3536 .cra_priority = SAFEXCEL_CRA_PRIORITY, 3537 .cra_flags = CRYPTO_ALG_ASYNC | 3538 CRYPTO_ALG_ALLOCATES_MEMORY | 3539 CRYPTO_ALG_KERN_DRIVER_ONLY, 3540 .cra_blocksize = 1, 3541 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 3542 .cra_alignmask = 0, 3543 .cra_init = safexcel_aead_sm4ctr_sm3_cra_init, 3544 .cra_exit = safexcel_aead_cra_exit, 3545 .cra_module = THIS_MODULE, 3546 }, 3547 }, 3548}; 3549 3550static int safexcel_rfc4106_gcm_setkey(struct crypto_aead *ctfm, const u8 *key, 3551 unsigned int len) 3552{ 3553 struct crypto_tfm *tfm = crypto_aead_tfm(ctfm); 3554 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 3555 3556 /* last 4 bytes of key are the nonce! */ 3557 ctx->nonce = *(u32 *)(key + len - CTR_RFC3686_NONCE_SIZE); 3558 3559 len -= CTR_RFC3686_NONCE_SIZE; 3560 return safexcel_aead_gcm_setkey(ctfm, key, len); 3561} 3562 3563static int safexcel_rfc4106_gcm_setauthsize(struct crypto_aead *tfm, 3564 unsigned int authsize) 3565{ 3566 return crypto_rfc4106_check_authsize(authsize); 3567} 3568 3569static int safexcel_rfc4106_encrypt(struct aead_request *req) 3570{ 3571 return crypto_ipsec_check_assoclen(req->assoclen) ?: 3572 safexcel_aead_encrypt(req); 3573} 3574 3575static int safexcel_rfc4106_decrypt(struct aead_request *req) 3576{ 3577 return crypto_ipsec_check_assoclen(req->assoclen) ?: 3578 safexcel_aead_decrypt(req); 3579} 3580 3581static int safexcel_rfc4106_gcm_cra_init(struct crypto_tfm *tfm) 3582{ 3583 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 3584 int ret; 3585 3586 ret = safexcel_aead_gcm_cra_init(tfm); 3587 ctx->aead = EIP197_AEAD_TYPE_IPSEC_ESP; 3588 ctx->aadskip = EIP197_AEAD_IPSEC_IV_SIZE; 3589 return ret; 3590} 3591 3592struct safexcel_alg_template safexcel_alg_rfc4106_gcm = { 3593 .type = SAFEXCEL_ALG_TYPE_AEAD, 3594 .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_GHASH, 3595 .alg.aead = { 3596 .setkey = safexcel_rfc4106_gcm_setkey, 3597 .setauthsize = safexcel_rfc4106_gcm_setauthsize, 3598 .encrypt = safexcel_rfc4106_encrypt, 3599 .decrypt = safexcel_rfc4106_decrypt, 3600 .ivsize = GCM_RFC4106_IV_SIZE, 3601 .maxauthsize = GHASH_DIGEST_SIZE, 3602 .base = { 3603 .cra_name = "rfc4106(gcm(aes))", 3604 .cra_driver_name = "safexcel-rfc4106-gcm-aes", 3605 .cra_priority = SAFEXCEL_CRA_PRIORITY, 3606 .cra_flags = CRYPTO_ALG_ASYNC | 3607 CRYPTO_ALG_ALLOCATES_MEMORY | 3608 CRYPTO_ALG_KERN_DRIVER_ONLY, 3609 .cra_blocksize = 1, 3610 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 3611 .cra_alignmask = 0, 3612 .cra_init = safexcel_rfc4106_gcm_cra_init, 3613 .cra_exit = safexcel_aead_gcm_cra_exit, 3614 }, 3615 }, 3616}; 3617 3618static int safexcel_rfc4543_gcm_setauthsize(struct crypto_aead *tfm, 3619 unsigned int authsize) 3620{ 3621 if (authsize != GHASH_DIGEST_SIZE) 3622 return -EINVAL; 3623 3624 return 0; 3625} 3626 3627static int safexcel_rfc4543_gcm_cra_init(struct crypto_tfm *tfm) 3628{ 3629 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 3630 int ret; 3631 3632 ret = safexcel_aead_gcm_cra_init(tfm); 3633 ctx->aead = EIP197_AEAD_TYPE_IPSEC_ESP_GMAC; 3634 return ret; 3635} 3636 3637struct safexcel_alg_template safexcel_alg_rfc4543_gcm = { 3638 .type = SAFEXCEL_ALG_TYPE_AEAD, 3639 .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_GHASH, 3640 .alg.aead = { 3641 .setkey = safexcel_rfc4106_gcm_setkey, 3642 .setauthsize = safexcel_rfc4543_gcm_setauthsize, 3643 .encrypt = safexcel_rfc4106_encrypt, 3644 .decrypt = safexcel_rfc4106_decrypt, 3645 .ivsize = GCM_RFC4543_IV_SIZE, 3646 .maxauthsize = GHASH_DIGEST_SIZE, 3647 .base = { 3648 .cra_name = "rfc4543(gcm(aes))", 3649 .cra_driver_name = "safexcel-rfc4543-gcm-aes", 3650 .cra_priority = SAFEXCEL_CRA_PRIORITY, 3651 .cra_flags = CRYPTO_ALG_ASYNC | 3652 CRYPTO_ALG_ALLOCATES_MEMORY | 3653 CRYPTO_ALG_KERN_DRIVER_ONLY, 3654 .cra_blocksize = 1, 3655 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 3656 .cra_alignmask = 0, 3657 .cra_init = safexcel_rfc4543_gcm_cra_init, 3658 .cra_exit = safexcel_aead_gcm_cra_exit, 3659 }, 3660 }, 3661}; 3662 3663static int safexcel_rfc4309_ccm_setkey(struct crypto_aead *ctfm, const u8 *key, 3664 unsigned int len) 3665{ 3666 struct crypto_tfm *tfm = crypto_aead_tfm(ctfm); 3667 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 3668 3669 /* First byte of the nonce = L = always 3 for RFC4309 (4 byte ctr) */ 3670 *(u8 *)&ctx->nonce = EIP197_AEAD_IPSEC_COUNTER_SIZE - 1; 3671 /* last 3 bytes of key are the nonce! */ 3672 memcpy((u8 *)&ctx->nonce + 1, key + len - 3673 EIP197_AEAD_IPSEC_CCM_NONCE_SIZE, 3674 EIP197_AEAD_IPSEC_CCM_NONCE_SIZE); 3675 3676 len -= EIP197_AEAD_IPSEC_CCM_NONCE_SIZE; 3677 return safexcel_aead_ccm_setkey(ctfm, key, len); 3678} 3679 3680static int safexcel_rfc4309_ccm_setauthsize(struct crypto_aead *tfm, 3681 unsigned int authsize) 3682{ 3683 /* Borrowed from crypto/ccm.c */ 3684 switch (authsize) { 3685 case 8: 3686 case 12: 3687 case 16: 3688 break; 3689 default: 3690 return -EINVAL; 3691 } 3692 3693 return 0; 3694} 3695 3696static int safexcel_rfc4309_ccm_encrypt(struct aead_request *req) 3697{ 3698 struct safexcel_cipher_req *creq = aead_request_ctx(req); 3699 3700 /* Borrowed from crypto/ccm.c */ 3701 if (req->assoclen != 16 && req->assoclen != 20) 3702 return -EINVAL; 3703 3704 return safexcel_queue_req(&req->base, creq, SAFEXCEL_ENCRYPT); 3705} 3706 3707static int safexcel_rfc4309_ccm_decrypt(struct aead_request *req) 3708{ 3709 struct safexcel_cipher_req *creq = aead_request_ctx(req); 3710 3711 /* Borrowed from crypto/ccm.c */ 3712 if (req->assoclen != 16 && req->assoclen != 20) 3713 return -EINVAL; 3714 3715 return safexcel_queue_req(&req->base, creq, SAFEXCEL_DECRYPT); 3716} 3717 3718static int safexcel_rfc4309_ccm_cra_init(struct crypto_tfm *tfm) 3719{ 3720 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 3721 int ret; 3722 3723 ret = safexcel_aead_ccm_cra_init(tfm); 3724 ctx->aead = EIP197_AEAD_TYPE_IPSEC_ESP; 3725 ctx->aadskip = EIP197_AEAD_IPSEC_IV_SIZE; 3726 return ret; 3727} 3728 3729struct safexcel_alg_template safexcel_alg_rfc4309_ccm = { 3730 .type = SAFEXCEL_ALG_TYPE_AEAD, 3731 .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_CBC_MAC_ALL, 3732 .alg.aead = { 3733 .setkey = safexcel_rfc4309_ccm_setkey, 3734 .setauthsize = safexcel_rfc4309_ccm_setauthsize, 3735 .encrypt = safexcel_rfc4309_ccm_encrypt, 3736 .decrypt = safexcel_rfc4309_ccm_decrypt, 3737 .ivsize = EIP197_AEAD_IPSEC_IV_SIZE, 3738 .maxauthsize = AES_BLOCK_SIZE, 3739 .base = { 3740 .cra_name = "rfc4309(ccm(aes))", 3741 .cra_driver_name = "safexcel-rfc4309-ccm-aes", 3742 .cra_priority = SAFEXCEL_CRA_PRIORITY, 3743 .cra_flags = CRYPTO_ALG_ASYNC | 3744 CRYPTO_ALG_ALLOCATES_MEMORY | 3745 CRYPTO_ALG_KERN_DRIVER_ONLY, 3746 .cra_blocksize = 1, 3747 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 3748 .cra_alignmask = 0, 3749 .cra_init = safexcel_rfc4309_ccm_cra_init, 3750 .cra_exit = safexcel_aead_cra_exit, 3751 .cra_module = THIS_MODULE, 3752 }, 3753 }, 3754}; 3755