1// SPDX-License-Identifier: GPL-2.0+ 2/* 3 * caam - Freescale FSL CAAM support for crypto API 4 * 5 * Copyright 2008-2011 Freescale Semiconductor, Inc. 6 * Copyright 2016-2019 NXP 7 * 8 * Based on talitos crypto API driver. 9 * 10 * relationship of job descriptors to shared descriptors (SteveC Dec 10 2008): 11 * 12 * --------------- --------------- 13 * | JobDesc #1 |-------------------->| ShareDesc | 14 * | *(packet 1) | | (PDB) | 15 * --------------- |------------->| (hashKey) | 16 * . | | (cipherKey) | 17 * . | |-------->| (operation) | 18 * --------------- | | --------------- 19 * | JobDesc #2 |------| | 20 * | *(packet 2) | | 21 * --------------- | 22 * . | 23 * . | 24 * --------------- | 25 * | JobDesc #3 |------------ 26 * | *(packet 3) | 27 * --------------- 28 * 29 * The SharedDesc never changes for a connection unless rekeyed, but 30 * each packet will likely be in a different place. So all we need 31 * to know to process the packet is where the input is, where the 32 * output goes, and what context we want to process with. Context is 33 * in the SharedDesc, packet references in the JobDesc. 34 * 35 * So, a job desc looks like: 36 * 37 * --------------------- 38 * | Header | 39 * | ShareDesc Pointer | 40 * | SEQ_OUT_PTR | 41 * | (output buffer) | 42 * | (output length) | 43 * | SEQ_IN_PTR | 44 * | (input buffer) | 45 * | (input length) | 46 * --------------------- 47 */ 48 49#include "compat.h" 50 51#include "regs.h" 52#include "intern.h" 53#include "desc_constr.h" 54#include "jr.h" 55#include "error.h" 56#include "sg_sw_sec4.h" 57#include "key_gen.h" 58#include "caamalg_desc.h" 59#include <crypto/engine.h> 60#include <crypto/xts.h> 61#include <asm/unaligned.h> 62 63/* 64 * crypto alg 65 */ 66#define CAAM_CRA_PRIORITY 3000 67/* max key is sum of AES_MAX_KEY_SIZE, max split key size */ 68#define CAAM_MAX_KEY_SIZE (AES_MAX_KEY_SIZE + \ 69 CTR_RFC3686_NONCE_SIZE + \ 70 SHA512_DIGEST_SIZE * 2) 71 72#define AEAD_DESC_JOB_IO_LEN (DESC_JOB_IO_LEN + CAAM_CMD_SZ * 2) 73#define GCM_DESC_JOB_IO_LEN (AEAD_DESC_JOB_IO_LEN + \ 74 CAAM_CMD_SZ * 4) 75#define AUTHENC_DESC_JOB_IO_LEN (AEAD_DESC_JOB_IO_LEN + \ 76 CAAM_CMD_SZ * 5) 77 78#define CHACHAPOLY_DESC_JOB_IO_LEN (AEAD_DESC_JOB_IO_LEN + CAAM_CMD_SZ * 6) 79 80#define DESC_MAX_USED_BYTES (CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN_MIN) 81#define DESC_MAX_USED_LEN (DESC_MAX_USED_BYTES / CAAM_CMD_SZ) 82 83struct caam_alg_entry { 84 int class1_alg_type; 85 int class2_alg_type; 86 bool rfc3686; 87 bool geniv; 88 bool nodkp; 89}; 90 91struct caam_aead_alg { 92 struct aead_alg aead; 93 struct caam_alg_entry caam; 94 bool registered; 95}; 96 97struct caam_skcipher_alg { 98 struct skcipher_alg skcipher; 99 struct caam_alg_entry caam; 100 bool registered; 101}; 102 103/* 104 * per-session context 105 */ 106struct caam_ctx { 107 struct crypto_engine_ctx enginectx; 108 u32 sh_desc_enc[DESC_MAX_USED_LEN]; 109 u32 sh_desc_dec[DESC_MAX_USED_LEN]; 110 u8 key[CAAM_MAX_KEY_SIZE]; 111 dma_addr_t sh_desc_enc_dma; 112 dma_addr_t sh_desc_dec_dma; 113 dma_addr_t key_dma; 114 enum dma_data_direction dir; 115 struct device *jrdev; 116 struct alginfo adata; 117 struct alginfo cdata; 118 unsigned int authsize; 119 bool xts_key_fallback; 120 struct crypto_skcipher *fallback; 121}; 122 123struct caam_skcipher_req_ctx { 124 struct skcipher_edesc *edesc; 125 struct skcipher_request fallback_req; 126}; 127 128struct caam_aead_req_ctx { 129 struct aead_edesc *edesc; 130}; 131 132static int aead_null_set_sh_desc(struct crypto_aead *aead) 133{ 134 struct caam_ctx *ctx = crypto_aead_ctx(aead); 135 struct device *jrdev = ctx->jrdev; 136 struct caam_drv_private *ctrlpriv = dev_get_drvdata(jrdev->parent); 137 u32 *desc; 138 int rem_bytes = CAAM_DESC_BYTES_MAX - AEAD_DESC_JOB_IO_LEN - 139 ctx->adata.keylen_pad; 140 141 /* 142 * Job Descriptor and Shared Descriptors 143 * must all fit into the 64-word Descriptor h/w Buffer 144 */ 145 if (rem_bytes >= DESC_AEAD_NULL_ENC_LEN) { 146 ctx->adata.key_inline = true; 147 ctx->adata.key_virt = ctx->key; 148 } else { 149 ctx->adata.key_inline = false; 150 ctx->adata.key_dma = ctx->key_dma; 151 } 152 153 /* aead_encrypt shared descriptor */ 154 desc = ctx->sh_desc_enc; 155 cnstr_shdsc_aead_null_encap(desc, &ctx->adata, ctx->authsize, 156 ctrlpriv->era); 157 dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma, 158 desc_bytes(desc), ctx->dir); 159 160 /* 161 * Job Descriptor and Shared Descriptors 162 * must all fit into the 64-word Descriptor h/w Buffer 163 */ 164 if (rem_bytes >= DESC_AEAD_NULL_DEC_LEN) { 165 ctx->adata.key_inline = true; 166 ctx->adata.key_virt = ctx->key; 167 } else { 168 ctx->adata.key_inline = false; 169 ctx->adata.key_dma = ctx->key_dma; 170 } 171 172 /* aead_decrypt shared descriptor */ 173 desc = ctx->sh_desc_dec; 174 cnstr_shdsc_aead_null_decap(desc, &ctx->adata, ctx->authsize, 175 ctrlpriv->era); 176 dma_sync_single_for_device(jrdev, ctx->sh_desc_dec_dma, 177 desc_bytes(desc), ctx->dir); 178 179 return 0; 180} 181 182static int aead_set_sh_desc(struct crypto_aead *aead) 183{ 184 struct caam_aead_alg *alg = container_of(crypto_aead_alg(aead), 185 struct caam_aead_alg, aead); 186 unsigned int ivsize = crypto_aead_ivsize(aead); 187 struct caam_ctx *ctx = crypto_aead_ctx(aead); 188 struct device *jrdev = ctx->jrdev; 189 struct caam_drv_private *ctrlpriv = dev_get_drvdata(jrdev->parent); 190 u32 ctx1_iv_off = 0; 191 u32 *desc, *nonce = NULL; 192 u32 inl_mask; 193 unsigned int data_len[2]; 194 const bool ctr_mode = ((ctx->cdata.algtype & OP_ALG_AAI_MASK) == 195 OP_ALG_AAI_CTR_MOD128); 196 const bool is_rfc3686 = alg->caam.rfc3686; 197 198 if (!ctx->authsize) 199 return 0; 200 201 /* NULL encryption / decryption */ 202 if (!ctx->cdata.keylen) 203 return aead_null_set_sh_desc(aead); 204 205 /* 206 * AES-CTR needs to load IV in CONTEXT1 reg 207 * at an offset of 128bits (16bytes) 208 * CONTEXT1[255:128] = IV 209 */ 210 if (ctr_mode) 211 ctx1_iv_off = 16; 212 213 /* 214 * RFC3686 specific: 215 * CONTEXT1[255:128] = {NONCE, IV, COUNTER} 216 */ 217 if (is_rfc3686) { 218 ctx1_iv_off = 16 + CTR_RFC3686_NONCE_SIZE; 219 nonce = (u32 *)((void *)ctx->key + ctx->adata.keylen_pad + 220 ctx->cdata.keylen - CTR_RFC3686_NONCE_SIZE); 221 } 222 223 /* 224 * In case |user key| > |derived key|, using DKP<imm,imm> 225 * would result in invalid opcodes (last bytes of user key) in 226 * the resulting descriptor. Use DKP<ptr,imm> instead => both 227 * virtual and dma key addresses are needed. 228 */ 229 ctx->adata.key_virt = ctx->key; 230 ctx->adata.key_dma = ctx->key_dma; 231 232 ctx->cdata.key_virt = ctx->key + ctx->adata.keylen_pad; 233 ctx->cdata.key_dma = ctx->key_dma + ctx->adata.keylen_pad; 234 235 data_len[0] = ctx->adata.keylen_pad; 236 data_len[1] = ctx->cdata.keylen; 237 238 if (alg->caam.geniv) 239 goto skip_enc; 240 241 /* 242 * Job Descriptor and Shared Descriptors 243 * must all fit into the 64-word Descriptor h/w Buffer 244 */ 245 if (desc_inline_query(DESC_AEAD_ENC_LEN + 246 (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0), 247 AUTHENC_DESC_JOB_IO_LEN, data_len, &inl_mask, 248 ARRAY_SIZE(data_len)) < 0) 249 return -EINVAL; 250 251 ctx->adata.key_inline = !!(inl_mask & 1); 252 ctx->cdata.key_inline = !!(inl_mask & 2); 253 254 /* aead_encrypt shared descriptor */ 255 desc = ctx->sh_desc_enc; 256 cnstr_shdsc_aead_encap(desc, &ctx->cdata, &ctx->adata, ivsize, 257 ctx->authsize, is_rfc3686, nonce, ctx1_iv_off, 258 false, ctrlpriv->era); 259 dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma, 260 desc_bytes(desc), ctx->dir); 261 262skip_enc: 263 /* 264 * Job Descriptor and Shared Descriptors 265 * must all fit into the 64-word Descriptor h/w Buffer 266 */ 267 if (desc_inline_query(DESC_AEAD_DEC_LEN + 268 (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0), 269 AUTHENC_DESC_JOB_IO_LEN, data_len, &inl_mask, 270 ARRAY_SIZE(data_len)) < 0) 271 return -EINVAL; 272 273 ctx->adata.key_inline = !!(inl_mask & 1); 274 ctx->cdata.key_inline = !!(inl_mask & 2); 275 276 /* aead_decrypt shared descriptor */ 277 desc = ctx->sh_desc_dec; 278 cnstr_shdsc_aead_decap(desc, &ctx->cdata, &ctx->adata, ivsize, 279 ctx->authsize, alg->caam.geniv, is_rfc3686, 280 nonce, ctx1_iv_off, false, ctrlpriv->era); 281 dma_sync_single_for_device(jrdev, ctx->sh_desc_dec_dma, 282 desc_bytes(desc), ctx->dir); 283 284 if (!alg->caam.geniv) 285 goto skip_givenc; 286 287 /* 288 * Job Descriptor and Shared Descriptors 289 * must all fit into the 64-word Descriptor h/w Buffer 290 */ 291 if (desc_inline_query(DESC_AEAD_GIVENC_LEN + 292 (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0), 293 AUTHENC_DESC_JOB_IO_LEN, data_len, &inl_mask, 294 ARRAY_SIZE(data_len)) < 0) 295 return -EINVAL; 296 297 ctx->adata.key_inline = !!(inl_mask & 1); 298 ctx->cdata.key_inline = !!(inl_mask & 2); 299 300 /* aead_givencrypt shared descriptor */ 301 desc = ctx->sh_desc_enc; 302 cnstr_shdsc_aead_givencap(desc, &ctx->cdata, &ctx->adata, ivsize, 303 ctx->authsize, is_rfc3686, nonce, 304 ctx1_iv_off, false, ctrlpriv->era); 305 dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma, 306 desc_bytes(desc), ctx->dir); 307 308skip_givenc: 309 return 0; 310} 311 312static int aead_setauthsize(struct crypto_aead *authenc, 313 unsigned int authsize) 314{ 315 struct caam_ctx *ctx = crypto_aead_ctx(authenc); 316 317 ctx->authsize = authsize; 318 aead_set_sh_desc(authenc); 319 320 return 0; 321} 322 323static int gcm_set_sh_desc(struct crypto_aead *aead) 324{ 325 struct caam_ctx *ctx = crypto_aead_ctx(aead); 326 struct device *jrdev = ctx->jrdev; 327 unsigned int ivsize = crypto_aead_ivsize(aead); 328 u32 *desc; 329 int rem_bytes = CAAM_DESC_BYTES_MAX - GCM_DESC_JOB_IO_LEN - 330 ctx->cdata.keylen; 331 332 if (!ctx->cdata.keylen || !ctx->authsize) 333 return 0; 334 335 /* 336 * AES GCM encrypt shared descriptor 337 * Job Descriptor and Shared Descriptor 338 * must fit into the 64-word Descriptor h/w Buffer 339 */ 340 if (rem_bytes >= DESC_GCM_ENC_LEN) { 341 ctx->cdata.key_inline = true; 342 ctx->cdata.key_virt = ctx->key; 343 } else { 344 ctx->cdata.key_inline = false; 345 ctx->cdata.key_dma = ctx->key_dma; 346 } 347 348 desc = ctx->sh_desc_enc; 349 cnstr_shdsc_gcm_encap(desc, &ctx->cdata, ivsize, ctx->authsize, false); 350 dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma, 351 desc_bytes(desc), ctx->dir); 352 353 /* 354 * Job Descriptor and Shared Descriptors 355 * must all fit into the 64-word Descriptor h/w Buffer 356 */ 357 if (rem_bytes >= DESC_GCM_DEC_LEN) { 358 ctx->cdata.key_inline = true; 359 ctx->cdata.key_virt = ctx->key; 360 } else { 361 ctx->cdata.key_inline = false; 362 ctx->cdata.key_dma = ctx->key_dma; 363 } 364 365 desc = ctx->sh_desc_dec; 366 cnstr_shdsc_gcm_decap(desc, &ctx->cdata, ivsize, ctx->authsize, false); 367 dma_sync_single_for_device(jrdev, ctx->sh_desc_dec_dma, 368 desc_bytes(desc), ctx->dir); 369 370 return 0; 371} 372 373static int gcm_setauthsize(struct crypto_aead *authenc, unsigned int authsize) 374{ 375 struct caam_ctx *ctx = crypto_aead_ctx(authenc); 376 int err; 377 378 err = crypto_gcm_check_authsize(authsize); 379 if (err) 380 return err; 381 382 ctx->authsize = authsize; 383 gcm_set_sh_desc(authenc); 384 385 return 0; 386} 387 388static int rfc4106_set_sh_desc(struct crypto_aead *aead) 389{ 390 struct caam_ctx *ctx = crypto_aead_ctx(aead); 391 struct device *jrdev = ctx->jrdev; 392 unsigned int ivsize = crypto_aead_ivsize(aead); 393 u32 *desc; 394 int rem_bytes = CAAM_DESC_BYTES_MAX - GCM_DESC_JOB_IO_LEN - 395 ctx->cdata.keylen; 396 397 if (!ctx->cdata.keylen || !ctx->authsize) 398 return 0; 399 400 /* 401 * RFC4106 encrypt shared descriptor 402 * Job Descriptor and Shared Descriptor 403 * must fit into the 64-word Descriptor h/w Buffer 404 */ 405 if (rem_bytes >= DESC_RFC4106_ENC_LEN) { 406 ctx->cdata.key_inline = true; 407 ctx->cdata.key_virt = ctx->key; 408 } else { 409 ctx->cdata.key_inline = false; 410 ctx->cdata.key_dma = ctx->key_dma; 411 } 412 413 desc = ctx->sh_desc_enc; 414 cnstr_shdsc_rfc4106_encap(desc, &ctx->cdata, ivsize, ctx->authsize, 415 false); 416 dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma, 417 desc_bytes(desc), ctx->dir); 418 419 /* 420 * Job Descriptor and Shared Descriptors 421 * must all fit into the 64-word Descriptor h/w Buffer 422 */ 423 if (rem_bytes >= DESC_RFC4106_DEC_LEN) { 424 ctx->cdata.key_inline = true; 425 ctx->cdata.key_virt = ctx->key; 426 } else { 427 ctx->cdata.key_inline = false; 428 ctx->cdata.key_dma = ctx->key_dma; 429 } 430 431 desc = ctx->sh_desc_dec; 432 cnstr_shdsc_rfc4106_decap(desc, &ctx->cdata, ivsize, ctx->authsize, 433 false); 434 dma_sync_single_for_device(jrdev, ctx->sh_desc_dec_dma, 435 desc_bytes(desc), ctx->dir); 436 437 return 0; 438} 439 440static int rfc4106_setauthsize(struct crypto_aead *authenc, 441 unsigned int authsize) 442{ 443 struct caam_ctx *ctx = crypto_aead_ctx(authenc); 444 int err; 445 446 err = crypto_rfc4106_check_authsize(authsize); 447 if (err) 448 return err; 449 450 ctx->authsize = authsize; 451 rfc4106_set_sh_desc(authenc); 452 453 return 0; 454} 455 456static int rfc4543_set_sh_desc(struct crypto_aead *aead) 457{ 458 struct caam_ctx *ctx = crypto_aead_ctx(aead); 459 struct device *jrdev = ctx->jrdev; 460 unsigned int ivsize = crypto_aead_ivsize(aead); 461 u32 *desc; 462 int rem_bytes = CAAM_DESC_BYTES_MAX - GCM_DESC_JOB_IO_LEN - 463 ctx->cdata.keylen; 464 465 if (!ctx->cdata.keylen || !ctx->authsize) 466 return 0; 467 468 /* 469 * RFC4543 encrypt shared descriptor 470 * Job Descriptor and Shared Descriptor 471 * must fit into the 64-word Descriptor h/w Buffer 472 */ 473 if (rem_bytes >= DESC_RFC4543_ENC_LEN) { 474 ctx->cdata.key_inline = true; 475 ctx->cdata.key_virt = ctx->key; 476 } else { 477 ctx->cdata.key_inline = false; 478 ctx->cdata.key_dma = ctx->key_dma; 479 } 480 481 desc = ctx->sh_desc_enc; 482 cnstr_shdsc_rfc4543_encap(desc, &ctx->cdata, ivsize, ctx->authsize, 483 false); 484 dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma, 485 desc_bytes(desc), ctx->dir); 486 487 /* 488 * Job Descriptor and Shared Descriptors 489 * must all fit into the 64-word Descriptor h/w Buffer 490 */ 491 if (rem_bytes >= DESC_RFC4543_DEC_LEN) { 492 ctx->cdata.key_inline = true; 493 ctx->cdata.key_virt = ctx->key; 494 } else { 495 ctx->cdata.key_inline = false; 496 ctx->cdata.key_dma = ctx->key_dma; 497 } 498 499 desc = ctx->sh_desc_dec; 500 cnstr_shdsc_rfc4543_decap(desc, &ctx->cdata, ivsize, ctx->authsize, 501 false); 502 dma_sync_single_for_device(jrdev, ctx->sh_desc_dec_dma, 503 desc_bytes(desc), ctx->dir); 504 505 return 0; 506} 507 508static int rfc4543_setauthsize(struct crypto_aead *authenc, 509 unsigned int authsize) 510{ 511 struct caam_ctx *ctx = crypto_aead_ctx(authenc); 512 513 if (authsize != 16) 514 return -EINVAL; 515 516 ctx->authsize = authsize; 517 rfc4543_set_sh_desc(authenc); 518 519 return 0; 520} 521 522static int chachapoly_set_sh_desc(struct crypto_aead *aead) 523{ 524 struct caam_ctx *ctx = crypto_aead_ctx(aead); 525 struct device *jrdev = ctx->jrdev; 526 unsigned int ivsize = crypto_aead_ivsize(aead); 527 u32 *desc; 528 529 if (!ctx->cdata.keylen || !ctx->authsize) 530 return 0; 531 532 desc = ctx->sh_desc_enc; 533 cnstr_shdsc_chachapoly(desc, &ctx->cdata, &ctx->adata, ivsize, 534 ctx->authsize, true, false); 535 dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma, 536 desc_bytes(desc), ctx->dir); 537 538 desc = ctx->sh_desc_dec; 539 cnstr_shdsc_chachapoly(desc, &ctx->cdata, &ctx->adata, ivsize, 540 ctx->authsize, false, false); 541 dma_sync_single_for_device(jrdev, ctx->sh_desc_dec_dma, 542 desc_bytes(desc), ctx->dir); 543 544 return 0; 545} 546 547static int chachapoly_setauthsize(struct crypto_aead *aead, 548 unsigned int authsize) 549{ 550 struct caam_ctx *ctx = crypto_aead_ctx(aead); 551 552 if (authsize != POLY1305_DIGEST_SIZE) 553 return -EINVAL; 554 555 ctx->authsize = authsize; 556 return chachapoly_set_sh_desc(aead); 557} 558 559static int chachapoly_setkey(struct crypto_aead *aead, const u8 *key, 560 unsigned int keylen) 561{ 562 struct caam_ctx *ctx = crypto_aead_ctx(aead); 563 unsigned int ivsize = crypto_aead_ivsize(aead); 564 unsigned int saltlen = CHACHAPOLY_IV_SIZE - ivsize; 565 566 if (keylen != CHACHA_KEY_SIZE + saltlen) 567 return -EINVAL; 568 569 memcpy(ctx->key, key, keylen); 570 ctx->cdata.key_virt = ctx->key; 571 ctx->cdata.keylen = keylen - saltlen; 572 573 return chachapoly_set_sh_desc(aead); 574} 575 576static int aead_setkey(struct crypto_aead *aead, 577 const u8 *key, unsigned int keylen) 578{ 579 struct caam_ctx *ctx = crypto_aead_ctx(aead); 580 struct device *jrdev = ctx->jrdev; 581 struct caam_drv_private *ctrlpriv = dev_get_drvdata(jrdev->parent); 582 struct crypto_authenc_keys keys; 583 int ret = 0; 584 585 if (crypto_authenc_extractkeys(&keys, key, keylen) != 0) 586 goto badkey; 587 588 dev_dbg(jrdev, "keylen %d enckeylen %d authkeylen %d\n", 589 keys.authkeylen + keys.enckeylen, keys.enckeylen, 590 keys.authkeylen); 591 print_hex_dump_debug("key in @"__stringify(__LINE__)": ", 592 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); 593 594 /* 595 * If DKP is supported, use it in the shared descriptor to generate 596 * the split key. 597 */ 598 if (ctrlpriv->era >= 6) { 599 ctx->adata.keylen = keys.authkeylen; 600 ctx->adata.keylen_pad = split_key_len(ctx->adata.algtype & 601 OP_ALG_ALGSEL_MASK); 602 603 if (ctx->adata.keylen_pad + keys.enckeylen > CAAM_MAX_KEY_SIZE) 604 goto badkey; 605 606 memcpy(ctx->key, keys.authkey, keys.authkeylen); 607 memcpy(ctx->key + ctx->adata.keylen_pad, keys.enckey, 608 keys.enckeylen); 609 dma_sync_single_for_device(jrdev, ctx->key_dma, 610 ctx->adata.keylen_pad + 611 keys.enckeylen, ctx->dir); 612 goto skip_split_key; 613 } 614 615 ret = gen_split_key(ctx->jrdev, ctx->key, &ctx->adata, keys.authkey, 616 keys.authkeylen, CAAM_MAX_KEY_SIZE - 617 keys.enckeylen); 618 if (ret) { 619 goto badkey; 620 } 621 622 /* postpend encryption key to auth split key */ 623 memcpy(ctx->key + ctx->adata.keylen_pad, keys.enckey, keys.enckeylen); 624 dma_sync_single_for_device(jrdev, ctx->key_dma, ctx->adata.keylen_pad + 625 keys.enckeylen, ctx->dir); 626 627 print_hex_dump_debug("ctx.key@"__stringify(__LINE__)": ", 628 DUMP_PREFIX_ADDRESS, 16, 4, ctx->key, 629 ctx->adata.keylen_pad + keys.enckeylen, 1); 630 631skip_split_key: 632 ctx->cdata.keylen = keys.enckeylen; 633 memzero_explicit(&keys, sizeof(keys)); 634 return aead_set_sh_desc(aead); 635badkey: 636 memzero_explicit(&keys, sizeof(keys)); 637 return -EINVAL; 638} 639 640static int des3_aead_setkey(struct crypto_aead *aead, const u8 *key, 641 unsigned int keylen) 642{ 643 struct crypto_authenc_keys keys; 644 int err; 645 646 err = crypto_authenc_extractkeys(&keys, key, keylen); 647 if (unlikely(err)) 648 return err; 649 650 err = verify_aead_des3_key(aead, keys.enckey, keys.enckeylen) ?: 651 aead_setkey(aead, key, keylen); 652 653 memzero_explicit(&keys, sizeof(keys)); 654 return err; 655} 656 657static int gcm_setkey(struct crypto_aead *aead, 658 const u8 *key, unsigned int keylen) 659{ 660 struct caam_ctx *ctx = crypto_aead_ctx(aead); 661 struct device *jrdev = ctx->jrdev; 662 int err; 663 664 err = aes_check_keylen(keylen); 665 if (err) 666 return err; 667 668 print_hex_dump_debug("key in @"__stringify(__LINE__)": ", 669 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); 670 671 memcpy(ctx->key, key, keylen); 672 dma_sync_single_for_device(jrdev, ctx->key_dma, keylen, ctx->dir); 673 ctx->cdata.keylen = keylen; 674 675 return gcm_set_sh_desc(aead); 676} 677 678static int rfc4106_setkey(struct crypto_aead *aead, 679 const u8 *key, unsigned int keylen) 680{ 681 struct caam_ctx *ctx = crypto_aead_ctx(aead); 682 struct device *jrdev = ctx->jrdev; 683 int err; 684 685 err = aes_check_keylen(keylen - 4); 686 if (err) 687 return err; 688 689 print_hex_dump_debug("key in @"__stringify(__LINE__)": ", 690 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); 691 692 memcpy(ctx->key, key, keylen); 693 694 /* 695 * The last four bytes of the key material are used as the salt value 696 * in the nonce. Update the AES key length. 697 */ 698 ctx->cdata.keylen = keylen - 4; 699 dma_sync_single_for_device(jrdev, ctx->key_dma, ctx->cdata.keylen, 700 ctx->dir); 701 return rfc4106_set_sh_desc(aead); 702} 703 704static int rfc4543_setkey(struct crypto_aead *aead, 705 const u8 *key, unsigned int keylen) 706{ 707 struct caam_ctx *ctx = crypto_aead_ctx(aead); 708 struct device *jrdev = ctx->jrdev; 709 int err; 710 711 err = aes_check_keylen(keylen - 4); 712 if (err) 713 return err; 714 715 print_hex_dump_debug("key in @"__stringify(__LINE__)": ", 716 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); 717 718 memcpy(ctx->key, key, keylen); 719 720 /* 721 * The last four bytes of the key material are used as the salt value 722 * in the nonce. Update the AES key length. 723 */ 724 ctx->cdata.keylen = keylen - 4; 725 dma_sync_single_for_device(jrdev, ctx->key_dma, ctx->cdata.keylen, 726 ctx->dir); 727 return rfc4543_set_sh_desc(aead); 728} 729 730static int skcipher_setkey(struct crypto_skcipher *skcipher, const u8 *key, 731 unsigned int keylen, const u32 ctx1_iv_off) 732{ 733 struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher); 734 struct caam_skcipher_alg *alg = 735 container_of(crypto_skcipher_alg(skcipher), typeof(*alg), 736 skcipher); 737 struct device *jrdev = ctx->jrdev; 738 unsigned int ivsize = crypto_skcipher_ivsize(skcipher); 739 u32 *desc; 740 const bool is_rfc3686 = alg->caam.rfc3686; 741 742 print_hex_dump_debug("key in @"__stringify(__LINE__)": ", 743 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); 744 745 ctx->cdata.keylen = keylen; 746 ctx->cdata.key_virt = key; 747 ctx->cdata.key_inline = true; 748 749 /* skcipher_encrypt shared descriptor */ 750 desc = ctx->sh_desc_enc; 751 cnstr_shdsc_skcipher_encap(desc, &ctx->cdata, ivsize, is_rfc3686, 752 ctx1_iv_off); 753 dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma, 754 desc_bytes(desc), ctx->dir); 755 756 /* skcipher_decrypt shared descriptor */ 757 desc = ctx->sh_desc_dec; 758 cnstr_shdsc_skcipher_decap(desc, &ctx->cdata, ivsize, is_rfc3686, 759 ctx1_iv_off); 760 dma_sync_single_for_device(jrdev, ctx->sh_desc_dec_dma, 761 desc_bytes(desc), ctx->dir); 762 763 return 0; 764} 765 766static int aes_skcipher_setkey(struct crypto_skcipher *skcipher, 767 const u8 *key, unsigned int keylen) 768{ 769 int err; 770 771 err = aes_check_keylen(keylen); 772 if (err) 773 return err; 774 775 return skcipher_setkey(skcipher, key, keylen, 0); 776} 777 778static int rfc3686_skcipher_setkey(struct crypto_skcipher *skcipher, 779 const u8 *key, unsigned int keylen) 780{ 781 u32 ctx1_iv_off; 782 int err; 783 784 /* 785 * RFC3686 specific: 786 * | CONTEXT1[255:128] = {NONCE, IV, COUNTER} 787 * | *key = {KEY, NONCE} 788 */ 789 ctx1_iv_off = 16 + CTR_RFC3686_NONCE_SIZE; 790 keylen -= CTR_RFC3686_NONCE_SIZE; 791 792 err = aes_check_keylen(keylen); 793 if (err) 794 return err; 795 796 return skcipher_setkey(skcipher, key, keylen, ctx1_iv_off); 797} 798 799static int ctr_skcipher_setkey(struct crypto_skcipher *skcipher, 800 const u8 *key, unsigned int keylen) 801{ 802 u32 ctx1_iv_off; 803 int err; 804 805 /* 806 * AES-CTR needs to load IV in CONTEXT1 reg 807 * at an offset of 128bits (16bytes) 808 * CONTEXT1[255:128] = IV 809 */ 810 ctx1_iv_off = 16; 811 812 err = aes_check_keylen(keylen); 813 if (err) 814 return err; 815 816 return skcipher_setkey(skcipher, key, keylen, ctx1_iv_off); 817} 818 819static int des_skcipher_setkey(struct crypto_skcipher *skcipher, 820 const u8 *key, unsigned int keylen) 821{ 822 return verify_skcipher_des_key(skcipher, key) ?: 823 skcipher_setkey(skcipher, key, keylen, 0); 824} 825 826static int des3_skcipher_setkey(struct crypto_skcipher *skcipher, 827 const u8 *key, unsigned int keylen) 828{ 829 return verify_skcipher_des3_key(skcipher, key) ?: 830 skcipher_setkey(skcipher, key, keylen, 0); 831} 832 833static int xts_skcipher_setkey(struct crypto_skcipher *skcipher, const u8 *key, 834 unsigned int keylen) 835{ 836 struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher); 837 struct device *jrdev = ctx->jrdev; 838 struct caam_drv_private *ctrlpriv = dev_get_drvdata(jrdev->parent); 839 u32 *desc; 840 int err; 841 842 err = xts_verify_key(skcipher, key, keylen); 843 if (err) { 844 dev_dbg(jrdev, "key size mismatch\n"); 845 return err; 846 } 847 848 if (keylen != 2 * AES_KEYSIZE_128 && keylen != 2 * AES_KEYSIZE_256) 849 ctx->xts_key_fallback = true; 850 851 if (ctrlpriv->era <= 8 || ctx->xts_key_fallback) { 852 err = crypto_skcipher_setkey(ctx->fallback, key, keylen); 853 if (err) 854 return err; 855 } 856 857 ctx->cdata.keylen = keylen; 858 ctx->cdata.key_virt = key; 859 ctx->cdata.key_inline = true; 860 861 /* xts_skcipher_encrypt shared descriptor */ 862 desc = ctx->sh_desc_enc; 863 cnstr_shdsc_xts_skcipher_encap(desc, &ctx->cdata); 864 dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma, 865 desc_bytes(desc), ctx->dir); 866 867 /* xts_skcipher_decrypt shared descriptor */ 868 desc = ctx->sh_desc_dec; 869 cnstr_shdsc_xts_skcipher_decap(desc, &ctx->cdata); 870 dma_sync_single_for_device(jrdev, ctx->sh_desc_dec_dma, 871 desc_bytes(desc), ctx->dir); 872 873 return 0; 874} 875 876/* 877 * aead_edesc - s/w-extended aead descriptor 878 * @src_nents: number of segments in input s/w scatterlist 879 * @dst_nents: number of segments in output s/w scatterlist 880 * @mapped_src_nents: number of segments in input h/w link table 881 * @mapped_dst_nents: number of segments in output h/w link table 882 * @sec4_sg_bytes: length of dma mapped sec4_sg space 883 * @bklog: stored to determine if the request needs backlog 884 * @sec4_sg_dma: bus physical mapped address of h/w link table 885 * @sec4_sg: pointer to h/w link table 886 * @hw_desc: the h/w job descriptor followed by any referenced link tables 887 */ 888struct aead_edesc { 889 int src_nents; 890 int dst_nents; 891 int mapped_src_nents; 892 int mapped_dst_nents; 893 int sec4_sg_bytes; 894 bool bklog; 895 dma_addr_t sec4_sg_dma; 896 struct sec4_sg_entry *sec4_sg; 897 u32 hw_desc[]; 898}; 899 900/* 901 * skcipher_edesc - s/w-extended skcipher descriptor 902 * @src_nents: number of segments in input s/w scatterlist 903 * @dst_nents: number of segments in output s/w scatterlist 904 * @mapped_src_nents: number of segments in input h/w link table 905 * @mapped_dst_nents: number of segments in output h/w link table 906 * @iv_dma: dma address of iv for checking continuity and link table 907 * @sec4_sg_bytes: length of dma mapped sec4_sg space 908 * @bklog: stored to determine if the request needs backlog 909 * @sec4_sg_dma: bus physical mapped address of h/w link table 910 * @sec4_sg: pointer to h/w link table 911 * @hw_desc: the h/w job descriptor followed by any referenced link tables 912 * and IV 913 */ 914struct skcipher_edesc { 915 int src_nents; 916 int dst_nents; 917 int mapped_src_nents; 918 int mapped_dst_nents; 919 dma_addr_t iv_dma; 920 int sec4_sg_bytes; 921 bool bklog; 922 dma_addr_t sec4_sg_dma; 923 struct sec4_sg_entry *sec4_sg; 924 u32 hw_desc[]; 925}; 926 927static void caam_unmap(struct device *dev, struct scatterlist *src, 928 struct scatterlist *dst, int src_nents, 929 int dst_nents, 930 dma_addr_t iv_dma, int ivsize, dma_addr_t sec4_sg_dma, 931 int sec4_sg_bytes) 932{ 933 if (dst != src) { 934 if (src_nents) 935 dma_unmap_sg(dev, src, src_nents, DMA_TO_DEVICE); 936 if (dst_nents) 937 dma_unmap_sg(dev, dst, dst_nents, DMA_FROM_DEVICE); 938 } else { 939 dma_unmap_sg(dev, src, src_nents, DMA_BIDIRECTIONAL); 940 } 941 942 if (iv_dma) 943 dma_unmap_single(dev, iv_dma, ivsize, DMA_BIDIRECTIONAL); 944 if (sec4_sg_bytes) 945 dma_unmap_single(dev, sec4_sg_dma, sec4_sg_bytes, 946 DMA_TO_DEVICE); 947} 948 949static void aead_unmap(struct device *dev, 950 struct aead_edesc *edesc, 951 struct aead_request *req) 952{ 953 caam_unmap(dev, req->src, req->dst, 954 edesc->src_nents, edesc->dst_nents, 0, 0, 955 edesc->sec4_sg_dma, edesc->sec4_sg_bytes); 956} 957 958static void skcipher_unmap(struct device *dev, struct skcipher_edesc *edesc, 959 struct skcipher_request *req) 960{ 961 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); 962 int ivsize = crypto_skcipher_ivsize(skcipher); 963 964 caam_unmap(dev, req->src, req->dst, 965 edesc->src_nents, edesc->dst_nents, 966 edesc->iv_dma, ivsize, 967 edesc->sec4_sg_dma, edesc->sec4_sg_bytes); 968} 969 970static void aead_crypt_done(struct device *jrdev, u32 *desc, u32 err, 971 void *context) 972{ 973 struct aead_request *req = context; 974 struct caam_aead_req_ctx *rctx = aead_request_ctx(req); 975 struct caam_drv_private_jr *jrp = dev_get_drvdata(jrdev); 976 struct aead_edesc *edesc; 977 int ecode = 0; 978 bool has_bklog; 979 980 dev_dbg(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err); 981 982 edesc = rctx->edesc; 983 has_bklog = edesc->bklog; 984 985 if (err) 986 ecode = caam_jr_strstatus(jrdev, err); 987 988 aead_unmap(jrdev, edesc, req); 989 990 kfree(edesc); 991 992 /* 993 * If no backlog flag, the completion of the request is done 994 * by CAAM, not crypto engine. 995 */ 996 if (!has_bklog) 997 aead_request_complete(req, ecode); 998 else 999 crypto_finalize_aead_request(jrp->engine, req, ecode); 1000} 1001 1002static void skcipher_crypt_done(struct device *jrdev, u32 *desc, u32 err, 1003 void *context) 1004{ 1005 struct skcipher_request *req = context; 1006 struct skcipher_edesc *edesc; 1007 struct caam_skcipher_req_ctx *rctx = skcipher_request_ctx(req); 1008 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); 1009 struct caam_drv_private_jr *jrp = dev_get_drvdata(jrdev); 1010 int ivsize = crypto_skcipher_ivsize(skcipher); 1011 int ecode = 0; 1012 bool has_bklog; 1013 1014 dev_dbg(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err); 1015 1016 edesc = rctx->edesc; 1017 has_bklog = edesc->bklog; 1018 if (err) 1019 ecode = caam_jr_strstatus(jrdev, err); 1020 1021 skcipher_unmap(jrdev, edesc, req); 1022 1023 /* 1024 * The crypto API expects us to set the IV (req->iv) to the last 1025 * ciphertext block (CBC mode) or last counter (CTR mode). 1026 * This is used e.g. by the CTS mode. 1027 */ 1028 if (ivsize && !ecode) { 1029 memcpy(req->iv, (u8 *)edesc->sec4_sg + edesc->sec4_sg_bytes, 1030 ivsize); 1031 1032 print_hex_dump_debug("dstiv @" __stringify(__LINE__)": ", 1033 DUMP_PREFIX_ADDRESS, 16, 4, req->iv, 1034 ivsize, 1); 1035 } 1036 1037 caam_dump_sg("dst @" __stringify(__LINE__)": ", 1038 DUMP_PREFIX_ADDRESS, 16, 4, req->dst, 1039 edesc->dst_nents > 1 ? 100 : req->cryptlen, 1); 1040 1041 kfree(edesc); 1042 1043 /* 1044 * If no backlog flag, the completion of the request is done 1045 * by CAAM, not crypto engine. 1046 */ 1047 if (!has_bklog) 1048 skcipher_request_complete(req, ecode); 1049 else 1050 crypto_finalize_skcipher_request(jrp->engine, req, ecode); 1051} 1052 1053/* 1054 * Fill in aead job descriptor 1055 */ 1056static void init_aead_job(struct aead_request *req, 1057 struct aead_edesc *edesc, 1058 bool all_contig, bool encrypt) 1059{ 1060 struct crypto_aead *aead = crypto_aead_reqtfm(req); 1061 struct caam_ctx *ctx = crypto_aead_ctx(aead); 1062 int authsize = ctx->authsize; 1063 u32 *desc = edesc->hw_desc; 1064 u32 out_options, in_options; 1065 dma_addr_t dst_dma, src_dma; 1066 int len, sec4_sg_index = 0; 1067 dma_addr_t ptr; 1068 u32 *sh_desc; 1069 1070 sh_desc = encrypt ? ctx->sh_desc_enc : ctx->sh_desc_dec; 1071 ptr = encrypt ? ctx->sh_desc_enc_dma : ctx->sh_desc_dec_dma; 1072 1073 len = desc_len(sh_desc); 1074 init_job_desc_shared(desc, ptr, len, HDR_SHARE_DEFER | HDR_REVERSE); 1075 1076 if (all_contig) { 1077 src_dma = edesc->mapped_src_nents ? sg_dma_address(req->src) : 1078 0; 1079 in_options = 0; 1080 } else { 1081 src_dma = edesc->sec4_sg_dma; 1082 sec4_sg_index += edesc->mapped_src_nents; 1083 in_options = LDST_SGF; 1084 } 1085 1086 append_seq_in_ptr(desc, src_dma, req->assoclen + req->cryptlen, 1087 in_options); 1088 1089 dst_dma = src_dma; 1090 out_options = in_options; 1091 1092 if (unlikely(req->src != req->dst)) { 1093 if (!edesc->mapped_dst_nents) { 1094 dst_dma = 0; 1095 out_options = 0; 1096 } else if (edesc->mapped_dst_nents == 1) { 1097 dst_dma = sg_dma_address(req->dst); 1098 out_options = 0; 1099 } else { 1100 dst_dma = edesc->sec4_sg_dma + 1101 sec4_sg_index * 1102 sizeof(struct sec4_sg_entry); 1103 out_options = LDST_SGF; 1104 } 1105 } 1106 1107 if (encrypt) 1108 append_seq_out_ptr(desc, dst_dma, 1109 req->assoclen + req->cryptlen + authsize, 1110 out_options); 1111 else 1112 append_seq_out_ptr(desc, dst_dma, 1113 req->assoclen + req->cryptlen - authsize, 1114 out_options); 1115} 1116 1117static void init_gcm_job(struct aead_request *req, 1118 struct aead_edesc *edesc, 1119 bool all_contig, bool encrypt) 1120{ 1121 struct crypto_aead *aead = crypto_aead_reqtfm(req); 1122 struct caam_ctx *ctx = crypto_aead_ctx(aead); 1123 unsigned int ivsize = crypto_aead_ivsize(aead); 1124 u32 *desc = edesc->hw_desc; 1125 bool generic_gcm = (ivsize == GCM_AES_IV_SIZE); 1126 unsigned int last; 1127 1128 init_aead_job(req, edesc, all_contig, encrypt); 1129 append_math_add_imm_u32(desc, REG3, ZERO, IMM, req->assoclen); 1130 1131 /* BUG This should not be specific to generic GCM. */ 1132 last = 0; 1133 if (encrypt && generic_gcm && !(req->assoclen + req->cryptlen)) 1134 last = FIFOLD_TYPE_LAST1; 1135 1136 /* Read GCM IV */ 1137 append_cmd(desc, CMD_FIFO_LOAD | FIFOLD_CLASS_CLASS1 | IMMEDIATE | 1138 FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1 | GCM_AES_IV_SIZE | last); 1139 /* Append Salt */ 1140 if (!generic_gcm) 1141 append_data(desc, ctx->key + ctx->cdata.keylen, 4); 1142 /* Append IV */ 1143 append_data(desc, req->iv, ivsize); 1144 /* End of blank commands */ 1145} 1146 1147static void init_chachapoly_job(struct aead_request *req, 1148 struct aead_edesc *edesc, bool all_contig, 1149 bool encrypt) 1150{ 1151 struct crypto_aead *aead = crypto_aead_reqtfm(req); 1152 unsigned int ivsize = crypto_aead_ivsize(aead); 1153 unsigned int assoclen = req->assoclen; 1154 u32 *desc = edesc->hw_desc; 1155 u32 ctx_iv_off = 4; 1156 1157 init_aead_job(req, edesc, all_contig, encrypt); 1158 1159 if (ivsize != CHACHAPOLY_IV_SIZE) { 1160 /* IPsec specific: CONTEXT1[223:128] = {NONCE, IV} */ 1161 ctx_iv_off += 4; 1162 1163 /* 1164 * The associated data comes already with the IV but we need 1165 * to skip it when we authenticate or encrypt... 1166 */ 1167 assoclen -= ivsize; 1168 } 1169 1170 append_math_add_imm_u32(desc, REG3, ZERO, IMM, assoclen); 1171 1172 /* 1173 * For IPsec load the IV further in the same register. 1174 * For RFC7539 simply load the 12 bytes nonce in a single operation 1175 */ 1176 append_load_as_imm(desc, req->iv, ivsize, LDST_CLASS_1_CCB | 1177 LDST_SRCDST_BYTE_CONTEXT | 1178 ctx_iv_off << LDST_OFFSET_SHIFT); 1179} 1180 1181static void init_authenc_job(struct aead_request *req, 1182 struct aead_edesc *edesc, 1183 bool all_contig, bool encrypt) 1184{ 1185 struct crypto_aead *aead = crypto_aead_reqtfm(req); 1186 struct caam_aead_alg *alg = container_of(crypto_aead_alg(aead), 1187 struct caam_aead_alg, aead); 1188 unsigned int ivsize = crypto_aead_ivsize(aead); 1189 struct caam_ctx *ctx = crypto_aead_ctx(aead); 1190 struct caam_drv_private *ctrlpriv = dev_get_drvdata(ctx->jrdev->parent); 1191 const bool ctr_mode = ((ctx->cdata.algtype & OP_ALG_AAI_MASK) == 1192 OP_ALG_AAI_CTR_MOD128); 1193 const bool is_rfc3686 = alg->caam.rfc3686; 1194 u32 *desc = edesc->hw_desc; 1195 u32 ivoffset = 0; 1196 1197 /* 1198 * AES-CTR needs to load IV in CONTEXT1 reg 1199 * at an offset of 128bits (16bytes) 1200 * CONTEXT1[255:128] = IV 1201 */ 1202 if (ctr_mode) 1203 ivoffset = 16; 1204 1205 /* 1206 * RFC3686 specific: 1207 * CONTEXT1[255:128] = {NONCE, IV, COUNTER} 1208 */ 1209 if (is_rfc3686) 1210 ivoffset = 16 + CTR_RFC3686_NONCE_SIZE; 1211 1212 init_aead_job(req, edesc, all_contig, encrypt); 1213 1214 /* 1215 * {REG3, DPOVRD} = assoclen, depending on whether MATH command supports 1216 * having DPOVRD as destination. 1217 */ 1218 if (ctrlpriv->era < 3) 1219 append_math_add_imm_u32(desc, REG3, ZERO, IMM, req->assoclen); 1220 else 1221 append_math_add_imm_u32(desc, DPOVRD, ZERO, IMM, req->assoclen); 1222 1223 if (ivsize && ((is_rfc3686 && encrypt) || !alg->caam.geniv)) 1224 append_load_as_imm(desc, req->iv, ivsize, 1225 LDST_CLASS_1_CCB | 1226 LDST_SRCDST_BYTE_CONTEXT | 1227 (ivoffset << LDST_OFFSET_SHIFT)); 1228} 1229 1230/* 1231 * Fill in skcipher job descriptor 1232 */ 1233static void init_skcipher_job(struct skcipher_request *req, 1234 struct skcipher_edesc *edesc, 1235 const bool encrypt) 1236{ 1237 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); 1238 struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher); 1239 struct device *jrdev = ctx->jrdev; 1240 int ivsize = crypto_skcipher_ivsize(skcipher); 1241 u32 *desc = edesc->hw_desc; 1242 u32 *sh_desc; 1243 u32 in_options = 0, out_options = 0; 1244 dma_addr_t src_dma, dst_dma, ptr; 1245 int len, sec4_sg_index = 0; 1246 1247 print_hex_dump_debug("presciv@"__stringify(__LINE__)": ", 1248 DUMP_PREFIX_ADDRESS, 16, 4, req->iv, ivsize, 1); 1249 dev_dbg(jrdev, "asked=%d, cryptlen%d\n", 1250 (int)edesc->src_nents > 1 ? 100 : req->cryptlen, req->cryptlen); 1251 1252 caam_dump_sg("src @" __stringify(__LINE__)": ", 1253 DUMP_PREFIX_ADDRESS, 16, 4, req->src, 1254 edesc->src_nents > 1 ? 100 : req->cryptlen, 1); 1255 1256 sh_desc = encrypt ? ctx->sh_desc_enc : ctx->sh_desc_dec; 1257 ptr = encrypt ? ctx->sh_desc_enc_dma : ctx->sh_desc_dec_dma; 1258 1259 len = desc_len(sh_desc); 1260 init_job_desc_shared(desc, ptr, len, HDR_SHARE_DEFER | HDR_REVERSE); 1261 1262 if (ivsize || edesc->mapped_src_nents > 1) { 1263 src_dma = edesc->sec4_sg_dma; 1264 sec4_sg_index = edesc->mapped_src_nents + !!ivsize; 1265 in_options = LDST_SGF; 1266 } else { 1267 src_dma = sg_dma_address(req->src); 1268 } 1269 1270 append_seq_in_ptr(desc, src_dma, req->cryptlen + ivsize, in_options); 1271 1272 if (likely(req->src == req->dst)) { 1273 dst_dma = src_dma + !!ivsize * sizeof(struct sec4_sg_entry); 1274 out_options = in_options; 1275 } else if (!ivsize && edesc->mapped_dst_nents == 1) { 1276 dst_dma = sg_dma_address(req->dst); 1277 } else { 1278 dst_dma = edesc->sec4_sg_dma + sec4_sg_index * 1279 sizeof(struct sec4_sg_entry); 1280 out_options = LDST_SGF; 1281 } 1282 1283 append_seq_out_ptr(desc, dst_dma, req->cryptlen + ivsize, out_options); 1284} 1285 1286/* 1287 * allocate and map the aead extended descriptor 1288 */ 1289static struct aead_edesc *aead_edesc_alloc(struct aead_request *req, 1290 int desc_bytes, bool *all_contig_ptr, 1291 bool encrypt) 1292{ 1293 struct crypto_aead *aead = crypto_aead_reqtfm(req); 1294 struct caam_ctx *ctx = crypto_aead_ctx(aead); 1295 struct device *jrdev = ctx->jrdev; 1296 struct caam_aead_req_ctx *rctx = aead_request_ctx(req); 1297 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 1298 GFP_KERNEL : GFP_ATOMIC; 1299 int src_nents, mapped_src_nents, dst_nents = 0, mapped_dst_nents = 0; 1300 int src_len, dst_len = 0; 1301 struct aead_edesc *edesc; 1302 int sec4_sg_index, sec4_sg_len, sec4_sg_bytes; 1303 unsigned int authsize = ctx->authsize; 1304 1305 if (unlikely(req->dst != req->src)) { 1306 src_len = req->assoclen + req->cryptlen; 1307 dst_len = src_len + (encrypt ? authsize : (-authsize)); 1308 1309 src_nents = sg_nents_for_len(req->src, src_len); 1310 if (unlikely(src_nents < 0)) { 1311 dev_err(jrdev, "Insufficient bytes (%d) in src S/G\n", 1312 src_len); 1313 return ERR_PTR(src_nents); 1314 } 1315 1316 dst_nents = sg_nents_for_len(req->dst, dst_len); 1317 if (unlikely(dst_nents < 0)) { 1318 dev_err(jrdev, "Insufficient bytes (%d) in dst S/G\n", 1319 dst_len); 1320 return ERR_PTR(dst_nents); 1321 } 1322 } else { 1323 src_len = req->assoclen + req->cryptlen + 1324 (encrypt ? authsize : 0); 1325 1326 src_nents = sg_nents_for_len(req->src, src_len); 1327 if (unlikely(src_nents < 0)) { 1328 dev_err(jrdev, "Insufficient bytes (%d) in src S/G\n", 1329 src_len); 1330 return ERR_PTR(src_nents); 1331 } 1332 } 1333 1334 if (likely(req->src == req->dst)) { 1335 mapped_src_nents = dma_map_sg(jrdev, req->src, src_nents, 1336 DMA_BIDIRECTIONAL); 1337 if (unlikely(!mapped_src_nents)) { 1338 dev_err(jrdev, "unable to map source\n"); 1339 return ERR_PTR(-ENOMEM); 1340 } 1341 } else { 1342 /* Cover also the case of null (zero length) input data */ 1343 if (src_nents) { 1344 mapped_src_nents = dma_map_sg(jrdev, req->src, 1345 src_nents, DMA_TO_DEVICE); 1346 if (unlikely(!mapped_src_nents)) { 1347 dev_err(jrdev, "unable to map source\n"); 1348 return ERR_PTR(-ENOMEM); 1349 } 1350 } else { 1351 mapped_src_nents = 0; 1352 } 1353 1354 /* Cover also the case of null (zero length) output data */ 1355 if (dst_nents) { 1356 mapped_dst_nents = dma_map_sg(jrdev, req->dst, 1357 dst_nents, 1358 DMA_FROM_DEVICE); 1359 if (unlikely(!mapped_dst_nents)) { 1360 dev_err(jrdev, "unable to map destination\n"); 1361 dma_unmap_sg(jrdev, req->src, src_nents, 1362 DMA_TO_DEVICE); 1363 return ERR_PTR(-ENOMEM); 1364 } 1365 } else { 1366 mapped_dst_nents = 0; 1367 } 1368 } 1369 1370 /* 1371 * HW reads 4 S/G entries at a time; make sure the reads don't go beyond 1372 * the end of the table by allocating more S/G entries. 1373 */ 1374 sec4_sg_len = mapped_src_nents > 1 ? mapped_src_nents : 0; 1375 if (mapped_dst_nents > 1) 1376 sec4_sg_len += pad_sg_nents(mapped_dst_nents); 1377 else 1378 sec4_sg_len = pad_sg_nents(sec4_sg_len); 1379 1380 sec4_sg_bytes = sec4_sg_len * sizeof(struct sec4_sg_entry); 1381 1382 /* allocate space for base edesc and hw desc commands, link tables */ 1383 edesc = kzalloc(sizeof(*edesc) + desc_bytes + sec4_sg_bytes, 1384 GFP_DMA | flags); 1385 if (!edesc) { 1386 caam_unmap(jrdev, req->src, req->dst, src_nents, dst_nents, 0, 1387 0, 0, 0); 1388 return ERR_PTR(-ENOMEM); 1389 } 1390 1391 edesc->src_nents = src_nents; 1392 edesc->dst_nents = dst_nents; 1393 edesc->mapped_src_nents = mapped_src_nents; 1394 edesc->mapped_dst_nents = mapped_dst_nents; 1395 edesc->sec4_sg = (void *)edesc + sizeof(struct aead_edesc) + 1396 desc_bytes; 1397 1398 rctx->edesc = edesc; 1399 1400 *all_contig_ptr = !(mapped_src_nents > 1); 1401 1402 sec4_sg_index = 0; 1403 if (mapped_src_nents > 1) { 1404 sg_to_sec4_sg_last(req->src, src_len, 1405 edesc->sec4_sg + sec4_sg_index, 0); 1406 sec4_sg_index += mapped_src_nents; 1407 } 1408 if (mapped_dst_nents > 1) { 1409 sg_to_sec4_sg_last(req->dst, dst_len, 1410 edesc->sec4_sg + sec4_sg_index, 0); 1411 } 1412 1413 if (!sec4_sg_bytes) 1414 return edesc; 1415 1416 edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg, 1417 sec4_sg_bytes, DMA_TO_DEVICE); 1418 if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) { 1419 dev_err(jrdev, "unable to map S/G table\n"); 1420 aead_unmap(jrdev, edesc, req); 1421 kfree(edesc); 1422 return ERR_PTR(-ENOMEM); 1423 } 1424 1425 edesc->sec4_sg_bytes = sec4_sg_bytes; 1426 1427 return edesc; 1428} 1429 1430static int aead_enqueue_req(struct device *jrdev, struct aead_request *req) 1431{ 1432 struct caam_drv_private_jr *jrpriv = dev_get_drvdata(jrdev); 1433 struct caam_aead_req_ctx *rctx = aead_request_ctx(req); 1434 struct aead_edesc *edesc = rctx->edesc; 1435 u32 *desc = edesc->hw_desc; 1436 int ret; 1437 1438 /* 1439 * Only the backlog request are sent to crypto-engine since the others 1440 * can be handled by CAAM, if free, especially since JR has up to 1024 1441 * entries (more than the 10 entries from crypto-engine). 1442 */ 1443 if (req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG) 1444 ret = crypto_transfer_aead_request_to_engine(jrpriv->engine, 1445 req); 1446 else 1447 ret = caam_jr_enqueue(jrdev, desc, aead_crypt_done, req); 1448 1449 if ((ret != -EINPROGRESS) && (ret != -EBUSY)) { 1450 aead_unmap(jrdev, edesc, req); 1451 kfree(rctx->edesc); 1452 } 1453 1454 return ret; 1455} 1456 1457static inline int chachapoly_crypt(struct aead_request *req, bool encrypt) 1458{ 1459 struct aead_edesc *edesc; 1460 struct crypto_aead *aead = crypto_aead_reqtfm(req); 1461 struct caam_ctx *ctx = crypto_aead_ctx(aead); 1462 struct device *jrdev = ctx->jrdev; 1463 bool all_contig; 1464 u32 *desc; 1465 1466 edesc = aead_edesc_alloc(req, CHACHAPOLY_DESC_JOB_IO_LEN, &all_contig, 1467 encrypt); 1468 if (IS_ERR(edesc)) 1469 return PTR_ERR(edesc); 1470 1471 desc = edesc->hw_desc; 1472 1473 init_chachapoly_job(req, edesc, all_contig, encrypt); 1474 print_hex_dump_debug("chachapoly jobdesc@" __stringify(__LINE__)": ", 1475 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1476 1); 1477 1478 return aead_enqueue_req(jrdev, req); 1479} 1480 1481static int chachapoly_encrypt(struct aead_request *req) 1482{ 1483 return chachapoly_crypt(req, true); 1484} 1485 1486static int chachapoly_decrypt(struct aead_request *req) 1487{ 1488 return chachapoly_crypt(req, false); 1489} 1490 1491static inline int aead_crypt(struct aead_request *req, bool encrypt) 1492{ 1493 struct aead_edesc *edesc; 1494 struct crypto_aead *aead = crypto_aead_reqtfm(req); 1495 struct caam_ctx *ctx = crypto_aead_ctx(aead); 1496 struct device *jrdev = ctx->jrdev; 1497 bool all_contig; 1498 1499 /* allocate extended descriptor */ 1500 edesc = aead_edesc_alloc(req, AUTHENC_DESC_JOB_IO_LEN, 1501 &all_contig, encrypt); 1502 if (IS_ERR(edesc)) 1503 return PTR_ERR(edesc); 1504 1505 /* Create and submit job descriptor */ 1506 init_authenc_job(req, edesc, all_contig, encrypt); 1507 1508 print_hex_dump_debug("aead jobdesc@"__stringify(__LINE__)": ", 1509 DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc, 1510 desc_bytes(edesc->hw_desc), 1); 1511 1512 return aead_enqueue_req(jrdev, req); 1513} 1514 1515static int aead_encrypt(struct aead_request *req) 1516{ 1517 return aead_crypt(req, true); 1518} 1519 1520static int aead_decrypt(struct aead_request *req) 1521{ 1522 return aead_crypt(req, false); 1523} 1524 1525static int aead_do_one_req(struct crypto_engine *engine, void *areq) 1526{ 1527 struct aead_request *req = aead_request_cast(areq); 1528 struct caam_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req)); 1529 struct caam_aead_req_ctx *rctx = aead_request_ctx(req); 1530 u32 *desc = rctx->edesc->hw_desc; 1531 int ret; 1532 1533 rctx->edesc->bklog = true; 1534 1535 ret = caam_jr_enqueue(ctx->jrdev, desc, aead_crypt_done, req); 1536 1537 if (ret != -EINPROGRESS) { 1538 aead_unmap(ctx->jrdev, rctx->edesc, req); 1539 kfree(rctx->edesc); 1540 } else { 1541 ret = 0; 1542 } 1543 1544 return ret; 1545} 1546 1547static inline int gcm_crypt(struct aead_request *req, bool encrypt) 1548{ 1549 struct aead_edesc *edesc; 1550 struct crypto_aead *aead = crypto_aead_reqtfm(req); 1551 struct caam_ctx *ctx = crypto_aead_ctx(aead); 1552 struct device *jrdev = ctx->jrdev; 1553 bool all_contig; 1554 1555 /* allocate extended descriptor */ 1556 edesc = aead_edesc_alloc(req, GCM_DESC_JOB_IO_LEN, &all_contig, 1557 encrypt); 1558 if (IS_ERR(edesc)) 1559 return PTR_ERR(edesc); 1560 1561 /* Create and submit job descriptor */ 1562 init_gcm_job(req, edesc, all_contig, encrypt); 1563 1564 print_hex_dump_debug("aead jobdesc@"__stringify(__LINE__)": ", 1565 DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc, 1566 desc_bytes(edesc->hw_desc), 1); 1567 1568 return aead_enqueue_req(jrdev, req); 1569} 1570 1571static int gcm_encrypt(struct aead_request *req) 1572{ 1573 return gcm_crypt(req, true); 1574} 1575 1576static int gcm_decrypt(struct aead_request *req) 1577{ 1578 return gcm_crypt(req, false); 1579} 1580 1581static int ipsec_gcm_encrypt(struct aead_request *req) 1582{ 1583 return crypto_ipsec_check_assoclen(req->assoclen) ? : gcm_encrypt(req); 1584} 1585 1586static int ipsec_gcm_decrypt(struct aead_request *req) 1587{ 1588 return crypto_ipsec_check_assoclen(req->assoclen) ? : gcm_decrypt(req); 1589} 1590 1591/* 1592 * allocate and map the skcipher extended descriptor for skcipher 1593 */ 1594static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req, 1595 int desc_bytes) 1596{ 1597 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); 1598 struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher); 1599 struct caam_skcipher_req_ctx *rctx = skcipher_request_ctx(req); 1600 struct device *jrdev = ctx->jrdev; 1601 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 1602 GFP_KERNEL : GFP_ATOMIC; 1603 int src_nents, mapped_src_nents, dst_nents = 0, mapped_dst_nents = 0; 1604 struct skcipher_edesc *edesc; 1605 dma_addr_t iv_dma = 0; 1606 u8 *iv; 1607 int ivsize = crypto_skcipher_ivsize(skcipher); 1608 int dst_sg_idx, sec4_sg_ents, sec4_sg_bytes; 1609 1610 src_nents = sg_nents_for_len(req->src, req->cryptlen); 1611 if (unlikely(src_nents < 0)) { 1612 dev_err(jrdev, "Insufficient bytes (%d) in src S/G\n", 1613 req->cryptlen); 1614 return ERR_PTR(src_nents); 1615 } 1616 1617 if (req->dst != req->src) { 1618 dst_nents = sg_nents_for_len(req->dst, req->cryptlen); 1619 if (unlikely(dst_nents < 0)) { 1620 dev_err(jrdev, "Insufficient bytes (%d) in dst S/G\n", 1621 req->cryptlen); 1622 return ERR_PTR(dst_nents); 1623 } 1624 } 1625 1626 if (likely(req->src == req->dst)) { 1627 mapped_src_nents = dma_map_sg(jrdev, req->src, src_nents, 1628 DMA_BIDIRECTIONAL); 1629 if (unlikely(!mapped_src_nents)) { 1630 dev_err(jrdev, "unable to map source\n"); 1631 return ERR_PTR(-ENOMEM); 1632 } 1633 } else { 1634 mapped_src_nents = dma_map_sg(jrdev, req->src, src_nents, 1635 DMA_TO_DEVICE); 1636 if (unlikely(!mapped_src_nents)) { 1637 dev_err(jrdev, "unable to map source\n"); 1638 return ERR_PTR(-ENOMEM); 1639 } 1640 mapped_dst_nents = dma_map_sg(jrdev, req->dst, dst_nents, 1641 DMA_FROM_DEVICE); 1642 if (unlikely(!mapped_dst_nents)) { 1643 dev_err(jrdev, "unable to map destination\n"); 1644 dma_unmap_sg(jrdev, req->src, src_nents, DMA_TO_DEVICE); 1645 return ERR_PTR(-ENOMEM); 1646 } 1647 } 1648 1649 if (!ivsize && mapped_src_nents == 1) 1650 sec4_sg_ents = 0; // no need for an input hw s/g table 1651 else 1652 sec4_sg_ents = mapped_src_nents + !!ivsize; 1653 dst_sg_idx = sec4_sg_ents; 1654 1655 /* 1656 * Input, output HW S/G tables: [IV, src][dst, IV] 1657 * IV entries point to the same buffer 1658 * If src == dst, S/G entries are reused (S/G tables overlap) 1659 * 1660 * HW reads 4 S/G entries at a time; make sure the reads don't go beyond 1661 * the end of the table by allocating more S/G entries. Logic: 1662 * if (output S/G) 1663 * pad output S/G, if needed 1664 * else if (input S/G) ... 1665 * pad input S/G, if needed 1666 */ 1667 if (ivsize || mapped_dst_nents > 1) { 1668 if (req->src == req->dst) 1669 sec4_sg_ents = !!ivsize + pad_sg_nents(sec4_sg_ents); 1670 else 1671 sec4_sg_ents += pad_sg_nents(mapped_dst_nents + 1672 !!ivsize); 1673 } else { 1674 sec4_sg_ents = pad_sg_nents(sec4_sg_ents); 1675 } 1676 1677 sec4_sg_bytes = sec4_sg_ents * sizeof(struct sec4_sg_entry); 1678 1679 /* 1680 * allocate space for base edesc and hw desc commands, link tables, IV 1681 */ 1682 edesc = kzalloc(sizeof(*edesc) + desc_bytes + sec4_sg_bytes + ivsize, 1683 GFP_DMA | flags); 1684 if (!edesc) { 1685 dev_err(jrdev, "could not allocate extended descriptor\n"); 1686 caam_unmap(jrdev, req->src, req->dst, src_nents, dst_nents, 0, 1687 0, 0, 0); 1688 return ERR_PTR(-ENOMEM); 1689 } 1690 1691 edesc->src_nents = src_nents; 1692 edesc->dst_nents = dst_nents; 1693 edesc->mapped_src_nents = mapped_src_nents; 1694 edesc->mapped_dst_nents = mapped_dst_nents; 1695 edesc->sec4_sg_bytes = sec4_sg_bytes; 1696 edesc->sec4_sg = (struct sec4_sg_entry *)((u8 *)edesc->hw_desc + 1697 desc_bytes); 1698 rctx->edesc = edesc; 1699 1700 /* Make sure IV is located in a DMAable area */ 1701 if (ivsize) { 1702 iv = (u8 *)edesc->sec4_sg + sec4_sg_bytes; 1703 memcpy(iv, req->iv, ivsize); 1704 1705 iv_dma = dma_map_single(jrdev, iv, ivsize, DMA_BIDIRECTIONAL); 1706 if (dma_mapping_error(jrdev, iv_dma)) { 1707 dev_err(jrdev, "unable to map IV\n"); 1708 caam_unmap(jrdev, req->src, req->dst, src_nents, 1709 dst_nents, 0, 0, 0, 0); 1710 kfree(edesc); 1711 return ERR_PTR(-ENOMEM); 1712 } 1713 1714 dma_to_sec4_sg_one(edesc->sec4_sg, iv_dma, ivsize, 0); 1715 } 1716 if (dst_sg_idx) 1717 sg_to_sec4_sg(req->src, req->cryptlen, edesc->sec4_sg + 1718 !!ivsize, 0); 1719 1720 if (req->src != req->dst && (ivsize || mapped_dst_nents > 1)) 1721 sg_to_sec4_sg(req->dst, req->cryptlen, edesc->sec4_sg + 1722 dst_sg_idx, 0); 1723 1724 if (ivsize) 1725 dma_to_sec4_sg_one(edesc->sec4_sg + dst_sg_idx + 1726 mapped_dst_nents, iv_dma, ivsize, 0); 1727 1728 if (ivsize || mapped_dst_nents > 1) 1729 sg_to_sec4_set_last(edesc->sec4_sg + dst_sg_idx + 1730 mapped_dst_nents - 1 + !!ivsize); 1731 1732 if (sec4_sg_bytes) { 1733 edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg, 1734 sec4_sg_bytes, 1735 DMA_TO_DEVICE); 1736 if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) { 1737 dev_err(jrdev, "unable to map S/G table\n"); 1738 caam_unmap(jrdev, req->src, req->dst, src_nents, 1739 dst_nents, iv_dma, ivsize, 0, 0); 1740 kfree(edesc); 1741 return ERR_PTR(-ENOMEM); 1742 } 1743 } 1744 1745 edesc->iv_dma = iv_dma; 1746 1747 print_hex_dump_debug("skcipher sec4_sg@" __stringify(__LINE__)": ", 1748 DUMP_PREFIX_ADDRESS, 16, 4, edesc->sec4_sg, 1749 sec4_sg_bytes, 1); 1750 1751 return edesc; 1752} 1753 1754static int skcipher_do_one_req(struct crypto_engine *engine, void *areq) 1755{ 1756 struct skcipher_request *req = skcipher_request_cast(areq); 1757 struct caam_ctx *ctx = crypto_skcipher_ctx(crypto_skcipher_reqtfm(req)); 1758 struct caam_skcipher_req_ctx *rctx = skcipher_request_ctx(req); 1759 u32 *desc = rctx->edesc->hw_desc; 1760 int ret; 1761 1762 rctx->edesc->bklog = true; 1763 1764 ret = caam_jr_enqueue(ctx->jrdev, desc, skcipher_crypt_done, req); 1765 1766 if (ret != -EINPROGRESS) { 1767 skcipher_unmap(ctx->jrdev, rctx->edesc, req); 1768 kfree(rctx->edesc); 1769 } else { 1770 ret = 0; 1771 } 1772 1773 return ret; 1774} 1775 1776static inline bool xts_skcipher_ivsize(struct skcipher_request *req) 1777{ 1778 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); 1779 unsigned int ivsize = crypto_skcipher_ivsize(skcipher); 1780 1781 return !!get_unaligned((u64 *)(req->iv + (ivsize / 2))); 1782} 1783 1784static inline int skcipher_crypt(struct skcipher_request *req, bool encrypt) 1785{ 1786 struct skcipher_edesc *edesc; 1787 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); 1788 struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher); 1789 struct device *jrdev = ctx->jrdev; 1790 struct caam_drv_private_jr *jrpriv = dev_get_drvdata(jrdev); 1791 struct caam_drv_private *ctrlpriv = dev_get_drvdata(jrdev->parent); 1792 u32 *desc; 1793 int ret = 0; 1794 1795 /* 1796 * XTS is expected to return an error even for input length = 0 1797 * Note that the case input length < block size will be caught during 1798 * HW offloading and return an error. 1799 */ 1800 if (!req->cryptlen && !ctx->fallback) 1801 return 0; 1802 1803 if (ctx->fallback && ((ctrlpriv->era <= 8 && xts_skcipher_ivsize(req)) || 1804 ctx->xts_key_fallback)) { 1805 struct caam_skcipher_req_ctx *rctx = skcipher_request_ctx(req); 1806 1807 skcipher_request_set_tfm(&rctx->fallback_req, ctx->fallback); 1808 skcipher_request_set_callback(&rctx->fallback_req, 1809 req->base.flags, 1810 req->base.complete, 1811 req->base.data); 1812 skcipher_request_set_crypt(&rctx->fallback_req, req->src, 1813 req->dst, req->cryptlen, req->iv); 1814 1815 return encrypt ? crypto_skcipher_encrypt(&rctx->fallback_req) : 1816 crypto_skcipher_decrypt(&rctx->fallback_req); 1817 } 1818 1819 /* allocate extended descriptor */ 1820 edesc = skcipher_edesc_alloc(req, DESC_JOB_IO_LEN * CAAM_CMD_SZ); 1821 if (IS_ERR(edesc)) 1822 return PTR_ERR(edesc); 1823 1824 /* Create and submit job descriptor*/ 1825 init_skcipher_job(req, edesc, encrypt); 1826 1827 print_hex_dump_debug("skcipher jobdesc@" __stringify(__LINE__)": ", 1828 DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc, 1829 desc_bytes(edesc->hw_desc), 1); 1830 1831 desc = edesc->hw_desc; 1832 /* 1833 * Only the backlog request are sent to crypto-engine since the others 1834 * can be handled by CAAM, if free, especially since JR has up to 1024 1835 * entries (more than the 10 entries from crypto-engine). 1836 */ 1837 if (req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG) 1838 ret = crypto_transfer_skcipher_request_to_engine(jrpriv->engine, 1839 req); 1840 else 1841 ret = caam_jr_enqueue(jrdev, desc, skcipher_crypt_done, req); 1842 1843 if ((ret != -EINPROGRESS) && (ret != -EBUSY)) { 1844 skcipher_unmap(jrdev, edesc, req); 1845 kfree(edesc); 1846 } 1847 1848 return ret; 1849} 1850 1851static int skcipher_encrypt(struct skcipher_request *req) 1852{ 1853 return skcipher_crypt(req, true); 1854} 1855 1856static int skcipher_decrypt(struct skcipher_request *req) 1857{ 1858 return skcipher_crypt(req, false); 1859} 1860 1861static struct caam_skcipher_alg driver_algs[] = { 1862 { 1863 .skcipher = { 1864 .base = { 1865 .cra_name = "cbc(aes)", 1866 .cra_driver_name = "cbc-aes-caam", 1867 .cra_blocksize = AES_BLOCK_SIZE, 1868 }, 1869 .setkey = aes_skcipher_setkey, 1870 .encrypt = skcipher_encrypt, 1871 .decrypt = skcipher_decrypt, 1872 .min_keysize = AES_MIN_KEY_SIZE, 1873 .max_keysize = AES_MAX_KEY_SIZE, 1874 .ivsize = AES_BLOCK_SIZE, 1875 }, 1876 .caam.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1877 }, 1878 { 1879 .skcipher = { 1880 .base = { 1881 .cra_name = "cbc(des3_ede)", 1882 .cra_driver_name = "cbc-3des-caam", 1883 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 1884 }, 1885 .setkey = des3_skcipher_setkey, 1886 .encrypt = skcipher_encrypt, 1887 .decrypt = skcipher_decrypt, 1888 .min_keysize = DES3_EDE_KEY_SIZE, 1889 .max_keysize = DES3_EDE_KEY_SIZE, 1890 .ivsize = DES3_EDE_BLOCK_SIZE, 1891 }, 1892 .caam.class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 1893 }, 1894 { 1895 .skcipher = { 1896 .base = { 1897 .cra_name = "cbc(des)", 1898 .cra_driver_name = "cbc-des-caam", 1899 .cra_blocksize = DES_BLOCK_SIZE, 1900 }, 1901 .setkey = des_skcipher_setkey, 1902 .encrypt = skcipher_encrypt, 1903 .decrypt = skcipher_decrypt, 1904 .min_keysize = DES_KEY_SIZE, 1905 .max_keysize = DES_KEY_SIZE, 1906 .ivsize = DES_BLOCK_SIZE, 1907 }, 1908 .caam.class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 1909 }, 1910 { 1911 .skcipher = { 1912 .base = { 1913 .cra_name = "ctr(aes)", 1914 .cra_driver_name = "ctr-aes-caam", 1915 .cra_blocksize = 1, 1916 }, 1917 .setkey = ctr_skcipher_setkey, 1918 .encrypt = skcipher_encrypt, 1919 .decrypt = skcipher_decrypt, 1920 .min_keysize = AES_MIN_KEY_SIZE, 1921 .max_keysize = AES_MAX_KEY_SIZE, 1922 .ivsize = AES_BLOCK_SIZE, 1923 .chunksize = AES_BLOCK_SIZE, 1924 }, 1925 .caam.class1_alg_type = OP_ALG_ALGSEL_AES | 1926 OP_ALG_AAI_CTR_MOD128, 1927 }, 1928 { 1929 .skcipher = { 1930 .base = { 1931 .cra_name = "rfc3686(ctr(aes))", 1932 .cra_driver_name = "rfc3686-ctr-aes-caam", 1933 .cra_blocksize = 1, 1934 }, 1935 .setkey = rfc3686_skcipher_setkey, 1936 .encrypt = skcipher_encrypt, 1937 .decrypt = skcipher_decrypt, 1938 .min_keysize = AES_MIN_KEY_SIZE + 1939 CTR_RFC3686_NONCE_SIZE, 1940 .max_keysize = AES_MAX_KEY_SIZE + 1941 CTR_RFC3686_NONCE_SIZE, 1942 .ivsize = CTR_RFC3686_IV_SIZE, 1943 .chunksize = AES_BLOCK_SIZE, 1944 }, 1945 .caam = { 1946 .class1_alg_type = OP_ALG_ALGSEL_AES | 1947 OP_ALG_AAI_CTR_MOD128, 1948 .rfc3686 = true, 1949 }, 1950 }, 1951 { 1952 .skcipher = { 1953 .base = { 1954 .cra_name = "xts(aes)", 1955 .cra_driver_name = "xts-aes-caam", 1956 .cra_flags = CRYPTO_ALG_NEED_FALLBACK, 1957 .cra_blocksize = AES_BLOCK_SIZE, 1958 }, 1959 .setkey = xts_skcipher_setkey, 1960 .encrypt = skcipher_encrypt, 1961 .decrypt = skcipher_decrypt, 1962 .min_keysize = 2 * AES_MIN_KEY_SIZE, 1963 .max_keysize = 2 * AES_MAX_KEY_SIZE, 1964 .ivsize = AES_BLOCK_SIZE, 1965 }, 1966 .caam.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_XTS, 1967 }, 1968 { 1969 .skcipher = { 1970 .base = { 1971 .cra_name = "ecb(des)", 1972 .cra_driver_name = "ecb-des-caam", 1973 .cra_blocksize = DES_BLOCK_SIZE, 1974 }, 1975 .setkey = des_skcipher_setkey, 1976 .encrypt = skcipher_encrypt, 1977 .decrypt = skcipher_decrypt, 1978 .min_keysize = DES_KEY_SIZE, 1979 .max_keysize = DES_KEY_SIZE, 1980 }, 1981 .caam.class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_ECB, 1982 }, 1983 { 1984 .skcipher = { 1985 .base = { 1986 .cra_name = "ecb(aes)", 1987 .cra_driver_name = "ecb-aes-caam", 1988 .cra_blocksize = AES_BLOCK_SIZE, 1989 }, 1990 .setkey = aes_skcipher_setkey, 1991 .encrypt = skcipher_encrypt, 1992 .decrypt = skcipher_decrypt, 1993 .min_keysize = AES_MIN_KEY_SIZE, 1994 .max_keysize = AES_MAX_KEY_SIZE, 1995 }, 1996 .caam.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_ECB, 1997 }, 1998 { 1999 .skcipher = { 2000 .base = { 2001 .cra_name = "ecb(des3_ede)", 2002 .cra_driver_name = "ecb-des3-caam", 2003 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2004 }, 2005 .setkey = des3_skcipher_setkey, 2006 .encrypt = skcipher_encrypt, 2007 .decrypt = skcipher_decrypt, 2008 .min_keysize = DES3_EDE_KEY_SIZE, 2009 .max_keysize = DES3_EDE_KEY_SIZE, 2010 }, 2011 .caam.class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_ECB, 2012 }, 2013}; 2014 2015static struct caam_aead_alg driver_aeads[] = { 2016 { 2017 .aead = { 2018 .base = { 2019 .cra_name = "rfc4106(gcm(aes))", 2020 .cra_driver_name = "rfc4106-gcm-aes-caam", 2021 .cra_blocksize = 1, 2022 }, 2023 .setkey = rfc4106_setkey, 2024 .setauthsize = rfc4106_setauthsize, 2025 .encrypt = ipsec_gcm_encrypt, 2026 .decrypt = ipsec_gcm_decrypt, 2027 .ivsize = GCM_RFC4106_IV_SIZE, 2028 .maxauthsize = AES_BLOCK_SIZE, 2029 }, 2030 .caam = { 2031 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM, 2032 .nodkp = true, 2033 }, 2034 }, 2035 { 2036 .aead = { 2037 .base = { 2038 .cra_name = "rfc4543(gcm(aes))", 2039 .cra_driver_name = "rfc4543-gcm-aes-caam", 2040 .cra_blocksize = 1, 2041 }, 2042 .setkey = rfc4543_setkey, 2043 .setauthsize = rfc4543_setauthsize, 2044 .encrypt = ipsec_gcm_encrypt, 2045 .decrypt = ipsec_gcm_decrypt, 2046 .ivsize = GCM_RFC4543_IV_SIZE, 2047 .maxauthsize = AES_BLOCK_SIZE, 2048 }, 2049 .caam = { 2050 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM, 2051 .nodkp = true, 2052 }, 2053 }, 2054 /* Galois Counter Mode */ 2055 { 2056 .aead = { 2057 .base = { 2058 .cra_name = "gcm(aes)", 2059 .cra_driver_name = "gcm-aes-caam", 2060 .cra_blocksize = 1, 2061 }, 2062 .setkey = gcm_setkey, 2063 .setauthsize = gcm_setauthsize, 2064 .encrypt = gcm_encrypt, 2065 .decrypt = gcm_decrypt, 2066 .ivsize = GCM_AES_IV_SIZE, 2067 .maxauthsize = AES_BLOCK_SIZE, 2068 }, 2069 .caam = { 2070 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM, 2071 .nodkp = true, 2072 }, 2073 }, 2074 /* single-pass ipsec_esp descriptor */ 2075 { 2076 .aead = { 2077 .base = { 2078 .cra_name = "authenc(hmac(md5)," 2079 "ecb(cipher_null))", 2080 .cra_driver_name = "authenc-hmac-md5-" 2081 "ecb-cipher_null-caam", 2082 .cra_blocksize = NULL_BLOCK_SIZE, 2083 }, 2084 .setkey = aead_setkey, 2085 .setauthsize = aead_setauthsize, 2086 .encrypt = aead_encrypt, 2087 .decrypt = aead_decrypt, 2088 .ivsize = NULL_IV_SIZE, 2089 .maxauthsize = MD5_DIGEST_SIZE, 2090 }, 2091 .caam = { 2092 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 2093 OP_ALG_AAI_HMAC_PRECOMP, 2094 }, 2095 }, 2096 { 2097 .aead = { 2098 .base = { 2099 .cra_name = "authenc(hmac(sha1)," 2100 "ecb(cipher_null))", 2101 .cra_driver_name = "authenc-hmac-sha1-" 2102 "ecb-cipher_null-caam", 2103 .cra_blocksize = NULL_BLOCK_SIZE, 2104 }, 2105 .setkey = aead_setkey, 2106 .setauthsize = aead_setauthsize, 2107 .encrypt = aead_encrypt, 2108 .decrypt = aead_decrypt, 2109 .ivsize = NULL_IV_SIZE, 2110 .maxauthsize = SHA1_DIGEST_SIZE, 2111 }, 2112 .caam = { 2113 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 2114 OP_ALG_AAI_HMAC_PRECOMP, 2115 }, 2116 }, 2117 { 2118 .aead = { 2119 .base = { 2120 .cra_name = "authenc(hmac(sha224)," 2121 "ecb(cipher_null))", 2122 .cra_driver_name = "authenc-hmac-sha224-" 2123 "ecb-cipher_null-caam", 2124 .cra_blocksize = NULL_BLOCK_SIZE, 2125 }, 2126 .setkey = aead_setkey, 2127 .setauthsize = aead_setauthsize, 2128 .encrypt = aead_encrypt, 2129 .decrypt = aead_decrypt, 2130 .ivsize = NULL_IV_SIZE, 2131 .maxauthsize = SHA224_DIGEST_SIZE, 2132 }, 2133 .caam = { 2134 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 2135 OP_ALG_AAI_HMAC_PRECOMP, 2136 }, 2137 }, 2138 { 2139 .aead = { 2140 .base = { 2141 .cra_name = "authenc(hmac(sha256)," 2142 "ecb(cipher_null))", 2143 .cra_driver_name = "authenc-hmac-sha256-" 2144 "ecb-cipher_null-caam", 2145 .cra_blocksize = NULL_BLOCK_SIZE, 2146 }, 2147 .setkey = aead_setkey, 2148 .setauthsize = aead_setauthsize, 2149 .encrypt = aead_encrypt, 2150 .decrypt = aead_decrypt, 2151 .ivsize = NULL_IV_SIZE, 2152 .maxauthsize = SHA256_DIGEST_SIZE, 2153 }, 2154 .caam = { 2155 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 2156 OP_ALG_AAI_HMAC_PRECOMP, 2157 }, 2158 }, 2159 { 2160 .aead = { 2161 .base = { 2162 .cra_name = "authenc(hmac(sha384)," 2163 "ecb(cipher_null))", 2164 .cra_driver_name = "authenc-hmac-sha384-" 2165 "ecb-cipher_null-caam", 2166 .cra_blocksize = NULL_BLOCK_SIZE, 2167 }, 2168 .setkey = aead_setkey, 2169 .setauthsize = aead_setauthsize, 2170 .encrypt = aead_encrypt, 2171 .decrypt = aead_decrypt, 2172 .ivsize = NULL_IV_SIZE, 2173 .maxauthsize = SHA384_DIGEST_SIZE, 2174 }, 2175 .caam = { 2176 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 2177 OP_ALG_AAI_HMAC_PRECOMP, 2178 }, 2179 }, 2180 { 2181 .aead = { 2182 .base = { 2183 .cra_name = "authenc(hmac(sha512)," 2184 "ecb(cipher_null))", 2185 .cra_driver_name = "authenc-hmac-sha512-" 2186 "ecb-cipher_null-caam", 2187 .cra_blocksize = NULL_BLOCK_SIZE, 2188 }, 2189 .setkey = aead_setkey, 2190 .setauthsize = aead_setauthsize, 2191 .encrypt = aead_encrypt, 2192 .decrypt = aead_decrypt, 2193 .ivsize = NULL_IV_SIZE, 2194 .maxauthsize = SHA512_DIGEST_SIZE, 2195 }, 2196 .caam = { 2197 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 2198 OP_ALG_AAI_HMAC_PRECOMP, 2199 }, 2200 }, 2201 { 2202 .aead = { 2203 .base = { 2204 .cra_name = "authenc(hmac(md5),cbc(aes))", 2205 .cra_driver_name = "authenc-hmac-md5-" 2206 "cbc-aes-caam", 2207 .cra_blocksize = AES_BLOCK_SIZE, 2208 }, 2209 .setkey = aead_setkey, 2210 .setauthsize = aead_setauthsize, 2211 .encrypt = aead_encrypt, 2212 .decrypt = aead_decrypt, 2213 .ivsize = AES_BLOCK_SIZE, 2214 .maxauthsize = MD5_DIGEST_SIZE, 2215 }, 2216 .caam = { 2217 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 2218 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 2219 OP_ALG_AAI_HMAC_PRECOMP, 2220 }, 2221 }, 2222 { 2223 .aead = { 2224 .base = { 2225 .cra_name = "echainiv(authenc(hmac(md5)," 2226 "cbc(aes)))", 2227 .cra_driver_name = "echainiv-authenc-hmac-md5-" 2228 "cbc-aes-caam", 2229 .cra_blocksize = AES_BLOCK_SIZE, 2230 }, 2231 .setkey = aead_setkey, 2232 .setauthsize = aead_setauthsize, 2233 .encrypt = aead_encrypt, 2234 .decrypt = aead_decrypt, 2235 .ivsize = AES_BLOCK_SIZE, 2236 .maxauthsize = MD5_DIGEST_SIZE, 2237 }, 2238 .caam = { 2239 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 2240 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 2241 OP_ALG_AAI_HMAC_PRECOMP, 2242 .geniv = true, 2243 }, 2244 }, 2245 { 2246 .aead = { 2247 .base = { 2248 .cra_name = "authenc(hmac(sha1),cbc(aes))", 2249 .cra_driver_name = "authenc-hmac-sha1-" 2250 "cbc-aes-caam", 2251 .cra_blocksize = AES_BLOCK_SIZE, 2252 }, 2253 .setkey = aead_setkey, 2254 .setauthsize = aead_setauthsize, 2255 .encrypt = aead_encrypt, 2256 .decrypt = aead_decrypt, 2257 .ivsize = AES_BLOCK_SIZE, 2258 .maxauthsize = SHA1_DIGEST_SIZE, 2259 }, 2260 .caam = { 2261 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 2262 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 2263 OP_ALG_AAI_HMAC_PRECOMP, 2264 }, 2265 }, 2266 { 2267 .aead = { 2268 .base = { 2269 .cra_name = "echainiv(authenc(hmac(sha1)," 2270 "cbc(aes)))", 2271 .cra_driver_name = "echainiv-authenc-" 2272 "hmac-sha1-cbc-aes-caam", 2273 .cra_blocksize = AES_BLOCK_SIZE, 2274 }, 2275 .setkey = aead_setkey, 2276 .setauthsize = aead_setauthsize, 2277 .encrypt = aead_encrypt, 2278 .decrypt = aead_decrypt, 2279 .ivsize = AES_BLOCK_SIZE, 2280 .maxauthsize = SHA1_DIGEST_SIZE, 2281 }, 2282 .caam = { 2283 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 2284 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 2285 OP_ALG_AAI_HMAC_PRECOMP, 2286 .geniv = true, 2287 }, 2288 }, 2289 { 2290 .aead = { 2291 .base = { 2292 .cra_name = "authenc(hmac(sha224),cbc(aes))", 2293 .cra_driver_name = "authenc-hmac-sha224-" 2294 "cbc-aes-caam", 2295 .cra_blocksize = AES_BLOCK_SIZE, 2296 }, 2297 .setkey = aead_setkey, 2298 .setauthsize = aead_setauthsize, 2299 .encrypt = aead_encrypt, 2300 .decrypt = aead_decrypt, 2301 .ivsize = AES_BLOCK_SIZE, 2302 .maxauthsize = SHA224_DIGEST_SIZE, 2303 }, 2304 .caam = { 2305 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 2306 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 2307 OP_ALG_AAI_HMAC_PRECOMP, 2308 }, 2309 }, 2310 { 2311 .aead = { 2312 .base = { 2313 .cra_name = "echainiv(authenc(hmac(sha224)," 2314 "cbc(aes)))", 2315 .cra_driver_name = "echainiv-authenc-" 2316 "hmac-sha224-cbc-aes-caam", 2317 .cra_blocksize = AES_BLOCK_SIZE, 2318 }, 2319 .setkey = aead_setkey, 2320 .setauthsize = aead_setauthsize, 2321 .encrypt = aead_encrypt, 2322 .decrypt = aead_decrypt, 2323 .ivsize = AES_BLOCK_SIZE, 2324 .maxauthsize = SHA224_DIGEST_SIZE, 2325 }, 2326 .caam = { 2327 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 2328 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 2329 OP_ALG_AAI_HMAC_PRECOMP, 2330 .geniv = true, 2331 }, 2332 }, 2333 { 2334 .aead = { 2335 .base = { 2336 .cra_name = "authenc(hmac(sha256),cbc(aes))", 2337 .cra_driver_name = "authenc-hmac-sha256-" 2338 "cbc-aes-caam", 2339 .cra_blocksize = AES_BLOCK_SIZE, 2340 }, 2341 .setkey = aead_setkey, 2342 .setauthsize = aead_setauthsize, 2343 .encrypt = aead_encrypt, 2344 .decrypt = aead_decrypt, 2345 .ivsize = AES_BLOCK_SIZE, 2346 .maxauthsize = SHA256_DIGEST_SIZE, 2347 }, 2348 .caam = { 2349 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 2350 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 2351 OP_ALG_AAI_HMAC_PRECOMP, 2352 }, 2353 }, 2354 { 2355 .aead = { 2356 .base = { 2357 .cra_name = "echainiv(authenc(hmac(sha256)," 2358 "cbc(aes)))", 2359 .cra_driver_name = "echainiv-authenc-" 2360 "hmac-sha256-cbc-aes-caam", 2361 .cra_blocksize = AES_BLOCK_SIZE, 2362 }, 2363 .setkey = aead_setkey, 2364 .setauthsize = aead_setauthsize, 2365 .encrypt = aead_encrypt, 2366 .decrypt = aead_decrypt, 2367 .ivsize = AES_BLOCK_SIZE, 2368 .maxauthsize = SHA256_DIGEST_SIZE, 2369 }, 2370 .caam = { 2371 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 2372 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 2373 OP_ALG_AAI_HMAC_PRECOMP, 2374 .geniv = true, 2375 }, 2376 }, 2377 { 2378 .aead = { 2379 .base = { 2380 .cra_name = "authenc(hmac(sha384),cbc(aes))", 2381 .cra_driver_name = "authenc-hmac-sha384-" 2382 "cbc-aes-caam", 2383 .cra_blocksize = AES_BLOCK_SIZE, 2384 }, 2385 .setkey = aead_setkey, 2386 .setauthsize = aead_setauthsize, 2387 .encrypt = aead_encrypt, 2388 .decrypt = aead_decrypt, 2389 .ivsize = AES_BLOCK_SIZE, 2390 .maxauthsize = SHA384_DIGEST_SIZE, 2391 }, 2392 .caam = { 2393 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 2394 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 2395 OP_ALG_AAI_HMAC_PRECOMP, 2396 }, 2397 }, 2398 { 2399 .aead = { 2400 .base = { 2401 .cra_name = "echainiv(authenc(hmac(sha384)," 2402 "cbc(aes)))", 2403 .cra_driver_name = "echainiv-authenc-" 2404 "hmac-sha384-cbc-aes-caam", 2405 .cra_blocksize = AES_BLOCK_SIZE, 2406 }, 2407 .setkey = aead_setkey, 2408 .setauthsize = aead_setauthsize, 2409 .encrypt = aead_encrypt, 2410 .decrypt = aead_decrypt, 2411 .ivsize = AES_BLOCK_SIZE, 2412 .maxauthsize = SHA384_DIGEST_SIZE, 2413 }, 2414 .caam = { 2415 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 2416 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 2417 OP_ALG_AAI_HMAC_PRECOMP, 2418 .geniv = true, 2419 }, 2420 }, 2421 { 2422 .aead = { 2423 .base = { 2424 .cra_name = "authenc(hmac(sha512),cbc(aes))", 2425 .cra_driver_name = "authenc-hmac-sha512-" 2426 "cbc-aes-caam", 2427 .cra_blocksize = AES_BLOCK_SIZE, 2428 }, 2429 .setkey = aead_setkey, 2430 .setauthsize = aead_setauthsize, 2431 .encrypt = aead_encrypt, 2432 .decrypt = aead_decrypt, 2433 .ivsize = AES_BLOCK_SIZE, 2434 .maxauthsize = SHA512_DIGEST_SIZE, 2435 }, 2436 .caam = { 2437 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 2438 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 2439 OP_ALG_AAI_HMAC_PRECOMP, 2440 }, 2441 }, 2442 { 2443 .aead = { 2444 .base = { 2445 .cra_name = "echainiv(authenc(hmac(sha512)," 2446 "cbc(aes)))", 2447 .cra_driver_name = "echainiv-authenc-" 2448 "hmac-sha512-cbc-aes-caam", 2449 .cra_blocksize = AES_BLOCK_SIZE, 2450 }, 2451 .setkey = aead_setkey, 2452 .setauthsize = aead_setauthsize, 2453 .encrypt = aead_encrypt, 2454 .decrypt = aead_decrypt, 2455 .ivsize = AES_BLOCK_SIZE, 2456 .maxauthsize = SHA512_DIGEST_SIZE, 2457 }, 2458 .caam = { 2459 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 2460 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 2461 OP_ALG_AAI_HMAC_PRECOMP, 2462 .geniv = true, 2463 }, 2464 }, 2465 { 2466 .aead = { 2467 .base = { 2468 .cra_name = "authenc(hmac(md5),cbc(des3_ede))", 2469 .cra_driver_name = "authenc-hmac-md5-" 2470 "cbc-des3_ede-caam", 2471 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2472 }, 2473 .setkey = des3_aead_setkey, 2474 .setauthsize = aead_setauthsize, 2475 .encrypt = aead_encrypt, 2476 .decrypt = aead_decrypt, 2477 .ivsize = DES3_EDE_BLOCK_SIZE, 2478 .maxauthsize = MD5_DIGEST_SIZE, 2479 }, 2480 .caam = { 2481 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2482 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 2483 OP_ALG_AAI_HMAC_PRECOMP, 2484 } 2485 }, 2486 { 2487 .aead = { 2488 .base = { 2489 .cra_name = "echainiv(authenc(hmac(md5)," 2490 "cbc(des3_ede)))", 2491 .cra_driver_name = "echainiv-authenc-hmac-md5-" 2492 "cbc-des3_ede-caam", 2493 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2494 }, 2495 .setkey = des3_aead_setkey, 2496 .setauthsize = aead_setauthsize, 2497 .encrypt = aead_encrypt, 2498 .decrypt = aead_decrypt, 2499 .ivsize = DES3_EDE_BLOCK_SIZE, 2500 .maxauthsize = MD5_DIGEST_SIZE, 2501 }, 2502 .caam = { 2503 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2504 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 2505 OP_ALG_AAI_HMAC_PRECOMP, 2506 .geniv = true, 2507 } 2508 }, 2509 { 2510 .aead = { 2511 .base = { 2512 .cra_name = "authenc(hmac(sha1)," 2513 "cbc(des3_ede))", 2514 .cra_driver_name = "authenc-hmac-sha1-" 2515 "cbc-des3_ede-caam", 2516 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2517 }, 2518 .setkey = des3_aead_setkey, 2519 .setauthsize = aead_setauthsize, 2520 .encrypt = aead_encrypt, 2521 .decrypt = aead_decrypt, 2522 .ivsize = DES3_EDE_BLOCK_SIZE, 2523 .maxauthsize = SHA1_DIGEST_SIZE, 2524 }, 2525 .caam = { 2526 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2527 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 2528 OP_ALG_AAI_HMAC_PRECOMP, 2529 }, 2530 }, 2531 { 2532 .aead = { 2533 .base = { 2534 .cra_name = "echainiv(authenc(hmac(sha1)," 2535 "cbc(des3_ede)))", 2536 .cra_driver_name = "echainiv-authenc-" 2537 "hmac-sha1-" 2538 "cbc-des3_ede-caam", 2539 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2540 }, 2541 .setkey = des3_aead_setkey, 2542 .setauthsize = aead_setauthsize, 2543 .encrypt = aead_encrypt, 2544 .decrypt = aead_decrypt, 2545 .ivsize = DES3_EDE_BLOCK_SIZE, 2546 .maxauthsize = SHA1_DIGEST_SIZE, 2547 }, 2548 .caam = { 2549 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2550 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 2551 OP_ALG_AAI_HMAC_PRECOMP, 2552 .geniv = true, 2553 }, 2554 }, 2555 { 2556 .aead = { 2557 .base = { 2558 .cra_name = "authenc(hmac(sha224)," 2559 "cbc(des3_ede))", 2560 .cra_driver_name = "authenc-hmac-sha224-" 2561 "cbc-des3_ede-caam", 2562 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2563 }, 2564 .setkey = des3_aead_setkey, 2565 .setauthsize = aead_setauthsize, 2566 .encrypt = aead_encrypt, 2567 .decrypt = aead_decrypt, 2568 .ivsize = DES3_EDE_BLOCK_SIZE, 2569 .maxauthsize = SHA224_DIGEST_SIZE, 2570 }, 2571 .caam = { 2572 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2573 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 2574 OP_ALG_AAI_HMAC_PRECOMP, 2575 }, 2576 }, 2577 { 2578 .aead = { 2579 .base = { 2580 .cra_name = "echainiv(authenc(hmac(sha224)," 2581 "cbc(des3_ede)))", 2582 .cra_driver_name = "echainiv-authenc-" 2583 "hmac-sha224-" 2584 "cbc-des3_ede-caam", 2585 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2586 }, 2587 .setkey = des3_aead_setkey, 2588 .setauthsize = aead_setauthsize, 2589 .encrypt = aead_encrypt, 2590 .decrypt = aead_decrypt, 2591 .ivsize = DES3_EDE_BLOCK_SIZE, 2592 .maxauthsize = SHA224_DIGEST_SIZE, 2593 }, 2594 .caam = { 2595 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2596 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 2597 OP_ALG_AAI_HMAC_PRECOMP, 2598 .geniv = true, 2599 }, 2600 }, 2601 { 2602 .aead = { 2603 .base = { 2604 .cra_name = "authenc(hmac(sha256)," 2605 "cbc(des3_ede))", 2606 .cra_driver_name = "authenc-hmac-sha256-" 2607 "cbc-des3_ede-caam", 2608 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2609 }, 2610 .setkey = des3_aead_setkey, 2611 .setauthsize = aead_setauthsize, 2612 .encrypt = aead_encrypt, 2613 .decrypt = aead_decrypt, 2614 .ivsize = DES3_EDE_BLOCK_SIZE, 2615 .maxauthsize = SHA256_DIGEST_SIZE, 2616 }, 2617 .caam = { 2618 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2619 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 2620 OP_ALG_AAI_HMAC_PRECOMP, 2621 }, 2622 }, 2623 { 2624 .aead = { 2625 .base = { 2626 .cra_name = "echainiv(authenc(hmac(sha256)," 2627 "cbc(des3_ede)))", 2628 .cra_driver_name = "echainiv-authenc-" 2629 "hmac-sha256-" 2630 "cbc-des3_ede-caam", 2631 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2632 }, 2633 .setkey = des3_aead_setkey, 2634 .setauthsize = aead_setauthsize, 2635 .encrypt = aead_encrypt, 2636 .decrypt = aead_decrypt, 2637 .ivsize = DES3_EDE_BLOCK_SIZE, 2638 .maxauthsize = SHA256_DIGEST_SIZE, 2639 }, 2640 .caam = { 2641 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2642 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 2643 OP_ALG_AAI_HMAC_PRECOMP, 2644 .geniv = true, 2645 }, 2646 }, 2647 { 2648 .aead = { 2649 .base = { 2650 .cra_name = "authenc(hmac(sha384)," 2651 "cbc(des3_ede))", 2652 .cra_driver_name = "authenc-hmac-sha384-" 2653 "cbc-des3_ede-caam", 2654 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2655 }, 2656 .setkey = des3_aead_setkey, 2657 .setauthsize = aead_setauthsize, 2658 .encrypt = aead_encrypt, 2659 .decrypt = aead_decrypt, 2660 .ivsize = DES3_EDE_BLOCK_SIZE, 2661 .maxauthsize = SHA384_DIGEST_SIZE, 2662 }, 2663 .caam = { 2664 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2665 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 2666 OP_ALG_AAI_HMAC_PRECOMP, 2667 }, 2668 }, 2669 { 2670 .aead = { 2671 .base = { 2672 .cra_name = "echainiv(authenc(hmac(sha384)," 2673 "cbc(des3_ede)))", 2674 .cra_driver_name = "echainiv-authenc-" 2675 "hmac-sha384-" 2676 "cbc-des3_ede-caam", 2677 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2678 }, 2679 .setkey = des3_aead_setkey, 2680 .setauthsize = aead_setauthsize, 2681 .encrypt = aead_encrypt, 2682 .decrypt = aead_decrypt, 2683 .ivsize = DES3_EDE_BLOCK_SIZE, 2684 .maxauthsize = SHA384_DIGEST_SIZE, 2685 }, 2686 .caam = { 2687 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2688 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 2689 OP_ALG_AAI_HMAC_PRECOMP, 2690 .geniv = true, 2691 }, 2692 }, 2693 { 2694 .aead = { 2695 .base = { 2696 .cra_name = "authenc(hmac(sha512)," 2697 "cbc(des3_ede))", 2698 .cra_driver_name = "authenc-hmac-sha512-" 2699 "cbc-des3_ede-caam", 2700 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2701 }, 2702 .setkey = des3_aead_setkey, 2703 .setauthsize = aead_setauthsize, 2704 .encrypt = aead_encrypt, 2705 .decrypt = aead_decrypt, 2706 .ivsize = DES3_EDE_BLOCK_SIZE, 2707 .maxauthsize = SHA512_DIGEST_SIZE, 2708 }, 2709 .caam = { 2710 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2711 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 2712 OP_ALG_AAI_HMAC_PRECOMP, 2713 }, 2714 }, 2715 { 2716 .aead = { 2717 .base = { 2718 .cra_name = "echainiv(authenc(hmac(sha512)," 2719 "cbc(des3_ede)))", 2720 .cra_driver_name = "echainiv-authenc-" 2721 "hmac-sha512-" 2722 "cbc-des3_ede-caam", 2723 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2724 }, 2725 .setkey = des3_aead_setkey, 2726 .setauthsize = aead_setauthsize, 2727 .encrypt = aead_encrypt, 2728 .decrypt = aead_decrypt, 2729 .ivsize = DES3_EDE_BLOCK_SIZE, 2730 .maxauthsize = SHA512_DIGEST_SIZE, 2731 }, 2732 .caam = { 2733 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2734 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 2735 OP_ALG_AAI_HMAC_PRECOMP, 2736 .geniv = true, 2737 }, 2738 }, 2739 { 2740 .aead = { 2741 .base = { 2742 .cra_name = "authenc(hmac(md5),cbc(des))", 2743 .cra_driver_name = "authenc-hmac-md5-" 2744 "cbc-des-caam", 2745 .cra_blocksize = DES_BLOCK_SIZE, 2746 }, 2747 .setkey = aead_setkey, 2748 .setauthsize = aead_setauthsize, 2749 .encrypt = aead_encrypt, 2750 .decrypt = aead_decrypt, 2751 .ivsize = DES_BLOCK_SIZE, 2752 .maxauthsize = MD5_DIGEST_SIZE, 2753 }, 2754 .caam = { 2755 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2756 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 2757 OP_ALG_AAI_HMAC_PRECOMP, 2758 }, 2759 }, 2760 { 2761 .aead = { 2762 .base = { 2763 .cra_name = "echainiv(authenc(hmac(md5)," 2764 "cbc(des)))", 2765 .cra_driver_name = "echainiv-authenc-hmac-md5-" 2766 "cbc-des-caam", 2767 .cra_blocksize = DES_BLOCK_SIZE, 2768 }, 2769 .setkey = aead_setkey, 2770 .setauthsize = aead_setauthsize, 2771 .encrypt = aead_encrypt, 2772 .decrypt = aead_decrypt, 2773 .ivsize = DES_BLOCK_SIZE, 2774 .maxauthsize = MD5_DIGEST_SIZE, 2775 }, 2776 .caam = { 2777 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2778 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 2779 OP_ALG_AAI_HMAC_PRECOMP, 2780 .geniv = true, 2781 }, 2782 }, 2783 { 2784 .aead = { 2785 .base = { 2786 .cra_name = "authenc(hmac(sha1),cbc(des))", 2787 .cra_driver_name = "authenc-hmac-sha1-" 2788 "cbc-des-caam", 2789 .cra_blocksize = DES_BLOCK_SIZE, 2790 }, 2791 .setkey = aead_setkey, 2792 .setauthsize = aead_setauthsize, 2793 .encrypt = aead_encrypt, 2794 .decrypt = aead_decrypt, 2795 .ivsize = DES_BLOCK_SIZE, 2796 .maxauthsize = SHA1_DIGEST_SIZE, 2797 }, 2798 .caam = { 2799 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2800 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 2801 OP_ALG_AAI_HMAC_PRECOMP, 2802 }, 2803 }, 2804 { 2805 .aead = { 2806 .base = { 2807 .cra_name = "echainiv(authenc(hmac(sha1)," 2808 "cbc(des)))", 2809 .cra_driver_name = "echainiv-authenc-" 2810 "hmac-sha1-cbc-des-caam", 2811 .cra_blocksize = DES_BLOCK_SIZE, 2812 }, 2813 .setkey = aead_setkey, 2814 .setauthsize = aead_setauthsize, 2815 .encrypt = aead_encrypt, 2816 .decrypt = aead_decrypt, 2817 .ivsize = DES_BLOCK_SIZE, 2818 .maxauthsize = SHA1_DIGEST_SIZE, 2819 }, 2820 .caam = { 2821 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2822 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 2823 OP_ALG_AAI_HMAC_PRECOMP, 2824 .geniv = true, 2825 }, 2826 }, 2827 { 2828 .aead = { 2829 .base = { 2830 .cra_name = "authenc(hmac(sha224),cbc(des))", 2831 .cra_driver_name = "authenc-hmac-sha224-" 2832 "cbc-des-caam", 2833 .cra_blocksize = DES_BLOCK_SIZE, 2834 }, 2835 .setkey = aead_setkey, 2836 .setauthsize = aead_setauthsize, 2837 .encrypt = aead_encrypt, 2838 .decrypt = aead_decrypt, 2839 .ivsize = DES_BLOCK_SIZE, 2840 .maxauthsize = SHA224_DIGEST_SIZE, 2841 }, 2842 .caam = { 2843 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2844 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 2845 OP_ALG_AAI_HMAC_PRECOMP, 2846 }, 2847 }, 2848 { 2849 .aead = { 2850 .base = { 2851 .cra_name = "echainiv(authenc(hmac(sha224)," 2852 "cbc(des)))", 2853 .cra_driver_name = "echainiv-authenc-" 2854 "hmac-sha224-cbc-des-caam", 2855 .cra_blocksize = DES_BLOCK_SIZE, 2856 }, 2857 .setkey = aead_setkey, 2858 .setauthsize = aead_setauthsize, 2859 .encrypt = aead_encrypt, 2860 .decrypt = aead_decrypt, 2861 .ivsize = DES_BLOCK_SIZE, 2862 .maxauthsize = SHA224_DIGEST_SIZE, 2863 }, 2864 .caam = { 2865 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2866 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 2867 OP_ALG_AAI_HMAC_PRECOMP, 2868 .geniv = true, 2869 }, 2870 }, 2871 { 2872 .aead = { 2873 .base = { 2874 .cra_name = "authenc(hmac(sha256),cbc(des))", 2875 .cra_driver_name = "authenc-hmac-sha256-" 2876 "cbc-des-caam", 2877 .cra_blocksize = DES_BLOCK_SIZE, 2878 }, 2879 .setkey = aead_setkey, 2880 .setauthsize = aead_setauthsize, 2881 .encrypt = aead_encrypt, 2882 .decrypt = aead_decrypt, 2883 .ivsize = DES_BLOCK_SIZE, 2884 .maxauthsize = SHA256_DIGEST_SIZE, 2885 }, 2886 .caam = { 2887 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2888 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 2889 OP_ALG_AAI_HMAC_PRECOMP, 2890 }, 2891 }, 2892 { 2893 .aead = { 2894 .base = { 2895 .cra_name = "echainiv(authenc(hmac(sha256)," 2896 "cbc(des)))", 2897 .cra_driver_name = "echainiv-authenc-" 2898 "hmac-sha256-cbc-des-caam", 2899 .cra_blocksize = DES_BLOCK_SIZE, 2900 }, 2901 .setkey = aead_setkey, 2902 .setauthsize = aead_setauthsize, 2903 .encrypt = aead_encrypt, 2904 .decrypt = aead_decrypt, 2905 .ivsize = DES_BLOCK_SIZE, 2906 .maxauthsize = SHA256_DIGEST_SIZE, 2907 }, 2908 .caam = { 2909 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2910 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 2911 OP_ALG_AAI_HMAC_PRECOMP, 2912 .geniv = true, 2913 }, 2914 }, 2915 { 2916 .aead = { 2917 .base = { 2918 .cra_name = "authenc(hmac(sha384),cbc(des))", 2919 .cra_driver_name = "authenc-hmac-sha384-" 2920 "cbc-des-caam", 2921 .cra_blocksize = DES_BLOCK_SIZE, 2922 }, 2923 .setkey = aead_setkey, 2924 .setauthsize = aead_setauthsize, 2925 .encrypt = aead_encrypt, 2926 .decrypt = aead_decrypt, 2927 .ivsize = DES_BLOCK_SIZE, 2928 .maxauthsize = SHA384_DIGEST_SIZE, 2929 }, 2930 .caam = { 2931 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2932 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 2933 OP_ALG_AAI_HMAC_PRECOMP, 2934 }, 2935 }, 2936 { 2937 .aead = { 2938 .base = { 2939 .cra_name = "echainiv(authenc(hmac(sha384)," 2940 "cbc(des)))", 2941 .cra_driver_name = "echainiv-authenc-" 2942 "hmac-sha384-cbc-des-caam", 2943 .cra_blocksize = DES_BLOCK_SIZE, 2944 }, 2945 .setkey = aead_setkey, 2946 .setauthsize = aead_setauthsize, 2947 .encrypt = aead_encrypt, 2948 .decrypt = aead_decrypt, 2949 .ivsize = DES_BLOCK_SIZE, 2950 .maxauthsize = SHA384_DIGEST_SIZE, 2951 }, 2952 .caam = { 2953 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2954 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 2955 OP_ALG_AAI_HMAC_PRECOMP, 2956 .geniv = true, 2957 }, 2958 }, 2959 { 2960 .aead = { 2961 .base = { 2962 .cra_name = "authenc(hmac(sha512),cbc(des))", 2963 .cra_driver_name = "authenc-hmac-sha512-" 2964 "cbc-des-caam", 2965 .cra_blocksize = DES_BLOCK_SIZE, 2966 }, 2967 .setkey = aead_setkey, 2968 .setauthsize = aead_setauthsize, 2969 .encrypt = aead_encrypt, 2970 .decrypt = aead_decrypt, 2971 .ivsize = DES_BLOCK_SIZE, 2972 .maxauthsize = SHA512_DIGEST_SIZE, 2973 }, 2974 .caam = { 2975 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2976 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 2977 OP_ALG_AAI_HMAC_PRECOMP, 2978 }, 2979 }, 2980 { 2981 .aead = { 2982 .base = { 2983 .cra_name = "echainiv(authenc(hmac(sha512)," 2984 "cbc(des)))", 2985 .cra_driver_name = "echainiv-authenc-" 2986 "hmac-sha512-cbc-des-caam", 2987 .cra_blocksize = DES_BLOCK_SIZE, 2988 }, 2989 .setkey = aead_setkey, 2990 .setauthsize = aead_setauthsize, 2991 .encrypt = aead_encrypt, 2992 .decrypt = aead_decrypt, 2993 .ivsize = DES_BLOCK_SIZE, 2994 .maxauthsize = SHA512_DIGEST_SIZE, 2995 }, 2996 .caam = { 2997 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2998 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 2999 OP_ALG_AAI_HMAC_PRECOMP, 3000 .geniv = true, 3001 }, 3002 }, 3003 { 3004 .aead = { 3005 .base = { 3006 .cra_name = "authenc(hmac(md5)," 3007 "rfc3686(ctr(aes)))", 3008 .cra_driver_name = "authenc-hmac-md5-" 3009 "rfc3686-ctr-aes-caam", 3010 .cra_blocksize = 1, 3011 }, 3012 .setkey = aead_setkey, 3013 .setauthsize = aead_setauthsize, 3014 .encrypt = aead_encrypt, 3015 .decrypt = aead_decrypt, 3016 .ivsize = CTR_RFC3686_IV_SIZE, 3017 .maxauthsize = MD5_DIGEST_SIZE, 3018 }, 3019 .caam = { 3020 .class1_alg_type = OP_ALG_ALGSEL_AES | 3021 OP_ALG_AAI_CTR_MOD128, 3022 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 3023 OP_ALG_AAI_HMAC_PRECOMP, 3024 .rfc3686 = true, 3025 }, 3026 }, 3027 { 3028 .aead = { 3029 .base = { 3030 .cra_name = "seqiv(authenc(" 3031 "hmac(md5),rfc3686(ctr(aes))))", 3032 .cra_driver_name = "seqiv-authenc-hmac-md5-" 3033 "rfc3686-ctr-aes-caam", 3034 .cra_blocksize = 1, 3035 }, 3036 .setkey = aead_setkey, 3037 .setauthsize = aead_setauthsize, 3038 .encrypt = aead_encrypt, 3039 .decrypt = aead_decrypt, 3040 .ivsize = CTR_RFC3686_IV_SIZE, 3041 .maxauthsize = MD5_DIGEST_SIZE, 3042 }, 3043 .caam = { 3044 .class1_alg_type = OP_ALG_ALGSEL_AES | 3045 OP_ALG_AAI_CTR_MOD128, 3046 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 3047 OP_ALG_AAI_HMAC_PRECOMP, 3048 .rfc3686 = true, 3049 .geniv = true, 3050 }, 3051 }, 3052 { 3053 .aead = { 3054 .base = { 3055 .cra_name = "authenc(hmac(sha1)," 3056 "rfc3686(ctr(aes)))", 3057 .cra_driver_name = "authenc-hmac-sha1-" 3058 "rfc3686-ctr-aes-caam", 3059 .cra_blocksize = 1, 3060 }, 3061 .setkey = aead_setkey, 3062 .setauthsize = aead_setauthsize, 3063 .encrypt = aead_encrypt, 3064 .decrypt = aead_decrypt, 3065 .ivsize = CTR_RFC3686_IV_SIZE, 3066 .maxauthsize = SHA1_DIGEST_SIZE, 3067 }, 3068 .caam = { 3069 .class1_alg_type = OP_ALG_ALGSEL_AES | 3070 OP_ALG_AAI_CTR_MOD128, 3071 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 3072 OP_ALG_AAI_HMAC_PRECOMP, 3073 .rfc3686 = true, 3074 }, 3075 }, 3076 { 3077 .aead = { 3078 .base = { 3079 .cra_name = "seqiv(authenc(" 3080 "hmac(sha1),rfc3686(ctr(aes))))", 3081 .cra_driver_name = "seqiv-authenc-hmac-sha1-" 3082 "rfc3686-ctr-aes-caam", 3083 .cra_blocksize = 1, 3084 }, 3085 .setkey = aead_setkey, 3086 .setauthsize = aead_setauthsize, 3087 .encrypt = aead_encrypt, 3088 .decrypt = aead_decrypt, 3089 .ivsize = CTR_RFC3686_IV_SIZE, 3090 .maxauthsize = SHA1_DIGEST_SIZE, 3091 }, 3092 .caam = { 3093 .class1_alg_type = OP_ALG_ALGSEL_AES | 3094 OP_ALG_AAI_CTR_MOD128, 3095 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 3096 OP_ALG_AAI_HMAC_PRECOMP, 3097 .rfc3686 = true, 3098 .geniv = true, 3099 }, 3100 }, 3101 { 3102 .aead = { 3103 .base = { 3104 .cra_name = "authenc(hmac(sha224)," 3105 "rfc3686(ctr(aes)))", 3106 .cra_driver_name = "authenc-hmac-sha224-" 3107 "rfc3686-ctr-aes-caam", 3108 .cra_blocksize = 1, 3109 }, 3110 .setkey = aead_setkey, 3111 .setauthsize = aead_setauthsize, 3112 .encrypt = aead_encrypt, 3113 .decrypt = aead_decrypt, 3114 .ivsize = CTR_RFC3686_IV_SIZE, 3115 .maxauthsize = SHA224_DIGEST_SIZE, 3116 }, 3117 .caam = { 3118 .class1_alg_type = OP_ALG_ALGSEL_AES | 3119 OP_ALG_AAI_CTR_MOD128, 3120 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 3121 OP_ALG_AAI_HMAC_PRECOMP, 3122 .rfc3686 = true, 3123 }, 3124 }, 3125 { 3126 .aead = { 3127 .base = { 3128 .cra_name = "seqiv(authenc(" 3129 "hmac(sha224),rfc3686(ctr(aes))))", 3130 .cra_driver_name = "seqiv-authenc-hmac-sha224-" 3131 "rfc3686-ctr-aes-caam", 3132 .cra_blocksize = 1, 3133 }, 3134 .setkey = aead_setkey, 3135 .setauthsize = aead_setauthsize, 3136 .encrypt = aead_encrypt, 3137 .decrypt = aead_decrypt, 3138 .ivsize = CTR_RFC3686_IV_SIZE, 3139 .maxauthsize = SHA224_DIGEST_SIZE, 3140 }, 3141 .caam = { 3142 .class1_alg_type = OP_ALG_ALGSEL_AES | 3143 OP_ALG_AAI_CTR_MOD128, 3144 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 3145 OP_ALG_AAI_HMAC_PRECOMP, 3146 .rfc3686 = true, 3147 .geniv = true, 3148 }, 3149 }, 3150 { 3151 .aead = { 3152 .base = { 3153 .cra_name = "authenc(hmac(sha256)," 3154 "rfc3686(ctr(aes)))", 3155 .cra_driver_name = "authenc-hmac-sha256-" 3156 "rfc3686-ctr-aes-caam", 3157 .cra_blocksize = 1, 3158 }, 3159 .setkey = aead_setkey, 3160 .setauthsize = aead_setauthsize, 3161 .encrypt = aead_encrypt, 3162 .decrypt = aead_decrypt, 3163 .ivsize = CTR_RFC3686_IV_SIZE, 3164 .maxauthsize = SHA256_DIGEST_SIZE, 3165 }, 3166 .caam = { 3167 .class1_alg_type = OP_ALG_ALGSEL_AES | 3168 OP_ALG_AAI_CTR_MOD128, 3169 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 3170 OP_ALG_AAI_HMAC_PRECOMP, 3171 .rfc3686 = true, 3172 }, 3173 }, 3174 { 3175 .aead = { 3176 .base = { 3177 .cra_name = "seqiv(authenc(hmac(sha256)," 3178 "rfc3686(ctr(aes))))", 3179 .cra_driver_name = "seqiv-authenc-hmac-sha256-" 3180 "rfc3686-ctr-aes-caam", 3181 .cra_blocksize = 1, 3182 }, 3183 .setkey = aead_setkey, 3184 .setauthsize = aead_setauthsize, 3185 .encrypt = aead_encrypt, 3186 .decrypt = aead_decrypt, 3187 .ivsize = CTR_RFC3686_IV_SIZE, 3188 .maxauthsize = SHA256_DIGEST_SIZE, 3189 }, 3190 .caam = { 3191 .class1_alg_type = OP_ALG_ALGSEL_AES | 3192 OP_ALG_AAI_CTR_MOD128, 3193 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 3194 OP_ALG_AAI_HMAC_PRECOMP, 3195 .rfc3686 = true, 3196 .geniv = true, 3197 }, 3198 }, 3199 { 3200 .aead = { 3201 .base = { 3202 .cra_name = "authenc(hmac(sha384)," 3203 "rfc3686(ctr(aes)))", 3204 .cra_driver_name = "authenc-hmac-sha384-" 3205 "rfc3686-ctr-aes-caam", 3206 .cra_blocksize = 1, 3207 }, 3208 .setkey = aead_setkey, 3209 .setauthsize = aead_setauthsize, 3210 .encrypt = aead_encrypt, 3211 .decrypt = aead_decrypt, 3212 .ivsize = CTR_RFC3686_IV_SIZE, 3213 .maxauthsize = SHA384_DIGEST_SIZE, 3214 }, 3215 .caam = { 3216 .class1_alg_type = OP_ALG_ALGSEL_AES | 3217 OP_ALG_AAI_CTR_MOD128, 3218 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 3219 OP_ALG_AAI_HMAC_PRECOMP, 3220 .rfc3686 = true, 3221 }, 3222 }, 3223 { 3224 .aead = { 3225 .base = { 3226 .cra_name = "seqiv(authenc(hmac(sha384)," 3227 "rfc3686(ctr(aes))))", 3228 .cra_driver_name = "seqiv-authenc-hmac-sha384-" 3229 "rfc3686-ctr-aes-caam", 3230 .cra_blocksize = 1, 3231 }, 3232 .setkey = aead_setkey, 3233 .setauthsize = aead_setauthsize, 3234 .encrypt = aead_encrypt, 3235 .decrypt = aead_decrypt, 3236 .ivsize = CTR_RFC3686_IV_SIZE, 3237 .maxauthsize = SHA384_DIGEST_SIZE, 3238 }, 3239 .caam = { 3240 .class1_alg_type = OP_ALG_ALGSEL_AES | 3241 OP_ALG_AAI_CTR_MOD128, 3242 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 3243 OP_ALG_AAI_HMAC_PRECOMP, 3244 .rfc3686 = true, 3245 .geniv = true, 3246 }, 3247 }, 3248 { 3249 .aead = { 3250 .base = { 3251 .cra_name = "authenc(hmac(sha512)," 3252 "rfc3686(ctr(aes)))", 3253 .cra_driver_name = "authenc-hmac-sha512-" 3254 "rfc3686-ctr-aes-caam", 3255 .cra_blocksize = 1, 3256 }, 3257 .setkey = aead_setkey, 3258 .setauthsize = aead_setauthsize, 3259 .encrypt = aead_encrypt, 3260 .decrypt = aead_decrypt, 3261 .ivsize = CTR_RFC3686_IV_SIZE, 3262 .maxauthsize = SHA512_DIGEST_SIZE, 3263 }, 3264 .caam = { 3265 .class1_alg_type = OP_ALG_ALGSEL_AES | 3266 OP_ALG_AAI_CTR_MOD128, 3267 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 3268 OP_ALG_AAI_HMAC_PRECOMP, 3269 .rfc3686 = true, 3270 }, 3271 }, 3272 { 3273 .aead = { 3274 .base = { 3275 .cra_name = "seqiv(authenc(hmac(sha512)," 3276 "rfc3686(ctr(aes))))", 3277 .cra_driver_name = "seqiv-authenc-hmac-sha512-" 3278 "rfc3686-ctr-aes-caam", 3279 .cra_blocksize = 1, 3280 }, 3281 .setkey = aead_setkey, 3282 .setauthsize = aead_setauthsize, 3283 .encrypt = aead_encrypt, 3284 .decrypt = aead_decrypt, 3285 .ivsize = CTR_RFC3686_IV_SIZE, 3286 .maxauthsize = SHA512_DIGEST_SIZE, 3287 }, 3288 .caam = { 3289 .class1_alg_type = OP_ALG_ALGSEL_AES | 3290 OP_ALG_AAI_CTR_MOD128, 3291 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 3292 OP_ALG_AAI_HMAC_PRECOMP, 3293 .rfc3686 = true, 3294 .geniv = true, 3295 }, 3296 }, 3297 { 3298 .aead = { 3299 .base = { 3300 .cra_name = "rfc7539(chacha20,poly1305)", 3301 .cra_driver_name = "rfc7539-chacha20-poly1305-" 3302 "caam", 3303 .cra_blocksize = 1, 3304 }, 3305 .setkey = chachapoly_setkey, 3306 .setauthsize = chachapoly_setauthsize, 3307 .encrypt = chachapoly_encrypt, 3308 .decrypt = chachapoly_decrypt, 3309 .ivsize = CHACHAPOLY_IV_SIZE, 3310 .maxauthsize = POLY1305_DIGEST_SIZE, 3311 }, 3312 .caam = { 3313 .class1_alg_type = OP_ALG_ALGSEL_CHACHA20 | 3314 OP_ALG_AAI_AEAD, 3315 .class2_alg_type = OP_ALG_ALGSEL_POLY1305 | 3316 OP_ALG_AAI_AEAD, 3317 .nodkp = true, 3318 }, 3319 }, 3320 { 3321 .aead = { 3322 .base = { 3323 .cra_name = "rfc7539esp(chacha20,poly1305)", 3324 .cra_driver_name = "rfc7539esp-chacha20-" 3325 "poly1305-caam", 3326 .cra_blocksize = 1, 3327 }, 3328 .setkey = chachapoly_setkey, 3329 .setauthsize = chachapoly_setauthsize, 3330 .encrypt = chachapoly_encrypt, 3331 .decrypt = chachapoly_decrypt, 3332 .ivsize = 8, 3333 .maxauthsize = POLY1305_DIGEST_SIZE, 3334 }, 3335 .caam = { 3336 .class1_alg_type = OP_ALG_ALGSEL_CHACHA20 | 3337 OP_ALG_AAI_AEAD, 3338 .class2_alg_type = OP_ALG_ALGSEL_POLY1305 | 3339 OP_ALG_AAI_AEAD, 3340 .nodkp = true, 3341 }, 3342 }, 3343}; 3344 3345static int caam_init_common(struct caam_ctx *ctx, struct caam_alg_entry *caam, 3346 bool uses_dkp) 3347{ 3348 dma_addr_t dma_addr; 3349 struct caam_drv_private *priv; 3350 const size_t sh_desc_enc_offset = offsetof(struct caam_ctx, 3351 sh_desc_enc); 3352 3353 ctx->jrdev = caam_jr_alloc(); 3354 if (IS_ERR(ctx->jrdev)) { 3355 pr_err("Job Ring Device allocation for transform failed\n"); 3356 return PTR_ERR(ctx->jrdev); 3357 } 3358 3359 priv = dev_get_drvdata(ctx->jrdev->parent); 3360 if (priv->era >= 6 && uses_dkp) 3361 ctx->dir = DMA_BIDIRECTIONAL; 3362 else 3363 ctx->dir = DMA_TO_DEVICE; 3364 3365 dma_addr = dma_map_single_attrs(ctx->jrdev, ctx->sh_desc_enc, 3366 offsetof(struct caam_ctx, 3367 sh_desc_enc_dma) - 3368 sh_desc_enc_offset, 3369 ctx->dir, DMA_ATTR_SKIP_CPU_SYNC); 3370 if (dma_mapping_error(ctx->jrdev, dma_addr)) { 3371 dev_err(ctx->jrdev, "unable to map key, shared descriptors\n"); 3372 caam_jr_free(ctx->jrdev); 3373 return -ENOMEM; 3374 } 3375 3376 ctx->sh_desc_enc_dma = dma_addr; 3377 ctx->sh_desc_dec_dma = dma_addr + offsetof(struct caam_ctx, 3378 sh_desc_dec) - 3379 sh_desc_enc_offset; 3380 ctx->key_dma = dma_addr + offsetof(struct caam_ctx, key) - 3381 sh_desc_enc_offset; 3382 3383 /* copy descriptor header template value */ 3384 ctx->cdata.algtype = OP_TYPE_CLASS1_ALG | caam->class1_alg_type; 3385 ctx->adata.algtype = OP_TYPE_CLASS2_ALG | caam->class2_alg_type; 3386 3387 return 0; 3388} 3389 3390static int caam_cra_init(struct crypto_skcipher *tfm) 3391{ 3392 struct skcipher_alg *alg = crypto_skcipher_alg(tfm); 3393 struct caam_skcipher_alg *caam_alg = 3394 container_of(alg, typeof(*caam_alg), skcipher); 3395 struct caam_ctx *ctx = crypto_skcipher_ctx(tfm); 3396 u32 alg_aai = caam_alg->caam.class1_alg_type & OP_ALG_AAI_MASK; 3397 int ret = 0; 3398 3399 ctx->enginectx.op.do_one_request = skcipher_do_one_req; 3400 3401 if (alg_aai == OP_ALG_AAI_XTS) { 3402 const char *tfm_name = crypto_tfm_alg_name(&tfm->base); 3403 struct crypto_skcipher *fallback; 3404 3405 fallback = crypto_alloc_skcipher(tfm_name, 0, 3406 CRYPTO_ALG_NEED_FALLBACK); 3407 if (IS_ERR(fallback)) { 3408 pr_err("Failed to allocate %s fallback: %ld\n", 3409 tfm_name, PTR_ERR(fallback)); 3410 return PTR_ERR(fallback); 3411 } 3412 3413 ctx->fallback = fallback; 3414 crypto_skcipher_set_reqsize(tfm, sizeof(struct caam_skcipher_req_ctx) + 3415 crypto_skcipher_reqsize(fallback)); 3416 } else { 3417 crypto_skcipher_set_reqsize(tfm, sizeof(struct caam_skcipher_req_ctx)); 3418 } 3419 3420 ret = caam_init_common(ctx, &caam_alg->caam, false); 3421 if (ret && ctx->fallback) 3422 crypto_free_skcipher(ctx->fallback); 3423 3424 return ret; 3425} 3426 3427static int caam_aead_init(struct crypto_aead *tfm) 3428{ 3429 struct aead_alg *alg = crypto_aead_alg(tfm); 3430 struct caam_aead_alg *caam_alg = 3431 container_of(alg, struct caam_aead_alg, aead); 3432 struct caam_ctx *ctx = crypto_aead_ctx(tfm); 3433 3434 crypto_aead_set_reqsize(tfm, sizeof(struct caam_aead_req_ctx)); 3435 3436 ctx->enginectx.op.do_one_request = aead_do_one_req; 3437 3438 return caam_init_common(ctx, &caam_alg->caam, !caam_alg->caam.nodkp); 3439} 3440 3441static void caam_exit_common(struct caam_ctx *ctx) 3442{ 3443 dma_unmap_single_attrs(ctx->jrdev, ctx->sh_desc_enc_dma, 3444 offsetof(struct caam_ctx, sh_desc_enc_dma) - 3445 offsetof(struct caam_ctx, sh_desc_enc), 3446 ctx->dir, DMA_ATTR_SKIP_CPU_SYNC); 3447 caam_jr_free(ctx->jrdev); 3448} 3449 3450static void caam_cra_exit(struct crypto_skcipher *tfm) 3451{ 3452 struct caam_ctx *ctx = crypto_skcipher_ctx(tfm); 3453 3454 if (ctx->fallback) 3455 crypto_free_skcipher(ctx->fallback); 3456 caam_exit_common(ctx); 3457} 3458 3459static void caam_aead_exit(struct crypto_aead *tfm) 3460{ 3461 caam_exit_common(crypto_aead_ctx(tfm)); 3462} 3463 3464void caam_algapi_exit(void) 3465{ 3466 int i; 3467 3468 for (i = 0; i < ARRAY_SIZE(driver_aeads); i++) { 3469 struct caam_aead_alg *t_alg = driver_aeads + i; 3470 3471 if (t_alg->registered) 3472 crypto_unregister_aead(&t_alg->aead); 3473 } 3474 3475 for (i = 0; i < ARRAY_SIZE(driver_algs); i++) { 3476 struct caam_skcipher_alg *t_alg = driver_algs + i; 3477 3478 if (t_alg->registered) 3479 crypto_unregister_skcipher(&t_alg->skcipher); 3480 } 3481} 3482 3483static void caam_skcipher_alg_init(struct caam_skcipher_alg *t_alg) 3484{ 3485 struct skcipher_alg *alg = &t_alg->skcipher; 3486 3487 alg->base.cra_module = THIS_MODULE; 3488 alg->base.cra_priority = CAAM_CRA_PRIORITY; 3489 alg->base.cra_ctxsize = sizeof(struct caam_ctx); 3490 alg->base.cra_flags |= (CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY | 3491 CRYPTO_ALG_KERN_DRIVER_ONLY); 3492 3493 alg->init = caam_cra_init; 3494 alg->exit = caam_cra_exit; 3495} 3496 3497static void caam_aead_alg_init(struct caam_aead_alg *t_alg) 3498{ 3499 struct aead_alg *alg = &t_alg->aead; 3500 3501 alg->base.cra_module = THIS_MODULE; 3502 alg->base.cra_priority = CAAM_CRA_PRIORITY; 3503 alg->base.cra_ctxsize = sizeof(struct caam_ctx); 3504 alg->base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY | 3505 CRYPTO_ALG_KERN_DRIVER_ONLY; 3506 3507 alg->init = caam_aead_init; 3508 alg->exit = caam_aead_exit; 3509} 3510 3511int caam_algapi_init(struct device *ctrldev) 3512{ 3513 struct caam_drv_private *priv = dev_get_drvdata(ctrldev); 3514 int i = 0, err = 0; 3515 u32 aes_vid, aes_inst, des_inst, md_vid, md_inst, ccha_inst, ptha_inst; 3516 unsigned int md_limit = SHA512_DIGEST_SIZE; 3517 bool registered = false, gcm_support; 3518 3519 /* 3520 * Register crypto algorithms the device supports. 3521 * First, detect presence and attributes of DES, AES, and MD blocks. 3522 */ 3523 if (priv->era < 10) { 3524 u32 cha_vid, cha_inst, aes_rn; 3525 3526 cha_vid = rd_reg32(&priv->ctrl->perfmon.cha_id_ls); 3527 aes_vid = cha_vid & CHA_ID_LS_AES_MASK; 3528 md_vid = (cha_vid & CHA_ID_LS_MD_MASK) >> CHA_ID_LS_MD_SHIFT; 3529 3530 cha_inst = rd_reg32(&priv->ctrl->perfmon.cha_num_ls); 3531 des_inst = (cha_inst & CHA_ID_LS_DES_MASK) >> 3532 CHA_ID_LS_DES_SHIFT; 3533 aes_inst = cha_inst & CHA_ID_LS_AES_MASK; 3534 md_inst = (cha_inst & CHA_ID_LS_MD_MASK) >> CHA_ID_LS_MD_SHIFT; 3535 ccha_inst = 0; 3536 ptha_inst = 0; 3537 3538 aes_rn = rd_reg32(&priv->ctrl->perfmon.cha_rev_ls) & 3539 CHA_ID_LS_AES_MASK; 3540 gcm_support = !(aes_vid == CHA_VER_VID_AES_LP && aes_rn < 8); 3541 } else { 3542 u32 aesa, mdha; 3543 3544 aesa = rd_reg32(&priv->ctrl->vreg.aesa); 3545 mdha = rd_reg32(&priv->ctrl->vreg.mdha); 3546 3547 aes_vid = (aesa & CHA_VER_VID_MASK) >> CHA_VER_VID_SHIFT; 3548 md_vid = (mdha & CHA_VER_VID_MASK) >> CHA_VER_VID_SHIFT; 3549 3550 des_inst = rd_reg32(&priv->ctrl->vreg.desa) & CHA_VER_NUM_MASK; 3551 aes_inst = aesa & CHA_VER_NUM_MASK; 3552 md_inst = mdha & CHA_VER_NUM_MASK; 3553 ccha_inst = rd_reg32(&priv->ctrl->vreg.ccha) & CHA_VER_NUM_MASK; 3554 ptha_inst = rd_reg32(&priv->ctrl->vreg.ptha) & CHA_VER_NUM_MASK; 3555 3556 gcm_support = aesa & CHA_VER_MISC_AES_GCM; 3557 } 3558 3559 /* If MD is present, limit digest size based on LP256 */ 3560 if (md_inst && md_vid == CHA_VER_VID_MD_LP256) 3561 md_limit = SHA256_DIGEST_SIZE; 3562 3563 for (i = 0; i < ARRAY_SIZE(driver_algs); i++) { 3564 struct caam_skcipher_alg *t_alg = driver_algs + i; 3565 u32 alg_sel = t_alg->caam.class1_alg_type & OP_ALG_ALGSEL_MASK; 3566 3567 /* Skip DES algorithms if not supported by device */ 3568 if (!des_inst && 3569 ((alg_sel == OP_ALG_ALGSEL_3DES) || 3570 (alg_sel == OP_ALG_ALGSEL_DES))) 3571 continue; 3572 3573 /* Skip AES algorithms if not supported by device */ 3574 if (!aes_inst && (alg_sel == OP_ALG_ALGSEL_AES)) 3575 continue; 3576 3577 /* 3578 * Check support for AES modes not available 3579 * on LP devices. 3580 */ 3581 if (aes_vid == CHA_VER_VID_AES_LP && 3582 (t_alg->caam.class1_alg_type & OP_ALG_AAI_MASK) == 3583 OP_ALG_AAI_XTS) 3584 continue; 3585 3586 caam_skcipher_alg_init(t_alg); 3587 3588 err = crypto_register_skcipher(&t_alg->skcipher); 3589 if (err) { 3590 pr_warn("%s alg registration failed\n", 3591 t_alg->skcipher.base.cra_driver_name); 3592 continue; 3593 } 3594 3595 t_alg->registered = true; 3596 registered = true; 3597 } 3598 3599 for (i = 0; i < ARRAY_SIZE(driver_aeads); i++) { 3600 struct caam_aead_alg *t_alg = driver_aeads + i; 3601 u32 c1_alg_sel = t_alg->caam.class1_alg_type & 3602 OP_ALG_ALGSEL_MASK; 3603 u32 c2_alg_sel = t_alg->caam.class2_alg_type & 3604 OP_ALG_ALGSEL_MASK; 3605 u32 alg_aai = t_alg->caam.class1_alg_type & OP_ALG_AAI_MASK; 3606 3607 /* Skip DES algorithms if not supported by device */ 3608 if (!des_inst && 3609 ((c1_alg_sel == OP_ALG_ALGSEL_3DES) || 3610 (c1_alg_sel == OP_ALG_ALGSEL_DES))) 3611 continue; 3612 3613 /* Skip AES algorithms if not supported by device */ 3614 if (!aes_inst && (c1_alg_sel == OP_ALG_ALGSEL_AES)) 3615 continue; 3616 3617 /* Skip CHACHA20 algorithms if not supported by device */ 3618 if (c1_alg_sel == OP_ALG_ALGSEL_CHACHA20 && !ccha_inst) 3619 continue; 3620 3621 /* Skip POLY1305 algorithms if not supported by device */ 3622 if (c2_alg_sel == OP_ALG_ALGSEL_POLY1305 && !ptha_inst) 3623 continue; 3624 3625 /* Skip GCM algorithms if not supported by device */ 3626 if (c1_alg_sel == OP_ALG_ALGSEL_AES && 3627 alg_aai == OP_ALG_AAI_GCM && !gcm_support) 3628 continue; 3629 3630 /* 3631 * Skip algorithms requiring message digests 3632 * if MD or MD size is not supported by device. 3633 */ 3634 if (is_mdha(c2_alg_sel) && 3635 (!md_inst || t_alg->aead.maxauthsize > md_limit)) 3636 continue; 3637 3638 caam_aead_alg_init(t_alg); 3639 3640 err = crypto_register_aead(&t_alg->aead); 3641 if (err) { 3642 pr_warn("%s alg registration failed\n", 3643 t_alg->aead.base.cra_driver_name); 3644 continue; 3645 } 3646 3647 t_alg->registered = true; 3648 registered = true; 3649 } 3650 3651 if (registered) 3652 pr_info("caam algorithms registered in /proc/crypto\n"); 3653 3654 return err; 3655} 3656