1// SPDX-License-Identifier: GPL-2.0-only 2/* 3 * GCM: Galois/Counter Mode. 4 * 5 * Copyright (c) 2007 Nokia Siemens Networks - Mikko Herranen <mh1@iki.fi> 6 */ 7 8#include <crypto/gf128mul.h> 9#include <crypto/internal/aead.h> 10#include <crypto/internal/skcipher.h> 11#include <crypto/internal/hash.h> 12#include <crypto/null.h> 13#include <crypto/scatterwalk.h> 14#include <crypto/gcm.h> 15#include <crypto/hash.h> 16#include <linux/err.h> 17#include <linux/init.h> 18#include <linux/kernel.h> 19#include <linux/module.h> 20#include <linux/slab.h> 21 22struct gcm_instance_ctx { 23 struct crypto_skcipher_spawn ctr; 24 struct crypto_ahash_spawn ghash; 25}; 26 27struct crypto_gcm_ctx { 28 struct crypto_skcipher *ctr; 29 struct crypto_ahash *ghash; 30}; 31 32struct crypto_rfc4106_ctx { 33 struct crypto_aead *child; 34 u8 nonce[4]; 35}; 36 37struct crypto_rfc4106_req_ctx { 38 struct scatterlist src[3]; 39 struct scatterlist dst[3]; 40 struct aead_request subreq; 41}; 42 43struct crypto_rfc4543_instance_ctx { 44 struct crypto_aead_spawn aead; 45}; 46 47struct crypto_rfc4543_ctx { 48 struct crypto_aead *child; 49 struct crypto_sync_skcipher *null; 50 u8 nonce[4]; 51}; 52 53struct crypto_rfc4543_req_ctx { 54 struct aead_request subreq; 55}; 56 57struct crypto_gcm_ghash_ctx { 58 unsigned int cryptlen; 59 struct scatterlist *src; 60 int (*complete)(struct aead_request *req, u32 flags); 61}; 62 63struct crypto_gcm_req_priv_ctx { 64 u8 iv[16]; 65 u8 auth_tag[16]; 66 u8 iauth_tag[16]; 67 struct scatterlist src[3]; 68 struct scatterlist dst[3]; 69 struct scatterlist sg; 70 struct crypto_gcm_ghash_ctx ghash_ctx; 71 union { 72 struct ahash_request ahreq; 73 struct skcipher_request skreq; 74 } u; 75}; 76 77static struct { 78 u8 buf[16]; 79 struct scatterlist sg; 80} *gcm_zeroes; 81 82static int crypto_rfc4543_copy_src_to_dst(struct aead_request *req, bool enc); 83 84static inline struct crypto_gcm_req_priv_ctx *crypto_gcm_reqctx( 85 struct aead_request *req) 86{ 87 unsigned long align = crypto_aead_alignmask(crypto_aead_reqtfm(req)); 88 89 return (void *)PTR_ALIGN((u8 *)aead_request_ctx(req), align + 1); 90} 91 92static int crypto_gcm_setkey(struct crypto_aead *aead, const u8 *key, 93 unsigned int keylen) 94{ 95 struct crypto_gcm_ctx *ctx = crypto_aead_ctx(aead); 96 struct crypto_ahash *ghash = ctx->ghash; 97 struct crypto_skcipher *ctr = ctx->ctr; 98 struct { 99 be128 hash; 100 u8 iv[16]; 101 102 struct crypto_wait wait; 103 104 struct scatterlist sg[1]; 105 struct skcipher_request req; 106 } *data; 107 int err; 108 109 crypto_skcipher_clear_flags(ctr, CRYPTO_TFM_REQ_MASK); 110 crypto_skcipher_set_flags(ctr, crypto_aead_get_flags(aead) & 111 CRYPTO_TFM_REQ_MASK); 112 err = crypto_skcipher_setkey(ctr, key, keylen); 113 if (err) 114 return err; 115 116 data = kzalloc(sizeof(*data) + crypto_skcipher_reqsize(ctr), 117 GFP_KERNEL); 118 if (!data) 119 return -ENOMEM; 120 121 crypto_init_wait(&data->wait); 122 sg_init_one(data->sg, &data->hash, sizeof(data->hash)); 123 skcipher_request_set_tfm(&data->req, ctr); 124 skcipher_request_set_callback(&data->req, CRYPTO_TFM_REQ_MAY_SLEEP | 125 CRYPTO_TFM_REQ_MAY_BACKLOG, 126 crypto_req_done, 127 &data->wait); 128 skcipher_request_set_crypt(&data->req, data->sg, data->sg, 129 sizeof(data->hash), data->iv); 130 131 err = crypto_wait_req(crypto_skcipher_encrypt(&data->req), 132 &data->wait); 133 134 if (err) 135 goto out; 136 137 crypto_ahash_clear_flags(ghash, CRYPTO_TFM_REQ_MASK); 138 crypto_ahash_set_flags(ghash, crypto_aead_get_flags(aead) & 139 CRYPTO_TFM_REQ_MASK); 140 err = crypto_ahash_setkey(ghash, (u8 *)&data->hash, sizeof(be128)); 141out: 142 kfree_sensitive(data); 143 return err; 144} 145 146static int crypto_gcm_setauthsize(struct crypto_aead *tfm, 147 unsigned int authsize) 148{ 149 return crypto_gcm_check_authsize(authsize); 150} 151 152static void crypto_gcm_init_common(struct aead_request *req) 153{ 154 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); 155 __be32 counter = cpu_to_be32(1); 156 struct scatterlist *sg; 157 158 memset(pctx->auth_tag, 0, sizeof(pctx->auth_tag)); 159 memcpy(pctx->iv, req->iv, GCM_AES_IV_SIZE); 160 memcpy(pctx->iv + GCM_AES_IV_SIZE, &counter, 4); 161 162 sg_init_table(pctx->src, 3); 163 sg_set_buf(pctx->src, pctx->auth_tag, sizeof(pctx->auth_tag)); 164 sg = scatterwalk_ffwd(pctx->src + 1, req->src, req->assoclen); 165 if (sg != pctx->src + 1) 166 sg_chain(pctx->src, 2, sg); 167 168 if (req->src != req->dst) { 169 sg_init_table(pctx->dst, 3); 170 sg_set_buf(pctx->dst, pctx->auth_tag, sizeof(pctx->auth_tag)); 171 sg = scatterwalk_ffwd(pctx->dst + 1, req->dst, req->assoclen); 172 if (sg != pctx->dst + 1) 173 sg_chain(pctx->dst, 2, sg); 174 } 175} 176 177static void crypto_gcm_init_crypt(struct aead_request *req, 178 unsigned int cryptlen) 179{ 180 struct crypto_aead *aead = crypto_aead_reqtfm(req); 181 struct crypto_gcm_ctx *ctx = crypto_aead_ctx(aead); 182 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); 183 struct skcipher_request *skreq = &pctx->u.skreq; 184 struct scatterlist *dst; 185 186 dst = req->src == req->dst ? pctx->src : pctx->dst; 187 188 skcipher_request_set_tfm(skreq, ctx->ctr); 189 skcipher_request_set_crypt(skreq, pctx->src, dst, 190 cryptlen + sizeof(pctx->auth_tag), 191 pctx->iv); 192} 193 194static inline unsigned int gcm_remain(unsigned int len) 195{ 196 len &= 0xfU; 197 return len ? 16 - len : 0; 198} 199 200static void gcm_hash_len_done(void *data, int err); 201 202static int gcm_hash_update(struct aead_request *req, 203 crypto_completion_t compl, 204 struct scatterlist *src, 205 unsigned int len, u32 flags) 206{ 207 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); 208 struct ahash_request *ahreq = &pctx->u.ahreq; 209 210 ahash_request_set_callback(ahreq, flags, compl, req); 211 ahash_request_set_crypt(ahreq, src, NULL, len); 212 213 return crypto_ahash_update(ahreq); 214} 215 216static int gcm_hash_remain(struct aead_request *req, 217 unsigned int remain, 218 crypto_completion_t compl, u32 flags) 219{ 220 return gcm_hash_update(req, compl, &gcm_zeroes->sg, remain, flags); 221} 222 223static int gcm_hash_len(struct aead_request *req, u32 flags) 224{ 225 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); 226 struct ahash_request *ahreq = &pctx->u.ahreq; 227 struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx; 228 be128 lengths; 229 230 lengths.a = cpu_to_be64(req->assoclen * 8); 231 lengths.b = cpu_to_be64(gctx->cryptlen * 8); 232 memcpy(pctx->iauth_tag, &lengths, 16); 233 sg_init_one(&pctx->sg, pctx->iauth_tag, 16); 234 ahash_request_set_callback(ahreq, flags, gcm_hash_len_done, req); 235 ahash_request_set_crypt(ahreq, &pctx->sg, 236 pctx->iauth_tag, sizeof(lengths)); 237 238 return crypto_ahash_finup(ahreq); 239} 240 241static int gcm_hash_len_continue(struct aead_request *req, u32 flags) 242{ 243 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); 244 struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx; 245 246 return gctx->complete(req, flags); 247} 248 249static void gcm_hash_len_done(void *data, int err) 250{ 251 struct aead_request *req = data; 252 253 if (err) 254 goto out; 255 256 err = gcm_hash_len_continue(req, 0); 257 if (err == -EINPROGRESS) 258 return; 259 260out: 261 aead_request_complete(req, err); 262} 263 264static int gcm_hash_crypt_remain_continue(struct aead_request *req, u32 flags) 265{ 266 return gcm_hash_len(req, flags) ?: 267 gcm_hash_len_continue(req, flags); 268} 269 270static void gcm_hash_crypt_remain_done(void *data, int err) 271{ 272 struct aead_request *req = data; 273 274 if (err) 275 goto out; 276 277 err = gcm_hash_crypt_remain_continue(req, 0); 278 if (err == -EINPROGRESS) 279 return; 280 281out: 282 aead_request_complete(req, err); 283} 284 285static int gcm_hash_crypt_continue(struct aead_request *req, u32 flags) 286{ 287 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); 288 struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx; 289 unsigned int remain; 290 291 remain = gcm_remain(gctx->cryptlen); 292 if (remain) 293 return gcm_hash_remain(req, remain, 294 gcm_hash_crypt_remain_done, flags) ?: 295 gcm_hash_crypt_remain_continue(req, flags); 296 297 return gcm_hash_crypt_remain_continue(req, flags); 298} 299 300static void gcm_hash_crypt_done(void *data, int err) 301{ 302 struct aead_request *req = data; 303 304 if (err) 305 goto out; 306 307 err = gcm_hash_crypt_continue(req, 0); 308 if (err == -EINPROGRESS) 309 return; 310 311out: 312 aead_request_complete(req, err); 313} 314 315static int gcm_hash_assoc_remain_continue(struct aead_request *req, u32 flags) 316{ 317 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); 318 struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx; 319 320 if (gctx->cryptlen) 321 return gcm_hash_update(req, gcm_hash_crypt_done, 322 gctx->src, gctx->cryptlen, flags) ?: 323 gcm_hash_crypt_continue(req, flags); 324 325 return gcm_hash_crypt_remain_continue(req, flags); 326} 327 328static void gcm_hash_assoc_remain_done(void *data, int err) 329{ 330 struct aead_request *req = data; 331 332 if (err) 333 goto out; 334 335 err = gcm_hash_assoc_remain_continue(req, 0); 336 if (err == -EINPROGRESS) 337 return; 338 339out: 340 aead_request_complete(req, err); 341} 342 343static int gcm_hash_assoc_continue(struct aead_request *req, u32 flags) 344{ 345 unsigned int remain; 346 347 remain = gcm_remain(req->assoclen); 348 if (remain) 349 return gcm_hash_remain(req, remain, 350 gcm_hash_assoc_remain_done, flags) ?: 351 gcm_hash_assoc_remain_continue(req, flags); 352 353 return gcm_hash_assoc_remain_continue(req, flags); 354} 355 356static void gcm_hash_assoc_done(void *data, int err) 357{ 358 struct aead_request *req = data; 359 360 if (err) 361 goto out; 362 363 err = gcm_hash_assoc_continue(req, 0); 364 if (err == -EINPROGRESS) 365 return; 366 367out: 368 aead_request_complete(req, err); 369} 370 371static int gcm_hash_init_continue(struct aead_request *req, u32 flags) 372{ 373 if (req->assoclen) 374 return gcm_hash_update(req, gcm_hash_assoc_done, 375 req->src, req->assoclen, flags) ?: 376 gcm_hash_assoc_continue(req, flags); 377 378 return gcm_hash_assoc_remain_continue(req, flags); 379} 380 381static void gcm_hash_init_done(void *data, int err) 382{ 383 struct aead_request *req = data; 384 385 if (err) 386 goto out; 387 388 err = gcm_hash_init_continue(req, 0); 389 if (err == -EINPROGRESS) 390 return; 391 392out: 393 aead_request_complete(req, err); 394} 395 396static int gcm_hash(struct aead_request *req, u32 flags) 397{ 398 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); 399 struct ahash_request *ahreq = &pctx->u.ahreq; 400 struct crypto_gcm_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req)); 401 402 ahash_request_set_tfm(ahreq, ctx->ghash); 403 404 ahash_request_set_callback(ahreq, flags, gcm_hash_init_done, req); 405 return crypto_ahash_init(ahreq) ?: 406 gcm_hash_init_continue(req, flags); 407} 408 409static int gcm_enc_copy_hash(struct aead_request *req, u32 flags) 410{ 411 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); 412 struct crypto_aead *aead = crypto_aead_reqtfm(req); 413 u8 *auth_tag = pctx->auth_tag; 414 415 crypto_xor(auth_tag, pctx->iauth_tag, 16); 416 scatterwalk_map_and_copy(auth_tag, req->dst, 417 req->assoclen + req->cryptlen, 418 crypto_aead_authsize(aead), 1); 419 return 0; 420} 421 422static int gcm_encrypt_continue(struct aead_request *req, u32 flags) 423{ 424 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); 425 struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx; 426 427 gctx->src = sg_next(req->src == req->dst ? pctx->src : pctx->dst); 428 gctx->cryptlen = req->cryptlen; 429 gctx->complete = gcm_enc_copy_hash; 430 431 return gcm_hash(req, flags); 432} 433 434static void gcm_encrypt_done(void *data, int err) 435{ 436 struct aead_request *req = data; 437 438 if (err) 439 goto out; 440 441 err = gcm_encrypt_continue(req, 0); 442 if (err == -EINPROGRESS) 443 return; 444 445out: 446 aead_request_complete(req, err); 447} 448 449static int crypto_gcm_encrypt(struct aead_request *req) 450{ 451 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); 452 struct skcipher_request *skreq = &pctx->u.skreq; 453 u32 flags = aead_request_flags(req); 454 455 crypto_gcm_init_common(req); 456 crypto_gcm_init_crypt(req, req->cryptlen); 457 skcipher_request_set_callback(skreq, flags, gcm_encrypt_done, req); 458 459 return crypto_skcipher_encrypt(skreq) ?: 460 gcm_encrypt_continue(req, flags); 461} 462 463static int crypto_gcm_verify(struct aead_request *req) 464{ 465 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); 466 struct crypto_aead *aead = crypto_aead_reqtfm(req); 467 u8 *auth_tag = pctx->auth_tag; 468 u8 *iauth_tag = pctx->iauth_tag; 469 unsigned int authsize = crypto_aead_authsize(aead); 470 unsigned int cryptlen = req->cryptlen - authsize; 471 472 crypto_xor(auth_tag, iauth_tag, 16); 473 scatterwalk_map_and_copy(iauth_tag, req->src, 474 req->assoclen + cryptlen, authsize, 0); 475 return crypto_memneq(iauth_tag, auth_tag, authsize) ? -EBADMSG : 0; 476} 477 478static void gcm_decrypt_done(void *data, int err) 479{ 480 struct aead_request *req = data; 481 482 if (!err) 483 err = crypto_gcm_verify(req); 484 485 aead_request_complete(req, err); 486} 487 488static int gcm_dec_hash_continue(struct aead_request *req, u32 flags) 489{ 490 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); 491 struct skcipher_request *skreq = &pctx->u.skreq; 492 struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx; 493 494 crypto_gcm_init_crypt(req, gctx->cryptlen); 495 skcipher_request_set_callback(skreq, flags, gcm_decrypt_done, req); 496 return crypto_skcipher_decrypt(skreq) ?: crypto_gcm_verify(req); 497} 498 499static int crypto_gcm_decrypt(struct aead_request *req) 500{ 501 struct crypto_aead *aead = crypto_aead_reqtfm(req); 502 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); 503 struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx; 504 unsigned int authsize = crypto_aead_authsize(aead); 505 unsigned int cryptlen = req->cryptlen; 506 u32 flags = aead_request_flags(req); 507 508 cryptlen -= authsize; 509 510 crypto_gcm_init_common(req); 511 512 gctx->src = sg_next(pctx->src); 513 gctx->cryptlen = cryptlen; 514 gctx->complete = gcm_dec_hash_continue; 515 516 return gcm_hash(req, flags); 517} 518 519static int crypto_gcm_init_tfm(struct crypto_aead *tfm) 520{ 521 struct aead_instance *inst = aead_alg_instance(tfm); 522 struct gcm_instance_ctx *ictx = aead_instance_ctx(inst); 523 struct crypto_gcm_ctx *ctx = crypto_aead_ctx(tfm); 524 struct crypto_skcipher *ctr; 525 struct crypto_ahash *ghash; 526 unsigned long align; 527 int err; 528 529 ghash = crypto_spawn_ahash(&ictx->ghash); 530 if (IS_ERR(ghash)) 531 return PTR_ERR(ghash); 532 533 ctr = crypto_spawn_skcipher(&ictx->ctr); 534 err = PTR_ERR(ctr); 535 if (IS_ERR(ctr)) 536 goto err_free_hash; 537 538 ctx->ctr = ctr; 539 ctx->ghash = ghash; 540 541 align = crypto_aead_alignmask(tfm); 542 align &= ~(crypto_tfm_ctx_alignment() - 1); 543 crypto_aead_set_reqsize(tfm, 544 align + offsetof(struct crypto_gcm_req_priv_ctx, u) + 545 max(sizeof(struct skcipher_request) + 546 crypto_skcipher_reqsize(ctr), 547 sizeof(struct ahash_request) + 548 crypto_ahash_reqsize(ghash))); 549 550 return 0; 551 552err_free_hash: 553 crypto_free_ahash(ghash); 554 return err; 555} 556 557static void crypto_gcm_exit_tfm(struct crypto_aead *tfm) 558{ 559 struct crypto_gcm_ctx *ctx = crypto_aead_ctx(tfm); 560 561 crypto_free_ahash(ctx->ghash); 562 crypto_free_skcipher(ctx->ctr); 563} 564 565static void crypto_gcm_free(struct aead_instance *inst) 566{ 567 struct gcm_instance_ctx *ctx = aead_instance_ctx(inst); 568 569 crypto_drop_skcipher(&ctx->ctr); 570 crypto_drop_ahash(&ctx->ghash); 571 kfree(inst); 572} 573 574static int crypto_gcm_create_common(struct crypto_template *tmpl, 575 struct rtattr **tb, 576 const char *ctr_name, 577 const char *ghash_name) 578{ 579 u32 mask; 580 struct aead_instance *inst; 581 struct gcm_instance_ctx *ctx; 582 struct skcipher_alg *ctr; 583 struct hash_alg_common *ghash; 584 int err; 585 586 err = crypto_check_attr_type(tb, CRYPTO_ALG_TYPE_AEAD, &mask); 587 if (err) 588 return err; 589 590 inst = kzalloc(sizeof(*inst) + sizeof(*ctx), GFP_KERNEL); 591 if (!inst) 592 return -ENOMEM; 593 ctx = aead_instance_ctx(inst); 594 595 err = crypto_grab_ahash(&ctx->ghash, aead_crypto_instance(inst), 596 ghash_name, 0, mask); 597 if (err) 598 goto err_free_inst; 599 ghash = crypto_spawn_ahash_alg(&ctx->ghash); 600 601 err = -EINVAL; 602 if (strcmp(ghash->base.cra_name, "ghash") != 0 || 603 ghash->digestsize != 16) 604 goto err_free_inst; 605 606 err = crypto_grab_skcipher(&ctx->ctr, aead_crypto_instance(inst), 607 ctr_name, 0, mask); 608 if (err) 609 goto err_free_inst; 610 ctr = crypto_spawn_skcipher_alg(&ctx->ctr); 611 612 /* The skcipher algorithm must be CTR mode, using 16-byte blocks. */ 613 err = -EINVAL; 614 if (strncmp(ctr->base.cra_name, "ctr(", 4) != 0 || 615 crypto_skcipher_alg_ivsize(ctr) != 16 || 616 ctr->base.cra_blocksize != 1) 617 goto err_free_inst; 618 619 err = -ENAMETOOLONG; 620 if (snprintf(inst->alg.base.cra_name, CRYPTO_MAX_ALG_NAME, 621 "gcm(%s", ctr->base.cra_name + 4) >= CRYPTO_MAX_ALG_NAME) 622 goto err_free_inst; 623 624 if (snprintf(inst->alg.base.cra_driver_name, CRYPTO_MAX_ALG_NAME, 625 "gcm_base(%s,%s)", ctr->base.cra_driver_name, 626 ghash->base.cra_driver_name) >= 627 CRYPTO_MAX_ALG_NAME) 628 goto err_free_inst; 629 630 inst->alg.base.cra_priority = (ghash->base.cra_priority + 631 ctr->base.cra_priority) / 2; 632 inst->alg.base.cra_blocksize = 1; 633 inst->alg.base.cra_alignmask = ghash->base.cra_alignmask | 634 ctr->base.cra_alignmask; 635 inst->alg.base.cra_ctxsize = sizeof(struct crypto_gcm_ctx); 636 inst->alg.ivsize = GCM_AES_IV_SIZE; 637 inst->alg.chunksize = crypto_skcipher_alg_chunksize(ctr); 638 inst->alg.maxauthsize = 16; 639 inst->alg.init = crypto_gcm_init_tfm; 640 inst->alg.exit = crypto_gcm_exit_tfm; 641 inst->alg.setkey = crypto_gcm_setkey; 642 inst->alg.setauthsize = crypto_gcm_setauthsize; 643 inst->alg.encrypt = crypto_gcm_encrypt; 644 inst->alg.decrypt = crypto_gcm_decrypt; 645 646 inst->free = crypto_gcm_free; 647 648 err = aead_register_instance(tmpl, inst); 649 if (err) { 650err_free_inst: 651 crypto_gcm_free(inst); 652 } 653 return err; 654} 655 656static int crypto_gcm_create(struct crypto_template *tmpl, struct rtattr **tb) 657{ 658 const char *cipher_name; 659 char ctr_name[CRYPTO_MAX_ALG_NAME]; 660 661 cipher_name = crypto_attr_alg_name(tb[1]); 662 if (IS_ERR(cipher_name)) 663 return PTR_ERR(cipher_name); 664 665 if (snprintf(ctr_name, CRYPTO_MAX_ALG_NAME, "ctr(%s)", cipher_name) >= 666 CRYPTO_MAX_ALG_NAME) 667 return -ENAMETOOLONG; 668 669 return crypto_gcm_create_common(tmpl, tb, ctr_name, "ghash"); 670} 671 672static int crypto_gcm_base_create(struct crypto_template *tmpl, 673 struct rtattr **tb) 674{ 675 const char *ctr_name; 676 const char *ghash_name; 677 678 ctr_name = crypto_attr_alg_name(tb[1]); 679 if (IS_ERR(ctr_name)) 680 return PTR_ERR(ctr_name); 681 682 ghash_name = crypto_attr_alg_name(tb[2]); 683 if (IS_ERR(ghash_name)) 684 return PTR_ERR(ghash_name); 685 686 return crypto_gcm_create_common(tmpl, tb, ctr_name, ghash_name); 687} 688 689static int crypto_rfc4106_setkey(struct crypto_aead *parent, const u8 *key, 690 unsigned int keylen) 691{ 692 struct crypto_rfc4106_ctx *ctx = crypto_aead_ctx(parent); 693 struct crypto_aead *child = ctx->child; 694 695 if (keylen < 4) 696 return -EINVAL; 697 698 keylen -= 4; 699 memcpy(ctx->nonce, key + keylen, 4); 700 701 crypto_aead_clear_flags(child, CRYPTO_TFM_REQ_MASK); 702 crypto_aead_set_flags(child, crypto_aead_get_flags(parent) & 703 CRYPTO_TFM_REQ_MASK); 704 return crypto_aead_setkey(child, key, keylen); 705} 706 707static int crypto_rfc4106_setauthsize(struct crypto_aead *parent, 708 unsigned int authsize) 709{ 710 struct crypto_rfc4106_ctx *ctx = crypto_aead_ctx(parent); 711 int err; 712 713 err = crypto_rfc4106_check_authsize(authsize); 714 if (err) 715 return err; 716 717 return crypto_aead_setauthsize(ctx->child, authsize); 718} 719 720static struct aead_request *crypto_rfc4106_crypt(struct aead_request *req) 721{ 722 struct crypto_rfc4106_req_ctx *rctx = aead_request_ctx(req); 723 struct crypto_aead *aead = crypto_aead_reqtfm(req); 724 struct crypto_rfc4106_ctx *ctx = crypto_aead_ctx(aead); 725 struct aead_request *subreq = &rctx->subreq; 726 struct crypto_aead *child = ctx->child; 727 struct scatterlist *sg; 728 u8 *iv = PTR_ALIGN((u8 *)(subreq + 1) + crypto_aead_reqsize(child), 729 crypto_aead_alignmask(child) + 1); 730 731 scatterwalk_map_and_copy(iv + GCM_AES_IV_SIZE, req->src, 0, req->assoclen - 8, 0); 732 733 memcpy(iv, ctx->nonce, 4); 734 memcpy(iv + 4, req->iv, 8); 735 736 sg_init_table(rctx->src, 3); 737 sg_set_buf(rctx->src, iv + GCM_AES_IV_SIZE, req->assoclen - 8); 738 sg = scatterwalk_ffwd(rctx->src + 1, req->src, req->assoclen); 739 if (sg != rctx->src + 1) 740 sg_chain(rctx->src, 2, sg); 741 742 if (req->src != req->dst) { 743 sg_init_table(rctx->dst, 3); 744 sg_set_buf(rctx->dst, iv + GCM_AES_IV_SIZE, req->assoclen - 8); 745 sg = scatterwalk_ffwd(rctx->dst + 1, req->dst, req->assoclen); 746 if (sg != rctx->dst + 1) 747 sg_chain(rctx->dst, 2, sg); 748 } 749 750 aead_request_set_tfm(subreq, child); 751 aead_request_set_callback(subreq, req->base.flags, req->base.complete, 752 req->base.data); 753 aead_request_set_crypt(subreq, rctx->src, 754 req->src == req->dst ? rctx->src : rctx->dst, 755 req->cryptlen, iv); 756 aead_request_set_ad(subreq, req->assoclen - 8); 757 758 return subreq; 759} 760 761static int crypto_rfc4106_encrypt(struct aead_request *req) 762{ 763 int err; 764 765 err = crypto_ipsec_check_assoclen(req->assoclen); 766 if (err) 767 return err; 768 769 req = crypto_rfc4106_crypt(req); 770 771 return crypto_aead_encrypt(req); 772} 773 774static int crypto_rfc4106_decrypt(struct aead_request *req) 775{ 776 int err; 777 778 err = crypto_ipsec_check_assoclen(req->assoclen); 779 if (err) 780 return err; 781 782 req = crypto_rfc4106_crypt(req); 783 784 return crypto_aead_decrypt(req); 785} 786 787static int crypto_rfc4106_init_tfm(struct crypto_aead *tfm) 788{ 789 struct aead_instance *inst = aead_alg_instance(tfm); 790 struct crypto_aead_spawn *spawn = aead_instance_ctx(inst); 791 struct crypto_rfc4106_ctx *ctx = crypto_aead_ctx(tfm); 792 struct crypto_aead *aead; 793 unsigned long align; 794 795 aead = crypto_spawn_aead(spawn); 796 if (IS_ERR(aead)) 797 return PTR_ERR(aead); 798 799 ctx->child = aead; 800 801 align = crypto_aead_alignmask(aead); 802 align &= ~(crypto_tfm_ctx_alignment() - 1); 803 crypto_aead_set_reqsize( 804 tfm, 805 sizeof(struct crypto_rfc4106_req_ctx) + 806 ALIGN(crypto_aead_reqsize(aead), crypto_tfm_ctx_alignment()) + 807 align + 24); 808 809 return 0; 810} 811 812static void crypto_rfc4106_exit_tfm(struct crypto_aead *tfm) 813{ 814 struct crypto_rfc4106_ctx *ctx = crypto_aead_ctx(tfm); 815 816 crypto_free_aead(ctx->child); 817} 818 819static void crypto_rfc4106_free(struct aead_instance *inst) 820{ 821 crypto_drop_aead(aead_instance_ctx(inst)); 822 kfree(inst); 823} 824 825static int crypto_rfc4106_create(struct crypto_template *tmpl, 826 struct rtattr **tb) 827{ 828 u32 mask; 829 struct aead_instance *inst; 830 struct crypto_aead_spawn *spawn; 831 struct aead_alg *alg; 832 int err; 833 834 err = crypto_check_attr_type(tb, CRYPTO_ALG_TYPE_AEAD, &mask); 835 if (err) 836 return err; 837 838 inst = kzalloc(sizeof(*inst) + sizeof(*spawn), GFP_KERNEL); 839 if (!inst) 840 return -ENOMEM; 841 842 spawn = aead_instance_ctx(inst); 843 err = crypto_grab_aead(spawn, aead_crypto_instance(inst), 844 crypto_attr_alg_name(tb[1]), 0, mask); 845 if (err) 846 goto err_free_inst; 847 848 alg = crypto_spawn_aead_alg(spawn); 849 850 err = -EINVAL; 851 852 /* Underlying IV size must be 12. */ 853 if (crypto_aead_alg_ivsize(alg) != GCM_AES_IV_SIZE) 854 goto err_free_inst; 855 856 /* Not a stream cipher? */ 857 if (alg->base.cra_blocksize != 1) 858 goto err_free_inst; 859 860 err = -ENAMETOOLONG; 861 if (snprintf(inst->alg.base.cra_name, CRYPTO_MAX_ALG_NAME, 862 "rfc4106(%s)", alg->base.cra_name) >= 863 CRYPTO_MAX_ALG_NAME || 864 snprintf(inst->alg.base.cra_driver_name, CRYPTO_MAX_ALG_NAME, 865 "rfc4106(%s)", alg->base.cra_driver_name) >= 866 CRYPTO_MAX_ALG_NAME) 867 goto err_free_inst; 868 869 inst->alg.base.cra_priority = alg->base.cra_priority; 870 inst->alg.base.cra_blocksize = 1; 871 inst->alg.base.cra_alignmask = alg->base.cra_alignmask; 872 873 inst->alg.base.cra_ctxsize = sizeof(struct crypto_rfc4106_ctx); 874 875 inst->alg.ivsize = GCM_RFC4106_IV_SIZE; 876 inst->alg.chunksize = crypto_aead_alg_chunksize(alg); 877 inst->alg.maxauthsize = crypto_aead_alg_maxauthsize(alg); 878 879 inst->alg.init = crypto_rfc4106_init_tfm; 880 inst->alg.exit = crypto_rfc4106_exit_tfm; 881 882 inst->alg.setkey = crypto_rfc4106_setkey; 883 inst->alg.setauthsize = crypto_rfc4106_setauthsize; 884 inst->alg.encrypt = crypto_rfc4106_encrypt; 885 inst->alg.decrypt = crypto_rfc4106_decrypt; 886 887 inst->free = crypto_rfc4106_free; 888 889 err = aead_register_instance(tmpl, inst); 890 if (err) { 891err_free_inst: 892 crypto_rfc4106_free(inst); 893 } 894 return err; 895} 896 897static int crypto_rfc4543_setkey(struct crypto_aead *parent, const u8 *key, 898 unsigned int keylen) 899{ 900 struct crypto_rfc4543_ctx *ctx = crypto_aead_ctx(parent); 901 struct crypto_aead *child = ctx->child; 902 903 if (keylen < 4) 904 return -EINVAL; 905 906 keylen -= 4; 907 memcpy(ctx->nonce, key + keylen, 4); 908 909 crypto_aead_clear_flags(child, CRYPTO_TFM_REQ_MASK); 910 crypto_aead_set_flags(child, crypto_aead_get_flags(parent) & 911 CRYPTO_TFM_REQ_MASK); 912 return crypto_aead_setkey(child, key, keylen); 913} 914 915static int crypto_rfc4543_setauthsize(struct crypto_aead *parent, 916 unsigned int authsize) 917{ 918 struct crypto_rfc4543_ctx *ctx = crypto_aead_ctx(parent); 919 920 if (authsize != 16) 921 return -EINVAL; 922 923 return crypto_aead_setauthsize(ctx->child, authsize); 924} 925 926static int crypto_rfc4543_crypt(struct aead_request *req, bool enc) 927{ 928 struct crypto_aead *aead = crypto_aead_reqtfm(req); 929 struct crypto_rfc4543_ctx *ctx = crypto_aead_ctx(aead); 930 struct crypto_rfc4543_req_ctx *rctx = aead_request_ctx(req); 931 struct aead_request *subreq = &rctx->subreq; 932 unsigned int authsize = crypto_aead_authsize(aead); 933 u8 *iv = PTR_ALIGN((u8 *)(rctx + 1) + crypto_aead_reqsize(ctx->child), 934 crypto_aead_alignmask(ctx->child) + 1); 935 int err; 936 937 if (req->src != req->dst) { 938 err = crypto_rfc4543_copy_src_to_dst(req, enc); 939 if (err) 940 return err; 941 } 942 943 memcpy(iv, ctx->nonce, 4); 944 memcpy(iv + 4, req->iv, 8); 945 946 aead_request_set_tfm(subreq, ctx->child); 947 aead_request_set_callback(subreq, req->base.flags, 948 req->base.complete, req->base.data); 949 aead_request_set_crypt(subreq, req->src, req->dst, 950 enc ? 0 : authsize, iv); 951 aead_request_set_ad(subreq, req->assoclen + req->cryptlen - 952 subreq->cryptlen); 953 954 return enc ? crypto_aead_encrypt(subreq) : crypto_aead_decrypt(subreq); 955} 956 957static int crypto_rfc4543_copy_src_to_dst(struct aead_request *req, bool enc) 958{ 959 struct crypto_aead *aead = crypto_aead_reqtfm(req); 960 struct crypto_rfc4543_ctx *ctx = crypto_aead_ctx(aead); 961 unsigned int authsize = crypto_aead_authsize(aead); 962 unsigned int nbytes = req->assoclen + req->cryptlen - 963 (enc ? 0 : authsize); 964 SYNC_SKCIPHER_REQUEST_ON_STACK(nreq, ctx->null); 965 966 skcipher_request_set_sync_tfm(nreq, ctx->null); 967 skcipher_request_set_callback(nreq, req->base.flags, NULL, NULL); 968 skcipher_request_set_crypt(nreq, req->src, req->dst, nbytes, NULL); 969 970 return crypto_skcipher_encrypt(nreq); 971} 972 973static int crypto_rfc4543_encrypt(struct aead_request *req) 974{ 975 return crypto_ipsec_check_assoclen(req->assoclen) ?: 976 crypto_rfc4543_crypt(req, true); 977} 978 979static int crypto_rfc4543_decrypt(struct aead_request *req) 980{ 981 return crypto_ipsec_check_assoclen(req->assoclen) ?: 982 crypto_rfc4543_crypt(req, false); 983} 984 985static int crypto_rfc4543_init_tfm(struct crypto_aead *tfm) 986{ 987 struct aead_instance *inst = aead_alg_instance(tfm); 988 struct crypto_rfc4543_instance_ctx *ictx = aead_instance_ctx(inst); 989 struct crypto_aead_spawn *spawn = &ictx->aead; 990 struct crypto_rfc4543_ctx *ctx = crypto_aead_ctx(tfm); 991 struct crypto_aead *aead; 992 struct crypto_sync_skcipher *null; 993 unsigned long align; 994 int err = 0; 995 996 aead = crypto_spawn_aead(spawn); 997 if (IS_ERR(aead)) 998 return PTR_ERR(aead); 999 1000 null = crypto_get_default_null_skcipher(); 1001 err = PTR_ERR(null); 1002 if (IS_ERR(null)) 1003 goto err_free_aead; 1004 1005 ctx->child = aead; 1006 ctx->null = null; 1007 1008 align = crypto_aead_alignmask(aead); 1009 align &= ~(crypto_tfm_ctx_alignment() - 1); 1010 crypto_aead_set_reqsize( 1011 tfm, 1012 sizeof(struct crypto_rfc4543_req_ctx) + 1013 ALIGN(crypto_aead_reqsize(aead), crypto_tfm_ctx_alignment()) + 1014 align + GCM_AES_IV_SIZE); 1015 1016 return 0; 1017 1018err_free_aead: 1019 crypto_free_aead(aead); 1020 return err; 1021} 1022 1023static void crypto_rfc4543_exit_tfm(struct crypto_aead *tfm) 1024{ 1025 struct crypto_rfc4543_ctx *ctx = crypto_aead_ctx(tfm); 1026 1027 crypto_free_aead(ctx->child); 1028 crypto_put_default_null_skcipher(); 1029} 1030 1031static void crypto_rfc4543_free(struct aead_instance *inst) 1032{ 1033 struct crypto_rfc4543_instance_ctx *ctx = aead_instance_ctx(inst); 1034 1035 crypto_drop_aead(&ctx->aead); 1036 1037 kfree(inst); 1038} 1039 1040static int crypto_rfc4543_create(struct crypto_template *tmpl, 1041 struct rtattr **tb) 1042{ 1043 u32 mask; 1044 struct aead_instance *inst; 1045 struct aead_alg *alg; 1046 struct crypto_rfc4543_instance_ctx *ctx; 1047 int err; 1048 1049 err = crypto_check_attr_type(tb, CRYPTO_ALG_TYPE_AEAD, &mask); 1050 if (err) 1051 return err; 1052 1053 inst = kzalloc(sizeof(*inst) + sizeof(*ctx), GFP_KERNEL); 1054 if (!inst) 1055 return -ENOMEM; 1056 1057 ctx = aead_instance_ctx(inst); 1058 err = crypto_grab_aead(&ctx->aead, aead_crypto_instance(inst), 1059 crypto_attr_alg_name(tb[1]), 0, mask); 1060 if (err) 1061 goto err_free_inst; 1062 1063 alg = crypto_spawn_aead_alg(&ctx->aead); 1064 1065 err = -EINVAL; 1066 1067 /* Underlying IV size must be 12. */ 1068 if (crypto_aead_alg_ivsize(alg) != GCM_AES_IV_SIZE) 1069 goto err_free_inst; 1070 1071 /* Not a stream cipher? */ 1072 if (alg->base.cra_blocksize != 1) 1073 goto err_free_inst; 1074 1075 err = -ENAMETOOLONG; 1076 if (snprintf(inst->alg.base.cra_name, CRYPTO_MAX_ALG_NAME, 1077 "rfc4543(%s)", alg->base.cra_name) >= 1078 CRYPTO_MAX_ALG_NAME || 1079 snprintf(inst->alg.base.cra_driver_name, CRYPTO_MAX_ALG_NAME, 1080 "rfc4543(%s)", alg->base.cra_driver_name) >= 1081 CRYPTO_MAX_ALG_NAME) 1082 goto err_free_inst; 1083 1084 inst->alg.base.cra_priority = alg->base.cra_priority; 1085 inst->alg.base.cra_blocksize = 1; 1086 inst->alg.base.cra_alignmask = alg->base.cra_alignmask; 1087 1088 inst->alg.base.cra_ctxsize = sizeof(struct crypto_rfc4543_ctx); 1089 1090 inst->alg.ivsize = GCM_RFC4543_IV_SIZE; 1091 inst->alg.chunksize = crypto_aead_alg_chunksize(alg); 1092 inst->alg.maxauthsize = crypto_aead_alg_maxauthsize(alg); 1093 1094 inst->alg.init = crypto_rfc4543_init_tfm; 1095 inst->alg.exit = crypto_rfc4543_exit_tfm; 1096 1097 inst->alg.setkey = crypto_rfc4543_setkey; 1098 inst->alg.setauthsize = crypto_rfc4543_setauthsize; 1099 inst->alg.encrypt = crypto_rfc4543_encrypt; 1100 inst->alg.decrypt = crypto_rfc4543_decrypt; 1101 1102 inst->free = crypto_rfc4543_free; 1103 1104 err = aead_register_instance(tmpl, inst); 1105 if (err) { 1106err_free_inst: 1107 crypto_rfc4543_free(inst); 1108 } 1109 return err; 1110} 1111 1112static struct crypto_template crypto_gcm_tmpls[] = { 1113 { 1114 .name = "gcm_base", 1115 .create = crypto_gcm_base_create, 1116 .module = THIS_MODULE, 1117 }, { 1118 .name = "gcm", 1119 .create = crypto_gcm_create, 1120 .module = THIS_MODULE, 1121 }, { 1122 .name = "rfc4106", 1123 .create = crypto_rfc4106_create, 1124 .module = THIS_MODULE, 1125 }, { 1126 .name = "rfc4543", 1127 .create = crypto_rfc4543_create, 1128 .module = THIS_MODULE, 1129 }, 1130}; 1131 1132static int __init crypto_gcm_module_init(void) 1133{ 1134 int err; 1135 1136 gcm_zeroes = kzalloc(sizeof(*gcm_zeroes), GFP_KERNEL); 1137 if (!gcm_zeroes) 1138 return -ENOMEM; 1139 1140 sg_init_one(&gcm_zeroes->sg, gcm_zeroes->buf, sizeof(gcm_zeroes->buf)); 1141 1142 err = crypto_register_templates(crypto_gcm_tmpls, 1143 ARRAY_SIZE(crypto_gcm_tmpls)); 1144 if (err) 1145 kfree(gcm_zeroes); 1146 1147 return err; 1148} 1149 1150static void __exit crypto_gcm_module_exit(void) 1151{ 1152 kfree(gcm_zeroes); 1153 crypto_unregister_templates(crypto_gcm_tmpls, 1154 ARRAY_SIZE(crypto_gcm_tmpls)); 1155} 1156 1157subsys_initcall(crypto_gcm_module_init); 1158module_exit(crypto_gcm_module_exit); 1159 1160MODULE_LICENSE("GPL"); 1161MODULE_DESCRIPTION("Galois/Counter Mode"); 1162MODULE_AUTHOR("Mikko Herranen <mh1@iki.fi>"); 1163MODULE_ALIAS_CRYPTO("gcm_base"); 1164MODULE_ALIAS_CRYPTO("rfc4106"); 1165MODULE_ALIAS_CRYPTO("rfc4543"); 1166MODULE_ALIAS_CRYPTO("gcm"); 1167