1// SPDX-License-Identifier: GPL-2.0 2/* 3 * Cryptographic API. 4 * 5 * s390 implementation of the AES Cipher Algorithm with protected keys. 6 * 7 * s390 Version: 8 * Copyright IBM Corp. 2017,2020 9 * Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com> 10 * Harald Freudenberger <freude@de.ibm.com> 11 */ 12 13#define KMSG_COMPONENT "paes_s390" 14#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt 15 16#include <crypto/aes.h> 17#include <crypto/algapi.h> 18#include <linux/bug.h> 19#include <linux/err.h> 20#include <linux/module.h> 21#include <linux/cpufeature.h> 22#include <linux/init.h> 23#include <linux/mutex.h> 24#include <linux/spinlock.h> 25#include <crypto/internal/skcipher.h> 26#include <crypto/xts.h> 27#include <asm/cpacf.h> 28#include <asm/pkey.h> 29 30/* 31 * Key blobs smaller/bigger than these defines are rejected 32 * by the common code even before the individual setkey function 33 * is called. As paes can handle different kinds of key blobs 34 * and padding is also possible, the limits need to be generous. 35 */ 36#define PAES_MIN_KEYSIZE 16 37#define PAES_MAX_KEYSIZE MAXEP11AESKEYBLOBSIZE 38 39static u8 *ctrblk; 40static DEFINE_MUTEX(ctrblk_lock); 41 42static cpacf_mask_t km_functions, kmc_functions, kmctr_functions; 43 44struct key_blob { 45 /* 46 * Small keys will be stored in the keybuf. Larger keys are 47 * stored in extra allocated memory. In both cases does 48 * key point to the memory where the key is stored. 49 * The code distinguishes by checking keylen against 50 * sizeof(keybuf). See the two following helper functions. 51 */ 52 u8 *key; 53 u8 keybuf[128]; 54 unsigned int keylen; 55}; 56 57static inline int _key_to_kb(struct key_blob *kb, 58 const u8 *key, 59 unsigned int keylen) 60{ 61 struct clearkey_header { 62 u8 type; 63 u8 res0[3]; 64 u8 version; 65 u8 res1[3]; 66 u32 keytype; 67 u32 len; 68 } __packed * h; 69 70 switch (keylen) { 71 case 16: 72 case 24: 73 case 32: 74 /* clear key value, prepare pkey clear key token in keybuf */ 75 memset(kb->keybuf, 0, sizeof(kb->keybuf)); 76 h = (struct clearkey_header *) kb->keybuf; 77 h->version = 0x02; /* TOKVER_CLEAR_KEY */ 78 h->keytype = (keylen - 8) >> 3; 79 h->len = keylen; 80 memcpy(kb->keybuf + sizeof(*h), key, keylen); 81 kb->keylen = sizeof(*h) + keylen; 82 kb->key = kb->keybuf; 83 break; 84 default: 85 /* other key material, let pkey handle this */ 86 if (keylen <= sizeof(kb->keybuf)) 87 kb->key = kb->keybuf; 88 else { 89 kb->key = kmalloc(keylen, GFP_KERNEL); 90 if (!kb->key) 91 return -ENOMEM; 92 } 93 memcpy(kb->key, key, keylen); 94 kb->keylen = keylen; 95 break; 96 } 97 98 return 0; 99} 100 101static inline void _free_kb_keybuf(struct key_blob *kb) 102{ 103 if (kb->key && kb->key != kb->keybuf 104 && kb->keylen > sizeof(kb->keybuf)) { 105 kfree(kb->key); 106 kb->key = NULL; 107 } 108} 109 110struct s390_paes_ctx { 111 struct key_blob kb; 112 struct pkey_protkey pk; 113 spinlock_t pk_lock; 114 unsigned long fc; 115}; 116 117struct s390_pxts_ctx { 118 struct key_blob kb[2]; 119 struct pkey_protkey pk[2]; 120 spinlock_t pk_lock; 121 unsigned long fc; 122}; 123 124static inline int __paes_keyblob2pkey(struct key_blob *kb, 125 struct pkey_protkey *pk) 126{ 127 int i, ret; 128 129 /* try three times in case of failure */ 130 for (i = 0; i < 3; i++) { 131 ret = pkey_keyblob2pkey(kb->key, kb->keylen, pk); 132 if (ret == 0) 133 break; 134 } 135 136 return ret; 137} 138 139static inline int __paes_convert_key(struct s390_paes_ctx *ctx) 140{ 141 struct pkey_protkey pkey; 142 143 if (__paes_keyblob2pkey(&ctx->kb, &pkey)) 144 return -EINVAL; 145 146 spin_lock_bh(&ctx->pk_lock); 147 memcpy(&ctx->pk, &pkey, sizeof(pkey)); 148 spin_unlock_bh(&ctx->pk_lock); 149 150 return 0; 151} 152 153static int ecb_paes_init(struct crypto_skcipher *tfm) 154{ 155 struct s390_paes_ctx *ctx = crypto_skcipher_ctx(tfm); 156 157 ctx->kb.key = NULL; 158 spin_lock_init(&ctx->pk_lock); 159 160 return 0; 161} 162 163static void ecb_paes_exit(struct crypto_skcipher *tfm) 164{ 165 struct s390_paes_ctx *ctx = crypto_skcipher_ctx(tfm); 166 167 _free_kb_keybuf(&ctx->kb); 168} 169 170static inline int __ecb_paes_set_key(struct s390_paes_ctx *ctx) 171{ 172 unsigned long fc; 173 174 if (__paes_convert_key(ctx)) 175 return -EINVAL; 176 177 /* Pick the correct function code based on the protected key type */ 178 fc = (ctx->pk.type == PKEY_KEYTYPE_AES_128) ? CPACF_KM_PAES_128 : 179 (ctx->pk.type == PKEY_KEYTYPE_AES_192) ? CPACF_KM_PAES_192 : 180 (ctx->pk.type == PKEY_KEYTYPE_AES_256) ? CPACF_KM_PAES_256 : 0; 181 182 /* Check if the function code is available */ 183 ctx->fc = (fc && cpacf_test_func(&km_functions, fc)) ? fc : 0; 184 185 return ctx->fc ? 0 : -EINVAL; 186} 187 188static int ecb_paes_set_key(struct crypto_skcipher *tfm, const u8 *in_key, 189 unsigned int key_len) 190{ 191 int rc; 192 struct s390_paes_ctx *ctx = crypto_skcipher_ctx(tfm); 193 194 _free_kb_keybuf(&ctx->kb); 195 rc = _key_to_kb(&ctx->kb, in_key, key_len); 196 if (rc) 197 return rc; 198 199 return __ecb_paes_set_key(ctx); 200} 201 202static int ecb_paes_crypt(struct skcipher_request *req, unsigned long modifier) 203{ 204 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 205 struct s390_paes_ctx *ctx = crypto_skcipher_ctx(tfm); 206 struct skcipher_walk walk; 207 unsigned int nbytes, n, k; 208 int ret; 209 struct { 210 u8 key[MAXPROTKEYSIZE]; 211 } param; 212 213 ret = skcipher_walk_virt(&walk, req, false); 214 if (ret) 215 return ret; 216 217 spin_lock_bh(&ctx->pk_lock); 218 memcpy(param.key, ctx->pk.protkey, MAXPROTKEYSIZE); 219 spin_unlock_bh(&ctx->pk_lock); 220 221 while ((nbytes = walk.nbytes) != 0) { 222 /* only use complete blocks */ 223 n = nbytes & ~(AES_BLOCK_SIZE - 1); 224 k = cpacf_km(ctx->fc | modifier, ¶m, 225 walk.dst.virt.addr, walk.src.virt.addr, n); 226 if (k) 227 ret = skcipher_walk_done(&walk, nbytes - k); 228 if (k < n) { 229 if (__paes_convert_key(ctx)) 230 return skcipher_walk_done(&walk, -EIO); 231 spin_lock_bh(&ctx->pk_lock); 232 memcpy(param.key, ctx->pk.protkey, MAXPROTKEYSIZE); 233 spin_unlock_bh(&ctx->pk_lock); 234 } 235 } 236 return ret; 237} 238 239static int ecb_paes_encrypt(struct skcipher_request *req) 240{ 241 return ecb_paes_crypt(req, 0); 242} 243 244static int ecb_paes_decrypt(struct skcipher_request *req) 245{ 246 return ecb_paes_crypt(req, CPACF_DECRYPT); 247} 248 249static struct skcipher_alg ecb_paes_alg = { 250 .base.cra_name = "ecb(paes)", 251 .base.cra_driver_name = "ecb-paes-s390", 252 .base.cra_priority = 401, /* combo: aes + ecb + 1 */ 253 .base.cra_blocksize = AES_BLOCK_SIZE, 254 .base.cra_ctxsize = sizeof(struct s390_paes_ctx), 255 .base.cra_module = THIS_MODULE, 256 .base.cra_list = LIST_HEAD_INIT(ecb_paes_alg.base.cra_list), 257 .init = ecb_paes_init, 258 .exit = ecb_paes_exit, 259 .min_keysize = PAES_MIN_KEYSIZE, 260 .max_keysize = PAES_MAX_KEYSIZE, 261 .setkey = ecb_paes_set_key, 262 .encrypt = ecb_paes_encrypt, 263 .decrypt = ecb_paes_decrypt, 264}; 265 266static int cbc_paes_init(struct crypto_skcipher *tfm) 267{ 268 struct s390_paes_ctx *ctx = crypto_skcipher_ctx(tfm); 269 270 ctx->kb.key = NULL; 271 spin_lock_init(&ctx->pk_lock); 272 273 return 0; 274} 275 276static void cbc_paes_exit(struct crypto_skcipher *tfm) 277{ 278 struct s390_paes_ctx *ctx = crypto_skcipher_ctx(tfm); 279 280 _free_kb_keybuf(&ctx->kb); 281} 282 283static inline int __cbc_paes_set_key(struct s390_paes_ctx *ctx) 284{ 285 unsigned long fc; 286 287 if (__paes_convert_key(ctx)) 288 return -EINVAL; 289 290 /* Pick the correct function code based on the protected key type */ 291 fc = (ctx->pk.type == PKEY_KEYTYPE_AES_128) ? CPACF_KMC_PAES_128 : 292 (ctx->pk.type == PKEY_KEYTYPE_AES_192) ? CPACF_KMC_PAES_192 : 293 (ctx->pk.type == PKEY_KEYTYPE_AES_256) ? CPACF_KMC_PAES_256 : 0; 294 295 /* Check if the function code is available */ 296 ctx->fc = (fc && cpacf_test_func(&kmc_functions, fc)) ? fc : 0; 297 298 return ctx->fc ? 0 : -EINVAL; 299} 300 301static int cbc_paes_set_key(struct crypto_skcipher *tfm, const u8 *in_key, 302 unsigned int key_len) 303{ 304 int rc; 305 struct s390_paes_ctx *ctx = crypto_skcipher_ctx(tfm); 306 307 _free_kb_keybuf(&ctx->kb); 308 rc = _key_to_kb(&ctx->kb, in_key, key_len); 309 if (rc) 310 return rc; 311 312 return __cbc_paes_set_key(ctx); 313} 314 315static int cbc_paes_crypt(struct skcipher_request *req, unsigned long modifier) 316{ 317 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 318 struct s390_paes_ctx *ctx = crypto_skcipher_ctx(tfm); 319 struct skcipher_walk walk; 320 unsigned int nbytes, n, k; 321 int ret; 322 struct { 323 u8 iv[AES_BLOCK_SIZE]; 324 u8 key[MAXPROTKEYSIZE]; 325 } param; 326 327 ret = skcipher_walk_virt(&walk, req, false); 328 if (ret) 329 return ret; 330 331 memcpy(param.iv, walk.iv, AES_BLOCK_SIZE); 332 spin_lock_bh(&ctx->pk_lock); 333 memcpy(param.key, ctx->pk.protkey, MAXPROTKEYSIZE); 334 spin_unlock_bh(&ctx->pk_lock); 335 336 while ((nbytes = walk.nbytes) != 0) { 337 /* only use complete blocks */ 338 n = nbytes & ~(AES_BLOCK_SIZE - 1); 339 k = cpacf_kmc(ctx->fc | modifier, ¶m, 340 walk.dst.virt.addr, walk.src.virt.addr, n); 341 if (k) { 342 memcpy(walk.iv, param.iv, AES_BLOCK_SIZE); 343 ret = skcipher_walk_done(&walk, nbytes - k); 344 } 345 if (k < n) { 346 if (__paes_convert_key(ctx)) 347 return skcipher_walk_done(&walk, -EIO); 348 spin_lock_bh(&ctx->pk_lock); 349 memcpy(param.key, ctx->pk.protkey, MAXPROTKEYSIZE); 350 spin_unlock_bh(&ctx->pk_lock); 351 } 352 } 353 return ret; 354} 355 356static int cbc_paes_encrypt(struct skcipher_request *req) 357{ 358 return cbc_paes_crypt(req, 0); 359} 360 361static int cbc_paes_decrypt(struct skcipher_request *req) 362{ 363 return cbc_paes_crypt(req, CPACF_DECRYPT); 364} 365 366static struct skcipher_alg cbc_paes_alg = { 367 .base.cra_name = "cbc(paes)", 368 .base.cra_driver_name = "cbc-paes-s390", 369 .base.cra_priority = 402, /* ecb-paes-s390 + 1 */ 370 .base.cra_blocksize = AES_BLOCK_SIZE, 371 .base.cra_ctxsize = sizeof(struct s390_paes_ctx), 372 .base.cra_module = THIS_MODULE, 373 .base.cra_list = LIST_HEAD_INIT(cbc_paes_alg.base.cra_list), 374 .init = cbc_paes_init, 375 .exit = cbc_paes_exit, 376 .min_keysize = PAES_MIN_KEYSIZE, 377 .max_keysize = PAES_MAX_KEYSIZE, 378 .ivsize = AES_BLOCK_SIZE, 379 .setkey = cbc_paes_set_key, 380 .encrypt = cbc_paes_encrypt, 381 .decrypt = cbc_paes_decrypt, 382}; 383 384static int xts_paes_init(struct crypto_skcipher *tfm) 385{ 386 struct s390_pxts_ctx *ctx = crypto_skcipher_ctx(tfm); 387 388 ctx->kb[0].key = NULL; 389 ctx->kb[1].key = NULL; 390 spin_lock_init(&ctx->pk_lock); 391 392 return 0; 393} 394 395static void xts_paes_exit(struct crypto_skcipher *tfm) 396{ 397 struct s390_pxts_ctx *ctx = crypto_skcipher_ctx(tfm); 398 399 _free_kb_keybuf(&ctx->kb[0]); 400 _free_kb_keybuf(&ctx->kb[1]); 401} 402 403static inline int __xts_paes_convert_key(struct s390_pxts_ctx *ctx) 404{ 405 struct pkey_protkey pkey0, pkey1; 406 407 if (__paes_keyblob2pkey(&ctx->kb[0], &pkey0) || 408 __paes_keyblob2pkey(&ctx->kb[1], &pkey1)) 409 return -EINVAL; 410 411 spin_lock_bh(&ctx->pk_lock); 412 memcpy(&ctx->pk[0], &pkey0, sizeof(pkey0)); 413 memcpy(&ctx->pk[1], &pkey1, sizeof(pkey1)); 414 spin_unlock_bh(&ctx->pk_lock); 415 416 return 0; 417} 418 419static inline int __xts_paes_set_key(struct s390_pxts_ctx *ctx) 420{ 421 unsigned long fc; 422 423 if (__xts_paes_convert_key(ctx)) 424 return -EINVAL; 425 426 if (ctx->pk[0].type != ctx->pk[1].type) 427 return -EINVAL; 428 429 /* Pick the correct function code based on the protected key type */ 430 fc = (ctx->pk[0].type == PKEY_KEYTYPE_AES_128) ? CPACF_KM_PXTS_128 : 431 (ctx->pk[0].type == PKEY_KEYTYPE_AES_256) ? 432 CPACF_KM_PXTS_256 : 0; 433 434 /* Check if the function code is available */ 435 ctx->fc = (fc && cpacf_test_func(&km_functions, fc)) ? fc : 0; 436 437 return ctx->fc ? 0 : -EINVAL; 438} 439 440static int xts_paes_set_key(struct crypto_skcipher *tfm, const u8 *in_key, 441 unsigned int xts_key_len) 442{ 443 int rc; 444 struct s390_pxts_ctx *ctx = crypto_skcipher_ctx(tfm); 445 u8 ckey[2 * AES_MAX_KEY_SIZE]; 446 unsigned int ckey_len, key_len; 447 448 if (xts_key_len % 2) 449 return -EINVAL; 450 451 key_len = xts_key_len / 2; 452 453 _free_kb_keybuf(&ctx->kb[0]); 454 _free_kb_keybuf(&ctx->kb[1]); 455 rc = _key_to_kb(&ctx->kb[0], in_key, key_len); 456 if (rc) 457 return rc; 458 rc = _key_to_kb(&ctx->kb[1], in_key + key_len, key_len); 459 if (rc) 460 return rc; 461 462 rc = __xts_paes_set_key(ctx); 463 if (rc) 464 return rc; 465 466 /* 467 * xts_check_key verifies the key length is not odd and makes 468 * sure that the two keys are not the same. This can be done 469 * on the two protected keys as well 470 */ 471 ckey_len = (ctx->pk[0].type == PKEY_KEYTYPE_AES_128) ? 472 AES_KEYSIZE_128 : AES_KEYSIZE_256; 473 memcpy(ckey, ctx->pk[0].protkey, ckey_len); 474 memcpy(ckey + ckey_len, ctx->pk[1].protkey, ckey_len); 475 return xts_verify_key(tfm, ckey, 2*ckey_len); 476} 477 478static int xts_paes_crypt(struct skcipher_request *req, unsigned long modifier) 479{ 480 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 481 struct s390_pxts_ctx *ctx = crypto_skcipher_ctx(tfm); 482 struct skcipher_walk walk; 483 unsigned int keylen, offset, nbytes, n, k; 484 int ret; 485 struct { 486 u8 key[MAXPROTKEYSIZE]; /* key + verification pattern */ 487 u8 tweak[16]; 488 u8 block[16]; 489 u8 bit[16]; 490 u8 xts[16]; 491 } pcc_param; 492 struct { 493 u8 key[MAXPROTKEYSIZE]; /* key + verification pattern */ 494 u8 init[16]; 495 } xts_param; 496 497 ret = skcipher_walk_virt(&walk, req, false); 498 if (ret) 499 return ret; 500 501 keylen = (ctx->pk[0].type == PKEY_KEYTYPE_AES_128) ? 48 : 64; 502 offset = (ctx->pk[0].type == PKEY_KEYTYPE_AES_128) ? 16 : 0; 503 504 memset(&pcc_param, 0, sizeof(pcc_param)); 505 memcpy(pcc_param.tweak, walk.iv, sizeof(pcc_param.tweak)); 506 spin_lock_bh(&ctx->pk_lock); 507 memcpy(pcc_param.key + offset, ctx->pk[1].protkey, keylen); 508 memcpy(xts_param.key + offset, ctx->pk[0].protkey, keylen); 509 spin_unlock_bh(&ctx->pk_lock); 510 cpacf_pcc(ctx->fc, pcc_param.key + offset); 511 memcpy(xts_param.init, pcc_param.xts, 16); 512 513 while ((nbytes = walk.nbytes) != 0) { 514 /* only use complete blocks */ 515 n = nbytes & ~(AES_BLOCK_SIZE - 1); 516 k = cpacf_km(ctx->fc | modifier, xts_param.key + offset, 517 walk.dst.virt.addr, walk.src.virt.addr, n); 518 if (k) 519 ret = skcipher_walk_done(&walk, nbytes - k); 520 if (k < n) { 521 if (__xts_paes_convert_key(ctx)) 522 return skcipher_walk_done(&walk, -EIO); 523 spin_lock_bh(&ctx->pk_lock); 524 memcpy(xts_param.key + offset, 525 ctx->pk[0].protkey, keylen); 526 spin_unlock_bh(&ctx->pk_lock); 527 } 528 } 529 530 return ret; 531} 532 533static int xts_paes_encrypt(struct skcipher_request *req) 534{ 535 return xts_paes_crypt(req, 0); 536} 537 538static int xts_paes_decrypt(struct skcipher_request *req) 539{ 540 return xts_paes_crypt(req, CPACF_DECRYPT); 541} 542 543static struct skcipher_alg xts_paes_alg = { 544 .base.cra_name = "xts(paes)", 545 .base.cra_driver_name = "xts-paes-s390", 546 .base.cra_priority = 402, /* ecb-paes-s390 + 1 */ 547 .base.cra_blocksize = AES_BLOCK_SIZE, 548 .base.cra_ctxsize = sizeof(struct s390_pxts_ctx), 549 .base.cra_module = THIS_MODULE, 550 .base.cra_list = LIST_HEAD_INIT(xts_paes_alg.base.cra_list), 551 .init = xts_paes_init, 552 .exit = xts_paes_exit, 553 .min_keysize = 2 * PAES_MIN_KEYSIZE, 554 .max_keysize = 2 * PAES_MAX_KEYSIZE, 555 .ivsize = AES_BLOCK_SIZE, 556 .setkey = xts_paes_set_key, 557 .encrypt = xts_paes_encrypt, 558 .decrypt = xts_paes_decrypt, 559}; 560 561static int ctr_paes_init(struct crypto_skcipher *tfm) 562{ 563 struct s390_paes_ctx *ctx = crypto_skcipher_ctx(tfm); 564 565 ctx->kb.key = NULL; 566 spin_lock_init(&ctx->pk_lock); 567 568 return 0; 569} 570 571static void ctr_paes_exit(struct crypto_skcipher *tfm) 572{ 573 struct s390_paes_ctx *ctx = crypto_skcipher_ctx(tfm); 574 575 _free_kb_keybuf(&ctx->kb); 576} 577 578static inline int __ctr_paes_set_key(struct s390_paes_ctx *ctx) 579{ 580 unsigned long fc; 581 582 if (__paes_convert_key(ctx)) 583 return -EINVAL; 584 585 /* Pick the correct function code based on the protected key type */ 586 fc = (ctx->pk.type == PKEY_KEYTYPE_AES_128) ? CPACF_KMCTR_PAES_128 : 587 (ctx->pk.type == PKEY_KEYTYPE_AES_192) ? CPACF_KMCTR_PAES_192 : 588 (ctx->pk.type == PKEY_KEYTYPE_AES_256) ? 589 CPACF_KMCTR_PAES_256 : 0; 590 591 /* Check if the function code is available */ 592 ctx->fc = (fc && cpacf_test_func(&kmctr_functions, fc)) ? fc : 0; 593 594 return ctx->fc ? 0 : -EINVAL; 595} 596 597static int ctr_paes_set_key(struct crypto_skcipher *tfm, const u8 *in_key, 598 unsigned int key_len) 599{ 600 int rc; 601 struct s390_paes_ctx *ctx = crypto_skcipher_ctx(tfm); 602 603 _free_kb_keybuf(&ctx->kb); 604 rc = _key_to_kb(&ctx->kb, in_key, key_len); 605 if (rc) 606 return rc; 607 608 return __ctr_paes_set_key(ctx); 609} 610 611static unsigned int __ctrblk_init(u8 *ctrptr, u8 *iv, unsigned int nbytes) 612{ 613 unsigned int i, n; 614 615 /* only use complete blocks, max. PAGE_SIZE */ 616 memcpy(ctrptr, iv, AES_BLOCK_SIZE); 617 n = (nbytes > PAGE_SIZE) ? PAGE_SIZE : nbytes & ~(AES_BLOCK_SIZE - 1); 618 for (i = (n / AES_BLOCK_SIZE) - 1; i > 0; i--) { 619 memcpy(ctrptr + AES_BLOCK_SIZE, ctrptr, AES_BLOCK_SIZE); 620 crypto_inc(ctrptr + AES_BLOCK_SIZE, AES_BLOCK_SIZE); 621 ctrptr += AES_BLOCK_SIZE; 622 } 623 return n; 624} 625 626static int ctr_paes_crypt(struct skcipher_request *req) 627{ 628 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 629 struct s390_paes_ctx *ctx = crypto_skcipher_ctx(tfm); 630 u8 buf[AES_BLOCK_SIZE], *ctrptr; 631 struct skcipher_walk walk; 632 unsigned int nbytes, n, k; 633 int ret, locked; 634 struct { 635 u8 key[MAXPROTKEYSIZE]; 636 } param; 637 638 ret = skcipher_walk_virt(&walk, req, false); 639 if (ret) 640 return ret; 641 642 spin_lock_bh(&ctx->pk_lock); 643 memcpy(param.key, ctx->pk.protkey, MAXPROTKEYSIZE); 644 spin_unlock_bh(&ctx->pk_lock); 645 646 locked = mutex_trylock(&ctrblk_lock); 647 648 while ((nbytes = walk.nbytes) >= AES_BLOCK_SIZE) { 649 n = AES_BLOCK_SIZE; 650 if (nbytes >= 2*AES_BLOCK_SIZE && locked) 651 n = __ctrblk_init(ctrblk, walk.iv, nbytes); 652 ctrptr = (n > AES_BLOCK_SIZE) ? ctrblk : walk.iv; 653 k = cpacf_kmctr(ctx->fc, ¶m, walk.dst.virt.addr, 654 walk.src.virt.addr, n, ctrptr); 655 if (k) { 656 if (ctrptr == ctrblk) 657 memcpy(walk.iv, ctrptr + k - AES_BLOCK_SIZE, 658 AES_BLOCK_SIZE); 659 crypto_inc(walk.iv, AES_BLOCK_SIZE); 660 ret = skcipher_walk_done(&walk, nbytes - k); 661 } 662 if (k < n) { 663 if (__paes_convert_key(ctx)) { 664 if (locked) 665 mutex_unlock(&ctrblk_lock); 666 return skcipher_walk_done(&walk, -EIO); 667 } 668 spin_lock_bh(&ctx->pk_lock); 669 memcpy(param.key, ctx->pk.protkey, MAXPROTKEYSIZE); 670 spin_unlock_bh(&ctx->pk_lock); 671 } 672 } 673 if (locked) 674 mutex_unlock(&ctrblk_lock); 675 /* 676 * final block may be < AES_BLOCK_SIZE, copy only nbytes 677 */ 678 if (nbytes) { 679 memset(buf, 0, AES_BLOCK_SIZE); 680 memcpy(buf, walk.src.virt.addr, nbytes); 681 while (1) { 682 if (cpacf_kmctr(ctx->fc, ¶m, buf, 683 buf, AES_BLOCK_SIZE, 684 walk.iv) == AES_BLOCK_SIZE) 685 break; 686 if (__paes_convert_key(ctx)) 687 return skcipher_walk_done(&walk, -EIO); 688 spin_lock_bh(&ctx->pk_lock); 689 memcpy(param.key, ctx->pk.protkey, MAXPROTKEYSIZE); 690 spin_unlock_bh(&ctx->pk_lock); 691 } 692 memcpy(walk.dst.virt.addr, buf, nbytes); 693 crypto_inc(walk.iv, AES_BLOCK_SIZE); 694 ret = skcipher_walk_done(&walk, nbytes); 695 } 696 697 return ret; 698} 699 700static struct skcipher_alg ctr_paes_alg = { 701 .base.cra_name = "ctr(paes)", 702 .base.cra_driver_name = "ctr-paes-s390", 703 .base.cra_priority = 402, /* ecb-paes-s390 + 1 */ 704 .base.cra_blocksize = 1, 705 .base.cra_ctxsize = sizeof(struct s390_paes_ctx), 706 .base.cra_module = THIS_MODULE, 707 .base.cra_list = LIST_HEAD_INIT(ctr_paes_alg.base.cra_list), 708 .init = ctr_paes_init, 709 .exit = ctr_paes_exit, 710 .min_keysize = PAES_MIN_KEYSIZE, 711 .max_keysize = PAES_MAX_KEYSIZE, 712 .ivsize = AES_BLOCK_SIZE, 713 .setkey = ctr_paes_set_key, 714 .encrypt = ctr_paes_crypt, 715 .decrypt = ctr_paes_crypt, 716 .chunksize = AES_BLOCK_SIZE, 717}; 718 719static inline void __crypto_unregister_skcipher(struct skcipher_alg *alg) 720{ 721 if (!list_empty(&alg->base.cra_list)) 722 crypto_unregister_skcipher(alg); 723} 724 725static void paes_s390_fini(void) 726{ 727 __crypto_unregister_skcipher(&ctr_paes_alg); 728 __crypto_unregister_skcipher(&xts_paes_alg); 729 __crypto_unregister_skcipher(&cbc_paes_alg); 730 __crypto_unregister_skcipher(&ecb_paes_alg); 731 if (ctrblk) 732 free_page((unsigned long) ctrblk); 733} 734 735static int __init paes_s390_init(void) 736{ 737 int ret; 738 739 /* Query available functions for KM, KMC and KMCTR */ 740 cpacf_query(CPACF_KM, &km_functions); 741 cpacf_query(CPACF_KMC, &kmc_functions); 742 cpacf_query(CPACF_KMCTR, &kmctr_functions); 743 744 if (cpacf_test_func(&km_functions, CPACF_KM_PAES_128) || 745 cpacf_test_func(&km_functions, CPACF_KM_PAES_192) || 746 cpacf_test_func(&km_functions, CPACF_KM_PAES_256)) { 747 ret = crypto_register_skcipher(&ecb_paes_alg); 748 if (ret) 749 goto out_err; 750 } 751 752 if (cpacf_test_func(&kmc_functions, CPACF_KMC_PAES_128) || 753 cpacf_test_func(&kmc_functions, CPACF_KMC_PAES_192) || 754 cpacf_test_func(&kmc_functions, CPACF_KMC_PAES_256)) { 755 ret = crypto_register_skcipher(&cbc_paes_alg); 756 if (ret) 757 goto out_err; 758 } 759 760 if (cpacf_test_func(&km_functions, CPACF_KM_PXTS_128) || 761 cpacf_test_func(&km_functions, CPACF_KM_PXTS_256)) { 762 ret = crypto_register_skcipher(&xts_paes_alg); 763 if (ret) 764 goto out_err; 765 } 766 767 if (cpacf_test_func(&kmctr_functions, CPACF_KMCTR_PAES_128) || 768 cpacf_test_func(&kmctr_functions, CPACF_KMCTR_PAES_192) || 769 cpacf_test_func(&kmctr_functions, CPACF_KMCTR_PAES_256)) { 770 ctrblk = (u8 *) __get_free_page(GFP_KERNEL); 771 if (!ctrblk) { 772 ret = -ENOMEM; 773 goto out_err; 774 } 775 ret = crypto_register_skcipher(&ctr_paes_alg); 776 if (ret) 777 goto out_err; 778 } 779 780 return 0; 781out_err: 782 paes_s390_fini(); 783 return ret; 784} 785 786module_init(paes_s390_init); 787module_exit(paes_s390_fini); 788 789MODULE_ALIAS_CRYPTO("paes"); 790 791MODULE_DESCRIPTION("Rijndael (AES) Cipher Algorithm with protected keys"); 792MODULE_LICENSE("GPL"); 793