1 /* SPDX-License-Identifier: GPL-2.0-or-later */ 2 /* 3 * SM4 Cipher Algorithm for ARMv8 with Crypto Extensions 4 * as specified in 5 * https://tools.ietf.org/id/draft-ribose-cfrg-sm4-10.html 6 * 7 * Copyright (C) 2022, Alibaba Group. 8 * Copyright (C) 2022 Tianjia Zhang <tianjia.zhang@linux.alibaba.com> 9 */ 10 11 #include <linux/linkage.h> 12 #include <asm/assembler.h> 13 #include "sm4-ce-asm.h" 14 15 .arch armv8-a+crypto 16 17 .irp b, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, \ 18 20, 24, 25, 26, 27, 28, 29, 30, 31 19 .set .Lv\b\().4s, \b 20 .endr 21 22 .macro sm4e, vd, vn 23 .inst 0xcec08400 | (.L\vn << 5) | .L\vd 24 .endm 25 26 .macro sm4ekey, vd, vn, vm 27 .inst 0xce60c800 | (.L\vm << 16) | (.L\vn << 5) | .L\vd 28 .endm 29 30 /* Register macros */ 31 32 #define RTMP0 v16 33 #define RTMP1 v17 34 #define RTMP2 v18 35 #define RTMP3 v19 36 37 #define RIV v20 38 #define RMAC v20 39 #define RMASK v21 40 41 42 .align 3 43 SYM_FUNC_START(sm4_ce_expand_key) 44 /* input: 45 * x0: 128-bit key 46 * x1: rkey_enc 47 * x2: rkey_dec 48 * x3: fk array 49 * x4: ck array 50 */ 51 ld1 {v0.16b}, [x0]; 52 rev32 v0.16b, v0.16b; 53 ld1 {v1.16b}, [x3]; 54 /* load ck */ 55 ld1 {v24.16b-v27.16b}, [x4], #64; 56 ld1 {v28.16b-v31.16b}, [x4]; 57 58 /* input ^ fk */ 59 eor v0.16b, v0.16b, v1.16b; 60 61 sm4ekey v0.4s, v0.4s, v24.4s; 62 sm4ekey v1.4s, v0.4s, v25.4s; 63 sm4ekey v2.4s, v1.4s, v26.4s; 64 sm4ekey v3.4s, v2.4s, v27.4s; 65 sm4ekey v4.4s, v3.4s, v28.4s; 66 sm4ekey v5.4s, v4.4s, v29.4s; 67 sm4ekey v6.4s, v5.4s, v30.4s; 68 sm4ekey v7.4s, v6.4s, v31.4s; 69 70 adr_l x5, .Lbswap128_mask 71 ld1 {v24.16b}, [x5] 72 73 st1 {v0.16b-v3.16b}, [x1], #64; 74 st1 {v4.16b-v7.16b}, [x1]; 75 76 tbl v16.16b, {v7.16b}, v24.16b 77 tbl v17.16b, {v6.16b}, v24.16b 78 tbl v18.16b, {v5.16b}, v24.16b 79 tbl v19.16b, {v4.16b}, v24.16b 80 tbl v20.16b, {v3.16b}, v24.16b 81 tbl v21.16b, {v2.16b}, v24.16b 82 tbl v22.16b, {v1.16b}, v24.16b 83 tbl v23.16b, {v0.16b}, v24.16b 84 85 st1 {v16.16b-v19.16b}, [x2], #64 86 st1 {v20.16b-v23.16b}, [x2] 87 88 ret; 89 SYM_FUNC_END(sm4_ce_expand_key) 90 91 .align 3 92 SYM_FUNC_START(sm4_ce_crypt_block) 93 /* input: 94 * x0: round key array, CTX 95 * x1: dst 96 * x2: src 97 */ 98 SM4_PREPARE(x0) 99 100 ld1 {v0.16b}, [x2]; 101 SM4_CRYPT_BLK(v0); 102 st1 {v0.16b}, [x1]; 103 104 ret; 105 SYM_FUNC_END(sm4_ce_crypt_block) 106 107 .align 3 108 SYM_FUNC_START(sm4_ce_crypt) 109 /* input: 110 * x0: round key array, CTX 111 * x1: dst 112 * x2: src 113 * w3: nblocks 114 */ 115 SM4_PREPARE(x0) 116 117 .Lcrypt_loop_blk: 118 sub w3, w3, #8; 119 tbnz w3, #31, .Lcrypt_tail8; 120 121 ld1 {v0.16b-v3.16b}, [x2], #64; 122 ld1 {v4.16b-v7.16b}, [x2], #64; 123 124 SM4_CRYPT_BLK8(v0, v1, v2, v3, v4, v5, v6, v7); 125 126 st1 {v0.16b-v3.16b}, [x1], #64; 127 st1 {v4.16b-v7.16b}, [x1], #64; 128 129 cbz w3, .Lcrypt_end; 130 b .Lcrypt_loop_blk; 131 132 .Lcrypt_tail8: 133 add w3, w3, #8; 134 cmp w3, #4; 135 blt .Lcrypt_tail4; 136 137 sub w3, w3, #4; 138 139 ld1 {v0.16b-v3.16b}, [x2], #64; 140 SM4_CRYPT_BLK4(v0, v1, v2, v3); 141 st1 {v0.16b-v3.16b}, [x1], #64; 142 143 cbz w3, .Lcrypt_end; 144 145 .Lcrypt_tail4: 146 sub w3, w3, #1; 147 148 ld1 {v0.16b}, [x2], #16; 149 SM4_CRYPT_BLK(v0); 150 st1 {v0.16b}, [x1], #16; 151 152 cbnz w3, .Lcrypt_tail4; 153 154 .Lcrypt_end: 155 ret; 156 SYM_FUNC_END(sm4_ce_crypt) 157 158 .align 3 159 SYM_FUNC_START(sm4_ce_cbc_enc) 160 /* input: 161 * x0: round key array, CTX 162 * x1: dst 163 * x2: src 164 * x3: iv (big endian, 128 bit) 165 * w4: nblocks 166 */ 167 SM4_PREPARE(x0) 168 169 ld1 {RIV.16b}, [x3] 170 171 .Lcbc_enc_loop_4x: 172 cmp w4, #4 173 blt .Lcbc_enc_loop_1x 174 175 sub w4, w4, #4 176 177 ld1 {v0.16b-v3.16b}, [x2], #64 178 179 eor v0.16b, v0.16b, RIV.16b 180 SM4_CRYPT_BLK(v0) 181 eor v1.16b, v1.16b, v0.16b 182 SM4_CRYPT_BLK(v1) 183 eor v2.16b, v2.16b, v1.16b 184 SM4_CRYPT_BLK(v2) 185 eor v3.16b, v3.16b, v2.16b 186 SM4_CRYPT_BLK(v3) 187 188 st1 {v0.16b-v3.16b}, [x1], #64 189 mov RIV.16b, v3.16b 190 191 cbz w4, .Lcbc_enc_end 192 b .Lcbc_enc_loop_4x 193 194 .Lcbc_enc_loop_1x: 195 sub w4, w4, #1 196 197 ld1 {v0.16b}, [x2], #16 198 199 eor RIV.16b, RIV.16b, v0.16b 200 SM4_CRYPT_BLK(RIV) 201 202 st1 {RIV.16b}, [x1], #16 203 204 cbnz w4, .Lcbc_enc_loop_1x 205 206 .Lcbc_enc_end: 207 /* store new IV */ 208 st1 {RIV.16b}, [x3] 209 210 ret 211 SYM_FUNC_END(sm4_ce_cbc_enc) 212 213 .align 3 214 SYM_FUNC_START(sm4_ce_cbc_dec) 215 /* input: 216 * x0: round key array, CTX 217 * x1: dst 218 * x2: src 219 * x3: iv (big endian, 128 bit) 220 * w4: nblocks 221 */ 222 SM4_PREPARE(x0) 223 224 ld1 {RIV.16b}, [x3] 225 226 .Lcbc_dec_loop_8x: 227 sub w4, w4, #8 228 tbnz w4, #31, .Lcbc_dec_4x 229 230 ld1 {v0.16b-v3.16b}, [x2], #64 231 ld1 {v4.16b-v7.16b}, [x2], #64 232 233 rev32 v8.16b, v0.16b 234 rev32 v9.16b, v1.16b 235 rev32 v10.16b, v2.16b 236 rev32 v11.16b, v3.16b 237 rev32 v12.16b, v4.16b 238 rev32 v13.16b, v5.16b 239 rev32 v14.16b, v6.16b 240 rev32 v15.16b, v7.16b 241 242 SM4_CRYPT_BLK8_BE(v8, v9, v10, v11, v12, v13, v14, v15) 243 244 eor v8.16b, v8.16b, RIV.16b 245 eor v9.16b, v9.16b, v0.16b 246 eor v10.16b, v10.16b, v1.16b 247 eor v11.16b, v11.16b, v2.16b 248 eor v12.16b, v12.16b, v3.16b 249 eor v13.16b, v13.16b, v4.16b 250 eor v14.16b, v14.16b, v5.16b 251 eor v15.16b, v15.16b, v6.16b 252 253 st1 {v8.16b-v11.16b}, [x1], #64 254 st1 {v12.16b-v15.16b}, [x1], #64 255 256 mov RIV.16b, v7.16b 257 258 cbz w4, .Lcbc_dec_end 259 b .Lcbc_dec_loop_8x 260 261 .Lcbc_dec_4x: 262 add w4, w4, #8 263 cmp w4, #4 264 blt .Lcbc_dec_loop_1x 265 266 sub w4, w4, #4 267 268 ld1 {v0.16b-v3.16b}, [x2], #64 269 270 rev32 v8.16b, v0.16b 271 rev32 v9.16b, v1.16b 272 rev32 v10.16b, v2.16b 273 rev32 v11.16b, v3.16b 274 275 SM4_CRYPT_BLK4_BE(v8, v9, v10, v11) 276 277 eor v8.16b, v8.16b, RIV.16b 278 eor v9.16b, v9.16b, v0.16b 279 eor v10.16b, v10.16b, v1.16b 280 eor v11.16b, v11.16b, v2.16b 281 282 st1 {v8.16b-v11.16b}, [x1], #64 283 284 mov RIV.16b, v3.16b 285 286 cbz w4, .Lcbc_dec_end 287 288 .Lcbc_dec_loop_1x: 289 sub w4, w4, #1 290 291 ld1 {v0.16b}, [x2], #16 292 293 rev32 v8.16b, v0.16b 294 295 SM4_CRYPT_BLK_BE(v8) 296 297 eor v8.16b, v8.16b, RIV.16b 298 st1 {v8.16b}, [x1], #16 299 300 mov RIV.16b, v0.16b 301 302 cbnz w4, .Lcbc_dec_loop_1x 303 304 .Lcbc_dec_end: 305 /* store new IV */ 306 st1 {RIV.16b}, [x3] 307 308 ret 309 SYM_FUNC_END(sm4_ce_cbc_dec) 310 311 .align 3 312 SYM_FUNC_START(sm4_ce_cbc_cts_enc) 313 /* input: 314 * x0: round key array, CTX 315 * x1: dst 316 * x2: src 317 * x3: iv (big endian, 128 bit) 318 * w4: nbytes 319 */ 320 SM4_PREPARE(x0) 321 322 sub w5, w4, #16 323 uxtw x5, w5 324 325 ld1 {RIV.16b}, [x3] 326 327 ld1 {v0.16b}, [x2] 328 eor RIV.16b, RIV.16b, v0.16b 329 SM4_CRYPT_BLK(RIV) 330 331 /* load permute table */ 332 adr_l x6, .Lcts_permute_table 333 add x7, x6, #32 334 add x6, x6, x5 335 sub x7, x7, x5 336 ld1 {v3.16b}, [x6] 337 ld1 {v4.16b}, [x7] 338 339 /* overlapping loads */ 340 add x2, x2, x5 341 ld1 {v1.16b}, [x2] 342 343 /* create Cn from En-1 */ 344 tbl v0.16b, {RIV.16b}, v3.16b 345 /* padding Pn with zeros */ 346 tbl v1.16b, {v1.16b}, v4.16b 347 348 eor v1.16b, v1.16b, RIV.16b 349 SM4_CRYPT_BLK(v1) 350 351 /* overlapping stores */ 352 add x5, x1, x5 353 st1 {v0.16b}, [x5] 354 st1 {v1.16b}, [x1] 355 356 ret 357 SYM_FUNC_END(sm4_ce_cbc_cts_enc) 358 359 .align 3 360 SYM_FUNC_START(sm4_ce_cbc_cts_dec) 361 /* input: 362 * x0: round key array, CTX 363 * x1: dst 364 * x2: src 365 * x3: iv (big endian, 128 bit) 366 * w4: nbytes 367 */ 368 SM4_PREPARE(x0) 369 370 sub w5, w4, #16 371 uxtw x5, w5 372 373 ld1 {RIV.16b}, [x3] 374 375 /* load permute table */ 376 adr_l x6, .Lcts_permute_table 377 add x7, x6, #32 378 add x6, x6, x5 379 sub x7, x7, x5 380 ld1 {v3.16b}, [x6] 381 ld1 {v4.16b}, [x7] 382 383 /* overlapping loads */ 384 ld1 {v0.16b}, [x2], x5 385 ld1 {v1.16b}, [x2] 386 387 SM4_CRYPT_BLK(v0) 388 /* select the first Ln bytes of Xn to create Pn */ 389 tbl v2.16b, {v0.16b}, v3.16b 390 eor v2.16b, v2.16b, v1.16b 391 392 /* overwrite the first Ln bytes with Cn to create En-1 */ 393 tbx v0.16b, {v1.16b}, v4.16b 394 SM4_CRYPT_BLK(v0) 395 eor v0.16b, v0.16b, RIV.16b 396 397 /* overlapping stores */ 398 add x5, x1, x5 399 st1 {v2.16b}, [x5] 400 st1 {v0.16b}, [x1] 401 402 ret 403 SYM_FUNC_END(sm4_ce_cbc_cts_dec) 404 405 .align 3 406 SYM_FUNC_START(sm4_ce_cfb_enc) 407 /* input: 408 * x0: round key array, CTX 409 * x1: dst 410 * x2: src 411 * x3: iv (big endian, 128 bit) 412 * w4: nblocks 413 */ 414 SM4_PREPARE(x0) 415 416 ld1 {RIV.16b}, [x3] 417 418 .Lcfb_enc_loop_4x: 419 cmp w4, #4 420 blt .Lcfb_enc_loop_1x 421 422 sub w4, w4, #4 423 424 ld1 {v0.16b-v3.16b}, [x2], #64 425 426 rev32 v8.16b, RIV.16b 427 SM4_CRYPT_BLK_BE(v8) 428 eor v0.16b, v0.16b, v8.16b 429 430 rev32 v8.16b, v0.16b 431 SM4_CRYPT_BLK_BE(v8) 432 eor v1.16b, v1.16b, v8.16b 433 434 rev32 v8.16b, v1.16b 435 SM4_CRYPT_BLK_BE(v8) 436 eor v2.16b, v2.16b, v8.16b 437 438 rev32 v8.16b, v2.16b 439 SM4_CRYPT_BLK_BE(v8) 440 eor v3.16b, v3.16b, v8.16b 441 442 st1 {v0.16b-v3.16b}, [x1], #64 443 mov RIV.16b, v3.16b 444 445 cbz w4, .Lcfb_enc_end 446 b .Lcfb_enc_loop_4x 447 448 .Lcfb_enc_loop_1x: 449 sub w4, w4, #1 450 451 ld1 {v0.16b}, [x2], #16 452 453 SM4_CRYPT_BLK(RIV) 454 eor RIV.16b, RIV.16b, v0.16b 455 456 st1 {RIV.16b}, [x1], #16 457 458 cbnz w4, .Lcfb_enc_loop_1x 459 460 .Lcfb_enc_end: 461 /* store new IV */ 462 st1 {RIV.16b}, [x3] 463 464 ret 465 SYM_FUNC_END(sm4_ce_cfb_enc) 466 467 .align 3 468 SYM_FUNC_START(sm4_ce_cfb_dec) 469 /* input: 470 * x0: round key array, CTX 471 * x1: dst 472 * x2: src 473 * x3: iv (big endian, 128 bit) 474 * w4: nblocks 475 */ 476 SM4_PREPARE(x0) 477 478 ld1 {RIV.16b}, [x3] 479 480 .Lcfb_dec_loop_8x: 481 sub w4, w4, #8 482 tbnz w4, #31, .Lcfb_dec_4x 483 484 ld1 {v0.16b-v3.16b}, [x2], #64 485 ld1 {v4.16b-v7.16b}, [x2], #64 486 487 rev32 v8.16b, RIV.16b 488 rev32 v9.16b, v0.16b 489 rev32 v10.16b, v1.16b 490 rev32 v11.16b, v2.16b 491 rev32 v12.16b, v3.16b 492 rev32 v13.16b, v4.16b 493 rev32 v14.16b, v5.16b 494 rev32 v15.16b, v6.16b 495 496 SM4_CRYPT_BLK8_BE(v8, v9, v10, v11, v12, v13, v14, v15) 497 498 mov RIV.16b, v7.16b 499 500 eor v0.16b, v0.16b, v8.16b 501 eor v1.16b, v1.16b, v9.16b 502 eor v2.16b, v2.16b, v10.16b 503 eor v3.16b, v3.16b, v11.16b 504 eor v4.16b, v4.16b, v12.16b 505 eor v5.16b, v5.16b, v13.16b 506 eor v6.16b, v6.16b, v14.16b 507 eor v7.16b, v7.16b, v15.16b 508 509 st1 {v0.16b-v3.16b}, [x1], #64 510 st1 {v4.16b-v7.16b}, [x1], #64 511 512 cbz w4, .Lcfb_dec_end 513 b .Lcfb_dec_loop_8x 514 515 .Lcfb_dec_4x: 516 add w4, w4, #8 517 cmp w4, #4 518 blt .Lcfb_dec_loop_1x 519 520 sub w4, w4, #4 521 522 ld1 {v0.16b-v3.16b}, [x2], #64 523 524 rev32 v8.16b, RIV.16b 525 rev32 v9.16b, v0.16b 526 rev32 v10.16b, v1.16b 527 rev32 v11.16b, v2.16b 528 529 SM4_CRYPT_BLK4_BE(v8, v9, v10, v11) 530 531 mov RIV.16b, v3.16b 532 533 eor v0.16b, v0.16b, v8.16b 534 eor v1.16b, v1.16b, v9.16b 535 eor v2.16b, v2.16b, v10.16b 536 eor v3.16b, v3.16b, v11.16b 537 538 st1 {v0.16b-v3.16b}, [x1], #64 539 540 cbz w4, .Lcfb_dec_end 541 542 .Lcfb_dec_loop_1x: 543 sub w4, w4, #1 544 545 ld1 {v0.16b}, [x2], #16 546 547 SM4_CRYPT_BLK(RIV) 548 549 eor RIV.16b, RIV.16b, v0.16b 550 st1 {RIV.16b}, [x1], #16 551 552 mov RIV.16b, v0.16b 553 554 cbnz w4, .Lcfb_dec_loop_1x 555 556 .Lcfb_dec_end: 557 /* store new IV */ 558 st1 {RIV.16b}, [x3] 559 560 ret 561 SYM_FUNC_END(sm4_ce_cfb_dec) 562 563 .align 3 564 SYM_FUNC_START(sm4_ce_ctr_enc) 565 /* input: 566 * x0: round key array, CTX 567 * x1: dst 568 * x2: src 569 * x3: ctr (big endian, 128 bit) 570 * w4: nblocks 571 */ 572 SM4_PREPARE(x0) 573 574 ldp x7, x8, [x3] 575 rev x7, x7 576 rev x8, x8 577 578 .Lctr_loop_8x: 579 sub w4, w4, #8 580 tbnz w4, #31, .Lctr_4x 581 582 #define inc_le128(vctr) \ 583 mov vctr.d[1], x8; \ 584 mov vctr.d[0], x7; \ 585 adds x8, x8, #1; \ 586 rev64 vctr.16b, vctr.16b; \ 587 adc x7, x7, xzr; 588 589 /* construct CTRs */ 590 inc_le128(v0) /* +0 */ 591 inc_le128(v1) /* +1 */ 592 inc_le128(v2) /* +2 */ 593 inc_le128(v3) /* +3 */ 594 inc_le128(v4) /* +4 */ 595 inc_le128(v5) /* +5 */ 596 inc_le128(v6) /* +6 */ 597 inc_le128(v7) /* +7 */ 598 599 ld1 {v8.16b-v11.16b}, [x2], #64 600 ld1 {v12.16b-v15.16b}, [x2], #64 601 602 SM4_CRYPT_BLK8(v0, v1, v2, v3, v4, v5, v6, v7) 603 604 eor v0.16b, v0.16b, v8.16b 605 eor v1.16b, v1.16b, v9.16b 606 eor v2.16b, v2.16b, v10.16b 607 eor v3.16b, v3.16b, v11.16b 608 eor v4.16b, v4.16b, v12.16b 609 eor v5.16b, v5.16b, v13.16b 610 eor v6.16b, v6.16b, v14.16b 611 eor v7.16b, v7.16b, v15.16b 612 613 st1 {v0.16b-v3.16b}, [x1], #64 614 st1 {v4.16b-v7.16b}, [x1], #64 615 616 cbz w4, .Lctr_end 617 b .Lctr_loop_8x 618 619 .Lctr_4x: 620 add w4, w4, #8 621 cmp w4, #4 622 blt .Lctr_loop_1x 623 624 sub w4, w4, #4 625 626 /* construct CTRs */ 627 inc_le128(v0) /* +0 */ 628 inc_le128(v1) /* +1 */ 629 inc_le128(v2) /* +2 */ 630 inc_le128(v3) /* +3 */ 631 632 ld1 {v8.16b-v11.16b}, [x2], #64 633 634 SM4_CRYPT_BLK4(v0, v1, v2, v3) 635 636 eor v0.16b, v0.16b, v8.16b 637 eor v1.16b, v1.16b, v9.16b 638 eor v2.16b, v2.16b, v10.16b 639 eor v3.16b, v3.16b, v11.16b 640 641 st1 {v0.16b-v3.16b}, [x1], #64 642 643 cbz w4, .Lctr_end 644 645 .Lctr_loop_1x: 646 sub w4, w4, #1 647 648 /* construct CTRs */ 649 inc_le128(v0) 650 651 ld1 {v8.16b}, [x2], #16 652 653 SM4_CRYPT_BLK(v0) 654 655 eor v0.16b, v0.16b, v8.16b 656 st1 {v0.16b}, [x1], #16 657 658 cbnz w4, .Lctr_loop_1x 659 660 .Lctr_end: 661 /* store new CTR */ 662 rev x7, x7 663 rev x8, x8 664 stp x7, x8, [x3] 665 666 ret 667 SYM_FUNC_END(sm4_ce_ctr_enc) 668 669 670 #define tweak_next(vt, vin, RTMP) \ 671 sshr RTMP.2d, vin.2d, #63; \ 672 and RTMP.16b, RTMP.16b, RMASK.16b; \ 673 add vt.2d, vin.2d, vin.2d; \ 674 ext RTMP.16b, RTMP.16b, RTMP.16b, #8; \ 675 eor vt.16b, vt.16b, RTMP.16b; 676 677 .align 3 678 SYM_FUNC_START(sm4_ce_xts_enc) 679 /* input: 680 * x0: round key array, CTX 681 * x1: dst 682 * x2: src 683 * x3: tweak (big endian, 128 bit) 684 * w4: nbytes 685 * x5: round key array for IV 686 */ 687 ld1 {v8.16b}, [x3] 688 689 cbz x5, .Lxts_enc_nofirst 690 691 SM4_PREPARE(x5) 692 693 /* Generate first tweak */ 694 SM4_CRYPT_BLK(v8) 695 696 .Lxts_enc_nofirst: 697 SM4_PREPARE(x0) 698 699 ands w5, w4, #15 700 lsr w4, w4, #4 701 sub w6, w4, #1 702 csel w4, w4, w6, eq 703 uxtw x5, w5 704 705 movi RMASK.2s, #0x1 706 movi RTMP0.2s, #0x87 707 uzp1 RMASK.4s, RMASK.4s, RTMP0.4s 708 709 cbz w4, .Lxts_enc_cts 710 711 .Lxts_enc_loop_8x: 712 sub w4, w4, #8 713 tbnz w4, #31, .Lxts_enc_4x 714 715 tweak_next( v9, v8, RTMP0) 716 tweak_next(v10, v9, RTMP1) 717 tweak_next(v11, v10, RTMP2) 718 tweak_next(v12, v11, RTMP3) 719 tweak_next(v13, v12, RTMP0) 720 tweak_next(v14, v13, RTMP1) 721 tweak_next(v15, v14, RTMP2) 722 723 ld1 {v0.16b-v3.16b}, [x2], #64 724 ld1 {v4.16b-v7.16b}, [x2], #64 725 eor v0.16b, v0.16b, v8.16b 726 eor v1.16b, v1.16b, v9.16b 727 eor v2.16b, v2.16b, v10.16b 728 eor v3.16b, v3.16b, v11.16b 729 eor v4.16b, v4.16b, v12.16b 730 eor v5.16b, v5.16b, v13.16b 731 eor v6.16b, v6.16b, v14.16b 732 eor v7.16b, v7.16b, v15.16b 733 734 SM4_CRYPT_BLK8(v0, v1, v2, v3, v4, v5, v6, v7) 735 736 eor v0.16b, v0.16b, v8.16b 737 eor v1.16b, v1.16b, v9.16b 738 eor v2.16b, v2.16b, v10.16b 739 eor v3.16b, v3.16b, v11.16b 740 eor v4.16b, v4.16b, v12.16b 741 eor v5.16b, v5.16b, v13.16b 742 eor v6.16b, v6.16b, v14.16b 743 eor v7.16b, v7.16b, v15.16b 744 st1 {v0.16b-v3.16b}, [x1], #64 745 st1 {v4.16b-v7.16b}, [x1], #64 746 747 tweak_next(v8, v15, RTMP3) 748 749 cbz w4, .Lxts_enc_cts 750 b .Lxts_enc_loop_8x 751 752 .Lxts_enc_4x: 753 add w4, w4, #8 754 cmp w4, #4 755 blt .Lxts_enc_loop_1x 756 757 sub w4, w4, #4 758 759 tweak_next( v9, v8, RTMP0) 760 tweak_next(v10, v9, RTMP1) 761 tweak_next(v11, v10, RTMP2) 762 763 ld1 {v0.16b-v3.16b}, [x2], #64 764 eor v0.16b, v0.16b, v8.16b 765 eor v1.16b, v1.16b, v9.16b 766 eor v2.16b, v2.16b, v10.16b 767 eor v3.16b, v3.16b, v11.16b 768 769 SM4_CRYPT_BLK4(v0, v1, v2, v3) 770 771 eor v0.16b, v0.16b, v8.16b 772 eor v1.16b, v1.16b, v9.16b 773 eor v2.16b, v2.16b, v10.16b 774 eor v3.16b, v3.16b, v11.16b 775 st1 {v0.16b-v3.16b}, [x1], #64 776 777 tweak_next(v8, v11, RTMP3) 778 779 cbz w4, .Lxts_enc_cts 780 781 .Lxts_enc_loop_1x: 782 sub w4, w4, #1 783 784 ld1 {v0.16b}, [x2], #16 785 eor v0.16b, v0.16b, v8.16b 786 787 SM4_CRYPT_BLK(v0) 788 789 eor v0.16b, v0.16b, v8.16b 790 st1 {v0.16b}, [x1], #16 791 792 tweak_next(v8, v8, RTMP0) 793 794 cbnz w4, .Lxts_enc_loop_1x 795 796 .Lxts_enc_cts: 797 cbz x5, .Lxts_enc_end 798 799 /* cipher text stealing */ 800 801 tweak_next(v9, v8, RTMP0) 802 ld1 {v0.16b}, [x2] 803 eor v0.16b, v0.16b, v8.16b 804 SM4_CRYPT_BLK(v0) 805 eor v0.16b, v0.16b, v8.16b 806 807 /* load permute table */ 808 adr_l x6, .Lcts_permute_table 809 add x7, x6, #32 810 add x6, x6, x5 811 sub x7, x7, x5 812 ld1 {v3.16b}, [x6] 813 ld1 {v4.16b}, [x7] 814 815 /* overlapping loads */ 816 add x2, x2, x5 817 ld1 {v1.16b}, [x2] 818 819 /* create Cn from En-1 */ 820 tbl v2.16b, {v0.16b}, v3.16b 821 /* padding Pn with En-1 at the end */ 822 tbx v0.16b, {v1.16b}, v4.16b 823 824 eor v0.16b, v0.16b, v9.16b 825 SM4_CRYPT_BLK(v0) 826 eor v0.16b, v0.16b, v9.16b 827 828 829 /* overlapping stores */ 830 add x5, x1, x5 831 st1 {v2.16b}, [x5] 832 st1 {v0.16b}, [x1] 833 834 b .Lxts_enc_ret 835 836 .Lxts_enc_end: 837 /* store new tweak */ 838 st1 {v8.16b}, [x3] 839 840 .Lxts_enc_ret: 841 ret 842 SYM_FUNC_END(sm4_ce_xts_enc) 843 844 .align 3 845 SYM_FUNC_START(sm4_ce_xts_dec) 846 /* input: 847 * x0: round key array, CTX 848 * x1: dst 849 * x2: src 850 * x3: tweak (big endian, 128 bit) 851 * w4: nbytes 852 * x5: round key array for IV 853 */ 854 ld1 {v8.16b}, [x3] 855 856 cbz x5, .Lxts_dec_nofirst 857 858 SM4_PREPARE(x5) 859 860 /* Generate first tweak */ 861 SM4_CRYPT_BLK(v8) 862 863 .Lxts_dec_nofirst: 864 SM4_PREPARE(x0) 865 866 ands w5, w4, #15 867 lsr w4, w4, #4 868 sub w6, w4, #1 869 csel w4, w4, w6, eq 870 uxtw x5, w5 871 872 movi RMASK.2s, #0x1 873 movi RTMP0.2s, #0x87 874 uzp1 RMASK.4s, RMASK.4s, RTMP0.4s 875 876 cbz w4, .Lxts_dec_cts 877 878 .Lxts_dec_loop_8x: 879 sub w4, w4, #8 880 tbnz w4, #31, .Lxts_dec_4x 881 882 tweak_next( v9, v8, RTMP0) 883 tweak_next(v10, v9, RTMP1) 884 tweak_next(v11, v10, RTMP2) 885 tweak_next(v12, v11, RTMP3) 886 tweak_next(v13, v12, RTMP0) 887 tweak_next(v14, v13, RTMP1) 888 tweak_next(v15, v14, RTMP2) 889 890 ld1 {v0.16b-v3.16b}, [x2], #64 891 ld1 {v4.16b-v7.16b}, [x2], #64 892 eor v0.16b, v0.16b, v8.16b 893 eor v1.16b, v1.16b, v9.16b 894 eor v2.16b, v2.16b, v10.16b 895 eor v3.16b, v3.16b, v11.16b 896 eor v4.16b, v4.16b, v12.16b 897 eor v5.16b, v5.16b, v13.16b 898 eor v6.16b, v6.16b, v14.16b 899 eor v7.16b, v7.16b, v15.16b 900 901 SM4_CRYPT_BLK8(v0, v1, v2, v3, v4, v5, v6, v7) 902 903 eor v0.16b, v0.16b, v8.16b 904 eor v1.16b, v1.16b, v9.16b 905 eor v2.16b, v2.16b, v10.16b 906 eor v3.16b, v3.16b, v11.16b 907 eor v4.16b, v4.16b, v12.16b 908 eor v5.16b, v5.16b, v13.16b 909 eor v6.16b, v6.16b, v14.16b 910 eor v7.16b, v7.16b, v15.16b 911 st1 {v0.16b-v3.16b}, [x1], #64 912 st1 {v4.16b-v7.16b}, [x1], #64 913 914 tweak_next(v8, v15, RTMP3) 915 916 cbz w4, .Lxts_dec_cts 917 b .Lxts_dec_loop_8x 918 919 .Lxts_dec_4x: 920 add w4, w4, #8 921 cmp w4, #4 922 blt .Lxts_dec_loop_1x 923 924 sub w4, w4, #4 925 926 tweak_next( v9, v8, RTMP0) 927 tweak_next(v10, v9, RTMP1) 928 tweak_next(v11, v10, RTMP2) 929 930 ld1 {v0.16b-v3.16b}, [x2], #64 931 eor v0.16b, v0.16b, v8.16b 932 eor v1.16b, v1.16b, v9.16b 933 eor v2.16b, v2.16b, v10.16b 934 eor v3.16b, v3.16b, v11.16b 935 936 SM4_CRYPT_BLK4(v0, v1, v2, v3) 937 938 eor v0.16b, v0.16b, v8.16b 939 eor v1.16b, v1.16b, v9.16b 940 eor v2.16b, v2.16b, v10.16b 941 eor v3.16b, v3.16b, v11.16b 942 st1 {v0.16b-v3.16b}, [x1], #64 943 944 tweak_next(v8, v11, RTMP3) 945 946 cbz w4, .Lxts_dec_cts 947 948 .Lxts_dec_loop_1x: 949 sub w4, w4, #1 950 951 ld1 {v0.16b}, [x2], #16 952 eor v0.16b, v0.16b, v8.16b 953 954 SM4_CRYPT_BLK(v0) 955 956 eor v0.16b, v0.16b, v8.16b 957 st1 {v0.16b}, [x1], #16 958 959 tweak_next(v8, v8, RTMP0) 960 961 cbnz w4, .Lxts_dec_loop_1x 962 963 .Lxts_dec_cts: 964 cbz x5, .Lxts_dec_end 965 966 /* cipher text stealing */ 967 968 tweak_next(v9, v8, RTMP0) 969 ld1 {v0.16b}, [x2] 970 eor v0.16b, v0.16b, v9.16b 971 SM4_CRYPT_BLK(v0) 972 eor v0.16b, v0.16b, v9.16b 973 974 /* load permute table */ 975 adr_l x6, .Lcts_permute_table 976 add x7, x6, #32 977 add x6, x6, x5 978 sub x7, x7, x5 979 ld1 {v3.16b}, [x6] 980 ld1 {v4.16b}, [x7] 981 982 /* overlapping loads */ 983 add x2, x2, x5 984 ld1 {v1.16b}, [x2] 985 986 /* create Cn from En-1 */ 987 tbl v2.16b, {v0.16b}, v3.16b 988 /* padding Pn with En-1 at the end */ 989 tbx v0.16b, {v1.16b}, v4.16b 990 991 eor v0.16b, v0.16b, v8.16b 992 SM4_CRYPT_BLK(v0) 993 eor v0.16b, v0.16b, v8.16b 994 995 996 /* overlapping stores */ 997 add x5, x1, x5 998 st1 {v2.16b}, [x5] 999 st1 {v0.16b}, [x1] 1000 1001 b .Lxts_dec_ret 1002 1003 .Lxts_dec_end: 1004 /* store new tweak */ 1005 st1 {v8.16b}, [x3] 1006 1007 .Lxts_dec_ret: 1008 ret 1009 SYM_FUNC_END(sm4_ce_xts_dec) 1010 1011 .align 3 1012 SYM_FUNC_START(sm4_ce_mac_update) 1013 /* input: 1014 * x0: round key array, CTX 1015 * x1: digest 1016 * x2: src 1017 * w3: nblocks 1018 * w4: enc_before 1019 * w5: enc_after 1020 */ 1021 SM4_PREPARE(x0) 1022 1023 ld1 {RMAC.16b}, [x1] 1024 1025 cbz w4, .Lmac_update 1026 1027 SM4_CRYPT_BLK(RMAC) 1028 1029 .Lmac_update: 1030 cbz w3, .Lmac_ret 1031 1032 sub w6, w3, #1 1033 cmp w5, wzr 1034 csel w3, w3, w6, ne 1035 1036 cbz w3, .Lmac_end 1037 1038 .Lmac_loop_4x: 1039 cmp w3, #4 1040 blt .Lmac_loop_1x 1041 1042 sub w3, w3, #4 1043 1044 ld1 {v0.16b-v3.16b}, [x2], #64 1045 1046 eor RMAC.16b, RMAC.16b, v0.16b 1047 SM4_CRYPT_BLK(RMAC) 1048 eor RMAC.16b, RMAC.16b, v1.16b 1049 SM4_CRYPT_BLK(RMAC) 1050 eor RMAC.16b, RMAC.16b, v2.16b 1051 SM4_CRYPT_BLK(RMAC) 1052 eor RMAC.16b, RMAC.16b, v3.16b 1053 SM4_CRYPT_BLK(RMAC) 1054 1055 cbz w3, .Lmac_end 1056 b .Lmac_loop_4x 1057 1058 .Lmac_loop_1x: 1059 sub w3, w3, #1 1060 1061 ld1 {v0.16b}, [x2], #16 1062 1063 eor RMAC.16b, RMAC.16b, v0.16b 1064 SM4_CRYPT_BLK(RMAC) 1065 1066 cbnz w3, .Lmac_loop_1x 1067 1068 1069 .Lmac_end: 1070 cbnz w5, .Lmac_ret 1071 1072 ld1 {v0.16b}, [x2], #16 1073 eor RMAC.16b, RMAC.16b, v0.16b 1074 1075 .Lmac_ret: 1076 st1 {RMAC.16b}, [x1] 1077 ret 1078 SYM_FUNC_END(sm4_ce_mac_update) 1079 1080 1081 .section ".rodata", "a" 1082 .align 4 1083 .Lbswap128_mask: 1084 .byte 0x0c, 0x0d, 0x0e, 0x0f, 0x08, 0x09, 0x0a, 0x0b 1085 .byte 0x04, 0x05, 0x06, 0x07, 0x00, 0x01, 0x02, 0x03 1086 1087 .Lcts_permute_table: 1088 .byte 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff 1089 .byte 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff 1090 .byte 0x0, 0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7 1091 .byte 0x8, 0x9, 0xa, 0xb, 0xc, 0xd, 0xe, 0xf 1092 .byte 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff 1093 .byte 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff 1094