1 /* SPDX-License-Identifier: GPL-2.0-only */ 2 /* 3 * Accelerated GHASH implementation with NEON/ARMv8 vmull.p8/64 instructions. 4 * 5 * Copyright (C) 2015 - 2017 Linaro Ltd. 6 * Copyright (C) 2023 Google LLC. <ardb@google.com> 7 */ 8 9 #include <linux/linkage.h> 10 #include <asm/assembler.h> 11 12 .arch armv8-a 13 .fpu crypto-neon-fp-armv8 14 15 SHASH .req q0 16 T1 .req q1 17 XL .req q2 18 XM .req q3 19 XH .req q4 20 IN1 .req q4 21 22 SHASH_L .req d0 23 SHASH_H .req d1 24 T1_L .req d2 25 T1_H .req d3 26 XL_L .req d4 27 XL_H .req d5 28 XM_L .req d6 29 XM_H .req d7 30 XH_L .req d8 31 32 t0l .req d10 33 t0h .req d11 34 t1l .req d12 35 t1h .req d13 36 t2l .req d14 37 t2h .req d15 38 t3l .req d16 39 t3h .req d17 40 t4l .req d18 41 t4h .req d19 42 43 t0q .req q5 44 t1q .req q6 45 t2q .req q7 46 t3q .req q8 47 t4q .req q9 48 XH2 .req q9 49 50 s1l .req d20 51 s1h .req d21 52 s2l .req d22 53 s2h .req d23 54 s3l .req d24 55 s3h .req d25 56 s4l .req d26 57 s4h .req d27 58 59 MASK .req d28 60 SHASH2_p8 .req d28 61 62 k16 .req d29 63 k32 .req d30 64 k48 .req d31 65 SHASH2_p64 .req d31 66 67 HH .req q10 68 HH3 .req q11 69 HH4 .req q12 70 HH34 .req q13 71 72 HH_L .req d20 73 HH_H .req d21 74 HH3_L .req d22 75 HH3_H .req d23 76 HH4_L .req d24 77 HH4_H .req d25 78 HH34_L .req d26 79 HH34_H .req d27 80 SHASH2_H .req d29 81 82 XL2 .req q5 83 XM2 .req q6 84 T2 .req q7 85 T3 .req q8 86 87 XL2_L .req d10 88 XL2_H .req d11 89 XM2_L .req d12 90 XM2_H .req d13 91 T3_L .req d16 92 T3_H .req d17 93 94 .text 95 96 .macro __pmull_p64, rd, rn, rm, b1, b2, b3, b4 97 vmull.p64 \rd, \rn, \rm 98 .endm 99 100 /* 101 * This implementation of 64x64 -> 128 bit polynomial multiplication 102 * using vmull.p8 instructions (8x8 -> 16) is taken from the paper 103 * "Fast Software Polynomial Multiplication on ARM Processors Using 104 * the NEON Engine" by Danilo Camara, Conrado Gouvea, Julio Lopez and 105 * Ricardo Dahab (https://hal.inria.fr/hal-01506572) 106 * 107 * It has been slightly tweaked for in-order performance, and to allow 108 * 'rq' to overlap with 'ad' or 'bd'. 109 */ 110 .macro __pmull_p8, rq, ad, bd, b1=t4l, b2=t3l, b3=t4l, b4=t3l 111 vext.8 t0l, \ad, \ad, #1 @ A1 112 .ifc \b1, t4l 113 vext.8 t4l, \bd, \bd, #1 @ B1 114 .endif 115 vmull.p8 t0q, t0l, \bd @ F = A1*B 116 vext.8 t1l, \ad, \ad, #2 @ A2 117 vmull.p8 t4q, \ad, \b1 @ E = A*B1 118 .ifc \b2, t3l 119 vext.8 t3l, \bd, \bd, #2 @ B2 120 .endif 121 vmull.p8 t1q, t1l, \bd @ H = A2*B 122 vext.8 t2l, \ad, \ad, #3 @ A3 123 vmull.p8 t3q, \ad, \b2 @ G = A*B2 124 veor t0q, t0q, t4q @ L = E + F 125 .ifc \b3, t4l 126 vext.8 t4l, \bd, \bd, #3 @ B3 127 .endif 128 vmull.p8 t2q, t2l, \bd @ J = A3*B 129 veor t0l, t0l, t0h @ t0 = (L) (P0 + P1) << 8 130 veor t1q, t1q, t3q @ M = G + H 131 .ifc \b4, t3l 132 vext.8 t3l, \bd, \bd, #4 @ B4 133 .endif 134 vmull.p8 t4q, \ad, \b3 @ I = A*B3 135 veor t1l, t1l, t1h @ t1 = (M) (P2 + P3) << 16 136 vmull.p8 t3q, \ad, \b4 @ K = A*B4 137 vand t0h, t0h, k48 138 vand t1h, t1h, k32 139 veor t2q, t2q, t4q @ N = I + J 140 veor t0l, t0l, t0h 141 veor t1l, t1l, t1h 142 veor t2l, t2l, t2h @ t2 = (N) (P4 + P5) << 24 143 vand t2h, t2h, k16 144 veor t3l, t3l, t3h @ t3 = (K) (P6 + P7) << 32 145 vmov.i64 t3h, #0 146 vext.8 t0q, t0q, t0q, #15 147 veor t2l, t2l, t2h 148 vext.8 t1q, t1q, t1q, #14 149 vmull.p8 \rq, \ad, \bd @ D = A*B 150 vext.8 t2q, t2q, t2q, #13 151 vext.8 t3q, t3q, t3q, #12 152 veor t0q, t0q, t1q 153 veor t2q, t2q, t3q 154 veor \rq, \rq, t0q 155 veor \rq, \rq, t2q 156 .endm 157 158 // 159 // PMULL (64x64->128) based reduction for CPUs that can do 160 // it in a single instruction. 161 // 162 .macro __pmull_reduce_p64 163 vmull.p64 T1, XL_L, MASK 164 165 veor XH_L, XH_L, XM_H 166 vext.8 T1, T1, T1, #8 167 veor XL_H, XL_H, XM_L 168 veor T1, T1, XL 169 170 vmull.p64 XL, T1_H, MASK 171 .endm 172 173 // 174 // Alternative reduction for CPUs that lack support for the 175 // 64x64->128 PMULL instruction 176 // 177 .macro __pmull_reduce_p8 178 veor XL_H, XL_H, XM_L 179 veor XH_L, XH_L, XM_H 180 181 vshl.i64 T1, XL, #57 182 vshl.i64 T2, XL, #62 183 veor T1, T1, T2 184 vshl.i64 T2, XL, #63 185 veor T1, T1, T2 186 veor XL_H, XL_H, T1_L 187 veor XH_L, XH_L, T1_H 188 189 vshr.u64 T1, XL, #1 190 veor XH, XH, XL 191 veor XL, XL, T1 192 vshr.u64 T1, T1, #6 193 vshr.u64 XL, XL, #1 194 .endm 195 196 .macro ghash_update, pn, enc, aggregate=1, head=1 197 vld1.64 {XL}, [r1] 198 199 .if \head 200 /* do the head block first, if supplied */ 201 ldr ip, [sp] 202 teq ip, #0 203 beq 0f 204 vld1.64 {T1}, [ip] 205 teq r0, #0 206 b 3f 207 .endif 208 209 0: .ifc \pn, p64 210 .if \aggregate 211 tst r0, #3 // skip until #blocks is a 212 bne 2f // round multiple of 4 213 214 vld1.8 {XL2-XM2}, [r2]! 215 1: vld1.8 {T2-T3}, [r2]! 216 217 .ifnb \enc 218 \enc\()_4x XL2, XM2, T2, T3 219 220 add ip, r3, #16 221 vld1.64 {HH}, [ip, :128]! 222 vld1.64 {HH3-HH4}, [ip, :128] 223 224 veor SHASH2_p64, SHASH_L, SHASH_H 225 veor SHASH2_H, HH_L, HH_H 226 veor HH34_L, HH3_L, HH3_H 227 veor HH34_H, HH4_L, HH4_H 228 229 vmov.i8 MASK, #0xe1 230 vshl.u64 MASK, MASK, #57 231 .endif 232 233 vrev64.8 XL2, XL2 234 vrev64.8 XM2, XM2 235 236 subs r0, r0, #4 237 238 vext.8 T1, XL2, XL2, #8 239 veor XL2_H, XL2_H, XL_L 240 veor XL, XL, T1 241 242 vrev64.8 T1, T3 243 vrev64.8 T3, T2 244 245 vmull.p64 XH, HH4_H, XL_H // a1 * b1 246 veor XL2_H, XL2_H, XL_H 247 vmull.p64 XL, HH4_L, XL_L // a0 * b0 248 vmull.p64 XM, HH34_H, XL2_H // (a1 + a0)(b1 + b0) 249 250 vmull.p64 XH2, HH3_H, XM2_L // a1 * b1 251 veor XM2_L, XM2_L, XM2_H 252 vmull.p64 XL2, HH3_L, XM2_H // a0 * b0 253 vmull.p64 XM2, HH34_L, XM2_L // (a1 + a0)(b1 + b0) 254 255 veor XH, XH, XH2 256 veor XL, XL, XL2 257 veor XM, XM, XM2 258 259 vmull.p64 XH2, HH_H, T3_L // a1 * b1 260 veor T3_L, T3_L, T3_H 261 vmull.p64 XL2, HH_L, T3_H // a0 * b0 262 vmull.p64 XM2, SHASH2_H, T3_L // (a1 + a0)(b1 + b0) 263 264 veor XH, XH, XH2 265 veor XL, XL, XL2 266 veor XM, XM, XM2 267 268 vmull.p64 XH2, SHASH_H, T1_L // a1 * b1 269 veor T1_L, T1_L, T1_H 270 vmull.p64 XL2, SHASH_L, T1_H // a0 * b0 271 vmull.p64 XM2, SHASH2_p64, T1_L // (a1 + a0)(b1 + b0) 272 273 veor XH, XH, XH2 274 veor XL, XL, XL2 275 veor XM, XM, XM2 276 277 beq 4f 278 279 vld1.8 {XL2-XM2}, [r2]! 280 281 veor T1, XL, XH 282 veor XM, XM, T1 283 284 __pmull_reduce_p64 285 286 veor T1, T1, XH 287 veor XL, XL, T1 288 289 b 1b 290 .endif 291 .endif 292 293 2: vld1.8 {T1}, [r2]! 294 295 .ifnb \enc 296 \enc\()_1x T1 297 veor SHASH2_p64, SHASH_L, SHASH_H 298 vmov.i8 MASK, #0xe1 299 vshl.u64 MASK, MASK, #57 300 .endif 301 302 subs r0, r0, #1 303 304 3: /* multiply XL by SHASH in GF(2^128) */ 305 vrev64.8 T1, T1 306 307 vext.8 IN1, T1, T1, #8 308 veor T1_L, T1_L, XL_H 309 veor XL, XL, IN1 310 311 __pmull_\pn XH, XL_H, SHASH_H, s1h, s2h, s3h, s4h @ a1 * b1 312 veor T1, T1, XL 313 __pmull_\pn XL, XL_L, SHASH_L, s1l, s2l, s3l, s4l @ a0 * b0 314 __pmull_\pn XM, T1_L, SHASH2_\pn @ (a1+a0)(b1+b0) 315 316 4: veor T1, XL, XH 317 veor XM, XM, T1 318 319 __pmull_reduce_\pn 320 321 veor T1, T1, XH 322 veor XL, XL, T1 323 324 bne 0b 325 .endm 326 327 /* 328 * void pmull_ghash_update(int blocks, u64 dg[], const char *src, 329 * struct ghash_key const *k, const char *head) 330 */ 331 ENTRY(pmull_ghash_update_p64) 332 vld1.64 {SHASH}, [r3]! 333 vld1.64 {HH}, [r3]! 334 vld1.64 {HH3-HH4}, [r3] 335 336 veor SHASH2_p64, SHASH_L, SHASH_H 337 veor SHASH2_H, HH_L, HH_H 338 veor HH34_L, HH3_L, HH3_H 339 veor HH34_H, HH4_L, HH4_H 340 341 vmov.i8 MASK, #0xe1 342 vshl.u64 MASK, MASK, #57 343 344 ghash_update p64 345 vst1.64 {XL}, [r1] 346 347 bx lr 348 ENDPROC(pmull_ghash_update_p64) 349 350 ENTRY(pmull_ghash_update_p8) 351 vld1.64 {SHASH}, [r3] 352 veor SHASH2_p8, SHASH_L, SHASH_H 353 354 vext.8 s1l, SHASH_L, SHASH_L, #1 355 vext.8 s2l, SHASH_L, SHASH_L, #2 356 vext.8 s3l, SHASH_L, SHASH_L, #3 357 vext.8 s4l, SHASH_L, SHASH_L, #4 358 vext.8 s1h, SHASH_H, SHASH_H, #1 359 vext.8 s2h, SHASH_H, SHASH_H, #2 360 vext.8 s3h, SHASH_H, SHASH_H, #3 361 vext.8 s4h, SHASH_H, SHASH_H, #4 362 363 vmov.i64 k16, #0xffff 364 vmov.i64 k32, #0xffffffff 365 vmov.i64 k48, #0xffffffffffff 366 367 ghash_update p8 368 vst1.64 {XL}, [r1] 369 370 bx lr 371 ENDPROC(pmull_ghash_update_p8) 372 373 e0 .req q9 374 e1 .req q10 375 e2 .req q11 376 e3 .req q12 377 e0l .req d18 378 e0h .req d19 379 e2l .req d22 380 e2h .req d23 381 e3l .req d24 382 e3h .req d25 383 ctr .req q13 384 ctr0 .req d26 385 ctr1 .req d27 386 387 ek0 .req q14 388 ek1 .req q15 389 390 .macro round, rk:req, regs:vararg 391 .irp r, \regs 392 aese.8 \r, \rk 393 aesmc.8 \r, \r 394 .endr 395 .endm 396 397 .macro aes_encrypt, rkp, rounds, regs:vararg 398 vld1.8 {ek0-ek1}, [\rkp, :128]! 399 cmp \rounds, #12 400 blt .L\@ // AES-128 401 402 round ek0, \regs 403 vld1.8 {ek0}, [\rkp, :128]! 404 round ek1, \regs 405 vld1.8 {ek1}, [\rkp, :128]! 406 407 beq .L\@ // AES-192 408 409 round ek0, \regs 410 vld1.8 {ek0}, [\rkp, :128]! 411 round ek1, \regs 412 vld1.8 {ek1}, [\rkp, :128]! 413 414 .L\@: .rept 4 415 round ek0, \regs 416 vld1.8 {ek0}, [\rkp, :128]! 417 round ek1, \regs 418 vld1.8 {ek1}, [\rkp, :128]! 419 .endr 420 421 round ek0, \regs 422 vld1.8 {ek0}, [\rkp, :128] 423 424 .irp r, \regs 425 aese.8 \r, ek1 426 .endr 427 .irp r, \regs 428 veor \r, \r, ek0 429 .endr 430 .endm 431 432 pmull_aes_encrypt: 433 add ip, r5, #4 434 vld1.8 {ctr0}, [r5] // load 12 byte IV 435 vld1.8 {ctr1}, [ip] 436 rev r8, r7 437 vext.8 ctr1, ctr1, ctr1, #4 438 add r7, r7, #1 439 vmov.32 ctr1[1], r8 440 vmov e0, ctr 441 442 add ip, r3, #64 443 aes_encrypt ip, r6, e0 444 bx lr 445 ENDPROC(pmull_aes_encrypt) 446 447 pmull_aes_encrypt_4x: 448 add ip, r5, #4 449 vld1.8 {ctr0}, [r5] 450 vld1.8 {ctr1}, [ip] 451 rev r8, r7 452 vext.8 ctr1, ctr1, ctr1, #4 453 add r7, r7, #1 454 vmov.32 ctr1[1], r8 455 rev ip, r7 456 vmov e0, ctr 457 add r7, r7, #1 458 vmov.32 ctr1[1], ip 459 rev r8, r7 460 vmov e1, ctr 461 add r7, r7, #1 462 vmov.32 ctr1[1], r8 463 rev ip, r7 464 vmov e2, ctr 465 add r7, r7, #1 466 vmov.32 ctr1[1], ip 467 vmov e3, ctr 468 469 add ip, r3, #64 470 aes_encrypt ip, r6, e0, e1, e2, e3 471 bx lr 472 ENDPROC(pmull_aes_encrypt_4x) 473 474 pmull_aes_encrypt_final: 475 add ip, r5, #4 476 vld1.8 {ctr0}, [r5] 477 vld1.8 {ctr1}, [ip] 478 rev r8, r7 479 vext.8 ctr1, ctr1, ctr1, #4 480 mov r7, #1 << 24 // BE #1 for the tag 481 vmov.32 ctr1[1], r8 482 vmov e0, ctr 483 vmov.32 ctr1[1], r7 484 vmov e1, ctr 485 486 add ip, r3, #64 487 aes_encrypt ip, r6, e0, e1 488 bx lr 489 ENDPROC(pmull_aes_encrypt_final) 490 491 .macro enc_1x, in0 492 bl pmull_aes_encrypt 493 veor \in0, \in0, e0 494 vst1.8 {\in0}, [r4]! 495 .endm 496 497 .macro dec_1x, in0 498 bl pmull_aes_encrypt 499 veor e0, e0, \in0 500 vst1.8 {e0}, [r4]! 501 .endm 502 503 .macro enc_4x, in0, in1, in2, in3 504 bl pmull_aes_encrypt_4x 505 506 veor \in0, \in0, e0 507 veor \in1, \in1, e1 508 veor \in2, \in2, e2 509 veor \in3, \in3, e3 510 511 vst1.8 {\in0-\in1}, [r4]! 512 vst1.8 {\in2-\in3}, [r4]! 513 .endm 514 515 .macro dec_4x, in0, in1, in2, in3 516 bl pmull_aes_encrypt_4x 517 518 veor e0, e0, \in0 519 veor e1, e1, \in1 520 veor e2, e2, \in2 521 veor e3, e3, \in3 522 523 vst1.8 {e0-e1}, [r4]! 524 vst1.8 {e2-e3}, [r4]! 525 .endm 526 527 /* 528 * void pmull_gcm_encrypt(int blocks, u64 dg[], const char *src, 529 * struct gcm_key const *k, char *dst, 530 * char *iv, int rounds, u32 counter) 531 */ 532 ENTRY(pmull_gcm_encrypt) 533 push {r4-r8, lr} 534 ldrd r4, r5, [sp, #24] 535 ldrd r6, r7, [sp, #32] 536 537 vld1.64 {SHASH}, [r3] 538 539 ghash_update p64, enc, head=0 540 vst1.64 {XL}, [r1] 541 542 pop {r4-r8, pc} 543 ENDPROC(pmull_gcm_encrypt) 544 545 /* 546 * void pmull_gcm_decrypt(int blocks, u64 dg[], const char *src, 547 * struct gcm_key const *k, char *dst, 548 * char *iv, int rounds, u32 counter) 549 */ 550 ENTRY(pmull_gcm_decrypt) 551 push {r4-r8, lr} 552 ldrd r4, r5, [sp, #24] 553 ldrd r6, r7, [sp, #32] 554 555 vld1.64 {SHASH}, [r3] 556 557 ghash_update p64, dec, head=0 558 vst1.64 {XL}, [r1] 559 560 pop {r4-r8, pc} 561 ENDPROC(pmull_gcm_decrypt) 562 563 /* 564 * void pmull_gcm_enc_final(int bytes, u64 dg[], char *tag, 565 * struct gcm_key const *k, char *head, 566 * char *iv, int rounds, u32 counter) 567 */ 568 ENTRY(pmull_gcm_enc_final) 569 push {r4-r8, lr} 570 ldrd r4, r5, [sp, #24] 571 ldrd r6, r7, [sp, #32] 572 573 bl pmull_aes_encrypt_final 574 575 cmp r0, #0 576 beq .Lenc_final 577 578 mov_l ip, .Lpermute 579 sub r4, r4, #16 580 add r8, ip, r0 581 add ip, ip, #32 582 add r4, r4, r0 583 sub ip, ip, r0 584 585 vld1.8 {e3}, [r8] // permute vector for key stream 586 vld1.8 {e2}, [ip] // permute vector for ghash input 587 588 vtbl.8 e3l, {e0}, e3l 589 vtbl.8 e3h, {e0}, e3h 590 591 vld1.8 {e0}, [r4] // encrypt tail block 592 veor e0, e0, e3 593 vst1.8 {e0}, [r4] 594 595 vtbl.8 T1_L, {e0}, e2l 596 vtbl.8 T1_H, {e0}, e2h 597 598 vld1.64 {XL}, [r1] 599 .Lenc_final: 600 vld1.64 {SHASH}, [r3, :128] 601 vmov.i8 MASK, #0xe1 602 veor SHASH2_p64, SHASH_L, SHASH_H 603 vshl.u64 MASK, MASK, #57 604 mov r0, #1 605 bne 3f // process head block first 606 ghash_update p64, aggregate=0, head=0 607 608 vrev64.8 XL, XL 609 vext.8 XL, XL, XL, #8 610 veor XL, XL, e1 611 612 sub r2, r2, #16 // rewind src pointer 613 vst1.8 {XL}, [r2] // store tag 614 615 pop {r4-r8, pc} 616 ENDPROC(pmull_gcm_enc_final) 617 618 /* 619 * int pmull_gcm_dec_final(int bytes, u64 dg[], char *tag, 620 * struct gcm_key const *k, char *head, 621 * char *iv, int rounds, u32 counter, 622 * const char *otag, int authsize) 623 */ 624 ENTRY(pmull_gcm_dec_final) 625 push {r4-r8, lr} 626 ldrd r4, r5, [sp, #24] 627 ldrd r6, r7, [sp, #32] 628 629 bl pmull_aes_encrypt_final 630 631 cmp r0, #0 632 beq .Ldec_final 633 634 mov_l ip, .Lpermute 635 sub r4, r4, #16 636 add r8, ip, r0 637 add ip, ip, #32 638 add r4, r4, r0 639 sub ip, ip, r0 640 641 vld1.8 {e3}, [r8] // permute vector for key stream 642 vld1.8 {e2}, [ip] // permute vector for ghash input 643 644 vtbl.8 e3l, {e0}, e3l 645 vtbl.8 e3h, {e0}, e3h 646 647 vld1.8 {e0}, [r4] 648 649 vtbl.8 T1_L, {e0}, e2l 650 vtbl.8 T1_H, {e0}, e2h 651 652 veor e0, e0, e3 653 vst1.8 {e0}, [r4] 654 655 vld1.64 {XL}, [r1] 656 .Ldec_final: 657 vld1.64 {SHASH}, [r3] 658 vmov.i8 MASK, #0xe1 659 veor SHASH2_p64, SHASH_L, SHASH_H 660 vshl.u64 MASK, MASK, #57 661 mov r0, #1 662 bne 3f // process head block first 663 ghash_update p64, aggregate=0, head=0 664 665 vrev64.8 XL, XL 666 vext.8 XL, XL, XL, #8 667 veor XL, XL, e1 668 669 mov_l ip, .Lpermute 670 ldrd r2, r3, [sp, #40] // otag and authsize 671 vld1.8 {T1}, [r2] 672 add ip, ip, r3 673 vceq.i8 T1, T1, XL // compare tags 674 vmvn T1, T1 // 0 for eq, -1 for ne 675 676 vld1.8 {e0}, [ip] 677 vtbl.8 XL_L, {T1}, e0l // keep authsize bytes only 678 vtbl.8 XL_H, {T1}, e0h 679 680 vpmin.s8 XL_L, XL_L, XL_H // take the minimum s8 across the vector 681 vpmin.s8 XL_L, XL_L, XL_L 682 vmov.32 r0, XL_L[0] // fail if != 0x0 683 684 pop {r4-r8, pc} 685 ENDPROC(pmull_gcm_dec_final) 686 687 .section ".rodata", "a", %progbits 688 .align 5 689 .Lpermute: 690 .byte 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff 691 .byte 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff 692 .byte 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07 693 .byte 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f 694 .byte 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff 695 .byte 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff 696