Lines Matching refs:block
27 # | CTR block 4k+8 | AES block 4k+4 | GHASH block 4k+0 |
30 # | CTR block 4k+9 | AES block 4k+5 | GHASH block 4k+1 |
33 # | CTR block 4k+10| AES block 4k+6 | GHASH block 4k+2 |
36 # | CTR block 4k+11| AES block 4k+7 | GHASH block 4k+3 |
47 # CTR block:
55 # AES block:
56 # Do AES encryption/decryption on CTR block X and EOR it with input block X. Take 256 bytes key below for example.
107 # GHASH block X:
108 # do 128b karatsuba polynomial multiplication on block
161 $input_ptr="x0"; #argument block
296 fmov $ctr1d, $ctr96_b64x @ CTR block 1
303 rev $ctr32w, $rctr32w @ CTR block 1
304 add $rctr32w, $rctr32w, #1 @ CTR block 1
305 fmov $ctr3d, $ctr96_b64x @ CTR block 3
307 orr $ctr32x, $ctr96_t32x, $ctr32x, lsl #32 @ CTR block 1
308 ld1 { $ctr0b}, [$counter] @ special case vector load initial counter so we can start first AES block as quickly as possible
310 fmov $ctr1.d[1], $ctr32x @ CTR block 1
311 rev $ctr32w, $rctr32w @ CTR block 2
313 fmov $ctr2d, $ctr96_b64x @ CTR block 2
314 orr $ctr32x, $ctr96_t32x, $ctr32x, lsl #32 @ CTR block 2
315 add $rctr32w, $rctr32w, #1 @ CTR block 2
317 fmov $ctr2.d[1], $ctr32x @ CTR block 2
318 rev $ctr32w, $rctr32w @ CTR block 3
320 orr $ctr32x, $ctr96_t32x, $ctr32x, lsl #32 @ CTR block 3
323 add $rctr32w, $rctr32w, #1 @ CTR block 3
324 fmov $ctr3.d[1], $ctr32x @ CTR block 3
330 aese $ctr1b, $rk0 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 0
333 aese $ctr2b, $rk0 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 0
339 aese $ctr0b, $rk0 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 0
342 aese $ctr3b, $rk0 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 0
345 aese $ctr2b, $rk1 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 1
348 aese $ctr0b, $rk1 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 1
351 aese $ctr1b, $rk1 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 1
354 aese $ctr3b, $rk1 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 1
357 aese $ctr0b, $rk2 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 2
360 aese $ctr1b, $rk2 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 2
366 aese $ctr3b, $rk2 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 2
368 aese $ctr2b, $rk2 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 2
371 aese $ctr0b, $rk3 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 3
373 aese $ctr1b, $rk3 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 3
375 aese $ctr2b, $rk3 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 3
378 aese $ctr3b, $rk3 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 3
383 aese $ctr3b, $rk4 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 4
386 aese $ctr2b, $rk4 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 4
389 aese $ctr0b, $rk4 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 4
391 aese $ctr3b, $rk5 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 5
393 aese $ctr2b, $rk5 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 5
395 aese $ctr0b, $rk5 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 5
397 aese $ctr3b, $rk6 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 6
399 aese $ctr1b, $rk4 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 4
401 aese $ctr2b, $rk6 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 6
404 aese $ctr0b, $rk6 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 6
406 aese $ctr1b, $rk5 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 5
408 aese $ctr3b, $rk7 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 7
410 aese $ctr0b, $rk7 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 7
412 aese $ctr1b, $rk6 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 6
414 aese $ctr2b, $rk7 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 7
416 aese $ctr0b, $rk8 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 8
418 aese $ctr1b, $rk7 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 7
420 aese $ctr2b, $rk8 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 8
422 aese $ctr3b, $rk8 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 8
424 aese $ctr1b, $rk8 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 8
426 aese $ctr2b, $rk9 @ AES block 2 - round 9
428 aese $ctr0b, $rk9 @ AES block 0 - round 9
432 aese $ctr1b, $rk9 @ AES block 1 - round 9
434 aese $ctr3b, $rk9 @ AES block 3 - round 9
437 ldp $input_l0, $input_h0, [$input_ptr, #0] @ AES block 0 - load plaintext
442 ldp $input_l2, $input_h2, [$input_ptr, #32] @ AES block 2 - load plaintext
447 ldp $input_l1, $input_h1, [$input_ptr, #16] @ AES block 1 - load plaintext
452 ldp $input_l3, $input_h3, [$input_ptr, #48] @ AES block 3 - load plaintext
457 eor $input_l0, $input_l0, $rk10_l @ AES block 0 - round 10 low
458 eor $input_h0, $input_h0, $rk10_h @ AES block 0 - round 10 high
460 eor $input_l2, $input_l2, $rk10_l @ AES block 2 - round 10 low
461 fmov $ctr_t0d, $input_l0 @ AES block 0 - mov low
463 eor $input_l1, $input_l1, $rk10_l @ AES block 1 - round 10 low
464 eor $input_h2, $input_h2, $rk10_h @ AES block 2 - round 10 high
465 fmov $ctr_t0.d[1], $input_h0 @ AES block 0 - mov high
467 fmov $ctr_t1d, $input_l1 @ AES block 1 - mov low
468 eor $input_h1, $input_h1, $rk10_h @ AES block 1 - round 10 high
470 eor $input_l3, $input_l3, $rk10_l @ AES block 3 - round 10 low
471 fmov $ctr_t1.d[1], $input_h1 @ AES block 1 - mov high
473 fmov $ctr_t2d, $input_l2 @ AES block 2 - mov low
474 eor $input_h3, $input_h3, $rk10_h @ AES block 3 - round 10 high
475 rev $ctr32w, $rctr32w @ CTR block 4
477 fmov $ctr_t2.d[1], $input_h2 @ AES block 2 - mov high
478 orr $ctr32x, $ctr96_t32x, $ctr32x, lsl #32 @ CTR block 4
480 eor $res0b, $ctr_t0b, $ctr0b @ AES block 0 - result
481 fmov $ctr0d, $ctr96_b64x @ CTR block 4
482 add $rctr32w, $rctr32w, #1 @ CTR block 4
484 fmov $ctr0.d[1], $ctr32x @ CTR block 4
485 rev $ctr32w, $rctr32w @ CTR block 5
487 eor $res1b, $ctr_t1b, $ctr1b @ AES block 1 - result
488 fmov $ctr1d, $ctr96_b64x @ CTR block 5
489 orr $ctr32x, $ctr96_t32x, $ctr32x, lsl #32 @ CTR block 5
491 add $rctr32w, $rctr32w, #1 @ CTR block 5
493 fmov $ctr1.d[1], $ctr32x @ CTR block 5
495 fmov $ctr_t3d, $input_l3 @ AES block 3 - mov low
496 rev $ctr32w, $rctr32w @ CTR block 6
497 st1 { $res0b}, [$output_ptr], #16 @ AES block 0 - store result
499 fmov $ctr_t3.d[1], $input_h3 @ AES block 3 - mov high
500 orr $ctr32x, $ctr96_t32x, $ctr32x, lsl #32 @ CTR block 6
502 add $rctr32w, $rctr32w, #1 @ CTR block 6
503 eor $res2b, $ctr_t2b, $ctr2b @ AES block 2 - result
504 st1 { $res1b}, [$output_ptr], #16 @ AES block 1 - store result
506 fmov $ctr2d, $ctr96_b64x @ CTR block 6
509 fmov $ctr2.d[1], $ctr32x @ CTR block 6
510 rev $ctr32w, $rctr32w @ CTR block 7
511 st1 { $res2b}, [$output_ptr], #16 @ AES block 2 - store result
513 orr $ctr32x, $ctr96_t32x, $ctr32x, lsl #32 @ CTR block 7
515 eor $res3b, $ctr_t3b, $ctr3b @ AES block 3 - result
516 st1 { $res3b}, [$output_ptr], #16 @ AES block 3 - store result
520 ldp $input_l3, $input_h3, [$input_ptr, #48] @ AES block 4k+3 - load plaintext
525 rev64 $res0b, $res0b @ GHASH block 4k (only t0 is free)
526 rev64 $res2b, $res2b @ GHASH block 4k+2 (t0, t1, and t2 free)
528 aese $ctr2b, $rk0 \n aesmc $ctr2b, $ctr2b @ AES block 4k+6 - round 0
529 fmov $ctr3d, $ctr96_b64x @ CTR block 4k+3
532 rev64 $res1b, $res1b @ GHASH block 4k+1 (t0 and t1 free)
534 aese $ctr1b, $rk0 \n aesmc $ctr1b, $ctr1b @ AES block 4k+5 - round 0
535 add $rctr32w, $rctr32w, #1 @ CTR block 4k+3
536 fmov $ctr3.d[1], $ctr32x @ CTR block 4k+3
538 aese $ctr0b, $rk0 \n aesmc $ctr0b, $ctr0b @ AES block 4k+4 - round 0
539 mov $t6d, $res2.d[1] @ GHASH block 4k+2 - mid
541 aese $ctr2b, $rk1 \n aesmc $ctr2b, $ctr2b @ AES block 4k+6 - round 1
542 mov $t3d, $res1.d[1] @ GHASH block 4k+1 - mid
544 aese $ctr1b, $rk1 \n aesmc $ctr1b, $ctr1b @ AES block 4k+5 - round 1
547 aese $ctr3b, $rk0 \n aesmc $ctr3b, $ctr3b @ AES block 4k+7 - round 0
548 eor $input_h3, $input_h3, $rk10_h @ AES block 4k+3 - round 10 high
550 pmull2 $t1.1q, $res1.2d, $h3.2d @ GHASH block 4k+1 - high
551 eor $t6.8b, $t6.8b, $res2.8b @ GHASH block 4k+2 - mid
552 ldp $input_l0, $input_h0, [$input_ptr, #0] @ AES block 4k+4 - load plaintext
557 aese $ctr0b, $rk1 \n aesmc $ctr0b, $ctr0b @ AES block 4k+4 - round 1
558 rev $ctr32w, $rctr32w @ CTR block 4k+8
560 eor $t3.8b, $t3.8b, $res1.8b @ GHASH block 4k+1 - mid
561 mov $t0d, $res0.d[1] @ GHASH block 4k - mid
562 orr $ctr32x, $ctr96_t32x, $ctr32x, lsl #32 @ CTR block 4k+8
564 pmull2 $acc_h.1q, $res0.2d, $h4.2d @ GHASH block 4k - high
565 add $rctr32w, $rctr32w, #1 @ CTR block 4k+8
566 mov $acc_md, $h34k.d[1] @ GHASH block 4k - mid
568 aese $ctr0b, $rk2 \n aesmc $ctr0b, $ctr0b @ AES block 4k+4 - round 2
570 pmull $acc_l.1q, $res0.1d, $h4.1d @ GHASH block 4k - low
571 eor $t0.8b, $t0.8b, $res0.8b @ GHASH block 4k - mid
573 aese $ctr1b, $rk2 \n aesmc $ctr1b, $ctr1b @ AES block 4k+5 - round 2
575 aese $ctr0b, $rk3 \n aesmc $ctr0b, $ctr0b @ AES block 4k+4 - round 3
576 eor $acc_hb, $acc_hb, $t1.16b @ GHASH block 4k+1 - high
578 pmull $t5.1q, $res2.1d, $h2.1d @ GHASH block 4k+2 - low
580 pmull $acc_m.1q, $t0.1d, $acc_m.1d @ GHASH block 4k - mid
581 rev64 $res3b, $res3b @ GHASH block 4k+3 (t0, t1, t2 and t3 free)
583 pmull $t3.1q, $t3.1d, $h34k.1d @ GHASH block 4k+1 - mid
585 pmull $t2.1q, $res1.1d, $h3.1d @ GHASH block 4k+1 - low
586 ins $t6.d[1], $t6.d[0] @ GHASH block 4k+2 - mid
588 pmull2 $t4.1q, $res2.2d, $h2.2d @ GHASH block 4k+2 - high
589 eor $input_h0, $input_h0, $rk10_h @ AES block 4k+4 - round 10 high
591 eor $acc_mb, $acc_mb, $t3.16b @ GHASH block 4k+1 - mid
592 mov $t9d, $res3.d[1] @ GHASH block 4k+3 - mid
594 aese $ctr3b, $rk1 \n aesmc $ctr3b, $ctr3b @ AES block 4k+7 - round 1
595 eor $acc_lb, $acc_lb, $t2.16b @ GHASH block 4k+1 - low
597 aese $ctr2b, $rk2 \n aesmc $ctr2b, $ctr2b @ AES block 4k+6 - round 2
598 eor $input_l0, $input_l0, $rk10_l @ AES block 4k+4 - round 10 low
600 aese $ctr1b, $rk3 \n aesmc $ctr1b, $ctr1b @ AES block 4k+5 - round 3
601 eor $t9.8b, $t9.8b, $res3.8b @ GHASH block 4k+3 - mid
603 pmull2 $t7.1q, $res3.2d, $h1.2d @ GHASH block 4k+3 - high
605 aese $ctr2b, $rk3 \n aesmc $ctr2b, $ctr2b @ AES block 4k+6 - round 3
606 eor $acc_hb, $acc_hb, $t4.16b @ GHASH block 4k+2 - high
608 pmull2 $t6.1q, $t6.2d, $h12k.2d @ GHASH block 4k+2 - mid
610 pmull $t8.1q, $res3.1d, $h1.1d @ GHASH block 4k+3 - low
613 pmull $t9.1q, $t9.1d, $h12k.1d @ GHASH block 4k+3 - mid
614 eor $acc_lb, $acc_lb, $t5.16b @ GHASH block 4k+2 - low
616 aese $ctr1b, $rk4 \n aesmc $ctr1b, $ctr1b @ AES block 4k+5 - round 4
618 aese $ctr3b, $rk2 \n aesmc $ctr3b, $ctr3b @ AES block 4k+7 - round 2
621 aese $ctr0b, $rk4 \n aesmc $ctr0b, $ctr0b @ AES block 4k+4 - round 4
622 eor $acc_hb, $acc_hb, $t7.16b @ GHASH block 4k+3 - high
624 aese $ctr1b, $rk5 \n aesmc $ctr1b, $ctr1b @ AES block 4k+5 - round 5
625 ldp $input_l1, $input_h1, [$input_ptr, #16] @ AES block 4k+5 - load plaintext
630 aese $ctr3b, $rk3 \n aesmc $ctr3b, $ctr3b @ AES block 4k+7 - round 3
631 eor $acc_mb, $acc_mb, $t6.16b @ GHASH block 4k+2 - mid
633 aese $ctr0b, $rk5 \n aesmc $ctr0b, $ctr0b @ AES block 4k+4 - round 5
634 ldp $input_l2, $input_h2, [$input_ptr, #32] @ AES block 4k+6 - load plaintext
640 eor $acc_lb, $acc_lb, $t8.16b @ GHASH block 4k+3 - low
642 aese $ctr2b, $rk4 \n aesmc $ctr2b, $ctr2b @ AES block 4k+6 - round 4
643 eor $input_l1, $input_l1, $rk10_l @ AES block 4k+5 - round 10 low
645 aese $ctr3b, $rk4 \n aesmc $ctr3b, $ctr3b @ AES block 4k+7 - round 4
646 eor $acc_mb, $acc_mb, $t9.16b @ GHASH block 4k+3 - mid
648 aese $ctr1b, $rk6 \n aesmc $ctr1b, $ctr1b @ AES block 4k+5 - round 6
649 eor $input_l3, $input_l3, $rk10_l @ AES block 4k+3 - round 10 low
651 aese $ctr2b, $rk5 \n aesmc $ctr2b, $ctr2b @ AES block 4k+6 - round 5
654 fmov $ctr_t0d, $input_l0 @ AES block 4k+4 - mov low
655 aese $ctr0b, $rk6 \n aesmc $ctr0b, $ctr0b @ AES block 4k+4 - round 6
656 fmov $ctr_t0.d[1], $input_h0 @ AES block 4k+4 - mov high
659 fmov $ctr_t3d, $input_l3 @ AES block 4k+3 - mov low
662 aese $ctr3b, $rk5 \n aesmc $ctr3b, $ctr3b @ AES block 4k+7 - round 5
663 fmov $ctr_t1d, $input_l1 @ AES block 4k+5 - mov low
665 aese $ctr0b, $rk7 \n aesmc $ctr0b, $ctr0b @ AES block 4k+4 - round 7
668 aese $ctr2b, $rk6 \n aesmc $ctr2b, $ctr2b @ AES block 4k+6 - round 6
669 eor $input_h1, $input_h1, $rk10_h @ AES block 4k+5 - round 10 high
671 aese $ctr1b, $rk7 \n aesmc $ctr1b, $ctr1b @ AES block 4k+5 - round 7
672 fmov $ctr_t1.d[1], $input_h1 @ AES block 4k+5 - mov high
674 aese $ctr0b, $rk8 \n aesmc $ctr0b, $ctr0b @ AES block 4k+4 - round 8
675 fmov $ctr_t3.d[1], $input_h3 @ AES block 4k+3 - mov high
677 aese $ctr3b, $rk6 \n aesmc $ctr3b, $ctr3b @ AES block 4k+7 - round 6
680 aese $ctr1b, $rk8 \n aesmc $ctr1b, $ctr1b @ AES block 4k+5 - round 8
683 aese $ctr0b, $rk9 @ AES block 4k+4 - round 9
684 eor $input_l2, $input_l2, $rk10_l @ AES block 4k+6 - round 10 low
685 eor $input_h2, $input_h2, $rk10_h @ AES block 4k+6 - round 10 high
687 aese $ctr3b, $rk7 \n aesmc $ctr3b, $ctr3b @ AES block 4k+7 - round 7
688 fmov $ctr_t2d, $input_l2 @ AES block 4k+6 - mov low
690 aese $ctr1b, $rk9 @ AES block 4k+5 - round 9
691 fmov $ctr_t2.d[1], $input_h2 @ AES block 4k+6 - mov high
693 aese $ctr2b, $rk7 \n aesmc $ctr2b, $ctr2b @ AES block 4k+6 - round 7
694 eor $res0b, $ctr_t0b, $ctr0b @ AES block 4k+4 - result
696 fmov $ctr0d, $ctr96_b64x @ CTR block 4k+8
697 aese $ctr3b, $rk8 \n aesmc $ctr3b, $ctr3b @ AES block 4k+7 - round 8
699 fmov $ctr0.d[1], $ctr32x @ CTR block 4k+8
700 rev $ctr32w, $rctr32w @ CTR block 4k+9
703 aese $ctr2b, $rk8 \n aesmc $ctr2b, $ctr2b @ AES block 4k+6 - round 8
704 eor $res1b, $ctr_t1b, $ctr1b @ AES block 4k+5 - result
706 add $rctr32w, $rctr32w, #1 @ CTR block 4k+9
707 orr $ctr32x, $ctr96_t32x, $ctr32x, lsl #32 @ CTR block 4k+9
708 fmov $ctr1d, $ctr96_b64x @ CTR block 4k+9
711 fmov $ctr1.d[1], $ctr32x @ CTR block 4k+9
712 rev $ctr32w, $rctr32w @ CTR block 4k+10
714 aese $ctr2b, $rk9 @ AES block 4k+6 - round 9
715 st1 { $res0b}, [$output_ptr], #16 @ AES block 4k+4 - store result
716 eor $res2b, $ctr_t2b, $ctr2b @ AES block 4k+6 - result
717 orr $ctr32x, $ctr96_t32x, $ctr32x, lsl #32 @ CTR block 4k+10
719 aese $ctr3b, $rk9 @ AES block 4k+7 - round 9
720 add $rctr32w, $rctr32w, #1 @ CTR block 4k+10
722 fmov $ctr2d, $ctr96_b64x @ CTR block 4k+10
725 st1 { $res1b}, [$output_ptr], #16 @ AES block 4k+5 - store result
727 fmov $ctr2.d[1], $ctr32x @ CTR block 4k+10
728 st1 { $res2b}, [$output_ptr], #16 @ AES block 4k+6 - store result
729 rev $ctr32w, $rctr32w @ CTR block 4k+11
731 orr $ctr32x, $ctr96_t32x, $ctr32x, lsl #32 @ CTR block 4k+11
732 eor $res3b, $ctr_t3b, $ctr3b @ AES block 4k+3 - result
735 st1 { $res3b}, [$output_ptr], #16 @ AES block 4k+3 - store result
739 rev64 $res0b, $res0b @ GHASH block 4k (only t0 is free)
740 fmov $ctr3d, $ctr96_b64x @ CTR block 4k+3
741 rev64 $res1b, $res1b @ GHASH block 4k+1 (t0 and t1 free)
744 add $rctr32w, $rctr32w, #1 @ CTR block 4k+3
745 fmov $ctr3.d[1], $ctr32x @ CTR block 4k+3
747 aese $ctr1b, $rk0 \n aesmc $ctr1b, $ctr1b @ AES block 4k+5 - round 0
748 rev64 $res2b, $res2b @ GHASH block 4k+2 (t0, t1, and t2 free)
750 pmull $t2.1q, $res1.1d, $h3.1d @ GHASH block 4k+1 - low
752 rev64 $res3b, $res3b @ GHASH block 4k+3 (t0, t1, t2 and t3 free)
755 pmull2 $t1.1q, $res1.2d, $h3.2d @ GHASH block 4k+1 - high
757 aese $ctr3b, $rk0 \n aesmc $ctr3b, $ctr3b @ AES block 4k+7 - round 0
758 mov $t3d, $res1.d[1] @ GHASH block 4k+1 - mid
760 pmull $acc_l.1q, $res0.1d, $h4.1d @ GHASH block 4k - low
761 mov $t0d, $res0.d[1] @ GHASH block 4k - mid
763 mov $t6d, $res2.d[1] @ GHASH block 4k+2 - mid
764 mov $acc_md, $h34k.d[1] @ GHASH block 4k - mid
766 aese $ctr1b, $rk1 \n aesmc $ctr1b, $ctr1b @ AES block 4k+5 - round 1
767 eor $t3.8b, $t3.8b, $res1.8b @ GHASH block 4k+1 - mid
769 eor $t0.8b, $t0.8b, $res0.8b @ GHASH block 4k - mid
771 pmull2 $acc_h.1q, $res0.2d, $h4.2d @ GHASH block 4k - high
772 eor $t6.8b, $t6.8b, $res2.8b @ GHASH block 4k+2 - mid
774 aese $ctr3b, $rk1 \n aesmc $ctr3b, $ctr3b @ AES block 4k+7 - round 1
776 pmull $t3.1q, $t3.1d, $h34k.1d @ GHASH block 4k+1 - mid
777 eor $acc_lb, $acc_lb, $t2.16b @ GHASH block 4k+1 - low
779 pmull $acc_m.1q, $t0.1d, $acc_m.1d @ GHASH block 4k - mid
781 aese $ctr0b, $rk0 \n aesmc $ctr0b, $ctr0b @ AES block 4k+4 - round 0
782 ins $t6.d[1], $t6.d[0] @ GHASH block 4k+2 - mid
784 aese $ctr2b, $rk0 \n aesmc $ctr2b, $ctr2b @ AES block 4k+6 - round 0
786 eor $acc_mb, $acc_mb, $t3.16b @ GHASH block 4k+1 - mid
787 mov $t9d, $res3.d[1] @ GHASH block 4k+3 - mid
789 aese $ctr0b, $rk1 \n aesmc $ctr0b, $ctr0b @ AES block 4k+4 - round 1
790 eor $acc_hb, $acc_hb, $t1.16b @ GHASH block 4k+1 - high
792 pmull2 $t6.1q, $t6.2d, $h12k.2d @ GHASH block 4k+2 - mid
794 pmull2 $t4.1q, $res2.2d, $h2.2d @ GHASH block 4k+2 - high
795 eor $t9.8b, $t9.8b, $res3.8b @ GHASH block 4k+3 - mid
797 pmull2 $t7.1q, $res3.2d, $h1.2d @ GHASH block 4k+3 - high
799 pmull $t5.1q, $res2.1d, $h2.1d @ GHASH block 4k+2 - low
801 aese $ctr2b, $rk1 \n aesmc $ctr2b, $ctr2b @ AES block 4k+6 - round 1
802 eor $acc_hb, $acc_hb, $t4.16b @ GHASH block 4k+2 - high
804 aese $ctr0b, $rk2 \n aesmc $ctr0b, $ctr0b @ AES block 4k+4 - round 2
806 pmull $t8.1q, $res3.1d, $h1.1d @ GHASH block 4k+3 - low
809 aese $ctr2b, $rk2 \n aesmc $ctr2b, $ctr2b @ AES block 4k+6 - round 2
810 eor $acc_lb, $acc_lb, $t5.16b @ GHASH block 4k+2 - low
812 aese $ctr3b, $rk2 \n aesmc $ctr3b, $ctr3b @ AES block 4k+7 - round 2
814 pmull $t9.1q, $t9.1d, $h12k.1d @ GHASH block 4k+3 - mid
815 eor $acc_mb, $acc_mb, $t6.16b @ GHASH block 4k+2 - mid
817 aese $ctr2b, $rk3 \n aesmc $ctr2b, $ctr2b @ AES block 4k+6 - round 3
819 aese $ctr1b, $rk2 \n aesmc $ctr1b, $ctr1b @ AES block 4k+5 - round 2
820 eor $acc_hb, $acc_hb, $t7.16b @ GHASH block 4k+3 - high
822 aese $ctr0b, $rk3 \n aesmc $ctr0b, $ctr0b @ AES block 4k+4 - round 3
824 eor $acc_mb, $acc_mb, $t9.16b @ GHASH block 4k+3 - mid
827 aese $ctr1b, $rk3 \n aesmc $ctr1b, $ctr1b @ AES block 4k+5 - round 3
828 eor $acc_lb, $acc_lb, $t8.16b @ GHASH block 4k+3 - low
830 aese $ctr0b, $rk4 \n aesmc $ctr0b, $ctr0b @ AES block 4k+4 - round 4
835 aese $ctr1b, $rk4 \n aesmc $ctr1b, $ctr1b @ AES block 4k+5 - round 4
837 aese $ctr0b, $rk5 \n aesmc $ctr0b, $ctr0b @ AES block 4k+4 - round 5
840 aese $ctr3b, $rk3 \n aesmc $ctr3b, $ctr3b @ AES block 4k+7 - round 3
842 aese $ctr2b, $rk4 \n aesmc $ctr2b, $ctr2b @ AES block 4k+6 - round 4
845 aese $ctr0b, $rk6 \n aesmc $ctr0b, $ctr0b @ AES block 4k+4 - round 6
847 aese $ctr3b, $rk4 \n aesmc $ctr3b, $ctr3b @ AES block 4k+7 - round 4
849 aese $ctr1b, $rk5 \n aesmc $ctr1b, $ctr1b @ AES block 4k+5 - round 5
851 aese $ctr2b, $rk5 \n aesmc $ctr2b, $ctr2b @ AES block 4k+6 - round 5
854 aese $ctr3b, $rk5 \n aesmc $ctr3b, $ctr3b @ AES block 4k+7 - round 5
856 aese $ctr1b, $rk6 \n aesmc $ctr1b, $ctr1b @ AES block 4k+5 - round 6
858 aese $ctr2b, $rk6 \n aesmc $ctr2b, $ctr2b @ AES block 4k+6 - round 6
860 aese $ctr3b, $rk6 \n aesmc $ctr3b, $ctr3b @ AES block 4k+7 - round 6
863 aese $ctr0b, $rk7 \n aesmc $ctr0b, $ctr0b @ AES block 4k+4 - round 7
865 aese $ctr2b, $rk7 \n aesmc $ctr2b, $ctr2b @ AES block 4k+6 - round 7
867 aese $ctr3b, $rk7 \n aesmc $ctr3b, $ctr3b @ AES block 4k+7 - round 7
871 aese $ctr1b, $rk7 \n aesmc $ctr1b, $ctr1b @ AES block 4k+5 - round 7
874 aese $ctr3b, $rk8 \n aesmc $ctr3b, $ctr3b @ AES block 4k+7 - round 8
876 aese $ctr0b, $rk8 \n aesmc $ctr0b, $ctr0b @ AES block 4k+4 - round 8
879 aese $ctr1b, $rk8 \n aesmc $ctr1b, $ctr1b @ AES block 4k+5 - round 8
881 aese $ctr3b, $rk9 @ AES block 4k+7 - round 9
883 aese $ctr2b, $rk8 \n aesmc $ctr2b, $ctr2b @ AES block 4k+6 - round 8
885 aese $ctr0b, $rk9 @ AES block 4k+4 - round 9
887 aese $ctr1b, $rk9 @ AES block 4k+5 - round 9
890 aese $ctr2b, $rk9 @ AES block 4k+6 - round 9
894 ldp $input_l0, $input_h0, [$input_ptr], #16 @ AES block 4k+4 - load plaintext
902 eor $input_l0, $input_l0, $rk10_l @ AES block 4k+4 - round 10 low
903 eor $input_h0, $input_h0, $rk10_h @ AES block 4k+4 - round 10 high
905 fmov $ctr_t0d, $input_l0 @ AES block 4k+4 - mov low
907 fmov $ctr_t0.d[1], $input_h0 @ AES block 4k+4 - mov high
909 eor $res1b, $ctr_t0b, $ctr0b @ AES block 4k+4 - result
933 st1 { $res1b}, [$output_ptr], #16 @ AES final-3 block - store result
935 ldp $input_l0, $input_h0, [$input_ptr], #16 @ AES final-2 block - load input low & high
940 rev64 $res0b, $res1b @ GHASH final-3 block
943 eor $input_h0, $input_h0, $rk10_h @ AES final-2 block - round 10 high
944 eor $input_l0, $input_l0, $rk10_l @ AES final-2 block - round 10 low
946 fmov $res1d, $input_l0 @ AES final-2 block - mov low
949 fmov $res1.d[1], $input_h0 @ AES final-2 block - mov high
951 pmull $acc_l.1q, $res0.1d, $h4.1d @ GHASH final-3 block - low
952 mov $rk4d, $res0.d[1] @ GHASH final-3 block - mid
954 pmull2 $acc_h.1q, $res0.2d, $h4.2d @ GHASH final-3 block - high
956 mov $acc_md, $h34k.d[1] @ GHASH final-3 block - mid
958 eor $res1b, $res1b, $ctr1b @ AES final-2 block - result
959 eor $rk4v.8b, $rk4v.8b, $res0.8b @ GHASH final-3 block - mid
961 pmull $acc_m.1q, $rk4v.1d, $acc_m.1d @ GHASH final-3 block - mid
964 st1 { $res1b}, [$output_ptr], #16 @ AES final-2 block - store result
966 rev64 $res0b, $res1b @ GHASH final-2 block
967 ldp $input_l0, $input_h0, [$input_ptr], #16 @ AES final-1 block - load input low & high
974 eor $input_l0, $input_l0, $rk10_l @ AES final-1 block - round 10 low
976 fmov $res1d, $input_l0 @ AES final-1 block - mov low
977 eor $input_h0, $input_h0, $rk10_h @ AES final-1 block - round 10 high
979 pmull2 $rk2q1, $res0.2d, $h3.2d @ GHASH final-2 block - high
980 fmov $res1.d[1], $input_h0 @ AES final-1 block - mov high
982 mov $rk4d, $res0.d[1] @ GHASH final-2 block - mid
984 pmull $rk3q1, $res0.1d, $h3.1d @ GHASH final-2 block - low
986 eor $acc_hb, $acc_hb, $rk2 @ GHASH final-2 block - high
988 eor $rk4v.8b, $rk4v.8b, $res0.8b @ GHASH final-2 block - mid
990 eor $res1b, $res1b, $ctr2b @ AES final-1 block - result
992 eor $acc_lb, $acc_lb, $rk3 @ GHASH final-2 block - low
994 pmull $rk4v.1q, $rk4v.1d, $h34k.1d @ GHASH final-2 block - mid
998 eor $acc_mb, $acc_mb, $rk4v.16b @ GHASH final-2 block - mid
1001 st1 { $res1b}, [$output_ptr], #16 @ AES final-1 block - store result
1003 rev64 $res0b, $res1b @ GHASH final-1 block
1004 ldp $input_l0, $input_h0, [$input_ptr], #16 @ AES final block - load input low & high
1011 eor $input_h0, $input_h0, $rk10_h @ AES final block - round 10 high
1012 eor $input_l0, $input_l0, $rk10_l @ AES final block - round 10 low
1014 fmov $res1d, $input_l0 @ AES final block - mov low
1016 pmull2 $rk2q1, $res0.2d, $h2.2d @ GHASH final-1 block - high
1017 fmov $res1.d[1], $input_h0 @ AES final block - mov high
1019 mov $rk4d, $res0.d[1] @ GHASH final-1 block - mid
1021 pmull $rk3q1, $res0.1d, $h2.1d @ GHASH final-1 block - low
1023 eor $rk4v.8b, $rk4v.8b, $res0.8b @ GHASH final-1 block - mid
1025 eor $res1b, $res1b, $ctr3b @ AES final block - result
1027 ins $rk4v.d[1], $rk4v.d[0] @ GHASH final-1 block - mid
1029 pmull2 $rk4v.1q, $rk4v.2d, $h12k.2d @ GHASH final-1 block - mid
1031 eor $acc_lb, $acc_lb, $rk3 @ GHASH final-1 block - low
1033 eor $acc_hb, $acc_hb, $rk2 @ GHASH final-1 block - high
1035 eor $acc_mb, $acc_mb, $rk4v.16b @ GHASH final-1 block - mid
1049 lsr $rk10_h, $rk10_h, $bit_length @ rk10_h is mask for top 64b of last block
1055 fmov $ctr0d, $input_l0 @ ctr0b is mask for last block
1059 and $res1b, $res1b, $ctr0b @ possibly partial last block has zeroes in highest bits
1061 rev64 $res0b, $res1b @ GHASH final block
1065 mov $t0d, $res0.d[1] @ GHASH final block - mid
1067 pmull $rk3q1, $res0.1d, $h1.1d @ GHASH final block - low
1068 ld1 { $rk0}, [$output_ptr] @ load existing bytes where the possibly partial last block is to be stored
1070 eor $t0.8b, $t0.8b, $res0.8b @ GHASH final block - mid
1076 pmull2 $rk2q1, $res0.2d, $h1.2d @ GHASH final block - high
1078 pmull $t0.1q, $t0.1d, $h12k.1d @ GHASH final block - mid
1080 eor $acc_lb, $acc_lb, $rk3 @ GHASH final block - low
1082 eor $acc_hb, $acc_hb, $rk2 @ GHASH final block - high
1084 eor $acc_mb, $acc_mb, $t0.16b @ GHASH final block - mid
1172 ld1 { $ctr0b}, [$counter] @ special case vector load initial counter so we can start first AES block as quickly as possible
1179 fmov $ctr2d, $ctr96_b64x @ CTR block 2
1185 fmov $ctr1d, $ctr96_b64x @ CTR block 1
1188 aese $ctr0b, $rk0 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 0
1189 rev $ctr32w, $rctr32w @ CTR block 1
1191 orr $ctr32x, $ctr96_t32x, $ctr32x, lsl #32 @ CTR block 1
1193 add $rctr32w, $rctr32w, #1 @ CTR block 1
1195 fmov $ctr1.d[1], $ctr32x @ CTR block 1
1196 rev $ctr32w, $rctr32w @ CTR block 2
1197 add $rctr32w, $rctr32w, #1 @ CTR block 2
1199 aese $ctr0b, $rk1 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 1
1200 orr $ctr32x, $ctr96_t32x, $ctr32x, lsl #32 @ CTR block 2
1202 fmov $ctr2.d[1], $ctr32x @ CTR block 2
1203 rev $ctr32w, $rctr32w @ CTR block 3
1205 fmov $ctr3d, $ctr96_b64x @ CTR block 3
1206 orr $ctr32x, $ctr96_t32x, $ctr32x, lsl #32 @ CTR block 3
1207 add $rctr32w, $rctr32w, #1 @ CTR block 3
1209 fmov $ctr3.d[1], $ctr32x @ CTR block 3
1212 aese $ctr1b, $rk0 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 0
1215 aese $ctr0b, $rk2 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 2
1218 aese $ctr2b, $rk0 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 0
1221 aese $ctr1b, $rk1 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 1
1224 aese $ctr3b, $rk0 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 0
1226 aese $ctr2b, $rk1 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 1
1228 aese $ctr1b, $rk2 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 2
1230 aese $ctr3b, $rk1 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 1
1235 aese $ctr0b, $rk3 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 3
1238 aese $ctr1b, $rk3 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 3
1240 aese $ctr3b, $rk2 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 2
1242 aese $ctr2b, $rk2 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 2
1245 aese $ctr1b, $rk4 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 4
1247 aese $ctr3b, $rk3 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 3
1249 aese $ctr2b, $rk3 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 3
1254 aese $ctr0b, $rk4 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 4
1257 aese $ctr1b, $rk5 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 5
1259 aese $ctr2b, $rk4 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 4
1261 aese $ctr3b, $rk4 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 4
1263 aese $ctr0b, $rk5 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 5
1265 aese $ctr2b, $rk5 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 5
1270 aese $ctr3b, $rk5 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 5
1272 aese $ctr0b, $rk6 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 6
1274 aese $ctr1b, $rk6 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 6
1276 aese $ctr3b, $rk6 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 6
1278 aese $ctr2b, $rk6 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 6
1288 aese $ctr1b, $rk7 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 7
1290 aese $ctr2b, $rk7 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 7
1292 aese $ctr0b, $rk7 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 7
1295 aese $ctr3b, $rk7 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 7
1297 aese $ctr1b, $rk8 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 8
1300 aese $ctr2b, $rk8 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 8
1302 aese $ctr3b, $rk8 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 8
1304 aese $ctr0b, $rk8 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 8
1307 aese $ctr2b, $rk9 @ AES block 2 - round 9
1309 aese $ctr3b, $rk9 @ AES block 3 - round 9
1311 aese $ctr0b, $rk9 @ AES block 0 - round 9
1314 aese $ctr1b, $rk9 @ AES block 1 - round 9
1318 ld1 {$res0b, $res1b}, [$input_ptr], #32 @ AES block 0 - load ciphertext; AES block 1 - load ciphertext
1320 eor $ctr1b, $res1b, $ctr1b @ AES block 1 - result
1321 ld1 {$res2b}, [$input_ptr], #16 @ AES block 2 - load ciphertext
1323 eor $ctr0b, $res0b, $ctr0b @ AES block 0 - result
1324 rev64 $res0b, $res0b @ GHASH block 0
1325 rev $ctr32w, $rctr32w @ CTR block 4
1327 orr $ctr32x, $ctr96_t32x, $ctr32x, lsl #32 @ CTR block 4
1328 add $rctr32w, $rctr32w, #1 @ CTR block 4
1329 ld1 {$res3b}, [$input_ptr], #16 @ AES block 3 - load ciphertext
1331 rev64 $res1b, $res1b @ GHASH block 1
1332 mov $output_l1, $ctr1.d[0] @ AES block 1 - mov low
1334 mov $output_h1, $ctr1.d[1] @ AES block 1 - mov high
1336 mov $output_l0, $ctr0.d[0] @ AES block 0 - mov low
1339 mov $output_h0, $ctr0.d[1] @ AES block 0 - mov high
1341 fmov $ctr0d, $ctr96_b64x @ CTR block 4
1343 fmov $ctr0.d[1], $ctr32x @ CTR block 4
1344 rev $ctr32w, $rctr32w @ CTR block 5
1345 eor $output_l1, $output_l1, $rk10_l @ AES block 1 - round 10 low
1349 fmov $ctr1d, $ctr96_b64x @ CTR block 5
1350 add $rctr32w, $rctr32w, #1 @ CTR block 5
1351 orr $ctr32x, $ctr96_t32x, $ctr32x, lsl #32 @ CTR block 5
1353 fmov $ctr1.d[1], $ctr32x @ CTR block 5
1354 rev $ctr32w, $rctr32w @ CTR block 6
1355 add $rctr32w, $rctr32w, #1 @ CTR block 6
1357 orr $ctr32x, $ctr96_t32x, $ctr32x, lsl #32 @ CTR block 6
1359 eor $output_h1, $output_h1, $rk10_h @ AES block 1 - round 10 high
1363 eor $output_l0, $output_l0, $rk10_l @ AES block 0 - round 10 low
1367 eor $ctr2b, $res2b, $ctr2b @ AES block 2 - result
1369 eor $output_h0, $output_h0, $rk10_h @ AES block 0 - round 10 high
1373 stp $output_l0, $output_h0, [$output_ptr], #16 @ AES block 0 - store result
1375 stp $output_l1, $output_h1, [$output_ptr], #16 @ AES block 1 - store result
1379 eor $ctr3b, $res3b, $ctr3b @ AES block 4k+3 - result
1381 mov $output_l2, $ctr2.d[0] @ AES block 4k+2 - mov low
1383 pmull2 $t1.1q, $res1.2d, $h3.2d @ GHASH block 4k+1 - high
1384 mov $output_h2, $ctr2.d[1] @ AES block 4k+2 - mov high
1386 aese $ctr1b, $rk0 \n aesmc $ctr1b, $ctr1b @ AES block 4k+5 - round 0
1387 fmov $ctr2d, $ctr96_b64x @ CTR block 4k+6
1389 rev64 $res2b, $res2b @ GHASH block 4k+2
1390 fmov $ctr2.d[1], $ctr32x @ CTR block 4k+6
1391 rev $ctr32w, $rctr32w @ CTR block 4k+7
1393 mov $output_l3, $ctr3.d[0] @ AES block 4k+3 - mov low
1395 mov $t3d, $res1.d[1] @ GHASH block 4k+1 - mid
1397 aese $ctr1b, $rk1 \n aesmc $ctr1b, $ctr1b @ AES block 4k+5 - round 1
1398 rev64 $res3b, $res3b @ GHASH block 4k+3
1400 pmull $t2.1q, $res1.1d, $h3.1d @ GHASH block 4k+1 - low
1401 mov $output_h3, $ctr3.d[1] @ AES block 4k+3 - mov high
1402 orr $ctr32x, $ctr96_t32x, $ctr32x, lsl #32 @ CTR block 4k+7
1404 pmull $acc_l.1q, $res0.1d, $h4.1d @ GHASH block 4k - low
1405 fmov $ctr3d, $ctr96_b64x @ CTR block 4k+7
1406 eor $t3.8b, $t3.8b, $res1.8b @ GHASH block 4k+1 - mid
1408 aese $ctr1b, $rk2 \n aesmc $ctr1b, $ctr1b @ AES block 4k+5 - round 2
1409 fmov $ctr3.d[1], $ctr32x @ CTR block 4k+7
1411 aese $ctr2b, $rk0 \n aesmc $ctr2b, $ctr2b @ AES block 4k+6 - round 0
1412 mov $acc_md, $h34k.d[1] @ GHASH block 4k - mid
1414 pmull2 $acc_h.1q, $res0.2d, $h4.2d @ GHASH block 4k - high
1415 eor $acc_lb, $acc_lb, $t2.16b @ GHASH block 4k+1 - low
1417 pmull $t8.1q, $res3.1d, $h1.1d @ GHASH block 4k+3 - low
1419 aese $ctr1b, $rk3 \n aesmc $ctr1b, $ctr1b @ AES block 4k+5 - round 3
1420 mov $t0d, $res0.d[1] @ GHASH block 4k - mid
1422 aese $ctr3b, $rk0 \n aesmc $ctr3b, $ctr3b @ AES block 4k+7 - round 0
1423 eor $acc_hb, $acc_hb, $t1.16b @ GHASH block 4k+1 - high
1425 aese $ctr0b, $rk0 \n aesmc $ctr0b, $ctr0b @ AES block 4k+4 - round 0
1427 pmull $t5.1q, $res2.1d, $h2.1d @ GHASH block 4k+2 - low
1428 eor $t0.8b, $t0.8b, $res0.8b @ GHASH block 4k - mid
1430 aese $ctr3b, $rk1 \n aesmc $ctr3b, $ctr3b @ AES block 4k+7 - round 1
1431 eor $output_l3, $output_l3, $rk10_l @ AES block 4k+3 - round 10 low
1435 pmull $t3.1q, $t3.1d, $h34k.1d @ GHASH block 4k+1 - mid
1436 eor $output_h2, $output_h2, $rk10_h @ AES block 4k+2 - round 10 high
1440 mov $t6d, $res2.d[1] @ GHASH block 4k+2 - mid
1442 aese $ctr0b, $rk1 \n aesmc $ctr0b, $ctr0b @ AES block 4k+4 - round 1
1443 eor $acc_lb, $acc_lb, $t5.16b @ GHASH block 4k+2 - low
1445 pmull $acc_m.1q, $t0.1d, $acc_m.1d @ GHASH block 4k - mid
1447 aese $ctr3b, $rk2 \n aesmc $ctr3b, $ctr3b @ AES block 4k+7 - round 2
1448 eor $t6.8b, $t6.8b, $res2.8b @ GHASH block 4k+2 - mid
1450 aese $ctr0b, $rk2 \n aesmc $ctr0b, $ctr0b @ AES block 4k+4 - round 2
1452 aese $ctr1b, $rk4 \n aesmc $ctr1b, $ctr1b @ AES block 4k+5 - round 4
1453 eor $acc_mb, $acc_mb, $t3.16b @ GHASH block 4k+1 - mid
1455 pmull2 $t4.1q, $res2.2d, $h2.2d @ GHASH block 4k+2 - high
1457 aese $ctr0b, $rk3 \n aesmc $ctr0b, $ctr0b @ AES block 4k+4 - round 3
1458 ins $t6.d[1], $t6.d[0] @ GHASH block 4k+2 - mid
1460 pmull2 $t7.1q, $res3.2d, $h1.2d @ GHASH block 4k+3 - high
1462 aese $ctr2b, $rk1 \n aesmc $ctr2b, $ctr2b @ AES block 4k+6 - round 1
1463 mov $t9d, $res3.d[1] @ GHASH block 4k+3 - mid
1465 aese $ctr0b, $rk4 \n aesmc $ctr0b, $ctr0b @ AES block 4k+4 - round 4
1466 eor $acc_hb, $acc_hb, $t4.16b @ GHASH block 4k+2 - high
1468 pmull2 $t6.1q, $t6.2d, $h12k.2d @ GHASH block 4k+2 - mid
1469 eor $output_h3, $output_h3, $rk10_h @ AES block 4k+3 - round 10 high
1473 aese $ctr2b, $rk2 \n aesmc $ctr2b, $ctr2b @ AES block 4k+6 - round 2
1474 eor $t9.8b, $t9.8b, $res3.8b @ GHASH block 4k+3 - mid
1476 aese $ctr1b, $rk5 \n aesmc $ctr1b, $ctr1b @ AES block 4k+5 - round 5
1477 eor $output_l2, $output_l2, $rk10_l @ AES block 4k+2 - round 10 low
1481 aese $ctr0b, $rk5 \n aesmc $ctr0b, $ctr0b @ AES block 4k+4 - round 5
1484 aese $ctr2b, $rk3 \n aesmc $ctr2b, $ctr2b @ AES block 4k+6 - round 3
1485 eor $acc_lb, $acc_lb, $t8.16b @ GHASH block 4k+3 - low
1487 aese $ctr1b, $rk6 \n aesmc $ctr1b, $ctr1b @ AES block 4k+5 - round 6
1489 aese $ctr0b, $rk6 \n aesmc $ctr0b, $ctr0b @ AES block 4k+4 - round 6
1490 eor $acc_mb, $acc_mb, $t6.16b @ GHASH block 4k+2 - mid
1492 aese $ctr2b, $rk4 \n aesmc $ctr2b, $ctr2b @ AES block 4k+6 - round 4
1493 stp $output_l2, $output_h2, [$output_ptr], #16 @ AES block 4k+2 - store result
1495 pmull $t9.1q, $t9.1d, $h12k.1d @ GHASH block 4k+3 - mid
1496 eor $acc_hb, $acc_hb, $t7.16b @ GHASH block 4k+3 - high
1497 ld1 {$res0b}, [$input_ptr], #16 @ AES block 4k+3 - load ciphertext
1499 aese $ctr1b, $rk7 \n aesmc $ctr1b, $ctr1b @ AES block 4k+5 - round 7
1500 add $rctr32w, $rctr32w, #1 @ CTR block 4k+7
1502 aese $ctr0b, $rk7 \n aesmc $ctr0b, $ctr0b @ AES block 4k+4 - round 7
1505 aese $ctr2b, $rk5 \n aesmc $ctr2b, $ctr2b @ AES block 4k+6 - round 5
1506 eor $acc_mb, $acc_mb, $t9.16b @ GHASH block 4k+3 - mid
1508 aese $ctr1b, $rk8 \n aesmc $ctr1b, $ctr1b @ AES block 4k+5 - round 8
1509 stp $output_l3, $output_h3, [$output_ptr], #16 @ AES block 4k+3 - store result
1511 aese $ctr0b, $rk8 \n aesmc $ctr0b, $ctr0b @ AES block 4k+4 - round 8
1514 aese $ctr3b, $rk3 \n aesmc $ctr3b, $ctr3b @ AES block 4k+7 - round 3
1515 rev $ctr32w, $rctr32w @ CTR block 4k+8
1518 ld1 {$res1b}, [$input_ptr], #16 @ AES block 4k+4 - load ciphertext
1521 aese $ctr0b, $rk9 @ AES block 4k+4 - round 9
1522 orr $ctr32x, $ctr96_t32x, $ctr32x, lsl #32 @ CTR block 4k+8
1524 aese $ctr3b, $rk4 \n aesmc $ctr3b, $ctr3b @ AES block 4k+7 - round 4
1527 aese $ctr1b, $rk9 @ AES block 4k+5 - round 9
1529 aese $ctr2b, $rk6 \n aesmc $ctr2b, $ctr2b @ AES block 4k+6 - round 6
1530 eor $ctr0b, $res0b, $ctr0b @ AES block 4k+4 - result
1532 aese $ctr3b, $rk5 \n aesmc $ctr3b, $ctr3b @ AES block 4k+7 - round 5
1533 ld1 {$res2b}, [$input_ptr], #16 @ AES block 4k+5 - load ciphertext
1535 add $rctr32w, $rctr32w, #1 @ CTR block 4k+8
1537 eor $ctr1b, $res1b, $ctr1b @ AES block 4k+5 - result
1539 aese $ctr2b, $rk7 \n aesmc $ctr2b, $ctr2b @ AES block 4k+6 - round 7
1540 ld1 {$res3b}, [$input_ptr], #16 @ AES block 4k+6 - load ciphertext
1542 aese $ctr3b, $rk6 \n aesmc $ctr3b, $ctr3b @ AES block 4k+7 - round 6
1544 rev64 $res1b, $res1b @ GHASH block 4k+5
1546 mov $output_h0, $ctr0.d[1] @ AES block 4k+4 - mov high
1548 aese $ctr2b, $rk8 \n aesmc $ctr2b, $ctr2b @ AES block 4k+6 - round 8
1549 mov $output_l0, $ctr0.d[0] @ AES block 4k+4 - mov low
1551 aese $ctr3b, $rk7 \n aesmc $ctr3b, $ctr3b @ AES block 4k+7 - round 7
1552 fmov $ctr0d, $ctr96_b64x @ CTR block 4k+8
1555 fmov $ctr0.d[1], $ctr32x @ CTR block 4k+8
1556 rev $ctr32w, $rctr32w @ CTR block 4k+9
1558 aese $ctr2b, $rk9 @ AES block 4k+6 - round 9
1559 orr $ctr32x, $ctr96_t32x, $ctr32x, lsl #32 @ CTR block 4k+9
1562 aese $ctr3b, $rk8 \n aesmc $ctr3b, $ctr3b @ AES block 4k+7 - round 8
1563 eor $output_h0, $output_h0, $rk10_h @ AES block 4k+4 - round 10 high
1568 mov $output_h1, $ctr1.d[1] @ AES block 4k+5 - mov high
1569 eor $output_l0, $output_l0, $rk10_l @ AES block 4k+4 - round 10 low
1573 eor $ctr2b, $res2b, $ctr2b @ AES block 4k+6 - result
1574 mov $output_l1, $ctr1.d[0] @ AES block 4k+5 - mov low
1575 add $rctr32w, $rctr32w, #1 @ CTR block 4k+9
1577 aese $ctr3b, $rk9 @ AES block 4k+7 - round 9
1578 fmov $ctr1d, $ctr96_b64x @ CTR block 4k+9
1581 rev64 $res0b, $res0b @ GHASH block 4k+4
1583 fmov $ctr1.d[1], $ctr32x @ CTR block 4k+9
1585 rev $ctr32w, $rctr32w @ CTR block 4k+10
1586 add $rctr32w, $rctr32w, #1 @ CTR block 4k+10
1588 eor $output_h1, $output_h1, $rk10_h @ AES block 4k+5 - round 10 high
1592 stp $output_l0, $output_h0, [$output_ptr], #16 @ AES block 4k+4 - store result
1594 eor $output_l1, $output_l1, $rk10_l @ AES block 4k+5 - round 10 low
1598 stp $output_l1, $output_h1, [$output_ptr], #16 @ AES block 4k+5 - store result
1600 orr $ctr32x, $ctr96_t32x, $ctr32x, lsl #32 @ CTR block 4k+10
1605 mov $output_l2, $ctr2.d[0] @ AES block 4k+2 - mov low
1606 mov $t3d, $res1.d[1] @ GHASH block 4k+1 - mid
1608 aese $ctr0b, $rk0 \n aesmc $ctr0b, $ctr0b @ AES block 4k+4 - round 0
1609 eor $ctr3b, $res3b, $ctr3b @ AES block 4k+3 - result
1611 aese $ctr1b, $rk0 \n aesmc $ctr1b, $ctr1b @ AES block 4k+5 - round 0
1612 mov $output_h2, $ctr2.d[1] @ AES block 4k+2 - mov high
1615 fmov $ctr2d, $ctr96_b64x @ CTR block 4k+6
1616 rev64 $res2b, $res2b @ GHASH block 4k+2
1618 aese $ctr0b, $rk1 \n aesmc $ctr0b, $ctr0b @ AES block 4k+4 - round 1
1619 fmov $ctr2.d[1], $ctr32x @ CTR block 4k+6
1621 rev $ctr32w, $rctr32w @ CTR block 4k+7
1622 mov $output_l3, $ctr3.d[0] @ AES block 4k+3 - mov low
1623 eor $t3.8b, $t3.8b, $res1.8b @ GHASH block 4k+1 - mid
1625 pmull $acc_l.1q, $res0.1d, $h4.1d @ GHASH block 4k - low
1626 mov $acc_md, $h34k.d[1] @ GHASH block 4k - mid
1627 mov $output_h3, $ctr3.d[1] @ AES block 4k+3 - mov high
1629 aese $ctr1b, $rk1 \n aesmc $ctr1b, $ctr1b @ AES block 4k+5 - round 1
1630 mov $t6d, $res2.d[1] @ GHASH block 4k+2 - mid
1632 aese $ctr0b, $rk2 \n aesmc $ctr0b, $ctr0b @ AES block 4k+4 - round 2
1633 orr $ctr32x, $ctr96_t32x, $ctr32x, lsl #32 @ CTR block 4k+7
1635 pmull $t2.1q, $res1.1d, $h3.1d @ GHASH block 4k+1 - low
1636 mov $t0d, $res0.d[1] @ GHASH block 4k - mid
1637 fmov $ctr3d, $ctr96_b64x @ CTR block 4k+7
1639 aese $ctr2b, $rk0 \n aesmc $ctr2b, $ctr2b @ AES block 4k+6 - round 0
1640 fmov $ctr3.d[1], $ctr32x @ CTR block 4k+7
1642 pmull $t3.1q, $t3.1d, $h34k.1d @ GHASH block 4k+1 - mid
1643 eor $t6.8b, $t6.8b, $res2.8b @ GHASH block 4k+2 - mid
1645 rev64 $res3b, $res3b @ GHASH block 4k+3
1647 aese $ctr2b, $rk1 \n aesmc $ctr2b, $ctr2b @ AES block 4k+6 - round 1
1648 eor $t0.8b, $t0.8b, $res0.8b @ GHASH block 4k - mid
1650 pmull2 $acc_h.1q, $res0.2d, $h4.2d @ GHASH block 4k - high
1652 aese $ctr3b, $rk0 \n aesmc $ctr3b, $ctr3b @ AES block 4k+7 - round 0
1653 ins $t6.d[1], $t6.d[0] @ GHASH block 4k+2 - mid
1655 pmull2 $t1.1q, $res1.2d, $h3.2d @ GHASH block 4k+1 - high
1657 pmull $acc_m.1q, $t0.1d, $acc_m.1d @ GHASH block 4k - mid
1658 eor $acc_lb, $acc_lb, $t2.16b @ GHASH block 4k+1 - low
1660 pmull $t8.1q, $res3.1d, $h1.1d @ GHASH block 4k+3 - low
1662 pmull2 $t6.1q, $t6.2d, $h12k.2d @ GHASH block 4k+2 - mid
1663 eor $acc_hb, $acc_hb, $t1.16b @ GHASH block 4k+1 - high
1665 eor $acc_mb, $acc_mb, $t3.16b @ GHASH block 4k+1 - mid
1667 pmull2 $t7.1q, $res3.2d, $h1.2d @ GHASH block 4k+3 - high
1669 pmull2 $t4.1q, $res2.2d, $h2.2d @ GHASH block 4k+2 - high
1670 mov $t9d, $res3.d[1] @ GHASH block 4k+3 - mid
1672 aese $ctr1b, $rk2 \n aesmc $ctr1b, $ctr1b @ AES block 4k+5 - round 2
1673 eor $acc_mb, $acc_mb, $t6.16b @ GHASH block 4k+2 - mid
1675 pmull $t5.1q, $res2.1d, $h2.1d @ GHASH block 4k+2 - low
1677 eor $acc_hb, $acc_hb, $t4.16b @ GHASH block 4k+2 - high
1680 aese $ctr3b, $rk1 \n aesmc $ctr3b, $ctr3b @ AES block 4k+7 - round 1
1681 eor $t9.8b, $t9.8b, $res3.8b @ GHASH block 4k+3 - mid
1683 eor $acc_lb, $acc_lb, $t5.16b @ GHASH block 4k+2 - low
1685 aese $ctr2b, $rk2 \n aesmc $ctr2b, $ctr2b @ AES block 4k+6 - round 2
1686 eor $acc_hb, $acc_hb, $t7.16b @ GHASH block 4k+3 - high
1688 aese $ctr3b, $rk2 \n aesmc $ctr3b, $ctr3b @ AES block 4k+7 - round 2
1689 eor $output_l3, $output_l3, $rk10_l @ AES block 4k+3 - round 10 low
1693 pmull $t9.1q, $t9.1d, $h12k.1d @ GHASH block 4k+3 - mid
1694 eor $output_l2, $output_l2, $rk10_l @ AES block 4k+2 - round 10 low
1698 eor $acc_lb, $acc_lb, $t8.16b @ GHASH block 4k+3 - low
1700 aese $ctr2b, $rk3 \n aesmc $ctr2b, $ctr2b @ AES block 4k+6 - round 3
1702 aese $ctr1b, $rk3 \n aesmc $ctr1b, $ctr1b @ AES block 4k+5 - round 3
1705 aese $ctr0b, $rk3 \n aesmc $ctr0b, $ctr0b @ AES block 4k+4 - round 3
1707 aese $ctr2b, $rk4 \n aesmc $ctr2b, $ctr2b @ AES block 4k+6 - round 4
1708 eor $acc_mb, $acc_mb, $t9.16b @ GHASH block 4k+3 - mid
1710 aese $ctr1b, $rk4 \n aesmc $ctr1b, $ctr1b @ AES block 4k+5 - round 4
1712 aese $ctr3b, $rk3 \n aesmc $ctr3b, $ctr3b @ AES block 4k+7 - round 3
1715 aese $ctr2b, $rk5 \n aesmc $ctr2b, $ctr2b @ AES block 4k+6 - round 5
1717 aese $ctr1b, $rk5 \n aesmc $ctr1b, $ctr1b @ AES block 4k+5 - round 5
1719 aese $ctr3b, $rk4 \n aesmc $ctr3b, $ctr3b @ AES block 4k+7 - round 4
1721 aese $ctr0b, $rk4 \n aesmc $ctr0b, $ctr0b @ AES block 4k+4 - round 4
1726 aese $ctr1b, $rk6 \n aesmc $ctr1b, $ctr1b @ AES block 4k+5 - round 6
1729 aese $ctr3b, $rk5 \n aesmc $ctr3b, $ctr3b @ AES block 4k+7 - round 5
1731 aese $ctr0b, $rk5 \n aesmc $ctr0b, $ctr0b @ AES block 4k+4 - round 5
1734 aese $ctr1b, $rk7 \n aesmc $ctr1b, $ctr1b @ AES block 4k+5 - round 7
1736 aese $ctr2b, $rk6 \n aesmc $ctr2b, $ctr2b @ AES block 4k+6 - round 6
1738 aese $ctr0b, $rk6 \n aesmc $ctr0b, $ctr0b @ AES block 4k+4 - round 6
1740 aese $ctr1b, $rk8 \n aesmc $ctr1b, $ctr1b @ AES block 4k+5 - round 8
1743 aese $ctr3b, $rk6 \n aesmc $ctr3b, $ctr3b @ AES block 4k+7 - round 6
1745 aese $ctr0b, $rk7 \n aesmc $ctr0b, $ctr0b @ AES block 4k+4 - round 7
1747 aese $ctr1b, $rk9 @ AES block 4k+5 - round 9
1750 eor $output_h3, $output_h3, $rk10_h @ AES block 4k+3 - round 10 high
1754 aese $ctr2b, $rk7 \n aesmc $ctr2b, $ctr2b @ AES block 4k+6 - round 7
1757 aese $ctr3b, $rk7 \n aesmc $ctr3b, $ctr3b @ AES block 4k+7 - round 7
1759 aese $ctr0b, $rk8 \n aesmc $ctr0b, $ctr0b @ AES block 4k+4 - round 8
1762 aese $ctr2b, $rk8 \n aesmc $ctr2b, $ctr2b @ AES block 4k+6 - round 8
1764 aese $ctr3b, $rk8 \n aesmc $ctr3b, $ctr3b @ AES block 4k+7 - round 8
1765 eor $output_h2, $output_h2, $rk10_h @ AES block 4k+2 - round 10 high
1769 aese $ctr0b, $rk9 @ AES block 4k+4 - round 9
1770 stp $output_l2, $output_h2, [$output_ptr], #16 @ AES block 4k+2 - store result
1772 aese $ctr2b, $rk9 @ AES block 4k+6 - round 9
1773 add $rctr32w, $rctr32w, #1 @ CTR block 4k+7
1774 stp $output_l3, $output_h3, [$output_ptr], #16 @ AES block 4k+3 - store result
1776 aese $ctr3b, $rk9 @ AES block 4k+7 - round 9
1781 ld1 { $res1b}, [$input_ptr], #16 @ AES block 4k+4 - load ciphertext
1783 eor $ctr0b, $res1b, $ctr0b @ AES block 4k+4 - result
1785 mov $output_h0, $ctr0.d[1] @ AES block 4k+4 - mov high
1787 mov $output_l0, $ctr0.d[0] @ AES block 4k+4 - mov low
1791 eor $output_h0, $output_h0, $rk10_h @ AES block 4k+4 - round 10 high
1796 eor $output_l0, $output_l0, $rk10_l @ AES block 4k+4 - round 10 low
1822 rev64 $res0b, $res1b @ GHASH final-3 block
1823 ld1 { $res1b}, [$input_ptr], #16 @ AES final-2 block - load ciphertext
1827 mov $acc_md, $h34k.d[1] @ GHASH final-3 block - mid
1828 stp $output_l0, $output_h0, [$output_ptr], #16 @ AES final-3 block - store result
1829 eor $ctr0b, $res1b, $ctr1b @ AES final-2 block - result
1831 mov $rk4d, $res0.d[1] @ GHASH final-3 block - mid
1832 mov $output_h0, $ctr0.d[1] @ AES final-2 block - mov high
1834 pmull $acc_l.1q, $res0.1d, $h4.1d @ GHASH final-3 block - low
1835 mov $output_l0, $ctr0.d[0] @ AES final-2 block - mov low
1837 pmull2 $acc_h.1q, $res0.2d, $h4.2d @ GHASH final-3 block - high
1839 eor $rk4v.8b, $rk4v.8b, $res0.8b @ GHASH final-3 block - mid
1842 eor $output_h0, $output_h0, $rk10_h @ AES final-2 block - round 10 high
1846 pmull $acc_m.1q, $rk4v.1d, $acc_m.1d @ GHASH final-3 block - mid
1847 eor $output_l0, $output_l0, $rk10_l @ AES final-2 block - round 10 low
1853 rev64 $res0b, $res1b @ GHASH final-2 block
1854 ld1 { $res1b}, [$input_ptr], #16 @ AES final-1 block - load ciphertext
1858 eor $ctr0b, $res1b, $ctr2b @ AES final-1 block - result
1859 stp $output_l0, $output_h0, [$output_ptr], #16 @ AES final-2 block - store result
1861 mov $rk4d, $res0.d[1] @ GHASH final-2 block - mid
1863 pmull $rk3q1, $res0.1d, $h3.1d @ GHASH final-2 block - low
1865 pmull2 $rk2q1, $res0.2d, $h3.2d @ GHASH final-2 block - high
1866 mov $output_l0, $ctr0.d[0] @ AES final-1 block - mov low
1868 mov $output_h0, $ctr0.d[1] @ AES final-1 block - mov high
1869 eor $rk4v.8b, $rk4v.8b, $res0.8b @ GHASH final-2 block - mid
1873 pmull $rk4v.1q, $rk4v.1d, $h34k.1d @ GHASH final-2 block - mid
1875 eor $output_l0, $output_l0, $rk10_l @ AES final-1 block - round 10 low
1879 eor $acc_lb, $acc_lb, $rk3 @ GHASH final-2 block - low
1881 eor $acc_hb, $acc_hb, $rk2 @ GHASH final-2 block - high
1883 eor $acc_mb, $acc_mb, $rk4v.16b @ GHASH final-2 block - mid
1884 eor $output_h0, $output_h0, $rk10_h @ AES final-1 block - round 10 high
1890 rev64 $res0b, $res1b @ GHASH final-1 block
1892 ld1 { $res1b}, [$input_ptr], #16 @ AES final block - load ciphertext
1895 mov $rk4d, $res0.d[1] @ GHASH final-1 block - mid
1897 eor $ctr0b, $res1b, $ctr3b @ AES final block - result
1899 eor $rk4v.8b, $rk4v.8b, $res0.8b @ GHASH final-1 block - mid
1901 stp $output_l0, $output_h0, [$output_ptr], #16 @ AES final-1 block - store result
1902 mov $output_l0, $ctr0.d[0] @ AES final block - mov low
1904 mov $output_h0, $ctr0.d[1] @ AES final block - mov high
1905 ins $rk4v.d[1], $rk4v.d[0] @ GHASH final-1 block - mid
1907 pmull $rk3q1, $res0.1d, $h2.1d @ GHASH final-1 block - low
1909 pmull2 $rk2q1, $res0.2d, $h2.2d @ GHASH final-1 block - high
1911 pmull2 $rk4v.1q, $rk4v.2d, $h12k.2d @ GHASH final-1 block - mid
1914 eor $acc_lb, $acc_lb, $rk3 @ GHASH final-1 block - low
1916 eor $acc_hb, $acc_hb, $rk2 @ GHASH final-1 block - high
1917 eor $output_h0, $output_h0, $rk10_h @ AES final block - round 10 high
1921 eor $output_l0, $output_l0, $rk10_l @ AES final block - round 10 low
1925 eor $acc_mb, $acc_mb, $rk4v.16b @ GHASH final-1 block - mid
1938 lsr $rk10_h, $rk10_h, $bit_length @ rk10_h is mask for top 64b of last block
1944 fmov $ctr0d, $ctr32x @ ctr0b is mask for last block
1948 and $res1b, $res1b, $ctr0b @ possibly partial last block has zeroes in highest bits
1950 rev64 $res0b, $res1b @ GHASH final block
1958 pmull2 $rk2q1, $res0.2d, $h1.2d @ GHASH final block - high
1959 mov $t0d, $res0.d[1] @ GHASH final block - mid
1961 eor $t0.8b, $t0.8b, $res0.8b @ GHASH final block - mid
1962 eor $acc_hb, $acc_hb, $rk2 @ GHASH final block - high
1964 pmull $t0.1q, $t0.1d, $h12k.1d @ GHASH final block - mid
1966 pmull $rk3q1, $res0.1d, $h1.1d @ GHASH final block - low
1976 eor $acc_mb, $acc_mb, $t0.16b @ GHASH final block - mid
1979 eor $acc_lb, $acc_lb, $rk3 @ GHASH final block - low
2137 fmov $ctr3d, $ctr96_b64x @ CTR block 3
2139 rev $ctr32w, $rctr32w @ CTR block 1
2140 add $rctr32w, $rctr32w, #1 @ CTR block 1
2141 fmov $ctr1d, $ctr96_b64x @ CTR block 1
2143 orr $ctr32x, $ctr96_t32x, $ctr32x, lsl #32 @ CTR block 1
2144 ld1 { $ctr0b}, [$counter] @ special case vector load initial counter so we can start first AES block as quickly as possible
2146 fmov $ctr1.d[1], $ctr32x @ CTR block 1
2147 rev $ctr32w, $rctr32w @ CTR block 2
2148 add $rctr32w, $rctr32w, #1 @ CTR block 2
2150 fmov $ctr2d, $ctr96_b64x @ CTR block 2
2151 orr $ctr32x, $ctr96_t32x, $ctr32x, lsl #32 @ CTR block 2
2153 fmov $ctr2.d[1], $ctr32x @ CTR block 2
2154 rev $ctr32w, $rctr32w @ CTR block 3
2156 orr $ctr32x, $ctr96_t32x, $ctr32x, lsl #32 @ CTR block 3
2159 fmov $ctr3.d[1], $ctr32x @ CTR block 3
2165 aese $ctr0b, $rk0 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 0
2170 aese $ctr3b, $rk0 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 0
2173 aese $ctr1b, $rk0 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 0
2178 aese $ctr2b, $rk0 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 0
2181 aese $ctr0b, $rk1 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 1
2184 aese $ctr1b, $rk1 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 1
2189 aese $ctr2b, $rk1 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 1
2192 aese $ctr3b, $rk1 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 1
2197 aese $ctr0b, $rk2 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 2
2199 aese $ctr2b, $rk2 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 2
2201 aese $ctr3b, $rk2 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 2
2203 aese $ctr0b, $rk3 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 3
2206 aese $ctr2b, $rk3 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 3
2208 aese $ctr1b, $rk2 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 2
2211 aese $ctr0b, $rk4 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 4
2213 aese $ctr3b, $rk3 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 3
2215 aese $ctr1b, $rk3 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 3
2217 aese $ctr0b, $rk5 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 5
2219 aese $ctr2b, $rk4 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 4
2221 aese $ctr1b, $rk4 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 4
2223 aese $ctr0b, $rk6 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 6
2225 aese $ctr3b, $rk4 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 4
2227 aese $ctr2b, $rk5 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 5
2229 aese $ctr1b, $rk5 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 5
2231 aese $ctr3b, $rk5 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 5
2233 aese $ctr2b, $rk6 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 6
2238 aese $ctr1b, $rk6 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 6
2240 aese $ctr3b, $rk6 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 6
2242 aese $ctr0b, $rk7 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 7
2244 aese $ctr1b, $rk7 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 7
2247 aese $ctr3b, $rk7 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 7
2249 aese $ctr0b, $rk8 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 8
2251 aese $ctr2b, $rk7 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 7
2254 aese $ctr1b, $rk8 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 8
2256 aese $ctr3b, $rk8 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 8
2258 aese $ctr2b, $rk8 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 8
2260 aese $ctr0b, $rk9 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 9
2262 aese $ctr3b, $rk9 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 9
2264 aese $ctr2b, $rk9 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 9
2266 aese $ctr1b, $rk9 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 9
2268 aese $ctr0b, $rk10 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 10
2270 aese $ctr2b, $rk10 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 10
2272 aese $ctr1b, $rk10 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 10
2276 aese $ctr3b, $rk10 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 10
2284 aese $ctr2b, $rk11 @ AES block 2 - round 11
2288 aese $ctr1b, $rk11 @ AES block 1 - round 11
2291 aese $ctr0b, $rk11 @ AES block 0 - round 11
2292 add $rctr32w, $rctr32w, #1 @ CTR block 3
2294 aese $ctr3b, $rk11 @ AES block 3 - round 11
2297 rev $ctr32w, $rctr32w @ CTR block 4
2298 ldp $input_l0, $input_h0, [$input_ptr, #0] @ AES block 0 - load plaintext
2303 orr $ctr32x, $ctr96_t32x, $ctr32x, lsl #32 @ CTR block 4
2304 ldp $input_l2, $input_h2, [$input_ptr, #32] @ AES block 2 - load plaintext
2309 ldp $input_l3, $input_h3, [$input_ptr, #48] @ AES block 3 - load plaintext
2314 ldp $input_l1, $input_h1, [$input_ptr, #16] @ AES block 1 - load plaintext
2322 eor $input_l0, $input_l0, $rk12_l @ AES block 0 - round 12 low
2324 eor $input_h0, $input_h0, $rk12_h @ AES block 0 - round 12 high
2325 eor $input_h2, $input_h2, $rk12_h @ AES block 2 - round 12 high
2326 fmov $ctr_t0d, $input_l0 @ AES block 0 - mov low
2328 eor $input_h3, $input_h3, $rk12_h @ AES block 3 - round 12 high
2329 fmov $ctr_t0.d[1], $input_h0 @ AES block 0 - mov high
2331 eor $input_l2, $input_l2, $rk12_l @ AES block 2 - round 12 low
2332 eor $input_l1, $input_l1, $rk12_l @ AES block 1 - round 12 low
2334 fmov $ctr_t1d, $input_l1 @ AES block 1 - mov low
2335 eor $input_h1, $input_h1, $rk12_h @ AES block 1 - round 12 high
2337 fmov $ctr_t1.d[1], $input_h1 @ AES block 1 - mov high
2339 eor $input_l3, $input_l3, $rk12_l @ AES block 3 - round 12 low
2340 fmov $ctr_t2d, $input_l2 @ AES block 2 - mov low
2342 add $rctr32w, $rctr32w, #1 @ CTR block 4
2343 eor $res0b, $ctr_t0b, $ctr0b @ AES block 0 - result
2344 fmov $ctr0d, $ctr96_b64x @ CTR block 4
2346 fmov $ctr0.d[1], $ctr32x @ CTR block 4
2347 rev $ctr32w, $rctr32w @ CTR block 5
2349 orr $ctr32x, $ctr96_t32x, $ctr32x, lsl #32 @ CTR block 5
2350 add $rctr32w, $rctr32w, #1 @ CTR block 5
2352 fmov $ctr_t3d, $input_l3 @ AES block 3 - mov low
2353 st1 { $res0b}, [$output_ptr], #16 @ AES block 0 - store result
2355 fmov $ctr_t2.d[1], $input_h2 @ AES block 2 - mov high
2357 eor $res1b, $ctr_t1b, $ctr1b @ AES block 1 - result
2358 fmov $ctr1d, $ctr96_b64x @ CTR block 5
2359 st1 { $res1b}, [$output_ptr], #16 @ AES block 1 - store result
2361 fmov $ctr_t3.d[1], $input_h3 @ AES block 3 - mov high
2363 fmov $ctr1.d[1], $ctr32x @ CTR block 5
2364 rev $ctr32w, $rctr32w @ CTR block 6
2366 orr $ctr32x, $ctr96_t32x, $ctr32x, lsl #32 @ CTR block 6
2368 add $rctr32w, $rctr32w, #1 @ CTR block 6
2369 eor $res2b, $ctr_t2b, $ctr2b @ AES block 2 - result
2370 fmov $ctr2d, $ctr96_b64x @ CTR block 6
2372 fmov $ctr2.d[1], $ctr32x @ CTR block 6
2373 rev $ctr32w, $rctr32w @ CTR block 7
2375 orr $ctr32x, $ctr96_t32x, $ctr32x, lsl #32 @ CTR block 7
2376 st1 { $res2b}, [$output_ptr], #16 @ AES block 2 - store result
2378 eor $res3b, $ctr_t3b, $ctr3b @ AES block 3 - result
2379 st1 { $res3b}, [$output_ptr], #16 @ AES block 3 - store result
2383 aese $ctr2b, $rk0 \n aesmc $ctr2b, $ctr2b @ AES block 4k+6 - round 0
2384 rev64 $res1b, $res1b @ GHASH block 4k+1 (t0 and t1 free)
2386 aese $ctr1b, $rk0 \n aesmc $ctr1b, $ctr1b @ AES block 4k+5 - round 0
2387 ldp $input_l1, $input_h1, [$input_ptr, #16] @ AES block 4k+5 - load plaintext
2393 fmov $ctr3d, $ctr96_b64x @ CTR block 4k+3
2394 rev64 $res0b, $res0b @ GHASH block 4k (only t0 is free)
2396 aese $ctr2b, $rk1 \n aesmc $ctr2b, $ctr2b @ AES block 4k+6 - round 1
2397 fmov $ctr3.d[1], $ctr32x @ CTR block 4k+3
2399 pmull2 $t1.1q, $res1.2d, $h3.2d @ GHASH block 4k+1 - high
2400 rev64 $res3b, $res3b @ GHASH block 4k+3 (t0, t1, t2 and t3 free)
2401 ldp $input_l2, $input_h2, [$input_ptr, #32] @ AES block 4k+6 - load plaintext
2406 aese $ctr0b, $rk0 \n aesmc $ctr0b, $ctr0b @ AES block 4k+4 - round 0
2407 ldp $input_l3, $input_h3, [$input_ptr, #48] @ AES block 4k+3 - load plaintext
2412 pmull $t2.1q, $res1.1d, $h3.1d @ GHASH block 4k+1 - low
2415 aese $ctr1b, $rk1 \n aesmc $ctr1b, $ctr1b @ AES block 4k+5 - round 1
2417 aese $ctr0b, $rk1 \n aesmc $ctr0b, $ctr0b @ AES block 4k+4 - round 1
2418 rev64 $res2b, $res2b @ GHASH block 4k+2 (t0, t1, and t2 free)
2420 aese $ctr3b, $rk0 \n aesmc $ctr3b, $ctr3b @ AES block 4k+7 - round 0
2421 eor $input_h3, $input_h3, $rk12_h @ AES block 4k+3 - round 12 high
2423 pmull $acc_l.1q, $res0.1d, $h4.1d @ GHASH block 4k - low
2424 mov $t0d, $res0.d[1] @ GHASH block 4k - mid
2426 aese $ctr0b, $rk2 \n aesmc $ctr0b, $ctr0b @ AES block 4k+4 - round 2
2428 aese $ctr3b, $rk1 \n aesmc $ctr3b, $ctr3b @ AES block 4k+7 - round 1
2429 eor $input_l2, $input_l2, $rk12_l @ AES block 4k+6 - round 12 low
2431 eor $t0.8b, $t0.8b, $res0.8b @ GHASH block 4k - mid
2432 eor $acc_lb, $acc_lb, $t2.16b @ GHASH block 4k+1 - low
2434 aese $ctr0b, $rk3 \n aesmc $ctr0b, $ctr0b @ AES block 4k+4 - round 3
2435 eor $input_l1, $input_l1, $rk12_l @ AES block 4k+5 - round 12 low
2437 aese $ctr1b, $rk2 \n aesmc $ctr1b, $ctr1b @ AES block 4k+5 - round 2
2438 mov $t6d, $res2.d[1] @ GHASH block 4k+2 - mid
2440 pmull2 $acc_h.1q, $res0.2d, $h4.2d @ GHASH block 4k - high
2441 mov $t3d, $res1.d[1] @ GHASH block 4k+1 - mid
2443 aese $ctr2b, $rk2 \n aesmc $ctr2b, $ctr2b @ AES block 4k+6 - round 2
2445 aese $ctr1b, $rk3 \n aesmc $ctr1b, $ctr1b @ AES block 4k+5 - round 3
2447 mov $acc_md, $h34k.d[1] @ GHASH block 4k - mid
2448 eor $acc_hb, $acc_hb, $t1.16b @ GHASH block 4k+1 - high
2450 aese $ctr3b, $rk2 \n aesmc $ctr3b, $ctr3b @ AES block 4k+7 - round 2
2451 eor $t6.8b, $t6.8b, $res2.8b @ GHASH block 4k+2 - mid
2453 pmull2 $t4.1q, $res2.2d, $h2.2d @ GHASH block 4k+2 - high
2455 aese $ctr0b, $rk4 \n aesmc $ctr0b, $ctr0b @ AES block 4k+4 - round 4
2456 eor $t3.8b, $t3.8b, $res1.8b @ GHASH block 4k+1 - mid
2458 aese $ctr3b, $rk3 \n aesmc $ctr3b, $ctr3b @ AES block 4k+7 - round 3
2460 pmull2 $t7.1q, $res3.2d, $h1.2d @ GHASH block 4k+3 - high
2461 eor $input_h1, $input_h1, $rk12_h @ AES block 4k+5 - round 12 high
2462 ins $t6.d[1], $t6.d[0] @ GHASH block 4k+2 - mid
2464 aese $ctr0b, $rk5 \n aesmc $ctr0b, $ctr0b @ AES block 4k+4 - round 5
2465 add $rctr32w, $rctr32w, #1 @ CTR block 4k+3
2467 aese $ctr3b, $rk4 \n aesmc $ctr3b, $ctr3b @ AES block 4k+7 - round 4
2468 eor $acc_hb, $acc_hb, $t4.16b @ GHASH block 4k+2 - high
2470 pmull $t3.1q, $t3.1d, $h34k.1d @ GHASH block 4k+1 - mid
2471 eor $input_h2, $input_h2, $rk12_h @ AES block 4k+6 - round 12 high
2473 pmull2 $t6.1q, $t6.2d, $h12k.2d @ GHASH block 4k+2 - mid
2474 eor $input_l3, $input_l3, $rk12_l @ AES block 4k+3 - round 12 low
2475 mov $t9d, $res3.d[1] @ GHASH block 4k+3 - mid
2477 pmull $acc_m.1q, $t0.1d, $acc_m.1d @ GHASH block 4k - mid
2478 rev $ctr32w, $rctr32w @ CTR block 4k+8
2480 pmull $t5.1q, $res2.1d, $h2.1d @ GHASH block 4k+2 - low
2481 orr $ctr32x, $ctr96_t32x, $ctr32x, lsl #32 @ CTR block 4k+8
2483 aese $ctr2b, $rk3 \n aesmc $ctr2b, $ctr2b @ AES block 4k+6 - round 3
2484 eor $t9.8b, $t9.8b, $res3.8b @ GHASH block 4k+3 - mid
2486 aese $ctr1b, $rk4 \n aesmc $ctr1b, $ctr1b @ AES block 4k+5 - round 4
2487 ldp $input_l0, $input_h0, [$input_ptr, #0] @ AES block 4k+4 - load plaintext
2492 aese $ctr0b, $rk6 \n aesmc $ctr0b, $ctr0b @ AES block 4k+4 - round 6
2493 eor $acc_lb, $acc_lb, $t5.16b @ GHASH block 4k+2 - low
2495 aese $ctr2b, $rk4 \n aesmc $ctr2b, $ctr2b @ AES block 4k+6 - round 4
2498 aese $ctr1b, $rk5 \n aesmc $ctr1b, $ctr1b @ AES block 4k+5 - round 5
2501 pmull $t8.1q, $res3.1d, $h1.1d @ GHASH block 4k+3 - low
2502 eor $input_h0, $input_h0, $rk12_h @ AES block 4k+4 - round 12 high
2503 eor $acc_mb, $acc_mb, $t3.16b @ GHASH block 4k+1 - mid
2505 aese $ctr2b, $rk5 \n aesmc $ctr2b, $ctr2b @ AES block 4k+6 - round 5
2506 eor $input_l0, $input_l0, $rk12_l @ AES block 4k+4 - round 12 low
2508 aese $ctr1b, $rk6 \n aesmc $ctr1b, $ctr1b @ AES block 4k+5 - round 6
2511 aese $ctr3b, $rk5 \n aesmc $ctr3b, $ctr3b @ AES block 4k+7 - round 5
2512 eor $acc_hb, $acc_hb, $t7.16b @ GHASH block 4k+3 - high
2514 aese $ctr0b, $rk7 \n aesmc $ctr0b, $ctr0b @ AES block 4k+4 - round 7
2515 fmov $ctr_t1d, $input_l1 @ AES block 4k+5 - mov low
2517 aese $ctr1b, $rk7 \n aesmc $ctr1b, $ctr1b @ AES block 4k+5 - round 7
2518 eor $acc_mb, $acc_mb, $t6.16b @ GHASH block 4k+2 - mid
2520 aese $ctr3b, $rk6 \n aesmc $ctr3b, $ctr3b @ AES block 4k+7 - round 6
2521 fmov $ctr_t1.d[1], $input_h1 @ AES block 4k+5 - mov high
2523 aese $ctr0b, $rk8 \n aesmc $ctr0b, $ctr0b @ AES block 4k+4 - round 8
2524 eor $acc_lb, $acc_lb, $t8.16b @ GHASH block 4k+3 - low
2526 pmull $t9.1q, $t9.1d, $h12k.1d @ GHASH block 4k+3 - mid
2528 fmov $ctr_t0d, $input_l0 @ AES block 4k+4 - mov low
2530 aese $ctr2b, $rk6 \n aesmc $ctr2b, $ctr2b @ AES block 4k+6 - round 6
2531 fmov $ctr_t0.d[1], $input_h0 @ AES block 4k+4 - mov high
2533 aese $ctr1b, $rk8 \n aesmc $ctr1b, $ctr1b @ AES block 4k+5 - round 8
2534 fmov $ctr_t3d, $input_l3 @ AES block 4k+3 - mov low
2536 eor $acc_mb, $acc_mb, $t9.16b @ GHASH block 4k+3 - mid
2538 add $rctr32w, $rctr32w, #1 @ CTR block 4k+8
2540 aese $ctr2b, $rk7 \n aesmc $ctr2b, $ctr2b @ AES block 4k+6 - round 7
2541 fmov $ctr_t3.d[1], $input_h3 @ AES block 4k+3 - mov high
2545 fmov $ctr_t2d, $input_l2 @ AES block 4k+6 - mov low
2547 aese $ctr3b, $rk7 \n aesmc $ctr3b, $ctr3b @ AES block 4k+7 - round 7
2549 aese $ctr0b, $rk9 \n aesmc $ctr0b, $ctr0b @ AES block 4k+4 - round 9
2552 aese $ctr2b, $rk8 \n aesmc $ctr2b, $ctr2b @ AES block 4k+6 - round 8
2554 aese $ctr3b, $rk8 \n aesmc $ctr3b, $ctr3b @ AES block 4k+7 - round 8
2556 aese $ctr1b, $rk9 \n aesmc $ctr1b, $ctr1b @ AES block 4k+5 - round 9
2558 aese $ctr0b, $rk10 \n aesmc $ctr0b, $ctr0b @ AES block 4k+4 - round 10
2561 aese $ctr3b, $rk9 \n aesmc $ctr3b, $ctr3b @ AES block 4k+7 - round 9
2563 aese $ctr2b, $rk9 \n aesmc $ctr2b, $ctr2b @ AES block 4k+6 - round 9
2565 aese $ctr0b, $rk11 @ AES block 4k+4 - round 11
2567 aese $ctr1b, $rk10 \n aesmc $ctr1b, $ctr1b @ AES block 4k+5 - round 10
2570 aese $ctr2b, $rk10 \n aesmc $ctr2b, $ctr2b @ AES block 4k+6 - round 10
2572 eor $res0b, $ctr_t0b, $ctr0b @ AES block 4k+4 - result
2573 fmov $ctr0d, $ctr96_b64x @ CTR block 4k+8
2575 aese $ctr1b, $rk11 @ AES block 4k+5 - round 11
2576 fmov $ctr0.d[1], $ctr32x @ CTR block 4k+8
2577 rev $ctr32w, $rctr32w @ CTR block 4k+9
2580 fmov $ctr_t2.d[1], $input_h2 @ AES block 4k+6 - mov high
2581 st1 { $res0b}, [$output_ptr], #16 @ AES block 4k+4 - store result
2583 aese $ctr3b, $rk10 \n aesmc $ctr3b, $ctr3b @ AES block 4k+7 - round 10
2584 orr $ctr32x, $ctr96_t32x, $ctr32x, lsl #32 @ CTR block 4k+9
2586 eor $res1b, $ctr_t1b, $ctr1b @ AES block 4k+5 - result
2587 add $rctr32w, $rctr32w, #1 @ CTR block 4k+9
2588 fmov $ctr1d, $ctr96_b64x @ CTR block 4k+9
2590 aese $ctr2b, $rk11 @ AES block 4k+6 - round 11
2591 fmov $ctr1.d[1], $ctr32x @ CTR block 4k+9
2592 rev $ctr32w, $rctr32w @ CTR block 4k+10
2594 add $rctr32w, $rctr32w, #1 @ CTR block 4k+10
2596 orr $ctr32x, $ctr96_t32x, $ctr32x, lsl #32 @ CTR block 4k+10
2598 st1 { $res1b}, [$output_ptr], #16 @ AES block 4k+5 - store result
2601 aese $ctr3b, $rk11 @ AES block 4k+7 - round 11
2602 eor $res2b, $ctr_t2b, $ctr2b @ AES block 4k+6 - result
2603 fmov $ctr2d, $ctr96_b64x @ CTR block 4k+10
2605 st1 { $res2b}, [$output_ptr], #16 @ AES block 4k+6 - store result
2606 fmov $ctr2.d[1], $ctr32x @ CTR block 4k+10
2607 rev $ctr32w, $rctr32w @ CTR block 4k+11
2610 orr $ctr32x, $ctr96_t32x, $ctr32x, lsl #32 @ CTR block 4k+11
2612 eor $res3b, $ctr_t3b, $ctr3b @ AES block 4k+3 - result
2613 st1 { $res3b}, [$output_ptr], #16 @ AES block 4k+3 - store result
2617 aese $ctr0b, $rk0 \n aesmc $ctr0b, $ctr0b @ AES block 4k+4 - round 0
2618 rev64 $res0b, $res0b @ GHASH block 4k (only t0 is free)
2620 fmov $ctr3d, $ctr96_b64x @ CTR block 4k+3
2622 add $rctr32w, $rctr32w, #1 @ CTR block 4k+3
2624 aese $ctr1b, $rk0 \n aesmc $ctr1b, $ctr1b @ AES block 4k+5 - round 0
2625 rev64 $res1b, $res1b @ GHASH block 4k+1 (t0 and t1 free)
2627 aese $ctr2b, $rk0 \n aesmc $ctr2b, $ctr2b @ AES block 4k+6 - round 0
2629 fmov $ctr3.d[1], $ctr32x @ CTR block 4k+3
2631 mov $acc_md, $h34k.d[1] @ GHASH block 4k - mid
2633 aese $ctr1b, $rk1 \n aesmc $ctr1b, $ctr1b @ AES block 4k+5 - round 1
2634 rev64 $res2b, $res2b @ GHASH block 4k+2 (t0, t1, and t2 free)
2636 pmull2 $t1.1q, $res1.2d, $h3.2d @ GHASH block 4k+1 - high
2638 pmull $acc_l.1q, $res0.1d, $h4.1d @ GHASH block 4k - low
2639 mov $t0d, $res0.d[1] @ GHASH block 4k - mid
2641 pmull $t2.1q, $res1.1d, $h3.1d @ GHASH block 4k+1 - low
2642 rev64 $res3b, $res3b @ GHASH block 4k+3 (t0, t1, t2 and t3 free)
2644 pmull2 $acc_h.1q, $res0.2d, $h4.2d @ GHASH block 4k - high
2646 eor $t0.8b, $t0.8b, $res0.8b @ GHASH block 4k - mid
2647 mov $t3d, $res1.d[1] @ GHASH block 4k+1 - mid
2649 eor $acc_lb, $acc_lb, $t2.16b @ GHASH block 4k+1 - low
2650 mov $t6d, $res2.d[1] @ GHASH block 4k+2 - mid
2652 aese $ctr3b, $rk0 \n aesmc $ctr3b, $ctr3b @ AES block 4k+7 - round 0
2653 eor $acc_hb, $acc_hb, $t1.16b @ GHASH block 4k+1 - high
2655 pmull2 $t4.1q, $res2.2d, $h2.2d @ GHASH block 4k+2 - high
2657 eor $t3.8b, $t3.8b, $res1.8b @ GHASH block 4k+1 - mid
2658 eor $t6.8b, $t6.8b, $res2.8b @ GHASH block 4k+2 - mid
2660 aese $ctr3b, $rk1 \n aesmc $ctr3b, $ctr3b @ AES block 4k+7 - round 1
2662 aese $ctr2b, $rk1 \n aesmc $ctr2b, $ctr2b @ AES block 4k+6 - round 1
2663 eor $acc_hb, $acc_hb, $t4.16b @ GHASH block 4k+2 - high
2665 aese $ctr0b, $rk1 \n aesmc $ctr0b, $ctr0b @ AES block 4k+4 - round 1
2667 aese $ctr1b, $rk2 \n aesmc $ctr1b, $ctr1b @ AES block 4k+5 - round 2
2668 mov $t9d, $res3.d[1] @ GHASH block 4k+3 - mid
2670 pmull2 $t7.1q, $res3.2d, $h1.2d @ GHASH block 4k+3 - high
2671 ins $t6.d[1], $t6.d[0] @ GHASH block 4k+2 - mid
2673 aese $ctr0b, $rk2 \n aesmc $ctr0b, $ctr0b @ AES block 4k+4 - round 2
2675 pmull $acc_m.1q, $t0.1d, $acc_m.1d @ GHASH block 4k - mid
2676 eor $t9.8b, $t9.8b, $res3.8b @ GHASH block 4k+3 - mid
2678 aese $ctr1b, $rk3 \n aesmc $ctr1b, $ctr1b @ AES block 4k+5 - round 3
2680 pmull2 $t6.1q, $t6.2d, $h12k.2d @ GHASH block 4k+2 - mid
2682 pmull $t3.1q, $t3.1d, $h34k.1d @ GHASH block 4k+1 - mid
2684 pmull $t9.1q, $t9.1d, $h12k.1d @ GHASH block 4k+3 - mid
2685 eor $acc_hb, $acc_hb, $t7.16b @ GHASH block 4k+3 - high
2687 pmull $t5.1q, $res2.1d, $h2.1d @ GHASH block 4k+2 - low
2689 aese $ctr0b, $rk3 \n aesmc $ctr0b, $ctr0b @ AES block 4k+4 - round 3
2690 eor $acc_mb, $acc_mb, $t3.16b @ GHASH block 4k+1 - mid
2692 aese $ctr3b, $rk2 \n aesmc $ctr3b, $ctr3b @ AES block 4k+7 - round 2
2694 aese $ctr2b, $rk2 \n aesmc $ctr2b, $ctr2b @ AES block 4k+6 - round 2
2695 eor $acc_lb, $acc_lb, $t5.16b @ GHASH block 4k+2 - low
2697 aese $ctr0b, $rk4 \n aesmc $ctr0b, $ctr0b @ AES block 4k+4 - round 4
2699 aese $ctr3b, $rk3 \n aesmc $ctr3b, $ctr3b @ AES block 4k+7 - round 3
2700 eor $acc_mb, $acc_mb, $t6.16b @ GHASH block 4k+2 - mid
2702 aese $ctr2b, $rk3 \n aesmc $ctr2b, $ctr2b @ AES block 4k+6 - round 3
2704 pmull $t8.1q, $res3.1d, $h1.1d @ GHASH block 4k+3 - low
2707 aese $ctr3b, $rk4 \n aesmc $ctr3b, $ctr3b @ AES block 4k+7 - round 4
2709 aese $ctr2b, $rk4 \n aesmc $ctr2b, $ctr2b @ AES block 4k+6 - round 4
2711 aese $ctr1b, $rk4 \n aesmc $ctr1b, $ctr1b @ AES block 4k+5 - round 4
2712 eor $acc_mb, $acc_mb, $t9.16b @ GHASH block 4k+3 - mid
2714 aese $ctr3b, $rk5 \n aesmc $ctr3b, $ctr3b @ AES block 4k+7 - round 5
2716 aese $ctr2b, $rk5 \n aesmc $ctr2b, $ctr2b @ AES block 4k+6 - round 5
2718 aese $ctr1b, $rk5 \n aesmc $ctr1b, $ctr1b @ AES block 4k+5 - round 5
2719 eor $acc_lb, $acc_lb, $t8.16b @ GHASH block 4k+3 - low
2721 aese $ctr0b, $rk5 \n aesmc $ctr0b, $ctr0b @ AES block 4k+4 - round 5
2723 aese $ctr3b, $rk6 \n aesmc $ctr3b, $ctr3b @ AES block 4k+7 - round 6
2726 aese $ctr1b, $rk6 \n aesmc $ctr1b, $ctr1b @ AES block 4k+5 - round 6
2728 aese $ctr0b, $rk6 \n aesmc $ctr0b, $ctr0b @ AES block 4k+4 - round 6
2731 aese $ctr3b, $rk7 \n aesmc $ctr3b, $ctr3b @ AES block 4k+7 - round 7
2733 aese $ctr1b, $rk7 \n aesmc $ctr1b, $ctr1b @ AES block 4k+5 - round 7
2736 aese $ctr0b, $rk7 \n aesmc $ctr0b, $ctr0b @ AES block 4k+4 - round 7
2740 aese $ctr2b, $rk6 \n aesmc $ctr2b, $ctr2b @ AES block 4k+6 - round 6
2743 aese $ctr0b, $rk8 \n aesmc $ctr0b, $ctr0b @ AES block 4k+4 - round 8
2745 aese $ctr1b, $rk8 \n aesmc $ctr1b, $ctr1b @ AES block 4k+5 - round 8
2748 aese $ctr2b, $rk7 \n aesmc $ctr2b, $ctr2b @ AES block 4k+6 - round 7
2750 aese $ctr3b, $rk8 \n aesmc $ctr3b, $ctr3b @ AES block 4k+7 - round 8
2752 aese $ctr0b, $rk9 \n aesmc $ctr0b, $ctr0b @ AES block 4k+4 - round 9
2754 aese $ctr2b, $rk8 \n aesmc $ctr2b, $ctr2b @ AES block 4k+6 - round 8
2757 aese $ctr3b, $rk9 \n aesmc $ctr3b, $ctr3b @ AES block 4k+7 - round 9
2759 aese $ctr1b, $rk9 \n aesmc $ctr1b, $ctr1b @ AES block 4k+5 - round 9
2761 aese $ctr2b, $rk9 \n aesmc $ctr2b, $ctr2b @ AES block 4k+6 - round 9
2767 aese $ctr3b, $rk10 \n aesmc $ctr3b, $ctr3b @ AES block 4k+7 - round 10
2769 aese $ctr0b, $rk10 \n aesmc $ctr0b, $ctr0b @ AES block 4k+4 - round 10
2771 aese $ctr2b, $rk10 \n aesmc $ctr2b, $ctr2b @ AES block 4k+6 - round 10
2773 aese $ctr1b, $rk10 \n aesmc $ctr1b, $ctr1b @ AES block 4k+5 - round 10
2776 aese $ctr0b, $rk11 @ AES block 4k+4 - round 11
2778 aese $ctr3b, $rk11 @ AES block 4k+7 - round 11
2780 aese $ctr2b, $rk11 @ AES block 4k+6 - round 11
2782 aese $ctr1b, $rk11 @ AES block 4k+5 - round 11
2787 ldp $input_l0, $input_h0, [$input_ptr], #16 @ AES block 4k+4 - load plaintext
2792 eor $input_l0, $input_l0, $rk12_l @ AES block 4k+4 - round 12 low
2793 eor $input_h0, $input_h0, $rk12_h @ AES block 4k+4 - round 12 high
2795 fmov $ctr_t0d, $input_l0 @ AES block 4k+4 - mov low
2797 fmov $ctr_t0.d[1], $input_h0 @ AES block 4k+4 - mov high
2800 eor $res1b, $ctr_t0b, $ctr0b @ AES block 4k+4 - result
2825 st1 { $res1b}, [$output_ptr], #16 @ AES final-3 block - store result
2827 ldp $input_l0, $input_h0, [$input_ptr], #16 @ AES final-2 block - load input low & high
2832 rev64 $res0b, $res1b @ GHASH final-3 block
2834 eor $input_l0, $input_l0, $rk12_l @ AES final-2 block - round 12 low
2837 eor $input_h0, $input_h0, $rk12_h @ AES final-2 block - round 12 high
2838 fmov $res1d, $input_l0 @ AES final-2 block - mov low
2840 fmov $res1.d[1], $input_h0 @ AES final-2 block - mov high
2842 mov $rk4d, $res0.d[1] @ GHASH final-3 block - mid
2844 pmull $acc_l.1q, $res0.1d, $h4.1d @ GHASH final-3 block - low
2846 mov $acc_md, $h34k.d[1] @ GHASH final-3 block - mid
2848 eor $rk4v.8b, $rk4v.8b, $res0.8b @ GHASH final-3 block - mid
2852 pmull2 $acc_h.1q, $res0.2d, $h4.2d @ GHASH final-3 block - high
2854 pmull $acc_m.1q, $rk4v.1d, $acc_m.1d @ GHASH final-3 block - mid
2855 eor $res1b, $res1b, $ctr1b @ AES final-2 block - result
2858 st1 { $res1b}, [$output_ptr], #16 @ AES final-2 block - store result
2860 rev64 $res0b, $res1b @ GHASH final-2 block
2861 ldp $input_l0, $input_h0, [$input_ptr], #16 @ AES final-1 block - load input low & high
2868 eor $input_h0, $input_h0, $rk12_h @ AES final-1 block - round 12 high
2870 pmull2 $rk2q1, $res0.2d, $h3.2d @ GHASH final-2 block - high
2871 mov $rk4d, $res0.d[1] @ GHASH final-2 block - mid
2873 pmull $rk3q1, $res0.1d, $h3.1d @ GHASH final-2 block - low
2874 eor $input_l0, $input_l0, $rk12_l @ AES final-1 block - round 12 low
2876 fmov $res1d, $input_l0 @ AES final-1 block - mov low
2878 fmov $res1.d[1], $input_h0 @ AES final-1 block - mov high
2879 eor $acc_hb, $acc_hb, $rk2 @ GHASH final-2 block - high
2880 eor $rk4v.8b, $rk4v.8b, $res0.8b @ GHASH final-2 block - mid
2882 eor $acc_lb, $acc_lb, $rk3 @ GHASH final-2 block - low
2884 pmull $rk4v.1q, $rk4v.1d, $h34k.1d @ GHASH final-2 block - mid
2888 eor $res1b, $res1b, $ctr2b @ AES final-1 block - result
2890 eor $acc_mb, $acc_mb, $rk4v.16b @ GHASH final-2 block - mid
2893 st1 { $res1b}, [$output_ptr], #16 @ AES final-1 block - store result
2895 ldp $input_l0, $input_h0, [$input_ptr], #16 @ AES final block - load input low & high
2900 rev64 $res0b, $res1b @ GHASH final-1 block
2902 eor $input_l0, $input_l0, $rk12_l @ AES final block - round 12 low
2906 mov $rk4d, $res0.d[1] @ GHASH final-1 block - mid
2908 eor $rk4v.8b, $rk4v.8b, $res0.8b @ GHASH final-1 block - mid
2909 eor $input_h0, $input_h0, $rk12_h @ AES final block - round 12 high
2910 fmov $res1d, $input_l0 @ AES final block - mov low
2912 pmull2 $rk2q1, $res0.2d, $h2.2d @ GHASH final-1 block - high
2913 fmov $res1.d[1], $input_h0 @ AES final block - mov high
2915 ins $rk4v.d[1], $rk4v.d[0] @ GHASH final-1 block - mid
2917 eor $acc_hb, $acc_hb, $rk2 @ GHASH final-1 block - high
2919 pmull $rk3q1, $res0.1d, $h2.1d @ GHASH final-1 block - low
2921 pmull2 $rk4v.1q, $rk4v.2d, $h12k.2d @ GHASH final-1 block - mid
2923 eor $res1b, $res1b, $ctr3b @ AES final block - result
2925 eor $acc_lb, $acc_lb, $rk3 @ GHASH final-1 block - low
2927 eor $acc_mb, $acc_mb, $rk4v.16b @ GHASH final-1 block - mid
2930 ld1 { $rk0}, [$output_ptr] @ load existing bytes where the possibly partial last block is to be stored
2946 lsr $rk12_h, $rk12_h, $bit_length @ rk12_h is mask for top 64b of last block
2952 fmov $ctr0d, $input_l0 @ ctr0b is mask for last block
2956 and $res1b, $res1b, $ctr0b @ possibly partial last block has zeroes in highest bits
2958 rev64 $res0b, $res1b @ GHASH final block
2962 mov $t0d, $res0.d[1] @ GHASH final block - mid
2964 pmull $rk3q1, $res0.1d, $h1.1d @ GHASH final block - low
2966 pmull2 $rk2q1, $res0.2d, $h1.2d @ GHASH final block - high
2968 eor $t0.8b, $t0.8b, $res0.8b @ GHASH final block - mid
2970 eor $acc_lb, $acc_lb, $rk3 @ GHASH final block - low
2972 eor $acc_hb, $acc_hb, $rk2 @ GHASH final block - high
2974 pmull $t0.1q, $t0.1d, $h12k.1d @ GHASH final block - mid
2976 eor $acc_mb, $acc_mb, $t0.16b @ GHASH final block - mid
3060 ld1 { $ctr0b}, [$counter] @ special case vector load initial counter so we can start first AES block as quickly as possible
3070 fmov $ctr3d, $ctr96_b64x @ CTR block 3
3073 fmov $ctr1d, $ctr96_b64x @ CTR block 1
3078 aese $ctr0b, $rk0 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 0
3079 rev $ctr32w, $rctr32w @ CTR block 1
3081 add $rctr32w, $rctr32w, #1 @ CTR block 1
3082 orr $ctr32x, $ctr96_t32x, $ctr32x, lsl #32 @ CTR block 1
3085 fmov $ctr1.d[1], $ctr32x @ CTR block 1
3086 rev $ctr32w, $rctr32w @ CTR block 2
3087 add $rctr32w, $rctr32w, #1 @ CTR block 2
3089 fmov $ctr2d, $ctr96_b64x @ CTR block 2
3090 orr $ctr32x, $ctr96_t32x, $ctr32x, lsl #32 @ CTR block 2
3092 fmov $ctr2.d[1], $ctr32x @ CTR block 2
3093 rev $ctr32w, $rctr32w @ CTR block 3
3095 aese $ctr0b, $rk1 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 1
3096 orr $ctr32x, $ctr96_t32x, $ctr32x, lsl #32 @ CTR block 3
3098 fmov $ctr3.d[1], $ctr32x @ CTR block 3
3102 aese $ctr0b, $rk2 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 2
3104 aese $ctr2b, $rk0 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 0
3107 aese $ctr1b, $rk0 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 0
3112 aese $ctr3b, $rk0 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 0
3117 aese $ctr2b, $rk1 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 1
3122 aese $ctr1b, $rk1 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 1
3124 aese $ctr3b, $rk1 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 1
3129 aese $ctr2b, $rk2 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 2
3132 aese $ctr0b, $rk3 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 3
3135 aese $ctr1b, $rk2 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 2
3138 aese $ctr3b, $rk2 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 2
3141 aese $ctr2b, $rk3 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 3
3146 aese $ctr1b, $rk3 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 3
3147 add $rctr32w, $rctr32w, #1 @ CTR block 3
3149 aese $ctr3b, $rk3 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 3
3152 aese $ctr0b, $rk4 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 4
3155 aese $ctr1b, $rk4 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 4
3158 aese $ctr2b, $rk4 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 4
3160 aese $ctr3b, $rk4 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 4
3163 aese $ctr0b, $rk5 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 5
3166 aese $ctr1b, $rk5 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 5
3168 aese $ctr2b, $rk5 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 5
3170 aese $ctr3b, $rk5 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 5
3172 aese $ctr0b, $rk6 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 6
3174 aese $ctr2b, $rk6 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 6
3176 aese $ctr3b, $rk6 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 6
3178 aese $ctr0b, $rk7 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 7
3180 aese $ctr2b, $rk7 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 7
3182 aese $ctr3b, $rk7 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 7
3184 aese $ctr1b, $rk6 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 6
3186 aese $ctr2b, $rk8 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 8
3188 aese $ctr3b, $rk8 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 8
3190 aese $ctr1b, $rk7 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 7
3192 aese $ctr2b, $rk9 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 9
3194 aese $ctr3b, $rk9 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 9
3196 aese $ctr1b, $rk8 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 8
3199 aese $ctr0b, $rk8 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 8
3202 aese $ctr3b, $rk10 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 10
3205 aese $ctr1b, $rk9 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 9
3208 aese $ctr0b, $rk9 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 9
3211 aese $ctr3b, $rk11 @ AES block 3 - round 11
3213 aese $ctr2b, $rk10 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 10
3215 aese $ctr1b, $rk10 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 10
3217 aese $ctr0b, $rk10 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 10
3220 aese $ctr2b, $rk11 @ AES block 2 - round 11
3222 aese $ctr1b, $rk11 @ AES block 1 - round 11
3225 aese $ctr0b, $rk11 @ AES block 0 - round 11
3228 ld1 {$res0b, $res1b}, [$input_ptr], #32 @ AES block 0,1 - load ciphertext
3230 eor $ctr1b, $res1b, $ctr1b @ AES block 1 - result
3232 eor $ctr0b, $res0b, $ctr0b @ AES block 0 - result
3233 rev $ctr32w, $rctr32w @ CTR block 4
3234 ld1 {$res2b, $res3b}, [$input_ptr], #32 @ AES block 2,3 - load ciphertext
3236 mov $output_l1, $ctr1.d[0] @ AES block 1 - mov low
3238 mov $output_h1, $ctr1.d[1] @ AES block 1 - mov high
3240 mov $output_l0, $ctr0.d[0] @ AES block 0 - mov low
3241 orr $ctr32x, $ctr96_t32x, $ctr32x, lsl #32 @ CTR block 4
3242 add $rctr32w, $rctr32w, #1 @ CTR block 4
3244 mov $output_h0, $ctr0.d[1] @ AES block 0 - mov high
3245 rev64 $res0b, $res0b @ GHASH block 0
3247 fmov $ctr0d, $ctr96_b64x @ CTR block 4
3248 rev64 $res1b, $res1b @ GHASH block 1
3251 eor $output_l1, $output_l1, $rk12_l @ AES block 1 - round 12 low
3255 fmov $ctr0.d[1], $ctr32x @ CTR block 4
3256 rev $ctr32w, $rctr32w @ CTR block 5
3258 orr $ctr32x, $ctr96_t32x, $ctr32x, lsl #32 @ CTR block 5
3259 fmov $ctr1d, $ctr96_b64x @ CTR block 5
3260 eor $output_h1, $output_h1, $rk12_h @ AES block 1 - round 12 high
3264 add $rctr32w, $rctr32w, #1 @ CTR block 5
3265 fmov $ctr1.d[1], $ctr32x @ CTR block 5
3266 eor $output_l0, $output_l0, $rk12_l @ AES block 0 - round 12 low
3270 rev $ctr32w, $rctr32w @ CTR block 6
3271 eor $output_h0, $output_h0, $rk12_h @ AES block 0 - round 12 high
3275 stp $output_l0, $output_h0, [$output_ptr], #16 @ AES block 0 - store result
3276 orr $ctr32x, $ctr96_t32x, $ctr32x, lsl #32 @ CTR block 6
3278 stp $output_l1, $output_h1, [$output_ptr], #16 @ AES block 1 - store result
3280 add $rctr32w, $rctr32w, #1 @ CTR block 6
3281 eor $ctr2b, $res2b, $ctr2b @ AES block 2 - result
3285 aese $ctr1b, $rk0 \n aesmc $ctr1b, $ctr1b @ AES block 4k+5 - round 0
3288 pmull $t2.1q, $res1.1d, $h3.1d @ GHASH block 4k+1 - low
3289 mov $output_l2, $ctr2.d[0] @ AES block 4k+2 - mov low
3291 mov $output_h2, $ctr2.d[1] @ AES block 4k+2 - mov high
3292 eor $ctr3b, $res3b, $ctr3b @ AES block 4k+3 - result
3293 rev64 $res3b, $res3b @ GHASH block 4k+3
3295 aese $ctr1b, $rk1 \n aesmc $ctr1b, $ctr1b @ AES block 4k+5 - round 1
3296 fmov $ctr2d, $ctr96_b64x @ CTR block 4k+6
3298 aese $ctr0b, $rk0 \n aesmc $ctr0b, $ctr0b @ AES block 4k+4 - round 0
3301 pmull2 $t1.1q, $res1.2d, $h3.2d @ GHASH block 4k+1 - high
3302 fmov $ctr2.d[1], $ctr32x @ CTR block 4k+6
3304 aese $ctr1b, $rk2 \n aesmc $ctr1b, $ctr1b @ AES block 4k+5 - round 2
3305 mov $output_h3, $ctr3.d[1] @ AES block 4k+3 - mov high
3307 aese $ctr0b, $rk1 \n aesmc $ctr0b, $ctr0b @ AES block 4k+4 - round 1
3308 mov $output_l3, $ctr3.d[0] @ AES block 4k+3 - mov low
3310 pmull2 $acc_h.1q, $res0.2d, $h4.2d @ GHASH block 4k - high
3311 fmov $ctr3d, $ctr96_b64x @ CTR block 4k+7
3312 mov $t0d, $res0.d[1] @ GHASH block 4k - mid
3314 pmull $acc_l.1q, $res0.1d, $h4.1d @ GHASH block 4k - low
3315 mov $acc_md, $h34k.d[1] @ GHASH block 4k - mid
3316 rev $ctr32w, $rctr32w @ CTR block 4k+7
3318 aese $ctr2b, $rk0 \n aesmc $ctr2b, $ctr2b @ AES block 4k+6 - round 0
3319 orr $ctr32x, $ctr96_t32x, $ctr32x, lsl #32 @ CTR block 4k+7
3321 fmov $ctr3.d[1], $ctr32x @ CTR block 4k+7
3322 eor $t0.8b, $t0.8b, $res0.8b @ GHASH block 4k - mid
3323 mov $t3d, $res1.d[1] @ GHASH block 4k+1 - mid
3325 aese $ctr1b, $rk3 \n aesmc $ctr1b, $ctr1b @ AES block 4k+5 - round 3
3327 aese $ctr0b, $rk2 \n aesmc $ctr0b, $ctr0b @ AES block 4k+4 - round 2
3328 eor $output_h2, $output_h2, $rk12_h @ AES block 4k+2 - round 12 high
3332 aese $ctr2b, $rk1 \n aesmc $ctr2b, $ctr2b @ AES block 4k+6 - round 1
3333 eor $t3.8b, $t3.8b, $res1.8b @ GHASH block 4k+1 - mid
3335 pmull $acc_m.1q, $t0.1d, $acc_m.1d @ GHASH block 4k - mid
3337 aese $ctr3b, $rk0 \n aesmc $ctr3b, $ctr3b @ AES block 4k+7 - round 0
3338 rev64 $res2b, $res2b @ GHASH block 4k+2
3340 aese $ctr2b, $rk2 \n aesmc $ctr2b, $ctr2b @ AES block 4k+6 - round 2
3342 pmull $t3.1q, $t3.1d, $h34k.1d @ GHASH block 4k+1 - mid
3343 eor $acc_lb, $acc_lb, $t2.16b @ GHASH block 4k+1 - low
3344 eor $output_l2, $output_l2, $rk12_l @ AES block 4k+2 - round 12 low
3348 aese $ctr1b, $rk4 \n aesmc $ctr1b, $ctr1b @ AES block 4k+5 - round 4
3350 aese $ctr0b, $rk3 \n aesmc $ctr0b, $ctr0b @ AES block 4k+4 - round 3
3352 eor $acc_mb, $acc_mb, $t3.16b @ GHASH block 4k+1 - mid
3353 mov $t6d, $res2.d[1] @ GHASH block 4k+2 - mid
3355 aese $ctr3b, $rk1 \n aesmc $ctr3b, $ctr3b @ AES block 4k+7 - round 1
3356 eor $acc_hb, $acc_hb, $t1.16b @ GHASH block 4k+1 - high
3358 aese $ctr0b, $rk4 \n aesmc $ctr0b, $ctr0b @ AES block 4k+4 - round 4
3360 pmull2 $t4.1q, $res2.2d, $h2.2d @ GHASH block 4k+2 - high
3361 eor $t6.8b, $t6.8b, $res2.8b @ GHASH block 4k+2 - mid
3363 pmull $t5.1q, $res2.1d, $h2.1d @ GHASH block 4k+2 - low
3365 aese $ctr0b, $rk5 \n aesmc $ctr0b, $ctr0b @ AES block 4k+4 - round 5
3367 eor $acc_hb, $acc_hb, $t4.16b @ GHASH block 4k+2 - high
3368 mov $t9d, $res3.d[1] @ GHASH block 4k+3 - mid
3370 aese $ctr1b, $rk5 \n aesmc $ctr1b, $ctr1b @ AES block 4k+5 - round 5
3372 pmull2 $t7.1q, $res3.2d, $h1.2d @ GHASH block 4k+3 - high
3374 aese $ctr3b, $rk2 \n aesmc $ctr3b, $ctr3b @ AES block 4k+7 - round 2
3375 eor $t9.8b, $t9.8b, $res3.8b @ GHASH block 4k+3 - mid
3377 aese $ctr1b, $rk6 \n aesmc $ctr1b, $ctr1b @ AES block 4k+5 - round 6
3379 aese $ctr0b, $rk6 \n aesmc $ctr0b, $ctr0b @ AES block 4k+4 - round 6
3380 ins $t6.d[1], $t6.d[0] @ GHASH block 4k+2 - mid
3382 aese $ctr3b, $rk3 \n aesmc $ctr3b, $ctr3b @ AES block 4k+7 - round 3
3384 pmull $t9.1q, $t9.1d, $h12k.1d @ GHASH block 4k+3 - mid
3385 eor $acc_lb, $acc_lb, $t5.16b @ GHASH block 4k+2 - low
3387 aese $ctr0b, $rk7 \n aesmc $ctr0b, $ctr0b @ AES block 4k+4 - round 7
3389 pmull2 $t6.1q, $t6.2d, $h12k.2d @ GHASH block 4k+2 - mid
3390 eor $acc_hb, $acc_hb, $t7.16b @ GHASH block 4k+3 - high
3392 aese $ctr1b, $rk7 \n aesmc $ctr1b, $ctr1b @ AES block 4k+5 - round 7
3394 aese $ctr0b, $rk8 \n aesmc $ctr0b, $ctr0b @ AES block 4k+4 - round 8
3397 pmull $t8.1q, $res3.1d, $h1.1d @ GHASH block 4k+3 - low
3399 aese $ctr1b, $rk8 \n aesmc $ctr1b, $ctr1b @ AES block 4k+5 - round 8
3400 eor $acc_mb, $acc_mb, $t6.16b @ GHASH block 4k+2 - mid
3402 aese $ctr2b, $rk3 \n aesmc $ctr2b, $ctr2b @ AES block 4k+6 - round 3
3404 aese $ctr0b, $rk9 \n aesmc $ctr0b, $ctr0b @ AES block 4k+4 - round 9
3405 eor $acc_lb, $acc_lb, $t8.16b @ GHASH block 4k+3 - low
3407 aese $ctr3b, $rk4 \n aesmc $ctr3b, $ctr3b @ AES block 4k+7 - round 4
3409 aese $ctr2b, $rk4 \n aesmc $ctr2b, $ctr2b @ AES block 4k+6 - round 4
3410 eor $acc_mb, $acc_mb, $t9.16b @ GHASH block 4k+3 - mid
3412 aese $ctr0b, $rk10 \n aesmc $ctr0b, $ctr0b @ AES block 4k+4 - round 10
3414 aese $ctr1b, $rk9 \n aesmc $ctr1b, $ctr1b @ AES block 4k+5 - round 9
3417 aese $ctr2b, $rk5 \n aesmc $ctr2b, $ctr2b @ AES block 4k+6 - round 5
3419 aese $ctr3b, $rk5 \n aesmc $ctr3b, $ctr3b @ AES block 4k+7 - round 5
3422 aese $ctr1b, $rk10 \n aesmc $ctr1b, $ctr1b @ AES block 4k+5 - round 10
3424 aese $ctr2b, $rk6 \n aesmc $ctr2b, $ctr2b @ AES block 4k+6 - round 6
3425 ld1 {$res0b}, [$input_ptr], #16 @ AES block 4k+4 - load ciphertext
3427 aese $ctr3b, $rk6 \n aesmc $ctr3b, $ctr3b @ AES block 4k+7 - round 6
3431 ld1 {$res1b}, [$input_ptr], #16 @ AES block 4k+5 - load ciphertext
3432 eor $output_l3, $output_l3, $rk12_l @ AES block 4k+3 - round 12 low
3436 aese $ctr2b, $rk7 \n aesmc $ctr2b, $ctr2b @ AES block 4k+6 - round 7
3439 aese $ctr0b, $rk11 @ AES block 4k+4 - round 11
3440 add $rctr32w, $rctr32w, #1 @ CTR block 4k+7
3442 aese $ctr3b, $rk7 \n aesmc $ctr3b, $ctr3b @ AES block 4k+7 - round 7
3445 aese $ctr2b, $rk8 \n aesmc $ctr2b, $ctr2b @ AES block 4k+6 - round 8
3446 ld1 {$res2b}, [$input_ptr], #16 @ AES block 4k+6 - load ciphertext
3448 aese $ctr1b, $rk11 @ AES block 4k+5 - round 11
3449 ld1 {$res3b}, [$input_ptr], #16 @ AES block 4k+7 - load ciphertext
3450 rev $ctr32w, $rctr32w @ CTR block 4k+8
3452 aese $ctr3b, $rk8 \n aesmc $ctr3b, $ctr3b @ AES block 4k+7 - round 8
3453 stp $output_l2, $output_h2, [$output_ptr], #16 @ AES block 4k+2 - store result
3455 aese $ctr2b, $rk9 \n aesmc $ctr2b, $ctr2b @ AES block 4k+6 - round 9
3460 eor $ctr0b, $res0b, $ctr0b @ AES block 4k+4 - result
3461 eor $output_h3, $output_h3, $rk12_h @ AES block 4k+3 - round 12 high
3465 eor $ctr1b, $res1b, $ctr1b @ AES block 4k+5 - result
3467 aese $ctr2b, $rk10 \n aesmc $ctr2b, $ctr2b @ AES block 4k+6 - round 10
3468 orr $ctr32x, $ctr96_t32x, $ctr32x, lsl #32 @ CTR block 4k+8
3470 aese $ctr3b, $rk9 \n aesmc $ctr3b, $ctr3b @ AES block 4k+7 - round 9
3473 mov $output_l1, $ctr1.d[0] @ AES block 4k+5 - mov low
3475 mov $output_l0, $ctr0.d[0] @ AES block 4k+4 - mov low
3476 stp $output_l3, $output_h3, [$output_ptr], #16 @ AES block 4k+3 - store result
3477 rev64 $res1b, $res1b @ GHASH block 4k+5
3479 aese $ctr2b, $rk11 @ AES block 4k+6 - round 11
3480 mov $output_h0, $ctr0.d[1] @ AES block 4k+4 - mov high
3482 aese $ctr3b, $rk10 \n aesmc $ctr3b, $ctr3b @ AES block 4k+7 - round 10
3483 mov $output_h1, $ctr1.d[1] @ AES block 4k+5 - mov high
3485 fmov $ctr0d, $ctr96_b64x @ CTR block 4k+8
3486 add $rctr32w, $rctr32w, #1 @ CTR block 4k+8
3489 eor $ctr2b, $res2b, $ctr2b @ AES block 4k+6 - result
3490 fmov $ctr0.d[1], $ctr32x @ CTR block 4k+8
3491 rev $ctr32w, $rctr32w @ CTR block 4k+9
3493 eor $output_l0, $output_l0, $rk12_l @ AES block 4k+4 - round 12 low
3497 orr $ctr32x, $ctr96_t32x, $ctr32x, lsl #32 @ CTR block 4k+9
3500 fmov $ctr1d, $ctr96_b64x @ CTR block 4k+9
3501 add $rctr32w, $rctr32w, #1 @ CTR block 4k+9
3502 eor $output_l1, $output_l1, $rk12_l @ AES block 4k+5 - round 12 low
3506 fmov $ctr1.d[1], $ctr32x @ CTR block 4k+9
3507 rev $ctr32w, $rctr32w @ CTR block 4k+10
3508 eor $output_h1, $output_h1, $rk12_h @ AES block 4k+5 - round 12 high
3512 eor $output_h0, $output_h0, $rk12_h @ AES block 4k+4 - round 12 high
3516 stp $output_l0, $output_h0, [$output_ptr], #16 @ AES block 4k+4 - store result
3519 add $rctr32w, $rctr32w, #1 @ CTR block 4k+10
3520 rev64 $res0b, $res0b @ GHASH block 4k+4
3521 orr $ctr32x, $ctr96_t32x, $ctr32x, lsl #32 @ CTR block 4k+10
3523 aese $ctr3b, $rk11 @ AES block 4k+7 - round 11
3524 stp $output_l1, $output_h1, [$output_ptr], #16 @ AES block 4k+5 - store result
3528 mov $output_h2, $ctr2.d[1] @ AES block 4k+2 - mov high
3530 eor $ctr3b, $res3b, $ctr3b @ AES block 4k+3 - result
3532 aese $ctr1b, $rk0 \n aesmc $ctr1b, $ctr1b @ AES block 4k+5 - round 0
3533 mov $output_l2, $ctr2.d[0] @ AES block 4k+2 - mov low
3535 aese $ctr0b, $rk0 \n aesmc $ctr0b, $ctr0b @ AES block 4k+4 - round 0
3536 mov $acc_md, $h34k.d[1] @ GHASH block 4k - mid
3539 fmov $ctr2d, $ctr96_b64x @ CTR block 4k+6
3541 aese $ctr1b, $rk1 \n aesmc $ctr1b, $ctr1b @ AES block 4k+5 - round 1
3542 mov $output_l3, $ctr3.d[0] @ AES block 4k+3 - mov low
3544 aese $ctr0b, $rk1 \n aesmc $ctr0b, $ctr0b @ AES block 4k+4 - round 1
3545 mov $output_h3, $ctr3.d[1] @ AES block 4k+3 - mov high
3547 pmull $acc_l.1q, $res0.1d, $h4.1d @ GHASH block 4k - low
3548 mov $t0d, $res0.d[1] @ GHASH block 4k - mid
3549 fmov $ctr3d, $ctr96_b64x @ CTR block 4k+7
3551 aese $ctr1b, $rk2 \n aesmc $ctr1b, $ctr1b @ AES block 4k+5 - round 2
3552 rev64 $res2b, $res2b @ GHASH block 4k+2
3554 pmull2 $acc_h.1q, $res0.2d, $h4.2d @ GHASH block 4k - high
3555 fmov $ctr2.d[1], $ctr32x @ CTR block 4k+6
3556 rev $ctr32w, $rctr32w @ CTR block 4k+7
3558 orr $ctr32x, $ctr96_t32x, $ctr32x, lsl #32 @ CTR block 4k+7
3559 eor $t0.8b, $t0.8b, $res0.8b @ GHASH block 4k - mid
3560 mov $t3d, $res1.d[1] @ GHASH block 4k+1 - mid
3562 pmull $t2.1q, $res1.1d, $h3.1d @ GHASH block 4k+1 - low
3563 eor $output_h3, $output_h3, $rk12_h @ AES block 4k+3 - round 12 high
3567 fmov $ctr3.d[1], $ctr32x @ CTR block 4k+7
3569 aese $ctr0b, $rk2 \n aesmc $ctr0b, $ctr0b @ AES block 4k+4 - round 2
3570 eor $output_l2, $output_l2, $rk12_l @ AES block 4k+2 - round 12 low
3574 pmull2 $t1.1q, $res1.2d, $h3.2d @ GHASH block 4k+1 - high
3575 eor $output_h2, $output_h2, $rk12_h @ AES block 4k+2 - round 12 high
3579 eor $t3.8b, $t3.8b, $res1.8b @ GHASH block 4k+1 - mid
3581 pmull $acc_m.1q, $t0.1d, $acc_m.1d @ GHASH block 4k - mid
3582 eor $output_l3, $output_l3, $rk12_l @ AES block 4k+3 - round 12 low
3586 stp $output_l2, $output_h2, [$output_ptr], #16 @ AES block 4k+2 - store result
3588 rev64 $res3b, $res3b @ GHASH block 4k+3
3589 stp $output_l3, $output_h3, [$output_ptr], #16 @ AES block 4k+3 - store result
3591 aese $ctr3b, $rk0 \n aesmc $ctr3b, $ctr3b @ AES block 4k+7 - round 0
3592 eor $acc_hb, $acc_hb, $t1.16b @ GHASH block 4k+1 - high
3594 pmull $t3.1q, $t3.1d, $h34k.1d @ GHASH block 4k+1 - mid
3595 add $rctr32w, $rctr32w, #1 @ CTR block 4k+7
3597 pmull2 $t4.1q, $res2.2d, $h2.2d @ GHASH block 4k+2 - high
3598 eor $acc_lb, $acc_lb, $t2.16b @ GHASH block 4k+1 - low
3600 aese $ctr2b, $rk0 \n aesmc $ctr2b, $ctr2b @ AES block 4k+6 - round 0
3602 eor $acc_mb, $acc_mb, $t3.16b @ GHASH block 4k+1 - mid
3603 mov $t6d, $res2.d[1] @ GHASH block 4k+2 - mid
3605 aese $ctr3b, $rk1 \n aesmc $ctr3b, $ctr3b @ AES block 4k+7 - round 1
3607 aese $ctr2b, $rk1 \n aesmc $ctr2b, $ctr2b @ AES block 4k+6 - round 1
3608 eor $acc_hb, $acc_hb, $t4.16b @ GHASH block 4k+2 - high
3610 eor $t6.8b, $t6.8b, $res2.8b @ GHASH block 4k+2 - mid
3612 pmull $t5.1q, $res2.1d, $h2.1d @ GHASH block 4k+2 - low
3614 aese $ctr2b, $rk2 \n aesmc $ctr2b, $ctr2b @ AES block 4k+6 - round 2
3615 mov $t9d, $res3.d[1] @ GHASH block 4k+3 - mid
3617 aese $ctr3b, $rk2 \n aesmc $ctr3b, $ctr3b @ AES block 4k+7 - round 2
3618 ins $t6.d[1], $t6.d[0] @ GHASH block 4k+2 - mid
3620 pmull $t8.1q, $res3.1d, $h1.1d @ GHASH block 4k+3 - low
3622 aese $ctr0b, $rk3 \n aesmc $ctr0b, $ctr0b @ AES block 4k+4 - round 3
3623 eor $t9.8b, $t9.8b, $res3.8b @ GHASH block 4k+3 - mid
3625 aese $ctr1b, $rk3 \n aesmc $ctr1b, $ctr1b @ AES block 4k+5 - round 3
3627 pmull2 $t6.1q, $t6.2d, $h12k.2d @ GHASH block 4k+2 - mid
3628 eor $acc_lb, $acc_lb, $t5.16b @ GHASH block 4k+2 - low
3630 aese $ctr0b, $rk4 \n aesmc $ctr0b, $ctr0b @ AES block 4k+4 - round 4
3632 pmull2 $t7.1q, $res3.2d, $h1.2d @ GHASH block 4k+3 - high
3635 pmull $t9.1q, $t9.1d, $h12k.1d @ GHASH block 4k+3 - mid
3637 aese $ctr2b, $rk3 \n aesmc $ctr2b, $ctr2b @ AES block 4k+6 - round 3
3640 eor $acc_hb, $acc_hb, $t7.16b @ GHASH block 4k+3 - high
3642 aese $ctr0b, $rk5 \n aesmc $ctr0b, $ctr0b @ AES block 4k+4 - round 5
3643 eor $acc_mb, $acc_mb, $t6.16b @ GHASH block 4k+2 - mid
3645 aese $ctr2b, $rk4 \n aesmc $ctr2b, $ctr2b @ AES block 4k+6 - round 4
3648 eor $acc_lb, $acc_lb, $t8.16b @ GHASH block 4k+3 - low
3650 aese $ctr0b, $rk6 \n aesmc $ctr0b, $ctr0b @ AES block 4k+4 - round 6
3652 aese $ctr3b, $rk3 \n aesmc $ctr3b, $ctr3b @ AES block 4k+7 - round 3
3653 eor $acc_mb, $acc_mb, $t9.16b @ GHASH block 4k+3 - mid
3655 aese $ctr2b, $rk5 \n aesmc $ctr2b, $ctr2b @ AES block 4k+6 - round 5
3657 aese $ctr0b, $rk7 \n aesmc $ctr0b, $ctr0b @ AES block 4k+4 - round 7
3660 aese $ctr3b, $rk4 \n aesmc $ctr3b, $ctr3b @ AES block 4k+7 - round 4
3662 aese $ctr2b, $rk6 \n aesmc $ctr2b, $ctr2b @ AES block 4k+6 - round 6
3665 aese $ctr0b, $rk8 \n aesmc $ctr0b, $ctr0b @ AES block 4k+4 - round 8
3667 aese $ctr3b, $rk5 \n aesmc $ctr3b, $ctr3b @ AES block 4k+7 - round 5
3670 aese $ctr1b, $rk4 \n aesmc $ctr1b, $ctr1b @ AES block 4k+5 - round 4
3672 aese $ctr2b, $rk7 \n aesmc $ctr2b, $ctr2b @ AES block 4k+6 - round 7
3674 aese $ctr0b, $rk9 \n aesmc $ctr0b, $ctr0b @ AES block 4k+4 - round 9
3676 aese $ctr1b, $rk5 \n aesmc $ctr1b, $ctr1b @ AES block 4k+5 - round 5
3678 aese $ctr3b, $rk6 \n aesmc $ctr3b, $ctr3b @ AES block 4k+7 - round 6
3681 aese $ctr0b, $rk10 \n aesmc $ctr0b, $ctr0b @ AES block 4k+4 - round 10
3683 aese $ctr1b, $rk6 \n aesmc $ctr1b, $ctr1b @ AES block 4k+5 - round 6
3685 aese $ctr3b, $rk7 \n aesmc $ctr3b, $ctr3b @ AES block 4k+7 - round 7
3687 aese $ctr2b, $rk8 \n aesmc $ctr2b, $ctr2b @ AES block 4k+6 - round 8
3690 aese $ctr1b, $rk7 \n aesmc $ctr1b, $ctr1b @ AES block 4k+5 - round 7
3692 aese $ctr3b, $rk8 \n aesmc $ctr3b, $ctr3b @ AES block 4k+7 - round 8
3694 aese $ctr2b, $rk9 \n aesmc $ctr2b, $ctr2b @ AES block 4k+6 - round 9
3696 aese $ctr1b, $rk8 \n aesmc $ctr1b, $ctr1b @ AES block 4k+5 - round 8
3698 aese $ctr3b, $rk9 \n aesmc $ctr3b, $ctr3b @ AES block 4k+7 - round 9
3702 aese $ctr1b, $rk9 \n aesmc $ctr1b, $ctr1b @ AES block 4k+5 - round 9
3704 aese $ctr2b, $rk10 \n aesmc $ctr2b, $ctr2b @ AES block 4k+6 - round 10
3706 aese $ctr3b, $rk10 \n aesmc $ctr3b, $ctr3b @ AES block 4k+7 - round 10
3709 aese $ctr1b, $rk10 \n aesmc $ctr1b, $ctr1b @ AES block 4k+5 - round 10
3724 ld1 { $res1b}, [$input_ptr], #16 @ AES block 4k+4 - load ciphertext
3726 eor $ctr0b, $res1b, $ctr0b @ AES block 4k+4 - result
3728 mov $output_h0, $ctr0.d[1] @ AES block 4k+4 - mov high
3730 mov $output_l0, $ctr0.d[0] @ AES block 4k+4 - mov low
3736 eor $output_h0, $output_h0, $rk12_h @ AES block 4k+4 - round 12 high
3740 eor $output_l0, $output_l0, $rk12_l @ AES block 4k+4 - round 12 low
3766 rev64 $res0b, $res1b @ GHASH final-3 block
3767 ld1 { $res1b}, [$input_ptr], #16 @ AES final-2 block - load ciphertext
3769 stp $output_l0, $output_h0, [$output_ptr], #16 @ AES final-3 block - store result
3773 eor $ctr0b, $res1b, $ctr1b @ AES final-2 block - result
3775 pmull $acc_l.1q, $res0.1d, $h4.1d @ GHASH final-3 block - low
3776 mov $output_l0, $ctr0.d[0] @ AES final-2 block - mov low
3777 mov $rk4d, $res0.d[1] @ GHASH final-3 block - mid
3779 mov $output_h0, $ctr0.d[1] @ AES final-2 block - mov high
3781 mov $acc_md, $h34k.d[1] @ GHASH final-3 block - mid
3782 eor $rk4v.8b, $rk4v.8b, $res0.8b @ GHASH final-3 block - mid
3784 pmull2 $acc_h.1q, $res0.2d, $h4.2d @ GHASH final-3 block - high
3786 eor $output_l0, $output_l0, $rk12_l @ AES final-2 block - round 12 low
3792 pmull $acc_m.1q, $rk4v.1d, $acc_m.1d @ GHASH final-3 block - mid
3793 eor $output_h0, $output_h0, $rk12_h @ AES final-2 block - round 12 high
3799 rev64 $res0b, $res1b @ GHASH final-2 block
3800 ld1 { $res1b}, [$input_ptr], #16 @ AES final-1 block - load ciphertext
3806 eor $ctr0b, $res1b, $ctr2b @ AES final-1 block - result
3808 mov $rk4d, $res0.d[1] @ GHASH final-2 block - mid
3810 pmull $rk3q1, $res0.1d, $h3.1d @ GHASH final-2 block - low
3812 stp $output_l0, $output_h0, [$output_ptr], #16 @ AES final-2 block - store result
3814 eor $rk4v.8b, $rk4v.8b, $res0.8b @ GHASH final-2 block - mid
3815 mov $output_h0, $ctr0.d[1] @ AES final-1 block - mov high
3817 eor $acc_lb, $acc_lb, $rk3 @ GHASH final-2 block - low
3818 mov $output_l0, $ctr0.d[0] @ AES final-1 block - mov low
3820 pmull2 $rk2q1, $res0.2d, $h3.2d @ GHASH final-2 block - high
3822 pmull $rk4v.1q, $rk4v.1d, $h34k.1d @ GHASH final-2 block - mid
3824 eor $acc_hb, $acc_hb, $rk2 @ GHASH final-2 block - high
3825 eor $output_h0, $output_h0, $rk12_h @ AES final-1 block - round 12 high
3829 eor $output_l0, $output_l0, $rk12_l @ AES final-1 block - round 12 low
3833 eor $acc_mb, $acc_mb, $rk4v.16b @ GHASH final-2 block - mid
3836 rev64 $res0b, $res1b @ GHASH final-1 block
3839 ld1 { $res1b}, [$input_ptr], #16 @ AES final block - load ciphertext
3841 mov $rk4d, $res0.d[1] @ GHASH final-1 block - mid
3843 pmull2 $rk2q1, $res0.2d, $h2.2d @ GHASH final-1 block - high
3845 eor $ctr0b, $res1b, $ctr3b @ AES final block - result
3846 stp $output_l0, $output_h0, [$output_ptr], #16 @ AES final-1 block - store result
3848 eor $rk4v.8b, $rk4v.8b, $res0.8b @ GHASH final-1 block - mid
3850 eor $acc_hb, $acc_hb, $rk2 @ GHASH final-1 block - high
3852 pmull $rk3q1, $res0.1d, $h2.1d @ GHASH final-1 block - low
3853 mov $output_h0, $ctr0.d[1] @ AES final block - mov high
3855 ins $rk4v.d[1], $rk4v.d[0] @ GHASH final-1 block - mid
3856 mov $output_l0, $ctr0.d[0] @ AES final block - mov low
3858 pmull2 $rk4v.1q, $rk4v.2d, $h12k.2d @ GHASH final-1 block - mid
3861 eor $acc_lb, $acc_lb, $rk3 @ GHASH final-1 block - low
3862 eor $output_h0, $output_h0, $rk12_h @ AES final block - round 12 high
3866 eor $output_l0, $output_l0, $rk12_l @ AES final block - round 12 low
3870 eor $acc_mb, $acc_mb, $rk4v.16b @ GHASH final-1 block - mid
3884 lsr $rk12_h, $rk12_h, $bit_length @ rk12_h is mask for top 64b of last block
3890 fmov $ctr0d, $ctr32x @ ctr0b is mask for last block
3902 and $res1b, $res1b, $ctr0b @ possibly partial last block has zeroes in highest bits
3905 rev64 $res0b, $res1b @ GHASH final block
3912 pmull2 $rk2q1, $res0.2d, $h1.2d @ GHASH final block - high
3913 mov $t0d, $res0.d[1] @ GHASH final block - mid
3915 pmull $rk3q1, $res0.1d, $h1.1d @ GHASH final block - low
3917 eor $t0.8b, $t0.8b, $res0.8b @ GHASH final block - mid
3919 eor $acc_hb, $acc_hb, $rk2 @ GHASH final block - high
3921 pmull $t0.1q, $t0.1d, $h12k.1d @ GHASH final block - mid
3923 eor $acc_lb, $acc_lb, $rk3 @ GHASH final block - low
3925 eor $acc_mb, $acc_mb, $t0.16b @ GHASH final block - mid
4069 ld1 { $ctr0b}, [$counter] @ special case vector load initial counter so we can start first AES block as quickly as possible
4079 fmov $ctr2d, $ctr96_b64x @ CTR block 2
4084 fmov $ctr1d, $ctr96_b64x @ CTR block 1
4086 aese $ctr0b, $rk0 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 0
4089 rev $ctr32w, $rctr32w @ CTR block 1
4090 fmov $ctr3d, $ctr96_b64x @ CTR block 3
4092 orr $ctr32x, $ctr96_t32x, $ctr32x, lsl #32 @ CTR block 1
4093 add $rctr32w, $rctr32w, #1 @ CTR block 1
4096 fmov $ctr1.d[1], $ctr32x @ CTR block 1
4097 rev $ctr32w, $rctr32w @ CTR block 2
4098 add $rctr32w, $rctr32w, #1 @ CTR block 2
4100 orr $ctr32x, $ctr96_t32x, $ctr32x, lsl #32 @ CTR block 2
4103 fmov $ctr2.d[1], $ctr32x @ CTR block 2
4104 rev $ctr32w, $rctr32w @ CTR block 3
4106 aese $ctr0b, $rk1 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 1
4107 orr $ctr32x, $ctr96_t32x, $ctr32x, lsl #32 @ CTR block 3
4109 fmov $ctr3.d[1], $ctr32x @ CTR block 3
4111 aese $ctr1b, $rk0 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 0
4114 aese $ctr0b, $rk2 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 2
4117 aese $ctr2b, $rk0 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 0
4120 aese $ctr1b, $rk1 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 1
4125 aese $ctr3b, $rk0 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 0
4128 aese $ctr2b, $rk1 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 1
4131 aese $ctr1b, $rk2 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 2
4136 aese $ctr3b, $rk1 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 1
4139 aese $ctr2b, $rk2 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 2
4144 aese $ctr1b, $rk3 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 3
4147 aese $ctr3b, $rk2 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 2
4150 aese $ctr2b, $rk3 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 3
4151 add $rctr32w, $rctr32w, #1 @ CTR block 3
4153 aese $ctr0b, $rk3 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 3
4155 aese $ctr3b, $rk3 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 3
4160 aese $ctr2b, $rk4 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 4
4162 aese $ctr0b, $rk4 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 4
4164 aese $ctr1b, $rk4 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 4
4166 aese $ctr3b, $rk4 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 4
4168 aese $ctr0b, $rk5 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 5
4170 aese $ctr1b, $rk5 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 5
4172 aese $ctr3b, $rk5 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 5
4174 aese $ctr2b, $rk5 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 5
4176 aese $ctr1b, $rk6 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 6
4179 aese $ctr3b, $rk6 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 6
4182 aese $ctr0b, $rk6 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 6
4187 aese $ctr2b, $rk6 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 6
4190 aese $ctr1b, $rk7 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 7
4193 aese $ctr0b, $rk7 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 7
4195 aese $ctr2b, $rk7 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 7
4197 aese $ctr3b, $rk7 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 7
4200 aese $ctr1b, $rk8 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 8
4202 aese $ctr2b, $rk8 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 8
4204 aese $ctr3b, $rk8 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 8
4206 aese $ctr1b, $rk9 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 9
4208 aese $ctr2b, $rk9 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 9
4210 aese $ctr0b, $rk8 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 8
4212 aese $ctr1b, $rk10 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 10
4214 aese $ctr3b, $rk9 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 9
4216 aese $ctr0b, $rk9 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 9
4218 aese $ctr2b, $rk10 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 10
4220 aese $ctr3b, $rk10 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 10
4222 aese $ctr1b, $rk11 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 11
4224 aese $ctr2b, $rk11 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 11
4226 aese $ctr0b, $rk10 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 10
4228 aese $ctr1b, $rk12 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 12
4230 aese $ctr2b, $rk12 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 12
4232 aese $ctr0b, $rk11 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 11
4235 aese $ctr3b, $rk11 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 11
4237 aese $ctr2b, $rk13 @ AES block 2 - round 13
4240 aese $ctr0b, $rk12 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 12
4242 aese $ctr3b, $rk12 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 12
4244 aese $ctr1b, $rk13 @ AES block 1 - round 13
4246 aese $ctr0b, $rk13 @ AES block 0 - round 13
4248 aese $ctr3b, $rk13 @ AES block 3 - round 13
4252 ldp $input_l1, $input_h1, [$input_ptr, #16] @ AES block 1 - load plaintext
4257 rev $ctr32w, $rctr32w @ CTR block 4
4258 ldp $input_l0, $input_h0, [$input_ptr, #0] @ AES block 0 - load plaintext
4263 ldp $input_l3, $input_h3, [$input_ptr, #48] @ AES block 3 - load plaintext
4268 ldp $input_l2, $input_h2, [$input_ptr, #32] @ AES block 2 - load plaintext
4275 eor $input_l1, $input_l1, $rk14_l @ AES block 1 - round 14 low
4276 eor $input_h1, $input_h1, $rk14_h @ AES block 1 - round 14 high
4278 fmov $ctr_t1d, $input_l1 @ AES block 1 - mov low
4279 eor $input_l0, $input_l0, $rk14_l @ AES block 0 - round 14 low
4281 eor $input_h0, $input_h0, $rk14_h @ AES block 0 - round 14 high
4282 eor $input_h3, $input_h3, $rk14_h @ AES block 3 - round 14 high
4283 fmov $ctr_t0d, $input_l0 @ AES block 0 - mov low
4286 fmov $ctr_t0.d[1], $input_h0 @ AES block 0 - mov high
4287 eor $input_l3, $input_l3, $rk14_l @ AES block 3 - round 14 low
4289 eor $input_l2, $input_l2, $rk14_l @ AES block 2 - round 14 low
4290 fmov $ctr_t1.d[1], $input_h1 @ AES block 1 - mov high
4292 fmov $ctr_t2d, $input_l2 @ AES block 2 - mov low
4293 add $rctr32w, $rctr32w, #1 @ CTR block 4
4295 orr $ctr32x, $ctr96_t32x, $ctr32x, lsl #32 @ CTR block 4
4296 fmov $ctr_t3d, $input_l3 @ AES block 3 - mov low
4297 eor $input_h2, $input_h2, $rk14_h @ AES block 2 - round 14 high
4299 fmov $ctr_t2.d[1], $input_h2 @ AES block 2 - mov high
4301 eor $res0b, $ctr_t0b, $ctr0b @ AES block 0 - result
4302 fmov $ctr0d, $ctr96_b64x @ CTR block 4
4304 fmov $ctr0.d[1], $ctr32x @ CTR block 4
4305 rev $ctr32w, $rctr32w @ CTR block 5
4306 add $rctr32w, $rctr32w, #1 @ CTR block 5
4308 eor $res1b, $ctr_t1b, $ctr1b @ AES block 1 - result
4309 fmov $ctr1d, $ctr96_b64x @ CTR block 5
4310 orr $ctr32x, $ctr96_t32x, $ctr32x, lsl #32 @ CTR block 5
4312 fmov $ctr1.d[1], $ctr32x @ CTR block 5
4313 rev $ctr32w, $rctr32w @ CTR block 6
4314 st1 { $res0b}, [$output_ptr], #16 @ AES block 0 - store result
4316 fmov $ctr_t3.d[1], $input_h3 @ AES block 3 - mov high
4317 orr $ctr32x, $ctr96_t32x, $ctr32x, lsl #32 @ CTR block 6
4318 eor $res2b, $ctr_t2b, $ctr2b @ AES block 2 - result
4320 st1 { $res1b}, [$output_ptr], #16 @ AES block 1 - store result
4322 add $rctr32w, $rctr32w, #1 @ CTR block 6
4323 fmov $ctr2d, $ctr96_b64x @ CTR block 6
4325 fmov $ctr2.d[1], $ctr32x @ CTR block 6
4326 st1 { $res2b}, [$output_ptr], #16 @ AES block 2 - store result
4327 rev $ctr32w, $rctr32w @ CTR block 7
4329 orr $ctr32x, $ctr96_t32x, $ctr32x, lsl #32 @ CTR block 7
4331 eor $res3b, $ctr_t3b, $ctr3b @ AES block 3 - result
4332 st1 { $res3b}, [$output_ptr], #16 @ AES block 3 - store result
4336 aese $ctr0b, $rk0 \n aesmc $ctr0b, $ctr0b @ AES block 4k+4 - round 0
4337 rev64 $res0b, $res0b @ GHASH block 4k (only t0 is free)
4339 aese $ctr1b, $rk0 \n aesmc $ctr1b, $ctr1b @ AES block 4k+5 - round 0
4340 fmov $ctr3d, $ctr96_b64x @ CTR block 4k+3
4342 aese $ctr2b, $rk0 \n aesmc $ctr2b, $ctr2b @ AES block 4k+6 - round 0
4345 aese $ctr0b, $rk1 \n aesmc $ctr0b, $ctr0b @ AES block 4k+4 - round 1
4346 fmov $ctr3.d[1], $ctr32x @ CTR block 4k+3
4348 aese $ctr1b, $rk1 \n aesmc $ctr1b, $ctr1b @ AES block 4k+5 - round 1
4349 ldp $input_l3, $input_h3, [$input_ptr, #48] @ AES block 4k+7 - load plaintext
4354 aese $ctr2b, $rk1 \n aesmc $ctr2b, $ctr2b @ AES block 4k+6 - round 1
4355 ldp $input_l2, $input_h2, [$input_ptr, #32] @ AES block 4k+6 - load plaintext
4360 aese $ctr0b, $rk2 \n aesmc $ctr0b, $ctr0b @ AES block 4k+4 - round 2
4363 aese $ctr1b, $rk2 \n aesmc $ctr1b, $ctr1b @ AES block 4k+5 - round 2
4365 aese $ctr3b, $rk0 \n aesmc $ctr3b, $ctr3b @ AES block 4k+7 - round 0
4366 eor $input_l3, $input_l3, $rk14_l @ AES block 4k+7 - round 14 low
4368 aese $ctr0b, $rk3 \n aesmc $ctr0b, $ctr0b @ AES block 4k+4 - round 3
4369 mov $acc_md, $h34k.d[1] @ GHASH block 4k - mid
4371 pmull2 $acc_h.1q, $res0.2d, $h4.2d @ GHASH block 4k - high
4372 eor $input_h2, $input_h2, $rk14_h @ AES block 4k+6 - round 14 high
4373 mov $t0d, $res0.d[1] @ GHASH block 4k - mid
4375 aese $ctr3b, $rk1 \n aesmc $ctr3b, $ctr3b @ AES block 4k+7 - round 1
4376 rev64 $res1b, $res1b @ GHASH block 4k+1 (t0 and t1 free)
4378 aese $ctr0b, $rk4 \n aesmc $ctr0b, $ctr0b @ AES block 4k+4 - round 4
4380 pmull $acc_l.1q, $res0.1d, $h4.1d @ GHASH block 4k - low
4381 eor $t0.8b, $t0.8b, $res0.8b @ GHASH block 4k - mid
4383 aese $ctr2b, $rk2 \n aesmc $ctr2b, $ctr2b @ AES block 4k+6 - round 2
4385 aese $ctr0b, $rk5 \n aesmc $ctr0b, $ctr0b @ AES block 4k+4 - round 5
4386 rev64 $res3b, $res3b @ GHASH block 4k+3 (t0, t1, t2 and t3 free)
4388 pmull2 $t1.1q, $res1.2d, $h3.2d @ GHASH block 4k+1 - high
4390 pmull $acc_m.1q, $t0.1d, $acc_m.1d @ GHASH block 4k - mid
4391 rev64 $res2b, $res2b @ GHASH block 4k+2 (t0, t1, and t2 free)
4393 pmull $t2.1q, $res1.1d, $h3.1d @ GHASH block 4k+1 - low
4395 eor $acc_hb, $acc_hb, $t1.16b @ GHASH block 4k+1 - high
4396 mov $t3d, $res1.d[1] @ GHASH block 4k+1 - mid
4398 aese $ctr1b, $rk3 \n aesmc $ctr1b, $ctr1b @ AES block 4k+5 - round 3
4400 aese $ctr3b, $rk2 \n aesmc $ctr3b, $ctr3b @ AES block 4k+7 - round 2
4401 eor $acc_lb, $acc_lb, $t2.16b @ GHASH block 4k+1 - low
4403 aese $ctr2b, $rk3 \n aesmc $ctr2b, $ctr2b @ AES block 4k+6 - round 3
4405 aese $ctr1b, $rk4 \n aesmc $ctr1b, $ctr1b @ AES block 4k+5 - round 4
4406 mov $t6d, $res2.d[1] @ GHASH block 4k+2 - mid
4408 aese $ctr3b, $rk3 \n aesmc $ctr3b, $ctr3b @ AES block 4k+7 - round 3
4409 eor $t3.8b, $t3.8b, $res1.8b @ GHASH block 4k+1 - mid
4411 aese $ctr2b, $rk4 \n aesmc $ctr2b, $ctr2b @ AES block 4k+6 - round 4
4413 aese $ctr0b, $rk6 \n aesmc $ctr0b, $ctr0b @ AES block 4k+4 - round 6
4414 eor $t6.8b, $t6.8b, $res2.8b @ GHASH block 4k+2 - mid
4416 aese $ctr3b, $rk4 \n aesmc $ctr3b, $ctr3b @ AES block 4k+7 - round 4
4418 pmull $t3.1q, $t3.1d, $h34k.1d @ GHASH block 4k+1 - mid
4420 aese $ctr0b, $rk7 \n aesmc $ctr0b, $ctr0b @ AES block 4k+4 - round 7
4422 aese $ctr3b, $rk5 \n aesmc $ctr3b, $ctr3b @ AES block 4k+7 - round 5
4423 ins $t6.d[1], $t6.d[0] @ GHASH block 4k+2 - mid
4425 aese $ctr1b, $rk5 \n aesmc $ctr1b, $ctr1b @ AES block 4k+5 - round 5
4427 aese $ctr0b, $rk8 \n aesmc $ctr0b, $ctr0b @ AES block 4k+4 - round 8
4429 aese $ctr2b, $rk5 \n aesmc $ctr2b, $ctr2b @ AES block 4k+6 - round 5
4431 aese $ctr1b, $rk6 \n aesmc $ctr1b, $ctr1b @ AES block 4k+5 - round 6
4432 eor $acc_mb, $acc_mb, $t3.16b @ GHASH block 4k+1 - mid
4434 pmull2 $t4.1q, $res2.2d, $h2.2d @ GHASH block 4k+2 - high
4436 pmull $t5.1q, $res2.1d, $h2.1d @ GHASH block 4k+2 - low
4438 aese $ctr1b, $rk7 \n aesmc $ctr1b, $ctr1b @ AES block 4k+5 - round 7
4440 pmull $t8.1q, $res3.1d, $h1.1d @ GHASH block 4k+3 - low
4441 eor $acc_hb, $acc_hb, $t4.16b @ GHASH block 4k+2 - high
4443 aese $ctr3b, $rk6 \n aesmc $ctr3b, $ctr3b @ AES block 4k+7 - round 6
4444 ldp $input_l1, $input_h1, [$input_ptr, #16] @ AES block 4k+5 - load plaintext
4449 aese $ctr1b, $rk8 \n aesmc $ctr1b, $ctr1b @ AES block 4k+5 - round 8
4450 mov $t9d, $res3.d[1] @ GHASH block 4k+3 - mid
4452 aese $ctr2b, $rk6 \n aesmc $ctr2b, $ctr2b @ AES block 4k+6 - round 6
4453 eor $acc_lb, $acc_lb, $t5.16b @ GHASH block 4k+2 - low
4455 pmull2 $t6.1q, $t6.2d, $h12k.2d @ GHASH block 4k+2 - mid
4457 pmull2 $t7.1q, $res3.2d, $h1.2d @ GHASH block 4k+3 - high
4458 eor $t9.8b, $t9.8b, $res3.8b @ GHASH block 4k+3 - mid
4460 aese $ctr2b, $rk7 \n aesmc $ctr2b, $ctr2b @ AES block 4k+6 - round 7
4461 eor $input_l1, $input_l1, $rk14_l @ AES block 4k+5 - round 14 low
4463 aese $ctr1b, $rk9 \n aesmc $ctr1b, $ctr1b @ AES block 4k+5 - round 9
4464 eor $acc_mb, $acc_mb, $t6.16b @ GHASH block 4k+2 - mid
4466 aese $ctr3b, $rk7 \n aesmc $ctr3b, $ctr3b @ AES block 4k+7 - round 7
4467 eor $input_l2, $input_l2, $rk14_l @ AES block 4k+6 - round 14 low
4469 aese $ctr0b, $rk9 \n aesmc $ctr0b, $ctr0b @ AES block 4k+4 - round 9
4472 pmull $t9.1q, $t9.1d, $h12k.1d @ GHASH block 4k+3 - mid
4473 eor $acc_hb, $acc_hb, $t7.16b @ GHASH block 4k+3 - high
4474 fmov $ctr_t1d, $input_l1 @ AES block 4k+5 - mov low
4476 aese $ctr2b, $rk8 \n aesmc $ctr2b, $ctr2b @ AES block 4k+6 - round 8
4477 ldp $input_l0, $input_h0, [$input_ptr, #0] @ AES block 4k+4 - load plaintext
4482 aese $ctr0b, $rk10 \n aesmc $ctr0b, $ctr0b @ AES block 4k+4 - round 10
4485 aese $ctr3b, $rk8 \n aesmc $ctr3b, $ctr3b @ AES block 4k+7 - round 8
4486 eor $acc_lb, $acc_lb, $t8.16b @ GHASH block 4k+3 - low
4488 aese $ctr2b, $rk9 \n aesmc $ctr2b, $ctr2b @ AES block 4k+6 - round 9
4490 aese $ctr1b, $rk10 \n aesmc $ctr1b, $ctr1b @ AES block 4k+5 - round 10
4491 eor $acc_mb, $acc_mb, $t9.16b @ GHASH block 4k+3 - mid
4493 aese $ctr3b, $rk9 \n aesmc $ctr3b, $ctr3b @ AES block 4k+7 - round 9
4494 add $rctr32w, $rctr32w, #1 @ CTR block 4k+3
4496 aese $ctr0b, $rk11 \n aesmc $ctr0b, $ctr0b @ AES block 4k+4 - round 11
4499 aese $ctr1b, $rk11 \n aesmc $ctr1b, $ctr1b @ AES block 4k+5 - round 11
4503 rev $ctr32w, $rctr32w @ CTR block 4k+8
4506 aese $ctr2b, $rk10 \n aesmc $ctr2b, $ctr2b @ AES block 4k+6 - round 10
4507 eor $input_l0, $input_l0, $rk14_l @ AES block 4k+4 - round 14 low
4509 aese $ctr1b, $rk12 \n aesmc $ctr1b, $ctr1b @ AES block 4k+5 - round 12
4512 aese $ctr3b, $rk10 \n aesmc $ctr3b, $ctr3b @ AES block 4k+7 - round 10
4513 eor $input_h0, $input_h0, $rk14_h @ AES block 4k+4 - round 14 high
4515 fmov $ctr_t0d, $input_l0 @ AES block 4k+4 - mov low
4516 orr $ctr32x, $ctr96_t32x, $ctr32x, lsl #32 @ CTR block 4k+8
4519 aese $ctr0b, $rk12 \n aesmc $ctr0b, $ctr0b @ AES block 4k+4 - round 12
4520 eor $input_h1, $input_h1, $rk14_h @ AES block 4k+5 - round 14 high
4522 aese $ctr2b, $rk11 \n aesmc $ctr2b, $ctr2b @ AES block 4k+6 - round 11
4523 eor $input_h3, $input_h3, $rk14_h @ AES block 4k+7 - round 14 high
4525 aese $ctr3b, $rk11 \n aesmc $ctr3b, $ctr3b @ AES block 4k+7 - round 11
4526 add $rctr32w, $rctr32w, #1 @ CTR block 4k+8
4528 aese $ctr0b, $rk13 @ AES block 4k+4 - round 13
4529 fmov $ctr_t0.d[1], $input_h0 @ AES block 4k+4 - mov high
4532 aese $ctr2b, $rk12 \n aesmc $ctr2b, $ctr2b @ AES block 4k+6 - round 12
4533 fmov $ctr_t3d, $input_l3 @ AES block 4k+7 - mov low
4535 aese $ctr1b, $rk13 @ AES block 4k+5 - round 13
4536 fmov $ctr_t1.d[1], $input_h1 @ AES block 4k+5 - mov high
4538 fmov $ctr_t2d, $input_l2 @ AES block 4k+6 - mov low
4541 fmov $ctr_t2.d[1], $input_h2 @ AES block 4k+6 - mov high
4544 eor $res0b, $ctr_t0b, $ctr0b @ AES block 4k+4 - result
4545 fmov $ctr0d, $ctr96_b64x @ CTR block 4k+8
4547 fmov $ctr0.d[1], $ctr32x @ CTR block 4k+8
4548 rev $ctr32w, $rctr32w @ CTR block 4k+9
4549 add $rctr32w, $rctr32w, #1 @ CTR block 4k+9
4551 eor $res1b, $ctr_t1b, $ctr1b @ AES block 4k+5 - result
4552 fmov $ctr1d, $ctr96_b64x @ CTR block 4k+9
4553 orr $ctr32x, $ctr96_t32x, $ctr32x, lsl #32 @ CTR block 4k+9
4555 aese $ctr3b, $rk12 \n aesmc $ctr3b, $ctr3b @ AES block 4k+7 - round 12
4556 fmov $ctr1.d[1], $ctr32x @ CTR block 4k+9
4558 aese $ctr2b, $rk13 @ AES block 4k+6 - round 13
4559 rev $ctr32w, $rctr32w @ CTR block 4k+10
4560 st1 { $res0b}, [$output_ptr], #16 @ AES block 4k+4 - store result
4562 orr $ctr32x, $ctr96_t32x, $ctr32x, lsl #32 @ CTR block 4k+10
4564 fmov $ctr_t3.d[1], $input_h3 @ AES block 4k+7 - mov high
4567 st1 { $res1b}, [$output_ptr], #16 @ AES block 4k+5 - store result
4568 add $rctr32w, $rctr32w, #1 @ CTR block 4k+10
4570 aese $ctr3b, $rk13 @ AES block 4k+7 - round 13
4571 eor $res2b, $ctr_t2b, $ctr2b @ AES block 4k+6 - result
4572 fmov $ctr2d, $ctr96_b64x @ CTR block 4k+10
4574 st1 { $res2b}, [$output_ptr], #16 @ AES block 4k+6 - store result
4575 fmov $ctr2.d[1], $ctr32x @ CTR block 4k+10
4576 rev $ctr32w, $rctr32w @ CTR block 4k+11
4579 orr $ctr32x, $ctr96_t32x, $ctr32x, lsl #32 @ CTR block 4k+11
4581 eor $res3b, $ctr_t3b, $ctr3b @ AES block 4k+7 - result
4582 st1 { $res3b}, [$output_ptr], #16 @ AES block 4k+7 - store result
4586 aese $ctr1b, $rk0 \n aesmc $ctr1b, $ctr1b @ AES block 4k+5 - round 0
4587 rev64 $res2b, $res2b @ GHASH block 4k+2 (t0, t1, and t2 free)
4589 aese $ctr2b, $rk0 \n aesmc $ctr2b, $ctr2b @ AES block 4k+6 - round 0
4590 fmov $ctr3d, $ctr96_b64x @ CTR block 4k+3
4592 aese $ctr0b, $rk0 \n aesmc $ctr0b, $ctr0b @ AES block 4k+4 - round 0
4593 rev64 $res0b, $res0b @ GHASH block 4k (only t0 is free)
4595 fmov $ctr3.d[1], $ctr32x @ CTR block 4k+3
4598 aese $ctr2b, $rk1 \n aesmc $ctr2b, $ctr2b @ AES block 4k+6 - round 1
4600 aese $ctr0b, $rk1 \n aesmc $ctr0b, $ctr0b @ AES block 4k+4 - round 1
4603 rev64 $res1b, $res1b @ GHASH block 4k+1 (t0 and t1 free)
4605 aese $ctr2b, $rk2 \n aesmc $ctr2b, $ctr2b @ AES block 4k+6 - round 2
4607 aese $ctr3b, $rk0 \n aesmc $ctr3b, $ctr3b @ AES block 4k+7 - round 0
4608 mov $acc_md, $h34k.d[1] @ GHASH block 4k - mid
4610 aese $ctr1b, $rk1 \n aesmc $ctr1b, $ctr1b @ AES block 4k+5 - round 1
4612 pmull $acc_l.1q, $res0.1d, $h4.1d @ GHASH block 4k - low
4613 mov $t0d, $res0.d[1] @ GHASH block 4k - mid
4615 pmull2 $acc_h.1q, $res0.2d, $h4.2d @ GHASH block 4k - high
4617 aese $ctr2b, $rk3 \n aesmc $ctr2b, $ctr2b @ AES block 4k+6 - round 3
4619 aese $ctr1b, $rk2 \n aesmc $ctr1b, $ctr1b @ AES block 4k+5 - round 2
4620 eor $t0.8b, $t0.8b, $res0.8b @ GHASH block 4k - mid
4622 aese $ctr0b, $rk2 \n aesmc $ctr0b, $ctr0b @ AES block 4k+4 - round 2
4624 aese $ctr3b, $rk1 \n aesmc $ctr3b, $ctr3b @ AES block 4k+7 - round 1
4626 aese $ctr1b, $rk3 \n aesmc $ctr1b, $ctr1b @ AES block 4k+5 - round 3
4628 pmull $acc_m.1q, $t0.1d, $acc_m.1d @ GHASH block 4k - mid
4630 pmull2 $t1.1q, $res1.2d, $h3.2d @ GHASH block 4k+1 - high
4632 pmull $t2.1q, $res1.1d, $h3.1d @ GHASH block 4k+1 - low
4634 aese $ctr3b, $rk2 \n aesmc $ctr3b, $ctr3b @ AES block 4k+7 - round 2
4636 eor $acc_hb, $acc_hb, $t1.16b @ GHASH block 4k+1 - high
4637 mov $t3d, $res1.d[1] @ GHASH block 4k+1 - mid
4639 aese $ctr0b, $rk3 \n aesmc $ctr0b, $ctr0b @ AES block 4k+4 - round 3
4640 eor $acc_lb, $acc_lb, $t2.16b @ GHASH block 4k+1 - low
4642 aese $ctr3b, $rk3 \n aesmc $ctr3b, $ctr3b @ AES block 4k+7 - round 3
4644 eor $t3.8b, $t3.8b, $res1.8b @ GHASH block 4k+1 - mid
4645 mov $t6d, $res2.d[1] @ GHASH block 4k+2 - mid
4647 aese $ctr0b, $rk4 \n aesmc $ctr0b, $ctr0b @ AES block 4k+4 - round 4
4648 rev64 $res3b, $res3b @ GHASH block 4k+3 (t0, t1, t2 and t3 free)
4650 aese $ctr3b, $rk4 \n aesmc $ctr3b, $ctr3b @ AES block 4k+7 - round 4
4652 pmull $t3.1q, $t3.1d, $h34k.1d @ GHASH block 4k+1 - mid
4653 eor $t6.8b, $t6.8b, $res2.8b @ GHASH block 4k+2 - mid
4654 add $rctr32w, $rctr32w, #1 @ CTR block 4k+3
4656 pmull $t5.1q, $res2.1d, $h2.1d @ GHASH block 4k+2 - low
4658 aese $ctr3b, $rk5 \n aesmc $ctr3b, $ctr3b @ AES block 4k+7 - round 5
4660 aese $ctr2b, $rk4 \n aesmc $ctr2b, $ctr2b @ AES block 4k+6 - round 4
4661 eor $acc_mb, $acc_mb, $t3.16b @ GHASH block 4k+1 - mid
4663 pmull2 $t4.1q, $res2.2d, $h2.2d @ GHASH block 4k+2 - high
4665 eor $acc_lb, $acc_lb, $t5.16b @ GHASH block 4k+2 - low
4666 ins $t6.d[1], $t6.d[0] @ GHASH block 4k+2 - mid
4668 aese $ctr2b, $rk5 \n aesmc $ctr2b, $ctr2b @ AES block 4k+6 - round 5
4670 eor $acc_hb, $acc_hb, $t4.16b @ GHASH block 4k+2 - high
4671 mov $t9d, $res3.d[1] @ GHASH block 4k+3 - mid
4673 aese $ctr1b, $rk4 \n aesmc $ctr1b, $ctr1b @ AES block 4k+5 - round 4
4675 pmull2 $t6.1q, $t6.2d, $h12k.2d @ GHASH block 4k+2 - mid
4677 eor $t9.8b, $t9.8b, $res3.8b @ GHASH block 4k+3 - mid
4679 pmull2 $t7.1q, $res3.2d, $h1.2d @ GHASH block 4k+3 - high
4681 aese $ctr1b, $rk5 \n aesmc $ctr1b, $ctr1b @ AES block 4k+5 - round 5
4683 pmull $t9.1q, $t9.1d, $h12k.1d @ GHASH block 4k+3 - mid
4684 eor $acc_mb, $acc_mb, $t6.16b @ GHASH block 4k+2 - mid
4686 aese $ctr0b, $rk5 \n aesmc $ctr0b, $ctr0b @ AES block 4k+4 - round 5
4688 aese $ctr1b, $rk6 \n aesmc $ctr1b, $ctr1b @ AES block 4k+5 - round 6
4690 aese $ctr2b, $rk6 \n aesmc $ctr2b, $ctr2b @ AES block 4k+6 - round 6
4692 aese $ctr0b, $rk6 \n aesmc $ctr0b, $ctr0b @ AES block 4k+4 - round 6
4695 aese $ctr3b, $rk6 \n aesmc $ctr3b, $ctr3b @ AES block 4k+7 - round 6
4697 aese $ctr1b, $rk7 \n aesmc $ctr1b, $ctr1b @ AES block 4k+5 - round 7
4698 eor $acc_hb, $acc_hb, $t7.16b @ GHASH block 4k+3 - high
4700 aese $ctr0b, $rk7 \n aesmc $ctr0b, $ctr0b @ AES block 4k+4 - round 7
4702 aese $ctr3b, $rk7 \n aesmc $ctr3b, $ctr3b @ AES block 4k+7 - round 7
4705 aese $ctr1b, $rk8 \n aesmc $ctr1b, $ctr1b @ AES block 4k+5 - round 8
4706 eor $acc_mb, $acc_mb, $t9.16b @ GHASH block 4k+3 - mid
4708 pmull $t8.1q, $res3.1d, $h1.1d @ GHASH block 4k+3 - low
4710 aese $ctr3b, $rk8 \n aesmc $ctr3b, $ctr3b @ AES block 4k+7 - round 8
4712 aese $ctr1b, $rk9 \n aesmc $ctr1b, $ctr1b @ AES block 4k+5 - round 9
4714 aese $ctr0b, $rk8 \n aesmc $ctr0b, $ctr0b @ AES block 4k+4 - round 8
4715 eor $acc_lb, $acc_lb, $t8.16b @ GHASH block 4k+3 - low
4717 aese $ctr3b, $rk9 \n aesmc $ctr3b, $ctr3b @ AES block 4k+7 - round 9
4724 aese $ctr3b, $rk10 \n aesmc $ctr3b, $ctr3b @ AES block 4k+7 - round 10
4726 aese $ctr2b, $rk7 \n aesmc $ctr2b, $ctr2b @ AES block 4k+6 - round 7
4729 aese $ctr1b, $rk10 \n aesmc $ctr1b, $ctr1b @ AES block 4k+5 - round 10
4731 aese $ctr0b, $rk9 \n aesmc $ctr0b, $ctr0b @ AES block 4k+4 - round 9
4733 aese $ctr2b, $rk8 \n aesmc $ctr2b, $ctr2b @ AES block 4k+6 - round 8
4735 aese $ctr1b, $rk11 \n aesmc $ctr1b, $ctr1b @ AES block 4k+5 - round 11
4738 aese $ctr0b, $rk10 \n aesmc $ctr0b, $ctr0b @ AES block 4k+4 - round 10
4740 aese $ctr2b, $rk9 \n aesmc $ctr2b, $ctr2b @ AES block 4k+6 - round 9
4742 aese $ctr1b, $rk12 \n aesmc $ctr1b, $ctr1b @ AES block 4k+5 - round 12
4744 aese $ctr0b, $rk11 \n aesmc $ctr0b, $ctr0b @ AES block 4k+4 - round 11
4747 aese $ctr3b, $rk11 \n aesmc $ctr3b, $ctr3b @ AES block 4k+7 - round 11
4749 aese $ctr2b, $rk10 \n aesmc $ctr2b, $ctr2b @ AES block 4k+6 - round 10
4751 aese $ctr0b, $rk12 \n aesmc $ctr0b, $ctr0b @ AES block 4k+4 - round 12
4755 aese $ctr2b, $rk11 \n aesmc $ctr2b, $ctr2b @ AES block 4k+6 - round 11
4758 aese $ctr3b, $rk12 \n aesmc $ctr3b, $ctr3b @ AES block 4k+7 - round 12
4760 aese $ctr1b, $rk13 @ AES block 4k+5 - round 13
4763 aese $ctr2b, $rk12 \n aesmc $ctr2b, $ctr2b @ AES block 4k+6 - round 12
4765 aese $ctr3b, $rk13 @ AES block 4k+7 - round 13
4767 aese $ctr0b, $rk13 @ AES block 4k+4 - round 13
4769 aese $ctr2b, $rk13 @ AES block 4k+6 - round 13
4775 ldp $input_l0, $input_h0, [$input_ptr], #16 @ AES block 4k+4 - load plaintext
4780 eor $input_l0, $input_l0, $rk14_l @ AES block 4k+4 - round 14 low
4781 eor $input_h0, $input_h0, $rk14_h @ AES block 4k+4 - round 14 high
4784 fmov $ctr_t0d, $input_l0 @ AES block 4k+4 - mov low
4786 fmov $ctr_t0.d[1], $input_h0 @ AES block 4k+4 - mov high
4788 eor $res1b, $ctr_t0b, $ctr0b @ AES block 4k+4 - result
4811 st1 { $res1b}, [$output_ptr], #16 @ AES final-3 block - store result
4813 ldp $input_l0, $input_h0, [$input_ptr], #16 @ AES final-2 block - load input low & high
4818 rev64 $res0b, $res1b @ GHASH final-3 block
4820 eor $input_l0, $input_l0, $rk14_l @ AES final-2 block - round 14 low
4823 eor $input_h0, $input_h0, $rk14_h @ AES final-2 block - round 14 high
4825 mov $rk4d, $res0.d[1] @ GHASH final-3 block - mid
4826 fmov $res1d, $input_l0 @ AES final-2 block - mov low
4828 fmov $res1.d[1], $input_h0 @ AES final-2 block - mov high
4830 eor $rk4v.8b, $rk4v.8b, $res0.8b @ GHASH final-3 block - mid
4833 mov $acc_md, $h34k.d[1] @ GHASH final-3 block - mid
4835 pmull $acc_l.1q, $res0.1d, $h4.1d @ GHASH final-3 block - low
4837 pmull2 $acc_h.1q, $res0.2d, $h4.2d @ GHASH final-3 block - high
4839 pmull $acc_m.1q, $rk4v.1d, $acc_m.1d @ GHASH final-3 block - mid
4840 eor $res1b, $res1b, $ctr1b @ AES final-2 block - result
4843 st1 { $res1b}, [$output_ptr], #16 @ AES final-2 block - store result
4845 ldp $input_l0, $input_h0, [$input_ptr], #16 @ AES final-1 block - load input low & high
4850 rev64 $res0b, $res1b @ GHASH final-2 block
4852 eor $input_l0, $input_l0, $rk14_l @ AES final-1 block - round 14 low
4855 fmov $res1d, $input_l0 @ AES final-1 block - mov low
4856 eor $input_h0, $input_h0, $rk14_h @ AES final-1 block - round 14 high
4858 fmov $res1.d[1], $input_h0 @ AES final-1 block - mov high
4862 pmull2 $rk2q1, $res0.2d, $h3.2d @ GHASH final-2 block - high
4863 mov $rk4d, $res0.d[1] @ GHASH final-2 block - mid
4865 pmull $rk3q1, $res0.1d, $h3.1d @ GHASH final-2 block - low
4867 eor $rk4v.8b, $rk4v.8b, $res0.8b @ GHASH final-2 block - mid
4869 eor $res1b, $res1b, $ctr2b @ AES final-1 block - result
4871 eor $acc_hb, $acc_hb, $rk2 @ GHASH final-2 block - high
4873 pmull $rk4v.1q, $rk4v.1d, $h34k.1d @ GHASH final-2 block - mid
4875 eor $acc_lb, $acc_lb, $rk3 @ GHASH final-2 block - low
4877 eor $acc_mb, $acc_mb, $rk4v.16b @ GHASH final-2 block - mid
4880 st1 { $res1b}, [$output_ptr], #16 @ AES final-1 block - store result
4882 rev64 $res0b, $res1b @ GHASH final-1 block
4884 ldp $input_l0, $input_h0, [$input_ptr], #16 @ AES final block - load input low & high
4893 eor $input_l0, $input_l0, $rk14_l @ AES final block - round 14 low
4894 mov $rk4d, $res0.d[1] @ GHASH final-1 block - mid
4896 pmull2 $rk2q1, $res0.2d, $h2.2d @ GHASH final-1 block - high
4897 eor $input_h0, $input_h0, $rk14_h @ AES final block - round 14 high
4899 eor $rk4v.8b, $rk4v.8b, $res0.8b @ GHASH final-1 block - mid
4901 eor $acc_hb, $acc_hb, $rk2 @ GHASH final-1 block - high
4903 ins $rk4v.d[1], $rk4v.d[0] @ GHASH final-1 block - mid
4904 fmov $res1d, $input_l0 @ AES final block - mov low
4906 fmov $res1.d[1], $input_h0 @ AES final block - mov high
4908 pmull2 $rk4v.1q, $rk4v.2d, $h12k.2d @ GHASH final-1 block - mid
4910 pmull $rk3q1, $res0.1d, $h2.1d @ GHASH final-1 block - low
4912 eor $res1b, $res1b, $ctr3b @ AES final block - result
4913 eor $acc_mb, $acc_mb, $rk4v.16b @ GHASH final-1 block - mid
4915 eor $acc_lb, $acc_lb, $rk3 @ GHASH final-1 block - low
4924 ld1 { $rk0}, [$output_ptr] @ load existing bytes where the possibly partial last block is to be stored
4929 lsr $rk14_h, $rk14_h, $bit_length @ rk14_h is mask for top 64b of last block
4935 fmov $ctr0d, $input_l0 @ ctr0b is mask for last block
4939 and $res1b, $res1b, $ctr0b @ possibly partial last block has zeroes in highest bits
4941 rev64 $res0b, $res1b @ GHASH final block
4947 pmull2 $rk2q1, $res0.2d, $h1.2d @ GHASH final block - high
4948 mov $t0d, $res0.d[1] @ GHASH final block - mid
4955 pmull $rk3q1, $res0.1d, $h1.1d @ GHASH final block - low
4957 eor $acc_hb, $acc_hb, $rk2 @ GHASH final block - high
4958 eor $t0.8b, $t0.8b, $res0.8b @ GHASH final block - mid
4960 pmull $t0.1q, $t0.1d, $h12k.1d @ GHASH final block - mid
4962 eor $acc_lb, $acc_lb, $rk3 @ GHASH final block - low
4964 eor $acc_mb, $acc_mb, $t0.16b @ GHASH final block - mid
5070 fmov $ctr3d, $ctr96_b64x @ CTR block 3
5072 rev $ctr32w, $rctr32w @ CTR block 1
5073 add $rctr32w, $rctr32w, #1 @ CTR block 1
5074 fmov $ctr1d, $ctr96_b64x @ CTR block 1
5076 orr $ctr32x, $ctr96_t32x, $ctr32x, lsl #32 @ CTR block 1
5077 ld1 { $ctr0b}, [$counter] @ special case vector load initial counter so we can start first AES block as quickly as possible
5079 fmov $ctr1.d[1], $ctr32x @ CTR block 1
5080 rev $ctr32w, $rctr32w @ CTR block 2
5081 add $rctr32w, $rctr32w, #1 @ CTR block 2
5083 fmov $ctr2d, $ctr96_b64x @ CTR block 2
5084 orr $ctr32x, $ctr96_t32x, $ctr32x, lsl #32 @ CTR block 2
5086 fmov $ctr2.d[1], $ctr32x @ CTR block 2
5087 rev $ctr32w, $rctr32w @ CTR block 3
5089 orr $ctr32x, $ctr96_t32x, $ctr32x, lsl #32 @ CTR block 3
5092 fmov $ctr3.d[1], $ctr32x @ CTR block 3
5093 add $rctr32w, $rctr32w, #1 @ CTR block 3
5101 aese $ctr0b, $rk0 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 0
5107 aese $ctr3b, $rk0 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 0
5113 aese $ctr1b, $rk0 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 0
5119 aese $ctr2b, $rk0 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 0
5122 aese $ctr0b, $rk1 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 1
5124 aese $ctr1b, $rk1 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 1
5129 aese $ctr2b, $rk1 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 1
5132 aese $ctr3b, $rk1 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 1
5135 aese $ctr0b, $rk2 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 2
5140 aese $ctr2b, $rk2 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 2
5143 aese $ctr3b, $rk2 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 2
5145 aese $ctr0b, $rk3 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 3
5147 aese $ctr1b, $rk2 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 2
5149 aese $ctr3b, $rk3 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 3
5151 aese $ctr0b, $rk4 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 4
5154 aese $ctr2b, $rk3 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 3
5156 aese $ctr1b, $rk3 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 3
5158 aese $ctr3b, $rk4 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 4
5160 aese $ctr2b, $rk4 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 4
5162 aese $ctr1b, $rk4 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 4
5164 aese $ctr3b, $rk5 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 5
5166 aese $ctr0b, $rk5 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 5
5168 aese $ctr1b, $rk5 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 5
5170 aese $ctr2b, $rk5 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 5
5172 aese $ctr0b, $rk6 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 6
5174 aese $ctr3b, $rk6 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 6
5176 aese $ctr1b, $rk6 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 6
5178 aese $ctr2b, $rk6 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 6
5180 aese $ctr0b, $rk7 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 7
5182 aese $ctr1b, $rk7 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 7
5184 aese $ctr3b, $rk7 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 7
5186 aese $ctr0b, $rk8 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 8
5188 aese $ctr2b, $rk7 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 7
5190 aese $ctr3b, $rk8 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 8
5192 aese $ctr1b, $rk8 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 8
5194 aese $ctr0b, $rk9 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 9
5196 aese $ctr2b, $rk8 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 8
5199 aese $ctr1b, $rk9 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 9
5201 aese $ctr0b, $rk10 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 10
5203 aese $ctr3b, $rk9 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 9
5205 aese $ctr1b, $rk10 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 10
5207 aese $ctr2b, $rk9 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 9
5209 aese $ctr3b, $rk10 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 10
5211 aese $ctr0b, $rk11 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 11
5213 aese $ctr2b, $rk10 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 10
5215 aese $ctr3b, $rk11 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 11
5217 aese $ctr1b, $rk11 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 11
5219 aese $ctr2b, $rk11 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 11
5228 aese $ctr1b, $rk12 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 12
5230 aese $ctr0b, $rk12 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 12
5232 aese $ctr2b, $rk12 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 12
5234 aese $ctr3b, $rk12 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 12
5237 aese $ctr1b, $rk13 @ AES block 1 - round 13
5239 aese $ctr2b, $rk13 @ AES block 2 - round 13
5242 aese $ctr3b, $rk13 @ AES block 3 - round 13
5244 aese $ctr0b, $rk13 @ AES block 0 - round 13
5247 ld1 {$res0b, $res1b}, [$input_ptr], #32 @ AES block 0,1 - load ciphertext
5249 rev $ctr32w, $rctr32w @ CTR block 4
5251 eor $ctr0b, $res0b, $ctr0b @ AES block 0 - result
5253 eor $ctr1b, $res1b, $ctr1b @ AES block 1 - result
5254 rev64 $res1b, $res1b @ GHASH block 1
5255 ld1 {$res2b}, [$input_ptr], #16 @ AES block 2 - load ciphertext
5257 mov $output_h0, $ctr0.d[1] @ AES block 0 - mov high
5259 mov $output_l0, $ctr0.d[0] @ AES block 0 - mov low
5260 rev64 $res0b, $res0b @ GHASH block 0
5261 add $rctr32w, $rctr32w, #1 @ CTR block 4
5263 fmov $ctr0d, $ctr96_b64x @ CTR block 4
5264 orr $ctr32x, $ctr96_t32x, $ctr32x, lsl #32 @ CTR block 4
5266 fmov $ctr0.d[1], $ctr32x @ CTR block 4
5267 rev $ctr32w, $rctr32w @ CTR block 5
5268 add $rctr32w, $rctr32w, #1 @ CTR block 5
5270 mov $output_l1, $ctr1.d[0] @ AES block 1 - mov low
5272 orr $ctr32x, $ctr96_t32x, $ctr32x, lsl #32 @ CTR block 5
5273 mov $output_h1, $ctr1.d[1] @ AES block 1 - mov high
5274 eor $output_h0, $output_h0, $rk14_h @ AES block 0 - round 14 high
5278 eor $output_l0, $output_l0, $rk14_l @ AES block 0 - round 14 low
5282 stp $output_l0, $output_h0, [$output_ptr], #16 @ AES block 0 - store result
5283 fmov $ctr1d, $ctr96_b64x @ CTR block 5
5285 ld1 {$res3b}, [$input_ptr], #16 @ AES block 3 - load ciphertext
5287 fmov $ctr1.d[1], $ctr32x @ CTR block 5
5288 rev $ctr32w, $rctr32w @ CTR block 6
5289 add $rctr32w, $rctr32w, #1 @ CTR block 6
5291 eor $output_l1, $output_l1, $rk14_l @ AES block 1 - round 14 low
5295 orr $ctr32x, $ctr96_t32x, $ctr32x, lsl #32 @ CTR block 6
5297 eor $output_h1, $output_h1, $rk14_h @ AES block 1 - round 14 high
5301 stp $output_l1, $output_h1, [$output_ptr], #16 @ AES block 1 - store result
5303 eor $ctr2b, $res2b, $ctr2b @ AES block 2 - result
5308 mov $output_l2, $ctr2.d[0] @ AES block 4k+2 - mov low
5310 eor $ctr3b, $res3b, $ctr3b @ AES block 4k+3 - result
5312 aese $ctr0b, $rk0 \n aesmc $ctr0b, $ctr0b @ AES block 4k+4 - round 0
5313 mov $output_h2, $ctr2.d[1] @ AES block 4k+2 - mov high
5315 aese $ctr1b, $rk0 \n aesmc $ctr1b, $ctr1b @ AES block 4k+5 - round 0
5316 fmov $ctr2d, $ctr96_b64x @ CTR block 4k+6
5318 fmov $ctr2.d[1], $ctr32x @ CTR block 4k+6
5320 rev $ctr32w, $rctr32w @ CTR block 4k+7
5322 aese $ctr0b, $rk1 \n aesmc $ctr0b, $ctr0b @ AES block 4k+4 - round 1
5323 mov $output_h3, $ctr3.d[1] @ AES block 4k+3 - mov high
5325 aese $ctr1b, $rk1 \n aesmc $ctr1b, $ctr1b @ AES block 4k+5 - round 1
5326 mov $output_l3, $ctr3.d[0] @ AES block 4k+3 - mov low
5328 pmull2 $acc_h.1q, $res0.2d, $h4.2d @ GHASH block 4k - high
5329 mov $t0d, $res0.d[1] @ GHASH block 4k - mid
5330 fmov $ctr3d, $ctr96_b64x @ CTR block 4k+7
5332 aese $ctr0b, $rk2 \n aesmc $ctr0b, $ctr0b @ AES block 4k+4 - round 2
5333 orr $ctr32x, $ctr96_t32x, $ctr32x, lsl #32 @ CTR block 4k+7
5335 aese $ctr2b, $rk0 \n aesmc $ctr2b, $ctr2b @ AES block 4k+6 - round 0
5336 fmov $ctr3.d[1], $ctr32x @ CTR block 4k+7
5338 aese $ctr1b, $rk2 \n aesmc $ctr1b, $ctr1b @ AES block 4k+5 - round 2
5339 eor $t0.8b, $t0.8b, $res0.8b @ GHASH block 4k - mid
5341 aese $ctr0b, $rk3 \n aesmc $ctr0b, $ctr0b @ AES block 4k+4 - round 3
5342 eor $output_h2, $output_h2, $rk14_h @ AES block 4k+2 - round 14 high
5346 aese $ctr2b, $rk1 \n aesmc $ctr2b, $ctr2b @ AES block 4k+6 - round 1
5347 mov $acc_md, $h34k.d[1] @ GHASH block 4k - mid
5349 aese $ctr1b, $rk3 \n aesmc $ctr1b, $ctr1b @ AES block 4k+5 - round 3
5350 rev64 $res2b, $res2b @ GHASH block 4k+2
5352 aese $ctr3b, $rk0 \n aesmc $ctr3b, $ctr3b @ AES block 4k+7 - round 0
5353 eor $output_l2, $output_l2, $rk14_l @ AES block 4k+2 - round 14 low
5357 aese $ctr2b, $rk2 \n aesmc $ctr2b, $ctr2b @ AES block 4k+6 - round 2
5358 stp $output_l2, $output_h2, [$output_ptr], #16 @ AES block 4k+2 - store result
5360 pmull $acc_l.1q, $res0.1d, $h4.1d @ GHASH block 4k - low
5362 pmull2 $t1.1q, $res1.2d, $h3.2d @ GHASH block 4k+1 - high
5364 aese $ctr2b, $rk3 \n aesmc $ctr2b, $ctr2b @ AES block 4k+6 - round 3
5365 rev64 $res3b, $res3b @ GHASH block 4k+3
5367 pmull $acc_m.1q, $t0.1d, $acc_m.1d @ GHASH block 4k - mid
5368 eor $output_l3, $output_l3, $rk14_l @ AES block 4k+3 - round 14 low
5372 pmull $t2.1q, $res1.1d, $h3.1d @ GHASH block 4k+1 - low
5373 eor $output_h3, $output_h3, $rk14_h @ AES block 4k+3 - round 14 high
5377 eor $acc_hb, $acc_hb, $t1.16b @ GHASH block 4k+1 - high
5379 aese $ctr2b, $rk4 \n aesmc $ctr2b, $ctr2b @ AES block 4k+6 - round 4
5381 aese $ctr3b, $rk1 \n aesmc $ctr3b, $ctr3b @ AES block 4k+7 - round 1
5382 mov $t3d, $res1.d[1] @ GHASH block 4k+1 - mid
5384 aese $ctr0b, $rk4 \n aesmc $ctr0b, $ctr0b @ AES block 4k+4 - round 4
5385 eor $acc_lb, $acc_lb, $t2.16b @ GHASH block 4k+1 - low
5387 aese $ctr2b, $rk5 \n aesmc $ctr2b, $ctr2b @ AES block 4k+6 - round 5
5388 add $rctr32w, $rctr32w, #1 @ CTR block 4k+7
5390 aese $ctr3b, $rk2 \n aesmc $ctr3b, $ctr3b @ AES block 4k+7 - round 2
5391 mov $t6d, $res2.d[1] @ GHASH block 4k+2 - mid
5393 aese $ctr1b, $rk4 \n aesmc $ctr1b, $ctr1b @ AES block 4k+5 - round 4
5394 eor $t3.8b, $t3.8b, $res1.8b @ GHASH block 4k+1 - mid
5396 pmull $t5.1q, $res2.1d, $h2.1d @ GHASH block 4k+2 - low
5398 aese $ctr3b, $rk3 \n aesmc $ctr3b, $ctr3b @ AES block 4k+7 - round 3
5399 eor $t6.8b, $t6.8b, $res2.8b @ GHASH block 4k+2 - mid
5401 aese $ctr1b, $rk5 \n aesmc $ctr1b, $ctr1b @ AES block 4k+5 - round 5
5403 aese $ctr0b, $rk5 \n aesmc $ctr0b, $ctr0b @ AES block 4k+4 - round 5
5404 eor $acc_lb, $acc_lb, $t5.16b @ GHASH block 4k+2 - low
5406 pmull $t3.1q, $t3.1d, $h34k.1d @ GHASH block 4k+1 - mid
5407 rev $ctr32w, $rctr32w @ CTR block 4k+8
5409 aese $ctr1b, $rk6 \n aesmc $ctr1b, $ctr1b @ AES block 4k+5 - round 6
5410 ins $t6.d[1], $t6.d[0] @ GHASH block 4k+2 - mid
5412 aese $ctr0b, $rk6 \n aesmc $ctr0b, $ctr0b @ AES block 4k+4 - round 6
5413 add $rctr32w, $rctr32w, #1 @ CTR block 4k+8
5415 aese $ctr3b, $rk4 \n aesmc $ctr3b, $ctr3b @ AES block 4k+7 - round 4
5417 aese $ctr1b, $rk7 \n aesmc $ctr1b, $ctr1b @ AES block 4k+5 - round 7
5418 eor $acc_mb, $acc_mb, $t3.16b @ GHASH block 4k+1 - mid
5420 aese $ctr0b, $rk7 \n aesmc $ctr0b, $ctr0b @ AES block 4k+4 - round 7
5422 pmull2 $t4.1q, $res2.2d, $h2.2d @ GHASH block 4k+2 - high
5423 mov $t9d, $res3.d[1] @ GHASH block 4k+3 - mid
5425 aese $ctr3b, $rk5 \n aesmc $ctr3b, $ctr3b @ AES block 4k+7 - round 5
5427 pmull2 $t6.1q, $t6.2d, $h12k.2d @ GHASH block 4k+2 - mid
5429 aese $ctr0b, $rk8 \n aesmc $ctr0b, $ctr0b @ AES block 4k+4 - round 8
5430 eor $acc_hb, $acc_hb, $t4.16b @ GHASH block 4k+2 - high
5432 aese $ctr3b, $rk6 \n aesmc $ctr3b, $ctr3b @ AES block 4k+7 - round 6
5434 pmull $t8.1q, $res3.1d, $h1.1d @ GHASH block 4k+3 - low
5435 orr $ctr32x, $ctr96_t32x, $ctr32x, lsl #32 @ CTR block 4k+8
5436 eor $acc_mb, $acc_mb, $t6.16b @ GHASH block 4k+2 - mid
5438 pmull2 $t7.1q, $res3.2d, $h1.2d @ GHASH block 4k+3 - high
5440 aese $ctr0b, $rk9 \n aesmc $ctr0b, $ctr0b @ AES block 4k+4 - round 9
5441 eor $t9.8b, $t9.8b, $res3.8b @ GHASH block 4k+3 - mid
5443 aese $ctr1b, $rk8 \n aesmc $ctr1b, $ctr1b @ AES block 4k+5 - round 8
5445 aese $ctr2b, $rk6 \n aesmc $ctr2b, $ctr2b @ AES block 4k+6 - round 6
5446 eor $acc_hb, $acc_hb, $t7.16b @ GHASH block 4k+3 - high
5448 aese $ctr0b, $rk10 \n aesmc $ctr0b, $ctr0b @ AES block 4k+4 - round 10
5450 pmull $t9.1q, $t9.1d, $h12k.1d @ GHASH block 4k+3 - mid
5453 aese $ctr2b, $rk7 \n aesmc $ctr2b, $ctr2b @ AES block 4k+6 - round 7
5454 eor $acc_lb, $acc_lb, $t8.16b @ GHASH block 4k+3 - low
5456 aese $ctr0b, $rk11 \n aesmc $ctr0b, $ctr0b @ AES block 4k+4 - round 11
5458 aese $ctr3b, $rk7 \n aesmc $ctr3b, $ctr3b @ AES block 4k+7 - round 7
5461 aese $ctr2b, $rk8 \n aesmc $ctr2b, $ctr2b @ AES block 4k+6 - round 8
5462 eor $acc_mb, $acc_mb, $t9.16b @ GHASH block 4k+3 - mid
5464 aese $ctr0b, $rk12 \n aesmc $ctr0b, $ctr0b @ AES block 4k+4 - round 12
5469 aese $ctr1b, $rk9 \n aesmc $ctr1b, $ctr1b @ AES block 4k+5 - round 9
5470 ld1 {$res0b}, [$input_ptr], #16 @ AES block 4k+4 - load ciphertext
5472 aese $ctr0b, $rk13 @ AES block 4k+4 - round 13
5475 aese $ctr1b, $rk10 \n aesmc $ctr1b, $ctr1b @ AES block 4k+5 - round 10
5478 aese $ctr2b, $rk9 \n aesmc $ctr2b, $ctr2b @ AES block 4k+6 - round 9
5479 ld1 {$res1b}, [$input_ptr], #16 @ AES block 4k+5 - load ciphertext
5481 aese $ctr3b, $rk8 \n aesmc $ctr3b, $ctr3b @ AES block 4k+7 - round 8
5482 eor $ctr0b, $res0b, $ctr0b @ AES block 4k+4 - result
5484 aese $ctr1b, $rk11 \n aesmc $ctr1b, $ctr1b @ AES block 4k+5 - round 11
5485 stp $output_l3, $output_h3, [$output_ptr], #16 @ AES block 4k+3 - store result
5487 aese $ctr2b, $rk10 \n aesmc $ctr2b, $ctr2b @ AES block 4k+6 - round 10
5490 aese $ctr3b, $rk9 \n aesmc $ctr3b, $ctr3b @ AES block 4k+7 - round 9
5491 ld1 {$res2b}, [$input_ptr], #16 @ AES block 4k+6 - load ciphertext
5493 aese $ctr1b, $rk12 \n aesmc $ctr1b, $ctr1b @ AES block 4k+5 - round 12
5494 ld1 {$res3b}, [$input_ptr], #16 @ AES block 4k+7 - load ciphertext
5496 aese $ctr2b, $rk11 \n aesmc $ctr2b, $ctr2b @ AES block 4k+6 - round 11
5497 mov $output_h0, $ctr0.d[1] @ AES block 4k+4 - mov high
5499 aese $ctr3b, $rk10 \n aesmc $ctr3b, $ctr3b @ AES block 4k+7 - round 10
5502 aese $ctr1b, $rk13 @ AES block 4k+5 - round 13
5503 mov $output_l0, $ctr0.d[0] @ AES block 4k+4 - mov low
5505 aese $ctr2b, $rk12 \n aesmc $ctr2b, $ctr2b @ AES block 4k+6 - round 12
5506 fmov $ctr0d, $ctr96_b64x @ CTR block 4k+8
5508 aese $ctr3b, $rk11 \n aesmc $ctr3b, $ctr3b @ AES block 4k+7 - round 11
5509 fmov $ctr0.d[1], $ctr32x @ CTR block 4k+8
5512 eor $ctr1b, $res1b, $ctr1b @ AES block 4k+5 - result
5513 rev $ctr32w, $rctr32w @ CTR block 4k+9
5515 aese $ctr2b, $rk13 @ AES block 4k+6 - round 13
5516 orr $ctr32x, $ctr96_t32x, $ctr32x, lsl #32 @ CTR block 4k+9
5519 add $rctr32w, $rctr32w, #1 @ CTR block 4k+9
5521 eor $output_l0, $output_l0, $rk14_l @ AES block 4k+4 - round 14 low
5525 eor $output_h0, $output_h0, $rk14_h @ AES block 4k+4 - round 14 high
5529 mov $output_h1, $ctr1.d[1] @ AES block 4k+5 - mov high
5530 eor $ctr2b, $res2b, $ctr2b @ AES block 4k+6 - result
5533 aese $ctr3b, $rk12 \n aesmc $ctr3b, $ctr3b @ AES block 4k+7 - round 12
5534 mov $output_l1, $ctr1.d[0] @ AES block 4k+5 - mov low
5536 fmov $ctr1d, $ctr96_b64x @ CTR block 4k+9
5539 fmov $ctr1.d[1], $ctr32x @ CTR block 4k+9
5540 rev $ctr32w, $rctr32w @ CTR block 4k+10
5541 add $rctr32w, $rctr32w, #1 @ CTR block 4k+10
5543 aese $ctr3b, $rk13 @ AES block 4k+7 - round 13
5544 orr $ctr32x, $ctr96_t32x, $ctr32x, lsl #32 @ CTR block 4k+10
5546 rev64 $res1b, $res1b @ GHASH block 4k+5
5547 eor $output_h1, $output_h1, $rk14_h @ AES block 4k+5 - round 14 high
5551 stp $output_l0, $output_h0, [$output_ptr], #16 @ AES block 4k+4 - store result
5553 eor $output_l1, $output_l1, $rk14_l @ AES block 4k+5 - round 14 low
5557 stp $output_l1, $output_h1, [$output_ptr], #16 @ AES block 4k+5 - store result
5559 rev64 $res0b, $res0b @ GHASH block 4k+4
5566 mov $output_l2, $ctr2.d[0] @ AES block 4k+2 - mov low
5567 eor $ctr3b, $res3b, $ctr3b @ AES block 4k+3 - result
5569 aese $ctr0b, $rk0 \n aesmc $ctr0b, $ctr0b @ AES block 4k+4 - round 0
5570 mov $output_h2, $ctr2.d[1] @ AES block 4k+2 - mov high
5572 aese $ctr1b, $rk0 \n aesmc $ctr1b, $ctr1b @ AES block 4k+5 - round 0
5573 fmov $ctr2d, $ctr96_b64x @ CTR block 4k+6
5575 fmov $ctr2.d[1], $ctr32x @ CTR block 4k+6
5576 rev $ctr32w, $rctr32w @ CTR block 4k+7
5579 rev64 $res2b, $res2b @ GHASH block 4k+2
5580 orr $ctr32x, $ctr96_t32x, $ctr32x, lsl #32 @ CTR block 4k+7
5581 mov $output_l3, $ctr3.d[0] @ AES block 4k+3 - mov low
5583 aese $ctr1b, $rk1 \n aesmc $ctr1b, $ctr1b @ AES block 4k+5 - round 1
5584 mov $output_h3, $ctr3.d[1] @ AES block 4k+3 - mov high
5586 pmull $acc_l.1q, $res0.1d, $h4.1d @ GHASH block 4k - low
5587 mov $t0d, $res0.d[1] @ GHASH block 4k - mid
5588 fmov $ctr3d, $ctr96_b64x @ CTR block 4k+7
5590 pmull2 $acc_h.1q, $res0.2d, $h4.2d @ GHASH block 4k - high
5591 fmov $ctr3.d[1], $ctr32x @ CTR block 4k+7
5593 aese $ctr2b, $rk0 \n aesmc $ctr2b, $ctr2b @ AES block 4k+6 - round 0
5594 mov $acc_md, $h34k.d[1] @ GHASH block 4k - mid
5596 aese $ctr0b, $rk1 \n aesmc $ctr0b, $ctr0b @ AES block 4k+4 - round 1
5597 eor $t0.8b, $t0.8b, $res0.8b @ GHASH block 4k - mid
5599 pmull2 $t1.1q, $res1.2d, $h3.2d @ GHASH block 4k+1 - high
5601 aese $ctr2b, $rk1 \n aesmc $ctr2b, $ctr2b @ AES block 4k+6 - round 1
5602 rev64 $res3b, $res3b @ GHASH block 4k+3
5604 aese $ctr3b, $rk0 \n aesmc $ctr3b, $ctr3b @ AES block 4k+7 - round 0
5606 pmull $acc_m.1q, $t0.1d, $acc_m.1d @ GHASH block 4k - mid
5607 eor $acc_hb, $acc_hb, $t1.16b @ GHASH block 4k+1 - high
5609 pmull $t2.1q, $res1.1d, $h3.1d @ GHASH block 4k+1 - low
5611 aese $ctr3b, $rk1 \n aesmc $ctr3b, $ctr3b @ AES block 4k+7 - round 1
5612 mov $t3d, $res1.d[1] @ GHASH block 4k+1 - mid
5614 aese $ctr0b, $rk2 \n aesmc $ctr0b, $ctr0b @ AES block 4k+4 - round 2
5616 aese $ctr1b, $rk2 \n aesmc $ctr1b, $ctr1b @ AES block 4k+5 - round 2
5617 eor $acc_lb, $acc_lb, $t2.16b @ GHASH block 4k+1 - low
5619 aese $ctr2b, $rk2 \n aesmc $ctr2b, $ctr2b @ AES block 4k+6 - round 2
5621 aese $ctr0b, $rk3 \n aesmc $ctr0b, $ctr0b @ AES block 4k+4 - round 3
5622 mov $t6d, $res2.d[1] @ GHASH block 4k+2 - mid
5624 aese $ctr3b, $rk2 \n aesmc $ctr3b, $ctr3b @ AES block 4k+7 - round 2
5625 eor $t3.8b, $t3.8b, $res1.8b @ GHASH block 4k+1 - mid
5627 pmull $t5.1q, $res2.1d, $h2.1d @ GHASH block 4k+2 - low
5629 aese $ctr0b, $rk4 \n aesmc $ctr0b, $ctr0b @ AES block 4k+4 - round 4
5631 aese $ctr3b, $rk3 \n aesmc $ctr3b, $ctr3b @ AES block 4k+7 - round 3
5632 eor $t6.8b, $t6.8b, $res2.8b @ GHASH block 4k+2 - mid
5634 pmull $t3.1q, $t3.1d, $h34k.1d @ GHASH block 4k+1 - mid
5636 aese $ctr0b, $rk5 \n aesmc $ctr0b, $ctr0b @ AES block 4k+4 - round 5
5637 eor $acc_lb, $acc_lb, $t5.16b @ GHASH block 4k+2 - low
5639 aese $ctr3b, $rk4 \n aesmc $ctr3b, $ctr3b @ AES block 4k+7 - round 4
5641 pmull2 $t7.1q, $res3.2d, $h1.2d @ GHASH block 4k+3 - high
5642 eor $acc_mb, $acc_mb, $t3.16b @ GHASH block 4k+1 - mid
5644 pmull2 $t4.1q, $res2.2d, $h2.2d @ GHASH block 4k+2 - high
5646 aese $ctr3b, $rk5 \n aesmc $ctr3b, $ctr3b @ AES block 4k+7 - round 5
5647 ins $t6.d[1], $t6.d[0] @ GHASH block 4k+2 - mid
5649 aese $ctr2b, $rk3 \n aesmc $ctr2b, $ctr2b @ AES block 4k+6 - round 3
5651 aese $ctr1b, $rk3 \n aesmc $ctr1b, $ctr1b @ AES block 4k+5 - round 3
5652 eor $acc_hb, $acc_hb, $t4.16b @ GHASH block 4k+2 - high
5654 pmull $t8.1q, $res3.1d, $h1.1d @ GHASH block 4k+3 - low
5656 aese $ctr2b, $rk4 \n aesmc $ctr2b, $ctr2b @ AES block 4k+6 - round 4
5657 mov $t9d, $res3.d[1] @ GHASH block 4k+3 - mid
5659 aese $ctr1b, $rk4 \n aesmc $ctr1b, $ctr1b @ AES block 4k+5 - round 4
5661 pmull2 $t6.1q, $t6.2d, $h12k.2d @ GHASH block 4k+2 - mid
5663 aese $ctr2b, $rk5 \n aesmc $ctr2b, $ctr2b @ AES block 4k+6 - round 5
5664 eor $t9.8b, $t9.8b, $res3.8b @ GHASH block 4k+3 - mid
5666 aese $ctr1b, $rk5 \n aesmc $ctr1b, $ctr1b @ AES block 4k+5 - round 5
5668 aese $ctr3b, $rk6 \n aesmc $ctr3b, $ctr3b @ AES block 4k+7 - round 6
5669 eor $acc_mb, $acc_mb, $t6.16b @ GHASH block 4k+2 - mid
5671 aese $ctr2b, $rk6 \n aesmc $ctr2b, $ctr2b @ AES block 4k+6 - round 6
5673 aese $ctr0b, $rk6 \n aesmc $ctr0b, $ctr0b @ AES block 4k+4 - round 6
5676 aese $ctr1b, $rk6 \n aesmc $ctr1b, $ctr1b @ AES block 4k+5 - round 6
5677 eor $acc_lb, $acc_lb, $t8.16b @ GHASH block 4k+3 - low
5679 pmull $t9.1q, $t9.1d, $h12k.1d @ GHASH block 4k+3 - mid
5681 aese $ctr3b, $rk7 \n aesmc $ctr3b, $ctr3b @ AES block 4k+7 - round 7
5682 eor $acc_hb, $acc_hb, $t7.16b @ GHASH block 4k+3 - high
5684 aese $ctr1b, $rk7 \n aesmc $ctr1b, $ctr1b @ AES block 4k+5 - round 7
5686 aese $ctr0b, $rk7 \n aesmc $ctr0b, $ctr0b @ AES block 4k+4 - round 7
5687 eor $acc_mb, $acc_mb, $t9.16b @ GHASH block 4k+3 - mid
5689 aese $ctr3b, $rk8 \n aesmc $ctr3b, $ctr3b @ AES block 4k+7 - round 8
5691 aese $ctr2b, $rk7 \n aesmc $ctr2b, $ctr2b @ AES block 4k+6 - round 7
5694 aese $ctr1b, $rk8 \n aesmc $ctr1b, $ctr1b @ AES block 4k+5 - round 8
5696 aese $ctr0b, $rk8 \n aesmc $ctr0b, $ctr0b @ AES block 4k+4 - round 8
5699 aese $ctr2b, $rk8 \n aesmc $ctr2b, $ctr2b @ AES block 4k+6 - round 8
5701 aese $ctr1b, $rk9 \n aesmc $ctr1b, $ctr1b @ AES block 4k+5 - round 9
5706 aese $ctr2b, $rk9 \n aesmc $ctr2b, $ctr2b @ AES block 4k+6 - round 9
5709 aese $ctr3b, $rk9 \n aesmc $ctr3b, $ctr3b @ AES block 4k+7 - round 9
5711 aese $ctr0b, $rk9 \n aesmc $ctr0b, $ctr0b @ AES block 4k+4 - round 9
5714 aese $ctr2b, $rk10 \n aesmc $ctr2b, $ctr2b @ AES block 4k+6 - round 10
5716 aese $ctr3b, $rk10 \n aesmc $ctr3b, $ctr3b @ AES block 4k+7 - round 10
5718 aese $ctr0b, $rk10 \n aesmc $ctr0b, $ctr0b @ AES block 4k+4 - round 10
5719 eor $output_h2, $output_h2, $rk14_h @ AES block 4k+2 - round 14 high
5723 aese $ctr1b, $rk10 \n aesmc $ctr1b, $ctr1b @ AES block 4k+5 - round 10
5724 eor $output_l3, $output_l3, $rk14_l @ AES block 4k+3 - round 14 low
5728 aese $ctr2b, $rk11 \n aesmc $ctr2b, $ctr2b @ AES block 4k+6 - round 11
5731 aese $ctr0b, $rk11 \n aesmc $ctr0b, $ctr0b @ AES block 4k+4 - round 11
5732 add $rctr32w, $rctr32w, #1 @ CTR block 4k+7
5734 aese $ctr1b, $rk11 \n aesmc $ctr1b, $ctr1b @ AES block 4k+5 - round 11
5735 eor $output_l2, $output_l2, $rk14_l @ AES block 4k+2 - round 14 low
5740 aese $ctr2b, $rk12 \n aesmc $ctr2b, $ctr2b @ AES block 4k+6 - round 12
5743 eor $output_h3, $output_h3, $rk14_h @ AES block 4k+3 - round 14 high
5748 aese $ctr3b, $rk11 \n aesmc $ctr3b, $ctr3b @ AES block 4k+7 - round 11
5749 stp $output_l2, $output_h2, [$output_ptr], #16 @ AES block 4k+2 - store result
5751 aese $ctr1b, $rk12 \n aesmc $ctr1b, $ctr1b @ AES block 4k+5 - round 12
5754 aese $ctr0b, $rk12 \n aesmc $ctr0b, $ctr0b @ AES block 4k+4 - round 12
5755 stp $output_l3, $output_h3, [$output_ptr], #16 @ AES block 4k+3 - store result
5757 aese $ctr3b, $rk12 \n aesmc $ctr3b, $ctr3b @ AES block 4k+7 - round 12
5760 aese $ctr1b, $rk13 @ AES block 4k+5 - round 13
5762 aese $ctr0b, $rk13 @ AES block 4k+4 - round 13
5764 aese $ctr3b, $rk13 @ AES block 4k+7 - round 13
5766 aese $ctr2b, $rk13 @ AES block 4k+6 - round 13
5771 ld1 { $res1b}, [$input_ptr], #16 @ AES block 4k+4 - load ciphertext
5773 eor $ctr0b, $res1b, $ctr0b @ AES block 4k+4 - result
5775 mov $output_l0, $ctr0.d[0] @ AES block 4k+4 - mov low
5777 mov $output_h0, $ctr0.d[1] @ AES block 4k+4 - mov high
5782 eor $output_l0, $output_l0, $rk14_l @ AES block 4k+4 - round 14 low
5787 eor $output_h0, $output_h0, $rk14_h @ AES block 4k+4 - round 14 high
5813 rev64 $res0b, $res1b @ GHASH final-3 block
5814 ld1 { $res1b}, [$input_ptr], #16 @ AES final-2 block - load ciphertext
5816 stp $output_l0, $output_h0, [$output_ptr], #16 @ AES final-3 block - store result
5818 mov $acc_md, $h34k.d[1] @ GHASH final-3 block - mid
5822 eor $ctr0b, $res1b, $ctr1b @ AES final-2 block - result
5824 mov $rk4d, $res0.d[1] @ GHASH final-3 block - mid
5826 mov $output_l0, $ctr0.d[0] @ AES final-2 block - mov low
5828 mov $output_h0, $ctr0.d[1] @ AES final-2 block - mov high
5830 eor $rk4v.8b, $rk4v.8b, $res0.8b @ GHASH final-3 block - mid
5834 pmull2 $acc_h.1q, $res0.2d, $h4.2d @ GHASH final-3 block - high
5836 pmull $acc_m.1q, $rk4v.1d, $acc_m.1d @ GHASH final-3 block - mid
5837 eor $output_l0, $output_l0, $rk14_l @ AES final-2 block - round 14 low
5842 pmull $acc_l.1q, $res0.1d, $h4.1d @ GHASH final-3 block - low
5843 eor $output_h0, $output_h0, $rk14_h @ AES final-2 block - round 14 high
5849 rev64 $res0b, $res1b @ GHASH final-2 block
5850 ld1 { $res1b}, [$input_ptr], #16 @ AES final-1 block - load ciphertext
5853 stp $output_l0, $output_h0, [$output_ptr], #16 @ AES final-2 block - store result
5855 eor $ctr0b, $res1b, $ctr2b @ AES final-1 block - result
5857 mov $rk4d, $res0.d[1] @ GHASH final-2 block - mid
5859 pmull $rk3q1, $res0.1d, $h3.1d @ GHASH final-2 block - low
5861 pmull2 $rk2q1, $res0.2d, $h3.2d @ GHASH final-2 block - high
5863 eor $rk4v.8b, $rk4v.8b, $res0.8b @ GHASH final-2 block - mid
5864 mov $output_l0, $ctr0.d[0] @ AES final-1 block - mov low
5866 mov $output_h0, $ctr0.d[1] @ AES final-1 block - mov high
5867 eor $acc_lb, $acc_lb, $rk3 @ GHASH final-2 block - low
5870 pmull $rk4v.1q, $rk4v.1d, $h34k.1d @ GHASH final-2 block - mid
5872 eor $acc_hb, $acc_hb, $rk2 @ GHASH final-2 block - high
5873 eor $output_l0, $output_l0, $rk14_l @ AES final-1 block - round 14 low
5878 eor $acc_mb, $acc_mb, $rk4v.16b @ GHASH final-2 block - mid
5879 eor $output_h0, $output_h0, $rk14_h @ AES final-1 block - round 14 high
5885 stp $output_l0, $output_h0, [$output_ptr], #16 @ AES final-1 block - store result
5886 rev64 $res0b, $res1b @ GHASH final-1 block
5888 ld1 { $res1b}, [$input_ptr], #16 @ AES final block - load ciphertext
5893 mov $rk4d, $res0.d[1] @ GHASH final-1 block - mid
5895 eor $ctr0b, $res1b, $ctr3b @ AES final block - result
5897 pmull2 $rk2q1, $res0.2d, $h2.2d @ GHASH final-1 block - high
5899 eor $rk4v.8b, $rk4v.8b, $res0.8b @ GHASH final-1 block - mid
5901 pmull $rk3q1, $res0.1d, $h2.1d @ GHASH final-1 block - low
5902 mov $output_l0, $ctr0.d[0] @ AES final block - mov low
5904 ins $rk4v.d[1], $rk4v.d[0] @ GHASH final-1 block - mid
5906 mov $output_h0, $ctr0.d[1] @ AES final block - mov high
5908 pmull2 $rk4v.1q, $rk4v.2d, $h12k.2d @ GHASH final-1 block - mid
5909 eor $output_l0, $output_l0, $rk14_l @ AES final block - round 14 low
5913 eor $acc_lb, $acc_lb, $rk3 @ GHASH final-1 block - low
5915 eor $acc_hb, $acc_hb, $rk2 @ GHASH final-1 block - high
5917 eor $acc_mb, $acc_mb, $rk4v.16b @ GHASH final-1 block - mid
5918 eor $output_h0, $output_h0, $rk14_h @ AES final block - round 14 high
5935 lsr $rk14_h, $rk14_h, $bit_length @ rk14_h is mask for top 64b of last block
5941 fmov $ctr0d, $ctr32x @ ctr0b is mask for last block
5961 and $res1b, $res1b, $ctr0b @ possibly partial last block has zeroes in highest bits
5963 rev64 $res0b, $res1b @ GHASH final block
5967 pmull $rk3q1, $res0.1d, $h1.1d @ GHASH final block - low
5969 mov $t0d, $res0.d[1] @ GHASH final block - mid
5971 eor $t0.8b, $t0.8b, $res0.8b @ GHASH final block - mid
5973 pmull2 $rk2q1, $res0.2d, $h1.2d @ GHASH final block - high
5975 pmull $t0.1q, $t0.1d, $h12k.1d @ GHASH final block - mid
5977 eor $acc_hb, $acc_hb, $rk2 @ GHASH final block - high
5979 eor $acc_lb, $acc_lb, $rk3 @ GHASH final block - low
5981 eor $acc_mb, $acc_mb, $t0.16b @ GHASH final block - mid