1 /* SPDX-License-Identifier: GPL-2.0-only */ 2 /* 3 * Copyright (C) 2013 - 2017 Linaro Ltd <ard.biesheuvel@linaro.org> 4 */ 5 6 #include <linux/linkage.h> 7 #include <asm/assembler.h> 8 9 .arch armv8-a+crypto 10 11 SYM_FUNC_START(__aes_ce_encrypt) 12 sub w3, w3, #2 13 ld1 {v0.16b}, [x2] 14 ld1 {v1.4s}, [x0], #16 15 cmp w3, #10 16 bmi 0f 17 bne 3f 18 mov v3.16b, v1.16b 19 b 2f 20 0: mov v2.16b, v1.16b 21 ld1 {v3.4s}, [x0], #16 22 1: aese v0.16b, v2.16b 23 aesmc v0.16b, v0.16b 24 2: ld1 {v1.4s}, [x0], #16 25 aese v0.16b, v3.16b 26 aesmc v0.16b, v0.16b 27 3: ld1 {v2.4s}, [x0], #16 28 subs w3, w3, #3 29 aese v0.16b, v1.16b 30 aesmc v0.16b, v0.16b 31 ld1 {v3.4s}, [x0], #16 32 bpl 1b 33 aese v0.16b, v2.16b 34 eor v0.16b, v0.16b, v3.16b 35 st1 {v0.16b}, [x1] 36 ret 37 SYM_FUNC_END(__aes_ce_encrypt) 38 39 SYM_FUNC_START(__aes_ce_decrypt) 40 sub w3, w3, #2 41 ld1 {v0.16b}, [x2] 42 ld1 {v1.4s}, [x0], #16 43 cmp w3, #10 44 bmi 0f 45 bne 3f 46 mov v3.16b, v1.16b 47 b 2f 48 0: mov v2.16b, v1.16b 49 ld1 {v3.4s}, [x0], #16 50 1: aesd v0.16b, v2.16b 51 aesimc v0.16b, v0.16b 52 2: ld1 {v1.4s}, [x0], #16 53 aesd v0.16b, v3.16b 54 aesimc v0.16b, v0.16b 55 3: ld1 {v2.4s}, [x0], #16 56 subs w3, w3, #3 57 aesd v0.16b, v1.16b 58 aesimc v0.16b, v0.16b 59 ld1 {v3.4s}, [x0], #16 60 bpl 1b 61 aesd v0.16b, v2.16b 62 eor v0.16b, v0.16b, v3.16b 63 st1 {v0.16b}, [x1] 64 ret 65 SYM_FUNC_END(__aes_ce_decrypt) 66 67 /* 68 * __aes_ce_sub() - use the aese instruction to perform the AES sbox 69 * substitution on each byte in 'input' 70 */ 71 SYM_FUNC_START(__aes_ce_sub) 72 dup v1.4s, w0 73 movi v0.16b, #0 74 aese v0.16b, v1.16b 75 umov w0, v0.s[0] 76 ret 77 SYM_FUNC_END(__aes_ce_sub) 78 79 SYM_FUNC_START(__aes_ce_invert) 80 ld1 {v0.4s}, [x1] 81 aesimc v1.16b, v0.16b 82 st1 {v1.4s}, [x0] 83 ret 84 SYM_FUNC_END(__aes_ce_invert) 85