1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * sm3-neon-core.S - SM3 secure hash using NEON instructions 4 * 5 * Linux/arm64 port of the libgcrypt SM3 implementation for AArch64 6 * 7 * Copyright (C) 2021 Jussi Kivilinna <jussi.kivilinna@iki.fi> 8 * Copyright (c) 2022 Tianjia Zhang <tianjia.zhang@linux.alibaba.com> 9 */ 10 11 #include <linux/linkage.h> 12 #include <linux/cfi_types.h> 13 #include <asm/assembler.h> 14 15 /* Context structure */ 16 17 #define state_h0 0 18 #define state_h1 4 19 #define state_h2 8 20 #define state_h3 12 21 #define state_h4 16 22 #define state_h5 20 23 #define state_h6 24 24 #define state_h7 28 25 26 /* Stack structure */ 27 28 #define STACK_W_SIZE (32 * 2 * 3) 29 30 #define STACK_W (0) 31 #define STACK_SIZE (STACK_W + STACK_W_SIZE) 32 33 /* Register macros */ 34 35 #define RSTATE x0 36 #define RDATA x1 37 #define RNBLKS x2 38 #define RKPTR x28 39 #define RFRAME x29 40 41 #define ra w3 42 #define rb w4 43 #define rc w5 44 #define rd w6 45 #define re w7 46 #define rf w8 47 #define rg w9 48 #define rh w10 49 50 #define t0 w11 51 #define t1 w12 52 #define t2 w13 53 #define t3 w14 54 #define t4 w15 55 #define t5 w16 56 #define t6 w17 57 58 #define k_even w19 59 #define k_odd w20 60 61 #define addr0 x21 62 #define addr1 x22 63 64 #define s0 w23 65 #define s1 w24 66 #define s2 w25 67 #define s3 w26 68 69 #define W0 v0 70 #define W1 v1 71 #define W2 v2 72 #define W3 v3 73 #define W4 v4 74 #define W5 v5 75 76 #define XTMP0 v6 77 #define XTMP1 v7 78 #define XTMP2 v16 79 #define XTMP3 v17 80 #define XTMP4 v18 81 #define XTMP5 v19 82 #define XTMP6 v20 83 84 /* Helper macros. */ 85 86 #define _(...) /*_*/ 87 88 #define clear_vec(x) \ 89 movi x.8h, #0; 90 91 #define rolw(o, a, n) \ 92 ror o, a, #(32 - n); 93 94 /* Round function macros. */ 95 96 #define GG1_1(x, y, z, o, t) \ 97 eor o, x, y; 98 #define GG1_2(x, y, z, o, t) \ 99 eor o, o, z; 100 #define GG1_3(x, y, z, o, t) 101 102 #define FF1_1(x, y, z, o, t) GG1_1(x, y, z, o, t) 103 #define FF1_2(x, y, z, o, t) 104 #define FF1_3(x, y, z, o, t) GG1_2(x, y, z, o, t) 105 106 #define GG2_1(x, y, z, o, t) \ 107 bic o, z, x; 108 #define GG2_2(x, y, z, o, t) \ 109 and t, y, x; 110 #define GG2_3(x, y, z, o, t) \ 111 eor o, o, t; 112 113 #define FF2_1(x, y, z, o, t) \ 114 eor o, x, y; 115 #define FF2_2(x, y, z, o, t) \ 116 and t, x, y; \ 117 and o, o, z; 118 #define FF2_3(x, y, z, o, t) \ 119 eor o, o, t; 120 121 #define R(i, a, b, c, d, e, f, g, h, k, K_LOAD, round, widx, wtype, IOP, iop_param) \ 122 K_LOAD(round); \ 123 ldr t5, [sp, #(wtype##_W1_ADDR(round, widx))]; \ 124 rolw(t0, a, 12); /* rol(a, 12) => t0 */ \ 125 IOP(1, iop_param); \ 126 FF##i##_1(a, b, c, t1, t2); \ 127 ldr t6, [sp, #(wtype##_W1W2_ADDR(round, widx))]; \ 128 add k, k, e; \ 129 IOP(2, iop_param); \ 130 GG##i##_1(e, f, g, t3, t4); \ 131 FF##i##_2(a, b, c, t1, t2); \ 132 IOP(3, iop_param); \ 133 add k, k, t0; \ 134 add h, h, t5; \ 135 add d, d, t6; /* w1w2 + d => d */ \ 136 IOP(4, iop_param); \ 137 rolw(k, k, 7); /* rol (t0 + e + t), 7) => k */ \ 138 GG##i##_2(e, f, g, t3, t4); \ 139 add h, h, k; /* h + w1 + k => h */ \ 140 IOP(5, iop_param); \ 141 FF##i##_3(a, b, c, t1, t2); \ 142 eor t0, t0, k; /* k ^ t0 => t0 */ \ 143 GG##i##_3(e, f, g, t3, t4); \ 144 add d, d, t1; /* FF(a,b,c) + d => d */ \ 145 IOP(6, iop_param); \ 146 add t3, t3, h; /* GG(e,f,g) + h => t3 */ \ 147 rolw(b, b, 9); /* rol(b, 9) => b */ \ 148 eor h, t3, t3, ror #(32-9); \ 149 IOP(7, iop_param); \ 150 add d, d, t0; /* t0 + d => d */ \ 151 rolw(f, f, 19); /* rol(f, 19) => f */ \ 152 IOP(8, iop_param); \ 153 eor h, h, t3, ror #(32-17); /* P0(t3) => h */ 154 155 #define R1(a, b, c, d, e, f, g, h, k, K_LOAD, round, widx, wtype, IOP, iop_param) \ 156 R(1, ##a, ##b, ##c, ##d, ##e, ##f, ##g, ##h, ##k, K_LOAD, round, widx, wtype, IOP, iop_param) 157 158 #define R2(a, b, c, d, e, f, g, h, k, K_LOAD, round, widx, wtype, IOP, iop_param) \ 159 R(2, ##a, ##b, ##c, ##d, ##e, ##f, ##g, ##h, ##k, K_LOAD, round, widx, wtype, IOP, iop_param) 160 161 #define KL(round) \ 162 ldp k_even, k_odd, [RKPTR, #(4*(round))]; 163 164 /* Input expansion macros. */ 165 166 /* Byte-swapped input address. */ 167 #define IW_W_ADDR(round, widx, offs) \ 168 (STACK_W + ((round) / 4) * 64 + (offs) + ((widx) * 4)) 169 170 /* Expanded input address. */ 171 #define XW_W_ADDR(round, widx, offs) \ 172 (STACK_W + ((((round) / 3) - 4) % 2) * 64 + (offs) + ((widx) * 4)) 173 174 /* Rounds 1-12, byte-swapped input block addresses. */ 175 #define IW_W1_ADDR(round, widx) IW_W_ADDR(round, widx, 32) 176 #define IW_W1W2_ADDR(round, widx) IW_W_ADDR(round, widx, 48) 177 178 /* Rounds 1-12, expanded input block addresses. */ 179 #define XW_W1_ADDR(round, widx) XW_W_ADDR(round, widx, 0) 180 #define XW_W1W2_ADDR(round, widx) XW_W_ADDR(round, widx, 16) 181 182 /* Input block loading. 183 * Interleaving within round function needed for in-order CPUs. */ 184 #define LOAD_W_VEC_1_1() \ 185 add addr0, sp, #IW_W1_ADDR(0, 0); 186 #define LOAD_W_VEC_1_2() \ 187 add addr1, sp, #IW_W1_ADDR(4, 0); 188 #define LOAD_W_VEC_1_3() \ 189 ld1 {W0.16b}, [RDATA], #16; 190 #define LOAD_W_VEC_1_4() \ 191 ld1 {W1.16b}, [RDATA], #16; 192 #define LOAD_W_VEC_1_5() \ 193 ld1 {W2.16b}, [RDATA], #16; 194 #define LOAD_W_VEC_1_6() \ 195 ld1 {W3.16b}, [RDATA], #16; 196 #define LOAD_W_VEC_1_7() \ 197 rev32 XTMP0.16b, W0.16b; 198 #define LOAD_W_VEC_1_8() \ 199 rev32 XTMP1.16b, W1.16b; 200 #define LOAD_W_VEC_2_1() \ 201 rev32 XTMP2.16b, W2.16b; 202 #define LOAD_W_VEC_2_2() \ 203 rev32 XTMP3.16b, W3.16b; 204 #define LOAD_W_VEC_2_3() \ 205 eor XTMP4.16b, XTMP1.16b, XTMP0.16b; 206 #define LOAD_W_VEC_2_4() \ 207 eor XTMP5.16b, XTMP2.16b, XTMP1.16b; 208 #define LOAD_W_VEC_2_5() \ 209 st1 {XTMP0.16b}, [addr0], #16; 210 #define LOAD_W_VEC_2_6() \ 211 st1 {XTMP4.16b}, [addr0]; \ 212 add addr0, sp, #IW_W1_ADDR(8, 0); 213 #define LOAD_W_VEC_2_7() \ 214 eor XTMP6.16b, XTMP3.16b, XTMP2.16b; 215 #define LOAD_W_VEC_2_8() \ 216 ext W0.16b, XTMP0.16b, XTMP0.16b, #8; /* W0: xx, w0, xx, xx */ 217 #define LOAD_W_VEC_3_1() \ 218 mov W2.16b, XTMP1.16b; /* W2: xx, w6, w5, w4 */ 219 #define LOAD_W_VEC_3_2() \ 220 st1 {XTMP1.16b}, [addr1], #16; 221 #define LOAD_W_VEC_3_3() \ 222 st1 {XTMP5.16b}, [addr1]; \ 223 ext W1.16b, XTMP0.16b, XTMP0.16b, #4; /* W1: xx, w3, w2, w1 */ 224 #define LOAD_W_VEC_3_4() \ 225 ext W3.16b, XTMP1.16b, XTMP2.16b, #12; /* W3: xx, w9, w8, w7 */ 226 #define LOAD_W_VEC_3_5() \ 227 ext W4.16b, XTMP2.16b, XTMP3.16b, #8; /* W4: xx, w12, w11, w10 */ 228 #define LOAD_W_VEC_3_6() \ 229 st1 {XTMP2.16b}, [addr0], #16; 230 #define LOAD_W_VEC_3_7() \ 231 st1 {XTMP6.16b}, [addr0]; 232 #define LOAD_W_VEC_3_8() \ 233 ext W5.16b, XTMP3.16b, XTMP3.16b, #4; /* W5: xx, w15, w14, w13 */ 234 235 #define LOAD_W_VEC_1(iop_num, ...) \ 236 LOAD_W_VEC_1_##iop_num() 237 #define LOAD_W_VEC_2(iop_num, ...) \ 238 LOAD_W_VEC_2_##iop_num() 239 #define LOAD_W_VEC_3(iop_num, ...) \ 240 LOAD_W_VEC_3_##iop_num() 241 242 /* Message scheduling. Note: 3 words per vector register. 243 * Interleaving within round function needed for in-order CPUs. */ 244 #define SCHED_W_1_1(round, w0, w1, w2, w3, w4, w5) \ 245 /* Load (w[i - 16]) => XTMP0 */ \ 246 /* Load (w[i - 13]) => XTMP5 */ \ 247 ext XTMP0.16b, w0.16b, w0.16b, #12; /* XTMP0: w0, xx, xx, xx */ 248 #define SCHED_W_1_2(round, w0, w1, w2, w3, w4, w5) \ 249 ext XTMP5.16b, w1.16b, w1.16b, #12; 250 #define SCHED_W_1_3(round, w0, w1, w2, w3, w4, w5) \ 251 ext XTMP0.16b, XTMP0.16b, w1.16b, #12; /* XTMP0: xx, w2, w1, w0 */ 252 #define SCHED_W_1_4(round, w0, w1, w2, w3, w4, w5) \ 253 ext XTMP5.16b, XTMP5.16b, w2.16b, #12; 254 #define SCHED_W_1_5(round, w0, w1, w2, w3, w4, w5) \ 255 /* w[i - 9] == w3 */ \ 256 /* W3 ^ XTMP0 => XTMP0 */ \ 257 eor XTMP0.16b, XTMP0.16b, w3.16b; 258 #define SCHED_W_1_6(round, w0, w1, w2, w3, w4, w5) \ 259 /* w[i - 3] == w5 */ \ 260 /* rol(XMM5, 15) ^ XTMP0 => XTMP0 */ \ 261 /* rol(XTMP5, 7) => XTMP1 */ \ 262 add addr0, sp, #XW_W1_ADDR((round), 0); \ 263 shl XTMP2.4s, w5.4s, #15; 264 #define SCHED_W_1_7(round, w0, w1, w2, w3, w4, w5) \ 265 shl XTMP1.4s, XTMP5.4s, #7; 266 #define SCHED_W_1_8(round, w0, w1, w2, w3, w4, w5) \ 267 sri XTMP2.4s, w5.4s, #(32-15); 268 #define SCHED_W_2_1(round, w0, w1, w2, w3, w4, w5) \ 269 sri XTMP1.4s, XTMP5.4s, #(32-7); 270 #define SCHED_W_2_2(round, w0, w1, w2, w3, w4, w5) \ 271 eor XTMP0.16b, XTMP0.16b, XTMP2.16b; 272 #define SCHED_W_2_3(round, w0, w1, w2, w3, w4, w5) \ 273 /* w[i - 6] == W4 */ \ 274 /* W4 ^ XTMP1 => XTMP1 */ \ 275 eor XTMP1.16b, XTMP1.16b, w4.16b; 276 #define SCHED_W_2_4(round, w0, w1, w2, w3, w4, w5) \ 277 /* P1(XTMP0) ^ XTMP1 => W0 */ \ 278 shl XTMP3.4s, XTMP0.4s, #15; 279 #define SCHED_W_2_5(round, w0, w1, w2, w3, w4, w5) \ 280 shl XTMP4.4s, XTMP0.4s, #23; 281 #define SCHED_W_2_6(round, w0, w1, w2, w3, w4, w5) \ 282 eor w0.16b, XTMP1.16b, XTMP0.16b; 283 #define SCHED_W_2_7(round, w0, w1, w2, w3, w4, w5) \ 284 sri XTMP3.4s, XTMP0.4s, #(32-15); 285 #define SCHED_W_2_8(round, w0, w1, w2, w3, w4, w5) \ 286 sri XTMP4.4s, XTMP0.4s, #(32-23); 287 #define SCHED_W_3_1(round, w0, w1, w2, w3, w4, w5) \ 288 eor w0.16b, w0.16b, XTMP3.16b; 289 #define SCHED_W_3_2(round, w0, w1, w2, w3, w4, w5) \ 290 /* Load (w[i - 3]) => XTMP2 */ \ 291 ext XTMP2.16b, w4.16b, w4.16b, #12; 292 #define SCHED_W_3_3(round, w0, w1, w2, w3, w4, w5) \ 293 eor w0.16b, w0.16b, XTMP4.16b; 294 #define SCHED_W_3_4(round, w0, w1, w2, w3, w4, w5) \ 295 ext XTMP2.16b, XTMP2.16b, w5.16b, #12; 296 #define SCHED_W_3_5(round, w0, w1, w2, w3, w4, w5) \ 297 /* W1 ^ W2 => XTMP3 */ \ 298 eor XTMP3.16b, XTMP2.16b, w0.16b; 299 #define SCHED_W_3_6(round, w0, w1, w2, w3, w4, w5) 300 #define SCHED_W_3_7(round, w0, w1, w2, w3, w4, w5) \ 301 st1 {XTMP2.16b-XTMP3.16b}, [addr0]; 302 #define SCHED_W_3_8(round, w0, w1, w2, w3, w4, w5) 303 304 #define SCHED_W_W0W1W2W3W4W5_1(iop_num, round) \ 305 SCHED_W_1_##iop_num(round, W0, W1, W2, W3, W4, W5) 306 #define SCHED_W_W0W1W2W3W4W5_2(iop_num, round) \ 307 SCHED_W_2_##iop_num(round, W0, W1, W2, W3, W4, W5) 308 #define SCHED_W_W0W1W2W3W4W5_3(iop_num, round) \ 309 SCHED_W_3_##iop_num(round, W0, W1, W2, W3, W4, W5) 310 311 #define SCHED_W_W1W2W3W4W5W0_1(iop_num, round) \ 312 SCHED_W_1_##iop_num(round, W1, W2, W3, W4, W5, W0) 313 #define SCHED_W_W1W2W3W4W5W0_2(iop_num, round) \ 314 SCHED_W_2_##iop_num(round, W1, W2, W3, W4, W5, W0) 315 #define SCHED_W_W1W2W3W4W5W0_3(iop_num, round) \ 316 SCHED_W_3_##iop_num(round, W1, W2, W3, W4, W5, W0) 317 318 #define SCHED_W_W2W3W4W5W0W1_1(iop_num, round) \ 319 SCHED_W_1_##iop_num(round, W2, W3, W4, W5, W0, W1) 320 #define SCHED_W_W2W3W4W5W0W1_2(iop_num, round) \ 321 SCHED_W_2_##iop_num(round, W2, W3, W4, W5, W0, W1) 322 #define SCHED_W_W2W3W4W5W0W1_3(iop_num, round) \ 323 SCHED_W_3_##iop_num(round, W2, W3, W4, W5, W0, W1) 324 325 #define SCHED_W_W3W4W5W0W1W2_1(iop_num, round) \ 326 SCHED_W_1_##iop_num(round, W3, W4, W5, W0, W1, W2) 327 #define SCHED_W_W3W4W5W0W1W2_2(iop_num, round) \ 328 SCHED_W_2_##iop_num(round, W3, W4, W5, W0, W1, W2) 329 #define SCHED_W_W3W4W5W0W1W2_3(iop_num, round) \ 330 SCHED_W_3_##iop_num(round, W3, W4, W5, W0, W1, W2) 331 332 #define SCHED_W_W4W5W0W1W2W3_1(iop_num, round) \ 333 SCHED_W_1_##iop_num(round, W4, W5, W0, W1, W2, W3) 334 #define SCHED_W_W4W5W0W1W2W3_2(iop_num, round) \ 335 SCHED_W_2_##iop_num(round, W4, W5, W0, W1, W2, W3) 336 #define SCHED_W_W4W5W0W1W2W3_3(iop_num, round) \ 337 SCHED_W_3_##iop_num(round, W4, W5, W0, W1, W2, W3) 338 339 #define SCHED_W_W5W0W1W2W3W4_1(iop_num, round) \ 340 SCHED_W_1_##iop_num(round, W5, W0, W1, W2, W3, W4) 341 #define SCHED_W_W5W0W1W2W3W4_2(iop_num, round) \ 342 SCHED_W_2_##iop_num(round, W5, W0, W1, W2, W3, W4) 343 #define SCHED_W_W5W0W1W2W3W4_3(iop_num, round) \ 344 SCHED_W_3_##iop_num(round, W5, W0, W1, W2, W3, W4) 345 346 347 /* 348 * Transform blocks*64 bytes (blocks*16 32-bit words) at 'src'. 349 * 350 * void sm3_neon_transform(struct sm3_state *sst, u8 const *src, 351 * int blocks) 352 */ 353 .text 354 .align 3 355 SYM_TYPED_FUNC_START(sm3_neon_transform) 356 ldp ra, rb, [RSTATE, #0] 357 ldp rc, rd, [RSTATE, #8] 358 ldp re, rf, [RSTATE, #16] 359 ldp rg, rh, [RSTATE, #24] 360 361 stp x28, x29, [sp, #-16]! 362 stp x19, x20, [sp, #-16]! 363 stp x21, x22, [sp, #-16]! 364 stp x23, x24, [sp, #-16]! 365 stp x25, x26, [sp, #-16]! 366 mov RFRAME, sp 367 368 sub addr0, sp, #STACK_SIZE 369 adr_l RKPTR, .LKtable 370 and sp, addr0, #(~63) 371 372 /* Preload first block. */ 373 LOAD_W_VEC_1(1, 0) 374 LOAD_W_VEC_1(2, 0) 375 LOAD_W_VEC_1(3, 0) 376 LOAD_W_VEC_1(4, 0) 377 LOAD_W_VEC_1(5, 0) 378 LOAD_W_VEC_1(6, 0) 379 LOAD_W_VEC_1(7, 0) 380 LOAD_W_VEC_1(8, 0) 381 LOAD_W_VEC_2(1, 0) 382 LOAD_W_VEC_2(2, 0) 383 LOAD_W_VEC_2(3, 0) 384 LOAD_W_VEC_2(4, 0) 385 LOAD_W_VEC_2(5, 0) 386 LOAD_W_VEC_2(6, 0) 387 LOAD_W_VEC_2(7, 0) 388 LOAD_W_VEC_2(8, 0) 389 LOAD_W_VEC_3(1, 0) 390 LOAD_W_VEC_3(2, 0) 391 LOAD_W_VEC_3(3, 0) 392 LOAD_W_VEC_3(4, 0) 393 LOAD_W_VEC_3(5, 0) 394 LOAD_W_VEC_3(6, 0) 395 LOAD_W_VEC_3(7, 0) 396 LOAD_W_VEC_3(8, 0) 397 398 .balign 16 399 .Loop: 400 /* Transform 0-3 */ 401 R1(ra, rb, rc, rd, re, rf, rg, rh, k_even, KL, 0, 0, IW, _, 0) 402 R1(rd, ra, rb, rc, rh, re, rf, rg, k_odd, _, 1, 1, IW, _, 0) 403 R1(rc, rd, ra, rb, rg, rh, re, rf, k_even, KL, 2, 2, IW, _, 0) 404 R1(rb, rc, rd, ra, rf, rg, rh, re, k_odd, _, 3, 3, IW, _, 0) 405 406 /* Transform 4-7 + Precalc 12-14 */ 407 R1(ra, rb, rc, rd, re, rf, rg, rh, k_even, KL, 4, 0, IW, _, 0) 408 R1(rd, ra, rb, rc, rh, re, rf, rg, k_odd, _, 5, 1, IW, _, 0) 409 R1(rc, rd, ra, rb, rg, rh, re, rf, k_even, KL, 6, 2, IW, SCHED_W_W0W1W2W3W4W5_1, 12) 410 R1(rb, rc, rd, ra, rf, rg, rh, re, k_odd, _, 7, 3, IW, SCHED_W_W0W1W2W3W4W5_2, 12) 411 412 /* Transform 8-11 + Precalc 12-17 */ 413 R1(ra, rb, rc, rd, re, rf, rg, rh, k_even, KL, 8, 0, IW, SCHED_W_W0W1W2W3W4W5_3, 12) 414 R1(rd, ra, rb, rc, rh, re, rf, rg, k_odd, _, 9, 1, IW, SCHED_W_W1W2W3W4W5W0_1, 15) 415 R1(rc, rd, ra, rb, rg, rh, re, rf, k_even, KL, 10, 2, IW, SCHED_W_W1W2W3W4W5W0_2, 15) 416 R1(rb, rc, rd, ra, rf, rg, rh, re, k_odd, _, 11, 3, IW, SCHED_W_W1W2W3W4W5W0_3, 15) 417 418 /* Transform 12-14 + Precalc 18-20 */ 419 R1(ra, rb, rc, rd, re, rf, rg, rh, k_even, KL, 12, 0, XW, SCHED_W_W2W3W4W5W0W1_1, 18) 420 R1(rd, ra, rb, rc, rh, re, rf, rg, k_odd, _, 13, 1, XW, SCHED_W_W2W3W4W5W0W1_2, 18) 421 R1(rc, rd, ra, rb, rg, rh, re, rf, k_even, KL, 14, 2, XW, SCHED_W_W2W3W4W5W0W1_3, 18) 422 423 /* Transform 15-17 + Precalc 21-23 */ 424 R1(rb, rc, rd, ra, rf, rg, rh, re, k_odd, _, 15, 0, XW, SCHED_W_W3W4W5W0W1W2_1, 21) 425 R2(ra, rb, rc, rd, re, rf, rg, rh, k_even, KL, 16, 1, XW, SCHED_W_W3W4W5W0W1W2_2, 21) 426 R2(rd, ra, rb, rc, rh, re, rf, rg, k_odd, _, 17, 2, XW, SCHED_W_W3W4W5W0W1W2_3, 21) 427 428 /* Transform 18-20 + Precalc 24-26 */ 429 R2(rc, rd, ra, rb, rg, rh, re, rf, k_even, KL, 18, 0, XW, SCHED_W_W4W5W0W1W2W3_1, 24) 430 R2(rb, rc, rd, ra, rf, rg, rh, re, k_odd, _, 19, 1, XW, SCHED_W_W4W5W0W1W2W3_2, 24) 431 R2(ra, rb, rc, rd, re, rf, rg, rh, k_even, KL, 20, 2, XW, SCHED_W_W4W5W0W1W2W3_3, 24) 432 433 /* Transform 21-23 + Precalc 27-29 */ 434 R2(rd, ra, rb, rc, rh, re, rf, rg, k_odd, _, 21, 0, XW, SCHED_W_W5W0W1W2W3W4_1, 27) 435 R2(rc, rd, ra, rb, rg, rh, re, rf, k_even, KL, 22, 1, XW, SCHED_W_W5W0W1W2W3W4_2, 27) 436 R2(rb, rc, rd, ra, rf, rg, rh, re, k_odd, _, 23, 2, XW, SCHED_W_W5W0W1W2W3W4_3, 27) 437 438 /* Transform 24-26 + Precalc 30-32 */ 439 R2(ra, rb, rc, rd, re, rf, rg, rh, k_even, KL, 24, 0, XW, SCHED_W_W0W1W2W3W4W5_1, 30) 440 R2(rd, ra, rb, rc, rh, re, rf, rg, k_odd, _, 25, 1, XW, SCHED_W_W0W1W2W3W4W5_2, 30) 441 R2(rc, rd, ra, rb, rg, rh, re, rf, k_even, KL, 26, 2, XW, SCHED_W_W0W1W2W3W4W5_3, 30) 442 443 /* Transform 27-29 + Precalc 33-35 */ 444 R2(rb, rc, rd, ra, rf, rg, rh, re, k_odd, _, 27, 0, XW, SCHED_W_W1W2W3W4W5W0_1, 33) 445 R2(ra, rb, rc, rd, re, rf, rg, rh, k_even, KL, 28, 1, XW, SCHED_W_W1W2W3W4W5W0_2, 33) 446 R2(rd, ra, rb, rc, rh, re, rf, rg, k_odd, _, 29, 2, XW, SCHED_W_W1W2W3W4W5W0_3, 33) 447 448 /* Transform 30-32 + Precalc 36-38 */ 449 R2(rc, rd, ra, rb, rg, rh, re, rf, k_even, KL, 30, 0, XW, SCHED_W_W2W3W4W5W0W1_1, 36) 450 R2(rb, rc, rd, ra, rf, rg, rh, re, k_odd, _, 31, 1, XW, SCHED_W_W2W3W4W5W0W1_2, 36) 451 R2(ra, rb, rc, rd, re, rf, rg, rh, k_even, KL, 32, 2, XW, SCHED_W_W2W3W4W5W0W1_3, 36) 452 453 /* Transform 33-35 + Precalc 39-41 */ 454 R2(rd, ra, rb, rc, rh, re, rf, rg, k_odd, _, 33, 0, XW, SCHED_W_W3W4W5W0W1W2_1, 39) 455 R2(rc, rd, ra, rb, rg, rh, re, rf, k_even, KL, 34, 1, XW, SCHED_W_W3W4W5W0W1W2_2, 39) 456 R2(rb, rc, rd, ra, rf, rg, rh, re, k_odd, _, 35, 2, XW, SCHED_W_W3W4W5W0W1W2_3, 39) 457 458 /* Transform 36-38 + Precalc 42-44 */ 459 R2(ra, rb, rc, rd, re, rf, rg, rh, k_even, KL, 36, 0, XW, SCHED_W_W4W5W0W1W2W3_1, 42) 460 R2(rd, ra, rb, rc, rh, re, rf, rg, k_odd, _, 37, 1, XW, SCHED_W_W4W5W0W1W2W3_2, 42) 461 R2(rc, rd, ra, rb, rg, rh, re, rf, k_even, KL, 38, 2, XW, SCHED_W_W4W5W0W1W2W3_3, 42) 462 463 /* Transform 39-41 + Precalc 45-47 */ 464 R2(rb, rc, rd, ra, rf, rg, rh, re, k_odd, _, 39, 0, XW, SCHED_W_W5W0W1W2W3W4_1, 45) 465 R2(ra, rb, rc, rd, re, rf, rg, rh, k_even, KL, 40, 1, XW, SCHED_W_W5W0W1W2W3W4_2, 45) 466 R2(rd, ra, rb, rc, rh, re, rf, rg, k_odd, _, 41, 2, XW, SCHED_W_W5W0W1W2W3W4_3, 45) 467 468 /* Transform 42-44 + Precalc 48-50 */ 469 R2(rc, rd, ra, rb, rg, rh, re, rf, k_even, KL, 42, 0, XW, SCHED_W_W0W1W2W3W4W5_1, 48) 470 R2(rb, rc, rd, ra, rf, rg, rh, re, k_odd, _, 43, 1, XW, SCHED_W_W0W1W2W3W4W5_2, 48) 471 R2(ra, rb, rc, rd, re, rf, rg, rh, k_even, KL, 44, 2, XW, SCHED_W_W0W1W2W3W4W5_3, 48) 472 473 /* Transform 45-47 + Precalc 51-53 */ 474 R2(rd, ra, rb, rc, rh, re, rf, rg, k_odd, _, 45, 0, XW, SCHED_W_W1W2W3W4W5W0_1, 51) 475 R2(rc, rd, ra, rb, rg, rh, re, rf, k_even, KL, 46, 1, XW, SCHED_W_W1W2W3W4W5W0_2, 51) 476 R2(rb, rc, rd, ra, rf, rg, rh, re, k_odd, _, 47, 2, XW, SCHED_W_W1W2W3W4W5W0_3, 51) 477 478 /* Transform 48-50 + Precalc 54-56 */ 479 R2(ra, rb, rc, rd, re, rf, rg, rh, k_even, KL, 48, 0, XW, SCHED_W_W2W3W4W5W0W1_1, 54) 480 R2(rd, ra, rb, rc, rh, re, rf, rg, k_odd, _, 49, 1, XW, SCHED_W_W2W3W4W5W0W1_2, 54) 481 R2(rc, rd, ra, rb, rg, rh, re, rf, k_even, KL, 50, 2, XW, SCHED_W_W2W3W4W5W0W1_3, 54) 482 483 /* Transform 51-53 + Precalc 57-59 */ 484 R2(rb, rc, rd, ra, rf, rg, rh, re, k_odd, _, 51, 0, XW, SCHED_W_W3W4W5W0W1W2_1, 57) 485 R2(ra, rb, rc, rd, re, rf, rg, rh, k_even, KL, 52, 1, XW, SCHED_W_W3W4W5W0W1W2_2, 57) 486 R2(rd, ra, rb, rc, rh, re, rf, rg, k_odd, _, 53, 2, XW, SCHED_W_W3W4W5W0W1W2_3, 57) 487 488 /* Transform 54-56 + Precalc 60-62 */ 489 R2(rc, rd, ra, rb, rg, rh, re, rf, k_even, KL, 54, 0, XW, SCHED_W_W4W5W0W1W2W3_1, 60) 490 R2(rb, rc, rd, ra, rf, rg, rh, re, k_odd, _, 55, 1, XW, SCHED_W_W4W5W0W1W2W3_2, 60) 491 R2(ra, rb, rc, rd, re, rf, rg, rh, k_even, KL, 56, 2, XW, SCHED_W_W4W5W0W1W2W3_3, 60) 492 493 /* Transform 57-59 + Precalc 63 */ 494 R2(rd, ra, rb, rc, rh, re, rf, rg, k_odd, _, 57, 0, XW, SCHED_W_W5W0W1W2W3W4_1, 63) 495 R2(rc, rd, ra, rb, rg, rh, re, rf, k_even, KL, 58, 1, XW, SCHED_W_W5W0W1W2W3W4_2, 63) 496 R2(rb, rc, rd, ra, rf, rg, rh, re, k_odd, _, 59, 2, XW, SCHED_W_W5W0W1W2W3W4_3, 63) 497 498 /* Transform 60 */ 499 R2(ra, rb, rc, rd, re, rf, rg, rh, k_even, KL, 60, 0, XW, _, _) 500 subs RNBLKS, RNBLKS, #1 501 b.eq .Lend 502 503 /* Transform 61-63 + Preload next block */ 504 R2(rd, ra, rb, rc, rh, re, rf, rg, k_odd, _, 61, 1, XW, LOAD_W_VEC_1, _) 505 ldp s0, s1, [RSTATE, #0] 506 R2(rc, rd, ra, rb, rg, rh, re, rf, k_even, KL, 62, 2, XW, LOAD_W_VEC_2, _) 507 ldp s2, s3, [RSTATE, #8] 508 R2(rb, rc, rd, ra, rf, rg, rh, re, k_odd, _, 63, 0, XW, LOAD_W_VEC_3, _) 509 510 /* Update the chaining variables. */ 511 eor ra, ra, s0 512 eor rb, rb, s1 513 ldp s0, s1, [RSTATE, #16] 514 eor rc, rc, s2 515 ldp k_even, k_odd, [RSTATE, #24] 516 eor rd, rd, s3 517 eor re, re, s0 518 stp ra, rb, [RSTATE, #0] 519 eor rf, rf, s1 520 stp rc, rd, [RSTATE, #8] 521 eor rg, rg, k_even 522 stp re, rf, [RSTATE, #16] 523 eor rh, rh, k_odd 524 stp rg, rh, [RSTATE, #24] 525 b .Loop 526 527 .Lend: 528 /* Transform 61-63 */ 529 R2(rd, ra, rb, rc, rh, re, rf, rg, k_odd, _, 61, 1, XW, _, _) 530 ldp s0, s1, [RSTATE, #0] 531 R2(rc, rd, ra, rb, rg, rh, re, rf, k_even, KL, 62, 2, XW, _, _) 532 ldp s2, s3, [RSTATE, #8] 533 R2(rb, rc, rd, ra, rf, rg, rh, re, k_odd, _, 63, 0, XW, _, _) 534 535 /* Update the chaining variables. */ 536 eor ra, ra, s0 537 clear_vec(W0) 538 eor rb, rb, s1 539 clear_vec(W1) 540 ldp s0, s1, [RSTATE, #16] 541 clear_vec(W2) 542 eor rc, rc, s2 543 clear_vec(W3) 544 ldp k_even, k_odd, [RSTATE, #24] 545 clear_vec(W4) 546 eor rd, rd, s3 547 clear_vec(W5) 548 eor re, re, s0 549 clear_vec(XTMP0) 550 stp ra, rb, [RSTATE, #0] 551 clear_vec(XTMP1) 552 eor rf, rf, s1 553 clear_vec(XTMP2) 554 stp rc, rd, [RSTATE, #8] 555 clear_vec(XTMP3) 556 eor rg, rg, k_even 557 clear_vec(XTMP4) 558 stp re, rf, [RSTATE, #16] 559 clear_vec(XTMP5) 560 eor rh, rh, k_odd 561 clear_vec(XTMP6) 562 stp rg, rh, [RSTATE, #24] 563 564 /* Clear message expansion area */ 565 add addr0, sp, #STACK_W 566 st1 {W0.16b-W3.16b}, [addr0], #64 567 st1 {W0.16b-W3.16b}, [addr0], #64 568 st1 {W0.16b-W3.16b}, [addr0] 569 570 mov sp, RFRAME 571 572 ldp x25, x26, [sp], #16 573 ldp x23, x24, [sp], #16 574 ldp x21, x22, [sp], #16 575 ldp x19, x20, [sp], #16 576 ldp x28, x29, [sp], #16 577 578 ret 579 SYM_FUNC_END(sm3_neon_transform) 580 581 582 .section ".rodata", "a" 583 584 .align 4 585 .LKtable: 586 .long 0x79cc4519, 0xf3988a32, 0xe7311465, 0xce6228cb 587 .long 0x9cc45197, 0x3988a32f, 0x7311465e, 0xe6228cbc 588 .long 0xcc451979, 0x988a32f3, 0x311465e7, 0x6228cbce 589 .long 0xc451979c, 0x88a32f39, 0x11465e73, 0x228cbce6 590 .long 0x9d8a7a87, 0x3b14f50f, 0x7629ea1e, 0xec53d43c 591 .long 0xd8a7a879, 0xb14f50f3, 0x629ea1e7, 0xc53d43ce 592 .long 0x8a7a879d, 0x14f50f3b, 0x29ea1e76, 0x53d43cec 593 .long 0xa7a879d8, 0x4f50f3b1, 0x9ea1e762, 0x3d43cec5 594 .long 0x7a879d8a, 0xf50f3b14, 0xea1e7629, 0xd43cec53 595 .long 0xa879d8a7, 0x50f3b14f, 0xa1e7629e, 0x43cec53d 596 .long 0x879d8a7a, 0x0f3b14f5, 0x1e7629ea, 0x3cec53d4 597 .long 0x79d8a7a8, 0xf3b14f50, 0xe7629ea1, 0xcec53d43 598 .long 0x9d8a7a87, 0x3b14f50f, 0x7629ea1e, 0xec53d43c 599 .long 0xd8a7a879, 0xb14f50f3, 0x629ea1e7, 0xc53d43ce 600 .long 0x8a7a879d, 0x14f50f3b, 0x29ea1e76, 0x53d43cec 601 .long 0xa7a879d8, 0x4f50f3b1, 0x9ea1e762, 0x3d43cec5 602