1 /* SPDX-License-Identifier: GPL-2.0-or-later */ 2 /* 3 * SM4 Cipher Algorithm for ARMv8 NEON 4 * as specified in 5 * https://tools.ietf.org/id/draft-ribose-cfrg-sm4-10.html 6 * 7 * Copyright (C) 2022, Alibaba Group. 8 * Copyright (C) 2022 Tianjia Zhang <tianjia.zhang@linux.alibaba.com> 9 */ 10 11 #include <linux/linkage.h> 12 #include <asm/assembler.h> 13 14 /* Register macros */ 15 16 #define RTMP0 v8 17 #define RTMP1 v9 18 #define RTMP2 v10 19 #define RTMP3 v11 20 21 #define RTMP4 v12 22 #define RTMP5 v13 23 #define RTMP6 v14 24 #define RTMP7 v15 25 26 #define RX0 v12 27 #define RX1 v13 28 #define RKEY v14 29 #define RIV v15 30 31 /* Helper macros. */ 32 33 #define SM4_PREPARE() \ 34 adr_l x5, crypto_sm4_sbox; \ 35 ld1 {v16.16b-v19.16b}, [x5], #64; \ 36 ld1 {v20.16b-v23.16b}, [x5], #64; \ 37 ld1 {v24.16b-v27.16b}, [x5], #64; \ 38 ld1 {v28.16b-v31.16b}, [x5]; 39 40 #define transpose_4x4(s0, s1, s2, s3) \ 41 zip1 RTMP0.4s, s0.4s, s1.4s; \ 42 zip1 RTMP1.4s, s2.4s, s3.4s; \ 43 zip2 RTMP2.4s, s0.4s, s1.4s; \ 44 zip2 RTMP3.4s, s2.4s, s3.4s; \ 45 zip1 s0.2d, RTMP0.2d, RTMP1.2d; \ 46 zip2 s1.2d, RTMP0.2d, RTMP1.2d; \ 47 zip1 s2.2d, RTMP2.2d, RTMP3.2d; \ 48 zip2 s3.2d, RTMP2.2d, RTMP3.2d; 49 50 #define transpose_4x4_2x(s0, s1, s2, s3, s4, s5, s6, s7) \ 51 zip1 RTMP0.4s, s0.4s, s1.4s; \ 52 zip1 RTMP1.4s, s2.4s, s3.4s; \ 53 zip2 RTMP2.4s, s0.4s, s1.4s; \ 54 zip2 RTMP3.4s, s2.4s, s3.4s; \ 55 zip1 RTMP4.4s, s4.4s, s5.4s; \ 56 zip1 RTMP5.4s, s6.4s, s7.4s; \ 57 zip2 RTMP6.4s, s4.4s, s5.4s; \ 58 zip2 RTMP7.4s, s6.4s, s7.4s; \ 59 zip1 s0.2d, RTMP0.2d, RTMP1.2d; \ 60 zip2 s1.2d, RTMP0.2d, RTMP1.2d; \ 61 zip1 s2.2d, RTMP2.2d, RTMP3.2d; \ 62 zip2 s3.2d, RTMP2.2d, RTMP3.2d; \ 63 zip1 s4.2d, RTMP4.2d, RTMP5.2d; \ 64 zip2 s5.2d, RTMP4.2d, RTMP5.2d; \ 65 zip1 s6.2d, RTMP6.2d, RTMP7.2d; \ 66 zip2 s7.2d, RTMP6.2d, RTMP7.2d; 67 68 #define rotate_clockwise_4x4(s0, s1, s2, s3) \ 69 zip1 RTMP0.4s, s1.4s, s0.4s; \ 70 zip2 RTMP1.4s, s1.4s, s0.4s; \ 71 zip1 RTMP2.4s, s3.4s, s2.4s; \ 72 zip2 RTMP3.4s, s3.4s, s2.4s; \ 73 zip1 s0.2d, RTMP2.2d, RTMP0.2d; \ 74 zip2 s1.2d, RTMP2.2d, RTMP0.2d; \ 75 zip1 s2.2d, RTMP3.2d, RTMP1.2d; \ 76 zip2 s3.2d, RTMP3.2d, RTMP1.2d; 77 78 #define rotate_clockwise_4x4_2x(s0, s1, s2, s3, s4, s5, s6, s7) \ 79 zip1 RTMP0.4s, s1.4s, s0.4s; \ 80 zip1 RTMP2.4s, s3.4s, s2.4s; \ 81 zip2 RTMP1.4s, s1.4s, s0.4s; \ 82 zip2 RTMP3.4s, s3.4s, s2.4s; \ 83 zip1 RTMP4.4s, s5.4s, s4.4s; \ 84 zip1 RTMP6.4s, s7.4s, s6.4s; \ 85 zip2 RTMP5.4s, s5.4s, s4.4s; \ 86 zip2 RTMP7.4s, s7.4s, s6.4s; \ 87 zip1 s0.2d, RTMP2.2d, RTMP0.2d; \ 88 zip2 s1.2d, RTMP2.2d, RTMP0.2d; \ 89 zip1 s2.2d, RTMP3.2d, RTMP1.2d; \ 90 zip2 s3.2d, RTMP3.2d, RTMP1.2d; \ 91 zip1 s4.2d, RTMP6.2d, RTMP4.2d; \ 92 zip2 s5.2d, RTMP6.2d, RTMP4.2d; \ 93 zip1 s6.2d, RTMP7.2d, RTMP5.2d; \ 94 zip2 s7.2d, RTMP7.2d, RTMP5.2d; 95 96 #define ROUND4(round, s0, s1, s2, s3) \ 97 dup RX0.4s, RKEY.s[round]; \ 98 /* rk ^ s1 ^ s2 ^ s3 */ \ 99 eor RTMP1.16b, s2.16b, s3.16b; \ 100 eor RX0.16b, RX0.16b, s1.16b; \ 101 eor RX0.16b, RX0.16b, RTMP1.16b; \ 102 \ 103 /* sbox, non-linear part */ \ 104 movi RTMP3.16b, #64; /* sizeof(sbox) / 4 */ \ 105 tbl RTMP0.16b, {v16.16b-v19.16b}, RX0.16b; \ 106 sub RX0.16b, RX0.16b, RTMP3.16b; \ 107 tbx RTMP0.16b, {v20.16b-v23.16b}, RX0.16b; \ 108 sub RX0.16b, RX0.16b, RTMP3.16b; \ 109 tbx RTMP0.16b, {v24.16b-v27.16b}, RX0.16b; \ 110 sub RX0.16b, RX0.16b, RTMP3.16b; \ 111 tbx RTMP0.16b, {v28.16b-v31.16b}, RX0.16b; \ 112 \ 113 /* linear part */ \ 114 shl RTMP1.4s, RTMP0.4s, #8; \ 115 shl RTMP2.4s, RTMP0.4s, #16; \ 116 shl RTMP3.4s, RTMP0.4s, #24; \ 117 sri RTMP1.4s, RTMP0.4s, #(32-8); \ 118 sri RTMP2.4s, RTMP0.4s, #(32-16); \ 119 sri RTMP3.4s, RTMP0.4s, #(32-24); \ 120 /* RTMP1 = x ^ rol32(x, 8) ^ rol32(x, 16) */ \ 121 eor RTMP1.16b, RTMP1.16b, RTMP0.16b; \ 122 eor RTMP1.16b, RTMP1.16b, RTMP2.16b; \ 123 /* RTMP3 = x ^ rol32(x, 24) ^ rol32(RTMP1, 2) */ \ 124 eor RTMP3.16b, RTMP3.16b, RTMP0.16b; \ 125 shl RTMP2.4s, RTMP1.4s, 2; \ 126 sri RTMP2.4s, RTMP1.4s, #(32-2); \ 127 eor RTMP3.16b, RTMP3.16b, RTMP2.16b; \ 128 /* s0 ^= RTMP3 */ \ 129 eor s0.16b, s0.16b, RTMP3.16b; 130 131 #define SM4_CRYPT_BLK4_BE(b0, b1, b2, b3) \ 132 mov x6, 8; \ 133 4: \ 134 ld1 {RKEY.4s}, [x0], #16; \ 135 subs x6, x6, #1; \ 136 \ 137 ROUND4(0, b0, b1, b2, b3); \ 138 ROUND4(1, b1, b2, b3, b0); \ 139 ROUND4(2, b2, b3, b0, b1); \ 140 ROUND4(3, b3, b0, b1, b2); \ 141 \ 142 bne 4b; \ 143 \ 144 rev32 b0.16b, b0.16b; \ 145 rev32 b1.16b, b1.16b; \ 146 rev32 b2.16b, b2.16b; \ 147 rev32 b3.16b, b3.16b; \ 148 \ 149 rotate_clockwise_4x4(b0, b1, b2, b3); \ 150 \ 151 /* repoint to rkey */ \ 152 sub x0, x0, #128; 153 154 #define SM4_CRYPT_BLK4(b0, b1, b2, b3) \ 155 rev32 b0.16b, b0.16b; \ 156 rev32 b1.16b, b1.16b; \ 157 rev32 b2.16b, b2.16b; \ 158 rev32 b3.16b, b3.16b; \ 159 SM4_CRYPT_BLK4_BE(b0, b1, b2, b3); 160 161 #define ROUND8(round, s0, s1, s2, s3, t0, t1, t2, t3) \ 162 /* rk ^ s1 ^ s2 ^ s3 */ \ 163 dup RX0.4s, RKEY.s[round]; \ 164 eor RTMP0.16b, s2.16b, s3.16b; \ 165 mov RX1.16b, RX0.16b; \ 166 eor RTMP1.16b, t2.16b, t3.16b; \ 167 eor RX0.16b, RX0.16b, s1.16b; \ 168 eor RX1.16b, RX1.16b, t1.16b; \ 169 eor RX0.16b, RX0.16b, RTMP0.16b; \ 170 eor RX1.16b, RX1.16b, RTMP1.16b; \ 171 \ 172 /* sbox, non-linear part */ \ 173 movi RTMP3.16b, #64; /* sizeof(sbox) / 4 */ \ 174 tbl RTMP0.16b, {v16.16b-v19.16b}, RX0.16b; \ 175 tbl RTMP1.16b, {v16.16b-v19.16b}, RX1.16b; \ 176 sub RX0.16b, RX0.16b, RTMP3.16b; \ 177 sub RX1.16b, RX1.16b, RTMP3.16b; \ 178 tbx RTMP0.16b, {v20.16b-v23.16b}, RX0.16b; \ 179 tbx RTMP1.16b, {v20.16b-v23.16b}, RX1.16b; \ 180 sub RX0.16b, RX0.16b, RTMP3.16b; \ 181 sub RX1.16b, RX1.16b, RTMP3.16b; \ 182 tbx RTMP0.16b, {v24.16b-v27.16b}, RX0.16b; \ 183 tbx RTMP1.16b, {v24.16b-v27.16b}, RX1.16b; \ 184 sub RX0.16b, RX0.16b, RTMP3.16b; \ 185 sub RX1.16b, RX1.16b, RTMP3.16b; \ 186 tbx RTMP0.16b, {v28.16b-v31.16b}, RX0.16b; \ 187 tbx RTMP1.16b, {v28.16b-v31.16b}, RX1.16b; \ 188 \ 189 /* linear part */ \ 190 shl RX0.4s, RTMP0.4s, #8; \ 191 shl RX1.4s, RTMP1.4s, #8; \ 192 shl RTMP2.4s, RTMP0.4s, #16; \ 193 shl RTMP3.4s, RTMP1.4s, #16; \ 194 sri RX0.4s, RTMP0.4s, #(32 - 8); \ 195 sri RX1.4s, RTMP1.4s, #(32 - 8); \ 196 sri RTMP2.4s, RTMP0.4s, #(32 - 16); \ 197 sri RTMP3.4s, RTMP1.4s, #(32 - 16); \ 198 /* RX = x ^ rol32(x, 8) ^ rol32(x, 16) */ \ 199 eor RX0.16b, RX0.16b, RTMP0.16b; \ 200 eor RX1.16b, RX1.16b, RTMP1.16b; \ 201 eor RX0.16b, RX0.16b, RTMP2.16b; \ 202 eor RX1.16b, RX1.16b, RTMP3.16b; \ 203 /* RTMP0/1 ^= x ^ rol32(x, 24) ^ rol32(RX, 2) */ \ 204 shl RTMP2.4s, RTMP0.4s, #24; \ 205 shl RTMP3.4s, RTMP1.4s, #24; \ 206 sri RTMP2.4s, RTMP0.4s, #(32 - 24); \ 207 sri RTMP3.4s, RTMP1.4s, #(32 - 24); \ 208 eor RTMP0.16b, RTMP0.16b, RTMP2.16b; \ 209 eor RTMP1.16b, RTMP1.16b, RTMP3.16b; \ 210 shl RTMP2.4s, RX0.4s, #2; \ 211 shl RTMP3.4s, RX1.4s, #2; \ 212 sri RTMP2.4s, RX0.4s, #(32 - 2); \ 213 sri RTMP3.4s, RX1.4s, #(32 - 2); \ 214 eor RTMP0.16b, RTMP0.16b, RTMP2.16b; \ 215 eor RTMP1.16b, RTMP1.16b, RTMP3.16b; \ 216 /* s0/t0 ^= RTMP0/1 */ \ 217 eor s0.16b, s0.16b, RTMP0.16b; \ 218 eor t0.16b, t0.16b, RTMP1.16b; 219 220 #define SM4_CRYPT_BLK8_norotate(b0, b1, b2, b3, b4, b5, b6, b7) \ 221 rev32 b0.16b, b0.16b; \ 222 rev32 b1.16b, b1.16b; \ 223 rev32 b2.16b, b2.16b; \ 224 rev32 b3.16b, b3.16b; \ 225 rev32 b4.16b, b4.16b; \ 226 rev32 b5.16b, b5.16b; \ 227 rev32 b6.16b, b6.16b; \ 228 rev32 b7.16b, b7.16b; \ 229 \ 230 mov x6, 8; \ 231 8: \ 232 ld1 {RKEY.4s}, [x0], #16; \ 233 subs x6, x6, #1; \ 234 \ 235 ROUND8(0, b0, b1, b2, b3, b4, b5, b6, b7); \ 236 ROUND8(1, b1, b2, b3, b0, b5, b6, b7, b4); \ 237 ROUND8(2, b2, b3, b0, b1, b6, b7, b4, b5); \ 238 ROUND8(3, b3, b0, b1, b2, b7, b4, b5, b6); \ 239 \ 240 bne 8b; \ 241 \ 242 rev32 b0.16b, b0.16b; \ 243 rev32 b1.16b, b1.16b; \ 244 rev32 b2.16b, b2.16b; \ 245 rev32 b3.16b, b3.16b; \ 246 rev32 b4.16b, b4.16b; \ 247 rev32 b5.16b, b5.16b; \ 248 rev32 b6.16b, b6.16b; \ 249 rev32 b7.16b, b7.16b; \ 250 \ 251 /* repoint to rkey */ \ 252 sub x0, x0, #128; 253 254 #define SM4_CRYPT_BLK8(b0, b1, b2, b3, b4, b5, b6, b7) \ 255 SM4_CRYPT_BLK8_norotate(b0, b1, b2, b3, b4, b5, b6, b7); \ 256 rotate_clockwise_4x4_2x(b0, b1, b2, b3, b4, b5, b6, b7); \ 257 258 259 .align 3 260 SYM_FUNC_START(sm4_neon_crypt) 261 /* input: 262 * x0: round key array, CTX 263 * x1: dst 264 * x2: src 265 * w3: nblocks 266 */ 267 SM4_PREPARE() 268 269 .Lcrypt_loop_8x: 270 sub w3, w3, #8 271 tbnz w3, #31, .Lcrypt_4x 272 273 ld4 {v0.4s-v3.4s}, [x2], #64 274 ld4 {v4.4s-v7.4s}, [x2], #64 275 276 SM4_CRYPT_BLK8(v0, v1, v2, v3, v4, v5, v6, v7) 277 278 st1 {v0.16b-v3.16b}, [x1], #64 279 st1 {v4.16b-v7.16b}, [x1], #64 280 281 cbz w3, .Lcrypt_end 282 b .Lcrypt_loop_8x 283 284 .Lcrypt_4x: 285 add w3, w3, #8 286 cmp w3, #4 287 blt .Lcrypt_tail 288 289 sub w3, w3, #4 290 291 ld4 {v0.4s-v3.4s}, [x2], #64 292 293 SM4_CRYPT_BLK4(v0, v1, v2, v3) 294 295 st1 {v0.16b-v3.16b}, [x1], #64 296 297 cbz w3, .Lcrypt_end 298 299 .Lcrypt_tail: 300 cmp w3, #2 301 ld1 {v0.16b}, [x2], #16 302 blt .Lcrypt_tail_load_done 303 ld1 {v1.16b}, [x2], #16 304 beq .Lcrypt_tail_load_done 305 ld1 {v2.16b}, [x2], #16 306 307 .Lcrypt_tail_load_done: 308 transpose_4x4(v0, v1, v2, v3) 309 310 SM4_CRYPT_BLK4(v0, v1, v2, v3) 311 312 cmp w3, #2 313 st1 {v0.16b}, [x1], #16 314 blt .Lcrypt_end 315 st1 {v1.16b}, [x1], #16 316 beq .Lcrypt_end 317 st1 {v2.16b}, [x1], #16 318 319 .Lcrypt_end: 320 ret 321 SYM_FUNC_END(sm4_neon_crypt) 322 323 .align 3 324 SYM_FUNC_START(sm4_neon_cbc_dec) 325 /* input: 326 * x0: round key array, CTX 327 * x1: dst 328 * x2: src 329 * x3: iv (big endian, 128 bit) 330 * w4: nblocks 331 */ 332 SM4_PREPARE() 333 334 ld1 {RIV.16b}, [x3] 335 336 .Lcbc_dec_loop_8x: 337 sub w4, w4, #8 338 tbnz w4, #31, .Lcbc_dec_4x 339 340 ld4 {v0.4s-v3.4s}, [x2], #64 341 ld4 {v4.4s-v7.4s}, [x2] 342 343 SM4_CRYPT_BLK8_norotate(v0, v1, v2, v3, v4, v5, v6, v7) 344 345 /* Avoid overwriting the RIV register */ 346 rotate_clockwise_4x4(v0, v1, v2, v3) 347 rotate_clockwise_4x4(v4, v5, v6, v7) 348 349 sub x2, x2, #64 350 351 eor v0.16b, v0.16b, RIV.16b 352 353 ld1 {RTMP0.16b-RTMP3.16b}, [x2], #64 354 ld1 {RTMP4.16b-RTMP7.16b}, [x2], #64 355 356 eor v1.16b, v1.16b, RTMP0.16b 357 eor v2.16b, v2.16b, RTMP1.16b 358 eor v3.16b, v3.16b, RTMP2.16b 359 eor v4.16b, v4.16b, RTMP3.16b 360 eor v5.16b, v5.16b, RTMP4.16b 361 eor v6.16b, v6.16b, RTMP5.16b 362 eor v7.16b, v7.16b, RTMP6.16b 363 364 mov RIV.16b, RTMP7.16b 365 366 st1 {v0.16b-v3.16b}, [x1], #64 367 st1 {v4.16b-v7.16b}, [x1], #64 368 369 cbz w4, .Lcbc_dec_end 370 b .Lcbc_dec_loop_8x 371 372 .Lcbc_dec_4x: 373 add w4, w4, #8 374 cmp w4, #4 375 blt .Lcbc_dec_tail 376 377 sub w4, w4, #4 378 379 ld1 {v0.16b-v3.16b}, [x2], #64 380 381 rev32 v4.16b, v0.16b 382 rev32 v5.16b, v1.16b 383 rev32 v6.16b, v2.16b 384 rev32 v7.16b, v3.16b 385 386 transpose_4x4(v4, v5, v6, v7) 387 388 SM4_CRYPT_BLK4_BE(v4, v5, v6, v7) 389 390 eor v4.16b, v4.16b, RIV.16b 391 eor v5.16b, v5.16b, v0.16b 392 eor v6.16b, v6.16b, v1.16b 393 eor v7.16b, v7.16b, v2.16b 394 395 mov RIV.16b, v3.16b 396 397 st1 {v4.16b-v7.16b}, [x1], #64 398 399 cbz w4, .Lcbc_dec_end 400 401 .Lcbc_dec_tail: 402 cmp w4, #2 403 ld1 {v0.16b}, [x2], #16 404 blt .Lcbc_dec_tail_load_done 405 ld1 {v1.16b}, [x2], #16 406 beq .Lcbc_dec_tail_load_done 407 ld1 {v2.16b}, [x2], #16 408 409 .Lcbc_dec_tail_load_done: 410 rev32 v4.16b, v0.16b 411 rev32 v5.16b, v1.16b 412 rev32 v6.16b, v2.16b 413 414 transpose_4x4(v4, v5, v6, v7) 415 416 SM4_CRYPT_BLK4_BE(v4, v5, v6, v7) 417 418 cmp w4, #2 419 eor v4.16b, v4.16b, RIV.16b 420 mov RIV.16b, v0.16b 421 st1 {v4.16b}, [x1], #16 422 blt .Lcbc_dec_end 423 424 eor v5.16b, v5.16b, v0.16b 425 mov RIV.16b, v1.16b 426 st1 {v5.16b}, [x1], #16 427 beq .Lcbc_dec_end 428 429 eor v6.16b, v6.16b, v1.16b 430 mov RIV.16b, v2.16b 431 st1 {v6.16b}, [x1], #16 432 433 .Lcbc_dec_end: 434 /* store new IV */ 435 st1 {RIV.16b}, [x3] 436 437 ret 438 SYM_FUNC_END(sm4_neon_cbc_dec) 439 440 .align 3 441 SYM_FUNC_START(sm4_neon_cfb_dec) 442 /* input: 443 * x0: round key array, CTX 444 * x1: dst 445 * x2: src 446 * x3: iv (big endian, 128 bit) 447 * w4: nblocks 448 */ 449 SM4_PREPARE() 450 451 ld1 {v0.16b}, [x3] 452 453 .Lcfb_dec_loop_8x: 454 sub w4, w4, #8 455 tbnz w4, #31, .Lcfb_dec_4x 456 457 ld1 {v1.16b-v3.16b}, [x2], #48 458 ld4 {v4.4s-v7.4s}, [x2] 459 460 transpose_4x4(v0, v1, v2, v3) 461 462 SM4_CRYPT_BLK8(v0, v1, v2, v3, v4, v5, v6, v7) 463 464 sub x2, x2, #48 465 ld1 {RTMP0.16b-RTMP3.16b}, [x2], #64 466 ld1 {RTMP4.16b-RTMP7.16b}, [x2], #64 467 468 eor v0.16b, v0.16b, RTMP0.16b 469 eor v1.16b, v1.16b, RTMP1.16b 470 eor v2.16b, v2.16b, RTMP2.16b 471 eor v3.16b, v3.16b, RTMP3.16b 472 eor v4.16b, v4.16b, RTMP4.16b 473 eor v5.16b, v5.16b, RTMP5.16b 474 eor v6.16b, v6.16b, RTMP6.16b 475 eor v7.16b, v7.16b, RTMP7.16b 476 477 st1 {v0.16b-v3.16b}, [x1], #64 478 st1 {v4.16b-v7.16b}, [x1], #64 479 480 mov v0.16b, RTMP7.16b 481 482 cbz w4, .Lcfb_dec_end 483 b .Lcfb_dec_loop_8x 484 485 .Lcfb_dec_4x: 486 add w4, w4, #8 487 cmp w4, #4 488 blt .Lcfb_dec_tail 489 490 sub w4, w4, #4 491 492 ld1 {v4.16b-v7.16b}, [x2], #64 493 494 rev32 v0.16b, v0.16b /* v0 is IV register */ 495 rev32 v1.16b, v4.16b 496 rev32 v2.16b, v5.16b 497 rev32 v3.16b, v6.16b 498 499 transpose_4x4(v0, v1, v2, v3) 500 501 SM4_CRYPT_BLK4_BE(v0, v1, v2, v3) 502 503 eor v0.16b, v0.16b, v4.16b 504 eor v1.16b, v1.16b, v5.16b 505 eor v2.16b, v2.16b, v6.16b 506 eor v3.16b, v3.16b, v7.16b 507 508 st1 {v0.16b-v3.16b}, [x1], #64 509 510 mov v0.16b, v7.16b 511 512 cbz w4, .Lcfb_dec_end 513 514 .Lcfb_dec_tail: 515 cmp w4, #2 516 ld1 {v4.16b}, [x2], #16 517 blt .Lcfb_dec_tail_load_done 518 ld1 {v5.16b}, [x2], #16 519 beq .Lcfb_dec_tail_load_done 520 ld1 {v6.16b}, [x2], #16 521 522 .Lcfb_dec_tail_load_done: 523 rev32 v0.16b, v0.16b /* v0 is IV register */ 524 rev32 v1.16b, v4.16b 525 rev32 v2.16b, v5.16b 526 527 transpose_4x4(v0, v1, v2, v3) 528 529 SM4_CRYPT_BLK4_BE(v0, v1, v2, v3) 530 531 cmp w4, #2 532 eor v0.16b, v0.16b, v4.16b 533 st1 {v0.16b}, [x1], #16 534 mov v0.16b, v4.16b 535 blt .Lcfb_dec_end 536 537 eor v1.16b, v1.16b, v5.16b 538 st1 {v1.16b}, [x1], #16 539 mov v0.16b, v5.16b 540 beq .Lcfb_dec_end 541 542 eor v2.16b, v2.16b, v6.16b 543 st1 {v2.16b}, [x1], #16 544 mov v0.16b, v6.16b 545 546 .Lcfb_dec_end: 547 /* store new IV */ 548 st1 {v0.16b}, [x3] 549 550 ret 551 SYM_FUNC_END(sm4_neon_cfb_dec) 552 553 .align 3 554 SYM_FUNC_START(sm4_neon_ctr_crypt) 555 /* input: 556 * x0: round key array, CTX 557 * x1: dst 558 * x2: src 559 * x3: ctr (big endian, 128 bit) 560 * w4: nblocks 561 */ 562 SM4_PREPARE() 563 564 ldp x7, x8, [x3] 565 rev x7, x7 566 rev x8, x8 567 568 .Lctr_crypt_loop_8x: 569 sub w4, w4, #8 570 tbnz w4, #31, .Lctr_crypt_4x 571 572 #define inc_le128(vctr) \ 573 mov vctr.d[1], x8; \ 574 mov vctr.d[0], x7; \ 575 adds x8, x8, #1; \ 576 rev64 vctr.16b, vctr.16b; \ 577 adc x7, x7, xzr; 578 579 /* construct CTRs */ 580 inc_le128(v0) /* +0 */ 581 inc_le128(v1) /* +1 */ 582 inc_le128(v2) /* +2 */ 583 inc_le128(v3) /* +3 */ 584 inc_le128(v4) /* +4 */ 585 inc_le128(v5) /* +5 */ 586 inc_le128(v6) /* +6 */ 587 inc_le128(v7) /* +7 */ 588 589 transpose_4x4_2x(v0, v1, v2, v3, v4, v5, v6, v7) 590 591 SM4_CRYPT_BLK8(v0, v1, v2, v3, v4, v5, v6, v7) 592 593 ld1 {RTMP0.16b-RTMP3.16b}, [x2], #64 594 ld1 {RTMP4.16b-RTMP7.16b}, [x2], #64 595 596 eor v0.16b, v0.16b, RTMP0.16b 597 eor v1.16b, v1.16b, RTMP1.16b 598 eor v2.16b, v2.16b, RTMP2.16b 599 eor v3.16b, v3.16b, RTMP3.16b 600 eor v4.16b, v4.16b, RTMP4.16b 601 eor v5.16b, v5.16b, RTMP5.16b 602 eor v6.16b, v6.16b, RTMP6.16b 603 eor v7.16b, v7.16b, RTMP7.16b 604 605 st1 {v0.16b-v3.16b}, [x1], #64 606 st1 {v4.16b-v7.16b}, [x1], #64 607 608 cbz w4, .Lctr_crypt_end 609 b .Lctr_crypt_loop_8x 610 611 .Lctr_crypt_4x: 612 add w4, w4, #8 613 cmp w4, #4 614 blt .Lctr_crypt_tail 615 616 sub w4, w4, #4 617 618 /* construct CTRs */ 619 inc_le128(v0) /* +0 */ 620 inc_le128(v1) /* +1 */ 621 inc_le128(v2) /* +2 */ 622 inc_le128(v3) /* +3 */ 623 624 ld1 {v4.16b-v7.16b}, [x2], #64 625 626 transpose_4x4(v0, v1, v2, v3) 627 628 SM4_CRYPT_BLK4(v0, v1, v2, v3) 629 630 eor v0.16b, v0.16b, v4.16b 631 eor v1.16b, v1.16b, v5.16b 632 eor v2.16b, v2.16b, v6.16b 633 eor v3.16b, v3.16b, v7.16b 634 635 st1 {v0.16b-v3.16b}, [x1], #64 636 637 cbz w4, .Lctr_crypt_end 638 639 .Lctr_crypt_tail: 640 /* inc_le128 will change the sign bit */ 641 ld1 {v4.16b}, [x2], #16 642 inc_le128(v0) 643 cmp w4, #2 644 blt .Lctr_crypt_tail_load_done 645 646 ld1 {v5.16b}, [x2], #16 647 inc_le128(v1) 648 cmp w4, #2 649 beq .Lctr_crypt_tail_load_done 650 651 ld1 {v6.16b}, [x2], #16 652 inc_le128(v2) 653 654 .Lctr_crypt_tail_load_done: 655 transpose_4x4(v0, v1, v2, v3) 656 657 SM4_CRYPT_BLK4(v0, v1, v2, v3) 658 659 cmp w4, #2 660 661 eor v0.16b, v0.16b, v4.16b 662 st1 {v0.16b}, [x1], #16 663 blt .Lctr_crypt_end 664 665 eor v1.16b, v1.16b, v5.16b 666 st1 {v1.16b}, [x1], #16 667 beq .Lctr_crypt_end 668 669 eor v2.16b, v2.16b, v6.16b 670 st1 {v2.16b}, [x1], #16 671 672 .Lctr_crypt_end: 673 /* store new CTR */ 674 rev x7, x7 675 rev x8, x8 676 stp x7, x8, [x3] 677 678 ret 679 SYM_FUNC_END(sm4_neon_ctr_crypt) 680