1 /* 2 * ChaCha/XChaCha NEON helper functions 3 * 4 * Copyright (C) 2016-2018 Linaro, Ltd. <ard.biesheuvel@linaro.org> 5 * 6 * This program is free software; you can redistribute it and/or modify 7 * it under the terms of the GNU General Public License version 2 as 8 * published by the Free Software Foundation. 9 * 10 * Originally based on: 11 * ChaCha20 256-bit cipher algorithm, RFC7539, x64 SSSE3 functions 12 * 13 * Copyright (C) 2015 Martin Willi 14 * 15 * This program is free software; you can redistribute it and/or modify 16 * it under the terms of the GNU General Public License as published by 17 * the Free Software Foundation; either version 2 of the License, or 18 * (at your option) any later version. 19 */ 20 21 #include <linux/linkage.h> 22 #include <asm/assembler.h> 23 #include <asm/cache.h> 24 25 .text 26 .align 6 27 28 /* 29 * chacha_permute - permute one block 30 * 31 * Permute one 64-byte block where the state matrix is stored in the four NEON 32 * registers v0-v3. It performs matrix operations on four words in parallel, 33 * but requires shuffling to rearrange the words after each round. 34 * 35 * The round count is given in w3. 36 * 37 * Clobbers: w3, x10, v4, v12 38 */ 39 SYM_FUNC_START_LOCAL(chacha_permute) 40 41 adr_l x10, ROT8 42 ld1 {v12.4s}, [x10] 43 44 .Ldoubleround: 45 // x0 += x1, x3 = rotl32(x3 ^ x0, 16) 46 add v0.4s, v0.4s, v1.4s 47 eor v3.16b, v3.16b, v0.16b 48 rev32 v3.8h, v3.8h 49 50 // x2 += x3, x1 = rotl32(x1 ^ x2, 12) 51 add v2.4s, v2.4s, v3.4s 52 eor v4.16b, v1.16b, v2.16b 53 shl v1.4s, v4.4s, #12 54 sri v1.4s, v4.4s, #20 55 56 // x0 += x1, x3 = rotl32(x3 ^ x0, 8) 57 add v0.4s, v0.4s, v1.4s 58 eor v3.16b, v3.16b, v0.16b 59 tbl v3.16b, {v3.16b}, v12.16b 60 61 // x2 += x3, x1 = rotl32(x1 ^ x2, 7) 62 add v2.4s, v2.4s, v3.4s 63 eor v4.16b, v1.16b, v2.16b 64 shl v1.4s, v4.4s, #7 65 sri v1.4s, v4.4s, #25 66 67 // x1 = shuffle32(x1, MASK(0, 3, 2, 1)) 68 ext v1.16b, v1.16b, v1.16b, #4 69 // x2 = shuffle32(x2, MASK(1, 0, 3, 2)) 70 ext v2.16b, v2.16b, v2.16b, #8 71 // x3 = shuffle32(x3, MASK(2, 1, 0, 3)) 72 ext v3.16b, v3.16b, v3.16b, #12 73 74 // x0 += x1, x3 = rotl32(x3 ^ x0, 16) 75 add v0.4s, v0.4s, v1.4s 76 eor v3.16b, v3.16b, v0.16b 77 rev32 v3.8h, v3.8h 78 79 // x2 += x3, x1 = rotl32(x1 ^ x2, 12) 80 add v2.4s, v2.4s, v3.4s 81 eor v4.16b, v1.16b, v2.16b 82 shl v1.4s, v4.4s, #12 83 sri v1.4s, v4.4s, #20 84 85 // x0 += x1, x3 = rotl32(x3 ^ x0, 8) 86 add v0.4s, v0.4s, v1.4s 87 eor v3.16b, v3.16b, v0.16b 88 tbl v3.16b, {v3.16b}, v12.16b 89 90 // x2 += x3, x1 = rotl32(x1 ^ x2, 7) 91 add v2.4s, v2.4s, v3.4s 92 eor v4.16b, v1.16b, v2.16b 93 shl v1.4s, v4.4s, #7 94 sri v1.4s, v4.4s, #25 95 96 // x1 = shuffle32(x1, MASK(2, 1, 0, 3)) 97 ext v1.16b, v1.16b, v1.16b, #12 98 // x2 = shuffle32(x2, MASK(1, 0, 3, 2)) 99 ext v2.16b, v2.16b, v2.16b, #8 100 // x3 = shuffle32(x3, MASK(0, 3, 2, 1)) 101 ext v3.16b, v3.16b, v3.16b, #4 102 103 subs w3, w3, #2 104 b.ne .Ldoubleround 105 106 ret 107 SYM_FUNC_END(chacha_permute) 108 109 SYM_FUNC_START(chacha_block_xor_neon) 110 // x0: Input state matrix, s 111 // x1: 1 data block output, o 112 // x2: 1 data block input, i 113 // w3: nrounds 114 115 stp x29, x30, [sp, #-16]! 116 mov x29, sp 117 118 // x0..3 = s0..3 119 ld1 {v0.4s-v3.4s}, [x0] 120 ld1 {v8.4s-v11.4s}, [x0] 121 122 bl chacha_permute 123 124 ld1 {v4.16b-v7.16b}, [x2] 125 126 // o0 = i0 ^ (x0 + s0) 127 add v0.4s, v0.4s, v8.4s 128 eor v0.16b, v0.16b, v4.16b 129 130 // o1 = i1 ^ (x1 + s1) 131 add v1.4s, v1.4s, v9.4s 132 eor v1.16b, v1.16b, v5.16b 133 134 // o2 = i2 ^ (x2 + s2) 135 add v2.4s, v2.4s, v10.4s 136 eor v2.16b, v2.16b, v6.16b 137 138 // o3 = i3 ^ (x3 + s3) 139 add v3.4s, v3.4s, v11.4s 140 eor v3.16b, v3.16b, v7.16b 141 142 st1 {v0.16b-v3.16b}, [x1] 143 144 ldp x29, x30, [sp], #16 145 ret 146 SYM_FUNC_END(chacha_block_xor_neon) 147 148 SYM_FUNC_START(hchacha_block_neon) 149 // x0: Input state matrix, s 150 // x1: output (8 32-bit words) 151 // w2: nrounds 152 153 stp x29, x30, [sp, #-16]! 154 mov x29, sp 155 156 ld1 {v0.4s-v3.4s}, [x0] 157 158 mov w3, w2 159 bl chacha_permute 160 161 st1 {v0.4s}, [x1], #16 162 st1 {v3.4s}, [x1] 163 164 ldp x29, x30, [sp], #16 165 ret 166 SYM_FUNC_END(hchacha_block_neon) 167 168 a0 .req w12 169 a1 .req w13 170 a2 .req w14 171 a3 .req w15 172 a4 .req w16 173 a5 .req w17 174 a6 .req w19 175 a7 .req w20 176 a8 .req w21 177 a9 .req w22 178 a10 .req w23 179 a11 .req w24 180 a12 .req w25 181 a13 .req w26 182 a14 .req w27 183 a15 .req w28 184 185 .align 6 186 SYM_FUNC_START(chacha_4block_xor_neon) 187 frame_push 10 188 189 // x0: Input state matrix, s 190 // x1: 4 data blocks output, o 191 // x2: 4 data blocks input, i 192 // w3: nrounds 193 // x4: byte count 194 195 adr_l x10, .Lpermute 196 and x5, x4, #63 197 add x10, x10, x5 198 add x11, x10, #64 199 200 // 201 // This function encrypts four consecutive ChaCha blocks by loading 202 // the state matrix in NEON registers four times. The algorithm performs 203 // each operation on the corresponding word of each state matrix, hence 204 // requires no word shuffling. For final XORing step we transpose the 205 // matrix by interleaving 32- and then 64-bit words, which allows us to 206 // do XOR in NEON registers. 207 // 208 // At the same time, a fifth block is encrypted in parallel using 209 // scalar registers 210 // 211 adr_l x9, CTRINC // ... and ROT8 212 ld1 {v30.4s-v31.4s}, [x9] 213 214 // x0..15[0-3] = s0..3[0..3] 215 add x8, x0, #16 216 ld4r { v0.4s- v3.4s}, [x0] 217 ld4r { v4.4s- v7.4s}, [x8], #16 218 ld4r { v8.4s-v11.4s}, [x8], #16 219 ld4r {v12.4s-v15.4s}, [x8] 220 221 mov a0, v0.s[0] 222 mov a1, v1.s[0] 223 mov a2, v2.s[0] 224 mov a3, v3.s[0] 225 mov a4, v4.s[0] 226 mov a5, v5.s[0] 227 mov a6, v6.s[0] 228 mov a7, v7.s[0] 229 mov a8, v8.s[0] 230 mov a9, v9.s[0] 231 mov a10, v10.s[0] 232 mov a11, v11.s[0] 233 mov a12, v12.s[0] 234 mov a13, v13.s[0] 235 mov a14, v14.s[0] 236 mov a15, v15.s[0] 237 238 // x12 += counter values 1-4 239 add v12.4s, v12.4s, v30.4s 240 241 .Ldoubleround4: 242 // x0 += x4, x12 = rotl32(x12 ^ x0, 16) 243 // x1 += x5, x13 = rotl32(x13 ^ x1, 16) 244 // x2 += x6, x14 = rotl32(x14 ^ x2, 16) 245 // x3 += x7, x15 = rotl32(x15 ^ x3, 16) 246 add v0.4s, v0.4s, v4.4s 247 add a0, a0, a4 248 add v1.4s, v1.4s, v5.4s 249 add a1, a1, a5 250 add v2.4s, v2.4s, v6.4s 251 add a2, a2, a6 252 add v3.4s, v3.4s, v7.4s 253 add a3, a3, a7 254 255 eor v12.16b, v12.16b, v0.16b 256 eor a12, a12, a0 257 eor v13.16b, v13.16b, v1.16b 258 eor a13, a13, a1 259 eor v14.16b, v14.16b, v2.16b 260 eor a14, a14, a2 261 eor v15.16b, v15.16b, v3.16b 262 eor a15, a15, a3 263 264 rev32 v12.8h, v12.8h 265 ror a12, a12, #16 266 rev32 v13.8h, v13.8h 267 ror a13, a13, #16 268 rev32 v14.8h, v14.8h 269 ror a14, a14, #16 270 rev32 v15.8h, v15.8h 271 ror a15, a15, #16 272 273 // x8 += x12, x4 = rotl32(x4 ^ x8, 12) 274 // x9 += x13, x5 = rotl32(x5 ^ x9, 12) 275 // x10 += x14, x6 = rotl32(x6 ^ x10, 12) 276 // x11 += x15, x7 = rotl32(x7 ^ x11, 12) 277 add v8.4s, v8.4s, v12.4s 278 add a8, a8, a12 279 add v9.4s, v9.4s, v13.4s 280 add a9, a9, a13 281 add v10.4s, v10.4s, v14.4s 282 add a10, a10, a14 283 add v11.4s, v11.4s, v15.4s 284 add a11, a11, a15 285 286 eor v16.16b, v4.16b, v8.16b 287 eor a4, a4, a8 288 eor v17.16b, v5.16b, v9.16b 289 eor a5, a5, a9 290 eor v18.16b, v6.16b, v10.16b 291 eor a6, a6, a10 292 eor v19.16b, v7.16b, v11.16b 293 eor a7, a7, a11 294 295 shl v4.4s, v16.4s, #12 296 shl v5.4s, v17.4s, #12 297 shl v6.4s, v18.4s, #12 298 shl v7.4s, v19.4s, #12 299 300 sri v4.4s, v16.4s, #20 301 ror a4, a4, #20 302 sri v5.4s, v17.4s, #20 303 ror a5, a5, #20 304 sri v6.4s, v18.4s, #20 305 ror a6, a6, #20 306 sri v7.4s, v19.4s, #20 307 ror a7, a7, #20 308 309 // x0 += x4, x12 = rotl32(x12 ^ x0, 8) 310 // x1 += x5, x13 = rotl32(x13 ^ x1, 8) 311 // x2 += x6, x14 = rotl32(x14 ^ x2, 8) 312 // x3 += x7, x15 = rotl32(x15 ^ x3, 8) 313 add v0.4s, v0.4s, v4.4s 314 add a0, a0, a4 315 add v1.4s, v1.4s, v5.4s 316 add a1, a1, a5 317 add v2.4s, v2.4s, v6.4s 318 add a2, a2, a6 319 add v3.4s, v3.4s, v7.4s 320 add a3, a3, a7 321 322 eor v12.16b, v12.16b, v0.16b 323 eor a12, a12, a0 324 eor v13.16b, v13.16b, v1.16b 325 eor a13, a13, a1 326 eor v14.16b, v14.16b, v2.16b 327 eor a14, a14, a2 328 eor v15.16b, v15.16b, v3.16b 329 eor a15, a15, a3 330 331 tbl v12.16b, {v12.16b}, v31.16b 332 ror a12, a12, #24 333 tbl v13.16b, {v13.16b}, v31.16b 334 ror a13, a13, #24 335 tbl v14.16b, {v14.16b}, v31.16b 336 ror a14, a14, #24 337 tbl v15.16b, {v15.16b}, v31.16b 338 ror a15, a15, #24 339 340 // x8 += x12, x4 = rotl32(x4 ^ x8, 7) 341 // x9 += x13, x5 = rotl32(x5 ^ x9, 7) 342 // x10 += x14, x6 = rotl32(x6 ^ x10, 7) 343 // x11 += x15, x7 = rotl32(x7 ^ x11, 7) 344 add v8.4s, v8.4s, v12.4s 345 add a8, a8, a12 346 add v9.4s, v9.4s, v13.4s 347 add a9, a9, a13 348 add v10.4s, v10.4s, v14.4s 349 add a10, a10, a14 350 add v11.4s, v11.4s, v15.4s 351 add a11, a11, a15 352 353 eor v16.16b, v4.16b, v8.16b 354 eor a4, a4, a8 355 eor v17.16b, v5.16b, v9.16b 356 eor a5, a5, a9 357 eor v18.16b, v6.16b, v10.16b 358 eor a6, a6, a10 359 eor v19.16b, v7.16b, v11.16b 360 eor a7, a7, a11 361 362 shl v4.4s, v16.4s, #7 363 shl v5.4s, v17.4s, #7 364 shl v6.4s, v18.4s, #7 365 shl v7.4s, v19.4s, #7 366 367 sri v4.4s, v16.4s, #25 368 ror a4, a4, #25 369 sri v5.4s, v17.4s, #25 370 ror a5, a5, #25 371 sri v6.4s, v18.4s, #25 372 ror a6, a6, #25 373 sri v7.4s, v19.4s, #25 374 ror a7, a7, #25 375 376 // x0 += x5, x15 = rotl32(x15 ^ x0, 16) 377 // x1 += x6, x12 = rotl32(x12 ^ x1, 16) 378 // x2 += x7, x13 = rotl32(x13 ^ x2, 16) 379 // x3 += x4, x14 = rotl32(x14 ^ x3, 16) 380 add v0.4s, v0.4s, v5.4s 381 add a0, a0, a5 382 add v1.4s, v1.4s, v6.4s 383 add a1, a1, a6 384 add v2.4s, v2.4s, v7.4s 385 add a2, a2, a7 386 add v3.4s, v3.4s, v4.4s 387 add a3, a3, a4 388 389 eor v15.16b, v15.16b, v0.16b 390 eor a15, a15, a0 391 eor v12.16b, v12.16b, v1.16b 392 eor a12, a12, a1 393 eor v13.16b, v13.16b, v2.16b 394 eor a13, a13, a2 395 eor v14.16b, v14.16b, v3.16b 396 eor a14, a14, a3 397 398 rev32 v15.8h, v15.8h 399 ror a15, a15, #16 400 rev32 v12.8h, v12.8h 401 ror a12, a12, #16 402 rev32 v13.8h, v13.8h 403 ror a13, a13, #16 404 rev32 v14.8h, v14.8h 405 ror a14, a14, #16 406 407 // x10 += x15, x5 = rotl32(x5 ^ x10, 12) 408 // x11 += x12, x6 = rotl32(x6 ^ x11, 12) 409 // x8 += x13, x7 = rotl32(x7 ^ x8, 12) 410 // x9 += x14, x4 = rotl32(x4 ^ x9, 12) 411 add v10.4s, v10.4s, v15.4s 412 add a10, a10, a15 413 add v11.4s, v11.4s, v12.4s 414 add a11, a11, a12 415 add v8.4s, v8.4s, v13.4s 416 add a8, a8, a13 417 add v9.4s, v9.4s, v14.4s 418 add a9, a9, a14 419 420 eor v16.16b, v5.16b, v10.16b 421 eor a5, a5, a10 422 eor v17.16b, v6.16b, v11.16b 423 eor a6, a6, a11 424 eor v18.16b, v7.16b, v8.16b 425 eor a7, a7, a8 426 eor v19.16b, v4.16b, v9.16b 427 eor a4, a4, a9 428 429 shl v5.4s, v16.4s, #12 430 shl v6.4s, v17.4s, #12 431 shl v7.4s, v18.4s, #12 432 shl v4.4s, v19.4s, #12 433 434 sri v5.4s, v16.4s, #20 435 ror a5, a5, #20 436 sri v6.4s, v17.4s, #20 437 ror a6, a6, #20 438 sri v7.4s, v18.4s, #20 439 ror a7, a7, #20 440 sri v4.4s, v19.4s, #20 441 ror a4, a4, #20 442 443 // x0 += x5, x15 = rotl32(x15 ^ x0, 8) 444 // x1 += x6, x12 = rotl32(x12 ^ x1, 8) 445 // x2 += x7, x13 = rotl32(x13 ^ x2, 8) 446 // x3 += x4, x14 = rotl32(x14 ^ x3, 8) 447 add v0.4s, v0.4s, v5.4s 448 add a0, a0, a5 449 add v1.4s, v1.4s, v6.4s 450 add a1, a1, a6 451 add v2.4s, v2.4s, v7.4s 452 add a2, a2, a7 453 add v3.4s, v3.4s, v4.4s 454 add a3, a3, a4 455 456 eor v15.16b, v15.16b, v0.16b 457 eor a15, a15, a0 458 eor v12.16b, v12.16b, v1.16b 459 eor a12, a12, a1 460 eor v13.16b, v13.16b, v2.16b 461 eor a13, a13, a2 462 eor v14.16b, v14.16b, v3.16b 463 eor a14, a14, a3 464 465 tbl v15.16b, {v15.16b}, v31.16b 466 ror a15, a15, #24 467 tbl v12.16b, {v12.16b}, v31.16b 468 ror a12, a12, #24 469 tbl v13.16b, {v13.16b}, v31.16b 470 ror a13, a13, #24 471 tbl v14.16b, {v14.16b}, v31.16b 472 ror a14, a14, #24 473 474 // x10 += x15, x5 = rotl32(x5 ^ x10, 7) 475 // x11 += x12, x6 = rotl32(x6 ^ x11, 7) 476 // x8 += x13, x7 = rotl32(x7 ^ x8, 7) 477 // x9 += x14, x4 = rotl32(x4 ^ x9, 7) 478 add v10.4s, v10.4s, v15.4s 479 add a10, a10, a15 480 add v11.4s, v11.4s, v12.4s 481 add a11, a11, a12 482 add v8.4s, v8.4s, v13.4s 483 add a8, a8, a13 484 add v9.4s, v9.4s, v14.4s 485 add a9, a9, a14 486 487 eor v16.16b, v5.16b, v10.16b 488 eor a5, a5, a10 489 eor v17.16b, v6.16b, v11.16b 490 eor a6, a6, a11 491 eor v18.16b, v7.16b, v8.16b 492 eor a7, a7, a8 493 eor v19.16b, v4.16b, v9.16b 494 eor a4, a4, a9 495 496 shl v5.4s, v16.4s, #7 497 shl v6.4s, v17.4s, #7 498 shl v7.4s, v18.4s, #7 499 shl v4.4s, v19.4s, #7 500 501 sri v5.4s, v16.4s, #25 502 ror a5, a5, #25 503 sri v6.4s, v17.4s, #25 504 ror a6, a6, #25 505 sri v7.4s, v18.4s, #25 506 ror a7, a7, #25 507 sri v4.4s, v19.4s, #25 508 ror a4, a4, #25 509 510 subs w3, w3, #2 511 b.ne .Ldoubleround4 512 513 ld4r {v16.4s-v19.4s}, [x0], #16 514 ld4r {v20.4s-v23.4s}, [x0], #16 515 516 // x12 += counter values 0-3 517 add v12.4s, v12.4s, v30.4s 518 519 // x0[0-3] += s0[0] 520 // x1[0-3] += s0[1] 521 // x2[0-3] += s0[2] 522 // x3[0-3] += s0[3] 523 add v0.4s, v0.4s, v16.4s 524 mov w6, v16.s[0] 525 mov w7, v17.s[0] 526 add v1.4s, v1.4s, v17.4s 527 mov w8, v18.s[0] 528 mov w9, v19.s[0] 529 add v2.4s, v2.4s, v18.4s 530 add a0, a0, w6 531 add a1, a1, w7 532 add v3.4s, v3.4s, v19.4s 533 add a2, a2, w8 534 add a3, a3, w9 535 CPU_BE( rev a0, a0 ) 536 CPU_BE( rev a1, a1 ) 537 CPU_BE( rev a2, a2 ) 538 CPU_BE( rev a3, a3 ) 539 540 ld4r {v24.4s-v27.4s}, [x0], #16 541 ld4r {v28.4s-v31.4s}, [x0] 542 543 // x4[0-3] += s1[0] 544 // x5[0-3] += s1[1] 545 // x6[0-3] += s1[2] 546 // x7[0-3] += s1[3] 547 add v4.4s, v4.4s, v20.4s 548 mov w6, v20.s[0] 549 mov w7, v21.s[0] 550 add v5.4s, v5.4s, v21.4s 551 mov w8, v22.s[0] 552 mov w9, v23.s[0] 553 add v6.4s, v6.4s, v22.4s 554 add a4, a4, w6 555 add a5, a5, w7 556 add v7.4s, v7.4s, v23.4s 557 add a6, a6, w8 558 add a7, a7, w9 559 CPU_BE( rev a4, a4 ) 560 CPU_BE( rev a5, a5 ) 561 CPU_BE( rev a6, a6 ) 562 CPU_BE( rev a7, a7 ) 563 564 // x8[0-3] += s2[0] 565 // x9[0-3] += s2[1] 566 // x10[0-3] += s2[2] 567 // x11[0-3] += s2[3] 568 add v8.4s, v8.4s, v24.4s 569 mov w6, v24.s[0] 570 mov w7, v25.s[0] 571 add v9.4s, v9.4s, v25.4s 572 mov w8, v26.s[0] 573 mov w9, v27.s[0] 574 add v10.4s, v10.4s, v26.4s 575 add a8, a8, w6 576 add a9, a9, w7 577 add v11.4s, v11.4s, v27.4s 578 add a10, a10, w8 579 add a11, a11, w9 580 CPU_BE( rev a8, a8 ) 581 CPU_BE( rev a9, a9 ) 582 CPU_BE( rev a10, a10 ) 583 CPU_BE( rev a11, a11 ) 584 585 // x12[0-3] += s3[0] 586 // x13[0-3] += s3[1] 587 // x14[0-3] += s3[2] 588 // x15[0-3] += s3[3] 589 add v12.4s, v12.4s, v28.4s 590 mov w6, v28.s[0] 591 mov w7, v29.s[0] 592 add v13.4s, v13.4s, v29.4s 593 mov w8, v30.s[0] 594 mov w9, v31.s[0] 595 add v14.4s, v14.4s, v30.4s 596 add a12, a12, w6 597 add a13, a13, w7 598 add v15.4s, v15.4s, v31.4s 599 add a14, a14, w8 600 add a15, a15, w9 601 CPU_BE( rev a12, a12 ) 602 CPU_BE( rev a13, a13 ) 603 CPU_BE( rev a14, a14 ) 604 CPU_BE( rev a15, a15 ) 605 606 // interleave 32-bit words in state n, n+1 607 ldp w6, w7, [x2], #64 608 zip1 v16.4s, v0.4s, v1.4s 609 ldp w8, w9, [x2, #-56] 610 eor a0, a0, w6 611 zip2 v17.4s, v0.4s, v1.4s 612 eor a1, a1, w7 613 zip1 v18.4s, v2.4s, v3.4s 614 eor a2, a2, w8 615 zip2 v19.4s, v2.4s, v3.4s 616 eor a3, a3, w9 617 ldp w6, w7, [x2, #-48] 618 zip1 v20.4s, v4.4s, v5.4s 619 ldp w8, w9, [x2, #-40] 620 eor a4, a4, w6 621 zip2 v21.4s, v4.4s, v5.4s 622 eor a5, a5, w7 623 zip1 v22.4s, v6.4s, v7.4s 624 eor a6, a6, w8 625 zip2 v23.4s, v6.4s, v7.4s 626 eor a7, a7, w9 627 ldp w6, w7, [x2, #-32] 628 zip1 v24.4s, v8.4s, v9.4s 629 ldp w8, w9, [x2, #-24] 630 eor a8, a8, w6 631 zip2 v25.4s, v8.4s, v9.4s 632 eor a9, a9, w7 633 zip1 v26.4s, v10.4s, v11.4s 634 eor a10, a10, w8 635 zip2 v27.4s, v10.4s, v11.4s 636 eor a11, a11, w9 637 ldp w6, w7, [x2, #-16] 638 zip1 v28.4s, v12.4s, v13.4s 639 ldp w8, w9, [x2, #-8] 640 eor a12, a12, w6 641 zip2 v29.4s, v12.4s, v13.4s 642 eor a13, a13, w7 643 zip1 v30.4s, v14.4s, v15.4s 644 eor a14, a14, w8 645 zip2 v31.4s, v14.4s, v15.4s 646 eor a15, a15, w9 647 648 mov x3, #64 649 subs x5, x4, #128 650 add x6, x5, x2 651 csel x3, x3, xzr, ge 652 csel x2, x2, x6, ge 653 654 // interleave 64-bit words in state n, n+2 655 zip1 v0.2d, v16.2d, v18.2d 656 zip2 v4.2d, v16.2d, v18.2d 657 stp a0, a1, [x1], #64 658 zip1 v8.2d, v17.2d, v19.2d 659 zip2 v12.2d, v17.2d, v19.2d 660 stp a2, a3, [x1, #-56] 661 ld1 {v16.16b-v19.16b}, [x2], x3 662 663 subs x6, x4, #192 664 ccmp x3, xzr, #4, lt 665 add x7, x6, x2 666 csel x3, x3, xzr, eq 667 csel x2, x2, x7, eq 668 669 zip1 v1.2d, v20.2d, v22.2d 670 zip2 v5.2d, v20.2d, v22.2d 671 stp a4, a5, [x1, #-48] 672 zip1 v9.2d, v21.2d, v23.2d 673 zip2 v13.2d, v21.2d, v23.2d 674 stp a6, a7, [x1, #-40] 675 ld1 {v20.16b-v23.16b}, [x2], x3 676 677 subs x7, x4, #256 678 ccmp x3, xzr, #4, lt 679 add x8, x7, x2 680 csel x3, x3, xzr, eq 681 csel x2, x2, x8, eq 682 683 zip1 v2.2d, v24.2d, v26.2d 684 zip2 v6.2d, v24.2d, v26.2d 685 stp a8, a9, [x1, #-32] 686 zip1 v10.2d, v25.2d, v27.2d 687 zip2 v14.2d, v25.2d, v27.2d 688 stp a10, a11, [x1, #-24] 689 ld1 {v24.16b-v27.16b}, [x2], x3 690 691 subs x8, x4, #320 692 ccmp x3, xzr, #4, lt 693 add x9, x8, x2 694 csel x2, x2, x9, eq 695 696 zip1 v3.2d, v28.2d, v30.2d 697 zip2 v7.2d, v28.2d, v30.2d 698 stp a12, a13, [x1, #-16] 699 zip1 v11.2d, v29.2d, v31.2d 700 zip2 v15.2d, v29.2d, v31.2d 701 stp a14, a15, [x1, #-8] 702 ld1 {v28.16b-v31.16b}, [x2] 703 704 // xor with corresponding input, write to output 705 tbnz x5, #63, 0f 706 eor v16.16b, v16.16b, v0.16b 707 eor v17.16b, v17.16b, v1.16b 708 eor v18.16b, v18.16b, v2.16b 709 eor v19.16b, v19.16b, v3.16b 710 st1 {v16.16b-v19.16b}, [x1], #64 711 cbz x5, .Lout 712 713 tbnz x6, #63, 1f 714 eor v20.16b, v20.16b, v4.16b 715 eor v21.16b, v21.16b, v5.16b 716 eor v22.16b, v22.16b, v6.16b 717 eor v23.16b, v23.16b, v7.16b 718 st1 {v20.16b-v23.16b}, [x1], #64 719 cbz x6, .Lout 720 721 tbnz x7, #63, 2f 722 eor v24.16b, v24.16b, v8.16b 723 eor v25.16b, v25.16b, v9.16b 724 eor v26.16b, v26.16b, v10.16b 725 eor v27.16b, v27.16b, v11.16b 726 st1 {v24.16b-v27.16b}, [x1], #64 727 cbz x7, .Lout 728 729 tbnz x8, #63, 3f 730 eor v28.16b, v28.16b, v12.16b 731 eor v29.16b, v29.16b, v13.16b 732 eor v30.16b, v30.16b, v14.16b 733 eor v31.16b, v31.16b, v15.16b 734 st1 {v28.16b-v31.16b}, [x1] 735 736 .Lout: frame_pop 737 ret 738 739 // fewer than 128 bytes of in/output 740 0: ld1 {v8.16b}, [x10] 741 ld1 {v9.16b}, [x11] 742 movi v10.16b, #16 743 sub x2, x1, #64 744 add x1, x1, x5 745 ld1 {v16.16b-v19.16b}, [x2] 746 tbl v4.16b, {v0.16b-v3.16b}, v8.16b 747 tbx v20.16b, {v16.16b-v19.16b}, v9.16b 748 add v8.16b, v8.16b, v10.16b 749 add v9.16b, v9.16b, v10.16b 750 tbl v5.16b, {v0.16b-v3.16b}, v8.16b 751 tbx v21.16b, {v16.16b-v19.16b}, v9.16b 752 add v8.16b, v8.16b, v10.16b 753 add v9.16b, v9.16b, v10.16b 754 tbl v6.16b, {v0.16b-v3.16b}, v8.16b 755 tbx v22.16b, {v16.16b-v19.16b}, v9.16b 756 add v8.16b, v8.16b, v10.16b 757 add v9.16b, v9.16b, v10.16b 758 tbl v7.16b, {v0.16b-v3.16b}, v8.16b 759 tbx v23.16b, {v16.16b-v19.16b}, v9.16b 760 761 eor v20.16b, v20.16b, v4.16b 762 eor v21.16b, v21.16b, v5.16b 763 eor v22.16b, v22.16b, v6.16b 764 eor v23.16b, v23.16b, v7.16b 765 st1 {v20.16b-v23.16b}, [x1] 766 b .Lout 767 768 // fewer than 192 bytes of in/output 769 1: ld1 {v8.16b}, [x10] 770 ld1 {v9.16b}, [x11] 771 movi v10.16b, #16 772 add x1, x1, x6 773 tbl v0.16b, {v4.16b-v7.16b}, v8.16b 774 tbx v20.16b, {v16.16b-v19.16b}, v9.16b 775 add v8.16b, v8.16b, v10.16b 776 add v9.16b, v9.16b, v10.16b 777 tbl v1.16b, {v4.16b-v7.16b}, v8.16b 778 tbx v21.16b, {v16.16b-v19.16b}, v9.16b 779 add v8.16b, v8.16b, v10.16b 780 add v9.16b, v9.16b, v10.16b 781 tbl v2.16b, {v4.16b-v7.16b}, v8.16b 782 tbx v22.16b, {v16.16b-v19.16b}, v9.16b 783 add v8.16b, v8.16b, v10.16b 784 add v9.16b, v9.16b, v10.16b 785 tbl v3.16b, {v4.16b-v7.16b}, v8.16b 786 tbx v23.16b, {v16.16b-v19.16b}, v9.16b 787 788 eor v20.16b, v20.16b, v0.16b 789 eor v21.16b, v21.16b, v1.16b 790 eor v22.16b, v22.16b, v2.16b 791 eor v23.16b, v23.16b, v3.16b 792 st1 {v20.16b-v23.16b}, [x1] 793 b .Lout 794 795 // fewer than 256 bytes of in/output 796 2: ld1 {v4.16b}, [x10] 797 ld1 {v5.16b}, [x11] 798 movi v6.16b, #16 799 add x1, x1, x7 800 tbl v0.16b, {v8.16b-v11.16b}, v4.16b 801 tbx v24.16b, {v20.16b-v23.16b}, v5.16b 802 add v4.16b, v4.16b, v6.16b 803 add v5.16b, v5.16b, v6.16b 804 tbl v1.16b, {v8.16b-v11.16b}, v4.16b 805 tbx v25.16b, {v20.16b-v23.16b}, v5.16b 806 add v4.16b, v4.16b, v6.16b 807 add v5.16b, v5.16b, v6.16b 808 tbl v2.16b, {v8.16b-v11.16b}, v4.16b 809 tbx v26.16b, {v20.16b-v23.16b}, v5.16b 810 add v4.16b, v4.16b, v6.16b 811 add v5.16b, v5.16b, v6.16b 812 tbl v3.16b, {v8.16b-v11.16b}, v4.16b 813 tbx v27.16b, {v20.16b-v23.16b}, v5.16b 814 815 eor v24.16b, v24.16b, v0.16b 816 eor v25.16b, v25.16b, v1.16b 817 eor v26.16b, v26.16b, v2.16b 818 eor v27.16b, v27.16b, v3.16b 819 st1 {v24.16b-v27.16b}, [x1] 820 b .Lout 821 822 // fewer than 320 bytes of in/output 823 3: ld1 {v4.16b}, [x10] 824 ld1 {v5.16b}, [x11] 825 movi v6.16b, #16 826 add x1, x1, x8 827 tbl v0.16b, {v12.16b-v15.16b}, v4.16b 828 tbx v28.16b, {v24.16b-v27.16b}, v5.16b 829 add v4.16b, v4.16b, v6.16b 830 add v5.16b, v5.16b, v6.16b 831 tbl v1.16b, {v12.16b-v15.16b}, v4.16b 832 tbx v29.16b, {v24.16b-v27.16b}, v5.16b 833 add v4.16b, v4.16b, v6.16b 834 add v5.16b, v5.16b, v6.16b 835 tbl v2.16b, {v12.16b-v15.16b}, v4.16b 836 tbx v30.16b, {v24.16b-v27.16b}, v5.16b 837 add v4.16b, v4.16b, v6.16b 838 add v5.16b, v5.16b, v6.16b 839 tbl v3.16b, {v12.16b-v15.16b}, v4.16b 840 tbx v31.16b, {v24.16b-v27.16b}, v5.16b 841 842 eor v28.16b, v28.16b, v0.16b 843 eor v29.16b, v29.16b, v1.16b 844 eor v30.16b, v30.16b, v2.16b 845 eor v31.16b, v31.16b, v3.16b 846 st1 {v28.16b-v31.16b}, [x1] 847 b .Lout 848 SYM_FUNC_END(chacha_4block_xor_neon) 849 850 .section ".rodata", "a", %progbits 851 .align L1_CACHE_SHIFT 852 .Lpermute: 853 .set .Li, 0 854 .rept 192 855 .byte (.Li - 64) 856 .set .Li, .Li + 1 857 .endr 858 859 CTRINC: .word 1, 2, 3, 4 860 ROT8: .word 0x02010003, 0x06050407, 0x0a09080b, 0x0e0d0c0f 861