1 /* SPDX-License-Identifier: GPL-2.0+ */ 2 /* 3 * ChaCha 256-bit cipher algorithm, x64 AVX-512VL functions 4 * 5 * Copyright (C) 2018 Martin Willi 6 */ 7 8 #include <linux/linkage.h> 9 10 .section .rodata.cst32.CTR2BL, "aM", @progbits, 32 11 .align 32 12 CTR2BL: .octa 0x00000000000000000000000000000000 13 .octa 0x00000000000000000000000000000001 14 15 .section .rodata.cst32.CTR4BL, "aM", @progbits, 32 16 .align 32 17 CTR4BL: .octa 0x00000000000000000000000000000002 18 .octa 0x00000000000000000000000000000003 19 20 .section .rodata.cst32.CTR8BL, "aM", @progbits, 32 21 .align 32 22 CTR8BL: .octa 0x00000003000000020000000100000000 23 .octa 0x00000007000000060000000500000004 24 25 .text 26 27 SYM_FUNC_START(chacha_2block_xor_avx512vl) 28 # %rdi: Input state matrix, s 29 # %rsi: up to 2 data blocks output, o 30 # %rdx: up to 2 data blocks input, i 31 # %rcx: input/output length in bytes 32 # %r8d: nrounds 33 34 # This function encrypts two ChaCha blocks by loading the state 35 # matrix twice across four AVX registers. It performs matrix operations 36 # on four words in each matrix in parallel, but requires shuffling to 37 # rearrange the words after each round. 38 39 vzeroupper 40 41 # x0..3[0-2] = s0..3 42 vbroadcasti128 0x00(%rdi),%ymm0 43 vbroadcasti128 0x10(%rdi),%ymm1 44 vbroadcasti128 0x20(%rdi),%ymm2 45 vbroadcasti128 0x30(%rdi),%ymm3 46 47 vpaddd CTR2BL(%rip),%ymm3,%ymm3 48 49 vmovdqa %ymm0,%ymm8 50 vmovdqa %ymm1,%ymm9 51 vmovdqa %ymm2,%ymm10 52 vmovdqa %ymm3,%ymm11 53 54 .Ldoubleround: 55 56 # x0 += x1, x3 = rotl32(x3 ^ x0, 16) 57 vpaddd %ymm1,%ymm0,%ymm0 58 vpxord %ymm0,%ymm3,%ymm3 59 vprold $16,%ymm3,%ymm3 60 61 # x2 += x3, x1 = rotl32(x1 ^ x2, 12) 62 vpaddd %ymm3,%ymm2,%ymm2 63 vpxord %ymm2,%ymm1,%ymm1 64 vprold $12,%ymm1,%ymm1 65 66 # x0 += x1, x3 = rotl32(x3 ^ x0, 8) 67 vpaddd %ymm1,%ymm0,%ymm0 68 vpxord %ymm0,%ymm3,%ymm3 69 vprold $8,%ymm3,%ymm3 70 71 # x2 += x3, x1 = rotl32(x1 ^ x2, 7) 72 vpaddd %ymm3,%ymm2,%ymm2 73 vpxord %ymm2,%ymm1,%ymm1 74 vprold $7,%ymm1,%ymm1 75 76 # x1 = shuffle32(x1, MASK(0, 3, 2, 1)) 77 vpshufd $0x39,%ymm1,%ymm1 78 # x2 = shuffle32(x2, MASK(1, 0, 3, 2)) 79 vpshufd $0x4e,%ymm2,%ymm2 80 # x3 = shuffle32(x3, MASK(2, 1, 0, 3)) 81 vpshufd $0x93,%ymm3,%ymm3 82 83 # x0 += x1, x3 = rotl32(x3 ^ x0, 16) 84 vpaddd %ymm1,%ymm0,%ymm0 85 vpxord %ymm0,%ymm3,%ymm3 86 vprold $16,%ymm3,%ymm3 87 88 # x2 += x3, x1 = rotl32(x1 ^ x2, 12) 89 vpaddd %ymm3,%ymm2,%ymm2 90 vpxord %ymm2,%ymm1,%ymm1 91 vprold $12,%ymm1,%ymm1 92 93 # x0 += x1, x3 = rotl32(x3 ^ x0, 8) 94 vpaddd %ymm1,%ymm0,%ymm0 95 vpxord %ymm0,%ymm3,%ymm3 96 vprold $8,%ymm3,%ymm3 97 98 # x2 += x3, x1 = rotl32(x1 ^ x2, 7) 99 vpaddd %ymm3,%ymm2,%ymm2 100 vpxord %ymm2,%ymm1,%ymm1 101 vprold $7,%ymm1,%ymm1 102 103 # x1 = shuffle32(x1, MASK(2, 1, 0, 3)) 104 vpshufd $0x93,%ymm1,%ymm1 105 # x2 = shuffle32(x2, MASK(1, 0, 3, 2)) 106 vpshufd $0x4e,%ymm2,%ymm2 107 # x3 = shuffle32(x3, MASK(0, 3, 2, 1)) 108 vpshufd $0x39,%ymm3,%ymm3 109 110 sub $2,%r8d 111 jnz .Ldoubleround 112 113 # o0 = i0 ^ (x0 + s0) 114 vpaddd %ymm8,%ymm0,%ymm7 115 cmp $0x10,%rcx 116 jl .Lxorpart2 117 vpxord 0x00(%rdx),%xmm7,%xmm6 118 vmovdqu %xmm6,0x00(%rsi) 119 vextracti128 $1,%ymm7,%xmm0 120 # o1 = i1 ^ (x1 + s1) 121 vpaddd %ymm9,%ymm1,%ymm7 122 cmp $0x20,%rcx 123 jl .Lxorpart2 124 vpxord 0x10(%rdx),%xmm7,%xmm6 125 vmovdqu %xmm6,0x10(%rsi) 126 vextracti128 $1,%ymm7,%xmm1 127 # o2 = i2 ^ (x2 + s2) 128 vpaddd %ymm10,%ymm2,%ymm7 129 cmp $0x30,%rcx 130 jl .Lxorpart2 131 vpxord 0x20(%rdx),%xmm7,%xmm6 132 vmovdqu %xmm6,0x20(%rsi) 133 vextracti128 $1,%ymm7,%xmm2 134 # o3 = i3 ^ (x3 + s3) 135 vpaddd %ymm11,%ymm3,%ymm7 136 cmp $0x40,%rcx 137 jl .Lxorpart2 138 vpxord 0x30(%rdx),%xmm7,%xmm6 139 vmovdqu %xmm6,0x30(%rsi) 140 vextracti128 $1,%ymm7,%xmm3 141 142 # xor and write second block 143 vmovdqa %xmm0,%xmm7 144 cmp $0x50,%rcx 145 jl .Lxorpart2 146 vpxord 0x40(%rdx),%xmm7,%xmm6 147 vmovdqu %xmm6,0x40(%rsi) 148 149 vmovdqa %xmm1,%xmm7 150 cmp $0x60,%rcx 151 jl .Lxorpart2 152 vpxord 0x50(%rdx),%xmm7,%xmm6 153 vmovdqu %xmm6,0x50(%rsi) 154 155 vmovdqa %xmm2,%xmm7 156 cmp $0x70,%rcx 157 jl .Lxorpart2 158 vpxord 0x60(%rdx),%xmm7,%xmm6 159 vmovdqu %xmm6,0x60(%rsi) 160 161 vmovdqa %xmm3,%xmm7 162 cmp $0x80,%rcx 163 jl .Lxorpart2 164 vpxord 0x70(%rdx),%xmm7,%xmm6 165 vmovdqu %xmm6,0x70(%rsi) 166 167 .Ldone2: 168 vzeroupper 169 RET 170 171 .Lxorpart2: 172 # xor remaining bytes from partial register into output 173 mov %rcx,%rax 174 and $0xf,%rcx 175 jz .Ldone2 176 mov %rax,%r9 177 and $~0xf,%r9 178 179 mov $1,%rax 180 shld %cl,%rax,%rax 181 sub $1,%rax 182 kmovq %rax,%k1 183 184 vmovdqu8 (%rdx,%r9),%xmm1{%k1}{z} 185 vpxord %xmm7,%xmm1,%xmm1 186 vmovdqu8 %xmm1,(%rsi,%r9){%k1} 187 188 jmp .Ldone2 189 190 SYM_FUNC_END(chacha_2block_xor_avx512vl) 191 192 SYM_FUNC_START(chacha_4block_xor_avx512vl) 193 # %rdi: Input state matrix, s 194 # %rsi: up to 4 data blocks output, o 195 # %rdx: up to 4 data blocks input, i 196 # %rcx: input/output length in bytes 197 # %r8d: nrounds 198 199 # This function encrypts four ChaCha blocks by loading the state 200 # matrix four times across eight AVX registers. It performs matrix 201 # operations on four words in two matrices in parallel, sequentially 202 # to the operations on the four words of the other two matrices. The 203 # required word shuffling has a rather high latency, we can do the 204 # arithmetic on two matrix-pairs without much slowdown. 205 206 vzeroupper 207 208 # x0..3[0-4] = s0..3 209 vbroadcasti128 0x00(%rdi),%ymm0 210 vbroadcasti128 0x10(%rdi),%ymm1 211 vbroadcasti128 0x20(%rdi),%ymm2 212 vbroadcasti128 0x30(%rdi),%ymm3 213 214 vmovdqa %ymm0,%ymm4 215 vmovdqa %ymm1,%ymm5 216 vmovdqa %ymm2,%ymm6 217 vmovdqa %ymm3,%ymm7 218 219 vpaddd CTR2BL(%rip),%ymm3,%ymm3 220 vpaddd CTR4BL(%rip),%ymm7,%ymm7 221 222 vmovdqa %ymm0,%ymm11 223 vmovdqa %ymm1,%ymm12 224 vmovdqa %ymm2,%ymm13 225 vmovdqa %ymm3,%ymm14 226 vmovdqa %ymm7,%ymm15 227 228 .Ldoubleround4: 229 230 # x0 += x1, x3 = rotl32(x3 ^ x0, 16) 231 vpaddd %ymm1,%ymm0,%ymm0 232 vpxord %ymm0,%ymm3,%ymm3 233 vprold $16,%ymm3,%ymm3 234 235 vpaddd %ymm5,%ymm4,%ymm4 236 vpxord %ymm4,%ymm7,%ymm7 237 vprold $16,%ymm7,%ymm7 238 239 # x2 += x3, x1 = rotl32(x1 ^ x2, 12) 240 vpaddd %ymm3,%ymm2,%ymm2 241 vpxord %ymm2,%ymm1,%ymm1 242 vprold $12,%ymm1,%ymm1 243 244 vpaddd %ymm7,%ymm6,%ymm6 245 vpxord %ymm6,%ymm5,%ymm5 246 vprold $12,%ymm5,%ymm5 247 248 # x0 += x1, x3 = rotl32(x3 ^ x0, 8) 249 vpaddd %ymm1,%ymm0,%ymm0 250 vpxord %ymm0,%ymm3,%ymm3 251 vprold $8,%ymm3,%ymm3 252 253 vpaddd %ymm5,%ymm4,%ymm4 254 vpxord %ymm4,%ymm7,%ymm7 255 vprold $8,%ymm7,%ymm7 256 257 # x2 += x3, x1 = rotl32(x1 ^ x2, 7) 258 vpaddd %ymm3,%ymm2,%ymm2 259 vpxord %ymm2,%ymm1,%ymm1 260 vprold $7,%ymm1,%ymm1 261 262 vpaddd %ymm7,%ymm6,%ymm6 263 vpxord %ymm6,%ymm5,%ymm5 264 vprold $7,%ymm5,%ymm5 265 266 # x1 = shuffle32(x1, MASK(0, 3, 2, 1)) 267 vpshufd $0x39,%ymm1,%ymm1 268 vpshufd $0x39,%ymm5,%ymm5 269 # x2 = shuffle32(x2, MASK(1, 0, 3, 2)) 270 vpshufd $0x4e,%ymm2,%ymm2 271 vpshufd $0x4e,%ymm6,%ymm6 272 # x3 = shuffle32(x3, MASK(2, 1, 0, 3)) 273 vpshufd $0x93,%ymm3,%ymm3 274 vpshufd $0x93,%ymm7,%ymm7 275 276 # x0 += x1, x3 = rotl32(x3 ^ x0, 16) 277 vpaddd %ymm1,%ymm0,%ymm0 278 vpxord %ymm0,%ymm3,%ymm3 279 vprold $16,%ymm3,%ymm3 280 281 vpaddd %ymm5,%ymm4,%ymm4 282 vpxord %ymm4,%ymm7,%ymm7 283 vprold $16,%ymm7,%ymm7 284 285 # x2 += x3, x1 = rotl32(x1 ^ x2, 12) 286 vpaddd %ymm3,%ymm2,%ymm2 287 vpxord %ymm2,%ymm1,%ymm1 288 vprold $12,%ymm1,%ymm1 289 290 vpaddd %ymm7,%ymm6,%ymm6 291 vpxord %ymm6,%ymm5,%ymm5 292 vprold $12,%ymm5,%ymm5 293 294 # x0 += x1, x3 = rotl32(x3 ^ x0, 8) 295 vpaddd %ymm1,%ymm0,%ymm0 296 vpxord %ymm0,%ymm3,%ymm3 297 vprold $8,%ymm3,%ymm3 298 299 vpaddd %ymm5,%ymm4,%ymm4 300 vpxord %ymm4,%ymm7,%ymm7 301 vprold $8,%ymm7,%ymm7 302 303 # x2 += x3, x1 = rotl32(x1 ^ x2, 7) 304 vpaddd %ymm3,%ymm2,%ymm2 305 vpxord %ymm2,%ymm1,%ymm1 306 vprold $7,%ymm1,%ymm1 307 308 vpaddd %ymm7,%ymm6,%ymm6 309 vpxord %ymm6,%ymm5,%ymm5 310 vprold $7,%ymm5,%ymm5 311 312 # x1 = shuffle32(x1, MASK(2, 1, 0, 3)) 313 vpshufd $0x93,%ymm1,%ymm1 314 vpshufd $0x93,%ymm5,%ymm5 315 # x2 = shuffle32(x2, MASK(1, 0, 3, 2)) 316 vpshufd $0x4e,%ymm2,%ymm2 317 vpshufd $0x4e,%ymm6,%ymm6 318 # x3 = shuffle32(x3, MASK(0, 3, 2, 1)) 319 vpshufd $0x39,%ymm3,%ymm3 320 vpshufd $0x39,%ymm7,%ymm7 321 322 sub $2,%r8d 323 jnz .Ldoubleround4 324 325 # o0 = i0 ^ (x0 + s0), first block 326 vpaddd %ymm11,%ymm0,%ymm10 327 cmp $0x10,%rcx 328 jl .Lxorpart4 329 vpxord 0x00(%rdx),%xmm10,%xmm9 330 vmovdqu %xmm9,0x00(%rsi) 331 vextracti128 $1,%ymm10,%xmm0 332 # o1 = i1 ^ (x1 + s1), first block 333 vpaddd %ymm12,%ymm1,%ymm10 334 cmp $0x20,%rcx 335 jl .Lxorpart4 336 vpxord 0x10(%rdx),%xmm10,%xmm9 337 vmovdqu %xmm9,0x10(%rsi) 338 vextracti128 $1,%ymm10,%xmm1 339 # o2 = i2 ^ (x2 + s2), first block 340 vpaddd %ymm13,%ymm2,%ymm10 341 cmp $0x30,%rcx 342 jl .Lxorpart4 343 vpxord 0x20(%rdx),%xmm10,%xmm9 344 vmovdqu %xmm9,0x20(%rsi) 345 vextracti128 $1,%ymm10,%xmm2 346 # o3 = i3 ^ (x3 + s3), first block 347 vpaddd %ymm14,%ymm3,%ymm10 348 cmp $0x40,%rcx 349 jl .Lxorpart4 350 vpxord 0x30(%rdx),%xmm10,%xmm9 351 vmovdqu %xmm9,0x30(%rsi) 352 vextracti128 $1,%ymm10,%xmm3 353 354 # xor and write second block 355 vmovdqa %xmm0,%xmm10 356 cmp $0x50,%rcx 357 jl .Lxorpart4 358 vpxord 0x40(%rdx),%xmm10,%xmm9 359 vmovdqu %xmm9,0x40(%rsi) 360 361 vmovdqa %xmm1,%xmm10 362 cmp $0x60,%rcx 363 jl .Lxorpart4 364 vpxord 0x50(%rdx),%xmm10,%xmm9 365 vmovdqu %xmm9,0x50(%rsi) 366 367 vmovdqa %xmm2,%xmm10 368 cmp $0x70,%rcx 369 jl .Lxorpart4 370 vpxord 0x60(%rdx),%xmm10,%xmm9 371 vmovdqu %xmm9,0x60(%rsi) 372 373 vmovdqa %xmm3,%xmm10 374 cmp $0x80,%rcx 375 jl .Lxorpart4 376 vpxord 0x70(%rdx),%xmm10,%xmm9 377 vmovdqu %xmm9,0x70(%rsi) 378 379 # o0 = i0 ^ (x0 + s0), third block 380 vpaddd %ymm11,%ymm4,%ymm10 381 cmp $0x90,%rcx 382 jl .Lxorpart4 383 vpxord 0x80(%rdx),%xmm10,%xmm9 384 vmovdqu %xmm9,0x80(%rsi) 385 vextracti128 $1,%ymm10,%xmm4 386 # o1 = i1 ^ (x1 + s1), third block 387 vpaddd %ymm12,%ymm5,%ymm10 388 cmp $0xa0,%rcx 389 jl .Lxorpart4 390 vpxord 0x90(%rdx),%xmm10,%xmm9 391 vmovdqu %xmm9,0x90(%rsi) 392 vextracti128 $1,%ymm10,%xmm5 393 # o2 = i2 ^ (x2 + s2), third block 394 vpaddd %ymm13,%ymm6,%ymm10 395 cmp $0xb0,%rcx 396 jl .Lxorpart4 397 vpxord 0xa0(%rdx),%xmm10,%xmm9 398 vmovdqu %xmm9,0xa0(%rsi) 399 vextracti128 $1,%ymm10,%xmm6 400 # o3 = i3 ^ (x3 + s3), third block 401 vpaddd %ymm15,%ymm7,%ymm10 402 cmp $0xc0,%rcx 403 jl .Lxorpart4 404 vpxord 0xb0(%rdx),%xmm10,%xmm9 405 vmovdqu %xmm9,0xb0(%rsi) 406 vextracti128 $1,%ymm10,%xmm7 407 408 # xor and write fourth block 409 vmovdqa %xmm4,%xmm10 410 cmp $0xd0,%rcx 411 jl .Lxorpart4 412 vpxord 0xc0(%rdx),%xmm10,%xmm9 413 vmovdqu %xmm9,0xc0(%rsi) 414 415 vmovdqa %xmm5,%xmm10 416 cmp $0xe0,%rcx 417 jl .Lxorpart4 418 vpxord 0xd0(%rdx),%xmm10,%xmm9 419 vmovdqu %xmm9,0xd0(%rsi) 420 421 vmovdqa %xmm6,%xmm10 422 cmp $0xf0,%rcx 423 jl .Lxorpart4 424 vpxord 0xe0(%rdx),%xmm10,%xmm9 425 vmovdqu %xmm9,0xe0(%rsi) 426 427 vmovdqa %xmm7,%xmm10 428 cmp $0x100,%rcx 429 jl .Lxorpart4 430 vpxord 0xf0(%rdx),%xmm10,%xmm9 431 vmovdqu %xmm9,0xf0(%rsi) 432 433 .Ldone4: 434 vzeroupper 435 RET 436 437 .Lxorpart4: 438 # xor remaining bytes from partial register into output 439 mov %rcx,%rax 440 and $0xf,%rcx 441 jz .Ldone4 442 mov %rax,%r9 443 and $~0xf,%r9 444 445 mov $1,%rax 446 shld %cl,%rax,%rax 447 sub $1,%rax 448 kmovq %rax,%k1 449 450 vmovdqu8 (%rdx,%r9),%xmm1{%k1}{z} 451 vpxord %xmm10,%xmm1,%xmm1 452 vmovdqu8 %xmm1,(%rsi,%r9){%k1} 453 454 jmp .Ldone4 455 456 SYM_FUNC_END(chacha_4block_xor_avx512vl) 457 458 SYM_FUNC_START(chacha_8block_xor_avx512vl) 459 # %rdi: Input state matrix, s 460 # %rsi: up to 8 data blocks output, o 461 # %rdx: up to 8 data blocks input, i 462 # %rcx: input/output length in bytes 463 # %r8d: nrounds 464 465 # This function encrypts eight consecutive ChaCha blocks by loading 466 # the state matrix in AVX registers eight times. Compared to AVX2, this 467 # mostly benefits from the new rotate instructions in VL and the 468 # additional registers. 469 470 vzeroupper 471 472 # x0..15[0-7] = s[0..15] 473 vpbroadcastd 0x00(%rdi),%ymm0 474 vpbroadcastd 0x04(%rdi),%ymm1 475 vpbroadcastd 0x08(%rdi),%ymm2 476 vpbroadcastd 0x0c(%rdi),%ymm3 477 vpbroadcastd 0x10(%rdi),%ymm4 478 vpbroadcastd 0x14(%rdi),%ymm5 479 vpbroadcastd 0x18(%rdi),%ymm6 480 vpbroadcastd 0x1c(%rdi),%ymm7 481 vpbroadcastd 0x20(%rdi),%ymm8 482 vpbroadcastd 0x24(%rdi),%ymm9 483 vpbroadcastd 0x28(%rdi),%ymm10 484 vpbroadcastd 0x2c(%rdi),%ymm11 485 vpbroadcastd 0x30(%rdi),%ymm12 486 vpbroadcastd 0x34(%rdi),%ymm13 487 vpbroadcastd 0x38(%rdi),%ymm14 488 vpbroadcastd 0x3c(%rdi),%ymm15 489 490 # x12 += counter values 0-3 491 vpaddd CTR8BL(%rip),%ymm12,%ymm12 492 493 vmovdqa64 %ymm0,%ymm16 494 vmovdqa64 %ymm1,%ymm17 495 vmovdqa64 %ymm2,%ymm18 496 vmovdqa64 %ymm3,%ymm19 497 vmovdqa64 %ymm4,%ymm20 498 vmovdqa64 %ymm5,%ymm21 499 vmovdqa64 %ymm6,%ymm22 500 vmovdqa64 %ymm7,%ymm23 501 vmovdqa64 %ymm8,%ymm24 502 vmovdqa64 %ymm9,%ymm25 503 vmovdqa64 %ymm10,%ymm26 504 vmovdqa64 %ymm11,%ymm27 505 vmovdqa64 %ymm12,%ymm28 506 vmovdqa64 %ymm13,%ymm29 507 vmovdqa64 %ymm14,%ymm30 508 vmovdqa64 %ymm15,%ymm31 509 510 .Ldoubleround8: 511 # x0 += x4, x12 = rotl32(x12 ^ x0, 16) 512 vpaddd %ymm0,%ymm4,%ymm0 513 vpxord %ymm0,%ymm12,%ymm12 514 vprold $16,%ymm12,%ymm12 515 # x1 += x5, x13 = rotl32(x13 ^ x1, 16) 516 vpaddd %ymm1,%ymm5,%ymm1 517 vpxord %ymm1,%ymm13,%ymm13 518 vprold $16,%ymm13,%ymm13 519 # x2 += x6, x14 = rotl32(x14 ^ x2, 16) 520 vpaddd %ymm2,%ymm6,%ymm2 521 vpxord %ymm2,%ymm14,%ymm14 522 vprold $16,%ymm14,%ymm14 523 # x3 += x7, x15 = rotl32(x15 ^ x3, 16) 524 vpaddd %ymm3,%ymm7,%ymm3 525 vpxord %ymm3,%ymm15,%ymm15 526 vprold $16,%ymm15,%ymm15 527 528 # x8 += x12, x4 = rotl32(x4 ^ x8, 12) 529 vpaddd %ymm12,%ymm8,%ymm8 530 vpxord %ymm8,%ymm4,%ymm4 531 vprold $12,%ymm4,%ymm4 532 # x9 += x13, x5 = rotl32(x5 ^ x9, 12) 533 vpaddd %ymm13,%ymm9,%ymm9 534 vpxord %ymm9,%ymm5,%ymm5 535 vprold $12,%ymm5,%ymm5 536 # x10 += x14, x6 = rotl32(x6 ^ x10, 12) 537 vpaddd %ymm14,%ymm10,%ymm10 538 vpxord %ymm10,%ymm6,%ymm6 539 vprold $12,%ymm6,%ymm6 540 # x11 += x15, x7 = rotl32(x7 ^ x11, 12) 541 vpaddd %ymm15,%ymm11,%ymm11 542 vpxord %ymm11,%ymm7,%ymm7 543 vprold $12,%ymm7,%ymm7 544 545 # x0 += x4, x12 = rotl32(x12 ^ x0, 8) 546 vpaddd %ymm0,%ymm4,%ymm0 547 vpxord %ymm0,%ymm12,%ymm12 548 vprold $8,%ymm12,%ymm12 549 # x1 += x5, x13 = rotl32(x13 ^ x1, 8) 550 vpaddd %ymm1,%ymm5,%ymm1 551 vpxord %ymm1,%ymm13,%ymm13 552 vprold $8,%ymm13,%ymm13 553 # x2 += x6, x14 = rotl32(x14 ^ x2, 8) 554 vpaddd %ymm2,%ymm6,%ymm2 555 vpxord %ymm2,%ymm14,%ymm14 556 vprold $8,%ymm14,%ymm14 557 # x3 += x7, x15 = rotl32(x15 ^ x3, 8) 558 vpaddd %ymm3,%ymm7,%ymm3 559 vpxord %ymm3,%ymm15,%ymm15 560 vprold $8,%ymm15,%ymm15 561 562 # x8 += x12, x4 = rotl32(x4 ^ x8, 7) 563 vpaddd %ymm12,%ymm8,%ymm8 564 vpxord %ymm8,%ymm4,%ymm4 565 vprold $7,%ymm4,%ymm4 566 # x9 += x13, x5 = rotl32(x5 ^ x9, 7) 567 vpaddd %ymm13,%ymm9,%ymm9 568 vpxord %ymm9,%ymm5,%ymm5 569 vprold $7,%ymm5,%ymm5 570 # x10 += x14, x6 = rotl32(x6 ^ x10, 7) 571 vpaddd %ymm14,%ymm10,%ymm10 572 vpxord %ymm10,%ymm6,%ymm6 573 vprold $7,%ymm6,%ymm6 574 # x11 += x15, x7 = rotl32(x7 ^ x11, 7) 575 vpaddd %ymm15,%ymm11,%ymm11 576 vpxord %ymm11,%ymm7,%ymm7 577 vprold $7,%ymm7,%ymm7 578 579 # x0 += x5, x15 = rotl32(x15 ^ x0, 16) 580 vpaddd %ymm0,%ymm5,%ymm0 581 vpxord %ymm0,%ymm15,%ymm15 582 vprold $16,%ymm15,%ymm15 583 # x1 += x6, x12 = rotl32(x12 ^ x1, 16) 584 vpaddd %ymm1,%ymm6,%ymm1 585 vpxord %ymm1,%ymm12,%ymm12 586 vprold $16,%ymm12,%ymm12 587 # x2 += x7, x13 = rotl32(x13 ^ x2, 16) 588 vpaddd %ymm2,%ymm7,%ymm2 589 vpxord %ymm2,%ymm13,%ymm13 590 vprold $16,%ymm13,%ymm13 591 # x3 += x4, x14 = rotl32(x14 ^ x3, 16) 592 vpaddd %ymm3,%ymm4,%ymm3 593 vpxord %ymm3,%ymm14,%ymm14 594 vprold $16,%ymm14,%ymm14 595 596 # x10 += x15, x5 = rotl32(x5 ^ x10, 12) 597 vpaddd %ymm15,%ymm10,%ymm10 598 vpxord %ymm10,%ymm5,%ymm5 599 vprold $12,%ymm5,%ymm5 600 # x11 += x12, x6 = rotl32(x6 ^ x11, 12) 601 vpaddd %ymm12,%ymm11,%ymm11 602 vpxord %ymm11,%ymm6,%ymm6 603 vprold $12,%ymm6,%ymm6 604 # x8 += x13, x7 = rotl32(x7 ^ x8, 12) 605 vpaddd %ymm13,%ymm8,%ymm8 606 vpxord %ymm8,%ymm7,%ymm7 607 vprold $12,%ymm7,%ymm7 608 # x9 += x14, x4 = rotl32(x4 ^ x9, 12) 609 vpaddd %ymm14,%ymm9,%ymm9 610 vpxord %ymm9,%ymm4,%ymm4 611 vprold $12,%ymm4,%ymm4 612 613 # x0 += x5, x15 = rotl32(x15 ^ x0, 8) 614 vpaddd %ymm0,%ymm5,%ymm0 615 vpxord %ymm0,%ymm15,%ymm15 616 vprold $8,%ymm15,%ymm15 617 # x1 += x6, x12 = rotl32(x12 ^ x1, 8) 618 vpaddd %ymm1,%ymm6,%ymm1 619 vpxord %ymm1,%ymm12,%ymm12 620 vprold $8,%ymm12,%ymm12 621 # x2 += x7, x13 = rotl32(x13 ^ x2, 8) 622 vpaddd %ymm2,%ymm7,%ymm2 623 vpxord %ymm2,%ymm13,%ymm13 624 vprold $8,%ymm13,%ymm13 625 # x3 += x4, x14 = rotl32(x14 ^ x3, 8) 626 vpaddd %ymm3,%ymm4,%ymm3 627 vpxord %ymm3,%ymm14,%ymm14 628 vprold $8,%ymm14,%ymm14 629 630 # x10 += x15, x5 = rotl32(x5 ^ x10, 7) 631 vpaddd %ymm15,%ymm10,%ymm10 632 vpxord %ymm10,%ymm5,%ymm5 633 vprold $7,%ymm5,%ymm5 634 # x11 += x12, x6 = rotl32(x6 ^ x11, 7) 635 vpaddd %ymm12,%ymm11,%ymm11 636 vpxord %ymm11,%ymm6,%ymm6 637 vprold $7,%ymm6,%ymm6 638 # x8 += x13, x7 = rotl32(x7 ^ x8, 7) 639 vpaddd %ymm13,%ymm8,%ymm8 640 vpxord %ymm8,%ymm7,%ymm7 641 vprold $7,%ymm7,%ymm7 642 # x9 += x14, x4 = rotl32(x4 ^ x9, 7) 643 vpaddd %ymm14,%ymm9,%ymm9 644 vpxord %ymm9,%ymm4,%ymm4 645 vprold $7,%ymm4,%ymm4 646 647 sub $2,%r8d 648 jnz .Ldoubleround8 649 650 # x0..15[0-3] += s[0..15] 651 vpaddd %ymm16,%ymm0,%ymm0 652 vpaddd %ymm17,%ymm1,%ymm1 653 vpaddd %ymm18,%ymm2,%ymm2 654 vpaddd %ymm19,%ymm3,%ymm3 655 vpaddd %ymm20,%ymm4,%ymm4 656 vpaddd %ymm21,%ymm5,%ymm5 657 vpaddd %ymm22,%ymm6,%ymm6 658 vpaddd %ymm23,%ymm7,%ymm7 659 vpaddd %ymm24,%ymm8,%ymm8 660 vpaddd %ymm25,%ymm9,%ymm9 661 vpaddd %ymm26,%ymm10,%ymm10 662 vpaddd %ymm27,%ymm11,%ymm11 663 vpaddd %ymm28,%ymm12,%ymm12 664 vpaddd %ymm29,%ymm13,%ymm13 665 vpaddd %ymm30,%ymm14,%ymm14 666 vpaddd %ymm31,%ymm15,%ymm15 667 668 # interleave 32-bit words in state n, n+1 669 vpunpckldq %ymm1,%ymm0,%ymm16 670 vpunpckhdq %ymm1,%ymm0,%ymm17 671 vpunpckldq %ymm3,%ymm2,%ymm18 672 vpunpckhdq %ymm3,%ymm2,%ymm19 673 vpunpckldq %ymm5,%ymm4,%ymm20 674 vpunpckhdq %ymm5,%ymm4,%ymm21 675 vpunpckldq %ymm7,%ymm6,%ymm22 676 vpunpckhdq %ymm7,%ymm6,%ymm23 677 vpunpckldq %ymm9,%ymm8,%ymm24 678 vpunpckhdq %ymm9,%ymm8,%ymm25 679 vpunpckldq %ymm11,%ymm10,%ymm26 680 vpunpckhdq %ymm11,%ymm10,%ymm27 681 vpunpckldq %ymm13,%ymm12,%ymm28 682 vpunpckhdq %ymm13,%ymm12,%ymm29 683 vpunpckldq %ymm15,%ymm14,%ymm30 684 vpunpckhdq %ymm15,%ymm14,%ymm31 685 686 # interleave 64-bit words in state n, n+2 687 vpunpcklqdq %ymm18,%ymm16,%ymm0 688 vpunpcklqdq %ymm19,%ymm17,%ymm1 689 vpunpckhqdq %ymm18,%ymm16,%ymm2 690 vpunpckhqdq %ymm19,%ymm17,%ymm3 691 vpunpcklqdq %ymm22,%ymm20,%ymm4 692 vpunpcklqdq %ymm23,%ymm21,%ymm5 693 vpunpckhqdq %ymm22,%ymm20,%ymm6 694 vpunpckhqdq %ymm23,%ymm21,%ymm7 695 vpunpcklqdq %ymm26,%ymm24,%ymm8 696 vpunpcklqdq %ymm27,%ymm25,%ymm9 697 vpunpckhqdq %ymm26,%ymm24,%ymm10 698 vpunpckhqdq %ymm27,%ymm25,%ymm11 699 vpunpcklqdq %ymm30,%ymm28,%ymm12 700 vpunpcklqdq %ymm31,%ymm29,%ymm13 701 vpunpckhqdq %ymm30,%ymm28,%ymm14 702 vpunpckhqdq %ymm31,%ymm29,%ymm15 703 704 # interleave 128-bit words in state n, n+4 705 # xor/write first four blocks 706 vmovdqa64 %ymm0,%ymm16 707 vperm2i128 $0x20,%ymm4,%ymm0,%ymm0 708 cmp $0x0020,%rcx 709 jl .Lxorpart8 710 vpxord 0x0000(%rdx),%ymm0,%ymm0 711 vmovdqu64 %ymm0,0x0000(%rsi) 712 vmovdqa64 %ymm16,%ymm0 713 vperm2i128 $0x31,%ymm4,%ymm0,%ymm4 714 715 vperm2i128 $0x20,%ymm12,%ymm8,%ymm0 716 cmp $0x0040,%rcx 717 jl .Lxorpart8 718 vpxord 0x0020(%rdx),%ymm0,%ymm0 719 vmovdqu64 %ymm0,0x0020(%rsi) 720 vperm2i128 $0x31,%ymm12,%ymm8,%ymm12 721 722 vperm2i128 $0x20,%ymm6,%ymm2,%ymm0 723 cmp $0x0060,%rcx 724 jl .Lxorpart8 725 vpxord 0x0040(%rdx),%ymm0,%ymm0 726 vmovdqu64 %ymm0,0x0040(%rsi) 727 vperm2i128 $0x31,%ymm6,%ymm2,%ymm6 728 729 vperm2i128 $0x20,%ymm14,%ymm10,%ymm0 730 cmp $0x0080,%rcx 731 jl .Lxorpart8 732 vpxord 0x0060(%rdx),%ymm0,%ymm0 733 vmovdqu64 %ymm0,0x0060(%rsi) 734 vperm2i128 $0x31,%ymm14,%ymm10,%ymm14 735 736 vperm2i128 $0x20,%ymm5,%ymm1,%ymm0 737 cmp $0x00a0,%rcx 738 jl .Lxorpart8 739 vpxord 0x0080(%rdx),%ymm0,%ymm0 740 vmovdqu64 %ymm0,0x0080(%rsi) 741 vperm2i128 $0x31,%ymm5,%ymm1,%ymm5 742 743 vperm2i128 $0x20,%ymm13,%ymm9,%ymm0 744 cmp $0x00c0,%rcx 745 jl .Lxorpart8 746 vpxord 0x00a0(%rdx),%ymm0,%ymm0 747 vmovdqu64 %ymm0,0x00a0(%rsi) 748 vperm2i128 $0x31,%ymm13,%ymm9,%ymm13 749 750 vperm2i128 $0x20,%ymm7,%ymm3,%ymm0 751 cmp $0x00e0,%rcx 752 jl .Lxorpart8 753 vpxord 0x00c0(%rdx),%ymm0,%ymm0 754 vmovdqu64 %ymm0,0x00c0(%rsi) 755 vperm2i128 $0x31,%ymm7,%ymm3,%ymm7 756 757 vperm2i128 $0x20,%ymm15,%ymm11,%ymm0 758 cmp $0x0100,%rcx 759 jl .Lxorpart8 760 vpxord 0x00e0(%rdx),%ymm0,%ymm0 761 vmovdqu64 %ymm0,0x00e0(%rsi) 762 vperm2i128 $0x31,%ymm15,%ymm11,%ymm15 763 764 # xor remaining blocks, write to output 765 vmovdqa64 %ymm4,%ymm0 766 cmp $0x0120,%rcx 767 jl .Lxorpart8 768 vpxord 0x0100(%rdx),%ymm0,%ymm0 769 vmovdqu64 %ymm0,0x0100(%rsi) 770 771 vmovdqa64 %ymm12,%ymm0 772 cmp $0x0140,%rcx 773 jl .Lxorpart8 774 vpxord 0x0120(%rdx),%ymm0,%ymm0 775 vmovdqu64 %ymm0,0x0120(%rsi) 776 777 vmovdqa64 %ymm6,%ymm0 778 cmp $0x0160,%rcx 779 jl .Lxorpart8 780 vpxord 0x0140(%rdx),%ymm0,%ymm0 781 vmovdqu64 %ymm0,0x0140(%rsi) 782 783 vmovdqa64 %ymm14,%ymm0 784 cmp $0x0180,%rcx 785 jl .Lxorpart8 786 vpxord 0x0160(%rdx),%ymm0,%ymm0 787 vmovdqu64 %ymm0,0x0160(%rsi) 788 789 vmovdqa64 %ymm5,%ymm0 790 cmp $0x01a0,%rcx 791 jl .Lxorpart8 792 vpxord 0x0180(%rdx),%ymm0,%ymm0 793 vmovdqu64 %ymm0,0x0180(%rsi) 794 795 vmovdqa64 %ymm13,%ymm0 796 cmp $0x01c0,%rcx 797 jl .Lxorpart8 798 vpxord 0x01a0(%rdx),%ymm0,%ymm0 799 vmovdqu64 %ymm0,0x01a0(%rsi) 800 801 vmovdqa64 %ymm7,%ymm0 802 cmp $0x01e0,%rcx 803 jl .Lxorpart8 804 vpxord 0x01c0(%rdx),%ymm0,%ymm0 805 vmovdqu64 %ymm0,0x01c0(%rsi) 806 807 vmovdqa64 %ymm15,%ymm0 808 cmp $0x0200,%rcx 809 jl .Lxorpart8 810 vpxord 0x01e0(%rdx),%ymm0,%ymm0 811 vmovdqu64 %ymm0,0x01e0(%rsi) 812 813 .Ldone8: 814 vzeroupper 815 RET 816 817 .Lxorpart8: 818 # xor remaining bytes from partial register into output 819 mov %rcx,%rax 820 and $0x1f,%rcx 821 jz .Ldone8 822 mov %rax,%r9 823 and $~0x1f,%r9 824 825 mov $1,%rax 826 shld %cl,%rax,%rax 827 sub $1,%rax 828 kmovq %rax,%k1 829 830 vmovdqu8 (%rdx,%r9),%ymm1{%k1}{z} 831 vpxord %ymm0,%ymm1,%ymm1 832 vmovdqu8 %ymm1,(%rsi,%r9){%k1} 833 834 jmp .Ldone8 835 836 SYM_FUNC_END(chacha_8block_xor_avx512vl) 837