1 ######################################################################## 2 # Copyright (c) 2013, Intel Corporation 3 # 4 # This software is available to you under a choice of one of two 5 # licenses. You may choose to be licensed under the terms of the GNU 6 # General Public License (GPL) Version 2, available from the file 7 # COPYING in the main directory of this source tree, or the 8 # OpenIB.org BSD license below: 9 # 10 # Redistribution and use in source and binary forms, with or without 11 # modification, are permitted provided that the following conditions are 12 # met: 13 # 14 # * Redistributions of source code must retain the above copyright 15 # notice, this list of conditions and the following disclaimer. 16 # 17 # * Redistributions in binary form must reproduce the above copyright 18 # notice, this list of conditions and the following disclaimer in the 19 # documentation and/or other materials provided with the 20 # distribution. 21 # 22 # * Neither the name of the Intel Corporation nor the names of its 23 # contributors may be used to endorse or promote products derived from 24 # this software without specific prior written permission. 25 # 26 # 27 # THIS SOFTWARE IS PROVIDED BY INTEL CORPORATION ""AS IS"" AND ANY 28 # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 29 # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 30 # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL INTEL CORPORATION OR 31 # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 32 # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 33 # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES# LOSS OF USE, DATA, OR 34 # PROFITS# OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 35 # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 36 # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 37 # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 38 ######################################################################## 39 ## 40 ## Authors: 41 ## Erdinc Ozturk <erdinc.ozturk@intel.com> 42 ## Vinodh Gopal <vinodh.gopal@intel.com> 43 ## James Guilford <james.guilford@intel.com> 44 ## Tim Chen <tim.c.chen@linux.intel.com> 45 ## 46 ## References: 47 ## This code was derived and highly optimized from the code described in paper: 48 ## Vinodh Gopal et. al. Optimized Galois-Counter-Mode Implementation 49 ## on Intel Architecture Processors. August, 2010 50 ## The details of the implementation is explained in: 51 ## Erdinc Ozturk et. al. Enabling High-Performance Galois-Counter-Mode 52 ## on Intel Architecture Processors. October, 2012. 53 ## 54 ## Assumptions: 55 ## 56 ## 57 ## 58 ## iv: 59 ## 0 1 2 3 60 ## 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 61 ## +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ 62 ## | Salt (From the SA) | 63 ## +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ 64 ## | Initialization Vector | 65 ## | (This is the sequence number from IPSec header) | 66 ## +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ 67 ## | 0x1 | 68 ## +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ 69 ## 70 ## 71 ## 72 ## AAD: 73 ## AAD padded to 128 bits with 0 74 ## for example, assume AAD is a u32 vector 75 ## 76 ## if AAD is 8 bytes: 77 ## AAD[3] = {A0, A1}# 78 ## padded AAD in xmm register = {A1 A0 0 0} 79 ## 80 ## 0 1 2 3 81 ## 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 82 ## +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ 83 ## | SPI (A1) | 84 ## +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ 85 ## | 32-bit Sequence Number (A0) | 86 ## +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ 87 ## | 0x0 | 88 ## +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ 89 ## 90 ## AAD Format with 32-bit Sequence Number 91 ## 92 ## if AAD is 12 bytes: 93 ## AAD[3] = {A0, A1, A2}# 94 ## padded AAD in xmm register = {A2 A1 A0 0} 95 ## 96 ## 0 1 2 3 97 ## 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 98 ## +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ 99 ## | SPI (A2) | 100 ## +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ 101 ## | 64-bit Extended Sequence Number {A1,A0} | 102 ## | | 103 ## +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ 104 ## | 0x0 | 105 ## +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ 106 ## 107 ## AAD Format with 64-bit Extended Sequence Number 108 ## 109 ## 110 ## aadLen: 111 ## from the definition of the spec, aadLen can only be 8 or 12 bytes. 112 ## The code additionally supports aadLen of length 16 bytes. 113 ## 114 ## TLen: 115 ## from the definition of the spec, TLen can only be 8, 12 or 16 bytes. 116 ## 117 ## poly = x^128 + x^127 + x^126 + x^121 + 1 118 ## throughout the code, one tab and two tab indentations are used. one tab is 119 ## for GHASH part, two tabs is for AES part. 120 ## 121 122 #include <linux/linkage.h> 123 124 # constants in mergeable sections, linker can reorder and merge 125 .section .rodata.cst16.POLY, "aM", @progbits, 16 126 .align 16 127 POLY: .octa 0xC2000000000000000000000000000001 128 129 .section .rodata.cst16.POLY2, "aM", @progbits, 16 130 .align 16 131 POLY2: .octa 0xC20000000000000000000001C2000000 132 133 .section .rodata.cst16.TWOONE, "aM", @progbits, 16 134 .align 16 135 TWOONE: .octa 0x00000001000000000000000000000001 136 137 .section .rodata.cst16.SHUF_MASK, "aM", @progbits, 16 138 .align 16 139 SHUF_MASK: .octa 0x000102030405060708090A0B0C0D0E0F 140 141 .section .rodata.cst16.ONE, "aM", @progbits, 16 142 .align 16 143 ONE: .octa 0x00000000000000000000000000000001 144 145 .section .rodata.cst16.ONEf, "aM", @progbits, 16 146 .align 16 147 ONEf: .octa 0x01000000000000000000000000000000 148 149 # order of these constants should not change. 150 # more specifically, ALL_F should follow SHIFT_MASK, and zero should follow ALL_F 151 .section .rodata, "a", @progbits 152 .align 16 153 SHIFT_MASK: .octa 0x0f0e0d0c0b0a09080706050403020100 154 ALL_F: .octa 0xffffffffffffffffffffffffffffffff 155 .octa 0x00000000000000000000000000000000 156 157 .text 158 159 160 #define AadHash 16*0 161 #define AadLen 16*1 162 #define InLen (16*1)+8 163 #define PBlockEncKey 16*2 164 #define OrigIV 16*3 165 #define CurCount 16*4 166 #define PBlockLen 16*5 167 168 HashKey = 16*6 # store HashKey <<1 mod poly here 169 HashKey_2 = 16*7 # store HashKey^2 <<1 mod poly here 170 HashKey_3 = 16*8 # store HashKey^3 <<1 mod poly here 171 HashKey_4 = 16*9 # store HashKey^4 <<1 mod poly here 172 HashKey_5 = 16*10 # store HashKey^5 <<1 mod poly here 173 HashKey_6 = 16*11 # store HashKey^6 <<1 mod poly here 174 HashKey_7 = 16*12 # store HashKey^7 <<1 mod poly here 175 HashKey_8 = 16*13 # store HashKey^8 <<1 mod poly here 176 HashKey_k = 16*14 # store XOR of HashKey <<1 mod poly here (for Karatsuba purposes) 177 HashKey_2_k = 16*15 # store XOR of HashKey^2 <<1 mod poly here (for Karatsuba purposes) 178 HashKey_3_k = 16*16 # store XOR of HashKey^3 <<1 mod poly here (for Karatsuba purposes) 179 HashKey_4_k = 16*17 # store XOR of HashKey^4 <<1 mod poly here (for Karatsuba purposes) 180 HashKey_5_k = 16*18 # store XOR of HashKey^5 <<1 mod poly here (for Karatsuba purposes) 181 HashKey_6_k = 16*19 # store XOR of HashKey^6 <<1 mod poly here (for Karatsuba purposes) 182 HashKey_7_k = 16*20 # store XOR of HashKey^7 <<1 mod poly here (for Karatsuba purposes) 183 HashKey_8_k = 16*21 # store XOR of HashKey^8 <<1 mod poly here (for Karatsuba purposes) 184 185 #define arg1 %rdi 186 #define arg2 %rsi 187 #define arg3 %rdx 188 #define arg4 %rcx 189 #define arg5 %r8 190 #define arg6 %r9 191 #define keysize 2*15*16(arg1) 192 193 i = 0 194 j = 0 195 196 out_order = 0 197 in_order = 1 198 DEC = 0 199 ENC = 1 200 201 .macro define_reg r n 202 reg_\r = %xmm\n 203 .endm 204 205 .macro setreg 206 .altmacro 207 define_reg i %i 208 define_reg j %j 209 .noaltmacro 210 .endm 211 212 TMP1 = 16*0 # Temporary storage for AAD 213 TMP2 = 16*1 # Temporary storage for AES State 2 (State 1 is stored in an XMM register) 214 TMP3 = 16*2 # Temporary storage for AES State 3 215 TMP4 = 16*3 # Temporary storage for AES State 4 216 TMP5 = 16*4 # Temporary storage for AES State 5 217 TMP6 = 16*5 # Temporary storage for AES State 6 218 TMP7 = 16*6 # Temporary storage for AES State 7 219 TMP8 = 16*7 # Temporary storage for AES State 8 220 221 VARIABLE_OFFSET = 16*8 222 223 ################################ 224 # Utility Macros 225 ################################ 226 227 .macro FUNC_SAVE 228 push %r12 229 push %r13 230 push %r15 231 232 push %rbp 233 mov %rsp, %rbp 234 235 sub $VARIABLE_OFFSET, %rsp 236 and $~63, %rsp # align rsp to 64 bytes 237 .endm 238 239 .macro FUNC_RESTORE 240 mov %rbp, %rsp 241 pop %rbp 242 243 pop %r15 244 pop %r13 245 pop %r12 246 .endm 247 248 # Encryption of a single block 249 .macro ENCRYPT_SINGLE_BLOCK REP XMM0 250 vpxor (arg1), \XMM0, \XMM0 251 i = 1 252 setreg 253 .rep \REP 254 vaesenc 16*i(arg1), \XMM0, \XMM0 255 i = (i+1) 256 setreg 257 .endr 258 vaesenclast 16*i(arg1), \XMM0, \XMM0 259 .endm 260 261 # combined for GCM encrypt and decrypt functions 262 # clobbering all xmm registers 263 # clobbering r10, r11, r12, r13, r15, rax 264 .macro GCM_ENC_DEC INITIAL_BLOCKS GHASH_8_ENCRYPT_8_PARALLEL GHASH_LAST_8 GHASH_MUL ENC_DEC REP 265 vmovdqu AadHash(arg2), %xmm8 266 vmovdqu HashKey(arg2), %xmm13 # xmm13 = HashKey 267 add arg5, InLen(arg2) 268 269 # initialize the data pointer offset as zero 270 xor %r11d, %r11d 271 272 PARTIAL_BLOCK \GHASH_MUL, arg3, arg4, arg5, %r11, %xmm8, \ENC_DEC 273 sub %r11, arg5 274 275 mov arg5, %r13 # save the number of bytes of plaintext/ciphertext 276 and $-16, %r13 # r13 = r13 - (r13 mod 16) 277 278 mov %r13, %r12 279 shr $4, %r12 280 and $7, %r12 281 jz .L_initial_num_blocks_is_0\@ 282 283 cmp $7, %r12 284 je .L_initial_num_blocks_is_7\@ 285 cmp $6, %r12 286 je .L_initial_num_blocks_is_6\@ 287 cmp $5, %r12 288 je .L_initial_num_blocks_is_5\@ 289 cmp $4, %r12 290 je .L_initial_num_blocks_is_4\@ 291 cmp $3, %r12 292 je .L_initial_num_blocks_is_3\@ 293 cmp $2, %r12 294 je .L_initial_num_blocks_is_2\@ 295 296 jmp .L_initial_num_blocks_is_1\@ 297 298 .L_initial_num_blocks_is_7\@: 299 \INITIAL_BLOCKS \REP, 7, %xmm12, %xmm13, %xmm14, %xmm15, %xmm11, %xmm9, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7, %xmm8, %xmm10, %xmm0, \ENC_DEC 300 sub $16*7, %r13 301 jmp .L_initial_blocks_encrypted\@ 302 303 .L_initial_num_blocks_is_6\@: 304 \INITIAL_BLOCKS \REP, 6, %xmm12, %xmm13, %xmm14, %xmm15, %xmm11, %xmm9, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7, %xmm8, %xmm10, %xmm0, \ENC_DEC 305 sub $16*6, %r13 306 jmp .L_initial_blocks_encrypted\@ 307 308 .L_initial_num_blocks_is_5\@: 309 \INITIAL_BLOCKS \REP, 5, %xmm12, %xmm13, %xmm14, %xmm15, %xmm11, %xmm9, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7, %xmm8, %xmm10, %xmm0, \ENC_DEC 310 sub $16*5, %r13 311 jmp .L_initial_blocks_encrypted\@ 312 313 .L_initial_num_blocks_is_4\@: 314 \INITIAL_BLOCKS \REP, 4, %xmm12, %xmm13, %xmm14, %xmm15, %xmm11, %xmm9, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7, %xmm8, %xmm10, %xmm0, \ENC_DEC 315 sub $16*4, %r13 316 jmp .L_initial_blocks_encrypted\@ 317 318 .L_initial_num_blocks_is_3\@: 319 \INITIAL_BLOCKS \REP, 3, %xmm12, %xmm13, %xmm14, %xmm15, %xmm11, %xmm9, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7, %xmm8, %xmm10, %xmm0, \ENC_DEC 320 sub $16*3, %r13 321 jmp .L_initial_blocks_encrypted\@ 322 323 .L_initial_num_blocks_is_2\@: 324 \INITIAL_BLOCKS \REP, 2, %xmm12, %xmm13, %xmm14, %xmm15, %xmm11, %xmm9, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7, %xmm8, %xmm10, %xmm0, \ENC_DEC 325 sub $16*2, %r13 326 jmp .L_initial_blocks_encrypted\@ 327 328 .L_initial_num_blocks_is_1\@: 329 \INITIAL_BLOCKS \REP, 1, %xmm12, %xmm13, %xmm14, %xmm15, %xmm11, %xmm9, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7, %xmm8, %xmm10, %xmm0, \ENC_DEC 330 sub $16*1, %r13 331 jmp .L_initial_blocks_encrypted\@ 332 333 .L_initial_num_blocks_is_0\@: 334 \INITIAL_BLOCKS \REP, 0, %xmm12, %xmm13, %xmm14, %xmm15, %xmm11, %xmm9, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7, %xmm8, %xmm10, %xmm0, \ENC_DEC 335 336 337 .L_initial_blocks_encrypted\@: 338 test %r13, %r13 339 je .L_zero_cipher_left\@ 340 341 sub $128, %r13 342 je .L_eight_cipher_left\@ 343 344 345 346 347 vmovd %xmm9, %r15d 348 and $255, %r15d 349 vpshufb SHUF_MASK(%rip), %xmm9, %xmm9 350 351 352 .L_encrypt_by_8_new\@: 353 cmp $(255-8), %r15d 354 jg .L_encrypt_by_8\@ 355 356 357 358 add $8, %r15b 359 \GHASH_8_ENCRYPT_8_PARALLEL \REP, %xmm0, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14, %xmm9, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7, %xmm8, %xmm15, out_order, \ENC_DEC 360 add $128, %r11 361 sub $128, %r13 362 jne .L_encrypt_by_8_new\@ 363 364 vpshufb SHUF_MASK(%rip), %xmm9, %xmm9 365 jmp .L_eight_cipher_left\@ 366 367 .L_encrypt_by_8\@: 368 vpshufb SHUF_MASK(%rip), %xmm9, %xmm9 369 add $8, %r15b 370 \GHASH_8_ENCRYPT_8_PARALLEL \REP, %xmm0, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14, %xmm9, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7, %xmm8, %xmm15, in_order, \ENC_DEC 371 vpshufb SHUF_MASK(%rip), %xmm9, %xmm9 372 add $128, %r11 373 sub $128, %r13 374 jne .L_encrypt_by_8_new\@ 375 376 vpshufb SHUF_MASK(%rip), %xmm9, %xmm9 377 378 379 380 381 .L_eight_cipher_left\@: 382 \GHASH_LAST_8 %xmm0, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14, %xmm15, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7, %xmm8 383 384 385 .L_zero_cipher_left\@: 386 vmovdqu %xmm14, AadHash(arg2) 387 vmovdqu %xmm9, CurCount(arg2) 388 389 # check for 0 length 390 mov arg5, %r13 391 and $15, %r13 # r13 = (arg5 mod 16) 392 393 je .L_multiple_of_16_bytes\@ 394 395 # handle the last <16 Byte block separately 396 397 mov %r13, PBlockLen(arg2) 398 399 vpaddd ONE(%rip), %xmm9, %xmm9 # INCR CNT to get Yn 400 vmovdqu %xmm9, CurCount(arg2) 401 vpshufb SHUF_MASK(%rip), %xmm9, %xmm9 402 403 ENCRYPT_SINGLE_BLOCK \REP, %xmm9 # E(K, Yn) 404 vmovdqu %xmm9, PBlockEncKey(arg2) 405 406 cmp $16, arg5 407 jge .L_large_enough_update\@ 408 409 lea (arg4,%r11,1), %r10 410 mov %r13, %r12 411 412 READ_PARTIAL_BLOCK %r10 %r12 %xmm1 413 414 lea SHIFT_MASK+16(%rip), %r12 415 sub %r13, %r12 # adjust the shuffle mask pointer to be 416 # able to shift 16-r13 bytes (r13 is the 417 # number of bytes in plaintext mod 16) 418 419 jmp .L_final_ghash_mul\@ 420 421 .L_large_enough_update\@: 422 sub $16, %r11 423 add %r13, %r11 424 425 # receive the last <16 Byte block 426 vmovdqu (arg4, %r11, 1), %xmm1 427 428 sub %r13, %r11 429 add $16, %r11 430 431 lea SHIFT_MASK+16(%rip), %r12 432 # adjust the shuffle mask pointer to be able to shift 16-r13 bytes 433 # (r13 is the number of bytes in plaintext mod 16) 434 sub %r13, %r12 435 # get the appropriate shuffle mask 436 vmovdqu (%r12), %xmm2 437 # shift right 16-r13 bytes 438 vpshufb %xmm2, %xmm1, %xmm1 439 440 .L_final_ghash_mul\@: 441 .if \ENC_DEC == DEC 442 vmovdqa %xmm1, %xmm2 443 vpxor %xmm1, %xmm9, %xmm9 # Plaintext XOR E(K, Yn) 444 vmovdqu ALL_F-SHIFT_MASK(%r12), %xmm1 # get the appropriate mask to 445 # mask out top 16-r13 bytes of xmm9 446 vpand %xmm1, %xmm9, %xmm9 # mask out top 16-r13 bytes of xmm9 447 vpand %xmm1, %xmm2, %xmm2 448 vpshufb SHUF_MASK(%rip), %xmm2, %xmm2 449 vpxor %xmm2, %xmm14, %xmm14 450 451 vmovdqu %xmm14, AadHash(arg2) 452 .else 453 vpxor %xmm1, %xmm9, %xmm9 # Plaintext XOR E(K, Yn) 454 vmovdqu ALL_F-SHIFT_MASK(%r12), %xmm1 # get the appropriate mask to 455 # mask out top 16-r13 bytes of xmm9 456 vpand %xmm1, %xmm9, %xmm9 # mask out top 16-r13 bytes of xmm9 457 vpshufb SHUF_MASK(%rip), %xmm9, %xmm9 458 vpxor %xmm9, %xmm14, %xmm14 459 460 vmovdqu %xmm14, AadHash(arg2) 461 vpshufb SHUF_MASK(%rip), %xmm9, %xmm9 # shuffle xmm9 back to output as ciphertext 462 .endif 463 464 465 ############################# 466 # output r13 Bytes 467 vmovq %xmm9, %rax 468 cmp $8, %r13 469 jle .L_less_than_8_bytes_left\@ 470 471 mov %rax, (arg3 , %r11) 472 add $8, %r11 473 vpsrldq $8, %xmm9, %xmm9 474 vmovq %xmm9, %rax 475 sub $8, %r13 476 477 .L_less_than_8_bytes_left\@: 478 movb %al, (arg3 , %r11) 479 add $1, %r11 480 shr $8, %rax 481 sub $1, %r13 482 jne .L_less_than_8_bytes_left\@ 483 ############################# 484 485 .L_multiple_of_16_bytes\@: 486 .endm 487 488 489 # GCM_COMPLETE Finishes update of tag of last partial block 490 # Output: Authorization Tag (AUTH_TAG) 491 # Clobbers rax, r10-r12, and xmm0, xmm1, xmm5-xmm15 492 .macro GCM_COMPLETE GHASH_MUL REP AUTH_TAG AUTH_TAG_LEN 493 vmovdqu AadHash(arg2), %xmm14 494 vmovdqu HashKey(arg2), %xmm13 495 496 mov PBlockLen(arg2), %r12 497 test %r12, %r12 498 je .L_partial_done\@ 499 500 #GHASH computation for the last <16 Byte block 501 \GHASH_MUL %xmm14, %xmm13, %xmm0, %xmm10, %xmm11, %xmm5, %xmm6 502 503 .L_partial_done\@: 504 mov AadLen(arg2), %r12 # r12 = aadLen (number of bytes) 505 shl $3, %r12 # convert into number of bits 506 vmovd %r12d, %xmm15 # len(A) in xmm15 507 508 mov InLen(arg2), %r12 509 shl $3, %r12 # len(C) in bits (*128) 510 vmovq %r12, %xmm1 511 vpslldq $8, %xmm15, %xmm15 # xmm15 = len(A)|| 0x0000000000000000 512 vpxor %xmm1, %xmm15, %xmm15 # xmm15 = len(A)||len(C) 513 514 vpxor %xmm15, %xmm14, %xmm14 515 \GHASH_MUL %xmm14, %xmm13, %xmm0, %xmm10, %xmm11, %xmm5, %xmm6 # final GHASH computation 516 vpshufb SHUF_MASK(%rip), %xmm14, %xmm14 # perform a 16Byte swap 517 518 vmovdqu OrigIV(arg2), %xmm9 519 520 ENCRYPT_SINGLE_BLOCK \REP, %xmm9 # E(K, Y0) 521 522 vpxor %xmm14, %xmm9, %xmm9 523 524 525 526 .L_return_T\@: 527 mov \AUTH_TAG, %r10 # r10 = authTag 528 mov \AUTH_TAG_LEN, %r11 # r11 = auth_tag_len 529 530 cmp $16, %r11 531 je .L_T_16\@ 532 533 cmp $8, %r11 534 jl .L_T_4\@ 535 536 .L_T_8\@: 537 vmovq %xmm9, %rax 538 mov %rax, (%r10) 539 add $8, %r10 540 sub $8, %r11 541 vpsrldq $8, %xmm9, %xmm9 542 test %r11, %r11 543 je .L_return_T_done\@ 544 .L_T_4\@: 545 vmovd %xmm9, %eax 546 mov %eax, (%r10) 547 add $4, %r10 548 sub $4, %r11 549 vpsrldq $4, %xmm9, %xmm9 550 test %r11, %r11 551 je .L_return_T_done\@ 552 .L_T_123\@: 553 vmovd %xmm9, %eax 554 cmp $2, %r11 555 jl .L_T_1\@ 556 mov %ax, (%r10) 557 cmp $2, %r11 558 je .L_return_T_done\@ 559 add $2, %r10 560 sar $16, %eax 561 .L_T_1\@: 562 mov %al, (%r10) 563 jmp .L_return_T_done\@ 564 565 .L_T_16\@: 566 vmovdqu %xmm9, (%r10) 567 568 .L_return_T_done\@: 569 .endm 570 571 .macro CALC_AAD_HASH GHASH_MUL AAD AADLEN T1 T2 T3 T4 T5 T6 T7 T8 572 573 mov \AAD, %r10 # r10 = AAD 574 mov \AADLEN, %r12 # r12 = aadLen 575 576 577 mov %r12, %r11 578 579 vpxor \T8, \T8, \T8 580 vpxor \T7, \T7, \T7 581 cmp $16, %r11 582 jl .L_get_AAD_rest8\@ 583 .L_get_AAD_blocks\@: 584 vmovdqu (%r10), \T7 585 vpshufb SHUF_MASK(%rip), \T7, \T7 586 vpxor \T7, \T8, \T8 587 \GHASH_MUL \T8, \T2, \T1, \T3, \T4, \T5, \T6 588 add $16, %r10 589 sub $16, %r12 590 sub $16, %r11 591 cmp $16, %r11 592 jge .L_get_AAD_blocks\@ 593 vmovdqu \T8, \T7 594 test %r11, %r11 595 je .L_get_AAD_done\@ 596 597 vpxor \T7, \T7, \T7 598 599 /* read the last <16B of AAD. since we have at least 4B of 600 data right after the AAD (the ICV, and maybe some CT), we can 601 read 4B/8B blocks safely, and then get rid of the extra stuff */ 602 .L_get_AAD_rest8\@: 603 cmp $4, %r11 604 jle .L_get_AAD_rest4\@ 605 movq (%r10), \T1 606 add $8, %r10 607 sub $8, %r11 608 vpslldq $8, \T1, \T1 609 vpsrldq $8, \T7, \T7 610 vpxor \T1, \T7, \T7 611 jmp .L_get_AAD_rest8\@ 612 .L_get_AAD_rest4\@: 613 test %r11, %r11 614 jle .L_get_AAD_rest0\@ 615 mov (%r10), %eax 616 movq %rax, \T1 617 add $4, %r10 618 sub $4, %r11 619 vpslldq $12, \T1, \T1 620 vpsrldq $4, \T7, \T7 621 vpxor \T1, \T7, \T7 622 .L_get_AAD_rest0\@: 623 /* finalize: shift out the extra bytes we read, and align 624 left. since pslldq can only shift by an immediate, we use 625 vpshufb and a pair of shuffle masks */ 626 leaq ALL_F(%rip), %r11 627 subq %r12, %r11 628 vmovdqu 16(%r11), \T1 629 andq $~3, %r11 630 vpshufb (%r11), \T7, \T7 631 vpand \T1, \T7, \T7 632 .L_get_AAD_rest_final\@: 633 vpshufb SHUF_MASK(%rip), \T7, \T7 634 vpxor \T8, \T7, \T7 635 \GHASH_MUL \T7, \T2, \T1, \T3, \T4, \T5, \T6 636 637 .L_get_AAD_done\@: 638 vmovdqu \T7, AadHash(arg2) 639 .endm 640 641 .macro INIT GHASH_MUL PRECOMPUTE 642 mov arg6, %r11 643 mov %r11, AadLen(arg2) # ctx_data.aad_length = aad_length 644 xor %r11d, %r11d 645 mov %r11, InLen(arg2) # ctx_data.in_length = 0 646 647 mov %r11, PBlockLen(arg2) # ctx_data.partial_block_length = 0 648 mov %r11, PBlockEncKey(arg2) # ctx_data.partial_block_enc_key = 0 649 mov arg3, %rax 650 movdqu (%rax), %xmm0 651 movdqu %xmm0, OrigIV(arg2) # ctx_data.orig_IV = iv 652 653 vpshufb SHUF_MASK(%rip), %xmm0, %xmm0 654 movdqu %xmm0, CurCount(arg2) # ctx_data.current_counter = iv 655 656 vmovdqu (arg4), %xmm6 # xmm6 = HashKey 657 658 vpshufb SHUF_MASK(%rip), %xmm6, %xmm6 659 ############### PRECOMPUTATION of HashKey<<1 mod poly from the HashKey 660 vmovdqa %xmm6, %xmm2 661 vpsllq $1, %xmm6, %xmm6 662 vpsrlq $63, %xmm2, %xmm2 663 vmovdqa %xmm2, %xmm1 664 vpslldq $8, %xmm2, %xmm2 665 vpsrldq $8, %xmm1, %xmm1 666 vpor %xmm2, %xmm6, %xmm6 667 #reduction 668 vpshufd $0b00100100, %xmm1, %xmm2 669 vpcmpeqd TWOONE(%rip), %xmm2, %xmm2 670 vpand POLY(%rip), %xmm2, %xmm2 671 vpxor %xmm2, %xmm6, %xmm6 # xmm6 holds the HashKey<<1 mod poly 672 ####################################################################### 673 vmovdqu %xmm6, HashKey(arg2) # store HashKey<<1 mod poly 674 675 CALC_AAD_HASH \GHASH_MUL, arg5, arg6, %xmm2, %xmm6, %xmm3, %xmm4, %xmm5, %xmm7, %xmm1, %xmm0 676 677 \PRECOMPUTE %xmm6, %xmm0, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5 678 .endm 679 680 681 # Reads DLEN bytes starting at DPTR and stores in XMMDst 682 # where 0 < DLEN < 16 683 # Clobbers %rax, DLEN 684 .macro READ_PARTIAL_BLOCK DPTR DLEN XMMDst 685 vpxor \XMMDst, \XMMDst, \XMMDst 686 687 cmp $8, \DLEN 688 jl .L_read_lt8_\@ 689 mov (\DPTR), %rax 690 vpinsrq $0, %rax, \XMMDst, \XMMDst 691 sub $8, \DLEN 692 jz .L_done_read_partial_block_\@ 693 xor %eax, %eax 694 .L_read_next_byte_\@: 695 shl $8, %rax 696 mov 7(\DPTR, \DLEN, 1), %al 697 dec \DLEN 698 jnz .L_read_next_byte_\@ 699 vpinsrq $1, %rax, \XMMDst, \XMMDst 700 jmp .L_done_read_partial_block_\@ 701 .L_read_lt8_\@: 702 xor %eax, %eax 703 .L_read_next_byte_lt8_\@: 704 shl $8, %rax 705 mov -1(\DPTR, \DLEN, 1), %al 706 dec \DLEN 707 jnz .L_read_next_byte_lt8_\@ 708 vpinsrq $0, %rax, \XMMDst, \XMMDst 709 .L_done_read_partial_block_\@: 710 .endm 711 712 # PARTIAL_BLOCK: Handles encryption/decryption and the tag partial blocks 713 # between update calls. 714 # Requires the input data be at least 1 byte long due to READ_PARTIAL_BLOCK 715 # Outputs encrypted bytes, and updates hash and partial info in gcm_data_context 716 # Clobbers rax, r10, r12, r13, xmm0-6, xmm9-13 717 .macro PARTIAL_BLOCK GHASH_MUL CYPH_PLAIN_OUT PLAIN_CYPH_IN PLAIN_CYPH_LEN DATA_OFFSET \ 718 AAD_HASH ENC_DEC 719 mov PBlockLen(arg2), %r13 720 test %r13, %r13 721 je .L_partial_block_done_\@ # Leave Macro if no partial blocks 722 # Read in input data without over reading 723 cmp $16, \PLAIN_CYPH_LEN 724 jl .L_fewer_than_16_bytes_\@ 725 vmovdqu (\PLAIN_CYPH_IN), %xmm1 # If more than 16 bytes, just fill xmm 726 jmp .L_data_read_\@ 727 728 .L_fewer_than_16_bytes_\@: 729 lea (\PLAIN_CYPH_IN, \DATA_OFFSET, 1), %r10 730 mov \PLAIN_CYPH_LEN, %r12 731 READ_PARTIAL_BLOCK %r10 %r12 %xmm1 732 733 mov PBlockLen(arg2), %r13 734 735 .L_data_read_\@: # Finished reading in data 736 737 vmovdqu PBlockEncKey(arg2), %xmm9 738 vmovdqu HashKey(arg2), %xmm13 739 740 lea SHIFT_MASK(%rip), %r12 741 742 # adjust the shuffle mask pointer to be able to shift r13 bytes 743 # r16-r13 is the number of bytes in plaintext mod 16) 744 add %r13, %r12 745 vmovdqu (%r12), %xmm2 # get the appropriate shuffle mask 746 vpshufb %xmm2, %xmm9, %xmm9 # shift right r13 bytes 747 748 .if \ENC_DEC == DEC 749 vmovdqa %xmm1, %xmm3 750 pxor %xmm1, %xmm9 # Cyphertext XOR E(K, Yn) 751 752 mov \PLAIN_CYPH_LEN, %r10 753 add %r13, %r10 754 # Set r10 to be the amount of data left in CYPH_PLAIN_IN after filling 755 sub $16, %r10 756 # Determine if if partial block is not being filled and 757 # shift mask accordingly 758 jge .L_no_extra_mask_1_\@ 759 sub %r10, %r12 760 .L_no_extra_mask_1_\@: 761 762 vmovdqu ALL_F-SHIFT_MASK(%r12), %xmm1 763 # get the appropriate mask to mask out bottom r13 bytes of xmm9 764 vpand %xmm1, %xmm9, %xmm9 # mask out bottom r13 bytes of xmm9 765 766 vpand %xmm1, %xmm3, %xmm3 767 vmovdqa SHUF_MASK(%rip), %xmm10 768 vpshufb %xmm10, %xmm3, %xmm3 769 vpshufb %xmm2, %xmm3, %xmm3 770 vpxor %xmm3, \AAD_HASH, \AAD_HASH 771 772 test %r10, %r10 773 jl .L_partial_incomplete_1_\@ 774 775 # GHASH computation for the last <16 Byte block 776 \GHASH_MUL \AAD_HASH, %xmm13, %xmm0, %xmm10, %xmm11, %xmm5, %xmm6 777 xor %eax,%eax 778 779 mov %rax, PBlockLen(arg2) 780 jmp .L_dec_done_\@ 781 .L_partial_incomplete_1_\@: 782 add \PLAIN_CYPH_LEN, PBlockLen(arg2) 783 .L_dec_done_\@: 784 vmovdqu \AAD_HASH, AadHash(arg2) 785 .else 786 vpxor %xmm1, %xmm9, %xmm9 # Plaintext XOR E(K, Yn) 787 788 mov \PLAIN_CYPH_LEN, %r10 789 add %r13, %r10 790 # Set r10 to be the amount of data left in CYPH_PLAIN_IN after filling 791 sub $16, %r10 792 # Determine if if partial block is not being filled and 793 # shift mask accordingly 794 jge .L_no_extra_mask_2_\@ 795 sub %r10, %r12 796 .L_no_extra_mask_2_\@: 797 798 vmovdqu ALL_F-SHIFT_MASK(%r12), %xmm1 799 # get the appropriate mask to mask out bottom r13 bytes of xmm9 800 vpand %xmm1, %xmm9, %xmm9 801 802 vmovdqa SHUF_MASK(%rip), %xmm1 803 vpshufb %xmm1, %xmm9, %xmm9 804 vpshufb %xmm2, %xmm9, %xmm9 805 vpxor %xmm9, \AAD_HASH, \AAD_HASH 806 807 test %r10, %r10 808 jl .L_partial_incomplete_2_\@ 809 810 # GHASH computation for the last <16 Byte block 811 \GHASH_MUL \AAD_HASH, %xmm13, %xmm0, %xmm10, %xmm11, %xmm5, %xmm6 812 xor %eax,%eax 813 814 mov %rax, PBlockLen(arg2) 815 jmp .L_encode_done_\@ 816 .L_partial_incomplete_2_\@: 817 add \PLAIN_CYPH_LEN, PBlockLen(arg2) 818 .L_encode_done_\@: 819 vmovdqu \AAD_HASH, AadHash(arg2) 820 821 vmovdqa SHUF_MASK(%rip), %xmm10 822 # shuffle xmm9 back to output as ciphertext 823 vpshufb %xmm10, %xmm9, %xmm9 824 vpshufb %xmm2, %xmm9, %xmm9 825 .endif 826 # output encrypted Bytes 827 test %r10, %r10 828 jl .L_partial_fill_\@ 829 mov %r13, %r12 830 mov $16, %r13 831 # Set r13 to be the number of bytes to write out 832 sub %r12, %r13 833 jmp .L_count_set_\@ 834 .L_partial_fill_\@: 835 mov \PLAIN_CYPH_LEN, %r13 836 .L_count_set_\@: 837 vmovdqa %xmm9, %xmm0 838 vmovq %xmm0, %rax 839 cmp $8, %r13 840 jle .L_less_than_8_bytes_left_\@ 841 842 mov %rax, (\CYPH_PLAIN_OUT, \DATA_OFFSET, 1) 843 add $8, \DATA_OFFSET 844 psrldq $8, %xmm0 845 vmovq %xmm0, %rax 846 sub $8, %r13 847 .L_less_than_8_bytes_left_\@: 848 movb %al, (\CYPH_PLAIN_OUT, \DATA_OFFSET, 1) 849 add $1, \DATA_OFFSET 850 shr $8, %rax 851 sub $1, %r13 852 jne .L_less_than_8_bytes_left_\@ 853 .L_partial_block_done_\@: 854 .endm # PARTIAL_BLOCK 855 856 ############################################################################### 857 # GHASH_MUL MACRO to implement: Data*HashKey mod (128,127,126,121,0) 858 # Input: A and B (128-bits each, bit-reflected) 859 # Output: C = A*B*x mod poly, (i.e. >>1 ) 860 # To compute GH = GH*HashKey mod poly, give HK = HashKey<<1 mod poly as input 861 # GH = GH * HK * x mod poly which is equivalent to GH*HashKey mod poly. 862 ############################################################################### 863 .macro GHASH_MUL_AVX GH HK T1 T2 T3 T4 T5 864 865 vpshufd $0b01001110, \GH, \T2 866 vpshufd $0b01001110, \HK, \T3 867 vpxor \GH , \T2, \T2 # T2 = (a1+a0) 868 vpxor \HK , \T3, \T3 # T3 = (b1+b0) 869 870 vpclmulqdq $0x11, \HK, \GH, \T1 # T1 = a1*b1 871 vpclmulqdq $0x00, \HK, \GH, \GH # GH = a0*b0 872 vpclmulqdq $0x00, \T3, \T2, \T2 # T2 = (a1+a0)*(b1+b0) 873 vpxor \GH, \T2,\T2 874 vpxor \T1, \T2,\T2 # T2 = a0*b1+a1*b0 875 876 vpslldq $8, \T2,\T3 # shift-L T3 2 DWs 877 vpsrldq $8, \T2,\T2 # shift-R T2 2 DWs 878 vpxor \T3, \GH, \GH 879 vpxor \T2, \T1, \T1 # <T1:GH> = GH x HK 880 881 #first phase of the reduction 882 vpslld $31, \GH, \T2 # packed right shifting << 31 883 vpslld $30, \GH, \T3 # packed right shifting shift << 30 884 vpslld $25, \GH, \T4 # packed right shifting shift << 25 885 886 vpxor \T3, \T2, \T2 # xor the shifted versions 887 vpxor \T4, \T2, \T2 888 889 vpsrldq $4, \T2, \T5 # shift-R T5 1 DW 890 891 vpslldq $12, \T2, \T2 # shift-L T2 3 DWs 892 vpxor \T2, \GH, \GH # first phase of the reduction complete 893 894 #second phase of the reduction 895 896 vpsrld $1,\GH, \T2 # packed left shifting >> 1 897 vpsrld $2,\GH, \T3 # packed left shifting >> 2 898 vpsrld $7,\GH, \T4 # packed left shifting >> 7 899 vpxor \T3, \T2, \T2 # xor the shifted versions 900 vpxor \T4, \T2, \T2 901 902 vpxor \T5, \T2, \T2 903 vpxor \T2, \GH, \GH 904 vpxor \T1, \GH, \GH # the result is in GH 905 906 907 .endm 908 909 .macro PRECOMPUTE_AVX HK T1 T2 T3 T4 T5 T6 910 911 # Haskey_i_k holds XORed values of the low and high parts of the Haskey_i 912 vmovdqa \HK, \T5 913 914 vpshufd $0b01001110, \T5, \T1 915 vpxor \T5, \T1, \T1 916 vmovdqu \T1, HashKey_k(arg2) 917 918 GHASH_MUL_AVX \T5, \HK, \T1, \T3, \T4, \T6, \T2 # T5 = HashKey^2<<1 mod poly 919 vmovdqu \T5, HashKey_2(arg2) # [HashKey_2] = HashKey^2<<1 mod poly 920 vpshufd $0b01001110, \T5, \T1 921 vpxor \T5, \T1, \T1 922 vmovdqu \T1, HashKey_2_k(arg2) 923 924 GHASH_MUL_AVX \T5, \HK, \T1, \T3, \T4, \T6, \T2 # T5 = HashKey^3<<1 mod poly 925 vmovdqu \T5, HashKey_3(arg2) 926 vpshufd $0b01001110, \T5, \T1 927 vpxor \T5, \T1, \T1 928 vmovdqu \T1, HashKey_3_k(arg2) 929 930 GHASH_MUL_AVX \T5, \HK, \T1, \T3, \T4, \T6, \T2 # T5 = HashKey^4<<1 mod poly 931 vmovdqu \T5, HashKey_4(arg2) 932 vpshufd $0b01001110, \T5, \T1 933 vpxor \T5, \T1, \T1 934 vmovdqu \T1, HashKey_4_k(arg2) 935 936 GHASH_MUL_AVX \T5, \HK, \T1, \T3, \T4, \T6, \T2 # T5 = HashKey^5<<1 mod poly 937 vmovdqu \T5, HashKey_5(arg2) 938 vpshufd $0b01001110, \T5, \T1 939 vpxor \T5, \T1, \T1 940 vmovdqu \T1, HashKey_5_k(arg2) 941 942 GHASH_MUL_AVX \T5, \HK, \T1, \T3, \T4, \T6, \T2 # T5 = HashKey^6<<1 mod poly 943 vmovdqu \T5, HashKey_6(arg2) 944 vpshufd $0b01001110, \T5, \T1 945 vpxor \T5, \T1, \T1 946 vmovdqu \T1, HashKey_6_k(arg2) 947 948 GHASH_MUL_AVX \T5, \HK, \T1, \T3, \T4, \T6, \T2 # T5 = HashKey^7<<1 mod poly 949 vmovdqu \T5, HashKey_7(arg2) 950 vpshufd $0b01001110, \T5, \T1 951 vpxor \T5, \T1, \T1 952 vmovdqu \T1, HashKey_7_k(arg2) 953 954 GHASH_MUL_AVX \T5, \HK, \T1, \T3, \T4, \T6, \T2 # T5 = HashKey^8<<1 mod poly 955 vmovdqu \T5, HashKey_8(arg2) 956 vpshufd $0b01001110, \T5, \T1 957 vpxor \T5, \T1, \T1 958 vmovdqu \T1, HashKey_8_k(arg2) 959 960 .endm 961 962 ## if a = number of total plaintext bytes 963 ## b = floor(a/16) 964 ## num_initial_blocks = b mod 4# 965 ## encrypt the initial num_initial_blocks blocks and apply ghash on the ciphertext 966 ## r10, r11, r12, rax are clobbered 967 ## arg1, arg2, arg3, arg4 are used as pointers only, not modified 968 969 .macro INITIAL_BLOCKS_AVX REP num_initial_blocks T1 T2 T3 T4 T5 CTR XMM1 XMM2 XMM3 XMM4 XMM5 XMM6 XMM7 XMM8 T6 T_key ENC_DEC 970 i = (8-\num_initial_blocks) 971 setreg 972 vmovdqu AadHash(arg2), reg_i 973 974 # start AES for num_initial_blocks blocks 975 vmovdqu CurCount(arg2), \CTR 976 977 i = (9-\num_initial_blocks) 978 setreg 979 .rep \num_initial_blocks 980 vpaddd ONE(%rip), \CTR, \CTR # INCR Y0 981 vmovdqa \CTR, reg_i 982 vpshufb SHUF_MASK(%rip), reg_i, reg_i # perform a 16Byte swap 983 i = (i+1) 984 setreg 985 .endr 986 987 vmovdqa (arg1), \T_key 988 i = (9-\num_initial_blocks) 989 setreg 990 .rep \num_initial_blocks 991 vpxor \T_key, reg_i, reg_i 992 i = (i+1) 993 setreg 994 .endr 995 996 j = 1 997 setreg 998 .rep \REP 999 vmovdqa 16*j(arg1), \T_key 1000 i = (9-\num_initial_blocks) 1001 setreg 1002 .rep \num_initial_blocks 1003 vaesenc \T_key, reg_i, reg_i 1004 i = (i+1) 1005 setreg 1006 .endr 1007 1008 j = (j+1) 1009 setreg 1010 .endr 1011 1012 vmovdqa 16*j(arg1), \T_key 1013 i = (9-\num_initial_blocks) 1014 setreg 1015 .rep \num_initial_blocks 1016 vaesenclast \T_key, reg_i, reg_i 1017 i = (i+1) 1018 setreg 1019 .endr 1020 1021 i = (9-\num_initial_blocks) 1022 setreg 1023 .rep \num_initial_blocks 1024 vmovdqu (arg4, %r11), \T1 1025 vpxor \T1, reg_i, reg_i 1026 vmovdqu reg_i, (arg3 , %r11) # write back ciphertext for num_initial_blocks blocks 1027 add $16, %r11 1028 .if \ENC_DEC == DEC 1029 vmovdqa \T1, reg_i 1030 .endif 1031 vpshufb SHUF_MASK(%rip), reg_i, reg_i # prepare ciphertext for GHASH computations 1032 i = (i+1) 1033 setreg 1034 .endr 1035 1036 1037 i = (8-\num_initial_blocks) 1038 j = (9-\num_initial_blocks) 1039 setreg 1040 1041 .rep \num_initial_blocks 1042 vpxor reg_i, reg_j, reg_j 1043 GHASH_MUL_AVX reg_j, \T2, \T1, \T3, \T4, \T5, \T6 # apply GHASH on num_initial_blocks blocks 1044 i = (i+1) 1045 j = (j+1) 1046 setreg 1047 .endr 1048 # XMM8 has the combined result here 1049 1050 vmovdqa \XMM8, TMP1(%rsp) 1051 vmovdqa \XMM8, \T3 1052 1053 cmp $128, %r13 1054 jl .L_initial_blocks_done\@ # no need for precomputed constants 1055 1056 ############################################################################### 1057 # Haskey_i_k holds XORed values of the low and high parts of the Haskey_i 1058 vpaddd ONE(%rip), \CTR, \CTR # INCR Y0 1059 vmovdqa \CTR, \XMM1 1060 vpshufb SHUF_MASK(%rip), \XMM1, \XMM1 # perform a 16Byte swap 1061 1062 vpaddd ONE(%rip), \CTR, \CTR # INCR Y0 1063 vmovdqa \CTR, \XMM2 1064 vpshufb SHUF_MASK(%rip), \XMM2, \XMM2 # perform a 16Byte swap 1065 1066 vpaddd ONE(%rip), \CTR, \CTR # INCR Y0 1067 vmovdqa \CTR, \XMM3 1068 vpshufb SHUF_MASK(%rip), \XMM3, \XMM3 # perform a 16Byte swap 1069 1070 vpaddd ONE(%rip), \CTR, \CTR # INCR Y0 1071 vmovdqa \CTR, \XMM4 1072 vpshufb SHUF_MASK(%rip), \XMM4, \XMM4 # perform a 16Byte swap 1073 1074 vpaddd ONE(%rip), \CTR, \CTR # INCR Y0 1075 vmovdqa \CTR, \XMM5 1076 vpshufb SHUF_MASK(%rip), \XMM5, \XMM5 # perform a 16Byte swap 1077 1078 vpaddd ONE(%rip), \CTR, \CTR # INCR Y0 1079 vmovdqa \CTR, \XMM6 1080 vpshufb SHUF_MASK(%rip), \XMM6, \XMM6 # perform a 16Byte swap 1081 1082 vpaddd ONE(%rip), \CTR, \CTR # INCR Y0 1083 vmovdqa \CTR, \XMM7 1084 vpshufb SHUF_MASK(%rip), \XMM7, \XMM7 # perform a 16Byte swap 1085 1086 vpaddd ONE(%rip), \CTR, \CTR # INCR Y0 1087 vmovdqa \CTR, \XMM8 1088 vpshufb SHUF_MASK(%rip), \XMM8, \XMM8 # perform a 16Byte swap 1089 1090 vmovdqa (arg1), \T_key 1091 vpxor \T_key, \XMM1, \XMM1 1092 vpxor \T_key, \XMM2, \XMM2 1093 vpxor \T_key, \XMM3, \XMM3 1094 vpxor \T_key, \XMM4, \XMM4 1095 vpxor \T_key, \XMM5, \XMM5 1096 vpxor \T_key, \XMM6, \XMM6 1097 vpxor \T_key, \XMM7, \XMM7 1098 vpxor \T_key, \XMM8, \XMM8 1099 1100 i = 1 1101 setreg 1102 .rep \REP # do REP rounds 1103 vmovdqa 16*i(arg1), \T_key 1104 vaesenc \T_key, \XMM1, \XMM1 1105 vaesenc \T_key, \XMM2, \XMM2 1106 vaesenc \T_key, \XMM3, \XMM3 1107 vaesenc \T_key, \XMM4, \XMM4 1108 vaesenc \T_key, \XMM5, \XMM5 1109 vaesenc \T_key, \XMM6, \XMM6 1110 vaesenc \T_key, \XMM7, \XMM7 1111 vaesenc \T_key, \XMM8, \XMM8 1112 i = (i+1) 1113 setreg 1114 .endr 1115 1116 vmovdqa 16*i(arg1), \T_key 1117 vaesenclast \T_key, \XMM1, \XMM1 1118 vaesenclast \T_key, \XMM2, \XMM2 1119 vaesenclast \T_key, \XMM3, \XMM3 1120 vaesenclast \T_key, \XMM4, \XMM4 1121 vaesenclast \T_key, \XMM5, \XMM5 1122 vaesenclast \T_key, \XMM6, \XMM6 1123 vaesenclast \T_key, \XMM7, \XMM7 1124 vaesenclast \T_key, \XMM8, \XMM8 1125 1126 vmovdqu (arg4, %r11), \T1 1127 vpxor \T1, \XMM1, \XMM1 1128 vmovdqu \XMM1, (arg3 , %r11) 1129 .if \ENC_DEC == DEC 1130 vmovdqa \T1, \XMM1 1131 .endif 1132 1133 vmovdqu 16*1(arg4, %r11), \T1 1134 vpxor \T1, \XMM2, \XMM2 1135 vmovdqu \XMM2, 16*1(arg3 , %r11) 1136 .if \ENC_DEC == DEC 1137 vmovdqa \T1, \XMM2 1138 .endif 1139 1140 vmovdqu 16*2(arg4, %r11), \T1 1141 vpxor \T1, \XMM3, \XMM3 1142 vmovdqu \XMM3, 16*2(arg3 , %r11) 1143 .if \ENC_DEC == DEC 1144 vmovdqa \T1, \XMM3 1145 .endif 1146 1147 vmovdqu 16*3(arg4, %r11), \T1 1148 vpxor \T1, \XMM4, \XMM4 1149 vmovdqu \XMM4, 16*3(arg3 , %r11) 1150 .if \ENC_DEC == DEC 1151 vmovdqa \T1, \XMM4 1152 .endif 1153 1154 vmovdqu 16*4(arg4, %r11), \T1 1155 vpxor \T1, \XMM5, \XMM5 1156 vmovdqu \XMM5, 16*4(arg3 , %r11) 1157 .if \ENC_DEC == DEC 1158 vmovdqa \T1, \XMM5 1159 .endif 1160 1161 vmovdqu 16*5(arg4, %r11), \T1 1162 vpxor \T1, \XMM6, \XMM6 1163 vmovdqu \XMM6, 16*5(arg3 , %r11) 1164 .if \ENC_DEC == DEC 1165 vmovdqa \T1, \XMM6 1166 .endif 1167 1168 vmovdqu 16*6(arg4, %r11), \T1 1169 vpxor \T1, \XMM7, \XMM7 1170 vmovdqu \XMM7, 16*6(arg3 , %r11) 1171 .if \ENC_DEC == DEC 1172 vmovdqa \T1, \XMM7 1173 .endif 1174 1175 vmovdqu 16*7(arg4, %r11), \T1 1176 vpxor \T1, \XMM8, \XMM8 1177 vmovdqu \XMM8, 16*7(arg3 , %r11) 1178 .if \ENC_DEC == DEC 1179 vmovdqa \T1, \XMM8 1180 .endif 1181 1182 add $128, %r11 1183 1184 vpshufb SHUF_MASK(%rip), \XMM1, \XMM1 # perform a 16Byte swap 1185 vpxor TMP1(%rsp), \XMM1, \XMM1 # combine GHASHed value with the corresponding ciphertext 1186 vpshufb SHUF_MASK(%rip), \XMM2, \XMM2 # perform a 16Byte swap 1187 vpshufb SHUF_MASK(%rip), \XMM3, \XMM3 # perform a 16Byte swap 1188 vpshufb SHUF_MASK(%rip), \XMM4, \XMM4 # perform a 16Byte swap 1189 vpshufb SHUF_MASK(%rip), \XMM5, \XMM5 # perform a 16Byte swap 1190 vpshufb SHUF_MASK(%rip), \XMM6, \XMM6 # perform a 16Byte swap 1191 vpshufb SHUF_MASK(%rip), \XMM7, \XMM7 # perform a 16Byte swap 1192 vpshufb SHUF_MASK(%rip), \XMM8, \XMM8 # perform a 16Byte swap 1193 1194 ############################################################################### 1195 1196 .L_initial_blocks_done\@: 1197 1198 .endm 1199 1200 # encrypt 8 blocks at a time 1201 # ghash the 8 previously encrypted ciphertext blocks 1202 # arg1, arg2, arg3, arg4 are used as pointers only, not modified 1203 # r11 is the data offset value 1204 .macro GHASH_8_ENCRYPT_8_PARALLEL_AVX REP T1 T2 T3 T4 T5 T6 CTR XMM1 XMM2 XMM3 XMM4 XMM5 XMM6 XMM7 XMM8 T7 loop_idx ENC_DEC 1205 1206 vmovdqa \XMM1, \T2 1207 vmovdqa \XMM2, TMP2(%rsp) 1208 vmovdqa \XMM3, TMP3(%rsp) 1209 vmovdqa \XMM4, TMP4(%rsp) 1210 vmovdqa \XMM5, TMP5(%rsp) 1211 vmovdqa \XMM6, TMP6(%rsp) 1212 vmovdqa \XMM7, TMP7(%rsp) 1213 vmovdqa \XMM8, TMP8(%rsp) 1214 1215 .if \loop_idx == in_order 1216 vpaddd ONE(%rip), \CTR, \XMM1 # INCR CNT 1217 vpaddd ONE(%rip), \XMM1, \XMM2 1218 vpaddd ONE(%rip), \XMM2, \XMM3 1219 vpaddd ONE(%rip), \XMM3, \XMM4 1220 vpaddd ONE(%rip), \XMM4, \XMM5 1221 vpaddd ONE(%rip), \XMM5, \XMM6 1222 vpaddd ONE(%rip), \XMM6, \XMM7 1223 vpaddd ONE(%rip), \XMM7, \XMM8 1224 vmovdqa \XMM8, \CTR 1225 1226 vpshufb SHUF_MASK(%rip), \XMM1, \XMM1 # perform a 16Byte swap 1227 vpshufb SHUF_MASK(%rip), \XMM2, \XMM2 # perform a 16Byte swap 1228 vpshufb SHUF_MASK(%rip), \XMM3, \XMM3 # perform a 16Byte swap 1229 vpshufb SHUF_MASK(%rip), \XMM4, \XMM4 # perform a 16Byte swap 1230 vpshufb SHUF_MASK(%rip), \XMM5, \XMM5 # perform a 16Byte swap 1231 vpshufb SHUF_MASK(%rip), \XMM6, \XMM6 # perform a 16Byte swap 1232 vpshufb SHUF_MASK(%rip), \XMM7, \XMM7 # perform a 16Byte swap 1233 vpshufb SHUF_MASK(%rip), \XMM8, \XMM8 # perform a 16Byte swap 1234 .else 1235 vpaddd ONEf(%rip), \CTR, \XMM1 # INCR CNT 1236 vpaddd ONEf(%rip), \XMM1, \XMM2 1237 vpaddd ONEf(%rip), \XMM2, \XMM3 1238 vpaddd ONEf(%rip), \XMM3, \XMM4 1239 vpaddd ONEf(%rip), \XMM4, \XMM5 1240 vpaddd ONEf(%rip), \XMM5, \XMM6 1241 vpaddd ONEf(%rip), \XMM6, \XMM7 1242 vpaddd ONEf(%rip), \XMM7, \XMM8 1243 vmovdqa \XMM8, \CTR 1244 .endif 1245 1246 1247 ####################################################################### 1248 1249 vmovdqu (arg1), \T1 1250 vpxor \T1, \XMM1, \XMM1 1251 vpxor \T1, \XMM2, \XMM2 1252 vpxor \T1, \XMM3, \XMM3 1253 vpxor \T1, \XMM4, \XMM4 1254 vpxor \T1, \XMM5, \XMM5 1255 vpxor \T1, \XMM6, \XMM6 1256 vpxor \T1, \XMM7, \XMM7 1257 vpxor \T1, \XMM8, \XMM8 1258 1259 ####################################################################### 1260 1261 1262 1263 1264 1265 vmovdqu 16*1(arg1), \T1 1266 vaesenc \T1, \XMM1, \XMM1 1267 vaesenc \T1, \XMM2, \XMM2 1268 vaesenc \T1, \XMM3, \XMM3 1269 vaesenc \T1, \XMM4, \XMM4 1270 vaesenc \T1, \XMM5, \XMM5 1271 vaesenc \T1, \XMM6, \XMM6 1272 vaesenc \T1, \XMM7, \XMM7 1273 vaesenc \T1, \XMM8, \XMM8 1274 1275 vmovdqu 16*2(arg1), \T1 1276 vaesenc \T1, \XMM1, \XMM1 1277 vaesenc \T1, \XMM2, \XMM2 1278 vaesenc \T1, \XMM3, \XMM3 1279 vaesenc \T1, \XMM4, \XMM4 1280 vaesenc \T1, \XMM5, \XMM5 1281 vaesenc \T1, \XMM6, \XMM6 1282 vaesenc \T1, \XMM7, \XMM7 1283 vaesenc \T1, \XMM8, \XMM8 1284 1285 1286 ####################################################################### 1287 1288 vmovdqu HashKey_8(arg2), \T5 1289 vpclmulqdq $0x11, \T5, \T2, \T4 # T4 = a1*b1 1290 vpclmulqdq $0x00, \T5, \T2, \T7 # T7 = a0*b0 1291 1292 vpshufd $0b01001110, \T2, \T6 1293 vpxor \T2, \T6, \T6 1294 1295 vmovdqu HashKey_8_k(arg2), \T5 1296 vpclmulqdq $0x00, \T5, \T6, \T6 1297 1298 vmovdqu 16*3(arg1), \T1 1299 vaesenc \T1, \XMM1, \XMM1 1300 vaesenc \T1, \XMM2, \XMM2 1301 vaesenc \T1, \XMM3, \XMM3 1302 vaesenc \T1, \XMM4, \XMM4 1303 vaesenc \T1, \XMM5, \XMM5 1304 vaesenc \T1, \XMM6, \XMM6 1305 vaesenc \T1, \XMM7, \XMM7 1306 vaesenc \T1, \XMM8, \XMM8 1307 1308 vmovdqa TMP2(%rsp), \T1 1309 vmovdqu HashKey_7(arg2), \T5 1310 vpclmulqdq $0x11, \T5, \T1, \T3 1311 vpxor \T3, \T4, \T4 1312 vpclmulqdq $0x00, \T5, \T1, \T3 1313 vpxor \T3, \T7, \T7 1314 1315 vpshufd $0b01001110, \T1, \T3 1316 vpxor \T1, \T3, \T3 1317 vmovdqu HashKey_7_k(arg2), \T5 1318 vpclmulqdq $0x10, \T5, \T3, \T3 1319 vpxor \T3, \T6, \T6 1320 1321 vmovdqu 16*4(arg1), \T1 1322 vaesenc \T1, \XMM1, \XMM1 1323 vaesenc \T1, \XMM2, \XMM2 1324 vaesenc \T1, \XMM3, \XMM3 1325 vaesenc \T1, \XMM4, \XMM4 1326 vaesenc \T1, \XMM5, \XMM5 1327 vaesenc \T1, \XMM6, \XMM6 1328 vaesenc \T1, \XMM7, \XMM7 1329 vaesenc \T1, \XMM8, \XMM8 1330 1331 ####################################################################### 1332 1333 vmovdqa TMP3(%rsp), \T1 1334 vmovdqu HashKey_6(arg2), \T5 1335 vpclmulqdq $0x11, \T5, \T1, \T3 1336 vpxor \T3, \T4, \T4 1337 vpclmulqdq $0x00, \T5, \T1, \T3 1338 vpxor \T3, \T7, \T7 1339 1340 vpshufd $0b01001110, \T1, \T3 1341 vpxor \T1, \T3, \T3 1342 vmovdqu HashKey_6_k(arg2), \T5 1343 vpclmulqdq $0x10, \T5, \T3, \T3 1344 vpxor \T3, \T6, \T6 1345 1346 vmovdqu 16*5(arg1), \T1 1347 vaesenc \T1, \XMM1, \XMM1 1348 vaesenc \T1, \XMM2, \XMM2 1349 vaesenc \T1, \XMM3, \XMM3 1350 vaesenc \T1, \XMM4, \XMM4 1351 vaesenc \T1, \XMM5, \XMM5 1352 vaesenc \T1, \XMM6, \XMM6 1353 vaesenc \T1, \XMM7, \XMM7 1354 vaesenc \T1, \XMM8, \XMM8 1355 1356 vmovdqa TMP4(%rsp), \T1 1357 vmovdqu HashKey_5(arg2), \T5 1358 vpclmulqdq $0x11, \T5, \T1, \T3 1359 vpxor \T3, \T4, \T4 1360 vpclmulqdq $0x00, \T5, \T1, \T3 1361 vpxor \T3, \T7, \T7 1362 1363 vpshufd $0b01001110, \T1, \T3 1364 vpxor \T1, \T3, \T3 1365 vmovdqu HashKey_5_k(arg2), \T5 1366 vpclmulqdq $0x10, \T5, \T3, \T3 1367 vpxor \T3, \T6, \T6 1368 1369 vmovdqu 16*6(arg1), \T1 1370 vaesenc \T1, \XMM1, \XMM1 1371 vaesenc \T1, \XMM2, \XMM2 1372 vaesenc \T1, \XMM3, \XMM3 1373 vaesenc \T1, \XMM4, \XMM4 1374 vaesenc \T1, \XMM5, \XMM5 1375 vaesenc \T1, \XMM6, \XMM6 1376 vaesenc \T1, \XMM7, \XMM7 1377 vaesenc \T1, \XMM8, \XMM8 1378 1379 1380 vmovdqa TMP5(%rsp), \T1 1381 vmovdqu HashKey_4(arg2), \T5 1382 vpclmulqdq $0x11, \T5, \T1, \T3 1383 vpxor \T3, \T4, \T4 1384 vpclmulqdq $0x00, \T5, \T1, \T3 1385 vpxor \T3, \T7, \T7 1386 1387 vpshufd $0b01001110, \T1, \T3 1388 vpxor \T1, \T3, \T3 1389 vmovdqu HashKey_4_k(arg2), \T5 1390 vpclmulqdq $0x10, \T5, \T3, \T3 1391 vpxor \T3, \T6, \T6 1392 1393 vmovdqu 16*7(arg1), \T1 1394 vaesenc \T1, \XMM1, \XMM1 1395 vaesenc \T1, \XMM2, \XMM2 1396 vaesenc \T1, \XMM3, \XMM3 1397 vaesenc \T1, \XMM4, \XMM4 1398 vaesenc \T1, \XMM5, \XMM5 1399 vaesenc \T1, \XMM6, \XMM6 1400 vaesenc \T1, \XMM7, \XMM7 1401 vaesenc \T1, \XMM8, \XMM8 1402 1403 vmovdqa TMP6(%rsp), \T1 1404 vmovdqu HashKey_3(arg2), \T5 1405 vpclmulqdq $0x11, \T5, \T1, \T3 1406 vpxor \T3, \T4, \T4 1407 vpclmulqdq $0x00, \T5, \T1, \T3 1408 vpxor \T3, \T7, \T7 1409 1410 vpshufd $0b01001110, \T1, \T3 1411 vpxor \T1, \T3, \T3 1412 vmovdqu HashKey_3_k(arg2), \T5 1413 vpclmulqdq $0x10, \T5, \T3, \T3 1414 vpxor \T3, \T6, \T6 1415 1416 1417 vmovdqu 16*8(arg1), \T1 1418 vaesenc \T1, \XMM1, \XMM1 1419 vaesenc \T1, \XMM2, \XMM2 1420 vaesenc \T1, \XMM3, \XMM3 1421 vaesenc \T1, \XMM4, \XMM4 1422 vaesenc \T1, \XMM5, \XMM5 1423 vaesenc \T1, \XMM6, \XMM6 1424 vaesenc \T1, \XMM7, \XMM7 1425 vaesenc \T1, \XMM8, \XMM8 1426 1427 vmovdqa TMP7(%rsp), \T1 1428 vmovdqu HashKey_2(arg2), \T5 1429 vpclmulqdq $0x11, \T5, \T1, \T3 1430 vpxor \T3, \T4, \T4 1431 vpclmulqdq $0x00, \T5, \T1, \T3 1432 vpxor \T3, \T7, \T7 1433 1434 vpshufd $0b01001110, \T1, \T3 1435 vpxor \T1, \T3, \T3 1436 vmovdqu HashKey_2_k(arg2), \T5 1437 vpclmulqdq $0x10, \T5, \T3, \T3 1438 vpxor \T3, \T6, \T6 1439 1440 ####################################################################### 1441 1442 vmovdqu 16*9(arg1), \T5 1443 vaesenc \T5, \XMM1, \XMM1 1444 vaesenc \T5, \XMM2, \XMM2 1445 vaesenc \T5, \XMM3, \XMM3 1446 vaesenc \T5, \XMM4, \XMM4 1447 vaesenc \T5, \XMM5, \XMM5 1448 vaesenc \T5, \XMM6, \XMM6 1449 vaesenc \T5, \XMM7, \XMM7 1450 vaesenc \T5, \XMM8, \XMM8 1451 1452 vmovdqa TMP8(%rsp), \T1 1453 vmovdqu HashKey(arg2), \T5 1454 vpclmulqdq $0x11, \T5, \T1, \T3 1455 vpxor \T3, \T4, \T4 1456 vpclmulqdq $0x00, \T5, \T1, \T3 1457 vpxor \T3, \T7, \T7 1458 1459 vpshufd $0b01001110, \T1, \T3 1460 vpxor \T1, \T3, \T3 1461 vmovdqu HashKey_k(arg2), \T5 1462 vpclmulqdq $0x10, \T5, \T3, \T3 1463 vpxor \T3, \T6, \T6 1464 1465 vpxor \T4, \T6, \T6 1466 vpxor \T7, \T6, \T6 1467 1468 vmovdqu 16*10(arg1), \T5 1469 1470 i = 11 1471 setreg 1472 .rep (\REP-9) 1473 1474 vaesenc \T5, \XMM1, \XMM1 1475 vaesenc \T5, \XMM2, \XMM2 1476 vaesenc \T5, \XMM3, \XMM3 1477 vaesenc \T5, \XMM4, \XMM4 1478 vaesenc \T5, \XMM5, \XMM5 1479 vaesenc \T5, \XMM6, \XMM6 1480 vaesenc \T5, \XMM7, \XMM7 1481 vaesenc \T5, \XMM8, \XMM8 1482 1483 vmovdqu 16*i(arg1), \T5 1484 i = i + 1 1485 setreg 1486 .endr 1487 1488 i = 0 1489 j = 1 1490 setreg 1491 .rep 8 1492 vpxor 16*i(arg4, %r11), \T5, \T2 1493 .if \ENC_DEC == ENC 1494 vaesenclast \T2, reg_j, reg_j 1495 .else 1496 vaesenclast \T2, reg_j, \T3 1497 vmovdqu 16*i(arg4, %r11), reg_j 1498 vmovdqu \T3, 16*i(arg3, %r11) 1499 .endif 1500 i = (i+1) 1501 j = (j+1) 1502 setreg 1503 .endr 1504 ####################################################################### 1505 1506 1507 vpslldq $8, \T6, \T3 # shift-L T3 2 DWs 1508 vpsrldq $8, \T6, \T6 # shift-R T2 2 DWs 1509 vpxor \T3, \T7, \T7 1510 vpxor \T4, \T6, \T6 # accumulate the results in T6:T7 1511 1512 1513 1514 ####################################################################### 1515 #first phase of the reduction 1516 ####################################################################### 1517 vpslld $31, \T7, \T2 # packed right shifting << 31 1518 vpslld $30, \T7, \T3 # packed right shifting shift << 30 1519 vpslld $25, \T7, \T4 # packed right shifting shift << 25 1520 1521 vpxor \T3, \T2, \T2 # xor the shifted versions 1522 vpxor \T4, \T2, \T2 1523 1524 vpsrldq $4, \T2, \T1 # shift-R T1 1 DW 1525 1526 vpslldq $12, \T2, \T2 # shift-L T2 3 DWs 1527 vpxor \T2, \T7, \T7 # first phase of the reduction complete 1528 ####################################################################### 1529 .if \ENC_DEC == ENC 1530 vmovdqu \XMM1, 16*0(arg3,%r11) # Write to the Ciphertext buffer 1531 vmovdqu \XMM2, 16*1(arg3,%r11) # Write to the Ciphertext buffer 1532 vmovdqu \XMM3, 16*2(arg3,%r11) # Write to the Ciphertext buffer 1533 vmovdqu \XMM4, 16*3(arg3,%r11) # Write to the Ciphertext buffer 1534 vmovdqu \XMM5, 16*4(arg3,%r11) # Write to the Ciphertext buffer 1535 vmovdqu \XMM6, 16*5(arg3,%r11) # Write to the Ciphertext buffer 1536 vmovdqu \XMM7, 16*6(arg3,%r11) # Write to the Ciphertext buffer 1537 vmovdqu \XMM8, 16*7(arg3,%r11) # Write to the Ciphertext buffer 1538 .endif 1539 1540 ####################################################################### 1541 #second phase of the reduction 1542 vpsrld $1, \T7, \T2 # packed left shifting >> 1 1543 vpsrld $2, \T7, \T3 # packed left shifting >> 2 1544 vpsrld $7, \T7, \T4 # packed left shifting >> 7 1545 vpxor \T3, \T2, \T2 # xor the shifted versions 1546 vpxor \T4, \T2, \T2 1547 1548 vpxor \T1, \T2, \T2 1549 vpxor \T2, \T7, \T7 1550 vpxor \T7, \T6, \T6 # the result is in T6 1551 ####################################################################### 1552 1553 vpshufb SHUF_MASK(%rip), \XMM1, \XMM1 # perform a 16Byte swap 1554 vpshufb SHUF_MASK(%rip), \XMM2, \XMM2 # perform a 16Byte swap 1555 vpshufb SHUF_MASK(%rip), \XMM3, \XMM3 # perform a 16Byte swap 1556 vpshufb SHUF_MASK(%rip), \XMM4, \XMM4 # perform a 16Byte swap 1557 vpshufb SHUF_MASK(%rip), \XMM5, \XMM5 # perform a 16Byte swap 1558 vpshufb SHUF_MASK(%rip), \XMM6, \XMM6 # perform a 16Byte swap 1559 vpshufb SHUF_MASK(%rip), \XMM7, \XMM7 # perform a 16Byte swap 1560 vpshufb SHUF_MASK(%rip), \XMM8, \XMM8 # perform a 16Byte swap 1561 1562 1563 vpxor \T6, \XMM1, \XMM1 1564 1565 1566 1567 .endm 1568 1569 1570 # GHASH the last 4 ciphertext blocks. 1571 .macro GHASH_LAST_8_AVX T1 T2 T3 T4 T5 T6 T7 XMM1 XMM2 XMM3 XMM4 XMM5 XMM6 XMM7 XMM8 1572 1573 ## Karatsuba Method 1574 1575 1576 vpshufd $0b01001110, \XMM1, \T2 1577 vpxor \XMM1, \T2, \T2 1578 vmovdqu HashKey_8(arg2), \T5 1579 vpclmulqdq $0x11, \T5, \XMM1, \T6 1580 vpclmulqdq $0x00, \T5, \XMM1, \T7 1581 1582 vmovdqu HashKey_8_k(arg2), \T3 1583 vpclmulqdq $0x00, \T3, \T2, \XMM1 1584 1585 ###################### 1586 1587 vpshufd $0b01001110, \XMM2, \T2 1588 vpxor \XMM2, \T2, \T2 1589 vmovdqu HashKey_7(arg2), \T5 1590 vpclmulqdq $0x11, \T5, \XMM2, \T4 1591 vpxor \T4, \T6, \T6 1592 1593 vpclmulqdq $0x00, \T5, \XMM2, \T4 1594 vpxor \T4, \T7, \T7 1595 1596 vmovdqu HashKey_7_k(arg2), \T3 1597 vpclmulqdq $0x00, \T3, \T2, \T2 1598 vpxor \T2, \XMM1, \XMM1 1599 1600 ###################### 1601 1602 vpshufd $0b01001110, \XMM3, \T2 1603 vpxor \XMM3, \T2, \T2 1604 vmovdqu HashKey_6(arg2), \T5 1605 vpclmulqdq $0x11, \T5, \XMM3, \T4 1606 vpxor \T4, \T6, \T6 1607 1608 vpclmulqdq $0x00, \T5, \XMM3, \T4 1609 vpxor \T4, \T7, \T7 1610 1611 vmovdqu HashKey_6_k(arg2), \T3 1612 vpclmulqdq $0x00, \T3, \T2, \T2 1613 vpxor \T2, \XMM1, \XMM1 1614 1615 ###################### 1616 1617 vpshufd $0b01001110, \XMM4, \T2 1618 vpxor \XMM4, \T2, \T2 1619 vmovdqu HashKey_5(arg2), \T5 1620 vpclmulqdq $0x11, \T5, \XMM4, \T4 1621 vpxor \T4, \T6, \T6 1622 1623 vpclmulqdq $0x00, \T5, \XMM4, \T4 1624 vpxor \T4, \T7, \T7 1625 1626 vmovdqu HashKey_5_k(arg2), \T3 1627 vpclmulqdq $0x00, \T3, \T2, \T2 1628 vpxor \T2, \XMM1, \XMM1 1629 1630 ###################### 1631 1632 vpshufd $0b01001110, \XMM5, \T2 1633 vpxor \XMM5, \T2, \T2 1634 vmovdqu HashKey_4(arg2), \T5 1635 vpclmulqdq $0x11, \T5, \XMM5, \T4 1636 vpxor \T4, \T6, \T6 1637 1638 vpclmulqdq $0x00, \T5, \XMM5, \T4 1639 vpxor \T4, \T7, \T7 1640 1641 vmovdqu HashKey_4_k(arg2), \T3 1642 vpclmulqdq $0x00, \T3, \T2, \T2 1643 vpxor \T2, \XMM1, \XMM1 1644 1645 ###################### 1646 1647 vpshufd $0b01001110, \XMM6, \T2 1648 vpxor \XMM6, \T2, \T2 1649 vmovdqu HashKey_3(arg2), \T5 1650 vpclmulqdq $0x11, \T5, \XMM6, \T4 1651 vpxor \T4, \T6, \T6 1652 1653 vpclmulqdq $0x00, \T5, \XMM6, \T4 1654 vpxor \T4, \T7, \T7 1655 1656 vmovdqu HashKey_3_k(arg2), \T3 1657 vpclmulqdq $0x00, \T3, \T2, \T2 1658 vpxor \T2, \XMM1, \XMM1 1659 1660 ###################### 1661 1662 vpshufd $0b01001110, \XMM7, \T2 1663 vpxor \XMM7, \T2, \T2 1664 vmovdqu HashKey_2(arg2), \T5 1665 vpclmulqdq $0x11, \T5, \XMM7, \T4 1666 vpxor \T4, \T6, \T6 1667 1668 vpclmulqdq $0x00, \T5, \XMM7, \T4 1669 vpxor \T4, \T7, \T7 1670 1671 vmovdqu HashKey_2_k(arg2), \T3 1672 vpclmulqdq $0x00, \T3, \T2, \T2 1673 vpxor \T2, \XMM1, \XMM1 1674 1675 ###################### 1676 1677 vpshufd $0b01001110, \XMM8, \T2 1678 vpxor \XMM8, \T2, \T2 1679 vmovdqu HashKey(arg2), \T5 1680 vpclmulqdq $0x11, \T5, \XMM8, \T4 1681 vpxor \T4, \T6, \T6 1682 1683 vpclmulqdq $0x00, \T5, \XMM8, \T4 1684 vpxor \T4, \T7, \T7 1685 1686 vmovdqu HashKey_k(arg2), \T3 1687 vpclmulqdq $0x00, \T3, \T2, \T2 1688 1689 vpxor \T2, \XMM1, \XMM1 1690 vpxor \T6, \XMM1, \XMM1 1691 vpxor \T7, \XMM1, \T2 1692 1693 1694 1695 1696 vpslldq $8, \T2, \T4 1697 vpsrldq $8, \T2, \T2 1698 1699 vpxor \T4, \T7, \T7 1700 vpxor \T2, \T6, \T6 # <T6:T7> holds the result of 1701 # the accumulated carry-less multiplications 1702 1703 ####################################################################### 1704 #first phase of the reduction 1705 vpslld $31, \T7, \T2 # packed right shifting << 31 1706 vpslld $30, \T7, \T3 # packed right shifting shift << 30 1707 vpslld $25, \T7, \T4 # packed right shifting shift << 25 1708 1709 vpxor \T3, \T2, \T2 # xor the shifted versions 1710 vpxor \T4, \T2, \T2 1711 1712 vpsrldq $4, \T2, \T1 # shift-R T1 1 DW 1713 1714 vpslldq $12, \T2, \T2 # shift-L T2 3 DWs 1715 vpxor \T2, \T7, \T7 # first phase of the reduction complete 1716 ####################################################################### 1717 1718 1719 #second phase of the reduction 1720 vpsrld $1, \T7, \T2 # packed left shifting >> 1 1721 vpsrld $2, \T7, \T3 # packed left shifting >> 2 1722 vpsrld $7, \T7, \T4 # packed left shifting >> 7 1723 vpxor \T3, \T2, \T2 # xor the shifted versions 1724 vpxor \T4, \T2, \T2 1725 1726 vpxor \T1, \T2, \T2 1727 vpxor \T2, \T7, \T7 1728 vpxor \T7, \T6, \T6 # the result is in T6 1729 1730 .endm 1731 1732 ############################################################# 1733 #void aesni_gcm_precomp_avx_gen2 1734 # (gcm_data *my_ctx_data, 1735 # gcm_context_data *data, 1736 # u8 *hash_subkey# /* H, the Hash sub key input. Data starts on a 16-byte boundary. */ 1737 # u8 *iv, /* Pre-counter block j0: 4 byte salt 1738 # (from Security Association) concatenated with 8 byte 1739 # Initialisation Vector (from IPSec ESP Payload) 1740 # concatenated with 0x00000001. 16-byte aligned pointer. */ 1741 # const u8 *aad, /* Additional Authentication Data (AAD)*/ 1742 # u64 aad_len) /* Length of AAD in bytes. With RFC4106 this is going to be 8 or 12 Bytes */ 1743 ############################################################# 1744 SYM_FUNC_START(aesni_gcm_init_avx_gen2) 1745 FUNC_SAVE 1746 INIT GHASH_MUL_AVX, PRECOMPUTE_AVX 1747 FUNC_RESTORE 1748 RET 1749 SYM_FUNC_END(aesni_gcm_init_avx_gen2) 1750 1751 ############################################################################### 1752 #void aesni_gcm_enc_update_avx_gen2( 1753 # gcm_data *my_ctx_data, /* aligned to 16 Bytes */ 1754 # gcm_context_data *data, 1755 # u8 *out, /* Ciphertext output. Encrypt in-place is allowed. */ 1756 # const u8 *in, /* Plaintext input */ 1757 # u64 plaintext_len) /* Length of data in Bytes for encryption. */ 1758 ############################################################################### 1759 SYM_FUNC_START(aesni_gcm_enc_update_avx_gen2) 1760 FUNC_SAVE 1761 mov keysize, %eax 1762 cmp $32, %eax 1763 je key_256_enc_update 1764 cmp $16, %eax 1765 je key_128_enc_update 1766 # must be 192 1767 GCM_ENC_DEC INITIAL_BLOCKS_AVX, GHASH_8_ENCRYPT_8_PARALLEL_AVX, GHASH_LAST_8_AVX, GHASH_MUL_AVX, ENC, 11 1768 FUNC_RESTORE 1769 RET 1770 key_128_enc_update: 1771 GCM_ENC_DEC INITIAL_BLOCKS_AVX, GHASH_8_ENCRYPT_8_PARALLEL_AVX, GHASH_LAST_8_AVX, GHASH_MUL_AVX, ENC, 9 1772 FUNC_RESTORE 1773 RET 1774 key_256_enc_update: 1775 GCM_ENC_DEC INITIAL_BLOCKS_AVX, GHASH_8_ENCRYPT_8_PARALLEL_AVX, GHASH_LAST_8_AVX, GHASH_MUL_AVX, ENC, 13 1776 FUNC_RESTORE 1777 RET 1778 SYM_FUNC_END(aesni_gcm_enc_update_avx_gen2) 1779 1780 ############################################################################### 1781 #void aesni_gcm_dec_update_avx_gen2( 1782 # gcm_data *my_ctx_data, /* aligned to 16 Bytes */ 1783 # gcm_context_data *data, 1784 # u8 *out, /* Plaintext output. Decrypt in-place is allowed. */ 1785 # const u8 *in, /* Ciphertext input */ 1786 # u64 plaintext_len) /* Length of data in Bytes for encryption. */ 1787 ############################################################################### 1788 SYM_FUNC_START(aesni_gcm_dec_update_avx_gen2) 1789 FUNC_SAVE 1790 mov keysize,%eax 1791 cmp $32, %eax 1792 je key_256_dec_update 1793 cmp $16, %eax 1794 je key_128_dec_update 1795 # must be 192 1796 GCM_ENC_DEC INITIAL_BLOCKS_AVX, GHASH_8_ENCRYPT_8_PARALLEL_AVX, GHASH_LAST_8_AVX, GHASH_MUL_AVX, DEC, 11 1797 FUNC_RESTORE 1798 RET 1799 key_128_dec_update: 1800 GCM_ENC_DEC INITIAL_BLOCKS_AVX, GHASH_8_ENCRYPT_8_PARALLEL_AVX, GHASH_LAST_8_AVX, GHASH_MUL_AVX, DEC, 9 1801 FUNC_RESTORE 1802 RET 1803 key_256_dec_update: 1804 GCM_ENC_DEC INITIAL_BLOCKS_AVX, GHASH_8_ENCRYPT_8_PARALLEL_AVX, GHASH_LAST_8_AVX, GHASH_MUL_AVX, DEC, 13 1805 FUNC_RESTORE 1806 RET 1807 SYM_FUNC_END(aesni_gcm_dec_update_avx_gen2) 1808 1809 ############################################################################### 1810 #void aesni_gcm_finalize_avx_gen2( 1811 # gcm_data *my_ctx_data, /* aligned to 16 Bytes */ 1812 # gcm_context_data *data, 1813 # u8 *auth_tag, /* Authenticated Tag output. */ 1814 # u64 auth_tag_len)# /* Authenticated Tag Length in bytes. 1815 # Valid values are 16 (most likely), 12 or 8. */ 1816 ############################################################################### 1817 SYM_FUNC_START(aesni_gcm_finalize_avx_gen2) 1818 FUNC_SAVE 1819 mov keysize,%eax 1820 cmp $32, %eax 1821 je key_256_finalize 1822 cmp $16, %eax 1823 je key_128_finalize 1824 # must be 192 1825 GCM_COMPLETE GHASH_MUL_AVX, 11, arg3, arg4 1826 FUNC_RESTORE 1827 RET 1828 key_128_finalize: 1829 GCM_COMPLETE GHASH_MUL_AVX, 9, arg3, arg4 1830 FUNC_RESTORE 1831 RET 1832 key_256_finalize: 1833 GCM_COMPLETE GHASH_MUL_AVX, 13, arg3, arg4 1834 FUNC_RESTORE 1835 RET 1836 SYM_FUNC_END(aesni_gcm_finalize_avx_gen2) 1837 1838 ############################################################################### 1839 # GHASH_MUL MACRO to implement: Data*HashKey mod (128,127,126,121,0) 1840 # Input: A and B (128-bits each, bit-reflected) 1841 # Output: C = A*B*x mod poly, (i.e. >>1 ) 1842 # To compute GH = GH*HashKey mod poly, give HK = HashKey<<1 mod poly as input 1843 # GH = GH * HK * x mod poly which is equivalent to GH*HashKey mod poly. 1844 ############################################################################### 1845 .macro GHASH_MUL_AVX2 GH HK T1 T2 T3 T4 T5 1846 1847 vpclmulqdq $0x11,\HK,\GH,\T1 # T1 = a1*b1 1848 vpclmulqdq $0x00,\HK,\GH,\T2 # T2 = a0*b0 1849 vpclmulqdq $0x01,\HK,\GH,\T3 # T3 = a1*b0 1850 vpclmulqdq $0x10,\HK,\GH,\GH # GH = a0*b1 1851 vpxor \T3, \GH, \GH 1852 1853 1854 vpsrldq $8 , \GH, \T3 # shift-R GH 2 DWs 1855 vpslldq $8 , \GH, \GH # shift-L GH 2 DWs 1856 1857 vpxor \T3, \T1, \T1 1858 vpxor \T2, \GH, \GH 1859 1860 ####################################################################### 1861 #first phase of the reduction 1862 vmovdqa POLY2(%rip), \T3 1863 1864 vpclmulqdq $0x01, \GH, \T3, \T2 1865 vpslldq $8, \T2, \T2 # shift-L T2 2 DWs 1866 1867 vpxor \T2, \GH, \GH # first phase of the reduction complete 1868 ####################################################################### 1869 #second phase of the reduction 1870 vpclmulqdq $0x00, \GH, \T3, \T2 1871 vpsrldq $4, \T2, \T2 # shift-R T2 1 DW (Shift-R only 1-DW to obtain 2-DWs shift-R) 1872 1873 vpclmulqdq $0x10, \GH, \T3, \GH 1874 vpslldq $4, \GH, \GH # shift-L GH 1 DW (Shift-L 1-DW to obtain result with no shifts) 1875 1876 vpxor \T2, \GH, \GH # second phase of the reduction complete 1877 ####################################################################### 1878 vpxor \T1, \GH, \GH # the result is in GH 1879 1880 1881 .endm 1882 1883 .macro PRECOMPUTE_AVX2 HK T1 T2 T3 T4 T5 T6 1884 1885 # Haskey_i_k holds XORed values of the low and high parts of the Haskey_i 1886 vmovdqa \HK, \T5 1887 GHASH_MUL_AVX2 \T5, \HK, \T1, \T3, \T4, \T6, \T2 # T5 = HashKey^2<<1 mod poly 1888 vmovdqu \T5, HashKey_2(arg2) # [HashKey_2] = HashKey^2<<1 mod poly 1889 1890 GHASH_MUL_AVX2 \T5, \HK, \T1, \T3, \T4, \T6, \T2 # T5 = HashKey^3<<1 mod poly 1891 vmovdqu \T5, HashKey_3(arg2) 1892 1893 GHASH_MUL_AVX2 \T5, \HK, \T1, \T3, \T4, \T6, \T2 # T5 = HashKey^4<<1 mod poly 1894 vmovdqu \T5, HashKey_4(arg2) 1895 1896 GHASH_MUL_AVX2 \T5, \HK, \T1, \T3, \T4, \T6, \T2 # T5 = HashKey^5<<1 mod poly 1897 vmovdqu \T5, HashKey_5(arg2) 1898 1899 GHASH_MUL_AVX2 \T5, \HK, \T1, \T3, \T4, \T6, \T2 # T5 = HashKey^6<<1 mod poly 1900 vmovdqu \T5, HashKey_6(arg2) 1901 1902 GHASH_MUL_AVX2 \T5, \HK, \T1, \T3, \T4, \T6, \T2 # T5 = HashKey^7<<1 mod poly 1903 vmovdqu \T5, HashKey_7(arg2) 1904 1905 GHASH_MUL_AVX2 \T5, \HK, \T1, \T3, \T4, \T6, \T2 # T5 = HashKey^8<<1 mod poly 1906 vmovdqu \T5, HashKey_8(arg2) 1907 1908 .endm 1909 1910 ## if a = number of total plaintext bytes 1911 ## b = floor(a/16) 1912 ## num_initial_blocks = b mod 4# 1913 ## encrypt the initial num_initial_blocks blocks and apply ghash on the ciphertext 1914 ## r10, r11, r12, rax are clobbered 1915 ## arg1, arg2, arg3, arg4 are used as pointers only, not modified 1916 1917 .macro INITIAL_BLOCKS_AVX2 REP num_initial_blocks T1 T2 T3 T4 T5 CTR XMM1 XMM2 XMM3 XMM4 XMM5 XMM6 XMM7 XMM8 T6 T_key ENC_DEC VER 1918 i = (8-\num_initial_blocks) 1919 setreg 1920 vmovdqu AadHash(arg2), reg_i 1921 1922 # start AES for num_initial_blocks blocks 1923 vmovdqu CurCount(arg2), \CTR 1924 1925 i = (9-\num_initial_blocks) 1926 setreg 1927 .rep \num_initial_blocks 1928 vpaddd ONE(%rip), \CTR, \CTR # INCR Y0 1929 vmovdqa \CTR, reg_i 1930 vpshufb SHUF_MASK(%rip), reg_i, reg_i # perform a 16Byte swap 1931 i = (i+1) 1932 setreg 1933 .endr 1934 1935 vmovdqa (arg1), \T_key 1936 i = (9-\num_initial_blocks) 1937 setreg 1938 .rep \num_initial_blocks 1939 vpxor \T_key, reg_i, reg_i 1940 i = (i+1) 1941 setreg 1942 .endr 1943 1944 j = 1 1945 setreg 1946 .rep \REP 1947 vmovdqa 16*j(arg1), \T_key 1948 i = (9-\num_initial_blocks) 1949 setreg 1950 .rep \num_initial_blocks 1951 vaesenc \T_key, reg_i, reg_i 1952 i = (i+1) 1953 setreg 1954 .endr 1955 1956 j = (j+1) 1957 setreg 1958 .endr 1959 1960 1961 vmovdqa 16*j(arg1), \T_key 1962 i = (9-\num_initial_blocks) 1963 setreg 1964 .rep \num_initial_blocks 1965 vaesenclast \T_key, reg_i, reg_i 1966 i = (i+1) 1967 setreg 1968 .endr 1969 1970 i = (9-\num_initial_blocks) 1971 setreg 1972 .rep \num_initial_blocks 1973 vmovdqu (arg4, %r11), \T1 1974 vpxor \T1, reg_i, reg_i 1975 vmovdqu reg_i, (arg3 , %r11) # write back ciphertext for 1976 # num_initial_blocks blocks 1977 add $16, %r11 1978 .if \ENC_DEC == DEC 1979 vmovdqa \T1, reg_i 1980 .endif 1981 vpshufb SHUF_MASK(%rip), reg_i, reg_i # prepare ciphertext for GHASH computations 1982 i = (i+1) 1983 setreg 1984 .endr 1985 1986 1987 i = (8-\num_initial_blocks) 1988 j = (9-\num_initial_blocks) 1989 setreg 1990 1991 .rep \num_initial_blocks 1992 vpxor reg_i, reg_j, reg_j 1993 GHASH_MUL_AVX2 reg_j, \T2, \T1, \T3, \T4, \T5, \T6 # apply GHASH on num_initial_blocks blocks 1994 i = (i+1) 1995 j = (j+1) 1996 setreg 1997 .endr 1998 # XMM8 has the combined result here 1999 2000 vmovdqa \XMM8, TMP1(%rsp) 2001 vmovdqa \XMM8, \T3 2002 2003 cmp $128, %r13 2004 jl .L_initial_blocks_done\@ # no need for precomputed constants 2005 2006 ############################################################################### 2007 # Haskey_i_k holds XORed values of the low and high parts of the Haskey_i 2008 vpaddd ONE(%rip), \CTR, \CTR # INCR Y0 2009 vmovdqa \CTR, \XMM1 2010 vpshufb SHUF_MASK(%rip), \XMM1, \XMM1 # perform a 16Byte swap 2011 2012 vpaddd ONE(%rip), \CTR, \CTR # INCR Y0 2013 vmovdqa \CTR, \XMM2 2014 vpshufb SHUF_MASK(%rip), \XMM2, \XMM2 # perform a 16Byte swap 2015 2016 vpaddd ONE(%rip), \CTR, \CTR # INCR Y0 2017 vmovdqa \CTR, \XMM3 2018 vpshufb SHUF_MASK(%rip), \XMM3, \XMM3 # perform a 16Byte swap 2019 2020 vpaddd ONE(%rip), \CTR, \CTR # INCR Y0 2021 vmovdqa \CTR, \XMM4 2022 vpshufb SHUF_MASK(%rip), \XMM4, \XMM4 # perform a 16Byte swap 2023 2024 vpaddd ONE(%rip), \CTR, \CTR # INCR Y0 2025 vmovdqa \CTR, \XMM5 2026 vpshufb SHUF_MASK(%rip), \XMM5, \XMM5 # perform a 16Byte swap 2027 2028 vpaddd ONE(%rip), \CTR, \CTR # INCR Y0 2029 vmovdqa \CTR, \XMM6 2030 vpshufb SHUF_MASK(%rip), \XMM6, \XMM6 # perform a 16Byte swap 2031 2032 vpaddd ONE(%rip), \CTR, \CTR # INCR Y0 2033 vmovdqa \CTR, \XMM7 2034 vpshufb SHUF_MASK(%rip), \XMM7, \XMM7 # perform a 16Byte swap 2035 2036 vpaddd ONE(%rip), \CTR, \CTR # INCR Y0 2037 vmovdqa \CTR, \XMM8 2038 vpshufb SHUF_MASK(%rip), \XMM8, \XMM8 # perform a 16Byte swap 2039 2040 vmovdqa (arg1), \T_key 2041 vpxor \T_key, \XMM1, \XMM1 2042 vpxor \T_key, \XMM2, \XMM2 2043 vpxor \T_key, \XMM3, \XMM3 2044 vpxor \T_key, \XMM4, \XMM4 2045 vpxor \T_key, \XMM5, \XMM5 2046 vpxor \T_key, \XMM6, \XMM6 2047 vpxor \T_key, \XMM7, \XMM7 2048 vpxor \T_key, \XMM8, \XMM8 2049 2050 i = 1 2051 setreg 2052 .rep \REP # do REP rounds 2053 vmovdqa 16*i(arg1), \T_key 2054 vaesenc \T_key, \XMM1, \XMM1 2055 vaesenc \T_key, \XMM2, \XMM2 2056 vaesenc \T_key, \XMM3, \XMM3 2057 vaesenc \T_key, \XMM4, \XMM4 2058 vaesenc \T_key, \XMM5, \XMM5 2059 vaesenc \T_key, \XMM6, \XMM6 2060 vaesenc \T_key, \XMM7, \XMM7 2061 vaesenc \T_key, \XMM8, \XMM8 2062 i = (i+1) 2063 setreg 2064 .endr 2065 2066 2067 vmovdqa 16*i(arg1), \T_key 2068 vaesenclast \T_key, \XMM1, \XMM1 2069 vaesenclast \T_key, \XMM2, \XMM2 2070 vaesenclast \T_key, \XMM3, \XMM3 2071 vaesenclast \T_key, \XMM4, \XMM4 2072 vaesenclast \T_key, \XMM5, \XMM5 2073 vaesenclast \T_key, \XMM6, \XMM6 2074 vaesenclast \T_key, \XMM7, \XMM7 2075 vaesenclast \T_key, \XMM8, \XMM8 2076 2077 vmovdqu (arg4, %r11), \T1 2078 vpxor \T1, \XMM1, \XMM1 2079 vmovdqu \XMM1, (arg3 , %r11) 2080 .if \ENC_DEC == DEC 2081 vmovdqa \T1, \XMM1 2082 .endif 2083 2084 vmovdqu 16*1(arg4, %r11), \T1 2085 vpxor \T1, \XMM2, \XMM2 2086 vmovdqu \XMM2, 16*1(arg3 , %r11) 2087 .if \ENC_DEC == DEC 2088 vmovdqa \T1, \XMM2 2089 .endif 2090 2091 vmovdqu 16*2(arg4, %r11), \T1 2092 vpxor \T1, \XMM3, \XMM3 2093 vmovdqu \XMM3, 16*2(arg3 , %r11) 2094 .if \ENC_DEC == DEC 2095 vmovdqa \T1, \XMM3 2096 .endif 2097 2098 vmovdqu 16*3(arg4, %r11), \T1 2099 vpxor \T1, \XMM4, \XMM4 2100 vmovdqu \XMM4, 16*3(arg3 , %r11) 2101 .if \ENC_DEC == DEC 2102 vmovdqa \T1, \XMM4 2103 .endif 2104 2105 vmovdqu 16*4(arg4, %r11), \T1 2106 vpxor \T1, \XMM5, \XMM5 2107 vmovdqu \XMM5, 16*4(arg3 , %r11) 2108 .if \ENC_DEC == DEC 2109 vmovdqa \T1, \XMM5 2110 .endif 2111 2112 vmovdqu 16*5(arg4, %r11), \T1 2113 vpxor \T1, \XMM6, \XMM6 2114 vmovdqu \XMM6, 16*5(arg3 , %r11) 2115 .if \ENC_DEC == DEC 2116 vmovdqa \T1, \XMM6 2117 .endif 2118 2119 vmovdqu 16*6(arg4, %r11), \T1 2120 vpxor \T1, \XMM7, \XMM7 2121 vmovdqu \XMM7, 16*6(arg3 , %r11) 2122 .if \ENC_DEC == DEC 2123 vmovdqa \T1, \XMM7 2124 .endif 2125 2126 vmovdqu 16*7(arg4, %r11), \T1 2127 vpxor \T1, \XMM8, \XMM8 2128 vmovdqu \XMM8, 16*7(arg3 , %r11) 2129 .if \ENC_DEC == DEC 2130 vmovdqa \T1, \XMM8 2131 .endif 2132 2133 add $128, %r11 2134 2135 vpshufb SHUF_MASK(%rip), \XMM1, \XMM1 # perform a 16Byte swap 2136 vpxor TMP1(%rsp), \XMM1, \XMM1 # combine GHASHed value with 2137 # the corresponding ciphertext 2138 vpshufb SHUF_MASK(%rip), \XMM2, \XMM2 # perform a 16Byte swap 2139 vpshufb SHUF_MASK(%rip), \XMM3, \XMM3 # perform a 16Byte swap 2140 vpshufb SHUF_MASK(%rip), \XMM4, \XMM4 # perform a 16Byte swap 2141 vpshufb SHUF_MASK(%rip), \XMM5, \XMM5 # perform a 16Byte swap 2142 vpshufb SHUF_MASK(%rip), \XMM6, \XMM6 # perform a 16Byte swap 2143 vpshufb SHUF_MASK(%rip), \XMM7, \XMM7 # perform a 16Byte swap 2144 vpshufb SHUF_MASK(%rip), \XMM8, \XMM8 # perform a 16Byte swap 2145 2146 ############################################################################### 2147 2148 .L_initial_blocks_done\@: 2149 2150 2151 .endm 2152 2153 2154 2155 # encrypt 8 blocks at a time 2156 # ghash the 8 previously encrypted ciphertext blocks 2157 # arg1, arg2, arg3, arg4 are used as pointers only, not modified 2158 # r11 is the data offset value 2159 .macro GHASH_8_ENCRYPT_8_PARALLEL_AVX2 REP T1 T2 T3 T4 T5 T6 CTR XMM1 XMM2 XMM3 XMM4 XMM5 XMM6 XMM7 XMM8 T7 loop_idx ENC_DEC 2160 2161 vmovdqa \XMM1, \T2 2162 vmovdqa \XMM2, TMP2(%rsp) 2163 vmovdqa \XMM3, TMP3(%rsp) 2164 vmovdqa \XMM4, TMP4(%rsp) 2165 vmovdqa \XMM5, TMP5(%rsp) 2166 vmovdqa \XMM6, TMP6(%rsp) 2167 vmovdqa \XMM7, TMP7(%rsp) 2168 vmovdqa \XMM8, TMP8(%rsp) 2169 2170 .if \loop_idx == in_order 2171 vpaddd ONE(%rip), \CTR, \XMM1 # INCR CNT 2172 vpaddd ONE(%rip), \XMM1, \XMM2 2173 vpaddd ONE(%rip), \XMM2, \XMM3 2174 vpaddd ONE(%rip), \XMM3, \XMM4 2175 vpaddd ONE(%rip), \XMM4, \XMM5 2176 vpaddd ONE(%rip), \XMM5, \XMM6 2177 vpaddd ONE(%rip), \XMM6, \XMM7 2178 vpaddd ONE(%rip), \XMM7, \XMM8 2179 vmovdqa \XMM8, \CTR 2180 2181 vpshufb SHUF_MASK(%rip), \XMM1, \XMM1 # perform a 16Byte swap 2182 vpshufb SHUF_MASK(%rip), \XMM2, \XMM2 # perform a 16Byte swap 2183 vpshufb SHUF_MASK(%rip), \XMM3, \XMM3 # perform a 16Byte swap 2184 vpshufb SHUF_MASK(%rip), \XMM4, \XMM4 # perform a 16Byte swap 2185 vpshufb SHUF_MASK(%rip), \XMM5, \XMM5 # perform a 16Byte swap 2186 vpshufb SHUF_MASK(%rip), \XMM6, \XMM6 # perform a 16Byte swap 2187 vpshufb SHUF_MASK(%rip), \XMM7, \XMM7 # perform a 16Byte swap 2188 vpshufb SHUF_MASK(%rip), \XMM8, \XMM8 # perform a 16Byte swap 2189 .else 2190 vpaddd ONEf(%rip), \CTR, \XMM1 # INCR CNT 2191 vpaddd ONEf(%rip), \XMM1, \XMM2 2192 vpaddd ONEf(%rip), \XMM2, \XMM3 2193 vpaddd ONEf(%rip), \XMM3, \XMM4 2194 vpaddd ONEf(%rip), \XMM4, \XMM5 2195 vpaddd ONEf(%rip), \XMM5, \XMM6 2196 vpaddd ONEf(%rip), \XMM6, \XMM7 2197 vpaddd ONEf(%rip), \XMM7, \XMM8 2198 vmovdqa \XMM8, \CTR 2199 .endif 2200 2201 2202 ####################################################################### 2203 2204 vmovdqu (arg1), \T1 2205 vpxor \T1, \XMM1, \XMM1 2206 vpxor \T1, \XMM2, \XMM2 2207 vpxor \T1, \XMM3, \XMM3 2208 vpxor \T1, \XMM4, \XMM4 2209 vpxor \T1, \XMM5, \XMM5 2210 vpxor \T1, \XMM6, \XMM6 2211 vpxor \T1, \XMM7, \XMM7 2212 vpxor \T1, \XMM8, \XMM8 2213 2214 ####################################################################### 2215 2216 2217 2218 2219 2220 vmovdqu 16*1(arg1), \T1 2221 vaesenc \T1, \XMM1, \XMM1 2222 vaesenc \T1, \XMM2, \XMM2 2223 vaesenc \T1, \XMM3, \XMM3 2224 vaesenc \T1, \XMM4, \XMM4 2225 vaesenc \T1, \XMM5, \XMM5 2226 vaesenc \T1, \XMM6, \XMM6 2227 vaesenc \T1, \XMM7, \XMM7 2228 vaesenc \T1, \XMM8, \XMM8 2229 2230 vmovdqu 16*2(arg1), \T1 2231 vaesenc \T1, \XMM1, \XMM1 2232 vaesenc \T1, \XMM2, \XMM2 2233 vaesenc \T1, \XMM3, \XMM3 2234 vaesenc \T1, \XMM4, \XMM4 2235 vaesenc \T1, \XMM5, \XMM5 2236 vaesenc \T1, \XMM6, \XMM6 2237 vaesenc \T1, \XMM7, \XMM7 2238 vaesenc \T1, \XMM8, \XMM8 2239 2240 2241 ####################################################################### 2242 2243 vmovdqu HashKey_8(arg2), \T5 2244 vpclmulqdq $0x11, \T5, \T2, \T4 # T4 = a1*b1 2245 vpclmulqdq $0x00, \T5, \T2, \T7 # T7 = a0*b0 2246 vpclmulqdq $0x01, \T5, \T2, \T6 # T6 = a1*b0 2247 vpclmulqdq $0x10, \T5, \T2, \T5 # T5 = a0*b1 2248 vpxor \T5, \T6, \T6 2249 2250 vmovdqu 16*3(arg1), \T1 2251 vaesenc \T1, \XMM1, \XMM1 2252 vaesenc \T1, \XMM2, \XMM2 2253 vaesenc \T1, \XMM3, \XMM3 2254 vaesenc \T1, \XMM4, \XMM4 2255 vaesenc \T1, \XMM5, \XMM5 2256 vaesenc \T1, \XMM6, \XMM6 2257 vaesenc \T1, \XMM7, \XMM7 2258 vaesenc \T1, \XMM8, \XMM8 2259 2260 vmovdqa TMP2(%rsp), \T1 2261 vmovdqu HashKey_7(arg2), \T5 2262 vpclmulqdq $0x11, \T5, \T1, \T3 2263 vpxor \T3, \T4, \T4 2264 2265 vpclmulqdq $0x00, \T5, \T1, \T3 2266 vpxor \T3, \T7, \T7 2267 2268 vpclmulqdq $0x01, \T5, \T1, \T3 2269 vpxor \T3, \T6, \T6 2270 2271 vpclmulqdq $0x10, \T5, \T1, \T3 2272 vpxor \T3, \T6, \T6 2273 2274 vmovdqu 16*4(arg1), \T1 2275 vaesenc \T1, \XMM1, \XMM1 2276 vaesenc \T1, \XMM2, \XMM2 2277 vaesenc \T1, \XMM3, \XMM3 2278 vaesenc \T1, \XMM4, \XMM4 2279 vaesenc \T1, \XMM5, \XMM5 2280 vaesenc \T1, \XMM6, \XMM6 2281 vaesenc \T1, \XMM7, \XMM7 2282 vaesenc \T1, \XMM8, \XMM8 2283 2284 ####################################################################### 2285 2286 vmovdqa TMP3(%rsp), \T1 2287 vmovdqu HashKey_6(arg2), \T5 2288 vpclmulqdq $0x11, \T5, \T1, \T3 2289 vpxor \T3, \T4, \T4 2290 2291 vpclmulqdq $0x00, \T5, \T1, \T3 2292 vpxor \T3, \T7, \T7 2293 2294 vpclmulqdq $0x01, \T5, \T1, \T3 2295 vpxor \T3, \T6, \T6 2296 2297 vpclmulqdq $0x10, \T5, \T1, \T3 2298 vpxor \T3, \T6, \T6 2299 2300 vmovdqu 16*5(arg1), \T1 2301 vaesenc \T1, \XMM1, \XMM1 2302 vaesenc \T1, \XMM2, \XMM2 2303 vaesenc \T1, \XMM3, \XMM3 2304 vaesenc \T1, \XMM4, \XMM4 2305 vaesenc \T1, \XMM5, \XMM5 2306 vaesenc \T1, \XMM6, \XMM6 2307 vaesenc \T1, \XMM7, \XMM7 2308 vaesenc \T1, \XMM8, \XMM8 2309 2310 vmovdqa TMP4(%rsp), \T1 2311 vmovdqu HashKey_5(arg2), \T5 2312 vpclmulqdq $0x11, \T5, \T1, \T3 2313 vpxor \T3, \T4, \T4 2314 2315 vpclmulqdq $0x00, \T5, \T1, \T3 2316 vpxor \T3, \T7, \T7 2317 2318 vpclmulqdq $0x01, \T5, \T1, \T3 2319 vpxor \T3, \T6, \T6 2320 2321 vpclmulqdq $0x10, \T5, \T1, \T3 2322 vpxor \T3, \T6, \T6 2323 2324 vmovdqu 16*6(arg1), \T1 2325 vaesenc \T1, \XMM1, \XMM1 2326 vaesenc \T1, \XMM2, \XMM2 2327 vaesenc \T1, \XMM3, \XMM3 2328 vaesenc \T1, \XMM4, \XMM4 2329 vaesenc \T1, \XMM5, \XMM5 2330 vaesenc \T1, \XMM6, \XMM6 2331 vaesenc \T1, \XMM7, \XMM7 2332 vaesenc \T1, \XMM8, \XMM8 2333 2334 2335 vmovdqa TMP5(%rsp), \T1 2336 vmovdqu HashKey_4(arg2), \T5 2337 vpclmulqdq $0x11, \T5, \T1, \T3 2338 vpxor \T3, \T4, \T4 2339 2340 vpclmulqdq $0x00, \T5, \T1, \T3 2341 vpxor \T3, \T7, \T7 2342 2343 vpclmulqdq $0x01, \T5, \T1, \T3 2344 vpxor \T3, \T6, \T6 2345 2346 vpclmulqdq $0x10, \T5, \T1, \T3 2347 vpxor \T3, \T6, \T6 2348 2349 vmovdqu 16*7(arg1), \T1 2350 vaesenc \T1, \XMM1, \XMM1 2351 vaesenc \T1, \XMM2, \XMM2 2352 vaesenc \T1, \XMM3, \XMM3 2353 vaesenc \T1, \XMM4, \XMM4 2354 vaesenc \T1, \XMM5, \XMM5 2355 vaesenc \T1, \XMM6, \XMM6 2356 vaesenc \T1, \XMM7, \XMM7 2357 vaesenc \T1, \XMM8, \XMM8 2358 2359 vmovdqa TMP6(%rsp), \T1 2360 vmovdqu HashKey_3(arg2), \T5 2361 vpclmulqdq $0x11, \T5, \T1, \T3 2362 vpxor \T3, \T4, \T4 2363 2364 vpclmulqdq $0x00, \T5, \T1, \T3 2365 vpxor \T3, \T7, \T7 2366 2367 vpclmulqdq $0x01, \T5, \T1, \T3 2368 vpxor \T3, \T6, \T6 2369 2370 vpclmulqdq $0x10, \T5, \T1, \T3 2371 vpxor \T3, \T6, \T6 2372 2373 vmovdqu 16*8(arg1), \T1 2374 vaesenc \T1, \XMM1, \XMM1 2375 vaesenc \T1, \XMM2, \XMM2 2376 vaesenc \T1, \XMM3, \XMM3 2377 vaesenc \T1, \XMM4, \XMM4 2378 vaesenc \T1, \XMM5, \XMM5 2379 vaesenc \T1, \XMM6, \XMM6 2380 vaesenc \T1, \XMM7, \XMM7 2381 vaesenc \T1, \XMM8, \XMM8 2382 2383 vmovdqa TMP7(%rsp), \T1 2384 vmovdqu HashKey_2(arg2), \T5 2385 vpclmulqdq $0x11, \T5, \T1, \T3 2386 vpxor \T3, \T4, \T4 2387 2388 vpclmulqdq $0x00, \T5, \T1, \T3 2389 vpxor \T3, \T7, \T7 2390 2391 vpclmulqdq $0x01, \T5, \T1, \T3 2392 vpxor \T3, \T6, \T6 2393 2394 vpclmulqdq $0x10, \T5, \T1, \T3 2395 vpxor \T3, \T6, \T6 2396 2397 2398 ####################################################################### 2399 2400 vmovdqu 16*9(arg1), \T5 2401 vaesenc \T5, \XMM1, \XMM1 2402 vaesenc \T5, \XMM2, \XMM2 2403 vaesenc \T5, \XMM3, \XMM3 2404 vaesenc \T5, \XMM4, \XMM4 2405 vaesenc \T5, \XMM5, \XMM5 2406 vaesenc \T5, \XMM6, \XMM6 2407 vaesenc \T5, \XMM7, \XMM7 2408 vaesenc \T5, \XMM8, \XMM8 2409 2410 vmovdqa TMP8(%rsp), \T1 2411 vmovdqu HashKey(arg2), \T5 2412 2413 vpclmulqdq $0x00, \T5, \T1, \T3 2414 vpxor \T3, \T7, \T7 2415 2416 vpclmulqdq $0x01, \T5, \T1, \T3 2417 vpxor \T3, \T6, \T6 2418 2419 vpclmulqdq $0x10, \T5, \T1, \T3 2420 vpxor \T3, \T6, \T6 2421 2422 vpclmulqdq $0x11, \T5, \T1, \T3 2423 vpxor \T3, \T4, \T1 2424 2425 2426 vmovdqu 16*10(arg1), \T5 2427 2428 i = 11 2429 setreg 2430 .rep (\REP-9) 2431 vaesenc \T5, \XMM1, \XMM1 2432 vaesenc \T5, \XMM2, \XMM2 2433 vaesenc \T5, \XMM3, \XMM3 2434 vaesenc \T5, \XMM4, \XMM4 2435 vaesenc \T5, \XMM5, \XMM5 2436 vaesenc \T5, \XMM6, \XMM6 2437 vaesenc \T5, \XMM7, \XMM7 2438 vaesenc \T5, \XMM8, \XMM8 2439 2440 vmovdqu 16*i(arg1), \T5 2441 i = i + 1 2442 setreg 2443 .endr 2444 2445 i = 0 2446 j = 1 2447 setreg 2448 .rep 8 2449 vpxor 16*i(arg4, %r11), \T5, \T2 2450 .if \ENC_DEC == ENC 2451 vaesenclast \T2, reg_j, reg_j 2452 .else 2453 vaesenclast \T2, reg_j, \T3 2454 vmovdqu 16*i(arg4, %r11), reg_j 2455 vmovdqu \T3, 16*i(arg3, %r11) 2456 .endif 2457 i = (i+1) 2458 j = (j+1) 2459 setreg 2460 .endr 2461 ####################################################################### 2462 2463 2464 vpslldq $8, \T6, \T3 # shift-L T3 2 DWs 2465 vpsrldq $8, \T6, \T6 # shift-R T2 2 DWs 2466 vpxor \T3, \T7, \T7 2467 vpxor \T6, \T1, \T1 # accumulate the results in T1:T7 2468 2469 2470 2471 ####################################################################### 2472 #first phase of the reduction 2473 vmovdqa POLY2(%rip), \T3 2474 2475 vpclmulqdq $0x01, \T7, \T3, \T2 2476 vpslldq $8, \T2, \T2 # shift-L xmm2 2 DWs 2477 2478 vpxor \T2, \T7, \T7 # first phase of the reduction complete 2479 ####################################################################### 2480 .if \ENC_DEC == ENC 2481 vmovdqu \XMM1, 16*0(arg3,%r11) # Write to the Ciphertext buffer 2482 vmovdqu \XMM2, 16*1(arg3,%r11) # Write to the Ciphertext buffer 2483 vmovdqu \XMM3, 16*2(arg3,%r11) # Write to the Ciphertext buffer 2484 vmovdqu \XMM4, 16*3(arg3,%r11) # Write to the Ciphertext buffer 2485 vmovdqu \XMM5, 16*4(arg3,%r11) # Write to the Ciphertext buffer 2486 vmovdqu \XMM6, 16*5(arg3,%r11) # Write to the Ciphertext buffer 2487 vmovdqu \XMM7, 16*6(arg3,%r11) # Write to the Ciphertext buffer 2488 vmovdqu \XMM8, 16*7(arg3,%r11) # Write to the Ciphertext buffer 2489 .endif 2490 2491 ####################################################################### 2492 #second phase of the reduction 2493 vpclmulqdq $0x00, \T7, \T3, \T2 2494 vpsrldq $4, \T2, \T2 # shift-R xmm2 1 DW (Shift-R only 1-DW to obtain 2-DWs shift-R) 2495 2496 vpclmulqdq $0x10, \T7, \T3, \T4 2497 vpslldq $4, \T4, \T4 # shift-L xmm0 1 DW (Shift-L 1-DW to obtain result with no shifts) 2498 2499 vpxor \T2, \T4, \T4 # second phase of the reduction complete 2500 ####################################################################### 2501 vpxor \T4, \T1, \T1 # the result is in T1 2502 2503 vpshufb SHUF_MASK(%rip), \XMM1, \XMM1 # perform a 16Byte swap 2504 vpshufb SHUF_MASK(%rip), \XMM2, \XMM2 # perform a 16Byte swap 2505 vpshufb SHUF_MASK(%rip), \XMM3, \XMM3 # perform a 16Byte swap 2506 vpshufb SHUF_MASK(%rip), \XMM4, \XMM4 # perform a 16Byte swap 2507 vpshufb SHUF_MASK(%rip), \XMM5, \XMM5 # perform a 16Byte swap 2508 vpshufb SHUF_MASK(%rip), \XMM6, \XMM6 # perform a 16Byte swap 2509 vpshufb SHUF_MASK(%rip), \XMM7, \XMM7 # perform a 16Byte swap 2510 vpshufb SHUF_MASK(%rip), \XMM8, \XMM8 # perform a 16Byte swap 2511 2512 2513 vpxor \T1, \XMM1, \XMM1 2514 2515 2516 2517 .endm 2518 2519 2520 # GHASH the last 4 ciphertext blocks. 2521 .macro GHASH_LAST_8_AVX2 T1 T2 T3 T4 T5 T6 T7 XMM1 XMM2 XMM3 XMM4 XMM5 XMM6 XMM7 XMM8 2522 2523 ## Karatsuba Method 2524 2525 vmovdqu HashKey_8(arg2), \T5 2526 2527 vpshufd $0b01001110, \XMM1, \T2 2528 vpshufd $0b01001110, \T5, \T3 2529 vpxor \XMM1, \T2, \T2 2530 vpxor \T5, \T3, \T3 2531 2532 vpclmulqdq $0x11, \T5, \XMM1, \T6 2533 vpclmulqdq $0x00, \T5, \XMM1, \T7 2534 2535 vpclmulqdq $0x00, \T3, \T2, \XMM1 2536 2537 ###################### 2538 2539 vmovdqu HashKey_7(arg2), \T5 2540 vpshufd $0b01001110, \XMM2, \T2 2541 vpshufd $0b01001110, \T5, \T3 2542 vpxor \XMM2, \T2, \T2 2543 vpxor \T5, \T3, \T3 2544 2545 vpclmulqdq $0x11, \T5, \XMM2, \T4 2546 vpxor \T4, \T6, \T6 2547 2548 vpclmulqdq $0x00, \T5, \XMM2, \T4 2549 vpxor \T4, \T7, \T7 2550 2551 vpclmulqdq $0x00, \T3, \T2, \T2 2552 2553 vpxor \T2, \XMM1, \XMM1 2554 2555 ###################### 2556 2557 vmovdqu HashKey_6(arg2), \T5 2558 vpshufd $0b01001110, \XMM3, \T2 2559 vpshufd $0b01001110, \T5, \T3 2560 vpxor \XMM3, \T2, \T2 2561 vpxor \T5, \T3, \T3 2562 2563 vpclmulqdq $0x11, \T5, \XMM3, \T4 2564 vpxor \T4, \T6, \T6 2565 2566 vpclmulqdq $0x00, \T5, \XMM3, \T4 2567 vpxor \T4, \T7, \T7 2568 2569 vpclmulqdq $0x00, \T3, \T2, \T2 2570 2571 vpxor \T2, \XMM1, \XMM1 2572 2573 ###################### 2574 2575 vmovdqu HashKey_5(arg2), \T5 2576 vpshufd $0b01001110, \XMM4, \T2 2577 vpshufd $0b01001110, \T5, \T3 2578 vpxor \XMM4, \T2, \T2 2579 vpxor \T5, \T3, \T3 2580 2581 vpclmulqdq $0x11, \T5, \XMM4, \T4 2582 vpxor \T4, \T6, \T6 2583 2584 vpclmulqdq $0x00, \T5, \XMM4, \T4 2585 vpxor \T4, \T7, \T7 2586 2587 vpclmulqdq $0x00, \T3, \T2, \T2 2588 2589 vpxor \T2, \XMM1, \XMM1 2590 2591 ###################### 2592 2593 vmovdqu HashKey_4(arg2), \T5 2594 vpshufd $0b01001110, \XMM5, \T2 2595 vpshufd $0b01001110, \T5, \T3 2596 vpxor \XMM5, \T2, \T2 2597 vpxor \T5, \T3, \T3 2598 2599 vpclmulqdq $0x11, \T5, \XMM5, \T4 2600 vpxor \T4, \T6, \T6 2601 2602 vpclmulqdq $0x00, \T5, \XMM5, \T4 2603 vpxor \T4, \T7, \T7 2604 2605 vpclmulqdq $0x00, \T3, \T2, \T2 2606 2607 vpxor \T2, \XMM1, \XMM1 2608 2609 ###################### 2610 2611 vmovdqu HashKey_3(arg2), \T5 2612 vpshufd $0b01001110, \XMM6, \T2 2613 vpshufd $0b01001110, \T5, \T3 2614 vpxor \XMM6, \T2, \T2 2615 vpxor \T5, \T3, \T3 2616 2617 vpclmulqdq $0x11, \T5, \XMM6, \T4 2618 vpxor \T4, \T6, \T6 2619 2620 vpclmulqdq $0x00, \T5, \XMM6, \T4 2621 vpxor \T4, \T7, \T7 2622 2623 vpclmulqdq $0x00, \T3, \T2, \T2 2624 2625 vpxor \T2, \XMM1, \XMM1 2626 2627 ###################### 2628 2629 vmovdqu HashKey_2(arg2), \T5 2630 vpshufd $0b01001110, \XMM7, \T2 2631 vpshufd $0b01001110, \T5, \T3 2632 vpxor \XMM7, \T2, \T2 2633 vpxor \T5, \T3, \T3 2634 2635 vpclmulqdq $0x11, \T5, \XMM7, \T4 2636 vpxor \T4, \T6, \T6 2637 2638 vpclmulqdq $0x00, \T5, \XMM7, \T4 2639 vpxor \T4, \T7, \T7 2640 2641 vpclmulqdq $0x00, \T3, \T2, \T2 2642 2643 vpxor \T2, \XMM1, \XMM1 2644 2645 ###################### 2646 2647 vmovdqu HashKey(arg2), \T5 2648 vpshufd $0b01001110, \XMM8, \T2 2649 vpshufd $0b01001110, \T5, \T3 2650 vpxor \XMM8, \T2, \T2 2651 vpxor \T5, \T3, \T3 2652 2653 vpclmulqdq $0x11, \T5, \XMM8, \T4 2654 vpxor \T4, \T6, \T6 2655 2656 vpclmulqdq $0x00, \T5, \XMM8, \T4 2657 vpxor \T4, \T7, \T7 2658 2659 vpclmulqdq $0x00, \T3, \T2, \T2 2660 2661 vpxor \T2, \XMM1, \XMM1 2662 vpxor \T6, \XMM1, \XMM1 2663 vpxor \T7, \XMM1, \T2 2664 2665 2666 2667 2668 vpslldq $8, \T2, \T4 2669 vpsrldq $8, \T2, \T2 2670 2671 vpxor \T4, \T7, \T7 2672 vpxor \T2, \T6, \T6 # <T6:T7> holds the result of the 2673 # accumulated carry-less multiplications 2674 2675 ####################################################################### 2676 #first phase of the reduction 2677 vmovdqa POLY2(%rip), \T3 2678 2679 vpclmulqdq $0x01, \T7, \T3, \T2 2680 vpslldq $8, \T2, \T2 # shift-L xmm2 2 DWs 2681 2682 vpxor \T2, \T7, \T7 # first phase of the reduction complete 2683 ####################################################################### 2684 2685 2686 #second phase of the reduction 2687 vpclmulqdq $0x00, \T7, \T3, \T2 2688 vpsrldq $4, \T2, \T2 # shift-R T2 1 DW (Shift-R only 1-DW to obtain 2-DWs shift-R) 2689 2690 vpclmulqdq $0x10, \T7, \T3, \T4 2691 vpslldq $4, \T4, \T4 # shift-L T4 1 DW (Shift-L 1-DW to obtain result with no shifts) 2692 2693 vpxor \T2, \T4, \T4 # second phase of the reduction complete 2694 ####################################################################### 2695 vpxor \T4, \T6, \T6 # the result is in T6 2696 .endm 2697 2698 2699 2700 ############################################################# 2701 #void aesni_gcm_init_avx_gen4 2702 # (gcm_data *my_ctx_data, 2703 # gcm_context_data *data, 2704 # u8 *iv, /* Pre-counter block j0: 4 byte salt 2705 # (from Security Association) concatenated with 8 byte 2706 # Initialisation Vector (from IPSec ESP Payload) 2707 # concatenated with 0x00000001. 16-byte aligned pointer. */ 2708 # u8 *hash_subkey# /* H, the Hash sub key input. Data starts on a 16-byte boundary. */ 2709 # const u8 *aad, /* Additional Authentication Data (AAD)*/ 2710 # u64 aad_len) /* Length of AAD in bytes. With RFC4106 this is going to be 8 or 12 Bytes */ 2711 ############################################################# 2712 SYM_FUNC_START(aesni_gcm_init_avx_gen4) 2713 FUNC_SAVE 2714 INIT GHASH_MUL_AVX2, PRECOMPUTE_AVX2 2715 FUNC_RESTORE 2716 RET 2717 SYM_FUNC_END(aesni_gcm_init_avx_gen4) 2718 2719 ############################################################################### 2720 #void aesni_gcm_enc_avx_gen4( 2721 # gcm_data *my_ctx_data, /* aligned to 16 Bytes */ 2722 # gcm_context_data *data, 2723 # u8 *out, /* Ciphertext output. Encrypt in-place is allowed. */ 2724 # const u8 *in, /* Plaintext input */ 2725 # u64 plaintext_len) /* Length of data in Bytes for encryption. */ 2726 ############################################################################### 2727 SYM_FUNC_START(aesni_gcm_enc_update_avx_gen4) 2728 FUNC_SAVE 2729 mov keysize,%eax 2730 cmp $32, %eax 2731 je key_256_enc_update4 2732 cmp $16, %eax 2733 je key_128_enc_update4 2734 # must be 192 2735 GCM_ENC_DEC INITIAL_BLOCKS_AVX2, GHASH_8_ENCRYPT_8_PARALLEL_AVX2, GHASH_LAST_8_AVX2, GHASH_MUL_AVX2, ENC, 11 2736 FUNC_RESTORE 2737 RET 2738 key_128_enc_update4: 2739 GCM_ENC_DEC INITIAL_BLOCKS_AVX2, GHASH_8_ENCRYPT_8_PARALLEL_AVX2, GHASH_LAST_8_AVX2, GHASH_MUL_AVX2, ENC, 9 2740 FUNC_RESTORE 2741 RET 2742 key_256_enc_update4: 2743 GCM_ENC_DEC INITIAL_BLOCKS_AVX2, GHASH_8_ENCRYPT_8_PARALLEL_AVX2, GHASH_LAST_8_AVX2, GHASH_MUL_AVX2, ENC, 13 2744 FUNC_RESTORE 2745 RET 2746 SYM_FUNC_END(aesni_gcm_enc_update_avx_gen4) 2747 2748 ############################################################################### 2749 #void aesni_gcm_dec_update_avx_gen4( 2750 # gcm_data *my_ctx_data, /* aligned to 16 Bytes */ 2751 # gcm_context_data *data, 2752 # u8 *out, /* Plaintext output. Decrypt in-place is allowed. */ 2753 # const u8 *in, /* Ciphertext input */ 2754 # u64 plaintext_len) /* Length of data in Bytes for encryption. */ 2755 ############################################################################### 2756 SYM_FUNC_START(aesni_gcm_dec_update_avx_gen4) 2757 FUNC_SAVE 2758 mov keysize,%eax 2759 cmp $32, %eax 2760 je key_256_dec_update4 2761 cmp $16, %eax 2762 je key_128_dec_update4 2763 # must be 192 2764 GCM_ENC_DEC INITIAL_BLOCKS_AVX2, GHASH_8_ENCRYPT_8_PARALLEL_AVX2, GHASH_LAST_8_AVX2, GHASH_MUL_AVX2, DEC, 11 2765 FUNC_RESTORE 2766 RET 2767 key_128_dec_update4: 2768 GCM_ENC_DEC INITIAL_BLOCKS_AVX2, GHASH_8_ENCRYPT_8_PARALLEL_AVX2, GHASH_LAST_8_AVX2, GHASH_MUL_AVX2, DEC, 9 2769 FUNC_RESTORE 2770 RET 2771 key_256_dec_update4: 2772 GCM_ENC_DEC INITIAL_BLOCKS_AVX2, GHASH_8_ENCRYPT_8_PARALLEL_AVX2, GHASH_LAST_8_AVX2, GHASH_MUL_AVX2, DEC, 13 2773 FUNC_RESTORE 2774 RET 2775 SYM_FUNC_END(aesni_gcm_dec_update_avx_gen4) 2776 2777 ############################################################################### 2778 #void aesni_gcm_finalize_avx_gen4( 2779 # gcm_data *my_ctx_data, /* aligned to 16 Bytes */ 2780 # gcm_context_data *data, 2781 # u8 *auth_tag, /* Authenticated Tag output. */ 2782 # u64 auth_tag_len)# /* Authenticated Tag Length in bytes. 2783 # Valid values are 16 (most likely), 12 or 8. */ 2784 ############################################################################### 2785 SYM_FUNC_START(aesni_gcm_finalize_avx_gen4) 2786 FUNC_SAVE 2787 mov keysize,%eax 2788 cmp $32, %eax 2789 je key_256_finalize4 2790 cmp $16, %eax 2791 je key_128_finalize4 2792 # must be 192 2793 GCM_COMPLETE GHASH_MUL_AVX2, 11, arg3, arg4 2794 FUNC_RESTORE 2795 RET 2796 key_128_finalize4: 2797 GCM_COMPLETE GHASH_MUL_AVX2, 9, arg3, arg4 2798 FUNC_RESTORE 2799 RET 2800 key_256_finalize4: 2801 GCM_COMPLETE GHASH_MUL_AVX2, 13, arg3, arg4 2802 FUNC_RESTORE 2803 RET 2804 SYM_FUNC_END(aesni_gcm_finalize_avx_gen4) 2805