xref: /kernel/linux/linux-6.6/arch/x86/crypto/aes_ctrby8_avx-x86_64.S

/* SPDX-License-Identifier: GPL-2.0-only OR BSD-3-Clause */
/*
 * AES CTR mode by8 optimization with AVX instructions. (x86_64)
 *
 * Copyright(c) 2014 Intel Corporation.
 *
 * Contact Information:
 * James Guilford <james.guilford@intel.com>
 * Sean Gulley <sean.m.gulley@intel.com>
 * Chandramouli Narayanan <mouli@linux.intel.com>
 */
/*
 * This is AES128/192/256 CTR mode optimization implementation. It requires
 * the support of Intel(R) AESNI and AVX instructions.
 *
 * This work was inspired by the AES CTR mode optimization published
 * in Intel Optimized IPSEC Cryptographic library.
 * Additional information on it can be found at:
 *    https://github.com/intel/intel-ipsec-mb
 */

#include <linux/linkage.h>

#define VMOVDQ		vmovdqu

/*
 * Note: the "x" prefix in these aliases means "this is an xmm register".  The
 * alias prefixes have no relation to XCTR where the "X" prefix means "XOR
 * counter".
 */
#define xdata0		%xmm0
#define xdata1		%xmm1
#define xdata2		%xmm2
#define xdata3		%xmm3
#define xdata4		%xmm4
#define xdata5		%xmm5
#define xdata6		%xmm6
#define xdata7		%xmm7
#define xcounter	%xmm8	// CTR mode only
#define xiv		%xmm8	// XCTR mode only
#define xbyteswap	%xmm9	// CTR mode only
#define xtmp		%xmm9	// XCTR mode only
#define xkey0		%xmm10
#define xkey4		%xmm11
#define xkey8		%xmm12
#define xkey12		%xmm13
#define xkeyA		%xmm14
#define xkeyB		%xmm15

#define p_in		%rdi
#define p_iv		%rsi
#define p_keys		%rdx
#define p_out		%rcx
#define num_bytes	%r8
#define counter		%r9	// XCTR mode only
#define tmp		%r10
#define	DDQ_DATA	0
#define	XDATA		1
#define KEY_128		1
#define KEY_192		2
#define KEY_256		3

.section .rodata
.align 16

byteswap_const:
	.octa 0x000102030405060708090A0B0C0D0E0F
ddq_low_msk:
	.octa 0x0000000000000000FFFFFFFFFFFFFFFF
ddq_high_add_1:
	.octa 0x00000000000000010000000000000000
ddq_add_1:
	.octa 0x00000000000000000000000000000001
ddq_add_2:
	.octa 0x00000000000000000000000000000002
ddq_add_3:
	.octa 0x00000000000000000000000000000003
ddq_add_4:
	.octa 0x00000000000000000000000000000004
ddq_add_5:
	.octa 0x00000000000000000000000000000005
ddq_add_6:
	.octa 0x00000000000000000000000000000006
ddq_add_7:
	.octa 0x00000000000000000000000000000007
ddq_add_8:
	.octa 0x00000000000000000000000000000008

.text

/* generate a unique variable for ddq_add_x */

/* generate a unique variable for xmm register */
.macro setxdata n
	var_xdata = %xmm\n
.endm

/* club the numeric 'id' to the symbol 'name' */

.macro club name, id
.altmacro
	.if \name == XDATA
		setxdata %\id
	.endif
.noaltmacro
.endm

/*
 * do_aes num_in_par load_keys key_len
 * This increments p_in, but not p_out
 */
.macro do_aes b, k, key_len, xctr
	.set by, \b
	.set load_keys, \k
	.set klen, \key_len

	.if (load_keys)
		vmovdqa	0*16(p_keys), xkey0
	.endif

	.if \xctr
		movq counter, xtmp
		.set i, 0
		.rept (by)
			club XDATA, i
			vpaddq	(ddq_add_1 + 16 * i)(%rip), xtmp, var_xdata
			.set i, (i +1)
		.endr
		.set i, 0
		.rept (by)
			club	XDATA, i
			vpxor	xiv, var_xdata, var_xdata
			.set i, (i +1)
		.endr
	.else
		vpshufb	xbyteswap, xcounter, xdata0
		.set i, 1
		.rept (by - 1)
			club XDATA, i
			vpaddq	(ddq_add_1 + 16 * (i - 1))(%rip), xcounter, var_xdata
			vptest	ddq_low_msk(%rip), var_xdata
			jnz 1f
			vpaddq	ddq_high_add_1(%rip), var_xdata, var_xdata
			vpaddq	ddq_high_add_1(%rip), xcounter, xcounter
			1:
			vpshufb	xbyteswap, var_xdata, var_xdata
			.set i, (i +1)
		.endr
	.endif

	vmovdqa	1*16(p_keys), xkeyA

	vpxor	xkey0, xdata0, xdata0
	.if \xctr
		add $by, counter
	.else
		vpaddq	(ddq_add_1 + 16 * (by - 1))(%rip), xcounter, xcounter
		vptest	ddq_low_msk(%rip), xcounter
		jnz	1f
		vpaddq	ddq_high_add_1(%rip), xcounter, xcounter
		1:
	.endif

	.set i, 1
	.rept (by - 1)
		club XDATA, i
		vpxor	xkey0, var_xdata, var_xdata
		.set i, (i +1)
	.endr

	vmovdqa	2*16(p_keys), xkeyB

	.set i, 0
	.rept by
		club XDATA, i
		vaesenc	xkeyA, var_xdata, var_xdata		/* key 1 */
		.set i, (i +1)
	.endr

	.if (klen == KEY_128)
		.if (load_keys)
			vmovdqa	3*16(p_keys), xkey4
		.endif
	.else
		vmovdqa	3*16(p_keys), xkeyA
	.endif

	.set i, 0
	.rept by
		club XDATA, i
		vaesenc	xkeyB, var_xdata, var_xdata		/* key 2 */
		.set i, (i +1)
	.endr

	add	$(16*by), p_in

	.if (klen == KEY_128)
		vmovdqa	4*16(p_keys), xkeyB
	.else
		.if (load_keys)
			vmovdqa	4*16(p_keys), xkey4
		.endif
	.endif

	.set i, 0
	.rept by
		club XDATA, i
		/* key 3 */
		.if (klen == KEY_128)
			vaesenc	xkey4, var_xdata, var_xdata
		.else
			vaesenc	xkeyA, var_xdata, var_xdata
		.endif
		.set i, (i +1)
	.endr

	vmovdqa	5*16(p_keys), xkeyA

	.set i, 0
	.rept by
		club XDATA, i
		/* key 4 */
		.if (klen == KEY_128)
			vaesenc	xkeyB, var_xdata, var_xdata
		.else
			vaesenc	xkey4, var_xdata, var_xdata
		.endif
		.set i, (i +1)
	.endr

	.if (klen == KEY_128)
		.if (load_keys)
			vmovdqa	6*16(p_keys), xkey8
		.endif
	.else
		vmovdqa	6*16(p_keys), xkeyB
	.endif

	.set i, 0
	.rept by
		club XDATA, i
		vaesenc	xkeyA, var_xdata, var_xdata		/* key 5 */
		.set i, (i +1)
	.endr

	vmovdqa	7*16(p_keys), xkeyA

	.set i, 0
	.rept by
		club XDATA, i
		/* key 6 */
		.if (klen == KEY_128)
			vaesenc	xkey8, var_xdata, var_xdata
		.else
			vaesenc	xkeyB, var_xdata, var_xdata
		.endif
		.set i, (i +1)
	.endr

	.if (klen == KEY_128)
		vmovdqa	8*16(p_keys), xkeyB
	.else
		.if (load_keys)
			vmovdqa	8*16(p_keys), xkey8
		.endif
	.endif

	.set i, 0
	.rept by
		club XDATA, i
		vaesenc	xkeyA, var_xdata, var_xdata		/* key 7 */
		.set i, (i +1)
	.endr

	.if (klen == KEY_128)
		.if (load_keys)
			vmovdqa	9*16(p_keys), xkey12
		.endif
	.else
		vmovdqa	9*16(p_keys), xkeyA
	.endif

	.set i, 0
	.rept by
		club XDATA, i
		/* key 8 */
		.if (klen == KEY_128)
			vaesenc	xkeyB, var_xdata, var_xdata
		.else
			vaesenc	xkey8, var_xdata, var_xdata
		.endif
		.set i, (i +1)
	.endr

	vmovdqa	10*16(p_keys), xkeyB

	.set i, 0
	.rept by
		club XDATA, i
		/* key 9 */
		.if (klen == KEY_128)
			vaesenc	xkey12, var_xdata, var_xdata
		.else
			vaesenc	xkeyA, var_xdata, var_xdata
		.endif
		.set i, (i +1)
	.endr

	.if (klen != KEY_128)
		vmovdqa	11*16(p_keys), xkeyA
	.endif

	.set i, 0
	.rept by
		club XDATA, i
		/* key 10 */
		.if (klen == KEY_128)
			vaesenclast	xkeyB, var_xdata, var_xdata
		.else
			vaesenc	xkeyB, var_xdata, var_xdata
		.endif
		.set i, (i +1)
	.endr

	.if (klen != KEY_128)
		.if (load_keys)
			vmovdqa	12*16(p_keys), xkey12
		.endif

		.set i, 0
		.rept by
			club XDATA, i
			vaesenc	xkeyA, var_xdata, var_xdata	/* key 11 */
			.set i, (i +1)
		.endr

		.if (klen == KEY_256)
			vmovdqa	13*16(p_keys), xkeyA
		.endif

		.set i, 0
		.rept by
			club XDATA, i
			.if (klen == KEY_256)
				/* key 12 */
				vaesenc	xkey12, var_xdata, var_xdata
			.else
				vaesenclast xkey12, var_xdata, var_xdata
			.endif
			.set i, (i +1)
		.endr

		.if (klen == KEY_256)
			vmovdqa	14*16(p_keys), xkeyB

			.set i, 0
			.rept by
				club XDATA, i
				/* key 13 */
				vaesenc	xkeyA, var_xdata, var_xdata
				.set i, (i +1)
			.endr

			.set i, 0
			.rept by
				club XDATA, i
				/* key 14 */
				vaesenclast	xkeyB, var_xdata, var_xdata
				.set i, (i +1)
			.endr
		.endif
	.endif

	.set i, 0
	.rept (by / 2)
		.set j, (i+1)
		VMOVDQ	(i*16 - 16*by)(p_in), xkeyA
		VMOVDQ	(j*16 - 16*by)(p_in), xkeyB
		club XDATA, i
		vpxor	xkeyA, var_xdata, var_xdata
		club XDATA, j
		vpxor	xkeyB, var_xdata, var_xdata
		.set i, (i+2)
	.endr

	.if (i < by)
		VMOVDQ	(i*16 - 16*by)(p_in), xkeyA
		club XDATA, i
		vpxor	xkeyA, var_xdata, var_xdata
	.endif

	.set i, 0
	.rept by
		club XDATA, i
		VMOVDQ	var_xdata, i*16(p_out)
		.set i, (i+1)
	.endr
.endm

.macro do_aes_load val, key_len, xctr
	do_aes \val, 1, \key_len, \xctr
.endm

.macro do_aes_noload val, key_len, xctr
	do_aes \val, 0, \key_len, \xctr
.endm

/* main body of aes ctr load */

.macro do_aes_ctrmain key_len, xctr
	cmp	$16, num_bytes
	jb	.Ldo_return2\xctr\key_len

	.if \xctr
		shr	$4, counter
		vmovdqu	(p_iv), xiv
	.else
		vmovdqa	byteswap_const(%rip), xbyteswap
		vmovdqu	(p_iv), xcounter
		vpshufb	xbyteswap, xcounter, xcounter
	.endif

	mov	num_bytes, tmp
	and	$(7*16), tmp
	jz	.Lmult_of_8_blks\xctr\key_len

	/* 1 <= tmp <= 7 */
	cmp	$(4*16), tmp
	jg	.Lgt4\xctr\key_len
	je	.Leq4\xctr\key_len

.Llt4\xctr\key_len:
	cmp	$(2*16), tmp
	jg	.Leq3\xctr\key_len
	je	.Leq2\xctr\key_len

.Leq1\xctr\key_len:
	do_aes_load	1, \key_len, \xctr
	add	$(1*16), p_out
	and	$(~7*16), num_bytes
	jz	.Ldo_return2\xctr\key_len
	jmp	.Lmain_loop2\xctr\key_len

.Leq2\xctr\key_len:
	do_aes_load	2, \key_len, \xctr
	add	$(2*16), p_out
	and	$(~7*16), num_bytes
	jz	.Ldo_return2\xctr\key_len
	jmp	.Lmain_loop2\xctr\key_len


.Leq3\xctr\key_len:
	do_aes_load	3, \key_len, \xctr
	add	$(3*16), p_out
	and	$(~7*16), num_bytes
	jz	.Ldo_return2\xctr\key_len
	jmp	.Lmain_loop2\xctr\key_len

.Leq4\xctr\key_len:
	do_aes_load	4, \key_len, \xctr
	add	$(4*16), p_out
	and	$(~7*16), num_bytes
	jz	.Ldo_return2\xctr\key_len
	jmp	.Lmain_loop2\xctr\key_len

.Lgt4\xctr\key_len:
	cmp	$(6*16), tmp
	jg	.Leq7\xctr\key_len
	je	.Leq6\xctr\key_len

.Leq5\xctr\key_len:
	do_aes_load	5, \key_len, \xctr
	add	$(5*16), p_out
	and	$(~7*16), num_bytes
	jz	.Ldo_return2\xctr\key_len
	jmp	.Lmain_loop2\xctr\key_len

.Leq6\xctr\key_len:
	do_aes_load	6, \key_len, \xctr
	add	$(6*16), p_out
	and	$(~7*16), num_bytes
	jz	.Ldo_return2\xctr\key_len
	jmp	.Lmain_loop2\xctr\key_len

.Leq7\xctr\key_len:
	do_aes_load	7, \key_len, \xctr
	add	$(7*16), p_out
	and	$(~7*16), num_bytes
	jz	.Ldo_return2\xctr\key_len
	jmp	.Lmain_loop2\xctr\key_len

.Lmult_of_8_blks\xctr\key_len:
	.if (\key_len != KEY_128)
		vmovdqa	0*16(p_keys), xkey0
		vmovdqa	4*16(p_keys), xkey4
		vmovdqa	8*16(p_keys), xkey8
		vmovdqa	12*16(p_keys), xkey12
	.else
		vmovdqa	0*16(p_keys), xkey0
		vmovdqa	3*16(p_keys), xkey4
		vmovdqa	6*16(p_keys), xkey8
		vmovdqa	9*16(p_keys), xkey12
	.endif
.align 16
.Lmain_loop2\xctr\key_len:
	/* num_bytes is a multiple of 8 and >0 */
	do_aes_noload	8, \key_len, \xctr
	add	$(8*16), p_out
	sub	$(8*16), num_bytes
	jne	.Lmain_loop2\xctr\key_len

.Ldo_return2\xctr\key_len:
	.if !\xctr
		/* return updated IV */
		vpshufb	xbyteswap, xcounter, xcounter
		vmovdqu	xcounter, (p_iv)
	.endif
	RET
.endm

/*
 * routine to do AES128 CTR enc/decrypt "by8"
 * XMM registers are clobbered.
 * Saving/restoring must be done at a higher level
 * aes_ctr_enc_128_avx_by8(void *in, void *iv, void *keys, void *out,
 *			unsigned int num_bytes)
 */
SYM_FUNC_START(aes_ctr_enc_128_avx_by8)
	/* call the aes main loop */
	do_aes_ctrmain KEY_128 0

SYM_FUNC_END(aes_ctr_enc_128_avx_by8)

/*
 * routine to do AES192 CTR enc/decrypt "by8"
 * XMM registers are clobbered.
 * Saving/restoring must be done at a higher level
 * aes_ctr_enc_192_avx_by8(void *in, void *iv, void *keys, void *out,
 *			unsigned int num_bytes)
 */
SYM_FUNC_START(aes_ctr_enc_192_avx_by8)
	/* call the aes main loop */
	do_aes_ctrmain KEY_192 0

SYM_FUNC_END(aes_ctr_enc_192_avx_by8)

/*
 * routine to do AES256 CTR enc/decrypt "by8"
 * XMM registers are clobbered.
 * Saving/restoring must be done at a higher level
 * aes_ctr_enc_256_avx_by8(void *in, void *iv, void *keys, void *out,
 *			unsigned int num_bytes)
 */
SYM_FUNC_START(aes_ctr_enc_256_avx_by8)
	/* call the aes main loop */
	do_aes_ctrmain KEY_256 0

SYM_FUNC_END(aes_ctr_enc_256_avx_by8)

/*
 * routine to do AES128 XCTR enc/decrypt "by8"
 * XMM registers are clobbered.
 * Saving/restoring must be done at a higher level
 * aes_xctr_enc_128_avx_by8(const u8 *in, const u8 *iv, const void *keys,
 * 	u8* out, unsigned int num_bytes, unsigned int byte_ctr)
 */
SYM_FUNC_START(aes_xctr_enc_128_avx_by8)
	/* call the aes main loop */
	do_aes_ctrmain KEY_128 1

SYM_FUNC_END(aes_xctr_enc_128_avx_by8)

/*
 * routine to do AES192 XCTR enc/decrypt "by8"
 * XMM registers are clobbered.
 * Saving/restoring must be done at a higher level
 * aes_xctr_enc_192_avx_by8(const u8 *in, const u8 *iv, const void *keys,
 * 	u8* out, unsigned int num_bytes, unsigned int byte_ctr)
 */
SYM_FUNC_START(aes_xctr_enc_192_avx_by8)
	/* call the aes main loop */
	do_aes_ctrmain KEY_192 1

SYM_FUNC_END(aes_xctr_enc_192_avx_by8)

/*
 * routine to do AES256 XCTR enc/decrypt "by8"
 * XMM registers are clobbered.
 * Saving/restoring must be done at a higher level
 * aes_xctr_enc_256_avx_by8(const u8 *in, const u8 *iv, const void *keys,
 * 	u8* out, unsigned int num_bytes, unsigned int byte_ctr)
 */
SYM_FUNC_START(aes_xctr_enc_256_avx_by8)
	/* call the aes main loop */
	do_aes_ctrmain KEY_256 1

SYM_FUNC_END(aes_xctr_enc_256_avx_by8)