xref: /kernel/linux/linux-6.6/arch/x86/lib/checksum_32.S (revision 62306a36)

/* SPDX-License-Identifier: GPL-2.0-or-later */
/*
 * INET		An implementation of the TCP/IP protocol suite for the LINUX
 *		operating system.  INET is implemented using the  BSD Socket
 *		interface as the means of communication with the user level.
 *
 *		IP/TCP/UDP checksumming routines
 *
 * Authors:	Jorge Cwik, <jorge@laser.satlink.net>
 *		Arnt Gulbrandsen, <agulbra@nvg.unit.no>
 *		Tom May, <ftom@netcom.com>
 *              Pentium Pro/II routines:
 *              Alexander Kjeldaas <astor@guardian.no>
 *              Finn Arne Gangstad <finnag@guardian.no>
 *		Lots of code moved from tcp.c and ip.c; see those files
 *		for more names.
 *
 * Changes:     Ingo Molnar, converted csum_partial_copy() to 2.1 exception
 *			     handling.
 *		Andi Kleen,  add zeroing on error
 *                   converted to pure assembler
 */

#include <linux/linkage.h>
#include <asm/errno.h>
#include <asm/asm.h>
#include <asm/export.h>
#include <asm/nospec-branch.h>

/*
 * computes a partial checksum, e.g. for TCP/UDP fragments
 */

/*
unsigned int csum_partial(const unsigned char * buff, int len, unsigned int sum)
 */

.text

#ifndef CONFIG_X86_USE_PPRO_CHECKSUM

	  /*
	   * Experiments with Ethernet and SLIP connections show that buff
	   * is aligned on either a 2-byte or 4-byte boundary.  We get at
	   * least a twofold speedup on 486 and Pentium if it is 4-byte aligned.
	   * Fortunately, it is easy to convert 2-byte alignment to 4-byte
	   * alignment for the unrolled loop.
	   */
SYM_FUNC_START(csum_partial)
	pushl %esi
	pushl %ebx
	movl 20(%esp),%eax	# Function arg: unsigned int sum
	movl 16(%esp),%ecx	# Function arg: int len
	movl 12(%esp),%esi	# Function arg: unsigned char *buff
	testl $3, %esi		# Check alignment.
	jz 2f			# Jump if alignment is ok.
	testl $1, %esi		# Check alignment.
	jz 10f			# Jump if alignment is boundary of 2 bytes.

	# buf is odd
	dec %ecx
	jl 8f
	movzbl (%esi), %ebx
	adcl %ebx, %eax
	roll $8, %eax
	inc %esi
	testl $2, %esi
	jz 2f
10:
	subl $2, %ecx		# Alignment uses up two bytes.
	jae 1f			# Jump if we had at least two bytes.
	addl $2, %ecx		# ecx was < 2.  Deal with it.
	jmp 4f
1:	movw (%esi), %bx
	addl $2, %esi
	addw %bx, %ax
	adcl $0, %eax
2:
	movl %ecx, %edx
	shrl $5, %ecx
	jz 2f
	testl %esi, %esi
1:	movl (%esi), %ebx
	adcl %ebx, %eax
	movl 4(%esi), %ebx
	adcl %ebx, %eax
	movl 8(%esi), %ebx
	adcl %ebx, %eax
	movl 12(%esi), %ebx
	adcl %ebx, %eax
	movl 16(%esi), %ebx
	adcl %ebx, %eax
	movl 20(%esi), %ebx
	adcl %ebx, %eax
	movl 24(%esi), %ebx
	adcl %ebx, %eax
	movl 28(%esi), %ebx
	adcl %ebx, %eax
	lea 32(%esi), %esi
	dec %ecx
	jne 1b
	adcl $0, %eax
2:	movl %edx, %ecx
	andl $0x1c, %edx
	je 4f
	shrl $2, %edx		# This clears CF
3:	adcl (%esi), %eax
	lea 4(%esi), %esi
	dec %edx
	jne 3b
	adcl $0, %eax
4:	andl $3, %ecx
	jz 7f
	cmpl $2, %ecx
	jb 5f
	movw (%esi),%cx
	leal 2(%esi),%esi
	je 6f
	shll $16,%ecx
5:	movb (%esi),%cl
6:	addl %ecx,%eax
	adcl $0, %eax
7:
	testb $1, 12(%esp)
	jz 8f
	roll $8, %eax
8:
	popl %ebx
	popl %esi
	RET
SYM_FUNC_END(csum_partial)

#else

/* Version for PentiumII/PPro */

SYM_FUNC_START(csum_partial)
	pushl %esi
	pushl %ebx
	movl 20(%esp),%eax	# Function arg: unsigned int sum
	movl 16(%esp),%ecx	# Function arg: int len
	movl 12(%esp),%esi	# Function arg:	const unsigned char *buf

	testl $3, %esi
	jnz 25f
10:
	movl %ecx, %edx
	movl %ecx, %ebx
	andl $0x7c, %ebx
	shrl $7, %ecx
	addl %ebx,%esi
	shrl $2, %ebx
	negl %ebx
	lea 45f(%ebx,%ebx,2), %ebx
	testl %esi, %esi
	JMP_NOSPEC ebx

	# Handle 2-byte-aligned regions
20:	addw (%esi), %ax
	lea 2(%esi), %esi
	adcl $0, %eax
	jmp 10b
25:
	testl $1, %esi
	jz 30f
	# buf is odd
	dec %ecx
	jl 90f
	movzbl (%esi), %ebx
	addl %ebx, %eax
	adcl $0, %eax
	roll $8, %eax
	inc %esi
	testl $2, %esi
	jz 10b

30:	subl $2, %ecx
	ja 20b
	je 32f
	addl $2, %ecx
	jz 80f
	movzbl (%esi),%ebx	# csumming 1 byte, 2-aligned
	addl %ebx, %eax
	adcl $0, %eax
	jmp 80f
32:
	addw (%esi), %ax	# csumming 2 bytes, 2-aligned
	adcl $0, %eax
	jmp 80f

40:
	addl -128(%esi), %eax
	adcl -124(%esi), %eax
	adcl -120(%esi), %eax
	adcl -116(%esi), %eax
	adcl -112(%esi), %eax
	adcl -108(%esi), %eax
	adcl -104(%esi), %eax
	adcl -100(%esi), %eax
	adcl -96(%esi), %eax
	adcl -92(%esi), %eax
	adcl -88(%esi), %eax
	adcl -84(%esi), %eax
	adcl -80(%esi), %eax
	adcl -76(%esi), %eax
	adcl -72(%esi), %eax
	adcl -68(%esi), %eax
	adcl -64(%esi), %eax
	adcl -60(%esi), %eax
	adcl -56(%esi), %eax
	adcl -52(%esi), %eax
	adcl -48(%esi), %eax
	adcl -44(%esi), %eax
	adcl -40(%esi), %eax
	adcl -36(%esi), %eax
	adcl -32(%esi), %eax
	adcl -28(%esi), %eax
	adcl -24(%esi), %eax
	adcl -20(%esi), %eax
	adcl -16(%esi), %eax
	adcl -12(%esi), %eax
	adcl -8(%esi), %eax
	adcl -4(%esi), %eax
45:
	lea 128(%esi), %esi
	adcl $0, %eax
	dec %ecx
	jge 40b
	movl %edx, %ecx
50:	andl $3, %ecx
	jz 80f

	# Handle the last 1-3 bytes without jumping
	notl %ecx		# 1->2, 2->1, 3->0, higher bits are masked
	movl $0xffffff,%ebx	# by the shll and shrl instructions
	shll $3,%ecx
	shrl %cl,%ebx
	andl -128(%esi),%ebx	# esi is 4-aligned so should be ok
	addl %ebx,%eax
	adcl $0,%eax
80:
	testb $1, 12(%esp)
	jz 90f
	roll $8, %eax
90:
	popl %ebx
	popl %esi
	RET
SYM_FUNC_END(csum_partial)

#endif
EXPORT_SYMBOL(csum_partial)

/*
unsigned int csum_partial_copy_generic (const char *src, char *dst,
				  int len)
 */

/*
 * Copy from ds while checksumming, otherwise like csum_partial
 */

#define EXC(y...)						\
	9999: y;						\
	_ASM_EXTABLE_TYPE(9999b, 7f, EX_TYPE_UACCESS | EX_FLAG_CLEAR_AX)

#ifndef CONFIG_X86_USE_PPRO_CHECKSUM

#define ARGBASE 16
#define FP		12

SYM_FUNC_START(csum_partial_copy_generic)
	subl  $4,%esp
	pushl %edi
	pushl %esi
	pushl %ebx
	movl ARGBASE+12(%esp),%ecx	# len
	movl ARGBASE+4(%esp),%esi	# src
	movl ARGBASE+8(%esp),%edi	# dst

	movl $-1, %eax			# sum
	testl $2, %edi			# Check alignment.
	jz 2f				# Jump if alignment is ok.
	subl $2, %ecx			# Alignment uses up two bytes.
	jae 1f				# Jump if we had at least two bytes.
	addl $2, %ecx			# ecx was < 2.  Deal with it.
	jmp 4f
EXC(1:	movw (%esi), %bx	)
	addl $2, %esi
EXC(	movw %bx, (%edi)	)
	addl $2, %edi
	addw %bx, %ax
	adcl $0, %eax
2:
	movl %ecx, FP(%esp)
	shrl $5, %ecx
	jz 2f
	testl %esi, %esi		# what's wrong with clc?
EXC(1:	movl (%esi), %ebx	)
EXC(	movl 4(%esi), %edx	)
	adcl %ebx, %eax
EXC(	movl %ebx, (%edi)	)
	adcl %edx, %eax
EXC(	movl %edx, 4(%edi)	)

EXC(	movl 8(%esi), %ebx	)
EXC(	movl 12(%esi), %edx	)
	adcl %ebx, %eax
EXC(	movl %ebx, 8(%edi)	)
	adcl %edx, %eax
EXC(	movl %edx, 12(%edi)	)

EXC(	movl 16(%esi), %ebx 	)
EXC(	movl 20(%esi), %edx	)
	adcl %ebx, %eax
EXC(	movl %ebx, 16(%edi)	)
	adcl %edx, %eax
EXC(	movl %edx, 20(%edi)	)

EXC(	movl 24(%esi), %ebx	)
EXC(	movl 28(%esi), %edx	)
	adcl %ebx, %eax
EXC(	movl %ebx, 24(%edi)	)
	adcl %edx, %eax
EXC(	movl %edx, 28(%edi)	)

	lea 32(%esi), %esi
	lea 32(%edi), %edi
	dec %ecx
	jne 1b
	adcl $0, %eax
2:	movl FP(%esp), %edx
	movl %edx, %ecx
	andl $0x1c, %edx
	je 4f
	shrl $2, %edx			# This clears CF
EXC(3:	movl (%esi), %ebx	)
	adcl %ebx, %eax
EXC(	movl %ebx, (%edi)	)
	lea 4(%esi), %esi
	lea 4(%edi), %edi
	dec %edx
	jne 3b
	adcl $0, %eax
4:	andl $3, %ecx
	jz 7f
	cmpl $2, %ecx
	jb 5f
EXC(	movw (%esi), %cx	)
	leal 2(%esi), %esi
EXC(	movw %cx, (%edi)	)
	leal 2(%edi), %edi
	je 6f
	shll $16,%ecx
EXC(5:	movb (%esi), %cl	)
EXC(	movb %cl, (%edi)	)
6:	addl %ecx, %eax
	adcl $0, %eax
7:

	popl %ebx
	popl %esi
	popl %edi
	popl %ecx			# equivalent to addl $4,%esp
	RET
SYM_FUNC_END(csum_partial_copy_generic)

#else

/* Version for PentiumII/PPro */

#define ROUND1(x) \
	EXC(movl x(%esi), %ebx	)	;	\
	addl %ebx, %eax			;	\
	EXC(movl %ebx, x(%edi)	)	;

#define ROUND(x) \
	EXC(movl x(%esi), %ebx	)	;	\
	adcl %ebx, %eax			;	\
	EXC(movl %ebx, x(%edi)	)	;

#define ARGBASE 12

SYM_FUNC_START(csum_partial_copy_generic)
	pushl %ebx
	pushl %edi
	pushl %esi
	movl ARGBASE+4(%esp),%esi	#src
	movl ARGBASE+8(%esp),%edi	#dst
	movl ARGBASE+12(%esp),%ecx	#len
	movl $-1, %eax			#sum
#	movl %ecx, %edx
	movl %ecx, %ebx
	movl %esi, %edx
	shrl $6, %ecx
	andl $0x3c, %ebx
	negl %ebx
	subl %ebx, %esi
	subl %ebx, %edi
	lea  -1(%esi),%edx
	andl $-32,%edx
	lea 3f(%ebx,%ebx), %ebx
	testl %esi, %esi
	JMP_NOSPEC ebx
1:	addl $64,%esi
	addl $64,%edi
	EXC(movb -32(%edx),%bl)	; EXC(movb (%edx),%bl)
	ROUND1(-64) ROUND(-60) ROUND(-56) ROUND(-52)
	ROUND (-48) ROUND(-44) ROUND(-40) ROUND(-36)
	ROUND (-32) ROUND(-28) ROUND(-24) ROUND(-20)
	ROUND (-16) ROUND(-12) ROUND(-8)  ROUND(-4)
3:	adcl $0,%eax
	addl $64, %edx
	dec %ecx
	jge 1b
4:	movl ARGBASE+12(%esp),%edx	#len
	andl $3, %edx
	jz 7f
	cmpl $2, %edx
	jb 5f
EXC(	movw (%esi), %dx         )
	leal 2(%esi), %esi
EXC(	movw %dx, (%edi)         )
	leal 2(%edi), %edi
	je 6f
	shll $16,%edx
5:
EXC(	movb (%esi), %dl         )
EXC(	movb %dl, (%edi)         )
6:	addl %edx, %eax
	adcl $0, %eax
7:

	popl %esi
	popl %edi
	popl %ebx
	RET
SYM_FUNC_END(csum_partial_copy_generic)

#undef ROUND
#undef ROUND1

#endif
EXPORT_SYMBOL(csum_partial_copy_generic)