xref: /kernel/linux/linux-6.6/arch/powerpc/crypto/poly1305-p10le_64.S (revision 62306a36)

/* SPDX-License-Identifier: GPL-2.0-or-later */
#
# Accelerated poly1305 implementation for ppc64le.
#
# Copyright 2023- IBM Corp. All rights reserved
#
#===================================================================================
# Written by Danny Tsen <dtsen@us.ibm.com>
#
# Poly1305 - this version mainly using vector/VSX/Scalar
#  - 26 bits limbs
#  - Handle multiple 64 byte blcok.
#
# Block size 16 bytes
# key = (r, s)
# clamp r &= 0x0FFFFFFC0FFFFFFC 0x0FFFFFFC0FFFFFFF
# p = 2^130 - 5
# a += m
# a = (r + a) % p
# a += s
#
# Improve performance by breaking down polynominal to the sum of products with
#     h4 = m1 * r⁴ + m2 * r³ + m3 * r² + m4 * r
#
#  07/22/21 - this revison based on the above sum of products.  Setup r^4, r^3, r^2, r and s3, s2, s1, s0
#             to 9 vectors for multiplications.
#
# setup r^4, r^3, r^2, r vectors
#    vs    [r^1, r^3, r^2, r^4]
#    vs0 = [r0,.....]
#    vs1 = [r1,.....]
#    vs2 = [r2,.....]
#    vs3 = [r3,.....]
#    vs4 = [r4,.....]
#    vs5 = [r1*5,...]
#    vs6 = [r2*5,...]
#    vs7 = [r2*5,...]
#    vs8 = [r4*5,...]
#
#  Each word in a vector consists a member of a "r/s" in [a * r/s].
#
# r0, r4*5, r3*5, r2*5, r1*5;
# r1, r0,   r4*5, r3*5, r2*5;
# r2, r1,   r0,   r4*5, r3*5;
# r3, r2,   r1,   r0,   r4*5;
# r4, r3,   r2,   r1,   r0  ;
#
#
# poly1305_p10le_4blocks( uint8_t *k, uint32_t mlen, uint8_t *m)
#  k = 32 bytes key
#  r3 = k (r, s)
#  r4 = mlen
#  r5 = m
#
#include <asm/ppc_asm.h>
#include <asm/asm-offsets.h>
#include <asm/asm-compat.h>
#include <linux/linkage.h>

.machine "any"

.text

.macro	SAVE_GPR GPR OFFSET FRAME
	std	\GPR,\OFFSET(\FRAME)
.endm

.macro	SAVE_VRS VRS OFFSET FRAME
	li	16, \OFFSET
	stvx	\VRS, 16, \FRAME
.endm

.macro	SAVE_VSX VSX OFFSET FRAME
	li	16, \OFFSET
	stxvx	\VSX, 16, \FRAME
.endm

.macro	RESTORE_GPR GPR OFFSET FRAME
	ld	\GPR,\OFFSET(\FRAME)
.endm

.macro	RESTORE_VRS VRS OFFSET FRAME
	li	16, \OFFSET
	lvx	\VRS, 16, \FRAME
.endm

.macro	RESTORE_VSX VSX OFFSET FRAME
	li	16, \OFFSET
	lxvx	\VSX, 16, \FRAME
.endm

.macro SAVE_REGS
	mflr 0
	std 0, 16(1)
	stdu 1,-752(1)

	SAVE_GPR 14, 112, 1
	SAVE_GPR 15, 120, 1
	SAVE_GPR 16, 128, 1
	SAVE_GPR 17, 136, 1
	SAVE_GPR 18, 144, 1
	SAVE_GPR 19, 152, 1
	SAVE_GPR 20, 160, 1
	SAVE_GPR 21, 168, 1
	SAVE_GPR 22, 176, 1
	SAVE_GPR 23, 184, 1
	SAVE_GPR 24, 192, 1
	SAVE_GPR 25, 200, 1
	SAVE_GPR 26, 208, 1
	SAVE_GPR 27, 216, 1
	SAVE_GPR 28, 224, 1
	SAVE_GPR 29, 232, 1
	SAVE_GPR 30, 240, 1
	SAVE_GPR 31, 248, 1

	addi	9, 1, 256
	SAVE_VRS 20, 0, 9
	SAVE_VRS 21, 16, 9
	SAVE_VRS 22, 32, 9
	SAVE_VRS 23, 48, 9
	SAVE_VRS 24, 64, 9
	SAVE_VRS 25, 80, 9
	SAVE_VRS 26, 96, 9
	SAVE_VRS 27, 112, 9
	SAVE_VRS 28, 128, 9
	SAVE_VRS 29, 144, 9
	SAVE_VRS 30, 160, 9
	SAVE_VRS 31, 176, 9

	SAVE_VSX 14, 192, 9
	SAVE_VSX 15, 208, 9
	SAVE_VSX 16, 224, 9
	SAVE_VSX 17, 240, 9
	SAVE_VSX 18, 256, 9
	SAVE_VSX 19, 272, 9
	SAVE_VSX 20, 288, 9
	SAVE_VSX 21, 304, 9
	SAVE_VSX 22, 320, 9
	SAVE_VSX 23, 336, 9
	SAVE_VSX 24, 352, 9
	SAVE_VSX 25, 368, 9
	SAVE_VSX 26, 384, 9
	SAVE_VSX 27, 400, 9
	SAVE_VSX 28, 416, 9
	SAVE_VSX 29, 432, 9
	SAVE_VSX 30, 448, 9
	SAVE_VSX 31, 464, 9
.endm # SAVE_REGS

.macro RESTORE_REGS
	addi	9, 1, 256
	RESTORE_VRS 20, 0, 9
	RESTORE_VRS 21, 16, 9
	RESTORE_VRS 22, 32, 9
	RESTORE_VRS 23, 48, 9
	RESTORE_VRS 24, 64, 9
	RESTORE_VRS 25, 80, 9
	RESTORE_VRS 26, 96, 9
	RESTORE_VRS 27, 112, 9
	RESTORE_VRS 28, 128, 9
	RESTORE_VRS 29, 144, 9
	RESTORE_VRS 30, 160, 9
	RESTORE_VRS 31, 176, 9

	RESTORE_VSX 14, 192, 9
	RESTORE_VSX 15, 208, 9
	RESTORE_VSX 16, 224, 9
	RESTORE_VSX 17, 240, 9
	RESTORE_VSX 18, 256, 9
	RESTORE_VSX 19, 272, 9
	RESTORE_VSX 20, 288, 9
	RESTORE_VSX 21, 304, 9
	RESTORE_VSX 22, 320, 9
	RESTORE_VSX 23, 336, 9
	RESTORE_VSX 24, 352, 9
	RESTORE_VSX 25, 368, 9
	RESTORE_VSX 26, 384, 9
	RESTORE_VSX 27, 400, 9
	RESTORE_VSX 28, 416, 9
	RESTORE_VSX 29, 432, 9
	RESTORE_VSX 30, 448, 9
	RESTORE_VSX 31, 464, 9

	RESTORE_GPR 14, 112, 1
	RESTORE_GPR 15, 120, 1
	RESTORE_GPR 16, 128, 1
	RESTORE_GPR 17, 136, 1
	RESTORE_GPR 18, 144, 1
	RESTORE_GPR 19, 152, 1
	RESTORE_GPR 20, 160, 1
	RESTORE_GPR 21, 168, 1
	RESTORE_GPR 22, 176, 1
	RESTORE_GPR 23, 184, 1
	RESTORE_GPR 24, 192, 1
	RESTORE_GPR 25, 200, 1
	RESTORE_GPR 26, 208, 1
	RESTORE_GPR 27, 216, 1
	RESTORE_GPR 28, 224, 1
	RESTORE_GPR 29, 232, 1
	RESTORE_GPR 30, 240, 1
	RESTORE_GPR 31, 248, 1

	addi    1, 1, 752
	ld 0, 16(1)
	mtlr 0
.endm # RESTORE_REGS

#
# p[0] = a0*r0 + a1*r4*5 + a2*r3*5 + a3*r2*5 + a4*r1*5;
# p[1] = a0*r1 + a1*r0   + a2*r4*5 + a3*r3*5 + a4*r2*5;
# p[2] = a0*r2 + a1*r1   + a2*r0   + a3*r4*5 + a4*r3*5;
# p[3] = a0*r3 + a1*r2   + a2*r1   + a3*r0   + a4*r4*5;
# p[4] = a0*r4 + a1*r3   + a2*r2   + a3*r1   + a4*r0  ;
#
#    [r^2, r^3, r^1, r^4]
#    [m3,  m2,  m4,  m1]
#
# multiply odd and even words
.macro mul_odd
	vmulouw	14, 4, 26
	vmulouw	10, 5, 3
	vmulouw	11, 6, 2
	vmulouw	12, 7, 1
	vmulouw	13, 8, 0
	vmulouw	15, 4, 27
	vaddudm	14, 14, 10
	vaddudm	14, 14, 11
	vmulouw	10, 5, 26
	vmulouw	11, 6, 3
	vaddudm	14, 14, 12
	vaddudm	14, 14, 13	# x0
	vaddudm	15, 15, 10
	vaddudm	15, 15, 11
	vmulouw	12, 7, 2
	vmulouw	13, 8, 1
	vaddudm	15, 15, 12
	vaddudm	15, 15, 13	# x1
	vmulouw	16, 4, 28
	vmulouw	10, 5, 27
	vmulouw	11, 6, 26
	vaddudm	16, 16, 10
	vaddudm	16, 16, 11
	vmulouw	12, 7, 3
	vmulouw	13, 8, 2
	vaddudm	16, 16, 12
	vaddudm	16, 16, 13	# x2
	vmulouw	17, 4, 29
	vmulouw	10, 5, 28
	vmulouw	11, 6, 27
	vaddudm	17, 17, 10
	vaddudm	17, 17, 11
	vmulouw	12, 7, 26
	vmulouw	13, 8, 3
	vaddudm	17, 17, 12
	vaddudm	17, 17, 13	# x3
	vmulouw	18, 4, 30
	vmulouw	10, 5, 29
	vmulouw	11, 6, 28
	vaddudm	18, 18, 10
	vaddudm	18, 18, 11
	vmulouw	12, 7, 27
	vmulouw	13, 8, 26
	vaddudm	18, 18, 12
	vaddudm	18, 18, 13	# x4
.endm

.macro mul_even
	vmuleuw	9, 4, 26
	vmuleuw	10, 5, 3
	vmuleuw	11, 6, 2
	vmuleuw	12, 7, 1
	vmuleuw	13, 8, 0
	vaddudm	14, 14, 9
	vaddudm	14, 14, 10
	vaddudm	14, 14, 11
	vaddudm	14, 14, 12
	vaddudm	14, 14, 13	# x0

	vmuleuw	9, 4, 27
	vmuleuw	10, 5, 26
	vmuleuw	11, 6, 3
	vmuleuw	12, 7, 2
	vmuleuw	13, 8, 1
	vaddudm	15, 15, 9
	vaddudm	15, 15, 10
	vaddudm	15, 15, 11
	vaddudm	15, 15, 12
	vaddudm	15, 15, 13	# x1

	vmuleuw	9, 4, 28
	vmuleuw	10, 5, 27
	vmuleuw	11, 6, 26
	vmuleuw	12, 7, 3
	vmuleuw	13, 8, 2
	vaddudm	16, 16, 9
	vaddudm	16, 16, 10
	vaddudm	16, 16, 11
	vaddudm	16, 16, 12
	vaddudm	16, 16, 13	# x2

	vmuleuw	9, 4, 29
	vmuleuw	10, 5, 28
	vmuleuw	11, 6, 27
	vmuleuw	12, 7, 26
	vmuleuw	13, 8, 3
	vaddudm	17, 17, 9
	vaddudm	17, 17, 10
	vaddudm	17, 17, 11
	vaddudm	17, 17, 12
	vaddudm	17, 17, 13	# x3

	vmuleuw	9, 4, 30
	vmuleuw	10, 5, 29
	vmuleuw	11, 6, 28
	vmuleuw	12, 7, 27
	vmuleuw	13, 8, 26
	vaddudm	18, 18, 9
	vaddudm	18, 18, 10
	vaddudm	18, 18, 11
	vaddudm	18, 18, 12
	vaddudm	18, 18, 13	# x4
.endm

#
# poly1305_setup_r
#
# setup r^4, r^3, r^2, r vectors
#    [r, r^3, r^2, r^4]
#    vs0 = [r0,...]
#    vs1 = [r1,...]
#    vs2 = [r2,...]
#    vs3 = [r3,...]
#    vs4 = [r4,...]
#    vs5 = [r4*5,...]
#    vs6 = [r3*5,...]
#    vs7 = [r2*5,...]
#    vs8 = [r1*5,...]
#
# r0, r4*5, r3*5, r2*5, r1*5;
# r1, r0,   r4*5, r3*5, r2*5;
# r2, r1,   r0,   r4*5, r3*5;
# r3, r2,   r1,   r0,   r4*5;
# r4, r3,   r2,   r1,   r0  ;
#
.macro poly1305_setup_r

	# save r
	xxlor	26, 58, 58
	xxlor	27, 59, 59
	xxlor	28, 60, 60
	xxlor	29, 61, 61
	xxlor	30, 62, 62

	xxlxor	31, 31, 31

#    [r, r^3, r^2, r^4]
	# compute r^2
	vmr	4, 26
	vmr	5, 27
	vmr	6, 28
	vmr	7, 29
	vmr	8, 30
	bl	do_mul		# r^2 r^1
	xxpermdi 58, 58, 36, 0x3		# r0
	xxpermdi 59, 59, 37, 0x3		# r1
	xxpermdi 60, 60, 38, 0x3		# r2
	xxpermdi 61, 61, 39, 0x3		# r3
	xxpermdi 62, 62, 40, 0x3		# r4
	xxpermdi 36, 36, 36, 0x3
	xxpermdi 37, 37, 37, 0x3
	xxpermdi 38, 38, 38, 0x3
	xxpermdi 39, 39, 39, 0x3
	xxpermdi 40, 40, 40, 0x3
	vspltisb 13, 2
	vsld	9, 27, 13
	vsld	10, 28, 13
	vsld	11, 29, 13
	vsld	12, 30, 13
	vaddudm	0, 9, 27
	vaddudm	1, 10, 28
	vaddudm	2, 11, 29
	vaddudm	3, 12, 30

	bl	do_mul		# r^4 r^3
	vmrgow	26, 26, 4
	vmrgow	27, 27, 5
	vmrgow	28, 28, 6
	vmrgow	29, 29, 7
	vmrgow	30, 30, 8
	vspltisb 13, 2
	vsld	9, 27, 13
	vsld	10, 28, 13
	vsld	11, 29, 13
	vsld	12, 30, 13
	vaddudm	0, 9, 27
	vaddudm	1, 10, 28
	vaddudm	2, 11, 29
	vaddudm	3, 12, 30

	# r^2 r^4
	xxlor	0, 58, 58
	xxlor	1, 59, 59
	xxlor	2, 60, 60
	xxlor	3, 61, 61
	xxlor	4, 62, 62
	xxlor	5, 32, 32
	xxlor	6, 33, 33
	xxlor	7, 34, 34
	xxlor	8, 35, 35

	vspltw	9, 26, 3
	vspltw	10, 26, 2
	vmrgow	26, 10, 9
	vspltw	9, 27, 3
	vspltw	10, 27, 2
	vmrgow	27, 10, 9
	vspltw	9, 28, 3
	vspltw	10, 28, 2
	vmrgow	28, 10, 9
	vspltw	9, 29, 3
	vspltw	10, 29, 2
	vmrgow	29, 10, 9
	vspltw	9, 30, 3
	vspltw	10, 30, 2
	vmrgow	30, 10, 9

	vsld	9, 27, 13
	vsld	10, 28, 13
	vsld	11, 29, 13
	vsld	12, 30, 13
	vaddudm	0, 9, 27
	vaddudm	1, 10, 28
	vaddudm	2, 11, 29
	vaddudm	3, 12, 30
.endm

SYM_FUNC_START_LOCAL(do_mul)
	mul_odd

	# do reduction ( h %= p )
	# carry reduction
	vspltisb 9, 2
	vsrd	10, 14, 31
	vsrd	11, 17, 31
	vand	7, 17, 25
	vand	4, 14, 25
	vaddudm	18, 18, 11
	vsrd	12, 18, 31
	vaddudm	15, 15, 10

	vsrd	11, 15, 31
	vand	8, 18, 25
	vand	5, 15, 25
	vaddudm	4, 4, 12
	vsld	10, 12, 9
	vaddudm	6, 16, 11

	vsrd	13, 6, 31
	vand	6, 6, 25
	vaddudm	4, 4, 10
	vsrd	10, 4, 31
	vaddudm	7, 7, 13

	vsrd	11, 7, 31
	vand	7, 7, 25
	vand	4, 4, 25
	vaddudm	5, 5, 10
	vaddudm	8, 8, 11
	blr
SYM_FUNC_END(do_mul)

#
# init key
#
.macro do_poly1305_init
	addis	10, 2, rmask@toc@ha
	addi	10, 10, rmask@toc@l

	ld	11, 0(10)
	ld	12, 8(10)

	li	14, 16
	li	15, 32
	addis	10, 2, cnum@toc@ha
	addi	10, 10, cnum@toc@l
	lvx	25, 0, 10	# v25 - mask
	lvx	31, 14, 10	# v31 = 1a
	lvx	19, 15, 10	# v19 = 1 << 24
	lxv	24, 48(10)	# vs24
	lxv	25, 64(10)	# vs25

	# initialize
	# load key from r3 to vectors
	ld	9, 24(3)
	ld	10, 32(3)
	and.	9, 9, 11
	and.	10, 10, 12

	# break 26 bits
	extrdi	14, 9, 26, 38
	extrdi	15, 9, 26, 12
	extrdi	16, 9, 12, 0
	mtvsrdd	58, 0, 14
	insrdi	16, 10, 14, 38
	mtvsrdd	59, 0, 15
	extrdi	17, 10, 26, 24
	mtvsrdd	60, 0, 16
	extrdi	18, 10, 24, 0
	mtvsrdd	61, 0, 17
	mtvsrdd	62, 0, 18

	# r1 = r1 * 5, r2 = r2 * 5, r3 = r3 * 5, r4 = r4 * 5
	li	9, 5
	mtvsrdd	36, 0, 9
	vmulouw	0, 27, 4		# v0 = rr0
	vmulouw	1, 28, 4		# v1 = rr1
	vmulouw	2, 29, 4		# v2 = rr2
	vmulouw	3, 30, 4		# v3 = rr3
.endm

#
# poly1305_p10le_4blocks( uint8_t *k, uint32_t mlen, uint8_t *m)
#  k = 32 bytes key
#  r3 = k (r, s)
#  r4 = mlen
#  r5 = m
#
SYM_FUNC_START(poly1305_p10le_4blocks)
.align 5
	cmpdi	5, 64
	blt	Out_no_poly1305

	SAVE_REGS

	do_poly1305_init

	li	21, 0	# counter to message

	poly1305_setup_r

	# load previous H state
	# break/convert r6 to 26 bits
	ld	9, 0(3)
	ld	10, 8(3)
	ld	19, 16(3)
	sldi	19, 19, 24
	mtvsrdd	41, 0, 19
	extrdi	14, 9, 26, 38
	extrdi	15, 9, 26, 12
	extrdi	16, 9, 12, 0
	mtvsrdd	36, 0, 14
	insrdi	16, 10, 14, 38
	mtvsrdd	37, 0, 15
	extrdi	17, 10, 26, 24
	mtvsrdd	38, 0, 16
	extrdi	18, 10, 24, 0
	mtvsrdd	39, 0, 17
	mtvsrdd	40, 0, 18
	vor	8, 8, 9

	# input m1 m2
	add	20, 4, 21
	xxlor	49, 24, 24
	xxlor	50, 25, 25
	lxvw4x	43, 0, 20
	addi	17, 20, 16
	lxvw4x	44, 0, 17
	vperm	14, 11, 12, 17
	vperm	15, 11, 12, 18
	vand	9, 14, 25	# a0
	vsrd	10, 14, 31	# >> 26
	vsrd	11, 10, 31	# 12 bits left
	vand	10, 10, 25	# a1
	vspltisb 13, 12
	vand	16, 15, 25
	vsld	12, 16, 13
	vor	11, 11, 12
	vand	11, 11, 25	# a2
	vspltisb 13, 14
	vsrd	12, 15, 13	# >> 14
	vsrd	13, 12, 31	# >> 26, a4
	vand	12, 12, 25	# a3

	vaddudm	20, 4, 9
	vaddudm	21, 5, 10
	vaddudm	22, 6, 11
	vaddudm	23, 7, 12
	vaddudm	24, 8, 13

	# m3 m4
	addi	17, 17, 16
	lxvw4x	43, 0, 17
	addi	17, 17, 16
	lxvw4x	44, 0, 17
	vperm	14, 11, 12, 17
	vperm	15, 11, 12, 18
	vand	9, 14, 25	# a0
	vsrd	10, 14, 31	# >> 26
	vsrd	11, 10, 31	# 12 bits left
	vand	10, 10, 25	# a1
	vspltisb 13, 12
	vand	16, 15, 25
	vsld	12, 16, 13
	vspltisb 13, 14
	vor	11, 11, 12
	vand	11, 11, 25	# a2
	vsrd	12, 15, 13	# >> 14
	vsrd	13, 12, 31	# >> 26, a4
	vand	12, 12, 25	# a3

	# Smash 4 message blocks into 5 vectors of [m4,  m2,  m3,  m1]
	vmrgow	4, 9, 20
	vmrgow	5, 10, 21
	vmrgow	6, 11, 22
	vmrgow	7, 12, 23
	vmrgow	8, 13, 24
	vaddudm	8, 8, 19

	addi	5, 5, -64	# len -= 64
	addi	21, 21, 64	# offset += 64

	li      9, 64
	divdu   31, 5, 9

	cmpdi	31, 0
	ble	Skip_block_loop

	mtctr	31

# h4 =   m1 * r⁴ + m2 * r³ + m3 * r² + m4 * r
# Rewrite the polynominal sum of product as follows,
# h1 = (h0 + m1) * r^2,	h2 = (h0 + m2) * r^2
# h3 = (h1 + m3) * r^2,	h4 = (h2 + m4) * r^2  --> (h0 + m1) r*4 + (h3 + m3) r^2, (h0 + m2) r^4 + (h0 + m4) r^2
#  .... Repeat
# h5 = (h3 + m5) * r^2,	h6 = (h4 + m6) * r^2  -->
# h7 = (h5 + m7) * r^2,	h8 = (h6 + m8) * r^1  --> m5 * r^4 + m6 * r^3 + m7 * r^2 + m8 * r
#
loop_4blocks:

	# Multiply odd words and even words
	mul_odd
	mul_even
	# carry reduction
	vspltisb 9, 2
	vsrd	10, 14, 31
	vsrd	11, 17, 31
	vand	7, 17, 25
	vand	4, 14, 25
	vaddudm	18, 18, 11
	vsrd	12, 18, 31
	vaddudm	15, 15, 10

	vsrd	11, 15, 31
	vand	8, 18, 25
	vand	5, 15, 25
	vaddudm	4, 4, 12
	vsld	10, 12, 9
	vaddudm	6, 16, 11

	vsrd	13, 6, 31
	vand	6, 6, 25
	vaddudm	4, 4, 10
	vsrd	10, 4, 31
	vaddudm	7, 7, 13

	vsrd	11, 7, 31
	vand	7, 7, 25
	vand	4, 4, 25
	vaddudm	5, 5, 10
	vaddudm	8, 8, 11

	# input m1  m2  m3  m4
	add	20, 4, 21
	xxlor	49, 24, 24
	xxlor	50, 25, 25
	lxvw4x	43, 0, 20
	addi	17, 20, 16
	lxvw4x	44, 0, 17
	vperm	14, 11, 12, 17
	vperm	15, 11, 12, 18
	addi	17, 17, 16
	lxvw4x	43, 0, 17
	addi	17, 17, 16
	lxvw4x	44, 0, 17
	vperm	17, 11, 12, 17
	vperm	18, 11, 12, 18

	vand	20, 14, 25	# a0
	vand	9, 17, 25	# a0
	vsrd	21, 14, 31	# >> 26
	vsrd	22, 21, 31	# 12 bits left
	vsrd	10, 17, 31	# >> 26
	vsrd	11, 10, 31	# 12 bits left

	vand	21, 21, 25	# a1
	vand	10, 10, 25	# a1

	vspltisb 13, 12
	vand	16, 15, 25
	vsld	23, 16, 13
	vor	22, 22, 23
	vand	22, 22, 25	# a2
	vand	16, 18, 25
	vsld	12, 16, 13
	vor	11, 11, 12
	vand	11, 11, 25	# a2
	vspltisb 13, 14
	vsrd	23, 15, 13	# >> 14
	vsrd	24, 23, 31	# >> 26, a4
	vand	23, 23, 25	# a3
	vsrd	12, 18, 13	# >> 14
	vsrd	13, 12, 31	# >> 26, a4
	vand	12, 12, 25	# a3

	vaddudm	4, 4, 20
	vaddudm	5, 5, 21
	vaddudm	6, 6, 22
	vaddudm	7, 7, 23
	vaddudm	8, 8, 24

	# Smash 4 message blocks into 5 vectors of [m4,  m2,  m3,  m1]
	vmrgow	4, 9, 4
	vmrgow	5, 10, 5
	vmrgow	6, 11, 6
	vmrgow	7, 12, 7
	vmrgow	8, 13, 8
	vaddudm	8, 8, 19

	addi	5, 5, -64	# len -= 64
	addi	21, 21, 64	# offset += 64

	bdnz	loop_4blocks

Skip_block_loop:
	xxlor	58, 0, 0
	xxlor	59, 1, 1
	xxlor	60, 2, 2
	xxlor	61, 3, 3
	xxlor	62, 4, 4
	xxlor	32, 5, 5
	xxlor	33, 6, 6
	xxlor	34, 7, 7
	xxlor	35, 8, 8

	# Multiply odd words and even words
	mul_odd
	mul_even

	# Sum the products.
	xxpermdi 41, 31, 46, 0
	xxpermdi 42, 31, 47, 0
	vaddudm	4, 14, 9
	xxpermdi 36, 31, 36, 3
	vaddudm	5, 15, 10
	xxpermdi 37, 31, 37, 3
	xxpermdi 43, 31, 48, 0
	vaddudm	6, 16, 11
	xxpermdi 38, 31, 38, 3
	xxpermdi 44, 31, 49, 0
	vaddudm	7, 17, 12
	xxpermdi 39, 31, 39, 3
	xxpermdi 45, 31, 50, 0
	vaddudm	8, 18, 13
	xxpermdi 40, 31, 40, 3

	# carry reduction
	vspltisb 9, 2
	vsrd	10, 4, 31
	vsrd	11, 7, 31
	vand	7, 7, 25
	vand	4, 4, 25
	vaddudm	8, 8, 11
	vsrd	12, 8, 31
	vaddudm	5, 5, 10

	vsrd	11, 5, 31
	vand	8, 8, 25
	vand	5, 5, 25
	vaddudm	4, 4, 12
	vsld	10, 12, 9
	vaddudm	6, 6, 11

	vsrd	13, 6, 31
	vand	6, 6, 25
	vaddudm	4, 4, 10
	vsrd	10, 4, 31
	vaddudm	7, 7, 13

	vsrd	11, 7, 31
	vand	7, 7, 25
	vand	4, 4, 25
	vaddudm	5, 5, 10
	vsrd	10, 5, 31
	vand	5, 5, 25
	vaddudm	6, 6, 10
	vaddudm	8, 8, 11

	b	do_final_update

do_final_update:
	# combine 26 bit limbs
	# v4, v5, v6, v7 and v8 are 26 bit vectors
	vsld	5, 5, 31
	vor	20, 4, 5
	vspltisb 11, 12
	vsrd	12, 6, 11
	vsld	6, 6, 31
	vsld	6, 6, 31
	vor	20, 20, 6
	vspltisb 11, 14
	vsld	7, 7, 11
	vor	21, 7, 12
	mfvsrld	16, 40		# save last 2 bytes
	vsld	8, 8, 11
	vsld	8, 8, 31
	vor	21, 21, 8
	mfvsrld	17, 52
	mfvsrld	19, 53
	srdi	16, 16, 24

	std	17, 0(3)
	std	19, 8(3)
	stw	16, 16(3)

Out_loop:
	li	3, 0

	RESTORE_REGS

	blr

Out_no_poly1305:
	li	3, 0
	blr
SYM_FUNC_END(poly1305_p10le_4blocks)

#
# =======================================================================
# The following functions implement 64 x 64 bits multiplication poly1305.
#
SYM_FUNC_START_LOCAL(Poly1305_init_64)
	#  mask 0x0FFFFFFC0FFFFFFC
	#  mask 0x0FFFFFFC0FFFFFFF
	addis	10, 2, rmask@toc@ha
	addi	10, 10, rmask@toc@l
	ld	11, 0(10)
	ld	12, 8(10)

	# initialize
	# load key from r3
	ld	9, 24(3)
	ld	10, 32(3)
	and.	9, 9, 11	# cramp mask r0
	and.	10, 10, 12	# cramp mask r1

        srdi    21, 10, 2
        add     19, 21, 10      # s1: r19 - (r1 >> 2) *5

        # setup r and s
        li      25, 0
	mtvsrdd 32+0, 9, 19	# r0, s1
	mtvsrdd 32+1, 10, 9	# r1, r0
	mtvsrdd 32+2, 19, 25	# s1
	mtvsrdd 32+3, 9, 25	# r0

	blr
SYM_FUNC_END(Poly1305_init_64)

# Poly1305_mult
# v6 = (h0, h1), v8 = h2
# v0 = (r0, s1), v1 = (r1, r0), v2 = s1, v3 = r0
#
# Output: v7, v10, v11
#
SYM_FUNC_START_LOCAL(Poly1305_mult)
	#
	#	d0 = h0 * r0 + h1 * s1
	vmsumudm	7, 6, 0, 9		# h0 * r0, h1 * s1

	#	d1 = h0 * r1 + h1 * r0 + h2 * s1
	vmsumudm	11, 6, 1, 9		# h0 * r1, h1 * r0
	vmsumudm	10, 8, 2, 11		# d1 += h2 * s1

	#       d2 = r0
	vmsumudm	11, 8, 3, 9		# d2 = h2 * r0
	blr
SYM_FUNC_END(Poly1305_mult)

#
# carry reduction
# h %=p
#
# Input: v7, v10, v11
# Output: r27, r28, r29
#
SYM_FUNC_START_LOCAL(Carry_reduction)
	mfvsrld	27, 32+7
	mfvsrld	28, 32+10
	mfvsrld	29, 32+11
	mfvsrd	20, 32+7	# h0.h
	mfvsrd	21, 32+10	# h1.h

	addc	28, 28, 20
	adde	29, 29, 21
	srdi	22, 29, 0x2
	sldi	23, 22, 0x2
	add	23, 23, 22	# (h2 & 3) * 5
	addc	27, 27, 23	# h0
	addze	28, 28		# h1
	andi.	29, 29, 0x3	# h2
	blr
SYM_FUNC_END(Carry_reduction)

#
# poly1305 multiplication
# h *= r, h %= p
#	d0 = h0 * r0 + h1 * s1
#	d1 = h0 * r1 + h1 * r0 + h2 * s1
#       d2 = h0 * r0
#
#
# unsigned int poly1305_test_64s(unisgned char *state, const byte *src, size_t len, highbit)
#   - no highbit if final leftover block (highbit = 0)
#
SYM_FUNC_START(poly1305_64s)
	cmpdi	5, 0
	ble	Out_no_poly1305_64

	mflr 0
	std 0, 16(1)
	stdu 1,-400(1)

	SAVE_GPR 14, 112, 1
	SAVE_GPR 15, 120, 1
	SAVE_GPR 16, 128, 1
	SAVE_GPR 17, 136, 1
	SAVE_GPR 18, 144, 1
	SAVE_GPR 19, 152, 1
	SAVE_GPR 20, 160, 1
	SAVE_GPR 21, 168, 1
	SAVE_GPR 22, 176, 1
	SAVE_GPR 23, 184, 1
	SAVE_GPR 24, 192, 1
	SAVE_GPR 25, 200, 1
	SAVE_GPR 26, 208, 1
	SAVE_GPR 27, 216, 1
	SAVE_GPR 28, 224, 1
	SAVE_GPR 29, 232, 1
	SAVE_GPR 30, 240, 1
	SAVE_GPR 31, 248, 1

	# Init poly1305
	bl Poly1305_init_64

	li 25, 0			# offset to inp and outp

	add 11, 25, 4

	# load h
	# h0, h1, h2?
        ld	27, 0(3)
        ld	28, 8(3)
        lwz	29, 16(3)

        li      30, 16
        divdu   31, 5, 30

        mtctr   31

        mr      24, 6		# highbit

Loop_block_64:
	vxor	9, 9, 9

	ld	20, 0(11)
	ld	21, 8(11)
	addi	11, 11, 16

	addc	27, 27, 20
	adde	28, 28, 21
	adde	29, 29, 24

	li	22, 0
	mtvsrdd	32+6, 27, 28	# h0, h1
	mtvsrdd	32+8, 29, 22	# h2

	bl	Poly1305_mult

	bl	Carry_reduction

	bdnz	Loop_block_64

	std	27, 0(3)
	std	28, 8(3)
	stw	29, 16(3)

	li	3, 0

	RESTORE_GPR 14, 112, 1
	RESTORE_GPR 15, 120, 1
	RESTORE_GPR 16, 128, 1
	RESTORE_GPR 17, 136, 1
	RESTORE_GPR 18, 144, 1
	RESTORE_GPR 19, 152, 1
	RESTORE_GPR 20, 160, 1
	RESTORE_GPR 21, 168, 1
	RESTORE_GPR 22, 176, 1
	RESTORE_GPR 23, 184, 1
	RESTORE_GPR 24, 192, 1
	RESTORE_GPR 25, 200, 1
	RESTORE_GPR 26, 208, 1
	RESTORE_GPR 27, 216, 1
	RESTORE_GPR 28, 224, 1
	RESTORE_GPR 29, 232, 1
	RESTORE_GPR 30, 240, 1
	RESTORE_GPR 31, 248, 1

	addi    1, 1, 400
	ld 0, 16(1)
	mtlr 0

	blr

Out_no_poly1305_64:
	li	3, 0
	blr
SYM_FUNC_END(poly1305_64s)

#
# Input: r3 = h, r4 = s, r5 = mac
# mac = h + s
#
SYM_FUNC_START(poly1305_emit_64)
	ld	10, 0(3)
	ld	11, 8(3)
	ld	12, 16(3)

	# compare modulus
	# h + 5 + (-p)
	mr	6, 10
	mr	7, 11
	mr	8, 12
	addic.	6, 6, 5
	addze	7, 7
	addze	8, 8
	srdi	9, 8, 2		# overflow?
	cmpdi	9, 0
	beq	Skip_h64
	mr	10, 6
	mr	11, 7
	mr	12, 8

Skip_h64:
	ld	6, 0(4)
	ld	7, 8(4)
	addc	10, 10, 6
	adde	11, 11, 7
	addze	12, 12

	std	10, 0(5)
	std	11, 8(5)
	blr
SYM_FUNC_END(poly1305_emit_64)

SYM_DATA_START_LOCAL(RMASK)
.align 5
rmask:
.byte	0xff, 0xff, 0xff, 0x0f, 0xfc, 0xff, 0xff, 0x0f, 0xfc, 0xff, 0xff, 0x0f, 0xfc, 0xff, 0xff, 0x0f
cnum:
.long	0x03ffffff, 0x00000000, 0x03ffffff, 0x00000000
.long	0x1a, 0x00, 0x1a, 0x00
.long	0x01000000, 0x01000000, 0x01000000, 0x01000000
.long	0x00010203, 0x04050607, 0x10111213, 0x14151617
.long	0x08090a0b, 0x0c0d0e0f, 0x18191a1b, 0x1c1d1e1f
SYM_DATA_END(RMASK)