162306a36Sopenharmony_ci/* SPDX-License-Identifier: GPL-2.0-or-later */
262306a36Sopenharmony_ci/*
362306a36Sopenharmony_ci *
462306a36Sopenharmony_ci * Copyright (C) IBM Corporation, 2012
562306a36Sopenharmony_ci *
662306a36Sopenharmony_ci * Author: Anton Blanchard <anton@au.ibm.com>
762306a36Sopenharmony_ci */
862306a36Sopenharmony_ci#include <asm/page.h>
962306a36Sopenharmony_ci#include <asm/ppc_asm.h>
1062306a36Sopenharmony_ci
1162306a36Sopenharmony_ci_GLOBAL(copypage_power7)
1262306a36Sopenharmony_ci	/*
1362306a36Sopenharmony_ci	 * We prefetch both the source and destination using enhanced touch
1462306a36Sopenharmony_ci	 * instructions. We use a stream ID of 0 for the load side and
1562306a36Sopenharmony_ci	 * 1 for the store side. Since source and destination are page
1662306a36Sopenharmony_ci	 * aligned we don't need to clear the bottom 7 bits of either
1762306a36Sopenharmony_ci	 * address.
1862306a36Sopenharmony_ci	 */
1962306a36Sopenharmony_ci	ori	r9,r3,1		/* stream=1 => to */
2062306a36Sopenharmony_ci
2162306a36Sopenharmony_ci#ifdef CONFIG_PPC_64K_PAGES
2262306a36Sopenharmony_ci	lis	r7,0x0E01	/* depth=7
2362306a36Sopenharmony_ci				 * units/cachelines=512 */
2462306a36Sopenharmony_ci#else
2562306a36Sopenharmony_ci	lis	r7,0x0E00	/* depth=7 */
2662306a36Sopenharmony_ci	ori	r7,r7,0x1000	/* units/cachelines=32 */
2762306a36Sopenharmony_ci#endif
2862306a36Sopenharmony_ci	ori	r10,r7,1	/* stream=1 */
2962306a36Sopenharmony_ci
3062306a36Sopenharmony_ci	lis	r8,0x8000	/* GO=1 */
3162306a36Sopenharmony_ci	clrldi	r8,r8,32
3262306a36Sopenharmony_ci
3362306a36Sopenharmony_ci	/* setup read stream 0  */
3462306a36Sopenharmony_ci	dcbt	0,r4,0b01000  	/* addr from */
3562306a36Sopenharmony_ci	dcbt	0,r7,0b01010   /* length and depth from */
3662306a36Sopenharmony_ci	/* setup write stream 1 */
3762306a36Sopenharmony_ci	dcbtst	0,r9,0b01000   /* addr to */
3862306a36Sopenharmony_ci	dcbtst	0,r10,0b01010  /* length and depth to */
3962306a36Sopenharmony_ci	eieio
4062306a36Sopenharmony_ci	dcbt	0,r8,0b01010	/* all streams GO */
4162306a36Sopenharmony_ci
4262306a36Sopenharmony_ci#ifdef CONFIG_ALTIVEC
4362306a36Sopenharmony_ci	mflr	r0
4462306a36Sopenharmony_ci	std	r3,-STACKFRAMESIZE+STK_REG(R31)(r1)
4562306a36Sopenharmony_ci	std	r4,-STACKFRAMESIZE+STK_REG(R30)(r1)
4662306a36Sopenharmony_ci	std	r0,16(r1)
4762306a36Sopenharmony_ci	stdu	r1,-STACKFRAMESIZE(r1)
4862306a36Sopenharmony_ci	bl	CFUNC(enter_vmx_ops)
4962306a36Sopenharmony_ci	cmpwi	r3,0
5062306a36Sopenharmony_ci	ld	r0,STACKFRAMESIZE+16(r1)
5162306a36Sopenharmony_ci	ld	r3,STK_REG(R31)(r1)
5262306a36Sopenharmony_ci	ld	r4,STK_REG(R30)(r1)
5362306a36Sopenharmony_ci	mtlr	r0
5462306a36Sopenharmony_ci
5562306a36Sopenharmony_ci	li	r0,(PAGE_SIZE/128)
5662306a36Sopenharmony_ci	mtctr	r0
5762306a36Sopenharmony_ci
5862306a36Sopenharmony_ci	beq	.Lnonvmx_copy
5962306a36Sopenharmony_ci
6062306a36Sopenharmony_ci	addi	r1,r1,STACKFRAMESIZE
6162306a36Sopenharmony_ci
6262306a36Sopenharmony_ci	li	r6,16
6362306a36Sopenharmony_ci	li	r7,32
6462306a36Sopenharmony_ci	li	r8,48
6562306a36Sopenharmony_ci	li	r9,64
6662306a36Sopenharmony_ci	li	r10,80
6762306a36Sopenharmony_ci	li	r11,96
6862306a36Sopenharmony_ci	li	r12,112
6962306a36Sopenharmony_ci
7062306a36Sopenharmony_ci	.align	5
7162306a36Sopenharmony_ci1:	lvx	v7,0,r4
7262306a36Sopenharmony_ci	lvx	v6,r4,r6
7362306a36Sopenharmony_ci	lvx	v5,r4,r7
7462306a36Sopenharmony_ci	lvx	v4,r4,r8
7562306a36Sopenharmony_ci	lvx	v3,r4,r9
7662306a36Sopenharmony_ci	lvx	v2,r4,r10
7762306a36Sopenharmony_ci	lvx	v1,r4,r11
7862306a36Sopenharmony_ci	lvx	v0,r4,r12
7962306a36Sopenharmony_ci	addi	r4,r4,128
8062306a36Sopenharmony_ci	stvx	v7,0,r3
8162306a36Sopenharmony_ci	stvx	v6,r3,r6
8262306a36Sopenharmony_ci	stvx	v5,r3,r7
8362306a36Sopenharmony_ci	stvx	v4,r3,r8
8462306a36Sopenharmony_ci	stvx	v3,r3,r9
8562306a36Sopenharmony_ci	stvx	v2,r3,r10
8662306a36Sopenharmony_ci	stvx	v1,r3,r11
8762306a36Sopenharmony_ci	stvx	v0,r3,r12
8862306a36Sopenharmony_ci	addi	r3,r3,128
8962306a36Sopenharmony_ci	bdnz	1b
9062306a36Sopenharmony_ci
9162306a36Sopenharmony_ci	b	CFUNC(exit_vmx_ops)		/* tail call optimise */
9262306a36Sopenharmony_ci
9362306a36Sopenharmony_ci#else
9462306a36Sopenharmony_ci	li	r0,(PAGE_SIZE/128)
9562306a36Sopenharmony_ci	mtctr	r0
9662306a36Sopenharmony_ci
9762306a36Sopenharmony_ci	stdu	r1,-STACKFRAMESIZE(r1)
9862306a36Sopenharmony_ci#endif
9962306a36Sopenharmony_ci
10062306a36Sopenharmony_ci.Lnonvmx_copy:
10162306a36Sopenharmony_ci	std	r14,STK_REG(R14)(r1)
10262306a36Sopenharmony_ci	std	r15,STK_REG(R15)(r1)
10362306a36Sopenharmony_ci	std	r16,STK_REG(R16)(r1)
10462306a36Sopenharmony_ci	std	r17,STK_REG(R17)(r1)
10562306a36Sopenharmony_ci	std	r18,STK_REG(R18)(r1)
10662306a36Sopenharmony_ci	std	r19,STK_REG(R19)(r1)
10762306a36Sopenharmony_ci	std	r20,STK_REG(R20)(r1)
10862306a36Sopenharmony_ci
10962306a36Sopenharmony_ci1:	ld	r0,0(r4)
11062306a36Sopenharmony_ci	ld	r5,8(r4)
11162306a36Sopenharmony_ci	ld	r6,16(r4)
11262306a36Sopenharmony_ci	ld	r7,24(r4)
11362306a36Sopenharmony_ci	ld	r8,32(r4)
11462306a36Sopenharmony_ci	ld	r9,40(r4)
11562306a36Sopenharmony_ci	ld	r10,48(r4)
11662306a36Sopenharmony_ci	ld	r11,56(r4)
11762306a36Sopenharmony_ci	ld	r12,64(r4)
11862306a36Sopenharmony_ci	ld	r14,72(r4)
11962306a36Sopenharmony_ci	ld	r15,80(r4)
12062306a36Sopenharmony_ci	ld	r16,88(r4)
12162306a36Sopenharmony_ci	ld	r17,96(r4)
12262306a36Sopenharmony_ci	ld	r18,104(r4)
12362306a36Sopenharmony_ci	ld	r19,112(r4)
12462306a36Sopenharmony_ci	ld	r20,120(r4)
12562306a36Sopenharmony_ci	addi	r4,r4,128
12662306a36Sopenharmony_ci	std	r0,0(r3)
12762306a36Sopenharmony_ci	std	r5,8(r3)
12862306a36Sopenharmony_ci	std	r6,16(r3)
12962306a36Sopenharmony_ci	std	r7,24(r3)
13062306a36Sopenharmony_ci	std	r8,32(r3)
13162306a36Sopenharmony_ci	std	r9,40(r3)
13262306a36Sopenharmony_ci	std	r10,48(r3)
13362306a36Sopenharmony_ci	std	r11,56(r3)
13462306a36Sopenharmony_ci	std	r12,64(r3)
13562306a36Sopenharmony_ci	std	r14,72(r3)
13662306a36Sopenharmony_ci	std	r15,80(r3)
13762306a36Sopenharmony_ci	std	r16,88(r3)
13862306a36Sopenharmony_ci	std	r17,96(r3)
13962306a36Sopenharmony_ci	std	r18,104(r3)
14062306a36Sopenharmony_ci	std	r19,112(r3)
14162306a36Sopenharmony_ci	std	r20,120(r3)
14262306a36Sopenharmony_ci	addi	r3,r3,128
14362306a36Sopenharmony_ci	bdnz	1b
14462306a36Sopenharmony_ci
14562306a36Sopenharmony_ci	ld	r14,STK_REG(R14)(r1)
14662306a36Sopenharmony_ci	ld	r15,STK_REG(R15)(r1)
14762306a36Sopenharmony_ci	ld	r16,STK_REG(R16)(r1)
14862306a36Sopenharmony_ci	ld	r17,STK_REG(R17)(r1)
14962306a36Sopenharmony_ci	ld	r18,STK_REG(R18)(r1)
15062306a36Sopenharmony_ci	ld	r19,STK_REG(R19)(r1)
15162306a36Sopenharmony_ci	ld	r20,STK_REG(R20)(r1)
15262306a36Sopenharmony_ci	addi	r1,r1,STACKFRAMESIZE
15362306a36Sopenharmony_ci	blr
154