162306a36Sopenharmony_ci/* SPDX-License-Identifier: GPL-2.0 */ 262306a36Sopenharmony_ci/* 362306a36Sopenharmony_ci * 462306a36Sopenharmony_ci * Optimized version of the standard copy_page() function 562306a36Sopenharmony_ci * 662306a36Sopenharmony_ci * Inputs: 762306a36Sopenharmony_ci * in0: address of target page 862306a36Sopenharmony_ci * in1: address of source page 962306a36Sopenharmony_ci * Output: 1062306a36Sopenharmony_ci * no return value 1162306a36Sopenharmony_ci * 1262306a36Sopenharmony_ci * Copyright (C) 1999, 2001 Hewlett-Packard Co 1362306a36Sopenharmony_ci * Stephane Eranian <eranian@hpl.hp.com> 1462306a36Sopenharmony_ci * David Mosberger <davidm@hpl.hp.com> 1562306a36Sopenharmony_ci * 1662306a36Sopenharmony_ci * 4/06/01 davidm Tuned to make it perform well both for cached and uncached copies. 1762306a36Sopenharmony_ci */ 1862306a36Sopenharmony_ci#include <linux/export.h> 1962306a36Sopenharmony_ci#include <asm/asmmacro.h> 2062306a36Sopenharmony_ci#include <asm/page.h> 2162306a36Sopenharmony_ci 2262306a36Sopenharmony_ci#define PIPE_DEPTH 3 2362306a36Sopenharmony_ci#define EPI p[PIPE_DEPTH-1] 2462306a36Sopenharmony_ci 2562306a36Sopenharmony_ci#define lcount r16 2662306a36Sopenharmony_ci#define saved_pr r17 2762306a36Sopenharmony_ci#define saved_lc r18 2862306a36Sopenharmony_ci#define saved_pfs r19 2962306a36Sopenharmony_ci#define src1 r20 3062306a36Sopenharmony_ci#define src2 r21 3162306a36Sopenharmony_ci#define tgt1 r22 3262306a36Sopenharmony_ci#define tgt2 r23 3362306a36Sopenharmony_ci#define srcf r24 3462306a36Sopenharmony_ci#define tgtf r25 3562306a36Sopenharmony_ci#define tgt_last r26 3662306a36Sopenharmony_ci 3762306a36Sopenharmony_ci#define Nrot ((8*PIPE_DEPTH+7)&~7) 3862306a36Sopenharmony_ci 3962306a36Sopenharmony_ciGLOBAL_ENTRY(copy_page) 4062306a36Sopenharmony_ci .prologue 4162306a36Sopenharmony_ci .save ar.pfs, saved_pfs 4262306a36Sopenharmony_ci alloc saved_pfs=ar.pfs,3,Nrot-3,0,Nrot 4362306a36Sopenharmony_ci 4462306a36Sopenharmony_ci .rotr t1[PIPE_DEPTH], t2[PIPE_DEPTH], t3[PIPE_DEPTH], t4[PIPE_DEPTH], \ 4562306a36Sopenharmony_ci t5[PIPE_DEPTH], t6[PIPE_DEPTH], t7[PIPE_DEPTH], t8[PIPE_DEPTH] 4662306a36Sopenharmony_ci .rotp p[PIPE_DEPTH] 4762306a36Sopenharmony_ci 4862306a36Sopenharmony_ci .save ar.lc, saved_lc 4962306a36Sopenharmony_ci mov saved_lc=ar.lc 5062306a36Sopenharmony_ci mov ar.ec=PIPE_DEPTH 5162306a36Sopenharmony_ci 5262306a36Sopenharmony_ci mov lcount=PAGE_SIZE/64-1 5362306a36Sopenharmony_ci .save pr, saved_pr 5462306a36Sopenharmony_ci mov saved_pr=pr 5562306a36Sopenharmony_ci mov pr.rot=1<<16 5662306a36Sopenharmony_ci 5762306a36Sopenharmony_ci .body 5862306a36Sopenharmony_ci 5962306a36Sopenharmony_ci mov src1=in1 6062306a36Sopenharmony_ci adds src2=8,in1 6162306a36Sopenharmony_ci mov tgt_last = PAGE_SIZE 6262306a36Sopenharmony_ci ;; 6362306a36Sopenharmony_ci adds tgt2=8,in0 6462306a36Sopenharmony_ci add srcf=512,in1 6562306a36Sopenharmony_ci mov ar.lc=lcount 6662306a36Sopenharmony_ci mov tgt1=in0 6762306a36Sopenharmony_ci add tgtf=512,in0 6862306a36Sopenharmony_ci add tgt_last = tgt_last, in0 6962306a36Sopenharmony_ci ;; 7062306a36Sopenharmony_ci1: 7162306a36Sopenharmony_ci(p[0]) ld8 t1[0]=[src1],16 7262306a36Sopenharmony_ci(EPI) st8 [tgt1]=t1[PIPE_DEPTH-1],16 7362306a36Sopenharmony_ci(p[0]) ld8 t2[0]=[src2],16 7462306a36Sopenharmony_ci(EPI) st8 [tgt2]=t2[PIPE_DEPTH-1],16 7562306a36Sopenharmony_ci cmp.ltu p6,p0 = tgtf, tgt_last 7662306a36Sopenharmony_ci ;; 7762306a36Sopenharmony_ci(p[0]) ld8 t3[0]=[src1],16 7862306a36Sopenharmony_ci(EPI) st8 [tgt1]=t3[PIPE_DEPTH-1],16 7962306a36Sopenharmony_ci(p[0]) ld8 t4[0]=[src2],16 8062306a36Sopenharmony_ci(EPI) st8 [tgt2]=t4[PIPE_DEPTH-1],16 8162306a36Sopenharmony_ci ;; 8262306a36Sopenharmony_ci(p[0]) ld8 t5[0]=[src1],16 8362306a36Sopenharmony_ci(EPI) st8 [tgt1]=t5[PIPE_DEPTH-1],16 8462306a36Sopenharmony_ci(p[0]) ld8 t6[0]=[src2],16 8562306a36Sopenharmony_ci(EPI) st8 [tgt2]=t6[PIPE_DEPTH-1],16 8662306a36Sopenharmony_ci ;; 8762306a36Sopenharmony_ci(p[0]) ld8 t7[0]=[src1],16 8862306a36Sopenharmony_ci(EPI) st8 [tgt1]=t7[PIPE_DEPTH-1],16 8962306a36Sopenharmony_ci(p[0]) ld8 t8[0]=[src2],16 9062306a36Sopenharmony_ci(EPI) st8 [tgt2]=t8[PIPE_DEPTH-1],16 9162306a36Sopenharmony_ci 9262306a36Sopenharmony_ci(p6) lfetch [srcf], 64 9362306a36Sopenharmony_ci(p6) lfetch [tgtf], 64 9462306a36Sopenharmony_ci br.ctop.sptk.few 1b 9562306a36Sopenharmony_ci ;; 9662306a36Sopenharmony_ci mov pr=saved_pr,0xffffffffffff0000 // restore predicates 9762306a36Sopenharmony_ci mov ar.pfs=saved_pfs 9862306a36Sopenharmony_ci mov ar.lc=saved_lc 9962306a36Sopenharmony_ci br.ret.sptk.many rp 10062306a36Sopenharmony_ciEND(copy_page) 10162306a36Sopenharmony_ciEXPORT_SYMBOL(copy_page) 102