1 /* SPDX-License-Identifier: GPL-2.0 */ 2 /* 3 * 4 * Optimized version of the standard copy_page() function 5 * 6 * Inputs: 7 * in0: address of target page 8 * in1: address of source page 9 * Output: 10 * no return value 11 * 12 * Copyright (C) 1999, 2001 Hewlett-Packard Co 13 * Stephane Eranian <eranian@hpl.hp.com> 14 * David Mosberger <davidm@hpl.hp.com> 15 * 16 * 4/06/01 davidm Tuned to make it perform well both for cached and uncached copies. 17 */ 18 #include <linux/export.h> 19 #include <asm/asmmacro.h> 20 #include <asm/page.h> 21 22 #define PIPE_DEPTH 3 23 #define EPI p[PIPE_DEPTH-1] 24 25 #define lcount r16 26 #define saved_pr r17 27 #define saved_lc r18 28 #define saved_pfs r19 29 #define src1 r20 30 #define src2 r21 31 #define tgt1 r22 32 #define tgt2 r23 33 #define srcf r24 34 #define tgtf r25 35 #define tgt_last r26 36 37 #define Nrot ((8*PIPE_DEPTH+7)&~7) 38 39 GLOBAL_ENTRY(copy_page) 40 .prologue 41 .save ar.pfs, saved_pfs 42 alloc saved_pfs=ar.pfs,3,Nrot-3,0,Nrot 43 44 .rotr t1[PIPE_DEPTH], t2[PIPE_DEPTH], t3[PIPE_DEPTH], t4[PIPE_DEPTH], \ 45 t5[PIPE_DEPTH], t6[PIPE_DEPTH], t7[PIPE_DEPTH], t8[PIPE_DEPTH] 46 .rotp p[PIPE_DEPTH] 47 48 .save ar.lc, saved_lc 49 mov saved_lc=ar.lc 50 mov ar.ec=PIPE_DEPTH 51 52 mov lcount=PAGE_SIZE/64-1 53 .save pr, saved_pr 54 mov saved_pr=pr 55 mov pr.rot=1<<16 56 57 .body 58 59 mov src1=in1 60 adds src2=8,in1 61 mov tgt_last = PAGE_SIZE 62 ;; 63 adds tgt2=8,in0 64 add srcf=512,in1 65 mov ar.lc=lcount 66 mov tgt1=in0 67 add tgtf=512,in0 68 add tgt_last = tgt_last, in0 69 ;; 70 1: 71 (p[0]) ld8 t1[0]=[src1],16 72 (EPI) st8 [tgt1]=t1[PIPE_DEPTH-1],16 73 (p[0]) ld8 t2[0]=[src2],16 74 (EPI) st8 [tgt2]=t2[PIPE_DEPTH-1],16 75 cmp.ltu p6,p0 = tgtf, tgt_last 76 ;; 77 (p[0]) ld8 t3[0]=[src1],16 78 (EPI) st8 [tgt1]=t3[PIPE_DEPTH-1],16 79 (p[0]) ld8 t4[0]=[src2],16 80 (EPI) st8 [tgt2]=t4[PIPE_DEPTH-1],16 81 ;; 82 (p[0]) ld8 t5[0]=[src1],16 83 (EPI) st8 [tgt1]=t5[PIPE_DEPTH-1],16 84 (p[0]) ld8 t6[0]=[src2],16 85 (EPI) st8 [tgt2]=t6[PIPE_DEPTH-1],16 86 ;; 87 (p[0]) ld8 t7[0]=[src1],16 88 (EPI) st8 [tgt1]=t7[PIPE_DEPTH-1],16 89 (p[0]) ld8 t8[0]=[src2],16 90 (EPI) st8 [tgt2]=t8[PIPE_DEPTH-1],16 91 92 (p6) lfetch [srcf], 64 93 (p6) lfetch [tgtf], 64 94 br.ctop.sptk.few 1b 95 ;; 96 mov pr=saved_pr,0xffffffffffff0000 // restore predicates 97 mov ar.pfs=saved_pfs 98 mov ar.lc=saved_lc 99 br.ret.sptk.many rp 100 END(copy_page) 101 EXPORT_SYMBOL(copy_page) 102