1 /* SPDX-License-Identifier: GPL-2.0-only */ 2 /* 3 * Copyright (C) 2012 ARM Ltd. 4 */ 5 6 #include <linux/linkage.h> 7 #include <linux/const.h> 8 #include <asm/assembler.h> 9 #include <asm/page.h> 10 #include <asm/cpufeature.h> 11 #include <asm/alternative.h> 12 13 /* 14 * Copy a page from src to dest (both are page aligned) 15 * 16 * Parameters: 17 * x0 - dest 18 * x1 - src 19 */ 20 SYM_FUNC_START_PI(copy_page) 21 alternative_if ARM64_HAS_NO_HW_PREFETCH 22 // Prefetch three cache lines ahead. 23 prfm pldl1strm, [x1, #128] 24 prfm pldl1strm, [x1, #256] 25 prfm pldl1strm, [x1, #384] 26 alternative_else_nop_endif 27 28 ldp x2, x3, [x1] 29 ldp x4, x5, [x1, #16] 30 ldp x6, x7, [x1, #32] 31 ldp x8, x9, [x1, #48] 32 ldp x10, x11, [x1, #64] 33 ldp x12, x13, [x1, #80] 34 ldp x14, x15, [x1, #96] 35 ldp x16, x17, [x1, #112] 36 37 add x0, x0, #256 38 add x1, x1, #128 39 1: 40 tst x0, #(PAGE_SIZE - 1) 41 42 alternative_if ARM64_HAS_NO_HW_PREFETCH 43 prfm pldl1strm, [x1, #384] 44 alternative_else_nop_endif 45 46 stnp x2, x3, [x0, #-256] 47 ldp x2, x3, [x1] 48 stnp x4, x5, [x0, #16 - 256] 49 ldp x4, x5, [x1, #16] 50 stnp x6, x7, [x0, #32 - 256] 51 ldp x6, x7, [x1, #32] 52 stnp x8, x9, [x0, #48 - 256] 53 ldp x8, x9, [x1, #48] 54 stnp x10, x11, [x0, #64 - 256] 55 ldp x10, x11, [x1, #64] 56 stnp x12, x13, [x0, #80 - 256] 57 ldp x12, x13, [x1, #80] 58 stnp x14, x15, [x0, #96 - 256] 59 ldp x14, x15, [x1, #96] 60 stnp x16, x17, [x0, #112 - 256] 61 ldp x16, x17, [x1, #112] 62 63 add x0, x0, #128 64 add x1, x1, #128 65 66 b.ne 1b 67 68 stnp x2, x3, [x0, #-256] 69 stnp x4, x5, [x0, #16 - 256] 70 stnp x6, x7, [x0, #32 - 256] 71 stnp x8, x9, [x0, #48 - 256] 72 stnp x10, x11, [x0, #64 - 256] 73 stnp x12, x13, [x0, #80 - 256] 74 stnp x14, x15, [x0, #96 - 256] 75 stnp x16, x17, [x0, #112 - 256] 76 77 ret 78 SYM_FUNC_END_PI(copy_page) 79 EXPORT_SYMBOL(copy_page) 80