162306a36Sopenharmony_ci/* SPDX-License-Identifier: GPL-2.0-only */ 262306a36Sopenharmony_ci/* 362306a36Sopenharmony_ci * Copyright (C) 2012 ARM Ltd. 462306a36Sopenharmony_ci */ 562306a36Sopenharmony_ci 662306a36Sopenharmony_ci#include <linux/linkage.h> 762306a36Sopenharmony_ci#include <linux/const.h> 862306a36Sopenharmony_ci#include <asm/assembler.h> 962306a36Sopenharmony_ci#include <asm/page.h> 1062306a36Sopenharmony_ci#include <asm/cpufeature.h> 1162306a36Sopenharmony_ci#include <asm/alternative.h> 1262306a36Sopenharmony_ci 1362306a36Sopenharmony_ci/* 1462306a36Sopenharmony_ci * Copy a page from src to dest (both are page aligned) 1562306a36Sopenharmony_ci * 1662306a36Sopenharmony_ci * Parameters: 1762306a36Sopenharmony_ci * x0 - dest 1862306a36Sopenharmony_ci * x1 - src 1962306a36Sopenharmony_ci */ 2062306a36Sopenharmony_ciSYM_FUNC_START(__pi_copy_page) 2162306a36Sopenharmony_cialternative_if ARM64_HAS_NO_HW_PREFETCH 2262306a36Sopenharmony_ci // Prefetch three cache lines ahead. 2362306a36Sopenharmony_ci prfm pldl1strm, [x1, #128] 2462306a36Sopenharmony_ci prfm pldl1strm, [x1, #256] 2562306a36Sopenharmony_ci prfm pldl1strm, [x1, #384] 2662306a36Sopenharmony_cialternative_else_nop_endif 2762306a36Sopenharmony_ci 2862306a36Sopenharmony_ci ldp x2, x3, [x1] 2962306a36Sopenharmony_ci ldp x4, x5, [x1, #16] 3062306a36Sopenharmony_ci ldp x6, x7, [x1, #32] 3162306a36Sopenharmony_ci ldp x8, x9, [x1, #48] 3262306a36Sopenharmony_ci ldp x10, x11, [x1, #64] 3362306a36Sopenharmony_ci ldp x12, x13, [x1, #80] 3462306a36Sopenharmony_ci ldp x14, x15, [x1, #96] 3562306a36Sopenharmony_ci ldp x16, x17, [x1, #112] 3662306a36Sopenharmony_ci 3762306a36Sopenharmony_ci add x0, x0, #256 3862306a36Sopenharmony_ci add x1, x1, #128 3962306a36Sopenharmony_ci1: 4062306a36Sopenharmony_ci tst x0, #(PAGE_SIZE - 1) 4162306a36Sopenharmony_ci 4262306a36Sopenharmony_cialternative_if ARM64_HAS_NO_HW_PREFETCH 4362306a36Sopenharmony_ci prfm pldl1strm, [x1, #384] 4462306a36Sopenharmony_cialternative_else_nop_endif 4562306a36Sopenharmony_ci 4662306a36Sopenharmony_ci stnp x2, x3, [x0, #-256] 4762306a36Sopenharmony_ci ldp x2, x3, [x1] 4862306a36Sopenharmony_ci stnp x4, x5, [x0, #16 - 256] 4962306a36Sopenharmony_ci ldp x4, x5, [x1, #16] 5062306a36Sopenharmony_ci stnp x6, x7, [x0, #32 - 256] 5162306a36Sopenharmony_ci ldp x6, x7, [x1, #32] 5262306a36Sopenharmony_ci stnp x8, x9, [x0, #48 - 256] 5362306a36Sopenharmony_ci ldp x8, x9, [x1, #48] 5462306a36Sopenharmony_ci stnp x10, x11, [x0, #64 - 256] 5562306a36Sopenharmony_ci ldp x10, x11, [x1, #64] 5662306a36Sopenharmony_ci stnp x12, x13, [x0, #80 - 256] 5762306a36Sopenharmony_ci ldp x12, x13, [x1, #80] 5862306a36Sopenharmony_ci stnp x14, x15, [x0, #96 - 256] 5962306a36Sopenharmony_ci ldp x14, x15, [x1, #96] 6062306a36Sopenharmony_ci stnp x16, x17, [x0, #112 - 256] 6162306a36Sopenharmony_ci ldp x16, x17, [x1, #112] 6262306a36Sopenharmony_ci 6362306a36Sopenharmony_ci add x0, x0, #128 6462306a36Sopenharmony_ci add x1, x1, #128 6562306a36Sopenharmony_ci 6662306a36Sopenharmony_ci b.ne 1b 6762306a36Sopenharmony_ci 6862306a36Sopenharmony_ci stnp x2, x3, [x0, #-256] 6962306a36Sopenharmony_ci stnp x4, x5, [x0, #16 - 256] 7062306a36Sopenharmony_ci stnp x6, x7, [x0, #32 - 256] 7162306a36Sopenharmony_ci stnp x8, x9, [x0, #48 - 256] 7262306a36Sopenharmony_ci stnp x10, x11, [x0, #64 - 256] 7362306a36Sopenharmony_ci stnp x12, x13, [x0, #80 - 256] 7462306a36Sopenharmony_ci stnp x14, x15, [x0, #96 - 256] 7562306a36Sopenharmony_ci stnp x16, x17, [x0, #112 - 256] 7662306a36Sopenharmony_ci 7762306a36Sopenharmony_ci ret 7862306a36Sopenharmony_ciSYM_FUNC_END(__pi_copy_page) 7962306a36Sopenharmony_ciSYM_FUNC_ALIAS(copy_page, __pi_copy_page) 8062306a36Sopenharmony_ciEXPORT_SYMBOL(copy_page) 81