162306a36Sopenharmony_ci/* SPDX-License-Identifier: GPL-2.0-only */ 262306a36Sopenharmony_ci/* 362306a36Sopenharmony_ci * Copyright 2008 Vitaly Mayatskikh <vmayatsk@redhat.com> 462306a36Sopenharmony_ci * Copyright 2002 Andi Kleen, SuSE Labs. 562306a36Sopenharmony_ci * 662306a36Sopenharmony_ci * Functions to copy from and to user space. 762306a36Sopenharmony_ci */ 862306a36Sopenharmony_ci 962306a36Sopenharmony_ci#include <linux/linkage.h> 1062306a36Sopenharmony_ci#include <asm/cpufeatures.h> 1162306a36Sopenharmony_ci#include <asm/alternative.h> 1262306a36Sopenharmony_ci#include <asm/asm.h> 1362306a36Sopenharmony_ci#include <asm/export.h> 1462306a36Sopenharmony_ci 1562306a36Sopenharmony_ci/* 1662306a36Sopenharmony_ci * rep_movs_alternative - memory copy with exception handling. 1762306a36Sopenharmony_ci * This version is for CPUs that don't have FSRM (Fast Short Rep Movs) 1862306a36Sopenharmony_ci * 1962306a36Sopenharmony_ci * Input: 2062306a36Sopenharmony_ci * rdi destination 2162306a36Sopenharmony_ci * rsi source 2262306a36Sopenharmony_ci * rcx count 2362306a36Sopenharmony_ci * 2462306a36Sopenharmony_ci * Output: 2562306a36Sopenharmony_ci * rcx uncopied bytes or 0 if successful. 2662306a36Sopenharmony_ci * 2762306a36Sopenharmony_ci * NOTE! The calling convention is very intentionally the same as 2862306a36Sopenharmony_ci * for 'rep movs', so that we can rewrite the function call with 2962306a36Sopenharmony_ci * just a plain 'rep movs' on machines that have FSRM. But to make 3062306a36Sopenharmony_ci * it simpler for us, we can clobber rsi/rdi and rax freely. 3162306a36Sopenharmony_ci */ 3262306a36Sopenharmony_ciSYM_FUNC_START(rep_movs_alternative) 3362306a36Sopenharmony_ci cmpq $64,%rcx 3462306a36Sopenharmony_ci jae .Llarge 3562306a36Sopenharmony_ci 3662306a36Sopenharmony_ci cmp $8,%ecx 3762306a36Sopenharmony_ci jae .Lword 3862306a36Sopenharmony_ci 3962306a36Sopenharmony_ci testl %ecx,%ecx 4062306a36Sopenharmony_ci je .Lexit 4162306a36Sopenharmony_ci 4262306a36Sopenharmony_ci.Lcopy_user_tail: 4362306a36Sopenharmony_ci0: movb (%rsi),%al 4462306a36Sopenharmony_ci1: movb %al,(%rdi) 4562306a36Sopenharmony_ci inc %rdi 4662306a36Sopenharmony_ci inc %rsi 4762306a36Sopenharmony_ci dec %rcx 4862306a36Sopenharmony_ci jne .Lcopy_user_tail 4962306a36Sopenharmony_ci.Lexit: 5062306a36Sopenharmony_ci RET 5162306a36Sopenharmony_ci 5262306a36Sopenharmony_ci _ASM_EXTABLE_UA( 0b, .Lexit) 5362306a36Sopenharmony_ci _ASM_EXTABLE_UA( 1b, .Lexit) 5462306a36Sopenharmony_ci 5562306a36Sopenharmony_ci .p2align 4 5662306a36Sopenharmony_ci.Lword: 5762306a36Sopenharmony_ci2: movq (%rsi),%rax 5862306a36Sopenharmony_ci3: movq %rax,(%rdi) 5962306a36Sopenharmony_ci addq $8,%rsi 6062306a36Sopenharmony_ci addq $8,%rdi 6162306a36Sopenharmony_ci sub $8,%ecx 6262306a36Sopenharmony_ci je .Lexit 6362306a36Sopenharmony_ci cmp $8,%ecx 6462306a36Sopenharmony_ci jae .Lword 6562306a36Sopenharmony_ci jmp .Lcopy_user_tail 6662306a36Sopenharmony_ci 6762306a36Sopenharmony_ci _ASM_EXTABLE_UA( 2b, .Lcopy_user_tail) 6862306a36Sopenharmony_ci _ASM_EXTABLE_UA( 3b, .Lcopy_user_tail) 6962306a36Sopenharmony_ci 7062306a36Sopenharmony_ci.Llarge: 7162306a36Sopenharmony_ci0: ALTERNATIVE "jmp .Llarge_movsq", "rep movsb", X86_FEATURE_ERMS 7262306a36Sopenharmony_ci1: RET 7362306a36Sopenharmony_ci 7462306a36Sopenharmony_ci _ASM_EXTABLE_UA( 0b, 1b) 7562306a36Sopenharmony_ci 7662306a36Sopenharmony_ci.Llarge_movsq: 7762306a36Sopenharmony_ci movq %rcx,%rax 7862306a36Sopenharmony_ci shrq $3,%rcx 7962306a36Sopenharmony_ci andl $7,%eax 8062306a36Sopenharmony_ci0: rep movsq 8162306a36Sopenharmony_ci movl %eax,%ecx 8262306a36Sopenharmony_ci testl %ecx,%ecx 8362306a36Sopenharmony_ci jne .Lcopy_user_tail 8462306a36Sopenharmony_ci RET 8562306a36Sopenharmony_ci 8662306a36Sopenharmony_ci1: leaq (%rax,%rcx,8),%rcx 8762306a36Sopenharmony_ci jmp .Lcopy_user_tail 8862306a36Sopenharmony_ci 8962306a36Sopenharmony_ci _ASM_EXTABLE_UA( 0b, 1b) 9062306a36Sopenharmony_ciSYM_FUNC_END(rep_movs_alternative) 9162306a36Sopenharmony_ciEXPORT_SYMBOL(rep_movs_alternative) 92