1 /* SPDX-License-Identifier: GPL-2.0-only */ 2 /* 3 * Copyright 2008 Vitaly Mayatskikh <vmayatsk@redhat.com> 4 * Copyright 2002 Andi Kleen, SuSE Labs. 5 * 6 * Functions to copy from and to user space. 7 */ 8 9 #include <linux/linkage.h> 10 #include <asm/cpufeatures.h> 11 #include <asm/alternative.h> 12 #include <asm/asm.h> 13 #include <asm/export.h> 14 15 /* 16 * rep_movs_alternative - memory copy with exception handling. 17 * This version is for CPUs that don't have FSRM (Fast Short Rep Movs) 18 * 19 * Input: 20 * rdi destination 21 * rsi source 22 * rcx count 23 * 24 * Output: 25 * rcx uncopied bytes or 0 if successful. 26 * 27 * NOTE! The calling convention is very intentionally the same as 28 * for 'rep movs', so that we can rewrite the function call with 29 * just a plain 'rep movs' on machines that have FSRM. But to make 30 * it simpler for us, we can clobber rsi/rdi and rax freely. 31 */ 32 SYM_FUNC_START(rep_movs_alternative) 33 cmpq $64,%rcx 34 jae .Llarge 35 36 cmp $8,%ecx 37 jae .Lword 38 39 testl %ecx,%ecx 40 je .Lexit 41 42 .Lcopy_user_tail: 43 0: movb (%rsi),%al 44 1: movb %al,(%rdi) 45 inc %rdi 46 inc %rsi 47 dec %rcx 48 jne .Lcopy_user_tail 49 .Lexit: 50 RET 51 52 _ASM_EXTABLE_UA( 0b, .Lexit) 53 _ASM_EXTABLE_UA( 1b, .Lexit) 54 55 .p2align 4 56 .Lword: 57 2: movq (%rsi),%rax 58 3: movq %rax,(%rdi) 59 addq $8,%rsi 60 addq $8,%rdi 61 sub $8,%ecx 62 je .Lexit 63 cmp $8,%ecx 64 jae .Lword 65 jmp .Lcopy_user_tail 66 67 _ASM_EXTABLE_UA( 2b, .Lcopy_user_tail) 68 _ASM_EXTABLE_UA( 3b, .Lcopy_user_tail) 69 70 .Llarge: 71 0: ALTERNATIVE "jmp .Llarge_movsq", "rep movsb", X86_FEATURE_ERMS 72 1: RET 73 74 _ASM_EXTABLE_UA( 0b, 1b) 75 76 .Llarge_movsq: 77 movq %rcx,%rax 78 shrq $3,%rcx 79 andl $7,%eax 80 0: rep movsq 81 movl %eax,%ecx 82 testl %ecx,%ecx 83 jne .Lcopy_user_tail 84 RET 85 86 1: leaq (%rax,%rcx,8),%rcx 87 jmp .Lcopy_user_tail 88 89 _ASM_EXTABLE_UA( 0b, 1b) 90 SYM_FUNC_END(rep_movs_alternative) 91 EXPORT_SYMBOL(rep_movs_alternative) 92