1 /* SPDX-License-Identifier: GPL-2.0 */ 2 /* 3 * __get_user functions. 4 * 5 * (C) Copyright 1998 Linus Torvalds 6 * (C) Copyright 2005 Andi Kleen 7 * (C) Copyright 2008 Glauber Costa 8 * 9 * These functions have a non-standard call interface 10 * to make them more efficient, especially as they 11 * return an error value in addition to the "real" 12 * return value. 13 */ 14 15 /* 16 * __get_user_X 17 * 18 * Inputs: %[r|e]ax contains the address. 19 * 20 * Outputs: %[r|e]ax is error code (0 or -EFAULT) 21 * %[r|e]dx contains zero-extended value 22 * %ecx contains the high half for 32-bit __get_user_8 23 * 24 * 25 * These functions should not modify any other registers, 26 * as they get called from within inline assembly. 27 */ 28 29 #include <linux/linkage.h> 30 #include <asm/page_types.h> 31 #include <asm/errno.h> 32 #include <asm/asm-offsets.h> 33 #include <asm/thread_info.h> 34 #include <asm/asm.h> 35 #include <asm/smap.h> 36 #include <asm/export.h> 37 38 #define ASM_BARRIER_NOSPEC ALTERNATIVE "", "lfence", X86_FEATURE_LFENCE_RDTSC 39 40 #ifdef CONFIG_X86_5LEVEL 41 #define LOAD_TASK_SIZE_MINUS_N(n) \ 42 ALTERNATIVE __stringify(mov $((1 << 47) - 4096 - (n)),%rdx), \ 43 __stringify(mov $((1 << 56) - 4096 - (n)),%rdx), X86_FEATURE_LA57 44 #else 45 #define LOAD_TASK_SIZE_MINUS_N(n) \ 46 mov $(TASK_SIZE_MAX - (n)),%_ASM_DX 47 #endif 48 49 .text 50 SYM_FUNC_START(__get_user_1) 51 LOAD_TASK_SIZE_MINUS_N(0) 52 cmp %_ASM_DX,%_ASM_AX 53 jae bad_get_user 54 sbb %_ASM_DX, %_ASM_DX /* array_index_mask_nospec() */ 55 and %_ASM_DX, %_ASM_AX 56 ASM_STAC 57 1: movzbl (%_ASM_AX),%edx 58 xor %eax,%eax 59 ASM_CLAC 60 RET 61 SYM_FUNC_END(__get_user_1) 62 EXPORT_SYMBOL(__get_user_1) 63 64 SYM_FUNC_START(__get_user_2) 65 LOAD_TASK_SIZE_MINUS_N(1) 66 cmp %_ASM_DX,%_ASM_AX 67 jae bad_get_user 68 sbb %_ASM_DX, %_ASM_DX /* array_index_mask_nospec() */ 69 and %_ASM_DX, %_ASM_AX 70 ASM_STAC 71 2: movzwl (%_ASM_AX),%edx 72 xor %eax,%eax 73 ASM_CLAC 74 RET 75 SYM_FUNC_END(__get_user_2) 76 EXPORT_SYMBOL(__get_user_2) 77 78 SYM_FUNC_START(__get_user_4) 79 LOAD_TASK_SIZE_MINUS_N(3) 80 cmp %_ASM_DX,%_ASM_AX 81 jae bad_get_user 82 sbb %_ASM_DX, %_ASM_DX /* array_index_mask_nospec() */ 83 and %_ASM_DX, %_ASM_AX 84 ASM_STAC 85 3: movl (%_ASM_AX),%edx 86 xor %eax,%eax 87 ASM_CLAC 88 RET 89 SYM_FUNC_END(__get_user_4) 90 EXPORT_SYMBOL(__get_user_4) 91 92 SYM_FUNC_START(__get_user_8) 93 #ifdef CONFIG_X86_64 94 LOAD_TASK_SIZE_MINUS_N(7) 95 cmp %_ASM_DX,%_ASM_AX 96 jae bad_get_user 97 sbb %_ASM_DX, %_ASM_DX /* array_index_mask_nospec() */ 98 and %_ASM_DX, %_ASM_AX 99 ASM_STAC 100 4: movq (%_ASM_AX),%rdx 101 xor %eax,%eax 102 ASM_CLAC 103 RET 104 #else 105 LOAD_TASK_SIZE_MINUS_N(7) 106 cmp %_ASM_DX,%_ASM_AX 107 jae bad_get_user_8 108 sbb %_ASM_DX, %_ASM_DX /* array_index_mask_nospec() */ 109 and %_ASM_DX, %_ASM_AX 110 ASM_STAC 111 4: movl (%_ASM_AX),%edx 112 5: movl 4(%_ASM_AX),%ecx 113 xor %eax,%eax 114 ASM_CLAC 115 RET 116 #endif 117 SYM_FUNC_END(__get_user_8) 118 EXPORT_SYMBOL(__get_user_8) 119 120 /* .. and the same for __get_user, just without the range checks */ 121 SYM_FUNC_START(__get_user_nocheck_1) 122 ASM_STAC 123 ASM_BARRIER_NOSPEC 124 6: movzbl (%_ASM_AX),%edx 125 xor %eax,%eax 126 ASM_CLAC 127 RET 128 SYM_FUNC_END(__get_user_nocheck_1) 129 EXPORT_SYMBOL(__get_user_nocheck_1) 130 131 SYM_FUNC_START(__get_user_nocheck_2) 132 ASM_STAC 133 ASM_BARRIER_NOSPEC 134 7: movzwl (%_ASM_AX),%edx 135 xor %eax,%eax 136 ASM_CLAC 137 RET 138 SYM_FUNC_END(__get_user_nocheck_2) 139 EXPORT_SYMBOL(__get_user_nocheck_2) 140 141 SYM_FUNC_START(__get_user_nocheck_4) 142 ASM_STAC 143 ASM_BARRIER_NOSPEC 144 8: movl (%_ASM_AX),%edx 145 xor %eax,%eax 146 ASM_CLAC 147 RET 148 SYM_FUNC_END(__get_user_nocheck_4) 149 EXPORT_SYMBOL(__get_user_nocheck_4) 150 151 SYM_FUNC_START(__get_user_nocheck_8) 152 ASM_STAC 153 ASM_BARRIER_NOSPEC 154 #ifdef CONFIG_X86_64 155 9: movq (%_ASM_AX),%rdx 156 #else 157 9: movl (%_ASM_AX),%edx 158 10: movl 4(%_ASM_AX),%ecx 159 #endif 160 xor %eax,%eax 161 ASM_CLAC 162 RET 163 SYM_FUNC_END(__get_user_nocheck_8) 164 EXPORT_SYMBOL(__get_user_nocheck_8) 165 166 167 SYM_CODE_START_LOCAL(.Lbad_get_user_clac) 168 ASM_CLAC 169 bad_get_user: 170 xor %edx,%edx 171 mov $(-EFAULT),%_ASM_AX 172 RET 173 SYM_CODE_END(.Lbad_get_user_clac) 174 175 #ifdef CONFIG_X86_32 176 SYM_CODE_START_LOCAL(.Lbad_get_user_8_clac) 177 ASM_CLAC 178 bad_get_user_8: 179 xor %edx,%edx 180 xor %ecx,%ecx 181 mov $(-EFAULT),%_ASM_AX 182 RET 183 SYM_CODE_END(.Lbad_get_user_8_clac) 184 #endif 185 186 /* get_user */ 187 _ASM_EXTABLE_UA(1b, .Lbad_get_user_clac) 188 _ASM_EXTABLE_UA(2b, .Lbad_get_user_clac) 189 _ASM_EXTABLE_UA(3b, .Lbad_get_user_clac) 190 #ifdef CONFIG_X86_64 191 _ASM_EXTABLE_UA(4b, .Lbad_get_user_clac) 192 #else 193 _ASM_EXTABLE_UA(4b, .Lbad_get_user_8_clac) 194 _ASM_EXTABLE_UA(5b, .Lbad_get_user_8_clac) 195 #endif 196 197 /* __get_user */ 198 _ASM_EXTABLE_UA(6b, .Lbad_get_user_clac) 199 _ASM_EXTABLE_UA(7b, .Lbad_get_user_clac) 200 _ASM_EXTABLE_UA(8b, .Lbad_get_user_clac) 201 #ifdef CONFIG_X86_64 202 _ASM_EXTABLE_UA(9b, .Lbad_get_user_clac) 203 #else 204 _ASM_EXTABLE_UA(9b, .Lbad_get_user_8_clac) 205 _ASM_EXTABLE_UA(10b, .Lbad_get_user_8_clac) 206 #endif 207