1// SPDX-License-Identifier: GPL-2.0 2#include <linux/string.h> 3#include <linux/export.h> 4 5#undef memcpy 6#undef memset 7 8__visible void *memcpy(void *to, const void *from, size_t n) 9{ 10#if defined(CONFIG_X86_USE_3DNOW) && !defined(CONFIG_FORTIFY_SOURCE) 11 return __memcpy3d(to, from, n); 12#else 13 return __memcpy(to, from, n); 14#endif 15} 16EXPORT_SYMBOL(memcpy); 17 18__visible void *memset(void *s, int c, size_t count) 19{ 20 return __memset(s, c, count); 21} 22EXPORT_SYMBOL(memset); 23 24__visible void *memmove(void *dest, const void *src, size_t n) 25{ 26 int d0,d1,d2,d3,d4,d5; 27 char *ret = dest; 28 29 __asm__ __volatile__( 30 /* Handle more 16 bytes in loop */ 31 "cmp $0x10, %0\n\t" 32 "jb 1f\n\t" 33 34 /* Decide forward/backward copy mode */ 35 "cmp %2, %1\n\t" 36 "jb 2f\n\t" 37 38 /* 39 * movs instruction have many startup latency 40 * so we handle small size by general register. 41 */ 42 "cmp $680, %0\n\t" 43 "jb 3f\n\t" 44 /* 45 * movs instruction is only good for aligned case. 46 */ 47 "mov %1, %3\n\t" 48 "xor %2, %3\n\t" 49 "and $0xff, %3\n\t" 50 "jz 4f\n\t" 51 "3:\n\t" 52 "sub $0x10, %0\n\t" 53 54 /* 55 * We gobble 16 bytes forward in each loop. 56 */ 57 "3:\n\t" 58 "sub $0x10, %0\n\t" 59 "mov 0*4(%1), %3\n\t" 60 "mov 1*4(%1), %4\n\t" 61 "mov %3, 0*4(%2)\n\t" 62 "mov %4, 1*4(%2)\n\t" 63 "mov 2*4(%1), %3\n\t" 64 "mov 3*4(%1), %4\n\t" 65 "mov %3, 2*4(%2)\n\t" 66 "mov %4, 3*4(%2)\n\t" 67 "lea 0x10(%1), %1\n\t" 68 "lea 0x10(%2), %2\n\t" 69 "jae 3b\n\t" 70 "add $0x10, %0\n\t" 71 "jmp 1f\n\t" 72 73 /* 74 * Handle data forward by movs. 75 */ 76 ".p2align 4\n\t" 77 "4:\n\t" 78 "mov -4(%1, %0), %3\n\t" 79 "lea -4(%2, %0), %4\n\t" 80 "shr $2, %0\n\t" 81 "rep movsl\n\t" 82 "mov %3, (%4)\n\t" 83 "jmp 11f\n\t" 84 /* 85 * Handle data backward by movs. 86 */ 87 ".p2align 4\n\t" 88 "6:\n\t" 89 "mov (%1), %3\n\t" 90 "mov %2, %4\n\t" 91 "lea -4(%1, %0), %1\n\t" 92 "lea -4(%2, %0), %2\n\t" 93 "shr $2, %0\n\t" 94 "std\n\t" 95 "rep movsl\n\t" 96 "mov %3,(%4)\n\t" 97 "cld\n\t" 98 "jmp 11f\n\t" 99 100 /* 101 * Start to prepare for backward copy. 102 */ 103 ".p2align 4\n\t" 104 "2:\n\t" 105 "cmp $680, %0\n\t" 106 "jb 5f\n\t" 107 "mov %1, %3\n\t" 108 "xor %2, %3\n\t" 109 "and $0xff, %3\n\t" 110 "jz 6b\n\t" 111 112 /* 113 * Calculate copy position to tail. 114 */ 115 "5:\n\t" 116 "add %0, %1\n\t" 117 "add %0, %2\n\t" 118 "sub $0x10, %0\n\t" 119 120 /* 121 * We gobble 16 bytes backward in each loop. 122 */ 123 "7:\n\t" 124 "sub $0x10, %0\n\t" 125 126 "mov -1*4(%1), %3\n\t" 127 "mov -2*4(%1), %4\n\t" 128 "mov %3, -1*4(%2)\n\t" 129 "mov %4, -2*4(%2)\n\t" 130 "mov -3*4(%1), %3\n\t" 131 "mov -4*4(%1), %4\n\t" 132 "mov %3, -3*4(%2)\n\t" 133 "mov %4, -4*4(%2)\n\t" 134 "lea -0x10(%1), %1\n\t" 135 "lea -0x10(%2), %2\n\t" 136 "jae 7b\n\t" 137 /* 138 * Calculate copy position to head. 139 */ 140 "add $0x10, %0\n\t" 141 "sub %0, %1\n\t" 142 "sub %0, %2\n\t" 143 144 /* 145 * Move data from 8 bytes to 15 bytes. 146 */ 147 ".p2align 4\n\t" 148 "1:\n\t" 149 "cmp $8, %0\n\t" 150 "jb 8f\n\t" 151 "mov 0*4(%1), %3\n\t" 152 "mov 1*4(%1), %4\n\t" 153 "mov -2*4(%1, %0), %5\n\t" 154 "mov -1*4(%1, %0), %1\n\t" 155 156 "mov %3, 0*4(%2)\n\t" 157 "mov %4, 1*4(%2)\n\t" 158 "mov %5, -2*4(%2, %0)\n\t" 159 "mov %1, -1*4(%2, %0)\n\t" 160 "jmp 11f\n\t" 161 162 /* 163 * Move data from 4 bytes to 7 bytes. 164 */ 165 ".p2align 4\n\t" 166 "8:\n\t" 167 "cmp $4, %0\n\t" 168 "jb 9f\n\t" 169 "mov 0*4(%1), %3\n\t" 170 "mov -1*4(%1, %0), %4\n\t" 171 "mov %3, 0*4(%2)\n\t" 172 "mov %4, -1*4(%2, %0)\n\t" 173 "jmp 11f\n\t" 174 175 /* 176 * Move data from 2 bytes to 3 bytes. 177 */ 178 ".p2align 4\n\t" 179 "9:\n\t" 180 "cmp $2, %0\n\t" 181 "jb 10f\n\t" 182 "movw 0*2(%1), %%dx\n\t" 183 "movw -1*2(%1, %0), %%bx\n\t" 184 "movw %%dx, 0*2(%2)\n\t" 185 "movw %%bx, -1*2(%2, %0)\n\t" 186 "jmp 11f\n\t" 187 188 /* 189 * Move data for 1 byte. 190 */ 191 ".p2align 4\n\t" 192 "10:\n\t" 193 "cmp $1, %0\n\t" 194 "jb 11f\n\t" 195 "movb (%1), %%cl\n\t" 196 "movb %%cl, (%2)\n\t" 197 ".p2align 4\n\t" 198 "11:" 199 : "=&c" (d0), "=&S" (d1), "=&D" (d2), 200 "=r" (d3),"=r" (d4), "=r"(d5) 201 :"0" (n), 202 "1" (src), 203 "2" (dest) 204 :"memory"); 205 206 return ret; 207 208} 209EXPORT_SYMBOL(memmove); 210