1 /* SPDX-License-Identifier: GPL-2.0 */ 2 /* 3 * String handling functions. 4 * 5 * Copyright IBM Corp. 2012 6 */ 7 8 #include <linux/linkage.h> 9 #include <asm/export.h> 10 #include <asm/nospec-insn.h> 11 12 GEN_BR_THUNK %r14 13 14 /* 15 * void *memmove(void *dest, const void *src, size_t n) 16 */ 17 WEAK(memmove) 18 ENTRY(__memmove) 19 ltgr %r4,%r4 20 lgr %r1,%r2 21 jz .Lmemmove_exit 22 aghi %r4,-1 23 clgr %r2,%r3 24 jnh .Lmemmove_forward 25 la %r5,1(%r4,%r3) 26 clgr %r2,%r5 27 jl .Lmemmove_reverse 28 .Lmemmove_forward: 29 srlg %r0,%r4,8 30 ltgr %r0,%r0 31 jz .Lmemmove_forward_remainder 32 .Lmemmove_forward_loop: 33 mvc 0(256,%r1),0(%r3) 34 la %r1,256(%r1) 35 la %r3,256(%r3) 36 brctg %r0,.Lmemmove_forward_loop 37 .Lmemmove_forward_remainder: 38 larl %r5,.Lmemmove_mvc 39 ex %r4,0(%r5) 40 .Lmemmove_exit: 41 BR_EX %r14 42 .Lmemmove_reverse: 43 ic %r0,0(%r4,%r3) 44 stc %r0,0(%r4,%r1) 45 brctg %r4,.Lmemmove_reverse 46 ic %r0,0(%r4,%r3) 47 stc %r0,0(%r4,%r1) 48 BR_EX %r14 49 .Lmemmove_mvc: 50 mvc 0(1,%r1),0(%r3) 51 ENDPROC(__memmove) 52 EXPORT_SYMBOL(memmove) 53 54 /* 55 * memset implementation 56 * 57 * This code corresponds to the C construct below. We do distinguish 58 * between clearing (c == 0) and setting a memory array (c != 0) simply 59 * because nearly all memset invocations in the kernel clear memory and 60 * the xc instruction is preferred in such cases. 61 * 62 * void *memset(void *s, int c, size_t n) 63 * { 64 * if (likely(c == 0)) 65 * return __builtin_memset(s, 0, n); 66 * return __builtin_memset(s, c, n); 67 * } 68 */ 69 WEAK(memset) 70 ENTRY(__memset) 71 ltgr %r4,%r4 72 jz .Lmemset_exit 73 ltgr %r3,%r3 74 jnz .Lmemset_fill 75 aghi %r4,-1 76 srlg %r3,%r4,8 77 ltgr %r3,%r3 78 lgr %r1,%r2 79 jz .Lmemset_clear_remainder 80 .Lmemset_clear_loop: 81 xc 0(256,%r1),0(%r1) 82 la %r1,256(%r1) 83 brctg %r3,.Lmemset_clear_loop 84 .Lmemset_clear_remainder: 85 larl %r3,.Lmemset_xc 86 ex %r4,0(%r3) 87 .Lmemset_exit: 88 BR_EX %r14 89 .Lmemset_fill: 90 cghi %r4,1 91 lgr %r1,%r2 92 je .Lmemset_fill_exit 93 aghi %r4,-2 94 srlg %r5,%r4,8 95 ltgr %r5,%r5 96 jz .Lmemset_fill_remainder 97 .Lmemset_fill_loop: 98 stc %r3,0(%r1) 99 mvc 1(255,%r1),0(%r1) 100 la %r1,256(%r1) 101 brctg %r5,.Lmemset_fill_loop 102 .Lmemset_fill_remainder: 103 stc %r3,0(%r1) 104 larl %r5,.Lmemset_mvc 105 ex %r4,0(%r5) 106 BR_EX %r14 107 .Lmemset_fill_exit: 108 stc %r3,0(%r1) 109 BR_EX %r14 110 .Lmemset_xc: 111 xc 0(1,%r1),0(%r1) 112 .Lmemset_mvc: 113 mvc 1(1,%r1),0(%r1) 114 ENDPROC(__memset) 115 EXPORT_SYMBOL(memset) 116 117 /* 118 * memcpy implementation 119 * 120 * void *memcpy(void *dest, const void *src, size_t n) 121 */ 122 WEAK(memcpy) 123 ENTRY(__memcpy) 124 ltgr %r4,%r4 125 jz .Lmemcpy_exit 126 aghi %r4,-1 127 srlg %r5,%r4,8 128 ltgr %r5,%r5 129 lgr %r1,%r2 130 jnz .Lmemcpy_loop 131 .Lmemcpy_remainder: 132 larl %r5,.Lmemcpy_mvc 133 ex %r4,0(%r5) 134 .Lmemcpy_exit: 135 BR_EX %r14 136 .Lmemcpy_loop: 137 mvc 0(256,%r1),0(%r3) 138 la %r1,256(%r1) 139 la %r3,256(%r3) 140 brctg %r5,.Lmemcpy_loop 141 j .Lmemcpy_remainder 142 .Lmemcpy_mvc: 143 mvc 0(1,%r1),0(%r3) 144 ENDPROC(__memcpy) 145 EXPORT_SYMBOL(memcpy) 146 147 /* 148 * __memset16/32/64 149 * 150 * void *__memset16(uint16_t *s, uint16_t v, size_t count) 151 * void *__memset32(uint32_t *s, uint32_t v, size_t count) 152 * void *__memset64(uint64_t *s, uint64_t v, size_t count) 153 */ 154 .macro __MEMSET bits,bytes,insn 155 ENTRY(__memset\bits) 156 ltgr %r4,%r4 157 jz .L__memset_exit\bits 158 cghi %r4,\bytes 159 je .L__memset_store\bits 160 aghi %r4,-(\bytes+1) 161 srlg %r5,%r4,8 162 ltgr %r5,%r5 163 lgr %r1,%r2 164 jz .L__memset_remainder\bits 165 .L__memset_loop\bits: 166 \insn %r3,0(%r1) 167 mvc \bytes(256-\bytes,%r1),0(%r1) 168 la %r1,256(%r1) 169 brctg %r5,.L__memset_loop\bits 170 .L__memset_remainder\bits: 171 \insn %r3,0(%r1) 172 larl %r5,.L__memset_mvc\bits 173 ex %r4,0(%r5) 174 BR_EX %r14 175 .L__memset_store\bits: 176 \insn %r3,0(%r2) 177 .L__memset_exit\bits: 178 BR_EX %r14 179 .L__memset_mvc\bits: 180 mvc \bytes(1,%r1),0(%r1) 181 ENDPROC(__memset\bits) 182 .endm 183 184 __MEMSET 16,2,sth 185 EXPORT_SYMBOL(__memset16) 186 187 __MEMSET 32,4,st 188 EXPORT_SYMBOL(__memset32) 189 190 __MEMSET 64,8,stg 191 EXPORT_SYMBOL(__memset64) 192