1 /* SPDX-License-Identifier: GPL-2.0-or-later */ 2 /* 3 * 4 * Copyright (C) IBM Corporation, 2012 5 * 6 * Author: Anton Blanchard <anton@au.ibm.com> 7 */ 8 9 #include <linux/export.h> 10 #include <asm/ppc_asm.h> 11 #include <asm/linkage.h> 12 #include <asm/asm-offsets.h> 13 14 /** 15 * __arch_clear_user: - Zero a block of memory in user space, with less checking. 16 * @to: Destination address, in user space. 17 * @n: Number of bytes to zero. 18 * 19 * Zero a block of memory in user space. Caller must check 20 * the specified block with access_ok() before calling this function. 21 * 22 * Returns number of bytes that could not be cleared. 23 * On success, this will be zero. 24 */ 25 26 .macro err1 27 100: 28 EX_TABLE(100b,.Ldo_err1) 29 .endm 30 31 .macro err2 32 200: 33 EX_TABLE(200b,.Ldo_err2) 34 .endm 35 36 .macro err3 37 300: 38 EX_TABLE(300b,.Ldo_err3) 39 .endm 40 41 .Ldo_err1: 42 mr r3,r8 43 44 .Ldo_err2: 45 mtctr r4 46 1: 47 err3; stb r0,0(r3) 48 addi r3,r3,1 49 addi r4,r4,-1 50 bdnz 1b 51 52 .Ldo_err3: 53 mr r3,r4 54 blr 55 56 _GLOBAL_TOC(__arch_clear_user) 57 cmpdi r4,32 58 neg r6,r3 59 li r0,0 60 blt .Lshort_clear 61 mr r8,r3 62 mtocrf 0x01,r6 63 clrldi r6,r6,(64-3) 64 65 /* Get the destination 8 byte aligned */ 66 bf cr7*4+3,1f 67 err1; stb r0,0(r3) 68 addi r3,r3,1 69 70 1: bf cr7*4+2,2f 71 err1; sth r0,0(r3) 72 addi r3,r3,2 73 74 2: bf cr7*4+1,3f 75 err1; stw r0,0(r3) 76 addi r3,r3,4 77 78 3: sub r4,r4,r6 79 80 cmpdi r4,32 81 cmpdi cr1,r4,512 82 blt .Lshort_clear 83 bgt cr1,.Llong_clear 84 85 .Lmedium_clear: 86 srdi r6,r4,5 87 mtctr r6 88 89 /* Do 32 byte chunks */ 90 4: 91 err2; std r0,0(r3) 92 err2; std r0,8(r3) 93 err2; std r0,16(r3) 94 err2; std r0,24(r3) 95 addi r3,r3,32 96 addi r4,r4,-32 97 bdnz 4b 98 99 .Lshort_clear: 100 /* up to 31 bytes to go */ 101 cmpdi r4,16 102 blt 6f 103 err2; std r0,0(r3) 104 err2; std r0,8(r3) 105 addi r3,r3,16 106 addi r4,r4,-16 107 108 /* Up to 15 bytes to go */ 109 6: mr r8,r3 110 clrldi r4,r4,(64-4) 111 mtocrf 0x01,r4 112 bf cr7*4+0,7f 113 err1; std r0,0(r3) 114 addi r3,r3,8 115 116 7: bf cr7*4+1,8f 117 err1; stw r0,0(r3) 118 addi r3,r3,4 119 120 8: bf cr7*4+2,9f 121 err1; sth r0,0(r3) 122 addi r3,r3,2 123 124 9: bf cr7*4+3,10f 125 err1; stb r0,0(r3) 126 127 10: li r3,0 128 blr 129 130 .Llong_clear: 131 LOAD_REG_ADDR(r5, ppc64_caches) 132 133 bf cr7*4+0,11f 134 err2; std r0,0(r3) 135 addi r3,r3,8 136 addi r4,r4,-8 137 138 /* Destination is 16 byte aligned, need to get it cache block aligned */ 139 11: lwz r7,DCACHEL1LOGBLOCKSIZE(r5) 140 lwz r9,DCACHEL1BLOCKSIZE(r5) 141 142 /* 143 * With worst case alignment the long clear loop takes a minimum 144 * of 1 byte less than 2 cachelines. 145 */ 146 sldi r10,r9,2 147 cmpd r4,r10 148 blt .Lmedium_clear 149 150 neg r6,r3 151 addi r10,r9,-1 152 and. r5,r6,r10 153 beq 13f 154 155 srdi r6,r5,4 156 mtctr r6 157 mr r8,r3 158 12: 159 err1; std r0,0(r3) 160 err1; std r0,8(r3) 161 addi r3,r3,16 162 bdnz 12b 163 164 sub r4,r4,r5 165 166 13: srd r6,r4,r7 167 mtctr r6 168 mr r8,r3 169 14: 170 err1; dcbz 0,r3 171 add r3,r3,r9 172 bdnz 14b 173 174 and r4,r4,r10 175 176 cmpdi r4,32 177 blt .Lshort_clear 178 b .Lmedium_clear 179 EXPORT_SYMBOL(__arch_clear_user) 180