1/* SPDX-License-Identifier: LGPL-2.1 OR MIT */ 2/* 3 * rseq-arm64-bits.h 4 * 5 * (C) Copyright 2016-2022 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com> 6 * (C) Copyright 2018 - Will Deacon <will.deacon@arm.com> 7 */ 8 9#include "rseq-bits-template.h" 10 11#if defined(RSEQ_TEMPLATE_MO_RELAXED) && \ 12 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) 13 14static inline __attribute__((always_inline)) 15int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_storev)(intptr_t *v, intptr_t expect, intptr_t newv, int cpu) 16{ 17 RSEQ_INJECT_C(9) 18 19 __asm__ __volatile__ goto ( 20 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f) 21 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail]) 22#ifdef RSEQ_COMPARE_TWICE 23 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1]) 24 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2]) 25#endif 26 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) 27 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 28 RSEQ_INJECT_ASM(3) 29 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail]) 30 RSEQ_INJECT_ASM(4) 31#ifdef RSEQ_COMPARE_TWICE 32 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 33 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2]) 34#endif 35 RSEQ_ASM_OP_FINAL_STORE(newv, v, 3) 36 RSEQ_INJECT_ASM(5) 37 RSEQ_ASM_DEFINE_ABORT(4, abort) 38 : /* gcc asm goto does not allow outputs */ 39 : [cpu_id] "r" (cpu), 40 [current_cpu_id] "Qo" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD), 41 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), 42 [v] "Qo" (*v), 43 [expect] "r" (expect), 44 [newv] "r" (newv) 45 RSEQ_INJECT_INPUT 46 : "memory", RSEQ_ASM_TMP_REG 47 : abort, cmpfail 48#ifdef RSEQ_COMPARE_TWICE 49 , error1, error2 50#endif 51 ); 52 rseq_after_asm_goto(); 53 return 0; 54abort: 55 rseq_after_asm_goto(); 56 RSEQ_INJECT_FAILED 57 return -1; 58cmpfail: 59 rseq_after_asm_goto(); 60 return 1; 61#ifdef RSEQ_COMPARE_TWICE 62error1: 63 rseq_after_asm_goto(); 64 rseq_bug("cpu_id comparison failed"); 65error2: 66 rseq_after_asm_goto(); 67 rseq_bug("expected value comparison failed"); 68#endif 69} 70 71static inline __attribute__((always_inline)) 72int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpnev_storeoffp_load)(intptr_t *v, intptr_t expectnot, 73 long voffp, intptr_t *load, int cpu) 74{ 75 RSEQ_INJECT_C(9) 76 77 __asm__ __volatile__ goto ( 78 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f) 79 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail]) 80#ifdef RSEQ_COMPARE_TWICE 81 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1]) 82 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2]) 83#endif 84 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) 85 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 86 RSEQ_INJECT_ASM(3) 87 RSEQ_ASM_OP_CMPNE(v, expectnot, %l[cmpfail]) 88 RSEQ_INJECT_ASM(4) 89#ifdef RSEQ_COMPARE_TWICE 90 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 91 RSEQ_ASM_OP_CMPNE(v, expectnot, %l[error2]) 92#endif 93 RSEQ_ASM_OP_R_LOAD(v) 94 RSEQ_ASM_OP_R_STORE(load) 95 RSEQ_ASM_OP_R_LOAD_OFF(voffp) 96 RSEQ_ASM_OP_R_FINAL_STORE(v, 3) 97 RSEQ_INJECT_ASM(5) 98 RSEQ_ASM_DEFINE_ABORT(4, abort) 99 : /* gcc asm goto does not allow outputs */ 100 : [cpu_id] "r" (cpu), 101 [current_cpu_id] "Qo" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD), 102 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), 103 [v] "Qo" (*v), 104 [expectnot] "r" (expectnot), 105 [load] "Qo" (*load), 106 [voffp] "r" (voffp) 107 RSEQ_INJECT_INPUT 108 : "memory", RSEQ_ASM_TMP_REG 109 : abort, cmpfail 110#ifdef RSEQ_COMPARE_TWICE 111 , error1, error2 112#endif 113 ); 114 rseq_after_asm_goto(); 115 return 0; 116abort: 117 rseq_after_asm_goto(); 118 RSEQ_INJECT_FAILED 119 return -1; 120cmpfail: 121 rseq_after_asm_goto(); 122 return 1; 123#ifdef RSEQ_COMPARE_TWICE 124error1: 125 rseq_after_asm_goto(); 126 rseq_bug("cpu_id comparison failed"); 127error2: 128 rseq_after_asm_goto(); 129 rseq_bug("expected value comparison failed"); 130#endif 131} 132 133static inline __attribute__((always_inline)) 134int RSEQ_TEMPLATE_IDENTIFIER(rseq_addv)(intptr_t *v, intptr_t count, int cpu) 135{ 136 RSEQ_INJECT_C(9) 137 138 __asm__ __volatile__ goto ( 139 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f) 140#ifdef RSEQ_COMPARE_TWICE 141 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1]) 142#endif 143 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) 144 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 145 RSEQ_INJECT_ASM(3) 146#ifdef RSEQ_COMPARE_TWICE 147 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 148#endif 149 RSEQ_ASM_OP_R_LOAD(v) 150 RSEQ_ASM_OP_R_ADD(count) 151 RSEQ_ASM_OP_R_FINAL_STORE(v, 3) 152 RSEQ_INJECT_ASM(4) 153 RSEQ_ASM_DEFINE_ABORT(4, abort) 154 : /* gcc asm goto does not allow outputs */ 155 : [cpu_id] "r" (cpu), 156 [current_cpu_id] "Qo" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD), 157 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), 158 [v] "Qo" (*v), 159 [count] "r" (count) 160 RSEQ_INJECT_INPUT 161 : "memory", RSEQ_ASM_TMP_REG 162 : abort 163#ifdef RSEQ_COMPARE_TWICE 164 , error1 165#endif 166 ); 167 rseq_after_asm_goto(); 168 return 0; 169abort: 170 rseq_after_asm_goto(); 171 RSEQ_INJECT_FAILED 172 return -1; 173#ifdef RSEQ_COMPARE_TWICE 174error1: 175 rseq_after_asm_goto(); 176 rseq_bug("cpu_id comparison failed"); 177#endif 178} 179 180static inline __attribute__((always_inline)) 181int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_cmpeqv_storev)(intptr_t *v, intptr_t expect, 182 intptr_t *v2, intptr_t expect2, 183 intptr_t newv, int cpu) 184{ 185 RSEQ_INJECT_C(9) 186 187 __asm__ __volatile__ goto ( 188 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f) 189 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail]) 190#ifdef RSEQ_COMPARE_TWICE 191 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1]) 192 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2]) 193 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error3]) 194#endif 195 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) 196 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 197 RSEQ_INJECT_ASM(3) 198 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail]) 199 RSEQ_INJECT_ASM(4) 200 RSEQ_ASM_OP_CMPEQ(v2, expect2, %l[cmpfail]) 201 RSEQ_INJECT_ASM(5) 202#ifdef RSEQ_COMPARE_TWICE 203 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 204 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2]) 205 RSEQ_ASM_OP_CMPEQ(v2, expect2, %l[error3]) 206#endif 207 RSEQ_ASM_OP_FINAL_STORE(newv, v, 3) 208 RSEQ_INJECT_ASM(6) 209 RSEQ_ASM_DEFINE_ABORT(4, abort) 210 : /* gcc asm goto does not allow outputs */ 211 : [cpu_id] "r" (cpu), 212 [current_cpu_id] "Qo" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD), 213 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), 214 [v] "Qo" (*v), 215 [expect] "r" (expect), 216 [v2] "Qo" (*v2), 217 [expect2] "r" (expect2), 218 [newv] "r" (newv) 219 RSEQ_INJECT_INPUT 220 : "memory", RSEQ_ASM_TMP_REG 221 : abort, cmpfail 222#ifdef RSEQ_COMPARE_TWICE 223 , error1, error2, error3 224#endif 225 ); 226 rseq_after_asm_goto(); 227 return 0; 228abort: 229 rseq_after_asm_goto(); 230 RSEQ_INJECT_FAILED 231 return -1; 232cmpfail: 233 rseq_after_asm_goto(); 234 return 1; 235#ifdef RSEQ_COMPARE_TWICE 236error1: 237 rseq_after_asm_goto(); 238 rseq_bug("cpu_id comparison failed"); 239error2: 240 rseq_after_asm_goto(); 241 rseq_bug("expected value comparison failed"); 242error3: 243 rseq_after_asm_goto(); 244 rseq_bug("2nd expected value comparison failed"); 245#endif 246} 247 248#endif /* #if defined(RSEQ_TEMPLATE_MO_RELAXED) && 249 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */ 250 251#if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) && \ 252 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) 253 254static inline __attribute__((always_inline)) 255int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trystorev_storev)(intptr_t *v, intptr_t expect, 256 intptr_t *v2, intptr_t newv2, 257 intptr_t newv, int cpu) 258{ 259 RSEQ_INJECT_C(9) 260 261 __asm__ __volatile__ goto ( 262 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f) 263 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail]) 264#ifdef RSEQ_COMPARE_TWICE 265 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1]) 266 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2]) 267#endif 268 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) 269 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 270 RSEQ_INJECT_ASM(3) 271 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail]) 272 RSEQ_INJECT_ASM(4) 273#ifdef RSEQ_COMPARE_TWICE 274 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 275 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2]) 276#endif 277 RSEQ_ASM_OP_STORE(newv2, v2) 278 RSEQ_INJECT_ASM(5) 279#ifdef RSEQ_TEMPLATE_MO_RELEASE 280 RSEQ_ASM_OP_FINAL_STORE_RELEASE(newv, v, 3) 281#else 282 RSEQ_ASM_OP_FINAL_STORE(newv, v, 3) 283#endif 284 RSEQ_INJECT_ASM(6) 285 RSEQ_ASM_DEFINE_ABORT(4, abort) 286 : /* gcc asm goto does not allow outputs */ 287 : [cpu_id] "r" (cpu), 288 [current_cpu_id] "Qo" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD), 289 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), 290 [expect] "r" (expect), 291 [v] "Qo" (*v), 292 [newv] "r" (newv), 293 [v2] "Qo" (*v2), 294 [newv2] "r" (newv2) 295 RSEQ_INJECT_INPUT 296 : "memory", RSEQ_ASM_TMP_REG 297 : abort, cmpfail 298#ifdef RSEQ_COMPARE_TWICE 299 , error1, error2 300#endif 301 ); 302 rseq_after_asm_goto(); 303 return 0; 304abort: 305 rseq_after_asm_goto(); 306 RSEQ_INJECT_FAILED 307 return -1; 308cmpfail: 309 rseq_after_asm_goto(); 310 return 1; 311#ifdef RSEQ_COMPARE_TWICE 312error1: 313 rseq_after_asm_goto(); 314 rseq_bug("cpu_id comparison failed"); 315error2: 316 rseq_after_asm_goto(); 317 rseq_bug("expected value comparison failed"); 318#endif 319} 320 321static inline __attribute__((always_inline)) 322int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trymemcpy_storev)(intptr_t *v, intptr_t expect, 323 void *dst, void *src, size_t len, 324 intptr_t newv, int cpu) 325{ 326 RSEQ_INJECT_C(9) 327 328 __asm__ __volatile__ goto ( 329 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f) 330 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail]) 331#ifdef RSEQ_COMPARE_TWICE 332 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1]) 333 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2]) 334#endif 335 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) 336 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 337 RSEQ_INJECT_ASM(3) 338 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail]) 339 RSEQ_INJECT_ASM(4) 340#ifdef RSEQ_COMPARE_TWICE 341 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 342 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2]) 343#endif 344 RSEQ_ASM_OP_R_BAD_MEMCPY(dst, src, len) 345 RSEQ_INJECT_ASM(5) 346#ifdef RSEQ_TEMPLATE_MO_RELEASE 347 RSEQ_ASM_OP_FINAL_STORE_RELEASE(newv, v, 3) 348#else 349 RSEQ_ASM_OP_FINAL_STORE(newv, v, 3) 350#endif 351 RSEQ_INJECT_ASM(6) 352 RSEQ_ASM_DEFINE_ABORT(4, abort) 353 : /* gcc asm goto does not allow outputs */ 354 : [cpu_id] "r" (cpu), 355 [current_cpu_id] "Qo" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD), 356 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), 357 [expect] "r" (expect), 358 [v] "Qo" (*v), 359 [newv] "r" (newv), 360 [dst] "r" (dst), 361 [src] "r" (src), 362 [len] "r" (len) 363 RSEQ_INJECT_INPUT 364 : "memory", RSEQ_ASM_TMP_REG, RSEQ_ASM_TMP_REG_2 365 : abort, cmpfail 366#ifdef RSEQ_COMPARE_TWICE 367 , error1, error2 368#endif 369 ); 370 rseq_after_asm_goto(); 371 return 0; 372abort: 373 rseq_after_asm_goto(); 374 RSEQ_INJECT_FAILED 375 return -1; 376cmpfail: 377 rseq_after_asm_goto(); 378 return 1; 379#ifdef RSEQ_COMPARE_TWICE 380error1: 381 rseq_after_asm_goto(); 382 rseq_bug("cpu_id comparison failed"); 383error2: 384 rseq_after_asm_goto(); 385 rseq_bug("expected value comparison failed"); 386#endif 387} 388 389#endif /* #if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) && 390 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */ 391 392#include "rseq-bits-reset.h" 393