1/* SPDX-License-Identifier: LGPL-2.1 OR MIT */ 2/* 3 * rseq-ppc.h 4 * 5 * (C) Copyright 2016-2018 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com> 6 * (C) Copyright 2016-2018 - Boqun Feng <boqun.feng@gmail.com> 7 */ 8 9/* 10 * RSEQ_SIG is used with the following trap instruction: 11 * 12 * powerpc-be: 0f e5 00 0b twui r5,11 13 * powerpc64-le: 0b 00 e5 0f twui r5,11 14 * powerpc64-be: 0f e5 00 0b twui r5,11 15 */ 16 17#define RSEQ_SIG 0x0fe5000b 18 19#define rseq_smp_mb() __asm__ __volatile__ ("sync" ::: "memory", "cc") 20#define rseq_smp_lwsync() __asm__ __volatile__ ("lwsync" ::: "memory", "cc") 21#define rseq_smp_rmb() rseq_smp_lwsync() 22#define rseq_smp_wmb() rseq_smp_lwsync() 23 24#define rseq_smp_load_acquire(p) \ 25__extension__ ({ \ 26 __typeof(*p) ____p1 = RSEQ_READ_ONCE(*p); \ 27 rseq_smp_lwsync(); \ 28 ____p1; \ 29}) 30 31#define rseq_smp_acquire__after_ctrl_dep() rseq_smp_lwsync() 32 33#define rseq_smp_store_release(p, v) \ 34do { \ 35 rseq_smp_lwsync(); \ 36 RSEQ_WRITE_ONCE(*p, v); \ 37} while (0) 38 39#ifdef RSEQ_SKIP_FASTPATH 40#include "rseq-skip.h" 41#else /* !RSEQ_SKIP_FASTPATH */ 42 43/* 44 * The __rseq_cs_ptr_array and __rseq_cs sections can be used by debuggers to 45 * better handle single-stepping through the restartable critical sections. 46 */ 47 48#ifdef __PPC64__ 49 50#define RSEQ_STORE_LONG(arg) "std%U[" __rseq_str(arg) "]%X[" __rseq_str(arg) "] " /* To memory ("m" constraint) */ 51#define RSEQ_STORE_INT(arg) "stw%U[" __rseq_str(arg) "]%X[" __rseq_str(arg) "] " /* To memory ("m" constraint) */ 52#define RSEQ_LOAD_LONG(arg) "ld%U[" __rseq_str(arg) "]%X[" __rseq_str(arg) "] " /* From memory ("m" constraint) */ 53#define RSEQ_LOAD_INT(arg) "lwz%U[" __rseq_str(arg) "]%X[" __rseq_str(arg) "] " /* From memory ("m" constraint) */ 54#define RSEQ_LOADX_LONG "ldx " /* From base register ("b" constraint) */ 55#define RSEQ_CMP_LONG "cmpd " 56#define RSEQ_CMP_LONG_INT "cmpdi " 57 58#define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, \ 59 start_ip, post_commit_offset, abort_ip) \ 60 ".pushsection __rseq_cs, \"aw\"\n\t" \ 61 ".balign 32\n\t" \ 62 __rseq_str(label) ":\n\t" \ 63 ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \ 64 ".quad " __rseq_str(start_ip) ", " __rseq_str(post_commit_offset) ", " __rseq_str(abort_ip) "\n\t" \ 65 ".popsection\n\t" \ 66 ".pushsection __rseq_cs_ptr_array, \"aw\"\n\t" \ 67 ".quad " __rseq_str(label) "b\n\t" \ 68 ".popsection\n\t" 69 70#define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \ 71 RSEQ_INJECT_ASM(1) \ 72 "lis %%r17, (" __rseq_str(cs_label) ")@highest\n\t" \ 73 "ori %%r17, %%r17, (" __rseq_str(cs_label) ")@higher\n\t" \ 74 "rldicr %%r17, %%r17, 32, 31\n\t" \ 75 "oris %%r17, %%r17, (" __rseq_str(cs_label) ")@high\n\t" \ 76 "ori %%r17, %%r17, (" __rseq_str(cs_label) ")@l\n\t" \ 77 "std %%r17, %[" __rseq_str(rseq_cs) "]\n\t" \ 78 __rseq_str(label) ":\n\t" 79 80/* 81 * Exit points of a rseq critical section consist of all instructions outside 82 * of the critical section where a critical section can either branch to or 83 * reach through the normal course of its execution. The abort IP and the 84 * post-commit IP are already part of the __rseq_cs section and should not be 85 * explicitly defined as additional exit points. Knowing all exit points is 86 * useful to assist debuggers stepping over the critical section. 87 */ 88#define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \ 89 ".pushsection __rseq_exit_point_array, \"aw\"\n\t" \ 90 ".quad " __rseq_str(start_ip) ", " __rseq_str(exit_ip) "\n\t" \ 91 ".popsection\n\t" 92 93#else /* #ifdef __PPC64__ */ 94 95#define RSEQ_STORE_LONG(arg) "stw%U[" __rseq_str(arg) "]%X[" __rseq_str(arg) "] " /* To memory ("m" constraint) */ 96#define RSEQ_STORE_INT(arg) RSEQ_STORE_LONG(arg) /* To memory ("m" constraint) */ 97#define RSEQ_LOAD_LONG(arg) "lwz%U[" __rseq_str(arg) "]%X[" __rseq_str(arg) "] " /* From memory ("m" constraint) */ 98#define RSEQ_LOAD_INT(arg) RSEQ_LOAD_LONG(arg) /* From memory ("m" constraint) */ 99#define RSEQ_LOADX_LONG "lwzx " /* From base register ("b" constraint) */ 100#define RSEQ_CMP_LONG "cmpw " 101#define RSEQ_CMP_LONG_INT "cmpwi " 102 103#define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, \ 104 start_ip, post_commit_offset, abort_ip) \ 105 ".pushsection __rseq_cs, \"aw\"\n\t" \ 106 ".balign 32\n\t" \ 107 __rseq_str(label) ":\n\t" \ 108 ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \ 109 /* 32-bit only supported on BE */ \ 110 ".long 0x0, " __rseq_str(start_ip) ", 0x0, " __rseq_str(post_commit_offset) ", 0x0, " __rseq_str(abort_ip) "\n\t" \ 111 ".popsection\n\t" \ 112 ".pushsection __rseq_cs_ptr_array, \"aw\"\n\t" \ 113 ".long 0x0, " __rseq_str(label) "b\n\t" \ 114 ".popsection\n\t" 115 116/* 117 * Exit points of a rseq critical section consist of all instructions outside 118 * of the critical section where a critical section can either branch to or 119 * reach through the normal course of its execution. The abort IP and the 120 * post-commit IP are already part of the __rseq_cs section and should not be 121 * explicitly defined as additional exit points. Knowing all exit points is 122 * useful to assist debuggers stepping over the critical section. 123 */ 124#define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \ 125 ".pushsection __rseq_exit_point_array, \"aw\"\n\t" \ 126 /* 32-bit only supported on BE */ \ 127 ".long 0x0, " __rseq_str(start_ip) ", 0x0, " __rseq_str(exit_ip) "\n\t" \ 128 ".popsection\n\t" 129 130#define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \ 131 RSEQ_INJECT_ASM(1) \ 132 "lis %%r17, (" __rseq_str(cs_label) ")@ha\n\t" \ 133 "addi %%r17, %%r17, (" __rseq_str(cs_label) ")@l\n\t" \ 134 RSEQ_STORE_INT(rseq_cs) "%%r17, %[" __rseq_str(rseq_cs) "]\n\t" \ 135 __rseq_str(label) ":\n\t" 136 137#endif /* #ifdef __PPC64__ */ 138 139#define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \ 140 __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip, \ 141 (post_commit_ip - start_ip), abort_ip) 142 143#define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label) \ 144 RSEQ_INJECT_ASM(2) \ 145 RSEQ_LOAD_INT(current_cpu_id) "%%r17, %[" __rseq_str(current_cpu_id) "]\n\t" \ 146 "cmpw cr7, %[" __rseq_str(cpu_id) "], %%r17\n\t" \ 147 "bne- cr7, " __rseq_str(label) "\n\t" 148 149#define RSEQ_ASM_DEFINE_ABORT(label, abort_label) \ 150 ".pushsection __rseq_failure, \"ax\"\n\t" \ 151 ".long " __rseq_str(RSEQ_SIG) "\n\t" \ 152 __rseq_str(label) ":\n\t" \ 153 "b %l[" __rseq_str(abort_label) "]\n\t" \ 154 ".popsection\n\t" 155 156/* 157 * RSEQ_ASM_OPs: asm operations for rseq 158 * RSEQ_ASM_OP_R_*: has hard-code registers in it 159 * RSEQ_ASM_OP_* (else): doesn't have hard-code registers(unless cr7) 160 */ 161#define RSEQ_ASM_OP_CMPEQ(var, expect, label) \ 162 RSEQ_LOAD_LONG(var) "%%r17, %[" __rseq_str(var) "]\n\t" \ 163 RSEQ_CMP_LONG "cr7, %%r17, %[" __rseq_str(expect) "]\n\t" \ 164 "bne- cr7, " __rseq_str(label) "\n\t" 165 166#define RSEQ_ASM_OP_CMPNE(var, expectnot, label) \ 167 RSEQ_LOAD_LONG(var) "%%r17, %[" __rseq_str(var) "]\n\t" \ 168 RSEQ_CMP_LONG "cr7, %%r17, %[" __rseq_str(expectnot) "]\n\t" \ 169 "beq- cr7, " __rseq_str(label) "\n\t" 170 171#define RSEQ_ASM_OP_STORE(value, var) \ 172 RSEQ_STORE_LONG(var) "%[" __rseq_str(value) "], %[" __rseq_str(var) "]\n\t" 173 174/* Load @var to r17 */ 175#define RSEQ_ASM_OP_R_LOAD(var) \ 176 RSEQ_LOAD_LONG(var) "%%r17, %[" __rseq_str(var) "]\n\t" 177 178/* Store r17 to @var */ 179#define RSEQ_ASM_OP_R_STORE(var) \ 180 RSEQ_STORE_LONG(var) "%%r17, %[" __rseq_str(var) "]\n\t" 181 182/* Add @count to r17 */ 183#define RSEQ_ASM_OP_R_ADD(count) \ 184 "add %%r17, %[" __rseq_str(count) "], %%r17\n\t" 185 186/* Load (r17 + voffp) to r17 */ 187#define RSEQ_ASM_OP_R_LOADX(voffp) \ 188 RSEQ_LOADX_LONG "%%r17, %[" __rseq_str(voffp) "], %%r17\n\t" 189 190/* TODO: implement a faster memcpy. */ 191#define RSEQ_ASM_OP_R_MEMCPY() \ 192 RSEQ_CMP_LONG_INT "%%r19, 0\n\t" \ 193 "beq 333f\n\t" \ 194 "addi %%r20, %%r20, -1\n\t" \ 195 "addi %%r21, %%r21, -1\n\t" \ 196 "222:\n\t" \ 197 "lbzu %%r18, 1(%%r20)\n\t" \ 198 "stbu %%r18, 1(%%r21)\n\t" \ 199 "addi %%r19, %%r19, -1\n\t" \ 200 RSEQ_CMP_LONG_INT "%%r19, 0\n\t" \ 201 "bne 222b\n\t" \ 202 "333:\n\t" \ 203 204#define RSEQ_ASM_OP_R_FINAL_STORE(var, post_commit_label) \ 205 RSEQ_STORE_LONG(var) "%%r17, %[" __rseq_str(var) "]\n\t" \ 206 __rseq_str(post_commit_label) ":\n\t" 207 208#define RSEQ_ASM_OP_FINAL_STORE(value, var, post_commit_label) \ 209 RSEQ_STORE_LONG(var) "%[" __rseq_str(value) "], %[" __rseq_str(var) "]\n\t" \ 210 __rseq_str(post_commit_label) ":\n\t" 211 212static inline __attribute__((always_inline)) 213int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu) 214{ 215 RSEQ_INJECT_C(9) 216 217 __asm__ __volatile__ goto ( 218 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 219 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 220#ifdef RSEQ_COMPARE_TWICE 221 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 222 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 223#endif 224 /* Start rseq by storing table entry pointer into rseq_cs. */ 225 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) 226 /* cmp cpuid */ 227 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 228 RSEQ_INJECT_ASM(3) 229 /* cmp @v equal to @expect */ 230 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail]) 231 RSEQ_INJECT_ASM(4) 232#ifdef RSEQ_COMPARE_TWICE 233 /* cmp cpuid */ 234 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 235 /* cmp @v equal to @expect */ 236 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2]) 237#endif 238 /* final store */ 239 RSEQ_ASM_OP_FINAL_STORE(newv, v, 2) 240 RSEQ_INJECT_ASM(5) 241 RSEQ_ASM_DEFINE_ABORT(4, abort) 242 : /* gcc asm goto does not allow outputs */ 243 : [cpu_id] "r" (cpu), 244 [current_cpu_id] "m" (rseq_get_abi()->cpu_id), 245 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), 246 [v] "m" (*v), 247 [expect] "r" (expect), 248 [newv] "r" (newv) 249 RSEQ_INJECT_INPUT 250 : "memory", "cc", "r17" 251 RSEQ_INJECT_CLOBBER 252 : abort, cmpfail 253#ifdef RSEQ_COMPARE_TWICE 254 , error1, error2 255#endif 256 ); 257 rseq_after_asm_goto(); 258 return 0; 259abort: 260 rseq_after_asm_goto(); 261 RSEQ_INJECT_FAILED 262 return -1; 263cmpfail: 264 rseq_after_asm_goto(); 265 return 1; 266#ifdef RSEQ_COMPARE_TWICE 267error1: 268 rseq_after_asm_goto(); 269 rseq_bug("cpu_id comparison failed"); 270error2: 271 rseq_after_asm_goto(); 272 rseq_bug("expected value comparison failed"); 273#endif 274} 275 276static inline __attribute__((always_inline)) 277int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot, 278 long voffp, intptr_t *load, int cpu) 279{ 280 RSEQ_INJECT_C(9) 281 282 __asm__ __volatile__ goto ( 283 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 284 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 285#ifdef RSEQ_COMPARE_TWICE 286 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 287 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 288#endif 289 /* Start rseq by storing table entry pointer into rseq_cs. */ 290 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) 291 /* cmp cpuid */ 292 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 293 RSEQ_INJECT_ASM(3) 294 /* cmp @v not equal to @expectnot */ 295 RSEQ_ASM_OP_CMPNE(v, expectnot, %l[cmpfail]) 296 RSEQ_INJECT_ASM(4) 297#ifdef RSEQ_COMPARE_TWICE 298 /* cmp cpuid */ 299 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 300 /* cmp @v not equal to @expectnot */ 301 RSEQ_ASM_OP_CMPNE(v, expectnot, %l[error2]) 302#endif 303 /* load the value of @v */ 304 RSEQ_ASM_OP_R_LOAD(v) 305 /* store it in @load */ 306 RSEQ_ASM_OP_R_STORE(load) 307 /* dereference voffp(v) */ 308 RSEQ_ASM_OP_R_LOADX(voffp) 309 /* final store the value at voffp(v) */ 310 RSEQ_ASM_OP_R_FINAL_STORE(v, 2) 311 RSEQ_INJECT_ASM(5) 312 RSEQ_ASM_DEFINE_ABORT(4, abort) 313 : /* gcc asm goto does not allow outputs */ 314 : [cpu_id] "r" (cpu), 315 [current_cpu_id] "m" (rseq_get_abi()->cpu_id), 316 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), 317 /* final store input */ 318 [v] "m" (*v), 319 [expectnot] "r" (expectnot), 320 [voffp] "b" (voffp), 321 [load] "m" (*load) 322 RSEQ_INJECT_INPUT 323 : "memory", "cc", "r17" 324 RSEQ_INJECT_CLOBBER 325 : abort, cmpfail 326#ifdef RSEQ_COMPARE_TWICE 327 , error1, error2 328#endif 329 ); 330 rseq_after_asm_goto(); 331 return 0; 332abort: 333 rseq_after_asm_goto(); 334 RSEQ_INJECT_FAILED 335 return -1; 336cmpfail: 337 rseq_after_asm_goto(); 338 return 1; 339#ifdef RSEQ_COMPARE_TWICE 340error1: 341 rseq_after_asm_goto(); 342 rseq_bug("cpu_id comparison failed"); 343error2: 344 rseq_after_asm_goto(); 345 rseq_bug("expected value comparison failed"); 346#endif 347} 348 349static inline __attribute__((always_inline)) 350int rseq_addv(intptr_t *v, intptr_t count, int cpu) 351{ 352 RSEQ_INJECT_C(9) 353 354 __asm__ __volatile__ goto ( 355 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 356#ifdef RSEQ_COMPARE_TWICE 357 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 358#endif 359 /* Start rseq by storing table entry pointer into rseq_cs. */ 360 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) 361 /* cmp cpuid */ 362 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 363 RSEQ_INJECT_ASM(3) 364#ifdef RSEQ_COMPARE_TWICE 365 /* cmp cpuid */ 366 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 367#endif 368 /* load the value of @v */ 369 RSEQ_ASM_OP_R_LOAD(v) 370 /* add @count to it */ 371 RSEQ_ASM_OP_R_ADD(count) 372 /* final store */ 373 RSEQ_ASM_OP_R_FINAL_STORE(v, 2) 374 RSEQ_INJECT_ASM(4) 375 RSEQ_ASM_DEFINE_ABORT(4, abort) 376 : /* gcc asm goto does not allow outputs */ 377 : [cpu_id] "r" (cpu), 378 [current_cpu_id] "m" (rseq_get_abi()->cpu_id), 379 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), 380 /* final store input */ 381 [v] "m" (*v), 382 [count] "r" (count) 383 RSEQ_INJECT_INPUT 384 : "memory", "cc", "r17" 385 RSEQ_INJECT_CLOBBER 386 : abort 387#ifdef RSEQ_COMPARE_TWICE 388 , error1 389#endif 390 ); 391 rseq_after_asm_goto(); 392 return 0; 393abort: 394 rseq_after_asm_goto(); 395 RSEQ_INJECT_FAILED 396 return -1; 397#ifdef RSEQ_COMPARE_TWICE 398error1: 399 rseq_after_asm_goto(); 400 rseq_bug("cpu_id comparison failed"); 401#endif 402} 403 404static inline __attribute__((always_inline)) 405int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect, 406 intptr_t *v2, intptr_t newv2, 407 intptr_t newv, int cpu) 408{ 409 RSEQ_INJECT_C(9) 410 411 __asm__ __volatile__ goto ( 412 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 413 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 414#ifdef RSEQ_COMPARE_TWICE 415 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 416 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 417#endif 418 /* Start rseq by storing table entry pointer into rseq_cs. */ 419 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) 420 /* cmp cpuid */ 421 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 422 RSEQ_INJECT_ASM(3) 423 /* cmp @v equal to @expect */ 424 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail]) 425 RSEQ_INJECT_ASM(4) 426#ifdef RSEQ_COMPARE_TWICE 427 /* cmp cpuid */ 428 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 429 /* cmp @v equal to @expect */ 430 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2]) 431#endif 432 /* try store */ 433 RSEQ_ASM_OP_STORE(newv2, v2) 434 RSEQ_INJECT_ASM(5) 435 /* final store */ 436 RSEQ_ASM_OP_FINAL_STORE(newv, v, 2) 437 RSEQ_INJECT_ASM(6) 438 RSEQ_ASM_DEFINE_ABORT(4, abort) 439 : /* gcc asm goto does not allow outputs */ 440 : [cpu_id] "r" (cpu), 441 [current_cpu_id] "m" (rseq_get_abi()->cpu_id), 442 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), 443 /* try store input */ 444 [v2] "m" (*v2), 445 [newv2] "r" (newv2), 446 /* final store input */ 447 [v] "m" (*v), 448 [expect] "r" (expect), 449 [newv] "r" (newv) 450 RSEQ_INJECT_INPUT 451 : "memory", "cc", "r17" 452 RSEQ_INJECT_CLOBBER 453 : abort, cmpfail 454#ifdef RSEQ_COMPARE_TWICE 455 , error1, error2 456#endif 457 ); 458 rseq_after_asm_goto(); 459 return 0; 460abort: 461 rseq_after_asm_goto(); 462 RSEQ_INJECT_FAILED 463 return -1; 464cmpfail: 465 rseq_after_asm_goto(); 466 return 1; 467#ifdef RSEQ_COMPARE_TWICE 468error1: 469 rseq_after_asm_goto(); 470 rseq_bug("cpu_id comparison failed"); 471error2: 472 rseq_after_asm_goto(); 473 rseq_bug("expected value comparison failed"); 474#endif 475} 476 477static inline __attribute__((always_inline)) 478int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect, 479 intptr_t *v2, intptr_t newv2, 480 intptr_t newv, int cpu) 481{ 482 RSEQ_INJECT_C(9) 483 484 __asm__ __volatile__ goto ( 485 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 486 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 487#ifdef RSEQ_COMPARE_TWICE 488 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 489 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 490#endif 491 /* Start rseq by storing table entry pointer into rseq_cs. */ 492 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) 493 /* cmp cpuid */ 494 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 495 RSEQ_INJECT_ASM(3) 496 /* cmp @v equal to @expect */ 497 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail]) 498 RSEQ_INJECT_ASM(4) 499#ifdef RSEQ_COMPARE_TWICE 500 /* cmp cpuid */ 501 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 502 /* cmp @v equal to @expect */ 503 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2]) 504#endif 505 /* try store */ 506 RSEQ_ASM_OP_STORE(newv2, v2) 507 RSEQ_INJECT_ASM(5) 508 /* for 'release' */ 509 "lwsync\n\t" 510 /* final store */ 511 RSEQ_ASM_OP_FINAL_STORE(newv, v, 2) 512 RSEQ_INJECT_ASM(6) 513 RSEQ_ASM_DEFINE_ABORT(4, abort) 514 : /* gcc asm goto does not allow outputs */ 515 : [cpu_id] "r" (cpu), 516 [current_cpu_id] "m" (rseq_get_abi()->cpu_id), 517 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), 518 /* try store input */ 519 [v2] "m" (*v2), 520 [newv2] "r" (newv2), 521 /* final store input */ 522 [v] "m" (*v), 523 [expect] "r" (expect), 524 [newv] "r" (newv) 525 RSEQ_INJECT_INPUT 526 : "memory", "cc", "r17" 527 RSEQ_INJECT_CLOBBER 528 : abort, cmpfail 529#ifdef RSEQ_COMPARE_TWICE 530 , error1, error2 531#endif 532 ); 533 rseq_after_asm_goto(); 534 return 0; 535abort: 536 rseq_after_asm_goto(); 537 RSEQ_INJECT_FAILED 538 return -1; 539cmpfail: 540 rseq_after_asm_goto(); 541 return 1; 542#ifdef RSEQ_COMPARE_TWICE 543error1: 544 rseq_after_asm_goto(); 545 rseq_bug("cpu_id comparison failed"); 546error2: 547 rseq_after_asm_goto(); 548 rseq_bug("expected value comparison failed"); 549#endif 550} 551 552static inline __attribute__((always_inline)) 553int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect, 554 intptr_t *v2, intptr_t expect2, 555 intptr_t newv, int cpu) 556{ 557 RSEQ_INJECT_C(9) 558 559 __asm__ __volatile__ goto ( 560 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 561 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 562#ifdef RSEQ_COMPARE_TWICE 563 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 564 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 565 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3]) 566#endif 567 /* Start rseq by storing table entry pointer into rseq_cs. */ 568 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) 569 /* cmp cpuid */ 570 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 571 RSEQ_INJECT_ASM(3) 572 /* cmp @v equal to @expect */ 573 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail]) 574 RSEQ_INJECT_ASM(4) 575 /* cmp @v2 equal to @expct2 */ 576 RSEQ_ASM_OP_CMPEQ(v2, expect2, %l[cmpfail]) 577 RSEQ_INJECT_ASM(5) 578#ifdef RSEQ_COMPARE_TWICE 579 /* cmp cpuid */ 580 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 581 /* cmp @v equal to @expect */ 582 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2]) 583 /* cmp @v2 equal to @expct2 */ 584 RSEQ_ASM_OP_CMPEQ(v2, expect2, %l[error3]) 585#endif 586 /* final store */ 587 RSEQ_ASM_OP_FINAL_STORE(newv, v, 2) 588 RSEQ_INJECT_ASM(6) 589 RSEQ_ASM_DEFINE_ABORT(4, abort) 590 : /* gcc asm goto does not allow outputs */ 591 : [cpu_id] "r" (cpu), 592 [current_cpu_id] "m" (rseq_get_abi()->cpu_id), 593 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), 594 /* cmp2 input */ 595 [v2] "m" (*v2), 596 [expect2] "r" (expect2), 597 /* final store input */ 598 [v] "m" (*v), 599 [expect] "r" (expect), 600 [newv] "r" (newv) 601 RSEQ_INJECT_INPUT 602 : "memory", "cc", "r17" 603 RSEQ_INJECT_CLOBBER 604 : abort, cmpfail 605#ifdef RSEQ_COMPARE_TWICE 606 , error1, error2, error3 607#endif 608 ); 609 rseq_after_asm_goto(); 610 return 0; 611abort: 612 rseq_after_asm_goto(); 613 RSEQ_INJECT_FAILED 614 return -1; 615cmpfail: 616 rseq_after_asm_goto(); 617 return 1; 618#ifdef RSEQ_COMPARE_TWICE 619error1: 620 rseq_after_asm_goto(); 621 rseq_bug("cpu_id comparison failed"); 622error2: 623 rseq_after_asm_goto(); 624 rseq_bug("1st expected value comparison failed"); 625error3: 626 rseq_after_asm_goto(); 627 rseq_bug("2nd expected value comparison failed"); 628#endif 629} 630 631static inline __attribute__((always_inline)) 632int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect, 633 void *dst, void *src, size_t len, 634 intptr_t newv, int cpu) 635{ 636 RSEQ_INJECT_C(9) 637 638 __asm__ __volatile__ goto ( 639 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 640 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 641#ifdef RSEQ_COMPARE_TWICE 642 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 643 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 644#endif 645 /* setup for mempcy */ 646 "mr %%r19, %[len]\n\t" 647 "mr %%r20, %[src]\n\t" 648 "mr %%r21, %[dst]\n\t" 649 /* Start rseq by storing table entry pointer into rseq_cs. */ 650 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) 651 /* cmp cpuid */ 652 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 653 RSEQ_INJECT_ASM(3) 654 /* cmp @v equal to @expect */ 655 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail]) 656 RSEQ_INJECT_ASM(4) 657#ifdef RSEQ_COMPARE_TWICE 658 /* cmp cpuid */ 659 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 660 /* cmp @v equal to @expect */ 661 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2]) 662#endif 663 /* try memcpy */ 664 RSEQ_ASM_OP_R_MEMCPY() 665 RSEQ_INJECT_ASM(5) 666 /* final store */ 667 RSEQ_ASM_OP_FINAL_STORE(newv, v, 2) 668 RSEQ_INJECT_ASM(6) 669 /* teardown */ 670 RSEQ_ASM_DEFINE_ABORT(4, abort) 671 : /* gcc asm goto does not allow outputs */ 672 : [cpu_id] "r" (cpu), 673 [current_cpu_id] "m" (rseq_get_abi()->cpu_id), 674 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), 675 /* final store input */ 676 [v] "m" (*v), 677 [expect] "r" (expect), 678 [newv] "r" (newv), 679 /* try memcpy input */ 680 [dst] "r" (dst), 681 [src] "r" (src), 682 [len] "r" (len) 683 RSEQ_INJECT_INPUT 684 : "memory", "cc", "r17", "r18", "r19", "r20", "r21" 685 RSEQ_INJECT_CLOBBER 686 : abort, cmpfail 687#ifdef RSEQ_COMPARE_TWICE 688 , error1, error2 689#endif 690 ); 691 rseq_after_asm_goto(); 692 return 0; 693abort: 694 rseq_after_asm_goto(); 695 RSEQ_INJECT_FAILED 696 return -1; 697cmpfail: 698 rseq_after_asm_goto(); 699 return 1; 700#ifdef RSEQ_COMPARE_TWICE 701error1: 702 rseq_after_asm_goto(); 703 rseq_bug("cpu_id comparison failed"); 704error2: 705 rseq_after_asm_goto(); 706 rseq_bug("expected value comparison failed"); 707#endif 708} 709 710static inline __attribute__((always_inline)) 711int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect, 712 void *dst, void *src, size_t len, 713 intptr_t newv, int cpu) 714{ 715 RSEQ_INJECT_C(9) 716 717 __asm__ __volatile__ goto ( 718 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 719 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 720#ifdef RSEQ_COMPARE_TWICE 721 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 722 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 723#endif 724 /* setup for mempcy */ 725 "mr %%r19, %[len]\n\t" 726 "mr %%r20, %[src]\n\t" 727 "mr %%r21, %[dst]\n\t" 728 /* Start rseq by storing table entry pointer into rseq_cs. */ 729 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) 730 /* cmp cpuid */ 731 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 732 RSEQ_INJECT_ASM(3) 733 /* cmp @v equal to @expect */ 734 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail]) 735 RSEQ_INJECT_ASM(4) 736#ifdef RSEQ_COMPARE_TWICE 737 /* cmp cpuid */ 738 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 739 /* cmp @v equal to @expect */ 740 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2]) 741#endif 742 /* try memcpy */ 743 RSEQ_ASM_OP_R_MEMCPY() 744 RSEQ_INJECT_ASM(5) 745 /* for 'release' */ 746 "lwsync\n\t" 747 /* final store */ 748 RSEQ_ASM_OP_FINAL_STORE(newv, v, 2) 749 RSEQ_INJECT_ASM(6) 750 /* teardown */ 751 RSEQ_ASM_DEFINE_ABORT(4, abort) 752 : /* gcc asm goto does not allow outputs */ 753 : [cpu_id] "r" (cpu), 754 [current_cpu_id] "m" (rseq_get_abi()->cpu_id), 755 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), 756 /* final store input */ 757 [v] "m" (*v), 758 [expect] "r" (expect), 759 [newv] "r" (newv), 760 /* try memcpy input */ 761 [dst] "r" (dst), 762 [src] "r" (src), 763 [len] "r" (len) 764 RSEQ_INJECT_INPUT 765 : "memory", "cc", "r17", "r18", "r19", "r20", "r21" 766 RSEQ_INJECT_CLOBBER 767 : abort, cmpfail 768#ifdef RSEQ_COMPARE_TWICE 769 , error1, error2 770#endif 771 ); 772 rseq_after_asm_goto(); 773 return 0; 774abort: 775 rseq_after_asm_goto(); 776 RSEQ_INJECT_FAILED 777 return -1; 778cmpfail: 779 rseq_after_asm_goto(); 780 return 1; 781#ifdef RSEQ_COMPARE_TWICE 782error1: 783 rseq_after_asm_goto(); 784 rseq_bug("cpu_id comparison failed"); 785error2: 786 rseq_after_asm_goto(); 787 rseq_bug("expected value comparison failed"); 788#endif 789} 790 791#endif /* !RSEQ_SKIP_FASTPATH */ 792