1/* SPDX-License-Identifier: LGPL-2.1 OR MIT */ 2/* 3 * Author: Paul Burton <paul.burton@mips.com> 4 * (C) Copyright 2018 MIPS Tech LLC 5 * 6 * Based on rseq-arm.h: 7 * (C) Copyright 2016-2018 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com> 8 */ 9 10/* 11 * RSEQ_SIG uses the break instruction. The instruction pattern is: 12 * 13 * On MIPS: 14 * 0350000d break 0x350 15 * 16 * On nanoMIPS: 17 * 00100350 break 0x350 18 * 19 * On microMIPS: 20 * 0000d407 break 0x350 21 * 22 * For nanoMIPS32 and microMIPS, the instruction stream is encoded as 16-bit 23 * halfwords, so the signature halfwords need to be swapped accordingly for 24 * little-endian. 25 */ 26#if defined(__nanomips__) 27# ifdef __MIPSEL__ 28# define RSEQ_SIG 0x03500010 29# else 30# define RSEQ_SIG 0x00100350 31# endif 32#elif defined(__mips_micromips) 33# ifdef __MIPSEL__ 34# define RSEQ_SIG 0xd4070000 35# else 36# define RSEQ_SIG 0x0000d407 37# endif 38#elif defined(__mips__) 39# define RSEQ_SIG 0x0350000d 40#else 41/* Unknown MIPS architecture. */ 42#endif 43 44#define rseq_smp_mb() __asm__ __volatile__ ("sync" ::: "memory") 45#define rseq_smp_rmb() rseq_smp_mb() 46#define rseq_smp_wmb() rseq_smp_mb() 47 48#define rseq_smp_load_acquire(p) \ 49__extension__ ({ \ 50 __typeof(*p) ____p1 = RSEQ_READ_ONCE(*p); \ 51 rseq_smp_mb(); \ 52 ____p1; \ 53}) 54 55#define rseq_smp_acquire__after_ctrl_dep() rseq_smp_rmb() 56 57#define rseq_smp_store_release(p, v) \ 58do { \ 59 rseq_smp_mb(); \ 60 RSEQ_WRITE_ONCE(*p, v); \ 61} while (0) 62 63#ifdef RSEQ_SKIP_FASTPATH 64#include "rseq-skip.h" 65#else /* !RSEQ_SKIP_FASTPATH */ 66 67#if _MIPS_SZLONG == 64 68# define LONG ".dword" 69# define LONG_LA "dla" 70# define LONG_L "ld" 71# define LONG_S "sd" 72# define LONG_ADDI "daddiu" 73# define U32_U64_PAD(x) x 74#elif _MIPS_SZLONG == 32 75# define LONG ".word" 76# define LONG_LA "la" 77# define LONG_L "lw" 78# define LONG_S "sw" 79# define LONG_ADDI "addiu" 80# ifdef __BIG_ENDIAN 81# define U32_U64_PAD(x) "0x0, " x 82# else 83# define U32_U64_PAD(x) x ", 0x0" 84# endif 85#else 86# error unsupported _MIPS_SZLONG 87#endif 88 89#define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, start_ip, \ 90 post_commit_offset, abort_ip) \ 91 ".pushsection __rseq_cs, \"aw\"\n\t" \ 92 ".balign 32\n\t" \ 93 __rseq_str(label) ":\n\t" \ 94 ".word " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \ 95 LONG " " U32_U64_PAD(__rseq_str(start_ip)) "\n\t" \ 96 LONG " " U32_U64_PAD(__rseq_str(post_commit_offset)) "\n\t" \ 97 LONG " " U32_U64_PAD(__rseq_str(abort_ip)) "\n\t" \ 98 ".popsection\n\t" \ 99 ".pushsection __rseq_cs_ptr_array, \"aw\"\n\t" \ 100 LONG " " U32_U64_PAD(__rseq_str(label) "b") "\n\t" \ 101 ".popsection\n\t" 102 103#define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \ 104 __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip, \ 105 (post_commit_ip - start_ip), abort_ip) 106 107/* 108 * Exit points of a rseq critical section consist of all instructions outside 109 * of the critical section where a critical section can either branch to or 110 * reach through the normal course of its execution. The abort IP and the 111 * post-commit IP are already part of the __rseq_cs section and should not be 112 * explicitly defined as additional exit points. Knowing all exit points is 113 * useful to assist debuggers stepping over the critical section. 114 */ 115#define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \ 116 ".pushsection __rseq_exit_point_array, \"aw\"\n\t" \ 117 LONG " " U32_U64_PAD(__rseq_str(start_ip)) "\n\t" \ 118 LONG " " U32_U64_PAD(__rseq_str(exit_ip)) "\n\t" \ 119 ".popsection\n\t" 120 121#define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \ 122 RSEQ_INJECT_ASM(1) \ 123 LONG_LA " $4, " __rseq_str(cs_label) "\n\t" \ 124 LONG_S " $4, %[" __rseq_str(rseq_cs) "]\n\t" \ 125 __rseq_str(label) ":\n\t" 126 127#define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label) \ 128 RSEQ_INJECT_ASM(2) \ 129 "lw $4, %[" __rseq_str(current_cpu_id) "]\n\t" \ 130 "bne $4, %[" __rseq_str(cpu_id) "], " __rseq_str(label) "\n\t" 131 132#define __RSEQ_ASM_DEFINE_ABORT(table_label, label, teardown, \ 133 abort_label, version, flags, \ 134 start_ip, post_commit_offset, abort_ip) \ 135 ".balign 32\n\t" \ 136 __rseq_str(table_label) ":\n\t" \ 137 ".word " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \ 138 LONG " " U32_U64_PAD(__rseq_str(start_ip)) "\n\t" \ 139 LONG " " U32_U64_PAD(__rseq_str(post_commit_offset)) "\n\t" \ 140 LONG " " U32_U64_PAD(__rseq_str(abort_ip)) "\n\t" \ 141 ".word " __rseq_str(RSEQ_SIG) "\n\t" \ 142 __rseq_str(label) ":\n\t" \ 143 teardown \ 144 "b %l[" __rseq_str(abort_label) "]\n\t" 145 146#define RSEQ_ASM_DEFINE_ABORT(table_label, label, teardown, abort_label, \ 147 start_ip, post_commit_ip, abort_ip) \ 148 __RSEQ_ASM_DEFINE_ABORT(table_label, label, teardown, \ 149 abort_label, 0x0, 0x0, start_ip, \ 150 (post_commit_ip - start_ip), abort_ip) 151 152#define RSEQ_ASM_DEFINE_CMPFAIL(label, teardown, cmpfail_label) \ 153 __rseq_str(label) ":\n\t" \ 154 teardown \ 155 "b %l[" __rseq_str(cmpfail_label) "]\n\t" 156 157static inline __attribute__((always_inline)) 158int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu) 159{ 160 RSEQ_INJECT_C(9) 161 162 __asm__ __volatile__ goto ( 163 RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ 164 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 165#ifdef RSEQ_COMPARE_TWICE 166 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 167 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 168#endif 169 /* Start rseq by storing table entry pointer into rseq_cs. */ 170 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) 171 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 172 RSEQ_INJECT_ASM(3) 173 LONG_L " $4, %[v]\n\t" 174 "bne $4, %[expect], %l[cmpfail]\n\t" 175 RSEQ_INJECT_ASM(4) 176#ifdef RSEQ_COMPARE_TWICE 177 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 178 LONG_L " $4, %[v]\n\t" 179 "bne $4, %[expect], %l[error2]\n\t" 180#endif 181 /* final store */ 182 LONG_S " %[newv], %[v]\n\t" 183 "2:\n\t" 184 RSEQ_INJECT_ASM(5) 185 "b 5f\n\t" 186 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f) 187 "5:\n\t" 188 : /* gcc asm goto does not allow outputs */ 189 : [cpu_id] "r" (cpu), 190 [current_cpu_id] "m" (rseq_get_abi()->cpu_id), 191 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), 192 [v] "m" (*v), 193 [expect] "r" (expect), 194 [newv] "r" (newv) 195 RSEQ_INJECT_INPUT 196 : "$4", "memory" 197 RSEQ_INJECT_CLOBBER 198 : abort, cmpfail 199#ifdef RSEQ_COMPARE_TWICE 200 , error1, error2 201#endif 202 ); 203 return 0; 204abort: 205 RSEQ_INJECT_FAILED 206 return -1; 207cmpfail: 208 return 1; 209#ifdef RSEQ_COMPARE_TWICE 210error1: 211 rseq_bug("cpu_id comparison failed"); 212error2: 213 rseq_bug("expected value comparison failed"); 214#endif 215} 216 217static inline __attribute__((always_inline)) 218int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot, 219 long voffp, intptr_t *load, int cpu) 220{ 221 RSEQ_INJECT_C(9) 222 223 __asm__ __volatile__ goto ( 224 RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ 225 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 226#ifdef RSEQ_COMPARE_TWICE 227 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 228 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 229#endif 230 /* Start rseq by storing table entry pointer into rseq_cs. */ 231 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) 232 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 233 RSEQ_INJECT_ASM(3) 234 LONG_L " $4, %[v]\n\t" 235 "beq $4, %[expectnot], %l[cmpfail]\n\t" 236 RSEQ_INJECT_ASM(4) 237#ifdef RSEQ_COMPARE_TWICE 238 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 239 LONG_L " $4, %[v]\n\t" 240 "beq $4, %[expectnot], %l[error2]\n\t" 241#endif 242 LONG_S " $4, %[load]\n\t" 243 LONG_ADDI " $4, %[voffp]\n\t" 244 LONG_L " $4, 0($4)\n\t" 245 /* final store */ 246 LONG_S " $4, %[v]\n\t" 247 "2:\n\t" 248 RSEQ_INJECT_ASM(5) 249 "b 5f\n\t" 250 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f) 251 "5:\n\t" 252 : /* gcc asm goto does not allow outputs */ 253 : [cpu_id] "r" (cpu), 254 [current_cpu_id] "m" (rseq_get_abi()->cpu_id), 255 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), 256 /* final store input */ 257 [v] "m" (*v), 258 [expectnot] "r" (expectnot), 259 [voffp] "Ir" (voffp), 260 [load] "m" (*load) 261 RSEQ_INJECT_INPUT 262 : "$4", "memory" 263 RSEQ_INJECT_CLOBBER 264 : abort, cmpfail 265#ifdef RSEQ_COMPARE_TWICE 266 , error1, error2 267#endif 268 ); 269 return 0; 270abort: 271 RSEQ_INJECT_FAILED 272 return -1; 273cmpfail: 274 return 1; 275#ifdef RSEQ_COMPARE_TWICE 276error1: 277 rseq_bug("cpu_id comparison failed"); 278error2: 279 rseq_bug("expected value comparison failed"); 280#endif 281} 282 283static inline __attribute__((always_inline)) 284int rseq_addv(intptr_t *v, intptr_t count, int cpu) 285{ 286 RSEQ_INJECT_C(9) 287 288 __asm__ __volatile__ goto ( 289 RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ 290#ifdef RSEQ_COMPARE_TWICE 291 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 292#endif 293 /* Start rseq by storing table entry pointer into rseq_cs. */ 294 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) 295 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 296 RSEQ_INJECT_ASM(3) 297#ifdef RSEQ_COMPARE_TWICE 298 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 299#endif 300 LONG_L " $4, %[v]\n\t" 301 LONG_ADDI " $4, %[count]\n\t" 302 /* final store */ 303 LONG_S " $4, %[v]\n\t" 304 "2:\n\t" 305 RSEQ_INJECT_ASM(4) 306 "b 5f\n\t" 307 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f) 308 "5:\n\t" 309 : /* gcc asm goto does not allow outputs */ 310 : [cpu_id] "r" (cpu), 311 [current_cpu_id] "m" (rseq_get_abi()->cpu_id), 312 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), 313 [v] "m" (*v), 314 [count] "Ir" (count) 315 RSEQ_INJECT_INPUT 316 : "$4", "memory" 317 RSEQ_INJECT_CLOBBER 318 : abort 319#ifdef RSEQ_COMPARE_TWICE 320 , error1 321#endif 322 ); 323 return 0; 324abort: 325 RSEQ_INJECT_FAILED 326 return -1; 327#ifdef RSEQ_COMPARE_TWICE 328error1: 329 rseq_bug("cpu_id comparison failed"); 330#endif 331} 332 333static inline __attribute__((always_inline)) 334int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect, 335 intptr_t *v2, intptr_t newv2, 336 intptr_t newv, int cpu) 337{ 338 RSEQ_INJECT_C(9) 339 340 __asm__ __volatile__ goto ( 341 RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ 342 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 343#ifdef RSEQ_COMPARE_TWICE 344 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 345 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 346#endif 347 /* Start rseq by storing table entry pointer into rseq_cs. */ 348 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) 349 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 350 RSEQ_INJECT_ASM(3) 351 LONG_L " $4, %[v]\n\t" 352 "bne $4, %[expect], %l[cmpfail]\n\t" 353 RSEQ_INJECT_ASM(4) 354#ifdef RSEQ_COMPARE_TWICE 355 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 356 LONG_L " $4, %[v]\n\t" 357 "bne $4, %[expect], %l[error2]\n\t" 358#endif 359 /* try store */ 360 LONG_S " %[newv2], %[v2]\n\t" 361 RSEQ_INJECT_ASM(5) 362 /* final store */ 363 LONG_S " %[newv], %[v]\n\t" 364 "2:\n\t" 365 RSEQ_INJECT_ASM(6) 366 "b 5f\n\t" 367 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f) 368 "5:\n\t" 369 : /* gcc asm goto does not allow outputs */ 370 : [cpu_id] "r" (cpu), 371 [current_cpu_id] "m" (rseq_get_abi()->cpu_id), 372 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), 373 /* try store input */ 374 [v2] "m" (*v2), 375 [newv2] "r" (newv2), 376 /* final store input */ 377 [v] "m" (*v), 378 [expect] "r" (expect), 379 [newv] "r" (newv) 380 RSEQ_INJECT_INPUT 381 : "$4", "memory" 382 RSEQ_INJECT_CLOBBER 383 : abort, cmpfail 384#ifdef RSEQ_COMPARE_TWICE 385 , error1, error2 386#endif 387 ); 388 return 0; 389abort: 390 RSEQ_INJECT_FAILED 391 return -1; 392cmpfail: 393 return 1; 394#ifdef RSEQ_COMPARE_TWICE 395error1: 396 rseq_bug("cpu_id comparison failed"); 397error2: 398 rseq_bug("expected value comparison failed"); 399#endif 400} 401 402static inline __attribute__((always_inline)) 403int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect, 404 intptr_t *v2, intptr_t newv2, 405 intptr_t newv, int cpu) 406{ 407 RSEQ_INJECT_C(9) 408 409 __asm__ __volatile__ goto ( 410 RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ 411 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 412#ifdef RSEQ_COMPARE_TWICE 413 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 414 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 415#endif 416 /* Start rseq by storing table entry pointer into rseq_cs. */ 417 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) 418 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 419 RSEQ_INJECT_ASM(3) 420 LONG_L " $4, %[v]\n\t" 421 "bne $4, %[expect], %l[cmpfail]\n\t" 422 RSEQ_INJECT_ASM(4) 423#ifdef RSEQ_COMPARE_TWICE 424 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 425 LONG_L " $4, %[v]\n\t" 426 "bne $4, %[expect], %l[error2]\n\t" 427#endif 428 /* try store */ 429 LONG_S " %[newv2], %[v2]\n\t" 430 RSEQ_INJECT_ASM(5) 431 "sync\n\t" /* full sync provides store-release */ 432 /* final store */ 433 LONG_S " %[newv], %[v]\n\t" 434 "2:\n\t" 435 RSEQ_INJECT_ASM(6) 436 "b 5f\n\t" 437 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f) 438 "5:\n\t" 439 : /* gcc asm goto does not allow outputs */ 440 : [cpu_id] "r" (cpu), 441 [current_cpu_id] "m" (rseq_get_abi()->cpu_id), 442 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), 443 /* try store input */ 444 [v2] "m" (*v2), 445 [newv2] "r" (newv2), 446 /* final store input */ 447 [v] "m" (*v), 448 [expect] "r" (expect), 449 [newv] "r" (newv) 450 RSEQ_INJECT_INPUT 451 : "$4", "memory" 452 RSEQ_INJECT_CLOBBER 453 : abort, cmpfail 454#ifdef RSEQ_COMPARE_TWICE 455 , error1, error2 456#endif 457 ); 458 return 0; 459abort: 460 RSEQ_INJECT_FAILED 461 return -1; 462cmpfail: 463 return 1; 464#ifdef RSEQ_COMPARE_TWICE 465error1: 466 rseq_bug("cpu_id comparison failed"); 467error2: 468 rseq_bug("expected value comparison failed"); 469#endif 470} 471 472static inline __attribute__((always_inline)) 473int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect, 474 intptr_t *v2, intptr_t expect2, 475 intptr_t newv, int cpu) 476{ 477 RSEQ_INJECT_C(9) 478 479 __asm__ __volatile__ goto ( 480 RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ 481 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 482#ifdef RSEQ_COMPARE_TWICE 483 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 484 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 485 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3]) 486#endif 487 /* Start rseq by storing table entry pointer into rseq_cs. */ 488 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) 489 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 490 RSEQ_INJECT_ASM(3) 491 LONG_L " $4, %[v]\n\t" 492 "bne $4, %[expect], %l[cmpfail]\n\t" 493 RSEQ_INJECT_ASM(4) 494 LONG_L " $4, %[v2]\n\t" 495 "bne $4, %[expect2], %l[cmpfail]\n\t" 496 RSEQ_INJECT_ASM(5) 497#ifdef RSEQ_COMPARE_TWICE 498 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 499 LONG_L " $4, %[v]\n\t" 500 "bne $4, %[expect], %l[error2]\n\t" 501 LONG_L " $4, %[v2]\n\t" 502 "bne $4, %[expect2], %l[error3]\n\t" 503#endif 504 /* final store */ 505 LONG_S " %[newv], %[v]\n\t" 506 "2:\n\t" 507 RSEQ_INJECT_ASM(6) 508 "b 5f\n\t" 509 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f) 510 "5:\n\t" 511 : /* gcc asm goto does not allow outputs */ 512 : [cpu_id] "r" (cpu), 513 [current_cpu_id] "m" (rseq_get_abi()->cpu_id), 514 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), 515 /* cmp2 input */ 516 [v2] "m" (*v2), 517 [expect2] "r" (expect2), 518 /* final store input */ 519 [v] "m" (*v), 520 [expect] "r" (expect), 521 [newv] "r" (newv) 522 RSEQ_INJECT_INPUT 523 : "$4", "memory" 524 RSEQ_INJECT_CLOBBER 525 : abort, cmpfail 526#ifdef RSEQ_COMPARE_TWICE 527 , error1, error2, error3 528#endif 529 ); 530 return 0; 531abort: 532 RSEQ_INJECT_FAILED 533 return -1; 534cmpfail: 535 return 1; 536#ifdef RSEQ_COMPARE_TWICE 537error1: 538 rseq_bug("cpu_id comparison failed"); 539error2: 540 rseq_bug("1st expected value comparison failed"); 541error3: 542 rseq_bug("2nd expected value comparison failed"); 543#endif 544} 545 546static inline __attribute__((always_inline)) 547int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect, 548 void *dst, void *src, size_t len, 549 intptr_t newv, int cpu) 550{ 551 uintptr_t rseq_scratch[3]; 552 553 RSEQ_INJECT_C(9) 554 555 __asm__ __volatile__ goto ( 556 RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ 557 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 558#ifdef RSEQ_COMPARE_TWICE 559 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 560 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 561#endif 562 LONG_S " %[src], %[rseq_scratch0]\n\t" 563 LONG_S " %[dst], %[rseq_scratch1]\n\t" 564 LONG_S " %[len], %[rseq_scratch2]\n\t" 565 /* Start rseq by storing table entry pointer into rseq_cs. */ 566 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) 567 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 568 RSEQ_INJECT_ASM(3) 569 LONG_L " $4, %[v]\n\t" 570 "bne $4, %[expect], 5f\n\t" 571 RSEQ_INJECT_ASM(4) 572#ifdef RSEQ_COMPARE_TWICE 573 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 6f) 574 LONG_L " $4, %[v]\n\t" 575 "bne $4, %[expect], 7f\n\t" 576#endif 577 /* try memcpy */ 578 "beqz %[len], 333f\n\t" \ 579 "222:\n\t" \ 580 "lb $4, 0(%[src])\n\t" \ 581 "sb $4, 0(%[dst])\n\t" \ 582 LONG_ADDI " %[src], 1\n\t" \ 583 LONG_ADDI " %[dst], 1\n\t" \ 584 LONG_ADDI " %[len], -1\n\t" \ 585 "bnez %[len], 222b\n\t" \ 586 "333:\n\t" \ 587 RSEQ_INJECT_ASM(5) 588 /* final store */ 589 LONG_S " %[newv], %[v]\n\t" 590 "2:\n\t" 591 RSEQ_INJECT_ASM(6) 592 /* teardown */ 593 LONG_L " %[len], %[rseq_scratch2]\n\t" 594 LONG_L " %[dst], %[rseq_scratch1]\n\t" 595 LONG_L " %[src], %[rseq_scratch0]\n\t" 596 "b 8f\n\t" 597 RSEQ_ASM_DEFINE_ABORT(3, 4, 598 /* teardown */ 599 LONG_L " %[len], %[rseq_scratch2]\n\t" 600 LONG_L " %[dst], %[rseq_scratch1]\n\t" 601 LONG_L " %[src], %[rseq_scratch0]\n\t", 602 abort, 1b, 2b, 4f) 603 RSEQ_ASM_DEFINE_CMPFAIL(5, 604 /* teardown */ 605 LONG_L " %[len], %[rseq_scratch2]\n\t" 606 LONG_L " %[dst], %[rseq_scratch1]\n\t" 607 LONG_L " %[src], %[rseq_scratch0]\n\t", 608 cmpfail) 609#ifdef RSEQ_COMPARE_TWICE 610 RSEQ_ASM_DEFINE_CMPFAIL(6, 611 /* teardown */ 612 LONG_L " %[len], %[rseq_scratch2]\n\t" 613 LONG_L " %[dst], %[rseq_scratch1]\n\t" 614 LONG_L " %[src], %[rseq_scratch0]\n\t", 615 error1) 616 RSEQ_ASM_DEFINE_CMPFAIL(7, 617 /* teardown */ 618 LONG_L " %[len], %[rseq_scratch2]\n\t" 619 LONG_L " %[dst], %[rseq_scratch1]\n\t" 620 LONG_L " %[src], %[rseq_scratch0]\n\t", 621 error2) 622#endif 623 "8:\n\t" 624 : /* gcc asm goto does not allow outputs */ 625 : [cpu_id] "r" (cpu), 626 [current_cpu_id] "m" (rseq_get_abi()->cpu_id), 627 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), 628 /* final store input */ 629 [v] "m" (*v), 630 [expect] "r" (expect), 631 [newv] "r" (newv), 632 /* try memcpy input */ 633 [dst] "r" (dst), 634 [src] "r" (src), 635 [len] "r" (len), 636 [rseq_scratch0] "m" (rseq_scratch[0]), 637 [rseq_scratch1] "m" (rseq_scratch[1]), 638 [rseq_scratch2] "m" (rseq_scratch[2]) 639 RSEQ_INJECT_INPUT 640 : "$4", "memory" 641 RSEQ_INJECT_CLOBBER 642 : abort, cmpfail 643#ifdef RSEQ_COMPARE_TWICE 644 , error1, error2 645#endif 646 ); 647 return 0; 648abort: 649 RSEQ_INJECT_FAILED 650 return -1; 651cmpfail: 652 return 1; 653#ifdef RSEQ_COMPARE_TWICE 654error1: 655 rseq_bug("cpu_id comparison failed"); 656error2: 657 rseq_bug("expected value comparison failed"); 658#endif 659} 660 661static inline __attribute__((always_inline)) 662int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect, 663 void *dst, void *src, size_t len, 664 intptr_t newv, int cpu) 665{ 666 uintptr_t rseq_scratch[3]; 667 668 RSEQ_INJECT_C(9) 669 670 __asm__ __volatile__ goto ( 671 RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ 672 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 673#ifdef RSEQ_COMPARE_TWICE 674 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 675 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 676#endif 677 LONG_S " %[src], %[rseq_scratch0]\n\t" 678 LONG_S " %[dst], %[rseq_scratch1]\n\t" 679 LONG_S " %[len], %[rseq_scratch2]\n\t" 680 /* Start rseq by storing table entry pointer into rseq_cs. */ 681 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) 682 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 683 RSEQ_INJECT_ASM(3) 684 LONG_L " $4, %[v]\n\t" 685 "bne $4, %[expect], 5f\n\t" 686 RSEQ_INJECT_ASM(4) 687#ifdef RSEQ_COMPARE_TWICE 688 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 6f) 689 LONG_L " $4, %[v]\n\t" 690 "bne $4, %[expect], 7f\n\t" 691#endif 692 /* try memcpy */ 693 "beqz %[len], 333f\n\t" \ 694 "222:\n\t" \ 695 "lb $4, 0(%[src])\n\t" \ 696 "sb $4, 0(%[dst])\n\t" \ 697 LONG_ADDI " %[src], 1\n\t" \ 698 LONG_ADDI " %[dst], 1\n\t" \ 699 LONG_ADDI " %[len], -1\n\t" \ 700 "bnez %[len], 222b\n\t" \ 701 "333:\n\t" \ 702 RSEQ_INJECT_ASM(5) 703 "sync\n\t" /* full sync provides store-release */ 704 /* final store */ 705 LONG_S " %[newv], %[v]\n\t" 706 "2:\n\t" 707 RSEQ_INJECT_ASM(6) 708 /* teardown */ 709 LONG_L " %[len], %[rseq_scratch2]\n\t" 710 LONG_L " %[dst], %[rseq_scratch1]\n\t" 711 LONG_L " %[src], %[rseq_scratch0]\n\t" 712 "b 8f\n\t" 713 RSEQ_ASM_DEFINE_ABORT(3, 4, 714 /* teardown */ 715 LONG_L " %[len], %[rseq_scratch2]\n\t" 716 LONG_L " %[dst], %[rseq_scratch1]\n\t" 717 LONG_L " %[src], %[rseq_scratch0]\n\t", 718 abort, 1b, 2b, 4f) 719 RSEQ_ASM_DEFINE_CMPFAIL(5, 720 /* teardown */ 721 LONG_L " %[len], %[rseq_scratch2]\n\t" 722 LONG_L " %[dst], %[rseq_scratch1]\n\t" 723 LONG_L " %[src], %[rseq_scratch0]\n\t", 724 cmpfail) 725#ifdef RSEQ_COMPARE_TWICE 726 RSEQ_ASM_DEFINE_CMPFAIL(6, 727 /* teardown */ 728 LONG_L " %[len], %[rseq_scratch2]\n\t" 729 LONG_L " %[dst], %[rseq_scratch1]\n\t" 730 LONG_L " %[src], %[rseq_scratch0]\n\t", 731 error1) 732 RSEQ_ASM_DEFINE_CMPFAIL(7, 733 /* teardown */ 734 LONG_L " %[len], %[rseq_scratch2]\n\t" 735 LONG_L " %[dst], %[rseq_scratch1]\n\t" 736 LONG_L " %[src], %[rseq_scratch0]\n\t", 737 error2) 738#endif 739 "8:\n\t" 740 : /* gcc asm goto does not allow outputs */ 741 : [cpu_id] "r" (cpu), 742 [current_cpu_id] "m" (rseq_get_abi()->cpu_id), 743 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), 744 /* final store input */ 745 [v] "m" (*v), 746 [expect] "r" (expect), 747 [newv] "r" (newv), 748 /* try memcpy input */ 749 [dst] "r" (dst), 750 [src] "r" (src), 751 [len] "r" (len), 752 [rseq_scratch0] "m" (rseq_scratch[0]), 753 [rseq_scratch1] "m" (rseq_scratch[1]), 754 [rseq_scratch2] "m" (rseq_scratch[2]) 755 RSEQ_INJECT_INPUT 756 : "$4", "memory" 757 RSEQ_INJECT_CLOBBER 758 : abort, cmpfail 759#ifdef RSEQ_COMPARE_TWICE 760 , error1, error2 761#endif 762 ); 763 return 0; 764abort: 765 RSEQ_INJECT_FAILED 766 return -1; 767cmpfail: 768 return 1; 769#ifdef RSEQ_COMPARE_TWICE 770error1: 771 rseq_bug("cpu_id comparison failed"); 772error2: 773 rseq_bug("expected value comparison failed"); 774#endif 775} 776 777#endif /* !RSEQ_SKIP_FASTPATH */ 778