1/* SPDX-License-Identifier: LGPL-2.1 OR MIT */ 2/* 3 * rseq-x86.h 4 * 5 * (C) Copyright 2016-2018 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com> 6 */ 7 8#include <stdint.h> 9 10/* 11 * RSEQ_SIG is used with the following reserved undefined instructions, which 12 * trap in user-space: 13 * 14 * x86-32: 0f b9 3d 53 30 05 53 ud1 0x53053053,%edi 15 * x86-64: 0f b9 3d 53 30 05 53 ud1 0x53053053(%rip),%edi 16 */ 17#define RSEQ_SIG 0x53053053 18 19/* 20 * Due to a compiler optimization bug in gcc-8 with asm goto and TLS asm input 21 * operands, we cannot use "m" input operands, and rather pass the __rseq_abi 22 * address through a "r" input operand. 23 */ 24 25/* Offset of cpu_id and rseq_cs fields in struct rseq. */ 26#define RSEQ_CPU_ID_OFFSET 4 27#define RSEQ_CS_OFFSET 8 28 29#ifdef __x86_64__ 30 31#define RSEQ_ASM_TP_SEGMENT %%fs 32 33#define rseq_smp_mb() \ 34 __asm__ __volatile__ ("lock; addl $0,-128(%%rsp)" ::: "memory", "cc") 35#define rseq_smp_rmb() rseq_barrier() 36#define rseq_smp_wmb() rseq_barrier() 37 38#define rseq_smp_load_acquire(p) \ 39__extension__ ({ \ 40 __typeof(*p) ____p1 = RSEQ_READ_ONCE(*p); \ 41 rseq_barrier(); \ 42 ____p1; \ 43}) 44 45#define rseq_smp_acquire__after_ctrl_dep() rseq_smp_rmb() 46 47#define rseq_smp_store_release(p, v) \ 48do { \ 49 rseq_barrier(); \ 50 RSEQ_WRITE_ONCE(*p, v); \ 51} while (0) 52 53#ifdef RSEQ_SKIP_FASTPATH 54#include "rseq-skip.h" 55#else /* !RSEQ_SKIP_FASTPATH */ 56 57#define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, \ 58 start_ip, post_commit_offset, abort_ip) \ 59 ".pushsection __rseq_cs, \"aw\"\n\t" \ 60 ".balign 32\n\t" \ 61 __rseq_str(label) ":\n\t" \ 62 ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \ 63 ".quad " __rseq_str(start_ip) ", " __rseq_str(post_commit_offset) ", " __rseq_str(abort_ip) "\n\t" \ 64 ".popsection\n\t" \ 65 ".pushsection __rseq_cs_ptr_array, \"aw\"\n\t" \ 66 ".quad " __rseq_str(label) "b\n\t" \ 67 ".popsection\n\t" 68 69 70#define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \ 71 __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip, \ 72 (post_commit_ip - start_ip), abort_ip) 73 74/* 75 * Exit points of a rseq critical section consist of all instructions outside 76 * of the critical section where a critical section can either branch to or 77 * reach through the normal course of its execution. The abort IP and the 78 * post-commit IP are already part of the __rseq_cs section and should not be 79 * explicitly defined as additional exit points. Knowing all exit points is 80 * useful to assist debuggers stepping over the critical section. 81 */ 82#define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \ 83 ".pushsection __rseq_exit_point_array, \"aw\"\n\t" \ 84 ".quad " __rseq_str(start_ip) ", " __rseq_str(exit_ip) "\n\t" \ 85 ".popsection\n\t" 86 87#define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \ 88 RSEQ_INJECT_ASM(1) \ 89 "leaq " __rseq_str(cs_label) "(%%rip), %%rax\n\t" \ 90 "movq %%rax, " __rseq_str(rseq_cs) "\n\t" \ 91 __rseq_str(label) ":\n\t" 92 93#define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label) \ 94 RSEQ_INJECT_ASM(2) \ 95 "cmpl %[" __rseq_str(cpu_id) "], " __rseq_str(current_cpu_id) "\n\t" \ 96 "jnz " __rseq_str(label) "\n\t" 97 98#define RSEQ_ASM_DEFINE_ABORT(label, teardown, abort_label) \ 99 ".pushsection __rseq_failure, \"ax\"\n\t" \ 100 /* Disassembler-friendly signature: ud1 <sig>(%rip),%edi. */ \ 101 ".byte 0x0f, 0xb9, 0x3d\n\t" \ 102 ".long " __rseq_str(RSEQ_SIG) "\n\t" \ 103 __rseq_str(label) ":\n\t" \ 104 teardown \ 105 "jmp %l[" __rseq_str(abort_label) "]\n\t" \ 106 ".popsection\n\t" 107 108#define RSEQ_ASM_DEFINE_CMPFAIL(label, teardown, cmpfail_label) \ 109 ".pushsection __rseq_failure, \"ax\"\n\t" \ 110 __rseq_str(label) ":\n\t" \ 111 teardown \ 112 "jmp %l[" __rseq_str(cmpfail_label) "]\n\t" \ 113 ".popsection\n\t" 114 115static inline __attribute__((always_inline)) 116int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu) 117{ 118 RSEQ_INJECT_C(9) 119 120 __asm__ __volatile__ goto ( 121 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 122 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 123#ifdef RSEQ_COMPARE_TWICE 124 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 125 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 126#endif 127 /* Start rseq by storing table entry pointer into rseq_cs. */ 128 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) 129 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f) 130 RSEQ_INJECT_ASM(3) 131 "cmpq %[v], %[expect]\n\t" 132 "jnz %l[cmpfail]\n\t" 133 RSEQ_INJECT_ASM(4) 134#ifdef RSEQ_COMPARE_TWICE 135 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1]) 136 "cmpq %[v], %[expect]\n\t" 137 "jnz %l[error2]\n\t" 138#endif 139 /* final store */ 140 "movq %[newv], %[v]\n\t" 141 "2:\n\t" 142 RSEQ_INJECT_ASM(5) 143 RSEQ_ASM_DEFINE_ABORT(4, "", abort) 144 : /* gcc asm goto does not allow outputs */ 145 : [cpu_id] "r" (cpu), 146 [rseq_offset] "r" (rseq_offset), 147 [v] "m" (*v), 148 [expect] "r" (expect), 149 [newv] "r" (newv) 150 : "memory", "cc", "rax" 151 RSEQ_INJECT_CLOBBER 152 : abort, cmpfail 153#ifdef RSEQ_COMPARE_TWICE 154 , error1, error2 155#endif 156 ); 157 rseq_after_asm_goto(); 158 return 0; 159abort: 160 rseq_after_asm_goto(); 161 RSEQ_INJECT_FAILED 162 return -1; 163cmpfail: 164 rseq_after_asm_goto(); 165 return 1; 166#ifdef RSEQ_COMPARE_TWICE 167error1: 168 rseq_after_asm_goto(); 169 rseq_bug("cpu_id comparison failed"); 170error2: 171 rseq_after_asm_goto(); 172 rseq_bug("expected value comparison failed"); 173#endif 174} 175 176/* 177 * Compare @v against @expectnot. When it does _not_ match, load @v 178 * into @load, and store the content of *@v + voffp into @v. 179 */ 180static inline __attribute__((always_inline)) 181int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot, 182 long voffp, intptr_t *load, int cpu) 183{ 184 RSEQ_INJECT_C(9) 185 186 __asm__ __volatile__ goto ( 187 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 188 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 189#ifdef RSEQ_COMPARE_TWICE 190 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 191 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 192#endif 193 /* Start rseq by storing table entry pointer into rseq_cs. */ 194 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) 195 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f) 196 RSEQ_INJECT_ASM(3) 197 "movq %[v], %%rbx\n\t" 198 "cmpq %%rbx, %[expectnot]\n\t" 199 "je %l[cmpfail]\n\t" 200 RSEQ_INJECT_ASM(4) 201#ifdef RSEQ_COMPARE_TWICE 202 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1]) 203 "movq %[v], %%rbx\n\t" 204 "cmpq %%rbx, %[expectnot]\n\t" 205 "je %l[error2]\n\t" 206#endif 207 "movq %%rbx, %[load]\n\t" 208 "addq %[voffp], %%rbx\n\t" 209 "movq (%%rbx), %%rbx\n\t" 210 /* final store */ 211 "movq %%rbx, %[v]\n\t" 212 "2:\n\t" 213 RSEQ_INJECT_ASM(5) 214 RSEQ_ASM_DEFINE_ABORT(4, "", abort) 215 : /* gcc asm goto does not allow outputs */ 216 : [cpu_id] "r" (cpu), 217 [rseq_offset] "r" (rseq_offset), 218 /* final store input */ 219 [v] "m" (*v), 220 [expectnot] "r" (expectnot), 221 [voffp] "er" (voffp), 222 [load] "m" (*load) 223 : "memory", "cc", "rax", "rbx" 224 RSEQ_INJECT_CLOBBER 225 : abort, cmpfail 226#ifdef RSEQ_COMPARE_TWICE 227 , error1, error2 228#endif 229 ); 230 rseq_after_asm_goto(); 231 return 0; 232abort: 233 rseq_after_asm_goto(); 234 RSEQ_INJECT_FAILED 235 return -1; 236cmpfail: 237 rseq_after_asm_goto(); 238 return 1; 239#ifdef RSEQ_COMPARE_TWICE 240error1: 241 rseq_after_asm_goto(); 242 rseq_bug("cpu_id comparison failed"); 243error2: 244 rseq_after_asm_goto(); 245 rseq_bug("expected value comparison failed"); 246#endif 247} 248 249static inline __attribute__((always_inline)) 250int rseq_addv(intptr_t *v, intptr_t count, int cpu) 251{ 252 RSEQ_INJECT_C(9) 253 254 __asm__ __volatile__ goto ( 255 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 256#ifdef RSEQ_COMPARE_TWICE 257 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 258#endif 259 /* Start rseq by storing table entry pointer into rseq_cs. */ 260 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) 261 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f) 262 RSEQ_INJECT_ASM(3) 263#ifdef RSEQ_COMPARE_TWICE 264 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1]) 265#endif 266 /* final store */ 267 "addq %[count], %[v]\n\t" 268 "2:\n\t" 269 RSEQ_INJECT_ASM(4) 270 RSEQ_ASM_DEFINE_ABORT(4, "", abort) 271 : /* gcc asm goto does not allow outputs */ 272 : [cpu_id] "r" (cpu), 273 [rseq_offset] "r" (rseq_offset), 274 /* final store input */ 275 [v] "m" (*v), 276 [count] "er" (count) 277 : "memory", "cc", "rax" 278 RSEQ_INJECT_CLOBBER 279 : abort 280#ifdef RSEQ_COMPARE_TWICE 281 , error1 282#endif 283 ); 284 rseq_after_asm_goto(); 285 return 0; 286abort: 287 rseq_after_asm_goto(); 288 RSEQ_INJECT_FAILED 289 return -1; 290#ifdef RSEQ_COMPARE_TWICE 291error1: 292 rseq_after_asm_goto(); 293 rseq_bug("cpu_id comparison failed"); 294#endif 295} 296 297#define RSEQ_ARCH_HAS_OFFSET_DEREF_ADDV 298 299/* 300 * pval = *(ptr+off) 301 * *pval += inc; 302 */ 303static inline __attribute__((always_inline)) 304int rseq_offset_deref_addv(intptr_t *ptr, long off, intptr_t inc, int cpu) 305{ 306 RSEQ_INJECT_C(9) 307 308 __asm__ __volatile__ goto ( 309 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 310#ifdef RSEQ_COMPARE_TWICE 311 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 312#endif 313 /* Start rseq by storing table entry pointer into rseq_cs. */ 314 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) 315 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f) 316 RSEQ_INJECT_ASM(3) 317#ifdef RSEQ_COMPARE_TWICE 318 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1]) 319#endif 320 /* get p+v */ 321 "movq %[ptr], %%rbx\n\t" 322 "addq %[off], %%rbx\n\t" 323 /* get pv */ 324 "movq (%%rbx), %%rcx\n\t" 325 /* *pv += inc */ 326 "addq %[inc], (%%rcx)\n\t" 327 "2:\n\t" 328 RSEQ_INJECT_ASM(4) 329 RSEQ_ASM_DEFINE_ABORT(4, "", abort) 330 : /* gcc asm goto does not allow outputs */ 331 : [cpu_id] "r" (cpu), 332 [rseq_offset] "r" (rseq_offset), 333 /* final store input */ 334 [ptr] "m" (*ptr), 335 [off] "er" (off), 336 [inc] "er" (inc) 337 : "memory", "cc", "rax", "rbx", "rcx" 338 RSEQ_INJECT_CLOBBER 339 : abort 340#ifdef RSEQ_COMPARE_TWICE 341 , error1 342#endif 343 ); 344 return 0; 345abort: 346 RSEQ_INJECT_FAILED 347 return -1; 348#ifdef RSEQ_COMPARE_TWICE 349error1: 350 rseq_bug("cpu_id comparison failed"); 351#endif 352} 353 354static inline __attribute__((always_inline)) 355int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect, 356 intptr_t *v2, intptr_t newv2, 357 intptr_t newv, int cpu) 358{ 359 RSEQ_INJECT_C(9) 360 361 __asm__ __volatile__ goto ( 362 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 363 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 364#ifdef RSEQ_COMPARE_TWICE 365 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 366 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 367#endif 368 /* Start rseq by storing table entry pointer into rseq_cs. */ 369 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) 370 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f) 371 RSEQ_INJECT_ASM(3) 372 "cmpq %[v], %[expect]\n\t" 373 "jnz %l[cmpfail]\n\t" 374 RSEQ_INJECT_ASM(4) 375#ifdef RSEQ_COMPARE_TWICE 376 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1]) 377 "cmpq %[v], %[expect]\n\t" 378 "jnz %l[error2]\n\t" 379#endif 380 /* try store */ 381 "movq %[newv2], %[v2]\n\t" 382 RSEQ_INJECT_ASM(5) 383 /* final store */ 384 "movq %[newv], %[v]\n\t" 385 "2:\n\t" 386 RSEQ_INJECT_ASM(6) 387 RSEQ_ASM_DEFINE_ABORT(4, "", abort) 388 : /* gcc asm goto does not allow outputs */ 389 : [cpu_id] "r" (cpu), 390 [rseq_offset] "r" (rseq_offset), 391 /* try store input */ 392 [v2] "m" (*v2), 393 [newv2] "r" (newv2), 394 /* final store input */ 395 [v] "m" (*v), 396 [expect] "r" (expect), 397 [newv] "r" (newv) 398 : "memory", "cc", "rax" 399 RSEQ_INJECT_CLOBBER 400 : abort, cmpfail 401#ifdef RSEQ_COMPARE_TWICE 402 , error1, error2 403#endif 404 ); 405 rseq_after_asm_goto(); 406 return 0; 407abort: 408 rseq_after_asm_goto(); 409 RSEQ_INJECT_FAILED 410 return -1; 411cmpfail: 412 rseq_after_asm_goto(); 413 return 1; 414#ifdef RSEQ_COMPARE_TWICE 415error1: 416 rseq_after_asm_goto(); 417 rseq_bug("cpu_id comparison failed"); 418error2: 419 rseq_after_asm_goto(); 420 rseq_bug("expected value comparison failed"); 421#endif 422} 423 424/* x86-64 is TSO. */ 425static inline __attribute__((always_inline)) 426int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect, 427 intptr_t *v2, intptr_t newv2, 428 intptr_t newv, int cpu) 429{ 430 return rseq_cmpeqv_trystorev_storev(v, expect, v2, newv2, newv, cpu); 431} 432 433static inline __attribute__((always_inline)) 434int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect, 435 intptr_t *v2, intptr_t expect2, 436 intptr_t newv, int cpu) 437{ 438 RSEQ_INJECT_C(9) 439 440 __asm__ __volatile__ goto ( 441 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 442 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 443#ifdef RSEQ_COMPARE_TWICE 444 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 445 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 446 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3]) 447#endif 448 /* Start rseq by storing table entry pointer into rseq_cs. */ 449 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) 450 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f) 451 RSEQ_INJECT_ASM(3) 452 "cmpq %[v], %[expect]\n\t" 453 "jnz %l[cmpfail]\n\t" 454 RSEQ_INJECT_ASM(4) 455 "cmpq %[v2], %[expect2]\n\t" 456 "jnz %l[cmpfail]\n\t" 457 RSEQ_INJECT_ASM(5) 458#ifdef RSEQ_COMPARE_TWICE 459 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1]) 460 "cmpq %[v], %[expect]\n\t" 461 "jnz %l[error2]\n\t" 462 "cmpq %[v2], %[expect2]\n\t" 463 "jnz %l[error3]\n\t" 464#endif 465 /* final store */ 466 "movq %[newv], %[v]\n\t" 467 "2:\n\t" 468 RSEQ_INJECT_ASM(6) 469 RSEQ_ASM_DEFINE_ABORT(4, "", abort) 470 : /* gcc asm goto does not allow outputs */ 471 : [cpu_id] "r" (cpu), 472 [rseq_offset] "r" (rseq_offset), 473 /* cmp2 input */ 474 [v2] "m" (*v2), 475 [expect2] "r" (expect2), 476 /* final store input */ 477 [v] "m" (*v), 478 [expect] "r" (expect), 479 [newv] "r" (newv) 480 : "memory", "cc", "rax" 481 RSEQ_INJECT_CLOBBER 482 : abort, cmpfail 483#ifdef RSEQ_COMPARE_TWICE 484 , error1, error2, error3 485#endif 486 ); 487 rseq_after_asm_goto(); 488 return 0; 489abort: 490 rseq_after_asm_goto(); 491 RSEQ_INJECT_FAILED 492 return -1; 493cmpfail: 494 rseq_after_asm_goto(); 495 return 1; 496#ifdef RSEQ_COMPARE_TWICE 497error1: 498 rseq_after_asm_goto(); 499 rseq_bug("cpu_id comparison failed"); 500error2: 501 rseq_after_asm_goto(); 502 rseq_bug("1st expected value comparison failed"); 503error3: 504 rseq_after_asm_goto(); 505 rseq_bug("2nd expected value comparison failed"); 506#endif 507} 508 509static inline __attribute__((always_inline)) 510int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect, 511 void *dst, void *src, size_t len, 512 intptr_t newv, int cpu) 513{ 514 uint64_t rseq_scratch[3]; 515 516 RSEQ_INJECT_C(9) 517 518 __asm__ __volatile__ goto ( 519 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 520 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 521#ifdef RSEQ_COMPARE_TWICE 522 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 523 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 524#endif 525 "movq %[src], %[rseq_scratch0]\n\t" 526 "movq %[dst], %[rseq_scratch1]\n\t" 527 "movq %[len], %[rseq_scratch2]\n\t" 528 /* Start rseq by storing table entry pointer into rseq_cs. */ 529 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) 530 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f) 531 RSEQ_INJECT_ASM(3) 532 "cmpq %[v], %[expect]\n\t" 533 "jnz 5f\n\t" 534 RSEQ_INJECT_ASM(4) 535#ifdef RSEQ_COMPARE_TWICE 536 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 6f) 537 "cmpq %[v], %[expect]\n\t" 538 "jnz 7f\n\t" 539#endif 540 /* try memcpy */ 541 "test %[len], %[len]\n\t" \ 542 "jz 333f\n\t" \ 543 "222:\n\t" \ 544 "movb (%[src]), %%al\n\t" \ 545 "movb %%al, (%[dst])\n\t" \ 546 "inc %[src]\n\t" \ 547 "inc %[dst]\n\t" \ 548 "dec %[len]\n\t" \ 549 "jnz 222b\n\t" \ 550 "333:\n\t" \ 551 RSEQ_INJECT_ASM(5) 552 /* final store */ 553 "movq %[newv], %[v]\n\t" 554 "2:\n\t" 555 RSEQ_INJECT_ASM(6) 556 /* teardown */ 557 "movq %[rseq_scratch2], %[len]\n\t" 558 "movq %[rseq_scratch1], %[dst]\n\t" 559 "movq %[rseq_scratch0], %[src]\n\t" 560 RSEQ_ASM_DEFINE_ABORT(4, 561 "movq %[rseq_scratch2], %[len]\n\t" 562 "movq %[rseq_scratch1], %[dst]\n\t" 563 "movq %[rseq_scratch0], %[src]\n\t", 564 abort) 565 RSEQ_ASM_DEFINE_CMPFAIL(5, 566 "movq %[rseq_scratch2], %[len]\n\t" 567 "movq %[rseq_scratch1], %[dst]\n\t" 568 "movq %[rseq_scratch0], %[src]\n\t", 569 cmpfail) 570#ifdef RSEQ_COMPARE_TWICE 571 RSEQ_ASM_DEFINE_CMPFAIL(6, 572 "movq %[rseq_scratch2], %[len]\n\t" 573 "movq %[rseq_scratch1], %[dst]\n\t" 574 "movq %[rseq_scratch0], %[src]\n\t", 575 error1) 576 RSEQ_ASM_DEFINE_CMPFAIL(7, 577 "movq %[rseq_scratch2], %[len]\n\t" 578 "movq %[rseq_scratch1], %[dst]\n\t" 579 "movq %[rseq_scratch0], %[src]\n\t", 580 error2) 581#endif 582 : /* gcc asm goto does not allow outputs */ 583 : [cpu_id] "r" (cpu), 584 [rseq_offset] "r" (rseq_offset), 585 /* final store input */ 586 [v] "m" (*v), 587 [expect] "r" (expect), 588 [newv] "r" (newv), 589 /* try memcpy input */ 590 [dst] "r" (dst), 591 [src] "r" (src), 592 [len] "r" (len), 593 [rseq_scratch0] "m" (rseq_scratch[0]), 594 [rseq_scratch1] "m" (rseq_scratch[1]), 595 [rseq_scratch2] "m" (rseq_scratch[2]) 596 : "memory", "cc", "rax" 597 RSEQ_INJECT_CLOBBER 598 : abort, cmpfail 599#ifdef RSEQ_COMPARE_TWICE 600 , error1, error2 601#endif 602 ); 603 rseq_after_asm_goto(); 604 return 0; 605abort: 606 rseq_after_asm_goto(); 607 RSEQ_INJECT_FAILED 608 return -1; 609cmpfail: 610 rseq_after_asm_goto(); 611 return 1; 612#ifdef RSEQ_COMPARE_TWICE 613error1: 614 rseq_after_asm_goto(); 615 rseq_bug("cpu_id comparison failed"); 616error2: 617 rseq_after_asm_goto(); 618 rseq_bug("expected value comparison failed"); 619#endif 620} 621 622/* x86-64 is TSO. */ 623static inline __attribute__((always_inline)) 624int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect, 625 void *dst, void *src, size_t len, 626 intptr_t newv, int cpu) 627{ 628 return rseq_cmpeqv_trymemcpy_storev(v, expect, dst, src, len, 629 newv, cpu); 630} 631 632#endif /* !RSEQ_SKIP_FASTPATH */ 633 634#elif defined(__i386__) 635 636#define RSEQ_ASM_TP_SEGMENT %%gs 637 638#define rseq_smp_mb() \ 639 __asm__ __volatile__ ("lock; addl $0,-128(%%esp)" ::: "memory", "cc") 640#define rseq_smp_rmb() \ 641 __asm__ __volatile__ ("lock; addl $0,-128(%%esp)" ::: "memory", "cc") 642#define rseq_smp_wmb() \ 643 __asm__ __volatile__ ("lock; addl $0,-128(%%esp)" ::: "memory", "cc") 644 645#define rseq_smp_load_acquire(p) \ 646__extension__ ({ \ 647 __typeof(*p) ____p1 = RSEQ_READ_ONCE(*p); \ 648 rseq_smp_mb(); \ 649 ____p1; \ 650}) 651 652#define rseq_smp_acquire__after_ctrl_dep() rseq_smp_rmb() 653 654#define rseq_smp_store_release(p, v) \ 655do { \ 656 rseq_smp_mb(); \ 657 RSEQ_WRITE_ONCE(*p, v); \ 658} while (0) 659 660#ifdef RSEQ_SKIP_FASTPATH 661#include "rseq-skip.h" 662#else /* !RSEQ_SKIP_FASTPATH */ 663 664/* 665 * Use eax as scratch register and take memory operands as input to 666 * lessen register pressure. Especially needed when compiling in O0. 667 */ 668#define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, \ 669 start_ip, post_commit_offset, abort_ip) \ 670 ".pushsection __rseq_cs, \"aw\"\n\t" \ 671 ".balign 32\n\t" \ 672 __rseq_str(label) ":\n\t" \ 673 ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \ 674 ".long " __rseq_str(start_ip) ", 0x0, " __rseq_str(post_commit_offset) ", 0x0, " __rseq_str(abort_ip) ", 0x0\n\t" \ 675 ".popsection\n\t" \ 676 ".pushsection __rseq_cs_ptr_array, \"aw\"\n\t" \ 677 ".long " __rseq_str(label) "b, 0x0\n\t" \ 678 ".popsection\n\t" 679 680#define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \ 681 __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip, \ 682 (post_commit_ip - start_ip), abort_ip) 683 684/* 685 * Exit points of a rseq critical section consist of all instructions outside 686 * of the critical section where a critical section can either branch to or 687 * reach through the normal course of its execution. The abort IP and the 688 * post-commit IP are already part of the __rseq_cs section and should not be 689 * explicitly defined as additional exit points. Knowing all exit points is 690 * useful to assist debuggers stepping over the critical section. 691 */ 692#define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \ 693 ".pushsection __rseq_exit_point_array, \"aw\"\n\t" \ 694 ".long " __rseq_str(start_ip) ", 0x0, " __rseq_str(exit_ip) ", 0x0\n\t" \ 695 ".popsection\n\t" 696 697#define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \ 698 RSEQ_INJECT_ASM(1) \ 699 "movl $" __rseq_str(cs_label) ", " __rseq_str(rseq_cs) "\n\t" \ 700 __rseq_str(label) ":\n\t" 701 702#define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label) \ 703 RSEQ_INJECT_ASM(2) \ 704 "cmpl %[" __rseq_str(cpu_id) "], " __rseq_str(current_cpu_id) "\n\t" \ 705 "jnz " __rseq_str(label) "\n\t" 706 707#define RSEQ_ASM_DEFINE_ABORT(label, teardown, abort_label) \ 708 ".pushsection __rseq_failure, \"ax\"\n\t" \ 709 /* Disassembler-friendly signature: ud1 <sig>,%edi. */ \ 710 ".byte 0x0f, 0xb9, 0x3d\n\t" \ 711 ".long " __rseq_str(RSEQ_SIG) "\n\t" \ 712 __rseq_str(label) ":\n\t" \ 713 teardown \ 714 "jmp %l[" __rseq_str(abort_label) "]\n\t" \ 715 ".popsection\n\t" 716 717#define RSEQ_ASM_DEFINE_CMPFAIL(label, teardown, cmpfail_label) \ 718 ".pushsection __rseq_failure, \"ax\"\n\t" \ 719 __rseq_str(label) ":\n\t" \ 720 teardown \ 721 "jmp %l[" __rseq_str(cmpfail_label) "]\n\t" \ 722 ".popsection\n\t" 723 724static inline __attribute__((always_inline)) 725int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu) 726{ 727 RSEQ_INJECT_C(9) 728 729 __asm__ __volatile__ goto ( 730 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 731 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 732#ifdef RSEQ_COMPARE_TWICE 733 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 734 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 735#endif 736 /* Start rseq by storing table entry pointer into rseq_cs. */ 737 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) 738 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f) 739 RSEQ_INJECT_ASM(3) 740 "cmpl %[v], %[expect]\n\t" 741 "jnz %l[cmpfail]\n\t" 742 RSEQ_INJECT_ASM(4) 743#ifdef RSEQ_COMPARE_TWICE 744 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1]) 745 "cmpl %[v], %[expect]\n\t" 746 "jnz %l[error2]\n\t" 747#endif 748 /* final store */ 749 "movl %[newv], %[v]\n\t" 750 "2:\n\t" 751 RSEQ_INJECT_ASM(5) 752 RSEQ_ASM_DEFINE_ABORT(4, "", abort) 753 : /* gcc asm goto does not allow outputs */ 754 : [cpu_id] "r" (cpu), 755 [rseq_offset] "r" (rseq_offset), 756 [v] "m" (*v), 757 [expect] "r" (expect), 758 [newv] "r" (newv) 759 : "memory", "cc", "eax" 760 RSEQ_INJECT_CLOBBER 761 : abort, cmpfail 762#ifdef RSEQ_COMPARE_TWICE 763 , error1, error2 764#endif 765 ); 766 rseq_after_asm_goto(); 767 return 0; 768abort: 769 rseq_after_asm_goto(); 770 RSEQ_INJECT_FAILED 771 return -1; 772cmpfail: 773 rseq_after_asm_goto(); 774 return 1; 775#ifdef RSEQ_COMPARE_TWICE 776error1: 777 rseq_after_asm_goto(); 778 rseq_bug("cpu_id comparison failed"); 779error2: 780 rseq_after_asm_goto(); 781 rseq_bug("expected value comparison failed"); 782#endif 783} 784 785/* 786 * Compare @v against @expectnot. When it does _not_ match, load @v 787 * into @load, and store the content of *@v + voffp into @v. 788 */ 789static inline __attribute__((always_inline)) 790int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot, 791 long voffp, intptr_t *load, int cpu) 792{ 793 RSEQ_INJECT_C(9) 794 795 __asm__ __volatile__ goto ( 796 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 797 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 798#ifdef RSEQ_COMPARE_TWICE 799 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 800 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 801#endif 802 /* Start rseq by storing table entry pointer into rseq_cs. */ 803 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) 804 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f) 805 RSEQ_INJECT_ASM(3) 806 "movl %[v], %%ebx\n\t" 807 "cmpl %%ebx, %[expectnot]\n\t" 808 "je %l[cmpfail]\n\t" 809 RSEQ_INJECT_ASM(4) 810#ifdef RSEQ_COMPARE_TWICE 811 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1]) 812 "movl %[v], %%ebx\n\t" 813 "cmpl %%ebx, %[expectnot]\n\t" 814 "je %l[error2]\n\t" 815#endif 816 "movl %%ebx, %[load]\n\t" 817 "addl %[voffp], %%ebx\n\t" 818 "movl (%%ebx), %%ebx\n\t" 819 /* final store */ 820 "movl %%ebx, %[v]\n\t" 821 "2:\n\t" 822 RSEQ_INJECT_ASM(5) 823 RSEQ_ASM_DEFINE_ABORT(4, "", abort) 824 : /* gcc asm goto does not allow outputs */ 825 : [cpu_id] "r" (cpu), 826 [rseq_offset] "r" (rseq_offset), 827 /* final store input */ 828 [v] "m" (*v), 829 [expectnot] "r" (expectnot), 830 [voffp] "ir" (voffp), 831 [load] "m" (*load) 832 : "memory", "cc", "eax", "ebx" 833 RSEQ_INJECT_CLOBBER 834 : abort, cmpfail 835#ifdef RSEQ_COMPARE_TWICE 836 , error1, error2 837#endif 838 ); 839 rseq_after_asm_goto(); 840 return 0; 841abort: 842 rseq_after_asm_goto(); 843 RSEQ_INJECT_FAILED 844 return -1; 845cmpfail: 846 rseq_after_asm_goto(); 847 return 1; 848#ifdef RSEQ_COMPARE_TWICE 849error1: 850 rseq_after_asm_goto(); 851 rseq_bug("cpu_id comparison failed"); 852error2: 853 rseq_after_asm_goto(); 854 rseq_bug("expected value comparison failed"); 855#endif 856} 857 858static inline __attribute__((always_inline)) 859int rseq_addv(intptr_t *v, intptr_t count, int cpu) 860{ 861 RSEQ_INJECT_C(9) 862 863 __asm__ __volatile__ goto ( 864 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 865#ifdef RSEQ_COMPARE_TWICE 866 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 867#endif 868 /* Start rseq by storing table entry pointer into rseq_cs. */ 869 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) 870 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f) 871 RSEQ_INJECT_ASM(3) 872#ifdef RSEQ_COMPARE_TWICE 873 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1]) 874#endif 875 /* final store */ 876 "addl %[count], %[v]\n\t" 877 "2:\n\t" 878 RSEQ_INJECT_ASM(4) 879 RSEQ_ASM_DEFINE_ABORT(4, "", abort) 880 : /* gcc asm goto does not allow outputs */ 881 : [cpu_id] "r" (cpu), 882 [rseq_offset] "r" (rseq_offset), 883 /* final store input */ 884 [v] "m" (*v), 885 [count] "ir" (count) 886 : "memory", "cc", "eax" 887 RSEQ_INJECT_CLOBBER 888 : abort 889#ifdef RSEQ_COMPARE_TWICE 890 , error1 891#endif 892 ); 893 rseq_after_asm_goto(); 894 return 0; 895abort: 896 rseq_after_asm_goto(); 897 RSEQ_INJECT_FAILED 898 return -1; 899#ifdef RSEQ_COMPARE_TWICE 900error1: 901 rseq_after_asm_goto(); 902 rseq_bug("cpu_id comparison failed"); 903#endif 904} 905 906static inline __attribute__((always_inline)) 907int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect, 908 intptr_t *v2, intptr_t newv2, 909 intptr_t newv, int cpu) 910{ 911 RSEQ_INJECT_C(9) 912 913 __asm__ __volatile__ goto ( 914 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 915 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 916#ifdef RSEQ_COMPARE_TWICE 917 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 918 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 919#endif 920 /* Start rseq by storing table entry pointer into rseq_cs. */ 921 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) 922 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f) 923 RSEQ_INJECT_ASM(3) 924 "cmpl %[v], %[expect]\n\t" 925 "jnz %l[cmpfail]\n\t" 926 RSEQ_INJECT_ASM(4) 927#ifdef RSEQ_COMPARE_TWICE 928 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1]) 929 "cmpl %[v], %[expect]\n\t" 930 "jnz %l[error2]\n\t" 931#endif 932 /* try store */ 933 "movl %[newv2], %%eax\n\t" 934 "movl %%eax, %[v2]\n\t" 935 RSEQ_INJECT_ASM(5) 936 /* final store */ 937 "movl %[newv], %[v]\n\t" 938 "2:\n\t" 939 RSEQ_INJECT_ASM(6) 940 RSEQ_ASM_DEFINE_ABORT(4, "", abort) 941 : /* gcc asm goto does not allow outputs */ 942 : [cpu_id] "r" (cpu), 943 [rseq_offset] "r" (rseq_offset), 944 /* try store input */ 945 [v2] "m" (*v2), 946 [newv2] "m" (newv2), 947 /* final store input */ 948 [v] "m" (*v), 949 [expect] "r" (expect), 950 [newv] "r" (newv) 951 : "memory", "cc", "eax" 952 RSEQ_INJECT_CLOBBER 953 : abort, cmpfail 954#ifdef RSEQ_COMPARE_TWICE 955 , error1, error2 956#endif 957 ); 958 rseq_after_asm_goto(); 959 return 0; 960abort: 961 rseq_after_asm_goto(); 962 RSEQ_INJECT_FAILED 963 return -1; 964cmpfail: 965 rseq_after_asm_goto(); 966 return 1; 967#ifdef RSEQ_COMPARE_TWICE 968error1: 969 rseq_after_asm_goto(); 970 rseq_bug("cpu_id comparison failed"); 971error2: 972 rseq_after_asm_goto(); 973 rseq_bug("expected value comparison failed"); 974#endif 975} 976 977static inline __attribute__((always_inline)) 978int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect, 979 intptr_t *v2, intptr_t newv2, 980 intptr_t newv, int cpu) 981{ 982 RSEQ_INJECT_C(9) 983 984 __asm__ __volatile__ goto ( 985 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 986 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 987#ifdef RSEQ_COMPARE_TWICE 988 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 989 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 990#endif 991 /* Start rseq by storing table entry pointer into rseq_cs. */ 992 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) 993 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f) 994 RSEQ_INJECT_ASM(3) 995 "movl %[expect], %%eax\n\t" 996 "cmpl %[v], %%eax\n\t" 997 "jnz %l[cmpfail]\n\t" 998 RSEQ_INJECT_ASM(4) 999#ifdef RSEQ_COMPARE_TWICE 1000 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1]) 1001 "movl %[expect], %%eax\n\t" 1002 "cmpl %[v], %%eax\n\t" 1003 "jnz %l[error2]\n\t" 1004#endif 1005 /* try store */ 1006 "movl %[newv2], %[v2]\n\t" 1007 RSEQ_INJECT_ASM(5) 1008 "lock; addl $0,-128(%%esp)\n\t" 1009 /* final store */ 1010 "movl %[newv], %[v]\n\t" 1011 "2:\n\t" 1012 RSEQ_INJECT_ASM(6) 1013 RSEQ_ASM_DEFINE_ABORT(4, "", abort) 1014 : /* gcc asm goto does not allow outputs */ 1015 : [cpu_id] "r" (cpu), 1016 [rseq_offset] "r" (rseq_offset), 1017 /* try store input */ 1018 [v2] "m" (*v2), 1019 [newv2] "r" (newv2), 1020 /* final store input */ 1021 [v] "m" (*v), 1022 [expect] "m" (expect), 1023 [newv] "r" (newv) 1024 : "memory", "cc", "eax" 1025 RSEQ_INJECT_CLOBBER 1026 : abort, cmpfail 1027#ifdef RSEQ_COMPARE_TWICE 1028 , error1, error2 1029#endif 1030 ); 1031 rseq_after_asm_goto(); 1032 return 0; 1033abort: 1034 rseq_after_asm_goto(); 1035 RSEQ_INJECT_FAILED 1036 return -1; 1037cmpfail: 1038 rseq_after_asm_goto(); 1039 return 1; 1040#ifdef RSEQ_COMPARE_TWICE 1041error1: 1042 rseq_after_asm_goto(); 1043 rseq_bug("cpu_id comparison failed"); 1044error2: 1045 rseq_after_asm_goto(); 1046 rseq_bug("expected value comparison failed"); 1047#endif 1048 1049} 1050 1051static inline __attribute__((always_inline)) 1052int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect, 1053 intptr_t *v2, intptr_t expect2, 1054 intptr_t newv, int cpu) 1055{ 1056 RSEQ_INJECT_C(9) 1057 1058 __asm__ __volatile__ goto ( 1059 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 1060 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 1061#ifdef RSEQ_COMPARE_TWICE 1062 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 1063 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 1064 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3]) 1065#endif 1066 /* Start rseq by storing table entry pointer into rseq_cs. */ 1067 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) 1068 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f) 1069 RSEQ_INJECT_ASM(3) 1070 "cmpl %[v], %[expect]\n\t" 1071 "jnz %l[cmpfail]\n\t" 1072 RSEQ_INJECT_ASM(4) 1073 "cmpl %[expect2], %[v2]\n\t" 1074 "jnz %l[cmpfail]\n\t" 1075 RSEQ_INJECT_ASM(5) 1076#ifdef RSEQ_COMPARE_TWICE 1077 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1]) 1078 "cmpl %[v], %[expect]\n\t" 1079 "jnz %l[error2]\n\t" 1080 "cmpl %[expect2], %[v2]\n\t" 1081 "jnz %l[error3]\n\t" 1082#endif 1083 "movl %[newv], %%eax\n\t" 1084 /* final store */ 1085 "movl %%eax, %[v]\n\t" 1086 "2:\n\t" 1087 RSEQ_INJECT_ASM(6) 1088 RSEQ_ASM_DEFINE_ABORT(4, "", abort) 1089 : /* gcc asm goto does not allow outputs */ 1090 : [cpu_id] "r" (cpu), 1091 [rseq_offset] "r" (rseq_offset), 1092 /* cmp2 input */ 1093 [v2] "m" (*v2), 1094 [expect2] "r" (expect2), 1095 /* final store input */ 1096 [v] "m" (*v), 1097 [expect] "r" (expect), 1098 [newv] "m" (newv) 1099 : "memory", "cc", "eax" 1100 RSEQ_INJECT_CLOBBER 1101 : abort, cmpfail 1102#ifdef RSEQ_COMPARE_TWICE 1103 , error1, error2, error3 1104#endif 1105 ); 1106 rseq_after_asm_goto(); 1107 return 0; 1108abort: 1109 rseq_after_asm_goto(); 1110 RSEQ_INJECT_FAILED 1111 return -1; 1112cmpfail: 1113 rseq_after_asm_goto(); 1114 return 1; 1115#ifdef RSEQ_COMPARE_TWICE 1116error1: 1117 rseq_after_asm_goto(); 1118 rseq_bug("cpu_id comparison failed"); 1119error2: 1120 rseq_after_asm_goto(); 1121 rseq_bug("1st expected value comparison failed"); 1122error3: 1123 rseq_after_asm_goto(); 1124 rseq_bug("2nd expected value comparison failed"); 1125#endif 1126} 1127 1128/* TODO: implement a faster memcpy. */ 1129static inline __attribute__((always_inline)) 1130int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect, 1131 void *dst, void *src, size_t len, 1132 intptr_t newv, int cpu) 1133{ 1134 uint32_t rseq_scratch[3]; 1135 1136 RSEQ_INJECT_C(9) 1137 1138 __asm__ __volatile__ goto ( 1139 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 1140 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 1141#ifdef RSEQ_COMPARE_TWICE 1142 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 1143 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 1144#endif 1145 "movl %[src], %[rseq_scratch0]\n\t" 1146 "movl %[dst], %[rseq_scratch1]\n\t" 1147 "movl %[len], %[rseq_scratch2]\n\t" 1148 /* Start rseq by storing table entry pointer into rseq_cs. */ 1149 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) 1150 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f) 1151 RSEQ_INJECT_ASM(3) 1152 "movl %[expect], %%eax\n\t" 1153 "cmpl %%eax, %[v]\n\t" 1154 "jnz 5f\n\t" 1155 RSEQ_INJECT_ASM(4) 1156#ifdef RSEQ_COMPARE_TWICE 1157 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 6f) 1158 "movl %[expect], %%eax\n\t" 1159 "cmpl %%eax, %[v]\n\t" 1160 "jnz 7f\n\t" 1161#endif 1162 /* try memcpy */ 1163 "test %[len], %[len]\n\t" \ 1164 "jz 333f\n\t" \ 1165 "222:\n\t" \ 1166 "movb (%[src]), %%al\n\t" \ 1167 "movb %%al, (%[dst])\n\t" \ 1168 "inc %[src]\n\t" \ 1169 "inc %[dst]\n\t" \ 1170 "dec %[len]\n\t" \ 1171 "jnz 222b\n\t" \ 1172 "333:\n\t" \ 1173 RSEQ_INJECT_ASM(5) 1174 "movl %[newv], %%eax\n\t" 1175 /* final store */ 1176 "movl %%eax, %[v]\n\t" 1177 "2:\n\t" 1178 RSEQ_INJECT_ASM(6) 1179 /* teardown */ 1180 "movl %[rseq_scratch2], %[len]\n\t" 1181 "movl %[rseq_scratch1], %[dst]\n\t" 1182 "movl %[rseq_scratch0], %[src]\n\t" 1183 RSEQ_ASM_DEFINE_ABORT(4, 1184 "movl %[rseq_scratch2], %[len]\n\t" 1185 "movl %[rseq_scratch1], %[dst]\n\t" 1186 "movl %[rseq_scratch0], %[src]\n\t", 1187 abort) 1188 RSEQ_ASM_DEFINE_CMPFAIL(5, 1189 "movl %[rseq_scratch2], %[len]\n\t" 1190 "movl %[rseq_scratch1], %[dst]\n\t" 1191 "movl %[rseq_scratch0], %[src]\n\t", 1192 cmpfail) 1193#ifdef RSEQ_COMPARE_TWICE 1194 RSEQ_ASM_DEFINE_CMPFAIL(6, 1195 "movl %[rseq_scratch2], %[len]\n\t" 1196 "movl %[rseq_scratch1], %[dst]\n\t" 1197 "movl %[rseq_scratch0], %[src]\n\t", 1198 error1) 1199 RSEQ_ASM_DEFINE_CMPFAIL(7, 1200 "movl %[rseq_scratch2], %[len]\n\t" 1201 "movl %[rseq_scratch1], %[dst]\n\t" 1202 "movl %[rseq_scratch0], %[src]\n\t", 1203 error2) 1204#endif 1205 : /* gcc asm goto does not allow outputs */ 1206 : [cpu_id] "r" (cpu), 1207 [rseq_offset] "r" (rseq_offset), 1208 /* final store input */ 1209 [v] "m" (*v), 1210 [expect] "m" (expect), 1211 [newv] "m" (newv), 1212 /* try memcpy input */ 1213 [dst] "r" (dst), 1214 [src] "r" (src), 1215 [len] "r" (len), 1216 [rseq_scratch0] "m" (rseq_scratch[0]), 1217 [rseq_scratch1] "m" (rseq_scratch[1]), 1218 [rseq_scratch2] "m" (rseq_scratch[2]) 1219 : "memory", "cc", "eax" 1220 RSEQ_INJECT_CLOBBER 1221 : abort, cmpfail 1222#ifdef RSEQ_COMPARE_TWICE 1223 , error1, error2 1224#endif 1225 ); 1226 rseq_after_asm_goto(); 1227 return 0; 1228abort: 1229 rseq_after_asm_goto(); 1230 RSEQ_INJECT_FAILED 1231 return -1; 1232cmpfail: 1233 rseq_after_asm_goto(); 1234 return 1; 1235#ifdef RSEQ_COMPARE_TWICE 1236error1: 1237 rseq_after_asm_goto(); 1238 rseq_bug("cpu_id comparison failed"); 1239error2: 1240 rseq_after_asm_goto(); 1241 rseq_bug("expected value comparison failed"); 1242#endif 1243} 1244 1245/* TODO: implement a faster memcpy. */ 1246static inline __attribute__((always_inline)) 1247int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect, 1248 void *dst, void *src, size_t len, 1249 intptr_t newv, int cpu) 1250{ 1251 uint32_t rseq_scratch[3]; 1252 1253 RSEQ_INJECT_C(9) 1254 1255 __asm__ __volatile__ goto ( 1256 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 1257 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 1258#ifdef RSEQ_COMPARE_TWICE 1259 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 1260 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 1261#endif 1262 "movl %[src], %[rseq_scratch0]\n\t" 1263 "movl %[dst], %[rseq_scratch1]\n\t" 1264 "movl %[len], %[rseq_scratch2]\n\t" 1265 /* Start rseq by storing table entry pointer into rseq_cs. */ 1266 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) 1267 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f) 1268 RSEQ_INJECT_ASM(3) 1269 "movl %[expect], %%eax\n\t" 1270 "cmpl %%eax, %[v]\n\t" 1271 "jnz 5f\n\t" 1272 RSEQ_INJECT_ASM(4) 1273#ifdef RSEQ_COMPARE_TWICE 1274 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 6f) 1275 "movl %[expect], %%eax\n\t" 1276 "cmpl %%eax, %[v]\n\t" 1277 "jnz 7f\n\t" 1278#endif 1279 /* try memcpy */ 1280 "test %[len], %[len]\n\t" \ 1281 "jz 333f\n\t" \ 1282 "222:\n\t" \ 1283 "movb (%[src]), %%al\n\t" \ 1284 "movb %%al, (%[dst])\n\t" \ 1285 "inc %[src]\n\t" \ 1286 "inc %[dst]\n\t" \ 1287 "dec %[len]\n\t" \ 1288 "jnz 222b\n\t" \ 1289 "333:\n\t" \ 1290 RSEQ_INJECT_ASM(5) 1291 "lock; addl $0,-128(%%esp)\n\t" 1292 "movl %[newv], %%eax\n\t" 1293 /* final store */ 1294 "movl %%eax, %[v]\n\t" 1295 "2:\n\t" 1296 RSEQ_INJECT_ASM(6) 1297 /* teardown */ 1298 "movl %[rseq_scratch2], %[len]\n\t" 1299 "movl %[rseq_scratch1], %[dst]\n\t" 1300 "movl %[rseq_scratch0], %[src]\n\t" 1301 RSEQ_ASM_DEFINE_ABORT(4, 1302 "movl %[rseq_scratch2], %[len]\n\t" 1303 "movl %[rseq_scratch1], %[dst]\n\t" 1304 "movl %[rseq_scratch0], %[src]\n\t", 1305 abort) 1306 RSEQ_ASM_DEFINE_CMPFAIL(5, 1307 "movl %[rseq_scratch2], %[len]\n\t" 1308 "movl %[rseq_scratch1], %[dst]\n\t" 1309 "movl %[rseq_scratch0], %[src]\n\t", 1310 cmpfail) 1311#ifdef RSEQ_COMPARE_TWICE 1312 RSEQ_ASM_DEFINE_CMPFAIL(6, 1313 "movl %[rseq_scratch2], %[len]\n\t" 1314 "movl %[rseq_scratch1], %[dst]\n\t" 1315 "movl %[rseq_scratch0], %[src]\n\t", 1316 error1) 1317 RSEQ_ASM_DEFINE_CMPFAIL(7, 1318 "movl %[rseq_scratch2], %[len]\n\t" 1319 "movl %[rseq_scratch1], %[dst]\n\t" 1320 "movl %[rseq_scratch0], %[src]\n\t", 1321 error2) 1322#endif 1323 : /* gcc asm goto does not allow outputs */ 1324 : [cpu_id] "r" (cpu), 1325 [rseq_offset] "r" (rseq_offset), 1326 /* final store input */ 1327 [v] "m" (*v), 1328 [expect] "m" (expect), 1329 [newv] "m" (newv), 1330 /* try memcpy input */ 1331 [dst] "r" (dst), 1332 [src] "r" (src), 1333 [len] "r" (len), 1334 [rseq_scratch0] "m" (rseq_scratch[0]), 1335 [rseq_scratch1] "m" (rseq_scratch[1]), 1336 [rseq_scratch2] "m" (rseq_scratch[2]) 1337 : "memory", "cc", "eax" 1338 RSEQ_INJECT_CLOBBER 1339 : abort, cmpfail 1340#ifdef RSEQ_COMPARE_TWICE 1341 , error1, error2 1342#endif 1343 ); 1344 rseq_after_asm_goto(); 1345 return 0; 1346abort: 1347 rseq_after_asm_goto(); 1348 RSEQ_INJECT_FAILED 1349 return -1; 1350cmpfail: 1351 rseq_after_asm_goto(); 1352 return 1; 1353#ifdef RSEQ_COMPARE_TWICE 1354error1: 1355 rseq_after_asm_goto(); 1356 rseq_bug("cpu_id comparison failed"); 1357error2: 1358 rseq_after_asm_goto(); 1359 rseq_bug("expected value comparison failed"); 1360#endif 1361} 1362 1363#endif /* !RSEQ_SKIP_FASTPATH */ 1364 1365#endif 1366