1/* SPDX-License-Identifier: LGPL-2.1 OR MIT */ 2/* 3 * rseq-arm.h 4 * 5 * (C) Copyright 2016-2018 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com> 6 */ 7 8/* 9 * - ARM little endian 10 * 11 * RSEQ_SIG uses the udf A32 instruction with an uncommon immediate operand 12 * value 0x5de3. This traps if user-space reaches this instruction by mistake, 13 * and the uncommon operand ensures the kernel does not move the instruction 14 * pointer to attacker-controlled code on rseq abort. 15 * 16 * The instruction pattern in the A32 instruction set is: 17 * 18 * e7f5def3 udf #24035 ; 0x5de3 19 * 20 * This translates to the following instruction pattern in the T16 instruction 21 * set: 22 * 23 * little endian: 24 * def3 udf #243 ; 0xf3 25 * e7f5 b.n <7f5> 26 * 27 * - ARMv6+ big endian (BE8): 28 * 29 * ARMv6+ -mbig-endian generates mixed endianness code vs data: little-endian 30 * code and big-endian data. The data value of the signature needs to have its 31 * byte order reversed to generate the trap instruction: 32 * 33 * Data: 0xf3def5e7 34 * 35 * Translates to this A32 instruction pattern: 36 * 37 * e7f5def3 udf #24035 ; 0x5de3 38 * 39 * Translates to this T16 instruction pattern: 40 * 41 * def3 udf #243 ; 0xf3 42 * e7f5 b.n <7f5> 43 * 44 * - Prior to ARMv6 big endian (BE32): 45 * 46 * Prior to ARMv6, -mbig-endian generates big-endian code and data 47 * (which match), so the endianness of the data representation of the 48 * signature should not be reversed. However, the choice between BE32 49 * and BE8 is done by the linker, so we cannot know whether code and 50 * data endianness will be mixed before the linker is invoked. So rather 51 * than try to play tricks with the linker, the rseq signature is simply 52 * data (not a trap instruction) prior to ARMv6 on big endian. This is 53 * why the signature is expressed as data (.word) rather than as 54 * instruction (.inst) in assembler. 55 */ 56 57#ifdef __ARMEB__ 58#define RSEQ_SIG 0xf3def5e7 /* udf #24035 ; 0x5de3 (ARMv6+) */ 59#else 60#define RSEQ_SIG 0xe7f5def3 /* udf #24035 ; 0x5de3 */ 61#endif 62 63#define rseq_smp_mb() __asm__ __volatile__ ("dmb" ::: "memory", "cc") 64#define rseq_smp_rmb() __asm__ __volatile__ ("dmb" ::: "memory", "cc") 65#define rseq_smp_wmb() __asm__ __volatile__ ("dmb" ::: "memory", "cc") 66 67#define rseq_smp_load_acquire(p) \ 68__extension__ ({ \ 69 __typeof(*p) ____p1 = RSEQ_READ_ONCE(*p); \ 70 rseq_smp_mb(); \ 71 ____p1; \ 72}) 73 74#define rseq_smp_acquire__after_ctrl_dep() rseq_smp_rmb() 75 76#define rseq_smp_store_release(p, v) \ 77do { \ 78 rseq_smp_mb(); \ 79 RSEQ_WRITE_ONCE(*p, v); \ 80} while (0) 81 82#ifdef RSEQ_SKIP_FASTPATH 83#include "rseq-skip.h" 84#else /* !RSEQ_SKIP_FASTPATH */ 85 86#define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, start_ip, \ 87 post_commit_offset, abort_ip) \ 88 ".pushsection __rseq_cs, \"aw\"\n\t" \ 89 ".balign 32\n\t" \ 90 __rseq_str(label) ":\n\t" \ 91 ".word " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \ 92 ".word " __rseq_str(start_ip) ", 0x0, " __rseq_str(post_commit_offset) ", 0x0, " __rseq_str(abort_ip) ", 0x0\n\t" \ 93 ".popsection\n\t" \ 94 ".pushsection __rseq_cs_ptr_array, \"aw\"\n\t" \ 95 ".word " __rseq_str(label) "b, 0x0\n\t" \ 96 ".popsection\n\t" 97 98#define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \ 99 __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip, \ 100 (post_commit_ip - start_ip), abort_ip) 101 102/* 103 * Exit points of a rseq critical section consist of all instructions outside 104 * of the critical section where a critical section can either branch to or 105 * reach through the normal course of its execution. The abort IP and the 106 * post-commit IP are already part of the __rseq_cs section and should not be 107 * explicitly defined as additional exit points. Knowing all exit points is 108 * useful to assist debuggers stepping over the critical section. 109 */ 110#define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \ 111 ".pushsection __rseq_exit_point_array, \"aw\"\n\t" \ 112 ".word " __rseq_str(start_ip) ", 0x0, " __rseq_str(exit_ip) ", 0x0\n\t" \ 113 ".popsection\n\t" 114 115#define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \ 116 RSEQ_INJECT_ASM(1) \ 117 "adr r0, " __rseq_str(cs_label) "\n\t" \ 118 "str r0, %[" __rseq_str(rseq_cs) "]\n\t" \ 119 __rseq_str(label) ":\n\t" 120 121#define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label) \ 122 RSEQ_INJECT_ASM(2) \ 123 "ldr r0, %[" __rseq_str(current_cpu_id) "]\n\t" \ 124 "cmp %[" __rseq_str(cpu_id) "], r0\n\t" \ 125 "bne " __rseq_str(label) "\n\t" 126 127#define __RSEQ_ASM_DEFINE_ABORT(table_label, label, teardown, \ 128 abort_label, version, flags, \ 129 start_ip, post_commit_offset, abort_ip) \ 130 ".balign 32\n\t" \ 131 __rseq_str(table_label) ":\n\t" \ 132 ".word " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \ 133 ".word " __rseq_str(start_ip) ", 0x0, " __rseq_str(post_commit_offset) ", 0x0, " __rseq_str(abort_ip) ", 0x0\n\t" \ 134 ".word " __rseq_str(RSEQ_SIG) "\n\t" \ 135 __rseq_str(label) ":\n\t" \ 136 teardown \ 137 "b %l[" __rseq_str(abort_label) "]\n\t" 138 139#define RSEQ_ASM_DEFINE_ABORT(table_label, label, teardown, abort_label, \ 140 start_ip, post_commit_ip, abort_ip) \ 141 __RSEQ_ASM_DEFINE_ABORT(table_label, label, teardown, \ 142 abort_label, 0x0, 0x0, start_ip, \ 143 (post_commit_ip - start_ip), abort_ip) 144 145#define RSEQ_ASM_DEFINE_CMPFAIL(label, teardown, cmpfail_label) \ 146 __rseq_str(label) ":\n\t" \ 147 teardown \ 148 "b %l[" __rseq_str(cmpfail_label) "]\n\t" 149 150static inline __attribute__((always_inline)) 151int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu) 152{ 153 RSEQ_INJECT_C(9) 154 155 __asm__ __volatile__ goto ( 156 RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ 157 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 158#ifdef RSEQ_COMPARE_TWICE 159 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 160 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 161#endif 162 /* Start rseq by storing table entry pointer into rseq_cs. */ 163 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) 164 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 165 RSEQ_INJECT_ASM(3) 166 "ldr r0, %[v]\n\t" 167 "cmp %[expect], r0\n\t" 168 "bne %l[cmpfail]\n\t" 169 RSEQ_INJECT_ASM(4) 170#ifdef RSEQ_COMPARE_TWICE 171 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 172 "ldr r0, %[v]\n\t" 173 "cmp %[expect], r0\n\t" 174 "bne %l[error2]\n\t" 175#endif 176 /* final store */ 177 "str %[newv], %[v]\n\t" 178 "2:\n\t" 179 RSEQ_INJECT_ASM(5) 180 "b 5f\n\t" 181 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f) 182 "5:\n\t" 183 : /* gcc asm goto does not allow outputs */ 184 : [cpu_id] "r" (cpu), 185 [current_cpu_id] "m" (rseq_get_abi()->cpu_id), 186 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), 187 [v] "m" (*v), 188 [expect] "r" (expect), 189 [newv] "r" (newv) 190 RSEQ_INJECT_INPUT 191 : "r0", "memory", "cc" 192 RSEQ_INJECT_CLOBBER 193 : abort, cmpfail 194#ifdef RSEQ_COMPARE_TWICE 195 , error1, error2 196#endif 197 ); 198 rseq_after_asm_goto(); 199 return 0; 200abort: 201 rseq_after_asm_goto(); 202 RSEQ_INJECT_FAILED 203 return -1; 204cmpfail: 205 rseq_after_asm_goto(); 206 return 1; 207#ifdef RSEQ_COMPARE_TWICE 208error1: 209 rseq_after_asm_goto(); 210 rseq_bug("cpu_id comparison failed"); 211error2: 212 rseq_after_asm_goto(); 213 rseq_bug("expected value comparison failed"); 214#endif 215} 216 217static inline __attribute__((always_inline)) 218int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot, 219 long voffp, intptr_t *load, int cpu) 220{ 221 RSEQ_INJECT_C(9) 222 223 __asm__ __volatile__ goto ( 224 RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ 225 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 226#ifdef RSEQ_COMPARE_TWICE 227 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 228 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 229#endif 230 /* Start rseq by storing table entry pointer into rseq_cs. */ 231 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) 232 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 233 RSEQ_INJECT_ASM(3) 234 "ldr r0, %[v]\n\t" 235 "cmp %[expectnot], r0\n\t" 236 "beq %l[cmpfail]\n\t" 237 RSEQ_INJECT_ASM(4) 238#ifdef RSEQ_COMPARE_TWICE 239 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 240 "ldr r0, %[v]\n\t" 241 "cmp %[expectnot], r0\n\t" 242 "beq %l[error2]\n\t" 243#endif 244 "str r0, %[load]\n\t" 245 "add r0, %[voffp]\n\t" 246 "ldr r0, [r0]\n\t" 247 /* final store */ 248 "str r0, %[v]\n\t" 249 "2:\n\t" 250 RSEQ_INJECT_ASM(5) 251 "b 5f\n\t" 252 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f) 253 "5:\n\t" 254 : /* gcc asm goto does not allow outputs */ 255 : [cpu_id] "r" (cpu), 256 [current_cpu_id] "m" (rseq_get_abi()->cpu_id), 257 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), 258 /* final store input */ 259 [v] "m" (*v), 260 [expectnot] "r" (expectnot), 261 [voffp] "Ir" (voffp), 262 [load] "m" (*load) 263 RSEQ_INJECT_INPUT 264 : "r0", "memory", "cc" 265 RSEQ_INJECT_CLOBBER 266 : abort, cmpfail 267#ifdef RSEQ_COMPARE_TWICE 268 , error1, error2 269#endif 270 ); 271 rseq_after_asm_goto(); 272 return 0; 273abort: 274 rseq_after_asm_goto(); 275 RSEQ_INJECT_FAILED 276 return -1; 277cmpfail: 278 rseq_after_asm_goto(); 279 return 1; 280#ifdef RSEQ_COMPARE_TWICE 281error1: 282 rseq_after_asm_goto(); 283 rseq_bug("cpu_id comparison failed"); 284error2: 285 rseq_after_asm_goto(); 286 rseq_bug("expected value comparison failed"); 287#endif 288} 289 290static inline __attribute__((always_inline)) 291int rseq_addv(intptr_t *v, intptr_t count, int cpu) 292{ 293 RSEQ_INJECT_C(9) 294 295 __asm__ __volatile__ goto ( 296 RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ 297#ifdef RSEQ_COMPARE_TWICE 298 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 299#endif 300 /* Start rseq by storing table entry pointer into rseq_cs. */ 301 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) 302 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 303 RSEQ_INJECT_ASM(3) 304#ifdef RSEQ_COMPARE_TWICE 305 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 306#endif 307 "ldr r0, %[v]\n\t" 308 "add r0, %[count]\n\t" 309 /* final store */ 310 "str r0, %[v]\n\t" 311 "2:\n\t" 312 RSEQ_INJECT_ASM(4) 313 "b 5f\n\t" 314 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f) 315 "5:\n\t" 316 : /* gcc asm goto does not allow outputs */ 317 : [cpu_id] "r" (cpu), 318 [current_cpu_id] "m" (rseq_get_abi()->cpu_id), 319 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), 320 [v] "m" (*v), 321 [count] "Ir" (count) 322 RSEQ_INJECT_INPUT 323 : "r0", "memory", "cc" 324 RSEQ_INJECT_CLOBBER 325 : abort 326#ifdef RSEQ_COMPARE_TWICE 327 , error1 328#endif 329 ); 330 rseq_after_asm_goto(); 331 return 0; 332abort: 333 rseq_after_asm_goto(); 334 RSEQ_INJECT_FAILED 335 return -1; 336#ifdef RSEQ_COMPARE_TWICE 337error1: 338 rseq_after_asm_goto(); 339 rseq_bug("cpu_id comparison failed"); 340#endif 341} 342 343static inline __attribute__((always_inline)) 344int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect, 345 intptr_t *v2, intptr_t newv2, 346 intptr_t newv, int cpu) 347{ 348 RSEQ_INJECT_C(9) 349 350 __asm__ __volatile__ goto ( 351 RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ 352 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 353#ifdef RSEQ_COMPARE_TWICE 354 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 355 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 356#endif 357 /* Start rseq by storing table entry pointer into rseq_cs. */ 358 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) 359 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 360 RSEQ_INJECT_ASM(3) 361 "ldr r0, %[v]\n\t" 362 "cmp %[expect], r0\n\t" 363 "bne %l[cmpfail]\n\t" 364 RSEQ_INJECT_ASM(4) 365#ifdef RSEQ_COMPARE_TWICE 366 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 367 "ldr r0, %[v]\n\t" 368 "cmp %[expect], r0\n\t" 369 "bne %l[error2]\n\t" 370#endif 371 /* try store */ 372 "str %[newv2], %[v2]\n\t" 373 RSEQ_INJECT_ASM(5) 374 /* final store */ 375 "str %[newv], %[v]\n\t" 376 "2:\n\t" 377 RSEQ_INJECT_ASM(6) 378 "b 5f\n\t" 379 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f) 380 "5:\n\t" 381 : /* gcc asm goto does not allow outputs */ 382 : [cpu_id] "r" (cpu), 383 [current_cpu_id] "m" (rseq_get_abi()->cpu_id), 384 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), 385 /* try store input */ 386 [v2] "m" (*v2), 387 [newv2] "r" (newv2), 388 /* final store input */ 389 [v] "m" (*v), 390 [expect] "r" (expect), 391 [newv] "r" (newv) 392 RSEQ_INJECT_INPUT 393 : "r0", "memory", "cc" 394 RSEQ_INJECT_CLOBBER 395 : abort, cmpfail 396#ifdef RSEQ_COMPARE_TWICE 397 , error1, error2 398#endif 399 ); 400 rseq_after_asm_goto(); 401 return 0; 402abort: 403 rseq_after_asm_goto(); 404 RSEQ_INJECT_FAILED 405 return -1; 406cmpfail: 407 rseq_after_asm_goto(); 408 return 1; 409#ifdef RSEQ_COMPARE_TWICE 410error1: 411 rseq_after_asm_goto(); 412 rseq_bug("cpu_id comparison failed"); 413error2: 414 rseq_after_asm_goto(); 415 rseq_bug("expected value comparison failed"); 416#endif 417} 418 419static inline __attribute__((always_inline)) 420int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect, 421 intptr_t *v2, intptr_t newv2, 422 intptr_t newv, int cpu) 423{ 424 RSEQ_INJECT_C(9) 425 426 __asm__ __volatile__ goto ( 427 RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ 428 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 429#ifdef RSEQ_COMPARE_TWICE 430 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 431 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 432#endif 433 /* Start rseq by storing table entry pointer into rseq_cs. */ 434 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) 435 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 436 RSEQ_INJECT_ASM(3) 437 "ldr r0, %[v]\n\t" 438 "cmp %[expect], r0\n\t" 439 "bne %l[cmpfail]\n\t" 440 RSEQ_INJECT_ASM(4) 441#ifdef RSEQ_COMPARE_TWICE 442 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 443 "ldr r0, %[v]\n\t" 444 "cmp %[expect], r0\n\t" 445 "bne %l[error2]\n\t" 446#endif 447 /* try store */ 448 "str %[newv2], %[v2]\n\t" 449 RSEQ_INJECT_ASM(5) 450 "dmb\n\t" /* full mb provides store-release */ 451 /* final store */ 452 "str %[newv], %[v]\n\t" 453 "2:\n\t" 454 RSEQ_INJECT_ASM(6) 455 "b 5f\n\t" 456 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f) 457 "5:\n\t" 458 : /* gcc asm goto does not allow outputs */ 459 : [cpu_id] "r" (cpu), 460 [current_cpu_id] "m" (rseq_get_abi()->cpu_id), 461 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), 462 /* try store input */ 463 [v2] "m" (*v2), 464 [newv2] "r" (newv2), 465 /* final store input */ 466 [v] "m" (*v), 467 [expect] "r" (expect), 468 [newv] "r" (newv) 469 RSEQ_INJECT_INPUT 470 : "r0", "memory", "cc" 471 RSEQ_INJECT_CLOBBER 472 : abort, cmpfail 473#ifdef RSEQ_COMPARE_TWICE 474 , error1, error2 475#endif 476 ); 477 rseq_after_asm_goto(); 478 return 0; 479abort: 480 rseq_after_asm_goto(); 481 RSEQ_INJECT_FAILED 482 return -1; 483cmpfail: 484 rseq_after_asm_goto(); 485 return 1; 486#ifdef RSEQ_COMPARE_TWICE 487error1: 488 rseq_after_asm_goto(); 489 rseq_bug("cpu_id comparison failed"); 490error2: 491 rseq_after_asm_goto(); 492 rseq_bug("expected value comparison failed"); 493#endif 494} 495 496static inline __attribute__((always_inline)) 497int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect, 498 intptr_t *v2, intptr_t expect2, 499 intptr_t newv, int cpu) 500{ 501 RSEQ_INJECT_C(9) 502 503 __asm__ __volatile__ goto ( 504 RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ 505 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 506#ifdef RSEQ_COMPARE_TWICE 507 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 508 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 509 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3]) 510#endif 511 /* Start rseq by storing table entry pointer into rseq_cs. */ 512 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) 513 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 514 RSEQ_INJECT_ASM(3) 515 "ldr r0, %[v]\n\t" 516 "cmp %[expect], r0\n\t" 517 "bne %l[cmpfail]\n\t" 518 RSEQ_INJECT_ASM(4) 519 "ldr r0, %[v2]\n\t" 520 "cmp %[expect2], r0\n\t" 521 "bne %l[cmpfail]\n\t" 522 RSEQ_INJECT_ASM(5) 523#ifdef RSEQ_COMPARE_TWICE 524 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 525 "ldr r0, %[v]\n\t" 526 "cmp %[expect], r0\n\t" 527 "bne %l[error2]\n\t" 528 "ldr r0, %[v2]\n\t" 529 "cmp %[expect2], r0\n\t" 530 "bne %l[error3]\n\t" 531#endif 532 /* final store */ 533 "str %[newv], %[v]\n\t" 534 "2:\n\t" 535 RSEQ_INJECT_ASM(6) 536 "b 5f\n\t" 537 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f) 538 "5:\n\t" 539 : /* gcc asm goto does not allow outputs */ 540 : [cpu_id] "r" (cpu), 541 [current_cpu_id] "m" (rseq_get_abi()->cpu_id), 542 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), 543 /* cmp2 input */ 544 [v2] "m" (*v2), 545 [expect2] "r" (expect2), 546 /* final store input */ 547 [v] "m" (*v), 548 [expect] "r" (expect), 549 [newv] "r" (newv) 550 RSEQ_INJECT_INPUT 551 : "r0", "memory", "cc" 552 RSEQ_INJECT_CLOBBER 553 : abort, cmpfail 554#ifdef RSEQ_COMPARE_TWICE 555 , error1, error2, error3 556#endif 557 ); 558 rseq_after_asm_goto(); 559 return 0; 560abort: 561 rseq_after_asm_goto(); 562 RSEQ_INJECT_FAILED 563 return -1; 564cmpfail: 565 rseq_after_asm_goto(); 566 return 1; 567#ifdef RSEQ_COMPARE_TWICE 568error1: 569 rseq_after_asm_goto(); 570 rseq_bug("cpu_id comparison failed"); 571error2: 572 rseq_after_asm_goto(); 573 rseq_bug("1st expected value comparison failed"); 574error3: 575 rseq_after_asm_goto(); 576 rseq_bug("2nd expected value comparison failed"); 577#endif 578} 579 580static inline __attribute__((always_inline)) 581int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect, 582 void *dst, void *src, size_t len, 583 intptr_t newv, int cpu) 584{ 585 uint32_t rseq_scratch[3]; 586 587 RSEQ_INJECT_C(9) 588 589 __asm__ __volatile__ goto ( 590 RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ 591 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 592#ifdef RSEQ_COMPARE_TWICE 593 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 594 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 595#endif 596 "str %[src], %[rseq_scratch0]\n\t" 597 "str %[dst], %[rseq_scratch1]\n\t" 598 "str %[len], %[rseq_scratch2]\n\t" 599 /* Start rseq by storing table entry pointer into rseq_cs. */ 600 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) 601 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 602 RSEQ_INJECT_ASM(3) 603 "ldr r0, %[v]\n\t" 604 "cmp %[expect], r0\n\t" 605 "bne 5f\n\t" 606 RSEQ_INJECT_ASM(4) 607#ifdef RSEQ_COMPARE_TWICE 608 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 6f) 609 "ldr r0, %[v]\n\t" 610 "cmp %[expect], r0\n\t" 611 "bne 7f\n\t" 612#endif 613 /* try memcpy */ 614 "cmp %[len], #0\n\t" \ 615 "beq 333f\n\t" \ 616 "222:\n\t" \ 617 "ldrb %%r0, [%[src]]\n\t" \ 618 "strb %%r0, [%[dst]]\n\t" \ 619 "adds %[src], #1\n\t" \ 620 "adds %[dst], #1\n\t" \ 621 "subs %[len], #1\n\t" \ 622 "bne 222b\n\t" \ 623 "333:\n\t" \ 624 RSEQ_INJECT_ASM(5) 625 /* final store */ 626 "str %[newv], %[v]\n\t" 627 "2:\n\t" 628 RSEQ_INJECT_ASM(6) 629 /* teardown */ 630 "ldr %[len], %[rseq_scratch2]\n\t" 631 "ldr %[dst], %[rseq_scratch1]\n\t" 632 "ldr %[src], %[rseq_scratch0]\n\t" 633 "b 8f\n\t" 634 RSEQ_ASM_DEFINE_ABORT(3, 4, 635 /* teardown */ 636 "ldr %[len], %[rseq_scratch2]\n\t" 637 "ldr %[dst], %[rseq_scratch1]\n\t" 638 "ldr %[src], %[rseq_scratch0]\n\t", 639 abort, 1b, 2b, 4f) 640 RSEQ_ASM_DEFINE_CMPFAIL(5, 641 /* teardown */ 642 "ldr %[len], %[rseq_scratch2]\n\t" 643 "ldr %[dst], %[rseq_scratch1]\n\t" 644 "ldr %[src], %[rseq_scratch0]\n\t", 645 cmpfail) 646#ifdef RSEQ_COMPARE_TWICE 647 RSEQ_ASM_DEFINE_CMPFAIL(6, 648 /* teardown */ 649 "ldr %[len], %[rseq_scratch2]\n\t" 650 "ldr %[dst], %[rseq_scratch1]\n\t" 651 "ldr %[src], %[rseq_scratch0]\n\t", 652 error1) 653 RSEQ_ASM_DEFINE_CMPFAIL(7, 654 /* teardown */ 655 "ldr %[len], %[rseq_scratch2]\n\t" 656 "ldr %[dst], %[rseq_scratch1]\n\t" 657 "ldr %[src], %[rseq_scratch0]\n\t", 658 error2) 659#endif 660 "8:\n\t" 661 : /* gcc asm goto does not allow outputs */ 662 : [cpu_id] "r" (cpu), 663 [current_cpu_id] "m" (rseq_get_abi()->cpu_id), 664 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), 665 /* final store input */ 666 [v] "m" (*v), 667 [expect] "r" (expect), 668 [newv] "r" (newv), 669 /* try memcpy input */ 670 [dst] "r" (dst), 671 [src] "r" (src), 672 [len] "r" (len), 673 [rseq_scratch0] "m" (rseq_scratch[0]), 674 [rseq_scratch1] "m" (rseq_scratch[1]), 675 [rseq_scratch2] "m" (rseq_scratch[2]) 676 RSEQ_INJECT_INPUT 677 : "r0", "memory", "cc" 678 RSEQ_INJECT_CLOBBER 679 : abort, cmpfail 680#ifdef RSEQ_COMPARE_TWICE 681 , error1, error2 682#endif 683 ); 684 rseq_after_asm_goto(); 685 return 0; 686abort: 687 rseq_after_asm_goto(); 688 RSEQ_INJECT_FAILED 689 return -1; 690cmpfail: 691 rseq_after_asm_goto(); 692 return 1; 693#ifdef RSEQ_COMPARE_TWICE 694error1: 695 rseq_after_asm_goto(); 696 rseq_bug("cpu_id comparison failed"); 697error2: 698 rseq_after_asm_goto(); 699 rseq_bug("expected value comparison failed"); 700#endif 701} 702 703static inline __attribute__((always_inline)) 704int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect, 705 void *dst, void *src, size_t len, 706 intptr_t newv, int cpu) 707{ 708 uint32_t rseq_scratch[3]; 709 710 RSEQ_INJECT_C(9) 711 712 __asm__ __volatile__ goto ( 713 RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ 714 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 715#ifdef RSEQ_COMPARE_TWICE 716 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 717 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 718#endif 719 "str %[src], %[rseq_scratch0]\n\t" 720 "str %[dst], %[rseq_scratch1]\n\t" 721 "str %[len], %[rseq_scratch2]\n\t" 722 /* Start rseq by storing table entry pointer into rseq_cs. */ 723 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) 724 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 725 RSEQ_INJECT_ASM(3) 726 "ldr r0, %[v]\n\t" 727 "cmp %[expect], r0\n\t" 728 "bne 5f\n\t" 729 RSEQ_INJECT_ASM(4) 730#ifdef RSEQ_COMPARE_TWICE 731 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 6f) 732 "ldr r0, %[v]\n\t" 733 "cmp %[expect], r0\n\t" 734 "bne 7f\n\t" 735#endif 736 /* try memcpy */ 737 "cmp %[len], #0\n\t" \ 738 "beq 333f\n\t" \ 739 "222:\n\t" \ 740 "ldrb %%r0, [%[src]]\n\t" \ 741 "strb %%r0, [%[dst]]\n\t" \ 742 "adds %[src], #1\n\t" \ 743 "adds %[dst], #1\n\t" \ 744 "subs %[len], #1\n\t" \ 745 "bne 222b\n\t" \ 746 "333:\n\t" \ 747 RSEQ_INJECT_ASM(5) 748 "dmb\n\t" /* full mb provides store-release */ 749 /* final store */ 750 "str %[newv], %[v]\n\t" 751 "2:\n\t" 752 RSEQ_INJECT_ASM(6) 753 /* teardown */ 754 "ldr %[len], %[rseq_scratch2]\n\t" 755 "ldr %[dst], %[rseq_scratch1]\n\t" 756 "ldr %[src], %[rseq_scratch0]\n\t" 757 "b 8f\n\t" 758 RSEQ_ASM_DEFINE_ABORT(3, 4, 759 /* teardown */ 760 "ldr %[len], %[rseq_scratch2]\n\t" 761 "ldr %[dst], %[rseq_scratch1]\n\t" 762 "ldr %[src], %[rseq_scratch0]\n\t", 763 abort, 1b, 2b, 4f) 764 RSEQ_ASM_DEFINE_CMPFAIL(5, 765 /* teardown */ 766 "ldr %[len], %[rseq_scratch2]\n\t" 767 "ldr %[dst], %[rseq_scratch1]\n\t" 768 "ldr %[src], %[rseq_scratch0]\n\t", 769 cmpfail) 770#ifdef RSEQ_COMPARE_TWICE 771 RSEQ_ASM_DEFINE_CMPFAIL(6, 772 /* teardown */ 773 "ldr %[len], %[rseq_scratch2]\n\t" 774 "ldr %[dst], %[rseq_scratch1]\n\t" 775 "ldr %[src], %[rseq_scratch0]\n\t", 776 error1) 777 RSEQ_ASM_DEFINE_CMPFAIL(7, 778 /* teardown */ 779 "ldr %[len], %[rseq_scratch2]\n\t" 780 "ldr %[dst], %[rseq_scratch1]\n\t" 781 "ldr %[src], %[rseq_scratch0]\n\t", 782 error2) 783#endif 784 "8:\n\t" 785 : /* gcc asm goto does not allow outputs */ 786 : [cpu_id] "r" (cpu), 787 [current_cpu_id] "m" (rseq_get_abi()->cpu_id), 788 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), 789 /* final store input */ 790 [v] "m" (*v), 791 [expect] "r" (expect), 792 [newv] "r" (newv), 793 /* try memcpy input */ 794 [dst] "r" (dst), 795 [src] "r" (src), 796 [len] "r" (len), 797 [rseq_scratch0] "m" (rseq_scratch[0]), 798 [rseq_scratch1] "m" (rseq_scratch[1]), 799 [rseq_scratch2] "m" (rseq_scratch[2]) 800 RSEQ_INJECT_INPUT 801 : "r0", "memory", "cc" 802 RSEQ_INJECT_CLOBBER 803 : abort, cmpfail 804#ifdef RSEQ_COMPARE_TWICE 805 , error1, error2 806#endif 807 ); 808 rseq_after_asm_goto(); 809 return 0; 810abort: 811 rseq_after_asm_goto(); 812 RSEQ_INJECT_FAILED 813 return -1; 814cmpfail: 815 rseq_after_asm_goto(); 816 return 1; 817#ifdef RSEQ_COMPARE_TWICE 818error1: 819 rseq_after_asm_goto(); 820 rseq_bug("cpu_id comparison failed"); 821error2: 822 rseq_after_asm_goto(); 823 rseq_bug("expected value comparison failed"); 824#endif 825} 826 827#endif /* !RSEQ_SKIP_FASTPATH */ 828