1/* SPDX-License-Identifier: GPL-2.0 */ 2#ifndef _ASM_POWERPC_ATOMIC_H_ 3#define _ASM_POWERPC_ATOMIC_H_ 4 5/* 6 * PowerPC atomic operations 7 */ 8 9#ifdef __KERNEL__ 10#include <linux/types.h> 11#include <asm/cmpxchg.h> 12#include <asm/barrier.h> 13 14/* 15 * Since *_return_relaxed and {cmp}xchg_relaxed are implemented with 16 * a "bne-" instruction at the end, so an isync is enough as a acquire barrier 17 * on the platform without lwsync. 18 */ 19#define __atomic_acquire_fence() \ 20 __asm__ __volatile__(PPC_ACQUIRE_BARRIER "" : : : "memory") 21 22#define __atomic_release_fence() \ 23 __asm__ __volatile__(PPC_RELEASE_BARRIER "" : : : "memory") 24 25static __inline__ int atomic_read(const atomic_t *v) 26{ 27 int t; 28 29 __asm__ __volatile__("lwz%U1%X1 %0,%1" : "=r"(t) : "m"(v->counter)); 30 31 return t; 32} 33 34static __inline__ void atomic_set(atomic_t *v, int i) 35{ 36 __asm__ __volatile__("stw%U0%X0 %1,%0" : "=m"(v->counter) : "r"(i)); 37} 38 39#define ATOMIC_OP(op, asm_op) \ 40static __inline__ void atomic_##op(int a, atomic_t *v) \ 41{ \ 42 int t; \ 43 \ 44 __asm__ __volatile__( \ 45"1: lwarx %0,0,%3 # atomic_" #op "\n" \ 46 #asm_op " %0,%2,%0\n" \ 47" stwcx. %0,0,%3 \n" \ 48" bne- 1b\n" \ 49 : "=&r" (t), "+m" (v->counter) \ 50 : "r" (a), "r" (&v->counter) \ 51 : "cc"); \ 52} \ 53 54#define ATOMIC_OP_RETURN_RELAXED(op, asm_op) \ 55static inline int atomic_##op##_return_relaxed(int a, atomic_t *v) \ 56{ \ 57 int t; \ 58 \ 59 __asm__ __volatile__( \ 60"1: lwarx %0,0,%3 # atomic_" #op "_return_relaxed\n" \ 61 #asm_op " %0,%2,%0\n" \ 62" stwcx. %0,0,%3\n" \ 63" bne- 1b\n" \ 64 : "=&r" (t), "+m" (v->counter) \ 65 : "r" (a), "r" (&v->counter) \ 66 : "cc"); \ 67 \ 68 return t; \ 69} 70 71#define ATOMIC_FETCH_OP_RELAXED(op, asm_op) \ 72static inline int atomic_fetch_##op##_relaxed(int a, atomic_t *v) \ 73{ \ 74 int res, t; \ 75 \ 76 __asm__ __volatile__( \ 77"1: lwarx %0,0,%4 # atomic_fetch_" #op "_relaxed\n" \ 78 #asm_op " %1,%3,%0\n" \ 79" stwcx. %1,0,%4\n" \ 80" bne- 1b\n" \ 81 : "=&r" (res), "=&r" (t), "+m" (v->counter) \ 82 : "r" (a), "r" (&v->counter) \ 83 : "cc"); \ 84 \ 85 return res; \ 86} 87 88#define ATOMIC_OPS(op, asm_op) \ 89 ATOMIC_OP(op, asm_op) \ 90 ATOMIC_OP_RETURN_RELAXED(op, asm_op) \ 91 ATOMIC_FETCH_OP_RELAXED(op, asm_op) 92 93ATOMIC_OPS(add, add) 94ATOMIC_OPS(sub, subf) 95 96#define atomic_add_return_relaxed atomic_add_return_relaxed 97#define atomic_sub_return_relaxed atomic_sub_return_relaxed 98 99#define atomic_fetch_add_relaxed atomic_fetch_add_relaxed 100#define atomic_fetch_sub_relaxed atomic_fetch_sub_relaxed 101 102#undef ATOMIC_OPS 103#define ATOMIC_OPS(op, asm_op) \ 104 ATOMIC_OP(op, asm_op) \ 105 ATOMIC_FETCH_OP_RELAXED(op, asm_op) 106 107ATOMIC_OPS(and, and) 108ATOMIC_OPS(or, or) 109ATOMIC_OPS(xor, xor) 110 111#define atomic_fetch_and_relaxed atomic_fetch_and_relaxed 112#define atomic_fetch_or_relaxed atomic_fetch_or_relaxed 113#define atomic_fetch_xor_relaxed atomic_fetch_xor_relaxed 114 115#undef ATOMIC_OPS 116#undef ATOMIC_FETCH_OP_RELAXED 117#undef ATOMIC_OP_RETURN_RELAXED 118#undef ATOMIC_OP 119 120static __inline__ void atomic_inc(atomic_t *v) 121{ 122 int t; 123 124 __asm__ __volatile__( 125"1: lwarx %0,0,%2 # atomic_inc\n\ 126 addic %0,%0,1\n" 127" stwcx. %0,0,%2 \n\ 128 bne- 1b" 129 : "=&r" (t), "+m" (v->counter) 130 : "r" (&v->counter) 131 : "cc", "xer"); 132} 133#define atomic_inc atomic_inc 134 135static __inline__ int atomic_inc_return_relaxed(atomic_t *v) 136{ 137 int t; 138 139 __asm__ __volatile__( 140"1: lwarx %0,0,%2 # atomic_inc_return_relaxed\n" 141" addic %0,%0,1\n" 142" stwcx. %0,0,%2\n" 143" bne- 1b" 144 : "=&r" (t), "+m" (v->counter) 145 : "r" (&v->counter) 146 : "cc", "xer"); 147 148 return t; 149} 150 151static __inline__ void atomic_dec(atomic_t *v) 152{ 153 int t; 154 155 __asm__ __volatile__( 156"1: lwarx %0,0,%2 # atomic_dec\n\ 157 addic %0,%0,-1\n" 158" stwcx. %0,0,%2\n\ 159 bne- 1b" 160 : "=&r" (t), "+m" (v->counter) 161 : "r" (&v->counter) 162 : "cc", "xer"); 163} 164#define atomic_dec atomic_dec 165 166static __inline__ int atomic_dec_return_relaxed(atomic_t *v) 167{ 168 int t; 169 170 __asm__ __volatile__( 171"1: lwarx %0,0,%2 # atomic_dec_return_relaxed\n" 172" addic %0,%0,-1\n" 173" stwcx. %0,0,%2\n" 174" bne- 1b" 175 : "=&r" (t), "+m" (v->counter) 176 : "r" (&v->counter) 177 : "cc", "xer"); 178 179 return t; 180} 181 182#define atomic_inc_return_relaxed atomic_inc_return_relaxed 183#define atomic_dec_return_relaxed atomic_dec_return_relaxed 184 185#define atomic_cmpxchg(v, o, n) (cmpxchg(&((v)->counter), (o), (n))) 186#define atomic_cmpxchg_relaxed(v, o, n) \ 187 cmpxchg_relaxed(&((v)->counter), (o), (n)) 188#define atomic_cmpxchg_acquire(v, o, n) \ 189 cmpxchg_acquire(&((v)->counter), (o), (n)) 190 191#define atomic_xchg(v, new) (xchg(&((v)->counter), new)) 192#define atomic_xchg_relaxed(v, new) xchg_relaxed(&((v)->counter), (new)) 193 194/* 195 * Don't want to override the generic atomic_try_cmpxchg_acquire, because 196 * we add a lock hint to the lwarx, which may not be wanted for the 197 * _acquire case (and is not used by the other _acquire variants so it 198 * would be a surprise). 199 */ 200static __always_inline bool 201atomic_try_cmpxchg_lock(atomic_t *v, int *old, int new) 202{ 203 int r, o = *old; 204 205 __asm__ __volatile__ ( 206"1:\t" PPC_LWARX(%0,0,%2,1) " # atomic_try_cmpxchg_acquire \n" 207" cmpw 0,%0,%3 \n" 208" bne- 2f \n" 209" stwcx. %4,0,%2 \n" 210" bne- 1b \n" 211"\t" PPC_ACQUIRE_BARRIER " \n" 212"2: \n" 213 : "=&r" (r), "+m" (v->counter) 214 : "r" (&v->counter), "r" (o), "r" (new) 215 : "cr0", "memory"); 216 217 if (unlikely(r != o)) 218 *old = r; 219 return likely(r == o); 220} 221 222/** 223 * atomic_fetch_add_unless - add unless the number is a given value 224 * @v: pointer of type atomic_t 225 * @a: the amount to add to v... 226 * @u: ...unless v is equal to u. 227 * 228 * Atomically adds @a to @v, so long as it was not @u. 229 * Returns the old value of @v. 230 */ 231static __inline__ int atomic_fetch_add_unless(atomic_t *v, int a, int u) 232{ 233 int t; 234 235 __asm__ __volatile__ ( 236 PPC_ATOMIC_ENTRY_BARRIER 237"1: lwarx %0,0,%1 # atomic_fetch_add_unless\n\ 238 cmpw 0,%0,%3 \n\ 239 beq 2f \n\ 240 add %0,%2,%0 \n" 241" stwcx. %0,0,%1 \n\ 242 bne- 1b \n" 243 PPC_ATOMIC_EXIT_BARRIER 244" subf %0,%2,%0 \n\ 2452:" 246 : "=&r" (t) 247 : "r" (&v->counter), "r" (a), "r" (u) 248 : "cc", "memory"); 249 250 return t; 251} 252#define atomic_fetch_add_unless atomic_fetch_add_unless 253 254/** 255 * atomic_inc_not_zero - increment unless the number is zero 256 * @v: pointer of type atomic_t 257 * 258 * Atomically increments @v by 1, so long as @v is non-zero. 259 * Returns non-zero if @v was non-zero, and zero otherwise. 260 */ 261static __inline__ int atomic_inc_not_zero(atomic_t *v) 262{ 263 int t1, t2; 264 265 __asm__ __volatile__ ( 266 PPC_ATOMIC_ENTRY_BARRIER 267"1: lwarx %0,0,%2 # atomic_inc_not_zero\n\ 268 cmpwi 0,%0,0\n\ 269 beq- 2f\n\ 270 addic %1,%0,1\n" 271" stwcx. %1,0,%2\n\ 272 bne- 1b\n" 273 PPC_ATOMIC_EXIT_BARRIER 274 "\n\ 2752:" 276 : "=&r" (t1), "=&r" (t2) 277 : "r" (&v->counter) 278 : "cc", "xer", "memory"); 279 280 return t1; 281} 282#define atomic_inc_not_zero(v) atomic_inc_not_zero((v)) 283 284/* 285 * Atomically test *v and decrement if it is greater than 0. 286 * The function returns the old value of *v minus 1, even if 287 * the atomic variable, v, was not decremented. 288 */ 289static __inline__ int atomic_dec_if_positive(atomic_t *v) 290{ 291 int t; 292 293 __asm__ __volatile__( 294 PPC_ATOMIC_ENTRY_BARRIER 295"1: lwarx %0,0,%1 # atomic_dec_if_positive\n\ 296 cmpwi %0,1\n\ 297 addi %0,%0,-1\n\ 298 blt- 2f\n" 299" stwcx. %0,0,%1\n\ 300 bne- 1b" 301 PPC_ATOMIC_EXIT_BARRIER 302 "\n\ 3032:" : "=&b" (t) 304 : "r" (&v->counter) 305 : "cc", "memory"); 306 307 return t; 308} 309#define atomic_dec_if_positive atomic_dec_if_positive 310 311#ifdef __powerpc64__ 312 313#define ATOMIC64_INIT(i) { (i) } 314 315static __inline__ s64 atomic64_read(const atomic64_t *v) 316{ 317 s64 t; 318 319 __asm__ __volatile__("ld%U1%X1 %0,%1" : "=r"(t) : "m"(v->counter)); 320 321 return t; 322} 323 324static __inline__ void atomic64_set(atomic64_t *v, s64 i) 325{ 326 __asm__ __volatile__("std%U0%X0 %1,%0" : "=m"(v->counter) : "r"(i)); 327} 328 329#define ATOMIC64_OP(op, asm_op) \ 330static __inline__ void atomic64_##op(s64 a, atomic64_t *v) \ 331{ \ 332 s64 t; \ 333 \ 334 __asm__ __volatile__( \ 335"1: ldarx %0,0,%3 # atomic64_" #op "\n" \ 336 #asm_op " %0,%2,%0\n" \ 337" stdcx. %0,0,%3 \n" \ 338" bne- 1b\n" \ 339 : "=&r" (t), "+m" (v->counter) \ 340 : "r" (a), "r" (&v->counter) \ 341 : "cc"); \ 342} 343 344#define ATOMIC64_OP_RETURN_RELAXED(op, asm_op) \ 345static inline s64 \ 346atomic64_##op##_return_relaxed(s64 a, atomic64_t *v) \ 347{ \ 348 s64 t; \ 349 \ 350 __asm__ __volatile__( \ 351"1: ldarx %0,0,%3 # atomic64_" #op "_return_relaxed\n" \ 352 #asm_op " %0,%2,%0\n" \ 353" stdcx. %0,0,%3\n" \ 354" bne- 1b\n" \ 355 : "=&r" (t), "+m" (v->counter) \ 356 : "r" (a), "r" (&v->counter) \ 357 : "cc"); \ 358 \ 359 return t; \ 360} 361 362#define ATOMIC64_FETCH_OP_RELAXED(op, asm_op) \ 363static inline s64 \ 364atomic64_fetch_##op##_relaxed(s64 a, atomic64_t *v) \ 365{ \ 366 s64 res, t; \ 367 \ 368 __asm__ __volatile__( \ 369"1: ldarx %0,0,%4 # atomic64_fetch_" #op "_relaxed\n" \ 370 #asm_op " %1,%3,%0\n" \ 371" stdcx. %1,0,%4\n" \ 372" bne- 1b\n" \ 373 : "=&r" (res), "=&r" (t), "+m" (v->counter) \ 374 : "r" (a), "r" (&v->counter) \ 375 : "cc"); \ 376 \ 377 return res; \ 378} 379 380#define ATOMIC64_OPS(op, asm_op) \ 381 ATOMIC64_OP(op, asm_op) \ 382 ATOMIC64_OP_RETURN_RELAXED(op, asm_op) \ 383 ATOMIC64_FETCH_OP_RELAXED(op, asm_op) 384 385ATOMIC64_OPS(add, add) 386ATOMIC64_OPS(sub, subf) 387 388#define atomic64_add_return_relaxed atomic64_add_return_relaxed 389#define atomic64_sub_return_relaxed atomic64_sub_return_relaxed 390 391#define atomic64_fetch_add_relaxed atomic64_fetch_add_relaxed 392#define atomic64_fetch_sub_relaxed atomic64_fetch_sub_relaxed 393 394#undef ATOMIC64_OPS 395#define ATOMIC64_OPS(op, asm_op) \ 396 ATOMIC64_OP(op, asm_op) \ 397 ATOMIC64_FETCH_OP_RELAXED(op, asm_op) 398 399ATOMIC64_OPS(and, and) 400ATOMIC64_OPS(or, or) 401ATOMIC64_OPS(xor, xor) 402 403#define atomic64_fetch_and_relaxed atomic64_fetch_and_relaxed 404#define atomic64_fetch_or_relaxed atomic64_fetch_or_relaxed 405#define atomic64_fetch_xor_relaxed atomic64_fetch_xor_relaxed 406 407#undef ATOPIC64_OPS 408#undef ATOMIC64_FETCH_OP_RELAXED 409#undef ATOMIC64_OP_RETURN_RELAXED 410#undef ATOMIC64_OP 411 412static __inline__ void atomic64_inc(atomic64_t *v) 413{ 414 s64 t; 415 416 __asm__ __volatile__( 417"1: ldarx %0,0,%2 # atomic64_inc\n\ 418 addic %0,%0,1\n\ 419 stdcx. %0,0,%2 \n\ 420 bne- 1b" 421 : "=&r" (t), "+m" (v->counter) 422 : "r" (&v->counter) 423 : "cc", "xer"); 424} 425#define atomic64_inc atomic64_inc 426 427static __inline__ s64 atomic64_inc_return_relaxed(atomic64_t *v) 428{ 429 s64 t; 430 431 __asm__ __volatile__( 432"1: ldarx %0,0,%2 # atomic64_inc_return_relaxed\n" 433" addic %0,%0,1\n" 434" stdcx. %0,0,%2\n" 435" bne- 1b" 436 : "=&r" (t), "+m" (v->counter) 437 : "r" (&v->counter) 438 : "cc", "xer"); 439 440 return t; 441} 442 443static __inline__ void atomic64_dec(atomic64_t *v) 444{ 445 s64 t; 446 447 __asm__ __volatile__( 448"1: ldarx %0,0,%2 # atomic64_dec\n\ 449 addic %0,%0,-1\n\ 450 stdcx. %0,0,%2\n\ 451 bne- 1b" 452 : "=&r" (t), "+m" (v->counter) 453 : "r" (&v->counter) 454 : "cc", "xer"); 455} 456#define atomic64_dec atomic64_dec 457 458static __inline__ s64 atomic64_dec_return_relaxed(atomic64_t *v) 459{ 460 s64 t; 461 462 __asm__ __volatile__( 463"1: ldarx %0,0,%2 # atomic64_dec_return_relaxed\n" 464" addic %0,%0,-1\n" 465" stdcx. %0,0,%2\n" 466" bne- 1b" 467 : "=&r" (t), "+m" (v->counter) 468 : "r" (&v->counter) 469 : "cc", "xer"); 470 471 return t; 472} 473 474#define atomic64_inc_return_relaxed atomic64_inc_return_relaxed 475#define atomic64_dec_return_relaxed atomic64_dec_return_relaxed 476 477/* 478 * Atomically test *v and decrement if it is greater than 0. 479 * The function returns the old value of *v minus 1. 480 */ 481static __inline__ s64 atomic64_dec_if_positive(atomic64_t *v) 482{ 483 s64 t; 484 485 __asm__ __volatile__( 486 PPC_ATOMIC_ENTRY_BARRIER 487"1: ldarx %0,0,%1 # atomic64_dec_if_positive\n\ 488 addic. %0,%0,-1\n\ 489 blt- 2f\n\ 490 stdcx. %0,0,%1\n\ 491 bne- 1b" 492 PPC_ATOMIC_EXIT_BARRIER 493 "\n\ 4942:" : "=&r" (t) 495 : "r" (&v->counter) 496 : "cc", "xer", "memory"); 497 498 return t; 499} 500#define atomic64_dec_if_positive atomic64_dec_if_positive 501 502#define atomic64_cmpxchg(v, o, n) (cmpxchg(&((v)->counter), (o), (n))) 503#define atomic64_cmpxchg_relaxed(v, o, n) \ 504 cmpxchg_relaxed(&((v)->counter), (o), (n)) 505#define atomic64_cmpxchg_acquire(v, o, n) \ 506 cmpxchg_acquire(&((v)->counter), (o), (n)) 507 508#define atomic64_xchg(v, new) (xchg(&((v)->counter), new)) 509#define atomic64_xchg_relaxed(v, new) xchg_relaxed(&((v)->counter), (new)) 510 511/** 512 * atomic64_fetch_add_unless - add unless the number is a given value 513 * @v: pointer of type atomic64_t 514 * @a: the amount to add to v... 515 * @u: ...unless v is equal to u. 516 * 517 * Atomically adds @a to @v, so long as it was not @u. 518 * Returns the old value of @v. 519 */ 520static __inline__ s64 atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u) 521{ 522 s64 t; 523 524 __asm__ __volatile__ ( 525 PPC_ATOMIC_ENTRY_BARRIER 526"1: ldarx %0,0,%1 # atomic64_fetch_add_unless\n\ 527 cmpd 0,%0,%3 \n\ 528 beq 2f \n\ 529 add %0,%2,%0 \n" 530" stdcx. %0,0,%1 \n\ 531 bne- 1b \n" 532 PPC_ATOMIC_EXIT_BARRIER 533" subf %0,%2,%0 \n\ 5342:" 535 : "=&r" (t) 536 : "r" (&v->counter), "r" (a), "r" (u) 537 : "cc", "memory"); 538 539 return t; 540} 541#define atomic64_fetch_add_unless atomic64_fetch_add_unless 542 543/** 544 * atomic_inc64_not_zero - increment unless the number is zero 545 * @v: pointer of type atomic64_t 546 * 547 * Atomically increments @v by 1, so long as @v is non-zero. 548 * Returns non-zero if @v was non-zero, and zero otherwise. 549 */ 550static __inline__ int atomic64_inc_not_zero(atomic64_t *v) 551{ 552 s64 t1, t2; 553 554 __asm__ __volatile__ ( 555 PPC_ATOMIC_ENTRY_BARRIER 556"1: ldarx %0,0,%2 # atomic64_inc_not_zero\n\ 557 cmpdi 0,%0,0\n\ 558 beq- 2f\n\ 559 addic %1,%0,1\n\ 560 stdcx. %1,0,%2\n\ 561 bne- 1b\n" 562 PPC_ATOMIC_EXIT_BARRIER 563 "\n\ 5642:" 565 : "=&r" (t1), "=&r" (t2) 566 : "r" (&v->counter) 567 : "cc", "xer", "memory"); 568 569 return t1 != 0; 570} 571#define atomic64_inc_not_zero(v) atomic64_inc_not_zero((v)) 572 573#endif /* __powerpc64__ */ 574 575#endif /* __KERNEL__ */ 576#endif /* _ASM_POWERPC_ATOMIC_H_ */ 577