1/** 2 * Many similar implementations exist. See for example libwsbm 3 * or the linux kernel include/atomic.h 4 * 5 * No copyright claimed on this file. 6 * 7 */ 8 9#include "no_extern_c.h" 10 11#ifndef U_ATOMIC_H 12#define U_ATOMIC_H 13 14#include <stdbool.h> 15#include <stdint.h> 16 17/* Favor OS-provided implementations. 18 * 19 * Where no OS-provided implementation is available, fall back to 20 * locally coded assembly, compiler intrinsic or ultimately a 21 * mutex-based implementation. 22 */ 23#if defined(__sun) 24#define PIPE_ATOMIC_OS_SOLARIS 25#elif defined(_MSC_VER) 26#define PIPE_ATOMIC_MSVC_INTRINSIC 27#elif defined(__GNUC__) 28#define PIPE_ATOMIC_GCC_INTRINSIC 29#else 30#error "Unsupported platform" 31#endif 32 33 34/* Implementation using GCC-provided synchronization intrinsics 35 */ 36#if defined(PIPE_ATOMIC_GCC_INTRINSIC) 37 38#define PIPE_ATOMIC "GCC Sync Intrinsics" 39 40#if defined(USE_GCC_ATOMIC_BUILTINS) 41 42/* The builtins with explicit memory model are available since GCC 4.7. */ 43#define p_atomic_set(_v, _i) __atomic_store_n((_v), (_i), __ATOMIC_RELEASE) 44#define p_atomic_read(_v) __atomic_load_n((_v), __ATOMIC_ACQUIRE) 45#define p_atomic_read_relaxed(_v) __atomic_load_n((_v), __ATOMIC_RELAXED) 46#define p_atomic_dec_zero(v) (__atomic_sub_fetch((v), 1, __ATOMIC_ACQ_REL) == 0) 47#define p_atomic_inc(v) (void) __atomic_add_fetch((v), 1, __ATOMIC_ACQ_REL) 48#define p_atomic_dec(v) (void) __atomic_sub_fetch((v), 1, __ATOMIC_ACQ_REL) 49#define p_atomic_add(v, i) (void) __atomic_add_fetch((v), (i), __ATOMIC_ACQ_REL) 50#define p_atomic_inc_return(v) __atomic_add_fetch((v), 1, __ATOMIC_ACQ_REL) 51#define p_atomic_dec_return(v) __atomic_sub_fetch((v), 1, __ATOMIC_ACQ_REL) 52#define p_atomic_add_return(v, i) __atomic_add_fetch((v), (i), __ATOMIC_ACQ_REL) 53#define p_atomic_fetch_add(v, i) __atomic_fetch_add((v), (i), __ATOMIC_ACQ_REL) 54#define p_atomic_xchg(v, i) __atomic_exchange_n((v), (i), __ATOMIC_ACQ_REL) 55#define PIPE_NATIVE_ATOMIC_XCHG 56 57#else 58 59#define p_atomic_set(_v, _i) (*(_v) = (_i)) 60#define p_atomic_read(_v) (*(_v)) 61#define p_atomic_read_relaxed(_v) (*(_v)) 62#define p_atomic_dec_zero(v) (__sync_sub_and_fetch((v), 1) == 0) 63#define p_atomic_inc(v) (void) __sync_add_and_fetch((v), 1) 64#define p_atomic_dec(v) (void) __sync_sub_and_fetch((v), 1) 65#define p_atomic_add(v, i) (void) __sync_add_and_fetch((v), (i)) 66#define p_atomic_inc_return(v) __sync_add_and_fetch((v), 1) 67#define p_atomic_dec_return(v) __sync_sub_and_fetch((v), 1) 68#define p_atomic_add_return(v, i) __sync_add_and_fetch((v), (i)) 69#define p_atomic_fetch_add(v, i) __sync_fetch_and_add((v), (i)) 70 71#endif 72 73/* There is no __atomic_* compare and exchange that returns the current value. 74 * Also, GCC 5.4 seems unable to optimize a compound statement expression that 75 * uses an additional stack variable with __atomic_compare_exchange[_n]. 76 */ 77#define p_atomic_cmpxchg(v, old, _new) \ 78 __sync_val_compare_and_swap((v), (old), (_new)) 79#define p_atomic_cmpxchg_ptr(v, old, _new) p_atomic_cmpxchg(v, old, _new) 80 81#endif 82 83 84 85/* Unlocked version for single threaded environments, such as some 86 * windows kernel modules. 87 */ 88#if defined(PIPE_ATOMIC_OS_UNLOCKED) 89 90#define PIPE_ATOMIC "Unlocked" 91 92#define p_atomic_set(_v, _i) (*(_v) = (_i)) 93#define p_atomic_read(_v) (*(_v)) 94#define p_atomic_read_relaxed(_v) (*(_v)) 95#define p_atomic_dec_zero(_v) (p_atomic_dec_return(_v) == 0) 96#define p_atomic_inc(_v) ((void) p_atomic_inc_return(_v)) 97#define p_atomic_dec(_v) ((void) p_atomic_dec_return(_v)) 98#define p_atomic_add(_v, _i) ((void) p_atomic_add_return((_v), (_i))) 99#define p_atomic_inc_return(_v) (++(*(_v))) 100#define p_atomic_dec_return(_v) (--(*(_v))) 101#define p_atomic_add_return(_v, _i) (*(_v) = *(_v) + (_i)) 102#define p_atomic_fetch_add(_v, _i) (*(_v) = *(_v) + (_i), *(_v) - (_i)) 103#define p_atomic_cmpxchg(_v, _old, _new) (*(_v) == (_old) ? (*(_v) = (_new), (_old)) : *(_v)) 104#define p_atomic_cmpxchg_ptr(_v, _old, _new) p_atomic_cmpxchg(_v, _old, _new) 105 106#endif 107 108 109#if defined(PIPE_ATOMIC_MSVC_INTRINSIC) 110 111#define PIPE_ATOMIC "MSVC Intrinsics" 112 113/* We use the Windows header's Interlocked*64 functions instead of the 114 * _Interlocked*64 intrinsics wherever we can, as support for the latter varies 115 * with target CPU, whereas Windows headers take care of all portability 116 * issues: using intrinsics where available, falling back to library 117 * implementations where not. 118 */ 119#ifndef WIN32_LEAN_AND_MEAN 120#define WIN32_LEAN_AND_MEAN 1 121#endif 122#include <windows.h> 123#include <intrin.h> 124#include <assert.h> 125 126/* MSVC supports decltype keyword, but it's only supported on C++ and doesn't 127 * quite work here; and if a C++-only solution is worthwhile, then it would be 128 * better to use templates / function overloading, instead of decltype magic. 129 * Therefore, we rely on implicit casting to LONGLONG for the functions that return 130 */ 131 132#define p_atomic_set(_v, _i) (*(_v) = (_i)) 133#define p_atomic_read(_v) (*(_v)) 134#define p_atomic_read_relaxed(_v) (*(_v)) 135 136#define p_atomic_dec_zero(_v) \ 137 (p_atomic_dec_return(_v) == 0) 138 139#define p_atomic_inc(_v) \ 140 ((void) p_atomic_inc_return(_v)) 141 142#define p_atomic_inc_return(_v) (\ 143 sizeof *(_v) == sizeof(short) ? _InterlockedIncrement16((short *) (_v)) : \ 144 sizeof *(_v) == sizeof(long) ? _InterlockedIncrement ((long *) (_v)) : \ 145 sizeof *(_v) == sizeof(__int64) ? InterlockedIncrement64 ((__int64 *)(_v)) : \ 146 (assert(!"should not get here"), 0)) 147 148#define p_atomic_dec(_v) \ 149 ((void) p_atomic_dec_return(_v)) 150 151#define p_atomic_dec_return(_v) (\ 152 sizeof *(_v) == sizeof(short) ? _InterlockedDecrement16((short *) (_v)) : \ 153 sizeof *(_v) == sizeof(long) ? _InterlockedDecrement ((long *) (_v)) : \ 154 sizeof *(_v) == sizeof(__int64) ? InterlockedDecrement64 ((__int64 *)(_v)) : \ 155 (assert(!"should not get here"), 0)) 156 157#define p_atomic_add(_v, _i) \ 158 ((void) p_atomic_fetch_add((_v), (_i))) 159 160#define p_atomic_add_return(_v, _i) (\ 161 sizeof *(_v) == sizeof(long) ? InterlockedAdd ((long *) (_v), (_i)) : \ 162 sizeof *(_v) == sizeof(__int64) ? InterlockedAdd64((__int64 *)(_v), (_i)) : \ 163 (assert(!"should not get here"), 0)) 164 165#define p_atomic_fetch_add(_v, _i) (\ 166 sizeof *(_v) == sizeof(char) ? _InterlockedExchangeAdd8 ((char *) (_v), (_i)) : \ 167 sizeof *(_v) == sizeof(short) ? _InterlockedExchangeAdd16((short *) (_v), (_i)) : \ 168 sizeof *(_v) == sizeof(long) ? InterlockedExchangeAdd ((long *) (_v), (_i)) : \ 169 sizeof *(_v) == sizeof(__int64) ? InterlockedExchangeAdd64((__int64 *)(_v), (_i)) : \ 170 (assert(!"should not get here"), 0)) 171 172#define p_atomic_cmpxchg(_v, _old, _new) (\ 173 sizeof *(_v) == sizeof(char) ? _InterlockedCompareExchange8 ((char *) (_v), (char) (_new), (char) (_old)) : \ 174 sizeof *(_v) == sizeof(short) ? _InterlockedCompareExchange16((short *) (_v), (short) (_new), (short) (_old)) : \ 175 sizeof *(_v) == sizeof(long) ? _InterlockedCompareExchange ((long *) (_v), (long) (_new), (long) (_old)) : \ 176 sizeof *(_v) == sizeof(__int64) ? InterlockedCompareExchange64 ((__int64 *)(_v), (__int64)(_new), (__int64)(_old)) : \ 177 (assert(!"should not get here"), 0)) 178 179#if defined(_WIN64) 180#define p_atomic_cmpxchg_ptr(_v, _old, _new) (void *)InterlockedCompareExchange64((__int64 *)(_v), (__int64)(_new), (__int64)(_old)) 181#else 182#define p_atomic_cmpxchg_ptr(_v, _old, _new) (void *)InterlockedCompareExchange((long *)(_v), (long)(_new), (long)(_old)) 183#endif 184 185#define PIPE_NATIVE_ATOMIC_XCHG 186#define p_atomic_xchg(_v, _new) (\ 187 sizeof *(_v) == sizeof(long) ? InterlockedExchange ((long *) (_v), (long) (_new)) : \ 188 sizeof *(_v) == sizeof(__int64) ? InterlockedExchange64((__int64 *)(_v), (__int64)(_new)) : \ 189 (assert(!"should not get here"), 0)) 190 191#endif 192 193#if defined(PIPE_ATOMIC_OS_SOLARIS) 194 195#define PIPE_ATOMIC "Solaris OS atomic functions" 196 197#include <atomic.h> 198#include <assert.h> 199 200#define p_atomic_set(_v, _i) (*(_v) = (_i)) 201#define p_atomic_read(_v) (*(_v)) 202 203#define p_atomic_dec_zero(v) (\ 204 sizeof(*v) == sizeof(uint8_t) ? atomic_dec_8_nv ((uint8_t *)(v)) == 0 : \ 205 sizeof(*v) == sizeof(uint16_t) ? atomic_dec_16_nv((uint16_t *)(v)) == 0 : \ 206 sizeof(*v) == sizeof(uint32_t) ? atomic_dec_32_nv((uint32_t *)(v)) == 0 : \ 207 sizeof(*v) == sizeof(uint64_t) ? atomic_dec_64_nv((uint64_t *)(v)) == 0 : \ 208 (assert(!"should not get here"), 0)) 209 210#define p_atomic_inc(v) (void) (\ 211 sizeof(*v) == sizeof(uint8_t) ? atomic_inc_8 ((uint8_t *)(v)) : \ 212 sizeof(*v) == sizeof(uint16_t) ? atomic_inc_16((uint16_t *)(v)) : \ 213 sizeof(*v) == sizeof(uint32_t) ? atomic_inc_32((uint32_t *)(v)) : \ 214 sizeof(*v) == sizeof(uint64_t) ? atomic_inc_64((uint64_t *)(v)) : \ 215 (assert(!"should not get here"), 0)) 216 217#define p_atomic_inc_return(v) (__typeof(*v))( \ 218 sizeof(*v) == sizeof(uint8_t) ? atomic_inc_8_nv ((uint8_t *)(v)) : \ 219 sizeof(*v) == sizeof(uint16_t) ? atomic_inc_16_nv((uint16_t *)(v)) : \ 220 sizeof(*v) == sizeof(uint32_t) ? atomic_inc_32_nv((uint32_t *)(v)) : \ 221 sizeof(*v) == sizeof(uint64_t) ? atomic_inc_64_nv((uint64_t *)(v)) : \ 222 (assert(!"should not get here"), 0)) 223 224#define p_atomic_dec(v) (void) ( \ 225 sizeof(*v) == sizeof(uint8_t) ? atomic_dec_8 ((uint8_t *)(v)) : \ 226 sizeof(*v) == sizeof(uint16_t) ? atomic_dec_16((uint16_t *)(v)) : \ 227 sizeof(*v) == sizeof(uint32_t) ? atomic_dec_32((uint32_t *)(v)) : \ 228 sizeof(*v) == sizeof(uint64_t) ? atomic_dec_64((uint64_t *)(v)) : \ 229 (assert(!"should not get here"), 0)) 230 231#define p_atomic_dec_return(v) (__typeof(*v))( \ 232 sizeof(*v) == sizeof(uint8_t) ? atomic_dec_8_nv ((uint8_t *)(v)) : \ 233 sizeof(*v) == sizeof(uint16_t) ? atomic_dec_16_nv((uint16_t *)(v)) : \ 234 sizeof(*v) == sizeof(uint32_t) ? atomic_dec_32_nv((uint32_t *)(v)) : \ 235 sizeof(*v) == sizeof(uint64_t) ? atomic_dec_64_nv((uint64_t *)(v)) : \ 236 (assert(!"should not get here"), 0)) 237 238#define p_atomic_add(v, i) (void) ( \ 239 sizeof(*v) == sizeof(uint8_t) ? atomic_add_8 ((uint8_t *)(v), (i)) : \ 240 sizeof(*v) == sizeof(uint16_t) ? atomic_add_16((uint16_t *)(v), (i)) : \ 241 sizeof(*v) == sizeof(uint32_t) ? atomic_add_32((uint32_t *)(v), (i)) : \ 242 sizeof(*v) == sizeof(uint64_t) ? atomic_add_64((uint64_t *)(v), (i)) : \ 243 (assert(!"should not get here"), 0)) 244 245#define p_atomic_add_return(v, i) (__typeof(*v)) ( \ 246 sizeof(*v) == sizeof(uint8_t) ? atomic_add_8_nv ((uint8_t *)(v), (i)) : \ 247 sizeof(*v) == sizeof(uint16_t) ? atomic_add_16_nv((uint16_t *)(v), (i)) : \ 248 sizeof(*v) == sizeof(uint32_t) ? atomic_add_32_nv((uint32_t *)(v), (i)) : \ 249 sizeof(*v) == sizeof(uint64_t) ? atomic_add_64_nv((uint64_t *)(v), (i)) : \ 250 (assert(!"should not get here"), 0)) 251 252#define p_atomic_fetch_add(v, i) (__typeof(*v)) ( \ 253 sizeof(*v) == sizeof(uint8_t) ? atomic_add_8_nv ((uint8_t *)(v), (i)) - (i) : \ 254 sizeof(*v) == sizeof(uint16_t) ? atomic_add_16_nv((uint16_t *)(v), (i)) - (i) : \ 255 sizeof(*v) == sizeof(uint32_t) ? atomic_add_32_nv((uint32_t *)(v), (i)) - (i) : \ 256 sizeof(*v) == sizeof(uint64_t) ? atomic_add_64_nv((uint64_t *)(v), (i)) - (i) : \ 257 (assert(!"should not get here"), 0)) 258 259#define p_atomic_cmpxchg(v, old, _new) (__typeof(*v))( \ 260 sizeof(*v) == sizeof(uint8_t) ? atomic_cas_8 ((uint8_t *)(v), (uint8_t )(old), (uint8_t )(_new)) : \ 261 sizeof(*v) == sizeof(uint16_t) ? atomic_cas_16((uint16_t *)(v), (uint16_t)(old), (uint16_t)(_new)) : \ 262 sizeof(*v) == sizeof(uint32_t) ? atomic_cas_32((uint32_t *)(v), (uint32_t)(old), (uint32_t)(_new)) : \ 263 sizeof(*v) == sizeof(uint64_t) ? atomic_cas_64((uint64_t *)(v), (uint64_t)(old), (uint64_t)(_new)) : \ 264 (assert(!"should not get here"), 0)) 265 266#if INTPTR_MAX == INT32_MAX 267#define p_atomic_cmpxchg_ptr(v, old, _new) (__typeof(*v))(atomic_cas_32((uint32_t *)(v), (uint32_t)(old), (uint32_t)(_new))) 268#else 269#define p_atomic_cmpxchg_ptr(v, old, _new) (__typeof(*v))(atomic_cas_64((uint64_t *)(v), (uint64_t)(old), (uint64_t)(_new))) 270#endif 271 272#endif 273 274#ifndef PIPE_ATOMIC 275#error "No pipe_atomic implementation selected" 276#endif 277 278#ifndef PIPE_NATIVE_ATOMIC_XCHG 279static inline uint32_t p_atomic_xchg_32(uint32_t *v, uint32_t i) 280{ 281 uint32_t actual = p_atomic_read(v); 282 uint32_t expected; 283 do { 284 expected = actual; 285 actual = p_atomic_cmpxchg(v, expected, i); 286 } while (expected != actual); 287 return actual; 288} 289 290static inline uint64_t p_atomic_xchg_64(uint64_t *v, uint64_t i) 291{ 292 uint64_t actual = p_atomic_read(v); 293 uint64_t expected; 294 do { 295 expected = actual; 296 actual = p_atomic_cmpxchg(v, expected, i); 297 } while (expected != actual); 298 return actual; 299} 300 301#define p_atomic_xchg(v, i) (__typeof(*(v)))( \ 302 sizeof(*(v)) == sizeof(uint32_t) ? p_atomic_xchg_32((uint32_t *)(v), (uint32_t)(i)) : \ 303 sizeof(*(v)) == sizeof(uint64_t) ? p_atomic_xchg_64((uint64_t *)(v), (uint64_t)(i)) : \ 304 (assert(!"should not get here"), 0)) 305#endif 306 307#endif /* U_ATOMIC_H */ 308