1// SPDX-License-Identifier: GPL-2.0-or-later 2/* 3 * Generic implementation of 64-bit atomics using spinlocks, 4 * useful on processors that don't have 64-bit atomic instructions. 5 * 6 * Copyright © 2009 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com> 7 */ 8#include <linux/types.h> 9#include <linux/cache.h> 10#include <linux/spinlock.h> 11#include <linux/init.h> 12#include <linux/export.h> 13#include <linux/atomic.h> 14 15/* 16 * We use a hashed array of spinlocks to provide exclusive access 17 * to each atomic64_t variable. Since this is expected to used on 18 * systems with small numbers of CPUs (<= 4 or so), we use a 19 * relatively small array of 16 spinlocks to avoid wasting too much 20 * memory on the spinlock array. 21 */ 22#define NR_LOCKS 16 23 24/* 25 * Ensure each lock is in a separate cacheline. 26 */ 27static union { 28 raw_spinlock_t lock; 29 char pad[L1_CACHE_BYTES]; 30} atomic64_lock[NR_LOCKS] __cacheline_aligned_in_smp = { 31 [0 ... (NR_LOCKS - 1)] = { 32 .lock = __RAW_SPIN_LOCK_UNLOCKED(atomic64_lock.lock), 33 }, 34}; 35 36static inline raw_spinlock_t *lock_addr(const atomic64_t *v) 37{ 38 unsigned long addr = (unsigned long) v; 39 40 addr >>= L1_CACHE_SHIFT; 41 addr ^= (addr >> 8) ^ (addr >> 16); 42 return &atomic64_lock[addr & (NR_LOCKS - 1)].lock; 43} 44 45s64 atomic64_read(const atomic64_t *v) 46{ 47 unsigned long flags; 48 raw_spinlock_t *lock = lock_addr(v); 49 s64 val; 50 51 raw_spin_lock_irqsave(lock, flags); 52 val = v->counter; 53 raw_spin_unlock_irqrestore(lock, flags); 54 return val; 55} 56EXPORT_SYMBOL(atomic64_read); 57 58void atomic64_set(atomic64_t *v, s64 i) 59{ 60 unsigned long flags; 61 raw_spinlock_t *lock = lock_addr(v); 62 63 raw_spin_lock_irqsave(lock, flags); 64 v->counter = i; 65 raw_spin_unlock_irqrestore(lock, flags); 66} 67EXPORT_SYMBOL(atomic64_set); 68 69#define ATOMIC64_OP(op, c_op) \ 70void atomic64_##op(s64 a, atomic64_t *v) \ 71{ \ 72 unsigned long flags; \ 73 raw_spinlock_t *lock = lock_addr(v); \ 74 \ 75 raw_spin_lock_irqsave(lock, flags); \ 76 v->counter c_op a; \ 77 raw_spin_unlock_irqrestore(lock, flags); \ 78} \ 79EXPORT_SYMBOL(atomic64_##op); 80 81#define ATOMIC64_OP_RETURN(op, c_op) \ 82s64 atomic64_##op##_return(s64 a, atomic64_t *v) \ 83{ \ 84 unsigned long flags; \ 85 raw_spinlock_t *lock = lock_addr(v); \ 86 s64 val; \ 87 \ 88 raw_spin_lock_irqsave(lock, flags); \ 89 val = (v->counter c_op a); \ 90 raw_spin_unlock_irqrestore(lock, flags); \ 91 return val; \ 92} \ 93EXPORT_SYMBOL(atomic64_##op##_return); 94 95#define ATOMIC64_FETCH_OP(op, c_op) \ 96s64 atomic64_fetch_##op(s64 a, atomic64_t *v) \ 97{ \ 98 unsigned long flags; \ 99 raw_spinlock_t *lock = lock_addr(v); \ 100 s64 val; \ 101 \ 102 raw_spin_lock_irqsave(lock, flags); \ 103 val = v->counter; \ 104 v->counter c_op a; \ 105 raw_spin_unlock_irqrestore(lock, flags); \ 106 return val; \ 107} \ 108EXPORT_SYMBOL(atomic64_fetch_##op); 109 110#define ATOMIC64_OPS(op, c_op) \ 111 ATOMIC64_OP(op, c_op) \ 112 ATOMIC64_OP_RETURN(op, c_op) \ 113 ATOMIC64_FETCH_OP(op, c_op) 114 115ATOMIC64_OPS(add, +=) 116ATOMIC64_OPS(sub, -=) 117 118#undef ATOMIC64_OPS 119#define ATOMIC64_OPS(op, c_op) \ 120 ATOMIC64_OP(op, c_op) \ 121 ATOMIC64_OP_RETURN(op, c_op) \ 122 ATOMIC64_FETCH_OP(op, c_op) 123 124ATOMIC64_OPS(and, &=) 125ATOMIC64_OPS(or, |=) 126ATOMIC64_OPS(xor, ^=) 127 128#undef ATOMIC64_OPS 129#undef ATOMIC64_FETCH_OP 130#undef ATOMIC64_OP_RETURN 131#undef ATOMIC64_OP 132 133s64 atomic64_dec_if_positive(atomic64_t *v) 134{ 135 unsigned long flags; 136 raw_spinlock_t *lock = lock_addr(v); 137 s64 val; 138 139 raw_spin_lock_irqsave(lock, flags); 140 val = v->counter - 1; 141 if (val >= 0) 142 v->counter = val; 143 raw_spin_unlock_irqrestore(lock, flags); 144 return val; 145} 146EXPORT_SYMBOL(atomic64_dec_if_positive); 147 148s64 atomic64_cmpxchg(atomic64_t *v, s64 o, s64 n) 149{ 150 unsigned long flags; 151 raw_spinlock_t *lock = lock_addr(v); 152 s64 val; 153 154 raw_spin_lock_irqsave(lock, flags); 155 val = v->counter; 156 if (val == o) 157 v->counter = n; 158 raw_spin_unlock_irqrestore(lock, flags); 159 return val; 160} 161EXPORT_SYMBOL(atomic64_cmpxchg); 162 163s64 atomic64_xchg(atomic64_t *v, s64 new) 164{ 165 unsigned long flags; 166 raw_spinlock_t *lock = lock_addr(v); 167 s64 val; 168 169 raw_spin_lock_irqsave(lock, flags); 170 val = v->counter; 171 v->counter = new; 172 raw_spin_unlock_irqrestore(lock, flags); 173 return val; 174} 175EXPORT_SYMBOL(atomic64_xchg); 176 177s64 atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u) 178{ 179 unsigned long flags; 180 raw_spinlock_t *lock = lock_addr(v); 181 s64 val; 182 183 raw_spin_lock_irqsave(lock, flags); 184 val = v->counter; 185 if (val != u) 186 v->counter += a; 187 raw_spin_unlock_irqrestore(lock, flags); 188 189 return val; 190} 191EXPORT_SYMBOL(atomic64_fetch_add_unless); 192