1/* SPDX-License-Identifier: GPL-2.0-or-later */ 2/* 3 * Generic barrier definitions. 4 * 5 * It should be possible to use these on really simple architectures, 6 * but it serves more as a starting point for new ports. 7 * 8 * Copyright (C) 2007 Red Hat, Inc. All Rights Reserved. 9 * Written by David Howells (dhowells@redhat.com) 10 */ 11#ifndef __ASM_GENERIC_BARRIER_H 12#define __ASM_GENERIC_BARRIER_H 13 14#ifndef __ASSEMBLY__ 15 16#include <linux/compiler.h> 17#include <asm/rwonce.h> 18 19#ifndef nop 20#define nop() asm volatile ("nop") 21#endif 22 23/* 24 * Force strict CPU ordering. And yes, this is required on UP too when we're 25 * talking to devices. 26 * 27 * Fall back to compiler barriers if nothing better is provided. 28 */ 29 30#ifndef mb 31#define mb() barrier() 32#endif 33 34#ifndef rmb 35#define rmb() mb() 36#endif 37 38#ifndef wmb 39#define wmb() mb() 40#endif 41 42#ifndef dma_rmb 43#define dma_rmb() rmb() 44#endif 45 46#ifndef dma_wmb 47#define dma_wmb() wmb() 48#endif 49 50#ifndef __smp_mb 51#define __smp_mb() mb() 52#endif 53 54#ifndef __smp_rmb 55#define __smp_rmb() rmb() 56#endif 57 58#ifndef __smp_wmb 59#define __smp_wmb() wmb() 60#endif 61 62#ifdef CONFIG_SMP 63 64#ifndef smp_mb 65#define smp_mb() __smp_mb() 66#endif 67 68#ifndef smp_rmb 69#define smp_rmb() __smp_rmb() 70#endif 71 72#ifndef smp_wmb 73#define smp_wmb() __smp_wmb() 74#endif 75 76#else /* !CONFIG_SMP */ 77 78#ifndef smp_mb 79#define smp_mb() barrier() 80#endif 81 82#ifndef smp_rmb 83#define smp_rmb() barrier() 84#endif 85 86#ifndef smp_wmb 87#define smp_wmb() barrier() 88#endif 89 90#endif /* CONFIG_SMP */ 91 92#ifndef __smp_store_mb 93#define __smp_store_mb(var, value) do { WRITE_ONCE(var, value); __smp_mb(); } while (0) 94#endif 95 96#ifndef __smp_mb__before_atomic 97#define __smp_mb__before_atomic() __smp_mb() 98#endif 99 100#ifndef __smp_mb__after_atomic 101#define __smp_mb__after_atomic() __smp_mb() 102#endif 103 104#ifndef __smp_store_release 105#define __smp_store_release(p, v) \ 106do { \ 107 compiletime_assert_atomic_type(*p); \ 108 __smp_mb(); \ 109 WRITE_ONCE(*p, v); \ 110} while (0) 111#endif 112 113#ifndef __smp_load_acquire 114#define __smp_load_acquire(p) \ 115({ \ 116 __unqual_scalar_typeof(*p) ___p1 = READ_ONCE(*p); \ 117 compiletime_assert_atomic_type(*p); \ 118 __smp_mb(); \ 119 (typeof(*p))___p1; \ 120}) 121#endif 122 123#ifdef CONFIG_SMP 124 125#ifndef smp_store_mb 126#define smp_store_mb(var, value) __smp_store_mb(var, value) 127#endif 128 129#ifndef smp_mb__before_atomic 130#define smp_mb__before_atomic() __smp_mb__before_atomic() 131#endif 132 133#ifndef smp_mb__after_atomic 134#define smp_mb__after_atomic() __smp_mb__after_atomic() 135#endif 136 137#ifndef smp_store_release 138#define smp_store_release(p, v) __smp_store_release(p, v) 139#endif 140 141#ifndef smp_load_acquire 142#define smp_load_acquire(p) __smp_load_acquire(p) 143#endif 144 145#else /* !CONFIG_SMP */ 146 147#ifndef smp_store_mb 148#define smp_store_mb(var, value) do { WRITE_ONCE(var, value); barrier(); } while (0) 149#endif 150 151#ifndef smp_mb__before_atomic 152#define smp_mb__before_atomic() barrier() 153#endif 154 155#ifndef smp_mb__after_atomic 156#define smp_mb__after_atomic() barrier() 157#endif 158 159#ifndef smp_store_release 160#define smp_store_release(p, v) \ 161do { \ 162 compiletime_assert_atomic_type(*p); \ 163 barrier(); \ 164 WRITE_ONCE(*p, v); \ 165} while (0) 166#endif 167 168#ifndef smp_load_acquire 169#define smp_load_acquire(p) \ 170({ \ 171 __unqual_scalar_typeof(*p) ___p1 = READ_ONCE(*p); \ 172 compiletime_assert_atomic_type(*p); \ 173 barrier(); \ 174 (typeof(*p))___p1; \ 175}) 176#endif 177 178#endif /* CONFIG_SMP */ 179 180/* Barriers for virtual machine guests when talking to an SMP host */ 181#define virt_mb() __smp_mb() 182#define virt_rmb() __smp_rmb() 183#define virt_wmb() __smp_wmb() 184#define virt_store_mb(var, value) __smp_store_mb(var, value) 185#define virt_mb__before_atomic() __smp_mb__before_atomic() 186#define virt_mb__after_atomic() __smp_mb__after_atomic() 187#define virt_store_release(p, v) __smp_store_release(p, v) 188#define virt_load_acquire(p) __smp_load_acquire(p) 189 190/** 191 * smp_acquire__after_ctrl_dep() - Provide ACQUIRE ordering after a control dependency 192 * 193 * A control dependency provides a LOAD->STORE order, the additional RMB 194 * provides LOAD->LOAD order, together they provide LOAD->{LOAD,STORE} order, 195 * aka. (load)-ACQUIRE. 196 * 197 * Architectures that do not do load speculation can have this be barrier(). 198 */ 199#ifndef smp_acquire__after_ctrl_dep 200#define smp_acquire__after_ctrl_dep() smp_rmb() 201#endif 202 203/** 204 * smp_cond_load_relaxed() - (Spin) wait for cond with no ordering guarantees 205 * @ptr: pointer to the variable to wait on 206 * @cond: boolean expression to wait for 207 * 208 * Equivalent to using READ_ONCE() on the condition variable. 209 * 210 * Due to C lacking lambda expressions we load the value of *ptr into a 211 * pre-named variable @VAL to be used in @cond. 212 */ 213#ifndef smp_cond_load_relaxed 214#define smp_cond_load_relaxed(ptr, cond_expr) ({ \ 215 typeof(ptr) __PTR = (ptr); \ 216 __unqual_scalar_typeof(*ptr) VAL; \ 217 for (;;) { \ 218 VAL = READ_ONCE(*__PTR); \ 219 if (cond_expr) \ 220 break; \ 221 cpu_relax(); \ 222 } \ 223 (typeof(*ptr))VAL; \ 224}) 225#endif 226 227/** 228 * smp_cond_load_acquire() - (Spin) wait for cond with ACQUIRE ordering 229 * @ptr: pointer to the variable to wait on 230 * @cond: boolean expression to wait for 231 * 232 * Equivalent to using smp_load_acquire() on the condition variable but employs 233 * the control dependency of the wait to reduce the barrier on many platforms. 234 */ 235#ifndef smp_cond_load_acquire 236#define smp_cond_load_acquire(ptr, cond_expr) ({ \ 237 __unqual_scalar_typeof(*ptr) _val; \ 238 _val = smp_cond_load_relaxed(ptr, cond_expr); \ 239 smp_acquire__after_ctrl_dep(); \ 240 (typeof(*ptr))_val; \ 241}) 242#endif 243 244/* 245 * pmem_wmb() ensures that all stores for which the modification 246 * are written to persistent storage by preceding instructions have 247 * updated persistent storage before any data access or data transfer 248 * caused by subsequent instructions is initiated. 249 */ 250#ifndef pmem_wmb 251#define pmem_wmb() wmb() 252#endif 253 254#endif /* !__ASSEMBLY__ */ 255#endif /* __ASM_GENERIC_BARRIER_H */ 256