1/* SPDX-License-Identifier: GPL-2.0 */
2/*
3 * Low level function for atomic operations
4 *
5 * Copyright IBM Corp. 1999, 2016
6 */
7
8#ifndef __ARCH_S390_ATOMIC_OPS__
9#define __ARCH_S390_ATOMIC_OPS__
10
11#ifdef CONFIG_HAVE_MARCH_Z196_FEATURES
12
13#define __ATOMIC_OP(op_name, op_type, op_string, op_barrier)		\
14static inline op_type op_name(op_type val, op_type *ptr)		\
15{									\
16	op_type old;							\
17									\
18	asm volatile(							\
19		op_string "	%[old],%[val],%[ptr]\n"			\
20		op_barrier						\
21		: [old] "=d" (old), [ptr] "+Q" (*ptr)			\
22		: [val] "d" (val) : "cc", "memory");			\
23	return old;							\
24}									\
25
26#define __ATOMIC_OPS(op_name, op_type, op_string)			\
27	__ATOMIC_OP(op_name, op_type, op_string, "\n")			\
28	__ATOMIC_OP(op_name##_barrier, op_type, op_string, "bcr 14,0\n")
29
30__ATOMIC_OPS(__atomic_add, int, "laa")
31__ATOMIC_OPS(__atomic_and, int, "lan")
32__ATOMIC_OPS(__atomic_or,  int, "lao")
33__ATOMIC_OPS(__atomic_xor, int, "lax")
34
35__ATOMIC_OPS(__atomic64_add, long, "laag")
36__ATOMIC_OPS(__atomic64_and, long, "lang")
37__ATOMIC_OPS(__atomic64_or,  long, "laog")
38__ATOMIC_OPS(__atomic64_xor, long, "laxg")
39
40#undef __ATOMIC_OPS
41#undef __ATOMIC_OP
42
43#define __ATOMIC_CONST_OP(op_name, op_type, op_string, op_barrier)	\
44static __always_inline void op_name(op_type val, op_type *ptr)		\
45{									\
46	asm volatile(							\
47		op_string "	%[ptr],%[val]\n"			\
48		op_barrier						\
49		: [ptr] "+Q" (*ptr) : [val] "i" (val) : "cc", "memory");\
50}
51
52#define __ATOMIC_CONST_OPS(op_name, op_type, op_string)			\
53	__ATOMIC_CONST_OP(op_name, op_type, op_string, "\n")		\
54	__ATOMIC_CONST_OP(op_name##_barrier, op_type, op_string, "bcr 14,0\n")
55
56__ATOMIC_CONST_OPS(__atomic_add_const, int, "asi")
57__ATOMIC_CONST_OPS(__atomic64_add_const, long, "agsi")
58
59#undef __ATOMIC_CONST_OPS
60#undef __ATOMIC_CONST_OP
61
62#else /* CONFIG_HAVE_MARCH_Z196_FEATURES */
63
64#define __ATOMIC_OP(op_name, op_string)					\
65static inline int op_name(int val, int *ptr)				\
66{									\
67	int old, new;							\
68									\
69	asm volatile(							\
70		"0:	lr	%[new],%[old]\n"			\
71		op_string "	%[new],%[val]\n"			\
72		"	cs	%[old],%[new],%[ptr]\n"			\
73		"	jl	0b"					\
74		: [old] "=d" (old), [new] "=&d" (new), [ptr] "+Q" (*ptr)\
75		: [val] "d" (val), "0" (*ptr) : "cc", "memory");	\
76	return old;							\
77}
78
79#define __ATOMIC_OPS(op_name, op_string)				\
80	__ATOMIC_OP(op_name, op_string)					\
81	__ATOMIC_OP(op_name##_barrier, op_string)
82
83__ATOMIC_OPS(__atomic_add, "ar")
84__ATOMIC_OPS(__atomic_and, "nr")
85__ATOMIC_OPS(__atomic_or,  "or")
86__ATOMIC_OPS(__atomic_xor, "xr")
87
88#undef __ATOMIC_OPS
89
90#define __ATOMIC64_OP(op_name, op_string)				\
91static inline long op_name(long val, long *ptr)				\
92{									\
93	long old, new;							\
94									\
95	asm volatile(							\
96		"0:	lgr	%[new],%[old]\n"			\
97		op_string "	%[new],%[val]\n"			\
98		"	csg	%[old],%[new],%[ptr]\n"			\
99		"	jl	0b"					\
100		: [old] "=d" (old), [new] "=&d" (new), [ptr] "+Q" (*ptr)\
101		: [val] "d" (val), "0" (*ptr) : "cc", "memory");	\
102	return old;							\
103}
104
105#define __ATOMIC64_OPS(op_name, op_string)				\
106	__ATOMIC64_OP(op_name, op_string)				\
107	__ATOMIC64_OP(op_name##_barrier, op_string)
108
109__ATOMIC64_OPS(__atomic64_add, "agr")
110__ATOMIC64_OPS(__atomic64_and, "ngr")
111__ATOMIC64_OPS(__atomic64_or,  "ogr")
112__ATOMIC64_OPS(__atomic64_xor, "xgr")
113
114#undef __ATOMIC64_OPS
115
116#define __atomic_add_const(val, ptr)		__atomic_add(val, ptr)
117#define __atomic_add_const_barrier(val, ptr)	__atomic_add(val, ptr)
118#define __atomic64_add_const(val, ptr)		__atomic64_add(val, ptr)
119#define __atomic64_add_const_barrier(val, ptr)	__atomic64_add(val, ptr)
120
121#endif /* CONFIG_HAVE_MARCH_Z196_FEATURES */
122
123static inline int __atomic_cmpxchg(int *ptr, int old, int new)
124{
125	return __sync_val_compare_and_swap(ptr, old, new);
126}
127
128static inline int __atomic_cmpxchg_bool(int *ptr, int old, int new)
129{
130	return __sync_bool_compare_and_swap(ptr, old, new);
131}
132
133static inline long __atomic64_cmpxchg(long *ptr, long old, long new)
134{
135	return __sync_val_compare_and_swap(ptr, old, new);
136}
137
138static inline long __atomic64_cmpxchg_bool(long *ptr, long old, long new)
139{
140	return __sync_bool_compare_and_swap(ptr, old, new);
141}
142
143#endif /* __ARCH_S390_ATOMIC_OPS__  */
144