1 /* $OpenBSD: atomic.h,v 1.19 2022/08/29 02:01:18 jsg Exp $ */
2
3 /* Public Domain */
4
5 #ifndef _ARM_ATOMIC_H_
6 #define _ARM_ATOMIC_H_
7
8 /*
9 * Compare and set:
10 * ret = *ptr
11 * if (ret == expect)
12 * *ptr = new
13 * return (ret)
14 */
15 #define _def_atomic_cas(_f, _t) \
16 static inline _t \
17 _f(volatile _t *p, _t e, _t n) \
18 { \
19 _t ret, modified; \
20 \
21 __asm volatile ( \
22 "1: ldrex %0, [%4] \n\t" \
23 " cmp %0, %3 \n\t" \
24 " bne 2f \n\t" \
25 " strex %1, %2, [%4] \n\t" \
26 " cmp %1, #0 \n\t" \
27 " bne 1b \n\t" \
28 " b 3f \n\t" \
29 "2: clrex \n\t" \
30 "3: \n\t" \
31 : "=&r" (ret), "=&r" (modified) \
32 : "r" (n), "r" (e), "r" (p) \
33 : "memory", "cc" \
34 ); \
35 return (ret); \
36 }
_def_atomic_cas(_atomic_cas_uint,unsigned int)37 _def_atomic_cas(_atomic_cas_uint, unsigned int)
38 _def_atomic_cas(_atomic_cas_ulong, unsigned long)
39 #undef _def_atomic_cas
40
41 #define atomic_cas_uint(_p, _e, _n) _atomic_cas_uint((_p), (_e), (_n))
42 #define atomic_cas_ulong(_p, _e, _n) _atomic_cas_ulong((_p), (_e), (_n))
43
44 static inline void *
45 _atomic_cas_ptr(volatile void *p, void *e, void *n)
46 {
47 void *ret;
48 unsigned long modified;
49
50 __asm volatile (
51 "1: ldrex %0, [%4] \n\t"
52 " cmp %0, %3 \n\t"
53 " bne 2f \n\t"
54 " strex %1, %2, [%4] \n\t"
55 " cmp %1, #0 \n\t"
56 " bne 1b \n\t"
57 " b 3f \n\t"
58 "2: clrex \n\t"
59 "3: \n\t"
60 : "=&r" (ret), "=&r" (modified)
61 : "r" (n), "r" (e), "r" (p)
62 : "memory", "cc"
63 );
64 return (ret);
65 }
66 #define atomic_cas_ptr(_p, _e, _n) _atomic_cas_ptr((_p), (_e), (_n))
67
68 /*
69 * Swap:
70 * ret = *p
71 * *p = val
72 * return (ret)
73 */
74 #define _def_atomic_swap(_f, _t) \
75 static inline _t \
76 _f(volatile _t *p, _t v) \
77 { \
78 _t ret, modified; \
79 \
80 __asm volatile ( \
81 "1: ldrex %0, [%3] \n\t" \
82 " strex %1, %2, [%3] \n\t" \
83 " cmp %1, #0 \n\t" \
84 " bne 1b \n\t" \
85 : "=&r" (ret), "=&r" (modified) \
86 : "r" (v), "r" (p) \
87 : "memory", "cc" \
88 ); \
89 return (ret); \
90 }
_def_atomic_swap(_atomic_swap_uint,unsigned int)91 _def_atomic_swap(_atomic_swap_uint, unsigned int)
92 _def_atomic_swap(_atomic_swap_ulong, unsigned long)
93 #undef _def_atomic_swap
94
95 #define atomic_swap_uint(_p, _v) _atomic_swap_uint((_p), (_v))
96 #define atomic_swap_ulong(_p, _v) _atomic_swap_ulong((_p), (_v))
97
98 static inline void *
99 _atomic_swap_ptr(volatile void *p, void *v)
100 {
101 void *ret;
102 unsigned long modified;
103
104 __asm volatile (
105 "1: ldrex %0, [%3] \n\t"
106 " strex %1, %2, [%3] \n\t"
107 " cmp %1, #0 \n\t"
108 " bne 1b \n\t"
109 : "=&r" (ret), "=&r" (modified)
110 : "r" (v), "r" (p)
111 : "memory", "cc"
112 );
113 return (ret);
114 }
115 #define atomic_swap_ptr(_p, _v) _atomic_swap_ptr((_p), (_v))
116
117 /*
118 * Increment returning the new value
119 * *p += 1
120 * return (*p)
121 */
122 #define _def_atomic_inc_nv(_f, _t) \
123 static inline _t \
124 _f(volatile _t *p) \
125 { \
126 _t ret, modified; \
127 \
128 __asm volatile ( \
129 "1: ldrex %0, [%2] \n\t" \
130 " add %0, %0, #1 \n\t" \
131 " strex %1, %0, [%2] \n\t" \
132 " cmp %1, #0 \n\t" \
133 " bne 1b \n\t" \
134 : "=&r" (ret), "=&r" (modified) \
135 : "r" (p) \
136 : "memory", "cc" \
137 ); \
138 return (ret); \
139 }
_def_atomic_inc_nv(_atomic_inc_int_nv,unsigned int)140 _def_atomic_inc_nv(_atomic_inc_int_nv, unsigned int)
141 _def_atomic_inc_nv(_atomic_inc_long_nv, unsigned long)
142 #undef _def_atomic_inc_nv
143
144 #define atomic_inc_int_nv(_p) _atomic_inc_int_nv((_p))
145 #define atomic_inc_long_nv(_p) _atomic_inc_long_nv((_p))
146
147 /*
148 * Decrement returning the new value
149 * *p -= 1
150 * return (*p)
151 */
152 #define _def_atomic_dec_nv(_f, _t) \
153 static inline _t \
154 _f(volatile _t *p) \
155 { \
156 _t ret, modified; \
157 \
158 __asm volatile ( \
159 "1: ldrex %0, [%2] \n\t" \
160 " sub %0, %0, #1 \n\t" \
161 " strex %1, %0, [%2] \n\t" \
162 " cmp %1, #0 \n\t" \
163 " bne 1b \n\t" \
164 : "=&r" (ret), "=&r" (modified) \
165 : "r" (p) \
166 : "memory", "cc" \
167 ); \
168 return (ret); \
169 }
170 _def_atomic_dec_nv(_atomic_dec_int_nv, unsigned int)
171 _def_atomic_dec_nv(_atomic_dec_long_nv, unsigned long)
172 #undef _def_atomic_dec_nv
173
174 #define atomic_dec_int_nv(_p) _atomic_dec_int_nv((_p))
175 #define atomic_dec_long_nv(_p) _atomic_dec_long_nv((_p))
176
177 /*
178 * Addition returning the new value
179 * *p += v
180 * return (*p)
181 */
182 #define _def_atomic_add_nv(_f, _t) \
183 static inline _t \
184 _f(volatile _t *p, _t v) \
185 { \
186 _t ret, modified; \
187 \
188 __asm volatile ( \
189 "1: ldrex %0, [%2] \n\t" \
190 " add %0, %0, %3 \n\t" \
191 " strex %1, %0, [%2] \n\t" \
192 " cmp %1, #0 \n\t" \
193 " bne 1b \n\t" \
194 : "=&r" (ret), "=&r" (modified) \
195 : "r" (p), "r" (v) \
196 : "memory", "cc" \
197 ); \
198 return (ret); \
199 }
200 _def_atomic_add_nv(_atomic_add_int_nv, unsigned int)
201 _def_atomic_add_nv(_atomic_add_long_nv, unsigned long)
202 #undef _def_atomic_add_nv
203
204 #define atomic_add_int_nv(_p, _v) _atomic_add_int_nv((_p), (_v))
205 #define atomic_add_long_nv(_p, _v) _atomic_add_long_nv((_p), (_v))
206
207 /*
208 * Subtraction returning the new value
209 * *p -= v
210 * return (*p)
211 */
212 #define _def_atomic_sub_nv(_f, _t) \
213 static inline _t \
214 _f(volatile _t *p, _t v) \
215 { \
216 _t ret, modified; \
217 \
218 __asm volatile ( \
219 "1: ldrex %0, [%2] \n\t" \
220 " sub %0, %0, %3 \n\t" \
221 " strex %1, %0, [%2] \n\t" \
222 " cmp %1, #0 \n\t" \
223 " bne 1b \n\t" \
224 : "=&r" (ret), "=&r" (modified) \
225 : "r" (p), "r" (v) \
226 : "memory", "cc" \
227 ); \
228 return (ret); \
229 }
230 _def_atomic_sub_nv(_atomic_sub_int_nv, unsigned int)
231 _def_atomic_sub_nv(_atomic_sub_long_nv, unsigned long)
232 #undef _def_atomic_sub_nv
233
234 #define atomic_sub_int_nv(_p, _v) _atomic_sub_int_nv((_p), (_v))
235 #define atomic_sub_long_nv(_p, _v) _atomic_sub_long_nv((_p), (_v))
236
237 #define __membar(_f) do { __asm volatile(_f ::: "memory"); } while (0)
238
239 #define membar_enter() __membar("dmb sy")
240 #define membar_exit() __membar("dmb sy")
241 #define membar_producer() __membar("dmb st")
242 #define membar_consumer() __membar("dmb sy")
243 #define membar_sync() __membar("dmb sy")
244
245 #if defined(_KERNEL)
246
247 /* virtio needs MP membars even on SP kernels */
248 #define virtio_membar_producer() __membar("dmb st")
249 #define virtio_membar_consumer() __membar("dmb sy")
250 #define virtio_membar_sync() __membar("dmb sy")
251
252 /*
253 * Set bits
254 * *p = *p | v
255 */
256 static inline void
257 atomic_setbits_int(volatile unsigned int *p, unsigned int v)
258 {
259 unsigned int modified, tmp;
260
261 __asm volatile (
262 "1: ldrex %0, [%3] \n\t"
263 " orr %0, %0, %2 \n\t"
264 " strex %1, %0, [%3] \n\t"
265 " cmp %1, #0 \n\t"
266 " bne 1b \n\t"
267 : "=&r" (tmp), "=&r" (modified)
268 : "r" (v), "r" (p)
269 : "memory", "cc"
270 );
271 }
272
273 /*
274 * Clear bits
275 * *p = *p & (~v)
276 */
277 static inline void
atomic_clearbits_int(volatile unsigned int * p,unsigned int v)278 atomic_clearbits_int(volatile unsigned int *p, unsigned int v)
279 {
280 unsigned int modified, tmp;
281
282 __asm volatile (
283 "1: ldrex %0, [%3] \n\t"
284 " bic %0, %0, %2 \n\t"
285 " strex %1, %0, [%3] \n\t"
286 " cmp %1, #0 \n\t"
287 " bne 1b \n\t"
288 : "=&r" (tmp), "=&r" (modified)
289 : "r" (v), "r" (p)
290 : "memory", "cc"
291 );
292 }
293
294 #endif /* defined(_KERNEL) */
295 #endif /* _ARM_ATOMIC_H_ */
296