1 /* $NetBSD: atomic.h,v 1.42 2021/12/19 12:21:30 riastradh Exp $ */
2
3 /*-
4 * Copyright (c) 2013 The NetBSD Foundation, Inc.
5 * All rights reserved.
6 *
7 * This code is derived from software contributed to The NetBSD Foundation
8 * by Taylor R. Campbell.
9 *
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions
12 * are met:
13 * 1. Redistributions of source code must retain the above copyright
14 * notice, this list of conditions and the following disclaimer.
15 * 2. Redistributions in binary form must reproduce the above copyright
16 * notice, this list of conditions and the following disclaimer in the
17 * documentation and/or other materials provided with the distribution.
18 *
19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29 * POSSIBILITY OF SUCH DAMAGE.
30 */
31
32 #ifndef _LINUX_ATOMIC_H_
33 #define _LINUX_ATOMIC_H_
34
35 #include <sys/atomic.h>
36
37 #include <machine/limits.h>
38
39 #include <asm/barrier.h>
40
41 /* XXX Hope the GCC __sync builtins work everywhere we care about! */
42 #define xchg(P, V) __sync_lock_test_and_set(P, V)
43 #define cmpxchg(P, O, N) __sync_val_compare_and_swap(P, O, N)
44 #define try_cmpxchg(P, V, N) \
45 ({ \
46 __typeof__(*(V)) *__tcx_v = (V), __tcx_expected = *__tcx_v; \
47 (*__tcx_v = cmpxchg((P), __tcx_expected, (N))) == __tcx_expected; \
48 })
49
50 /*
51 * atomic (u)int operations
52 *
53 * Atomics that return a value, other than atomic_read, imply a
54 * full memory_sync barrier. Those that do not return a value
55 * imply no memory barrier.
56 */
57
58 struct atomic {
59 union {
60 volatile int au_int;
61 volatile unsigned int au_uint;
62 } a_u;
63 };
64
65 #define ATOMIC_INIT(i) { .a_u = { .au_int = (i) } }
66
67 typedef struct atomic atomic_t;
68
69 static inline int
atomic_read(const atomic_t * atomic)70 atomic_read(const atomic_t *atomic)
71 {
72 /* no membar */
73 return atomic->a_u.au_int;
74 }
75
76 static inline void
atomic_set(atomic_t * atomic,int value)77 atomic_set(atomic_t *atomic, int value)
78 {
79 /* no membar */
80 atomic->a_u.au_int = value;
81 }
82
83 static inline void
atomic_set_release(atomic_t * atomic,int value)84 atomic_set_release(atomic_t *atomic, int value)
85 {
86 atomic_store_release(&atomic->a_u.au_int, value);
87 }
88
89 static inline void
atomic_add(int addend,atomic_t * atomic)90 atomic_add(int addend, atomic_t *atomic)
91 {
92 /* no membar */
93 atomic_add_int(&atomic->a_u.au_uint, addend);
94 }
95
96 static inline void
atomic_sub(int subtrahend,atomic_t * atomic)97 atomic_sub(int subtrahend, atomic_t *atomic)
98 {
99 /* no membar */
100 atomic_add_int(&atomic->a_u.au_uint, -subtrahend);
101 }
102
103 static inline int
atomic_add_return(int addend,atomic_t * atomic)104 atomic_add_return(int addend, atomic_t *atomic)
105 {
106 int v;
107
108 smp_mb__before_atomic();
109 v = (int)atomic_add_int_nv(&atomic->a_u.au_uint, addend);
110 smp_mb__after_atomic();
111
112 return v;
113 }
114
115 static inline int
atomic_sub_return(int subtrahend,atomic_t * atomic)116 atomic_sub_return(int subtrahend, atomic_t *atomic)
117 {
118 int v;
119
120 smp_mb__before_atomic();
121 v = (int)atomic_add_int_nv(&atomic->a_u.au_uint, -subtrahend);
122 smp_mb__after_atomic();
123
124 return v;
125 }
126
127 static inline void
atomic_inc(atomic_t * atomic)128 atomic_inc(atomic_t *atomic)
129 {
130 /* no membar */
131 atomic_inc_uint(&atomic->a_u.au_uint);
132 }
133
134 static inline void
atomic_dec(atomic_t * atomic)135 atomic_dec(atomic_t *atomic)
136 {
137 /* no membar */
138 atomic_dec_uint(&atomic->a_u.au_uint);
139 }
140
141 static inline int
atomic_inc_return(atomic_t * atomic)142 atomic_inc_return(atomic_t *atomic)
143 {
144 int v;
145
146 smp_mb__before_atomic();
147 v = (int)atomic_inc_uint_nv(&atomic->a_u.au_uint);
148 smp_mb__after_atomic();
149
150 return v;
151 }
152
153 static inline int
atomic_dec_return(atomic_t * atomic)154 atomic_dec_return(atomic_t *atomic)
155 {
156 int v;
157
158 smp_mb__before_atomic();
159 v = (int)atomic_dec_uint_nv(&atomic->a_u.au_uint);
160 smp_mb__after_atomic();
161
162 return v;
163 }
164
165 static inline int
atomic_dec_and_test(atomic_t * atomic)166 atomic_dec_and_test(atomic_t *atomic)
167 {
168 /* membar implied by atomic_dec_return */
169 return atomic_dec_return(atomic) == 0;
170 }
171
172 static inline int
atomic_dec_if_positive(atomic_t * atomic)173 atomic_dec_if_positive(atomic_t *atomic)
174 {
175 int v;
176
177 smp_mb__before_atomic();
178 do {
179 v = atomic->a_u.au_uint;
180 if (v <= 0)
181 break;
182 } while (atomic_cas_uint(&atomic->a_u.au_uint, v, v - 1) != v);
183 smp_mb__after_atomic();
184
185 return v - 1;
186 }
187
188 static inline void
atomic_or(int value,atomic_t * atomic)189 atomic_or(int value, atomic_t *atomic)
190 {
191 /* no membar */
192 atomic_or_uint(&atomic->a_u.au_uint, value);
193 }
194
195 static inline void
atomic_and(int value,atomic_t * atomic)196 atomic_and(int value, atomic_t *atomic)
197 {
198 /* no membar */
199 atomic_and_uint(&atomic->a_u.au_uint, value);
200 }
201
202 static inline void
atomic_andnot(int value,atomic_t * atomic)203 atomic_andnot(int value, atomic_t *atomic)
204 {
205 /* no membar */
206 atomic_and_uint(&atomic->a_u.au_uint, ~value);
207 }
208
209 static inline int
atomic_fetch_add(int value,atomic_t * atomic)210 atomic_fetch_add(int value, atomic_t *atomic)
211 {
212 unsigned old, new;
213
214 smp_mb__before_atomic();
215 do {
216 old = atomic->a_u.au_uint;
217 new = old + value;
218 } while (atomic_cas_uint(&atomic->a_u.au_uint, old, new) != old);
219 smp_mb__after_atomic();
220
221 return old;
222 }
223
224 static inline int
atomic_fetch_inc(atomic_t * atomic)225 atomic_fetch_inc(atomic_t *atomic)
226 {
227 return atomic_fetch_add(1, atomic);
228 }
229
230 static inline int
atomic_fetch_xor(int value,atomic_t * atomic)231 atomic_fetch_xor(int value, atomic_t *atomic)
232 {
233 unsigned old, new;
234
235 smp_mb__before_atomic();
236 do {
237 old = atomic->a_u.au_uint;
238 new = old ^ value;
239 } while (atomic_cas_uint(&atomic->a_u.au_uint, old, new) != old);
240 smp_mb__after_atomic();
241
242 return old;
243 }
244
245 static inline void
atomic_set_mask(unsigned long mask,atomic_t * atomic)246 atomic_set_mask(unsigned long mask, atomic_t *atomic)
247 {
248 /* no membar */
249 atomic_or_uint(&atomic->a_u.au_uint, mask);
250 }
251
252 static inline void
atomic_clear_mask(unsigned long mask,atomic_t * atomic)253 atomic_clear_mask(unsigned long mask, atomic_t *atomic)
254 {
255 /* no membar */
256 atomic_and_uint(&atomic->a_u.au_uint, ~mask);
257 }
258
259 static inline int
atomic_add_unless(atomic_t * atomic,int addend,int zero)260 atomic_add_unless(atomic_t *atomic, int addend, int zero)
261 {
262 int value;
263
264 smp_mb__before_atomic();
265 do {
266 value = atomic->a_u.au_int;
267 if (value == zero)
268 break;
269 } while (atomic_cas_uint(&atomic->a_u.au_uint, value, (value + addend))
270 != (unsigned)value);
271 smp_mb__after_atomic();
272
273 return value != zero;
274 }
275
276 static inline int
atomic_inc_not_zero(atomic_t * atomic)277 atomic_inc_not_zero(atomic_t *atomic)
278 {
279 /* membar implied by atomic_add_unless */
280 return atomic_add_unless(atomic, 1, 0);
281 }
282
283 static inline int
atomic_xchg(atomic_t * atomic,int new)284 atomic_xchg(atomic_t *atomic, int new)
285 {
286 int old;
287
288 smp_mb__before_atomic();
289 old = (int)atomic_swap_uint(&atomic->a_u.au_uint, (unsigned)new);
290 smp_mb__after_atomic();
291
292 return old;
293 }
294
295 static inline int
atomic_cmpxchg(atomic_t * atomic,int expect,int new)296 atomic_cmpxchg(atomic_t *atomic, int expect, int new)
297 {
298 int old;
299
300 /*
301 * XXX As an optimization, under Linux's semantics we are
302 * allowed to skip the memory barrier if the comparison fails,
303 * but taking advantage of that is not convenient here.
304 */
305 smp_mb__before_atomic();
306 old = (int)atomic_cas_uint(&atomic->a_u.au_uint, (unsigned)expect,
307 (unsigned)new);
308 smp_mb__after_atomic();
309
310 return old;
311 }
312
313 static inline bool
atomic_try_cmpxchg(atomic_t * atomic,int * valuep,int new)314 atomic_try_cmpxchg(atomic_t *atomic, int *valuep, int new)
315 {
316 int expect = *valuep;
317
318 *valuep = atomic_cmpxchg(atomic, expect, new);
319
320 return *valuep == expect;
321 }
322
323 struct atomic64 {
324 volatile uint64_t a_v;
325 };
326
327 typedef struct atomic64 atomic64_t;
328
329 #define ATOMIC64_INIT(v) { .a_v = (v) }
330
331 int linux_atomic64_init(void);
332 void linux_atomic64_fini(void);
333
334 #ifdef __HAVE_ATOMIC64_OPS
335
336 static inline uint64_t
atomic64_read(const struct atomic64 * a)337 atomic64_read(const struct atomic64 *a)
338 {
339 /* no membar */
340 return a->a_v;
341 }
342
343 static inline void
atomic64_set(struct atomic64 * a,uint64_t v)344 atomic64_set(struct atomic64 *a, uint64_t v)
345 {
346 /* no membar */
347 a->a_v = v;
348 }
349
350 static inline void
atomic64_add(int64_t d,struct atomic64 * a)351 atomic64_add(int64_t d, struct atomic64 *a)
352 {
353 /* no membar */
354 atomic_add_64(&a->a_v, d);
355 }
356
357 static inline void
atomic64_sub(int64_t d,struct atomic64 * a)358 atomic64_sub(int64_t d, struct atomic64 *a)
359 {
360 /* no membar */
361 atomic_add_64(&a->a_v, -d);
362 }
363
364 static inline int64_t
atomic64_add_return(int64_t d,struct atomic64 * a)365 atomic64_add_return(int64_t d, struct atomic64 *a)
366 {
367 int64_t v;
368
369 smp_mb__before_atomic();
370 v = (int64_t)atomic_add_64_nv(&a->a_v, d);
371 smp_mb__after_atomic();
372
373 return v;
374 }
375
376 static inline uint64_t
atomic64_xchg(struct atomic64 * a,uint64_t new)377 atomic64_xchg(struct atomic64 *a, uint64_t new)
378 {
379 uint64_t old;
380
381 smp_mb__before_atomic();
382 old = atomic_swap_64(&a->a_v, new);
383 smp_mb__after_atomic();
384
385 return old;
386 }
387
388 static inline uint64_t
atomic64_cmpxchg(struct atomic64 * atomic,uint64_t expect,uint64_t new)389 atomic64_cmpxchg(struct atomic64 *atomic, uint64_t expect, uint64_t new)
390 {
391 uint64_t old;
392
393 /*
394 * XXX As an optimization, under Linux's semantics we are
395 * allowed to skip the memory barrier if the comparison fails,
396 * but taking advantage of that is not convenient here.
397 */
398 smp_mb__before_atomic();
399 old = atomic_cas_64(&atomic->a_v, expect, new);
400 smp_mb__after_atomic();
401
402 return old;
403 }
404
405 #else /* !defined(__HAVE_ATOMIC64_OPS) */
406
407 #define atomic64_add linux_atomic64_add
408 #define atomic64_add_return linux_atomic64_add_return
409 #define atomic64_cmpxchg linux_atomic64_cmpxchg
410 #define atomic64_read linux_atomic64_read
411 #define atomic64_set linux_atomic64_set
412 #define atomic64_sub linux_atomic64_sub
413 #define atomic64_xchg linux_atomic64_xchg
414
415 uint64_t atomic64_read(const struct atomic64 *);
416 void atomic64_set(struct atomic64 *, uint64_t);
417 void atomic64_add(int64_t, struct atomic64 *);
418 void atomic64_sub(int64_t, struct atomic64 *);
419 int64_t atomic64_add_return(int64_t, struct atomic64 *);
420 uint64_t atomic64_xchg(struct atomic64 *, uint64_t);
421 uint64_t atomic64_cmpxchg(struct atomic64 *, uint64_t, uint64_t);
422
423 #endif
424
425 static inline void
atomic64_inc(struct atomic64 * a)426 atomic64_inc(struct atomic64 *a)
427 {
428 atomic64_add(1, a);
429 }
430
431 static inline int64_t
atomic64_inc_return(struct atomic64 * a)432 atomic64_inc_return(struct atomic64 *a)
433 {
434 return atomic64_add_return(1, a);
435 }
436
437 struct atomic_long {
438 volatile unsigned long al_v;
439 };
440
441 typedef struct atomic_long atomic_long_t;
442
443 static inline long
atomic_long_read(struct atomic_long * a)444 atomic_long_read(struct atomic_long *a)
445 {
446 /* no membar */
447 return (unsigned long)a->al_v;
448 }
449
450 static inline void
atomic_long_set(struct atomic_long * a,long v)451 atomic_long_set(struct atomic_long *a, long v)
452 {
453 /* no membar */
454 a->al_v = v;
455 }
456
457 static inline long
atomic_long_add_unless(struct atomic_long * a,long addend,long zero)458 atomic_long_add_unless(struct atomic_long *a, long addend, long zero)
459 {
460 long value;
461
462 smp_mb__before_atomic();
463 do {
464 value = (long)a->al_v;
465 if (value == zero)
466 break;
467 } while (atomic_cas_ulong(&a->al_v, (unsigned long)value,
468 (unsigned long)(value + addend)) != (unsigned long)value);
469 smp_mb__after_atomic();
470
471 return value != zero;
472 }
473
474 static inline long
atomic_long_inc_not_zero(struct atomic_long * a)475 atomic_long_inc_not_zero(struct atomic_long *a)
476 {
477 /* membar implied by atomic_long_add_unless */
478 return atomic_long_add_unless(a, 1, 0);
479 }
480
481 static inline long
atomic_long_xchg(struct atomic_long * a,long new)482 atomic_long_xchg(struct atomic_long *a, long new)
483 {
484 long old;
485
486 smp_mb__before_atomic();
487 old = (long)atomic_swap_ulong(&a->al_v, (unsigned long)new);
488 smp_mb__after_atomic();
489
490 return old;
491 }
492
493 static inline long
atomic_long_cmpxchg(struct atomic_long * a,long expect,long new)494 atomic_long_cmpxchg(struct atomic_long *a, long expect, long new)
495 {
496 long old;
497
498 /*
499 * XXX As an optimization, under Linux's semantics we are
500 * allowed to skip the memory barrier if the comparison fails,
501 * but taking advantage of that is not convenient here.
502 */
503 smp_mb__before_atomic();
504 old = (long)atomic_cas_ulong(&a->al_v, (unsigned long)expect,
505 (unsigned long)new);
506 smp_mb__after_atomic();
507
508 return old;
509 }
510
511 #endif /* _LINUX_ATOMIC_H_ */
512