1 /* $OpenBSD: atomic.h,v 1.20 2022/08/29 02:01:18 jsg Exp $ */
2 /* $NetBSD: atomic.h,v 1.1.2.2 2000/02/21 18:54:07 sommerfeld Exp $ */
3
4 /*-
5 * Copyright (c) 2000 The NetBSD Foundation, Inc.
6 * All rights reserved.
7 *
8 * This code is derived from software contributed to The NetBSD Foundation
9 * by RedBack Networks Inc.
10 *
11 * Author: Bill Sommerfeld
12 *
13 * Redistribution and use in source and binary forms, with or without
14 * modification, are permitted provided that the following conditions
15 * are met:
16 * 1. Redistributions of source code must retain the above copyright
17 * notice, this list of conditions and the following disclaimer.
18 * 2. Redistributions in binary form must reproduce the above copyright
19 * notice, this list of conditions and the following disclaimer in the
20 * documentation and/or other materials provided with the distribution.
21 *
22 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
23 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
24 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
25 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
26 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
27 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
28 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
29 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
30 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
31 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
32 * POSSIBILITY OF SUCH DAMAGE.
33 */
34
35 #ifndef _MACHINE_ATOMIC_H_
36 #define _MACHINE_ATOMIC_H_
37
38 /*
39 * Perform atomic operations on memory. Should be atomic with respect
40 * to interrupts and multiple processors.
41 *
42 * void atomic_setbits_int(volatile u_int *a, u_int mask) { *a |= mask; }
43 * void atomic_clearbits_int(volatile u_int *a, u_int mas) { *a &= ~mask; }
44 */
45 #if !defined(_LOCORE)
46
47 #if defined(MULTIPROCESSOR) || !defined(_KERNEL)
48 #define _LOCK "lock"
49 #else
50 #define _LOCK
51 #endif
52
53 static inline unsigned int
_atomic_cas_uint(volatile unsigned int * p,unsigned int e,unsigned int n)54 _atomic_cas_uint(volatile unsigned int *p, unsigned int e, unsigned int n)
55 {
56 __asm volatile(_LOCK " cmpxchgl %2, %1"
57 : "=a" (n), "=m" (*p)
58 : "r" (n), "a" (e), "m" (*p));
59
60 return (n);
61 }
62 #define atomic_cas_uint(_p, _e, _n) _atomic_cas_uint((_p), (_e), (_n))
63
64 static inline unsigned long
_atomic_cas_ulong(volatile unsigned long * p,unsigned long e,unsigned long n)65 _atomic_cas_ulong(volatile unsigned long *p, unsigned long e, unsigned long n)
66 {
67 __asm volatile(_LOCK " cmpxchgl %2, %1"
68 : "=a" (n), "=m" (*p)
69 : "r" (n), "a" (e), "m" (*p));
70
71 return (n);
72 }
73 #define atomic_cas_ulong(_p, _e, _n) _atomic_cas_ulong((_p), (_e), (_n))
74
75 static inline void *
_atomic_cas_ptr(volatile void * p,void * e,void * n)76 _atomic_cas_ptr(volatile void *p, void *e, void *n)
77 {
78 __asm volatile(_LOCK " cmpxchgl %2, %1"
79 : "=a" (n), "=m" (*(unsigned long *)p)
80 : "r" (n), "a" (e), "m" (*(unsigned long *)p));
81
82 return (n);
83 }
84 #define atomic_cas_ptr(_p, _e, _n) _atomic_cas_ptr((_p), (_e), (_n))
85
86 static inline unsigned int
_atomic_swap_uint(volatile unsigned int * p,unsigned int n)87 _atomic_swap_uint(volatile unsigned int *p, unsigned int n)
88 {
89 __asm volatile("xchgl %0, %1"
90 : "=a" (n), "=m" (*p)
91 : "0" (n), "m" (*p));
92
93 return (n);
94 }
95 #define atomic_swap_uint(_p, _n) _atomic_swap_uint((_p), (_n))
96 #define atomic_swap_32(_p, _n) _atomic_swap_uint((_p), (_n))
97
98 static inline unsigned long
_atomic_swap_ulong(volatile unsigned long * p,unsigned long n)99 _atomic_swap_ulong(volatile unsigned long *p, unsigned long n)
100 {
101 __asm volatile("xchgl %0, %1"
102 : "=a" (n), "=m" (*p)
103 : "0" (n), "m" (*p));
104
105 return (n);
106 }
107 #define atomic_swap_ulong(_p, _n) _atomic_swap_ulong((_p), (_n))
108
109 static inline void *
_atomic_swap_ptr(volatile void * p,void * n)110 _atomic_swap_ptr(volatile void *p, void *n)
111 {
112 __asm volatile("xchgl %0, %1"
113 : "=a" (n), "=m" (*(unsigned long *)p)
114 : "0" (n), "m" (*(unsigned long *)p));
115
116 return (n);
117 }
118 #define atomic_swap_ptr(_p, _n) _atomic_swap_ptr((_p), (_n))
119
120 static inline void
_atomic_inc_int(volatile unsigned int * p)121 _atomic_inc_int(volatile unsigned int *p)
122 {
123 __asm volatile(_LOCK " incl %0"
124 : "+m" (*p));
125 }
126 #define atomic_inc_int(_p) _atomic_inc_int(_p)
127
128 static inline void
_atomic_inc_long(volatile unsigned long * p)129 _atomic_inc_long(volatile unsigned long *p)
130 {
131 __asm volatile(_LOCK " incl %0"
132 : "+m" (*p));
133 }
134 #define atomic_inc_long(_p) _atomic_inc_long(_p)
135
136 static inline void
_atomic_dec_int(volatile unsigned int * p)137 _atomic_dec_int(volatile unsigned int *p)
138 {
139 __asm volatile(_LOCK " decl %0"
140 : "+m" (*p));
141 }
142 #define atomic_dec_int(_p) _atomic_dec_int(_p)
143
144 static inline void
_atomic_dec_long(volatile unsigned long * p)145 _atomic_dec_long(volatile unsigned long *p)
146 {
147 __asm volatile(_LOCK " decl %0"
148 : "+m" (*p));
149 }
150 #define atomic_dec_long(_p) _atomic_dec_long(_p)
151
152 static inline void
_atomic_add_int(volatile unsigned int * p,unsigned int v)153 _atomic_add_int(volatile unsigned int *p, unsigned int v)
154 {
155 __asm volatile(_LOCK " addl %1,%0"
156 : "+m" (*p)
157 : "a" (v));
158 }
159 #define atomic_add_int(_p, _v) _atomic_add_int(_p, _v)
160
161 static inline void
_atomic_add_long(volatile unsigned long * p,unsigned long v)162 _atomic_add_long(volatile unsigned long *p, unsigned long v)
163 {
164 __asm volatile(_LOCK " addl %1,%0"
165 : "+m" (*p)
166 : "a" (v));
167 }
168 #define atomic_add_long(_p, _v) _atomic_add_long(_p, _v)
169
170 static inline void
_atomic_sub_int(volatile unsigned int * p,unsigned int v)171 _atomic_sub_int(volatile unsigned int *p, unsigned int v)
172 {
173 __asm volatile(_LOCK " subl %1,%0"
174 : "+m" (*p)
175 : "a" (v));
176 }
177 #define atomic_sub_int(_p, _v) _atomic_sub_int(_p, _v)
178
179 static inline void
_atomic_sub_long(volatile unsigned long * p,unsigned long v)180 _atomic_sub_long(volatile unsigned long *p, unsigned long v)
181 {
182 __asm volatile(_LOCK " subl %1,%0"
183 : "+m" (*p)
184 : "a" (v));
185 }
186 #define atomic_sub_long(_p, _v) _atomic_sub_long(_p, _v)
187
188
189 static inline unsigned long
_atomic_add_int_nv(volatile unsigned int * p,unsigned int v)190 _atomic_add_int_nv(volatile unsigned int *p, unsigned int v)
191 {
192 unsigned int rv = v;
193
194 __asm volatile(_LOCK " xaddl %0,%1"
195 : "+a" (rv), "+m" (*p));
196
197 return (rv + v);
198 }
199 #define atomic_add_int_nv(_p, _v) _atomic_add_int_nv(_p, _v)
200
201 static inline unsigned long
_atomic_add_long_nv(volatile unsigned long * p,unsigned long v)202 _atomic_add_long_nv(volatile unsigned long *p, unsigned long v)
203 {
204 unsigned long rv = v;
205
206 __asm volatile(_LOCK " xaddl %0,%1"
207 : "+a" (rv), "+m" (*p));
208
209 return (rv + v);
210 }
211 #define atomic_add_long_nv(_p, _v) _atomic_add_long_nv(_p, _v)
212
213 static inline unsigned long
_atomic_sub_int_nv(volatile unsigned int * p,unsigned int v)214 _atomic_sub_int_nv(volatile unsigned int *p, unsigned int v)
215 {
216 unsigned int rv = 0 - v;
217
218 __asm volatile(_LOCK " xaddl %0,%1"
219 : "+a" (rv), "+m" (*p));
220
221 return (rv - v);
222 }
223 #define atomic_sub_int_nv(_p, _v) _atomic_sub_int_nv(_p, _v)
224
225 static inline unsigned long
_atomic_sub_long_nv(volatile unsigned long * p,unsigned long v)226 _atomic_sub_long_nv(volatile unsigned long *p, unsigned long v)
227 {
228 unsigned long rv = 0 - v;
229
230 __asm volatile(_LOCK " xaddl %0,%1"
231 : "+a" (rv), "+m" (*p));
232
233 return (rv - v);
234 }
235 #define atomic_sub_long_nv(_p, _v) _atomic_sub_long_nv(_p, _v)
236
237 /*
238 * The IA-32 architecture is rather strongly ordered. When accessing
239 * normal write-back cacheable memory, only reads may be reordered with
240 * older writes to different locations. There are a few instructions
241 * (clfush, non-temporal move instructions) that obey weaker ordering
242 * rules, but those instructions will only be used in (inline)
243 * assembly code where we can add the necessary fence instructions
244 * ourselves.
245 */
246
247 #define __membar(_f) do { __asm volatile(_f ::: "memory"); } while (0)
248
249 #if defined(MULTIPROCESSOR) || !defined(_KERNEL)
250 #define membar_enter() __membar("lock; addl $0,0(%%esp)")
251 #define membar_exit() __membar("")
252 #define membar_producer() __membar("")
253 #define membar_consumer() __membar("")
254 #define membar_sync() __membar("lock; addl $0,0(%%esp)")
255 #else
256 #define membar_enter() __membar("")
257 #define membar_exit() __membar("")
258 #define membar_producer() __membar("")
259 #define membar_consumer() __membar("")
260 #define membar_sync() __membar("")
261 #endif
262
263 #define membar_enter_after_atomic() __membar("")
264 #define membar_exit_before_atomic() __membar("")
265
266 #ifdef _KERNEL
267
268 /* virtio needs MP membars even on SP kernels */
269 #define virtio_membar_producer() __membar("")
270 #define virtio_membar_consumer() __membar("")
271 #define virtio_membar_sync() __membar("lock; addl $0,0(%%esp)")
272
273 static __inline u_int64_t
i386_atomic_testset_uq(volatile u_int64_t * ptr,u_int64_t val)274 i386_atomic_testset_uq(volatile u_int64_t *ptr, u_int64_t val)
275 {
276 __asm__ volatile ("\n1:\t" _LOCK " cmpxchg8b (%1); jnz 1b" : "+A" (val) :
277 "r" (ptr), "b" ((u_int32_t)val), "c" ((u_int32_t)(val >> 32)));
278 return val;
279 }
280
281 static __inline u_int32_t
i386_atomic_testset_ul(volatile u_int32_t * ptr,unsigned long val)282 i386_atomic_testset_ul(volatile u_int32_t *ptr, unsigned long val)
283 {
284 __asm__ volatile ("xchgl %0,(%2)" :"=r" (val):"0" (val),"r" (ptr));
285 return val;
286 }
287
288 static __inline int
i386_atomic_testset_i(volatile int * ptr,unsigned long val)289 i386_atomic_testset_i(volatile int *ptr, unsigned long val)
290 {
291 __asm__ volatile ("xchgl %0,(%2)" :"=r" (val):"0" (val),"r" (ptr));
292 return val;
293 }
294
295 static __inline void
i386_atomic_setbits_l(volatile u_int32_t * ptr,unsigned long bits)296 i386_atomic_setbits_l(volatile u_int32_t *ptr, unsigned long bits)
297 {
298 __asm volatile(_LOCK " orl %1,%0" : "=m" (*ptr) : "ir" (bits));
299 }
300
301 static __inline void
i386_atomic_clearbits_l(volatile u_int32_t * ptr,unsigned long bits)302 i386_atomic_clearbits_l(volatile u_int32_t *ptr, unsigned long bits)
303 {
304 bits = ~bits;
305 __asm volatile(_LOCK " andl %1,%0" : "=m" (*ptr) : "ir" (bits));
306 }
307
308 #define atomic_setbits_int i386_atomic_setbits_l
309 #define atomic_clearbits_int i386_atomic_clearbits_l
310
311 #endif /* _KERNEL */
312
313 #undef _LOCK
314
315 #endif /* !defined(_LOCORE) */
316 #endif /* _MACHINE_ATOMIC_H_ */
317