1 /* $OpenBSD: atomic.h,v 1.15 2015/05/30 08:41:30 kettenis Exp $ */ 2 /* $NetBSD: atomic.h,v 1.1.2.2 2000/02/21 18:54:07 sommerfeld Exp $ */ 3 4 /*- 5 * Copyright (c) 2000 The NetBSD Foundation, Inc. 6 * All rights reserved. 7 * 8 * This code is derived from software contributed to The NetBSD Foundation 9 * by RedBack Networks Inc. 10 * 11 * Author: Bill Sommerfeld 12 * 13 * Redistribution and use in source and binary forms, with or without 14 * modification, are permitted provided that the following conditions 15 * are met: 16 * 1. Redistributions of source code must retain the above copyright 17 * notice, this list of conditions and the following disclaimer. 18 * 2. Redistributions in binary form must reproduce the above copyright 19 * notice, this list of conditions and the following disclaimer in the 20 * documentation and/or other materials provided with the distribution. 21 * 22 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 23 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 24 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 25 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 26 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 27 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 28 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 29 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 30 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 31 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 32 * POSSIBILITY OF SUCH DAMAGE. 33 */ 34 35 #ifndef _MACHINE_ATOMIC_H_ 36 #define _MACHINE_ATOMIC_H_ 37 38 /* 39 * Perform atomic operations on memory. Should be atomic with respect 40 * to interrupts and multiple processors. 41 * 42 * void atomic_setbits_int(volatile u_int *a, u_int mask) { *a |= mask; } 43 * void atomic_clearbits_int(volatile u_int *a, u_int mas) { *a &= ~mask; } 44 */ 45 #if defined(_KERNEL) && !defined(_LOCORE) 46 47 #ifdef MULTIPROCESSOR 48 #define LOCK "lock" 49 #else 50 #define LOCK 51 #endif 52 53 static inline unsigned int 54 _atomic_cas_uint(volatile unsigned int *p, unsigned int e, unsigned int n) 55 { 56 __asm volatile(LOCK " cmpxchgl %2, %1" 57 : "=a" (n), "=m" (*p) 58 : "r" (n), "a" (e), "m" (*p)); 59 60 return (n); 61 } 62 #define atomic_cas_uint(_p, _e, _n) _atomic_cas_uint((_p), (_e), (_n)) 63 64 static inline unsigned long 65 _atomic_cas_ulong(volatile unsigned long *p, unsigned long e, unsigned long n) 66 { 67 __asm volatile(LOCK " cmpxchgl %2, %1" 68 : "=a" (n), "=m" (*p) 69 : "r" (n), "a" (e), "m" (*p)); 70 71 return (n); 72 } 73 #define atomic_cas_ulong(_p, _e, _n) _atomic_cas_ulong((_p), (_e), (_n)) 74 75 static inline void * 76 _atomic_cas_ptr(volatile void *p, void *e, void *n) 77 { 78 __asm volatile(LOCK " cmpxchgl %2, %1" 79 : "=a" (n), "=m" (*(unsigned long *)p) 80 : "r" (n), "a" (e), "m" (*(unsigned long *)p)); 81 82 return (n); 83 } 84 #define atomic_cas_ptr(_p, _e, _n) _atomic_cas_ptr((_p), (_e), (_n)) 85 86 static inline unsigned int 87 _atomic_swap_uint(volatile unsigned int *p, unsigned int n) 88 { 89 __asm volatile("xchgl %0, %1" 90 : "=a" (n), "=m" (*p) 91 : "0" (n), "m" (*p)); 92 93 return (n); 94 } 95 #define atomic_swap_uint(_p, _n) _atomic_swap_uint((_p), (_n)) 96 #define atomic_swap_32(_p, _n) _atomic_swap_uint((_p), (_n)) 97 98 static inline unsigned long 99 _atomic_swap_ulong(volatile unsigned long *p, unsigned long n) 100 { 101 __asm volatile("xchgl %0, %1" 102 : "=a" (n), "=m" (*p) 103 : "0" (n), "m" (*p)); 104 105 return (n); 106 } 107 #define atomic_swap_ulong(_p, _n) _atomic_swap_ulong((_p), (_n)) 108 109 static inline uint64_t 110 _atomic_swap_64(volatile uint64_t *p, uint64_t n) 111 { 112 __asm volatile("xchgl %0, %1" 113 : "=a" (n), "=m" (*p) 114 : "0" (n), "m" (*p)); 115 116 return (n); 117 } 118 #define atomic_swap_64(_p, _n) _atomic_swap_64((_p), (_n)) 119 120 static inline void * 121 _atomic_swap_ptr(volatile void *p, void *n) 122 { 123 __asm volatile("xchgl %0, %1" 124 : "=a" (n), "=m" (*(unsigned long *)p) 125 : "0" (n), "m" (*(unsigned long *)p)); 126 127 return (n); 128 } 129 #define atomic_swap_ptr(_p, _n) _atomic_swap_ptr((_p), (_n)) 130 131 static inline void 132 _atomic_inc_int(volatile unsigned int *p) 133 { 134 __asm volatile(LOCK " incl %0" 135 : "+m" (*p)); 136 } 137 #define atomic_inc_int(_p) _atomic_inc_int(_p) 138 139 static inline void 140 _atomic_inc_long(volatile unsigned long *p) 141 { 142 __asm volatile(LOCK " incl %0" 143 : "+m" (*p)); 144 } 145 #define atomic_inc_long(_p) _atomic_inc_long(_p) 146 147 static inline void 148 _atomic_dec_int(volatile unsigned int *p) 149 { 150 __asm volatile(LOCK " decl %0" 151 : "+m" (*p)); 152 } 153 #define atomic_dec_int(_p) _atomic_dec_int(_p) 154 155 static inline void 156 _atomic_dec_long(volatile unsigned long *p) 157 { 158 __asm volatile(LOCK " decl %0" 159 : "+m" (*p)); 160 } 161 #define atomic_dec_long(_p) _atomic_dec_long(_p) 162 163 static inline void 164 _atomic_add_int(volatile unsigned int *p, unsigned int v) 165 { 166 __asm volatile(LOCK " addl %1,%0" 167 : "+m" (*p) 168 : "a" (v)); 169 } 170 #define atomic_add_int(_p, _v) _atomic_add_int(_p, _v) 171 172 static inline void 173 _atomic_add_long(volatile unsigned long *p, unsigned long v) 174 { 175 __asm volatile(LOCK " addl %1,%0" 176 : "+m" (*p) 177 : "a" (v)); 178 } 179 #define atomic_add_long(_p, _v) _atomic_add_long(_p, _v) 180 181 static inline void 182 _atomic_sub_int(volatile unsigned int *p, unsigned int v) 183 { 184 __asm volatile(LOCK " subl %1,%0" 185 : "+m" (*p) 186 : "a" (v)); 187 } 188 #define atomic_sub_int(_p, _v) _atomic_sub_int(_p, _v) 189 190 static inline void 191 _atomic_sub_long(volatile unsigned long *p, unsigned long v) 192 { 193 __asm volatile(LOCK " subl %1,%0" 194 : "+m" (*p) 195 : "a" (v)); 196 } 197 #define atomic_sub_long(_p, _v) _atomic_sub_long(_p, _v) 198 199 200 static inline unsigned long 201 _atomic_add_int_nv(volatile unsigned int *p, unsigned int v) 202 { 203 unsigned int rv = v; 204 205 __asm volatile(LOCK " xaddl %0,%1" 206 : "+a" (rv), "+m" (*p)); 207 208 return (rv + v); 209 } 210 #define atomic_add_int_nv(_p, _v) _atomic_add_int_nv(_p, _v) 211 212 static inline unsigned long 213 _atomic_add_long_nv(volatile unsigned long *p, unsigned long v) 214 { 215 unsigned long rv = v; 216 217 __asm volatile(LOCK " xaddl %0,%1" 218 : "+a" (rv), "+m" (*p)); 219 220 return (rv + v); 221 } 222 #define atomic_add_long_nv(_p, _v) _atomic_add_long_nv(_p, _v) 223 224 static inline unsigned long 225 _atomic_sub_int_nv(volatile unsigned int *p, unsigned int v) 226 { 227 unsigned int rv = 0 - v; 228 229 __asm volatile(LOCK " xaddl %0,%1" 230 : "+a" (rv), "+m" (*p)); 231 232 return (rv - v); 233 } 234 #define atomic_sub_int_nv(_p, _v) _atomic_sub_int_nv(_p, _v) 235 236 static inline unsigned long 237 _atomic_sub_long_nv(volatile unsigned long *p, unsigned long v) 238 { 239 unsigned long rv = 0 - v; 240 241 __asm volatile(LOCK " xaddl %0,%1" 242 : "+a" (rv), "+m" (*p)); 243 244 return (rv - v); 245 } 246 #define atomic_sub_long_nv(_p, _v) _atomic_sub_long_nv(_p, _v) 247 248 /* 249 * The IA-32 architecture is rather strongly ordered. When accessing 250 * normal write-back cachable memory, only reads may be reordered with 251 * older writes to different locations. There are a few instructions 252 * (clfush, non-temporal move instructions) that obey weaker ordering 253 * rules, but those instructions will only be used in (inline) 254 * assembly code where we can add the necessary fence instructions 255 * ourselves. 256 */ 257 258 #define __membar(_f) do { __asm __volatile(_f ::: "memory"); } while (0) 259 260 #ifdef MULTIPROCESSOR 261 #define membar_enter() __membar("lock; addl $0,0(%%esp)") 262 #define membar_exit() __membar("") 263 #define membar_producer() __membar("") 264 #define membar_consumer() __membar("") 265 #define membar_sync() __membar("lock; addl $0,0(%%esp)") 266 #else 267 #define membar_enter() __membar("") 268 #define membar_exit() __membar("") 269 #define membar_producer() __membar("") 270 #define membar_consumer() __membar("") 271 #define membar_sync() __membar("") 272 #endif 273 274 /* virtio needs MP membars even on SP kernels */ 275 #define virtio_membar_producer() __membar("") 276 #define virtio_membar_consumer() __membar("") 277 #define virtio_membar_sync() __membar("lock; addl $0,0(%%esp)") 278 279 static __inline u_int64_t 280 i386_atomic_testset_uq(volatile u_int64_t *ptr, u_int64_t val) 281 { 282 __asm__ volatile ("\n1:\t" LOCK " cmpxchg8b (%1); jnz 1b" : "+A" (val) : 283 "r" (ptr), "b" ((u_int32_t)val), "c" ((u_int32_t)(val >> 32))); 284 return val; 285 } 286 287 static __inline u_int32_t 288 i386_atomic_testset_ul(volatile u_int32_t *ptr, unsigned long val) 289 { 290 __asm__ volatile ("xchgl %0,(%2)" :"=r" (val):"0" (val),"r" (ptr)); 291 return val; 292 } 293 294 static __inline int 295 i386_atomic_testset_i(volatile int *ptr, unsigned long val) 296 { 297 __asm__ volatile ("xchgl %0,(%2)" :"=r" (val):"0" (val),"r" (ptr)); 298 return val; 299 } 300 301 static __inline void 302 i386_atomic_setbits_l(volatile u_int32_t *ptr, unsigned long bits) 303 { 304 __asm volatile(LOCK " orl %1,%0" : "=m" (*ptr) : "ir" (bits)); 305 } 306 307 static __inline void 308 i386_atomic_clearbits_l(volatile u_int32_t *ptr, unsigned long bits) 309 { 310 bits = ~bits; 311 __asm volatile(LOCK " andl %1,%0" : "=m" (*ptr) : "ir" (bits)); 312 } 313 314 int ucas_32(volatile int32_t *, int32_t, int32_t); 315 #define futex_atomic_ucas_int32 ucas_32 316 317 #define atomic_setbits_int i386_atomic_setbits_l 318 #define atomic_clearbits_int i386_atomic_clearbits_l 319 320 #undef LOCK 321 322 #endif /* defined(_KERNEL) && !defined(_LOCORE) */ 323 #endif /* _MACHINE_ATOMIC_H_ */ 324