1 /* $OpenBSD: atomic.h,v 1.24 2025/01/19 11:15:52 jsg Exp $ */ 2 /** 3 * \file drm_atomic.h 4 * Atomic operations used in the DRM which may or may not be provided by the OS. 5 * 6 * \author Eric Anholt <anholt@FreeBSD.org> 7 */ 8 9 /*- 10 * Copyright 2004 Eric Anholt 11 * All Rights Reserved. 12 * 13 * Permission is hereby granted, free of charge, to any person obtaining a 14 * copy of this software and associated documentation files (the "Software"), 15 * to deal in the Software without restriction, including without limitation 16 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 17 * and/or sell copies of the Software, and to permit persons to whom the 18 * Software is furnished to do so, subject to the following conditions: 19 * 20 * The above copyright notice and this permission notice (including the next 21 * paragraph) shall be included in all copies or substantial portions of the 22 * Software. 23 * 24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 25 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 26 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 27 * VA LINUX SYSTEMS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR 28 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 29 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 30 * OTHER DEALINGS IN THE SOFTWARE. 31 */ 32 33 #ifndef _DRM_LINUX_ATOMIC_H_ 34 #define _DRM_LINUX_ATOMIC_H_ 35 36 #include <sys/types.h> 37 #include <sys/mutex.h> 38 #include <machine/intr.h> 39 #include <linux/types.h> 40 #include <linux/compiler.h> /* via x86/include/asm/atomic.h */ 41 42 #define ATOMIC_INIT(x) (x) 43 44 #define atomic_set(p, v) WRITE_ONCE(*(p), (v)) 45 #define atomic_read(p) READ_ONCE(*(p)) 46 #define atomic_inc(p) __sync_fetch_and_add(p, 1) 47 #define atomic_dec(p) __sync_fetch_and_sub(p, 1) 48 #define atomic_add(n, p) __sync_fetch_and_add(p, n) 49 #define atomic_sub(n, p) __sync_fetch_and_sub(p, n) 50 #define atomic_and(n, p) __sync_fetch_and_and(p, n) 51 #define atomic_or(n, p) atomic_setbits_int(p, n) 52 #define atomic_add_return(n, p) __sync_add_and_fetch(p, n) 53 #define atomic_sub_return(n, p) __sync_sub_and_fetch(p, n) 54 #define atomic_sub_and_test(n, p) (atomic_sub_return(n, p) == 0) 55 #define atomic_inc_return(v) atomic_add_return(1, (v)) 56 #define atomic_dec_return(v) atomic_sub_return(1, (v)) 57 #define atomic_dec_and_test(v) (atomic_dec_return(v) == 0) 58 #define atomic_inc_and_test(v) (atomic_inc_return(v) == 0) 59 #define atomic_cmpxchg(p, o, n) __sync_val_compare_and_swap(p, o, n) 60 #define cmpxchg(p, o, n) __sync_val_compare_and_swap(p, o, n) 61 #define cmpxchg64(p, o, n) __sync_val_compare_and_swap(p, o, n) 62 #define atomic_set_release(p, v) atomic_set((p), (v)) 63 #define atomic_andnot(bits, p) atomic_clearbits_int(p,bits) 64 #define atomic_fetch_inc(p) __sync_fetch_and_add(p, 1) 65 #define atomic_fetch_xor(n, p) __sync_fetch_and_xor(p, n) 66 67 #define try_cmpxchg(p, op, n) \ 68 ({ \ 69 __typeof(p) __op = (__typeof((p)))(op); \ 70 __typeof(*(p)) __o = *__op; \ 71 __typeof(*(p)) __p = __sync_val_compare_and_swap((p), (__o), (n)); \ 72 if (__p != __o) \ 73 *__op = __p; \ 74 (__p == __o); \ 75 }) 76 77 static inline bool 78 atomic_try_cmpxchg(volatile int *p, int *op, int n) 79 { 80 return try_cmpxchg(p, op, n); 81 } 82 83 static inline int 84 atomic_xchg(volatile int *v, int n) 85 { 86 __sync_synchronize(); 87 return __sync_lock_test_and_set(v, n); 88 } 89 90 #define xchg(v, n) __sync_lock_test_and_set(v, n) 91 92 static inline int 93 atomic_add_unless(volatile int *v, int n, int u) 94 { 95 int o; 96 97 do { 98 o = *v; 99 if (o == u) 100 return 0; 101 } while (__sync_val_compare_and_swap(v, o, o +n) != o); 102 103 return 1; 104 } 105 106 #define atomic_inc_not_zero(v) atomic_add_unless((v), 1, 0) 107 108 static inline int 109 atomic_dec_if_positive(volatile int *v) 110 { 111 int r, o; 112 113 do { 114 o = *v; 115 r = o - 1; 116 if (r < 0) 117 break; 118 } while (__sync_val_compare_and_swap(v, o, r) != o); 119 120 return r; 121 } 122 123 #define atomic_long_read(p) READ_ONCE(*(p)) 124 125 /* 32 bit powerpc lacks 64 bit atomics */ 126 #if !defined(__powerpc__) || defined(__powerpc64__) 127 128 typedef int64_t atomic64_t __aligned(8); 129 130 #define ATOMIC64_INIT(x) (x) 131 132 #define atomic64_set(p, v) WRITE_ONCE(*(p), (v)) 133 #define atomic64_read(p) READ_ONCE(*(p)) 134 135 static inline int64_t 136 atomic64_xchg(atomic64_t *v, int64_t n) 137 { 138 __sync_synchronize(); 139 return __sync_lock_test_and_set(v, n); 140 } 141 142 static inline int64_t 143 atomic64_cmpxchg(atomic64_t *v, int64_t o, int64_t n) 144 { 145 return __sync_val_compare_and_swap(v, o, n); 146 } 147 148 #define atomic64_add(n, p) __sync_fetch_and_add_8(p, n) 149 #define atomic64_sub(n, p) __sync_fetch_and_sub_8(p, n) 150 #define atomic64_inc(p) __sync_fetch_and_add_8(p, 1) 151 #define atomic64_add_return(n, p) __sync_add_and_fetch_8(p, n) 152 #define atomic64_inc_return(p) __sync_add_and_fetch_8(p, 1) 153 154 #else 155 156 extern struct mutex atomic64_mtx; 157 158 typedef struct { 159 volatile int64_t val; 160 } atomic64_t; 161 162 #define ATOMIC64_INIT(x) { (x) } 163 164 static inline void 165 atomic64_set(atomic64_t *v, int64_t i) 166 { 167 mtx_enter(&atomic64_mtx); 168 v->val = i; 169 mtx_leave(&atomic64_mtx); 170 } 171 172 static inline int64_t 173 atomic64_read(atomic64_t *v) 174 { 175 int64_t val; 176 177 mtx_enter(&atomic64_mtx); 178 val = v->val; 179 mtx_leave(&atomic64_mtx); 180 181 return val; 182 } 183 184 static inline int64_t 185 atomic64_xchg(atomic64_t *v, int64_t n) 186 { 187 int64_t val; 188 189 mtx_enter(&atomic64_mtx); 190 val = v->val; 191 v->val = n; 192 mtx_leave(&atomic64_mtx); 193 194 return val; 195 } 196 197 static inline void 198 atomic64_add(int i, atomic64_t *v) 199 { 200 mtx_enter(&atomic64_mtx); 201 v->val += i; 202 mtx_leave(&atomic64_mtx); 203 } 204 205 #define atomic64_inc(p) atomic64_add(p, 1) 206 207 static inline int64_t 208 atomic64_add_return(int i, atomic64_t *v) 209 { 210 int64_t val; 211 212 mtx_enter(&atomic64_mtx); 213 val = v->val + i; 214 v->val = val; 215 mtx_leave(&atomic64_mtx); 216 217 return val; 218 } 219 220 #define atomic64_inc_return(p) atomic64_add_return(1, p) 221 222 static inline void 223 atomic64_sub(int i, atomic64_t *v) 224 { 225 mtx_enter(&atomic64_mtx); 226 v->val -= i; 227 mtx_leave(&atomic64_mtx); 228 } 229 #endif 230 231 #ifdef __LP64__ 232 typedef int64_t atomic_long_t; 233 #define atomic_long_set(p, v) atomic64_set(p, v) 234 #define atomic_long_xchg(v, n) atomic64_xchg(v, n) 235 #define atomic_long_cmpxchg(p, o, n) atomic_cmpxchg(p, o, n) 236 #define atomic_long_add(i, v) atomic64_add(i, v) 237 #define atomic_long_sub(i, v) atomic64_sub(i, v) 238 #else 239 typedef int32_t atomic_long_t; 240 #define atomic_long_set(p, v) atomic_set(p, v) 241 #define atomic_long_xchg(v, n) atomic_xchg(v, n) 242 #define atomic_long_cmpxchg(p, o, n) atomic_cmpxchg(p, o, n) 243 #define atomic_long_add(i, v) atomic_add(i, v) 244 #define atomic_long_sub(i, v) atomic_sub(i, v) 245 #endif 246 247 static inline atomic_t 248 test_and_set_bit(u_int b, volatile void *p) 249 { 250 unsigned int m = 1 << (b & 0x1f); 251 unsigned int prev = __sync_fetch_and_or((volatile u_int *)p + (b >> 5), m); 252 return (prev & m) != 0; 253 } 254 255 static inline void 256 clear_bit(u_int b, volatile void *p) 257 { 258 atomic_clearbits_int(((volatile u_int *)p) + (b >> 5), 1 << (b & 0x1f)); 259 } 260 261 static inline void 262 clear_bit_unlock(u_int b, volatile void *p) 263 { 264 membar_enter(); 265 clear_bit(b, p); 266 } 267 268 static inline void 269 set_bit(u_int b, volatile void *p) 270 { 271 atomic_setbits_int(((volatile u_int *)p) + (b >> 5), 1 << (b & 0x1f)); 272 } 273 274 static inline void 275 __clear_bit(u_int b, volatile void *p) 276 { 277 volatile u_int *ptr = (volatile u_int *)p; 278 ptr[b >> 5] &= ~(1 << (b & 0x1f)); 279 } 280 281 static inline void 282 __set_bit(u_int b, volatile void *p) 283 { 284 volatile u_int *ptr = (volatile u_int *)p; 285 ptr[b >> 5] |= (1 << (b & 0x1f)); 286 } 287 288 static inline int 289 test_bit(u_int b, const volatile void *p) 290 { 291 return !!(((volatile u_int *)p)[b >> 5] & (1 << (b & 0x1f))); 292 } 293 294 static inline int 295 __test_and_set_bit(u_int b, volatile void *p) 296 { 297 unsigned int m = 1 << (b & 0x1f); 298 volatile u_int *ptr = (volatile u_int *)p; 299 unsigned int prev = ptr[b >> 5]; 300 ptr[b >> 5] |= m; 301 302 return (prev & m) != 0; 303 } 304 305 static inline int 306 test_and_clear_bit(u_int b, volatile void *p) 307 { 308 unsigned int m = 1 << (b & 0x1f); 309 unsigned int prev = __sync_fetch_and_and((volatile u_int *)p + (b >> 5), ~m); 310 return (prev & m) != 0; 311 } 312 313 static inline int 314 __test_and_clear_bit(u_int b, volatile void *p) 315 { 316 volatile u_int *ptr = (volatile u_int *)p; 317 int rv = !!(ptr[b >> 5] & (1 << (b & 0x1f))); 318 ptr[b >> 5] &= ~(1 << (b & 0x1f)); 319 return rv; 320 } 321 322 static inline int 323 find_first_zero_bit(volatile void *p, int max) 324 { 325 int b; 326 volatile u_int *ptr = (volatile u_int *)p; 327 328 for (b = 0; b < max; b += 32) { 329 if (ptr[b >> 5] != ~0) { 330 for (;;) { 331 if ((ptr[b >> 5] & (1 << (b & 0x1f))) == 0) 332 return b; 333 b++; 334 } 335 } 336 } 337 return max; 338 } 339 340 static inline int 341 find_next_zero_bit(volatile void *p, int max, int b) 342 { 343 volatile u_int *ptr = (volatile u_int *)p; 344 345 for (; b < max; b += 32) { 346 if (ptr[b >> 5] != ~0) { 347 for (;;) { 348 if ((ptr[b >> 5] & (1 << (b & 0x1f))) == 0) 349 return b; 350 b++; 351 } 352 } 353 } 354 return max; 355 } 356 357 static inline int 358 find_first_bit(volatile void *p, int max) 359 { 360 int b; 361 volatile u_int *ptr = (volatile u_int *)p; 362 363 for (b = 0; b < max; b += 32) { 364 if (ptr[b >> 5] != 0) { 365 for (;;) { 366 if (ptr[b >> 5] & (1 << (b & 0x1f))) 367 return b; 368 b++; 369 } 370 } 371 } 372 return max; 373 } 374 375 static inline int 376 find_next_bit(const volatile void *p, int max, int b) 377 { 378 volatile u_int *ptr = (volatile u_int *)p; 379 380 for (; b < max; b+= 32) { 381 if (ptr[b >> 5] != 0) { 382 for (;;) { 383 if (ptr[b >> 5] & (1 << (b & 0x1f))) 384 return b; 385 b++; 386 } 387 } 388 } 389 return max; 390 } 391 392 #define for_each_set_bit(b, p, max) \ 393 for ((b) = find_first_bit((p), (max)); \ 394 (b) < (max); \ 395 (b) = find_next_bit((p), (max), (b) + 1)) 396 397 #define for_each_clear_bit(b, p, max) \ 398 for ((b) = find_first_zero_bit((p), (max)); \ 399 (b) < (max); \ 400 (b) = find_next_zero_bit((p), (max), (b) + 1)) 401 402 #if defined(__i386__) 403 #define rmb() __asm volatile("lock; addl $0,-4(%%esp)" : : : "memory", "cc") 404 #define wmb() __asm volatile("lock; addl $0,-4(%%esp)" : : : "memory", "cc") 405 #define mb() __asm volatile("lock; addl $0,-4(%%esp)" : : : "memory", "cc") 406 #define smp_mb() __asm volatile("lock; addl $0,-4(%%esp)" : : : "memory", "cc") 407 #define smp_rmb() __membar("") 408 #define smp_wmb() __membar("") 409 #define __smp_store_mb(var, value) do { (void)xchg(&var, value); } while (0) 410 #define smp_mb__after_atomic() do { } while (0) 411 #define smp_mb__before_atomic() do { } while (0) 412 #elif defined(__amd64__) 413 #define rmb() __membar("lfence") 414 #define wmb() __membar("sfence") 415 #define mb() __membar("mfence") 416 #define smp_mb() __asm volatile("lock; addl $0,-4(%%rsp)" : : : "memory", "cc") 417 #define smp_rmb() __membar("") 418 #define smp_wmb() __membar("") 419 #define __smp_store_mb(var, value) do { (void)xchg(&var, value); } while (0) 420 #define smp_mb__after_atomic() do { } while (0) 421 #define smp_mb__before_atomic() do { } while (0) 422 #elif defined(__aarch64__) 423 #define rmb() __membar("dsb ld") 424 #define wmb() __membar("dsb st") 425 #define mb() __membar("dsb sy") 426 #define dma_rmb() __membar("dmb oshld") 427 #define dma_wmb() __membar("dmb oshst") 428 #define dma_mb() __membar("dmb osh") 429 #define smp_mb() __membar("dmb ish") 430 #elif defined(__arm__) 431 #define rmb() __membar("dsb sy") 432 #define wmb() __membar("dsb sy") 433 #define mb() __membar("dsb sy") 434 #elif defined(__mips64__) 435 #define rmb() mips_sync() 436 #define wmb() mips_sync() 437 #define mb() mips_sync() 438 #elif defined(__powerpc64__) 439 #define rmb() __membar("sync") 440 #define wmb() __membar("sync") 441 #define mb() __membar("sync") 442 #define smp_rmb() __membar("lwsync") 443 #define smp_wmb() __membar("lwsync") 444 #define smp_mb() __membar("sync") 445 #elif defined(__powerpc__) 446 #define rmb() __membar("sync") 447 #define wmb() __membar("sync") 448 #define mb() __membar("sync") 449 #define smp_wmb() __membar("eieio") 450 #elif defined(__riscv) 451 #define rmb() __membar("fence ir,ir") 452 #define wmb() __membar("fence ow,ow") 453 #define mb() __membar("fence iorw,iorw") 454 #define smp_rmb() __membar("fence r,r") 455 #define smp_wmb() __membar("fence w,w") 456 #define smp_mb() __membar("fence rw,rw") 457 #elif defined(__sparc64__) 458 #define rmb() membar_sync() 459 #define wmb() membar_sync() 460 #define mb() membar_sync() 461 #endif 462 463 #ifndef smp_rmb 464 #define smp_rmb() rmb() 465 #endif 466 467 #ifndef smp_wmb 468 #define smp_wmb() wmb() 469 #endif 470 471 #ifndef mmiowb 472 #define mmiowb() wmb() 473 #endif 474 475 #ifndef smp_mb__before_atomic 476 #define smp_mb__before_atomic() mb() 477 #endif 478 479 #ifndef smp_mb__after_atomic 480 #define smp_mb__after_atomic() mb() 481 #endif 482 483 #ifndef smp_store_mb 484 #define smp_store_mb(x, v) do { x = v; mb(); } while (0) 485 #endif 486 487 #ifndef smp_store_release 488 #define smp_store_release(x, v) do { smp_mb(); WRITE_ONCE(*x, v); } while(0) 489 #endif 490 491 #ifndef smp_load_acquire 492 #define smp_load_acquire(x) \ 493 ({ \ 494 __typeof(*x) _v = READ_ONCE(*x); \ 495 smp_mb(); \ 496 _v; \ 497 }) 498 #endif 499 500 #endif 501