1 //===-- sanitizer_atomic_msvc.h ---------------------------------*- C++ -*-===// 2 // 3 // This file is distributed under the University of Illinois Open Source 4 // License. See LICENSE.TXT for details. 5 // 6 //===----------------------------------------------------------------------===// 7 // 8 // This file is a part of ThreadSanitizer/AddressSanitizer runtime. 9 // Not intended for direct inclusion. Include sanitizer_atomic.h. 10 // 11 //===----------------------------------------------------------------------===// 12 13 #ifndef SANITIZER_ATOMIC_MSVC_H 14 #define SANITIZER_ATOMIC_MSVC_H 15 16 extern "C" void _ReadWriteBarrier(); 17 #pragma intrinsic(_ReadWriteBarrier) 18 extern "C" void _mm_mfence(); 19 #pragma intrinsic(_mm_mfence) 20 extern "C" void _mm_pause(); 21 #pragma intrinsic(_mm_pause) 22 extern "C" char _InterlockedExchange8( // NOLINT 23 char volatile *Addend, char Value); // NOLINT 24 #pragma intrinsic(_InterlockedExchange8) 25 extern "C" short _InterlockedExchange16( // NOLINT 26 short volatile *Addend, short Value); // NOLINT 27 #pragma intrinsic(_InterlockedExchange16) 28 extern "C" long _InterlockedExchange( // NOLINT 29 long volatile *Addend, long Value); // NOLINT 30 #pragma intrinsic(_InterlockedExchange) 31 extern "C" long _InterlockedExchangeAdd( // NOLINT 32 long volatile * Addend, long Value); // NOLINT 33 #pragma intrinsic(_InterlockedExchangeAdd) 34 extern "C" char _InterlockedCompareExchange8( // NOLINT 35 char volatile *Destination, // NOLINT 36 char Exchange, char Comparand); // NOLINT 37 #pragma intrinsic(_InterlockedCompareExchange8) 38 extern "C" short _InterlockedCompareExchange16( // NOLINT 39 short volatile *Destination, // NOLINT 40 short Exchange, short Comparand); // NOLINT 41 #pragma intrinsic(_InterlockedCompareExchange16) 42 extern "C" 43 long long _InterlockedCompareExchange64( // NOLINT 44 long long volatile *Destination, // NOLINT 45 long long Exchange, long long Comparand); // NOLINT 46 #pragma intrinsic(_InterlockedCompareExchange64) 47 extern "C" void *_InterlockedCompareExchangePointer( 48 void *volatile *Destination, 49 void *Exchange, void *Comparand); 50 #pragma intrinsic(_InterlockedCompareExchangePointer) 51 extern "C" 52 long __cdecl _InterlockedCompareExchange( // NOLINT 53 long volatile *Destination, // NOLINT 54 long Exchange, long Comparand); // NOLINT 55 #pragma intrinsic(_InterlockedCompareExchange) 56 57 #ifdef _WIN64 58 extern "C" long long _InterlockedExchangeAdd64( // NOLINT 59 long long volatile * Addend, long long Value); // NOLINT 60 #pragma intrinsic(_InterlockedExchangeAdd64) 61 #endif 62 63 namespace __sanitizer { 64 65 INLINE void atomic_signal_fence(memory_order) { 66 _ReadWriteBarrier(); 67 } 68 69 INLINE void atomic_thread_fence(memory_order) { 70 _mm_mfence(); 71 } 72 73 INLINE void proc_yield(int cnt) { 74 for (int i = 0; i < cnt; i++) 75 _mm_pause(); 76 } 77 78 template<typename T> 79 INLINE typename T::Type atomic_load( 80 const volatile T *a, memory_order mo) { 81 DCHECK(mo & (memory_order_relaxed | memory_order_consume 82 | memory_order_acquire | memory_order_seq_cst)); 83 DCHECK(!((uptr)a % sizeof(*a))); 84 typename T::Type v; 85 // FIXME(dvyukov): 64-bit load is not atomic on 32-bits. 86 if (mo == memory_order_relaxed) { 87 v = a->val_dont_use; 88 } else { 89 atomic_signal_fence(memory_order_seq_cst); 90 v = a->val_dont_use; 91 atomic_signal_fence(memory_order_seq_cst); 92 } 93 return v; 94 } 95 96 template<typename T> 97 INLINE void atomic_store(volatile T *a, typename T::Type v, memory_order mo) { 98 DCHECK(mo & (memory_order_relaxed | memory_order_release 99 | memory_order_seq_cst)); 100 DCHECK(!((uptr)a % sizeof(*a))); 101 // FIXME(dvyukov): 64-bit store is not atomic on 32-bits. 102 if (mo == memory_order_relaxed) { 103 a->val_dont_use = v; 104 } else { 105 atomic_signal_fence(memory_order_seq_cst); 106 a->val_dont_use = v; 107 atomic_signal_fence(memory_order_seq_cst); 108 } 109 if (mo == memory_order_seq_cst) 110 atomic_thread_fence(memory_order_seq_cst); 111 } 112 113 INLINE u32 atomic_fetch_add(volatile atomic_uint32_t *a, 114 u32 v, memory_order mo) { 115 (void)mo; 116 DCHECK(!((uptr)a % sizeof(*a))); 117 return (u32)_InterlockedExchangeAdd( 118 (volatile long*)&a->val_dont_use, (long)v); // NOLINT 119 } 120 121 INLINE uptr atomic_fetch_add(volatile atomic_uintptr_t *a, 122 uptr v, memory_order mo) { 123 (void)mo; 124 DCHECK(!((uptr)a % sizeof(*a))); 125 #ifdef _WIN64 126 return (uptr)_InterlockedExchangeAdd64( 127 (volatile long long*)&a->val_dont_use, (long long)v); // NOLINT 128 #else 129 return (uptr)_InterlockedExchangeAdd( 130 (volatile long*)&a->val_dont_use, (long)v); // NOLINT 131 #endif 132 } 133 134 INLINE u32 atomic_fetch_sub(volatile atomic_uint32_t *a, 135 u32 v, memory_order mo) { 136 (void)mo; 137 DCHECK(!((uptr)a % sizeof(*a))); 138 return (u32)_InterlockedExchangeAdd( 139 (volatile long*)&a->val_dont_use, -(long)v); // NOLINT 140 } 141 142 INLINE uptr atomic_fetch_sub(volatile atomic_uintptr_t *a, 143 uptr v, memory_order mo) { 144 (void)mo; 145 DCHECK(!((uptr)a % sizeof(*a))); 146 #ifdef _WIN64 147 return (uptr)_InterlockedExchangeAdd64( 148 (volatile long long*)&a->val_dont_use, -(long long)v); // NOLINT 149 #else 150 return (uptr)_InterlockedExchangeAdd( 151 (volatile long*)&a->val_dont_use, -(long)v); // NOLINT 152 #endif 153 } 154 155 INLINE u8 atomic_exchange(volatile atomic_uint8_t *a, 156 u8 v, memory_order mo) { 157 (void)mo; 158 DCHECK(!((uptr)a % sizeof(*a))); 159 return (u8)_InterlockedExchange8((volatile char*)&a->val_dont_use, v); 160 } 161 162 INLINE u16 atomic_exchange(volatile atomic_uint16_t *a, 163 u16 v, memory_order mo) { 164 (void)mo; 165 DCHECK(!((uptr)a % sizeof(*a))); 166 return (u16)_InterlockedExchange16((volatile short*)&a->val_dont_use, v); 167 } 168 169 INLINE u32 atomic_exchange(volatile atomic_uint32_t *a, 170 u32 v, memory_order mo) { 171 (void)mo; 172 DCHECK(!((uptr)a % sizeof(*a))); 173 return (u32)_InterlockedExchange((volatile long*)&a->val_dont_use, v); 174 } 175 176 INLINE bool atomic_compare_exchange_strong(volatile atomic_uint8_t *a, 177 u8 *cmp, 178 u8 xchgv, 179 memory_order mo) { 180 (void)mo; 181 DCHECK(!((uptr)a % sizeof(*a))); 182 u8 cmpv = *cmp; 183 #ifdef _WIN64 184 u8 prev = (u8)_InterlockedCompareExchange8( 185 (volatile char*)&a->val_dont_use, (char)xchgv, (char)cmpv); 186 #else 187 u8 prev; 188 __asm { 189 mov al, cmpv 190 mov ecx, a 191 mov dl, xchgv 192 lock cmpxchg [ecx], dl 193 mov prev, al 194 } 195 #endif 196 if (prev == cmpv) 197 return true; 198 *cmp = prev; 199 return false; 200 } 201 202 INLINE bool atomic_compare_exchange_strong(volatile atomic_uintptr_t *a, 203 uptr *cmp, 204 uptr xchg, 205 memory_order mo) { 206 uptr cmpv = *cmp; 207 uptr prev = (uptr)_InterlockedCompareExchangePointer( 208 (void*volatile*)&a->val_dont_use, (void*)xchg, (void*)cmpv); 209 if (prev == cmpv) 210 return true; 211 *cmp = prev; 212 return false; 213 } 214 215 INLINE bool atomic_compare_exchange_strong(volatile atomic_uint16_t *a, 216 u16 *cmp, 217 u16 xchg, 218 memory_order mo) { 219 u16 cmpv = *cmp; 220 u16 prev = (u16)_InterlockedCompareExchange16( 221 (volatile short*)&a->val_dont_use, (short)xchg, (short)cmpv); 222 if (prev == cmpv) 223 return true; 224 *cmp = prev; 225 return false; 226 } 227 228 INLINE bool atomic_compare_exchange_strong(volatile atomic_uint32_t *a, 229 u32 *cmp, 230 u32 xchg, 231 memory_order mo) { 232 u32 cmpv = *cmp; 233 u32 prev = (u32)_InterlockedCompareExchange( 234 (volatile long*)&a->val_dont_use, (long)xchg, (long)cmpv); 235 if (prev == cmpv) 236 return true; 237 *cmp = prev; 238 return false; 239 } 240 241 INLINE bool atomic_compare_exchange_strong(volatile atomic_uint64_t *a, 242 u64 *cmp, 243 u64 xchg, 244 memory_order mo) { 245 u64 cmpv = *cmp; 246 u64 prev = (u64)_InterlockedCompareExchange64( 247 (volatile long long*)&a->val_dont_use, (long long)xchg, (long long)cmpv); 248 if (prev == cmpv) 249 return true; 250 *cmp = prev; 251 return false; 252 } 253 254 template<typename T> 255 INLINE bool atomic_compare_exchange_weak(volatile T *a, 256 typename T::Type *cmp, 257 typename T::Type xchg, 258 memory_order mo) { 259 return atomic_compare_exchange_strong(a, cmp, xchg, mo); 260 } 261 262 } // namespace __sanitizer 263 264 #endif // SANITIZER_ATOMIC_CLANG_H 265