1 //===----------------------------------------------------------------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 9 #ifndef _LIBCPP___ATOMIC_ATOMIC_H 10 #define _LIBCPP___ATOMIC_ATOMIC_H 11 12 #include <__atomic/atomic_sync.h> 13 #include <__atomic/check_memory_order.h> 14 #include <__atomic/is_always_lock_free.h> 15 #include <__atomic/memory_order.h> 16 #include <__atomic/support.h> 17 #include <__config> 18 #include <__cstddef/ptrdiff_t.h> 19 #include <__memory/addressof.h> 20 #include <__type_traits/enable_if.h> 21 #include <__type_traits/is_floating_point.h> 22 #include <__type_traits/is_function.h> 23 #include <__type_traits/is_integral.h> 24 #include <__type_traits/is_nothrow_constructible.h> 25 #include <__type_traits/is_same.h> 26 #include <__type_traits/remove_const.h> 27 #include <__type_traits/remove_pointer.h> 28 #include <__type_traits/remove_volatile.h> 29 #include <__utility/forward.h> 30 #include <cstring> 31 32 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) 33 # pragma GCC system_header 34 #endif 35 36 _LIBCPP_BEGIN_NAMESPACE_STD 37 38 template <class _Tp, bool = is_integral<_Tp>::value && !is_same<_Tp, bool>::value> 39 struct __atomic_base // false 40 { 41 mutable __cxx_atomic_impl<_Tp> __a_; 42 43 #if _LIBCPP_STD_VER >= 17 44 static constexpr bool is_always_lock_free = __libcpp_is_always_lock_free<__cxx_atomic_impl<_Tp> >::__value; 45 #endif 46 47 _LIBCPP_HIDE_FROM_ABI bool is_lock_free() const volatile _NOEXCEPT { 48 return __cxx_atomic_is_lock_free(sizeof(__cxx_atomic_impl<_Tp>)); 49 } 50 _LIBCPP_HIDE_FROM_ABI bool is_lock_free() const _NOEXCEPT { 51 return static_cast<__atomic_base const volatile*>(this)->is_lock_free(); 52 } 53 _LIBCPP_HIDE_FROM_ABI void store(_Tp __d, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT 54 _LIBCPP_CHECK_STORE_MEMORY_ORDER(__m) { 55 std::__cxx_atomic_store(std::addressof(__a_), __d, __m); 56 } 57 _LIBCPP_HIDE_FROM_ABI void store(_Tp __d, memory_order __m = memory_order_seq_cst) _NOEXCEPT 58 _LIBCPP_CHECK_STORE_MEMORY_ORDER(__m) { 59 std::__cxx_atomic_store(std::addressof(__a_), __d, __m); 60 } 61 _LIBCPP_HIDE_FROM_ABI _Tp load(memory_order __m = memory_order_seq_cst) const volatile _NOEXCEPT 62 _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m) { 63 return std::__cxx_atomic_load(std::addressof(__a_), __m); 64 } 65 _LIBCPP_HIDE_FROM_ABI _Tp load(memory_order __m = memory_order_seq_cst) const _NOEXCEPT 66 _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m) { 67 return std::__cxx_atomic_load(std::addressof(__a_), __m); 68 } 69 _LIBCPP_HIDE_FROM_ABI operator _Tp() const volatile _NOEXCEPT { return load(); } 70 _LIBCPP_HIDE_FROM_ABI operator _Tp() const _NOEXCEPT { return load(); } 71 _LIBCPP_HIDE_FROM_ABI _Tp exchange(_Tp __d, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT { 72 return std::__cxx_atomic_exchange(std::addressof(__a_), __d, __m); 73 } 74 _LIBCPP_HIDE_FROM_ABI _Tp exchange(_Tp __d, memory_order __m = memory_order_seq_cst) _NOEXCEPT { 75 return std::__cxx_atomic_exchange(std::addressof(__a_), __d, __m); 76 } 77 _LIBCPP_HIDE_FROM_ABI bool 78 compare_exchange_weak(_Tp& __e, _Tp __d, memory_order __s, memory_order __f) volatile _NOEXCEPT 79 _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) { 80 return std::__cxx_atomic_compare_exchange_weak(std::addressof(__a_), std::addressof(__e), __d, __s, __f); 81 } 82 _LIBCPP_HIDE_FROM_ABI bool compare_exchange_weak(_Tp& __e, _Tp __d, memory_order __s, memory_order __f) _NOEXCEPT 83 _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) { 84 return std::__cxx_atomic_compare_exchange_weak(std::addressof(__a_), std::addressof(__e), __d, __s, __f); 85 } 86 _LIBCPP_HIDE_FROM_ABI bool 87 compare_exchange_strong(_Tp& __e, _Tp __d, memory_order __s, memory_order __f) volatile _NOEXCEPT 88 _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) { 89 return std::__cxx_atomic_compare_exchange_strong(std::addressof(__a_), std::addressof(__e), __d, __s, __f); 90 } 91 _LIBCPP_HIDE_FROM_ABI bool compare_exchange_strong(_Tp& __e, _Tp __d, memory_order __s, memory_order __f) _NOEXCEPT 92 _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) { 93 return std::__cxx_atomic_compare_exchange_strong(std::addressof(__a_), std::addressof(__e), __d, __s, __f); 94 } 95 _LIBCPP_HIDE_FROM_ABI bool 96 compare_exchange_weak(_Tp& __e, _Tp __d, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT { 97 return std::__cxx_atomic_compare_exchange_weak(std::addressof(__a_), std::addressof(__e), __d, __m, __m); 98 } 99 _LIBCPP_HIDE_FROM_ABI bool 100 compare_exchange_weak(_Tp& __e, _Tp __d, memory_order __m = memory_order_seq_cst) _NOEXCEPT { 101 return std::__cxx_atomic_compare_exchange_weak(std::addressof(__a_), std::addressof(__e), __d, __m, __m); 102 } 103 _LIBCPP_HIDE_FROM_ABI bool 104 compare_exchange_strong(_Tp& __e, _Tp __d, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT { 105 return std::__cxx_atomic_compare_exchange_strong(std::addressof(__a_), std::addressof(__e), __d, __m, __m); 106 } 107 _LIBCPP_HIDE_FROM_ABI bool 108 compare_exchange_strong(_Tp& __e, _Tp __d, memory_order __m = memory_order_seq_cst) _NOEXCEPT { 109 return std::__cxx_atomic_compare_exchange_strong(std::addressof(__a_), std::addressof(__e), __d, __m, __m); 110 } 111 112 #if _LIBCPP_STD_VER >= 20 113 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void wait(_Tp __v, memory_order __m = memory_order_seq_cst) const 114 volatile _NOEXCEPT { 115 std::__atomic_wait(*this, __v, __m); 116 } 117 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void 118 wait(_Tp __v, memory_order __m = memory_order_seq_cst) const _NOEXCEPT { 119 std::__atomic_wait(*this, __v, __m); 120 } 121 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void notify_one() volatile _NOEXCEPT { 122 std::__atomic_notify_one(*this); 123 } 124 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void notify_one() _NOEXCEPT { std::__atomic_notify_one(*this); } 125 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void notify_all() volatile _NOEXCEPT { 126 std::__atomic_notify_all(*this); 127 } 128 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void notify_all() _NOEXCEPT { std::__atomic_notify_all(*this); } 129 #endif // _LIBCPP_STD_VER >= 20 130 131 #if _LIBCPP_STD_VER >= 20 132 _LIBCPP_HIDE_FROM_ABI constexpr __atomic_base() noexcept(is_nothrow_default_constructible_v<_Tp>) : __a_(_Tp()) {} 133 #else 134 _LIBCPP_HIDE_FROM_ABI __atomic_base() _NOEXCEPT = default; 135 #endif 136 137 _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR __atomic_base(_Tp __d) _NOEXCEPT : __a_(__d) {} 138 139 __atomic_base(const __atomic_base&) = delete; 140 }; 141 142 // atomic<Integral> 143 144 template <class _Tp> 145 struct __atomic_base<_Tp, true> : public __atomic_base<_Tp, false> { 146 using __base _LIBCPP_NODEBUG = __atomic_base<_Tp, false>; 147 148 _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 __atomic_base() _NOEXCEPT = default; 149 150 _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR __atomic_base(_Tp __d) _NOEXCEPT : __base(__d) {} 151 152 _LIBCPP_HIDE_FROM_ABI _Tp fetch_add(_Tp __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT { 153 return std::__cxx_atomic_fetch_add(std::addressof(this->__a_), __op, __m); 154 } 155 _LIBCPP_HIDE_FROM_ABI _Tp fetch_add(_Tp __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT { 156 return std::__cxx_atomic_fetch_add(std::addressof(this->__a_), __op, __m); 157 } 158 _LIBCPP_HIDE_FROM_ABI _Tp fetch_sub(_Tp __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT { 159 return std::__cxx_atomic_fetch_sub(std::addressof(this->__a_), __op, __m); 160 } 161 _LIBCPP_HIDE_FROM_ABI _Tp fetch_sub(_Tp __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT { 162 return std::__cxx_atomic_fetch_sub(std::addressof(this->__a_), __op, __m); 163 } 164 _LIBCPP_HIDE_FROM_ABI _Tp fetch_and(_Tp __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT { 165 return std::__cxx_atomic_fetch_and(std::addressof(this->__a_), __op, __m); 166 } 167 _LIBCPP_HIDE_FROM_ABI _Tp fetch_and(_Tp __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT { 168 return std::__cxx_atomic_fetch_and(std::addressof(this->__a_), __op, __m); 169 } 170 _LIBCPP_HIDE_FROM_ABI _Tp fetch_or(_Tp __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT { 171 return std::__cxx_atomic_fetch_or(std::addressof(this->__a_), __op, __m); 172 } 173 _LIBCPP_HIDE_FROM_ABI _Tp fetch_or(_Tp __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT { 174 return std::__cxx_atomic_fetch_or(std::addressof(this->__a_), __op, __m); 175 } 176 _LIBCPP_HIDE_FROM_ABI _Tp fetch_xor(_Tp __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT { 177 return std::__cxx_atomic_fetch_xor(std::addressof(this->__a_), __op, __m); 178 } 179 _LIBCPP_HIDE_FROM_ABI _Tp fetch_xor(_Tp __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT { 180 return std::__cxx_atomic_fetch_xor(std::addressof(this->__a_), __op, __m); 181 } 182 183 _LIBCPP_HIDE_FROM_ABI _Tp operator++(int) volatile _NOEXCEPT { return fetch_add(_Tp(1)); } 184 _LIBCPP_HIDE_FROM_ABI _Tp operator++(int) _NOEXCEPT { return fetch_add(_Tp(1)); } 185 _LIBCPP_HIDE_FROM_ABI _Tp operator--(int) volatile _NOEXCEPT { return fetch_sub(_Tp(1)); } 186 _LIBCPP_HIDE_FROM_ABI _Tp operator--(int) _NOEXCEPT { return fetch_sub(_Tp(1)); } 187 _LIBCPP_HIDE_FROM_ABI _Tp operator++() volatile _NOEXCEPT { return fetch_add(_Tp(1)) + _Tp(1); } 188 _LIBCPP_HIDE_FROM_ABI _Tp operator++() _NOEXCEPT { return fetch_add(_Tp(1)) + _Tp(1); } 189 _LIBCPP_HIDE_FROM_ABI _Tp operator--() volatile _NOEXCEPT { return fetch_sub(_Tp(1)) - _Tp(1); } 190 _LIBCPP_HIDE_FROM_ABI _Tp operator--() _NOEXCEPT { return fetch_sub(_Tp(1)) - _Tp(1); } 191 _LIBCPP_HIDE_FROM_ABI _Tp operator+=(_Tp __op) volatile _NOEXCEPT { return fetch_add(__op) + __op; } 192 _LIBCPP_HIDE_FROM_ABI _Tp operator+=(_Tp __op) _NOEXCEPT { return fetch_add(__op) + __op; } 193 _LIBCPP_HIDE_FROM_ABI _Tp operator-=(_Tp __op) volatile _NOEXCEPT { return fetch_sub(__op) - __op; } 194 _LIBCPP_HIDE_FROM_ABI _Tp operator-=(_Tp __op) _NOEXCEPT { return fetch_sub(__op) - __op; } 195 _LIBCPP_HIDE_FROM_ABI _Tp operator&=(_Tp __op) volatile _NOEXCEPT { return fetch_and(__op) & __op; } 196 _LIBCPP_HIDE_FROM_ABI _Tp operator&=(_Tp __op) _NOEXCEPT { return fetch_and(__op) & __op; } 197 _LIBCPP_HIDE_FROM_ABI _Tp operator|=(_Tp __op) volatile _NOEXCEPT { return fetch_or(__op) | __op; } 198 _LIBCPP_HIDE_FROM_ABI _Tp operator|=(_Tp __op) _NOEXCEPT { return fetch_or(__op) | __op; } 199 _LIBCPP_HIDE_FROM_ABI _Tp operator^=(_Tp __op) volatile _NOEXCEPT { return fetch_xor(__op) ^ __op; } 200 _LIBCPP_HIDE_FROM_ABI _Tp operator^=(_Tp __op) _NOEXCEPT { return fetch_xor(__op) ^ __op; } 201 }; 202 203 // Here we need _IsIntegral because the default template argument is not enough 204 // e.g __atomic_base<int> is __atomic_base<int, true>, which inherits from 205 // __atomic_base<int, false> and the caller of the wait function is 206 // __atomic_base<int, false>. So specializing __atomic_base<_Tp> does not work 207 template <class _Tp, bool _IsIntegral> 208 struct __atomic_waitable_traits<__atomic_base<_Tp, _IsIntegral> > { 209 static _LIBCPP_HIDE_FROM_ABI _Tp __atomic_load(const __atomic_base<_Tp, _IsIntegral>& __a, memory_order __order) { 210 return __a.load(__order); 211 } 212 213 static _LIBCPP_HIDE_FROM_ABI _Tp 214 __atomic_load(const volatile __atomic_base<_Tp, _IsIntegral>& __this, memory_order __order) { 215 return __this.load(__order); 216 } 217 218 static _LIBCPP_HIDE_FROM_ABI const __cxx_atomic_impl<_Tp>* 219 __atomic_contention_address(const __atomic_base<_Tp, _IsIntegral>& __a) { 220 return std::addressof(__a.__a_); 221 } 222 223 static _LIBCPP_HIDE_FROM_ABI const volatile __cxx_atomic_impl<_Tp>* 224 __atomic_contention_address(const volatile __atomic_base<_Tp, _IsIntegral>& __this) { 225 return std::addressof(__this.__a_); 226 } 227 }; 228 229 template <class _Tp> 230 struct atomic : public __atomic_base<_Tp> { 231 using __base _LIBCPP_NODEBUG = __atomic_base<_Tp>; 232 using value_type = _Tp; 233 using difference_type = value_type; 234 235 #if _LIBCPP_STD_VER >= 20 236 _LIBCPP_HIDE_FROM_ABI atomic() = default; 237 #else 238 _LIBCPP_HIDE_FROM_ABI atomic() _NOEXCEPT = default; 239 #endif 240 241 _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR atomic(_Tp __d) _NOEXCEPT : __base(__d) {} 242 243 _LIBCPP_HIDE_FROM_ABI _Tp operator=(_Tp __d) volatile _NOEXCEPT { 244 __base::store(__d); 245 return __d; 246 } 247 _LIBCPP_HIDE_FROM_ABI _Tp operator=(_Tp __d) _NOEXCEPT { 248 __base::store(__d); 249 return __d; 250 } 251 252 atomic& operator=(const atomic&) = delete; 253 atomic& operator=(const atomic&) volatile = delete; 254 }; 255 256 // atomic<T*> 257 258 template <class _Tp> 259 struct atomic<_Tp*> : public __atomic_base<_Tp*> { 260 using __base _LIBCPP_NODEBUG = __atomic_base<_Tp*>; 261 using value_type = _Tp*; 262 using difference_type = ptrdiff_t; 263 264 _LIBCPP_HIDE_FROM_ABI atomic() _NOEXCEPT = default; 265 266 _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR atomic(_Tp* __d) _NOEXCEPT : __base(__d) {} 267 268 _LIBCPP_HIDE_FROM_ABI _Tp* operator=(_Tp* __d) volatile _NOEXCEPT { 269 __base::store(__d); 270 return __d; 271 } 272 _LIBCPP_HIDE_FROM_ABI _Tp* operator=(_Tp* __d) _NOEXCEPT { 273 __base::store(__d); 274 return __d; 275 } 276 277 _LIBCPP_HIDE_FROM_ABI _Tp* fetch_add(ptrdiff_t __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT { 278 // __atomic_fetch_add accepts function pointers, guard against them. 279 static_assert(!is_function<__remove_pointer_t<_Tp> >::value, "Pointer to function isn't allowed"); 280 return std::__cxx_atomic_fetch_add(std::addressof(this->__a_), __op, __m); 281 } 282 283 _LIBCPP_HIDE_FROM_ABI _Tp* fetch_add(ptrdiff_t __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT { 284 // __atomic_fetch_add accepts function pointers, guard against them. 285 static_assert(!is_function<__remove_pointer_t<_Tp> >::value, "Pointer to function isn't allowed"); 286 return std::__cxx_atomic_fetch_add(std::addressof(this->__a_), __op, __m); 287 } 288 289 _LIBCPP_HIDE_FROM_ABI _Tp* fetch_sub(ptrdiff_t __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT { 290 // __atomic_fetch_add accepts function pointers, guard against them. 291 static_assert(!is_function<__remove_pointer_t<_Tp> >::value, "Pointer to function isn't allowed"); 292 return std::__cxx_atomic_fetch_sub(std::addressof(this->__a_), __op, __m); 293 } 294 295 _LIBCPP_HIDE_FROM_ABI _Tp* fetch_sub(ptrdiff_t __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT { 296 // __atomic_fetch_add accepts function pointers, guard against them. 297 static_assert(!is_function<__remove_pointer_t<_Tp> >::value, "Pointer to function isn't allowed"); 298 return std::__cxx_atomic_fetch_sub(std::addressof(this->__a_), __op, __m); 299 } 300 301 _LIBCPP_HIDE_FROM_ABI _Tp* operator++(int) volatile _NOEXCEPT { return fetch_add(1); } 302 _LIBCPP_HIDE_FROM_ABI _Tp* operator++(int) _NOEXCEPT { return fetch_add(1); } 303 _LIBCPP_HIDE_FROM_ABI _Tp* operator--(int) volatile _NOEXCEPT { return fetch_sub(1); } 304 _LIBCPP_HIDE_FROM_ABI _Tp* operator--(int) _NOEXCEPT { return fetch_sub(1); } 305 _LIBCPP_HIDE_FROM_ABI _Tp* operator++() volatile _NOEXCEPT { return fetch_add(1) + 1; } 306 _LIBCPP_HIDE_FROM_ABI _Tp* operator++() _NOEXCEPT { return fetch_add(1) + 1; } 307 _LIBCPP_HIDE_FROM_ABI _Tp* operator--() volatile _NOEXCEPT { return fetch_sub(1) - 1; } 308 _LIBCPP_HIDE_FROM_ABI _Tp* operator--() _NOEXCEPT { return fetch_sub(1) - 1; } 309 _LIBCPP_HIDE_FROM_ABI _Tp* operator+=(ptrdiff_t __op) volatile _NOEXCEPT { return fetch_add(__op) + __op; } 310 _LIBCPP_HIDE_FROM_ABI _Tp* operator+=(ptrdiff_t __op) _NOEXCEPT { return fetch_add(__op) + __op; } 311 _LIBCPP_HIDE_FROM_ABI _Tp* operator-=(ptrdiff_t __op) volatile _NOEXCEPT { return fetch_sub(__op) - __op; } 312 _LIBCPP_HIDE_FROM_ABI _Tp* operator-=(ptrdiff_t __op) _NOEXCEPT { return fetch_sub(__op) - __op; } 313 314 atomic& operator=(const atomic&) = delete; 315 atomic& operator=(const atomic&) volatile = delete; 316 }; 317 318 template <class _Tp> 319 struct __atomic_waitable_traits<atomic<_Tp> > : __atomic_waitable_traits<__atomic_base<_Tp> > {}; 320 321 #if _LIBCPP_STD_VER >= 20 322 template <class _Tp> 323 requires is_floating_point_v<_Tp> 324 struct atomic<_Tp> : __atomic_base<_Tp> { 325 private: 326 _LIBCPP_HIDE_FROM_ABI static constexpr bool __is_fp80_long_double() { 327 // Only x87-fp80 long double has 64-bit mantissa 328 return __LDBL_MANT_DIG__ == 64 && std::is_same_v<_Tp, long double>; 329 } 330 331 _LIBCPP_HIDE_FROM_ABI static constexpr bool __has_rmw_builtin() { 332 # ifndef _LIBCPP_COMPILER_CLANG_BASED 333 return false; 334 # else 335 // The builtin __cxx_atomic_fetch_add errors during compilation for 336 // long double on platforms with fp80 format. 337 // For more details, see 338 // lib/Sema/SemaChecking.cpp function IsAllowedValueType 339 // LLVM Parser does not allow atomicrmw with x86_fp80 type. 340 // if (ValType->isSpecificBuiltinType(BuiltinType::LongDouble) && 341 // &Context.getTargetInfo().getLongDoubleFormat() == 342 // &llvm::APFloat::x87DoubleExtended()) 343 // For more info 344 // https://github.com/llvm/llvm-project/issues/68602 345 // https://reviews.llvm.org/D53965 346 return !__is_fp80_long_double(); 347 # endif 348 } 349 350 template <class _This, class _Operation, class _BuiltinOp> 351 _LIBCPP_HIDE_FROM_ABI static _Tp 352 __rmw_op(_This&& __self, _Tp __operand, memory_order __m, _Operation __operation, _BuiltinOp __builtin_op) { 353 if constexpr (__has_rmw_builtin()) { 354 return __builtin_op(std::addressof(std::forward<_This>(__self).__a_), __operand, __m); 355 } else { 356 _Tp __old = __self.load(memory_order_relaxed); 357 _Tp __new = __operation(__old, __operand); 358 while (!__self.compare_exchange_weak(__old, __new, __m, memory_order_relaxed)) { 359 # ifdef _LIBCPP_COMPILER_CLANG_BASED 360 if constexpr (__is_fp80_long_double()) { 361 // https://github.com/llvm/llvm-project/issues/47978 362 // clang bug: __old is not updated on failure for atomic<long double>::compare_exchange_weak 363 // Note __old = __self.load(memory_order_relaxed) will not work 364 std::__cxx_atomic_load_inplace(std::addressof(__self.__a_), &__old, memory_order_relaxed); 365 } 366 # endif 367 __new = __operation(__old, __operand); 368 } 369 return __old; 370 } 371 } 372 373 template <class _This> 374 _LIBCPP_HIDE_FROM_ABI static _Tp __fetch_add(_This&& __self, _Tp __operand, memory_order __m) { 375 auto __builtin_op = [](auto __a, auto __builtin_operand, auto __order) { 376 return std::__cxx_atomic_fetch_add(__a, __builtin_operand, __order); 377 }; 378 auto __plus = [](auto __a, auto __b) { return __a + __b; }; 379 return __rmw_op(std::forward<_This>(__self), __operand, __m, __plus, __builtin_op); 380 } 381 382 template <class _This> 383 _LIBCPP_HIDE_FROM_ABI static _Tp __fetch_sub(_This&& __self, _Tp __operand, memory_order __m) { 384 auto __builtin_op = [](auto __a, auto __builtin_operand, auto __order) { 385 return std::__cxx_atomic_fetch_sub(__a, __builtin_operand, __order); 386 }; 387 auto __minus = [](auto __a, auto __b) { return __a - __b; }; 388 return __rmw_op(std::forward<_This>(__self), __operand, __m, __minus, __builtin_op); 389 } 390 391 public: 392 using __base _LIBCPP_NODEBUG = __atomic_base<_Tp>; 393 using value_type = _Tp; 394 using difference_type = value_type; 395 396 _LIBCPP_HIDE_FROM_ABI constexpr atomic() noexcept = default; 397 _LIBCPP_HIDE_FROM_ABI constexpr atomic(_Tp __d) noexcept : __base(__d) {} 398 399 atomic(const atomic&) = delete; 400 atomic& operator=(const atomic&) = delete; 401 atomic& operator=(const atomic&) volatile = delete; 402 403 _LIBCPP_HIDE_FROM_ABI _Tp operator=(_Tp __d) volatile noexcept 404 requires __base::is_always_lock_free 405 { 406 __base::store(__d); 407 return __d; 408 } 409 _LIBCPP_HIDE_FROM_ABI _Tp operator=(_Tp __d) noexcept { 410 __base::store(__d); 411 return __d; 412 } 413 414 _LIBCPP_HIDE_FROM_ABI _Tp fetch_add(_Tp __op, memory_order __m = memory_order_seq_cst) volatile noexcept 415 requires __base::is_always_lock_free 416 { 417 return __fetch_add(*this, __op, __m); 418 } 419 420 _LIBCPP_HIDE_FROM_ABI _Tp fetch_add(_Tp __op, memory_order __m = memory_order_seq_cst) noexcept { 421 return __fetch_add(*this, __op, __m); 422 } 423 424 _LIBCPP_HIDE_FROM_ABI _Tp fetch_sub(_Tp __op, memory_order __m = memory_order_seq_cst) volatile noexcept 425 requires __base::is_always_lock_free 426 { 427 return __fetch_sub(*this, __op, __m); 428 } 429 430 _LIBCPP_HIDE_FROM_ABI _Tp fetch_sub(_Tp __op, memory_order __m = memory_order_seq_cst) noexcept { 431 return __fetch_sub(*this, __op, __m); 432 } 433 434 _LIBCPP_HIDE_FROM_ABI _Tp operator+=(_Tp __op) volatile noexcept 435 requires __base::is_always_lock_free 436 { 437 return fetch_add(__op) + __op; 438 } 439 440 _LIBCPP_HIDE_FROM_ABI _Tp operator+=(_Tp __op) noexcept { return fetch_add(__op) + __op; } 441 442 _LIBCPP_HIDE_FROM_ABI _Tp operator-=(_Tp __op) volatile noexcept 443 requires __base::is_always_lock_free 444 { 445 return fetch_sub(__op) - __op; 446 } 447 448 _LIBCPP_HIDE_FROM_ABI _Tp operator-=(_Tp __op) noexcept { return fetch_sub(__op) - __op; } 449 }; 450 451 #endif // _LIBCPP_STD_VER >= 20 452 453 // atomic_is_lock_free 454 455 template <class _Tp> 456 _LIBCPP_HIDE_FROM_ABI bool atomic_is_lock_free(const volatile atomic<_Tp>* __o) _NOEXCEPT { 457 return __o->is_lock_free(); 458 } 459 460 template <class _Tp> 461 _LIBCPP_HIDE_FROM_ABI bool atomic_is_lock_free(const atomic<_Tp>* __o) _NOEXCEPT { 462 return __o->is_lock_free(); 463 } 464 465 // atomic_init 466 467 template <class _Tp> 468 _LIBCPP_DEPRECATED_IN_CXX20 _LIBCPP_HIDE_FROM_ABI void 469 atomic_init(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d) _NOEXCEPT { 470 std::__cxx_atomic_init(std::addressof(__o->__a_), __d); 471 } 472 473 template <class _Tp> 474 _LIBCPP_DEPRECATED_IN_CXX20 _LIBCPP_HIDE_FROM_ABI void 475 atomic_init(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d) _NOEXCEPT { 476 std::__cxx_atomic_init(std::addressof(__o->__a_), __d); 477 } 478 479 // atomic_store 480 481 template <class _Tp> 482 _LIBCPP_HIDE_FROM_ABI void atomic_store(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d) _NOEXCEPT { 483 __o->store(__d); 484 } 485 486 template <class _Tp> 487 _LIBCPP_HIDE_FROM_ABI void atomic_store(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d) _NOEXCEPT { 488 __o->store(__d); 489 } 490 491 // atomic_store_explicit 492 493 template <class _Tp> 494 _LIBCPP_HIDE_FROM_ABI void 495 atomic_store_explicit(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d, memory_order __m) _NOEXCEPT 496 _LIBCPP_CHECK_STORE_MEMORY_ORDER(__m) { 497 __o->store(__d, __m); 498 } 499 500 template <class _Tp> 501 _LIBCPP_HIDE_FROM_ABI void 502 atomic_store_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d, memory_order __m) _NOEXCEPT 503 _LIBCPP_CHECK_STORE_MEMORY_ORDER(__m) { 504 __o->store(__d, __m); 505 } 506 507 // atomic_load 508 509 template <class _Tp> 510 _LIBCPP_HIDE_FROM_ABI _Tp atomic_load(const volatile atomic<_Tp>* __o) _NOEXCEPT { 511 return __o->load(); 512 } 513 514 template <class _Tp> 515 _LIBCPP_HIDE_FROM_ABI _Tp atomic_load(const atomic<_Tp>* __o) _NOEXCEPT { 516 return __o->load(); 517 } 518 519 // atomic_load_explicit 520 521 template <class _Tp> 522 _LIBCPP_HIDE_FROM_ABI _Tp atomic_load_explicit(const volatile atomic<_Tp>* __o, memory_order __m) _NOEXCEPT 523 _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m) { 524 return __o->load(__m); 525 } 526 527 template <class _Tp> 528 _LIBCPP_HIDE_FROM_ABI _Tp atomic_load_explicit(const atomic<_Tp>* __o, memory_order __m) _NOEXCEPT 529 _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m) { 530 return __o->load(__m); 531 } 532 533 // atomic_exchange 534 535 template <class _Tp> 536 _LIBCPP_HIDE_FROM_ABI _Tp atomic_exchange(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d) _NOEXCEPT { 537 return __o->exchange(__d); 538 } 539 540 template <class _Tp> 541 _LIBCPP_HIDE_FROM_ABI _Tp atomic_exchange(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d) _NOEXCEPT { 542 return __o->exchange(__d); 543 } 544 545 // atomic_exchange_explicit 546 547 template <class _Tp> 548 _LIBCPP_HIDE_FROM_ABI _Tp 549 atomic_exchange_explicit(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d, memory_order __m) _NOEXCEPT { 550 return __o->exchange(__d, __m); 551 } 552 553 template <class _Tp> 554 _LIBCPP_HIDE_FROM_ABI _Tp 555 atomic_exchange_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d, memory_order __m) _NOEXCEPT { 556 return __o->exchange(__d, __m); 557 } 558 559 // atomic_compare_exchange_weak 560 561 template <class _Tp> 562 _LIBCPP_HIDE_FROM_ABI bool atomic_compare_exchange_weak( 563 volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type* __e, typename atomic<_Tp>::value_type __d) _NOEXCEPT { 564 return __o->compare_exchange_weak(*__e, __d); 565 } 566 567 template <class _Tp> 568 _LIBCPP_HIDE_FROM_ABI bool atomic_compare_exchange_weak( 569 atomic<_Tp>* __o, typename atomic<_Tp>::value_type* __e, typename atomic<_Tp>::value_type __d) _NOEXCEPT { 570 return __o->compare_exchange_weak(*__e, __d); 571 } 572 573 // atomic_compare_exchange_strong 574 575 template <class _Tp> 576 _LIBCPP_HIDE_FROM_ABI bool atomic_compare_exchange_strong( 577 volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type* __e, typename atomic<_Tp>::value_type __d) _NOEXCEPT { 578 return __o->compare_exchange_strong(*__e, __d); 579 } 580 581 template <class _Tp> 582 _LIBCPP_HIDE_FROM_ABI bool atomic_compare_exchange_strong( 583 atomic<_Tp>* __o, typename atomic<_Tp>::value_type* __e, typename atomic<_Tp>::value_type __d) _NOEXCEPT { 584 return __o->compare_exchange_strong(*__e, __d); 585 } 586 587 // atomic_compare_exchange_weak_explicit 588 589 template <class _Tp> 590 _LIBCPP_HIDE_FROM_ABI bool atomic_compare_exchange_weak_explicit( 591 volatile atomic<_Tp>* __o, 592 typename atomic<_Tp>::value_type* __e, 593 typename atomic<_Tp>::value_type __d, 594 memory_order __s, 595 memory_order __f) _NOEXCEPT _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) { 596 return __o->compare_exchange_weak(*__e, __d, __s, __f); 597 } 598 599 template <class _Tp> 600 _LIBCPP_HIDE_FROM_ABI bool atomic_compare_exchange_weak_explicit( 601 atomic<_Tp>* __o, 602 typename atomic<_Tp>::value_type* __e, 603 typename atomic<_Tp>::value_type __d, 604 memory_order __s, 605 memory_order __f) _NOEXCEPT _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) { 606 return __o->compare_exchange_weak(*__e, __d, __s, __f); 607 } 608 609 // atomic_compare_exchange_strong_explicit 610 611 template <class _Tp> 612 _LIBCPP_HIDE_FROM_ABI bool atomic_compare_exchange_strong_explicit( 613 volatile atomic<_Tp>* __o, 614 typename atomic<_Tp>::value_type* __e, 615 typename atomic<_Tp>::value_type __d, 616 memory_order __s, 617 memory_order __f) _NOEXCEPT _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) { 618 return __o->compare_exchange_strong(*__e, __d, __s, __f); 619 } 620 621 template <class _Tp> 622 _LIBCPP_HIDE_FROM_ABI bool atomic_compare_exchange_strong_explicit( 623 atomic<_Tp>* __o, 624 typename atomic<_Tp>::value_type* __e, 625 typename atomic<_Tp>::value_type __d, 626 memory_order __s, 627 memory_order __f) _NOEXCEPT _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) { 628 return __o->compare_exchange_strong(*__e, __d, __s, __f); 629 } 630 631 #if _LIBCPP_STD_VER >= 20 632 633 // atomic_wait 634 635 template <class _Tp> 636 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void 637 atomic_wait(const volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __v) _NOEXCEPT { 638 return __o->wait(__v); 639 } 640 641 template <class _Tp> 642 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void 643 atomic_wait(const atomic<_Tp>* __o, typename atomic<_Tp>::value_type __v) _NOEXCEPT { 644 return __o->wait(__v); 645 } 646 647 // atomic_wait_explicit 648 649 template <class _Tp> 650 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void 651 atomic_wait_explicit(const volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __v, memory_order __m) _NOEXCEPT 652 _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m) { 653 return __o->wait(__v, __m); 654 } 655 656 template <class _Tp> 657 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void 658 atomic_wait_explicit(const atomic<_Tp>* __o, typename atomic<_Tp>::value_type __v, memory_order __m) _NOEXCEPT 659 _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m) { 660 return __o->wait(__v, __m); 661 } 662 663 // atomic_notify_one 664 665 template <class _Tp> 666 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void atomic_notify_one(volatile atomic<_Tp>* __o) _NOEXCEPT { 667 __o->notify_one(); 668 } 669 template <class _Tp> 670 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void atomic_notify_one(atomic<_Tp>* __o) _NOEXCEPT { 671 __o->notify_one(); 672 } 673 674 // atomic_notify_all 675 676 template <class _Tp> 677 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void atomic_notify_all(volatile atomic<_Tp>* __o) _NOEXCEPT { 678 __o->notify_all(); 679 } 680 template <class _Tp> 681 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void atomic_notify_all(atomic<_Tp>* __o) _NOEXCEPT { 682 __o->notify_all(); 683 } 684 685 #endif // _LIBCPP_STD_VER >= 20 686 687 // atomic_fetch_add 688 689 template <class _Tp> 690 _LIBCPP_HIDE_FROM_ABI _Tp 691 atomic_fetch_add(volatile atomic<_Tp>* __o, typename atomic<_Tp>::difference_type __op) _NOEXCEPT { 692 return __o->fetch_add(__op); 693 } 694 695 template <class _Tp> 696 _LIBCPP_HIDE_FROM_ABI _Tp atomic_fetch_add(atomic<_Tp>* __o, typename atomic<_Tp>::difference_type __op) _NOEXCEPT { 697 return __o->fetch_add(__op); 698 } 699 700 // atomic_fetch_add_explicit 701 702 template <class _Tp> 703 _LIBCPP_HIDE_FROM_ABI _Tp atomic_fetch_add_explicit( 704 volatile atomic<_Tp>* __o, typename atomic<_Tp>::difference_type __op, memory_order __m) _NOEXCEPT { 705 return __o->fetch_add(__op, __m); 706 } 707 708 template <class _Tp> 709 _LIBCPP_HIDE_FROM_ABI _Tp 710 atomic_fetch_add_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::difference_type __op, memory_order __m) _NOEXCEPT { 711 return __o->fetch_add(__op, __m); 712 } 713 714 // atomic_fetch_sub 715 716 template <class _Tp> 717 _LIBCPP_HIDE_FROM_ABI _Tp 718 atomic_fetch_sub(volatile atomic<_Tp>* __o, typename atomic<_Tp>::difference_type __op) _NOEXCEPT { 719 return __o->fetch_sub(__op); 720 } 721 722 template <class _Tp> 723 _LIBCPP_HIDE_FROM_ABI _Tp atomic_fetch_sub(atomic<_Tp>* __o, typename atomic<_Tp>::difference_type __op) _NOEXCEPT { 724 return __o->fetch_sub(__op); 725 } 726 727 // atomic_fetch_sub_explicit 728 729 template <class _Tp> 730 _LIBCPP_HIDE_FROM_ABI _Tp atomic_fetch_sub_explicit( 731 volatile atomic<_Tp>* __o, typename atomic<_Tp>::difference_type __op, memory_order __m) _NOEXCEPT { 732 return __o->fetch_sub(__op, __m); 733 } 734 735 template <class _Tp> 736 _LIBCPP_HIDE_FROM_ABI _Tp 737 atomic_fetch_sub_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::difference_type __op, memory_order __m) _NOEXCEPT { 738 return __o->fetch_sub(__op, __m); 739 } 740 741 // atomic_fetch_and 742 743 template <class _Tp, __enable_if_t<is_integral<_Tp>::value && !is_same<_Tp, bool>::value, int> = 0> 744 _LIBCPP_HIDE_FROM_ABI _Tp atomic_fetch_and(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op) _NOEXCEPT { 745 return __o->fetch_and(__op); 746 } 747 748 template <class _Tp, __enable_if_t<is_integral<_Tp>::value && !is_same<_Tp, bool>::value, int> = 0> 749 _LIBCPP_HIDE_FROM_ABI _Tp atomic_fetch_and(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op) _NOEXCEPT { 750 return __o->fetch_and(__op); 751 } 752 753 // atomic_fetch_and_explicit 754 755 template <class _Tp, __enable_if_t<is_integral<_Tp>::value && !is_same<_Tp, bool>::value, int> = 0> 756 _LIBCPP_HIDE_FROM_ABI _Tp atomic_fetch_and_explicit( 757 volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op, memory_order __m) _NOEXCEPT { 758 return __o->fetch_and(__op, __m); 759 } 760 761 template <class _Tp, __enable_if_t<is_integral<_Tp>::value && !is_same<_Tp, bool>::value, int> = 0> 762 _LIBCPP_HIDE_FROM_ABI _Tp 763 atomic_fetch_and_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op, memory_order __m) _NOEXCEPT { 764 return __o->fetch_and(__op, __m); 765 } 766 767 // atomic_fetch_or 768 769 template <class _Tp, __enable_if_t<is_integral<_Tp>::value && !is_same<_Tp, bool>::value, int> = 0> 770 _LIBCPP_HIDE_FROM_ABI _Tp atomic_fetch_or(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op) _NOEXCEPT { 771 return __o->fetch_or(__op); 772 } 773 774 template <class _Tp, __enable_if_t<is_integral<_Tp>::value && !is_same<_Tp, bool>::value, int> = 0> 775 _LIBCPP_HIDE_FROM_ABI _Tp atomic_fetch_or(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op) _NOEXCEPT { 776 return __o->fetch_or(__op); 777 } 778 779 // atomic_fetch_or_explicit 780 781 template <class _Tp, __enable_if_t<is_integral<_Tp>::value && !is_same<_Tp, bool>::value, int> = 0> 782 _LIBCPP_HIDE_FROM_ABI _Tp 783 atomic_fetch_or_explicit(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op, memory_order __m) _NOEXCEPT { 784 return __o->fetch_or(__op, __m); 785 } 786 787 template <class _Tp, __enable_if_t<is_integral<_Tp>::value && !is_same<_Tp, bool>::value, int> = 0> 788 _LIBCPP_HIDE_FROM_ABI _Tp 789 atomic_fetch_or_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op, memory_order __m) _NOEXCEPT { 790 return __o->fetch_or(__op, __m); 791 } 792 793 // atomic_fetch_xor 794 795 template <class _Tp, __enable_if_t<is_integral<_Tp>::value && !is_same<_Tp, bool>::value, int> = 0> 796 _LIBCPP_HIDE_FROM_ABI _Tp atomic_fetch_xor(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op) _NOEXCEPT { 797 return __o->fetch_xor(__op); 798 } 799 800 template <class _Tp, __enable_if_t<is_integral<_Tp>::value && !is_same<_Tp, bool>::value, int> = 0> 801 _LIBCPP_HIDE_FROM_ABI _Tp atomic_fetch_xor(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op) _NOEXCEPT { 802 return __o->fetch_xor(__op); 803 } 804 805 // atomic_fetch_xor_explicit 806 807 template <class _Tp, __enable_if_t<is_integral<_Tp>::value && !is_same<_Tp, bool>::value, int> = 0> 808 _LIBCPP_HIDE_FROM_ABI _Tp atomic_fetch_xor_explicit( 809 volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op, memory_order __m) _NOEXCEPT { 810 return __o->fetch_xor(__op, __m); 811 } 812 813 template <class _Tp, __enable_if_t<is_integral<_Tp>::value && !is_same<_Tp, bool>::value, int> = 0> 814 _LIBCPP_HIDE_FROM_ABI _Tp 815 atomic_fetch_xor_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op, memory_order __m) _NOEXCEPT { 816 return __o->fetch_xor(__op, __m); 817 } 818 819 _LIBCPP_END_NAMESPACE_STD 820 821 #endif // _LIBCPP___ATOMIC_ATOMIC_H 822