1 //===----------------------------------------------------------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8
9 #include <__config>
10 #ifdef _LIBCPP_DEPRECATED_ABI_LEGACY_LIBRARY_DEFINITIONS_FOR_INLINE_FUNCTIONS
11 # define _LIBCPP_SHARED_PTR_DEFINE_LEGACY_INLINE_FUNCTIONS
12 #endif
13
14 #include <memory>
15
16 #ifndef _LIBCPP_HAS_NO_THREADS
17 # include <mutex>
18 # include <thread>
19 # if defined(__ELF__) && defined(_LIBCPP_LINK_PTHREAD_LIB)
20 # pragma comment(lib, "pthread")
21 # endif
22 #endif
23
24 #include "include/atomic_support.h"
25
26 _LIBCPP_BEGIN_NAMESPACE_STD
27
~bad_weak_ptr()28 bad_weak_ptr::~bad_weak_ptr() noexcept {}
29
what() const30 const char* bad_weak_ptr::what() const noexcept { return "bad_weak_ptr"; }
31
~__shared_count()32 __shared_count::~__shared_count() {}
33
~__shared_weak_count()34 __shared_weak_count::~__shared_weak_count() {}
35
36 #if defined(_LIBCPP_SHARED_PTR_DEFINE_LEGACY_INLINE_FUNCTIONS)
__add_shared()37 void __shared_count::__add_shared() noexcept { __libcpp_atomic_refcount_increment(__shared_owners_); }
38
__release_shared()39 bool __shared_count::__release_shared() noexcept {
40 if (__libcpp_atomic_refcount_decrement(__shared_owners_) == -1) {
41 __on_zero_shared();
42 return true;
43 }
44 return false;
45 }
46
__add_shared()47 void __shared_weak_count::__add_shared() noexcept { __shared_count::__add_shared(); }
48
__add_weak()49 void __shared_weak_count::__add_weak() noexcept { __libcpp_atomic_refcount_increment(__shared_weak_owners_); }
50
__release_shared()51 void __shared_weak_count::__release_shared() noexcept {
52 if (__shared_count::__release_shared())
53 __release_weak();
54 }
55 #endif // _LIBCPP_SHARED_PTR_DEFINE_LEGACY_INLINE_FUNCTIONS
56
__release_weak()57 void __shared_weak_count::__release_weak() noexcept {
58 // NOTE: The acquire load here is an optimization of the very
59 // common case where a shared pointer is being destructed while
60 // having no other contended references.
61 //
62 // BENEFIT: We avoid expensive atomic stores like XADD and STREX
63 // in a common case. Those instructions are slow and do nasty
64 // things to caches.
65 //
66 // IS THIS SAFE? Yes. During weak destruction, if we see that we
67 // are the last reference, we know that no-one else is accessing
68 // us. If someone were accessing us, then they would be doing so
69 // while the last shared / weak_ptr was being destructed, and
70 // that's undefined anyway.
71 //
72 // If we see anything other than a 0, then we have possible
73 // contention, and need to use an atomicrmw primitive.
74 // The same arguments don't apply for increment, where it is legal
75 // (though inadvisable) to share shared_ptr references between
76 // threads, and have them all get copied at once. The argument
77 // also doesn't apply for __release_shared, because an outstanding
78 // weak_ptr::lock() could read / modify the shared count.
79 if (__libcpp_atomic_load(&__shared_weak_owners_, _AO_Acquire) == 0) {
80 // no need to do this store, because we are about
81 // to destroy everything.
82 //__libcpp_atomic_store(&__shared_weak_owners_, -1, _AO_Release);
83 __on_zero_shared_weak();
84 } else if (__libcpp_atomic_refcount_decrement(__shared_weak_owners_) == -1)
85 __on_zero_shared_weak();
86 }
87
lock()88 __shared_weak_count* __shared_weak_count::lock() noexcept {
89 long object_owners = __libcpp_atomic_load(&__shared_owners_);
90 while (object_owners != -1) {
91 if (__libcpp_atomic_compare_exchange(&__shared_owners_, &object_owners, object_owners + 1))
92 return this;
93 }
94 return nullptr;
95 }
96
__get_deleter(const type_info &) const97 const void* __shared_weak_count::__get_deleter(const type_info&) const noexcept { return nullptr; }
98
99 #if !defined(_LIBCPP_HAS_NO_THREADS)
100
101 static constexpr std::size_t __sp_mut_count = 32;
102 static constinit __libcpp_mutex_t mut_back[__sp_mut_count] = {
103 _LIBCPP_MUTEX_INITIALIZER, _LIBCPP_MUTEX_INITIALIZER, _LIBCPP_MUTEX_INITIALIZER, _LIBCPP_MUTEX_INITIALIZER,
104 _LIBCPP_MUTEX_INITIALIZER, _LIBCPP_MUTEX_INITIALIZER, _LIBCPP_MUTEX_INITIALIZER, _LIBCPP_MUTEX_INITIALIZER,
105 _LIBCPP_MUTEX_INITIALIZER, _LIBCPP_MUTEX_INITIALIZER, _LIBCPP_MUTEX_INITIALIZER, _LIBCPP_MUTEX_INITIALIZER,
106 _LIBCPP_MUTEX_INITIALIZER, _LIBCPP_MUTEX_INITIALIZER, _LIBCPP_MUTEX_INITIALIZER, _LIBCPP_MUTEX_INITIALIZER,
107 _LIBCPP_MUTEX_INITIALIZER, _LIBCPP_MUTEX_INITIALIZER, _LIBCPP_MUTEX_INITIALIZER, _LIBCPP_MUTEX_INITIALIZER,
108 _LIBCPP_MUTEX_INITIALIZER, _LIBCPP_MUTEX_INITIALIZER, _LIBCPP_MUTEX_INITIALIZER, _LIBCPP_MUTEX_INITIALIZER,
109 _LIBCPP_MUTEX_INITIALIZER, _LIBCPP_MUTEX_INITIALIZER, _LIBCPP_MUTEX_INITIALIZER, _LIBCPP_MUTEX_INITIALIZER,
110 _LIBCPP_MUTEX_INITIALIZER, _LIBCPP_MUTEX_INITIALIZER, _LIBCPP_MUTEX_INITIALIZER, _LIBCPP_MUTEX_INITIALIZER};
111
__sp_mut(void * p)112 constexpr __sp_mut::__sp_mut(void* p) noexcept : __lx_(p) {}
113
lock()114 void __sp_mut::lock() noexcept {
115 auto m = static_cast<__libcpp_mutex_t*>(__lx_);
116 __libcpp_mutex_lock(m);
117 }
118
unlock()119 void __sp_mut::unlock() noexcept { __libcpp_mutex_unlock(static_cast<__libcpp_mutex_t*>(__lx_)); }
120
__get_sp_mut(const void * p)121 __sp_mut& __get_sp_mut(const void* p) {
122 static constinit __sp_mut muts[__sp_mut_count] = {
123 &mut_back[0], &mut_back[1], &mut_back[2], &mut_back[3], &mut_back[4], &mut_back[5], &mut_back[6],
124 &mut_back[7], &mut_back[8], &mut_back[9], &mut_back[10], &mut_back[11], &mut_back[12], &mut_back[13],
125 &mut_back[14], &mut_back[15], &mut_back[16], &mut_back[17], &mut_back[18], &mut_back[19], &mut_back[20],
126 &mut_back[21], &mut_back[22], &mut_back[23], &mut_back[24], &mut_back[25], &mut_back[26], &mut_back[27],
127 &mut_back[28], &mut_back[29], &mut_back[30], &mut_back[31]};
128 return muts[hash<const void*>()(p) & (__sp_mut_count - 1)];
129 }
130
131 #endif // !defined(_LIBCPP_HAS_NO_THREADS)
132
align(size_t alignment,size_t size,void * & ptr,size_t & space)133 void* align(size_t alignment, size_t size, void*& ptr, size_t& space) {
134 void* r = nullptr;
135 if (size <= space) {
136 char* p1 = static_cast<char*>(ptr);
137 char* p2 = reinterpret_cast<char*>(reinterpret_cast<uintptr_t>(p1 + (alignment - 1)) & -alignment);
138 size_t d = static_cast<size_t>(p2 - p1);
139 if (d <= space - size) {
140 r = p2;
141 ptr = r;
142 space -= d;
143 }
144 }
145 return r;
146 }
147
148 _LIBCPP_END_NAMESPACE_STD
149