1 //===----------------------------------------------------------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8
9 #ifndef LIBCXXABI_SRC_INCLUDE_CXA_GUARD_IMPL_H
10 #define LIBCXXABI_SRC_INCLUDE_CXA_GUARD_IMPL_H
11
12 /* cxa_guard_impl.h - Implements the C++ runtime support for function local
13 * static guards.
14 * The layout of the guard object is the same across ARM and Itanium.
15 *
16 * The first "guard byte" (which is checked by the compiler) is set only upon
17 * the completion of cxa release.
18 *
19 * The second "init byte" does the rest of the bookkeeping. It tracks if
20 * initialization is complete or pending, and if there are waiting threads.
21 *
22 * If the guard variable is 64-bits and the platforms supplies a 32-bit thread
23 * identifier, it is used to detect recursive initialization. The thread ID of
24 * the thread currently performing initialization is stored in the second word.
25 *
26 * Guard Object Layout:
27 * ---------------------------------------------------------------------------
28 * | a+0: guard byte | a+1: init byte | a+2: unused ... | a+4: thread-id ... |
29 * ---------------------------------------------------------------------------
30 *
31 * Note that we don't do what the ABI docs suggest (put a mutex in the guard
32 * object which we acquire in cxa_guard_acquire and release in
33 * cxa_guard_release). Instead we use the init byte to imitate that behaviour,
34 * but without actually holding anything mutex related between aquire and
35 * release/abort.
36 *
37 * Access Protocol:
38 * For each implementation the guard byte is checked and set before accessing
39 * the init byte.
40 *
41 * Overall Design:
42 * The implementation was designed to allow each implementation to be tested
43 * independent of the C++ runtime or platform support.
44 *
45 */
46
47 #include "__cxxabi_config.h"
48 #include "include/atomic_support.h" // from libc++
49 #if defined(__has_include)
50 # if __has_include(<sys/syscall.h>)
51 # include <sys/syscall.h>
52 # endif
53 # if __has_include(<unistd.h>)
54 # include <unistd.h>
55 # endif
56 #endif
57
58 #include <__threading_support>
59 #include <cstdint>
60 #include <cstring>
61 #include <limits.h>
62 #include <stdlib.h>
63
64 #ifndef _LIBCXXABI_HAS_NO_THREADS
65 # if defined(__ELF__) && defined(_LIBCXXABI_LINK_PTHREAD_LIB)
66 # pragma comment(lib, "pthread")
67 # endif
68 #endif
69
70 #if defined(__clang__)
71 # pragma clang diagnostic push
72 # pragma clang diagnostic ignored "-Wtautological-pointer-compare"
73 #elif defined(__GNUC__)
74 # pragma GCC diagnostic push
75 # pragma GCC diagnostic ignored "-Waddress"
76 #endif
77
78 // To make testing possible, this header is included from both cxa_guard.cpp
79 // and a number of tests.
80 //
81 // For this reason we place everything in an anonymous namespace -- even though
82 // we're in a header. We want the actual implementation and the tests to have
83 // unique definitions of the types in this header (since the tests may depend
84 // on function local statics).
85 //
86 // To enforce this either `BUILDING_CXA_GUARD` or `TESTING_CXA_GUARD` must be
87 // defined when including this file. Only `src/cxa_guard.cpp` should define
88 // the former.
89 #ifdef BUILDING_CXA_GUARD
90 # include "abort_message.h"
91 # define ABORT_WITH_MESSAGE(...) ::abort_message(__VA_ARGS__)
92 #elif defined(TESTING_CXA_GUARD)
93 # define ABORT_WITH_MESSAGE(...) ::abort()
94 #else
95 # error "Either BUILDING_CXA_GUARD or TESTING_CXA_GUARD must be defined"
96 #endif
97
98 #if __has_feature(thread_sanitizer)
99 extern "C" void __tsan_acquire(void*);
100 extern "C" void __tsan_release(void*);
101 #else
102 # define __tsan_acquire(addr) ((void)0)
103 # define __tsan_release(addr) ((void)0)
104 #endif
105
106 namespace __cxxabiv1 {
107 // Use an anonymous namespace to ensure that the tests and actual implementation
108 // have unique definitions of these symbols.
109 namespace {
110
111 //===----------------------------------------------------------------------===//
112 // Misc Utilities
113 //===----------------------------------------------------------------------===//
114
115 template <class T, T (*Init)()>
116 struct LazyValue {
LazyValueLazyValue117 LazyValue() : is_init(false) {}
118
getLazyValue119 T& get() {
120 if (!is_init) {
121 value = Init();
122 is_init = true;
123 }
124 return value;
125 }
126
127 private:
128 T value;
129 bool is_init = false;
130 };
131
132 template <class IntType>
133 class AtomicInt {
134 public:
135 using MemoryOrder = std::__libcpp_atomic_order;
136
AtomicInt(IntType * b)137 explicit AtomicInt(IntType* b) : b_(b) {}
138 AtomicInt(AtomicInt const&) = delete;
139 AtomicInt& operator=(AtomicInt const&) = delete;
140
load(MemoryOrder ord)141 IntType load(MemoryOrder ord) { return std::__libcpp_atomic_load(b_, ord); }
store(IntType val,MemoryOrder ord)142 void store(IntType val, MemoryOrder ord) { std::__libcpp_atomic_store(b_, val, ord); }
exchange(IntType new_val,MemoryOrder ord)143 IntType exchange(IntType new_val, MemoryOrder ord) { return std::__libcpp_atomic_exchange(b_, new_val, ord); }
compare_exchange(IntType * expected,IntType desired,MemoryOrder ord_success,MemoryOrder ord_failure)144 bool compare_exchange(IntType* expected, IntType desired, MemoryOrder ord_success, MemoryOrder ord_failure) {
145 return std::__libcpp_atomic_compare_exchange(b_, expected, desired, ord_success, ord_failure);
146 }
147
148 private:
149 IntType* b_;
150 };
151
152 //===----------------------------------------------------------------------===//
153 // PlatformGetThreadID
154 //===----------------------------------------------------------------------===//
155
156 #if defined(__APPLE__) && defined(_LIBCPP_HAS_THREAD_API_PTHREAD)
PlatformThreadID()157 uint32_t PlatformThreadID() {
158 static_assert(sizeof(mach_port_t) == sizeof(uint32_t), "");
159 return static_cast<uint32_t>(pthread_mach_thread_np(std::__libcpp_thread_get_current_id()));
160 }
161 #elif defined(SYS_gettid) && defined(_LIBCPP_HAS_THREAD_API_PTHREAD)
PlatformThreadID()162 uint32_t PlatformThreadID() {
163 static_assert(sizeof(pid_t) == sizeof(uint32_t), "");
164 return static_cast<uint32_t>(syscall(SYS_gettid));
165 }
166 #else
167 constexpr uint32_t (*PlatformThreadID)() = nullptr;
168 #endif
169
170 //===----------------------------------------------------------------------===//
171 // GuardByte
172 //===----------------------------------------------------------------------===//
173
174 static constexpr uint8_t UNSET = 0;
175 static constexpr uint8_t COMPLETE_BIT = (1 << 0);
176 static constexpr uint8_t PENDING_BIT = (1 << 1);
177 static constexpr uint8_t WAITING_BIT = (1 << 2);
178
179 /// Manages reads and writes to the guard byte.
180 struct GuardByte {
181 GuardByte() = delete;
182 GuardByte(GuardByte const&) = delete;
183 GuardByte& operator=(GuardByte const&) = delete;
184
GuardByteGuardByte185 explicit GuardByte(uint8_t* const guard_byte_address) : guard_byte(guard_byte_address) {}
186
187 public:
188 /// The guard byte portion of cxa_guard_acquire. Returns true if
189 /// initialization has already been completed.
acquireGuardByte190 bool acquire() {
191 // if guard_byte is non-zero, we have already completed initialization
192 // (i.e. release has been called)
193 return guard_byte.load(std::_AO_Acquire) != UNSET;
194 }
195
196 /// The guard byte portion of cxa_guard_release.
releaseGuardByte197 void release() { guard_byte.store(COMPLETE_BIT, std::_AO_Release); }
198
199 /// The guard byte portion of cxa_guard_abort.
abortGuardByte200 void abort() {} // Nothing to do
201
202 private:
203 AtomicInt<uint8_t> guard_byte;
204 };
205
206 //===----------------------------------------------------------------------===//
207 // InitByte Implementations
208 //===----------------------------------------------------------------------===//
209 //
210 // Each initialization byte implementation supports the following methods:
211 //
212 // InitByte(uint8_t* _init_byte_address, uint32_t* _thread_id_address)
213 // Construct the InitByte object, initializing our member variables
214 //
215 // bool acquire()
216 // Called before we start the initialization. Check if someone else has already started, and if
217 // not to signal our intent to start it ourselves. We determine the current status from the init
218 // byte, which is one of 4 possible values:
219 // COMPLETE: Initialization was finished by somebody else. Return true.
220 // PENDING: Somebody has started the initialization already, set the WAITING bit,
221 // then wait for the init byte to get updated with a new value.
222 // (PENDING|WAITING): Somebody has started the initialization already, and we're not the
223 // first one waiting. Wait for the init byte to get updated.
224 // UNSET: Initialization hasn't successfully completed, and nobody is currently
225 // performing the initialization. Set the PENDING bit to indicate our
226 // intention to start the initialization, and return false.
227 // The return value indicates whether initialization has already been completed.
228 //
229 // void release()
230 // Called after successfully completing the initialization. Update the init byte to reflect
231 // that, then if anybody else is waiting, wake them up.
232 //
233 // void abort()
234 // Called after an error is thrown during the initialization. Reset the init byte to UNSET to
235 // indicate that we're no longer performing the initialization, then if anybody is waiting, wake
236 // them up so they can try performing the initialization.
237 //
238
239 //===----------------------------------------------------------------------===//
240 // Single Threaded Implementation
241 //===----------------------------------------------------------------------===//
242
243 /// InitByteNoThreads - Doesn't use any inter-thread synchronization when
244 /// managing reads and writes to the init byte.
245 struct InitByteNoThreads {
246 InitByteNoThreads() = delete;
247 InitByteNoThreads(InitByteNoThreads const&) = delete;
248 InitByteNoThreads& operator=(InitByteNoThreads const&) = delete;
249
InitByteNoThreadsInitByteNoThreads250 explicit InitByteNoThreads(uint8_t* _init_byte_address, uint32_t*) : init_byte_address(_init_byte_address) {}
251
252 /// The init byte portion of cxa_guard_acquire. Returns true if
253 /// initialization has already been completed.
acquireInitByteNoThreads254 bool acquire() {
255 if (*init_byte_address == COMPLETE_BIT)
256 return true;
257 if (*init_byte_address & PENDING_BIT)
258 ABORT_WITH_MESSAGE("__cxa_guard_acquire detected recursive initialization");
259 *init_byte_address = PENDING_BIT;
260 return false;
261 }
262
263 /// The init byte portion of cxa_guard_release.
releaseInitByteNoThreads264 void release() { *init_byte_address = COMPLETE_BIT; }
265 /// The init byte portion of cxa_guard_abort.
abortInitByteNoThreads266 void abort() { *init_byte_address = UNSET; }
267
268 private:
269 /// The address of the byte used during initialization.
270 uint8_t* const init_byte_address;
271 };
272
273 //===----------------------------------------------------------------------===//
274 // Global Mutex Implementation
275 //===----------------------------------------------------------------------===//
276
277 struct LibcppMutex;
278 struct LibcppCondVar;
279
280 #ifndef _LIBCXXABI_HAS_NO_THREADS
281 struct LibcppMutex {
282 LibcppMutex() = default;
283 LibcppMutex(LibcppMutex const&) = delete;
284 LibcppMutex& operator=(LibcppMutex const&) = delete;
285
lockLibcppMutex286 bool lock() { return std::__libcpp_mutex_lock(&mutex); }
unlockLibcppMutex287 bool unlock() { return std::__libcpp_mutex_unlock(&mutex); }
288
289 private:
290 friend struct LibcppCondVar;
291 std::__libcpp_mutex_t mutex = _LIBCPP_MUTEX_INITIALIZER;
292 };
293
294 struct LibcppCondVar {
295 LibcppCondVar() = default;
296 LibcppCondVar(LibcppCondVar const&) = delete;
297 LibcppCondVar& operator=(LibcppCondVar const&) = delete;
298
waitLibcppCondVar299 bool wait(LibcppMutex& mut) { return std::__libcpp_condvar_wait(&cond, &mut.mutex); }
broadcastLibcppCondVar300 bool broadcast() { return std::__libcpp_condvar_broadcast(&cond); }
301
302 private:
303 std::__libcpp_condvar_t cond = _LIBCPP_CONDVAR_INITIALIZER;
304 };
305 #else
306 struct LibcppMutex {};
307 struct LibcppCondVar {};
308 #endif // !defined(_LIBCXXABI_HAS_NO_THREADS)
309
310 /// InitByteGlobalMutex - Uses a global mutex and condition variable (common to
311 /// all static local variables) to manage reads and writes to the init byte.
312 template <class Mutex, class CondVar, Mutex& global_mutex, CondVar& global_cond,
313 uint32_t (*GetThreadID)() = PlatformThreadID>
314 struct InitByteGlobalMutex {
315
InitByteGlobalMutexInitByteGlobalMutex316 explicit InitByteGlobalMutex(uint8_t* _init_byte_address, uint32_t* _thread_id_address)
317 : init_byte_address(_init_byte_address), thread_id_address(_thread_id_address),
318 has_thread_id_support(_thread_id_address != nullptr && GetThreadID != nullptr) {}
319
320 public:
321 /// The init byte portion of cxa_guard_acquire. Returns true if
322 /// initialization has already been completed.
acquireInitByteGlobalMutex323 bool acquire() {
324 LockGuard g("__cxa_guard_acquire");
325 // Check for possible recursive initialization.
326 if (has_thread_id_support && (*init_byte_address & PENDING_BIT)) {
327 if (*thread_id_address == current_thread_id.get())
328 ABORT_WITH_MESSAGE("__cxa_guard_acquire detected recursive initialization");
329 }
330
331 // Wait until the pending bit is not set.
332 while (*init_byte_address & PENDING_BIT) {
333 *init_byte_address |= WAITING_BIT;
334 global_cond.wait(global_mutex);
335 }
336
337 if (*init_byte_address == COMPLETE_BIT)
338 return true;
339
340 if (has_thread_id_support)
341 *thread_id_address = current_thread_id.get();
342
343 *init_byte_address = PENDING_BIT;
344 return false;
345 }
346
347 /// The init byte portion of cxa_guard_release.
releaseInitByteGlobalMutex348 void release() {
349 bool has_waiting;
350 {
351 LockGuard g("__cxa_guard_release");
352 has_waiting = *init_byte_address & WAITING_BIT;
353 *init_byte_address = COMPLETE_BIT;
354 }
355 if (has_waiting) {
356 if (global_cond.broadcast()) {
357 ABORT_WITH_MESSAGE("%s failed to broadcast", "__cxa_guard_release");
358 }
359 }
360 }
361
362 /// The init byte portion of cxa_guard_abort.
abortInitByteGlobalMutex363 void abort() {
364 bool has_waiting;
365 {
366 LockGuard g("__cxa_guard_abort");
367 if (has_thread_id_support)
368 *thread_id_address = 0;
369 has_waiting = *init_byte_address & WAITING_BIT;
370 *init_byte_address = UNSET;
371 }
372 if (has_waiting) {
373 if (global_cond.broadcast()) {
374 ABORT_WITH_MESSAGE("%s failed to broadcast", "__cxa_guard_abort");
375 }
376 }
377 }
378
379 private:
380 /// The address of the byte used during initialization.
381 uint8_t* const init_byte_address;
382 /// An optional address storing an identifier for the thread performing initialization.
383 /// It's used to detect recursive initialization.
384 uint32_t* const thread_id_address;
385
386 const bool has_thread_id_support;
387 LazyValue<uint32_t, GetThreadID> current_thread_id;
388
389 private:
390 struct LockGuard {
391 LockGuard() = delete;
392 LockGuard(LockGuard const&) = delete;
393 LockGuard& operator=(LockGuard const&) = delete;
394
LockGuardInitByteGlobalMutex::LockGuard395 explicit LockGuard(const char* calling_func) : calling_func_(calling_func) {
396 if (global_mutex.lock())
397 ABORT_WITH_MESSAGE("%s failed to acquire mutex", calling_func_);
398 }
399
~LockGuardInitByteGlobalMutex::LockGuard400 ~LockGuard() {
401 if (global_mutex.unlock())
402 ABORT_WITH_MESSAGE("%s failed to release mutex", calling_func_);
403 }
404
405 private:
406 const char* const calling_func_;
407 };
408 };
409
410 //===----------------------------------------------------------------------===//
411 // Futex Implementation
412 //===----------------------------------------------------------------------===//
413
414 #ifdef __OpenBSD__
415 #include <sys/futex.h>
416
PlatformFutexWait(int * addr,int expect)417 void PlatformFutexWait(int* addr, int expect) {
418 constexpr int WAIT = 0;
419 futex((volatile uint32_t *)addr, WAIT, expect, NULL, NULL);
420 __tsan_acquire(addr);
421 }
PlatformFutexWake(int * addr)422 void PlatformFutexWake(int* addr) {
423 constexpr int WAKE = 1;
424 __tsan_release(addr);
425 futex((volatile uint32_t *)addr, WAKE, INT_MAX, NULL, NULL);
426 }
427 #elif defined(SYS_futex)
PlatformFutexWait(int * addr,int expect)428 void PlatformFutexWait(int* addr, int expect) {
429 constexpr int WAIT = 0;
430 syscall(SYS_futex, addr, WAIT, expect, 0);
431 __tsan_acquire(addr);
432 }
PlatformFutexWake(int * addr)433 void PlatformFutexWake(int* addr) {
434 constexpr int WAKE = 1;
435 __tsan_release(addr);
436 syscall(SYS_futex, addr, WAKE, INT_MAX);
437 }
438 #else
439 constexpr void (*PlatformFutexWait)(int*, int) = nullptr;
440 constexpr void (*PlatformFutexWake)(int*) = nullptr;
441 #endif
442
PlatformSupportsFutex()443 constexpr bool PlatformSupportsFutex() { return +PlatformFutexWait != nullptr; }
444
445 /// InitByteFutex - Uses a futex to manage reads and writes to the init byte.
446 template <void (*Wait)(int*, int) = PlatformFutexWait, void (*Wake)(int*) = PlatformFutexWake,
447 uint32_t (*GetThreadIDArg)() = PlatformThreadID>
448 struct InitByteFutex {
449
InitByteFutexInitByteFutex450 explicit InitByteFutex(uint8_t* _init_byte_address, uint32_t* _thread_id_address)
451 : init_byte(_init_byte_address),
452 has_thread_id_support(_thread_id_address != nullptr && GetThreadIDArg != nullptr),
453 thread_id(_thread_id_address),
454 base_address(reinterpret_cast<int*>(/*_init_byte_address & ~0x3*/ _init_byte_address - 1)) {}
455
456 public:
457 /// The init byte portion of cxa_guard_acquire. Returns true if
458 /// initialization has already been completed.
acquireInitByteFutex459 bool acquire() {
460 while (true) {
461 uint8_t last_val = UNSET;
462 if (init_byte.compare_exchange(&last_val, PENDING_BIT, std::_AO_Acq_Rel, std::_AO_Acquire)) {
463 if (has_thread_id_support) {
464 thread_id.store(current_thread_id.get(), std::_AO_Relaxed);
465 }
466 return false;
467 }
468
469 if (last_val == COMPLETE_BIT)
470 return true;
471
472 if (last_val & PENDING_BIT) {
473
474 // Check for recursive initialization
475 if (has_thread_id_support && thread_id.load(std::_AO_Relaxed) == current_thread_id.get()) {
476 ABORT_WITH_MESSAGE("__cxa_guard_acquire detected recursive initialization");
477 }
478
479 if ((last_val & WAITING_BIT) == 0) {
480 // This compare exchange can fail for several reasons
481 // (1) another thread finished the whole thing before we got here
482 // (2) another thread set the waiting bit we were trying to thread
483 // (3) another thread had an exception and failed to finish
484 if (!init_byte.compare_exchange(&last_val, PENDING_BIT | WAITING_BIT, std::_AO_Acq_Rel, std::_AO_Release)) {
485 // (1) success, via someone else's work!
486 if (last_val == COMPLETE_BIT)
487 return true;
488
489 // (3) someone else, bailed on doing the work, retry from the start!
490 if (last_val == UNSET)
491 continue;
492
493 // (2) the waiting bit got set, so we are happy to keep waiting
494 }
495 }
496 wait_on_initialization();
497 }
498 }
499 }
500
501 /// The init byte portion of cxa_guard_release.
releaseInitByteFutex502 void release() {
503 uint8_t old = init_byte.exchange(COMPLETE_BIT, std::_AO_Acq_Rel);
504 if (old & WAITING_BIT)
505 wake_all();
506 }
507
508 /// The init byte portion of cxa_guard_abort.
abortInitByteFutex509 void abort() {
510 if (has_thread_id_support)
511 thread_id.store(0, std::_AO_Relaxed);
512
513 uint8_t old = init_byte.exchange(UNSET, std::_AO_Acq_Rel);
514 if (old & WAITING_BIT)
515 wake_all();
516 }
517
518 private:
519 /// Use the futex to wait on the current guard variable. Futex expects a
520 /// 32-bit 4-byte aligned address as the first argument, so we use the 4-byte
521 /// aligned address that encompasses the init byte (i.e. the address of the
522 /// raw guard object that was passed to __cxa_guard_acquire/release/abort).
wait_on_initializationInitByteFutex523 void wait_on_initialization() { Wait(base_address, expected_value_for_futex(PENDING_BIT | WAITING_BIT)); }
wake_allInitByteFutex524 void wake_all() { Wake(base_address); }
525
526 private:
527 AtomicInt<uint8_t> init_byte;
528
529 const bool has_thread_id_support;
530 // Unsafe to use unless has_thread_id_support
531 AtomicInt<uint32_t> thread_id;
532 LazyValue<uint32_t, GetThreadIDArg> current_thread_id;
533
534 /// the 4-byte-aligned address that encompasses the init byte (i.e. the
535 /// address of the raw guard object).
536 int* const base_address;
537
538 /// Create the expected integer value for futex `wait(int* addr, int expected)`.
539 /// We pass the base address as the first argument, So this function creates
540 /// an zero-initialized integer with `b` copied at the correct offset.
expected_value_for_futexInitByteFutex541 static int expected_value_for_futex(uint8_t b) {
542 int dest_val = 0;
543 std::memcpy(reinterpret_cast<char*>(&dest_val) + 1, &b, 1);
544 return dest_val;
545 }
546
547 static_assert(Wait != nullptr && Wake != nullptr, "");
548 };
549
550 //===----------------------------------------------------------------------===//
551 // GuardObject
552 //===----------------------------------------------------------------------===//
553
554 enum class AcquireResult {
555 INIT_IS_DONE,
556 INIT_IS_PENDING,
557 };
558 constexpr AcquireResult INIT_IS_DONE = AcquireResult::INIT_IS_DONE;
559 constexpr AcquireResult INIT_IS_PENDING = AcquireResult::INIT_IS_PENDING;
560
561 /// Co-ordinates between GuardByte and InitByte.
562 template <class InitByteT>
563 struct GuardObject {
564 GuardObject() = delete;
565 GuardObject(GuardObject const&) = delete;
566 GuardObject& operator=(GuardObject const&) = delete;
567
568 private:
569 GuardByte guard_byte;
570 InitByteT init_byte;
571
572 public:
573 /// ARM Constructor
GuardObjectGuardObject574 explicit GuardObject(uint32_t* raw_guard_object)
575 : guard_byte(reinterpret_cast<uint8_t*>(raw_guard_object)),
576 init_byte(reinterpret_cast<uint8_t*>(raw_guard_object) + 1, nullptr) {}
577
578 /// Itanium Constructor
GuardObjectGuardObject579 explicit GuardObject(uint64_t* raw_guard_object)
580 : guard_byte(reinterpret_cast<uint8_t*>(raw_guard_object)),
581 init_byte(reinterpret_cast<uint8_t*>(raw_guard_object) + 1, reinterpret_cast<uint32_t*>(raw_guard_object) + 1) {
582 }
583
584 /// Implements __cxa_guard_acquire.
cxa_guard_acquireGuardObject585 AcquireResult cxa_guard_acquire() {
586 // Use short-circuit evaluation to avoid calling init_byte.acquire when
587 // guard_byte.acquire returns true. (i.e. don't call it when we know from
588 // the guard byte that initialization has already been completed)
589 if (guard_byte.acquire() || init_byte.acquire())
590 return INIT_IS_DONE;
591 return INIT_IS_PENDING;
592 }
593
594 /// Implements __cxa_guard_release.
cxa_guard_releaseGuardObject595 void cxa_guard_release() {
596 // Update guard byte first, so if somebody is woken up by init_byte.release
597 // and comes all the way back around to __cxa_guard_acquire again, they see
598 // it as having completed initialization.
599 guard_byte.release();
600 init_byte.release();
601 }
602
603 /// Implements __cxa_guard_abort.
cxa_guard_abortGuardObject604 void cxa_guard_abort() {
605 guard_byte.abort();
606 init_byte.abort();
607 }
608 };
609
610 //===----------------------------------------------------------------------===//
611 // Convenience Classes
612 //===----------------------------------------------------------------------===//
613
614 /// NoThreadsGuard - Manages initialization without performing any inter-thread
615 /// synchronization.
616 using NoThreadsGuard = GuardObject<InitByteNoThreads>;
617
618 /// GlobalMutexGuard - Manages initialization using a global mutex and
619 /// condition variable.
620 template <class Mutex, class CondVar, Mutex& global_mutex, CondVar& global_cond,
621 uint32_t (*GetThreadID)() = PlatformThreadID>
622 using GlobalMutexGuard = GuardObject<InitByteGlobalMutex<Mutex, CondVar, global_mutex, global_cond, GetThreadID>>;
623
624 /// FutexGuard - Manages initialization using atomics and the futex syscall for
625 /// waiting and waking.
626 template <void (*Wait)(int*, int) = PlatformFutexWait, void (*Wake)(int*) = PlatformFutexWake,
627 uint32_t (*GetThreadIDArg)() = PlatformThreadID>
628 using FutexGuard = GuardObject<InitByteFutex<Wait, Wake, GetThreadIDArg>>;
629
630 //===----------------------------------------------------------------------===//
631 //
632 //===----------------------------------------------------------------------===//
633
634 template <class T>
635 struct GlobalStatic {
636 static T instance;
637 };
638 template <class T>
639 _LIBCPP_CONSTINIT T GlobalStatic<T>::instance = {};
640
641 enum class Implementation { NoThreads, GlobalMutex, Futex };
642
643 template <Implementation Impl>
644 struct SelectImplementation;
645
646 template <>
647 struct SelectImplementation<Implementation::NoThreads> {
648 using type = NoThreadsGuard;
649 };
650
651 template <>
652 struct SelectImplementation<Implementation::GlobalMutex> {
653 using type = GlobalMutexGuard<LibcppMutex, LibcppCondVar, GlobalStatic<LibcppMutex>::instance,
654 GlobalStatic<LibcppCondVar>::instance, PlatformThreadID>;
655 };
656
657 template <>
658 struct SelectImplementation<Implementation::Futex> {
659 using type = FutexGuard<PlatformFutexWait, PlatformFutexWake, PlatformThreadID>;
660 };
661
662 // TODO(EricWF): We should prefer the futex implementation when available. But
663 // it should be done in a separate step from adding the implementation.
664 constexpr Implementation CurrentImplementation =
665 #if defined(_LIBCXXABI_HAS_NO_THREADS)
666 Implementation::NoThreads;
667 #elif defined(_LIBCXXABI_USE_FUTEX)
668 Implementation::Futex;
669 #else
670 Implementation::GlobalMutex;
671 #endif
672
673 static_assert(CurrentImplementation != Implementation::Futex || PlatformSupportsFutex(),
674 "Futex selected but not supported");
675
676 using SelectedImplementation = SelectImplementation<CurrentImplementation>::type;
677
678 } // end namespace
679 } // end namespace __cxxabiv1
680
681 #if defined(__clang__)
682 # pragma clang diagnostic pop
683 #elif defined(__GNUC__)
684 # pragma GCC diagnostic pop
685 #endif
686
687 #endif // LIBCXXABI_SRC_INCLUDE_CXA_GUARD_IMPL_H
688