xref: /freebsd-src/contrib/llvm-project/compiler-rt/lib/xray/xray_profiling.cpp (revision 0fca6ea1d4eea4c934cfff25ac9ee8ad6fe95583)
168d75effSDimitry Andric //===-- xray_profiling.cpp --------------------------------------*- C++ -*-===//
268d75effSDimitry Andric //
368d75effSDimitry Andric // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
468d75effSDimitry Andric // See https://llvm.org/LICENSE.txt for license information.
568d75effSDimitry Andric // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
668d75effSDimitry Andric //
768d75effSDimitry Andric //===----------------------------------------------------------------------===//
868d75effSDimitry Andric //
968d75effSDimitry Andric // This file is a part of XRay, a dynamic runtime instrumentation system.
1068d75effSDimitry Andric //
1168d75effSDimitry Andric // This is the implementation of a profiling handler.
1268d75effSDimitry Andric //
1368d75effSDimitry Andric //===----------------------------------------------------------------------===//
1468d75effSDimitry Andric #include <memory>
1568d75effSDimitry Andric #include <time.h>
1668d75effSDimitry Andric 
1768d75effSDimitry Andric #include "sanitizer_common/sanitizer_atomic.h"
1868d75effSDimitry Andric #include "sanitizer_common/sanitizer_flags.h"
1968d75effSDimitry Andric #include "xray/xray_interface.h"
2068d75effSDimitry Andric #include "xray/xray_log_interface.h"
2168d75effSDimitry Andric #include "xray_buffer_queue.h"
2268d75effSDimitry Andric #include "xray_flags.h"
2368d75effSDimitry Andric #include "xray_profile_collector.h"
2468d75effSDimitry Andric #include "xray_profiling_flags.h"
2568d75effSDimitry Andric #include "xray_recursion_guard.h"
2668d75effSDimitry Andric #include "xray_tsc.h"
2768d75effSDimitry Andric #include "xray_utils.h"
2868d75effSDimitry Andric #include <pthread.h>
2968d75effSDimitry Andric 
3068d75effSDimitry Andric namespace __xray {
3168d75effSDimitry Andric 
3268d75effSDimitry Andric namespace {
3368d75effSDimitry Andric 
3468d75effSDimitry Andric static atomic_sint32_t ProfilerLogFlushStatus = {
3568d75effSDimitry Andric     XRayLogFlushStatus::XRAY_LOG_NOT_FLUSHING};
3668d75effSDimitry Andric 
3768d75effSDimitry Andric static atomic_sint32_t ProfilerLogStatus = {
3868d75effSDimitry Andric     XRayLogInitStatus::XRAY_LOG_UNINITIALIZED};
3968d75effSDimitry Andric 
4068d75effSDimitry Andric static SpinMutex ProfilerOptionsMutex;
4168d75effSDimitry Andric 
4268d75effSDimitry Andric struct ProfilingData {
4368d75effSDimitry Andric   atomic_uintptr_t Allocators;
4468d75effSDimitry Andric   atomic_uintptr_t FCT;
4568d75effSDimitry Andric };
4668d75effSDimitry Andric 
4768d75effSDimitry Andric static pthread_key_t ProfilingKey;
4868d75effSDimitry Andric 
4968d75effSDimitry Andric // We use a global buffer queue, which gets initialized once at initialisation
5068d75effSDimitry Andric // time, and gets reset when profiling is "done".
51*0fca6ea1SDimitry Andric alignas(BufferQueue) static std::byte BufferQueueStorage[sizeof(BufferQueue)];
5268d75effSDimitry Andric static BufferQueue *BQ = nullptr;
5368d75effSDimitry Andric 
5468d75effSDimitry Andric thread_local FunctionCallTrie::Allocators::Buffers ThreadBuffers;
55*0fca6ea1SDimitry Andric alignas(FunctionCallTrie::Allocators) thread_local std::byte
56*0fca6ea1SDimitry Andric     AllocatorsStorage[sizeof(FunctionCallTrie::Allocators)];
57*0fca6ea1SDimitry Andric alignas(FunctionCallTrie) thread_local std::byte
58*0fca6ea1SDimitry Andric     FunctionCallTrieStorage[sizeof(FunctionCallTrie)];
5968d75effSDimitry Andric thread_local ProfilingData TLD{{0}, {0}};
6068d75effSDimitry Andric thread_local atomic_uint8_t ReentranceGuard{0};
6168d75effSDimitry Andric 
6268d75effSDimitry Andric // We use a separate guard for ensuring that for this thread, if we're already
6368d75effSDimitry Andric // cleaning up, that any signal handlers don't attempt to cleanup nor
6468d75effSDimitry Andric // initialise.
6568d75effSDimitry Andric thread_local atomic_uint8_t TLDInitGuard{0};
6668d75effSDimitry Andric 
6768d75effSDimitry Andric // We also use a separate latch to signal that the thread is exiting, and
6868d75effSDimitry Andric // non-essential work should be ignored (things like recording events, etc.).
6968d75effSDimitry Andric thread_local atomic_uint8_t ThreadExitingLatch{0};
7068d75effSDimitry Andric 
7168d75effSDimitry Andric static ProfilingData *getThreadLocalData() XRAY_NEVER_INSTRUMENT {
7268d75effSDimitry Andric   thread_local auto ThreadOnce = []() XRAY_NEVER_INSTRUMENT {
7368d75effSDimitry Andric     pthread_setspecific(ProfilingKey, &TLD);
7468d75effSDimitry Andric     return false;
7568d75effSDimitry Andric   }();
7668d75effSDimitry Andric   (void)ThreadOnce;
7768d75effSDimitry Andric 
7868d75effSDimitry Andric   RecursionGuard TLDInit(TLDInitGuard);
7968d75effSDimitry Andric   if (!TLDInit)
8068d75effSDimitry Andric     return nullptr;
8168d75effSDimitry Andric 
8268d75effSDimitry Andric   if (atomic_load_relaxed(&ThreadExitingLatch))
8368d75effSDimitry Andric     return nullptr;
8468d75effSDimitry Andric 
8568d75effSDimitry Andric   uptr Allocators = 0;
8668d75effSDimitry Andric   if (atomic_compare_exchange_strong(&TLD.Allocators, &Allocators, 1,
8768d75effSDimitry Andric                                      memory_order_acq_rel)) {
8868d75effSDimitry Andric     bool Success = false;
8968d75effSDimitry Andric     auto AllocatorsUndo = at_scope_exit([&]() XRAY_NEVER_INSTRUMENT {
9068d75effSDimitry Andric       if (!Success)
9168d75effSDimitry Andric         atomic_store(&TLD.Allocators, 0, memory_order_release);
9268d75effSDimitry Andric     });
9368d75effSDimitry Andric 
9468d75effSDimitry Andric     // Acquire a set of buffers for this thread.
9568d75effSDimitry Andric     if (BQ == nullptr)
9668d75effSDimitry Andric       return nullptr;
9768d75effSDimitry Andric 
9868d75effSDimitry Andric     if (BQ->getBuffer(ThreadBuffers.NodeBuffer) != BufferQueue::ErrorCode::Ok)
9968d75effSDimitry Andric       return nullptr;
10068d75effSDimitry Andric     auto NodeBufferUndo = at_scope_exit([&]() XRAY_NEVER_INSTRUMENT {
10168d75effSDimitry Andric       if (!Success)
10268d75effSDimitry Andric         BQ->releaseBuffer(ThreadBuffers.NodeBuffer);
10368d75effSDimitry Andric     });
10468d75effSDimitry Andric 
10568d75effSDimitry Andric     if (BQ->getBuffer(ThreadBuffers.RootsBuffer) != BufferQueue::ErrorCode::Ok)
10668d75effSDimitry Andric       return nullptr;
10768d75effSDimitry Andric     auto RootsBufferUndo = at_scope_exit([&]() XRAY_NEVER_INSTRUMENT {
10868d75effSDimitry Andric       if (!Success)
10968d75effSDimitry Andric         BQ->releaseBuffer(ThreadBuffers.RootsBuffer);
11068d75effSDimitry Andric     });
11168d75effSDimitry Andric 
11268d75effSDimitry Andric     if (BQ->getBuffer(ThreadBuffers.ShadowStackBuffer) !=
11368d75effSDimitry Andric         BufferQueue::ErrorCode::Ok)
11468d75effSDimitry Andric       return nullptr;
11568d75effSDimitry Andric     auto ShadowStackBufferUndo = at_scope_exit([&]() XRAY_NEVER_INSTRUMENT {
11668d75effSDimitry Andric       if (!Success)
11768d75effSDimitry Andric         BQ->releaseBuffer(ThreadBuffers.ShadowStackBuffer);
11868d75effSDimitry Andric     });
11968d75effSDimitry Andric 
12068d75effSDimitry Andric     if (BQ->getBuffer(ThreadBuffers.NodeIdPairBuffer) !=
12168d75effSDimitry Andric         BufferQueue::ErrorCode::Ok)
12268d75effSDimitry Andric       return nullptr;
12368d75effSDimitry Andric 
12468d75effSDimitry Andric     Success = true;
12568d75effSDimitry Andric     new (&AllocatorsStorage) FunctionCallTrie::Allocators(
12668d75effSDimitry Andric         FunctionCallTrie::InitAllocatorsFromBuffers(ThreadBuffers));
12768d75effSDimitry Andric     Allocators = reinterpret_cast<uptr>(
12868d75effSDimitry Andric         reinterpret_cast<FunctionCallTrie::Allocators *>(&AllocatorsStorage));
12968d75effSDimitry Andric     atomic_store(&TLD.Allocators, Allocators, memory_order_release);
13068d75effSDimitry Andric   }
13168d75effSDimitry Andric 
13268d75effSDimitry Andric   if (Allocators == 1)
13368d75effSDimitry Andric     return nullptr;
13468d75effSDimitry Andric 
13568d75effSDimitry Andric   uptr FCT = 0;
13668d75effSDimitry Andric   if (atomic_compare_exchange_strong(&TLD.FCT, &FCT, 1, memory_order_acq_rel)) {
13768d75effSDimitry Andric     new (&FunctionCallTrieStorage)
13868d75effSDimitry Andric         FunctionCallTrie(*reinterpret_cast<FunctionCallTrie::Allocators *>(
13968d75effSDimitry Andric             atomic_load_relaxed(&TLD.Allocators)));
14068d75effSDimitry Andric     FCT = reinterpret_cast<uptr>(
14168d75effSDimitry Andric         reinterpret_cast<FunctionCallTrie *>(&FunctionCallTrieStorage));
14268d75effSDimitry Andric     atomic_store(&TLD.FCT, FCT, memory_order_release);
14368d75effSDimitry Andric   }
14468d75effSDimitry Andric 
14568d75effSDimitry Andric   if (FCT == 1)
14668d75effSDimitry Andric     return nullptr;
14768d75effSDimitry Andric 
14868d75effSDimitry Andric   return &TLD;
14968d75effSDimitry Andric }
15068d75effSDimitry Andric 
15168d75effSDimitry Andric static void cleanupTLD() XRAY_NEVER_INSTRUMENT {
15268d75effSDimitry Andric   auto FCT = atomic_exchange(&TLD.FCT, 0, memory_order_acq_rel);
15368d75effSDimitry Andric   if (FCT == reinterpret_cast<uptr>(reinterpret_cast<FunctionCallTrie *>(
15468d75effSDimitry Andric                  &FunctionCallTrieStorage)))
15568d75effSDimitry Andric     reinterpret_cast<FunctionCallTrie *>(FCT)->~FunctionCallTrie();
15668d75effSDimitry Andric 
15768d75effSDimitry Andric   auto Allocators = atomic_exchange(&TLD.Allocators, 0, memory_order_acq_rel);
15868d75effSDimitry Andric   if (Allocators ==
15968d75effSDimitry Andric       reinterpret_cast<uptr>(
16068d75effSDimitry Andric           reinterpret_cast<FunctionCallTrie::Allocators *>(&AllocatorsStorage)))
16168d75effSDimitry Andric     reinterpret_cast<FunctionCallTrie::Allocators *>(Allocators)->~Allocators();
16268d75effSDimitry Andric }
16368d75effSDimitry Andric 
16468d75effSDimitry Andric static void postCurrentThreadFCT(ProfilingData &T) XRAY_NEVER_INSTRUMENT {
16568d75effSDimitry Andric   RecursionGuard TLDInit(TLDInitGuard);
16668d75effSDimitry Andric   if (!TLDInit)
16768d75effSDimitry Andric     return;
16868d75effSDimitry Andric 
16968d75effSDimitry Andric   uptr P = atomic_exchange(&T.FCT, 0, memory_order_acq_rel);
17068d75effSDimitry Andric   if (P != reinterpret_cast<uptr>(
17168d75effSDimitry Andric                reinterpret_cast<FunctionCallTrie *>(&FunctionCallTrieStorage)))
17268d75effSDimitry Andric     return;
17368d75effSDimitry Andric 
17468d75effSDimitry Andric   auto FCT = reinterpret_cast<FunctionCallTrie *>(P);
17568d75effSDimitry Andric   DCHECK_NE(FCT, nullptr);
17668d75effSDimitry Andric 
17768d75effSDimitry Andric   uptr A = atomic_exchange(&T.Allocators, 0, memory_order_acq_rel);
17868d75effSDimitry Andric   if (A !=
17968d75effSDimitry Andric       reinterpret_cast<uptr>(
18068d75effSDimitry Andric           reinterpret_cast<FunctionCallTrie::Allocators *>(&AllocatorsStorage)))
18168d75effSDimitry Andric     return;
18268d75effSDimitry Andric 
18368d75effSDimitry Andric   auto Allocators = reinterpret_cast<FunctionCallTrie::Allocators *>(A);
18468d75effSDimitry Andric   DCHECK_NE(Allocators, nullptr);
18568d75effSDimitry Andric 
18668d75effSDimitry Andric   // Always move the data into the profile collector.
18768d75effSDimitry Andric   profileCollectorService::post(BQ, std::move(*FCT), std::move(*Allocators),
18868d75effSDimitry Andric                                 std::move(ThreadBuffers), GetTid());
18968d75effSDimitry Andric 
19068d75effSDimitry Andric   // Re-initialize the ThreadBuffers object to a known "default" state.
19168d75effSDimitry Andric   ThreadBuffers = FunctionCallTrie::Allocators::Buffers{};
19268d75effSDimitry Andric }
19368d75effSDimitry Andric 
19468d75effSDimitry Andric } // namespace
19568d75effSDimitry Andric 
19668d75effSDimitry Andric const char *profilingCompilerDefinedFlags() XRAY_NEVER_INSTRUMENT {
19768d75effSDimitry Andric #ifdef XRAY_PROFILER_DEFAULT_OPTIONS
19868d75effSDimitry Andric   return SANITIZER_STRINGIFY(XRAY_PROFILER_DEFAULT_OPTIONS);
19968d75effSDimitry Andric #else
20068d75effSDimitry Andric   return "";
20168d75effSDimitry Andric #endif
20268d75effSDimitry Andric }
20368d75effSDimitry Andric 
20468d75effSDimitry Andric XRayLogFlushStatus profilingFlush() XRAY_NEVER_INSTRUMENT {
20568d75effSDimitry Andric   if (atomic_load(&ProfilerLogStatus, memory_order_acquire) !=
20668d75effSDimitry Andric       XRayLogInitStatus::XRAY_LOG_FINALIZED) {
20768d75effSDimitry Andric     if (Verbosity())
20868d75effSDimitry Andric       Report("Not flushing profiles, profiling not been finalized.\n");
20968d75effSDimitry Andric     return XRayLogFlushStatus::XRAY_LOG_NOT_FLUSHING;
21068d75effSDimitry Andric   }
21168d75effSDimitry Andric 
21268d75effSDimitry Andric   RecursionGuard SignalGuard(ReentranceGuard);
21368d75effSDimitry Andric   if (!SignalGuard) {
21468d75effSDimitry Andric     if (Verbosity())
21568d75effSDimitry Andric       Report("Cannot finalize properly inside a signal handler!\n");
21668d75effSDimitry Andric     atomic_store(&ProfilerLogFlushStatus,
21768d75effSDimitry Andric                  XRayLogFlushStatus::XRAY_LOG_NOT_FLUSHING,
21868d75effSDimitry Andric                  memory_order_release);
21968d75effSDimitry Andric     return XRayLogFlushStatus::XRAY_LOG_NOT_FLUSHING;
22068d75effSDimitry Andric   }
22168d75effSDimitry Andric 
22268d75effSDimitry Andric   s32 Previous = atomic_exchange(&ProfilerLogFlushStatus,
22368d75effSDimitry Andric                                  XRayLogFlushStatus::XRAY_LOG_FLUSHING,
22468d75effSDimitry Andric                                  memory_order_acq_rel);
22568d75effSDimitry Andric   if (Previous == XRayLogFlushStatus::XRAY_LOG_FLUSHING) {
22668d75effSDimitry Andric     if (Verbosity())
22768d75effSDimitry Andric       Report("Not flushing profiles, implementation still flushing.\n");
22868d75effSDimitry Andric     return XRayLogFlushStatus::XRAY_LOG_FLUSHING;
22968d75effSDimitry Andric   }
23068d75effSDimitry Andric 
23168d75effSDimitry Andric   // At this point, we'll create the file that will contain the profile, but
23268d75effSDimitry Andric   // only if the options say so.
23368d75effSDimitry Andric   if (!profilingFlags()->no_flush) {
23468d75effSDimitry Andric     // First check whether we have data in the profile collector service
23568d75effSDimitry Andric     // before we try and write anything down.
23668d75effSDimitry Andric     XRayBuffer B = profileCollectorService::nextBuffer({nullptr, 0});
23768d75effSDimitry Andric     if (B.Data == nullptr) {
23868d75effSDimitry Andric       if (Verbosity())
23968d75effSDimitry Andric         Report("profiling: No data to flush.\n");
24068d75effSDimitry Andric     } else {
24168d75effSDimitry Andric       LogWriter *LW = LogWriter::Open();
24268d75effSDimitry Andric       if (LW == nullptr) {
24368d75effSDimitry Andric         if (Verbosity())
24468d75effSDimitry Andric           Report("profiling: Failed to flush to file, dropping data.\n");
24568d75effSDimitry Andric       } else {
24668d75effSDimitry Andric         // Now for each of the buffers, write out the profile data as we would
24768d75effSDimitry Andric         // see it in memory, verbatim.
24868d75effSDimitry Andric         while (B.Data != nullptr && B.Size != 0) {
24968d75effSDimitry Andric           LW->WriteAll(reinterpret_cast<const char *>(B.Data),
25068d75effSDimitry Andric                        reinterpret_cast<const char *>(B.Data) + B.Size);
25168d75effSDimitry Andric           B = profileCollectorService::nextBuffer(B);
25268d75effSDimitry Andric         }
25368d75effSDimitry Andric         LogWriter::Close(LW);
25468d75effSDimitry Andric       }
25568d75effSDimitry Andric     }
25606c3fb27SDimitry Andric   }
25768d75effSDimitry Andric 
25868d75effSDimitry Andric   profileCollectorService::reset();
25968d75effSDimitry Andric 
26068d75effSDimitry Andric   atomic_store(&ProfilerLogFlushStatus, XRayLogFlushStatus::XRAY_LOG_FLUSHED,
26168d75effSDimitry Andric                memory_order_release);
26268d75effSDimitry Andric   atomic_store(&ProfilerLogStatus, XRayLogInitStatus::XRAY_LOG_UNINITIALIZED,
26368d75effSDimitry Andric                memory_order_release);
26468d75effSDimitry Andric 
26568d75effSDimitry Andric   return XRayLogFlushStatus::XRAY_LOG_FLUSHED;
26668d75effSDimitry Andric }
26768d75effSDimitry Andric 
26868d75effSDimitry Andric void profilingHandleArg0(int32_t FuncId,
26968d75effSDimitry Andric                          XRayEntryType Entry) XRAY_NEVER_INSTRUMENT {
27068d75effSDimitry Andric   unsigned char CPU;
27168d75effSDimitry Andric   auto TSC = readTSC(CPU);
27268d75effSDimitry Andric   RecursionGuard G(ReentranceGuard);
27368d75effSDimitry Andric   if (!G)
27468d75effSDimitry Andric     return;
27568d75effSDimitry Andric 
27668d75effSDimitry Andric   auto Status = atomic_load(&ProfilerLogStatus, memory_order_acquire);
27768d75effSDimitry Andric   if (UNLIKELY(Status == XRayLogInitStatus::XRAY_LOG_UNINITIALIZED ||
27868d75effSDimitry Andric                Status == XRayLogInitStatus::XRAY_LOG_INITIALIZING))
27968d75effSDimitry Andric     return;
28068d75effSDimitry Andric 
28168d75effSDimitry Andric   if (UNLIKELY(Status == XRayLogInitStatus::XRAY_LOG_FINALIZED ||
28268d75effSDimitry Andric                Status == XRayLogInitStatus::XRAY_LOG_FINALIZING)) {
28368d75effSDimitry Andric     postCurrentThreadFCT(TLD);
28468d75effSDimitry Andric     return;
28568d75effSDimitry Andric   }
28668d75effSDimitry Andric 
28768d75effSDimitry Andric   auto T = getThreadLocalData();
28868d75effSDimitry Andric   if (T == nullptr)
28968d75effSDimitry Andric     return;
29068d75effSDimitry Andric 
29168d75effSDimitry Andric   auto FCT = reinterpret_cast<FunctionCallTrie *>(atomic_load_relaxed(&T->FCT));
29268d75effSDimitry Andric   switch (Entry) {
29368d75effSDimitry Andric   case XRayEntryType::ENTRY:
29468d75effSDimitry Andric   case XRayEntryType::LOG_ARGS_ENTRY:
29568d75effSDimitry Andric     FCT->enterFunction(FuncId, TSC, CPU);
29668d75effSDimitry Andric     break;
29768d75effSDimitry Andric   case XRayEntryType::EXIT:
29868d75effSDimitry Andric   case XRayEntryType::TAIL:
29968d75effSDimitry Andric     FCT->exitFunction(FuncId, TSC, CPU);
30068d75effSDimitry Andric     break;
30168d75effSDimitry Andric   default:
30268d75effSDimitry Andric     // FIXME: Handle bugs.
30368d75effSDimitry Andric     break;
30468d75effSDimitry Andric   }
30568d75effSDimitry Andric }
30668d75effSDimitry Andric 
30768d75effSDimitry Andric void profilingHandleArg1(int32_t FuncId, XRayEntryType Entry,
30868d75effSDimitry Andric                          uint64_t) XRAY_NEVER_INSTRUMENT {
30968d75effSDimitry Andric   return profilingHandleArg0(FuncId, Entry);
31068d75effSDimitry Andric }
31168d75effSDimitry Andric 
31268d75effSDimitry Andric XRayLogInitStatus profilingFinalize() XRAY_NEVER_INSTRUMENT {
31368d75effSDimitry Andric   s32 CurrentStatus = XRayLogInitStatus::XRAY_LOG_INITIALIZED;
31468d75effSDimitry Andric   if (!atomic_compare_exchange_strong(&ProfilerLogStatus, &CurrentStatus,
31568d75effSDimitry Andric                                       XRayLogInitStatus::XRAY_LOG_FINALIZING,
31668d75effSDimitry Andric                                       memory_order_release)) {
31768d75effSDimitry Andric     if (Verbosity())
31868d75effSDimitry Andric       Report("Cannot finalize profile, the profiling is not initialized.\n");
31968d75effSDimitry Andric     return static_cast<XRayLogInitStatus>(CurrentStatus);
32068d75effSDimitry Andric   }
32168d75effSDimitry Andric 
32268d75effSDimitry Andric   // Mark then finalize the current generation of buffers. This allows us to let
32368d75effSDimitry Andric   // the threads currently holding onto new buffers still use them, but let the
32468d75effSDimitry Andric   // last reference do the memory cleanup.
32568d75effSDimitry Andric   DCHECK_NE(BQ, nullptr);
32668d75effSDimitry Andric   BQ->finalize();
32768d75effSDimitry Andric 
32868d75effSDimitry Andric   // Wait a grace period to allow threads to see that we're finalizing.
32968d75effSDimitry Andric   SleepForMillis(profilingFlags()->grace_period_ms);
33068d75effSDimitry Andric 
33168d75effSDimitry Andric   // If we for some reason are entering this function from an instrumented
33268d75effSDimitry Andric   // handler, we bail out.
33368d75effSDimitry Andric   RecursionGuard G(ReentranceGuard);
33468d75effSDimitry Andric   if (!G)
33568d75effSDimitry Andric     return static_cast<XRayLogInitStatus>(CurrentStatus);
33668d75effSDimitry Andric 
33768d75effSDimitry Andric   // Post the current thread's data if we have any.
33868d75effSDimitry Andric   postCurrentThreadFCT(TLD);
33968d75effSDimitry Andric 
34068d75effSDimitry Andric   // Then we force serialize the log data.
34168d75effSDimitry Andric   profileCollectorService::serialize();
34268d75effSDimitry Andric 
34368d75effSDimitry Andric   atomic_store(&ProfilerLogStatus, XRayLogInitStatus::XRAY_LOG_FINALIZED,
34468d75effSDimitry Andric                memory_order_release);
34568d75effSDimitry Andric   return XRayLogInitStatus::XRAY_LOG_FINALIZED;
34668d75effSDimitry Andric }
34768d75effSDimitry Andric 
34868d75effSDimitry Andric XRayLogInitStatus
34968d75effSDimitry Andric profilingLoggingInit(size_t, size_t, void *Options,
35068d75effSDimitry Andric                      size_t OptionsSize) XRAY_NEVER_INSTRUMENT {
35168d75effSDimitry Andric   RecursionGuard G(ReentranceGuard);
35268d75effSDimitry Andric   if (!G)
35368d75effSDimitry Andric     return XRayLogInitStatus::XRAY_LOG_UNINITIALIZED;
35468d75effSDimitry Andric 
35568d75effSDimitry Andric   s32 CurrentStatus = XRayLogInitStatus::XRAY_LOG_UNINITIALIZED;
35668d75effSDimitry Andric   if (!atomic_compare_exchange_strong(&ProfilerLogStatus, &CurrentStatus,
35768d75effSDimitry Andric                                       XRayLogInitStatus::XRAY_LOG_INITIALIZING,
35868d75effSDimitry Andric                                       memory_order_acq_rel)) {
35968d75effSDimitry Andric     if (Verbosity())
36068d75effSDimitry Andric       Report("Cannot initialize already initialised profiling "
36168d75effSDimitry Andric              "implementation.\n");
36268d75effSDimitry Andric     return static_cast<XRayLogInitStatus>(CurrentStatus);
36368d75effSDimitry Andric   }
36468d75effSDimitry Andric 
36568d75effSDimitry Andric   {
36668d75effSDimitry Andric     SpinMutexLock Lock(&ProfilerOptionsMutex);
36768d75effSDimitry Andric     FlagParser ConfigParser;
36868d75effSDimitry Andric     ProfilerFlags Flags;
36968d75effSDimitry Andric     Flags.setDefaults();
37068d75effSDimitry Andric     registerProfilerFlags(&ConfigParser, &Flags);
37168d75effSDimitry Andric     ConfigParser.ParseString(profilingCompilerDefinedFlags());
37268d75effSDimitry Andric     const char *Env = GetEnv("XRAY_PROFILING_OPTIONS");
37368d75effSDimitry Andric     if (Env == nullptr)
37468d75effSDimitry Andric       Env = "";
37568d75effSDimitry Andric     ConfigParser.ParseString(Env);
37668d75effSDimitry Andric 
37768d75effSDimitry Andric     // Then parse the configuration string provided.
37868d75effSDimitry Andric     ConfigParser.ParseString(static_cast<const char *>(Options));
37968d75effSDimitry Andric     if (Verbosity())
38068d75effSDimitry Andric       ReportUnrecognizedFlags();
38168d75effSDimitry Andric     *profilingFlags() = Flags;
38268d75effSDimitry Andric   }
38368d75effSDimitry Andric 
38468d75effSDimitry Andric   // We need to reset the profile data collection implementation now.
38568d75effSDimitry Andric   profileCollectorService::reset();
38668d75effSDimitry Andric 
38768d75effSDimitry Andric   // Then also reset the buffer queue implementation.
38868d75effSDimitry Andric   if (BQ == nullptr) {
38968d75effSDimitry Andric     bool Success = false;
39068d75effSDimitry Andric     new (&BufferQueueStorage)
39168d75effSDimitry Andric         BufferQueue(profilingFlags()->per_thread_allocator_max,
39268d75effSDimitry Andric                     profilingFlags()->buffers_max, Success);
39368d75effSDimitry Andric     if (!Success) {
39468d75effSDimitry Andric       if (Verbosity())
39568d75effSDimitry Andric         Report("Failed to initialize preallocated memory buffers!");
39668d75effSDimitry Andric       atomic_store(&ProfilerLogStatus,
39768d75effSDimitry Andric                    XRayLogInitStatus::XRAY_LOG_UNINITIALIZED,
39868d75effSDimitry Andric                    memory_order_release);
39968d75effSDimitry Andric       return XRayLogInitStatus::XRAY_LOG_UNINITIALIZED;
40068d75effSDimitry Andric     }
40168d75effSDimitry Andric 
402349cc55cSDimitry Andric     // If we've succeeded, set the global pointer to the initialised storage.
40368d75effSDimitry Andric     BQ = reinterpret_cast<BufferQueue *>(&BufferQueueStorage);
40468d75effSDimitry Andric   } else {
40568d75effSDimitry Andric     BQ->finalize();
40668d75effSDimitry Andric     auto InitStatus = BQ->init(profilingFlags()->per_thread_allocator_max,
40768d75effSDimitry Andric                                profilingFlags()->buffers_max);
40868d75effSDimitry Andric 
40968d75effSDimitry Andric     if (InitStatus != BufferQueue::ErrorCode::Ok) {
41068d75effSDimitry Andric       if (Verbosity())
41168d75effSDimitry Andric         Report("Failed to initialize preallocated memory buffers; error: %s",
41268d75effSDimitry Andric                BufferQueue::getErrorString(InitStatus));
41368d75effSDimitry Andric       atomic_store(&ProfilerLogStatus,
41468d75effSDimitry Andric                    XRayLogInitStatus::XRAY_LOG_UNINITIALIZED,
41568d75effSDimitry Andric                    memory_order_release);
41668d75effSDimitry Andric       return XRayLogInitStatus::XRAY_LOG_UNINITIALIZED;
41768d75effSDimitry Andric     }
41868d75effSDimitry Andric 
41968d75effSDimitry Andric     DCHECK(!BQ->finalizing());
42068d75effSDimitry Andric   }
42168d75effSDimitry Andric 
42268d75effSDimitry Andric   // We need to set up the exit handlers.
42368d75effSDimitry Andric   static pthread_once_t Once = PTHREAD_ONCE_INIT;
42468d75effSDimitry Andric   pthread_once(
42568d75effSDimitry Andric       &Once, +[] {
42668d75effSDimitry Andric         pthread_key_create(
42768d75effSDimitry Andric             &ProfilingKey, +[](void *P) XRAY_NEVER_INSTRUMENT {
42868d75effSDimitry Andric               if (atomic_exchange(&ThreadExitingLatch, 1, memory_order_acq_rel))
42968d75effSDimitry Andric                 return;
43068d75effSDimitry Andric 
43168d75effSDimitry Andric               if (P == nullptr)
43268d75effSDimitry Andric                 return;
43368d75effSDimitry Andric 
43468d75effSDimitry Andric               auto T = reinterpret_cast<ProfilingData *>(P);
43568d75effSDimitry Andric               if (atomic_load_relaxed(&T->Allocators) == 0)
43668d75effSDimitry Andric                 return;
43768d75effSDimitry Andric 
43868d75effSDimitry Andric               {
43968d75effSDimitry Andric                 // If we're somehow executing this while inside a
44068d75effSDimitry Andric                 // non-reentrant-friendly context, we skip attempting to post
44168d75effSDimitry Andric                 // the current thread's data.
44268d75effSDimitry Andric                 RecursionGuard G(ReentranceGuard);
44368d75effSDimitry Andric                 if (!G)
44468d75effSDimitry Andric                   return;
44568d75effSDimitry Andric 
44668d75effSDimitry Andric                 postCurrentThreadFCT(*T);
44768d75effSDimitry Andric               }
44868d75effSDimitry Andric             });
44968d75effSDimitry Andric 
45068d75effSDimitry Andric         // We also need to set up an exit handler, so that we can get the
45168d75effSDimitry Andric         // profile information at exit time. We use the C API to do this, to not
45268d75effSDimitry Andric         // rely on C++ ABI functions for registering exit handlers.
45368d75effSDimitry Andric         Atexit(+[]() XRAY_NEVER_INSTRUMENT {
45468d75effSDimitry Andric           if (atomic_exchange(&ThreadExitingLatch, 1, memory_order_acq_rel))
45568d75effSDimitry Andric             return;
45668d75effSDimitry Andric 
45768d75effSDimitry Andric           auto Cleanup =
45868d75effSDimitry Andric               at_scope_exit([]() XRAY_NEVER_INSTRUMENT { cleanupTLD(); });
45968d75effSDimitry Andric 
46068d75effSDimitry Andric           // Finalize and flush.
46168d75effSDimitry Andric           if (profilingFinalize() != XRAY_LOG_FINALIZED ||
46268d75effSDimitry Andric               profilingFlush() != XRAY_LOG_FLUSHED)
46368d75effSDimitry Andric             return;
46468d75effSDimitry Andric 
46568d75effSDimitry Andric           if (Verbosity())
46668d75effSDimitry Andric             Report("XRay Profile flushed at exit.");
46768d75effSDimitry Andric         });
46868d75effSDimitry Andric       });
46968d75effSDimitry Andric 
47068d75effSDimitry Andric   __xray_log_set_buffer_iterator(profileCollectorService::nextBuffer);
47168d75effSDimitry Andric   __xray_set_handler(profilingHandleArg0);
47268d75effSDimitry Andric   __xray_set_handler_arg1(profilingHandleArg1);
47368d75effSDimitry Andric 
47468d75effSDimitry Andric   atomic_store(&ProfilerLogStatus, XRayLogInitStatus::XRAY_LOG_INITIALIZED,
47568d75effSDimitry Andric                memory_order_release);
47668d75effSDimitry Andric   if (Verbosity())
47768d75effSDimitry Andric     Report("XRay Profiling init successful.\n");
47868d75effSDimitry Andric 
47968d75effSDimitry Andric   return XRayLogInitStatus::XRAY_LOG_INITIALIZED;
48068d75effSDimitry Andric }
48168d75effSDimitry Andric 
48268d75effSDimitry Andric bool profilingDynamicInitializer() XRAY_NEVER_INSTRUMENT {
48368d75effSDimitry Andric   // Set up the flag defaults from the static defaults and the
48468d75effSDimitry Andric   // compiler-provided defaults.
48568d75effSDimitry Andric   {
48668d75effSDimitry Andric     SpinMutexLock Lock(&ProfilerOptionsMutex);
48768d75effSDimitry Andric     auto *F = profilingFlags();
48868d75effSDimitry Andric     F->setDefaults();
48968d75effSDimitry Andric     FlagParser ProfilingParser;
49068d75effSDimitry Andric     registerProfilerFlags(&ProfilingParser, F);
49168d75effSDimitry Andric     ProfilingParser.ParseString(profilingCompilerDefinedFlags());
49268d75effSDimitry Andric   }
49368d75effSDimitry Andric 
49468d75effSDimitry Andric   XRayLogImpl Impl{
49568d75effSDimitry Andric       profilingLoggingInit,
49668d75effSDimitry Andric       profilingFinalize,
49768d75effSDimitry Andric       profilingHandleArg0,
49868d75effSDimitry Andric       profilingFlush,
49968d75effSDimitry Andric   };
50068d75effSDimitry Andric   auto RegistrationResult = __xray_log_register_mode("xray-profiling", Impl);
50168d75effSDimitry Andric   if (RegistrationResult != XRayLogRegisterStatus::XRAY_REGISTRATION_OK) {
50268d75effSDimitry Andric     if (Verbosity())
50368d75effSDimitry Andric       Report("Cannot register XRay Profiling mode to 'xray-profiling'; error = "
50468d75effSDimitry Andric              "%d\n",
50568d75effSDimitry Andric              RegistrationResult);
50668d75effSDimitry Andric     return false;
50768d75effSDimitry Andric   }
50868d75effSDimitry Andric 
50968d75effSDimitry Andric   if (!internal_strcmp(flags()->xray_mode, "xray-profiling"))
51068d75effSDimitry Andric     __xray_log_select_mode("xray_profiling");
51168d75effSDimitry Andric   return true;
51268d75effSDimitry Andric }
51368d75effSDimitry Andric 
51468d75effSDimitry Andric } // namespace __xray
51568d75effSDimitry Andric 
51668d75effSDimitry Andric static auto UNUSED Unused = __xray::profilingDynamicInitializer();
517