168d75effSDimitry Andric //===-- xray_profiling.cpp --------------------------------------*- C++ -*-===// 268d75effSDimitry Andric // 368d75effSDimitry Andric // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 468d75effSDimitry Andric // See https://llvm.org/LICENSE.txt for license information. 568d75effSDimitry Andric // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 668d75effSDimitry Andric // 768d75effSDimitry Andric //===----------------------------------------------------------------------===// 868d75effSDimitry Andric // 968d75effSDimitry Andric // This file is a part of XRay, a dynamic runtime instrumentation system. 1068d75effSDimitry Andric // 1168d75effSDimitry Andric // This is the implementation of a profiling handler. 1268d75effSDimitry Andric // 1368d75effSDimitry Andric //===----------------------------------------------------------------------===// 1468d75effSDimitry Andric #include <memory> 1568d75effSDimitry Andric #include <time.h> 1668d75effSDimitry Andric 1768d75effSDimitry Andric #include "sanitizer_common/sanitizer_atomic.h" 1868d75effSDimitry Andric #include "sanitizer_common/sanitizer_flags.h" 1968d75effSDimitry Andric #include "xray/xray_interface.h" 2068d75effSDimitry Andric #include "xray/xray_log_interface.h" 2168d75effSDimitry Andric #include "xray_buffer_queue.h" 2268d75effSDimitry Andric #include "xray_flags.h" 2368d75effSDimitry Andric #include "xray_profile_collector.h" 2468d75effSDimitry Andric #include "xray_profiling_flags.h" 2568d75effSDimitry Andric #include "xray_recursion_guard.h" 2668d75effSDimitry Andric #include "xray_tsc.h" 2768d75effSDimitry Andric #include "xray_utils.h" 2868d75effSDimitry Andric #include <pthread.h> 2968d75effSDimitry Andric 3068d75effSDimitry Andric namespace __xray { 3168d75effSDimitry Andric 3268d75effSDimitry Andric namespace { 3368d75effSDimitry Andric 3468d75effSDimitry Andric static atomic_sint32_t ProfilerLogFlushStatus = { 3568d75effSDimitry Andric XRayLogFlushStatus::XRAY_LOG_NOT_FLUSHING}; 3668d75effSDimitry Andric 3768d75effSDimitry Andric static atomic_sint32_t ProfilerLogStatus = { 3868d75effSDimitry Andric XRayLogInitStatus::XRAY_LOG_UNINITIALIZED}; 3968d75effSDimitry Andric 4068d75effSDimitry Andric static SpinMutex ProfilerOptionsMutex; 4168d75effSDimitry Andric 4268d75effSDimitry Andric struct ProfilingData { 4368d75effSDimitry Andric atomic_uintptr_t Allocators; 4468d75effSDimitry Andric atomic_uintptr_t FCT; 4568d75effSDimitry Andric }; 4668d75effSDimitry Andric 4768d75effSDimitry Andric static pthread_key_t ProfilingKey; 4868d75effSDimitry Andric 4968d75effSDimitry Andric // We use a global buffer queue, which gets initialized once at initialisation 5068d75effSDimitry Andric // time, and gets reset when profiling is "done". 51*0fca6ea1SDimitry Andric alignas(BufferQueue) static std::byte BufferQueueStorage[sizeof(BufferQueue)]; 5268d75effSDimitry Andric static BufferQueue *BQ = nullptr; 5368d75effSDimitry Andric 5468d75effSDimitry Andric thread_local FunctionCallTrie::Allocators::Buffers ThreadBuffers; 55*0fca6ea1SDimitry Andric alignas(FunctionCallTrie::Allocators) thread_local std::byte 56*0fca6ea1SDimitry Andric AllocatorsStorage[sizeof(FunctionCallTrie::Allocators)]; 57*0fca6ea1SDimitry Andric alignas(FunctionCallTrie) thread_local std::byte 58*0fca6ea1SDimitry Andric FunctionCallTrieStorage[sizeof(FunctionCallTrie)]; 5968d75effSDimitry Andric thread_local ProfilingData TLD{{0}, {0}}; 6068d75effSDimitry Andric thread_local atomic_uint8_t ReentranceGuard{0}; 6168d75effSDimitry Andric 6268d75effSDimitry Andric // We use a separate guard for ensuring that for this thread, if we're already 6368d75effSDimitry Andric // cleaning up, that any signal handlers don't attempt to cleanup nor 6468d75effSDimitry Andric // initialise. 6568d75effSDimitry Andric thread_local atomic_uint8_t TLDInitGuard{0}; 6668d75effSDimitry Andric 6768d75effSDimitry Andric // We also use a separate latch to signal that the thread is exiting, and 6868d75effSDimitry Andric // non-essential work should be ignored (things like recording events, etc.). 6968d75effSDimitry Andric thread_local atomic_uint8_t ThreadExitingLatch{0}; 7068d75effSDimitry Andric 7168d75effSDimitry Andric static ProfilingData *getThreadLocalData() XRAY_NEVER_INSTRUMENT { 7268d75effSDimitry Andric thread_local auto ThreadOnce = []() XRAY_NEVER_INSTRUMENT { 7368d75effSDimitry Andric pthread_setspecific(ProfilingKey, &TLD); 7468d75effSDimitry Andric return false; 7568d75effSDimitry Andric }(); 7668d75effSDimitry Andric (void)ThreadOnce; 7768d75effSDimitry Andric 7868d75effSDimitry Andric RecursionGuard TLDInit(TLDInitGuard); 7968d75effSDimitry Andric if (!TLDInit) 8068d75effSDimitry Andric return nullptr; 8168d75effSDimitry Andric 8268d75effSDimitry Andric if (atomic_load_relaxed(&ThreadExitingLatch)) 8368d75effSDimitry Andric return nullptr; 8468d75effSDimitry Andric 8568d75effSDimitry Andric uptr Allocators = 0; 8668d75effSDimitry Andric if (atomic_compare_exchange_strong(&TLD.Allocators, &Allocators, 1, 8768d75effSDimitry Andric memory_order_acq_rel)) { 8868d75effSDimitry Andric bool Success = false; 8968d75effSDimitry Andric auto AllocatorsUndo = at_scope_exit([&]() XRAY_NEVER_INSTRUMENT { 9068d75effSDimitry Andric if (!Success) 9168d75effSDimitry Andric atomic_store(&TLD.Allocators, 0, memory_order_release); 9268d75effSDimitry Andric }); 9368d75effSDimitry Andric 9468d75effSDimitry Andric // Acquire a set of buffers for this thread. 9568d75effSDimitry Andric if (BQ == nullptr) 9668d75effSDimitry Andric return nullptr; 9768d75effSDimitry Andric 9868d75effSDimitry Andric if (BQ->getBuffer(ThreadBuffers.NodeBuffer) != BufferQueue::ErrorCode::Ok) 9968d75effSDimitry Andric return nullptr; 10068d75effSDimitry Andric auto NodeBufferUndo = at_scope_exit([&]() XRAY_NEVER_INSTRUMENT { 10168d75effSDimitry Andric if (!Success) 10268d75effSDimitry Andric BQ->releaseBuffer(ThreadBuffers.NodeBuffer); 10368d75effSDimitry Andric }); 10468d75effSDimitry Andric 10568d75effSDimitry Andric if (BQ->getBuffer(ThreadBuffers.RootsBuffer) != BufferQueue::ErrorCode::Ok) 10668d75effSDimitry Andric return nullptr; 10768d75effSDimitry Andric auto RootsBufferUndo = at_scope_exit([&]() XRAY_NEVER_INSTRUMENT { 10868d75effSDimitry Andric if (!Success) 10968d75effSDimitry Andric BQ->releaseBuffer(ThreadBuffers.RootsBuffer); 11068d75effSDimitry Andric }); 11168d75effSDimitry Andric 11268d75effSDimitry Andric if (BQ->getBuffer(ThreadBuffers.ShadowStackBuffer) != 11368d75effSDimitry Andric BufferQueue::ErrorCode::Ok) 11468d75effSDimitry Andric return nullptr; 11568d75effSDimitry Andric auto ShadowStackBufferUndo = at_scope_exit([&]() XRAY_NEVER_INSTRUMENT { 11668d75effSDimitry Andric if (!Success) 11768d75effSDimitry Andric BQ->releaseBuffer(ThreadBuffers.ShadowStackBuffer); 11868d75effSDimitry Andric }); 11968d75effSDimitry Andric 12068d75effSDimitry Andric if (BQ->getBuffer(ThreadBuffers.NodeIdPairBuffer) != 12168d75effSDimitry Andric BufferQueue::ErrorCode::Ok) 12268d75effSDimitry Andric return nullptr; 12368d75effSDimitry Andric 12468d75effSDimitry Andric Success = true; 12568d75effSDimitry Andric new (&AllocatorsStorage) FunctionCallTrie::Allocators( 12668d75effSDimitry Andric FunctionCallTrie::InitAllocatorsFromBuffers(ThreadBuffers)); 12768d75effSDimitry Andric Allocators = reinterpret_cast<uptr>( 12868d75effSDimitry Andric reinterpret_cast<FunctionCallTrie::Allocators *>(&AllocatorsStorage)); 12968d75effSDimitry Andric atomic_store(&TLD.Allocators, Allocators, memory_order_release); 13068d75effSDimitry Andric } 13168d75effSDimitry Andric 13268d75effSDimitry Andric if (Allocators == 1) 13368d75effSDimitry Andric return nullptr; 13468d75effSDimitry Andric 13568d75effSDimitry Andric uptr FCT = 0; 13668d75effSDimitry Andric if (atomic_compare_exchange_strong(&TLD.FCT, &FCT, 1, memory_order_acq_rel)) { 13768d75effSDimitry Andric new (&FunctionCallTrieStorage) 13868d75effSDimitry Andric FunctionCallTrie(*reinterpret_cast<FunctionCallTrie::Allocators *>( 13968d75effSDimitry Andric atomic_load_relaxed(&TLD.Allocators))); 14068d75effSDimitry Andric FCT = reinterpret_cast<uptr>( 14168d75effSDimitry Andric reinterpret_cast<FunctionCallTrie *>(&FunctionCallTrieStorage)); 14268d75effSDimitry Andric atomic_store(&TLD.FCT, FCT, memory_order_release); 14368d75effSDimitry Andric } 14468d75effSDimitry Andric 14568d75effSDimitry Andric if (FCT == 1) 14668d75effSDimitry Andric return nullptr; 14768d75effSDimitry Andric 14868d75effSDimitry Andric return &TLD; 14968d75effSDimitry Andric } 15068d75effSDimitry Andric 15168d75effSDimitry Andric static void cleanupTLD() XRAY_NEVER_INSTRUMENT { 15268d75effSDimitry Andric auto FCT = atomic_exchange(&TLD.FCT, 0, memory_order_acq_rel); 15368d75effSDimitry Andric if (FCT == reinterpret_cast<uptr>(reinterpret_cast<FunctionCallTrie *>( 15468d75effSDimitry Andric &FunctionCallTrieStorage))) 15568d75effSDimitry Andric reinterpret_cast<FunctionCallTrie *>(FCT)->~FunctionCallTrie(); 15668d75effSDimitry Andric 15768d75effSDimitry Andric auto Allocators = atomic_exchange(&TLD.Allocators, 0, memory_order_acq_rel); 15868d75effSDimitry Andric if (Allocators == 15968d75effSDimitry Andric reinterpret_cast<uptr>( 16068d75effSDimitry Andric reinterpret_cast<FunctionCallTrie::Allocators *>(&AllocatorsStorage))) 16168d75effSDimitry Andric reinterpret_cast<FunctionCallTrie::Allocators *>(Allocators)->~Allocators(); 16268d75effSDimitry Andric } 16368d75effSDimitry Andric 16468d75effSDimitry Andric static void postCurrentThreadFCT(ProfilingData &T) XRAY_NEVER_INSTRUMENT { 16568d75effSDimitry Andric RecursionGuard TLDInit(TLDInitGuard); 16668d75effSDimitry Andric if (!TLDInit) 16768d75effSDimitry Andric return; 16868d75effSDimitry Andric 16968d75effSDimitry Andric uptr P = atomic_exchange(&T.FCT, 0, memory_order_acq_rel); 17068d75effSDimitry Andric if (P != reinterpret_cast<uptr>( 17168d75effSDimitry Andric reinterpret_cast<FunctionCallTrie *>(&FunctionCallTrieStorage))) 17268d75effSDimitry Andric return; 17368d75effSDimitry Andric 17468d75effSDimitry Andric auto FCT = reinterpret_cast<FunctionCallTrie *>(P); 17568d75effSDimitry Andric DCHECK_NE(FCT, nullptr); 17668d75effSDimitry Andric 17768d75effSDimitry Andric uptr A = atomic_exchange(&T.Allocators, 0, memory_order_acq_rel); 17868d75effSDimitry Andric if (A != 17968d75effSDimitry Andric reinterpret_cast<uptr>( 18068d75effSDimitry Andric reinterpret_cast<FunctionCallTrie::Allocators *>(&AllocatorsStorage))) 18168d75effSDimitry Andric return; 18268d75effSDimitry Andric 18368d75effSDimitry Andric auto Allocators = reinterpret_cast<FunctionCallTrie::Allocators *>(A); 18468d75effSDimitry Andric DCHECK_NE(Allocators, nullptr); 18568d75effSDimitry Andric 18668d75effSDimitry Andric // Always move the data into the profile collector. 18768d75effSDimitry Andric profileCollectorService::post(BQ, std::move(*FCT), std::move(*Allocators), 18868d75effSDimitry Andric std::move(ThreadBuffers), GetTid()); 18968d75effSDimitry Andric 19068d75effSDimitry Andric // Re-initialize the ThreadBuffers object to a known "default" state. 19168d75effSDimitry Andric ThreadBuffers = FunctionCallTrie::Allocators::Buffers{}; 19268d75effSDimitry Andric } 19368d75effSDimitry Andric 19468d75effSDimitry Andric } // namespace 19568d75effSDimitry Andric 19668d75effSDimitry Andric const char *profilingCompilerDefinedFlags() XRAY_NEVER_INSTRUMENT { 19768d75effSDimitry Andric #ifdef XRAY_PROFILER_DEFAULT_OPTIONS 19868d75effSDimitry Andric return SANITIZER_STRINGIFY(XRAY_PROFILER_DEFAULT_OPTIONS); 19968d75effSDimitry Andric #else 20068d75effSDimitry Andric return ""; 20168d75effSDimitry Andric #endif 20268d75effSDimitry Andric } 20368d75effSDimitry Andric 20468d75effSDimitry Andric XRayLogFlushStatus profilingFlush() XRAY_NEVER_INSTRUMENT { 20568d75effSDimitry Andric if (atomic_load(&ProfilerLogStatus, memory_order_acquire) != 20668d75effSDimitry Andric XRayLogInitStatus::XRAY_LOG_FINALIZED) { 20768d75effSDimitry Andric if (Verbosity()) 20868d75effSDimitry Andric Report("Not flushing profiles, profiling not been finalized.\n"); 20968d75effSDimitry Andric return XRayLogFlushStatus::XRAY_LOG_NOT_FLUSHING; 21068d75effSDimitry Andric } 21168d75effSDimitry Andric 21268d75effSDimitry Andric RecursionGuard SignalGuard(ReentranceGuard); 21368d75effSDimitry Andric if (!SignalGuard) { 21468d75effSDimitry Andric if (Verbosity()) 21568d75effSDimitry Andric Report("Cannot finalize properly inside a signal handler!\n"); 21668d75effSDimitry Andric atomic_store(&ProfilerLogFlushStatus, 21768d75effSDimitry Andric XRayLogFlushStatus::XRAY_LOG_NOT_FLUSHING, 21868d75effSDimitry Andric memory_order_release); 21968d75effSDimitry Andric return XRayLogFlushStatus::XRAY_LOG_NOT_FLUSHING; 22068d75effSDimitry Andric } 22168d75effSDimitry Andric 22268d75effSDimitry Andric s32 Previous = atomic_exchange(&ProfilerLogFlushStatus, 22368d75effSDimitry Andric XRayLogFlushStatus::XRAY_LOG_FLUSHING, 22468d75effSDimitry Andric memory_order_acq_rel); 22568d75effSDimitry Andric if (Previous == XRayLogFlushStatus::XRAY_LOG_FLUSHING) { 22668d75effSDimitry Andric if (Verbosity()) 22768d75effSDimitry Andric Report("Not flushing profiles, implementation still flushing.\n"); 22868d75effSDimitry Andric return XRayLogFlushStatus::XRAY_LOG_FLUSHING; 22968d75effSDimitry Andric } 23068d75effSDimitry Andric 23168d75effSDimitry Andric // At this point, we'll create the file that will contain the profile, but 23268d75effSDimitry Andric // only if the options say so. 23368d75effSDimitry Andric if (!profilingFlags()->no_flush) { 23468d75effSDimitry Andric // First check whether we have data in the profile collector service 23568d75effSDimitry Andric // before we try and write anything down. 23668d75effSDimitry Andric XRayBuffer B = profileCollectorService::nextBuffer({nullptr, 0}); 23768d75effSDimitry Andric if (B.Data == nullptr) { 23868d75effSDimitry Andric if (Verbosity()) 23968d75effSDimitry Andric Report("profiling: No data to flush.\n"); 24068d75effSDimitry Andric } else { 24168d75effSDimitry Andric LogWriter *LW = LogWriter::Open(); 24268d75effSDimitry Andric if (LW == nullptr) { 24368d75effSDimitry Andric if (Verbosity()) 24468d75effSDimitry Andric Report("profiling: Failed to flush to file, dropping data.\n"); 24568d75effSDimitry Andric } else { 24668d75effSDimitry Andric // Now for each of the buffers, write out the profile data as we would 24768d75effSDimitry Andric // see it in memory, verbatim. 24868d75effSDimitry Andric while (B.Data != nullptr && B.Size != 0) { 24968d75effSDimitry Andric LW->WriteAll(reinterpret_cast<const char *>(B.Data), 25068d75effSDimitry Andric reinterpret_cast<const char *>(B.Data) + B.Size); 25168d75effSDimitry Andric B = profileCollectorService::nextBuffer(B); 25268d75effSDimitry Andric } 25368d75effSDimitry Andric LogWriter::Close(LW); 25468d75effSDimitry Andric } 25568d75effSDimitry Andric } 25606c3fb27SDimitry Andric } 25768d75effSDimitry Andric 25868d75effSDimitry Andric profileCollectorService::reset(); 25968d75effSDimitry Andric 26068d75effSDimitry Andric atomic_store(&ProfilerLogFlushStatus, XRayLogFlushStatus::XRAY_LOG_FLUSHED, 26168d75effSDimitry Andric memory_order_release); 26268d75effSDimitry Andric atomic_store(&ProfilerLogStatus, XRayLogInitStatus::XRAY_LOG_UNINITIALIZED, 26368d75effSDimitry Andric memory_order_release); 26468d75effSDimitry Andric 26568d75effSDimitry Andric return XRayLogFlushStatus::XRAY_LOG_FLUSHED; 26668d75effSDimitry Andric } 26768d75effSDimitry Andric 26868d75effSDimitry Andric void profilingHandleArg0(int32_t FuncId, 26968d75effSDimitry Andric XRayEntryType Entry) XRAY_NEVER_INSTRUMENT { 27068d75effSDimitry Andric unsigned char CPU; 27168d75effSDimitry Andric auto TSC = readTSC(CPU); 27268d75effSDimitry Andric RecursionGuard G(ReentranceGuard); 27368d75effSDimitry Andric if (!G) 27468d75effSDimitry Andric return; 27568d75effSDimitry Andric 27668d75effSDimitry Andric auto Status = atomic_load(&ProfilerLogStatus, memory_order_acquire); 27768d75effSDimitry Andric if (UNLIKELY(Status == XRayLogInitStatus::XRAY_LOG_UNINITIALIZED || 27868d75effSDimitry Andric Status == XRayLogInitStatus::XRAY_LOG_INITIALIZING)) 27968d75effSDimitry Andric return; 28068d75effSDimitry Andric 28168d75effSDimitry Andric if (UNLIKELY(Status == XRayLogInitStatus::XRAY_LOG_FINALIZED || 28268d75effSDimitry Andric Status == XRayLogInitStatus::XRAY_LOG_FINALIZING)) { 28368d75effSDimitry Andric postCurrentThreadFCT(TLD); 28468d75effSDimitry Andric return; 28568d75effSDimitry Andric } 28668d75effSDimitry Andric 28768d75effSDimitry Andric auto T = getThreadLocalData(); 28868d75effSDimitry Andric if (T == nullptr) 28968d75effSDimitry Andric return; 29068d75effSDimitry Andric 29168d75effSDimitry Andric auto FCT = reinterpret_cast<FunctionCallTrie *>(atomic_load_relaxed(&T->FCT)); 29268d75effSDimitry Andric switch (Entry) { 29368d75effSDimitry Andric case XRayEntryType::ENTRY: 29468d75effSDimitry Andric case XRayEntryType::LOG_ARGS_ENTRY: 29568d75effSDimitry Andric FCT->enterFunction(FuncId, TSC, CPU); 29668d75effSDimitry Andric break; 29768d75effSDimitry Andric case XRayEntryType::EXIT: 29868d75effSDimitry Andric case XRayEntryType::TAIL: 29968d75effSDimitry Andric FCT->exitFunction(FuncId, TSC, CPU); 30068d75effSDimitry Andric break; 30168d75effSDimitry Andric default: 30268d75effSDimitry Andric // FIXME: Handle bugs. 30368d75effSDimitry Andric break; 30468d75effSDimitry Andric } 30568d75effSDimitry Andric } 30668d75effSDimitry Andric 30768d75effSDimitry Andric void profilingHandleArg1(int32_t FuncId, XRayEntryType Entry, 30868d75effSDimitry Andric uint64_t) XRAY_NEVER_INSTRUMENT { 30968d75effSDimitry Andric return profilingHandleArg0(FuncId, Entry); 31068d75effSDimitry Andric } 31168d75effSDimitry Andric 31268d75effSDimitry Andric XRayLogInitStatus profilingFinalize() XRAY_NEVER_INSTRUMENT { 31368d75effSDimitry Andric s32 CurrentStatus = XRayLogInitStatus::XRAY_LOG_INITIALIZED; 31468d75effSDimitry Andric if (!atomic_compare_exchange_strong(&ProfilerLogStatus, &CurrentStatus, 31568d75effSDimitry Andric XRayLogInitStatus::XRAY_LOG_FINALIZING, 31668d75effSDimitry Andric memory_order_release)) { 31768d75effSDimitry Andric if (Verbosity()) 31868d75effSDimitry Andric Report("Cannot finalize profile, the profiling is not initialized.\n"); 31968d75effSDimitry Andric return static_cast<XRayLogInitStatus>(CurrentStatus); 32068d75effSDimitry Andric } 32168d75effSDimitry Andric 32268d75effSDimitry Andric // Mark then finalize the current generation of buffers. This allows us to let 32368d75effSDimitry Andric // the threads currently holding onto new buffers still use them, but let the 32468d75effSDimitry Andric // last reference do the memory cleanup. 32568d75effSDimitry Andric DCHECK_NE(BQ, nullptr); 32668d75effSDimitry Andric BQ->finalize(); 32768d75effSDimitry Andric 32868d75effSDimitry Andric // Wait a grace period to allow threads to see that we're finalizing. 32968d75effSDimitry Andric SleepForMillis(profilingFlags()->grace_period_ms); 33068d75effSDimitry Andric 33168d75effSDimitry Andric // If we for some reason are entering this function from an instrumented 33268d75effSDimitry Andric // handler, we bail out. 33368d75effSDimitry Andric RecursionGuard G(ReentranceGuard); 33468d75effSDimitry Andric if (!G) 33568d75effSDimitry Andric return static_cast<XRayLogInitStatus>(CurrentStatus); 33668d75effSDimitry Andric 33768d75effSDimitry Andric // Post the current thread's data if we have any. 33868d75effSDimitry Andric postCurrentThreadFCT(TLD); 33968d75effSDimitry Andric 34068d75effSDimitry Andric // Then we force serialize the log data. 34168d75effSDimitry Andric profileCollectorService::serialize(); 34268d75effSDimitry Andric 34368d75effSDimitry Andric atomic_store(&ProfilerLogStatus, XRayLogInitStatus::XRAY_LOG_FINALIZED, 34468d75effSDimitry Andric memory_order_release); 34568d75effSDimitry Andric return XRayLogInitStatus::XRAY_LOG_FINALIZED; 34668d75effSDimitry Andric } 34768d75effSDimitry Andric 34868d75effSDimitry Andric XRayLogInitStatus 34968d75effSDimitry Andric profilingLoggingInit(size_t, size_t, void *Options, 35068d75effSDimitry Andric size_t OptionsSize) XRAY_NEVER_INSTRUMENT { 35168d75effSDimitry Andric RecursionGuard G(ReentranceGuard); 35268d75effSDimitry Andric if (!G) 35368d75effSDimitry Andric return XRayLogInitStatus::XRAY_LOG_UNINITIALIZED; 35468d75effSDimitry Andric 35568d75effSDimitry Andric s32 CurrentStatus = XRayLogInitStatus::XRAY_LOG_UNINITIALIZED; 35668d75effSDimitry Andric if (!atomic_compare_exchange_strong(&ProfilerLogStatus, &CurrentStatus, 35768d75effSDimitry Andric XRayLogInitStatus::XRAY_LOG_INITIALIZING, 35868d75effSDimitry Andric memory_order_acq_rel)) { 35968d75effSDimitry Andric if (Verbosity()) 36068d75effSDimitry Andric Report("Cannot initialize already initialised profiling " 36168d75effSDimitry Andric "implementation.\n"); 36268d75effSDimitry Andric return static_cast<XRayLogInitStatus>(CurrentStatus); 36368d75effSDimitry Andric } 36468d75effSDimitry Andric 36568d75effSDimitry Andric { 36668d75effSDimitry Andric SpinMutexLock Lock(&ProfilerOptionsMutex); 36768d75effSDimitry Andric FlagParser ConfigParser; 36868d75effSDimitry Andric ProfilerFlags Flags; 36968d75effSDimitry Andric Flags.setDefaults(); 37068d75effSDimitry Andric registerProfilerFlags(&ConfigParser, &Flags); 37168d75effSDimitry Andric ConfigParser.ParseString(profilingCompilerDefinedFlags()); 37268d75effSDimitry Andric const char *Env = GetEnv("XRAY_PROFILING_OPTIONS"); 37368d75effSDimitry Andric if (Env == nullptr) 37468d75effSDimitry Andric Env = ""; 37568d75effSDimitry Andric ConfigParser.ParseString(Env); 37668d75effSDimitry Andric 37768d75effSDimitry Andric // Then parse the configuration string provided. 37868d75effSDimitry Andric ConfigParser.ParseString(static_cast<const char *>(Options)); 37968d75effSDimitry Andric if (Verbosity()) 38068d75effSDimitry Andric ReportUnrecognizedFlags(); 38168d75effSDimitry Andric *profilingFlags() = Flags; 38268d75effSDimitry Andric } 38368d75effSDimitry Andric 38468d75effSDimitry Andric // We need to reset the profile data collection implementation now. 38568d75effSDimitry Andric profileCollectorService::reset(); 38668d75effSDimitry Andric 38768d75effSDimitry Andric // Then also reset the buffer queue implementation. 38868d75effSDimitry Andric if (BQ == nullptr) { 38968d75effSDimitry Andric bool Success = false; 39068d75effSDimitry Andric new (&BufferQueueStorage) 39168d75effSDimitry Andric BufferQueue(profilingFlags()->per_thread_allocator_max, 39268d75effSDimitry Andric profilingFlags()->buffers_max, Success); 39368d75effSDimitry Andric if (!Success) { 39468d75effSDimitry Andric if (Verbosity()) 39568d75effSDimitry Andric Report("Failed to initialize preallocated memory buffers!"); 39668d75effSDimitry Andric atomic_store(&ProfilerLogStatus, 39768d75effSDimitry Andric XRayLogInitStatus::XRAY_LOG_UNINITIALIZED, 39868d75effSDimitry Andric memory_order_release); 39968d75effSDimitry Andric return XRayLogInitStatus::XRAY_LOG_UNINITIALIZED; 40068d75effSDimitry Andric } 40168d75effSDimitry Andric 402349cc55cSDimitry Andric // If we've succeeded, set the global pointer to the initialised storage. 40368d75effSDimitry Andric BQ = reinterpret_cast<BufferQueue *>(&BufferQueueStorage); 40468d75effSDimitry Andric } else { 40568d75effSDimitry Andric BQ->finalize(); 40668d75effSDimitry Andric auto InitStatus = BQ->init(profilingFlags()->per_thread_allocator_max, 40768d75effSDimitry Andric profilingFlags()->buffers_max); 40868d75effSDimitry Andric 40968d75effSDimitry Andric if (InitStatus != BufferQueue::ErrorCode::Ok) { 41068d75effSDimitry Andric if (Verbosity()) 41168d75effSDimitry Andric Report("Failed to initialize preallocated memory buffers; error: %s", 41268d75effSDimitry Andric BufferQueue::getErrorString(InitStatus)); 41368d75effSDimitry Andric atomic_store(&ProfilerLogStatus, 41468d75effSDimitry Andric XRayLogInitStatus::XRAY_LOG_UNINITIALIZED, 41568d75effSDimitry Andric memory_order_release); 41668d75effSDimitry Andric return XRayLogInitStatus::XRAY_LOG_UNINITIALIZED; 41768d75effSDimitry Andric } 41868d75effSDimitry Andric 41968d75effSDimitry Andric DCHECK(!BQ->finalizing()); 42068d75effSDimitry Andric } 42168d75effSDimitry Andric 42268d75effSDimitry Andric // We need to set up the exit handlers. 42368d75effSDimitry Andric static pthread_once_t Once = PTHREAD_ONCE_INIT; 42468d75effSDimitry Andric pthread_once( 42568d75effSDimitry Andric &Once, +[] { 42668d75effSDimitry Andric pthread_key_create( 42768d75effSDimitry Andric &ProfilingKey, +[](void *P) XRAY_NEVER_INSTRUMENT { 42868d75effSDimitry Andric if (atomic_exchange(&ThreadExitingLatch, 1, memory_order_acq_rel)) 42968d75effSDimitry Andric return; 43068d75effSDimitry Andric 43168d75effSDimitry Andric if (P == nullptr) 43268d75effSDimitry Andric return; 43368d75effSDimitry Andric 43468d75effSDimitry Andric auto T = reinterpret_cast<ProfilingData *>(P); 43568d75effSDimitry Andric if (atomic_load_relaxed(&T->Allocators) == 0) 43668d75effSDimitry Andric return; 43768d75effSDimitry Andric 43868d75effSDimitry Andric { 43968d75effSDimitry Andric // If we're somehow executing this while inside a 44068d75effSDimitry Andric // non-reentrant-friendly context, we skip attempting to post 44168d75effSDimitry Andric // the current thread's data. 44268d75effSDimitry Andric RecursionGuard G(ReentranceGuard); 44368d75effSDimitry Andric if (!G) 44468d75effSDimitry Andric return; 44568d75effSDimitry Andric 44668d75effSDimitry Andric postCurrentThreadFCT(*T); 44768d75effSDimitry Andric } 44868d75effSDimitry Andric }); 44968d75effSDimitry Andric 45068d75effSDimitry Andric // We also need to set up an exit handler, so that we can get the 45168d75effSDimitry Andric // profile information at exit time. We use the C API to do this, to not 45268d75effSDimitry Andric // rely on C++ ABI functions for registering exit handlers. 45368d75effSDimitry Andric Atexit(+[]() XRAY_NEVER_INSTRUMENT { 45468d75effSDimitry Andric if (atomic_exchange(&ThreadExitingLatch, 1, memory_order_acq_rel)) 45568d75effSDimitry Andric return; 45668d75effSDimitry Andric 45768d75effSDimitry Andric auto Cleanup = 45868d75effSDimitry Andric at_scope_exit([]() XRAY_NEVER_INSTRUMENT { cleanupTLD(); }); 45968d75effSDimitry Andric 46068d75effSDimitry Andric // Finalize and flush. 46168d75effSDimitry Andric if (profilingFinalize() != XRAY_LOG_FINALIZED || 46268d75effSDimitry Andric profilingFlush() != XRAY_LOG_FLUSHED) 46368d75effSDimitry Andric return; 46468d75effSDimitry Andric 46568d75effSDimitry Andric if (Verbosity()) 46668d75effSDimitry Andric Report("XRay Profile flushed at exit."); 46768d75effSDimitry Andric }); 46868d75effSDimitry Andric }); 46968d75effSDimitry Andric 47068d75effSDimitry Andric __xray_log_set_buffer_iterator(profileCollectorService::nextBuffer); 47168d75effSDimitry Andric __xray_set_handler(profilingHandleArg0); 47268d75effSDimitry Andric __xray_set_handler_arg1(profilingHandleArg1); 47368d75effSDimitry Andric 47468d75effSDimitry Andric atomic_store(&ProfilerLogStatus, XRayLogInitStatus::XRAY_LOG_INITIALIZED, 47568d75effSDimitry Andric memory_order_release); 47668d75effSDimitry Andric if (Verbosity()) 47768d75effSDimitry Andric Report("XRay Profiling init successful.\n"); 47868d75effSDimitry Andric 47968d75effSDimitry Andric return XRayLogInitStatus::XRAY_LOG_INITIALIZED; 48068d75effSDimitry Andric } 48168d75effSDimitry Andric 48268d75effSDimitry Andric bool profilingDynamicInitializer() XRAY_NEVER_INSTRUMENT { 48368d75effSDimitry Andric // Set up the flag defaults from the static defaults and the 48468d75effSDimitry Andric // compiler-provided defaults. 48568d75effSDimitry Andric { 48668d75effSDimitry Andric SpinMutexLock Lock(&ProfilerOptionsMutex); 48768d75effSDimitry Andric auto *F = profilingFlags(); 48868d75effSDimitry Andric F->setDefaults(); 48968d75effSDimitry Andric FlagParser ProfilingParser; 49068d75effSDimitry Andric registerProfilerFlags(&ProfilingParser, F); 49168d75effSDimitry Andric ProfilingParser.ParseString(profilingCompilerDefinedFlags()); 49268d75effSDimitry Andric } 49368d75effSDimitry Andric 49468d75effSDimitry Andric XRayLogImpl Impl{ 49568d75effSDimitry Andric profilingLoggingInit, 49668d75effSDimitry Andric profilingFinalize, 49768d75effSDimitry Andric profilingHandleArg0, 49868d75effSDimitry Andric profilingFlush, 49968d75effSDimitry Andric }; 50068d75effSDimitry Andric auto RegistrationResult = __xray_log_register_mode("xray-profiling", Impl); 50168d75effSDimitry Andric if (RegistrationResult != XRayLogRegisterStatus::XRAY_REGISTRATION_OK) { 50268d75effSDimitry Andric if (Verbosity()) 50368d75effSDimitry Andric Report("Cannot register XRay Profiling mode to 'xray-profiling'; error = " 50468d75effSDimitry Andric "%d\n", 50568d75effSDimitry Andric RegistrationResult); 50668d75effSDimitry Andric return false; 50768d75effSDimitry Andric } 50868d75effSDimitry Andric 50968d75effSDimitry Andric if (!internal_strcmp(flags()->xray_mode, "xray-profiling")) 51068d75effSDimitry Andric __xray_log_select_mode("xray_profiling"); 51168d75effSDimitry Andric return true; 51268d75effSDimitry Andric } 51368d75effSDimitry Andric 51468d75effSDimitry Andric } // namespace __xray 51568d75effSDimitry Andric 51668d75effSDimitry Andric static auto UNUSED Unused = __xray::profilingDynamicInitializer(); 517