168d75effSDimitry Andric //===-- xray_basic_logging.cpp ----------------------------------*- C++ -*-===//
268d75effSDimitry Andric //
368d75effSDimitry Andric // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
468d75effSDimitry Andric // See https://llvm.org/LICENSE.txt for license information.
568d75effSDimitry Andric // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
668d75effSDimitry Andric //
768d75effSDimitry Andric //===----------------------------------------------------------------------===//
868d75effSDimitry Andric //
968d75effSDimitry Andric // This file is a part of XRay, a dynamic runtime instrumentation system.
1068d75effSDimitry Andric //
1168d75effSDimitry Andric // Implementation of a simple in-memory log of XRay events. This defines a
1268d75effSDimitry Andric // logging function that's compatible with the XRay handler interface, and
1368d75effSDimitry Andric // routines for exporting data to files.
1468d75effSDimitry Andric //
1568d75effSDimitry Andric //===----------------------------------------------------------------------===//
1668d75effSDimitry Andric
1768d75effSDimitry Andric #include <errno.h>
1868d75effSDimitry Andric #include <fcntl.h>
1968d75effSDimitry Andric #include <pthread.h>
2068d75effSDimitry Andric #include <sys/stat.h>
21*81ad6265SDimitry Andric #if SANITIZER_FREEBSD || SANITIZER_NETBSD || SANITIZER_APPLE
2268d75effSDimitry Andric #include <sys/syscall.h>
2368d75effSDimitry Andric #endif
2468d75effSDimitry Andric #include <sys/types.h>
2568d75effSDimitry Andric #include <time.h>
2668d75effSDimitry Andric #include <unistd.h>
2768d75effSDimitry Andric
2868d75effSDimitry Andric #include "sanitizer_common/sanitizer_allocator_internal.h"
2968d75effSDimitry Andric #include "sanitizer_common/sanitizer_libc.h"
3068d75effSDimitry Andric #include "xray/xray_records.h"
3168d75effSDimitry Andric #include "xray_recursion_guard.h"
3268d75effSDimitry Andric #include "xray_basic_flags.h"
3368d75effSDimitry Andric #include "xray_basic_logging.h"
3468d75effSDimitry Andric #include "xray_defs.h"
3568d75effSDimitry Andric #include "xray_flags.h"
3668d75effSDimitry Andric #include "xray_interface_internal.h"
3768d75effSDimitry Andric #include "xray_tsc.h"
3868d75effSDimitry Andric #include "xray_utils.h"
3968d75effSDimitry Andric
4068d75effSDimitry Andric namespace __xray {
4168d75effSDimitry Andric
4268d75effSDimitry Andric static SpinMutex LogMutex;
4368d75effSDimitry Andric
4468d75effSDimitry Andric namespace {
4568d75effSDimitry Andric // We use elements of this type to record the entry TSC of every function ID we
4668d75effSDimitry Andric // see as we're tracing a particular thread's execution.
4768d75effSDimitry Andric struct alignas(16) StackEntry {
4868d75effSDimitry Andric int32_t FuncId;
4968d75effSDimitry Andric uint16_t Type;
5068d75effSDimitry Andric uint8_t CPU;
5168d75effSDimitry Andric uint8_t Padding;
5268d75effSDimitry Andric uint64_t TSC;
5368d75effSDimitry Andric };
5468d75effSDimitry Andric
5568d75effSDimitry Andric static_assert(sizeof(StackEntry) == 16, "Wrong size for StackEntry");
5668d75effSDimitry Andric
5768d75effSDimitry Andric struct XRAY_TLS_ALIGNAS(64) ThreadLocalData {
5868d75effSDimitry Andric void *InMemoryBuffer = nullptr;
5968d75effSDimitry Andric size_t BufferSize = 0;
6068d75effSDimitry Andric size_t BufferOffset = 0;
6168d75effSDimitry Andric void *ShadowStack = nullptr;
6268d75effSDimitry Andric size_t StackSize = 0;
6368d75effSDimitry Andric size_t StackEntries = 0;
6468d75effSDimitry Andric __xray::LogWriter *LogWriter = nullptr;
6568d75effSDimitry Andric };
6668d75effSDimitry Andric
6768d75effSDimitry Andric struct BasicLoggingOptions {
6868d75effSDimitry Andric int DurationFilterMicros = 0;
6968d75effSDimitry Andric size_t MaxStackDepth = 0;
7068d75effSDimitry Andric size_t ThreadBufferSize = 0;
7168d75effSDimitry Andric };
7268d75effSDimitry Andric } // namespace
7368d75effSDimitry Andric
7468d75effSDimitry Andric static pthread_key_t PThreadKey;
7568d75effSDimitry Andric
7668d75effSDimitry Andric static atomic_uint8_t BasicInitialized{0};
7768d75effSDimitry Andric
7868d75effSDimitry Andric struct BasicLoggingOptions GlobalOptions;
7968d75effSDimitry Andric
8068d75effSDimitry Andric thread_local atomic_uint8_t Guard{0};
8168d75effSDimitry Andric
8268d75effSDimitry Andric static atomic_uint8_t UseRealTSC{0};
8368d75effSDimitry Andric static atomic_uint64_t ThresholdTicks{0};
8468d75effSDimitry Andric static atomic_uint64_t TicksPerSec{0};
8568d75effSDimitry Andric static atomic_uint64_t CycleFrequency{NanosecondsPerSecond};
8668d75effSDimitry Andric
getLog()8768d75effSDimitry Andric static LogWriter *getLog() XRAY_NEVER_INSTRUMENT {
8868d75effSDimitry Andric LogWriter* LW = LogWriter::Open();
8968d75effSDimitry Andric if (LW == nullptr)
9068d75effSDimitry Andric return LW;
9168d75effSDimitry Andric
9268d75effSDimitry Andric static pthread_once_t DetectOnce = PTHREAD_ONCE_INIT;
9368d75effSDimitry Andric pthread_once(&DetectOnce, +[] {
9468d75effSDimitry Andric if (atomic_load(&UseRealTSC, memory_order_acquire))
9568d75effSDimitry Andric atomic_store(&CycleFrequency, getTSCFrequency(), memory_order_release);
9668d75effSDimitry Andric });
9768d75effSDimitry Andric
9868d75effSDimitry Andric // Since we're here, we get to write the header. We set it up so that the
9968d75effSDimitry Andric // header will only be written once, at the start, and let the threads
10068d75effSDimitry Andric // logging do writes which just append.
10168d75effSDimitry Andric XRayFileHeader Header;
10268d75effSDimitry Andric // Version 2 includes tail exit records.
10368d75effSDimitry Andric // Version 3 includes pid inside records.
10468d75effSDimitry Andric Header.Version = 3;
10568d75effSDimitry Andric Header.Type = FileTypes::NAIVE_LOG;
10668d75effSDimitry Andric Header.CycleFrequency = atomic_load(&CycleFrequency, memory_order_acquire);
10768d75effSDimitry Andric
10868d75effSDimitry Andric // FIXME: Actually check whether we have 'constant_tsc' and 'nonstop_tsc'
10968d75effSDimitry Andric // before setting the values in the header.
11068d75effSDimitry Andric Header.ConstantTSC = 1;
11168d75effSDimitry Andric Header.NonstopTSC = 1;
11268d75effSDimitry Andric LW->WriteAll(reinterpret_cast<char *>(&Header),
11368d75effSDimitry Andric reinterpret_cast<char *>(&Header) + sizeof(Header));
11468d75effSDimitry Andric return LW;
11568d75effSDimitry Andric }
11668d75effSDimitry Andric
getGlobalLog()11768d75effSDimitry Andric static LogWriter *getGlobalLog() XRAY_NEVER_INSTRUMENT {
11868d75effSDimitry Andric static pthread_once_t OnceInit = PTHREAD_ONCE_INIT;
11968d75effSDimitry Andric static LogWriter *LW = nullptr;
12068d75effSDimitry Andric pthread_once(&OnceInit, +[] { LW = getLog(); });
12168d75effSDimitry Andric return LW;
12268d75effSDimitry Andric }
12368d75effSDimitry Andric
getThreadLocalData()12468d75effSDimitry Andric static ThreadLocalData &getThreadLocalData() XRAY_NEVER_INSTRUMENT {
12568d75effSDimitry Andric thread_local ThreadLocalData TLD;
12668d75effSDimitry Andric thread_local bool UNUSED TOnce = [] {
12768d75effSDimitry Andric if (GlobalOptions.ThreadBufferSize == 0) {
12868d75effSDimitry Andric if (Verbosity())
12968d75effSDimitry Andric Report("Not initializing TLD since ThreadBufferSize == 0.\n");
13068d75effSDimitry Andric return false;
13168d75effSDimitry Andric }
13268d75effSDimitry Andric pthread_setspecific(PThreadKey, &TLD);
13368d75effSDimitry Andric TLD.LogWriter = getGlobalLog();
13468d75effSDimitry Andric TLD.InMemoryBuffer = reinterpret_cast<XRayRecord *>(
13568d75effSDimitry Andric InternalAlloc(sizeof(XRayRecord) * GlobalOptions.ThreadBufferSize,
13668d75effSDimitry Andric nullptr, alignof(XRayRecord)));
13768d75effSDimitry Andric TLD.BufferSize = GlobalOptions.ThreadBufferSize;
13868d75effSDimitry Andric TLD.BufferOffset = 0;
13968d75effSDimitry Andric if (GlobalOptions.MaxStackDepth == 0) {
14068d75effSDimitry Andric if (Verbosity())
14168d75effSDimitry Andric Report("Not initializing the ShadowStack since MaxStackDepth == 0.\n");
14268d75effSDimitry Andric TLD.StackSize = 0;
14368d75effSDimitry Andric TLD.StackEntries = 0;
14468d75effSDimitry Andric TLD.ShadowStack = nullptr;
14568d75effSDimitry Andric return false;
14668d75effSDimitry Andric }
14768d75effSDimitry Andric TLD.ShadowStack = reinterpret_cast<StackEntry *>(
14868d75effSDimitry Andric InternalAlloc(sizeof(StackEntry) * GlobalOptions.MaxStackDepth, nullptr,
14968d75effSDimitry Andric alignof(StackEntry)));
15068d75effSDimitry Andric TLD.StackSize = GlobalOptions.MaxStackDepth;
15168d75effSDimitry Andric TLD.StackEntries = 0;
15268d75effSDimitry Andric return false;
15368d75effSDimitry Andric }();
15468d75effSDimitry Andric return TLD;
15568d75effSDimitry Andric }
15668d75effSDimitry Andric
15768d75effSDimitry Andric template <class RDTSC>
InMemoryRawLog(int32_t FuncId,XRayEntryType Type,RDTSC ReadTSC)15868d75effSDimitry Andric void InMemoryRawLog(int32_t FuncId, XRayEntryType Type,
15968d75effSDimitry Andric RDTSC ReadTSC) XRAY_NEVER_INSTRUMENT {
16068d75effSDimitry Andric auto &TLD = getThreadLocalData();
16168d75effSDimitry Andric LogWriter *LW = getGlobalLog();
16268d75effSDimitry Andric if (LW == nullptr)
16368d75effSDimitry Andric return;
16468d75effSDimitry Andric
16568d75effSDimitry Andric // Use a simple recursion guard, to handle cases where we're already logging
16668d75effSDimitry Andric // and for one reason or another, this function gets called again in the same
16768d75effSDimitry Andric // thread.
16868d75effSDimitry Andric RecursionGuard G(Guard);
16968d75effSDimitry Andric if (!G)
17068d75effSDimitry Andric return;
17168d75effSDimitry Andric
17268d75effSDimitry Andric uint8_t CPU = 0;
17368d75effSDimitry Andric uint64_t TSC = ReadTSC(CPU);
17468d75effSDimitry Andric
17568d75effSDimitry Andric switch (Type) {
17668d75effSDimitry Andric case XRayEntryType::ENTRY:
17768d75effSDimitry Andric case XRayEntryType::LOG_ARGS_ENTRY: {
17868d75effSDimitry Andric // Short circuit if we've reached the maximum depth of the stack.
17968d75effSDimitry Andric if (TLD.StackEntries++ >= TLD.StackSize)
18068d75effSDimitry Andric return;
18168d75effSDimitry Andric
18268d75effSDimitry Andric // When we encounter an entry event, we keep track of the TSC and the CPU,
18368d75effSDimitry Andric // and put it in the stack.
18468d75effSDimitry Andric StackEntry E;
18568d75effSDimitry Andric E.FuncId = FuncId;
18668d75effSDimitry Andric E.CPU = CPU;
18768d75effSDimitry Andric E.Type = Type;
18868d75effSDimitry Andric E.TSC = TSC;
18968d75effSDimitry Andric auto StackEntryPtr = static_cast<char *>(TLD.ShadowStack) +
19068d75effSDimitry Andric (sizeof(StackEntry) * (TLD.StackEntries - 1));
19168d75effSDimitry Andric internal_memcpy(StackEntryPtr, &E, sizeof(StackEntry));
19268d75effSDimitry Andric break;
19368d75effSDimitry Andric }
19468d75effSDimitry Andric case XRayEntryType::EXIT:
19568d75effSDimitry Andric case XRayEntryType::TAIL: {
19668d75effSDimitry Andric if (TLD.StackEntries == 0)
19768d75effSDimitry Andric break;
19868d75effSDimitry Andric
19968d75effSDimitry Andric if (--TLD.StackEntries >= TLD.StackSize)
20068d75effSDimitry Andric return;
20168d75effSDimitry Andric
20268d75effSDimitry Andric // When we encounter an exit event, we check whether all the following are
20368d75effSDimitry Andric // true:
20468d75effSDimitry Andric //
20568d75effSDimitry Andric // - The Function ID is the same as the most recent entry in the stack.
20668d75effSDimitry Andric // - The CPU is the same as the most recent entry in the stack.
20768d75effSDimitry Andric // - The Delta of the TSCs is less than the threshold amount of time we're
20868d75effSDimitry Andric // looking to record.
20968d75effSDimitry Andric //
21068d75effSDimitry Andric // If all of these conditions are true, we pop the stack and don't write a
21168d75effSDimitry Andric // record and move the record offset back.
21268d75effSDimitry Andric StackEntry StackTop;
21368d75effSDimitry Andric auto StackEntryPtr = static_cast<char *>(TLD.ShadowStack) +
21468d75effSDimitry Andric (sizeof(StackEntry) * TLD.StackEntries);
21568d75effSDimitry Andric internal_memcpy(&StackTop, StackEntryPtr, sizeof(StackEntry));
21668d75effSDimitry Andric if (StackTop.FuncId == FuncId && StackTop.CPU == CPU &&
21768d75effSDimitry Andric StackTop.TSC < TSC) {
21868d75effSDimitry Andric auto Delta = TSC - StackTop.TSC;
21968d75effSDimitry Andric if (Delta < atomic_load(&ThresholdTicks, memory_order_relaxed)) {
22068d75effSDimitry Andric DCHECK(TLD.BufferOffset > 0);
22168d75effSDimitry Andric TLD.BufferOffset -= StackTop.Type == XRayEntryType::ENTRY ? 1 : 2;
22268d75effSDimitry Andric return;
22368d75effSDimitry Andric }
22468d75effSDimitry Andric }
22568d75effSDimitry Andric break;
22668d75effSDimitry Andric }
22768d75effSDimitry Andric default:
22868d75effSDimitry Andric // Should be unreachable.
22968d75effSDimitry Andric DCHECK(false && "Unsupported XRayEntryType encountered.");
23068d75effSDimitry Andric break;
23168d75effSDimitry Andric }
23268d75effSDimitry Andric
23368d75effSDimitry Andric // First determine whether the delta between the function's enter record and
23468d75effSDimitry Andric // the exit record is higher than the threshold.
23568d75effSDimitry Andric XRayRecord R;
23668d75effSDimitry Andric R.RecordType = RecordTypes::NORMAL;
23768d75effSDimitry Andric R.CPU = CPU;
23868d75effSDimitry Andric R.TSC = TSC;
23968d75effSDimitry Andric R.TId = GetTid();
24068d75effSDimitry Andric R.PId = internal_getpid();
24168d75effSDimitry Andric R.Type = Type;
24268d75effSDimitry Andric R.FuncId = FuncId;
24368d75effSDimitry Andric auto FirstEntry = reinterpret_cast<XRayRecord *>(TLD.InMemoryBuffer);
24468d75effSDimitry Andric internal_memcpy(FirstEntry + TLD.BufferOffset, &R, sizeof(R));
24568d75effSDimitry Andric if (++TLD.BufferOffset == TLD.BufferSize) {
24668d75effSDimitry Andric SpinMutexLock Lock(&LogMutex);
24768d75effSDimitry Andric LW->WriteAll(reinterpret_cast<char *>(FirstEntry),
24868d75effSDimitry Andric reinterpret_cast<char *>(FirstEntry + TLD.BufferOffset));
24968d75effSDimitry Andric TLD.BufferOffset = 0;
25068d75effSDimitry Andric TLD.StackEntries = 0;
25168d75effSDimitry Andric }
25268d75effSDimitry Andric }
25368d75effSDimitry Andric
25468d75effSDimitry Andric template <class RDTSC>
InMemoryRawLogWithArg(int32_t FuncId,XRayEntryType Type,uint64_t Arg1,RDTSC ReadTSC)25568d75effSDimitry Andric void InMemoryRawLogWithArg(int32_t FuncId, XRayEntryType Type, uint64_t Arg1,
25668d75effSDimitry Andric RDTSC ReadTSC) XRAY_NEVER_INSTRUMENT {
25768d75effSDimitry Andric auto &TLD = getThreadLocalData();
25868d75effSDimitry Andric auto FirstEntry =
25968d75effSDimitry Andric reinterpret_cast<XRayArgPayload *>(TLD.InMemoryBuffer);
26068d75effSDimitry Andric const auto &BuffLen = TLD.BufferSize;
26168d75effSDimitry Andric LogWriter *LW = getGlobalLog();
26268d75effSDimitry Andric if (LW == nullptr)
26368d75effSDimitry Andric return;
26468d75effSDimitry Andric
26568d75effSDimitry Andric // First we check whether there's enough space to write the data consecutively
26668d75effSDimitry Andric // in the thread-local buffer. If not, we first flush the buffer before
26768d75effSDimitry Andric // attempting to write the two records that must be consecutive.
26868d75effSDimitry Andric if (TLD.BufferOffset + 2 > BuffLen) {
26968d75effSDimitry Andric SpinMutexLock Lock(&LogMutex);
27068d75effSDimitry Andric LW->WriteAll(reinterpret_cast<char *>(FirstEntry),
27168d75effSDimitry Andric reinterpret_cast<char *>(FirstEntry + TLD.BufferOffset));
27268d75effSDimitry Andric TLD.BufferOffset = 0;
27368d75effSDimitry Andric TLD.StackEntries = 0;
27468d75effSDimitry Andric }
27568d75effSDimitry Andric
27668d75effSDimitry Andric // Then we write the "we have an argument" record.
27768d75effSDimitry Andric InMemoryRawLog(FuncId, Type, ReadTSC);
27868d75effSDimitry Andric
27968d75effSDimitry Andric RecursionGuard G(Guard);
28068d75effSDimitry Andric if (!G)
28168d75effSDimitry Andric return;
28268d75effSDimitry Andric
28368d75effSDimitry Andric // And, from here on write the arg payload.
28468d75effSDimitry Andric XRayArgPayload R;
28568d75effSDimitry Andric R.RecordType = RecordTypes::ARG_PAYLOAD;
28668d75effSDimitry Andric R.FuncId = FuncId;
28768d75effSDimitry Andric R.TId = GetTid();
28868d75effSDimitry Andric R.PId = internal_getpid();
28968d75effSDimitry Andric R.Arg = Arg1;
29068d75effSDimitry Andric internal_memcpy(FirstEntry + TLD.BufferOffset, &R, sizeof(R));
29168d75effSDimitry Andric if (++TLD.BufferOffset == BuffLen) {
29268d75effSDimitry Andric SpinMutexLock Lock(&LogMutex);
29368d75effSDimitry Andric LW->WriteAll(reinterpret_cast<char *>(FirstEntry),
29468d75effSDimitry Andric reinterpret_cast<char *>(FirstEntry + TLD.BufferOffset));
29568d75effSDimitry Andric TLD.BufferOffset = 0;
29668d75effSDimitry Andric TLD.StackEntries = 0;
29768d75effSDimitry Andric }
29868d75effSDimitry Andric }
29968d75effSDimitry Andric
basicLoggingHandleArg0RealTSC(int32_t FuncId,XRayEntryType Type)30068d75effSDimitry Andric void basicLoggingHandleArg0RealTSC(int32_t FuncId,
30168d75effSDimitry Andric XRayEntryType Type) XRAY_NEVER_INSTRUMENT {
30268d75effSDimitry Andric InMemoryRawLog(FuncId, Type, readTSC);
30368d75effSDimitry Andric }
30468d75effSDimitry Andric
basicLoggingHandleArg0EmulateTSC(int32_t FuncId,XRayEntryType Type)30568d75effSDimitry Andric void basicLoggingHandleArg0EmulateTSC(int32_t FuncId, XRayEntryType Type)
30668d75effSDimitry Andric XRAY_NEVER_INSTRUMENT {
30768d75effSDimitry Andric InMemoryRawLog(FuncId, Type, [](uint8_t &CPU) XRAY_NEVER_INSTRUMENT {
30868d75effSDimitry Andric timespec TS;
30968d75effSDimitry Andric int result = clock_gettime(CLOCK_REALTIME, &TS);
31068d75effSDimitry Andric if (result != 0) {
31168d75effSDimitry Andric Report("clock_gettimg(2) return %d, errno=%d.", result, int(errno));
31268d75effSDimitry Andric TS = {0, 0};
31368d75effSDimitry Andric }
31468d75effSDimitry Andric CPU = 0;
31568d75effSDimitry Andric return TS.tv_sec * NanosecondsPerSecond + TS.tv_nsec;
31668d75effSDimitry Andric });
31768d75effSDimitry Andric }
31868d75effSDimitry Andric
basicLoggingHandleArg1RealTSC(int32_t FuncId,XRayEntryType Type,uint64_t Arg1)31968d75effSDimitry Andric void basicLoggingHandleArg1RealTSC(int32_t FuncId, XRayEntryType Type,
32068d75effSDimitry Andric uint64_t Arg1) XRAY_NEVER_INSTRUMENT {
32168d75effSDimitry Andric InMemoryRawLogWithArg(FuncId, Type, Arg1, readTSC);
32268d75effSDimitry Andric }
32368d75effSDimitry Andric
basicLoggingHandleArg1EmulateTSC(int32_t FuncId,XRayEntryType Type,uint64_t Arg1)32468d75effSDimitry Andric void basicLoggingHandleArg1EmulateTSC(int32_t FuncId, XRayEntryType Type,
32568d75effSDimitry Andric uint64_t Arg1) XRAY_NEVER_INSTRUMENT {
32668d75effSDimitry Andric InMemoryRawLogWithArg(
32768d75effSDimitry Andric FuncId, Type, Arg1, [](uint8_t &CPU) XRAY_NEVER_INSTRUMENT {
32868d75effSDimitry Andric timespec TS;
32968d75effSDimitry Andric int result = clock_gettime(CLOCK_REALTIME, &TS);
33068d75effSDimitry Andric if (result != 0) {
33168d75effSDimitry Andric Report("clock_gettimg(2) return %d, errno=%d.", result, int(errno));
33268d75effSDimitry Andric TS = {0, 0};
33368d75effSDimitry Andric }
33468d75effSDimitry Andric CPU = 0;
33568d75effSDimitry Andric return TS.tv_sec * NanosecondsPerSecond + TS.tv_nsec;
33668d75effSDimitry Andric });
33768d75effSDimitry Andric }
33868d75effSDimitry Andric
TLDDestructor(void * P)33968d75effSDimitry Andric static void TLDDestructor(void *P) XRAY_NEVER_INSTRUMENT {
34068d75effSDimitry Andric ThreadLocalData &TLD = *reinterpret_cast<ThreadLocalData *>(P);
34168d75effSDimitry Andric auto ExitGuard = at_scope_exit([&TLD] {
34268d75effSDimitry Andric // Clean up dynamic resources.
34368d75effSDimitry Andric if (TLD.InMemoryBuffer)
34468d75effSDimitry Andric InternalFree(TLD.InMemoryBuffer);
34568d75effSDimitry Andric if (TLD.ShadowStack)
34668d75effSDimitry Andric InternalFree(TLD.ShadowStack);
34768d75effSDimitry Andric if (Verbosity())
3480eae32dcSDimitry Andric Report("Cleaned up log for TID: %llu\n", GetTid());
34968d75effSDimitry Andric });
35068d75effSDimitry Andric
35168d75effSDimitry Andric if (TLD.LogWriter == nullptr || TLD.BufferOffset == 0) {
35268d75effSDimitry Andric if (Verbosity())
3530eae32dcSDimitry Andric Report("Skipping buffer for TID: %llu; Offset = %zu\n", GetTid(),
35468d75effSDimitry Andric TLD.BufferOffset);
35568d75effSDimitry Andric return;
35668d75effSDimitry Andric }
35768d75effSDimitry Andric
35868d75effSDimitry Andric {
35968d75effSDimitry Andric SpinMutexLock L(&LogMutex);
36068d75effSDimitry Andric TLD.LogWriter->WriteAll(reinterpret_cast<char *>(TLD.InMemoryBuffer),
36168d75effSDimitry Andric reinterpret_cast<char *>(TLD.InMemoryBuffer) +
36268d75effSDimitry Andric (sizeof(XRayRecord) * TLD.BufferOffset));
36368d75effSDimitry Andric }
36468d75effSDimitry Andric
36568d75effSDimitry Andric // Because this thread's exit could be the last one trying to write to
36668d75effSDimitry Andric // the file and that we're not able to close out the file properly, we
36768d75effSDimitry Andric // sync instead and hope that the pending writes are flushed as the
36868d75effSDimitry Andric // thread exits.
36968d75effSDimitry Andric TLD.LogWriter->Flush();
37068d75effSDimitry Andric }
37168d75effSDimitry Andric
basicLoggingInit(UNUSED size_t BufferSize,UNUSED size_t BufferMax,void * Options,size_t OptionsSize)37268d75effSDimitry Andric XRayLogInitStatus basicLoggingInit(UNUSED size_t BufferSize,
37368d75effSDimitry Andric UNUSED size_t BufferMax, void *Options,
37468d75effSDimitry Andric size_t OptionsSize) XRAY_NEVER_INSTRUMENT {
37568d75effSDimitry Andric uint8_t Expected = 0;
37668d75effSDimitry Andric if (!atomic_compare_exchange_strong(&BasicInitialized, &Expected, 1,
37768d75effSDimitry Andric memory_order_acq_rel)) {
37868d75effSDimitry Andric if (Verbosity())
37968d75effSDimitry Andric Report("Basic logging already initialized.\n");
38068d75effSDimitry Andric return XRayLogInitStatus::XRAY_LOG_INITIALIZED;
38168d75effSDimitry Andric }
38268d75effSDimitry Andric
38368d75effSDimitry Andric static pthread_once_t OnceInit = PTHREAD_ONCE_INIT;
38468d75effSDimitry Andric pthread_once(&OnceInit, +[] {
38568d75effSDimitry Andric pthread_key_create(&PThreadKey, TLDDestructor);
38668d75effSDimitry Andric atomic_store(&UseRealTSC, probeRequiredCPUFeatures(), memory_order_release);
38768d75effSDimitry Andric // Initialize the global TicksPerSec value.
38868d75effSDimitry Andric atomic_store(&TicksPerSec,
38968d75effSDimitry Andric probeRequiredCPUFeatures() ? getTSCFrequency()
39068d75effSDimitry Andric : NanosecondsPerSecond,
39168d75effSDimitry Andric memory_order_release);
39268d75effSDimitry Andric if (!atomic_load(&UseRealTSC, memory_order_relaxed) && Verbosity())
39368d75effSDimitry Andric Report("WARNING: Required CPU features missing for XRay instrumentation, "
39468d75effSDimitry Andric "using emulation instead.\n");
39568d75effSDimitry Andric });
39668d75effSDimitry Andric
39768d75effSDimitry Andric FlagParser P;
39868d75effSDimitry Andric BasicFlags F;
39968d75effSDimitry Andric F.setDefaults();
40068d75effSDimitry Andric registerXRayBasicFlags(&P, &F);
40168d75effSDimitry Andric P.ParseString(useCompilerDefinedBasicFlags());
40268d75effSDimitry Andric auto *EnvOpts = GetEnv("XRAY_BASIC_OPTIONS");
40368d75effSDimitry Andric if (EnvOpts == nullptr)
40468d75effSDimitry Andric EnvOpts = "";
40568d75effSDimitry Andric
40668d75effSDimitry Andric P.ParseString(EnvOpts);
40768d75effSDimitry Andric
40868d75effSDimitry Andric // If XRAY_BASIC_OPTIONS was not defined, then we use the deprecated options
40968d75effSDimitry Andric // set through XRAY_OPTIONS instead.
41068d75effSDimitry Andric if (internal_strlen(EnvOpts) == 0) {
41168d75effSDimitry Andric F.func_duration_threshold_us =
41268d75effSDimitry Andric flags()->xray_naive_log_func_duration_threshold_us;
41368d75effSDimitry Andric F.max_stack_depth = flags()->xray_naive_log_max_stack_depth;
41468d75effSDimitry Andric F.thread_buffer_size = flags()->xray_naive_log_thread_buffer_size;
41568d75effSDimitry Andric }
41668d75effSDimitry Andric
41768d75effSDimitry Andric P.ParseString(static_cast<const char *>(Options));
41868d75effSDimitry Andric GlobalOptions.ThreadBufferSize = F.thread_buffer_size;
41968d75effSDimitry Andric GlobalOptions.DurationFilterMicros = F.func_duration_threshold_us;
42068d75effSDimitry Andric GlobalOptions.MaxStackDepth = F.max_stack_depth;
42168d75effSDimitry Andric *basicFlags() = F;
42268d75effSDimitry Andric
42368d75effSDimitry Andric atomic_store(&ThresholdTicks,
42468d75effSDimitry Andric atomic_load(&TicksPerSec, memory_order_acquire) *
42568d75effSDimitry Andric GlobalOptions.DurationFilterMicros / 1000000,
42668d75effSDimitry Andric memory_order_release);
42768d75effSDimitry Andric __xray_set_handler_arg1(atomic_load(&UseRealTSC, memory_order_acquire)
42868d75effSDimitry Andric ? basicLoggingHandleArg1RealTSC
42968d75effSDimitry Andric : basicLoggingHandleArg1EmulateTSC);
43068d75effSDimitry Andric __xray_set_handler(atomic_load(&UseRealTSC, memory_order_acquire)
43168d75effSDimitry Andric ? basicLoggingHandleArg0RealTSC
43268d75effSDimitry Andric : basicLoggingHandleArg0EmulateTSC);
43368d75effSDimitry Andric
43468d75effSDimitry Andric // TODO: Implement custom event and typed event handling support in Basic
43568d75effSDimitry Andric // Mode.
43668d75effSDimitry Andric __xray_remove_customevent_handler();
43768d75effSDimitry Andric __xray_remove_typedevent_handler();
43868d75effSDimitry Andric
43968d75effSDimitry Andric return XRayLogInitStatus::XRAY_LOG_INITIALIZED;
44068d75effSDimitry Andric }
44168d75effSDimitry Andric
basicLoggingFinalize()44268d75effSDimitry Andric XRayLogInitStatus basicLoggingFinalize() XRAY_NEVER_INSTRUMENT {
44368d75effSDimitry Andric uint8_t Expected = 0;
44468d75effSDimitry Andric if (!atomic_compare_exchange_strong(&BasicInitialized, &Expected, 0,
44568d75effSDimitry Andric memory_order_acq_rel) &&
44668d75effSDimitry Andric Verbosity())
44768d75effSDimitry Andric Report("Basic logging already finalized.\n");
44868d75effSDimitry Andric
44968d75effSDimitry Andric // Nothing really to do aside from marking state of the global to be
45068d75effSDimitry Andric // uninitialized.
45168d75effSDimitry Andric
45268d75effSDimitry Andric return XRayLogInitStatus::XRAY_LOG_FINALIZED;
45368d75effSDimitry Andric }
45468d75effSDimitry Andric
basicLoggingFlush()45568d75effSDimitry Andric XRayLogFlushStatus basicLoggingFlush() XRAY_NEVER_INSTRUMENT {
45668d75effSDimitry Andric // This really does nothing, since flushing the logs happen at the end of a
45768d75effSDimitry Andric // thread's lifetime, or when the buffers are full.
45868d75effSDimitry Andric return XRayLogFlushStatus::XRAY_LOG_FLUSHED;
45968d75effSDimitry Andric }
46068d75effSDimitry Andric
46168d75effSDimitry Andric // This is a handler that, effectively, does nothing.
basicLoggingHandleArg0Empty(int32_t,XRayEntryType)46268d75effSDimitry Andric void basicLoggingHandleArg0Empty(int32_t, XRayEntryType) XRAY_NEVER_INSTRUMENT {
46368d75effSDimitry Andric }
46468d75effSDimitry Andric
basicLogDynamicInitializer()46568d75effSDimitry Andric bool basicLogDynamicInitializer() XRAY_NEVER_INSTRUMENT {
46668d75effSDimitry Andric XRayLogImpl Impl{
46768d75effSDimitry Andric basicLoggingInit,
46868d75effSDimitry Andric basicLoggingFinalize,
46968d75effSDimitry Andric basicLoggingHandleArg0Empty,
47068d75effSDimitry Andric basicLoggingFlush,
47168d75effSDimitry Andric };
47268d75effSDimitry Andric auto RegistrationResult = __xray_log_register_mode("xray-basic", Impl);
47368d75effSDimitry Andric if (RegistrationResult != XRayLogRegisterStatus::XRAY_REGISTRATION_OK &&
47468d75effSDimitry Andric Verbosity())
47568d75effSDimitry Andric Report("Cannot register XRay Basic Mode to 'xray-basic'; error = %d\n",
47668d75effSDimitry Andric RegistrationResult);
47768d75effSDimitry Andric if (flags()->xray_naive_log ||
47868d75effSDimitry Andric !internal_strcmp(flags()->xray_mode, "xray-basic")) {
47968d75effSDimitry Andric auto SelectResult = __xray_log_select_mode("xray-basic");
48068d75effSDimitry Andric if (SelectResult != XRayLogRegisterStatus::XRAY_REGISTRATION_OK) {
48168d75effSDimitry Andric if (Verbosity())
48268d75effSDimitry Andric Report("Failed selecting XRay Basic Mode; error = %d\n", SelectResult);
48368d75effSDimitry Andric return false;
48468d75effSDimitry Andric }
48568d75effSDimitry Andric
48668d75effSDimitry Andric // We initialize the implementation using the data we get from the
48768d75effSDimitry Andric // XRAY_BASIC_OPTIONS environment variable, at this point of the
48868d75effSDimitry Andric // implementation.
48968d75effSDimitry Andric auto *Env = GetEnv("XRAY_BASIC_OPTIONS");
49068d75effSDimitry Andric auto InitResult =
49168d75effSDimitry Andric __xray_log_init_mode("xray-basic", Env == nullptr ? "" : Env);
49268d75effSDimitry Andric if (InitResult != XRayLogInitStatus::XRAY_LOG_INITIALIZED) {
49368d75effSDimitry Andric if (Verbosity())
49468d75effSDimitry Andric Report("Failed initializing XRay Basic Mode; error = %d\n", InitResult);
49568d75effSDimitry Andric return false;
49668d75effSDimitry Andric }
49768d75effSDimitry Andric
49868d75effSDimitry Andric // At this point we know that we've successfully initialized Basic mode
49968d75effSDimitry Andric // tracing, and the only chance we're going to get for the current thread to
50068d75effSDimitry Andric // clean-up may be at thread/program exit. To ensure that we're going to get
50168d75effSDimitry Andric // the cleanup even without calling the finalization routines, we're
50268d75effSDimitry Andric // registering a program exit function that will do the cleanup.
50368d75effSDimitry Andric static pthread_once_t DynamicOnce = PTHREAD_ONCE_INIT;
50468d75effSDimitry Andric pthread_once(&DynamicOnce, +[] {
50568d75effSDimitry Andric static void *FakeTLD = nullptr;
50668d75effSDimitry Andric FakeTLD = &getThreadLocalData();
50768d75effSDimitry Andric Atexit(+[] { TLDDestructor(FakeTLD); });
50868d75effSDimitry Andric });
50968d75effSDimitry Andric }
51068d75effSDimitry Andric return true;
51168d75effSDimitry Andric }
51268d75effSDimitry Andric
51368d75effSDimitry Andric } // namespace __xray
51468d75effSDimitry Andric
51568d75effSDimitry Andric static auto UNUSED Unused = __xray::basicLogDynamicInitializer();
516