xref: /freebsd-src/contrib/llvm-project/compiler-rt/lib/xray/xray_basic_logging.cpp (revision 81ad626541db97eb356e2c1d4a20eb2a26a766ab)
168d75effSDimitry Andric //===-- xray_basic_logging.cpp ----------------------------------*- C++ -*-===//
268d75effSDimitry Andric //
368d75effSDimitry Andric // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
468d75effSDimitry Andric // See https://llvm.org/LICENSE.txt for license information.
568d75effSDimitry Andric // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
668d75effSDimitry Andric //
768d75effSDimitry Andric //===----------------------------------------------------------------------===//
868d75effSDimitry Andric //
968d75effSDimitry Andric // This file is a part of XRay, a dynamic runtime instrumentation system.
1068d75effSDimitry Andric //
1168d75effSDimitry Andric // Implementation of a simple in-memory log of XRay events. This defines a
1268d75effSDimitry Andric // logging function that's compatible with the XRay handler interface, and
1368d75effSDimitry Andric // routines for exporting data to files.
1468d75effSDimitry Andric //
1568d75effSDimitry Andric //===----------------------------------------------------------------------===//
1668d75effSDimitry Andric 
1768d75effSDimitry Andric #include <errno.h>
1868d75effSDimitry Andric #include <fcntl.h>
1968d75effSDimitry Andric #include <pthread.h>
2068d75effSDimitry Andric #include <sys/stat.h>
21*81ad6265SDimitry Andric #if SANITIZER_FREEBSD || SANITIZER_NETBSD || SANITIZER_APPLE
2268d75effSDimitry Andric #include <sys/syscall.h>
2368d75effSDimitry Andric #endif
2468d75effSDimitry Andric #include <sys/types.h>
2568d75effSDimitry Andric #include <time.h>
2668d75effSDimitry Andric #include <unistd.h>
2768d75effSDimitry Andric 
2868d75effSDimitry Andric #include "sanitizer_common/sanitizer_allocator_internal.h"
2968d75effSDimitry Andric #include "sanitizer_common/sanitizer_libc.h"
3068d75effSDimitry Andric #include "xray/xray_records.h"
3168d75effSDimitry Andric #include "xray_recursion_guard.h"
3268d75effSDimitry Andric #include "xray_basic_flags.h"
3368d75effSDimitry Andric #include "xray_basic_logging.h"
3468d75effSDimitry Andric #include "xray_defs.h"
3568d75effSDimitry Andric #include "xray_flags.h"
3668d75effSDimitry Andric #include "xray_interface_internal.h"
3768d75effSDimitry Andric #include "xray_tsc.h"
3868d75effSDimitry Andric #include "xray_utils.h"
3968d75effSDimitry Andric 
4068d75effSDimitry Andric namespace __xray {
4168d75effSDimitry Andric 
4268d75effSDimitry Andric static SpinMutex LogMutex;
4368d75effSDimitry Andric 
4468d75effSDimitry Andric namespace {
4568d75effSDimitry Andric // We use elements of this type to record the entry TSC of every function ID we
4668d75effSDimitry Andric // see as we're tracing a particular thread's execution.
4768d75effSDimitry Andric struct alignas(16) StackEntry {
4868d75effSDimitry Andric   int32_t FuncId;
4968d75effSDimitry Andric   uint16_t Type;
5068d75effSDimitry Andric   uint8_t CPU;
5168d75effSDimitry Andric   uint8_t Padding;
5268d75effSDimitry Andric   uint64_t TSC;
5368d75effSDimitry Andric };
5468d75effSDimitry Andric 
5568d75effSDimitry Andric static_assert(sizeof(StackEntry) == 16, "Wrong size for StackEntry");
5668d75effSDimitry Andric 
5768d75effSDimitry Andric struct XRAY_TLS_ALIGNAS(64) ThreadLocalData {
5868d75effSDimitry Andric   void *InMemoryBuffer = nullptr;
5968d75effSDimitry Andric   size_t BufferSize = 0;
6068d75effSDimitry Andric   size_t BufferOffset = 0;
6168d75effSDimitry Andric   void *ShadowStack = nullptr;
6268d75effSDimitry Andric   size_t StackSize = 0;
6368d75effSDimitry Andric   size_t StackEntries = 0;
6468d75effSDimitry Andric   __xray::LogWriter *LogWriter = nullptr;
6568d75effSDimitry Andric };
6668d75effSDimitry Andric 
6768d75effSDimitry Andric struct BasicLoggingOptions {
6868d75effSDimitry Andric   int DurationFilterMicros = 0;
6968d75effSDimitry Andric   size_t MaxStackDepth = 0;
7068d75effSDimitry Andric   size_t ThreadBufferSize = 0;
7168d75effSDimitry Andric };
7268d75effSDimitry Andric } // namespace
7368d75effSDimitry Andric 
7468d75effSDimitry Andric static pthread_key_t PThreadKey;
7568d75effSDimitry Andric 
7668d75effSDimitry Andric static atomic_uint8_t BasicInitialized{0};
7768d75effSDimitry Andric 
7868d75effSDimitry Andric struct BasicLoggingOptions GlobalOptions;
7968d75effSDimitry Andric 
8068d75effSDimitry Andric thread_local atomic_uint8_t Guard{0};
8168d75effSDimitry Andric 
8268d75effSDimitry Andric static atomic_uint8_t UseRealTSC{0};
8368d75effSDimitry Andric static atomic_uint64_t ThresholdTicks{0};
8468d75effSDimitry Andric static atomic_uint64_t TicksPerSec{0};
8568d75effSDimitry Andric static atomic_uint64_t CycleFrequency{NanosecondsPerSecond};
8668d75effSDimitry Andric 
getLog()8768d75effSDimitry Andric static LogWriter *getLog() XRAY_NEVER_INSTRUMENT {
8868d75effSDimitry Andric   LogWriter* LW = LogWriter::Open();
8968d75effSDimitry Andric   if (LW == nullptr)
9068d75effSDimitry Andric     return LW;
9168d75effSDimitry Andric 
9268d75effSDimitry Andric   static pthread_once_t DetectOnce = PTHREAD_ONCE_INIT;
9368d75effSDimitry Andric   pthread_once(&DetectOnce, +[] {
9468d75effSDimitry Andric     if (atomic_load(&UseRealTSC, memory_order_acquire))
9568d75effSDimitry Andric       atomic_store(&CycleFrequency, getTSCFrequency(), memory_order_release);
9668d75effSDimitry Andric   });
9768d75effSDimitry Andric 
9868d75effSDimitry Andric   // Since we're here, we get to write the header. We set it up so that the
9968d75effSDimitry Andric   // header will only be written once, at the start, and let the threads
10068d75effSDimitry Andric   // logging do writes which just append.
10168d75effSDimitry Andric   XRayFileHeader Header;
10268d75effSDimitry Andric   // Version 2 includes tail exit records.
10368d75effSDimitry Andric   // Version 3 includes pid inside records.
10468d75effSDimitry Andric   Header.Version = 3;
10568d75effSDimitry Andric   Header.Type = FileTypes::NAIVE_LOG;
10668d75effSDimitry Andric   Header.CycleFrequency = atomic_load(&CycleFrequency, memory_order_acquire);
10768d75effSDimitry Andric 
10868d75effSDimitry Andric   // FIXME: Actually check whether we have 'constant_tsc' and 'nonstop_tsc'
10968d75effSDimitry Andric   // before setting the values in the header.
11068d75effSDimitry Andric   Header.ConstantTSC = 1;
11168d75effSDimitry Andric   Header.NonstopTSC = 1;
11268d75effSDimitry Andric   LW->WriteAll(reinterpret_cast<char *>(&Header),
11368d75effSDimitry Andric                reinterpret_cast<char *>(&Header) + sizeof(Header));
11468d75effSDimitry Andric   return LW;
11568d75effSDimitry Andric }
11668d75effSDimitry Andric 
getGlobalLog()11768d75effSDimitry Andric static LogWriter *getGlobalLog() XRAY_NEVER_INSTRUMENT {
11868d75effSDimitry Andric   static pthread_once_t OnceInit = PTHREAD_ONCE_INIT;
11968d75effSDimitry Andric   static LogWriter *LW = nullptr;
12068d75effSDimitry Andric   pthread_once(&OnceInit, +[] { LW = getLog(); });
12168d75effSDimitry Andric   return LW;
12268d75effSDimitry Andric }
12368d75effSDimitry Andric 
getThreadLocalData()12468d75effSDimitry Andric static ThreadLocalData &getThreadLocalData() XRAY_NEVER_INSTRUMENT {
12568d75effSDimitry Andric   thread_local ThreadLocalData TLD;
12668d75effSDimitry Andric   thread_local bool UNUSED TOnce = [] {
12768d75effSDimitry Andric     if (GlobalOptions.ThreadBufferSize == 0) {
12868d75effSDimitry Andric       if (Verbosity())
12968d75effSDimitry Andric         Report("Not initializing TLD since ThreadBufferSize == 0.\n");
13068d75effSDimitry Andric       return false;
13168d75effSDimitry Andric     }
13268d75effSDimitry Andric     pthread_setspecific(PThreadKey, &TLD);
13368d75effSDimitry Andric     TLD.LogWriter = getGlobalLog();
13468d75effSDimitry Andric     TLD.InMemoryBuffer = reinterpret_cast<XRayRecord *>(
13568d75effSDimitry Andric         InternalAlloc(sizeof(XRayRecord) * GlobalOptions.ThreadBufferSize,
13668d75effSDimitry Andric                       nullptr, alignof(XRayRecord)));
13768d75effSDimitry Andric     TLD.BufferSize = GlobalOptions.ThreadBufferSize;
13868d75effSDimitry Andric     TLD.BufferOffset = 0;
13968d75effSDimitry Andric     if (GlobalOptions.MaxStackDepth == 0) {
14068d75effSDimitry Andric       if (Verbosity())
14168d75effSDimitry Andric         Report("Not initializing the ShadowStack since MaxStackDepth == 0.\n");
14268d75effSDimitry Andric       TLD.StackSize = 0;
14368d75effSDimitry Andric       TLD.StackEntries = 0;
14468d75effSDimitry Andric       TLD.ShadowStack = nullptr;
14568d75effSDimitry Andric       return false;
14668d75effSDimitry Andric     }
14768d75effSDimitry Andric     TLD.ShadowStack = reinterpret_cast<StackEntry *>(
14868d75effSDimitry Andric         InternalAlloc(sizeof(StackEntry) * GlobalOptions.MaxStackDepth, nullptr,
14968d75effSDimitry Andric                       alignof(StackEntry)));
15068d75effSDimitry Andric     TLD.StackSize = GlobalOptions.MaxStackDepth;
15168d75effSDimitry Andric     TLD.StackEntries = 0;
15268d75effSDimitry Andric     return false;
15368d75effSDimitry Andric   }();
15468d75effSDimitry Andric   return TLD;
15568d75effSDimitry Andric }
15668d75effSDimitry Andric 
15768d75effSDimitry Andric template <class RDTSC>
InMemoryRawLog(int32_t FuncId,XRayEntryType Type,RDTSC ReadTSC)15868d75effSDimitry Andric void InMemoryRawLog(int32_t FuncId, XRayEntryType Type,
15968d75effSDimitry Andric                     RDTSC ReadTSC) XRAY_NEVER_INSTRUMENT {
16068d75effSDimitry Andric   auto &TLD = getThreadLocalData();
16168d75effSDimitry Andric   LogWriter *LW = getGlobalLog();
16268d75effSDimitry Andric   if (LW == nullptr)
16368d75effSDimitry Andric     return;
16468d75effSDimitry Andric 
16568d75effSDimitry Andric   // Use a simple recursion guard, to handle cases where we're already logging
16668d75effSDimitry Andric   // and for one reason or another, this function gets called again in the same
16768d75effSDimitry Andric   // thread.
16868d75effSDimitry Andric   RecursionGuard G(Guard);
16968d75effSDimitry Andric   if (!G)
17068d75effSDimitry Andric     return;
17168d75effSDimitry Andric 
17268d75effSDimitry Andric   uint8_t CPU = 0;
17368d75effSDimitry Andric   uint64_t TSC = ReadTSC(CPU);
17468d75effSDimitry Andric 
17568d75effSDimitry Andric   switch (Type) {
17668d75effSDimitry Andric   case XRayEntryType::ENTRY:
17768d75effSDimitry Andric   case XRayEntryType::LOG_ARGS_ENTRY: {
17868d75effSDimitry Andric     // Short circuit if we've reached the maximum depth of the stack.
17968d75effSDimitry Andric     if (TLD.StackEntries++ >= TLD.StackSize)
18068d75effSDimitry Andric       return;
18168d75effSDimitry Andric 
18268d75effSDimitry Andric     // When we encounter an entry event, we keep track of the TSC and the CPU,
18368d75effSDimitry Andric     // and put it in the stack.
18468d75effSDimitry Andric     StackEntry E;
18568d75effSDimitry Andric     E.FuncId = FuncId;
18668d75effSDimitry Andric     E.CPU = CPU;
18768d75effSDimitry Andric     E.Type = Type;
18868d75effSDimitry Andric     E.TSC = TSC;
18968d75effSDimitry Andric     auto StackEntryPtr = static_cast<char *>(TLD.ShadowStack) +
19068d75effSDimitry Andric                          (sizeof(StackEntry) * (TLD.StackEntries - 1));
19168d75effSDimitry Andric     internal_memcpy(StackEntryPtr, &E, sizeof(StackEntry));
19268d75effSDimitry Andric     break;
19368d75effSDimitry Andric   }
19468d75effSDimitry Andric   case XRayEntryType::EXIT:
19568d75effSDimitry Andric   case XRayEntryType::TAIL: {
19668d75effSDimitry Andric     if (TLD.StackEntries == 0)
19768d75effSDimitry Andric       break;
19868d75effSDimitry Andric 
19968d75effSDimitry Andric     if (--TLD.StackEntries >= TLD.StackSize)
20068d75effSDimitry Andric       return;
20168d75effSDimitry Andric 
20268d75effSDimitry Andric     // When we encounter an exit event, we check whether all the following are
20368d75effSDimitry Andric     // true:
20468d75effSDimitry Andric     //
20568d75effSDimitry Andric     // - The Function ID is the same as the most recent entry in the stack.
20668d75effSDimitry Andric     // - The CPU is the same as the most recent entry in the stack.
20768d75effSDimitry Andric     // - The Delta of the TSCs is less than the threshold amount of time we're
20868d75effSDimitry Andric     //   looking to record.
20968d75effSDimitry Andric     //
21068d75effSDimitry Andric     // If all of these conditions are true, we pop the stack and don't write a
21168d75effSDimitry Andric     // record and move the record offset back.
21268d75effSDimitry Andric     StackEntry StackTop;
21368d75effSDimitry Andric     auto StackEntryPtr = static_cast<char *>(TLD.ShadowStack) +
21468d75effSDimitry Andric                          (sizeof(StackEntry) * TLD.StackEntries);
21568d75effSDimitry Andric     internal_memcpy(&StackTop, StackEntryPtr, sizeof(StackEntry));
21668d75effSDimitry Andric     if (StackTop.FuncId == FuncId && StackTop.CPU == CPU &&
21768d75effSDimitry Andric         StackTop.TSC < TSC) {
21868d75effSDimitry Andric       auto Delta = TSC - StackTop.TSC;
21968d75effSDimitry Andric       if (Delta < atomic_load(&ThresholdTicks, memory_order_relaxed)) {
22068d75effSDimitry Andric         DCHECK(TLD.BufferOffset > 0);
22168d75effSDimitry Andric         TLD.BufferOffset -= StackTop.Type == XRayEntryType::ENTRY ? 1 : 2;
22268d75effSDimitry Andric         return;
22368d75effSDimitry Andric       }
22468d75effSDimitry Andric     }
22568d75effSDimitry Andric     break;
22668d75effSDimitry Andric   }
22768d75effSDimitry Andric   default:
22868d75effSDimitry Andric     // Should be unreachable.
22968d75effSDimitry Andric     DCHECK(false && "Unsupported XRayEntryType encountered.");
23068d75effSDimitry Andric     break;
23168d75effSDimitry Andric   }
23268d75effSDimitry Andric 
23368d75effSDimitry Andric   // First determine whether the delta between the function's enter record and
23468d75effSDimitry Andric   // the exit record is higher than the threshold.
23568d75effSDimitry Andric   XRayRecord R;
23668d75effSDimitry Andric   R.RecordType = RecordTypes::NORMAL;
23768d75effSDimitry Andric   R.CPU = CPU;
23868d75effSDimitry Andric   R.TSC = TSC;
23968d75effSDimitry Andric   R.TId = GetTid();
24068d75effSDimitry Andric   R.PId = internal_getpid();
24168d75effSDimitry Andric   R.Type = Type;
24268d75effSDimitry Andric   R.FuncId = FuncId;
24368d75effSDimitry Andric   auto FirstEntry = reinterpret_cast<XRayRecord *>(TLD.InMemoryBuffer);
24468d75effSDimitry Andric   internal_memcpy(FirstEntry + TLD.BufferOffset, &R, sizeof(R));
24568d75effSDimitry Andric   if (++TLD.BufferOffset == TLD.BufferSize) {
24668d75effSDimitry Andric     SpinMutexLock Lock(&LogMutex);
24768d75effSDimitry Andric     LW->WriteAll(reinterpret_cast<char *>(FirstEntry),
24868d75effSDimitry Andric                  reinterpret_cast<char *>(FirstEntry + TLD.BufferOffset));
24968d75effSDimitry Andric     TLD.BufferOffset = 0;
25068d75effSDimitry Andric     TLD.StackEntries = 0;
25168d75effSDimitry Andric   }
25268d75effSDimitry Andric }
25368d75effSDimitry Andric 
25468d75effSDimitry Andric template <class RDTSC>
InMemoryRawLogWithArg(int32_t FuncId,XRayEntryType Type,uint64_t Arg1,RDTSC ReadTSC)25568d75effSDimitry Andric void InMemoryRawLogWithArg(int32_t FuncId, XRayEntryType Type, uint64_t Arg1,
25668d75effSDimitry Andric                            RDTSC ReadTSC) XRAY_NEVER_INSTRUMENT {
25768d75effSDimitry Andric   auto &TLD = getThreadLocalData();
25868d75effSDimitry Andric   auto FirstEntry =
25968d75effSDimitry Andric       reinterpret_cast<XRayArgPayload *>(TLD.InMemoryBuffer);
26068d75effSDimitry Andric   const auto &BuffLen = TLD.BufferSize;
26168d75effSDimitry Andric   LogWriter *LW = getGlobalLog();
26268d75effSDimitry Andric   if (LW == nullptr)
26368d75effSDimitry Andric     return;
26468d75effSDimitry Andric 
26568d75effSDimitry Andric   // First we check whether there's enough space to write the data consecutively
26668d75effSDimitry Andric   // in the thread-local buffer. If not, we first flush the buffer before
26768d75effSDimitry Andric   // attempting to write the two records that must be consecutive.
26868d75effSDimitry Andric   if (TLD.BufferOffset + 2 > BuffLen) {
26968d75effSDimitry Andric     SpinMutexLock Lock(&LogMutex);
27068d75effSDimitry Andric     LW->WriteAll(reinterpret_cast<char *>(FirstEntry),
27168d75effSDimitry Andric                  reinterpret_cast<char *>(FirstEntry + TLD.BufferOffset));
27268d75effSDimitry Andric     TLD.BufferOffset = 0;
27368d75effSDimitry Andric     TLD.StackEntries = 0;
27468d75effSDimitry Andric   }
27568d75effSDimitry Andric 
27668d75effSDimitry Andric   // Then we write the "we have an argument" record.
27768d75effSDimitry Andric   InMemoryRawLog(FuncId, Type, ReadTSC);
27868d75effSDimitry Andric 
27968d75effSDimitry Andric   RecursionGuard G(Guard);
28068d75effSDimitry Andric   if (!G)
28168d75effSDimitry Andric     return;
28268d75effSDimitry Andric 
28368d75effSDimitry Andric   // And, from here on write the arg payload.
28468d75effSDimitry Andric   XRayArgPayload R;
28568d75effSDimitry Andric   R.RecordType = RecordTypes::ARG_PAYLOAD;
28668d75effSDimitry Andric   R.FuncId = FuncId;
28768d75effSDimitry Andric   R.TId = GetTid();
28868d75effSDimitry Andric   R.PId = internal_getpid();
28968d75effSDimitry Andric   R.Arg = Arg1;
29068d75effSDimitry Andric   internal_memcpy(FirstEntry + TLD.BufferOffset, &R, sizeof(R));
29168d75effSDimitry Andric   if (++TLD.BufferOffset == BuffLen) {
29268d75effSDimitry Andric     SpinMutexLock Lock(&LogMutex);
29368d75effSDimitry Andric     LW->WriteAll(reinterpret_cast<char *>(FirstEntry),
29468d75effSDimitry Andric                  reinterpret_cast<char *>(FirstEntry + TLD.BufferOffset));
29568d75effSDimitry Andric     TLD.BufferOffset = 0;
29668d75effSDimitry Andric     TLD.StackEntries = 0;
29768d75effSDimitry Andric   }
29868d75effSDimitry Andric }
29968d75effSDimitry Andric 
basicLoggingHandleArg0RealTSC(int32_t FuncId,XRayEntryType Type)30068d75effSDimitry Andric void basicLoggingHandleArg0RealTSC(int32_t FuncId,
30168d75effSDimitry Andric                                    XRayEntryType Type) XRAY_NEVER_INSTRUMENT {
30268d75effSDimitry Andric   InMemoryRawLog(FuncId, Type, readTSC);
30368d75effSDimitry Andric }
30468d75effSDimitry Andric 
basicLoggingHandleArg0EmulateTSC(int32_t FuncId,XRayEntryType Type)30568d75effSDimitry Andric void basicLoggingHandleArg0EmulateTSC(int32_t FuncId, XRayEntryType Type)
30668d75effSDimitry Andric     XRAY_NEVER_INSTRUMENT {
30768d75effSDimitry Andric   InMemoryRawLog(FuncId, Type, [](uint8_t &CPU) XRAY_NEVER_INSTRUMENT {
30868d75effSDimitry Andric     timespec TS;
30968d75effSDimitry Andric     int result = clock_gettime(CLOCK_REALTIME, &TS);
31068d75effSDimitry Andric     if (result != 0) {
31168d75effSDimitry Andric       Report("clock_gettimg(2) return %d, errno=%d.", result, int(errno));
31268d75effSDimitry Andric       TS = {0, 0};
31368d75effSDimitry Andric     }
31468d75effSDimitry Andric     CPU = 0;
31568d75effSDimitry Andric     return TS.tv_sec * NanosecondsPerSecond + TS.tv_nsec;
31668d75effSDimitry Andric   });
31768d75effSDimitry Andric }
31868d75effSDimitry Andric 
basicLoggingHandleArg1RealTSC(int32_t FuncId,XRayEntryType Type,uint64_t Arg1)31968d75effSDimitry Andric void basicLoggingHandleArg1RealTSC(int32_t FuncId, XRayEntryType Type,
32068d75effSDimitry Andric                                    uint64_t Arg1) XRAY_NEVER_INSTRUMENT {
32168d75effSDimitry Andric   InMemoryRawLogWithArg(FuncId, Type, Arg1, readTSC);
32268d75effSDimitry Andric }
32368d75effSDimitry Andric 
basicLoggingHandleArg1EmulateTSC(int32_t FuncId,XRayEntryType Type,uint64_t Arg1)32468d75effSDimitry Andric void basicLoggingHandleArg1EmulateTSC(int32_t FuncId, XRayEntryType Type,
32568d75effSDimitry Andric                                       uint64_t Arg1) XRAY_NEVER_INSTRUMENT {
32668d75effSDimitry Andric   InMemoryRawLogWithArg(
32768d75effSDimitry Andric       FuncId, Type, Arg1, [](uint8_t &CPU) XRAY_NEVER_INSTRUMENT {
32868d75effSDimitry Andric         timespec TS;
32968d75effSDimitry Andric         int result = clock_gettime(CLOCK_REALTIME, &TS);
33068d75effSDimitry Andric         if (result != 0) {
33168d75effSDimitry Andric           Report("clock_gettimg(2) return %d, errno=%d.", result, int(errno));
33268d75effSDimitry Andric           TS = {0, 0};
33368d75effSDimitry Andric         }
33468d75effSDimitry Andric         CPU = 0;
33568d75effSDimitry Andric         return TS.tv_sec * NanosecondsPerSecond + TS.tv_nsec;
33668d75effSDimitry Andric       });
33768d75effSDimitry Andric }
33868d75effSDimitry Andric 
TLDDestructor(void * P)33968d75effSDimitry Andric static void TLDDestructor(void *P) XRAY_NEVER_INSTRUMENT {
34068d75effSDimitry Andric   ThreadLocalData &TLD = *reinterpret_cast<ThreadLocalData *>(P);
34168d75effSDimitry Andric   auto ExitGuard = at_scope_exit([&TLD] {
34268d75effSDimitry Andric     // Clean up dynamic resources.
34368d75effSDimitry Andric     if (TLD.InMemoryBuffer)
34468d75effSDimitry Andric       InternalFree(TLD.InMemoryBuffer);
34568d75effSDimitry Andric     if (TLD.ShadowStack)
34668d75effSDimitry Andric       InternalFree(TLD.ShadowStack);
34768d75effSDimitry Andric     if (Verbosity())
3480eae32dcSDimitry Andric       Report("Cleaned up log for TID: %llu\n", GetTid());
34968d75effSDimitry Andric   });
35068d75effSDimitry Andric 
35168d75effSDimitry Andric   if (TLD.LogWriter == nullptr || TLD.BufferOffset == 0) {
35268d75effSDimitry Andric     if (Verbosity())
3530eae32dcSDimitry Andric       Report("Skipping buffer for TID: %llu; Offset = %zu\n", GetTid(),
35468d75effSDimitry Andric              TLD.BufferOffset);
35568d75effSDimitry Andric     return;
35668d75effSDimitry Andric   }
35768d75effSDimitry Andric 
35868d75effSDimitry Andric   {
35968d75effSDimitry Andric     SpinMutexLock L(&LogMutex);
36068d75effSDimitry Andric     TLD.LogWriter->WriteAll(reinterpret_cast<char *>(TLD.InMemoryBuffer),
36168d75effSDimitry Andric                             reinterpret_cast<char *>(TLD.InMemoryBuffer) +
36268d75effSDimitry Andric                             (sizeof(XRayRecord) * TLD.BufferOffset));
36368d75effSDimitry Andric   }
36468d75effSDimitry Andric 
36568d75effSDimitry Andric   // Because this thread's exit could be the last one trying to write to
36668d75effSDimitry Andric   // the file and that we're not able to close out the file properly, we
36768d75effSDimitry Andric   // sync instead and hope that the pending writes are flushed as the
36868d75effSDimitry Andric   // thread exits.
36968d75effSDimitry Andric   TLD.LogWriter->Flush();
37068d75effSDimitry Andric }
37168d75effSDimitry Andric 
basicLoggingInit(UNUSED size_t BufferSize,UNUSED size_t BufferMax,void * Options,size_t OptionsSize)37268d75effSDimitry Andric XRayLogInitStatus basicLoggingInit(UNUSED size_t BufferSize,
37368d75effSDimitry Andric                                    UNUSED size_t BufferMax, void *Options,
37468d75effSDimitry Andric                                    size_t OptionsSize) XRAY_NEVER_INSTRUMENT {
37568d75effSDimitry Andric   uint8_t Expected = 0;
37668d75effSDimitry Andric   if (!atomic_compare_exchange_strong(&BasicInitialized, &Expected, 1,
37768d75effSDimitry Andric                                       memory_order_acq_rel)) {
37868d75effSDimitry Andric     if (Verbosity())
37968d75effSDimitry Andric       Report("Basic logging already initialized.\n");
38068d75effSDimitry Andric     return XRayLogInitStatus::XRAY_LOG_INITIALIZED;
38168d75effSDimitry Andric   }
38268d75effSDimitry Andric 
38368d75effSDimitry Andric   static pthread_once_t OnceInit = PTHREAD_ONCE_INIT;
38468d75effSDimitry Andric   pthread_once(&OnceInit, +[] {
38568d75effSDimitry Andric     pthread_key_create(&PThreadKey, TLDDestructor);
38668d75effSDimitry Andric     atomic_store(&UseRealTSC, probeRequiredCPUFeatures(), memory_order_release);
38768d75effSDimitry Andric     // Initialize the global TicksPerSec value.
38868d75effSDimitry Andric     atomic_store(&TicksPerSec,
38968d75effSDimitry Andric                  probeRequiredCPUFeatures() ? getTSCFrequency()
39068d75effSDimitry Andric                                             : NanosecondsPerSecond,
39168d75effSDimitry Andric                  memory_order_release);
39268d75effSDimitry Andric     if (!atomic_load(&UseRealTSC, memory_order_relaxed) && Verbosity())
39368d75effSDimitry Andric       Report("WARNING: Required CPU features missing for XRay instrumentation, "
39468d75effSDimitry Andric              "using emulation instead.\n");
39568d75effSDimitry Andric   });
39668d75effSDimitry Andric 
39768d75effSDimitry Andric   FlagParser P;
39868d75effSDimitry Andric   BasicFlags F;
39968d75effSDimitry Andric   F.setDefaults();
40068d75effSDimitry Andric   registerXRayBasicFlags(&P, &F);
40168d75effSDimitry Andric   P.ParseString(useCompilerDefinedBasicFlags());
40268d75effSDimitry Andric   auto *EnvOpts = GetEnv("XRAY_BASIC_OPTIONS");
40368d75effSDimitry Andric   if (EnvOpts == nullptr)
40468d75effSDimitry Andric     EnvOpts = "";
40568d75effSDimitry Andric 
40668d75effSDimitry Andric   P.ParseString(EnvOpts);
40768d75effSDimitry Andric 
40868d75effSDimitry Andric   // If XRAY_BASIC_OPTIONS was not defined, then we use the deprecated options
40968d75effSDimitry Andric   // set through XRAY_OPTIONS instead.
41068d75effSDimitry Andric   if (internal_strlen(EnvOpts) == 0) {
41168d75effSDimitry Andric     F.func_duration_threshold_us =
41268d75effSDimitry Andric         flags()->xray_naive_log_func_duration_threshold_us;
41368d75effSDimitry Andric     F.max_stack_depth = flags()->xray_naive_log_max_stack_depth;
41468d75effSDimitry Andric     F.thread_buffer_size = flags()->xray_naive_log_thread_buffer_size;
41568d75effSDimitry Andric   }
41668d75effSDimitry Andric 
41768d75effSDimitry Andric   P.ParseString(static_cast<const char *>(Options));
41868d75effSDimitry Andric   GlobalOptions.ThreadBufferSize = F.thread_buffer_size;
41968d75effSDimitry Andric   GlobalOptions.DurationFilterMicros = F.func_duration_threshold_us;
42068d75effSDimitry Andric   GlobalOptions.MaxStackDepth = F.max_stack_depth;
42168d75effSDimitry Andric   *basicFlags() = F;
42268d75effSDimitry Andric 
42368d75effSDimitry Andric   atomic_store(&ThresholdTicks,
42468d75effSDimitry Andric                atomic_load(&TicksPerSec, memory_order_acquire) *
42568d75effSDimitry Andric                    GlobalOptions.DurationFilterMicros / 1000000,
42668d75effSDimitry Andric                memory_order_release);
42768d75effSDimitry Andric   __xray_set_handler_arg1(atomic_load(&UseRealTSC, memory_order_acquire)
42868d75effSDimitry Andric                               ? basicLoggingHandleArg1RealTSC
42968d75effSDimitry Andric                               : basicLoggingHandleArg1EmulateTSC);
43068d75effSDimitry Andric   __xray_set_handler(atomic_load(&UseRealTSC, memory_order_acquire)
43168d75effSDimitry Andric                          ? basicLoggingHandleArg0RealTSC
43268d75effSDimitry Andric                          : basicLoggingHandleArg0EmulateTSC);
43368d75effSDimitry Andric 
43468d75effSDimitry Andric   // TODO: Implement custom event and typed event handling support in Basic
43568d75effSDimitry Andric   // Mode.
43668d75effSDimitry Andric   __xray_remove_customevent_handler();
43768d75effSDimitry Andric   __xray_remove_typedevent_handler();
43868d75effSDimitry Andric 
43968d75effSDimitry Andric   return XRayLogInitStatus::XRAY_LOG_INITIALIZED;
44068d75effSDimitry Andric }
44168d75effSDimitry Andric 
basicLoggingFinalize()44268d75effSDimitry Andric XRayLogInitStatus basicLoggingFinalize() XRAY_NEVER_INSTRUMENT {
44368d75effSDimitry Andric   uint8_t Expected = 0;
44468d75effSDimitry Andric   if (!atomic_compare_exchange_strong(&BasicInitialized, &Expected, 0,
44568d75effSDimitry Andric                                       memory_order_acq_rel) &&
44668d75effSDimitry Andric       Verbosity())
44768d75effSDimitry Andric     Report("Basic logging already finalized.\n");
44868d75effSDimitry Andric 
44968d75effSDimitry Andric   // Nothing really to do aside from marking state of the global to be
45068d75effSDimitry Andric   // uninitialized.
45168d75effSDimitry Andric 
45268d75effSDimitry Andric   return XRayLogInitStatus::XRAY_LOG_FINALIZED;
45368d75effSDimitry Andric }
45468d75effSDimitry Andric 
basicLoggingFlush()45568d75effSDimitry Andric XRayLogFlushStatus basicLoggingFlush() XRAY_NEVER_INSTRUMENT {
45668d75effSDimitry Andric   // This really does nothing, since flushing the logs happen at the end of a
45768d75effSDimitry Andric   // thread's lifetime, or when the buffers are full.
45868d75effSDimitry Andric   return XRayLogFlushStatus::XRAY_LOG_FLUSHED;
45968d75effSDimitry Andric }
46068d75effSDimitry Andric 
46168d75effSDimitry Andric // This is a handler that, effectively, does nothing.
basicLoggingHandleArg0Empty(int32_t,XRayEntryType)46268d75effSDimitry Andric void basicLoggingHandleArg0Empty(int32_t, XRayEntryType) XRAY_NEVER_INSTRUMENT {
46368d75effSDimitry Andric }
46468d75effSDimitry Andric 
basicLogDynamicInitializer()46568d75effSDimitry Andric bool basicLogDynamicInitializer() XRAY_NEVER_INSTRUMENT {
46668d75effSDimitry Andric   XRayLogImpl Impl{
46768d75effSDimitry Andric       basicLoggingInit,
46868d75effSDimitry Andric       basicLoggingFinalize,
46968d75effSDimitry Andric       basicLoggingHandleArg0Empty,
47068d75effSDimitry Andric       basicLoggingFlush,
47168d75effSDimitry Andric   };
47268d75effSDimitry Andric   auto RegistrationResult = __xray_log_register_mode("xray-basic", Impl);
47368d75effSDimitry Andric   if (RegistrationResult != XRayLogRegisterStatus::XRAY_REGISTRATION_OK &&
47468d75effSDimitry Andric       Verbosity())
47568d75effSDimitry Andric     Report("Cannot register XRay Basic Mode to 'xray-basic'; error = %d\n",
47668d75effSDimitry Andric            RegistrationResult);
47768d75effSDimitry Andric   if (flags()->xray_naive_log ||
47868d75effSDimitry Andric       !internal_strcmp(flags()->xray_mode, "xray-basic")) {
47968d75effSDimitry Andric     auto SelectResult = __xray_log_select_mode("xray-basic");
48068d75effSDimitry Andric     if (SelectResult != XRayLogRegisterStatus::XRAY_REGISTRATION_OK) {
48168d75effSDimitry Andric       if (Verbosity())
48268d75effSDimitry Andric         Report("Failed selecting XRay Basic Mode; error = %d\n", SelectResult);
48368d75effSDimitry Andric       return false;
48468d75effSDimitry Andric     }
48568d75effSDimitry Andric 
48668d75effSDimitry Andric     // We initialize the implementation using the data we get from the
48768d75effSDimitry Andric     // XRAY_BASIC_OPTIONS environment variable, at this point of the
48868d75effSDimitry Andric     // implementation.
48968d75effSDimitry Andric     auto *Env = GetEnv("XRAY_BASIC_OPTIONS");
49068d75effSDimitry Andric     auto InitResult =
49168d75effSDimitry Andric         __xray_log_init_mode("xray-basic", Env == nullptr ? "" : Env);
49268d75effSDimitry Andric     if (InitResult != XRayLogInitStatus::XRAY_LOG_INITIALIZED) {
49368d75effSDimitry Andric       if (Verbosity())
49468d75effSDimitry Andric         Report("Failed initializing XRay Basic Mode; error = %d\n", InitResult);
49568d75effSDimitry Andric       return false;
49668d75effSDimitry Andric     }
49768d75effSDimitry Andric 
49868d75effSDimitry Andric     // At this point we know that we've successfully initialized Basic mode
49968d75effSDimitry Andric     // tracing, and the only chance we're going to get for the current thread to
50068d75effSDimitry Andric     // clean-up may be at thread/program exit. To ensure that we're going to get
50168d75effSDimitry Andric     // the cleanup even without calling the finalization routines, we're
50268d75effSDimitry Andric     // registering a program exit function that will do the cleanup.
50368d75effSDimitry Andric     static pthread_once_t DynamicOnce = PTHREAD_ONCE_INIT;
50468d75effSDimitry Andric     pthread_once(&DynamicOnce, +[] {
50568d75effSDimitry Andric       static void *FakeTLD = nullptr;
50668d75effSDimitry Andric       FakeTLD = &getThreadLocalData();
50768d75effSDimitry Andric       Atexit(+[] { TLDDestructor(FakeTLD); });
50868d75effSDimitry Andric     });
50968d75effSDimitry Andric   }
51068d75effSDimitry Andric   return true;
51168d75effSDimitry Andric }
51268d75effSDimitry Andric 
51368d75effSDimitry Andric } // namespace __xray
51468d75effSDimitry Andric 
51568d75effSDimitry Andric static auto UNUSED Unused = __xray::basicLogDynamicInitializer();
516