xref: /openbsd-src/gnu/llvm/compiler-rt/lib/memprof/memprof_rtl.cpp (revision 810390e339a5425391477d5d41c78d7cab2424ac)
1d89ec533Spatrick //===-- memprof_rtl.cpp --------------------------------------------------===//
2d89ec533Spatrick //
3d89ec533Spatrick // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4d89ec533Spatrick // See https://llvm.org/LICENSE.txt for license information.
5d89ec533Spatrick // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6d89ec533Spatrick //
7d89ec533Spatrick //===----------------------------------------------------------------------===//
8d89ec533Spatrick //
9d89ec533Spatrick // This file is a part of MemProfiler, a memory profiler.
10d89ec533Spatrick //
11d89ec533Spatrick // Main file of the MemProf run-time library.
12d89ec533Spatrick //===----------------------------------------------------------------------===//
13d89ec533Spatrick 
14d89ec533Spatrick #include "memprof_allocator.h"
15d89ec533Spatrick #include "memprof_interceptors.h"
16d89ec533Spatrick #include "memprof_interface_internal.h"
17d89ec533Spatrick #include "memprof_internal.h"
18d89ec533Spatrick #include "memprof_mapping.h"
19d89ec533Spatrick #include "memprof_stack.h"
20d89ec533Spatrick #include "memprof_stats.h"
21d89ec533Spatrick #include "memprof_thread.h"
22d89ec533Spatrick #include "sanitizer_common/sanitizer_atomic.h"
23d89ec533Spatrick #include "sanitizer_common/sanitizer_flags.h"
24*810390e3Srobert #include "sanitizer_common/sanitizer_interface_internal.h"
25d89ec533Spatrick #include "sanitizer_common/sanitizer_libc.h"
26d89ec533Spatrick #include "sanitizer_common/sanitizer_symbolizer.h"
27d89ec533Spatrick 
28d89ec533Spatrick #include <time.h>
29d89ec533Spatrick 
30d89ec533Spatrick uptr __memprof_shadow_memory_dynamic_address; // Global interface symbol.
31d89ec533Spatrick 
32d89ec533Spatrick // Allow the user to specify a profile output file via the binary.
33d89ec533Spatrick SANITIZER_WEAK_ATTRIBUTE char __memprof_profile_filename[1];
34d89ec533Spatrick 
35d89ec533Spatrick namespace __memprof {
36d89ec533Spatrick 
MemprofDie()37d89ec533Spatrick static void MemprofDie() {
38d89ec533Spatrick   static atomic_uint32_t num_calls;
39d89ec533Spatrick   if (atomic_fetch_add(&num_calls, 1, memory_order_relaxed) != 0) {
40d89ec533Spatrick     // Don't die twice - run a busy loop.
41d89ec533Spatrick     while (1) {
42*810390e3Srobert       internal_sched_yield();
43d89ec533Spatrick     }
44d89ec533Spatrick   }
45d89ec533Spatrick   if (common_flags()->print_module_map >= 1)
46d89ec533Spatrick     DumpProcessMap();
47d89ec533Spatrick   if (flags()->unmap_shadow_on_exit) {
48d89ec533Spatrick     if (kHighShadowEnd)
49d89ec533Spatrick       UnmapOrDie((void *)kLowShadowBeg, kHighShadowEnd - kLowShadowBeg);
50d89ec533Spatrick   }
51d89ec533Spatrick }
52d89ec533Spatrick 
MemprofOnDeadlySignal(int signo,void * siginfo,void * context)53*810390e3Srobert static void MemprofOnDeadlySignal(int signo, void *siginfo, void *context) {
54*810390e3Srobert   // We call StartReportDeadlySignal not HandleDeadlySignal so we get the
55*810390e3Srobert   // deadly signal message to stderr but no writing to the profile output file
56*810390e3Srobert   StartReportDeadlySignal();
57*810390e3Srobert   __memprof_profile_dump();
58*810390e3Srobert   Die();
59*810390e3Srobert }
60*810390e3Srobert 
CheckUnwind()61d89ec533Spatrick static void CheckUnwind() {
62d89ec533Spatrick   GET_STACK_TRACE(kStackTraceMax, common_flags()->fast_unwind_on_check);
63d89ec533Spatrick   stack.Print();
64d89ec533Spatrick }
65d89ec533Spatrick 
66d89ec533Spatrick // -------------------------- Globals --------------------- {{{1
67d89ec533Spatrick int memprof_inited;
68d89ec533Spatrick int memprof_init_done;
69d89ec533Spatrick bool memprof_init_is_running;
70d89ec533Spatrick int memprof_timestamp_inited;
71d89ec533Spatrick long memprof_init_timestamp_s;
72d89ec533Spatrick 
73d89ec533Spatrick uptr kHighMemEnd;
74d89ec533Spatrick 
75d89ec533Spatrick // -------------------------- Run-time entry ------------------- {{{1
76d89ec533Spatrick // exported functions
77d89ec533Spatrick 
78d89ec533Spatrick #define MEMPROF_MEMORY_ACCESS_CALLBACK_BODY() __memprof::RecordAccess(addr);
79d89ec533Spatrick 
80d89ec533Spatrick #define MEMPROF_MEMORY_ACCESS_CALLBACK(type)                                   \
81d89ec533Spatrick   extern "C" NOINLINE INTERFACE_ATTRIBUTE void __memprof_##type(uptr addr) {   \
82d89ec533Spatrick     MEMPROF_MEMORY_ACCESS_CALLBACK_BODY()                                      \
83d89ec533Spatrick   }
84d89ec533Spatrick 
85d89ec533Spatrick MEMPROF_MEMORY_ACCESS_CALLBACK(load)
MEMPROF_MEMORY_ACCESS_CALLBACK(store)86d89ec533Spatrick MEMPROF_MEMORY_ACCESS_CALLBACK(store)
87d89ec533Spatrick 
88d89ec533Spatrick // Force the linker to keep the symbols for various MemProf interface
89d89ec533Spatrick // functions. We want to keep those in the executable in order to let the
90d89ec533Spatrick // instrumented dynamic libraries access the symbol even if it is not used by
91d89ec533Spatrick // the executable itself. This should help if the build system is removing dead
92d89ec533Spatrick // code at link time.
93d89ec533Spatrick static NOINLINE void force_interface_symbols() {
94d89ec533Spatrick   volatile int fake_condition = 0; // prevent dead condition elimination.
95d89ec533Spatrick   // clang-format off
96d89ec533Spatrick   switch (fake_condition) {
97d89ec533Spatrick     case 1: __memprof_record_access(nullptr); break;
98d89ec533Spatrick     case 2: __memprof_record_access_range(nullptr, 0); break;
99d89ec533Spatrick   }
100d89ec533Spatrick   // clang-format on
101d89ec533Spatrick }
102d89ec533Spatrick 
memprof_atexit()103d89ec533Spatrick static void memprof_atexit() {
104d89ec533Spatrick   Printf("MemProfiler exit stats:\n");
105d89ec533Spatrick   __memprof_print_accumulated_stats();
106d89ec533Spatrick }
107d89ec533Spatrick 
InitializeHighMemEnd()108d89ec533Spatrick static void InitializeHighMemEnd() {
109d89ec533Spatrick   kHighMemEnd = GetMaxUserVirtualAddress();
110d89ec533Spatrick   // Increase kHighMemEnd to make sure it's properly
111d89ec533Spatrick   // aligned together with kHighMemBeg:
112d89ec533Spatrick   kHighMemEnd |= (GetMmapGranularity() << SHADOW_SCALE) - 1;
113d89ec533Spatrick }
114d89ec533Spatrick 
PrintAddressSpaceLayout()115d89ec533Spatrick void PrintAddressSpaceLayout() {
116d89ec533Spatrick   if (kHighMemBeg) {
117d89ec533Spatrick     Printf("|| `[%p, %p]` || HighMem    ||\n", (void *)kHighMemBeg,
118d89ec533Spatrick            (void *)kHighMemEnd);
119d89ec533Spatrick     Printf("|| `[%p, %p]` || HighShadow ||\n", (void *)kHighShadowBeg,
120d89ec533Spatrick            (void *)kHighShadowEnd);
121d89ec533Spatrick   }
122d89ec533Spatrick   Printf("|| `[%p, %p]` || ShadowGap  ||\n", (void *)kShadowGapBeg,
123d89ec533Spatrick          (void *)kShadowGapEnd);
124d89ec533Spatrick   if (kLowShadowBeg) {
125d89ec533Spatrick     Printf("|| `[%p, %p]` || LowShadow  ||\n", (void *)kLowShadowBeg,
126d89ec533Spatrick            (void *)kLowShadowEnd);
127d89ec533Spatrick     Printf("|| `[%p, %p]` || LowMem     ||\n", (void *)kLowMemBeg,
128d89ec533Spatrick            (void *)kLowMemEnd);
129d89ec533Spatrick   }
130d89ec533Spatrick   Printf("MemToShadow(shadow): %p %p", (void *)MEM_TO_SHADOW(kLowShadowBeg),
131d89ec533Spatrick          (void *)MEM_TO_SHADOW(kLowShadowEnd));
132d89ec533Spatrick   if (kHighMemBeg) {
133d89ec533Spatrick     Printf(" %p %p", (void *)MEM_TO_SHADOW(kHighShadowBeg),
134d89ec533Spatrick            (void *)MEM_TO_SHADOW(kHighShadowEnd));
135d89ec533Spatrick   }
136d89ec533Spatrick   Printf("\n");
137d89ec533Spatrick   Printf("malloc_context_size=%zu\n",
138d89ec533Spatrick          (uptr)common_flags()->malloc_context_size);
139d89ec533Spatrick 
140d89ec533Spatrick   Printf("SHADOW_SCALE: %d\n", (int)SHADOW_SCALE);
141d89ec533Spatrick   Printf("SHADOW_GRANULARITY: %d\n", (int)SHADOW_GRANULARITY);
142d89ec533Spatrick   Printf("SHADOW_OFFSET: 0x%zx\n", (uptr)SHADOW_OFFSET);
143d89ec533Spatrick   CHECK(SHADOW_SCALE >= 3 && SHADOW_SCALE <= 7);
144d89ec533Spatrick }
145d89ec533Spatrick 
MemprofInitInternal()146d89ec533Spatrick static void MemprofInitInternal() {
147d89ec533Spatrick   if (LIKELY(memprof_inited))
148d89ec533Spatrick     return;
149d89ec533Spatrick   SanitizerToolName = "MemProfiler";
150d89ec533Spatrick   CHECK(!memprof_init_is_running && "MemProf init calls itself!");
151d89ec533Spatrick   memprof_init_is_running = true;
152d89ec533Spatrick 
153d89ec533Spatrick   CacheBinaryName();
154d89ec533Spatrick 
155d89ec533Spatrick   // Initialize flags. This must be done early, because most of the
156d89ec533Spatrick   // initialization steps look at flags().
157d89ec533Spatrick   InitializeFlags();
158d89ec533Spatrick 
159d89ec533Spatrick   AvoidCVE_2016_2143();
160d89ec533Spatrick 
161d89ec533Spatrick   SetMallocContextSize(common_flags()->malloc_context_size);
162d89ec533Spatrick 
163d89ec533Spatrick   InitializeHighMemEnd();
164d89ec533Spatrick 
165d89ec533Spatrick   // Make sure we are not statically linked.
166d89ec533Spatrick   MemprofDoesNotSupportStaticLinkage();
167d89ec533Spatrick 
168d89ec533Spatrick   // Install tool-specific callbacks in sanitizer_common.
169d89ec533Spatrick   AddDieCallback(MemprofDie);
170d89ec533Spatrick   SetCheckUnwindCallback(CheckUnwind);
171d89ec533Spatrick 
172d89ec533Spatrick   // Use profile name specified via the binary itself if it exists, and hasn't
173d89ec533Spatrick   // been overrriden by a flag at runtime.
174d89ec533Spatrick   if (__memprof_profile_filename[0] != 0 && !common_flags()->log_path)
175d89ec533Spatrick     __sanitizer_set_report_path(__memprof_profile_filename);
176d89ec533Spatrick   else
177d89ec533Spatrick     __sanitizer_set_report_path(common_flags()->log_path);
178d89ec533Spatrick 
179d89ec533Spatrick   __sanitizer::InitializePlatformEarly();
180d89ec533Spatrick 
181d89ec533Spatrick   // Setup internal allocator callback.
182d89ec533Spatrick   SetLowLevelAllocateMinAlignment(SHADOW_GRANULARITY);
183d89ec533Spatrick 
184d89ec533Spatrick   InitializeMemprofInterceptors();
185d89ec533Spatrick   CheckASLR();
186d89ec533Spatrick 
187d89ec533Spatrick   ReplaceSystemMalloc();
188d89ec533Spatrick 
189d89ec533Spatrick   DisableCoreDumperIfNecessary();
190d89ec533Spatrick 
191d89ec533Spatrick   InitializeShadowMemory();
192d89ec533Spatrick 
193d89ec533Spatrick   TSDInit(PlatformTSDDtor);
194*810390e3Srobert   InstallDeadlySignalHandlers(MemprofOnDeadlySignal);
195d89ec533Spatrick 
196d89ec533Spatrick   InitializeAllocator();
197d89ec533Spatrick 
198d89ec533Spatrick   // On Linux MemprofThread::ThreadStart() calls malloc() that's why
199d89ec533Spatrick   // memprof_inited should be set to 1 prior to initializing the threads.
200d89ec533Spatrick   memprof_inited = 1;
201d89ec533Spatrick   memprof_init_is_running = false;
202d89ec533Spatrick 
203d89ec533Spatrick   if (flags()->atexit)
204d89ec533Spatrick     Atexit(memprof_atexit);
205d89ec533Spatrick 
206d89ec533Spatrick   InitializeCoverage(common_flags()->coverage, common_flags()->coverage_dir);
207d89ec533Spatrick 
208d89ec533Spatrick   // interceptors
209d89ec533Spatrick   InitTlsSize();
210d89ec533Spatrick 
211d89ec533Spatrick   // Create main thread.
212d89ec533Spatrick   MemprofThread *main_thread = CreateMainThread();
213d89ec533Spatrick   CHECK_EQ(0, main_thread->tid());
214d89ec533Spatrick   force_interface_symbols(); // no-op.
215d89ec533Spatrick   SanitizerInitializeUnwinder();
216d89ec533Spatrick 
217d89ec533Spatrick   Symbolizer::LateInitialize();
218d89ec533Spatrick 
219d89ec533Spatrick   VReport(1, "MemProfiler Init done\n");
220d89ec533Spatrick 
221d89ec533Spatrick   memprof_init_done = 1;
222d89ec533Spatrick }
223d89ec533Spatrick 
MemprofInitTime()224d89ec533Spatrick void MemprofInitTime() {
225d89ec533Spatrick   if (LIKELY(memprof_timestamp_inited))
226d89ec533Spatrick     return;
227d89ec533Spatrick   timespec ts;
228d89ec533Spatrick   clock_gettime(CLOCK_REALTIME, &ts);
229d89ec533Spatrick   memprof_init_timestamp_s = ts.tv_sec;
230d89ec533Spatrick   memprof_timestamp_inited = 1;
231d89ec533Spatrick }
232d89ec533Spatrick 
233d89ec533Spatrick // Initialize as requested from some part of MemProf runtime library
234d89ec533Spatrick // (interceptors, allocator, etc).
MemprofInitFromRtl()235d89ec533Spatrick void MemprofInitFromRtl() { MemprofInitInternal(); }
236d89ec533Spatrick 
237d89ec533Spatrick #if MEMPROF_DYNAMIC
238d89ec533Spatrick // Initialize runtime in case it's LD_PRELOAD-ed into uninstrumented executable
239d89ec533Spatrick // (and thus normal initializers from .preinit_array or modules haven't run).
240d89ec533Spatrick 
241d89ec533Spatrick class MemprofInitializer {
242d89ec533Spatrick public:
MemprofInitializer()243d89ec533Spatrick   MemprofInitializer() { MemprofInitFromRtl(); }
244d89ec533Spatrick };
245d89ec533Spatrick 
246d89ec533Spatrick static MemprofInitializer memprof_initializer;
247d89ec533Spatrick #endif // MEMPROF_DYNAMIC
248d89ec533Spatrick 
249d89ec533Spatrick } // namespace __memprof
250d89ec533Spatrick 
251d89ec533Spatrick // ---------------------- Interface ---------------- {{{1
252d89ec533Spatrick using namespace __memprof;
253d89ec533Spatrick 
254d89ec533Spatrick // Initialize as requested from instrumented application code.
__memprof_init()255d89ec533Spatrick void __memprof_init() {
256d89ec533Spatrick   MemprofInitTime();
257d89ec533Spatrick   MemprofInitInternal();
258d89ec533Spatrick }
259d89ec533Spatrick 
__memprof_preinit()260d89ec533Spatrick void __memprof_preinit() { MemprofInitInternal(); }
261d89ec533Spatrick 
__memprof_version_mismatch_check_v1()262d89ec533Spatrick void __memprof_version_mismatch_check_v1() {}
263d89ec533Spatrick 
__memprof_record_access(void const volatile * addr)264d89ec533Spatrick void __memprof_record_access(void const volatile *addr) {
265d89ec533Spatrick   __memprof::RecordAccess((uptr)addr);
266d89ec533Spatrick }
267d89ec533Spatrick 
__memprof_record_access_range(void const volatile * addr,uptr size)268*810390e3Srobert void __memprof_record_access_range(void const volatile *addr, uptr size) {
269*810390e3Srobert   for (uptr a = (uptr)addr; a < (uptr)addr + size; a += kWordSize)
270*810390e3Srobert     __memprof::RecordAccess(a);
271d89ec533Spatrick }
272d89ec533Spatrick 
273d89ec533Spatrick extern "C" SANITIZER_INTERFACE_ATTRIBUTE u16
__sanitizer_unaligned_load16(const uu16 * p)274d89ec533Spatrick __sanitizer_unaligned_load16(const uu16 *p) {
275d89ec533Spatrick   __memprof_record_access(p);
276d89ec533Spatrick   return *p;
277d89ec533Spatrick }
278d89ec533Spatrick 
279d89ec533Spatrick extern "C" SANITIZER_INTERFACE_ATTRIBUTE u32
__sanitizer_unaligned_load32(const uu32 * p)280d89ec533Spatrick __sanitizer_unaligned_load32(const uu32 *p) {
281d89ec533Spatrick   __memprof_record_access(p);
282d89ec533Spatrick   return *p;
283d89ec533Spatrick }
284d89ec533Spatrick 
285d89ec533Spatrick extern "C" SANITIZER_INTERFACE_ATTRIBUTE u64
__sanitizer_unaligned_load64(const uu64 * p)286d89ec533Spatrick __sanitizer_unaligned_load64(const uu64 *p) {
287d89ec533Spatrick   __memprof_record_access(p);
288d89ec533Spatrick   return *p;
289d89ec533Spatrick }
290d89ec533Spatrick 
291d89ec533Spatrick extern "C" SANITIZER_INTERFACE_ATTRIBUTE void
__sanitizer_unaligned_store16(uu16 * p,u16 x)292d89ec533Spatrick __sanitizer_unaligned_store16(uu16 *p, u16 x) {
293d89ec533Spatrick   __memprof_record_access(p);
294d89ec533Spatrick   *p = x;
295d89ec533Spatrick }
296d89ec533Spatrick 
297d89ec533Spatrick extern "C" SANITIZER_INTERFACE_ATTRIBUTE void
__sanitizer_unaligned_store32(uu32 * p,u32 x)298d89ec533Spatrick __sanitizer_unaligned_store32(uu32 *p, u32 x) {
299d89ec533Spatrick   __memprof_record_access(p);
300d89ec533Spatrick   *p = x;
301d89ec533Spatrick }
302d89ec533Spatrick 
303d89ec533Spatrick extern "C" SANITIZER_INTERFACE_ATTRIBUTE void
__sanitizer_unaligned_store64(uu64 * p,u64 x)304d89ec533Spatrick __sanitizer_unaligned_store64(uu64 *p, u64 x) {
305d89ec533Spatrick   __memprof_record_access(p);
306d89ec533Spatrick   *p = x;
307d89ec533Spatrick }
308