1 //===-- xray_interface.cpp --------------------------------------*- C++ -*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file is a part of XRay, a dynamic runtime instrumentation system.
10 //
11 // Implementation of the API functions.
12 //
13 //===----------------------------------------------------------------------===//
14
15 #include "xray_interface_internal.h"
16
17 #include <cinttypes>
18 #include <cstdio>
19 #include <errno.h>
20 #include <limits>
21 #include <string.h>
22 #include <sys/mman.h>
23
24 #if SANITIZER_FUCHSIA
25 #include <zircon/process.h>
26 #include <zircon/sanitizer.h>
27 #include <zircon/status.h>
28 #include <zircon/syscalls.h>
29 #endif
30
31 #include "sanitizer_common/sanitizer_addrhashmap.h"
32 #include "sanitizer_common/sanitizer_common.h"
33
34 #include "xray_defs.h"
35 #include "xray_flags.h"
36
37 extern __sanitizer::SpinMutex XRayInstrMapMutex;
38 extern __sanitizer::atomic_uint8_t XRayInitialized;
39 extern __xray::XRaySledMap XRayInstrMap;
40
41 namespace __xray {
42
43 #if defined(__x86_64__)
44 static const int16_t cSledLength = 12;
45 #elif defined(__aarch64__)
46 static const int16_t cSledLength = 32;
47 #elif defined(__arm__)
48 static const int16_t cSledLength = 28;
49 #elif SANITIZER_MIPS32
50 static const int16_t cSledLength = 48;
51 #elif SANITIZER_MIPS64
52 static const int16_t cSledLength = 64;
53 #elif defined(__powerpc64__)
54 static const int16_t cSledLength = 8;
55 #elif defined(__hexagon__)
56 static const int16_t cSledLength = 20;
57 #else
58 #error "Unsupported CPU Architecture"
59 #endif /* CPU architecture */
60
61 // This is the function to call when we encounter the entry or exit sleds.
62 atomic_uintptr_t XRayPatchedFunction{0};
63
64 // This is the function to call from the arg1-enabled sleds/trampolines.
65 atomic_uintptr_t XRayArgLogger{0};
66
67 // This is the function to call when we encounter a custom event log call.
68 atomic_uintptr_t XRayPatchedCustomEvent{0};
69
70 // This is the function to call when we encounter a typed event log call.
71 atomic_uintptr_t XRayPatchedTypedEvent{0};
72
73 // This is the global status to determine whether we are currently
74 // patching/unpatching.
75 atomic_uint8_t XRayPatching{0};
76
77 struct TypeDescription {
78 uint32_t type_id;
79 std::size_t description_string_length;
80 };
81
82 using TypeDescriptorMapType = AddrHashMap<TypeDescription, 11>;
83 // An address map from immutable descriptors to type ids.
84 TypeDescriptorMapType TypeDescriptorAddressMap{};
85
86 atomic_uint32_t TypeEventDescriptorCounter{0};
87
88 // MProtectHelper is an RAII wrapper for calls to mprotect(...) that will
89 // undo any successful mprotect(...) changes. This is used to make a page
90 // writeable and executable, and upon destruction if it was successful in
91 // doing so returns the page into a read-only and executable page.
92 //
93 // This is only used specifically for runtime-patching of the XRay
94 // instrumentation points. This assumes that the executable pages are
95 // originally read-and-execute only.
96 class MProtectHelper {
97 void *PageAlignedAddr;
98 std::size_t MProtectLen;
99 bool MustCleanup;
100
101 public:
MProtectHelper(void * PageAlignedAddr,std::size_t MProtectLen,std::size_t PageSize)102 explicit MProtectHelper(void *PageAlignedAddr,
103 std::size_t MProtectLen,
104 std::size_t PageSize) XRAY_NEVER_INSTRUMENT
105 : PageAlignedAddr(PageAlignedAddr),
106 MProtectLen(MProtectLen),
107 MustCleanup(false) {
108 #if SANITIZER_FUCHSIA
109 MProtectLen = RoundUpTo(MProtectLen, PageSize);
110 #endif
111 }
112
MakeWriteable()113 int MakeWriteable() XRAY_NEVER_INSTRUMENT {
114 #if SANITIZER_FUCHSIA
115 auto R = __sanitizer_change_code_protection(
116 reinterpret_cast<uintptr_t>(PageAlignedAddr), MProtectLen, true);
117 if (R != ZX_OK) {
118 Report("XRay: cannot change code protection: %s\n",
119 _zx_status_get_string(R));
120 return -1;
121 }
122 MustCleanup = true;
123 return 0;
124 #else
125 auto R = mprotect(PageAlignedAddr, MProtectLen,
126 PROT_READ | PROT_WRITE | PROT_EXEC);
127 if (R != -1)
128 MustCleanup = true;
129 return R;
130 #endif
131 }
132
~MProtectHelper()133 ~MProtectHelper() XRAY_NEVER_INSTRUMENT {
134 if (MustCleanup) {
135 #if SANITIZER_FUCHSIA
136 auto R = __sanitizer_change_code_protection(
137 reinterpret_cast<uintptr_t>(PageAlignedAddr), MProtectLen, false);
138 if (R != ZX_OK) {
139 Report("XRay: cannot change code protection: %s\n",
140 _zx_status_get_string(R));
141 }
142 #else
143 mprotect(PageAlignedAddr, MProtectLen, PROT_READ | PROT_EXEC);
144 #endif
145 }
146 }
147 };
148
149 namespace {
150
patchSled(const XRaySledEntry & Sled,bool Enable,int32_t FuncId)151 bool patchSled(const XRaySledEntry &Sled, bool Enable,
152 int32_t FuncId) XRAY_NEVER_INSTRUMENT {
153 bool Success = false;
154 switch (Sled.Kind) {
155 case XRayEntryType::ENTRY:
156 Success = patchFunctionEntry(Enable, FuncId, Sled, __xray_FunctionEntry);
157 break;
158 case XRayEntryType::EXIT:
159 Success = patchFunctionExit(Enable, FuncId, Sled);
160 break;
161 case XRayEntryType::TAIL:
162 Success = patchFunctionTailExit(Enable, FuncId, Sled);
163 break;
164 case XRayEntryType::LOG_ARGS_ENTRY:
165 Success = patchFunctionEntry(Enable, FuncId, Sled, __xray_ArgLoggerEntry);
166 break;
167 case XRayEntryType::CUSTOM_EVENT:
168 Success = patchCustomEvent(Enable, FuncId, Sled);
169 break;
170 case XRayEntryType::TYPED_EVENT:
171 Success = patchTypedEvent(Enable, FuncId, Sled);
172 break;
173 default:
174 Report("Unsupported sled kind '%" PRIu64 "' @%04x\n", Sled.Address,
175 int(Sled.Kind));
176 return false;
177 }
178 return Success;
179 }
180
181 const XRayFunctionSledIndex
findFunctionSleds(int32_t FuncId,const XRaySledMap & InstrMap)182 findFunctionSleds(int32_t FuncId,
183 const XRaySledMap &InstrMap) XRAY_NEVER_INSTRUMENT {
184 int32_t CurFn = 0;
185 uint64_t LastFnAddr = 0;
186 XRayFunctionSledIndex Index = {nullptr, nullptr};
187
188 for (std::size_t I = 0; I < InstrMap.Entries && CurFn <= FuncId; I++) {
189 const auto &Sled = InstrMap.Sleds[I];
190 const auto Function = Sled.function();
191 if (Function != LastFnAddr) {
192 CurFn++;
193 LastFnAddr = Function;
194 }
195
196 if (CurFn == FuncId) {
197 if (Index.Begin == nullptr)
198 Index.Begin = &Sled;
199 Index.End = &Sled;
200 }
201 }
202
203 Index.End += 1;
204
205 return Index;
206 }
207
patchFunction(int32_t FuncId,bool Enable)208 XRayPatchingStatus patchFunction(int32_t FuncId,
209 bool Enable) XRAY_NEVER_INSTRUMENT {
210 if (!atomic_load(&XRayInitialized,
211 memory_order_acquire))
212 return XRayPatchingStatus::NOT_INITIALIZED; // Not initialized.
213
214 uint8_t NotPatching = false;
215 if (!atomic_compare_exchange_strong(
216 &XRayPatching, &NotPatching, true, memory_order_acq_rel))
217 return XRayPatchingStatus::ONGOING; // Already patching.
218
219 // Next, we look for the function index.
220 XRaySledMap InstrMap;
221 {
222 SpinMutexLock Guard(&XRayInstrMapMutex);
223 InstrMap = XRayInstrMap;
224 }
225
226 // If we don't have an index, we can't patch individual functions.
227 if (InstrMap.Functions == 0)
228 return XRayPatchingStatus::NOT_INITIALIZED;
229
230 // FuncId must be a positive number, less than the number of functions
231 // instrumented.
232 if (FuncId <= 0 || static_cast<size_t>(FuncId) > InstrMap.Functions) {
233 Report("Invalid function id provided: %d\n", FuncId);
234 return XRayPatchingStatus::FAILED;
235 }
236
237 // Now we patch ths sleds for this specific function.
238 auto SledRange = InstrMap.SledsIndex ? InstrMap.SledsIndex[FuncId - 1]
239 : findFunctionSleds(FuncId, InstrMap);
240 auto *f = SledRange.Begin;
241 auto *e = SledRange.End;
242 bool SucceedOnce = false;
243 while (f != e)
244 SucceedOnce |= patchSled(*f++, Enable, FuncId);
245
246 atomic_store(&XRayPatching, false,
247 memory_order_release);
248
249 if (!SucceedOnce) {
250 Report("Failed patching any sled for function '%d'.", FuncId);
251 return XRayPatchingStatus::FAILED;
252 }
253
254 return XRayPatchingStatus::SUCCESS;
255 }
256
257 // controlPatching implements the common internals of the patching/unpatching
258 // implementation. |Enable| defines whether we're enabling or disabling the
259 // runtime XRay instrumentation.
controlPatching(bool Enable)260 XRayPatchingStatus controlPatching(bool Enable) XRAY_NEVER_INSTRUMENT {
261 if (!atomic_load(&XRayInitialized,
262 memory_order_acquire))
263 return XRayPatchingStatus::NOT_INITIALIZED; // Not initialized.
264
265 uint8_t NotPatching = false;
266 if (!atomic_compare_exchange_strong(
267 &XRayPatching, &NotPatching, true, memory_order_acq_rel))
268 return XRayPatchingStatus::ONGOING; // Already patching.
269
270 uint8_t PatchingSuccess = false;
271 auto XRayPatchingStatusResetter =
272 at_scope_exit([&PatchingSuccess] {
273 if (!PatchingSuccess)
274 atomic_store(&XRayPatching, false,
275 memory_order_release);
276 });
277
278 XRaySledMap InstrMap;
279 {
280 SpinMutexLock Guard(&XRayInstrMapMutex);
281 InstrMap = XRayInstrMap;
282 }
283 if (InstrMap.Entries == 0)
284 return XRayPatchingStatus::NOT_INITIALIZED;
285
286 uint32_t FuncId = 1;
287 uint64_t CurFun = 0;
288
289 // First we want to find the bounds for which we have instrumentation points,
290 // and try to get as few calls to mprotect(...) as possible. We're assuming
291 // that all the sleds for the instrumentation map are contiguous as a single
292 // set of pages. When we do support dynamic shared object instrumentation,
293 // we'll need to do this for each set of page load offsets per DSO loaded. For
294 // now we're assuming we can mprotect the whole section of text between the
295 // minimum sled address and the maximum sled address (+ the largest sled
296 // size).
297 auto *MinSled = &InstrMap.Sleds[0];
298 auto *MaxSled = &InstrMap.Sleds[InstrMap.Entries - 1];
299 for (std::size_t I = 0; I < InstrMap.Entries; I++) {
300 const auto &Sled = InstrMap.Sleds[I];
301 if (Sled.address() < MinSled->address())
302 MinSled = &Sled;
303 if (Sled.address() > MaxSled->address())
304 MaxSled = &Sled;
305 }
306
307 const size_t PageSize = flags()->xray_page_size_override > 0
308 ? flags()->xray_page_size_override
309 : GetPageSizeCached();
310 if ((PageSize == 0) || ((PageSize & (PageSize - 1)) != 0)) {
311 Report("System page size is not a power of two: %zu\n", PageSize);
312 return XRayPatchingStatus::FAILED;
313 }
314
315 void *PageAlignedAddr =
316 reinterpret_cast<void *>(MinSled->address() & ~(PageSize - 1));
317 size_t MProtectLen =
318 (MaxSled->address() - reinterpret_cast<uptr>(PageAlignedAddr)) +
319 cSledLength;
320 MProtectHelper Protector(PageAlignedAddr, MProtectLen, PageSize);
321 if (Protector.MakeWriteable() == -1) {
322 Report("Failed mprotect: %d\n", errno);
323 return XRayPatchingStatus::FAILED;
324 }
325
326 for (std::size_t I = 0; I < InstrMap.Entries; ++I) {
327 auto &Sled = InstrMap.Sleds[I];
328 auto F = Sled.function();
329 if (CurFun == 0)
330 CurFun = F;
331 if (F != CurFun) {
332 ++FuncId;
333 CurFun = F;
334 }
335 patchSled(Sled, Enable, FuncId);
336 }
337 atomic_store(&XRayPatching, false,
338 memory_order_release);
339 PatchingSuccess = true;
340 return XRayPatchingStatus::SUCCESS;
341 }
342
mprotectAndPatchFunction(int32_t FuncId,bool Enable)343 XRayPatchingStatus mprotectAndPatchFunction(int32_t FuncId,
344 bool Enable) XRAY_NEVER_INSTRUMENT {
345 XRaySledMap InstrMap;
346 {
347 SpinMutexLock Guard(&XRayInstrMapMutex);
348 InstrMap = XRayInstrMap;
349 }
350
351 // FuncId must be a positive number, less than the number of functions
352 // instrumented.
353 if (FuncId <= 0 || static_cast<size_t>(FuncId) > InstrMap.Functions) {
354 Report("Invalid function id provided: %d\n", FuncId);
355 return XRayPatchingStatus::FAILED;
356 }
357
358 const size_t PageSize = flags()->xray_page_size_override > 0
359 ? flags()->xray_page_size_override
360 : GetPageSizeCached();
361 if ((PageSize == 0) || ((PageSize & (PageSize - 1)) != 0)) {
362 Report("Provided page size is not a power of two: %zu\n", PageSize);
363 return XRayPatchingStatus::FAILED;
364 }
365
366 // Here we compute the minimum sled and maximum sled associated with a
367 // particular function ID.
368 auto SledRange = InstrMap.SledsIndex ? InstrMap.SledsIndex[FuncId - 1]
369 : findFunctionSleds(FuncId, InstrMap);
370 auto *f = SledRange.Begin;
371 auto *e = SledRange.End;
372 auto *MinSled = f;
373 auto *MaxSled = (SledRange.End - 1);
374 while (f != e) {
375 if (f->address() < MinSled->address())
376 MinSled = f;
377 if (f->address() > MaxSled->address())
378 MaxSled = f;
379 ++f;
380 }
381
382 void *PageAlignedAddr =
383 reinterpret_cast<void *>(MinSled->address() & ~(PageSize - 1));
384 size_t MProtectLen =
385 (MaxSled->address() - reinterpret_cast<uptr>(PageAlignedAddr)) +
386 cSledLength;
387 MProtectHelper Protector(PageAlignedAddr, MProtectLen, PageSize);
388 if (Protector.MakeWriteable() == -1) {
389 Report("Failed mprotect: %d\n", errno);
390 return XRayPatchingStatus::FAILED;
391 }
392 return patchFunction(FuncId, Enable);
393 }
394
395 } // namespace
396
397 } // namespace __xray
398
399 using namespace __xray;
400
401 // The following functions are declared `extern "C" {...}` in the header, hence
402 // they're defined in the global namespace.
403
__xray_set_handler(void (* entry)(int32_t,XRayEntryType))404 int __xray_set_handler(void (*entry)(int32_t,
405 XRayEntryType)) XRAY_NEVER_INSTRUMENT {
406 if (atomic_load(&XRayInitialized,
407 memory_order_acquire)) {
408
409 atomic_store(&__xray::XRayPatchedFunction,
410 reinterpret_cast<uintptr_t>(entry),
411 memory_order_release);
412 return 1;
413 }
414 return 0;
415 }
416
__xray_set_customevent_handler(void (* entry)(void *,size_t))417 int __xray_set_customevent_handler(void (*entry)(void *, size_t))
418 XRAY_NEVER_INSTRUMENT {
419 if (atomic_load(&XRayInitialized,
420 memory_order_acquire)) {
421 atomic_store(&__xray::XRayPatchedCustomEvent,
422 reinterpret_cast<uintptr_t>(entry),
423 memory_order_release);
424 return 1;
425 }
426 return 0;
427 }
428
__xray_set_typedevent_handler(void (* entry)(uint16_t,const void *,size_t))429 int __xray_set_typedevent_handler(void (*entry)(
430 uint16_t, const void *, size_t)) XRAY_NEVER_INSTRUMENT {
431 if (atomic_load(&XRayInitialized,
432 memory_order_acquire)) {
433 atomic_store(&__xray::XRayPatchedTypedEvent,
434 reinterpret_cast<uintptr_t>(entry),
435 memory_order_release);
436 return 1;
437 }
438 return 0;
439 }
440
__xray_remove_handler()441 int __xray_remove_handler() XRAY_NEVER_INSTRUMENT {
442 return __xray_set_handler(nullptr);
443 }
444
__xray_remove_customevent_handler()445 int __xray_remove_customevent_handler() XRAY_NEVER_INSTRUMENT {
446 return __xray_set_customevent_handler(nullptr);
447 }
448
__xray_remove_typedevent_handler()449 int __xray_remove_typedevent_handler() XRAY_NEVER_INSTRUMENT {
450 return __xray_set_typedevent_handler(nullptr);
451 }
452
__xray_register_event_type(const char * const event_type)453 uint16_t __xray_register_event_type(
454 const char *const event_type) XRAY_NEVER_INSTRUMENT {
455 TypeDescriptorMapType::Handle h(&TypeDescriptorAddressMap, (uptr)event_type);
456 if (h.created()) {
457 h->type_id = atomic_fetch_add(
458 &TypeEventDescriptorCounter, 1, memory_order_acq_rel);
459 h->description_string_length = strnlen(event_type, 1024);
460 }
461 return h->type_id;
462 }
463
__xray_patch()464 XRayPatchingStatus __xray_patch() XRAY_NEVER_INSTRUMENT {
465 return controlPatching(true);
466 }
467
__xray_unpatch()468 XRayPatchingStatus __xray_unpatch() XRAY_NEVER_INSTRUMENT {
469 return controlPatching(false);
470 }
471
__xray_patch_function(int32_t FuncId)472 XRayPatchingStatus __xray_patch_function(int32_t FuncId) XRAY_NEVER_INSTRUMENT {
473 return mprotectAndPatchFunction(FuncId, true);
474 }
475
476 XRayPatchingStatus
__xray_unpatch_function(int32_t FuncId)477 __xray_unpatch_function(int32_t FuncId) XRAY_NEVER_INSTRUMENT {
478 return mprotectAndPatchFunction(FuncId, false);
479 }
480
__xray_set_handler_arg1(void (* entry)(int32_t,XRayEntryType,uint64_t))481 int __xray_set_handler_arg1(void (*entry)(int32_t, XRayEntryType, uint64_t)) {
482 if (!atomic_load(&XRayInitialized,
483 memory_order_acquire))
484 return 0;
485
486 // A relaxed write might not be visible even if the current thread gets
487 // scheduled on a different CPU/NUMA node. We need to wait for everyone to
488 // have this handler installed for consistency of collected data across CPUs.
489 atomic_store(&XRayArgLogger, reinterpret_cast<uint64_t>(entry),
490 memory_order_release);
491 return 1;
492 }
493
__xray_remove_handler_arg1()494 int __xray_remove_handler_arg1() { return __xray_set_handler_arg1(nullptr); }
495
__xray_function_address(int32_t FuncId)496 uintptr_t __xray_function_address(int32_t FuncId) XRAY_NEVER_INSTRUMENT {
497 XRaySledMap InstrMap;
498 {
499 SpinMutexLock Guard(&XRayInstrMapMutex);
500 InstrMap = XRayInstrMap;
501 }
502
503 if (FuncId <= 0 || static_cast<size_t>(FuncId) > InstrMap.Functions)
504 return 0;
505 const XRaySledEntry *Sled = InstrMap.SledsIndex
506 ? InstrMap.SledsIndex[FuncId - 1].Begin
507 : findFunctionSleds(FuncId, InstrMap).Begin;
508 return Sled->function()
509 // On PPC, function entries are always aligned to 16 bytes. The beginning of a
510 // sled might be a local entry, which is always +8 based on the global entry.
511 // Always return the global entry.
512 #ifdef __PPC__
513 & ~0xf
514 #endif
515 ;
516 }
517
__xray_max_function_id()518 size_t __xray_max_function_id() XRAY_NEVER_INSTRUMENT {
519 SpinMutexLock Guard(&XRayInstrMapMutex);
520 return XRayInstrMap.Functions;
521 }
522