1 //===-- tsan_rtl.h ----------------------------------------------*- C++ -*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file is a part of ThreadSanitizer (TSan), a race detector.
10 //
11 // Main internal TSan header file.
12 //
13 // Ground rules:
14 // - C++ run-time should not be used (static CTORs, RTTI, exceptions, static
15 // function-scope locals)
16 // - All functions/classes/etc reside in namespace __tsan, except for those
17 // declared in tsan_interface.h.
18 // - Platform-specific files should be used instead of ifdefs (*).
19 // - No system headers included in header files (*).
20 // - Platform specific headres included only into platform-specific files (*).
21 //
22 // (*) Except when inlining is critical for performance.
23 //===----------------------------------------------------------------------===//
24
25 #ifndef TSAN_RTL_H
26 #define TSAN_RTL_H
27
28 #include "sanitizer_common/sanitizer_allocator.h"
29 #include "sanitizer_common/sanitizer_allocator_internal.h"
30 #include "sanitizer_common/sanitizer_asm.h"
31 #include "sanitizer_common/sanitizer_common.h"
32 #include "sanitizer_common/sanitizer_deadlock_detector_interface.h"
33 #include "sanitizer_common/sanitizer_libignore.h"
34 #include "sanitizer_common/sanitizer_suppressions.h"
35 #include "sanitizer_common/sanitizer_thread_registry.h"
36 #include "sanitizer_common/sanitizer_vector.h"
37 #include "tsan_clock.h"
38 #include "tsan_defs.h"
39 #include "tsan_flags.h"
40 #include "tsan_ignoreset.h"
41 #include "tsan_mman.h"
42 #include "tsan_mutexset.h"
43 #include "tsan_platform.h"
44 #include "tsan_report.h"
45 #include "tsan_shadow.h"
46 #include "tsan_stack_trace.h"
47 #include "tsan_sync.h"
48 #include "tsan_trace.h"
49
50 #if SANITIZER_WORDSIZE != 64
51 # error "ThreadSanitizer is supported only on 64-bit platforms"
52 #endif
53
54 namespace __tsan {
55
56 #if !SANITIZER_GO
57 struct MapUnmapCallback;
58 #if defined(__mips64) || defined(__aarch64__) || defined(__powerpc__)
59
60 struct AP32 {
61 static const uptr kSpaceBeg = 0;
62 static const u64 kSpaceSize = SANITIZER_MMAP_RANGE_SIZE;
63 static const uptr kMetadataSize = 0;
64 typedef __sanitizer::CompactSizeClassMap SizeClassMap;
65 static const uptr kRegionSizeLog = 20;
66 using AddressSpaceView = LocalAddressSpaceView;
67 typedef __tsan::MapUnmapCallback MapUnmapCallback;
68 static const uptr kFlags = 0;
69 };
70 typedef SizeClassAllocator32<AP32> PrimaryAllocator;
71 #else
72 struct AP64 { // Allocator64 parameters. Deliberately using a short name.
73 # if defined(__s390x__)
74 typedef MappingS390x Mapping;
75 # else
76 typedef Mapping48AddressSpace Mapping;
77 # endif
78 static const uptr kSpaceBeg = Mapping::kHeapMemBeg;
79 static const uptr kSpaceSize = Mapping::kHeapMemEnd - Mapping::kHeapMemBeg;
80 static const uptr kMetadataSize = 0;
81 typedef DefaultSizeClassMap SizeClassMap;
82 typedef __tsan::MapUnmapCallback MapUnmapCallback;
83 static const uptr kFlags = 0;
84 using AddressSpaceView = LocalAddressSpaceView;
85 };
86 typedef SizeClassAllocator64<AP64> PrimaryAllocator;
87 #endif
88 typedef CombinedAllocator<PrimaryAllocator> Allocator;
89 typedef Allocator::AllocatorCache AllocatorCache;
90 Allocator *allocator();
91 #endif
92
93 struct ThreadSignalContext;
94
95 struct JmpBuf {
96 uptr sp;
97 int int_signal_send;
98 bool in_blocking_func;
99 uptr in_signal_handler;
100 uptr *shadow_stack_pos;
101 };
102
103 // A Processor represents a physical thread, or a P for Go.
104 // It is used to store internal resources like allocate cache, and does not
105 // participate in race-detection logic (invisible to end user).
106 // In C++ it is tied to an OS thread just like ThreadState, however ideally
107 // it should be tied to a CPU (this way we will have fewer allocator caches).
108 // In Go it is tied to a P, so there are significantly fewer Processor's than
109 // ThreadState's (which are tied to Gs).
110 // A ThreadState must be wired with a Processor to handle events.
111 struct Processor {
112 ThreadState *thr; // currently wired thread, or nullptr
113 #if !SANITIZER_GO
114 AllocatorCache alloc_cache;
115 InternalAllocatorCache internal_alloc_cache;
116 #endif
117 DenseSlabAllocCache block_cache;
118 DenseSlabAllocCache sync_cache;
119 DenseSlabAllocCache clock_cache;
120 DDPhysicalThread *dd_pt;
121 };
122
123 #if !SANITIZER_GO
124 // ScopedGlobalProcessor temporary setups a global processor for the current
125 // thread, if it does not have one. Intended for interceptors that can run
126 // at the very thread end, when we already destroyed the thread processor.
127 struct ScopedGlobalProcessor {
128 ScopedGlobalProcessor();
129 ~ScopedGlobalProcessor();
130 };
131 #endif
132
133 // This struct is stored in TLS.
134 struct ThreadState {
135 FastState fast_state ALIGNED(SANITIZER_CACHE_LINE_SIZE);
136 // Synch epoch represents the threads's epoch before the last synchronization
137 // action. It allows to reduce number of shadow state updates.
138 // For example, fast_synch_epoch=100, last write to addr X was at epoch=150,
139 // if we are processing write to X from the same thread at epoch=200,
140 // we do nothing, because both writes happen in the same 'synch epoch'.
141 // That is, if another memory access does not race with the former write,
142 // it does not race with the latter as well.
143 // QUESTION: can we can squeeze this into ThreadState::Fast?
144 // E.g. ThreadState::Fast is a 44-bit, 32 are taken by synch_epoch and 12 are
145 // taken by epoch between synchs.
146 // This way we can save one load from tls.
147 u64 fast_synch_epoch;
148 // Technically `current` should be a separate THREADLOCAL variable;
149 // but it is placed here in order to share cache line with previous fields.
150 ThreadState* current;
151 // This is a slow path flag. On fast path, fast_state.GetIgnoreBit() is read.
152 // We do not distinguish beteween ignoring reads and writes
153 // for better performance.
154 int ignore_reads_and_writes;
155 atomic_sint32_t pending_signals;
156 int ignore_sync;
157 int suppress_reports;
158 // Go does not support ignores.
159 #if !SANITIZER_GO
160 IgnoreSet mop_ignore_set;
161 IgnoreSet sync_ignore_set;
162 // C/C++ uses fixed size shadow stack.
163 uptr shadow_stack[kShadowStackSize];
164 #else
165 // Go uses malloc-allocated shadow stack with dynamic size.
166 uptr *shadow_stack;
167 #endif
168 uptr *shadow_stack_end;
169 uptr *shadow_stack_pos;
170 RawShadow *racy_shadow_addr;
171 RawShadow racy_state[2];
172 MutexSet mset;
173 ThreadClock clock;
174 #if !SANITIZER_GO
175 Vector<JmpBuf> jmp_bufs;
176 int ignore_interceptors;
177 #endif
178 const Tid tid;
179 const int unique_id;
180 bool in_symbolizer;
181 bool in_ignored_lib;
182 bool is_inited;
183 bool is_dead;
184 bool is_freeing;
185 bool is_vptr_access;
186 const uptr stk_addr;
187 const uptr stk_size;
188 const uptr tls_addr;
189 const uptr tls_size;
190 ThreadContext *tctx;
191
192 DDLogicalThread *dd_lt;
193
194 // Current wired Processor, or nullptr. Required to handle any events.
195 Processor *proc1;
196 #if !SANITIZER_GO
procThreadState197 Processor *proc() { return proc1; }
198 #else
199 Processor *proc();
200 #endif
201
202 atomic_uintptr_t in_signal_handler;
203 ThreadSignalContext *signal_ctx;
204
205 #if !SANITIZER_GO
206 StackID last_sleep_stack_id;
207 ThreadClock last_sleep_clock;
208 #endif
209
210 // Set in regions of runtime that must be signal-safe and fork-safe.
211 // If set, malloc must not be called.
212 int nomalloc;
213
214 const ReportDesc *current_report;
215
216 // Current position in tctx->trace.Back()->events (Event*).
217 atomic_uintptr_t trace_pos;
218 // PC of the last memory access, used to compute PC deltas in the trace.
219 uptr trace_prev_pc;
220 Sid sid;
221 Epoch epoch;
222
223 explicit ThreadState(Context *ctx, Tid tid, int unique_id, u64 epoch,
224 unsigned reuse_count, uptr stk_addr, uptr stk_size,
225 uptr tls_addr, uptr tls_size);
226 } ALIGNED(SANITIZER_CACHE_LINE_SIZE);
227
228 #if !SANITIZER_GO
229 #if SANITIZER_MAC || SANITIZER_ANDROID
230 ThreadState *cur_thread();
231 void set_cur_thread(ThreadState *thr);
232 void cur_thread_finalize();
cur_thread_init()233 inline ThreadState *cur_thread_init() { return cur_thread(); }
234 # else
235 __attribute__((tls_model("initial-exec")))
236 extern THREADLOCAL char cur_thread_placeholder[];
cur_thread()237 inline ThreadState *cur_thread() {
238 return reinterpret_cast<ThreadState *>((reinterpret_cast<uptr>(cur_thread_placeholder) + SANITIZER_CACHE_LINE_SIZE - 1) & ~static_cast<uptr>(SANITIZER_CACHE_LINE_SIZE - 1))->current;
239 }
cur_thread_init()240 inline ThreadState *cur_thread_init() {
241 ThreadState *thr = reinterpret_cast<ThreadState *>((reinterpret_cast<uptr>(cur_thread_placeholder) + SANITIZER_CACHE_LINE_SIZE - 1) & ~static_cast<uptr>(SANITIZER_CACHE_LINE_SIZE - 1));
242 if (UNLIKELY(!thr->current))
243 thr->current = thr;
244 return thr->current;
245 }
set_cur_thread(ThreadState * thr)246 inline void set_cur_thread(ThreadState *thr) {
247 reinterpret_cast<ThreadState *>((reinterpret_cast<uptr>(cur_thread_placeholder) + SANITIZER_CACHE_LINE_SIZE - 1) & ~static_cast<uptr>(SANITIZER_CACHE_LINE_SIZE - 1))->current = thr;
248 }
cur_thread_finalize()249 inline void cur_thread_finalize() { }
250 # endif // SANITIZER_MAC || SANITIZER_ANDROID
251 #endif // SANITIZER_GO
252
253 class ThreadContext final : public ThreadContextBase {
254 public:
255 explicit ThreadContext(Tid tid);
256 ~ThreadContext();
257 ThreadState *thr;
258 StackID creation_stack_id;
259 SyncClock sync;
260 // Epoch at which the thread had started.
261 // If we see an event from the thread stamped by an older epoch,
262 // the event is from a dead thread that shared tid with this thread.
263 u64 epoch0;
264 u64 epoch1;
265
266 v3::Trace trace;
267
268 // Override superclass callbacks.
269 void OnDead() override;
270 void OnJoined(void *arg) override;
271 void OnFinished() override;
272 void OnStarted(void *arg) override;
273 void OnCreated(void *arg) override;
274 void OnReset() override;
275 void OnDetached(void *arg) override;
276 };
277
278 struct RacyStacks {
279 MD5Hash hash[2];
280 bool operator==(const RacyStacks &other) const;
281 };
282
283 struct RacyAddress {
284 uptr addr_min;
285 uptr addr_max;
286 };
287
288 struct FiredSuppression {
289 ReportType type;
290 uptr pc_or_addr;
291 Suppression *supp;
292 };
293
294 struct Context {
295 Context();
296
297 bool initialized;
298 #if !SANITIZER_GO
299 bool after_multithreaded_fork;
300 #endif
301
302 MetaMap metamap;
303
304 Mutex report_mtx;
305 int nreported;
306 atomic_uint64_t last_symbolize_time_ns;
307
308 void *background_thread;
309 atomic_uint32_t stop_background_thread;
310
311 ThreadRegistry thread_registry;
312
313 Mutex racy_mtx;
314 Vector<RacyStacks> racy_stacks;
315 Vector<RacyAddress> racy_addresses;
316 // Number of fired suppressions may be large enough.
317 Mutex fired_suppressions_mtx;
318 InternalMmapVector<FiredSuppression> fired_suppressions;
319 DDetector *dd;
320
321 ClockAlloc clock_alloc;
322
323 Flags flags;
324 fd_t memprof_fd;
325
326 Mutex slot_mtx;
327 };
328
329 extern Context *ctx; // The one and the only global runtime context.
330
flags()331 ALWAYS_INLINE Flags *flags() {
332 return &ctx->flags;
333 }
334
335 struct ScopedIgnoreInterceptors {
ScopedIgnoreInterceptorsScopedIgnoreInterceptors336 ScopedIgnoreInterceptors() {
337 #if !SANITIZER_GO
338 cur_thread()->ignore_interceptors++;
339 #endif
340 }
341
~ScopedIgnoreInterceptorsScopedIgnoreInterceptors342 ~ScopedIgnoreInterceptors() {
343 #if !SANITIZER_GO
344 cur_thread()->ignore_interceptors--;
345 #endif
346 }
347 };
348
349 const char *GetObjectTypeFromTag(uptr tag);
350 const char *GetReportHeaderFromTag(uptr tag);
351 uptr TagFromShadowStackFrame(uptr pc);
352
353 class ScopedReportBase {
354 public:
355 void AddMemoryAccess(uptr addr, uptr external_tag, Shadow s, StackTrace stack,
356 const MutexSet *mset);
357 void AddStack(StackTrace stack, bool suppressable = false);
358 void AddThread(const ThreadContext *tctx, bool suppressable = false);
359 void AddThread(Tid unique_tid, bool suppressable = false);
360 void AddUniqueTid(Tid unique_tid);
361 void AddMutex(const SyncVar *s);
362 u64 AddMutex(u64 id);
363 void AddLocation(uptr addr, uptr size);
364 void AddSleep(StackID stack_id);
365 void SetCount(int count);
366
367 const ReportDesc *GetReport() const;
368
369 protected:
370 ScopedReportBase(ReportType typ, uptr tag);
371 ~ScopedReportBase();
372
373 private:
374 ReportDesc *rep_;
375 // Symbolizer makes lots of intercepted calls. If we try to process them,
376 // at best it will cause deadlocks on internal mutexes.
377 ScopedIgnoreInterceptors ignore_interceptors_;
378
379 void AddDeadMutex(u64 id);
380
381 ScopedReportBase(const ScopedReportBase &) = delete;
382 void operator=(const ScopedReportBase &) = delete;
383 };
384
385 class ScopedReport : public ScopedReportBase {
386 public:
387 explicit ScopedReport(ReportType typ, uptr tag = kExternalTagNone);
388 ~ScopedReport();
389
390 private:
391 ScopedErrorReportLock lock_;
392 };
393
394 bool ShouldReport(ThreadState *thr, ReportType typ);
395 ThreadContext *IsThreadStackOrTls(uptr addr, bool *is_stack);
396 void RestoreStack(Tid tid, const u64 epoch, VarSizeStackTrace *stk,
397 MutexSet *mset, uptr *tag = nullptr);
398
399 // The stack could look like:
400 // <start> | <main> | <foo> | tag | <bar>
401 // This will extract the tag and keep:
402 // <start> | <main> | <foo> | <bar>
403 template<typename StackTraceTy>
404 void ExtractTagFromStack(StackTraceTy *stack, uptr *tag = nullptr) {
405 if (stack->size < 2) return;
406 uptr possible_tag_pc = stack->trace[stack->size - 2];
407 uptr possible_tag = TagFromShadowStackFrame(possible_tag_pc);
408 if (possible_tag == kExternalTagNone) return;
409 stack->trace_buffer[stack->size - 2] = stack->trace_buffer[stack->size - 1];
410 stack->size -= 1;
411 if (tag) *tag = possible_tag;
412 }
413
414 template<typename StackTraceTy>
415 void ObtainCurrentStack(ThreadState *thr, uptr toppc, StackTraceTy *stack,
416 uptr *tag = nullptr) {
417 uptr size = thr->shadow_stack_pos - thr->shadow_stack;
418 uptr start = 0;
419 if (size + !!toppc > kStackTraceMax) {
420 start = size + !!toppc - kStackTraceMax;
421 size = kStackTraceMax - !!toppc;
422 }
423 stack->Init(&thr->shadow_stack[start], size, toppc);
424 ExtractTagFromStack(stack, tag);
425 }
426
427 #define GET_STACK_TRACE_FATAL(thr, pc) \
428 VarSizeStackTrace stack; \
429 ObtainCurrentStack(thr, pc, &stack); \
430 stack.ReverseOrder();
431
432 void MapShadow(uptr addr, uptr size);
433 void MapThreadTrace(uptr addr, uptr size, const char *name);
434 void DontNeedShadowFor(uptr addr, uptr size);
435 void UnmapShadow(ThreadState *thr, uptr addr, uptr size);
436 void InitializeShadowMemory();
437 void InitializeInterceptors();
438 void InitializeLibIgnore();
439 void InitializeDynamicAnnotations();
440
441 void ForkBefore(ThreadState *thr, uptr pc);
442 void ForkParentAfter(ThreadState *thr, uptr pc);
443 void ForkChildAfter(ThreadState *thr, uptr pc, bool start_thread);
444
445 void ReportRace(ThreadState *thr);
446 bool OutputReport(ThreadState *thr, const ScopedReport &srep);
447 bool IsFiredSuppression(Context *ctx, ReportType type, StackTrace trace);
448 bool IsExpectedReport(uptr addr, uptr size);
449
450 #if defined(TSAN_DEBUG_OUTPUT) && TSAN_DEBUG_OUTPUT >= 1
451 # define DPrintf Printf
452 #else
453 # define DPrintf(...)
454 #endif
455
456 #if defined(TSAN_DEBUG_OUTPUT) && TSAN_DEBUG_OUTPUT >= 2
457 # define DPrintf2 Printf
458 #else
459 # define DPrintf2(...)
460 #endif
461
462 StackID CurrentStackId(ThreadState *thr, uptr pc);
463 ReportStack *SymbolizeStackId(StackID stack_id);
464 void PrintCurrentStack(ThreadState *thr, uptr pc);
465 void PrintCurrentStackSlow(uptr pc); // uses libunwind
466 MBlock *JavaHeapBlock(uptr addr, uptr *start);
467
468 void Initialize(ThreadState *thr);
469 void MaybeSpawnBackgroundThread();
470 int Finalize(ThreadState *thr);
471
472 void OnUserAlloc(ThreadState *thr, uptr pc, uptr p, uptr sz, bool write);
473 void OnUserFree(ThreadState *thr, uptr pc, uptr p, bool write);
474
475 void MemoryAccess(ThreadState *thr, uptr pc, uptr addr,
476 int kAccessSizeLog, bool kAccessIsWrite, bool kIsAtomic);
477 void MemoryAccessImpl(ThreadState *thr, uptr addr,
478 int kAccessSizeLog, bool kAccessIsWrite, bool kIsAtomic,
479 u64 *shadow_mem, Shadow cur);
480 void MemoryAccessRange(ThreadState *thr, uptr pc, uptr addr,
481 uptr size, bool is_write);
482 void UnalignedMemoryAccess(ThreadState *thr, uptr pc, uptr addr, uptr size,
483 AccessType typ);
484
485 const int kSizeLog1 = 0;
486 const int kSizeLog2 = 1;
487 const int kSizeLog4 = 2;
488 const int kSizeLog8 = 3;
489
490 ALWAYS_INLINE
MemoryAccess(ThreadState * thr,uptr pc,uptr addr,uptr size,AccessType typ)491 void MemoryAccess(ThreadState *thr, uptr pc, uptr addr, uptr size,
492 AccessType typ) {
493 int size_log;
494 switch (size) {
495 case 1:
496 size_log = kSizeLog1;
497 break;
498 case 2:
499 size_log = kSizeLog2;
500 break;
501 case 4:
502 size_log = kSizeLog4;
503 break;
504 default:
505 DCHECK_EQ(size, 8);
506 size_log = kSizeLog8;
507 break;
508 }
509 bool is_write = !(typ & kAccessRead);
510 bool is_atomic = typ & kAccessAtomic;
511 if (typ & kAccessVptr)
512 thr->is_vptr_access = true;
513 if (typ & kAccessFree)
514 thr->is_freeing = true;
515 MemoryAccess(thr, pc, addr, size_log, is_write, is_atomic);
516 if (typ & kAccessVptr)
517 thr->is_vptr_access = false;
518 if (typ & kAccessFree)
519 thr->is_freeing = false;
520 }
521
522 void MemoryResetRange(ThreadState *thr, uptr pc, uptr addr, uptr size);
523 void MemoryRangeFreed(ThreadState *thr, uptr pc, uptr addr, uptr size);
524 void MemoryRangeImitateWrite(ThreadState *thr, uptr pc, uptr addr, uptr size);
525 void MemoryRangeImitateWriteOrResetRange(ThreadState *thr, uptr pc, uptr addr,
526 uptr size);
527
528 void ThreadIgnoreBegin(ThreadState *thr, uptr pc);
529 void ThreadIgnoreEnd(ThreadState *thr);
530 void ThreadIgnoreSyncBegin(ThreadState *thr, uptr pc);
531 void ThreadIgnoreSyncEnd(ThreadState *thr);
532
533 void FuncEntry(ThreadState *thr, uptr pc);
534 void FuncExit(ThreadState *thr);
535
536 Tid ThreadCreate(ThreadState *thr, uptr pc, uptr uid, bool detached);
537 void ThreadStart(ThreadState *thr, Tid tid, tid_t os_id,
538 ThreadType thread_type);
539 void ThreadFinish(ThreadState *thr);
540 Tid ThreadConsumeTid(ThreadState *thr, uptr pc, uptr uid);
541 void ThreadJoin(ThreadState *thr, uptr pc, Tid tid);
542 void ThreadDetach(ThreadState *thr, uptr pc, Tid tid);
543 void ThreadFinalize(ThreadState *thr);
544 void ThreadSetName(ThreadState *thr, const char *name);
545 int ThreadCount(ThreadState *thr);
546 void ProcessPendingSignalsImpl(ThreadState *thr);
547 void ThreadNotJoined(ThreadState *thr, uptr pc, Tid tid, uptr uid);
548
549 Processor *ProcCreate();
550 void ProcDestroy(Processor *proc);
551 void ProcWire(Processor *proc, ThreadState *thr);
552 void ProcUnwire(Processor *proc, ThreadState *thr);
553
554 // Note: the parameter is called flagz, because flags is already taken
555 // by the global function that returns flags.
556 void MutexCreate(ThreadState *thr, uptr pc, uptr addr, u32 flagz = 0);
557 void MutexDestroy(ThreadState *thr, uptr pc, uptr addr, u32 flagz = 0);
558 void MutexPreLock(ThreadState *thr, uptr pc, uptr addr, u32 flagz = 0);
559 void MutexPostLock(ThreadState *thr, uptr pc, uptr addr, u32 flagz = 0,
560 int rec = 1);
561 int MutexUnlock(ThreadState *thr, uptr pc, uptr addr, u32 flagz = 0);
562 void MutexPreReadLock(ThreadState *thr, uptr pc, uptr addr, u32 flagz = 0);
563 void MutexPostReadLock(ThreadState *thr, uptr pc, uptr addr, u32 flagz = 0);
564 void MutexReadUnlock(ThreadState *thr, uptr pc, uptr addr);
565 void MutexReadOrWriteUnlock(ThreadState *thr, uptr pc, uptr addr);
566 void MutexRepair(ThreadState *thr, uptr pc, uptr addr); // call on EOWNERDEAD
567 void MutexInvalidAccess(ThreadState *thr, uptr pc, uptr addr);
568
569 void Acquire(ThreadState *thr, uptr pc, uptr addr);
570 // AcquireGlobal synchronizes the current thread with all other threads.
571 // In terms of happens-before relation, it draws a HB edge from all threads
572 // (where they happen to execute right now) to the current thread. We use it to
573 // handle Go finalizers. Namely, finalizer goroutine executes AcquireGlobal
574 // right before executing finalizers. This provides a coarse, but simple
575 // approximation of the actual required synchronization.
576 void AcquireGlobal(ThreadState *thr);
577 void Release(ThreadState *thr, uptr pc, uptr addr);
578 void ReleaseStoreAcquire(ThreadState *thr, uptr pc, uptr addr);
579 void ReleaseStore(ThreadState *thr, uptr pc, uptr addr);
580 void AfterSleep(ThreadState *thr, uptr pc);
581 void AcquireImpl(ThreadState *thr, uptr pc, SyncClock *c);
582 void ReleaseImpl(ThreadState *thr, uptr pc, SyncClock *c);
583 void ReleaseStoreAcquireImpl(ThreadState *thr, uptr pc, SyncClock *c);
584 void ReleaseStoreImpl(ThreadState *thr, uptr pc, SyncClock *c);
585 void AcquireReleaseImpl(ThreadState *thr, uptr pc, SyncClock *c);
586
587 // The hacky call uses custom calling convention and an assembly thunk.
588 // It is considerably faster that a normal call for the caller
589 // if it is not executed (it is intended for slow paths from hot functions).
590 // The trick is that the call preserves all registers and the compiler
591 // does not treat it as a call.
592 // If it does not work for you, use normal call.
593 #if !SANITIZER_DEBUG && defined(__x86_64__) && !SANITIZER_MAC
594 // The caller may not create the stack frame for itself at all,
595 // so we create a reserve stack frame for it (1024b must be enough).
596 #define HACKY_CALL(f) \
597 __asm__ __volatile__("sub $1024, %%rsp;" \
598 CFI_INL_ADJUST_CFA_OFFSET(1024) \
599 ".hidden " #f "_thunk;" \
600 "call " #f "_thunk;" \
601 "add $1024, %%rsp;" \
602 CFI_INL_ADJUST_CFA_OFFSET(-1024) \
603 ::: "memory", "cc");
604 #else
605 #define HACKY_CALL(f) f()
606 #endif
607
608 void TraceSwitch(ThreadState *thr);
609 uptr TraceTopPC(ThreadState *thr);
610 uptr TraceSize();
611 uptr TraceParts();
612 Trace *ThreadTrace(Tid tid);
613
614 extern "C" void __tsan_trace_switch();
TraceAddEvent(ThreadState * thr,FastState fs,EventType typ,u64 addr)615 void ALWAYS_INLINE TraceAddEvent(ThreadState *thr, FastState fs,
616 EventType typ, u64 addr) {
617 if (!kCollectHistory)
618 return;
619 DCHECK_GE((int)typ, 0);
620 DCHECK_LE((int)typ, 7);
621 DCHECK_EQ(GetLsb(addr, kEventPCBits), addr);
622 u64 pos = fs.GetTracePos();
623 if (UNLIKELY((pos % kTracePartSize) == 0)) {
624 #if !SANITIZER_GO
625 HACKY_CALL(__tsan_trace_switch);
626 #else
627 TraceSwitch(thr);
628 #endif
629 }
630 Event *trace = (Event*)GetThreadTrace(fs.tid());
631 Event *evp = &trace[pos];
632 Event ev = (u64)addr | ((u64)typ << kEventPCBits);
633 *evp = ev;
634 }
635
636 #if !SANITIZER_GO
HeapEnd()637 uptr ALWAYS_INLINE HeapEnd() {
638 return HeapMemEnd() + PrimaryAllocator::AdditionalSize();
639 }
640 #endif
641
642 ThreadState *FiberCreate(ThreadState *thr, uptr pc, unsigned flags);
643 void FiberDestroy(ThreadState *thr, uptr pc, ThreadState *fiber);
644 void FiberSwitch(ThreadState *thr, uptr pc, ThreadState *fiber, unsigned flags);
645
646 // These need to match __tsan_switch_to_fiber_* flags defined in
647 // tsan_interface.h. See documentation there as well.
648 enum FiberSwitchFlags {
649 FiberSwitchFlagNoSync = 1 << 0, // __tsan_switch_to_fiber_no_sync
650 };
651
ProcessPendingSignals(ThreadState * thr)652 ALWAYS_INLINE void ProcessPendingSignals(ThreadState *thr) {
653 if (UNLIKELY(atomic_load_relaxed(&thr->pending_signals)))
654 ProcessPendingSignalsImpl(thr);
655 }
656
657 extern bool is_initialized;
658
659 ALWAYS_INLINE
LazyInitialize(ThreadState * thr)660 void LazyInitialize(ThreadState *thr) {
661 // If we can use .preinit_array, assume that __tsan_init
662 // called from .preinit_array initializes runtime before
663 // any instrumented code.
664 #if !SANITIZER_CAN_USE_PREINIT_ARRAY
665 if (UNLIKELY(!is_initialized))
666 Initialize(thr);
667 #endif
668 }
669
670 namespace v3 {
671
672 void TraceSwitchPart(ThreadState *thr);
673 bool RestoreStack(Tid tid, EventType type, Sid sid, Epoch epoch, uptr addr,
674 uptr size, AccessType typ, VarSizeStackTrace *pstk,
675 MutexSet *pmset, uptr *ptag);
676
677 template <typename EventT>
TraceAcquire(ThreadState * thr,EventT ** ev)678 ALWAYS_INLINE WARN_UNUSED_RESULT bool TraceAcquire(ThreadState *thr,
679 EventT **ev) {
680 Event *pos = reinterpret_cast<Event *>(atomic_load_relaxed(&thr->trace_pos));
681 #if SANITIZER_DEBUG
682 // TraceSwitch acquires these mutexes,
683 // so we lock them here to detect deadlocks more reliably.
684 { Lock lock(&ctx->slot_mtx); }
685 { Lock lock(&thr->tctx->trace.mtx); }
686 TracePart *current = thr->tctx->trace.parts.Back();
687 if (current) {
688 DCHECK_GE(pos, ¤t->events[0]);
689 DCHECK_LE(pos, ¤t->events[TracePart::kSize]);
690 } else {
691 DCHECK_EQ(pos, nullptr);
692 }
693 #endif
694 // TracePart is allocated with mmap and is at least 4K aligned.
695 // So the following check is a faster way to check for part end.
696 // It may have false positives in the middle of the trace,
697 // they are filtered out in TraceSwitch.
698 if (UNLIKELY(((uptr)(pos + 1) & TracePart::kAlignment) == 0))
699 return false;
700 *ev = reinterpret_cast<EventT *>(pos);
701 return true;
702 }
703
704 template <typename EventT>
TraceRelease(ThreadState * thr,EventT * evp)705 ALWAYS_INLINE void TraceRelease(ThreadState *thr, EventT *evp) {
706 DCHECK_LE(evp + 1, &thr->tctx->trace.parts.Back()->events[TracePart::kSize]);
707 atomic_store_relaxed(&thr->trace_pos, (uptr)(evp + 1));
708 }
709
710 template <typename EventT>
TraceEvent(ThreadState * thr,EventT ev)711 void TraceEvent(ThreadState *thr, EventT ev) {
712 EventT *evp;
713 if (!TraceAcquire(thr, &evp)) {
714 TraceSwitchPart(thr);
715 UNUSED bool res = TraceAcquire(thr, &evp);
716 DCHECK(res);
717 }
718 *evp = ev;
719 TraceRelease(thr, evp);
720 }
721
722 ALWAYS_INLINE WARN_UNUSED_RESULT bool TryTraceFunc(ThreadState *thr,
723 uptr pc = 0) {
724 if (!kCollectHistory)
725 return true;
726 EventFunc *ev;
727 if (UNLIKELY(!TraceAcquire(thr, &ev)))
728 return false;
729 ev->is_access = 0;
730 ev->is_func = 1;
731 ev->pc = pc;
732 TraceRelease(thr, ev);
733 return true;
734 }
735
736 WARN_UNUSED_RESULT
737 bool TryTraceMemoryAccess(ThreadState *thr, uptr pc, uptr addr, uptr size,
738 AccessType typ);
739 WARN_UNUSED_RESULT
740 bool TryTraceMemoryAccessRange(ThreadState *thr, uptr pc, uptr addr, uptr size,
741 AccessType typ);
742 void TraceMemoryAccessRange(ThreadState *thr, uptr pc, uptr addr, uptr size,
743 AccessType typ);
744 void TraceFunc(ThreadState *thr, uptr pc = 0);
745 void TraceMutexLock(ThreadState *thr, EventType type, uptr pc, uptr addr,
746 StackID stk);
747 void TraceMutexUnlock(ThreadState *thr, uptr addr);
748 void TraceTime(ThreadState *thr);
749
750 } // namespace v3
751
752 void GrowShadowStack(ThreadState *thr);
753
754 ALWAYS_INLINE
FuncEntry(ThreadState * thr,uptr pc)755 void FuncEntry(ThreadState *thr, uptr pc) {
756 DPrintf2("#%d: FuncEntry %p\n", (int)thr->fast_state.tid(), (void *)pc);
757 if (kCollectHistory) {
758 thr->fast_state.IncrementEpoch();
759 TraceAddEvent(thr, thr->fast_state, EventTypeFuncEnter, pc);
760 }
761
762 // Shadow stack maintenance can be replaced with
763 // stack unwinding during trace switch (which presumably must be faster).
764 DCHECK_GE(thr->shadow_stack_pos, thr->shadow_stack);
765 #if !SANITIZER_GO
766 DCHECK_LT(thr->shadow_stack_pos, thr->shadow_stack_end);
767 #else
768 if (thr->shadow_stack_pos == thr->shadow_stack_end)
769 GrowShadowStack(thr);
770 #endif
771 thr->shadow_stack_pos[0] = pc;
772 thr->shadow_stack_pos++;
773 }
774
775 ALWAYS_INLINE
FuncExit(ThreadState * thr)776 void FuncExit(ThreadState *thr) {
777 DPrintf2("#%d: FuncExit\n", (int)thr->fast_state.tid());
778 if (kCollectHistory) {
779 thr->fast_state.IncrementEpoch();
780 TraceAddEvent(thr, thr->fast_state, EventTypeFuncExit, 0);
781 }
782
783 DCHECK_GT(thr->shadow_stack_pos, thr->shadow_stack);
784 #if !SANITIZER_GO
785 DCHECK_LT(thr->shadow_stack_pos, thr->shadow_stack_end);
786 #endif
787 thr->shadow_stack_pos--;
788 }
789
790 #if !SANITIZER_GO
791 extern void (*on_initialize)(void);
792 extern int (*on_finalize)(int);
793 #endif
794
795 } // namespace __tsan
796
797 #endif // TSAN_RTL_H
798