1 //===-- tsan_interceptors.cc ----------------------------------------------===// 2 // 3 // This file is distributed under the University of Illinois Open Source 4 // License. See LICENSE.TXT for details. 5 // 6 //===----------------------------------------------------------------------===// 7 // 8 // This file is a part of ThreadSanitizer (TSan), a race detector. 9 // 10 // FIXME: move as many interceptors as possible into 11 // sanitizer_common/sanitizer_common_interceptors.inc 12 //===----------------------------------------------------------------------===// 13 14 #include "sanitizer_common/sanitizer_atomic.h" 15 #include "sanitizer_common/sanitizer_libc.h" 16 #include "sanitizer_common/sanitizer_linux.h" 17 #include "sanitizer_common/sanitizer_platform_limits_posix.h" 18 #include "sanitizer_common/sanitizer_placement_new.h" 19 #include "sanitizer_common/sanitizer_stacktrace.h" 20 #include "interception/interception.h" 21 #include "tsan_interface.h" 22 #include "tsan_platform.h" 23 #include "tsan_suppressions.h" 24 #include "tsan_rtl.h" 25 #include "tsan_mman.h" 26 #include "tsan_fd.h" 27 28 using namespace __tsan; // NOLINT 29 30 #if SANITIZER_FREEBSD 31 #define __errno_location __error 32 #define __libc_malloc __malloc 33 #define __libc_realloc __realloc 34 #define __libc_calloc __calloc 35 #define __libc_free __free 36 #define stdout __stdoutp 37 #define stderr __stderrp 38 #endif 39 #if SANITIZER_NETBSD 40 #define __errno_location __errno 41 #define pthread_yield sched_yield 42 #define fileno_unlocked fileno 43 #define stdout __sF[1] 44 #define stderr __sF[2] 45 #endif 46 47 const int kSigCount = 65; 48 49 struct my_siginfo_t { 50 // The size is determined by looking at sizeof of real siginfo_t on linux. 51 u64 opaque[128 / sizeof(u64)]; 52 }; 53 54 struct ucontext_t { 55 // The size is determined by looking at sizeof of real ucontext_t on linux. 56 u64 opaque[936 / sizeof(u64) + 1]; 57 }; 58 59 extern "C" int pthread_attr_init(void *attr); 60 extern "C" int pthread_attr_destroy(void *attr); 61 DECLARE_REAL(int, pthread_attr_getdetachstate, void *, void *) 62 extern "C" int pthread_attr_setstacksize(void *attr, uptr stacksize); 63 extern "C" int pthread_key_create(unsigned *key, void (*destructor)(void* v)); 64 extern "C" int pthread_setspecific(unsigned key, const void *v); 65 DECLARE_REAL(int, pthread_mutexattr_gettype, void *, void *) 66 extern "C" int pthread_yield(); 67 extern "C" int pthread_sigmask(int how, const __sanitizer_sigset_t *set, 68 __sanitizer_sigset_t *oldset); 69 // REAL(sigfillset) defined in common interceptors. 70 DECLARE_REAL(int, sigfillset, __sanitizer_sigset_t *set) 71 DECLARE_REAL(int, fflush, __sanitizer_FILE *fp) 72 extern "C" void *pthread_self(); 73 extern "C" void _exit(int status); 74 extern "C" int *__errno_location(); 75 extern "C" int fileno_unlocked(void *stream); 76 extern "C" void *__libc_malloc(uptr size); 77 extern "C" void *__libc_calloc(uptr size, uptr n); 78 extern "C" void *__libc_realloc(void *ptr, uptr size); 79 extern "C" void __libc_free(void *ptr); 80 #if !SANITIZER_FREEBSD && !SANITIZER_NETBSD 81 extern "C" int mallopt(int param, int value); 82 #endif 83 #if SANITIZER_NETBSD 84 extern __sanitizer_FILE **__sF; 85 #else 86 extern __sanitizer_FILE *stdout, *stderr; 87 #endif 88 const int PTHREAD_MUTEX_RECURSIVE = 1; 89 const int PTHREAD_MUTEX_RECURSIVE_NP = 1; 90 const int EINVAL = 22; 91 const int EBUSY = 16; 92 const int EOWNERDEAD = 130; 93 const int EPOLL_CTL_ADD = 1; 94 const int SIGILL = 4; 95 const int SIGABRT = 6; 96 const int SIGFPE = 8; 97 const int SIGSEGV = 11; 98 const int SIGPIPE = 13; 99 const int SIGTERM = 15; 100 const int SIGBUS = 7; 101 const int SIGSYS = 31; 102 void *const MAP_FAILED = (void*)-1; 103 const int PTHREAD_BARRIER_SERIAL_THREAD = -1; 104 const int MAP_FIXED = 0x10; 105 typedef long long_t; // NOLINT 106 107 // From /usr/include/unistd.h 108 # define F_ULOCK 0 /* Unlock a previously locked region. */ 109 # define F_LOCK 1 /* Lock a region for exclusive use. */ 110 # define F_TLOCK 2 /* Test and lock a region for exclusive use. */ 111 # define F_TEST 3 /* Test a region for other processes locks. */ 112 113 typedef void (*sighandler_t)(int sig); 114 115 #define errno (*__errno_location()) 116 117 struct sigaction_t { 118 union { 119 sighandler_t sa_handler; 120 void (*sa_sigaction)(int sig, my_siginfo_t *siginfo, void *uctx); 121 }; 122 #if SANITIZER_FREEBSD || SANITIZER_NETBSD 123 int sa_flags; 124 __sanitizer_sigset_t sa_mask; 125 #else 126 __sanitizer_sigset_t sa_mask; 127 int sa_flags; 128 void (*sa_restorer)(); 129 #endif 130 }; 131 132 const sighandler_t SIG_DFL = (sighandler_t)0; 133 const sighandler_t SIG_IGN = (sighandler_t)1; 134 const sighandler_t SIG_ERR = (sighandler_t)-1; 135 const int SA_SIGINFO = 4; 136 const int SIG_SETMASK = 2; 137 138 namespace std { 139 struct nothrow_t {}; 140 } // namespace std 141 142 static sigaction_t sigactions[kSigCount]; 143 144 namespace __tsan { 145 struct SignalDesc { 146 bool armed; 147 bool sigaction; 148 my_siginfo_t siginfo; 149 ucontext_t ctx; 150 }; 151 152 struct SignalContext { 153 int int_signal_send; 154 atomic_uintptr_t in_blocking_func; 155 atomic_uintptr_t have_pending_signals; 156 SignalDesc pending_signals[kSigCount]; 157 }; 158 159 // The object is 64-byte aligned, because we want hot data to be located in 160 // a single cache line if possible (it's accessed in every interceptor). 161 static ALIGNED(64) char libignore_placeholder[sizeof(LibIgnore)]; 162 static LibIgnore *libignore() { 163 return reinterpret_cast<LibIgnore*>(&libignore_placeholder[0]); 164 } 165 166 void InitializeLibIgnore() { 167 libignore()->Init(*SuppressionContext::Get()); 168 libignore()->OnLibraryLoaded(0); 169 } 170 171 } // namespace __tsan 172 173 static SignalContext *SigCtx(ThreadState *thr) { 174 SignalContext *ctx = (SignalContext*)thr->signal_ctx; 175 if (ctx == 0 && !thr->is_dead) { 176 ctx = (SignalContext*)MmapOrDie(sizeof(*ctx), "SignalContext"); 177 MemoryResetRange(thr, (uptr)&SigCtx, (uptr)ctx, sizeof(*ctx)); 178 thr->signal_ctx = ctx; 179 } 180 return ctx; 181 } 182 183 static unsigned g_thread_finalize_key; 184 185 class ScopedInterceptor { 186 public: 187 ScopedInterceptor(ThreadState *thr, const char *fname, uptr pc); 188 ~ScopedInterceptor(); 189 private: 190 ThreadState *const thr_; 191 const uptr pc_; 192 bool in_ignored_lib_; 193 }; 194 195 ScopedInterceptor::ScopedInterceptor(ThreadState *thr, const char *fname, 196 uptr pc) 197 : thr_(thr) 198 , pc_(pc) 199 , in_ignored_lib_(false) { 200 if (!thr_->ignore_interceptors) { 201 Initialize(thr); 202 FuncEntry(thr, pc); 203 } 204 DPrintf("#%d: intercept %s()\n", thr_->tid, fname); 205 if (!thr_->in_ignored_lib && libignore()->IsIgnored(pc)) { 206 in_ignored_lib_ = true; 207 thr_->in_ignored_lib = true; 208 ThreadIgnoreBegin(thr_, pc_); 209 } 210 } 211 212 ScopedInterceptor::~ScopedInterceptor() { 213 if (in_ignored_lib_) { 214 thr_->in_ignored_lib = false; 215 ThreadIgnoreEnd(thr_, pc_); 216 } 217 if (!thr_->ignore_interceptors) { 218 ProcessPendingSignals(thr_); 219 FuncExit(thr_); 220 CheckNoLocks(thr_); 221 } 222 } 223 224 #define SCOPED_INTERCEPTOR_RAW(func, ...) \ 225 ThreadState *thr = cur_thread(); \ 226 const uptr caller_pc = GET_CALLER_PC(); \ 227 ScopedInterceptor si(thr, #func, caller_pc); \ 228 const uptr pc = StackTrace::GetCurrentPc(); \ 229 (void)pc; \ 230 /**/ 231 232 #define SCOPED_TSAN_INTERCEPTOR(func, ...) \ 233 SCOPED_INTERCEPTOR_RAW(func, __VA_ARGS__); \ 234 if (REAL(func) == 0) { \ 235 Report("FATAL: ThreadSanitizer: failed to intercept %s\n", #func); \ 236 Die(); \ 237 } \ 238 if (thr->ignore_interceptors || thr->in_ignored_lib) \ 239 return REAL(func)(__VA_ARGS__); \ 240 /**/ 241 242 #define TSAN_INTERCEPTOR(ret, func, ...) INTERCEPTOR(ret, func, __VA_ARGS__) 243 #define TSAN_INTERCEPT(func) INTERCEPT_FUNCTION(func) 244 #if SANITIZER_FREEBSD || SANITIZER_NETBSD 245 # define TSAN_INTERCEPT_VER(func, ver) INTERCEPT_FUNCTION(func) 246 #else 247 # define TSAN_INTERCEPT_VER(func, ver) INTERCEPT_FUNCTION_VER(func, ver) 248 #endif 249 250 #define BLOCK_REAL(name) (BlockingCall(thr), REAL(name)) 251 252 struct BlockingCall { 253 explicit BlockingCall(ThreadState *thr) 254 : thr(thr) 255 , ctx(SigCtx(thr)) { 256 for (;;) { 257 atomic_store(&ctx->in_blocking_func, 1, memory_order_relaxed); 258 if (atomic_load(&ctx->have_pending_signals, memory_order_relaxed) == 0) 259 break; 260 atomic_store(&ctx->in_blocking_func, 0, memory_order_relaxed); 261 ProcessPendingSignals(thr); 262 } 263 // When we are in a "blocking call", we process signals asynchronously 264 // (right when they arrive). In this context we do not expect to be 265 // executing any user/runtime code. The known interceptor sequence when 266 // this is not true is: pthread_join -> munmap(stack). It's fine 267 // to ignore munmap in this case -- we handle stack shadow separately. 268 thr->ignore_interceptors++; 269 } 270 271 ~BlockingCall() { 272 thr->ignore_interceptors--; 273 atomic_store(&ctx->in_blocking_func, 0, memory_order_relaxed); 274 } 275 276 ThreadState *thr; 277 SignalContext *ctx; 278 }; 279 280 TSAN_INTERCEPTOR(unsigned, sleep, unsigned sec) { 281 SCOPED_TSAN_INTERCEPTOR(sleep, sec); 282 unsigned res = BLOCK_REAL(sleep)(sec); 283 AfterSleep(thr, pc); 284 return res; 285 } 286 287 TSAN_INTERCEPTOR(int, usleep, long_t usec) { 288 SCOPED_TSAN_INTERCEPTOR(usleep, usec); 289 int res = BLOCK_REAL(usleep)(usec); 290 AfterSleep(thr, pc); 291 return res; 292 } 293 294 TSAN_INTERCEPTOR(int, nanosleep, void *req, void *rem) { 295 SCOPED_TSAN_INTERCEPTOR(nanosleep, req, rem); 296 int res = BLOCK_REAL(nanosleep)(req, rem); 297 AfterSleep(thr, pc); 298 return res; 299 } 300 301 // The sole reason tsan wraps atexit callbacks is to establish synchronization 302 // between callback setup and callback execution. 303 struct AtExitCtx { 304 void (*f)(); 305 void *arg; 306 }; 307 308 static void at_exit_wrapper(void *arg) { 309 ThreadState *thr = cur_thread(); 310 uptr pc = 0; 311 Acquire(thr, pc, (uptr)arg); 312 AtExitCtx *ctx = (AtExitCtx*)arg; 313 ((void(*)(void *arg))ctx->f)(ctx->arg); 314 __libc_free(ctx); 315 } 316 317 static int setup_at_exit_wrapper(ThreadState *thr, uptr pc, void(*f)(), 318 void *arg, void *dso); 319 320 TSAN_INTERCEPTOR(int, atexit, void (*f)()) { 321 if (cur_thread()->in_symbolizer) 322 return 0; 323 // We want to setup the atexit callback even if we are in ignored lib 324 // or after fork. 325 SCOPED_INTERCEPTOR_RAW(atexit, f); 326 return setup_at_exit_wrapper(thr, pc, (void(*)())f, 0, 0); 327 } 328 329 TSAN_INTERCEPTOR(int, __cxa_atexit, void (*f)(void *a), void *arg, void *dso) { 330 if (cur_thread()->in_symbolizer) 331 return 0; 332 SCOPED_TSAN_INTERCEPTOR(__cxa_atexit, f, arg, dso); 333 return setup_at_exit_wrapper(thr, pc, (void(*)())f, arg, dso); 334 } 335 336 static int setup_at_exit_wrapper(ThreadState *thr, uptr pc, void(*f)(), 337 void *arg, void *dso) { 338 AtExitCtx *ctx = (AtExitCtx*)__libc_malloc(sizeof(AtExitCtx)); 339 ctx->f = f; 340 ctx->arg = arg; 341 Release(thr, pc, (uptr)ctx); 342 // Memory allocation in __cxa_atexit will race with free during exit, 343 // because we do not see synchronization around atexit callback list. 344 ThreadIgnoreBegin(thr, pc); 345 int res = REAL(__cxa_atexit)(at_exit_wrapper, ctx, dso); 346 ThreadIgnoreEnd(thr, pc); 347 return res; 348 } 349 350 static void on_exit_wrapper(int status, void *arg) { 351 ThreadState *thr = cur_thread(); 352 uptr pc = 0; 353 Acquire(thr, pc, (uptr)arg); 354 AtExitCtx *ctx = (AtExitCtx*)arg; 355 ((void(*)(int status, void *arg))ctx->f)(status, ctx->arg); 356 __libc_free(ctx); 357 } 358 359 TSAN_INTERCEPTOR(int, on_exit, void(*f)(int, void*), void *arg) { 360 if (cur_thread()->in_symbolizer) 361 return 0; 362 SCOPED_TSAN_INTERCEPTOR(on_exit, f, arg); 363 AtExitCtx *ctx = (AtExitCtx*)__libc_malloc(sizeof(AtExitCtx)); 364 ctx->f = (void(*)())f; 365 ctx->arg = arg; 366 Release(thr, pc, (uptr)ctx); 367 // Memory allocation in __cxa_atexit will race with free during exit, 368 // because we do not see synchronization around atexit callback list. 369 ThreadIgnoreBegin(thr, pc); 370 int res = REAL(on_exit)(on_exit_wrapper, ctx); 371 ThreadIgnoreEnd(thr, pc); 372 return res; 373 } 374 375 // Cleanup old bufs. 376 static void JmpBufGarbageCollect(ThreadState *thr, uptr sp) { 377 for (uptr i = 0; i < thr->jmp_bufs.Size(); i++) { 378 JmpBuf *buf = &thr->jmp_bufs[i]; 379 if (buf->sp <= sp) { 380 uptr sz = thr->jmp_bufs.Size(); 381 thr->jmp_bufs[i] = thr->jmp_bufs[sz - 1]; 382 thr->jmp_bufs.PopBack(); 383 i--; 384 } 385 } 386 } 387 388 static void SetJmp(ThreadState *thr, uptr sp, uptr mangled_sp) { 389 if (thr->shadow_stack_pos == 0) // called from libc guts during bootstrap 390 return; 391 // Cleanup old bufs. 392 JmpBufGarbageCollect(thr, sp); 393 // Remember the buf. 394 JmpBuf *buf = thr->jmp_bufs.PushBack(); 395 buf->sp = sp; 396 buf->mangled_sp = mangled_sp; 397 buf->shadow_stack_pos = thr->shadow_stack_pos; 398 SignalContext *sctx = SigCtx(thr); 399 buf->int_signal_send = sctx ? sctx->int_signal_send : 0; 400 buf->in_blocking_func = sctx ? 401 atomic_load(&sctx->in_blocking_func, memory_order_relaxed) : 402 false; 403 buf->in_signal_handler = atomic_load(&thr->in_signal_handler, 404 memory_order_relaxed); 405 } 406 407 static void LongJmp(ThreadState *thr, uptr *env) { 408 #if SANITIZER_FREEBSD || SANITIZER_NETBSD 409 uptr mangled_sp = env[2]; 410 #else 411 uptr mangled_sp = env[6]; 412 #endif // SANITIZER_FREEBSD 413 // Find the saved buf by mangled_sp. 414 for (uptr i = 0; i < thr->jmp_bufs.Size(); i++) { 415 JmpBuf *buf = &thr->jmp_bufs[i]; 416 if (buf->mangled_sp == mangled_sp) { 417 CHECK_GE(thr->shadow_stack_pos, buf->shadow_stack_pos); 418 // Unwind the stack. 419 while (thr->shadow_stack_pos > buf->shadow_stack_pos) 420 FuncExit(thr); 421 SignalContext *sctx = SigCtx(thr); 422 if (sctx) { 423 sctx->int_signal_send = buf->int_signal_send; 424 atomic_store(&sctx->in_blocking_func, buf->in_blocking_func, 425 memory_order_relaxed); 426 } 427 atomic_store(&thr->in_signal_handler, buf->in_signal_handler, 428 memory_order_relaxed); 429 JmpBufGarbageCollect(thr, buf->sp - 1); // do not collect buf->sp 430 return; 431 } 432 } 433 Printf("ThreadSanitizer: can't find longjmp buf\n"); 434 CHECK(0); 435 } 436 437 // FIXME: put everything below into a common extern "C" block? 438 extern "C" void __tsan_setjmp(uptr sp, uptr mangled_sp) { 439 SetJmp(cur_thread(), sp, mangled_sp); 440 } 441 442 // Not called. Merely to satisfy TSAN_INTERCEPT(). 443 extern "C" SANITIZER_INTERFACE_ATTRIBUTE 444 int __interceptor_setjmp(void *env); 445 extern "C" int __interceptor_setjmp(void *env) { 446 CHECK(0); 447 return 0; 448 } 449 450 // FIXME: any reason to have a separate declaration? 451 extern "C" SANITIZER_INTERFACE_ATTRIBUTE 452 int __interceptor__setjmp(void *env); 453 extern "C" int __interceptor__setjmp(void *env) { 454 CHECK(0); 455 return 0; 456 } 457 458 extern "C" SANITIZER_INTERFACE_ATTRIBUTE 459 int __interceptor_sigsetjmp(void *env); 460 extern "C" int __interceptor_sigsetjmp(void *env) { 461 CHECK(0); 462 return 0; 463 } 464 465 extern "C" SANITIZER_INTERFACE_ATTRIBUTE 466 int __interceptor___sigsetjmp(void *env); 467 extern "C" int __interceptor___sigsetjmp(void *env) { 468 CHECK(0); 469 return 0; 470 } 471 472 extern "C" int setjmp(void *env); 473 extern "C" int _setjmp(void *env); 474 extern "C" int sigsetjmp(void *env); 475 extern "C" int __sigsetjmp(void *env); 476 DEFINE_REAL(int, setjmp, void *env) 477 DEFINE_REAL(int, _setjmp, void *env) 478 DEFINE_REAL(int, sigsetjmp, void *env) 479 DEFINE_REAL(int, __sigsetjmp, void *env) 480 481 TSAN_INTERCEPTOR(void, longjmp, uptr *env, int val) { 482 { 483 SCOPED_TSAN_INTERCEPTOR(longjmp, env, val); 484 } 485 LongJmp(cur_thread(), env); 486 REAL(longjmp)(env, val); 487 } 488 489 TSAN_INTERCEPTOR(void, siglongjmp, uptr *env, int val) { 490 { 491 SCOPED_TSAN_INTERCEPTOR(siglongjmp, env, val); 492 } 493 LongJmp(cur_thread(), env); 494 REAL(siglongjmp)(env, val); 495 } 496 497 TSAN_INTERCEPTOR(void*, malloc, uptr size) { 498 if (cur_thread()->in_symbolizer) 499 return __libc_malloc(size); 500 void *p = 0; 501 { 502 SCOPED_INTERCEPTOR_RAW(malloc, size); 503 p = user_alloc(thr, pc, size); 504 } 505 invoke_malloc_hook(p, size); 506 return p; 507 } 508 509 TSAN_INTERCEPTOR(void*, __libc_memalign, uptr align, uptr sz) { 510 SCOPED_TSAN_INTERCEPTOR(__libc_memalign, align, sz); 511 return user_alloc(thr, pc, sz, align); 512 } 513 514 TSAN_INTERCEPTOR(void*, calloc, uptr size, uptr n) { 515 if (cur_thread()->in_symbolizer) 516 return __libc_calloc(size, n); 517 if (__sanitizer::CallocShouldReturnNullDueToOverflow(size, n)) 518 return AllocatorReturnNull(); 519 void *p = 0; 520 { 521 SCOPED_INTERCEPTOR_RAW(calloc, size, n); 522 p = user_alloc(thr, pc, n * size); 523 if (p) 524 internal_memset(p, 0, n * size); 525 } 526 invoke_malloc_hook(p, n * size); 527 return p; 528 } 529 530 TSAN_INTERCEPTOR(void*, realloc, void *p, uptr size) { 531 if (cur_thread()->in_symbolizer) 532 return __libc_realloc(p, size); 533 if (p) 534 invoke_free_hook(p); 535 { 536 SCOPED_INTERCEPTOR_RAW(realloc, p, size); 537 p = user_realloc(thr, pc, p, size); 538 } 539 invoke_malloc_hook(p, size); 540 return p; 541 } 542 543 TSAN_INTERCEPTOR(void, free, void *p) { 544 if (p == 0) 545 return; 546 if (cur_thread()->in_symbolizer) 547 return __libc_free(p); 548 invoke_free_hook(p); 549 SCOPED_INTERCEPTOR_RAW(free, p); 550 user_free(thr, pc, p); 551 } 552 553 TSAN_INTERCEPTOR(void, cfree, void *p) { 554 if (p == 0) 555 return; 556 if (cur_thread()->in_symbolizer) 557 return __libc_free(p); 558 invoke_free_hook(p); 559 SCOPED_INTERCEPTOR_RAW(cfree, p); 560 user_free(thr, pc, p); 561 } 562 563 TSAN_INTERCEPTOR(uptr, malloc_usable_size, void *p) { 564 SCOPED_INTERCEPTOR_RAW(malloc_usable_size, p); 565 return user_alloc_usable_size(p); 566 } 567 568 #define OPERATOR_NEW_BODY(mangled_name) \ 569 if (cur_thread()->in_symbolizer) \ 570 return __libc_malloc(size); \ 571 void *p = 0; \ 572 { \ 573 SCOPED_INTERCEPTOR_RAW(mangled_name, size); \ 574 p = user_alloc(thr, pc, size); \ 575 } \ 576 invoke_malloc_hook(p, size); \ 577 return p; 578 579 SANITIZER_INTERFACE_ATTRIBUTE 580 void *operator new(__sanitizer::uptr size); 581 void *operator new(__sanitizer::uptr size) { 582 OPERATOR_NEW_BODY(_Znwm); 583 } 584 585 SANITIZER_INTERFACE_ATTRIBUTE 586 void *operator new[](__sanitizer::uptr size); 587 void *operator new[](__sanitizer::uptr size) { 588 OPERATOR_NEW_BODY(_Znam); 589 } 590 591 SANITIZER_INTERFACE_ATTRIBUTE 592 void *operator new(__sanitizer::uptr size, std::nothrow_t const&); 593 void *operator new(__sanitizer::uptr size, std::nothrow_t const&) { 594 OPERATOR_NEW_BODY(_ZnwmRKSt9nothrow_t); 595 } 596 597 SANITIZER_INTERFACE_ATTRIBUTE 598 void *operator new[](__sanitizer::uptr size, std::nothrow_t const&); 599 void *operator new[](__sanitizer::uptr size, std::nothrow_t const&) { 600 OPERATOR_NEW_BODY(_ZnamRKSt9nothrow_t); 601 } 602 603 #define OPERATOR_DELETE_BODY(mangled_name) \ 604 if (ptr == 0) return; \ 605 if (cur_thread()->in_symbolizer) \ 606 return __libc_free(ptr); \ 607 invoke_free_hook(ptr); \ 608 SCOPED_INTERCEPTOR_RAW(mangled_name, ptr); \ 609 user_free(thr, pc, ptr); 610 611 SANITIZER_INTERFACE_ATTRIBUTE 612 void operator delete(void *ptr) throw(); 613 void operator delete(void *ptr) throw() { 614 OPERATOR_DELETE_BODY(_ZdlPv); 615 } 616 617 SANITIZER_INTERFACE_ATTRIBUTE 618 void operator delete[](void *ptr) throw(); 619 void operator delete[](void *ptr) throw() { 620 OPERATOR_DELETE_BODY(_ZdaPv); 621 } 622 623 SANITIZER_INTERFACE_ATTRIBUTE 624 void operator delete(void *ptr, std::nothrow_t const&); 625 void operator delete(void *ptr, std::nothrow_t const&) { 626 OPERATOR_DELETE_BODY(_ZdlPvRKSt9nothrow_t); 627 } 628 629 SANITIZER_INTERFACE_ATTRIBUTE 630 void operator delete[](void *ptr, std::nothrow_t const&); 631 void operator delete[](void *ptr, std::nothrow_t const&) { 632 OPERATOR_DELETE_BODY(_ZdaPvRKSt9nothrow_t); 633 } 634 635 TSAN_INTERCEPTOR(uptr, strlen, const char *s) { 636 SCOPED_TSAN_INTERCEPTOR(strlen, s); 637 uptr len = internal_strlen(s); 638 MemoryAccessRange(thr, pc, (uptr)s, len + 1, false); 639 return len; 640 } 641 642 TSAN_INTERCEPTOR(void*, memset, void *dst, int v, uptr size) { 643 SCOPED_TSAN_INTERCEPTOR(memset, dst, v, size); 644 MemoryAccessRange(thr, pc, (uptr)dst, size, true); 645 return internal_memset(dst, v, size); 646 } 647 648 TSAN_INTERCEPTOR(void*, memcpy, void *dst, const void *src, uptr size) { 649 SCOPED_TSAN_INTERCEPTOR(memcpy, dst, src, size); 650 MemoryAccessRange(thr, pc, (uptr)dst, size, true); 651 MemoryAccessRange(thr, pc, (uptr)src, size, false); 652 return internal_memcpy(dst, src, size); 653 } 654 655 TSAN_INTERCEPTOR(int, memcmp, const void *s1, const void *s2, uptr n) { 656 SCOPED_TSAN_INTERCEPTOR(memcmp, s1, s2, n); 657 int res = 0; 658 uptr len = 0; 659 for (; len < n; len++) { 660 if ((res = ((unsigned char*)s1)[len] - ((unsigned char*)s2)[len])) 661 break; 662 } 663 MemoryAccessRange(thr, pc, (uptr)s1, len < n ? len + 1 : n, false); 664 MemoryAccessRange(thr, pc, (uptr)s2, len < n ? len + 1 : n, false); 665 return res; 666 } 667 668 TSAN_INTERCEPTOR(void*, memmove, void *dst, void *src, uptr n) { 669 SCOPED_TSAN_INTERCEPTOR(memmove, dst, src, n); 670 MemoryAccessRange(thr, pc, (uptr)dst, n, true); 671 MemoryAccessRange(thr, pc, (uptr)src, n, false); 672 return REAL(memmove)(dst, src, n); 673 } 674 675 TSAN_INTERCEPTOR(char*, strchr, char *s, int c) { 676 SCOPED_TSAN_INTERCEPTOR(strchr, s, c); 677 char *res = REAL(strchr)(s, c); 678 uptr len = res ? (char*)res - (char*)s + 1 : internal_strlen(s) + 1; 679 MemoryAccessRange(thr, pc, (uptr)s, len, false); 680 return res; 681 } 682 683 TSAN_INTERCEPTOR(char*, strchrnul, char *s, int c) { 684 SCOPED_TSAN_INTERCEPTOR(strchrnul, s, c); 685 char *res = REAL(strchrnul)(s, c); 686 uptr len = (char*)res - (char*)s + 1; 687 MemoryAccessRange(thr, pc, (uptr)s, len, false); 688 return res; 689 } 690 691 TSAN_INTERCEPTOR(char*, strrchr, char *s, int c) { 692 SCOPED_TSAN_INTERCEPTOR(strrchr, s, c); 693 MemoryAccessRange(thr, pc, (uptr)s, internal_strlen(s) + 1, false); 694 return REAL(strrchr)(s, c); 695 } 696 697 TSAN_INTERCEPTOR(char*, strcpy, char *dst, const char *src) { // NOLINT 698 SCOPED_TSAN_INTERCEPTOR(strcpy, dst, src); // NOLINT 699 uptr srclen = internal_strlen(src); 700 MemoryAccessRange(thr, pc, (uptr)dst, srclen + 1, true); 701 MemoryAccessRange(thr, pc, (uptr)src, srclen + 1, false); 702 return REAL(strcpy)(dst, src); // NOLINT 703 } 704 705 TSAN_INTERCEPTOR(char*, strncpy, char *dst, char *src, uptr n) { 706 SCOPED_TSAN_INTERCEPTOR(strncpy, dst, src, n); 707 uptr srclen = internal_strnlen(src, n); 708 MemoryAccessRange(thr, pc, (uptr)dst, n, true); 709 MemoryAccessRange(thr, pc, (uptr)src, min(srclen + 1, n), false); 710 return REAL(strncpy)(dst, src, n); 711 } 712 713 TSAN_INTERCEPTOR(const char*, strstr, const char *s1, const char *s2) { 714 SCOPED_TSAN_INTERCEPTOR(strstr, s1, s2); 715 const char *res = REAL(strstr)(s1, s2); 716 uptr len1 = internal_strlen(s1); 717 uptr len2 = internal_strlen(s2); 718 MemoryAccessRange(thr, pc, (uptr)s1, len1 + 1, false); 719 MemoryAccessRange(thr, pc, (uptr)s2, len2 + 1, false); 720 return res; 721 } 722 723 TSAN_INTERCEPTOR(char*, strdup, const char *str) { 724 SCOPED_TSAN_INTERCEPTOR(strdup, str); 725 // strdup will call malloc, so no instrumentation is required here. 726 return REAL(strdup)(str); 727 } 728 729 static bool fix_mmap_addr(void **addr, long_t sz, int flags) { 730 if (*addr) { 731 if (!IsAppMem((uptr)*addr) || !IsAppMem((uptr)*addr + sz - 1)) { 732 if (flags & MAP_FIXED) { 733 errno = EINVAL; 734 return false; 735 } else { 736 *addr = 0; 737 } 738 } 739 } 740 return true; 741 } 742 743 TSAN_INTERCEPTOR(void*, mmap, void *addr, long_t sz, int prot, 744 int flags, int fd, unsigned off) { 745 SCOPED_TSAN_INTERCEPTOR(mmap, addr, sz, prot, flags, fd, off); 746 if (!fix_mmap_addr(&addr, sz, flags)) 747 return MAP_FAILED; 748 void *res = REAL(mmap)(addr, sz, prot, flags, fd, off); 749 if (res != MAP_FAILED) { 750 if (fd > 0) 751 FdAccess(thr, pc, fd); 752 MemoryRangeImitateWrite(thr, pc, (uptr)res, sz); 753 } 754 return res; 755 } 756 757 #if !SANITIZER_FREEBSD && !SANITIZER_NETBSD 758 TSAN_INTERCEPTOR(void*, mmap64, void *addr, long_t sz, int prot, 759 int flags, int fd, u64 off) { 760 SCOPED_TSAN_INTERCEPTOR(mmap64, addr, sz, prot, flags, fd, off); 761 if (!fix_mmap_addr(&addr, sz, flags)) 762 return MAP_FAILED; 763 void *res = REAL(mmap64)(addr, sz, prot, flags, fd, off); 764 if (res != MAP_FAILED) { 765 if (fd > 0) 766 FdAccess(thr, pc, fd); 767 MemoryRangeImitateWrite(thr, pc, (uptr)res, sz); 768 } 769 return res; 770 } 771 #define TSAN_MAYBE_INTERCEPT_MMAP64 TSAN_INTERCEPT(mmap64) 772 #else 773 #define TSAN_MAYBE_INTERCEPT_MMAP64 774 #endif 775 776 TSAN_INTERCEPTOR(int, munmap, void *addr, long_t sz) { 777 SCOPED_TSAN_INTERCEPTOR(munmap, addr, sz); 778 DontNeedShadowFor((uptr)addr, sz); 779 int res = REAL(munmap)(addr, sz); 780 return res; 781 } 782 783 #if !SANITIZER_FREEBSD && !SANITIZER_NETBSD 784 TSAN_INTERCEPTOR(void*, memalign, uptr align, uptr sz) { 785 SCOPED_INTERCEPTOR_RAW(memalign, align, sz); 786 return user_alloc(thr, pc, sz, align); 787 } 788 #define TSAN_MAYBE_INTERCEPT_MEMALIGN TSAN_INTERCEPT(memalign) 789 #else 790 #define TSAN_MAYBE_INTERCEPT_MEMALIGN 791 #endif 792 793 TSAN_INTERCEPTOR(void*, aligned_alloc, uptr align, uptr sz) { 794 SCOPED_INTERCEPTOR_RAW(memalign, align, sz); 795 return user_alloc(thr, pc, sz, align); 796 } 797 798 TSAN_INTERCEPTOR(void*, valloc, uptr sz) { 799 SCOPED_INTERCEPTOR_RAW(valloc, sz); 800 return user_alloc(thr, pc, sz, GetPageSizeCached()); 801 } 802 803 #if !SANITIZER_FREEBSD && !SANITIZER_NETBSD 804 TSAN_INTERCEPTOR(void*, pvalloc, uptr sz) { 805 SCOPED_INTERCEPTOR_RAW(pvalloc, sz); 806 sz = RoundUp(sz, GetPageSizeCached()); 807 return user_alloc(thr, pc, sz, GetPageSizeCached()); 808 } 809 #define TSAN_MAYBE_INTERCEPT_PVALLOC TSAN_INTERCEPT(pvalloc) 810 #else 811 #define TSAN_MAYBE_INTERCEPT_PVALLOC 812 #endif 813 814 TSAN_INTERCEPTOR(int, posix_memalign, void **memptr, uptr align, uptr sz) { 815 SCOPED_INTERCEPTOR_RAW(posix_memalign, memptr, align, sz); 816 *memptr = user_alloc(thr, pc, sz, align); 817 return 0; 818 } 819 820 // Used in thread-safe function static initialization. 821 extern "C" int INTERFACE_ATTRIBUTE __cxa_guard_acquire(atomic_uint32_t *g) { 822 SCOPED_INTERCEPTOR_RAW(__cxa_guard_acquire, g); 823 for (;;) { 824 u32 cmp = atomic_load(g, memory_order_acquire); 825 if (cmp == 0) { 826 if (atomic_compare_exchange_strong(g, &cmp, 1<<16, memory_order_relaxed)) 827 return 1; 828 } else if (cmp == 1) { 829 Acquire(thr, pc, (uptr)g); 830 return 0; 831 } else { 832 internal_sched_yield(); 833 } 834 } 835 } 836 837 extern "C" void INTERFACE_ATTRIBUTE __cxa_guard_release(atomic_uint32_t *g) { 838 SCOPED_INTERCEPTOR_RAW(__cxa_guard_release, g); 839 Release(thr, pc, (uptr)g); 840 atomic_store(g, 1, memory_order_release); 841 } 842 843 extern "C" void INTERFACE_ATTRIBUTE __cxa_guard_abort(atomic_uint32_t *g) { 844 SCOPED_INTERCEPTOR_RAW(__cxa_guard_abort, g); 845 atomic_store(g, 0, memory_order_relaxed); 846 } 847 848 static void thread_finalize(void *v) { 849 uptr iter = (uptr)v; 850 if (iter > 1) { 851 if (pthread_setspecific(g_thread_finalize_key, (void*)(iter - 1))) { 852 Printf("ThreadSanitizer: failed to set thread key\n"); 853 Die(); 854 } 855 return; 856 } 857 { 858 ThreadState *thr = cur_thread(); 859 ThreadFinish(thr); 860 SignalContext *sctx = thr->signal_ctx; 861 if (sctx) { 862 thr->signal_ctx = 0; 863 UnmapOrDie(sctx, sizeof(*sctx)); 864 } 865 } 866 } 867 868 869 struct ThreadParam { 870 void* (*callback)(void *arg); 871 void *param; 872 atomic_uintptr_t tid; 873 }; 874 875 extern "C" void *__tsan_thread_start_func(void *arg) { 876 ThreadParam *p = (ThreadParam*)arg; 877 void* (*callback)(void *arg) = p->callback; 878 void *param = p->param; 879 int tid = 0; 880 { 881 ThreadState *thr = cur_thread(); 882 // Thread-local state is not initialized yet. 883 ScopedIgnoreInterceptors ignore; 884 ThreadIgnoreBegin(thr, 0); 885 if (pthread_setspecific(g_thread_finalize_key, 886 (void *)kPthreadDestructorIterations)) { 887 Printf("ThreadSanitizer: failed to set thread key\n"); 888 Die(); 889 } 890 ThreadIgnoreEnd(thr, 0); 891 while ((tid = atomic_load(&p->tid, memory_order_acquire)) == 0) 892 pthread_yield(); 893 atomic_store(&p->tid, 0, memory_order_release); 894 ThreadStart(thr, tid, GetTid()); 895 } 896 void *res = callback(param); 897 // Prevent the callback from being tail called, 898 // it mixes up stack traces. 899 volatile int foo = 42; 900 foo++; 901 return res; 902 } 903 904 TSAN_INTERCEPTOR(int, pthread_create, 905 void *th, void *attr, void *(*callback)(void*), void * param) { 906 SCOPED_INTERCEPTOR_RAW(pthread_create, th, attr, callback, param); 907 if (ctx->after_multithreaded_fork) { 908 if (flags()->die_after_fork) { 909 Report("ThreadSanitizer: starting new threads after multi-threaded " 910 "fork is not supported. Dying (set die_after_fork=0 to override)\n"); 911 Die(); 912 } else { 913 VPrintf(1, "ThreadSanitizer: starting new threads after multi-threaded " 914 "fork is not supported (pid %d). Continuing because of " 915 "die_after_fork=0, but you are on your own\n", internal_getpid()); 916 } 917 } 918 __sanitizer_pthread_attr_t myattr; 919 if (attr == 0) { 920 pthread_attr_init(&myattr); 921 attr = &myattr; 922 } 923 int detached = 0; 924 REAL(pthread_attr_getdetachstate)(attr, &detached); 925 AdjustStackSize(attr); 926 927 ThreadParam p; 928 p.callback = callback; 929 p.param = param; 930 atomic_store(&p.tid, 0, memory_order_relaxed); 931 int res = -1; 932 { 933 // Otherwise we see false positives in pthread stack manipulation. 934 ScopedIgnoreInterceptors ignore; 935 ThreadIgnoreBegin(thr, pc); 936 res = REAL(pthread_create)(th, attr, __tsan_thread_start_func, &p); 937 ThreadIgnoreEnd(thr, pc); 938 } 939 if (res == 0) { 940 int tid = ThreadCreate(thr, pc, *(uptr*)th, detached); 941 CHECK_NE(tid, 0); 942 atomic_store(&p.tid, tid, memory_order_release); 943 while (atomic_load(&p.tid, memory_order_acquire) != 0) 944 pthread_yield(); 945 } 946 if (attr == &myattr) 947 pthread_attr_destroy(&myattr); 948 return res; 949 } 950 951 TSAN_INTERCEPTOR(int, pthread_join, void *th, void **ret) { 952 SCOPED_INTERCEPTOR_RAW(pthread_join, th, ret); 953 int tid = ThreadTid(thr, pc, (uptr)th); 954 ThreadIgnoreBegin(thr, pc); 955 int res = BLOCK_REAL(pthread_join)(th, ret); 956 ThreadIgnoreEnd(thr, pc); 957 if (res == 0) { 958 ThreadJoin(thr, pc, tid); 959 } 960 return res; 961 } 962 963 TSAN_INTERCEPTOR(int, pthread_detach, void *th) { 964 SCOPED_TSAN_INTERCEPTOR(pthread_detach, th); 965 int tid = ThreadTid(thr, pc, (uptr)th); 966 int res = REAL(pthread_detach)(th); 967 if (res == 0) { 968 ThreadDetach(thr, pc, tid); 969 } 970 return res; 971 } 972 973 // Problem: 974 // NPTL implementation of pthread_cond has 2 versions (2.2.5 and 2.3.2). 975 // pthread_cond_t has different size in the different versions. 976 // If call new REAL functions for old pthread_cond_t, they will corrupt memory 977 // after pthread_cond_t (old cond is smaller). 978 // If we call old REAL functions for new pthread_cond_t, we will lose some 979 // functionality (e.g. old functions do not support waiting against 980 // CLOCK_REALTIME). 981 // Proper handling would require to have 2 versions of interceptors as well. 982 // But this is messy, in particular requires linker scripts when sanitizer 983 // runtime is linked into a shared library. 984 // Instead we assume we don't have dynamic libraries built against old 985 // pthread (2.2.5 is dated by 2002). And provide legacy_pthread_cond flag 986 // that allows to work with old libraries (but this mode does not support 987 // some features, e.g. pthread_condattr_getpshared). 988 static void *init_cond(void *c, bool force = false) { 989 // sizeof(pthread_cond_t) >= sizeof(uptr) in both versions. 990 // So we allocate additional memory on the side large enough to hold 991 // any pthread_cond_t object. Always call new REAL functions, but pass 992 // the aux object to them. 993 // Note: the code assumes that PTHREAD_COND_INITIALIZER initializes 994 // first word of pthread_cond_t to zero. 995 // It's all relevant only for linux. 996 if (!common_flags()->legacy_pthread_cond) 997 return c; 998 atomic_uintptr_t *p = (atomic_uintptr_t*)c; 999 uptr cond = atomic_load(p, memory_order_acquire); 1000 if (!force && cond != 0) 1001 return (void*)cond; 1002 void *newcond = WRAP(malloc)(pthread_cond_t_sz); 1003 internal_memset(newcond, 0, pthread_cond_t_sz); 1004 if (atomic_compare_exchange_strong(p, &cond, (uptr)newcond, 1005 memory_order_acq_rel)) 1006 return newcond; 1007 WRAP(free)(newcond); 1008 return (void*)cond; 1009 } 1010 1011 struct CondMutexUnlockCtx { 1012 ThreadState *thr; 1013 uptr pc; 1014 void *m; 1015 }; 1016 1017 static void cond_mutex_unlock(CondMutexUnlockCtx *arg) { 1018 MutexLock(arg->thr, arg->pc, (uptr)arg->m); 1019 } 1020 1021 INTERCEPTOR(int, pthread_cond_init, void *c, void *a) { 1022 void *cond = init_cond(c, true); 1023 SCOPED_TSAN_INTERCEPTOR(pthread_cond_init, cond, a); 1024 MemoryAccessRange(thr, pc, (uptr)c, sizeof(uptr), true); 1025 return REAL(pthread_cond_init)(cond, a); 1026 } 1027 1028 INTERCEPTOR(int, pthread_cond_wait, void *c, void *m) { 1029 void *cond = init_cond(c); 1030 SCOPED_TSAN_INTERCEPTOR(pthread_cond_wait, cond, m); 1031 MutexUnlock(thr, pc, (uptr)m); 1032 MemoryAccessRange(thr, pc, (uptr)c, sizeof(uptr), false); 1033 CondMutexUnlockCtx arg = {thr, pc, m}; 1034 // This ensures that we handle mutex lock even in case of pthread_cancel. 1035 // See test/tsan/cond_cancel.cc. 1036 int res = call_pthread_cancel_with_cleanup( 1037 (int(*)(void *c, void *m, void *abstime))REAL(pthread_cond_wait), 1038 cond, m, 0, (void(*)(void *arg))cond_mutex_unlock, &arg); 1039 if (res == errno_EOWNERDEAD) 1040 MutexRepair(thr, pc, (uptr)m); 1041 MutexLock(thr, pc, (uptr)m); 1042 return res; 1043 } 1044 1045 INTERCEPTOR(int, pthread_cond_timedwait, void *c, void *m, void *abstime) { 1046 void *cond = init_cond(c); 1047 SCOPED_TSAN_INTERCEPTOR(pthread_cond_timedwait, cond, m, abstime); 1048 MutexUnlock(thr, pc, (uptr)m); 1049 MemoryAccessRange(thr, pc, (uptr)c, sizeof(uptr), false); 1050 CondMutexUnlockCtx arg = {thr, pc, m}; 1051 // This ensures that we handle mutex lock even in case of pthread_cancel. 1052 // See test/tsan/cond_cancel.cc. 1053 int res = call_pthread_cancel_with_cleanup( 1054 REAL(pthread_cond_timedwait), cond, m, abstime, 1055 (void(*)(void *arg))cond_mutex_unlock, &arg); 1056 if (res == errno_EOWNERDEAD) 1057 MutexRepair(thr, pc, (uptr)m); 1058 MutexLock(thr, pc, (uptr)m); 1059 return res; 1060 } 1061 1062 INTERCEPTOR(int, pthread_cond_signal, void *c) { 1063 void *cond = init_cond(c); 1064 SCOPED_TSAN_INTERCEPTOR(pthread_cond_signal, cond); 1065 MemoryAccessRange(thr, pc, (uptr)c, sizeof(uptr), false); 1066 return REAL(pthread_cond_signal)(cond); 1067 } 1068 1069 INTERCEPTOR(int, pthread_cond_broadcast, void *c) { 1070 void *cond = init_cond(c); 1071 SCOPED_TSAN_INTERCEPTOR(pthread_cond_broadcast, cond); 1072 MemoryAccessRange(thr, pc, (uptr)c, sizeof(uptr), false); 1073 return REAL(pthread_cond_broadcast)(cond); 1074 } 1075 1076 INTERCEPTOR(int, pthread_cond_destroy, void *c) { 1077 void *cond = init_cond(c); 1078 SCOPED_TSAN_INTERCEPTOR(pthread_cond_destroy, cond); 1079 MemoryAccessRange(thr, pc, (uptr)c, sizeof(uptr), true); 1080 int res = REAL(pthread_cond_destroy)(cond); 1081 if (common_flags()->legacy_pthread_cond) { 1082 // Free our aux cond and zero the pointer to not leave dangling pointers. 1083 WRAP(free)(cond); 1084 atomic_store((atomic_uintptr_t*)c, 0, memory_order_relaxed); 1085 } 1086 return res; 1087 } 1088 1089 TSAN_INTERCEPTOR(int, pthread_mutex_init, void *m, void *a) { 1090 SCOPED_TSAN_INTERCEPTOR(pthread_mutex_init, m, a); 1091 int res = REAL(pthread_mutex_init)(m, a); 1092 if (res == 0) { 1093 bool recursive = false; 1094 if (a) { 1095 int type = 0; 1096 if (REAL(pthread_mutexattr_gettype)(a, &type) == 0) 1097 recursive = (type == PTHREAD_MUTEX_RECURSIVE 1098 || type == PTHREAD_MUTEX_RECURSIVE_NP); 1099 } 1100 MutexCreate(thr, pc, (uptr)m, false, recursive, false); 1101 } 1102 return res; 1103 } 1104 1105 TSAN_INTERCEPTOR(int, pthread_mutex_destroy, void *m) { 1106 SCOPED_TSAN_INTERCEPTOR(pthread_mutex_destroy, m); 1107 int res = REAL(pthread_mutex_destroy)(m); 1108 if (res == 0 || res == EBUSY) { 1109 MutexDestroy(thr, pc, (uptr)m); 1110 } 1111 return res; 1112 } 1113 1114 TSAN_INTERCEPTOR(int, pthread_mutex_trylock, void *m) { 1115 SCOPED_TSAN_INTERCEPTOR(pthread_mutex_trylock, m); 1116 int res = REAL(pthread_mutex_trylock)(m); 1117 if (res == EOWNERDEAD) 1118 MutexRepair(thr, pc, (uptr)m); 1119 if (res == 0 || res == EOWNERDEAD) 1120 MutexLock(thr, pc, (uptr)m, /*rec=*/1, /*try_lock=*/true); 1121 return res; 1122 } 1123 1124 TSAN_INTERCEPTOR(int, pthread_mutex_timedlock, void *m, void *abstime) { 1125 SCOPED_TSAN_INTERCEPTOR(pthread_mutex_timedlock, m, abstime); 1126 int res = REAL(pthread_mutex_timedlock)(m, abstime); 1127 if (res == 0) { 1128 MutexLock(thr, pc, (uptr)m); 1129 } 1130 return res; 1131 } 1132 1133 TSAN_INTERCEPTOR(int, pthread_spin_init, void *m, int pshared) { 1134 SCOPED_TSAN_INTERCEPTOR(pthread_spin_init, m, pshared); 1135 int res = REAL(pthread_spin_init)(m, pshared); 1136 if (res == 0) { 1137 MutexCreate(thr, pc, (uptr)m, false, false, false); 1138 } 1139 return res; 1140 } 1141 1142 TSAN_INTERCEPTOR(int, pthread_spin_destroy, void *m) { 1143 SCOPED_TSAN_INTERCEPTOR(pthread_spin_destroy, m); 1144 int res = REAL(pthread_spin_destroy)(m); 1145 if (res == 0) { 1146 MutexDestroy(thr, pc, (uptr)m); 1147 } 1148 return res; 1149 } 1150 1151 TSAN_INTERCEPTOR(int, pthread_spin_lock, void *m) { 1152 SCOPED_TSAN_INTERCEPTOR(pthread_spin_lock, m); 1153 int res = REAL(pthread_spin_lock)(m); 1154 if (res == 0) { 1155 MutexLock(thr, pc, (uptr)m); 1156 } 1157 return res; 1158 } 1159 1160 TSAN_INTERCEPTOR(int, pthread_spin_trylock, void *m) { 1161 SCOPED_TSAN_INTERCEPTOR(pthread_spin_trylock, m); 1162 int res = REAL(pthread_spin_trylock)(m); 1163 if (res == 0) { 1164 MutexLock(thr, pc, (uptr)m, /*rec=*/1, /*try_lock=*/true); 1165 } 1166 return res; 1167 } 1168 1169 TSAN_INTERCEPTOR(int, pthread_spin_unlock, void *m) { 1170 SCOPED_TSAN_INTERCEPTOR(pthread_spin_unlock, m); 1171 MutexUnlock(thr, pc, (uptr)m); 1172 int res = REAL(pthread_spin_unlock)(m); 1173 return res; 1174 } 1175 1176 TSAN_INTERCEPTOR(int, pthread_rwlock_init, void *m, void *a) { 1177 SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_init, m, a); 1178 int res = REAL(pthread_rwlock_init)(m, a); 1179 if (res == 0) { 1180 MutexCreate(thr, pc, (uptr)m, true, false, false); 1181 } 1182 return res; 1183 } 1184 1185 TSAN_INTERCEPTOR(int, pthread_rwlock_destroy, void *m) { 1186 SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_destroy, m); 1187 int res = REAL(pthread_rwlock_destroy)(m); 1188 if (res == 0) { 1189 MutexDestroy(thr, pc, (uptr)m); 1190 } 1191 return res; 1192 } 1193 1194 TSAN_INTERCEPTOR(int, pthread_rwlock_rdlock, void *m) { 1195 SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_rdlock, m); 1196 int res = REAL(pthread_rwlock_rdlock)(m); 1197 if (res == 0) { 1198 MutexReadLock(thr, pc, (uptr)m); 1199 } 1200 return res; 1201 } 1202 1203 TSAN_INTERCEPTOR(int, pthread_rwlock_tryrdlock, void *m) { 1204 SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_tryrdlock, m); 1205 int res = REAL(pthread_rwlock_tryrdlock)(m); 1206 if (res == 0) { 1207 MutexReadLock(thr, pc, (uptr)m, /*try_lock=*/true); 1208 } 1209 return res; 1210 } 1211 1212 TSAN_INTERCEPTOR(int, pthread_rwlock_timedrdlock, void *m, void *abstime) { 1213 SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_timedrdlock, m, abstime); 1214 int res = REAL(pthread_rwlock_timedrdlock)(m, abstime); 1215 if (res == 0) { 1216 MutexReadLock(thr, pc, (uptr)m); 1217 } 1218 return res; 1219 } 1220 1221 TSAN_INTERCEPTOR(int, pthread_rwlock_wrlock, void *m) { 1222 SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_wrlock, m); 1223 int res = REAL(pthread_rwlock_wrlock)(m); 1224 if (res == 0) { 1225 MutexLock(thr, pc, (uptr)m); 1226 } 1227 return res; 1228 } 1229 1230 TSAN_INTERCEPTOR(int, pthread_rwlock_trywrlock, void *m) { 1231 SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_trywrlock, m); 1232 int res = REAL(pthread_rwlock_trywrlock)(m); 1233 if (res == 0) { 1234 MutexLock(thr, pc, (uptr)m, /*rec=*/1, /*try_lock=*/true); 1235 } 1236 return res; 1237 } 1238 1239 TSAN_INTERCEPTOR(int, pthread_rwlock_timedwrlock, void *m, void *abstime) { 1240 SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_timedwrlock, m, abstime); 1241 int res = REAL(pthread_rwlock_timedwrlock)(m, abstime); 1242 if (res == 0) { 1243 MutexLock(thr, pc, (uptr)m); 1244 } 1245 return res; 1246 } 1247 1248 TSAN_INTERCEPTOR(int, pthread_rwlock_unlock, void *m) { 1249 SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_unlock, m); 1250 MutexReadOrWriteUnlock(thr, pc, (uptr)m); 1251 int res = REAL(pthread_rwlock_unlock)(m); 1252 return res; 1253 } 1254 1255 TSAN_INTERCEPTOR(int, pthread_barrier_init, void *b, void *a, unsigned count) { 1256 SCOPED_TSAN_INTERCEPTOR(pthread_barrier_init, b, a, count); 1257 MemoryWrite(thr, pc, (uptr)b, kSizeLog1); 1258 int res = REAL(pthread_barrier_init)(b, a, count); 1259 return res; 1260 } 1261 1262 TSAN_INTERCEPTOR(int, pthread_barrier_destroy, void *b) { 1263 SCOPED_TSAN_INTERCEPTOR(pthread_barrier_destroy, b); 1264 MemoryWrite(thr, pc, (uptr)b, kSizeLog1); 1265 int res = REAL(pthread_barrier_destroy)(b); 1266 return res; 1267 } 1268 1269 TSAN_INTERCEPTOR(int, pthread_barrier_wait, void *b) { 1270 SCOPED_TSAN_INTERCEPTOR(pthread_barrier_wait, b); 1271 Release(thr, pc, (uptr)b); 1272 MemoryRead(thr, pc, (uptr)b, kSizeLog1); 1273 int res = REAL(pthread_barrier_wait)(b); 1274 MemoryRead(thr, pc, (uptr)b, kSizeLog1); 1275 if (res == 0 || res == PTHREAD_BARRIER_SERIAL_THREAD) { 1276 Acquire(thr, pc, (uptr)b); 1277 } 1278 return res; 1279 } 1280 1281 TSAN_INTERCEPTOR(int, pthread_once, void *o, void (*f)()) { 1282 SCOPED_INTERCEPTOR_RAW(pthread_once, o, f); 1283 if (o == 0 || f == 0) 1284 return EINVAL; 1285 atomic_uint32_t *a = static_cast<atomic_uint32_t*>(o); 1286 u32 v = atomic_load(a, memory_order_acquire); 1287 if (v == 0 && atomic_compare_exchange_strong(a, &v, 1, 1288 memory_order_relaxed)) { 1289 (*f)(); 1290 if (!thr->in_ignored_lib) 1291 Release(thr, pc, (uptr)o); 1292 atomic_store(a, 2, memory_order_release); 1293 } else { 1294 while (v != 2) { 1295 pthread_yield(); 1296 v = atomic_load(a, memory_order_acquire); 1297 } 1298 if (!thr->in_ignored_lib) 1299 Acquire(thr, pc, (uptr)o); 1300 } 1301 return 0; 1302 } 1303 1304 TSAN_INTERCEPTOR(int, sem_init, void *s, int pshared, unsigned value) { 1305 SCOPED_TSAN_INTERCEPTOR(sem_init, s, pshared, value); 1306 int res = REAL(sem_init)(s, pshared, value); 1307 return res; 1308 } 1309 1310 TSAN_INTERCEPTOR(int, sem_destroy, void *s) { 1311 SCOPED_TSAN_INTERCEPTOR(sem_destroy, s); 1312 int res = REAL(sem_destroy)(s); 1313 return res; 1314 } 1315 1316 TSAN_INTERCEPTOR(int, sem_wait, void *s) { 1317 SCOPED_TSAN_INTERCEPTOR(sem_wait, s); 1318 int res = BLOCK_REAL(sem_wait)(s); 1319 if (res == 0) { 1320 Acquire(thr, pc, (uptr)s); 1321 } 1322 return res; 1323 } 1324 1325 TSAN_INTERCEPTOR(int, sem_trywait, void *s) { 1326 SCOPED_TSAN_INTERCEPTOR(sem_trywait, s); 1327 int res = BLOCK_REAL(sem_trywait)(s); 1328 if (res == 0) { 1329 Acquire(thr, pc, (uptr)s); 1330 } 1331 return res; 1332 } 1333 1334 TSAN_INTERCEPTOR(int, sem_timedwait, void *s, void *abstime) { 1335 SCOPED_TSAN_INTERCEPTOR(sem_timedwait, s, abstime); 1336 int res = BLOCK_REAL(sem_timedwait)(s, abstime); 1337 if (res == 0) { 1338 Acquire(thr, pc, (uptr)s); 1339 } 1340 return res; 1341 } 1342 1343 TSAN_INTERCEPTOR(int, sem_post, void *s) { 1344 SCOPED_TSAN_INTERCEPTOR(sem_post, s); 1345 Release(thr, pc, (uptr)s); 1346 int res = REAL(sem_post)(s); 1347 return res; 1348 } 1349 1350 TSAN_INTERCEPTOR(int, sem_getvalue, void *s, int *sval) { 1351 SCOPED_TSAN_INTERCEPTOR(sem_getvalue, s, sval); 1352 int res = REAL(sem_getvalue)(s, sval); 1353 if (res == 0) { 1354 Acquire(thr, pc, (uptr)s); 1355 } 1356 return res; 1357 } 1358 1359 #if !SANITIZER_FREEBSD && !SANITIZER_NETBSD 1360 TSAN_INTERCEPTOR(int, __xstat, int version, const char *path, void *buf) { 1361 SCOPED_TSAN_INTERCEPTOR(__xstat, version, path, buf); 1362 return REAL(__xstat)(version, path, buf); 1363 } 1364 #define TSAN_MAYBE_INTERCEPT___XSTAT TSAN_INTERCEPT(__xstat) 1365 #else 1366 #define TSAN_MAYBE_INTERCEPT___XSTAT 1367 #endif 1368 1369 TSAN_INTERCEPTOR(int, stat, const char *path, void *buf) { 1370 #if SANITIZER_FREEBSD || SANITIZER_NETBSD 1371 SCOPED_TSAN_INTERCEPTOR(stat, path, buf); 1372 return REAL(stat)(path, buf); 1373 #else 1374 SCOPED_TSAN_INTERCEPTOR(__xstat, 0, path, buf); 1375 return REAL(__xstat)(0, path, buf); 1376 #endif 1377 } 1378 1379 #if !SANITIZER_FREEBSD && !SANITIZER_NETBSD 1380 TSAN_INTERCEPTOR(int, __xstat64, int version, const char *path, void *buf) { 1381 SCOPED_TSAN_INTERCEPTOR(__xstat64, version, path, buf); 1382 return REAL(__xstat64)(version, path, buf); 1383 } 1384 #define TSAN_MAYBE_INTERCEPT___XSTAT64 TSAN_INTERCEPT(__xstat64) 1385 #else 1386 #define TSAN_MAYBE_INTERCEPT___XSTAT64 1387 #endif 1388 1389 #if !SANITIZER_FREEBSD && !SANITIZER_NETBSD 1390 TSAN_INTERCEPTOR(int, stat64, const char *path, void *buf) { 1391 SCOPED_TSAN_INTERCEPTOR(__xstat64, 0, path, buf); 1392 return REAL(__xstat64)(0, path, buf); 1393 } 1394 #define TSAN_MAYBE_INTERCEPT_STAT64 TSAN_INTERCEPT(stat64) 1395 #else 1396 #define TSAN_MAYBE_INTERCEPT_STAT64 1397 #endif 1398 1399 #if !SANITIZER_FREEBSD && !SANITIZER_NETBSD 1400 TSAN_INTERCEPTOR(int, __lxstat, int version, const char *path, void *buf) { 1401 SCOPED_TSAN_INTERCEPTOR(__lxstat, version, path, buf); 1402 return REAL(__lxstat)(version, path, buf); 1403 } 1404 #define TSAN_MAYBE_INTERCEPT___LXSTAT TSAN_INTERCEPT(__lxstat) 1405 #else 1406 #define TSAN_MAYBE_INTERCEPT___LXSTAT 1407 #endif 1408 1409 TSAN_INTERCEPTOR(int, lstat, const char *path, void *buf) { 1410 #if SANITIZER_FREEBSD || SANITIZER_NETBSD 1411 SCOPED_TSAN_INTERCEPTOR(lstat, path, buf); 1412 return REAL(lstat)(path, buf); 1413 #else 1414 SCOPED_TSAN_INTERCEPTOR(__lxstat, 0, path, buf); 1415 return REAL(__lxstat)(0, path, buf); 1416 #endif 1417 } 1418 1419 #if !SANITIZER_FREEBSD && !SANITIZER_NETBSD 1420 TSAN_INTERCEPTOR(int, __lxstat64, int version, const char *path, void *buf) { 1421 SCOPED_TSAN_INTERCEPTOR(__lxstat64, version, path, buf); 1422 return REAL(__lxstat64)(version, path, buf); 1423 } 1424 #define TSAN_MAYBE_INTERCEPT___LXSTAT64 TSAN_INTERCEPT(__lxstat64) 1425 #else 1426 #define TSAN_MAYBE_INTERCEPT___LXSTAT64 1427 #endif 1428 1429 #if !SANITIZER_FREEBSD && !SANITIZER_NETBSD 1430 TSAN_INTERCEPTOR(int, lstat64, const char *path, void *buf) { 1431 SCOPED_TSAN_INTERCEPTOR(__lxstat64, 0, path, buf); 1432 return REAL(__lxstat64)(0, path, buf); 1433 } 1434 #define TSAN_MAYBE_INTERCEPT_LSTAT64 TSAN_INTERCEPT(lstat64) 1435 #else 1436 #define TSAN_MAYBE_INTERCEPT_LSTAT64 1437 #endif 1438 1439 #if !SANITIZER_FREEBSD && !SANITIZER_NETBSD 1440 TSAN_INTERCEPTOR(int, __fxstat, int version, int fd, void *buf) { 1441 SCOPED_TSAN_INTERCEPTOR(__fxstat, version, fd, buf); 1442 if (fd > 0) 1443 FdAccess(thr, pc, fd); 1444 return REAL(__fxstat)(version, fd, buf); 1445 } 1446 #define TSAN_MAYBE_INTERCEPT___FXSTAT TSAN_INTERCEPT(__fxstat) 1447 #else 1448 #define TSAN_MAYBE_INTERCEPT___FXSTAT 1449 #endif 1450 1451 TSAN_INTERCEPTOR(int, fstat, int fd, void *buf) { 1452 #if SANITIZER_FREEBSD || SANITIZER_NETBSD 1453 SCOPED_TSAN_INTERCEPTOR(fstat, fd, buf); 1454 if (fd > 0) 1455 FdAccess(thr, pc, fd); 1456 return REAL(fstat)(fd, buf); 1457 #else 1458 SCOPED_TSAN_INTERCEPTOR(__fxstat, 0, fd, buf); 1459 if (fd > 0) 1460 FdAccess(thr, pc, fd); 1461 return REAL(__fxstat)(0, fd, buf); 1462 #endif 1463 } 1464 1465 #if !SANITIZER_FREEBSD && !SANITIZER_NETBSD 1466 TSAN_INTERCEPTOR(int, __fxstat64, int version, int fd, void *buf) { 1467 SCOPED_TSAN_INTERCEPTOR(__fxstat64, version, fd, buf); 1468 if (fd > 0) 1469 FdAccess(thr, pc, fd); 1470 return REAL(__fxstat64)(version, fd, buf); 1471 } 1472 #define TSAN_MAYBE_INTERCEPT___FXSTAT64 TSAN_INTERCEPT(__fxstat64) 1473 #else 1474 #define TSAN_MAYBE_INTERCEPT___FXSTAT64 1475 #endif 1476 1477 #if !SANITIZER_FREEBSD && !SANITIZER_NETBSD 1478 TSAN_INTERCEPTOR(int, fstat64, int fd, void *buf) { 1479 SCOPED_TSAN_INTERCEPTOR(__fxstat64, 0, fd, buf); 1480 if (fd > 0) 1481 FdAccess(thr, pc, fd); 1482 return REAL(__fxstat64)(0, fd, buf); 1483 } 1484 #define TSAN_MAYBE_INTERCEPT_FSTAT64 TSAN_INTERCEPT(fstat64) 1485 #else 1486 #define TSAN_MAYBE_INTERCEPT_FSTAT64 1487 #endif 1488 1489 TSAN_INTERCEPTOR(int, open, const char *name, int flags, int mode) { 1490 SCOPED_TSAN_INTERCEPTOR(open, name, flags, mode); 1491 int fd = REAL(open)(name, flags, mode); 1492 if (fd >= 0) 1493 FdFileCreate(thr, pc, fd); 1494 return fd; 1495 } 1496 1497 #if !SANITIZER_FREEBSD && !SANITIZER_NETBSD 1498 TSAN_INTERCEPTOR(int, open64, const char *name, int flags, int mode) { 1499 SCOPED_TSAN_INTERCEPTOR(open64, name, flags, mode); 1500 int fd = REAL(open64)(name, flags, mode); 1501 if (fd >= 0) 1502 FdFileCreate(thr, pc, fd); 1503 return fd; 1504 } 1505 #define TSAN_MAYBE_INTERCEPT_OPEN64 TSAN_INTERCEPT(open64) 1506 #else 1507 #define TSAN_MAYBE_INTERCEPT_OPEN64 1508 #endif 1509 1510 TSAN_INTERCEPTOR(int, creat, const char *name, int mode) { 1511 SCOPED_TSAN_INTERCEPTOR(creat, name, mode); 1512 int fd = REAL(creat)(name, mode); 1513 if (fd >= 0) 1514 FdFileCreate(thr, pc, fd); 1515 return fd; 1516 } 1517 1518 #if !SANITIZER_FREEBSD && !SANITIZER_NETBSD 1519 TSAN_INTERCEPTOR(int, creat64, const char *name, int mode) { 1520 SCOPED_TSAN_INTERCEPTOR(creat64, name, mode); 1521 int fd = REAL(creat64)(name, mode); 1522 if (fd >= 0) 1523 FdFileCreate(thr, pc, fd); 1524 return fd; 1525 } 1526 #define TSAN_MAYBE_INTERCEPT_CREAT64 TSAN_INTERCEPT(creat64) 1527 #else 1528 #define TSAN_MAYBE_INTERCEPT_CREAT64 1529 #endif 1530 1531 TSAN_INTERCEPTOR(int, dup, int oldfd) { 1532 SCOPED_TSAN_INTERCEPTOR(dup, oldfd); 1533 int newfd = REAL(dup)(oldfd); 1534 if (oldfd >= 0 && newfd >= 0 && newfd != oldfd) 1535 FdDup(thr, pc, oldfd, newfd); 1536 return newfd; 1537 } 1538 1539 TSAN_INTERCEPTOR(int, dup2, int oldfd, int newfd) { 1540 SCOPED_TSAN_INTERCEPTOR(dup2, oldfd, newfd); 1541 int newfd2 = REAL(dup2)(oldfd, newfd); 1542 if (oldfd >= 0 && newfd2 >= 0 && newfd2 != oldfd) 1543 FdDup(thr, pc, oldfd, newfd2); 1544 return newfd2; 1545 } 1546 1547 TSAN_INTERCEPTOR(int, dup3, int oldfd, int newfd, int flags) { 1548 SCOPED_TSAN_INTERCEPTOR(dup3, oldfd, newfd, flags); 1549 int newfd2 = REAL(dup3)(oldfd, newfd, flags); 1550 if (oldfd >= 0 && newfd2 >= 0 && newfd2 != oldfd) 1551 FdDup(thr, pc, oldfd, newfd2); 1552 return newfd2; 1553 } 1554 1555 #if !SANITIZER_FREEBSD && !SANITIZER_NETBSD 1556 TSAN_INTERCEPTOR(int, eventfd, unsigned initval, int flags) { 1557 SCOPED_TSAN_INTERCEPTOR(eventfd, initval, flags); 1558 int fd = REAL(eventfd)(initval, flags); 1559 if (fd >= 0) 1560 FdEventCreate(thr, pc, fd); 1561 return fd; 1562 } 1563 #define TSAN_MAYBE_INTERCEPT_EVENTFD TSAN_INTERCEPT(eventfd) 1564 #else 1565 #define TSAN_MAYBE_INTERCEPT_EVENTFD 1566 #endif 1567 1568 #if !SANITIZER_FREEBSD && !SANITIZER_NETBSD 1569 TSAN_INTERCEPTOR(int, signalfd, int fd, void *mask, int flags) { 1570 SCOPED_TSAN_INTERCEPTOR(signalfd, fd, mask, flags); 1571 if (fd >= 0) 1572 FdClose(thr, pc, fd); 1573 fd = REAL(signalfd)(fd, mask, flags); 1574 if (fd >= 0) 1575 FdSignalCreate(thr, pc, fd); 1576 return fd; 1577 } 1578 #define TSAN_MAYBE_INTERCEPT_SIGNALFD TSAN_INTERCEPT(signalfd) 1579 #else 1580 #define TSAN_MAYBE_INTERCEPT_SIGNALFD 1581 #endif 1582 1583 #if !SANITIZER_FREEBSD && !SANITIZER_NETBSD 1584 TSAN_INTERCEPTOR(int, inotify_init, int fake) { 1585 SCOPED_TSAN_INTERCEPTOR(inotify_init, fake); 1586 int fd = REAL(inotify_init)(fake); 1587 if (fd >= 0) 1588 FdInotifyCreate(thr, pc, fd); 1589 return fd; 1590 } 1591 #define TSAN_MAYBE_INTERCEPT_INOTIFY_INIT TSAN_INTERCEPT(inotify_init) 1592 #else 1593 #define TSAN_MAYBE_INTERCEPT_INOTIFY_INIT 1594 #endif 1595 1596 #if !SANITIZER_FREEBSD && !SANITIZER_NETBSD 1597 TSAN_INTERCEPTOR(int, inotify_init1, int flags) { 1598 SCOPED_TSAN_INTERCEPTOR(inotify_init1, flags); 1599 int fd = REAL(inotify_init1)(flags); 1600 if (fd >= 0) 1601 FdInotifyCreate(thr, pc, fd); 1602 return fd; 1603 } 1604 #define TSAN_MAYBE_INTERCEPT_INOTIFY_INIT1 TSAN_INTERCEPT(inotify_init1) 1605 #else 1606 #define TSAN_MAYBE_INTERCEPT_INOTIFY_INIT1 1607 #endif 1608 1609 TSAN_INTERCEPTOR(int, socket, int domain, int type, int protocol) { 1610 SCOPED_TSAN_INTERCEPTOR(socket, domain, type, protocol); 1611 int fd = REAL(socket)(domain, type, protocol); 1612 if (fd >= 0) 1613 FdSocketCreate(thr, pc, fd); 1614 return fd; 1615 } 1616 1617 TSAN_INTERCEPTOR(int, socketpair, int domain, int type, int protocol, int *fd) { 1618 SCOPED_TSAN_INTERCEPTOR(socketpair, domain, type, protocol, fd); 1619 int res = REAL(socketpair)(domain, type, protocol, fd); 1620 if (res == 0 && fd[0] >= 0 && fd[1] >= 0) 1621 FdPipeCreate(thr, pc, fd[0], fd[1]); 1622 return res; 1623 } 1624 1625 TSAN_INTERCEPTOR(int, connect, int fd, void *addr, unsigned addrlen) { 1626 SCOPED_TSAN_INTERCEPTOR(connect, fd, addr, addrlen); 1627 FdSocketConnecting(thr, pc, fd); 1628 int res = REAL(connect)(fd, addr, addrlen); 1629 if (res == 0 && fd >= 0) 1630 FdSocketConnect(thr, pc, fd); 1631 return res; 1632 } 1633 1634 TSAN_INTERCEPTOR(int, bind, int fd, void *addr, unsigned addrlen) { 1635 SCOPED_TSAN_INTERCEPTOR(bind, fd, addr, addrlen); 1636 int res = REAL(bind)(fd, addr, addrlen); 1637 if (fd > 0 && res == 0) 1638 FdAccess(thr, pc, fd); 1639 return res; 1640 } 1641 1642 TSAN_INTERCEPTOR(int, listen, int fd, int backlog) { 1643 SCOPED_TSAN_INTERCEPTOR(listen, fd, backlog); 1644 int res = REAL(listen)(fd, backlog); 1645 if (fd > 0 && res == 0) 1646 FdAccess(thr, pc, fd); 1647 return res; 1648 } 1649 1650 #if !SANITIZER_FREEBSD && !SANITIZER_NETBSD 1651 TSAN_INTERCEPTOR(int, epoll_create, int size) { 1652 SCOPED_TSAN_INTERCEPTOR(epoll_create, size); 1653 int fd = REAL(epoll_create)(size); 1654 if (fd >= 0) 1655 FdPollCreate(thr, pc, fd); 1656 return fd; 1657 } 1658 #define TSAN_MAYBE_INTERCEPT_EPOLL_CREATE TSAN_INTERCEPT(epoll_create) 1659 #else 1660 #define TSAN_MAYBE_INTERCEPT_EPOLL_CREATE 1661 #endif 1662 1663 #if !SANITIZER_FREEBSD && !SANITIZER_NETBSD 1664 TSAN_INTERCEPTOR(int, epoll_create1, int flags) { 1665 SCOPED_TSAN_INTERCEPTOR(epoll_create1, flags); 1666 int fd = REAL(epoll_create1)(flags); 1667 if (fd >= 0) 1668 FdPollCreate(thr, pc, fd); 1669 return fd; 1670 } 1671 #define TSAN_MAYBE_INTERCEPT_EPOLL_CREATE1 TSAN_INTERCEPT(epoll_create1) 1672 #else 1673 #define TSAN_MAYBE_INTERCEPT_EPOLL_CREATE1 1674 #endif 1675 1676 TSAN_INTERCEPTOR(int, close, int fd) { 1677 SCOPED_TSAN_INTERCEPTOR(close, fd); 1678 if (fd >= 0) 1679 FdClose(thr, pc, fd); 1680 return REAL(close)(fd); 1681 } 1682 1683 #if !SANITIZER_FREEBSD && !SANITIZER_NETBSD 1684 TSAN_INTERCEPTOR(int, __close, int fd) { 1685 SCOPED_TSAN_INTERCEPTOR(__close, fd); 1686 if (fd >= 0) 1687 FdClose(thr, pc, fd); 1688 return REAL(__close)(fd); 1689 } 1690 #define TSAN_MAYBE_INTERCEPT___CLOSE TSAN_INTERCEPT(__close) 1691 #else 1692 #define TSAN_MAYBE_INTERCEPT___CLOSE 1693 #endif 1694 1695 // glibc guts 1696 #if !SANITIZER_FREEBSD && !SANITIZER_NETBSD 1697 TSAN_INTERCEPTOR(void, __res_iclose, void *state, bool free_addr) { 1698 SCOPED_TSAN_INTERCEPTOR(__res_iclose, state, free_addr); 1699 int fds[64]; 1700 int cnt = ExtractResolvFDs(state, fds, ARRAY_SIZE(fds)); 1701 for (int i = 0; i < cnt; i++) { 1702 if (fds[i] > 0) 1703 FdClose(thr, pc, fds[i]); 1704 } 1705 REAL(__res_iclose)(state, free_addr); 1706 } 1707 #define TSAN_MAYBE_INTERCEPT___RES_ICLOSE TSAN_INTERCEPT(__res_iclose) 1708 #else 1709 #define TSAN_MAYBE_INTERCEPT___RES_ICLOSE 1710 #endif 1711 1712 TSAN_INTERCEPTOR(int, pipe, int *pipefd) { 1713 SCOPED_TSAN_INTERCEPTOR(pipe, pipefd); 1714 int res = REAL(pipe)(pipefd); 1715 if (res == 0 && pipefd[0] >= 0 && pipefd[1] >= 0) 1716 FdPipeCreate(thr, pc, pipefd[0], pipefd[1]); 1717 return res; 1718 } 1719 1720 TSAN_INTERCEPTOR(int, pipe2, int *pipefd, int flags) { 1721 SCOPED_TSAN_INTERCEPTOR(pipe2, pipefd, flags); 1722 int res = REAL(pipe2)(pipefd, flags); 1723 if (res == 0 && pipefd[0] >= 0 && pipefd[1] >= 0) 1724 FdPipeCreate(thr, pc, pipefd[0], pipefd[1]); 1725 return res; 1726 } 1727 1728 TSAN_INTERCEPTOR(long_t, send, int fd, void *buf, long_t len, int flags) { 1729 SCOPED_TSAN_INTERCEPTOR(send, fd, buf, len, flags); 1730 if (fd >= 0) { 1731 FdAccess(thr, pc, fd); 1732 FdRelease(thr, pc, fd); 1733 } 1734 int res = REAL(send)(fd, buf, len, flags); 1735 return res; 1736 } 1737 1738 TSAN_INTERCEPTOR(long_t, sendmsg, int fd, void *msg, int flags) { 1739 SCOPED_TSAN_INTERCEPTOR(sendmsg, fd, msg, flags); 1740 if (fd >= 0) { 1741 FdAccess(thr, pc, fd); 1742 FdRelease(thr, pc, fd); 1743 } 1744 int res = REAL(sendmsg)(fd, msg, flags); 1745 return res; 1746 } 1747 1748 TSAN_INTERCEPTOR(long_t, recv, int fd, void *buf, long_t len, int flags) { 1749 SCOPED_TSAN_INTERCEPTOR(recv, fd, buf, len, flags); 1750 if (fd >= 0) 1751 FdAccess(thr, pc, fd); 1752 int res = REAL(recv)(fd, buf, len, flags); 1753 if (res >= 0 && fd >= 0) { 1754 FdAcquire(thr, pc, fd); 1755 } 1756 return res; 1757 } 1758 1759 TSAN_INTERCEPTOR(int, unlink, char *path) { 1760 SCOPED_TSAN_INTERCEPTOR(unlink, path); 1761 Release(thr, pc, File2addr(path)); 1762 int res = REAL(unlink)(path); 1763 return res; 1764 } 1765 1766 TSAN_INTERCEPTOR(void*, tmpfile, int fake) { 1767 SCOPED_TSAN_INTERCEPTOR(tmpfile, fake); 1768 void *res = REAL(tmpfile)(fake); 1769 if (res) { 1770 int fd = fileno_unlocked(res); 1771 if (fd >= 0) 1772 FdFileCreate(thr, pc, fd); 1773 } 1774 return res; 1775 } 1776 1777 #if !SANITIZER_FREEBSD && !SANITIZER_NETBSD 1778 TSAN_INTERCEPTOR(void*, tmpfile64, int fake) { 1779 SCOPED_TSAN_INTERCEPTOR(tmpfile64, fake); 1780 void *res = REAL(tmpfile64)(fake); 1781 if (res) { 1782 int fd = fileno_unlocked(res); 1783 if (fd >= 0) 1784 FdFileCreate(thr, pc, fd); 1785 } 1786 return res; 1787 } 1788 #define TSAN_MAYBE_INTERCEPT_TMPFILE64 TSAN_INTERCEPT(tmpfile64) 1789 #else 1790 #define TSAN_MAYBE_INTERCEPT_TMPFILE64 1791 #endif 1792 1793 TSAN_INTERCEPTOR(uptr, fread, void *ptr, uptr size, uptr nmemb, void *f) { 1794 // libc file streams can call user-supplied functions, see fopencookie. 1795 { 1796 SCOPED_TSAN_INTERCEPTOR(fread, ptr, size, nmemb, f); 1797 MemoryAccessRange(thr, pc, (uptr)ptr, size * nmemb, true); 1798 } 1799 return REAL(fread)(ptr, size, nmemb, f); 1800 } 1801 1802 TSAN_INTERCEPTOR(uptr, fwrite, const void *p, uptr size, uptr nmemb, void *f) { 1803 // libc file streams can call user-supplied functions, see fopencookie. 1804 { 1805 SCOPED_TSAN_INTERCEPTOR(fwrite, p, size, nmemb, f); 1806 MemoryAccessRange(thr, pc, (uptr)p, size * nmemb, false); 1807 } 1808 return REAL(fwrite)(p, size, nmemb, f); 1809 } 1810 1811 TSAN_INTERCEPTOR(void, abort, int fake) { 1812 SCOPED_TSAN_INTERCEPTOR(abort, fake); 1813 REAL(fflush)(0); 1814 REAL(abort)(fake); 1815 } 1816 1817 TSAN_INTERCEPTOR(int, puts, const char *s) { 1818 SCOPED_TSAN_INTERCEPTOR(puts, s); 1819 MemoryAccessRange(thr, pc, (uptr)s, internal_strlen(s), false); 1820 return REAL(puts)(s); 1821 } 1822 1823 TSAN_INTERCEPTOR(int, rmdir, char *path) { 1824 SCOPED_TSAN_INTERCEPTOR(rmdir, path); 1825 Release(thr, pc, Dir2addr(path)); 1826 int res = REAL(rmdir)(path); 1827 return res; 1828 } 1829 1830 TSAN_INTERCEPTOR(void*, opendir, char *path) { 1831 SCOPED_TSAN_INTERCEPTOR(opendir, path); 1832 void *res = REAL(opendir)(path); 1833 if (res != 0) 1834 Acquire(thr, pc, Dir2addr(path)); 1835 return res; 1836 } 1837 1838 #if !SANITIZER_FREEBSD && !SANITIZER_NETBSD 1839 TSAN_INTERCEPTOR(int, epoll_ctl, int epfd, int op, int fd, void *ev) { 1840 SCOPED_TSAN_INTERCEPTOR(epoll_ctl, epfd, op, fd, ev); 1841 if (epfd >= 0) 1842 FdAccess(thr, pc, epfd); 1843 if (epfd >= 0 && fd >= 0) 1844 FdAccess(thr, pc, fd); 1845 if (op == EPOLL_CTL_ADD && epfd >= 0) 1846 FdRelease(thr, pc, epfd); 1847 int res = REAL(epoll_ctl)(epfd, op, fd, ev); 1848 return res; 1849 } 1850 #define TSAN_MAYBE_INTERCEPT_EPOLL_CTL TSAN_INTERCEPT(epoll_ctl) 1851 #else 1852 #define TSAN_MAYBE_INTERCEPT_EPOLL_CTL 1853 #endif 1854 1855 #if !SANITIZER_FREEBSD && !SANITIZER_NETBSD 1856 TSAN_INTERCEPTOR(int, epoll_wait, int epfd, void *ev, int cnt, int timeout) { 1857 SCOPED_TSAN_INTERCEPTOR(epoll_wait, epfd, ev, cnt, timeout); 1858 if (epfd >= 0) 1859 FdAccess(thr, pc, epfd); 1860 int res = BLOCK_REAL(epoll_wait)(epfd, ev, cnt, timeout); 1861 if (res > 0 && epfd >= 0) 1862 FdAcquire(thr, pc, epfd); 1863 return res; 1864 } 1865 #define TSAN_MAYBE_INTERCEPT_EPOLL_WAIT TSAN_INTERCEPT(epoll_wait) 1866 #else 1867 #define TSAN_MAYBE_INTERCEPT_EPOLL_WAIT 1868 #endif 1869 1870 namespace __tsan { 1871 1872 static void CallUserSignalHandler(ThreadState *thr, bool sync, bool acquire, 1873 bool sigact, int sig, my_siginfo_t *info, void *uctx) { 1874 if (acquire) 1875 Acquire(thr, 0, (uptr)&sigactions[sig]); 1876 // Ensure that the handler does not spoil errno. 1877 const int saved_errno = errno; 1878 errno = 99; 1879 // Need to remember pc before the call, because the handler can reset it. 1880 uptr pc = sigact ? 1881 (uptr)sigactions[sig].sa_sigaction : 1882 (uptr)sigactions[sig].sa_handler; 1883 pc += 1; // return address is expected, OutputReport() will undo this 1884 if (sigact) 1885 sigactions[sig].sa_sigaction(sig, info, uctx); 1886 else 1887 sigactions[sig].sa_handler(sig); 1888 // We do not detect errno spoiling for SIGTERM, 1889 // because some SIGTERM handlers do spoil errno but reraise SIGTERM, 1890 // tsan reports false positive in such case. 1891 // It's difficult to properly detect this situation (reraise), 1892 // because in async signal processing case (when handler is called directly 1893 // from rtl_generic_sighandler) we have not yet received the reraised 1894 // signal; and it looks too fragile to intercept all ways to reraise a signal. 1895 if (flags()->report_bugs && !sync && sig != SIGTERM && errno != 99) { 1896 VarSizeStackTrace stack; 1897 ObtainCurrentStack(thr, pc, &stack); 1898 ThreadRegistryLock l(ctx->thread_registry); 1899 ScopedReport rep(ReportTypeErrnoInSignal); 1900 if (!IsFiredSuppression(ctx, rep, stack)) { 1901 rep.AddStack(stack, true); 1902 OutputReport(thr, rep); 1903 } 1904 } 1905 errno = saved_errno; 1906 } 1907 1908 void ProcessPendingSignals(ThreadState *thr) { 1909 SignalContext *sctx = SigCtx(thr); 1910 if (sctx == 0 || 1911 atomic_load(&sctx->have_pending_signals, memory_order_relaxed) == 0) 1912 return; 1913 atomic_store(&sctx->have_pending_signals, 0, memory_order_relaxed); 1914 atomic_fetch_add(&thr->in_signal_handler, 1, memory_order_relaxed); 1915 // These are too big for stack. 1916 static THREADLOCAL __sanitizer_sigset_t emptyset, oldset; 1917 REAL(sigfillset)(&emptyset); 1918 pthread_sigmask(SIG_SETMASK, &emptyset, &oldset); 1919 for (int sig = 0; sig < kSigCount; sig++) { 1920 SignalDesc *signal = &sctx->pending_signals[sig]; 1921 if (signal->armed) { 1922 signal->armed = false; 1923 if (sigactions[sig].sa_handler != SIG_DFL 1924 && sigactions[sig].sa_handler != SIG_IGN) { 1925 CallUserSignalHandler(thr, false, true, signal->sigaction, 1926 sig, &signal->siginfo, &signal->ctx); 1927 } 1928 } 1929 } 1930 pthread_sigmask(SIG_SETMASK, &oldset, 0); 1931 atomic_fetch_add(&thr->in_signal_handler, -1, memory_order_relaxed); 1932 } 1933 1934 } // namespace __tsan 1935 1936 static bool is_sync_signal(SignalContext *sctx, int sig) { 1937 return sig == SIGSEGV || sig == SIGBUS || sig == SIGILL || 1938 sig == SIGABRT || sig == SIGFPE || sig == SIGPIPE || sig == SIGSYS || 1939 // If we are sending signal to ourselves, we must process it now. 1940 (sctx && sig == sctx->int_signal_send); 1941 } 1942 1943 void ALWAYS_INLINE rtl_generic_sighandler(bool sigact, int sig, 1944 my_siginfo_t *info, void *ctx) { 1945 ThreadState *thr = cur_thread(); 1946 SignalContext *sctx = SigCtx(thr); 1947 if (sig < 0 || sig >= kSigCount) { 1948 VPrintf(1, "ThreadSanitizer: ignoring signal %d\n", sig); 1949 return; 1950 } 1951 // Don't mess with synchronous signals. 1952 const bool sync = is_sync_signal(sctx, sig); 1953 if (sync || 1954 // If we are in blocking function, we can safely process it now 1955 // (but check if we are in a recursive interceptor, 1956 // i.e. pthread_join()->munmap()). 1957 (sctx && atomic_load(&sctx->in_blocking_func, memory_order_relaxed))) { 1958 atomic_fetch_add(&thr->in_signal_handler, 1, memory_order_relaxed); 1959 if (sctx && atomic_load(&sctx->in_blocking_func, memory_order_relaxed)) { 1960 // We ignore interceptors in blocking functions, 1961 // temporary enbled them again while we are calling user function. 1962 int const i = thr->ignore_interceptors; 1963 thr->ignore_interceptors = 0; 1964 atomic_store(&sctx->in_blocking_func, 0, memory_order_relaxed); 1965 CallUserSignalHandler(thr, sync, true, sigact, sig, info, ctx); 1966 thr->ignore_interceptors = i; 1967 atomic_store(&sctx->in_blocking_func, 1, memory_order_relaxed); 1968 } else { 1969 // Be very conservative with when we do acquire in this case. 1970 // It's unsafe to do acquire in async handlers, because ThreadState 1971 // can be in inconsistent state. 1972 // SIGSYS looks relatively safe -- it's synchronous and can actually 1973 // need some global state. 1974 bool acq = (sig == SIGSYS); 1975 CallUserSignalHandler(thr, sync, acq, sigact, sig, info, ctx); 1976 } 1977 atomic_fetch_add(&thr->in_signal_handler, -1, memory_order_relaxed); 1978 return; 1979 } 1980 1981 if (sctx == 0) 1982 return; 1983 SignalDesc *signal = &sctx->pending_signals[sig]; 1984 if (signal->armed == false) { 1985 signal->armed = true; 1986 signal->sigaction = sigact; 1987 if (info) 1988 internal_memcpy(&signal->siginfo, info, sizeof(*info)); 1989 if (ctx) 1990 internal_memcpy(&signal->ctx, ctx, sizeof(signal->ctx)); 1991 atomic_store(&sctx->have_pending_signals, 1, memory_order_relaxed); 1992 } 1993 } 1994 1995 static void rtl_sighandler(int sig) { 1996 rtl_generic_sighandler(false, sig, 0, 0); 1997 } 1998 1999 static void rtl_sigaction(int sig, my_siginfo_t *info, void *ctx) { 2000 rtl_generic_sighandler(true, sig, info, ctx); 2001 } 2002 2003 TSAN_INTERCEPTOR(int, sigaction, int sig, sigaction_t *act, sigaction_t *old) { 2004 SCOPED_TSAN_INTERCEPTOR(sigaction, sig, act, old); 2005 if (old) 2006 internal_memcpy(old, &sigactions[sig], sizeof(*old)); 2007 if (act == 0) 2008 return 0; 2009 internal_memcpy(&sigactions[sig], act, sizeof(*act)); 2010 sigaction_t newact; 2011 internal_memcpy(&newact, act, sizeof(newact)); 2012 REAL(sigfillset)(&newact.sa_mask); 2013 if (act->sa_handler != SIG_IGN && act->sa_handler != SIG_DFL) { 2014 if (newact.sa_flags & SA_SIGINFO) 2015 newact.sa_sigaction = rtl_sigaction; 2016 else 2017 newact.sa_handler = rtl_sighandler; 2018 } 2019 ReleaseStore(thr, pc, (uptr)&sigactions[sig]); 2020 int res = REAL(sigaction)(sig, &newact, 0); 2021 return res; 2022 } 2023 2024 TSAN_INTERCEPTOR(sighandler_t, signal, int sig, sighandler_t h) { 2025 sigaction_t act; 2026 act.sa_handler = h; 2027 REAL(memset)(&act.sa_mask, -1, sizeof(act.sa_mask)); 2028 act.sa_flags = 0; 2029 sigaction_t old; 2030 int res = sigaction(sig, &act, &old); 2031 if (res) 2032 return SIG_ERR; 2033 return old.sa_handler; 2034 } 2035 2036 TSAN_INTERCEPTOR(int, sigsuspend, const __sanitizer_sigset_t *mask) { 2037 SCOPED_TSAN_INTERCEPTOR(sigsuspend, mask); 2038 return REAL(sigsuspend)(mask); 2039 } 2040 2041 TSAN_INTERCEPTOR(int, raise, int sig) { 2042 SCOPED_TSAN_INTERCEPTOR(raise, sig); 2043 SignalContext *sctx = SigCtx(thr); 2044 CHECK_NE(sctx, 0); 2045 int prev = sctx->int_signal_send; 2046 sctx->int_signal_send = sig; 2047 int res = REAL(raise)(sig); 2048 CHECK_EQ(sctx->int_signal_send, sig); 2049 sctx->int_signal_send = prev; 2050 return res; 2051 } 2052 2053 TSAN_INTERCEPTOR(int, kill, int pid, int sig) { 2054 SCOPED_TSAN_INTERCEPTOR(kill, pid, sig); 2055 SignalContext *sctx = SigCtx(thr); 2056 CHECK_NE(sctx, 0); 2057 int prev = sctx->int_signal_send; 2058 if (pid == (int)internal_getpid()) { 2059 sctx->int_signal_send = sig; 2060 } 2061 int res = REAL(kill)(pid, sig); 2062 if (pid == (int)internal_getpid()) { 2063 CHECK_EQ(sctx->int_signal_send, sig); 2064 sctx->int_signal_send = prev; 2065 } 2066 return res; 2067 } 2068 2069 TSAN_INTERCEPTOR(int, pthread_kill, void *tid, int sig) { 2070 SCOPED_TSAN_INTERCEPTOR(pthread_kill, tid, sig); 2071 SignalContext *sctx = SigCtx(thr); 2072 CHECK_NE(sctx, 0); 2073 int prev = sctx->int_signal_send; 2074 if (tid == pthread_self()) { 2075 sctx->int_signal_send = sig; 2076 } 2077 int res = REAL(pthread_kill)(tid, sig); 2078 if (tid == pthread_self()) { 2079 CHECK_EQ(sctx->int_signal_send, sig); 2080 sctx->int_signal_send = prev; 2081 } 2082 return res; 2083 } 2084 2085 TSAN_INTERCEPTOR(int, gettimeofday, void *tv, void *tz) { 2086 SCOPED_TSAN_INTERCEPTOR(gettimeofday, tv, tz); 2087 // It's intercepted merely to process pending signals. 2088 return REAL(gettimeofday)(tv, tz); 2089 } 2090 2091 TSAN_INTERCEPTOR(int, getaddrinfo, void *node, void *service, 2092 void *hints, void *rv) { 2093 SCOPED_TSAN_INTERCEPTOR(getaddrinfo, node, service, hints, rv); 2094 // We miss atomic synchronization in getaddrinfo, 2095 // and can report false race between malloc and free 2096 // inside of getaddrinfo. So ignore memory accesses. 2097 ThreadIgnoreBegin(thr, pc); 2098 int res = REAL(getaddrinfo)(node, service, hints, rv); 2099 ThreadIgnoreEnd(thr, pc); 2100 return res; 2101 } 2102 2103 TSAN_INTERCEPTOR(int, fork, int fake) { 2104 if (cur_thread()->in_symbolizer) 2105 return REAL(fork)(fake); 2106 SCOPED_INTERCEPTOR_RAW(fork, fake); 2107 ForkBefore(thr, pc); 2108 int pid = REAL(fork)(fake); 2109 if (pid == 0) { 2110 // child 2111 ForkChildAfter(thr, pc); 2112 FdOnFork(thr, pc); 2113 } else if (pid > 0) { 2114 // parent 2115 ForkParentAfter(thr, pc); 2116 } else { 2117 // error 2118 ForkParentAfter(thr, pc); 2119 } 2120 return pid; 2121 } 2122 2123 TSAN_INTERCEPTOR(int, vfork, int fake) { 2124 // Some programs (e.g. openjdk) call close for all file descriptors 2125 // in the child process. Under tsan it leads to false positives, because 2126 // address space is shared, so the parent process also thinks that 2127 // the descriptors are closed (while they are actually not). 2128 // This leads to false positives due to missed synchronization. 2129 // Strictly saying this is undefined behavior, because vfork child is not 2130 // allowed to call any functions other than exec/exit. But this is what 2131 // openjdk does, so we want to handle it. 2132 // We could disable interceptors in the child process. But it's not possible 2133 // to simply intercept and wrap vfork, because vfork child is not allowed 2134 // to return from the function that calls vfork, and that's exactly what 2135 // we would do. So this would require some assembly trickery as well. 2136 // Instead we simply turn vfork into fork. 2137 return WRAP(fork)(fake); 2138 } 2139 2140 static int OnExit(ThreadState *thr) { 2141 int status = Finalize(thr); 2142 REAL(fflush)(0); 2143 return status; 2144 } 2145 2146 struct TsanInterceptorContext { 2147 ThreadState *thr; 2148 const uptr caller_pc; 2149 const uptr pc; 2150 }; 2151 2152 static void HandleRecvmsg(ThreadState *thr, uptr pc, 2153 __sanitizer_msghdr *msg) { 2154 int fds[64]; 2155 int cnt = ExtractRecvmsgFDs(msg, fds, ARRAY_SIZE(fds)); 2156 for (int i = 0; i < cnt; i++) 2157 FdEventCreate(thr, pc, fds[i]); 2158 } 2159 2160 #include "sanitizer_common/sanitizer_platform_interceptors.h" 2161 // Causes interceptor recursion (getaddrinfo() and fopen()) 2162 #undef SANITIZER_INTERCEPT_GETADDRINFO 2163 // There interceptors do not seem to be strictly necessary for tsan. 2164 // But we see cases where the interceptors consume 70% of execution time. 2165 // Memory blocks passed to fgetgrent_r are "written to" by tsan several times. 2166 // First, there is some recursion (getgrnam_r calls fgetgrent_r), and each 2167 // function "writes to" the buffer. Then, the same memory is "written to" 2168 // twice, first as buf and then as pwbufp (both of them refer to the same 2169 // addresses). 2170 #undef SANITIZER_INTERCEPT_GETPWENT 2171 #undef SANITIZER_INTERCEPT_GETPWENT_R 2172 #undef SANITIZER_INTERCEPT_FGETPWENT 2173 #undef SANITIZER_INTERCEPT_GETPWNAM_AND_FRIENDS 2174 #undef SANITIZER_INTERCEPT_GETPWNAM_R_AND_FRIENDS 2175 2176 #define COMMON_INTERCEPT_FUNCTION(name) INTERCEPT_FUNCTION(name) 2177 2178 #define COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ptr, size) \ 2179 MemoryAccessRange(((TsanInterceptorContext *)ctx)->thr, \ 2180 ((TsanInterceptorContext *)ctx)->pc, (uptr)ptr, size, \ 2181 true) 2182 2183 #define COMMON_INTERCEPTOR_READ_RANGE(ctx, ptr, size) \ 2184 MemoryAccessRange(((TsanInterceptorContext *) ctx)->thr, \ 2185 ((TsanInterceptorContext *) ctx)->pc, (uptr) ptr, size, \ 2186 false) 2187 2188 #define COMMON_INTERCEPTOR_ENTER(ctx, func, ...) \ 2189 SCOPED_TSAN_INTERCEPTOR(func, __VA_ARGS__); \ 2190 TsanInterceptorContext _ctx = {thr, caller_pc, pc}; \ 2191 ctx = (void *)&_ctx; \ 2192 (void) ctx; 2193 2194 #define COMMON_INTERCEPTOR_ENTER_NOIGNORE(ctx, func, ...) \ 2195 SCOPED_INTERCEPTOR_RAW(func, __VA_ARGS__); \ 2196 TsanInterceptorContext _ctx = {thr, caller_pc, pc}; \ 2197 ctx = (void *)&_ctx; \ 2198 (void) ctx; 2199 2200 #define COMMON_INTERCEPTOR_FILE_OPEN(ctx, file, path) \ 2201 Acquire(thr, pc, File2addr(path)); \ 2202 if (file) { \ 2203 int fd = fileno_unlocked(file); \ 2204 if (fd >= 0) FdFileCreate(thr, pc, fd); \ 2205 } 2206 2207 #define COMMON_INTERCEPTOR_FILE_CLOSE(ctx, file) \ 2208 if (file) { \ 2209 int fd = fileno_unlocked(file); \ 2210 if (fd >= 0) FdClose(thr, pc, fd); \ 2211 } 2212 2213 #define COMMON_INTERCEPTOR_LIBRARY_LOADED(filename, res) \ 2214 libignore()->OnLibraryLoaded(filename) 2215 2216 #define COMMON_INTERCEPTOR_LIBRARY_UNLOADED() \ 2217 libignore()->OnLibraryUnloaded() 2218 2219 #define COMMON_INTERCEPTOR_FD_ACQUIRE(ctx, fd) \ 2220 FdAcquire(((TsanInterceptorContext *) ctx)->thr, pc, fd) 2221 2222 #define COMMON_INTERCEPTOR_FD_RELEASE(ctx, fd) \ 2223 FdRelease(((TsanInterceptorContext *) ctx)->thr, pc, fd) 2224 2225 #define COMMON_INTERCEPTOR_FD_ACCESS(ctx, fd) \ 2226 FdAccess(((TsanInterceptorContext *) ctx)->thr, pc, fd) 2227 2228 #define COMMON_INTERCEPTOR_FD_SOCKET_ACCEPT(ctx, fd, newfd) \ 2229 FdSocketAccept(((TsanInterceptorContext *) ctx)->thr, pc, fd, newfd) 2230 2231 #define COMMON_INTERCEPTOR_SET_THREAD_NAME(ctx, name) \ 2232 ThreadSetName(((TsanInterceptorContext *) ctx)->thr, name) 2233 2234 #define COMMON_INTERCEPTOR_SET_PTHREAD_NAME(ctx, thread, name) \ 2235 __tsan::ctx->thread_registry->SetThreadNameByUserId(thread, name) 2236 2237 #define COMMON_INTERCEPTOR_BLOCK_REAL(name) BLOCK_REAL(name) 2238 2239 #define COMMON_INTERCEPTOR_ON_EXIT(ctx) \ 2240 OnExit(((TsanInterceptorContext *) ctx)->thr) 2241 2242 #define COMMON_INTERCEPTOR_MUTEX_LOCK(ctx, m) \ 2243 MutexLock(((TsanInterceptorContext *)ctx)->thr, \ 2244 ((TsanInterceptorContext *)ctx)->pc, (uptr)m) 2245 2246 #define COMMON_INTERCEPTOR_MUTEX_UNLOCK(ctx, m) \ 2247 MutexUnlock(((TsanInterceptorContext *)ctx)->thr, \ 2248 ((TsanInterceptorContext *)ctx)->pc, (uptr)m) 2249 2250 #define COMMON_INTERCEPTOR_MUTEX_REPAIR(ctx, m) \ 2251 MutexRepair(((TsanInterceptorContext *)ctx)->thr, \ 2252 ((TsanInterceptorContext *)ctx)->pc, (uptr)m) 2253 2254 #define COMMON_INTERCEPTOR_HANDLE_RECVMSG(ctx, msg) \ 2255 HandleRecvmsg(((TsanInterceptorContext *)ctx)->thr, \ 2256 ((TsanInterceptorContext *)ctx)->pc, msg) 2257 2258 #include "sanitizer_common/sanitizer_common_interceptors.inc" 2259 2260 #define TSAN_SYSCALL() \ 2261 ThreadState *thr = cur_thread(); \ 2262 if (thr->ignore_interceptors) \ 2263 return; \ 2264 ScopedSyscall scoped_syscall(thr) \ 2265 /**/ 2266 2267 struct ScopedSyscall { 2268 ThreadState *thr; 2269 2270 explicit ScopedSyscall(ThreadState *thr) 2271 : thr(thr) { 2272 Initialize(thr); 2273 } 2274 2275 ~ScopedSyscall() { 2276 ProcessPendingSignals(thr); 2277 } 2278 }; 2279 2280 #if !SANITIZER_NETBSD 2281 static void syscall_access_range(uptr pc, uptr p, uptr s, bool write) { 2282 TSAN_SYSCALL(); 2283 MemoryAccessRange(thr, pc, p, s, write); 2284 } 2285 2286 static void syscall_acquire(uptr pc, uptr addr) { 2287 TSAN_SYSCALL(); 2288 Acquire(thr, pc, addr); 2289 DPrintf("syscall_acquire(%p)\n", addr); 2290 } 2291 2292 static void syscall_release(uptr pc, uptr addr) { 2293 TSAN_SYSCALL(); 2294 DPrintf("syscall_release(%p)\n", addr); 2295 Release(thr, pc, addr); 2296 } 2297 2298 static void syscall_fd_close(uptr pc, int fd) { 2299 TSAN_SYSCALL(); 2300 FdClose(thr, pc, fd); 2301 } 2302 #endif 2303 2304 static USED void syscall_fd_acquire(uptr pc, int fd) { 2305 TSAN_SYSCALL(); 2306 FdAcquire(thr, pc, fd); 2307 DPrintf("syscall_fd_acquire(%p)\n", fd); 2308 } 2309 2310 static USED void syscall_fd_release(uptr pc, int fd) { 2311 TSAN_SYSCALL(); 2312 DPrintf("syscall_fd_release(%p)\n", fd); 2313 FdRelease(thr, pc, fd); 2314 } 2315 2316 #if !SANITIZER_NETBSD 2317 static void syscall_pre_fork(uptr pc) { 2318 TSAN_SYSCALL(); 2319 ForkBefore(thr, pc); 2320 } 2321 2322 static void syscall_post_fork(uptr pc, int pid) { 2323 TSAN_SYSCALL(); 2324 if (pid == 0) { 2325 // child 2326 ForkChildAfter(thr, pc); 2327 FdOnFork(thr, pc); 2328 } else if (pid > 0) { 2329 // parent 2330 ForkParentAfter(thr, pc); 2331 } else { 2332 // error 2333 ForkParentAfter(thr, pc); 2334 } 2335 } 2336 #endif 2337 2338 #define COMMON_SYSCALL_PRE_READ_RANGE(p, s) \ 2339 syscall_access_range(GET_CALLER_PC(), (uptr)(p), (uptr)(s), false) 2340 2341 #define COMMON_SYSCALL_PRE_WRITE_RANGE(p, s) \ 2342 syscall_access_range(GET_CALLER_PC(), (uptr)(p), (uptr)(s), true) 2343 2344 #define COMMON_SYSCALL_POST_READ_RANGE(p, s) \ 2345 do { \ 2346 (void)(p); \ 2347 (void)(s); \ 2348 } while (false) 2349 2350 #define COMMON_SYSCALL_POST_WRITE_RANGE(p, s) \ 2351 do { \ 2352 (void)(p); \ 2353 (void)(s); \ 2354 } while (false) 2355 2356 #define COMMON_SYSCALL_ACQUIRE(addr) \ 2357 syscall_acquire(GET_CALLER_PC(), (uptr)(addr)) 2358 2359 #define COMMON_SYSCALL_RELEASE(addr) \ 2360 syscall_release(GET_CALLER_PC(), (uptr)(addr)) 2361 2362 #define COMMON_SYSCALL_FD_CLOSE(fd) syscall_fd_close(GET_CALLER_PC(), fd) 2363 2364 #define COMMON_SYSCALL_FD_ACQUIRE(fd) syscall_fd_acquire(GET_CALLER_PC(), fd) 2365 2366 #define COMMON_SYSCALL_FD_RELEASE(fd) syscall_fd_release(GET_CALLER_PC(), fd) 2367 2368 #define COMMON_SYSCALL_PRE_FORK() \ 2369 syscall_pre_fork(GET_CALLER_PC()) 2370 2371 #define COMMON_SYSCALL_POST_FORK(res) \ 2372 syscall_post_fork(GET_CALLER_PC(), res) 2373 2374 #include "sanitizer_common/sanitizer_common_syscalls.inc" 2375 2376 namespace __tsan { 2377 2378 static void finalize(void *arg) { 2379 ThreadState *thr = cur_thread(); 2380 int status = Finalize(thr); 2381 // Make sure the output is not lost. 2382 // Flushing all the streams here may freeze the process if a child thread is 2383 // performing file stream operations at the same time. 2384 REAL(fflush)(stdout); 2385 REAL(fflush)(stderr); 2386 if (status) 2387 REAL(_exit)(status); 2388 } 2389 2390 static void unreachable() { 2391 Report("FATAL: ThreadSanitizer: unreachable called\n"); 2392 Die(); 2393 } 2394 2395 void InitializeInterceptors() { 2396 // We need to setup it early, because functions like dlsym() can call it. 2397 REAL(memset) = internal_memset; 2398 REAL(memcpy) = internal_memcpy; 2399 REAL(memcmp) = internal_memcmp; 2400 2401 // Instruct libc malloc to consume less memory. 2402 #if !SANITIZER_FREEBSD && !SANITIZER_NETBSD 2403 mallopt(1, 0); // M_MXFAST 2404 mallopt(-3, 32*1024); // M_MMAP_THRESHOLD 2405 #endif 2406 2407 InitializeCommonInterceptors(); 2408 2409 // We can not use TSAN_INTERCEPT to get setjmp addr, 2410 // because it does &setjmp and setjmp is not present in some versions of libc. 2411 using __interception::GetRealFunctionAddress; 2412 GetRealFunctionAddress("setjmp", (uptr*)&REAL(setjmp), 0, 0); 2413 GetRealFunctionAddress("_setjmp", (uptr*)&REAL(_setjmp), 0, 0); 2414 GetRealFunctionAddress("sigsetjmp", (uptr*)&REAL(sigsetjmp), 0, 0); 2415 GetRealFunctionAddress("__sigsetjmp", (uptr*)&REAL(__sigsetjmp), 0, 0); 2416 2417 TSAN_INTERCEPT(longjmp); 2418 TSAN_INTERCEPT(siglongjmp); 2419 2420 TSAN_INTERCEPT(malloc); 2421 TSAN_INTERCEPT(__libc_memalign); 2422 TSAN_INTERCEPT(calloc); 2423 TSAN_INTERCEPT(realloc); 2424 TSAN_INTERCEPT(free); 2425 TSAN_INTERCEPT(cfree); 2426 TSAN_INTERCEPT(mmap); 2427 TSAN_MAYBE_INTERCEPT_MMAP64; 2428 TSAN_INTERCEPT(munmap); 2429 TSAN_MAYBE_INTERCEPT_MEMALIGN; 2430 TSAN_INTERCEPT(valloc); 2431 TSAN_MAYBE_INTERCEPT_PVALLOC; 2432 TSAN_INTERCEPT(posix_memalign); 2433 2434 TSAN_INTERCEPT(strlen); 2435 TSAN_INTERCEPT(memset); 2436 TSAN_INTERCEPT(memcpy); 2437 TSAN_INTERCEPT(memmove); 2438 TSAN_INTERCEPT(memcmp); 2439 TSAN_INTERCEPT(strchr); 2440 TSAN_INTERCEPT(strchrnul); 2441 TSAN_INTERCEPT(strrchr); 2442 TSAN_INTERCEPT(strcpy); // NOLINT 2443 TSAN_INTERCEPT(strncpy); 2444 TSAN_INTERCEPT(strstr); 2445 TSAN_INTERCEPT(strdup); 2446 2447 TSAN_INTERCEPT(pthread_create); 2448 TSAN_INTERCEPT(pthread_join); 2449 TSAN_INTERCEPT(pthread_detach); 2450 2451 TSAN_INTERCEPT_VER(pthread_cond_init, "GLIBC_2.3.2"); 2452 TSAN_INTERCEPT_VER(pthread_cond_signal, "GLIBC_2.3.2"); 2453 TSAN_INTERCEPT_VER(pthread_cond_broadcast, "GLIBC_2.3.2"); 2454 TSAN_INTERCEPT_VER(pthread_cond_wait, "GLIBC_2.3.2"); 2455 TSAN_INTERCEPT_VER(pthread_cond_timedwait, "GLIBC_2.3.2"); 2456 TSAN_INTERCEPT_VER(pthread_cond_destroy, "GLIBC_2.3.2"); 2457 2458 TSAN_INTERCEPT(pthread_mutex_init); 2459 TSAN_INTERCEPT(pthread_mutex_destroy); 2460 TSAN_INTERCEPT(pthread_mutex_trylock); 2461 TSAN_INTERCEPT(pthread_mutex_timedlock); 2462 2463 TSAN_INTERCEPT(pthread_spin_init); 2464 TSAN_INTERCEPT(pthread_spin_destroy); 2465 TSAN_INTERCEPT(pthread_spin_lock); 2466 TSAN_INTERCEPT(pthread_spin_trylock); 2467 TSAN_INTERCEPT(pthread_spin_unlock); 2468 2469 TSAN_INTERCEPT(pthread_rwlock_init); 2470 TSAN_INTERCEPT(pthread_rwlock_destroy); 2471 TSAN_INTERCEPT(pthread_rwlock_rdlock); 2472 TSAN_INTERCEPT(pthread_rwlock_tryrdlock); 2473 TSAN_INTERCEPT(pthread_rwlock_timedrdlock); 2474 TSAN_INTERCEPT(pthread_rwlock_wrlock); 2475 TSAN_INTERCEPT(pthread_rwlock_trywrlock); 2476 TSAN_INTERCEPT(pthread_rwlock_timedwrlock); 2477 TSAN_INTERCEPT(pthread_rwlock_unlock); 2478 2479 TSAN_INTERCEPT(pthread_barrier_init); 2480 TSAN_INTERCEPT(pthread_barrier_destroy); 2481 TSAN_INTERCEPT(pthread_barrier_wait); 2482 2483 TSAN_INTERCEPT(pthread_once); 2484 2485 TSAN_INTERCEPT(sem_init); 2486 TSAN_INTERCEPT(sem_destroy); 2487 TSAN_INTERCEPT(sem_wait); 2488 TSAN_INTERCEPT(sem_trywait); 2489 TSAN_INTERCEPT(sem_timedwait); 2490 TSAN_INTERCEPT(sem_post); 2491 TSAN_INTERCEPT(sem_getvalue); 2492 2493 TSAN_INTERCEPT(stat); 2494 TSAN_MAYBE_INTERCEPT___XSTAT; 2495 TSAN_MAYBE_INTERCEPT_STAT64; 2496 TSAN_MAYBE_INTERCEPT___XSTAT64; 2497 TSAN_INTERCEPT(lstat); 2498 TSAN_MAYBE_INTERCEPT___LXSTAT; 2499 TSAN_MAYBE_INTERCEPT_LSTAT64; 2500 TSAN_MAYBE_INTERCEPT___LXSTAT64; 2501 TSAN_INTERCEPT(fstat); 2502 TSAN_MAYBE_INTERCEPT___FXSTAT; 2503 TSAN_MAYBE_INTERCEPT_FSTAT64; 2504 TSAN_MAYBE_INTERCEPT___FXSTAT64; 2505 TSAN_INTERCEPT(open); 2506 TSAN_MAYBE_INTERCEPT_OPEN64; 2507 TSAN_INTERCEPT(creat); 2508 TSAN_MAYBE_INTERCEPT_CREAT64; 2509 TSAN_INTERCEPT(dup); 2510 TSAN_INTERCEPT(dup2); 2511 TSAN_INTERCEPT(dup3); 2512 TSAN_MAYBE_INTERCEPT_EVENTFD; 2513 TSAN_MAYBE_INTERCEPT_SIGNALFD; 2514 TSAN_MAYBE_INTERCEPT_INOTIFY_INIT; 2515 TSAN_MAYBE_INTERCEPT_INOTIFY_INIT1; 2516 TSAN_INTERCEPT(socket); 2517 TSAN_INTERCEPT(socketpair); 2518 TSAN_INTERCEPT(connect); 2519 TSAN_INTERCEPT(bind); 2520 TSAN_INTERCEPT(listen); 2521 TSAN_MAYBE_INTERCEPT_EPOLL_CREATE; 2522 TSAN_MAYBE_INTERCEPT_EPOLL_CREATE1; 2523 TSAN_INTERCEPT(close); 2524 TSAN_MAYBE_INTERCEPT___CLOSE; 2525 TSAN_MAYBE_INTERCEPT___RES_ICLOSE; 2526 TSAN_INTERCEPT(pipe); 2527 TSAN_INTERCEPT(pipe2); 2528 2529 TSAN_INTERCEPT(send); 2530 TSAN_INTERCEPT(sendmsg); 2531 TSAN_INTERCEPT(recv); 2532 2533 TSAN_INTERCEPT(unlink); 2534 TSAN_INTERCEPT(tmpfile); 2535 TSAN_MAYBE_INTERCEPT_TMPFILE64; 2536 TSAN_INTERCEPT(fread); 2537 TSAN_INTERCEPT(fwrite); 2538 TSAN_INTERCEPT(abort); 2539 TSAN_INTERCEPT(puts); 2540 TSAN_INTERCEPT(rmdir); 2541 TSAN_INTERCEPT(opendir); 2542 2543 TSAN_MAYBE_INTERCEPT_EPOLL_CTL; 2544 TSAN_MAYBE_INTERCEPT_EPOLL_WAIT; 2545 2546 TSAN_INTERCEPT(sigaction); 2547 TSAN_INTERCEPT(signal); 2548 TSAN_INTERCEPT(sigsuspend); 2549 TSAN_INTERCEPT(raise); 2550 TSAN_INTERCEPT(kill); 2551 TSAN_INTERCEPT(pthread_kill); 2552 TSAN_INTERCEPT(sleep); 2553 TSAN_INTERCEPT(usleep); 2554 TSAN_INTERCEPT(nanosleep); 2555 TSAN_INTERCEPT(gettimeofday); 2556 TSAN_INTERCEPT(getaddrinfo); 2557 2558 TSAN_INTERCEPT(fork); 2559 TSAN_INTERCEPT(vfork); 2560 TSAN_INTERCEPT(on_exit); 2561 TSAN_INTERCEPT(__cxa_atexit); 2562 TSAN_INTERCEPT(_exit); 2563 2564 // Need to setup it, because interceptors check that the function is resolved. 2565 // But atexit is emitted directly into the module, so can't be resolved. 2566 REAL(atexit) = (int(*)(void(*)()))unreachable; 2567 if (REAL(__cxa_atexit)(&finalize, 0, 0)) { 2568 Printf("ThreadSanitizer: failed to setup atexit callback\n"); 2569 Die(); 2570 } 2571 2572 if (pthread_key_create(&g_thread_finalize_key, &thread_finalize)) { 2573 Printf("ThreadSanitizer: failed to create thread key\n"); 2574 Die(); 2575 } 2576 2577 FdInit(); 2578 } 2579 2580 void *internal_start_thread(void(*func)(void *arg), void *arg) { 2581 // Start the thread with signals blocked, otherwise it can steal user signals. 2582 __sanitizer_sigset_t set, old; 2583 internal_sigfillset(&set); 2584 internal_sigprocmask(SIG_SETMASK, &set, &old); 2585 void *th; 2586 REAL(pthread_create)(&th, 0, (void*(*)(void *arg))func, arg); 2587 internal_sigprocmask(SIG_SETMASK, &old, 0); 2588 return th; 2589 } 2590 2591 void internal_join_thread(void *th) { 2592 REAL(pthread_join)(th, 0); 2593 } 2594 2595 } // namespace __tsan 2596