1 //===-- tsan_platform_linux.cpp -------------------------------------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file is a part of ThreadSanitizer (TSan), a race detector. 10 // 11 // Linux- and BSD-specific code. 12 //===----------------------------------------------------------------------===// 13 14 #include "sanitizer_common/sanitizer_platform.h" 15 #if SANITIZER_LINUX || SANITIZER_FREEBSD || SANITIZER_NETBSD 16 17 #include "sanitizer_common/sanitizer_common.h" 18 #include "sanitizer_common/sanitizer_libc.h" 19 #include "sanitizer_common/sanitizer_linux.h" 20 #include "sanitizer_common/sanitizer_platform_limits_netbsd.h" 21 #include "sanitizer_common/sanitizer_platform_limits_posix.h" 22 #include "sanitizer_common/sanitizer_posix.h" 23 #include "sanitizer_common/sanitizer_procmaps.h" 24 #include "sanitizer_common/sanitizer_stackdepot.h" 25 #include "sanitizer_common/sanitizer_stoptheworld.h" 26 #include "tsan_flags.h" 27 #include "tsan_platform.h" 28 #include "tsan_rtl.h" 29 30 #include <fcntl.h> 31 #include <pthread.h> 32 #include <signal.h> 33 #include <stdio.h> 34 #include <stdlib.h> 35 #include <string.h> 36 #include <stdarg.h> 37 #include <sys/mman.h> 38 #if SANITIZER_LINUX 39 #include <sys/personality.h> 40 #include <setjmp.h> 41 #endif 42 #include <sys/syscall.h> 43 #include <sys/socket.h> 44 #include <sys/time.h> 45 #include <sys/types.h> 46 #include <sys/resource.h> 47 #include <sys/stat.h> 48 #include <unistd.h> 49 #include <sched.h> 50 #include <dlfcn.h> 51 #if SANITIZER_LINUX 52 #define __need_res_state 53 #include <resolv.h> 54 #endif 55 56 #ifdef sa_handler 57 # undef sa_handler 58 #endif 59 60 #ifdef sa_sigaction 61 # undef sa_sigaction 62 #endif 63 64 #if SANITIZER_FREEBSD 65 extern "C" void *__libc_stack_end; 66 void *__libc_stack_end = 0; 67 #endif 68 69 #if SANITIZER_LINUX && defined(__aarch64__) && !SANITIZER_GO 70 # define INIT_LONGJMP_XOR_KEY 1 71 #else 72 # define INIT_LONGJMP_XOR_KEY 0 73 #endif 74 75 #if INIT_LONGJMP_XOR_KEY 76 #include "interception/interception.h" 77 // Must be declared outside of other namespaces. 78 DECLARE_REAL(int, _setjmp, void *env) 79 #endif 80 81 namespace __tsan { 82 83 #if INIT_LONGJMP_XOR_KEY 84 static void InitializeLongjmpXorKey(); 85 static uptr longjmp_xor_key; 86 #endif 87 88 #ifdef TSAN_RUNTIME_VMA 89 // Runtime detected VMA size. 90 uptr vmaSize; 91 #endif 92 93 enum { 94 MemTotal = 0, 95 MemShadow = 1, 96 MemMeta = 2, 97 MemFile = 3, 98 MemMmap = 4, 99 MemTrace = 5, 100 MemHeap = 6, 101 MemOther = 7, 102 MemCount = 8, 103 }; 104 105 void FillProfileCallback(uptr p, uptr rss, bool file, 106 uptr *mem, uptr stats_size) { 107 mem[MemTotal] += rss; 108 if (p >= ShadowBeg() && p < ShadowEnd()) 109 mem[MemShadow] += rss; 110 else if (p >= MetaShadowBeg() && p < MetaShadowEnd()) 111 mem[MemMeta] += rss; 112 #if !SANITIZER_GO 113 else if (p >= HeapMemBeg() && p < HeapMemEnd()) 114 mem[MemHeap] += rss; 115 else if (p >= LoAppMemBeg() && p < LoAppMemEnd()) 116 mem[file ? MemFile : MemMmap] += rss; 117 else if (p >= HiAppMemBeg() && p < HiAppMemEnd()) 118 mem[file ? MemFile : MemMmap] += rss; 119 #else 120 else if (p >= AppMemBeg() && p < AppMemEnd()) 121 mem[file ? MemFile : MemMmap] += rss; 122 #endif 123 else if (p >= TraceMemBeg() && p < TraceMemEnd()) 124 mem[MemTrace] += rss; 125 else 126 mem[MemOther] += rss; 127 } 128 129 void WriteMemoryProfile(char *buf, uptr buf_size, uptr nthread, uptr nlive) { 130 uptr mem[MemCount]; 131 internal_memset(mem, 0, sizeof(mem[0]) * MemCount); 132 __sanitizer::GetMemoryProfile(FillProfileCallback, mem, 7); 133 StackDepotStats *stacks = StackDepotGetStats(); 134 internal_snprintf(buf, buf_size, 135 "RSS %zd MB: shadow:%zd meta:%zd file:%zd mmap:%zd" 136 " trace:%zd heap:%zd other:%zd stacks=%zd[%zd] nthr=%zd/%zd\n", 137 mem[MemTotal] >> 20, mem[MemShadow] >> 20, mem[MemMeta] >> 20, 138 mem[MemFile] >> 20, mem[MemMmap] >> 20, mem[MemTrace] >> 20, 139 mem[MemHeap] >> 20, mem[MemOther] >> 20, 140 stacks->allocated >> 20, stacks->n_uniq_ids, 141 nlive, nthread); 142 } 143 144 #if SANITIZER_LINUX 145 void FlushShadowMemoryCallback( 146 const SuspendedThreadsList &suspended_threads_list, 147 void *argument) { 148 ReleaseMemoryPagesToOS(ShadowBeg(), ShadowEnd()); 149 } 150 #endif 151 152 void FlushShadowMemory() { 153 #if SANITIZER_LINUX 154 StopTheWorld(FlushShadowMemoryCallback, 0); 155 #endif 156 } 157 158 #if !SANITIZER_GO 159 // Mark shadow for .rodata sections with the special kShadowRodata marker. 160 // Accesses to .rodata can't race, so this saves time, memory and trace space. 161 static void MapRodata() { 162 // First create temp file. 163 const char *tmpdir = GetEnv("TMPDIR"); 164 if (tmpdir == 0) 165 tmpdir = GetEnv("TEST_TMPDIR"); 166 #ifdef P_tmpdir 167 if (tmpdir == 0) 168 tmpdir = P_tmpdir; 169 #endif 170 if (tmpdir == 0) 171 return; 172 char name[256]; 173 internal_snprintf(name, sizeof(name), "%s/tsan.rodata.%d", 174 tmpdir, (int)internal_getpid()); 175 uptr openrv = internal_open(name, O_RDWR | O_CREAT | O_EXCL, 0600); 176 if (internal_iserror(openrv)) 177 return; 178 internal_unlink(name); // Unlink it now, so that we can reuse the buffer. 179 fd_t fd = openrv; 180 // Fill the file with kShadowRodata. 181 const uptr kMarkerSize = 512 * 1024 / sizeof(u64); 182 InternalMmapVector<u64> marker(kMarkerSize); 183 // volatile to prevent insertion of memset 184 for (volatile u64 *p = marker.data(); p < marker.data() + kMarkerSize; p++) 185 *p = kShadowRodata; 186 internal_write(fd, marker.data(), marker.size() * sizeof(u64)); 187 // Map the file into memory. 188 uptr page = internal_mmap(0, GetPageSizeCached(), PROT_READ | PROT_WRITE, 189 MAP_PRIVATE | MAP_ANONYMOUS, fd, 0); 190 if (internal_iserror(page)) { 191 internal_close(fd); 192 return; 193 } 194 // Map the file into shadow of .rodata sections. 195 MemoryMappingLayout proc_maps(/*cache_enabled*/true); 196 // Reusing the buffer 'name'. 197 MemoryMappedSegment segment(name, ARRAY_SIZE(name)); 198 while (proc_maps.Next(&segment)) { 199 if (segment.filename[0] != 0 && segment.filename[0] != '[' && 200 segment.IsReadable() && segment.IsExecutable() && 201 !segment.IsWritable() && IsAppMem(segment.start)) { 202 // Assume it's .rodata 203 char *shadow_start = (char *)MemToShadow(segment.start); 204 char *shadow_end = (char *)MemToShadow(segment.end); 205 for (char *p = shadow_start; p < shadow_end; 206 p += marker.size() * sizeof(u64)) { 207 internal_mmap(p, Min<uptr>(marker.size() * sizeof(u64), shadow_end - p), 208 PROT_READ, MAP_PRIVATE | MAP_FIXED, fd, 0); 209 } 210 } 211 } 212 internal_close(fd); 213 } 214 215 void InitializeShadowMemoryPlatform() { 216 MapRodata(); 217 } 218 219 #endif // #if !SANITIZER_GO 220 221 void InitializePlatformEarly() { 222 #ifdef TSAN_RUNTIME_VMA 223 vmaSize = 224 (MostSignificantSetBitIndex(GET_CURRENT_FRAME()) + 1); 225 #if defined(__aarch64__) 226 # if !SANITIZER_GO 227 if (vmaSize != 39 && vmaSize != 42 && vmaSize != 48) { 228 Printf("FATAL: ThreadSanitizer: unsupported VMA range\n"); 229 Printf("FATAL: Found %zd - Supported 39, 42 and 48\n", vmaSize); 230 Die(); 231 } 232 #else 233 if (vmaSize != 48) { 234 Printf("FATAL: ThreadSanitizer: unsupported VMA range\n"); 235 Printf("FATAL: Found %zd - Supported 48\n", vmaSize); 236 Die(); 237 } 238 #endif 239 #elif defined(__powerpc64__) 240 # if !SANITIZER_GO 241 if (vmaSize != 44 && vmaSize != 46 && vmaSize != 47) { 242 Printf("FATAL: ThreadSanitizer: unsupported VMA range\n"); 243 Printf("FATAL: Found %zd - Supported 44, 46, and 47\n", vmaSize); 244 Die(); 245 } 246 # else 247 if (vmaSize != 46 && vmaSize != 47) { 248 Printf("FATAL: ThreadSanitizer: unsupported VMA range\n"); 249 Printf("FATAL: Found %zd - Supported 46, and 47\n", vmaSize); 250 Die(); 251 } 252 # endif 253 #endif 254 #endif 255 } 256 257 void InitializePlatform() { 258 DisableCoreDumperIfNecessary(); 259 260 // Go maps shadow memory lazily and works fine with limited address space. 261 // Unlimited stack is not a problem as well, because the executable 262 // is not compiled with -pie. 263 #if !SANITIZER_GO 264 { 265 bool reexec = false; 266 // TSan doesn't play well with unlimited stack size (as stack 267 // overlaps with shadow memory). If we detect unlimited stack size, 268 // we re-exec the program with limited stack size as a best effort. 269 if (StackSizeIsUnlimited()) { 270 const uptr kMaxStackSize = 32 * 1024 * 1024; 271 VReport(1, "Program is run with unlimited stack size, which wouldn't " 272 "work with ThreadSanitizer.\n" 273 "Re-execing with stack size limited to %zd bytes.\n", 274 kMaxStackSize); 275 SetStackSizeLimitInBytes(kMaxStackSize); 276 reexec = true; 277 } 278 279 if (!AddressSpaceIsUnlimited()) { 280 Report("WARNING: Program is run with limited virtual address space," 281 " which wouldn't work with ThreadSanitizer.\n"); 282 Report("Re-execing with unlimited virtual address space.\n"); 283 SetAddressSpaceUnlimited(); 284 reexec = true; 285 } 286 #if SANITIZER_LINUX && defined(__aarch64__) 287 // After patch "arm64: mm: support ARCH_MMAP_RND_BITS." is introduced in 288 // linux kernel, the random gap between stack and mapped area is increased 289 // from 128M to 36G on 39-bit aarch64. As it is almost impossible to cover 290 // this big range, we should disable randomized virtual space on aarch64. 291 int old_personality = personality(0xffffffff); 292 if (old_personality != -1 && (old_personality & ADDR_NO_RANDOMIZE) == 0) { 293 VReport(1, "WARNING: Program is run with randomized virtual address " 294 "space, which wouldn't work with ThreadSanitizer.\n" 295 "Re-execing with fixed virtual address space.\n"); 296 CHECK_NE(personality(old_personality | ADDR_NO_RANDOMIZE), -1); 297 reexec = true; 298 } 299 // Initialize the xor key used in {sig}{set,long}jump. 300 InitializeLongjmpXorKey(); 301 #endif 302 if (reexec) 303 ReExec(); 304 } 305 306 CheckAndProtect(); 307 InitTlsSize(); 308 #endif // !SANITIZER_GO 309 } 310 311 #if !SANITIZER_GO 312 // Extract file descriptors passed to glibc internal __res_iclose function. 313 // This is required to properly "close" the fds, because we do not see internal 314 // closes within glibc. The code is a pure hack. 315 int ExtractResolvFDs(void *state, int *fds, int nfd) { 316 #if SANITIZER_LINUX && !SANITIZER_ANDROID 317 int cnt = 0; 318 struct __res_state *statp = (struct __res_state*)state; 319 for (int i = 0; i < MAXNS && cnt < nfd; i++) { 320 if (statp->_u._ext.nsaddrs[i] && statp->_u._ext.nssocks[i] != -1) 321 fds[cnt++] = statp->_u._ext.nssocks[i]; 322 } 323 return cnt; 324 #else 325 return 0; 326 #endif 327 } 328 329 // Extract file descriptors passed via UNIX domain sockets. 330 // This is requried to properly handle "open" of these fds. 331 // see 'man recvmsg' and 'man 3 cmsg'. 332 int ExtractRecvmsgFDs(void *msgp, int *fds, int nfd) { 333 int res = 0; 334 msghdr *msg = (msghdr*)msgp; 335 struct cmsghdr *cmsg = CMSG_FIRSTHDR(msg); 336 for (; cmsg; cmsg = CMSG_NXTHDR(msg, cmsg)) { 337 if (cmsg->cmsg_level != SOL_SOCKET || cmsg->cmsg_type != SCM_RIGHTS) 338 continue; 339 int n = (cmsg->cmsg_len - CMSG_LEN(0)) / sizeof(fds[0]); 340 for (int i = 0; i < n; i++) { 341 fds[res++] = ((int*)CMSG_DATA(cmsg))[i]; 342 if (res == nfd) 343 return res; 344 } 345 } 346 return res; 347 } 348 349 // Reverse operation of libc stack pointer mangling 350 static uptr UnmangleLongJmpSp(uptr mangled_sp) { 351 #if defined(__x86_64__) 352 # if SANITIZER_LINUX 353 // Reverse of: 354 // xor %fs:0x30, %rsi 355 // rol $0x11, %rsi 356 uptr sp; 357 asm("ror $0x11, %0 \n" 358 "xor %%fs:0x30, %0 \n" 359 : "=r" (sp) 360 : "0" (mangled_sp)); 361 return sp; 362 # else 363 return mangled_sp; 364 # endif 365 #elif defined(__aarch64__) 366 # if SANITIZER_LINUX 367 return mangled_sp ^ longjmp_xor_key; 368 # else 369 return mangled_sp; 370 # endif 371 #elif defined(__powerpc64__) 372 // Reverse of: 373 // ld r4, -28696(r13) 374 // xor r4, r3, r4 375 uptr xor_key; 376 asm("ld %0, -28696(%%r13)" : "=r" (xor_key)); 377 return mangled_sp ^ xor_key; 378 #elif defined(__mips__) 379 return mangled_sp; 380 #else 381 #error "Unknown platform" 382 #endif 383 } 384 385 #if SANITIZER_NETBSD 386 # ifdef __x86_64__ 387 # define LONG_JMP_SP_ENV_SLOT 6 388 # else 389 # error unsupported 390 # endif 391 #elif defined(__powerpc__) 392 # define LONG_JMP_SP_ENV_SLOT 0 393 #elif SANITIZER_FREEBSD 394 # define LONG_JMP_SP_ENV_SLOT 2 395 #elif SANITIZER_LINUX 396 # ifdef __aarch64__ 397 # define LONG_JMP_SP_ENV_SLOT 13 398 # elif defined(__mips64) 399 # define LONG_JMP_SP_ENV_SLOT 1 400 # else 401 # define LONG_JMP_SP_ENV_SLOT 6 402 # endif 403 #endif 404 405 uptr ExtractLongJmpSp(uptr *env) { 406 uptr mangled_sp = env[LONG_JMP_SP_ENV_SLOT]; 407 return UnmangleLongJmpSp(mangled_sp); 408 } 409 410 #if INIT_LONGJMP_XOR_KEY 411 // GLIBC mangles the function pointers in jmp_buf (used in {set,long}*jmp 412 // functions) by XORing them with a random key. For AArch64 it is a global 413 // variable rather than a TCB one (as for x86_64/powerpc). We obtain the key by 414 // issuing a setjmp and XORing the SP pointer values to derive the key. 415 static void InitializeLongjmpXorKey() { 416 // 1. Call REAL(setjmp), which stores the mangled SP in env. 417 jmp_buf env; 418 REAL(_setjmp)(env); 419 420 // 2. Retrieve vanilla/mangled SP. 421 uptr sp; 422 asm("mov %0, sp" : "=r" (sp)); 423 uptr mangled_sp = ((uptr *)&env)[LONG_JMP_SP_ENV_SLOT]; 424 425 // 3. xor SPs to obtain key. 426 longjmp_xor_key = mangled_sp ^ sp; 427 } 428 #endif 429 430 void ImitateTlsWrite(ThreadState *thr, uptr tls_addr, uptr tls_size) { 431 // Check that the thr object is in tls; 432 const uptr thr_beg = (uptr)thr; 433 const uptr thr_end = (uptr)thr + sizeof(*thr); 434 CHECK_GE(thr_beg, tls_addr); 435 CHECK_LE(thr_beg, tls_addr + tls_size); 436 CHECK_GE(thr_end, tls_addr); 437 CHECK_LE(thr_end, tls_addr + tls_size); 438 // Since the thr object is huge, skip it. 439 MemoryRangeImitateWrite(thr, /*pc=*/2, tls_addr, thr_beg - tls_addr); 440 MemoryRangeImitateWrite(thr, /*pc=*/2, thr_end, 441 tls_addr + tls_size - thr_end); 442 } 443 444 // Note: this function runs with async signals enabled, 445 // so it must not touch any tsan state. 446 int call_pthread_cancel_with_cleanup(int (*fn)(void *arg), 447 void (*cleanup)(void *arg), void *arg) { 448 // pthread_cleanup_push/pop are hardcore macros mess. 449 // We can't intercept nor call them w/o including pthread.h. 450 int res; 451 pthread_cleanup_push(cleanup, arg); 452 res = fn(arg); 453 pthread_cleanup_pop(0); 454 return res; 455 } 456 #endif // !SANITIZER_GO 457 458 #if !SANITIZER_GO 459 void ReplaceSystemMalloc() { } 460 #endif 461 462 #if !SANITIZER_GO 463 #if SANITIZER_ANDROID 464 // On Android, one thread can call intercepted functions after 465 // DestroyThreadState(), so add a fake thread state for "dead" threads. 466 static ThreadState *dead_thread_state = nullptr; 467 468 ThreadState *cur_thread() { 469 ThreadState* thr = reinterpret_cast<ThreadState*>(*get_android_tls_ptr()); 470 if (thr == nullptr) { 471 __sanitizer_sigset_t emptyset; 472 internal_sigfillset(&emptyset); 473 __sanitizer_sigset_t oldset; 474 CHECK_EQ(0, internal_sigprocmask(SIG_SETMASK, &emptyset, &oldset)); 475 thr = reinterpret_cast<ThreadState*>(*get_android_tls_ptr()); 476 if (thr == nullptr) { 477 thr = reinterpret_cast<ThreadState*>(MmapOrDie(sizeof(ThreadState), 478 "ThreadState")); 479 *get_android_tls_ptr() = reinterpret_cast<uptr>(thr); 480 if (dead_thread_state == nullptr) { 481 dead_thread_state = reinterpret_cast<ThreadState*>( 482 MmapOrDie(sizeof(ThreadState), "ThreadState")); 483 dead_thread_state->fast_state.SetIgnoreBit(); 484 dead_thread_state->ignore_interceptors = 1; 485 dead_thread_state->is_dead = true; 486 *const_cast<int*>(&dead_thread_state->tid) = -1; 487 CHECK_EQ(0, internal_mprotect(dead_thread_state, sizeof(ThreadState), 488 PROT_READ)); 489 } 490 } 491 CHECK_EQ(0, internal_sigprocmask(SIG_SETMASK, &oldset, nullptr)); 492 } 493 return thr; 494 } 495 496 void set_cur_thread(ThreadState *thr) { 497 *get_android_tls_ptr() = reinterpret_cast<uptr>(thr); 498 } 499 500 void cur_thread_finalize() { 501 __sanitizer_sigset_t emptyset; 502 internal_sigfillset(&emptyset); 503 __sanitizer_sigset_t oldset; 504 CHECK_EQ(0, internal_sigprocmask(SIG_SETMASK, &emptyset, &oldset)); 505 ThreadState* thr = reinterpret_cast<ThreadState*>(*get_android_tls_ptr()); 506 if (thr != dead_thread_state) { 507 *get_android_tls_ptr() = reinterpret_cast<uptr>(dead_thread_state); 508 UnmapOrDie(thr, sizeof(ThreadState)); 509 } 510 CHECK_EQ(0, internal_sigprocmask(SIG_SETMASK, &oldset, nullptr)); 511 } 512 #endif // SANITIZER_ANDROID 513 #endif // if !SANITIZER_GO 514 515 } // namespace __tsan 516 517 #endif // SANITIZER_LINUX || SANITIZER_FREEBSD || SANITIZER_NETBSD 518