1 //===-- ThreadSanitizer.cpp - race detector -------------------------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file is a part of ThreadSanitizer, a race detector. 10 // 11 // The tool is under development, for the details about previous versions see 12 // http://code.google.com/p/data-race-test 13 // 14 // The instrumentation phase is quite simple: 15 // - Insert calls to run-time library before every memory access. 16 // - Optimizations may apply to avoid instrumenting some of the accesses. 17 // - Insert calls at function entry/exit. 18 // The rest is handled by the run-time library. 19 //===----------------------------------------------------------------------===// 20 21 #include "llvm/Transforms/Instrumentation/ThreadSanitizer.h" 22 #include "llvm/ADT/DenseMap.h" 23 #include "llvm/ADT/SmallString.h" 24 #include "llvm/ADT/SmallVector.h" 25 #include "llvm/ADT/Statistic.h" 26 #include "llvm/ADT/StringExtras.h" 27 #include "llvm/Analysis/CaptureTracking.h" 28 #include "llvm/Analysis/TargetLibraryInfo.h" 29 #include "llvm/Analysis/ValueTracking.h" 30 #include "llvm/IR/DataLayout.h" 31 #include "llvm/IR/Function.h" 32 #include "llvm/IR/IRBuilder.h" 33 #include "llvm/IR/Instructions.h" 34 #include "llvm/IR/IntrinsicInst.h" 35 #include "llvm/IR/Intrinsics.h" 36 #include "llvm/IR/LLVMContext.h" 37 #include "llvm/IR/Metadata.h" 38 #include "llvm/IR/Module.h" 39 #include "llvm/IR/Type.h" 40 #include "llvm/ProfileData/InstrProf.h" 41 #include "llvm/Support/CommandLine.h" 42 #include "llvm/Support/Debug.h" 43 #include "llvm/Support/raw_ostream.h" 44 #include "llvm/Transforms/Utils/EscapeEnumerator.h" 45 #include "llvm/Transforms/Utils/Instrumentation.h" 46 #include "llvm/Transforms/Utils/Local.h" 47 #include "llvm/Transforms/Utils/ModuleUtils.h" 48 49 using namespace llvm; 50 51 #define DEBUG_TYPE "tsan" 52 53 static cl::opt<bool> ClInstrumentMemoryAccesses( 54 "tsan-instrument-memory-accesses", cl::init(true), 55 cl::desc("Instrument memory accesses"), cl::Hidden); 56 static cl::opt<bool> 57 ClInstrumentFuncEntryExit("tsan-instrument-func-entry-exit", cl::init(true), 58 cl::desc("Instrument function entry and exit"), 59 cl::Hidden); 60 static cl::opt<bool> ClHandleCxxExceptions( 61 "tsan-handle-cxx-exceptions", cl::init(true), 62 cl::desc("Handle C++ exceptions (insert cleanup blocks for unwinding)"), 63 cl::Hidden); 64 static cl::opt<bool> ClInstrumentAtomics("tsan-instrument-atomics", 65 cl::init(true), 66 cl::desc("Instrument atomics"), 67 cl::Hidden); 68 static cl::opt<bool> ClInstrumentMemIntrinsics( 69 "tsan-instrument-memintrinsics", cl::init(true), 70 cl::desc("Instrument memintrinsics (memset/memcpy/memmove)"), cl::Hidden); 71 static cl::opt<bool> ClDistinguishVolatile( 72 "tsan-distinguish-volatile", cl::init(false), 73 cl::desc("Emit special instrumentation for accesses to volatiles"), 74 cl::Hidden); 75 static cl::opt<bool> ClInstrumentReadBeforeWrite( 76 "tsan-instrument-read-before-write", cl::init(false), 77 cl::desc("Do not eliminate read instrumentation for read-before-writes"), 78 cl::Hidden); 79 static cl::opt<bool> ClCompoundReadBeforeWrite( 80 "tsan-compound-read-before-write", cl::init(false), 81 cl::desc("Emit special compound instrumentation for reads-before-writes"), 82 cl::Hidden); 83 84 STATISTIC(NumInstrumentedReads, "Number of instrumented reads"); 85 STATISTIC(NumInstrumentedWrites, "Number of instrumented writes"); 86 STATISTIC(NumOmittedReadsBeforeWrite, 87 "Number of reads ignored due to following writes"); 88 STATISTIC(NumAccessesWithBadSize, "Number of accesses with bad size"); 89 STATISTIC(NumInstrumentedVtableWrites, "Number of vtable ptr writes"); 90 STATISTIC(NumInstrumentedVtableReads, "Number of vtable ptr reads"); 91 STATISTIC(NumOmittedReadsFromConstantGlobals, 92 "Number of reads from constant globals"); 93 STATISTIC(NumOmittedReadsFromVtable, "Number of vtable reads"); 94 STATISTIC(NumOmittedNonCaptured, "Number of accesses ignored due to capturing"); 95 96 const char kTsanModuleCtorName[] = "tsan.module_ctor"; 97 const char kTsanInitName[] = "__tsan_init"; 98 99 namespace { 100 101 /// ThreadSanitizer: instrument the code in module to find races. 102 /// 103 /// Instantiating ThreadSanitizer inserts the tsan runtime library API function 104 /// declarations into the module if they don't exist already. Instantiating 105 /// ensures the __tsan_init function is in the list of global constructors for 106 /// the module. 107 struct ThreadSanitizer { 108 ThreadSanitizer() { 109 // Check options and warn user. 110 if (ClInstrumentReadBeforeWrite && ClCompoundReadBeforeWrite) { 111 errs() 112 << "warning: Option -tsan-compound-read-before-write has no effect " 113 "when -tsan-instrument-read-before-write is set.\n"; 114 } 115 } 116 117 bool sanitizeFunction(Function &F, const TargetLibraryInfo &TLI); 118 119 private: 120 // Internal Instruction wrapper that contains more information about the 121 // Instruction from prior analysis. 122 struct InstructionInfo { 123 // Instrumentation emitted for this instruction is for a compounded set of 124 // read and write operations in the same basic block. 125 static constexpr unsigned kCompoundRW = (1U << 0); 126 127 explicit InstructionInfo(Instruction *Inst) : Inst(Inst) {} 128 129 Instruction *Inst; 130 unsigned Flags = 0; 131 }; 132 133 void initialize(Module &M, const TargetLibraryInfo &TLI); 134 bool instrumentLoadOrStore(const InstructionInfo &II, const DataLayout &DL); 135 bool instrumentAtomic(Instruction *I, const DataLayout &DL); 136 bool instrumentMemIntrinsic(Instruction *I); 137 void chooseInstructionsToInstrument(SmallVectorImpl<Instruction *> &Local, 138 SmallVectorImpl<InstructionInfo> &All, 139 const DataLayout &DL); 140 bool addrPointsToConstantData(Value *Addr); 141 int getMemoryAccessFuncIndex(Type *OrigTy, Value *Addr, const DataLayout &DL); 142 void InsertRuntimeIgnores(Function &F); 143 144 Type *IntptrTy; 145 FunctionCallee TsanFuncEntry; 146 FunctionCallee TsanFuncExit; 147 FunctionCallee TsanIgnoreBegin; 148 FunctionCallee TsanIgnoreEnd; 149 // Accesses sizes are powers of two: 1, 2, 4, 8, 16. 150 static const size_t kNumberOfAccessSizes = 5; 151 FunctionCallee TsanRead[kNumberOfAccessSizes]; 152 FunctionCallee TsanWrite[kNumberOfAccessSizes]; 153 FunctionCallee TsanUnalignedRead[kNumberOfAccessSizes]; 154 FunctionCallee TsanUnalignedWrite[kNumberOfAccessSizes]; 155 FunctionCallee TsanVolatileRead[kNumberOfAccessSizes]; 156 FunctionCallee TsanVolatileWrite[kNumberOfAccessSizes]; 157 FunctionCallee TsanUnalignedVolatileRead[kNumberOfAccessSizes]; 158 FunctionCallee TsanUnalignedVolatileWrite[kNumberOfAccessSizes]; 159 FunctionCallee TsanCompoundRW[kNumberOfAccessSizes]; 160 FunctionCallee TsanUnalignedCompoundRW[kNumberOfAccessSizes]; 161 FunctionCallee TsanAtomicLoad[kNumberOfAccessSizes]; 162 FunctionCallee TsanAtomicStore[kNumberOfAccessSizes]; 163 FunctionCallee TsanAtomicRMW[AtomicRMWInst::LAST_BINOP + 1] 164 [kNumberOfAccessSizes]; 165 FunctionCallee TsanAtomicCAS[kNumberOfAccessSizes]; 166 FunctionCallee TsanAtomicThreadFence; 167 FunctionCallee TsanAtomicSignalFence; 168 FunctionCallee TsanVptrUpdate; 169 FunctionCallee TsanVptrLoad; 170 FunctionCallee MemmoveFn, MemcpyFn, MemsetFn; 171 }; 172 173 void insertModuleCtor(Module &M) { 174 getOrCreateSanitizerCtorAndInitFunctions( 175 M, kTsanModuleCtorName, kTsanInitName, /*InitArgTypes=*/{}, 176 /*InitArgs=*/{}, 177 // This callback is invoked when the functions are created the first 178 // time. Hook them into the global ctors list in that case: 179 [&](Function *Ctor, FunctionCallee) { appendToGlobalCtors(M, Ctor, 0); }); 180 } 181 } // namespace 182 183 PreservedAnalyses ThreadSanitizerPass::run(Function &F, 184 FunctionAnalysisManager &FAM) { 185 ThreadSanitizer TSan; 186 if (TSan.sanitizeFunction(F, FAM.getResult<TargetLibraryAnalysis>(F))) 187 return PreservedAnalyses::none(); 188 return PreservedAnalyses::all(); 189 } 190 191 PreservedAnalyses ModuleThreadSanitizerPass::run(Module &M, 192 ModuleAnalysisManager &MAM) { 193 // Return early if nosanitize_thread module flag is present for the module. 194 if (checkIfAlreadyInstrumented(M, "nosanitize_thread")) 195 return PreservedAnalyses::all(); 196 insertModuleCtor(M); 197 return PreservedAnalyses::none(); 198 } 199 void ThreadSanitizer::initialize(Module &M, const TargetLibraryInfo &TLI) { 200 const DataLayout &DL = M.getDataLayout(); 201 LLVMContext &Ctx = M.getContext(); 202 IntptrTy = DL.getIntPtrType(Ctx); 203 204 IRBuilder<> IRB(Ctx); 205 AttributeList Attr; 206 Attr = Attr.addFnAttribute(Ctx, Attribute::NoUnwind); 207 // Initialize the callbacks. 208 TsanFuncEntry = M.getOrInsertFunction("__tsan_func_entry", Attr, 209 IRB.getVoidTy(), IRB.getPtrTy()); 210 TsanFuncExit = 211 M.getOrInsertFunction("__tsan_func_exit", Attr, IRB.getVoidTy()); 212 TsanIgnoreBegin = M.getOrInsertFunction("__tsan_ignore_thread_begin", Attr, 213 IRB.getVoidTy()); 214 TsanIgnoreEnd = 215 M.getOrInsertFunction("__tsan_ignore_thread_end", Attr, IRB.getVoidTy()); 216 IntegerType *OrdTy = IRB.getInt32Ty(); 217 for (size_t i = 0; i < kNumberOfAccessSizes; ++i) { 218 const unsigned ByteSize = 1U << i; 219 const unsigned BitSize = ByteSize * 8; 220 std::string ByteSizeStr = utostr(ByteSize); 221 std::string BitSizeStr = utostr(BitSize); 222 SmallString<32> ReadName("__tsan_read" + ByteSizeStr); 223 TsanRead[i] = M.getOrInsertFunction(ReadName, Attr, IRB.getVoidTy(), 224 IRB.getPtrTy()); 225 226 SmallString<32> WriteName("__tsan_write" + ByteSizeStr); 227 TsanWrite[i] = M.getOrInsertFunction(WriteName, Attr, IRB.getVoidTy(), 228 IRB.getPtrTy()); 229 230 SmallString<64> UnalignedReadName("__tsan_unaligned_read" + ByteSizeStr); 231 TsanUnalignedRead[i] = M.getOrInsertFunction( 232 UnalignedReadName, Attr, IRB.getVoidTy(), IRB.getPtrTy()); 233 234 SmallString<64> UnalignedWriteName("__tsan_unaligned_write" + ByteSizeStr); 235 TsanUnalignedWrite[i] = M.getOrInsertFunction( 236 UnalignedWriteName, Attr, IRB.getVoidTy(), IRB.getPtrTy()); 237 238 SmallString<64> VolatileReadName("__tsan_volatile_read" + ByteSizeStr); 239 TsanVolatileRead[i] = M.getOrInsertFunction( 240 VolatileReadName, Attr, IRB.getVoidTy(), IRB.getPtrTy()); 241 242 SmallString<64> VolatileWriteName("__tsan_volatile_write" + ByteSizeStr); 243 TsanVolatileWrite[i] = M.getOrInsertFunction( 244 VolatileWriteName, Attr, IRB.getVoidTy(), IRB.getPtrTy()); 245 246 SmallString<64> UnalignedVolatileReadName("__tsan_unaligned_volatile_read" + 247 ByteSizeStr); 248 TsanUnalignedVolatileRead[i] = M.getOrInsertFunction( 249 UnalignedVolatileReadName, Attr, IRB.getVoidTy(), IRB.getPtrTy()); 250 251 SmallString<64> UnalignedVolatileWriteName( 252 "__tsan_unaligned_volatile_write" + ByteSizeStr); 253 TsanUnalignedVolatileWrite[i] = M.getOrInsertFunction( 254 UnalignedVolatileWriteName, Attr, IRB.getVoidTy(), IRB.getPtrTy()); 255 256 SmallString<64> CompoundRWName("__tsan_read_write" + ByteSizeStr); 257 TsanCompoundRW[i] = M.getOrInsertFunction( 258 CompoundRWName, Attr, IRB.getVoidTy(), IRB.getPtrTy()); 259 260 SmallString<64> UnalignedCompoundRWName("__tsan_unaligned_read_write" + 261 ByteSizeStr); 262 TsanUnalignedCompoundRW[i] = M.getOrInsertFunction( 263 UnalignedCompoundRWName, Attr, IRB.getVoidTy(), IRB.getPtrTy()); 264 265 Type *Ty = Type::getIntNTy(Ctx, BitSize); 266 Type *PtrTy = PointerType::get(Ctx, 0); 267 SmallString<32> AtomicLoadName("__tsan_atomic" + BitSizeStr + "_load"); 268 TsanAtomicLoad[i] = 269 M.getOrInsertFunction(AtomicLoadName, 270 TLI.getAttrList(&Ctx, {1}, /*Signed=*/true, 271 /*Ret=*/BitSize <= 32, Attr), 272 Ty, PtrTy, OrdTy); 273 274 // Args of type Ty need extension only when BitSize is 32 or less. 275 using Idxs = std::vector<unsigned>; 276 Idxs Idxs2Or12 ((BitSize <= 32) ? Idxs({1, 2}) : Idxs({2})); 277 Idxs Idxs34Or1234((BitSize <= 32) ? Idxs({1, 2, 3, 4}) : Idxs({3, 4})); 278 SmallString<32> AtomicStoreName("__tsan_atomic" + BitSizeStr + "_store"); 279 TsanAtomicStore[i] = M.getOrInsertFunction( 280 AtomicStoreName, 281 TLI.getAttrList(&Ctx, Idxs2Or12, /*Signed=*/true, /*Ret=*/false, Attr), 282 IRB.getVoidTy(), PtrTy, Ty, OrdTy); 283 284 for (unsigned Op = AtomicRMWInst::FIRST_BINOP; 285 Op <= AtomicRMWInst::LAST_BINOP; ++Op) { 286 TsanAtomicRMW[Op][i] = nullptr; 287 const char *NamePart = nullptr; 288 if (Op == AtomicRMWInst::Xchg) 289 NamePart = "_exchange"; 290 else if (Op == AtomicRMWInst::Add) 291 NamePart = "_fetch_add"; 292 else if (Op == AtomicRMWInst::Sub) 293 NamePart = "_fetch_sub"; 294 else if (Op == AtomicRMWInst::And) 295 NamePart = "_fetch_and"; 296 else if (Op == AtomicRMWInst::Or) 297 NamePart = "_fetch_or"; 298 else if (Op == AtomicRMWInst::Xor) 299 NamePart = "_fetch_xor"; 300 else if (Op == AtomicRMWInst::Nand) 301 NamePart = "_fetch_nand"; 302 else 303 continue; 304 SmallString<32> RMWName("__tsan_atomic" + itostr(BitSize) + NamePart); 305 TsanAtomicRMW[Op][i] = M.getOrInsertFunction( 306 RMWName, 307 TLI.getAttrList(&Ctx, Idxs2Or12, /*Signed=*/true, 308 /*Ret=*/BitSize <= 32, Attr), 309 Ty, PtrTy, Ty, OrdTy); 310 } 311 312 SmallString<32> AtomicCASName("__tsan_atomic" + BitSizeStr + 313 "_compare_exchange_val"); 314 TsanAtomicCAS[i] = M.getOrInsertFunction( 315 AtomicCASName, 316 TLI.getAttrList(&Ctx, Idxs34Or1234, /*Signed=*/true, 317 /*Ret=*/BitSize <= 32, Attr), 318 Ty, PtrTy, Ty, Ty, OrdTy, OrdTy); 319 } 320 TsanVptrUpdate = 321 M.getOrInsertFunction("__tsan_vptr_update", Attr, IRB.getVoidTy(), 322 IRB.getPtrTy(), IRB.getPtrTy()); 323 TsanVptrLoad = M.getOrInsertFunction("__tsan_vptr_read", Attr, 324 IRB.getVoidTy(), IRB.getPtrTy()); 325 TsanAtomicThreadFence = M.getOrInsertFunction( 326 "__tsan_atomic_thread_fence", 327 TLI.getAttrList(&Ctx, {0}, /*Signed=*/true, /*Ret=*/false, Attr), 328 IRB.getVoidTy(), OrdTy); 329 330 TsanAtomicSignalFence = M.getOrInsertFunction( 331 "__tsan_atomic_signal_fence", 332 TLI.getAttrList(&Ctx, {0}, /*Signed=*/true, /*Ret=*/false, Attr), 333 IRB.getVoidTy(), OrdTy); 334 335 MemmoveFn = 336 M.getOrInsertFunction("__tsan_memmove", Attr, IRB.getPtrTy(), 337 IRB.getPtrTy(), IRB.getPtrTy(), IntptrTy); 338 MemcpyFn = 339 M.getOrInsertFunction("__tsan_memcpy", Attr, IRB.getPtrTy(), 340 IRB.getPtrTy(), IRB.getPtrTy(), IntptrTy); 341 MemsetFn = M.getOrInsertFunction( 342 "__tsan_memset", 343 TLI.getAttrList(&Ctx, {1}, /*Signed=*/true, /*Ret=*/false, Attr), 344 IRB.getPtrTy(), IRB.getPtrTy(), IRB.getInt32Ty(), IntptrTy); 345 } 346 347 static bool isVtableAccess(Instruction *I) { 348 if (MDNode *Tag = I->getMetadata(LLVMContext::MD_tbaa)) 349 return Tag->isTBAAVtableAccess(); 350 return false; 351 } 352 353 // Do not instrument known races/"benign races" that come from compiler 354 // instrumentatin. The user has no way of suppressing them. 355 static bool shouldInstrumentReadWriteFromAddress(const Module *M, Value *Addr) { 356 // Peel off GEPs and BitCasts. 357 Addr = Addr->stripInBoundsOffsets(); 358 359 if (GlobalVariable *GV = dyn_cast<GlobalVariable>(Addr)) { 360 if (GV->hasSection()) { 361 StringRef SectionName = GV->getSection(); 362 // Check if the global is in the PGO counters section. 363 auto OF = Triple(M->getTargetTriple()).getObjectFormat(); 364 if (SectionName.ends_with( 365 getInstrProfSectionName(IPSK_cnts, OF, /*AddSegmentInfo=*/false))) 366 return false; 367 } 368 } 369 370 // Do not instrument accesses from different address spaces; we cannot deal 371 // with them. 372 if (Addr) { 373 Type *PtrTy = cast<PointerType>(Addr->getType()->getScalarType()); 374 if (PtrTy->getPointerAddressSpace() != 0) 375 return false; 376 } 377 378 return true; 379 } 380 381 bool ThreadSanitizer::addrPointsToConstantData(Value *Addr) { 382 // If this is a GEP, just analyze its pointer operand. 383 if (GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(Addr)) 384 Addr = GEP->getPointerOperand(); 385 386 if (GlobalVariable *GV = dyn_cast<GlobalVariable>(Addr)) { 387 if (GV->isConstant()) { 388 // Reads from constant globals can not race with any writes. 389 NumOmittedReadsFromConstantGlobals++; 390 return true; 391 } 392 } else if (LoadInst *L = dyn_cast<LoadInst>(Addr)) { 393 if (isVtableAccess(L)) { 394 // Reads from a vtable pointer can not race with any writes. 395 NumOmittedReadsFromVtable++; 396 return true; 397 } 398 } 399 return false; 400 } 401 402 // Instrumenting some of the accesses may be proven redundant. 403 // Currently handled: 404 // - read-before-write (within same BB, no calls between) 405 // - not captured variables 406 // 407 // We do not handle some of the patterns that should not survive 408 // after the classic compiler optimizations. 409 // E.g. two reads from the same temp should be eliminated by CSE, 410 // two writes should be eliminated by DSE, etc. 411 // 412 // 'Local' is a vector of insns within the same BB (no calls between). 413 // 'All' is a vector of insns that will be instrumented. 414 void ThreadSanitizer::chooseInstructionsToInstrument( 415 SmallVectorImpl<Instruction *> &Local, 416 SmallVectorImpl<InstructionInfo> &All, const DataLayout &DL) { 417 DenseMap<Value *, size_t> WriteTargets; // Map of addresses to index in All 418 // Iterate from the end. 419 for (Instruction *I : reverse(Local)) { 420 const bool IsWrite = isa<StoreInst>(*I); 421 Value *Addr = IsWrite ? cast<StoreInst>(I)->getPointerOperand() 422 : cast<LoadInst>(I)->getPointerOperand(); 423 424 if (!shouldInstrumentReadWriteFromAddress(I->getModule(), Addr)) 425 continue; 426 427 if (!IsWrite) { 428 const auto WriteEntry = WriteTargets.find(Addr); 429 if (!ClInstrumentReadBeforeWrite && WriteEntry != WriteTargets.end()) { 430 auto &WI = All[WriteEntry->second]; 431 // If we distinguish volatile accesses and if either the read or write 432 // is volatile, do not omit any instrumentation. 433 const bool AnyVolatile = 434 ClDistinguishVolatile && (cast<LoadInst>(I)->isVolatile() || 435 cast<StoreInst>(WI.Inst)->isVolatile()); 436 if (!AnyVolatile) { 437 // We will write to this temp, so no reason to analyze the read. 438 // Mark the write instruction as compound. 439 WI.Flags |= InstructionInfo::kCompoundRW; 440 NumOmittedReadsBeforeWrite++; 441 continue; 442 } 443 } 444 445 if (addrPointsToConstantData(Addr)) { 446 // Addr points to some constant data -- it can not race with any writes. 447 continue; 448 } 449 } 450 451 if (isa<AllocaInst>(getUnderlyingObject(Addr)) && 452 !PointerMayBeCaptured(Addr, true, true)) { 453 // The variable is addressable but not captured, so it cannot be 454 // referenced from a different thread and participate in a data race 455 // (see llvm/Analysis/CaptureTracking.h for details). 456 NumOmittedNonCaptured++; 457 continue; 458 } 459 460 // Instrument this instruction. 461 All.emplace_back(I); 462 if (IsWrite) { 463 // For read-before-write and compound instrumentation we only need one 464 // write target, and we can override any previous entry if it exists. 465 WriteTargets[Addr] = All.size() - 1; 466 } 467 } 468 Local.clear(); 469 } 470 471 static bool isTsanAtomic(const Instruction *I) { 472 // TODO: Ask TTI whether synchronization scope is between threads. 473 auto SSID = getAtomicSyncScopeID(I); 474 if (!SSID) 475 return false; 476 if (isa<LoadInst>(I) || isa<StoreInst>(I)) 477 return *SSID != SyncScope::SingleThread; 478 return true; 479 } 480 481 void ThreadSanitizer::InsertRuntimeIgnores(Function &F) { 482 InstrumentationIRBuilder IRB(&F.getEntryBlock(), 483 F.getEntryBlock().getFirstNonPHIIt()); 484 IRB.CreateCall(TsanIgnoreBegin); 485 EscapeEnumerator EE(F, "tsan_ignore_cleanup", ClHandleCxxExceptions); 486 while (IRBuilder<> *AtExit = EE.Next()) { 487 InstrumentationIRBuilder::ensureDebugInfo(*AtExit, F); 488 AtExit->CreateCall(TsanIgnoreEnd); 489 } 490 } 491 492 bool ThreadSanitizer::sanitizeFunction(Function &F, 493 const TargetLibraryInfo &TLI) { 494 // This is required to prevent instrumenting call to __tsan_init from within 495 // the module constructor. 496 if (F.getName() == kTsanModuleCtorName) 497 return false; 498 // Naked functions can not have prologue/epilogue 499 // (__tsan_func_entry/__tsan_func_exit) generated, so don't instrument them at 500 // all. 501 if (F.hasFnAttribute(Attribute::Naked)) 502 return false; 503 504 // __attribute__(disable_sanitizer_instrumentation) prevents all kinds of 505 // instrumentation. 506 if (F.hasFnAttribute(Attribute::DisableSanitizerInstrumentation)) 507 return false; 508 509 initialize(*F.getParent(), TLI); 510 SmallVector<InstructionInfo, 8> AllLoadsAndStores; 511 SmallVector<Instruction*, 8> LocalLoadsAndStores; 512 SmallVector<Instruction*, 8> AtomicAccesses; 513 SmallVector<Instruction*, 8> MemIntrinCalls; 514 bool Res = false; 515 bool HasCalls = false; 516 bool SanitizeFunction = F.hasFnAttribute(Attribute::SanitizeThread); 517 const DataLayout &DL = F.getDataLayout(); 518 519 // Traverse all instructions, collect loads/stores/returns, check for calls. 520 for (auto &BB : F) { 521 for (auto &Inst : BB) { 522 // Skip instructions inserted by another instrumentation. 523 if (Inst.hasMetadata(LLVMContext::MD_nosanitize)) 524 continue; 525 if (isTsanAtomic(&Inst)) 526 AtomicAccesses.push_back(&Inst); 527 else if (isa<LoadInst>(Inst) || isa<StoreInst>(Inst)) 528 LocalLoadsAndStores.push_back(&Inst); 529 else if ((isa<CallInst>(Inst) && !isa<DbgInfoIntrinsic>(Inst)) || 530 isa<InvokeInst>(Inst)) { 531 if (CallInst *CI = dyn_cast<CallInst>(&Inst)) 532 maybeMarkSanitizerLibraryCallNoBuiltin(CI, &TLI); 533 if (isa<MemIntrinsic>(Inst)) 534 MemIntrinCalls.push_back(&Inst); 535 HasCalls = true; 536 chooseInstructionsToInstrument(LocalLoadsAndStores, AllLoadsAndStores, 537 DL); 538 } 539 } 540 chooseInstructionsToInstrument(LocalLoadsAndStores, AllLoadsAndStores, DL); 541 } 542 543 // We have collected all loads and stores. 544 // FIXME: many of these accesses do not need to be checked for races 545 // (e.g. variables that do not escape, etc). 546 547 // Instrument memory accesses only if we want to report bugs in the function. 548 if (ClInstrumentMemoryAccesses && SanitizeFunction) 549 for (const auto &II : AllLoadsAndStores) { 550 Res |= instrumentLoadOrStore(II, DL); 551 } 552 553 // Instrument atomic memory accesses in any case (they can be used to 554 // implement synchronization). 555 if (ClInstrumentAtomics) 556 for (auto *Inst : AtomicAccesses) { 557 Res |= instrumentAtomic(Inst, DL); 558 } 559 560 if (ClInstrumentMemIntrinsics && SanitizeFunction) 561 for (auto *Inst : MemIntrinCalls) { 562 Res |= instrumentMemIntrinsic(Inst); 563 } 564 565 if (F.hasFnAttribute("sanitize_thread_no_checking_at_run_time")) { 566 assert(!F.hasFnAttribute(Attribute::SanitizeThread)); 567 if (HasCalls) 568 InsertRuntimeIgnores(F); 569 } 570 571 // Instrument function entry/exit points if there were instrumented accesses. 572 if ((Res || HasCalls) && ClInstrumentFuncEntryExit) { 573 InstrumentationIRBuilder IRB(&F.getEntryBlock(), 574 F.getEntryBlock().getFirstNonPHIIt()); 575 Value *ReturnAddress = 576 IRB.CreateIntrinsic(Intrinsic::returnaddress, {}, IRB.getInt32(0)); 577 IRB.CreateCall(TsanFuncEntry, ReturnAddress); 578 579 EscapeEnumerator EE(F, "tsan_cleanup", ClHandleCxxExceptions); 580 while (IRBuilder<> *AtExit = EE.Next()) { 581 InstrumentationIRBuilder::ensureDebugInfo(*AtExit, F); 582 AtExit->CreateCall(TsanFuncExit, {}); 583 } 584 Res = true; 585 } 586 return Res; 587 } 588 589 bool ThreadSanitizer::instrumentLoadOrStore(const InstructionInfo &II, 590 const DataLayout &DL) { 591 InstrumentationIRBuilder IRB(II.Inst); 592 const bool IsWrite = isa<StoreInst>(*II.Inst); 593 Value *Addr = IsWrite ? cast<StoreInst>(II.Inst)->getPointerOperand() 594 : cast<LoadInst>(II.Inst)->getPointerOperand(); 595 Type *OrigTy = getLoadStoreType(II.Inst); 596 597 // swifterror memory addresses are mem2reg promoted by instruction selection. 598 // As such they cannot have regular uses like an instrumentation function and 599 // it makes no sense to track them as memory. 600 if (Addr->isSwiftError()) 601 return false; 602 603 int Idx = getMemoryAccessFuncIndex(OrigTy, Addr, DL); 604 if (Idx < 0) 605 return false; 606 if (IsWrite && isVtableAccess(II.Inst)) { 607 LLVM_DEBUG(dbgs() << " VPTR : " << *II.Inst << "\n"); 608 Value *StoredValue = cast<StoreInst>(II.Inst)->getValueOperand(); 609 // StoredValue may be a vector type if we are storing several vptrs at once. 610 // In this case, just take the first element of the vector since this is 611 // enough to find vptr races. 612 if (isa<VectorType>(StoredValue->getType())) 613 StoredValue = IRB.CreateExtractElement( 614 StoredValue, ConstantInt::get(IRB.getInt32Ty(), 0)); 615 if (StoredValue->getType()->isIntegerTy()) 616 StoredValue = IRB.CreateIntToPtr(StoredValue, IRB.getPtrTy()); 617 // Call TsanVptrUpdate. 618 IRB.CreateCall(TsanVptrUpdate, {Addr, StoredValue}); 619 NumInstrumentedVtableWrites++; 620 return true; 621 } 622 if (!IsWrite && isVtableAccess(II.Inst)) { 623 IRB.CreateCall(TsanVptrLoad, Addr); 624 NumInstrumentedVtableReads++; 625 return true; 626 } 627 628 const Align Alignment = IsWrite ? cast<StoreInst>(II.Inst)->getAlign() 629 : cast<LoadInst>(II.Inst)->getAlign(); 630 const bool IsCompoundRW = 631 ClCompoundReadBeforeWrite && (II.Flags & InstructionInfo::kCompoundRW); 632 const bool IsVolatile = ClDistinguishVolatile && 633 (IsWrite ? cast<StoreInst>(II.Inst)->isVolatile() 634 : cast<LoadInst>(II.Inst)->isVolatile()); 635 assert((!IsVolatile || !IsCompoundRW) && "Compound volatile invalid!"); 636 637 const uint32_t TypeSize = DL.getTypeStoreSizeInBits(OrigTy); 638 FunctionCallee OnAccessFunc = nullptr; 639 if (Alignment >= Align(8) || (Alignment.value() % (TypeSize / 8)) == 0) { 640 if (IsCompoundRW) 641 OnAccessFunc = TsanCompoundRW[Idx]; 642 else if (IsVolatile) 643 OnAccessFunc = IsWrite ? TsanVolatileWrite[Idx] : TsanVolatileRead[Idx]; 644 else 645 OnAccessFunc = IsWrite ? TsanWrite[Idx] : TsanRead[Idx]; 646 } else { 647 if (IsCompoundRW) 648 OnAccessFunc = TsanUnalignedCompoundRW[Idx]; 649 else if (IsVolatile) 650 OnAccessFunc = IsWrite ? TsanUnalignedVolatileWrite[Idx] 651 : TsanUnalignedVolatileRead[Idx]; 652 else 653 OnAccessFunc = IsWrite ? TsanUnalignedWrite[Idx] : TsanUnalignedRead[Idx]; 654 } 655 IRB.CreateCall(OnAccessFunc, Addr); 656 if (IsCompoundRW || IsWrite) 657 NumInstrumentedWrites++; 658 if (IsCompoundRW || !IsWrite) 659 NumInstrumentedReads++; 660 return true; 661 } 662 663 static ConstantInt *createOrdering(IRBuilder<> *IRB, AtomicOrdering ord) { 664 uint32_t v = 0; 665 switch (ord) { 666 case AtomicOrdering::NotAtomic: 667 llvm_unreachable("unexpected atomic ordering!"); 668 case AtomicOrdering::Unordered: [[fallthrough]]; 669 case AtomicOrdering::Monotonic: v = 0; break; 670 // Not specified yet: 671 // case AtomicOrdering::Consume: v = 1; break; 672 case AtomicOrdering::Acquire: v = 2; break; 673 case AtomicOrdering::Release: v = 3; break; 674 case AtomicOrdering::AcquireRelease: v = 4; break; 675 case AtomicOrdering::SequentiallyConsistent: v = 5; break; 676 } 677 return IRB->getInt32(v); 678 } 679 680 // If a memset intrinsic gets inlined by the code gen, we will miss races on it. 681 // So, we either need to ensure the intrinsic is not inlined, or instrument it. 682 // We do not instrument memset/memmove/memcpy intrinsics (too complicated), 683 // instead we simply replace them with regular function calls, which are then 684 // intercepted by the run-time. 685 // Since tsan is running after everyone else, the calls should not be 686 // replaced back with intrinsics. If that becomes wrong at some point, 687 // we will need to call e.g. __tsan_memset to avoid the intrinsics. 688 bool ThreadSanitizer::instrumentMemIntrinsic(Instruction *I) { 689 InstrumentationIRBuilder IRB(I); 690 if (MemSetInst *M = dyn_cast<MemSetInst>(I)) { 691 Value *Cast1 = IRB.CreateIntCast(M->getArgOperand(1), IRB.getInt32Ty(), false); 692 Value *Cast2 = IRB.CreateIntCast(M->getArgOperand(2), IntptrTy, false); 693 IRB.CreateCall( 694 MemsetFn, 695 {M->getArgOperand(0), 696 Cast1, 697 Cast2}); 698 I->eraseFromParent(); 699 } else if (MemTransferInst *M = dyn_cast<MemTransferInst>(I)) { 700 IRB.CreateCall( 701 isa<MemCpyInst>(M) ? MemcpyFn : MemmoveFn, 702 {M->getArgOperand(0), 703 M->getArgOperand(1), 704 IRB.CreateIntCast(M->getArgOperand(2), IntptrTy, false)}); 705 I->eraseFromParent(); 706 } 707 return false; 708 } 709 710 // Both llvm and ThreadSanitizer atomic operations are based on C++11/C1x 711 // standards. For background see C++11 standard. A slightly older, publicly 712 // available draft of the standard (not entirely up-to-date, but close enough 713 // for casual browsing) is available here: 714 // http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2011/n3242.pdf 715 // The following page contains more background information: 716 // http://www.hpl.hp.com/personal/Hans_Boehm/c++mm/ 717 718 bool ThreadSanitizer::instrumentAtomic(Instruction *I, const DataLayout &DL) { 719 InstrumentationIRBuilder IRB(I); 720 if (LoadInst *LI = dyn_cast<LoadInst>(I)) { 721 Value *Addr = LI->getPointerOperand(); 722 Type *OrigTy = LI->getType(); 723 int Idx = getMemoryAccessFuncIndex(OrigTy, Addr, DL); 724 if (Idx < 0) 725 return false; 726 Value *Args[] = {Addr, 727 createOrdering(&IRB, LI->getOrdering())}; 728 Value *C = IRB.CreateCall(TsanAtomicLoad[Idx], Args); 729 Value *Cast = IRB.CreateBitOrPointerCast(C, OrigTy); 730 I->replaceAllUsesWith(Cast); 731 } else if (StoreInst *SI = dyn_cast<StoreInst>(I)) { 732 Value *Addr = SI->getPointerOperand(); 733 int Idx = 734 getMemoryAccessFuncIndex(SI->getValueOperand()->getType(), Addr, DL); 735 if (Idx < 0) 736 return false; 737 const unsigned ByteSize = 1U << Idx; 738 const unsigned BitSize = ByteSize * 8; 739 Type *Ty = Type::getIntNTy(IRB.getContext(), BitSize); 740 Value *Args[] = {Addr, 741 IRB.CreateBitOrPointerCast(SI->getValueOperand(), Ty), 742 createOrdering(&IRB, SI->getOrdering())}; 743 IRB.CreateCall(TsanAtomicStore[Idx], Args); 744 SI->eraseFromParent(); 745 } else if (AtomicRMWInst *RMWI = dyn_cast<AtomicRMWInst>(I)) { 746 Value *Addr = RMWI->getPointerOperand(); 747 int Idx = 748 getMemoryAccessFuncIndex(RMWI->getValOperand()->getType(), Addr, DL); 749 if (Idx < 0) 750 return false; 751 FunctionCallee F = TsanAtomicRMW[RMWI->getOperation()][Idx]; 752 if (!F) 753 return false; 754 const unsigned ByteSize = 1U << Idx; 755 const unsigned BitSize = ByteSize * 8; 756 Type *Ty = Type::getIntNTy(IRB.getContext(), BitSize); 757 Value *Val = RMWI->getValOperand(); 758 Value *Args[] = {Addr, IRB.CreateBitOrPointerCast(Val, Ty), 759 createOrdering(&IRB, RMWI->getOrdering())}; 760 Value *C = IRB.CreateCall(F, Args); 761 I->replaceAllUsesWith(IRB.CreateBitOrPointerCast(C, Val->getType())); 762 I->eraseFromParent(); 763 } else if (AtomicCmpXchgInst *CASI = dyn_cast<AtomicCmpXchgInst>(I)) { 764 Value *Addr = CASI->getPointerOperand(); 765 Type *OrigOldValTy = CASI->getNewValOperand()->getType(); 766 int Idx = getMemoryAccessFuncIndex(OrigOldValTy, Addr, DL); 767 if (Idx < 0) 768 return false; 769 const unsigned ByteSize = 1U << Idx; 770 const unsigned BitSize = ByteSize * 8; 771 Type *Ty = Type::getIntNTy(IRB.getContext(), BitSize); 772 Value *CmpOperand = 773 IRB.CreateBitOrPointerCast(CASI->getCompareOperand(), Ty); 774 Value *NewOperand = 775 IRB.CreateBitOrPointerCast(CASI->getNewValOperand(), Ty); 776 Value *Args[] = {Addr, 777 CmpOperand, 778 NewOperand, 779 createOrdering(&IRB, CASI->getSuccessOrdering()), 780 createOrdering(&IRB, CASI->getFailureOrdering())}; 781 CallInst *C = IRB.CreateCall(TsanAtomicCAS[Idx], Args); 782 Value *Success = IRB.CreateICmpEQ(C, CmpOperand); 783 Value *OldVal = C; 784 if (Ty != OrigOldValTy) { 785 // The value is a pointer, so we need to cast the return value. 786 OldVal = IRB.CreateIntToPtr(C, OrigOldValTy); 787 } 788 789 Value *Res = 790 IRB.CreateInsertValue(PoisonValue::get(CASI->getType()), OldVal, 0); 791 Res = IRB.CreateInsertValue(Res, Success, 1); 792 793 I->replaceAllUsesWith(Res); 794 I->eraseFromParent(); 795 } else if (FenceInst *FI = dyn_cast<FenceInst>(I)) { 796 Value *Args[] = {createOrdering(&IRB, FI->getOrdering())}; 797 FunctionCallee F = FI->getSyncScopeID() == SyncScope::SingleThread 798 ? TsanAtomicSignalFence 799 : TsanAtomicThreadFence; 800 IRB.CreateCall(F, Args); 801 FI->eraseFromParent(); 802 } 803 return true; 804 } 805 806 int ThreadSanitizer::getMemoryAccessFuncIndex(Type *OrigTy, Value *Addr, 807 const DataLayout &DL) { 808 assert(OrigTy->isSized()); 809 if (OrigTy->isScalableTy()) { 810 // FIXME: support vscale. 811 return -1; 812 } 813 uint32_t TypeSize = DL.getTypeStoreSizeInBits(OrigTy); 814 if (TypeSize != 8 && TypeSize != 16 && 815 TypeSize != 32 && TypeSize != 64 && TypeSize != 128) { 816 NumAccessesWithBadSize++; 817 // Ignore all unusual sizes. 818 return -1; 819 } 820 size_t Idx = llvm::countr_zero(TypeSize / 8); 821 assert(Idx < kNumberOfAccessSizes); 822 return Idx; 823 } 824