1 //===-- SanitizerCoverage.cpp - coverage instrumentation for sanitizers ---===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // Coverage instrumentation done on LLVM IR level, works with Sanitizers.
10 //
11 //===----------------------------------------------------------------------===//
12
13 #include "llvm/Transforms/Instrumentation/SanitizerCoverage.h"
14 #include "llvm/ADT/ArrayRef.h"
15 #include "llvm/ADT/SmallVector.h"
16 #include "llvm/ADT/Triple.h"
17 #include "llvm/Analysis/EHPersonalities.h"
18 #include "llvm/Analysis/GlobalsModRef.h"
19 #include "llvm/Analysis/PostDominators.h"
20 #include "llvm/IR/Constant.h"
21 #include "llvm/IR/DataLayout.h"
22 #include "llvm/IR/Dominators.h"
23 #include "llvm/IR/Function.h"
24 #include "llvm/IR/GlobalVariable.h"
25 #include "llvm/IR/IRBuilder.h"
26 #include "llvm/IR/IntrinsicInst.h"
27 #include "llvm/IR/Intrinsics.h"
28 #include "llvm/IR/LLVMContext.h"
29 #include "llvm/IR/Module.h"
30 #include "llvm/IR/Type.h"
31 #include "llvm/InitializePasses.h"
32 #include "llvm/Support/CommandLine.h"
33 #include "llvm/Support/SpecialCaseList.h"
34 #include "llvm/Support/VirtualFileSystem.h"
35 #include "llvm/Transforms/Instrumentation.h"
36 #include "llvm/Transforms/Utils/BasicBlockUtils.h"
37 #include "llvm/Transforms/Utils/ModuleUtils.h"
38
39 using namespace llvm;
40
41 #define DEBUG_TYPE "sancov"
42
43 const char SanCovTracePCIndirName[] = "__sanitizer_cov_trace_pc_indir";
44 const char SanCovTracePCName[] = "__sanitizer_cov_trace_pc";
45 const char SanCovTraceCmp1[] = "__sanitizer_cov_trace_cmp1";
46 const char SanCovTraceCmp2[] = "__sanitizer_cov_trace_cmp2";
47 const char SanCovTraceCmp4[] = "__sanitizer_cov_trace_cmp4";
48 const char SanCovTraceCmp8[] = "__sanitizer_cov_trace_cmp8";
49 const char SanCovTraceConstCmp1[] = "__sanitizer_cov_trace_const_cmp1";
50 const char SanCovTraceConstCmp2[] = "__sanitizer_cov_trace_const_cmp2";
51 const char SanCovTraceConstCmp4[] = "__sanitizer_cov_trace_const_cmp4";
52 const char SanCovTraceConstCmp8[] = "__sanitizer_cov_trace_const_cmp8";
53 const char SanCovLoad1[] = "__sanitizer_cov_load1";
54 const char SanCovLoad2[] = "__sanitizer_cov_load2";
55 const char SanCovLoad4[] = "__sanitizer_cov_load4";
56 const char SanCovLoad8[] = "__sanitizer_cov_load8";
57 const char SanCovLoad16[] = "__sanitizer_cov_load16";
58 const char SanCovStore1[] = "__sanitizer_cov_store1";
59 const char SanCovStore2[] = "__sanitizer_cov_store2";
60 const char SanCovStore4[] = "__sanitizer_cov_store4";
61 const char SanCovStore8[] = "__sanitizer_cov_store8";
62 const char SanCovStore16[] = "__sanitizer_cov_store16";
63 const char SanCovTraceDiv4[] = "__sanitizer_cov_trace_div4";
64 const char SanCovTraceDiv8[] = "__sanitizer_cov_trace_div8";
65 const char SanCovTraceGep[] = "__sanitizer_cov_trace_gep";
66 const char SanCovTraceSwitchName[] = "__sanitizer_cov_trace_switch";
67 const char SanCovModuleCtorTracePcGuardName[] =
68 "sancov.module_ctor_trace_pc_guard";
69 const char SanCovModuleCtor8bitCountersName[] =
70 "sancov.module_ctor_8bit_counters";
71 const char SanCovModuleCtorBoolFlagName[] = "sancov.module_ctor_bool_flag";
72 static const uint64_t SanCtorAndDtorPriority = 2;
73
74 const char SanCovTracePCGuardName[] = "__sanitizer_cov_trace_pc_guard";
75 const char SanCovTracePCGuardInitName[] = "__sanitizer_cov_trace_pc_guard_init";
76 const char SanCov8bitCountersInitName[] = "__sanitizer_cov_8bit_counters_init";
77 const char SanCovBoolFlagInitName[] = "__sanitizer_cov_bool_flag_init";
78 const char SanCovPCsInitName[] = "__sanitizer_cov_pcs_init";
79 const char SanCovCFsInitName[] = "__sanitizer_cov_cfs_init";
80
81 const char SanCovGuardsSectionName[] = "sancov_guards";
82 const char SanCovCountersSectionName[] = "sancov_cntrs";
83 const char SanCovBoolFlagSectionName[] = "sancov_bools";
84 const char SanCovPCsSectionName[] = "sancov_pcs";
85 const char SanCovCFsSectionName[] = "sancov_cfs";
86
87 const char SanCovLowestStackName[] = "__sancov_lowest_stack";
88
89 static cl::opt<int> ClCoverageLevel(
90 "sanitizer-coverage-level",
91 cl::desc("Sanitizer Coverage. 0: none, 1: entry block, 2: all blocks, "
92 "3: all blocks and critical edges"),
93 cl::Hidden, cl::init(0));
94
95 static cl::opt<bool> ClTracePC("sanitizer-coverage-trace-pc",
96 cl::desc("Experimental pc tracing"), cl::Hidden,
97 cl::init(false));
98
99 static cl::opt<bool> ClTracePCGuard("sanitizer-coverage-trace-pc-guard",
100 cl::desc("pc tracing with a guard"),
101 cl::Hidden, cl::init(false));
102
103 // If true, we create a global variable that contains PCs of all instrumented
104 // BBs, put this global into a named section, and pass this section's bounds
105 // to __sanitizer_cov_pcs_init.
106 // This way the coverage instrumentation does not need to acquire the PCs
107 // at run-time. Works with trace-pc-guard, inline-8bit-counters, and
108 // inline-bool-flag.
109 static cl::opt<bool> ClCreatePCTable("sanitizer-coverage-pc-table",
110 cl::desc("create a static PC table"),
111 cl::Hidden, cl::init(false));
112
113 static cl::opt<bool>
114 ClInline8bitCounters("sanitizer-coverage-inline-8bit-counters",
115 cl::desc("increments 8-bit counter for every edge"),
116 cl::Hidden, cl::init(false));
117
118 static cl::opt<bool>
119 ClInlineBoolFlag("sanitizer-coverage-inline-bool-flag",
120 cl::desc("sets a boolean flag for every edge"), cl::Hidden,
121 cl::init(false));
122
123 static cl::opt<bool>
124 ClCMPTracing("sanitizer-coverage-trace-compares",
125 cl::desc("Tracing of CMP and similar instructions"),
126 cl::Hidden, cl::init(false));
127
128 static cl::opt<bool> ClDIVTracing("sanitizer-coverage-trace-divs",
129 cl::desc("Tracing of DIV instructions"),
130 cl::Hidden, cl::init(false));
131
132 static cl::opt<bool> ClLoadTracing("sanitizer-coverage-trace-loads",
133 cl::desc("Tracing of load instructions"),
134 cl::Hidden, cl::init(false));
135
136 static cl::opt<bool> ClStoreTracing("sanitizer-coverage-trace-stores",
137 cl::desc("Tracing of store instructions"),
138 cl::Hidden, cl::init(false));
139
140 static cl::opt<bool> ClGEPTracing("sanitizer-coverage-trace-geps",
141 cl::desc("Tracing of GEP instructions"),
142 cl::Hidden, cl::init(false));
143
144 static cl::opt<bool>
145 ClPruneBlocks("sanitizer-coverage-prune-blocks",
146 cl::desc("Reduce the number of instrumented blocks"),
147 cl::Hidden, cl::init(true));
148
149 static cl::opt<bool> ClStackDepth("sanitizer-coverage-stack-depth",
150 cl::desc("max stack depth tracing"),
151 cl::Hidden, cl::init(false));
152
153 static cl::opt<bool>
154 ClCollectCF("sanitizer-coverage-control-flow",
155 cl::desc("collect control flow for each function"), cl::Hidden,
156 cl::init(false));
157
158 namespace {
159
getOptions(int LegacyCoverageLevel)160 SanitizerCoverageOptions getOptions(int LegacyCoverageLevel) {
161 SanitizerCoverageOptions Res;
162 switch (LegacyCoverageLevel) {
163 case 0:
164 Res.CoverageType = SanitizerCoverageOptions::SCK_None;
165 break;
166 case 1:
167 Res.CoverageType = SanitizerCoverageOptions::SCK_Function;
168 break;
169 case 2:
170 Res.CoverageType = SanitizerCoverageOptions::SCK_BB;
171 break;
172 case 3:
173 Res.CoverageType = SanitizerCoverageOptions::SCK_Edge;
174 break;
175 case 4:
176 Res.CoverageType = SanitizerCoverageOptions::SCK_Edge;
177 Res.IndirectCalls = true;
178 break;
179 }
180 return Res;
181 }
182
OverrideFromCL(SanitizerCoverageOptions Options)183 SanitizerCoverageOptions OverrideFromCL(SanitizerCoverageOptions Options) {
184 // Sets CoverageType and IndirectCalls.
185 SanitizerCoverageOptions CLOpts = getOptions(ClCoverageLevel);
186 Options.CoverageType = std::max(Options.CoverageType, CLOpts.CoverageType);
187 Options.IndirectCalls |= CLOpts.IndirectCalls;
188 Options.TraceCmp |= ClCMPTracing;
189 Options.TraceDiv |= ClDIVTracing;
190 Options.TraceGep |= ClGEPTracing;
191 Options.TracePC |= ClTracePC;
192 Options.TracePCGuard |= ClTracePCGuard;
193 Options.Inline8bitCounters |= ClInline8bitCounters;
194 Options.InlineBoolFlag |= ClInlineBoolFlag;
195 Options.PCTable |= ClCreatePCTable;
196 Options.NoPrune |= !ClPruneBlocks;
197 Options.StackDepth |= ClStackDepth;
198 Options.TraceLoads |= ClLoadTracing;
199 Options.TraceStores |= ClStoreTracing;
200 if (!Options.TracePCGuard && !Options.TracePC &&
201 !Options.Inline8bitCounters && !Options.StackDepth &&
202 !Options.InlineBoolFlag && !Options.TraceLoads && !Options.TraceStores)
203 Options.TracePCGuard = true; // TracePCGuard is default.
204 Options.CollectControlFlow |= ClCollectCF;
205 return Options;
206 }
207
208 using DomTreeCallback = function_ref<const DominatorTree *(Function &F)>;
209 using PostDomTreeCallback =
210 function_ref<const PostDominatorTree *(Function &F)>;
211
212 class ModuleSanitizerCoverage {
213 public:
ModuleSanitizerCoverage(const SanitizerCoverageOptions & Options=SanitizerCoverageOptions (),const SpecialCaseList * Allowlist=nullptr,const SpecialCaseList * Blocklist=nullptr)214 ModuleSanitizerCoverage(
215 const SanitizerCoverageOptions &Options = SanitizerCoverageOptions(),
216 const SpecialCaseList *Allowlist = nullptr,
217 const SpecialCaseList *Blocklist = nullptr)
218 : Options(OverrideFromCL(Options)), Allowlist(Allowlist),
219 Blocklist(Blocklist) {}
220 bool instrumentModule(Module &M, DomTreeCallback DTCallback,
221 PostDomTreeCallback PDTCallback);
222
223 private:
224 void createFunctionControlFlow(Function &F);
225 void instrumentFunction(Function &F, DomTreeCallback DTCallback,
226 PostDomTreeCallback PDTCallback);
227 void InjectCoverageForIndirectCalls(Function &F,
228 ArrayRef<Instruction *> IndirCalls);
229 void InjectTraceForCmp(Function &F, ArrayRef<Instruction *> CmpTraceTargets);
230 void InjectTraceForDiv(Function &F,
231 ArrayRef<BinaryOperator *> DivTraceTargets);
232 void InjectTraceForGep(Function &F,
233 ArrayRef<GetElementPtrInst *> GepTraceTargets);
234 void InjectTraceForLoadsAndStores(Function &F, ArrayRef<LoadInst *> Loads,
235 ArrayRef<StoreInst *> Stores);
236 void InjectTraceForSwitch(Function &F,
237 ArrayRef<Instruction *> SwitchTraceTargets);
238 bool InjectCoverage(Function &F, ArrayRef<BasicBlock *> AllBlocks,
239 bool IsLeafFunc = true);
240 GlobalVariable *CreateFunctionLocalArrayInSection(size_t NumElements,
241 Function &F, Type *Ty,
242 const char *Section);
243 GlobalVariable *CreatePCArray(Function &F, ArrayRef<BasicBlock *> AllBlocks);
244 void CreateFunctionLocalArrays(Function &F, ArrayRef<BasicBlock *> AllBlocks);
245 void InjectCoverageAtBlock(Function &F, BasicBlock &BB, size_t Idx,
246 bool IsLeafFunc = true);
247 Function *CreateInitCallsForSections(Module &M, const char *CtorName,
248 const char *InitFunctionName, Type *Ty,
249 const char *Section);
250 std::pair<Value *, Value *> CreateSecStartEnd(Module &M, const char *Section,
251 Type *Ty);
252
SetNoSanitizeMetadata(Instruction * I)253 void SetNoSanitizeMetadata(Instruction *I) {
254 I->setMetadata(LLVMContext::MD_nosanitize, MDNode::get(*C, std::nullopt));
255 }
256
257 std::string getSectionName(const std::string &Section) const;
258 std::string getSectionStart(const std::string &Section) const;
259 std::string getSectionEnd(const std::string &Section) const;
260 FunctionCallee SanCovTracePCIndir;
261 FunctionCallee SanCovTracePC, SanCovTracePCGuard;
262 std::array<FunctionCallee, 4> SanCovTraceCmpFunction;
263 std::array<FunctionCallee, 4> SanCovTraceConstCmpFunction;
264 std::array<FunctionCallee, 5> SanCovLoadFunction;
265 std::array<FunctionCallee, 5> SanCovStoreFunction;
266 std::array<FunctionCallee, 2> SanCovTraceDivFunction;
267 FunctionCallee SanCovTraceGepFunction;
268 FunctionCallee SanCovTraceSwitchFunction;
269 GlobalVariable *SanCovLowestStack;
270 Type *Int128PtrTy, *IntptrTy, *IntptrPtrTy, *Int64Ty, *Int64PtrTy, *Int32Ty,
271 *Int32PtrTy, *Int16PtrTy, *Int16Ty, *Int8Ty, *Int8PtrTy, *Int1Ty,
272 *Int1PtrTy;
273 Module *CurModule;
274 std::string CurModuleUniqueId;
275 Triple TargetTriple;
276 LLVMContext *C;
277 const DataLayout *DL;
278
279 GlobalVariable *FunctionGuardArray; // for trace-pc-guard.
280 GlobalVariable *Function8bitCounterArray; // for inline-8bit-counters.
281 GlobalVariable *FunctionBoolArray; // for inline-bool-flag.
282 GlobalVariable *FunctionPCsArray; // for pc-table.
283 GlobalVariable *FunctionCFsArray; // for control flow table
284 SmallVector<GlobalValue *, 20> GlobalsToAppendToUsed;
285 SmallVector<GlobalValue *, 20> GlobalsToAppendToCompilerUsed;
286
287 SanitizerCoverageOptions Options;
288
289 const SpecialCaseList *Allowlist;
290 const SpecialCaseList *Blocklist;
291 };
292 } // namespace
293
run(Module & M,ModuleAnalysisManager & MAM)294 PreservedAnalyses SanitizerCoveragePass::run(Module &M,
295 ModuleAnalysisManager &MAM) {
296 ModuleSanitizerCoverage ModuleSancov(Options, Allowlist.get(),
297 Blocklist.get());
298 auto &FAM = MAM.getResult<FunctionAnalysisManagerModuleProxy>(M).getManager();
299 auto DTCallback = [&FAM](Function &F) -> const DominatorTree * {
300 return &FAM.getResult<DominatorTreeAnalysis>(F);
301 };
302 auto PDTCallback = [&FAM](Function &F) -> const PostDominatorTree * {
303 return &FAM.getResult<PostDominatorTreeAnalysis>(F);
304 };
305 if (!ModuleSancov.instrumentModule(M, DTCallback, PDTCallback))
306 return PreservedAnalyses::all();
307
308 PreservedAnalyses PA = PreservedAnalyses::none();
309 // GlobalsAA is considered stateless and does not get invalidated unless
310 // explicitly invalidated; PreservedAnalyses::none() is not enough. Sanitizers
311 // make changes that require GlobalsAA to be invalidated.
312 PA.abandon<GlobalsAA>();
313 return PA;
314 }
315
316 std::pair<Value *, Value *>
CreateSecStartEnd(Module & M,const char * Section,Type * Ty)317 ModuleSanitizerCoverage::CreateSecStartEnd(Module &M, const char *Section,
318 Type *Ty) {
319 // Use ExternalWeak so that if all sections are discarded due to section
320 // garbage collection, the linker will not report undefined symbol errors.
321 // Windows defines the start/stop symbols in compiler-rt so no need for
322 // ExternalWeak.
323 GlobalValue::LinkageTypes Linkage = TargetTriple.isOSBinFormatCOFF()
324 ? GlobalVariable::ExternalLinkage
325 : GlobalVariable::ExternalWeakLinkage;
326 GlobalVariable *SecStart =
327 new GlobalVariable(M, Ty, false, Linkage, nullptr,
328 getSectionStart(Section));
329 SecStart->setVisibility(GlobalValue::HiddenVisibility);
330 GlobalVariable *SecEnd =
331 new GlobalVariable(M, Ty, false, Linkage, nullptr,
332 getSectionEnd(Section));
333 SecEnd->setVisibility(GlobalValue::HiddenVisibility);
334 IRBuilder<> IRB(M.getContext());
335 if (!TargetTriple.isOSBinFormatCOFF())
336 return std::make_pair(SecStart, SecEnd);
337
338 // Account for the fact that on windows-msvc __start_* symbols actually
339 // point to a uint64_t before the start of the array.
340 auto SecStartI8Ptr = IRB.CreatePointerCast(SecStart, Int8PtrTy);
341 auto GEP = IRB.CreateGEP(Int8Ty, SecStartI8Ptr,
342 ConstantInt::get(IntptrTy, sizeof(uint64_t)));
343 return std::make_pair(IRB.CreatePointerCast(GEP, PointerType::getUnqual(Ty)),
344 SecEnd);
345 }
346
CreateInitCallsForSections(Module & M,const char * CtorName,const char * InitFunctionName,Type * Ty,const char * Section)347 Function *ModuleSanitizerCoverage::CreateInitCallsForSections(
348 Module &M, const char *CtorName, const char *InitFunctionName, Type *Ty,
349 const char *Section) {
350 auto SecStartEnd = CreateSecStartEnd(M, Section, Ty);
351 auto SecStart = SecStartEnd.first;
352 auto SecEnd = SecStartEnd.second;
353 Function *CtorFunc;
354 Type *PtrTy = PointerType::getUnqual(Ty);
355 std::tie(CtorFunc, std::ignore) = createSanitizerCtorAndInitFunctions(
356 M, CtorName, InitFunctionName, {PtrTy, PtrTy}, {SecStart, SecEnd});
357 assert(CtorFunc->getName() == CtorName);
358
359 if (TargetTriple.supportsCOMDAT()) {
360 // Use comdat to dedup CtorFunc.
361 CtorFunc->setComdat(M.getOrInsertComdat(CtorName));
362 appendToGlobalCtors(M, CtorFunc, SanCtorAndDtorPriority, CtorFunc);
363 } else {
364 appendToGlobalCtors(M, CtorFunc, SanCtorAndDtorPriority);
365 }
366
367 if (TargetTriple.isOSBinFormatCOFF()) {
368 // In COFF files, if the contructors are set as COMDAT (they are because
369 // COFF supports COMDAT) and the linker flag /OPT:REF (strip unreferenced
370 // functions and data) is used, the constructors get stripped. To prevent
371 // this, give the constructors weak ODR linkage and ensure the linker knows
372 // to include the sancov constructor. This way the linker can deduplicate
373 // the constructors but always leave one copy.
374 CtorFunc->setLinkage(GlobalValue::WeakODRLinkage);
375 }
376 return CtorFunc;
377 }
378
instrumentModule(Module & M,DomTreeCallback DTCallback,PostDomTreeCallback PDTCallback)379 bool ModuleSanitizerCoverage::instrumentModule(
380 Module &M, DomTreeCallback DTCallback, PostDomTreeCallback PDTCallback) {
381 if (Options.CoverageType == SanitizerCoverageOptions::SCK_None)
382 return false;
383 if (Allowlist &&
384 !Allowlist->inSection("coverage", "src", M.getSourceFileName()))
385 return false;
386 if (Blocklist &&
387 Blocklist->inSection("coverage", "src", M.getSourceFileName()))
388 return false;
389 C = &(M.getContext());
390 DL = &M.getDataLayout();
391 CurModule = &M;
392 CurModuleUniqueId = getUniqueModuleId(CurModule);
393 TargetTriple = Triple(M.getTargetTriple());
394 FunctionGuardArray = nullptr;
395 Function8bitCounterArray = nullptr;
396 FunctionBoolArray = nullptr;
397 FunctionPCsArray = nullptr;
398 FunctionCFsArray = nullptr;
399 IntptrTy = Type::getIntNTy(*C, DL->getPointerSizeInBits());
400 IntptrPtrTy = PointerType::getUnqual(IntptrTy);
401 Type *VoidTy = Type::getVoidTy(*C);
402 IRBuilder<> IRB(*C);
403 Int128PtrTy = PointerType::getUnqual(IRB.getInt128Ty());
404 Int64PtrTy = PointerType::getUnqual(IRB.getInt64Ty());
405 Int16PtrTy = PointerType::getUnqual(IRB.getInt16Ty());
406 Int32PtrTy = PointerType::getUnqual(IRB.getInt32Ty());
407 Int8PtrTy = PointerType::getUnqual(IRB.getInt8Ty());
408 Int1PtrTy = PointerType::getUnqual(IRB.getInt1Ty());
409 Int64Ty = IRB.getInt64Ty();
410 Int32Ty = IRB.getInt32Ty();
411 Int16Ty = IRB.getInt16Ty();
412 Int8Ty = IRB.getInt8Ty();
413 Int1Ty = IRB.getInt1Ty();
414
415 SanCovTracePCIndir =
416 M.getOrInsertFunction(SanCovTracePCIndirName, VoidTy, IntptrTy);
417 // Make sure smaller parameters are zero-extended to i64 if required by the
418 // target ABI.
419 AttributeList SanCovTraceCmpZeroExtAL;
420 SanCovTraceCmpZeroExtAL =
421 SanCovTraceCmpZeroExtAL.addParamAttribute(*C, 0, Attribute::ZExt);
422 SanCovTraceCmpZeroExtAL =
423 SanCovTraceCmpZeroExtAL.addParamAttribute(*C, 1, Attribute::ZExt);
424
425 SanCovTraceCmpFunction[0] =
426 M.getOrInsertFunction(SanCovTraceCmp1, SanCovTraceCmpZeroExtAL, VoidTy,
427 IRB.getInt8Ty(), IRB.getInt8Ty());
428 SanCovTraceCmpFunction[1] =
429 M.getOrInsertFunction(SanCovTraceCmp2, SanCovTraceCmpZeroExtAL, VoidTy,
430 IRB.getInt16Ty(), IRB.getInt16Ty());
431 SanCovTraceCmpFunction[2] =
432 M.getOrInsertFunction(SanCovTraceCmp4, SanCovTraceCmpZeroExtAL, VoidTy,
433 IRB.getInt32Ty(), IRB.getInt32Ty());
434 SanCovTraceCmpFunction[3] =
435 M.getOrInsertFunction(SanCovTraceCmp8, VoidTy, Int64Ty, Int64Ty);
436
437 SanCovTraceConstCmpFunction[0] = M.getOrInsertFunction(
438 SanCovTraceConstCmp1, SanCovTraceCmpZeroExtAL, VoidTy, Int8Ty, Int8Ty);
439 SanCovTraceConstCmpFunction[1] = M.getOrInsertFunction(
440 SanCovTraceConstCmp2, SanCovTraceCmpZeroExtAL, VoidTy, Int16Ty, Int16Ty);
441 SanCovTraceConstCmpFunction[2] = M.getOrInsertFunction(
442 SanCovTraceConstCmp4, SanCovTraceCmpZeroExtAL, VoidTy, Int32Ty, Int32Ty);
443 SanCovTraceConstCmpFunction[3] =
444 M.getOrInsertFunction(SanCovTraceConstCmp8, VoidTy, Int64Ty, Int64Ty);
445
446 // Loads.
447 SanCovLoadFunction[0] = M.getOrInsertFunction(SanCovLoad1, VoidTy, Int8PtrTy);
448 SanCovLoadFunction[1] =
449 M.getOrInsertFunction(SanCovLoad2, VoidTy, Int16PtrTy);
450 SanCovLoadFunction[2] =
451 M.getOrInsertFunction(SanCovLoad4, VoidTy, Int32PtrTy);
452 SanCovLoadFunction[3] =
453 M.getOrInsertFunction(SanCovLoad8, VoidTy, Int64PtrTy);
454 SanCovLoadFunction[4] =
455 M.getOrInsertFunction(SanCovLoad16, VoidTy, Int128PtrTy);
456 // Stores.
457 SanCovStoreFunction[0] =
458 M.getOrInsertFunction(SanCovStore1, VoidTy, Int8PtrTy);
459 SanCovStoreFunction[1] =
460 M.getOrInsertFunction(SanCovStore2, VoidTy, Int16PtrTy);
461 SanCovStoreFunction[2] =
462 M.getOrInsertFunction(SanCovStore4, VoidTy, Int32PtrTy);
463 SanCovStoreFunction[3] =
464 M.getOrInsertFunction(SanCovStore8, VoidTy, Int64PtrTy);
465 SanCovStoreFunction[4] =
466 M.getOrInsertFunction(SanCovStore16, VoidTy, Int128PtrTy);
467
468 {
469 AttributeList AL;
470 AL = AL.addParamAttribute(*C, 0, Attribute::ZExt);
471 SanCovTraceDivFunction[0] =
472 M.getOrInsertFunction(SanCovTraceDiv4, AL, VoidTy, IRB.getInt32Ty());
473 }
474 SanCovTraceDivFunction[1] =
475 M.getOrInsertFunction(SanCovTraceDiv8, VoidTy, Int64Ty);
476 SanCovTraceGepFunction =
477 M.getOrInsertFunction(SanCovTraceGep, VoidTy, IntptrTy);
478 SanCovTraceSwitchFunction =
479 M.getOrInsertFunction(SanCovTraceSwitchName, VoidTy, Int64Ty, Int64PtrTy);
480
481 Constant *SanCovLowestStackConstant =
482 M.getOrInsertGlobal(SanCovLowestStackName, IntptrTy);
483 SanCovLowestStack = dyn_cast<GlobalVariable>(SanCovLowestStackConstant);
484 if (!SanCovLowestStack || SanCovLowestStack->getValueType() != IntptrTy) {
485 C->emitError(StringRef("'") + SanCovLowestStackName +
486 "' should not be declared by the user");
487 return true;
488 }
489 SanCovLowestStack->setThreadLocalMode(
490 GlobalValue::ThreadLocalMode::InitialExecTLSModel);
491 if (Options.StackDepth && !SanCovLowestStack->isDeclaration())
492 SanCovLowestStack->setInitializer(Constant::getAllOnesValue(IntptrTy));
493
494 SanCovTracePC = M.getOrInsertFunction(SanCovTracePCName, VoidTy);
495 SanCovTracePCGuard =
496 M.getOrInsertFunction(SanCovTracePCGuardName, VoidTy, Int32PtrTy);
497
498 for (auto &F : M)
499 instrumentFunction(F, DTCallback, PDTCallback);
500
501 Function *Ctor = nullptr;
502
503 if (FunctionGuardArray)
504 Ctor = CreateInitCallsForSections(M, SanCovModuleCtorTracePcGuardName,
505 SanCovTracePCGuardInitName, Int32Ty,
506 SanCovGuardsSectionName);
507 if (Function8bitCounterArray)
508 Ctor = CreateInitCallsForSections(M, SanCovModuleCtor8bitCountersName,
509 SanCov8bitCountersInitName, Int8Ty,
510 SanCovCountersSectionName);
511 if (FunctionBoolArray) {
512 Ctor = CreateInitCallsForSections(M, SanCovModuleCtorBoolFlagName,
513 SanCovBoolFlagInitName, Int1Ty,
514 SanCovBoolFlagSectionName);
515 }
516 if (Ctor && Options.PCTable) {
517 auto SecStartEnd = CreateSecStartEnd(M, SanCovPCsSectionName, IntptrTy);
518 FunctionCallee InitFunction = declareSanitizerInitFunction(
519 M, SanCovPCsInitName, {IntptrPtrTy, IntptrPtrTy});
520 IRBuilder<> IRBCtor(Ctor->getEntryBlock().getTerminator());
521 IRBCtor.CreateCall(InitFunction, {SecStartEnd.first, SecStartEnd.second});
522 }
523
524 if (Ctor && Options.CollectControlFlow) {
525 auto SecStartEnd = CreateSecStartEnd(M, SanCovCFsSectionName, IntptrTy);
526 FunctionCallee InitFunction = declareSanitizerInitFunction(
527 M, SanCovCFsInitName, {IntptrPtrTy, IntptrPtrTy});
528 IRBuilder<> IRBCtor(Ctor->getEntryBlock().getTerminator());
529 IRBCtor.CreateCall(InitFunction, {SecStartEnd.first, SecStartEnd.second});
530 }
531
532 appendToUsed(M, GlobalsToAppendToUsed);
533 appendToCompilerUsed(M, GlobalsToAppendToCompilerUsed);
534 return true;
535 }
536
537 // True if block has successors and it dominates all of them.
isFullDominator(const BasicBlock * BB,const DominatorTree * DT)538 static bool isFullDominator(const BasicBlock *BB, const DominatorTree *DT) {
539 if (succ_empty(BB))
540 return false;
541
542 return llvm::all_of(successors(BB), [&](const BasicBlock *SUCC) {
543 return DT->dominates(BB, SUCC);
544 });
545 }
546
547 // True if block has predecessors and it postdominates all of them.
isFullPostDominator(const BasicBlock * BB,const PostDominatorTree * PDT)548 static bool isFullPostDominator(const BasicBlock *BB,
549 const PostDominatorTree *PDT) {
550 if (pred_empty(BB))
551 return false;
552
553 return llvm::all_of(predecessors(BB), [&](const BasicBlock *PRED) {
554 return PDT->dominates(BB, PRED);
555 });
556 }
557
shouldInstrumentBlock(const Function & F,const BasicBlock * BB,const DominatorTree * DT,const PostDominatorTree * PDT,const SanitizerCoverageOptions & Options)558 static bool shouldInstrumentBlock(const Function &F, const BasicBlock *BB,
559 const DominatorTree *DT,
560 const PostDominatorTree *PDT,
561 const SanitizerCoverageOptions &Options) {
562 // Don't insert coverage for blocks containing nothing but unreachable: we
563 // will never call __sanitizer_cov() for them, so counting them in
564 // NumberOfInstrumentedBlocks() might complicate calculation of code coverage
565 // percentage. Also, unreachable instructions frequently have no debug
566 // locations.
567 if (isa<UnreachableInst>(BB->getFirstNonPHIOrDbgOrLifetime()))
568 return false;
569
570 // Don't insert coverage into blocks without a valid insertion point
571 // (catchswitch blocks).
572 if (BB->getFirstInsertionPt() == BB->end())
573 return false;
574
575 if (Options.NoPrune || &F.getEntryBlock() == BB)
576 return true;
577
578 if (Options.CoverageType == SanitizerCoverageOptions::SCK_Function &&
579 &F.getEntryBlock() != BB)
580 return false;
581
582 // Do not instrument full dominators, or full post-dominators with multiple
583 // predecessors.
584 return !isFullDominator(BB, DT)
585 && !(isFullPostDominator(BB, PDT) && !BB->getSinglePredecessor());
586 }
587
588
589 // Returns true iff From->To is a backedge.
590 // A twist here is that we treat From->To as a backedge if
591 // * To dominates From or
592 // * To->UniqueSuccessor dominates From
IsBackEdge(BasicBlock * From,BasicBlock * To,const DominatorTree * DT)593 static bool IsBackEdge(BasicBlock *From, BasicBlock *To,
594 const DominatorTree *DT) {
595 if (DT->dominates(To, From))
596 return true;
597 if (auto Next = To->getUniqueSuccessor())
598 if (DT->dominates(Next, From))
599 return true;
600 return false;
601 }
602
603 // Prunes uninteresting Cmp instrumentation:
604 // * CMP instructions that feed into loop backedge branch.
605 //
606 // Note that Cmp pruning is controlled by the same flag as the
607 // BB pruning.
IsInterestingCmp(ICmpInst * CMP,const DominatorTree * DT,const SanitizerCoverageOptions & Options)608 static bool IsInterestingCmp(ICmpInst *CMP, const DominatorTree *DT,
609 const SanitizerCoverageOptions &Options) {
610 if (!Options.NoPrune)
611 if (CMP->hasOneUse())
612 if (auto BR = dyn_cast<BranchInst>(CMP->user_back()))
613 for (BasicBlock *B : BR->successors())
614 if (IsBackEdge(BR->getParent(), B, DT))
615 return false;
616 return true;
617 }
618
instrumentFunction(Function & F,DomTreeCallback DTCallback,PostDomTreeCallback PDTCallback)619 void ModuleSanitizerCoverage::instrumentFunction(
620 Function &F, DomTreeCallback DTCallback, PostDomTreeCallback PDTCallback) {
621 if (F.empty())
622 return;
623 if (F.getName().find(".module_ctor") != std::string::npos)
624 return; // Should not instrument sanitizer init functions.
625 if (F.getName().startswith("__sanitizer_"))
626 return; // Don't instrument __sanitizer_* callbacks.
627 // Don't touch available_externally functions, their actual body is elewhere.
628 if (F.getLinkage() == GlobalValue::AvailableExternallyLinkage)
629 return;
630 // Don't instrument MSVC CRT configuration helpers. They may run before normal
631 // initialization.
632 if (F.getName() == "__local_stdio_printf_options" ||
633 F.getName() == "__local_stdio_scanf_options")
634 return;
635 if (isa<UnreachableInst>(F.getEntryBlock().getTerminator()))
636 return;
637 // Don't instrument functions using SEH for now. Splitting basic blocks like
638 // we do for coverage breaks WinEHPrepare.
639 // FIXME: Remove this when SEH no longer uses landingpad pattern matching.
640 if (F.hasPersonalityFn() &&
641 isAsynchronousEHPersonality(classifyEHPersonality(F.getPersonalityFn())))
642 return;
643 if (Allowlist && !Allowlist->inSection("coverage", "fun", F.getName()))
644 return;
645 if (Blocklist && Blocklist->inSection("coverage", "fun", F.getName()))
646 return;
647 if (F.hasFnAttribute(Attribute::NoSanitizeCoverage))
648 return;
649 if (Options.CoverageType >= SanitizerCoverageOptions::SCK_Edge)
650 SplitAllCriticalEdges(F, CriticalEdgeSplittingOptions().setIgnoreUnreachableDests());
651 SmallVector<Instruction *, 8> IndirCalls;
652 SmallVector<BasicBlock *, 16> BlocksToInstrument;
653 SmallVector<Instruction *, 8> CmpTraceTargets;
654 SmallVector<Instruction *, 8> SwitchTraceTargets;
655 SmallVector<BinaryOperator *, 8> DivTraceTargets;
656 SmallVector<GetElementPtrInst *, 8> GepTraceTargets;
657 SmallVector<LoadInst *, 8> Loads;
658 SmallVector<StoreInst *, 8> Stores;
659
660 const DominatorTree *DT = DTCallback(F);
661 const PostDominatorTree *PDT = PDTCallback(F);
662 bool IsLeafFunc = true;
663
664 for (auto &BB : F) {
665 if (shouldInstrumentBlock(F, &BB, DT, PDT, Options))
666 BlocksToInstrument.push_back(&BB);
667 for (auto &Inst : BB) {
668 if (Options.IndirectCalls) {
669 CallBase *CB = dyn_cast<CallBase>(&Inst);
670 if (CB && CB->isIndirectCall())
671 IndirCalls.push_back(&Inst);
672 }
673 if (Options.TraceCmp) {
674 if (ICmpInst *CMP = dyn_cast<ICmpInst>(&Inst))
675 if (IsInterestingCmp(CMP, DT, Options))
676 CmpTraceTargets.push_back(&Inst);
677 if (isa<SwitchInst>(&Inst))
678 SwitchTraceTargets.push_back(&Inst);
679 }
680 if (Options.TraceDiv)
681 if (BinaryOperator *BO = dyn_cast<BinaryOperator>(&Inst))
682 if (BO->getOpcode() == Instruction::SDiv ||
683 BO->getOpcode() == Instruction::UDiv)
684 DivTraceTargets.push_back(BO);
685 if (Options.TraceGep)
686 if (GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(&Inst))
687 GepTraceTargets.push_back(GEP);
688 if (Options.TraceLoads)
689 if (LoadInst *LI = dyn_cast<LoadInst>(&Inst))
690 Loads.push_back(LI);
691 if (Options.TraceStores)
692 if (StoreInst *SI = dyn_cast<StoreInst>(&Inst))
693 Stores.push_back(SI);
694 if (Options.StackDepth)
695 if (isa<InvokeInst>(Inst) ||
696 (isa<CallInst>(Inst) && !isa<IntrinsicInst>(Inst)))
697 IsLeafFunc = false;
698 }
699 }
700
701 if (Options.CollectControlFlow)
702 createFunctionControlFlow(F);
703
704 InjectCoverage(F, BlocksToInstrument, IsLeafFunc);
705 InjectCoverageForIndirectCalls(F, IndirCalls);
706 InjectTraceForCmp(F, CmpTraceTargets);
707 InjectTraceForSwitch(F, SwitchTraceTargets);
708 InjectTraceForDiv(F, DivTraceTargets);
709 InjectTraceForGep(F, GepTraceTargets);
710 InjectTraceForLoadsAndStores(F, Loads, Stores);
711 }
712
CreateFunctionLocalArrayInSection(size_t NumElements,Function & F,Type * Ty,const char * Section)713 GlobalVariable *ModuleSanitizerCoverage::CreateFunctionLocalArrayInSection(
714 size_t NumElements, Function &F, Type *Ty, const char *Section) {
715 ArrayType *ArrayTy = ArrayType::get(Ty, NumElements);
716 auto Array = new GlobalVariable(
717 *CurModule, ArrayTy, false, GlobalVariable::PrivateLinkage,
718 Constant::getNullValue(ArrayTy), "__sancov_gen_");
719
720 if (TargetTriple.supportsCOMDAT() &&
721 (TargetTriple.isOSBinFormatELF() || !F.isInterposable()))
722 if (auto Comdat = getOrCreateFunctionComdat(F, TargetTriple))
723 Array->setComdat(Comdat);
724 Array->setSection(getSectionName(Section));
725 Array->setAlignment(Align(DL->getTypeStoreSize(Ty).getFixedValue()));
726
727 // sancov_pcs parallels the other metadata section(s). Optimizers (e.g.
728 // GlobalOpt/ConstantMerge) may not discard sancov_pcs and the other
729 // section(s) as a unit, so we conservatively retain all unconditionally in
730 // the compiler.
731 //
732 // With comdat (COFF/ELF), the linker can guarantee the associated sections
733 // will be retained or discarded as a unit, so llvm.compiler.used is
734 // sufficient. Otherwise, conservatively make all of them retained by the
735 // linker.
736 if (Array->hasComdat())
737 GlobalsToAppendToCompilerUsed.push_back(Array);
738 else
739 GlobalsToAppendToUsed.push_back(Array);
740
741 return Array;
742 }
743
744 GlobalVariable *
CreatePCArray(Function & F,ArrayRef<BasicBlock * > AllBlocks)745 ModuleSanitizerCoverage::CreatePCArray(Function &F,
746 ArrayRef<BasicBlock *> AllBlocks) {
747 size_t N = AllBlocks.size();
748 assert(N);
749 SmallVector<Constant *, 32> PCs;
750 IRBuilder<> IRB(&*F.getEntryBlock().getFirstInsertionPt());
751 for (size_t i = 0; i < N; i++) {
752 if (&F.getEntryBlock() == AllBlocks[i]) {
753 PCs.push_back((Constant *)IRB.CreatePointerCast(&F, IntptrPtrTy));
754 PCs.push_back((Constant *)IRB.CreateIntToPtr(
755 ConstantInt::get(IntptrTy, 1), IntptrPtrTy));
756 } else {
757 PCs.push_back((Constant *)IRB.CreatePointerCast(
758 BlockAddress::get(AllBlocks[i]), IntptrPtrTy));
759 PCs.push_back(Constant::getNullValue(IntptrPtrTy));
760 }
761 }
762 auto *PCArray = CreateFunctionLocalArrayInSection(N * 2, F, IntptrPtrTy,
763 SanCovPCsSectionName);
764 PCArray->setInitializer(
765 ConstantArray::get(ArrayType::get(IntptrPtrTy, N * 2), PCs));
766 PCArray->setConstant(true);
767
768 return PCArray;
769 }
770
CreateFunctionLocalArrays(Function & F,ArrayRef<BasicBlock * > AllBlocks)771 void ModuleSanitizerCoverage::CreateFunctionLocalArrays(
772 Function &F, ArrayRef<BasicBlock *> AllBlocks) {
773 if (Options.TracePCGuard)
774 FunctionGuardArray = CreateFunctionLocalArrayInSection(
775 AllBlocks.size(), F, Int32Ty, SanCovGuardsSectionName);
776
777 if (Options.Inline8bitCounters)
778 Function8bitCounterArray = CreateFunctionLocalArrayInSection(
779 AllBlocks.size(), F, Int8Ty, SanCovCountersSectionName);
780 if (Options.InlineBoolFlag)
781 FunctionBoolArray = CreateFunctionLocalArrayInSection(
782 AllBlocks.size(), F, Int1Ty, SanCovBoolFlagSectionName);
783
784 if (Options.PCTable)
785 FunctionPCsArray = CreatePCArray(F, AllBlocks);
786 }
787
InjectCoverage(Function & F,ArrayRef<BasicBlock * > AllBlocks,bool IsLeafFunc)788 bool ModuleSanitizerCoverage::InjectCoverage(Function &F,
789 ArrayRef<BasicBlock *> AllBlocks,
790 bool IsLeafFunc) {
791 if (AllBlocks.empty()) return false;
792 CreateFunctionLocalArrays(F, AllBlocks);
793 for (size_t i = 0, N = AllBlocks.size(); i < N; i++)
794 InjectCoverageAtBlock(F, *AllBlocks[i], i, IsLeafFunc);
795 return true;
796 }
797
798 // On every indirect call we call a run-time function
799 // __sanitizer_cov_indir_call* with two parameters:
800 // - callee address,
801 // - global cache array that contains CacheSize pointers (zero-initialized).
802 // The cache is used to speed up recording the caller-callee pairs.
803 // The address of the caller is passed implicitly via caller PC.
804 // CacheSize is encoded in the name of the run-time function.
InjectCoverageForIndirectCalls(Function & F,ArrayRef<Instruction * > IndirCalls)805 void ModuleSanitizerCoverage::InjectCoverageForIndirectCalls(
806 Function &F, ArrayRef<Instruction *> IndirCalls) {
807 if (IndirCalls.empty())
808 return;
809 assert(Options.TracePC || Options.TracePCGuard ||
810 Options.Inline8bitCounters || Options.InlineBoolFlag);
811 for (auto *I : IndirCalls) {
812 IRBuilder<> IRB(I);
813 CallBase &CB = cast<CallBase>(*I);
814 Value *Callee = CB.getCalledOperand();
815 if (isa<InlineAsm>(Callee))
816 continue;
817 IRB.CreateCall(SanCovTracePCIndir, IRB.CreatePointerCast(Callee, IntptrTy));
818 }
819 }
820
821 // For every switch statement we insert a call:
822 // __sanitizer_cov_trace_switch(CondValue,
823 // {NumCases, ValueSizeInBits, Case0Value, Case1Value, Case2Value, ... })
824
InjectTraceForSwitch(Function &,ArrayRef<Instruction * > SwitchTraceTargets)825 void ModuleSanitizerCoverage::InjectTraceForSwitch(
826 Function &, ArrayRef<Instruction *> SwitchTraceTargets) {
827 for (auto *I : SwitchTraceTargets) {
828 if (SwitchInst *SI = dyn_cast<SwitchInst>(I)) {
829 IRBuilder<> IRB(I);
830 SmallVector<Constant *, 16> Initializers;
831 Value *Cond = SI->getCondition();
832 if (Cond->getType()->getScalarSizeInBits() >
833 Int64Ty->getScalarSizeInBits())
834 continue;
835 Initializers.push_back(ConstantInt::get(Int64Ty, SI->getNumCases()));
836 Initializers.push_back(
837 ConstantInt::get(Int64Ty, Cond->getType()->getScalarSizeInBits()));
838 if (Cond->getType()->getScalarSizeInBits() <
839 Int64Ty->getScalarSizeInBits())
840 Cond = IRB.CreateIntCast(Cond, Int64Ty, false);
841 for (auto It : SI->cases()) {
842 Constant *C = It.getCaseValue();
843 if (C->getType()->getScalarSizeInBits() <
844 Int64Ty->getScalarSizeInBits())
845 C = ConstantExpr::getCast(CastInst::ZExt, It.getCaseValue(), Int64Ty);
846 Initializers.push_back(C);
847 }
848 llvm::sort(drop_begin(Initializers, 2),
849 [](const Constant *A, const Constant *B) {
850 return cast<ConstantInt>(A)->getLimitedValue() <
851 cast<ConstantInt>(B)->getLimitedValue();
852 });
853 ArrayType *ArrayOfInt64Ty = ArrayType::get(Int64Ty, Initializers.size());
854 GlobalVariable *GV = new GlobalVariable(
855 *CurModule, ArrayOfInt64Ty, false, GlobalVariable::InternalLinkage,
856 ConstantArray::get(ArrayOfInt64Ty, Initializers),
857 "__sancov_gen_cov_switch_values");
858 IRB.CreateCall(SanCovTraceSwitchFunction,
859 {Cond, IRB.CreatePointerCast(GV, Int64PtrTy)});
860 }
861 }
862 }
863
InjectTraceForDiv(Function &,ArrayRef<BinaryOperator * > DivTraceTargets)864 void ModuleSanitizerCoverage::InjectTraceForDiv(
865 Function &, ArrayRef<BinaryOperator *> DivTraceTargets) {
866 for (auto *BO : DivTraceTargets) {
867 IRBuilder<> IRB(BO);
868 Value *A1 = BO->getOperand(1);
869 if (isa<ConstantInt>(A1)) continue;
870 if (!A1->getType()->isIntegerTy())
871 continue;
872 uint64_t TypeSize = DL->getTypeStoreSizeInBits(A1->getType());
873 int CallbackIdx = TypeSize == 32 ? 0 :
874 TypeSize == 64 ? 1 : -1;
875 if (CallbackIdx < 0) continue;
876 auto Ty = Type::getIntNTy(*C, TypeSize);
877 IRB.CreateCall(SanCovTraceDivFunction[CallbackIdx],
878 {IRB.CreateIntCast(A1, Ty, true)});
879 }
880 }
881
InjectTraceForGep(Function &,ArrayRef<GetElementPtrInst * > GepTraceTargets)882 void ModuleSanitizerCoverage::InjectTraceForGep(
883 Function &, ArrayRef<GetElementPtrInst *> GepTraceTargets) {
884 for (auto *GEP : GepTraceTargets) {
885 IRBuilder<> IRB(GEP);
886 for (Use &Idx : GEP->indices())
887 if (!isa<ConstantInt>(Idx) && Idx->getType()->isIntegerTy())
888 IRB.CreateCall(SanCovTraceGepFunction,
889 {IRB.CreateIntCast(Idx, IntptrTy, true)});
890 }
891 }
892
InjectTraceForLoadsAndStores(Function &,ArrayRef<LoadInst * > Loads,ArrayRef<StoreInst * > Stores)893 void ModuleSanitizerCoverage::InjectTraceForLoadsAndStores(
894 Function &, ArrayRef<LoadInst *> Loads, ArrayRef<StoreInst *> Stores) {
895 auto CallbackIdx = [&](Type *ElementTy) -> int {
896 uint64_t TypeSize = DL->getTypeStoreSizeInBits(ElementTy);
897 return TypeSize == 8 ? 0
898 : TypeSize == 16 ? 1
899 : TypeSize == 32 ? 2
900 : TypeSize == 64 ? 3
901 : TypeSize == 128 ? 4
902 : -1;
903 };
904 Type *PointerType[5] = {Int8PtrTy, Int16PtrTy, Int32PtrTy, Int64PtrTy,
905 Int128PtrTy};
906 for (auto *LI : Loads) {
907 IRBuilder<> IRB(LI);
908 auto Ptr = LI->getPointerOperand();
909 int Idx = CallbackIdx(LI->getType());
910 if (Idx < 0)
911 continue;
912 IRB.CreateCall(SanCovLoadFunction[Idx],
913 IRB.CreatePointerCast(Ptr, PointerType[Idx]));
914 }
915 for (auto *SI : Stores) {
916 IRBuilder<> IRB(SI);
917 auto Ptr = SI->getPointerOperand();
918 int Idx = CallbackIdx(SI->getValueOperand()->getType());
919 if (Idx < 0)
920 continue;
921 IRB.CreateCall(SanCovStoreFunction[Idx],
922 IRB.CreatePointerCast(Ptr, PointerType[Idx]));
923 }
924 }
925
InjectTraceForCmp(Function &,ArrayRef<Instruction * > CmpTraceTargets)926 void ModuleSanitizerCoverage::InjectTraceForCmp(
927 Function &, ArrayRef<Instruction *> CmpTraceTargets) {
928 for (auto *I : CmpTraceTargets) {
929 if (ICmpInst *ICMP = dyn_cast<ICmpInst>(I)) {
930 IRBuilder<> IRB(ICMP);
931 Value *A0 = ICMP->getOperand(0);
932 Value *A1 = ICMP->getOperand(1);
933 if (!A0->getType()->isIntegerTy())
934 continue;
935 uint64_t TypeSize = DL->getTypeStoreSizeInBits(A0->getType());
936 int CallbackIdx = TypeSize == 8 ? 0 :
937 TypeSize == 16 ? 1 :
938 TypeSize == 32 ? 2 :
939 TypeSize == 64 ? 3 : -1;
940 if (CallbackIdx < 0) continue;
941 // __sanitizer_cov_trace_cmp((type_size << 32) | predicate, A0, A1);
942 auto CallbackFunc = SanCovTraceCmpFunction[CallbackIdx];
943 bool FirstIsConst = isa<ConstantInt>(A0);
944 bool SecondIsConst = isa<ConstantInt>(A1);
945 // If both are const, then we don't need such a comparison.
946 if (FirstIsConst && SecondIsConst) continue;
947 // If only one is const, then make it the first callback argument.
948 if (FirstIsConst || SecondIsConst) {
949 CallbackFunc = SanCovTraceConstCmpFunction[CallbackIdx];
950 if (SecondIsConst)
951 std::swap(A0, A1);
952 }
953
954 auto Ty = Type::getIntNTy(*C, TypeSize);
955 IRB.CreateCall(CallbackFunc, {IRB.CreateIntCast(A0, Ty, true),
956 IRB.CreateIntCast(A1, Ty, true)});
957 }
958 }
959 }
960
InjectCoverageAtBlock(Function & F,BasicBlock & BB,size_t Idx,bool IsLeafFunc)961 void ModuleSanitizerCoverage::InjectCoverageAtBlock(Function &F, BasicBlock &BB,
962 size_t Idx,
963 bool IsLeafFunc) {
964 BasicBlock::iterator IP = BB.getFirstInsertionPt();
965 bool IsEntryBB = &BB == &F.getEntryBlock();
966 DebugLoc EntryLoc;
967 if (IsEntryBB) {
968 if (auto SP = F.getSubprogram())
969 EntryLoc = DILocation::get(SP->getContext(), SP->getScopeLine(), 0, SP);
970 // Keep static allocas and llvm.localescape calls in the entry block. Even
971 // if we aren't splitting the block, it's nice for allocas to be before
972 // calls.
973 IP = PrepareToSplitEntryBlock(BB, IP);
974 }
975
976 InstrumentationIRBuilder IRB(&*IP);
977 if (EntryLoc)
978 IRB.SetCurrentDebugLocation(EntryLoc);
979 if (Options.TracePC) {
980 IRB.CreateCall(SanCovTracePC)
981 ->setCannotMerge(); // gets the PC using GET_CALLER_PC.
982 }
983 if (Options.TracePCGuard) {
984 auto GuardPtr = IRB.CreateIntToPtr(
985 IRB.CreateAdd(IRB.CreatePointerCast(FunctionGuardArray, IntptrTy),
986 ConstantInt::get(IntptrTy, Idx * 4)),
987 Int32PtrTy);
988 IRB.CreateCall(SanCovTracePCGuard, GuardPtr)->setCannotMerge();
989 }
990 if (Options.Inline8bitCounters) {
991 auto CounterPtr = IRB.CreateGEP(
992 Function8bitCounterArray->getValueType(), Function8bitCounterArray,
993 {ConstantInt::get(IntptrTy, 0), ConstantInt::get(IntptrTy, Idx)});
994 auto Load = IRB.CreateLoad(Int8Ty, CounterPtr);
995 auto Inc = IRB.CreateAdd(Load, ConstantInt::get(Int8Ty, 1));
996 auto Store = IRB.CreateStore(Inc, CounterPtr);
997 SetNoSanitizeMetadata(Load);
998 SetNoSanitizeMetadata(Store);
999 }
1000 if (Options.InlineBoolFlag) {
1001 auto FlagPtr = IRB.CreateGEP(
1002 FunctionBoolArray->getValueType(), FunctionBoolArray,
1003 {ConstantInt::get(IntptrTy, 0), ConstantInt::get(IntptrTy, Idx)});
1004 auto Load = IRB.CreateLoad(Int1Ty, FlagPtr);
1005 auto ThenTerm =
1006 SplitBlockAndInsertIfThen(IRB.CreateIsNull(Load), &*IP, false);
1007 IRBuilder<> ThenIRB(ThenTerm);
1008 auto Store = ThenIRB.CreateStore(ConstantInt::getTrue(Int1Ty), FlagPtr);
1009 SetNoSanitizeMetadata(Load);
1010 SetNoSanitizeMetadata(Store);
1011 }
1012 if (Options.StackDepth && IsEntryBB && !IsLeafFunc) {
1013 // Check stack depth. If it's the deepest so far, record it.
1014 Module *M = F.getParent();
1015 Function *GetFrameAddr = Intrinsic::getDeclaration(
1016 M, Intrinsic::frameaddress,
1017 IRB.getInt8PtrTy(M->getDataLayout().getAllocaAddrSpace()));
1018 auto FrameAddrPtr =
1019 IRB.CreateCall(GetFrameAddr, {Constant::getNullValue(Int32Ty)});
1020 auto FrameAddrInt = IRB.CreatePtrToInt(FrameAddrPtr, IntptrTy);
1021 auto LowestStack = IRB.CreateLoad(IntptrTy, SanCovLowestStack);
1022 auto IsStackLower = IRB.CreateICmpULT(FrameAddrInt, LowestStack);
1023 auto ThenTerm = SplitBlockAndInsertIfThen(IsStackLower, &*IP, false);
1024 IRBuilder<> ThenIRB(ThenTerm);
1025 auto Store = ThenIRB.CreateStore(FrameAddrInt, SanCovLowestStack);
1026 SetNoSanitizeMetadata(LowestStack);
1027 SetNoSanitizeMetadata(Store);
1028 }
1029 }
1030
1031 std::string
getSectionName(const std::string & Section) const1032 ModuleSanitizerCoverage::getSectionName(const std::string &Section) const {
1033 if (TargetTriple.isOSBinFormatCOFF()) {
1034 if (Section == SanCovCountersSectionName)
1035 return ".SCOV$CM";
1036 if (Section == SanCovBoolFlagSectionName)
1037 return ".SCOV$BM";
1038 if (Section == SanCovPCsSectionName)
1039 return ".SCOVP$M";
1040 return ".SCOV$GM"; // For SanCovGuardsSectionName.
1041 }
1042 if (TargetTriple.isOSBinFormatMachO())
1043 return "__DATA,__" + Section;
1044 return "__" + Section;
1045 }
1046
1047 std::string
getSectionStart(const std::string & Section) const1048 ModuleSanitizerCoverage::getSectionStart(const std::string &Section) const {
1049 if (TargetTriple.isOSBinFormatMachO())
1050 return "\1section$start$__DATA$__" + Section;
1051 return "__start___" + Section;
1052 }
1053
1054 std::string
getSectionEnd(const std::string & Section) const1055 ModuleSanitizerCoverage::getSectionEnd(const std::string &Section) const {
1056 if (TargetTriple.isOSBinFormatMachO())
1057 return "\1section$end$__DATA$__" + Section;
1058 return "__stop___" + Section;
1059 }
1060
createFunctionControlFlow(Function & F)1061 void ModuleSanitizerCoverage::createFunctionControlFlow(Function &F) {
1062 SmallVector<Constant *, 32> CFs;
1063 IRBuilder<> IRB(&*F.getEntryBlock().getFirstInsertionPt());
1064
1065 for (auto &BB : F) {
1066 // blockaddress can not be used on function's entry block.
1067 if (&BB == &F.getEntryBlock())
1068 CFs.push_back((Constant *)IRB.CreatePointerCast(&F, IntptrPtrTy));
1069 else
1070 CFs.push_back((Constant *)IRB.CreatePointerCast(BlockAddress::get(&BB),
1071 IntptrPtrTy));
1072
1073 for (auto SuccBB : successors(&BB)) {
1074 assert(SuccBB != &F.getEntryBlock());
1075 CFs.push_back((Constant *)IRB.CreatePointerCast(BlockAddress::get(SuccBB),
1076 IntptrPtrTy));
1077 }
1078
1079 CFs.push_back((Constant *)Constant::getNullValue(IntptrPtrTy));
1080
1081 for (auto &Inst : BB) {
1082 if (CallBase *CB = dyn_cast<CallBase>(&Inst)) {
1083 if (CB->isIndirectCall()) {
1084 // TODO(navidem): handle indirect calls, for now mark its existence.
1085 CFs.push_back((Constant *)IRB.CreateIntToPtr(
1086 ConstantInt::get(IntptrTy, -1), IntptrPtrTy));
1087 } else {
1088 auto CalledF = CB->getCalledFunction();
1089 if (CalledF && !CalledF->isIntrinsic())
1090 CFs.push_back(
1091 (Constant *)IRB.CreatePointerCast(CalledF, IntptrPtrTy));
1092 }
1093 }
1094 }
1095
1096 CFs.push_back((Constant *)Constant::getNullValue(IntptrPtrTy));
1097 }
1098
1099 FunctionCFsArray = CreateFunctionLocalArrayInSection(
1100 CFs.size(), F, IntptrPtrTy, SanCovCFsSectionName);
1101 FunctionCFsArray->setInitializer(
1102 ConstantArray::get(ArrayType::get(IntptrPtrTy, CFs.size()), CFs));
1103 FunctionCFsArray->setConstant(true);
1104 }
1105