xref: /freebsd-src/contrib/llvm-project/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp (revision 0fca6ea1d4eea4c934cfff25ac9ee8ad6fe95583)
10b57cec5SDimitry Andric //===- AddressSanitizer.cpp - memory error detector -----------------------===//
20b57cec5SDimitry Andric //
30b57cec5SDimitry Andric // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
40b57cec5SDimitry Andric // See https://llvm.org/LICENSE.txt for license information.
50b57cec5SDimitry Andric // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
60b57cec5SDimitry Andric //
70b57cec5SDimitry Andric //===----------------------------------------------------------------------===//
80b57cec5SDimitry Andric //
94824e7fdSDimitry Andric // This file is a part of AddressSanitizer, an address basic correctness
104824e7fdSDimitry Andric // checker.
110b57cec5SDimitry Andric // Details of the algorithm:
120b57cec5SDimitry Andric //  https://github.com/google/sanitizers/wiki/AddressSanitizerAlgorithm
130b57cec5SDimitry Andric //
145ffd83dbSDimitry Andric // FIXME: This sanitizer does not yet handle scalable vectors
155ffd83dbSDimitry Andric //
160b57cec5SDimitry Andric //===----------------------------------------------------------------------===//
170b57cec5SDimitry Andric 
180b57cec5SDimitry Andric #include "llvm/Transforms/Instrumentation/AddressSanitizer.h"
190b57cec5SDimitry Andric #include "llvm/ADT/ArrayRef.h"
200b57cec5SDimitry Andric #include "llvm/ADT/DenseMap.h"
210b57cec5SDimitry Andric #include "llvm/ADT/DepthFirstIterator.h"
220b57cec5SDimitry Andric #include "llvm/ADT/SmallPtrSet.h"
230b57cec5SDimitry Andric #include "llvm/ADT/SmallVector.h"
240b57cec5SDimitry Andric #include "llvm/ADT/Statistic.h"
250b57cec5SDimitry Andric #include "llvm/ADT/StringExtras.h"
260b57cec5SDimitry Andric #include "llvm/ADT/StringRef.h"
270b57cec5SDimitry Andric #include "llvm/ADT/Twine.h"
28bdd1243dSDimitry Andric #include "llvm/Analysis/GlobalsModRef.h"
290b57cec5SDimitry Andric #include "llvm/Analysis/MemoryBuiltins.h"
30349cc55cSDimitry Andric #include "llvm/Analysis/StackSafetyAnalysis.h"
310b57cec5SDimitry Andric #include "llvm/Analysis/TargetLibraryInfo.h"
320b57cec5SDimitry Andric #include "llvm/Analysis/ValueTracking.h"
330b57cec5SDimitry Andric #include "llvm/BinaryFormat/MachO.h"
3481ad6265SDimitry Andric #include "llvm/Demangle/Demangle.h"
350b57cec5SDimitry Andric #include "llvm/IR/Argument.h"
360b57cec5SDimitry Andric #include "llvm/IR/Attributes.h"
370b57cec5SDimitry Andric #include "llvm/IR/BasicBlock.h"
380b57cec5SDimitry Andric #include "llvm/IR/Comdat.h"
390b57cec5SDimitry Andric #include "llvm/IR/Constant.h"
400b57cec5SDimitry Andric #include "llvm/IR/Constants.h"
410b57cec5SDimitry Andric #include "llvm/IR/DIBuilder.h"
420b57cec5SDimitry Andric #include "llvm/IR/DataLayout.h"
430b57cec5SDimitry Andric #include "llvm/IR/DebugInfoMetadata.h"
440b57cec5SDimitry Andric #include "llvm/IR/DebugLoc.h"
450b57cec5SDimitry Andric #include "llvm/IR/DerivedTypes.h"
46*0fca6ea1SDimitry Andric #include "llvm/IR/EHPersonalities.h"
470b57cec5SDimitry Andric #include "llvm/IR/Function.h"
480b57cec5SDimitry Andric #include "llvm/IR/GlobalAlias.h"
490b57cec5SDimitry Andric #include "llvm/IR/GlobalValue.h"
500b57cec5SDimitry Andric #include "llvm/IR/GlobalVariable.h"
510b57cec5SDimitry Andric #include "llvm/IR/IRBuilder.h"
520b57cec5SDimitry Andric #include "llvm/IR/InlineAsm.h"
530b57cec5SDimitry Andric #include "llvm/IR/InstVisitor.h"
540b57cec5SDimitry Andric #include "llvm/IR/InstrTypes.h"
550b57cec5SDimitry Andric #include "llvm/IR/Instruction.h"
560b57cec5SDimitry Andric #include "llvm/IR/Instructions.h"
570b57cec5SDimitry Andric #include "llvm/IR/IntrinsicInst.h"
580b57cec5SDimitry Andric #include "llvm/IR/Intrinsics.h"
590b57cec5SDimitry Andric #include "llvm/IR/LLVMContext.h"
600b57cec5SDimitry Andric #include "llvm/IR/MDBuilder.h"
610b57cec5SDimitry Andric #include "llvm/IR/Metadata.h"
620b57cec5SDimitry Andric #include "llvm/IR/Module.h"
630b57cec5SDimitry Andric #include "llvm/IR/Type.h"
640b57cec5SDimitry Andric #include "llvm/IR/Use.h"
650b57cec5SDimitry Andric #include "llvm/IR/Value.h"
660b57cec5SDimitry Andric #include "llvm/MC/MCSectionMachO.h"
670b57cec5SDimitry Andric #include "llvm/Support/Casting.h"
680b57cec5SDimitry Andric #include "llvm/Support/CommandLine.h"
690b57cec5SDimitry Andric #include "llvm/Support/Debug.h"
700b57cec5SDimitry Andric #include "llvm/Support/ErrorHandling.h"
710b57cec5SDimitry Andric #include "llvm/Support/MathExtras.h"
720b57cec5SDimitry Andric #include "llvm/Support/raw_ostream.h"
7306c3fb27SDimitry Andric #include "llvm/TargetParser/Triple.h"
740b57cec5SDimitry Andric #include "llvm/Transforms/Instrumentation.h"
755ffd83dbSDimitry Andric #include "llvm/Transforms/Instrumentation/AddressSanitizerCommon.h"
76fe6060f1SDimitry Andric #include "llvm/Transforms/Instrumentation/AddressSanitizerOptions.h"
770b57cec5SDimitry Andric #include "llvm/Transforms/Utils/ASanStackFrameLayout.h"
780b57cec5SDimitry Andric #include "llvm/Transforms/Utils/BasicBlockUtils.h"
790b57cec5SDimitry Andric #include "llvm/Transforms/Utils/Local.h"
800b57cec5SDimitry Andric #include "llvm/Transforms/Utils/ModuleUtils.h"
810b57cec5SDimitry Andric #include "llvm/Transforms/Utils/PromoteMemToReg.h"
820b57cec5SDimitry Andric #include <algorithm>
830b57cec5SDimitry Andric #include <cassert>
840b57cec5SDimitry Andric #include <cstddef>
850b57cec5SDimitry Andric #include <cstdint>
860b57cec5SDimitry Andric #include <iomanip>
870b57cec5SDimitry Andric #include <limits>
880b57cec5SDimitry Andric #include <sstream>
890b57cec5SDimitry Andric #include <string>
900b57cec5SDimitry Andric #include <tuple>
910b57cec5SDimitry Andric 
920b57cec5SDimitry Andric using namespace llvm;
930b57cec5SDimitry Andric 
940b57cec5SDimitry Andric #define DEBUG_TYPE "asan"
950b57cec5SDimitry Andric 
960b57cec5SDimitry Andric static const uint64_t kDefaultShadowScale = 3;
970b57cec5SDimitry Andric static const uint64_t kDefaultShadowOffset32 = 1ULL << 29;
980b57cec5SDimitry Andric static const uint64_t kDefaultShadowOffset64 = 1ULL << 44;
990b57cec5SDimitry Andric static const uint64_t kDynamicShadowSentinel =
1000b57cec5SDimitry Andric     std::numeric_limits<uint64_t>::max();
1010b57cec5SDimitry Andric static const uint64_t kSmallX86_64ShadowOffsetBase = 0x7FFFFFFF;  // < 2G.
1020b57cec5SDimitry Andric static const uint64_t kSmallX86_64ShadowOffsetAlignMask = ~0xFFFULL;
1030b57cec5SDimitry Andric static const uint64_t kLinuxKasan_ShadowOffset64 = 0xdffffc0000000000;
1040b57cec5SDimitry Andric static const uint64_t kPPC64_ShadowOffset64 = 1ULL << 44;
1050b57cec5SDimitry Andric static const uint64_t kSystemZ_ShadowOffset64 = 1ULL << 52;
106753f127fSDimitry Andric static const uint64_t kMIPS_ShadowOffsetN32 = 1ULL << 29;
1070b57cec5SDimitry Andric static const uint64_t kMIPS32_ShadowOffset32 = 0x0aaa0000;
1080b57cec5SDimitry Andric static const uint64_t kMIPS64_ShadowOffset64 = 1ULL << 37;
1090b57cec5SDimitry Andric static const uint64_t kAArch64_ShadowOffset64 = 1ULL << 36;
110bdd1243dSDimitry Andric static const uint64_t kLoongArch64_ShadowOffset64 = 1ULL << 46;
111*0fca6ea1SDimitry Andric static const uint64_t kRISCV64_ShadowOffset64 = kDynamicShadowSentinel;
1120b57cec5SDimitry Andric static const uint64_t kFreeBSD_ShadowOffset32 = 1ULL << 30;
1130b57cec5SDimitry Andric static const uint64_t kFreeBSD_ShadowOffset64 = 1ULL << 46;
114fcaf7f86SDimitry Andric static const uint64_t kFreeBSDAArch64_ShadowOffset64 = 1ULL << 47;
1151b00608bSDimitry Andric static const uint64_t kFreeBSDKasan_ShadowOffset64 = 0xdffff7c000000000;
1160b57cec5SDimitry Andric static const uint64_t kNetBSD_ShadowOffset32 = 1ULL << 30;
1170b57cec5SDimitry Andric static const uint64_t kNetBSD_ShadowOffset64 = 1ULL << 46;
1180b57cec5SDimitry Andric static const uint64_t kNetBSDKasan_ShadowOffset64 = 0xdfff900000000000;
11981ad6265SDimitry Andric static const uint64_t kPS_ShadowOffset64 = 1ULL << 40;
1200b57cec5SDimitry Andric static const uint64_t kWindowsShadowOffset32 = 3ULL << 28;
1210b57cec5SDimitry Andric static const uint64_t kEmscriptenShadowOffset = 0;
1220b57cec5SDimitry Andric 
1230b57cec5SDimitry Andric // The shadow memory space is dynamically allocated.
1240b57cec5SDimitry Andric static const uint64_t kWindowsShadowOffset64 = kDynamicShadowSentinel;
1250b57cec5SDimitry Andric 
1260b57cec5SDimitry Andric static const size_t kMinStackMallocSize = 1 << 6;   // 64B
1270b57cec5SDimitry Andric static const size_t kMaxStackMallocSize = 1 << 16;  // 64K
1280b57cec5SDimitry Andric static const uintptr_t kCurrentStackFrameMagic = 0x41B58AB3;
1290b57cec5SDimitry Andric static const uintptr_t kRetiredStackFrameMagic = 0x45E0360E;
1300b57cec5SDimitry Andric 
131e8d8bef9SDimitry Andric const char kAsanModuleCtorName[] = "asan.module_ctor";
132e8d8bef9SDimitry Andric const char kAsanModuleDtorName[] = "asan.module_dtor";
1330b57cec5SDimitry Andric static const uint64_t kAsanCtorAndDtorPriority = 1;
1348bcb0991SDimitry Andric // On Emscripten, the system needs more than one priorities for constructors.
1358bcb0991SDimitry Andric static const uint64_t kAsanEmscriptenCtorAndDtorPriority = 50;
136e8d8bef9SDimitry Andric const char kAsanReportErrorTemplate[] = "__asan_report_";
137e8d8bef9SDimitry Andric const char kAsanRegisterGlobalsName[] = "__asan_register_globals";
138e8d8bef9SDimitry Andric const char kAsanUnregisterGlobalsName[] = "__asan_unregister_globals";
139e8d8bef9SDimitry Andric const char kAsanRegisterImageGlobalsName[] = "__asan_register_image_globals";
140e8d8bef9SDimitry Andric const char kAsanUnregisterImageGlobalsName[] =
1410b57cec5SDimitry Andric     "__asan_unregister_image_globals";
142e8d8bef9SDimitry Andric const char kAsanRegisterElfGlobalsName[] = "__asan_register_elf_globals";
143e8d8bef9SDimitry Andric const char kAsanUnregisterElfGlobalsName[] = "__asan_unregister_elf_globals";
144e8d8bef9SDimitry Andric const char kAsanPoisonGlobalsName[] = "__asan_before_dynamic_init";
145e8d8bef9SDimitry Andric const char kAsanUnpoisonGlobalsName[] = "__asan_after_dynamic_init";
146e8d8bef9SDimitry Andric const char kAsanInitName[] = "__asan_init";
147e8d8bef9SDimitry Andric const char kAsanVersionCheckNamePrefix[] = "__asan_version_mismatch_check_v";
148e8d8bef9SDimitry Andric const char kAsanPtrCmp[] = "__sanitizer_ptr_cmp";
149e8d8bef9SDimitry Andric const char kAsanPtrSub[] = "__sanitizer_ptr_sub";
150e8d8bef9SDimitry Andric const char kAsanHandleNoReturnName[] = "__asan_handle_no_return";
1510b57cec5SDimitry Andric static const int kMaxAsanStackMallocSizeClass = 10;
152e8d8bef9SDimitry Andric const char kAsanStackMallocNameTemplate[] = "__asan_stack_malloc_";
153fe6060f1SDimitry Andric const char kAsanStackMallocAlwaysNameTemplate[] =
154fe6060f1SDimitry Andric     "__asan_stack_malloc_always_";
155e8d8bef9SDimitry Andric const char kAsanStackFreeNameTemplate[] = "__asan_stack_free_";
156e8d8bef9SDimitry Andric const char kAsanGenPrefix[] = "___asan_gen_";
157e8d8bef9SDimitry Andric const char kODRGenPrefix[] = "__odr_asan_gen_";
158e8d8bef9SDimitry Andric const char kSanCovGenPrefix[] = "__sancov_gen_";
159e8d8bef9SDimitry Andric const char kAsanSetShadowPrefix[] = "__asan_set_shadow_";
160e8d8bef9SDimitry Andric const char kAsanPoisonStackMemoryName[] = "__asan_poison_stack_memory";
161e8d8bef9SDimitry Andric const char kAsanUnpoisonStackMemoryName[] = "__asan_unpoison_stack_memory";
1620b57cec5SDimitry Andric 
1630b57cec5SDimitry Andric // ASan version script has __asan_* wildcard. Triple underscore prevents a
1640b57cec5SDimitry Andric // linker (gold) warning about attempting to export a local symbol.
165e8d8bef9SDimitry Andric const char kAsanGlobalsRegisteredFlagName[] = "___asan_globals_registered";
1660b57cec5SDimitry Andric 
167e8d8bef9SDimitry Andric const char kAsanOptionDetectUseAfterReturn[] =
1680b57cec5SDimitry Andric     "__asan_option_detect_stack_use_after_return";
1690b57cec5SDimitry Andric 
170e8d8bef9SDimitry Andric const char kAsanShadowMemoryDynamicAddress[] =
1710b57cec5SDimitry Andric     "__asan_shadow_memory_dynamic_address";
1720b57cec5SDimitry Andric 
173e8d8bef9SDimitry Andric const char kAsanAllocaPoison[] = "__asan_alloca_poison";
174e8d8bef9SDimitry Andric const char kAsanAllocasUnpoison[] = "__asan_allocas_unpoison";
1750b57cec5SDimitry Andric 
176fe6060f1SDimitry Andric const char kAMDGPUAddressSharedName[] = "llvm.amdgcn.is.shared";
177fe6060f1SDimitry Andric const char kAMDGPUAddressPrivateName[] = "llvm.amdgcn.is.private";
1781db9f3b2SDimitry Andric const char kAMDGPUBallotName[] = "llvm.amdgcn.ballot.i64";
1791db9f3b2SDimitry Andric const char kAMDGPUUnreachableName[] = "llvm.amdgcn.unreachable";
180fe6060f1SDimitry Andric 
1810b57cec5SDimitry Andric // Accesses sizes are powers of two: 1, 2, 4, 8, 16.
1820b57cec5SDimitry Andric static const size_t kNumberOfAccessSizes = 5;
1830b57cec5SDimitry Andric 
184349cc55cSDimitry Andric static const uint64_t kAllocaRzSize = 32;
185349cc55cSDimitry Andric 
186349cc55cSDimitry Andric // ASanAccessInfo implementation constants.
187349cc55cSDimitry Andric constexpr size_t kCompileKernelShift = 0;
188349cc55cSDimitry Andric constexpr size_t kCompileKernelMask = 0x1;
189349cc55cSDimitry Andric constexpr size_t kAccessSizeIndexShift = 1;
190349cc55cSDimitry Andric constexpr size_t kAccessSizeIndexMask = 0xf;
191349cc55cSDimitry Andric constexpr size_t kIsWriteShift = 5;
192349cc55cSDimitry Andric constexpr size_t kIsWriteMask = 0x1;
1930b57cec5SDimitry Andric 
1940b57cec5SDimitry Andric // Command-line flags.
1950b57cec5SDimitry Andric 
1960b57cec5SDimitry Andric static cl::opt<bool> ClEnableKasan(
1970b57cec5SDimitry Andric     "asan-kernel", cl::desc("Enable KernelAddressSanitizer instrumentation"),
1980b57cec5SDimitry Andric     cl::Hidden, cl::init(false));
1990b57cec5SDimitry Andric 
2000b57cec5SDimitry Andric static cl::opt<bool> ClRecover(
2010b57cec5SDimitry Andric     "asan-recover",
2020b57cec5SDimitry Andric     cl::desc("Enable recovery mode (continue-after-error)."),
2030b57cec5SDimitry Andric     cl::Hidden, cl::init(false));
2040b57cec5SDimitry Andric 
2058bcb0991SDimitry Andric static cl::opt<bool> ClInsertVersionCheck(
2068bcb0991SDimitry Andric     "asan-guard-against-version-mismatch",
2075f757f3fSDimitry Andric     cl::desc("Guard against compiler/runtime version mismatch."), cl::Hidden,
2085f757f3fSDimitry Andric     cl::init(true));
2098bcb0991SDimitry Andric 
2100b57cec5SDimitry Andric // This flag may need to be replaced with -f[no-]asan-reads.
2110b57cec5SDimitry Andric static cl::opt<bool> ClInstrumentReads("asan-instrument-reads",
2120b57cec5SDimitry Andric                                        cl::desc("instrument read instructions"),
2130b57cec5SDimitry Andric                                        cl::Hidden, cl::init(true));
2140b57cec5SDimitry Andric 
2150b57cec5SDimitry Andric static cl::opt<bool> ClInstrumentWrites(
2160b57cec5SDimitry Andric     "asan-instrument-writes", cl::desc("instrument write instructions"),
2170b57cec5SDimitry Andric     cl::Hidden, cl::init(true));
2180b57cec5SDimitry Andric 
219349cc55cSDimitry Andric static cl::opt<bool>
220297eecfbSDimitry Andric     ClUseStackSafety("asan-use-stack-safety", cl::Hidden, cl::init(true),
221349cc55cSDimitry Andric                      cl::Hidden, cl::desc("Use Stack Safety analysis results"),
222349cc55cSDimitry Andric                      cl::Optional);
223349cc55cSDimitry Andric 
2240b57cec5SDimitry Andric static cl::opt<bool> ClInstrumentAtomics(
2250b57cec5SDimitry Andric     "asan-instrument-atomics",
2260b57cec5SDimitry Andric     cl::desc("instrument atomic instructions (rmw, cmpxchg)"), cl::Hidden,
2270b57cec5SDimitry Andric     cl::init(true));
2280b57cec5SDimitry Andric 
2295ffd83dbSDimitry Andric static cl::opt<bool>
2305ffd83dbSDimitry Andric     ClInstrumentByval("asan-instrument-byval",
2315ffd83dbSDimitry Andric                       cl::desc("instrument byval call arguments"), cl::Hidden,
2325ffd83dbSDimitry Andric                       cl::init(true));
2335ffd83dbSDimitry Andric 
2340b57cec5SDimitry Andric static cl::opt<bool> ClAlwaysSlowPath(
2350b57cec5SDimitry Andric     "asan-always-slow-path",
2360b57cec5SDimitry Andric     cl::desc("use instrumentation with slow path for all accesses"), cl::Hidden,
2370b57cec5SDimitry Andric     cl::init(false));
2380b57cec5SDimitry Andric 
2390b57cec5SDimitry Andric static cl::opt<bool> ClForceDynamicShadow(
2400b57cec5SDimitry Andric     "asan-force-dynamic-shadow",
2410b57cec5SDimitry Andric     cl::desc("Load shadow address into a local variable for each function"),
2420b57cec5SDimitry Andric     cl::Hidden, cl::init(false));
2430b57cec5SDimitry Andric 
2440b57cec5SDimitry Andric static cl::opt<bool>
2450b57cec5SDimitry Andric     ClWithIfunc("asan-with-ifunc",
2460b57cec5SDimitry Andric                 cl::desc("Access dynamic shadow through an ifunc global on "
2470b57cec5SDimitry Andric                          "platforms that support this"),
2480b57cec5SDimitry Andric                 cl::Hidden, cl::init(true));
2490b57cec5SDimitry Andric 
2500b57cec5SDimitry Andric static cl::opt<bool> ClWithIfuncSuppressRemat(
2510b57cec5SDimitry Andric     "asan-with-ifunc-suppress-remat",
2520b57cec5SDimitry Andric     cl::desc("Suppress rematerialization of dynamic shadow address by passing "
2530b57cec5SDimitry Andric              "it through inline asm in prologue."),
2540b57cec5SDimitry Andric     cl::Hidden, cl::init(true));
2550b57cec5SDimitry Andric 
2560b57cec5SDimitry Andric // This flag limits the number of instructions to be instrumented
2570b57cec5SDimitry Andric // in any given BB. Normally, this should be set to unlimited (INT_MAX),
2580b57cec5SDimitry Andric // but due to http://llvm.org/bugs/show_bug.cgi?id=12652 we temporary
2590b57cec5SDimitry Andric // set it to 10000.
2600b57cec5SDimitry Andric static cl::opt<int> ClMaxInsnsToInstrumentPerBB(
2610b57cec5SDimitry Andric     "asan-max-ins-per-bb", cl::init(10000),
2620b57cec5SDimitry Andric     cl::desc("maximal number of instructions to instrument in any given BB"),
2630b57cec5SDimitry Andric     cl::Hidden);
2640b57cec5SDimitry Andric 
2650b57cec5SDimitry Andric // This flag may need to be replaced with -f[no]asan-stack.
2660b57cec5SDimitry Andric static cl::opt<bool> ClStack("asan-stack", cl::desc("Handle stack memory"),
2670b57cec5SDimitry Andric                              cl::Hidden, cl::init(true));
2680b57cec5SDimitry Andric static cl::opt<uint32_t> ClMaxInlinePoisoningSize(
2690b57cec5SDimitry Andric     "asan-max-inline-poisoning-size",
2700b57cec5SDimitry Andric     cl::desc(
2710b57cec5SDimitry Andric         "Inline shadow poisoning for blocks up to the given size in bytes."),
2720b57cec5SDimitry Andric     cl::Hidden, cl::init(64));
2730b57cec5SDimitry Andric 
274fe6060f1SDimitry Andric static cl::opt<AsanDetectStackUseAfterReturnMode> ClUseAfterReturn(
275fe6060f1SDimitry Andric     "asan-use-after-return",
276fe6060f1SDimitry Andric     cl::desc("Sets the mode of detection for stack-use-after-return."),
277fe6060f1SDimitry Andric     cl::values(
278fe6060f1SDimitry Andric         clEnumValN(AsanDetectStackUseAfterReturnMode::Never, "never",
279fe6060f1SDimitry Andric                    "Never detect stack use after return."),
280fe6060f1SDimitry Andric         clEnumValN(
281fe6060f1SDimitry Andric             AsanDetectStackUseAfterReturnMode::Runtime, "runtime",
282fe6060f1SDimitry Andric             "Detect stack use after return if "
283fe6060f1SDimitry Andric             "binary flag 'ASAN_OPTIONS=detect_stack_use_after_return' is set."),
284fe6060f1SDimitry Andric         clEnumValN(AsanDetectStackUseAfterReturnMode::Always, "always",
285fe6060f1SDimitry Andric                    "Always detect stack use after return.")),
286fe6060f1SDimitry Andric     cl::Hidden, cl::init(AsanDetectStackUseAfterReturnMode::Runtime));
2870b57cec5SDimitry Andric 
2880b57cec5SDimitry Andric static cl::opt<bool> ClRedzoneByvalArgs("asan-redzone-byval-args",
2890b57cec5SDimitry Andric                                         cl::desc("Create redzones for byval "
2900b57cec5SDimitry Andric                                                  "arguments (extra copy "
2910b57cec5SDimitry Andric                                                  "required)"), cl::Hidden,
2920b57cec5SDimitry Andric                                         cl::init(true));
2930b57cec5SDimitry Andric 
2940b57cec5SDimitry Andric static cl::opt<bool> ClUseAfterScope("asan-use-after-scope",
2950b57cec5SDimitry Andric                                      cl::desc("Check stack-use-after-scope"),
2960b57cec5SDimitry Andric                                      cl::Hidden, cl::init(false));
2970b57cec5SDimitry Andric 
2980b57cec5SDimitry Andric // This flag may need to be replaced with -f[no]asan-globals.
2990b57cec5SDimitry Andric static cl::opt<bool> ClGlobals("asan-globals",
3000b57cec5SDimitry Andric                                cl::desc("Handle global objects"), cl::Hidden,
3010b57cec5SDimitry Andric                                cl::init(true));
3020b57cec5SDimitry Andric 
3030b57cec5SDimitry Andric static cl::opt<bool> ClInitializers("asan-initialization-order",
3040b57cec5SDimitry Andric                                     cl::desc("Handle C++ initializer order"),
3050b57cec5SDimitry Andric                                     cl::Hidden, cl::init(true));
3060b57cec5SDimitry Andric 
3070b57cec5SDimitry Andric static cl::opt<bool> ClInvalidPointerPairs(
3080b57cec5SDimitry Andric     "asan-detect-invalid-pointer-pair",
3090b57cec5SDimitry Andric     cl::desc("Instrument <, <=, >, >=, - with pointer operands"), cl::Hidden,
3100b57cec5SDimitry Andric     cl::init(false));
3110b57cec5SDimitry Andric 
3120b57cec5SDimitry Andric static cl::opt<bool> ClInvalidPointerCmp(
3130b57cec5SDimitry Andric     "asan-detect-invalid-pointer-cmp",
3140b57cec5SDimitry Andric     cl::desc("Instrument <, <=, >, >= with pointer operands"), cl::Hidden,
3150b57cec5SDimitry Andric     cl::init(false));
3160b57cec5SDimitry Andric 
3170b57cec5SDimitry Andric static cl::opt<bool> ClInvalidPointerSub(
3180b57cec5SDimitry Andric     "asan-detect-invalid-pointer-sub",
3190b57cec5SDimitry Andric     cl::desc("Instrument - operations with pointer operands"), cl::Hidden,
3200b57cec5SDimitry Andric     cl::init(false));
3210b57cec5SDimitry Andric 
3220b57cec5SDimitry Andric static cl::opt<unsigned> ClRealignStack(
3230b57cec5SDimitry Andric     "asan-realign-stack",
3240b57cec5SDimitry Andric     cl::desc("Realign stack to the value of this flag (power of two)"),
3250b57cec5SDimitry Andric     cl::Hidden, cl::init(32));
3260b57cec5SDimitry Andric 
3270b57cec5SDimitry Andric static cl::opt<int> ClInstrumentationWithCallsThreshold(
3280b57cec5SDimitry Andric     "asan-instrumentation-with-call-threshold",
3295f757f3fSDimitry Andric     cl::desc("If the function being instrumented contains more than "
3300b57cec5SDimitry Andric              "this number of memory accesses, use callbacks instead of "
3310b57cec5SDimitry Andric              "inline checks (-1 means never use callbacks)."),
3320b57cec5SDimitry Andric     cl::Hidden, cl::init(7000));
3330b57cec5SDimitry Andric 
3340b57cec5SDimitry Andric static cl::opt<std::string> ClMemoryAccessCallbackPrefix(
3350b57cec5SDimitry Andric     "asan-memory-access-callback-prefix",
3360b57cec5SDimitry Andric     cl::desc("Prefix for memory access callbacks"), cl::Hidden,
3370b57cec5SDimitry Andric     cl::init("__asan_"));
3380b57cec5SDimitry Andric 
33981ad6265SDimitry Andric static cl::opt<bool> ClKasanMemIntrinCallbackPrefix(
34081ad6265SDimitry Andric     "asan-kernel-mem-intrinsic-prefix",
34181ad6265SDimitry Andric     cl::desc("Use prefix for memory intrinsics in KASAN mode"), cl::Hidden,
34281ad6265SDimitry Andric     cl::init(false));
34381ad6265SDimitry Andric 
3440b57cec5SDimitry Andric static cl::opt<bool>
3450b57cec5SDimitry Andric     ClInstrumentDynamicAllocas("asan-instrument-dynamic-allocas",
3460b57cec5SDimitry Andric                                cl::desc("instrument dynamic allocas"),
3470b57cec5SDimitry Andric                                cl::Hidden, cl::init(true));
3480b57cec5SDimitry Andric 
3490b57cec5SDimitry Andric static cl::opt<bool> ClSkipPromotableAllocas(
3500b57cec5SDimitry Andric     "asan-skip-promotable-allocas",
3510b57cec5SDimitry Andric     cl::desc("Do not instrument promotable allocas"), cl::Hidden,
3520b57cec5SDimitry Andric     cl::init(true));
3530b57cec5SDimitry Andric 
354bdd1243dSDimitry Andric static cl::opt<AsanCtorKind> ClConstructorKind(
355bdd1243dSDimitry Andric     "asan-constructor-kind",
356bdd1243dSDimitry Andric     cl::desc("Sets the ASan constructor kind"),
357bdd1243dSDimitry Andric     cl::values(clEnumValN(AsanCtorKind::None, "none", "No constructors"),
358bdd1243dSDimitry Andric                clEnumValN(AsanCtorKind::Global, "global",
359bdd1243dSDimitry Andric                           "Use global constructors")),
360bdd1243dSDimitry Andric     cl::init(AsanCtorKind::Global), cl::Hidden);
3610b57cec5SDimitry Andric // These flags allow to change the shadow mapping.
3620b57cec5SDimitry Andric // The shadow mapping looks like
3630b57cec5SDimitry Andric //    Shadow = (Mem >> scale) + offset
3640b57cec5SDimitry Andric 
3650b57cec5SDimitry Andric static cl::opt<int> ClMappingScale("asan-mapping-scale",
3660b57cec5SDimitry Andric                                    cl::desc("scale of asan shadow mapping"),
3670b57cec5SDimitry Andric                                    cl::Hidden, cl::init(0));
3680b57cec5SDimitry Andric 
3690b57cec5SDimitry Andric static cl::opt<uint64_t>
3700b57cec5SDimitry Andric     ClMappingOffset("asan-mapping-offset",
3710b57cec5SDimitry Andric                     cl::desc("offset of asan shadow mapping [EXPERIMENTAL]"),
3720b57cec5SDimitry Andric                     cl::Hidden, cl::init(0));
3730b57cec5SDimitry Andric 
3740b57cec5SDimitry Andric // Optimization flags. Not user visible, used mostly for testing
3750b57cec5SDimitry Andric // and benchmarking the tool.
3760b57cec5SDimitry Andric 
3770b57cec5SDimitry Andric static cl::opt<bool> ClOpt("asan-opt", cl::desc("Optimize instrumentation"),
3780b57cec5SDimitry Andric                            cl::Hidden, cl::init(true));
3790b57cec5SDimitry Andric 
380349cc55cSDimitry Andric static cl::opt<bool> ClOptimizeCallbacks("asan-optimize-callbacks",
381349cc55cSDimitry Andric                                          cl::desc("Optimize callbacks"),
382349cc55cSDimitry Andric                                          cl::Hidden, cl::init(false));
383349cc55cSDimitry Andric 
3840b57cec5SDimitry Andric static cl::opt<bool> ClOptSameTemp(
3850b57cec5SDimitry Andric     "asan-opt-same-temp", cl::desc("Instrument the same temp just once"),
3860b57cec5SDimitry Andric     cl::Hidden, cl::init(true));
3870b57cec5SDimitry Andric 
3880b57cec5SDimitry Andric static cl::opt<bool> ClOptGlobals("asan-opt-globals",
3890b57cec5SDimitry Andric                                   cl::desc("Don't instrument scalar globals"),
3900b57cec5SDimitry Andric                                   cl::Hidden, cl::init(true));
3910b57cec5SDimitry Andric 
3920b57cec5SDimitry Andric static cl::opt<bool> ClOptStack(
3930b57cec5SDimitry Andric     "asan-opt-stack", cl::desc("Don't instrument scalar stack variables"),
3940b57cec5SDimitry Andric     cl::Hidden, cl::init(false));
3950b57cec5SDimitry Andric 
3960b57cec5SDimitry Andric static cl::opt<bool> ClDynamicAllocaStack(
3970b57cec5SDimitry Andric     "asan-stack-dynamic-alloca",
3980b57cec5SDimitry Andric     cl::desc("Use dynamic alloca to represent stack variables"), cl::Hidden,
3990b57cec5SDimitry Andric     cl::init(true));
4000b57cec5SDimitry Andric 
4010b57cec5SDimitry Andric static cl::opt<uint32_t> ClForceExperiment(
4020b57cec5SDimitry Andric     "asan-force-experiment",
4030b57cec5SDimitry Andric     cl::desc("Force optimization experiment (for testing)"), cl::Hidden,
4040b57cec5SDimitry Andric     cl::init(0));
4050b57cec5SDimitry Andric 
4060b57cec5SDimitry Andric static cl::opt<bool>
4070b57cec5SDimitry Andric     ClUsePrivateAlias("asan-use-private-alias",
4080b57cec5SDimitry Andric                       cl::desc("Use private aliases for global variables"),
409bdd1243dSDimitry Andric                       cl::Hidden, cl::init(true));
4100b57cec5SDimitry Andric 
4110b57cec5SDimitry Andric static cl::opt<bool>
4120b57cec5SDimitry Andric     ClUseOdrIndicator("asan-use-odr-indicator",
4130b57cec5SDimitry Andric                       cl::desc("Use odr indicators to improve ODR reporting"),
414bdd1243dSDimitry Andric                       cl::Hidden, cl::init(true));
4150b57cec5SDimitry Andric 
4160b57cec5SDimitry Andric static cl::opt<bool>
4170b57cec5SDimitry Andric     ClUseGlobalsGC("asan-globals-live-support",
4180b57cec5SDimitry Andric                    cl::desc("Use linker features to support dead "
4190b57cec5SDimitry Andric                             "code stripping of globals"),
4200b57cec5SDimitry Andric                    cl::Hidden, cl::init(true));
4210b57cec5SDimitry Andric 
4220b57cec5SDimitry Andric // This is on by default even though there is a bug in gold:
4230b57cec5SDimitry Andric // https://sourceware.org/bugzilla/show_bug.cgi?id=19002
4240b57cec5SDimitry Andric static cl::opt<bool>
4250b57cec5SDimitry Andric     ClWithComdat("asan-with-comdat",
4260b57cec5SDimitry Andric                  cl::desc("Place ASan constructors in comdat sections"),
4270b57cec5SDimitry Andric                  cl::Hidden, cl::init(true));
4280b57cec5SDimitry Andric 
429fe6060f1SDimitry Andric static cl::opt<AsanDtorKind> ClOverrideDestructorKind(
430fe6060f1SDimitry Andric     "asan-destructor-kind",
431fe6060f1SDimitry Andric     cl::desc("Sets the ASan destructor kind. The default is to use the value "
432fe6060f1SDimitry Andric              "provided to the pass constructor"),
433fe6060f1SDimitry Andric     cl::values(clEnumValN(AsanDtorKind::None, "none", "No destructors"),
434fe6060f1SDimitry Andric                clEnumValN(AsanDtorKind::Global, "global",
435fe6060f1SDimitry Andric                           "Use global destructors")),
436fe6060f1SDimitry Andric     cl::init(AsanDtorKind::Invalid), cl::Hidden);
437fe6060f1SDimitry Andric 
4380b57cec5SDimitry Andric // Debug flags.
4390b57cec5SDimitry Andric 
4400b57cec5SDimitry Andric static cl::opt<int> ClDebug("asan-debug", cl::desc("debug"), cl::Hidden,
4410b57cec5SDimitry Andric                             cl::init(0));
4420b57cec5SDimitry Andric 
4430b57cec5SDimitry Andric static cl::opt<int> ClDebugStack("asan-debug-stack", cl::desc("debug stack"),
4440b57cec5SDimitry Andric                                  cl::Hidden, cl::init(0));
4450b57cec5SDimitry Andric 
4460b57cec5SDimitry Andric static cl::opt<std::string> ClDebugFunc("asan-debug-func", cl::Hidden,
4470b57cec5SDimitry Andric                                         cl::desc("Debug func"));
4480b57cec5SDimitry Andric 
4490b57cec5SDimitry Andric static cl::opt<int> ClDebugMin("asan-debug-min", cl::desc("Debug min inst"),
4500b57cec5SDimitry Andric                                cl::Hidden, cl::init(-1));
4510b57cec5SDimitry Andric 
4520b57cec5SDimitry Andric static cl::opt<int> ClDebugMax("asan-debug-max", cl::desc("Debug max inst"),
4530b57cec5SDimitry Andric                                cl::Hidden, cl::init(-1));
4540b57cec5SDimitry Andric 
4550b57cec5SDimitry Andric STATISTIC(NumInstrumentedReads, "Number of instrumented reads");
4560b57cec5SDimitry Andric STATISTIC(NumInstrumentedWrites, "Number of instrumented writes");
4570b57cec5SDimitry Andric STATISTIC(NumOptimizedAccessesToGlobalVar,
4580b57cec5SDimitry Andric           "Number of optimized accesses to global vars");
4590b57cec5SDimitry Andric STATISTIC(NumOptimizedAccessesToStackVar,
4600b57cec5SDimitry Andric           "Number of optimized accesses to stack vars");
4610b57cec5SDimitry Andric 
4620b57cec5SDimitry Andric namespace {
4630b57cec5SDimitry Andric 
4640b57cec5SDimitry Andric /// This struct defines the shadow mapping using the rule:
4650b57cec5SDimitry Andric ///   shadow = (mem >> Scale) ADD-or-OR Offset.
4660b57cec5SDimitry Andric /// If InGlobal is true, then
4670b57cec5SDimitry Andric ///   extern char __asan_shadow[];
4680b57cec5SDimitry Andric ///   shadow = (mem >> Scale) + &__asan_shadow
4690b57cec5SDimitry Andric struct ShadowMapping {
4700b57cec5SDimitry Andric   int Scale;
4710b57cec5SDimitry Andric   uint64_t Offset;
4720b57cec5SDimitry Andric   bool OrShadowOffset;
4730b57cec5SDimitry Andric   bool InGlobal;
4740b57cec5SDimitry Andric };
4750b57cec5SDimitry Andric 
4760b57cec5SDimitry Andric } // end anonymous namespace
4770b57cec5SDimitry Andric 
478349cc55cSDimitry Andric static ShadowMapping getShadowMapping(const Triple &TargetTriple, int LongSize,
4790b57cec5SDimitry Andric                                       bool IsKasan) {
4800b57cec5SDimitry Andric   bool IsAndroid = TargetTriple.isAndroid();
48181ad6265SDimitry Andric   bool IsIOS = TargetTriple.isiOS() || TargetTriple.isWatchOS() ||
48281ad6265SDimitry Andric                TargetTriple.isDriverKit();
483e8d8bef9SDimitry Andric   bool IsMacOS = TargetTriple.isMacOSX();
4840b57cec5SDimitry Andric   bool IsFreeBSD = TargetTriple.isOSFreeBSD();
4850b57cec5SDimitry Andric   bool IsNetBSD = TargetTriple.isOSNetBSD();
48681ad6265SDimitry Andric   bool IsPS = TargetTriple.isPS();
4870b57cec5SDimitry Andric   bool IsLinux = TargetTriple.isOSLinux();
4880b57cec5SDimitry Andric   bool IsPPC64 = TargetTriple.getArch() == Triple::ppc64 ||
4890b57cec5SDimitry Andric                  TargetTriple.getArch() == Triple::ppc64le;
4900b57cec5SDimitry Andric   bool IsSystemZ = TargetTriple.getArch() == Triple::systemz;
4910b57cec5SDimitry Andric   bool IsX86_64 = TargetTriple.getArch() == Triple::x86_64;
492753f127fSDimitry Andric   bool IsMIPSN32ABI = TargetTriple.getEnvironment() == Triple::GNUABIN32;
4930b57cec5SDimitry Andric   bool IsMIPS32 = TargetTriple.isMIPS32();
4940b57cec5SDimitry Andric   bool IsMIPS64 = TargetTriple.isMIPS64();
4950b57cec5SDimitry Andric   bool IsArmOrThumb = TargetTriple.isARM() || TargetTriple.isThumb();
4965f757f3fSDimitry Andric   bool IsAArch64 = TargetTriple.getArch() == Triple::aarch64 ||
4975f757f3fSDimitry Andric                    TargetTriple.getArch() == Triple::aarch64_be;
49806c3fb27SDimitry Andric   bool IsLoongArch64 = TargetTriple.isLoongArch64();
499e8d8bef9SDimitry Andric   bool IsRISCV64 = TargetTriple.getArch() == Triple::riscv64;
5000b57cec5SDimitry Andric   bool IsWindows = TargetTriple.isOSWindows();
5010b57cec5SDimitry Andric   bool IsFuchsia = TargetTriple.isOSFuchsia();
5020b57cec5SDimitry Andric   bool IsEmscripten = TargetTriple.isOSEmscripten();
503fe6060f1SDimitry Andric   bool IsAMDGPU = TargetTriple.isAMDGPU();
5040b57cec5SDimitry Andric 
5050b57cec5SDimitry Andric   ShadowMapping Mapping;
5060b57cec5SDimitry Andric 
507fe6060f1SDimitry Andric   Mapping.Scale = kDefaultShadowScale;
5080b57cec5SDimitry Andric   if (ClMappingScale.getNumOccurrences() > 0) {
5090b57cec5SDimitry Andric     Mapping.Scale = ClMappingScale;
5100b57cec5SDimitry Andric   }
5110b57cec5SDimitry Andric 
5120b57cec5SDimitry Andric   if (LongSize == 32) {
5130b57cec5SDimitry Andric     if (IsAndroid)
5140b57cec5SDimitry Andric       Mapping.Offset = kDynamicShadowSentinel;
515753f127fSDimitry Andric     else if (IsMIPSN32ABI)
516753f127fSDimitry Andric       Mapping.Offset = kMIPS_ShadowOffsetN32;
5170b57cec5SDimitry Andric     else if (IsMIPS32)
5180b57cec5SDimitry Andric       Mapping.Offset = kMIPS32_ShadowOffset32;
5190b57cec5SDimitry Andric     else if (IsFreeBSD)
5200b57cec5SDimitry Andric       Mapping.Offset = kFreeBSD_ShadowOffset32;
5210b57cec5SDimitry Andric     else if (IsNetBSD)
5220b57cec5SDimitry Andric       Mapping.Offset = kNetBSD_ShadowOffset32;
5230b57cec5SDimitry Andric     else if (IsIOS)
5240b57cec5SDimitry Andric       Mapping.Offset = kDynamicShadowSentinel;
5250b57cec5SDimitry Andric     else if (IsWindows)
5260b57cec5SDimitry Andric       Mapping.Offset = kWindowsShadowOffset32;
5270b57cec5SDimitry Andric     else if (IsEmscripten)
5280b57cec5SDimitry Andric       Mapping.Offset = kEmscriptenShadowOffset;
5290b57cec5SDimitry Andric     else
5300b57cec5SDimitry Andric       Mapping.Offset = kDefaultShadowOffset32;
5310b57cec5SDimitry Andric   } else {  // LongSize == 64
5320b57cec5SDimitry Andric     // Fuchsia is always PIE, which means that the beginning of the address
5330b57cec5SDimitry Andric     // space is always available.
5340b57cec5SDimitry Andric     if (IsFuchsia)
5350b57cec5SDimitry Andric       Mapping.Offset = 0;
5360b57cec5SDimitry Andric     else if (IsPPC64)
5370b57cec5SDimitry Andric       Mapping.Offset = kPPC64_ShadowOffset64;
5380b57cec5SDimitry Andric     else if (IsSystemZ)
5390b57cec5SDimitry Andric       Mapping.Offset = kSystemZ_ShadowOffset64;
540fcaf7f86SDimitry Andric     else if (IsFreeBSD && IsAArch64)
541fcaf7f86SDimitry Andric         Mapping.Offset = kFreeBSDAArch64_ShadowOffset64;
5421b00608bSDimitry Andric     else if (IsFreeBSD && !IsMIPS64) {
5431b00608bSDimitry Andric       if (IsKasan)
5441b00608bSDimitry Andric         Mapping.Offset = kFreeBSDKasan_ShadowOffset64;
5451b00608bSDimitry Andric       else
5460b57cec5SDimitry Andric         Mapping.Offset = kFreeBSD_ShadowOffset64;
5471b00608bSDimitry Andric     } else if (IsNetBSD) {
5480b57cec5SDimitry Andric       if (IsKasan)
5490b57cec5SDimitry Andric         Mapping.Offset = kNetBSDKasan_ShadowOffset64;
5500b57cec5SDimitry Andric       else
5510b57cec5SDimitry Andric         Mapping.Offset = kNetBSD_ShadowOffset64;
55281ad6265SDimitry Andric     } else if (IsPS)
55381ad6265SDimitry Andric       Mapping.Offset = kPS_ShadowOffset64;
5540b57cec5SDimitry Andric     else if (IsLinux && IsX86_64) {
5550b57cec5SDimitry Andric       if (IsKasan)
5560b57cec5SDimitry Andric         Mapping.Offset = kLinuxKasan_ShadowOffset64;
5570b57cec5SDimitry Andric       else
5580b57cec5SDimitry Andric         Mapping.Offset = (kSmallX86_64ShadowOffsetBase &
5590b57cec5SDimitry Andric                           (kSmallX86_64ShadowOffsetAlignMask << Mapping.Scale));
5600b57cec5SDimitry Andric     } else if (IsWindows && IsX86_64) {
5610b57cec5SDimitry Andric       Mapping.Offset = kWindowsShadowOffset64;
5620b57cec5SDimitry Andric     } else if (IsMIPS64)
5630b57cec5SDimitry Andric       Mapping.Offset = kMIPS64_ShadowOffset64;
5640b57cec5SDimitry Andric     else if (IsIOS)
5650b57cec5SDimitry Andric       Mapping.Offset = kDynamicShadowSentinel;
566e8d8bef9SDimitry Andric     else if (IsMacOS && IsAArch64)
567e8d8bef9SDimitry Andric       Mapping.Offset = kDynamicShadowSentinel;
5680b57cec5SDimitry Andric     else if (IsAArch64)
5690b57cec5SDimitry Andric       Mapping.Offset = kAArch64_ShadowOffset64;
570bdd1243dSDimitry Andric     else if (IsLoongArch64)
571bdd1243dSDimitry Andric       Mapping.Offset = kLoongArch64_ShadowOffset64;
572e8d8bef9SDimitry Andric     else if (IsRISCV64)
573e8d8bef9SDimitry Andric       Mapping.Offset = kRISCV64_ShadowOffset64;
574fe6060f1SDimitry Andric     else if (IsAMDGPU)
575fe6060f1SDimitry Andric       Mapping.Offset = (kSmallX86_64ShadowOffsetBase &
576fe6060f1SDimitry Andric                         (kSmallX86_64ShadowOffsetAlignMask << Mapping.Scale));
5770b57cec5SDimitry Andric     else
5780b57cec5SDimitry Andric       Mapping.Offset = kDefaultShadowOffset64;
5790b57cec5SDimitry Andric   }
5800b57cec5SDimitry Andric 
5810b57cec5SDimitry Andric   if (ClForceDynamicShadow) {
5820b57cec5SDimitry Andric     Mapping.Offset = kDynamicShadowSentinel;
5830b57cec5SDimitry Andric   }
5840b57cec5SDimitry Andric 
5850b57cec5SDimitry Andric   if (ClMappingOffset.getNumOccurrences() > 0) {
5860b57cec5SDimitry Andric     Mapping.Offset = ClMappingOffset;
5870b57cec5SDimitry Andric   }
5880b57cec5SDimitry Andric 
5890b57cec5SDimitry Andric   // OR-ing shadow offset if more efficient (at least on x86) if the offset
590bdd1243dSDimitry Andric   // is a power of two, but on ppc64 and loongarch64 we have to use add since
591bdd1243dSDimitry Andric   // the shadow offset is not necessarily 1/8-th of the address space.  On
592bdd1243dSDimitry Andric   // SystemZ, we could OR the constant in a single instruction, but it's more
5930b57cec5SDimitry Andric   // efficient to load it once and use indexed addressing.
59481ad6265SDimitry Andric   Mapping.OrShadowOffset = !IsAArch64 && !IsPPC64 && !IsSystemZ && !IsPS &&
595bdd1243dSDimitry Andric                            !IsRISCV64 && !IsLoongArch64 &&
5960b57cec5SDimitry Andric                            !(Mapping.Offset & (Mapping.Offset - 1)) &&
5970b57cec5SDimitry Andric                            Mapping.Offset != kDynamicShadowSentinel;
5980b57cec5SDimitry Andric   bool IsAndroidWithIfuncSupport =
5990b57cec5SDimitry Andric       IsAndroid && !TargetTriple.isAndroidVersionLT(21);
6000b57cec5SDimitry Andric   Mapping.InGlobal = ClWithIfunc && IsAndroidWithIfuncSupport && IsArmOrThumb;
6010b57cec5SDimitry Andric 
6020b57cec5SDimitry Andric   return Mapping;
6030b57cec5SDimitry Andric }
6040b57cec5SDimitry Andric 
605349cc55cSDimitry Andric namespace llvm {
606349cc55cSDimitry Andric void getAddressSanitizerParams(const Triple &TargetTriple, int LongSize,
607349cc55cSDimitry Andric                                bool IsKasan, uint64_t *ShadowBase,
608349cc55cSDimitry Andric                                int *MappingScale, bool *OrShadowOffset) {
609349cc55cSDimitry Andric   auto Mapping = getShadowMapping(TargetTriple, LongSize, IsKasan);
610349cc55cSDimitry Andric   *ShadowBase = Mapping.Offset;
611349cc55cSDimitry Andric   *MappingScale = Mapping.Scale;
612349cc55cSDimitry Andric   *OrShadowOffset = Mapping.OrShadowOffset;
613349cc55cSDimitry Andric }
614349cc55cSDimitry Andric 
615349cc55cSDimitry Andric ASanAccessInfo::ASanAccessInfo(int32_t Packed)
616349cc55cSDimitry Andric     : Packed(Packed),
617349cc55cSDimitry Andric       AccessSizeIndex((Packed >> kAccessSizeIndexShift) & kAccessSizeIndexMask),
618349cc55cSDimitry Andric       IsWrite((Packed >> kIsWriteShift) & kIsWriteMask),
619349cc55cSDimitry Andric       CompileKernel((Packed >> kCompileKernelShift) & kCompileKernelMask) {}
620349cc55cSDimitry Andric 
621349cc55cSDimitry Andric ASanAccessInfo::ASanAccessInfo(bool IsWrite, bool CompileKernel,
622349cc55cSDimitry Andric                                uint8_t AccessSizeIndex)
623349cc55cSDimitry Andric     : Packed((IsWrite << kIsWriteShift) +
624349cc55cSDimitry Andric              (CompileKernel << kCompileKernelShift) +
625349cc55cSDimitry Andric              (AccessSizeIndex << kAccessSizeIndexShift)),
626349cc55cSDimitry Andric       AccessSizeIndex(AccessSizeIndex), IsWrite(IsWrite),
627349cc55cSDimitry Andric       CompileKernel(CompileKernel) {}
628349cc55cSDimitry Andric 
629349cc55cSDimitry Andric } // namespace llvm
630349cc55cSDimitry Andric 
6315ffd83dbSDimitry Andric static uint64_t getRedzoneSizeForScale(int MappingScale) {
6320b57cec5SDimitry Andric   // Redzone used for stack and globals is at least 32 bytes.
6330b57cec5SDimitry Andric   // For scales 6 and 7, the redzone has to be 64 and 128 bytes respectively.
6340b57cec5SDimitry Andric   return std::max(32U, 1U << MappingScale);
6350b57cec5SDimitry Andric }
6360b57cec5SDimitry Andric 
6378bcb0991SDimitry Andric static uint64_t GetCtorAndDtorPriority(Triple &TargetTriple) {
6388bcb0991SDimitry Andric   if (TargetTriple.isOSEmscripten()) {
6398bcb0991SDimitry Andric     return kAsanEmscriptenCtorAndDtorPriority;
6408bcb0991SDimitry Andric   } else {
6418bcb0991SDimitry Andric     return kAsanCtorAndDtorPriority;
6428bcb0991SDimitry Andric   }
6438bcb0991SDimitry Andric }
6448bcb0991SDimitry Andric 
6450b57cec5SDimitry Andric namespace {
646*0fca6ea1SDimitry Andric /// Helper RAII class to post-process inserted asan runtime calls during a
647*0fca6ea1SDimitry Andric /// pass on a single Function. Upon end of scope, detects and applies the
648*0fca6ea1SDimitry Andric /// required funclet OpBundle.
649*0fca6ea1SDimitry Andric class RuntimeCallInserter {
650*0fca6ea1SDimitry Andric   Function *OwnerFn = nullptr;
651*0fca6ea1SDimitry Andric   bool TrackInsertedCalls = false;
652*0fca6ea1SDimitry Andric   SmallVector<CallInst *> InsertedCalls;
653*0fca6ea1SDimitry Andric 
654*0fca6ea1SDimitry Andric public:
655*0fca6ea1SDimitry Andric   RuntimeCallInserter(Function &Fn) : OwnerFn(&Fn) {
656*0fca6ea1SDimitry Andric     if (Fn.hasPersonalityFn()) {
657*0fca6ea1SDimitry Andric       auto Personality = classifyEHPersonality(Fn.getPersonalityFn());
658*0fca6ea1SDimitry Andric       if (isScopedEHPersonality(Personality))
659*0fca6ea1SDimitry Andric         TrackInsertedCalls = true;
660*0fca6ea1SDimitry Andric     }
661*0fca6ea1SDimitry Andric   }
662*0fca6ea1SDimitry Andric 
663*0fca6ea1SDimitry Andric   ~RuntimeCallInserter() {
664*0fca6ea1SDimitry Andric     if (InsertedCalls.empty())
665*0fca6ea1SDimitry Andric       return;
666*0fca6ea1SDimitry Andric     assert(TrackInsertedCalls && "Calls were wrongly tracked");
667*0fca6ea1SDimitry Andric 
668*0fca6ea1SDimitry Andric     DenseMap<BasicBlock *, ColorVector> BlockColors = colorEHFunclets(*OwnerFn);
669*0fca6ea1SDimitry Andric     for (CallInst *CI : InsertedCalls) {
670*0fca6ea1SDimitry Andric       BasicBlock *BB = CI->getParent();
671*0fca6ea1SDimitry Andric       assert(BB && "Instruction doesn't belong to a BasicBlock");
672*0fca6ea1SDimitry Andric       assert(BB->getParent() == OwnerFn &&
673*0fca6ea1SDimitry Andric              "Instruction doesn't belong to the expected Function!");
674*0fca6ea1SDimitry Andric 
675*0fca6ea1SDimitry Andric       ColorVector &Colors = BlockColors[BB];
676*0fca6ea1SDimitry Andric       // funclet opbundles are only valid in monochromatic BBs.
677*0fca6ea1SDimitry Andric       // Note that unreachable BBs are seen as colorless by colorEHFunclets()
678*0fca6ea1SDimitry Andric       // and will be DCE'ed later.
679*0fca6ea1SDimitry Andric       if (Colors.empty())
680*0fca6ea1SDimitry Andric         continue;
681*0fca6ea1SDimitry Andric       if (Colors.size() != 1) {
682*0fca6ea1SDimitry Andric         OwnerFn->getContext().emitError(
683*0fca6ea1SDimitry Andric             "Instruction's BasicBlock is not monochromatic");
684*0fca6ea1SDimitry Andric         continue;
685*0fca6ea1SDimitry Andric       }
686*0fca6ea1SDimitry Andric 
687*0fca6ea1SDimitry Andric       BasicBlock *Color = Colors.front();
688*0fca6ea1SDimitry Andric       Instruction *EHPad = Color->getFirstNonPHI();
689*0fca6ea1SDimitry Andric 
690*0fca6ea1SDimitry Andric       if (EHPad && EHPad->isEHPad()) {
691*0fca6ea1SDimitry Andric         // Replace CI with a clone with an added funclet OperandBundle
692*0fca6ea1SDimitry Andric         OperandBundleDef OB("funclet", EHPad);
693*0fca6ea1SDimitry Andric         auto *NewCall =
694*0fca6ea1SDimitry Andric             CallBase::addOperandBundle(CI, LLVMContext::OB_funclet, OB, CI);
695*0fca6ea1SDimitry Andric         NewCall->copyMetadata(*CI);
696*0fca6ea1SDimitry Andric         CI->replaceAllUsesWith(NewCall);
697*0fca6ea1SDimitry Andric         CI->eraseFromParent();
698*0fca6ea1SDimitry Andric       }
699*0fca6ea1SDimitry Andric     }
700*0fca6ea1SDimitry Andric   }
701*0fca6ea1SDimitry Andric 
702*0fca6ea1SDimitry Andric   CallInst *createRuntimeCall(IRBuilder<> &IRB, FunctionCallee Callee,
703*0fca6ea1SDimitry Andric                               ArrayRef<Value *> Args = {},
704*0fca6ea1SDimitry Andric                               const Twine &Name = "") {
705*0fca6ea1SDimitry Andric     assert(IRB.GetInsertBlock()->getParent() == OwnerFn);
706*0fca6ea1SDimitry Andric 
707*0fca6ea1SDimitry Andric     CallInst *Inst = IRB.CreateCall(Callee, Args, Name, nullptr);
708*0fca6ea1SDimitry Andric     if (TrackInsertedCalls)
709*0fca6ea1SDimitry Andric       InsertedCalls.push_back(Inst);
710*0fca6ea1SDimitry Andric     return Inst;
711*0fca6ea1SDimitry Andric   }
712*0fca6ea1SDimitry Andric };
7130b57cec5SDimitry Andric 
7140b57cec5SDimitry Andric /// AddressSanitizer: instrument the code in module to find memory bugs.
7150b57cec5SDimitry Andric struct AddressSanitizer {
71681ad6265SDimitry Andric   AddressSanitizer(Module &M, const StackSafetyGlobalInfo *SSGI,
7175f757f3fSDimitry Andric                    int InstrumentationWithCallsThreshold,
7185f757f3fSDimitry Andric                    uint32_t MaxInlinePoisoningSize, bool CompileKernel = false,
7195f757f3fSDimitry Andric                    bool Recover = false, bool UseAfterScope = false,
720fe6060f1SDimitry Andric                    AsanDetectStackUseAfterReturnMode UseAfterReturn =
721fe6060f1SDimitry Andric                        AsanDetectStackUseAfterReturnMode::Runtime)
7225ffd83dbSDimitry Andric       : CompileKernel(ClEnableKasan.getNumOccurrences() > 0 ? ClEnableKasan
7235ffd83dbSDimitry Andric                                                             : CompileKernel),
7245ffd83dbSDimitry Andric         Recover(ClRecover.getNumOccurrences() > 0 ? ClRecover : Recover),
725fe6060f1SDimitry Andric         UseAfterScope(UseAfterScope || ClUseAfterScope),
726fe6060f1SDimitry Andric         UseAfterReturn(ClUseAfterReturn.getNumOccurrences() ? ClUseAfterReturn
727fe6060f1SDimitry Andric                                                             : UseAfterReturn),
7285f757f3fSDimitry Andric         SSGI(SSGI),
7295f757f3fSDimitry Andric         InstrumentationWithCallsThreshold(
7305f757f3fSDimitry Andric             ClInstrumentationWithCallsThreshold.getNumOccurrences() > 0
7315f757f3fSDimitry Andric                 ? ClInstrumentationWithCallsThreshold
7325f757f3fSDimitry Andric                 : InstrumentationWithCallsThreshold),
7335f757f3fSDimitry Andric         MaxInlinePoisoningSize(ClMaxInlinePoisoningSize.getNumOccurrences() > 0
7345f757f3fSDimitry Andric                                    ? ClMaxInlinePoisoningSize
7355f757f3fSDimitry Andric                                    : MaxInlinePoisoningSize) {
7360b57cec5SDimitry Andric     C = &(M.getContext());
73706c3fb27SDimitry Andric     DL = &M.getDataLayout();
7380b57cec5SDimitry Andric     LongSize = M.getDataLayout().getPointerSizeInBits();
7390b57cec5SDimitry Andric     IntptrTy = Type::getIntNTy(*C, LongSize);
7405f757f3fSDimitry Andric     PtrTy = PointerType::getUnqual(*C);
741349cc55cSDimitry Andric     Int32Ty = Type::getInt32Ty(*C);
7420b57cec5SDimitry Andric     TargetTriple = Triple(M.getTargetTriple());
7430b57cec5SDimitry Andric 
7440b57cec5SDimitry Andric     Mapping = getShadowMapping(TargetTriple, LongSize, this->CompileKernel);
745fe6060f1SDimitry Andric 
746fe6060f1SDimitry Andric     assert(this->UseAfterReturn != AsanDetectStackUseAfterReturnMode::Invalid);
7470b57cec5SDimitry Andric   }
7480b57cec5SDimitry Andric 
74906c3fb27SDimitry Andric   TypeSize getAllocaSizeInBytes(const AllocaInst &AI) const {
750*0fca6ea1SDimitry Andric     return *AI.getAllocationSize(AI.getDataLayout());
7510b57cec5SDimitry Andric   }
7520b57cec5SDimitry Andric 
7530b57cec5SDimitry Andric   /// Check if we want (and can) handle this alloca.
7540b57cec5SDimitry Andric   bool isInterestingAlloca(const AllocaInst &AI);
7550b57cec5SDimitry Andric 
756349cc55cSDimitry Andric   bool ignoreAccess(Instruction *Inst, Value *Ptr);
7575ffd83dbSDimitry Andric   void getInterestingMemoryOperands(
7585ffd83dbSDimitry Andric       Instruction *I, SmallVectorImpl<InterestingMemoryOperand> &Interesting);
7590b57cec5SDimitry Andric 
7605ffd83dbSDimitry Andric   void instrumentMop(ObjectSizeOffsetVisitor &ObjSizeVis,
7615ffd83dbSDimitry Andric                      InterestingMemoryOperand &O, bool UseCalls,
762*0fca6ea1SDimitry Andric                      const DataLayout &DL, RuntimeCallInserter &RTCI);
763*0fca6ea1SDimitry Andric   void instrumentPointerComparisonOrSubtraction(Instruction *I,
764*0fca6ea1SDimitry Andric                                                 RuntimeCallInserter &RTCI);
7650b57cec5SDimitry Andric   void instrumentAddress(Instruction *OrigIns, Instruction *InsertBefore,
76606c3fb27SDimitry Andric                          Value *Addr, MaybeAlign Alignment,
76706c3fb27SDimitry Andric                          uint32_t TypeStoreSize, bool IsWrite,
768*0fca6ea1SDimitry Andric                          Value *SizeArgument, bool UseCalls, uint32_t Exp,
769*0fca6ea1SDimitry Andric                          RuntimeCallInserter &RTCI);
770fe6060f1SDimitry Andric   Instruction *instrumentAMDGPUAddress(Instruction *OrigIns,
771fe6060f1SDimitry Andric                                        Instruction *InsertBefore, Value *Addr,
77206c3fb27SDimitry Andric                                        uint32_t TypeStoreSize, bool IsWrite,
773fe6060f1SDimitry Andric                                        Value *SizeArgument);
7741db9f3b2SDimitry Andric   Instruction *genAMDGPUReportBlock(IRBuilder<> &IRB, Value *Cond,
7751db9f3b2SDimitry Andric                                     bool Recover);
7760b57cec5SDimitry Andric   void instrumentUnusualSizeOrAlignment(Instruction *I,
7770b57cec5SDimitry Andric                                         Instruction *InsertBefore, Value *Addr,
77806c3fb27SDimitry Andric                                         TypeSize TypeStoreSize, bool IsWrite,
77906c3fb27SDimitry Andric                                         Value *SizeArgument, bool UseCalls,
780*0fca6ea1SDimitry Andric                                         uint32_t Exp,
781*0fca6ea1SDimitry Andric                                         RuntimeCallInserter &RTCI);
78206c3fb27SDimitry Andric   void instrumentMaskedLoadOrStore(AddressSanitizer *Pass, const DataLayout &DL,
78306c3fb27SDimitry Andric                                    Type *IntptrTy, Value *Mask, Value *EVL,
78406c3fb27SDimitry Andric                                    Value *Stride, Instruction *I, Value *Addr,
78506c3fb27SDimitry Andric                                    MaybeAlign Alignment, unsigned Granularity,
78606c3fb27SDimitry Andric                                    Type *OpType, bool IsWrite,
7870b57cec5SDimitry Andric                                    Value *SizeArgument, bool UseCalls,
788*0fca6ea1SDimitry Andric                                    uint32_t Exp, RuntimeCallInserter &RTCI);
7890b57cec5SDimitry Andric   Value *createSlowPathCmp(IRBuilder<> &IRB, Value *AddrLong,
79006c3fb27SDimitry Andric                            Value *ShadowValue, uint32_t TypeStoreSize);
7910b57cec5SDimitry Andric   Instruction *generateCrashCode(Instruction *InsertBefore, Value *Addr,
7920b57cec5SDimitry Andric                                  bool IsWrite, size_t AccessSizeIndex,
793*0fca6ea1SDimitry Andric                                  Value *SizeArgument, uint32_t Exp,
794*0fca6ea1SDimitry Andric                                  RuntimeCallInserter &RTCI);
795*0fca6ea1SDimitry Andric   void instrumentMemIntrinsic(MemIntrinsic *MI, RuntimeCallInserter &RTCI);
7960b57cec5SDimitry Andric   Value *memToShadow(Value *Shadow, IRBuilder<> &IRB);
7975ffd83dbSDimitry Andric   bool suppressInstrumentationSiteForDebug(int &Instrumented);
7980b57cec5SDimitry Andric   bool instrumentFunction(Function &F, const TargetLibraryInfo *TLI);
7990b57cec5SDimitry Andric   bool maybeInsertAsanInitAtFunctionEntry(Function &F);
8005ffd83dbSDimitry Andric   bool maybeInsertDynamicShadowAtFunctionEntry(Function &F);
8010b57cec5SDimitry Andric   void markEscapedLocalAllocas(Function &F);
8020b57cec5SDimitry Andric 
8030b57cec5SDimitry Andric private:
8040b57cec5SDimitry Andric   friend struct FunctionStackPoisoner;
8050b57cec5SDimitry Andric 
806bdd1243dSDimitry Andric   void initializeCallbacks(Module &M, const TargetLibraryInfo *TLI);
8070b57cec5SDimitry Andric 
8080b57cec5SDimitry Andric   bool LooksLikeCodeInBug11395(Instruction *I);
8090b57cec5SDimitry Andric   bool GlobalIsLinkerInitialized(GlobalVariable *G);
8100b57cec5SDimitry Andric   bool isSafeAccess(ObjectSizeOffsetVisitor &ObjSizeVis, Value *Addr,
81106c3fb27SDimitry Andric                     TypeSize TypeStoreSize) const;
8120b57cec5SDimitry Andric 
8130b57cec5SDimitry Andric   /// Helper to cleanup per-function state.
8140b57cec5SDimitry Andric   struct FunctionStateRAII {
8150b57cec5SDimitry Andric     AddressSanitizer *Pass;
8160b57cec5SDimitry Andric 
8170b57cec5SDimitry Andric     FunctionStateRAII(AddressSanitizer *Pass) : Pass(Pass) {
8180b57cec5SDimitry Andric       assert(Pass->ProcessedAllocas.empty() &&
8190b57cec5SDimitry Andric              "last pass forgot to clear cache");
8200b57cec5SDimitry Andric       assert(!Pass->LocalDynamicShadow);
8210b57cec5SDimitry Andric     }
8220b57cec5SDimitry Andric 
8230b57cec5SDimitry Andric     ~FunctionStateRAII() {
8240b57cec5SDimitry Andric       Pass->LocalDynamicShadow = nullptr;
8250b57cec5SDimitry Andric       Pass->ProcessedAllocas.clear();
8260b57cec5SDimitry Andric     }
8270b57cec5SDimitry Andric   };
8280b57cec5SDimitry Andric 
8290b57cec5SDimitry Andric   LLVMContext *C;
83006c3fb27SDimitry Andric   const DataLayout *DL;
8310b57cec5SDimitry Andric   Triple TargetTriple;
8320b57cec5SDimitry Andric   int LongSize;
8330b57cec5SDimitry Andric   bool CompileKernel;
8340b57cec5SDimitry Andric   bool Recover;
8350b57cec5SDimitry Andric   bool UseAfterScope;
836fe6060f1SDimitry Andric   AsanDetectStackUseAfterReturnMode UseAfterReturn;
8370b57cec5SDimitry Andric   Type *IntptrTy;
838349cc55cSDimitry Andric   Type *Int32Ty;
8395f757f3fSDimitry Andric   PointerType *PtrTy;
8400b57cec5SDimitry Andric   ShadowMapping Mapping;
8410b57cec5SDimitry Andric   FunctionCallee AsanHandleNoReturnFunc;
8420b57cec5SDimitry Andric   FunctionCallee AsanPtrCmpFunction, AsanPtrSubFunction;
8430b57cec5SDimitry Andric   Constant *AsanShadowGlobal;
8440b57cec5SDimitry Andric 
8450b57cec5SDimitry Andric   // These arrays is indexed by AccessIsWrite, Experiment and log2(AccessSize).
8460b57cec5SDimitry Andric   FunctionCallee AsanErrorCallback[2][2][kNumberOfAccessSizes];
8470b57cec5SDimitry Andric   FunctionCallee AsanMemoryAccessCallback[2][2][kNumberOfAccessSizes];
8480b57cec5SDimitry Andric 
8490b57cec5SDimitry Andric   // These arrays is indexed by AccessIsWrite and Experiment.
8500b57cec5SDimitry Andric   FunctionCallee AsanErrorCallbackSized[2][2];
8510b57cec5SDimitry Andric   FunctionCallee AsanMemoryAccessCallbackSized[2][2];
8520b57cec5SDimitry Andric 
8530b57cec5SDimitry Andric   FunctionCallee AsanMemmove, AsanMemcpy, AsanMemset;
8540b57cec5SDimitry Andric   Value *LocalDynamicShadow = nullptr;
855349cc55cSDimitry Andric   const StackSafetyGlobalInfo *SSGI;
8560b57cec5SDimitry Andric   DenseMap<const AllocaInst *, bool> ProcessedAllocas;
857fe6060f1SDimitry Andric 
858fe6060f1SDimitry Andric   FunctionCallee AMDGPUAddressShared;
859fe6060f1SDimitry Andric   FunctionCallee AMDGPUAddressPrivate;
8605f757f3fSDimitry Andric   int InstrumentationWithCallsThreshold;
8615f757f3fSDimitry Andric   uint32_t MaxInlinePoisoningSize;
8620b57cec5SDimitry Andric };
8630b57cec5SDimitry Andric 
8640b57cec5SDimitry Andric class ModuleAddressSanitizer {
8650b57cec5SDimitry Andric public:
8665f757f3fSDimitry Andric   ModuleAddressSanitizer(Module &M, bool InsertVersionCheck,
8675f757f3fSDimitry Andric                          bool CompileKernel = false, bool Recover = false,
8685f757f3fSDimitry Andric                          bool UseGlobalsGC = true, bool UseOdrIndicator = true,
869bdd1243dSDimitry Andric                          AsanDtorKind DestructorKind = AsanDtorKind::Global,
870bdd1243dSDimitry Andric                          AsanCtorKind ConstructorKind = AsanCtorKind::Global)
87181ad6265SDimitry Andric       : CompileKernel(ClEnableKasan.getNumOccurrences() > 0 ? ClEnableKasan
8725ffd83dbSDimitry Andric                                                             : CompileKernel),
8735f757f3fSDimitry Andric         InsertVersionCheck(ClInsertVersionCheck.getNumOccurrences() > 0
8745f757f3fSDimitry Andric                                ? ClInsertVersionCheck
8755f757f3fSDimitry Andric                                : InsertVersionCheck),
8765ffd83dbSDimitry Andric         Recover(ClRecover.getNumOccurrences() > 0 ? ClRecover : Recover),
8775ffd83dbSDimitry Andric         UseGlobalsGC(UseGlobalsGC && ClUseGlobalsGC && !this->CompileKernel),
8780b57cec5SDimitry Andric         // Enable aliases as they should have no downside with ODR indicators.
879bdd1243dSDimitry Andric         UsePrivateAlias(ClUsePrivateAlias.getNumOccurrences() > 0
880bdd1243dSDimitry Andric                             ? ClUsePrivateAlias
881bdd1243dSDimitry Andric                             : UseOdrIndicator),
882bdd1243dSDimitry Andric         UseOdrIndicator(ClUseOdrIndicator.getNumOccurrences() > 0
883bdd1243dSDimitry Andric                             ? ClUseOdrIndicator
884bdd1243dSDimitry Andric                             : UseOdrIndicator),
8850b57cec5SDimitry Andric         // Not a typo: ClWithComdat is almost completely pointless without
8860b57cec5SDimitry Andric         // ClUseGlobalsGC (because then it only works on modules without
8870b57cec5SDimitry Andric         // globals, which are rare); it is a prerequisite for ClUseGlobalsGC;
8880b57cec5SDimitry Andric         // and both suffer from gold PR19002 for which UseGlobalsGC constructor
8890b57cec5SDimitry Andric         // argument is designed as workaround. Therefore, disable both
8900b57cec5SDimitry Andric         // ClWithComdat and ClUseGlobalsGC unless the frontend says it's ok to
8910b57cec5SDimitry Andric         // do globals-gc.
892fe6060f1SDimitry Andric         UseCtorComdat(UseGlobalsGC && ClWithComdat && !this->CompileKernel),
893bdd1243dSDimitry Andric         DestructorKind(DestructorKind),
8945f757f3fSDimitry Andric         ConstructorKind(ClConstructorKind.getNumOccurrences() > 0
8955f757f3fSDimitry Andric                             ? ClConstructorKind
8965f757f3fSDimitry Andric                             : ConstructorKind) {
8970b57cec5SDimitry Andric     C = &(M.getContext());
8980b57cec5SDimitry Andric     int LongSize = M.getDataLayout().getPointerSizeInBits();
8990b57cec5SDimitry Andric     IntptrTy = Type::getIntNTy(*C, LongSize);
9005f757f3fSDimitry Andric     PtrTy = PointerType::getUnqual(*C);
9010b57cec5SDimitry Andric     TargetTriple = Triple(M.getTargetTriple());
9020b57cec5SDimitry Andric     Mapping = getShadowMapping(TargetTriple, LongSize, this->CompileKernel);
903fe6060f1SDimitry Andric 
904fe6060f1SDimitry Andric     if (ClOverrideDestructorKind != AsanDtorKind::Invalid)
905fe6060f1SDimitry Andric       this->DestructorKind = ClOverrideDestructorKind;
906fe6060f1SDimitry Andric     assert(this->DestructorKind != AsanDtorKind::Invalid);
9070b57cec5SDimitry Andric   }
9080b57cec5SDimitry Andric 
9090b57cec5SDimitry Andric   bool instrumentModule(Module &);
9100b57cec5SDimitry Andric 
9110b57cec5SDimitry Andric private:
9120b57cec5SDimitry Andric   void initializeCallbacks(Module &M);
9130b57cec5SDimitry Andric 
914bdb86d1aSDimitry Andric   void instrumentGlobals(IRBuilder<> &IRB, Module &M, bool *CtorComdat);
9150b57cec5SDimitry Andric   void InstrumentGlobalsCOFF(IRBuilder<> &IRB, Module &M,
9160b57cec5SDimitry Andric                              ArrayRef<GlobalVariable *> ExtendedGlobals,
9170b57cec5SDimitry Andric                              ArrayRef<Constant *> MetadataInitializers);
918bdb86d1aSDimitry Andric   void instrumentGlobalsELF(IRBuilder<> &IRB, Module &M,
9190b57cec5SDimitry Andric                             ArrayRef<GlobalVariable *> ExtendedGlobals,
9200b57cec5SDimitry Andric                             ArrayRef<Constant *> MetadataInitializers,
9210b57cec5SDimitry Andric                             const std::string &UniqueModuleId);
9220b57cec5SDimitry Andric   void InstrumentGlobalsMachO(IRBuilder<> &IRB, Module &M,
9230b57cec5SDimitry Andric                               ArrayRef<GlobalVariable *> ExtendedGlobals,
9240b57cec5SDimitry Andric                               ArrayRef<Constant *> MetadataInitializers);
9250b57cec5SDimitry Andric   void
9260b57cec5SDimitry Andric   InstrumentGlobalsWithMetadataArray(IRBuilder<> &IRB, Module &M,
9270b57cec5SDimitry Andric                                      ArrayRef<GlobalVariable *> ExtendedGlobals,
9280b57cec5SDimitry Andric                                      ArrayRef<Constant *> MetadataInitializers);
9290b57cec5SDimitry Andric 
9300b57cec5SDimitry Andric   GlobalVariable *CreateMetadataGlobal(Module &M, Constant *Initializer,
9310b57cec5SDimitry Andric                                        StringRef OriginalName);
9320b57cec5SDimitry Andric   void SetComdatForGlobalMetadata(GlobalVariable *G, GlobalVariable *Metadata,
9330b57cec5SDimitry Andric                                   StringRef InternalSuffix);
9345ffd83dbSDimitry Andric   Instruction *CreateAsanModuleDtor(Module &M);
9350b57cec5SDimitry Andric 
936eaeb601bSDimitry Andric   const GlobalVariable *getExcludedAliasedGlobal(const GlobalAlias &GA) const;
9375ffd83dbSDimitry Andric   bool shouldInstrumentGlobal(GlobalVariable *G) const;
9380b57cec5SDimitry Andric   bool ShouldUseMachOGlobalsSection() const;
9390b57cec5SDimitry Andric   StringRef getGlobalMetadataSection() const;
9400b57cec5SDimitry Andric   void poisonOneInitializer(Function &GlobalInit, GlobalValue *ModuleName);
9410b57cec5SDimitry Andric   void createInitializerPoisonCalls(Module &M, GlobalValue *ModuleName);
9425ffd83dbSDimitry Andric   uint64_t getMinRedzoneSizeForGlobal() const {
9435ffd83dbSDimitry Andric     return getRedzoneSizeForScale(Mapping.Scale);
9440b57cec5SDimitry Andric   }
9455ffd83dbSDimitry Andric   uint64_t getRedzoneSizeForGlobal(uint64_t SizeInBytes) const;
9460b57cec5SDimitry Andric   int GetAsanVersion(const Module &M) const;
9470b57cec5SDimitry Andric 
9480b57cec5SDimitry Andric   bool CompileKernel;
9495f757f3fSDimitry Andric   bool InsertVersionCheck;
9500b57cec5SDimitry Andric   bool Recover;
9510b57cec5SDimitry Andric   bool UseGlobalsGC;
9520b57cec5SDimitry Andric   bool UsePrivateAlias;
9530b57cec5SDimitry Andric   bool UseOdrIndicator;
9540b57cec5SDimitry Andric   bool UseCtorComdat;
955fe6060f1SDimitry Andric   AsanDtorKind DestructorKind;
956bdd1243dSDimitry Andric   AsanCtorKind ConstructorKind;
9570b57cec5SDimitry Andric   Type *IntptrTy;
9585f757f3fSDimitry Andric   PointerType *PtrTy;
9590b57cec5SDimitry Andric   LLVMContext *C;
9600b57cec5SDimitry Andric   Triple TargetTriple;
9610b57cec5SDimitry Andric   ShadowMapping Mapping;
9620b57cec5SDimitry Andric   FunctionCallee AsanPoisonGlobals;
9630b57cec5SDimitry Andric   FunctionCallee AsanUnpoisonGlobals;
9640b57cec5SDimitry Andric   FunctionCallee AsanRegisterGlobals;
9650b57cec5SDimitry Andric   FunctionCallee AsanUnregisterGlobals;
9660b57cec5SDimitry Andric   FunctionCallee AsanRegisterImageGlobals;
9670b57cec5SDimitry Andric   FunctionCallee AsanUnregisterImageGlobals;
9680b57cec5SDimitry Andric   FunctionCallee AsanRegisterElfGlobals;
9690b57cec5SDimitry Andric   FunctionCallee AsanUnregisterElfGlobals;
9700b57cec5SDimitry Andric 
9710b57cec5SDimitry Andric   Function *AsanCtorFunction = nullptr;
9720b57cec5SDimitry Andric   Function *AsanDtorFunction = nullptr;
9730b57cec5SDimitry Andric };
9740b57cec5SDimitry Andric 
9750b57cec5SDimitry Andric // Stack poisoning does not play well with exception handling.
9760b57cec5SDimitry Andric // When an exception is thrown, we essentially bypass the code
9770b57cec5SDimitry Andric // that unpoisones the stack. This is why the run-time library has
9780b57cec5SDimitry Andric // to intercept __cxa_throw (as well as longjmp, etc) and unpoison the entire
9790b57cec5SDimitry Andric // stack in the interceptor. This however does not work inside the
9800b57cec5SDimitry Andric // actual function which catches the exception. Most likely because the
9810b57cec5SDimitry Andric // compiler hoists the load of the shadow value somewhere too high.
9820b57cec5SDimitry Andric // This causes asan to report a non-existing bug on 453.povray.
9830b57cec5SDimitry Andric // It sounds like an LLVM bug.
9840b57cec5SDimitry Andric struct FunctionStackPoisoner : public InstVisitor<FunctionStackPoisoner> {
9850b57cec5SDimitry Andric   Function &F;
9860b57cec5SDimitry Andric   AddressSanitizer &ASan;
987*0fca6ea1SDimitry Andric   RuntimeCallInserter &RTCI;
9880b57cec5SDimitry Andric   DIBuilder DIB;
9890b57cec5SDimitry Andric   LLVMContext *C;
9900b57cec5SDimitry Andric   Type *IntptrTy;
9910b57cec5SDimitry Andric   Type *IntptrPtrTy;
9920b57cec5SDimitry Andric   ShadowMapping Mapping;
9930b57cec5SDimitry Andric 
9940b57cec5SDimitry Andric   SmallVector<AllocaInst *, 16> AllocaVec;
9950b57cec5SDimitry Andric   SmallVector<AllocaInst *, 16> StaticAllocasToMoveUp;
9960b57cec5SDimitry Andric   SmallVector<Instruction *, 8> RetVec;
9970b57cec5SDimitry Andric 
9980b57cec5SDimitry Andric   FunctionCallee AsanStackMallocFunc[kMaxAsanStackMallocSizeClass + 1],
9990b57cec5SDimitry Andric       AsanStackFreeFunc[kMaxAsanStackMallocSizeClass + 1];
10000b57cec5SDimitry Andric   FunctionCallee AsanSetShadowFunc[0x100] = {};
10010b57cec5SDimitry Andric   FunctionCallee AsanPoisonStackMemoryFunc, AsanUnpoisonStackMemoryFunc;
10020b57cec5SDimitry Andric   FunctionCallee AsanAllocaPoisonFunc, AsanAllocasUnpoisonFunc;
10030b57cec5SDimitry Andric 
10040b57cec5SDimitry Andric   // Stores a place and arguments of poisoning/unpoisoning call for alloca.
10050b57cec5SDimitry Andric   struct AllocaPoisonCall {
10060b57cec5SDimitry Andric     IntrinsicInst *InsBefore;
10070b57cec5SDimitry Andric     AllocaInst *AI;
10080b57cec5SDimitry Andric     uint64_t Size;
10090b57cec5SDimitry Andric     bool DoPoison;
10100b57cec5SDimitry Andric   };
10110b57cec5SDimitry Andric   SmallVector<AllocaPoisonCall, 8> DynamicAllocaPoisonCallVec;
10120b57cec5SDimitry Andric   SmallVector<AllocaPoisonCall, 8> StaticAllocaPoisonCallVec;
10130b57cec5SDimitry Andric   bool HasUntracedLifetimeIntrinsic = false;
10140b57cec5SDimitry Andric 
10150b57cec5SDimitry Andric   SmallVector<AllocaInst *, 1> DynamicAllocaVec;
10160b57cec5SDimitry Andric   SmallVector<IntrinsicInst *, 1> StackRestoreVec;
10170b57cec5SDimitry Andric   AllocaInst *DynamicAllocaLayout = nullptr;
10180b57cec5SDimitry Andric   IntrinsicInst *LocalEscapeCall = nullptr;
10190b57cec5SDimitry Andric 
10205ffd83dbSDimitry Andric   bool HasInlineAsm = false;
10210b57cec5SDimitry Andric   bool HasReturnsTwiceCall = false;
1022fe6060f1SDimitry Andric   bool PoisonStack;
10230b57cec5SDimitry Andric 
1024*0fca6ea1SDimitry Andric   FunctionStackPoisoner(Function &F, AddressSanitizer &ASan,
1025*0fca6ea1SDimitry Andric                         RuntimeCallInserter &RTCI)
1026*0fca6ea1SDimitry Andric       : F(F), ASan(ASan), RTCI(RTCI),
1027*0fca6ea1SDimitry Andric         DIB(*F.getParent(), /*AllowUnresolved*/ false), C(ASan.C),
1028*0fca6ea1SDimitry Andric         IntptrTy(ASan.IntptrTy), IntptrPtrTy(PointerType::get(IntptrTy, 0)),
1029*0fca6ea1SDimitry Andric         Mapping(ASan.Mapping),
1030fe6060f1SDimitry Andric         PoisonStack(ClStack &&
1031fe6060f1SDimitry Andric                     !Triple(F.getParent()->getTargetTriple()).isAMDGPU()) {}
10320b57cec5SDimitry Andric 
10330b57cec5SDimitry Andric   bool runOnFunction() {
1034fe6060f1SDimitry Andric     if (!PoisonStack)
1035fe6060f1SDimitry Andric       return false;
10360b57cec5SDimitry Andric 
10370b57cec5SDimitry Andric     if (ClRedzoneByvalArgs)
10380b57cec5SDimitry Andric       copyArgsPassedByValToAllocas();
10390b57cec5SDimitry Andric 
10400b57cec5SDimitry Andric     // Collect alloca, ret, lifetime instructions etc.
10410b57cec5SDimitry Andric     for (BasicBlock *BB : depth_first(&F.getEntryBlock())) visit(*BB);
10420b57cec5SDimitry Andric 
10430b57cec5SDimitry Andric     if (AllocaVec.empty() && DynamicAllocaVec.empty()) return false;
10440b57cec5SDimitry Andric 
10450b57cec5SDimitry Andric     initializeCallbacks(*F.getParent());
10460b57cec5SDimitry Andric 
10470b57cec5SDimitry Andric     if (HasUntracedLifetimeIntrinsic) {
10480b57cec5SDimitry Andric       // If there are lifetime intrinsics which couldn't be traced back to an
10490b57cec5SDimitry Andric       // alloca, we may not know exactly when a variable enters scope, and
10500b57cec5SDimitry Andric       // therefore should "fail safe" by not poisoning them.
10510b57cec5SDimitry Andric       StaticAllocaPoisonCallVec.clear();
10520b57cec5SDimitry Andric       DynamicAllocaPoisonCallVec.clear();
10530b57cec5SDimitry Andric     }
10540b57cec5SDimitry Andric 
10550b57cec5SDimitry Andric     processDynamicAllocas();
10560b57cec5SDimitry Andric     processStaticAllocas();
10570b57cec5SDimitry Andric 
10580b57cec5SDimitry Andric     if (ClDebugStack) {
10590b57cec5SDimitry Andric       LLVM_DEBUG(dbgs() << F);
10600b57cec5SDimitry Andric     }
10610b57cec5SDimitry Andric     return true;
10620b57cec5SDimitry Andric   }
10630b57cec5SDimitry Andric 
10640b57cec5SDimitry Andric   // Arguments marked with the "byval" attribute are implicitly copied without
10650b57cec5SDimitry Andric   // using an alloca instruction.  To produce redzones for those arguments, we
10660b57cec5SDimitry Andric   // copy them a second time into memory allocated with an alloca instruction.
10670b57cec5SDimitry Andric   void copyArgsPassedByValToAllocas();
10680b57cec5SDimitry Andric 
10690b57cec5SDimitry Andric   // Finds all Alloca instructions and puts
10700b57cec5SDimitry Andric   // poisoned red zones around all of them.
10710b57cec5SDimitry Andric   // Then unpoison everything back before the function returns.
10720b57cec5SDimitry Andric   void processStaticAllocas();
10730b57cec5SDimitry Andric   void processDynamicAllocas();
10740b57cec5SDimitry Andric 
10750b57cec5SDimitry Andric   void createDynamicAllocasInitStorage();
10760b57cec5SDimitry Andric 
10770b57cec5SDimitry Andric   // ----------------------- Visitors.
1078e8d8bef9SDimitry Andric   /// Collect all Ret instructions, or the musttail call instruction if it
1079e8d8bef9SDimitry Andric   /// precedes the return instruction.
1080e8d8bef9SDimitry Andric   void visitReturnInst(ReturnInst &RI) {
1081e8d8bef9SDimitry Andric     if (CallInst *CI = RI.getParent()->getTerminatingMustTailCall())
1082e8d8bef9SDimitry Andric       RetVec.push_back(CI);
1083e8d8bef9SDimitry Andric     else
1084e8d8bef9SDimitry Andric       RetVec.push_back(&RI);
1085e8d8bef9SDimitry Andric   }
10860b57cec5SDimitry Andric 
10870b57cec5SDimitry Andric   /// Collect all Resume instructions.
10880b57cec5SDimitry Andric   void visitResumeInst(ResumeInst &RI) { RetVec.push_back(&RI); }
10890b57cec5SDimitry Andric 
10900b57cec5SDimitry Andric   /// Collect all CatchReturnInst instructions.
10910b57cec5SDimitry Andric   void visitCleanupReturnInst(CleanupReturnInst &CRI) { RetVec.push_back(&CRI); }
10920b57cec5SDimitry Andric 
10930b57cec5SDimitry Andric   void unpoisonDynamicAllocasBeforeInst(Instruction *InstBefore,
10940b57cec5SDimitry Andric                                         Value *SavedStack) {
10950b57cec5SDimitry Andric     IRBuilder<> IRB(InstBefore);
10960b57cec5SDimitry Andric     Value *DynamicAreaPtr = IRB.CreatePtrToInt(SavedStack, IntptrTy);
10970b57cec5SDimitry Andric     // When we insert _asan_allocas_unpoison before @llvm.stackrestore, we
10980b57cec5SDimitry Andric     // need to adjust extracted SP to compute the address of the most recent
10990b57cec5SDimitry Andric     // alloca. We have a special @llvm.get.dynamic.area.offset intrinsic for
11000b57cec5SDimitry Andric     // this purpose.
11010b57cec5SDimitry Andric     if (!isa<ReturnInst>(InstBefore)) {
11020b57cec5SDimitry Andric       Function *DynamicAreaOffsetFunc = Intrinsic::getDeclaration(
11030b57cec5SDimitry Andric           InstBefore->getModule(), Intrinsic::get_dynamic_area_offset,
11040b57cec5SDimitry Andric           {IntptrTy});
11050b57cec5SDimitry Andric 
11060b57cec5SDimitry Andric       Value *DynamicAreaOffset = IRB.CreateCall(DynamicAreaOffsetFunc, {});
11070b57cec5SDimitry Andric 
11080b57cec5SDimitry Andric       DynamicAreaPtr = IRB.CreateAdd(IRB.CreatePtrToInt(SavedStack, IntptrTy),
11090b57cec5SDimitry Andric                                      DynamicAreaOffset);
11100b57cec5SDimitry Andric     }
11110b57cec5SDimitry Andric 
1112*0fca6ea1SDimitry Andric     RTCI.createRuntimeCall(
1113*0fca6ea1SDimitry Andric         IRB, AsanAllocasUnpoisonFunc,
11140b57cec5SDimitry Andric         {IRB.CreateLoad(IntptrTy, DynamicAllocaLayout), DynamicAreaPtr});
11150b57cec5SDimitry Andric   }
11160b57cec5SDimitry Andric 
11170b57cec5SDimitry Andric   // Unpoison dynamic allocas redzones.
11180b57cec5SDimitry Andric   void unpoisonDynamicAllocas() {
1119e8d8bef9SDimitry Andric     for (Instruction *Ret : RetVec)
11200b57cec5SDimitry Andric       unpoisonDynamicAllocasBeforeInst(Ret, DynamicAllocaLayout);
11210b57cec5SDimitry Andric 
1122e8d8bef9SDimitry Andric     for (Instruction *StackRestoreInst : StackRestoreVec)
11230b57cec5SDimitry Andric       unpoisonDynamicAllocasBeforeInst(StackRestoreInst,
11240b57cec5SDimitry Andric                                        StackRestoreInst->getOperand(0));
11250b57cec5SDimitry Andric   }
11260b57cec5SDimitry Andric 
11270b57cec5SDimitry Andric   // Deploy and poison redzones around dynamic alloca call. To do this, we
11280b57cec5SDimitry Andric   // should replace this call with another one with changed parameters and
11290b57cec5SDimitry Andric   // replace all its uses with new address, so
11300b57cec5SDimitry Andric   //   addr = alloca type, old_size, align
11310b57cec5SDimitry Andric   // is replaced by
11320b57cec5SDimitry Andric   //   new_size = (old_size + additional_size) * sizeof(type)
11330b57cec5SDimitry Andric   //   tmp = alloca i8, new_size, max(align, 32)
11340b57cec5SDimitry Andric   //   addr = tmp + 32 (first 32 bytes are for the left redzone).
11350b57cec5SDimitry Andric   // Additional_size is added to make new memory allocation contain not only
11360b57cec5SDimitry Andric   // requested memory, but also left, partial and right redzones.
11370b57cec5SDimitry Andric   void handleDynamicAllocaCall(AllocaInst *AI);
11380b57cec5SDimitry Andric 
11390b57cec5SDimitry Andric   /// Collect Alloca instructions we want (and can) handle.
11400b57cec5SDimitry Andric   void visitAllocaInst(AllocaInst &AI) {
114106c3fb27SDimitry Andric     // FIXME: Handle scalable vectors instead of ignoring them.
1142*0fca6ea1SDimitry Andric     const Type *AllocaType = AI.getAllocatedType();
1143*0fca6ea1SDimitry Andric     const auto *STy = dyn_cast<StructType>(AllocaType);
1144*0fca6ea1SDimitry Andric     if (!ASan.isInterestingAlloca(AI) || isa<ScalableVectorType>(AllocaType) ||
1145*0fca6ea1SDimitry Andric         (STy && STy->containsHomogeneousScalableVectorTypes())) {
11460b57cec5SDimitry Andric       if (AI.isStaticAlloca()) {
11470b57cec5SDimitry Andric         // Skip over allocas that are present *before* the first instrumented
11480b57cec5SDimitry Andric         // alloca, we don't want to move those around.
11490b57cec5SDimitry Andric         if (AllocaVec.empty())
11500b57cec5SDimitry Andric           return;
11510b57cec5SDimitry Andric 
11520b57cec5SDimitry Andric         StaticAllocasToMoveUp.push_back(&AI);
11530b57cec5SDimitry Andric       }
11540b57cec5SDimitry Andric       return;
11550b57cec5SDimitry Andric     }
11560b57cec5SDimitry Andric 
11570b57cec5SDimitry Andric     if (!AI.isStaticAlloca())
11580b57cec5SDimitry Andric       DynamicAllocaVec.push_back(&AI);
11590b57cec5SDimitry Andric     else
11600b57cec5SDimitry Andric       AllocaVec.push_back(&AI);
11610b57cec5SDimitry Andric   }
11620b57cec5SDimitry Andric 
11630b57cec5SDimitry Andric   /// Collect lifetime intrinsic calls to check for use-after-scope
11640b57cec5SDimitry Andric   /// errors.
11650b57cec5SDimitry Andric   void visitIntrinsicInst(IntrinsicInst &II) {
11660b57cec5SDimitry Andric     Intrinsic::ID ID = II.getIntrinsicID();
11670b57cec5SDimitry Andric     if (ID == Intrinsic::stackrestore) StackRestoreVec.push_back(&II);
11680b57cec5SDimitry Andric     if (ID == Intrinsic::localescape) LocalEscapeCall = &II;
11690b57cec5SDimitry Andric     if (!ASan.UseAfterScope)
11700b57cec5SDimitry Andric       return;
11710b57cec5SDimitry Andric     if (!II.isLifetimeStartOrEnd())
11720b57cec5SDimitry Andric       return;
11730b57cec5SDimitry Andric     // Found lifetime intrinsic, add ASan instrumentation if necessary.
11748bcb0991SDimitry Andric     auto *Size = cast<ConstantInt>(II.getArgOperand(0));
11750b57cec5SDimitry Andric     // If size argument is undefined, don't do anything.
11760b57cec5SDimitry Andric     if (Size->isMinusOne()) return;
11770b57cec5SDimitry Andric     // Check that size doesn't saturate uint64_t and can
11780b57cec5SDimitry Andric     // be stored in IntptrTy.
11790b57cec5SDimitry Andric     const uint64_t SizeValue = Size->getValue().getLimitedValue();
11800b57cec5SDimitry Andric     if (SizeValue == ~0ULL ||
11810b57cec5SDimitry Andric         !ConstantInt::isValueValidForType(IntptrTy, SizeValue))
11820b57cec5SDimitry Andric       return;
11830b57cec5SDimitry Andric     // Find alloca instruction that corresponds to llvm.lifetime argument.
1184e8d8bef9SDimitry Andric     // Currently we can only handle lifetime markers pointing to the
1185e8d8bef9SDimitry Andric     // beginning of the alloca.
1186e8d8bef9SDimitry Andric     AllocaInst *AI = findAllocaForValue(II.getArgOperand(1), true);
11870b57cec5SDimitry Andric     if (!AI) {
11880b57cec5SDimitry Andric       HasUntracedLifetimeIntrinsic = true;
11890b57cec5SDimitry Andric       return;
11900b57cec5SDimitry Andric     }
11910b57cec5SDimitry Andric     // We're interested only in allocas we can handle.
11920b57cec5SDimitry Andric     if (!ASan.isInterestingAlloca(*AI))
11930b57cec5SDimitry Andric       return;
11940b57cec5SDimitry Andric     bool DoPoison = (ID == Intrinsic::lifetime_end);
11950b57cec5SDimitry Andric     AllocaPoisonCall APC = {&II, AI, SizeValue, DoPoison};
11960b57cec5SDimitry Andric     if (AI->isStaticAlloca())
11970b57cec5SDimitry Andric       StaticAllocaPoisonCallVec.push_back(APC);
11980b57cec5SDimitry Andric     else if (ClInstrumentDynamicAllocas)
11990b57cec5SDimitry Andric       DynamicAllocaPoisonCallVec.push_back(APC);
12000b57cec5SDimitry Andric   }
12010b57cec5SDimitry Andric 
12025ffd83dbSDimitry Andric   void visitCallBase(CallBase &CB) {
12035ffd83dbSDimitry Andric     if (CallInst *CI = dyn_cast<CallInst>(&CB)) {
12045ffd83dbSDimitry Andric       HasInlineAsm |= CI->isInlineAsm() && &CB != ASan.LocalDynamicShadow;
12050b57cec5SDimitry Andric       HasReturnsTwiceCall |= CI->canReturnTwice();
12060b57cec5SDimitry Andric     }
12070b57cec5SDimitry Andric   }
12080b57cec5SDimitry Andric 
12090b57cec5SDimitry Andric   // ---------------------- Helpers.
12100b57cec5SDimitry Andric   void initializeCallbacks(Module &M);
12110b57cec5SDimitry Andric 
12120b57cec5SDimitry Andric   // Copies bytes from ShadowBytes into shadow memory for indexes where
12130b57cec5SDimitry Andric   // ShadowMask is not zero. If ShadowMask[i] is zero, we assume that
12140b57cec5SDimitry Andric   // ShadowBytes[i] is constantly zero and doesn't need to be overwritten.
12150b57cec5SDimitry Andric   void copyToShadow(ArrayRef<uint8_t> ShadowMask, ArrayRef<uint8_t> ShadowBytes,
12160b57cec5SDimitry Andric                     IRBuilder<> &IRB, Value *ShadowBase);
12170b57cec5SDimitry Andric   void copyToShadow(ArrayRef<uint8_t> ShadowMask, ArrayRef<uint8_t> ShadowBytes,
12180b57cec5SDimitry Andric                     size_t Begin, size_t End, IRBuilder<> &IRB,
12190b57cec5SDimitry Andric                     Value *ShadowBase);
12200b57cec5SDimitry Andric   void copyToShadowInline(ArrayRef<uint8_t> ShadowMask,
12210b57cec5SDimitry Andric                           ArrayRef<uint8_t> ShadowBytes, size_t Begin,
12220b57cec5SDimitry Andric                           size_t End, IRBuilder<> &IRB, Value *ShadowBase);
12230b57cec5SDimitry Andric 
12240b57cec5SDimitry Andric   void poisonAlloca(Value *V, uint64_t Size, IRBuilder<> &IRB, bool DoPoison);
12250b57cec5SDimitry Andric 
12260b57cec5SDimitry Andric   Value *createAllocaForLayout(IRBuilder<> &IRB, const ASanStackFrameLayout &L,
12270b57cec5SDimitry Andric                                bool Dynamic);
12280b57cec5SDimitry Andric   PHINode *createPHI(IRBuilder<> &IRB, Value *Cond, Value *ValueIfTrue,
12290b57cec5SDimitry Andric                      Instruction *ThenTerm, Value *ValueIfFalse);
12300b57cec5SDimitry Andric };
12310b57cec5SDimitry Andric 
12320b57cec5SDimitry Andric } // end anonymous namespace
12330b57cec5SDimitry Andric 
1234bdd1243dSDimitry Andric void AddressSanitizerPass::printPipeline(
1235349cc55cSDimitry Andric     raw_ostream &OS, function_ref<StringRef(StringRef)> MapClassName2PassName) {
1236bdd1243dSDimitry Andric   static_cast<PassInfoMixin<AddressSanitizerPass> *>(this)->printPipeline(
1237349cc55cSDimitry Andric       OS, MapClassName2PassName);
123806c3fb27SDimitry Andric   OS << '<';
1239349cc55cSDimitry Andric   if (Options.CompileKernel)
1240349cc55cSDimitry Andric     OS << "kernel";
124106c3fb27SDimitry Andric   OS << '>';
1242349cc55cSDimitry Andric }
1243349cc55cSDimitry Andric 
1244bdd1243dSDimitry Andric AddressSanitizerPass::AddressSanitizerPass(
1245349cc55cSDimitry Andric     const AddressSanitizerOptions &Options, bool UseGlobalGC,
1246bdd1243dSDimitry Andric     bool UseOdrIndicator, AsanDtorKind DestructorKind,
1247bdd1243dSDimitry Andric     AsanCtorKind ConstructorKind)
1248349cc55cSDimitry Andric     : Options(Options), UseGlobalGC(UseGlobalGC),
1249bdd1243dSDimitry Andric       UseOdrIndicator(UseOdrIndicator), DestructorKind(DestructorKind),
12505f757f3fSDimitry Andric       ConstructorKind(ConstructorKind) {}
12510b57cec5SDimitry Andric 
1252bdd1243dSDimitry Andric PreservedAnalyses AddressSanitizerPass::run(Module &M,
1253349cc55cSDimitry Andric                                             ModuleAnalysisManager &MAM) {
12545f757f3fSDimitry Andric   ModuleAddressSanitizer ModuleSanitizer(
12555f757f3fSDimitry Andric       M, Options.InsertVersionCheck, Options.CompileKernel, Options.Recover,
12565f757f3fSDimitry Andric       UseGlobalGC, UseOdrIndicator, DestructorKind, ConstructorKind);
1257349cc55cSDimitry Andric   bool Modified = false;
1258349cc55cSDimitry Andric   auto &FAM = MAM.getResult<FunctionAnalysisManagerModuleProxy>(M).getManager();
1259349cc55cSDimitry Andric   const StackSafetyGlobalInfo *const SSGI =
1260349cc55cSDimitry Andric       ClUseStackSafety ? &MAM.getResult<StackSafetyGlobalAnalysis>(M) : nullptr;
1261349cc55cSDimitry Andric   for (Function &F : M) {
12625f757f3fSDimitry Andric     AddressSanitizer FunctionSanitizer(
12635f757f3fSDimitry Andric         M, SSGI, Options.InstrumentationWithCallsThreshold,
12645f757f3fSDimitry Andric         Options.MaxInlinePoisoningSize, Options.CompileKernel, Options.Recover,
12655f757f3fSDimitry Andric         Options.UseAfterScope, Options.UseAfterReturn);
1266349cc55cSDimitry Andric     const TargetLibraryInfo &TLI = FAM.getResult<TargetLibraryAnalysis>(F);
1267349cc55cSDimitry Andric     Modified |= FunctionSanitizer.instrumentFunction(F, &TLI);
1268349cc55cSDimitry Andric   }
1269349cc55cSDimitry Andric   Modified |= ModuleSanitizer.instrumentModule(M);
1270bdd1243dSDimitry Andric   if (!Modified)
1271bdd1243dSDimitry Andric     return PreservedAnalyses::all();
1272bdd1243dSDimitry Andric 
1273bdd1243dSDimitry Andric   PreservedAnalyses PA = PreservedAnalyses::none();
1274bdd1243dSDimitry Andric   // GlobalsAA is considered stateless and does not get invalidated unless
1275bdd1243dSDimitry Andric   // explicitly invalidated; PreservedAnalyses::none() is not enough. Sanitizers
1276bdd1243dSDimitry Andric   // make changes that require GlobalsAA to be invalidated.
1277bdd1243dSDimitry Andric   PA.abandon<GlobalsAA>();
1278bdd1243dSDimitry Andric   return PA;
12790b57cec5SDimitry Andric }
12800b57cec5SDimitry Andric 
128106c3fb27SDimitry Andric static size_t TypeStoreSizeToSizeIndex(uint32_t TypeSize) {
128206c3fb27SDimitry Andric   size_t Res = llvm::countr_zero(TypeSize / 8);
12830b57cec5SDimitry Andric   assert(Res < kNumberOfAccessSizes);
12840b57cec5SDimitry Andric   return Res;
12850b57cec5SDimitry Andric }
12860b57cec5SDimitry Andric 
12870b57cec5SDimitry Andric /// Check if \p G has been created by a trusted compiler pass.
12880b57cec5SDimitry Andric static bool GlobalWasGeneratedByCompiler(GlobalVariable *G) {
12890b57cec5SDimitry Andric   // Do not instrument @llvm.global_ctors, @llvm.used, etc.
12905f757f3fSDimitry Andric   if (G->getName().starts_with("llvm.") ||
129181ad6265SDimitry Andric       // Do not instrument gcov counter arrays.
12925f757f3fSDimitry Andric       G->getName().starts_with("__llvm_gcov_ctr") ||
129381ad6265SDimitry Andric       // Do not instrument rtti proxy symbols for function sanitizer.
12945f757f3fSDimitry Andric       G->getName().starts_with("__llvm_rtti_proxy"))
12950b57cec5SDimitry Andric     return true;
12960b57cec5SDimitry Andric 
12970b57cec5SDimitry Andric   // Do not instrument asan globals.
12985f757f3fSDimitry Andric   if (G->getName().starts_with(kAsanGenPrefix) ||
12995f757f3fSDimitry Andric       G->getName().starts_with(kSanCovGenPrefix) ||
13005f757f3fSDimitry Andric       G->getName().starts_with(kODRGenPrefix))
13010b57cec5SDimitry Andric     return true;
13020b57cec5SDimitry Andric 
13030b57cec5SDimitry Andric   return false;
13040b57cec5SDimitry Andric }
13050b57cec5SDimitry Andric 
1306fe6060f1SDimitry Andric static bool isUnsupportedAMDGPUAddrspace(Value *Addr) {
1307fe6060f1SDimitry Andric   Type *PtrTy = cast<PointerType>(Addr->getType()->getScalarType());
1308fe6060f1SDimitry Andric   unsigned int AddrSpace = PtrTy->getPointerAddressSpace();
1309fe6060f1SDimitry Andric   if (AddrSpace == 3 || AddrSpace == 5)
1310fe6060f1SDimitry Andric     return true;
1311fe6060f1SDimitry Andric   return false;
1312fe6060f1SDimitry Andric }
1313fe6060f1SDimitry Andric 
13140b57cec5SDimitry Andric Value *AddressSanitizer::memToShadow(Value *Shadow, IRBuilder<> &IRB) {
13150b57cec5SDimitry Andric   // Shadow >> scale
13160b57cec5SDimitry Andric   Shadow = IRB.CreateLShr(Shadow, Mapping.Scale);
13170b57cec5SDimitry Andric   if (Mapping.Offset == 0) return Shadow;
13180b57cec5SDimitry Andric   // (Shadow >> scale) | offset
13190b57cec5SDimitry Andric   Value *ShadowBase;
13200b57cec5SDimitry Andric   if (LocalDynamicShadow)
13210b57cec5SDimitry Andric     ShadowBase = LocalDynamicShadow;
13220b57cec5SDimitry Andric   else
13230b57cec5SDimitry Andric     ShadowBase = ConstantInt::get(IntptrTy, Mapping.Offset);
13240b57cec5SDimitry Andric   if (Mapping.OrShadowOffset)
13250b57cec5SDimitry Andric     return IRB.CreateOr(Shadow, ShadowBase);
13260b57cec5SDimitry Andric   else
13270b57cec5SDimitry Andric     return IRB.CreateAdd(Shadow, ShadowBase);
13280b57cec5SDimitry Andric }
13290b57cec5SDimitry Andric 
13300b57cec5SDimitry Andric // Instrument memset/memmove/memcpy
1331*0fca6ea1SDimitry Andric void AddressSanitizer::instrumentMemIntrinsic(MemIntrinsic *MI,
1332*0fca6ea1SDimitry Andric                                               RuntimeCallInserter &RTCI) {
133306c3fb27SDimitry Andric   InstrumentationIRBuilder IRB(MI);
13340b57cec5SDimitry Andric   if (isa<MemTransferInst>(MI)) {
1335*0fca6ea1SDimitry Andric     RTCI.createRuntimeCall(
1336*0fca6ea1SDimitry Andric         IRB, isa<MemMoveInst>(MI) ? AsanMemmove : AsanMemcpy,
1337*0fca6ea1SDimitry Andric         {IRB.CreateAddrSpaceCast(MI->getOperand(0), PtrTy),
1338*0fca6ea1SDimitry Andric          IRB.CreateAddrSpaceCast(MI->getOperand(1), PtrTy),
13390b57cec5SDimitry Andric          IRB.CreateIntCast(MI->getOperand(2), IntptrTy, false)});
13400b57cec5SDimitry Andric   } else if (isa<MemSetInst>(MI)) {
1341*0fca6ea1SDimitry Andric     RTCI.createRuntimeCall(
1342*0fca6ea1SDimitry Andric         IRB, AsanMemset,
1343*0fca6ea1SDimitry Andric         {IRB.CreateAddrSpaceCast(MI->getOperand(0), PtrTy),
13440b57cec5SDimitry Andric          IRB.CreateIntCast(MI->getOperand(1), IRB.getInt32Ty(), false),
13450b57cec5SDimitry Andric          IRB.CreateIntCast(MI->getOperand(2), IntptrTy, false)});
13460b57cec5SDimitry Andric   }
13470b57cec5SDimitry Andric   MI->eraseFromParent();
13480b57cec5SDimitry Andric }
13490b57cec5SDimitry Andric 
13500b57cec5SDimitry Andric /// Check if we want (and can) handle this alloca.
13510b57cec5SDimitry Andric bool AddressSanitizer::isInterestingAlloca(const AllocaInst &AI) {
13520b57cec5SDimitry Andric   auto PreviouslySeenAllocaInfo = ProcessedAllocas.find(&AI);
13530b57cec5SDimitry Andric 
13540b57cec5SDimitry Andric   if (PreviouslySeenAllocaInfo != ProcessedAllocas.end())
13550b57cec5SDimitry Andric     return PreviouslySeenAllocaInfo->getSecond();
13560b57cec5SDimitry Andric 
13570b57cec5SDimitry Andric   bool IsInteresting =
13580b57cec5SDimitry Andric       (AI.getAllocatedType()->isSized() &&
13590b57cec5SDimitry Andric        // alloca() may be called with 0 size, ignore it.
136006c3fb27SDimitry Andric        ((!AI.isStaticAlloca()) || !getAllocaSizeInBytes(AI).isZero()) &&
13610b57cec5SDimitry Andric        // We are only interested in allocas not promotable to registers.
13620b57cec5SDimitry Andric        // Promotable allocas are common under -O0.
13630b57cec5SDimitry Andric        (!ClSkipPromotableAllocas || !isAllocaPromotable(&AI)) &&
13640b57cec5SDimitry Andric        // inalloca allocas are not treated as static, and we don't want
13650b57cec5SDimitry Andric        // dynamic alloca instrumentation for them as well.
13660b57cec5SDimitry Andric        !AI.isUsedWithInAlloca() &&
13670b57cec5SDimitry Andric        // swifterror allocas are register promoted by ISel
1368972a253aSDimitry Andric        !AI.isSwiftError() &&
1369972a253aSDimitry Andric        // safe allocas are not interesting
1370972a253aSDimitry Andric        !(SSGI && SSGI->isSafe(AI)));
13710b57cec5SDimitry Andric 
13720b57cec5SDimitry Andric   ProcessedAllocas[&AI] = IsInteresting;
13730b57cec5SDimitry Andric   return IsInteresting;
13740b57cec5SDimitry Andric }
13750b57cec5SDimitry Andric 
1376349cc55cSDimitry Andric bool AddressSanitizer::ignoreAccess(Instruction *Inst, Value *Ptr) {
1377bdd1243dSDimitry Andric   // Instrument accesses from different address spaces only for AMDGPU.
13785ffd83dbSDimitry Andric   Type *PtrTy = cast<PointerType>(Ptr->getType()->getScalarType());
1379fe6060f1SDimitry Andric   if (PtrTy->getPointerAddressSpace() != 0 &&
1380fe6060f1SDimitry Andric       !(TargetTriple.isAMDGPU() && !isUnsupportedAMDGPUAddrspace(Ptr)))
13815ffd83dbSDimitry Andric     return true;
13820b57cec5SDimitry Andric 
13830b57cec5SDimitry Andric   // Ignore swifterror addresses.
13840b57cec5SDimitry Andric   // swifterror memory addresses are mem2reg promoted by instruction
13850b57cec5SDimitry Andric   // selection. As such they cannot have regular uses like an instrumentation
13860b57cec5SDimitry Andric   // function and it makes no sense to track them as memory.
13875ffd83dbSDimitry Andric   if (Ptr->isSwiftError())
13885ffd83dbSDimitry Andric     return true;
13890b57cec5SDimitry Andric 
13900b57cec5SDimitry Andric   // Treat memory accesses to promotable allocas as non-interesting since they
13910b57cec5SDimitry Andric   // will not cause memory violations. This greatly speeds up the instrumented
13920b57cec5SDimitry Andric   // executable at -O0.
13935ffd83dbSDimitry Andric   if (auto AI = dyn_cast_or_null<AllocaInst>(Ptr))
13945ffd83dbSDimitry Andric     if (ClSkipPromotableAllocas && !isInterestingAlloca(*AI))
13955ffd83dbSDimitry Andric       return true;
13960b57cec5SDimitry Andric 
1397349cc55cSDimitry Andric   if (SSGI != nullptr && SSGI->stackAccessIsSafe(*Inst) &&
1398349cc55cSDimitry Andric       findAllocaForValue(Ptr))
1399349cc55cSDimitry Andric     return true;
1400349cc55cSDimitry Andric 
14015ffd83dbSDimitry Andric   return false;
14025ffd83dbSDimitry Andric }
14035ffd83dbSDimitry Andric 
14045ffd83dbSDimitry Andric void AddressSanitizer::getInterestingMemoryOperands(
14055ffd83dbSDimitry Andric     Instruction *I, SmallVectorImpl<InterestingMemoryOperand> &Interesting) {
14065ffd83dbSDimitry Andric   // Do not instrument the load fetching the dynamic shadow address.
14075ffd83dbSDimitry Andric   if (LocalDynamicShadow == I)
14085ffd83dbSDimitry Andric     return;
14095ffd83dbSDimitry Andric 
14105ffd83dbSDimitry Andric   if (LoadInst *LI = dyn_cast<LoadInst>(I)) {
14111fd87a68SDimitry Andric     if (!ClInstrumentReads || ignoreAccess(I, LI->getPointerOperand()))
14125ffd83dbSDimitry Andric       return;
14135ffd83dbSDimitry Andric     Interesting.emplace_back(I, LI->getPointerOperandIndex(), false,
14145ffd83dbSDimitry Andric                              LI->getType(), LI->getAlign());
14155ffd83dbSDimitry Andric   } else if (StoreInst *SI = dyn_cast<StoreInst>(I)) {
14161fd87a68SDimitry Andric     if (!ClInstrumentWrites || ignoreAccess(I, SI->getPointerOperand()))
14175ffd83dbSDimitry Andric       return;
14185ffd83dbSDimitry Andric     Interesting.emplace_back(I, SI->getPointerOperandIndex(), true,
14195ffd83dbSDimitry Andric                              SI->getValueOperand()->getType(), SI->getAlign());
14205ffd83dbSDimitry Andric   } else if (AtomicRMWInst *RMW = dyn_cast<AtomicRMWInst>(I)) {
14211fd87a68SDimitry Andric     if (!ClInstrumentAtomics || ignoreAccess(I, RMW->getPointerOperand()))
14225ffd83dbSDimitry Andric       return;
14235ffd83dbSDimitry Andric     Interesting.emplace_back(I, RMW->getPointerOperandIndex(), true,
1424bdd1243dSDimitry Andric                              RMW->getValOperand()->getType(), std::nullopt);
14255ffd83dbSDimitry Andric   } else if (AtomicCmpXchgInst *XCHG = dyn_cast<AtomicCmpXchgInst>(I)) {
14261fd87a68SDimitry Andric     if (!ClInstrumentAtomics || ignoreAccess(I, XCHG->getPointerOperand()))
14275ffd83dbSDimitry Andric       return;
14285ffd83dbSDimitry Andric     Interesting.emplace_back(I, XCHG->getPointerOperandIndex(), true,
1429bdd1243dSDimitry Andric                              XCHG->getCompareOperand()->getType(),
1430bdd1243dSDimitry Andric                              std::nullopt);
14315ffd83dbSDimitry Andric   } else if (auto CI = dyn_cast<CallInst>(I)) {
143206c3fb27SDimitry Andric     switch (CI->getIntrinsicID()) {
143306c3fb27SDimitry Andric     case Intrinsic::masked_load:
143406c3fb27SDimitry Andric     case Intrinsic::masked_store:
143506c3fb27SDimitry Andric     case Intrinsic::masked_gather:
143606c3fb27SDimitry Andric     case Intrinsic::masked_scatter: {
143706c3fb27SDimitry Andric       bool IsWrite = CI->getType()->isVoidTy();
14385ffd83dbSDimitry Andric       // Masked store has an initial operand for the value.
14395ffd83dbSDimitry Andric       unsigned OpOffset = IsWrite ? 1 : 0;
14405ffd83dbSDimitry Andric       if (IsWrite ? !ClInstrumentWrites : !ClInstrumentReads)
14415ffd83dbSDimitry Andric         return;
14425ffd83dbSDimitry Andric 
14435ffd83dbSDimitry Andric       auto BasePtr = CI->getOperand(OpOffset);
14441fd87a68SDimitry Andric       if (ignoreAccess(I, BasePtr))
14455ffd83dbSDimitry Andric         return;
144604eeddc0SDimitry Andric       Type *Ty = IsWrite ? CI->getArgOperand(0)->getType() : CI->getType();
14475ffd83dbSDimitry Andric       MaybeAlign Alignment = Align(1);
14485ffd83dbSDimitry Andric       // Otherwise no alignment guarantees. We probably got Undef.
14495ffd83dbSDimitry Andric       if (auto *Op = dyn_cast<ConstantInt>(CI->getOperand(1 + OpOffset)))
14505ffd83dbSDimitry Andric         Alignment = Op->getMaybeAlignValue();
14515ffd83dbSDimitry Andric       Value *Mask = CI->getOperand(2 + OpOffset);
14525ffd83dbSDimitry Andric       Interesting.emplace_back(I, OpOffset, IsWrite, Ty, Alignment, Mask);
145306c3fb27SDimitry Andric       break;
145406c3fb27SDimitry Andric     }
145506c3fb27SDimitry Andric     case Intrinsic::masked_expandload:
145606c3fb27SDimitry Andric     case Intrinsic::masked_compressstore: {
145706c3fb27SDimitry Andric       bool IsWrite = CI->getIntrinsicID() == Intrinsic::masked_compressstore;
145806c3fb27SDimitry Andric       unsigned OpOffset = IsWrite ? 1 : 0;
145906c3fb27SDimitry Andric       if (IsWrite ? !ClInstrumentWrites : !ClInstrumentReads)
146006c3fb27SDimitry Andric         return;
146106c3fb27SDimitry Andric       auto BasePtr = CI->getOperand(OpOffset);
146206c3fb27SDimitry Andric       if (ignoreAccess(I, BasePtr))
146306c3fb27SDimitry Andric         return;
146406c3fb27SDimitry Andric       MaybeAlign Alignment = BasePtr->getPointerAlignment(*DL);
146506c3fb27SDimitry Andric       Type *Ty = IsWrite ? CI->getArgOperand(0)->getType() : CI->getType();
146606c3fb27SDimitry Andric 
146706c3fb27SDimitry Andric       IRBuilder IB(I);
146806c3fb27SDimitry Andric       Value *Mask = CI->getOperand(1 + OpOffset);
146906c3fb27SDimitry Andric       // Use the popcount of Mask as the effective vector length.
147006c3fb27SDimitry Andric       Type *ExtTy = VectorType::get(IntptrTy, cast<VectorType>(Ty));
147106c3fb27SDimitry Andric       Value *ExtMask = IB.CreateZExt(Mask, ExtTy);
147206c3fb27SDimitry Andric       Value *EVL = IB.CreateAddReduce(ExtMask);
147306c3fb27SDimitry Andric       Value *TrueMask = ConstantInt::get(Mask->getType(), 1);
147406c3fb27SDimitry Andric       Interesting.emplace_back(I, OpOffset, IsWrite, Ty, Alignment, TrueMask,
147506c3fb27SDimitry Andric                                EVL);
147606c3fb27SDimitry Andric       break;
147706c3fb27SDimitry Andric     }
147806c3fb27SDimitry Andric     case Intrinsic::vp_load:
147906c3fb27SDimitry Andric     case Intrinsic::vp_store:
148006c3fb27SDimitry Andric     case Intrinsic::experimental_vp_strided_load:
148106c3fb27SDimitry Andric     case Intrinsic::experimental_vp_strided_store: {
148206c3fb27SDimitry Andric       auto *VPI = cast<VPIntrinsic>(CI);
148306c3fb27SDimitry Andric       unsigned IID = CI->getIntrinsicID();
148406c3fb27SDimitry Andric       bool IsWrite = CI->getType()->isVoidTy();
148506c3fb27SDimitry Andric       if (IsWrite ? !ClInstrumentWrites : !ClInstrumentReads)
148606c3fb27SDimitry Andric         return;
148706c3fb27SDimitry Andric       unsigned PtrOpNo = *VPI->getMemoryPointerParamPos(IID);
148806c3fb27SDimitry Andric       Type *Ty = IsWrite ? CI->getArgOperand(0)->getType() : CI->getType();
148906c3fb27SDimitry Andric       MaybeAlign Alignment = VPI->getOperand(PtrOpNo)->getPointerAlignment(*DL);
149006c3fb27SDimitry Andric       Value *Stride = nullptr;
149106c3fb27SDimitry Andric       if (IID == Intrinsic::experimental_vp_strided_store ||
149206c3fb27SDimitry Andric           IID == Intrinsic::experimental_vp_strided_load) {
149306c3fb27SDimitry Andric         Stride = VPI->getOperand(PtrOpNo + 1);
149406c3fb27SDimitry Andric         // Use the pointer alignment as the element alignment if the stride is a
149506c3fb27SDimitry Andric         // mutiple of the pointer alignment. Otherwise, the element alignment
149606c3fb27SDimitry Andric         // should be Align(1).
149706c3fb27SDimitry Andric         unsigned PointerAlign = Alignment.valueOrOne().value();
149806c3fb27SDimitry Andric         if (!isa<ConstantInt>(Stride) ||
149906c3fb27SDimitry Andric             cast<ConstantInt>(Stride)->getZExtValue() % PointerAlign != 0)
150006c3fb27SDimitry Andric           Alignment = Align(1);
150106c3fb27SDimitry Andric       }
150206c3fb27SDimitry Andric       Interesting.emplace_back(I, PtrOpNo, IsWrite, Ty, Alignment,
150306c3fb27SDimitry Andric                                VPI->getMaskParam(), VPI->getVectorLengthParam(),
150406c3fb27SDimitry Andric                                Stride);
150506c3fb27SDimitry Andric       break;
150606c3fb27SDimitry Andric     }
150706c3fb27SDimitry Andric     case Intrinsic::vp_gather:
150806c3fb27SDimitry Andric     case Intrinsic::vp_scatter: {
150906c3fb27SDimitry Andric       auto *VPI = cast<VPIntrinsic>(CI);
151006c3fb27SDimitry Andric       unsigned IID = CI->getIntrinsicID();
151106c3fb27SDimitry Andric       bool IsWrite = IID == Intrinsic::vp_scatter;
151206c3fb27SDimitry Andric       if (IsWrite ? !ClInstrumentWrites : !ClInstrumentReads)
151306c3fb27SDimitry Andric         return;
151406c3fb27SDimitry Andric       unsigned PtrOpNo = *VPI->getMemoryPointerParamPos(IID);
151506c3fb27SDimitry Andric       Type *Ty = IsWrite ? CI->getArgOperand(0)->getType() : CI->getType();
151606c3fb27SDimitry Andric       MaybeAlign Alignment = VPI->getPointerAlignment();
151706c3fb27SDimitry Andric       Interesting.emplace_back(I, PtrOpNo, IsWrite, Ty, Alignment,
151806c3fb27SDimitry Andric                                VPI->getMaskParam(),
151906c3fb27SDimitry Andric                                VPI->getVectorLengthParam());
152006c3fb27SDimitry Andric       break;
152106c3fb27SDimitry Andric     }
152206c3fb27SDimitry Andric     default:
1523349cc55cSDimitry Andric       for (unsigned ArgNo = 0; ArgNo < CI->arg_size(); ArgNo++) {
15245ffd83dbSDimitry Andric         if (!ClInstrumentByval || !CI->isByValArgument(ArgNo) ||
15251fd87a68SDimitry Andric             ignoreAccess(I, CI->getArgOperand(ArgNo)))
15265ffd83dbSDimitry Andric           continue;
15275ffd83dbSDimitry Andric         Type *Ty = CI->getParamByValType(ArgNo);
15285ffd83dbSDimitry Andric         Interesting.emplace_back(I, ArgNo, false, Ty, Align(1));
15295ffd83dbSDimitry Andric       }
15305ffd83dbSDimitry Andric     }
15315ffd83dbSDimitry Andric   }
15320b57cec5SDimitry Andric }
15330b57cec5SDimitry Andric 
15340b57cec5SDimitry Andric static bool isPointerOperand(Value *V) {
15350b57cec5SDimitry Andric   return V->getType()->isPointerTy() || isa<PtrToIntInst>(V);
15360b57cec5SDimitry Andric }
15370b57cec5SDimitry Andric 
15380b57cec5SDimitry Andric // This is a rough heuristic; it may cause both false positives and
15390b57cec5SDimitry Andric // false negatives. The proper implementation requires cooperation with
15400b57cec5SDimitry Andric // the frontend.
15410b57cec5SDimitry Andric static bool isInterestingPointerComparison(Instruction *I) {
15420b57cec5SDimitry Andric   if (ICmpInst *Cmp = dyn_cast<ICmpInst>(I)) {
15430b57cec5SDimitry Andric     if (!Cmp->isRelational())
15440b57cec5SDimitry Andric       return false;
15450b57cec5SDimitry Andric   } else {
15460b57cec5SDimitry Andric     return false;
15470b57cec5SDimitry Andric   }
15480b57cec5SDimitry Andric   return isPointerOperand(I->getOperand(0)) &&
15490b57cec5SDimitry Andric          isPointerOperand(I->getOperand(1));
15500b57cec5SDimitry Andric }
15510b57cec5SDimitry Andric 
15520b57cec5SDimitry Andric // This is a rough heuristic; it may cause both false positives and
15530b57cec5SDimitry Andric // false negatives. The proper implementation requires cooperation with
15540b57cec5SDimitry Andric // the frontend.
15550b57cec5SDimitry Andric static bool isInterestingPointerSubtraction(Instruction *I) {
15560b57cec5SDimitry Andric   if (BinaryOperator *BO = dyn_cast<BinaryOperator>(I)) {
15570b57cec5SDimitry Andric     if (BO->getOpcode() != Instruction::Sub)
15580b57cec5SDimitry Andric       return false;
15590b57cec5SDimitry Andric   } else {
15600b57cec5SDimitry Andric     return false;
15610b57cec5SDimitry Andric   }
15620b57cec5SDimitry Andric   return isPointerOperand(I->getOperand(0)) &&
15630b57cec5SDimitry Andric          isPointerOperand(I->getOperand(1));
15640b57cec5SDimitry Andric }
15650b57cec5SDimitry Andric 
15660b57cec5SDimitry Andric bool AddressSanitizer::GlobalIsLinkerInitialized(GlobalVariable *G) {
15670b57cec5SDimitry Andric   // If a global variable does not have dynamic initialization we don't
15680b57cec5SDimitry Andric   // have to instrument it.  However, if a global does not have initializer
15690b57cec5SDimitry Andric   // at all, we assume it has dynamic initializer (in other TU).
157081ad6265SDimitry Andric   if (!G->hasInitializer())
157181ad6265SDimitry Andric     return false;
157281ad6265SDimitry Andric 
157381ad6265SDimitry Andric   if (G->hasSanitizerMetadata() && G->getSanitizerMetadata().IsDynInit)
157481ad6265SDimitry Andric     return false;
157581ad6265SDimitry Andric 
157681ad6265SDimitry Andric   return true;
15770b57cec5SDimitry Andric }
15780b57cec5SDimitry Andric 
15790b57cec5SDimitry Andric void AddressSanitizer::instrumentPointerComparisonOrSubtraction(
1580*0fca6ea1SDimitry Andric     Instruction *I, RuntimeCallInserter &RTCI) {
15810b57cec5SDimitry Andric   IRBuilder<> IRB(I);
15820b57cec5SDimitry Andric   FunctionCallee F = isa<ICmpInst>(I) ? AsanPtrCmpFunction : AsanPtrSubFunction;
15830b57cec5SDimitry Andric   Value *Param[2] = {I->getOperand(0), I->getOperand(1)};
15840b57cec5SDimitry Andric   for (Value *&i : Param) {
15850b57cec5SDimitry Andric     if (i->getType()->isPointerTy())
15860b57cec5SDimitry Andric       i = IRB.CreatePointerCast(i, IntptrTy);
15870b57cec5SDimitry Andric   }
1588*0fca6ea1SDimitry Andric   RTCI.createRuntimeCall(IRB, F, Param);
15890b57cec5SDimitry Andric }
15900b57cec5SDimitry Andric 
15910b57cec5SDimitry Andric static void doInstrumentAddress(AddressSanitizer *Pass, Instruction *I,
15920b57cec5SDimitry Andric                                 Instruction *InsertBefore, Value *Addr,
15935ffd83dbSDimitry Andric                                 MaybeAlign Alignment, unsigned Granularity,
159406c3fb27SDimitry Andric                                 TypeSize TypeStoreSize, bool IsWrite,
15950b57cec5SDimitry Andric                                 Value *SizeArgument, bool UseCalls,
1596*0fca6ea1SDimitry Andric                                 uint32_t Exp, RuntimeCallInserter &RTCI) {
15970b57cec5SDimitry Andric   // Instrument a 1-, 2-, 4-, 8-, or 16- byte access with one check
15980b57cec5SDimitry Andric   // if the data is properly aligned.
159906c3fb27SDimitry Andric   if (!TypeStoreSize.isScalable()) {
160006c3fb27SDimitry Andric     const auto FixedSize = TypeStoreSize.getFixedValue();
160106c3fb27SDimitry Andric     switch (FixedSize) {
160206c3fb27SDimitry Andric     case 8:
160306c3fb27SDimitry Andric     case 16:
160406c3fb27SDimitry Andric     case 32:
160506c3fb27SDimitry Andric     case 64:
160606c3fb27SDimitry Andric     case 128:
160706c3fb27SDimitry Andric       if (!Alignment || *Alignment >= Granularity ||
160806c3fb27SDimitry Andric           *Alignment >= FixedSize / 8)
160906c3fb27SDimitry Andric         return Pass->instrumentAddress(I, InsertBefore, Addr, Alignment,
161006c3fb27SDimitry Andric                                        FixedSize, IsWrite, nullptr, UseCalls,
1611*0fca6ea1SDimitry Andric                                        Exp, RTCI);
161206c3fb27SDimitry Andric     }
161306c3fb27SDimitry Andric   }
161406c3fb27SDimitry Andric   Pass->instrumentUnusualSizeOrAlignment(I, InsertBefore, Addr, TypeStoreSize,
1615*0fca6ea1SDimitry Andric                                          IsWrite, nullptr, UseCalls, Exp, RTCI);
16160b57cec5SDimitry Andric }
16170b57cec5SDimitry Andric 
161806c3fb27SDimitry Andric void AddressSanitizer::instrumentMaskedLoadOrStore(
161906c3fb27SDimitry Andric     AddressSanitizer *Pass, const DataLayout &DL, Type *IntptrTy, Value *Mask,
162006c3fb27SDimitry Andric     Value *EVL, Value *Stride, Instruction *I, Value *Addr,
162106c3fb27SDimitry Andric     MaybeAlign Alignment, unsigned Granularity, Type *OpType, bool IsWrite,
1622*0fca6ea1SDimitry Andric     Value *SizeArgument, bool UseCalls, uint32_t Exp,
1623*0fca6ea1SDimitry Andric     RuntimeCallInserter &RTCI) {
162406c3fb27SDimitry Andric   auto *VTy = cast<VectorType>(OpType);
162506c3fb27SDimitry Andric   TypeSize ElemTypeSize = DL.getTypeStoreSizeInBits(VTy->getScalarType());
16260b57cec5SDimitry Andric   auto Zero = ConstantInt::get(IntptrTy, 0);
162706c3fb27SDimitry Andric 
162806c3fb27SDimitry Andric   IRBuilder IB(I);
162906c3fb27SDimitry Andric   Instruction *LoopInsertBefore = I;
163006c3fb27SDimitry Andric   if (EVL) {
163106c3fb27SDimitry Andric     // The end argument of SplitBlockAndInsertForLane is assumed bigger
163206c3fb27SDimitry Andric     // than zero, so we should check whether EVL is zero here.
163306c3fb27SDimitry Andric     Type *EVLType = EVL->getType();
163406c3fb27SDimitry Andric     Value *IsEVLZero = IB.CreateICmpNE(EVL, ConstantInt::get(EVLType, 0));
163506c3fb27SDimitry Andric     LoopInsertBefore = SplitBlockAndInsertIfThen(IsEVLZero, I, false);
163606c3fb27SDimitry Andric     IB.SetInsertPoint(LoopInsertBefore);
163706c3fb27SDimitry Andric     // Cast EVL to IntptrTy.
163806c3fb27SDimitry Andric     EVL = IB.CreateZExtOrTrunc(EVL, IntptrTy);
163906c3fb27SDimitry Andric     // To avoid undefined behavior for extracting with out of range index, use
164006c3fb27SDimitry Andric     // the minimum of evl and element count as trip count.
164106c3fb27SDimitry Andric     Value *EC = IB.CreateElementCount(IntptrTy, VTy->getElementCount());
164206c3fb27SDimitry Andric     EVL = IB.CreateBinaryIntrinsic(Intrinsic::umin, EVL, EC);
16430b57cec5SDimitry Andric   } else {
164406c3fb27SDimitry Andric     EVL = IB.CreateElementCount(IntptrTy, VTy->getElementCount());
16450b57cec5SDimitry Andric   }
16460b57cec5SDimitry Andric 
164706c3fb27SDimitry Andric   // Cast Stride to IntptrTy.
164806c3fb27SDimitry Andric   if (Stride)
164906c3fb27SDimitry Andric     Stride = IB.CreateZExtOrTrunc(Stride, IntptrTy);
165006c3fb27SDimitry Andric 
165106c3fb27SDimitry Andric   SplitBlockAndInsertForEachLane(EVL, LoopInsertBefore,
165206c3fb27SDimitry Andric                                  [&](IRBuilderBase &IRB, Value *Index) {
165306c3fb27SDimitry Andric     Value *MaskElem = IRB.CreateExtractElement(Mask, Index);
165406c3fb27SDimitry Andric     if (auto *MaskElemC = dyn_cast<ConstantInt>(MaskElem)) {
165506c3fb27SDimitry Andric       if (MaskElemC->isZero())
165606c3fb27SDimitry Andric         // No check
165706c3fb27SDimitry Andric         return;
165806c3fb27SDimitry Andric       // Unconditional check
165906c3fb27SDimitry Andric     } else {
166006c3fb27SDimitry Andric       // Conditional check
166106c3fb27SDimitry Andric       Instruction *ThenTerm = SplitBlockAndInsertIfThen(
166206c3fb27SDimitry Andric           MaskElem, &*IRB.GetInsertPoint(), false);
166306c3fb27SDimitry Andric       IRB.SetInsertPoint(ThenTerm);
16640b57cec5SDimitry Andric     }
166506c3fb27SDimitry Andric 
166606c3fb27SDimitry Andric     Value *InstrumentedAddress;
166706c3fb27SDimitry Andric     if (isa<VectorType>(Addr->getType())) {
166806c3fb27SDimitry Andric       assert(
166906c3fb27SDimitry Andric           cast<VectorType>(Addr->getType())->getElementType()->isPointerTy() &&
167006c3fb27SDimitry Andric           "Expected vector of pointer.");
167106c3fb27SDimitry Andric       InstrumentedAddress = IRB.CreateExtractElement(Addr, Index);
167206c3fb27SDimitry Andric     } else if (Stride) {
167306c3fb27SDimitry Andric       Index = IRB.CreateMul(Index, Stride);
16747a6dacacSDimitry Andric       InstrumentedAddress = IRB.CreatePtrAdd(Addr, Index);
167506c3fb27SDimitry Andric     } else {
167606c3fb27SDimitry Andric       InstrumentedAddress = IRB.CreateGEP(VTy, Addr, {Zero, Index});
167706c3fb27SDimitry Andric     }
1678*0fca6ea1SDimitry Andric     doInstrumentAddress(Pass, I, &*IRB.GetInsertPoint(), InstrumentedAddress,
1679*0fca6ea1SDimitry Andric                         Alignment, Granularity, ElemTypeSize, IsWrite,
1680*0fca6ea1SDimitry Andric                         SizeArgument, UseCalls, Exp, RTCI);
168106c3fb27SDimitry Andric   });
16820b57cec5SDimitry Andric }
16830b57cec5SDimitry Andric 
16840b57cec5SDimitry Andric void AddressSanitizer::instrumentMop(ObjectSizeOffsetVisitor &ObjSizeVis,
16855ffd83dbSDimitry Andric                                      InterestingMemoryOperand &O, bool UseCalls,
1686*0fca6ea1SDimitry Andric                                      const DataLayout &DL,
1687*0fca6ea1SDimitry Andric                                      RuntimeCallInserter &RTCI) {
16885ffd83dbSDimitry Andric   Value *Addr = O.getPtr();
16890b57cec5SDimitry Andric 
16900b57cec5SDimitry Andric   // Optimization experiments.
16910b57cec5SDimitry Andric   // The experiments can be used to evaluate potential optimizations that remove
16920b57cec5SDimitry Andric   // instrumentation (assess false negatives). Instead of completely removing
16930b57cec5SDimitry Andric   // some instrumentation, you set Exp to a non-zero value (mask of optimization
16940b57cec5SDimitry Andric   // experiments that want to remove instrumentation of this instruction).
16950b57cec5SDimitry Andric   // If Exp is non-zero, this pass will emit special calls into runtime
16960b57cec5SDimitry Andric   // (e.g. __asan_report_exp_load1 instead of __asan_report_load1). These calls
16970b57cec5SDimitry Andric   // make runtime terminate the program in a special way (with a different
16980b57cec5SDimitry Andric   // exit status). Then you run the new compiler on a buggy corpus, collect
16990b57cec5SDimitry Andric   // the special terminations (ideally, you don't see them at all -- no false
17000b57cec5SDimitry Andric   // negatives) and make the decision on the optimization.
17010b57cec5SDimitry Andric   uint32_t Exp = ClForceExperiment;
17020b57cec5SDimitry Andric 
17030b57cec5SDimitry Andric   if (ClOpt && ClOptGlobals) {
17040b57cec5SDimitry Andric     // If initialization order checking is disabled, a simple access to a
17050b57cec5SDimitry Andric     // dynamically initialized global is always valid.
1706e8d8bef9SDimitry Andric     GlobalVariable *G = dyn_cast<GlobalVariable>(getUnderlyingObject(Addr));
17070b57cec5SDimitry Andric     if (G && (!ClInitializers || GlobalIsLinkerInitialized(G)) &&
170806c3fb27SDimitry Andric         isSafeAccess(ObjSizeVis, Addr, O.TypeStoreSize)) {
17090b57cec5SDimitry Andric       NumOptimizedAccessesToGlobalVar++;
17100b57cec5SDimitry Andric       return;
17110b57cec5SDimitry Andric     }
17120b57cec5SDimitry Andric   }
17130b57cec5SDimitry Andric 
17140b57cec5SDimitry Andric   if (ClOpt && ClOptStack) {
17150b57cec5SDimitry Andric     // A direct inbounds access to a stack variable is always valid.
1716e8d8bef9SDimitry Andric     if (isa<AllocaInst>(getUnderlyingObject(Addr)) &&
171706c3fb27SDimitry Andric         isSafeAccess(ObjSizeVis, Addr, O.TypeStoreSize)) {
17180b57cec5SDimitry Andric       NumOptimizedAccessesToStackVar++;
17190b57cec5SDimitry Andric       return;
17200b57cec5SDimitry Andric     }
17210b57cec5SDimitry Andric   }
17220b57cec5SDimitry Andric 
17235ffd83dbSDimitry Andric   if (O.IsWrite)
17240b57cec5SDimitry Andric     NumInstrumentedWrites++;
17250b57cec5SDimitry Andric   else
17260b57cec5SDimitry Andric     NumInstrumentedReads++;
17270b57cec5SDimitry Andric 
17280b57cec5SDimitry Andric   unsigned Granularity = 1 << Mapping.Scale;
17295ffd83dbSDimitry Andric   if (O.MaybeMask) {
173006c3fb27SDimitry Andric     instrumentMaskedLoadOrStore(this, DL, IntptrTy, O.MaybeMask, O.MaybeEVL,
173106c3fb27SDimitry Andric                                 O.MaybeStride, O.getInsn(), Addr, O.Alignment,
173206c3fb27SDimitry Andric                                 Granularity, O.OpType, O.IsWrite, nullptr,
1733*0fca6ea1SDimitry Andric                                 UseCalls, Exp, RTCI);
17340b57cec5SDimitry Andric   } else {
17355ffd83dbSDimitry Andric     doInstrumentAddress(this, O.getInsn(), O.getInsn(), Addr, O.Alignment,
1736*0fca6ea1SDimitry Andric                         Granularity, O.TypeStoreSize, O.IsWrite, nullptr,
1737*0fca6ea1SDimitry Andric                         UseCalls, Exp, RTCI);
17380b57cec5SDimitry Andric   }
17390b57cec5SDimitry Andric }
17400b57cec5SDimitry Andric 
17410b57cec5SDimitry Andric Instruction *AddressSanitizer::generateCrashCode(Instruction *InsertBefore,
17420b57cec5SDimitry Andric                                                  Value *Addr, bool IsWrite,
17430b57cec5SDimitry Andric                                                  size_t AccessSizeIndex,
17440b57cec5SDimitry Andric                                                  Value *SizeArgument,
1745*0fca6ea1SDimitry Andric                                                  uint32_t Exp,
1746*0fca6ea1SDimitry Andric                                                  RuntimeCallInserter &RTCI) {
174706c3fb27SDimitry Andric   InstrumentationIRBuilder IRB(InsertBefore);
17480b57cec5SDimitry Andric   Value *ExpVal = Exp == 0 ? nullptr : ConstantInt::get(IRB.getInt32Ty(), Exp);
17490b57cec5SDimitry Andric   CallInst *Call = nullptr;
17500b57cec5SDimitry Andric   if (SizeArgument) {
17510b57cec5SDimitry Andric     if (Exp == 0)
1752*0fca6ea1SDimitry Andric       Call = RTCI.createRuntimeCall(IRB, AsanErrorCallbackSized[IsWrite][0],
17530b57cec5SDimitry Andric                                     {Addr, SizeArgument});
17540b57cec5SDimitry Andric     else
1755*0fca6ea1SDimitry Andric       Call = RTCI.createRuntimeCall(IRB, AsanErrorCallbackSized[IsWrite][1],
17560b57cec5SDimitry Andric                                     {Addr, SizeArgument, ExpVal});
17570b57cec5SDimitry Andric   } else {
17580b57cec5SDimitry Andric     if (Exp == 0)
1759*0fca6ea1SDimitry Andric       Call = RTCI.createRuntimeCall(
1760*0fca6ea1SDimitry Andric           IRB, AsanErrorCallback[IsWrite][0][AccessSizeIndex], Addr);
17610b57cec5SDimitry Andric     else
1762*0fca6ea1SDimitry Andric       Call = RTCI.createRuntimeCall(
1763*0fca6ea1SDimitry Andric           IRB, AsanErrorCallback[IsWrite][1][AccessSizeIndex], {Addr, ExpVal});
17640b57cec5SDimitry Andric   }
17650b57cec5SDimitry Andric 
17665ffd83dbSDimitry Andric   Call->setCannotMerge();
17670b57cec5SDimitry Andric   return Call;
17680b57cec5SDimitry Andric }
17690b57cec5SDimitry Andric 
17700b57cec5SDimitry Andric Value *AddressSanitizer::createSlowPathCmp(IRBuilder<> &IRB, Value *AddrLong,
17710b57cec5SDimitry Andric                                            Value *ShadowValue,
177206c3fb27SDimitry Andric                                            uint32_t TypeStoreSize) {
17730b57cec5SDimitry Andric   size_t Granularity = static_cast<size_t>(1) << Mapping.Scale;
17740b57cec5SDimitry Andric   // Addr & (Granularity - 1)
17750b57cec5SDimitry Andric   Value *LastAccessedByte =
17760b57cec5SDimitry Andric       IRB.CreateAnd(AddrLong, ConstantInt::get(IntptrTy, Granularity - 1));
17770b57cec5SDimitry Andric   // (Addr & (Granularity - 1)) + size - 1
177806c3fb27SDimitry Andric   if (TypeStoreSize / 8 > 1)
17790b57cec5SDimitry Andric     LastAccessedByte = IRB.CreateAdd(
178006c3fb27SDimitry Andric         LastAccessedByte, ConstantInt::get(IntptrTy, TypeStoreSize / 8 - 1));
17810b57cec5SDimitry Andric   // (uint8_t) ((Addr & (Granularity-1)) + size - 1)
17820b57cec5SDimitry Andric   LastAccessedByte =
17830b57cec5SDimitry Andric       IRB.CreateIntCast(LastAccessedByte, ShadowValue->getType(), false);
17840b57cec5SDimitry Andric   // ((uint8_t) ((Addr & (Granularity-1)) + size - 1)) >= ShadowValue
17850b57cec5SDimitry Andric   return IRB.CreateICmpSGE(LastAccessedByte, ShadowValue);
17860b57cec5SDimitry Andric }
17870b57cec5SDimitry Andric 
1788fe6060f1SDimitry Andric Instruction *AddressSanitizer::instrumentAMDGPUAddress(
1789fe6060f1SDimitry Andric     Instruction *OrigIns, Instruction *InsertBefore, Value *Addr,
179006c3fb27SDimitry Andric     uint32_t TypeStoreSize, bool IsWrite, Value *SizeArgument) {
1791fe6060f1SDimitry Andric   // Do not instrument unsupported addrspaces.
1792fe6060f1SDimitry Andric   if (isUnsupportedAMDGPUAddrspace(Addr))
1793fe6060f1SDimitry Andric     return nullptr;
1794fe6060f1SDimitry Andric   Type *PtrTy = cast<PointerType>(Addr->getType()->getScalarType());
1795fe6060f1SDimitry Andric   // Follow host instrumentation for global and constant addresses.
1796fe6060f1SDimitry Andric   if (PtrTy->getPointerAddressSpace() != 0)
1797fe6060f1SDimitry Andric     return InsertBefore;
1798fe6060f1SDimitry Andric   // Instrument generic addresses in supported addressspaces.
1799fe6060f1SDimitry Andric   IRBuilder<> IRB(InsertBefore);
18005f757f3fSDimitry Andric   Value *IsShared = IRB.CreateCall(AMDGPUAddressShared, {Addr});
18015f757f3fSDimitry Andric   Value *IsPrivate = IRB.CreateCall(AMDGPUAddressPrivate, {Addr});
1802fe6060f1SDimitry Andric   Value *IsSharedOrPrivate = IRB.CreateOr(IsShared, IsPrivate);
1803bdd1243dSDimitry Andric   Value *Cmp = IRB.CreateNot(IsSharedOrPrivate);
1804fe6060f1SDimitry Andric   Value *AddrSpaceZeroLanding =
1805fe6060f1SDimitry Andric       SplitBlockAndInsertIfThen(Cmp, InsertBefore, false);
1806fe6060f1SDimitry Andric   InsertBefore = cast<Instruction>(AddrSpaceZeroLanding);
1807fe6060f1SDimitry Andric   return InsertBefore;
1808fe6060f1SDimitry Andric }
1809fe6060f1SDimitry Andric 
18101db9f3b2SDimitry Andric Instruction *AddressSanitizer::genAMDGPUReportBlock(IRBuilder<> &IRB,
18111db9f3b2SDimitry Andric                                                     Value *Cond, bool Recover) {
18121db9f3b2SDimitry Andric   Module &M = *IRB.GetInsertBlock()->getModule();
18131db9f3b2SDimitry Andric   Value *ReportCond = Cond;
18141db9f3b2SDimitry Andric   if (!Recover) {
18151db9f3b2SDimitry Andric     auto Ballot = M.getOrInsertFunction(kAMDGPUBallotName, IRB.getInt64Ty(),
18161db9f3b2SDimitry Andric                                         IRB.getInt1Ty());
18171db9f3b2SDimitry Andric     ReportCond = IRB.CreateIsNotNull(IRB.CreateCall(Ballot, {Cond}));
18181db9f3b2SDimitry Andric   }
18191db9f3b2SDimitry Andric 
18201db9f3b2SDimitry Andric   auto *Trm =
18211db9f3b2SDimitry Andric       SplitBlockAndInsertIfThen(ReportCond, &*IRB.GetInsertPoint(), false,
1822*0fca6ea1SDimitry Andric                                 MDBuilder(*C).createUnlikelyBranchWeights());
18231db9f3b2SDimitry Andric   Trm->getParent()->setName("asan.report");
18241db9f3b2SDimitry Andric 
18251db9f3b2SDimitry Andric   if (Recover)
18261db9f3b2SDimitry Andric     return Trm;
18271db9f3b2SDimitry Andric 
18281db9f3b2SDimitry Andric   Trm = SplitBlockAndInsertIfThen(Cond, Trm, false);
18291db9f3b2SDimitry Andric   IRB.SetInsertPoint(Trm);
18301db9f3b2SDimitry Andric   return IRB.CreateCall(
18311db9f3b2SDimitry Andric       M.getOrInsertFunction(kAMDGPUUnreachableName, IRB.getVoidTy()), {});
18321db9f3b2SDimitry Andric }
18331db9f3b2SDimitry Andric 
18340b57cec5SDimitry Andric void AddressSanitizer::instrumentAddress(Instruction *OrigIns,
18350b57cec5SDimitry Andric                                          Instruction *InsertBefore, Value *Addr,
183606c3fb27SDimitry Andric                                          MaybeAlign Alignment,
183706c3fb27SDimitry Andric                                          uint32_t TypeStoreSize, bool IsWrite,
18380b57cec5SDimitry Andric                                          Value *SizeArgument, bool UseCalls,
1839*0fca6ea1SDimitry Andric                                          uint32_t Exp,
1840*0fca6ea1SDimitry Andric                                          RuntimeCallInserter &RTCI) {
1841fe6060f1SDimitry Andric   if (TargetTriple.isAMDGPU()) {
1842fe6060f1SDimitry Andric     InsertBefore = instrumentAMDGPUAddress(OrigIns, InsertBefore, Addr,
184306c3fb27SDimitry Andric                                            TypeStoreSize, IsWrite, SizeArgument);
1844fe6060f1SDimitry Andric     if (!InsertBefore)
1845fe6060f1SDimitry Andric       return;
1846fe6060f1SDimitry Andric   }
18470b57cec5SDimitry Andric 
184806c3fb27SDimitry Andric   InstrumentationIRBuilder IRB(InsertBefore);
184906c3fb27SDimitry Andric   size_t AccessSizeIndex = TypeStoreSizeToSizeIndex(TypeStoreSize);
1850349cc55cSDimitry Andric   const ASanAccessInfo AccessInfo(IsWrite, CompileKernel, AccessSizeIndex);
18510b57cec5SDimitry Andric 
1852349cc55cSDimitry Andric   if (UseCalls && ClOptimizeCallbacks) {
1853349cc55cSDimitry Andric     const ASanAccessInfo AccessInfo(IsWrite, CompileKernel, AccessSizeIndex);
1854349cc55cSDimitry Andric     Module *M = IRB.GetInsertBlock()->getParent()->getParent();
1855349cc55cSDimitry Andric     IRB.CreateCall(
1856349cc55cSDimitry Andric         Intrinsic::getDeclaration(M, Intrinsic::asan_check_memaccess),
18575f757f3fSDimitry Andric         {IRB.CreatePointerCast(Addr, PtrTy),
1858349cc55cSDimitry Andric          ConstantInt::get(Int32Ty, AccessInfo.Packed)});
1859349cc55cSDimitry Andric     return;
1860349cc55cSDimitry Andric   }
1861349cc55cSDimitry Andric 
1862349cc55cSDimitry Andric   Value *AddrLong = IRB.CreatePointerCast(Addr, IntptrTy);
18630b57cec5SDimitry Andric   if (UseCalls) {
18640b57cec5SDimitry Andric     if (Exp == 0)
1865*0fca6ea1SDimitry Andric       RTCI.createRuntimeCall(
1866*0fca6ea1SDimitry Andric           IRB, AsanMemoryAccessCallback[IsWrite][0][AccessSizeIndex], AddrLong);
18670b57cec5SDimitry Andric     else
1868*0fca6ea1SDimitry Andric       RTCI.createRuntimeCall(
1869*0fca6ea1SDimitry Andric           IRB, AsanMemoryAccessCallback[IsWrite][1][AccessSizeIndex],
18700b57cec5SDimitry Andric           {AddrLong, ConstantInt::get(IRB.getInt32Ty(), Exp)});
18710b57cec5SDimitry Andric     return;
18720b57cec5SDimitry Andric   }
18730b57cec5SDimitry Andric 
18740b57cec5SDimitry Andric   Type *ShadowTy =
187506c3fb27SDimitry Andric       IntegerType::get(*C, std::max(8U, TypeStoreSize >> Mapping.Scale));
18760b57cec5SDimitry Andric   Type *ShadowPtrTy = PointerType::get(ShadowTy, 0);
18770b57cec5SDimitry Andric   Value *ShadowPtr = memToShadow(AddrLong, IRB);
187806c3fb27SDimitry Andric   const uint64_t ShadowAlign =
187906c3fb27SDimitry Andric       std::max<uint64_t>(Alignment.valueOrOne().value() >> Mapping.Scale, 1);
188006c3fb27SDimitry Andric   Value *ShadowValue = IRB.CreateAlignedLoad(
188106c3fb27SDimitry Andric       ShadowTy, IRB.CreateIntToPtr(ShadowPtr, ShadowPtrTy), Align(ShadowAlign));
18820b57cec5SDimitry Andric 
1883bdd1243dSDimitry Andric   Value *Cmp = IRB.CreateIsNotNull(ShadowValue);
18840b57cec5SDimitry Andric   size_t Granularity = 1ULL << Mapping.Scale;
18850b57cec5SDimitry Andric   Instruction *CrashTerm = nullptr;
18860b57cec5SDimitry Andric 
18871db9f3b2SDimitry Andric   bool GenSlowPath = (ClAlwaysSlowPath || (TypeStoreSize < 8 * Granularity));
18881db9f3b2SDimitry Andric 
18891db9f3b2SDimitry Andric   if (TargetTriple.isAMDGCN()) {
18901db9f3b2SDimitry Andric     if (GenSlowPath) {
18911db9f3b2SDimitry Andric       auto *Cmp2 = createSlowPathCmp(IRB, AddrLong, ShadowValue, TypeStoreSize);
18921db9f3b2SDimitry Andric       Cmp = IRB.CreateAnd(Cmp, Cmp2);
18931db9f3b2SDimitry Andric     }
18941db9f3b2SDimitry Andric     CrashTerm = genAMDGPUReportBlock(IRB, Cmp, Recover);
18951db9f3b2SDimitry Andric   } else if (GenSlowPath) {
18960b57cec5SDimitry Andric     // We use branch weights for the slow path check, to indicate that the slow
18970b57cec5SDimitry Andric     // path is rarely taken. This seems to be the case for SPEC benchmarks.
18980b57cec5SDimitry Andric     Instruction *CheckTerm = SplitBlockAndInsertIfThen(
1899*0fca6ea1SDimitry Andric         Cmp, InsertBefore, false, MDBuilder(*C).createUnlikelyBranchWeights());
19000b57cec5SDimitry Andric     assert(cast<BranchInst>(CheckTerm)->isUnconditional());
19010b57cec5SDimitry Andric     BasicBlock *NextBB = CheckTerm->getSuccessor(0);
19020b57cec5SDimitry Andric     IRB.SetInsertPoint(CheckTerm);
190306c3fb27SDimitry Andric     Value *Cmp2 = createSlowPathCmp(IRB, AddrLong, ShadowValue, TypeStoreSize);
19040b57cec5SDimitry Andric     if (Recover) {
19050b57cec5SDimitry Andric       CrashTerm = SplitBlockAndInsertIfThen(Cmp2, CheckTerm, false);
19060b57cec5SDimitry Andric     } else {
19070b57cec5SDimitry Andric       BasicBlock *CrashBlock =
19080b57cec5SDimitry Andric         BasicBlock::Create(*C, "", NextBB->getParent(), NextBB);
19090b57cec5SDimitry Andric       CrashTerm = new UnreachableInst(*C, CrashBlock);
19100b57cec5SDimitry Andric       BranchInst *NewTerm = BranchInst::Create(CrashBlock, NextBB, Cmp2);
19110b57cec5SDimitry Andric       ReplaceInstWithInst(CheckTerm, NewTerm);
19120b57cec5SDimitry Andric     }
19130b57cec5SDimitry Andric   } else {
19140b57cec5SDimitry Andric     CrashTerm = SplitBlockAndInsertIfThen(Cmp, InsertBefore, !Recover);
19150b57cec5SDimitry Andric   }
19160b57cec5SDimitry Andric 
1917*0fca6ea1SDimitry Andric   Instruction *Crash = generateCrashCode(
1918*0fca6ea1SDimitry Andric       CrashTerm, AddrLong, IsWrite, AccessSizeIndex, SizeArgument, Exp, RTCI);
191906c3fb27SDimitry Andric   if (OrigIns->getDebugLoc())
19200b57cec5SDimitry Andric     Crash->setDebugLoc(OrigIns->getDebugLoc());
19210b57cec5SDimitry Andric }
19220b57cec5SDimitry Andric 
19230b57cec5SDimitry Andric // Instrument unusual size or unusual alignment.
19240b57cec5SDimitry Andric // We can not do it with a single check, so we do 1-byte check for the first
19250b57cec5SDimitry Andric // and the last bytes. We call __asan_report_*_n(addr, real_size) to be able
19260b57cec5SDimitry Andric // to report the actual access size.
19270b57cec5SDimitry Andric void AddressSanitizer::instrumentUnusualSizeOrAlignment(
1928*0fca6ea1SDimitry Andric     Instruction *I, Instruction *InsertBefore, Value *Addr,
1929*0fca6ea1SDimitry Andric     TypeSize TypeStoreSize, bool IsWrite, Value *SizeArgument, bool UseCalls,
1930*0fca6ea1SDimitry Andric     uint32_t Exp, RuntimeCallInserter &RTCI) {
193106c3fb27SDimitry Andric   InstrumentationIRBuilder IRB(InsertBefore);
193206c3fb27SDimitry Andric   Value *NumBits = IRB.CreateTypeSize(IntptrTy, TypeStoreSize);
193306c3fb27SDimitry Andric   Value *Size = IRB.CreateLShr(NumBits, ConstantInt::get(IntptrTy, 3));
193406c3fb27SDimitry Andric 
19350b57cec5SDimitry Andric   Value *AddrLong = IRB.CreatePointerCast(Addr, IntptrTy);
19360b57cec5SDimitry Andric   if (UseCalls) {
19370b57cec5SDimitry Andric     if (Exp == 0)
1938*0fca6ea1SDimitry Andric       RTCI.createRuntimeCall(IRB, AsanMemoryAccessCallbackSized[IsWrite][0],
19390b57cec5SDimitry Andric                              {AddrLong, Size});
19400b57cec5SDimitry Andric     else
1941*0fca6ea1SDimitry Andric       RTCI.createRuntimeCall(
1942*0fca6ea1SDimitry Andric           IRB, AsanMemoryAccessCallbackSized[IsWrite][1],
19430b57cec5SDimitry Andric           {AddrLong, Size, ConstantInt::get(IRB.getInt32Ty(), Exp)});
19440b57cec5SDimitry Andric   } else {
194506c3fb27SDimitry Andric     Value *SizeMinusOne = IRB.CreateSub(Size, ConstantInt::get(IntptrTy, 1));
19460b57cec5SDimitry Andric     Value *LastByte = IRB.CreateIntToPtr(
194706c3fb27SDimitry Andric         IRB.CreateAdd(AddrLong, SizeMinusOne),
19480b57cec5SDimitry Andric         Addr->getType());
1949*0fca6ea1SDimitry Andric     instrumentAddress(I, InsertBefore, Addr, {}, 8, IsWrite, Size, false, Exp,
1950*0fca6ea1SDimitry Andric                       RTCI);
195106c3fb27SDimitry Andric     instrumentAddress(I, InsertBefore, LastByte, {}, 8, IsWrite, Size, false,
1952*0fca6ea1SDimitry Andric                       Exp, RTCI);
19530b57cec5SDimitry Andric   }
19540b57cec5SDimitry Andric }
19550b57cec5SDimitry Andric 
19560b57cec5SDimitry Andric void ModuleAddressSanitizer::poisonOneInitializer(Function &GlobalInit,
19570b57cec5SDimitry Andric                                                   GlobalValue *ModuleName) {
19580b57cec5SDimitry Andric   // Set up the arguments to our poison/unpoison functions.
19590b57cec5SDimitry Andric   IRBuilder<> IRB(&GlobalInit.front(),
19600b57cec5SDimitry Andric                   GlobalInit.front().getFirstInsertionPt());
19610b57cec5SDimitry Andric 
19620b57cec5SDimitry Andric   // Add a call to poison all external globals before the given function starts.
19630b57cec5SDimitry Andric   Value *ModuleNameAddr = ConstantExpr::getPointerCast(ModuleName, IntptrTy);
19640b57cec5SDimitry Andric   IRB.CreateCall(AsanPoisonGlobals, ModuleNameAddr);
19650b57cec5SDimitry Andric 
19660b57cec5SDimitry Andric   // Add calls to unpoison all globals before each return instruction.
1967bdd1243dSDimitry Andric   for (auto &BB : GlobalInit)
19680b57cec5SDimitry Andric     if (ReturnInst *RI = dyn_cast<ReturnInst>(BB.getTerminator()))
1969*0fca6ea1SDimitry Andric       CallInst::Create(AsanUnpoisonGlobals, "", RI->getIterator());
19700b57cec5SDimitry Andric }
19710b57cec5SDimitry Andric 
19720b57cec5SDimitry Andric void ModuleAddressSanitizer::createInitializerPoisonCalls(
19730b57cec5SDimitry Andric     Module &M, GlobalValue *ModuleName) {
19740b57cec5SDimitry Andric   GlobalVariable *GV = M.getGlobalVariable("llvm.global_ctors");
19750b57cec5SDimitry Andric   if (!GV)
19760b57cec5SDimitry Andric     return;
19770b57cec5SDimitry Andric 
19780b57cec5SDimitry Andric   ConstantArray *CA = dyn_cast<ConstantArray>(GV->getInitializer());
19790b57cec5SDimitry Andric   if (!CA)
19800b57cec5SDimitry Andric     return;
19810b57cec5SDimitry Andric 
19820b57cec5SDimitry Andric   for (Use &OP : CA->operands()) {
19830b57cec5SDimitry Andric     if (isa<ConstantAggregateZero>(OP)) continue;
19840b57cec5SDimitry Andric     ConstantStruct *CS = cast<ConstantStruct>(OP);
19850b57cec5SDimitry Andric 
19860b57cec5SDimitry Andric     // Must have a function or null ptr.
19870b57cec5SDimitry Andric     if (Function *F = dyn_cast<Function>(CS->getOperand(1))) {
19880b57cec5SDimitry Andric       if (F->getName() == kAsanModuleCtorName) continue;
19898bcb0991SDimitry Andric       auto *Priority = cast<ConstantInt>(CS->getOperand(0));
19900b57cec5SDimitry Andric       // Don't instrument CTORs that will run before asan.module_ctor.
19918bcb0991SDimitry Andric       if (Priority->getLimitedValue() <= GetCtorAndDtorPriority(TargetTriple))
19928bcb0991SDimitry Andric         continue;
19930b57cec5SDimitry Andric       poisonOneInitializer(*F, ModuleName);
19940b57cec5SDimitry Andric     }
19950b57cec5SDimitry Andric   }
19960b57cec5SDimitry Andric }
19970b57cec5SDimitry Andric 
1998eaeb601bSDimitry Andric const GlobalVariable *
1999eaeb601bSDimitry Andric ModuleAddressSanitizer::getExcludedAliasedGlobal(const GlobalAlias &GA) const {
20005ffd83dbSDimitry Andric   // In case this function should be expanded to include rules that do not just
20015ffd83dbSDimitry Andric   // apply when CompileKernel is true, either guard all existing rules with an
20025ffd83dbSDimitry Andric   // 'if (CompileKernel) { ... }' or be absolutely sure that all these rules
20035ffd83dbSDimitry Andric   // should also apply to user space.
20045ffd83dbSDimitry Andric   assert(CompileKernel && "Only expecting to be called when compiling kernel");
20055ffd83dbSDimitry Andric 
2006eaeb601bSDimitry Andric   const Constant *C = GA.getAliasee();
2007eaeb601bSDimitry Andric 
20085ffd83dbSDimitry Andric   // When compiling the kernel, globals that are aliased by symbols prefixed
20095ffd83dbSDimitry Andric   // by "__" are special and cannot be padded with a redzone.
20105f757f3fSDimitry Andric   if (GA.getName().starts_with("__"))
2011eaeb601bSDimitry Andric     return dyn_cast<GlobalVariable>(C->stripPointerCastsAndAliases());
20125ffd83dbSDimitry Andric 
2013eaeb601bSDimitry Andric   return nullptr;
20145ffd83dbSDimitry Andric }
20155ffd83dbSDimitry Andric 
20165ffd83dbSDimitry Andric bool ModuleAddressSanitizer::shouldInstrumentGlobal(GlobalVariable *G) const {
20170b57cec5SDimitry Andric   Type *Ty = G->getValueType();
20180b57cec5SDimitry Andric   LLVM_DEBUG(dbgs() << "GLOBAL: " << *G << "\n");
20190b57cec5SDimitry Andric 
202081ad6265SDimitry Andric   if (G->hasSanitizerMetadata() && G->getSanitizerMetadata().NoAddress)
202181ad6265SDimitry Andric     return false;
20220b57cec5SDimitry Andric   if (!Ty->isSized()) return false;
20230b57cec5SDimitry Andric   if (!G->hasInitializer()) return false;
2024fe6060f1SDimitry Andric   // Globals in address space 1 and 4 are supported for AMDGPU.
2025fe6060f1SDimitry Andric   if (G->getAddressSpace() &&
2026fe6060f1SDimitry Andric       !(TargetTriple.isAMDGPU() && !isUnsupportedAMDGPUAddrspace(G)))
2027fe6060f1SDimitry Andric     return false;
20280b57cec5SDimitry Andric   if (GlobalWasGeneratedByCompiler(G)) return false; // Our own globals.
20290b57cec5SDimitry Andric   // Two problems with thread-locals:
20300b57cec5SDimitry Andric   //   - The address of the main thread's copy can't be computed at link-time.
20310b57cec5SDimitry Andric   //   - Need to poison all copies, not just the main thread's one.
20320b57cec5SDimitry Andric   if (G->isThreadLocal()) return false;
20330b57cec5SDimitry Andric   // For now, just ignore this Global if the alignment is large.
2034bdd1243dSDimitry Andric   if (G->getAlign() && *G->getAlign() > getMinRedzoneSizeForGlobal()) return false;
20350b57cec5SDimitry Andric 
20360b57cec5SDimitry Andric   // For non-COFF targets, only instrument globals known to be defined by this
20370b57cec5SDimitry Andric   // TU.
20380b57cec5SDimitry Andric   // FIXME: We can instrument comdat globals on ELF if we are using the
20390b57cec5SDimitry Andric   // GC-friendly metadata scheme.
20400b57cec5SDimitry Andric   if (!TargetTriple.isOSBinFormatCOFF()) {
20410b57cec5SDimitry Andric     if (!G->hasExactDefinition() || G->hasComdat())
20420b57cec5SDimitry Andric       return false;
20430b57cec5SDimitry Andric   } else {
20440b57cec5SDimitry Andric     // On COFF, don't instrument non-ODR linkages.
20450b57cec5SDimitry Andric     if (G->isInterposable())
20460b57cec5SDimitry Andric       return false;
2047*0fca6ea1SDimitry Andric     // If the global has AvailableExternally linkage, then it is not in this
2048*0fca6ea1SDimitry Andric     // module, which means it does not need to be instrumented.
2049*0fca6ea1SDimitry Andric     if (G->hasAvailableExternallyLinkage())
2050*0fca6ea1SDimitry Andric       return false;
20510b57cec5SDimitry Andric   }
20520b57cec5SDimitry Andric 
20530b57cec5SDimitry Andric   // If a comdat is present, it must have a selection kind that implies ODR
20540b57cec5SDimitry Andric   // semantics: no duplicates, any, or exact match.
20550b57cec5SDimitry Andric   if (Comdat *C = G->getComdat()) {
20560b57cec5SDimitry Andric     switch (C->getSelectionKind()) {
20570b57cec5SDimitry Andric     case Comdat::Any:
20580b57cec5SDimitry Andric     case Comdat::ExactMatch:
2059fe6060f1SDimitry Andric     case Comdat::NoDeduplicate:
20600b57cec5SDimitry Andric       break;
20610b57cec5SDimitry Andric     case Comdat::Largest:
20620b57cec5SDimitry Andric     case Comdat::SameSize:
20630b57cec5SDimitry Andric       return false;
20640b57cec5SDimitry Andric     }
20650b57cec5SDimitry Andric   }
20660b57cec5SDimitry Andric 
20670b57cec5SDimitry Andric   if (G->hasSection()) {
20685ffd83dbSDimitry Andric     // The kernel uses explicit sections for mostly special global variables
20695ffd83dbSDimitry Andric     // that we should not instrument. E.g. the kernel may rely on their layout
20705ffd83dbSDimitry Andric     // without redzones, or remove them at link time ("discard.*"), etc.
20715ffd83dbSDimitry Andric     if (CompileKernel)
20725ffd83dbSDimitry Andric       return false;
20735ffd83dbSDimitry Andric 
20740b57cec5SDimitry Andric     StringRef Section = G->getSection();
20750b57cec5SDimitry Andric 
20760b57cec5SDimitry Andric     // Globals from llvm.metadata aren't emitted, do not instrument them.
20770b57cec5SDimitry Andric     if (Section == "llvm.metadata") return false;
20780b57cec5SDimitry Andric     // Do not instrument globals from special LLVM sections.
2079349cc55cSDimitry Andric     if (Section.contains("__llvm") || Section.contains("__LLVM"))
2080349cc55cSDimitry Andric       return false;
20810b57cec5SDimitry Andric 
20820b57cec5SDimitry Andric     // Do not instrument function pointers to initialization and termination
20830b57cec5SDimitry Andric     // routines: dynamic linker will not properly handle redzones.
20845f757f3fSDimitry Andric     if (Section.starts_with(".preinit_array") ||
20855f757f3fSDimitry Andric         Section.starts_with(".init_array") ||
20865f757f3fSDimitry Andric         Section.starts_with(".fini_array")) {
20870b57cec5SDimitry Andric       return false;
20880b57cec5SDimitry Andric     }
20890b57cec5SDimitry Andric 
2090e8d8bef9SDimitry Andric     // Do not instrument user-defined sections (with names resembling
2091e8d8bef9SDimitry Andric     // valid C identifiers)
2092e8d8bef9SDimitry Andric     if (TargetTriple.isOSBinFormatELF()) {
2093e8d8bef9SDimitry Andric       if (llvm::all_of(Section,
2094e8d8bef9SDimitry Andric                        [](char c) { return llvm::isAlnum(c) || c == '_'; }))
2095e8d8bef9SDimitry Andric         return false;
2096e8d8bef9SDimitry Andric     }
2097e8d8bef9SDimitry Andric 
20980b57cec5SDimitry Andric     // On COFF, if the section name contains '$', it is highly likely that the
20990b57cec5SDimitry Andric     // user is using section sorting to create an array of globals similar to
21000b57cec5SDimitry Andric     // the way initialization callbacks are registered in .init_array and
21010b57cec5SDimitry Andric     // .CRT$XCU. The ATL also registers things in .ATL$__[azm]. Adding redzones
21020b57cec5SDimitry Andric     // to such globals is counterproductive, because the intent is that they
21030b57cec5SDimitry Andric     // will form an array, and out-of-bounds accesses are expected.
21040b57cec5SDimitry Andric     // See https://github.com/google/sanitizers/issues/305
21050b57cec5SDimitry Andric     // and http://msdn.microsoft.com/en-US/en-en/library/bb918180(v=vs.120).aspx
21060b57cec5SDimitry Andric     if (TargetTriple.isOSBinFormatCOFF() && Section.contains('$')) {
21070b57cec5SDimitry Andric       LLVM_DEBUG(dbgs() << "Ignoring global in sorted section (contains '$'): "
21080b57cec5SDimitry Andric                         << *G << "\n");
21090b57cec5SDimitry Andric       return false;
21100b57cec5SDimitry Andric     }
21110b57cec5SDimitry Andric 
21120b57cec5SDimitry Andric     if (TargetTriple.isOSBinFormatMachO()) {
21130b57cec5SDimitry Andric       StringRef ParsedSegment, ParsedSection;
21140b57cec5SDimitry Andric       unsigned TAA = 0, StubSize = 0;
21150b57cec5SDimitry Andric       bool TAAParsed;
2116fe6060f1SDimitry Andric       cantFail(MCSectionMachO::ParseSectionSpecifier(
2117fe6060f1SDimitry Andric           Section, ParsedSegment, ParsedSection, TAA, TAAParsed, StubSize));
21180b57cec5SDimitry Andric 
21190b57cec5SDimitry Andric       // Ignore the globals from the __OBJC section. The ObjC runtime assumes
21200b57cec5SDimitry Andric       // those conform to /usr/lib/objc/runtime.h, so we can't add redzones to
21210b57cec5SDimitry Andric       // them.
21220b57cec5SDimitry Andric       if (ParsedSegment == "__OBJC" ||
21235f757f3fSDimitry Andric           (ParsedSegment == "__DATA" && ParsedSection.starts_with("__objc_"))) {
21240b57cec5SDimitry Andric         LLVM_DEBUG(dbgs() << "Ignoring ObjC runtime global: " << *G << "\n");
21250b57cec5SDimitry Andric         return false;
21260b57cec5SDimitry Andric       }
21270b57cec5SDimitry Andric       // See https://github.com/google/sanitizers/issues/32
21280b57cec5SDimitry Andric       // Constant CFString instances are compiled in the following way:
21290b57cec5SDimitry Andric       //  -- the string buffer is emitted into
21300b57cec5SDimitry Andric       //     __TEXT,__cstring,cstring_literals
21310b57cec5SDimitry Andric       //  -- the constant NSConstantString structure referencing that buffer
21320b57cec5SDimitry Andric       //     is placed into __DATA,__cfstring
21330b57cec5SDimitry Andric       // Therefore there's no point in placing redzones into __DATA,__cfstring.
21340b57cec5SDimitry Andric       // Moreover, it causes the linker to crash on OS X 10.7
21350b57cec5SDimitry Andric       if (ParsedSegment == "__DATA" && ParsedSection == "__cfstring") {
21360b57cec5SDimitry Andric         LLVM_DEBUG(dbgs() << "Ignoring CFString: " << *G << "\n");
21370b57cec5SDimitry Andric         return false;
21380b57cec5SDimitry Andric       }
21390b57cec5SDimitry Andric       // The linker merges the contents of cstring_literals and removes the
21400b57cec5SDimitry Andric       // trailing zeroes.
21410b57cec5SDimitry Andric       if (ParsedSegment == "__TEXT" && (TAA & MachO::S_CSTRING_LITERALS)) {
21420b57cec5SDimitry Andric         LLVM_DEBUG(dbgs() << "Ignoring a cstring literal: " << *G << "\n");
21430b57cec5SDimitry Andric         return false;
21440b57cec5SDimitry Andric       }
21450b57cec5SDimitry Andric     }
21460b57cec5SDimitry Andric   }
21470b57cec5SDimitry Andric 
21485ffd83dbSDimitry Andric   if (CompileKernel) {
21495ffd83dbSDimitry Andric     // Globals that prefixed by "__" are special and cannot be padded with a
21505ffd83dbSDimitry Andric     // redzone.
21515f757f3fSDimitry Andric     if (G->getName().starts_with("__"))
21525ffd83dbSDimitry Andric       return false;
21535ffd83dbSDimitry Andric   }
21545ffd83dbSDimitry Andric 
21550b57cec5SDimitry Andric   return true;
21560b57cec5SDimitry Andric }
21570b57cec5SDimitry Andric 
21580b57cec5SDimitry Andric // On Mach-O platforms, we emit global metadata in a separate section of the
21590b57cec5SDimitry Andric // binary in order to allow the linker to properly dead strip. This is only
21600b57cec5SDimitry Andric // supported on recent versions of ld64.
21610b57cec5SDimitry Andric bool ModuleAddressSanitizer::ShouldUseMachOGlobalsSection() const {
21620b57cec5SDimitry Andric   if (!TargetTriple.isOSBinFormatMachO())
21630b57cec5SDimitry Andric     return false;
21640b57cec5SDimitry Andric 
21650b57cec5SDimitry Andric   if (TargetTriple.isMacOSX() && !TargetTriple.isMacOSXVersionLT(10, 11))
21660b57cec5SDimitry Andric     return true;
21670b57cec5SDimitry Andric   if (TargetTriple.isiOS() /* or tvOS */ && !TargetTriple.isOSVersionLT(9))
21680b57cec5SDimitry Andric     return true;
21690b57cec5SDimitry Andric   if (TargetTriple.isWatchOS() && !TargetTriple.isOSVersionLT(2))
21700b57cec5SDimitry Andric     return true;
217181ad6265SDimitry Andric   if (TargetTriple.isDriverKit())
217281ad6265SDimitry Andric     return true;
21737a6dacacSDimitry Andric   if (TargetTriple.isXROS())
21747a6dacacSDimitry Andric     return true;
21750b57cec5SDimitry Andric 
21760b57cec5SDimitry Andric   return false;
21770b57cec5SDimitry Andric }
21780b57cec5SDimitry Andric 
21790b57cec5SDimitry Andric StringRef ModuleAddressSanitizer::getGlobalMetadataSection() const {
21800b57cec5SDimitry Andric   switch (TargetTriple.getObjectFormat()) {
21810b57cec5SDimitry Andric   case Triple::COFF:  return ".ASAN$GL";
21820b57cec5SDimitry Andric   case Triple::ELF:   return "asan_globals";
21830b57cec5SDimitry Andric   case Triple::MachO: return "__DATA,__asan_globals,regular";
21848bcb0991SDimitry Andric   case Triple::Wasm:
2185e8d8bef9SDimitry Andric   case Triple::GOFF:
218681ad6265SDimitry Andric   case Triple::SPIRV:
21878bcb0991SDimitry Andric   case Triple::XCOFF:
218881ad6265SDimitry Andric   case Triple::DXContainer:
21898bcb0991SDimitry Andric     report_fatal_error(
2190e8d8bef9SDimitry Andric         "ModuleAddressSanitizer not implemented for object file format");
21918bcb0991SDimitry Andric   case Triple::UnknownObjectFormat:
21928bcb0991SDimitry Andric     break;
21930b57cec5SDimitry Andric   }
21940b57cec5SDimitry Andric   llvm_unreachable("unsupported object format");
21950b57cec5SDimitry Andric }
21960b57cec5SDimitry Andric 
21970b57cec5SDimitry Andric void ModuleAddressSanitizer::initializeCallbacks(Module &M) {
21980b57cec5SDimitry Andric   IRBuilder<> IRB(*C);
21990b57cec5SDimitry Andric 
22000b57cec5SDimitry Andric   // Declare our poisoning and unpoisoning functions.
22010b57cec5SDimitry Andric   AsanPoisonGlobals =
22020b57cec5SDimitry Andric       M.getOrInsertFunction(kAsanPoisonGlobalsName, IRB.getVoidTy(), IntptrTy);
22030b57cec5SDimitry Andric   AsanUnpoisonGlobals =
22040b57cec5SDimitry Andric       M.getOrInsertFunction(kAsanUnpoisonGlobalsName, IRB.getVoidTy());
22050b57cec5SDimitry Andric 
22060b57cec5SDimitry Andric   // Declare functions that register/unregister globals.
22070b57cec5SDimitry Andric   AsanRegisterGlobals = M.getOrInsertFunction(
22080b57cec5SDimitry Andric       kAsanRegisterGlobalsName, IRB.getVoidTy(), IntptrTy, IntptrTy);
22090b57cec5SDimitry Andric   AsanUnregisterGlobals = M.getOrInsertFunction(
22100b57cec5SDimitry Andric       kAsanUnregisterGlobalsName, IRB.getVoidTy(), IntptrTy, IntptrTy);
22110b57cec5SDimitry Andric 
22120b57cec5SDimitry Andric   // Declare the functions that find globals in a shared object and then invoke
22130b57cec5SDimitry Andric   // the (un)register function on them.
22140b57cec5SDimitry Andric   AsanRegisterImageGlobals = M.getOrInsertFunction(
22150b57cec5SDimitry Andric       kAsanRegisterImageGlobalsName, IRB.getVoidTy(), IntptrTy);
22160b57cec5SDimitry Andric   AsanUnregisterImageGlobals = M.getOrInsertFunction(
22170b57cec5SDimitry Andric       kAsanUnregisterImageGlobalsName, IRB.getVoidTy(), IntptrTy);
22180b57cec5SDimitry Andric 
22190b57cec5SDimitry Andric   AsanRegisterElfGlobals =
22200b57cec5SDimitry Andric       M.getOrInsertFunction(kAsanRegisterElfGlobalsName, IRB.getVoidTy(),
22210b57cec5SDimitry Andric                             IntptrTy, IntptrTy, IntptrTy);
22220b57cec5SDimitry Andric   AsanUnregisterElfGlobals =
22230b57cec5SDimitry Andric       M.getOrInsertFunction(kAsanUnregisterElfGlobalsName, IRB.getVoidTy(),
22240b57cec5SDimitry Andric                             IntptrTy, IntptrTy, IntptrTy);
22250b57cec5SDimitry Andric }
22260b57cec5SDimitry Andric 
22270b57cec5SDimitry Andric // Put the metadata and the instrumented global in the same group. This ensures
22280b57cec5SDimitry Andric // that the metadata is discarded if the instrumented global is discarded.
22290b57cec5SDimitry Andric void ModuleAddressSanitizer::SetComdatForGlobalMetadata(
22300b57cec5SDimitry Andric     GlobalVariable *G, GlobalVariable *Metadata, StringRef InternalSuffix) {
22310b57cec5SDimitry Andric   Module &M = *G->getParent();
22320b57cec5SDimitry Andric   Comdat *C = G->getComdat();
22330b57cec5SDimitry Andric   if (!C) {
22340b57cec5SDimitry Andric     if (!G->hasName()) {
22350b57cec5SDimitry Andric       // If G is unnamed, it must be internal. Give it an artificial name
22360b57cec5SDimitry Andric       // so we can put it in a comdat.
22370b57cec5SDimitry Andric       assert(G->hasLocalLinkage());
22380b57cec5SDimitry Andric       G->setName(Twine(kAsanGenPrefix) + "_anon_global");
22390b57cec5SDimitry Andric     }
22400b57cec5SDimitry Andric 
22410b57cec5SDimitry Andric     if (!InternalSuffix.empty() && G->hasLocalLinkage()) {
22425ffd83dbSDimitry Andric       std::string Name = std::string(G->getName());
22430b57cec5SDimitry Andric       Name += InternalSuffix;
22440b57cec5SDimitry Andric       C = M.getOrInsertComdat(Name);
22450b57cec5SDimitry Andric     } else {
22460b57cec5SDimitry Andric       C = M.getOrInsertComdat(G->getName());
22470b57cec5SDimitry Andric     }
22480b57cec5SDimitry Andric 
22490b57cec5SDimitry Andric     // Make this IMAGE_COMDAT_SELECT_NODUPLICATES on COFF. Also upgrade private
22500b57cec5SDimitry Andric     // linkage to internal linkage so that a symbol table entry is emitted. This
22510b57cec5SDimitry Andric     // is necessary in order to create the comdat group.
22520b57cec5SDimitry Andric     if (TargetTriple.isOSBinFormatCOFF()) {
2253fe6060f1SDimitry Andric       C->setSelectionKind(Comdat::NoDeduplicate);
22540b57cec5SDimitry Andric       if (G->hasPrivateLinkage())
22550b57cec5SDimitry Andric         G->setLinkage(GlobalValue::InternalLinkage);
22560b57cec5SDimitry Andric     }
22570b57cec5SDimitry Andric     G->setComdat(C);
22580b57cec5SDimitry Andric   }
22590b57cec5SDimitry Andric 
22600b57cec5SDimitry Andric   assert(G->hasComdat());
22610b57cec5SDimitry Andric   Metadata->setComdat(G->getComdat());
22620b57cec5SDimitry Andric }
22630b57cec5SDimitry Andric 
22640b57cec5SDimitry Andric // Create a separate metadata global and put it in the appropriate ASan
22650b57cec5SDimitry Andric // global registration section.
22660b57cec5SDimitry Andric GlobalVariable *
22670b57cec5SDimitry Andric ModuleAddressSanitizer::CreateMetadataGlobal(Module &M, Constant *Initializer,
22680b57cec5SDimitry Andric                                              StringRef OriginalName) {
22690b57cec5SDimitry Andric   auto Linkage = TargetTriple.isOSBinFormatMachO()
22700b57cec5SDimitry Andric                      ? GlobalVariable::InternalLinkage
22710b57cec5SDimitry Andric                      : GlobalVariable::PrivateLinkage;
22720b57cec5SDimitry Andric   GlobalVariable *Metadata = new GlobalVariable(
22730b57cec5SDimitry Andric       M, Initializer->getType(), false, Linkage, Initializer,
22740b57cec5SDimitry Andric       Twine("__asan_global_") + GlobalValue::dropLLVMManglingEscape(OriginalName));
22750b57cec5SDimitry Andric   Metadata->setSection(getGlobalMetadataSection());
22765f757f3fSDimitry Andric   // Place metadata in a large section for x86-64 ELF binaries to mitigate
22775f757f3fSDimitry Andric   // relocation pressure.
22785f757f3fSDimitry Andric   setGlobalVariableLargeSection(TargetTriple, *Metadata);
22790b57cec5SDimitry Andric   return Metadata;
22800b57cec5SDimitry Andric }
22810b57cec5SDimitry Andric 
22825ffd83dbSDimitry Andric Instruction *ModuleAddressSanitizer::CreateAsanModuleDtor(Module &M) {
2283fe6060f1SDimitry Andric   AsanDtorFunction = Function::createWithDefaultAttr(
2284fe6060f1SDimitry Andric       FunctionType::get(Type::getVoidTy(*C), false),
2285fe6060f1SDimitry Andric       GlobalValue::InternalLinkage, 0, kAsanModuleDtorName, &M);
2286349cc55cSDimitry Andric   AsanDtorFunction->addFnAttr(Attribute::NoUnwind);
2287fe6060f1SDimitry Andric   // Ensure Dtor cannot be discarded, even if in a comdat.
2288fe6060f1SDimitry Andric   appendToUsed(M, {AsanDtorFunction});
22890b57cec5SDimitry Andric   BasicBlock *AsanDtorBB = BasicBlock::Create(*C, "", AsanDtorFunction);
22900b57cec5SDimitry Andric 
22915ffd83dbSDimitry Andric   return ReturnInst::Create(*C, AsanDtorBB);
22920b57cec5SDimitry Andric }
22930b57cec5SDimitry Andric 
22940b57cec5SDimitry Andric void ModuleAddressSanitizer::InstrumentGlobalsCOFF(
22950b57cec5SDimitry Andric     IRBuilder<> &IRB, Module &M, ArrayRef<GlobalVariable *> ExtendedGlobals,
22960b57cec5SDimitry Andric     ArrayRef<Constant *> MetadataInitializers) {
22970b57cec5SDimitry Andric   assert(ExtendedGlobals.size() == MetadataInitializers.size());
22980b57cec5SDimitry Andric   auto &DL = M.getDataLayout();
22990b57cec5SDimitry Andric 
23005ffd83dbSDimitry Andric   SmallVector<GlobalValue *, 16> MetadataGlobals(ExtendedGlobals.size());
23010b57cec5SDimitry Andric   for (size_t i = 0; i < ExtendedGlobals.size(); i++) {
23020b57cec5SDimitry Andric     Constant *Initializer = MetadataInitializers[i];
23030b57cec5SDimitry Andric     GlobalVariable *G = ExtendedGlobals[i];
23040b57cec5SDimitry Andric     GlobalVariable *Metadata =
23050b57cec5SDimitry Andric         CreateMetadataGlobal(M, Initializer, G->getName());
23065ffd83dbSDimitry Andric     MDNode *MD = MDNode::get(M.getContext(), ValueAsMetadata::get(G));
23075ffd83dbSDimitry Andric     Metadata->setMetadata(LLVMContext::MD_associated, MD);
23085ffd83dbSDimitry Andric     MetadataGlobals[i] = Metadata;
23090b57cec5SDimitry Andric 
23100b57cec5SDimitry Andric     // The MSVC linker always inserts padding when linking incrementally. We
23110b57cec5SDimitry Andric     // cope with that by aligning each struct to its size, which must be a power
23120b57cec5SDimitry Andric     // of two.
23130b57cec5SDimitry Andric     unsigned SizeOfGlobalStruct = DL.getTypeAllocSize(Initializer->getType());
23140b57cec5SDimitry Andric     assert(isPowerOf2_32(SizeOfGlobalStruct) &&
23150b57cec5SDimitry Andric            "global metadata will not be padded appropriately");
23168bcb0991SDimitry Andric     Metadata->setAlignment(assumeAligned(SizeOfGlobalStruct));
23170b57cec5SDimitry Andric 
23180b57cec5SDimitry Andric     SetComdatForGlobalMetadata(G, Metadata, "");
23190b57cec5SDimitry Andric   }
23205ffd83dbSDimitry Andric 
23215ffd83dbSDimitry Andric   // Update llvm.compiler.used, adding the new metadata globals. This is
23225ffd83dbSDimitry Andric   // needed so that during LTO these variables stay alive.
23235ffd83dbSDimitry Andric   if (!MetadataGlobals.empty())
23245ffd83dbSDimitry Andric     appendToCompilerUsed(M, MetadataGlobals);
23250b57cec5SDimitry Andric }
23260b57cec5SDimitry Andric 
2327bdb86d1aSDimitry Andric void ModuleAddressSanitizer::instrumentGlobalsELF(
23280b57cec5SDimitry Andric     IRBuilder<> &IRB, Module &M, ArrayRef<GlobalVariable *> ExtendedGlobals,
23290b57cec5SDimitry Andric     ArrayRef<Constant *> MetadataInitializers,
23300b57cec5SDimitry Andric     const std::string &UniqueModuleId) {
23310b57cec5SDimitry Andric   assert(ExtendedGlobals.size() == MetadataInitializers.size());
23320b57cec5SDimitry Andric 
2333fe6060f1SDimitry Andric   // Putting globals in a comdat changes the semantic and potentially cause
2334fe6060f1SDimitry Andric   // false negative odr violations at link time. If odr indicators are used, we
2335fe6060f1SDimitry Andric   // keep the comdat sections, as link time odr violations will be dectected on
2336fe6060f1SDimitry Andric   // the odr indicator symbols.
2337bdb86d1aSDimitry Andric   bool UseComdatForGlobalsGC = UseOdrIndicator && !UniqueModuleId.empty();
2338fe6060f1SDimitry Andric 
23390b57cec5SDimitry Andric   SmallVector<GlobalValue *, 16> MetadataGlobals(ExtendedGlobals.size());
23400b57cec5SDimitry Andric   for (size_t i = 0; i < ExtendedGlobals.size(); i++) {
23410b57cec5SDimitry Andric     GlobalVariable *G = ExtendedGlobals[i];
23420b57cec5SDimitry Andric     GlobalVariable *Metadata =
23430b57cec5SDimitry Andric         CreateMetadataGlobal(M, MetadataInitializers[i], G->getName());
23440b57cec5SDimitry Andric     MDNode *MD = MDNode::get(M.getContext(), ValueAsMetadata::get(G));
23450b57cec5SDimitry Andric     Metadata->setMetadata(LLVMContext::MD_associated, MD);
23460b57cec5SDimitry Andric     MetadataGlobals[i] = Metadata;
23470b57cec5SDimitry Andric 
2348fe6060f1SDimitry Andric     if (UseComdatForGlobalsGC)
23490b57cec5SDimitry Andric       SetComdatForGlobalMetadata(G, Metadata, UniqueModuleId);
23500b57cec5SDimitry Andric   }
23510b57cec5SDimitry Andric 
23520b57cec5SDimitry Andric   // Update llvm.compiler.used, adding the new metadata globals. This is
23530b57cec5SDimitry Andric   // needed so that during LTO these variables stay alive.
2354e8d8bef9SDimitry Andric   if (!MetadataGlobals.empty())
23550b57cec5SDimitry Andric     appendToCompilerUsed(M, MetadataGlobals);
23560b57cec5SDimitry Andric 
23570b57cec5SDimitry Andric   // RegisteredFlag serves two purposes. First, we can pass it to dladdr()
23580b57cec5SDimitry Andric   // to look up the loaded image that contains it. Second, we can store in it
23590b57cec5SDimitry Andric   // whether registration has already occurred, to prevent duplicate
23600b57cec5SDimitry Andric   // registration.
23610b57cec5SDimitry Andric   //
23620b57cec5SDimitry Andric   // Common linkage ensures that there is only one global per shared library.
23630b57cec5SDimitry Andric   GlobalVariable *RegisteredFlag = new GlobalVariable(
23640b57cec5SDimitry Andric       M, IntptrTy, false, GlobalVariable::CommonLinkage,
23650b57cec5SDimitry Andric       ConstantInt::get(IntptrTy, 0), kAsanGlobalsRegisteredFlagName);
23660b57cec5SDimitry Andric   RegisteredFlag->setVisibility(GlobalVariable::HiddenVisibility);
23670b57cec5SDimitry Andric 
2368e8d8bef9SDimitry Andric   // Create start and stop symbols.
2369e8d8bef9SDimitry Andric   GlobalVariable *StartELFMetadata = new GlobalVariable(
2370e8d8bef9SDimitry Andric       M, IntptrTy, false, GlobalVariable::ExternalWeakLinkage, nullptr,
2371e8d8bef9SDimitry Andric       "__start_" + getGlobalMetadataSection());
2372e8d8bef9SDimitry Andric   StartELFMetadata->setVisibility(GlobalVariable::HiddenVisibility);
2373e8d8bef9SDimitry Andric   GlobalVariable *StopELFMetadata = new GlobalVariable(
2374e8d8bef9SDimitry Andric       M, IntptrTy, false, GlobalVariable::ExternalWeakLinkage, nullptr,
2375e8d8bef9SDimitry Andric       "__stop_" + getGlobalMetadataSection());
2376e8d8bef9SDimitry Andric   StopELFMetadata->setVisibility(GlobalVariable::HiddenVisibility);
23770b57cec5SDimitry Andric 
23780b57cec5SDimitry Andric   // Create a call to register the globals with the runtime.
2379bdd1243dSDimitry Andric   if (ConstructorKind == AsanCtorKind::Global)
23800b57cec5SDimitry Andric     IRB.CreateCall(AsanRegisterElfGlobals,
23810b57cec5SDimitry Andric                  {IRB.CreatePointerCast(RegisteredFlag, IntptrTy),
23820b57cec5SDimitry Andric                   IRB.CreatePointerCast(StartELFMetadata, IntptrTy),
23830b57cec5SDimitry Andric                   IRB.CreatePointerCast(StopELFMetadata, IntptrTy)});
23840b57cec5SDimitry Andric 
23850b57cec5SDimitry Andric   // We also need to unregister globals at the end, e.g., when a shared library
23860b57cec5SDimitry Andric   // gets closed.
2387bdb86d1aSDimitry Andric   if (DestructorKind != AsanDtorKind::None && !MetadataGlobals.empty()) {
2388fe6060f1SDimitry Andric     IRBuilder<> IrbDtor(CreateAsanModuleDtor(M));
2389fe6060f1SDimitry Andric     IrbDtor.CreateCall(AsanUnregisterElfGlobals,
23900b57cec5SDimitry Andric                        {IRB.CreatePointerCast(RegisteredFlag, IntptrTy),
23910b57cec5SDimitry Andric                         IRB.CreatePointerCast(StartELFMetadata, IntptrTy),
23920b57cec5SDimitry Andric                         IRB.CreatePointerCast(StopELFMetadata, IntptrTy)});
23930b57cec5SDimitry Andric   }
2394fe6060f1SDimitry Andric }
23950b57cec5SDimitry Andric 
23960b57cec5SDimitry Andric void ModuleAddressSanitizer::InstrumentGlobalsMachO(
23970b57cec5SDimitry Andric     IRBuilder<> &IRB, Module &M, ArrayRef<GlobalVariable *> ExtendedGlobals,
23980b57cec5SDimitry Andric     ArrayRef<Constant *> MetadataInitializers) {
23990b57cec5SDimitry Andric   assert(ExtendedGlobals.size() == MetadataInitializers.size());
24000b57cec5SDimitry Andric 
24010b57cec5SDimitry Andric   // On recent Mach-O platforms, use a structure which binds the liveness of
24020b57cec5SDimitry Andric   // the global variable to the metadata struct. Keep the list of "Liveness" GV
24030b57cec5SDimitry Andric   // created to be added to llvm.compiler.used
24040b57cec5SDimitry Andric   StructType *LivenessTy = StructType::get(IntptrTy, IntptrTy);
24050b57cec5SDimitry Andric   SmallVector<GlobalValue *, 16> LivenessGlobals(ExtendedGlobals.size());
24060b57cec5SDimitry Andric 
24070b57cec5SDimitry Andric   for (size_t i = 0; i < ExtendedGlobals.size(); i++) {
24080b57cec5SDimitry Andric     Constant *Initializer = MetadataInitializers[i];
24090b57cec5SDimitry Andric     GlobalVariable *G = ExtendedGlobals[i];
24100b57cec5SDimitry Andric     GlobalVariable *Metadata =
24110b57cec5SDimitry Andric         CreateMetadataGlobal(M, Initializer, G->getName());
24120b57cec5SDimitry Andric 
24130b57cec5SDimitry Andric     // On recent Mach-O platforms, we emit the global metadata in a way that
24140b57cec5SDimitry Andric     // allows the linker to properly strip dead globals.
24150b57cec5SDimitry Andric     auto LivenessBinder =
24160b57cec5SDimitry Andric         ConstantStruct::get(LivenessTy, Initializer->getAggregateElement(0u),
24170b57cec5SDimitry Andric                             ConstantExpr::getPointerCast(Metadata, IntptrTy));
24180b57cec5SDimitry Andric     GlobalVariable *Liveness = new GlobalVariable(
24190b57cec5SDimitry Andric         M, LivenessTy, false, GlobalVariable::InternalLinkage, LivenessBinder,
24200b57cec5SDimitry Andric         Twine("__asan_binder_") + G->getName());
24210b57cec5SDimitry Andric     Liveness->setSection("__DATA,__asan_liveness,regular,live_support");
24220b57cec5SDimitry Andric     LivenessGlobals[i] = Liveness;
24230b57cec5SDimitry Andric   }
24240b57cec5SDimitry Andric 
24250b57cec5SDimitry Andric   // Update llvm.compiler.used, adding the new liveness globals. This is
24260b57cec5SDimitry Andric   // needed so that during LTO these variables stay alive. The alternative
24270b57cec5SDimitry Andric   // would be to have the linker handling the LTO symbols, but libLTO
24280b57cec5SDimitry Andric   // current API does not expose access to the section for each symbol.
24290b57cec5SDimitry Andric   if (!LivenessGlobals.empty())
24300b57cec5SDimitry Andric     appendToCompilerUsed(M, LivenessGlobals);
24310b57cec5SDimitry Andric 
24320b57cec5SDimitry Andric   // RegisteredFlag serves two purposes. First, we can pass it to dladdr()
24330b57cec5SDimitry Andric   // to look up the loaded image that contains it. Second, we can store in it
24340b57cec5SDimitry Andric   // whether registration has already occurred, to prevent duplicate
24350b57cec5SDimitry Andric   // registration.
24360b57cec5SDimitry Andric   //
24370b57cec5SDimitry Andric   // common linkage ensures that there is only one global per shared library.
24380b57cec5SDimitry Andric   GlobalVariable *RegisteredFlag = new GlobalVariable(
24390b57cec5SDimitry Andric       M, IntptrTy, false, GlobalVariable::CommonLinkage,
24400b57cec5SDimitry Andric       ConstantInt::get(IntptrTy, 0), kAsanGlobalsRegisteredFlagName);
24410b57cec5SDimitry Andric   RegisteredFlag->setVisibility(GlobalVariable::HiddenVisibility);
24420b57cec5SDimitry Andric 
2443bdd1243dSDimitry Andric   if (ConstructorKind == AsanCtorKind::Global)
24440b57cec5SDimitry Andric     IRB.CreateCall(AsanRegisterImageGlobals,
24450b57cec5SDimitry Andric                  {IRB.CreatePointerCast(RegisteredFlag, IntptrTy)});
24460b57cec5SDimitry Andric 
24470b57cec5SDimitry Andric   // We also need to unregister globals at the end, e.g., when a shared library
24480b57cec5SDimitry Andric   // gets closed.
2449fe6060f1SDimitry Andric   if (DestructorKind != AsanDtorKind::None) {
2450fe6060f1SDimitry Andric     IRBuilder<> IrbDtor(CreateAsanModuleDtor(M));
2451fe6060f1SDimitry Andric     IrbDtor.CreateCall(AsanUnregisterImageGlobals,
24520b57cec5SDimitry Andric                        {IRB.CreatePointerCast(RegisteredFlag, IntptrTy)});
24530b57cec5SDimitry Andric   }
2454fe6060f1SDimitry Andric }
24550b57cec5SDimitry Andric 
24560b57cec5SDimitry Andric void ModuleAddressSanitizer::InstrumentGlobalsWithMetadataArray(
24570b57cec5SDimitry Andric     IRBuilder<> &IRB, Module &M, ArrayRef<GlobalVariable *> ExtendedGlobals,
24580b57cec5SDimitry Andric     ArrayRef<Constant *> MetadataInitializers) {
24590b57cec5SDimitry Andric   assert(ExtendedGlobals.size() == MetadataInitializers.size());
24600b57cec5SDimitry Andric   unsigned N = ExtendedGlobals.size();
24610b57cec5SDimitry Andric   assert(N > 0);
24620b57cec5SDimitry Andric 
24630b57cec5SDimitry Andric   // On platforms that don't have a custom metadata section, we emit an array
24640b57cec5SDimitry Andric   // of global metadata structures.
24650b57cec5SDimitry Andric   ArrayType *ArrayOfGlobalStructTy =
24660b57cec5SDimitry Andric       ArrayType::get(MetadataInitializers[0]->getType(), N);
24670b57cec5SDimitry Andric   auto AllGlobals = new GlobalVariable(
24680b57cec5SDimitry Andric       M, ArrayOfGlobalStructTy, false, GlobalVariable::InternalLinkage,
24690b57cec5SDimitry Andric       ConstantArray::get(ArrayOfGlobalStructTy, MetadataInitializers), "");
24700b57cec5SDimitry Andric   if (Mapping.Scale > 3)
24718bcb0991SDimitry Andric     AllGlobals->setAlignment(Align(1ULL << Mapping.Scale));
24720b57cec5SDimitry Andric 
2473bdd1243dSDimitry Andric   if (ConstructorKind == AsanCtorKind::Global)
24740b57cec5SDimitry Andric     IRB.CreateCall(AsanRegisterGlobals,
24750b57cec5SDimitry Andric                  {IRB.CreatePointerCast(AllGlobals, IntptrTy),
24760b57cec5SDimitry Andric                   ConstantInt::get(IntptrTy, N)});
24770b57cec5SDimitry Andric 
24780b57cec5SDimitry Andric   // We also need to unregister globals at the end, e.g., when a shared library
24790b57cec5SDimitry Andric   // gets closed.
2480fe6060f1SDimitry Andric   if (DestructorKind != AsanDtorKind::None) {
2481fe6060f1SDimitry Andric     IRBuilder<> IrbDtor(CreateAsanModuleDtor(M));
2482fe6060f1SDimitry Andric     IrbDtor.CreateCall(AsanUnregisterGlobals,
24830b57cec5SDimitry Andric                        {IRB.CreatePointerCast(AllGlobals, IntptrTy),
24840b57cec5SDimitry Andric                         ConstantInt::get(IntptrTy, N)});
24850b57cec5SDimitry Andric   }
2486fe6060f1SDimitry Andric }
24870b57cec5SDimitry Andric 
24880b57cec5SDimitry Andric // This function replaces all global variables with new variables that have
24890b57cec5SDimitry Andric // trailing redzones. It also creates a function that poisons
24900b57cec5SDimitry Andric // redzones and inserts this function into llvm.global_ctors.
24910b57cec5SDimitry Andric // Sets *CtorComdat to true if the global registration code emitted into the
24920b57cec5SDimitry Andric // asan constructor is comdat-compatible.
2493bdb86d1aSDimitry Andric void ModuleAddressSanitizer::instrumentGlobals(IRBuilder<> &IRB, Module &M,
24940b57cec5SDimitry Andric                                                bool *CtorComdat) {
24955ffd83dbSDimitry Andric   // Build set of globals that are aliased by some GA, where
2496eaeb601bSDimitry Andric   // getExcludedAliasedGlobal(GA) returns the relevant GlobalVariable.
24975ffd83dbSDimitry Andric   SmallPtrSet<const GlobalVariable *, 16> AliasedGlobalExclusions;
24985ffd83dbSDimitry Andric   if (CompileKernel) {
24995ffd83dbSDimitry Andric     for (auto &GA : M.aliases()) {
2500eaeb601bSDimitry Andric       if (const GlobalVariable *GV = getExcludedAliasedGlobal(GA))
25015ffd83dbSDimitry Andric         AliasedGlobalExclusions.insert(GV);
25025ffd83dbSDimitry Andric     }
25035ffd83dbSDimitry Andric   }
25040b57cec5SDimitry Andric 
25055ffd83dbSDimitry Andric   SmallVector<GlobalVariable *, 16> GlobalsToChange;
25060b57cec5SDimitry Andric   for (auto &G : M.globals()) {
25075ffd83dbSDimitry Andric     if (!AliasedGlobalExclusions.count(&G) && shouldInstrumentGlobal(&G))
25085ffd83dbSDimitry Andric       GlobalsToChange.push_back(&G);
25090b57cec5SDimitry Andric   }
25100b57cec5SDimitry Andric 
25110b57cec5SDimitry Andric   size_t n = GlobalsToChange.size();
25120b57cec5SDimitry Andric   auto &DL = M.getDataLayout();
25130b57cec5SDimitry Andric 
25140b57cec5SDimitry Andric   // A global is described by a structure
25150b57cec5SDimitry Andric   //   size_t beg;
25160b57cec5SDimitry Andric   //   size_t size;
25170b57cec5SDimitry Andric   //   size_t size_with_redzone;
25180b57cec5SDimitry Andric   //   const char *name;
25190b57cec5SDimitry Andric   //   const char *module_name;
25200b57cec5SDimitry Andric   //   size_t has_dynamic_init;
252181ad6265SDimitry Andric   //   size_t padding_for_windows_msvc_incremental_link;
25220b57cec5SDimitry Andric   //   size_t odr_indicator;
25230b57cec5SDimitry Andric   // We initialize an array of such structures and pass it to a run-time call.
25240b57cec5SDimitry Andric   StructType *GlobalStructTy =
25250b57cec5SDimitry Andric       StructType::get(IntptrTy, IntptrTy, IntptrTy, IntptrTy, IntptrTy,
25260b57cec5SDimitry Andric                       IntptrTy, IntptrTy, IntptrTy);
25270b57cec5SDimitry Andric   SmallVector<GlobalVariable *, 16> NewGlobals(n);
25280b57cec5SDimitry Andric   SmallVector<Constant *, 16> Initializers(n);
25290b57cec5SDimitry Andric 
25300b57cec5SDimitry Andric   bool HasDynamicallyInitializedGlobals = false;
25310b57cec5SDimitry Andric 
25320b57cec5SDimitry Andric   // We shouldn't merge same module names, as this string serves as unique
25330b57cec5SDimitry Andric   // module ID in runtime.
2534bdb86d1aSDimitry Andric   GlobalVariable *ModuleName =
2535bdb86d1aSDimitry Andric       n != 0
2536bdb86d1aSDimitry Andric           ? createPrivateGlobalForString(M, M.getModuleIdentifier(),
2537bdb86d1aSDimitry Andric                                          /*AllowMerging*/ false, kAsanGenPrefix)
2538bdb86d1aSDimitry Andric           : nullptr;
25390b57cec5SDimitry Andric 
25400b57cec5SDimitry Andric   for (size_t i = 0; i < n; i++) {
25410b57cec5SDimitry Andric     GlobalVariable *G = GlobalsToChange[i];
25420b57cec5SDimitry Andric 
254381ad6265SDimitry Andric     GlobalValue::SanitizerMetadata MD;
254481ad6265SDimitry Andric     if (G->hasSanitizerMetadata())
254581ad6265SDimitry Andric       MD = G->getSanitizerMetadata();
254681ad6265SDimitry Andric 
2547bdd1243dSDimitry Andric     // The runtime library tries demangling symbol names in the descriptor but
2548bdd1243dSDimitry Andric     // functionality like __cxa_demangle may be unavailable (e.g.
2549bdd1243dSDimitry Andric     // -static-libstdc++). So we demangle the symbol names here.
2550bdd1243dSDimitry Andric     std::string NameForGlobal = G->getName().str();
255181ad6265SDimitry Andric     GlobalVariable *Name =
2552bdd1243dSDimitry Andric         createPrivateGlobalForString(M, llvm::demangle(NameForGlobal),
25530b57cec5SDimitry Andric                                      /*AllowMerging*/ true, kAsanGenPrefix);
25540b57cec5SDimitry Andric 
25550b57cec5SDimitry Andric     Type *Ty = G->getValueType();
25565ffd83dbSDimitry Andric     const uint64_t SizeInBytes = DL.getTypeAllocSize(Ty);
25575ffd83dbSDimitry Andric     const uint64_t RightRedzoneSize = getRedzoneSizeForGlobal(SizeInBytes);
25580b57cec5SDimitry Andric     Type *RightRedZoneTy = ArrayType::get(IRB.getInt8Ty(), RightRedzoneSize);
25590b57cec5SDimitry Andric 
25600b57cec5SDimitry Andric     StructType *NewTy = StructType::get(Ty, RightRedZoneTy);
25610b57cec5SDimitry Andric     Constant *NewInitializer = ConstantStruct::get(
25620b57cec5SDimitry Andric         NewTy, G->getInitializer(), Constant::getNullValue(RightRedZoneTy));
25630b57cec5SDimitry Andric 
25640b57cec5SDimitry Andric     // Create a new global variable with enough space for a redzone.
25650b57cec5SDimitry Andric     GlobalValue::LinkageTypes Linkage = G->getLinkage();
25660b57cec5SDimitry Andric     if (G->isConstant() && Linkage == GlobalValue::PrivateLinkage)
25670b57cec5SDimitry Andric       Linkage = GlobalValue::InternalLinkage;
2568fe6060f1SDimitry Andric     GlobalVariable *NewGlobal = new GlobalVariable(
2569fe6060f1SDimitry Andric         M, NewTy, G->isConstant(), Linkage, NewInitializer, "", G,
2570fe6060f1SDimitry Andric         G->getThreadLocalMode(), G->getAddressSpace());
25710b57cec5SDimitry Andric     NewGlobal->copyAttributesFrom(G);
25720b57cec5SDimitry Andric     NewGlobal->setComdat(G->getComdat());
257306c3fb27SDimitry Andric     NewGlobal->setAlignment(Align(getMinRedzoneSizeForGlobal()));
25740b57cec5SDimitry Andric     // Don't fold globals with redzones. ODR violation detector and redzone
25750b57cec5SDimitry Andric     // poisoning implicitly creates a dependence on the global's address, so it
25760b57cec5SDimitry Andric     // is no longer valid for it to be marked unnamed_addr.
25770b57cec5SDimitry Andric     NewGlobal->setUnnamedAddr(GlobalValue::UnnamedAddr::None);
25780b57cec5SDimitry Andric 
25790b57cec5SDimitry Andric     // Move null-terminated C strings to "__asan_cstring" section on Darwin.
25800b57cec5SDimitry Andric     if (TargetTriple.isOSBinFormatMachO() && !G->hasSection() &&
25810b57cec5SDimitry Andric         G->isConstant()) {
25820b57cec5SDimitry Andric       auto Seq = dyn_cast<ConstantDataSequential>(G->getInitializer());
25830b57cec5SDimitry Andric       if (Seq && Seq->isCString())
25840b57cec5SDimitry Andric         NewGlobal->setSection("__TEXT,__asan_cstring,regular");
25850b57cec5SDimitry Andric     }
25860b57cec5SDimitry Andric 
2587e8d8bef9SDimitry Andric     // Transfer the debug info and type metadata.  The payload starts at offset
2588e8d8bef9SDimitry Andric     // zero so we can copy the metadata over as is.
2589e8d8bef9SDimitry Andric     NewGlobal->copyMetadata(G, 0);
25900b57cec5SDimitry Andric 
25910b57cec5SDimitry Andric     Value *Indices2[2];
25920b57cec5SDimitry Andric     Indices2[0] = IRB.getInt32(0);
25930b57cec5SDimitry Andric     Indices2[1] = IRB.getInt32(0);
25940b57cec5SDimitry Andric 
25950b57cec5SDimitry Andric     G->replaceAllUsesWith(
25960b57cec5SDimitry Andric         ConstantExpr::getGetElementPtr(NewTy, NewGlobal, Indices2, true));
25970b57cec5SDimitry Andric     NewGlobal->takeName(G);
25980b57cec5SDimitry Andric     G->eraseFromParent();
25990b57cec5SDimitry Andric     NewGlobals[i] = NewGlobal;
26000b57cec5SDimitry Andric 
26015f757f3fSDimitry Andric     Constant *ODRIndicator = ConstantPointerNull::get(PtrTy);
26020b57cec5SDimitry Andric     GlobalValue *InstrumentedGlobal = NewGlobal;
26030b57cec5SDimitry Andric 
26040b57cec5SDimitry Andric     bool CanUsePrivateAliases =
26050b57cec5SDimitry Andric         TargetTriple.isOSBinFormatELF() || TargetTriple.isOSBinFormatMachO() ||
26060b57cec5SDimitry Andric         TargetTriple.isOSBinFormatWasm();
26070b57cec5SDimitry Andric     if (CanUsePrivateAliases && UsePrivateAlias) {
26080b57cec5SDimitry Andric       // Create local alias for NewGlobal to avoid crash on ODR between
26090b57cec5SDimitry Andric       // instrumented and non-instrumented libraries.
26100b57cec5SDimitry Andric       InstrumentedGlobal =
26110b57cec5SDimitry Andric           GlobalAlias::create(GlobalValue::PrivateLinkage, "", NewGlobal);
26120b57cec5SDimitry Andric     }
26130b57cec5SDimitry Andric 
26140b57cec5SDimitry Andric     // ODR should not happen for local linkage.
26150b57cec5SDimitry Andric     if (NewGlobal->hasLocalLinkage()) {
26165f757f3fSDimitry Andric       ODRIndicator =
26175f757f3fSDimitry Andric           ConstantExpr::getIntToPtr(ConstantInt::get(IntptrTy, -1), PtrTy);
26180b57cec5SDimitry Andric     } else if (UseOdrIndicator) {
26190b57cec5SDimitry Andric       // With local aliases, we need to provide another externally visible
26200b57cec5SDimitry Andric       // symbol __odr_asan_XXX to detect ODR violation.
26210b57cec5SDimitry Andric       auto *ODRIndicatorSym =
26220b57cec5SDimitry Andric           new GlobalVariable(M, IRB.getInt8Ty(), false, Linkage,
26230b57cec5SDimitry Andric                              Constant::getNullValue(IRB.getInt8Ty()),
26240b57cec5SDimitry Andric                              kODRGenPrefix + NameForGlobal, nullptr,
26250b57cec5SDimitry Andric                              NewGlobal->getThreadLocalMode());
26260b57cec5SDimitry Andric 
26270b57cec5SDimitry Andric       // Set meaningful attributes for indicator symbol.
26280b57cec5SDimitry Andric       ODRIndicatorSym->setVisibility(NewGlobal->getVisibility());
26290b57cec5SDimitry Andric       ODRIndicatorSym->setDLLStorageClass(NewGlobal->getDLLStorageClass());
26305ffd83dbSDimitry Andric       ODRIndicatorSym->setAlignment(Align(1));
26310b57cec5SDimitry Andric       ODRIndicator = ODRIndicatorSym;
26320b57cec5SDimitry Andric     }
26330b57cec5SDimitry Andric 
26340b57cec5SDimitry Andric     Constant *Initializer = ConstantStruct::get(
26350b57cec5SDimitry Andric         GlobalStructTy,
26360b57cec5SDimitry Andric         ConstantExpr::getPointerCast(InstrumentedGlobal, IntptrTy),
26370b57cec5SDimitry Andric         ConstantInt::get(IntptrTy, SizeInBytes),
26380b57cec5SDimitry Andric         ConstantInt::get(IntptrTy, SizeInBytes + RightRedzoneSize),
26390b57cec5SDimitry Andric         ConstantExpr::getPointerCast(Name, IntptrTy),
26400b57cec5SDimitry Andric         ConstantExpr::getPointerCast(ModuleName, IntptrTy),
264181ad6265SDimitry Andric         ConstantInt::get(IntptrTy, MD.IsDynInit),
264281ad6265SDimitry Andric         Constant::getNullValue(IntptrTy),
26430b57cec5SDimitry Andric         ConstantExpr::getPointerCast(ODRIndicator, IntptrTy));
26440b57cec5SDimitry Andric 
264581ad6265SDimitry Andric     if (ClInitializers && MD.IsDynInit)
264681ad6265SDimitry Andric       HasDynamicallyInitializedGlobals = true;
26470b57cec5SDimitry Andric 
26480b57cec5SDimitry Andric     LLVM_DEBUG(dbgs() << "NEW GLOBAL: " << *NewGlobal << "\n");
26490b57cec5SDimitry Andric 
26500b57cec5SDimitry Andric     Initializers[i] = Initializer;
26510b57cec5SDimitry Andric   }
26520b57cec5SDimitry Andric 
26530b57cec5SDimitry Andric   // Add instrumented globals to llvm.compiler.used list to avoid LTO from
26540b57cec5SDimitry Andric   // ConstantMerge'ing them.
26550b57cec5SDimitry Andric   SmallVector<GlobalValue *, 16> GlobalsToAddToUsedList;
26560b57cec5SDimitry Andric   for (size_t i = 0; i < n; i++) {
26570b57cec5SDimitry Andric     GlobalVariable *G = NewGlobals[i];
26580b57cec5SDimitry Andric     if (G->getName().empty()) continue;
26590b57cec5SDimitry Andric     GlobalsToAddToUsedList.push_back(G);
26600b57cec5SDimitry Andric   }
26610b57cec5SDimitry Andric   appendToCompilerUsed(M, ArrayRef<GlobalValue *>(GlobalsToAddToUsedList));
26620b57cec5SDimitry Andric 
2663bdb86d1aSDimitry Andric   if (UseGlobalsGC && TargetTriple.isOSBinFormatELF()) {
2664bdb86d1aSDimitry Andric     // Use COMDAT and register globals even if n == 0 to ensure that (a) the
2665bdb86d1aSDimitry Andric     // linkage unit will only have one module constructor, and (b) the register
2666bdb86d1aSDimitry Andric     // function will be called. The module destructor is not created when n ==
2667bdb86d1aSDimitry Andric     // 0.
26680b57cec5SDimitry Andric     *CtorComdat = true;
2669bdb86d1aSDimitry Andric     instrumentGlobalsELF(IRB, M, NewGlobals, Initializers,
2670bdb86d1aSDimitry Andric                          getUniqueModuleId(&M));
2671bdb86d1aSDimitry Andric   } else if (n == 0) {
2672bdb86d1aSDimitry Andric     // When UseGlobalsGC is false, COMDAT can still be used if n == 0, because
2673bdb86d1aSDimitry Andric     // all compile units will have identical module constructor/destructor.
2674bdb86d1aSDimitry Andric     *CtorComdat = TargetTriple.isOSBinFormatELF();
2675bdb86d1aSDimitry Andric   } else {
2676bdb86d1aSDimitry Andric     *CtorComdat = false;
2677bdb86d1aSDimitry Andric     if (UseGlobalsGC && TargetTriple.isOSBinFormatCOFF()) {
26780b57cec5SDimitry Andric       InstrumentGlobalsCOFF(IRB, M, NewGlobals, Initializers);
26790b57cec5SDimitry Andric     } else if (UseGlobalsGC && ShouldUseMachOGlobalsSection()) {
26800b57cec5SDimitry Andric       InstrumentGlobalsMachO(IRB, M, NewGlobals, Initializers);
26810b57cec5SDimitry Andric     } else {
26820b57cec5SDimitry Andric       InstrumentGlobalsWithMetadataArray(IRB, M, NewGlobals, Initializers);
26830b57cec5SDimitry Andric     }
2684bdb86d1aSDimitry Andric   }
26850b57cec5SDimitry Andric 
26860b57cec5SDimitry Andric   // Create calls for poisoning before initializers run and unpoisoning after.
26870b57cec5SDimitry Andric   if (HasDynamicallyInitializedGlobals)
26880b57cec5SDimitry Andric     createInitializerPoisonCalls(M, ModuleName);
26890b57cec5SDimitry Andric 
26900b57cec5SDimitry Andric   LLVM_DEBUG(dbgs() << M);
26910b57cec5SDimitry Andric }
26920b57cec5SDimitry Andric 
26935ffd83dbSDimitry Andric uint64_t
26945ffd83dbSDimitry Andric ModuleAddressSanitizer::getRedzoneSizeForGlobal(uint64_t SizeInBytes) const {
26955ffd83dbSDimitry Andric   constexpr uint64_t kMaxRZ = 1 << 18;
26965ffd83dbSDimitry Andric   const uint64_t MinRZ = getMinRedzoneSizeForGlobal();
26975ffd83dbSDimitry Andric 
2698fe6060f1SDimitry Andric   uint64_t RZ = 0;
2699fe6060f1SDimitry Andric   if (SizeInBytes <= MinRZ / 2) {
2700fe6060f1SDimitry Andric     // Reduce redzone size for small size objects, e.g. int, char[1]. MinRZ is
2701fe6060f1SDimitry Andric     // at least 32 bytes, optimize when SizeInBytes is less than or equal to
2702fe6060f1SDimitry Andric     // half of MinRZ.
2703fe6060f1SDimitry Andric     RZ = MinRZ - SizeInBytes;
2704fe6060f1SDimitry Andric   } else {
27055ffd83dbSDimitry Andric     // Calculate RZ, where MinRZ <= RZ <= MaxRZ, and RZ ~ 1/4 * SizeInBytes.
2706bdd1243dSDimitry Andric     RZ = std::clamp((SizeInBytes / MinRZ / 4) * MinRZ, MinRZ, kMaxRZ);
27075ffd83dbSDimitry Andric 
27085ffd83dbSDimitry Andric     // Round up to multiple of MinRZ.
27095ffd83dbSDimitry Andric     if (SizeInBytes % MinRZ)
27105ffd83dbSDimitry Andric       RZ += MinRZ - (SizeInBytes % MinRZ);
2711fe6060f1SDimitry Andric   }
2712fe6060f1SDimitry Andric 
27135ffd83dbSDimitry Andric   assert((RZ + SizeInBytes) % MinRZ == 0);
27145ffd83dbSDimitry Andric 
27155ffd83dbSDimitry Andric   return RZ;
27165ffd83dbSDimitry Andric }
27175ffd83dbSDimitry Andric 
27180b57cec5SDimitry Andric int ModuleAddressSanitizer::GetAsanVersion(const Module &M) const {
27190b57cec5SDimitry Andric   int LongSize = M.getDataLayout().getPointerSizeInBits();
27200b57cec5SDimitry Andric   bool isAndroid = Triple(M.getTargetTriple()).isAndroid();
27210b57cec5SDimitry Andric   int Version = 8;
27220b57cec5SDimitry Andric   // 32-bit Android is one version ahead because of the switch to dynamic
27230b57cec5SDimitry Andric   // shadow.
27240b57cec5SDimitry Andric   Version += (LongSize == 32 && isAndroid);
27250b57cec5SDimitry Andric   return Version;
27260b57cec5SDimitry Andric }
27270b57cec5SDimitry Andric 
27280b57cec5SDimitry Andric bool ModuleAddressSanitizer::instrumentModule(Module &M) {
27290b57cec5SDimitry Andric   initializeCallbacks(M);
27300b57cec5SDimitry Andric 
27310b57cec5SDimitry Andric   // Create a module constructor. A destructor is created lazily because not all
27320b57cec5SDimitry Andric   // platforms, and not all modules need it.
2733bdd1243dSDimitry Andric   if (ConstructorKind == AsanCtorKind::Global) {
27345ffd83dbSDimitry Andric     if (CompileKernel) {
27355ffd83dbSDimitry Andric       // The kernel always builds with its own runtime, and therefore does not
27365ffd83dbSDimitry Andric       // need the init and version check calls.
27375ffd83dbSDimitry Andric       AsanCtorFunction = createSanitizerCtor(M, kAsanModuleCtorName);
27385ffd83dbSDimitry Andric     } else {
27398bcb0991SDimitry Andric       std::string AsanVersion = std::to_string(GetAsanVersion(M));
27400b57cec5SDimitry Andric       std::string VersionCheckName =
27415f757f3fSDimitry Andric           InsertVersionCheck ? (kAsanVersionCheckNamePrefix + AsanVersion) : "";
27425ffd83dbSDimitry Andric       std::tie(AsanCtorFunction, std::ignore) =
27435ffd83dbSDimitry Andric           createSanitizerCtorAndInitFunctions(M, kAsanModuleCtorName,
27445ffd83dbSDimitry Andric                                               kAsanInitName, /*InitArgTypes=*/{},
27450b57cec5SDimitry Andric                                               /*InitArgs=*/{}, VersionCheckName);
27465ffd83dbSDimitry Andric     }
2747bdd1243dSDimitry Andric   }
27480b57cec5SDimitry Andric 
27490b57cec5SDimitry Andric   bool CtorComdat = true;
27500b57cec5SDimitry Andric   if (ClGlobals) {
2751bdd1243dSDimitry Andric     assert(AsanCtorFunction || ConstructorKind == AsanCtorKind::None);
2752bdd1243dSDimitry Andric     if (AsanCtorFunction) {
27530b57cec5SDimitry Andric       IRBuilder<> IRB(AsanCtorFunction->getEntryBlock().getTerminator());
2754bdb86d1aSDimitry Andric       instrumentGlobals(IRB, M, &CtorComdat);
2755bdd1243dSDimitry Andric     } else {
2756bdd1243dSDimitry Andric       IRBuilder<> IRB(*C);
2757bdb86d1aSDimitry Andric       instrumentGlobals(IRB, M, &CtorComdat);
2758bdd1243dSDimitry Andric     }
27590b57cec5SDimitry Andric   }
27600b57cec5SDimitry Andric 
27618bcb0991SDimitry Andric   const uint64_t Priority = GetCtorAndDtorPriority(TargetTriple);
27628bcb0991SDimitry Andric 
27630b57cec5SDimitry Andric   // Put the constructor and destructor in comdat if both
27640b57cec5SDimitry Andric   // (1) global instrumentation is not TU-specific
27650b57cec5SDimitry Andric   // (2) target is ELF.
27660b57cec5SDimitry Andric   if (UseCtorComdat && TargetTriple.isOSBinFormatELF() && CtorComdat) {
2767bdd1243dSDimitry Andric     if (AsanCtorFunction) {
27680b57cec5SDimitry Andric       AsanCtorFunction->setComdat(M.getOrInsertComdat(kAsanModuleCtorName));
27698bcb0991SDimitry Andric       appendToGlobalCtors(M, AsanCtorFunction, Priority, AsanCtorFunction);
2770bdd1243dSDimitry Andric     }
27710b57cec5SDimitry Andric     if (AsanDtorFunction) {
27720b57cec5SDimitry Andric       AsanDtorFunction->setComdat(M.getOrInsertComdat(kAsanModuleDtorName));
27738bcb0991SDimitry Andric       appendToGlobalDtors(M, AsanDtorFunction, Priority, AsanDtorFunction);
27740b57cec5SDimitry Andric     }
27750b57cec5SDimitry Andric   } else {
2776bdd1243dSDimitry Andric     if (AsanCtorFunction)
27778bcb0991SDimitry Andric       appendToGlobalCtors(M, AsanCtorFunction, Priority);
27780b57cec5SDimitry Andric     if (AsanDtorFunction)
27798bcb0991SDimitry Andric       appendToGlobalDtors(M, AsanDtorFunction, Priority);
27800b57cec5SDimitry Andric   }
27810b57cec5SDimitry Andric 
27828bcb0991SDimitry Andric   return true;
27830b57cec5SDimitry Andric }
27840b57cec5SDimitry Andric 
2785bdd1243dSDimitry Andric void AddressSanitizer::initializeCallbacks(Module &M, const TargetLibraryInfo *TLI) {
27860b57cec5SDimitry Andric   IRBuilder<> IRB(*C);
27870b57cec5SDimitry Andric   // Create __asan_report* callbacks.
27880b57cec5SDimitry Andric   // IsWrite, TypeSize and Exp are encoded in the function name.
27890b57cec5SDimitry Andric   for (int Exp = 0; Exp < 2; Exp++) {
27900b57cec5SDimitry Andric     for (size_t AccessIsWrite = 0; AccessIsWrite <= 1; AccessIsWrite++) {
27910b57cec5SDimitry Andric       const std::string TypeStr = AccessIsWrite ? "store" : "load";
27920b57cec5SDimitry Andric       const std::string ExpStr = Exp ? "exp_" : "";
27930b57cec5SDimitry Andric       const std::string EndingStr = Recover ? "_noabort" : "";
27940b57cec5SDimitry Andric 
27950b57cec5SDimitry Andric       SmallVector<Type *, 3> Args2 = {IntptrTy, IntptrTy};
27960b57cec5SDimitry Andric       SmallVector<Type *, 2> Args1{1, IntptrTy};
2797bdd1243dSDimitry Andric       AttributeList AL2;
2798bdd1243dSDimitry Andric       AttributeList AL1;
27990b57cec5SDimitry Andric       if (Exp) {
28000b57cec5SDimitry Andric         Type *ExpType = Type::getInt32Ty(*C);
28010b57cec5SDimitry Andric         Args2.push_back(ExpType);
28020b57cec5SDimitry Andric         Args1.push_back(ExpType);
2803bdd1243dSDimitry Andric         if (auto AK = TLI->getExtAttrForI32Param(false)) {
2804bdd1243dSDimitry Andric           AL2 = AL2.addParamAttribute(*C, 2, AK);
2805bdd1243dSDimitry Andric           AL1 = AL1.addParamAttribute(*C, 1, AK);
2806bdd1243dSDimitry Andric         }
28070b57cec5SDimitry Andric       }
28080b57cec5SDimitry Andric       AsanErrorCallbackSized[AccessIsWrite][Exp] = M.getOrInsertFunction(
28090b57cec5SDimitry Andric           kAsanReportErrorTemplate + ExpStr + TypeStr + "_n" + EndingStr,
2810bdd1243dSDimitry Andric           FunctionType::get(IRB.getVoidTy(), Args2, false), AL2);
28110b57cec5SDimitry Andric 
28120b57cec5SDimitry Andric       AsanMemoryAccessCallbackSized[AccessIsWrite][Exp] = M.getOrInsertFunction(
28130b57cec5SDimitry Andric           ClMemoryAccessCallbackPrefix + ExpStr + TypeStr + "N" + EndingStr,
2814bdd1243dSDimitry Andric           FunctionType::get(IRB.getVoidTy(), Args2, false), AL2);
28150b57cec5SDimitry Andric 
28160b57cec5SDimitry Andric       for (size_t AccessSizeIndex = 0; AccessSizeIndex < kNumberOfAccessSizes;
28170b57cec5SDimitry Andric            AccessSizeIndex++) {
28180b57cec5SDimitry Andric         const std::string Suffix = TypeStr + itostr(1ULL << AccessSizeIndex);
28190b57cec5SDimitry Andric         AsanErrorCallback[AccessIsWrite][Exp][AccessSizeIndex] =
28200b57cec5SDimitry Andric             M.getOrInsertFunction(
28210b57cec5SDimitry Andric                 kAsanReportErrorTemplate + ExpStr + Suffix + EndingStr,
2822bdd1243dSDimitry Andric                 FunctionType::get(IRB.getVoidTy(), Args1, false), AL1);
28230b57cec5SDimitry Andric 
28240b57cec5SDimitry Andric         AsanMemoryAccessCallback[AccessIsWrite][Exp][AccessSizeIndex] =
28250b57cec5SDimitry Andric             M.getOrInsertFunction(
28260b57cec5SDimitry Andric                 ClMemoryAccessCallbackPrefix + ExpStr + Suffix + EndingStr,
2827bdd1243dSDimitry Andric                 FunctionType::get(IRB.getVoidTy(), Args1, false), AL1);
28280b57cec5SDimitry Andric       }
28290b57cec5SDimitry Andric     }
28300b57cec5SDimitry Andric   }
28310b57cec5SDimitry Andric 
28320b57cec5SDimitry Andric   const std::string MemIntrinCallbackPrefix =
283381ad6265SDimitry Andric       (CompileKernel && !ClKasanMemIntrinCallbackPrefix)
283481ad6265SDimitry Andric           ? std::string("")
283581ad6265SDimitry Andric           : ClMemoryAccessCallbackPrefix;
28360b57cec5SDimitry Andric   AsanMemmove = M.getOrInsertFunction(MemIntrinCallbackPrefix + "memmove",
28375f757f3fSDimitry Andric                                       PtrTy, PtrTy, PtrTy, IntptrTy);
28385f757f3fSDimitry Andric   AsanMemcpy = M.getOrInsertFunction(MemIntrinCallbackPrefix + "memcpy", PtrTy,
28395f757f3fSDimitry Andric                                      PtrTy, PtrTy, IntptrTy);
28400b57cec5SDimitry Andric   AsanMemset = M.getOrInsertFunction(MemIntrinCallbackPrefix + "memset",
2841bdd1243dSDimitry Andric                                      TLI->getAttrList(C, {1}, /*Signed=*/false),
28425f757f3fSDimitry Andric                                      PtrTy, PtrTy, IRB.getInt32Ty(), IntptrTy);
28430b57cec5SDimitry Andric 
28440b57cec5SDimitry Andric   AsanHandleNoReturnFunc =
28450b57cec5SDimitry Andric       M.getOrInsertFunction(kAsanHandleNoReturnName, IRB.getVoidTy());
28460b57cec5SDimitry Andric 
28470b57cec5SDimitry Andric   AsanPtrCmpFunction =
28480b57cec5SDimitry Andric       M.getOrInsertFunction(kAsanPtrCmp, IRB.getVoidTy(), IntptrTy, IntptrTy);
28490b57cec5SDimitry Andric   AsanPtrSubFunction =
28500b57cec5SDimitry Andric       M.getOrInsertFunction(kAsanPtrSub, IRB.getVoidTy(), IntptrTy, IntptrTy);
28510b57cec5SDimitry Andric   if (Mapping.InGlobal)
28520b57cec5SDimitry Andric     AsanShadowGlobal = M.getOrInsertGlobal("__asan_shadow",
28530b57cec5SDimitry Andric                                            ArrayType::get(IRB.getInt8Ty(), 0));
2854fe6060f1SDimitry Andric 
28555f757f3fSDimitry Andric   AMDGPUAddressShared =
28565f757f3fSDimitry Andric       M.getOrInsertFunction(kAMDGPUAddressSharedName, IRB.getInt1Ty(), PtrTy);
28575f757f3fSDimitry Andric   AMDGPUAddressPrivate =
28585f757f3fSDimitry Andric       M.getOrInsertFunction(kAMDGPUAddressPrivateName, IRB.getInt1Ty(), PtrTy);
28590b57cec5SDimitry Andric }
28600b57cec5SDimitry Andric 
28610b57cec5SDimitry Andric bool AddressSanitizer::maybeInsertAsanInitAtFunctionEntry(Function &F) {
28620b57cec5SDimitry Andric   // For each NSObject descendant having a +load method, this method is invoked
28630b57cec5SDimitry Andric   // by the ObjC runtime before any of the static constructors is called.
28640b57cec5SDimitry Andric   // Therefore we need to instrument such methods with a call to __asan_init
28650b57cec5SDimitry Andric   // at the beginning in order to initialize our runtime before any access to
28660b57cec5SDimitry Andric   // the shadow memory.
28670b57cec5SDimitry Andric   // We cannot just ignore these methods, because they may call other
28680b57cec5SDimitry Andric   // instrumented functions.
2869cb14a3feSDimitry Andric   if (F.getName().contains(" load]")) {
28700b57cec5SDimitry Andric     FunctionCallee AsanInitFunction =
28710b57cec5SDimitry Andric         declareSanitizerInitFunction(*F.getParent(), kAsanInitName, {});
28720b57cec5SDimitry Andric     IRBuilder<> IRB(&F.front(), F.front().begin());
28730b57cec5SDimitry Andric     IRB.CreateCall(AsanInitFunction, {});
28740b57cec5SDimitry Andric     return true;
28750b57cec5SDimitry Andric   }
28760b57cec5SDimitry Andric   return false;
28770b57cec5SDimitry Andric }
28780b57cec5SDimitry Andric 
28795ffd83dbSDimitry Andric bool AddressSanitizer::maybeInsertDynamicShadowAtFunctionEntry(Function &F) {
28800b57cec5SDimitry Andric   // Generate code only when dynamic addressing is needed.
28810b57cec5SDimitry Andric   if (Mapping.Offset != kDynamicShadowSentinel)
28825ffd83dbSDimitry Andric     return false;
28830b57cec5SDimitry Andric 
28840b57cec5SDimitry Andric   IRBuilder<> IRB(&F.front().front());
28850b57cec5SDimitry Andric   if (Mapping.InGlobal) {
28860b57cec5SDimitry Andric     if (ClWithIfuncSuppressRemat) {
28870b57cec5SDimitry Andric       // An empty inline asm with input reg == output reg.
28880b57cec5SDimitry Andric       // An opaque pointer-to-int cast, basically.
28890b57cec5SDimitry Andric       InlineAsm *Asm = InlineAsm::get(
28900b57cec5SDimitry Andric           FunctionType::get(IntptrTy, {AsanShadowGlobal->getType()}, false),
28910b57cec5SDimitry Andric           StringRef(""), StringRef("=r,0"),
28920b57cec5SDimitry Andric           /*hasSideEffects=*/false);
28930b57cec5SDimitry Andric       LocalDynamicShadow =
28940b57cec5SDimitry Andric           IRB.CreateCall(Asm, {AsanShadowGlobal}, ".asan.shadow");
28950b57cec5SDimitry Andric     } else {
28960b57cec5SDimitry Andric       LocalDynamicShadow =
28970b57cec5SDimitry Andric           IRB.CreatePointerCast(AsanShadowGlobal, IntptrTy, ".asan.shadow");
28980b57cec5SDimitry Andric     }
28990b57cec5SDimitry Andric   } else {
29000b57cec5SDimitry Andric     Value *GlobalDynamicAddress = F.getParent()->getOrInsertGlobal(
29010b57cec5SDimitry Andric         kAsanShadowMemoryDynamicAddress, IntptrTy);
29020b57cec5SDimitry Andric     LocalDynamicShadow = IRB.CreateLoad(IntptrTy, GlobalDynamicAddress);
29030b57cec5SDimitry Andric   }
29045ffd83dbSDimitry Andric   return true;
29050b57cec5SDimitry Andric }
29060b57cec5SDimitry Andric 
29070b57cec5SDimitry Andric void AddressSanitizer::markEscapedLocalAllocas(Function &F) {
29080b57cec5SDimitry Andric   // Find the one possible call to llvm.localescape and pre-mark allocas passed
29090b57cec5SDimitry Andric   // to it as uninteresting. This assumes we haven't started processing allocas
29100b57cec5SDimitry Andric   // yet. This check is done up front because iterating the use list in
29110b57cec5SDimitry Andric   // isInterestingAlloca would be algorithmically slower.
29120b57cec5SDimitry Andric   assert(ProcessedAllocas.empty() && "must process localescape before allocas");
29130b57cec5SDimitry Andric 
29140b57cec5SDimitry Andric   // Try to get the declaration of llvm.localescape. If it's not in the module,
29150b57cec5SDimitry Andric   // we can exit early.
29160b57cec5SDimitry Andric   if (!F.getParent()->getFunction("llvm.localescape")) return;
29170b57cec5SDimitry Andric 
29180b57cec5SDimitry Andric   // Look for a call to llvm.localescape call in the entry block. It can't be in
29190b57cec5SDimitry Andric   // any other block.
29200b57cec5SDimitry Andric   for (Instruction &I : F.getEntryBlock()) {
29210b57cec5SDimitry Andric     IntrinsicInst *II = dyn_cast<IntrinsicInst>(&I);
29220b57cec5SDimitry Andric     if (II && II->getIntrinsicID() == Intrinsic::localescape) {
29230b57cec5SDimitry Andric       // We found a call. Mark all the allocas passed in as uninteresting.
2924349cc55cSDimitry Andric       for (Value *Arg : II->args()) {
29250b57cec5SDimitry Andric         AllocaInst *AI = dyn_cast<AllocaInst>(Arg->stripPointerCasts());
29260b57cec5SDimitry Andric         assert(AI && AI->isStaticAlloca() &&
29270b57cec5SDimitry Andric                "non-static alloca arg to localescape");
29280b57cec5SDimitry Andric         ProcessedAllocas[AI] = false;
29290b57cec5SDimitry Andric       }
29300b57cec5SDimitry Andric       break;
29310b57cec5SDimitry Andric     }
29320b57cec5SDimitry Andric   }
29330b57cec5SDimitry Andric }
29340b57cec5SDimitry Andric 
29355ffd83dbSDimitry Andric bool AddressSanitizer::suppressInstrumentationSiteForDebug(int &Instrumented) {
29365ffd83dbSDimitry Andric   bool ShouldInstrument =
29375ffd83dbSDimitry Andric       ClDebugMin < 0 || ClDebugMax < 0 ||
29385ffd83dbSDimitry Andric       (Instrumented >= ClDebugMin && Instrumented <= ClDebugMax);
29395ffd83dbSDimitry Andric   Instrumented++;
29405ffd83dbSDimitry Andric   return !ShouldInstrument;
29415ffd83dbSDimitry Andric }
29425ffd83dbSDimitry Andric 
29430b57cec5SDimitry Andric bool AddressSanitizer::instrumentFunction(Function &F,
29440b57cec5SDimitry Andric                                           const TargetLibraryInfo *TLI) {
2945349cc55cSDimitry Andric   if (F.empty())
2946349cc55cSDimitry Andric     return false;
29470b57cec5SDimitry Andric   if (F.getLinkage() == GlobalValue::AvailableExternallyLinkage) return false;
29480b57cec5SDimitry Andric   if (!ClDebugFunc.empty() && ClDebugFunc == F.getName()) return false;
29495f757f3fSDimitry Andric   if (F.getName().starts_with("__asan_")) return false;
2950*0fca6ea1SDimitry Andric   if (F.isPresplitCoroutine())
2951*0fca6ea1SDimitry Andric     return false;
29520b57cec5SDimitry Andric 
29530b57cec5SDimitry Andric   bool FunctionModified = false;
29540b57cec5SDimitry Andric 
29550b57cec5SDimitry Andric   // If needed, insert __asan_init before checking for SanitizeAddress attr.
29560b57cec5SDimitry Andric   // This function needs to be called even if the function body is not
29570b57cec5SDimitry Andric   // instrumented.
29580b57cec5SDimitry Andric   if (maybeInsertAsanInitAtFunctionEntry(F))
29590b57cec5SDimitry Andric     FunctionModified = true;
29600b57cec5SDimitry Andric 
29610b57cec5SDimitry Andric   // Leave if the function doesn't need instrumentation.
29620b57cec5SDimitry Andric   if (!F.hasFnAttribute(Attribute::SanitizeAddress)) return FunctionModified;
29630b57cec5SDimitry Andric 
296481ad6265SDimitry Andric   if (F.hasFnAttribute(Attribute::DisableSanitizerInstrumentation))
296581ad6265SDimitry Andric     return FunctionModified;
296681ad6265SDimitry Andric 
29670b57cec5SDimitry Andric   LLVM_DEBUG(dbgs() << "ASAN instrumenting:\n" << F << "\n");
29680b57cec5SDimitry Andric 
2969bdd1243dSDimitry Andric   initializeCallbacks(*F.getParent(), TLI);
29700b57cec5SDimitry Andric 
29710b57cec5SDimitry Andric   FunctionStateRAII CleanupObj(this);
29720b57cec5SDimitry Andric 
2973*0fca6ea1SDimitry Andric   RuntimeCallInserter RTCI(F);
2974*0fca6ea1SDimitry Andric 
29755ffd83dbSDimitry Andric   FunctionModified |= maybeInsertDynamicShadowAtFunctionEntry(F);
29760b57cec5SDimitry Andric 
29770b57cec5SDimitry Andric   // We can't instrument allocas used with llvm.localescape. Only static allocas
29780b57cec5SDimitry Andric   // can be passed to that intrinsic.
29790b57cec5SDimitry Andric   markEscapedLocalAllocas(F);
29800b57cec5SDimitry Andric 
29810b57cec5SDimitry Andric   // We want to instrument every address only once per basic block (unless there
29820b57cec5SDimitry Andric   // are calls between uses).
29830b57cec5SDimitry Andric   SmallPtrSet<Value *, 16> TempsToInstrument;
29845ffd83dbSDimitry Andric   SmallVector<InterestingMemoryOperand, 16> OperandsToInstrument;
29855ffd83dbSDimitry Andric   SmallVector<MemIntrinsic *, 16> IntrinToInstrument;
29860b57cec5SDimitry Andric   SmallVector<Instruction *, 8> NoReturnCalls;
29870b57cec5SDimitry Andric   SmallVector<BasicBlock *, 16> AllBlocks;
29880b57cec5SDimitry Andric   SmallVector<Instruction *, 16> PointerComparisonsOrSubtracts;
29890b57cec5SDimitry Andric 
29900b57cec5SDimitry Andric   // Fill the set of memory operations to instrument.
29910b57cec5SDimitry Andric   for (auto &BB : F) {
29920b57cec5SDimitry Andric     AllBlocks.push_back(&BB);
29930b57cec5SDimitry Andric     TempsToInstrument.clear();
29940b57cec5SDimitry Andric     int NumInsnsPerBB = 0;
29950b57cec5SDimitry Andric     for (auto &Inst : BB) {
29960b57cec5SDimitry Andric       if (LooksLikeCodeInBug11395(&Inst)) return false;
299781ad6265SDimitry Andric       // Skip instructions inserted by another instrumentation.
299881ad6265SDimitry Andric       if (Inst.hasMetadata(LLVMContext::MD_nosanitize))
299981ad6265SDimitry Andric         continue;
30005ffd83dbSDimitry Andric       SmallVector<InterestingMemoryOperand, 1> InterestingOperands;
30015ffd83dbSDimitry Andric       getInterestingMemoryOperands(&Inst, InterestingOperands);
30025ffd83dbSDimitry Andric 
30035ffd83dbSDimitry Andric       if (!InterestingOperands.empty()) {
30045ffd83dbSDimitry Andric         for (auto &Operand : InterestingOperands) {
30050b57cec5SDimitry Andric           if (ClOpt && ClOptSameTemp) {
30065ffd83dbSDimitry Andric             Value *Ptr = Operand.getPtr();
30070b57cec5SDimitry Andric             // If we have a mask, skip instrumentation if we've already
30080b57cec5SDimitry Andric             // instrumented the full object. But don't add to TempsToInstrument
30090b57cec5SDimitry Andric             // because we might get another load/store with a different mask.
30105ffd83dbSDimitry Andric             if (Operand.MaybeMask) {
30115ffd83dbSDimitry Andric               if (TempsToInstrument.count(Ptr))
30120b57cec5SDimitry Andric                 continue; // We've seen this (whole) temp in the current BB.
30130b57cec5SDimitry Andric             } else {
30145ffd83dbSDimitry Andric               if (!TempsToInstrument.insert(Ptr).second)
30150b57cec5SDimitry Andric                 continue; // We've seen this temp in the current BB.
30160b57cec5SDimitry Andric             }
30170b57cec5SDimitry Andric           }
30185ffd83dbSDimitry Andric           OperandsToInstrument.push_back(Operand);
30195ffd83dbSDimitry Andric           NumInsnsPerBB++;
30205ffd83dbSDimitry Andric         }
30210b57cec5SDimitry Andric       } else if (((ClInvalidPointerPairs || ClInvalidPointerCmp) &&
30220b57cec5SDimitry Andric                   isInterestingPointerComparison(&Inst)) ||
30230b57cec5SDimitry Andric                  ((ClInvalidPointerPairs || ClInvalidPointerSub) &&
30240b57cec5SDimitry Andric                   isInterestingPointerSubtraction(&Inst))) {
30250b57cec5SDimitry Andric         PointerComparisonsOrSubtracts.push_back(&Inst);
30265ffd83dbSDimitry Andric       } else if (MemIntrinsic *MI = dyn_cast<MemIntrinsic>(&Inst)) {
30270b57cec5SDimitry Andric         // ok, take it.
30285ffd83dbSDimitry Andric         IntrinToInstrument.push_back(MI);
30295ffd83dbSDimitry Andric         NumInsnsPerBB++;
30300b57cec5SDimitry Andric       } else {
30315ffd83dbSDimitry Andric         if (auto *CB = dyn_cast<CallBase>(&Inst)) {
30320b57cec5SDimitry Andric           // A call inside BB.
30330b57cec5SDimitry Andric           TempsToInstrument.clear();
303481ad6265SDimitry Andric           if (CB->doesNotReturn())
30355ffd83dbSDimitry Andric             NoReturnCalls.push_back(CB);
30360b57cec5SDimitry Andric         }
30370b57cec5SDimitry Andric         if (CallInst *CI = dyn_cast<CallInst>(&Inst))
30380b57cec5SDimitry Andric           maybeMarkSanitizerLibraryCallNoBuiltin(CI, TLI);
30390b57cec5SDimitry Andric       }
30400b57cec5SDimitry Andric       if (NumInsnsPerBB >= ClMaxInsnsToInstrumentPerBB) break;
30410b57cec5SDimitry Andric     }
30420b57cec5SDimitry Andric   }
30430b57cec5SDimitry Andric 
30445f757f3fSDimitry Andric   bool UseCalls = (InstrumentationWithCallsThreshold >= 0 &&
30455ffd83dbSDimitry Andric                    OperandsToInstrument.size() + IntrinToInstrument.size() >
30465f757f3fSDimitry Andric                        (unsigned)InstrumentationWithCallsThreshold);
3047*0fca6ea1SDimitry Andric   const DataLayout &DL = F.getDataLayout();
30480b57cec5SDimitry Andric   ObjectSizeOpts ObjSizeOpts;
30490b57cec5SDimitry Andric   ObjSizeOpts.RoundToAlign = true;
30500b57cec5SDimitry Andric   ObjectSizeOffsetVisitor ObjSizeVis(DL, TLI, F.getContext(), ObjSizeOpts);
30510b57cec5SDimitry Andric 
30520b57cec5SDimitry Andric   // Instrument.
30530b57cec5SDimitry Andric   int NumInstrumented = 0;
30545ffd83dbSDimitry Andric   for (auto &Operand : OperandsToInstrument) {
30555ffd83dbSDimitry Andric     if (!suppressInstrumentationSiteForDebug(NumInstrumented))
30565ffd83dbSDimitry Andric       instrumentMop(ObjSizeVis, Operand, UseCalls,
3057*0fca6ea1SDimitry Andric                     F.getDataLayout(), RTCI);
30585ffd83dbSDimitry Andric     FunctionModified = true;
30590b57cec5SDimitry Andric   }
3060bdd1243dSDimitry Andric   for (auto *Inst : IntrinToInstrument) {
30615ffd83dbSDimitry Andric     if (!suppressInstrumentationSiteForDebug(NumInstrumented))
3062*0fca6ea1SDimitry Andric       instrumentMemIntrinsic(Inst, RTCI);
30635ffd83dbSDimitry Andric     FunctionModified = true;
30640b57cec5SDimitry Andric   }
30650b57cec5SDimitry Andric 
3066*0fca6ea1SDimitry Andric   FunctionStackPoisoner FSP(F, *this, RTCI);
30670b57cec5SDimitry Andric   bool ChangedStack = FSP.runOnFunction();
30680b57cec5SDimitry Andric 
30690b57cec5SDimitry Andric   // We must unpoison the stack before NoReturn calls (throw, _exit, etc).
30700b57cec5SDimitry Andric   // See e.g. https://github.com/google/sanitizers/issues/37
3071bdd1243dSDimitry Andric   for (auto *CI : NoReturnCalls) {
30720b57cec5SDimitry Andric     IRBuilder<> IRB(CI);
3073*0fca6ea1SDimitry Andric     RTCI.createRuntimeCall(IRB, AsanHandleNoReturnFunc, {});
30740b57cec5SDimitry Andric   }
30750b57cec5SDimitry Andric 
3076bdd1243dSDimitry Andric   for (auto *Inst : PointerComparisonsOrSubtracts) {
3077*0fca6ea1SDimitry Andric     instrumentPointerComparisonOrSubtraction(Inst, RTCI);
30785ffd83dbSDimitry Andric     FunctionModified = true;
30790b57cec5SDimitry Andric   }
30800b57cec5SDimitry Andric 
30815ffd83dbSDimitry Andric   if (ChangedStack || !NoReturnCalls.empty())
30820b57cec5SDimitry Andric     FunctionModified = true;
30830b57cec5SDimitry Andric 
30840b57cec5SDimitry Andric   LLVM_DEBUG(dbgs() << "ASAN done instrumenting: " << FunctionModified << " "
30850b57cec5SDimitry Andric                     << F << "\n");
30860b57cec5SDimitry Andric 
30870b57cec5SDimitry Andric   return FunctionModified;
30880b57cec5SDimitry Andric }
30890b57cec5SDimitry Andric 
30900b57cec5SDimitry Andric // Workaround for bug 11395: we don't want to instrument stack in functions
30910b57cec5SDimitry Andric // with large assembly blobs (32-bit only), otherwise reg alloc may crash.
30920b57cec5SDimitry Andric // FIXME: remove once the bug 11395 is fixed.
30930b57cec5SDimitry Andric bool AddressSanitizer::LooksLikeCodeInBug11395(Instruction *I) {
30940b57cec5SDimitry Andric   if (LongSize != 32) return false;
30950b57cec5SDimitry Andric   CallInst *CI = dyn_cast<CallInst>(I);
30960b57cec5SDimitry Andric   if (!CI || !CI->isInlineAsm()) return false;
3097349cc55cSDimitry Andric   if (CI->arg_size() <= 5)
3098349cc55cSDimitry Andric     return false;
30990b57cec5SDimitry Andric   // We have inline assembly with quite a few arguments.
31000b57cec5SDimitry Andric   return true;
31010b57cec5SDimitry Andric }
31020b57cec5SDimitry Andric 
31030b57cec5SDimitry Andric void FunctionStackPoisoner::initializeCallbacks(Module &M) {
31040b57cec5SDimitry Andric   IRBuilder<> IRB(*C);
3105fe6060f1SDimitry Andric   if (ASan.UseAfterReturn == AsanDetectStackUseAfterReturnMode::Always ||
3106fe6060f1SDimitry Andric       ASan.UseAfterReturn == AsanDetectStackUseAfterReturnMode::Runtime) {
3107fe6060f1SDimitry Andric     const char *MallocNameTemplate =
3108fe6060f1SDimitry Andric         ASan.UseAfterReturn == AsanDetectStackUseAfterReturnMode::Always
3109fe6060f1SDimitry Andric             ? kAsanStackMallocAlwaysNameTemplate
3110fe6060f1SDimitry Andric             : kAsanStackMallocNameTemplate;
3111fe6060f1SDimitry Andric     for (int Index = 0; Index <= kMaxAsanStackMallocSizeClass; Index++) {
3112fe6060f1SDimitry Andric       std::string Suffix = itostr(Index);
3113fe6060f1SDimitry Andric       AsanStackMallocFunc[Index] = M.getOrInsertFunction(
3114fe6060f1SDimitry Andric           MallocNameTemplate + Suffix, IntptrTy, IntptrTy);
3115fe6060f1SDimitry Andric       AsanStackFreeFunc[Index] =
31160b57cec5SDimitry Andric           M.getOrInsertFunction(kAsanStackFreeNameTemplate + Suffix,
31170b57cec5SDimitry Andric                                 IRB.getVoidTy(), IntptrTy, IntptrTy);
31180b57cec5SDimitry Andric     }
3119fe6060f1SDimitry Andric   }
31200b57cec5SDimitry Andric   if (ASan.UseAfterScope) {
31210b57cec5SDimitry Andric     AsanPoisonStackMemoryFunc = M.getOrInsertFunction(
31220b57cec5SDimitry Andric         kAsanPoisonStackMemoryName, IRB.getVoidTy(), IntptrTy, IntptrTy);
31230b57cec5SDimitry Andric     AsanUnpoisonStackMemoryFunc = M.getOrInsertFunction(
31240b57cec5SDimitry Andric         kAsanUnpoisonStackMemoryName, IRB.getVoidTy(), IntptrTy, IntptrTy);
31250b57cec5SDimitry Andric   }
31260b57cec5SDimitry Andric 
3127bdd1243dSDimitry Andric   for (size_t Val : {0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0xf1, 0xf2,
3128bdd1243dSDimitry Andric                      0xf3, 0xf5, 0xf8}) {
31290b57cec5SDimitry Andric     std::ostringstream Name;
31300b57cec5SDimitry Andric     Name << kAsanSetShadowPrefix;
31310b57cec5SDimitry Andric     Name << std::setw(2) << std::setfill('0') << std::hex << Val;
31320b57cec5SDimitry Andric     AsanSetShadowFunc[Val] =
31330b57cec5SDimitry Andric         M.getOrInsertFunction(Name.str(), IRB.getVoidTy(), IntptrTy, IntptrTy);
31340b57cec5SDimitry Andric   }
31350b57cec5SDimitry Andric 
31360b57cec5SDimitry Andric   AsanAllocaPoisonFunc = M.getOrInsertFunction(
31370b57cec5SDimitry Andric       kAsanAllocaPoison, IRB.getVoidTy(), IntptrTy, IntptrTy);
31380b57cec5SDimitry Andric   AsanAllocasUnpoisonFunc = M.getOrInsertFunction(
31390b57cec5SDimitry Andric       kAsanAllocasUnpoison, IRB.getVoidTy(), IntptrTy, IntptrTy);
31400b57cec5SDimitry Andric }
31410b57cec5SDimitry Andric 
31420b57cec5SDimitry Andric void FunctionStackPoisoner::copyToShadowInline(ArrayRef<uint8_t> ShadowMask,
31430b57cec5SDimitry Andric                                                ArrayRef<uint8_t> ShadowBytes,
31440b57cec5SDimitry Andric                                                size_t Begin, size_t End,
31450b57cec5SDimitry Andric                                                IRBuilder<> &IRB,
31460b57cec5SDimitry Andric                                                Value *ShadowBase) {
31470b57cec5SDimitry Andric   if (Begin >= End)
31480b57cec5SDimitry Andric     return;
31490b57cec5SDimitry Andric 
31500b57cec5SDimitry Andric   const size_t LargestStoreSizeInBytes =
31510b57cec5SDimitry Andric       std::min<size_t>(sizeof(uint64_t), ASan.LongSize / 8);
31520b57cec5SDimitry Andric 
3153*0fca6ea1SDimitry Andric   const bool IsLittleEndian = F.getDataLayout().isLittleEndian();
31540b57cec5SDimitry Andric 
31550b57cec5SDimitry Andric   // Poison given range in shadow using larges store size with out leading and
31560b57cec5SDimitry Andric   // trailing zeros in ShadowMask. Zeros never change, so they need neither
31570b57cec5SDimitry Andric   // poisoning nor up-poisoning. Still we don't mind if some of them get into a
31580b57cec5SDimitry Andric   // middle of a store.
31590b57cec5SDimitry Andric   for (size_t i = Begin; i < End;) {
31600b57cec5SDimitry Andric     if (!ShadowMask[i]) {
31610b57cec5SDimitry Andric       assert(!ShadowBytes[i]);
31620b57cec5SDimitry Andric       ++i;
31630b57cec5SDimitry Andric       continue;
31640b57cec5SDimitry Andric     }
31650b57cec5SDimitry Andric 
31660b57cec5SDimitry Andric     size_t StoreSizeInBytes = LargestStoreSizeInBytes;
31670b57cec5SDimitry Andric     // Fit store size into the range.
31680b57cec5SDimitry Andric     while (StoreSizeInBytes > End - i)
31690b57cec5SDimitry Andric       StoreSizeInBytes /= 2;
31700b57cec5SDimitry Andric 
31710b57cec5SDimitry Andric     // Minimize store size by trimming trailing zeros.
31720b57cec5SDimitry Andric     for (size_t j = StoreSizeInBytes - 1; j && !ShadowMask[i + j]; --j) {
31730b57cec5SDimitry Andric       while (j <= StoreSizeInBytes / 2)
31740b57cec5SDimitry Andric         StoreSizeInBytes /= 2;
31750b57cec5SDimitry Andric     }
31760b57cec5SDimitry Andric 
31770b57cec5SDimitry Andric     uint64_t Val = 0;
31780b57cec5SDimitry Andric     for (size_t j = 0; j < StoreSizeInBytes; j++) {
31790b57cec5SDimitry Andric       if (IsLittleEndian)
31800b57cec5SDimitry Andric         Val |= (uint64_t)ShadowBytes[i + j] << (8 * j);
31810b57cec5SDimitry Andric       else
31820b57cec5SDimitry Andric         Val = (Val << 8) | ShadowBytes[i + j];
31830b57cec5SDimitry Andric     }
31840b57cec5SDimitry Andric 
31850b57cec5SDimitry Andric     Value *Ptr = IRB.CreateAdd(ShadowBase, ConstantInt::get(IntptrTy, i));
31860b57cec5SDimitry Andric     Value *Poison = IRB.getIntN(StoreSizeInBytes * 8, Val);
31870b57cec5SDimitry Andric     IRB.CreateAlignedStore(
31885f757f3fSDimitry Andric         Poison, IRB.CreateIntToPtr(Ptr, PointerType::getUnqual(Poison->getContext())),
31895ffd83dbSDimitry Andric         Align(1));
31900b57cec5SDimitry Andric 
31910b57cec5SDimitry Andric     i += StoreSizeInBytes;
31920b57cec5SDimitry Andric   }
31930b57cec5SDimitry Andric }
31940b57cec5SDimitry Andric 
31950b57cec5SDimitry Andric void FunctionStackPoisoner::copyToShadow(ArrayRef<uint8_t> ShadowMask,
31960b57cec5SDimitry Andric                                          ArrayRef<uint8_t> ShadowBytes,
31970b57cec5SDimitry Andric                                          IRBuilder<> &IRB, Value *ShadowBase) {
31980b57cec5SDimitry Andric   copyToShadow(ShadowMask, ShadowBytes, 0, ShadowMask.size(), IRB, ShadowBase);
31990b57cec5SDimitry Andric }
32000b57cec5SDimitry Andric 
32010b57cec5SDimitry Andric void FunctionStackPoisoner::copyToShadow(ArrayRef<uint8_t> ShadowMask,
32020b57cec5SDimitry Andric                                          ArrayRef<uint8_t> ShadowBytes,
32030b57cec5SDimitry Andric                                          size_t Begin, size_t End,
32040b57cec5SDimitry Andric                                          IRBuilder<> &IRB, Value *ShadowBase) {
32050b57cec5SDimitry Andric   assert(ShadowMask.size() == ShadowBytes.size());
32060b57cec5SDimitry Andric   size_t Done = Begin;
32070b57cec5SDimitry Andric   for (size_t i = Begin, j = Begin + 1; i < End; i = j++) {
32080b57cec5SDimitry Andric     if (!ShadowMask[i]) {
32090b57cec5SDimitry Andric       assert(!ShadowBytes[i]);
32100b57cec5SDimitry Andric       continue;
32110b57cec5SDimitry Andric     }
32120b57cec5SDimitry Andric     uint8_t Val = ShadowBytes[i];
32130b57cec5SDimitry Andric     if (!AsanSetShadowFunc[Val])
32140b57cec5SDimitry Andric       continue;
32150b57cec5SDimitry Andric 
32160b57cec5SDimitry Andric     // Skip same values.
32170b57cec5SDimitry Andric     for (; j < End && ShadowMask[j] && Val == ShadowBytes[j]; ++j) {
32180b57cec5SDimitry Andric     }
32190b57cec5SDimitry Andric 
32205f757f3fSDimitry Andric     if (j - i >= ASan.MaxInlinePoisoningSize) {
32210b57cec5SDimitry Andric       copyToShadowInline(ShadowMask, ShadowBytes, Done, i, IRB, ShadowBase);
3222*0fca6ea1SDimitry Andric       RTCI.createRuntimeCall(
3223*0fca6ea1SDimitry Andric           IRB, AsanSetShadowFunc[Val],
32240b57cec5SDimitry Andric           {IRB.CreateAdd(ShadowBase, ConstantInt::get(IntptrTy, i)),
32250b57cec5SDimitry Andric            ConstantInt::get(IntptrTy, j - i)});
32260b57cec5SDimitry Andric       Done = j;
32270b57cec5SDimitry Andric     }
32280b57cec5SDimitry Andric   }
32290b57cec5SDimitry Andric 
32300b57cec5SDimitry Andric   copyToShadowInline(ShadowMask, ShadowBytes, Done, End, IRB, ShadowBase);
32310b57cec5SDimitry Andric }
32320b57cec5SDimitry Andric 
32330b57cec5SDimitry Andric // Fake stack allocator (asan_fake_stack.h) has 11 size classes
32340b57cec5SDimitry Andric // for every power of 2 from kMinStackMallocSize to kMaxAsanStackMallocSizeClass
32350b57cec5SDimitry Andric static int StackMallocSizeClass(uint64_t LocalStackSize) {
32360b57cec5SDimitry Andric   assert(LocalStackSize <= kMaxStackMallocSize);
32370b57cec5SDimitry Andric   uint64_t MaxSize = kMinStackMallocSize;
32380b57cec5SDimitry Andric   for (int i = 0;; i++, MaxSize *= 2)
32390b57cec5SDimitry Andric     if (LocalStackSize <= MaxSize) return i;
32400b57cec5SDimitry Andric   llvm_unreachable("impossible LocalStackSize");
32410b57cec5SDimitry Andric }
32420b57cec5SDimitry Andric 
32430b57cec5SDimitry Andric void FunctionStackPoisoner::copyArgsPassedByValToAllocas() {
32440b57cec5SDimitry Andric   Instruction *CopyInsertPoint = &F.front().front();
32450b57cec5SDimitry Andric   if (CopyInsertPoint == ASan.LocalDynamicShadow) {
32460b57cec5SDimitry Andric     // Insert after the dynamic shadow location is determined
32470b57cec5SDimitry Andric     CopyInsertPoint = CopyInsertPoint->getNextNode();
32480b57cec5SDimitry Andric     assert(CopyInsertPoint);
32490b57cec5SDimitry Andric   }
32500b57cec5SDimitry Andric   IRBuilder<> IRB(CopyInsertPoint);
3251*0fca6ea1SDimitry Andric   const DataLayout &DL = F.getDataLayout();
32520b57cec5SDimitry Andric   for (Argument &Arg : F.args()) {
32530b57cec5SDimitry Andric     if (Arg.hasByValAttr()) {
32545ffd83dbSDimitry Andric       Type *Ty = Arg.getParamByValType();
3255480093f4SDimitry Andric       const Align Alignment =
3256480093f4SDimitry Andric           DL.getValueOrABITypeAlignment(Arg.getParamAlign(), Ty);
32570b57cec5SDimitry Andric 
32580b57cec5SDimitry Andric       AllocaInst *AI = IRB.CreateAlloca(
32590b57cec5SDimitry Andric           Ty, nullptr,
32600b57cec5SDimitry Andric           (Arg.hasName() ? Arg.getName() : "Arg" + Twine(Arg.getArgNo())) +
32610b57cec5SDimitry Andric               ".byval");
3262480093f4SDimitry Andric       AI->setAlignment(Alignment);
32630b57cec5SDimitry Andric       Arg.replaceAllUsesWith(AI);
32640b57cec5SDimitry Andric 
32650b57cec5SDimitry Andric       uint64_t AllocSize = DL.getTypeAllocSize(Ty);
32668bcb0991SDimitry Andric       IRB.CreateMemCpy(AI, Alignment, &Arg, Alignment, AllocSize);
32670b57cec5SDimitry Andric     }
32680b57cec5SDimitry Andric   }
32690b57cec5SDimitry Andric }
32700b57cec5SDimitry Andric 
32710b57cec5SDimitry Andric PHINode *FunctionStackPoisoner::createPHI(IRBuilder<> &IRB, Value *Cond,
32720b57cec5SDimitry Andric                                           Value *ValueIfTrue,
32730b57cec5SDimitry Andric                                           Instruction *ThenTerm,
32740b57cec5SDimitry Andric                                           Value *ValueIfFalse) {
32750b57cec5SDimitry Andric   PHINode *PHI = IRB.CreatePHI(IntptrTy, 2);
32760b57cec5SDimitry Andric   BasicBlock *CondBlock = cast<Instruction>(Cond)->getParent();
32770b57cec5SDimitry Andric   PHI->addIncoming(ValueIfFalse, CondBlock);
32780b57cec5SDimitry Andric   BasicBlock *ThenBlock = ThenTerm->getParent();
32790b57cec5SDimitry Andric   PHI->addIncoming(ValueIfTrue, ThenBlock);
32800b57cec5SDimitry Andric   return PHI;
32810b57cec5SDimitry Andric }
32820b57cec5SDimitry Andric 
32830b57cec5SDimitry Andric Value *FunctionStackPoisoner::createAllocaForLayout(
32840b57cec5SDimitry Andric     IRBuilder<> &IRB, const ASanStackFrameLayout &L, bool Dynamic) {
32850b57cec5SDimitry Andric   AllocaInst *Alloca;
32860b57cec5SDimitry Andric   if (Dynamic) {
32870b57cec5SDimitry Andric     Alloca = IRB.CreateAlloca(IRB.getInt8Ty(),
32880b57cec5SDimitry Andric                               ConstantInt::get(IRB.getInt64Ty(), L.FrameSize),
32890b57cec5SDimitry Andric                               "MyAlloca");
32900b57cec5SDimitry Andric   } else {
32910b57cec5SDimitry Andric     Alloca = IRB.CreateAlloca(ArrayType::get(IRB.getInt8Ty(), L.FrameSize),
32920b57cec5SDimitry Andric                               nullptr, "MyAlloca");
32930b57cec5SDimitry Andric     assert(Alloca->isStaticAlloca());
32940b57cec5SDimitry Andric   }
32950b57cec5SDimitry Andric   assert((ClRealignStack & (ClRealignStack - 1)) == 0);
3296349cc55cSDimitry Andric   uint64_t FrameAlignment = std::max(L.FrameAlignment, uint64_t(ClRealignStack));
32975ffd83dbSDimitry Andric   Alloca->setAlignment(Align(FrameAlignment));
32980b57cec5SDimitry Andric   return IRB.CreatePointerCast(Alloca, IntptrTy);
32990b57cec5SDimitry Andric }
33000b57cec5SDimitry Andric 
33010b57cec5SDimitry Andric void FunctionStackPoisoner::createDynamicAllocasInitStorage() {
33020b57cec5SDimitry Andric   BasicBlock &FirstBB = *F.begin();
33030b57cec5SDimitry Andric   IRBuilder<> IRB(dyn_cast<Instruction>(FirstBB.begin()));
33040b57cec5SDimitry Andric   DynamicAllocaLayout = IRB.CreateAlloca(IntptrTy, nullptr);
33050b57cec5SDimitry Andric   IRB.CreateStore(Constant::getNullValue(IntptrTy), DynamicAllocaLayout);
33068bcb0991SDimitry Andric   DynamicAllocaLayout->setAlignment(Align(32));
33070b57cec5SDimitry Andric }
33080b57cec5SDimitry Andric 
33090b57cec5SDimitry Andric void FunctionStackPoisoner::processDynamicAllocas() {
33100b57cec5SDimitry Andric   if (!ClInstrumentDynamicAllocas || DynamicAllocaVec.empty()) {
33110b57cec5SDimitry Andric     assert(DynamicAllocaPoisonCallVec.empty());
33120b57cec5SDimitry Andric     return;
33130b57cec5SDimitry Andric   }
33140b57cec5SDimitry Andric 
33150b57cec5SDimitry Andric   // Insert poison calls for lifetime intrinsics for dynamic allocas.
33160b57cec5SDimitry Andric   for (const auto &APC : DynamicAllocaPoisonCallVec) {
33170b57cec5SDimitry Andric     assert(APC.InsBefore);
33180b57cec5SDimitry Andric     assert(APC.AI);
33190b57cec5SDimitry Andric     assert(ASan.isInterestingAlloca(*APC.AI));
33200b57cec5SDimitry Andric     assert(!APC.AI->isStaticAlloca());
33210b57cec5SDimitry Andric 
33220b57cec5SDimitry Andric     IRBuilder<> IRB(APC.InsBefore);
33230b57cec5SDimitry Andric     poisonAlloca(APC.AI, APC.Size, IRB, APC.DoPoison);
33240b57cec5SDimitry Andric     // Dynamic allocas will be unpoisoned unconditionally below in
33250b57cec5SDimitry Andric     // unpoisonDynamicAllocas.
33260b57cec5SDimitry Andric     // Flag that we need unpoison static allocas.
33270b57cec5SDimitry Andric   }
33280b57cec5SDimitry Andric 
33290b57cec5SDimitry Andric   // Handle dynamic allocas.
33300b57cec5SDimitry Andric   createDynamicAllocasInitStorage();
33310b57cec5SDimitry Andric   for (auto &AI : DynamicAllocaVec)
33320b57cec5SDimitry Andric     handleDynamicAllocaCall(AI);
33330b57cec5SDimitry Andric   unpoisonDynamicAllocas();
33340b57cec5SDimitry Andric }
33350b57cec5SDimitry Andric 
33365ffd83dbSDimitry Andric /// Collect instructions in the entry block after \p InsBefore which initialize
33375ffd83dbSDimitry Andric /// permanent storage for a function argument. These instructions must remain in
33385ffd83dbSDimitry Andric /// the entry block so that uninitialized values do not appear in backtraces. An
33395ffd83dbSDimitry Andric /// added benefit is that this conserves spill slots. This does not move stores
33405ffd83dbSDimitry Andric /// before instrumented / "interesting" allocas.
33415ffd83dbSDimitry Andric static void findStoresToUninstrumentedArgAllocas(
33425ffd83dbSDimitry Andric     AddressSanitizer &ASan, Instruction &InsBefore,
33435ffd83dbSDimitry Andric     SmallVectorImpl<Instruction *> &InitInsts) {
33445ffd83dbSDimitry Andric   Instruction *Start = InsBefore.getNextNonDebugInstruction();
33455ffd83dbSDimitry Andric   for (Instruction *It = Start; It; It = It->getNextNonDebugInstruction()) {
33465ffd83dbSDimitry Andric     // Argument initialization looks like:
33475ffd83dbSDimitry Andric     // 1) store <Argument>, <Alloca> OR
33485ffd83dbSDimitry Andric     // 2) <CastArgument> = cast <Argument> to ...
33495ffd83dbSDimitry Andric     //    store <CastArgument> to <Alloca>
33505ffd83dbSDimitry Andric     // Do not consider any other kind of instruction.
33515ffd83dbSDimitry Andric     //
33525ffd83dbSDimitry Andric     // Note: This covers all known cases, but may not be exhaustive. An
33535ffd83dbSDimitry Andric     // alternative to pattern-matching stores is to DFS over all Argument uses:
33545ffd83dbSDimitry Andric     // this might be more general, but is probably much more complicated.
33555ffd83dbSDimitry Andric     if (isa<AllocaInst>(It) || isa<CastInst>(It))
33565ffd83dbSDimitry Andric       continue;
33575ffd83dbSDimitry Andric     if (auto *Store = dyn_cast<StoreInst>(It)) {
33585ffd83dbSDimitry Andric       // The store destination must be an alloca that isn't interesting for
33595ffd83dbSDimitry Andric       // ASan to instrument. These are moved up before InsBefore, and they're
33605ffd83dbSDimitry Andric       // not interesting because allocas for arguments can be mem2reg'd.
33615ffd83dbSDimitry Andric       auto *Alloca = dyn_cast<AllocaInst>(Store->getPointerOperand());
33625ffd83dbSDimitry Andric       if (!Alloca || ASan.isInterestingAlloca(*Alloca))
33635ffd83dbSDimitry Andric         continue;
33645ffd83dbSDimitry Andric 
33655ffd83dbSDimitry Andric       Value *Val = Store->getValueOperand();
33665ffd83dbSDimitry Andric       bool IsDirectArgInit = isa<Argument>(Val);
33675ffd83dbSDimitry Andric       bool IsArgInitViaCast =
33685ffd83dbSDimitry Andric           isa<CastInst>(Val) &&
33695ffd83dbSDimitry Andric           isa<Argument>(cast<CastInst>(Val)->getOperand(0)) &&
33705ffd83dbSDimitry Andric           // Check that the cast appears directly before the store. Otherwise
33715ffd83dbSDimitry Andric           // moving the cast before InsBefore may break the IR.
33725ffd83dbSDimitry Andric           Val == It->getPrevNonDebugInstruction();
33735ffd83dbSDimitry Andric       bool IsArgInit = IsDirectArgInit || IsArgInitViaCast;
33745ffd83dbSDimitry Andric       if (!IsArgInit)
33755ffd83dbSDimitry Andric         continue;
33765ffd83dbSDimitry Andric 
33775ffd83dbSDimitry Andric       if (IsArgInitViaCast)
33785ffd83dbSDimitry Andric         InitInsts.push_back(cast<Instruction>(Val));
33795ffd83dbSDimitry Andric       InitInsts.push_back(Store);
33805ffd83dbSDimitry Andric       continue;
33815ffd83dbSDimitry Andric     }
33825ffd83dbSDimitry Andric 
33835ffd83dbSDimitry Andric     // Do not reorder past unknown instructions: argument initialization should
33845ffd83dbSDimitry Andric     // only involve casts and stores.
33855ffd83dbSDimitry Andric     return;
33865ffd83dbSDimitry Andric   }
33875ffd83dbSDimitry Andric }
33885ffd83dbSDimitry Andric 
33890b57cec5SDimitry Andric void FunctionStackPoisoner::processStaticAllocas() {
33900b57cec5SDimitry Andric   if (AllocaVec.empty()) {
33910b57cec5SDimitry Andric     assert(StaticAllocaPoisonCallVec.empty());
33920b57cec5SDimitry Andric     return;
33930b57cec5SDimitry Andric   }
33940b57cec5SDimitry Andric 
33950b57cec5SDimitry Andric   int StackMallocIdx = -1;
33960b57cec5SDimitry Andric   DebugLoc EntryDebugLocation;
33970b57cec5SDimitry Andric   if (auto SP = F.getSubprogram())
3398e8d8bef9SDimitry Andric     EntryDebugLocation =
3399e8d8bef9SDimitry Andric         DILocation::get(SP->getContext(), SP->getScopeLine(), 0, SP);
34000b57cec5SDimitry Andric 
34010b57cec5SDimitry Andric   Instruction *InsBefore = AllocaVec[0];
34020b57cec5SDimitry Andric   IRBuilder<> IRB(InsBefore);
34030b57cec5SDimitry Andric 
34040b57cec5SDimitry Andric   // Make sure non-instrumented allocas stay in the entry block. Otherwise,
34050b57cec5SDimitry Andric   // debug info is broken, because only entry-block allocas are treated as
34060b57cec5SDimitry Andric   // regular stack slots.
34070b57cec5SDimitry Andric   auto InsBeforeB = InsBefore->getParent();
34080b57cec5SDimitry Andric   assert(InsBeforeB == &F.getEntryBlock());
34090b57cec5SDimitry Andric   for (auto *AI : StaticAllocasToMoveUp)
34100b57cec5SDimitry Andric     if (AI->getParent() == InsBeforeB)
34110b57cec5SDimitry Andric       AI->moveBefore(InsBefore);
34120b57cec5SDimitry Andric 
34135ffd83dbSDimitry Andric   // Move stores of arguments into entry-block allocas as well. This prevents
34145ffd83dbSDimitry Andric   // extra stack slots from being generated (to house the argument values until
34155ffd83dbSDimitry Andric   // they can be stored into the allocas). This also prevents uninitialized
34165ffd83dbSDimitry Andric   // values from being shown in backtraces.
34175ffd83dbSDimitry Andric   SmallVector<Instruction *, 8> ArgInitInsts;
34185ffd83dbSDimitry Andric   findStoresToUninstrumentedArgAllocas(ASan, *InsBefore, ArgInitInsts);
34195ffd83dbSDimitry Andric   for (Instruction *ArgInitInst : ArgInitInsts)
34205ffd83dbSDimitry Andric     ArgInitInst->moveBefore(InsBefore);
34215ffd83dbSDimitry Andric 
34220b57cec5SDimitry Andric   // If we have a call to llvm.localescape, keep it in the entry block.
34230b57cec5SDimitry Andric   if (LocalEscapeCall) LocalEscapeCall->moveBefore(InsBefore);
34240b57cec5SDimitry Andric 
34250b57cec5SDimitry Andric   SmallVector<ASanStackVariableDescription, 16> SVD;
34260b57cec5SDimitry Andric   SVD.reserve(AllocaVec.size());
34270b57cec5SDimitry Andric   for (AllocaInst *AI : AllocaVec) {
34280b57cec5SDimitry Andric     ASanStackVariableDescription D = {AI->getName().data(),
34290b57cec5SDimitry Andric                                       ASan.getAllocaSizeInBytes(*AI),
34300b57cec5SDimitry Andric                                       0,
343181ad6265SDimitry Andric                                       AI->getAlign().value(),
34320b57cec5SDimitry Andric                                       AI,
34330b57cec5SDimitry Andric                                       0,
34340b57cec5SDimitry Andric                                       0};
34350b57cec5SDimitry Andric     SVD.push_back(D);
34360b57cec5SDimitry Andric   }
34370b57cec5SDimitry Andric 
34380b57cec5SDimitry Andric   // Minimal header size (left redzone) is 4 pointers,
34390b57cec5SDimitry Andric   // i.e. 32 bytes on 64-bit platforms and 16 bytes in 32-bit platforms.
3440349cc55cSDimitry Andric   uint64_t Granularity = 1ULL << Mapping.Scale;
3441349cc55cSDimitry Andric   uint64_t MinHeaderSize = std::max((uint64_t)ASan.LongSize / 2, Granularity);
34420b57cec5SDimitry Andric   const ASanStackFrameLayout &L =
34430b57cec5SDimitry Andric       ComputeASanStackFrameLayout(SVD, Granularity, MinHeaderSize);
34440b57cec5SDimitry Andric 
34450b57cec5SDimitry Andric   // Build AllocaToSVDMap for ASanStackVariableDescription lookup.
34460b57cec5SDimitry Andric   DenseMap<const AllocaInst *, ASanStackVariableDescription *> AllocaToSVDMap;
34470b57cec5SDimitry Andric   for (auto &Desc : SVD)
34480b57cec5SDimitry Andric     AllocaToSVDMap[Desc.AI] = &Desc;
34490b57cec5SDimitry Andric 
34500b57cec5SDimitry Andric   // Update SVD with information from lifetime intrinsics.
34510b57cec5SDimitry Andric   for (const auto &APC : StaticAllocaPoisonCallVec) {
34520b57cec5SDimitry Andric     assert(APC.InsBefore);
34530b57cec5SDimitry Andric     assert(APC.AI);
34540b57cec5SDimitry Andric     assert(ASan.isInterestingAlloca(*APC.AI));
34550b57cec5SDimitry Andric     assert(APC.AI->isStaticAlloca());
34560b57cec5SDimitry Andric 
34570b57cec5SDimitry Andric     ASanStackVariableDescription &Desc = *AllocaToSVDMap[APC.AI];
34580b57cec5SDimitry Andric     Desc.LifetimeSize = Desc.Size;
34590b57cec5SDimitry Andric     if (const DILocation *FnLoc = EntryDebugLocation.get()) {
34600b57cec5SDimitry Andric       if (const DILocation *LifetimeLoc = APC.InsBefore->getDebugLoc().get()) {
34610b57cec5SDimitry Andric         if (LifetimeLoc->getFile() == FnLoc->getFile())
34620b57cec5SDimitry Andric           if (unsigned Line = LifetimeLoc->getLine())
34630b57cec5SDimitry Andric             Desc.Line = std::min(Desc.Line ? Desc.Line : Line, Line);
34640b57cec5SDimitry Andric       }
34650b57cec5SDimitry Andric     }
34660b57cec5SDimitry Andric   }
34670b57cec5SDimitry Andric 
34680b57cec5SDimitry Andric   auto DescriptionString = ComputeASanStackFrameDescription(SVD);
34690b57cec5SDimitry Andric   LLVM_DEBUG(dbgs() << DescriptionString << " --- " << L.FrameSize << "\n");
34700b57cec5SDimitry Andric   uint64_t LocalStackSize = L.FrameSize;
3471fe6060f1SDimitry Andric   bool DoStackMalloc =
3472fe6060f1SDimitry Andric       ASan.UseAfterReturn != AsanDetectStackUseAfterReturnMode::Never &&
3473fe6060f1SDimitry Andric       !ASan.CompileKernel && LocalStackSize <= kMaxStackMallocSize;
34740b57cec5SDimitry Andric   bool DoDynamicAlloca = ClDynamicAllocaStack;
34750b57cec5SDimitry Andric   // Don't do dynamic alloca or stack malloc if:
34760b57cec5SDimitry Andric   // 1) There is inline asm: too often it makes assumptions on which registers
34770b57cec5SDimitry Andric   //    are available.
34780b57cec5SDimitry Andric   // 2) There is a returns_twice call (typically setjmp), which is
34790b57cec5SDimitry Andric   //    optimization-hostile, and doesn't play well with introduced indirect
34800b57cec5SDimitry Andric   //    register-relative calculation of local variable addresses.
34815ffd83dbSDimitry Andric   DoDynamicAlloca &= !HasInlineAsm && !HasReturnsTwiceCall;
34825ffd83dbSDimitry Andric   DoStackMalloc &= !HasInlineAsm && !HasReturnsTwiceCall;
34830b57cec5SDimitry Andric 
34840b57cec5SDimitry Andric   Value *StaticAlloca =
34850b57cec5SDimitry Andric       DoDynamicAlloca ? nullptr : createAllocaForLayout(IRB, L, false);
34860b57cec5SDimitry Andric 
34870b57cec5SDimitry Andric   Value *FakeStack;
34880b57cec5SDimitry Andric   Value *LocalStackBase;
34890b57cec5SDimitry Andric   Value *LocalStackBaseAlloca;
34900b57cec5SDimitry Andric   uint8_t DIExprFlags = DIExpression::ApplyOffset;
34910b57cec5SDimitry Andric 
34920b57cec5SDimitry Andric   if (DoStackMalloc) {
34930b57cec5SDimitry Andric     LocalStackBaseAlloca =
34940b57cec5SDimitry Andric         IRB.CreateAlloca(IntptrTy, nullptr, "asan_local_stack_base");
3495fe6060f1SDimitry Andric     if (ASan.UseAfterReturn == AsanDetectStackUseAfterReturnMode::Runtime) {
34960b57cec5SDimitry Andric       // void *FakeStack = __asan_option_detect_stack_use_after_return
34970b57cec5SDimitry Andric       //     ? __asan_stack_malloc_N(LocalStackSize)
34980b57cec5SDimitry Andric       //     : nullptr;
3499fe6060f1SDimitry Andric       // void *LocalStackBase = (FakeStack) ? FakeStack :
3500fe6060f1SDimitry Andric       //                        alloca(LocalStackSize);
35010b57cec5SDimitry Andric       Constant *OptionDetectUseAfterReturn = F.getParent()->getOrInsertGlobal(
35020b57cec5SDimitry Andric           kAsanOptionDetectUseAfterReturn, IRB.getInt32Ty());
35030b57cec5SDimitry Andric       Value *UseAfterReturnIsEnabled = IRB.CreateICmpNE(
35040b57cec5SDimitry Andric           IRB.CreateLoad(IRB.getInt32Ty(), OptionDetectUseAfterReturn),
35050b57cec5SDimitry Andric           Constant::getNullValue(IRB.getInt32Ty()));
35060b57cec5SDimitry Andric       Instruction *Term =
35070b57cec5SDimitry Andric           SplitBlockAndInsertIfThen(UseAfterReturnIsEnabled, InsBefore, false);
35080b57cec5SDimitry Andric       IRBuilder<> IRBIf(Term);
35090b57cec5SDimitry Andric       StackMallocIdx = StackMallocSizeClass(LocalStackSize);
35100b57cec5SDimitry Andric       assert(StackMallocIdx <= kMaxAsanStackMallocSizeClass);
35110b57cec5SDimitry Andric       Value *FakeStackValue =
3512*0fca6ea1SDimitry Andric           RTCI.createRuntimeCall(IRBIf, AsanStackMallocFunc[StackMallocIdx],
35130b57cec5SDimitry Andric                                  ConstantInt::get(IntptrTy, LocalStackSize));
35140b57cec5SDimitry Andric       IRB.SetInsertPoint(InsBefore);
35150b57cec5SDimitry Andric       FakeStack = createPHI(IRB, UseAfterReturnIsEnabled, FakeStackValue, Term,
35160b57cec5SDimitry Andric                             ConstantInt::get(IntptrTy, 0));
3517fe6060f1SDimitry Andric     } else {
3518fe6060f1SDimitry Andric       // assert(ASan.UseAfterReturn == AsanDetectStackUseAfterReturnMode:Always)
3519fe6060f1SDimitry Andric       // void *FakeStack = __asan_stack_malloc_N(LocalStackSize);
3520fe6060f1SDimitry Andric       // void *LocalStackBase = (FakeStack) ? FakeStack :
3521fe6060f1SDimitry Andric       //                        alloca(LocalStackSize);
3522fe6060f1SDimitry Andric       StackMallocIdx = StackMallocSizeClass(LocalStackSize);
3523*0fca6ea1SDimitry Andric       FakeStack =
3524*0fca6ea1SDimitry Andric           RTCI.createRuntimeCall(IRB, AsanStackMallocFunc[StackMallocIdx],
3525fe6060f1SDimitry Andric                                  ConstantInt::get(IntptrTy, LocalStackSize));
3526fe6060f1SDimitry Andric     }
35270b57cec5SDimitry Andric     Value *NoFakeStack =
35280b57cec5SDimitry Andric         IRB.CreateICmpEQ(FakeStack, Constant::getNullValue(IntptrTy));
3529fe6060f1SDimitry Andric     Instruction *Term =
3530fe6060f1SDimitry Andric         SplitBlockAndInsertIfThen(NoFakeStack, InsBefore, false);
3531fe6060f1SDimitry Andric     IRBuilder<> IRBIf(Term);
35320b57cec5SDimitry Andric     Value *AllocaValue =
35330b57cec5SDimitry Andric         DoDynamicAlloca ? createAllocaForLayout(IRBIf, L, true) : StaticAlloca;
35340b57cec5SDimitry Andric 
35350b57cec5SDimitry Andric     IRB.SetInsertPoint(InsBefore);
35360b57cec5SDimitry Andric     LocalStackBase = createPHI(IRB, NoFakeStack, AllocaValue, Term, FakeStack);
35370b57cec5SDimitry Andric     IRB.CreateStore(LocalStackBase, LocalStackBaseAlloca);
35380b57cec5SDimitry Andric     DIExprFlags |= DIExpression::DerefBefore;
35390b57cec5SDimitry Andric   } else {
35400b57cec5SDimitry Andric     // void *FakeStack = nullptr;
35410b57cec5SDimitry Andric     // void *LocalStackBase = alloca(LocalStackSize);
35420b57cec5SDimitry Andric     FakeStack = ConstantInt::get(IntptrTy, 0);
35430b57cec5SDimitry Andric     LocalStackBase =
35440b57cec5SDimitry Andric         DoDynamicAlloca ? createAllocaForLayout(IRB, L, true) : StaticAlloca;
35450b57cec5SDimitry Andric     LocalStackBaseAlloca = LocalStackBase;
35460b57cec5SDimitry Andric   }
35470b57cec5SDimitry Andric 
35485ffd83dbSDimitry Andric   // It shouldn't matter whether we pass an `alloca` or a `ptrtoint` as the
35495ffd83dbSDimitry Andric   // dbg.declare address opereand, but passing a `ptrtoint` seems to confuse
35505ffd83dbSDimitry Andric   // later passes and can result in dropped variable coverage in debug info.
35515ffd83dbSDimitry Andric   Value *LocalStackBaseAllocaPtr =
35525ffd83dbSDimitry Andric       isa<PtrToIntInst>(LocalStackBaseAlloca)
35535ffd83dbSDimitry Andric           ? cast<PtrToIntInst>(LocalStackBaseAlloca)->getPointerOperand()
35545ffd83dbSDimitry Andric           : LocalStackBaseAlloca;
35555ffd83dbSDimitry Andric   assert(isa<AllocaInst>(LocalStackBaseAllocaPtr) &&
35565ffd83dbSDimitry Andric          "Variable descriptions relative to ASan stack base will be dropped");
35575ffd83dbSDimitry Andric 
35580b57cec5SDimitry Andric   // Replace Alloca instructions with base+offset.
35590b57cec5SDimitry Andric   for (const auto &Desc : SVD) {
35600b57cec5SDimitry Andric     AllocaInst *AI = Desc.AI;
35615ffd83dbSDimitry Andric     replaceDbgDeclare(AI, LocalStackBaseAllocaPtr, DIB, DIExprFlags,
35620b57cec5SDimitry Andric                       Desc.Offset);
35630b57cec5SDimitry Andric     Value *NewAllocaPtr = IRB.CreateIntToPtr(
35640b57cec5SDimitry Andric         IRB.CreateAdd(LocalStackBase, ConstantInt::get(IntptrTy, Desc.Offset)),
35650b57cec5SDimitry Andric         AI->getType());
35660b57cec5SDimitry Andric     AI->replaceAllUsesWith(NewAllocaPtr);
35670b57cec5SDimitry Andric   }
35680b57cec5SDimitry Andric 
35690b57cec5SDimitry Andric   // The left-most redzone has enough space for at least 4 pointers.
35700b57cec5SDimitry Andric   // Write the Magic value to redzone[0].
35710b57cec5SDimitry Andric   Value *BasePlus0 = IRB.CreateIntToPtr(LocalStackBase, IntptrPtrTy);
35720b57cec5SDimitry Andric   IRB.CreateStore(ConstantInt::get(IntptrTy, kCurrentStackFrameMagic),
35730b57cec5SDimitry Andric                   BasePlus0);
35740b57cec5SDimitry Andric   // Write the frame description constant to redzone[1].
35750b57cec5SDimitry Andric   Value *BasePlus1 = IRB.CreateIntToPtr(
35760b57cec5SDimitry Andric       IRB.CreateAdd(LocalStackBase,
35770b57cec5SDimitry Andric                     ConstantInt::get(IntptrTy, ASan.LongSize / 8)),
35780b57cec5SDimitry Andric       IntptrPtrTy);
35790b57cec5SDimitry Andric   GlobalVariable *StackDescriptionGlobal =
35800b57cec5SDimitry Andric       createPrivateGlobalForString(*F.getParent(), DescriptionString,
35810b57cec5SDimitry Andric                                    /*AllowMerging*/ true, kAsanGenPrefix);
35820b57cec5SDimitry Andric   Value *Description = IRB.CreatePointerCast(StackDescriptionGlobal, IntptrTy);
35830b57cec5SDimitry Andric   IRB.CreateStore(Description, BasePlus1);
35840b57cec5SDimitry Andric   // Write the PC to redzone[2].
35850b57cec5SDimitry Andric   Value *BasePlus2 = IRB.CreateIntToPtr(
35860b57cec5SDimitry Andric       IRB.CreateAdd(LocalStackBase,
35870b57cec5SDimitry Andric                     ConstantInt::get(IntptrTy, 2 * ASan.LongSize / 8)),
35880b57cec5SDimitry Andric       IntptrPtrTy);
35890b57cec5SDimitry Andric   IRB.CreateStore(IRB.CreatePointerCast(&F, IntptrTy), BasePlus2);
35900b57cec5SDimitry Andric 
35910b57cec5SDimitry Andric   const auto &ShadowAfterScope = GetShadowBytesAfterScope(SVD, L);
35920b57cec5SDimitry Andric 
35930b57cec5SDimitry Andric   // Poison the stack red zones at the entry.
35940b57cec5SDimitry Andric   Value *ShadowBase = ASan.memToShadow(LocalStackBase, IRB);
35950b57cec5SDimitry Andric   // As mask we must use most poisoned case: red zones and after scope.
35960b57cec5SDimitry Andric   // As bytes we can use either the same or just red zones only.
35970b57cec5SDimitry Andric   copyToShadow(ShadowAfterScope, ShadowAfterScope, IRB, ShadowBase);
35980b57cec5SDimitry Andric 
35990b57cec5SDimitry Andric   if (!StaticAllocaPoisonCallVec.empty()) {
36000b57cec5SDimitry Andric     const auto &ShadowInScope = GetShadowBytes(SVD, L);
36010b57cec5SDimitry Andric 
36020b57cec5SDimitry Andric     // Poison static allocas near lifetime intrinsics.
36030b57cec5SDimitry Andric     for (const auto &APC : StaticAllocaPoisonCallVec) {
36040b57cec5SDimitry Andric       const ASanStackVariableDescription &Desc = *AllocaToSVDMap[APC.AI];
36050b57cec5SDimitry Andric       assert(Desc.Offset % L.Granularity == 0);
36060b57cec5SDimitry Andric       size_t Begin = Desc.Offset / L.Granularity;
36070b57cec5SDimitry Andric       size_t End = Begin + (APC.Size + L.Granularity - 1) / L.Granularity;
36080b57cec5SDimitry Andric 
36090b57cec5SDimitry Andric       IRBuilder<> IRB(APC.InsBefore);
36100b57cec5SDimitry Andric       copyToShadow(ShadowAfterScope,
36110b57cec5SDimitry Andric                    APC.DoPoison ? ShadowAfterScope : ShadowInScope, Begin, End,
36120b57cec5SDimitry Andric                    IRB, ShadowBase);
36130b57cec5SDimitry Andric     }
36140b57cec5SDimitry Andric   }
36150b57cec5SDimitry Andric 
36160b57cec5SDimitry Andric   SmallVector<uint8_t, 64> ShadowClean(ShadowAfterScope.size(), 0);
36170b57cec5SDimitry Andric   SmallVector<uint8_t, 64> ShadowAfterReturn;
36180b57cec5SDimitry Andric 
36190b57cec5SDimitry Andric   // (Un)poison the stack before all ret instructions.
3620e8d8bef9SDimitry Andric   for (Instruction *Ret : RetVec) {
36210b57cec5SDimitry Andric     IRBuilder<> IRBRet(Ret);
36220b57cec5SDimitry Andric     // Mark the current frame as retired.
36230b57cec5SDimitry Andric     IRBRet.CreateStore(ConstantInt::get(IntptrTy, kRetiredStackFrameMagic),
36240b57cec5SDimitry Andric                        BasePlus0);
36250b57cec5SDimitry Andric     if (DoStackMalloc) {
36260b57cec5SDimitry Andric       assert(StackMallocIdx >= 0);
36270b57cec5SDimitry Andric       // if FakeStack != 0  // LocalStackBase == FakeStack
36280b57cec5SDimitry Andric       //     // In use-after-return mode, poison the whole stack frame.
36290b57cec5SDimitry Andric       //     if StackMallocIdx <= 4
36300b57cec5SDimitry Andric       //         // For small sizes inline the whole thing:
36310b57cec5SDimitry Andric       //         memset(ShadowBase, kAsanStackAfterReturnMagic, ShadowSize);
36320b57cec5SDimitry Andric       //         **SavedFlagPtr(FakeStack) = 0
36330b57cec5SDimitry Andric       //     else
36340b57cec5SDimitry Andric       //         __asan_stack_free_N(FakeStack, LocalStackSize)
36350b57cec5SDimitry Andric       // else
36360b57cec5SDimitry Andric       //     <This is not a fake stack; unpoison the redzones>
36370b57cec5SDimitry Andric       Value *Cmp =
36380b57cec5SDimitry Andric           IRBRet.CreateICmpNE(FakeStack, Constant::getNullValue(IntptrTy));
36390b57cec5SDimitry Andric       Instruction *ThenTerm, *ElseTerm;
36400b57cec5SDimitry Andric       SplitBlockAndInsertIfThenElse(Cmp, Ret, &ThenTerm, &ElseTerm);
36410b57cec5SDimitry Andric 
36420b57cec5SDimitry Andric       IRBuilder<> IRBPoison(ThenTerm);
36435f757f3fSDimitry Andric       if (ASan.MaxInlinePoisoningSize != 0 && StackMallocIdx <= 4) {
36440b57cec5SDimitry Andric         int ClassSize = kMinStackMallocSize << StackMallocIdx;
36450b57cec5SDimitry Andric         ShadowAfterReturn.resize(ClassSize / L.Granularity,
36460b57cec5SDimitry Andric                                  kAsanStackUseAfterReturnMagic);
36470b57cec5SDimitry Andric         copyToShadow(ShadowAfterReturn, ShadowAfterReturn, IRBPoison,
36480b57cec5SDimitry Andric                      ShadowBase);
36490b57cec5SDimitry Andric         Value *SavedFlagPtrPtr = IRBPoison.CreateAdd(
36500b57cec5SDimitry Andric             FakeStack,
36510b57cec5SDimitry Andric             ConstantInt::get(IntptrTy, ClassSize - ASan.LongSize / 8));
36520b57cec5SDimitry Andric         Value *SavedFlagPtr = IRBPoison.CreateLoad(
36530b57cec5SDimitry Andric             IntptrTy, IRBPoison.CreateIntToPtr(SavedFlagPtrPtr, IntptrPtrTy));
36540b57cec5SDimitry Andric         IRBPoison.CreateStore(
36550b57cec5SDimitry Andric             Constant::getNullValue(IRBPoison.getInt8Ty()),
36565f757f3fSDimitry Andric             IRBPoison.CreateIntToPtr(SavedFlagPtr, IRBPoison.getPtrTy()));
36570b57cec5SDimitry Andric       } else {
36580b57cec5SDimitry Andric         // For larger frames call __asan_stack_free_*.
3659*0fca6ea1SDimitry Andric         RTCI.createRuntimeCall(
3660*0fca6ea1SDimitry Andric             IRBPoison, AsanStackFreeFunc[StackMallocIdx],
36610b57cec5SDimitry Andric             {FakeStack, ConstantInt::get(IntptrTy, LocalStackSize)});
36620b57cec5SDimitry Andric       }
36630b57cec5SDimitry Andric 
36640b57cec5SDimitry Andric       IRBuilder<> IRBElse(ElseTerm);
36650b57cec5SDimitry Andric       copyToShadow(ShadowAfterScope, ShadowClean, IRBElse, ShadowBase);
36660b57cec5SDimitry Andric     } else {
36670b57cec5SDimitry Andric       copyToShadow(ShadowAfterScope, ShadowClean, IRBRet, ShadowBase);
36680b57cec5SDimitry Andric     }
36690b57cec5SDimitry Andric   }
36700b57cec5SDimitry Andric 
36710b57cec5SDimitry Andric   // We are done. Remove the old unused alloca instructions.
3672bdd1243dSDimitry Andric   for (auto *AI : AllocaVec)
3673bdd1243dSDimitry Andric     AI->eraseFromParent();
36740b57cec5SDimitry Andric }
36750b57cec5SDimitry Andric 
36760b57cec5SDimitry Andric void FunctionStackPoisoner::poisonAlloca(Value *V, uint64_t Size,
36770b57cec5SDimitry Andric                                          IRBuilder<> &IRB, bool DoPoison) {
36780b57cec5SDimitry Andric   // For now just insert the call to ASan runtime.
36790b57cec5SDimitry Andric   Value *AddrArg = IRB.CreatePointerCast(V, IntptrTy);
36800b57cec5SDimitry Andric   Value *SizeArg = ConstantInt::get(IntptrTy, Size);
3681*0fca6ea1SDimitry Andric   RTCI.createRuntimeCall(
3682*0fca6ea1SDimitry Andric       IRB, DoPoison ? AsanPoisonStackMemoryFunc : AsanUnpoisonStackMemoryFunc,
36830b57cec5SDimitry Andric       {AddrArg, SizeArg});
36840b57cec5SDimitry Andric }
36850b57cec5SDimitry Andric 
36860b57cec5SDimitry Andric // Handling llvm.lifetime intrinsics for a given %alloca:
36870b57cec5SDimitry Andric // (1) collect all llvm.lifetime.xxx(%size, %value) describing the alloca.
36880b57cec5SDimitry Andric // (2) if %size is constant, poison memory for llvm.lifetime.end (to detect
36890b57cec5SDimitry Andric //     invalid accesses) and unpoison it for llvm.lifetime.start (the memory
36900b57cec5SDimitry Andric //     could be poisoned by previous llvm.lifetime.end instruction, as the
36910b57cec5SDimitry Andric //     variable may go in and out of scope several times, e.g. in loops).
36920b57cec5SDimitry Andric // (3) if we poisoned at least one %alloca in a function,
36930b57cec5SDimitry Andric //     unpoison the whole stack frame at function exit.
36940b57cec5SDimitry Andric void FunctionStackPoisoner::handleDynamicAllocaCall(AllocaInst *AI) {
36950b57cec5SDimitry Andric   IRBuilder<> IRB(AI);
36960b57cec5SDimitry Andric 
369781ad6265SDimitry Andric   const Align Alignment = std::max(Align(kAllocaRzSize), AI->getAlign());
36980b57cec5SDimitry Andric   const uint64_t AllocaRedzoneMask = kAllocaRzSize - 1;
36990b57cec5SDimitry Andric 
37000b57cec5SDimitry Andric   Value *Zero = Constant::getNullValue(IntptrTy);
37010b57cec5SDimitry Andric   Value *AllocaRzSize = ConstantInt::get(IntptrTy, kAllocaRzSize);
37020b57cec5SDimitry Andric   Value *AllocaRzMask = ConstantInt::get(IntptrTy, AllocaRedzoneMask);
37030b57cec5SDimitry Andric 
37040b57cec5SDimitry Andric   // Since we need to extend alloca with additional memory to locate
37050b57cec5SDimitry Andric   // redzones, and OldSize is number of allocated blocks with
37060b57cec5SDimitry Andric   // ElementSize size, get allocated memory size in bytes by
37070b57cec5SDimitry Andric   // OldSize * ElementSize.
37080b57cec5SDimitry Andric   const unsigned ElementSize =
3709*0fca6ea1SDimitry Andric       F.getDataLayout().getTypeAllocSize(AI->getAllocatedType());
37100b57cec5SDimitry Andric   Value *OldSize =
37110b57cec5SDimitry Andric       IRB.CreateMul(IRB.CreateIntCast(AI->getArraySize(), IntptrTy, false),
37120b57cec5SDimitry Andric                     ConstantInt::get(IntptrTy, ElementSize));
37130b57cec5SDimitry Andric 
37140b57cec5SDimitry Andric   // PartialSize = OldSize % 32
37150b57cec5SDimitry Andric   Value *PartialSize = IRB.CreateAnd(OldSize, AllocaRzMask);
37160b57cec5SDimitry Andric 
37170b57cec5SDimitry Andric   // Misalign = kAllocaRzSize - PartialSize;
37180b57cec5SDimitry Andric   Value *Misalign = IRB.CreateSub(AllocaRzSize, PartialSize);
37190b57cec5SDimitry Andric 
37200b57cec5SDimitry Andric   // PartialPadding = Misalign != kAllocaRzSize ? Misalign : 0;
37210b57cec5SDimitry Andric   Value *Cond = IRB.CreateICmpNE(Misalign, AllocaRzSize);
37220b57cec5SDimitry Andric   Value *PartialPadding = IRB.CreateSelect(Cond, Misalign, Zero);
37230b57cec5SDimitry Andric 
37245ffd83dbSDimitry Andric   // AdditionalChunkSize = Alignment + PartialPadding + kAllocaRzSize
37255ffd83dbSDimitry Andric   // Alignment is added to locate left redzone, PartialPadding for possible
37260b57cec5SDimitry Andric   // partial redzone and kAllocaRzSize for right redzone respectively.
37270b57cec5SDimitry Andric   Value *AdditionalChunkSize = IRB.CreateAdd(
372881ad6265SDimitry Andric       ConstantInt::get(IntptrTy, Alignment.value() + kAllocaRzSize),
372981ad6265SDimitry Andric       PartialPadding);
37300b57cec5SDimitry Andric 
37310b57cec5SDimitry Andric   Value *NewSize = IRB.CreateAdd(OldSize, AdditionalChunkSize);
37320b57cec5SDimitry Andric 
37335ffd83dbSDimitry Andric   // Insert new alloca with new NewSize and Alignment params.
37340b57cec5SDimitry Andric   AllocaInst *NewAlloca = IRB.CreateAlloca(IRB.getInt8Ty(), NewSize);
373581ad6265SDimitry Andric   NewAlloca->setAlignment(Alignment);
37360b57cec5SDimitry Andric 
37375ffd83dbSDimitry Andric   // NewAddress = Address + Alignment
373881ad6265SDimitry Andric   Value *NewAddress =
373981ad6265SDimitry Andric       IRB.CreateAdd(IRB.CreatePtrToInt(NewAlloca, IntptrTy),
374081ad6265SDimitry Andric                     ConstantInt::get(IntptrTy, Alignment.value()));
37410b57cec5SDimitry Andric 
37420b57cec5SDimitry Andric   // Insert __asan_alloca_poison call for new created alloca.
3743*0fca6ea1SDimitry Andric   RTCI.createRuntimeCall(IRB, AsanAllocaPoisonFunc, {NewAddress, OldSize});
37440b57cec5SDimitry Andric 
37450b57cec5SDimitry Andric   // Store the last alloca's address to DynamicAllocaLayout. We'll need this
37460b57cec5SDimitry Andric   // for unpoisoning stuff.
37470b57cec5SDimitry Andric   IRB.CreateStore(IRB.CreatePtrToInt(NewAlloca, IntptrTy), DynamicAllocaLayout);
37480b57cec5SDimitry Andric 
37490b57cec5SDimitry Andric   Value *NewAddressPtr = IRB.CreateIntToPtr(NewAddress, AI->getType());
37500b57cec5SDimitry Andric 
37510b57cec5SDimitry Andric   // Replace all uses of AddessReturnedByAlloca with NewAddressPtr.
37520b57cec5SDimitry Andric   AI->replaceAllUsesWith(NewAddressPtr);
37530b57cec5SDimitry Andric 
37540b57cec5SDimitry Andric   // We are done. Erase old alloca from parent.
37550b57cec5SDimitry Andric   AI->eraseFromParent();
37560b57cec5SDimitry Andric }
37570b57cec5SDimitry Andric 
37580b57cec5SDimitry Andric // isSafeAccess returns true if Addr is always inbounds with respect to its
37590b57cec5SDimitry Andric // base object. For example, it is a field access or an array access with
37600b57cec5SDimitry Andric // constant inbounds index.
37610b57cec5SDimitry Andric bool AddressSanitizer::isSafeAccess(ObjectSizeOffsetVisitor &ObjSizeVis,
376206c3fb27SDimitry Andric                                     Value *Addr, TypeSize TypeStoreSize) const {
376306c3fb27SDimitry Andric   if (TypeStoreSize.isScalable())
376406c3fb27SDimitry Andric     // TODO: We can use vscale_range to convert a scalable value to an
376506c3fb27SDimitry Andric     // upper bound on the access size.
376606c3fb27SDimitry Andric     return false;
37671db9f3b2SDimitry Andric 
37681db9f3b2SDimitry Andric   SizeOffsetAPInt SizeOffset = ObjSizeVis.compute(Addr);
37691db9f3b2SDimitry Andric   if (!SizeOffset.bothKnown())
37701db9f3b2SDimitry Andric     return false;
37711db9f3b2SDimitry Andric 
37721db9f3b2SDimitry Andric   uint64_t Size = SizeOffset.Size.getZExtValue();
37731db9f3b2SDimitry Andric   int64_t Offset = SizeOffset.Offset.getSExtValue();
37741db9f3b2SDimitry Andric 
37750b57cec5SDimitry Andric   // Three checks are required to ensure safety:
37760b57cec5SDimitry Andric   // . Offset >= 0  (since the offset is given from the base ptr)
37770b57cec5SDimitry Andric   // . Size >= Offset  (unsigned)
37780b57cec5SDimitry Andric   // . Size - Offset >= NeededSize  (unsigned)
37790b57cec5SDimitry Andric   return Offset >= 0 && Size >= uint64_t(Offset) &&
378006c3fb27SDimitry Andric          Size - uint64_t(Offset) >= TypeStoreSize / 8;
37810b57cec5SDimitry Andric }
3782