117106792SAlexander Shaposhnikov //===-- NumericalStabilitySanitizer.cpp -----------------------------------===// 217106792SAlexander Shaposhnikov // 317106792SAlexander Shaposhnikov // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 417106792SAlexander Shaposhnikov // See https://llvm.org/LICENSE.txt for license information. 517106792SAlexander Shaposhnikov // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 617106792SAlexander Shaposhnikov // 717106792SAlexander Shaposhnikov //===----------------------------------------------------------------------===// 817106792SAlexander Shaposhnikov // 917106792SAlexander Shaposhnikov // This file contains the instrumentation pass for the numerical sanitizer. 1017106792SAlexander Shaposhnikov // Conceptually the pass injects shadow computations using higher precision 1117106792SAlexander Shaposhnikov // types and inserts consistency checks. For details see the paper 1217106792SAlexander Shaposhnikov // https://arxiv.org/abs/2102.12782. 1317106792SAlexander Shaposhnikov // 1417106792SAlexander Shaposhnikov //===----------------------------------------------------------------------===// 1517106792SAlexander Shaposhnikov 1617106792SAlexander Shaposhnikov #include "llvm/Transforms/Instrumentation/NumericalStabilitySanitizer.h" 1717106792SAlexander Shaposhnikov 1817106792SAlexander Shaposhnikov #include "llvm/ADT/DenseMap.h" 1917106792SAlexander Shaposhnikov #include "llvm/ADT/SmallVector.h" 2017106792SAlexander Shaposhnikov #include "llvm/ADT/Statistic.h" 2117106792SAlexander Shaposhnikov #include "llvm/ADT/StringExtras.h" 2217106792SAlexander Shaposhnikov #include "llvm/Analysis/TargetLibraryInfo.h" 2317106792SAlexander Shaposhnikov #include "llvm/Analysis/ValueTracking.h" 2417106792SAlexander Shaposhnikov #include "llvm/IR/DataLayout.h" 2517106792SAlexander Shaposhnikov #include "llvm/IR/Function.h" 2617106792SAlexander Shaposhnikov #include "llvm/IR/IRBuilder.h" 2717106792SAlexander Shaposhnikov #include "llvm/IR/IntrinsicInst.h" 2817106792SAlexander Shaposhnikov #include "llvm/IR/Intrinsics.h" 2917106792SAlexander Shaposhnikov #include "llvm/IR/LLVMContext.h" 3017106792SAlexander Shaposhnikov #include "llvm/IR/MDBuilder.h" 3117106792SAlexander Shaposhnikov #include "llvm/IR/Metadata.h" 3217106792SAlexander Shaposhnikov #include "llvm/IR/Module.h" 3317106792SAlexander Shaposhnikov #include "llvm/IR/Type.h" 3417106792SAlexander Shaposhnikov #include "llvm/Support/CommandLine.h" 3517106792SAlexander Shaposhnikov #include "llvm/Support/Debug.h" 3617106792SAlexander Shaposhnikov #include "llvm/Support/Regex.h" 3717106792SAlexander Shaposhnikov #include "llvm/Support/raw_ostream.h" 3817106792SAlexander Shaposhnikov #include "llvm/Transforms/Utils/BasicBlockUtils.h" 392ae968a0SAntonio Frighetto #include "llvm/Transforms/Utils/Instrumentation.h" 4017106792SAlexander Shaposhnikov #include "llvm/Transforms/Utils/Local.h" 4117106792SAlexander Shaposhnikov #include "llvm/Transforms/Utils/ModuleUtils.h" 4217106792SAlexander Shaposhnikov 4317106792SAlexander Shaposhnikov #include <cstdint> 4417106792SAlexander Shaposhnikov 4517106792SAlexander Shaposhnikov using namespace llvm; 4617106792SAlexander Shaposhnikov 4717106792SAlexander Shaposhnikov #define DEBUG_TYPE "nsan" 4817106792SAlexander Shaposhnikov 4917106792SAlexander Shaposhnikov STATISTIC(NumInstrumentedFTLoads, 5017106792SAlexander Shaposhnikov "Number of instrumented floating-point loads"); 5117106792SAlexander Shaposhnikov 5217106792SAlexander Shaposhnikov STATISTIC(NumInstrumentedFTCalls, 5317106792SAlexander Shaposhnikov "Number of instrumented floating-point calls"); 5417106792SAlexander Shaposhnikov STATISTIC(NumInstrumentedFTRets, 5517106792SAlexander Shaposhnikov "Number of instrumented floating-point returns"); 5617106792SAlexander Shaposhnikov STATISTIC(NumInstrumentedFTStores, 5717106792SAlexander Shaposhnikov "Number of instrumented floating-point stores"); 5817106792SAlexander Shaposhnikov STATISTIC(NumInstrumentedNonFTStores, 5917106792SAlexander Shaposhnikov "Number of instrumented non floating-point stores"); 6017106792SAlexander Shaposhnikov STATISTIC( 6117106792SAlexander Shaposhnikov NumInstrumentedNonFTMemcpyStores, 6217106792SAlexander Shaposhnikov "Number of instrumented non floating-point stores with memcpy semantics"); 6317106792SAlexander Shaposhnikov STATISTIC(NumInstrumentedFCmp, "Number of instrumented fcmps"); 6417106792SAlexander Shaposhnikov 6517106792SAlexander Shaposhnikov // Using smaller shadow types types can help improve speed. For example, `dlq` 6617106792SAlexander Shaposhnikov // is 3x slower to 5x faster in opt mode and 2-6x faster in dbg mode compared to 6717106792SAlexander Shaposhnikov // `dqq`. 6817106792SAlexander Shaposhnikov static cl::opt<std::string> ClShadowMapping( 6917106792SAlexander Shaposhnikov "nsan-shadow-type-mapping", cl::init("dqq"), 7017106792SAlexander Shaposhnikov cl::desc("One shadow type id for each of `float`, `double`, `long double`. " 7117106792SAlexander Shaposhnikov "`d`,`l`,`q`,`e` mean double, x86_fp80, fp128 (quad) and " 7217106792SAlexander Shaposhnikov "ppc_fp128 (extended double) respectively. The default is to " 7317106792SAlexander Shaposhnikov "shadow `float` as `double`, and `double` and `x86_fp80` as " 7417106792SAlexander Shaposhnikov "`fp128`"), 7517106792SAlexander Shaposhnikov cl::Hidden); 7617106792SAlexander Shaposhnikov 7717106792SAlexander Shaposhnikov static cl::opt<bool> 7817106792SAlexander Shaposhnikov ClInstrumentFCmp("nsan-instrument-fcmp", cl::init(true), 7917106792SAlexander Shaposhnikov cl::desc("Instrument floating-point comparisons"), 8017106792SAlexander Shaposhnikov cl::Hidden); 8117106792SAlexander Shaposhnikov 8217106792SAlexander Shaposhnikov static cl::opt<std::string> ClCheckFunctionsFilter( 8317106792SAlexander Shaposhnikov "check-functions-filter", 8417106792SAlexander Shaposhnikov cl::desc("Only emit checks for arguments of functions " 8517106792SAlexander Shaposhnikov "whose names match the given regular expression"), 8617106792SAlexander Shaposhnikov cl::value_desc("regex")); 8717106792SAlexander Shaposhnikov 8817106792SAlexander Shaposhnikov static cl::opt<bool> ClTruncateFCmpEq( 8917106792SAlexander Shaposhnikov "nsan-truncate-fcmp-eq", cl::init(true), 9017106792SAlexander Shaposhnikov cl::desc( 9117106792SAlexander Shaposhnikov "This flag controls the behaviour of fcmp equality comparisons." 9217106792SAlexander Shaposhnikov "For equality comparisons such as `x == 0.0f`, we can perform the " 9317106792SAlexander Shaposhnikov "shadow check in the shadow (`x_shadow == 0.0) == (x == 0.0f)`) or app " 9417106792SAlexander Shaposhnikov " domain (`(trunc(x_shadow) == 0.0f) == (x == 0.0f)`). This helps " 9517106792SAlexander Shaposhnikov "catch the case when `x_shadow` is accurate enough (and therefore " 9617106792SAlexander Shaposhnikov "close enough to zero) so that `trunc(x_shadow)` is zero even though " 9717106792SAlexander Shaposhnikov "both `x` and `x_shadow` are not"), 9817106792SAlexander Shaposhnikov cl::Hidden); 9917106792SAlexander Shaposhnikov 10017106792SAlexander Shaposhnikov // When there is external, uninstrumented code writing to memory, the shadow 10117106792SAlexander Shaposhnikov // memory can get out of sync with the application memory. Enabling this flag 10217106792SAlexander Shaposhnikov // emits consistency checks for loads to catch this situation. 10317106792SAlexander Shaposhnikov // When everything is instrumented, this is not strictly necessary because any 10417106792SAlexander Shaposhnikov // load should have a corresponding store, but can help debug cases when the 10517106792SAlexander Shaposhnikov // framework did a bad job at tracking shadow memory modifications by failing on 10617106792SAlexander Shaposhnikov // load rather than store. 10717106792SAlexander Shaposhnikov // TODO: provide a way to resume computations from the FT value when the load 10817106792SAlexander Shaposhnikov // is inconsistent. This ensures that further computations are not polluted. 10917106792SAlexander Shaposhnikov static cl::opt<bool> ClCheckLoads("nsan-check-loads", 11017106792SAlexander Shaposhnikov cl::desc("Check floating-point load"), 11117106792SAlexander Shaposhnikov cl::Hidden); 11217106792SAlexander Shaposhnikov 11317106792SAlexander Shaposhnikov static cl::opt<bool> ClCheckStores("nsan-check-stores", cl::init(true), 11417106792SAlexander Shaposhnikov cl::desc("Check floating-point stores"), 11517106792SAlexander Shaposhnikov cl::Hidden); 11617106792SAlexander Shaposhnikov 11717106792SAlexander Shaposhnikov static cl::opt<bool> ClCheckRet("nsan-check-ret", cl::init(true), 11817106792SAlexander Shaposhnikov cl::desc("Check floating-point return values"), 11917106792SAlexander Shaposhnikov cl::Hidden); 12017106792SAlexander Shaposhnikov 12117106792SAlexander Shaposhnikov // LLVM may store constant floats as bitcasted ints. 12217106792SAlexander Shaposhnikov // It's not really necessary to shadow such stores, 12317106792SAlexander Shaposhnikov // if the shadow value is unknown the framework will re-extend it on load 12417106792SAlexander Shaposhnikov // anyway. Moreover, because of size collisions (e.g. bf16 vs f16) it is 12517106792SAlexander Shaposhnikov // impossible to determine the floating-point type based on the size. 12617106792SAlexander Shaposhnikov // However, for debugging purposes it can be useful to model such stores. 12717106792SAlexander Shaposhnikov static cl::opt<bool> ClPropagateNonFTConstStoresAsFT( 12817106792SAlexander Shaposhnikov "nsan-propagate-non-ft-const-stores-as-ft", 12917106792SAlexander Shaposhnikov cl::desc( 13017106792SAlexander Shaposhnikov "Propagate non floating-point const stores as floating point values." 13117106792SAlexander Shaposhnikov "For debugging purposes only"), 13217106792SAlexander Shaposhnikov cl::Hidden); 13317106792SAlexander Shaposhnikov 13417106792SAlexander Shaposhnikov constexpr StringLiteral kNsanModuleCtorName("nsan.module_ctor"); 13517106792SAlexander Shaposhnikov constexpr StringLiteral kNsanInitName("__nsan_init"); 13617106792SAlexander Shaposhnikov 13717106792SAlexander Shaposhnikov // The following values must be kept in sync with the runtime. 13817106792SAlexander Shaposhnikov constexpr int kShadowScale = 2; 13917106792SAlexander Shaposhnikov constexpr int kMaxVectorWidth = 8; 14017106792SAlexander Shaposhnikov constexpr int kMaxNumArgs = 128; 14117106792SAlexander Shaposhnikov constexpr int kMaxShadowTypeSizeBytes = 16; // fp128 14217106792SAlexander Shaposhnikov 14317106792SAlexander Shaposhnikov namespace { 14417106792SAlexander Shaposhnikov 14517106792SAlexander Shaposhnikov // Defines the characteristics (type id, type, and floating-point semantics) 14617106792SAlexander Shaposhnikov // attached for all possible shadow types. 14717106792SAlexander Shaposhnikov class ShadowTypeConfig { 14817106792SAlexander Shaposhnikov public: 14917106792SAlexander Shaposhnikov static std::unique_ptr<ShadowTypeConfig> fromNsanTypeId(char TypeId); 15017106792SAlexander Shaposhnikov 15117106792SAlexander Shaposhnikov // The LLVM Type corresponding to the shadow type. 15217106792SAlexander Shaposhnikov virtual Type *getType(LLVMContext &Context) const = 0; 15317106792SAlexander Shaposhnikov 15417106792SAlexander Shaposhnikov // The nsan type id of the shadow type (`d`, `l`, `q`, ...). 15517106792SAlexander Shaposhnikov virtual char getNsanTypeId() const = 0; 15617106792SAlexander Shaposhnikov 15717106792SAlexander Shaposhnikov virtual ~ShadowTypeConfig() = default; 15817106792SAlexander Shaposhnikov }; 15917106792SAlexander Shaposhnikov 16017106792SAlexander Shaposhnikov template <char NsanTypeId> 16117106792SAlexander Shaposhnikov class ShadowTypeConfigImpl : public ShadowTypeConfig { 16217106792SAlexander Shaposhnikov public: 16317106792SAlexander Shaposhnikov char getNsanTypeId() const override { return NsanTypeId; } 16417106792SAlexander Shaposhnikov static constexpr const char kNsanTypeId = NsanTypeId; 16517106792SAlexander Shaposhnikov }; 16617106792SAlexander Shaposhnikov 16717106792SAlexander Shaposhnikov // `double` (`d`) shadow type. 16817106792SAlexander Shaposhnikov class F64ShadowConfig : public ShadowTypeConfigImpl<'d'> { 16917106792SAlexander Shaposhnikov Type *getType(LLVMContext &Context) const override { 17017106792SAlexander Shaposhnikov return Type::getDoubleTy(Context); 17117106792SAlexander Shaposhnikov } 17217106792SAlexander Shaposhnikov }; 17317106792SAlexander Shaposhnikov 17417106792SAlexander Shaposhnikov // `x86_fp80` (`l`) shadow type: X86 long double. 17517106792SAlexander Shaposhnikov class F80ShadowConfig : public ShadowTypeConfigImpl<'l'> { 17617106792SAlexander Shaposhnikov Type *getType(LLVMContext &Context) const override { 17717106792SAlexander Shaposhnikov return Type::getX86_FP80Ty(Context); 17817106792SAlexander Shaposhnikov } 17917106792SAlexander Shaposhnikov }; 18017106792SAlexander Shaposhnikov 18117106792SAlexander Shaposhnikov // `fp128` (`q`) shadow type. 18217106792SAlexander Shaposhnikov class F128ShadowConfig : public ShadowTypeConfigImpl<'q'> { 18317106792SAlexander Shaposhnikov Type *getType(LLVMContext &Context) const override { 18417106792SAlexander Shaposhnikov return Type::getFP128Ty(Context); 18517106792SAlexander Shaposhnikov } 18617106792SAlexander Shaposhnikov }; 18717106792SAlexander Shaposhnikov 18817106792SAlexander Shaposhnikov // `ppc_fp128` (`e`) shadow type: IBM extended double with 106 bits of mantissa. 18917106792SAlexander Shaposhnikov class PPC128ShadowConfig : public ShadowTypeConfigImpl<'e'> { 19017106792SAlexander Shaposhnikov Type *getType(LLVMContext &Context) const override { 19117106792SAlexander Shaposhnikov return Type::getPPC_FP128Ty(Context); 19217106792SAlexander Shaposhnikov } 19317106792SAlexander Shaposhnikov }; 19417106792SAlexander Shaposhnikov 19517106792SAlexander Shaposhnikov // Creates a ShadowTypeConfig given its type id. 19617106792SAlexander Shaposhnikov std::unique_ptr<ShadowTypeConfig> 19717106792SAlexander Shaposhnikov ShadowTypeConfig::fromNsanTypeId(const char TypeId) { 19817106792SAlexander Shaposhnikov switch (TypeId) { 19917106792SAlexander Shaposhnikov case F64ShadowConfig::kNsanTypeId: 20017106792SAlexander Shaposhnikov return std::make_unique<F64ShadowConfig>(); 20117106792SAlexander Shaposhnikov case F80ShadowConfig::kNsanTypeId: 20217106792SAlexander Shaposhnikov return std::make_unique<F80ShadowConfig>(); 20317106792SAlexander Shaposhnikov case F128ShadowConfig::kNsanTypeId: 20417106792SAlexander Shaposhnikov return std::make_unique<F128ShadowConfig>(); 20517106792SAlexander Shaposhnikov case PPC128ShadowConfig::kNsanTypeId: 20617106792SAlexander Shaposhnikov return std::make_unique<PPC128ShadowConfig>(); 20717106792SAlexander Shaposhnikov } 20817106792SAlexander Shaposhnikov report_fatal_error("nsan: invalid shadow type id '" + Twine(TypeId) + "'"); 20917106792SAlexander Shaposhnikov } 21017106792SAlexander Shaposhnikov 21117106792SAlexander Shaposhnikov // An enum corresponding to shadow value types. Used as indices in arrays, so 21217106792SAlexander Shaposhnikov // not an `enum class`. 21317106792SAlexander Shaposhnikov enum FTValueType { kFloat, kDouble, kLongDouble, kNumValueTypes }; 21417106792SAlexander Shaposhnikov 21517106792SAlexander Shaposhnikov // If `FT` corresponds to a primitive FTValueType, return it. 21617106792SAlexander Shaposhnikov static std::optional<FTValueType> ftValueTypeFromType(Type *FT) { 21717106792SAlexander Shaposhnikov if (FT->isFloatTy()) 21817106792SAlexander Shaposhnikov return kFloat; 21917106792SAlexander Shaposhnikov if (FT->isDoubleTy()) 22017106792SAlexander Shaposhnikov return kDouble; 22117106792SAlexander Shaposhnikov if (FT->isX86_FP80Ty()) 22217106792SAlexander Shaposhnikov return kLongDouble; 22317106792SAlexander Shaposhnikov return {}; 22417106792SAlexander Shaposhnikov } 22517106792SAlexander Shaposhnikov 22617106792SAlexander Shaposhnikov // Returns the LLVM type for an FTValueType. 22717106792SAlexander Shaposhnikov static Type *typeFromFTValueType(FTValueType VT, LLVMContext &Context) { 22817106792SAlexander Shaposhnikov switch (VT) { 22917106792SAlexander Shaposhnikov case kFloat: 23017106792SAlexander Shaposhnikov return Type::getFloatTy(Context); 23117106792SAlexander Shaposhnikov case kDouble: 23217106792SAlexander Shaposhnikov return Type::getDoubleTy(Context); 23317106792SAlexander Shaposhnikov case kLongDouble: 23417106792SAlexander Shaposhnikov return Type::getX86_FP80Ty(Context); 23517106792SAlexander Shaposhnikov case kNumValueTypes: 23617106792SAlexander Shaposhnikov return nullptr; 23717106792SAlexander Shaposhnikov } 238d32d20f3SSimon Pilgrim llvm_unreachable("Unhandled FTValueType enum"); 23917106792SAlexander Shaposhnikov } 24017106792SAlexander Shaposhnikov 24117106792SAlexander Shaposhnikov // Returns the type name for an FTValueType. 24217106792SAlexander Shaposhnikov static const char *typeNameFromFTValueType(FTValueType VT) { 24317106792SAlexander Shaposhnikov switch (VT) { 24417106792SAlexander Shaposhnikov case kFloat: 24517106792SAlexander Shaposhnikov return "float"; 24617106792SAlexander Shaposhnikov case kDouble: 24717106792SAlexander Shaposhnikov return "double"; 24817106792SAlexander Shaposhnikov case kLongDouble: 24917106792SAlexander Shaposhnikov return "longdouble"; 25017106792SAlexander Shaposhnikov case kNumValueTypes: 25117106792SAlexander Shaposhnikov return nullptr; 25217106792SAlexander Shaposhnikov } 253d32d20f3SSimon Pilgrim llvm_unreachable("Unhandled FTValueType enum"); 25417106792SAlexander Shaposhnikov } 25517106792SAlexander Shaposhnikov 25617106792SAlexander Shaposhnikov // A specific mapping configuration of application type to shadow type for nsan 25717106792SAlexander Shaposhnikov // (see -nsan-shadow-mapping flag). 25817106792SAlexander Shaposhnikov class MappingConfig { 25917106792SAlexander Shaposhnikov public: 26017106792SAlexander Shaposhnikov explicit MappingConfig(LLVMContext &C) : Context(C) { 26117106792SAlexander Shaposhnikov if (ClShadowMapping.size() != 3) 26217106792SAlexander Shaposhnikov report_fatal_error("Invalid nsan mapping: " + Twine(ClShadowMapping)); 26317106792SAlexander Shaposhnikov unsigned ShadowTypeSizeBits[kNumValueTypes]; 26417106792SAlexander Shaposhnikov for (int VT = 0; VT < kNumValueTypes; ++VT) { 26517106792SAlexander Shaposhnikov auto Config = ShadowTypeConfig::fromNsanTypeId(ClShadowMapping[VT]); 26617106792SAlexander Shaposhnikov if (!Config) 26717106792SAlexander Shaposhnikov report_fatal_error("Failed to get ShadowTypeConfig for " + 26817106792SAlexander Shaposhnikov Twine(ClShadowMapping[VT])); 26917106792SAlexander Shaposhnikov const unsigned AppTypeSize = 27017106792SAlexander Shaposhnikov typeFromFTValueType(static_cast<FTValueType>(VT), Context) 27117106792SAlexander Shaposhnikov ->getScalarSizeInBits(); 27217106792SAlexander Shaposhnikov const unsigned ShadowTypeSize = 27317106792SAlexander Shaposhnikov Config->getType(Context)->getScalarSizeInBits(); 27417106792SAlexander Shaposhnikov // Check that the shadow type size is at most kShadowScale times the 27517106792SAlexander Shaposhnikov // application type size, so that shadow memory compoutations are valid. 27617106792SAlexander Shaposhnikov if (ShadowTypeSize > kShadowScale * AppTypeSize) 27717106792SAlexander Shaposhnikov report_fatal_error("Invalid nsan mapping f" + Twine(AppTypeSize) + 27817106792SAlexander Shaposhnikov "->f" + Twine(ShadowTypeSize) + 27917106792SAlexander Shaposhnikov ": The shadow type size should be at most " + 28017106792SAlexander Shaposhnikov Twine(kShadowScale) + 28117106792SAlexander Shaposhnikov " times the application type size"); 28217106792SAlexander Shaposhnikov ShadowTypeSizeBits[VT] = ShadowTypeSize; 28317106792SAlexander Shaposhnikov Configs[VT] = std::move(Config); 28417106792SAlexander Shaposhnikov } 28517106792SAlexander Shaposhnikov 28617106792SAlexander Shaposhnikov // Check that the mapping is monotonous. This is required because if one 28717106792SAlexander Shaposhnikov // does an fpextend of `float->long double` in application code, nsan is 28817106792SAlexander Shaposhnikov // going to do an fpextend of `shadow(float) -> shadow(long double)` in 28917106792SAlexander Shaposhnikov // shadow code. This will fail in `qql` mode, since nsan would be 29017106792SAlexander Shaposhnikov // fpextending `f128->long`, which is invalid. 29117106792SAlexander Shaposhnikov // TODO: Relax this. 29217106792SAlexander Shaposhnikov if (ShadowTypeSizeBits[kFloat] > ShadowTypeSizeBits[kDouble] || 29317106792SAlexander Shaposhnikov ShadowTypeSizeBits[kDouble] > ShadowTypeSizeBits[kLongDouble]) 29417106792SAlexander Shaposhnikov report_fatal_error("Invalid nsan mapping: { float->f" + 29517106792SAlexander Shaposhnikov Twine(ShadowTypeSizeBits[kFloat]) + "; double->f" + 29617106792SAlexander Shaposhnikov Twine(ShadowTypeSizeBits[kDouble]) + 29717106792SAlexander Shaposhnikov "; long double->f" + 29817106792SAlexander Shaposhnikov Twine(ShadowTypeSizeBits[kLongDouble]) + " }"); 29917106792SAlexander Shaposhnikov } 30017106792SAlexander Shaposhnikov 30117106792SAlexander Shaposhnikov const ShadowTypeConfig &byValueType(FTValueType VT) const { 30217106792SAlexander Shaposhnikov assert(VT < FTValueType::kNumValueTypes && "invalid value type"); 30317106792SAlexander Shaposhnikov return *Configs[VT]; 30417106792SAlexander Shaposhnikov } 30517106792SAlexander Shaposhnikov 30617106792SAlexander Shaposhnikov // Returns the extended shadow type for a given application type. 30717106792SAlexander Shaposhnikov Type *getExtendedFPType(Type *FT) const { 30817106792SAlexander Shaposhnikov if (const auto VT = ftValueTypeFromType(FT)) 30917106792SAlexander Shaposhnikov return Configs[*VT]->getType(Context); 31017106792SAlexander Shaposhnikov if (FT->isVectorTy()) { 31117106792SAlexander Shaposhnikov auto *VecTy = cast<VectorType>(FT); 31217106792SAlexander Shaposhnikov // TODO: add support for scalable vector types. 31317106792SAlexander Shaposhnikov if (VecTy->isScalableTy()) 31417106792SAlexander Shaposhnikov return nullptr; 31517106792SAlexander Shaposhnikov Type *ExtendedScalar = getExtendedFPType(VecTy->getElementType()); 31617106792SAlexander Shaposhnikov return ExtendedScalar 31717106792SAlexander Shaposhnikov ? VectorType::get(ExtendedScalar, VecTy->getElementCount()) 31817106792SAlexander Shaposhnikov : nullptr; 31917106792SAlexander Shaposhnikov } 32017106792SAlexander Shaposhnikov return nullptr; 32117106792SAlexander Shaposhnikov } 32217106792SAlexander Shaposhnikov 32317106792SAlexander Shaposhnikov private: 32417106792SAlexander Shaposhnikov LLVMContext &Context; 32517106792SAlexander Shaposhnikov std::unique_ptr<ShadowTypeConfig> Configs[FTValueType::kNumValueTypes]; 32617106792SAlexander Shaposhnikov }; 32717106792SAlexander Shaposhnikov 32817106792SAlexander Shaposhnikov // The memory extents of a type specifies how many elements of a given 32917106792SAlexander Shaposhnikov // FTValueType needs to be stored when storing this type. 33017106792SAlexander Shaposhnikov struct MemoryExtents { 33117106792SAlexander Shaposhnikov FTValueType ValueType; 33217106792SAlexander Shaposhnikov uint64_t NumElts; 33317106792SAlexander Shaposhnikov }; 33417106792SAlexander Shaposhnikov 33517106792SAlexander Shaposhnikov static MemoryExtents getMemoryExtentsOrDie(Type *FT) { 33617106792SAlexander Shaposhnikov if (const auto VT = ftValueTypeFromType(FT)) 33717106792SAlexander Shaposhnikov return {*VT, 1}; 33817106792SAlexander Shaposhnikov if (auto *VecTy = dyn_cast<VectorType>(FT)) { 33917106792SAlexander Shaposhnikov const auto ScalarExtents = getMemoryExtentsOrDie(VecTy->getElementType()); 34017106792SAlexander Shaposhnikov return {ScalarExtents.ValueType, 34117106792SAlexander Shaposhnikov ScalarExtents.NumElts * VecTy->getElementCount().getFixedValue()}; 34217106792SAlexander Shaposhnikov } 34317106792SAlexander Shaposhnikov llvm_unreachable("invalid value type"); 34417106792SAlexander Shaposhnikov } 34517106792SAlexander Shaposhnikov 34617106792SAlexander Shaposhnikov // The location of a check. Passed as parameters to runtime checking functions. 34717106792SAlexander Shaposhnikov class CheckLoc { 34817106792SAlexander Shaposhnikov public: 34917106792SAlexander Shaposhnikov // Creates a location that references an application memory location. 35017106792SAlexander Shaposhnikov static CheckLoc makeStore(Value *Address) { 35117106792SAlexander Shaposhnikov CheckLoc Result(kStore); 35217106792SAlexander Shaposhnikov Result.Address = Address; 35317106792SAlexander Shaposhnikov return Result; 35417106792SAlexander Shaposhnikov } 35517106792SAlexander Shaposhnikov static CheckLoc makeLoad(Value *Address) { 35617106792SAlexander Shaposhnikov CheckLoc Result(kLoad); 35717106792SAlexander Shaposhnikov Result.Address = Address; 35817106792SAlexander Shaposhnikov return Result; 35917106792SAlexander Shaposhnikov } 36017106792SAlexander Shaposhnikov 36117106792SAlexander Shaposhnikov // Creates a location that references an argument, given by id. 36217106792SAlexander Shaposhnikov static CheckLoc makeArg(int ArgId) { 36317106792SAlexander Shaposhnikov CheckLoc Result(kArg); 36417106792SAlexander Shaposhnikov Result.ArgId = ArgId; 36517106792SAlexander Shaposhnikov return Result; 36617106792SAlexander Shaposhnikov } 36717106792SAlexander Shaposhnikov 36817106792SAlexander Shaposhnikov // Creates a location that references the return value of a function. 36917106792SAlexander Shaposhnikov static CheckLoc makeRet() { return CheckLoc(kRet); } 37017106792SAlexander Shaposhnikov 37117106792SAlexander Shaposhnikov // Creates a location that references a vector insert. 37217106792SAlexander Shaposhnikov static CheckLoc makeInsert() { return CheckLoc(kInsert); } 37317106792SAlexander Shaposhnikov 37417106792SAlexander Shaposhnikov // Returns the CheckType of location this refers to, as an integer-typed LLVM 37517106792SAlexander Shaposhnikov // IR value. 37617106792SAlexander Shaposhnikov Value *getType(LLVMContext &C) const { 37717106792SAlexander Shaposhnikov return ConstantInt::get(Type::getInt32Ty(C), static_cast<int>(CheckTy)); 37817106792SAlexander Shaposhnikov } 37917106792SAlexander Shaposhnikov 38017106792SAlexander Shaposhnikov // Returns a CheckType-specific value representing details of the location 38117106792SAlexander Shaposhnikov // (e.g. application address for loads or stores), as an `IntptrTy`-typed LLVM 38217106792SAlexander Shaposhnikov // IR value. 38317106792SAlexander Shaposhnikov Value *getValue(Type *IntptrTy, IRBuilder<> &Builder) const { 38417106792SAlexander Shaposhnikov switch (CheckTy) { 38517106792SAlexander Shaposhnikov case kUnknown: 38617106792SAlexander Shaposhnikov llvm_unreachable("unknown type"); 38717106792SAlexander Shaposhnikov case kRet: 38817106792SAlexander Shaposhnikov case kInsert: 38917106792SAlexander Shaposhnikov return ConstantInt::get(IntptrTy, 0); 39017106792SAlexander Shaposhnikov case kArg: 39117106792SAlexander Shaposhnikov return ConstantInt::get(IntptrTy, ArgId); 39217106792SAlexander Shaposhnikov case kLoad: 39317106792SAlexander Shaposhnikov case kStore: 39417106792SAlexander Shaposhnikov return Builder.CreatePtrToInt(Address, IntptrTy); 39517106792SAlexander Shaposhnikov } 396d32d20f3SSimon Pilgrim llvm_unreachable("Unhandled CheckType enum"); 39717106792SAlexander Shaposhnikov } 39817106792SAlexander Shaposhnikov 39917106792SAlexander Shaposhnikov private: 40017106792SAlexander Shaposhnikov // Must be kept in sync with the runtime, 40117106792SAlexander Shaposhnikov // see compiler-rt/lib/nsan/nsan_stats.h 40217106792SAlexander Shaposhnikov enum CheckType { 40317106792SAlexander Shaposhnikov kUnknown = 0, 40417106792SAlexander Shaposhnikov kRet, 40517106792SAlexander Shaposhnikov kArg, 40617106792SAlexander Shaposhnikov kLoad, 40717106792SAlexander Shaposhnikov kStore, 40817106792SAlexander Shaposhnikov kInsert, 40917106792SAlexander Shaposhnikov }; 41017106792SAlexander Shaposhnikov explicit CheckLoc(CheckType CheckTy) : CheckTy(CheckTy) {} 41117106792SAlexander Shaposhnikov 41217106792SAlexander Shaposhnikov Value *Address = nullptr; 41317106792SAlexander Shaposhnikov const CheckType CheckTy; 41417106792SAlexander Shaposhnikov int ArgId = -1; 41517106792SAlexander Shaposhnikov }; 41617106792SAlexander Shaposhnikov 41717106792SAlexander Shaposhnikov // A map of LLVM IR values to shadow LLVM IR values. 41817106792SAlexander Shaposhnikov class ValueToShadowMap { 41917106792SAlexander Shaposhnikov public: 42017106792SAlexander Shaposhnikov explicit ValueToShadowMap(const MappingConfig &Config) : Config(Config) {} 42117106792SAlexander Shaposhnikov 42217106792SAlexander Shaposhnikov ValueToShadowMap(const ValueToShadowMap &) = delete; 42317106792SAlexander Shaposhnikov ValueToShadowMap &operator=(const ValueToShadowMap &) = delete; 42417106792SAlexander Shaposhnikov 42517106792SAlexander Shaposhnikov // Sets the shadow value for a value. Asserts that the value does not already 42617106792SAlexander Shaposhnikov // have a value. 42717106792SAlexander Shaposhnikov void setShadow(Value &V, Value &Shadow) { 42817106792SAlexander Shaposhnikov [[maybe_unused]] const bool Inserted = Map.try_emplace(&V, &Shadow).second; 42917106792SAlexander Shaposhnikov LLVM_DEBUG({ 43017106792SAlexander Shaposhnikov if (!Inserted) { 43117106792SAlexander Shaposhnikov if (auto *I = dyn_cast<Instruction>(&V)) 43217106792SAlexander Shaposhnikov errs() << I->getFunction()->getName() << ": "; 43317106792SAlexander Shaposhnikov errs() << "duplicate shadow (" << &V << "): "; 43417106792SAlexander Shaposhnikov V.dump(); 43517106792SAlexander Shaposhnikov } 43617106792SAlexander Shaposhnikov }); 43717106792SAlexander Shaposhnikov assert(Inserted && "duplicate shadow"); 43817106792SAlexander Shaposhnikov } 43917106792SAlexander Shaposhnikov 44017106792SAlexander Shaposhnikov // Returns true if the value already has a shadow (including if the value is a 44117106792SAlexander Shaposhnikov // constant). If true, calling getShadow() is valid. 44217106792SAlexander Shaposhnikov bool hasShadow(Value *V) const { 44317106792SAlexander Shaposhnikov return isa<Constant>(V) || (Map.find(V) != Map.end()); 44417106792SAlexander Shaposhnikov } 44517106792SAlexander Shaposhnikov 44617106792SAlexander Shaposhnikov // Returns the shadow value for a given value. Asserts that the value has 44717106792SAlexander Shaposhnikov // a shadow value. Lazily creates shadows for constant values. 44817106792SAlexander Shaposhnikov Value *getShadow(Value *V) const { 44917106792SAlexander Shaposhnikov if (Constant *C = dyn_cast<Constant>(V)) 45017106792SAlexander Shaposhnikov return getShadowConstant(C); 45117106792SAlexander Shaposhnikov return Map.find(V)->second; 45217106792SAlexander Shaposhnikov } 45317106792SAlexander Shaposhnikov 45417106792SAlexander Shaposhnikov bool empty() const { return Map.empty(); } 45517106792SAlexander Shaposhnikov 45617106792SAlexander Shaposhnikov private: 45717106792SAlexander Shaposhnikov // Extends a constant application value to its shadow counterpart. 45817106792SAlexander Shaposhnikov APFloat extendConstantFP(APFloat CV, const fltSemantics &To) const { 45917106792SAlexander Shaposhnikov bool LosesInfo = false; 46017106792SAlexander Shaposhnikov CV.convert(To, APFloatBase::rmTowardZero, &LosesInfo); 46117106792SAlexander Shaposhnikov return CV; 46217106792SAlexander Shaposhnikov } 46317106792SAlexander Shaposhnikov 46417106792SAlexander Shaposhnikov // Returns the shadow constant for the given application constant. 46517106792SAlexander Shaposhnikov Constant *getShadowConstant(Constant *C) const { 46617106792SAlexander Shaposhnikov if (UndefValue *U = dyn_cast<UndefValue>(C)) { 46717106792SAlexander Shaposhnikov return UndefValue::get(Config.getExtendedFPType(U->getType())); 46817106792SAlexander Shaposhnikov } 46917106792SAlexander Shaposhnikov if (ConstantFP *CFP = dyn_cast<ConstantFP>(C)) { 47017106792SAlexander Shaposhnikov // Floating-point constants. 47117106792SAlexander Shaposhnikov Type *Ty = Config.getExtendedFPType(CFP->getType()); 47217106792SAlexander Shaposhnikov return ConstantFP::get( 47317106792SAlexander Shaposhnikov Ty, extendConstantFP(CFP->getValueAPF(), Ty->getFltSemantics())); 47417106792SAlexander Shaposhnikov } 47517106792SAlexander Shaposhnikov // Vector, array, or aggregate constants. 47617106792SAlexander Shaposhnikov if (C->getType()->isVectorTy()) { 47717106792SAlexander Shaposhnikov SmallVector<Constant *, 8> Elements; 47817106792SAlexander Shaposhnikov for (int I = 0, E = cast<VectorType>(C->getType()) 47917106792SAlexander Shaposhnikov ->getElementCount() 48017106792SAlexander Shaposhnikov .getFixedValue(); 48117106792SAlexander Shaposhnikov I < E; ++I) 48217106792SAlexander Shaposhnikov Elements.push_back(getShadowConstant(C->getAggregateElement(I))); 48317106792SAlexander Shaposhnikov return ConstantVector::get(Elements); 48417106792SAlexander Shaposhnikov } 48517106792SAlexander Shaposhnikov llvm_unreachable("unimplemented"); 48617106792SAlexander Shaposhnikov } 48717106792SAlexander Shaposhnikov 48817106792SAlexander Shaposhnikov const MappingConfig &Config; 48917106792SAlexander Shaposhnikov DenseMap<Value *, Value *> Map; 49017106792SAlexander Shaposhnikov }; 49117106792SAlexander Shaposhnikov 492ddf5725eSDmitry Chestnykh class NsanMemOpFn { 493ddf5725eSDmitry Chestnykh public: 494ddf5725eSDmitry Chestnykh NsanMemOpFn(Module &M, ArrayRef<StringRef> Sized, StringRef Fallback, 495ddf5725eSDmitry Chestnykh size_t NumArgs); 496ddf5725eSDmitry Chestnykh FunctionCallee getFunctionFor(uint64_t MemOpSize) const; 497ddf5725eSDmitry Chestnykh FunctionCallee getFallback() const; 498ddf5725eSDmitry Chestnykh 499ddf5725eSDmitry Chestnykh private: 500ddf5725eSDmitry Chestnykh SmallVector<FunctionCallee> Funcs; 501ddf5725eSDmitry Chestnykh size_t NumSizedFuncs; 502ddf5725eSDmitry Chestnykh }; 503ddf5725eSDmitry Chestnykh 504ddf5725eSDmitry Chestnykh NsanMemOpFn::NsanMemOpFn(Module &M, ArrayRef<StringRef> Sized, 505ddf5725eSDmitry Chestnykh StringRef Fallback, size_t NumArgs) { 506ddf5725eSDmitry Chestnykh LLVMContext &Ctx = M.getContext(); 507ddf5725eSDmitry Chestnykh AttributeList Attr; 508ddf5725eSDmitry Chestnykh Attr = Attr.addFnAttribute(Ctx, Attribute::NoUnwind); 509ddf5725eSDmitry Chestnykh Type *PtrTy = PointerType::getUnqual(Ctx); 510ddf5725eSDmitry Chestnykh Type *VoidTy = Type::getVoidTy(Ctx); 511ddf5725eSDmitry Chestnykh IntegerType *IntptrTy = M.getDataLayout().getIntPtrType(Ctx); 512ddf5725eSDmitry Chestnykh FunctionType *SizedFnTy = nullptr; 513ddf5725eSDmitry Chestnykh 514ddf5725eSDmitry Chestnykh NumSizedFuncs = Sized.size(); 515ddf5725eSDmitry Chestnykh 516ddf5725eSDmitry Chestnykh // First entry is fallback function 517ddf5725eSDmitry Chestnykh if (NumArgs == 3) { 518ddf5725eSDmitry Chestnykh Funcs.push_back( 519ddf5725eSDmitry Chestnykh M.getOrInsertFunction(Fallback, Attr, VoidTy, PtrTy, PtrTy, IntptrTy)); 520ddf5725eSDmitry Chestnykh SizedFnTy = FunctionType::get(VoidTy, {PtrTy, PtrTy}, false); 521ddf5725eSDmitry Chestnykh } else if (NumArgs == 2) { 522ddf5725eSDmitry Chestnykh Funcs.push_back( 523ddf5725eSDmitry Chestnykh M.getOrInsertFunction(Fallback, Attr, VoidTy, PtrTy, IntptrTy)); 524ddf5725eSDmitry Chestnykh SizedFnTy = FunctionType::get(VoidTy, {PtrTy}, false); 525ddf5725eSDmitry Chestnykh } else { 526cd82fee3SDmitry Chestnykh llvm_unreachable("Unexpected value of sized functions arguments"); 527ddf5725eSDmitry Chestnykh } 528ddf5725eSDmitry Chestnykh 529ddf5725eSDmitry Chestnykh for (size_t i = 0; i < NumSizedFuncs; ++i) 530ddf5725eSDmitry Chestnykh Funcs.push_back(M.getOrInsertFunction(Sized[i], SizedFnTy, Attr)); 531ddf5725eSDmitry Chestnykh } 532ddf5725eSDmitry Chestnykh 533ddf5725eSDmitry Chestnykh FunctionCallee NsanMemOpFn::getFunctionFor(uint64_t MemOpSize) const { 534ddf5725eSDmitry Chestnykh // Now `getFunctionFor` operates on `Funcs` of size 4 (at least) and the 535ddf5725eSDmitry Chestnykh // following code assumes that the number of functions in `Func` is sufficient 536ddf5725eSDmitry Chestnykh assert(NumSizedFuncs >= 3 && "Unexpected number of sized functions"); 537ddf5725eSDmitry Chestnykh 538ddf5725eSDmitry Chestnykh size_t Idx = 539ddf5725eSDmitry Chestnykh MemOpSize == 4 ? 1 : (MemOpSize == 8 ? 2 : (MemOpSize == 16 ? 3 : 0)); 540ddf5725eSDmitry Chestnykh 541ddf5725eSDmitry Chestnykh return Funcs[Idx]; 542ddf5725eSDmitry Chestnykh } 543ddf5725eSDmitry Chestnykh 544ddf5725eSDmitry Chestnykh FunctionCallee NsanMemOpFn::getFallback() const { return Funcs[0]; } 545ddf5725eSDmitry Chestnykh 54617106792SAlexander Shaposhnikov /// Instantiating NumericalStabilitySanitizer inserts the nsan runtime library 54717106792SAlexander Shaposhnikov /// API function declarations into the module if they don't exist already. 54817106792SAlexander Shaposhnikov /// Instantiating ensures the __nsan_init function is in the list of global 54917106792SAlexander Shaposhnikov /// constructors for the module. 55017106792SAlexander Shaposhnikov class NumericalStabilitySanitizer { 55117106792SAlexander Shaposhnikov public: 55217106792SAlexander Shaposhnikov NumericalStabilitySanitizer(Module &M); 55317106792SAlexander Shaposhnikov bool sanitizeFunction(Function &F, const TargetLibraryInfo &TLI); 55417106792SAlexander Shaposhnikov 55517106792SAlexander Shaposhnikov private: 55617106792SAlexander Shaposhnikov bool instrumentMemIntrinsic(MemIntrinsic *MI); 55717106792SAlexander Shaposhnikov void maybeAddSuffixForNsanInterface(CallBase *CI); 55817106792SAlexander Shaposhnikov bool addrPointsToConstantData(Value *Addr); 55917106792SAlexander Shaposhnikov void maybeCreateShadowValue(Instruction &Root, const TargetLibraryInfo &TLI, 56017106792SAlexander Shaposhnikov ValueToShadowMap &Map); 56117106792SAlexander Shaposhnikov Value *createShadowValueWithOperandsAvailable(Instruction &Inst, 56217106792SAlexander Shaposhnikov const TargetLibraryInfo &TLI, 56317106792SAlexander Shaposhnikov const ValueToShadowMap &Map); 56417106792SAlexander Shaposhnikov PHINode *maybeCreateShadowPhi(PHINode &Phi, const TargetLibraryInfo &TLI); 56517106792SAlexander Shaposhnikov void createShadowArguments(Function &F, const TargetLibraryInfo &TLI, 56617106792SAlexander Shaposhnikov ValueToShadowMap &Map); 56717106792SAlexander Shaposhnikov 56817106792SAlexander Shaposhnikov void populateShadowStack(CallBase &CI, const TargetLibraryInfo &TLI, 56917106792SAlexander Shaposhnikov const ValueToShadowMap &Map); 57017106792SAlexander Shaposhnikov 57117106792SAlexander Shaposhnikov void propagateShadowValues(Instruction &Inst, const TargetLibraryInfo &TLI, 57217106792SAlexander Shaposhnikov const ValueToShadowMap &Map); 57317106792SAlexander Shaposhnikov Value *emitCheck(Value *V, Value *ShadowV, IRBuilder<> &Builder, 57417106792SAlexander Shaposhnikov CheckLoc Loc); 57517106792SAlexander Shaposhnikov Value *emitCheckInternal(Value *V, Value *ShadowV, IRBuilder<> &Builder, 57617106792SAlexander Shaposhnikov CheckLoc Loc); 57717106792SAlexander Shaposhnikov void emitFCmpCheck(FCmpInst &FCmp, const ValueToShadowMap &Map); 57817106792SAlexander Shaposhnikov 57917106792SAlexander Shaposhnikov // Value creation handlers. 58017106792SAlexander Shaposhnikov Value *handleLoad(LoadInst &Load, Type *VT, Type *ExtendedVT); 58117106792SAlexander Shaposhnikov Value *handleCallBase(CallBase &Call, Type *VT, Type *ExtendedVT, 58217106792SAlexander Shaposhnikov const TargetLibraryInfo &TLI, 58317106792SAlexander Shaposhnikov const ValueToShadowMap &Map, IRBuilder<> &Builder); 58417106792SAlexander Shaposhnikov Value *maybeHandleKnownCallBase(CallBase &Call, Type *VT, Type *ExtendedVT, 58517106792SAlexander Shaposhnikov const TargetLibraryInfo &TLI, 58617106792SAlexander Shaposhnikov const ValueToShadowMap &Map, 58717106792SAlexander Shaposhnikov IRBuilder<> &Builder); 58817106792SAlexander Shaposhnikov Value *handleTrunc(const FPTruncInst &Trunc, Type *VT, Type *ExtendedVT, 58917106792SAlexander Shaposhnikov const ValueToShadowMap &Map, IRBuilder<> &Builder); 59017106792SAlexander Shaposhnikov Value *handleExt(const FPExtInst &Ext, Type *VT, Type *ExtendedVT, 59117106792SAlexander Shaposhnikov const ValueToShadowMap &Map, IRBuilder<> &Builder); 59217106792SAlexander Shaposhnikov 59317106792SAlexander Shaposhnikov // Value propagation handlers. 59417106792SAlexander Shaposhnikov void propagateFTStore(StoreInst &Store, Type *VT, Type *ExtendedVT, 59517106792SAlexander Shaposhnikov const ValueToShadowMap &Map); 59617106792SAlexander Shaposhnikov void propagateNonFTStore(StoreInst &Store, Type *VT, 59717106792SAlexander Shaposhnikov const ValueToShadowMap &Map); 59817106792SAlexander Shaposhnikov 59917106792SAlexander Shaposhnikov const DataLayout &DL; 60017106792SAlexander Shaposhnikov LLVMContext &Context; 60117106792SAlexander Shaposhnikov MappingConfig Config; 60217106792SAlexander Shaposhnikov IntegerType *IntptrTy = nullptr; 603ddf5725eSDmitry Chestnykh 604ddf5725eSDmitry Chestnykh // TODO: Use std::array instead? 60517106792SAlexander Shaposhnikov FunctionCallee NsanGetShadowPtrForStore[FTValueType::kNumValueTypes] = {}; 60617106792SAlexander Shaposhnikov FunctionCallee NsanGetShadowPtrForLoad[FTValueType::kNumValueTypes] = {}; 60717106792SAlexander Shaposhnikov FunctionCallee NsanCheckValue[FTValueType::kNumValueTypes] = {}; 60817106792SAlexander Shaposhnikov FunctionCallee NsanFCmpFail[FTValueType::kNumValueTypes] = {}; 609ddf5725eSDmitry Chestnykh 610ddf5725eSDmitry Chestnykh NsanMemOpFn NsanCopyFns; 611ddf5725eSDmitry Chestnykh NsanMemOpFn NsanSetUnknownFns; 612ddf5725eSDmitry Chestnykh 61317106792SAlexander Shaposhnikov FunctionCallee NsanGetRawShadowTypePtr; 61417106792SAlexander Shaposhnikov FunctionCallee NsanGetRawShadowPtr; 61517106792SAlexander Shaposhnikov GlobalValue *NsanShadowRetTag = nullptr; 61617106792SAlexander Shaposhnikov 61717106792SAlexander Shaposhnikov Type *NsanShadowRetType = nullptr; 61817106792SAlexander Shaposhnikov GlobalValue *NsanShadowRetPtr = nullptr; 61917106792SAlexander Shaposhnikov 62017106792SAlexander Shaposhnikov GlobalValue *NsanShadowArgsTag = nullptr; 62117106792SAlexander Shaposhnikov 62217106792SAlexander Shaposhnikov Type *NsanShadowArgsType = nullptr; 62317106792SAlexander Shaposhnikov GlobalValue *NsanShadowArgsPtr = nullptr; 62417106792SAlexander Shaposhnikov 62517106792SAlexander Shaposhnikov std::optional<Regex> CheckFunctionsFilter; 62617106792SAlexander Shaposhnikov }; 62717106792SAlexander Shaposhnikov } // end anonymous namespace 62817106792SAlexander Shaposhnikov 62917106792SAlexander Shaposhnikov PreservedAnalyses 63017106792SAlexander Shaposhnikov NumericalStabilitySanitizerPass::run(Module &M, ModuleAnalysisManager &MAM) { 63117106792SAlexander Shaposhnikov getOrCreateSanitizerCtorAndInitFunctions( 63217106792SAlexander Shaposhnikov M, kNsanModuleCtorName, kNsanInitName, /*InitArgTypes=*/{}, 63317106792SAlexander Shaposhnikov /*InitArgs=*/{}, 63417106792SAlexander Shaposhnikov // This callback is invoked when the functions are created the first 63517106792SAlexander Shaposhnikov // time. Hook them into the global ctors list in that case: 63617106792SAlexander Shaposhnikov [&](Function *Ctor, FunctionCallee) { appendToGlobalCtors(M, Ctor, 0); }); 63717106792SAlexander Shaposhnikov 63817106792SAlexander Shaposhnikov NumericalStabilitySanitizer Nsan(M); 63917106792SAlexander Shaposhnikov auto &FAM = MAM.getResult<FunctionAnalysisManagerModuleProxy>(M).getManager(); 64017106792SAlexander Shaposhnikov for (Function &F : M) 64117106792SAlexander Shaposhnikov Nsan.sanitizeFunction(F, FAM.getResult<TargetLibraryAnalysis>(F)); 64217106792SAlexander Shaposhnikov 64317106792SAlexander Shaposhnikov return PreservedAnalyses::none(); 64417106792SAlexander Shaposhnikov } 64517106792SAlexander Shaposhnikov 64617106792SAlexander Shaposhnikov static GlobalValue *createThreadLocalGV(const char *Name, Module &M, Type *Ty) { 64717106792SAlexander Shaposhnikov return dyn_cast<GlobalValue>(M.getOrInsertGlobal(Name, Ty, [&M, Ty, Name] { 64817106792SAlexander Shaposhnikov return new GlobalVariable(M, Ty, false, GlobalVariable::ExternalLinkage, 64917106792SAlexander Shaposhnikov nullptr, Name, nullptr, 65017106792SAlexander Shaposhnikov GlobalVariable::InitialExecTLSModel); 65117106792SAlexander Shaposhnikov })); 65217106792SAlexander Shaposhnikov } 65317106792SAlexander Shaposhnikov 65417106792SAlexander Shaposhnikov NumericalStabilitySanitizer::NumericalStabilitySanitizer(Module &M) 655ddf5725eSDmitry Chestnykh : DL(M.getDataLayout()), Context(M.getContext()), Config(Context), 656ddf5725eSDmitry Chestnykh NsanCopyFns(M, {"__nsan_copy_4", "__nsan_copy_8", "__nsan_copy_16"}, 657ddf5725eSDmitry Chestnykh "__nsan_copy_values", /*NumArgs=*/3), 658ddf5725eSDmitry Chestnykh NsanSetUnknownFns(M, 659ddf5725eSDmitry Chestnykh {"__nsan_set_value_unknown_4", 660ddf5725eSDmitry Chestnykh "__nsan_set_value_unknown_8", 661ddf5725eSDmitry Chestnykh "__nsan_set_value_unknown_16"}, 662ddf5725eSDmitry Chestnykh "__nsan_set_value_unknown", /*NumArgs=*/2) { 66317106792SAlexander Shaposhnikov IntptrTy = DL.getIntPtrType(Context); 66417106792SAlexander Shaposhnikov Type *PtrTy = PointerType::getUnqual(Context); 66517106792SAlexander Shaposhnikov Type *Int32Ty = Type::getInt32Ty(Context); 66617106792SAlexander Shaposhnikov Type *Int1Ty = Type::getInt1Ty(Context); 66717106792SAlexander Shaposhnikov Type *VoidTy = Type::getVoidTy(Context); 66817106792SAlexander Shaposhnikov 66917106792SAlexander Shaposhnikov AttributeList Attr; 67017106792SAlexander Shaposhnikov Attr = Attr.addFnAttribute(Context, Attribute::NoUnwind); 67117106792SAlexander Shaposhnikov // Initialize the runtime values (functions and global variables). 67217106792SAlexander Shaposhnikov for (int I = 0; I < kNumValueTypes; ++I) { 67317106792SAlexander Shaposhnikov const FTValueType VT = static_cast<FTValueType>(I); 67417106792SAlexander Shaposhnikov const char *VTName = typeNameFromFTValueType(VT); 67517106792SAlexander Shaposhnikov Type *VTTy = typeFromFTValueType(VT, Context); 67617106792SAlexander Shaposhnikov 67717106792SAlexander Shaposhnikov // Load/store. 67817106792SAlexander Shaposhnikov const std::string GetterPrefix = 67917106792SAlexander Shaposhnikov std::string("__nsan_get_shadow_ptr_for_") + VTName; 68017106792SAlexander Shaposhnikov NsanGetShadowPtrForStore[VT] = M.getOrInsertFunction( 68117106792SAlexander Shaposhnikov GetterPrefix + "_store", Attr, PtrTy, PtrTy, IntptrTy); 68217106792SAlexander Shaposhnikov NsanGetShadowPtrForLoad[VT] = M.getOrInsertFunction( 68317106792SAlexander Shaposhnikov GetterPrefix + "_load", Attr, PtrTy, PtrTy, IntptrTy); 68417106792SAlexander Shaposhnikov 68517106792SAlexander Shaposhnikov // Check. 68617106792SAlexander Shaposhnikov const auto &ShadowConfig = Config.byValueType(VT); 68717106792SAlexander Shaposhnikov Type *ShadowTy = ShadowConfig.getType(Context); 68817106792SAlexander Shaposhnikov NsanCheckValue[VT] = 68917106792SAlexander Shaposhnikov M.getOrInsertFunction(std::string("__nsan_internal_check_") + VTName + 69017106792SAlexander Shaposhnikov "_" + ShadowConfig.getNsanTypeId(), 69117106792SAlexander Shaposhnikov Attr, Int32Ty, VTTy, ShadowTy, Int32Ty, IntptrTy); 69217106792SAlexander Shaposhnikov NsanFCmpFail[VT] = M.getOrInsertFunction( 69317106792SAlexander Shaposhnikov std::string("__nsan_fcmp_fail_") + VTName + "_" + 69417106792SAlexander Shaposhnikov ShadowConfig.getNsanTypeId(), 69517106792SAlexander Shaposhnikov Attr, VoidTy, VTTy, VTTy, ShadowTy, ShadowTy, Int32Ty, Int1Ty, Int1Ty); 69617106792SAlexander Shaposhnikov } 69717106792SAlexander Shaposhnikov 69817106792SAlexander Shaposhnikov // TODO: Add attributes nofree, nosync, readnone, readonly, 69917106792SAlexander Shaposhnikov NsanGetRawShadowTypePtr = M.getOrInsertFunction( 70017106792SAlexander Shaposhnikov "__nsan_internal_get_raw_shadow_type_ptr", Attr, PtrTy, PtrTy); 70117106792SAlexander Shaposhnikov NsanGetRawShadowPtr = M.getOrInsertFunction( 70217106792SAlexander Shaposhnikov "__nsan_internal_get_raw_shadow_ptr", Attr, PtrTy, PtrTy); 70317106792SAlexander Shaposhnikov 70417106792SAlexander Shaposhnikov NsanShadowRetTag = createThreadLocalGV("__nsan_shadow_ret_tag", M, IntptrTy); 70517106792SAlexander Shaposhnikov 70617106792SAlexander Shaposhnikov NsanShadowRetType = ArrayType::get(Type::getInt8Ty(Context), 70717106792SAlexander Shaposhnikov kMaxVectorWidth * kMaxShadowTypeSizeBytes); 70817106792SAlexander Shaposhnikov NsanShadowRetPtr = 70917106792SAlexander Shaposhnikov createThreadLocalGV("__nsan_shadow_ret_ptr", M, NsanShadowRetType); 71017106792SAlexander Shaposhnikov 71117106792SAlexander Shaposhnikov NsanShadowArgsTag = 71217106792SAlexander Shaposhnikov createThreadLocalGV("__nsan_shadow_args_tag", M, IntptrTy); 71317106792SAlexander Shaposhnikov 71417106792SAlexander Shaposhnikov NsanShadowArgsType = 71517106792SAlexander Shaposhnikov ArrayType::get(Type::getInt8Ty(Context), 71617106792SAlexander Shaposhnikov kMaxVectorWidth * kMaxNumArgs * kMaxShadowTypeSizeBytes); 71717106792SAlexander Shaposhnikov 71817106792SAlexander Shaposhnikov NsanShadowArgsPtr = 71917106792SAlexander Shaposhnikov createThreadLocalGV("__nsan_shadow_args_ptr", M, NsanShadowArgsType); 72017106792SAlexander Shaposhnikov 72117106792SAlexander Shaposhnikov if (!ClCheckFunctionsFilter.empty()) { 72217106792SAlexander Shaposhnikov Regex R = Regex(ClCheckFunctionsFilter); 72317106792SAlexander Shaposhnikov std::string RegexError; 72417106792SAlexander Shaposhnikov assert(R.isValid(RegexError)); 72517106792SAlexander Shaposhnikov CheckFunctionsFilter = std::move(R); 72617106792SAlexander Shaposhnikov } 72717106792SAlexander Shaposhnikov } 72817106792SAlexander Shaposhnikov 72917106792SAlexander Shaposhnikov // Returns true if the given LLVM Value points to constant data (typically, a 73017106792SAlexander Shaposhnikov // global variable reference). 73117106792SAlexander Shaposhnikov bool NumericalStabilitySanitizer::addrPointsToConstantData(Value *Addr) { 73217106792SAlexander Shaposhnikov // If this is a GEP, just analyze its pointer operand. 73317106792SAlexander Shaposhnikov if (GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(Addr)) 73417106792SAlexander Shaposhnikov Addr = GEP->getPointerOperand(); 73517106792SAlexander Shaposhnikov 73617106792SAlexander Shaposhnikov if (GlobalVariable *GV = dyn_cast<GlobalVariable>(Addr)) 73717106792SAlexander Shaposhnikov return GV->isConstant(); 73817106792SAlexander Shaposhnikov return false; 73917106792SAlexander Shaposhnikov } 74017106792SAlexander Shaposhnikov 74117106792SAlexander Shaposhnikov // This instruments the function entry to create shadow arguments. 74217106792SAlexander Shaposhnikov // Pseudocode: 74317106792SAlexander Shaposhnikov // if (this_fn_ptr == __nsan_shadow_args_tag) { 74417106792SAlexander Shaposhnikov // s(arg0) = LOAD<sizeof(arg0)>(__nsan_shadow_args); 74517106792SAlexander Shaposhnikov // s(arg1) = LOAD<sizeof(arg1)>(__nsan_shadow_args + sizeof(arg0)); 74617106792SAlexander Shaposhnikov // ... 74717106792SAlexander Shaposhnikov // __nsan_shadow_args_tag = 0; 74817106792SAlexander Shaposhnikov // } else { 74917106792SAlexander Shaposhnikov // s(arg0) = fext(arg0); 75017106792SAlexander Shaposhnikov // s(arg1) = fext(arg1); 75117106792SAlexander Shaposhnikov // ... 75217106792SAlexander Shaposhnikov // } 75317106792SAlexander Shaposhnikov void NumericalStabilitySanitizer::createShadowArguments( 75417106792SAlexander Shaposhnikov Function &F, const TargetLibraryInfo &TLI, ValueToShadowMap &Map) { 75517106792SAlexander Shaposhnikov assert(!F.getIntrinsicID() && "found a definition of an intrinsic"); 75617106792SAlexander Shaposhnikov 75717106792SAlexander Shaposhnikov // Do not bother if there are no FP args. 75817106792SAlexander Shaposhnikov if (all_of(F.args(), [this](const Argument &Arg) { 75917106792SAlexander Shaposhnikov return Config.getExtendedFPType(Arg.getType()) == nullptr; 76017106792SAlexander Shaposhnikov })) 76117106792SAlexander Shaposhnikov return; 76217106792SAlexander Shaposhnikov 763*6292a808SJeremy Morse IRBuilder<> Builder(&F.getEntryBlock(), F.getEntryBlock().getFirstNonPHIIt()); 76417106792SAlexander Shaposhnikov // The function has shadow args if the shadow args tag matches the function 76517106792SAlexander Shaposhnikov // address. 76617106792SAlexander Shaposhnikov Value *HasShadowArgs = Builder.CreateICmpEQ( 76717106792SAlexander Shaposhnikov Builder.CreateLoad(IntptrTy, NsanShadowArgsTag, /*isVolatile=*/false), 76817106792SAlexander Shaposhnikov Builder.CreatePtrToInt(&F, IntptrTy)); 76917106792SAlexander Shaposhnikov 77017106792SAlexander Shaposhnikov unsigned ShadowArgsOffsetBytes = 0; 77117106792SAlexander Shaposhnikov for (Argument &Arg : F.args()) { 77217106792SAlexander Shaposhnikov Type *VT = Arg.getType(); 77317106792SAlexander Shaposhnikov Type *ExtendedVT = Config.getExtendedFPType(VT); 77417106792SAlexander Shaposhnikov if (ExtendedVT == nullptr) 77517106792SAlexander Shaposhnikov continue; // Not an FT value. 77617106792SAlexander Shaposhnikov Value *L = Builder.CreateAlignedLoad( 77717106792SAlexander Shaposhnikov ExtendedVT, 77817106792SAlexander Shaposhnikov Builder.CreateConstGEP2_64(NsanShadowArgsType, NsanShadowArgsPtr, 0, 77917106792SAlexander Shaposhnikov ShadowArgsOffsetBytes), 78017106792SAlexander Shaposhnikov Align(1), /*isVolatile=*/false); 78117106792SAlexander Shaposhnikov Value *Shadow = Builder.CreateSelect(HasShadowArgs, L, 78217106792SAlexander Shaposhnikov Builder.CreateFPExt(&Arg, ExtendedVT)); 78317106792SAlexander Shaposhnikov Map.setShadow(Arg, *Shadow); 78417106792SAlexander Shaposhnikov TypeSize SlotSize = DL.getTypeStoreSize(ExtendedVT); 78517106792SAlexander Shaposhnikov assert(!SlotSize.isScalable() && "unsupported"); 78617106792SAlexander Shaposhnikov ShadowArgsOffsetBytes += SlotSize; 78717106792SAlexander Shaposhnikov } 78817106792SAlexander Shaposhnikov Builder.CreateStore(ConstantInt::get(IntptrTy, 0), NsanShadowArgsTag); 78917106792SAlexander Shaposhnikov } 79017106792SAlexander Shaposhnikov 79117106792SAlexander Shaposhnikov // Returns true if the instrumentation should emit code to check arguments 79217106792SAlexander Shaposhnikov // before a function call. 79317106792SAlexander Shaposhnikov static bool shouldCheckArgs(CallBase &CI, const TargetLibraryInfo &TLI, 79417106792SAlexander Shaposhnikov const std::optional<Regex> &CheckFunctionsFilter) { 79517106792SAlexander Shaposhnikov 79617106792SAlexander Shaposhnikov Function *Fn = CI.getCalledFunction(); 79717106792SAlexander Shaposhnikov 79817106792SAlexander Shaposhnikov if (CheckFunctionsFilter) { 79917106792SAlexander Shaposhnikov // Skip checking args of indirect calls. 80017106792SAlexander Shaposhnikov if (Fn == nullptr) 80117106792SAlexander Shaposhnikov return false; 80217106792SAlexander Shaposhnikov if (CheckFunctionsFilter->match(Fn->getName())) 80317106792SAlexander Shaposhnikov return true; 80417106792SAlexander Shaposhnikov return false; 80517106792SAlexander Shaposhnikov } 80617106792SAlexander Shaposhnikov 80717106792SAlexander Shaposhnikov if (Fn == nullptr) 80817106792SAlexander Shaposhnikov return true; // Always check args of indirect calls. 80917106792SAlexander Shaposhnikov 81017106792SAlexander Shaposhnikov // Never check nsan functions, the user called them for a reason. 81117106792SAlexander Shaposhnikov if (Fn->getName().starts_with("__nsan_")) 81217106792SAlexander Shaposhnikov return false; 81317106792SAlexander Shaposhnikov 81417106792SAlexander Shaposhnikov const auto ID = Fn->getIntrinsicID(); 81517106792SAlexander Shaposhnikov LibFunc LFunc = LibFunc::NumLibFuncs; 81617106792SAlexander Shaposhnikov // Always check args of unknown functions. 81717106792SAlexander Shaposhnikov if (ID == Intrinsic::ID() && !TLI.getLibFunc(*Fn, LFunc)) 81817106792SAlexander Shaposhnikov return true; 81917106792SAlexander Shaposhnikov 82017106792SAlexander Shaposhnikov // Do not check args of an `fabs` call that is used for a comparison. 82117106792SAlexander Shaposhnikov // This is typically used for `fabs(a-b) < tolerance`, where what matters is 82217106792SAlexander Shaposhnikov // the result of the comparison, which is already caught be the fcmp checks. 82317106792SAlexander Shaposhnikov if (ID == Intrinsic::fabs || LFunc == LibFunc_fabsf || 82417106792SAlexander Shaposhnikov LFunc == LibFunc_fabs || LFunc == LibFunc_fabsl) 82517106792SAlexander Shaposhnikov for (const auto &U : CI.users()) 82617106792SAlexander Shaposhnikov if (isa<CmpInst>(U)) 82717106792SAlexander Shaposhnikov return false; 82817106792SAlexander Shaposhnikov 82917106792SAlexander Shaposhnikov return true; // Default is check. 83017106792SAlexander Shaposhnikov } 83117106792SAlexander Shaposhnikov 83217106792SAlexander Shaposhnikov // Populates the shadow call stack (which contains shadow values for every 83317106792SAlexander Shaposhnikov // floating-point parameter to the function). 83417106792SAlexander Shaposhnikov void NumericalStabilitySanitizer::populateShadowStack( 83517106792SAlexander Shaposhnikov CallBase &CI, const TargetLibraryInfo &TLI, const ValueToShadowMap &Map) { 83617106792SAlexander Shaposhnikov // Do not create a shadow stack for inline asm. 83717106792SAlexander Shaposhnikov if (CI.isInlineAsm()) 83817106792SAlexander Shaposhnikov return; 83917106792SAlexander Shaposhnikov 84017106792SAlexander Shaposhnikov // Do not bother if there are no FP args. 84117106792SAlexander Shaposhnikov if (all_of(CI.operands(), [this](const Value *Arg) { 84217106792SAlexander Shaposhnikov return Config.getExtendedFPType(Arg->getType()) == nullptr; 84317106792SAlexander Shaposhnikov })) 84417106792SAlexander Shaposhnikov return; 84517106792SAlexander Shaposhnikov 84617106792SAlexander Shaposhnikov IRBuilder<> Builder(&CI); 84717106792SAlexander Shaposhnikov SmallVector<Value *, 8> ArgShadows; 84817106792SAlexander Shaposhnikov const bool ShouldCheckArgs = shouldCheckArgs(CI, TLI, CheckFunctionsFilter); 84917106792SAlexander Shaposhnikov for (auto [ArgIdx, Arg] : enumerate(CI.operands())) { 85017106792SAlexander Shaposhnikov if (Config.getExtendedFPType(Arg->getType()) == nullptr) 85117106792SAlexander Shaposhnikov continue; // Not an FT value. 85217106792SAlexander Shaposhnikov Value *ArgShadow = Map.getShadow(Arg); 85317106792SAlexander Shaposhnikov ArgShadows.push_back(ShouldCheckArgs ? emitCheck(Arg, ArgShadow, Builder, 85417106792SAlexander Shaposhnikov CheckLoc::makeArg(ArgIdx)) 85517106792SAlexander Shaposhnikov : ArgShadow); 85617106792SAlexander Shaposhnikov } 85717106792SAlexander Shaposhnikov 85817106792SAlexander Shaposhnikov // Do not create shadow stacks for intrinsics/known lib funcs. 85917106792SAlexander Shaposhnikov if (Function *Fn = CI.getCalledFunction()) { 86017106792SAlexander Shaposhnikov LibFunc LFunc; 86117106792SAlexander Shaposhnikov if (Fn->isIntrinsic() || TLI.getLibFunc(*Fn, LFunc)) 86217106792SAlexander Shaposhnikov return; 86317106792SAlexander Shaposhnikov } 86417106792SAlexander Shaposhnikov 86517106792SAlexander Shaposhnikov // Set the shadow stack tag. 86617106792SAlexander Shaposhnikov Builder.CreateStore(CI.getCalledOperand(), NsanShadowArgsTag); 86717106792SAlexander Shaposhnikov TypeSize ShadowArgsOffsetBytes = TypeSize::getFixed(0); 86817106792SAlexander Shaposhnikov 86917106792SAlexander Shaposhnikov unsigned ShadowArgId = 0; 87017106792SAlexander Shaposhnikov for (const Value *Arg : CI.operands()) { 87117106792SAlexander Shaposhnikov Type *VT = Arg->getType(); 87217106792SAlexander Shaposhnikov Type *ExtendedVT = Config.getExtendedFPType(VT); 87317106792SAlexander Shaposhnikov if (ExtendedVT == nullptr) 87417106792SAlexander Shaposhnikov continue; // Not an FT value. 87517106792SAlexander Shaposhnikov Builder.CreateAlignedStore( 87617106792SAlexander Shaposhnikov ArgShadows[ShadowArgId++], 87717106792SAlexander Shaposhnikov Builder.CreateConstGEP2_64(NsanShadowArgsType, NsanShadowArgsPtr, 0, 87817106792SAlexander Shaposhnikov ShadowArgsOffsetBytes), 87917106792SAlexander Shaposhnikov Align(1), /*isVolatile=*/false); 88017106792SAlexander Shaposhnikov TypeSize SlotSize = DL.getTypeStoreSize(ExtendedVT); 88117106792SAlexander Shaposhnikov assert(!SlotSize.isScalable() && "unsupported"); 88217106792SAlexander Shaposhnikov ShadowArgsOffsetBytes += SlotSize; 88317106792SAlexander Shaposhnikov } 88417106792SAlexander Shaposhnikov } 88517106792SAlexander Shaposhnikov 88617106792SAlexander Shaposhnikov // Internal part of emitCheck(). Returns a value that indicates whether 88717106792SAlexander Shaposhnikov // computation should continue with the shadow or resume by re-fextending the 88817106792SAlexander Shaposhnikov // value. 88917106792SAlexander Shaposhnikov enum class ContinuationType { // Keep in sync with runtime. 89017106792SAlexander Shaposhnikov ContinueWithShadow = 0, 89117106792SAlexander Shaposhnikov ResumeFromValue = 1, 89217106792SAlexander Shaposhnikov }; 89317106792SAlexander Shaposhnikov 89417106792SAlexander Shaposhnikov Value *NumericalStabilitySanitizer::emitCheckInternal(Value *V, Value *ShadowV, 89517106792SAlexander Shaposhnikov IRBuilder<> &Builder, 89617106792SAlexander Shaposhnikov CheckLoc Loc) { 89717106792SAlexander Shaposhnikov // Do not emit checks for constant values, this is redundant. 89817106792SAlexander Shaposhnikov if (isa<Constant>(V)) 89917106792SAlexander Shaposhnikov return ConstantInt::get( 90017106792SAlexander Shaposhnikov Builder.getInt32Ty(), 90117106792SAlexander Shaposhnikov static_cast<int>(ContinuationType::ContinueWithShadow)); 90217106792SAlexander Shaposhnikov 90317106792SAlexander Shaposhnikov Type *Ty = V->getType(); 90417106792SAlexander Shaposhnikov if (const auto VT = ftValueTypeFromType(Ty)) 90517106792SAlexander Shaposhnikov return Builder.CreateCall( 90617106792SAlexander Shaposhnikov NsanCheckValue[*VT], 90717106792SAlexander Shaposhnikov {V, ShadowV, Loc.getType(Context), Loc.getValue(IntptrTy, Builder)}); 90817106792SAlexander Shaposhnikov 90917106792SAlexander Shaposhnikov if (Ty->isVectorTy()) { 91017106792SAlexander Shaposhnikov auto *VecTy = cast<VectorType>(Ty); 91117106792SAlexander Shaposhnikov // We currently skip scalable vector types in MappingConfig, 91217106792SAlexander Shaposhnikov // thus we should not encounter any such types here. 91317106792SAlexander Shaposhnikov assert(!VecTy->isScalableTy() && 91417106792SAlexander Shaposhnikov "Scalable vector types are not supported yet"); 91517106792SAlexander Shaposhnikov Value *CheckResult = nullptr; 91617106792SAlexander Shaposhnikov for (int I = 0, E = VecTy->getElementCount().getFixedValue(); I < E; ++I) { 91717106792SAlexander Shaposhnikov // We resume if any element resumes. Another option would be to create a 91817106792SAlexander Shaposhnikov // vector shuffle with the array of ContinueWithShadow, but that is too 91917106792SAlexander Shaposhnikov // complex. 92017106792SAlexander Shaposhnikov Value *ExtractV = Builder.CreateExtractElement(V, I); 92117106792SAlexander Shaposhnikov Value *ExtractShadowV = Builder.CreateExtractElement(ShadowV, I); 92217106792SAlexander Shaposhnikov Value *ComponentCheckResult = 92317106792SAlexander Shaposhnikov emitCheckInternal(ExtractV, ExtractShadowV, Builder, Loc); 92417106792SAlexander Shaposhnikov CheckResult = CheckResult 92517106792SAlexander Shaposhnikov ? Builder.CreateOr(CheckResult, ComponentCheckResult) 92617106792SAlexander Shaposhnikov : ComponentCheckResult; 92717106792SAlexander Shaposhnikov } 92817106792SAlexander Shaposhnikov return CheckResult; 92917106792SAlexander Shaposhnikov } 93017106792SAlexander Shaposhnikov if (Ty->isArrayTy()) { 93117106792SAlexander Shaposhnikov Value *CheckResult = nullptr; 93217106792SAlexander Shaposhnikov for (auto I : seq(Ty->getArrayNumElements())) { 93317106792SAlexander Shaposhnikov Value *ExtractV = Builder.CreateExtractElement(V, I); 93417106792SAlexander Shaposhnikov Value *ExtractShadowV = Builder.CreateExtractElement(ShadowV, I); 93517106792SAlexander Shaposhnikov Value *ComponentCheckResult = 93617106792SAlexander Shaposhnikov emitCheckInternal(ExtractV, ExtractShadowV, Builder, Loc); 93717106792SAlexander Shaposhnikov CheckResult = CheckResult 93817106792SAlexander Shaposhnikov ? Builder.CreateOr(CheckResult, ComponentCheckResult) 93917106792SAlexander Shaposhnikov : ComponentCheckResult; 94017106792SAlexander Shaposhnikov } 94117106792SAlexander Shaposhnikov return CheckResult; 94217106792SAlexander Shaposhnikov } 94317106792SAlexander Shaposhnikov if (Ty->isStructTy()) { 94417106792SAlexander Shaposhnikov Value *CheckResult = nullptr; 94517106792SAlexander Shaposhnikov for (auto I : seq(Ty->getStructNumElements())) { 94617106792SAlexander Shaposhnikov if (Config.getExtendedFPType(Ty->getStructElementType(I)) == nullptr) 94717106792SAlexander Shaposhnikov continue; // Only check FT values. 94817106792SAlexander Shaposhnikov Value *ExtractV = Builder.CreateExtractValue(V, I); 94917106792SAlexander Shaposhnikov Value *ExtractShadowV = Builder.CreateExtractElement(ShadowV, I); 95017106792SAlexander Shaposhnikov Value *ComponentCheckResult = 95117106792SAlexander Shaposhnikov emitCheckInternal(ExtractV, ExtractShadowV, Builder, Loc); 95217106792SAlexander Shaposhnikov CheckResult = CheckResult 95317106792SAlexander Shaposhnikov ? Builder.CreateOr(CheckResult, ComponentCheckResult) 95417106792SAlexander Shaposhnikov : ComponentCheckResult; 95517106792SAlexander Shaposhnikov } 95617106792SAlexander Shaposhnikov if (!CheckResult) 95717106792SAlexander Shaposhnikov return ConstantInt::get( 95817106792SAlexander Shaposhnikov Builder.getInt32Ty(), 95917106792SAlexander Shaposhnikov static_cast<int>(ContinuationType::ContinueWithShadow)); 96017106792SAlexander Shaposhnikov return CheckResult; 96117106792SAlexander Shaposhnikov } 96217106792SAlexander Shaposhnikov 96317106792SAlexander Shaposhnikov llvm_unreachable("not implemented"); 96417106792SAlexander Shaposhnikov } 96517106792SAlexander Shaposhnikov 96617106792SAlexander Shaposhnikov // Inserts a runtime check of V against its shadow value ShadowV. 96717106792SAlexander Shaposhnikov // We check values whenever they escape: on return, call, stores, and 96817106792SAlexander Shaposhnikov // insertvalue. 96917106792SAlexander Shaposhnikov // Returns the shadow value that should be used to continue the computations, 97017106792SAlexander Shaposhnikov // depending on the answer from the runtime. 97117106792SAlexander Shaposhnikov // TODO: Should we check on select ? phi ? 97217106792SAlexander Shaposhnikov Value *NumericalStabilitySanitizer::emitCheck(Value *V, Value *ShadowV, 97317106792SAlexander Shaposhnikov IRBuilder<> &Builder, 97417106792SAlexander Shaposhnikov CheckLoc Loc) { 97517106792SAlexander Shaposhnikov // Do not emit checks for constant values, this is redundant. 97617106792SAlexander Shaposhnikov if (isa<Constant>(V)) 97717106792SAlexander Shaposhnikov return ShadowV; 97817106792SAlexander Shaposhnikov 97917106792SAlexander Shaposhnikov if (Instruction *Inst = dyn_cast<Instruction>(V)) { 98017106792SAlexander Shaposhnikov Function *F = Inst->getFunction(); 98117106792SAlexander Shaposhnikov if (CheckFunctionsFilter && !CheckFunctionsFilter->match(F->getName())) { 98217106792SAlexander Shaposhnikov return ShadowV; 98317106792SAlexander Shaposhnikov } 98417106792SAlexander Shaposhnikov } 98517106792SAlexander Shaposhnikov 98617106792SAlexander Shaposhnikov Value *CheckResult = emitCheckInternal(V, ShadowV, Builder, Loc); 98717106792SAlexander Shaposhnikov Value *ICmpEQ = Builder.CreateICmpEQ( 98817106792SAlexander Shaposhnikov CheckResult, 98917106792SAlexander Shaposhnikov ConstantInt::get(Builder.getInt32Ty(), 99017106792SAlexander Shaposhnikov static_cast<int>(ContinuationType::ResumeFromValue))); 99117106792SAlexander Shaposhnikov return Builder.CreateSelect( 99217106792SAlexander Shaposhnikov ICmpEQ, Builder.CreateFPExt(V, Config.getExtendedFPType(V->getType())), 99317106792SAlexander Shaposhnikov ShadowV); 99417106792SAlexander Shaposhnikov } 99517106792SAlexander Shaposhnikov 99617106792SAlexander Shaposhnikov // Inserts a check that fcmp on shadow values are consistent with that on base 99717106792SAlexander Shaposhnikov // values. 99817106792SAlexander Shaposhnikov void NumericalStabilitySanitizer::emitFCmpCheck(FCmpInst &FCmp, 99917106792SAlexander Shaposhnikov const ValueToShadowMap &Map) { 100017106792SAlexander Shaposhnikov if (!ClInstrumentFCmp) 100117106792SAlexander Shaposhnikov return; 100217106792SAlexander Shaposhnikov 100317106792SAlexander Shaposhnikov Function *F = FCmp.getFunction(); 100417106792SAlexander Shaposhnikov if (CheckFunctionsFilter && !CheckFunctionsFilter->match(F->getName())) 100517106792SAlexander Shaposhnikov return; 100617106792SAlexander Shaposhnikov 100717106792SAlexander Shaposhnikov Value *LHS = FCmp.getOperand(0); 100817106792SAlexander Shaposhnikov if (Config.getExtendedFPType(LHS->getType()) == nullptr) 100917106792SAlexander Shaposhnikov return; 101017106792SAlexander Shaposhnikov Value *RHS = FCmp.getOperand(1); 101117106792SAlexander Shaposhnikov 101217106792SAlexander Shaposhnikov // Split the basic block. On mismatch, we'll jump to the new basic block with 101317106792SAlexander Shaposhnikov // a call to the runtime for error reporting. 101417106792SAlexander Shaposhnikov BasicBlock *FCmpBB = FCmp.getParent(); 101517106792SAlexander Shaposhnikov BasicBlock *NextBB = FCmpBB->splitBasicBlock(FCmp.getNextNode()); 101617106792SAlexander Shaposhnikov // Remove the newly created terminator unconditional branch. 101717106792SAlexander Shaposhnikov FCmpBB->back().eraseFromParent(); 101817106792SAlexander Shaposhnikov BasicBlock *FailBB = 101917106792SAlexander Shaposhnikov BasicBlock::Create(Context, "", FCmpBB->getParent(), NextBB); 102017106792SAlexander Shaposhnikov 102117106792SAlexander Shaposhnikov // Create the shadow fcmp and comparison between the fcmps. 102217106792SAlexander Shaposhnikov IRBuilder<> FCmpBuilder(FCmpBB); 102317106792SAlexander Shaposhnikov FCmpBuilder.SetCurrentDebugLocation(FCmp.getDebugLoc()); 102417106792SAlexander Shaposhnikov Value *ShadowLHS = Map.getShadow(LHS); 102517106792SAlexander Shaposhnikov Value *ShadowRHS = Map.getShadow(RHS); 102617106792SAlexander Shaposhnikov // See comment on ClTruncateFCmpEq. 102717106792SAlexander Shaposhnikov if (FCmp.isEquality() && ClTruncateFCmpEq) { 102817106792SAlexander Shaposhnikov Type *Ty = ShadowLHS->getType(); 102917106792SAlexander Shaposhnikov ShadowLHS = FCmpBuilder.CreateFPExt( 103017106792SAlexander Shaposhnikov FCmpBuilder.CreateFPTrunc(ShadowLHS, LHS->getType()), Ty); 103117106792SAlexander Shaposhnikov ShadowRHS = FCmpBuilder.CreateFPExt( 103217106792SAlexander Shaposhnikov FCmpBuilder.CreateFPTrunc(ShadowRHS, RHS->getType()), Ty); 103317106792SAlexander Shaposhnikov } 103417106792SAlexander Shaposhnikov Value *ShadowFCmp = 103517106792SAlexander Shaposhnikov FCmpBuilder.CreateFCmp(FCmp.getPredicate(), ShadowLHS, ShadowRHS); 103617106792SAlexander Shaposhnikov Value *OriginalAndShadowFcmpMatch = 103717106792SAlexander Shaposhnikov FCmpBuilder.CreateICmpEQ(&FCmp, ShadowFCmp); 103817106792SAlexander Shaposhnikov 103917106792SAlexander Shaposhnikov if (OriginalAndShadowFcmpMatch->getType()->isVectorTy()) { 104017106792SAlexander Shaposhnikov // If we have a vector type, `OriginalAndShadowFcmpMatch` is a vector of i1, 104117106792SAlexander Shaposhnikov // where an element is true if the corresponding elements in original and 104217106792SAlexander Shaposhnikov // shadow are the same. We want all elements to be 1. 104317106792SAlexander Shaposhnikov OriginalAndShadowFcmpMatch = 104417106792SAlexander Shaposhnikov FCmpBuilder.CreateAndReduce(OriginalAndShadowFcmpMatch); 104517106792SAlexander Shaposhnikov } 104617106792SAlexander Shaposhnikov 104717106792SAlexander Shaposhnikov // Use MDBuilder(*C).createLikelyBranchWeights() because "match" is the common 104817106792SAlexander Shaposhnikov // case. 104917106792SAlexander Shaposhnikov FCmpBuilder.CreateCondBr(OriginalAndShadowFcmpMatch, NextBB, FailBB, 105017106792SAlexander Shaposhnikov MDBuilder(Context).createLikelyBranchWeights()); 105117106792SAlexander Shaposhnikov 105217106792SAlexander Shaposhnikov // Fill in FailBB. 105317106792SAlexander Shaposhnikov IRBuilder<> FailBuilder(FailBB); 105417106792SAlexander Shaposhnikov FailBuilder.SetCurrentDebugLocation(FCmp.getDebugLoc()); 105517106792SAlexander Shaposhnikov 105617106792SAlexander Shaposhnikov const auto EmitFailCall = [this, &FCmp, &FCmpBuilder, 105717106792SAlexander Shaposhnikov &FailBuilder](Value *L, Value *R, Value *ShadowL, 105817106792SAlexander Shaposhnikov Value *ShadowR, Value *Result, 105917106792SAlexander Shaposhnikov Value *ShadowResult) { 106017106792SAlexander Shaposhnikov Type *FT = L->getType(); 106117106792SAlexander Shaposhnikov FunctionCallee *Callee = nullptr; 106217106792SAlexander Shaposhnikov if (FT->isFloatTy()) { 106317106792SAlexander Shaposhnikov Callee = &(NsanFCmpFail[kFloat]); 106417106792SAlexander Shaposhnikov } else if (FT->isDoubleTy()) { 106517106792SAlexander Shaposhnikov Callee = &(NsanFCmpFail[kDouble]); 106617106792SAlexander Shaposhnikov } else if (FT->isX86_FP80Ty()) { 106717106792SAlexander Shaposhnikov // TODO: make NsanFCmpFailLongDouble work. 106817106792SAlexander Shaposhnikov Callee = &(NsanFCmpFail[kDouble]); 106917106792SAlexander Shaposhnikov L = FailBuilder.CreateFPTrunc(L, Type::getDoubleTy(Context)); 107017106792SAlexander Shaposhnikov R = FailBuilder.CreateFPTrunc(L, Type::getDoubleTy(Context)); 107117106792SAlexander Shaposhnikov } else { 107217106792SAlexander Shaposhnikov llvm_unreachable("not implemented"); 107317106792SAlexander Shaposhnikov } 107417106792SAlexander Shaposhnikov FailBuilder.CreateCall(*Callee, {L, R, ShadowL, ShadowR, 107517106792SAlexander Shaposhnikov ConstantInt::get(FCmpBuilder.getInt32Ty(), 107617106792SAlexander Shaposhnikov FCmp.getPredicate()), 107717106792SAlexander Shaposhnikov Result, ShadowResult}); 107817106792SAlexander Shaposhnikov }; 107917106792SAlexander Shaposhnikov if (LHS->getType()->isVectorTy()) { 108017106792SAlexander Shaposhnikov for (int I = 0, E = cast<VectorType>(LHS->getType()) 108117106792SAlexander Shaposhnikov ->getElementCount() 108217106792SAlexander Shaposhnikov .getFixedValue(); 108317106792SAlexander Shaposhnikov I < E; ++I) { 108417106792SAlexander Shaposhnikov Value *ExtractLHS = FailBuilder.CreateExtractElement(LHS, I); 108517106792SAlexander Shaposhnikov Value *ExtractRHS = FailBuilder.CreateExtractElement(RHS, I); 108617106792SAlexander Shaposhnikov Value *ExtractShaodwLHS = FailBuilder.CreateExtractElement(ShadowLHS, I); 108717106792SAlexander Shaposhnikov Value *ExtractShaodwRHS = FailBuilder.CreateExtractElement(ShadowRHS, I); 108817106792SAlexander Shaposhnikov Value *ExtractFCmp = FailBuilder.CreateExtractElement(&FCmp, I); 108917106792SAlexander Shaposhnikov Value *ExtractShadowFCmp = 109017106792SAlexander Shaposhnikov FailBuilder.CreateExtractElement(ShadowFCmp, I); 109117106792SAlexander Shaposhnikov EmitFailCall(ExtractLHS, ExtractRHS, ExtractShaodwLHS, ExtractShaodwRHS, 109217106792SAlexander Shaposhnikov ExtractFCmp, ExtractShadowFCmp); 109317106792SAlexander Shaposhnikov } 109417106792SAlexander Shaposhnikov } else { 109517106792SAlexander Shaposhnikov EmitFailCall(LHS, RHS, ShadowLHS, ShadowRHS, &FCmp, ShadowFCmp); 109617106792SAlexander Shaposhnikov } 109717106792SAlexander Shaposhnikov FailBuilder.CreateBr(NextBB); 109817106792SAlexander Shaposhnikov 109917106792SAlexander Shaposhnikov ++NumInstrumentedFCmp; 110017106792SAlexander Shaposhnikov } 110117106792SAlexander Shaposhnikov 110217106792SAlexander Shaposhnikov // Creates a shadow phi value for any phi that defines a value of FT type. 110317106792SAlexander Shaposhnikov PHINode *NumericalStabilitySanitizer::maybeCreateShadowPhi( 110417106792SAlexander Shaposhnikov PHINode &Phi, const TargetLibraryInfo &TLI) { 110517106792SAlexander Shaposhnikov Type *VT = Phi.getType(); 110617106792SAlexander Shaposhnikov Type *ExtendedVT = Config.getExtendedFPType(VT); 110717106792SAlexander Shaposhnikov if (ExtendedVT == nullptr) 110817106792SAlexander Shaposhnikov return nullptr; // Not an FT value. 110917106792SAlexander Shaposhnikov // The phi operands are shadow values and are not available when the phi is 111017106792SAlexander Shaposhnikov // created. They will be populated in a final phase, once all shadow values 111117106792SAlexander Shaposhnikov // have been created. 111217106792SAlexander Shaposhnikov PHINode *Shadow = PHINode::Create(ExtendedVT, Phi.getNumIncomingValues()); 11138e702735SJeremy Morse Shadow->insertAfter(Phi.getIterator()); 111417106792SAlexander Shaposhnikov return Shadow; 111517106792SAlexander Shaposhnikov } 111617106792SAlexander Shaposhnikov 111717106792SAlexander Shaposhnikov Value *NumericalStabilitySanitizer::handleLoad(LoadInst &Load, Type *VT, 111817106792SAlexander Shaposhnikov Type *ExtendedVT) { 111917106792SAlexander Shaposhnikov IRBuilder<> Builder(Load.getNextNode()); 112017106792SAlexander Shaposhnikov Builder.SetCurrentDebugLocation(Load.getDebugLoc()); 112117106792SAlexander Shaposhnikov if (addrPointsToConstantData(Load.getPointerOperand())) { 112217106792SAlexander Shaposhnikov // No need to look into the shadow memory, the value is a constant. Just 112317106792SAlexander Shaposhnikov // convert from FT to 2FT. 112417106792SAlexander Shaposhnikov return Builder.CreateFPExt(&Load, ExtendedVT); 112517106792SAlexander Shaposhnikov } 112617106792SAlexander Shaposhnikov 112717106792SAlexander Shaposhnikov // if (%shadowptr == &) 112817106792SAlexander Shaposhnikov // %shadow = fpext %v 112917106792SAlexander Shaposhnikov // else 113017106792SAlexander Shaposhnikov // %shadow = load (ptrcast %shadow_ptr)) 113117106792SAlexander Shaposhnikov // Considered options here: 113217106792SAlexander Shaposhnikov // - Have `NsanGetShadowPtrForLoad` return a fixed address 113317106792SAlexander Shaposhnikov // &__nsan_unknown_value_shadow_address that is valid to load from, and 113417106792SAlexander Shaposhnikov // use a select. This has the advantage that the generated IR is simpler. 113517106792SAlexander Shaposhnikov // - Have `NsanGetShadowPtrForLoad` return nullptr. Because `select` does 113617106792SAlexander Shaposhnikov // not short-circuit, dereferencing the returned pointer is no longer an 113717106792SAlexander Shaposhnikov // option, have to split and create a separate basic block. This has the 113817106792SAlexander Shaposhnikov // advantage of being easier to debug because it crashes if we ever mess 113917106792SAlexander Shaposhnikov // up. 114017106792SAlexander Shaposhnikov 114117106792SAlexander Shaposhnikov const auto Extents = getMemoryExtentsOrDie(VT); 114217106792SAlexander Shaposhnikov Value *ShadowPtr = Builder.CreateCall( 114317106792SAlexander Shaposhnikov NsanGetShadowPtrForLoad[Extents.ValueType], 114417106792SAlexander Shaposhnikov {Load.getPointerOperand(), ConstantInt::get(IntptrTy, Extents.NumElts)}); 114517106792SAlexander Shaposhnikov ++NumInstrumentedFTLoads; 114617106792SAlexander Shaposhnikov 114717106792SAlexander Shaposhnikov // Split the basic block. 114817106792SAlexander Shaposhnikov BasicBlock *LoadBB = Load.getParent(); 114917106792SAlexander Shaposhnikov BasicBlock *NextBB = LoadBB->splitBasicBlock(Builder.GetInsertPoint()); 115017106792SAlexander Shaposhnikov // Create the two options for creating the shadow value. 115117106792SAlexander Shaposhnikov BasicBlock *ShadowLoadBB = 115217106792SAlexander Shaposhnikov BasicBlock::Create(Context, "", LoadBB->getParent(), NextBB); 115317106792SAlexander Shaposhnikov BasicBlock *FExtBB = 115417106792SAlexander Shaposhnikov BasicBlock::Create(Context, "", LoadBB->getParent(), NextBB); 115517106792SAlexander Shaposhnikov 115617106792SAlexander Shaposhnikov // Replace the newly created terminator unconditional branch by a conditional 115717106792SAlexander Shaposhnikov // branch to one of the options. 115817106792SAlexander Shaposhnikov { 115917106792SAlexander Shaposhnikov LoadBB->back().eraseFromParent(); 116017106792SAlexander Shaposhnikov IRBuilder<> LoadBBBuilder(LoadBB); // The old builder has been invalidated. 116117106792SAlexander Shaposhnikov LoadBBBuilder.SetCurrentDebugLocation(Load.getDebugLoc()); 116217106792SAlexander Shaposhnikov LoadBBBuilder.CreateCondBr(LoadBBBuilder.CreateIsNull(ShadowPtr), FExtBB, 116317106792SAlexander Shaposhnikov ShadowLoadBB); 116417106792SAlexander Shaposhnikov } 116517106792SAlexander Shaposhnikov 116617106792SAlexander Shaposhnikov // Fill in ShadowLoadBB. 116717106792SAlexander Shaposhnikov IRBuilder<> ShadowLoadBBBuilder(ShadowLoadBB); 116817106792SAlexander Shaposhnikov ShadowLoadBBBuilder.SetCurrentDebugLocation(Load.getDebugLoc()); 116917106792SAlexander Shaposhnikov Value *ShadowLoad = ShadowLoadBBBuilder.CreateAlignedLoad( 117017106792SAlexander Shaposhnikov ExtendedVT, ShadowPtr, Align(1), Load.isVolatile()); 117117106792SAlexander Shaposhnikov if (ClCheckLoads) { 117217106792SAlexander Shaposhnikov ShadowLoad = emitCheck(&Load, ShadowLoad, ShadowLoadBBBuilder, 117317106792SAlexander Shaposhnikov CheckLoc::makeLoad(Load.getPointerOperand())); 117417106792SAlexander Shaposhnikov } 117517106792SAlexander Shaposhnikov ShadowLoadBBBuilder.CreateBr(NextBB); 117617106792SAlexander Shaposhnikov 117717106792SAlexander Shaposhnikov // Fill in FExtBB. 117817106792SAlexander Shaposhnikov IRBuilder<> FExtBBBuilder(FExtBB); 117917106792SAlexander Shaposhnikov FExtBBBuilder.SetCurrentDebugLocation(Load.getDebugLoc()); 118017106792SAlexander Shaposhnikov Value *FExt = FExtBBBuilder.CreateFPExt(&Load, ExtendedVT); 118117106792SAlexander Shaposhnikov FExtBBBuilder.CreateBr(NextBB); 118217106792SAlexander Shaposhnikov 118317106792SAlexander Shaposhnikov // The shadow value come from any of the options. 118417106792SAlexander Shaposhnikov IRBuilder<> NextBBBuilder(&*NextBB->begin()); 118517106792SAlexander Shaposhnikov NextBBBuilder.SetCurrentDebugLocation(Load.getDebugLoc()); 118617106792SAlexander Shaposhnikov PHINode *ShadowPhi = NextBBBuilder.CreatePHI(ExtendedVT, 2); 118717106792SAlexander Shaposhnikov ShadowPhi->addIncoming(ShadowLoad, ShadowLoadBB); 118817106792SAlexander Shaposhnikov ShadowPhi->addIncoming(FExt, FExtBB); 118917106792SAlexander Shaposhnikov return ShadowPhi; 119017106792SAlexander Shaposhnikov } 119117106792SAlexander Shaposhnikov 119217106792SAlexander Shaposhnikov Value *NumericalStabilitySanitizer::handleTrunc(const FPTruncInst &Trunc, 119317106792SAlexander Shaposhnikov Type *VT, Type *ExtendedVT, 119417106792SAlexander Shaposhnikov const ValueToShadowMap &Map, 119517106792SAlexander Shaposhnikov IRBuilder<> &Builder) { 119617106792SAlexander Shaposhnikov Value *OrigSource = Trunc.getOperand(0); 119717106792SAlexander Shaposhnikov Type *OrigSourceTy = OrigSource->getType(); 119817106792SAlexander Shaposhnikov Type *ExtendedSourceTy = Config.getExtendedFPType(OrigSourceTy); 119917106792SAlexander Shaposhnikov 120017106792SAlexander Shaposhnikov // When truncating: 120117106792SAlexander Shaposhnikov // - (A) If the source has a shadow, we truncate from the shadow, else we 120217106792SAlexander Shaposhnikov // truncate from the original source. 120317106792SAlexander Shaposhnikov // - (B) If the shadow of the source is larger than the shadow of the dest, 120417106792SAlexander Shaposhnikov // we still need a truncate. Else, the shadow of the source is the same 120517106792SAlexander Shaposhnikov // type as the shadow of the dest (because mappings are non-decreasing), so 120617106792SAlexander Shaposhnikov // we don't need to emit a truncate. 120717106792SAlexander Shaposhnikov // Examples, 120817106792SAlexander Shaposhnikov // with a mapping of {f32->f64;f64->f80;f80->f128} 120917106792SAlexander Shaposhnikov // fptrunc double %1 to float -> fptrunc x86_fp80 s(%1) to double 121017106792SAlexander Shaposhnikov // fptrunc x86_fp80 %1 to float -> fptrunc fp128 s(%1) to double 121117106792SAlexander Shaposhnikov // fptrunc fp128 %1 to float -> fptrunc fp128 %1 to double 121217106792SAlexander Shaposhnikov // fptrunc x86_fp80 %1 to double -> x86_fp80 s(%1) 121317106792SAlexander Shaposhnikov // fptrunc fp128 %1 to double -> fptrunc fp128 %1 to x86_fp80 121417106792SAlexander Shaposhnikov // fptrunc fp128 %1 to x86_fp80 -> fp128 %1 121517106792SAlexander Shaposhnikov // with a mapping of {f32->f64;f64->f128;f80->f128} 121617106792SAlexander Shaposhnikov // fptrunc double %1 to float -> fptrunc fp128 s(%1) to double 121717106792SAlexander Shaposhnikov // fptrunc x86_fp80 %1 to float -> fptrunc fp128 s(%1) to double 121817106792SAlexander Shaposhnikov // fptrunc fp128 %1 to float -> fptrunc fp128 %1 to double 121917106792SAlexander Shaposhnikov // fptrunc x86_fp80 %1 to double -> fp128 %1 122017106792SAlexander Shaposhnikov // fptrunc fp128 %1 to double -> fp128 %1 122117106792SAlexander Shaposhnikov // fptrunc fp128 %1 to x86_fp80 -> fp128 %1 122217106792SAlexander Shaposhnikov // with a mapping of {f32->f32;f64->f32;f80->f64} 122317106792SAlexander Shaposhnikov // fptrunc double %1 to float -> float s(%1) 122417106792SAlexander Shaposhnikov // fptrunc x86_fp80 %1 to float -> fptrunc double s(%1) to float 122517106792SAlexander Shaposhnikov // fptrunc fp128 %1 to float -> fptrunc fp128 %1 to float 122617106792SAlexander Shaposhnikov // fptrunc x86_fp80 %1 to double -> fptrunc double s(%1) to float 122717106792SAlexander Shaposhnikov // fptrunc fp128 %1 to double -> fptrunc fp128 %1 to float 122817106792SAlexander Shaposhnikov // fptrunc fp128 %1 to x86_fp80 -> fptrunc fp128 %1 to double 122917106792SAlexander Shaposhnikov 123017106792SAlexander Shaposhnikov // See (A) above. 123117106792SAlexander Shaposhnikov Value *Source = ExtendedSourceTy ? Map.getShadow(OrigSource) : OrigSource; 123217106792SAlexander Shaposhnikov Type *SourceTy = ExtendedSourceTy ? ExtendedSourceTy : OrigSourceTy; 123317106792SAlexander Shaposhnikov // See (B) above. 123417106792SAlexander Shaposhnikov if (SourceTy == ExtendedVT) 123517106792SAlexander Shaposhnikov return Source; 123617106792SAlexander Shaposhnikov 123717106792SAlexander Shaposhnikov return Builder.CreateFPTrunc(Source, ExtendedVT); 123817106792SAlexander Shaposhnikov } 123917106792SAlexander Shaposhnikov 124017106792SAlexander Shaposhnikov Value *NumericalStabilitySanitizer::handleExt(const FPExtInst &Ext, Type *VT, 124117106792SAlexander Shaposhnikov Type *ExtendedVT, 124217106792SAlexander Shaposhnikov const ValueToShadowMap &Map, 124317106792SAlexander Shaposhnikov IRBuilder<> &Builder) { 124417106792SAlexander Shaposhnikov Value *OrigSource = Ext.getOperand(0); 124517106792SAlexander Shaposhnikov Type *OrigSourceTy = OrigSource->getType(); 124617106792SAlexander Shaposhnikov Type *ExtendedSourceTy = Config.getExtendedFPType(OrigSourceTy); 124717106792SAlexander Shaposhnikov // When extending: 124817106792SAlexander Shaposhnikov // - (A) If the source has a shadow, we extend from the shadow, else we 124917106792SAlexander Shaposhnikov // extend from the original source. 125017106792SAlexander Shaposhnikov // - (B) If the shadow of the dest is larger than the shadow of the source, 125117106792SAlexander Shaposhnikov // we still need an extend. Else, the shadow of the source is the same 125217106792SAlexander Shaposhnikov // type as the shadow of the dest (because mappings are non-decreasing), so 125317106792SAlexander Shaposhnikov // we don't need to emit an extend. 125417106792SAlexander Shaposhnikov // Examples, 125517106792SAlexander Shaposhnikov // with a mapping of {f32->f64;f64->f80;f80->f128} 125617106792SAlexander Shaposhnikov // fpext half %1 to float -> fpext half %1 to double 125717106792SAlexander Shaposhnikov // fpext half %1 to double -> fpext half %1 to x86_fp80 125817106792SAlexander Shaposhnikov // fpext half %1 to x86_fp80 -> fpext half %1 to fp128 125917106792SAlexander Shaposhnikov // fpext float %1 to double -> double s(%1) 126017106792SAlexander Shaposhnikov // fpext float %1 to x86_fp80 -> fpext double s(%1) to fp128 126117106792SAlexander Shaposhnikov // fpext double %1 to x86_fp80 -> fpext x86_fp80 s(%1) to fp128 126217106792SAlexander Shaposhnikov // with a mapping of {f32->f64;f64->f128;f80->f128} 126317106792SAlexander Shaposhnikov // fpext half %1 to float -> fpext half %1 to double 126417106792SAlexander Shaposhnikov // fpext half %1 to double -> fpext half %1 to fp128 126517106792SAlexander Shaposhnikov // fpext half %1 to x86_fp80 -> fpext half %1 to fp128 126617106792SAlexander Shaposhnikov // fpext float %1 to double -> fpext double s(%1) to fp128 126717106792SAlexander Shaposhnikov // fpext float %1 to x86_fp80 -> fpext double s(%1) to fp128 126817106792SAlexander Shaposhnikov // fpext double %1 to x86_fp80 -> fp128 s(%1) 126917106792SAlexander Shaposhnikov // with a mapping of {f32->f32;f64->f32;f80->f64} 127017106792SAlexander Shaposhnikov // fpext half %1 to float -> fpext half %1 to float 127117106792SAlexander Shaposhnikov // fpext half %1 to double -> fpext half %1 to float 127217106792SAlexander Shaposhnikov // fpext half %1 to x86_fp80 -> fpext half %1 to double 127317106792SAlexander Shaposhnikov // fpext float %1 to double -> s(%1) 127417106792SAlexander Shaposhnikov // fpext float %1 to x86_fp80 -> fpext float s(%1) to double 127517106792SAlexander Shaposhnikov // fpext double %1 to x86_fp80 -> fpext float s(%1) to double 127617106792SAlexander Shaposhnikov 127717106792SAlexander Shaposhnikov // See (A) above. 127817106792SAlexander Shaposhnikov Value *Source = ExtendedSourceTy ? Map.getShadow(OrigSource) : OrigSource; 127917106792SAlexander Shaposhnikov Type *SourceTy = ExtendedSourceTy ? ExtendedSourceTy : OrigSourceTy; 128017106792SAlexander Shaposhnikov // See (B) above. 128117106792SAlexander Shaposhnikov if (SourceTy == ExtendedVT) 128217106792SAlexander Shaposhnikov return Source; 128317106792SAlexander Shaposhnikov 128417106792SAlexander Shaposhnikov return Builder.CreateFPExt(Source, ExtendedVT); 128517106792SAlexander Shaposhnikov } 128617106792SAlexander Shaposhnikov 128717106792SAlexander Shaposhnikov namespace { 128817106792SAlexander Shaposhnikov // TODO: This should be tablegen-ed. 128917106792SAlexander Shaposhnikov struct KnownIntrinsic { 129017106792SAlexander Shaposhnikov struct WidenedIntrinsic { 129117106792SAlexander Shaposhnikov const char *NarrowName; 129217106792SAlexander Shaposhnikov Intrinsic::ID ID; // wide id. 129317106792SAlexander Shaposhnikov using FnTypeFactory = FunctionType *(*)(LLVMContext &); 129417106792SAlexander Shaposhnikov FnTypeFactory MakeFnTy; 129517106792SAlexander Shaposhnikov }; 129617106792SAlexander Shaposhnikov 129717106792SAlexander Shaposhnikov static const char *get(LibFunc LFunc); 129817106792SAlexander Shaposhnikov 129917106792SAlexander Shaposhnikov // Given an intrinsic with an `FT` argument, try to find a wider intrinsic 130017106792SAlexander Shaposhnikov // that applies the same operation on the shadow argument. 130117106792SAlexander Shaposhnikov // Options are: 130217106792SAlexander Shaposhnikov // - pass in the ID and full function type, 130317106792SAlexander Shaposhnikov // - pass in the name, which includes the function type through mangling. 130417106792SAlexander Shaposhnikov static const WidenedIntrinsic *widen(StringRef Name); 130517106792SAlexander Shaposhnikov 130617106792SAlexander Shaposhnikov private: 130717106792SAlexander Shaposhnikov struct LFEntry { 130817106792SAlexander Shaposhnikov LibFunc LFunc; 130917106792SAlexander Shaposhnikov const char *IntrinsicName; 131017106792SAlexander Shaposhnikov }; 131117106792SAlexander Shaposhnikov static const LFEntry kLibfuncIntrinsics[]; 131217106792SAlexander Shaposhnikov 131317106792SAlexander Shaposhnikov static const WidenedIntrinsic kWidenedIntrinsics[]; 131417106792SAlexander Shaposhnikov }; 131517106792SAlexander Shaposhnikov } // namespace 131617106792SAlexander Shaposhnikov 131717106792SAlexander Shaposhnikov static FunctionType *makeDoubleDouble(LLVMContext &C) { 131817106792SAlexander Shaposhnikov return FunctionType::get(Type::getDoubleTy(C), {Type::getDoubleTy(C)}, false); 131917106792SAlexander Shaposhnikov } 132017106792SAlexander Shaposhnikov 132117106792SAlexander Shaposhnikov static FunctionType *makeX86FP80X86FP80(LLVMContext &C) { 132217106792SAlexander Shaposhnikov return FunctionType::get(Type::getX86_FP80Ty(C), {Type::getX86_FP80Ty(C)}, 132317106792SAlexander Shaposhnikov false); 132417106792SAlexander Shaposhnikov } 132517106792SAlexander Shaposhnikov 132617106792SAlexander Shaposhnikov static FunctionType *makeDoubleDoubleI32(LLVMContext &C) { 132717106792SAlexander Shaposhnikov return FunctionType::get(Type::getDoubleTy(C), 132817106792SAlexander Shaposhnikov {Type::getDoubleTy(C), Type::getInt32Ty(C)}, false); 132917106792SAlexander Shaposhnikov } 133017106792SAlexander Shaposhnikov 133117106792SAlexander Shaposhnikov static FunctionType *makeX86FP80X86FP80I32(LLVMContext &C) { 133217106792SAlexander Shaposhnikov return FunctionType::get(Type::getX86_FP80Ty(C), 133317106792SAlexander Shaposhnikov {Type::getX86_FP80Ty(C), Type::getInt32Ty(C)}, 133417106792SAlexander Shaposhnikov false); 133517106792SAlexander Shaposhnikov } 133617106792SAlexander Shaposhnikov 133717106792SAlexander Shaposhnikov static FunctionType *makeDoubleDoubleDouble(LLVMContext &C) { 133817106792SAlexander Shaposhnikov return FunctionType::get(Type::getDoubleTy(C), 133917106792SAlexander Shaposhnikov {Type::getDoubleTy(C), Type::getDoubleTy(C)}, false); 134017106792SAlexander Shaposhnikov } 134117106792SAlexander Shaposhnikov 134217106792SAlexander Shaposhnikov static FunctionType *makeX86FP80X86FP80X86FP80(LLVMContext &C) { 134317106792SAlexander Shaposhnikov return FunctionType::get(Type::getX86_FP80Ty(C), 134417106792SAlexander Shaposhnikov {Type::getX86_FP80Ty(C), Type::getX86_FP80Ty(C)}, 134517106792SAlexander Shaposhnikov false); 134617106792SAlexander Shaposhnikov } 134717106792SAlexander Shaposhnikov 134817106792SAlexander Shaposhnikov static FunctionType *makeDoubleDoubleDoubleDouble(LLVMContext &C) { 134917106792SAlexander Shaposhnikov return FunctionType::get( 135017106792SAlexander Shaposhnikov Type::getDoubleTy(C), 135117106792SAlexander Shaposhnikov {Type::getDoubleTy(C), Type::getDoubleTy(C), Type::getDoubleTy(C)}, 135217106792SAlexander Shaposhnikov false); 135317106792SAlexander Shaposhnikov } 135417106792SAlexander Shaposhnikov 135517106792SAlexander Shaposhnikov static FunctionType *makeX86FP80X86FP80X86FP80X86FP80(LLVMContext &C) { 135617106792SAlexander Shaposhnikov return FunctionType::get( 135717106792SAlexander Shaposhnikov Type::getX86_FP80Ty(C), 135817106792SAlexander Shaposhnikov {Type::getX86_FP80Ty(C), Type::getX86_FP80Ty(C), Type::getX86_FP80Ty(C)}, 135917106792SAlexander Shaposhnikov false); 136017106792SAlexander Shaposhnikov } 136117106792SAlexander Shaposhnikov 136217106792SAlexander Shaposhnikov const KnownIntrinsic::WidenedIntrinsic KnownIntrinsic::kWidenedIntrinsics[] = { 136317106792SAlexander Shaposhnikov // TODO: Right now we ignore vector intrinsics. 136417106792SAlexander Shaposhnikov // This is hard because we have to model the semantics of the intrinsics, 136517106792SAlexander Shaposhnikov // e.g. llvm.x86.sse2.min.sd means extract first element, min, insert back. 136617106792SAlexander Shaposhnikov // Intrinsics that take any non-vector FT types: 136717106792SAlexander Shaposhnikov // NOTE: Right now because of 136817106792SAlexander Shaposhnikov // https://github.com/llvm/llvm-project/issues/44744 136917106792SAlexander Shaposhnikov // for f128 we need to use makeX86FP80X86FP80 (go to a lower precision and 137017106792SAlexander Shaposhnikov // come back). 137117106792SAlexander Shaposhnikov {"llvm.sqrt.f32", Intrinsic::sqrt, makeDoubleDouble}, 137217106792SAlexander Shaposhnikov {"llvm.sqrt.f64", Intrinsic::sqrt, makeX86FP80X86FP80}, 137317106792SAlexander Shaposhnikov {"llvm.sqrt.f80", Intrinsic::sqrt, makeX86FP80X86FP80}, 137417106792SAlexander Shaposhnikov {"llvm.powi.f32", Intrinsic::powi, makeDoubleDoubleI32}, 137517106792SAlexander Shaposhnikov {"llvm.powi.f64", Intrinsic::powi, makeX86FP80X86FP80I32}, 137617106792SAlexander Shaposhnikov {"llvm.powi.f80", Intrinsic::powi, makeX86FP80X86FP80I32}, 137717106792SAlexander Shaposhnikov {"llvm.sin.f32", Intrinsic::sin, makeDoubleDouble}, 137817106792SAlexander Shaposhnikov {"llvm.sin.f64", Intrinsic::sin, makeX86FP80X86FP80}, 137917106792SAlexander Shaposhnikov {"llvm.sin.f80", Intrinsic::sin, makeX86FP80X86FP80}, 138017106792SAlexander Shaposhnikov {"llvm.cos.f32", Intrinsic::cos, makeDoubleDouble}, 138117106792SAlexander Shaposhnikov {"llvm.cos.f64", Intrinsic::cos, makeX86FP80X86FP80}, 138217106792SAlexander Shaposhnikov {"llvm.cos.f80", Intrinsic::cos, makeX86FP80X86FP80}, 138317106792SAlexander Shaposhnikov {"llvm.pow.f32", Intrinsic::pow, makeDoubleDoubleDouble}, 138417106792SAlexander Shaposhnikov {"llvm.pow.f64", Intrinsic::pow, makeX86FP80X86FP80X86FP80}, 138517106792SAlexander Shaposhnikov {"llvm.pow.f80", Intrinsic::pow, makeX86FP80X86FP80X86FP80}, 138617106792SAlexander Shaposhnikov {"llvm.exp.f32", Intrinsic::exp, makeDoubleDouble}, 138717106792SAlexander Shaposhnikov {"llvm.exp.f64", Intrinsic::exp, makeX86FP80X86FP80}, 138817106792SAlexander Shaposhnikov {"llvm.exp.f80", Intrinsic::exp, makeX86FP80X86FP80}, 138917106792SAlexander Shaposhnikov {"llvm.exp2.f32", Intrinsic::exp2, makeDoubleDouble}, 139017106792SAlexander Shaposhnikov {"llvm.exp2.f64", Intrinsic::exp2, makeX86FP80X86FP80}, 139117106792SAlexander Shaposhnikov {"llvm.exp2.f80", Intrinsic::exp2, makeX86FP80X86FP80}, 139217106792SAlexander Shaposhnikov {"llvm.log.f32", Intrinsic::log, makeDoubleDouble}, 139317106792SAlexander Shaposhnikov {"llvm.log.f64", Intrinsic::log, makeX86FP80X86FP80}, 139417106792SAlexander Shaposhnikov {"llvm.log.f80", Intrinsic::log, makeX86FP80X86FP80}, 139517106792SAlexander Shaposhnikov {"llvm.log10.f32", Intrinsic::log10, makeDoubleDouble}, 139617106792SAlexander Shaposhnikov {"llvm.log10.f64", Intrinsic::log10, makeX86FP80X86FP80}, 139717106792SAlexander Shaposhnikov {"llvm.log10.f80", Intrinsic::log10, makeX86FP80X86FP80}, 139817106792SAlexander Shaposhnikov {"llvm.log2.f32", Intrinsic::log2, makeDoubleDouble}, 139917106792SAlexander Shaposhnikov {"llvm.log2.f64", Intrinsic::log2, makeX86FP80X86FP80}, 140017106792SAlexander Shaposhnikov {"llvm.log2.f80", Intrinsic::log2, makeX86FP80X86FP80}, 140117106792SAlexander Shaposhnikov {"llvm.fma.f32", Intrinsic::fma, makeDoubleDoubleDoubleDouble}, 140217106792SAlexander Shaposhnikov 140317106792SAlexander Shaposhnikov {"llvm.fmuladd.f32", Intrinsic::fmuladd, makeDoubleDoubleDoubleDouble}, 140417106792SAlexander Shaposhnikov 140517106792SAlexander Shaposhnikov {"llvm.fma.f64", Intrinsic::fma, makeX86FP80X86FP80X86FP80X86FP80}, 140617106792SAlexander Shaposhnikov 140717106792SAlexander Shaposhnikov {"llvm.fmuladd.f64", Intrinsic::fma, makeX86FP80X86FP80X86FP80X86FP80}, 140817106792SAlexander Shaposhnikov 140917106792SAlexander Shaposhnikov {"llvm.fma.f80", Intrinsic::fma, makeX86FP80X86FP80X86FP80X86FP80}, 141017106792SAlexander Shaposhnikov {"llvm.fabs.f32", Intrinsic::fabs, makeDoubleDouble}, 141117106792SAlexander Shaposhnikov {"llvm.fabs.f64", Intrinsic::fabs, makeX86FP80X86FP80}, 141217106792SAlexander Shaposhnikov {"llvm.fabs.f80", Intrinsic::fabs, makeX86FP80X86FP80}, 141317106792SAlexander Shaposhnikov {"llvm.minnum.f32", Intrinsic::minnum, makeDoubleDoubleDouble}, 141417106792SAlexander Shaposhnikov {"llvm.minnum.f64", Intrinsic::minnum, makeX86FP80X86FP80X86FP80}, 141517106792SAlexander Shaposhnikov {"llvm.minnum.f80", Intrinsic::minnum, makeX86FP80X86FP80X86FP80}, 141617106792SAlexander Shaposhnikov {"llvm.maxnum.f32", Intrinsic::maxnum, makeDoubleDoubleDouble}, 141717106792SAlexander Shaposhnikov {"llvm.maxnum.f64", Intrinsic::maxnum, makeX86FP80X86FP80X86FP80}, 141817106792SAlexander Shaposhnikov {"llvm.maxnum.f80", Intrinsic::maxnum, makeX86FP80X86FP80X86FP80}, 141917106792SAlexander Shaposhnikov {"llvm.minimum.f32", Intrinsic::minimum, makeDoubleDoubleDouble}, 142017106792SAlexander Shaposhnikov {"llvm.minimum.f64", Intrinsic::minimum, makeX86FP80X86FP80X86FP80}, 142117106792SAlexander Shaposhnikov {"llvm.minimum.f80", Intrinsic::minimum, makeX86FP80X86FP80X86FP80}, 142217106792SAlexander Shaposhnikov {"llvm.maximum.f32", Intrinsic::maximum, makeDoubleDoubleDouble}, 142317106792SAlexander Shaposhnikov {"llvm.maximum.f64", Intrinsic::maximum, makeX86FP80X86FP80X86FP80}, 142417106792SAlexander Shaposhnikov {"llvm.maximum.f80", Intrinsic::maximum, makeX86FP80X86FP80X86FP80}, 142517106792SAlexander Shaposhnikov {"llvm.copysign.f32", Intrinsic::copysign, makeDoubleDoubleDouble}, 142617106792SAlexander Shaposhnikov {"llvm.copysign.f64", Intrinsic::copysign, makeX86FP80X86FP80X86FP80}, 142717106792SAlexander Shaposhnikov {"llvm.copysign.f80", Intrinsic::copysign, makeX86FP80X86FP80X86FP80}, 142817106792SAlexander Shaposhnikov {"llvm.floor.f32", Intrinsic::floor, makeDoubleDouble}, 142917106792SAlexander Shaposhnikov {"llvm.floor.f64", Intrinsic::floor, makeX86FP80X86FP80}, 143017106792SAlexander Shaposhnikov {"llvm.floor.f80", Intrinsic::floor, makeX86FP80X86FP80}, 143117106792SAlexander Shaposhnikov {"llvm.ceil.f32", Intrinsic::ceil, makeDoubleDouble}, 143217106792SAlexander Shaposhnikov {"llvm.ceil.f64", Intrinsic::ceil, makeX86FP80X86FP80}, 143317106792SAlexander Shaposhnikov {"llvm.ceil.f80", Intrinsic::ceil, makeX86FP80X86FP80}, 143417106792SAlexander Shaposhnikov {"llvm.trunc.f32", Intrinsic::trunc, makeDoubleDouble}, 143517106792SAlexander Shaposhnikov {"llvm.trunc.f64", Intrinsic::trunc, makeX86FP80X86FP80}, 143617106792SAlexander Shaposhnikov {"llvm.trunc.f80", Intrinsic::trunc, makeX86FP80X86FP80}, 143717106792SAlexander Shaposhnikov {"llvm.rint.f32", Intrinsic::rint, makeDoubleDouble}, 143817106792SAlexander Shaposhnikov {"llvm.rint.f64", Intrinsic::rint, makeX86FP80X86FP80}, 143917106792SAlexander Shaposhnikov {"llvm.rint.f80", Intrinsic::rint, makeX86FP80X86FP80}, 144017106792SAlexander Shaposhnikov {"llvm.nearbyint.f32", Intrinsic::nearbyint, makeDoubleDouble}, 144117106792SAlexander Shaposhnikov {"llvm.nearbyint.f64", Intrinsic::nearbyint, makeX86FP80X86FP80}, 144217106792SAlexander Shaposhnikov {"llvm.nearbyin80f64", Intrinsic::nearbyint, makeX86FP80X86FP80}, 144317106792SAlexander Shaposhnikov {"llvm.round.f32", Intrinsic::round, makeDoubleDouble}, 144417106792SAlexander Shaposhnikov {"llvm.round.f64", Intrinsic::round, makeX86FP80X86FP80}, 144517106792SAlexander Shaposhnikov {"llvm.round.f80", Intrinsic::round, makeX86FP80X86FP80}, 144617106792SAlexander Shaposhnikov {"llvm.lround.f32", Intrinsic::lround, makeDoubleDouble}, 144717106792SAlexander Shaposhnikov {"llvm.lround.f64", Intrinsic::lround, makeX86FP80X86FP80}, 144817106792SAlexander Shaposhnikov {"llvm.lround.f80", Intrinsic::lround, makeX86FP80X86FP80}, 144917106792SAlexander Shaposhnikov {"llvm.llround.f32", Intrinsic::llround, makeDoubleDouble}, 145017106792SAlexander Shaposhnikov {"llvm.llround.f64", Intrinsic::llround, makeX86FP80X86FP80}, 145117106792SAlexander Shaposhnikov {"llvm.llround.f80", Intrinsic::llround, makeX86FP80X86FP80}, 145217106792SAlexander Shaposhnikov {"llvm.lrint.f32", Intrinsic::lrint, makeDoubleDouble}, 145317106792SAlexander Shaposhnikov {"llvm.lrint.f64", Intrinsic::lrint, makeX86FP80X86FP80}, 145417106792SAlexander Shaposhnikov {"llvm.lrint.f80", Intrinsic::lrint, makeX86FP80X86FP80}, 145517106792SAlexander Shaposhnikov {"llvm.llrint.f32", Intrinsic::llrint, makeDoubleDouble}, 145617106792SAlexander Shaposhnikov {"llvm.llrint.f64", Intrinsic::llrint, makeX86FP80X86FP80}, 145717106792SAlexander Shaposhnikov {"llvm.llrint.f80", Intrinsic::llrint, makeX86FP80X86FP80}, 145817106792SAlexander Shaposhnikov }; 145917106792SAlexander Shaposhnikov 146017106792SAlexander Shaposhnikov const KnownIntrinsic::LFEntry KnownIntrinsic::kLibfuncIntrinsics[] = { 146117106792SAlexander Shaposhnikov {LibFunc_sqrtf, "llvm.sqrt.f32"}, 146217106792SAlexander Shaposhnikov {LibFunc_sqrt, "llvm.sqrt.f64"}, 146317106792SAlexander Shaposhnikov {LibFunc_sqrtl, "llvm.sqrt.f80"}, 146417106792SAlexander Shaposhnikov {LibFunc_sinf, "llvm.sin.f32"}, 146517106792SAlexander Shaposhnikov {LibFunc_sin, "llvm.sin.f64"}, 146617106792SAlexander Shaposhnikov {LibFunc_sinl, "llvm.sin.f80"}, 146717106792SAlexander Shaposhnikov {LibFunc_cosf, "llvm.cos.f32"}, 146817106792SAlexander Shaposhnikov {LibFunc_cos, "llvm.cos.f64"}, 146917106792SAlexander Shaposhnikov {LibFunc_cosl, "llvm.cos.f80"}, 147017106792SAlexander Shaposhnikov {LibFunc_powf, "llvm.pow.f32"}, 147117106792SAlexander Shaposhnikov {LibFunc_pow, "llvm.pow.f64"}, 147217106792SAlexander Shaposhnikov {LibFunc_powl, "llvm.pow.f80"}, 147317106792SAlexander Shaposhnikov {LibFunc_expf, "llvm.exp.f32"}, 147417106792SAlexander Shaposhnikov {LibFunc_exp, "llvm.exp.f64"}, 147517106792SAlexander Shaposhnikov {LibFunc_expl, "llvm.exp.f80"}, 147617106792SAlexander Shaposhnikov {LibFunc_exp2f, "llvm.exp2.f32"}, 147717106792SAlexander Shaposhnikov {LibFunc_exp2, "llvm.exp2.f64"}, 147817106792SAlexander Shaposhnikov {LibFunc_exp2l, "llvm.exp2.f80"}, 147917106792SAlexander Shaposhnikov {LibFunc_logf, "llvm.log.f32"}, 148017106792SAlexander Shaposhnikov {LibFunc_log, "llvm.log.f64"}, 148117106792SAlexander Shaposhnikov {LibFunc_logl, "llvm.log.f80"}, 148217106792SAlexander Shaposhnikov {LibFunc_log10f, "llvm.log10.f32"}, 148317106792SAlexander Shaposhnikov {LibFunc_log10, "llvm.log10.f64"}, 148417106792SAlexander Shaposhnikov {LibFunc_log10l, "llvm.log10.f80"}, 148517106792SAlexander Shaposhnikov {LibFunc_log2f, "llvm.log2.f32"}, 148617106792SAlexander Shaposhnikov {LibFunc_log2, "llvm.log2.f64"}, 148717106792SAlexander Shaposhnikov {LibFunc_log2l, "llvm.log2.f80"}, 148817106792SAlexander Shaposhnikov {LibFunc_fabsf, "llvm.fabs.f32"}, 148917106792SAlexander Shaposhnikov {LibFunc_fabs, "llvm.fabs.f64"}, 149017106792SAlexander Shaposhnikov {LibFunc_fabsl, "llvm.fabs.f80"}, 149117106792SAlexander Shaposhnikov {LibFunc_copysignf, "llvm.copysign.f32"}, 149217106792SAlexander Shaposhnikov {LibFunc_copysign, "llvm.copysign.f64"}, 149317106792SAlexander Shaposhnikov {LibFunc_copysignl, "llvm.copysign.f80"}, 149417106792SAlexander Shaposhnikov {LibFunc_floorf, "llvm.floor.f32"}, 149517106792SAlexander Shaposhnikov {LibFunc_floor, "llvm.floor.f64"}, 149617106792SAlexander Shaposhnikov {LibFunc_floorl, "llvm.floor.f80"}, 149717106792SAlexander Shaposhnikov {LibFunc_fmaxf, "llvm.maxnum.f32"}, 149817106792SAlexander Shaposhnikov {LibFunc_fmax, "llvm.maxnum.f64"}, 149917106792SAlexander Shaposhnikov {LibFunc_fmaxl, "llvm.maxnum.f80"}, 150017106792SAlexander Shaposhnikov {LibFunc_fminf, "llvm.minnum.f32"}, 150117106792SAlexander Shaposhnikov {LibFunc_fmin, "llvm.minnum.f64"}, 150217106792SAlexander Shaposhnikov {LibFunc_fminl, "llvm.minnum.f80"}, 150317106792SAlexander Shaposhnikov {LibFunc_ceilf, "llvm.ceil.f32"}, 150417106792SAlexander Shaposhnikov {LibFunc_ceil, "llvm.ceil.f64"}, 150517106792SAlexander Shaposhnikov {LibFunc_ceill, "llvm.ceil.f80"}, 150617106792SAlexander Shaposhnikov {LibFunc_truncf, "llvm.trunc.f32"}, 150717106792SAlexander Shaposhnikov {LibFunc_trunc, "llvm.trunc.f64"}, 150817106792SAlexander Shaposhnikov {LibFunc_truncl, "llvm.trunc.f80"}, 150917106792SAlexander Shaposhnikov {LibFunc_rintf, "llvm.rint.f32"}, 151017106792SAlexander Shaposhnikov {LibFunc_rint, "llvm.rint.f64"}, 151117106792SAlexander Shaposhnikov {LibFunc_rintl, "llvm.rint.f80"}, 151217106792SAlexander Shaposhnikov {LibFunc_nearbyintf, "llvm.nearbyint.f32"}, 151317106792SAlexander Shaposhnikov {LibFunc_nearbyint, "llvm.nearbyint.f64"}, 151417106792SAlexander Shaposhnikov {LibFunc_nearbyintl, "llvm.nearbyint.f80"}, 151517106792SAlexander Shaposhnikov {LibFunc_roundf, "llvm.round.f32"}, 151617106792SAlexander Shaposhnikov {LibFunc_round, "llvm.round.f64"}, 151717106792SAlexander Shaposhnikov {LibFunc_roundl, "llvm.round.f80"}, 151817106792SAlexander Shaposhnikov }; 151917106792SAlexander Shaposhnikov 152017106792SAlexander Shaposhnikov const char *KnownIntrinsic::get(LibFunc LFunc) { 152117106792SAlexander Shaposhnikov for (const auto &E : kLibfuncIntrinsics) { 152217106792SAlexander Shaposhnikov if (E.LFunc == LFunc) 152317106792SAlexander Shaposhnikov return E.IntrinsicName; 152417106792SAlexander Shaposhnikov } 152517106792SAlexander Shaposhnikov return nullptr; 152617106792SAlexander Shaposhnikov } 152717106792SAlexander Shaposhnikov 152817106792SAlexander Shaposhnikov const KnownIntrinsic::WidenedIntrinsic *KnownIntrinsic::widen(StringRef Name) { 152917106792SAlexander Shaposhnikov for (const auto &E : kWidenedIntrinsics) { 153017106792SAlexander Shaposhnikov if (E.NarrowName == Name) 153117106792SAlexander Shaposhnikov return &E; 153217106792SAlexander Shaposhnikov } 153317106792SAlexander Shaposhnikov return nullptr; 153417106792SAlexander Shaposhnikov } 153517106792SAlexander Shaposhnikov 153617106792SAlexander Shaposhnikov // Returns the name of the LLVM intrinsic corresponding to the given function. 153717106792SAlexander Shaposhnikov static const char *getIntrinsicFromLibfunc(Function &Fn, Type *VT, 153817106792SAlexander Shaposhnikov const TargetLibraryInfo &TLI) { 153917106792SAlexander Shaposhnikov LibFunc LFunc; 154017106792SAlexander Shaposhnikov if (!TLI.getLibFunc(Fn, LFunc)) 154117106792SAlexander Shaposhnikov return nullptr; 154217106792SAlexander Shaposhnikov 154317106792SAlexander Shaposhnikov if (const char *Name = KnownIntrinsic::get(LFunc)) 154417106792SAlexander Shaposhnikov return Name; 154517106792SAlexander Shaposhnikov 154617106792SAlexander Shaposhnikov LLVM_DEBUG(errs() << "TODO: LibFunc: " << TLI.getName(LFunc) << "\n"); 154717106792SAlexander Shaposhnikov return nullptr; 154817106792SAlexander Shaposhnikov } 154917106792SAlexander Shaposhnikov 155017106792SAlexander Shaposhnikov // Try to handle a known function call. 155117106792SAlexander Shaposhnikov Value *NumericalStabilitySanitizer::maybeHandleKnownCallBase( 155217106792SAlexander Shaposhnikov CallBase &Call, Type *VT, Type *ExtendedVT, const TargetLibraryInfo &TLI, 155317106792SAlexander Shaposhnikov const ValueToShadowMap &Map, IRBuilder<> &Builder) { 155417106792SAlexander Shaposhnikov Function *Fn = Call.getCalledFunction(); 155517106792SAlexander Shaposhnikov if (Fn == nullptr) 155617106792SAlexander Shaposhnikov return nullptr; 155717106792SAlexander Shaposhnikov 155817106792SAlexander Shaposhnikov Intrinsic::ID WidenedId = Intrinsic::ID(); 155917106792SAlexander Shaposhnikov FunctionType *WidenedFnTy = nullptr; 156017106792SAlexander Shaposhnikov if (const auto ID = Fn->getIntrinsicID()) { 156117106792SAlexander Shaposhnikov const auto *Widened = KnownIntrinsic::widen(Fn->getName()); 156217106792SAlexander Shaposhnikov if (Widened) { 156317106792SAlexander Shaposhnikov WidenedId = Widened->ID; 156417106792SAlexander Shaposhnikov WidenedFnTy = Widened->MakeFnTy(Context); 156517106792SAlexander Shaposhnikov } else { 156617106792SAlexander Shaposhnikov // If we don't know how to widen the intrinsic, we have no choice but to 156717106792SAlexander Shaposhnikov // call the non-wide version on a truncated shadow and extend again 156817106792SAlexander Shaposhnikov // afterwards. 156917106792SAlexander Shaposhnikov WidenedId = ID; 157017106792SAlexander Shaposhnikov WidenedFnTy = Fn->getFunctionType(); 157117106792SAlexander Shaposhnikov } 157217106792SAlexander Shaposhnikov } else if (const char *Name = getIntrinsicFromLibfunc(*Fn, VT, TLI)) { 157317106792SAlexander Shaposhnikov // We might have a call to a library function that we can replace with a 157417106792SAlexander Shaposhnikov // wider Intrinsic. 157517106792SAlexander Shaposhnikov const auto *Widened = KnownIntrinsic::widen(Name); 157617106792SAlexander Shaposhnikov assert(Widened && "make sure KnownIntrinsic entries are consistent"); 157717106792SAlexander Shaposhnikov WidenedId = Widened->ID; 157817106792SAlexander Shaposhnikov WidenedFnTy = Widened->MakeFnTy(Context); 157917106792SAlexander Shaposhnikov } else { 158017106792SAlexander Shaposhnikov // This is not a known library function or intrinsic. 158117106792SAlexander Shaposhnikov return nullptr; 158217106792SAlexander Shaposhnikov } 158317106792SAlexander Shaposhnikov 158417106792SAlexander Shaposhnikov // Check that the widened intrinsic is valid. 158517106792SAlexander Shaposhnikov SmallVector<Intrinsic::IITDescriptor, 8> Table; 158617106792SAlexander Shaposhnikov getIntrinsicInfoTableEntries(WidenedId, Table); 158717106792SAlexander Shaposhnikov SmallVector<Type *, 4> ArgTys; 158817106792SAlexander Shaposhnikov ArrayRef<Intrinsic::IITDescriptor> TableRef = Table; 158917106792SAlexander Shaposhnikov [[maybe_unused]] Intrinsic::MatchIntrinsicTypesResult MatchResult = 159017106792SAlexander Shaposhnikov Intrinsic::matchIntrinsicSignature(WidenedFnTy, TableRef, ArgTys); 159117106792SAlexander Shaposhnikov assert(MatchResult == Intrinsic::MatchIntrinsicTypes_Match && 159217106792SAlexander Shaposhnikov "invalid widened intrinsic"); 159317106792SAlexander Shaposhnikov // For known intrinsic functions, we create a second call to the same 159417106792SAlexander Shaposhnikov // intrinsic with a different type. 159517106792SAlexander Shaposhnikov SmallVector<Value *, 4> Args; 159617106792SAlexander Shaposhnikov // The last operand is the intrinsic itself, skip it. 159717106792SAlexander Shaposhnikov for (unsigned I = 0, E = Call.getNumOperands() - 1; I < E; ++I) { 159817106792SAlexander Shaposhnikov Value *Arg = Call.getOperand(I); 159917106792SAlexander Shaposhnikov Type *OrigArgTy = Arg->getType(); 160017106792SAlexander Shaposhnikov Type *IntrinsicArgTy = WidenedFnTy->getParamType(I); 160117106792SAlexander Shaposhnikov if (OrigArgTy == IntrinsicArgTy) { 160217106792SAlexander Shaposhnikov Args.push_back(Arg); // The arg is passed as is. 160317106792SAlexander Shaposhnikov continue; 160417106792SAlexander Shaposhnikov } 160517106792SAlexander Shaposhnikov Type *ShadowArgTy = Config.getExtendedFPType(Arg->getType()); 160617106792SAlexander Shaposhnikov assert(ShadowArgTy && 160717106792SAlexander Shaposhnikov "don't know how to get the shadow value for a non-FT"); 160817106792SAlexander Shaposhnikov Value *Shadow = Map.getShadow(Arg); 160917106792SAlexander Shaposhnikov if (ShadowArgTy == IntrinsicArgTy) { 161017106792SAlexander Shaposhnikov // The shadow is the right type for the intrinsic. 161117106792SAlexander Shaposhnikov assert(Shadow->getType() == ShadowArgTy); 161217106792SAlexander Shaposhnikov Args.push_back(Shadow); 161317106792SAlexander Shaposhnikov continue; 161417106792SAlexander Shaposhnikov } 161517106792SAlexander Shaposhnikov // There is no intrinsic with his level of precision, truncate the shadow. 161617106792SAlexander Shaposhnikov Args.push_back(Builder.CreateFPTrunc(Shadow, IntrinsicArgTy)); 161717106792SAlexander Shaposhnikov } 161817106792SAlexander Shaposhnikov Value *IntrinsicCall = Builder.CreateIntrinsic(WidenedId, ArgTys, Args); 161917106792SAlexander Shaposhnikov return WidenedFnTy->getReturnType() == ExtendedVT 162017106792SAlexander Shaposhnikov ? IntrinsicCall 162117106792SAlexander Shaposhnikov : Builder.CreateFPExt(IntrinsicCall, ExtendedVT); 162217106792SAlexander Shaposhnikov } 162317106792SAlexander Shaposhnikov 162417106792SAlexander Shaposhnikov // Handle a CallBase, i.e. a function call, an inline asm sequence, or an 162517106792SAlexander Shaposhnikov // invoke. 162617106792SAlexander Shaposhnikov Value *NumericalStabilitySanitizer::handleCallBase(CallBase &Call, Type *VT, 162717106792SAlexander Shaposhnikov Type *ExtendedVT, 162817106792SAlexander Shaposhnikov const TargetLibraryInfo &TLI, 162917106792SAlexander Shaposhnikov const ValueToShadowMap &Map, 163017106792SAlexander Shaposhnikov IRBuilder<> &Builder) { 163117106792SAlexander Shaposhnikov // We cannot look inside inline asm, just expand the result again. 163217106792SAlexander Shaposhnikov if (Call.isInlineAsm()) 163317106792SAlexander Shaposhnikov return Builder.CreateFPExt(&Call, ExtendedVT); 163417106792SAlexander Shaposhnikov 163517106792SAlexander Shaposhnikov // Intrinsics and library functions (e.g. sin, exp) are handled 163617106792SAlexander Shaposhnikov // specifically, because we know their semantics and can do better than 163717106792SAlexander Shaposhnikov // blindly calling them (e.g. compute the sinus in the actual shadow domain). 163817106792SAlexander Shaposhnikov if (Value *V = 163917106792SAlexander Shaposhnikov maybeHandleKnownCallBase(Call, VT, ExtendedVT, TLI, Map, Builder)) 164017106792SAlexander Shaposhnikov return V; 164117106792SAlexander Shaposhnikov 164217106792SAlexander Shaposhnikov // If the return tag matches that of the called function, read the extended 164317106792SAlexander Shaposhnikov // return value from the shadow ret ptr. Else, just extend the return value. 164417106792SAlexander Shaposhnikov Value *L = 164517106792SAlexander Shaposhnikov Builder.CreateLoad(IntptrTy, NsanShadowRetTag, /*isVolatile=*/false); 164617106792SAlexander Shaposhnikov Value *HasShadowRet = Builder.CreateICmpEQ( 164717106792SAlexander Shaposhnikov L, Builder.CreatePtrToInt(Call.getCalledOperand(), IntptrTy)); 164817106792SAlexander Shaposhnikov 164917106792SAlexander Shaposhnikov Value *ShadowRetVal = Builder.CreateLoad( 165017106792SAlexander Shaposhnikov ExtendedVT, 165117106792SAlexander Shaposhnikov Builder.CreateConstGEP2_64(NsanShadowRetType, NsanShadowRetPtr, 0, 0), 165217106792SAlexander Shaposhnikov /*isVolatile=*/false); 165317106792SAlexander Shaposhnikov Value *Shadow = Builder.CreateSelect(HasShadowRet, ShadowRetVal, 165417106792SAlexander Shaposhnikov Builder.CreateFPExt(&Call, ExtendedVT)); 165517106792SAlexander Shaposhnikov ++NumInstrumentedFTCalls; 165617106792SAlexander Shaposhnikov return Shadow; 165717106792SAlexander Shaposhnikov } 165817106792SAlexander Shaposhnikov 165917106792SAlexander Shaposhnikov // Creates a shadow value for the given FT value. At that point all operands are 166017106792SAlexander Shaposhnikov // guaranteed to be available. 166117106792SAlexander Shaposhnikov Value *NumericalStabilitySanitizer::createShadowValueWithOperandsAvailable( 166217106792SAlexander Shaposhnikov Instruction &Inst, const TargetLibraryInfo &TLI, 166317106792SAlexander Shaposhnikov const ValueToShadowMap &Map) { 166417106792SAlexander Shaposhnikov Type *VT = Inst.getType(); 166517106792SAlexander Shaposhnikov Type *ExtendedVT = Config.getExtendedFPType(VT); 166617106792SAlexander Shaposhnikov assert(ExtendedVT != nullptr && "trying to create a shadow for a non-FT"); 166717106792SAlexander Shaposhnikov 166817106792SAlexander Shaposhnikov if (auto *Load = dyn_cast<LoadInst>(&Inst)) 166917106792SAlexander Shaposhnikov return handleLoad(*Load, VT, ExtendedVT); 167017106792SAlexander Shaposhnikov 167117106792SAlexander Shaposhnikov if (auto *Call = dyn_cast<CallInst>(&Inst)) { 167217106792SAlexander Shaposhnikov // Insert after the call. 167317106792SAlexander Shaposhnikov BasicBlock::iterator It(Inst); 167417106792SAlexander Shaposhnikov IRBuilder<> Builder(Call->getParent(), ++It); 167517106792SAlexander Shaposhnikov Builder.SetCurrentDebugLocation(Call->getDebugLoc()); 167617106792SAlexander Shaposhnikov return handleCallBase(*Call, VT, ExtendedVT, TLI, Map, Builder); 167717106792SAlexander Shaposhnikov } 167817106792SAlexander Shaposhnikov 167917106792SAlexander Shaposhnikov if (auto *Invoke = dyn_cast<InvokeInst>(&Inst)) { 168017106792SAlexander Shaposhnikov // The Invoke terminates the basic block, create a new basic block in 168117106792SAlexander Shaposhnikov // between the successful invoke and the next block. 168217106792SAlexander Shaposhnikov BasicBlock *InvokeBB = Invoke->getParent(); 168317106792SAlexander Shaposhnikov BasicBlock *NextBB = Invoke->getNormalDest(); 168417106792SAlexander Shaposhnikov BasicBlock *NewBB = 168517106792SAlexander Shaposhnikov BasicBlock::Create(Context, "", NextBB->getParent(), NextBB); 168617106792SAlexander Shaposhnikov Inst.replaceSuccessorWith(NextBB, NewBB); 168717106792SAlexander Shaposhnikov 168817106792SAlexander Shaposhnikov IRBuilder<> Builder(NewBB); 168917106792SAlexander Shaposhnikov Builder.SetCurrentDebugLocation(Invoke->getDebugLoc()); 169017106792SAlexander Shaposhnikov Value *Shadow = handleCallBase(*Invoke, VT, ExtendedVT, TLI, Map, Builder); 169117106792SAlexander Shaposhnikov Builder.CreateBr(NextBB); 169217106792SAlexander Shaposhnikov NewBB->replaceSuccessorsPhiUsesWith(InvokeBB, NewBB); 169317106792SAlexander Shaposhnikov return Shadow; 169417106792SAlexander Shaposhnikov } 169517106792SAlexander Shaposhnikov 169617106792SAlexander Shaposhnikov IRBuilder<> Builder(Inst.getNextNode()); 169717106792SAlexander Shaposhnikov Builder.SetCurrentDebugLocation(Inst.getDebugLoc()); 169817106792SAlexander Shaposhnikov 169917106792SAlexander Shaposhnikov if (auto *Trunc = dyn_cast<FPTruncInst>(&Inst)) 170017106792SAlexander Shaposhnikov return handleTrunc(*Trunc, VT, ExtendedVT, Map, Builder); 170117106792SAlexander Shaposhnikov if (auto *Ext = dyn_cast<FPExtInst>(&Inst)) 170217106792SAlexander Shaposhnikov return handleExt(*Ext, VT, ExtendedVT, Map, Builder); 170317106792SAlexander Shaposhnikov 170417106792SAlexander Shaposhnikov if (auto *UnaryOp = dyn_cast<UnaryOperator>(&Inst)) 170517106792SAlexander Shaposhnikov return Builder.CreateUnOp(UnaryOp->getOpcode(), 170617106792SAlexander Shaposhnikov Map.getShadow(UnaryOp->getOperand(0))); 170717106792SAlexander Shaposhnikov 170817106792SAlexander Shaposhnikov if (auto *BinOp = dyn_cast<BinaryOperator>(&Inst)) 170917106792SAlexander Shaposhnikov return Builder.CreateBinOp(BinOp->getOpcode(), 171017106792SAlexander Shaposhnikov Map.getShadow(BinOp->getOperand(0)), 171117106792SAlexander Shaposhnikov Map.getShadow(BinOp->getOperand(1))); 171217106792SAlexander Shaposhnikov 171317106792SAlexander Shaposhnikov if (isa<UIToFPInst>(&Inst) || isa<SIToFPInst>(&Inst)) { 1714430b90f0SWu Yingcong auto *Cast = cast<CastInst>(&Inst); 171517106792SAlexander Shaposhnikov return Builder.CreateCast(Cast->getOpcode(), Cast->getOperand(0), 171617106792SAlexander Shaposhnikov ExtendedVT); 171717106792SAlexander Shaposhnikov } 171817106792SAlexander Shaposhnikov 171917106792SAlexander Shaposhnikov if (auto *S = dyn_cast<SelectInst>(&Inst)) 172017106792SAlexander Shaposhnikov return Builder.CreateSelect(S->getCondition(), 172117106792SAlexander Shaposhnikov Map.getShadow(S->getTrueValue()), 172217106792SAlexander Shaposhnikov Map.getShadow(S->getFalseValue())); 172317106792SAlexander Shaposhnikov 172470a9535fSAlexander Shaposhnikov if (auto *Freeze = dyn_cast<FreezeInst>(&Inst)) 172570a9535fSAlexander Shaposhnikov return Builder.CreateFreeze(Map.getShadow(Freeze->getOperand(0))); 172670a9535fSAlexander Shaposhnikov 172717106792SAlexander Shaposhnikov if (auto *Extract = dyn_cast<ExtractElementInst>(&Inst)) 172817106792SAlexander Shaposhnikov return Builder.CreateExtractElement( 172917106792SAlexander Shaposhnikov Map.getShadow(Extract->getVectorOperand()), Extract->getIndexOperand()); 173017106792SAlexander Shaposhnikov 173117106792SAlexander Shaposhnikov if (auto *Insert = dyn_cast<InsertElementInst>(&Inst)) 173217106792SAlexander Shaposhnikov return Builder.CreateInsertElement(Map.getShadow(Insert->getOperand(0)), 173317106792SAlexander Shaposhnikov Map.getShadow(Insert->getOperand(1)), 173417106792SAlexander Shaposhnikov Insert->getOperand(2)); 173517106792SAlexander Shaposhnikov 173617106792SAlexander Shaposhnikov if (auto *Shuffle = dyn_cast<ShuffleVectorInst>(&Inst)) 173717106792SAlexander Shaposhnikov return Builder.CreateShuffleVector(Map.getShadow(Shuffle->getOperand(0)), 173817106792SAlexander Shaposhnikov Map.getShadow(Shuffle->getOperand(1)), 173917106792SAlexander Shaposhnikov Shuffle->getShuffleMask()); 174017106792SAlexander Shaposhnikov // TODO: We could make aggregate object first class citizens. For now we 174117106792SAlexander Shaposhnikov // just extend the extracted value. 174217106792SAlexander Shaposhnikov if (auto *Extract = dyn_cast<ExtractValueInst>(&Inst)) 174317106792SAlexander Shaposhnikov return Builder.CreateFPExt(Extract, ExtendedVT); 174417106792SAlexander Shaposhnikov 174517106792SAlexander Shaposhnikov if (auto *BC = dyn_cast<BitCastInst>(&Inst)) 174617106792SAlexander Shaposhnikov return Builder.CreateFPExt(BC, ExtendedVT); 174717106792SAlexander Shaposhnikov 174817106792SAlexander Shaposhnikov report_fatal_error("Unimplemented support for " + 174917106792SAlexander Shaposhnikov Twine(Inst.getOpcodeName())); 175017106792SAlexander Shaposhnikov } 175117106792SAlexander Shaposhnikov 175217106792SAlexander Shaposhnikov // Creates a shadow value for an instruction that defines a value of FT type. 175317106792SAlexander Shaposhnikov // FT operands that do not already have shadow values are created recursively. 175417106792SAlexander Shaposhnikov // The DFS is guaranteed to not loop as phis and arguments already have 175517106792SAlexander Shaposhnikov // shadows. 175617106792SAlexander Shaposhnikov void NumericalStabilitySanitizer::maybeCreateShadowValue( 175717106792SAlexander Shaposhnikov Instruction &Root, const TargetLibraryInfo &TLI, ValueToShadowMap &Map) { 175817106792SAlexander Shaposhnikov Type *VT = Root.getType(); 175917106792SAlexander Shaposhnikov Type *ExtendedVT = Config.getExtendedFPType(VT); 176017106792SAlexander Shaposhnikov if (ExtendedVT == nullptr) 176117106792SAlexander Shaposhnikov return; // Not an FT value. 176217106792SAlexander Shaposhnikov 176317106792SAlexander Shaposhnikov if (Map.hasShadow(&Root)) 176417106792SAlexander Shaposhnikov return; // Shadow already exists. 176517106792SAlexander Shaposhnikov 176617106792SAlexander Shaposhnikov assert(!isa<PHINode>(Root) && "phi nodes should already have shadows"); 176717106792SAlexander Shaposhnikov 176817106792SAlexander Shaposhnikov std::vector<Instruction *> DfsStack(1, &Root); 176917106792SAlexander Shaposhnikov while (!DfsStack.empty()) { 177017106792SAlexander Shaposhnikov // Ensure that all operands to the instruction have shadows before 177117106792SAlexander Shaposhnikov // proceeding. 177217106792SAlexander Shaposhnikov Instruction *I = DfsStack.back(); 177317106792SAlexander Shaposhnikov // The shadow for the instruction might have been created deeper in the DFS, 177417106792SAlexander Shaposhnikov // see `forward_use_with_two_uses` test. 177517106792SAlexander Shaposhnikov if (Map.hasShadow(I)) { 177617106792SAlexander Shaposhnikov DfsStack.pop_back(); 177717106792SAlexander Shaposhnikov continue; 177817106792SAlexander Shaposhnikov } 177917106792SAlexander Shaposhnikov 178017106792SAlexander Shaposhnikov bool MissingShadow = false; 178117106792SAlexander Shaposhnikov for (Value *Op : I->operands()) { 178217106792SAlexander Shaposhnikov Type *VT = Op->getType(); 178317106792SAlexander Shaposhnikov if (!Config.getExtendedFPType(VT)) 178417106792SAlexander Shaposhnikov continue; // Not an FT value. 178517106792SAlexander Shaposhnikov if (Map.hasShadow(Op)) 178617106792SAlexander Shaposhnikov continue; // Shadow is already available. 178717106792SAlexander Shaposhnikov MissingShadow = true; 178817106792SAlexander Shaposhnikov DfsStack.push_back(cast<Instruction>(Op)); 178917106792SAlexander Shaposhnikov } 179017106792SAlexander Shaposhnikov if (MissingShadow) 179117106792SAlexander Shaposhnikov continue; // Process operands and come back to this instruction later. 179217106792SAlexander Shaposhnikov 179317106792SAlexander Shaposhnikov // All operands have shadows. Create a shadow for the current value. 179417106792SAlexander Shaposhnikov Value *Shadow = createShadowValueWithOperandsAvailable(*I, TLI, Map); 179517106792SAlexander Shaposhnikov Map.setShadow(*I, *Shadow); 179617106792SAlexander Shaposhnikov DfsStack.pop_back(); 179717106792SAlexander Shaposhnikov } 179817106792SAlexander Shaposhnikov } 179917106792SAlexander Shaposhnikov 180017106792SAlexander Shaposhnikov // A floating-point store needs its value and type written to shadow memory. 180117106792SAlexander Shaposhnikov void NumericalStabilitySanitizer::propagateFTStore( 180217106792SAlexander Shaposhnikov StoreInst &Store, Type *VT, Type *ExtendedVT, const ValueToShadowMap &Map) { 180317106792SAlexander Shaposhnikov Value *StoredValue = Store.getValueOperand(); 180417106792SAlexander Shaposhnikov IRBuilder<> Builder(&Store); 180517106792SAlexander Shaposhnikov Builder.SetCurrentDebugLocation(Store.getDebugLoc()); 180617106792SAlexander Shaposhnikov const auto Extents = getMemoryExtentsOrDie(VT); 180717106792SAlexander Shaposhnikov Value *ShadowPtr = Builder.CreateCall( 180817106792SAlexander Shaposhnikov NsanGetShadowPtrForStore[Extents.ValueType], 180917106792SAlexander Shaposhnikov {Store.getPointerOperand(), ConstantInt::get(IntptrTy, Extents.NumElts)}); 181017106792SAlexander Shaposhnikov 181117106792SAlexander Shaposhnikov Value *StoredShadow = Map.getShadow(StoredValue); 181217106792SAlexander Shaposhnikov if (!Store.getParent()->getParent()->hasOptNone()) { 181317106792SAlexander Shaposhnikov // Only check stores when optimizing, because non-optimized code generates 181417106792SAlexander Shaposhnikov // too many stores to the stack, creating false positives. 181517106792SAlexander Shaposhnikov if (ClCheckStores) { 181617106792SAlexander Shaposhnikov StoredShadow = emitCheck(StoredValue, StoredShadow, Builder, 181717106792SAlexander Shaposhnikov CheckLoc::makeStore(Store.getPointerOperand())); 181817106792SAlexander Shaposhnikov ++NumInstrumentedFTStores; 181917106792SAlexander Shaposhnikov } 182017106792SAlexander Shaposhnikov } 182117106792SAlexander Shaposhnikov 182217106792SAlexander Shaposhnikov Builder.CreateAlignedStore(StoredShadow, ShadowPtr, Align(1), 182317106792SAlexander Shaposhnikov Store.isVolatile()); 182417106792SAlexander Shaposhnikov } 182517106792SAlexander Shaposhnikov 182617106792SAlexander Shaposhnikov // A non-ft store needs to invalidate shadow memory. Exceptions are: 182717106792SAlexander Shaposhnikov // - memory transfers of floating-point data through other pointer types (llvm 182817106792SAlexander Shaposhnikov // optimization passes transform `*(float*)a = *(float*)b` into 182917106792SAlexander Shaposhnikov // `*(i32*)a = *(i32*)b` ). These have the same semantics as memcpy. 183017106792SAlexander Shaposhnikov // - Writes of FT-sized constants. LLVM likes to do float stores as bitcasted 183117106792SAlexander Shaposhnikov // ints. Note that this is not really necessary because if the value is 183217106792SAlexander Shaposhnikov // unknown the framework will re-extend it on load anyway. It just felt 183317106792SAlexander Shaposhnikov // easier to debug tests with vectors of FTs. 183417106792SAlexander Shaposhnikov void NumericalStabilitySanitizer::propagateNonFTStore( 183517106792SAlexander Shaposhnikov StoreInst &Store, Type *VT, const ValueToShadowMap &Map) { 183617106792SAlexander Shaposhnikov Value *PtrOp = Store.getPointerOperand(); 183717106792SAlexander Shaposhnikov IRBuilder<> Builder(Store.getNextNode()); 183817106792SAlexander Shaposhnikov Builder.SetCurrentDebugLocation(Store.getDebugLoc()); 183917106792SAlexander Shaposhnikov Value *Dst = PtrOp; 184017106792SAlexander Shaposhnikov TypeSize SlotSize = DL.getTypeStoreSize(VT); 184117106792SAlexander Shaposhnikov assert(!SlotSize.isScalable() && "unsupported"); 184217106792SAlexander Shaposhnikov const auto LoadSizeBytes = SlotSize.getFixedValue(); 184317106792SAlexander Shaposhnikov Value *ValueSize = Constant::getIntegerValue( 184417106792SAlexander Shaposhnikov IntptrTy, APInt(IntptrTy->getPrimitiveSizeInBits(), LoadSizeBytes)); 184517106792SAlexander Shaposhnikov 184617106792SAlexander Shaposhnikov ++NumInstrumentedNonFTStores; 184717106792SAlexander Shaposhnikov Value *StoredValue = Store.getValueOperand(); 184817106792SAlexander Shaposhnikov if (LoadInst *Load = dyn_cast<LoadInst>(StoredValue)) { 184917106792SAlexander Shaposhnikov // TODO: Handle the case when the value is from a phi. 185017106792SAlexander Shaposhnikov // This is a memory transfer with memcpy semantics. Copy the type and 185117106792SAlexander Shaposhnikov // value from the source. Note that we cannot use __nsan_copy_values() 185217106792SAlexander Shaposhnikov // here, because that will not work when there is a write to memory in 185317106792SAlexander Shaposhnikov // between the load and the store, e.g. in the case of a swap. 185417106792SAlexander Shaposhnikov Type *ShadowTypeIntTy = Type::getIntNTy(Context, 8 * LoadSizeBytes); 185517106792SAlexander Shaposhnikov Type *ShadowValueIntTy = 185617106792SAlexander Shaposhnikov Type::getIntNTy(Context, 8 * kShadowScale * LoadSizeBytes); 185717106792SAlexander Shaposhnikov IRBuilder<> LoadBuilder(Load->getNextNode()); 185817106792SAlexander Shaposhnikov Builder.SetCurrentDebugLocation(Store.getDebugLoc()); 185917106792SAlexander Shaposhnikov Value *LoadSrc = Load->getPointerOperand(); 186017106792SAlexander Shaposhnikov // Read the shadow type and value at load time. The type has the same size 186117106792SAlexander Shaposhnikov // as the FT value, the value has twice its size. 186217106792SAlexander Shaposhnikov // TODO: cache them to avoid re-creating them when a load is used by 186317106792SAlexander Shaposhnikov // several stores. Maybe create them like the FT shadows when a load is 186417106792SAlexander Shaposhnikov // encountered. 186517106792SAlexander Shaposhnikov Value *RawShadowType = LoadBuilder.CreateAlignedLoad( 186617106792SAlexander Shaposhnikov ShadowTypeIntTy, 186717106792SAlexander Shaposhnikov LoadBuilder.CreateCall(NsanGetRawShadowTypePtr, {LoadSrc}), Align(1), 186817106792SAlexander Shaposhnikov /*isVolatile=*/false); 186917106792SAlexander Shaposhnikov Value *RawShadowValue = LoadBuilder.CreateAlignedLoad( 187017106792SAlexander Shaposhnikov ShadowValueIntTy, 187117106792SAlexander Shaposhnikov LoadBuilder.CreateCall(NsanGetRawShadowPtr, {LoadSrc}), Align(1), 187217106792SAlexander Shaposhnikov /*isVolatile=*/false); 187317106792SAlexander Shaposhnikov 187417106792SAlexander Shaposhnikov // Write back the shadow type and value at store time. 187517106792SAlexander Shaposhnikov Builder.CreateAlignedStore( 187617106792SAlexander Shaposhnikov RawShadowType, Builder.CreateCall(NsanGetRawShadowTypePtr, {Dst}), 187717106792SAlexander Shaposhnikov Align(1), 187817106792SAlexander Shaposhnikov /*isVolatile=*/false); 187917106792SAlexander Shaposhnikov Builder.CreateAlignedStore(RawShadowValue, 188017106792SAlexander Shaposhnikov Builder.CreateCall(NsanGetRawShadowPtr, {Dst}), 188117106792SAlexander Shaposhnikov Align(1), 188217106792SAlexander Shaposhnikov /*isVolatile=*/false); 188317106792SAlexander Shaposhnikov 188417106792SAlexander Shaposhnikov ++NumInstrumentedNonFTMemcpyStores; 188517106792SAlexander Shaposhnikov return; 188617106792SAlexander Shaposhnikov } 188717106792SAlexander Shaposhnikov // ClPropagateNonFTConstStoresAsFT is by default false. 188817106792SAlexander Shaposhnikov if (Constant *C; ClPropagateNonFTConstStoresAsFT && 188917106792SAlexander Shaposhnikov (C = dyn_cast<Constant>(StoredValue))) { 189017106792SAlexander Shaposhnikov // This might be a fp constant stored as an int. Bitcast and store if it has 189117106792SAlexander Shaposhnikov // appropriate size. 189217106792SAlexander Shaposhnikov Type *BitcastTy = nullptr; // The FT type to bitcast to. 189317106792SAlexander Shaposhnikov if (auto *CInt = dyn_cast<ConstantInt>(C)) { 189417106792SAlexander Shaposhnikov switch (CInt->getType()->getScalarSizeInBits()) { 189517106792SAlexander Shaposhnikov case 32: 189617106792SAlexander Shaposhnikov BitcastTy = Type::getFloatTy(Context); 189717106792SAlexander Shaposhnikov break; 189817106792SAlexander Shaposhnikov case 64: 189917106792SAlexander Shaposhnikov BitcastTy = Type::getDoubleTy(Context); 190017106792SAlexander Shaposhnikov break; 190117106792SAlexander Shaposhnikov case 80: 190217106792SAlexander Shaposhnikov BitcastTy = Type::getX86_FP80Ty(Context); 190317106792SAlexander Shaposhnikov break; 190417106792SAlexander Shaposhnikov default: 190517106792SAlexander Shaposhnikov break; 190617106792SAlexander Shaposhnikov } 190717106792SAlexander Shaposhnikov } else if (auto *CDV = dyn_cast<ConstantDataVector>(C)) { 190817106792SAlexander Shaposhnikov const int NumElements = 190917106792SAlexander Shaposhnikov cast<VectorType>(CDV->getType())->getElementCount().getFixedValue(); 191017106792SAlexander Shaposhnikov switch (CDV->getType()->getScalarSizeInBits()) { 191117106792SAlexander Shaposhnikov case 32: 191217106792SAlexander Shaposhnikov BitcastTy = 191317106792SAlexander Shaposhnikov VectorType::get(Type::getFloatTy(Context), NumElements, false); 191417106792SAlexander Shaposhnikov break; 191517106792SAlexander Shaposhnikov case 64: 191617106792SAlexander Shaposhnikov BitcastTy = 191717106792SAlexander Shaposhnikov VectorType::get(Type::getDoubleTy(Context), NumElements, false); 191817106792SAlexander Shaposhnikov break; 191917106792SAlexander Shaposhnikov case 80: 192017106792SAlexander Shaposhnikov BitcastTy = 192117106792SAlexander Shaposhnikov VectorType::get(Type::getX86_FP80Ty(Context), NumElements, false); 192217106792SAlexander Shaposhnikov break; 192317106792SAlexander Shaposhnikov default: 192417106792SAlexander Shaposhnikov break; 192517106792SAlexander Shaposhnikov } 192617106792SAlexander Shaposhnikov } 192717106792SAlexander Shaposhnikov if (BitcastTy) { 192817106792SAlexander Shaposhnikov const MemoryExtents Extents = getMemoryExtentsOrDie(BitcastTy); 192917106792SAlexander Shaposhnikov Value *ShadowPtr = Builder.CreateCall( 193017106792SAlexander Shaposhnikov NsanGetShadowPtrForStore[Extents.ValueType], 193117106792SAlexander Shaposhnikov {PtrOp, ConstantInt::get(IntptrTy, Extents.NumElts)}); 193217106792SAlexander Shaposhnikov // Bitcast the integer value to the appropriate FT type and extend to 2FT. 193317106792SAlexander Shaposhnikov Type *ExtVT = Config.getExtendedFPType(BitcastTy); 193417106792SAlexander Shaposhnikov Value *Shadow = 193517106792SAlexander Shaposhnikov Builder.CreateFPExt(Builder.CreateBitCast(C, BitcastTy), ExtVT); 193617106792SAlexander Shaposhnikov Builder.CreateAlignedStore(Shadow, ShadowPtr, Align(1), 193717106792SAlexander Shaposhnikov Store.isVolatile()); 193817106792SAlexander Shaposhnikov return; 193917106792SAlexander Shaposhnikov } 194017106792SAlexander Shaposhnikov } 194117106792SAlexander Shaposhnikov // All other stores just reset the shadow value to unknown. 1942ddf5725eSDmitry Chestnykh Builder.CreateCall(NsanSetUnknownFns.getFallback(), {Dst, ValueSize}); 194317106792SAlexander Shaposhnikov } 194417106792SAlexander Shaposhnikov 194517106792SAlexander Shaposhnikov void NumericalStabilitySanitizer::propagateShadowValues( 194617106792SAlexander Shaposhnikov Instruction &Inst, const TargetLibraryInfo &TLI, 194717106792SAlexander Shaposhnikov const ValueToShadowMap &Map) { 194817106792SAlexander Shaposhnikov if (auto *Store = dyn_cast<StoreInst>(&Inst)) { 194917106792SAlexander Shaposhnikov Value *StoredValue = Store->getValueOperand(); 195017106792SAlexander Shaposhnikov Type *VT = StoredValue->getType(); 195117106792SAlexander Shaposhnikov Type *ExtendedVT = Config.getExtendedFPType(VT); 195217106792SAlexander Shaposhnikov if (ExtendedVT == nullptr) 195317106792SAlexander Shaposhnikov return propagateNonFTStore(*Store, VT, Map); 195417106792SAlexander Shaposhnikov return propagateFTStore(*Store, VT, ExtendedVT, Map); 195517106792SAlexander Shaposhnikov } 195617106792SAlexander Shaposhnikov 195717106792SAlexander Shaposhnikov if (auto *FCmp = dyn_cast<FCmpInst>(&Inst)) { 195817106792SAlexander Shaposhnikov emitFCmpCheck(*FCmp, Map); 195917106792SAlexander Shaposhnikov return; 196017106792SAlexander Shaposhnikov } 196117106792SAlexander Shaposhnikov 196217106792SAlexander Shaposhnikov if (auto *CB = dyn_cast<CallBase>(&Inst)) { 196317106792SAlexander Shaposhnikov maybeAddSuffixForNsanInterface(CB); 196417106792SAlexander Shaposhnikov if (CallInst *CI = dyn_cast<CallInst>(&Inst)) 196517106792SAlexander Shaposhnikov maybeMarkSanitizerLibraryCallNoBuiltin(CI, &TLI); 196617106792SAlexander Shaposhnikov if (MemIntrinsic *MI = dyn_cast<MemIntrinsic>(&Inst)) { 196717106792SAlexander Shaposhnikov instrumentMemIntrinsic(MI); 196817106792SAlexander Shaposhnikov return; 196917106792SAlexander Shaposhnikov } 197017106792SAlexander Shaposhnikov populateShadowStack(*CB, TLI, Map); 197117106792SAlexander Shaposhnikov return; 197217106792SAlexander Shaposhnikov } 197317106792SAlexander Shaposhnikov 197417106792SAlexander Shaposhnikov if (auto *RetInst = dyn_cast<ReturnInst>(&Inst)) { 197517106792SAlexander Shaposhnikov if (!ClCheckRet) 197617106792SAlexander Shaposhnikov return; 197717106792SAlexander Shaposhnikov 197817106792SAlexander Shaposhnikov Value *RV = RetInst->getReturnValue(); 197917106792SAlexander Shaposhnikov if (RV == nullptr) 198017106792SAlexander Shaposhnikov return; // This is a `ret void`. 198117106792SAlexander Shaposhnikov Type *VT = RV->getType(); 198217106792SAlexander Shaposhnikov Type *ExtendedVT = Config.getExtendedFPType(VT); 198317106792SAlexander Shaposhnikov if (ExtendedVT == nullptr) 198417106792SAlexander Shaposhnikov return; // Not an FT ret. 198517106792SAlexander Shaposhnikov Value *RVShadow = Map.getShadow(RV); 198617106792SAlexander Shaposhnikov IRBuilder<> Builder(RetInst); 198717106792SAlexander Shaposhnikov 198817106792SAlexander Shaposhnikov RVShadow = emitCheck(RV, RVShadow, Builder, CheckLoc::makeRet()); 198917106792SAlexander Shaposhnikov ++NumInstrumentedFTRets; 199017106792SAlexander Shaposhnikov // Store tag. 199117106792SAlexander Shaposhnikov Value *FnAddr = 199217106792SAlexander Shaposhnikov Builder.CreatePtrToInt(Inst.getParent()->getParent(), IntptrTy); 199317106792SAlexander Shaposhnikov Builder.CreateStore(FnAddr, NsanShadowRetTag); 199417106792SAlexander Shaposhnikov // Store value. 199517106792SAlexander Shaposhnikov Value *ShadowRetValPtr = 199617106792SAlexander Shaposhnikov Builder.CreateConstGEP2_64(NsanShadowRetType, NsanShadowRetPtr, 0, 0); 199717106792SAlexander Shaposhnikov Builder.CreateStore(RVShadow, ShadowRetValPtr); 199817106792SAlexander Shaposhnikov return; 199917106792SAlexander Shaposhnikov } 200017106792SAlexander Shaposhnikov 200117106792SAlexander Shaposhnikov if (InsertValueInst *Insert = dyn_cast<InsertValueInst>(&Inst)) { 200217106792SAlexander Shaposhnikov Value *V = Insert->getOperand(1); 200317106792SAlexander Shaposhnikov Type *VT = V->getType(); 200417106792SAlexander Shaposhnikov Type *ExtendedVT = Config.getExtendedFPType(VT); 200517106792SAlexander Shaposhnikov if (ExtendedVT == nullptr) 200617106792SAlexander Shaposhnikov return; 200717106792SAlexander Shaposhnikov IRBuilder<> Builder(Insert); 200817106792SAlexander Shaposhnikov emitCheck(V, Map.getShadow(V), Builder, CheckLoc::makeInsert()); 200917106792SAlexander Shaposhnikov return; 201017106792SAlexander Shaposhnikov } 201117106792SAlexander Shaposhnikov } 201217106792SAlexander Shaposhnikov 201317106792SAlexander Shaposhnikov // Moves fast math flags from the function to individual instructions, and 201417106792SAlexander Shaposhnikov // removes the attribute from the function. 201517106792SAlexander Shaposhnikov // TODO: Make this controllable with a flag. 201617106792SAlexander Shaposhnikov static void moveFastMathFlags(Function &F, 201717106792SAlexander Shaposhnikov std::vector<Instruction *> &Instructions) { 201817106792SAlexander Shaposhnikov FastMathFlags FMF; 201917106792SAlexander Shaposhnikov #define MOVE_FLAG(attr, setter) \ 202017106792SAlexander Shaposhnikov if (F.getFnAttribute(attr).getValueAsString() == "true") { \ 202117106792SAlexander Shaposhnikov F.removeFnAttr(attr); \ 202217106792SAlexander Shaposhnikov FMF.set##setter(); \ 202317106792SAlexander Shaposhnikov } 202417106792SAlexander Shaposhnikov MOVE_FLAG("unsafe-fp-math", Fast) 202517106792SAlexander Shaposhnikov MOVE_FLAG("no-infs-fp-math", NoInfs) 202617106792SAlexander Shaposhnikov MOVE_FLAG("no-nans-fp-math", NoNaNs) 202717106792SAlexander Shaposhnikov MOVE_FLAG("no-signed-zeros-fp-math", NoSignedZeros) 202817106792SAlexander Shaposhnikov #undef MOVE_FLAG 202917106792SAlexander Shaposhnikov 203017106792SAlexander Shaposhnikov for (Instruction *I : Instructions) 203117106792SAlexander Shaposhnikov if (isa<FPMathOperator>(I)) 203217106792SAlexander Shaposhnikov I->setFastMathFlags(FMF); 203317106792SAlexander Shaposhnikov } 203417106792SAlexander Shaposhnikov 203517106792SAlexander Shaposhnikov bool NumericalStabilitySanitizer::sanitizeFunction( 203617106792SAlexander Shaposhnikov Function &F, const TargetLibraryInfo &TLI) { 2037d23c24f3SAlexander Shaposhnikov if (!F.hasFnAttribute(Attribute::SanitizeNumericalStability) || 2038d23c24f3SAlexander Shaposhnikov F.isDeclaration()) 203917106792SAlexander Shaposhnikov return false; 204017106792SAlexander Shaposhnikov 204117106792SAlexander Shaposhnikov // This is required to prevent instrumenting call to __nsan_init from within 204217106792SAlexander Shaposhnikov // the module constructor. 204317106792SAlexander Shaposhnikov if (F.getName() == kNsanModuleCtorName) 204417106792SAlexander Shaposhnikov return false; 204517106792SAlexander Shaposhnikov SmallVector<Instruction *, 8> AllLoadsAndStores; 204617106792SAlexander Shaposhnikov SmallVector<Instruction *, 8> LocalLoadsAndStores; 204717106792SAlexander Shaposhnikov 204817106792SAlexander Shaposhnikov // The instrumentation maintains: 204917106792SAlexander Shaposhnikov // - for each IR value `v` of floating-point (or vector floating-point) type 205017106792SAlexander Shaposhnikov // FT, a shadow IR value `s(v)` with twice the precision 2FT (e.g. 205117106792SAlexander Shaposhnikov // double for float and f128 for double). 205217106792SAlexander Shaposhnikov // - A shadow memory, which stores `s(v)` for any `v` that has been stored, 205317106792SAlexander Shaposhnikov // along with a shadow memory tag, which stores whether the value in the 205417106792SAlexander Shaposhnikov // corresponding shadow memory is valid. Note that this might be 205517106792SAlexander Shaposhnikov // incorrect if a non-instrumented function stores to memory, or if 205617106792SAlexander Shaposhnikov // memory is stored to through a char pointer. 205717106792SAlexander Shaposhnikov // - A shadow stack, which holds `s(v)` for any floating-point argument `v` 205817106792SAlexander Shaposhnikov // of a call to an instrumented function. This allows 205917106792SAlexander Shaposhnikov // instrumented functions to retrieve the shadow values for their 206017106792SAlexander Shaposhnikov // arguments. 206117106792SAlexander Shaposhnikov // Because instrumented functions can be called from non-instrumented 206217106792SAlexander Shaposhnikov // functions, the stack needs to include a tag so that the instrumented 206317106792SAlexander Shaposhnikov // function knows whether shadow values are available for their 206417106792SAlexander Shaposhnikov // parameters (i.e. whether is was called by an instrumented function). 206517106792SAlexander Shaposhnikov // When shadow arguments are not available, they have to be recreated by 206617106792SAlexander Shaposhnikov // extending the precision of the non-shadow arguments to the non-shadow 206717106792SAlexander Shaposhnikov // value. Non-instrumented functions do not modify (or even know about) the 206817106792SAlexander Shaposhnikov // shadow stack. The shadow stack pointer is __nsan_shadow_args. The shadow 206917106792SAlexander Shaposhnikov // stack tag is __nsan_shadow_args_tag. The tag is any unique identifier 207017106792SAlexander Shaposhnikov // for the function (we use the address of the function). Both variables 207117106792SAlexander Shaposhnikov // are thread local. 207217106792SAlexander Shaposhnikov // Example: 207317106792SAlexander Shaposhnikov // calls shadow stack tag shadow stack 207417106792SAlexander Shaposhnikov // ======================================================================= 207517106792SAlexander Shaposhnikov // non_instrumented_1() 0 0 207617106792SAlexander Shaposhnikov // | 207717106792SAlexander Shaposhnikov // v 207817106792SAlexander Shaposhnikov // instrumented_2(float a) 0 0 207917106792SAlexander Shaposhnikov // | 208017106792SAlexander Shaposhnikov // v 208117106792SAlexander Shaposhnikov // instrumented_3(float b, double c) &instrumented_3 s(b),s(c) 208217106792SAlexander Shaposhnikov // | 208317106792SAlexander Shaposhnikov // v 208417106792SAlexander Shaposhnikov // instrumented_4(float d) &instrumented_4 s(d) 208517106792SAlexander Shaposhnikov // | 208617106792SAlexander Shaposhnikov // v 208717106792SAlexander Shaposhnikov // non_instrumented_5(float e) &non_instrumented_5 s(e) 208817106792SAlexander Shaposhnikov // | 208917106792SAlexander Shaposhnikov // v 209017106792SAlexander Shaposhnikov // instrumented_6(float f) &non_instrumented_5 s(e) 209117106792SAlexander Shaposhnikov // 209217106792SAlexander Shaposhnikov // On entry, instrumented_2 checks whether the tag corresponds to its 209317106792SAlexander Shaposhnikov // function ptr. 209417106792SAlexander Shaposhnikov // Note that functions reset the tag to 0 after reading shadow parameters. 209517106792SAlexander Shaposhnikov // This ensures that the function does not erroneously read invalid data if 209617106792SAlexander Shaposhnikov // called twice in the same stack, once from an instrumented function and 209717106792SAlexander Shaposhnikov // once from an uninstrumented one. For example, in the following example, 209817106792SAlexander Shaposhnikov // resetting the tag in (A) ensures that (B) does not reuse the same the 209917106792SAlexander Shaposhnikov // shadow arguments (which would be incorrect). 210017106792SAlexander Shaposhnikov // instrumented_1(float a) 210117106792SAlexander Shaposhnikov // | 210217106792SAlexander Shaposhnikov // v 210317106792SAlexander Shaposhnikov // instrumented_2(float b) (A) 210417106792SAlexander Shaposhnikov // | 210517106792SAlexander Shaposhnikov // v 210617106792SAlexander Shaposhnikov // non_instrumented_3() 210717106792SAlexander Shaposhnikov // | 210817106792SAlexander Shaposhnikov // v 210917106792SAlexander Shaposhnikov // instrumented_2(float b) (B) 211017106792SAlexander Shaposhnikov // 211117106792SAlexander Shaposhnikov // - A shadow return slot. Any function that returns a floating-point value 211217106792SAlexander Shaposhnikov // places a shadow return value in __nsan_shadow_ret_val. Again, because 211317106792SAlexander Shaposhnikov // we might be calling non-instrumented functions, this value is guarded 211417106792SAlexander Shaposhnikov // by __nsan_shadow_ret_tag marker indicating which instrumented function 211517106792SAlexander Shaposhnikov // placed the value in __nsan_shadow_ret_val, so that the caller can check 211617106792SAlexander Shaposhnikov // that this corresponds to the callee. Both variables are thread local. 211717106792SAlexander Shaposhnikov // 211817106792SAlexander Shaposhnikov // For example, in the following example, the instrumentation in 211917106792SAlexander Shaposhnikov // `instrumented_1` rejects the shadow return value from `instrumented_3` 212017106792SAlexander Shaposhnikov // because is is not tagged as expected (`&instrumented_3` instead of 212117106792SAlexander Shaposhnikov // `non_instrumented_2`): 212217106792SAlexander Shaposhnikov // 212317106792SAlexander Shaposhnikov // instrumented_1() 212417106792SAlexander Shaposhnikov // | 212517106792SAlexander Shaposhnikov // v 212617106792SAlexander Shaposhnikov // float non_instrumented_2() 212717106792SAlexander Shaposhnikov // | 212817106792SAlexander Shaposhnikov // v 212917106792SAlexander Shaposhnikov // float instrumented_3() 213017106792SAlexander Shaposhnikov // 213117106792SAlexander Shaposhnikov // Calls of known math functions (sin, cos, exp, ...) are duplicated to call 213217106792SAlexander Shaposhnikov // their overload on the shadow type. 213317106792SAlexander Shaposhnikov 213417106792SAlexander Shaposhnikov // Collect all instructions before processing, as creating shadow values 213517106792SAlexander Shaposhnikov // creates new instructions inside the function. 213617106792SAlexander Shaposhnikov std::vector<Instruction *> OriginalInstructions; 213717106792SAlexander Shaposhnikov for (BasicBlock &BB : F) 213817106792SAlexander Shaposhnikov for (Instruction &Inst : BB) 213917106792SAlexander Shaposhnikov OriginalInstructions.emplace_back(&Inst); 214017106792SAlexander Shaposhnikov 214117106792SAlexander Shaposhnikov moveFastMathFlags(F, OriginalInstructions); 214217106792SAlexander Shaposhnikov ValueToShadowMap ValueToShadow(Config); 214317106792SAlexander Shaposhnikov 214417106792SAlexander Shaposhnikov // In the first pass, we create shadow values for all FT function arguments 214517106792SAlexander Shaposhnikov // and all phis. This ensures that the DFS of the next pass does not have 214617106792SAlexander Shaposhnikov // any loops. 214717106792SAlexander Shaposhnikov std::vector<PHINode *> OriginalPhis; 214817106792SAlexander Shaposhnikov createShadowArguments(F, TLI, ValueToShadow); 214917106792SAlexander Shaposhnikov for (Instruction *I : OriginalInstructions) { 215017106792SAlexander Shaposhnikov if (PHINode *Phi = dyn_cast<PHINode>(I)) { 215117106792SAlexander Shaposhnikov if (PHINode *Shadow = maybeCreateShadowPhi(*Phi, TLI)) { 215217106792SAlexander Shaposhnikov OriginalPhis.push_back(Phi); 215317106792SAlexander Shaposhnikov ValueToShadow.setShadow(*Phi, *Shadow); 215417106792SAlexander Shaposhnikov } 215517106792SAlexander Shaposhnikov } 215617106792SAlexander Shaposhnikov } 215717106792SAlexander Shaposhnikov 215817106792SAlexander Shaposhnikov // Create shadow values for all instructions creating FT values. 215917106792SAlexander Shaposhnikov for (Instruction *I : OriginalInstructions) 216017106792SAlexander Shaposhnikov maybeCreateShadowValue(*I, TLI, ValueToShadow); 216117106792SAlexander Shaposhnikov 216217106792SAlexander Shaposhnikov // Propagate shadow values across stores, calls and rets. 216317106792SAlexander Shaposhnikov for (Instruction *I : OriginalInstructions) 216417106792SAlexander Shaposhnikov propagateShadowValues(*I, TLI, ValueToShadow); 216517106792SAlexander Shaposhnikov 216617106792SAlexander Shaposhnikov // The last pass populates shadow phis with shadow values. 216717106792SAlexander Shaposhnikov for (PHINode *Phi : OriginalPhis) { 2168430b90f0SWu Yingcong PHINode *ShadowPhi = cast<PHINode>(ValueToShadow.getShadow(Phi)); 216917106792SAlexander Shaposhnikov for (unsigned I : seq(Phi->getNumOperands())) { 217017106792SAlexander Shaposhnikov Value *V = Phi->getOperand(I); 217117106792SAlexander Shaposhnikov Value *Shadow = ValueToShadow.getShadow(V); 217217106792SAlexander Shaposhnikov BasicBlock *IncomingBB = Phi->getIncomingBlock(I); 217317106792SAlexander Shaposhnikov // For some instructions (e.g. invoke), we create the shadow in a separate 217417106792SAlexander Shaposhnikov // block, different from the block where the original value is created. 217517106792SAlexander Shaposhnikov // In that case, the shadow phi might need to refer to this block instead 217617106792SAlexander Shaposhnikov // of the original block. 217717106792SAlexander Shaposhnikov // Note that this can only happen for instructions as constant shadows are 217817106792SAlexander Shaposhnikov // always created in the same block. 217917106792SAlexander Shaposhnikov ShadowPhi->addIncoming(Shadow, IncomingBB); 218017106792SAlexander Shaposhnikov } 218117106792SAlexander Shaposhnikov } 218217106792SAlexander Shaposhnikov 218317106792SAlexander Shaposhnikov return !ValueToShadow.empty(); 218417106792SAlexander Shaposhnikov } 218517106792SAlexander Shaposhnikov 2186ddf5725eSDmitry Chestnykh static uint64_t GetMemOpSize(Value *V) { 2187ddf5725eSDmitry Chestnykh uint64_t OpSize = 0; 2188ddf5725eSDmitry Chestnykh if (Constant *C = dyn_cast<Constant>(V)) { 2189ddf5725eSDmitry Chestnykh auto *CInt = dyn_cast<ConstantInt>(C); 2190ddf5725eSDmitry Chestnykh if (CInt && CInt->getValue().getBitWidth() <= 64) 2191ddf5725eSDmitry Chestnykh OpSize = CInt->getValue().getZExtValue(); 2192ddf5725eSDmitry Chestnykh } 2193ddf5725eSDmitry Chestnykh 2194ddf5725eSDmitry Chestnykh return OpSize; 2195ddf5725eSDmitry Chestnykh } 2196ddf5725eSDmitry Chestnykh 219717106792SAlexander Shaposhnikov // Instrument the memory intrinsics so that they properly modify the shadow 219817106792SAlexander Shaposhnikov // memory. 219917106792SAlexander Shaposhnikov bool NumericalStabilitySanitizer::instrumentMemIntrinsic(MemIntrinsic *MI) { 220017106792SAlexander Shaposhnikov IRBuilder<> Builder(MI); 220117106792SAlexander Shaposhnikov if (auto *M = dyn_cast<MemSetInst>(MI)) { 2202ddf5725eSDmitry Chestnykh FunctionCallee SetUnknownFn = 2203ddf5725eSDmitry Chestnykh NsanSetUnknownFns.getFunctionFor(GetMemOpSize(M->getArgOperand(2))); 2204ddf5725eSDmitry Chestnykh if (SetUnknownFn.getFunctionType()->getNumParams() == 1) 2205ddf5725eSDmitry Chestnykh Builder.CreateCall(SetUnknownFn, {/*Address=*/M->getArgOperand(0)}); 2206ddf5725eSDmitry Chestnykh else 2207ddf5725eSDmitry Chestnykh Builder.CreateCall(SetUnknownFn, 220817106792SAlexander Shaposhnikov {/*Address=*/M->getArgOperand(0), 2209ddf5725eSDmitry Chestnykh /*Size=*/Builder.CreateIntCast(M->getArgOperand(2), 2210ddf5725eSDmitry Chestnykh IntptrTy, false)}); 2211ddf5725eSDmitry Chestnykh 221217106792SAlexander Shaposhnikov } else if (auto *M = dyn_cast<MemTransferInst>(MI)) { 2213ddf5725eSDmitry Chestnykh FunctionCallee CopyFn = 2214ddf5725eSDmitry Chestnykh NsanCopyFns.getFunctionFor(GetMemOpSize(M->getArgOperand(2))); 2215ddf5725eSDmitry Chestnykh 2216ddf5725eSDmitry Chestnykh if (CopyFn.getFunctionType()->getNumParams() == 2) 2217ddf5725eSDmitry Chestnykh Builder.CreateCall(CopyFn, {/*Destination=*/M->getArgOperand(0), 2218ddf5725eSDmitry Chestnykh /*Source=*/M->getArgOperand(1)}); 2219ddf5725eSDmitry Chestnykh else 2220ddf5725eSDmitry Chestnykh Builder.CreateCall(CopyFn, {/*Destination=*/M->getArgOperand(0), 222117106792SAlexander Shaposhnikov /*Source=*/M->getArgOperand(1), 2222ddf5725eSDmitry Chestnykh /*Size=*/ 2223ddf5725eSDmitry Chestnykh Builder.CreateIntCast(M->getArgOperand(2), 2224ddf5725eSDmitry Chestnykh IntptrTy, false)}); 222517106792SAlexander Shaposhnikov } 222617106792SAlexander Shaposhnikov return false; 222717106792SAlexander Shaposhnikov } 222817106792SAlexander Shaposhnikov 222917106792SAlexander Shaposhnikov void NumericalStabilitySanitizer::maybeAddSuffixForNsanInterface(CallBase *CI) { 223017106792SAlexander Shaposhnikov Function *Fn = CI->getCalledFunction(); 223117106792SAlexander Shaposhnikov if (Fn == nullptr) 223217106792SAlexander Shaposhnikov return; 223317106792SAlexander Shaposhnikov 223417106792SAlexander Shaposhnikov if (!Fn->getName().starts_with("__nsan_")) 223517106792SAlexander Shaposhnikov return; 223617106792SAlexander Shaposhnikov 223717106792SAlexander Shaposhnikov if (Fn->getName() == "__nsan_dump_shadow_mem") { 223817106792SAlexander Shaposhnikov assert(CI->arg_size() == 4 && 223917106792SAlexander Shaposhnikov "invalid prototype for __nsan_dump_shadow_mem"); 224017106792SAlexander Shaposhnikov // __nsan_dump_shadow_mem requires an extra parameter with the dynamic 224117106792SAlexander Shaposhnikov // configuration: 224217106792SAlexander Shaposhnikov // (shadow_type_id_for_long_double << 16) | (shadow_type_id_for_double << 8) 224317106792SAlexander Shaposhnikov // | shadow_type_id_for_double 224417106792SAlexander Shaposhnikov const uint64_t shadow_value_type_ids = 224517106792SAlexander Shaposhnikov (static_cast<size_t>(Config.byValueType(kLongDouble).getNsanTypeId()) 224617106792SAlexander Shaposhnikov << 16) | 224717106792SAlexander Shaposhnikov (static_cast<size_t>(Config.byValueType(kDouble).getNsanTypeId()) 224817106792SAlexander Shaposhnikov << 8) | 224917106792SAlexander Shaposhnikov static_cast<size_t>(Config.byValueType(kFloat).getNsanTypeId()); 225017106792SAlexander Shaposhnikov CI->setArgOperand(3, ConstantInt::get(IntptrTy, shadow_value_type_ids)); 225117106792SAlexander Shaposhnikov } 225217106792SAlexander Shaposhnikov } 2253