xref: /llvm-project/llvm/lib/Transforms/Instrumentation/NumericalStabilitySanitizer.cpp (revision 6292a808b3524d9ba6f4ce55bc5b9e547b088dd8)
117106792SAlexander Shaposhnikov //===-- NumericalStabilitySanitizer.cpp -----------------------------------===//
217106792SAlexander Shaposhnikov //
317106792SAlexander Shaposhnikov // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
417106792SAlexander Shaposhnikov // See https://llvm.org/LICENSE.txt for license information.
517106792SAlexander Shaposhnikov // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
617106792SAlexander Shaposhnikov //
717106792SAlexander Shaposhnikov //===----------------------------------------------------------------------===//
817106792SAlexander Shaposhnikov //
917106792SAlexander Shaposhnikov // This file contains the instrumentation pass for the numerical sanitizer.
1017106792SAlexander Shaposhnikov // Conceptually the pass injects shadow computations using higher precision
1117106792SAlexander Shaposhnikov // types and inserts consistency checks. For details see the paper
1217106792SAlexander Shaposhnikov // https://arxiv.org/abs/2102.12782.
1317106792SAlexander Shaposhnikov //
1417106792SAlexander Shaposhnikov //===----------------------------------------------------------------------===//
1517106792SAlexander Shaposhnikov 
1617106792SAlexander Shaposhnikov #include "llvm/Transforms/Instrumentation/NumericalStabilitySanitizer.h"
1717106792SAlexander Shaposhnikov 
1817106792SAlexander Shaposhnikov #include "llvm/ADT/DenseMap.h"
1917106792SAlexander Shaposhnikov #include "llvm/ADT/SmallVector.h"
2017106792SAlexander Shaposhnikov #include "llvm/ADT/Statistic.h"
2117106792SAlexander Shaposhnikov #include "llvm/ADT/StringExtras.h"
2217106792SAlexander Shaposhnikov #include "llvm/Analysis/TargetLibraryInfo.h"
2317106792SAlexander Shaposhnikov #include "llvm/Analysis/ValueTracking.h"
2417106792SAlexander Shaposhnikov #include "llvm/IR/DataLayout.h"
2517106792SAlexander Shaposhnikov #include "llvm/IR/Function.h"
2617106792SAlexander Shaposhnikov #include "llvm/IR/IRBuilder.h"
2717106792SAlexander Shaposhnikov #include "llvm/IR/IntrinsicInst.h"
2817106792SAlexander Shaposhnikov #include "llvm/IR/Intrinsics.h"
2917106792SAlexander Shaposhnikov #include "llvm/IR/LLVMContext.h"
3017106792SAlexander Shaposhnikov #include "llvm/IR/MDBuilder.h"
3117106792SAlexander Shaposhnikov #include "llvm/IR/Metadata.h"
3217106792SAlexander Shaposhnikov #include "llvm/IR/Module.h"
3317106792SAlexander Shaposhnikov #include "llvm/IR/Type.h"
3417106792SAlexander Shaposhnikov #include "llvm/Support/CommandLine.h"
3517106792SAlexander Shaposhnikov #include "llvm/Support/Debug.h"
3617106792SAlexander Shaposhnikov #include "llvm/Support/Regex.h"
3717106792SAlexander Shaposhnikov #include "llvm/Support/raw_ostream.h"
3817106792SAlexander Shaposhnikov #include "llvm/Transforms/Utils/BasicBlockUtils.h"
392ae968a0SAntonio Frighetto #include "llvm/Transforms/Utils/Instrumentation.h"
4017106792SAlexander Shaposhnikov #include "llvm/Transforms/Utils/Local.h"
4117106792SAlexander Shaposhnikov #include "llvm/Transforms/Utils/ModuleUtils.h"
4217106792SAlexander Shaposhnikov 
4317106792SAlexander Shaposhnikov #include <cstdint>
4417106792SAlexander Shaposhnikov 
4517106792SAlexander Shaposhnikov using namespace llvm;
4617106792SAlexander Shaposhnikov 
4717106792SAlexander Shaposhnikov #define DEBUG_TYPE "nsan"
4817106792SAlexander Shaposhnikov 
4917106792SAlexander Shaposhnikov STATISTIC(NumInstrumentedFTLoads,
5017106792SAlexander Shaposhnikov           "Number of instrumented floating-point loads");
5117106792SAlexander Shaposhnikov 
5217106792SAlexander Shaposhnikov STATISTIC(NumInstrumentedFTCalls,
5317106792SAlexander Shaposhnikov           "Number of instrumented floating-point calls");
5417106792SAlexander Shaposhnikov STATISTIC(NumInstrumentedFTRets,
5517106792SAlexander Shaposhnikov           "Number of instrumented floating-point returns");
5617106792SAlexander Shaposhnikov STATISTIC(NumInstrumentedFTStores,
5717106792SAlexander Shaposhnikov           "Number of instrumented floating-point stores");
5817106792SAlexander Shaposhnikov STATISTIC(NumInstrumentedNonFTStores,
5917106792SAlexander Shaposhnikov           "Number of instrumented non floating-point stores");
6017106792SAlexander Shaposhnikov STATISTIC(
6117106792SAlexander Shaposhnikov     NumInstrumentedNonFTMemcpyStores,
6217106792SAlexander Shaposhnikov     "Number of instrumented non floating-point stores with memcpy semantics");
6317106792SAlexander Shaposhnikov STATISTIC(NumInstrumentedFCmp, "Number of instrumented fcmps");
6417106792SAlexander Shaposhnikov 
6517106792SAlexander Shaposhnikov // Using smaller shadow types types can help improve speed. For example, `dlq`
6617106792SAlexander Shaposhnikov // is 3x slower to 5x faster in opt mode and 2-6x faster in dbg mode compared to
6717106792SAlexander Shaposhnikov // `dqq`.
6817106792SAlexander Shaposhnikov static cl::opt<std::string> ClShadowMapping(
6917106792SAlexander Shaposhnikov     "nsan-shadow-type-mapping", cl::init("dqq"),
7017106792SAlexander Shaposhnikov     cl::desc("One shadow type id for each of `float`, `double`, `long double`. "
7117106792SAlexander Shaposhnikov              "`d`,`l`,`q`,`e` mean double, x86_fp80, fp128 (quad) and "
7217106792SAlexander Shaposhnikov              "ppc_fp128 (extended double) respectively. The default is to "
7317106792SAlexander Shaposhnikov              "shadow `float` as `double`, and `double` and `x86_fp80` as "
7417106792SAlexander Shaposhnikov              "`fp128`"),
7517106792SAlexander Shaposhnikov     cl::Hidden);
7617106792SAlexander Shaposhnikov 
7717106792SAlexander Shaposhnikov static cl::opt<bool>
7817106792SAlexander Shaposhnikov     ClInstrumentFCmp("nsan-instrument-fcmp", cl::init(true),
7917106792SAlexander Shaposhnikov                      cl::desc("Instrument floating-point comparisons"),
8017106792SAlexander Shaposhnikov                      cl::Hidden);
8117106792SAlexander Shaposhnikov 
8217106792SAlexander Shaposhnikov static cl::opt<std::string> ClCheckFunctionsFilter(
8317106792SAlexander Shaposhnikov     "check-functions-filter",
8417106792SAlexander Shaposhnikov     cl::desc("Only emit checks for arguments of functions "
8517106792SAlexander Shaposhnikov              "whose names match the given regular expression"),
8617106792SAlexander Shaposhnikov     cl::value_desc("regex"));
8717106792SAlexander Shaposhnikov 
8817106792SAlexander Shaposhnikov static cl::opt<bool> ClTruncateFCmpEq(
8917106792SAlexander Shaposhnikov     "nsan-truncate-fcmp-eq", cl::init(true),
9017106792SAlexander Shaposhnikov     cl::desc(
9117106792SAlexander Shaposhnikov         "This flag controls the behaviour of fcmp equality comparisons."
9217106792SAlexander Shaposhnikov         "For equality comparisons such as `x == 0.0f`, we can perform the "
9317106792SAlexander Shaposhnikov         "shadow check in the shadow (`x_shadow == 0.0) == (x == 0.0f)`) or app "
9417106792SAlexander Shaposhnikov         " domain (`(trunc(x_shadow) == 0.0f) == (x == 0.0f)`). This helps "
9517106792SAlexander Shaposhnikov         "catch the case when `x_shadow` is accurate enough (and therefore "
9617106792SAlexander Shaposhnikov         "close enough to zero) so that `trunc(x_shadow)` is zero even though "
9717106792SAlexander Shaposhnikov         "both `x` and `x_shadow` are not"),
9817106792SAlexander Shaposhnikov     cl::Hidden);
9917106792SAlexander Shaposhnikov 
10017106792SAlexander Shaposhnikov // When there is external, uninstrumented code writing to memory, the shadow
10117106792SAlexander Shaposhnikov // memory can get out of sync with the application memory. Enabling this flag
10217106792SAlexander Shaposhnikov // emits consistency checks for loads to catch this situation.
10317106792SAlexander Shaposhnikov // When everything is instrumented, this is not strictly necessary because any
10417106792SAlexander Shaposhnikov // load should have a corresponding store, but can help debug cases when the
10517106792SAlexander Shaposhnikov // framework did a bad job at tracking shadow memory modifications by failing on
10617106792SAlexander Shaposhnikov // load rather than store.
10717106792SAlexander Shaposhnikov // TODO: provide a way to resume computations from the FT value when the load
10817106792SAlexander Shaposhnikov // is inconsistent. This ensures that further computations are not polluted.
10917106792SAlexander Shaposhnikov static cl::opt<bool> ClCheckLoads("nsan-check-loads",
11017106792SAlexander Shaposhnikov                                   cl::desc("Check floating-point load"),
11117106792SAlexander Shaposhnikov                                   cl::Hidden);
11217106792SAlexander Shaposhnikov 
11317106792SAlexander Shaposhnikov static cl::opt<bool> ClCheckStores("nsan-check-stores", cl::init(true),
11417106792SAlexander Shaposhnikov                                    cl::desc("Check floating-point stores"),
11517106792SAlexander Shaposhnikov                                    cl::Hidden);
11617106792SAlexander Shaposhnikov 
11717106792SAlexander Shaposhnikov static cl::opt<bool> ClCheckRet("nsan-check-ret", cl::init(true),
11817106792SAlexander Shaposhnikov                                 cl::desc("Check floating-point return values"),
11917106792SAlexander Shaposhnikov                                 cl::Hidden);
12017106792SAlexander Shaposhnikov 
12117106792SAlexander Shaposhnikov // LLVM may store constant floats as bitcasted ints.
12217106792SAlexander Shaposhnikov // It's not really necessary to shadow such stores,
12317106792SAlexander Shaposhnikov // if the shadow value is unknown the framework will re-extend it on load
12417106792SAlexander Shaposhnikov // anyway. Moreover, because of size collisions (e.g. bf16 vs f16) it is
12517106792SAlexander Shaposhnikov // impossible to determine the floating-point type based on the size.
12617106792SAlexander Shaposhnikov // However, for debugging purposes it can be useful to model such stores.
12717106792SAlexander Shaposhnikov static cl::opt<bool> ClPropagateNonFTConstStoresAsFT(
12817106792SAlexander Shaposhnikov     "nsan-propagate-non-ft-const-stores-as-ft",
12917106792SAlexander Shaposhnikov     cl::desc(
13017106792SAlexander Shaposhnikov         "Propagate non floating-point const stores as floating point values."
13117106792SAlexander Shaposhnikov         "For debugging purposes only"),
13217106792SAlexander Shaposhnikov     cl::Hidden);
13317106792SAlexander Shaposhnikov 
13417106792SAlexander Shaposhnikov constexpr StringLiteral kNsanModuleCtorName("nsan.module_ctor");
13517106792SAlexander Shaposhnikov constexpr StringLiteral kNsanInitName("__nsan_init");
13617106792SAlexander Shaposhnikov 
13717106792SAlexander Shaposhnikov // The following values must be kept in sync with the runtime.
13817106792SAlexander Shaposhnikov constexpr int kShadowScale = 2;
13917106792SAlexander Shaposhnikov constexpr int kMaxVectorWidth = 8;
14017106792SAlexander Shaposhnikov constexpr int kMaxNumArgs = 128;
14117106792SAlexander Shaposhnikov constexpr int kMaxShadowTypeSizeBytes = 16; // fp128
14217106792SAlexander Shaposhnikov 
14317106792SAlexander Shaposhnikov namespace {
14417106792SAlexander Shaposhnikov 
14517106792SAlexander Shaposhnikov // Defines the characteristics (type id, type, and floating-point semantics)
14617106792SAlexander Shaposhnikov // attached for all possible shadow types.
14717106792SAlexander Shaposhnikov class ShadowTypeConfig {
14817106792SAlexander Shaposhnikov public:
14917106792SAlexander Shaposhnikov   static std::unique_ptr<ShadowTypeConfig> fromNsanTypeId(char TypeId);
15017106792SAlexander Shaposhnikov 
15117106792SAlexander Shaposhnikov   // The LLVM Type corresponding to the shadow type.
15217106792SAlexander Shaposhnikov   virtual Type *getType(LLVMContext &Context) const = 0;
15317106792SAlexander Shaposhnikov 
15417106792SAlexander Shaposhnikov   // The nsan type id of the shadow type (`d`, `l`, `q`, ...).
15517106792SAlexander Shaposhnikov   virtual char getNsanTypeId() const = 0;
15617106792SAlexander Shaposhnikov 
15717106792SAlexander Shaposhnikov   virtual ~ShadowTypeConfig() = default;
15817106792SAlexander Shaposhnikov };
15917106792SAlexander Shaposhnikov 
16017106792SAlexander Shaposhnikov template <char NsanTypeId>
16117106792SAlexander Shaposhnikov class ShadowTypeConfigImpl : public ShadowTypeConfig {
16217106792SAlexander Shaposhnikov public:
16317106792SAlexander Shaposhnikov   char getNsanTypeId() const override { return NsanTypeId; }
16417106792SAlexander Shaposhnikov   static constexpr const char kNsanTypeId = NsanTypeId;
16517106792SAlexander Shaposhnikov };
16617106792SAlexander Shaposhnikov 
16717106792SAlexander Shaposhnikov // `double` (`d`) shadow type.
16817106792SAlexander Shaposhnikov class F64ShadowConfig : public ShadowTypeConfigImpl<'d'> {
16917106792SAlexander Shaposhnikov   Type *getType(LLVMContext &Context) const override {
17017106792SAlexander Shaposhnikov     return Type::getDoubleTy(Context);
17117106792SAlexander Shaposhnikov   }
17217106792SAlexander Shaposhnikov };
17317106792SAlexander Shaposhnikov 
17417106792SAlexander Shaposhnikov // `x86_fp80` (`l`) shadow type: X86 long double.
17517106792SAlexander Shaposhnikov class F80ShadowConfig : public ShadowTypeConfigImpl<'l'> {
17617106792SAlexander Shaposhnikov   Type *getType(LLVMContext &Context) const override {
17717106792SAlexander Shaposhnikov     return Type::getX86_FP80Ty(Context);
17817106792SAlexander Shaposhnikov   }
17917106792SAlexander Shaposhnikov };
18017106792SAlexander Shaposhnikov 
18117106792SAlexander Shaposhnikov // `fp128` (`q`) shadow type.
18217106792SAlexander Shaposhnikov class F128ShadowConfig : public ShadowTypeConfigImpl<'q'> {
18317106792SAlexander Shaposhnikov   Type *getType(LLVMContext &Context) const override {
18417106792SAlexander Shaposhnikov     return Type::getFP128Ty(Context);
18517106792SAlexander Shaposhnikov   }
18617106792SAlexander Shaposhnikov };
18717106792SAlexander Shaposhnikov 
18817106792SAlexander Shaposhnikov // `ppc_fp128` (`e`) shadow type: IBM extended double with 106 bits of mantissa.
18917106792SAlexander Shaposhnikov class PPC128ShadowConfig : public ShadowTypeConfigImpl<'e'> {
19017106792SAlexander Shaposhnikov   Type *getType(LLVMContext &Context) const override {
19117106792SAlexander Shaposhnikov     return Type::getPPC_FP128Ty(Context);
19217106792SAlexander Shaposhnikov   }
19317106792SAlexander Shaposhnikov };
19417106792SAlexander Shaposhnikov 
19517106792SAlexander Shaposhnikov // Creates a ShadowTypeConfig given its type id.
19617106792SAlexander Shaposhnikov std::unique_ptr<ShadowTypeConfig>
19717106792SAlexander Shaposhnikov ShadowTypeConfig::fromNsanTypeId(const char TypeId) {
19817106792SAlexander Shaposhnikov   switch (TypeId) {
19917106792SAlexander Shaposhnikov   case F64ShadowConfig::kNsanTypeId:
20017106792SAlexander Shaposhnikov     return std::make_unique<F64ShadowConfig>();
20117106792SAlexander Shaposhnikov   case F80ShadowConfig::kNsanTypeId:
20217106792SAlexander Shaposhnikov     return std::make_unique<F80ShadowConfig>();
20317106792SAlexander Shaposhnikov   case F128ShadowConfig::kNsanTypeId:
20417106792SAlexander Shaposhnikov     return std::make_unique<F128ShadowConfig>();
20517106792SAlexander Shaposhnikov   case PPC128ShadowConfig::kNsanTypeId:
20617106792SAlexander Shaposhnikov     return std::make_unique<PPC128ShadowConfig>();
20717106792SAlexander Shaposhnikov   }
20817106792SAlexander Shaposhnikov   report_fatal_error("nsan: invalid shadow type id '" + Twine(TypeId) + "'");
20917106792SAlexander Shaposhnikov }
21017106792SAlexander Shaposhnikov 
21117106792SAlexander Shaposhnikov // An enum corresponding to shadow value types. Used as indices in arrays, so
21217106792SAlexander Shaposhnikov // not an `enum class`.
21317106792SAlexander Shaposhnikov enum FTValueType { kFloat, kDouble, kLongDouble, kNumValueTypes };
21417106792SAlexander Shaposhnikov 
21517106792SAlexander Shaposhnikov // If `FT` corresponds to a primitive FTValueType, return it.
21617106792SAlexander Shaposhnikov static std::optional<FTValueType> ftValueTypeFromType(Type *FT) {
21717106792SAlexander Shaposhnikov   if (FT->isFloatTy())
21817106792SAlexander Shaposhnikov     return kFloat;
21917106792SAlexander Shaposhnikov   if (FT->isDoubleTy())
22017106792SAlexander Shaposhnikov     return kDouble;
22117106792SAlexander Shaposhnikov   if (FT->isX86_FP80Ty())
22217106792SAlexander Shaposhnikov     return kLongDouble;
22317106792SAlexander Shaposhnikov   return {};
22417106792SAlexander Shaposhnikov }
22517106792SAlexander Shaposhnikov 
22617106792SAlexander Shaposhnikov // Returns the LLVM type for an FTValueType.
22717106792SAlexander Shaposhnikov static Type *typeFromFTValueType(FTValueType VT, LLVMContext &Context) {
22817106792SAlexander Shaposhnikov   switch (VT) {
22917106792SAlexander Shaposhnikov   case kFloat:
23017106792SAlexander Shaposhnikov     return Type::getFloatTy(Context);
23117106792SAlexander Shaposhnikov   case kDouble:
23217106792SAlexander Shaposhnikov     return Type::getDoubleTy(Context);
23317106792SAlexander Shaposhnikov   case kLongDouble:
23417106792SAlexander Shaposhnikov     return Type::getX86_FP80Ty(Context);
23517106792SAlexander Shaposhnikov   case kNumValueTypes:
23617106792SAlexander Shaposhnikov     return nullptr;
23717106792SAlexander Shaposhnikov   }
238d32d20f3SSimon Pilgrim   llvm_unreachable("Unhandled FTValueType enum");
23917106792SAlexander Shaposhnikov }
24017106792SAlexander Shaposhnikov 
24117106792SAlexander Shaposhnikov // Returns the type name for an FTValueType.
24217106792SAlexander Shaposhnikov static const char *typeNameFromFTValueType(FTValueType VT) {
24317106792SAlexander Shaposhnikov   switch (VT) {
24417106792SAlexander Shaposhnikov   case kFloat:
24517106792SAlexander Shaposhnikov     return "float";
24617106792SAlexander Shaposhnikov   case kDouble:
24717106792SAlexander Shaposhnikov     return "double";
24817106792SAlexander Shaposhnikov   case kLongDouble:
24917106792SAlexander Shaposhnikov     return "longdouble";
25017106792SAlexander Shaposhnikov   case kNumValueTypes:
25117106792SAlexander Shaposhnikov     return nullptr;
25217106792SAlexander Shaposhnikov   }
253d32d20f3SSimon Pilgrim   llvm_unreachable("Unhandled FTValueType enum");
25417106792SAlexander Shaposhnikov }
25517106792SAlexander Shaposhnikov 
25617106792SAlexander Shaposhnikov // A specific mapping configuration of application type to shadow type for nsan
25717106792SAlexander Shaposhnikov // (see -nsan-shadow-mapping flag).
25817106792SAlexander Shaposhnikov class MappingConfig {
25917106792SAlexander Shaposhnikov public:
26017106792SAlexander Shaposhnikov   explicit MappingConfig(LLVMContext &C) : Context(C) {
26117106792SAlexander Shaposhnikov     if (ClShadowMapping.size() != 3)
26217106792SAlexander Shaposhnikov       report_fatal_error("Invalid nsan mapping: " + Twine(ClShadowMapping));
26317106792SAlexander Shaposhnikov     unsigned ShadowTypeSizeBits[kNumValueTypes];
26417106792SAlexander Shaposhnikov     for (int VT = 0; VT < kNumValueTypes; ++VT) {
26517106792SAlexander Shaposhnikov       auto Config = ShadowTypeConfig::fromNsanTypeId(ClShadowMapping[VT]);
26617106792SAlexander Shaposhnikov       if (!Config)
26717106792SAlexander Shaposhnikov         report_fatal_error("Failed to get ShadowTypeConfig for " +
26817106792SAlexander Shaposhnikov                            Twine(ClShadowMapping[VT]));
26917106792SAlexander Shaposhnikov       const unsigned AppTypeSize =
27017106792SAlexander Shaposhnikov           typeFromFTValueType(static_cast<FTValueType>(VT), Context)
27117106792SAlexander Shaposhnikov               ->getScalarSizeInBits();
27217106792SAlexander Shaposhnikov       const unsigned ShadowTypeSize =
27317106792SAlexander Shaposhnikov           Config->getType(Context)->getScalarSizeInBits();
27417106792SAlexander Shaposhnikov       // Check that the shadow type size is at most kShadowScale times the
27517106792SAlexander Shaposhnikov       // application type size, so that shadow memory compoutations are valid.
27617106792SAlexander Shaposhnikov       if (ShadowTypeSize > kShadowScale * AppTypeSize)
27717106792SAlexander Shaposhnikov         report_fatal_error("Invalid nsan mapping f" + Twine(AppTypeSize) +
27817106792SAlexander Shaposhnikov                            "->f" + Twine(ShadowTypeSize) +
27917106792SAlexander Shaposhnikov                            ": The shadow type size should be at most " +
28017106792SAlexander Shaposhnikov                            Twine(kShadowScale) +
28117106792SAlexander Shaposhnikov                            " times the application type size");
28217106792SAlexander Shaposhnikov       ShadowTypeSizeBits[VT] = ShadowTypeSize;
28317106792SAlexander Shaposhnikov       Configs[VT] = std::move(Config);
28417106792SAlexander Shaposhnikov     }
28517106792SAlexander Shaposhnikov 
28617106792SAlexander Shaposhnikov     // Check that the mapping is monotonous. This is required because if one
28717106792SAlexander Shaposhnikov     // does an fpextend of `float->long double` in application code, nsan is
28817106792SAlexander Shaposhnikov     // going to do an fpextend of `shadow(float) -> shadow(long double)` in
28917106792SAlexander Shaposhnikov     // shadow code. This will fail in `qql` mode, since nsan would be
29017106792SAlexander Shaposhnikov     // fpextending `f128->long`, which is invalid.
29117106792SAlexander Shaposhnikov     // TODO: Relax this.
29217106792SAlexander Shaposhnikov     if (ShadowTypeSizeBits[kFloat] > ShadowTypeSizeBits[kDouble] ||
29317106792SAlexander Shaposhnikov         ShadowTypeSizeBits[kDouble] > ShadowTypeSizeBits[kLongDouble])
29417106792SAlexander Shaposhnikov       report_fatal_error("Invalid nsan mapping: { float->f" +
29517106792SAlexander Shaposhnikov                          Twine(ShadowTypeSizeBits[kFloat]) + "; double->f" +
29617106792SAlexander Shaposhnikov                          Twine(ShadowTypeSizeBits[kDouble]) +
29717106792SAlexander Shaposhnikov                          "; long double->f" +
29817106792SAlexander Shaposhnikov                          Twine(ShadowTypeSizeBits[kLongDouble]) + " }");
29917106792SAlexander Shaposhnikov   }
30017106792SAlexander Shaposhnikov 
30117106792SAlexander Shaposhnikov   const ShadowTypeConfig &byValueType(FTValueType VT) const {
30217106792SAlexander Shaposhnikov     assert(VT < FTValueType::kNumValueTypes && "invalid value type");
30317106792SAlexander Shaposhnikov     return *Configs[VT];
30417106792SAlexander Shaposhnikov   }
30517106792SAlexander Shaposhnikov 
30617106792SAlexander Shaposhnikov   // Returns the extended shadow type for a given application type.
30717106792SAlexander Shaposhnikov   Type *getExtendedFPType(Type *FT) const {
30817106792SAlexander Shaposhnikov     if (const auto VT = ftValueTypeFromType(FT))
30917106792SAlexander Shaposhnikov       return Configs[*VT]->getType(Context);
31017106792SAlexander Shaposhnikov     if (FT->isVectorTy()) {
31117106792SAlexander Shaposhnikov       auto *VecTy = cast<VectorType>(FT);
31217106792SAlexander Shaposhnikov       // TODO: add support for scalable vector types.
31317106792SAlexander Shaposhnikov       if (VecTy->isScalableTy())
31417106792SAlexander Shaposhnikov         return nullptr;
31517106792SAlexander Shaposhnikov       Type *ExtendedScalar = getExtendedFPType(VecTy->getElementType());
31617106792SAlexander Shaposhnikov       return ExtendedScalar
31717106792SAlexander Shaposhnikov                  ? VectorType::get(ExtendedScalar, VecTy->getElementCount())
31817106792SAlexander Shaposhnikov                  : nullptr;
31917106792SAlexander Shaposhnikov     }
32017106792SAlexander Shaposhnikov     return nullptr;
32117106792SAlexander Shaposhnikov   }
32217106792SAlexander Shaposhnikov 
32317106792SAlexander Shaposhnikov private:
32417106792SAlexander Shaposhnikov   LLVMContext &Context;
32517106792SAlexander Shaposhnikov   std::unique_ptr<ShadowTypeConfig> Configs[FTValueType::kNumValueTypes];
32617106792SAlexander Shaposhnikov };
32717106792SAlexander Shaposhnikov 
32817106792SAlexander Shaposhnikov // The memory extents of a type specifies how many elements of a given
32917106792SAlexander Shaposhnikov // FTValueType needs to be stored when storing this type.
33017106792SAlexander Shaposhnikov struct MemoryExtents {
33117106792SAlexander Shaposhnikov   FTValueType ValueType;
33217106792SAlexander Shaposhnikov   uint64_t NumElts;
33317106792SAlexander Shaposhnikov };
33417106792SAlexander Shaposhnikov 
33517106792SAlexander Shaposhnikov static MemoryExtents getMemoryExtentsOrDie(Type *FT) {
33617106792SAlexander Shaposhnikov   if (const auto VT = ftValueTypeFromType(FT))
33717106792SAlexander Shaposhnikov     return {*VT, 1};
33817106792SAlexander Shaposhnikov   if (auto *VecTy = dyn_cast<VectorType>(FT)) {
33917106792SAlexander Shaposhnikov     const auto ScalarExtents = getMemoryExtentsOrDie(VecTy->getElementType());
34017106792SAlexander Shaposhnikov     return {ScalarExtents.ValueType,
34117106792SAlexander Shaposhnikov             ScalarExtents.NumElts * VecTy->getElementCount().getFixedValue()};
34217106792SAlexander Shaposhnikov   }
34317106792SAlexander Shaposhnikov   llvm_unreachable("invalid value type");
34417106792SAlexander Shaposhnikov }
34517106792SAlexander Shaposhnikov 
34617106792SAlexander Shaposhnikov // The location of a check. Passed as parameters to runtime checking functions.
34717106792SAlexander Shaposhnikov class CheckLoc {
34817106792SAlexander Shaposhnikov public:
34917106792SAlexander Shaposhnikov   // Creates a location that references an application memory location.
35017106792SAlexander Shaposhnikov   static CheckLoc makeStore(Value *Address) {
35117106792SAlexander Shaposhnikov     CheckLoc Result(kStore);
35217106792SAlexander Shaposhnikov     Result.Address = Address;
35317106792SAlexander Shaposhnikov     return Result;
35417106792SAlexander Shaposhnikov   }
35517106792SAlexander Shaposhnikov   static CheckLoc makeLoad(Value *Address) {
35617106792SAlexander Shaposhnikov     CheckLoc Result(kLoad);
35717106792SAlexander Shaposhnikov     Result.Address = Address;
35817106792SAlexander Shaposhnikov     return Result;
35917106792SAlexander Shaposhnikov   }
36017106792SAlexander Shaposhnikov 
36117106792SAlexander Shaposhnikov   // Creates a location that references an argument, given by id.
36217106792SAlexander Shaposhnikov   static CheckLoc makeArg(int ArgId) {
36317106792SAlexander Shaposhnikov     CheckLoc Result(kArg);
36417106792SAlexander Shaposhnikov     Result.ArgId = ArgId;
36517106792SAlexander Shaposhnikov     return Result;
36617106792SAlexander Shaposhnikov   }
36717106792SAlexander Shaposhnikov 
36817106792SAlexander Shaposhnikov   // Creates a location that references the return value of a function.
36917106792SAlexander Shaposhnikov   static CheckLoc makeRet() { return CheckLoc(kRet); }
37017106792SAlexander Shaposhnikov 
37117106792SAlexander Shaposhnikov   // Creates a location that references a vector insert.
37217106792SAlexander Shaposhnikov   static CheckLoc makeInsert() { return CheckLoc(kInsert); }
37317106792SAlexander Shaposhnikov 
37417106792SAlexander Shaposhnikov   // Returns the CheckType of location this refers to, as an integer-typed LLVM
37517106792SAlexander Shaposhnikov   // IR value.
37617106792SAlexander Shaposhnikov   Value *getType(LLVMContext &C) const {
37717106792SAlexander Shaposhnikov     return ConstantInt::get(Type::getInt32Ty(C), static_cast<int>(CheckTy));
37817106792SAlexander Shaposhnikov   }
37917106792SAlexander Shaposhnikov 
38017106792SAlexander Shaposhnikov   // Returns a CheckType-specific value representing details of the location
38117106792SAlexander Shaposhnikov   // (e.g. application address for loads or stores), as an `IntptrTy`-typed LLVM
38217106792SAlexander Shaposhnikov   // IR value.
38317106792SAlexander Shaposhnikov   Value *getValue(Type *IntptrTy, IRBuilder<> &Builder) const {
38417106792SAlexander Shaposhnikov     switch (CheckTy) {
38517106792SAlexander Shaposhnikov     case kUnknown:
38617106792SAlexander Shaposhnikov       llvm_unreachable("unknown type");
38717106792SAlexander Shaposhnikov     case kRet:
38817106792SAlexander Shaposhnikov     case kInsert:
38917106792SAlexander Shaposhnikov       return ConstantInt::get(IntptrTy, 0);
39017106792SAlexander Shaposhnikov     case kArg:
39117106792SAlexander Shaposhnikov       return ConstantInt::get(IntptrTy, ArgId);
39217106792SAlexander Shaposhnikov     case kLoad:
39317106792SAlexander Shaposhnikov     case kStore:
39417106792SAlexander Shaposhnikov       return Builder.CreatePtrToInt(Address, IntptrTy);
39517106792SAlexander Shaposhnikov     }
396d32d20f3SSimon Pilgrim     llvm_unreachable("Unhandled CheckType enum");
39717106792SAlexander Shaposhnikov   }
39817106792SAlexander Shaposhnikov 
39917106792SAlexander Shaposhnikov private:
40017106792SAlexander Shaposhnikov   // Must be kept in sync with the runtime,
40117106792SAlexander Shaposhnikov   // see compiler-rt/lib/nsan/nsan_stats.h
40217106792SAlexander Shaposhnikov   enum CheckType {
40317106792SAlexander Shaposhnikov     kUnknown = 0,
40417106792SAlexander Shaposhnikov     kRet,
40517106792SAlexander Shaposhnikov     kArg,
40617106792SAlexander Shaposhnikov     kLoad,
40717106792SAlexander Shaposhnikov     kStore,
40817106792SAlexander Shaposhnikov     kInsert,
40917106792SAlexander Shaposhnikov   };
41017106792SAlexander Shaposhnikov   explicit CheckLoc(CheckType CheckTy) : CheckTy(CheckTy) {}
41117106792SAlexander Shaposhnikov 
41217106792SAlexander Shaposhnikov   Value *Address = nullptr;
41317106792SAlexander Shaposhnikov   const CheckType CheckTy;
41417106792SAlexander Shaposhnikov   int ArgId = -1;
41517106792SAlexander Shaposhnikov };
41617106792SAlexander Shaposhnikov 
41717106792SAlexander Shaposhnikov // A map of LLVM IR values to shadow LLVM IR values.
41817106792SAlexander Shaposhnikov class ValueToShadowMap {
41917106792SAlexander Shaposhnikov public:
42017106792SAlexander Shaposhnikov   explicit ValueToShadowMap(const MappingConfig &Config) : Config(Config) {}
42117106792SAlexander Shaposhnikov 
42217106792SAlexander Shaposhnikov   ValueToShadowMap(const ValueToShadowMap &) = delete;
42317106792SAlexander Shaposhnikov   ValueToShadowMap &operator=(const ValueToShadowMap &) = delete;
42417106792SAlexander Shaposhnikov 
42517106792SAlexander Shaposhnikov   // Sets the shadow value for a value. Asserts that the value does not already
42617106792SAlexander Shaposhnikov   // have a value.
42717106792SAlexander Shaposhnikov   void setShadow(Value &V, Value &Shadow) {
42817106792SAlexander Shaposhnikov     [[maybe_unused]] const bool Inserted = Map.try_emplace(&V, &Shadow).second;
42917106792SAlexander Shaposhnikov     LLVM_DEBUG({
43017106792SAlexander Shaposhnikov       if (!Inserted) {
43117106792SAlexander Shaposhnikov         if (auto *I = dyn_cast<Instruction>(&V))
43217106792SAlexander Shaposhnikov           errs() << I->getFunction()->getName() << ": ";
43317106792SAlexander Shaposhnikov         errs() << "duplicate shadow (" << &V << "): ";
43417106792SAlexander Shaposhnikov         V.dump();
43517106792SAlexander Shaposhnikov       }
43617106792SAlexander Shaposhnikov     });
43717106792SAlexander Shaposhnikov     assert(Inserted && "duplicate shadow");
43817106792SAlexander Shaposhnikov   }
43917106792SAlexander Shaposhnikov 
44017106792SAlexander Shaposhnikov   // Returns true if the value already has a shadow (including if the value is a
44117106792SAlexander Shaposhnikov   // constant). If true, calling getShadow() is valid.
44217106792SAlexander Shaposhnikov   bool hasShadow(Value *V) const {
44317106792SAlexander Shaposhnikov     return isa<Constant>(V) || (Map.find(V) != Map.end());
44417106792SAlexander Shaposhnikov   }
44517106792SAlexander Shaposhnikov 
44617106792SAlexander Shaposhnikov   // Returns the shadow value for a given value. Asserts that the value has
44717106792SAlexander Shaposhnikov   // a shadow value. Lazily creates shadows for constant values.
44817106792SAlexander Shaposhnikov   Value *getShadow(Value *V) const {
44917106792SAlexander Shaposhnikov     if (Constant *C = dyn_cast<Constant>(V))
45017106792SAlexander Shaposhnikov       return getShadowConstant(C);
45117106792SAlexander Shaposhnikov     return Map.find(V)->second;
45217106792SAlexander Shaposhnikov   }
45317106792SAlexander Shaposhnikov 
45417106792SAlexander Shaposhnikov   bool empty() const { return Map.empty(); }
45517106792SAlexander Shaposhnikov 
45617106792SAlexander Shaposhnikov private:
45717106792SAlexander Shaposhnikov   // Extends a constant application value to its shadow counterpart.
45817106792SAlexander Shaposhnikov   APFloat extendConstantFP(APFloat CV, const fltSemantics &To) const {
45917106792SAlexander Shaposhnikov     bool LosesInfo = false;
46017106792SAlexander Shaposhnikov     CV.convert(To, APFloatBase::rmTowardZero, &LosesInfo);
46117106792SAlexander Shaposhnikov     return CV;
46217106792SAlexander Shaposhnikov   }
46317106792SAlexander Shaposhnikov 
46417106792SAlexander Shaposhnikov   // Returns the shadow constant for the given application constant.
46517106792SAlexander Shaposhnikov   Constant *getShadowConstant(Constant *C) const {
46617106792SAlexander Shaposhnikov     if (UndefValue *U = dyn_cast<UndefValue>(C)) {
46717106792SAlexander Shaposhnikov       return UndefValue::get(Config.getExtendedFPType(U->getType()));
46817106792SAlexander Shaposhnikov     }
46917106792SAlexander Shaposhnikov     if (ConstantFP *CFP = dyn_cast<ConstantFP>(C)) {
47017106792SAlexander Shaposhnikov       // Floating-point constants.
47117106792SAlexander Shaposhnikov       Type *Ty = Config.getExtendedFPType(CFP->getType());
47217106792SAlexander Shaposhnikov       return ConstantFP::get(
47317106792SAlexander Shaposhnikov           Ty, extendConstantFP(CFP->getValueAPF(), Ty->getFltSemantics()));
47417106792SAlexander Shaposhnikov     }
47517106792SAlexander Shaposhnikov     // Vector, array, or aggregate constants.
47617106792SAlexander Shaposhnikov     if (C->getType()->isVectorTy()) {
47717106792SAlexander Shaposhnikov       SmallVector<Constant *, 8> Elements;
47817106792SAlexander Shaposhnikov       for (int I = 0, E = cast<VectorType>(C->getType())
47917106792SAlexander Shaposhnikov                               ->getElementCount()
48017106792SAlexander Shaposhnikov                               .getFixedValue();
48117106792SAlexander Shaposhnikov            I < E; ++I)
48217106792SAlexander Shaposhnikov         Elements.push_back(getShadowConstant(C->getAggregateElement(I)));
48317106792SAlexander Shaposhnikov       return ConstantVector::get(Elements);
48417106792SAlexander Shaposhnikov     }
48517106792SAlexander Shaposhnikov     llvm_unreachable("unimplemented");
48617106792SAlexander Shaposhnikov   }
48717106792SAlexander Shaposhnikov 
48817106792SAlexander Shaposhnikov   const MappingConfig &Config;
48917106792SAlexander Shaposhnikov   DenseMap<Value *, Value *> Map;
49017106792SAlexander Shaposhnikov };
49117106792SAlexander Shaposhnikov 
492ddf5725eSDmitry Chestnykh class NsanMemOpFn {
493ddf5725eSDmitry Chestnykh public:
494ddf5725eSDmitry Chestnykh   NsanMemOpFn(Module &M, ArrayRef<StringRef> Sized, StringRef Fallback,
495ddf5725eSDmitry Chestnykh               size_t NumArgs);
496ddf5725eSDmitry Chestnykh   FunctionCallee getFunctionFor(uint64_t MemOpSize) const;
497ddf5725eSDmitry Chestnykh   FunctionCallee getFallback() const;
498ddf5725eSDmitry Chestnykh 
499ddf5725eSDmitry Chestnykh private:
500ddf5725eSDmitry Chestnykh   SmallVector<FunctionCallee> Funcs;
501ddf5725eSDmitry Chestnykh   size_t NumSizedFuncs;
502ddf5725eSDmitry Chestnykh };
503ddf5725eSDmitry Chestnykh 
504ddf5725eSDmitry Chestnykh NsanMemOpFn::NsanMemOpFn(Module &M, ArrayRef<StringRef> Sized,
505ddf5725eSDmitry Chestnykh                          StringRef Fallback, size_t NumArgs) {
506ddf5725eSDmitry Chestnykh   LLVMContext &Ctx = M.getContext();
507ddf5725eSDmitry Chestnykh   AttributeList Attr;
508ddf5725eSDmitry Chestnykh   Attr = Attr.addFnAttribute(Ctx, Attribute::NoUnwind);
509ddf5725eSDmitry Chestnykh   Type *PtrTy = PointerType::getUnqual(Ctx);
510ddf5725eSDmitry Chestnykh   Type *VoidTy = Type::getVoidTy(Ctx);
511ddf5725eSDmitry Chestnykh   IntegerType *IntptrTy = M.getDataLayout().getIntPtrType(Ctx);
512ddf5725eSDmitry Chestnykh   FunctionType *SizedFnTy = nullptr;
513ddf5725eSDmitry Chestnykh 
514ddf5725eSDmitry Chestnykh   NumSizedFuncs = Sized.size();
515ddf5725eSDmitry Chestnykh 
516ddf5725eSDmitry Chestnykh   // First entry is fallback function
517ddf5725eSDmitry Chestnykh   if (NumArgs == 3) {
518ddf5725eSDmitry Chestnykh     Funcs.push_back(
519ddf5725eSDmitry Chestnykh         M.getOrInsertFunction(Fallback, Attr, VoidTy, PtrTy, PtrTy, IntptrTy));
520ddf5725eSDmitry Chestnykh     SizedFnTy = FunctionType::get(VoidTy, {PtrTy, PtrTy}, false);
521ddf5725eSDmitry Chestnykh   } else if (NumArgs == 2) {
522ddf5725eSDmitry Chestnykh     Funcs.push_back(
523ddf5725eSDmitry Chestnykh         M.getOrInsertFunction(Fallback, Attr, VoidTy, PtrTy, IntptrTy));
524ddf5725eSDmitry Chestnykh     SizedFnTy = FunctionType::get(VoidTy, {PtrTy}, false);
525ddf5725eSDmitry Chestnykh   } else {
526cd82fee3SDmitry Chestnykh     llvm_unreachable("Unexpected value of sized functions arguments");
527ddf5725eSDmitry Chestnykh   }
528ddf5725eSDmitry Chestnykh 
529ddf5725eSDmitry Chestnykh   for (size_t i = 0; i < NumSizedFuncs; ++i)
530ddf5725eSDmitry Chestnykh     Funcs.push_back(M.getOrInsertFunction(Sized[i], SizedFnTy, Attr));
531ddf5725eSDmitry Chestnykh }
532ddf5725eSDmitry Chestnykh 
533ddf5725eSDmitry Chestnykh FunctionCallee NsanMemOpFn::getFunctionFor(uint64_t MemOpSize) const {
534ddf5725eSDmitry Chestnykh   // Now `getFunctionFor` operates on `Funcs` of size 4 (at least) and the
535ddf5725eSDmitry Chestnykh   // following code assumes that the number of functions in `Func` is sufficient
536ddf5725eSDmitry Chestnykh   assert(NumSizedFuncs >= 3 && "Unexpected number of sized functions");
537ddf5725eSDmitry Chestnykh 
538ddf5725eSDmitry Chestnykh   size_t Idx =
539ddf5725eSDmitry Chestnykh       MemOpSize == 4 ? 1 : (MemOpSize == 8 ? 2 : (MemOpSize == 16 ? 3 : 0));
540ddf5725eSDmitry Chestnykh 
541ddf5725eSDmitry Chestnykh   return Funcs[Idx];
542ddf5725eSDmitry Chestnykh }
543ddf5725eSDmitry Chestnykh 
544ddf5725eSDmitry Chestnykh FunctionCallee NsanMemOpFn::getFallback() const { return Funcs[0]; }
545ddf5725eSDmitry Chestnykh 
54617106792SAlexander Shaposhnikov /// Instantiating NumericalStabilitySanitizer inserts the nsan runtime library
54717106792SAlexander Shaposhnikov /// API function declarations into the module if they don't exist already.
54817106792SAlexander Shaposhnikov /// Instantiating ensures the __nsan_init function is in the list of global
54917106792SAlexander Shaposhnikov /// constructors for the module.
55017106792SAlexander Shaposhnikov class NumericalStabilitySanitizer {
55117106792SAlexander Shaposhnikov public:
55217106792SAlexander Shaposhnikov   NumericalStabilitySanitizer(Module &M);
55317106792SAlexander Shaposhnikov   bool sanitizeFunction(Function &F, const TargetLibraryInfo &TLI);
55417106792SAlexander Shaposhnikov 
55517106792SAlexander Shaposhnikov private:
55617106792SAlexander Shaposhnikov   bool instrumentMemIntrinsic(MemIntrinsic *MI);
55717106792SAlexander Shaposhnikov   void maybeAddSuffixForNsanInterface(CallBase *CI);
55817106792SAlexander Shaposhnikov   bool addrPointsToConstantData(Value *Addr);
55917106792SAlexander Shaposhnikov   void maybeCreateShadowValue(Instruction &Root, const TargetLibraryInfo &TLI,
56017106792SAlexander Shaposhnikov                               ValueToShadowMap &Map);
56117106792SAlexander Shaposhnikov   Value *createShadowValueWithOperandsAvailable(Instruction &Inst,
56217106792SAlexander Shaposhnikov                                                 const TargetLibraryInfo &TLI,
56317106792SAlexander Shaposhnikov                                                 const ValueToShadowMap &Map);
56417106792SAlexander Shaposhnikov   PHINode *maybeCreateShadowPhi(PHINode &Phi, const TargetLibraryInfo &TLI);
56517106792SAlexander Shaposhnikov   void createShadowArguments(Function &F, const TargetLibraryInfo &TLI,
56617106792SAlexander Shaposhnikov                              ValueToShadowMap &Map);
56717106792SAlexander Shaposhnikov 
56817106792SAlexander Shaposhnikov   void populateShadowStack(CallBase &CI, const TargetLibraryInfo &TLI,
56917106792SAlexander Shaposhnikov                            const ValueToShadowMap &Map);
57017106792SAlexander Shaposhnikov 
57117106792SAlexander Shaposhnikov   void propagateShadowValues(Instruction &Inst, const TargetLibraryInfo &TLI,
57217106792SAlexander Shaposhnikov                              const ValueToShadowMap &Map);
57317106792SAlexander Shaposhnikov   Value *emitCheck(Value *V, Value *ShadowV, IRBuilder<> &Builder,
57417106792SAlexander Shaposhnikov                    CheckLoc Loc);
57517106792SAlexander Shaposhnikov   Value *emitCheckInternal(Value *V, Value *ShadowV, IRBuilder<> &Builder,
57617106792SAlexander Shaposhnikov                            CheckLoc Loc);
57717106792SAlexander Shaposhnikov   void emitFCmpCheck(FCmpInst &FCmp, const ValueToShadowMap &Map);
57817106792SAlexander Shaposhnikov 
57917106792SAlexander Shaposhnikov   // Value creation handlers.
58017106792SAlexander Shaposhnikov   Value *handleLoad(LoadInst &Load, Type *VT, Type *ExtendedVT);
58117106792SAlexander Shaposhnikov   Value *handleCallBase(CallBase &Call, Type *VT, Type *ExtendedVT,
58217106792SAlexander Shaposhnikov                         const TargetLibraryInfo &TLI,
58317106792SAlexander Shaposhnikov                         const ValueToShadowMap &Map, IRBuilder<> &Builder);
58417106792SAlexander Shaposhnikov   Value *maybeHandleKnownCallBase(CallBase &Call, Type *VT, Type *ExtendedVT,
58517106792SAlexander Shaposhnikov                                   const TargetLibraryInfo &TLI,
58617106792SAlexander Shaposhnikov                                   const ValueToShadowMap &Map,
58717106792SAlexander Shaposhnikov                                   IRBuilder<> &Builder);
58817106792SAlexander Shaposhnikov   Value *handleTrunc(const FPTruncInst &Trunc, Type *VT, Type *ExtendedVT,
58917106792SAlexander Shaposhnikov                      const ValueToShadowMap &Map, IRBuilder<> &Builder);
59017106792SAlexander Shaposhnikov   Value *handleExt(const FPExtInst &Ext, Type *VT, Type *ExtendedVT,
59117106792SAlexander Shaposhnikov                    const ValueToShadowMap &Map, IRBuilder<> &Builder);
59217106792SAlexander Shaposhnikov 
59317106792SAlexander Shaposhnikov   // Value propagation handlers.
59417106792SAlexander Shaposhnikov   void propagateFTStore(StoreInst &Store, Type *VT, Type *ExtendedVT,
59517106792SAlexander Shaposhnikov                         const ValueToShadowMap &Map);
59617106792SAlexander Shaposhnikov   void propagateNonFTStore(StoreInst &Store, Type *VT,
59717106792SAlexander Shaposhnikov                            const ValueToShadowMap &Map);
59817106792SAlexander Shaposhnikov 
59917106792SAlexander Shaposhnikov   const DataLayout &DL;
60017106792SAlexander Shaposhnikov   LLVMContext &Context;
60117106792SAlexander Shaposhnikov   MappingConfig Config;
60217106792SAlexander Shaposhnikov   IntegerType *IntptrTy = nullptr;
603ddf5725eSDmitry Chestnykh 
604ddf5725eSDmitry Chestnykh   // TODO: Use std::array instead?
60517106792SAlexander Shaposhnikov   FunctionCallee NsanGetShadowPtrForStore[FTValueType::kNumValueTypes] = {};
60617106792SAlexander Shaposhnikov   FunctionCallee NsanGetShadowPtrForLoad[FTValueType::kNumValueTypes] = {};
60717106792SAlexander Shaposhnikov   FunctionCallee NsanCheckValue[FTValueType::kNumValueTypes] = {};
60817106792SAlexander Shaposhnikov   FunctionCallee NsanFCmpFail[FTValueType::kNumValueTypes] = {};
609ddf5725eSDmitry Chestnykh 
610ddf5725eSDmitry Chestnykh   NsanMemOpFn NsanCopyFns;
611ddf5725eSDmitry Chestnykh   NsanMemOpFn NsanSetUnknownFns;
612ddf5725eSDmitry Chestnykh 
61317106792SAlexander Shaposhnikov   FunctionCallee NsanGetRawShadowTypePtr;
61417106792SAlexander Shaposhnikov   FunctionCallee NsanGetRawShadowPtr;
61517106792SAlexander Shaposhnikov   GlobalValue *NsanShadowRetTag = nullptr;
61617106792SAlexander Shaposhnikov 
61717106792SAlexander Shaposhnikov   Type *NsanShadowRetType = nullptr;
61817106792SAlexander Shaposhnikov   GlobalValue *NsanShadowRetPtr = nullptr;
61917106792SAlexander Shaposhnikov 
62017106792SAlexander Shaposhnikov   GlobalValue *NsanShadowArgsTag = nullptr;
62117106792SAlexander Shaposhnikov 
62217106792SAlexander Shaposhnikov   Type *NsanShadowArgsType = nullptr;
62317106792SAlexander Shaposhnikov   GlobalValue *NsanShadowArgsPtr = nullptr;
62417106792SAlexander Shaposhnikov 
62517106792SAlexander Shaposhnikov   std::optional<Regex> CheckFunctionsFilter;
62617106792SAlexander Shaposhnikov };
62717106792SAlexander Shaposhnikov } // end anonymous namespace
62817106792SAlexander Shaposhnikov 
62917106792SAlexander Shaposhnikov PreservedAnalyses
63017106792SAlexander Shaposhnikov NumericalStabilitySanitizerPass::run(Module &M, ModuleAnalysisManager &MAM) {
63117106792SAlexander Shaposhnikov   getOrCreateSanitizerCtorAndInitFunctions(
63217106792SAlexander Shaposhnikov       M, kNsanModuleCtorName, kNsanInitName, /*InitArgTypes=*/{},
63317106792SAlexander Shaposhnikov       /*InitArgs=*/{},
63417106792SAlexander Shaposhnikov       // This callback is invoked when the functions are created the first
63517106792SAlexander Shaposhnikov       // time. Hook them into the global ctors list in that case:
63617106792SAlexander Shaposhnikov       [&](Function *Ctor, FunctionCallee) { appendToGlobalCtors(M, Ctor, 0); });
63717106792SAlexander Shaposhnikov 
63817106792SAlexander Shaposhnikov   NumericalStabilitySanitizer Nsan(M);
63917106792SAlexander Shaposhnikov   auto &FAM = MAM.getResult<FunctionAnalysisManagerModuleProxy>(M).getManager();
64017106792SAlexander Shaposhnikov   for (Function &F : M)
64117106792SAlexander Shaposhnikov     Nsan.sanitizeFunction(F, FAM.getResult<TargetLibraryAnalysis>(F));
64217106792SAlexander Shaposhnikov 
64317106792SAlexander Shaposhnikov   return PreservedAnalyses::none();
64417106792SAlexander Shaposhnikov }
64517106792SAlexander Shaposhnikov 
64617106792SAlexander Shaposhnikov static GlobalValue *createThreadLocalGV(const char *Name, Module &M, Type *Ty) {
64717106792SAlexander Shaposhnikov   return dyn_cast<GlobalValue>(M.getOrInsertGlobal(Name, Ty, [&M, Ty, Name] {
64817106792SAlexander Shaposhnikov     return new GlobalVariable(M, Ty, false, GlobalVariable::ExternalLinkage,
64917106792SAlexander Shaposhnikov                               nullptr, Name, nullptr,
65017106792SAlexander Shaposhnikov                               GlobalVariable::InitialExecTLSModel);
65117106792SAlexander Shaposhnikov   }));
65217106792SAlexander Shaposhnikov }
65317106792SAlexander Shaposhnikov 
65417106792SAlexander Shaposhnikov NumericalStabilitySanitizer::NumericalStabilitySanitizer(Module &M)
655ddf5725eSDmitry Chestnykh     : DL(M.getDataLayout()), Context(M.getContext()), Config(Context),
656ddf5725eSDmitry Chestnykh       NsanCopyFns(M, {"__nsan_copy_4", "__nsan_copy_8", "__nsan_copy_16"},
657ddf5725eSDmitry Chestnykh                   "__nsan_copy_values", /*NumArgs=*/3),
658ddf5725eSDmitry Chestnykh       NsanSetUnknownFns(M,
659ddf5725eSDmitry Chestnykh                         {"__nsan_set_value_unknown_4",
660ddf5725eSDmitry Chestnykh                          "__nsan_set_value_unknown_8",
661ddf5725eSDmitry Chestnykh                          "__nsan_set_value_unknown_16"},
662ddf5725eSDmitry Chestnykh                         "__nsan_set_value_unknown", /*NumArgs=*/2) {
66317106792SAlexander Shaposhnikov   IntptrTy = DL.getIntPtrType(Context);
66417106792SAlexander Shaposhnikov   Type *PtrTy = PointerType::getUnqual(Context);
66517106792SAlexander Shaposhnikov   Type *Int32Ty = Type::getInt32Ty(Context);
66617106792SAlexander Shaposhnikov   Type *Int1Ty = Type::getInt1Ty(Context);
66717106792SAlexander Shaposhnikov   Type *VoidTy = Type::getVoidTy(Context);
66817106792SAlexander Shaposhnikov 
66917106792SAlexander Shaposhnikov   AttributeList Attr;
67017106792SAlexander Shaposhnikov   Attr = Attr.addFnAttribute(Context, Attribute::NoUnwind);
67117106792SAlexander Shaposhnikov   // Initialize the runtime values (functions and global variables).
67217106792SAlexander Shaposhnikov   for (int I = 0; I < kNumValueTypes; ++I) {
67317106792SAlexander Shaposhnikov     const FTValueType VT = static_cast<FTValueType>(I);
67417106792SAlexander Shaposhnikov     const char *VTName = typeNameFromFTValueType(VT);
67517106792SAlexander Shaposhnikov     Type *VTTy = typeFromFTValueType(VT, Context);
67617106792SAlexander Shaposhnikov 
67717106792SAlexander Shaposhnikov     // Load/store.
67817106792SAlexander Shaposhnikov     const std::string GetterPrefix =
67917106792SAlexander Shaposhnikov         std::string("__nsan_get_shadow_ptr_for_") + VTName;
68017106792SAlexander Shaposhnikov     NsanGetShadowPtrForStore[VT] = M.getOrInsertFunction(
68117106792SAlexander Shaposhnikov         GetterPrefix + "_store", Attr, PtrTy, PtrTy, IntptrTy);
68217106792SAlexander Shaposhnikov     NsanGetShadowPtrForLoad[VT] = M.getOrInsertFunction(
68317106792SAlexander Shaposhnikov         GetterPrefix + "_load", Attr, PtrTy, PtrTy, IntptrTy);
68417106792SAlexander Shaposhnikov 
68517106792SAlexander Shaposhnikov     // Check.
68617106792SAlexander Shaposhnikov     const auto &ShadowConfig = Config.byValueType(VT);
68717106792SAlexander Shaposhnikov     Type *ShadowTy = ShadowConfig.getType(Context);
68817106792SAlexander Shaposhnikov     NsanCheckValue[VT] =
68917106792SAlexander Shaposhnikov         M.getOrInsertFunction(std::string("__nsan_internal_check_") + VTName +
69017106792SAlexander Shaposhnikov                                   "_" + ShadowConfig.getNsanTypeId(),
69117106792SAlexander Shaposhnikov                               Attr, Int32Ty, VTTy, ShadowTy, Int32Ty, IntptrTy);
69217106792SAlexander Shaposhnikov     NsanFCmpFail[VT] = M.getOrInsertFunction(
69317106792SAlexander Shaposhnikov         std::string("__nsan_fcmp_fail_") + VTName + "_" +
69417106792SAlexander Shaposhnikov             ShadowConfig.getNsanTypeId(),
69517106792SAlexander Shaposhnikov         Attr, VoidTy, VTTy, VTTy, ShadowTy, ShadowTy, Int32Ty, Int1Ty, Int1Ty);
69617106792SAlexander Shaposhnikov   }
69717106792SAlexander Shaposhnikov 
69817106792SAlexander Shaposhnikov   // TODO: Add attributes nofree, nosync, readnone, readonly,
69917106792SAlexander Shaposhnikov   NsanGetRawShadowTypePtr = M.getOrInsertFunction(
70017106792SAlexander Shaposhnikov       "__nsan_internal_get_raw_shadow_type_ptr", Attr, PtrTy, PtrTy);
70117106792SAlexander Shaposhnikov   NsanGetRawShadowPtr = M.getOrInsertFunction(
70217106792SAlexander Shaposhnikov       "__nsan_internal_get_raw_shadow_ptr", Attr, PtrTy, PtrTy);
70317106792SAlexander Shaposhnikov 
70417106792SAlexander Shaposhnikov   NsanShadowRetTag = createThreadLocalGV("__nsan_shadow_ret_tag", M, IntptrTy);
70517106792SAlexander Shaposhnikov 
70617106792SAlexander Shaposhnikov   NsanShadowRetType = ArrayType::get(Type::getInt8Ty(Context),
70717106792SAlexander Shaposhnikov                                      kMaxVectorWidth * kMaxShadowTypeSizeBytes);
70817106792SAlexander Shaposhnikov   NsanShadowRetPtr =
70917106792SAlexander Shaposhnikov       createThreadLocalGV("__nsan_shadow_ret_ptr", M, NsanShadowRetType);
71017106792SAlexander Shaposhnikov 
71117106792SAlexander Shaposhnikov   NsanShadowArgsTag =
71217106792SAlexander Shaposhnikov       createThreadLocalGV("__nsan_shadow_args_tag", M, IntptrTy);
71317106792SAlexander Shaposhnikov 
71417106792SAlexander Shaposhnikov   NsanShadowArgsType =
71517106792SAlexander Shaposhnikov       ArrayType::get(Type::getInt8Ty(Context),
71617106792SAlexander Shaposhnikov                      kMaxVectorWidth * kMaxNumArgs * kMaxShadowTypeSizeBytes);
71717106792SAlexander Shaposhnikov 
71817106792SAlexander Shaposhnikov   NsanShadowArgsPtr =
71917106792SAlexander Shaposhnikov       createThreadLocalGV("__nsan_shadow_args_ptr", M, NsanShadowArgsType);
72017106792SAlexander Shaposhnikov 
72117106792SAlexander Shaposhnikov   if (!ClCheckFunctionsFilter.empty()) {
72217106792SAlexander Shaposhnikov     Regex R = Regex(ClCheckFunctionsFilter);
72317106792SAlexander Shaposhnikov     std::string RegexError;
72417106792SAlexander Shaposhnikov     assert(R.isValid(RegexError));
72517106792SAlexander Shaposhnikov     CheckFunctionsFilter = std::move(R);
72617106792SAlexander Shaposhnikov   }
72717106792SAlexander Shaposhnikov }
72817106792SAlexander Shaposhnikov 
72917106792SAlexander Shaposhnikov // Returns true if the given LLVM Value points to constant data (typically, a
73017106792SAlexander Shaposhnikov // global variable reference).
73117106792SAlexander Shaposhnikov bool NumericalStabilitySanitizer::addrPointsToConstantData(Value *Addr) {
73217106792SAlexander Shaposhnikov   // If this is a GEP, just analyze its pointer operand.
73317106792SAlexander Shaposhnikov   if (GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(Addr))
73417106792SAlexander Shaposhnikov     Addr = GEP->getPointerOperand();
73517106792SAlexander Shaposhnikov 
73617106792SAlexander Shaposhnikov   if (GlobalVariable *GV = dyn_cast<GlobalVariable>(Addr))
73717106792SAlexander Shaposhnikov     return GV->isConstant();
73817106792SAlexander Shaposhnikov   return false;
73917106792SAlexander Shaposhnikov }
74017106792SAlexander Shaposhnikov 
74117106792SAlexander Shaposhnikov // This instruments the function entry to create shadow arguments.
74217106792SAlexander Shaposhnikov // Pseudocode:
74317106792SAlexander Shaposhnikov //   if (this_fn_ptr == __nsan_shadow_args_tag) {
74417106792SAlexander Shaposhnikov //     s(arg0) = LOAD<sizeof(arg0)>(__nsan_shadow_args);
74517106792SAlexander Shaposhnikov //     s(arg1) = LOAD<sizeof(arg1)>(__nsan_shadow_args + sizeof(arg0));
74617106792SAlexander Shaposhnikov //     ...
74717106792SAlexander Shaposhnikov //     __nsan_shadow_args_tag = 0;
74817106792SAlexander Shaposhnikov //   } else {
74917106792SAlexander Shaposhnikov //     s(arg0) = fext(arg0);
75017106792SAlexander Shaposhnikov //     s(arg1) = fext(arg1);
75117106792SAlexander Shaposhnikov //     ...
75217106792SAlexander Shaposhnikov //   }
75317106792SAlexander Shaposhnikov void NumericalStabilitySanitizer::createShadowArguments(
75417106792SAlexander Shaposhnikov     Function &F, const TargetLibraryInfo &TLI, ValueToShadowMap &Map) {
75517106792SAlexander Shaposhnikov   assert(!F.getIntrinsicID() && "found a definition of an intrinsic");
75617106792SAlexander Shaposhnikov 
75717106792SAlexander Shaposhnikov   // Do not bother if there are no FP args.
75817106792SAlexander Shaposhnikov   if (all_of(F.args(), [this](const Argument &Arg) {
75917106792SAlexander Shaposhnikov         return Config.getExtendedFPType(Arg.getType()) == nullptr;
76017106792SAlexander Shaposhnikov       }))
76117106792SAlexander Shaposhnikov     return;
76217106792SAlexander Shaposhnikov 
763*6292a808SJeremy Morse   IRBuilder<> Builder(&F.getEntryBlock(), F.getEntryBlock().getFirstNonPHIIt());
76417106792SAlexander Shaposhnikov   // The function has shadow args if the shadow args tag matches the function
76517106792SAlexander Shaposhnikov   // address.
76617106792SAlexander Shaposhnikov   Value *HasShadowArgs = Builder.CreateICmpEQ(
76717106792SAlexander Shaposhnikov       Builder.CreateLoad(IntptrTy, NsanShadowArgsTag, /*isVolatile=*/false),
76817106792SAlexander Shaposhnikov       Builder.CreatePtrToInt(&F, IntptrTy));
76917106792SAlexander Shaposhnikov 
77017106792SAlexander Shaposhnikov   unsigned ShadowArgsOffsetBytes = 0;
77117106792SAlexander Shaposhnikov   for (Argument &Arg : F.args()) {
77217106792SAlexander Shaposhnikov     Type *VT = Arg.getType();
77317106792SAlexander Shaposhnikov     Type *ExtendedVT = Config.getExtendedFPType(VT);
77417106792SAlexander Shaposhnikov     if (ExtendedVT == nullptr)
77517106792SAlexander Shaposhnikov       continue; // Not an FT value.
77617106792SAlexander Shaposhnikov     Value *L = Builder.CreateAlignedLoad(
77717106792SAlexander Shaposhnikov         ExtendedVT,
77817106792SAlexander Shaposhnikov         Builder.CreateConstGEP2_64(NsanShadowArgsType, NsanShadowArgsPtr, 0,
77917106792SAlexander Shaposhnikov                                    ShadowArgsOffsetBytes),
78017106792SAlexander Shaposhnikov         Align(1), /*isVolatile=*/false);
78117106792SAlexander Shaposhnikov     Value *Shadow = Builder.CreateSelect(HasShadowArgs, L,
78217106792SAlexander Shaposhnikov                                          Builder.CreateFPExt(&Arg, ExtendedVT));
78317106792SAlexander Shaposhnikov     Map.setShadow(Arg, *Shadow);
78417106792SAlexander Shaposhnikov     TypeSize SlotSize = DL.getTypeStoreSize(ExtendedVT);
78517106792SAlexander Shaposhnikov     assert(!SlotSize.isScalable() && "unsupported");
78617106792SAlexander Shaposhnikov     ShadowArgsOffsetBytes += SlotSize;
78717106792SAlexander Shaposhnikov   }
78817106792SAlexander Shaposhnikov   Builder.CreateStore(ConstantInt::get(IntptrTy, 0), NsanShadowArgsTag);
78917106792SAlexander Shaposhnikov }
79017106792SAlexander Shaposhnikov 
79117106792SAlexander Shaposhnikov // Returns true if the instrumentation should emit code to check arguments
79217106792SAlexander Shaposhnikov // before a function call.
79317106792SAlexander Shaposhnikov static bool shouldCheckArgs(CallBase &CI, const TargetLibraryInfo &TLI,
79417106792SAlexander Shaposhnikov                             const std::optional<Regex> &CheckFunctionsFilter) {
79517106792SAlexander Shaposhnikov 
79617106792SAlexander Shaposhnikov   Function *Fn = CI.getCalledFunction();
79717106792SAlexander Shaposhnikov 
79817106792SAlexander Shaposhnikov   if (CheckFunctionsFilter) {
79917106792SAlexander Shaposhnikov     // Skip checking args of indirect calls.
80017106792SAlexander Shaposhnikov     if (Fn == nullptr)
80117106792SAlexander Shaposhnikov       return false;
80217106792SAlexander Shaposhnikov     if (CheckFunctionsFilter->match(Fn->getName()))
80317106792SAlexander Shaposhnikov       return true;
80417106792SAlexander Shaposhnikov     return false;
80517106792SAlexander Shaposhnikov   }
80617106792SAlexander Shaposhnikov 
80717106792SAlexander Shaposhnikov   if (Fn == nullptr)
80817106792SAlexander Shaposhnikov     return true; // Always check args of indirect calls.
80917106792SAlexander Shaposhnikov 
81017106792SAlexander Shaposhnikov   // Never check nsan functions, the user called them for a reason.
81117106792SAlexander Shaposhnikov   if (Fn->getName().starts_with("__nsan_"))
81217106792SAlexander Shaposhnikov     return false;
81317106792SAlexander Shaposhnikov 
81417106792SAlexander Shaposhnikov   const auto ID = Fn->getIntrinsicID();
81517106792SAlexander Shaposhnikov   LibFunc LFunc = LibFunc::NumLibFuncs;
81617106792SAlexander Shaposhnikov   // Always check args of unknown functions.
81717106792SAlexander Shaposhnikov   if (ID == Intrinsic::ID() && !TLI.getLibFunc(*Fn, LFunc))
81817106792SAlexander Shaposhnikov     return true;
81917106792SAlexander Shaposhnikov 
82017106792SAlexander Shaposhnikov   // Do not check args of an `fabs` call that is used for a comparison.
82117106792SAlexander Shaposhnikov   // This is typically used for `fabs(a-b) < tolerance`, where what matters is
82217106792SAlexander Shaposhnikov   // the result of the comparison, which is already caught be the fcmp checks.
82317106792SAlexander Shaposhnikov   if (ID == Intrinsic::fabs || LFunc == LibFunc_fabsf ||
82417106792SAlexander Shaposhnikov       LFunc == LibFunc_fabs || LFunc == LibFunc_fabsl)
82517106792SAlexander Shaposhnikov     for (const auto &U : CI.users())
82617106792SAlexander Shaposhnikov       if (isa<CmpInst>(U))
82717106792SAlexander Shaposhnikov         return false;
82817106792SAlexander Shaposhnikov 
82917106792SAlexander Shaposhnikov   return true; // Default is check.
83017106792SAlexander Shaposhnikov }
83117106792SAlexander Shaposhnikov 
83217106792SAlexander Shaposhnikov // Populates the shadow call stack (which contains shadow values for every
83317106792SAlexander Shaposhnikov // floating-point parameter to the function).
83417106792SAlexander Shaposhnikov void NumericalStabilitySanitizer::populateShadowStack(
83517106792SAlexander Shaposhnikov     CallBase &CI, const TargetLibraryInfo &TLI, const ValueToShadowMap &Map) {
83617106792SAlexander Shaposhnikov   // Do not create a shadow stack for inline asm.
83717106792SAlexander Shaposhnikov   if (CI.isInlineAsm())
83817106792SAlexander Shaposhnikov     return;
83917106792SAlexander Shaposhnikov 
84017106792SAlexander Shaposhnikov   // Do not bother if there are no FP args.
84117106792SAlexander Shaposhnikov   if (all_of(CI.operands(), [this](const Value *Arg) {
84217106792SAlexander Shaposhnikov         return Config.getExtendedFPType(Arg->getType()) == nullptr;
84317106792SAlexander Shaposhnikov       }))
84417106792SAlexander Shaposhnikov     return;
84517106792SAlexander Shaposhnikov 
84617106792SAlexander Shaposhnikov   IRBuilder<> Builder(&CI);
84717106792SAlexander Shaposhnikov   SmallVector<Value *, 8> ArgShadows;
84817106792SAlexander Shaposhnikov   const bool ShouldCheckArgs = shouldCheckArgs(CI, TLI, CheckFunctionsFilter);
84917106792SAlexander Shaposhnikov   for (auto [ArgIdx, Arg] : enumerate(CI.operands())) {
85017106792SAlexander Shaposhnikov     if (Config.getExtendedFPType(Arg->getType()) == nullptr)
85117106792SAlexander Shaposhnikov       continue; // Not an FT value.
85217106792SAlexander Shaposhnikov     Value *ArgShadow = Map.getShadow(Arg);
85317106792SAlexander Shaposhnikov     ArgShadows.push_back(ShouldCheckArgs ? emitCheck(Arg, ArgShadow, Builder,
85417106792SAlexander Shaposhnikov                                                      CheckLoc::makeArg(ArgIdx))
85517106792SAlexander Shaposhnikov                                          : ArgShadow);
85617106792SAlexander Shaposhnikov   }
85717106792SAlexander Shaposhnikov 
85817106792SAlexander Shaposhnikov   // Do not create shadow stacks for intrinsics/known lib funcs.
85917106792SAlexander Shaposhnikov   if (Function *Fn = CI.getCalledFunction()) {
86017106792SAlexander Shaposhnikov     LibFunc LFunc;
86117106792SAlexander Shaposhnikov     if (Fn->isIntrinsic() || TLI.getLibFunc(*Fn, LFunc))
86217106792SAlexander Shaposhnikov       return;
86317106792SAlexander Shaposhnikov   }
86417106792SAlexander Shaposhnikov 
86517106792SAlexander Shaposhnikov   // Set the shadow stack tag.
86617106792SAlexander Shaposhnikov   Builder.CreateStore(CI.getCalledOperand(), NsanShadowArgsTag);
86717106792SAlexander Shaposhnikov   TypeSize ShadowArgsOffsetBytes = TypeSize::getFixed(0);
86817106792SAlexander Shaposhnikov 
86917106792SAlexander Shaposhnikov   unsigned ShadowArgId = 0;
87017106792SAlexander Shaposhnikov   for (const Value *Arg : CI.operands()) {
87117106792SAlexander Shaposhnikov     Type *VT = Arg->getType();
87217106792SAlexander Shaposhnikov     Type *ExtendedVT = Config.getExtendedFPType(VT);
87317106792SAlexander Shaposhnikov     if (ExtendedVT == nullptr)
87417106792SAlexander Shaposhnikov       continue; // Not an FT value.
87517106792SAlexander Shaposhnikov     Builder.CreateAlignedStore(
87617106792SAlexander Shaposhnikov         ArgShadows[ShadowArgId++],
87717106792SAlexander Shaposhnikov         Builder.CreateConstGEP2_64(NsanShadowArgsType, NsanShadowArgsPtr, 0,
87817106792SAlexander Shaposhnikov                                    ShadowArgsOffsetBytes),
87917106792SAlexander Shaposhnikov         Align(1), /*isVolatile=*/false);
88017106792SAlexander Shaposhnikov     TypeSize SlotSize = DL.getTypeStoreSize(ExtendedVT);
88117106792SAlexander Shaposhnikov     assert(!SlotSize.isScalable() && "unsupported");
88217106792SAlexander Shaposhnikov     ShadowArgsOffsetBytes += SlotSize;
88317106792SAlexander Shaposhnikov   }
88417106792SAlexander Shaposhnikov }
88517106792SAlexander Shaposhnikov 
88617106792SAlexander Shaposhnikov // Internal part of emitCheck(). Returns a value that indicates whether
88717106792SAlexander Shaposhnikov // computation should continue with the shadow or resume by re-fextending the
88817106792SAlexander Shaposhnikov // value.
88917106792SAlexander Shaposhnikov enum class ContinuationType { // Keep in sync with runtime.
89017106792SAlexander Shaposhnikov   ContinueWithShadow = 0,
89117106792SAlexander Shaposhnikov   ResumeFromValue = 1,
89217106792SAlexander Shaposhnikov };
89317106792SAlexander Shaposhnikov 
89417106792SAlexander Shaposhnikov Value *NumericalStabilitySanitizer::emitCheckInternal(Value *V, Value *ShadowV,
89517106792SAlexander Shaposhnikov                                                       IRBuilder<> &Builder,
89617106792SAlexander Shaposhnikov                                                       CheckLoc Loc) {
89717106792SAlexander Shaposhnikov   // Do not emit checks for constant values, this is redundant.
89817106792SAlexander Shaposhnikov   if (isa<Constant>(V))
89917106792SAlexander Shaposhnikov     return ConstantInt::get(
90017106792SAlexander Shaposhnikov         Builder.getInt32Ty(),
90117106792SAlexander Shaposhnikov         static_cast<int>(ContinuationType::ContinueWithShadow));
90217106792SAlexander Shaposhnikov 
90317106792SAlexander Shaposhnikov   Type *Ty = V->getType();
90417106792SAlexander Shaposhnikov   if (const auto VT = ftValueTypeFromType(Ty))
90517106792SAlexander Shaposhnikov     return Builder.CreateCall(
90617106792SAlexander Shaposhnikov         NsanCheckValue[*VT],
90717106792SAlexander Shaposhnikov         {V, ShadowV, Loc.getType(Context), Loc.getValue(IntptrTy, Builder)});
90817106792SAlexander Shaposhnikov 
90917106792SAlexander Shaposhnikov   if (Ty->isVectorTy()) {
91017106792SAlexander Shaposhnikov     auto *VecTy = cast<VectorType>(Ty);
91117106792SAlexander Shaposhnikov     // We currently skip scalable vector types in MappingConfig,
91217106792SAlexander Shaposhnikov     // thus we should not encounter any such types here.
91317106792SAlexander Shaposhnikov     assert(!VecTy->isScalableTy() &&
91417106792SAlexander Shaposhnikov            "Scalable vector types are not supported yet");
91517106792SAlexander Shaposhnikov     Value *CheckResult = nullptr;
91617106792SAlexander Shaposhnikov     for (int I = 0, E = VecTy->getElementCount().getFixedValue(); I < E; ++I) {
91717106792SAlexander Shaposhnikov       // We resume if any element resumes. Another option would be to create a
91817106792SAlexander Shaposhnikov       // vector shuffle with the array of ContinueWithShadow, but that is too
91917106792SAlexander Shaposhnikov       // complex.
92017106792SAlexander Shaposhnikov       Value *ExtractV = Builder.CreateExtractElement(V, I);
92117106792SAlexander Shaposhnikov       Value *ExtractShadowV = Builder.CreateExtractElement(ShadowV, I);
92217106792SAlexander Shaposhnikov       Value *ComponentCheckResult =
92317106792SAlexander Shaposhnikov           emitCheckInternal(ExtractV, ExtractShadowV, Builder, Loc);
92417106792SAlexander Shaposhnikov       CheckResult = CheckResult
92517106792SAlexander Shaposhnikov                         ? Builder.CreateOr(CheckResult, ComponentCheckResult)
92617106792SAlexander Shaposhnikov                         : ComponentCheckResult;
92717106792SAlexander Shaposhnikov     }
92817106792SAlexander Shaposhnikov     return CheckResult;
92917106792SAlexander Shaposhnikov   }
93017106792SAlexander Shaposhnikov   if (Ty->isArrayTy()) {
93117106792SAlexander Shaposhnikov     Value *CheckResult = nullptr;
93217106792SAlexander Shaposhnikov     for (auto I : seq(Ty->getArrayNumElements())) {
93317106792SAlexander Shaposhnikov       Value *ExtractV = Builder.CreateExtractElement(V, I);
93417106792SAlexander Shaposhnikov       Value *ExtractShadowV = Builder.CreateExtractElement(ShadowV, I);
93517106792SAlexander Shaposhnikov       Value *ComponentCheckResult =
93617106792SAlexander Shaposhnikov           emitCheckInternal(ExtractV, ExtractShadowV, Builder, Loc);
93717106792SAlexander Shaposhnikov       CheckResult = CheckResult
93817106792SAlexander Shaposhnikov                         ? Builder.CreateOr(CheckResult, ComponentCheckResult)
93917106792SAlexander Shaposhnikov                         : ComponentCheckResult;
94017106792SAlexander Shaposhnikov     }
94117106792SAlexander Shaposhnikov     return CheckResult;
94217106792SAlexander Shaposhnikov   }
94317106792SAlexander Shaposhnikov   if (Ty->isStructTy()) {
94417106792SAlexander Shaposhnikov     Value *CheckResult = nullptr;
94517106792SAlexander Shaposhnikov     for (auto I : seq(Ty->getStructNumElements())) {
94617106792SAlexander Shaposhnikov       if (Config.getExtendedFPType(Ty->getStructElementType(I)) == nullptr)
94717106792SAlexander Shaposhnikov         continue; // Only check FT values.
94817106792SAlexander Shaposhnikov       Value *ExtractV = Builder.CreateExtractValue(V, I);
94917106792SAlexander Shaposhnikov       Value *ExtractShadowV = Builder.CreateExtractElement(ShadowV, I);
95017106792SAlexander Shaposhnikov       Value *ComponentCheckResult =
95117106792SAlexander Shaposhnikov           emitCheckInternal(ExtractV, ExtractShadowV, Builder, Loc);
95217106792SAlexander Shaposhnikov       CheckResult = CheckResult
95317106792SAlexander Shaposhnikov                         ? Builder.CreateOr(CheckResult, ComponentCheckResult)
95417106792SAlexander Shaposhnikov                         : ComponentCheckResult;
95517106792SAlexander Shaposhnikov     }
95617106792SAlexander Shaposhnikov     if (!CheckResult)
95717106792SAlexander Shaposhnikov       return ConstantInt::get(
95817106792SAlexander Shaposhnikov           Builder.getInt32Ty(),
95917106792SAlexander Shaposhnikov           static_cast<int>(ContinuationType::ContinueWithShadow));
96017106792SAlexander Shaposhnikov     return CheckResult;
96117106792SAlexander Shaposhnikov   }
96217106792SAlexander Shaposhnikov 
96317106792SAlexander Shaposhnikov   llvm_unreachable("not implemented");
96417106792SAlexander Shaposhnikov }
96517106792SAlexander Shaposhnikov 
96617106792SAlexander Shaposhnikov // Inserts a runtime check of V against its shadow value ShadowV.
96717106792SAlexander Shaposhnikov // We check values whenever they escape: on return, call, stores, and
96817106792SAlexander Shaposhnikov // insertvalue.
96917106792SAlexander Shaposhnikov // Returns the shadow value that should be used to continue the computations,
97017106792SAlexander Shaposhnikov // depending on the answer from the runtime.
97117106792SAlexander Shaposhnikov // TODO: Should we check on select ? phi ?
97217106792SAlexander Shaposhnikov Value *NumericalStabilitySanitizer::emitCheck(Value *V, Value *ShadowV,
97317106792SAlexander Shaposhnikov                                               IRBuilder<> &Builder,
97417106792SAlexander Shaposhnikov                                               CheckLoc Loc) {
97517106792SAlexander Shaposhnikov   // Do not emit checks for constant values, this is redundant.
97617106792SAlexander Shaposhnikov   if (isa<Constant>(V))
97717106792SAlexander Shaposhnikov     return ShadowV;
97817106792SAlexander Shaposhnikov 
97917106792SAlexander Shaposhnikov   if (Instruction *Inst = dyn_cast<Instruction>(V)) {
98017106792SAlexander Shaposhnikov     Function *F = Inst->getFunction();
98117106792SAlexander Shaposhnikov     if (CheckFunctionsFilter && !CheckFunctionsFilter->match(F->getName())) {
98217106792SAlexander Shaposhnikov       return ShadowV;
98317106792SAlexander Shaposhnikov     }
98417106792SAlexander Shaposhnikov   }
98517106792SAlexander Shaposhnikov 
98617106792SAlexander Shaposhnikov   Value *CheckResult = emitCheckInternal(V, ShadowV, Builder, Loc);
98717106792SAlexander Shaposhnikov   Value *ICmpEQ = Builder.CreateICmpEQ(
98817106792SAlexander Shaposhnikov       CheckResult,
98917106792SAlexander Shaposhnikov       ConstantInt::get(Builder.getInt32Ty(),
99017106792SAlexander Shaposhnikov                        static_cast<int>(ContinuationType::ResumeFromValue)));
99117106792SAlexander Shaposhnikov   return Builder.CreateSelect(
99217106792SAlexander Shaposhnikov       ICmpEQ, Builder.CreateFPExt(V, Config.getExtendedFPType(V->getType())),
99317106792SAlexander Shaposhnikov       ShadowV);
99417106792SAlexander Shaposhnikov }
99517106792SAlexander Shaposhnikov 
99617106792SAlexander Shaposhnikov // Inserts a check that fcmp on shadow values are consistent with that on base
99717106792SAlexander Shaposhnikov // values.
99817106792SAlexander Shaposhnikov void NumericalStabilitySanitizer::emitFCmpCheck(FCmpInst &FCmp,
99917106792SAlexander Shaposhnikov                                                 const ValueToShadowMap &Map) {
100017106792SAlexander Shaposhnikov   if (!ClInstrumentFCmp)
100117106792SAlexander Shaposhnikov     return;
100217106792SAlexander Shaposhnikov 
100317106792SAlexander Shaposhnikov   Function *F = FCmp.getFunction();
100417106792SAlexander Shaposhnikov   if (CheckFunctionsFilter && !CheckFunctionsFilter->match(F->getName()))
100517106792SAlexander Shaposhnikov     return;
100617106792SAlexander Shaposhnikov 
100717106792SAlexander Shaposhnikov   Value *LHS = FCmp.getOperand(0);
100817106792SAlexander Shaposhnikov   if (Config.getExtendedFPType(LHS->getType()) == nullptr)
100917106792SAlexander Shaposhnikov     return;
101017106792SAlexander Shaposhnikov   Value *RHS = FCmp.getOperand(1);
101117106792SAlexander Shaposhnikov 
101217106792SAlexander Shaposhnikov   // Split the basic block. On mismatch, we'll jump to the new basic block with
101317106792SAlexander Shaposhnikov   // a call to the runtime for error reporting.
101417106792SAlexander Shaposhnikov   BasicBlock *FCmpBB = FCmp.getParent();
101517106792SAlexander Shaposhnikov   BasicBlock *NextBB = FCmpBB->splitBasicBlock(FCmp.getNextNode());
101617106792SAlexander Shaposhnikov   // Remove the newly created terminator unconditional branch.
101717106792SAlexander Shaposhnikov   FCmpBB->back().eraseFromParent();
101817106792SAlexander Shaposhnikov   BasicBlock *FailBB =
101917106792SAlexander Shaposhnikov       BasicBlock::Create(Context, "", FCmpBB->getParent(), NextBB);
102017106792SAlexander Shaposhnikov 
102117106792SAlexander Shaposhnikov   // Create the shadow fcmp and comparison between the fcmps.
102217106792SAlexander Shaposhnikov   IRBuilder<> FCmpBuilder(FCmpBB);
102317106792SAlexander Shaposhnikov   FCmpBuilder.SetCurrentDebugLocation(FCmp.getDebugLoc());
102417106792SAlexander Shaposhnikov   Value *ShadowLHS = Map.getShadow(LHS);
102517106792SAlexander Shaposhnikov   Value *ShadowRHS = Map.getShadow(RHS);
102617106792SAlexander Shaposhnikov   // See comment on ClTruncateFCmpEq.
102717106792SAlexander Shaposhnikov   if (FCmp.isEquality() && ClTruncateFCmpEq) {
102817106792SAlexander Shaposhnikov     Type *Ty = ShadowLHS->getType();
102917106792SAlexander Shaposhnikov     ShadowLHS = FCmpBuilder.CreateFPExt(
103017106792SAlexander Shaposhnikov         FCmpBuilder.CreateFPTrunc(ShadowLHS, LHS->getType()), Ty);
103117106792SAlexander Shaposhnikov     ShadowRHS = FCmpBuilder.CreateFPExt(
103217106792SAlexander Shaposhnikov         FCmpBuilder.CreateFPTrunc(ShadowRHS, RHS->getType()), Ty);
103317106792SAlexander Shaposhnikov   }
103417106792SAlexander Shaposhnikov   Value *ShadowFCmp =
103517106792SAlexander Shaposhnikov       FCmpBuilder.CreateFCmp(FCmp.getPredicate(), ShadowLHS, ShadowRHS);
103617106792SAlexander Shaposhnikov   Value *OriginalAndShadowFcmpMatch =
103717106792SAlexander Shaposhnikov       FCmpBuilder.CreateICmpEQ(&FCmp, ShadowFCmp);
103817106792SAlexander Shaposhnikov 
103917106792SAlexander Shaposhnikov   if (OriginalAndShadowFcmpMatch->getType()->isVectorTy()) {
104017106792SAlexander Shaposhnikov     // If we have a vector type, `OriginalAndShadowFcmpMatch` is a vector of i1,
104117106792SAlexander Shaposhnikov     // where an element is true if the corresponding elements in original and
104217106792SAlexander Shaposhnikov     // shadow are the same. We want all elements to be 1.
104317106792SAlexander Shaposhnikov     OriginalAndShadowFcmpMatch =
104417106792SAlexander Shaposhnikov         FCmpBuilder.CreateAndReduce(OriginalAndShadowFcmpMatch);
104517106792SAlexander Shaposhnikov   }
104617106792SAlexander Shaposhnikov 
104717106792SAlexander Shaposhnikov   // Use MDBuilder(*C).createLikelyBranchWeights() because "match" is the common
104817106792SAlexander Shaposhnikov   // case.
104917106792SAlexander Shaposhnikov   FCmpBuilder.CreateCondBr(OriginalAndShadowFcmpMatch, NextBB, FailBB,
105017106792SAlexander Shaposhnikov                            MDBuilder(Context).createLikelyBranchWeights());
105117106792SAlexander Shaposhnikov 
105217106792SAlexander Shaposhnikov   // Fill in FailBB.
105317106792SAlexander Shaposhnikov   IRBuilder<> FailBuilder(FailBB);
105417106792SAlexander Shaposhnikov   FailBuilder.SetCurrentDebugLocation(FCmp.getDebugLoc());
105517106792SAlexander Shaposhnikov 
105617106792SAlexander Shaposhnikov   const auto EmitFailCall = [this, &FCmp, &FCmpBuilder,
105717106792SAlexander Shaposhnikov                              &FailBuilder](Value *L, Value *R, Value *ShadowL,
105817106792SAlexander Shaposhnikov                                            Value *ShadowR, Value *Result,
105917106792SAlexander Shaposhnikov                                            Value *ShadowResult) {
106017106792SAlexander Shaposhnikov     Type *FT = L->getType();
106117106792SAlexander Shaposhnikov     FunctionCallee *Callee = nullptr;
106217106792SAlexander Shaposhnikov     if (FT->isFloatTy()) {
106317106792SAlexander Shaposhnikov       Callee = &(NsanFCmpFail[kFloat]);
106417106792SAlexander Shaposhnikov     } else if (FT->isDoubleTy()) {
106517106792SAlexander Shaposhnikov       Callee = &(NsanFCmpFail[kDouble]);
106617106792SAlexander Shaposhnikov     } else if (FT->isX86_FP80Ty()) {
106717106792SAlexander Shaposhnikov       // TODO: make NsanFCmpFailLongDouble work.
106817106792SAlexander Shaposhnikov       Callee = &(NsanFCmpFail[kDouble]);
106917106792SAlexander Shaposhnikov       L = FailBuilder.CreateFPTrunc(L, Type::getDoubleTy(Context));
107017106792SAlexander Shaposhnikov       R = FailBuilder.CreateFPTrunc(L, Type::getDoubleTy(Context));
107117106792SAlexander Shaposhnikov     } else {
107217106792SAlexander Shaposhnikov       llvm_unreachable("not implemented");
107317106792SAlexander Shaposhnikov     }
107417106792SAlexander Shaposhnikov     FailBuilder.CreateCall(*Callee, {L, R, ShadowL, ShadowR,
107517106792SAlexander Shaposhnikov                                      ConstantInt::get(FCmpBuilder.getInt32Ty(),
107617106792SAlexander Shaposhnikov                                                       FCmp.getPredicate()),
107717106792SAlexander Shaposhnikov                                      Result, ShadowResult});
107817106792SAlexander Shaposhnikov   };
107917106792SAlexander Shaposhnikov   if (LHS->getType()->isVectorTy()) {
108017106792SAlexander Shaposhnikov     for (int I = 0, E = cast<VectorType>(LHS->getType())
108117106792SAlexander Shaposhnikov                             ->getElementCount()
108217106792SAlexander Shaposhnikov                             .getFixedValue();
108317106792SAlexander Shaposhnikov          I < E; ++I) {
108417106792SAlexander Shaposhnikov       Value *ExtractLHS = FailBuilder.CreateExtractElement(LHS, I);
108517106792SAlexander Shaposhnikov       Value *ExtractRHS = FailBuilder.CreateExtractElement(RHS, I);
108617106792SAlexander Shaposhnikov       Value *ExtractShaodwLHS = FailBuilder.CreateExtractElement(ShadowLHS, I);
108717106792SAlexander Shaposhnikov       Value *ExtractShaodwRHS = FailBuilder.CreateExtractElement(ShadowRHS, I);
108817106792SAlexander Shaposhnikov       Value *ExtractFCmp = FailBuilder.CreateExtractElement(&FCmp, I);
108917106792SAlexander Shaposhnikov       Value *ExtractShadowFCmp =
109017106792SAlexander Shaposhnikov           FailBuilder.CreateExtractElement(ShadowFCmp, I);
109117106792SAlexander Shaposhnikov       EmitFailCall(ExtractLHS, ExtractRHS, ExtractShaodwLHS, ExtractShaodwRHS,
109217106792SAlexander Shaposhnikov                    ExtractFCmp, ExtractShadowFCmp);
109317106792SAlexander Shaposhnikov     }
109417106792SAlexander Shaposhnikov   } else {
109517106792SAlexander Shaposhnikov     EmitFailCall(LHS, RHS, ShadowLHS, ShadowRHS, &FCmp, ShadowFCmp);
109617106792SAlexander Shaposhnikov   }
109717106792SAlexander Shaposhnikov   FailBuilder.CreateBr(NextBB);
109817106792SAlexander Shaposhnikov 
109917106792SAlexander Shaposhnikov   ++NumInstrumentedFCmp;
110017106792SAlexander Shaposhnikov }
110117106792SAlexander Shaposhnikov 
110217106792SAlexander Shaposhnikov // Creates a shadow phi value for any phi that defines a value of FT type.
110317106792SAlexander Shaposhnikov PHINode *NumericalStabilitySanitizer::maybeCreateShadowPhi(
110417106792SAlexander Shaposhnikov     PHINode &Phi, const TargetLibraryInfo &TLI) {
110517106792SAlexander Shaposhnikov   Type *VT = Phi.getType();
110617106792SAlexander Shaposhnikov   Type *ExtendedVT = Config.getExtendedFPType(VT);
110717106792SAlexander Shaposhnikov   if (ExtendedVT == nullptr)
110817106792SAlexander Shaposhnikov     return nullptr; // Not an FT value.
110917106792SAlexander Shaposhnikov   // The phi operands are shadow values and are not available when the phi is
111017106792SAlexander Shaposhnikov   // created. They will be populated in a final phase, once all shadow values
111117106792SAlexander Shaposhnikov   // have been created.
111217106792SAlexander Shaposhnikov   PHINode *Shadow = PHINode::Create(ExtendedVT, Phi.getNumIncomingValues());
11138e702735SJeremy Morse   Shadow->insertAfter(Phi.getIterator());
111417106792SAlexander Shaposhnikov   return Shadow;
111517106792SAlexander Shaposhnikov }
111617106792SAlexander Shaposhnikov 
111717106792SAlexander Shaposhnikov Value *NumericalStabilitySanitizer::handleLoad(LoadInst &Load, Type *VT,
111817106792SAlexander Shaposhnikov                                                Type *ExtendedVT) {
111917106792SAlexander Shaposhnikov   IRBuilder<> Builder(Load.getNextNode());
112017106792SAlexander Shaposhnikov   Builder.SetCurrentDebugLocation(Load.getDebugLoc());
112117106792SAlexander Shaposhnikov   if (addrPointsToConstantData(Load.getPointerOperand())) {
112217106792SAlexander Shaposhnikov     // No need to look into the shadow memory, the value is a constant. Just
112317106792SAlexander Shaposhnikov     // convert from FT to 2FT.
112417106792SAlexander Shaposhnikov     return Builder.CreateFPExt(&Load, ExtendedVT);
112517106792SAlexander Shaposhnikov   }
112617106792SAlexander Shaposhnikov 
112717106792SAlexander Shaposhnikov   // if (%shadowptr == &)
112817106792SAlexander Shaposhnikov   //    %shadow = fpext %v
112917106792SAlexander Shaposhnikov   // else
113017106792SAlexander Shaposhnikov   //    %shadow = load (ptrcast %shadow_ptr))
113117106792SAlexander Shaposhnikov   // Considered options here:
113217106792SAlexander Shaposhnikov   //  - Have `NsanGetShadowPtrForLoad` return a fixed address
113317106792SAlexander Shaposhnikov   //    &__nsan_unknown_value_shadow_address that is valid to load from, and
113417106792SAlexander Shaposhnikov   //    use a select. This has the advantage that the generated IR is simpler.
113517106792SAlexander Shaposhnikov   //  - Have `NsanGetShadowPtrForLoad` return nullptr.  Because `select` does
113617106792SAlexander Shaposhnikov   //    not short-circuit, dereferencing the returned pointer is no longer an
113717106792SAlexander Shaposhnikov   //    option, have to split and create a separate basic block. This has the
113817106792SAlexander Shaposhnikov   //    advantage of being easier to debug because it crashes if we ever mess
113917106792SAlexander Shaposhnikov   //    up.
114017106792SAlexander Shaposhnikov 
114117106792SAlexander Shaposhnikov   const auto Extents = getMemoryExtentsOrDie(VT);
114217106792SAlexander Shaposhnikov   Value *ShadowPtr = Builder.CreateCall(
114317106792SAlexander Shaposhnikov       NsanGetShadowPtrForLoad[Extents.ValueType],
114417106792SAlexander Shaposhnikov       {Load.getPointerOperand(), ConstantInt::get(IntptrTy, Extents.NumElts)});
114517106792SAlexander Shaposhnikov   ++NumInstrumentedFTLoads;
114617106792SAlexander Shaposhnikov 
114717106792SAlexander Shaposhnikov   // Split the basic block.
114817106792SAlexander Shaposhnikov   BasicBlock *LoadBB = Load.getParent();
114917106792SAlexander Shaposhnikov   BasicBlock *NextBB = LoadBB->splitBasicBlock(Builder.GetInsertPoint());
115017106792SAlexander Shaposhnikov   // Create the two options for creating the shadow value.
115117106792SAlexander Shaposhnikov   BasicBlock *ShadowLoadBB =
115217106792SAlexander Shaposhnikov       BasicBlock::Create(Context, "", LoadBB->getParent(), NextBB);
115317106792SAlexander Shaposhnikov   BasicBlock *FExtBB =
115417106792SAlexander Shaposhnikov       BasicBlock::Create(Context, "", LoadBB->getParent(), NextBB);
115517106792SAlexander Shaposhnikov 
115617106792SAlexander Shaposhnikov   // Replace the newly created terminator unconditional branch by a conditional
115717106792SAlexander Shaposhnikov   // branch to one of the options.
115817106792SAlexander Shaposhnikov   {
115917106792SAlexander Shaposhnikov     LoadBB->back().eraseFromParent();
116017106792SAlexander Shaposhnikov     IRBuilder<> LoadBBBuilder(LoadBB); // The old builder has been invalidated.
116117106792SAlexander Shaposhnikov     LoadBBBuilder.SetCurrentDebugLocation(Load.getDebugLoc());
116217106792SAlexander Shaposhnikov     LoadBBBuilder.CreateCondBr(LoadBBBuilder.CreateIsNull(ShadowPtr), FExtBB,
116317106792SAlexander Shaposhnikov                                ShadowLoadBB);
116417106792SAlexander Shaposhnikov   }
116517106792SAlexander Shaposhnikov 
116617106792SAlexander Shaposhnikov   // Fill in ShadowLoadBB.
116717106792SAlexander Shaposhnikov   IRBuilder<> ShadowLoadBBBuilder(ShadowLoadBB);
116817106792SAlexander Shaposhnikov   ShadowLoadBBBuilder.SetCurrentDebugLocation(Load.getDebugLoc());
116917106792SAlexander Shaposhnikov   Value *ShadowLoad = ShadowLoadBBBuilder.CreateAlignedLoad(
117017106792SAlexander Shaposhnikov       ExtendedVT, ShadowPtr, Align(1), Load.isVolatile());
117117106792SAlexander Shaposhnikov   if (ClCheckLoads) {
117217106792SAlexander Shaposhnikov     ShadowLoad = emitCheck(&Load, ShadowLoad, ShadowLoadBBBuilder,
117317106792SAlexander Shaposhnikov                            CheckLoc::makeLoad(Load.getPointerOperand()));
117417106792SAlexander Shaposhnikov   }
117517106792SAlexander Shaposhnikov   ShadowLoadBBBuilder.CreateBr(NextBB);
117617106792SAlexander Shaposhnikov 
117717106792SAlexander Shaposhnikov   // Fill in FExtBB.
117817106792SAlexander Shaposhnikov   IRBuilder<> FExtBBBuilder(FExtBB);
117917106792SAlexander Shaposhnikov   FExtBBBuilder.SetCurrentDebugLocation(Load.getDebugLoc());
118017106792SAlexander Shaposhnikov   Value *FExt = FExtBBBuilder.CreateFPExt(&Load, ExtendedVT);
118117106792SAlexander Shaposhnikov   FExtBBBuilder.CreateBr(NextBB);
118217106792SAlexander Shaposhnikov 
118317106792SAlexander Shaposhnikov   // The shadow value come from any of the options.
118417106792SAlexander Shaposhnikov   IRBuilder<> NextBBBuilder(&*NextBB->begin());
118517106792SAlexander Shaposhnikov   NextBBBuilder.SetCurrentDebugLocation(Load.getDebugLoc());
118617106792SAlexander Shaposhnikov   PHINode *ShadowPhi = NextBBBuilder.CreatePHI(ExtendedVT, 2);
118717106792SAlexander Shaposhnikov   ShadowPhi->addIncoming(ShadowLoad, ShadowLoadBB);
118817106792SAlexander Shaposhnikov   ShadowPhi->addIncoming(FExt, FExtBB);
118917106792SAlexander Shaposhnikov   return ShadowPhi;
119017106792SAlexander Shaposhnikov }
119117106792SAlexander Shaposhnikov 
119217106792SAlexander Shaposhnikov Value *NumericalStabilitySanitizer::handleTrunc(const FPTruncInst &Trunc,
119317106792SAlexander Shaposhnikov                                                 Type *VT, Type *ExtendedVT,
119417106792SAlexander Shaposhnikov                                                 const ValueToShadowMap &Map,
119517106792SAlexander Shaposhnikov                                                 IRBuilder<> &Builder) {
119617106792SAlexander Shaposhnikov   Value *OrigSource = Trunc.getOperand(0);
119717106792SAlexander Shaposhnikov   Type *OrigSourceTy = OrigSource->getType();
119817106792SAlexander Shaposhnikov   Type *ExtendedSourceTy = Config.getExtendedFPType(OrigSourceTy);
119917106792SAlexander Shaposhnikov 
120017106792SAlexander Shaposhnikov   // When truncating:
120117106792SAlexander Shaposhnikov   //  - (A) If the source has a shadow, we truncate from the shadow, else we
120217106792SAlexander Shaposhnikov   //    truncate from the original source.
120317106792SAlexander Shaposhnikov   //  - (B) If the shadow of the source is larger than the shadow of the dest,
120417106792SAlexander Shaposhnikov   //    we still need a truncate. Else, the shadow of the source is the same
120517106792SAlexander Shaposhnikov   //    type as the shadow of the dest (because mappings are non-decreasing), so
120617106792SAlexander Shaposhnikov   //   we don't need to emit a truncate.
120717106792SAlexander Shaposhnikov   // Examples,
120817106792SAlexander Shaposhnikov   //   with a mapping of {f32->f64;f64->f80;f80->f128}
120917106792SAlexander Shaposhnikov   //     fptrunc double   %1 to float     ->  fptrunc x86_fp80 s(%1) to double
121017106792SAlexander Shaposhnikov   //     fptrunc x86_fp80 %1 to float     ->  fptrunc fp128    s(%1) to double
121117106792SAlexander Shaposhnikov   //     fptrunc fp128    %1 to float     ->  fptrunc fp128    %1    to double
121217106792SAlexander Shaposhnikov   //     fptrunc x86_fp80 %1 to double    ->  x86_fp80 s(%1)
121317106792SAlexander Shaposhnikov   //     fptrunc fp128    %1 to double    ->  fptrunc fp128 %1 to x86_fp80
121417106792SAlexander Shaposhnikov   //     fptrunc fp128    %1 to x86_fp80  ->  fp128 %1
121517106792SAlexander Shaposhnikov   //   with a mapping of {f32->f64;f64->f128;f80->f128}
121617106792SAlexander Shaposhnikov   //     fptrunc double   %1 to float     ->  fptrunc fp128    s(%1) to double
121717106792SAlexander Shaposhnikov   //     fptrunc x86_fp80 %1 to float     ->  fptrunc fp128    s(%1) to double
121817106792SAlexander Shaposhnikov   //     fptrunc fp128    %1 to float     ->  fptrunc fp128    %1    to double
121917106792SAlexander Shaposhnikov   //     fptrunc x86_fp80 %1 to double    ->  fp128 %1
122017106792SAlexander Shaposhnikov   //     fptrunc fp128    %1 to double    ->  fp128 %1
122117106792SAlexander Shaposhnikov   //     fptrunc fp128    %1 to x86_fp80  ->  fp128 %1
122217106792SAlexander Shaposhnikov   //   with a mapping of {f32->f32;f64->f32;f80->f64}
122317106792SAlexander Shaposhnikov   //     fptrunc double   %1 to float     ->  float s(%1)
122417106792SAlexander Shaposhnikov   //     fptrunc x86_fp80 %1 to float     ->  fptrunc double    s(%1) to float
122517106792SAlexander Shaposhnikov   //     fptrunc fp128    %1 to float     ->  fptrunc fp128     %1    to float
122617106792SAlexander Shaposhnikov   //     fptrunc x86_fp80 %1 to double    ->  fptrunc double    s(%1) to float
122717106792SAlexander Shaposhnikov   //     fptrunc fp128    %1 to double    ->  fptrunc fp128     %1    to float
122817106792SAlexander Shaposhnikov   //     fptrunc fp128    %1 to x86_fp80  ->  fptrunc fp128     %1    to double
122917106792SAlexander Shaposhnikov 
123017106792SAlexander Shaposhnikov   // See (A) above.
123117106792SAlexander Shaposhnikov   Value *Source = ExtendedSourceTy ? Map.getShadow(OrigSource) : OrigSource;
123217106792SAlexander Shaposhnikov   Type *SourceTy = ExtendedSourceTy ? ExtendedSourceTy : OrigSourceTy;
123317106792SAlexander Shaposhnikov   // See (B) above.
123417106792SAlexander Shaposhnikov   if (SourceTy == ExtendedVT)
123517106792SAlexander Shaposhnikov     return Source;
123617106792SAlexander Shaposhnikov 
123717106792SAlexander Shaposhnikov   return Builder.CreateFPTrunc(Source, ExtendedVT);
123817106792SAlexander Shaposhnikov }
123917106792SAlexander Shaposhnikov 
124017106792SAlexander Shaposhnikov Value *NumericalStabilitySanitizer::handleExt(const FPExtInst &Ext, Type *VT,
124117106792SAlexander Shaposhnikov                                               Type *ExtendedVT,
124217106792SAlexander Shaposhnikov                                               const ValueToShadowMap &Map,
124317106792SAlexander Shaposhnikov                                               IRBuilder<> &Builder) {
124417106792SAlexander Shaposhnikov   Value *OrigSource = Ext.getOperand(0);
124517106792SAlexander Shaposhnikov   Type *OrigSourceTy = OrigSource->getType();
124617106792SAlexander Shaposhnikov   Type *ExtendedSourceTy = Config.getExtendedFPType(OrigSourceTy);
124717106792SAlexander Shaposhnikov   // When extending:
124817106792SAlexander Shaposhnikov   //  - (A) If the source has a shadow, we extend from the shadow, else we
124917106792SAlexander Shaposhnikov   //    extend from the original source.
125017106792SAlexander Shaposhnikov   //  - (B) If the shadow of the dest is larger than the shadow of the source,
125117106792SAlexander Shaposhnikov   //    we still need an extend. Else, the shadow of the source is the same
125217106792SAlexander Shaposhnikov   //    type as the shadow of the dest (because mappings are non-decreasing), so
125317106792SAlexander Shaposhnikov   //    we don't need to emit an extend.
125417106792SAlexander Shaposhnikov   // Examples,
125517106792SAlexander Shaposhnikov   //   with a mapping of {f32->f64;f64->f80;f80->f128}
125617106792SAlexander Shaposhnikov   //     fpext half    %1 to float     ->  fpext half     %1    to double
125717106792SAlexander Shaposhnikov   //     fpext half    %1 to double    ->  fpext half     %1    to x86_fp80
125817106792SAlexander Shaposhnikov   //     fpext half    %1 to x86_fp80  ->  fpext half     %1    to fp128
125917106792SAlexander Shaposhnikov   //     fpext float   %1 to double    ->  double s(%1)
126017106792SAlexander Shaposhnikov   //     fpext float   %1 to x86_fp80  ->  fpext double   s(%1) to fp128
126117106792SAlexander Shaposhnikov   //     fpext double  %1 to x86_fp80  ->  fpext x86_fp80 s(%1) to fp128
126217106792SAlexander Shaposhnikov   //   with a mapping of {f32->f64;f64->f128;f80->f128}
126317106792SAlexander Shaposhnikov   //     fpext half    %1 to float     ->  fpext half     %1    to double
126417106792SAlexander Shaposhnikov   //     fpext half    %1 to double    ->  fpext half     %1    to fp128
126517106792SAlexander Shaposhnikov   //     fpext half    %1 to x86_fp80  ->  fpext half     %1    to fp128
126617106792SAlexander Shaposhnikov   //     fpext float   %1 to double    ->  fpext double   s(%1) to fp128
126717106792SAlexander Shaposhnikov   //     fpext float   %1 to x86_fp80  ->  fpext double   s(%1) to fp128
126817106792SAlexander Shaposhnikov   //     fpext double  %1 to x86_fp80  ->  fp128 s(%1)
126917106792SAlexander Shaposhnikov   //   with a mapping of {f32->f32;f64->f32;f80->f64}
127017106792SAlexander Shaposhnikov   //     fpext half    %1 to float     ->  fpext half     %1    to float
127117106792SAlexander Shaposhnikov   //     fpext half    %1 to double    ->  fpext half     %1    to float
127217106792SAlexander Shaposhnikov   //     fpext half    %1 to x86_fp80  ->  fpext half     %1    to double
127317106792SAlexander Shaposhnikov   //     fpext float   %1 to double    ->  s(%1)
127417106792SAlexander Shaposhnikov   //     fpext float   %1 to x86_fp80  ->  fpext float    s(%1) to double
127517106792SAlexander Shaposhnikov   //     fpext double  %1 to x86_fp80  ->  fpext float    s(%1) to double
127617106792SAlexander Shaposhnikov 
127717106792SAlexander Shaposhnikov   // See (A) above.
127817106792SAlexander Shaposhnikov   Value *Source = ExtendedSourceTy ? Map.getShadow(OrigSource) : OrigSource;
127917106792SAlexander Shaposhnikov   Type *SourceTy = ExtendedSourceTy ? ExtendedSourceTy : OrigSourceTy;
128017106792SAlexander Shaposhnikov   // See (B) above.
128117106792SAlexander Shaposhnikov   if (SourceTy == ExtendedVT)
128217106792SAlexander Shaposhnikov     return Source;
128317106792SAlexander Shaposhnikov 
128417106792SAlexander Shaposhnikov   return Builder.CreateFPExt(Source, ExtendedVT);
128517106792SAlexander Shaposhnikov }
128617106792SAlexander Shaposhnikov 
128717106792SAlexander Shaposhnikov namespace {
128817106792SAlexander Shaposhnikov // TODO: This should be tablegen-ed.
128917106792SAlexander Shaposhnikov struct KnownIntrinsic {
129017106792SAlexander Shaposhnikov   struct WidenedIntrinsic {
129117106792SAlexander Shaposhnikov     const char *NarrowName;
129217106792SAlexander Shaposhnikov     Intrinsic::ID ID; // wide id.
129317106792SAlexander Shaposhnikov     using FnTypeFactory = FunctionType *(*)(LLVMContext &);
129417106792SAlexander Shaposhnikov     FnTypeFactory MakeFnTy;
129517106792SAlexander Shaposhnikov   };
129617106792SAlexander Shaposhnikov 
129717106792SAlexander Shaposhnikov   static const char *get(LibFunc LFunc);
129817106792SAlexander Shaposhnikov 
129917106792SAlexander Shaposhnikov   // Given an intrinsic with an `FT` argument, try to find a wider intrinsic
130017106792SAlexander Shaposhnikov   // that applies the same operation on the shadow argument.
130117106792SAlexander Shaposhnikov   // Options are:
130217106792SAlexander Shaposhnikov   //  - pass in the ID and full function type,
130317106792SAlexander Shaposhnikov   //  - pass in the name, which includes the function type through mangling.
130417106792SAlexander Shaposhnikov   static const WidenedIntrinsic *widen(StringRef Name);
130517106792SAlexander Shaposhnikov 
130617106792SAlexander Shaposhnikov private:
130717106792SAlexander Shaposhnikov   struct LFEntry {
130817106792SAlexander Shaposhnikov     LibFunc LFunc;
130917106792SAlexander Shaposhnikov     const char *IntrinsicName;
131017106792SAlexander Shaposhnikov   };
131117106792SAlexander Shaposhnikov   static const LFEntry kLibfuncIntrinsics[];
131217106792SAlexander Shaposhnikov 
131317106792SAlexander Shaposhnikov   static const WidenedIntrinsic kWidenedIntrinsics[];
131417106792SAlexander Shaposhnikov };
131517106792SAlexander Shaposhnikov } // namespace
131617106792SAlexander Shaposhnikov 
131717106792SAlexander Shaposhnikov static FunctionType *makeDoubleDouble(LLVMContext &C) {
131817106792SAlexander Shaposhnikov   return FunctionType::get(Type::getDoubleTy(C), {Type::getDoubleTy(C)}, false);
131917106792SAlexander Shaposhnikov }
132017106792SAlexander Shaposhnikov 
132117106792SAlexander Shaposhnikov static FunctionType *makeX86FP80X86FP80(LLVMContext &C) {
132217106792SAlexander Shaposhnikov   return FunctionType::get(Type::getX86_FP80Ty(C), {Type::getX86_FP80Ty(C)},
132317106792SAlexander Shaposhnikov                            false);
132417106792SAlexander Shaposhnikov }
132517106792SAlexander Shaposhnikov 
132617106792SAlexander Shaposhnikov static FunctionType *makeDoubleDoubleI32(LLVMContext &C) {
132717106792SAlexander Shaposhnikov   return FunctionType::get(Type::getDoubleTy(C),
132817106792SAlexander Shaposhnikov                            {Type::getDoubleTy(C), Type::getInt32Ty(C)}, false);
132917106792SAlexander Shaposhnikov }
133017106792SAlexander Shaposhnikov 
133117106792SAlexander Shaposhnikov static FunctionType *makeX86FP80X86FP80I32(LLVMContext &C) {
133217106792SAlexander Shaposhnikov   return FunctionType::get(Type::getX86_FP80Ty(C),
133317106792SAlexander Shaposhnikov                            {Type::getX86_FP80Ty(C), Type::getInt32Ty(C)},
133417106792SAlexander Shaposhnikov                            false);
133517106792SAlexander Shaposhnikov }
133617106792SAlexander Shaposhnikov 
133717106792SAlexander Shaposhnikov static FunctionType *makeDoubleDoubleDouble(LLVMContext &C) {
133817106792SAlexander Shaposhnikov   return FunctionType::get(Type::getDoubleTy(C),
133917106792SAlexander Shaposhnikov                            {Type::getDoubleTy(C), Type::getDoubleTy(C)}, false);
134017106792SAlexander Shaposhnikov }
134117106792SAlexander Shaposhnikov 
134217106792SAlexander Shaposhnikov static FunctionType *makeX86FP80X86FP80X86FP80(LLVMContext &C) {
134317106792SAlexander Shaposhnikov   return FunctionType::get(Type::getX86_FP80Ty(C),
134417106792SAlexander Shaposhnikov                            {Type::getX86_FP80Ty(C), Type::getX86_FP80Ty(C)},
134517106792SAlexander Shaposhnikov                            false);
134617106792SAlexander Shaposhnikov }
134717106792SAlexander Shaposhnikov 
134817106792SAlexander Shaposhnikov static FunctionType *makeDoubleDoubleDoubleDouble(LLVMContext &C) {
134917106792SAlexander Shaposhnikov   return FunctionType::get(
135017106792SAlexander Shaposhnikov       Type::getDoubleTy(C),
135117106792SAlexander Shaposhnikov       {Type::getDoubleTy(C), Type::getDoubleTy(C), Type::getDoubleTy(C)},
135217106792SAlexander Shaposhnikov       false);
135317106792SAlexander Shaposhnikov }
135417106792SAlexander Shaposhnikov 
135517106792SAlexander Shaposhnikov static FunctionType *makeX86FP80X86FP80X86FP80X86FP80(LLVMContext &C) {
135617106792SAlexander Shaposhnikov   return FunctionType::get(
135717106792SAlexander Shaposhnikov       Type::getX86_FP80Ty(C),
135817106792SAlexander Shaposhnikov       {Type::getX86_FP80Ty(C), Type::getX86_FP80Ty(C), Type::getX86_FP80Ty(C)},
135917106792SAlexander Shaposhnikov       false);
136017106792SAlexander Shaposhnikov }
136117106792SAlexander Shaposhnikov 
136217106792SAlexander Shaposhnikov const KnownIntrinsic::WidenedIntrinsic KnownIntrinsic::kWidenedIntrinsics[] = {
136317106792SAlexander Shaposhnikov     // TODO: Right now we ignore vector intrinsics.
136417106792SAlexander Shaposhnikov     // This is hard because we have to model the semantics of the intrinsics,
136517106792SAlexander Shaposhnikov     // e.g. llvm.x86.sse2.min.sd means extract first element, min, insert back.
136617106792SAlexander Shaposhnikov     // Intrinsics that take any non-vector FT types:
136717106792SAlexander Shaposhnikov     // NOTE: Right now because of
136817106792SAlexander Shaposhnikov     // https://github.com/llvm/llvm-project/issues/44744
136917106792SAlexander Shaposhnikov     // for f128 we need to use makeX86FP80X86FP80 (go to a lower precision and
137017106792SAlexander Shaposhnikov     // come back).
137117106792SAlexander Shaposhnikov     {"llvm.sqrt.f32", Intrinsic::sqrt, makeDoubleDouble},
137217106792SAlexander Shaposhnikov     {"llvm.sqrt.f64", Intrinsic::sqrt, makeX86FP80X86FP80},
137317106792SAlexander Shaposhnikov     {"llvm.sqrt.f80", Intrinsic::sqrt, makeX86FP80X86FP80},
137417106792SAlexander Shaposhnikov     {"llvm.powi.f32", Intrinsic::powi, makeDoubleDoubleI32},
137517106792SAlexander Shaposhnikov     {"llvm.powi.f64", Intrinsic::powi, makeX86FP80X86FP80I32},
137617106792SAlexander Shaposhnikov     {"llvm.powi.f80", Intrinsic::powi, makeX86FP80X86FP80I32},
137717106792SAlexander Shaposhnikov     {"llvm.sin.f32", Intrinsic::sin, makeDoubleDouble},
137817106792SAlexander Shaposhnikov     {"llvm.sin.f64", Intrinsic::sin, makeX86FP80X86FP80},
137917106792SAlexander Shaposhnikov     {"llvm.sin.f80", Intrinsic::sin, makeX86FP80X86FP80},
138017106792SAlexander Shaposhnikov     {"llvm.cos.f32", Intrinsic::cos, makeDoubleDouble},
138117106792SAlexander Shaposhnikov     {"llvm.cos.f64", Intrinsic::cos, makeX86FP80X86FP80},
138217106792SAlexander Shaposhnikov     {"llvm.cos.f80", Intrinsic::cos, makeX86FP80X86FP80},
138317106792SAlexander Shaposhnikov     {"llvm.pow.f32", Intrinsic::pow, makeDoubleDoubleDouble},
138417106792SAlexander Shaposhnikov     {"llvm.pow.f64", Intrinsic::pow, makeX86FP80X86FP80X86FP80},
138517106792SAlexander Shaposhnikov     {"llvm.pow.f80", Intrinsic::pow, makeX86FP80X86FP80X86FP80},
138617106792SAlexander Shaposhnikov     {"llvm.exp.f32", Intrinsic::exp, makeDoubleDouble},
138717106792SAlexander Shaposhnikov     {"llvm.exp.f64", Intrinsic::exp, makeX86FP80X86FP80},
138817106792SAlexander Shaposhnikov     {"llvm.exp.f80", Intrinsic::exp, makeX86FP80X86FP80},
138917106792SAlexander Shaposhnikov     {"llvm.exp2.f32", Intrinsic::exp2, makeDoubleDouble},
139017106792SAlexander Shaposhnikov     {"llvm.exp2.f64", Intrinsic::exp2, makeX86FP80X86FP80},
139117106792SAlexander Shaposhnikov     {"llvm.exp2.f80", Intrinsic::exp2, makeX86FP80X86FP80},
139217106792SAlexander Shaposhnikov     {"llvm.log.f32", Intrinsic::log, makeDoubleDouble},
139317106792SAlexander Shaposhnikov     {"llvm.log.f64", Intrinsic::log, makeX86FP80X86FP80},
139417106792SAlexander Shaposhnikov     {"llvm.log.f80", Intrinsic::log, makeX86FP80X86FP80},
139517106792SAlexander Shaposhnikov     {"llvm.log10.f32", Intrinsic::log10, makeDoubleDouble},
139617106792SAlexander Shaposhnikov     {"llvm.log10.f64", Intrinsic::log10, makeX86FP80X86FP80},
139717106792SAlexander Shaposhnikov     {"llvm.log10.f80", Intrinsic::log10, makeX86FP80X86FP80},
139817106792SAlexander Shaposhnikov     {"llvm.log2.f32", Intrinsic::log2, makeDoubleDouble},
139917106792SAlexander Shaposhnikov     {"llvm.log2.f64", Intrinsic::log2, makeX86FP80X86FP80},
140017106792SAlexander Shaposhnikov     {"llvm.log2.f80", Intrinsic::log2, makeX86FP80X86FP80},
140117106792SAlexander Shaposhnikov     {"llvm.fma.f32", Intrinsic::fma, makeDoubleDoubleDoubleDouble},
140217106792SAlexander Shaposhnikov 
140317106792SAlexander Shaposhnikov     {"llvm.fmuladd.f32", Intrinsic::fmuladd, makeDoubleDoubleDoubleDouble},
140417106792SAlexander Shaposhnikov 
140517106792SAlexander Shaposhnikov     {"llvm.fma.f64", Intrinsic::fma, makeX86FP80X86FP80X86FP80X86FP80},
140617106792SAlexander Shaposhnikov 
140717106792SAlexander Shaposhnikov     {"llvm.fmuladd.f64", Intrinsic::fma, makeX86FP80X86FP80X86FP80X86FP80},
140817106792SAlexander Shaposhnikov 
140917106792SAlexander Shaposhnikov     {"llvm.fma.f80", Intrinsic::fma, makeX86FP80X86FP80X86FP80X86FP80},
141017106792SAlexander Shaposhnikov     {"llvm.fabs.f32", Intrinsic::fabs, makeDoubleDouble},
141117106792SAlexander Shaposhnikov     {"llvm.fabs.f64", Intrinsic::fabs, makeX86FP80X86FP80},
141217106792SAlexander Shaposhnikov     {"llvm.fabs.f80", Intrinsic::fabs, makeX86FP80X86FP80},
141317106792SAlexander Shaposhnikov     {"llvm.minnum.f32", Intrinsic::minnum, makeDoubleDoubleDouble},
141417106792SAlexander Shaposhnikov     {"llvm.minnum.f64", Intrinsic::minnum, makeX86FP80X86FP80X86FP80},
141517106792SAlexander Shaposhnikov     {"llvm.minnum.f80", Intrinsic::minnum, makeX86FP80X86FP80X86FP80},
141617106792SAlexander Shaposhnikov     {"llvm.maxnum.f32", Intrinsic::maxnum, makeDoubleDoubleDouble},
141717106792SAlexander Shaposhnikov     {"llvm.maxnum.f64", Intrinsic::maxnum, makeX86FP80X86FP80X86FP80},
141817106792SAlexander Shaposhnikov     {"llvm.maxnum.f80", Intrinsic::maxnum, makeX86FP80X86FP80X86FP80},
141917106792SAlexander Shaposhnikov     {"llvm.minimum.f32", Intrinsic::minimum, makeDoubleDoubleDouble},
142017106792SAlexander Shaposhnikov     {"llvm.minimum.f64", Intrinsic::minimum, makeX86FP80X86FP80X86FP80},
142117106792SAlexander Shaposhnikov     {"llvm.minimum.f80", Intrinsic::minimum, makeX86FP80X86FP80X86FP80},
142217106792SAlexander Shaposhnikov     {"llvm.maximum.f32", Intrinsic::maximum, makeDoubleDoubleDouble},
142317106792SAlexander Shaposhnikov     {"llvm.maximum.f64", Intrinsic::maximum, makeX86FP80X86FP80X86FP80},
142417106792SAlexander Shaposhnikov     {"llvm.maximum.f80", Intrinsic::maximum, makeX86FP80X86FP80X86FP80},
142517106792SAlexander Shaposhnikov     {"llvm.copysign.f32", Intrinsic::copysign, makeDoubleDoubleDouble},
142617106792SAlexander Shaposhnikov     {"llvm.copysign.f64", Intrinsic::copysign, makeX86FP80X86FP80X86FP80},
142717106792SAlexander Shaposhnikov     {"llvm.copysign.f80", Intrinsic::copysign, makeX86FP80X86FP80X86FP80},
142817106792SAlexander Shaposhnikov     {"llvm.floor.f32", Intrinsic::floor, makeDoubleDouble},
142917106792SAlexander Shaposhnikov     {"llvm.floor.f64", Intrinsic::floor, makeX86FP80X86FP80},
143017106792SAlexander Shaposhnikov     {"llvm.floor.f80", Intrinsic::floor, makeX86FP80X86FP80},
143117106792SAlexander Shaposhnikov     {"llvm.ceil.f32", Intrinsic::ceil, makeDoubleDouble},
143217106792SAlexander Shaposhnikov     {"llvm.ceil.f64", Intrinsic::ceil, makeX86FP80X86FP80},
143317106792SAlexander Shaposhnikov     {"llvm.ceil.f80", Intrinsic::ceil, makeX86FP80X86FP80},
143417106792SAlexander Shaposhnikov     {"llvm.trunc.f32", Intrinsic::trunc, makeDoubleDouble},
143517106792SAlexander Shaposhnikov     {"llvm.trunc.f64", Intrinsic::trunc, makeX86FP80X86FP80},
143617106792SAlexander Shaposhnikov     {"llvm.trunc.f80", Intrinsic::trunc, makeX86FP80X86FP80},
143717106792SAlexander Shaposhnikov     {"llvm.rint.f32", Intrinsic::rint, makeDoubleDouble},
143817106792SAlexander Shaposhnikov     {"llvm.rint.f64", Intrinsic::rint, makeX86FP80X86FP80},
143917106792SAlexander Shaposhnikov     {"llvm.rint.f80", Intrinsic::rint, makeX86FP80X86FP80},
144017106792SAlexander Shaposhnikov     {"llvm.nearbyint.f32", Intrinsic::nearbyint, makeDoubleDouble},
144117106792SAlexander Shaposhnikov     {"llvm.nearbyint.f64", Intrinsic::nearbyint, makeX86FP80X86FP80},
144217106792SAlexander Shaposhnikov     {"llvm.nearbyin80f64", Intrinsic::nearbyint, makeX86FP80X86FP80},
144317106792SAlexander Shaposhnikov     {"llvm.round.f32", Intrinsic::round, makeDoubleDouble},
144417106792SAlexander Shaposhnikov     {"llvm.round.f64", Intrinsic::round, makeX86FP80X86FP80},
144517106792SAlexander Shaposhnikov     {"llvm.round.f80", Intrinsic::round, makeX86FP80X86FP80},
144617106792SAlexander Shaposhnikov     {"llvm.lround.f32", Intrinsic::lround, makeDoubleDouble},
144717106792SAlexander Shaposhnikov     {"llvm.lround.f64", Intrinsic::lround, makeX86FP80X86FP80},
144817106792SAlexander Shaposhnikov     {"llvm.lround.f80", Intrinsic::lround, makeX86FP80X86FP80},
144917106792SAlexander Shaposhnikov     {"llvm.llround.f32", Intrinsic::llround, makeDoubleDouble},
145017106792SAlexander Shaposhnikov     {"llvm.llround.f64", Intrinsic::llround, makeX86FP80X86FP80},
145117106792SAlexander Shaposhnikov     {"llvm.llround.f80", Intrinsic::llround, makeX86FP80X86FP80},
145217106792SAlexander Shaposhnikov     {"llvm.lrint.f32", Intrinsic::lrint, makeDoubleDouble},
145317106792SAlexander Shaposhnikov     {"llvm.lrint.f64", Intrinsic::lrint, makeX86FP80X86FP80},
145417106792SAlexander Shaposhnikov     {"llvm.lrint.f80", Intrinsic::lrint, makeX86FP80X86FP80},
145517106792SAlexander Shaposhnikov     {"llvm.llrint.f32", Intrinsic::llrint, makeDoubleDouble},
145617106792SAlexander Shaposhnikov     {"llvm.llrint.f64", Intrinsic::llrint, makeX86FP80X86FP80},
145717106792SAlexander Shaposhnikov     {"llvm.llrint.f80", Intrinsic::llrint, makeX86FP80X86FP80},
145817106792SAlexander Shaposhnikov };
145917106792SAlexander Shaposhnikov 
146017106792SAlexander Shaposhnikov const KnownIntrinsic::LFEntry KnownIntrinsic::kLibfuncIntrinsics[] = {
146117106792SAlexander Shaposhnikov     {LibFunc_sqrtf, "llvm.sqrt.f32"},
146217106792SAlexander Shaposhnikov     {LibFunc_sqrt, "llvm.sqrt.f64"},
146317106792SAlexander Shaposhnikov     {LibFunc_sqrtl, "llvm.sqrt.f80"},
146417106792SAlexander Shaposhnikov     {LibFunc_sinf, "llvm.sin.f32"},
146517106792SAlexander Shaposhnikov     {LibFunc_sin, "llvm.sin.f64"},
146617106792SAlexander Shaposhnikov     {LibFunc_sinl, "llvm.sin.f80"},
146717106792SAlexander Shaposhnikov     {LibFunc_cosf, "llvm.cos.f32"},
146817106792SAlexander Shaposhnikov     {LibFunc_cos, "llvm.cos.f64"},
146917106792SAlexander Shaposhnikov     {LibFunc_cosl, "llvm.cos.f80"},
147017106792SAlexander Shaposhnikov     {LibFunc_powf, "llvm.pow.f32"},
147117106792SAlexander Shaposhnikov     {LibFunc_pow, "llvm.pow.f64"},
147217106792SAlexander Shaposhnikov     {LibFunc_powl, "llvm.pow.f80"},
147317106792SAlexander Shaposhnikov     {LibFunc_expf, "llvm.exp.f32"},
147417106792SAlexander Shaposhnikov     {LibFunc_exp, "llvm.exp.f64"},
147517106792SAlexander Shaposhnikov     {LibFunc_expl, "llvm.exp.f80"},
147617106792SAlexander Shaposhnikov     {LibFunc_exp2f, "llvm.exp2.f32"},
147717106792SAlexander Shaposhnikov     {LibFunc_exp2, "llvm.exp2.f64"},
147817106792SAlexander Shaposhnikov     {LibFunc_exp2l, "llvm.exp2.f80"},
147917106792SAlexander Shaposhnikov     {LibFunc_logf, "llvm.log.f32"},
148017106792SAlexander Shaposhnikov     {LibFunc_log, "llvm.log.f64"},
148117106792SAlexander Shaposhnikov     {LibFunc_logl, "llvm.log.f80"},
148217106792SAlexander Shaposhnikov     {LibFunc_log10f, "llvm.log10.f32"},
148317106792SAlexander Shaposhnikov     {LibFunc_log10, "llvm.log10.f64"},
148417106792SAlexander Shaposhnikov     {LibFunc_log10l, "llvm.log10.f80"},
148517106792SAlexander Shaposhnikov     {LibFunc_log2f, "llvm.log2.f32"},
148617106792SAlexander Shaposhnikov     {LibFunc_log2, "llvm.log2.f64"},
148717106792SAlexander Shaposhnikov     {LibFunc_log2l, "llvm.log2.f80"},
148817106792SAlexander Shaposhnikov     {LibFunc_fabsf, "llvm.fabs.f32"},
148917106792SAlexander Shaposhnikov     {LibFunc_fabs, "llvm.fabs.f64"},
149017106792SAlexander Shaposhnikov     {LibFunc_fabsl, "llvm.fabs.f80"},
149117106792SAlexander Shaposhnikov     {LibFunc_copysignf, "llvm.copysign.f32"},
149217106792SAlexander Shaposhnikov     {LibFunc_copysign, "llvm.copysign.f64"},
149317106792SAlexander Shaposhnikov     {LibFunc_copysignl, "llvm.copysign.f80"},
149417106792SAlexander Shaposhnikov     {LibFunc_floorf, "llvm.floor.f32"},
149517106792SAlexander Shaposhnikov     {LibFunc_floor, "llvm.floor.f64"},
149617106792SAlexander Shaposhnikov     {LibFunc_floorl, "llvm.floor.f80"},
149717106792SAlexander Shaposhnikov     {LibFunc_fmaxf, "llvm.maxnum.f32"},
149817106792SAlexander Shaposhnikov     {LibFunc_fmax, "llvm.maxnum.f64"},
149917106792SAlexander Shaposhnikov     {LibFunc_fmaxl, "llvm.maxnum.f80"},
150017106792SAlexander Shaposhnikov     {LibFunc_fminf, "llvm.minnum.f32"},
150117106792SAlexander Shaposhnikov     {LibFunc_fmin, "llvm.minnum.f64"},
150217106792SAlexander Shaposhnikov     {LibFunc_fminl, "llvm.minnum.f80"},
150317106792SAlexander Shaposhnikov     {LibFunc_ceilf, "llvm.ceil.f32"},
150417106792SAlexander Shaposhnikov     {LibFunc_ceil, "llvm.ceil.f64"},
150517106792SAlexander Shaposhnikov     {LibFunc_ceill, "llvm.ceil.f80"},
150617106792SAlexander Shaposhnikov     {LibFunc_truncf, "llvm.trunc.f32"},
150717106792SAlexander Shaposhnikov     {LibFunc_trunc, "llvm.trunc.f64"},
150817106792SAlexander Shaposhnikov     {LibFunc_truncl, "llvm.trunc.f80"},
150917106792SAlexander Shaposhnikov     {LibFunc_rintf, "llvm.rint.f32"},
151017106792SAlexander Shaposhnikov     {LibFunc_rint, "llvm.rint.f64"},
151117106792SAlexander Shaposhnikov     {LibFunc_rintl, "llvm.rint.f80"},
151217106792SAlexander Shaposhnikov     {LibFunc_nearbyintf, "llvm.nearbyint.f32"},
151317106792SAlexander Shaposhnikov     {LibFunc_nearbyint, "llvm.nearbyint.f64"},
151417106792SAlexander Shaposhnikov     {LibFunc_nearbyintl, "llvm.nearbyint.f80"},
151517106792SAlexander Shaposhnikov     {LibFunc_roundf, "llvm.round.f32"},
151617106792SAlexander Shaposhnikov     {LibFunc_round, "llvm.round.f64"},
151717106792SAlexander Shaposhnikov     {LibFunc_roundl, "llvm.round.f80"},
151817106792SAlexander Shaposhnikov };
151917106792SAlexander Shaposhnikov 
152017106792SAlexander Shaposhnikov const char *KnownIntrinsic::get(LibFunc LFunc) {
152117106792SAlexander Shaposhnikov   for (const auto &E : kLibfuncIntrinsics) {
152217106792SAlexander Shaposhnikov     if (E.LFunc == LFunc)
152317106792SAlexander Shaposhnikov       return E.IntrinsicName;
152417106792SAlexander Shaposhnikov   }
152517106792SAlexander Shaposhnikov   return nullptr;
152617106792SAlexander Shaposhnikov }
152717106792SAlexander Shaposhnikov 
152817106792SAlexander Shaposhnikov const KnownIntrinsic::WidenedIntrinsic *KnownIntrinsic::widen(StringRef Name) {
152917106792SAlexander Shaposhnikov   for (const auto &E : kWidenedIntrinsics) {
153017106792SAlexander Shaposhnikov     if (E.NarrowName == Name)
153117106792SAlexander Shaposhnikov       return &E;
153217106792SAlexander Shaposhnikov   }
153317106792SAlexander Shaposhnikov   return nullptr;
153417106792SAlexander Shaposhnikov }
153517106792SAlexander Shaposhnikov 
153617106792SAlexander Shaposhnikov // Returns the name of the LLVM intrinsic corresponding to the given function.
153717106792SAlexander Shaposhnikov static const char *getIntrinsicFromLibfunc(Function &Fn, Type *VT,
153817106792SAlexander Shaposhnikov                                            const TargetLibraryInfo &TLI) {
153917106792SAlexander Shaposhnikov   LibFunc LFunc;
154017106792SAlexander Shaposhnikov   if (!TLI.getLibFunc(Fn, LFunc))
154117106792SAlexander Shaposhnikov     return nullptr;
154217106792SAlexander Shaposhnikov 
154317106792SAlexander Shaposhnikov   if (const char *Name = KnownIntrinsic::get(LFunc))
154417106792SAlexander Shaposhnikov     return Name;
154517106792SAlexander Shaposhnikov 
154617106792SAlexander Shaposhnikov   LLVM_DEBUG(errs() << "TODO: LibFunc: " << TLI.getName(LFunc) << "\n");
154717106792SAlexander Shaposhnikov   return nullptr;
154817106792SAlexander Shaposhnikov }
154917106792SAlexander Shaposhnikov 
155017106792SAlexander Shaposhnikov // Try to handle a known function call.
155117106792SAlexander Shaposhnikov Value *NumericalStabilitySanitizer::maybeHandleKnownCallBase(
155217106792SAlexander Shaposhnikov     CallBase &Call, Type *VT, Type *ExtendedVT, const TargetLibraryInfo &TLI,
155317106792SAlexander Shaposhnikov     const ValueToShadowMap &Map, IRBuilder<> &Builder) {
155417106792SAlexander Shaposhnikov   Function *Fn = Call.getCalledFunction();
155517106792SAlexander Shaposhnikov   if (Fn == nullptr)
155617106792SAlexander Shaposhnikov     return nullptr;
155717106792SAlexander Shaposhnikov 
155817106792SAlexander Shaposhnikov   Intrinsic::ID WidenedId = Intrinsic::ID();
155917106792SAlexander Shaposhnikov   FunctionType *WidenedFnTy = nullptr;
156017106792SAlexander Shaposhnikov   if (const auto ID = Fn->getIntrinsicID()) {
156117106792SAlexander Shaposhnikov     const auto *Widened = KnownIntrinsic::widen(Fn->getName());
156217106792SAlexander Shaposhnikov     if (Widened) {
156317106792SAlexander Shaposhnikov       WidenedId = Widened->ID;
156417106792SAlexander Shaposhnikov       WidenedFnTy = Widened->MakeFnTy(Context);
156517106792SAlexander Shaposhnikov     } else {
156617106792SAlexander Shaposhnikov       // If we don't know how to widen the intrinsic, we have no choice but to
156717106792SAlexander Shaposhnikov       // call the non-wide version on a truncated shadow and extend again
156817106792SAlexander Shaposhnikov       // afterwards.
156917106792SAlexander Shaposhnikov       WidenedId = ID;
157017106792SAlexander Shaposhnikov       WidenedFnTy = Fn->getFunctionType();
157117106792SAlexander Shaposhnikov     }
157217106792SAlexander Shaposhnikov   } else if (const char *Name = getIntrinsicFromLibfunc(*Fn, VT, TLI)) {
157317106792SAlexander Shaposhnikov     // We might have a call to a library function that we can replace with a
157417106792SAlexander Shaposhnikov     // wider Intrinsic.
157517106792SAlexander Shaposhnikov     const auto *Widened = KnownIntrinsic::widen(Name);
157617106792SAlexander Shaposhnikov     assert(Widened && "make sure KnownIntrinsic entries are consistent");
157717106792SAlexander Shaposhnikov     WidenedId = Widened->ID;
157817106792SAlexander Shaposhnikov     WidenedFnTy = Widened->MakeFnTy(Context);
157917106792SAlexander Shaposhnikov   } else {
158017106792SAlexander Shaposhnikov     // This is not a known library function or intrinsic.
158117106792SAlexander Shaposhnikov     return nullptr;
158217106792SAlexander Shaposhnikov   }
158317106792SAlexander Shaposhnikov 
158417106792SAlexander Shaposhnikov   // Check that the widened intrinsic is valid.
158517106792SAlexander Shaposhnikov   SmallVector<Intrinsic::IITDescriptor, 8> Table;
158617106792SAlexander Shaposhnikov   getIntrinsicInfoTableEntries(WidenedId, Table);
158717106792SAlexander Shaposhnikov   SmallVector<Type *, 4> ArgTys;
158817106792SAlexander Shaposhnikov   ArrayRef<Intrinsic::IITDescriptor> TableRef = Table;
158917106792SAlexander Shaposhnikov   [[maybe_unused]] Intrinsic::MatchIntrinsicTypesResult MatchResult =
159017106792SAlexander Shaposhnikov       Intrinsic::matchIntrinsicSignature(WidenedFnTy, TableRef, ArgTys);
159117106792SAlexander Shaposhnikov   assert(MatchResult == Intrinsic::MatchIntrinsicTypes_Match &&
159217106792SAlexander Shaposhnikov          "invalid widened intrinsic");
159317106792SAlexander Shaposhnikov   // For known intrinsic functions, we create a second call to the same
159417106792SAlexander Shaposhnikov   // intrinsic with a different type.
159517106792SAlexander Shaposhnikov   SmallVector<Value *, 4> Args;
159617106792SAlexander Shaposhnikov   // The last operand is the intrinsic itself, skip it.
159717106792SAlexander Shaposhnikov   for (unsigned I = 0, E = Call.getNumOperands() - 1; I < E; ++I) {
159817106792SAlexander Shaposhnikov     Value *Arg = Call.getOperand(I);
159917106792SAlexander Shaposhnikov     Type *OrigArgTy = Arg->getType();
160017106792SAlexander Shaposhnikov     Type *IntrinsicArgTy = WidenedFnTy->getParamType(I);
160117106792SAlexander Shaposhnikov     if (OrigArgTy == IntrinsicArgTy) {
160217106792SAlexander Shaposhnikov       Args.push_back(Arg); // The arg is passed as is.
160317106792SAlexander Shaposhnikov       continue;
160417106792SAlexander Shaposhnikov     }
160517106792SAlexander Shaposhnikov     Type *ShadowArgTy = Config.getExtendedFPType(Arg->getType());
160617106792SAlexander Shaposhnikov     assert(ShadowArgTy &&
160717106792SAlexander Shaposhnikov            "don't know how to get the shadow value for a non-FT");
160817106792SAlexander Shaposhnikov     Value *Shadow = Map.getShadow(Arg);
160917106792SAlexander Shaposhnikov     if (ShadowArgTy == IntrinsicArgTy) {
161017106792SAlexander Shaposhnikov       // The shadow is the right type for the intrinsic.
161117106792SAlexander Shaposhnikov       assert(Shadow->getType() == ShadowArgTy);
161217106792SAlexander Shaposhnikov       Args.push_back(Shadow);
161317106792SAlexander Shaposhnikov       continue;
161417106792SAlexander Shaposhnikov     }
161517106792SAlexander Shaposhnikov     // There is no intrinsic with his level of precision, truncate the shadow.
161617106792SAlexander Shaposhnikov     Args.push_back(Builder.CreateFPTrunc(Shadow, IntrinsicArgTy));
161717106792SAlexander Shaposhnikov   }
161817106792SAlexander Shaposhnikov   Value *IntrinsicCall = Builder.CreateIntrinsic(WidenedId, ArgTys, Args);
161917106792SAlexander Shaposhnikov   return WidenedFnTy->getReturnType() == ExtendedVT
162017106792SAlexander Shaposhnikov              ? IntrinsicCall
162117106792SAlexander Shaposhnikov              : Builder.CreateFPExt(IntrinsicCall, ExtendedVT);
162217106792SAlexander Shaposhnikov }
162317106792SAlexander Shaposhnikov 
162417106792SAlexander Shaposhnikov // Handle a CallBase, i.e. a function call, an inline asm sequence, or an
162517106792SAlexander Shaposhnikov // invoke.
162617106792SAlexander Shaposhnikov Value *NumericalStabilitySanitizer::handleCallBase(CallBase &Call, Type *VT,
162717106792SAlexander Shaposhnikov                                                    Type *ExtendedVT,
162817106792SAlexander Shaposhnikov                                                    const TargetLibraryInfo &TLI,
162917106792SAlexander Shaposhnikov                                                    const ValueToShadowMap &Map,
163017106792SAlexander Shaposhnikov                                                    IRBuilder<> &Builder) {
163117106792SAlexander Shaposhnikov   // We cannot look inside inline asm, just expand the result again.
163217106792SAlexander Shaposhnikov   if (Call.isInlineAsm())
163317106792SAlexander Shaposhnikov     return Builder.CreateFPExt(&Call, ExtendedVT);
163417106792SAlexander Shaposhnikov 
163517106792SAlexander Shaposhnikov   // Intrinsics and library functions (e.g. sin, exp) are handled
163617106792SAlexander Shaposhnikov   // specifically, because we know their semantics and can do better than
163717106792SAlexander Shaposhnikov   // blindly calling them (e.g. compute the sinus in the actual shadow domain).
163817106792SAlexander Shaposhnikov   if (Value *V =
163917106792SAlexander Shaposhnikov           maybeHandleKnownCallBase(Call, VT, ExtendedVT, TLI, Map, Builder))
164017106792SAlexander Shaposhnikov     return V;
164117106792SAlexander Shaposhnikov 
164217106792SAlexander Shaposhnikov   // If the return tag matches that of the called function, read the extended
164317106792SAlexander Shaposhnikov   // return value from the shadow ret ptr. Else, just extend the return value.
164417106792SAlexander Shaposhnikov   Value *L =
164517106792SAlexander Shaposhnikov       Builder.CreateLoad(IntptrTy, NsanShadowRetTag, /*isVolatile=*/false);
164617106792SAlexander Shaposhnikov   Value *HasShadowRet = Builder.CreateICmpEQ(
164717106792SAlexander Shaposhnikov       L, Builder.CreatePtrToInt(Call.getCalledOperand(), IntptrTy));
164817106792SAlexander Shaposhnikov 
164917106792SAlexander Shaposhnikov   Value *ShadowRetVal = Builder.CreateLoad(
165017106792SAlexander Shaposhnikov       ExtendedVT,
165117106792SAlexander Shaposhnikov       Builder.CreateConstGEP2_64(NsanShadowRetType, NsanShadowRetPtr, 0, 0),
165217106792SAlexander Shaposhnikov       /*isVolatile=*/false);
165317106792SAlexander Shaposhnikov   Value *Shadow = Builder.CreateSelect(HasShadowRet, ShadowRetVal,
165417106792SAlexander Shaposhnikov                                        Builder.CreateFPExt(&Call, ExtendedVT));
165517106792SAlexander Shaposhnikov   ++NumInstrumentedFTCalls;
165617106792SAlexander Shaposhnikov   return Shadow;
165717106792SAlexander Shaposhnikov }
165817106792SAlexander Shaposhnikov 
165917106792SAlexander Shaposhnikov // Creates a shadow value for the given FT value. At that point all operands are
166017106792SAlexander Shaposhnikov // guaranteed to be available.
166117106792SAlexander Shaposhnikov Value *NumericalStabilitySanitizer::createShadowValueWithOperandsAvailable(
166217106792SAlexander Shaposhnikov     Instruction &Inst, const TargetLibraryInfo &TLI,
166317106792SAlexander Shaposhnikov     const ValueToShadowMap &Map) {
166417106792SAlexander Shaposhnikov   Type *VT = Inst.getType();
166517106792SAlexander Shaposhnikov   Type *ExtendedVT = Config.getExtendedFPType(VT);
166617106792SAlexander Shaposhnikov   assert(ExtendedVT != nullptr && "trying to create a shadow for a non-FT");
166717106792SAlexander Shaposhnikov 
166817106792SAlexander Shaposhnikov   if (auto *Load = dyn_cast<LoadInst>(&Inst))
166917106792SAlexander Shaposhnikov     return handleLoad(*Load, VT, ExtendedVT);
167017106792SAlexander Shaposhnikov 
167117106792SAlexander Shaposhnikov   if (auto *Call = dyn_cast<CallInst>(&Inst)) {
167217106792SAlexander Shaposhnikov     // Insert after the call.
167317106792SAlexander Shaposhnikov     BasicBlock::iterator It(Inst);
167417106792SAlexander Shaposhnikov     IRBuilder<> Builder(Call->getParent(), ++It);
167517106792SAlexander Shaposhnikov     Builder.SetCurrentDebugLocation(Call->getDebugLoc());
167617106792SAlexander Shaposhnikov     return handleCallBase(*Call, VT, ExtendedVT, TLI, Map, Builder);
167717106792SAlexander Shaposhnikov   }
167817106792SAlexander Shaposhnikov 
167917106792SAlexander Shaposhnikov   if (auto *Invoke = dyn_cast<InvokeInst>(&Inst)) {
168017106792SAlexander Shaposhnikov     // The Invoke terminates the basic block, create a new basic block in
168117106792SAlexander Shaposhnikov     // between the successful invoke and the next block.
168217106792SAlexander Shaposhnikov     BasicBlock *InvokeBB = Invoke->getParent();
168317106792SAlexander Shaposhnikov     BasicBlock *NextBB = Invoke->getNormalDest();
168417106792SAlexander Shaposhnikov     BasicBlock *NewBB =
168517106792SAlexander Shaposhnikov         BasicBlock::Create(Context, "", NextBB->getParent(), NextBB);
168617106792SAlexander Shaposhnikov     Inst.replaceSuccessorWith(NextBB, NewBB);
168717106792SAlexander Shaposhnikov 
168817106792SAlexander Shaposhnikov     IRBuilder<> Builder(NewBB);
168917106792SAlexander Shaposhnikov     Builder.SetCurrentDebugLocation(Invoke->getDebugLoc());
169017106792SAlexander Shaposhnikov     Value *Shadow = handleCallBase(*Invoke, VT, ExtendedVT, TLI, Map, Builder);
169117106792SAlexander Shaposhnikov     Builder.CreateBr(NextBB);
169217106792SAlexander Shaposhnikov     NewBB->replaceSuccessorsPhiUsesWith(InvokeBB, NewBB);
169317106792SAlexander Shaposhnikov     return Shadow;
169417106792SAlexander Shaposhnikov   }
169517106792SAlexander Shaposhnikov 
169617106792SAlexander Shaposhnikov   IRBuilder<> Builder(Inst.getNextNode());
169717106792SAlexander Shaposhnikov   Builder.SetCurrentDebugLocation(Inst.getDebugLoc());
169817106792SAlexander Shaposhnikov 
169917106792SAlexander Shaposhnikov   if (auto *Trunc = dyn_cast<FPTruncInst>(&Inst))
170017106792SAlexander Shaposhnikov     return handleTrunc(*Trunc, VT, ExtendedVT, Map, Builder);
170117106792SAlexander Shaposhnikov   if (auto *Ext = dyn_cast<FPExtInst>(&Inst))
170217106792SAlexander Shaposhnikov     return handleExt(*Ext, VT, ExtendedVT, Map, Builder);
170317106792SAlexander Shaposhnikov 
170417106792SAlexander Shaposhnikov   if (auto *UnaryOp = dyn_cast<UnaryOperator>(&Inst))
170517106792SAlexander Shaposhnikov     return Builder.CreateUnOp(UnaryOp->getOpcode(),
170617106792SAlexander Shaposhnikov                               Map.getShadow(UnaryOp->getOperand(0)));
170717106792SAlexander Shaposhnikov 
170817106792SAlexander Shaposhnikov   if (auto *BinOp = dyn_cast<BinaryOperator>(&Inst))
170917106792SAlexander Shaposhnikov     return Builder.CreateBinOp(BinOp->getOpcode(),
171017106792SAlexander Shaposhnikov                                Map.getShadow(BinOp->getOperand(0)),
171117106792SAlexander Shaposhnikov                                Map.getShadow(BinOp->getOperand(1)));
171217106792SAlexander Shaposhnikov 
171317106792SAlexander Shaposhnikov   if (isa<UIToFPInst>(&Inst) || isa<SIToFPInst>(&Inst)) {
1714430b90f0SWu Yingcong     auto *Cast = cast<CastInst>(&Inst);
171517106792SAlexander Shaposhnikov     return Builder.CreateCast(Cast->getOpcode(), Cast->getOperand(0),
171617106792SAlexander Shaposhnikov                               ExtendedVT);
171717106792SAlexander Shaposhnikov   }
171817106792SAlexander Shaposhnikov 
171917106792SAlexander Shaposhnikov   if (auto *S = dyn_cast<SelectInst>(&Inst))
172017106792SAlexander Shaposhnikov     return Builder.CreateSelect(S->getCondition(),
172117106792SAlexander Shaposhnikov                                 Map.getShadow(S->getTrueValue()),
172217106792SAlexander Shaposhnikov                                 Map.getShadow(S->getFalseValue()));
172317106792SAlexander Shaposhnikov 
172470a9535fSAlexander Shaposhnikov   if (auto *Freeze = dyn_cast<FreezeInst>(&Inst))
172570a9535fSAlexander Shaposhnikov     return Builder.CreateFreeze(Map.getShadow(Freeze->getOperand(0)));
172670a9535fSAlexander Shaposhnikov 
172717106792SAlexander Shaposhnikov   if (auto *Extract = dyn_cast<ExtractElementInst>(&Inst))
172817106792SAlexander Shaposhnikov     return Builder.CreateExtractElement(
172917106792SAlexander Shaposhnikov         Map.getShadow(Extract->getVectorOperand()), Extract->getIndexOperand());
173017106792SAlexander Shaposhnikov 
173117106792SAlexander Shaposhnikov   if (auto *Insert = dyn_cast<InsertElementInst>(&Inst))
173217106792SAlexander Shaposhnikov     return Builder.CreateInsertElement(Map.getShadow(Insert->getOperand(0)),
173317106792SAlexander Shaposhnikov                                        Map.getShadow(Insert->getOperand(1)),
173417106792SAlexander Shaposhnikov                                        Insert->getOperand(2));
173517106792SAlexander Shaposhnikov 
173617106792SAlexander Shaposhnikov   if (auto *Shuffle = dyn_cast<ShuffleVectorInst>(&Inst))
173717106792SAlexander Shaposhnikov     return Builder.CreateShuffleVector(Map.getShadow(Shuffle->getOperand(0)),
173817106792SAlexander Shaposhnikov                                        Map.getShadow(Shuffle->getOperand(1)),
173917106792SAlexander Shaposhnikov                                        Shuffle->getShuffleMask());
174017106792SAlexander Shaposhnikov   // TODO: We could make aggregate object first class citizens. For now we
174117106792SAlexander Shaposhnikov   // just extend the extracted value.
174217106792SAlexander Shaposhnikov   if (auto *Extract = dyn_cast<ExtractValueInst>(&Inst))
174317106792SAlexander Shaposhnikov     return Builder.CreateFPExt(Extract, ExtendedVT);
174417106792SAlexander Shaposhnikov 
174517106792SAlexander Shaposhnikov   if (auto *BC = dyn_cast<BitCastInst>(&Inst))
174617106792SAlexander Shaposhnikov     return Builder.CreateFPExt(BC, ExtendedVT);
174717106792SAlexander Shaposhnikov 
174817106792SAlexander Shaposhnikov   report_fatal_error("Unimplemented support for " +
174917106792SAlexander Shaposhnikov                      Twine(Inst.getOpcodeName()));
175017106792SAlexander Shaposhnikov }
175117106792SAlexander Shaposhnikov 
175217106792SAlexander Shaposhnikov // Creates a shadow value for an instruction that defines a value of FT type.
175317106792SAlexander Shaposhnikov // FT operands that do not already have shadow values are created recursively.
175417106792SAlexander Shaposhnikov // The DFS is guaranteed to not loop as phis and arguments already have
175517106792SAlexander Shaposhnikov // shadows.
175617106792SAlexander Shaposhnikov void NumericalStabilitySanitizer::maybeCreateShadowValue(
175717106792SAlexander Shaposhnikov     Instruction &Root, const TargetLibraryInfo &TLI, ValueToShadowMap &Map) {
175817106792SAlexander Shaposhnikov   Type *VT = Root.getType();
175917106792SAlexander Shaposhnikov   Type *ExtendedVT = Config.getExtendedFPType(VT);
176017106792SAlexander Shaposhnikov   if (ExtendedVT == nullptr)
176117106792SAlexander Shaposhnikov     return; // Not an FT value.
176217106792SAlexander Shaposhnikov 
176317106792SAlexander Shaposhnikov   if (Map.hasShadow(&Root))
176417106792SAlexander Shaposhnikov     return; // Shadow already exists.
176517106792SAlexander Shaposhnikov 
176617106792SAlexander Shaposhnikov   assert(!isa<PHINode>(Root) && "phi nodes should already have shadows");
176717106792SAlexander Shaposhnikov 
176817106792SAlexander Shaposhnikov   std::vector<Instruction *> DfsStack(1, &Root);
176917106792SAlexander Shaposhnikov   while (!DfsStack.empty()) {
177017106792SAlexander Shaposhnikov     // Ensure that all operands to the instruction have shadows before
177117106792SAlexander Shaposhnikov     // proceeding.
177217106792SAlexander Shaposhnikov     Instruction *I = DfsStack.back();
177317106792SAlexander Shaposhnikov     // The shadow for the instruction might have been created deeper in the DFS,
177417106792SAlexander Shaposhnikov     // see `forward_use_with_two_uses` test.
177517106792SAlexander Shaposhnikov     if (Map.hasShadow(I)) {
177617106792SAlexander Shaposhnikov       DfsStack.pop_back();
177717106792SAlexander Shaposhnikov       continue;
177817106792SAlexander Shaposhnikov     }
177917106792SAlexander Shaposhnikov 
178017106792SAlexander Shaposhnikov     bool MissingShadow = false;
178117106792SAlexander Shaposhnikov     for (Value *Op : I->operands()) {
178217106792SAlexander Shaposhnikov       Type *VT = Op->getType();
178317106792SAlexander Shaposhnikov       if (!Config.getExtendedFPType(VT))
178417106792SAlexander Shaposhnikov         continue; // Not an FT value.
178517106792SAlexander Shaposhnikov       if (Map.hasShadow(Op))
178617106792SAlexander Shaposhnikov         continue; // Shadow is already available.
178717106792SAlexander Shaposhnikov       MissingShadow = true;
178817106792SAlexander Shaposhnikov       DfsStack.push_back(cast<Instruction>(Op));
178917106792SAlexander Shaposhnikov     }
179017106792SAlexander Shaposhnikov     if (MissingShadow)
179117106792SAlexander Shaposhnikov       continue; // Process operands and come back to this instruction later.
179217106792SAlexander Shaposhnikov 
179317106792SAlexander Shaposhnikov     // All operands have shadows. Create a shadow for the current value.
179417106792SAlexander Shaposhnikov     Value *Shadow = createShadowValueWithOperandsAvailable(*I, TLI, Map);
179517106792SAlexander Shaposhnikov     Map.setShadow(*I, *Shadow);
179617106792SAlexander Shaposhnikov     DfsStack.pop_back();
179717106792SAlexander Shaposhnikov   }
179817106792SAlexander Shaposhnikov }
179917106792SAlexander Shaposhnikov 
180017106792SAlexander Shaposhnikov // A floating-point store needs its value and type written to shadow memory.
180117106792SAlexander Shaposhnikov void NumericalStabilitySanitizer::propagateFTStore(
180217106792SAlexander Shaposhnikov     StoreInst &Store, Type *VT, Type *ExtendedVT, const ValueToShadowMap &Map) {
180317106792SAlexander Shaposhnikov   Value *StoredValue = Store.getValueOperand();
180417106792SAlexander Shaposhnikov   IRBuilder<> Builder(&Store);
180517106792SAlexander Shaposhnikov   Builder.SetCurrentDebugLocation(Store.getDebugLoc());
180617106792SAlexander Shaposhnikov   const auto Extents = getMemoryExtentsOrDie(VT);
180717106792SAlexander Shaposhnikov   Value *ShadowPtr = Builder.CreateCall(
180817106792SAlexander Shaposhnikov       NsanGetShadowPtrForStore[Extents.ValueType],
180917106792SAlexander Shaposhnikov       {Store.getPointerOperand(), ConstantInt::get(IntptrTy, Extents.NumElts)});
181017106792SAlexander Shaposhnikov 
181117106792SAlexander Shaposhnikov   Value *StoredShadow = Map.getShadow(StoredValue);
181217106792SAlexander Shaposhnikov   if (!Store.getParent()->getParent()->hasOptNone()) {
181317106792SAlexander Shaposhnikov     // Only check stores when optimizing, because non-optimized code generates
181417106792SAlexander Shaposhnikov     // too many stores to the stack, creating false positives.
181517106792SAlexander Shaposhnikov     if (ClCheckStores) {
181617106792SAlexander Shaposhnikov       StoredShadow = emitCheck(StoredValue, StoredShadow, Builder,
181717106792SAlexander Shaposhnikov                                CheckLoc::makeStore(Store.getPointerOperand()));
181817106792SAlexander Shaposhnikov       ++NumInstrumentedFTStores;
181917106792SAlexander Shaposhnikov     }
182017106792SAlexander Shaposhnikov   }
182117106792SAlexander Shaposhnikov 
182217106792SAlexander Shaposhnikov   Builder.CreateAlignedStore(StoredShadow, ShadowPtr, Align(1),
182317106792SAlexander Shaposhnikov                              Store.isVolatile());
182417106792SAlexander Shaposhnikov }
182517106792SAlexander Shaposhnikov 
182617106792SAlexander Shaposhnikov // A non-ft store needs to invalidate shadow memory. Exceptions are:
182717106792SAlexander Shaposhnikov //   - memory transfers of floating-point data through other pointer types (llvm
182817106792SAlexander Shaposhnikov //     optimization passes transform `*(float*)a = *(float*)b` into
182917106792SAlexander Shaposhnikov //     `*(i32*)a = *(i32*)b` ). These have the same semantics as memcpy.
183017106792SAlexander Shaposhnikov //   - Writes of FT-sized constants. LLVM likes to do float stores as bitcasted
183117106792SAlexander Shaposhnikov //     ints. Note that this is not really necessary because if the value is
183217106792SAlexander Shaposhnikov //     unknown the framework will re-extend it on load anyway. It just felt
183317106792SAlexander Shaposhnikov //     easier to debug tests with vectors of FTs.
183417106792SAlexander Shaposhnikov void NumericalStabilitySanitizer::propagateNonFTStore(
183517106792SAlexander Shaposhnikov     StoreInst &Store, Type *VT, const ValueToShadowMap &Map) {
183617106792SAlexander Shaposhnikov   Value *PtrOp = Store.getPointerOperand();
183717106792SAlexander Shaposhnikov   IRBuilder<> Builder(Store.getNextNode());
183817106792SAlexander Shaposhnikov   Builder.SetCurrentDebugLocation(Store.getDebugLoc());
183917106792SAlexander Shaposhnikov   Value *Dst = PtrOp;
184017106792SAlexander Shaposhnikov   TypeSize SlotSize = DL.getTypeStoreSize(VT);
184117106792SAlexander Shaposhnikov   assert(!SlotSize.isScalable() && "unsupported");
184217106792SAlexander Shaposhnikov   const auto LoadSizeBytes = SlotSize.getFixedValue();
184317106792SAlexander Shaposhnikov   Value *ValueSize = Constant::getIntegerValue(
184417106792SAlexander Shaposhnikov       IntptrTy, APInt(IntptrTy->getPrimitiveSizeInBits(), LoadSizeBytes));
184517106792SAlexander Shaposhnikov 
184617106792SAlexander Shaposhnikov   ++NumInstrumentedNonFTStores;
184717106792SAlexander Shaposhnikov   Value *StoredValue = Store.getValueOperand();
184817106792SAlexander Shaposhnikov   if (LoadInst *Load = dyn_cast<LoadInst>(StoredValue)) {
184917106792SAlexander Shaposhnikov     // TODO: Handle the case when the value is from a phi.
185017106792SAlexander Shaposhnikov     // This is a memory transfer with memcpy semantics. Copy the type and
185117106792SAlexander Shaposhnikov     // value from the source. Note that we cannot use __nsan_copy_values()
185217106792SAlexander Shaposhnikov     // here, because that will not work when there is a write to memory in
185317106792SAlexander Shaposhnikov     // between the load and the store, e.g. in the case of a swap.
185417106792SAlexander Shaposhnikov     Type *ShadowTypeIntTy = Type::getIntNTy(Context, 8 * LoadSizeBytes);
185517106792SAlexander Shaposhnikov     Type *ShadowValueIntTy =
185617106792SAlexander Shaposhnikov         Type::getIntNTy(Context, 8 * kShadowScale * LoadSizeBytes);
185717106792SAlexander Shaposhnikov     IRBuilder<> LoadBuilder(Load->getNextNode());
185817106792SAlexander Shaposhnikov     Builder.SetCurrentDebugLocation(Store.getDebugLoc());
185917106792SAlexander Shaposhnikov     Value *LoadSrc = Load->getPointerOperand();
186017106792SAlexander Shaposhnikov     // Read the shadow type and value at load time. The type has the same size
186117106792SAlexander Shaposhnikov     // as the FT value, the value has twice its size.
186217106792SAlexander Shaposhnikov     // TODO: cache them to avoid re-creating them when a load is used by
186317106792SAlexander Shaposhnikov     // several stores. Maybe create them like the FT shadows when a load is
186417106792SAlexander Shaposhnikov     // encountered.
186517106792SAlexander Shaposhnikov     Value *RawShadowType = LoadBuilder.CreateAlignedLoad(
186617106792SAlexander Shaposhnikov         ShadowTypeIntTy,
186717106792SAlexander Shaposhnikov         LoadBuilder.CreateCall(NsanGetRawShadowTypePtr, {LoadSrc}), Align(1),
186817106792SAlexander Shaposhnikov         /*isVolatile=*/false);
186917106792SAlexander Shaposhnikov     Value *RawShadowValue = LoadBuilder.CreateAlignedLoad(
187017106792SAlexander Shaposhnikov         ShadowValueIntTy,
187117106792SAlexander Shaposhnikov         LoadBuilder.CreateCall(NsanGetRawShadowPtr, {LoadSrc}), Align(1),
187217106792SAlexander Shaposhnikov         /*isVolatile=*/false);
187317106792SAlexander Shaposhnikov 
187417106792SAlexander Shaposhnikov     // Write back the shadow type and value at store time.
187517106792SAlexander Shaposhnikov     Builder.CreateAlignedStore(
187617106792SAlexander Shaposhnikov         RawShadowType, Builder.CreateCall(NsanGetRawShadowTypePtr, {Dst}),
187717106792SAlexander Shaposhnikov         Align(1),
187817106792SAlexander Shaposhnikov         /*isVolatile=*/false);
187917106792SAlexander Shaposhnikov     Builder.CreateAlignedStore(RawShadowValue,
188017106792SAlexander Shaposhnikov                                Builder.CreateCall(NsanGetRawShadowPtr, {Dst}),
188117106792SAlexander Shaposhnikov                                Align(1),
188217106792SAlexander Shaposhnikov                                /*isVolatile=*/false);
188317106792SAlexander Shaposhnikov 
188417106792SAlexander Shaposhnikov     ++NumInstrumentedNonFTMemcpyStores;
188517106792SAlexander Shaposhnikov     return;
188617106792SAlexander Shaposhnikov   }
188717106792SAlexander Shaposhnikov   // ClPropagateNonFTConstStoresAsFT is by default false.
188817106792SAlexander Shaposhnikov   if (Constant *C; ClPropagateNonFTConstStoresAsFT &&
188917106792SAlexander Shaposhnikov                    (C = dyn_cast<Constant>(StoredValue))) {
189017106792SAlexander Shaposhnikov     // This might be a fp constant stored as an int. Bitcast and store if it has
189117106792SAlexander Shaposhnikov     // appropriate size.
189217106792SAlexander Shaposhnikov     Type *BitcastTy = nullptr; // The FT type to bitcast to.
189317106792SAlexander Shaposhnikov     if (auto *CInt = dyn_cast<ConstantInt>(C)) {
189417106792SAlexander Shaposhnikov       switch (CInt->getType()->getScalarSizeInBits()) {
189517106792SAlexander Shaposhnikov       case 32:
189617106792SAlexander Shaposhnikov         BitcastTy = Type::getFloatTy(Context);
189717106792SAlexander Shaposhnikov         break;
189817106792SAlexander Shaposhnikov       case 64:
189917106792SAlexander Shaposhnikov         BitcastTy = Type::getDoubleTy(Context);
190017106792SAlexander Shaposhnikov         break;
190117106792SAlexander Shaposhnikov       case 80:
190217106792SAlexander Shaposhnikov         BitcastTy = Type::getX86_FP80Ty(Context);
190317106792SAlexander Shaposhnikov         break;
190417106792SAlexander Shaposhnikov       default:
190517106792SAlexander Shaposhnikov         break;
190617106792SAlexander Shaposhnikov       }
190717106792SAlexander Shaposhnikov     } else if (auto *CDV = dyn_cast<ConstantDataVector>(C)) {
190817106792SAlexander Shaposhnikov       const int NumElements =
190917106792SAlexander Shaposhnikov           cast<VectorType>(CDV->getType())->getElementCount().getFixedValue();
191017106792SAlexander Shaposhnikov       switch (CDV->getType()->getScalarSizeInBits()) {
191117106792SAlexander Shaposhnikov       case 32:
191217106792SAlexander Shaposhnikov         BitcastTy =
191317106792SAlexander Shaposhnikov             VectorType::get(Type::getFloatTy(Context), NumElements, false);
191417106792SAlexander Shaposhnikov         break;
191517106792SAlexander Shaposhnikov       case 64:
191617106792SAlexander Shaposhnikov         BitcastTy =
191717106792SAlexander Shaposhnikov             VectorType::get(Type::getDoubleTy(Context), NumElements, false);
191817106792SAlexander Shaposhnikov         break;
191917106792SAlexander Shaposhnikov       case 80:
192017106792SAlexander Shaposhnikov         BitcastTy =
192117106792SAlexander Shaposhnikov             VectorType::get(Type::getX86_FP80Ty(Context), NumElements, false);
192217106792SAlexander Shaposhnikov         break;
192317106792SAlexander Shaposhnikov       default:
192417106792SAlexander Shaposhnikov         break;
192517106792SAlexander Shaposhnikov       }
192617106792SAlexander Shaposhnikov     }
192717106792SAlexander Shaposhnikov     if (BitcastTy) {
192817106792SAlexander Shaposhnikov       const MemoryExtents Extents = getMemoryExtentsOrDie(BitcastTy);
192917106792SAlexander Shaposhnikov       Value *ShadowPtr = Builder.CreateCall(
193017106792SAlexander Shaposhnikov           NsanGetShadowPtrForStore[Extents.ValueType],
193117106792SAlexander Shaposhnikov           {PtrOp, ConstantInt::get(IntptrTy, Extents.NumElts)});
193217106792SAlexander Shaposhnikov       // Bitcast the integer value to the appropriate FT type and extend to 2FT.
193317106792SAlexander Shaposhnikov       Type *ExtVT = Config.getExtendedFPType(BitcastTy);
193417106792SAlexander Shaposhnikov       Value *Shadow =
193517106792SAlexander Shaposhnikov           Builder.CreateFPExt(Builder.CreateBitCast(C, BitcastTy), ExtVT);
193617106792SAlexander Shaposhnikov       Builder.CreateAlignedStore(Shadow, ShadowPtr, Align(1),
193717106792SAlexander Shaposhnikov                                  Store.isVolatile());
193817106792SAlexander Shaposhnikov       return;
193917106792SAlexander Shaposhnikov     }
194017106792SAlexander Shaposhnikov   }
194117106792SAlexander Shaposhnikov   // All other stores just reset the shadow value to unknown.
1942ddf5725eSDmitry Chestnykh   Builder.CreateCall(NsanSetUnknownFns.getFallback(), {Dst, ValueSize});
194317106792SAlexander Shaposhnikov }
194417106792SAlexander Shaposhnikov 
194517106792SAlexander Shaposhnikov void NumericalStabilitySanitizer::propagateShadowValues(
194617106792SAlexander Shaposhnikov     Instruction &Inst, const TargetLibraryInfo &TLI,
194717106792SAlexander Shaposhnikov     const ValueToShadowMap &Map) {
194817106792SAlexander Shaposhnikov   if (auto *Store = dyn_cast<StoreInst>(&Inst)) {
194917106792SAlexander Shaposhnikov     Value *StoredValue = Store->getValueOperand();
195017106792SAlexander Shaposhnikov     Type *VT = StoredValue->getType();
195117106792SAlexander Shaposhnikov     Type *ExtendedVT = Config.getExtendedFPType(VT);
195217106792SAlexander Shaposhnikov     if (ExtendedVT == nullptr)
195317106792SAlexander Shaposhnikov       return propagateNonFTStore(*Store, VT, Map);
195417106792SAlexander Shaposhnikov     return propagateFTStore(*Store, VT, ExtendedVT, Map);
195517106792SAlexander Shaposhnikov   }
195617106792SAlexander Shaposhnikov 
195717106792SAlexander Shaposhnikov   if (auto *FCmp = dyn_cast<FCmpInst>(&Inst)) {
195817106792SAlexander Shaposhnikov     emitFCmpCheck(*FCmp, Map);
195917106792SAlexander Shaposhnikov     return;
196017106792SAlexander Shaposhnikov   }
196117106792SAlexander Shaposhnikov 
196217106792SAlexander Shaposhnikov   if (auto *CB = dyn_cast<CallBase>(&Inst)) {
196317106792SAlexander Shaposhnikov     maybeAddSuffixForNsanInterface(CB);
196417106792SAlexander Shaposhnikov     if (CallInst *CI = dyn_cast<CallInst>(&Inst))
196517106792SAlexander Shaposhnikov       maybeMarkSanitizerLibraryCallNoBuiltin(CI, &TLI);
196617106792SAlexander Shaposhnikov     if (MemIntrinsic *MI = dyn_cast<MemIntrinsic>(&Inst)) {
196717106792SAlexander Shaposhnikov       instrumentMemIntrinsic(MI);
196817106792SAlexander Shaposhnikov       return;
196917106792SAlexander Shaposhnikov     }
197017106792SAlexander Shaposhnikov     populateShadowStack(*CB, TLI, Map);
197117106792SAlexander Shaposhnikov     return;
197217106792SAlexander Shaposhnikov   }
197317106792SAlexander Shaposhnikov 
197417106792SAlexander Shaposhnikov   if (auto *RetInst = dyn_cast<ReturnInst>(&Inst)) {
197517106792SAlexander Shaposhnikov     if (!ClCheckRet)
197617106792SAlexander Shaposhnikov       return;
197717106792SAlexander Shaposhnikov 
197817106792SAlexander Shaposhnikov     Value *RV = RetInst->getReturnValue();
197917106792SAlexander Shaposhnikov     if (RV == nullptr)
198017106792SAlexander Shaposhnikov       return; // This is a `ret void`.
198117106792SAlexander Shaposhnikov     Type *VT = RV->getType();
198217106792SAlexander Shaposhnikov     Type *ExtendedVT = Config.getExtendedFPType(VT);
198317106792SAlexander Shaposhnikov     if (ExtendedVT == nullptr)
198417106792SAlexander Shaposhnikov       return; // Not an FT ret.
198517106792SAlexander Shaposhnikov     Value *RVShadow = Map.getShadow(RV);
198617106792SAlexander Shaposhnikov     IRBuilder<> Builder(RetInst);
198717106792SAlexander Shaposhnikov 
198817106792SAlexander Shaposhnikov     RVShadow = emitCheck(RV, RVShadow, Builder, CheckLoc::makeRet());
198917106792SAlexander Shaposhnikov     ++NumInstrumentedFTRets;
199017106792SAlexander Shaposhnikov     // Store tag.
199117106792SAlexander Shaposhnikov     Value *FnAddr =
199217106792SAlexander Shaposhnikov         Builder.CreatePtrToInt(Inst.getParent()->getParent(), IntptrTy);
199317106792SAlexander Shaposhnikov     Builder.CreateStore(FnAddr, NsanShadowRetTag);
199417106792SAlexander Shaposhnikov     // Store value.
199517106792SAlexander Shaposhnikov     Value *ShadowRetValPtr =
199617106792SAlexander Shaposhnikov         Builder.CreateConstGEP2_64(NsanShadowRetType, NsanShadowRetPtr, 0, 0);
199717106792SAlexander Shaposhnikov     Builder.CreateStore(RVShadow, ShadowRetValPtr);
199817106792SAlexander Shaposhnikov     return;
199917106792SAlexander Shaposhnikov   }
200017106792SAlexander Shaposhnikov 
200117106792SAlexander Shaposhnikov   if (InsertValueInst *Insert = dyn_cast<InsertValueInst>(&Inst)) {
200217106792SAlexander Shaposhnikov     Value *V = Insert->getOperand(1);
200317106792SAlexander Shaposhnikov     Type *VT = V->getType();
200417106792SAlexander Shaposhnikov     Type *ExtendedVT = Config.getExtendedFPType(VT);
200517106792SAlexander Shaposhnikov     if (ExtendedVT == nullptr)
200617106792SAlexander Shaposhnikov       return;
200717106792SAlexander Shaposhnikov     IRBuilder<> Builder(Insert);
200817106792SAlexander Shaposhnikov     emitCheck(V, Map.getShadow(V), Builder, CheckLoc::makeInsert());
200917106792SAlexander Shaposhnikov     return;
201017106792SAlexander Shaposhnikov   }
201117106792SAlexander Shaposhnikov }
201217106792SAlexander Shaposhnikov 
201317106792SAlexander Shaposhnikov // Moves fast math flags from the function to individual instructions, and
201417106792SAlexander Shaposhnikov // removes the attribute from the function.
201517106792SAlexander Shaposhnikov // TODO: Make this controllable with a flag.
201617106792SAlexander Shaposhnikov static void moveFastMathFlags(Function &F,
201717106792SAlexander Shaposhnikov                               std::vector<Instruction *> &Instructions) {
201817106792SAlexander Shaposhnikov   FastMathFlags FMF;
201917106792SAlexander Shaposhnikov #define MOVE_FLAG(attr, setter)                                                \
202017106792SAlexander Shaposhnikov   if (F.getFnAttribute(attr).getValueAsString() == "true") {                   \
202117106792SAlexander Shaposhnikov     F.removeFnAttr(attr);                                                      \
202217106792SAlexander Shaposhnikov     FMF.set##setter();                                                         \
202317106792SAlexander Shaposhnikov   }
202417106792SAlexander Shaposhnikov   MOVE_FLAG("unsafe-fp-math", Fast)
202517106792SAlexander Shaposhnikov   MOVE_FLAG("no-infs-fp-math", NoInfs)
202617106792SAlexander Shaposhnikov   MOVE_FLAG("no-nans-fp-math", NoNaNs)
202717106792SAlexander Shaposhnikov   MOVE_FLAG("no-signed-zeros-fp-math", NoSignedZeros)
202817106792SAlexander Shaposhnikov #undef MOVE_FLAG
202917106792SAlexander Shaposhnikov 
203017106792SAlexander Shaposhnikov   for (Instruction *I : Instructions)
203117106792SAlexander Shaposhnikov     if (isa<FPMathOperator>(I))
203217106792SAlexander Shaposhnikov       I->setFastMathFlags(FMF);
203317106792SAlexander Shaposhnikov }
203417106792SAlexander Shaposhnikov 
203517106792SAlexander Shaposhnikov bool NumericalStabilitySanitizer::sanitizeFunction(
203617106792SAlexander Shaposhnikov     Function &F, const TargetLibraryInfo &TLI) {
2037d23c24f3SAlexander Shaposhnikov   if (!F.hasFnAttribute(Attribute::SanitizeNumericalStability) ||
2038d23c24f3SAlexander Shaposhnikov       F.isDeclaration())
203917106792SAlexander Shaposhnikov     return false;
204017106792SAlexander Shaposhnikov 
204117106792SAlexander Shaposhnikov   // This is required to prevent instrumenting call to __nsan_init from within
204217106792SAlexander Shaposhnikov   // the module constructor.
204317106792SAlexander Shaposhnikov   if (F.getName() == kNsanModuleCtorName)
204417106792SAlexander Shaposhnikov     return false;
204517106792SAlexander Shaposhnikov   SmallVector<Instruction *, 8> AllLoadsAndStores;
204617106792SAlexander Shaposhnikov   SmallVector<Instruction *, 8> LocalLoadsAndStores;
204717106792SAlexander Shaposhnikov 
204817106792SAlexander Shaposhnikov   // The instrumentation maintains:
204917106792SAlexander Shaposhnikov   //  - for each IR value `v` of floating-point (or vector floating-point) type
205017106792SAlexander Shaposhnikov   //    FT, a shadow IR value `s(v)` with twice the precision 2FT (e.g.
205117106792SAlexander Shaposhnikov   //    double for float and f128 for double).
205217106792SAlexander Shaposhnikov   //  - A shadow memory, which stores `s(v)` for any `v` that has been stored,
205317106792SAlexander Shaposhnikov   //    along with a shadow memory tag, which stores whether the value in the
205417106792SAlexander Shaposhnikov   //    corresponding shadow memory is valid. Note that this might be
205517106792SAlexander Shaposhnikov   //    incorrect if a non-instrumented function stores to memory, or if
205617106792SAlexander Shaposhnikov   //    memory is stored to through a char pointer.
205717106792SAlexander Shaposhnikov   //  - A shadow stack, which holds `s(v)` for any floating-point argument `v`
205817106792SAlexander Shaposhnikov   //    of a call to an instrumented function. This allows
205917106792SAlexander Shaposhnikov   //    instrumented functions to retrieve the shadow values for their
206017106792SAlexander Shaposhnikov   //    arguments.
206117106792SAlexander Shaposhnikov   //    Because instrumented functions can be called from non-instrumented
206217106792SAlexander Shaposhnikov   //    functions, the stack needs to include a tag so that the instrumented
206317106792SAlexander Shaposhnikov   //    function knows whether shadow values are available for their
206417106792SAlexander Shaposhnikov   //    parameters (i.e. whether is was called by an instrumented function).
206517106792SAlexander Shaposhnikov   //    When shadow arguments are not available, they have to be recreated by
206617106792SAlexander Shaposhnikov   //    extending the precision of the non-shadow arguments to the non-shadow
206717106792SAlexander Shaposhnikov   //    value. Non-instrumented functions do not modify (or even know about) the
206817106792SAlexander Shaposhnikov   //    shadow stack. The shadow stack pointer is __nsan_shadow_args. The shadow
206917106792SAlexander Shaposhnikov   //    stack tag is __nsan_shadow_args_tag. The tag is any unique identifier
207017106792SAlexander Shaposhnikov   //    for the function (we use the address of the function). Both variables
207117106792SAlexander Shaposhnikov   //    are thread local.
207217106792SAlexander Shaposhnikov   //    Example:
207317106792SAlexander Shaposhnikov   //     calls                             shadow stack tag      shadow stack
207417106792SAlexander Shaposhnikov   //     =======================================================================
207517106792SAlexander Shaposhnikov   //     non_instrumented_1()              0                     0
207617106792SAlexander Shaposhnikov   //             |
207717106792SAlexander Shaposhnikov   //             v
207817106792SAlexander Shaposhnikov   //     instrumented_2(float a)           0                     0
207917106792SAlexander Shaposhnikov   //             |
208017106792SAlexander Shaposhnikov   //             v
208117106792SAlexander Shaposhnikov   //     instrumented_3(float b, double c) &instrumented_3       s(b),s(c)
208217106792SAlexander Shaposhnikov   //             |
208317106792SAlexander Shaposhnikov   //             v
208417106792SAlexander Shaposhnikov   //     instrumented_4(float d)           &instrumented_4       s(d)
208517106792SAlexander Shaposhnikov   //             |
208617106792SAlexander Shaposhnikov   //             v
208717106792SAlexander Shaposhnikov   //     non_instrumented_5(float e)       &non_instrumented_5   s(e)
208817106792SAlexander Shaposhnikov   //             |
208917106792SAlexander Shaposhnikov   //             v
209017106792SAlexander Shaposhnikov   //     instrumented_6(float f)           &non_instrumented_5   s(e)
209117106792SAlexander Shaposhnikov   //
209217106792SAlexander Shaposhnikov   //   On entry, instrumented_2 checks whether the tag corresponds to its
209317106792SAlexander Shaposhnikov   //   function ptr.
209417106792SAlexander Shaposhnikov   //   Note that functions reset the tag to 0 after reading shadow parameters.
209517106792SAlexander Shaposhnikov   //   This ensures that the function does not erroneously read invalid data if
209617106792SAlexander Shaposhnikov   //   called twice in the same stack, once from an instrumented function and
209717106792SAlexander Shaposhnikov   //   once from an uninstrumented one. For example, in the following example,
209817106792SAlexander Shaposhnikov   //   resetting the tag in (A) ensures that (B) does not reuse the same the
209917106792SAlexander Shaposhnikov   //   shadow arguments (which would be incorrect).
210017106792SAlexander Shaposhnikov   //      instrumented_1(float a)
210117106792SAlexander Shaposhnikov   //             |
210217106792SAlexander Shaposhnikov   //             v
210317106792SAlexander Shaposhnikov   //      instrumented_2(float b)  (A)
210417106792SAlexander Shaposhnikov   //             |
210517106792SAlexander Shaposhnikov   //             v
210617106792SAlexander Shaposhnikov   //      non_instrumented_3()
210717106792SAlexander Shaposhnikov   //             |
210817106792SAlexander Shaposhnikov   //             v
210917106792SAlexander Shaposhnikov   //      instrumented_2(float b)  (B)
211017106792SAlexander Shaposhnikov   //
211117106792SAlexander Shaposhnikov   //  - A shadow return slot. Any function that returns a floating-point value
211217106792SAlexander Shaposhnikov   //    places a shadow return value in __nsan_shadow_ret_val. Again, because
211317106792SAlexander Shaposhnikov   //    we might be calling non-instrumented functions, this value is guarded
211417106792SAlexander Shaposhnikov   //    by __nsan_shadow_ret_tag marker indicating which instrumented function
211517106792SAlexander Shaposhnikov   //    placed the value in __nsan_shadow_ret_val, so that the caller can check
211617106792SAlexander Shaposhnikov   //    that this corresponds to the callee. Both variables are thread local.
211717106792SAlexander Shaposhnikov   //
211817106792SAlexander Shaposhnikov   //    For example, in the following example, the instrumentation in
211917106792SAlexander Shaposhnikov   //    `instrumented_1` rejects the shadow return value from `instrumented_3`
212017106792SAlexander Shaposhnikov   //    because is is not tagged as expected (`&instrumented_3` instead of
212117106792SAlexander Shaposhnikov   //    `non_instrumented_2`):
212217106792SAlexander Shaposhnikov   //
212317106792SAlexander Shaposhnikov   //        instrumented_1()
212417106792SAlexander Shaposhnikov   //            |
212517106792SAlexander Shaposhnikov   //            v
212617106792SAlexander Shaposhnikov   //        float non_instrumented_2()
212717106792SAlexander Shaposhnikov   //            |
212817106792SAlexander Shaposhnikov   //            v
212917106792SAlexander Shaposhnikov   //        float instrumented_3()
213017106792SAlexander Shaposhnikov   //
213117106792SAlexander Shaposhnikov   // Calls of known math functions (sin, cos, exp, ...) are duplicated to call
213217106792SAlexander Shaposhnikov   // their overload on the shadow type.
213317106792SAlexander Shaposhnikov 
213417106792SAlexander Shaposhnikov   // Collect all instructions before processing, as creating shadow values
213517106792SAlexander Shaposhnikov   // creates new instructions inside the function.
213617106792SAlexander Shaposhnikov   std::vector<Instruction *> OriginalInstructions;
213717106792SAlexander Shaposhnikov   for (BasicBlock &BB : F)
213817106792SAlexander Shaposhnikov     for (Instruction &Inst : BB)
213917106792SAlexander Shaposhnikov       OriginalInstructions.emplace_back(&Inst);
214017106792SAlexander Shaposhnikov 
214117106792SAlexander Shaposhnikov   moveFastMathFlags(F, OriginalInstructions);
214217106792SAlexander Shaposhnikov   ValueToShadowMap ValueToShadow(Config);
214317106792SAlexander Shaposhnikov 
214417106792SAlexander Shaposhnikov   // In the first pass, we create shadow values for all FT function arguments
214517106792SAlexander Shaposhnikov   // and all phis. This ensures that the DFS of the next pass does not have
214617106792SAlexander Shaposhnikov   // any loops.
214717106792SAlexander Shaposhnikov   std::vector<PHINode *> OriginalPhis;
214817106792SAlexander Shaposhnikov   createShadowArguments(F, TLI, ValueToShadow);
214917106792SAlexander Shaposhnikov   for (Instruction *I : OriginalInstructions) {
215017106792SAlexander Shaposhnikov     if (PHINode *Phi = dyn_cast<PHINode>(I)) {
215117106792SAlexander Shaposhnikov       if (PHINode *Shadow = maybeCreateShadowPhi(*Phi, TLI)) {
215217106792SAlexander Shaposhnikov         OriginalPhis.push_back(Phi);
215317106792SAlexander Shaposhnikov         ValueToShadow.setShadow(*Phi, *Shadow);
215417106792SAlexander Shaposhnikov       }
215517106792SAlexander Shaposhnikov     }
215617106792SAlexander Shaposhnikov   }
215717106792SAlexander Shaposhnikov 
215817106792SAlexander Shaposhnikov   // Create shadow values for all instructions creating FT values.
215917106792SAlexander Shaposhnikov   for (Instruction *I : OriginalInstructions)
216017106792SAlexander Shaposhnikov     maybeCreateShadowValue(*I, TLI, ValueToShadow);
216117106792SAlexander Shaposhnikov 
216217106792SAlexander Shaposhnikov   // Propagate shadow values across stores, calls and rets.
216317106792SAlexander Shaposhnikov   for (Instruction *I : OriginalInstructions)
216417106792SAlexander Shaposhnikov     propagateShadowValues(*I, TLI, ValueToShadow);
216517106792SAlexander Shaposhnikov 
216617106792SAlexander Shaposhnikov   // The last pass populates shadow phis with shadow values.
216717106792SAlexander Shaposhnikov   for (PHINode *Phi : OriginalPhis) {
2168430b90f0SWu Yingcong     PHINode *ShadowPhi = cast<PHINode>(ValueToShadow.getShadow(Phi));
216917106792SAlexander Shaposhnikov     for (unsigned I : seq(Phi->getNumOperands())) {
217017106792SAlexander Shaposhnikov       Value *V = Phi->getOperand(I);
217117106792SAlexander Shaposhnikov       Value *Shadow = ValueToShadow.getShadow(V);
217217106792SAlexander Shaposhnikov       BasicBlock *IncomingBB = Phi->getIncomingBlock(I);
217317106792SAlexander Shaposhnikov       // For some instructions (e.g. invoke), we create the shadow in a separate
217417106792SAlexander Shaposhnikov       // block, different from the block where the original value is created.
217517106792SAlexander Shaposhnikov       // In that case, the shadow phi might need to refer to this block instead
217617106792SAlexander Shaposhnikov       // of the original block.
217717106792SAlexander Shaposhnikov       // Note that this can only happen for instructions as constant shadows are
217817106792SAlexander Shaposhnikov       // always created in the same block.
217917106792SAlexander Shaposhnikov       ShadowPhi->addIncoming(Shadow, IncomingBB);
218017106792SAlexander Shaposhnikov     }
218117106792SAlexander Shaposhnikov   }
218217106792SAlexander Shaposhnikov 
218317106792SAlexander Shaposhnikov   return !ValueToShadow.empty();
218417106792SAlexander Shaposhnikov }
218517106792SAlexander Shaposhnikov 
2186ddf5725eSDmitry Chestnykh static uint64_t GetMemOpSize(Value *V) {
2187ddf5725eSDmitry Chestnykh   uint64_t OpSize = 0;
2188ddf5725eSDmitry Chestnykh   if (Constant *C = dyn_cast<Constant>(V)) {
2189ddf5725eSDmitry Chestnykh     auto *CInt = dyn_cast<ConstantInt>(C);
2190ddf5725eSDmitry Chestnykh     if (CInt && CInt->getValue().getBitWidth() <= 64)
2191ddf5725eSDmitry Chestnykh       OpSize = CInt->getValue().getZExtValue();
2192ddf5725eSDmitry Chestnykh   }
2193ddf5725eSDmitry Chestnykh 
2194ddf5725eSDmitry Chestnykh   return OpSize;
2195ddf5725eSDmitry Chestnykh }
2196ddf5725eSDmitry Chestnykh 
219717106792SAlexander Shaposhnikov // Instrument the memory intrinsics so that they properly modify the shadow
219817106792SAlexander Shaposhnikov // memory.
219917106792SAlexander Shaposhnikov bool NumericalStabilitySanitizer::instrumentMemIntrinsic(MemIntrinsic *MI) {
220017106792SAlexander Shaposhnikov   IRBuilder<> Builder(MI);
220117106792SAlexander Shaposhnikov   if (auto *M = dyn_cast<MemSetInst>(MI)) {
2202ddf5725eSDmitry Chestnykh     FunctionCallee SetUnknownFn =
2203ddf5725eSDmitry Chestnykh         NsanSetUnknownFns.getFunctionFor(GetMemOpSize(M->getArgOperand(2)));
2204ddf5725eSDmitry Chestnykh     if (SetUnknownFn.getFunctionType()->getNumParams() == 1)
2205ddf5725eSDmitry Chestnykh       Builder.CreateCall(SetUnknownFn, {/*Address=*/M->getArgOperand(0)});
2206ddf5725eSDmitry Chestnykh     else
2207ddf5725eSDmitry Chestnykh       Builder.CreateCall(SetUnknownFn,
220817106792SAlexander Shaposhnikov                          {/*Address=*/M->getArgOperand(0),
2209ddf5725eSDmitry Chestnykh                           /*Size=*/Builder.CreateIntCast(M->getArgOperand(2),
2210ddf5725eSDmitry Chestnykh                                                          IntptrTy, false)});
2211ddf5725eSDmitry Chestnykh 
221217106792SAlexander Shaposhnikov   } else if (auto *M = dyn_cast<MemTransferInst>(MI)) {
2213ddf5725eSDmitry Chestnykh     FunctionCallee CopyFn =
2214ddf5725eSDmitry Chestnykh         NsanCopyFns.getFunctionFor(GetMemOpSize(M->getArgOperand(2)));
2215ddf5725eSDmitry Chestnykh 
2216ddf5725eSDmitry Chestnykh     if (CopyFn.getFunctionType()->getNumParams() == 2)
2217ddf5725eSDmitry Chestnykh       Builder.CreateCall(CopyFn, {/*Destination=*/M->getArgOperand(0),
2218ddf5725eSDmitry Chestnykh                                   /*Source=*/M->getArgOperand(1)});
2219ddf5725eSDmitry Chestnykh     else
2220ddf5725eSDmitry Chestnykh       Builder.CreateCall(CopyFn, {/*Destination=*/M->getArgOperand(0),
222117106792SAlexander Shaposhnikov                                   /*Source=*/M->getArgOperand(1),
2222ddf5725eSDmitry Chestnykh                                   /*Size=*/
2223ddf5725eSDmitry Chestnykh                                   Builder.CreateIntCast(M->getArgOperand(2),
2224ddf5725eSDmitry Chestnykh                                                         IntptrTy, false)});
222517106792SAlexander Shaposhnikov   }
222617106792SAlexander Shaposhnikov   return false;
222717106792SAlexander Shaposhnikov }
222817106792SAlexander Shaposhnikov 
222917106792SAlexander Shaposhnikov void NumericalStabilitySanitizer::maybeAddSuffixForNsanInterface(CallBase *CI) {
223017106792SAlexander Shaposhnikov   Function *Fn = CI->getCalledFunction();
223117106792SAlexander Shaposhnikov   if (Fn == nullptr)
223217106792SAlexander Shaposhnikov     return;
223317106792SAlexander Shaposhnikov 
223417106792SAlexander Shaposhnikov   if (!Fn->getName().starts_with("__nsan_"))
223517106792SAlexander Shaposhnikov     return;
223617106792SAlexander Shaposhnikov 
223717106792SAlexander Shaposhnikov   if (Fn->getName() == "__nsan_dump_shadow_mem") {
223817106792SAlexander Shaposhnikov     assert(CI->arg_size() == 4 &&
223917106792SAlexander Shaposhnikov            "invalid prototype for __nsan_dump_shadow_mem");
224017106792SAlexander Shaposhnikov     // __nsan_dump_shadow_mem requires an extra parameter with the dynamic
224117106792SAlexander Shaposhnikov     // configuration:
224217106792SAlexander Shaposhnikov     // (shadow_type_id_for_long_double << 16) | (shadow_type_id_for_double << 8)
224317106792SAlexander Shaposhnikov     // | shadow_type_id_for_double
224417106792SAlexander Shaposhnikov     const uint64_t shadow_value_type_ids =
224517106792SAlexander Shaposhnikov         (static_cast<size_t>(Config.byValueType(kLongDouble).getNsanTypeId())
224617106792SAlexander Shaposhnikov          << 16) |
224717106792SAlexander Shaposhnikov         (static_cast<size_t>(Config.byValueType(kDouble).getNsanTypeId())
224817106792SAlexander Shaposhnikov          << 8) |
224917106792SAlexander Shaposhnikov         static_cast<size_t>(Config.byValueType(kFloat).getNsanTypeId());
225017106792SAlexander Shaposhnikov     CI->setArgOperand(3, ConstantInt::get(IntptrTy, shadow_value_type_ids));
225117106792SAlexander Shaposhnikov   }
225217106792SAlexander Shaposhnikov }
2253