xref: /freebsd-src/contrib/llvm-project/llvm/lib/Transforms/Utils/InjectTLIMappings.cpp (revision 5ffd83dbcc34f10e07f6d3e968ae6365869615f4)
1480093f4SDimitry Andric //===- InjectTLIMAppings.cpp - TLI to VFABI attribute injection  ----------===//
2480093f4SDimitry Andric //
3480093f4SDimitry Andric // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4480093f4SDimitry Andric // See https://llvm.org/LICENSE.txt for license information.
5480093f4SDimitry Andric // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6480093f4SDimitry Andric //
7480093f4SDimitry Andric //===----------------------------------------------------------------------===//
8480093f4SDimitry Andric //
9480093f4SDimitry Andric // Populates the VFABI attribute with the scalar-to-vector mappings
10480093f4SDimitry Andric // from the TargetLibraryInfo.
11480093f4SDimitry Andric //
12480093f4SDimitry Andric //===----------------------------------------------------------------------===//
13480093f4SDimitry Andric 
14480093f4SDimitry Andric #include "llvm/Transforms/Utils/InjectTLIMappings.h"
15480093f4SDimitry Andric #include "llvm/ADT/Statistic.h"
16*5ffd83dbSDimitry Andric #include "llvm/Analysis/DemandedBits.h"
17*5ffd83dbSDimitry Andric #include "llvm/Analysis/GlobalsModRef.h"
18*5ffd83dbSDimitry Andric #include "llvm/Analysis/OptimizationRemarkEmitter.h"
19480093f4SDimitry Andric #include "llvm/Analysis/VectorUtils.h"
20480093f4SDimitry Andric #include "llvm/IR/InstIterator.h"
21*5ffd83dbSDimitry Andric #include "llvm/IR/IntrinsicInst.h"
22480093f4SDimitry Andric #include "llvm/Transforms/Utils.h"
23480093f4SDimitry Andric #include "llvm/Transforms/Utils/ModuleUtils.h"
24480093f4SDimitry Andric 
25480093f4SDimitry Andric using namespace llvm;
26480093f4SDimitry Andric 
27480093f4SDimitry Andric #define DEBUG_TYPE "inject-tli-mappings"
28480093f4SDimitry Andric 
29480093f4SDimitry Andric STATISTIC(NumCallInjected,
30480093f4SDimitry Andric           "Number of calls in which the mappings have been injected.");
31480093f4SDimitry Andric 
32480093f4SDimitry Andric STATISTIC(NumVFDeclAdded,
33480093f4SDimitry Andric           "Number of function declarations that have been added.");
34480093f4SDimitry Andric STATISTIC(NumCompUsedAdded,
35480093f4SDimitry Andric           "Number of `@llvm.compiler.used` operands that have been added.");
36480093f4SDimitry Andric 
37480093f4SDimitry Andric /// A helper function that adds the vector function declaration that
38480093f4SDimitry Andric /// vectorizes the CallInst CI with a vectorization factor of VF
39480093f4SDimitry Andric /// lanes. The TLI assumes that all parameters and the return type of
40480093f4SDimitry Andric /// CI (other than void) need to be widened to a VectorType of VF
41480093f4SDimitry Andric /// lanes.
42480093f4SDimitry Andric static void addVariantDeclaration(CallInst &CI, const unsigned VF,
43480093f4SDimitry Andric                                   const StringRef VFName) {
44480093f4SDimitry Andric   Module *M = CI.getModule();
45480093f4SDimitry Andric 
46480093f4SDimitry Andric   // Add function declaration.
47480093f4SDimitry Andric   Type *RetTy = ToVectorTy(CI.getType(), VF);
48480093f4SDimitry Andric   SmallVector<Type *, 4> Tys;
49480093f4SDimitry Andric   for (Value *ArgOperand : CI.arg_operands())
50480093f4SDimitry Andric     Tys.push_back(ToVectorTy(ArgOperand->getType(), VF));
51480093f4SDimitry Andric   assert(!CI.getFunctionType()->isVarArg() &&
52480093f4SDimitry Andric          "VarArg functions are not supported.");
53480093f4SDimitry Andric   FunctionType *FTy = FunctionType::get(RetTy, Tys, /*isVarArg=*/false);
54480093f4SDimitry Andric   Function *VectorF =
55480093f4SDimitry Andric       Function::Create(FTy, Function::ExternalLinkage, VFName, M);
56480093f4SDimitry Andric   VectorF->copyAttributesFrom(CI.getCalledFunction());
57480093f4SDimitry Andric   ++NumVFDeclAdded;
58480093f4SDimitry Andric   LLVM_DEBUG(dbgs() << DEBUG_TYPE << ": Added to the module: `" << VFName
59480093f4SDimitry Andric                     << "` of type " << *(VectorF->getType()) << "\n");
60480093f4SDimitry Andric 
61480093f4SDimitry Andric   // Make function declaration (without a body) "sticky" in the IR by
62480093f4SDimitry Andric   // listing it in the @llvm.compiler.used intrinsic.
63480093f4SDimitry Andric   assert(!VectorF->size() && "VFABI attribute requires `@llvm.compiler.used` "
64480093f4SDimitry Andric                              "only on declarations.");
65480093f4SDimitry Andric   appendToCompilerUsed(*M, {VectorF});
66480093f4SDimitry Andric   LLVM_DEBUG(dbgs() << DEBUG_TYPE << ": Adding `" << VFName
67480093f4SDimitry Andric                     << "` to `@llvm.compiler.used`.\n");
68480093f4SDimitry Andric   ++NumCompUsedAdded;
69480093f4SDimitry Andric }
70480093f4SDimitry Andric 
71480093f4SDimitry Andric static void addMappingsFromTLI(const TargetLibraryInfo &TLI, CallInst &CI) {
72480093f4SDimitry Andric   // This is needed to make sure we don't query the TLI for calls to
73480093f4SDimitry Andric   // bitcast of function pointers, like `%call = call i32 (i32*, ...)
74480093f4SDimitry Andric   // bitcast (i32 (...)* @goo to i32 (i32*, ...)*)(i32* nonnull %i)`,
75480093f4SDimitry Andric   // as such calls make the `isFunctionVectorizable` raise an
76480093f4SDimitry Andric   // exception.
77480093f4SDimitry Andric   if (CI.isNoBuiltin() || !CI.getCalledFunction())
78480093f4SDimitry Andric     return;
79480093f4SDimitry Andric 
80*5ffd83dbSDimitry Andric   const std::string ScalarName = std::string(CI.getCalledFunction()->getName());
81480093f4SDimitry Andric   // Nothing to be done if the TLI thinks the function is not
82480093f4SDimitry Andric   // vectorizable.
83480093f4SDimitry Andric   if (!TLI.isFunctionVectorizable(ScalarName))
84480093f4SDimitry Andric     return;
85480093f4SDimitry Andric   SmallVector<std::string, 8> Mappings;
86480093f4SDimitry Andric   VFABI::getVectorVariantNames(CI, Mappings);
87480093f4SDimitry Andric   Module *M = CI.getModule();
88480093f4SDimitry Andric   const SetVector<StringRef> OriginalSetOfMappings(Mappings.begin(),
89480093f4SDimitry Andric                                                    Mappings.end());
90480093f4SDimitry Andric   //  All VFs in the TLI are powers of 2.
91480093f4SDimitry Andric   for (unsigned VF = 2, WidestVF = TLI.getWidestVF(ScalarName); VF <= WidestVF;
92480093f4SDimitry Andric        VF *= 2) {
93*5ffd83dbSDimitry Andric     const std::string TLIName =
94*5ffd83dbSDimitry Andric         std::string(TLI.getVectorizedFunction(ScalarName, VF));
95480093f4SDimitry Andric     if (!TLIName.empty()) {
96*5ffd83dbSDimitry Andric       std::string MangledName = VFABI::mangleTLIVectorName(
97*5ffd83dbSDimitry Andric           TLIName, ScalarName, CI.getNumArgOperands(), VF);
98480093f4SDimitry Andric       if (!OriginalSetOfMappings.count(MangledName)) {
99480093f4SDimitry Andric         Mappings.push_back(MangledName);
100480093f4SDimitry Andric         ++NumCallInjected;
101480093f4SDimitry Andric       }
102480093f4SDimitry Andric       Function *VariantF = M->getFunction(TLIName);
103480093f4SDimitry Andric       if (!VariantF)
104480093f4SDimitry Andric         addVariantDeclaration(CI, VF, TLIName);
105480093f4SDimitry Andric     }
106480093f4SDimitry Andric   }
107480093f4SDimitry Andric 
108480093f4SDimitry Andric   VFABI::setVectorVariantNames(&CI, Mappings);
109480093f4SDimitry Andric }
110480093f4SDimitry Andric 
111480093f4SDimitry Andric static bool runImpl(const TargetLibraryInfo &TLI, Function &F) {
112480093f4SDimitry Andric   for (auto &I : instructions(F))
113480093f4SDimitry Andric     if (auto CI = dyn_cast<CallInst>(&I))
114480093f4SDimitry Andric       addMappingsFromTLI(TLI, *CI);
115480093f4SDimitry Andric   // Even if the pass adds IR attributes, the analyses are preserved.
116480093f4SDimitry Andric   return false;
117480093f4SDimitry Andric }
118480093f4SDimitry Andric 
119480093f4SDimitry Andric ////////////////////////////////////////////////////////////////////////////////
120480093f4SDimitry Andric // New pass manager implementation.
121480093f4SDimitry Andric ////////////////////////////////////////////////////////////////////////////////
122480093f4SDimitry Andric PreservedAnalyses InjectTLIMappings::run(Function &F,
123480093f4SDimitry Andric                                          FunctionAnalysisManager &AM) {
124480093f4SDimitry Andric   const TargetLibraryInfo &TLI = AM.getResult<TargetLibraryAnalysis>(F);
125480093f4SDimitry Andric   runImpl(TLI, F);
126480093f4SDimitry Andric   // Even if the pass adds IR attributes, the analyses are preserved.
127480093f4SDimitry Andric   return PreservedAnalyses::all();
128480093f4SDimitry Andric }
129480093f4SDimitry Andric 
130480093f4SDimitry Andric ////////////////////////////////////////////////////////////////////////////////
131480093f4SDimitry Andric // Legacy PM Implementation.
132480093f4SDimitry Andric ////////////////////////////////////////////////////////////////////////////////
133480093f4SDimitry Andric bool InjectTLIMappingsLegacy::runOnFunction(Function &F) {
134480093f4SDimitry Andric   const TargetLibraryInfo &TLI =
135480093f4SDimitry Andric       getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(F);
136480093f4SDimitry Andric   return runImpl(TLI, F);
137480093f4SDimitry Andric }
138480093f4SDimitry Andric 
139480093f4SDimitry Andric void InjectTLIMappingsLegacy::getAnalysisUsage(AnalysisUsage &AU) const {
140480093f4SDimitry Andric   AU.setPreservesCFG();
141480093f4SDimitry Andric   AU.addRequired<TargetLibraryInfoWrapperPass>();
142480093f4SDimitry Andric   AU.addPreserved<TargetLibraryInfoWrapperPass>();
143*5ffd83dbSDimitry Andric   AU.addPreserved<ScalarEvolutionWrapperPass>();
144*5ffd83dbSDimitry Andric   AU.addPreserved<AAResultsWrapperPass>();
145*5ffd83dbSDimitry Andric   AU.addPreserved<LoopAccessLegacyAnalysis>();
146*5ffd83dbSDimitry Andric   AU.addPreserved<DemandedBitsWrapperPass>();
147*5ffd83dbSDimitry Andric   AU.addPreserved<OptimizationRemarkEmitterWrapperPass>();
148*5ffd83dbSDimitry Andric   AU.addPreserved<GlobalsAAWrapperPass>();
149480093f4SDimitry Andric }
150480093f4SDimitry Andric 
151480093f4SDimitry Andric ////////////////////////////////////////////////////////////////////////////////
152480093f4SDimitry Andric // Legacy Pass manager initialization
153480093f4SDimitry Andric ////////////////////////////////////////////////////////////////////////////////
154480093f4SDimitry Andric char InjectTLIMappingsLegacy::ID = 0;
155480093f4SDimitry Andric 
156480093f4SDimitry Andric INITIALIZE_PASS_BEGIN(InjectTLIMappingsLegacy, DEBUG_TYPE,
157480093f4SDimitry Andric                       "Inject TLI Mappings", false, false)
158480093f4SDimitry Andric INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass)
159480093f4SDimitry Andric INITIALIZE_PASS_END(InjectTLIMappingsLegacy, DEBUG_TYPE, "Inject TLI Mappings",
160480093f4SDimitry Andric                     false, false)
161480093f4SDimitry Andric 
162480093f4SDimitry Andric FunctionPass *llvm::createInjectTLIMappingsLegacyPass() {
163480093f4SDimitry Andric   return new InjectTLIMappingsLegacy();
164480093f4SDimitry Andric }
165