1 //===- InjectTLIMAppings.cpp - TLI to VFABI attribute injection ----------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // Populates the VFABI attribute with the scalar-to-vector mappings
10 // from the TargetLibraryInfo.
11 //
12 //===----------------------------------------------------------------------===//
13
14 #include "llvm/Transforms/Utils/InjectTLIMappings.h"
15 #include "llvm/ADT/Statistic.h"
16 #include "llvm/Analysis/DemandedBits.h"
17 #include "llvm/Analysis/GlobalsModRef.h"
18 #include "llvm/Analysis/OptimizationRemarkEmitter.h"
19 #include "llvm/Analysis/TargetLibraryInfo.h"
20 #include "llvm/Analysis/VectorUtils.h"
21 #include "llvm/IR/InstIterator.h"
22 #include "llvm/Transforms/Utils.h"
23 #include "llvm/Transforms/Utils/ModuleUtils.h"
24
25 using namespace llvm;
26
27 #define DEBUG_TYPE "inject-tli-mappings"
28
29 STATISTIC(NumCallInjected,
30 "Number of calls in which the mappings have been injected.");
31
32 STATISTIC(NumVFDeclAdded,
33 "Number of function declarations that have been added.");
34 STATISTIC(NumCompUsedAdded,
35 "Number of `@llvm.compiler.used` operands that have been added.");
36
37 /// A helper function that adds the vector function declaration that
38 /// vectorizes the CallInst CI with a vectorization factor of VF
39 /// lanes. The TLI assumes that all parameters and the return type of
40 /// CI (other than void) need to be widened to a VectorType of VF
41 /// lanes.
addVariantDeclaration(CallInst & CI,const ElementCount & VF,const StringRef VFName)42 static void addVariantDeclaration(CallInst &CI, const ElementCount &VF,
43 const StringRef VFName) {
44 Module *M = CI.getModule();
45
46 // Add function declaration.
47 Type *RetTy = ToVectorTy(CI.getType(), VF);
48 SmallVector<Type *, 4> Tys;
49 for (Value *ArgOperand : CI.args())
50 Tys.push_back(ToVectorTy(ArgOperand->getType(), VF));
51 assert(!CI.getFunctionType()->isVarArg() &&
52 "VarArg functions are not supported.");
53 FunctionType *FTy = FunctionType::get(RetTy, Tys, /*isVarArg=*/false);
54 Function *VectorF =
55 Function::Create(FTy, Function::ExternalLinkage, VFName, M);
56 VectorF->copyAttributesFrom(CI.getCalledFunction());
57 ++NumVFDeclAdded;
58 LLVM_DEBUG(dbgs() << DEBUG_TYPE << ": Added to the module: `" << VFName
59 << "` of type " << *(VectorF->getType()) << "\n");
60
61 // Make function declaration (without a body) "sticky" in the IR by
62 // listing it in the @llvm.compiler.used intrinsic.
63 assert(!VectorF->size() && "VFABI attribute requires `@llvm.compiler.used` "
64 "only on declarations.");
65 appendToCompilerUsed(*M, {VectorF});
66 LLVM_DEBUG(dbgs() << DEBUG_TYPE << ": Adding `" << VFName
67 << "` to `@llvm.compiler.used`.\n");
68 ++NumCompUsedAdded;
69 }
70
addMappingsFromTLI(const TargetLibraryInfo & TLI,CallInst & CI)71 static void addMappingsFromTLI(const TargetLibraryInfo &TLI, CallInst &CI) {
72 // This is needed to make sure we don't query the TLI for calls to
73 // bitcast of function pointers, like `%call = call i32 (i32*, ...)
74 // bitcast (i32 (...)* @goo to i32 (i32*, ...)*)(i32* nonnull %i)`,
75 // as such calls make the `isFunctionVectorizable` raise an
76 // exception.
77 if (CI.isNoBuiltin() || !CI.getCalledFunction())
78 return;
79
80 StringRef ScalarName = CI.getCalledFunction()->getName();
81
82 // Nothing to be done if the TLI thinks the function is not
83 // vectorizable.
84 if (!TLI.isFunctionVectorizable(ScalarName))
85 return;
86 SmallVector<std::string, 8> Mappings;
87 VFABI::getVectorVariantNames(CI, Mappings);
88 Module *M = CI.getModule();
89 const SetVector<StringRef> OriginalSetOfMappings(Mappings.begin(),
90 Mappings.end());
91
92 auto AddVariantDecl = [&](const ElementCount &VF) {
93 const std::string TLIName =
94 std::string(TLI.getVectorizedFunction(ScalarName, VF));
95 if (!TLIName.empty()) {
96 std::string MangledName =
97 VFABI::mangleTLIVectorName(TLIName, ScalarName, CI.arg_size(), VF);
98 if (!OriginalSetOfMappings.count(MangledName)) {
99 Mappings.push_back(MangledName);
100 ++NumCallInjected;
101 }
102 Function *VariantF = M->getFunction(TLIName);
103 if (!VariantF)
104 addVariantDeclaration(CI, VF, TLIName);
105 }
106 };
107
108 // All VFs in the TLI are powers of 2.
109 ElementCount WidestFixedVF, WidestScalableVF;
110 TLI.getWidestVF(ScalarName, WidestFixedVF, WidestScalableVF);
111
112 for (ElementCount VF = ElementCount::getFixed(2);
113 ElementCount::isKnownLE(VF, WidestFixedVF); VF *= 2)
114 AddVariantDecl(VF);
115
116 // TODO: Add scalable variants once we're able to test them.
117 assert(WidestScalableVF.isZero() &&
118 "Scalable vector mappings not yet supported");
119
120 VFABI::setVectorVariantNames(&CI, Mappings);
121 }
122
runImpl(const TargetLibraryInfo & TLI,Function & F)123 static bool runImpl(const TargetLibraryInfo &TLI, Function &F) {
124 for (auto &I : instructions(F))
125 if (auto CI = dyn_cast<CallInst>(&I))
126 addMappingsFromTLI(TLI, *CI);
127 // Even if the pass adds IR attributes, the analyses are preserved.
128 return false;
129 }
130
131 ////////////////////////////////////////////////////////////////////////////////
132 // New pass manager implementation.
133 ////////////////////////////////////////////////////////////////////////////////
run(Function & F,FunctionAnalysisManager & AM)134 PreservedAnalyses InjectTLIMappings::run(Function &F,
135 FunctionAnalysisManager &AM) {
136 const TargetLibraryInfo &TLI = AM.getResult<TargetLibraryAnalysis>(F);
137 runImpl(TLI, F);
138 // Even if the pass adds IR attributes, the analyses are preserved.
139 return PreservedAnalyses::all();
140 }
141
142 ////////////////////////////////////////////////////////////////////////////////
143 // Legacy PM Implementation.
144 ////////////////////////////////////////////////////////////////////////////////
runOnFunction(Function & F)145 bool InjectTLIMappingsLegacy::runOnFunction(Function &F) {
146 const TargetLibraryInfo &TLI =
147 getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(F);
148 return runImpl(TLI, F);
149 }
150
getAnalysisUsage(AnalysisUsage & AU) const151 void InjectTLIMappingsLegacy::getAnalysisUsage(AnalysisUsage &AU) const {
152 AU.setPreservesCFG();
153 AU.addRequired<TargetLibraryInfoWrapperPass>();
154 AU.addPreserved<TargetLibraryInfoWrapperPass>();
155 AU.addPreserved<ScalarEvolutionWrapperPass>();
156 AU.addPreserved<AAResultsWrapperPass>();
157 AU.addPreserved<LoopAccessLegacyAnalysis>();
158 AU.addPreserved<DemandedBitsWrapperPass>();
159 AU.addPreserved<OptimizationRemarkEmitterWrapperPass>();
160 AU.addPreserved<GlobalsAAWrapperPass>();
161 }
162
163 ////////////////////////////////////////////////////////////////////////////////
164 // Legacy Pass manager initialization
165 ////////////////////////////////////////////////////////////////////////////////
166 char InjectTLIMappingsLegacy::ID = 0;
167
168 INITIALIZE_PASS_BEGIN(InjectTLIMappingsLegacy, DEBUG_TYPE,
169 "Inject TLI Mappings", false, false)
INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass)170 INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass)
171 INITIALIZE_PASS_END(InjectTLIMappingsLegacy, DEBUG_TYPE, "Inject TLI Mappings",
172 false, false)
173
174 FunctionPass *llvm::createInjectTLIMappingsLegacyPass() {
175 return new InjectTLIMappingsLegacy();
176 }
177