1 //===- AlwaysInliner.cpp - Code to inline always_inline functions ----------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file implements a custom inliner that handles only functions that 10 // are marked as "always inline". 11 // 12 //===----------------------------------------------------------------------===// 13 14 #include "llvm/Transforms/IPO/AlwaysInliner.h" 15 #include "llvm/ADT/SetVector.h" 16 #include "llvm/Analysis/AliasAnalysis.h" 17 #include "llvm/Analysis/AssumptionCache.h" 18 #include "llvm/Analysis/InlineAdvisor.h" 19 #include "llvm/Analysis/InlineCost.h" 20 #include "llvm/Analysis/OptimizationRemarkEmitter.h" 21 #include "llvm/Analysis/ProfileSummaryInfo.h" 22 #include "llvm/IR/Module.h" 23 #include "llvm/InitializePasses.h" 24 #include "llvm/Transforms/Utils/Cloning.h" 25 #include "llvm/Transforms/Utils/ModuleUtils.h" 26 27 using namespace llvm; 28 29 #define DEBUG_TYPE "inline" 30 31 namespace { 32 33 bool AlwaysInlineImpl( 34 Module &M, bool InsertLifetime, ProfileSummaryInfo &PSI, 35 FunctionAnalysisManager *FAM, 36 function_ref<AssumptionCache &(Function &)> GetAssumptionCache, 37 function_ref<AAResults &(Function &)> GetAAR) { 38 SmallSetVector<CallBase *, 16> Calls; 39 bool Changed = false; 40 SmallVector<Function *, 16> InlinedComdatFunctions; 41 42 for (Function &F : make_early_inc_range(M)) { 43 if (F.isPresplitCoroutine()) 44 continue; 45 46 if (F.isDeclaration() || !isInlineViable(F).isSuccess()) 47 continue; 48 49 Calls.clear(); 50 51 for (User *U : F.users()) 52 if (auto *CB = dyn_cast<CallBase>(U)) 53 if (CB->getCalledFunction() == &F && 54 CB->hasFnAttr(Attribute::AlwaysInline) && 55 !CB->getAttributes().hasFnAttr(Attribute::NoInline)) 56 Calls.insert(CB); 57 58 for (CallBase *CB : Calls) { 59 Function *Caller = CB->getCaller(); 60 OptimizationRemarkEmitter ORE(Caller); 61 DebugLoc DLoc = CB->getDebugLoc(); 62 BasicBlock *Block = CB->getParent(); 63 64 InlineFunctionInfo IFI(GetAssumptionCache, &PSI, nullptr, nullptr); 65 InlineResult Res = InlineFunction(*CB, IFI, /*MergeAttributes=*/true, 66 &GetAAR(F), InsertLifetime); 67 if (!Res.isSuccess()) { 68 ORE.emit([&]() { 69 return OptimizationRemarkMissed(DEBUG_TYPE, "NotInlined", DLoc, Block) 70 << "'" << ore::NV("Callee", &F) << "' is not inlined into '" 71 << ore::NV("Caller", Caller) 72 << "': " << ore::NV("Reason", Res.getFailureReason()); 73 }); 74 continue; 75 } 76 77 emitInlinedIntoBasedOnCost( 78 ORE, DLoc, Block, F, *Caller, 79 InlineCost::getAlways("always inline attribute"), 80 /*ForProfileContext=*/false, DEBUG_TYPE); 81 82 Changed = true; 83 if (FAM) 84 FAM->invalidate(*Caller, PreservedAnalyses::none()); 85 } 86 87 F.removeDeadConstantUsers(); 88 if (F.hasFnAttribute(Attribute::AlwaysInline) && F.isDefTriviallyDead()) { 89 // Remember to try and delete this function afterward. This allows to call 90 // filterDeadComdatFunctions() only once. 91 if (F.hasComdat()) { 92 InlinedComdatFunctions.push_back(&F); 93 } else { 94 if (FAM) 95 FAM->clear(F, F.getName()); 96 M.getFunctionList().erase(F); 97 Changed = true; 98 } 99 } 100 } 101 102 if (!InlinedComdatFunctions.empty()) { 103 // Now we just have the comdat functions. Filter out the ones whose comdats 104 // are not actually dead. 105 filterDeadComdatFunctions(InlinedComdatFunctions); 106 // The remaining functions are actually dead. 107 for (Function *F : InlinedComdatFunctions) { 108 if (FAM) 109 FAM->clear(*F, F->getName()); 110 M.getFunctionList().erase(F); 111 Changed = true; 112 } 113 } 114 115 return Changed; 116 } 117 118 struct AlwaysInlinerLegacyPass : public ModulePass { 119 bool InsertLifetime; 120 121 AlwaysInlinerLegacyPass() 122 : AlwaysInlinerLegacyPass(/*InsertLifetime*/ true) {} 123 124 AlwaysInlinerLegacyPass(bool InsertLifetime) 125 : ModulePass(ID), InsertLifetime(InsertLifetime) { 126 initializeAlwaysInlinerLegacyPassPass(*PassRegistry::getPassRegistry()); 127 } 128 129 /// Main run interface method. We override here to avoid calling skipSCC(). 130 bool runOnModule(Module &M) override { 131 132 auto &PSI = getAnalysis<ProfileSummaryInfoWrapperPass>().getPSI(); 133 auto GetAAR = [&](Function &F) -> AAResults & { 134 return getAnalysis<AAResultsWrapperPass>(F).getAAResults(); 135 }; 136 auto GetAssumptionCache = [&](Function &F) -> AssumptionCache & { 137 return getAnalysis<AssumptionCacheTracker>().getAssumptionCache(F); 138 }; 139 140 return AlwaysInlineImpl(M, InsertLifetime, PSI, /*FAM=*/nullptr, 141 GetAssumptionCache, GetAAR); 142 } 143 144 static char ID; // Pass identification, replacement for typeid 145 146 void getAnalysisUsage(AnalysisUsage &AU) const override { 147 AU.addRequired<AssumptionCacheTracker>(); 148 AU.addRequired<AAResultsWrapperPass>(); 149 AU.addRequired<ProfileSummaryInfoWrapperPass>(); 150 } 151 }; 152 153 } // namespace 154 155 char AlwaysInlinerLegacyPass::ID = 0; 156 INITIALIZE_PASS_BEGIN(AlwaysInlinerLegacyPass, "always-inline", 157 "Inliner for always_inline functions", false, false) 158 INITIALIZE_PASS_DEPENDENCY(AAResultsWrapperPass) 159 INITIALIZE_PASS_DEPENDENCY(AssumptionCacheTracker) 160 INITIALIZE_PASS_DEPENDENCY(ProfileSummaryInfoWrapperPass) 161 INITIALIZE_PASS_END(AlwaysInlinerLegacyPass, "always-inline", 162 "Inliner for always_inline functions", false, false) 163 164 Pass *llvm::createAlwaysInlinerLegacyPass(bool InsertLifetime) { 165 return new AlwaysInlinerLegacyPass(InsertLifetime); 166 } 167 168 PreservedAnalyses AlwaysInlinerPass::run(Module &M, 169 ModuleAnalysisManager &MAM) { 170 FunctionAnalysisManager &FAM = 171 MAM.getResult<FunctionAnalysisManagerModuleProxy>(M).getManager(); 172 auto GetAssumptionCache = [&](Function &F) -> AssumptionCache & { 173 return FAM.getResult<AssumptionAnalysis>(F); 174 }; 175 auto GetAAR = [&](Function &F) -> AAResults & { 176 return FAM.getResult<AAManager>(F); 177 }; 178 auto &PSI = MAM.getResult<ProfileSummaryAnalysis>(M); 179 180 bool Changed = AlwaysInlineImpl(M, InsertLifetime, PSI, &FAM, 181 GetAssumptionCache, GetAAR); 182 if (!Changed) 183 return PreservedAnalyses::all(); 184 185 PreservedAnalyses PA; 186 // We have already invalidated all analyses on modified functions. 187 PA.preserveSet<AllAnalysesOn<Function>>(); 188 return PA; 189 } 190