148fa355eSMircea Trofin //===- InlineAdvisor.cpp - analysis pass implementation -------------------===// 248fa355eSMircea Trofin // 3296e4773SMircea Trofin // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4296e4773SMircea Trofin // See https://llvm.org/LICENSE.txt for license information. 5296e4773SMircea Trofin // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 648fa355eSMircea Trofin // 748fa355eSMircea Trofin //===----------------------------------------------------------------------===// 848fa355eSMircea Trofin // 9d6695e18SMircea Trofin // This file implements InlineAdvisorAnalysis and DefaultInlineAdvisor, and 10d6695e18SMircea Trofin // related types. 1148fa355eSMircea Trofin // 1248fa355eSMircea Trofin //===----------------------------------------------------------------------===// 1348fa355eSMircea Trofin 1448fa355eSMircea Trofin #include "llvm/Analysis/InlineAdvisor.h" 1548fa355eSMircea Trofin #include "llvm/ADT/Statistic.h" 16b0abd489SElliot Goodrich #include "llvm/ADT/StringExtras.h" 1771c3a551Sserge-sans-paille #include "llvm/Analysis/AssumptionCache.h" 1848fa355eSMircea Trofin #include "llvm/Analysis/InlineCost.h" 1948fa355eSMircea Trofin #include "llvm/Analysis/OptimizationRemarkEmitter.h" 2048fa355eSMircea Trofin #include "llvm/Analysis/ProfileSummaryInfo.h" 215caad9b5Smodimo #include "llvm/Analysis/ReplayInlineAdvisor.h" 2248fa355eSMircea Trofin #include "llvm/Analysis/TargetLibraryInfo.h" 2348fa355eSMircea Trofin #include "llvm/Analysis/TargetTransformInfo.h" 2471c3a551Sserge-sans-paille #include "llvm/Analysis/Utils/ImportedFunctionsInliningStatistics.h" 257c8a6936SWenlei He #include "llvm/IR/DebugInfoMetadata.h" 264169338eSNikita Popov #include "llvm/IR/Module.h" 273e8553aaSMircea Trofin #include "llvm/IR/PassManager.h" 28cdceef4aSSimon Pilgrim #include "llvm/Support/CommandLine.h" 2948fa355eSMircea Trofin #include "llvm/Support/raw_ostream.h" 3048fa355eSMircea Trofin 3148fa355eSMircea Trofin using namespace llvm; 3248fa355eSMircea Trofin #define DEBUG_TYPE "inline" 33f29256a6SMircea Trofin #ifdef LLVM_HAVE_TF_AOT_INLINERSIZEMODEL 34f29256a6SMircea Trofin #define LLVM_HAVE_TF_AOT 35f29256a6SMircea Trofin #endif 3648fa355eSMircea Trofin 3748fa355eSMircea Trofin // This weirdly named statistic tracks the number of times that, when attempting 3848fa355eSMircea Trofin // to inline a function A into B, we analyze the callers of B in order to see 3948fa355eSMircea Trofin // if those would be more profitable and blocked inline steps. 4048fa355eSMircea Trofin STATISTIC(NumCallerCallersAnalyzed, "Number of caller-callers analyzed"); 4148fa355eSMircea Trofin 4248fa355eSMircea Trofin /// Flag to add inline messages as callsite attributes 'inline-remark'. 4348fa355eSMircea Trofin static cl::opt<bool> 4448fa355eSMircea Trofin InlineRemarkAttribute("inline-remark-attribute", cl::init(false), 4548fa355eSMircea Trofin cl::Hidden, 4648fa355eSMircea Trofin cl::desc("Enable adding inline-remark attribute to" 4748fa355eSMircea Trofin " callsites processed by inliner but decided" 4848fa355eSMircea Trofin " to be not inlined")); 4948fa355eSMircea Trofin 50a8c2ba10SNikita Popov static cl::opt<bool> EnableInlineDeferral("inline-deferral", cl::init(false), 517abf299fSNikita Popov cl::Hidden, 527abf299fSNikita Popov cl::desc("Enable deferred inlining")); 537abf299fSNikita Popov 5448fa355eSMircea Trofin // An integer used to limit the cost of inline deferral. The default negative 5548fa355eSMircea Trofin // number tells shouldBeDeferred to only take the secondary cost into account. 5648fa355eSMircea Trofin static cl::opt<int> 5748fa355eSMircea Trofin InlineDeferralScale("inline-deferral-scale", 5848fa355eSMircea Trofin cl::desc("Scale to limit the cost of inline deferral"), 59cec20db5SKazu Hirata cl::init(2), cl::Hidden); 6048fa355eSMircea Trofin 6130d3f56eSKazu Hirata static cl::opt<bool> 6230d3f56eSKazu Hirata AnnotateInlinePhase("annotate-inline-phase", cl::Hidden, cl::init(false), 63e0d06959SMingming Liu cl::desc("If true, annotate inline advisor remarks " 64e0d06959SMingming Liu "with LTO and pass information.")); 65e0d06959SMingming Liu 66fa71c164SFangrui Song namespace llvm { 67ccec2cf1SMircea Trofin extern cl::opt<InlinerFunctionImportStatsOpts> InlinerFunctionImportStats; 687b57a1b4SMohammed Keyvanzadeh } // namespace llvm 69ccec2cf1SMircea Trofin 707d541eb4SMircea Trofin namespace { 717d541eb4SMircea Trofin using namespace llvm::ore; 727d541eb4SMircea Trofin class MandatoryInlineAdvice : public InlineAdvice { 737d541eb4SMircea Trofin public: 747d541eb4SMircea Trofin MandatoryInlineAdvice(InlineAdvisor *Advisor, CallBase &CB, 757d541eb4SMircea Trofin OptimizationRemarkEmitter &ORE, 767d541eb4SMircea Trofin bool IsInliningMandatory) 777d541eb4SMircea Trofin : InlineAdvice(Advisor, CB, ORE, IsInliningMandatory) {} 787d541eb4SMircea Trofin 797d541eb4SMircea Trofin private: 807d541eb4SMircea Trofin void recordInliningWithCalleeDeletedImpl() override { recordInliningImpl(); } 817d541eb4SMircea Trofin 827d541eb4SMircea Trofin void recordInliningImpl() override { 837d541eb4SMircea Trofin if (IsInliningRecommended) 847d541eb4SMircea Trofin emitInlinedInto(ORE, DLoc, Block, *Callee, *Caller, IsInliningRecommended, 857d541eb4SMircea Trofin [&](OptimizationRemark &Remark) { 867d541eb4SMircea Trofin Remark << ": always inline attribute"; 877d541eb4SMircea Trofin }); 887d541eb4SMircea Trofin } 897d541eb4SMircea Trofin 907d541eb4SMircea Trofin void recordUnsuccessfulInliningImpl(const InlineResult &Result) override { 917d541eb4SMircea Trofin if (IsInliningRecommended) 927d541eb4SMircea Trofin ORE.emit([&]() { 938601f269SMingming Liu return OptimizationRemarkMissed(Advisor->getAnnotatedInlinePassName(), 948601f269SMingming Liu "NotInlined", DLoc, Block) 957d541eb4SMircea Trofin << "'" << NV("Callee", Callee) << "' is not AlwaysInline into '" 967d541eb4SMircea Trofin << NV("Caller", Caller) 977d541eb4SMircea Trofin << "': " << NV("Reason", Result.getFailureReason()); 987d541eb4SMircea Trofin }); 997d541eb4SMircea Trofin } 1007d541eb4SMircea Trofin 1017d541eb4SMircea Trofin void recordUnattemptedInliningImpl() override { 1027d541eb4SMircea Trofin assert(!IsInliningRecommended && "Expected to attempt inlining"); 1037d541eb4SMircea Trofin } 1047d541eb4SMircea Trofin }; 1057d541eb4SMircea Trofin } // namespace 1067d541eb4SMircea Trofin 1072a49b7c6Smodimo void DefaultInlineAdvice::recordUnsuccessfulInliningImpl( 1082a49b7c6Smodimo const InlineResult &Result) { 109d6695e18SMircea Trofin using namespace ore; 110d6695e18SMircea Trofin llvm::setInlineRemark(*OriginalCB, std::string(Result.getFailureReason()) + 111d6695e18SMircea Trofin "; " + inlineCostStr(*OIC)); 112d6695e18SMircea Trofin ORE.emit([&]() { 1138601f269SMingming Liu return OptimizationRemarkMissed(Advisor->getAnnotatedInlinePassName(), 1148601f269SMingming Liu "NotInlined", DLoc, Block) 11576093b17SFangrui Song << "'" << NV("Callee", Callee) << "' is not inlined into '" 1160bb767e7SFangrui Song << NV("Caller", Caller) 11776093b17SFangrui Song << "': " << NV("Reason", Result.getFailureReason()); 118d6695e18SMircea Trofin }); 119d6695e18SMircea Trofin } 120d6695e18SMircea Trofin 1212a49b7c6Smodimo void DefaultInlineAdvice::recordInliningWithCalleeDeletedImpl() { 1222a49b7c6Smodimo if (EmitRemarks) 1238601f269SMingming Liu emitInlinedIntoBasedOnCost(ORE, DLoc, Block, *Callee, *Caller, *OIC, 1248601f269SMingming Liu /* ForProfileContext= */ false, 1258601f269SMingming Liu Advisor->getAnnotatedInlinePassName()); 126d6695e18SMircea Trofin } 127d6695e18SMircea Trofin 1282a49b7c6Smodimo void DefaultInlineAdvice::recordInliningImpl() { 1292a49b7c6Smodimo if (EmitRemarks) 1308601f269SMingming Liu emitInlinedIntoBasedOnCost(ORE, DLoc, Block, *Callee, *Caller, *OIC, 1318601f269SMingming Liu /* ForProfileContext= */ false, 1328601f269SMingming Liu Advisor->getAnnotatedInlinePassName()); 133d6695e18SMircea Trofin } 134d6695e18SMircea Trofin 135d4b6fcb3SFangrui Song std::optional<llvm::InlineCost> static getDefaultInlineAdvice( 1369a0689e0SBenjamin Kramer CallBase &CB, FunctionAnalysisManager &FAM, const InlineParams &Params) { 1378a2e2a6aSMircea Trofin Function &Caller = *CB.getCaller(); 1388a2e2a6aSMircea Trofin ProfileSummaryInfo *PSI = 1398a2e2a6aSMircea Trofin FAM.getResult<ModuleAnalysisManagerFunctionProxy>(Caller) 140d6695e18SMircea Trofin .getCachedResult<ProfileSummaryAnalysis>( 141d6695e18SMircea Trofin *CB.getParent()->getParent()->getParent()); 142d6695e18SMircea Trofin 1438a2e2a6aSMircea Trofin auto &ORE = FAM.getResult<OptimizationRemarkEmitterAnalysis>(Caller); 14408e2386dSMircea Trofin auto GetAssumptionCache = [&](Function &F) -> AssumptionCache & { 1458a2e2a6aSMircea Trofin return FAM.getResult<AssumptionAnalysis>(F); 146d6695e18SMircea Trofin }; 147d6695e18SMircea Trofin auto GetBFI = [&](Function &F) -> BlockFrequencyInfo & { 148d6695e18SMircea Trofin return FAM.getResult<BlockFrequencyAnalysis>(F); 149d6695e18SMircea Trofin }; 150d6695e18SMircea Trofin auto GetTLI = [&](Function &F) -> const TargetLibraryInfo & { 151d6695e18SMircea Trofin return FAM.getResult<TargetLibraryAnalysis>(F); 152d6695e18SMircea Trofin }; 153d6695e18SMircea Trofin 154d6695e18SMircea Trofin Function &Callee = *CB.getCalledFunction(); 155d6695e18SMircea Trofin auto &CalleeTTI = FAM.getResult<TargetIRAnalysis>(Callee); 156e34e27f1SShilei Tian auto GetInlineCost = [&](CallBase &CB) { 157d6695e18SMircea Trofin bool RemarksEnabled = 158d6695e18SMircea Trofin Callee.getContext().getDiagHandlerPtr()->isMissedOptRemarkEnabled( 159d6695e18SMircea Trofin DEBUG_TYPE); 16008e2386dSMircea Trofin return getInlineCost(CB, Params, CalleeTTI, GetAssumptionCache, GetTLI, 16108e2386dSMircea Trofin GetBFI, PSI, RemarksEnabled ? &ORE : nullptr); 162d6695e18SMircea Trofin }; 1637abf299fSNikita Popov return llvm::shouldInline( 164e34e27f1SShilei Tian CB, CalleeTTI, GetInlineCost, ORE, 165129b531cSKazu Hirata Params.EnableDeferral.value_or(EnableInlineDeferral)); 16611046ef6SMircea Trofin } 16711046ef6SMircea Trofin 168e8049dc3SMircea Trofin std::unique_ptr<InlineAdvice> 169e8049dc3SMircea Trofin DefaultInlineAdvisor::getAdviceImpl(CallBase &CB) { 17011046ef6SMircea Trofin auto OIC = getDefaultInlineAdvice(CB, FAM, Params); 17111046ef6SMircea Trofin return std::make_unique<DefaultInlineAdvice>( 17211046ef6SMircea Trofin this, CB, OIC, 17311046ef6SMircea Trofin FAM.getResult<OptimizationRemarkEmitterAnalysis>(*CB.getCaller())); 174d6695e18SMircea Trofin } 175d6695e18SMircea Trofin 176d6695e18SMircea Trofin InlineAdvice::InlineAdvice(InlineAdvisor *Advisor, CallBase &CB, 177e82eff7aSMircea Trofin OptimizationRemarkEmitter &ORE, 178d6695e18SMircea Trofin bool IsInliningRecommended) 179d6695e18SMircea Trofin : Advisor(Advisor), Caller(CB.getCaller()), Callee(CB.getCalledFunction()), 180e82eff7aSMircea Trofin DLoc(CB.getDebugLoc()), Block(CB.getParent()), ORE(ORE), 181d6695e18SMircea Trofin IsInliningRecommended(IsInliningRecommended) {} 182d6695e18SMircea Trofin 183ccec2cf1SMircea Trofin void InlineAdvice::recordInlineStatsIfNeeded() { 184ccec2cf1SMircea Trofin if (Advisor->ImportedFunctionsStats) 185ccec2cf1SMircea Trofin Advisor->ImportedFunctionsStats->recordInline(*Caller, *Callee); 186ccec2cf1SMircea Trofin } 187ccec2cf1SMircea Trofin 188ccec2cf1SMircea Trofin void InlineAdvice::recordInlining() { 189ccec2cf1SMircea Trofin markRecorded(); 190ccec2cf1SMircea Trofin recordInlineStatsIfNeeded(); 191ccec2cf1SMircea Trofin recordInliningImpl(); 192ccec2cf1SMircea Trofin } 193ccec2cf1SMircea Trofin 194d6695e18SMircea Trofin void InlineAdvice::recordInliningWithCalleeDeleted() { 195d6695e18SMircea Trofin markRecorded(); 196ccec2cf1SMircea Trofin recordInlineStatsIfNeeded(); 197d6695e18SMircea Trofin recordInliningWithCalleeDeletedImpl(); 198d6695e18SMircea Trofin } 199d6695e18SMircea Trofin 200d6695e18SMircea Trofin AnalysisKey InlineAdvisorAnalysis::Key; 20107af0e2dSibricchi AnalysisKey PluginInlineAdvisorAnalysis::Key; 202d6695e18SMircea Trofin 2035caad9b5Smodimo bool InlineAdvisorAnalysis::Result::tryCreate( 2045caad9b5Smodimo InlineParams Params, InliningAdvisorMode Mode, 205e0d06959SMingming Liu const ReplayInlinerSettings &ReplaySettings, InlineContext IC) { 206999ea25aSMircea Trofin auto &FAM = MAM.getResult<FunctionAnalysisManagerModuleProxy>(M).getManager(); 207*ab4253f6SMichele Scandale if (MAM.isPassRegistered<PluginInlineAdvisorAnalysis>()) { 20807af0e2dSibricchi auto &DA = MAM.getResult<PluginInlineAdvisorAnalysis>(M); 20907af0e2dSibricchi Advisor.reset(DA.Factory(M, FAM, Params, IC)); 21007af0e2dSibricchi return !!Advisor; 21107af0e2dSibricchi } 212ab2e7666SMircea Trofin auto GetDefaultAdvice = [&FAM, Params](CallBase &CB) { 213ab2e7666SMircea Trofin auto OIC = getDefaultInlineAdvice(CB, FAM, Params); 214ab2e7666SMircea Trofin return OIC.has_value(); 215ab2e7666SMircea Trofin }; 216d6695e18SMircea Trofin switch (Mode) { 217d6695e18SMircea Trofin case InliningAdvisorMode::Default: 218935dea2cSMircea Trofin LLVM_DEBUG(dbgs() << "Using default inliner heuristic.\n"); 219e0d06959SMingming Liu Advisor.reset(new DefaultInlineAdvisor(M, FAM, Params, IC)); 220ce7f9cdbSmodimo // Restrict replay to default advisor, ML advisors are stateful so 221ce7f9cdbSmodimo // replay will need augmentations to interleave with them correctly. 2225caad9b5Smodimo if (!ReplaySettings.ReplayFile.empty()) { 2235caad9b5Smodimo Advisor = llvm::getReplayInlineAdvisor(M, FAM, M.getContext(), 2245caad9b5Smodimo std::move(Advisor), ReplaySettings, 225e0d06959SMingming Liu /* EmitRemarks =*/true, IC); 226ce7f9cdbSmodimo } 227d6695e18SMircea Trofin break; 228d6695e18SMircea Trofin case InliningAdvisorMode::Development: 229edc83a15SKazu Hirata #ifdef LLVM_HAVE_TFLITE 230935dea2cSMircea Trofin LLVM_DEBUG(dbgs() << "Using development-mode inliner policy.\n"); 231ab2e7666SMircea Trofin Advisor = llvm::getDevelopmentModeAdvisor(M, MAM, GetDefaultAdvice); 23270f8d0acSMircea Trofin #endif 233d6695e18SMircea Trofin break; 234d6695e18SMircea Trofin case InliningAdvisorMode::Release: 235935dea2cSMircea Trofin LLVM_DEBUG(dbgs() << "Using release-mode inliner policy.\n"); 236ab2e7666SMircea Trofin Advisor = llvm::getReleaseModeAdvisor(M, MAM, GetDefaultAdvice); 237d6695e18SMircea Trofin break; 238d6695e18SMircea Trofin } 239ce7f9cdbSmodimo 240d6695e18SMircea Trofin return !!Advisor; 241d6695e18SMircea Trofin } 242d6695e18SMircea Trofin 24348fa355eSMircea Trofin /// Return true if inlining of CB can block the caller from being 24448fa355eSMircea Trofin /// inlined which is proved to be more beneficial. \p IC is the 24548fa355eSMircea Trofin /// estimated inline cost associated with callsite \p CB. 24648fa355eSMircea Trofin /// \p TotalSecondaryCost will be set to the estimated cost of inlining the 24748fa355eSMircea Trofin /// caller if \p CB is suppressed for inlining. 2480205fabeSKazu Hirata static bool 249e34e27f1SShilei Tian shouldBeDeferred(Function *Caller, TargetTransformInfo &CalleeTTI, 250e34e27f1SShilei Tian InlineCost IC, int &TotalSecondaryCost, 25148fa355eSMircea Trofin function_ref<InlineCost(CallBase &CB)> GetInlineCost) { 25248fa355eSMircea Trofin // For now we only handle local or inline functions. 25348fa355eSMircea Trofin if (!Caller->hasLocalLinkage() && !Caller->hasLinkOnceODRLinkage()) 25448fa355eSMircea Trofin return false; 25548fa355eSMircea Trofin // If the cost of inlining CB is non-positive, it is not going to prevent the 25648fa355eSMircea Trofin // caller from being inlined into its callers and hence we don't need to 25748fa355eSMircea Trofin // defer. 25848fa355eSMircea Trofin if (IC.getCost() <= 0) 25948fa355eSMircea Trofin return false; 26048fa355eSMircea Trofin // Try to detect the case where the current inlining candidate caller (call 26148fa355eSMircea Trofin // it B) is a static or linkonce-ODR function and is an inlining candidate 26248fa355eSMircea Trofin // elsewhere, and the current candidate callee (call it C) is large enough 26348fa355eSMircea Trofin // that inlining it into B would make B too big to inline later. In these 26448fa355eSMircea Trofin // circumstances it may be best not to inline C into B, but to inline B into 26548fa355eSMircea Trofin // its callers. 26648fa355eSMircea Trofin // 26748fa355eSMircea Trofin // This only applies to static and linkonce-ODR functions because those are 26848fa355eSMircea Trofin // expected to be available for inlining in the translation units where they 26948fa355eSMircea Trofin // are used. Thus we will always have the opportunity to make local inlining 27048fa355eSMircea Trofin // decisions. Importantly the linkonce-ODR linkage covers inline functions 27148fa355eSMircea Trofin // and templates in C++. 27248fa355eSMircea Trofin // 27348fa355eSMircea Trofin // FIXME: All of this logic should be sunk into getInlineCost. It relies on 27448fa355eSMircea Trofin // the internal implementation of the inline cost metrics rather than 27548fa355eSMircea Trofin // treating them as truly abstract units etc. 27648fa355eSMircea Trofin TotalSecondaryCost = 0; 27748fa355eSMircea Trofin // The candidate cost to be imposed upon the current function. 27848fa355eSMircea Trofin int CandidateCost = IC.getCost() - 1; 27948fa355eSMircea Trofin // If the caller has local linkage and can be inlined to all its callers, we 28048fa355eSMircea Trofin // can apply a huge negative bonus to TotalSecondaryCost. 28148fa355eSMircea Trofin bool ApplyLastCallBonus = Caller->hasLocalLinkage() && !Caller->hasOneUse(); 28248fa355eSMircea Trofin // This bool tracks what happens if we DO inline C into B. 28348fa355eSMircea Trofin bool InliningPreventsSomeOuterInline = false; 28448fa355eSMircea Trofin unsigned NumCallerUsers = 0; 28548fa355eSMircea Trofin for (User *U : Caller->users()) { 28648fa355eSMircea Trofin CallBase *CS2 = dyn_cast<CallBase>(U); 28748fa355eSMircea Trofin 28848fa355eSMircea Trofin // If this isn't a call to Caller (it could be some other sort 28948fa355eSMircea Trofin // of reference) skip it. Such references will prevent the caller 29048fa355eSMircea Trofin // from being removed. 29148fa355eSMircea Trofin if (!CS2 || CS2->getCalledFunction() != Caller) { 29248fa355eSMircea Trofin ApplyLastCallBonus = false; 29348fa355eSMircea Trofin continue; 29448fa355eSMircea Trofin } 29548fa355eSMircea Trofin 29648fa355eSMircea Trofin InlineCost IC2 = GetInlineCost(*CS2); 29748fa355eSMircea Trofin ++NumCallerCallersAnalyzed; 29848fa355eSMircea Trofin if (!IC2) { 29948fa355eSMircea Trofin ApplyLastCallBonus = false; 30048fa355eSMircea Trofin continue; 30148fa355eSMircea Trofin } 30248fa355eSMircea Trofin if (IC2.isAlways()) 30348fa355eSMircea Trofin continue; 30448fa355eSMircea Trofin 30548fa355eSMircea Trofin // See if inlining of the original callsite would erase the cost delta of 30648fa355eSMircea Trofin // this callsite. We subtract off the penalty for the call instruction, 30748fa355eSMircea Trofin // which we would be deleting. 30848fa355eSMircea Trofin if (IC2.getCostDelta() <= CandidateCost) { 30948fa355eSMircea Trofin InliningPreventsSomeOuterInline = true; 31048fa355eSMircea Trofin TotalSecondaryCost += IC2.getCost(); 31148fa355eSMircea Trofin NumCallerUsers++; 31248fa355eSMircea Trofin } 31348fa355eSMircea Trofin } 31448fa355eSMircea Trofin 31548fa355eSMircea Trofin if (!InliningPreventsSomeOuterInline) 31648fa355eSMircea Trofin return false; 31748fa355eSMircea Trofin 31848fa355eSMircea Trofin // If all outer calls to Caller would get inlined, the cost for the last 31948fa355eSMircea Trofin // one is set very low by getInlineCost, in anticipation that Caller will 32048fa355eSMircea Trofin // be removed entirely. We did not account for this above unless there 32148fa355eSMircea Trofin // is only one caller of Caller. 32248fa355eSMircea Trofin if (ApplyLastCallBonus) 323e34e27f1SShilei Tian TotalSecondaryCost -= CalleeTTI.getInliningLastCallToStaticBonus(); 32448fa355eSMircea Trofin 32548fa355eSMircea Trofin // If InlineDeferralScale is negative, then ignore the cost of primary 32648fa355eSMircea Trofin // inlining -- IC.getCost() multiplied by the number of callers to Caller. 32748fa355eSMircea Trofin if (InlineDeferralScale < 0) 32848fa355eSMircea Trofin return TotalSecondaryCost < IC.getCost(); 32948fa355eSMircea Trofin 33048fa355eSMircea Trofin int TotalCost = TotalSecondaryCost + IC.getCost() * NumCallerUsers; 33148fa355eSMircea Trofin int Allowance = IC.getCost() * InlineDeferralScale; 33248fa355eSMircea Trofin return TotalCost < Allowance; 33348fa355eSMircea Trofin } 33448fa355eSMircea Trofin 33548fa355eSMircea Trofin namespace llvm { 3361ce2b584Sserge-sans-paille static raw_ostream &operator<<(raw_ostream &R, const ore::NV &Arg) { 33748fa355eSMircea Trofin return R << Arg.Val; 33848fa355eSMircea Trofin } 33948fa355eSMircea Trofin 34048fa355eSMircea Trofin template <class RemarkT> 34148fa355eSMircea Trofin RemarkT &operator<<(RemarkT &&R, const InlineCost &IC) { 34248fa355eSMircea Trofin using namespace ore; 34348fa355eSMircea Trofin if (IC.isAlways()) { 34448fa355eSMircea Trofin R << "(cost=always)"; 34548fa355eSMircea Trofin } else if (IC.isNever()) { 34648fa355eSMircea Trofin R << "(cost=never)"; 34748fa355eSMircea Trofin } else { 34848fa355eSMircea Trofin R << "(cost=" << ore::NV("Cost", IC.getCost()) 34948fa355eSMircea Trofin << ", threshold=" << ore::NV("Threshold", IC.getThreshold()) << ")"; 35048fa355eSMircea Trofin } 35148fa355eSMircea Trofin if (const char *Reason = IC.getReason()) 35248fa355eSMircea Trofin R << ": " << ore::NV("Reason", Reason); 35348fa355eSMircea Trofin return R; 35448fa355eSMircea Trofin } 35548fa355eSMircea Trofin } // namespace llvm 35648fa355eSMircea Trofin 35748fa355eSMircea Trofin std::string llvm::inlineCostStr(const InlineCost &IC) { 3581ce2b584Sserge-sans-paille std::string Buffer; 3591ce2b584Sserge-sans-paille raw_string_ostream Remark(Buffer); 36048fa355eSMircea Trofin Remark << IC; 36148fa355eSMircea Trofin return Remark.str(); 36248fa355eSMircea Trofin } 36348fa355eSMircea Trofin 36448fa355eSMircea Trofin void llvm::setInlineRemark(CallBase &CB, StringRef Message) { 36548fa355eSMircea Trofin if (!InlineRemarkAttribute) 36648fa355eSMircea Trofin return; 36748fa355eSMircea Trofin 36848fa355eSMircea Trofin Attribute Attr = Attribute::get(CB.getContext(), "inline-remark", Message); 3693f4d00bcSArthur Eubanks CB.addFnAttr(Attr); 37048fa355eSMircea Trofin } 37148fa355eSMircea Trofin 37248fa355eSMircea Trofin /// Return the cost only if the inliner should attempt to inline at the given 37348fa355eSMircea Trofin /// CallSite. If we return the cost, we will emit an optimisation remark later 3749f252e55SKazu Hirata /// using that cost, so we won't do so from this function. Return std::nullopt 3759f252e55SKazu Hirata /// if inlining should not be attempted. 376d4b6fcb3SFangrui Song std::optional<InlineCost> 377e34e27f1SShilei Tian llvm::shouldInline(CallBase &CB, TargetTransformInfo &CalleeTTI, 37848fa355eSMircea Trofin function_ref<InlineCost(CallBase &CB)> GetInlineCost, 379347a599eSKazu Hirata OptimizationRemarkEmitter &ORE, bool EnableDeferral) { 38048fa355eSMircea Trofin using namespace ore; 38148fa355eSMircea Trofin 38248fa355eSMircea Trofin InlineCost IC = GetInlineCost(CB); 38348fa355eSMircea Trofin Instruction *Call = &CB; 38448fa355eSMircea Trofin Function *Callee = CB.getCalledFunction(); 38548fa355eSMircea Trofin Function *Caller = CB.getCaller(); 38648fa355eSMircea Trofin 38748fa355eSMircea Trofin if (IC.isAlways()) { 38848fa355eSMircea Trofin LLVM_DEBUG(dbgs() << " Inlining " << inlineCostStr(IC) 38948fa355eSMircea Trofin << ", Call: " << CB << "\n"); 39048fa355eSMircea Trofin return IC; 39148fa355eSMircea Trofin } 39248fa355eSMircea Trofin 39348fa355eSMircea Trofin if (!IC) { 39448fa355eSMircea Trofin LLVM_DEBUG(dbgs() << " NOT Inlining " << inlineCostStr(IC) 39548fa355eSMircea Trofin << ", Call: " << CB << "\n"); 39648fa355eSMircea Trofin if (IC.isNever()) { 39748fa355eSMircea Trofin ORE.emit([&]() { 39848fa355eSMircea Trofin return OptimizationRemarkMissed(DEBUG_TYPE, "NeverInline", Call) 39976093b17SFangrui Song << "'" << NV("Callee", Callee) << "' not inlined into '" 40076093b17SFangrui Song << NV("Caller", Caller) 40176093b17SFangrui Song << "' because it should never be inlined " << IC; 40248fa355eSMircea Trofin }); 40348fa355eSMircea Trofin } else { 40448fa355eSMircea Trofin ORE.emit([&]() { 40548fa355eSMircea Trofin return OptimizationRemarkMissed(DEBUG_TYPE, "TooCostly", Call) 40676093b17SFangrui Song << "'" << NV("Callee", Callee) << "' not inlined into '" 40776093b17SFangrui Song << NV("Caller", Caller) << "' because too costly to inline " 40848fa355eSMircea Trofin << IC; 40948fa355eSMircea Trofin }); 41048fa355eSMircea Trofin } 41148fa355eSMircea Trofin setInlineRemark(CB, inlineCostStr(IC)); 41219aff0f3SKazu Hirata return std::nullopt; 41348fa355eSMircea Trofin } 41448fa355eSMircea Trofin 41548fa355eSMircea Trofin int TotalSecondaryCost = 0; 416e34e27f1SShilei Tian if (EnableDeferral && shouldBeDeferred(Caller, CalleeTTI, IC, 417e34e27f1SShilei Tian TotalSecondaryCost, GetInlineCost)) { 41848fa355eSMircea Trofin LLVM_DEBUG(dbgs() << " NOT Inlining: " << CB 41948fa355eSMircea Trofin << " Cost = " << IC.getCost() 42048fa355eSMircea Trofin << ", outer Cost = " << TotalSecondaryCost << '\n'); 42148fa355eSMircea Trofin ORE.emit([&]() { 42248fa355eSMircea Trofin return OptimizationRemarkMissed(DEBUG_TYPE, "IncreaseCostInOtherContexts", 42348fa355eSMircea Trofin Call) 42476093b17SFangrui Song << "Not inlining. Cost of inlining '" << NV("Callee", Callee) 42576093b17SFangrui Song << "' increases the cost of inlining '" << NV("Caller", Caller) 42676093b17SFangrui Song << "' in other contexts"; 42748fa355eSMircea Trofin }); 42848fa355eSMircea Trofin setInlineRemark(CB, "deferred"); 42919aff0f3SKazu Hirata return std::nullopt; 43048fa355eSMircea Trofin } 43148fa355eSMircea Trofin 43248fa355eSMircea Trofin LLVM_DEBUG(dbgs() << " Inlining " << inlineCostStr(IC) << ", Call: " << CB 43348fa355eSMircea Trofin << '\n'); 43448fa355eSMircea Trofin return IC; 43548fa355eSMircea Trofin } 43648fa355eSMircea Trofin 4375caad9b5Smodimo std::string llvm::formatCallSiteLocation(DebugLoc DLoc, 4385caad9b5Smodimo const CallSiteFormat &Format) { 4391ce2b584Sserge-sans-paille std::string Buffer; 4401ce2b584Sserge-sans-paille raw_string_ostream CallSiteLoc(Buffer); 441577e58bcSWenlei He bool First = true; 442577e58bcSWenlei He for (DILocation *DIL = DLoc.get(); DIL; DIL = DIL->getInlinedAt()) { 443577e58bcSWenlei He if (!First) 444577e58bcSWenlei He CallSiteLoc << " @ "; 445577e58bcSWenlei He // Note that negative line offset is actually possible, but we use 446577e58bcSWenlei He // unsigned int to match line offset representation in remarks so 447577e58bcSWenlei He // it's directly consumable by relay advisor. 448577e58bcSWenlei He uint32_t Offset = 449577e58bcSWenlei He DIL->getLine() - DIL->getScope()->getSubprogram()->getLine(); 450577e58bcSWenlei He uint32_t Discriminator = DIL->getBaseDiscriminator(); 451577e58bcSWenlei He StringRef Name = DIL->getScope()->getSubprogram()->getLinkageName(); 452577e58bcSWenlei He if (Name.empty()) 453577e58bcSWenlei He Name = DIL->getScope()->getSubprogram()->getName(); 4545caad9b5Smodimo CallSiteLoc << Name.str() << ":" << llvm::utostr(Offset); 4555caad9b5Smodimo if (Format.outputColumn()) 4565caad9b5Smodimo CallSiteLoc << ":" << llvm::utostr(DIL->getColumn()); 4575caad9b5Smodimo if (Format.outputDiscriminator() && Discriminator) 458577e58bcSWenlei He CallSiteLoc << "." << llvm::utostr(Discriminator); 459577e58bcSWenlei He First = false; 460577e58bcSWenlei He } 461577e58bcSWenlei He 462577e58bcSWenlei He return CallSiteLoc.str(); 463577e58bcSWenlei He } 464577e58bcSWenlei He 4657c8a6936SWenlei He void llvm::addLocationToRemarks(OptimizationRemark &Remark, DebugLoc DLoc) { 4669aa52ba5SKazu Hirata if (!DLoc) { 4677c8a6936SWenlei He return; 468577e58bcSWenlei He } 4697c8a6936SWenlei He 4707c8a6936SWenlei He bool First = true; 4717c8a6936SWenlei He Remark << " at callsite "; 4727c8a6936SWenlei He for (DILocation *DIL = DLoc.get(); DIL; DIL = DIL->getInlinedAt()) { 4737c8a6936SWenlei He if (!First) 4747c8a6936SWenlei He Remark << " @ "; 4757c8a6936SWenlei He unsigned int Offset = DIL->getLine(); 4767c8a6936SWenlei He Offset -= DIL->getScope()->getSubprogram()->getLine(); 4777c8a6936SWenlei He unsigned int Discriminator = DIL->getBaseDiscriminator(); 4787c8a6936SWenlei He StringRef Name = DIL->getScope()->getSubprogram()->getLinkageName(); 4797c8a6936SWenlei He if (Name.empty()) 4807c8a6936SWenlei He Name = DIL->getScope()->getSubprogram()->getName(); 4812a49b7c6Smodimo Remark << Name << ":" << ore::NV("Line", Offset) << ":" 4822a49b7c6Smodimo << ore::NV("Column", DIL->getColumn()); 4837c8a6936SWenlei He if (Discriminator) 4847c8a6936SWenlei He Remark << "." << ore::NV("Disc", Discriminator); 4857c8a6936SWenlei He First = false; 4867c8a6936SWenlei He } 4872a49b7c6Smodimo 4882a49b7c6Smodimo Remark << ";"; 4897c8a6936SWenlei He } 4907c8a6936SWenlei He 4917d541eb4SMircea Trofin void llvm::emitInlinedInto( 4927d541eb4SMircea Trofin OptimizationRemarkEmitter &ORE, DebugLoc DLoc, const BasicBlock *Block, 4937d541eb4SMircea Trofin const Function &Callee, const Function &Caller, bool AlwaysInline, 4947d541eb4SMircea Trofin function_ref<void(OptimizationRemark &)> ExtraContext, 4957d541eb4SMircea Trofin const char *PassName) { 49648fa355eSMircea Trofin ORE.emit([&]() { 49748fa355eSMircea Trofin StringRef RemarkName = AlwaysInline ? "AlwaysInline" : "Inlined"; 4987c8a6936SWenlei He OptimizationRemark Remark(PassName ? PassName : DEBUG_TYPE, RemarkName, 4997c8a6936SWenlei He DLoc, Block); 50076093b17SFangrui Song Remark << "'" << ore::NV("Callee", &Callee) << "' inlined into '" 50176093b17SFangrui Song << ore::NV("Caller", &Caller) << "'"; 5027d541eb4SMircea Trofin if (ExtraContext) 5037d541eb4SMircea Trofin ExtraContext(Remark); 5047c8a6936SWenlei He addLocationToRemarks(Remark, DLoc); 5057c8a6936SWenlei He return Remark; 50648fa355eSMircea Trofin }); 50748fa355eSMircea Trofin } 5085fe10263SMircea Trofin 5097d541eb4SMircea Trofin void llvm::emitInlinedIntoBasedOnCost( 5107d541eb4SMircea Trofin OptimizationRemarkEmitter &ORE, DebugLoc DLoc, const BasicBlock *Block, 5117d541eb4SMircea Trofin const Function &Callee, const Function &Caller, const InlineCost &IC, 5127d541eb4SMircea Trofin bool ForProfileContext, const char *PassName) { 5137d541eb4SMircea Trofin llvm::emitInlinedInto( 5147d541eb4SMircea Trofin ORE, DLoc, Block, Callee, Caller, IC.isAlways(), 5157d541eb4SMircea Trofin [&](OptimizationRemark &Remark) { 5167d541eb4SMircea Trofin if (ForProfileContext) 5177d541eb4SMircea Trofin Remark << " to match profiling context"; 5187d541eb4SMircea Trofin Remark << " with " << IC; 5197d541eb4SMircea Trofin }, 5207d541eb4SMircea Trofin PassName); 5217d541eb4SMircea Trofin } 5227d541eb4SMircea Trofin 5238601f269SMingming Liu InlineAdvisor::InlineAdvisor(Module &M, FunctionAnalysisManager &FAM, 524d4b6fcb3SFangrui Song std::optional<InlineContext> IC) 525bc856eb3SMingming Liu : M(M), FAM(FAM), IC(IC), 52630d3f56eSKazu Hirata AnnotatedInlinePassName((IC && AnnotateInlinePhase) 52730d3f56eSKazu Hirata ? llvm::AnnotateInlinePassName(*IC) 528bc856eb3SMingming Liu : DEBUG_TYPE) { 529ccec2cf1SMircea Trofin if (InlinerFunctionImportStats != InlinerFunctionImportStatsOpts::No) { 530ccec2cf1SMircea Trofin ImportedFunctionsStats = 531ccec2cf1SMircea Trofin std::make_unique<ImportedFunctionsInliningStatistics>(); 532ccec2cf1SMircea Trofin ImportedFunctionsStats->setModuleInfo(M); 533ccec2cf1SMircea Trofin } 534ccec2cf1SMircea Trofin } 535ccec2cf1SMircea Trofin 536ccec2cf1SMircea Trofin InlineAdvisor::~InlineAdvisor() { 537ccec2cf1SMircea Trofin if (ImportedFunctionsStats) { 538ccec2cf1SMircea Trofin assert(InlinerFunctionImportStats != InlinerFunctionImportStatsOpts::No); 539ccec2cf1SMircea Trofin ImportedFunctionsStats->dump(InlinerFunctionImportStats == 540ccec2cf1SMircea Trofin InlinerFunctionImportStatsOpts::Verbose); 541ccec2cf1SMircea Trofin } 542ccec2cf1SMircea Trofin } 543ccec2cf1SMircea Trofin 544e8049dc3SMircea Trofin std::unique_ptr<InlineAdvice> InlineAdvisor::getMandatoryAdvice(CallBase &CB, 545e8049dc3SMircea Trofin bool Advice) { 5467d541eb4SMircea Trofin return std::make_unique<MandatoryInlineAdvice>(this, CB, getCallerORE(CB), 5477d541eb4SMircea Trofin Advice); 5485fe10263SMircea Trofin } 5495fe10263SMircea Trofin 5508601f269SMingming Liu static inline const char *getLTOPhase(ThinOrFullLTOPhase LTOPhase) { 5518601f269SMingming Liu switch (LTOPhase) { 5528601f269SMingming Liu case (ThinOrFullLTOPhase::None): 5538601f269SMingming Liu return "main"; 5548601f269SMingming Liu case (ThinOrFullLTOPhase::ThinLTOPreLink): 5558601f269SMingming Liu case (ThinOrFullLTOPhase::FullLTOPreLink): 5568601f269SMingming Liu return "prelink"; 5578601f269SMingming Liu case (ThinOrFullLTOPhase::ThinLTOPostLink): 5588601f269SMingming Liu case (ThinOrFullLTOPhase::FullLTOPostLink): 5598601f269SMingming Liu return "postlink"; 5608601f269SMingming Liu } 5618601f269SMingming Liu llvm_unreachable("unreachable"); 5628601f269SMingming Liu } 5638601f269SMingming Liu 5648601f269SMingming Liu static inline const char *getInlineAdvisorContext(InlinePass IP) { 5658601f269SMingming Liu switch (IP) { 5668601f269SMingming Liu case (InlinePass::AlwaysInliner): 5678601f269SMingming Liu return "always-inline"; 5688601f269SMingming Liu case (InlinePass::CGSCCInliner): 5698601f269SMingming Liu return "cgscc-inline"; 5708601f269SMingming Liu case (InlinePass::EarlyInliner): 5718601f269SMingming Liu return "early-inline"; 5728601f269SMingming Liu case (InlinePass::MLInliner): 5738601f269SMingming Liu return "ml-inline"; 5748601f269SMingming Liu case (InlinePass::ModuleInliner): 5758601f269SMingming Liu return "module-inline"; 5768601f269SMingming Liu case (InlinePass::ReplayCGSCCInliner): 5778601f269SMingming Liu return "replay-cgscc-inline"; 5788601f269SMingming Liu case (InlinePass::ReplaySampleProfileInliner): 5798601f269SMingming Liu return "replay-sample-profile-inline"; 5808601f269SMingming Liu case (InlinePass::SampleProfileInliner): 5818601f269SMingming Liu return "sample-profile-inline"; 5828601f269SMingming Liu } 5838601f269SMingming Liu 5848601f269SMingming Liu llvm_unreachable("unreachable"); 5858601f269SMingming Liu } 5868601f269SMingming Liu 5878601f269SMingming Liu std::string llvm::AnnotateInlinePassName(InlineContext IC) { 5888601f269SMingming Liu return std::string(getLTOPhase(IC.LTOPhase)) + "-" + 5898601f269SMingming Liu std::string(getInlineAdvisorContext(IC.Pass)); 5908601f269SMingming Liu } 5918601f269SMingming Liu 592e8049dc3SMircea Trofin InlineAdvisor::MandatoryInliningKind 593e8049dc3SMircea Trofin InlineAdvisor::getMandatoryKind(CallBase &CB, FunctionAnalysisManager &FAM, 5945fe10263SMircea Trofin OptimizationRemarkEmitter &ORE) { 5955fe10263SMircea Trofin auto &Callee = *CB.getCalledFunction(); 5965fe10263SMircea Trofin 5975fe10263SMircea Trofin auto GetTLI = [&](Function &F) -> const TargetLibraryInfo & { 5985fe10263SMircea Trofin return FAM.getResult<TargetLibraryAnalysis>(F); 5995fe10263SMircea Trofin }; 6005fe10263SMircea Trofin 6015fe10263SMircea Trofin auto &TIR = FAM.getResult<TargetIRAnalysis>(Callee); 6025fe10263SMircea Trofin 6035fe10263SMircea Trofin auto TrivialDecision = 6045fe10263SMircea Trofin llvm::getAttributeBasedInliningDecision(CB, &Callee, TIR, GetTLI); 6055fe10263SMircea Trofin 606e0e687a6SKazu Hirata if (TrivialDecision) { 6075fe10263SMircea Trofin if (TrivialDecision->isSuccess()) 6085fe10263SMircea Trofin return MandatoryInliningKind::Always; 6095fe10263SMircea Trofin else 6105fe10263SMircea Trofin return MandatoryInliningKind::Never; 6115fe10263SMircea Trofin } 6125fe10263SMircea Trofin return MandatoryInliningKind::NotMandatory; 6135fe10263SMircea Trofin } 614e8049dc3SMircea Trofin 615e8049dc3SMircea Trofin std::unique_ptr<InlineAdvice> InlineAdvisor::getAdvice(CallBase &CB, 616e8049dc3SMircea Trofin bool MandatoryOnly) { 617e8049dc3SMircea Trofin if (!MandatoryOnly) 618e8049dc3SMircea Trofin return getAdviceImpl(CB); 619e8049dc3SMircea Trofin bool Advice = CB.getCaller() != CB.getCalledFunction() && 620e8049dc3SMircea Trofin MandatoryInliningKind::Always == 621e8049dc3SMircea Trofin getMandatoryKind(CB, FAM, getCallerORE(CB)); 622e8049dc3SMircea Trofin return getMandatoryAdvice(CB, Advice); 623e8049dc3SMircea Trofin } 624e8049dc3SMircea Trofin 625e8049dc3SMircea Trofin OptimizationRemarkEmitter &InlineAdvisor::getCallerORE(CallBase &CB) { 626e8049dc3SMircea Trofin return FAM.getResult<OptimizationRemarkEmitterAnalysis>(*CB.getCaller()); 627e8049dc3SMircea Trofin } 6283e8553aaSMircea Trofin 6293e8553aaSMircea Trofin PreservedAnalyses 6303e8553aaSMircea Trofin InlineAdvisorAnalysisPrinterPass::run(Module &M, ModuleAnalysisManager &MAM) { 6313e8553aaSMircea Trofin const auto *IA = MAM.getCachedResult<InlineAdvisorAnalysis>(M); 6323e8553aaSMircea Trofin if (!IA) 6333e8553aaSMircea Trofin OS << "No Inline Advisor\n"; 6343e8553aaSMircea Trofin else 6353e8553aaSMircea Trofin IA->getAdvisor()->print(OS); 6363e8553aaSMircea Trofin return PreservedAnalyses::all(); 6373e8553aaSMircea Trofin } 6389f2b873aSJin Xin Ng 6399f2b873aSJin Xin Ng PreservedAnalyses InlineAdvisorAnalysisPrinterPass::run( 6409f2b873aSJin Xin Ng LazyCallGraph::SCC &InitialC, CGSCCAnalysisManager &AM, LazyCallGraph &CG, 6419f2b873aSJin Xin Ng CGSCCUpdateResult &UR) { 6429f2b873aSJin Xin Ng const auto &MAMProxy = 6439f2b873aSJin Xin Ng AM.getResult<ModuleAnalysisManagerCGSCCProxy>(InitialC, CG); 6449f2b873aSJin Xin Ng 6459f2b873aSJin Xin Ng if (InitialC.size() == 0) { 6469f2b873aSJin Xin Ng OS << "SCC is empty!\n"; 6479f2b873aSJin Xin Ng return PreservedAnalyses::all(); 6489f2b873aSJin Xin Ng } 6499f2b873aSJin Xin Ng Module &M = *InitialC.begin()->getFunction().getParent(); 6509f2b873aSJin Xin Ng const auto *IA = MAMProxy.getCachedResult<InlineAdvisorAnalysis>(M); 6519f2b873aSJin Xin Ng if (!IA) 6529f2b873aSJin Xin Ng OS << "No Inline Advisor\n"; 6539f2b873aSJin Xin Ng else 6549f2b873aSJin Xin Ng IA->getAdvisor()->print(OS); 6559f2b873aSJin Xin Ng return PreservedAnalyses::all(); 6569f2b873aSJin Xin Ng } 657