1 //===- BlockFrequencyInfo.cpp - Block Frequency Analysis ------------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // Loops should be simplified before this analysis. 10 // 11 //===----------------------------------------------------------------------===// 12 13 #include "llvm/Analysis/BlockFrequencyInfo.h" 14 #include "llvm/ADT/APInt.h" 15 #include "llvm/ADT/None.h" 16 #include "llvm/ADT/iterator.h" 17 #include "llvm/Analysis/BlockFrequencyInfoImpl.h" 18 #include "llvm/Analysis/BranchProbabilityInfo.h" 19 #include "llvm/Analysis/LoopInfo.h" 20 #include "llvm/IR/CFG.h" 21 #include "llvm/IR/Function.h" 22 #include "llvm/IR/PassManager.h" 23 #include "llvm/Pass.h" 24 #include "llvm/Support/CommandLine.h" 25 #include "llvm/Support/GraphWriter.h" 26 #include "llvm/Support/raw_ostream.h" 27 #include <algorithm> 28 #include <cassert> 29 #include <string> 30 31 using namespace llvm; 32 33 #define DEBUG_TYPE "block-freq" 34 35 static cl::opt<GVDAGType> ViewBlockFreqPropagationDAG( 36 "view-block-freq-propagation-dags", cl::Hidden, 37 cl::desc("Pop up a window to show a dag displaying how block " 38 "frequencies propagation through the CFG."), 39 cl::values(clEnumValN(GVDT_None, "none", "do not display graphs."), 40 clEnumValN(GVDT_Fraction, "fraction", 41 "display a graph using the " 42 "fractional block frequency representation."), 43 clEnumValN(GVDT_Integer, "integer", 44 "display a graph using the raw " 45 "integer fractional block frequency representation."), 46 clEnumValN(GVDT_Count, "count", "display a graph using the real " 47 "profile count if available."))); 48 49 cl::opt<std::string> 50 ViewBlockFreqFuncName("view-bfi-func-name", cl::Hidden, 51 cl::desc("The option to specify " 52 "the name of the function " 53 "whose CFG will be displayed.")); 54 55 cl::opt<unsigned> 56 ViewHotFreqPercent("view-hot-freq-percent", cl::init(10), cl::Hidden, 57 cl::desc("An integer in percent used to specify " 58 "the hot blocks/edges to be displayed " 59 "in red: a block or edge whose frequency " 60 "is no less than the max frequency of the " 61 "function multiplied by this percent.")); 62 63 // Command line option to turn on CFG dot or text dump after profile annotation. 64 cl::opt<PGOViewCountsType> PGOViewCounts( 65 "pgo-view-counts", cl::Hidden, 66 cl::desc("A boolean option to show CFG dag or text with " 67 "block profile counts and branch probabilities " 68 "right after PGO profile annotation step. The " 69 "profile counts are computed using branch " 70 "probabilities from the runtime profile data and " 71 "block frequency propagation algorithm. To view " 72 "the raw counts from the profile, use option " 73 "-pgo-view-raw-counts instead. To limit graph " 74 "display to only one function, use filtering option " 75 "-view-bfi-func-name."), 76 cl::values(clEnumValN(PGOVCT_None, "none", "do not show."), 77 clEnumValN(PGOVCT_Graph, "graph", "show a graph."), 78 clEnumValN(PGOVCT_Text, "text", "show in text."))); 79 80 static cl::opt<bool> PrintBlockFreq( 81 "print-bfi", cl::init(false), cl::Hidden, 82 cl::desc("Print the block frequency info.")); 83 84 cl::opt<std::string> PrintBlockFreqFuncName( 85 "print-bfi-func-name", cl::Hidden, 86 cl::desc("The option to specify the name of the function " 87 "whose block frequency info is printed.")); 88 89 namespace llvm { 90 91 static GVDAGType getGVDT() { 92 if (PGOViewCounts == PGOVCT_Graph) 93 return GVDT_Count; 94 return ViewBlockFreqPropagationDAG; 95 } 96 97 template <> 98 struct GraphTraits<BlockFrequencyInfo *> { 99 using NodeRef = const BasicBlock *; 100 using ChildIteratorType = succ_const_iterator; 101 using nodes_iterator = pointer_iterator<Function::const_iterator>; 102 103 static NodeRef getEntryNode(const BlockFrequencyInfo *G) { 104 return &G->getFunction()->front(); 105 } 106 107 static ChildIteratorType child_begin(const NodeRef N) { 108 return succ_begin(N); 109 } 110 111 static ChildIteratorType child_end(const NodeRef N) { return succ_end(N); } 112 113 static nodes_iterator nodes_begin(const BlockFrequencyInfo *G) { 114 return nodes_iterator(G->getFunction()->begin()); 115 } 116 117 static nodes_iterator nodes_end(const BlockFrequencyInfo *G) { 118 return nodes_iterator(G->getFunction()->end()); 119 } 120 }; 121 122 using BFIDOTGTraitsBase = 123 BFIDOTGraphTraitsBase<BlockFrequencyInfo, BranchProbabilityInfo>; 124 125 template <> 126 struct DOTGraphTraits<BlockFrequencyInfo *> : public BFIDOTGTraitsBase { 127 explicit DOTGraphTraits(bool isSimple = false) 128 : BFIDOTGTraitsBase(isSimple) {} 129 130 std::string getNodeLabel(const BasicBlock *Node, 131 const BlockFrequencyInfo *Graph) { 132 133 return BFIDOTGTraitsBase::getNodeLabel(Node, Graph, getGVDT()); 134 } 135 136 std::string getNodeAttributes(const BasicBlock *Node, 137 const BlockFrequencyInfo *Graph) { 138 return BFIDOTGTraitsBase::getNodeAttributes(Node, Graph, 139 ViewHotFreqPercent); 140 } 141 142 std::string getEdgeAttributes(const BasicBlock *Node, EdgeIter EI, 143 const BlockFrequencyInfo *BFI) { 144 return BFIDOTGTraitsBase::getEdgeAttributes(Node, EI, BFI, BFI->getBPI(), 145 ViewHotFreqPercent); 146 } 147 }; 148 149 } // end namespace llvm 150 151 BlockFrequencyInfo::BlockFrequencyInfo() = default; 152 153 BlockFrequencyInfo::BlockFrequencyInfo(const Function &F, 154 const BranchProbabilityInfo &BPI, 155 const LoopInfo &LI) { 156 calculate(F, BPI, LI); 157 } 158 159 BlockFrequencyInfo::BlockFrequencyInfo(BlockFrequencyInfo &&Arg) 160 : BFI(std::move(Arg.BFI)) {} 161 162 BlockFrequencyInfo &BlockFrequencyInfo::operator=(BlockFrequencyInfo &&RHS) { 163 releaseMemory(); 164 BFI = std::move(RHS.BFI); 165 return *this; 166 } 167 168 // Explicitly define the default constructor otherwise it would be implicitly 169 // defined at the first ODR-use which is the BFI member in the 170 // LazyBlockFrequencyInfo header. The dtor needs the BlockFrequencyInfoImpl 171 // template instantiated which is not available in the header. 172 BlockFrequencyInfo::~BlockFrequencyInfo() = default; 173 174 bool BlockFrequencyInfo::invalidate(Function &F, const PreservedAnalyses &PA, 175 FunctionAnalysisManager::Invalidator &) { 176 // Check whether the analysis, all analyses on functions, or the function's 177 // CFG have been preserved. 178 auto PAC = PA.getChecker<BlockFrequencyAnalysis>(); 179 return !(PAC.preserved() || PAC.preservedSet<AllAnalysesOn<Function>>() || 180 PAC.preservedSet<CFGAnalyses>()); 181 } 182 183 void BlockFrequencyInfo::calculate(const Function &F, 184 const BranchProbabilityInfo &BPI, 185 const LoopInfo &LI) { 186 if (!BFI) 187 BFI.reset(new ImplType); 188 BFI->calculate(F, BPI, LI); 189 if (ViewBlockFreqPropagationDAG != GVDT_None && 190 (ViewBlockFreqFuncName.empty() || 191 F.getName().equals(ViewBlockFreqFuncName))) { 192 view(); 193 } 194 if (PrintBlockFreq && 195 (PrintBlockFreqFuncName.empty() || 196 F.getName().equals(PrintBlockFreqFuncName))) { 197 print(dbgs()); 198 } 199 } 200 201 BlockFrequency BlockFrequencyInfo::getBlockFreq(const BasicBlock *BB) const { 202 return BFI ? BFI->getBlockFreq(BB) : 0; 203 } 204 205 Optional<uint64_t> 206 BlockFrequencyInfo::getBlockProfileCount(const BasicBlock *BB, 207 bool AllowSynthetic) const { 208 if (!BFI) 209 return None; 210 211 return BFI->getBlockProfileCount(*getFunction(), BB, AllowSynthetic); 212 } 213 214 Optional<uint64_t> 215 BlockFrequencyInfo::getProfileCountFromFreq(uint64_t Freq) const { 216 if (!BFI) 217 return None; 218 return BFI->getProfileCountFromFreq(*getFunction(), Freq); 219 } 220 221 bool BlockFrequencyInfo::isIrrLoopHeader(const BasicBlock *BB) { 222 assert(BFI && "Expected analysis to be available"); 223 return BFI->isIrrLoopHeader(BB); 224 } 225 226 void BlockFrequencyInfo::setBlockFreq(const BasicBlock *BB, uint64_t Freq) { 227 assert(BFI && "Expected analysis to be available"); 228 BFI->setBlockFreq(BB, Freq); 229 } 230 231 void BlockFrequencyInfo::setBlockFreqAndScale( 232 const BasicBlock *ReferenceBB, uint64_t Freq, 233 SmallPtrSetImpl<BasicBlock *> &BlocksToScale) { 234 assert(BFI && "Expected analysis to be available"); 235 // Use 128 bits APInt to avoid overflow. 236 APInt NewFreq(128, Freq); 237 APInt OldFreq(128, BFI->getBlockFreq(ReferenceBB).getFrequency()); 238 APInt BBFreq(128, 0); 239 for (auto *BB : BlocksToScale) { 240 BBFreq = BFI->getBlockFreq(BB).getFrequency(); 241 // Multiply first by NewFreq and then divide by OldFreq 242 // to minimize loss of precision. 243 BBFreq *= NewFreq; 244 // udiv is an expensive operation in the general case. If this ends up being 245 // a hot spot, one of the options proposed in 246 // https://reviews.llvm.org/D28535#650071 could be used to avoid this. 247 BBFreq = BBFreq.udiv(OldFreq); 248 BFI->setBlockFreq(BB, BBFreq.getLimitedValue()); 249 } 250 BFI->setBlockFreq(ReferenceBB, Freq); 251 } 252 253 /// Pop up a ghostview window with the current block frequency propagation 254 /// rendered using dot. 255 void BlockFrequencyInfo::view(StringRef title) const { 256 ViewGraph(const_cast<BlockFrequencyInfo *>(this), title); 257 } 258 259 const Function *BlockFrequencyInfo::getFunction() const { 260 return BFI ? BFI->getFunction() : nullptr; 261 } 262 263 const BranchProbabilityInfo *BlockFrequencyInfo::getBPI() const { 264 return BFI ? &BFI->getBPI() : nullptr; 265 } 266 267 raw_ostream &BlockFrequencyInfo:: 268 printBlockFreq(raw_ostream &OS, const BlockFrequency Freq) const { 269 return BFI ? BFI->printBlockFreq(OS, Freq) : OS; 270 } 271 272 raw_ostream & 273 BlockFrequencyInfo::printBlockFreq(raw_ostream &OS, 274 const BasicBlock *BB) const { 275 return BFI ? BFI->printBlockFreq(OS, BB) : OS; 276 } 277 278 uint64_t BlockFrequencyInfo::getEntryFreq() const { 279 return BFI ? BFI->getEntryFreq() : 0; 280 } 281 282 void BlockFrequencyInfo::releaseMemory() { BFI.reset(); } 283 284 void BlockFrequencyInfo::print(raw_ostream &OS) const { 285 if (BFI) 286 BFI->print(OS); 287 } 288 289 INITIALIZE_PASS_BEGIN(BlockFrequencyInfoWrapperPass, "block-freq", 290 "Block Frequency Analysis", true, true) 291 INITIALIZE_PASS_DEPENDENCY(BranchProbabilityInfoWrapperPass) 292 INITIALIZE_PASS_DEPENDENCY(LoopInfoWrapperPass) 293 INITIALIZE_PASS_END(BlockFrequencyInfoWrapperPass, "block-freq", 294 "Block Frequency Analysis", true, true) 295 296 char BlockFrequencyInfoWrapperPass::ID = 0; 297 298 BlockFrequencyInfoWrapperPass::BlockFrequencyInfoWrapperPass() 299 : FunctionPass(ID) { 300 initializeBlockFrequencyInfoWrapperPassPass(*PassRegistry::getPassRegistry()); 301 } 302 303 BlockFrequencyInfoWrapperPass::~BlockFrequencyInfoWrapperPass() = default; 304 305 void BlockFrequencyInfoWrapperPass::print(raw_ostream &OS, 306 const Module *) const { 307 BFI.print(OS); 308 } 309 310 void BlockFrequencyInfoWrapperPass::getAnalysisUsage(AnalysisUsage &AU) const { 311 AU.addRequired<BranchProbabilityInfoWrapperPass>(); 312 AU.addRequired<LoopInfoWrapperPass>(); 313 AU.setPreservesAll(); 314 } 315 316 void BlockFrequencyInfoWrapperPass::releaseMemory() { BFI.releaseMemory(); } 317 318 bool BlockFrequencyInfoWrapperPass::runOnFunction(Function &F) { 319 BranchProbabilityInfo &BPI = 320 getAnalysis<BranchProbabilityInfoWrapperPass>().getBPI(); 321 LoopInfo &LI = getAnalysis<LoopInfoWrapperPass>().getLoopInfo(); 322 BFI.calculate(F, BPI, LI); 323 return false; 324 } 325 326 AnalysisKey BlockFrequencyAnalysis::Key; 327 BlockFrequencyInfo BlockFrequencyAnalysis::run(Function &F, 328 FunctionAnalysisManager &AM) { 329 BlockFrequencyInfo BFI; 330 BFI.calculate(F, AM.getResult<BranchProbabilityAnalysis>(F), 331 AM.getResult<LoopAnalysis>(F)); 332 return BFI; 333 } 334 335 PreservedAnalyses 336 BlockFrequencyPrinterPass::run(Function &F, FunctionAnalysisManager &AM) { 337 OS << "Printing analysis results of BFI for function " 338 << "'" << F.getName() << "':" 339 << "\n"; 340 AM.getResult<BlockFrequencyAnalysis>(F).print(OS); 341 return PreservedAnalyses::all(); 342 } 343