1 //===-- MachineFunctionSplitter.cpp - Split machine functions //-----------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // \file 10 // Uses profile information to split out cold blocks. 11 // 12 // This pass splits out cold machine basic blocks from the parent function. This 13 // implementation leverages the basic block section framework. Blocks marked 14 // cold by this pass are grouped together in a separate section prefixed with 15 // ".text.unlikely.*". The linker can then group these together as a cold 16 // section. The split part of the function is a contiguous region identified by 17 // the symbol "foo.cold". Grouping all cold blocks across functions together 18 // decreases fragmentation and improves icache and itlb utilization. Note that 19 // the overall changes to the binary size are negligible; only a small number of 20 // additional jump instructions may be introduced. 21 // 22 // For the original RFC of this pass please see 23 // https://groups.google.com/d/msg/llvm-dev/RUegaMg-iqc/wFAVxa6fCgAJ 24 //===----------------------------------------------------------------------===// 25 26 #include "llvm/ADT/Statistic.h" 27 #include "llvm/Analysis/ProfileSummaryInfo.h" 28 #include "llvm/CodeGen/BasicBlockSectionUtils.h" 29 #include "llvm/CodeGen/MachineBasicBlock.h" 30 #include "llvm/CodeGen/MachineBlockFrequencyInfo.h" 31 #include "llvm/CodeGen/MachineFunction.h" 32 #include "llvm/CodeGen/MachineFunctionPass.h" 33 #include "llvm/CodeGen/MachineModuleInfo.h" 34 #include "llvm/CodeGen/Passes.h" 35 #include "llvm/IR/Function.h" 36 #include "llvm/IR/Module.h" 37 #include "llvm/InitializePasses.h" 38 #include "llvm/Support/CommandLine.h" 39 40 using namespace llvm; 41 42 static cl::opt<unsigned> 43 PercentileCutoff("mfs-psi-cutoff", 44 cl::desc("Percentile profile summary cutoff used to " 45 "determine cold blocks. Unused if set to zero."), 46 cl::init(0), cl::Hidden); 47 48 static cl::opt<unsigned> ColdCountThreshold( 49 "mfs-count-threshold", 50 cl::desc( 51 "Minimum number of times a block must be executed to be retained."), 52 cl::init(1), cl::Hidden); 53 54 namespace { 55 56 class MachineFunctionSplitter : public MachineFunctionPass { 57 public: 58 static char ID; 59 MachineFunctionSplitter() : MachineFunctionPass(ID) { 60 initializeMachineFunctionSplitterPass(*PassRegistry::getPassRegistry()); 61 } 62 63 StringRef getPassName() const override { 64 return "Machine Function Splitter Transformation"; 65 } 66 67 void getAnalysisUsage(AnalysisUsage &AU) const override; 68 69 bool runOnMachineFunction(MachineFunction &F) override; 70 }; 71 } // end anonymous namespace 72 73 static bool isColdBlock(MachineBasicBlock &MBB, 74 const MachineBlockFrequencyInfo *MBFI, 75 ProfileSummaryInfo *PSI) { 76 Optional<uint64_t> Count = MBFI->getBlockProfileCount(&MBB); 77 if (!Count.hasValue()) 78 return true; 79 80 if (PercentileCutoff > 0) { 81 return PSI->isColdCountNthPercentile(PercentileCutoff, *Count); 82 } 83 return (*Count < ColdCountThreshold); 84 } 85 86 bool MachineFunctionSplitter::runOnMachineFunction(MachineFunction &MF) { 87 // TODO: We only target functions with profile data. Static information may 88 // also be considered but we don't see performance improvements yet. 89 if (!MF.getFunction().hasProfileData()) 90 return false; 91 92 // TODO: We don't split functions where a section attribute has been set 93 // since the split part may not be placed in a contiguous region. It may also 94 // be more beneficial to augment the linker to ensure contiguous layout of 95 // split functions within the same section as specified by the attribute. 96 if (!MF.getFunction().getSection().empty()) 97 return false; 98 99 // We don't want to proceed further for cold functions 100 // or functions of unknown hotness. Lukewarm functions have no prefix. 101 Optional<StringRef> SectionPrefix = MF.getFunction().getSectionPrefix(); 102 if (SectionPrefix.hasValue() && 103 (SectionPrefix.getValue().equals(".unlikely") || 104 SectionPrefix.getValue().equals(".unknown"))) { 105 return false; 106 } 107 108 // Renumbering blocks here preserves the order of the blocks as 109 // sortBasicBlocksAndUpdateBranches uses the numeric identifier to sort 110 // blocks. Preserving the order of blocks is essential to retaining decisions 111 // made by prior passes such as MachineBlockPlacement. 112 MF.RenumberBlocks(); 113 MF.setBBSectionsType(BasicBlockSection::Preset); 114 auto *MBFI = &getAnalysis<MachineBlockFrequencyInfo>(); 115 auto *PSI = &getAnalysis<ProfileSummaryInfoWrapperPass>().getPSI(); 116 117 for (auto &MBB : MF) { 118 // FIXME: We retain the entry block and conservatively keep all landing pad 119 // blocks as part of the original function. Once D73739 is submitted, we can 120 // improve the handling of ehpads. 121 if ((MBB.pred_empty() || MBB.isEHPad())) 122 continue; 123 if (isColdBlock(MBB, MBFI, PSI)) 124 MBB.setSectionID(MBBSectionID::ColdSectionID); 125 } 126 127 auto Comparator = [](const MachineBasicBlock &X, const MachineBasicBlock &Y) { 128 return X.getSectionID().Type < Y.getSectionID().Type; 129 }; 130 llvm::sortBasicBlocksAndUpdateBranches(MF, Comparator); 131 132 return true; 133 } 134 135 void MachineFunctionSplitter::getAnalysisUsage(AnalysisUsage &AU) const { 136 AU.addRequired<MachineModuleInfoWrapperPass>(); 137 AU.addRequired<MachineBlockFrequencyInfo>(); 138 AU.addRequired<ProfileSummaryInfoWrapperPass>(); 139 } 140 141 char MachineFunctionSplitter::ID = 0; 142 INITIALIZE_PASS(MachineFunctionSplitter, "machine-function-splitter", 143 "Split machine functions using profile information", false, 144 false) 145 146 MachineFunctionPass *llvm::createMachineFunctionSplitterPass() { 147 return new MachineFunctionSplitter(); 148 } 149