1 //===- StaticDataSplitter.cpp ---------------------------------------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // The pass uses branch profile data to assign hotness based section qualifiers 10 // for the following types of static data: 11 // - Jump tables 12 // - Constant pools (TODO) 13 // - Other module-internal data (TODO) 14 // 15 // For the original RFC of this pass please see 16 // https://discourse.llvm.org/t/rfc-profile-guided-static-data-partitioning/83744 17 18 #include "llvm/ADT/ScopeExit.h" 19 #include "llvm/ADT/Statistic.h" 20 #include "llvm/Analysis/ProfileSummaryInfo.h" 21 #include "llvm/CodeGen/MBFIWrapper.h" 22 #include "llvm/CodeGen/MachineBasicBlock.h" 23 #include "llvm/CodeGen/MachineBlockFrequencyInfo.h" 24 #include "llvm/CodeGen/MachineBranchProbabilityInfo.h" 25 #include "llvm/CodeGen/MachineConstantPool.h" 26 #include "llvm/CodeGen/MachineFunction.h" 27 #include "llvm/CodeGen/MachineFunctionPass.h" 28 #include "llvm/CodeGen/MachineJumpTableInfo.h" 29 #include "llvm/CodeGen/Passes.h" 30 #include "llvm/InitializePasses.h" 31 #include "llvm/Pass.h" 32 #include "llvm/Support/CommandLine.h" 33 34 using namespace llvm; 35 36 #define DEBUG_TYPE "static-data-splitter" 37 38 STATISTIC(NumHotJumpTables, "Number of hot jump tables seen."); 39 STATISTIC(NumColdJumpTables, "Number of cold jump tables seen."); 40 STATISTIC(NumUnknownJumpTables, 41 "Number of jump tables with unknown hotness. They are from functions " 42 "without profile information."); 43 44 class StaticDataSplitter : public MachineFunctionPass { 45 const MachineBranchProbabilityInfo *MBPI = nullptr; 46 const MachineBlockFrequencyInfo *MBFI = nullptr; 47 const ProfileSummaryInfo *PSI = nullptr; 48 49 // Returns true iff any jump table is hot-cold categorized. 50 bool splitJumpTables(MachineFunction &MF); 51 52 // Same as above but works on functions with profile information. 53 bool splitJumpTablesWithProfiles(const MachineFunction &MF, 54 MachineJumpTableInfo &MJTI); 55 56 public: 57 static char ID; 58 59 StaticDataSplitter() : MachineFunctionPass(ID) { 60 initializeStaticDataSplitterPass(*PassRegistry::getPassRegistry()); 61 } 62 63 StringRef getPassName() const override { return "Static Data Splitter"; } 64 65 void getAnalysisUsage(AnalysisUsage &AU) const override { 66 MachineFunctionPass::getAnalysisUsage(AU); 67 AU.addRequired<MachineBranchProbabilityInfoWrapperPass>(); 68 AU.addRequired<MachineBlockFrequencyInfoWrapperPass>(); 69 AU.addRequired<ProfileSummaryInfoWrapperPass>(); 70 } 71 72 bool runOnMachineFunction(MachineFunction &MF) override; 73 }; 74 75 bool StaticDataSplitter::runOnMachineFunction(MachineFunction &MF) { 76 MBPI = &getAnalysis<MachineBranchProbabilityInfoWrapperPass>().getMBPI(); 77 MBFI = &getAnalysis<MachineBlockFrequencyInfoWrapperPass>().getMBFI(); 78 PSI = &getAnalysis<ProfileSummaryInfoWrapperPass>().getPSI(); 79 80 return splitJumpTables(MF); 81 } 82 83 bool StaticDataSplitter::splitJumpTablesWithProfiles( 84 const MachineFunction &MF, MachineJumpTableInfo &MJTI) { 85 int NumChangedJumpTables = 0; 86 87 // Jump table could be used by either terminating instructions or 88 // non-terminating ones, so we walk all instructions and use 89 // `MachineOperand::isJTI()` to identify jump table operands. 90 // Similarly, `MachineOperand::isCPI()` can identify constant pool usages 91 // in the same loop. 92 for (const auto &MBB : MF) { 93 for (const MachineInstr &I : MBB) { 94 for (const MachineOperand &Op : I.operands()) { 95 if (!Op.isJTI()) 96 continue; 97 const int JTI = Op.getIndex(); 98 // This is not a source block of jump table. 99 if (JTI == -1) 100 continue; 101 102 auto Hotness = MachineFunctionDataHotness::Hot; 103 104 // Hotness is based on source basic block hotness. 105 // TODO: PSI APIs are about instruction hotness. Introduce API for data 106 // access hotness. 107 if (PSI->isColdBlock(&MBB, MBFI)) 108 Hotness = MachineFunctionDataHotness::Cold; 109 110 if (MJTI.updateJumpTableEntryHotness(JTI, Hotness)) 111 ++NumChangedJumpTables; 112 } 113 } 114 } 115 return NumChangedJumpTables > 0; 116 } 117 118 bool StaticDataSplitter::splitJumpTables(MachineFunction &MF) { 119 MachineJumpTableInfo *MJTI = MF.getJumpTableInfo(); 120 if (!MJTI || MJTI->getJumpTables().empty()) 121 return false; 122 123 const bool ProfileAvailable = PSI && PSI->hasProfileSummary() && MBFI && 124 MF.getFunction().hasProfileData(); 125 auto statOnExit = llvm::make_scope_exit([&] { 126 if (!AreStatisticsEnabled()) 127 return; 128 129 if (!ProfileAvailable) { 130 NumUnknownJumpTables += MJTI->getJumpTables().size(); 131 return; 132 } 133 134 for (size_t JTI = 0; JTI < MJTI->getJumpTables().size(); JTI++) { 135 auto Hotness = MJTI->getJumpTables()[JTI].Hotness; 136 if (Hotness == MachineFunctionDataHotness::Hot) { 137 ++NumHotJumpTables; 138 } else { 139 assert(Hotness == MachineFunctionDataHotness::Cold && 140 "A jump table is either hot or cold when profile information is " 141 "available."); 142 ++NumColdJumpTables; 143 } 144 } 145 }); 146 147 // Place jump tables according to block hotness if function has profile data. 148 if (ProfileAvailable) 149 return splitJumpTablesWithProfiles(MF, *MJTI); 150 151 return true; 152 } 153 154 char StaticDataSplitter::ID = 0; 155 156 INITIALIZE_PASS_BEGIN(StaticDataSplitter, DEBUG_TYPE, "Split static data", 157 false, false) 158 INITIALIZE_PASS_DEPENDENCY(MachineBranchProbabilityInfoWrapperPass) 159 INITIALIZE_PASS_DEPENDENCY(MachineBlockFrequencyInfoWrapperPass) 160 INITIALIZE_PASS_DEPENDENCY(ProfileSummaryInfoWrapperPass) 161 INITIALIZE_PASS_END(StaticDataSplitter, DEBUG_TYPE, "Split static data", false, 162 false) 163 164 MachineFunctionPass *llvm::createStaticDataSplitterPass() { 165 return new StaticDataSplitter(); 166 } 167