xref: /llvm-project/llvm/lib/CodeGen/StaticDataSplitter.cpp (revision 3feb724496238ce10d32e8c2bd84b4ea50f9977e)
1 //===- StaticDataSplitter.cpp ---------------------------------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // The pass uses branch profile data to assign hotness based section qualifiers
10 // for the following types of static data:
11 // - Jump tables
12 // - Constant pools (TODO)
13 // - Other module-internal data (TODO)
14 //
15 // For the original RFC of this pass please see
16 // https://discourse.llvm.org/t/rfc-profile-guided-static-data-partitioning/83744
17 
18 #include "llvm/ADT/ScopeExit.h"
19 #include "llvm/ADT/Statistic.h"
20 #include "llvm/Analysis/ProfileSummaryInfo.h"
21 #include "llvm/CodeGen/MBFIWrapper.h"
22 #include "llvm/CodeGen/MachineBasicBlock.h"
23 #include "llvm/CodeGen/MachineBlockFrequencyInfo.h"
24 #include "llvm/CodeGen/MachineBranchProbabilityInfo.h"
25 #include "llvm/CodeGen/MachineConstantPool.h"
26 #include "llvm/CodeGen/MachineFunction.h"
27 #include "llvm/CodeGen/MachineFunctionPass.h"
28 #include "llvm/CodeGen/MachineJumpTableInfo.h"
29 #include "llvm/CodeGen/Passes.h"
30 #include "llvm/InitializePasses.h"
31 #include "llvm/Pass.h"
32 #include "llvm/Support/CommandLine.h"
33 
34 using namespace llvm;
35 
36 #define DEBUG_TYPE "static-data-splitter"
37 
38 STATISTIC(NumHotJumpTables, "Number of hot jump tables seen.");
39 STATISTIC(NumColdJumpTables, "Number of cold jump tables seen.");
40 STATISTIC(NumUnknownJumpTables,
41           "Number of jump tables with unknown hotness. They are from functions "
42           "without profile information.");
43 
44 class StaticDataSplitter : public MachineFunctionPass {
45   const MachineBranchProbabilityInfo *MBPI = nullptr;
46   const MachineBlockFrequencyInfo *MBFI = nullptr;
47   const ProfileSummaryInfo *PSI = nullptr;
48 
49   // Returns true iff any jump table is hot-cold categorized.
50   bool splitJumpTables(MachineFunction &MF);
51 
52   // Same as above but works on functions with profile information.
53   bool splitJumpTablesWithProfiles(const MachineFunction &MF,
54                                    MachineJumpTableInfo &MJTI);
55 
56 public:
57   static char ID;
58 
59   StaticDataSplitter() : MachineFunctionPass(ID) {
60     initializeStaticDataSplitterPass(*PassRegistry::getPassRegistry());
61   }
62 
63   StringRef getPassName() const override { return "Static Data Splitter"; }
64 
65   void getAnalysisUsage(AnalysisUsage &AU) const override {
66     MachineFunctionPass::getAnalysisUsage(AU);
67     AU.addRequired<MachineBranchProbabilityInfoWrapperPass>();
68     AU.addRequired<MachineBlockFrequencyInfoWrapperPass>();
69     AU.addRequired<ProfileSummaryInfoWrapperPass>();
70   }
71 
72   bool runOnMachineFunction(MachineFunction &MF) override;
73 };
74 
75 bool StaticDataSplitter::runOnMachineFunction(MachineFunction &MF) {
76   MBPI = &getAnalysis<MachineBranchProbabilityInfoWrapperPass>().getMBPI();
77   MBFI = &getAnalysis<MachineBlockFrequencyInfoWrapperPass>().getMBFI();
78   PSI = &getAnalysis<ProfileSummaryInfoWrapperPass>().getPSI();
79 
80   return splitJumpTables(MF);
81 }
82 
83 bool StaticDataSplitter::splitJumpTablesWithProfiles(
84     const MachineFunction &MF, MachineJumpTableInfo &MJTI) {
85   int NumChangedJumpTables = 0;
86 
87   // Jump table could be used by either terminating instructions or
88   // non-terminating ones, so we walk all instructions and use
89   // `MachineOperand::isJTI()` to identify jump table operands.
90   // Similarly, `MachineOperand::isCPI()` can identify constant pool usages
91   // in the same loop.
92   for (const auto &MBB : MF) {
93     for (const MachineInstr &I : MBB) {
94       for (const MachineOperand &Op : I.operands()) {
95         if (!Op.isJTI())
96           continue;
97         const int JTI = Op.getIndex();
98         // This is not a source block of jump table.
99         if (JTI == -1)
100           continue;
101 
102         auto Hotness = MachineFunctionDataHotness::Hot;
103 
104         // Hotness is based on source basic block hotness.
105         // TODO: PSI APIs are about instruction hotness. Introduce API for data
106         // access hotness.
107         if (PSI->isColdBlock(&MBB, MBFI))
108           Hotness = MachineFunctionDataHotness::Cold;
109 
110         if (MJTI.updateJumpTableEntryHotness(JTI, Hotness))
111           ++NumChangedJumpTables;
112       }
113     }
114   }
115   return NumChangedJumpTables > 0;
116 }
117 
118 bool StaticDataSplitter::splitJumpTables(MachineFunction &MF) {
119   MachineJumpTableInfo *MJTI = MF.getJumpTableInfo();
120   if (!MJTI || MJTI->getJumpTables().empty())
121     return false;
122 
123   const bool ProfileAvailable = PSI && PSI->hasProfileSummary() && MBFI &&
124                                 MF.getFunction().hasProfileData();
125   auto statOnExit = llvm::make_scope_exit([&] {
126     if (!AreStatisticsEnabled())
127       return;
128 
129     if (!ProfileAvailable) {
130       NumUnknownJumpTables += MJTI->getJumpTables().size();
131       return;
132     }
133 
134     for (size_t JTI = 0; JTI < MJTI->getJumpTables().size(); JTI++) {
135       auto Hotness = MJTI->getJumpTables()[JTI].Hotness;
136       if (Hotness == MachineFunctionDataHotness::Hot) {
137         ++NumHotJumpTables;
138       } else {
139         assert(Hotness == MachineFunctionDataHotness::Cold &&
140                "A jump table is either hot or cold when profile information is "
141                "available.");
142         ++NumColdJumpTables;
143       }
144     }
145   });
146 
147   // Place jump tables according to block hotness if function has profile data.
148   if (ProfileAvailable)
149     return splitJumpTablesWithProfiles(MF, *MJTI);
150 
151   return true;
152 }
153 
154 char StaticDataSplitter::ID = 0;
155 
156 INITIALIZE_PASS_BEGIN(StaticDataSplitter, DEBUG_TYPE, "Split static data",
157                       false, false)
158 INITIALIZE_PASS_DEPENDENCY(MachineBranchProbabilityInfoWrapperPass)
159 INITIALIZE_PASS_DEPENDENCY(MachineBlockFrequencyInfoWrapperPass)
160 INITIALIZE_PASS_DEPENDENCY(ProfileSummaryInfoWrapperPass)
161 INITIALIZE_PASS_END(StaticDataSplitter, DEBUG_TYPE, "Split static data", false,
162                     false)
163 
164 MachineFunctionPass *llvm::createStaticDataSplitterPass() {
165   return new StaticDataSplitter();
166 }
167