1 //===- PseudoProbeInserter.cpp - Insert annotation for callsite profiling -===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements PseudoProbeInserter pass, which inserts pseudo probe
10 // annotations for call instructions with a pseudo-probe-specific dwarf
11 // discriminator. such discriminator indicates that the call instruction comes
12 // with a pseudo probe, and the discriminator value holds information to
13 // identify the corresponding counter.
14 //===----------------------------------------------------------------------===//
15
16 #include "llvm/CodeGen/MachineBasicBlock.h"
17 #include "llvm/CodeGen/MachineFunctionPass.h"
18 #include "llvm/CodeGen/MachineInstr.h"
19 #include "llvm/CodeGen/TargetInstrInfo.h"
20 #include "llvm/IR/DebugInfoMetadata.h"
21 #include "llvm/IR/PseudoProbe.h"
22 #include "llvm/InitializePasses.h"
23 #include "llvm/MC/MCPseudoProbe.h"
24 #include "llvm/Target/TargetMachine.h"
25 #include <unordered_set>
26
27 #define DEBUG_TYPE "pseudo-probe-inserter"
28
29 using namespace llvm;
30
31 namespace {
32 class PseudoProbeInserter : public MachineFunctionPass {
33 public:
34 static char ID;
35
PseudoProbeInserter()36 PseudoProbeInserter() : MachineFunctionPass(ID) {
37 initializePseudoProbeInserterPass(*PassRegistry::getPassRegistry());
38 }
39
getPassName() const40 StringRef getPassName() const override { return "Pseudo Probe Inserter"; }
41
getAnalysisUsage(AnalysisUsage & AU) const42 void getAnalysisUsage(AnalysisUsage &AU) const override {
43 AU.setPreservesAll();
44 MachineFunctionPass::getAnalysisUsage(AU);
45 }
46
runOnMachineFunction(MachineFunction & MF)47 bool runOnMachineFunction(MachineFunction &MF) override {
48 const TargetInstrInfo *TII = MF.getSubtarget().getInstrInfo();
49 bool Changed = false;
50 for (MachineBasicBlock &MBB : MF) {
51 MachineInstr *FirstInstr = nullptr;
52 for (MachineInstr &MI : MBB) {
53 if (!MI.isPseudo())
54 FirstInstr = &MI;
55 if (MI.isCall()) {
56 if (DILocation *DL = MI.getDebugLoc()) {
57 auto Value = DL->getDiscriminator();
58 if (DILocation::isPseudoProbeDiscriminator(Value)) {
59 BuildMI(MBB, MI, DL, TII->get(TargetOpcode::PSEUDO_PROBE))
60 .addImm(getFuncGUID(MF.getFunction().getParent(), DL))
61 .addImm(
62 PseudoProbeDwarfDiscriminator::extractProbeIndex(Value))
63 .addImm(
64 PseudoProbeDwarfDiscriminator::extractProbeType(Value))
65 .addImm(PseudoProbeDwarfDiscriminator::extractProbeAttributes(
66 Value));
67 Changed = true;
68 }
69 }
70 }
71 }
72
73 // Walk the block backwards, move PSEUDO_PROBE before the first real
74 // instruction to fix out-of-order probes. There is a problem with probes
75 // as the terminator of the block. During the offline counts processing,
76 // the samples collected on the first physical instruction following a
77 // probe will be counted towards the probe. This logically equals to
78 // treating the instruction next to a probe as if it is from the same
79 // block of the probe. This is accurate most of the time unless the
80 // instruction can be reached from multiple flows, which means it actually
81 // starts a new block. Samples collected on such probes may cause
82 // imprecision with the counts inference algorithm. Fortunately, if
83 // there are still other native instructions preceding the probe we can
84 // use them as a place holder to collect samples for the probe.
85 if (FirstInstr) {
86 auto MII = MBB.rbegin();
87 while (MII != MBB.rend()) {
88 // Skip all pseudo probes followed by a real instruction since they
89 // are not dangling.
90 if (!MII->isPseudo())
91 break;
92 auto Cur = MII++;
93 if (Cur->getOpcode() != TargetOpcode::PSEUDO_PROBE)
94 continue;
95 // Move the dangling probe before FirstInstr.
96 auto *ProbeInstr = &*Cur;
97 MBB.remove(ProbeInstr);
98 MBB.insert(FirstInstr, ProbeInstr);
99 Changed = true;
100 }
101 } else {
102 // Probes not surrounded by any real instructions in the same block are
103 // called dangling probes. Since there's no good way to pick up a sample
104 // collection point for dangling probes at compile time, they are being
105 // tagged so that the profile correlation tool will not report any
106 // samples collected for them and it's up to the counts inference tool
107 // to get them a reasonable count.
108 for (MachineInstr &MI : MBB) {
109 if (MI.isPseudoProbe())
110 MI.addPseudoProbeAttribute(PseudoProbeAttributes::Dangling);
111 }
112 }
113 }
114
115 // Remove redundant dangling probes. Same dangling probes are redundant
116 // since they all have the same semantic that is to rely on the counts
117 // inference too to get reasonable count for the same original block.
118 // Therefore, there's no need to keep multiple copies of them.
119 auto Hash = [](const MachineInstr *MI) {
120 return std::hash<uint64_t>()(MI->getOperand(0).getImm()) ^
121 std::hash<uint64_t>()(MI->getOperand(1).getImm());
122 };
123
124 auto IsEqual = [](const MachineInstr *Left, const MachineInstr *Right) {
125 return Left->getOperand(0).getImm() == Right->getOperand(0).getImm() &&
126 Left->getOperand(1).getImm() == Right->getOperand(1).getImm() &&
127 Left->getOperand(3).getImm() == Right->getOperand(3).getImm() &&
128 Left->getDebugLoc() == Right->getDebugLoc();
129 };
130
131 SmallVector<MachineInstr *, 4> ToBeRemoved;
132 std::unordered_set<MachineInstr *, decltype(Hash), decltype(IsEqual)>
133 DanglingProbes(0, Hash, IsEqual);
134
135 for (MachineBasicBlock &MBB : MF) {
136 for (MachineInstr &MI : MBB) {
137 if (MI.isPseudoProbe()) {
138 if ((uint32_t)MI.getPseudoProbeAttribute() &
139 (uint32_t)PseudoProbeAttributes::Dangling)
140 if (!DanglingProbes.insert(&MI).second)
141 ToBeRemoved.push_back(&MI);
142 }
143 }
144 }
145
146 for (auto *MI : ToBeRemoved)
147 MI->eraseFromParent();
148
149 Changed |= !ToBeRemoved.empty();
150 return Changed;
151 }
152
153 private:
getFuncGUID(Module * M,DILocation * DL)154 uint64_t getFuncGUID(Module *M, DILocation *DL) {
155 auto *SP = DL->getScope()->getSubprogram();
156 auto Name = SP->getLinkageName();
157 if (Name.empty())
158 Name = SP->getName();
159 return Function::getGUID(Name);
160 }
161 };
162 } // namespace
163
164 char PseudoProbeInserter::ID = 0;
165 INITIALIZE_PASS_BEGIN(PseudoProbeInserter, DEBUG_TYPE,
166 "Insert pseudo probe annotations for value profiling",
167 false, false)
INITIALIZE_PASS_DEPENDENCY(TargetPassConfig)168 INITIALIZE_PASS_DEPENDENCY(TargetPassConfig)
169 INITIALIZE_PASS_END(PseudoProbeInserter, DEBUG_TYPE,
170 "Insert pseudo probe annotations for value profiling",
171 false, false)
172
173 FunctionPass *llvm::createPseudoProbeInserter() {
174 return new PseudoProbeInserter();
175 }
176