1 //===- PseudoProbe.cpp - Pseudo Probe Helpers -----------------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements the helpers to manipulate pseudo probe IR intrinsic
10 // calls.
11 //
12 //===----------------------------------------------------------------------===//
13
14 #include "llvm/IR/PseudoProbe.h"
15 #include "llvm/IR/DebugInfoMetadata.h"
16 #include "llvm/IR/IRBuilder.h"
17 #include "llvm/IR/Instruction.h"
18 #include <unordered_set>
19
20 using namespace llvm;
21
22 namespace llvm {
23
extractProbeFromDiscriminator(const Instruction & Inst)24 Optional<PseudoProbe> extractProbeFromDiscriminator(const Instruction &Inst) {
25 assert(isa<CallBase>(&Inst) && !isa<IntrinsicInst>(&Inst) &&
26 "Only call instructions should have pseudo probe encodes as their "
27 "Dwarf discriminators");
28 if (const DebugLoc &DLoc = Inst.getDebugLoc()) {
29 const DILocation *DIL = DLoc;
30 auto Discriminator = DIL->getDiscriminator();
31 if (DILocation::isPseudoProbeDiscriminator(Discriminator)) {
32 PseudoProbe Probe;
33 Probe.Id =
34 PseudoProbeDwarfDiscriminator::extractProbeIndex(Discriminator);
35 Probe.Type =
36 PseudoProbeDwarfDiscriminator::extractProbeType(Discriminator);
37 Probe.Attr =
38 PseudoProbeDwarfDiscriminator::extractProbeAttributes(Discriminator);
39 Probe.Factor =
40 PseudoProbeDwarfDiscriminator::extractProbeFactor(Discriminator) /
41 (float)PseudoProbeDwarfDiscriminator::FullDistributionFactor;
42 return Probe;
43 }
44 }
45 return None;
46 }
47
extractProbe(const Instruction & Inst)48 Optional<PseudoProbe> extractProbe(const Instruction &Inst) {
49 if (const auto *II = dyn_cast<PseudoProbeInst>(&Inst)) {
50 PseudoProbe Probe;
51 Probe.Id = II->getIndex()->getZExtValue();
52 Probe.Type = (uint32_t)PseudoProbeType::Block;
53 Probe.Attr = II->getAttributes()->getZExtValue();
54 Probe.Factor = II->getFactor()->getZExtValue() /
55 (float)PseudoProbeFullDistributionFactor;
56 return Probe;
57 }
58
59 if (isa<CallBase>(&Inst) && !isa<IntrinsicInst>(&Inst))
60 return extractProbeFromDiscriminator(Inst);
61
62 return None;
63 }
64
setProbeDistributionFactor(Instruction & Inst,float Factor)65 void setProbeDistributionFactor(Instruction &Inst, float Factor) {
66 assert(Factor >= 0 && Factor <= 1 &&
67 "Distribution factor must be in [0, 1.0]");
68 if (auto *II = dyn_cast<PseudoProbeInst>(&Inst)) {
69 IRBuilder<> Builder(&Inst);
70 uint64_t IntFactor = PseudoProbeFullDistributionFactor;
71 if (Factor < 1)
72 IntFactor *= Factor;
73 auto OrigFactor = II->getFactor()->getZExtValue();
74 if (IntFactor != OrigFactor)
75 II->replaceUsesOfWith(II->getFactor(), Builder.getInt64(IntFactor));
76 } else if (isa<CallBase>(&Inst) && !isa<IntrinsicInst>(&Inst)) {
77 if (const DebugLoc &DLoc = Inst.getDebugLoc()) {
78 const DILocation *DIL = DLoc;
79 auto Discriminator = DIL->getDiscriminator();
80 if (DILocation::isPseudoProbeDiscriminator(Discriminator)) {
81 auto Index =
82 PseudoProbeDwarfDiscriminator::extractProbeIndex(Discriminator);
83 auto Type =
84 PseudoProbeDwarfDiscriminator::extractProbeType(Discriminator);
85 auto Attr = PseudoProbeDwarfDiscriminator::extractProbeAttributes(
86 Discriminator);
87 // Round small factors to 0 to avoid over-counting.
88 uint32_t IntFactor =
89 PseudoProbeDwarfDiscriminator::FullDistributionFactor;
90 if (Factor < 1)
91 IntFactor *= Factor;
92 uint32_t V = PseudoProbeDwarfDiscriminator::packProbeData(
93 Index, Type, Attr, IntFactor);
94 DIL = DIL->cloneWithDiscriminator(V);
95 Inst.setDebugLoc(DIL);
96 }
97 }
98 }
99 }
100
addPseudoProbeAttribute(PseudoProbeInst & Inst,PseudoProbeAttributes Attr)101 void addPseudoProbeAttribute(PseudoProbeInst &Inst,
102 PseudoProbeAttributes Attr) {
103 IRBuilder<> Builder(&Inst);
104 uint32_t OldAttr = Inst.getAttributes()->getZExtValue();
105 uint32_t NewAttr = OldAttr | (uint32_t)Attr;
106 if (OldAttr != NewAttr)
107 Inst.replaceUsesOfWith(Inst.getAttributes(), Builder.getInt32(NewAttr));
108 }
109
110 /// A block emptied (i.e., with all instructions moved out of it) won't be
111 /// sampled at run time. In such cases, AutoFDO will be informed of zero samples
112 /// collected for the block. This is not accurate and could lead to misleading
113 /// weights assigned for the block. A way to mitigate that is to treat such
114 /// block as having unknown counts in the AutoFDO profile loader and allow the
115 /// counts inference tool a chance to calculate a relatively reasonable weight
116 /// for it. This can be done by moving all pseudo probes in the emptied block
117 /// i.e, /c From, to before /c To and tag them dangling. Note that this is
118 /// not needed for dead blocks which really have a zero weight. It's per
119 /// transforms to decide whether to call this function or not.
moveAndDanglePseudoProbes(BasicBlock * From,Instruction * To)120 bool moveAndDanglePseudoProbes(BasicBlock *From, Instruction *To) {
121 SmallVector<PseudoProbeInst *, 4> ToBeMoved;
122 for (auto &I : *From) {
123 if (auto *II = dyn_cast<PseudoProbeInst>(&I)) {
124 addPseudoProbeAttribute(*II, PseudoProbeAttributes::Dangling);
125 ToBeMoved.push_back(II);
126 }
127 }
128
129 for (auto *I : ToBeMoved)
130 I->moveBefore(To);
131
132 return !ToBeMoved.empty();
133 }
134
135 /// Same dangling probes in one blocks are redundant since they all have the
136 /// same semantic that is to rely on the counts inference too to get reasonable
137 /// count for the same original block. Therefore, there's no need to keep
138 /// multiple copies of them.
removeRedundantPseudoProbes(BasicBlock * Block)139 bool removeRedundantPseudoProbes(BasicBlock *Block) {
140
141 auto Hash = [](const PseudoProbeInst *I) {
142 return std::hash<uint64_t>()(I->getFuncGuid()->getZExtValue()) ^
143 std::hash<uint64_t>()(I->getIndex()->getZExtValue());
144 };
145
146 auto IsEqual = [](const PseudoProbeInst *Left, const PseudoProbeInst *Right) {
147 return Left->getFuncGuid() == Right->getFuncGuid() &&
148 Left->getIndex() == Right->getIndex() &&
149 Left->getAttributes() == Right->getAttributes() &&
150 Left->getDebugLoc() == Right->getDebugLoc();
151 };
152
153 SmallVector<PseudoProbeInst *, 4> ToBeRemoved;
154 std::unordered_set<PseudoProbeInst *, decltype(Hash), decltype(IsEqual)>
155 DanglingProbes(0, Hash, IsEqual);
156
157 for (auto &I : *Block) {
158 if (auto *II = dyn_cast<PseudoProbeInst>(&I)) {
159 if (II->getAttributes()->getZExtValue() &
160 (uint32_t)PseudoProbeAttributes::Dangling)
161 if (!DanglingProbes.insert(II).second)
162 ToBeRemoved.push_back(II);
163 }
164 }
165
166 for (auto *I : ToBeRemoved)
167 I->eraseFromParent();
168 return !ToBeRemoved.empty();
169 }
170 } // namespace llvm
171