xref: /freebsd-src/contrib/llvm-project/llvm/lib/MC/MCPseudoProbe.cpp (revision 0fca6ea1d4eea4c934cfff25ac9ee8ad6fe95583)
1e8d8bef9SDimitry Andric //===- lib/MC/MCPseudoProbe.cpp - Pseudo probe encoding support ----------===//
2e8d8bef9SDimitry Andric //
3e8d8bef9SDimitry Andric // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4e8d8bef9SDimitry Andric // See https://llvm.org/LICENSE.txt for license information.
5e8d8bef9SDimitry Andric // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6e8d8bef9SDimitry Andric //
7e8d8bef9SDimitry Andric //===----------------------------------------------------------------------===//
8e8d8bef9SDimitry Andric 
9e8d8bef9SDimitry Andric #include "llvm/MC/MCPseudoProbe.h"
10fcaf7f86SDimitry Andric #include "llvm/ADT/STLExtras.h"
1106c3fb27SDimitry Andric #include "llvm/IR/PseudoProbe.h"
12e8d8bef9SDimitry Andric #include "llvm/MC/MCAsmInfo.h"
135f757f3fSDimitry Andric #include "llvm/MC/MCAssembler.h"
14e8d8bef9SDimitry Andric #include "llvm/MC/MCContext.h"
1581ad6265SDimitry Andric #include "llvm/MC/MCExpr.h"
1681ad6265SDimitry Andric #include "llvm/MC/MCFragment.h"
17e8d8bef9SDimitry Andric #include "llvm/MC/MCObjectFileInfo.h"
18e8d8bef9SDimitry Andric #include "llvm/MC/MCObjectStreamer.h"
19bdd1243dSDimitry Andric #include "llvm/MC/MCSymbol.h"
20349cc55cSDimitry Andric #include "llvm/Support/Endian.h"
21349cc55cSDimitry Andric #include "llvm/Support/LEB128.h"
22bdd1243dSDimitry Andric #include "llvm/Support/MD5.h"
23349cc55cSDimitry Andric #include "llvm/Support/raw_ostream.h"
24bdd1243dSDimitry Andric #include <algorithm>
25bdd1243dSDimitry Andric #include <cassert>
26349cc55cSDimitry Andric #include <limits>
27349cc55cSDimitry Andric #include <memory>
28349cc55cSDimitry Andric #include <sstream>
29bdd1243dSDimitry Andric #include <vector>
30e8d8bef9SDimitry Andric 
31e8d8bef9SDimitry Andric #define DEBUG_TYPE "mcpseudoprobe"
32e8d8bef9SDimitry Andric 
33e8d8bef9SDimitry Andric using namespace llvm;
34349cc55cSDimitry Andric using namespace support;
35e8d8bef9SDimitry Andric 
36e8d8bef9SDimitry Andric #ifndef NDEBUG
37e8d8bef9SDimitry Andric int MCPseudoProbeTable::DdgPrintIndent = 0;
38e8d8bef9SDimitry Andric #endif
39e8d8bef9SDimitry Andric 
40e8d8bef9SDimitry Andric static const MCExpr *buildSymbolDiff(MCObjectStreamer *MCOS, const MCSymbol *A,
41e8d8bef9SDimitry Andric                                      const MCSymbol *B) {
42e8d8bef9SDimitry Andric   MCContext &Context = MCOS->getContext();
43e8d8bef9SDimitry Andric   MCSymbolRefExpr::VariantKind Variant = MCSymbolRefExpr::VK_None;
44e8d8bef9SDimitry Andric   const MCExpr *ARef = MCSymbolRefExpr::create(A, Variant, Context);
45e8d8bef9SDimitry Andric   const MCExpr *BRef = MCSymbolRefExpr::create(B, Variant, Context);
46e8d8bef9SDimitry Andric   const MCExpr *AddrDelta =
47e8d8bef9SDimitry Andric       MCBinaryExpr::create(MCBinaryExpr::Sub, ARef, BRef, Context);
48e8d8bef9SDimitry Andric   return AddrDelta;
49e8d8bef9SDimitry Andric }
50e8d8bef9SDimitry Andric 
51e8d8bef9SDimitry Andric void MCPseudoProbe::emit(MCObjectStreamer *MCOS,
52e8d8bef9SDimitry Andric                          const MCPseudoProbe *LastProbe) const {
53bdd1243dSDimitry Andric   bool IsSentinel = isSentinelProbe(getAttributes());
54bdd1243dSDimitry Andric   assert((LastProbe || IsSentinel) &&
55bdd1243dSDimitry Andric          "Last probe should not be null for non-sentinel probes");
56bdd1243dSDimitry Andric 
57e8d8bef9SDimitry Andric   // Emit Index
58e8d8bef9SDimitry Andric   MCOS->emitULEB128IntValue(Index);
59e8d8bef9SDimitry Andric   // Emit Type and the flag:
60e8d8bef9SDimitry Andric   // Type (bit 0 to 3), with bit 4 to 6 for attributes.
61e8d8bef9SDimitry Andric   // Flag (bit 7, 0 - code address, 1 - address delta). This indicates whether
62e8d8bef9SDimitry Andric   // the following field is a symbolic code address or an address delta.
6306c3fb27SDimitry Andric   // Emit FS discriminator
64e8d8bef9SDimitry Andric   assert(Type <= 0xF && "Probe type too big to encode, exceeding 15");
6506c3fb27SDimitry Andric   auto NewAttributes = Attributes;
6606c3fb27SDimitry Andric   if (Discriminator)
6706c3fb27SDimitry Andric     NewAttributes |= (uint32_t)PseudoProbeAttributes::HasDiscriminator;
6806c3fb27SDimitry Andric   assert(NewAttributes <= 0x7 &&
69e8d8bef9SDimitry Andric          "Probe attributes too big to encode, exceeding 7");
7006c3fb27SDimitry Andric   uint8_t PackedType = Type | (NewAttributes << 4);
71bdd1243dSDimitry Andric   uint8_t Flag =
72bdd1243dSDimitry Andric       !IsSentinel ? ((int8_t)MCPseudoProbeFlag::AddressDelta << 7) : 0;
73e8d8bef9SDimitry Andric   MCOS->emitInt8(Flag | PackedType);
74e8d8bef9SDimitry Andric 
75bdd1243dSDimitry Andric   if (!IsSentinel) {
76e8d8bef9SDimitry Andric     // Emit the delta between the address label and LastProbe.
77e8d8bef9SDimitry Andric     const MCExpr *AddrDelta =
78e8d8bef9SDimitry Andric         buildSymbolDiff(MCOS, Label, LastProbe->getLabel());
79e8d8bef9SDimitry Andric     int64_t Delta;
80e8d8bef9SDimitry Andric     if (AddrDelta->evaluateAsAbsolute(Delta, MCOS->getAssemblerPtr())) {
81e8d8bef9SDimitry Andric       MCOS->emitSLEB128IntValue(Delta);
82e8d8bef9SDimitry Andric     } else {
83*0fca6ea1SDimitry Andric       MCOS->insert(MCOS->getContext().allocFragment<MCPseudoProbeAddrFragment>(
84*0fca6ea1SDimitry Andric           AddrDelta));
85e8d8bef9SDimitry Andric     }
86e8d8bef9SDimitry Andric   } else {
87bdd1243dSDimitry Andric     // Emit the GUID of the split function that the sentinel probe represents.
88bdd1243dSDimitry Andric     MCOS->emitInt64(Guid);
89e8d8bef9SDimitry Andric   }
90e8d8bef9SDimitry Andric 
9106c3fb27SDimitry Andric   if (Discriminator)
9206c3fb27SDimitry Andric     MCOS->emitULEB128IntValue(Discriminator);
9306c3fb27SDimitry Andric 
94e8d8bef9SDimitry Andric   LLVM_DEBUG({
95e8d8bef9SDimitry Andric     dbgs().indent(MCPseudoProbeTable::DdgPrintIndent);
96e8d8bef9SDimitry Andric     dbgs() << "Probe: " << Index << "\n";
97e8d8bef9SDimitry Andric   });
98e8d8bef9SDimitry Andric }
99e8d8bef9SDimitry Andric 
100e8d8bef9SDimitry Andric void MCPseudoProbeInlineTree::addPseudoProbe(
101e8d8bef9SDimitry Andric     const MCPseudoProbe &Probe, const MCPseudoProbeInlineStack &InlineStack) {
102e8d8bef9SDimitry Andric   // The function should not be called on the root.
103bdd1243dSDimitry Andric   assert(isRoot() && "Should only be called on root");
104e8d8bef9SDimitry Andric 
105e8d8bef9SDimitry Andric   // When it comes here, the input look like:
106e8d8bef9SDimitry Andric   //    Probe: GUID of C, ...
107e8d8bef9SDimitry Andric   //    InlineStack: [88, A], [66, B]
108e8d8bef9SDimitry Andric   // which means, Function A inlines function B at call site with a probe id of
109e8d8bef9SDimitry Andric   // 88, and B inlines C at probe 66. The tri-tree expects a tree path like {[0,
110e8d8bef9SDimitry Andric   // A], [88, B], [66, C]} to locate the tree node where the probe should be
111e8d8bef9SDimitry Andric   // added. Note that the edge [0, A] means A is the top-level function we are
112e8d8bef9SDimitry Andric   // emitting probes for.
113e8d8bef9SDimitry Andric 
114e8d8bef9SDimitry Andric   // Make a [0, A] edge.
115e8d8bef9SDimitry Andric   // An empty inline stack means the function that the probe originates from
116e8d8bef9SDimitry Andric   // is a top-level function.
117e8d8bef9SDimitry Andric   InlineSite Top;
118e8d8bef9SDimitry Andric   if (InlineStack.empty()) {
119e8d8bef9SDimitry Andric     Top = InlineSite(Probe.getGuid(), 0);
120e8d8bef9SDimitry Andric   } else {
121e8d8bef9SDimitry Andric     Top = InlineSite(std::get<0>(InlineStack.front()), 0);
122e8d8bef9SDimitry Andric   }
123e8d8bef9SDimitry Andric 
124e8d8bef9SDimitry Andric   auto *Cur = getOrAddNode(Top);
125e8d8bef9SDimitry Andric 
126e8d8bef9SDimitry Andric   // Make interior edges by walking the inline stack. Once it's done, Cur should
127e8d8bef9SDimitry Andric   // point to the node that the probe originates from.
128e8d8bef9SDimitry Andric   if (!InlineStack.empty()) {
129e8d8bef9SDimitry Andric     auto Iter = InlineStack.begin();
130e8d8bef9SDimitry Andric     auto Index = std::get<1>(*Iter);
131e8d8bef9SDimitry Andric     Iter++;
132e8d8bef9SDimitry Andric     for (; Iter != InlineStack.end(); Iter++) {
133e8d8bef9SDimitry Andric       // Make an edge by using the previous probe id and current GUID.
134e8d8bef9SDimitry Andric       Cur = Cur->getOrAddNode(InlineSite(std::get<0>(*Iter), Index));
135e8d8bef9SDimitry Andric       Index = std::get<1>(*Iter);
136e8d8bef9SDimitry Andric     }
137e8d8bef9SDimitry Andric     Cur = Cur->getOrAddNode(InlineSite(Probe.getGuid(), Index));
138e8d8bef9SDimitry Andric   }
139e8d8bef9SDimitry Andric 
140e8d8bef9SDimitry Andric   Cur->Probes.push_back(Probe);
141e8d8bef9SDimitry Andric }
142e8d8bef9SDimitry Andric 
143e8d8bef9SDimitry Andric void MCPseudoProbeInlineTree::emit(MCObjectStreamer *MCOS,
144e8d8bef9SDimitry Andric                                    const MCPseudoProbe *&LastProbe) {
145e8d8bef9SDimitry Andric   LLVM_DEBUG({
146e8d8bef9SDimitry Andric     dbgs().indent(MCPseudoProbeTable::DdgPrintIndent);
147e8d8bef9SDimitry Andric     dbgs() << "Group [\n";
148e8d8bef9SDimitry Andric     MCPseudoProbeTable::DdgPrintIndent += 2;
149e8d8bef9SDimitry Andric   });
150*0fca6ea1SDimitry Andric   assert(!isRoot() && "Root should be handled separately");
151bdd1243dSDimitry Andric 
152e8d8bef9SDimitry Andric   // Emit probes grouped by GUID.
153e8d8bef9SDimitry Andric   LLVM_DEBUG({
154e8d8bef9SDimitry Andric     dbgs().indent(MCPseudoProbeTable::DdgPrintIndent);
155e8d8bef9SDimitry Andric     dbgs() << "GUID: " << Guid << "\n";
156e8d8bef9SDimitry Andric   });
157e8d8bef9SDimitry Andric   // Emit Guid
158e8d8bef9SDimitry Andric   MCOS->emitInt64(Guid);
159bdd1243dSDimitry Andric   // Emit number of probes in this node, including a sentinel probe for
160bdd1243dSDimitry Andric   // top-level functions if needed.
161bdd1243dSDimitry Andric   bool NeedSentinel = false;
162bdd1243dSDimitry Andric   if (Parent->isRoot()) {
163bdd1243dSDimitry Andric     assert(isSentinelProbe(LastProbe->getAttributes()) &&
164bdd1243dSDimitry Andric            "Starting probe of a top-level function should be a sentinel probe");
165bdd1243dSDimitry Andric     // The main body of a split function doesn't need a sentinel probe.
166bdd1243dSDimitry Andric     if (LastProbe->getGuid() != Guid)
167bdd1243dSDimitry Andric       NeedSentinel = true;
168bdd1243dSDimitry Andric   }
169bdd1243dSDimitry Andric 
170bdd1243dSDimitry Andric   MCOS->emitULEB128IntValue(Probes.size() + NeedSentinel);
171e8d8bef9SDimitry Andric   // Emit number of direct inlinees
172349cc55cSDimitry Andric   MCOS->emitULEB128IntValue(Children.size());
173bdd1243dSDimitry Andric   // Emit sentinel probe for top-level functions
174bdd1243dSDimitry Andric   if (NeedSentinel)
175bdd1243dSDimitry Andric     LastProbe->emit(MCOS, nullptr);
176bdd1243dSDimitry Andric 
177e8d8bef9SDimitry Andric   // Emit probes in this group
178e8d8bef9SDimitry Andric   for (const auto &Probe : Probes) {
179e8d8bef9SDimitry Andric     Probe.emit(MCOS, LastProbe);
180e8d8bef9SDimitry Andric     LastProbe = &Probe;
181e8d8bef9SDimitry Andric   }
182e8d8bef9SDimitry Andric 
183bdd1243dSDimitry Andric   // Emit sorted descendant. InlineSite is unique for each pair, so there will
184bdd1243dSDimitry Andric   // be no ordering of Inlinee based on MCPseudoProbeInlineTree*
185bdd1243dSDimitry Andric   using InlineeType = std::pair<InlineSite, MCPseudoProbeInlineTree *>;
186bdd1243dSDimitry Andric   std::vector<InlineeType> Inlinees;
187bdd1243dSDimitry Andric   for (const auto &Child : Children)
188bdd1243dSDimitry Andric     Inlinees.emplace_back(Child.first, Child.second.get());
189*0fca6ea1SDimitry Andric   llvm::sort(Inlinees, llvm::less_first());
190349cc55cSDimitry Andric 
191e8d8bef9SDimitry Andric   for (const auto &Inlinee : Inlinees) {
192e8d8bef9SDimitry Andric     // Emit probe index
193e8d8bef9SDimitry Andric     MCOS->emitULEB128IntValue(std::get<1>(Inlinee.first));
194e8d8bef9SDimitry Andric     LLVM_DEBUG({
195e8d8bef9SDimitry Andric       dbgs().indent(MCPseudoProbeTable::DdgPrintIndent);
196e8d8bef9SDimitry Andric       dbgs() << "InlineSite: " << std::get<1>(Inlinee.first) << "\n";
197e8d8bef9SDimitry Andric     });
198e8d8bef9SDimitry Andric     // Emit the group
199e8d8bef9SDimitry Andric     Inlinee.second->emit(MCOS, LastProbe);
200e8d8bef9SDimitry Andric   }
201e8d8bef9SDimitry Andric 
202e8d8bef9SDimitry Andric   LLVM_DEBUG({
203e8d8bef9SDimitry Andric     MCPseudoProbeTable::DdgPrintIndent -= 2;
204e8d8bef9SDimitry Andric     dbgs().indent(MCPseudoProbeTable::DdgPrintIndent);
205e8d8bef9SDimitry Andric     dbgs() << "]\n";
206e8d8bef9SDimitry Andric   });
207e8d8bef9SDimitry Andric }
208e8d8bef9SDimitry Andric 
209bdd1243dSDimitry Andric void MCPseudoProbeSections::emit(MCObjectStreamer *MCOS) {
210e8d8bef9SDimitry Andric   MCContext &Ctx = MCOS->getContext();
2115f757f3fSDimitry Andric   SmallVector<std::pair<MCSymbol *, MCPseudoProbeInlineTree *>> Vec;
2125f757f3fSDimitry Andric   Vec.reserve(MCProbeDivisions.size());
2135f757f3fSDimitry Andric   for (auto &ProbeSec : MCProbeDivisions)
2145f757f3fSDimitry Andric     Vec.emplace_back(ProbeSec.first, &ProbeSec.second);
2155f757f3fSDimitry Andric   for (auto I : llvm::enumerate(MCOS->getAssembler()))
2165f757f3fSDimitry Andric     I.value().setOrdinal(I.index());
2175f757f3fSDimitry Andric   llvm::sort(Vec, [](auto A, auto B) {
2185f757f3fSDimitry Andric     return A.first->getSection().getOrdinal() <
2195f757f3fSDimitry Andric            B.first->getSection().getOrdinal();
2205f757f3fSDimitry Andric   });
2215f757f3fSDimitry Andric   for (auto [FuncSym, RootPtr] : Vec) {
2225f757f3fSDimitry Andric     const auto &Root = *RootPtr;
223bdd1243dSDimitry Andric     if (auto *S = Ctx.getObjectFileInfo()->getPseudoProbeSection(
224bdd1243dSDimitry Andric             FuncSym->getSection())) {
225e8d8bef9SDimitry Andric       // Switch to the .pseudoprobe section or a comdat group.
22681ad6265SDimitry Andric       MCOS->switchSection(S);
227e8d8bef9SDimitry Andric       // Emit probes grouped by GUID.
228bdd1243dSDimitry Andric       // Emit sorted descendant. InlineSite is unique for each pair, so there
229bdd1243dSDimitry Andric       // will be no ordering of Inlinee based on MCPseudoProbeInlineTree*
230bdd1243dSDimitry Andric       using InlineeType = std::pair<InlineSite, MCPseudoProbeInlineTree *>;
231bdd1243dSDimitry Andric       std::vector<InlineeType> Inlinees;
232bdd1243dSDimitry Andric       for (const auto &Child : Root.getChildren())
233bdd1243dSDimitry Andric         Inlinees.emplace_back(Child.first, Child.second.get());
234*0fca6ea1SDimitry Andric       llvm::sort(Inlinees, llvm::less_first());
235bdd1243dSDimitry Andric 
236bdd1243dSDimitry Andric       for (const auto &Inlinee : Inlinees) {
237bdd1243dSDimitry Andric         // Emit the group guarded by a sentinel probe.
23806c3fb27SDimitry Andric         MCPseudoProbe SentinelProbe(
23906c3fb27SDimitry Andric             const_cast<MCSymbol *>(FuncSym), MD5Hash(FuncSym->getName()),
240bdd1243dSDimitry Andric             (uint32_t)PseudoProbeReservedId::Invalid,
241bdd1243dSDimitry Andric             (uint32_t)PseudoProbeType::Block,
24206c3fb27SDimitry Andric             (uint32_t)PseudoProbeAttributes::Sentinel, 0);
243bdd1243dSDimitry Andric         const MCPseudoProbe *Probe = &SentinelProbe;
244bdd1243dSDimitry Andric         Inlinee.second->emit(MCOS, Probe);
245bdd1243dSDimitry Andric       }
246e8d8bef9SDimitry Andric     }
247e8d8bef9SDimitry Andric   }
248e8d8bef9SDimitry Andric }
249e8d8bef9SDimitry Andric 
250e8d8bef9SDimitry Andric //
251e8d8bef9SDimitry Andric // This emits the pseudo probe tables.
252e8d8bef9SDimitry Andric //
253e8d8bef9SDimitry Andric void MCPseudoProbeTable::emit(MCObjectStreamer *MCOS) {
254e8d8bef9SDimitry Andric   MCContext &Ctx = MCOS->getContext();
255e8d8bef9SDimitry Andric   auto &ProbeTable = Ctx.getMCPseudoProbeTable();
256e8d8bef9SDimitry Andric 
257e8d8bef9SDimitry Andric   // Bail out early so we don't switch to the pseudo_probe section needlessly
258e8d8bef9SDimitry Andric   // and in doing so create an unnecessary (if empty) section.
259e8d8bef9SDimitry Andric   auto &ProbeSections = ProbeTable.getProbeSections();
260e8d8bef9SDimitry Andric   if (ProbeSections.empty())
261e8d8bef9SDimitry Andric     return;
262e8d8bef9SDimitry Andric 
263e8d8bef9SDimitry Andric   LLVM_DEBUG(MCPseudoProbeTable::DdgPrintIndent = 0);
264e8d8bef9SDimitry Andric 
265e8d8bef9SDimitry Andric   // Put out the probe.
266e8d8bef9SDimitry Andric   ProbeSections.emit(MCOS);
267e8d8bef9SDimitry Andric }
268349cc55cSDimitry Andric 
269349cc55cSDimitry Andric static StringRef getProbeFNameForGUID(const GUIDProbeFunctionMap &GUID2FuncMAP,
270349cc55cSDimitry Andric                                       uint64_t GUID) {
271349cc55cSDimitry Andric   auto It = GUID2FuncMAP.find(GUID);
272349cc55cSDimitry Andric   assert(It != GUID2FuncMAP.end() &&
273349cc55cSDimitry Andric          "Probe function must exist for a valid GUID");
274349cc55cSDimitry Andric   return It->second.FuncName;
275349cc55cSDimitry Andric }
276349cc55cSDimitry Andric 
277349cc55cSDimitry Andric void MCPseudoProbeFuncDesc::print(raw_ostream &OS) {
278349cc55cSDimitry Andric   OS << "GUID: " << FuncGUID << " Name: " << FuncName << "\n";
279349cc55cSDimitry Andric   OS << "Hash: " << FuncHash << "\n";
280349cc55cSDimitry Andric }
281349cc55cSDimitry Andric 
282349cc55cSDimitry Andric void MCDecodedPseudoProbe::getInlineContext(
283*0fca6ea1SDimitry Andric     SmallVectorImpl<MCPseudoProbeFrameLocation> &ContextStack,
284349cc55cSDimitry Andric     const GUIDProbeFunctionMap &GUID2FuncMAP) const {
285349cc55cSDimitry Andric   uint32_t Begin = ContextStack.size();
286349cc55cSDimitry Andric   MCDecodedPseudoProbeInlineTree *Cur = InlineTree;
287349cc55cSDimitry Andric   // It will add the string of each node's inline site during iteration.
288349cc55cSDimitry Andric   // Note that it won't include the probe's belonging function(leaf location)
289349cc55cSDimitry Andric   while (Cur->hasInlineSite()) {
29081ad6265SDimitry Andric     StringRef FuncName = getProbeFNameForGUID(GUID2FuncMAP, Cur->Parent->Guid);
291349cc55cSDimitry Andric     ContextStack.emplace_back(
292*0fca6ea1SDimitry Andric         MCPseudoProbeFrameLocation(FuncName, std::get<1>(Cur->ISite)));
293349cc55cSDimitry Andric     Cur = static_cast<MCDecodedPseudoProbeInlineTree *>(Cur->Parent);
294349cc55cSDimitry Andric   }
295349cc55cSDimitry Andric   // Make the ContextStack in caller-callee order
296349cc55cSDimitry Andric   std::reverse(ContextStack.begin() + Begin, ContextStack.end());
297349cc55cSDimitry Andric }
298349cc55cSDimitry Andric 
299349cc55cSDimitry Andric std::string MCDecodedPseudoProbe::getInlineContextStr(
300349cc55cSDimitry Andric     const GUIDProbeFunctionMap &GUID2FuncMAP) const {
301349cc55cSDimitry Andric   std::ostringstream OContextStr;
302*0fca6ea1SDimitry Andric   SmallVector<MCPseudoProbeFrameLocation, 16> ContextStack;
303349cc55cSDimitry Andric   getInlineContext(ContextStack, GUID2FuncMAP);
304349cc55cSDimitry Andric   for (auto &Cxt : ContextStack) {
305349cc55cSDimitry Andric     if (OContextStr.str().size())
306349cc55cSDimitry Andric       OContextStr << " @ ";
307349cc55cSDimitry Andric     OContextStr << Cxt.first.str() << ":" << Cxt.second;
308349cc55cSDimitry Andric   }
309349cc55cSDimitry Andric   return OContextStr.str();
310349cc55cSDimitry Andric }
311349cc55cSDimitry Andric 
312349cc55cSDimitry Andric static const char *PseudoProbeTypeStr[3] = {"Block", "IndirectCall",
313349cc55cSDimitry Andric                                             "DirectCall"};
314349cc55cSDimitry Andric 
315349cc55cSDimitry Andric void MCDecodedPseudoProbe::print(raw_ostream &OS,
316349cc55cSDimitry Andric                                  const GUIDProbeFunctionMap &GUID2FuncMAP,
317349cc55cSDimitry Andric                                  bool ShowName) const {
318349cc55cSDimitry Andric   OS << "FUNC: ";
319349cc55cSDimitry Andric   if (ShowName) {
320349cc55cSDimitry Andric     StringRef FuncName = getProbeFNameForGUID(GUID2FuncMAP, Guid);
321349cc55cSDimitry Andric     OS << FuncName.str() << " ";
322349cc55cSDimitry Andric   } else {
323349cc55cSDimitry Andric     OS << Guid << " ";
324349cc55cSDimitry Andric   }
325349cc55cSDimitry Andric   OS << "Index: " << Index << "  ";
32606c3fb27SDimitry Andric   if (Discriminator)
32706c3fb27SDimitry Andric     OS << "Discriminator: " << Discriminator << "  ";
328349cc55cSDimitry Andric   OS << "Type: " << PseudoProbeTypeStr[static_cast<uint8_t>(Type)] << "  ";
329349cc55cSDimitry Andric   std::string InlineContextStr = getInlineContextStr(GUID2FuncMAP);
330349cc55cSDimitry Andric   if (InlineContextStr.size()) {
331349cc55cSDimitry Andric     OS << "Inlined: @ ";
332349cc55cSDimitry Andric     OS << InlineContextStr;
333349cc55cSDimitry Andric   }
334349cc55cSDimitry Andric   OS << "\n";
335349cc55cSDimitry Andric }
336349cc55cSDimitry Andric 
337349cc55cSDimitry Andric template <typename T> ErrorOr<T> MCPseudoProbeDecoder::readUnencodedNumber() {
338349cc55cSDimitry Andric   if (Data + sizeof(T) > End) {
339349cc55cSDimitry Andric     return std::error_code();
340349cc55cSDimitry Andric   }
341*0fca6ea1SDimitry Andric   T Val = endian::readNext<T, llvm::endianness::little>(Data);
342349cc55cSDimitry Andric   return ErrorOr<T>(Val);
343349cc55cSDimitry Andric }
344349cc55cSDimitry Andric 
345349cc55cSDimitry Andric template <typename T> ErrorOr<T> MCPseudoProbeDecoder::readUnsignedNumber() {
346349cc55cSDimitry Andric   unsigned NumBytesRead = 0;
347349cc55cSDimitry Andric   uint64_t Val = decodeULEB128(Data, &NumBytesRead);
348349cc55cSDimitry Andric   if (Val > std::numeric_limits<T>::max() || (Data + NumBytesRead > End)) {
349349cc55cSDimitry Andric     return std::error_code();
350349cc55cSDimitry Andric   }
351349cc55cSDimitry Andric   Data += NumBytesRead;
352349cc55cSDimitry Andric   return ErrorOr<T>(static_cast<T>(Val));
353349cc55cSDimitry Andric }
354349cc55cSDimitry Andric 
355349cc55cSDimitry Andric template <typename T> ErrorOr<T> MCPseudoProbeDecoder::readSignedNumber() {
356349cc55cSDimitry Andric   unsigned NumBytesRead = 0;
357349cc55cSDimitry Andric   int64_t Val = decodeSLEB128(Data, &NumBytesRead);
358349cc55cSDimitry Andric   if (Val > std::numeric_limits<T>::max() || (Data + NumBytesRead > End)) {
359349cc55cSDimitry Andric     return std::error_code();
360349cc55cSDimitry Andric   }
361349cc55cSDimitry Andric   Data += NumBytesRead;
362349cc55cSDimitry Andric   return ErrorOr<T>(static_cast<T>(Val));
363349cc55cSDimitry Andric }
364349cc55cSDimitry Andric 
365349cc55cSDimitry Andric ErrorOr<StringRef> MCPseudoProbeDecoder::readString(uint32_t Size) {
366349cc55cSDimitry Andric   StringRef Str(reinterpret_cast<const char *>(Data), Size);
367349cc55cSDimitry Andric   if (Data + Size > End) {
368349cc55cSDimitry Andric     return std::error_code();
369349cc55cSDimitry Andric   }
370349cc55cSDimitry Andric   Data += Size;
371349cc55cSDimitry Andric   return ErrorOr<StringRef>(Str);
372349cc55cSDimitry Andric }
373349cc55cSDimitry Andric 
374349cc55cSDimitry Andric bool MCPseudoProbeDecoder::buildGUID2FuncDescMap(const uint8_t *Start,
375349cc55cSDimitry Andric                                                  std::size_t Size) {
376349cc55cSDimitry Andric   // The pseudo_probe_desc section has a format like:
377349cc55cSDimitry Andric   // .section .pseudo_probe_desc,"",@progbits
378349cc55cSDimitry Andric   // .quad -5182264717993193164   // GUID
379349cc55cSDimitry Andric   // .quad 4294967295             // Hash
380349cc55cSDimitry Andric   // .uleb 3                      // Name size
381349cc55cSDimitry Andric   // .ascii "foo"                 // Name
382349cc55cSDimitry Andric   // .quad -2624081020897602054
383349cc55cSDimitry Andric   // .quad 174696971957
384349cc55cSDimitry Andric   // .uleb 34
385349cc55cSDimitry Andric   // .ascii "main"
386349cc55cSDimitry Andric 
387349cc55cSDimitry Andric   Data = Start;
388349cc55cSDimitry Andric   End = Data + Size;
389349cc55cSDimitry Andric 
390349cc55cSDimitry Andric   while (Data < End) {
391349cc55cSDimitry Andric     auto ErrorOrGUID = readUnencodedNumber<uint64_t>();
392349cc55cSDimitry Andric     if (!ErrorOrGUID)
393349cc55cSDimitry Andric       return false;
394349cc55cSDimitry Andric 
395349cc55cSDimitry Andric     auto ErrorOrHash = readUnencodedNumber<uint64_t>();
396349cc55cSDimitry Andric     if (!ErrorOrHash)
397349cc55cSDimitry Andric       return false;
398349cc55cSDimitry Andric 
399349cc55cSDimitry Andric     auto ErrorOrNameSize = readUnsignedNumber<uint32_t>();
400349cc55cSDimitry Andric     if (!ErrorOrNameSize)
401349cc55cSDimitry Andric       return false;
402349cc55cSDimitry Andric     uint32_t NameSize = std::move(*ErrorOrNameSize);
403349cc55cSDimitry Andric 
404349cc55cSDimitry Andric     auto ErrorOrName = readString(NameSize);
405349cc55cSDimitry Andric     if (!ErrorOrName)
406349cc55cSDimitry Andric       return false;
407349cc55cSDimitry Andric 
408349cc55cSDimitry Andric     uint64_t GUID = std::move(*ErrorOrGUID);
409349cc55cSDimitry Andric     uint64_t Hash = std::move(*ErrorOrHash);
410349cc55cSDimitry Andric     StringRef Name = std::move(*ErrorOrName);
411349cc55cSDimitry Andric 
412349cc55cSDimitry Andric     // Initialize PseudoProbeFuncDesc and populate it into GUID2FuncDescMap
413349cc55cSDimitry Andric     GUID2FuncDescMap.emplace(GUID, MCPseudoProbeFuncDesc(GUID, Hash, Name));
414349cc55cSDimitry Andric   }
415349cc55cSDimitry Andric   assert(Data == End && "Have unprocessed data in pseudo_probe_desc section");
416349cc55cSDimitry Andric   return true;
417349cc55cSDimitry Andric }
418349cc55cSDimitry Andric 
41981ad6265SDimitry Andric bool MCPseudoProbeDecoder::buildAddress2ProbeMap(
42081ad6265SDimitry Andric     MCDecodedPseudoProbeInlineTree *Cur, uint64_t &LastAddr,
421bdd1243dSDimitry Andric     const Uint64Set &GuidFilter, const Uint64Map &FuncStartAddrs) {
422349cc55cSDimitry Andric   // The pseudo_probe section encodes an inline forest and each tree has a
423bdd1243dSDimitry Andric   // format defined in MCPseudoProbe.h
424349cc55cSDimitry Andric 
425349cc55cSDimitry Andric   uint32_t Index = 0;
426bdd1243dSDimitry Andric   bool IsTopLevelFunc = Cur == &DummyInlineRoot;
427bdd1243dSDimitry Andric   if (IsTopLevelFunc) {
428349cc55cSDimitry Andric     // Use a sequential id for top level inliner.
42981ad6265SDimitry Andric     Index = Cur->getChildren().size();
430349cc55cSDimitry Andric   } else {
431349cc55cSDimitry Andric     // Read inline site for inlinees
432349cc55cSDimitry Andric     auto ErrorOrIndex = readUnsignedNumber<uint32_t>();
433349cc55cSDimitry Andric     if (!ErrorOrIndex)
434349cc55cSDimitry Andric       return false;
435349cc55cSDimitry Andric     Index = std::move(*ErrorOrIndex);
436349cc55cSDimitry Andric   }
43781ad6265SDimitry Andric 
438349cc55cSDimitry Andric   // Read guid
439349cc55cSDimitry Andric   auto ErrorOrCurGuid = readUnencodedNumber<uint64_t>();
440349cc55cSDimitry Andric   if (!ErrorOrCurGuid)
441349cc55cSDimitry Andric     return false;
44281ad6265SDimitry Andric   uint64_t Guid = std::move(*ErrorOrCurGuid);
44381ad6265SDimitry Andric 
44481ad6265SDimitry Andric   // Decide if top-level node should be disgarded.
445bdd1243dSDimitry Andric   if (IsTopLevelFunc && !GuidFilter.empty() && !GuidFilter.count(Guid))
44681ad6265SDimitry Andric     Cur = nullptr;
44781ad6265SDimitry Andric 
44881ad6265SDimitry Andric   // If the incoming node is null, all its children nodes should be disgarded.
44981ad6265SDimitry Andric   if (Cur) {
45081ad6265SDimitry Andric     // Switch/add to a new tree node(inlinee)
45181ad6265SDimitry Andric     Cur = Cur->getOrAddNode(std::make_tuple(Guid, Index));
45281ad6265SDimitry Andric     Cur->Guid = Guid;
453bdd1243dSDimitry Andric     if (IsTopLevelFunc && !EncodingIsAddrBased) {
454bdd1243dSDimitry Andric       if (auto V = FuncStartAddrs.lookup(Guid))
455bdd1243dSDimitry Andric         LastAddr = V;
456bdd1243dSDimitry Andric     }
45781ad6265SDimitry Andric   }
45881ad6265SDimitry Andric 
459349cc55cSDimitry Andric   // Read number of probes in the current node.
460349cc55cSDimitry Andric   auto ErrorOrNodeCount = readUnsignedNumber<uint32_t>();
461349cc55cSDimitry Andric   if (!ErrorOrNodeCount)
462349cc55cSDimitry Andric     return false;
463349cc55cSDimitry Andric   uint32_t NodeCount = std::move(*ErrorOrNodeCount);
464349cc55cSDimitry Andric   // Read number of direct inlinees
465349cc55cSDimitry Andric   auto ErrorOrCurChildrenToProcess = readUnsignedNumber<uint32_t>();
466349cc55cSDimitry Andric   if (!ErrorOrCurChildrenToProcess)
467349cc55cSDimitry Andric     return false;
468349cc55cSDimitry Andric   // Read all probes in this node
469349cc55cSDimitry Andric   for (std::size_t I = 0; I < NodeCount; I++) {
470349cc55cSDimitry Andric     // Read index
471349cc55cSDimitry Andric     auto ErrorOrIndex = readUnsignedNumber<uint32_t>();
472349cc55cSDimitry Andric     if (!ErrorOrIndex)
473349cc55cSDimitry Andric       return false;
474349cc55cSDimitry Andric     uint32_t Index = std::move(*ErrorOrIndex);
475349cc55cSDimitry Andric     // Read type | flag.
476349cc55cSDimitry Andric     auto ErrorOrValue = readUnencodedNumber<uint8_t>();
477349cc55cSDimitry Andric     if (!ErrorOrValue)
478349cc55cSDimitry Andric       return false;
479349cc55cSDimitry Andric     uint8_t Value = std::move(*ErrorOrValue);
480349cc55cSDimitry Andric     uint8_t Kind = Value & 0xf;
481349cc55cSDimitry Andric     uint8_t Attr = (Value & 0x70) >> 4;
482349cc55cSDimitry Andric     // Read address
483349cc55cSDimitry Andric     uint64_t Addr = 0;
484349cc55cSDimitry Andric     if (Value & 0x80) {
485349cc55cSDimitry Andric       auto ErrorOrOffset = readSignedNumber<int64_t>();
486349cc55cSDimitry Andric       if (!ErrorOrOffset)
487349cc55cSDimitry Andric         return false;
488349cc55cSDimitry Andric       int64_t Offset = std::move(*ErrorOrOffset);
489349cc55cSDimitry Andric       Addr = LastAddr + Offset;
490349cc55cSDimitry Andric     } else {
491349cc55cSDimitry Andric       auto ErrorOrAddr = readUnencodedNumber<int64_t>();
492349cc55cSDimitry Andric       if (!ErrorOrAddr)
493349cc55cSDimitry Andric         return false;
494349cc55cSDimitry Andric       Addr = std::move(*ErrorOrAddr);
495bdd1243dSDimitry Andric       if (isSentinelProbe(Attr)) {
496bdd1243dSDimitry Andric         // For sentinel probe, the addr field actually stores the GUID of the
497bdd1243dSDimitry Andric         // split function. Convert it to the real address.
498bdd1243dSDimitry Andric         if (auto V = FuncStartAddrs.lookup(Addr))
499bdd1243dSDimitry Andric           Addr = V;
500bdd1243dSDimitry Andric       } else {
501bdd1243dSDimitry Andric         // For now we assume all probe encoding should be either based on
502bdd1243dSDimitry Andric         // leading probe address or function start address.
503bdd1243dSDimitry Andric         // The scheme is for downwards compatibility.
504bdd1243dSDimitry Andric         // TODO: retire this scheme once compatibility is no longer an issue.
505bdd1243dSDimitry Andric         EncodingIsAddrBased = true;
506bdd1243dSDimitry Andric       }
507349cc55cSDimitry Andric     }
50881ad6265SDimitry Andric 
50906c3fb27SDimitry Andric     uint32_t Discriminator = 0;
51006c3fb27SDimitry Andric     if (hasDiscriminator(Attr)) {
51106c3fb27SDimitry Andric       auto ErrorOrDiscriminator = readUnsignedNumber<uint32_t>();
51206c3fb27SDimitry Andric       if (!ErrorOrDiscriminator)
51306c3fb27SDimitry Andric         return false;
51406c3fb27SDimitry Andric       Discriminator = std::move(*ErrorOrDiscriminator);
51506c3fb27SDimitry Andric     }
51606c3fb27SDimitry Andric 
517bdd1243dSDimitry Andric     if (Cur && !isSentinelProbe(Attr)) {
518349cc55cSDimitry Andric       // Populate Address2ProbesMap
519349cc55cSDimitry Andric       auto &Probes = Address2ProbesMap[Addr];
520349cc55cSDimitry Andric       Probes.emplace_back(Addr, Cur->Guid, Index, PseudoProbeType(Kind), Attr,
52106c3fb27SDimitry Andric                           Discriminator, Cur);
522349cc55cSDimitry Andric       Cur->addProbes(&Probes.back());
52381ad6265SDimitry Andric     }
524349cc55cSDimitry Andric     LastAddr = Addr;
525349cc55cSDimitry Andric   }
526349cc55cSDimitry Andric 
52781ad6265SDimitry Andric   uint32_t ChildrenToProcess = std::move(*ErrorOrCurChildrenToProcess);
52881ad6265SDimitry Andric   for (uint32_t I = 0; I < ChildrenToProcess; I++) {
529bdd1243dSDimitry Andric     buildAddress2ProbeMap(Cur, LastAddr, GuidFilter, FuncStartAddrs);
530349cc55cSDimitry Andric   }
531349cc55cSDimitry Andric 
532349cc55cSDimitry Andric   return true;
533349cc55cSDimitry Andric }
534349cc55cSDimitry Andric 
53581ad6265SDimitry Andric bool MCPseudoProbeDecoder::buildAddress2ProbeMap(
536bdd1243dSDimitry Andric     const uint8_t *Start, std::size_t Size, const Uint64Set &GuidFilter,
537bdd1243dSDimitry Andric     const Uint64Map &FuncStartAddrs) {
53881ad6265SDimitry Andric   Data = Start;
53981ad6265SDimitry Andric   End = Data + Size;
54081ad6265SDimitry Andric   uint64_t LastAddr = 0;
54181ad6265SDimitry Andric   while (Data < End)
542bdd1243dSDimitry Andric     buildAddress2ProbeMap(&DummyInlineRoot, LastAddr, GuidFilter,
543bdd1243dSDimitry Andric                           FuncStartAddrs);
54481ad6265SDimitry Andric   assert(Data == End && "Have unprocessed data in pseudo_probe section");
54581ad6265SDimitry Andric   return true;
54681ad6265SDimitry Andric }
54781ad6265SDimitry Andric 
548349cc55cSDimitry Andric void MCPseudoProbeDecoder::printGUID2FuncDescMap(raw_ostream &OS) {
549349cc55cSDimitry Andric   OS << "Pseudo Probe Desc:\n";
550349cc55cSDimitry Andric   // Make the output deterministic
551349cc55cSDimitry Andric   std::map<uint64_t, MCPseudoProbeFuncDesc> OrderedMap(GUID2FuncDescMap.begin(),
552349cc55cSDimitry Andric                                                        GUID2FuncDescMap.end());
553349cc55cSDimitry Andric   for (auto &I : OrderedMap) {
554349cc55cSDimitry Andric     I.second.print(OS);
555349cc55cSDimitry Andric   }
556349cc55cSDimitry Andric }
557349cc55cSDimitry Andric 
558349cc55cSDimitry Andric void MCPseudoProbeDecoder::printProbeForAddress(raw_ostream &OS,
559349cc55cSDimitry Andric                                                 uint64_t Address) {
560349cc55cSDimitry Andric   auto It = Address2ProbesMap.find(Address);
561349cc55cSDimitry Andric   if (It != Address2ProbesMap.end()) {
562349cc55cSDimitry Andric     for (auto &Probe : It->second) {
563349cc55cSDimitry Andric       OS << " [Probe]:\t";
564349cc55cSDimitry Andric       Probe.print(OS, GUID2FuncDescMap, true);
565349cc55cSDimitry Andric     }
566349cc55cSDimitry Andric   }
567349cc55cSDimitry Andric }
568349cc55cSDimitry Andric 
569349cc55cSDimitry Andric void MCPseudoProbeDecoder::printProbesForAllAddresses(raw_ostream &OS) {
5705f757f3fSDimitry Andric   auto Entries = make_first_range(Address2ProbesMap);
5715f757f3fSDimitry Andric   SmallVector<uint64_t, 0> Addresses(Entries.begin(), Entries.end());
572fcaf7f86SDimitry Andric   llvm::sort(Addresses);
573349cc55cSDimitry Andric   for (auto K : Addresses) {
574349cc55cSDimitry Andric     OS << "Address:\t";
575349cc55cSDimitry Andric     OS << K;
576349cc55cSDimitry Andric     OS << "\n";
577349cc55cSDimitry Andric     printProbeForAddress(OS, K);
578349cc55cSDimitry Andric   }
579349cc55cSDimitry Andric }
580349cc55cSDimitry Andric 
581349cc55cSDimitry Andric const MCDecodedPseudoProbe *
582349cc55cSDimitry Andric MCPseudoProbeDecoder::getCallProbeForAddr(uint64_t Address) const {
583349cc55cSDimitry Andric   auto It = Address2ProbesMap.find(Address);
584349cc55cSDimitry Andric   if (It == Address2ProbesMap.end())
585349cc55cSDimitry Andric     return nullptr;
586349cc55cSDimitry Andric   const auto &Probes = It->second;
587349cc55cSDimitry Andric 
588349cc55cSDimitry Andric   const MCDecodedPseudoProbe *CallProbe = nullptr;
589349cc55cSDimitry Andric   for (const auto &Probe : Probes) {
590349cc55cSDimitry Andric     if (Probe.isCall()) {
59106c3fb27SDimitry Andric       // Disabling the assert and returning first call probe seen so far.
59206c3fb27SDimitry Andric       // Subsequent call probes, if any, are ignored. Due to the the way
59306c3fb27SDimitry Andric       // .pseudo_probe section is decoded, probes of the same-named independent
59406c3fb27SDimitry Andric       // static functions are merged thus multiple call probes may be seen for a
59506c3fb27SDimitry Andric       // callsite. This should only happen to compiler-generated statics, with
59606c3fb27SDimitry Andric       // -funique-internal-linkage-names where user statics get unique names.
59706c3fb27SDimitry Andric       //
59806c3fb27SDimitry Andric       // TODO: re-enable or narrow down the assert to static functions only.
59906c3fb27SDimitry Andric       //
60006c3fb27SDimitry Andric       // assert(!CallProbe &&
60106c3fb27SDimitry Andric       //        "There should be only one call probe corresponding to address "
60206c3fb27SDimitry Andric       //        "which is a callsite.");
603349cc55cSDimitry Andric       CallProbe = &Probe;
60406c3fb27SDimitry Andric       break;
605349cc55cSDimitry Andric     }
606349cc55cSDimitry Andric   }
607349cc55cSDimitry Andric   return CallProbe;
608349cc55cSDimitry Andric }
609349cc55cSDimitry Andric 
610349cc55cSDimitry Andric const MCPseudoProbeFuncDesc *
611349cc55cSDimitry Andric MCPseudoProbeDecoder::getFuncDescForGUID(uint64_t GUID) const {
612349cc55cSDimitry Andric   auto It = GUID2FuncDescMap.find(GUID);
613349cc55cSDimitry Andric   assert(It != GUID2FuncDescMap.end() && "Function descriptor doesn't exist");
614349cc55cSDimitry Andric   return &It->second;
615349cc55cSDimitry Andric }
616349cc55cSDimitry Andric 
617349cc55cSDimitry Andric void MCPseudoProbeDecoder::getInlineContextForProbe(
618349cc55cSDimitry Andric     const MCDecodedPseudoProbe *Probe,
619*0fca6ea1SDimitry Andric     SmallVectorImpl<MCPseudoProbeFrameLocation> &InlineContextStack,
620349cc55cSDimitry Andric     bool IncludeLeaf) const {
621349cc55cSDimitry Andric   Probe->getInlineContext(InlineContextStack, GUID2FuncDescMap);
622349cc55cSDimitry Andric   if (!IncludeLeaf)
623349cc55cSDimitry Andric     return;
624349cc55cSDimitry Andric   // Note that the context from probe doesn't include leaf frame,
625349cc55cSDimitry Andric   // hence we need to retrieve and prepend leaf if requested.
626349cc55cSDimitry Andric   const auto *FuncDesc = getFuncDescForGUID(Probe->getGuid());
627349cc55cSDimitry Andric   InlineContextStack.emplace_back(
628*0fca6ea1SDimitry Andric       MCPseudoProbeFrameLocation(FuncDesc->FuncName, Probe->getIndex()));
629349cc55cSDimitry Andric }
630349cc55cSDimitry Andric 
631349cc55cSDimitry Andric const MCPseudoProbeFuncDesc *MCPseudoProbeDecoder::getInlinerDescForProbe(
632349cc55cSDimitry Andric     const MCDecodedPseudoProbe *Probe) const {
633349cc55cSDimitry Andric   MCDecodedPseudoProbeInlineTree *InlinerNode = Probe->getInlineTreeNode();
634349cc55cSDimitry Andric   if (!InlinerNode->hasInlineSite())
635349cc55cSDimitry Andric     return nullptr;
63681ad6265SDimitry Andric   return getFuncDescForGUID(InlinerNode->Parent->Guid);
637349cc55cSDimitry Andric }
638