xref: /freebsd-src/contrib/llvm-project/llvm/lib/MC/MCPseudoProbe.cpp (revision 06c3fb2749bda94cb5201f81ffdb8fa6c3161b2e)
1e8d8bef9SDimitry Andric //===- lib/MC/MCPseudoProbe.cpp - Pseudo probe encoding support ----------===//
2e8d8bef9SDimitry Andric //
3e8d8bef9SDimitry Andric // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4e8d8bef9SDimitry Andric // See https://llvm.org/LICENSE.txt for license information.
5e8d8bef9SDimitry Andric // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6e8d8bef9SDimitry Andric //
7e8d8bef9SDimitry Andric //===----------------------------------------------------------------------===//
8e8d8bef9SDimitry Andric 
9e8d8bef9SDimitry Andric #include "llvm/MC/MCPseudoProbe.h"
10fcaf7f86SDimitry Andric #include "llvm/ADT/STLExtras.h"
11*06c3fb27SDimitry Andric #include "llvm/IR/PseudoProbe.h"
12e8d8bef9SDimitry Andric #include "llvm/MC/MCAsmInfo.h"
13e8d8bef9SDimitry Andric #include "llvm/MC/MCContext.h"
1481ad6265SDimitry Andric #include "llvm/MC/MCExpr.h"
1581ad6265SDimitry Andric #include "llvm/MC/MCFragment.h"
16e8d8bef9SDimitry Andric #include "llvm/MC/MCObjectFileInfo.h"
17e8d8bef9SDimitry Andric #include "llvm/MC/MCObjectStreamer.h"
18bdd1243dSDimitry Andric #include "llvm/MC/MCSymbol.h"
19349cc55cSDimitry Andric #include "llvm/Support/Endian.h"
20349cc55cSDimitry Andric #include "llvm/Support/LEB128.h"
21bdd1243dSDimitry Andric #include "llvm/Support/MD5.h"
22349cc55cSDimitry Andric #include "llvm/Support/raw_ostream.h"
23bdd1243dSDimitry Andric #include <algorithm>
24bdd1243dSDimitry Andric #include <cassert>
25349cc55cSDimitry Andric #include <limits>
26349cc55cSDimitry Andric #include <memory>
27349cc55cSDimitry Andric #include <sstream>
28bdd1243dSDimitry Andric #include <vector>
29e8d8bef9SDimitry Andric 
30e8d8bef9SDimitry Andric #define DEBUG_TYPE "mcpseudoprobe"
31e8d8bef9SDimitry Andric 
32e8d8bef9SDimitry Andric using namespace llvm;
33349cc55cSDimitry Andric using namespace support;
34e8d8bef9SDimitry Andric 
35e8d8bef9SDimitry Andric #ifndef NDEBUG
36e8d8bef9SDimitry Andric int MCPseudoProbeTable::DdgPrintIndent = 0;
37e8d8bef9SDimitry Andric #endif
38e8d8bef9SDimitry Andric 
39e8d8bef9SDimitry Andric static const MCExpr *buildSymbolDiff(MCObjectStreamer *MCOS, const MCSymbol *A,
40e8d8bef9SDimitry Andric                                      const MCSymbol *B) {
41e8d8bef9SDimitry Andric   MCContext &Context = MCOS->getContext();
42e8d8bef9SDimitry Andric   MCSymbolRefExpr::VariantKind Variant = MCSymbolRefExpr::VK_None;
43e8d8bef9SDimitry Andric   const MCExpr *ARef = MCSymbolRefExpr::create(A, Variant, Context);
44e8d8bef9SDimitry Andric   const MCExpr *BRef = MCSymbolRefExpr::create(B, Variant, Context);
45e8d8bef9SDimitry Andric   const MCExpr *AddrDelta =
46e8d8bef9SDimitry Andric       MCBinaryExpr::create(MCBinaryExpr::Sub, ARef, BRef, Context);
47e8d8bef9SDimitry Andric   return AddrDelta;
48e8d8bef9SDimitry Andric }
49e8d8bef9SDimitry Andric 
50e8d8bef9SDimitry Andric void MCPseudoProbe::emit(MCObjectStreamer *MCOS,
51e8d8bef9SDimitry Andric                          const MCPseudoProbe *LastProbe) const {
52bdd1243dSDimitry Andric   bool IsSentinel = isSentinelProbe(getAttributes());
53bdd1243dSDimitry Andric   assert((LastProbe || IsSentinel) &&
54bdd1243dSDimitry Andric          "Last probe should not be null for non-sentinel probes");
55bdd1243dSDimitry Andric 
56e8d8bef9SDimitry Andric   // Emit Index
57e8d8bef9SDimitry Andric   MCOS->emitULEB128IntValue(Index);
58e8d8bef9SDimitry Andric   // Emit Type and the flag:
59e8d8bef9SDimitry Andric   // Type (bit 0 to 3), with bit 4 to 6 for attributes.
60e8d8bef9SDimitry Andric   // Flag (bit 7, 0 - code address, 1 - address delta). This indicates whether
61e8d8bef9SDimitry Andric   // the following field is a symbolic code address or an address delta.
62*06c3fb27SDimitry Andric   // Emit FS discriminator
63e8d8bef9SDimitry Andric   assert(Type <= 0xF && "Probe type too big to encode, exceeding 15");
64*06c3fb27SDimitry Andric   auto NewAttributes = Attributes;
65*06c3fb27SDimitry Andric   if (Discriminator)
66*06c3fb27SDimitry Andric     NewAttributes |= (uint32_t)PseudoProbeAttributes::HasDiscriminator;
67*06c3fb27SDimitry Andric   assert(NewAttributes <= 0x7 &&
68e8d8bef9SDimitry Andric          "Probe attributes too big to encode, exceeding 7");
69*06c3fb27SDimitry Andric   uint8_t PackedType = Type | (NewAttributes << 4);
70bdd1243dSDimitry Andric   uint8_t Flag =
71bdd1243dSDimitry Andric       !IsSentinel ? ((int8_t)MCPseudoProbeFlag::AddressDelta << 7) : 0;
72e8d8bef9SDimitry Andric   MCOS->emitInt8(Flag | PackedType);
73e8d8bef9SDimitry Andric 
74bdd1243dSDimitry Andric   if (!IsSentinel) {
75e8d8bef9SDimitry Andric     // Emit the delta between the address label and LastProbe.
76e8d8bef9SDimitry Andric     const MCExpr *AddrDelta =
77e8d8bef9SDimitry Andric         buildSymbolDiff(MCOS, Label, LastProbe->getLabel());
78e8d8bef9SDimitry Andric     int64_t Delta;
79e8d8bef9SDimitry Andric     if (AddrDelta->evaluateAsAbsolute(Delta, MCOS->getAssemblerPtr())) {
80e8d8bef9SDimitry Andric       MCOS->emitSLEB128IntValue(Delta);
81e8d8bef9SDimitry Andric     } else {
82e8d8bef9SDimitry Andric       MCOS->insert(new MCPseudoProbeAddrFragment(AddrDelta));
83e8d8bef9SDimitry Andric     }
84e8d8bef9SDimitry Andric   } else {
85bdd1243dSDimitry Andric     // Emit the GUID of the split function that the sentinel probe represents.
86bdd1243dSDimitry Andric     MCOS->emitInt64(Guid);
87e8d8bef9SDimitry Andric   }
88e8d8bef9SDimitry Andric 
89*06c3fb27SDimitry Andric   if (Discriminator)
90*06c3fb27SDimitry Andric     MCOS->emitULEB128IntValue(Discriminator);
91*06c3fb27SDimitry Andric 
92e8d8bef9SDimitry Andric   LLVM_DEBUG({
93e8d8bef9SDimitry Andric     dbgs().indent(MCPseudoProbeTable::DdgPrintIndent);
94e8d8bef9SDimitry Andric     dbgs() << "Probe: " << Index << "\n";
95e8d8bef9SDimitry Andric   });
96e8d8bef9SDimitry Andric }
97e8d8bef9SDimitry Andric 
98e8d8bef9SDimitry Andric void MCPseudoProbeInlineTree::addPseudoProbe(
99e8d8bef9SDimitry Andric     const MCPseudoProbe &Probe, const MCPseudoProbeInlineStack &InlineStack) {
100e8d8bef9SDimitry Andric   // The function should not be called on the root.
101bdd1243dSDimitry Andric   assert(isRoot() && "Should only be called on root");
102e8d8bef9SDimitry Andric 
103e8d8bef9SDimitry Andric   // When it comes here, the input look like:
104e8d8bef9SDimitry Andric   //    Probe: GUID of C, ...
105e8d8bef9SDimitry Andric   //    InlineStack: [88, A], [66, B]
106e8d8bef9SDimitry Andric   // which means, Function A inlines function B at call site with a probe id of
107e8d8bef9SDimitry Andric   // 88, and B inlines C at probe 66. The tri-tree expects a tree path like {[0,
108e8d8bef9SDimitry Andric   // A], [88, B], [66, C]} to locate the tree node where the probe should be
109e8d8bef9SDimitry Andric   // added. Note that the edge [0, A] means A is the top-level function we are
110e8d8bef9SDimitry Andric   // emitting probes for.
111e8d8bef9SDimitry Andric 
112e8d8bef9SDimitry Andric   // Make a [0, A] edge.
113e8d8bef9SDimitry Andric   // An empty inline stack means the function that the probe originates from
114e8d8bef9SDimitry Andric   // is a top-level function.
115e8d8bef9SDimitry Andric   InlineSite Top;
116e8d8bef9SDimitry Andric   if (InlineStack.empty()) {
117e8d8bef9SDimitry Andric     Top = InlineSite(Probe.getGuid(), 0);
118e8d8bef9SDimitry Andric   } else {
119e8d8bef9SDimitry Andric     Top = InlineSite(std::get<0>(InlineStack.front()), 0);
120e8d8bef9SDimitry Andric   }
121e8d8bef9SDimitry Andric 
122e8d8bef9SDimitry Andric   auto *Cur = getOrAddNode(Top);
123e8d8bef9SDimitry Andric 
124e8d8bef9SDimitry Andric   // Make interior edges by walking the inline stack. Once it's done, Cur should
125e8d8bef9SDimitry Andric   // point to the node that the probe originates from.
126e8d8bef9SDimitry Andric   if (!InlineStack.empty()) {
127e8d8bef9SDimitry Andric     auto Iter = InlineStack.begin();
128e8d8bef9SDimitry Andric     auto Index = std::get<1>(*Iter);
129e8d8bef9SDimitry Andric     Iter++;
130e8d8bef9SDimitry Andric     for (; Iter != InlineStack.end(); Iter++) {
131e8d8bef9SDimitry Andric       // Make an edge by using the previous probe id and current GUID.
132e8d8bef9SDimitry Andric       Cur = Cur->getOrAddNode(InlineSite(std::get<0>(*Iter), Index));
133e8d8bef9SDimitry Andric       Index = std::get<1>(*Iter);
134e8d8bef9SDimitry Andric     }
135e8d8bef9SDimitry Andric     Cur = Cur->getOrAddNode(InlineSite(Probe.getGuid(), Index));
136e8d8bef9SDimitry Andric   }
137e8d8bef9SDimitry Andric 
138e8d8bef9SDimitry Andric   Cur->Probes.push_back(Probe);
139e8d8bef9SDimitry Andric }
140e8d8bef9SDimitry Andric 
141e8d8bef9SDimitry Andric void MCPseudoProbeInlineTree::emit(MCObjectStreamer *MCOS,
142e8d8bef9SDimitry Andric                                    const MCPseudoProbe *&LastProbe) {
143e8d8bef9SDimitry Andric   LLVM_DEBUG({
144e8d8bef9SDimitry Andric     dbgs().indent(MCPseudoProbeTable::DdgPrintIndent);
145e8d8bef9SDimitry Andric     dbgs() << "Group [\n";
146e8d8bef9SDimitry Andric     MCPseudoProbeTable::DdgPrintIndent += 2;
147e8d8bef9SDimitry Andric   });
148bdd1243dSDimitry Andric   assert(!isRoot() && "Root should be handled seperately");
149bdd1243dSDimitry Andric 
150e8d8bef9SDimitry Andric   // Emit probes grouped by GUID.
151e8d8bef9SDimitry Andric   LLVM_DEBUG({
152e8d8bef9SDimitry Andric     dbgs().indent(MCPseudoProbeTable::DdgPrintIndent);
153e8d8bef9SDimitry Andric     dbgs() << "GUID: " << Guid << "\n";
154e8d8bef9SDimitry Andric   });
155e8d8bef9SDimitry Andric   // Emit Guid
156e8d8bef9SDimitry Andric   MCOS->emitInt64(Guid);
157bdd1243dSDimitry Andric   // Emit number of probes in this node, including a sentinel probe for
158bdd1243dSDimitry Andric   // top-level functions if needed.
159bdd1243dSDimitry Andric   bool NeedSentinel = false;
160bdd1243dSDimitry Andric   if (Parent->isRoot()) {
161bdd1243dSDimitry Andric     assert(isSentinelProbe(LastProbe->getAttributes()) &&
162bdd1243dSDimitry Andric            "Starting probe of a top-level function should be a sentinel probe");
163bdd1243dSDimitry Andric     // The main body of a split function doesn't need a sentinel probe.
164bdd1243dSDimitry Andric     if (LastProbe->getGuid() != Guid)
165bdd1243dSDimitry Andric       NeedSentinel = true;
166bdd1243dSDimitry Andric   }
167bdd1243dSDimitry Andric 
168bdd1243dSDimitry Andric   MCOS->emitULEB128IntValue(Probes.size() + NeedSentinel);
169e8d8bef9SDimitry Andric   // Emit number of direct inlinees
170349cc55cSDimitry Andric   MCOS->emitULEB128IntValue(Children.size());
171bdd1243dSDimitry Andric   // Emit sentinel probe for top-level functions
172bdd1243dSDimitry Andric   if (NeedSentinel)
173bdd1243dSDimitry Andric     LastProbe->emit(MCOS, nullptr);
174bdd1243dSDimitry Andric 
175e8d8bef9SDimitry Andric   // Emit probes in this group
176e8d8bef9SDimitry Andric   for (const auto &Probe : Probes) {
177e8d8bef9SDimitry Andric     Probe.emit(MCOS, LastProbe);
178e8d8bef9SDimitry Andric     LastProbe = &Probe;
179e8d8bef9SDimitry Andric   }
180e8d8bef9SDimitry Andric 
181bdd1243dSDimitry Andric   // Emit sorted descendant. InlineSite is unique for each pair, so there will
182bdd1243dSDimitry Andric   // be no ordering of Inlinee based on MCPseudoProbeInlineTree*
183bdd1243dSDimitry Andric   using InlineeType = std::pair<InlineSite, MCPseudoProbeInlineTree *>;
184bdd1243dSDimitry Andric   auto Comparer = [](const InlineeType &A, const InlineeType &B) {
185bdd1243dSDimitry Andric     return A.first < B.first;
186bdd1243dSDimitry Andric   };
187bdd1243dSDimitry Andric   std::vector<InlineeType> Inlinees;
188bdd1243dSDimitry Andric   for (const auto &Child : Children)
189bdd1243dSDimitry Andric     Inlinees.emplace_back(Child.first, Child.second.get());
190bdd1243dSDimitry Andric   std::sort(Inlinees.begin(), Inlinees.end(), Comparer);
191349cc55cSDimitry Andric 
192e8d8bef9SDimitry Andric   for (const auto &Inlinee : Inlinees) {
193e8d8bef9SDimitry Andric     // Emit probe index
194e8d8bef9SDimitry Andric     MCOS->emitULEB128IntValue(std::get<1>(Inlinee.first));
195e8d8bef9SDimitry Andric     LLVM_DEBUG({
196e8d8bef9SDimitry Andric       dbgs().indent(MCPseudoProbeTable::DdgPrintIndent);
197e8d8bef9SDimitry Andric       dbgs() << "InlineSite: " << std::get<1>(Inlinee.first) << "\n";
198e8d8bef9SDimitry Andric     });
199e8d8bef9SDimitry Andric     // Emit the group
200e8d8bef9SDimitry Andric     Inlinee.second->emit(MCOS, LastProbe);
201e8d8bef9SDimitry Andric   }
202e8d8bef9SDimitry Andric 
203e8d8bef9SDimitry Andric   LLVM_DEBUG({
204e8d8bef9SDimitry Andric     MCPseudoProbeTable::DdgPrintIndent -= 2;
205e8d8bef9SDimitry Andric     dbgs().indent(MCPseudoProbeTable::DdgPrintIndent);
206e8d8bef9SDimitry Andric     dbgs() << "]\n";
207e8d8bef9SDimitry Andric   });
208e8d8bef9SDimitry Andric }
209e8d8bef9SDimitry Andric 
210bdd1243dSDimitry Andric void MCPseudoProbeSections::emit(MCObjectStreamer *MCOS) {
211e8d8bef9SDimitry Andric   MCContext &Ctx = MCOS->getContext();
212e8d8bef9SDimitry Andric   for (auto &ProbeSec : MCProbeDivisions) {
213bdd1243dSDimitry Andric     const auto *FuncSym = ProbeSec.first;
214bdd1243dSDimitry Andric     const auto &Root = ProbeSec.second;
215bdd1243dSDimitry Andric     if (auto *S = Ctx.getObjectFileInfo()->getPseudoProbeSection(
216bdd1243dSDimitry Andric             FuncSym->getSection())) {
217e8d8bef9SDimitry Andric       // Switch to the .pseudoprobe section or a comdat group.
21881ad6265SDimitry Andric       MCOS->switchSection(S);
219e8d8bef9SDimitry Andric       // Emit probes grouped by GUID.
220bdd1243dSDimitry Andric       // Emit sorted descendant. InlineSite is unique for each pair, so there
221bdd1243dSDimitry Andric       // will be no ordering of Inlinee based on MCPseudoProbeInlineTree*
222bdd1243dSDimitry Andric       using InlineeType = std::pair<InlineSite, MCPseudoProbeInlineTree *>;
223bdd1243dSDimitry Andric       auto Comparer = [](const InlineeType &A, const InlineeType &B) {
224bdd1243dSDimitry Andric         return A.first < B.first;
225bdd1243dSDimitry Andric       };
226bdd1243dSDimitry Andric       std::vector<InlineeType> Inlinees;
227bdd1243dSDimitry Andric       for (const auto &Child : Root.getChildren())
228bdd1243dSDimitry Andric         Inlinees.emplace_back(Child.first, Child.second.get());
229bdd1243dSDimitry Andric       std::sort(Inlinees.begin(), Inlinees.end(), Comparer);
230bdd1243dSDimitry Andric 
231bdd1243dSDimitry Andric       for (const auto &Inlinee : Inlinees) {
232bdd1243dSDimitry Andric         // Emit the group guarded by a sentinel probe.
233*06c3fb27SDimitry Andric         MCPseudoProbe SentinelProbe(
234*06c3fb27SDimitry Andric             const_cast<MCSymbol *>(FuncSym), MD5Hash(FuncSym->getName()),
235bdd1243dSDimitry Andric             (uint32_t)PseudoProbeReservedId::Invalid,
236bdd1243dSDimitry Andric             (uint32_t)PseudoProbeType::Block,
237*06c3fb27SDimitry Andric             (uint32_t)PseudoProbeAttributes::Sentinel, 0);
238bdd1243dSDimitry Andric         const MCPseudoProbe *Probe = &SentinelProbe;
239bdd1243dSDimitry Andric         Inlinee.second->emit(MCOS, Probe);
240bdd1243dSDimitry Andric       }
241e8d8bef9SDimitry Andric     }
242e8d8bef9SDimitry Andric   }
243e8d8bef9SDimitry Andric }
244e8d8bef9SDimitry Andric 
245e8d8bef9SDimitry Andric //
246e8d8bef9SDimitry Andric // This emits the pseudo probe tables.
247e8d8bef9SDimitry Andric //
248e8d8bef9SDimitry Andric void MCPseudoProbeTable::emit(MCObjectStreamer *MCOS) {
249e8d8bef9SDimitry Andric   MCContext &Ctx = MCOS->getContext();
250e8d8bef9SDimitry Andric   auto &ProbeTable = Ctx.getMCPseudoProbeTable();
251e8d8bef9SDimitry Andric 
252e8d8bef9SDimitry Andric   // Bail out early so we don't switch to the pseudo_probe section needlessly
253e8d8bef9SDimitry Andric   // and in doing so create an unnecessary (if empty) section.
254e8d8bef9SDimitry Andric   auto &ProbeSections = ProbeTable.getProbeSections();
255e8d8bef9SDimitry Andric   if (ProbeSections.empty())
256e8d8bef9SDimitry Andric     return;
257e8d8bef9SDimitry Andric 
258e8d8bef9SDimitry Andric   LLVM_DEBUG(MCPseudoProbeTable::DdgPrintIndent = 0);
259e8d8bef9SDimitry Andric 
260e8d8bef9SDimitry Andric   // Put out the probe.
261e8d8bef9SDimitry Andric   ProbeSections.emit(MCOS);
262e8d8bef9SDimitry Andric }
263349cc55cSDimitry Andric 
264349cc55cSDimitry Andric static StringRef getProbeFNameForGUID(const GUIDProbeFunctionMap &GUID2FuncMAP,
265349cc55cSDimitry Andric                                       uint64_t GUID) {
266349cc55cSDimitry Andric   auto It = GUID2FuncMAP.find(GUID);
267349cc55cSDimitry Andric   assert(It != GUID2FuncMAP.end() &&
268349cc55cSDimitry Andric          "Probe function must exist for a valid GUID");
269349cc55cSDimitry Andric   return It->second.FuncName;
270349cc55cSDimitry Andric }
271349cc55cSDimitry Andric 
272349cc55cSDimitry Andric void MCPseudoProbeFuncDesc::print(raw_ostream &OS) {
273349cc55cSDimitry Andric   OS << "GUID: " << FuncGUID << " Name: " << FuncName << "\n";
274349cc55cSDimitry Andric   OS << "Hash: " << FuncHash << "\n";
275349cc55cSDimitry Andric }
276349cc55cSDimitry Andric 
277349cc55cSDimitry Andric void MCDecodedPseudoProbe::getInlineContext(
278349cc55cSDimitry Andric     SmallVectorImpl<MCPseduoProbeFrameLocation> &ContextStack,
279349cc55cSDimitry Andric     const GUIDProbeFunctionMap &GUID2FuncMAP) const {
280349cc55cSDimitry Andric   uint32_t Begin = ContextStack.size();
281349cc55cSDimitry Andric   MCDecodedPseudoProbeInlineTree *Cur = InlineTree;
282349cc55cSDimitry Andric   // It will add the string of each node's inline site during iteration.
283349cc55cSDimitry Andric   // Note that it won't include the probe's belonging function(leaf location)
284349cc55cSDimitry Andric   while (Cur->hasInlineSite()) {
28581ad6265SDimitry Andric     StringRef FuncName = getProbeFNameForGUID(GUID2FuncMAP, Cur->Parent->Guid);
286349cc55cSDimitry Andric     ContextStack.emplace_back(
287349cc55cSDimitry Andric         MCPseduoProbeFrameLocation(FuncName, std::get<1>(Cur->ISite)));
288349cc55cSDimitry Andric     Cur = static_cast<MCDecodedPseudoProbeInlineTree *>(Cur->Parent);
289349cc55cSDimitry Andric   }
290349cc55cSDimitry Andric   // Make the ContextStack in caller-callee order
291349cc55cSDimitry Andric   std::reverse(ContextStack.begin() + Begin, ContextStack.end());
292349cc55cSDimitry Andric }
293349cc55cSDimitry Andric 
294349cc55cSDimitry Andric std::string MCDecodedPseudoProbe::getInlineContextStr(
295349cc55cSDimitry Andric     const GUIDProbeFunctionMap &GUID2FuncMAP) const {
296349cc55cSDimitry Andric   std::ostringstream OContextStr;
297349cc55cSDimitry Andric   SmallVector<MCPseduoProbeFrameLocation, 16> ContextStack;
298349cc55cSDimitry Andric   getInlineContext(ContextStack, GUID2FuncMAP);
299349cc55cSDimitry Andric   for (auto &Cxt : ContextStack) {
300349cc55cSDimitry Andric     if (OContextStr.str().size())
301349cc55cSDimitry Andric       OContextStr << " @ ";
302349cc55cSDimitry Andric     OContextStr << Cxt.first.str() << ":" << Cxt.second;
303349cc55cSDimitry Andric   }
304349cc55cSDimitry Andric   return OContextStr.str();
305349cc55cSDimitry Andric }
306349cc55cSDimitry Andric 
307349cc55cSDimitry Andric static const char *PseudoProbeTypeStr[3] = {"Block", "IndirectCall",
308349cc55cSDimitry Andric                                             "DirectCall"};
309349cc55cSDimitry Andric 
310349cc55cSDimitry Andric void MCDecodedPseudoProbe::print(raw_ostream &OS,
311349cc55cSDimitry Andric                                  const GUIDProbeFunctionMap &GUID2FuncMAP,
312349cc55cSDimitry Andric                                  bool ShowName) const {
313349cc55cSDimitry Andric   OS << "FUNC: ";
314349cc55cSDimitry Andric   if (ShowName) {
315349cc55cSDimitry Andric     StringRef FuncName = getProbeFNameForGUID(GUID2FuncMAP, Guid);
316349cc55cSDimitry Andric     OS << FuncName.str() << " ";
317349cc55cSDimitry Andric   } else {
318349cc55cSDimitry Andric     OS << Guid << " ";
319349cc55cSDimitry Andric   }
320349cc55cSDimitry Andric   OS << "Index: " << Index << "  ";
321*06c3fb27SDimitry Andric   if (Discriminator)
322*06c3fb27SDimitry Andric     OS << "Discriminator: " << Discriminator << "  ";
323349cc55cSDimitry Andric   OS << "Type: " << PseudoProbeTypeStr[static_cast<uint8_t>(Type)] << "  ";
324349cc55cSDimitry Andric   std::string InlineContextStr = getInlineContextStr(GUID2FuncMAP);
325349cc55cSDimitry Andric   if (InlineContextStr.size()) {
326349cc55cSDimitry Andric     OS << "Inlined: @ ";
327349cc55cSDimitry Andric     OS << InlineContextStr;
328349cc55cSDimitry Andric   }
329349cc55cSDimitry Andric   OS << "\n";
330349cc55cSDimitry Andric }
331349cc55cSDimitry Andric 
332349cc55cSDimitry Andric template <typename T> ErrorOr<T> MCPseudoProbeDecoder::readUnencodedNumber() {
333349cc55cSDimitry Andric   if (Data + sizeof(T) > End) {
334349cc55cSDimitry Andric     return std::error_code();
335349cc55cSDimitry Andric   }
336349cc55cSDimitry Andric   T Val = endian::readNext<T, little, unaligned>(Data);
337349cc55cSDimitry Andric   return ErrorOr<T>(Val);
338349cc55cSDimitry Andric }
339349cc55cSDimitry Andric 
340349cc55cSDimitry Andric template <typename T> ErrorOr<T> MCPseudoProbeDecoder::readUnsignedNumber() {
341349cc55cSDimitry Andric   unsigned NumBytesRead = 0;
342349cc55cSDimitry Andric   uint64_t Val = decodeULEB128(Data, &NumBytesRead);
343349cc55cSDimitry Andric   if (Val > std::numeric_limits<T>::max() || (Data + NumBytesRead > End)) {
344349cc55cSDimitry Andric     return std::error_code();
345349cc55cSDimitry Andric   }
346349cc55cSDimitry Andric   Data += NumBytesRead;
347349cc55cSDimitry Andric   return ErrorOr<T>(static_cast<T>(Val));
348349cc55cSDimitry Andric }
349349cc55cSDimitry Andric 
350349cc55cSDimitry Andric template <typename T> ErrorOr<T> MCPseudoProbeDecoder::readSignedNumber() {
351349cc55cSDimitry Andric   unsigned NumBytesRead = 0;
352349cc55cSDimitry Andric   int64_t Val = decodeSLEB128(Data, &NumBytesRead);
353349cc55cSDimitry Andric   if (Val > std::numeric_limits<T>::max() || (Data + NumBytesRead > End)) {
354349cc55cSDimitry Andric     return std::error_code();
355349cc55cSDimitry Andric   }
356349cc55cSDimitry Andric   Data += NumBytesRead;
357349cc55cSDimitry Andric   return ErrorOr<T>(static_cast<T>(Val));
358349cc55cSDimitry Andric }
359349cc55cSDimitry Andric 
360349cc55cSDimitry Andric ErrorOr<StringRef> MCPseudoProbeDecoder::readString(uint32_t Size) {
361349cc55cSDimitry Andric   StringRef Str(reinterpret_cast<const char *>(Data), Size);
362349cc55cSDimitry Andric   if (Data + Size > End) {
363349cc55cSDimitry Andric     return std::error_code();
364349cc55cSDimitry Andric   }
365349cc55cSDimitry Andric   Data += Size;
366349cc55cSDimitry Andric   return ErrorOr<StringRef>(Str);
367349cc55cSDimitry Andric }
368349cc55cSDimitry Andric 
369349cc55cSDimitry Andric bool MCPseudoProbeDecoder::buildGUID2FuncDescMap(const uint8_t *Start,
370349cc55cSDimitry Andric                                                  std::size_t Size) {
371349cc55cSDimitry Andric   // The pseudo_probe_desc section has a format like:
372349cc55cSDimitry Andric   // .section .pseudo_probe_desc,"",@progbits
373349cc55cSDimitry Andric   // .quad -5182264717993193164   // GUID
374349cc55cSDimitry Andric   // .quad 4294967295             // Hash
375349cc55cSDimitry Andric   // .uleb 3                      // Name size
376349cc55cSDimitry Andric   // .ascii "foo"                 // Name
377349cc55cSDimitry Andric   // .quad -2624081020897602054
378349cc55cSDimitry Andric   // .quad 174696971957
379349cc55cSDimitry Andric   // .uleb 34
380349cc55cSDimitry Andric   // .ascii "main"
381349cc55cSDimitry Andric 
382349cc55cSDimitry Andric   Data = Start;
383349cc55cSDimitry Andric   End = Data + Size;
384349cc55cSDimitry Andric 
385349cc55cSDimitry Andric   while (Data < End) {
386349cc55cSDimitry Andric     auto ErrorOrGUID = readUnencodedNumber<uint64_t>();
387349cc55cSDimitry Andric     if (!ErrorOrGUID)
388349cc55cSDimitry Andric       return false;
389349cc55cSDimitry Andric 
390349cc55cSDimitry Andric     auto ErrorOrHash = readUnencodedNumber<uint64_t>();
391349cc55cSDimitry Andric     if (!ErrorOrHash)
392349cc55cSDimitry Andric       return false;
393349cc55cSDimitry Andric 
394349cc55cSDimitry Andric     auto ErrorOrNameSize = readUnsignedNumber<uint32_t>();
395349cc55cSDimitry Andric     if (!ErrorOrNameSize)
396349cc55cSDimitry Andric       return false;
397349cc55cSDimitry Andric     uint32_t NameSize = std::move(*ErrorOrNameSize);
398349cc55cSDimitry Andric 
399349cc55cSDimitry Andric     auto ErrorOrName = readString(NameSize);
400349cc55cSDimitry Andric     if (!ErrorOrName)
401349cc55cSDimitry Andric       return false;
402349cc55cSDimitry Andric 
403349cc55cSDimitry Andric     uint64_t GUID = std::move(*ErrorOrGUID);
404349cc55cSDimitry Andric     uint64_t Hash = std::move(*ErrorOrHash);
405349cc55cSDimitry Andric     StringRef Name = std::move(*ErrorOrName);
406349cc55cSDimitry Andric 
407349cc55cSDimitry Andric     // Initialize PseudoProbeFuncDesc and populate it into GUID2FuncDescMap
408349cc55cSDimitry Andric     GUID2FuncDescMap.emplace(GUID, MCPseudoProbeFuncDesc(GUID, Hash, Name));
409349cc55cSDimitry Andric   }
410349cc55cSDimitry Andric   assert(Data == End && "Have unprocessed data in pseudo_probe_desc section");
411349cc55cSDimitry Andric   return true;
412349cc55cSDimitry Andric }
413349cc55cSDimitry Andric 
41481ad6265SDimitry Andric bool MCPseudoProbeDecoder::buildAddress2ProbeMap(
41581ad6265SDimitry Andric     MCDecodedPseudoProbeInlineTree *Cur, uint64_t &LastAddr,
416bdd1243dSDimitry Andric     const Uint64Set &GuidFilter, const Uint64Map &FuncStartAddrs) {
417349cc55cSDimitry Andric   // The pseudo_probe section encodes an inline forest and each tree has a
418bdd1243dSDimitry Andric   // format defined in MCPseudoProbe.h
419349cc55cSDimitry Andric 
420349cc55cSDimitry Andric   uint32_t Index = 0;
421bdd1243dSDimitry Andric   bool IsTopLevelFunc = Cur == &DummyInlineRoot;
422bdd1243dSDimitry Andric   if (IsTopLevelFunc) {
423349cc55cSDimitry Andric     // Use a sequential id for top level inliner.
42481ad6265SDimitry Andric     Index = Cur->getChildren().size();
425349cc55cSDimitry Andric   } else {
426349cc55cSDimitry Andric     // Read inline site for inlinees
427349cc55cSDimitry Andric     auto ErrorOrIndex = readUnsignedNumber<uint32_t>();
428349cc55cSDimitry Andric     if (!ErrorOrIndex)
429349cc55cSDimitry Andric       return false;
430349cc55cSDimitry Andric     Index = std::move(*ErrorOrIndex);
431349cc55cSDimitry Andric   }
43281ad6265SDimitry Andric 
433349cc55cSDimitry Andric   // Read guid
434349cc55cSDimitry Andric   auto ErrorOrCurGuid = readUnencodedNumber<uint64_t>();
435349cc55cSDimitry Andric   if (!ErrorOrCurGuid)
436349cc55cSDimitry Andric     return false;
43781ad6265SDimitry Andric   uint64_t Guid = std::move(*ErrorOrCurGuid);
43881ad6265SDimitry Andric 
43981ad6265SDimitry Andric   // Decide if top-level node should be disgarded.
440bdd1243dSDimitry Andric   if (IsTopLevelFunc && !GuidFilter.empty() && !GuidFilter.count(Guid))
44181ad6265SDimitry Andric     Cur = nullptr;
44281ad6265SDimitry Andric 
44381ad6265SDimitry Andric   // If the incoming node is null, all its children nodes should be disgarded.
44481ad6265SDimitry Andric   if (Cur) {
44581ad6265SDimitry Andric     // Switch/add to a new tree node(inlinee)
44681ad6265SDimitry Andric     Cur = Cur->getOrAddNode(std::make_tuple(Guid, Index));
44781ad6265SDimitry Andric     Cur->Guid = Guid;
448bdd1243dSDimitry Andric     if (IsTopLevelFunc && !EncodingIsAddrBased) {
449bdd1243dSDimitry Andric       if (auto V = FuncStartAddrs.lookup(Guid))
450bdd1243dSDimitry Andric         LastAddr = V;
451bdd1243dSDimitry Andric     }
45281ad6265SDimitry Andric   }
45381ad6265SDimitry Andric 
454349cc55cSDimitry Andric   // Read number of probes in the current node.
455349cc55cSDimitry Andric   auto ErrorOrNodeCount = readUnsignedNumber<uint32_t>();
456349cc55cSDimitry Andric   if (!ErrorOrNodeCount)
457349cc55cSDimitry Andric     return false;
458349cc55cSDimitry Andric   uint32_t NodeCount = std::move(*ErrorOrNodeCount);
459349cc55cSDimitry Andric   // Read number of direct inlinees
460349cc55cSDimitry Andric   auto ErrorOrCurChildrenToProcess = readUnsignedNumber<uint32_t>();
461349cc55cSDimitry Andric   if (!ErrorOrCurChildrenToProcess)
462349cc55cSDimitry Andric     return false;
463349cc55cSDimitry Andric   // Read all probes in this node
464349cc55cSDimitry Andric   for (std::size_t I = 0; I < NodeCount; I++) {
465349cc55cSDimitry Andric     // Read index
466349cc55cSDimitry Andric     auto ErrorOrIndex = readUnsignedNumber<uint32_t>();
467349cc55cSDimitry Andric     if (!ErrorOrIndex)
468349cc55cSDimitry Andric       return false;
469349cc55cSDimitry Andric     uint32_t Index = std::move(*ErrorOrIndex);
470349cc55cSDimitry Andric     // Read type | flag.
471349cc55cSDimitry Andric     auto ErrorOrValue = readUnencodedNumber<uint8_t>();
472349cc55cSDimitry Andric     if (!ErrorOrValue)
473349cc55cSDimitry Andric       return false;
474349cc55cSDimitry Andric     uint8_t Value = std::move(*ErrorOrValue);
475349cc55cSDimitry Andric     uint8_t Kind = Value & 0xf;
476349cc55cSDimitry Andric     uint8_t Attr = (Value & 0x70) >> 4;
477349cc55cSDimitry Andric     // Read address
478349cc55cSDimitry Andric     uint64_t Addr = 0;
479349cc55cSDimitry Andric     if (Value & 0x80) {
480349cc55cSDimitry Andric       auto ErrorOrOffset = readSignedNumber<int64_t>();
481349cc55cSDimitry Andric       if (!ErrorOrOffset)
482349cc55cSDimitry Andric         return false;
483349cc55cSDimitry Andric       int64_t Offset = std::move(*ErrorOrOffset);
484349cc55cSDimitry Andric       Addr = LastAddr + Offset;
485349cc55cSDimitry Andric     } else {
486349cc55cSDimitry Andric       auto ErrorOrAddr = readUnencodedNumber<int64_t>();
487349cc55cSDimitry Andric       if (!ErrorOrAddr)
488349cc55cSDimitry Andric         return false;
489349cc55cSDimitry Andric       Addr = std::move(*ErrorOrAddr);
490bdd1243dSDimitry Andric       if (isSentinelProbe(Attr)) {
491bdd1243dSDimitry Andric         // For sentinel probe, the addr field actually stores the GUID of the
492bdd1243dSDimitry Andric         // split function. Convert it to the real address.
493bdd1243dSDimitry Andric         if (auto V = FuncStartAddrs.lookup(Addr))
494bdd1243dSDimitry Andric           Addr = V;
495bdd1243dSDimitry Andric       } else {
496bdd1243dSDimitry Andric         // For now we assume all probe encoding should be either based on
497bdd1243dSDimitry Andric         // leading probe address or function start address.
498bdd1243dSDimitry Andric         // The scheme is for downwards compatibility.
499bdd1243dSDimitry Andric         // TODO: retire this scheme once compatibility is no longer an issue.
500bdd1243dSDimitry Andric         EncodingIsAddrBased = true;
501bdd1243dSDimitry Andric       }
502349cc55cSDimitry Andric     }
50381ad6265SDimitry Andric 
504*06c3fb27SDimitry Andric     uint32_t Discriminator = 0;
505*06c3fb27SDimitry Andric     if (hasDiscriminator(Attr)) {
506*06c3fb27SDimitry Andric       auto ErrorOrDiscriminator = readUnsignedNumber<uint32_t>();
507*06c3fb27SDimitry Andric       if (!ErrorOrDiscriminator)
508*06c3fb27SDimitry Andric         return false;
509*06c3fb27SDimitry Andric       Discriminator = std::move(*ErrorOrDiscriminator);
510*06c3fb27SDimitry Andric     }
511*06c3fb27SDimitry Andric 
512bdd1243dSDimitry Andric     if (Cur && !isSentinelProbe(Attr)) {
513349cc55cSDimitry Andric       // Populate Address2ProbesMap
514349cc55cSDimitry Andric       auto &Probes = Address2ProbesMap[Addr];
515349cc55cSDimitry Andric       Probes.emplace_back(Addr, Cur->Guid, Index, PseudoProbeType(Kind), Attr,
516*06c3fb27SDimitry Andric                           Discriminator, Cur);
517349cc55cSDimitry Andric       Cur->addProbes(&Probes.back());
51881ad6265SDimitry Andric     }
519349cc55cSDimitry Andric     LastAddr = Addr;
520349cc55cSDimitry Andric   }
521349cc55cSDimitry Andric 
52281ad6265SDimitry Andric   uint32_t ChildrenToProcess = std::move(*ErrorOrCurChildrenToProcess);
52381ad6265SDimitry Andric   for (uint32_t I = 0; I < ChildrenToProcess; I++) {
524bdd1243dSDimitry Andric     buildAddress2ProbeMap(Cur, LastAddr, GuidFilter, FuncStartAddrs);
525349cc55cSDimitry Andric   }
526349cc55cSDimitry Andric 
527349cc55cSDimitry Andric   return true;
528349cc55cSDimitry Andric }
529349cc55cSDimitry Andric 
53081ad6265SDimitry Andric bool MCPseudoProbeDecoder::buildAddress2ProbeMap(
531bdd1243dSDimitry Andric     const uint8_t *Start, std::size_t Size, const Uint64Set &GuidFilter,
532bdd1243dSDimitry Andric     const Uint64Map &FuncStartAddrs) {
53381ad6265SDimitry Andric   Data = Start;
53481ad6265SDimitry Andric   End = Data + Size;
53581ad6265SDimitry Andric   uint64_t LastAddr = 0;
53681ad6265SDimitry Andric   while (Data < End)
537bdd1243dSDimitry Andric     buildAddress2ProbeMap(&DummyInlineRoot, LastAddr, GuidFilter,
538bdd1243dSDimitry Andric                           FuncStartAddrs);
53981ad6265SDimitry Andric   assert(Data == End && "Have unprocessed data in pseudo_probe section");
54081ad6265SDimitry Andric   return true;
54181ad6265SDimitry Andric }
54281ad6265SDimitry Andric 
543349cc55cSDimitry Andric void MCPseudoProbeDecoder::printGUID2FuncDescMap(raw_ostream &OS) {
544349cc55cSDimitry Andric   OS << "Pseudo Probe Desc:\n";
545349cc55cSDimitry Andric   // Make the output deterministic
546349cc55cSDimitry Andric   std::map<uint64_t, MCPseudoProbeFuncDesc> OrderedMap(GUID2FuncDescMap.begin(),
547349cc55cSDimitry Andric                                                        GUID2FuncDescMap.end());
548349cc55cSDimitry Andric   for (auto &I : OrderedMap) {
549349cc55cSDimitry Andric     I.second.print(OS);
550349cc55cSDimitry Andric   }
551349cc55cSDimitry Andric }
552349cc55cSDimitry Andric 
553349cc55cSDimitry Andric void MCPseudoProbeDecoder::printProbeForAddress(raw_ostream &OS,
554349cc55cSDimitry Andric                                                 uint64_t Address) {
555349cc55cSDimitry Andric   auto It = Address2ProbesMap.find(Address);
556349cc55cSDimitry Andric   if (It != Address2ProbesMap.end()) {
557349cc55cSDimitry Andric     for (auto &Probe : It->second) {
558349cc55cSDimitry Andric       OS << " [Probe]:\t";
559349cc55cSDimitry Andric       Probe.print(OS, GUID2FuncDescMap, true);
560349cc55cSDimitry Andric     }
561349cc55cSDimitry Andric   }
562349cc55cSDimitry Andric }
563349cc55cSDimitry Andric 
564349cc55cSDimitry Andric void MCPseudoProbeDecoder::printProbesForAllAddresses(raw_ostream &OS) {
565349cc55cSDimitry Andric   std::vector<uint64_t> Addresses;
566349cc55cSDimitry Andric   for (auto Entry : Address2ProbesMap)
567349cc55cSDimitry Andric     Addresses.push_back(Entry.first);
568fcaf7f86SDimitry Andric   llvm::sort(Addresses);
569349cc55cSDimitry Andric   for (auto K : Addresses) {
570349cc55cSDimitry Andric     OS << "Address:\t";
571349cc55cSDimitry Andric     OS << K;
572349cc55cSDimitry Andric     OS << "\n";
573349cc55cSDimitry Andric     printProbeForAddress(OS, K);
574349cc55cSDimitry Andric   }
575349cc55cSDimitry Andric }
576349cc55cSDimitry Andric 
577349cc55cSDimitry Andric const MCDecodedPseudoProbe *
578349cc55cSDimitry Andric MCPseudoProbeDecoder::getCallProbeForAddr(uint64_t Address) const {
579349cc55cSDimitry Andric   auto It = Address2ProbesMap.find(Address);
580349cc55cSDimitry Andric   if (It == Address2ProbesMap.end())
581349cc55cSDimitry Andric     return nullptr;
582349cc55cSDimitry Andric   const auto &Probes = It->second;
583349cc55cSDimitry Andric 
584349cc55cSDimitry Andric   const MCDecodedPseudoProbe *CallProbe = nullptr;
585349cc55cSDimitry Andric   for (const auto &Probe : Probes) {
586349cc55cSDimitry Andric     if (Probe.isCall()) {
587*06c3fb27SDimitry Andric       // Disabling the assert and returning first call probe seen so far.
588*06c3fb27SDimitry Andric       // Subsequent call probes, if any, are ignored. Due to the the way
589*06c3fb27SDimitry Andric       // .pseudo_probe section is decoded, probes of the same-named independent
590*06c3fb27SDimitry Andric       // static functions are merged thus multiple call probes may be seen for a
591*06c3fb27SDimitry Andric       // callsite. This should only happen to compiler-generated statics, with
592*06c3fb27SDimitry Andric       // -funique-internal-linkage-names where user statics get unique names.
593*06c3fb27SDimitry Andric       //
594*06c3fb27SDimitry Andric       // TODO: re-enable or narrow down the assert to static functions only.
595*06c3fb27SDimitry Andric       //
596*06c3fb27SDimitry Andric       // assert(!CallProbe &&
597*06c3fb27SDimitry Andric       //        "There should be only one call probe corresponding to address "
598*06c3fb27SDimitry Andric       //        "which is a callsite.");
599349cc55cSDimitry Andric       CallProbe = &Probe;
600*06c3fb27SDimitry Andric       break;
601349cc55cSDimitry Andric     }
602349cc55cSDimitry Andric   }
603349cc55cSDimitry Andric   return CallProbe;
604349cc55cSDimitry Andric }
605349cc55cSDimitry Andric 
606349cc55cSDimitry Andric const MCPseudoProbeFuncDesc *
607349cc55cSDimitry Andric MCPseudoProbeDecoder::getFuncDescForGUID(uint64_t GUID) const {
608349cc55cSDimitry Andric   auto It = GUID2FuncDescMap.find(GUID);
609349cc55cSDimitry Andric   assert(It != GUID2FuncDescMap.end() && "Function descriptor doesn't exist");
610349cc55cSDimitry Andric   return &It->second;
611349cc55cSDimitry Andric }
612349cc55cSDimitry Andric 
613349cc55cSDimitry Andric void MCPseudoProbeDecoder::getInlineContextForProbe(
614349cc55cSDimitry Andric     const MCDecodedPseudoProbe *Probe,
615349cc55cSDimitry Andric     SmallVectorImpl<MCPseduoProbeFrameLocation> &InlineContextStack,
616349cc55cSDimitry Andric     bool IncludeLeaf) const {
617349cc55cSDimitry Andric   Probe->getInlineContext(InlineContextStack, GUID2FuncDescMap);
618349cc55cSDimitry Andric   if (!IncludeLeaf)
619349cc55cSDimitry Andric     return;
620349cc55cSDimitry Andric   // Note that the context from probe doesn't include leaf frame,
621349cc55cSDimitry Andric   // hence we need to retrieve and prepend leaf if requested.
622349cc55cSDimitry Andric   const auto *FuncDesc = getFuncDescForGUID(Probe->getGuid());
623349cc55cSDimitry Andric   InlineContextStack.emplace_back(
624349cc55cSDimitry Andric       MCPseduoProbeFrameLocation(FuncDesc->FuncName, Probe->getIndex()));
625349cc55cSDimitry Andric }
626349cc55cSDimitry Andric 
627349cc55cSDimitry Andric const MCPseudoProbeFuncDesc *MCPseudoProbeDecoder::getInlinerDescForProbe(
628349cc55cSDimitry Andric     const MCDecodedPseudoProbe *Probe) const {
629349cc55cSDimitry Andric   MCDecodedPseudoProbeInlineTree *InlinerNode = Probe->getInlineTreeNode();
630349cc55cSDimitry Andric   if (!InlinerNode->hasInlineSite())
631349cc55cSDimitry Andric     return nullptr;
63281ad6265SDimitry Andric   return getFuncDescForGUID(InlinerNode->Parent->Guid);
633349cc55cSDimitry Andric }
634