1e8d8bef9SDimitry Andric //===- lib/MC/MCPseudoProbe.cpp - Pseudo probe encoding support ----------===// 2e8d8bef9SDimitry Andric // 3e8d8bef9SDimitry Andric // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4e8d8bef9SDimitry Andric // See https://llvm.org/LICENSE.txt for license information. 5e8d8bef9SDimitry Andric // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6e8d8bef9SDimitry Andric // 7e8d8bef9SDimitry Andric //===----------------------------------------------------------------------===// 8e8d8bef9SDimitry Andric 9e8d8bef9SDimitry Andric #include "llvm/MC/MCPseudoProbe.h" 10fcaf7f86SDimitry Andric #include "llvm/ADT/STLExtras.h" 11*06c3fb27SDimitry Andric #include "llvm/IR/PseudoProbe.h" 12e8d8bef9SDimitry Andric #include "llvm/MC/MCAsmInfo.h" 13e8d8bef9SDimitry Andric #include "llvm/MC/MCContext.h" 1481ad6265SDimitry Andric #include "llvm/MC/MCExpr.h" 1581ad6265SDimitry Andric #include "llvm/MC/MCFragment.h" 16e8d8bef9SDimitry Andric #include "llvm/MC/MCObjectFileInfo.h" 17e8d8bef9SDimitry Andric #include "llvm/MC/MCObjectStreamer.h" 18bdd1243dSDimitry Andric #include "llvm/MC/MCSymbol.h" 19349cc55cSDimitry Andric #include "llvm/Support/Endian.h" 20349cc55cSDimitry Andric #include "llvm/Support/LEB128.h" 21bdd1243dSDimitry Andric #include "llvm/Support/MD5.h" 22349cc55cSDimitry Andric #include "llvm/Support/raw_ostream.h" 23bdd1243dSDimitry Andric #include <algorithm> 24bdd1243dSDimitry Andric #include <cassert> 25349cc55cSDimitry Andric #include <limits> 26349cc55cSDimitry Andric #include <memory> 27349cc55cSDimitry Andric #include <sstream> 28bdd1243dSDimitry Andric #include <vector> 29e8d8bef9SDimitry Andric 30e8d8bef9SDimitry Andric #define DEBUG_TYPE "mcpseudoprobe" 31e8d8bef9SDimitry Andric 32e8d8bef9SDimitry Andric using namespace llvm; 33349cc55cSDimitry Andric using namespace support; 34e8d8bef9SDimitry Andric 35e8d8bef9SDimitry Andric #ifndef NDEBUG 36e8d8bef9SDimitry Andric int MCPseudoProbeTable::DdgPrintIndent = 0; 37e8d8bef9SDimitry Andric #endif 38e8d8bef9SDimitry Andric 39e8d8bef9SDimitry Andric static const MCExpr *buildSymbolDiff(MCObjectStreamer *MCOS, const MCSymbol *A, 40e8d8bef9SDimitry Andric const MCSymbol *B) { 41e8d8bef9SDimitry Andric MCContext &Context = MCOS->getContext(); 42e8d8bef9SDimitry Andric MCSymbolRefExpr::VariantKind Variant = MCSymbolRefExpr::VK_None; 43e8d8bef9SDimitry Andric const MCExpr *ARef = MCSymbolRefExpr::create(A, Variant, Context); 44e8d8bef9SDimitry Andric const MCExpr *BRef = MCSymbolRefExpr::create(B, Variant, Context); 45e8d8bef9SDimitry Andric const MCExpr *AddrDelta = 46e8d8bef9SDimitry Andric MCBinaryExpr::create(MCBinaryExpr::Sub, ARef, BRef, Context); 47e8d8bef9SDimitry Andric return AddrDelta; 48e8d8bef9SDimitry Andric } 49e8d8bef9SDimitry Andric 50e8d8bef9SDimitry Andric void MCPseudoProbe::emit(MCObjectStreamer *MCOS, 51e8d8bef9SDimitry Andric const MCPseudoProbe *LastProbe) const { 52bdd1243dSDimitry Andric bool IsSentinel = isSentinelProbe(getAttributes()); 53bdd1243dSDimitry Andric assert((LastProbe || IsSentinel) && 54bdd1243dSDimitry Andric "Last probe should not be null for non-sentinel probes"); 55bdd1243dSDimitry Andric 56e8d8bef9SDimitry Andric // Emit Index 57e8d8bef9SDimitry Andric MCOS->emitULEB128IntValue(Index); 58e8d8bef9SDimitry Andric // Emit Type and the flag: 59e8d8bef9SDimitry Andric // Type (bit 0 to 3), with bit 4 to 6 for attributes. 60e8d8bef9SDimitry Andric // Flag (bit 7, 0 - code address, 1 - address delta). This indicates whether 61e8d8bef9SDimitry Andric // the following field is a symbolic code address or an address delta. 62*06c3fb27SDimitry Andric // Emit FS discriminator 63e8d8bef9SDimitry Andric assert(Type <= 0xF && "Probe type too big to encode, exceeding 15"); 64*06c3fb27SDimitry Andric auto NewAttributes = Attributes; 65*06c3fb27SDimitry Andric if (Discriminator) 66*06c3fb27SDimitry Andric NewAttributes |= (uint32_t)PseudoProbeAttributes::HasDiscriminator; 67*06c3fb27SDimitry Andric assert(NewAttributes <= 0x7 && 68e8d8bef9SDimitry Andric "Probe attributes too big to encode, exceeding 7"); 69*06c3fb27SDimitry Andric uint8_t PackedType = Type | (NewAttributes << 4); 70bdd1243dSDimitry Andric uint8_t Flag = 71bdd1243dSDimitry Andric !IsSentinel ? ((int8_t)MCPseudoProbeFlag::AddressDelta << 7) : 0; 72e8d8bef9SDimitry Andric MCOS->emitInt8(Flag | PackedType); 73e8d8bef9SDimitry Andric 74bdd1243dSDimitry Andric if (!IsSentinel) { 75e8d8bef9SDimitry Andric // Emit the delta between the address label and LastProbe. 76e8d8bef9SDimitry Andric const MCExpr *AddrDelta = 77e8d8bef9SDimitry Andric buildSymbolDiff(MCOS, Label, LastProbe->getLabel()); 78e8d8bef9SDimitry Andric int64_t Delta; 79e8d8bef9SDimitry Andric if (AddrDelta->evaluateAsAbsolute(Delta, MCOS->getAssemblerPtr())) { 80e8d8bef9SDimitry Andric MCOS->emitSLEB128IntValue(Delta); 81e8d8bef9SDimitry Andric } else { 82e8d8bef9SDimitry Andric MCOS->insert(new MCPseudoProbeAddrFragment(AddrDelta)); 83e8d8bef9SDimitry Andric } 84e8d8bef9SDimitry Andric } else { 85bdd1243dSDimitry Andric // Emit the GUID of the split function that the sentinel probe represents. 86bdd1243dSDimitry Andric MCOS->emitInt64(Guid); 87e8d8bef9SDimitry Andric } 88e8d8bef9SDimitry Andric 89*06c3fb27SDimitry Andric if (Discriminator) 90*06c3fb27SDimitry Andric MCOS->emitULEB128IntValue(Discriminator); 91*06c3fb27SDimitry Andric 92e8d8bef9SDimitry Andric LLVM_DEBUG({ 93e8d8bef9SDimitry Andric dbgs().indent(MCPseudoProbeTable::DdgPrintIndent); 94e8d8bef9SDimitry Andric dbgs() << "Probe: " << Index << "\n"; 95e8d8bef9SDimitry Andric }); 96e8d8bef9SDimitry Andric } 97e8d8bef9SDimitry Andric 98e8d8bef9SDimitry Andric void MCPseudoProbeInlineTree::addPseudoProbe( 99e8d8bef9SDimitry Andric const MCPseudoProbe &Probe, const MCPseudoProbeInlineStack &InlineStack) { 100e8d8bef9SDimitry Andric // The function should not be called on the root. 101bdd1243dSDimitry Andric assert(isRoot() && "Should only be called on root"); 102e8d8bef9SDimitry Andric 103e8d8bef9SDimitry Andric // When it comes here, the input look like: 104e8d8bef9SDimitry Andric // Probe: GUID of C, ... 105e8d8bef9SDimitry Andric // InlineStack: [88, A], [66, B] 106e8d8bef9SDimitry Andric // which means, Function A inlines function B at call site with a probe id of 107e8d8bef9SDimitry Andric // 88, and B inlines C at probe 66. The tri-tree expects a tree path like {[0, 108e8d8bef9SDimitry Andric // A], [88, B], [66, C]} to locate the tree node where the probe should be 109e8d8bef9SDimitry Andric // added. Note that the edge [0, A] means A is the top-level function we are 110e8d8bef9SDimitry Andric // emitting probes for. 111e8d8bef9SDimitry Andric 112e8d8bef9SDimitry Andric // Make a [0, A] edge. 113e8d8bef9SDimitry Andric // An empty inline stack means the function that the probe originates from 114e8d8bef9SDimitry Andric // is a top-level function. 115e8d8bef9SDimitry Andric InlineSite Top; 116e8d8bef9SDimitry Andric if (InlineStack.empty()) { 117e8d8bef9SDimitry Andric Top = InlineSite(Probe.getGuid(), 0); 118e8d8bef9SDimitry Andric } else { 119e8d8bef9SDimitry Andric Top = InlineSite(std::get<0>(InlineStack.front()), 0); 120e8d8bef9SDimitry Andric } 121e8d8bef9SDimitry Andric 122e8d8bef9SDimitry Andric auto *Cur = getOrAddNode(Top); 123e8d8bef9SDimitry Andric 124e8d8bef9SDimitry Andric // Make interior edges by walking the inline stack. Once it's done, Cur should 125e8d8bef9SDimitry Andric // point to the node that the probe originates from. 126e8d8bef9SDimitry Andric if (!InlineStack.empty()) { 127e8d8bef9SDimitry Andric auto Iter = InlineStack.begin(); 128e8d8bef9SDimitry Andric auto Index = std::get<1>(*Iter); 129e8d8bef9SDimitry Andric Iter++; 130e8d8bef9SDimitry Andric for (; Iter != InlineStack.end(); Iter++) { 131e8d8bef9SDimitry Andric // Make an edge by using the previous probe id and current GUID. 132e8d8bef9SDimitry Andric Cur = Cur->getOrAddNode(InlineSite(std::get<0>(*Iter), Index)); 133e8d8bef9SDimitry Andric Index = std::get<1>(*Iter); 134e8d8bef9SDimitry Andric } 135e8d8bef9SDimitry Andric Cur = Cur->getOrAddNode(InlineSite(Probe.getGuid(), Index)); 136e8d8bef9SDimitry Andric } 137e8d8bef9SDimitry Andric 138e8d8bef9SDimitry Andric Cur->Probes.push_back(Probe); 139e8d8bef9SDimitry Andric } 140e8d8bef9SDimitry Andric 141e8d8bef9SDimitry Andric void MCPseudoProbeInlineTree::emit(MCObjectStreamer *MCOS, 142e8d8bef9SDimitry Andric const MCPseudoProbe *&LastProbe) { 143e8d8bef9SDimitry Andric LLVM_DEBUG({ 144e8d8bef9SDimitry Andric dbgs().indent(MCPseudoProbeTable::DdgPrintIndent); 145e8d8bef9SDimitry Andric dbgs() << "Group [\n"; 146e8d8bef9SDimitry Andric MCPseudoProbeTable::DdgPrintIndent += 2; 147e8d8bef9SDimitry Andric }); 148bdd1243dSDimitry Andric assert(!isRoot() && "Root should be handled seperately"); 149bdd1243dSDimitry Andric 150e8d8bef9SDimitry Andric // Emit probes grouped by GUID. 151e8d8bef9SDimitry Andric LLVM_DEBUG({ 152e8d8bef9SDimitry Andric dbgs().indent(MCPseudoProbeTable::DdgPrintIndent); 153e8d8bef9SDimitry Andric dbgs() << "GUID: " << Guid << "\n"; 154e8d8bef9SDimitry Andric }); 155e8d8bef9SDimitry Andric // Emit Guid 156e8d8bef9SDimitry Andric MCOS->emitInt64(Guid); 157bdd1243dSDimitry Andric // Emit number of probes in this node, including a sentinel probe for 158bdd1243dSDimitry Andric // top-level functions if needed. 159bdd1243dSDimitry Andric bool NeedSentinel = false; 160bdd1243dSDimitry Andric if (Parent->isRoot()) { 161bdd1243dSDimitry Andric assert(isSentinelProbe(LastProbe->getAttributes()) && 162bdd1243dSDimitry Andric "Starting probe of a top-level function should be a sentinel probe"); 163bdd1243dSDimitry Andric // The main body of a split function doesn't need a sentinel probe. 164bdd1243dSDimitry Andric if (LastProbe->getGuid() != Guid) 165bdd1243dSDimitry Andric NeedSentinel = true; 166bdd1243dSDimitry Andric } 167bdd1243dSDimitry Andric 168bdd1243dSDimitry Andric MCOS->emitULEB128IntValue(Probes.size() + NeedSentinel); 169e8d8bef9SDimitry Andric // Emit number of direct inlinees 170349cc55cSDimitry Andric MCOS->emitULEB128IntValue(Children.size()); 171bdd1243dSDimitry Andric // Emit sentinel probe for top-level functions 172bdd1243dSDimitry Andric if (NeedSentinel) 173bdd1243dSDimitry Andric LastProbe->emit(MCOS, nullptr); 174bdd1243dSDimitry Andric 175e8d8bef9SDimitry Andric // Emit probes in this group 176e8d8bef9SDimitry Andric for (const auto &Probe : Probes) { 177e8d8bef9SDimitry Andric Probe.emit(MCOS, LastProbe); 178e8d8bef9SDimitry Andric LastProbe = &Probe; 179e8d8bef9SDimitry Andric } 180e8d8bef9SDimitry Andric 181bdd1243dSDimitry Andric // Emit sorted descendant. InlineSite is unique for each pair, so there will 182bdd1243dSDimitry Andric // be no ordering of Inlinee based on MCPseudoProbeInlineTree* 183bdd1243dSDimitry Andric using InlineeType = std::pair<InlineSite, MCPseudoProbeInlineTree *>; 184bdd1243dSDimitry Andric auto Comparer = [](const InlineeType &A, const InlineeType &B) { 185bdd1243dSDimitry Andric return A.first < B.first; 186bdd1243dSDimitry Andric }; 187bdd1243dSDimitry Andric std::vector<InlineeType> Inlinees; 188bdd1243dSDimitry Andric for (const auto &Child : Children) 189bdd1243dSDimitry Andric Inlinees.emplace_back(Child.first, Child.second.get()); 190bdd1243dSDimitry Andric std::sort(Inlinees.begin(), Inlinees.end(), Comparer); 191349cc55cSDimitry Andric 192e8d8bef9SDimitry Andric for (const auto &Inlinee : Inlinees) { 193e8d8bef9SDimitry Andric // Emit probe index 194e8d8bef9SDimitry Andric MCOS->emitULEB128IntValue(std::get<1>(Inlinee.first)); 195e8d8bef9SDimitry Andric LLVM_DEBUG({ 196e8d8bef9SDimitry Andric dbgs().indent(MCPseudoProbeTable::DdgPrintIndent); 197e8d8bef9SDimitry Andric dbgs() << "InlineSite: " << std::get<1>(Inlinee.first) << "\n"; 198e8d8bef9SDimitry Andric }); 199e8d8bef9SDimitry Andric // Emit the group 200e8d8bef9SDimitry Andric Inlinee.second->emit(MCOS, LastProbe); 201e8d8bef9SDimitry Andric } 202e8d8bef9SDimitry Andric 203e8d8bef9SDimitry Andric LLVM_DEBUG({ 204e8d8bef9SDimitry Andric MCPseudoProbeTable::DdgPrintIndent -= 2; 205e8d8bef9SDimitry Andric dbgs().indent(MCPseudoProbeTable::DdgPrintIndent); 206e8d8bef9SDimitry Andric dbgs() << "]\n"; 207e8d8bef9SDimitry Andric }); 208e8d8bef9SDimitry Andric } 209e8d8bef9SDimitry Andric 210bdd1243dSDimitry Andric void MCPseudoProbeSections::emit(MCObjectStreamer *MCOS) { 211e8d8bef9SDimitry Andric MCContext &Ctx = MCOS->getContext(); 212e8d8bef9SDimitry Andric for (auto &ProbeSec : MCProbeDivisions) { 213bdd1243dSDimitry Andric const auto *FuncSym = ProbeSec.first; 214bdd1243dSDimitry Andric const auto &Root = ProbeSec.second; 215bdd1243dSDimitry Andric if (auto *S = Ctx.getObjectFileInfo()->getPseudoProbeSection( 216bdd1243dSDimitry Andric FuncSym->getSection())) { 217e8d8bef9SDimitry Andric // Switch to the .pseudoprobe section or a comdat group. 21881ad6265SDimitry Andric MCOS->switchSection(S); 219e8d8bef9SDimitry Andric // Emit probes grouped by GUID. 220bdd1243dSDimitry Andric // Emit sorted descendant. InlineSite is unique for each pair, so there 221bdd1243dSDimitry Andric // will be no ordering of Inlinee based on MCPseudoProbeInlineTree* 222bdd1243dSDimitry Andric using InlineeType = std::pair<InlineSite, MCPseudoProbeInlineTree *>; 223bdd1243dSDimitry Andric auto Comparer = [](const InlineeType &A, const InlineeType &B) { 224bdd1243dSDimitry Andric return A.first < B.first; 225bdd1243dSDimitry Andric }; 226bdd1243dSDimitry Andric std::vector<InlineeType> Inlinees; 227bdd1243dSDimitry Andric for (const auto &Child : Root.getChildren()) 228bdd1243dSDimitry Andric Inlinees.emplace_back(Child.first, Child.second.get()); 229bdd1243dSDimitry Andric std::sort(Inlinees.begin(), Inlinees.end(), Comparer); 230bdd1243dSDimitry Andric 231bdd1243dSDimitry Andric for (const auto &Inlinee : Inlinees) { 232bdd1243dSDimitry Andric // Emit the group guarded by a sentinel probe. 233*06c3fb27SDimitry Andric MCPseudoProbe SentinelProbe( 234*06c3fb27SDimitry Andric const_cast<MCSymbol *>(FuncSym), MD5Hash(FuncSym->getName()), 235bdd1243dSDimitry Andric (uint32_t)PseudoProbeReservedId::Invalid, 236bdd1243dSDimitry Andric (uint32_t)PseudoProbeType::Block, 237*06c3fb27SDimitry Andric (uint32_t)PseudoProbeAttributes::Sentinel, 0); 238bdd1243dSDimitry Andric const MCPseudoProbe *Probe = &SentinelProbe; 239bdd1243dSDimitry Andric Inlinee.second->emit(MCOS, Probe); 240bdd1243dSDimitry Andric } 241e8d8bef9SDimitry Andric } 242e8d8bef9SDimitry Andric } 243e8d8bef9SDimitry Andric } 244e8d8bef9SDimitry Andric 245e8d8bef9SDimitry Andric // 246e8d8bef9SDimitry Andric // This emits the pseudo probe tables. 247e8d8bef9SDimitry Andric // 248e8d8bef9SDimitry Andric void MCPseudoProbeTable::emit(MCObjectStreamer *MCOS) { 249e8d8bef9SDimitry Andric MCContext &Ctx = MCOS->getContext(); 250e8d8bef9SDimitry Andric auto &ProbeTable = Ctx.getMCPseudoProbeTable(); 251e8d8bef9SDimitry Andric 252e8d8bef9SDimitry Andric // Bail out early so we don't switch to the pseudo_probe section needlessly 253e8d8bef9SDimitry Andric // and in doing so create an unnecessary (if empty) section. 254e8d8bef9SDimitry Andric auto &ProbeSections = ProbeTable.getProbeSections(); 255e8d8bef9SDimitry Andric if (ProbeSections.empty()) 256e8d8bef9SDimitry Andric return; 257e8d8bef9SDimitry Andric 258e8d8bef9SDimitry Andric LLVM_DEBUG(MCPseudoProbeTable::DdgPrintIndent = 0); 259e8d8bef9SDimitry Andric 260e8d8bef9SDimitry Andric // Put out the probe. 261e8d8bef9SDimitry Andric ProbeSections.emit(MCOS); 262e8d8bef9SDimitry Andric } 263349cc55cSDimitry Andric 264349cc55cSDimitry Andric static StringRef getProbeFNameForGUID(const GUIDProbeFunctionMap &GUID2FuncMAP, 265349cc55cSDimitry Andric uint64_t GUID) { 266349cc55cSDimitry Andric auto It = GUID2FuncMAP.find(GUID); 267349cc55cSDimitry Andric assert(It != GUID2FuncMAP.end() && 268349cc55cSDimitry Andric "Probe function must exist for a valid GUID"); 269349cc55cSDimitry Andric return It->second.FuncName; 270349cc55cSDimitry Andric } 271349cc55cSDimitry Andric 272349cc55cSDimitry Andric void MCPseudoProbeFuncDesc::print(raw_ostream &OS) { 273349cc55cSDimitry Andric OS << "GUID: " << FuncGUID << " Name: " << FuncName << "\n"; 274349cc55cSDimitry Andric OS << "Hash: " << FuncHash << "\n"; 275349cc55cSDimitry Andric } 276349cc55cSDimitry Andric 277349cc55cSDimitry Andric void MCDecodedPseudoProbe::getInlineContext( 278349cc55cSDimitry Andric SmallVectorImpl<MCPseduoProbeFrameLocation> &ContextStack, 279349cc55cSDimitry Andric const GUIDProbeFunctionMap &GUID2FuncMAP) const { 280349cc55cSDimitry Andric uint32_t Begin = ContextStack.size(); 281349cc55cSDimitry Andric MCDecodedPseudoProbeInlineTree *Cur = InlineTree; 282349cc55cSDimitry Andric // It will add the string of each node's inline site during iteration. 283349cc55cSDimitry Andric // Note that it won't include the probe's belonging function(leaf location) 284349cc55cSDimitry Andric while (Cur->hasInlineSite()) { 28581ad6265SDimitry Andric StringRef FuncName = getProbeFNameForGUID(GUID2FuncMAP, Cur->Parent->Guid); 286349cc55cSDimitry Andric ContextStack.emplace_back( 287349cc55cSDimitry Andric MCPseduoProbeFrameLocation(FuncName, std::get<1>(Cur->ISite))); 288349cc55cSDimitry Andric Cur = static_cast<MCDecodedPseudoProbeInlineTree *>(Cur->Parent); 289349cc55cSDimitry Andric } 290349cc55cSDimitry Andric // Make the ContextStack in caller-callee order 291349cc55cSDimitry Andric std::reverse(ContextStack.begin() + Begin, ContextStack.end()); 292349cc55cSDimitry Andric } 293349cc55cSDimitry Andric 294349cc55cSDimitry Andric std::string MCDecodedPseudoProbe::getInlineContextStr( 295349cc55cSDimitry Andric const GUIDProbeFunctionMap &GUID2FuncMAP) const { 296349cc55cSDimitry Andric std::ostringstream OContextStr; 297349cc55cSDimitry Andric SmallVector<MCPseduoProbeFrameLocation, 16> ContextStack; 298349cc55cSDimitry Andric getInlineContext(ContextStack, GUID2FuncMAP); 299349cc55cSDimitry Andric for (auto &Cxt : ContextStack) { 300349cc55cSDimitry Andric if (OContextStr.str().size()) 301349cc55cSDimitry Andric OContextStr << " @ "; 302349cc55cSDimitry Andric OContextStr << Cxt.first.str() << ":" << Cxt.second; 303349cc55cSDimitry Andric } 304349cc55cSDimitry Andric return OContextStr.str(); 305349cc55cSDimitry Andric } 306349cc55cSDimitry Andric 307349cc55cSDimitry Andric static const char *PseudoProbeTypeStr[3] = {"Block", "IndirectCall", 308349cc55cSDimitry Andric "DirectCall"}; 309349cc55cSDimitry Andric 310349cc55cSDimitry Andric void MCDecodedPseudoProbe::print(raw_ostream &OS, 311349cc55cSDimitry Andric const GUIDProbeFunctionMap &GUID2FuncMAP, 312349cc55cSDimitry Andric bool ShowName) const { 313349cc55cSDimitry Andric OS << "FUNC: "; 314349cc55cSDimitry Andric if (ShowName) { 315349cc55cSDimitry Andric StringRef FuncName = getProbeFNameForGUID(GUID2FuncMAP, Guid); 316349cc55cSDimitry Andric OS << FuncName.str() << " "; 317349cc55cSDimitry Andric } else { 318349cc55cSDimitry Andric OS << Guid << " "; 319349cc55cSDimitry Andric } 320349cc55cSDimitry Andric OS << "Index: " << Index << " "; 321*06c3fb27SDimitry Andric if (Discriminator) 322*06c3fb27SDimitry Andric OS << "Discriminator: " << Discriminator << " "; 323349cc55cSDimitry Andric OS << "Type: " << PseudoProbeTypeStr[static_cast<uint8_t>(Type)] << " "; 324349cc55cSDimitry Andric std::string InlineContextStr = getInlineContextStr(GUID2FuncMAP); 325349cc55cSDimitry Andric if (InlineContextStr.size()) { 326349cc55cSDimitry Andric OS << "Inlined: @ "; 327349cc55cSDimitry Andric OS << InlineContextStr; 328349cc55cSDimitry Andric } 329349cc55cSDimitry Andric OS << "\n"; 330349cc55cSDimitry Andric } 331349cc55cSDimitry Andric 332349cc55cSDimitry Andric template <typename T> ErrorOr<T> MCPseudoProbeDecoder::readUnencodedNumber() { 333349cc55cSDimitry Andric if (Data + sizeof(T) > End) { 334349cc55cSDimitry Andric return std::error_code(); 335349cc55cSDimitry Andric } 336349cc55cSDimitry Andric T Val = endian::readNext<T, little, unaligned>(Data); 337349cc55cSDimitry Andric return ErrorOr<T>(Val); 338349cc55cSDimitry Andric } 339349cc55cSDimitry Andric 340349cc55cSDimitry Andric template <typename T> ErrorOr<T> MCPseudoProbeDecoder::readUnsignedNumber() { 341349cc55cSDimitry Andric unsigned NumBytesRead = 0; 342349cc55cSDimitry Andric uint64_t Val = decodeULEB128(Data, &NumBytesRead); 343349cc55cSDimitry Andric if (Val > std::numeric_limits<T>::max() || (Data + NumBytesRead > End)) { 344349cc55cSDimitry Andric return std::error_code(); 345349cc55cSDimitry Andric } 346349cc55cSDimitry Andric Data += NumBytesRead; 347349cc55cSDimitry Andric return ErrorOr<T>(static_cast<T>(Val)); 348349cc55cSDimitry Andric } 349349cc55cSDimitry Andric 350349cc55cSDimitry Andric template <typename T> ErrorOr<T> MCPseudoProbeDecoder::readSignedNumber() { 351349cc55cSDimitry Andric unsigned NumBytesRead = 0; 352349cc55cSDimitry Andric int64_t Val = decodeSLEB128(Data, &NumBytesRead); 353349cc55cSDimitry Andric if (Val > std::numeric_limits<T>::max() || (Data + NumBytesRead > End)) { 354349cc55cSDimitry Andric return std::error_code(); 355349cc55cSDimitry Andric } 356349cc55cSDimitry Andric Data += NumBytesRead; 357349cc55cSDimitry Andric return ErrorOr<T>(static_cast<T>(Val)); 358349cc55cSDimitry Andric } 359349cc55cSDimitry Andric 360349cc55cSDimitry Andric ErrorOr<StringRef> MCPseudoProbeDecoder::readString(uint32_t Size) { 361349cc55cSDimitry Andric StringRef Str(reinterpret_cast<const char *>(Data), Size); 362349cc55cSDimitry Andric if (Data + Size > End) { 363349cc55cSDimitry Andric return std::error_code(); 364349cc55cSDimitry Andric } 365349cc55cSDimitry Andric Data += Size; 366349cc55cSDimitry Andric return ErrorOr<StringRef>(Str); 367349cc55cSDimitry Andric } 368349cc55cSDimitry Andric 369349cc55cSDimitry Andric bool MCPseudoProbeDecoder::buildGUID2FuncDescMap(const uint8_t *Start, 370349cc55cSDimitry Andric std::size_t Size) { 371349cc55cSDimitry Andric // The pseudo_probe_desc section has a format like: 372349cc55cSDimitry Andric // .section .pseudo_probe_desc,"",@progbits 373349cc55cSDimitry Andric // .quad -5182264717993193164 // GUID 374349cc55cSDimitry Andric // .quad 4294967295 // Hash 375349cc55cSDimitry Andric // .uleb 3 // Name size 376349cc55cSDimitry Andric // .ascii "foo" // Name 377349cc55cSDimitry Andric // .quad -2624081020897602054 378349cc55cSDimitry Andric // .quad 174696971957 379349cc55cSDimitry Andric // .uleb 34 380349cc55cSDimitry Andric // .ascii "main" 381349cc55cSDimitry Andric 382349cc55cSDimitry Andric Data = Start; 383349cc55cSDimitry Andric End = Data + Size; 384349cc55cSDimitry Andric 385349cc55cSDimitry Andric while (Data < End) { 386349cc55cSDimitry Andric auto ErrorOrGUID = readUnencodedNumber<uint64_t>(); 387349cc55cSDimitry Andric if (!ErrorOrGUID) 388349cc55cSDimitry Andric return false; 389349cc55cSDimitry Andric 390349cc55cSDimitry Andric auto ErrorOrHash = readUnencodedNumber<uint64_t>(); 391349cc55cSDimitry Andric if (!ErrorOrHash) 392349cc55cSDimitry Andric return false; 393349cc55cSDimitry Andric 394349cc55cSDimitry Andric auto ErrorOrNameSize = readUnsignedNumber<uint32_t>(); 395349cc55cSDimitry Andric if (!ErrorOrNameSize) 396349cc55cSDimitry Andric return false; 397349cc55cSDimitry Andric uint32_t NameSize = std::move(*ErrorOrNameSize); 398349cc55cSDimitry Andric 399349cc55cSDimitry Andric auto ErrorOrName = readString(NameSize); 400349cc55cSDimitry Andric if (!ErrorOrName) 401349cc55cSDimitry Andric return false; 402349cc55cSDimitry Andric 403349cc55cSDimitry Andric uint64_t GUID = std::move(*ErrorOrGUID); 404349cc55cSDimitry Andric uint64_t Hash = std::move(*ErrorOrHash); 405349cc55cSDimitry Andric StringRef Name = std::move(*ErrorOrName); 406349cc55cSDimitry Andric 407349cc55cSDimitry Andric // Initialize PseudoProbeFuncDesc and populate it into GUID2FuncDescMap 408349cc55cSDimitry Andric GUID2FuncDescMap.emplace(GUID, MCPseudoProbeFuncDesc(GUID, Hash, Name)); 409349cc55cSDimitry Andric } 410349cc55cSDimitry Andric assert(Data == End && "Have unprocessed data in pseudo_probe_desc section"); 411349cc55cSDimitry Andric return true; 412349cc55cSDimitry Andric } 413349cc55cSDimitry Andric 41481ad6265SDimitry Andric bool MCPseudoProbeDecoder::buildAddress2ProbeMap( 41581ad6265SDimitry Andric MCDecodedPseudoProbeInlineTree *Cur, uint64_t &LastAddr, 416bdd1243dSDimitry Andric const Uint64Set &GuidFilter, const Uint64Map &FuncStartAddrs) { 417349cc55cSDimitry Andric // The pseudo_probe section encodes an inline forest and each tree has a 418bdd1243dSDimitry Andric // format defined in MCPseudoProbe.h 419349cc55cSDimitry Andric 420349cc55cSDimitry Andric uint32_t Index = 0; 421bdd1243dSDimitry Andric bool IsTopLevelFunc = Cur == &DummyInlineRoot; 422bdd1243dSDimitry Andric if (IsTopLevelFunc) { 423349cc55cSDimitry Andric // Use a sequential id for top level inliner. 42481ad6265SDimitry Andric Index = Cur->getChildren().size(); 425349cc55cSDimitry Andric } else { 426349cc55cSDimitry Andric // Read inline site for inlinees 427349cc55cSDimitry Andric auto ErrorOrIndex = readUnsignedNumber<uint32_t>(); 428349cc55cSDimitry Andric if (!ErrorOrIndex) 429349cc55cSDimitry Andric return false; 430349cc55cSDimitry Andric Index = std::move(*ErrorOrIndex); 431349cc55cSDimitry Andric } 43281ad6265SDimitry Andric 433349cc55cSDimitry Andric // Read guid 434349cc55cSDimitry Andric auto ErrorOrCurGuid = readUnencodedNumber<uint64_t>(); 435349cc55cSDimitry Andric if (!ErrorOrCurGuid) 436349cc55cSDimitry Andric return false; 43781ad6265SDimitry Andric uint64_t Guid = std::move(*ErrorOrCurGuid); 43881ad6265SDimitry Andric 43981ad6265SDimitry Andric // Decide if top-level node should be disgarded. 440bdd1243dSDimitry Andric if (IsTopLevelFunc && !GuidFilter.empty() && !GuidFilter.count(Guid)) 44181ad6265SDimitry Andric Cur = nullptr; 44281ad6265SDimitry Andric 44381ad6265SDimitry Andric // If the incoming node is null, all its children nodes should be disgarded. 44481ad6265SDimitry Andric if (Cur) { 44581ad6265SDimitry Andric // Switch/add to a new tree node(inlinee) 44681ad6265SDimitry Andric Cur = Cur->getOrAddNode(std::make_tuple(Guid, Index)); 44781ad6265SDimitry Andric Cur->Guid = Guid; 448bdd1243dSDimitry Andric if (IsTopLevelFunc && !EncodingIsAddrBased) { 449bdd1243dSDimitry Andric if (auto V = FuncStartAddrs.lookup(Guid)) 450bdd1243dSDimitry Andric LastAddr = V; 451bdd1243dSDimitry Andric } 45281ad6265SDimitry Andric } 45381ad6265SDimitry Andric 454349cc55cSDimitry Andric // Read number of probes in the current node. 455349cc55cSDimitry Andric auto ErrorOrNodeCount = readUnsignedNumber<uint32_t>(); 456349cc55cSDimitry Andric if (!ErrorOrNodeCount) 457349cc55cSDimitry Andric return false; 458349cc55cSDimitry Andric uint32_t NodeCount = std::move(*ErrorOrNodeCount); 459349cc55cSDimitry Andric // Read number of direct inlinees 460349cc55cSDimitry Andric auto ErrorOrCurChildrenToProcess = readUnsignedNumber<uint32_t>(); 461349cc55cSDimitry Andric if (!ErrorOrCurChildrenToProcess) 462349cc55cSDimitry Andric return false; 463349cc55cSDimitry Andric // Read all probes in this node 464349cc55cSDimitry Andric for (std::size_t I = 0; I < NodeCount; I++) { 465349cc55cSDimitry Andric // Read index 466349cc55cSDimitry Andric auto ErrorOrIndex = readUnsignedNumber<uint32_t>(); 467349cc55cSDimitry Andric if (!ErrorOrIndex) 468349cc55cSDimitry Andric return false; 469349cc55cSDimitry Andric uint32_t Index = std::move(*ErrorOrIndex); 470349cc55cSDimitry Andric // Read type | flag. 471349cc55cSDimitry Andric auto ErrorOrValue = readUnencodedNumber<uint8_t>(); 472349cc55cSDimitry Andric if (!ErrorOrValue) 473349cc55cSDimitry Andric return false; 474349cc55cSDimitry Andric uint8_t Value = std::move(*ErrorOrValue); 475349cc55cSDimitry Andric uint8_t Kind = Value & 0xf; 476349cc55cSDimitry Andric uint8_t Attr = (Value & 0x70) >> 4; 477349cc55cSDimitry Andric // Read address 478349cc55cSDimitry Andric uint64_t Addr = 0; 479349cc55cSDimitry Andric if (Value & 0x80) { 480349cc55cSDimitry Andric auto ErrorOrOffset = readSignedNumber<int64_t>(); 481349cc55cSDimitry Andric if (!ErrorOrOffset) 482349cc55cSDimitry Andric return false; 483349cc55cSDimitry Andric int64_t Offset = std::move(*ErrorOrOffset); 484349cc55cSDimitry Andric Addr = LastAddr + Offset; 485349cc55cSDimitry Andric } else { 486349cc55cSDimitry Andric auto ErrorOrAddr = readUnencodedNumber<int64_t>(); 487349cc55cSDimitry Andric if (!ErrorOrAddr) 488349cc55cSDimitry Andric return false; 489349cc55cSDimitry Andric Addr = std::move(*ErrorOrAddr); 490bdd1243dSDimitry Andric if (isSentinelProbe(Attr)) { 491bdd1243dSDimitry Andric // For sentinel probe, the addr field actually stores the GUID of the 492bdd1243dSDimitry Andric // split function. Convert it to the real address. 493bdd1243dSDimitry Andric if (auto V = FuncStartAddrs.lookup(Addr)) 494bdd1243dSDimitry Andric Addr = V; 495bdd1243dSDimitry Andric } else { 496bdd1243dSDimitry Andric // For now we assume all probe encoding should be either based on 497bdd1243dSDimitry Andric // leading probe address or function start address. 498bdd1243dSDimitry Andric // The scheme is for downwards compatibility. 499bdd1243dSDimitry Andric // TODO: retire this scheme once compatibility is no longer an issue. 500bdd1243dSDimitry Andric EncodingIsAddrBased = true; 501bdd1243dSDimitry Andric } 502349cc55cSDimitry Andric } 50381ad6265SDimitry Andric 504*06c3fb27SDimitry Andric uint32_t Discriminator = 0; 505*06c3fb27SDimitry Andric if (hasDiscriminator(Attr)) { 506*06c3fb27SDimitry Andric auto ErrorOrDiscriminator = readUnsignedNumber<uint32_t>(); 507*06c3fb27SDimitry Andric if (!ErrorOrDiscriminator) 508*06c3fb27SDimitry Andric return false; 509*06c3fb27SDimitry Andric Discriminator = std::move(*ErrorOrDiscriminator); 510*06c3fb27SDimitry Andric } 511*06c3fb27SDimitry Andric 512bdd1243dSDimitry Andric if (Cur && !isSentinelProbe(Attr)) { 513349cc55cSDimitry Andric // Populate Address2ProbesMap 514349cc55cSDimitry Andric auto &Probes = Address2ProbesMap[Addr]; 515349cc55cSDimitry Andric Probes.emplace_back(Addr, Cur->Guid, Index, PseudoProbeType(Kind), Attr, 516*06c3fb27SDimitry Andric Discriminator, Cur); 517349cc55cSDimitry Andric Cur->addProbes(&Probes.back()); 51881ad6265SDimitry Andric } 519349cc55cSDimitry Andric LastAddr = Addr; 520349cc55cSDimitry Andric } 521349cc55cSDimitry Andric 52281ad6265SDimitry Andric uint32_t ChildrenToProcess = std::move(*ErrorOrCurChildrenToProcess); 52381ad6265SDimitry Andric for (uint32_t I = 0; I < ChildrenToProcess; I++) { 524bdd1243dSDimitry Andric buildAddress2ProbeMap(Cur, LastAddr, GuidFilter, FuncStartAddrs); 525349cc55cSDimitry Andric } 526349cc55cSDimitry Andric 527349cc55cSDimitry Andric return true; 528349cc55cSDimitry Andric } 529349cc55cSDimitry Andric 53081ad6265SDimitry Andric bool MCPseudoProbeDecoder::buildAddress2ProbeMap( 531bdd1243dSDimitry Andric const uint8_t *Start, std::size_t Size, const Uint64Set &GuidFilter, 532bdd1243dSDimitry Andric const Uint64Map &FuncStartAddrs) { 53381ad6265SDimitry Andric Data = Start; 53481ad6265SDimitry Andric End = Data + Size; 53581ad6265SDimitry Andric uint64_t LastAddr = 0; 53681ad6265SDimitry Andric while (Data < End) 537bdd1243dSDimitry Andric buildAddress2ProbeMap(&DummyInlineRoot, LastAddr, GuidFilter, 538bdd1243dSDimitry Andric FuncStartAddrs); 53981ad6265SDimitry Andric assert(Data == End && "Have unprocessed data in pseudo_probe section"); 54081ad6265SDimitry Andric return true; 54181ad6265SDimitry Andric } 54281ad6265SDimitry Andric 543349cc55cSDimitry Andric void MCPseudoProbeDecoder::printGUID2FuncDescMap(raw_ostream &OS) { 544349cc55cSDimitry Andric OS << "Pseudo Probe Desc:\n"; 545349cc55cSDimitry Andric // Make the output deterministic 546349cc55cSDimitry Andric std::map<uint64_t, MCPseudoProbeFuncDesc> OrderedMap(GUID2FuncDescMap.begin(), 547349cc55cSDimitry Andric GUID2FuncDescMap.end()); 548349cc55cSDimitry Andric for (auto &I : OrderedMap) { 549349cc55cSDimitry Andric I.second.print(OS); 550349cc55cSDimitry Andric } 551349cc55cSDimitry Andric } 552349cc55cSDimitry Andric 553349cc55cSDimitry Andric void MCPseudoProbeDecoder::printProbeForAddress(raw_ostream &OS, 554349cc55cSDimitry Andric uint64_t Address) { 555349cc55cSDimitry Andric auto It = Address2ProbesMap.find(Address); 556349cc55cSDimitry Andric if (It != Address2ProbesMap.end()) { 557349cc55cSDimitry Andric for (auto &Probe : It->second) { 558349cc55cSDimitry Andric OS << " [Probe]:\t"; 559349cc55cSDimitry Andric Probe.print(OS, GUID2FuncDescMap, true); 560349cc55cSDimitry Andric } 561349cc55cSDimitry Andric } 562349cc55cSDimitry Andric } 563349cc55cSDimitry Andric 564349cc55cSDimitry Andric void MCPseudoProbeDecoder::printProbesForAllAddresses(raw_ostream &OS) { 565349cc55cSDimitry Andric std::vector<uint64_t> Addresses; 566349cc55cSDimitry Andric for (auto Entry : Address2ProbesMap) 567349cc55cSDimitry Andric Addresses.push_back(Entry.first); 568fcaf7f86SDimitry Andric llvm::sort(Addresses); 569349cc55cSDimitry Andric for (auto K : Addresses) { 570349cc55cSDimitry Andric OS << "Address:\t"; 571349cc55cSDimitry Andric OS << K; 572349cc55cSDimitry Andric OS << "\n"; 573349cc55cSDimitry Andric printProbeForAddress(OS, K); 574349cc55cSDimitry Andric } 575349cc55cSDimitry Andric } 576349cc55cSDimitry Andric 577349cc55cSDimitry Andric const MCDecodedPseudoProbe * 578349cc55cSDimitry Andric MCPseudoProbeDecoder::getCallProbeForAddr(uint64_t Address) const { 579349cc55cSDimitry Andric auto It = Address2ProbesMap.find(Address); 580349cc55cSDimitry Andric if (It == Address2ProbesMap.end()) 581349cc55cSDimitry Andric return nullptr; 582349cc55cSDimitry Andric const auto &Probes = It->second; 583349cc55cSDimitry Andric 584349cc55cSDimitry Andric const MCDecodedPseudoProbe *CallProbe = nullptr; 585349cc55cSDimitry Andric for (const auto &Probe : Probes) { 586349cc55cSDimitry Andric if (Probe.isCall()) { 587*06c3fb27SDimitry Andric // Disabling the assert and returning first call probe seen so far. 588*06c3fb27SDimitry Andric // Subsequent call probes, if any, are ignored. Due to the the way 589*06c3fb27SDimitry Andric // .pseudo_probe section is decoded, probes of the same-named independent 590*06c3fb27SDimitry Andric // static functions are merged thus multiple call probes may be seen for a 591*06c3fb27SDimitry Andric // callsite. This should only happen to compiler-generated statics, with 592*06c3fb27SDimitry Andric // -funique-internal-linkage-names where user statics get unique names. 593*06c3fb27SDimitry Andric // 594*06c3fb27SDimitry Andric // TODO: re-enable or narrow down the assert to static functions only. 595*06c3fb27SDimitry Andric // 596*06c3fb27SDimitry Andric // assert(!CallProbe && 597*06c3fb27SDimitry Andric // "There should be only one call probe corresponding to address " 598*06c3fb27SDimitry Andric // "which is a callsite."); 599349cc55cSDimitry Andric CallProbe = &Probe; 600*06c3fb27SDimitry Andric break; 601349cc55cSDimitry Andric } 602349cc55cSDimitry Andric } 603349cc55cSDimitry Andric return CallProbe; 604349cc55cSDimitry Andric } 605349cc55cSDimitry Andric 606349cc55cSDimitry Andric const MCPseudoProbeFuncDesc * 607349cc55cSDimitry Andric MCPseudoProbeDecoder::getFuncDescForGUID(uint64_t GUID) const { 608349cc55cSDimitry Andric auto It = GUID2FuncDescMap.find(GUID); 609349cc55cSDimitry Andric assert(It != GUID2FuncDescMap.end() && "Function descriptor doesn't exist"); 610349cc55cSDimitry Andric return &It->second; 611349cc55cSDimitry Andric } 612349cc55cSDimitry Andric 613349cc55cSDimitry Andric void MCPseudoProbeDecoder::getInlineContextForProbe( 614349cc55cSDimitry Andric const MCDecodedPseudoProbe *Probe, 615349cc55cSDimitry Andric SmallVectorImpl<MCPseduoProbeFrameLocation> &InlineContextStack, 616349cc55cSDimitry Andric bool IncludeLeaf) const { 617349cc55cSDimitry Andric Probe->getInlineContext(InlineContextStack, GUID2FuncDescMap); 618349cc55cSDimitry Andric if (!IncludeLeaf) 619349cc55cSDimitry Andric return; 620349cc55cSDimitry Andric // Note that the context from probe doesn't include leaf frame, 621349cc55cSDimitry Andric // hence we need to retrieve and prepend leaf if requested. 622349cc55cSDimitry Andric const auto *FuncDesc = getFuncDescForGUID(Probe->getGuid()); 623349cc55cSDimitry Andric InlineContextStack.emplace_back( 624349cc55cSDimitry Andric MCPseduoProbeFrameLocation(FuncDesc->FuncName, Probe->getIndex())); 625349cc55cSDimitry Andric } 626349cc55cSDimitry Andric 627349cc55cSDimitry Andric const MCPseudoProbeFuncDesc *MCPseudoProbeDecoder::getInlinerDescForProbe( 628349cc55cSDimitry Andric const MCDecodedPseudoProbe *Probe) const { 629349cc55cSDimitry Andric MCDecodedPseudoProbeInlineTree *InlinerNode = Probe->getInlineTreeNode(); 630349cc55cSDimitry Andric if (!InlinerNode->hasInlineSite()) 631349cc55cSDimitry Andric return nullptr; 63281ad6265SDimitry Andric return getFuncDescForGUID(InlinerNode->Parent->Guid); 633349cc55cSDimitry Andric } 634