1e8d8bef9SDimitry Andric //===- lib/MC/MCPseudoProbe.cpp - Pseudo probe encoding support ----------===// 2e8d8bef9SDimitry Andric // 3e8d8bef9SDimitry Andric // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4e8d8bef9SDimitry Andric // See https://llvm.org/LICENSE.txt for license information. 5e8d8bef9SDimitry Andric // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6e8d8bef9SDimitry Andric // 7e8d8bef9SDimitry Andric //===----------------------------------------------------------------------===// 8e8d8bef9SDimitry Andric 9e8d8bef9SDimitry Andric #include "llvm/MC/MCPseudoProbe.h" 10fcaf7f86SDimitry Andric #include "llvm/ADT/STLExtras.h" 1106c3fb27SDimitry Andric #include "llvm/IR/PseudoProbe.h" 12e8d8bef9SDimitry Andric #include "llvm/MC/MCAsmInfo.h" 135f757f3fSDimitry Andric #include "llvm/MC/MCAssembler.h" 14e8d8bef9SDimitry Andric #include "llvm/MC/MCContext.h" 1581ad6265SDimitry Andric #include "llvm/MC/MCExpr.h" 1681ad6265SDimitry Andric #include "llvm/MC/MCFragment.h" 17e8d8bef9SDimitry Andric #include "llvm/MC/MCObjectFileInfo.h" 18e8d8bef9SDimitry Andric #include "llvm/MC/MCObjectStreamer.h" 19bdd1243dSDimitry Andric #include "llvm/MC/MCSymbol.h" 20349cc55cSDimitry Andric #include "llvm/Support/Endian.h" 21349cc55cSDimitry Andric #include "llvm/Support/LEB128.h" 22bdd1243dSDimitry Andric #include "llvm/Support/MD5.h" 23349cc55cSDimitry Andric #include "llvm/Support/raw_ostream.h" 24bdd1243dSDimitry Andric #include <algorithm> 25bdd1243dSDimitry Andric #include <cassert> 26349cc55cSDimitry Andric #include <limits> 27349cc55cSDimitry Andric #include <memory> 28349cc55cSDimitry Andric #include <sstream> 29bdd1243dSDimitry Andric #include <vector> 30e8d8bef9SDimitry Andric 31e8d8bef9SDimitry Andric #define DEBUG_TYPE "mcpseudoprobe" 32e8d8bef9SDimitry Andric 33e8d8bef9SDimitry Andric using namespace llvm; 34349cc55cSDimitry Andric using namespace support; 35e8d8bef9SDimitry Andric 36e8d8bef9SDimitry Andric #ifndef NDEBUG 37e8d8bef9SDimitry Andric int MCPseudoProbeTable::DdgPrintIndent = 0; 38e8d8bef9SDimitry Andric #endif 39e8d8bef9SDimitry Andric 40e8d8bef9SDimitry Andric static const MCExpr *buildSymbolDiff(MCObjectStreamer *MCOS, const MCSymbol *A, 41e8d8bef9SDimitry Andric const MCSymbol *B) { 42e8d8bef9SDimitry Andric MCContext &Context = MCOS->getContext(); 43e8d8bef9SDimitry Andric MCSymbolRefExpr::VariantKind Variant = MCSymbolRefExpr::VK_None; 44e8d8bef9SDimitry Andric const MCExpr *ARef = MCSymbolRefExpr::create(A, Variant, Context); 45e8d8bef9SDimitry Andric const MCExpr *BRef = MCSymbolRefExpr::create(B, Variant, Context); 46e8d8bef9SDimitry Andric const MCExpr *AddrDelta = 47e8d8bef9SDimitry Andric MCBinaryExpr::create(MCBinaryExpr::Sub, ARef, BRef, Context); 48e8d8bef9SDimitry Andric return AddrDelta; 49e8d8bef9SDimitry Andric } 50e8d8bef9SDimitry Andric 51e8d8bef9SDimitry Andric void MCPseudoProbe::emit(MCObjectStreamer *MCOS, 52e8d8bef9SDimitry Andric const MCPseudoProbe *LastProbe) const { 53bdd1243dSDimitry Andric bool IsSentinel = isSentinelProbe(getAttributes()); 54bdd1243dSDimitry Andric assert((LastProbe || IsSentinel) && 55bdd1243dSDimitry Andric "Last probe should not be null for non-sentinel probes"); 56bdd1243dSDimitry Andric 57e8d8bef9SDimitry Andric // Emit Index 58e8d8bef9SDimitry Andric MCOS->emitULEB128IntValue(Index); 59e8d8bef9SDimitry Andric // Emit Type and the flag: 60e8d8bef9SDimitry Andric // Type (bit 0 to 3), with bit 4 to 6 for attributes. 61e8d8bef9SDimitry Andric // Flag (bit 7, 0 - code address, 1 - address delta). This indicates whether 62e8d8bef9SDimitry Andric // the following field is a symbolic code address or an address delta. 6306c3fb27SDimitry Andric // Emit FS discriminator 64e8d8bef9SDimitry Andric assert(Type <= 0xF && "Probe type too big to encode, exceeding 15"); 6506c3fb27SDimitry Andric auto NewAttributes = Attributes; 6606c3fb27SDimitry Andric if (Discriminator) 6706c3fb27SDimitry Andric NewAttributes |= (uint32_t)PseudoProbeAttributes::HasDiscriminator; 6806c3fb27SDimitry Andric assert(NewAttributes <= 0x7 && 69e8d8bef9SDimitry Andric "Probe attributes too big to encode, exceeding 7"); 7006c3fb27SDimitry Andric uint8_t PackedType = Type | (NewAttributes << 4); 71bdd1243dSDimitry Andric uint8_t Flag = 72bdd1243dSDimitry Andric !IsSentinel ? ((int8_t)MCPseudoProbeFlag::AddressDelta << 7) : 0; 73e8d8bef9SDimitry Andric MCOS->emitInt8(Flag | PackedType); 74e8d8bef9SDimitry Andric 75bdd1243dSDimitry Andric if (!IsSentinel) { 76e8d8bef9SDimitry Andric // Emit the delta between the address label and LastProbe. 77e8d8bef9SDimitry Andric const MCExpr *AddrDelta = 78e8d8bef9SDimitry Andric buildSymbolDiff(MCOS, Label, LastProbe->getLabel()); 79e8d8bef9SDimitry Andric int64_t Delta; 80e8d8bef9SDimitry Andric if (AddrDelta->evaluateAsAbsolute(Delta, MCOS->getAssemblerPtr())) { 81e8d8bef9SDimitry Andric MCOS->emitSLEB128IntValue(Delta); 82e8d8bef9SDimitry Andric } else { 83*0fca6ea1SDimitry Andric MCOS->insert(MCOS->getContext().allocFragment<MCPseudoProbeAddrFragment>( 84*0fca6ea1SDimitry Andric AddrDelta)); 85e8d8bef9SDimitry Andric } 86e8d8bef9SDimitry Andric } else { 87bdd1243dSDimitry Andric // Emit the GUID of the split function that the sentinel probe represents. 88bdd1243dSDimitry Andric MCOS->emitInt64(Guid); 89e8d8bef9SDimitry Andric } 90e8d8bef9SDimitry Andric 9106c3fb27SDimitry Andric if (Discriminator) 9206c3fb27SDimitry Andric MCOS->emitULEB128IntValue(Discriminator); 9306c3fb27SDimitry Andric 94e8d8bef9SDimitry Andric LLVM_DEBUG({ 95e8d8bef9SDimitry Andric dbgs().indent(MCPseudoProbeTable::DdgPrintIndent); 96e8d8bef9SDimitry Andric dbgs() << "Probe: " << Index << "\n"; 97e8d8bef9SDimitry Andric }); 98e8d8bef9SDimitry Andric } 99e8d8bef9SDimitry Andric 100e8d8bef9SDimitry Andric void MCPseudoProbeInlineTree::addPseudoProbe( 101e8d8bef9SDimitry Andric const MCPseudoProbe &Probe, const MCPseudoProbeInlineStack &InlineStack) { 102e8d8bef9SDimitry Andric // The function should not be called on the root. 103bdd1243dSDimitry Andric assert(isRoot() && "Should only be called on root"); 104e8d8bef9SDimitry Andric 105e8d8bef9SDimitry Andric // When it comes here, the input look like: 106e8d8bef9SDimitry Andric // Probe: GUID of C, ... 107e8d8bef9SDimitry Andric // InlineStack: [88, A], [66, B] 108e8d8bef9SDimitry Andric // which means, Function A inlines function B at call site with a probe id of 109e8d8bef9SDimitry Andric // 88, and B inlines C at probe 66. The tri-tree expects a tree path like {[0, 110e8d8bef9SDimitry Andric // A], [88, B], [66, C]} to locate the tree node where the probe should be 111e8d8bef9SDimitry Andric // added. Note that the edge [0, A] means A is the top-level function we are 112e8d8bef9SDimitry Andric // emitting probes for. 113e8d8bef9SDimitry Andric 114e8d8bef9SDimitry Andric // Make a [0, A] edge. 115e8d8bef9SDimitry Andric // An empty inline stack means the function that the probe originates from 116e8d8bef9SDimitry Andric // is a top-level function. 117e8d8bef9SDimitry Andric InlineSite Top; 118e8d8bef9SDimitry Andric if (InlineStack.empty()) { 119e8d8bef9SDimitry Andric Top = InlineSite(Probe.getGuid(), 0); 120e8d8bef9SDimitry Andric } else { 121e8d8bef9SDimitry Andric Top = InlineSite(std::get<0>(InlineStack.front()), 0); 122e8d8bef9SDimitry Andric } 123e8d8bef9SDimitry Andric 124e8d8bef9SDimitry Andric auto *Cur = getOrAddNode(Top); 125e8d8bef9SDimitry Andric 126e8d8bef9SDimitry Andric // Make interior edges by walking the inline stack. Once it's done, Cur should 127e8d8bef9SDimitry Andric // point to the node that the probe originates from. 128e8d8bef9SDimitry Andric if (!InlineStack.empty()) { 129e8d8bef9SDimitry Andric auto Iter = InlineStack.begin(); 130e8d8bef9SDimitry Andric auto Index = std::get<1>(*Iter); 131e8d8bef9SDimitry Andric Iter++; 132e8d8bef9SDimitry Andric for (; Iter != InlineStack.end(); Iter++) { 133e8d8bef9SDimitry Andric // Make an edge by using the previous probe id and current GUID. 134e8d8bef9SDimitry Andric Cur = Cur->getOrAddNode(InlineSite(std::get<0>(*Iter), Index)); 135e8d8bef9SDimitry Andric Index = std::get<1>(*Iter); 136e8d8bef9SDimitry Andric } 137e8d8bef9SDimitry Andric Cur = Cur->getOrAddNode(InlineSite(Probe.getGuid(), Index)); 138e8d8bef9SDimitry Andric } 139e8d8bef9SDimitry Andric 140e8d8bef9SDimitry Andric Cur->Probes.push_back(Probe); 141e8d8bef9SDimitry Andric } 142e8d8bef9SDimitry Andric 143e8d8bef9SDimitry Andric void MCPseudoProbeInlineTree::emit(MCObjectStreamer *MCOS, 144e8d8bef9SDimitry Andric const MCPseudoProbe *&LastProbe) { 145e8d8bef9SDimitry Andric LLVM_DEBUG({ 146e8d8bef9SDimitry Andric dbgs().indent(MCPseudoProbeTable::DdgPrintIndent); 147e8d8bef9SDimitry Andric dbgs() << "Group [\n"; 148e8d8bef9SDimitry Andric MCPseudoProbeTable::DdgPrintIndent += 2; 149e8d8bef9SDimitry Andric }); 150*0fca6ea1SDimitry Andric assert(!isRoot() && "Root should be handled separately"); 151bdd1243dSDimitry Andric 152e8d8bef9SDimitry Andric // Emit probes grouped by GUID. 153e8d8bef9SDimitry Andric LLVM_DEBUG({ 154e8d8bef9SDimitry Andric dbgs().indent(MCPseudoProbeTable::DdgPrintIndent); 155e8d8bef9SDimitry Andric dbgs() << "GUID: " << Guid << "\n"; 156e8d8bef9SDimitry Andric }); 157e8d8bef9SDimitry Andric // Emit Guid 158e8d8bef9SDimitry Andric MCOS->emitInt64(Guid); 159bdd1243dSDimitry Andric // Emit number of probes in this node, including a sentinel probe for 160bdd1243dSDimitry Andric // top-level functions if needed. 161bdd1243dSDimitry Andric bool NeedSentinel = false; 162bdd1243dSDimitry Andric if (Parent->isRoot()) { 163bdd1243dSDimitry Andric assert(isSentinelProbe(LastProbe->getAttributes()) && 164bdd1243dSDimitry Andric "Starting probe of a top-level function should be a sentinel probe"); 165bdd1243dSDimitry Andric // The main body of a split function doesn't need a sentinel probe. 166bdd1243dSDimitry Andric if (LastProbe->getGuid() != Guid) 167bdd1243dSDimitry Andric NeedSentinel = true; 168bdd1243dSDimitry Andric } 169bdd1243dSDimitry Andric 170bdd1243dSDimitry Andric MCOS->emitULEB128IntValue(Probes.size() + NeedSentinel); 171e8d8bef9SDimitry Andric // Emit number of direct inlinees 172349cc55cSDimitry Andric MCOS->emitULEB128IntValue(Children.size()); 173bdd1243dSDimitry Andric // Emit sentinel probe for top-level functions 174bdd1243dSDimitry Andric if (NeedSentinel) 175bdd1243dSDimitry Andric LastProbe->emit(MCOS, nullptr); 176bdd1243dSDimitry Andric 177e8d8bef9SDimitry Andric // Emit probes in this group 178e8d8bef9SDimitry Andric for (const auto &Probe : Probes) { 179e8d8bef9SDimitry Andric Probe.emit(MCOS, LastProbe); 180e8d8bef9SDimitry Andric LastProbe = &Probe; 181e8d8bef9SDimitry Andric } 182e8d8bef9SDimitry Andric 183bdd1243dSDimitry Andric // Emit sorted descendant. InlineSite is unique for each pair, so there will 184bdd1243dSDimitry Andric // be no ordering of Inlinee based on MCPseudoProbeInlineTree* 185bdd1243dSDimitry Andric using InlineeType = std::pair<InlineSite, MCPseudoProbeInlineTree *>; 186bdd1243dSDimitry Andric std::vector<InlineeType> Inlinees; 187bdd1243dSDimitry Andric for (const auto &Child : Children) 188bdd1243dSDimitry Andric Inlinees.emplace_back(Child.first, Child.second.get()); 189*0fca6ea1SDimitry Andric llvm::sort(Inlinees, llvm::less_first()); 190349cc55cSDimitry Andric 191e8d8bef9SDimitry Andric for (const auto &Inlinee : Inlinees) { 192e8d8bef9SDimitry Andric // Emit probe index 193e8d8bef9SDimitry Andric MCOS->emitULEB128IntValue(std::get<1>(Inlinee.first)); 194e8d8bef9SDimitry Andric LLVM_DEBUG({ 195e8d8bef9SDimitry Andric dbgs().indent(MCPseudoProbeTable::DdgPrintIndent); 196e8d8bef9SDimitry Andric dbgs() << "InlineSite: " << std::get<1>(Inlinee.first) << "\n"; 197e8d8bef9SDimitry Andric }); 198e8d8bef9SDimitry Andric // Emit the group 199e8d8bef9SDimitry Andric Inlinee.second->emit(MCOS, LastProbe); 200e8d8bef9SDimitry Andric } 201e8d8bef9SDimitry Andric 202e8d8bef9SDimitry Andric LLVM_DEBUG({ 203e8d8bef9SDimitry Andric MCPseudoProbeTable::DdgPrintIndent -= 2; 204e8d8bef9SDimitry Andric dbgs().indent(MCPseudoProbeTable::DdgPrintIndent); 205e8d8bef9SDimitry Andric dbgs() << "]\n"; 206e8d8bef9SDimitry Andric }); 207e8d8bef9SDimitry Andric } 208e8d8bef9SDimitry Andric 209bdd1243dSDimitry Andric void MCPseudoProbeSections::emit(MCObjectStreamer *MCOS) { 210e8d8bef9SDimitry Andric MCContext &Ctx = MCOS->getContext(); 2115f757f3fSDimitry Andric SmallVector<std::pair<MCSymbol *, MCPseudoProbeInlineTree *>> Vec; 2125f757f3fSDimitry Andric Vec.reserve(MCProbeDivisions.size()); 2135f757f3fSDimitry Andric for (auto &ProbeSec : MCProbeDivisions) 2145f757f3fSDimitry Andric Vec.emplace_back(ProbeSec.first, &ProbeSec.second); 2155f757f3fSDimitry Andric for (auto I : llvm::enumerate(MCOS->getAssembler())) 2165f757f3fSDimitry Andric I.value().setOrdinal(I.index()); 2175f757f3fSDimitry Andric llvm::sort(Vec, [](auto A, auto B) { 2185f757f3fSDimitry Andric return A.first->getSection().getOrdinal() < 2195f757f3fSDimitry Andric B.first->getSection().getOrdinal(); 2205f757f3fSDimitry Andric }); 2215f757f3fSDimitry Andric for (auto [FuncSym, RootPtr] : Vec) { 2225f757f3fSDimitry Andric const auto &Root = *RootPtr; 223bdd1243dSDimitry Andric if (auto *S = Ctx.getObjectFileInfo()->getPseudoProbeSection( 224bdd1243dSDimitry Andric FuncSym->getSection())) { 225e8d8bef9SDimitry Andric // Switch to the .pseudoprobe section or a comdat group. 22681ad6265SDimitry Andric MCOS->switchSection(S); 227e8d8bef9SDimitry Andric // Emit probes grouped by GUID. 228bdd1243dSDimitry Andric // Emit sorted descendant. InlineSite is unique for each pair, so there 229bdd1243dSDimitry Andric // will be no ordering of Inlinee based on MCPseudoProbeInlineTree* 230bdd1243dSDimitry Andric using InlineeType = std::pair<InlineSite, MCPseudoProbeInlineTree *>; 231bdd1243dSDimitry Andric std::vector<InlineeType> Inlinees; 232bdd1243dSDimitry Andric for (const auto &Child : Root.getChildren()) 233bdd1243dSDimitry Andric Inlinees.emplace_back(Child.first, Child.second.get()); 234*0fca6ea1SDimitry Andric llvm::sort(Inlinees, llvm::less_first()); 235bdd1243dSDimitry Andric 236bdd1243dSDimitry Andric for (const auto &Inlinee : Inlinees) { 237bdd1243dSDimitry Andric // Emit the group guarded by a sentinel probe. 23806c3fb27SDimitry Andric MCPseudoProbe SentinelProbe( 23906c3fb27SDimitry Andric const_cast<MCSymbol *>(FuncSym), MD5Hash(FuncSym->getName()), 240bdd1243dSDimitry Andric (uint32_t)PseudoProbeReservedId::Invalid, 241bdd1243dSDimitry Andric (uint32_t)PseudoProbeType::Block, 24206c3fb27SDimitry Andric (uint32_t)PseudoProbeAttributes::Sentinel, 0); 243bdd1243dSDimitry Andric const MCPseudoProbe *Probe = &SentinelProbe; 244bdd1243dSDimitry Andric Inlinee.second->emit(MCOS, Probe); 245bdd1243dSDimitry Andric } 246e8d8bef9SDimitry Andric } 247e8d8bef9SDimitry Andric } 248e8d8bef9SDimitry Andric } 249e8d8bef9SDimitry Andric 250e8d8bef9SDimitry Andric // 251e8d8bef9SDimitry Andric // This emits the pseudo probe tables. 252e8d8bef9SDimitry Andric // 253e8d8bef9SDimitry Andric void MCPseudoProbeTable::emit(MCObjectStreamer *MCOS) { 254e8d8bef9SDimitry Andric MCContext &Ctx = MCOS->getContext(); 255e8d8bef9SDimitry Andric auto &ProbeTable = Ctx.getMCPseudoProbeTable(); 256e8d8bef9SDimitry Andric 257e8d8bef9SDimitry Andric // Bail out early so we don't switch to the pseudo_probe section needlessly 258e8d8bef9SDimitry Andric // and in doing so create an unnecessary (if empty) section. 259e8d8bef9SDimitry Andric auto &ProbeSections = ProbeTable.getProbeSections(); 260e8d8bef9SDimitry Andric if (ProbeSections.empty()) 261e8d8bef9SDimitry Andric return; 262e8d8bef9SDimitry Andric 263e8d8bef9SDimitry Andric LLVM_DEBUG(MCPseudoProbeTable::DdgPrintIndent = 0); 264e8d8bef9SDimitry Andric 265e8d8bef9SDimitry Andric // Put out the probe. 266e8d8bef9SDimitry Andric ProbeSections.emit(MCOS); 267e8d8bef9SDimitry Andric } 268349cc55cSDimitry Andric 269349cc55cSDimitry Andric static StringRef getProbeFNameForGUID(const GUIDProbeFunctionMap &GUID2FuncMAP, 270349cc55cSDimitry Andric uint64_t GUID) { 271349cc55cSDimitry Andric auto It = GUID2FuncMAP.find(GUID); 272349cc55cSDimitry Andric assert(It != GUID2FuncMAP.end() && 273349cc55cSDimitry Andric "Probe function must exist for a valid GUID"); 274349cc55cSDimitry Andric return It->second.FuncName; 275349cc55cSDimitry Andric } 276349cc55cSDimitry Andric 277349cc55cSDimitry Andric void MCPseudoProbeFuncDesc::print(raw_ostream &OS) { 278349cc55cSDimitry Andric OS << "GUID: " << FuncGUID << " Name: " << FuncName << "\n"; 279349cc55cSDimitry Andric OS << "Hash: " << FuncHash << "\n"; 280349cc55cSDimitry Andric } 281349cc55cSDimitry Andric 282349cc55cSDimitry Andric void MCDecodedPseudoProbe::getInlineContext( 283*0fca6ea1SDimitry Andric SmallVectorImpl<MCPseudoProbeFrameLocation> &ContextStack, 284349cc55cSDimitry Andric const GUIDProbeFunctionMap &GUID2FuncMAP) const { 285349cc55cSDimitry Andric uint32_t Begin = ContextStack.size(); 286349cc55cSDimitry Andric MCDecodedPseudoProbeInlineTree *Cur = InlineTree; 287349cc55cSDimitry Andric // It will add the string of each node's inline site during iteration. 288349cc55cSDimitry Andric // Note that it won't include the probe's belonging function(leaf location) 289349cc55cSDimitry Andric while (Cur->hasInlineSite()) { 29081ad6265SDimitry Andric StringRef FuncName = getProbeFNameForGUID(GUID2FuncMAP, Cur->Parent->Guid); 291349cc55cSDimitry Andric ContextStack.emplace_back( 292*0fca6ea1SDimitry Andric MCPseudoProbeFrameLocation(FuncName, std::get<1>(Cur->ISite))); 293349cc55cSDimitry Andric Cur = static_cast<MCDecodedPseudoProbeInlineTree *>(Cur->Parent); 294349cc55cSDimitry Andric } 295349cc55cSDimitry Andric // Make the ContextStack in caller-callee order 296349cc55cSDimitry Andric std::reverse(ContextStack.begin() + Begin, ContextStack.end()); 297349cc55cSDimitry Andric } 298349cc55cSDimitry Andric 299349cc55cSDimitry Andric std::string MCDecodedPseudoProbe::getInlineContextStr( 300349cc55cSDimitry Andric const GUIDProbeFunctionMap &GUID2FuncMAP) const { 301349cc55cSDimitry Andric std::ostringstream OContextStr; 302*0fca6ea1SDimitry Andric SmallVector<MCPseudoProbeFrameLocation, 16> ContextStack; 303349cc55cSDimitry Andric getInlineContext(ContextStack, GUID2FuncMAP); 304349cc55cSDimitry Andric for (auto &Cxt : ContextStack) { 305349cc55cSDimitry Andric if (OContextStr.str().size()) 306349cc55cSDimitry Andric OContextStr << " @ "; 307349cc55cSDimitry Andric OContextStr << Cxt.first.str() << ":" << Cxt.second; 308349cc55cSDimitry Andric } 309349cc55cSDimitry Andric return OContextStr.str(); 310349cc55cSDimitry Andric } 311349cc55cSDimitry Andric 312349cc55cSDimitry Andric static const char *PseudoProbeTypeStr[3] = {"Block", "IndirectCall", 313349cc55cSDimitry Andric "DirectCall"}; 314349cc55cSDimitry Andric 315349cc55cSDimitry Andric void MCDecodedPseudoProbe::print(raw_ostream &OS, 316349cc55cSDimitry Andric const GUIDProbeFunctionMap &GUID2FuncMAP, 317349cc55cSDimitry Andric bool ShowName) const { 318349cc55cSDimitry Andric OS << "FUNC: "; 319349cc55cSDimitry Andric if (ShowName) { 320349cc55cSDimitry Andric StringRef FuncName = getProbeFNameForGUID(GUID2FuncMAP, Guid); 321349cc55cSDimitry Andric OS << FuncName.str() << " "; 322349cc55cSDimitry Andric } else { 323349cc55cSDimitry Andric OS << Guid << " "; 324349cc55cSDimitry Andric } 325349cc55cSDimitry Andric OS << "Index: " << Index << " "; 32606c3fb27SDimitry Andric if (Discriminator) 32706c3fb27SDimitry Andric OS << "Discriminator: " << Discriminator << " "; 328349cc55cSDimitry Andric OS << "Type: " << PseudoProbeTypeStr[static_cast<uint8_t>(Type)] << " "; 329349cc55cSDimitry Andric std::string InlineContextStr = getInlineContextStr(GUID2FuncMAP); 330349cc55cSDimitry Andric if (InlineContextStr.size()) { 331349cc55cSDimitry Andric OS << "Inlined: @ "; 332349cc55cSDimitry Andric OS << InlineContextStr; 333349cc55cSDimitry Andric } 334349cc55cSDimitry Andric OS << "\n"; 335349cc55cSDimitry Andric } 336349cc55cSDimitry Andric 337349cc55cSDimitry Andric template <typename T> ErrorOr<T> MCPseudoProbeDecoder::readUnencodedNumber() { 338349cc55cSDimitry Andric if (Data + sizeof(T) > End) { 339349cc55cSDimitry Andric return std::error_code(); 340349cc55cSDimitry Andric } 341*0fca6ea1SDimitry Andric T Val = endian::readNext<T, llvm::endianness::little>(Data); 342349cc55cSDimitry Andric return ErrorOr<T>(Val); 343349cc55cSDimitry Andric } 344349cc55cSDimitry Andric 345349cc55cSDimitry Andric template <typename T> ErrorOr<T> MCPseudoProbeDecoder::readUnsignedNumber() { 346349cc55cSDimitry Andric unsigned NumBytesRead = 0; 347349cc55cSDimitry Andric uint64_t Val = decodeULEB128(Data, &NumBytesRead); 348349cc55cSDimitry Andric if (Val > std::numeric_limits<T>::max() || (Data + NumBytesRead > End)) { 349349cc55cSDimitry Andric return std::error_code(); 350349cc55cSDimitry Andric } 351349cc55cSDimitry Andric Data += NumBytesRead; 352349cc55cSDimitry Andric return ErrorOr<T>(static_cast<T>(Val)); 353349cc55cSDimitry Andric } 354349cc55cSDimitry Andric 355349cc55cSDimitry Andric template <typename T> ErrorOr<T> MCPseudoProbeDecoder::readSignedNumber() { 356349cc55cSDimitry Andric unsigned NumBytesRead = 0; 357349cc55cSDimitry Andric int64_t Val = decodeSLEB128(Data, &NumBytesRead); 358349cc55cSDimitry Andric if (Val > std::numeric_limits<T>::max() || (Data + NumBytesRead > End)) { 359349cc55cSDimitry Andric return std::error_code(); 360349cc55cSDimitry Andric } 361349cc55cSDimitry Andric Data += NumBytesRead; 362349cc55cSDimitry Andric return ErrorOr<T>(static_cast<T>(Val)); 363349cc55cSDimitry Andric } 364349cc55cSDimitry Andric 365349cc55cSDimitry Andric ErrorOr<StringRef> MCPseudoProbeDecoder::readString(uint32_t Size) { 366349cc55cSDimitry Andric StringRef Str(reinterpret_cast<const char *>(Data), Size); 367349cc55cSDimitry Andric if (Data + Size > End) { 368349cc55cSDimitry Andric return std::error_code(); 369349cc55cSDimitry Andric } 370349cc55cSDimitry Andric Data += Size; 371349cc55cSDimitry Andric return ErrorOr<StringRef>(Str); 372349cc55cSDimitry Andric } 373349cc55cSDimitry Andric 374349cc55cSDimitry Andric bool MCPseudoProbeDecoder::buildGUID2FuncDescMap(const uint8_t *Start, 375349cc55cSDimitry Andric std::size_t Size) { 376349cc55cSDimitry Andric // The pseudo_probe_desc section has a format like: 377349cc55cSDimitry Andric // .section .pseudo_probe_desc,"",@progbits 378349cc55cSDimitry Andric // .quad -5182264717993193164 // GUID 379349cc55cSDimitry Andric // .quad 4294967295 // Hash 380349cc55cSDimitry Andric // .uleb 3 // Name size 381349cc55cSDimitry Andric // .ascii "foo" // Name 382349cc55cSDimitry Andric // .quad -2624081020897602054 383349cc55cSDimitry Andric // .quad 174696971957 384349cc55cSDimitry Andric // .uleb 34 385349cc55cSDimitry Andric // .ascii "main" 386349cc55cSDimitry Andric 387349cc55cSDimitry Andric Data = Start; 388349cc55cSDimitry Andric End = Data + Size; 389349cc55cSDimitry Andric 390349cc55cSDimitry Andric while (Data < End) { 391349cc55cSDimitry Andric auto ErrorOrGUID = readUnencodedNumber<uint64_t>(); 392349cc55cSDimitry Andric if (!ErrorOrGUID) 393349cc55cSDimitry Andric return false; 394349cc55cSDimitry Andric 395349cc55cSDimitry Andric auto ErrorOrHash = readUnencodedNumber<uint64_t>(); 396349cc55cSDimitry Andric if (!ErrorOrHash) 397349cc55cSDimitry Andric return false; 398349cc55cSDimitry Andric 399349cc55cSDimitry Andric auto ErrorOrNameSize = readUnsignedNumber<uint32_t>(); 400349cc55cSDimitry Andric if (!ErrorOrNameSize) 401349cc55cSDimitry Andric return false; 402349cc55cSDimitry Andric uint32_t NameSize = std::move(*ErrorOrNameSize); 403349cc55cSDimitry Andric 404349cc55cSDimitry Andric auto ErrorOrName = readString(NameSize); 405349cc55cSDimitry Andric if (!ErrorOrName) 406349cc55cSDimitry Andric return false; 407349cc55cSDimitry Andric 408349cc55cSDimitry Andric uint64_t GUID = std::move(*ErrorOrGUID); 409349cc55cSDimitry Andric uint64_t Hash = std::move(*ErrorOrHash); 410349cc55cSDimitry Andric StringRef Name = std::move(*ErrorOrName); 411349cc55cSDimitry Andric 412349cc55cSDimitry Andric // Initialize PseudoProbeFuncDesc and populate it into GUID2FuncDescMap 413349cc55cSDimitry Andric GUID2FuncDescMap.emplace(GUID, MCPseudoProbeFuncDesc(GUID, Hash, Name)); 414349cc55cSDimitry Andric } 415349cc55cSDimitry Andric assert(Data == End && "Have unprocessed data in pseudo_probe_desc section"); 416349cc55cSDimitry Andric return true; 417349cc55cSDimitry Andric } 418349cc55cSDimitry Andric 41981ad6265SDimitry Andric bool MCPseudoProbeDecoder::buildAddress2ProbeMap( 42081ad6265SDimitry Andric MCDecodedPseudoProbeInlineTree *Cur, uint64_t &LastAddr, 421bdd1243dSDimitry Andric const Uint64Set &GuidFilter, const Uint64Map &FuncStartAddrs) { 422349cc55cSDimitry Andric // The pseudo_probe section encodes an inline forest and each tree has a 423bdd1243dSDimitry Andric // format defined in MCPseudoProbe.h 424349cc55cSDimitry Andric 425349cc55cSDimitry Andric uint32_t Index = 0; 426bdd1243dSDimitry Andric bool IsTopLevelFunc = Cur == &DummyInlineRoot; 427bdd1243dSDimitry Andric if (IsTopLevelFunc) { 428349cc55cSDimitry Andric // Use a sequential id for top level inliner. 42981ad6265SDimitry Andric Index = Cur->getChildren().size(); 430349cc55cSDimitry Andric } else { 431349cc55cSDimitry Andric // Read inline site for inlinees 432349cc55cSDimitry Andric auto ErrorOrIndex = readUnsignedNumber<uint32_t>(); 433349cc55cSDimitry Andric if (!ErrorOrIndex) 434349cc55cSDimitry Andric return false; 435349cc55cSDimitry Andric Index = std::move(*ErrorOrIndex); 436349cc55cSDimitry Andric } 43781ad6265SDimitry Andric 438349cc55cSDimitry Andric // Read guid 439349cc55cSDimitry Andric auto ErrorOrCurGuid = readUnencodedNumber<uint64_t>(); 440349cc55cSDimitry Andric if (!ErrorOrCurGuid) 441349cc55cSDimitry Andric return false; 44281ad6265SDimitry Andric uint64_t Guid = std::move(*ErrorOrCurGuid); 44381ad6265SDimitry Andric 44481ad6265SDimitry Andric // Decide if top-level node should be disgarded. 445bdd1243dSDimitry Andric if (IsTopLevelFunc && !GuidFilter.empty() && !GuidFilter.count(Guid)) 44681ad6265SDimitry Andric Cur = nullptr; 44781ad6265SDimitry Andric 44881ad6265SDimitry Andric // If the incoming node is null, all its children nodes should be disgarded. 44981ad6265SDimitry Andric if (Cur) { 45081ad6265SDimitry Andric // Switch/add to a new tree node(inlinee) 45181ad6265SDimitry Andric Cur = Cur->getOrAddNode(std::make_tuple(Guid, Index)); 45281ad6265SDimitry Andric Cur->Guid = Guid; 453bdd1243dSDimitry Andric if (IsTopLevelFunc && !EncodingIsAddrBased) { 454bdd1243dSDimitry Andric if (auto V = FuncStartAddrs.lookup(Guid)) 455bdd1243dSDimitry Andric LastAddr = V; 456bdd1243dSDimitry Andric } 45781ad6265SDimitry Andric } 45881ad6265SDimitry Andric 459349cc55cSDimitry Andric // Read number of probes in the current node. 460349cc55cSDimitry Andric auto ErrorOrNodeCount = readUnsignedNumber<uint32_t>(); 461349cc55cSDimitry Andric if (!ErrorOrNodeCount) 462349cc55cSDimitry Andric return false; 463349cc55cSDimitry Andric uint32_t NodeCount = std::move(*ErrorOrNodeCount); 464349cc55cSDimitry Andric // Read number of direct inlinees 465349cc55cSDimitry Andric auto ErrorOrCurChildrenToProcess = readUnsignedNumber<uint32_t>(); 466349cc55cSDimitry Andric if (!ErrorOrCurChildrenToProcess) 467349cc55cSDimitry Andric return false; 468349cc55cSDimitry Andric // Read all probes in this node 469349cc55cSDimitry Andric for (std::size_t I = 0; I < NodeCount; I++) { 470349cc55cSDimitry Andric // Read index 471349cc55cSDimitry Andric auto ErrorOrIndex = readUnsignedNumber<uint32_t>(); 472349cc55cSDimitry Andric if (!ErrorOrIndex) 473349cc55cSDimitry Andric return false; 474349cc55cSDimitry Andric uint32_t Index = std::move(*ErrorOrIndex); 475349cc55cSDimitry Andric // Read type | flag. 476349cc55cSDimitry Andric auto ErrorOrValue = readUnencodedNumber<uint8_t>(); 477349cc55cSDimitry Andric if (!ErrorOrValue) 478349cc55cSDimitry Andric return false; 479349cc55cSDimitry Andric uint8_t Value = std::move(*ErrorOrValue); 480349cc55cSDimitry Andric uint8_t Kind = Value & 0xf; 481349cc55cSDimitry Andric uint8_t Attr = (Value & 0x70) >> 4; 482349cc55cSDimitry Andric // Read address 483349cc55cSDimitry Andric uint64_t Addr = 0; 484349cc55cSDimitry Andric if (Value & 0x80) { 485349cc55cSDimitry Andric auto ErrorOrOffset = readSignedNumber<int64_t>(); 486349cc55cSDimitry Andric if (!ErrorOrOffset) 487349cc55cSDimitry Andric return false; 488349cc55cSDimitry Andric int64_t Offset = std::move(*ErrorOrOffset); 489349cc55cSDimitry Andric Addr = LastAddr + Offset; 490349cc55cSDimitry Andric } else { 491349cc55cSDimitry Andric auto ErrorOrAddr = readUnencodedNumber<int64_t>(); 492349cc55cSDimitry Andric if (!ErrorOrAddr) 493349cc55cSDimitry Andric return false; 494349cc55cSDimitry Andric Addr = std::move(*ErrorOrAddr); 495bdd1243dSDimitry Andric if (isSentinelProbe(Attr)) { 496bdd1243dSDimitry Andric // For sentinel probe, the addr field actually stores the GUID of the 497bdd1243dSDimitry Andric // split function. Convert it to the real address. 498bdd1243dSDimitry Andric if (auto V = FuncStartAddrs.lookup(Addr)) 499bdd1243dSDimitry Andric Addr = V; 500bdd1243dSDimitry Andric } else { 501bdd1243dSDimitry Andric // For now we assume all probe encoding should be either based on 502bdd1243dSDimitry Andric // leading probe address or function start address. 503bdd1243dSDimitry Andric // The scheme is for downwards compatibility. 504bdd1243dSDimitry Andric // TODO: retire this scheme once compatibility is no longer an issue. 505bdd1243dSDimitry Andric EncodingIsAddrBased = true; 506bdd1243dSDimitry Andric } 507349cc55cSDimitry Andric } 50881ad6265SDimitry Andric 50906c3fb27SDimitry Andric uint32_t Discriminator = 0; 51006c3fb27SDimitry Andric if (hasDiscriminator(Attr)) { 51106c3fb27SDimitry Andric auto ErrorOrDiscriminator = readUnsignedNumber<uint32_t>(); 51206c3fb27SDimitry Andric if (!ErrorOrDiscriminator) 51306c3fb27SDimitry Andric return false; 51406c3fb27SDimitry Andric Discriminator = std::move(*ErrorOrDiscriminator); 51506c3fb27SDimitry Andric } 51606c3fb27SDimitry Andric 517bdd1243dSDimitry Andric if (Cur && !isSentinelProbe(Attr)) { 518349cc55cSDimitry Andric // Populate Address2ProbesMap 519349cc55cSDimitry Andric auto &Probes = Address2ProbesMap[Addr]; 520349cc55cSDimitry Andric Probes.emplace_back(Addr, Cur->Guid, Index, PseudoProbeType(Kind), Attr, 52106c3fb27SDimitry Andric Discriminator, Cur); 522349cc55cSDimitry Andric Cur->addProbes(&Probes.back()); 52381ad6265SDimitry Andric } 524349cc55cSDimitry Andric LastAddr = Addr; 525349cc55cSDimitry Andric } 526349cc55cSDimitry Andric 52781ad6265SDimitry Andric uint32_t ChildrenToProcess = std::move(*ErrorOrCurChildrenToProcess); 52881ad6265SDimitry Andric for (uint32_t I = 0; I < ChildrenToProcess; I++) { 529bdd1243dSDimitry Andric buildAddress2ProbeMap(Cur, LastAddr, GuidFilter, FuncStartAddrs); 530349cc55cSDimitry Andric } 531349cc55cSDimitry Andric 532349cc55cSDimitry Andric return true; 533349cc55cSDimitry Andric } 534349cc55cSDimitry Andric 53581ad6265SDimitry Andric bool MCPseudoProbeDecoder::buildAddress2ProbeMap( 536bdd1243dSDimitry Andric const uint8_t *Start, std::size_t Size, const Uint64Set &GuidFilter, 537bdd1243dSDimitry Andric const Uint64Map &FuncStartAddrs) { 53881ad6265SDimitry Andric Data = Start; 53981ad6265SDimitry Andric End = Data + Size; 54081ad6265SDimitry Andric uint64_t LastAddr = 0; 54181ad6265SDimitry Andric while (Data < End) 542bdd1243dSDimitry Andric buildAddress2ProbeMap(&DummyInlineRoot, LastAddr, GuidFilter, 543bdd1243dSDimitry Andric FuncStartAddrs); 54481ad6265SDimitry Andric assert(Data == End && "Have unprocessed data in pseudo_probe section"); 54581ad6265SDimitry Andric return true; 54681ad6265SDimitry Andric } 54781ad6265SDimitry Andric 548349cc55cSDimitry Andric void MCPseudoProbeDecoder::printGUID2FuncDescMap(raw_ostream &OS) { 549349cc55cSDimitry Andric OS << "Pseudo Probe Desc:\n"; 550349cc55cSDimitry Andric // Make the output deterministic 551349cc55cSDimitry Andric std::map<uint64_t, MCPseudoProbeFuncDesc> OrderedMap(GUID2FuncDescMap.begin(), 552349cc55cSDimitry Andric GUID2FuncDescMap.end()); 553349cc55cSDimitry Andric for (auto &I : OrderedMap) { 554349cc55cSDimitry Andric I.second.print(OS); 555349cc55cSDimitry Andric } 556349cc55cSDimitry Andric } 557349cc55cSDimitry Andric 558349cc55cSDimitry Andric void MCPseudoProbeDecoder::printProbeForAddress(raw_ostream &OS, 559349cc55cSDimitry Andric uint64_t Address) { 560349cc55cSDimitry Andric auto It = Address2ProbesMap.find(Address); 561349cc55cSDimitry Andric if (It != Address2ProbesMap.end()) { 562349cc55cSDimitry Andric for (auto &Probe : It->second) { 563349cc55cSDimitry Andric OS << " [Probe]:\t"; 564349cc55cSDimitry Andric Probe.print(OS, GUID2FuncDescMap, true); 565349cc55cSDimitry Andric } 566349cc55cSDimitry Andric } 567349cc55cSDimitry Andric } 568349cc55cSDimitry Andric 569349cc55cSDimitry Andric void MCPseudoProbeDecoder::printProbesForAllAddresses(raw_ostream &OS) { 5705f757f3fSDimitry Andric auto Entries = make_first_range(Address2ProbesMap); 5715f757f3fSDimitry Andric SmallVector<uint64_t, 0> Addresses(Entries.begin(), Entries.end()); 572fcaf7f86SDimitry Andric llvm::sort(Addresses); 573349cc55cSDimitry Andric for (auto K : Addresses) { 574349cc55cSDimitry Andric OS << "Address:\t"; 575349cc55cSDimitry Andric OS << K; 576349cc55cSDimitry Andric OS << "\n"; 577349cc55cSDimitry Andric printProbeForAddress(OS, K); 578349cc55cSDimitry Andric } 579349cc55cSDimitry Andric } 580349cc55cSDimitry Andric 581349cc55cSDimitry Andric const MCDecodedPseudoProbe * 582349cc55cSDimitry Andric MCPseudoProbeDecoder::getCallProbeForAddr(uint64_t Address) const { 583349cc55cSDimitry Andric auto It = Address2ProbesMap.find(Address); 584349cc55cSDimitry Andric if (It == Address2ProbesMap.end()) 585349cc55cSDimitry Andric return nullptr; 586349cc55cSDimitry Andric const auto &Probes = It->second; 587349cc55cSDimitry Andric 588349cc55cSDimitry Andric const MCDecodedPseudoProbe *CallProbe = nullptr; 589349cc55cSDimitry Andric for (const auto &Probe : Probes) { 590349cc55cSDimitry Andric if (Probe.isCall()) { 59106c3fb27SDimitry Andric // Disabling the assert and returning first call probe seen so far. 59206c3fb27SDimitry Andric // Subsequent call probes, if any, are ignored. Due to the the way 59306c3fb27SDimitry Andric // .pseudo_probe section is decoded, probes of the same-named independent 59406c3fb27SDimitry Andric // static functions are merged thus multiple call probes may be seen for a 59506c3fb27SDimitry Andric // callsite. This should only happen to compiler-generated statics, with 59606c3fb27SDimitry Andric // -funique-internal-linkage-names where user statics get unique names. 59706c3fb27SDimitry Andric // 59806c3fb27SDimitry Andric // TODO: re-enable or narrow down the assert to static functions only. 59906c3fb27SDimitry Andric // 60006c3fb27SDimitry Andric // assert(!CallProbe && 60106c3fb27SDimitry Andric // "There should be only one call probe corresponding to address " 60206c3fb27SDimitry Andric // "which is a callsite."); 603349cc55cSDimitry Andric CallProbe = &Probe; 60406c3fb27SDimitry Andric break; 605349cc55cSDimitry Andric } 606349cc55cSDimitry Andric } 607349cc55cSDimitry Andric return CallProbe; 608349cc55cSDimitry Andric } 609349cc55cSDimitry Andric 610349cc55cSDimitry Andric const MCPseudoProbeFuncDesc * 611349cc55cSDimitry Andric MCPseudoProbeDecoder::getFuncDescForGUID(uint64_t GUID) const { 612349cc55cSDimitry Andric auto It = GUID2FuncDescMap.find(GUID); 613349cc55cSDimitry Andric assert(It != GUID2FuncDescMap.end() && "Function descriptor doesn't exist"); 614349cc55cSDimitry Andric return &It->second; 615349cc55cSDimitry Andric } 616349cc55cSDimitry Andric 617349cc55cSDimitry Andric void MCPseudoProbeDecoder::getInlineContextForProbe( 618349cc55cSDimitry Andric const MCDecodedPseudoProbe *Probe, 619*0fca6ea1SDimitry Andric SmallVectorImpl<MCPseudoProbeFrameLocation> &InlineContextStack, 620349cc55cSDimitry Andric bool IncludeLeaf) const { 621349cc55cSDimitry Andric Probe->getInlineContext(InlineContextStack, GUID2FuncDescMap); 622349cc55cSDimitry Andric if (!IncludeLeaf) 623349cc55cSDimitry Andric return; 624349cc55cSDimitry Andric // Note that the context from probe doesn't include leaf frame, 625349cc55cSDimitry Andric // hence we need to retrieve and prepend leaf if requested. 626349cc55cSDimitry Andric const auto *FuncDesc = getFuncDescForGUID(Probe->getGuid()); 627349cc55cSDimitry Andric InlineContextStack.emplace_back( 628*0fca6ea1SDimitry Andric MCPseudoProbeFrameLocation(FuncDesc->FuncName, Probe->getIndex())); 629349cc55cSDimitry Andric } 630349cc55cSDimitry Andric 631349cc55cSDimitry Andric const MCPseudoProbeFuncDesc *MCPseudoProbeDecoder::getInlinerDescForProbe( 632349cc55cSDimitry Andric const MCDecodedPseudoProbe *Probe) const { 633349cc55cSDimitry Andric MCDecodedPseudoProbeInlineTree *InlinerNode = Probe->getInlineTreeNode(); 634349cc55cSDimitry Andric if (!InlinerNode->hasInlineSite()) 635349cc55cSDimitry Andric return nullptr; 63681ad6265SDimitry Andric return getFuncDescForGUID(InlinerNode->Parent->Guid); 637349cc55cSDimitry Andric } 638