1 //===-- InstrProfCorrelator.cpp -------------------------------------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 9 #include "llvm/ProfileData/InstrProfCorrelator.h" 10 #include "llvm/DebugInfo/DIContext.h" 11 #include "llvm/DebugInfo/DWARF/DWARFContext.h" 12 #include "llvm/DebugInfo/DWARF/DWARFDie.h" 13 #include "llvm/DebugInfo/DWARF/DWARFExpression.h" 14 #include "llvm/DebugInfo/DWARF/DWARFFormValue.h" 15 #include "llvm/DebugInfo/DWARF/DWARFLocationExpression.h" 16 #include "llvm/DebugInfo/DWARF/DWARFUnit.h" 17 #include "llvm/Object/MachO.h" 18 #include "llvm/Support/Debug.h" 19 #include <optional> 20 21 #define DEBUG_TYPE "correlator" 22 23 using namespace llvm; 24 25 /// Get the __llvm_prf_cnts section. 26 Expected<object::SectionRef> getCountersSection(const object::ObjectFile &Obj) { 27 for (auto &Section : Obj.sections()) 28 if (auto SectionName = Section.getName()) 29 if (SectionName.get() == INSTR_PROF_CNTS_SECT_NAME) 30 return Section; 31 return make_error<InstrProfError>( 32 instrprof_error::unable_to_correlate_profile, 33 "could not find counter section (" INSTR_PROF_CNTS_SECT_NAME ")"); 34 } 35 36 const char *InstrProfCorrelator::FunctionNameAttributeName = "Function Name"; 37 const char *InstrProfCorrelator::CFGHashAttributeName = "CFG Hash"; 38 const char *InstrProfCorrelator::NumCountersAttributeName = "Num Counters"; 39 40 llvm::Expected<std::unique_ptr<InstrProfCorrelator::Context>> 41 InstrProfCorrelator::Context::get(std::unique_ptr<MemoryBuffer> Buffer, 42 const object::ObjectFile &Obj) { 43 auto CountersSection = getCountersSection(Obj); 44 if (auto Err = CountersSection.takeError()) 45 return std::move(Err); 46 auto C = std::make_unique<Context>(); 47 C->Buffer = std::move(Buffer); 48 C->CountersSectionStart = CountersSection->getAddress(); 49 C->CountersSectionEnd = C->CountersSectionStart + CountersSection->getSize(); 50 C->ShouldSwapBytes = Obj.isLittleEndian() != sys::IsLittleEndianHost; 51 return Expected<std::unique_ptr<Context>>(std::move(C)); 52 } 53 54 llvm::Expected<std::unique_ptr<InstrProfCorrelator>> 55 InstrProfCorrelator::get(StringRef DebugInfoFilename) { 56 auto DsymObjectsOrErr = 57 object::MachOObjectFile::findDsymObjectMembers(DebugInfoFilename); 58 if (auto Err = DsymObjectsOrErr.takeError()) 59 return std::move(Err); 60 if (!DsymObjectsOrErr->empty()) { 61 // TODO: Enable profile correlation when there are multiple objects in a 62 // dSYM bundle. 63 if (DsymObjectsOrErr->size() > 1) 64 return make_error<InstrProfError>( 65 instrprof_error::unable_to_correlate_profile, 66 "using multiple objects is not yet supported"); 67 DebugInfoFilename = *DsymObjectsOrErr->begin(); 68 } 69 auto BufferOrErr = 70 errorOrToExpected(MemoryBuffer::getFile(DebugInfoFilename)); 71 if (auto Err = BufferOrErr.takeError()) 72 return std::move(Err); 73 74 return get(std::move(*BufferOrErr)); 75 } 76 77 llvm::Expected<std::unique_ptr<InstrProfCorrelator>> 78 InstrProfCorrelator::get(std::unique_ptr<MemoryBuffer> Buffer) { 79 auto BinOrErr = object::createBinary(*Buffer); 80 if (auto Err = BinOrErr.takeError()) 81 return std::move(Err); 82 83 if (auto *Obj = dyn_cast<object::ObjectFile>(BinOrErr->get())) { 84 auto CtxOrErr = Context::get(std::move(Buffer), *Obj); 85 if (auto Err = CtxOrErr.takeError()) 86 return std::move(Err); 87 auto T = Obj->makeTriple(); 88 if (T.isArch64Bit()) 89 return InstrProfCorrelatorImpl<uint64_t>::get(std::move(*CtxOrErr), *Obj); 90 if (T.isArch32Bit()) 91 return InstrProfCorrelatorImpl<uint32_t>::get(std::move(*CtxOrErr), *Obj); 92 } 93 return make_error<InstrProfError>( 94 instrprof_error::unable_to_correlate_profile, "not an object file"); 95 } 96 97 std::optional<size_t> InstrProfCorrelator::getDataSize() const { 98 if (auto *C = dyn_cast<InstrProfCorrelatorImpl<uint32_t>>(this)) { 99 return C->getDataSize(); 100 } else if (auto *C = dyn_cast<InstrProfCorrelatorImpl<uint64_t>>(this)) { 101 return C->getDataSize(); 102 } 103 return {}; 104 } 105 106 namespace llvm { 107 108 template <> 109 InstrProfCorrelatorImpl<uint32_t>::InstrProfCorrelatorImpl( 110 std::unique_ptr<InstrProfCorrelator::Context> Ctx) 111 : InstrProfCorrelatorImpl(InstrProfCorrelatorKind::CK_32Bit, 112 std::move(Ctx)) {} 113 template <> 114 InstrProfCorrelatorImpl<uint64_t>::InstrProfCorrelatorImpl( 115 std::unique_ptr<InstrProfCorrelator::Context> Ctx) 116 : InstrProfCorrelatorImpl(InstrProfCorrelatorKind::CK_64Bit, 117 std::move(Ctx)) {} 118 template <> 119 bool InstrProfCorrelatorImpl<uint32_t>::classof(const InstrProfCorrelator *C) { 120 return C->getKind() == InstrProfCorrelatorKind::CK_32Bit; 121 } 122 template <> 123 bool InstrProfCorrelatorImpl<uint64_t>::classof(const InstrProfCorrelator *C) { 124 return C->getKind() == InstrProfCorrelatorKind::CK_64Bit; 125 } 126 127 } // end namespace llvm 128 129 template <class IntPtrT> 130 llvm::Expected<std::unique_ptr<InstrProfCorrelatorImpl<IntPtrT>>> 131 InstrProfCorrelatorImpl<IntPtrT>::get( 132 std::unique_ptr<InstrProfCorrelator::Context> Ctx, 133 const object::ObjectFile &Obj) { 134 if (Obj.isELF() || Obj.isMachO()) { 135 auto DICtx = DWARFContext::create(Obj); 136 return std::make_unique<DwarfInstrProfCorrelator<IntPtrT>>(std::move(DICtx), 137 std::move(Ctx)); 138 } 139 return make_error<InstrProfError>( 140 instrprof_error::unable_to_correlate_profile, 141 "unsupported debug info format (only DWARF is supported)"); 142 } 143 144 template <class IntPtrT> 145 Error InstrProfCorrelatorImpl<IntPtrT>::correlateProfileData() { 146 assert(Data.empty() && Names.empty() && NamesVec.empty()); 147 correlateProfileDataImpl(); 148 if (Data.empty() || NamesVec.empty()) 149 return make_error<InstrProfError>( 150 instrprof_error::unable_to_correlate_profile, 151 "could not find any profile metadata in debug info"); 152 auto Result = 153 collectPGOFuncNameStrings(NamesVec, /*doCompression=*/false, Names); 154 CounterOffsets.clear(); 155 NamesVec.clear(); 156 return Result; 157 } 158 159 template <> struct yaml::MappingTraits<InstrProfCorrelator::CorrelationData> { 160 static void mapping(yaml::IO &io, 161 InstrProfCorrelator::CorrelationData &Data) { 162 io.mapRequired("Probes", Data.Probes); 163 } 164 }; 165 166 template <> struct yaml::MappingTraits<InstrProfCorrelator::Probe> { 167 static void mapping(yaml::IO &io, InstrProfCorrelator::Probe &P) { 168 io.mapRequired("Function Name", P.FunctionName); 169 io.mapOptional("Linkage Name", P.LinkageName); 170 io.mapRequired("CFG Hash", P.CFGHash); 171 io.mapRequired("Counter Offset", P.CounterOffset); 172 io.mapRequired("Num Counters", P.NumCounters); 173 io.mapOptional("File", P.FilePath); 174 io.mapOptional("Line", P.LineNumber); 175 } 176 }; 177 178 template <> struct yaml::SequenceElementTraits<InstrProfCorrelator::Probe> { 179 static const bool flow = false; 180 }; 181 182 template <class IntPtrT> 183 Error InstrProfCorrelatorImpl<IntPtrT>::dumpYaml(raw_ostream &OS) { 184 InstrProfCorrelator::CorrelationData Data; 185 correlateProfileDataImpl(&Data); 186 if (Data.Probes.empty()) 187 return make_error<InstrProfError>( 188 instrprof_error::unable_to_correlate_profile, 189 "could not find any profile metadata in debug info"); 190 yaml::Output YamlOS(OS); 191 YamlOS << Data; 192 return Error::success(); 193 } 194 195 template <class IntPtrT> 196 void InstrProfCorrelatorImpl<IntPtrT>::addProbe(StringRef FunctionName, 197 uint64_t CFGHash, 198 IntPtrT CounterOffset, 199 IntPtrT FunctionPtr, 200 uint32_t NumCounters) { 201 // Check if a probe was already added for this counter offset. 202 if (!CounterOffsets.insert(CounterOffset).second) 203 return; 204 Data.push_back({ 205 maybeSwap<uint64_t>(IndexedInstrProf::ComputeHash(FunctionName)), 206 maybeSwap<uint64_t>(CFGHash), 207 // In this mode, CounterPtr actually stores the section relative address 208 // of the counter. 209 maybeSwap<IntPtrT>(CounterOffset), 210 maybeSwap<IntPtrT>(FunctionPtr), 211 // TODO: Value profiling is not yet supported. 212 /*ValuesPtr=*/maybeSwap<IntPtrT>(0), 213 maybeSwap<uint32_t>(NumCounters), 214 /*NumValueSites=*/{maybeSwap<uint16_t>(0), maybeSwap<uint16_t>(0)}, 215 }); 216 NamesVec.push_back(FunctionName.str()); 217 } 218 219 template <class IntPtrT> 220 std::optional<uint64_t> 221 DwarfInstrProfCorrelator<IntPtrT>::getLocation(const DWARFDie &Die) const { 222 auto Locations = Die.getLocations(dwarf::DW_AT_location); 223 if (!Locations) { 224 consumeError(Locations.takeError()); 225 return {}; 226 } 227 auto &DU = *Die.getDwarfUnit(); 228 auto AddressSize = DU.getAddressByteSize(); 229 for (auto &Location : *Locations) { 230 DataExtractor Data(Location.Expr, DICtx->isLittleEndian(), AddressSize); 231 DWARFExpression Expr(Data, AddressSize); 232 for (auto &Op : Expr) { 233 if (Op.getCode() == dwarf::DW_OP_addr) { 234 return Op.getRawOperand(0); 235 } else if (Op.getCode() == dwarf::DW_OP_addrx) { 236 uint64_t Index = Op.getRawOperand(0); 237 if (auto SA = DU.getAddrOffsetSectionItem(Index)) 238 return SA->Address; 239 } 240 } 241 } 242 return {}; 243 } 244 245 template <class IntPtrT> 246 bool DwarfInstrProfCorrelator<IntPtrT>::isDIEOfProbe(const DWARFDie &Die) { 247 const auto &ParentDie = Die.getParent(); 248 if (!Die.isValid() || !ParentDie.isValid() || Die.isNULL()) 249 return false; 250 if (Die.getTag() != dwarf::DW_TAG_variable) 251 return false; 252 if (!ParentDie.isSubprogramDIE()) 253 return false; 254 if (!Die.hasChildren()) 255 return false; 256 if (const char *Name = Die.getName(DINameKind::ShortName)) 257 return StringRef(Name).startswith(getInstrProfCountersVarPrefix()); 258 return false; 259 } 260 261 template <class IntPtrT> 262 void DwarfInstrProfCorrelator<IntPtrT>::correlateProfileDataImpl( 263 InstrProfCorrelator::CorrelationData *Data) { 264 auto maybeAddProbe = [&](DWARFDie Die) { 265 if (!isDIEOfProbe(Die)) 266 return; 267 std::optional<const char *> FunctionName; 268 std::optional<uint64_t> CFGHash; 269 std::optional<uint64_t> CounterPtr = getLocation(Die); 270 auto FnDie = Die.getParent(); 271 auto FunctionPtr = dwarf::toAddress(FnDie.find(dwarf::DW_AT_low_pc)); 272 std::optional<uint64_t> NumCounters; 273 for (const DWARFDie &Child : Die.children()) { 274 if (Child.getTag() != dwarf::DW_TAG_LLVM_annotation) 275 continue; 276 auto AnnotationFormName = Child.find(dwarf::DW_AT_name); 277 auto AnnotationFormValue = Child.find(dwarf::DW_AT_const_value); 278 if (!AnnotationFormName || !AnnotationFormValue) 279 continue; 280 auto AnnotationNameOrErr = AnnotationFormName->getAsCString(); 281 if (auto Err = AnnotationNameOrErr.takeError()) { 282 consumeError(std::move(Err)); 283 continue; 284 } 285 StringRef AnnotationName = *AnnotationNameOrErr; 286 if (AnnotationName.compare( 287 InstrProfCorrelator::FunctionNameAttributeName) == 0) { 288 if (auto EC = 289 AnnotationFormValue->getAsCString().moveInto(FunctionName)) 290 consumeError(std::move(EC)); 291 } else if (AnnotationName.compare( 292 InstrProfCorrelator::CFGHashAttributeName) == 0) { 293 CFGHash = AnnotationFormValue->getAsUnsignedConstant(); 294 } else if (AnnotationName.compare( 295 InstrProfCorrelator::NumCountersAttributeName) == 0) { 296 NumCounters = AnnotationFormValue->getAsUnsignedConstant(); 297 } 298 } 299 if (!FunctionName || !CFGHash || !CounterPtr || !NumCounters) { 300 LLVM_DEBUG(dbgs() << "Incomplete DIE for probe\n\tFunctionName: " 301 << FunctionName << "\n\tCFGHash: " << CFGHash 302 << "\n\tCounterPtr: " << CounterPtr 303 << "\n\tNumCounters: " << NumCounters); 304 LLVM_DEBUG(Die.dump(dbgs())); 305 return; 306 } 307 uint64_t CountersStart = this->Ctx->CountersSectionStart; 308 uint64_t CountersEnd = this->Ctx->CountersSectionEnd; 309 if (*CounterPtr < CountersStart || *CounterPtr >= CountersEnd) { 310 LLVM_DEBUG( 311 dbgs() << "CounterPtr out of range for probe\n\tFunction Name: " 312 << FunctionName << "\n\tExpected: [0x" 313 << Twine::utohexstr(CountersStart) << ", 0x" 314 << Twine::utohexstr(CountersEnd) << ")\n\tActual: 0x" 315 << Twine::utohexstr(*CounterPtr)); 316 LLVM_DEBUG(Die.dump(dbgs())); 317 return; 318 } 319 if (!FunctionPtr) { 320 LLVM_DEBUG(dbgs() << "Could not find address of " << *FunctionName 321 << "\n"); 322 LLVM_DEBUG(Die.dump(dbgs())); 323 } 324 IntPtrT CounterOffset = *CounterPtr - CountersStart; 325 if (Data) { 326 InstrProfCorrelator::Probe P; 327 P.FunctionName = *FunctionName; 328 if (auto Name = FnDie.getName(DINameKind::LinkageName)) 329 P.LinkageName = Name; 330 P.CFGHash = *CFGHash; 331 P.CounterOffset = CounterOffset; 332 P.NumCounters = *NumCounters; 333 auto FilePath = FnDie.getDeclFile( 334 DILineInfoSpecifier::FileLineInfoKind::RelativeFilePath); 335 if (!FilePath.empty()) 336 P.FilePath = FilePath; 337 if (auto LineNumber = FnDie.getDeclLine()) 338 P.LineNumber = LineNumber; 339 Data->Probes.push_back(P); 340 } else { 341 this->addProbe(*FunctionName, *CFGHash, CounterOffset, 342 FunctionPtr.value_or(0), *NumCounters); 343 } 344 }; 345 for (auto &CU : DICtx->normal_units()) 346 for (const auto &Entry : CU->dies()) 347 maybeAddProbe(DWARFDie(CU.get(), &Entry)); 348 for (auto &CU : DICtx->dwo_units()) 349 for (const auto &Entry : CU->dies()) 350 maybeAddProbe(DWARFDie(CU.get(), &Entry)); 351 } 352