1 //===-- InstrProfCorrelator.cpp -------------------------------------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 9 #include "llvm/ProfileData/InstrProfCorrelator.h" 10 #include "llvm/DebugInfo/DIContext.h" 11 #include "llvm/DebugInfo/DWARF/DWARFContext.h" 12 #include "llvm/DebugInfo/DWARF/DWARFDie.h" 13 #include "llvm/DebugInfo/DWARF/DWARFExpression.h" 14 #include "llvm/DebugInfo/DWARF/DWARFFormValue.h" 15 #include "llvm/DebugInfo/DWARF/DWARFLocationExpression.h" 16 #include "llvm/DebugInfo/DWARF/DWARFUnit.h" 17 #include "llvm/Object/MachO.h" 18 #include "llvm/Support/Debug.h" 19 #include "llvm/Support/Format.h" 20 #include "llvm/Support/WithColor.h" 21 #include <optional> 22 23 #define DEBUG_TYPE "correlator" 24 25 using namespace llvm; 26 27 /// Get profile section. 28 Expected<object::SectionRef> getInstrProfSection(const object::ObjectFile &Obj, 29 InstrProfSectKind IPSK) { 30 // On COFF, the getInstrProfSectionName returns the section names may followed 31 // by "$M". The linker removes the dollar and everything after it in the final 32 // binary. Do the same to match. 33 Triple::ObjectFormatType ObjFormat = Obj.getTripleObjectFormat(); 34 auto StripSuffix = [ObjFormat](StringRef N) { 35 return ObjFormat == Triple::COFF ? N.split('$').first : N; 36 }; 37 std::string ExpectedSectionName = 38 getInstrProfSectionName(IPSK, ObjFormat, 39 /*AddSegmentInfo=*/false); 40 ExpectedSectionName = StripSuffix(ExpectedSectionName); 41 for (auto &Section : Obj.sections()) { 42 if (auto SectionName = Section.getName()) 43 if (*SectionName == ExpectedSectionName) 44 return Section; 45 } 46 return make_error<InstrProfError>( 47 instrprof_error::unable_to_correlate_profile, 48 "could not find section (" + Twine(ExpectedSectionName) + ")"); 49 } 50 51 const char *InstrProfCorrelator::FunctionNameAttributeName = "Function Name"; 52 const char *InstrProfCorrelator::CFGHashAttributeName = "CFG Hash"; 53 const char *InstrProfCorrelator::NumCountersAttributeName = "Num Counters"; 54 55 llvm::Expected<std::unique_ptr<InstrProfCorrelator::Context>> 56 InstrProfCorrelator::Context::get(std::unique_ptr<MemoryBuffer> Buffer, 57 const object::ObjectFile &Obj, 58 ProfCorrelatorKind FileKind) { 59 auto C = std::make_unique<Context>(); 60 auto CountersSection = getInstrProfSection(Obj, IPSK_cnts); 61 if (auto Err = CountersSection.takeError()) 62 return std::move(Err); 63 if (FileKind == InstrProfCorrelator::BINARY) { 64 auto DataSection = getInstrProfSection(Obj, IPSK_covdata); 65 if (auto Err = DataSection.takeError()) 66 return std::move(Err); 67 auto DataOrErr = DataSection->getContents(); 68 if (!DataOrErr) 69 return DataOrErr.takeError(); 70 auto NameSection = getInstrProfSection(Obj, IPSK_covname); 71 if (auto Err = NameSection.takeError()) 72 return std::move(Err); 73 auto NameOrErr = NameSection->getContents(); 74 if (!NameOrErr) 75 return NameOrErr.takeError(); 76 C->DataStart = DataOrErr->data(); 77 C->DataEnd = DataOrErr->data() + DataOrErr->size(); 78 C->NameStart = NameOrErr->data(); 79 C->NameSize = NameOrErr->size(); 80 } 81 C->Buffer = std::move(Buffer); 82 C->CountersSectionStart = CountersSection->getAddress(); 83 C->CountersSectionEnd = C->CountersSectionStart + CountersSection->getSize(); 84 // In COFF object file, there's a null byte at the beginning of the counter 85 // section which doesn't exist in raw profile. 86 if (Obj.getTripleObjectFormat() == Triple::COFF) 87 ++C->CountersSectionStart; 88 89 C->ShouldSwapBytes = Obj.isLittleEndian() != sys::IsLittleEndianHost; 90 return Expected<std::unique_ptr<Context>>(std::move(C)); 91 } 92 93 llvm::Expected<std::unique_ptr<InstrProfCorrelator>> 94 InstrProfCorrelator::get(StringRef Filename, ProfCorrelatorKind FileKind, 95 const object::BuildIDFetcher *BIDFetcher, 96 const ArrayRef<object::BuildID> BIs) { 97 std::optional<std::string> Path; 98 if (BIDFetcher) { 99 if (BIs.empty()) 100 return make_error<InstrProfError>( 101 instrprof_error::unable_to_correlate_profile, 102 "unsupported profile binary correlation when there is no build ID " 103 "in a profile"); 104 if (BIs.size() > 1) 105 return make_error<InstrProfError>( 106 instrprof_error::unable_to_correlate_profile, 107 "unsupported profile binary correlation when there are multiple " 108 "build IDs in a profile"); 109 110 Path = BIDFetcher->fetch(BIs.front()); 111 if (!Path) 112 return make_error<InstrProfError>( 113 instrprof_error::unable_to_correlate_profile, 114 "Missing build ID: " + llvm::toHex(BIs.front(), 115 /*LowerCase=*/true)); 116 Filename = *Path; 117 } 118 119 if (FileKind == DEBUG_INFO) { 120 auto DsymObjectsOrErr = 121 object::MachOObjectFile::findDsymObjectMembers(Filename); 122 if (auto Err = DsymObjectsOrErr.takeError()) 123 return std::move(Err); 124 if (!DsymObjectsOrErr->empty()) { 125 // TODO: Enable profile correlation when there are multiple objects in a 126 // dSYM bundle. 127 if (DsymObjectsOrErr->size() > 1) 128 return make_error<InstrProfError>( 129 instrprof_error::unable_to_correlate_profile, 130 "using multiple objects is not yet supported"); 131 Filename = *DsymObjectsOrErr->begin(); 132 } 133 auto BufferOrErr = errorOrToExpected(MemoryBuffer::getFile(Filename)); 134 if (auto Err = BufferOrErr.takeError()) 135 return std::move(Err); 136 137 return get(std::move(*BufferOrErr), FileKind); 138 } 139 if (FileKind == BINARY) { 140 auto BufferOrErr = errorOrToExpected(MemoryBuffer::getFile(Filename)); 141 if (auto Err = BufferOrErr.takeError()) 142 return std::move(Err); 143 144 return get(std::move(*BufferOrErr), FileKind); 145 } 146 return make_error<InstrProfError>( 147 instrprof_error::unable_to_correlate_profile, 148 "unsupported correlation kind (only DWARF debug info and Binary format " 149 "(ELF/COFF) are supported)"); 150 } 151 152 llvm::Expected<std::unique_ptr<InstrProfCorrelator>> 153 InstrProfCorrelator::get(std::unique_ptr<MemoryBuffer> Buffer, 154 ProfCorrelatorKind FileKind) { 155 auto BinOrErr = object::createBinary(*Buffer); 156 if (auto Err = BinOrErr.takeError()) 157 return std::move(Err); 158 159 if (auto *Obj = dyn_cast<object::ObjectFile>(BinOrErr->get())) { 160 auto CtxOrErr = Context::get(std::move(Buffer), *Obj, FileKind); 161 if (auto Err = CtxOrErr.takeError()) 162 return std::move(Err); 163 auto T = Obj->makeTriple(); 164 if (T.isArch64Bit()) 165 return InstrProfCorrelatorImpl<uint64_t>::get(std::move(*CtxOrErr), *Obj, 166 FileKind); 167 if (T.isArch32Bit()) 168 return InstrProfCorrelatorImpl<uint32_t>::get(std::move(*CtxOrErr), *Obj, 169 FileKind); 170 } 171 return make_error<InstrProfError>( 172 instrprof_error::unable_to_correlate_profile, "not an object file"); 173 } 174 175 std::optional<size_t> InstrProfCorrelator::getDataSize() const { 176 if (auto *C = dyn_cast<InstrProfCorrelatorImpl<uint32_t>>(this)) { 177 return C->getDataSize(); 178 } else if (auto *C = dyn_cast<InstrProfCorrelatorImpl<uint64_t>>(this)) { 179 return C->getDataSize(); 180 } 181 return {}; 182 } 183 184 namespace llvm { 185 186 template <> 187 InstrProfCorrelatorImpl<uint32_t>::InstrProfCorrelatorImpl( 188 std::unique_ptr<InstrProfCorrelator::Context> Ctx) 189 : InstrProfCorrelatorImpl(InstrProfCorrelatorKind::CK_32Bit, 190 std::move(Ctx)) {} 191 template <> 192 InstrProfCorrelatorImpl<uint64_t>::InstrProfCorrelatorImpl( 193 std::unique_ptr<InstrProfCorrelator::Context> Ctx) 194 : InstrProfCorrelatorImpl(InstrProfCorrelatorKind::CK_64Bit, 195 std::move(Ctx)) {} 196 template <> 197 bool InstrProfCorrelatorImpl<uint32_t>::classof(const InstrProfCorrelator *C) { 198 return C->getKind() == InstrProfCorrelatorKind::CK_32Bit; 199 } 200 template <> 201 bool InstrProfCorrelatorImpl<uint64_t>::classof(const InstrProfCorrelator *C) { 202 return C->getKind() == InstrProfCorrelatorKind::CK_64Bit; 203 } 204 205 } // end namespace llvm 206 207 template <class IntPtrT> 208 llvm::Expected<std::unique_ptr<InstrProfCorrelatorImpl<IntPtrT>>> 209 InstrProfCorrelatorImpl<IntPtrT>::get( 210 std::unique_ptr<InstrProfCorrelator::Context> Ctx, 211 const object::ObjectFile &Obj, ProfCorrelatorKind FileKind) { 212 if (FileKind == DEBUG_INFO) { 213 if (Obj.isELF() || Obj.isMachO()) { 214 auto DICtx = DWARFContext::create(Obj); 215 return std::make_unique<DwarfInstrProfCorrelator<IntPtrT>>( 216 std::move(DICtx), std::move(Ctx)); 217 } 218 return make_error<InstrProfError>( 219 instrprof_error::unable_to_correlate_profile, 220 "unsupported debug info format (only DWARF is supported)"); 221 } 222 if (Obj.isELF() || Obj.isCOFF()) 223 return std::make_unique<BinaryInstrProfCorrelator<IntPtrT>>(std::move(Ctx)); 224 return make_error<InstrProfError>( 225 instrprof_error::unable_to_correlate_profile, 226 "unsupported binary format (only ELF and COFF are supported)"); 227 } 228 229 template <class IntPtrT> 230 Error InstrProfCorrelatorImpl<IntPtrT>::correlateProfileData(int MaxWarnings) { 231 assert(Data.empty() && Names.empty() && NamesVec.empty()); 232 correlateProfileDataImpl(MaxWarnings); 233 if (this->Data.empty()) 234 return make_error<InstrProfError>( 235 instrprof_error::unable_to_correlate_profile, 236 "could not find any profile data metadata in correlated file"); 237 Error Result = correlateProfileNameImpl(); 238 this->CounterOffsets.clear(); 239 this->NamesVec.clear(); 240 return Result; 241 } 242 243 template <> struct yaml::MappingTraits<InstrProfCorrelator::CorrelationData> { 244 static void mapping(yaml::IO &io, 245 InstrProfCorrelator::CorrelationData &Data) { 246 io.mapRequired("Probes", Data.Probes); 247 } 248 }; 249 250 template <> struct yaml::MappingTraits<InstrProfCorrelator::Probe> { 251 static void mapping(yaml::IO &io, InstrProfCorrelator::Probe &P) { 252 io.mapRequired("Function Name", P.FunctionName); 253 io.mapOptional("Linkage Name", P.LinkageName); 254 io.mapRequired("CFG Hash", P.CFGHash); 255 io.mapRequired("Counter Offset", P.CounterOffset); 256 io.mapRequired("Num Counters", P.NumCounters); 257 io.mapOptional("File", P.FilePath); 258 io.mapOptional("Line", P.LineNumber); 259 } 260 }; 261 262 template <> struct yaml::SequenceElementTraits<InstrProfCorrelator::Probe> { 263 static const bool flow = false; 264 }; 265 266 template <class IntPtrT> 267 Error InstrProfCorrelatorImpl<IntPtrT>::dumpYaml(int MaxWarnings, 268 raw_ostream &OS) { 269 InstrProfCorrelator::CorrelationData Data; 270 correlateProfileDataImpl(MaxWarnings, &Data); 271 if (Data.Probes.empty()) 272 return make_error<InstrProfError>( 273 instrprof_error::unable_to_correlate_profile, 274 "could not find any profile data metadata in debug info"); 275 yaml::Output YamlOS(OS); 276 YamlOS << Data; 277 return Error::success(); 278 } 279 280 template <class IntPtrT> 281 void InstrProfCorrelatorImpl<IntPtrT>::addDataProbe(uint64_t NameRef, 282 uint64_t CFGHash, 283 IntPtrT CounterOffset, 284 IntPtrT FunctionPtr, 285 uint32_t NumCounters) { 286 // Check if a probe was already added for this counter offset. 287 if (!CounterOffsets.insert(CounterOffset).second) 288 return; 289 Data.push_back({ 290 maybeSwap<uint64_t>(NameRef), 291 maybeSwap<uint64_t>(CFGHash), 292 // In this mode, CounterPtr actually stores the section relative address 293 // of the counter. 294 maybeSwap<IntPtrT>(CounterOffset), 295 // TODO: MC/DC is not yet supported. 296 /*BitmapOffset=*/maybeSwap<IntPtrT>(0), 297 maybeSwap<IntPtrT>(FunctionPtr), 298 // TODO: Value profiling is not yet supported. 299 /*ValuesPtr=*/maybeSwap<IntPtrT>(0), 300 maybeSwap<uint32_t>(NumCounters), 301 /*NumValueSites=*/{maybeSwap<uint16_t>(0), maybeSwap<uint16_t>(0)}, 302 // TODO: MC/DC is not yet supported. 303 /*NumBitmapBytes=*/maybeSwap<uint32_t>(0), 304 }); 305 } 306 307 template <class IntPtrT> 308 std::optional<uint64_t> 309 DwarfInstrProfCorrelator<IntPtrT>::getLocation(const DWARFDie &Die) const { 310 auto Locations = Die.getLocations(dwarf::DW_AT_location); 311 if (!Locations) { 312 consumeError(Locations.takeError()); 313 return {}; 314 } 315 auto &DU = *Die.getDwarfUnit(); 316 auto AddressSize = DU.getAddressByteSize(); 317 for (auto &Location : *Locations) { 318 DataExtractor Data(Location.Expr, DICtx->isLittleEndian(), AddressSize); 319 DWARFExpression Expr(Data, AddressSize); 320 for (auto &Op : Expr) { 321 if (Op.getCode() == dwarf::DW_OP_addr) { 322 return Op.getRawOperand(0); 323 } else if (Op.getCode() == dwarf::DW_OP_addrx) { 324 uint64_t Index = Op.getRawOperand(0); 325 if (auto SA = DU.getAddrOffsetSectionItem(Index)) 326 return SA->Address; 327 } 328 } 329 } 330 return {}; 331 } 332 333 template <class IntPtrT> 334 bool DwarfInstrProfCorrelator<IntPtrT>::isDIEOfProbe(const DWARFDie &Die) { 335 const auto &ParentDie = Die.getParent(); 336 if (!Die.isValid() || !ParentDie.isValid() || Die.isNULL()) 337 return false; 338 if (Die.getTag() != dwarf::DW_TAG_variable) 339 return false; 340 if (!ParentDie.isSubprogramDIE()) 341 return false; 342 if (!Die.hasChildren()) 343 return false; 344 if (const char *Name = Die.getName(DINameKind::ShortName)) 345 return StringRef(Name).starts_with(getInstrProfCountersVarPrefix()); 346 return false; 347 } 348 349 template <class IntPtrT> 350 void DwarfInstrProfCorrelator<IntPtrT>::correlateProfileDataImpl( 351 int MaxWarnings, InstrProfCorrelator::CorrelationData *Data) { 352 bool UnlimitedWarnings = (MaxWarnings == 0); 353 // -N suppressed warnings means we can emit up to N (unsuppressed) warnings 354 int NumSuppressedWarnings = -MaxWarnings; 355 auto maybeAddProbe = [&](DWARFDie Die) { 356 if (!isDIEOfProbe(Die)) 357 return; 358 std::optional<const char *> FunctionName; 359 std::optional<uint64_t> CFGHash; 360 std::optional<uint64_t> CounterPtr = getLocation(Die); 361 auto FnDie = Die.getParent(); 362 auto FunctionPtr = dwarf::toAddress(FnDie.find(dwarf::DW_AT_low_pc)); 363 std::optional<uint64_t> NumCounters; 364 for (const DWARFDie &Child : Die.children()) { 365 if (Child.getTag() != dwarf::DW_TAG_LLVM_annotation) 366 continue; 367 auto AnnotationFormName = Child.find(dwarf::DW_AT_name); 368 auto AnnotationFormValue = Child.find(dwarf::DW_AT_const_value); 369 if (!AnnotationFormName || !AnnotationFormValue) 370 continue; 371 auto AnnotationNameOrErr = AnnotationFormName->getAsCString(); 372 if (auto Err = AnnotationNameOrErr.takeError()) { 373 consumeError(std::move(Err)); 374 continue; 375 } 376 StringRef AnnotationName = *AnnotationNameOrErr; 377 if (AnnotationName == InstrProfCorrelator::FunctionNameAttributeName) { 378 if (auto EC = 379 AnnotationFormValue->getAsCString().moveInto(FunctionName)) 380 consumeError(std::move(EC)); 381 } else if (AnnotationName == InstrProfCorrelator::CFGHashAttributeName) { 382 CFGHash = AnnotationFormValue->getAsUnsignedConstant(); 383 } else if (AnnotationName == 384 InstrProfCorrelator::NumCountersAttributeName) { 385 NumCounters = AnnotationFormValue->getAsUnsignedConstant(); 386 } 387 } 388 if (!FunctionName || !CFGHash || !CounterPtr || !NumCounters) { 389 if (UnlimitedWarnings || ++NumSuppressedWarnings < 1) { 390 WithColor::warning() 391 << "Incomplete DIE for function " << FunctionName 392 << ": CFGHash=" << CFGHash << " CounterPtr=" << CounterPtr 393 << " NumCounters=" << NumCounters << "\n"; 394 LLVM_DEBUG(Die.dump(dbgs())); 395 } 396 return; 397 } 398 uint64_t CountersStart = this->Ctx->CountersSectionStart; 399 uint64_t CountersEnd = this->Ctx->CountersSectionEnd; 400 if (*CounterPtr < CountersStart || *CounterPtr >= CountersEnd) { 401 if (UnlimitedWarnings || ++NumSuppressedWarnings < 1) { 402 WithColor::warning() 403 << format("CounterPtr out of range for function %s: Actual=0x%x " 404 "Expected=[0x%x, 0x%x)\n", 405 *FunctionName, *CounterPtr, CountersStart, CountersEnd); 406 LLVM_DEBUG(Die.dump(dbgs())); 407 } 408 return; 409 } 410 if (!FunctionPtr && (UnlimitedWarnings || ++NumSuppressedWarnings < 1)) { 411 WithColor::warning() << format("Could not find address of function %s\n", 412 *FunctionName); 413 LLVM_DEBUG(Die.dump(dbgs())); 414 } 415 // In debug info correlation mode, the CounterPtr is an absolute address of 416 // the counter, but it's expected to be relative later when iterating Data. 417 IntPtrT CounterOffset = *CounterPtr - CountersStart; 418 if (Data) { 419 InstrProfCorrelator::Probe P; 420 P.FunctionName = *FunctionName; 421 if (auto Name = FnDie.getName(DINameKind::LinkageName)) 422 P.LinkageName = Name; 423 P.CFGHash = *CFGHash; 424 P.CounterOffset = CounterOffset; 425 P.NumCounters = *NumCounters; 426 auto FilePath = FnDie.getDeclFile( 427 DILineInfoSpecifier::FileLineInfoKind::RelativeFilePath); 428 if (!FilePath.empty()) 429 P.FilePath = FilePath; 430 if (auto LineNumber = FnDie.getDeclLine()) 431 P.LineNumber = LineNumber; 432 Data->Probes.push_back(P); 433 } else { 434 this->addDataProbe(IndexedInstrProf::ComputeHash(*FunctionName), *CFGHash, 435 CounterOffset, FunctionPtr.value_or(0), *NumCounters); 436 this->NamesVec.push_back(*FunctionName); 437 } 438 }; 439 for (auto &CU : DICtx->normal_units()) 440 for (const auto &Entry : CU->dies()) 441 maybeAddProbe(DWARFDie(CU.get(), &Entry)); 442 for (auto &CU : DICtx->dwo_units()) 443 for (const auto &Entry : CU->dies()) 444 maybeAddProbe(DWARFDie(CU.get(), &Entry)); 445 446 if (!UnlimitedWarnings && NumSuppressedWarnings > 0) 447 WithColor::warning() << format("Suppressed %d additional warnings\n", 448 NumSuppressedWarnings); 449 } 450 451 template <class IntPtrT> 452 Error DwarfInstrProfCorrelator<IntPtrT>::correlateProfileNameImpl() { 453 if (this->NamesVec.empty()) { 454 return make_error<InstrProfError>( 455 instrprof_error::unable_to_correlate_profile, 456 "could not find any profile name metadata in debug info"); 457 } 458 auto Result = 459 collectGlobalObjectNameStrings(this->NamesVec, 460 /*doCompression=*/false, this->Names); 461 return Result; 462 } 463 464 template <class IntPtrT> 465 void BinaryInstrProfCorrelator<IntPtrT>::correlateProfileDataImpl( 466 int MaxWarnings, InstrProfCorrelator::CorrelationData *CorrelateData) { 467 using RawProfData = RawInstrProf::ProfileData<IntPtrT>; 468 bool UnlimitedWarnings = (MaxWarnings == 0); 469 // -N suppressed warnings means we can emit up to N (unsuppressed) warnings 470 int NumSuppressedWarnings = -MaxWarnings; 471 472 const RawProfData *DataStart = (const RawProfData *)this->Ctx->DataStart; 473 const RawProfData *DataEnd = (const RawProfData *)this->Ctx->DataEnd; 474 // We need to use < here because the last data record may have no padding. 475 for (const RawProfData *I = DataStart; I < DataEnd; ++I) { 476 uint64_t CounterPtr = this->template maybeSwap<IntPtrT>(I->CounterPtr); 477 uint64_t CountersStart = this->Ctx->CountersSectionStart; 478 uint64_t CountersEnd = this->Ctx->CountersSectionEnd; 479 if (CounterPtr < CountersStart || CounterPtr >= CountersEnd) { 480 if (UnlimitedWarnings || ++NumSuppressedWarnings < 1) { 481 WithColor::warning() 482 << format("CounterPtr out of range for function: Actual=0x%x " 483 "Expected=[0x%x, 0x%x) at data offset=0x%x\n", 484 CounterPtr, CountersStart, CountersEnd, 485 (I - DataStart) * sizeof(RawProfData)); 486 } 487 } 488 // In binary correlation mode, the CounterPtr is an absolute address of the 489 // counter, but it's expected to be relative later when iterating Data. 490 IntPtrT CounterOffset = CounterPtr - CountersStart; 491 this->addDataProbe(I->NameRef, I->FuncHash, CounterOffset, 492 I->FunctionPointer, I->NumCounters); 493 } 494 } 495 496 template <class IntPtrT> 497 Error BinaryInstrProfCorrelator<IntPtrT>::correlateProfileNameImpl() { 498 if (this->Ctx->NameSize == 0) { 499 return make_error<InstrProfError>( 500 instrprof_error::unable_to_correlate_profile, 501 "could not find any profile data metadata in object file"); 502 } 503 this->Names.append(this->Ctx->NameStart, this->Ctx->NameSize); 504 return Error::success(); 505 } 506