1 //===-- InstrProfCorrelator.cpp -------------------------------------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 9 #include "llvm/ProfileData/InstrProfCorrelator.h" 10 #include "llvm/Object/MachO.h" 11 #include "llvm/Support/Debug.h" 12 #include "llvm/Support/FileSystem.h" 13 #include "llvm/Support/Path.h" 14 15 #define DEBUG_TYPE "correlator" 16 17 using namespace llvm; 18 19 /// Get the __llvm_prf_cnts section. 20 Expected<object::SectionRef> getCountersSection(const object::ObjectFile &Obj) { 21 for (auto &Section : Obj.sections()) 22 if (auto SectionName = Section.getName()) 23 if (SectionName.get() == INSTR_PROF_CNTS_SECT_NAME) 24 return Section; 25 return make_error<InstrProfError>( 26 instrprof_error::unable_to_correlate_profile); 27 } 28 29 const char *InstrProfCorrelator::FunctionNameAttributeName = "Function Name"; 30 const char *InstrProfCorrelator::CFGHashAttributeName = "CFG Hash"; 31 const char *InstrProfCorrelator::NumCountersAttributeName = "Num Counters"; 32 33 llvm::Expected<std::unique_ptr<InstrProfCorrelator::Context>> 34 InstrProfCorrelator::Context::get(std::unique_ptr<MemoryBuffer> Buffer, 35 const object::ObjectFile &Obj) { 36 auto CountersSection = getCountersSection(Obj); 37 if (auto Err = CountersSection.takeError()) 38 return std::move(Err); 39 auto C = std::make_unique<Context>(); 40 C->Buffer = std::move(Buffer); 41 C->CountersSectionStart = CountersSection->getAddress(); 42 C->CountersSectionEnd = C->CountersSectionStart + CountersSection->getSize(); 43 C->ShouldSwapBytes = Obj.isLittleEndian() != sys::IsLittleEndianHost; 44 return Expected<std::unique_ptr<Context>>(std::move(C)); 45 } 46 47 llvm::Expected<std::unique_ptr<InstrProfCorrelator>> 48 InstrProfCorrelator::get(StringRef DebugInfoFilename) { 49 auto DsymObjectsOrErr = 50 object::MachOObjectFile::findDsymObjectMembers(DebugInfoFilename); 51 if (auto Err = DsymObjectsOrErr.takeError()) 52 return std::move(Err); 53 if (!DsymObjectsOrErr->empty()) { 54 // TODO: Enable profile correlation when there are multiple objects in a 55 // dSYM bundle. 56 if (DsymObjectsOrErr->size() > 1) 57 return createStringError( 58 std::error_code(), 59 "Profile correlation using multiple objects is not yet supported"); 60 DebugInfoFilename = *DsymObjectsOrErr->begin(); 61 } 62 auto BufferOrErr = 63 errorOrToExpected(MemoryBuffer::getFile(DebugInfoFilename)); 64 if (auto Err = BufferOrErr.takeError()) 65 return std::move(Err); 66 67 return get(std::move(*BufferOrErr)); 68 } 69 70 llvm::Expected<std::unique_ptr<InstrProfCorrelator>> 71 InstrProfCorrelator::get(std::unique_ptr<MemoryBuffer> Buffer) { 72 auto BinOrErr = object::createBinary(*Buffer); 73 if (auto Err = BinOrErr.takeError()) 74 return std::move(Err); 75 76 if (auto *Obj = dyn_cast<object::ObjectFile>(BinOrErr->get())) { 77 auto CtxOrErr = Context::get(std::move(Buffer), *Obj); 78 if (auto Err = CtxOrErr.takeError()) 79 return std::move(Err); 80 auto T = Obj->makeTriple(); 81 if (T.isArch64Bit()) 82 return InstrProfCorrelatorImpl<uint64_t>::get(std::move(*CtxOrErr), *Obj); 83 if (T.isArch32Bit()) 84 return InstrProfCorrelatorImpl<uint32_t>::get(std::move(*CtxOrErr), *Obj); 85 } 86 return make_error<InstrProfError>( 87 instrprof_error::unable_to_correlate_profile); 88 } 89 90 namespace llvm { 91 92 template <> 93 InstrProfCorrelatorImpl<uint32_t>::InstrProfCorrelatorImpl( 94 std::unique_ptr<InstrProfCorrelator::Context> Ctx) 95 : InstrProfCorrelatorImpl(InstrProfCorrelatorKind::CK_32Bit, 96 std::move(Ctx)) {} 97 template <> 98 InstrProfCorrelatorImpl<uint64_t>::InstrProfCorrelatorImpl( 99 std::unique_ptr<InstrProfCorrelator::Context> Ctx) 100 : InstrProfCorrelatorImpl(InstrProfCorrelatorKind::CK_64Bit, 101 std::move(Ctx)) {} 102 template <> 103 bool InstrProfCorrelatorImpl<uint32_t>::classof(const InstrProfCorrelator *C) { 104 return C->getKind() == InstrProfCorrelatorKind::CK_32Bit; 105 } 106 template <> 107 bool InstrProfCorrelatorImpl<uint64_t>::classof(const InstrProfCorrelator *C) { 108 return C->getKind() == InstrProfCorrelatorKind::CK_64Bit; 109 } 110 111 } // end namespace llvm 112 113 template <class IntPtrT> 114 llvm::Expected<std::unique_ptr<InstrProfCorrelatorImpl<IntPtrT>>> 115 InstrProfCorrelatorImpl<IntPtrT>::get( 116 std::unique_ptr<InstrProfCorrelator::Context> Ctx, 117 const object::ObjectFile &Obj) { 118 if (Obj.isELF() || Obj.isMachO()) { 119 auto DICtx = DWARFContext::create(Obj); 120 return std::make_unique<DwarfInstrProfCorrelator<IntPtrT>>(std::move(DICtx), 121 std::move(Ctx)); 122 } 123 return make_error<InstrProfError>(instrprof_error::unsupported_debug_format); 124 } 125 126 template <class IntPtrT> 127 Error InstrProfCorrelatorImpl<IntPtrT>::correlateProfileData() { 128 assert(Data.empty() && CompressedNames.empty() && Names.empty()); 129 correlateProfileDataImpl(); 130 auto Result = 131 collectPGOFuncNameStrings(Names, /*doCompression=*/true, CompressedNames); 132 Names.clear(); 133 return Result; 134 } 135 136 template <class IntPtrT> 137 void InstrProfCorrelatorImpl<IntPtrT>::addProbe(StringRef FunctionName, 138 uint64_t CFGHash, 139 IntPtrT CounterOffset, 140 IntPtrT FunctionPtr, 141 uint32_t NumCounters) { 142 Data.push_back({ 143 maybeSwap<uint64_t>(IndexedInstrProf::ComputeHash(FunctionName)), 144 maybeSwap<uint64_t>(CFGHash), 145 // In this mode, CounterPtr actually stores the section relative address 146 // of the counter. 147 maybeSwap<IntPtrT>(CounterOffset), 148 maybeSwap<IntPtrT>(FunctionPtr), 149 // TODO: Value profiling is not yet supported. 150 /*ValuesPtr=*/maybeSwap<IntPtrT>(0), 151 maybeSwap<uint32_t>(NumCounters), 152 /*NumValueSites=*/{maybeSwap<uint16_t>(0), maybeSwap<uint16_t>(0)}, 153 }); 154 Names.push_back(FunctionName.str()); 155 } 156 157 template <class IntPtrT> 158 llvm::Optional<uint64_t> 159 DwarfInstrProfCorrelator<IntPtrT>::getLocation(const DWARFDie &Die) const { 160 auto Locations = Die.getLocations(dwarf::DW_AT_location); 161 if (!Locations) { 162 consumeError(Locations.takeError()); 163 return {}; 164 } 165 auto &DU = *Die.getDwarfUnit(); 166 for (auto &Location : *Locations) { 167 auto AddressSize = DU.getAddressByteSize(); 168 DataExtractor Data(Location.Expr, DICtx->isLittleEndian(), AddressSize); 169 DWARFExpression Expr(Data, AddressSize); 170 for (auto &Op : Expr) 171 if (Op.getCode() == dwarf::DW_OP_addr) 172 return Op.getRawOperand(0); 173 } 174 return {}; 175 } 176 177 template <class IntPtrT> 178 bool DwarfInstrProfCorrelator<IntPtrT>::isDIEOfProbe(const DWARFDie &Die) { 179 const auto &ParentDie = Die.getParent(); 180 if (!Die.isValid() || !ParentDie.isValid() || Die.isNULL()) 181 return false; 182 if (Die.getTag() != dwarf::DW_TAG_variable) 183 return false; 184 if (!ParentDie.isSubprogramDIE()) 185 return false; 186 if (!Die.hasChildren()) 187 return false; 188 if (const char *Name = Die.getName(DINameKind::ShortName)) 189 return StringRef(Name).startswith(getInstrProfCountersVarPrefix()); 190 return false; 191 } 192 193 template <class IntPtrT> 194 void DwarfInstrProfCorrelator<IntPtrT>::correlateProfileDataImpl() { 195 auto maybeAddProbe = [&](DWARFDie Die) { 196 if (!isDIEOfProbe(Die)) 197 return; 198 Optional<const char *> FunctionName; 199 Optional<uint64_t> CFGHash; 200 Optional<uint64_t> CounterPtr = getLocation(Die); 201 auto FunctionPtr = 202 dwarf::toAddress(Die.getParent().find(dwarf::DW_AT_low_pc)); 203 Optional<uint64_t> NumCounters; 204 for (const DWARFDie &Child : Die.children()) { 205 if (Child.getTag() != dwarf::DW_TAG_LLVM_annotation) 206 continue; 207 auto AnnotationFormName = Child.find(dwarf::DW_AT_name); 208 auto AnnotationFormValue = Child.find(dwarf::DW_AT_const_value); 209 if (!AnnotationFormName || !AnnotationFormValue) 210 continue; 211 auto AnnotationNameOrErr = AnnotationFormName->getAsCString(); 212 if (auto Err = AnnotationNameOrErr.takeError()) { 213 consumeError(std::move(Err)); 214 continue; 215 } 216 StringRef AnnotationName = *AnnotationNameOrErr; 217 if (AnnotationName.compare( 218 InstrProfCorrelator::FunctionNameAttributeName) == 0) { 219 if (auto EC = 220 AnnotationFormValue->getAsCString().moveInto(FunctionName)) 221 consumeError(std::move(EC)); 222 } else if (AnnotationName.compare( 223 InstrProfCorrelator::CFGHashAttributeName) == 0) { 224 CFGHash = AnnotationFormValue->getAsUnsignedConstant(); 225 } else if (AnnotationName.compare( 226 InstrProfCorrelator::NumCountersAttributeName) == 0) { 227 NumCounters = AnnotationFormValue->getAsUnsignedConstant(); 228 } 229 } 230 if (!FunctionName || !CFGHash || !CounterPtr || !NumCounters) { 231 LLVM_DEBUG(dbgs() << "Incomplete DIE for probe\n\tFunctionName: " 232 << FunctionName << "\n\tCFGHash: " << CFGHash 233 << "\n\tCounterPtr: " << CounterPtr 234 << "\n\tNumCounters: " << NumCounters); 235 LLVM_DEBUG(Die.dump(dbgs())); 236 return; 237 } 238 uint64_t CountersStart = this->Ctx->CountersSectionStart; 239 uint64_t CountersEnd = this->Ctx->CountersSectionEnd; 240 if (*CounterPtr < CountersStart || *CounterPtr >= CountersEnd) { 241 LLVM_DEBUG( 242 dbgs() << "CounterPtr out of range for probe\n\tFunction Name: " 243 << FunctionName << "\n\tExpected: [0x" 244 << Twine::utohexstr(CountersStart) << ", 0x" 245 << Twine::utohexstr(CountersEnd) << ")\n\tActual: 0x" 246 << Twine::utohexstr(*CounterPtr)); 247 LLVM_DEBUG(Die.dump(dbgs())); 248 return; 249 } 250 if (!FunctionPtr) { 251 LLVM_DEBUG(dbgs() << "Could not find address of " << *FunctionName 252 << "\n"); 253 LLVM_DEBUG(Die.dump(dbgs())); 254 } 255 this->addProbe(*FunctionName, *CFGHash, *CounterPtr - CountersStart, 256 FunctionPtr.getValueOr(0), *NumCounters); 257 }; 258 for (auto &CU : DICtx->normal_units()) 259 for (const auto &Entry : CU->dies()) 260 maybeAddProbe(DWARFDie(CU.get(), &Entry)); 261 for (auto &CU : DICtx->dwo_units()) 262 for (const auto &Entry : CU->dies()) 263 maybeAddProbe(DWARFDie(CU.get(), &Entry)); 264 } 265