1 //===- bolt/Profile/YAMLProfileWriter.cpp - YAML profile serializer -------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 9 #include "bolt/Profile/YAMLProfileWriter.h" 10 #include "bolt/Core/BinaryBasicBlock.h" 11 #include "bolt/Core/BinaryFunction.h" 12 #include "bolt/Profile/BoltAddressTranslation.h" 13 #include "bolt/Profile/DataAggregator.h" 14 #include "bolt/Profile/ProfileReaderBase.h" 15 #include "bolt/Rewrite/RewriteInstance.h" 16 #include "llvm/Support/CommandLine.h" 17 #include "llvm/Support/FileSystem.h" 18 #include "llvm/Support/raw_ostream.h" 19 20 #undef DEBUG_TYPE 21 #define DEBUG_TYPE "bolt-prof" 22 23 namespace opts { 24 extern llvm::cl::opt<bool> ProfileUseDFS; 25 extern llvm::cl::opt<bool> ProfileUsePseudoProbes; 26 } // namespace opts 27 28 namespace llvm { 29 namespace bolt { 30 31 const BinaryFunction *YAMLProfileWriter::setCSIDestination( 32 const BinaryContext &BC, yaml::bolt::CallSiteInfo &CSI, 33 const MCSymbol *Symbol, const BoltAddressTranslation *BAT, 34 uint32_t Offset) { 35 CSI.DestId = 0; // designated for unknown functions 36 CSI.EntryDiscriminator = 0; 37 38 if (Symbol) { 39 uint64_t EntryID = 0; 40 if (const BinaryFunction *Callee = 41 BC.getFunctionForSymbol(Symbol, &EntryID)) { 42 if (BAT && BAT->isBATFunction(Callee->getAddress())) 43 std::tie(Callee, EntryID) = BAT->translateSymbol(BC, *Symbol, Offset); 44 else if (const BinaryBasicBlock *BB = 45 Callee->getBasicBlockContainingOffset(Offset)) 46 BC.getFunctionForSymbol(Callee->getSecondaryEntryPointSymbol(*BB), 47 &EntryID); 48 CSI.DestId = Callee->getFunctionNumber(); 49 CSI.EntryDiscriminator = EntryID; 50 return Callee; 51 } 52 } 53 return nullptr; 54 } 55 56 yaml::bolt::BinaryFunctionProfile 57 YAMLProfileWriter::convert(const BinaryFunction &BF, bool UseDFS, 58 const BoltAddressTranslation *BAT) { 59 yaml::bolt::BinaryFunctionProfile YamlBF; 60 const BinaryContext &BC = BF.getBinaryContext(); 61 const MCPseudoProbeDecoder *PseudoProbeDecoder = 62 opts::ProfileUsePseudoProbes ? BC.getPseudoProbeDecoder() : nullptr; 63 64 const uint16_t LBRProfile = BF.getProfileFlags() & BinaryFunction::PF_LBR; 65 66 // Prepare function and block hashes 67 BF.computeHash(UseDFS); 68 BF.computeBlockHashes(); 69 70 YamlBF.Name = DataAggregator::getLocationName(BF, BAT); 71 YamlBF.Id = BF.getFunctionNumber(); 72 YamlBF.Hash = BF.getHash(); 73 YamlBF.NumBasicBlocks = BF.size(); 74 YamlBF.ExecCount = BF.getKnownExecutionCount(); 75 if (PseudoProbeDecoder) { 76 if ((YamlBF.GUID = BF.getGUID())) { 77 const MCPseudoProbeFuncDesc *FuncDesc = 78 PseudoProbeDecoder->getFuncDescForGUID(YamlBF.GUID); 79 YamlBF.PseudoProbeDescHash = FuncDesc->FuncHash; 80 } 81 } 82 83 BinaryFunction::BasicBlockOrderType Order; 84 llvm::copy(UseDFS ? BF.dfs() : BF.getLayout().blocks(), 85 std::back_inserter(Order)); 86 87 const FunctionLayout Layout = BF.getLayout(); 88 Layout.updateLayoutIndices(Order); 89 90 for (const BinaryBasicBlock *BB : Order) { 91 yaml::bolt::BinaryBasicBlockProfile YamlBB; 92 YamlBB.Index = BB->getLayoutIndex(); 93 YamlBB.NumInstructions = BB->getNumNonPseudos(); 94 YamlBB.Hash = BB->getHash(); 95 96 if (!LBRProfile) { 97 YamlBB.EventCount = BB->getKnownExecutionCount(); 98 if (YamlBB.EventCount) 99 YamlBF.Blocks.emplace_back(YamlBB); 100 continue; 101 } 102 103 YamlBB.ExecCount = BB->getKnownExecutionCount(); 104 105 for (const MCInst &Instr : *BB) { 106 if (!BC.MIB->isCall(Instr) && !BC.MIB->isIndirectBranch(Instr)) 107 continue; 108 109 SmallVector<std::pair<StringRef, yaml::bolt::CallSiteInfo>> CSTargets; 110 yaml::bolt::CallSiteInfo CSI; 111 std::optional<uint32_t> Offset = BC.MIB->getOffset(Instr); 112 if (!Offset || *Offset < BB->getInputOffset()) 113 continue; 114 CSI.Offset = *Offset - BB->getInputOffset(); 115 116 if (BC.MIB->isIndirectCall(Instr) || BC.MIB->isIndirectBranch(Instr)) { 117 const auto ICSP = BC.MIB->tryGetAnnotationAs<IndirectCallSiteProfile>( 118 Instr, "CallProfile"); 119 if (!ICSP) 120 continue; 121 for (const IndirectCallProfile &CSP : ICSP.get()) { 122 StringRef TargetName = ""; 123 const BinaryFunction *Callee = 124 setCSIDestination(BC, CSI, CSP.Symbol, BAT); 125 if (Callee) 126 TargetName = Callee->getOneName(); 127 CSI.Count = CSP.Count; 128 CSI.Mispreds = CSP.Mispreds; 129 CSTargets.emplace_back(TargetName, CSI); 130 } 131 } else { // direct call or a tail call 132 StringRef TargetName = ""; 133 const MCSymbol *CalleeSymbol = BC.MIB->getTargetSymbol(Instr); 134 const BinaryFunction *const Callee = 135 setCSIDestination(BC, CSI, CalleeSymbol, BAT); 136 if (Callee) 137 TargetName = Callee->getOneName(); 138 139 auto getAnnotationWithDefault = [&](const MCInst &Inst, StringRef Ann) { 140 return BC.MIB->getAnnotationWithDefault(Instr, Ann, 0ull); 141 }; 142 if (BC.MIB->getConditionalTailCall(Instr)) { 143 CSI.Count = getAnnotationWithDefault(Instr, "CTCTakenCount"); 144 CSI.Mispreds = getAnnotationWithDefault(Instr, "CTCMispredCount"); 145 } else { 146 CSI.Count = getAnnotationWithDefault(Instr, "Count"); 147 } 148 149 if (CSI.Count) 150 CSTargets.emplace_back(TargetName, CSI); 151 } 152 // Sort targets in a similar way to getBranchData, see Location::operator< 153 llvm::sort(CSTargets, [](const auto &RHS, const auto &LHS) { 154 if (RHS.first != LHS.first) 155 return RHS.first < LHS.first; 156 return RHS.second.Offset < LHS.second.Offset; 157 }); 158 for (auto &KV : CSTargets) 159 YamlBB.CallSites.push_back(KV.second); 160 } 161 162 // Skip printing if there's no profile data for non-entry basic block. 163 // Include landing pads with non-zero execution count. 164 if (YamlBB.CallSites.empty() && !BB->isEntryPoint() && 165 !(BB->isLandingPad() && BB->getKnownExecutionCount() != 0)) { 166 // Include blocks having successors or predecessors with positive counts. 167 uint64_t SuccessorExecCount = 0; 168 for (const BinaryBasicBlock::BinaryBranchInfo &BranchInfo : 169 BB->branch_info()) 170 SuccessorExecCount += BranchInfo.Count; 171 uint64_t PredecessorExecCount = 0; 172 for (auto Pred : BB->predecessors()) 173 PredecessorExecCount += Pred->getBranchInfo(*BB).Count; 174 if (!SuccessorExecCount && !PredecessorExecCount) 175 continue; 176 } 177 178 auto BranchInfo = BB->branch_info_begin(); 179 for (const BinaryBasicBlock *Successor : BB->successors()) { 180 yaml::bolt::SuccessorInfo YamlSI; 181 YamlSI.Index = Successor->getLayoutIndex(); 182 YamlSI.Count = BranchInfo->Count; 183 YamlSI.Mispreds = BranchInfo->MispredictedCount; 184 185 YamlBB.Successors.emplace_back(YamlSI); 186 187 ++BranchInfo; 188 } 189 190 if (PseudoProbeDecoder) { 191 const AddressProbesMap &ProbeMap = 192 PseudoProbeDecoder->getAddress2ProbesMap(); 193 const uint64_t FuncAddr = BF.getAddress(); 194 const std::pair<uint64_t, uint64_t> &BlockRange = 195 BB->getInputAddressRange(); 196 const auto &BlockProbes = 197 llvm::make_range(ProbeMap.lower_bound(FuncAddr + BlockRange.first), 198 ProbeMap.lower_bound(FuncAddr + BlockRange.second)); 199 for (const auto &[_, Probes] : BlockProbes) 200 for (const MCDecodedPseudoProbe &Probe : Probes) 201 YamlBB.PseudoProbes.emplace_back(yaml::bolt::PseudoProbeInfo{ 202 Probe.getGuid(), Probe.getIndex(), Probe.getType()}); 203 } 204 205 YamlBF.Blocks.emplace_back(YamlBB); 206 } 207 return YamlBF; 208 } 209 210 std::error_code YAMLProfileWriter::writeProfile(const RewriteInstance &RI) { 211 const BinaryContext &BC = RI.getBinaryContext(); 212 const auto &Functions = BC.getBinaryFunctions(); 213 214 std::error_code EC; 215 OS = std::make_unique<raw_fd_ostream>(Filename, EC, sys::fs::OF_None); 216 if (EC) { 217 errs() << "BOLT-WARNING: " << EC.message() << " : unable to open " 218 << Filename << " for output.\n"; 219 return EC; 220 } 221 222 yaml::bolt::BinaryProfile BP; 223 224 // Fill out the header info. 225 BP.Header.Version = 1; 226 BP.Header.FileName = std::string(BC.getFilename()); 227 std::optional<StringRef> BuildID = BC.getFileBuildID(); 228 BP.Header.Id = BuildID ? std::string(*BuildID) : "<unknown>"; 229 BP.Header.Origin = std::string(RI.getProfileReader()->getReaderName()); 230 BP.Header.IsDFSOrder = opts::ProfileUseDFS; 231 BP.Header.HashFunction = HashFunction::Default; 232 233 StringSet<> EventNames = RI.getProfileReader()->getEventNames(); 234 if (!EventNames.empty()) { 235 std::string Sep; 236 for (const StringMapEntry<std::nullopt_t> &EventEntry : EventNames) { 237 BP.Header.EventNames += Sep + EventEntry.first().str(); 238 Sep = ","; 239 } 240 } 241 242 // Make sure the profile is consistent across all functions. 243 uint16_t ProfileFlags = BinaryFunction::PF_NONE; 244 for (const auto &BFI : Functions) { 245 const BinaryFunction &BF = BFI.second; 246 if (BF.hasProfile() && !BF.empty()) { 247 assert(BF.getProfileFlags() != BinaryFunction::PF_NONE); 248 if (ProfileFlags == BinaryFunction::PF_NONE) 249 ProfileFlags = BF.getProfileFlags(); 250 251 assert(BF.getProfileFlags() == ProfileFlags && 252 "expected consistent profile flags across all functions"); 253 } 254 } 255 BP.Header.Flags = ProfileFlags; 256 257 // Add all function objects. 258 for (const auto &BFI : Functions) { 259 const BinaryFunction &BF = BFI.second; 260 if (BF.hasProfile()) { 261 if (!BF.hasValidProfile() && !RI.getProfileReader()->isTrustedSource()) 262 continue; 263 264 BP.Functions.emplace_back(convert(BF, opts::ProfileUseDFS)); 265 } 266 } 267 268 // Write the profile. 269 yaml::Output Out(*OS, nullptr, 0); 270 Out << BP; 271 272 return std::error_code(); 273 } 274 275 } // namespace bolt 276 } // namespace llvm 277