1 //===- bolt/Profile/YAMLProfileWriter.cpp - YAML profile serializer -------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 9 #include "bolt/Profile/YAMLProfileWriter.h" 10 #include "bolt/Core/BinaryBasicBlock.h" 11 #include "bolt/Core/BinaryFunction.h" 12 #include "bolt/Profile/BoltAddressTranslation.h" 13 #include "bolt/Profile/DataAggregator.h" 14 #include "bolt/Profile/ProfileReaderBase.h" 15 #include "bolt/Rewrite/RewriteInstance.h" 16 #include "bolt/Utils/CommandLineOpts.h" 17 #include "llvm/Support/CommandLine.h" 18 #include "llvm/Support/FileSystem.h" 19 #include "llvm/Support/raw_ostream.h" 20 21 #undef DEBUG_TYPE 22 #define DEBUG_TYPE "bolt-prof" 23 24 namespace opts { 25 using namespace llvm; 26 extern cl::opt<bool> ProfileUseDFS; 27 cl::opt<bool> ProfileWritePseudoProbes( 28 "profile-write-pseudo-probes", 29 cl::desc("Use pseudo probes in profile generation"), cl::Hidden, 30 cl::cat(BoltOptCategory)); 31 } // namespace opts 32 33 namespace llvm { 34 namespace bolt { 35 36 const BinaryFunction *YAMLProfileWriter::setCSIDestination( 37 const BinaryContext &BC, yaml::bolt::CallSiteInfo &CSI, 38 const MCSymbol *Symbol, const BoltAddressTranslation *BAT, 39 uint32_t Offset) { 40 CSI.DestId = 0; // designated for unknown functions 41 CSI.EntryDiscriminator = 0; 42 43 if (Symbol) { 44 uint64_t EntryID = 0; 45 if (const BinaryFunction *Callee = 46 BC.getFunctionForSymbol(Symbol, &EntryID)) { 47 if (BAT && BAT->isBATFunction(Callee->getAddress())) 48 std::tie(Callee, EntryID) = BAT->translateSymbol(BC, *Symbol, Offset); 49 else if (const BinaryBasicBlock *BB = 50 Callee->getBasicBlockContainingOffset(Offset)) 51 BC.getFunctionForSymbol(Callee->getSecondaryEntryPointSymbol(*BB), 52 &EntryID); 53 CSI.DestId = Callee->getFunctionNumber(); 54 CSI.EntryDiscriminator = EntryID; 55 return Callee; 56 } 57 } 58 return nullptr; 59 } 60 61 yaml::bolt::BinaryFunctionProfile 62 YAMLProfileWriter::convert(const BinaryFunction &BF, bool UseDFS, 63 const BoltAddressTranslation *BAT) { 64 yaml::bolt::BinaryFunctionProfile YamlBF; 65 const BinaryContext &BC = BF.getBinaryContext(); 66 const MCPseudoProbeDecoder *PseudoProbeDecoder = 67 opts::ProfileWritePseudoProbes ? BC.getPseudoProbeDecoder() : nullptr; 68 69 const uint16_t LBRProfile = BF.getProfileFlags() & BinaryFunction::PF_LBR; 70 71 // Prepare function and block hashes 72 BF.computeHash(UseDFS); 73 BF.computeBlockHashes(); 74 75 YamlBF.Name = DataAggregator::getLocationName(BF, BAT); 76 YamlBF.Id = BF.getFunctionNumber(); 77 YamlBF.Hash = BF.getHash(); 78 YamlBF.NumBasicBlocks = BF.size(); 79 YamlBF.ExecCount = BF.getKnownExecutionCount(); 80 if (PseudoProbeDecoder) { 81 if ((YamlBF.GUID = BF.getGUID())) { 82 const MCPseudoProbeFuncDesc *FuncDesc = 83 PseudoProbeDecoder->getFuncDescForGUID(YamlBF.GUID); 84 YamlBF.PseudoProbeDescHash = FuncDesc->FuncHash; 85 } 86 } 87 88 BinaryFunction::BasicBlockOrderType Order; 89 llvm::copy(UseDFS ? BF.dfs() : BF.getLayout().blocks(), 90 std::back_inserter(Order)); 91 92 const FunctionLayout Layout = BF.getLayout(); 93 Layout.updateLayoutIndices(Order); 94 95 for (const BinaryBasicBlock *BB : Order) { 96 yaml::bolt::BinaryBasicBlockProfile YamlBB; 97 YamlBB.Index = BB->getLayoutIndex(); 98 YamlBB.NumInstructions = BB->getNumNonPseudos(); 99 YamlBB.Hash = BB->getHash(); 100 101 if (!LBRProfile) { 102 YamlBB.EventCount = BB->getKnownExecutionCount(); 103 if (YamlBB.EventCount) 104 YamlBF.Blocks.emplace_back(YamlBB); 105 continue; 106 } 107 108 YamlBB.ExecCount = BB->getKnownExecutionCount(); 109 110 for (const MCInst &Instr : *BB) { 111 if (!BC.MIB->isCall(Instr) && !BC.MIB->isIndirectBranch(Instr)) 112 continue; 113 114 SmallVector<std::pair<StringRef, yaml::bolt::CallSiteInfo>> CSTargets; 115 yaml::bolt::CallSiteInfo CSI; 116 std::optional<uint32_t> Offset = BC.MIB->getOffset(Instr); 117 if (!Offset || *Offset < BB->getInputOffset()) 118 continue; 119 CSI.Offset = *Offset - BB->getInputOffset(); 120 121 if (BC.MIB->isIndirectCall(Instr) || BC.MIB->isIndirectBranch(Instr)) { 122 const auto ICSP = BC.MIB->tryGetAnnotationAs<IndirectCallSiteProfile>( 123 Instr, "CallProfile"); 124 if (!ICSP) 125 continue; 126 for (const IndirectCallProfile &CSP : ICSP.get()) { 127 StringRef TargetName = ""; 128 const BinaryFunction *Callee = 129 setCSIDestination(BC, CSI, CSP.Symbol, BAT); 130 if (Callee) 131 TargetName = Callee->getOneName(); 132 CSI.Count = CSP.Count; 133 CSI.Mispreds = CSP.Mispreds; 134 CSTargets.emplace_back(TargetName, CSI); 135 } 136 } else { // direct call or a tail call 137 StringRef TargetName = ""; 138 const MCSymbol *CalleeSymbol = BC.MIB->getTargetSymbol(Instr); 139 const BinaryFunction *const Callee = 140 setCSIDestination(BC, CSI, CalleeSymbol, BAT); 141 if (Callee) 142 TargetName = Callee->getOneName(); 143 144 auto getAnnotationWithDefault = [&](const MCInst &Inst, StringRef Ann) { 145 return BC.MIB->getAnnotationWithDefault(Instr, Ann, 0ull); 146 }; 147 if (BC.MIB->getConditionalTailCall(Instr)) { 148 CSI.Count = getAnnotationWithDefault(Instr, "CTCTakenCount"); 149 CSI.Mispreds = getAnnotationWithDefault(Instr, "CTCMispredCount"); 150 } else { 151 CSI.Count = getAnnotationWithDefault(Instr, "Count"); 152 } 153 154 if (CSI.Count) 155 CSTargets.emplace_back(TargetName, CSI); 156 } 157 // Sort targets in a similar way to getBranchData, see Location::operator< 158 llvm::sort(CSTargets, [](const auto &RHS, const auto &LHS) { 159 if (RHS.first != LHS.first) 160 return RHS.first < LHS.first; 161 return RHS.second.Offset < LHS.second.Offset; 162 }); 163 for (auto &KV : CSTargets) 164 YamlBB.CallSites.push_back(KV.second); 165 } 166 167 // Skip printing if there's no profile data for non-entry basic block. 168 // Include landing pads with non-zero execution count. 169 if (YamlBB.CallSites.empty() && !BB->isEntryPoint() && 170 !(BB->isLandingPad() && BB->getKnownExecutionCount() != 0)) { 171 // Include blocks having successors or predecessors with positive counts. 172 uint64_t SuccessorExecCount = 0; 173 for (const BinaryBasicBlock::BinaryBranchInfo &BranchInfo : 174 BB->branch_info()) 175 SuccessorExecCount += BranchInfo.Count; 176 uint64_t PredecessorExecCount = 0; 177 for (auto Pred : BB->predecessors()) 178 PredecessorExecCount += Pred->getBranchInfo(*BB).Count; 179 if (!SuccessorExecCount && !PredecessorExecCount) 180 continue; 181 } 182 183 auto BranchInfo = BB->branch_info_begin(); 184 for (const BinaryBasicBlock *Successor : BB->successors()) { 185 yaml::bolt::SuccessorInfo YamlSI; 186 YamlSI.Index = Successor->getLayoutIndex(); 187 YamlSI.Count = BranchInfo->Count; 188 YamlSI.Mispreds = BranchInfo->MispredictedCount; 189 190 YamlBB.Successors.emplace_back(YamlSI); 191 192 ++BranchInfo; 193 } 194 195 if (PseudoProbeDecoder) { 196 const AddressProbesMap &ProbeMap = 197 PseudoProbeDecoder->getAddress2ProbesMap(); 198 const uint64_t FuncAddr = BF.getAddress(); 199 const std::pair<uint64_t, uint64_t> &BlockRange = 200 BB->getInputAddressRange(); 201 for (const MCDecodedPseudoProbe &Probe : ProbeMap.find( 202 FuncAddr + BlockRange.first, FuncAddr + BlockRange.second)) 203 YamlBB.PseudoProbes.emplace_back(yaml::bolt::PseudoProbeInfo{ 204 Probe.getGuid(), Probe.getIndex(), Probe.getType()}); 205 } 206 207 YamlBF.Blocks.emplace_back(YamlBB); 208 } 209 return YamlBF; 210 } 211 212 std::error_code YAMLProfileWriter::writeProfile(const RewriteInstance &RI) { 213 const BinaryContext &BC = RI.getBinaryContext(); 214 const auto &Functions = BC.getBinaryFunctions(); 215 216 std::error_code EC; 217 OS = std::make_unique<raw_fd_ostream>(Filename, EC, sys::fs::OF_None); 218 if (EC) { 219 errs() << "BOLT-WARNING: " << EC.message() << " : unable to open " 220 << Filename << " for output.\n"; 221 return EC; 222 } 223 224 yaml::bolt::BinaryProfile BP; 225 226 // Fill out the header info. 227 BP.Header.Version = 1; 228 BP.Header.FileName = std::string(BC.getFilename()); 229 std::optional<StringRef> BuildID = BC.getFileBuildID(); 230 BP.Header.Id = BuildID ? std::string(*BuildID) : "<unknown>"; 231 BP.Header.Origin = std::string(RI.getProfileReader()->getReaderName()); 232 BP.Header.IsDFSOrder = opts::ProfileUseDFS; 233 BP.Header.HashFunction = HashFunction::Default; 234 235 StringSet<> EventNames = RI.getProfileReader()->getEventNames(); 236 if (!EventNames.empty()) { 237 std::string Sep; 238 for (const StringMapEntry<std::nullopt_t> &EventEntry : EventNames) { 239 BP.Header.EventNames += Sep + EventEntry.first().str(); 240 Sep = ","; 241 } 242 } 243 244 // Make sure the profile is consistent across all functions. 245 uint16_t ProfileFlags = BinaryFunction::PF_NONE; 246 for (const auto &BFI : Functions) { 247 const BinaryFunction &BF = BFI.second; 248 if (BF.hasProfile() && !BF.empty()) { 249 assert(BF.getProfileFlags() != BinaryFunction::PF_NONE); 250 if (ProfileFlags == BinaryFunction::PF_NONE) 251 ProfileFlags = BF.getProfileFlags(); 252 253 assert(BF.getProfileFlags() == ProfileFlags && 254 "expected consistent profile flags across all functions"); 255 } 256 } 257 BP.Header.Flags = ProfileFlags; 258 259 // Add all function objects. 260 for (const auto &BFI : Functions) { 261 const BinaryFunction &BF = BFI.second; 262 if (BF.hasProfile()) { 263 if (!BF.hasValidProfile() && !RI.getProfileReader()->isTrustedSource()) 264 continue; 265 266 BP.Functions.emplace_back(convert(BF, opts::ProfileUseDFS)); 267 } 268 } 269 270 // Write the profile. 271 yaml::Output Out(*OS, nullptr, 0); 272 Out << BP; 273 274 return std::error_code(); 275 } 276 277 } // namespace bolt 278 } // namespace llvm 279