12f09f445SMaksim Panchenko //===- bolt/Core/BinaryFunction.cpp - Low-level function ------------------===// 2a34c753fSRafael Auler // 3a34c753fSRafael Auler // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4a34c753fSRafael Auler // See https://llvm.org/LICENSE.txt for license information. 5a34c753fSRafael Auler // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6a34c753fSRafael Auler // 7a34c753fSRafael Auler //===----------------------------------------------------------------------===// 8a34c753fSRafael Auler // 92f09f445SMaksim Panchenko // This file implements the BinaryFunction class. 102f09f445SMaksim Panchenko // 11a34c753fSRafael Auler //===----------------------------------------------------------------------===// 12a34c753fSRafael Auler 13a34c753fSRafael Auler #include "bolt/Core/BinaryFunction.h" 14a34c753fSRafael Auler #include "bolt/Core/BinaryBasicBlock.h" 15a7b69dbdSAmir Ayupov #include "bolt/Core/BinaryDomTree.h" 16a34c753fSRafael Auler #include "bolt/Core/DynoStats.h" 17a34c753fSRafael Auler #include "bolt/Core/MCPlusBuilder.h" 18a34c753fSRafael Auler #include "bolt/Utils/NameResolver.h" 19a34c753fSRafael Auler #include "bolt/Utils/NameShortener.h" 20a34c753fSRafael Auler #include "bolt/Utils/Utils.h" 215d8247d4SAmir Ayupov #include "llvm/ADT/STLExtras.h" 22a34c753fSRafael Auler #include "llvm/ADT/SmallSet.h" 23f7581a39SAmir Ayupov #include "llvm/ADT/StringExtras.h" 24a34c753fSRafael Auler #include "llvm/ADT/StringRef.h" 25a34c753fSRafael Auler #include "llvm/ADT/edit_distance.h" 26ae585be1SRafael Auler #include "llvm/Demangle/Demangle.h" 27a34c753fSRafael Auler #include "llvm/MC/MCAsmInfo.h" 28a34c753fSRafael Auler #include "llvm/MC/MCAsmLayout.h" 29a34c753fSRafael Auler #include "llvm/MC/MCContext.h" 30a34c753fSRafael Auler #include "llvm/MC/MCDisassembler/MCDisassembler.h" 31a34c753fSRafael Auler #include "llvm/MC/MCExpr.h" 32a34c753fSRafael Auler #include "llvm/MC/MCInst.h" 33a34c753fSRafael Auler #include "llvm/MC/MCInstPrinter.h" 3457f7c7d9Sserge-sans-paille #include "llvm/MC/MCRegisterInfo.h" 35a34c753fSRafael Auler #include "llvm/Object/ObjectFile.h" 36a34c753fSRafael Auler #include "llvm/Support/CommandLine.h" 37a34c753fSRafael Auler #include "llvm/Support/Debug.h" 38a34c753fSRafael Auler #include "llvm/Support/GraphWriter.h" 39a34c753fSRafael Auler #include "llvm/Support/LEB128.h" 40a34c753fSRafael Auler #include "llvm/Support/Regex.h" 41a34c753fSRafael Auler #include "llvm/Support/Timer.h" 42a34c753fSRafael Auler #include "llvm/Support/raw_ostream.h" 43a34c753fSRafael Auler #include <functional> 44a34c753fSRafael Auler #include <limits> 45a34c753fSRafael Auler #include <numeric> 46a34c753fSRafael Auler #include <string> 47a34c753fSRafael Auler 48a34c753fSRafael Auler #define DEBUG_TYPE "bolt" 49a34c753fSRafael Auler 50a34c753fSRafael Auler using namespace llvm; 51a34c753fSRafael Auler using namespace bolt; 52a34c753fSRafael Auler 53a34c753fSRafael Auler namespace opts { 54a34c753fSRafael Auler 55a34c753fSRafael Auler extern cl::OptionCategory BoltCategory; 56a34c753fSRafael Auler extern cl::OptionCategory BoltOptCategory; 57a34c753fSRafael Auler extern cl::OptionCategory BoltRelocCategory; 58a34c753fSRafael Auler 59a34c753fSRafael Auler extern cl::opt<bool> EnableBAT; 60a34c753fSRafael Auler extern cl::opt<bool> Instrument; 61a34c753fSRafael Auler extern cl::opt<bool> StrictMode; 62a34c753fSRafael Auler extern cl::opt<bool> UpdateDebugSections; 63a34c753fSRafael Auler extern cl::opt<unsigned> Verbosity; 64a34c753fSRafael Auler 65a34c753fSRafael Auler extern bool processAllFunctions(); 66a34c753fSRafael Auler 67b92436efSFangrui Song cl::opt<bool> CheckEncoding( 68b92436efSFangrui Song "check-encoding", 69a34c753fSRafael Auler cl::desc("perform verification of LLVM instruction encoding/decoding. " 70a34c753fSRafael Auler "Every instruction in the input is decoded and re-encoded. " 71a34c753fSRafael Auler "If the resulting bytes do not match the input, a warning message " 72a34c753fSRafael Auler "is printed."), 73b92436efSFangrui Song cl::Hidden, cl::cat(BoltCategory)); 74a34c753fSRafael Auler 75b92436efSFangrui Song static cl::opt<bool> DotToolTipCode( 76b92436efSFangrui Song "dot-tooltip-code", 77b92436efSFangrui Song cl::desc("add basic block instructions as tool tips on nodes"), cl::Hidden, 78a34c753fSRafael Auler cl::cat(BoltCategory)); 79a34c753fSRafael Auler 80a34c753fSRafael Auler cl::opt<JumpTableSupportLevel> 81a34c753fSRafael Auler JumpTables("jump-tables", 82a34c753fSRafael Auler cl::desc("jump tables support (default=basic)"), 83a34c753fSRafael Auler cl::init(JTS_BASIC), 84a34c753fSRafael Auler cl::values( 85a34c753fSRafael Auler clEnumValN(JTS_NONE, "none", 86a34c753fSRafael Auler "do not optimize functions with jump tables"), 87a34c753fSRafael Auler clEnumValN(JTS_BASIC, "basic", 88a34c753fSRafael Auler "optimize functions with jump tables"), 89a34c753fSRafael Auler clEnumValN(JTS_MOVE, "move", 90a34c753fSRafael Auler "move jump tables to a separate section"), 91a34c753fSRafael Auler clEnumValN(JTS_SPLIT, "split", 92a34c753fSRafael Auler "split jump tables section into hot and cold based on " 93a34c753fSRafael Auler "function execution frequency"), 94a34c753fSRafael Auler clEnumValN(JTS_AGGRESSIVE, "aggressive", 95a34c753fSRafael Auler "aggressively split jump tables section based on usage " 96a34c753fSRafael Auler "of the tables")), 97a34c753fSRafael Auler cl::ZeroOrMore, 98a34c753fSRafael Auler cl::cat(BoltOptCategory)); 99a34c753fSRafael Auler 100b92436efSFangrui Song static cl::opt<bool> NoScan( 101b92436efSFangrui Song "no-scan", 102b92436efSFangrui Song cl::desc( 103b92436efSFangrui Song "do not scan cold functions for external references (may result in " 104a34c753fSRafael Auler "slower binary)"), 105b92436efSFangrui Song cl::Hidden, cl::cat(BoltOptCategory)); 106a34c753fSRafael Auler 107a34c753fSRafael Auler cl::opt<bool> 108a34c753fSRafael Auler PreserveBlocksAlignment("preserve-blocks-alignment", 109a34c753fSRafael Auler cl::desc("try to preserve basic block alignment"), 110a34c753fSRafael Auler cl::cat(BoltOptCategory)); 111a34c753fSRafael Auler 112a34c753fSRafael Auler cl::opt<bool> 113a34c753fSRafael Auler PrintDynoStats("dyno-stats", 114a34c753fSRafael Auler cl::desc("print execution info based on profile"), 115a34c753fSRafael Auler cl::cat(BoltCategory)); 116a34c753fSRafael Auler 117a34c753fSRafael Auler static cl::opt<bool> 118a34c753fSRafael Auler PrintDynoStatsOnly("print-dyno-stats-only", 119a34c753fSRafael Auler cl::desc("while printing functions output dyno-stats and skip instructions"), 120a34c753fSRafael Auler cl::init(false), 121a34c753fSRafael Auler cl::Hidden, 122a34c753fSRafael Auler cl::cat(BoltCategory)); 123a34c753fSRafael Auler 124a34c753fSRafael Auler static cl::list<std::string> 125a34c753fSRafael Auler PrintOnly("print-only", 126a34c753fSRafael Auler cl::CommaSeparated, 127a34c753fSRafael Auler cl::desc("list of functions to print"), 128a34c753fSRafael Auler cl::value_desc("func1,func2,func3,..."), 129a34c753fSRafael Auler cl::Hidden, 130a34c753fSRafael Auler cl::cat(BoltCategory)); 131a34c753fSRafael Auler 132a34c753fSRafael Auler cl::opt<bool> 133a34c753fSRafael Auler TimeBuild("time-build", 134a34c753fSRafael Auler cl::desc("print time spent constructing binary functions"), 135b92436efSFangrui Song cl::Hidden, cl::cat(BoltCategory)); 136a34c753fSRafael Auler 137a34c753fSRafael Auler cl::opt<bool> 138a34c753fSRafael Auler TrapOnAVX512("trap-avx512", 139a34c753fSRafael Auler cl::desc("in relocation mode trap upon entry to any function that uses " 140a34c753fSRafael Auler "AVX-512 instructions"), 141a34c753fSRafael Auler cl::init(false), 142a34c753fSRafael Auler cl::ZeroOrMore, 143a34c753fSRafael Auler cl::Hidden, 144a34c753fSRafael Auler cl::cat(BoltCategory)); 145a34c753fSRafael Auler 146a34c753fSRafael Auler bool shouldPrint(const BinaryFunction &Function) { 147a34c753fSRafael Auler if (Function.isIgnored()) 148a34c753fSRafael Auler return false; 149a34c753fSRafael Auler 150a34c753fSRafael Auler if (PrintOnly.empty()) 151a34c753fSRafael Auler return true; 152a34c753fSRafael Auler 153a34c753fSRafael Auler for (std::string &Name : opts::PrintOnly) { 154a34c753fSRafael Auler if (Function.hasNameRegex(Name)) { 155a34c753fSRafael Auler return true; 156a34c753fSRafael Auler } 157a34c753fSRafael Auler } 158a34c753fSRafael Auler 159a34c753fSRafael Auler return false; 160a34c753fSRafael Auler } 161a34c753fSRafael Auler 162a34c753fSRafael Auler } // namespace opts 163a34c753fSRafael Auler 164a34c753fSRafael Auler namespace llvm { 165a34c753fSRafael Auler namespace bolt { 166a34c753fSRafael Auler 167a34c753fSRafael Auler constexpr unsigned BinaryFunction::MinAlign; 168a34c753fSRafael Auler 169a34c753fSRafael Auler namespace { 170a34c753fSRafael Auler 17140c2e0faSMaksim Panchenko template <typename R> bool emptyRange(const R &Range) { 172a34c753fSRafael Auler return Range.begin() == Range.end(); 173a34c753fSRafael Auler } 174a34c753fSRafael Auler 175a34c753fSRafael Auler /// Gets debug line information for the instruction located at the given 176a34c753fSRafael Auler /// address in the original binary. The SMLoc's pointer is used 177a34c753fSRafael Auler /// to point to this information, which is represented by a 178a34c753fSRafael Auler /// DebugLineTableRowRef. The returned pointer is null if no debug line 179a34c753fSRafael Auler /// information for this instruction was found. 18040c2e0faSMaksim Panchenko SMLoc findDebugLineInformationForInstructionAt( 18140c2e0faSMaksim Panchenko uint64_t Address, DWARFUnit *Unit, 18240c2e0faSMaksim Panchenko const DWARFDebugLine::LineTable *LineTable) { 183a34c753fSRafael Auler // We use the pointer in SMLoc to store an instance of DebugLineTableRowRef, 184a34c753fSRafael Auler // which occupies 64 bits. Thus, we can only proceed if the struct fits into 185a34c753fSRafael Auler // the pointer itself. 18640c2e0faSMaksim Panchenko assert(sizeof(decltype(SMLoc().getPointer())) >= 18740c2e0faSMaksim Panchenko sizeof(DebugLineTableRowRef) && 188a34c753fSRafael Auler "Cannot fit instruction debug line information into SMLoc's pointer"); 189a34c753fSRafael Auler 190a34c753fSRafael Auler SMLoc NullResult = DebugLineTableRowRef::NULL_ROW.toSMLoc(); 191a34c753fSRafael Auler uint32_t RowIndex = LineTable->lookupAddress( 192a34c753fSRafael Auler {Address, object::SectionedAddress::UndefSection}); 193a34c753fSRafael Auler if (RowIndex == LineTable->UnknownRowIndex) 194a34c753fSRafael Auler return NullResult; 195a34c753fSRafael Auler 196a34c753fSRafael Auler assert(RowIndex < LineTable->Rows.size() && 197a34c753fSRafael Auler "Line Table lookup returned invalid index."); 198a34c753fSRafael Auler 199a34c753fSRafael Auler decltype(SMLoc().getPointer()) Ptr; 200a34c753fSRafael Auler DebugLineTableRowRef *InstructionLocation = 201a34c753fSRafael Auler reinterpret_cast<DebugLineTableRowRef *>(&Ptr); 202a34c753fSRafael Auler 203a34c753fSRafael Auler InstructionLocation->DwCompileUnitIndex = Unit->getOffset(); 204a34c753fSRafael Auler InstructionLocation->RowIndex = RowIndex + 1; 205a34c753fSRafael Auler 206a34c753fSRafael Auler return SMLoc::getFromPointer(Ptr); 207a34c753fSRafael Auler } 208a34c753fSRafael Auler 209a34c753fSRafael Auler std::string buildSectionName(StringRef Prefix, StringRef Name, 210a34c753fSRafael Auler const BinaryContext &BC) { 211a34c753fSRafael Auler if (BC.isELF()) 212a34c753fSRafael Auler return (Prefix + Name).str(); 213a34c753fSRafael Auler static NameShortener NS; 214a34c753fSRafael Auler return (Prefix + Twine(NS.getID(Name))).str(); 215a34c753fSRafael Auler } 216a34c753fSRafael Auler 21740c2e0faSMaksim Panchenko raw_ostream &operator<<(raw_ostream &OS, const BinaryFunction::State State) { 21840c2e0faSMaksim Panchenko switch (State) { 21940c2e0faSMaksim Panchenko case BinaryFunction::State::Empty: OS << "empty"; break; 22040c2e0faSMaksim Panchenko case BinaryFunction::State::Disassembled: OS << "disassembled"; break; 22140c2e0faSMaksim Panchenko case BinaryFunction::State::CFG: OS << "CFG constructed"; break; 22240c2e0faSMaksim Panchenko case BinaryFunction::State::CFG_Finalized: OS << "CFG finalized"; break; 22340c2e0faSMaksim Panchenko case BinaryFunction::State::EmittedCFG: OS << "emitted with CFG"; break; 22440c2e0faSMaksim Panchenko case BinaryFunction::State::Emitted: OS << "emitted"; break; 22540c2e0faSMaksim Panchenko } 22640c2e0faSMaksim Panchenko 22740c2e0faSMaksim Panchenko return OS; 22840c2e0faSMaksim Panchenko } 22940c2e0faSMaksim Panchenko 230a34c753fSRafael Auler } // namespace 231a34c753fSRafael Auler 232a34c753fSRafael Auler std::string BinaryFunction::buildCodeSectionName(StringRef Name, 233a34c753fSRafael Auler const BinaryContext &BC) { 234a34c753fSRafael Auler return buildSectionName(BC.isELF() ? ".local.text." : ".l.text.", Name, BC); 235a34c753fSRafael Auler } 236a34c753fSRafael Auler 237a34c753fSRafael Auler std::string BinaryFunction::buildColdCodeSectionName(StringRef Name, 238a34c753fSRafael Auler const BinaryContext &BC) { 239a34c753fSRafael Auler return buildSectionName(BC.isELF() ? ".local.cold.text." : ".l.c.text.", Name, 240a34c753fSRafael Auler BC); 241a34c753fSRafael Auler } 242a34c753fSRafael Auler 243a34c753fSRafael Auler uint64_t BinaryFunction::Count = 0; 244a34c753fSRafael Auler 24540c2e0faSMaksim Panchenko Optional<StringRef> BinaryFunction::hasNameRegex(const StringRef Name) const { 246a34c753fSRafael Auler const std::string RegexName = (Twine("^") + StringRef(Name) + "$").str(); 247a34c753fSRafael Auler Regex MatchName(RegexName); 248a34c753fSRafael Auler Optional<StringRef> Match = forEachName( 249a34c753fSRafael Auler [&MatchName](StringRef Name) { return MatchName.match(Name); }); 250a34c753fSRafael Auler 251a34c753fSRafael Auler return Match; 252a34c753fSRafael Auler } 253a34c753fSRafael Auler 254a34c753fSRafael Auler Optional<StringRef> 255a34c753fSRafael Auler BinaryFunction::hasRestoredNameRegex(const StringRef Name) const { 256a34c753fSRafael Auler const std::string RegexName = (Twine("^") + StringRef(Name) + "$").str(); 257a34c753fSRafael Auler Regex MatchName(RegexName); 258a34c753fSRafael Auler Optional<StringRef> Match = forEachName([&MatchName](StringRef Name) { 259a34c753fSRafael Auler return MatchName.match(NameResolver::restore(Name)); 260a34c753fSRafael Auler }); 261a34c753fSRafael Auler 262a34c753fSRafael Auler return Match; 263a34c753fSRafael Auler } 264a34c753fSRafael Auler 265a34c753fSRafael Auler std::string BinaryFunction::getDemangledName() const { 266a34c753fSRafael Auler StringRef MangledName = NameResolver::restore(getOneName()); 267ae585be1SRafael Auler return demangle(MangledName.str()); 268a34c753fSRafael Auler } 269a34c753fSRafael Auler 270a34c753fSRafael Auler BinaryBasicBlock * 271a34c753fSRafael Auler BinaryFunction::getBasicBlockContainingOffset(uint64_t Offset) { 272a34c753fSRafael Auler if (Offset > Size) 273a34c753fSRafael Auler return nullptr; 274a34c753fSRafael Auler 275a34c753fSRafael Auler if (BasicBlockOffsets.empty()) 276a34c753fSRafael Auler return nullptr; 277a34c753fSRafael Auler 278a34c753fSRafael Auler /* 279a34c753fSRafael Auler * This is commented out because it makes BOLT too slow. 280a34c753fSRafael Auler * assert(std::is_sorted(BasicBlockOffsets.begin(), 281a34c753fSRafael Auler * BasicBlockOffsets.end(), 282a34c753fSRafael Auler * CompareBasicBlockOffsets()))); 283a34c753fSRafael Auler */ 284d2c87699SAmir Ayupov auto I = 285d2c87699SAmir Ayupov llvm::upper_bound(BasicBlockOffsets, BasicBlockOffset(Offset, nullptr), 286a34c753fSRafael Auler CompareBasicBlockOffsets()); 287a34c753fSRafael Auler assert(I != BasicBlockOffsets.begin() && "first basic block not at offset 0"); 288a34c753fSRafael Auler --I; 289a34c753fSRafael Auler BinaryBasicBlock *BB = I->second; 290a34c753fSRafael Auler return (Offset < BB->getOffset() + BB->getOriginalSize()) ? BB : nullptr; 291a34c753fSRafael Auler } 292a34c753fSRafael Auler 293a34c753fSRafael Auler void BinaryFunction::markUnreachableBlocks() { 294a34c753fSRafael Auler std::stack<BinaryBasicBlock *> Stack; 295a34c753fSRafael Auler 2963652483cSRafael Auler for (BinaryBasicBlock *BB : layout()) 297a34c753fSRafael Auler BB->markValid(false); 298a34c753fSRafael Auler 299a34c753fSRafael Auler // Add all entries and landing pads as roots. 300a34c753fSRafael Auler for (BinaryBasicBlock *BB : BasicBlocks) { 301a34c753fSRafael Auler if (isEntryPoint(*BB) || BB->isLandingPad()) { 302a34c753fSRafael Auler Stack.push(BB); 303a34c753fSRafael Auler BB->markValid(true); 304a34c753fSRafael Auler continue; 305a34c753fSRafael Auler } 306a34c753fSRafael Auler // FIXME: 307a34c753fSRafael Auler // Also mark BBs with indirect jumps as reachable, since we do not 308933df2a4SMaksim Panchenko // support removing unused jump tables yet (GH-issue20). 309a34c753fSRafael Auler for (const MCInst &Inst : *BB) { 310a34c753fSRafael Auler if (BC.MIB->getJumpTable(Inst)) { 311a34c753fSRafael Auler Stack.push(BB); 312a34c753fSRafael Auler BB->markValid(true); 313a34c753fSRafael Auler break; 314a34c753fSRafael Auler } 315a34c753fSRafael Auler } 316a34c753fSRafael Auler } 317a34c753fSRafael Auler 318a34c753fSRafael Auler // Determine reachable BBs from the entry point 319a34c753fSRafael Auler while (!Stack.empty()) { 320a34c753fSRafael Auler BinaryBasicBlock *BB = Stack.top(); 321a34c753fSRafael Auler Stack.pop(); 322a34c753fSRafael Auler for (BinaryBasicBlock *Succ : BB->successors()) { 323a34c753fSRafael Auler if (Succ->isValid()) 324a34c753fSRafael Auler continue; 325a34c753fSRafael Auler Succ->markValid(true); 326a34c753fSRafael Auler Stack.push(Succ); 327a34c753fSRafael Auler } 328a34c753fSRafael Auler } 329a34c753fSRafael Auler } 330a34c753fSRafael Auler 331a34c753fSRafael Auler // Any unnecessary fallthrough jumps revealed after calling eraseInvalidBBs 332a34c753fSRafael Auler // will be cleaned up by fixBranches(). 333a34c753fSRafael Auler std::pair<unsigned, uint64_t> BinaryFunction::eraseInvalidBBs() { 334a34c753fSRafael Auler BasicBlockOrderType NewLayout; 335a34c753fSRafael Auler unsigned Count = 0; 336a34c753fSRafael Auler uint64_t Bytes = 0; 337a34c753fSRafael Auler for (BinaryBasicBlock *BB : layout()) { 338a34c753fSRafael Auler if (BB->isValid()) { 339a34c753fSRafael Auler NewLayout.push_back(BB); 340a34c753fSRafael Auler } else { 341a34c753fSRafael Auler assert(!isEntryPoint(*BB) && "all entry blocks must be valid"); 342a34c753fSRafael Auler ++Count; 343a34c753fSRafael Auler Bytes += BC.computeCodeSize(BB->begin(), BB->end()); 344a34c753fSRafael Auler } 345a34c753fSRafael Auler } 346a34c753fSRafael Auler BasicBlocksLayout = std::move(NewLayout); 347a34c753fSRafael Auler 348a34c753fSRafael Auler BasicBlockListType NewBasicBlocks; 349a34c753fSRafael Auler for (auto I = BasicBlocks.begin(), E = BasicBlocks.end(); I != E; ++I) { 350a34c753fSRafael Auler BinaryBasicBlock *BB = *I; 351a34c753fSRafael Auler if (BB->isValid()) { 352a34c753fSRafael Auler NewBasicBlocks.push_back(BB); 353a34c753fSRafael Auler } else { 354a34c753fSRafael Auler // Make sure the block is removed from the list of predecessors. 355a34c753fSRafael Auler BB->removeAllSuccessors(); 356a34c753fSRafael Auler DeletedBasicBlocks.push_back(BB); 357a34c753fSRafael Auler } 358a34c753fSRafael Auler } 359a34c753fSRafael Auler BasicBlocks = std::move(NewBasicBlocks); 360a34c753fSRafael Auler 361a34c753fSRafael Auler assert(BasicBlocks.size() == BasicBlocksLayout.size()); 362a34c753fSRafael Auler 363a34c753fSRafael Auler // Update CFG state if needed 364a34c753fSRafael Auler if (Count > 0) 365a34c753fSRafael Auler recomputeLandingPads(); 366a34c753fSRafael Auler 367a34c753fSRafael Auler return std::make_pair(Count, Bytes); 368a34c753fSRafael Auler } 369a34c753fSRafael Auler 370a34c753fSRafael Auler bool BinaryFunction::isForwardCall(const MCSymbol *CalleeSymbol) const { 371a34c753fSRafael Auler // This function should work properly before and after function reordering. 372a34c753fSRafael Auler // In order to accomplish this, we use the function index (if it is valid). 373a34c753fSRafael Auler // If the function indices are not valid, we fall back to the original 374a34c753fSRafael Auler // addresses. This should be ok because the functions without valid indices 375a34c753fSRafael Auler // should have been ordered with a stable sort. 376a34c753fSRafael Auler const BinaryFunction *CalleeBF = BC.getFunctionForSymbol(CalleeSymbol); 377a34c753fSRafael Auler if (CalleeBF) { 378a34c753fSRafael Auler if (CalleeBF->isInjected()) 379a34c753fSRafael Auler return true; 380a34c753fSRafael Auler 381a34c753fSRafael Auler if (hasValidIndex() && CalleeBF->hasValidIndex()) { 382a34c753fSRafael Auler return getIndex() < CalleeBF->getIndex(); 383a34c753fSRafael Auler } else if (hasValidIndex() && !CalleeBF->hasValidIndex()) { 384a34c753fSRafael Auler return true; 385a34c753fSRafael Auler } else if (!hasValidIndex() && CalleeBF->hasValidIndex()) { 386a34c753fSRafael Auler return false; 387a34c753fSRafael Auler } else { 388a34c753fSRafael Auler return getAddress() < CalleeBF->getAddress(); 389a34c753fSRafael Auler } 390a34c753fSRafael Auler } else { 391a34c753fSRafael Auler // Absolute symbol. 392a34c753fSRafael Auler ErrorOr<uint64_t> CalleeAddressOrError = BC.getSymbolValue(*CalleeSymbol); 393a34c753fSRafael Auler assert(CalleeAddressOrError && "unregistered symbol found"); 394a34c753fSRafael Auler return *CalleeAddressOrError > getAddress(); 395a34c753fSRafael Auler } 396a34c753fSRafael Auler } 397a34c753fSRafael Auler 398a34c753fSRafael Auler void BinaryFunction::dump(bool PrintInstructions) const { 399a34c753fSRafael Auler print(dbgs(), "", PrintInstructions); 400a34c753fSRafael Auler } 401a34c753fSRafael Auler 402a34c753fSRafael Auler void BinaryFunction::print(raw_ostream &OS, std::string Annotation, 403a34c753fSRafael Auler bool PrintInstructions) const { 404a34c753fSRafael Auler if (!opts::shouldPrint(*this)) 405a34c753fSRafael Auler return; 406a34c753fSRafael Auler 407a34c753fSRafael Auler StringRef SectionName = 408a34c753fSRafael Auler OriginSection ? OriginSection->getName() : "<no origin section>"; 409a34c753fSRafael Auler OS << "Binary Function \"" << *this << "\" " << Annotation << " {"; 410a34c753fSRafael Auler std::vector<StringRef> AllNames = getNames(); 411a34c753fSRafael Auler if (AllNames.size() > 1) { 412a34c753fSRafael Auler OS << "\n All names : "; 413a34c753fSRafael Auler const char *Sep = ""; 414253b8f0aSAmir Ayupov for (const StringRef &Name : AllNames) { 415a34c753fSRafael Auler OS << Sep << Name; 416a34c753fSRafael Auler Sep = "\n "; 417a34c753fSRafael Auler } 418a34c753fSRafael Auler } 419a34c753fSRafael Auler OS << "\n Number : " << FunctionNumber 420a34c753fSRafael Auler << "\n State : " << CurrentState 421a34c753fSRafael Auler << "\n Address : 0x" << Twine::utohexstr(Address) 422a34c753fSRafael Auler << "\n Size : 0x" << Twine::utohexstr(Size) 423a34c753fSRafael Auler << "\n MaxSize : 0x" << Twine::utohexstr(MaxSize) 424a34c753fSRafael Auler << "\n Offset : 0x" << Twine::utohexstr(FileOffset) 425a34c753fSRafael Auler << "\n Section : " << SectionName 426a34c753fSRafael Auler << "\n Orc Section : " << getCodeSectionName() 427a34c753fSRafael Auler << "\n LSDA : 0x" << Twine::utohexstr(getLSDAAddress()) 428a34c753fSRafael Auler << "\n IsSimple : " << IsSimple 429a34c753fSRafael Auler << "\n IsMultiEntry: " << isMultiEntry() 430a34c753fSRafael Auler << "\n IsSplit : " << isSplit() 431a34c753fSRafael Auler << "\n BB Count : " << size(); 432a34c753fSRafael Auler 43340c2e0faSMaksim Panchenko if (HasFixedIndirectBranch) 434a34c753fSRafael Auler OS << "\n HasFixedIndirectBranch : true"; 43540c2e0faSMaksim Panchenko if (HasUnknownControlFlow) 436a34c753fSRafael Auler OS << "\n Unknown CF : true"; 43740c2e0faSMaksim Panchenko if (getPersonalityFunction()) 438a34c753fSRafael Auler OS << "\n Personality : " << getPersonalityFunction()->getName(); 43940c2e0faSMaksim Panchenko if (IsFragment) 440a34c753fSRafael Auler OS << "\n IsFragment : true"; 44140c2e0faSMaksim Panchenko if (isFolded()) 442a34c753fSRafael Auler OS << "\n FoldedInto : " << *getFoldedIntoFunction(); 44340c2e0faSMaksim Panchenko for (BinaryFunction *ParentFragment : ParentFragments) 444a34c753fSRafael Auler OS << "\n Parent : " << *ParentFragment; 445a34c753fSRafael Auler if (!Fragments.empty()) { 446a34c753fSRafael Auler OS << "\n Fragments : "; 447f7581a39SAmir Ayupov ListSeparator LS; 448f7581a39SAmir Ayupov for (BinaryFunction *Frag : Fragments) 449f7581a39SAmir Ayupov OS << LS << *Frag; 450a34c753fSRafael Auler } 45140c2e0faSMaksim Panchenko if (hasCFG()) 452a34c753fSRafael Auler OS << "\n Hash : " << Twine::utohexstr(computeHash()); 453a34c753fSRafael Auler if (isMultiEntry()) { 454a34c753fSRafael Auler OS << "\n Secondary Entry Points : "; 455f7581a39SAmir Ayupov ListSeparator LS; 456f7581a39SAmir Ayupov for (const auto &KV : SecondaryEntryPoints) 457f7581a39SAmir Ayupov OS << LS << KV.second->getName(); 458a34c753fSRafael Auler } 45940c2e0faSMaksim Panchenko if (FrameInstructions.size()) 460a34c753fSRafael Auler OS << "\n CFI Instrs : " << FrameInstructions.size(); 461a34c753fSRafael Auler if (BasicBlocksLayout.size()) { 462a34c753fSRafael Auler OS << "\n BB Layout : "; 463f7581a39SAmir Ayupov ListSeparator LS; 464f7581a39SAmir Ayupov for (BinaryBasicBlock *BB : BasicBlocksLayout) 465f7581a39SAmir Ayupov OS << LS << BB->getName(); 466a34c753fSRafael Auler } 467a34c753fSRafael Auler if (ImageAddress) 468a34c753fSRafael Auler OS << "\n Image : 0x" << Twine::utohexstr(ImageAddress); 469a34c753fSRafael Auler if (ExecutionCount != COUNT_NO_PROFILE) { 470a34c753fSRafael Auler OS << "\n Exec Count : " << ExecutionCount; 471a34c753fSRafael Auler OS << "\n Profile Acc : " << format("%.1f%%", ProfileMatchRatio * 100.0f); 472a34c753fSRafael Auler } 473a34c753fSRafael Auler 474a34c753fSRafael Auler if (opts::PrintDynoStats && !BasicBlocksLayout.empty()) { 475a34c753fSRafael Auler OS << '\n'; 476a34c753fSRafael Auler DynoStats dynoStats = getDynoStats(*this); 477a34c753fSRafael Auler OS << dynoStats; 478a34c753fSRafael Auler } 479a34c753fSRafael Auler 480a34c753fSRafael Auler OS << "\n}\n"; 481a34c753fSRafael Auler 482a34c753fSRafael Auler if (opts::PrintDynoStatsOnly || !PrintInstructions || !BC.InstPrinter) 483a34c753fSRafael Auler return; 484a34c753fSRafael Auler 485a34c753fSRafael Auler // Offset of the instruction in function. 486a34c753fSRafael Auler uint64_t Offset = 0; 487a34c753fSRafael Auler 488a34c753fSRafael Auler if (BasicBlocks.empty() && !Instructions.empty()) { 489a34c753fSRafael Auler // Print before CFG was built. 490a34c753fSRafael Auler for (const std::pair<const uint32_t, MCInst> &II : Instructions) { 491a34c753fSRafael Auler Offset = II.first; 492a34c753fSRafael Auler 493a34c753fSRafael Auler // Print label if exists at this offset. 494a34c753fSRafael Auler auto LI = Labels.find(Offset); 495a34c753fSRafael Auler if (LI != Labels.end()) { 496a34c753fSRafael Auler if (const MCSymbol *EntrySymbol = 497a34c753fSRafael Auler getSecondaryEntryPointSymbol(LI->second)) 498a34c753fSRafael Auler OS << EntrySymbol->getName() << " (Entry Point):\n"; 499a34c753fSRafael Auler OS << LI->second->getName() << ":\n"; 500a34c753fSRafael Auler } 501a34c753fSRafael Auler 502a34c753fSRafael Auler BC.printInstruction(OS, II.second, Offset, this); 503a34c753fSRafael Auler } 504a34c753fSRafael Auler } 505a34c753fSRafael Auler 506a34c753fSRafael Auler for (uint32_t I = 0, E = BasicBlocksLayout.size(); I != E; ++I) { 507a34c753fSRafael Auler BinaryBasicBlock *BB = BasicBlocksLayout[I]; 50840c2e0faSMaksim Panchenko if (I != 0 && BB->isCold() != BasicBlocksLayout[I - 1]->isCold()) 509a34c753fSRafael Auler OS << "------- HOT-COLD SPLIT POINT -------\n\n"; 510a34c753fSRafael Auler 51140c2e0faSMaksim Panchenko OS << BB->getName() << " (" << BB->size() 51240c2e0faSMaksim Panchenko << " instructions, align : " << BB->getAlignment() << ")\n"; 513a34c753fSRafael Auler 514a34c753fSRafael Auler if (isEntryPoint(*BB)) { 515a34c753fSRafael Auler if (MCSymbol *EntrySymbol = getSecondaryEntryPointSymbol(*BB)) 516a34c753fSRafael Auler OS << " Secondary Entry Point: " << EntrySymbol->getName() << '\n'; 517a34c753fSRafael Auler else 518a34c753fSRafael Auler OS << " Entry Point\n"; 519a34c753fSRafael Auler } 520a34c753fSRafael Auler 521a34c753fSRafael Auler if (BB->isLandingPad()) 522a34c753fSRafael Auler OS << " Landing Pad\n"; 523a34c753fSRafael Auler 524a34c753fSRafael Auler uint64_t BBExecCount = BB->getExecutionCount(); 525a34c753fSRafael Auler if (hasValidProfile()) { 526a34c753fSRafael Auler OS << " Exec Count : "; 527a34c753fSRafael Auler if (BB->getExecutionCount() != BinaryBasicBlock::COUNT_NO_PROFILE) 528a34c753fSRafael Auler OS << BBExecCount << '\n'; 529a34c753fSRafael Auler else 530a34c753fSRafael Auler OS << "<unknown>\n"; 531a34c753fSRafael Auler } 5323652483cSRafael Auler if (BB->getCFIState() >= 0) 533a34c753fSRafael Auler OS << " CFI State : " << BB->getCFIState() << '\n'; 534a34c753fSRafael Auler if (opts::EnableBAT) { 535a34c753fSRafael Auler OS << " Input offset: " << Twine::utohexstr(BB->getInputOffset()) 536a34c753fSRafael Auler << "\n"; 537a34c753fSRafael Auler } 538a34c753fSRafael Auler if (!BB->pred_empty()) { 539a34c753fSRafael Auler OS << " Predecessors: "; 540f7581a39SAmir Ayupov ListSeparator LS; 541f7581a39SAmir Ayupov for (BinaryBasicBlock *Pred : BB->predecessors()) 542f7581a39SAmir Ayupov OS << LS << Pred->getName(); 543a34c753fSRafael Auler OS << '\n'; 544a34c753fSRafael Auler } 545a34c753fSRafael Auler if (!BB->throw_empty()) { 546a34c753fSRafael Auler OS << " Throwers: "; 547f7581a39SAmir Ayupov ListSeparator LS; 548f7581a39SAmir Ayupov for (BinaryBasicBlock *Throw : BB->throwers()) 549f7581a39SAmir Ayupov OS << LS << Throw->getName(); 550a34c753fSRafael Auler OS << '\n'; 551a34c753fSRafael Auler } 552a34c753fSRafael Auler 553a34c753fSRafael Auler Offset = alignTo(Offset, BB->getAlignment()); 554a34c753fSRafael Auler 555a34c753fSRafael Auler // Note: offsets are imprecise since this is happening prior to relaxation. 556a34c753fSRafael Auler Offset = BC.printInstructions(OS, BB->begin(), BB->end(), Offset, this); 557a34c753fSRafael Auler 558a34c753fSRafael Auler if (!BB->succ_empty()) { 559a34c753fSRafael Auler OS << " Successors: "; 560a34c753fSRafael Auler // For more than 2 successors, sort them based on frequency. 561a34c753fSRafael Auler std::vector<uint64_t> Indices(BB->succ_size()); 562a34c753fSRafael Auler std::iota(Indices.begin(), Indices.end(), 0); 563a34c753fSRafael Auler if (BB->succ_size() > 2 && BB->getKnownExecutionCount()) { 564d2c87699SAmir Ayupov llvm::stable_sort(Indices, [&](const uint64_t A, const uint64_t B) { 565a34c753fSRafael Auler return BB->BranchInfo[B] < BB->BranchInfo[A]; 566a34c753fSRafael Auler }); 567a34c753fSRafael Auler } 568f7581a39SAmir Ayupov ListSeparator LS; 569a34c753fSRafael Auler for (unsigned I = 0; I < Indices.size(); ++I) { 570a34c753fSRafael Auler BinaryBasicBlock *Succ = BB->Successors[Indices[I]]; 571a34c753fSRafael Auler BinaryBasicBlock::BinaryBranchInfo &BI = BB->BranchInfo[Indices[I]]; 572f7581a39SAmir Ayupov OS << LS << Succ->getName(); 573a34c753fSRafael Auler if (ExecutionCount != COUNT_NO_PROFILE && 574a34c753fSRafael Auler BI.MispredictedCount != BinaryBasicBlock::COUNT_INFERRED) { 575a34c753fSRafael Auler OS << " (mispreds: " << BI.MispredictedCount 576a34c753fSRafael Auler << ", count: " << BI.Count << ")"; 577a34c753fSRafael Auler } else if (ExecutionCount != COUNT_NO_PROFILE && 578a34c753fSRafael Auler BI.Count != BinaryBasicBlock::COUNT_NO_PROFILE) { 579a34c753fSRafael Auler OS << " (inferred count: " << BI.Count << ")"; 580a34c753fSRafael Auler } 581a34c753fSRafael Auler } 582a34c753fSRafael Auler OS << '\n'; 583a34c753fSRafael Auler } 584a34c753fSRafael Auler 585a34c753fSRafael Auler if (!BB->lp_empty()) { 586a34c753fSRafael Auler OS << " Landing Pads: "; 587f7581a39SAmir Ayupov ListSeparator LS; 588a34c753fSRafael Auler for (BinaryBasicBlock *LP : BB->landing_pads()) { 589f7581a39SAmir Ayupov OS << LS << LP->getName(); 590a34c753fSRafael Auler if (ExecutionCount != COUNT_NO_PROFILE) { 591a34c753fSRafael Auler OS << " (count: " << LP->getExecutionCount() << ")"; 592a34c753fSRafael Auler } 593a34c753fSRafael Auler } 594a34c753fSRafael Auler OS << '\n'; 595a34c753fSRafael Auler } 596a34c753fSRafael Auler 597a34c753fSRafael Auler // In CFG_Finalized state we can miscalculate CFI state at exit. 598a34c753fSRafael Auler if (CurrentState == State::CFG) { 599a34c753fSRafael Auler const int32_t CFIStateAtExit = BB->getCFIStateAtExit(); 600a34c753fSRafael Auler if (CFIStateAtExit >= 0) 601a34c753fSRafael Auler OS << " CFI State: " << CFIStateAtExit << '\n'; 602a34c753fSRafael Auler } 603a34c753fSRafael Auler 604a34c753fSRafael Auler OS << '\n'; 605a34c753fSRafael Auler } 606a34c753fSRafael Auler 607a34c753fSRafael Auler // Dump new exception ranges for the function. 608a34c753fSRafael Auler if (!CallSites.empty()) { 609a34c753fSRafael Auler OS << "EH table:\n"; 610a34c753fSRafael Auler for (const CallSite &CSI : CallSites) { 611a34c753fSRafael Auler OS << " [" << *CSI.Start << ", " << *CSI.End << ") landing pad : "; 612a34c753fSRafael Auler if (CSI.LP) 613a34c753fSRafael Auler OS << *CSI.LP; 614a34c753fSRafael Auler else 615a34c753fSRafael Auler OS << "0"; 616a34c753fSRafael Auler OS << ", action : " << CSI.Action << '\n'; 617a34c753fSRafael Auler } 618a34c753fSRafael Auler OS << '\n'; 619a34c753fSRafael Auler } 620a34c753fSRafael Auler 621a34c753fSRafael Auler // Print all jump tables. 6223652483cSRafael Auler for (const std::pair<const uint64_t, JumpTable *> &JTI : JumpTables) 623a34c753fSRafael Auler JTI.second->print(OS); 624a34c753fSRafael Auler 625a34c753fSRafael Auler OS << "DWARF CFI Instructions:\n"; 626a34c753fSRafael Auler if (OffsetToCFI.size()) { 627a34c753fSRafael Auler // Pre-buildCFG information 628a34c753fSRafael Auler for (const std::pair<const uint32_t, uint32_t> &Elmt : OffsetToCFI) { 629a34c753fSRafael Auler OS << format(" %08x:\t", Elmt.first); 630a34c753fSRafael Auler assert(Elmt.second < FrameInstructions.size() && "Incorrect CFI offset"); 631a34c753fSRafael Auler BinaryContext::printCFI(OS, FrameInstructions[Elmt.second]); 632a34c753fSRafael Auler OS << "\n"; 633a34c753fSRafael Auler } 634a34c753fSRafael Auler } else { 635a34c753fSRafael Auler // Post-buildCFG information 636a34c753fSRafael Auler for (uint32_t I = 0, E = FrameInstructions.size(); I != E; ++I) { 637a34c753fSRafael Auler const MCCFIInstruction &CFI = FrameInstructions[I]; 638a34c753fSRafael Auler OS << format(" %d:\t", I); 639a34c753fSRafael Auler BinaryContext::printCFI(OS, CFI); 640a34c753fSRafael Auler OS << "\n"; 641a34c753fSRafael Auler } 642a34c753fSRafael Auler } 643a34c753fSRafael Auler if (FrameInstructions.empty()) 644a34c753fSRafael Auler OS << " <empty>\n"; 645a34c753fSRafael Auler 646a34c753fSRafael Auler OS << "End of Function \"" << *this << "\"\n\n"; 647a34c753fSRafael Auler } 648a34c753fSRafael Auler 64940c2e0faSMaksim Panchenko void BinaryFunction::printRelocations(raw_ostream &OS, uint64_t Offset, 650a34c753fSRafael Auler uint64_t Size) const { 651a34c753fSRafael Auler const char *Sep = " # Relocs: "; 652a34c753fSRafael Auler 653a34c753fSRafael Auler auto RI = Relocations.lower_bound(Offset); 654a34c753fSRafael Auler while (RI != Relocations.end() && RI->first < Offset + Size) { 655a34c753fSRafael Auler OS << Sep << "(R: " << RI->second << ")"; 656a34c753fSRafael Auler Sep = ", "; 657a34c753fSRafael Auler ++RI; 658a34c753fSRafael Auler } 659a34c753fSRafael Auler } 660a34c753fSRafael Auler 661a34c753fSRafael Auler namespace { 662a34c753fSRafael Auler std::string mutateDWARFExpressionTargetReg(const MCCFIInstruction &Instr, 663a34c753fSRafael Auler MCPhysReg NewReg) { 664a34c753fSRafael Auler StringRef ExprBytes = Instr.getValues(); 665a34c753fSRafael Auler assert(ExprBytes.size() > 1 && "DWARF expression CFI is too short"); 666a34c753fSRafael Auler uint8_t Opcode = ExprBytes[0]; 667a34c753fSRafael Auler assert((Opcode == dwarf::DW_CFA_expression || 668a34c753fSRafael Auler Opcode == dwarf::DW_CFA_val_expression) && 669a34c753fSRafael Auler "invalid DWARF expression CFI"); 670139744acSAmir Ayupov (void)Opcode; 671a34c753fSRafael Auler const uint8_t *const Start = 672a34c753fSRafael Auler reinterpret_cast<const uint8_t *>(ExprBytes.drop_front(1).data()); 673a34c753fSRafael Auler const uint8_t *const End = 674a34c753fSRafael Auler reinterpret_cast<const uint8_t *>(Start + ExprBytes.size() - 1); 675a34c753fSRafael Auler unsigned Size = 0; 676a34c753fSRafael Auler decodeULEB128(Start, &Size, End); 677a34c753fSRafael Auler assert(Size > 0 && "Invalid reg encoding for DWARF expression CFI"); 678a34c753fSRafael Auler SmallString<8> Tmp; 679a34c753fSRafael Auler raw_svector_ostream OSE(Tmp); 680a34c753fSRafael Auler encodeULEB128(NewReg, OSE); 681a34c753fSRafael Auler return Twine(ExprBytes.slice(0, 1)) 682a34c753fSRafael Auler .concat(OSE.str()) 683a34c753fSRafael Auler .concat(ExprBytes.drop_front(1 + Size)) 684a34c753fSRafael Auler .str(); 685a34c753fSRafael Auler } 686a34c753fSRafael Auler } // namespace 687a34c753fSRafael Auler 688a34c753fSRafael Auler void BinaryFunction::mutateCFIRegisterFor(const MCInst &Instr, 689a34c753fSRafael Auler MCPhysReg NewReg) { 690a34c753fSRafael Auler const MCCFIInstruction *OldCFI = getCFIFor(Instr); 691a34c753fSRafael Auler assert(OldCFI && "invalid CFI instr"); 692a34c753fSRafael Auler switch (OldCFI->getOperation()) { 693a34c753fSRafael Auler default: 694a34c753fSRafael Auler llvm_unreachable("Unexpected instruction"); 695a34c753fSRafael Auler case MCCFIInstruction::OpDefCfa: 696a34c753fSRafael Auler setCFIFor(Instr, MCCFIInstruction::cfiDefCfa(nullptr, NewReg, 697a34c753fSRafael Auler OldCFI->getOffset())); 698a34c753fSRafael Auler break; 699a34c753fSRafael Auler case MCCFIInstruction::OpDefCfaRegister: 700a34c753fSRafael Auler setCFIFor(Instr, MCCFIInstruction::createDefCfaRegister(nullptr, NewReg)); 701a34c753fSRafael Auler break; 702a34c753fSRafael Auler case MCCFIInstruction::OpOffset: 703a34c753fSRafael Auler setCFIFor(Instr, MCCFIInstruction::createOffset(nullptr, NewReg, 704a34c753fSRafael Auler OldCFI->getOffset())); 705a34c753fSRafael Auler break; 706a34c753fSRafael Auler case MCCFIInstruction::OpRegister: 707a34c753fSRafael Auler setCFIFor(Instr, MCCFIInstruction::createRegister(nullptr, NewReg, 708a34c753fSRafael Auler OldCFI->getRegister2())); 709a34c753fSRafael Auler break; 710a34c753fSRafael Auler case MCCFIInstruction::OpSameValue: 711a34c753fSRafael Auler setCFIFor(Instr, MCCFIInstruction::createSameValue(nullptr, NewReg)); 712a34c753fSRafael Auler break; 713a34c753fSRafael Auler case MCCFIInstruction::OpEscape: 714a34c753fSRafael Auler setCFIFor(Instr, 715a34c753fSRafael Auler MCCFIInstruction::createEscape( 716a34c753fSRafael Auler nullptr, 717a34c753fSRafael Auler StringRef(mutateDWARFExpressionTargetReg(*OldCFI, NewReg)))); 718a34c753fSRafael Auler break; 719a34c753fSRafael Auler case MCCFIInstruction::OpRestore: 720a34c753fSRafael Auler setCFIFor(Instr, MCCFIInstruction::createRestore(nullptr, NewReg)); 721a34c753fSRafael Auler break; 722a34c753fSRafael Auler case MCCFIInstruction::OpUndefined: 723a34c753fSRafael Auler setCFIFor(Instr, MCCFIInstruction::createUndefined(nullptr, NewReg)); 724a34c753fSRafael Auler break; 725a34c753fSRafael Auler } 726a34c753fSRafael Auler } 727a34c753fSRafael Auler 728a34c753fSRafael Auler const MCCFIInstruction *BinaryFunction::mutateCFIOffsetFor(const MCInst &Instr, 729a34c753fSRafael Auler int64_t NewOffset) { 730a34c753fSRafael Auler const MCCFIInstruction *OldCFI = getCFIFor(Instr); 731a34c753fSRafael Auler assert(OldCFI && "invalid CFI instr"); 732a34c753fSRafael Auler switch (OldCFI->getOperation()) { 733a34c753fSRafael Auler default: 734a34c753fSRafael Auler llvm_unreachable("Unexpected instruction"); 735a34c753fSRafael Auler case MCCFIInstruction::OpDefCfaOffset: 736a34c753fSRafael Auler setCFIFor(Instr, MCCFIInstruction::cfiDefCfaOffset(nullptr, NewOffset)); 737a34c753fSRafael Auler break; 738a34c753fSRafael Auler case MCCFIInstruction::OpAdjustCfaOffset: 739a34c753fSRafael Auler setCFIFor(Instr, 740a34c753fSRafael Auler MCCFIInstruction::createAdjustCfaOffset(nullptr, NewOffset)); 741a34c753fSRafael Auler break; 742a34c753fSRafael Auler case MCCFIInstruction::OpDefCfa: 743a34c753fSRafael Auler setCFIFor(Instr, MCCFIInstruction::cfiDefCfa(nullptr, OldCFI->getRegister(), 744a34c753fSRafael Auler NewOffset)); 745a34c753fSRafael Auler break; 746a34c753fSRafael Auler case MCCFIInstruction::OpOffset: 747a34c753fSRafael Auler setCFIFor(Instr, MCCFIInstruction::createOffset( 748a34c753fSRafael Auler nullptr, OldCFI->getRegister(), NewOffset)); 749a34c753fSRafael Auler break; 750a34c753fSRafael Auler } 751a34c753fSRafael Auler return getCFIFor(Instr); 752a34c753fSRafael Auler } 753a34c753fSRafael Auler 754a34c753fSRafael Auler IndirectBranchType 75540c2e0faSMaksim Panchenko BinaryFunction::processIndirectBranch(MCInst &Instruction, unsigned Size, 756a34c753fSRafael Auler uint64_t Offset, 757a34c753fSRafael Auler uint64_t &TargetAddress) { 758a34c753fSRafael Auler const unsigned PtrSize = BC.AsmInfo->getCodePointerSize(); 759a34c753fSRafael Auler 760a34c753fSRafael Auler // The instruction referencing memory used by the branch instruction. 761a34c753fSRafael Auler // It could be the branch instruction itself or one of the instructions 762a34c753fSRafael Auler // setting the value of the register used by the branch. 763a34c753fSRafael Auler MCInst *MemLocInstr; 764a34c753fSRafael Auler 765a34c753fSRafael Auler // Address of the table referenced by MemLocInstr. Could be either an 766a34c753fSRafael Auler // array of function pointers, or a jump table. 767a34c753fSRafael Auler uint64_t ArrayStart = 0; 768a34c753fSRafael Auler 769a34c753fSRafael Auler unsigned BaseRegNum, IndexRegNum; 770a34c753fSRafael Auler int64_t DispValue; 771a34c753fSRafael Auler const MCExpr *DispExpr; 772a34c753fSRafael Auler 773a34c753fSRafael Auler // In AArch, identify the instruction adding the PC-relative offset to 774a34c753fSRafael Auler // jump table entries to correctly decode it. 775a34c753fSRafael Auler MCInst *PCRelBaseInstr; 776a34c753fSRafael Auler uint64_t PCRelAddr = 0; 777a34c753fSRafael Auler 778a34c753fSRafael Auler auto Begin = Instructions.begin(); 779a34c753fSRafael Auler if (BC.isAArch64()) { 780a34c753fSRafael Auler PreserveNops = BC.HasRelocations; 781a34c753fSRafael Auler // Start at the last label as an approximation of the current basic block. 782a34c753fSRafael Auler // This is a heuristic, since the full set of labels have yet to be 783a34c753fSRafael Auler // determined 784a34c753fSRafael Auler for (auto LI = Labels.rbegin(); LI != Labels.rend(); ++LI) { 785a34c753fSRafael Auler auto II = Instructions.find(LI->first); 786a34c753fSRafael Auler if (II != Instructions.end()) { 787a34c753fSRafael Auler Begin = II; 788a34c753fSRafael Auler break; 789a34c753fSRafael Auler } 790a34c753fSRafael Auler } 791a34c753fSRafael Auler } 792a34c753fSRafael Auler 79340c2e0faSMaksim Panchenko IndirectBranchType BranchType = BC.MIB->analyzeIndirectBranch( 79440c2e0faSMaksim Panchenko Instruction, Begin, Instructions.end(), PtrSize, MemLocInstr, BaseRegNum, 79540c2e0faSMaksim Panchenko IndexRegNum, DispValue, DispExpr, PCRelBaseInstr); 796a34c753fSRafael Auler 797a34c753fSRafael Auler if (BranchType == IndirectBranchType::UNKNOWN && !MemLocInstr) 798a34c753fSRafael Auler return BranchType; 799a34c753fSRafael Auler 800a34c753fSRafael Auler if (MemLocInstr != &Instruction) 801a34c753fSRafael Auler IndexRegNum = BC.MIB->getNoRegister(); 802a34c753fSRafael Auler 803a34c753fSRafael Auler if (BC.isAArch64()) { 804a34c753fSRafael Auler const MCSymbol *Sym = BC.MIB->getTargetSymbol(*PCRelBaseInstr, 1); 805a34c753fSRafael Auler assert(Sym && "Symbol extraction failed"); 806a34c753fSRafael Auler ErrorOr<uint64_t> SymValueOrError = BC.getSymbolValue(*Sym); 807a34c753fSRafael Auler if (SymValueOrError) { 808a34c753fSRafael Auler PCRelAddr = *SymValueOrError; 809a34c753fSRafael Auler } else { 810a34c753fSRafael Auler for (std::pair<const uint32_t, MCSymbol *> &Elmt : Labels) { 811a34c753fSRafael Auler if (Elmt.second == Sym) { 812a34c753fSRafael Auler PCRelAddr = Elmt.first + getAddress(); 813a34c753fSRafael Auler break; 814a34c753fSRafael Auler } 815a34c753fSRafael Auler } 816a34c753fSRafael Auler } 817a34c753fSRafael Auler uint64_t InstrAddr = 0; 818a34c753fSRafael Auler for (auto II = Instructions.rbegin(); II != Instructions.rend(); ++II) { 819a34c753fSRafael Auler if (&II->second == PCRelBaseInstr) { 820a34c753fSRafael Auler InstrAddr = II->first + getAddress(); 821a34c753fSRafael Auler break; 822a34c753fSRafael Auler } 823a34c753fSRafael Auler } 824a34c753fSRafael Auler assert(InstrAddr != 0 && "instruction not found"); 825a34c753fSRafael Auler // We do this to avoid spurious references to code locations outside this 826a34c753fSRafael Auler // function (for example, if the indirect jump lives in the last basic 827a34c753fSRafael Auler // block of the function, it will create a reference to the next function). 828a34c753fSRafael Auler // This replaces a symbol reference with an immediate. 829a34c753fSRafael Auler BC.MIB->replaceMemOperandDisp(*PCRelBaseInstr, 830a34c753fSRafael Auler MCOperand::createImm(PCRelAddr - InstrAddr)); 831a34c753fSRafael Auler // FIXME: Disable full jump table processing for AArch64 until we have a 832a34c753fSRafael Auler // proper way of determining the jump table limits. 833a34c753fSRafael Auler return IndirectBranchType::UNKNOWN; 834a34c753fSRafael Auler } 835a34c753fSRafael Auler 836a34c753fSRafael Auler // RIP-relative addressing should be converted to symbol form by now 837a34c753fSRafael Auler // in processed instructions (but not in jump). 838a34c753fSRafael Auler if (DispExpr) { 839a34c753fSRafael Auler const MCSymbol *TargetSym; 840a34c753fSRafael Auler uint64_t TargetOffset; 841a34c753fSRafael Auler std::tie(TargetSym, TargetOffset) = BC.MIB->getTargetSymbolInfo(DispExpr); 842a34c753fSRafael Auler ErrorOr<uint64_t> SymValueOrError = BC.getSymbolValue(*TargetSym); 843a34c753fSRafael Auler assert(SymValueOrError && "global symbol needs a value"); 844a34c753fSRafael Auler ArrayStart = *SymValueOrError + TargetOffset; 845a34c753fSRafael Auler BaseRegNum = BC.MIB->getNoRegister(); 846a34c753fSRafael Auler if (BC.isAArch64()) { 847a34c753fSRafael Auler ArrayStart &= ~0xFFFULL; 848a34c753fSRafael Auler ArrayStart += DispValue & 0xFFFULL; 849a34c753fSRafael Auler } 850a34c753fSRafael Auler } else { 851a34c753fSRafael Auler ArrayStart = static_cast<uint64_t>(DispValue); 852a34c753fSRafael Auler } 853a34c753fSRafael Auler 854a34c753fSRafael Auler if (BaseRegNum == BC.MRI->getProgramCounter()) 855a34c753fSRafael Auler ArrayStart += getAddress() + Offset + Size; 856a34c753fSRafael Auler 857a34c753fSRafael Auler LLVM_DEBUG(dbgs() << "BOLT-DEBUG: addressed memory is 0x" 858a34c753fSRafael Auler << Twine::utohexstr(ArrayStart) << '\n'); 859a34c753fSRafael Auler 860a34c753fSRafael Auler ErrorOr<BinarySection &> Section = BC.getSectionForAddress(ArrayStart); 861a34c753fSRafael Auler if (!Section) { 862a34c753fSRafael Auler // No section - possibly an absolute address. Since we don't allow 863a34c753fSRafael Auler // internal function addresses to escape the function scope - we 864a34c753fSRafael Auler // consider it a tail call. 865a34c753fSRafael Auler if (opts::Verbosity >= 1) { 866a34c753fSRafael Auler errs() << "BOLT-WARNING: no section for address 0x" 867a34c753fSRafael Auler << Twine::utohexstr(ArrayStart) << " referenced from function " 868a34c753fSRafael Auler << *this << '\n'; 869a34c753fSRafael Auler } 870a34c753fSRafael Auler return IndirectBranchType::POSSIBLE_TAIL_CALL; 871a34c753fSRafael Auler } 872a34c753fSRafael Auler if (Section->isVirtual()) { 873a34c753fSRafael Auler // The contents are filled at runtime. 874a34c753fSRafael Auler return IndirectBranchType::POSSIBLE_TAIL_CALL; 875a34c753fSRafael Auler } 876a34c753fSRafael Auler 877a34c753fSRafael Auler if (BranchType == IndirectBranchType::POSSIBLE_FIXED_BRANCH) { 878a34c753fSRafael Auler ErrorOr<uint64_t> Value = BC.getPointerAtAddress(ArrayStart); 879a34c753fSRafael Auler if (!Value) 880a34c753fSRafael Auler return IndirectBranchType::UNKNOWN; 881a34c753fSRafael Auler 882a34c753fSRafael Auler if (!BC.getSectionForAddress(ArrayStart)->isReadOnly()) 883a34c753fSRafael Auler return IndirectBranchType::UNKNOWN; 884a34c753fSRafael Auler 885a34c753fSRafael Auler outs() << "BOLT-INFO: fixed indirect branch detected in " << *this 886a34c753fSRafael Auler << " at 0x" << Twine::utohexstr(getAddress() + Offset) 887a34c753fSRafael Auler << " referencing data at 0x" << Twine::utohexstr(ArrayStart) 888a34c753fSRafael Auler << " the destination value is 0x" << Twine::utohexstr(*Value) 889a34c753fSRafael Auler << '\n'; 890a34c753fSRafael Auler 891a34c753fSRafael Auler TargetAddress = *Value; 892a34c753fSRafael Auler return BranchType; 893a34c753fSRafael Auler } 894a34c753fSRafael Auler 895a34c753fSRafael Auler // Check if there's already a jump table registered at this address. 896a34c753fSRafael Auler MemoryContentsType MemType; 897a34c753fSRafael Auler if (JumpTable *JT = BC.getJumpTableContainingAddress(ArrayStart)) { 898a34c753fSRafael Auler switch (JT->Type) { 899a34c753fSRafael Auler case JumpTable::JTT_NORMAL: 900a34c753fSRafael Auler MemType = MemoryContentsType::POSSIBLE_JUMP_TABLE; 901a34c753fSRafael Auler break; 902a34c753fSRafael Auler case JumpTable::JTT_PIC: 903a34c753fSRafael Auler MemType = MemoryContentsType::POSSIBLE_PIC_JUMP_TABLE; 904a34c753fSRafael Auler break; 905a34c753fSRafael Auler } 906a34c753fSRafael Auler } else { 907a34c753fSRafael Auler MemType = BC.analyzeMemoryAt(ArrayStart, *this); 908a34c753fSRafael Auler } 909a34c753fSRafael Auler 910a34c753fSRafael Auler // Check that jump table type in instruction pattern matches memory contents. 911a34c753fSRafael Auler JumpTable::JumpTableType JTType; 912a34c753fSRafael Auler if (BranchType == IndirectBranchType::POSSIBLE_PIC_JUMP_TABLE) { 913a34c753fSRafael Auler if (MemType != MemoryContentsType::POSSIBLE_PIC_JUMP_TABLE) 914a34c753fSRafael Auler return IndirectBranchType::UNKNOWN; 915a34c753fSRafael Auler JTType = JumpTable::JTT_PIC; 916a34c753fSRafael Auler } else { 917a34c753fSRafael Auler if (MemType == MemoryContentsType::POSSIBLE_PIC_JUMP_TABLE) 918a34c753fSRafael Auler return IndirectBranchType::UNKNOWN; 919a34c753fSRafael Auler 920a34c753fSRafael Auler if (MemType == MemoryContentsType::UNKNOWN) 921a34c753fSRafael Auler return IndirectBranchType::POSSIBLE_TAIL_CALL; 922a34c753fSRafael Auler 923a34c753fSRafael Auler BranchType = IndirectBranchType::POSSIBLE_JUMP_TABLE; 924a34c753fSRafael Auler JTType = JumpTable::JTT_NORMAL; 925a34c753fSRafael Auler } 926a34c753fSRafael Auler 927a34c753fSRafael Auler // Convert the instruction into jump table branch. 928a34c753fSRafael Auler const MCSymbol *JTLabel = BC.getOrCreateJumpTable(*this, ArrayStart, JTType); 929a34c753fSRafael Auler BC.MIB->replaceMemOperandDisp(*MemLocInstr, JTLabel, BC.Ctx.get()); 930a34c753fSRafael Auler BC.MIB->setJumpTable(Instruction, ArrayStart, IndexRegNum); 931a34c753fSRafael Auler 932a34c753fSRafael Auler JTSites.emplace_back(Offset, ArrayStart); 933a34c753fSRafael Auler 934a34c753fSRafael Auler return BranchType; 935a34c753fSRafael Auler } 936a34c753fSRafael Auler 937a34c753fSRafael Auler MCSymbol *BinaryFunction::getOrCreateLocalLabel(uint64_t Address, 938a34c753fSRafael Auler bool CreatePastEnd) { 939a34c753fSRafael Auler const uint64_t Offset = Address - getAddress(); 940a34c753fSRafael Auler 941a34c753fSRafael Auler if ((Offset == getSize()) && CreatePastEnd) 942a34c753fSRafael Auler return getFunctionEndLabel(); 943a34c753fSRafael Auler 944a34c753fSRafael Auler auto LI = Labels.find(Offset); 945a34c753fSRafael Auler if (LI != Labels.end()) 946a34c753fSRafael Auler return LI->second; 947a34c753fSRafael Auler 948a34c753fSRafael Auler // For AArch64, check if this address is part of a constant island. 949a34c753fSRafael Auler if (BC.isAArch64()) { 9503652483cSRafael Auler if (MCSymbol *IslandSym = getOrCreateIslandAccess(Address)) 951a34c753fSRafael Auler return IslandSym; 952a34c753fSRafael Auler } 953a34c753fSRafael Auler 954a34c753fSRafael Auler MCSymbol *Label = BC.Ctx->createNamedTempSymbol(); 955a34c753fSRafael Auler Labels[Offset] = Label; 956a34c753fSRafael Auler 957a34c753fSRafael Auler return Label; 958a34c753fSRafael Auler } 959a34c753fSRafael Auler 960a34c753fSRafael Auler ErrorOr<ArrayRef<uint8_t>> BinaryFunction::getData() const { 961a34c753fSRafael Auler BinarySection &Section = *getOriginSection(); 962a34c753fSRafael Auler assert(Section.containsRange(getAddress(), getMaxSize()) && 963a34c753fSRafael Auler "wrong section for function"); 964a34c753fSRafael Auler 9653652483cSRafael Auler if (!Section.isText() || Section.isVirtual() || !Section.getSize()) 966a34c753fSRafael Auler return std::make_error_code(std::errc::bad_address); 967a34c753fSRafael Auler 968a34c753fSRafael Auler StringRef SectionContents = Section.getContents(); 969a34c753fSRafael Auler 970a34c753fSRafael Auler assert(SectionContents.size() == Section.getSize() && 971a34c753fSRafael Auler "section size mismatch"); 972a34c753fSRafael Auler 973a34c753fSRafael Auler // Function offset from the section start. 974a34c753fSRafael Auler uint64_t Offset = getAddress() - Section.getAddress(); 975a34c753fSRafael Auler auto *Bytes = reinterpret_cast<const uint8_t *>(SectionContents.data()); 976a34c753fSRafael Auler return ArrayRef<uint8_t>(Bytes + Offset, getMaxSize()); 977a34c753fSRafael Auler } 978a34c753fSRafael Auler 979a34c753fSRafael Auler size_t BinaryFunction::getSizeOfDataInCodeAt(uint64_t Offset) const { 980a34c753fSRafael Auler if (!Islands) 981a34c753fSRafael Auler return 0; 982a34c753fSRafael Auler 983a34c753fSRafael Auler if (Islands->DataOffsets.find(Offset) == Islands->DataOffsets.end()) 984a34c753fSRafael Auler return 0; 985a34c753fSRafael Auler 986a34c753fSRafael Auler auto Iter = Islands->CodeOffsets.upper_bound(Offset); 9873652483cSRafael Auler if (Iter != Islands->CodeOffsets.end()) 988a34c753fSRafael Auler return *Iter - Offset; 989a34c753fSRafael Auler return getSize() - Offset; 990a34c753fSRafael Auler } 991a34c753fSRafael Auler 992a34c753fSRafael Auler bool BinaryFunction::isZeroPaddingAt(uint64_t Offset) const { 993a34c753fSRafael Auler ArrayRef<uint8_t> FunctionData = *getData(); 994a34c753fSRafael Auler uint64_t EndOfCode = getSize(); 995a34c753fSRafael Auler if (Islands) { 996a34c753fSRafael Auler auto Iter = Islands->DataOffsets.upper_bound(Offset); 997a34c753fSRafael Auler if (Iter != Islands->DataOffsets.end()) 998a34c753fSRafael Auler EndOfCode = *Iter; 999a34c753fSRafael Auler } 10003652483cSRafael Auler for (uint64_t I = Offset; I < EndOfCode; ++I) 10013652483cSRafael Auler if (FunctionData[I] != 0) 1002a34c753fSRafael Auler return false; 1003a34c753fSRafael Auler 1004a34c753fSRafael Auler return true; 1005a34c753fSRafael Auler } 1006a34c753fSRafael Auler 1007a34c753fSRafael Auler bool BinaryFunction::disassemble() { 1008a34c753fSRafael Auler NamedRegionTimer T("disassemble", "Disassemble function", "buildfuncs", 1009a34c753fSRafael Auler "Build Binary Functions", opts::TimeBuild); 1010a34c753fSRafael Auler ErrorOr<ArrayRef<uint8_t>> ErrorOrFunctionData = getData(); 1011a34c753fSRafael Auler assert(ErrorOrFunctionData && "function data is not available"); 1012a34c753fSRafael Auler ArrayRef<uint8_t> FunctionData = *ErrorOrFunctionData; 1013a34c753fSRafael Auler assert(FunctionData.size() == getMaxSize() && 1014a34c753fSRafael Auler "function size does not match raw data size"); 1015a34c753fSRafael Auler 1016a34c753fSRafael Auler auto &Ctx = BC.Ctx; 1017a34c753fSRafael Auler auto &MIB = BC.MIB; 1018a34c753fSRafael Auler 1019e290133cSMaksim Panchenko BC.SymbolicDisAsm->setSymbolizer(MIB->createTargetSymbolizer(*this)); 1020e290133cSMaksim Panchenko 1021a34c753fSRafael Auler // Insert a label at the beginning of the function. This will be our first 1022a34c753fSRafael Auler // basic block. 1023a34c753fSRafael Auler Labels[0] = Ctx->createNamedTempSymbol("BB0"); 1024a34c753fSRafael Auler 102540c2e0faSMaksim Panchenko auto handlePCRelOperand = [&](MCInst &Instruction, uint64_t Address, 102640c2e0faSMaksim Panchenko uint64_t Size) { 1027a34c753fSRafael Auler uint64_t TargetAddress = 0; 1028a34c753fSRafael Auler if (!MIB->evaluateMemOperandTarget(Instruction, TargetAddress, Address, 1029a34c753fSRafael Auler Size)) { 1030a34c753fSRafael Auler errs() << "BOLT-ERROR: PC-relative operand can't be evaluated:\n"; 1031a34c753fSRafael Auler BC.InstPrinter->printInst(&Instruction, 0, "", *BC.STI, errs()); 1032a34c753fSRafael Auler errs() << '\n'; 1033a34c753fSRafael Auler Instruction.dump_pretty(errs(), BC.InstPrinter.get()); 1034a34c753fSRafael Auler errs() << '\n'; 1035a34c753fSRafael Auler errs() << "BOLT-ERROR: cannot handle PC-relative operand at 0x" 1036a34c753fSRafael Auler << Twine::utohexstr(Address) << ". Skipping function " << *this 1037a34c753fSRafael Auler << ".\n"; 1038a34c753fSRafael Auler if (BC.HasRelocations) 1039a34c753fSRafael Auler exit(1); 1040a34c753fSRafael Auler IsSimple = false; 1041a34c753fSRafael Auler return; 1042a34c753fSRafael Auler } 1043a34c753fSRafael Auler if (TargetAddress == 0 && opts::Verbosity >= 1) { 1044a34c753fSRafael Auler outs() << "BOLT-INFO: PC-relative operand is zero in function " << *this 1045a34c753fSRafael Auler << '\n'; 1046a34c753fSRafael Auler } 1047a34c753fSRafael Auler 1048a34c753fSRafael Auler const MCSymbol *TargetSymbol; 1049a34c753fSRafael Auler uint64_t TargetOffset; 1050a34c753fSRafael Auler std::tie(TargetSymbol, TargetOffset) = 1051a34c753fSRafael Auler BC.handleAddressRef(TargetAddress, *this, /*IsPCRel*/ true); 105240c2e0faSMaksim Panchenko const MCExpr *Expr = MCSymbolRefExpr::create( 105340c2e0faSMaksim Panchenko TargetSymbol, MCSymbolRefExpr::VK_None, *BC.Ctx); 1054a34c753fSRafael Auler if (TargetOffset) { 1055a34c753fSRafael Auler const MCConstantExpr *Offset = 1056a34c753fSRafael Auler MCConstantExpr::create(TargetOffset, *BC.Ctx); 1057a34c753fSRafael Auler Expr = MCBinaryExpr::createAdd(Expr, Offset, *BC.Ctx); 1058a34c753fSRafael Auler } 105940c2e0faSMaksim Panchenko MIB->replaceMemOperandDisp(Instruction, 106040c2e0faSMaksim Panchenko MCOperand::createExpr(BC.MIB->getTargetExprFor( 106140c2e0faSMaksim Panchenko Instruction, Expr, *BC.Ctx, 0))); 1062a34c753fSRafael Auler }; 1063a34c753fSRafael Auler 1064a34c753fSRafael Auler auto handleExternalReference = [&](MCInst &Instruction, uint64_t Size, 1065a34c753fSRafael Auler uint64_t Offset, uint64_t TargetAddress, 1066a34c753fSRafael Auler bool &IsCall) -> MCSymbol * { 1067a34c753fSRafael Auler const uint64_t AbsoluteInstrAddr = getAddress() + Offset; 1068a34c753fSRafael Auler MCSymbol *TargetSymbol = nullptr; 106935efe1d8SVladislav Khmelevsky BC.addInterproceduralReference(this, TargetAddress); 1070a34c753fSRafael Auler if (opts::Verbosity >= 2 && !IsCall && Size == 2 && !BC.HasRelocations) { 1071a34c753fSRafael Auler errs() << "BOLT-WARNING: relaxed tail call detected at 0x" 1072a34c753fSRafael Auler << Twine::utohexstr(AbsoluteInstrAddr) << " in function " << *this 1073a34c753fSRafael Auler << ". Code size will be increased.\n"; 1074a34c753fSRafael Auler } 1075a34c753fSRafael Auler 1076a34c753fSRafael Auler assert(!MIB->isTailCall(Instruction) && 1077a34c753fSRafael Auler "synthetic tail call instruction found"); 1078a34c753fSRafael Auler 1079a34c753fSRafael Auler // This is a call regardless of the opcode. 1080a34c753fSRafael Auler // Assign proper opcode for tail calls, so that they could be 1081a34c753fSRafael Auler // treated as calls. 1082a34c753fSRafael Auler if (!IsCall) { 1083a34c753fSRafael Auler if (!MIB->convertJmpToTailCall(Instruction)) { 1084139744acSAmir Ayupov assert(MIB->isConditionalBranch(Instruction) && 1085139744acSAmir Ayupov "unknown tail call instruction"); 1086a34c753fSRafael Auler if (opts::Verbosity >= 2) { 1087a34c753fSRafael Auler errs() << "BOLT-WARNING: conditional tail call detected in " 1088a34c753fSRafael Auler << "function " << *this << " at 0x" 1089a34c753fSRafael Auler << Twine::utohexstr(AbsoluteInstrAddr) << ".\n"; 1090a34c753fSRafael Auler } 1091a34c753fSRafael Auler } 1092a34c753fSRafael Auler IsCall = true; 1093a34c753fSRafael Auler } 1094a34c753fSRafael Auler 1095a34c753fSRafael Auler TargetSymbol = BC.getOrCreateGlobalSymbol(TargetAddress, "FUNCat"); 1096a34c753fSRafael Auler if (opts::Verbosity >= 2 && TargetAddress == 0) { 1097a34c753fSRafael Auler // We actually see calls to address 0 in presence of weak 1098a34c753fSRafael Auler // symbols originating from libraries. This code is never meant 1099a34c753fSRafael Auler // to be executed. 1100a34c753fSRafael Auler outs() << "BOLT-INFO: Function " << *this 1101a34c753fSRafael Auler << " has a call to address zero.\n"; 1102a34c753fSRafael Auler } 1103a34c753fSRafael Auler 1104a34c753fSRafael Auler return TargetSymbol; 1105a34c753fSRafael Auler }; 1106a34c753fSRafael Auler 1107a34c753fSRafael Auler auto handleIndirectBranch = [&](MCInst &Instruction, uint64_t Size, 1108a34c753fSRafael Auler uint64_t Offset) { 1109a34c753fSRafael Auler uint64_t IndirectTarget = 0; 1110a34c753fSRafael Auler IndirectBranchType Result = 1111a34c753fSRafael Auler processIndirectBranch(Instruction, Size, Offset, IndirectTarget); 1112a34c753fSRafael Auler switch (Result) { 1113a34c753fSRafael Auler default: 1114a34c753fSRafael Auler llvm_unreachable("unexpected result"); 1115a34c753fSRafael Auler case IndirectBranchType::POSSIBLE_TAIL_CALL: { 1116a34c753fSRafael Auler bool Result = MIB->convertJmpToTailCall(Instruction); 1117a34c753fSRafael Auler (void)Result; 1118a34c753fSRafael Auler assert(Result); 1119a34c753fSRafael Auler break; 1120a34c753fSRafael Auler } 1121a34c753fSRafael Auler case IndirectBranchType::POSSIBLE_JUMP_TABLE: 1122a34c753fSRafael Auler case IndirectBranchType::POSSIBLE_PIC_JUMP_TABLE: 1123a34c753fSRafael Auler if (opts::JumpTables == JTS_NONE) 1124a34c753fSRafael Auler IsSimple = false; 1125a34c753fSRafael Auler break; 1126a34c753fSRafael Auler case IndirectBranchType::POSSIBLE_FIXED_BRANCH: { 1127a34c753fSRafael Auler if (containsAddress(IndirectTarget)) { 1128a34c753fSRafael Auler const MCSymbol *TargetSymbol = getOrCreateLocalLabel(IndirectTarget); 1129a34c753fSRafael Auler Instruction.clear(); 1130a34c753fSRafael Auler MIB->createUncondBranch(Instruction, TargetSymbol, BC.Ctx.get()); 1131a34c753fSRafael Auler TakenBranches.emplace_back(Offset, IndirectTarget - getAddress()); 1132a34c753fSRafael Auler HasFixedIndirectBranch = true; 1133a34c753fSRafael Auler } else { 1134a34c753fSRafael Auler MIB->convertJmpToTailCall(Instruction); 113535efe1d8SVladislav Khmelevsky BC.addInterproceduralReference(this, IndirectTarget); 1136a34c753fSRafael Auler } 1137a34c753fSRafael Auler break; 1138a34c753fSRafael Auler } 1139a34c753fSRafael Auler case IndirectBranchType::UNKNOWN: 1140a34c753fSRafael Auler // Keep processing. We'll do more checks and fixes in 1141a34c753fSRafael Auler // postProcessIndirectBranches(). 1142a34c753fSRafael Auler UnknownIndirectBranchOffsets.emplace(Offset); 1143a34c753fSRafael Auler break; 1144a34c753fSRafael Auler } 1145a34c753fSRafael Auler }; 1146a34c753fSRafael Auler 1147a34c753fSRafael Auler // Check for linker veneers, which lack relocations and need manual 1148a34c753fSRafael Auler // adjustments. 1149a34c753fSRafael Auler auto handleAArch64IndirectCall = [&](MCInst &Instruction, uint64_t Offset) { 1150a34c753fSRafael Auler const uint64_t AbsoluteInstrAddr = getAddress() + Offset; 1151a34c753fSRafael Auler MCInst *TargetHiBits, *TargetLowBits; 115235efe1d8SVladislav Khmelevsky uint64_t TargetAddress, Count; 115335efe1d8SVladislav Khmelevsky Count = MIB->matchLinkerVeneer(Instructions.begin(), Instructions.end(), 1154a34c753fSRafael Auler AbsoluteInstrAddr, Instruction, TargetHiBits, 115535efe1d8SVladislav Khmelevsky TargetLowBits, TargetAddress); 115635efe1d8SVladislav Khmelevsky if (Count) { 1157a34c753fSRafael Auler MIB->addAnnotation(Instruction, "AArch64Veneer", true); 115835efe1d8SVladislav Khmelevsky --Count; 115935efe1d8SVladislav Khmelevsky for (auto It = std::prev(Instructions.end()); Count != 0; 116035efe1d8SVladislav Khmelevsky It = std::prev(It), --Count) { 1161a34c753fSRafael Auler MIB->addAnnotation(It->second, "AArch64Veneer", true); 1162a34c753fSRafael Auler } 1163a34c753fSRafael Auler 116435efe1d8SVladislav Khmelevsky BC.addAdrpAddRelocAArch64(*this, *TargetLowBits, *TargetHiBits, 116535efe1d8SVladislav Khmelevsky TargetAddress); 1166a34c753fSRafael Auler } 1167a34c753fSRafael Auler }; 1168a34c753fSRafael Auler 1169a34c753fSRafael Auler uint64_t Size = 0; // instruction size 1170a34c753fSRafael Auler for (uint64_t Offset = 0; Offset < getSize(); Offset += Size) { 1171a34c753fSRafael Auler MCInst Instruction; 1172a34c753fSRafael Auler const uint64_t AbsoluteInstrAddr = getAddress() + Offset; 1173a34c753fSRafael Auler 1174a34c753fSRafael Auler // Check for data inside code and ignore it 1175a34c753fSRafael Auler if (const size_t DataInCodeSize = getSizeOfDataInCodeAt(Offset)) { 1176a34c753fSRafael Auler Size = DataInCodeSize; 1177a34c753fSRafael Auler continue; 1178a34c753fSRafael Auler } 1179a34c753fSRafael Auler 1180e290133cSMaksim Panchenko if (!BC.SymbolicDisAsm->getInstruction(Instruction, Size, 1181a34c753fSRafael Auler FunctionData.slice(Offset), 118240c2e0faSMaksim Panchenko AbsoluteInstrAddr, nulls())) { 1183a34c753fSRafael Auler // Functions with "soft" boundaries, e.g. coming from assembly source, 1184a34c753fSRafael Auler // can have 0-byte padding at the end. 1185a34c753fSRafael Auler if (isZeroPaddingAt(Offset)) 1186a34c753fSRafael Auler break; 1187a34c753fSRafael Auler 1188a34c753fSRafael Auler errs() << "BOLT-WARNING: unable to disassemble instruction at offset 0x" 1189a34c753fSRafael Auler << Twine::utohexstr(Offset) << " (address 0x" 119040c2e0faSMaksim Panchenko << Twine::utohexstr(AbsoluteInstrAddr) << ") in function " << *this 119140c2e0faSMaksim Panchenko << '\n'; 1192a34c753fSRafael Auler // Some AVX-512 instructions could not be disassembled at all. 1193a34c753fSRafael Auler if (BC.HasRelocations && opts::TrapOnAVX512 && BC.isX86()) { 1194a34c753fSRafael Auler setTrapOnEntry(); 1195a34c753fSRafael Auler BC.TrappedFunctions.push_back(this); 1196a34c753fSRafael Auler } else { 1197a34c753fSRafael Auler setIgnored(); 1198a34c753fSRafael Auler } 1199a34c753fSRafael Auler 1200a34c753fSRafael Auler break; 1201a34c753fSRafael Auler } 1202a34c753fSRafael Auler 1203a34c753fSRafael Auler // Check integrity of LLVM assembler/disassembler. 1204a34c753fSRafael Auler if (opts::CheckEncoding && !BC.MIB->isBranch(Instruction) && 1205a34c753fSRafael Auler !BC.MIB->isCall(Instruction) && !BC.MIB->isNoop(Instruction)) { 1206a34c753fSRafael Auler if (!BC.validateEncoding(Instruction, FunctionData.slice(Offset, Size))) { 1207a34c753fSRafael Auler errs() << "BOLT-WARNING: mismatching LLVM encoding detected in " 1208a34c753fSRafael Auler << "function " << *this << " for instruction :\n"; 1209a34c753fSRafael Auler BC.printInstruction(errs(), Instruction, AbsoluteInstrAddr); 1210a34c753fSRafael Auler errs() << '\n'; 1211a34c753fSRafael Auler } 1212a34c753fSRafael Auler } 1213a34c753fSRafael Auler 1214a34c753fSRafael Auler // Special handling for AVX-512 instructions. 1215a34c753fSRafael Auler if (MIB->hasEVEXEncoding(Instruction)) { 1216a34c753fSRafael Auler if (BC.HasRelocations && opts::TrapOnAVX512) { 1217a34c753fSRafael Auler setTrapOnEntry(); 1218a34c753fSRafael Auler BC.TrappedFunctions.push_back(this); 1219a34c753fSRafael Auler break; 1220a34c753fSRafael Auler } 1221a34c753fSRafael Auler 1222e290133cSMaksim Panchenko // Disassemble again without the symbolizer and check that the disassembly 1223e290133cSMaksim Panchenko // matches the assembler output. 1224e290133cSMaksim Panchenko MCInst TempInst; 1225e290133cSMaksim Panchenko BC.DisAsm->getInstruction(TempInst, Size, FunctionData.slice(Offset), 1226e290133cSMaksim Panchenko AbsoluteInstrAddr, nulls()); 1227e290133cSMaksim Panchenko if (!BC.validateEncoding(TempInst, FunctionData.slice(Offset, Size))) { 1228e290133cSMaksim Panchenko if (opts::Verbosity >= 0) { 1229a34c753fSRafael Auler errs() << "BOLT-WARNING: internal assembler/disassembler error " 1230a34c753fSRafael Auler "detected for AVX512 instruction:\n"; 1231e290133cSMaksim Panchenko BC.printInstruction(errs(), TempInst, AbsoluteInstrAddr); 1232a34c753fSRafael Auler errs() << " in function " << *this << '\n'; 1233a34c753fSRafael Auler } 1234a34c753fSRafael Auler 1235a34c753fSRafael Auler setIgnored(); 1236a34c753fSRafael Auler break; 1237a34c753fSRafael Auler } 1238a34c753fSRafael Auler } 1239a34c753fSRafael Auler 1240a34c753fSRafael Auler if (MIB->isBranch(Instruction) || MIB->isCall(Instruction)) { 1241a34c753fSRafael Auler uint64_t TargetAddress = 0; 1242a34c753fSRafael Auler if (MIB->evaluateBranch(Instruction, AbsoluteInstrAddr, Size, 1243a34c753fSRafael Auler TargetAddress)) { 1244a34c753fSRafael Auler // Check if the target is within the same function. Otherwise it's 1245a34c753fSRafael Auler // a call, possibly a tail call. 1246a34c753fSRafael Auler // 1247a34c753fSRafael Auler // If the target *is* the function address it could be either a branch 1248a34c753fSRafael Auler // or a recursive call. 1249a34c753fSRafael Auler bool IsCall = MIB->isCall(Instruction); 1250a34c753fSRafael Auler const bool IsCondBranch = MIB->isConditionalBranch(Instruction); 1251a34c753fSRafael Auler MCSymbol *TargetSymbol = nullptr; 1252a34c753fSRafael Auler 1253a34c753fSRafael Auler if (BC.MIB->isUnsupportedBranch(Instruction.getOpcode())) { 1254a34c753fSRafael Auler setIgnored(); 1255a34c753fSRafael Auler if (BinaryFunction *TargetFunc = 1256a34c753fSRafael Auler BC.getBinaryFunctionContainingAddress(TargetAddress)) 1257a34c753fSRafael Auler TargetFunc->setIgnored(); 1258a34c753fSRafael Auler } 1259a34c753fSRafael Auler 1260a34c753fSRafael Auler if (IsCall && containsAddress(TargetAddress)) { 1261a34c753fSRafael Auler if (TargetAddress == getAddress()) { 1262a34c753fSRafael Auler // Recursive call. 1263a34c753fSRafael Auler TargetSymbol = getSymbol(); 1264a34c753fSRafael Auler } else { 1265a34c753fSRafael Auler if (BC.isX86()) { 1266a34c753fSRafael Auler // Dangerous old-style x86 PIC code. We may need to freeze this 1267a34c753fSRafael Auler // function, so preserve the function as is for now. 1268a34c753fSRafael Auler PreserveNops = true; 1269a34c753fSRafael Auler } else { 1270a34c753fSRafael Auler errs() << "BOLT-WARNING: internal call detected at 0x" 1271a34c753fSRafael Auler << Twine::utohexstr(AbsoluteInstrAddr) << " in function " 1272a34c753fSRafael Auler << *this << ". Skipping.\n"; 1273a34c753fSRafael Auler IsSimple = false; 1274a34c753fSRafael Auler } 1275a34c753fSRafael Auler } 1276a34c753fSRafael Auler } 1277a34c753fSRafael Auler 1278a34c753fSRafael Auler if (!TargetSymbol) { 1279a34c753fSRafael Auler // Create either local label or external symbol. 1280a34c753fSRafael Auler if (containsAddress(TargetAddress)) { 1281a34c753fSRafael Auler TargetSymbol = getOrCreateLocalLabel(TargetAddress); 1282a34c753fSRafael Auler } else { 1283a34c753fSRafael Auler if (TargetAddress == getAddress() + getSize() && 128435efe1d8SVladislav Khmelevsky TargetAddress < getAddress() + getMaxSize() && 128535efe1d8SVladislav Khmelevsky !(BC.isAArch64() && 128635efe1d8SVladislav Khmelevsky BC.handleAArch64Veneer(TargetAddress, /*MatchOnly*/ true))) { 1287a34c753fSRafael Auler // Result of __builtin_unreachable(). 1288a34c753fSRafael Auler LLVM_DEBUG(dbgs() << "BOLT-DEBUG: jump past end detected at 0x" 1289a34c753fSRafael Auler << Twine::utohexstr(AbsoluteInstrAddr) 1290a34c753fSRafael Auler << " in function " << *this 1291a34c753fSRafael Auler << " : replacing with nop.\n"); 1292a34c753fSRafael Auler BC.MIB->createNoop(Instruction); 1293a34c753fSRafael Auler if (IsCondBranch) { 1294a34c753fSRafael Auler // Register branch offset for profile validation. 1295a34c753fSRafael Auler IgnoredBranches.emplace_back(Offset, Offset + Size); 1296a34c753fSRafael Auler } 1297a34c753fSRafael Auler goto add_instruction; 1298a34c753fSRafael Auler } 1299a34c753fSRafael Auler // May update Instruction and IsCall 1300a34c753fSRafael Auler TargetSymbol = handleExternalReference(Instruction, Size, Offset, 1301a34c753fSRafael Auler TargetAddress, IsCall); 1302a34c753fSRafael Auler } 1303a34c753fSRafael Auler } 1304a34c753fSRafael Auler 1305a34c753fSRafael Auler if (!IsCall) { 1306a34c753fSRafael Auler // Add taken branch info. 1307a34c753fSRafael Auler TakenBranches.emplace_back(Offset, TargetAddress - getAddress()); 1308a34c753fSRafael Auler } 1309a34c753fSRafael Auler BC.MIB->replaceBranchTarget(Instruction, TargetSymbol, &*Ctx); 1310a34c753fSRafael Auler 1311a34c753fSRafael Auler // Mark CTC. 13123652483cSRafael Auler if (IsCondBranch && IsCall) 1313a34c753fSRafael Auler MIB->setConditionalTailCall(Instruction, TargetAddress); 1314a34c753fSRafael Auler } else { 1315a34c753fSRafael Auler // Could not evaluate branch. Should be an indirect call or an 1316a34c753fSRafael Auler // indirect branch. Bail out on the latter case. 1317a34c753fSRafael Auler if (MIB->isIndirectBranch(Instruction)) 1318a34c753fSRafael Auler handleIndirectBranch(Instruction, Size, Offset); 1319a34c753fSRafael Auler // Indirect call. We only need to fix it if the operand is RIP-relative. 1320a34c753fSRafael Auler if (IsSimple && MIB->hasPCRelOperand(Instruction)) 1321a34c753fSRafael Auler handlePCRelOperand(Instruction, AbsoluteInstrAddr, Size); 1322a34c753fSRafael Auler 1323a34c753fSRafael Auler if (BC.isAArch64()) 1324a34c753fSRafael Auler handleAArch64IndirectCall(Instruction, Offset); 1325a34c753fSRafael Auler } 1326e290133cSMaksim Panchenko } else if (BC.isAArch64()) { 13274101aa13SMaksim Panchenko // Check if there's a relocation associated with this instruction. 13284101aa13SMaksim Panchenko bool UsedReloc = false; 13294101aa13SMaksim Panchenko for (auto Itr = Relocations.lower_bound(Offset), 13304101aa13SMaksim Panchenko ItrE = Relocations.lower_bound(Offset + Size); 13314101aa13SMaksim Panchenko Itr != ItrE; ++Itr) { 13324101aa13SMaksim Panchenko const Relocation &Relocation = Itr->second; 13334101aa13SMaksim Panchenko int64_t Value = Relocation.Value; 13344101aa13SMaksim Panchenko const bool Result = BC.MIB->replaceImmWithSymbolRef( 1335e290133cSMaksim Panchenko Instruction, Relocation.Symbol, Relocation.Addend, Ctx.get(), Value, 1336e290133cSMaksim Panchenko Relocation.Type); 13374101aa13SMaksim Panchenko (void)Result; 13384101aa13SMaksim Panchenko assert(Result && "cannot replace immediate with relocation"); 13394101aa13SMaksim Panchenko 1340e290133cSMaksim Panchenko // For aarch64, if we replaced an immediate with a symbol from a 13414101aa13SMaksim Panchenko // relocation, we mark it so we do not try to further process a 13424101aa13SMaksim Panchenko // pc-relative operand. All we need is the symbol. 13434101aa13SMaksim Panchenko UsedReloc = true; 13444101aa13SMaksim Panchenko } 13454101aa13SMaksim Panchenko 13464101aa13SMaksim Panchenko if (MIB->hasPCRelOperand(Instruction) && !UsedReloc) 1347a34c753fSRafael Auler handlePCRelOperand(Instruction, AbsoluteInstrAddr, Size); 13483652483cSRafael Auler } 1349a34c753fSRafael Auler 1350a34c753fSRafael Auler add_instruction: 1351a34c753fSRafael Auler if (getDWARFLineTable()) { 135240c2e0faSMaksim Panchenko Instruction.setLoc(findDebugLineInformationForInstructionAt( 135340c2e0faSMaksim Panchenko AbsoluteInstrAddr, getDWARFUnit(), getDWARFLineTable())); 1354a34c753fSRafael Auler } 1355a34c753fSRafael Auler 1356a34c753fSRafael Auler // Record offset of the instruction for profile matching. 13573652483cSRafael Auler if (BC.keepOffsetForInstruction(Instruction)) 1358a9cd49d5SAmir Ayupov MIB->setOffset(Instruction, static_cast<uint32_t>(Offset)); 1359a34c753fSRafael Auler 136008f56926SVladislav Khmelevsky if (BC.MIB->isNoop(Instruction)) { 136108f56926SVladislav Khmelevsky // NOTE: disassembly loses the correct size information for noops. 136208f56926SVladislav Khmelevsky // E.g. nopw 0x0(%rax,%rax,1) is 9 bytes, but re-encoded it's only 136308f56926SVladislav Khmelevsky // 5 bytes. Preserve the size info using annotations. 136408f56926SVladislav Khmelevsky MIB->addAnnotation(Instruction, "Size", static_cast<uint32_t>(Size)); 136508f56926SVladislav Khmelevsky } 136608f56926SVladislav Khmelevsky 1367a34c753fSRafael Auler addInstruction(Offset, std::move(Instruction)); 1368a34c753fSRafael Auler } 1369a34c753fSRafael Auler 1370e290133cSMaksim Panchenko // Reset symbolizer for the disassembler. 1371e290133cSMaksim Panchenko BC.SymbolicDisAsm->setSymbolizer(nullptr); 1372e290133cSMaksim Panchenko 13730b7e8bafSDenis Revunov if (uint64_t Offset = getFirstInstructionOffset()) 13740b7e8bafSDenis Revunov Labels[Offset] = BC.Ctx->createNamedTempSymbol(); 13750b7e8bafSDenis Revunov 1376a34c753fSRafael Auler clearList(Relocations); 1377a34c753fSRafael Auler 1378a34c753fSRafael Auler if (!IsSimple) { 1379a34c753fSRafael Auler clearList(Instructions); 1380a34c753fSRafael Auler return false; 1381a34c753fSRafael Auler } 1382a34c753fSRafael Auler 1383a34c753fSRafael Auler updateState(State::Disassembled); 1384a34c753fSRafael Auler 1385a34c753fSRafael Auler return true; 1386a34c753fSRafael Auler } 1387a34c753fSRafael Auler 1388a34c753fSRafael Auler bool BinaryFunction::scanExternalRefs() { 1389a34c753fSRafael Auler bool Success = true; 1390a34c753fSRafael Auler bool DisassemblyFailed = false; 1391a34c753fSRafael Auler 1392a34c753fSRafael Auler // Ignore pseudo functions. 1393a34c753fSRafael Auler if (isPseudo()) 1394a34c753fSRafael Auler return Success; 1395a34c753fSRafael Auler 1396a34c753fSRafael Auler if (opts::NoScan) { 1397a34c753fSRafael Auler clearList(Relocations); 1398a34c753fSRafael Auler clearList(ExternallyReferencedOffsets); 1399a34c753fSRafael Auler 1400a34c753fSRafael Auler return false; 1401a34c753fSRafael Auler } 1402a34c753fSRafael Auler 1403a34c753fSRafael Auler // List of external references for this function. 1404a34c753fSRafael Auler std::vector<Relocation> FunctionRelocations; 1405a34c753fSRafael Auler 1406a34c753fSRafael Auler static BinaryContext::IndependentCodeEmitter Emitter = 1407a34c753fSRafael Auler BC.createIndependentMCCodeEmitter(); 1408a34c753fSRafael Auler 1409a34c753fSRafael Auler ErrorOr<ArrayRef<uint8_t>> ErrorOrFunctionData = getData(); 1410a34c753fSRafael Auler assert(ErrorOrFunctionData && "function data is not available"); 1411a34c753fSRafael Auler ArrayRef<uint8_t> FunctionData = *ErrorOrFunctionData; 1412a34c753fSRafael Auler assert(FunctionData.size() == getMaxSize() && 1413a34c753fSRafael Auler "function size does not match raw data size"); 1414a34c753fSRafael Auler 1415a34c753fSRafael Auler uint64_t Size = 0; // instruction size 1416a34c753fSRafael Auler for (uint64_t Offset = 0; Offset < getSize(); Offset += Size) { 1417a34c753fSRafael Auler // Check for data inside code and ignore it 1418a34c753fSRafael Auler if (const size_t DataInCodeSize = getSizeOfDataInCodeAt(Offset)) { 1419a34c753fSRafael Auler Size = DataInCodeSize; 1420a34c753fSRafael Auler continue; 1421a34c753fSRafael Auler } 1422a34c753fSRafael Auler 1423a34c753fSRafael Auler const uint64_t AbsoluteInstrAddr = getAddress() + Offset; 1424a34c753fSRafael Auler MCInst Instruction; 142540c2e0faSMaksim Panchenko if (!BC.DisAsm->getInstruction(Instruction, Size, 1426a34c753fSRafael Auler FunctionData.slice(Offset), 142740c2e0faSMaksim Panchenko AbsoluteInstrAddr, nulls())) { 1428a34c753fSRafael Auler if (opts::Verbosity >= 1 && !isZeroPaddingAt(Offset)) { 1429a34c753fSRafael Auler errs() << "BOLT-WARNING: unable to disassemble instruction at offset 0x" 1430a34c753fSRafael Auler << Twine::utohexstr(Offset) << " (address 0x" 1431a34c753fSRafael Auler << Twine::utohexstr(AbsoluteInstrAddr) << ") in function " 1432a34c753fSRafael Auler << *this << '\n'; 1433a34c753fSRafael Auler } 1434a34c753fSRafael Auler Success = false; 1435a34c753fSRafael Auler DisassemblyFailed = true; 1436a34c753fSRafael Auler break; 1437a34c753fSRafael Auler } 1438a34c753fSRafael Auler 1439a34c753fSRafael Auler // Return true if we can skip handling the Target function reference. 1440a34c753fSRafael Auler auto ignoreFunctionRef = [&](const BinaryFunction &Target) { 1441a34c753fSRafael Auler if (&Target == this) 1442a34c753fSRafael Auler return true; 1443a34c753fSRafael Auler 1444a34c753fSRafael Auler // Note that later we may decide not to emit Target function. In that 1445a34c753fSRafael Auler // case, we conservatively create references that will be ignored or 1446a34c753fSRafael Auler // resolved to the same function. 1447a34c753fSRafael Auler if (!BC.shouldEmit(Target)) 1448a34c753fSRafael Auler return true; 1449a34c753fSRafael Auler 1450a34c753fSRafael Auler return false; 1451a34c753fSRafael Auler }; 1452a34c753fSRafael Auler 1453a34c753fSRafael Auler // Return true if we can ignore reference to the symbol. 1454a34c753fSRafael Auler auto ignoreReference = [&](const MCSymbol *TargetSymbol) { 1455a34c753fSRafael Auler if (!TargetSymbol) 1456a34c753fSRafael Auler return true; 1457a34c753fSRafael Auler 1458a34c753fSRafael Auler if (BC.forceSymbolRelocations(TargetSymbol->getName())) 1459a34c753fSRafael Auler return false; 1460a34c753fSRafael Auler 1461a34c753fSRafael Auler BinaryFunction *TargetFunction = BC.getFunctionForSymbol(TargetSymbol); 1462a34c753fSRafael Auler if (!TargetFunction) 1463a34c753fSRafael Auler return true; 1464a34c753fSRafael Auler 1465a34c753fSRafael Auler return ignoreFunctionRef(*TargetFunction); 1466a34c753fSRafael Auler }; 1467a34c753fSRafael Auler 1468a34c753fSRafael Auler // Detect if the instruction references an address. 1469a34c753fSRafael Auler // Without relocations, we can only trust PC-relative address modes. 1470a34c753fSRafael Auler uint64_t TargetAddress = 0; 1471a34c753fSRafael Auler bool IsPCRel = false; 1472a34c753fSRafael Auler bool IsBranch = false; 1473a34c753fSRafael Auler if (BC.MIB->hasPCRelOperand(Instruction)) { 1474a34c753fSRafael Auler if (BC.MIB->evaluateMemOperandTarget(Instruction, TargetAddress, 1475a34c753fSRafael Auler AbsoluteInstrAddr, Size)) { 1476a34c753fSRafael Auler IsPCRel = true; 1477a34c753fSRafael Auler } 1478a34c753fSRafael Auler } else if (BC.MIB->isCall(Instruction) || BC.MIB->isBranch(Instruction)) { 1479a34c753fSRafael Auler if (BC.MIB->evaluateBranch(Instruction, AbsoluteInstrAddr, Size, 1480a34c753fSRafael Auler TargetAddress)) { 1481a34c753fSRafael Auler IsBranch = true; 1482a34c753fSRafael Auler } 1483a34c753fSRafael Auler } 1484a34c753fSRafael Auler 1485a34c753fSRafael Auler MCSymbol *TargetSymbol = nullptr; 1486a34c753fSRafael Auler 1487a34c753fSRafael Auler // Create an entry point at reference address if needed. 1488a34c753fSRafael Auler BinaryFunction *TargetFunction = 1489a34c753fSRafael Auler BC.getBinaryFunctionContainingAddress(TargetAddress); 1490a34c753fSRafael Auler if (TargetFunction && !ignoreFunctionRef(*TargetFunction)) { 1491a34c753fSRafael Auler const uint64_t FunctionOffset = 1492a34c753fSRafael Auler TargetAddress - TargetFunction->getAddress(); 1493a34c753fSRafael Auler TargetSymbol = FunctionOffset 1494a34c753fSRafael Auler ? TargetFunction->addEntryPointAtOffset(FunctionOffset) 1495a34c753fSRafael Auler : TargetFunction->getSymbol(); 1496a34c753fSRafael Auler } 1497a34c753fSRafael Auler 1498a34c753fSRafael Auler // Can't find more references and not creating relocations. 1499a34c753fSRafael Auler if (!BC.HasRelocations) 1500a34c753fSRafael Auler continue; 1501a34c753fSRafael Auler 1502a34c753fSRafael Auler // Create a relocation against the TargetSymbol as the symbol might get 1503a34c753fSRafael Auler // moved. 1504a34c753fSRafael Auler if (TargetSymbol) { 1505a34c753fSRafael Auler if (IsBranch) { 1506a34c753fSRafael Auler BC.MIB->replaceBranchTarget(Instruction, TargetSymbol, 1507a34c753fSRafael Auler Emitter.LocalCtx.get()); 1508a34c753fSRafael Auler } else if (IsPCRel) { 150940c2e0faSMaksim Panchenko const MCExpr *Expr = MCSymbolRefExpr::create( 151040c2e0faSMaksim Panchenko TargetSymbol, MCSymbolRefExpr::VK_None, *Emitter.LocalCtx.get()); 1511a34c753fSRafael Auler BC.MIB->replaceMemOperandDisp( 1512a34c753fSRafael Auler Instruction, MCOperand::createExpr(BC.MIB->getTargetExprFor( 151340c2e0faSMaksim Panchenko Instruction, Expr, *Emitter.LocalCtx.get(), 0))); 1514a34c753fSRafael Auler } 1515a34c753fSRafael Auler } 1516a34c753fSRafael Auler 1517a34c753fSRafael Auler // Create more relocations based on input file relocations. 1518a34c753fSRafael Auler bool HasRel = false; 1519a34c753fSRafael Auler for (auto Itr = Relocations.lower_bound(Offset), 152040c2e0faSMaksim Panchenko ItrE = Relocations.lower_bound(Offset + Size); 152140c2e0faSMaksim Panchenko Itr != ItrE; ++Itr) { 1522a34c753fSRafael Auler Relocation &Relocation = Itr->second; 15234101aa13SMaksim Panchenko if (Relocation.isPCRelative() && BC.isX86()) 15244101aa13SMaksim Panchenko continue; 1525a34c753fSRafael Auler if (ignoreReference(Relocation.Symbol)) 1526a34c753fSRafael Auler continue; 1527a34c753fSRafael Auler 1528a34c753fSRafael Auler int64_t Value = Relocation.Value; 152940c2e0faSMaksim Panchenko const bool Result = BC.MIB->replaceImmWithSymbolRef( 153040c2e0faSMaksim Panchenko Instruction, Relocation.Symbol, Relocation.Addend, 153140c2e0faSMaksim Panchenko Emitter.LocalCtx.get(), Value, Relocation.Type); 1532a34c753fSRafael Auler (void)Result; 1533a34c753fSRafael Auler assert(Result && "cannot replace immediate with relocation"); 1534a34c753fSRafael Auler 1535a34c753fSRafael Auler HasRel = true; 1536a34c753fSRafael Auler } 1537a34c753fSRafael Auler 1538a34c753fSRafael Auler if (!TargetSymbol && !HasRel) 1539a34c753fSRafael Auler continue; 1540a34c753fSRafael Auler 1541a34c753fSRafael Auler // Emit the instruction using temp emitter and generate relocations. 1542a34c753fSRafael Auler SmallString<256> Code; 1543a34c753fSRafael Auler SmallVector<MCFixup, 4> Fixups; 1544a34c753fSRafael Auler raw_svector_ostream VecOS(Code); 1545a34c753fSRafael Auler Emitter.MCE->encodeInstruction(Instruction, VecOS, Fixups, *BC.STI); 1546a34c753fSRafael Auler 1547a34c753fSRafael Auler // Create relocation for every fixup. 1548a34c753fSRafael Auler for (const MCFixup &Fixup : Fixups) { 1549a34c753fSRafael Auler Optional<Relocation> Rel = BC.MIB->createRelocation(Fixup, *BC.MAB); 1550a34c753fSRafael Auler if (!Rel) { 1551a34c753fSRafael Auler Success = false; 1552a34c753fSRafael Auler continue; 1553a34c753fSRafael Auler } 1554a34c753fSRafael Auler 1555a34c753fSRafael Auler if (Relocation::getSizeForType(Rel->Type) < 4) { 1556a34c753fSRafael Auler // If the instruction uses a short form, then we might not be able 1557a34c753fSRafael Auler // to handle the rewrite without relaxation, and hence cannot reliably 1558a34c753fSRafael Auler // create an external reference relocation. 1559a34c753fSRafael Auler Success = false; 1560a34c753fSRafael Auler continue; 1561a34c753fSRafael Auler } 1562a34c753fSRafael Auler Rel->Offset += getAddress() - getOriginSection()->getAddress() + Offset; 1563a34c753fSRafael Auler FunctionRelocations.push_back(*Rel); 1564a34c753fSRafael Auler } 1565a34c753fSRafael Auler 1566a34c753fSRafael Auler if (!Success) 1567a34c753fSRafael Auler break; 1568a34c753fSRafael Auler } 1569a34c753fSRafael Auler 1570a34c753fSRafael Auler // Add relocations unless disassembly failed for this function. 15713652483cSRafael Auler if (!DisassemblyFailed) 15723652483cSRafael Auler for (Relocation &Rel : FunctionRelocations) 1573a34c753fSRafael Auler getOriginSection()->addPendingRelocation(Rel); 1574a34c753fSRafael Auler 1575a34c753fSRafael Auler // Inform BinaryContext that this function symbols will not be defined and 1576a34c753fSRafael Auler // relocations should not be created against them. 1577a34c753fSRafael Auler if (BC.HasRelocations) { 15783652483cSRafael Auler for (std::pair<const uint32_t, MCSymbol *> &LI : Labels) 1579a34c753fSRafael Auler BC.UndefinedSymbols.insert(LI.second); 15803652483cSRafael Auler if (FunctionEndLabel) 1581a34c753fSRafael Auler BC.UndefinedSymbols.insert(FunctionEndLabel); 1582a34c753fSRafael Auler } 1583a34c753fSRafael Auler 1584a34c753fSRafael Auler clearList(Relocations); 1585a34c753fSRafael Auler clearList(ExternallyReferencedOffsets); 1586a34c753fSRafael Auler 15873652483cSRafael Auler if (Success && BC.HasRelocations) 1588a34c753fSRafael Auler HasExternalRefRelocations = true; 1589a34c753fSRafael Auler 15903652483cSRafael Auler if (opts::Verbosity >= 1 && !Success) 1591a34c753fSRafael Auler outs() << "BOLT-INFO: failed to scan refs for " << *this << '\n'; 1592a34c753fSRafael Auler 1593a34c753fSRafael Auler return Success; 1594a34c753fSRafael Auler } 1595a34c753fSRafael Auler 1596a34c753fSRafael Auler void BinaryFunction::postProcessEntryPoints() { 1597a34c753fSRafael Auler if (!isSimple()) 1598a34c753fSRafael Auler return; 1599a34c753fSRafael Auler 1600a34c753fSRafael Auler for (auto &KV : Labels) { 1601a34c753fSRafael Auler MCSymbol *Label = KV.second; 1602a34c753fSRafael Auler if (!getSecondaryEntryPointSymbol(Label)) 1603a34c753fSRafael Auler continue; 1604a34c753fSRafael Auler 1605a34c753fSRafael Auler // In non-relocation mode there's potentially an external undetectable 1606a34c753fSRafael Auler // reference to the entry point and hence we cannot move this entry 1607a34c753fSRafael Auler // point. Optimizing without moving could be difficult. 1608a34c753fSRafael Auler if (!BC.HasRelocations) 1609a34c753fSRafael Auler setSimple(false); 1610a34c753fSRafael Auler 1611a34c753fSRafael Auler const uint32_t Offset = KV.first; 1612a34c753fSRafael Auler 1613a34c753fSRafael Auler // If we are at Offset 0 and there is no instruction associated with it, 1614a34c753fSRafael Auler // this means this is an empty function. Just ignore. If we find an 1615a34c753fSRafael Auler // instruction at this offset, this entry point is valid. 16163652483cSRafael Auler if (!Offset || getInstructionAtOffset(Offset)) 1617a34c753fSRafael Auler continue; 1618a34c753fSRafael Auler 1619a34c753fSRafael Auler // On AArch64 there are legitimate reasons to have references past the 1620a34c753fSRafael Auler // end of the function, e.g. jump tables. 16213652483cSRafael Auler if (BC.isAArch64() && Offset == getSize()) 1622a34c753fSRafael Auler continue; 1623a34c753fSRafael Auler 1624a34c753fSRafael Auler errs() << "BOLT-WARNING: reference in the middle of instruction " 162540c2e0faSMaksim Panchenko "detected in function " 162640c2e0faSMaksim Panchenko << *this << " at offset 0x" << Twine::utohexstr(Offset) << '\n'; 16273652483cSRafael Auler if (BC.HasRelocations) 1628a34c753fSRafael Auler setIgnored(); 1629a34c753fSRafael Auler setSimple(false); 1630a34c753fSRafael Auler return; 1631a34c753fSRafael Auler } 1632a34c753fSRafael Auler } 1633a34c753fSRafael Auler 1634a34c753fSRafael Auler void BinaryFunction::postProcessJumpTables() { 1635a34c753fSRafael Auler // Create labels for all entries. 1636a34c753fSRafael Auler for (auto &JTI : JumpTables) { 1637a34c753fSRafael Auler JumpTable &JT = *JTI.second; 1638a34c753fSRafael Auler if (JT.Type == JumpTable::JTT_PIC && opts::JumpTables == JTS_BASIC) { 1639a34c753fSRafael Auler opts::JumpTables = JTS_MOVE; 1640a34c753fSRafael Auler outs() << "BOLT-INFO: forcing -jump-tables=move as PIC jump table was " 164140c2e0faSMaksim Panchenko "detected in function " 164240c2e0faSMaksim Panchenko << *this << '\n'; 1643a34c753fSRafael Auler } 164428b1dcb1SHuan Nguyen if (JT.Entries.empty()) { 1645*05523dc3SHuan Nguyen bool HasOneParent = (JT.Parents.size() == 1); 1646*05523dc3SHuan Nguyen for (unsigned I = 0; I < JT.EntriesAsAddress.size(); ++I) { 1647*05523dc3SHuan Nguyen uint64_t EntryAddress = JT.EntriesAsAddress[I]; 1648*05523dc3SHuan Nguyen // builtin_unreachable does not belong to any function 1649*05523dc3SHuan Nguyen // Need to handle separately 1650*05523dc3SHuan Nguyen bool IsBuiltIn = false; 1651*05523dc3SHuan Nguyen for (BinaryFunction *Parent : JT.Parents) { 1652*05523dc3SHuan Nguyen if (EntryAddress == Parent->getAddress() + Parent->getSize()) { 1653*05523dc3SHuan Nguyen IsBuiltIn = true; 1654*05523dc3SHuan Nguyen // Specify second parameter as true to accept builtin_unreachable 1655*05523dc3SHuan Nguyen MCSymbol *Label = getOrCreateLocalLabel(EntryAddress, true); 1656a34c753fSRafael Auler JT.Entries.push_back(Label); 1657*05523dc3SHuan Nguyen break; 1658*05523dc3SHuan Nguyen } 1659*05523dc3SHuan Nguyen } 1660*05523dc3SHuan Nguyen if (IsBuiltIn) 1661*05523dc3SHuan Nguyen continue; 1662*05523dc3SHuan Nguyen // Create local label for targets cannot be reached by other fragments 1663*05523dc3SHuan Nguyen // Otherwise, secondary entry point to target function 1664*05523dc3SHuan Nguyen BinaryFunction *TargetBF = 1665*05523dc3SHuan Nguyen BC.getBinaryFunctionContainingAddress(EntryAddress); 1666*05523dc3SHuan Nguyen if (TargetBF->getAddress() != EntryAddress) { 1667*05523dc3SHuan Nguyen MCSymbol *Label = 1668*05523dc3SHuan Nguyen (HasOneParent && TargetBF == this) 1669*05523dc3SHuan Nguyen ? getOrCreateLocalLabel(JT.EntriesAsAddress[I], true) 1670*05523dc3SHuan Nguyen : TargetBF->addEntryPointAtOffset(EntryAddress - 1671*05523dc3SHuan Nguyen TargetBF->getAddress()); 1672*05523dc3SHuan Nguyen JT.Entries.push_back(Label); 1673*05523dc3SHuan Nguyen } 1674a34c753fSRafael Auler } 167528b1dcb1SHuan Nguyen } 1676a34c753fSRafael Auler 1677a34c753fSRafael Auler const uint64_t BDSize = 1678a34c753fSRafael Auler BC.getBinaryDataAtAddress(JT.getAddress())->getSize(); 1679a34c753fSRafael Auler if (!BDSize) { 1680a34c753fSRafael Auler BC.setBinaryDataSize(JT.getAddress(), JT.getSize()); 1681a34c753fSRafael Auler } else { 1682a34c753fSRafael Auler assert(BDSize >= JT.getSize() && 1683a34c753fSRafael Auler "jump table cannot be larger than the containing object"); 1684a34c753fSRafael Auler } 1685a34c753fSRafael Auler } 1686a34c753fSRafael Auler 1687a34c753fSRafael Auler // Add TakenBranches from JumpTables. 1688a34c753fSRafael Auler // 1689a34c753fSRafael Auler // We want to do it after initial processing since we don't know jump tables' 1690a34c753fSRafael Auler // boundaries until we process them all. 1691a34c753fSRafael Auler for (auto &JTSite : JTSites) { 1692a34c753fSRafael Auler const uint64_t JTSiteOffset = JTSite.first; 1693a34c753fSRafael Auler const uint64_t JTAddress = JTSite.second; 1694a34c753fSRafael Auler const JumpTable *JT = getJumpTableContainingAddress(JTAddress); 1695a34c753fSRafael Auler assert(JT && "cannot find jump table for address"); 1696a34c753fSRafael Auler 1697a34c753fSRafael Auler uint64_t EntryOffset = JTAddress - JT->getAddress(); 1698a34c753fSRafael Auler while (EntryOffset < JT->getSize()) { 1699*05523dc3SHuan Nguyen uint64_t EntryAddress = JT->EntriesAsAddress[EntryOffset / JT->EntrySize]; 1700*05523dc3SHuan Nguyen uint64_t TargetOffset = EntryAddress - getAddress(); 1701a34c753fSRafael Auler if (TargetOffset < getSize()) { 1702a34c753fSRafael Auler TakenBranches.emplace_back(JTSiteOffset, TargetOffset); 1703a34c753fSRafael Auler 1704a34c753fSRafael Auler if (opts::StrictMode) 1705a34c753fSRafael Auler registerReferencedOffset(TargetOffset); 1706a34c753fSRafael Auler } 1707a34c753fSRafael Auler 1708a34c753fSRafael Auler EntryOffset += JT->EntrySize; 1709a34c753fSRafael Auler 1710a34c753fSRafael Auler // A label at the next entry means the end of this jump table. 1711a34c753fSRafael Auler if (JT->Labels.count(EntryOffset)) 1712a34c753fSRafael Auler break; 1713a34c753fSRafael Auler } 1714a34c753fSRafael Auler } 1715a34c753fSRafael Auler clearList(JTSites); 1716a34c753fSRafael Auler 1717a34c753fSRafael Auler // Conservatively populate all possible destinations for unknown indirect 1718a34c753fSRafael Auler // branches. 1719a34c753fSRafael Auler if (opts::StrictMode && hasInternalReference()) { 1720a34c753fSRafael Auler for (uint64_t Offset : UnknownIndirectBranchOffsets) { 1721a34c753fSRafael Auler for (uint64_t PossibleDestination : ExternallyReferencedOffsets) { 1722a34c753fSRafael Auler // Ignore __builtin_unreachable(). 1723a34c753fSRafael Auler if (PossibleDestination == getSize()) 1724a34c753fSRafael Auler continue; 1725a34c753fSRafael Auler TakenBranches.emplace_back(Offset, PossibleDestination); 1726a34c753fSRafael Auler } 1727a34c753fSRafael Auler } 1728a34c753fSRafael Auler } 1729a34c753fSRafael Auler 1730a34c753fSRafael Auler // Remove duplicates branches. We can get a bunch of them from jump tables. 1731a34c753fSRafael Auler // Without doing jump table value profiling we don't have use for extra 1732a34c753fSRafael Auler // (duplicate) branches. 1733d2c87699SAmir Ayupov llvm::sort(TakenBranches); 1734a34c753fSRafael Auler auto NewEnd = std::unique(TakenBranches.begin(), TakenBranches.end()); 1735a34c753fSRafael Auler TakenBranches.erase(NewEnd, TakenBranches.end()); 1736a34c753fSRafael Auler } 1737a34c753fSRafael Auler 1738a34c753fSRafael Auler bool BinaryFunction::postProcessIndirectBranches( 1739a34c753fSRafael Auler MCPlusBuilder::AllocatorIdTy AllocId) { 1740a34c753fSRafael Auler auto addUnknownControlFlow = [&](BinaryBasicBlock &BB) { 1741a34c753fSRafael Auler HasUnknownControlFlow = true; 1742a34c753fSRafael Auler BB.removeAllSuccessors(); 17433652483cSRafael Auler for (uint64_t PossibleDestination : ExternallyReferencedOffsets) 1744a34c753fSRafael Auler if (BinaryBasicBlock *SuccBB = getBasicBlockAtOffset(PossibleDestination)) 1745a34c753fSRafael Auler BB.addSuccessor(SuccBB); 1746a34c753fSRafael Auler }; 1747a34c753fSRafael Auler 1748a34c753fSRafael Auler uint64_t NumIndirectJumps = 0; 1749a34c753fSRafael Auler MCInst *LastIndirectJump = nullptr; 1750a34c753fSRafael Auler BinaryBasicBlock *LastIndirectJumpBB = nullptr; 1751a34c753fSRafael Auler uint64_t LastJT = 0; 1752a34c753fSRafael Auler uint16_t LastJTIndexReg = BC.MIB->getNoRegister(); 1753a34c753fSRafael Auler for (BinaryBasicBlock *BB : layout()) { 1754a34c753fSRafael Auler for (MCInst &Instr : *BB) { 1755a34c753fSRafael Auler if (!BC.MIB->isIndirectBranch(Instr)) 1756a34c753fSRafael Auler continue; 1757a34c753fSRafael Auler 1758a34c753fSRafael Auler // If there's an indirect branch in a single-block function - 1759a34c753fSRafael Auler // it must be a tail call. 1760a34c753fSRafael Auler if (layout_size() == 1) { 1761a34c753fSRafael Auler BC.MIB->convertJmpToTailCall(Instr); 1762a34c753fSRafael Auler return true; 1763a34c753fSRafael Auler } 1764a34c753fSRafael Auler 1765a34c753fSRafael Auler ++NumIndirectJumps; 1766a34c753fSRafael Auler 1767a34c753fSRafael Auler if (opts::StrictMode && !hasInternalReference()) { 1768a34c753fSRafael Auler BC.MIB->convertJmpToTailCall(Instr); 1769a34c753fSRafael Auler break; 1770a34c753fSRafael Auler } 1771a34c753fSRafael Auler 1772a34c753fSRafael Auler // Validate the tail call or jump table assumptions now that we know 1773a34c753fSRafael Auler // basic block boundaries. 1774a34c753fSRafael Auler if (BC.MIB->isTailCall(Instr) || BC.MIB->getJumpTable(Instr)) { 1775a34c753fSRafael Auler const unsigned PtrSize = BC.AsmInfo->getCodePointerSize(); 1776a34c753fSRafael Auler MCInst *MemLocInstr; 1777a34c753fSRafael Auler unsigned BaseRegNum, IndexRegNum; 1778a34c753fSRafael Auler int64_t DispValue; 1779a34c753fSRafael Auler const MCExpr *DispExpr; 1780a34c753fSRafael Auler MCInst *PCRelBaseInstr; 1781a34c753fSRafael Auler IndirectBranchType Type = BC.MIB->analyzeIndirectBranch( 1782a34c753fSRafael Auler Instr, BB->begin(), BB->end(), PtrSize, MemLocInstr, BaseRegNum, 1783a34c753fSRafael Auler IndexRegNum, DispValue, DispExpr, PCRelBaseInstr); 1784a34c753fSRafael Auler if (Type != IndirectBranchType::UNKNOWN || MemLocInstr != nullptr) 1785a34c753fSRafael Auler continue; 1786a34c753fSRafael Auler 1787a34c753fSRafael Auler if (!opts::StrictMode) 1788a34c753fSRafael Auler return false; 1789a34c753fSRafael Auler 1790a34c753fSRafael Auler if (BC.MIB->isTailCall(Instr)) { 1791a34c753fSRafael Auler BC.MIB->convertTailCallToJmp(Instr); 1792a34c753fSRafael Auler } else { 1793a34c753fSRafael Auler LastIndirectJump = &Instr; 1794a34c753fSRafael Auler LastIndirectJumpBB = BB; 1795a34c753fSRafael Auler LastJT = BC.MIB->getJumpTable(Instr); 1796a34c753fSRafael Auler LastJTIndexReg = BC.MIB->getJumpTableIndexReg(Instr); 1797a34c753fSRafael Auler BC.MIB->unsetJumpTable(Instr); 1798a34c753fSRafael Auler 1799a34c753fSRafael Auler JumpTable *JT = BC.getJumpTableContainingAddress(LastJT); 1800a34c753fSRafael Auler if (JT->Type == JumpTable::JTT_NORMAL) { 1801a34c753fSRafael Auler // Invalidating the jump table may also invalidate other jump table 1802a34c753fSRafael Auler // boundaries. Until we have/need a support for this, mark the 1803a34c753fSRafael Auler // function as non-simple. 1804a34c753fSRafael Auler LLVM_DEBUG(dbgs() << "BOLT-DEBUG: rejected jump table reference" 1805a34c753fSRafael Auler << JT->getName() << " in " << *this << '\n'); 1806a34c753fSRafael Auler return false; 1807a34c753fSRafael Auler } 1808a34c753fSRafael Auler } 1809a34c753fSRafael Auler 1810a34c753fSRafael Auler addUnknownControlFlow(*BB); 1811a34c753fSRafael Auler continue; 1812a34c753fSRafael Auler } 1813a34c753fSRafael Auler 1814a34c753fSRafael Auler // If this block contains an epilogue code and has an indirect branch, 1815a34c753fSRafael Auler // then most likely it's a tail call. Otherwise, we cannot tell for sure 1816a34c753fSRafael Auler // what it is and conservatively reject the function's CFG. 1817a34c753fSRafael Auler bool IsEpilogue = false; 1818a34c753fSRafael Auler for (const MCInst &Instr : *BB) { 1819a34c753fSRafael Auler if (BC.MIB->isLeave(Instr) || BC.MIB->isPop(Instr)) { 1820a34c753fSRafael Auler IsEpilogue = true; 1821a34c753fSRafael Auler break; 1822a34c753fSRafael Auler } 1823a34c753fSRafael Auler } 1824a34c753fSRafael Auler if (IsEpilogue) { 1825a34c753fSRafael Auler BC.MIB->convertJmpToTailCall(Instr); 1826a34c753fSRafael Auler BB->removeAllSuccessors(); 1827a34c753fSRafael Auler continue; 1828a34c753fSRafael Auler } 1829a34c753fSRafael Auler 1830a34c753fSRafael Auler if (opts::Verbosity >= 2) { 1831a34c753fSRafael Auler outs() << "BOLT-INFO: rejected potential indirect tail call in " 183240c2e0faSMaksim Panchenko << "function " << *this << " in basic block " << BB->getName() 183340c2e0faSMaksim Panchenko << ".\n"; 1834a34c753fSRafael Auler LLVM_DEBUG(BC.printInstructions(dbgs(), BB->begin(), BB->end(), 1835a34c753fSRafael Auler BB->getOffset(), this, true)); 1836a34c753fSRafael Auler } 1837a34c753fSRafael Auler 1838a34c753fSRafael Auler if (!opts::StrictMode) 1839a34c753fSRafael Auler return false; 1840a34c753fSRafael Auler 1841a34c753fSRafael Auler addUnknownControlFlow(*BB); 1842a34c753fSRafael Auler } 1843a34c753fSRafael Auler } 1844a34c753fSRafael Auler 1845a34c753fSRafael Auler if (HasInternalLabelReference) 1846a34c753fSRafael Auler return false; 1847a34c753fSRafael Auler 1848a34c753fSRafael Auler // If there's only one jump table, and one indirect jump, and no other 1849a34c753fSRafael Auler // references, then we should be able to derive the jump table even if we 1850a34c753fSRafael Auler // fail to match the pattern. 1851a34c753fSRafael Auler if (HasUnknownControlFlow && NumIndirectJumps == 1 && 1852a34c753fSRafael Auler JumpTables.size() == 1 && LastIndirectJump) { 1853a34c753fSRafael Auler BC.MIB->setJumpTable(*LastIndirectJump, LastJT, LastJTIndexReg, AllocId); 1854a34c753fSRafael Auler HasUnknownControlFlow = false; 1855a34c753fSRafael Auler 18565a343994SMaksim Panchenko LastIndirectJumpBB->updateJumpTableSuccessors(); 1857a34c753fSRafael Auler } 1858a34c753fSRafael Auler 1859a34c753fSRafael Auler if (HasFixedIndirectBranch) 1860a34c753fSRafael Auler return false; 1861a34c753fSRafael Auler 1862a34c753fSRafael Auler if (HasUnknownControlFlow && !BC.HasRelocations) 1863a34c753fSRafael Auler return false; 1864a34c753fSRafael Auler 1865a34c753fSRafael Auler return true; 1866a34c753fSRafael Auler } 1867a34c753fSRafael Auler 1868a34c753fSRafael Auler void BinaryFunction::recomputeLandingPads() { 1869a34c753fSRafael Auler updateBBIndices(0); 1870a34c753fSRafael Auler 1871a34c753fSRafael Auler for (BinaryBasicBlock *BB : BasicBlocks) { 1872a34c753fSRafael Auler BB->LandingPads.clear(); 1873a34c753fSRafael Auler BB->Throwers.clear(); 1874a34c753fSRafael Auler } 1875a34c753fSRafael Auler 1876a34c753fSRafael Auler for (BinaryBasicBlock *BB : BasicBlocks) { 1877a34c753fSRafael Auler std::unordered_set<const BinaryBasicBlock *> BBLandingPads; 1878a34c753fSRafael Auler for (MCInst &Instr : *BB) { 1879a34c753fSRafael Auler if (!BC.MIB->isInvoke(Instr)) 1880a34c753fSRafael Auler continue; 1881a34c753fSRafael Auler 1882a34c753fSRafael Auler const Optional<MCPlus::MCLandingPad> EHInfo = BC.MIB->getEHInfo(Instr); 1883a34c753fSRafael Auler if (!EHInfo || !EHInfo->first) 1884a34c753fSRafael Auler continue; 1885a34c753fSRafael Auler 1886a34c753fSRafael Auler BinaryBasicBlock *LPBlock = getBasicBlockForLabel(EHInfo->first); 1887a34c753fSRafael Auler if (!BBLandingPads.count(LPBlock)) { 1888a34c753fSRafael Auler BBLandingPads.insert(LPBlock); 1889a34c753fSRafael Auler BB->LandingPads.emplace_back(LPBlock); 1890a34c753fSRafael Auler LPBlock->Throwers.emplace_back(BB); 1891a34c753fSRafael Auler } 1892a34c753fSRafael Auler } 1893a34c753fSRafael Auler } 1894a34c753fSRafael Auler } 1895a34c753fSRafael Auler 1896a34c753fSRafael Auler bool BinaryFunction::buildCFG(MCPlusBuilder::AllocatorIdTy AllocatorId) { 1897a34c753fSRafael Auler auto &MIB = BC.MIB; 1898a34c753fSRafael Auler 1899a34c753fSRafael Auler if (!isSimple()) { 1900a34c753fSRafael Auler assert(!BC.HasRelocations && 1901a34c753fSRafael Auler "cannot process file with non-simple function in relocs mode"); 1902a34c753fSRafael Auler return false; 1903a34c753fSRafael Auler } 1904a34c753fSRafael Auler 1905a34c753fSRafael Auler if (CurrentState != State::Disassembled) 1906a34c753fSRafael Auler return false; 1907a34c753fSRafael Auler 1908a34c753fSRafael Auler assert(BasicBlocks.empty() && "basic block list should be empty"); 19090b7e8bafSDenis Revunov assert((Labels.find(getFirstInstructionOffset()) != Labels.end()) && 1910a34c753fSRafael Auler "first instruction should always have a label"); 1911a34c753fSRafael Auler 1912a34c753fSRafael Auler // Create basic blocks in the original layout order: 1913a34c753fSRafael Auler // 1914a34c753fSRafael Auler // * Every instruction with associated label marks 1915a34c753fSRafael Auler // the beginning of a basic block. 1916a34c753fSRafael Auler // * Conditional instruction marks the end of a basic block, 1917a34c753fSRafael Auler // except when the following instruction is an 1918a34c753fSRafael Auler // unconditional branch, and the unconditional branch is not 1919a34c753fSRafael Auler // a destination of another branch. In the latter case, the 1920a34c753fSRafael Auler // basic block will consist of a single unconditional branch 1921a34c753fSRafael Auler // (missed "double-jump" optimization). 1922a34c753fSRafael Auler // 1923a34c753fSRafael Auler // Created basic blocks are sorted in layout order since they are 1924a34c753fSRafael Auler // created in the same order as instructions, and instructions are 1925a34c753fSRafael Auler // sorted by offsets. 1926a34c753fSRafael Auler BinaryBasicBlock *InsertBB = nullptr; 1927a34c753fSRafael Auler BinaryBasicBlock *PrevBB = nullptr; 1928a34c753fSRafael Auler bool IsLastInstrNop = false; 1929ccb99dd1SMaksim Panchenko // Offset of the last non-nop instruction. 1930a34c753fSRafael Auler uint64_t LastInstrOffset = 0; 1931a34c753fSRafael Auler 193240c2e0faSMaksim Panchenko auto addCFIPlaceholders = [this](uint64_t CFIOffset, 193340c2e0faSMaksim Panchenko BinaryBasicBlock *InsertBB) { 1934a34c753fSRafael Auler for (auto FI = OffsetToCFI.lower_bound(CFIOffset), 1935a34c753fSRafael Auler FE = OffsetToCFI.upper_bound(CFIOffset); 1936a34c753fSRafael Auler FI != FE; ++FI) { 1937a34c753fSRafael Auler addCFIPseudo(InsertBB, InsertBB->end(), FI->second); 1938a34c753fSRafael Auler } 1939a34c753fSRafael Auler }; 1940a34c753fSRafael Auler 1941a34c753fSRafael Auler // For profiling purposes we need to save the offset of the last instruction 1942ccb99dd1SMaksim Panchenko // in the basic block. 1943ccb99dd1SMaksim Panchenko // NOTE: nops always have an Offset annotation. Annotate the last non-nop as 1944ccb99dd1SMaksim Panchenko // older profiles ignored nops. 1945a34c753fSRafael Auler auto updateOffset = [&](uint64_t Offset) { 1946a34c753fSRafael Auler assert(PrevBB && PrevBB != InsertBB && "invalid previous block"); 1947ccb99dd1SMaksim Panchenko MCInst *LastNonNop = nullptr; 1948ccb99dd1SMaksim Panchenko for (BinaryBasicBlock::reverse_iterator RII = PrevBB->getLastNonPseudo(), 1949ccb99dd1SMaksim Panchenko E = PrevBB->rend(); 1950ccb99dd1SMaksim Panchenko RII != E; ++RII) { 1951ccb99dd1SMaksim Panchenko if (!BC.MIB->isPseudo(*RII) && !BC.MIB->isNoop(*RII)) { 1952ccb99dd1SMaksim Panchenko LastNonNop = &*RII; 1953ccb99dd1SMaksim Panchenko break; 1954ccb99dd1SMaksim Panchenko } 1955ccb99dd1SMaksim Panchenko } 1956a9cd49d5SAmir Ayupov if (LastNonNop && !MIB->getOffset(*LastNonNop)) 1957a9cd49d5SAmir Ayupov MIB->setOffset(*LastNonNop, static_cast<uint32_t>(Offset), AllocatorId); 1958a34c753fSRafael Auler }; 1959a34c753fSRafael Auler 1960a34c753fSRafael Auler for (auto I = Instructions.begin(), E = Instructions.end(); I != E; ++I) { 1961a34c753fSRafael Auler const uint32_t Offset = I->first; 1962a34c753fSRafael Auler MCInst &Instr = I->second; 1963a34c753fSRafael Auler 1964a34c753fSRafael Auler auto LI = Labels.find(Offset); 1965a34c753fSRafael Auler if (LI != Labels.end()) { 1966a34c753fSRafael Auler // Always create new BB at branch destination. 1967ccb99dd1SMaksim Panchenko PrevBB = InsertBB ? InsertBB : PrevBB; 19688228c703SMaksim Panchenko InsertBB = addBasicBlockAt(LI->first, LI->second); 19698228c703SMaksim Panchenko if (opts::PreserveBlocksAlignment && IsLastInstrNop) 19708228c703SMaksim Panchenko InsertBB->setDerivedAlignment(); 19718228c703SMaksim Panchenko 1972a34c753fSRafael Auler if (PrevBB) 1973a34c753fSRafael Auler updateOffset(LastInstrOffset); 1974a34c753fSRafael Auler } 1975a34c753fSRafael Auler 1976a34c753fSRafael Auler const uint64_t InstrInputAddr = I->first + Address; 1977a34c753fSRafael Auler bool IsSDTMarker = 1978a34c753fSRafael Auler MIB->isNoop(Instr) && BC.SDTMarkers.count(InstrInputAddr); 1979a34c753fSRafael Auler bool IsLKMarker = BC.LKMarkers.count(InstrInputAddr); 1980ccb99dd1SMaksim Panchenko // Mark all nops with Offset for profile tracking purposes. 1981ccb99dd1SMaksim Panchenko if (MIB->isNoop(Instr) || IsLKMarker) { 1982a9cd49d5SAmir Ayupov if (!MIB->getOffset(Instr)) 1983a9cd49d5SAmir Ayupov MIB->setOffset(Instr, static_cast<uint32_t>(Offset), AllocatorId); 1984ccb99dd1SMaksim Panchenko if (IsSDTMarker || IsLKMarker) 1985ccb99dd1SMaksim Panchenko HasSDTMarker = true; 1986ccb99dd1SMaksim Panchenko else 1987ccb99dd1SMaksim Panchenko // Annotate ordinary nops, so we can safely delete them if required. 1988ccb99dd1SMaksim Panchenko MIB->addAnnotation(Instr, "NOP", static_cast<uint32_t>(1), AllocatorId); 1989a34c753fSRafael Auler } 1990a34c753fSRafael Auler 1991a34c753fSRafael Auler if (!InsertBB) { 1992a34c753fSRafael Auler // It must be a fallthrough or unreachable code. Create a new block unless 1993a34c753fSRafael Auler // we see an unconditional branch following a conditional one. The latter 1994a34c753fSRafael Auler // should not be a conditional tail call. 1995a34c753fSRafael Auler assert(PrevBB && "no previous basic block for a fall through"); 1996a34c753fSRafael Auler MCInst *PrevInstr = PrevBB->getLastNonPseudoInstr(); 1997a34c753fSRafael Auler assert(PrevInstr && "no previous instruction for a fall through"); 1998a34c753fSRafael Auler if (MIB->isUnconditionalBranch(Instr) && 1999a34c753fSRafael Auler !MIB->isUnconditionalBranch(*PrevInstr) && 2000bb8e7ebaSVladislav Khmelevsky !MIB->getConditionalTailCall(*PrevInstr) && 2001bb8e7ebaSVladislav Khmelevsky !MIB->isReturn(*PrevInstr)) { 2002a34c753fSRafael Auler // Temporarily restore inserter basic block. 2003a34c753fSRafael Auler InsertBB = PrevBB; 2004a34c753fSRafael Auler } else { 2005a34c753fSRafael Auler MCSymbol *Label; 2006a34c753fSRafael Auler { 2007a34c753fSRafael Auler auto L = BC.scopeLock(); 2008a34c753fSRafael Auler Label = BC.Ctx->createNamedTempSymbol("FT"); 2009a34c753fSRafael Auler } 20108228c703SMaksim Panchenko InsertBB = addBasicBlockAt(Offset, Label); 20118228c703SMaksim Panchenko if (opts::PreserveBlocksAlignment && IsLastInstrNop) 20128228c703SMaksim Panchenko InsertBB->setDerivedAlignment(); 2013a34c753fSRafael Auler updateOffset(LastInstrOffset); 2014a34c753fSRafael Auler } 2015a34c753fSRafael Auler } 20160b7e8bafSDenis Revunov if (Offset == getFirstInstructionOffset()) { 20170b7e8bafSDenis Revunov // Add associated CFI pseudos in the first offset 20180b7e8bafSDenis Revunov addCFIPlaceholders(Offset, InsertBB); 2019a34c753fSRafael Auler } 2020a34c753fSRafael Auler 2021a34c753fSRafael Auler const bool IsBlockEnd = MIB->isTerminator(Instr); 2022a34c753fSRafael Auler IsLastInstrNop = MIB->isNoop(Instr); 2023ccb99dd1SMaksim Panchenko if (!IsLastInstrNop) 2024a34c753fSRafael Auler LastInstrOffset = Offset; 2025a34c753fSRafael Auler InsertBB->addInstruction(std::move(Instr)); 2026a34c753fSRafael Auler 2027a34c753fSRafael Auler // Add associated CFI instrs. We always add the CFI instruction that is 2028a34c753fSRafael Auler // located immediately after this instruction, since the next CFI 2029a34c753fSRafael Auler // instruction reflects the change in state caused by this instruction. 2030a34c753fSRafael Auler auto NextInstr = std::next(I); 2031a34c753fSRafael Auler uint64_t CFIOffset; 2032a34c753fSRafael Auler if (NextInstr != E) 2033a34c753fSRafael Auler CFIOffset = NextInstr->first; 2034a34c753fSRafael Auler else 2035a34c753fSRafael Auler CFIOffset = getSize(); 2036a34c753fSRafael Auler 2037a34c753fSRafael Auler // Note: this potentially invalidates instruction pointers/iterators. 2038a34c753fSRafael Auler addCFIPlaceholders(CFIOffset, InsertBB); 2039a34c753fSRafael Auler 2040a34c753fSRafael Auler if (IsBlockEnd) { 2041a34c753fSRafael Auler PrevBB = InsertBB; 2042a34c753fSRafael Auler InsertBB = nullptr; 2043a34c753fSRafael Auler } 2044a34c753fSRafael Auler } 2045a34c753fSRafael Auler 2046a34c753fSRafael Auler if (BasicBlocks.empty()) { 2047a34c753fSRafael Auler setSimple(false); 2048a34c753fSRafael Auler return false; 2049a34c753fSRafael Auler } 2050a34c753fSRafael Auler 2051a34c753fSRafael Auler // Intermediate dump. 2052a34c753fSRafael Auler LLVM_DEBUG(print(dbgs(), "after creating basic blocks")); 2053a34c753fSRafael Auler 2054a34c753fSRafael Auler // TODO: handle properly calls to no-return functions, 2055a34c753fSRafael Auler // e.g. exit(3), etc. Otherwise we'll see a false fall-through 2056a34c753fSRafael Auler // blocks. 2057a34c753fSRafael Auler 2058a34c753fSRafael Auler for (std::pair<uint32_t, uint32_t> &Branch : TakenBranches) { 2059a34c753fSRafael Auler LLVM_DEBUG(dbgs() << "registering branch [0x" 2060a34c753fSRafael Auler << Twine::utohexstr(Branch.first) << "] -> [0x" 2061a34c753fSRafael Auler << Twine::utohexstr(Branch.second) << "]\n"); 2062a34c753fSRafael Auler BinaryBasicBlock *FromBB = getBasicBlockContainingOffset(Branch.first); 2063a34c753fSRafael Auler BinaryBasicBlock *ToBB = getBasicBlockAtOffset(Branch.second); 2064a34c753fSRafael Auler if (!FromBB || !ToBB) { 2065a34c753fSRafael Auler if (!FromBB) 2066a34c753fSRafael Auler errs() << "BOLT-ERROR: cannot find BB containing the branch.\n"; 2067a34c753fSRafael Auler if (!ToBB) 2068a34c753fSRafael Auler errs() << "BOLT-ERROR: cannot find BB containing branch destination.\n"; 2069a34c753fSRafael Auler BC.exitWithBugReport("disassembly failed - inconsistent branch found.", 2070a34c753fSRafael Auler *this); 2071a34c753fSRafael Auler } 2072a34c753fSRafael Auler 2073a34c753fSRafael Auler FromBB->addSuccessor(ToBB); 2074a34c753fSRafael Auler } 2075a34c753fSRafael Auler 2076a34c753fSRafael Auler // Add fall-through branches. 2077a34c753fSRafael Auler PrevBB = nullptr; 2078a34c753fSRafael Auler bool IsPrevFT = false; // Is previous block a fall-through. 2079a34c753fSRafael Auler for (BinaryBasicBlock *BB : BasicBlocks) { 20803652483cSRafael Auler if (IsPrevFT) 2081a34c753fSRafael Auler PrevBB->addSuccessor(BB); 20823652483cSRafael Auler 2083a34c753fSRafael Auler if (BB->empty()) { 2084a34c753fSRafael Auler IsPrevFT = true; 2085a34c753fSRafael Auler PrevBB = BB; 2086a34c753fSRafael Auler continue; 2087a34c753fSRafael Auler } 2088a34c753fSRafael Auler 2089a34c753fSRafael Auler MCInst *LastInstr = BB->getLastNonPseudoInstr(); 2090a34c753fSRafael Auler assert(LastInstr && 2091a34c753fSRafael Auler "should have non-pseudo instruction in non-empty block"); 2092a34c753fSRafael Auler 2093a34c753fSRafael Auler if (BB->succ_size() == 0) { 2094a34c753fSRafael Auler // Since there's no existing successors, we know the last instruction is 2095a34c753fSRafael Auler // not a conditional branch. Thus if it's a terminator, it shouldn't be a 2096a34c753fSRafael Auler // fall-through. 2097a34c753fSRafael Auler // 2098a34c753fSRafael Auler // Conditional tail call is a special case since we don't add a taken 2099a34c753fSRafael Auler // branch successor for it. 2100a34c753fSRafael Auler IsPrevFT = !MIB->isTerminator(*LastInstr) || 2101a34c753fSRafael Auler MIB->getConditionalTailCall(*LastInstr); 2102a34c753fSRafael Auler } else if (BB->succ_size() == 1) { 2103a34c753fSRafael Auler IsPrevFT = MIB->isConditionalBranch(*LastInstr); 2104a34c753fSRafael Auler } else { 2105a34c753fSRafael Auler IsPrevFT = false; 2106a34c753fSRafael Auler } 2107a34c753fSRafael Auler 2108a34c753fSRafael Auler PrevBB = BB; 2109a34c753fSRafael Auler } 2110a34c753fSRafael Auler 2111a34c753fSRafael Auler // Assign landing pads and throwers info. 2112a34c753fSRafael Auler recomputeLandingPads(); 2113a34c753fSRafael Auler 2114a34c753fSRafael Auler // Assign CFI information to each BB entry. 2115a34c753fSRafael Auler annotateCFIState(); 2116a34c753fSRafael Auler 2117a34c753fSRafael Auler // Annotate invoke instructions with GNU_args_size data. 2118a34c753fSRafael Auler propagateGnuArgsSizeInfo(AllocatorId); 2119a34c753fSRafael Auler 2120a34c753fSRafael Auler // Set the basic block layout to the original order and set end offsets. 2121a34c753fSRafael Auler PrevBB = nullptr; 2122a34c753fSRafael Auler for (BinaryBasicBlock *BB : BasicBlocks) { 2123a34c753fSRafael Auler BasicBlocksLayout.emplace_back(BB); 2124a34c753fSRafael Auler if (PrevBB) 2125a34c753fSRafael Auler PrevBB->setEndOffset(BB->getOffset()); 2126a34c753fSRafael Auler PrevBB = BB; 2127a34c753fSRafael Auler } 2128a34c753fSRafael Auler PrevBB->setEndOffset(getSize()); 2129a34c753fSRafael Auler 2130a34c753fSRafael Auler updateLayoutIndices(); 2131a34c753fSRafael Auler 2132a34c753fSRafael Auler normalizeCFIState(); 2133a34c753fSRafael Auler 2134a34c753fSRafael Auler // Clean-up memory taken by intermediate structures. 2135a34c753fSRafael Auler // 2136a34c753fSRafael Auler // NB: don't clear Labels list as we may need them if we mark the function 2137a34c753fSRafael Auler // as non-simple later in the process of discovering extra entry points. 2138a34c753fSRafael Auler clearList(Instructions); 2139a34c753fSRafael Auler clearList(OffsetToCFI); 2140a34c753fSRafael Auler clearList(TakenBranches); 2141a34c753fSRafael Auler 2142a34c753fSRafael Auler // Update the state. 2143a34c753fSRafael Auler CurrentState = State::CFG; 2144a34c753fSRafael Auler 2145a34c753fSRafael Auler // Make any necessary adjustments for indirect branches. 2146a34c753fSRafael Auler if (!postProcessIndirectBranches(AllocatorId)) { 2147a34c753fSRafael Auler if (opts::Verbosity) { 2148a34c753fSRafael Auler errs() << "BOLT-WARNING: failed to post-process indirect branches for " 2149a34c753fSRafael Auler << *this << '\n'; 2150a34c753fSRafael Auler } 2151a34c753fSRafael Auler // In relocation mode we want to keep processing the function but avoid 2152a34c753fSRafael Auler // optimizing it. 2153a34c753fSRafael Auler setSimple(false); 2154a34c753fSRafael Auler } 2155a34c753fSRafael Auler 2156a34c753fSRafael Auler clearList(ExternallyReferencedOffsets); 2157a34c753fSRafael Auler clearList(UnknownIndirectBranchOffsets); 2158a34c753fSRafael Auler 2159a34c753fSRafael Auler return true; 2160a34c753fSRafael Auler } 2161a34c753fSRafael Auler 2162a34c753fSRafael Auler void BinaryFunction::postProcessCFG() { 2163a34c753fSRafael Auler if (isSimple() && !BasicBlocks.empty()) { 2164a34c753fSRafael Auler // Convert conditional tail call branches to conditional branches that jump 2165a34c753fSRafael Auler // to a tail call. 2166a34c753fSRafael Auler removeConditionalTailCalls(); 2167a34c753fSRafael Auler 2168a34c753fSRafael Auler postProcessProfile(); 2169a34c753fSRafael Auler 2170a34c753fSRafael Auler // Eliminate inconsistencies between branch instructions and CFG. 2171a34c753fSRafael Auler postProcessBranches(); 2172a34c753fSRafael Auler } 2173a34c753fSRafael Auler 2174a34c753fSRafael Auler calculateMacroOpFusionStats(); 2175a34c753fSRafael Auler 2176a34c753fSRafael Auler // The final cleanup of intermediate structures. 2177a34c753fSRafael Auler clearList(IgnoredBranches); 2178a34c753fSRafael Auler 2179a34c753fSRafael Auler // Remove "Offset" annotations, unless we need an address-translation table 2180a34c753fSRafael Auler // later. This has no cost, since annotations are allocated by a bumpptr 2181a34c753fSRafael Auler // allocator and won't be released anyway until late in the pipeline. 21823652483cSRafael Auler if (!requiresAddressTranslation() && !opts::Instrument) { 2183a34c753fSRafael Auler for (BinaryBasicBlock *BB : layout()) 2184a34c753fSRafael Auler for (MCInst &Inst : *BB) 2185a9cd49d5SAmir Ayupov BC.MIB->clearOffset(Inst); 21863652483cSRafael Auler } 2187a34c753fSRafael Auler 2188a34c753fSRafael Auler assert((!isSimple() || validateCFG()) && 2189a34c753fSRafael Auler "invalid CFG detected after post-processing"); 2190a34c753fSRafael Auler } 2191a34c753fSRafael Auler 2192a34c753fSRafael Auler void BinaryFunction::calculateMacroOpFusionStats() { 2193a34c753fSRafael Auler if (!getBinaryContext().isX86()) 2194a34c753fSRafael Auler return; 2195a34c753fSRafael Auler for (BinaryBasicBlock *BB : layout()) { 2196a34c753fSRafael Auler auto II = BB->getMacroOpFusionPair(); 2197a34c753fSRafael Auler if (II == BB->end()) 2198a34c753fSRafael Auler continue; 2199a34c753fSRafael Auler 2200a34c753fSRafael Auler // Check offset of the second instruction. 2201a34c753fSRafael Auler // FIXME: arch-specific. 2202a9cd49d5SAmir Ayupov const uint32_t Offset = BC.MIB->getOffsetWithDefault(*std::next(II), 0); 2203a34c753fSRafael Auler if (!Offset || (getAddress() + Offset) % 64) 2204a34c753fSRafael Auler continue; 2205a34c753fSRafael Auler 2206a34c753fSRafael Auler LLVM_DEBUG(dbgs() << "\nmissed macro-op fusion at address 0x" 2207a34c753fSRafael Auler << Twine::utohexstr(getAddress() + Offset) 2208a34c753fSRafael Auler << " in function " << *this << "; executed " 2209a34c753fSRafael Auler << BB->getKnownExecutionCount() << " times.\n"); 2210a34c753fSRafael Auler ++BC.MissedMacroFusionPairs; 2211a34c753fSRafael Auler BC.MissedMacroFusionExecCount += BB->getKnownExecutionCount(); 2212a34c753fSRafael Auler } 2213a34c753fSRafael Auler } 2214a34c753fSRafael Auler 2215a34c753fSRafael Auler void BinaryFunction::removeTagsFromProfile() { 2216a34c753fSRafael Auler for (BinaryBasicBlock *BB : BasicBlocks) { 2217a34c753fSRafael Auler if (BB->ExecutionCount == BinaryBasicBlock::COUNT_NO_PROFILE) 2218a34c753fSRafael Auler BB->ExecutionCount = 0; 2219a34c753fSRafael Auler for (BinaryBasicBlock::BinaryBranchInfo &BI : BB->branch_info()) { 2220a34c753fSRafael Auler if (BI.Count != BinaryBasicBlock::COUNT_NO_PROFILE && 2221a34c753fSRafael Auler BI.MispredictedCount != BinaryBasicBlock::COUNT_NO_PROFILE) 2222a34c753fSRafael Auler continue; 2223a34c753fSRafael Auler BI.Count = 0; 2224a34c753fSRafael Auler BI.MispredictedCount = 0; 2225a34c753fSRafael Auler } 2226a34c753fSRafael Auler } 2227a34c753fSRafael Auler } 2228a34c753fSRafael Auler 2229a34c753fSRafael Auler void BinaryFunction::removeConditionalTailCalls() { 2230a34c753fSRafael Auler // Blocks to be appended at the end. 2231a34c753fSRafael Auler std::vector<std::unique_ptr<BinaryBasicBlock>> NewBlocks; 2232a34c753fSRafael Auler 2233a34c753fSRafael Auler for (auto BBI = begin(); BBI != end(); ++BBI) { 2234a34c753fSRafael Auler BinaryBasicBlock &BB = *BBI; 2235a34c753fSRafael Auler MCInst *CTCInstr = BB.getLastNonPseudoInstr(); 2236a34c753fSRafael Auler if (!CTCInstr) 2237a34c753fSRafael Auler continue; 2238a34c753fSRafael Auler 2239a34c753fSRafael Auler Optional<uint64_t> TargetAddressOrNone = 2240a34c753fSRafael Auler BC.MIB->getConditionalTailCall(*CTCInstr); 2241a34c753fSRafael Auler if (!TargetAddressOrNone) 2242a34c753fSRafael Auler continue; 2243a34c753fSRafael Auler 2244a34c753fSRafael Auler // Gather all necessary information about CTC instruction before 2245a34c753fSRafael Auler // annotations are destroyed. 2246a34c753fSRafael Auler const int32_t CFIStateBeforeCTC = BB.getCFIStateAtInstr(CTCInstr); 2247a34c753fSRafael Auler uint64_t CTCTakenCount = BinaryBasicBlock::COUNT_NO_PROFILE; 2248a34c753fSRafael Auler uint64_t CTCMispredCount = BinaryBasicBlock::COUNT_NO_PROFILE; 2249a34c753fSRafael Auler if (hasValidProfile()) { 225040c2e0faSMaksim Panchenko CTCTakenCount = BC.MIB->getAnnotationWithDefault<uint64_t>( 225140c2e0faSMaksim Panchenko *CTCInstr, "CTCTakenCount"); 225240c2e0faSMaksim Panchenko CTCMispredCount = BC.MIB->getAnnotationWithDefault<uint64_t>( 225340c2e0faSMaksim Panchenko *CTCInstr, "CTCMispredCount"); 2254a34c753fSRafael Auler } 2255a34c753fSRafael Auler 2256a34c753fSRafael Auler // Assert that the tail call does not throw. 2257a34c753fSRafael Auler assert(!BC.MIB->getEHInfo(*CTCInstr) && 2258a34c753fSRafael Auler "found tail call with associated landing pad"); 2259a34c753fSRafael Auler 2260a34c753fSRafael Auler // Create a basic block with an unconditional tail call instruction using 2261a34c753fSRafael Auler // the same destination. 2262a34c753fSRafael Auler const MCSymbol *CTCTargetLabel = BC.MIB->getTargetSymbol(*CTCInstr); 2263a34c753fSRafael Auler assert(CTCTargetLabel && "symbol expected for conditional tail call"); 2264a34c753fSRafael Auler MCInst TailCallInstr; 2265a34c753fSRafael Auler BC.MIB->createTailCall(TailCallInstr, CTCTargetLabel, BC.Ctx.get()); 2266a34c753fSRafael Auler // Link new BBs to the original input offset of the BB where the CTC 2267a34c753fSRafael Auler // is, so we can map samples recorded in new BBs back to the original BB 2268a34c753fSRafael Auler // seem in the input binary (if using BAT) 22698228c703SMaksim Panchenko std::unique_ptr<BinaryBasicBlock> TailCallBB = 22708228c703SMaksim Panchenko createBasicBlock(BC.Ctx->createNamedTempSymbol("TC")); 22718228c703SMaksim Panchenko TailCallBB->setOffset(BB.getInputOffset()); 2272a34c753fSRafael Auler TailCallBB->addInstruction(TailCallInstr); 2273a34c753fSRafael Auler TailCallBB->setCFIState(CFIStateBeforeCTC); 2274a34c753fSRafael Auler 2275a34c753fSRafael Auler // Add CFG edge with profile info from BB to TailCallBB. 2276a34c753fSRafael Auler BB.addSuccessor(TailCallBB.get(), CTCTakenCount, CTCMispredCount); 2277a34c753fSRafael Auler 2278a34c753fSRafael Auler // Add execution count for the block. 2279a34c753fSRafael Auler TailCallBB->setExecutionCount(CTCTakenCount); 2280a34c753fSRafael Auler 2281a34c753fSRafael Auler BC.MIB->convertTailCallToJmp(*CTCInstr); 2282a34c753fSRafael Auler 2283a34c753fSRafael Auler BC.MIB->replaceBranchTarget(*CTCInstr, TailCallBB->getLabel(), 2284a34c753fSRafael Auler BC.Ctx.get()); 2285a34c753fSRafael Auler 2286a34c753fSRafael Auler // Add basic block to the list that will be added to the end. 2287a34c753fSRafael Auler NewBlocks.emplace_back(std::move(TailCallBB)); 2288a34c753fSRafael Auler 2289a34c753fSRafael Auler // Swap edges as the TailCallBB corresponds to the taken branch. 2290a34c753fSRafael Auler BB.swapConditionalSuccessors(); 2291a34c753fSRafael Auler 2292a34c753fSRafael Auler // This branch is no longer a conditional tail call. 2293a34c753fSRafael Auler BC.MIB->unsetConditionalTailCall(*CTCInstr); 2294a34c753fSRafael Auler } 2295a34c753fSRafael Auler 229640c2e0faSMaksim Panchenko insertBasicBlocks(std::prev(end()), std::move(NewBlocks), 2297a34c753fSRafael Auler /* UpdateLayout */ true, 2298a34c753fSRafael Auler /* UpdateCFIState */ false); 2299a34c753fSRafael Auler } 2300a34c753fSRafael Auler 2301a34c753fSRafael Auler uint64_t BinaryFunction::getFunctionScore() const { 2302a34c753fSRafael Auler if (FunctionScore != -1) 2303a34c753fSRafael Auler return FunctionScore; 2304a34c753fSRafael Auler 2305a34c753fSRafael Auler if (!isSimple() || !hasValidProfile()) { 2306a34c753fSRafael Auler FunctionScore = 0; 2307a34c753fSRafael Auler return FunctionScore; 2308a34c753fSRafael Auler } 2309a34c753fSRafael Auler 2310a34c753fSRafael Auler uint64_t TotalScore = 0ULL; 2311a34c753fSRafael Auler for (BinaryBasicBlock *BB : layout()) { 2312a34c753fSRafael Auler uint64_t BBExecCount = BB->getExecutionCount(); 2313a34c753fSRafael Auler if (BBExecCount == BinaryBasicBlock::COUNT_NO_PROFILE) 2314a34c753fSRafael Auler continue; 231542465efdSRafael Auler TotalScore += BBExecCount * BB->getNumNonPseudos(); 2316a34c753fSRafael Auler } 2317a34c753fSRafael Auler FunctionScore = TotalScore; 2318a34c753fSRafael Auler return FunctionScore; 2319a34c753fSRafael Auler } 2320a34c753fSRafael Auler 2321a34c753fSRafael Auler void BinaryFunction::annotateCFIState() { 2322a34c753fSRafael Auler assert(CurrentState == State::Disassembled && "unexpected function state"); 2323a34c753fSRafael Auler assert(!BasicBlocks.empty() && "basic block list should not be empty"); 2324a34c753fSRafael Auler 2325a34c753fSRafael Auler // This is an index of the last processed CFI in FDE CFI program. 2326a34c753fSRafael Auler uint32_t State = 0; 2327a34c753fSRafael Auler 2328a34c753fSRafael Auler // This is an index of RememberState CFI reflecting effective state right 2329a34c753fSRafael Auler // after execution of RestoreState CFI. 2330a34c753fSRafael Auler // 2331a34c753fSRafael Auler // It differs from State iff the CFI at (State-1) 2332a34c753fSRafael Auler // was RestoreState (modulo GNU_args_size CFIs, which are ignored). 2333a34c753fSRafael Auler // 2334a34c753fSRafael Auler // This allows us to generate shorter replay sequences when producing new 2335a34c753fSRafael Auler // CFI programs. 2336a34c753fSRafael Auler uint32_t EffectiveState = 0; 2337a34c753fSRafael Auler 2338a34c753fSRafael Auler // For tracking RememberState/RestoreState sequences. 2339a34c753fSRafael Auler std::stack<uint32_t> StateStack; 2340a34c753fSRafael Auler 2341a34c753fSRafael Auler for (BinaryBasicBlock *BB : BasicBlocks) { 2342a34c753fSRafael Auler BB->setCFIState(EffectiveState); 2343a34c753fSRafael Auler 2344a34c753fSRafael Auler for (const MCInst &Instr : *BB) { 2345a34c753fSRafael Auler const MCCFIInstruction *CFI = getCFIFor(Instr); 2346a34c753fSRafael Auler if (!CFI) 2347a34c753fSRafael Auler continue; 2348a34c753fSRafael Auler 2349a34c753fSRafael Auler ++State; 2350a34c753fSRafael Auler 2351a34c753fSRafael Auler switch (CFI->getOperation()) { 2352a34c753fSRafael Auler case MCCFIInstruction::OpRememberState: 2353a34c753fSRafael Auler StateStack.push(EffectiveState); 2354a34c753fSRafael Auler EffectiveState = State; 2355a34c753fSRafael Auler break; 2356a34c753fSRafael Auler case MCCFIInstruction::OpRestoreState: 2357a34c753fSRafael Auler assert(!StateStack.empty() && "corrupt CFI stack"); 2358a34c753fSRafael Auler EffectiveState = StateStack.top(); 2359a34c753fSRafael Auler StateStack.pop(); 2360a34c753fSRafael Auler break; 2361a34c753fSRafael Auler case MCCFIInstruction::OpGnuArgsSize: 2362a34c753fSRafael Auler // OpGnuArgsSize CFIs do not affect the CFI state. 2363a34c753fSRafael Auler break; 2364a34c753fSRafael Auler default: 2365a34c753fSRafael Auler // Any other CFI updates the state. 2366a34c753fSRafael Auler EffectiveState = State; 2367a34c753fSRafael Auler break; 2368a34c753fSRafael Auler } 2369a34c753fSRafael Auler } 2370a34c753fSRafael Auler } 2371a34c753fSRafael Auler 2372a34c753fSRafael Auler assert(StateStack.empty() && "corrupt CFI stack"); 2373a34c753fSRafael Auler } 2374a34c753fSRafael Auler 2375a34c753fSRafael Auler namespace { 2376a34c753fSRafael Auler 2377a34c753fSRafael Auler /// Our full interpretation of a DWARF CFI machine state at a given point 2378a34c753fSRafael Auler struct CFISnapshot { 2379a34c753fSRafael Auler /// CFA register number and offset defining the canonical frame at this 2380a34c753fSRafael Auler /// point, or the number of a rule (CFI state) that computes it with a 2381a34c753fSRafael Auler /// DWARF expression. This number will be negative if it refers to a CFI 2382a34c753fSRafael Auler /// located in the CIE instead of the FDE. 2383a34c753fSRafael Auler uint32_t CFAReg; 2384a34c753fSRafael Auler int32_t CFAOffset; 2385a34c753fSRafael Auler int32_t CFARule; 2386a34c753fSRafael Auler /// Mapping of rules (CFI states) that define the location of each 2387a34c753fSRafael Auler /// register. If absent, no rule defining the location of such register 2388a34c753fSRafael Auler /// was ever read. This number will be negative if it refers to a CFI 2389a34c753fSRafael Auler /// located in the CIE instead of the FDE. 2390a34c753fSRafael Auler DenseMap<int32_t, int32_t> RegRule; 2391a34c753fSRafael Auler 2392a34c753fSRafael Auler /// References to CIE, FDE and expanded instructions after a restore state 2393ebe51c4dSMaksim Panchenko const BinaryFunction::CFIInstrMapType &CIE; 2394ebe51c4dSMaksim Panchenko const BinaryFunction::CFIInstrMapType &FDE; 2395a34c753fSRafael Auler const DenseMap<int32_t, SmallVector<int32_t, 4>> &FrameRestoreEquivalents; 2396a34c753fSRafael Auler 2397a34c753fSRafael Auler /// Current FDE CFI number representing the state where the snapshot is at 2398a34c753fSRafael Auler int32_t CurState; 2399a34c753fSRafael Auler 2400a34c753fSRafael Auler /// Used when we don't have information about which state/rule to apply 2401a34c753fSRafael Auler /// to recover the location of either the CFA or a specific register 2402a34c753fSRafael Auler constexpr static int32_t UNKNOWN = std::numeric_limits<int32_t>::min(); 2403a34c753fSRafael Auler 2404a34c753fSRafael Auler private: 2405a34c753fSRafael Auler /// Update our snapshot by executing a single CFI 2406a34c753fSRafael Auler void update(const MCCFIInstruction &Instr, int32_t RuleNumber) { 2407a34c753fSRafael Auler switch (Instr.getOperation()) { 2408a34c753fSRafael Auler case MCCFIInstruction::OpSameValue: 2409a34c753fSRafael Auler case MCCFIInstruction::OpRelOffset: 2410a34c753fSRafael Auler case MCCFIInstruction::OpOffset: 2411a34c753fSRafael Auler case MCCFIInstruction::OpRestore: 2412a34c753fSRafael Auler case MCCFIInstruction::OpUndefined: 2413a34c753fSRafael Auler case MCCFIInstruction::OpRegister: 2414a34c753fSRafael Auler RegRule[Instr.getRegister()] = RuleNumber; 2415a34c753fSRafael Auler break; 2416a34c753fSRafael Auler case MCCFIInstruction::OpDefCfaRegister: 2417a34c753fSRafael Auler CFAReg = Instr.getRegister(); 2418a34c753fSRafael Auler CFARule = UNKNOWN; 2419a34c753fSRafael Auler break; 2420a34c753fSRafael Auler case MCCFIInstruction::OpDefCfaOffset: 2421a34c753fSRafael Auler CFAOffset = Instr.getOffset(); 2422a34c753fSRafael Auler CFARule = UNKNOWN; 2423a34c753fSRafael Auler break; 2424a34c753fSRafael Auler case MCCFIInstruction::OpDefCfa: 2425a34c753fSRafael Auler CFAReg = Instr.getRegister(); 2426a34c753fSRafael Auler CFAOffset = Instr.getOffset(); 2427a34c753fSRafael Auler CFARule = UNKNOWN; 2428a34c753fSRafael Auler break; 2429a34c753fSRafael Auler case MCCFIInstruction::OpEscape: { 2430a34c753fSRafael Auler Optional<uint8_t> Reg = readDWARFExpressionTargetReg(Instr.getValues()); 2431a34c753fSRafael Auler // Handle DW_CFA_def_cfa_expression 2432a34c753fSRafael Auler if (!Reg) { 2433a34c753fSRafael Auler CFARule = RuleNumber; 2434a34c753fSRafael Auler break; 2435a34c753fSRafael Auler } 2436a34c753fSRafael Auler RegRule[*Reg] = RuleNumber; 2437a34c753fSRafael Auler break; 2438a34c753fSRafael Auler } 2439a34c753fSRafael Auler case MCCFIInstruction::OpAdjustCfaOffset: 2440a34c753fSRafael Auler case MCCFIInstruction::OpWindowSave: 2441a34c753fSRafael Auler case MCCFIInstruction::OpNegateRAState: 2442a34c753fSRafael Auler case MCCFIInstruction::OpLLVMDefAspaceCfa: 2443a34c753fSRafael Auler llvm_unreachable("unsupported CFI opcode"); 2444a34c753fSRafael Auler break; 2445a34c753fSRafael Auler case MCCFIInstruction::OpRememberState: 2446a34c753fSRafael Auler case MCCFIInstruction::OpRestoreState: 2447a34c753fSRafael Auler case MCCFIInstruction::OpGnuArgsSize: 2448a34c753fSRafael Auler // do not affect CFI state 2449a34c753fSRafael Auler break; 2450a34c753fSRafael Auler } 2451a34c753fSRafael Auler } 2452a34c753fSRafael Auler 2453a34c753fSRafael Auler public: 2454a34c753fSRafael Auler /// Advance state reading FDE CFI instructions up to State number 2455a34c753fSRafael Auler void advanceTo(int32_t State) { 2456a34c753fSRafael Auler for (int32_t I = CurState, E = State; I != E; ++I) { 2457a34c753fSRafael Auler const MCCFIInstruction &Instr = FDE[I]; 2458a34c753fSRafael Auler if (Instr.getOperation() != MCCFIInstruction::OpRestoreState) { 2459a34c753fSRafael Auler update(Instr, I); 2460a34c753fSRafael Auler continue; 2461a34c753fSRafael Auler } 2462a34c753fSRafael Auler // If restore state instruction, fetch the equivalent CFIs that have 2463a34c753fSRafael Auler // the same effect of this restore. This is used to ensure remember- 2464a34c753fSRafael Auler // restore pairs are completely removed. 2465a34c753fSRafael Auler auto Iter = FrameRestoreEquivalents.find(I); 2466a34c753fSRafael Auler if (Iter == FrameRestoreEquivalents.end()) 2467a34c753fSRafael Auler continue; 24683652483cSRafael Auler for (int32_t RuleNumber : Iter->second) 2469a34c753fSRafael Auler update(FDE[RuleNumber], RuleNumber); 2470a34c753fSRafael Auler } 2471a34c753fSRafael Auler 2472a34c753fSRafael Auler assert(((CFAReg != (uint32_t)UNKNOWN && CFAOffset != UNKNOWN) || 2473a34c753fSRafael Auler CFARule != UNKNOWN) && 2474a34c753fSRafael Auler "CIE did not define default CFA?"); 2475a34c753fSRafael Auler 2476a34c753fSRafael Auler CurState = State; 2477a34c753fSRafael Auler } 2478a34c753fSRafael Auler 2479a34c753fSRafael Auler /// Interpret all CIE and FDE instructions up until CFI State number and 2480a34c753fSRafael Auler /// populate this snapshot 2481a34c753fSRafael Auler CFISnapshot( 2482ebe51c4dSMaksim Panchenko const BinaryFunction::CFIInstrMapType &CIE, 2483ebe51c4dSMaksim Panchenko const BinaryFunction::CFIInstrMapType &FDE, 2484a34c753fSRafael Auler const DenseMap<int32_t, SmallVector<int32_t, 4>> &FrameRestoreEquivalents, 2485a34c753fSRafael Auler int32_t State) 2486a34c753fSRafael Auler : CIE(CIE), FDE(FDE), FrameRestoreEquivalents(FrameRestoreEquivalents) { 2487a34c753fSRafael Auler CFAReg = UNKNOWN; 2488a34c753fSRafael Auler CFAOffset = UNKNOWN; 2489a34c753fSRafael Auler CFARule = UNKNOWN; 2490a34c753fSRafael Auler CurState = 0; 2491a34c753fSRafael Auler 2492a34c753fSRafael Auler for (int32_t I = 0, E = CIE.size(); I != E; ++I) { 2493a34c753fSRafael Auler const MCCFIInstruction &Instr = CIE[I]; 2494a34c753fSRafael Auler update(Instr, -I); 2495a34c753fSRafael Auler } 2496a34c753fSRafael Auler 2497a34c753fSRafael Auler advanceTo(State); 2498a34c753fSRafael Auler } 2499a34c753fSRafael Auler }; 2500a34c753fSRafael Auler 2501a34c753fSRafael Auler /// A CFI snapshot with the capability of checking if incremental additions to 2502a34c753fSRafael Auler /// it are redundant. This is used to ensure we do not emit two CFI instructions 2503a34c753fSRafael Auler /// back-to-back that are doing the same state change, or to avoid emitting a 2504a34c753fSRafael Auler /// CFI at all when the state at that point would not be modified after that CFI 2505a34c753fSRafael Auler struct CFISnapshotDiff : public CFISnapshot { 2506a34c753fSRafael Auler bool RestoredCFAReg{false}; 2507a34c753fSRafael Auler bool RestoredCFAOffset{false}; 2508a34c753fSRafael Auler DenseMap<int32_t, bool> RestoredRegs; 2509a34c753fSRafael Auler 2510a34c753fSRafael Auler CFISnapshotDiff(const CFISnapshot &S) : CFISnapshot(S) {} 2511a34c753fSRafael Auler 2512a34c753fSRafael Auler CFISnapshotDiff( 2513ebe51c4dSMaksim Panchenko const BinaryFunction::CFIInstrMapType &CIE, 2514ebe51c4dSMaksim Panchenko const BinaryFunction::CFIInstrMapType &FDE, 2515a34c753fSRafael Auler const DenseMap<int32_t, SmallVector<int32_t, 4>> &FrameRestoreEquivalents, 2516a34c753fSRafael Auler int32_t State) 2517a34c753fSRafael Auler : CFISnapshot(CIE, FDE, FrameRestoreEquivalents, State) {} 2518a34c753fSRafael Auler 2519a34c753fSRafael Auler /// Return true if applying Instr to this state is redundant and can be 2520a34c753fSRafael Auler /// dismissed. 2521a34c753fSRafael Auler bool isRedundant(const MCCFIInstruction &Instr) { 2522a34c753fSRafael Auler switch (Instr.getOperation()) { 2523a34c753fSRafael Auler case MCCFIInstruction::OpSameValue: 2524a34c753fSRafael Auler case MCCFIInstruction::OpRelOffset: 2525a34c753fSRafael Auler case MCCFIInstruction::OpOffset: 2526a34c753fSRafael Auler case MCCFIInstruction::OpRestore: 2527a34c753fSRafael Auler case MCCFIInstruction::OpUndefined: 2528a34c753fSRafael Auler case MCCFIInstruction::OpRegister: 2529a34c753fSRafael Auler case MCCFIInstruction::OpEscape: { 2530a34c753fSRafael Auler uint32_t Reg; 2531a34c753fSRafael Auler if (Instr.getOperation() != MCCFIInstruction::OpEscape) { 2532a34c753fSRafael Auler Reg = Instr.getRegister(); 2533a34c753fSRafael Auler } else { 2534a34c753fSRafael Auler Optional<uint8_t> R = readDWARFExpressionTargetReg(Instr.getValues()); 2535a34c753fSRafael Auler // Handle DW_CFA_def_cfa_expression 2536a34c753fSRafael Auler if (!R) { 2537a34c753fSRafael Auler if (RestoredCFAReg && RestoredCFAOffset) 2538a34c753fSRafael Auler return true; 2539a34c753fSRafael Auler RestoredCFAReg = true; 2540a34c753fSRafael Auler RestoredCFAOffset = true; 2541a34c753fSRafael Auler return false; 2542a34c753fSRafael Auler } 2543a34c753fSRafael Auler Reg = *R; 2544a34c753fSRafael Auler } 2545a34c753fSRafael Auler if (RestoredRegs[Reg]) 2546a34c753fSRafael Auler return true; 2547a34c753fSRafael Auler RestoredRegs[Reg] = true; 2548a34c753fSRafael Auler const int32_t CurRegRule = 2549a34c753fSRafael Auler RegRule.find(Reg) != RegRule.end() ? RegRule[Reg] : UNKNOWN; 2550a34c753fSRafael Auler if (CurRegRule == UNKNOWN) { 2551a34c753fSRafael Auler if (Instr.getOperation() == MCCFIInstruction::OpRestore || 2552a34c753fSRafael Auler Instr.getOperation() == MCCFIInstruction::OpSameValue) 2553a34c753fSRafael Auler return true; 2554a34c753fSRafael Auler return false; 2555a34c753fSRafael Auler } 2556a34c753fSRafael Auler const MCCFIInstruction &LastDef = 2557a34c753fSRafael Auler CurRegRule < 0 ? CIE[-CurRegRule] : FDE[CurRegRule]; 2558a34c753fSRafael Auler return LastDef == Instr; 2559a34c753fSRafael Auler } 2560a34c753fSRafael Auler case MCCFIInstruction::OpDefCfaRegister: 2561a34c753fSRafael Auler if (RestoredCFAReg) 2562a34c753fSRafael Auler return true; 2563a34c753fSRafael Auler RestoredCFAReg = true; 2564a34c753fSRafael Auler return CFAReg == Instr.getRegister(); 2565a34c753fSRafael Auler case MCCFIInstruction::OpDefCfaOffset: 2566a34c753fSRafael Auler if (RestoredCFAOffset) 2567a34c753fSRafael Auler return true; 2568a34c753fSRafael Auler RestoredCFAOffset = true; 2569a34c753fSRafael Auler return CFAOffset == Instr.getOffset(); 2570a34c753fSRafael Auler case MCCFIInstruction::OpDefCfa: 2571a34c753fSRafael Auler if (RestoredCFAReg && RestoredCFAOffset) 2572a34c753fSRafael Auler return true; 2573a34c753fSRafael Auler RestoredCFAReg = true; 2574a34c753fSRafael Auler RestoredCFAOffset = true; 2575a34c753fSRafael Auler return CFAReg == Instr.getRegister() && CFAOffset == Instr.getOffset(); 2576a34c753fSRafael Auler case MCCFIInstruction::OpAdjustCfaOffset: 2577a34c753fSRafael Auler case MCCFIInstruction::OpWindowSave: 2578a34c753fSRafael Auler case MCCFIInstruction::OpNegateRAState: 2579a34c753fSRafael Auler case MCCFIInstruction::OpLLVMDefAspaceCfa: 2580a34c753fSRafael Auler llvm_unreachable("unsupported CFI opcode"); 2581a34c753fSRafael Auler return false; 2582a34c753fSRafael Auler case MCCFIInstruction::OpRememberState: 2583a34c753fSRafael Auler case MCCFIInstruction::OpRestoreState: 2584a34c753fSRafael Auler case MCCFIInstruction::OpGnuArgsSize: 2585a34c753fSRafael Auler // do not affect CFI state 2586a34c753fSRafael Auler return true; 2587a34c753fSRafael Auler } 2588a34c753fSRafael Auler return false; 2589a34c753fSRafael Auler } 2590a34c753fSRafael Auler }; 2591a34c753fSRafael Auler 2592a34c753fSRafael Auler } // end anonymous namespace 2593a34c753fSRafael Auler 2594a34c753fSRafael Auler bool BinaryFunction::replayCFIInstrs(int32_t FromState, int32_t ToState, 2595a34c753fSRafael Auler BinaryBasicBlock *InBB, 2596a34c753fSRafael Auler BinaryBasicBlock::iterator InsertIt) { 2597a34c753fSRafael Auler if (FromState == ToState) 2598a34c753fSRafael Auler return true; 2599a34c753fSRafael Auler assert(FromState < ToState && "can only replay CFIs forward"); 2600a34c753fSRafael Auler 2601a34c753fSRafael Auler CFISnapshotDiff CFIDiff(CIEFrameInstructions, FrameInstructions, 2602a34c753fSRafael Auler FrameRestoreEquivalents, FromState); 2603a34c753fSRafael Auler 2604a34c753fSRafael Auler std::vector<uint32_t> NewCFIs; 2605a34c753fSRafael Auler for (int32_t CurState = FromState; CurState < ToState; ++CurState) { 2606a34c753fSRafael Auler MCCFIInstruction *Instr = &FrameInstructions[CurState]; 2607a34c753fSRafael Auler if (Instr->getOperation() == MCCFIInstruction::OpRestoreState) { 2608a34c753fSRafael Auler auto Iter = FrameRestoreEquivalents.find(CurState); 2609a34c753fSRafael Auler assert(Iter != FrameRestoreEquivalents.end()); 261040c2e0faSMaksim Panchenko NewCFIs.insert(NewCFIs.end(), Iter->second.begin(), Iter->second.end()); 2611a34c753fSRafael Auler // RestoreState / Remember will be filtered out later by CFISnapshotDiff, 2612a34c753fSRafael Auler // so we might as well fall-through here. 2613a34c753fSRafael Auler } 2614a34c753fSRafael Auler NewCFIs.push_back(CurState); 2615a34c753fSRafael Auler continue; 2616a34c753fSRafael Auler } 2617a34c753fSRafael Auler 2618a34c753fSRafael Auler // Replay instructions while avoiding duplicates 2619a34c753fSRafael Auler for (auto I = NewCFIs.rbegin(), E = NewCFIs.rend(); I != E; ++I) { 2620a34c753fSRafael Auler if (CFIDiff.isRedundant(FrameInstructions[*I])) 2621a34c753fSRafael Auler continue; 2622a34c753fSRafael Auler InsertIt = addCFIPseudo(InBB, InsertIt, *I); 2623a34c753fSRafael Auler } 2624a34c753fSRafael Auler 2625a34c753fSRafael Auler return true; 2626a34c753fSRafael Auler } 2627a34c753fSRafael Auler 2628a34c753fSRafael Auler SmallVector<int32_t, 4> 2629a34c753fSRafael Auler BinaryFunction::unwindCFIState(int32_t FromState, int32_t ToState, 2630a34c753fSRafael Auler BinaryBasicBlock *InBB, 2631a34c753fSRafael Auler BinaryBasicBlock::iterator &InsertIt) { 2632a34c753fSRafael Auler SmallVector<int32_t, 4> NewStates; 2633a34c753fSRafael Auler 2634a34c753fSRafael Auler CFISnapshot ToCFITable(CIEFrameInstructions, FrameInstructions, 2635a34c753fSRafael Auler FrameRestoreEquivalents, ToState); 2636a34c753fSRafael Auler CFISnapshotDiff FromCFITable(ToCFITable); 2637a34c753fSRafael Auler FromCFITable.advanceTo(FromState); 2638a34c753fSRafael Auler 2639a34c753fSRafael Auler auto undoStateDefCfa = [&]() { 2640a34c753fSRafael Auler if (ToCFITable.CFARule == CFISnapshot::UNKNOWN) { 2641a34c753fSRafael Auler FrameInstructions.emplace_back(MCCFIInstruction::cfiDefCfa( 2642a34c753fSRafael Auler nullptr, ToCFITable.CFAReg, ToCFITable.CFAOffset)); 2643a34c753fSRafael Auler if (FromCFITable.isRedundant(FrameInstructions.back())) { 2644a34c753fSRafael Auler FrameInstructions.pop_back(); 2645a34c753fSRafael Auler return; 2646a34c753fSRafael Auler } 2647a34c753fSRafael Auler NewStates.push_back(FrameInstructions.size() - 1); 2648a34c753fSRafael Auler InsertIt = addCFIPseudo(InBB, InsertIt, FrameInstructions.size() - 1); 2649a34c753fSRafael Auler ++InsertIt; 2650a34c753fSRafael Auler } else if (ToCFITable.CFARule < 0) { 2651a34c753fSRafael Auler if (FromCFITable.isRedundant(CIEFrameInstructions[-ToCFITable.CFARule])) 2652a34c753fSRafael Auler return; 2653a34c753fSRafael Auler NewStates.push_back(FrameInstructions.size()); 2654a34c753fSRafael Auler InsertIt = addCFIPseudo(InBB, InsertIt, FrameInstructions.size()); 2655a34c753fSRafael Auler ++InsertIt; 2656a34c753fSRafael Auler FrameInstructions.emplace_back(CIEFrameInstructions[-ToCFITable.CFARule]); 2657a34c753fSRafael Auler } else if (!FromCFITable.isRedundant( 2658a34c753fSRafael Auler FrameInstructions[ToCFITable.CFARule])) { 2659a34c753fSRafael Auler NewStates.push_back(ToCFITable.CFARule); 2660a34c753fSRafael Auler InsertIt = addCFIPseudo(InBB, InsertIt, ToCFITable.CFARule); 2661a34c753fSRafael Auler ++InsertIt; 2662a34c753fSRafael Auler } 2663a34c753fSRafael Auler }; 2664a34c753fSRafael Auler 2665a34c753fSRafael Auler auto undoState = [&](const MCCFIInstruction &Instr) { 2666a34c753fSRafael Auler switch (Instr.getOperation()) { 2667a34c753fSRafael Auler case MCCFIInstruction::OpRememberState: 2668a34c753fSRafael Auler case MCCFIInstruction::OpRestoreState: 2669a34c753fSRafael Auler break; 2670a34c753fSRafael Auler case MCCFIInstruction::OpSameValue: 2671a34c753fSRafael Auler case MCCFIInstruction::OpRelOffset: 2672a34c753fSRafael Auler case MCCFIInstruction::OpOffset: 2673a34c753fSRafael Auler case MCCFIInstruction::OpRestore: 2674a34c753fSRafael Auler case MCCFIInstruction::OpUndefined: 2675a34c753fSRafael Auler case MCCFIInstruction::OpEscape: 2676a34c753fSRafael Auler case MCCFIInstruction::OpRegister: { 2677a34c753fSRafael Auler uint32_t Reg; 2678a34c753fSRafael Auler if (Instr.getOperation() != MCCFIInstruction::OpEscape) { 2679a34c753fSRafael Auler Reg = Instr.getRegister(); 2680a34c753fSRafael Auler } else { 2681a34c753fSRafael Auler Optional<uint8_t> R = readDWARFExpressionTargetReg(Instr.getValues()); 2682a34c753fSRafael Auler // Handle DW_CFA_def_cfa_expression 2683a34c753fSRafael Auler if (!R) { 2684a34c753fSRafael Auler undoStateDefCfa(); 2685a34c753fSRafael Auler return; 2686a34c753fSRafael Auler } 2687a34c753fSRafael Auler Reg = *R; 2688a34c753fSRafael Auler } 2689a34c753fSRafael Auler 2690a34c753fSRafael Auler if (ToCFITable.RegRule.find(Reg) == ToCFITable.RegRule.end()) { 2691a34c753fSRafael Auler FrameInstructions.emplace_back( 2692a34c753fSRafael Auler MCCFIInstruction::createRestore(nullptr, Reg)); 2693a34c753fSRafael Auler if (FromCFITable.isRedundant(FrameInstructions.back())) { 2694a34c753fSRafael Auler FrameInstructions.pop_back(); 2695a34c753fSRafael Auler break; 2696a34c753fSRafael Auler } 2697a34c753fSRafael Auler NewStates.push_back(FrameInstructions.size() - 1); 2698a34c753fSRafael Auler InsertIt = addCFIPseudo(InBB, InsertIt, FrameInstructions.size() - 1); 2699a34c753fSRafael Auler ++InsertIt; 2700a34c753fSRafael Auler break; 2701a34c753fSRafael Auler } 2702a34c753fSRafael Auler const int32_t Rule = ToCFITable.RegRule[Reg]; 2703a34c753fSRafael Auler if (Rule < 0) { 2704a34c753fSRafael Auler if (FromCFITable.isRedundant(CIEFrameInstructions[-Rule])) 2705a34c753fSRafael Auler break; 2706a34c753fSRafael Auler NewStates.push_back(FrameInstructions.size()); 2707a34c753fSRafael Auler InsertIt = addCFIPseudo(InBB, InsertIt, FrameInstructions.size()); 2708a34c753fSRafael Auler ++InsertIt; 2709a34c753fSRafael Auler FrameInstructions.emplace_back(CIEFrameInstructions[-Rule]); 2710a34c753fSRafael Auler break; 2711a34c753fSRafael Auler } 2712a34c753fSRafael Auler if (FromCFITable.isRedundant(FrameInstructions[Rule])) 2713a34c753fSRafael Auler break; 2714a34c753fSRafael Auler NewStates.push_back(Rule); 2715a34c753fSRafael Auler InsertIt = addCFIPseudo(InBB, InsertIt, Rule); 2716a34c753fSRafael Auler ++InsertIt; 2717a34c753fSRafael Auler break; 2718a34c753fSRafael Auler } 2719a34c753fSRafael Auler case MCCFIInstruction::OpDefCfaRegister: 2720a34c753fSRafael Auler case MCCFIInstruction::OpDefCfaOffset: 2721a34c753fSRafael Auler case MCCFIInstruction::OpDefCfa: 2722a34c753fSRafael Auler undoStateDefCfa(); 2723a34c753fSRafael Auler break; 2724a34c753fSRafael Auler case MCCFIInstruction::OpAdjustCfaOffset: 2725a34c753fSRafael Auler case MCCFIInstruction::OpWindowSave: 2726a34c753fSRafael Auler case MCCFIInstruction::OpNegateRAState: 2727a34c753fSRafael Auler case MCCFIInstruction::OpLLVMDefAspaceCfa: 2728a34c753fSRafael Auler llvm_unreachable("unsupported CFI opcode"); 2729a34c753fSRafael Auler break; 2730a34c753fSRafael Auler case MCCFIInstruction::OpGnuArgsSize: 2731a34c753fSRafael Auler // do not affect CFI state 2732a34c753fSRafael Auler break; 2733a34c753fSRafael Auler } 2734a34c753fSRafael Auler }; 2735a34c753fSRafael Auler 2736a34c753fSRafael Auler // Undo all modifications from ToState to FromState 2737a34c753fSRafael Auler for (int32_t I = ToState, E = FromState; I != E; ++I) { 2738a34c753fSRafael Auler const MCCFIInstruction &Instr = FrameInstructions[I]; 2739a34c753fSRafael Auler if (Instr.getOperation() != MCCFIInstruction::OpRestoreState) { 2740a34c753fSRafael Auler undoState(Instr); 2741a34c753fSRafael Auler continue; 2742a34c753fSRafael Auler } 2743a34c753fSRafael Auler auto Iter = FrameRestoreEquivalents.find(I); 2744a34c753fSRafael Auler if (Iter == FrameRestoreEquivalents.end()) 2745a34c753fSRafael Auler continue; 2746a34c753fSRafael Auler for (int32_t State : Iter->second) 2747a34c753fSRafael Auler undoState(FrameInstructions[State]); 2748a34c753fSRafael Auler } 2749a34c753fSRafael Auler 2750a34c753fSRafael Auler return NewStates; 2751a34c753fSRafael Auler } 2752a34c753fSRafael Auler 2753a34c753fSRafael Auler void BinaryFunction::normalizeCFIState() { 2754a34c753fSRafael Auler // Reordering blocks with remember-restore state instructions can be specially 2755a34c753fSRafael Auler // tricky. When rewriting the CFI, we omit remember-restore state instructions 2756a34c753fSRafael Auler // entirely. For restore state, we build a map expanding each restore to the 2757a34c753fSRafael Auler // equivalent unwindCFIState sequence required at that point to achieve the 2758a34c753fSRafael Auler // same effect of the restore. All remember state are then just ignored. 2759a34c753fSRafael Auler std::stack<int32_t> Stack; 2760a34c753fSRafael Auler for (BinaryBasicBlock *CurBB : BasicBlocksLayout) { 2761a34c753fSRafael Auler for (auto II = CurBB->begin(); II != CurBB->end(); ++II) { 2762a34c753fSRafael Auler if (const MCCFIInstruction *CFI = getCFIFor(*II)) { 2763a34c753fSRafael Auler if (CFI->getOperation() == MCCFIInstruction::OpRememberState) { 2764a34c753fSRafael Auler Stack.push(II->getOperand(0).getImm()); 2765a34c753fSRafael Auler continue; 2766a34c753fSRafael Auler } 2767a34c753fSRafael Auler if (CFI->getOperation() == MCCFIInstruction::OpRestoreState) { 2768a34c753fSRafael Auler const int32_t RememberState = Stack.top(); 2769a34c753fSRafael Auler const int32_t CurState = II->getOperand(0).getImm(); 2770a34c753fSRafael Auler FrameRestoreEquivalents[CurState] = 2771a34c753fSRafael Auler unwindCFIState(CurState, RememberState, CurBB, II); 2772a34c753fSRafael Auler Stack.pop(); 2773a34c753fSRafael Auler } 2774a34c753fSRafael Auler } 2775a34c753fSRafael Auler } 2776a34c753fSRafael Auler } 2777a34c753fSRafael Auler } 2778a34c753fSRafael Auler 2779a34c753fSRafael Auler bool BinaryFunction::finalizeCFIState() { 2780a34c753fSRafael Auler LLVM_DEBUG( 2781a34c753fSRafael Auler dbgs() << "Trying to fix CFI states for each BB after reordering.\n"); 2782a34c753fSRafael Auler LLVM_DEBUG(dbgs() << "This is the list of CFI states for each BB of " << *this 2783a34c753fSRafael Auler << ": "); 2784a34c753fSRafael Auler 2785a34c753fSRafael Auler int32_t State = 0; 2786a34c753fSRafael Auler bool SeenCold = false; 2787a34c753fSRafael Auler const char *Sep = ""; 2788a34c753fSRafael Auler (void)Sep; 2789a34c753fSRafael Auler for (BinaryBasicBlock *BB : BasicBlocksLayout) { 2790a34c753fSRafael Auler const int32_t CFIStateAtExit = BB->getCFIStateAtExit(); 2791a34c753fSRafael Auler 2792a34c753fSRafael Auler // Hot-cold border: check if this is the first BB to be allocated in a cold 2793a34c753fSRafael Auler // region (with a different FDE). If yes, we need to reset the CFI state. 2794a34c753fSRafael Auler if (!SeenCold && BB->isCold()) { 2795a34c753fSRafael Auler State = 0; 2796a34c753fSRafael Auler SeenCold = true; 2797a34c753fSRafael Auler } 2798a34c753fSRafael Auler 2799a34c753fSRafael Auler // We need to recover the correct state if it doesn't match expected 2800a34c753fSRafael Auler // state at BB entry point. 2801a34c753fSRafael Auler if (BB->getCFIState() < State) { 2802a34c753fSRafael Auler // In this case, State is currently higher than what this BB expect it 2803a34c753fSRafael Auler // to be. To solve this, we need to insert CFI instructions to undo 2804a34c753fSRafael Auler // the effect of all CFI from BB's state to current State. 2805a34c753fSRafael Auler auto InsertIt = BB->begin(); 2806a34c753fSRafael Auler unwindCFIState(State, BB->getCFIState(), BB, InsertIt); 2807a34c753fSRafael Auler } else if (BB->getCFIState() > State) { 2808a34c753fSRafael Auler // If BB's CFI state is greater than State, it means we are behind in the 2809a34c753fSRafael Auler // state. Just emit all instructions to reach this state at the 2810a34c753fSRafael Auler // beginning of this BB. If this sequence of instructions involve 2811a34c753fSRafael Auler // remember state or restore state, bail out. 2812a34c753fSRafael Auler if (!replayCFIInstrs(State, BB->getCFIState(), BB, BB->begin())) 2813a34c753fSRafael Auler return false; 2814a34c753fSRafael Auler } 2815a34c753fSRafael Auler 2816a34c753fSRafael Auler State = CFIStateAtExit; 2817a34c753fSRafael Auler LLVM_DEBUG(dbgs() << Sep << State; Sep = ", "); 2818a34c753fSRafael Auler } 2819a34c753fSRafael Auler LLVM_DEBUG(dbgs() << "\n"); 2820a34c753fSRafael Auler 2821a34c753fSRafael Auler for (BinaryBasicBlock *BB : BasicBlocksLayout) { 2822a34c753fSRafael Auler for (auto II = BB->begin(); II != BB->end();) { 2823a34c753fSRafael Auler const MCCFIInstruction *CFI = getCFIFor(*II); 282440c2e0faSMaksim Panchenko if (CFI && (CFI->getOperation() == MCCFIInstruction::OpRememberState || 2825a34c753fSRafael Auler CFI->getOperation() == MCCFIInstruction::OpRestoreState)) { 2826a34c753fSRafael Auler II = BB->eraseInstruction(II); 2827a34c753fSRafael Auler } else { 2828a34c753fSRafael Auler ++II; 2829a34c753fSRafael Auler } 2830a34c753fSRafael Auler } 2831a34c753fSRafael Auler } 2832a34c753fSRafael Auler 2833a34c753fSRafael Auler return true; 2834a34c753fSRafael Auler } 2835a34c753fSRafael Auler 2836a34c753fSRafael Auler bool BinaryFunction::requiresAddressTranslation() const { 2837a34c753fSRafael Auler return opts::EnableBAT || hasSDTMarker() || hasPseudoProbe(); 2838a34c753fSRafael Auler } 2839a34c753fSRafael Auler 2840a34c753fSRafael Auler uint64_t BinaryFunction::getInstructionCount() const { 2841a34c753fSRafael Auler uint64_t Count = 0; 28423652483cSRafael Auler for (BinaryBasicBlock *const &Block : BasicBlocksLayout) 2843a34c753fSRafael Auler Count += Block->getNumNonPseudos(); 2844a34c753fSRafael Auler return Count; 2845a34c753fSRafael Auler } 2846a34c753fSRafael Auler 284740c2e0faSMaksim Panchenko bool BinaryFunction::hasLayoutChanged() const { return ModifiedLayout; } 2848a34c753fSRafael Auler 2849a34c753fSRafael Auler uint64_t BinaryFunction::getEditDistance() const { 2850a34c753fSRafael Auler return ComputeEditDistance<BinaryBasicBlock *>(BasicBlocksPreviousLayout, 2851a34c753fSRafael Auler BasicBlocksLayout); 2852a34c753fSRafael Auler } 2853a34c753fSRafael Auler 2854a34c753fSRafael Auler void BinaryFunction::clearDisasmState() { 2855a34c753fSRafael Auler clearList(Instructions); 2856a34c753fSRafael Auler clearList(IgnoredBranches); 2857a34c753fSRafael Auler clearList(TakenBranches); 2858a34c753fSRafael Auler 2859a34c753fSRafael Auler if (BC.HasRelocations) { 28603652483cSRafael Auler for (std::pair<const uint32_t, MCSymbol *> &LI : Labels) 2861a34c753fSRafael Auler BC.UndefinedSymbols.insert(LI.second); 28623652483cSRafael Auler if (FunctionEndLabel) 2863a34c753fSRafael Auler BC.UndefinedSymbols.insert(FunctionEndLabel); 2864a34c753fSRafael Auler } 2865a34c753fSRafael Auler } 2866a34c753fSRafael Auler 2867a34c753fSRafael Auler void BinaryFunction::setTrapOnEntry() { 2868a34c753fSRafael Auler clearDisasmState(); 2869a34c753fSRafael Auler 2870a34c753fSRafael Auler auto addTrapAtOffset = [&](uint64_t Offset) { 2871a34c753fSRafael Auler MCInst TrapInstr; 2872a34c753fSRafael Auler BC.MIB->createTrap(TrapInstr); 2873a34c753fSRafael Auler addInstruction(Offset, std::move(TrapInstr)); 2874a34c753fSRafael Auler }; 2875a34c753fSRafael Auler 2876a34c753fSRafael Auler addTrapAtOffset(0); 28773652483cSRafael Auler for (const std::pair<const uint32_t, MCSymbol *> &KV : getLabels()) 28783652483cSRafael Auler if (getSecondaryEntryPointSymbol(KV.second)) 2879a34c753fSRafael Auler addTrapAtOffset(KV.first); 2880a34c753fSRafael Auler 2881a34c753fSRafael Auler TrapsOnEntry = true; 2882a34c753fSRafael Auler } 2883a34c753fSRafael Auler 2884a34c753fSRafael Auler void BinaryFunction::setIgnored() { 2885a34c753fSRafael Auler if (opts::processAllFunctions()) { 2886a34c753fSRafael Auler // We can accept ignored functions before they've been disassembled. 2887a34c753fSRafael Auler // In that case, they would still get disassembled and emited, but not 2888a34c753fSRafael Auler // optimized. 2889a34c753fSRafael Auler assert(CurrentState == State::Empty && 2890a34c753fSRafael Auler "cannot ignore non-empty functions in current mode"); 2891a34c753fSRafael Auler IsIgnored = true; 2892a34c753fSRafael Auler return; 2893a34c753fSRafael Auler } 2894a34c753fSRafael Auler 2895a34c753fSRafael Auler clearDisasmState(); 2896a34c753fSRafael Auler 2897a34c753fSRafael Auler // Clear CFG state too. 2898a34c753fSRafael Auler if (hasCFG()) { 2899a34c753fSRafael Auler releaseCFG(); 2900a34c753fSRafael Auler 29013652483cSRafael Auler for (BinaryBasicBlock *BB : BasicBlocks) 2902a34c753fSRafael Auler delete BB; 2903a34c753fSRafael Auler clearList(BasicBlocks); 2904a34c753fSRafael Auler 29053652483cSRafael Auler for (BinaryBasicBlock *BB : DeletedBasicBlocks) 2906a34c753fSRafael Auler delete BB; 2907a34c753fSRafael Auler clearList(DeletedBasicBlocks); 2908a34c753fSRafael Auler 2909a34c753fSRafael Auler clearList(BasicBlocksLayout); 2910a34c753fSRafael Auler clearList(BasicBlocksPreviousLayout); 2911a34c753fSRafael Auler } 2912a34c753fSRafael Auler 2913a34c753fSRafael Auler CurrentState = State::Empty; 2914a34c753fSRafael Auler 2915a34c753fSRafael Auler IsIgnored = true; 2916a34c753fSRafael Auler IsSimple = false; 2917a34c753fSRafael Auler LLVM_DEBUG(dbgs() << "Ignoring " << getPrintName() << '\n'); 2918a34c753fSRafael Auler } 2919a34c753fSRafael Auler 2920a34c753fSRafael Auler void BinaryFunction::duplicateConstantIslands() { 2921a34c753fSRafael Auler assert(Islands && "function expected to have constant islands"); 2922a34c753fSRafael Auler 2923a34c753fSRafael Auler for (BinaryBasicBlock *BB : layout()) { 2924a34c753fSRafael Auler if (!BB->isCold()) 2925a34c753fSRafael Auler continue; 2926a34c753fSRafael Auler 2927a34c753fSRafael Auler for (MCInst &Inst : *BB) { 2928a34c753fSRafael Auler int OpNum = 0; 2929a34c753fSRafael Auler for (MCOperand &Operand : Inst) { 2930a34c753fSRafael Auler if (!Operand.isExpr()) { 2931a34c753fSRafael Auler ++OpNum; 2932a34c753fSRafael Auler continue; 2933a34c753fSRafael Auler } 2934a34c753fSRafael Auler const MCSymbol *Symbol = BC.MIB->getTargetSymbol(Inst, OpNum); 2935a34c753fSRafael Auler // Check if this is an island symbol 2936a34c753fSRafael Auler if (!Islands->Symbols.count(Symbol) && 2937a34c753fSRafael Auler !Islands->ProxySymbols.count(Symbol)) 2938a34c753fSRafael Auler continue; 2939a34c753fSRafael Auler 2940a34c753fSRafael Auler // Create cold symbol, if missing 2941a34c753fSRafael Auler auto ISym = Islands->ColdSymbols.find(Symbol); 2942a34c753fSRafael Auler MCSymbol *ColdSymbol; 2943a34c753fSRafael Auler if (ISym != Islands->ColdSymbols.end()) { 2944a34c753fSRafael Auler ColdSymbol = ISym->second; 2945a34c753fSRafael Auler } else { 2946a34c753fSRafael Auler ColdSymbol = BC.Ctx->getOrCreateSymbol(Symbol->getName() + ".cold"); 2947a34c753fSRafael Auler Islands->ColdSymbols[Symbol] = ColdSymbol; 2948a34c753fSRafael Auler // Check if this is a proxy island symbol and update owner proxy map 2949a34c753fSRafael Auler if (Islands->ProxySymbols.count(Symbol)) { 2950a34c753fSRafael Auler BinaryFunction *Owner = Islands->ProxySymbols[Symbol]; 2951a34c753fSRafael Auler auto IProxiedSym = Owner->Islands->Proxies[this].find(Symbol); 2952a34c753fSRafael Auler Owner->Islands->ColdProxies[this][IProxiedSym->second] = ColdSymbol; 2953a34c753fSRafael Auler } 2954a34c753fSRafael Auler } 2955a34c753fSRafael Auler 2956a34c753fSRafael Auler // Update instruction reference 2957a34c753fSRafael Auler Operand = MCOperand::createExpr(BC.MIB->getTargetExprFor( 2958a34c753fSRafael Auler Inst, 2959a34c753fSRafael Auler MCSymbolRefExpr::create(ColdSymbol, MCSymbolRefExpr::VK_None, 2960a34c753fSRafael Auler *BC.Ctx), 2961a34c753fSRafael Auler *BC.Ctx, 0)); 2962a34c753fSRafael Auler ++OpNum; 2963a34c753fSRafael Auler } 2964a34c753fSRafael Auler } 2965a34c753fSRafael Auler } 2966a34c753fSRafael Auler } 2967a34c753fSRafael Auler 2968a34c753fSRafael Auler namespace { 2969a34c753fSRafael Auler 2970a34c753fSRafael Auler #ifndef MAX_PATH 2971a34c753fSRafael Auler #define MAX_PATH 255 2972a34c753fSRafael Auler #endif 2973a34c753fSRafael Auler 297440c2e0faSMaksim Panchenko std::string constructFilename(std::string Filename, std::string Annotation, 2975a34c753fSRafael Auler std::string Suffix) { 2976a34c753fSRafael Auler std::replace(Filename.begin(), Filename.end(), '/', '-'); 29773652483cSRafael Auler if (!Annotation.empty()) 2978a34c753fSRafael Auler Annotation.insert(0, "-"); 2979a34c753fSRafael Auler if (Filename.size() + Annotation.size() + Suffix.size() > MAX_PATH) { 2980a34c753fSRafael Auler assert(Suffix.size() + Annotation.size() <= MAX_PATH); 2981a34c753fSRafael Auler if (opts::Verbosity >= 1) { 2982a34c753fSRafael Auler errs() << "BOLT-WARNING: Filename \"" << Filename << Annotation << Suffix 2983a34c753fSRafael Auler << "\" exceeds the " << MAX_PATH << " size limit, truncating.\n"; 2984a34c753fSRafael Auler } 2985a34c753fSRafael Auler Filename.resize(MAX_PATH - (Suffix.size() + Annotation.size())); 2986a34c753fSRafael Auler } 2987a34c753fSRafael Auler Filename += Annotation; 2988a34c753fSRafael Auler Filename += Suffix; 2989a34c753fSRafael Auler return Filename; 2990a34c753fSRafael Auler } 2991a34c753fSRafael Auler 2992a34c753fSRafael Auler std::string formatEscapes(const std::string &Str) { 2993a34c753fSRafael Auler std::string Result; 2994a34c753fSRafael Auler for (unsigned I = 0; I < Str.size(); ++I) { 2995a34c753fSRafael Auler char C = Str[I]; 2996a34c753fSRafael Auler switch (C) { 2997a34c753fSRafael Auler case '\n': 2998a34c753fSRafael Auler Result += " "; 2999a34c753fSRafael Auler break; 3000a34c753fSRafael Auler case '"': 3001a34c753fSRafael Auler break; 3002a34c753fSRafael Auler default: 3003a34c753fSRafael Auler Result += C; 3004a34c753fSRafael Auler break; 3005a34c753fSRafael Auler } 3006a34c753fSRafael Auler } 3007a34c753fSRafael Auler return Result; 3008a34c753fSRafael Auler } 3009a34c753fSRafael Auler 301040c2e0faSMaksim Panchenko } // namespace 3011a34c753fSRafael Auler 3012a34c753fSRafael Auler void BinaryFunction::dumpGraph(raw_ostream &OS) const { 30136333e5ddSAmir Ayupov OS << "digraph \"" << getPrintName() << "\" {\n" 30146333e5ddSAmir Ayupov << "node [fontname=courier, shape=box, style=filled, colorscheme=brbg9]\n"; 3015a34c753fSRafael Auler uint64_t Offset = Address; 3016a34c753fSRafael Auler for (BinaryBasicBlock *BB : BasicBlocks) { 3017d2c87699SAmir Ayupov auto LayoutPos = llvm::find(BasicBlocksLayout, BB); 3018a34c753fSRafael Auler unsigned Layout = LayoutPos - BasicBlocksLayout.begin(); 3019a34c753fSRafael Auler const char *ColdStr = BB->isCold() ? " (cold)" : ""; 30206333e5ddSAmir Ayupov std::vector<std::string> Attrs; 30216333e5ddSAmir Ayupov // Bold box for entry points 30226333e5ddSAmir Ayupov if (isEntryPoint(*BB)) 30236333e5ddSAmir Ayupov Attrs.push_back("penwidth=2"); 30246333e5ddSAmir Ayupov if (BLI && BLI->getLoopFor(BB)) { 30256333e5ddSAmir Ayupov // Distinguish innermost loops 30266333e5ddSAmir Ayupov const BinaryLoop *Loop = BLI->getLoopFor(BB); 30276333e5ddSAmir Ayupov if (Loop->isInnermost()) 30286333e5ddSAmir Ayupov Attrs.push_back("fillcolor=6"); 30296333e5ddSAmir Ayupov else // some outer loop 30306333e5ddSAmir Ayupov Attrs.push_back("fillcolor=4"); 30316333e5ddSAmir Ayupov } else { // non-loopy code 30326333e5ddSAmir Ayupov Attrs.push_back("fillcolor=5"); 30336333e5ddSAmir Ayupov } 30346333e5ddSAmir Ayupov ListSeparator LS; 30356333e5ddSAmir Ayupov OS << "\"" << BB->getName() << "\" ["; 30366333e5ddSAmir Ayupov for (StringRef Attr : Attrs) 30376333e5ddSAmir Ayupov OS << LS << Attr; 30386333e5ddSAmir Ayupov OS << "]\n"; 3039cc23c64fSAmir Ayupov OS << format("\"%s\" [label=\"%s%s\\n(C:%lu,O:%lu,I:%u,L:%u,CFI:%u)\\n", 3040cc23c64fSAmir Ayupov BB->getName().data(), BB->getName().data(), ColdStr, 3041cc23c64fSAmir Ayupov BB->getKnownExecutionCount(), BB->getOffset(), getIndex(BB), 3042cc23c64fSAmir Ayupov Layout, BB->getCFIState()); 3043cc23c64fSAmir Ayupov 3044a34c753fSRafael Auler if (opts::DotToolTipCode) { 3045a34c753fSRafael Auler std::string Str; 3046a34c753fSRafael Auler raw_string_ostream CS(Str); 3047cc23c64fSAmir Ayupov Offset = BC.printInstructions(CS, BB->begin(), BB->end(), Offset, this, 3048cc23c64fSAmir Ayupov /* PrintMCInst = */ false, 3049cc23c64fSAmir Ayupov /* PrintMemData = */ false, 3050cc23c64fSAmir Ayupov /* PrintRelocations = */ false, 3051cc23c64fSAmir Ayupov /* Endl = */ R"(\\l)"); 3052cc23c64fSAmir Ayupov OS << formatEscapes(CS.str()) << '\n'; 3053a34c753fSRafael Auler } 3054cc23c64fSAmir Ayupov OS << "\"]\n"; 3055a34c753fSRafael Auler 3056a34c753fSRafael Auler // analyzeBranch is just used to get the names of the branch 3057a34c753fSRafael Auler // opcodes. 3058a34c753fSRafael Auler const MCSymbol *TBB = nullptr; 3059a34c753fSRafael Auler const MCSymbol *FBB = nullptr; 3060a34c753fSRafael Auler MCInst *CondBranch = nullptr; 3061a34c753fSRafael Auler MCInst *UncondBranch = nullptr; 306240c2e0faSMaksim Panchenko const bool Success = BB->analyzeBranch(TBB, FBB, CondBranch, UncondBranch); 3063a34c753fSRafael Auler 3064a34c753fSRafael Auler const MCInst *LastInstr = BB->getLastNonPseudoInstr(); 3065a34c753fSRafael Auler const bool IsJumpTable = LastInstr && BC.MIB->getJumpTable(*LastInstr); 3066a34c753fSRafael Auler 3067a34c753fSRafael Auler auto BI = BB->branch_info_begin(); 3068a34c753fSRafael Auler for (BinaryBasicBlock *Succ : BB->successors()) { 3069a34c753fSRafael Auler std::string Branch; 3070a34c753fSRafael Auler if (Success) { 3071a34c753fSRafael Auler if (Succ == BB->getConditionalSuccessor(true)) { 3072a34c753fSRafael Auler Branch = CondBranch ? std::string(BC.InstPrinter->getOpcodeName( 3073a34c753fSRafael Auler CondBranch->getOpcode())) 3074a34c753fSRafael Auler : "TB"; 3075a34c753fSRafael Auler } else if (Succ == BB->getConditionalSuccessor(false)) { 3076a34c753fSRafael Auler Branch = UncondBranch ? std::string(BC.InstPrinter->getOpcodeName( 3077a34c753fSRafael Auler UncondBranch->getOpcode())) 3078a34c753fSRafael Auler : "FB"; 3079a34c753fSRafael Auler } else { 3080a34c753fSRafael Auler Branch = "FT"; 3081a34c753fSRafael Auler } 3082a34c753fSRafael Auler } 30833652483cSRafael Auler if (IsJumpTable) 3084a34c753fSRafael Auler Branch = "JT"; 308540c2e0faSMaksim Panchenko OS << format("\"%s\" -> \"%s\" [label=\"%s", BB->getName().data(), 308640c2e0faSMaksim Panchenko Succ->getName().data(), Branch.c_str()); 3087a34c753fSRafael Auler 3088a34c753fSRafael Auler if (BB->getExecutionCount() != COUNT_NO_PROFILE && 3089a34c753fSRafael Auler BI->MispredictedCount != BinaryBasicBlock::COUNT_INFERRED) { 3090a34c753fSRafael Auler OS << "\\n(C:" << BI->Count << ",M:" << BI->MispredictedCount << ")"; 3091a34c753fSRafael Auler } else if (ExecutionCount != COUNT_NO_PROFILE && 3092a34c753fSRafael Auler BI->Count != BinaryBasicBlock::COUNT_NO_PROFILE) { 3093a34c753fSRafael Auler OS << "\\n(IC:" << BI->Count << ")"; 3094a34c753fSRafael Auler } 3095a34c753fSRafael Auler OS << "\"]\n"; 3096a34c753fSRafael Auler 3097a34c753fSRafael Auler ++BI; 3098a34c753fSRafael Auler } 3099a34c753fSRafael Auler for (BinaryBasicBlock *LP : BB->landing_pads()) { 3100a34c753fSRafael Auler OS << format("\"%s\" -> \"%s\" [constraint=false style=dashed]\n", 310140c2e0faSMaksim Panchenko BB->getName().data(), LP->getName().data()); 3102a34c753fSRafael Auler } 3103a34c753fSRafael Auler } 3104a34c753fSRafael Auler OS << "}\n"; 3105a34c753fSRafael Auler } 3106a34c753fSRafael Auler 3107a34c753fSRafael Auler void BinaryFunction::viewGraph() const { 3108a34c753fSRafael Auler SmallString<MAX_PATH> Filename; 3109a34c753fSRafael Auler if (std::error_code EC = 3110a34c753fSRafael Auler sys::fs::createTemporaryFile("bolt-cfg", "dot", Filename)) { 3111a34c753fSRafael Auler errs() << "BOLT-ERROR: " << EC.message() << ", unable to create " 3112a34c753fSRafael Auler << " bolt-cfg-XXXXX.dot temporary file.\n"; 3113a34c753fSRafael Auler return; 3114a34c753fSRafael Auler } 3115a34c753fSRafael Auler dumpGraphToFile(std::string(Filename)); 31163652483cSRafael Auler if (DisplayGraph(Filename)) 3117a34c753fSRafael Auler errs() << "BOLT-ERROR: Can't display " << Filename << " with graphviz.\n"; 3118a34c753fSRafael Auler if (std::error_code EC = sys::fs::remove(Filename)) { 3119a34c753fSRafael Auler errs() << "BOLT-WARNING: " << EC.message() << ", failed to remove " 3120a34c753fSRafael Auler << Filename << "\n"; 3121a34c753fSRafael Auler } 3122a34c753fSRafael Auler } 3123a34c753fSRafael Auler 3124a34c753fSRafael Auler void BinaryFunction::dumpGraphForPass(std::string Annotation) const { 3125798e92c6SAmir Ayupov if (!opts::shouldPrint(*this)) 3126798e92c6SAmir Ayupov return; 3127798e92c6SAmir Ayupov 3128a34c753fSRafael Auler std::string Filename = constructFilename(getPrintName(), Annotation, ".dot"); 3129798e92c6SAmir Ayupov if (opts::Verbosity >= 1) 3130798e92c6SAmir Ayupov outs() << "BOLT-INFO: dumping CFG to " << Filename << "\n"; 3131a34c753fSRafael Auler dumpGraphToFile(Filename); 3132a34c753fSRafael Auler } 3133a34c753fSRafael Auler 3134a34c753fSRafael Auler void BinaryFunction::dumpGraphToFile(std::string Filename) const { 3135a34c753fSRafael Auler std::error_code EC; 3136a34c753fSRafael Auler raw_fd_ostream of(Filename, EC, sys::fs::OF_None); 3137a34c753fSRafael Auler if (EC) { 3138a34c753fSRafael Auler if (opts::Verbosity >= 1) { 3139a34c753fSRafael Auler errs() << "BOLT-WARNING: " << EC.message() << ", unable to open " 3140a34c753fSRafael Auler << Filename << " for output.\n"; 3141a34c753fSRafael Auler } 3142a34c753fSRafael Auler return; 3143a34c753fSRafael Auler } 3144a34c753fSRafael Auler dumpGraph(of); 3145a34c753fSRafael Auler } 3146a34c753fSRafael Auler 3147a34c753fSRafael Auler bool BinaryFunction::validateCFG() const { 3148a34c753fSRafael Auler bool Valid = true; 31493652483cSRafael Auler for (BinaryBasicBlock *BB : BasicBlocks) 3150a34c753fSRafael Auler Valid &= BB->validateSuccessorInvariants(); 3151a34c753fSRafael Auler 3152a34c753fSRafael Auler if (!Valid) 3153a34c753fSRafael Auler return Valid; 3154a34c753fSRafael Auler 3155a34c753fSRafael Auler // Make sure all blocks in CFG are valid. 3156a34c753fSRafael Auler auto validateBlock = [this](const BinaryBasicBlock *BB, StringRef Desc) { 3157a34c753fSRafael Auler if (!BB->isValid()) { 3158a34c753fSRafael Auler errs() << "BOLT-ERROR: deleted " << Desc << " " << BB->getName() 3159a34c753fSRafael Auler << " detected in:\n"; 3160a34c753fSRafael Auler this->dump(); 3161a34c753fSRafael Auler return false; 3162a34c753fSRafael Auler } 3163a34c753fSRafael Auler return true; 3164a34c753fSRafael Auler }; 3165a34c753fSRafael Auler for (const BinaryBasicBlock *BB : BasicBlocks) { 3166a34c753fSRafael Auler if (!validateBlock(BB, "block")) 3167a34c753fSRafael Auler return false; 3168a34c753fSRafael Auler for (const BinaryBasicBlock *PredBB : BB->predecessors()) 3169a34c753fSRafael Auler if (!validateBlock(PredBB, "predecessor")) 3170a34c753fSRafael Auler return false; 3171a34c753fSRafael Auler for (const BinaryBasicBlock *SuccBB : BB->successors()) 3172a34c753fSRafael Auler if (!validateBlock(SuccBB, "successor")) 3173a34c753fSRafael Auler return false; 3174a34c753fSRafael Auler for (const BinaryBasicBlock *LP : BB->landing_pads()) 3175a34c753fSRafael Auler if (!validateBlock(LP, "landing pad")) 3176a34c753fSRafael Auler return false; 3177a34c753fSRafael Auler for (const BinaryBasicBlock *Thrower : BB->throwers()) 3178a34c753fSRafael Auler if (!validateBlock(Thrower, "thrower")) 3179a34c753fSRafael Auler return false; 3180a34c753fSRafael Auler } 3181a34c753fSRafael Auler 3182a34c753fSRafael Auler for (const BinaryBasicBlock *BB : BasicBlocks) { 3183a34c753fSRafael Auler std::unordered_set<const BinaryBasicBlock *> BBLandingPads; 3184a34c753fSRafael Auler for (const BinaryBasicBlock *LP : BB->landing_pads()) { 3185a34c753fSRafael Auler if (BBLandingPads.count(LP)) { 3186a34c753fSRafael Auler errs() << "BOLT-ERROR: duplicate landing pad detected in" 3187a34c753fSRafael Auler << BB->getName() << " in function " << *this << '\n'; 3188a34c753fSRafael Auler return false; 3189a34c753fSRafael Auler } 3190a34c753fSRafael Auler BBLandingPads.insert(LP); 3191a34c753fSRafael Auler } 3192a34c753fSRafael Auler 3193a34c753fSRafael Auler std::unordered_set<const BinaryBasicBlock *> BBThrowers; 3194a34c753fSRafael Auler for (const BinaryBasicBlock *Thrower : BB->throwers()) { 3195a34c753fSRafael Auler if (BBThrowers.count(Thrower)) { 319640c2e0faSMaksim Panchenko errs() << "BOLT-ERROR: duplicate thrower detected in" << BB->getName() 319740c2e0faSMaksim Panchenko << " in function " << *this << '\n'; 3198a34c753fSRafael Auler return false; 3199a34c753fSRafael Auler } 3200a34c753fSRafael Auler BBThrowers.insert(Thrower); 3201a34c753fSRafael Auler } 3202a34c753fSRafael Auler 3203a34c753fSRafael Auler for (const BinaryBasicBlock *LPBlock : BB->landing_pads()) { 3204d2c87699SAmir Ayupov if (!llvm::is_contained(LPBlock->throwers(), BB)) { 320540c2e0faSMaksim Panchenko errs() << "BOLT-ERROR: inconsistent landing pad detected in " << *this 320640c2e0faSMaksim Panchenko << ": " << BB->getName() << " is in LandingPads but not in " 320740c2e0faSMaksim Panchenko << LPBlock->getName() << " Throwers\n"; 3208a34c753fSRafael Auler return false; 3209a34c753fSRafael Auler } 3210a34c753fSRafael Auler } 3211a34c753fSRafael Auler for (const BinaryBasicBlock *Thrower : BB->throwers()) { 3212d2c87699SAmir Ayupov if (!llvm::is_contained(Thrower->landing_pads(), BB)) { 321340c2e0faSMaksim Panchenko errs() << "BOLT-ERROR: inconsistent thrower detected in " << *this 321440c2e0faSMaksim Panchenko << ": " << BB->getName() << " is in Throwers list but not in " 321540c2e0faSMaksim Panchenko << Thrower->getName() << " LandingPads\n"; 3216a34c753fSRafael Auler return false; 3217a34c753fSRafael Auler } 3218a34c753fSRafael Auler } 3219a34c753fSRafael Auler } 3220a34c753fSRafael Auler 3221a34c753fSRafael Auler return Valid; 3222a34c753fSRafael Auler } 3223a34c753fSRafael Auler 3224a34c753fSRafael Auler void BinaryFunction::fixBranches() { 3225a34c753fSRafael Auler auto &MIB = BC.MIB; 3226a34c753fSRafael Auler MCContext *Ctx = BC.Ctx.get(); 3227a34c753fSRafael Auler 3228a34c753fSRafael Auler for (unsigned I = 0, E = BasicBlocksLayout.size(); I != E; ++I) { 3229a34c753fSRafael Auler BinaryBasicBlock *BB = BasicBlocksLayout[I]; 3230a34c753fSRafael Auler const MCSymbol *TBB = nullptr; 3231a34c753fSRafael Auler const MCSymbol *FBB = nullptr; 3232a34c753fSRafael Auler MCInst *CondBranch = nullptr; 3233a34c753fSRafael Auler MCInst *UncondBranch = nullptr; 3234a34c753fSRafael Auler if (!BB->analyzeBranch(TBB, FBB, CondBranch, UncondBranch)) 3235a34c753fSRafael Auler continue; 3236a34c753fSRafael Auler 3237a34c753fSRafael Auler // We will create unconditional branch with correct destination if needed. 3238a34c753fSRafael Auler if (UncondBranch) 3239a34c753fSRafael Auler BB->eraseInstruction(BB->findInstruction(UncondBranch)); 3240a34c753fSRafael Auler 3241a34c753fSRafael Auler // Basic block that follows the current one in the final layout. 3242a34c753fSRafael Auler const BinaryBasicBlock *NextBB = nullptr; 3243a34c753fSRafael Auler if (I + 1 != E && BB->isCold() == BasicBlocksLayout[I + 1]->isCold()) 3244a34c753fSRafael Auler NextBB = BasicBlocksLayout[I + 1]; 3245a34c753fSRafael Auler 3246a34c753fSRafael Auler if (BB->succ_size() == 1) { 3247a34c753fSRafael Auler // __builtin_unreachable() could create a conditional branch that 3248a34c753fSRafael Auler // falls-through into the next function - hence the block will have only 3249a34c753fSRafael Auler // one valid successor. Since behaviour is undefined - we replace 3250a34c753fSRafael Auler // the conditional branch with an unconditional if required. 3251a34c753fSRafael Auler if (CondBranch) 3252a34c753fSRafael Auler BB->eraseInstruction(BB->findInstruction(CondBranch)); 3253a34c753fSRafael Auler if (BB->getSuccessor() == NextBB) 3254a34c753fSRafael Auler continue; 3255a34c753fSRafael Auler BB->addBranchInstruction(BB->getSuccessor()); 3256a34c753fSRafael Auler } else if (BB->succ_size() == 2) { 3257a34c753fSRafael Auler assert(CondBranch && "conditional branch expected"); 3258a34c753fSRafael Auler const BinaryBasicBlock *TSuccessor = BB->getConditionalSuccessor(true); 3259a34c753fSRafael Auler const BinaryBasicBlock *FSuccessor = BB->getConditionalSuccessor(false); 3260a34c753fSRafael Auler // Check whether we support reversing this branch direction 3261a34c753fSRafael Auler const bool IsSupported = 3262a34c753fSRafael Auler !MIB->isUnsupportedBranch(CondBranch->getOpcode()); 3263a34c753fSRafael Auler if (NextBB && NextBB == TSuccessor && IsSupported) { 3264a34c753fSRafael Auler std::swap(TSuccessor, FSuccessor); 3265a34c753fSRafael Auler { 3266a34c753fSRafael Auler auto L = BC.scopeLock(); 3267a34c753fSRafael Auler MIB->reverseBranchCondition(*CondBranch, TSuccessor->getLabel(), Ctx); 3268a34c753fSRafael Auler } 3269a34c753fSRafael Auler BB->swapConditionalSuccessors(); 3270a34c753fSRafael Auler } else { 3271a34c753fSRafael Auler auto L = BC.scopeLock(); 3272a34c753fSRafael Auler MIB->replaceBranchTarget(*CondBranch, TSuccessor->getLabel(), Ctx); 3273a34c753fSRafael Auler } 32743652483cSRafael Auler if (TSuccessor == FSuccessor) 3275a34c753fSRafael Auler BB->removeDuplicateConditionalSuccessor(CondBranch); 3276a34c753fSRafael Auler if (!NextBB || 3277a34c753fSRafael Auler ((NextBB != TSuccessor || !IsSupported) && NextBB != FSuccessor)) { 3278a34c753fSRafael Auler // If one of the branches is guaranteed to be "long" while the other 3279a34c753fSRafael Auler // could be "short", then prioritize short for "taken". This will 3280a34c753fSRafael Auler // generate a sequence 1 byte shorter on x86. 3281a34c753fSRafael Auler if (IsSupported && BC.isX86() && 3282a34c753fSRafael Auler TSuccessor->isCold() != FSuccessor->isCold() && 3283a34c753fSRafael Auler BB->isCold() != TSuccessor->isCold()) { 3284a34c753fSRafael Auler std::swap(TSuccessor, FSuccessor); 3285a34c753fSRafael Auler { 3286a34c753fSRafael Auler auto L = BC.scopeLock(); 3287a34c753fSRafael Auler MIB->reverseBranchCondition(*CondBranch, TSuccessor->getLabel(), 3288a34c753fSRafael Auler Ctx); 3289a34c753fSRafael Auler } 3290a34c753fSRafael Auler BB->swapConditionalSuccessors(); 3291a34c753fSRafael Auler } 3292a34c753fSRafael Auler BB->addBranchInstruction(FSuccessor); 3293a34c753fSRafael Auler } 3294a34c753fSRafael Auler } 3295a34c753fSRafael Auler // Cases where the number of successors is 0 (block ends with a 3296a34c753fSRafael Auler // terminator) or more than 2 (switch table) don't require branch 3297a34c753fSRafael Auler // instruction adjustments. 3298a34c753fSRafael Auler } 329940c2e0faSMaksim Panchenko assert((!isSimple() || validateCFG()) && 330040c2e0faSMaksim Panchenko "Invalid CFG detected after fixing branches"); 3301a34c753fSRafael Auler } 3302a34c753fSRafael Auler 3303a34c753fSRafael Auler void BinaryFunction::propagateGnuArgsSizeInfo( 3304a34c753fSRafael Auler MCPlusBuilder::AllocatorIdTy AllocId) { 3305a34c753fSRafael Auler assert(CurrentState == State::Disassembled && "unexpected function state"); 3306a34c753fSRafael Auler 3307a34c753fSRafael Auler if (!hasEHRanges() || !usesGnuArgsSize()) 3308a34c753fSRafael Auler return; 3309a34c753fSRafael Auler 3310a34c753fSRafael Auler // The current value of DW_CFA_GNU_args_size affects all following 3311a34c753fSRafael Auler // invoke instructions until the next CFI overrides it. 3312a34c753fSRafael Auler // It is important to iterate basic blocks in the original order when 3313a34c753fSRafael Auler // assigning the value. 3314a34c753fSRafael Auler uint64_t CurrentGnuArgsSize = 0; 3315a34c753fSRafael Auler for (BinaryBasicBlock *BB : BasicBlocks) { 3316a34c753fSRafael Auler for (auto II = BB->begin(); II != BB->end();) { 3317a34c753fSRafael Auler MCInst &Instr = *II; 3318a34c753fSRafael Auler if (BC.MIB->isCFI(Instr)) { 3319a34c753fSRafael Auler const MCCFIInstruction *CFI = getCFIFor(Instr); 3320a34c753fSRafael Auler if (CFI->getOperation() == MCCFIInstruction::OpGnuArgsSize) { 3321a34c753fSRafael Auler CurrentGnuArgsSize = CFI->getOffset(); 3322a34c753fSRafael Auler // Delete DW_CFA_GNU_args_size instructions and only regenerate 3323a34c753fSRafael Auler // during the final code emission. The information is embedded 3324a34c753fSRafael Auler // inside call instructions. 3325a34c753fSRafael Auler II = BB->erasePseudoInstruction(II); 3326a34c753fSRafael Auler continue; 3327a34c753fSRafael Auler } 3328a34c753fSRafael Auler } else if (BC.MIB->isInvoke(Instr)) { 3329a34c753fSRafael Auler // Add the value of GNU_args_size as an extra operand to invokes. 3330a34c753fSRafael Auler BC.MIB->addGnuArgsSize(Instr, CurrentGnuArgsSize, AllocId); 3331a34c753fSRafael Auler } 3332a34c753fSRafael Auler ++II; 3333a34c753fSRafael Auler } 3334a34c753fSRafael Auler } 3335a34c753fSRafael Auler } 3336a34c753fSRafael Auler 3337a34c753fSRafael Auler void BinaryFunction::postProcessBranches() { 3338a34c753fSRafael Auler if (!isSimple()) 3339a34c753fSRafael Auler return; 3340a34c753fSRafael Auler for (BinaryBasicBlock *BB : BasicBlocksLayout) { 3341a34c753fSRafael Auler auto LastInstrRI = BB->getLastNonPseudo(); 3342a34c753fSRafael Auler if (BB->succ_size() == 1) { 3343a34c753fSRafael Auler if (LastInstrRI != BB->rend() && 3344a34c753fSRafael Auler BC.MIB->isConditionalBranch(*LastInstrRI)) { 3345a34c753fSRafael Auler // __builtin_unreachable() could create a conditional branch that 3346a34c753fSRafael Auler // falls-through into the next function - hence the block will have only 3347a34c753fSRafael Auler // one valid successor. Such behaviour is undefined and thus we remove 3348a34c753fSRafael Auler // the conditional branch while leaving a valid successor. 3349a34c753fSRafael Auler BB->eraseInstruction(std::prev(LastInstrRI.base())); 3350a34c753fSRafael Auler LLVM_DEBUG(dbgs() << "BOLT-DEBUG: erasing conditional branch in " 3351a34c753fSRafael Auler << BB->getName() << " in function " << *this << '\n'); 3352a34c753fSRafael Auler } 3353a34c753fSRafael Auler } else if (BB->succ_size() == 0) { 3354a34c753fSRafael Auler // Ignore unreachable basic blocks. 3355a34c753fSRafael Auler if (BB->pred_size() == 0 || BB->isLandingPad()) 3356a34c753fSRafael Auler continue; 3357a34c753fSRafael Auler 3358a34c753fSRafael Auler // If it's the basic block that does not end up with a terminator - we 3359a34c753fSRafael Auler // insert a return instruction unless it's a call instruction. 3360a34c753fSRafael Auler if (LastInstrRI == BB->rend()) { 3361a34c753fSRafael Auler LLVM_DEBUG( 3362a34c753fSRafael Auler dbgs() << "BOLT-DEBUG: at least one instruction expected in BB " 3363a34c753fSRafael Auler << BB->getName() << " in function " << *this << '\n'); 3364a34c753fSRafael Auler continue; 3365a34c753fSRafael Auler } 3366a34c753fSRafael Auler if (!BC.MIB->isTerminator(*LastInstrRI) && 3367a34c753fSRafael Auler !BC.MIB->isCall(*LastInstrRI)) { 3368a34c753fSRafael Auler LLVM_DEBUG(dbgs() << "BOLT-DEBUG: adding return to basic block " 3369a34c753fSRafael Auler << BB->getName() << " in function " << *this << '\n'); 3370a34c753fSRafael Auler MCInst ReturnInstr; 3371a34c753fSRafael Auler BC.MIB->createReturn(ReturnInstr); 3372a34c753fSRafael Auler BB->addInstruction(ReturnInstr); 3373a34c753fSRafael Auler } 3374a34c753fSRafael Auler } 3375a34c753fSRafael Auler } 3376a34c753fSRafael Auler assert(validateCFG() && "invalid CFG"); 3377a34c753fSRafael Auler } 3378a34c753fSRafael Auler 3379a34c753fSRafael Auler MCSymbol *BinaryFunction::addEntryPointAtOffset(uint64_t Offset) { 3380a34c753fSRafael Auler assert(Offset && "cannot add primary entry point"); 3381a34c753fSRafael Auler assert(CurrentState == State::Empty || CurrentState == State::Disassembled); 3382a34c753fSRafael Auler 3383a34c753fSRafael Auler const uint64_t EntryPointAddress = getAddress() + Offset; 3384a34c753fSRafael Auler MCSymbol *LocalSymbol = getOrCreateLocalLabel(EntryPointAddress); 3385a34c753fSRafael Auler 3386a34c753fSRafael Auler MCSymbol *EntrySymbol = getSecondaryEntryPointSymbol(LocalSymbol); 3387a34c753fSRafael Auler if (EntrySymbol) 3388a34c753fSRafael Auler return EntrySymbol; 3389a34c753fSRafael Auler 3390a34c753fSRafael Auler if (BinaryData *EntryBD = BC.getBinaryDataAtAddress(EntryPointAddress)) { 3391a34c753fSRafael Auler EntrySymbol = EntryBD->getSymbol(); 3392a34c753fSRafael Auler } else { 339340c2e0faSMaksim Panchenko EntrySymbol = BC.getOrCreateGlobalSymbol( 339440c2e0faSMaksim Panchenko EntryPointAddress, Twine("__ENTRY_") + getOneName() + "@"); 3395a34c753fSRafael Auler } 3396a34c753fSRafael Auler SecondaryEntryPoints[LocalSymbol] = EntrySymbol; 3397a34c753fSRafael Auler 3398a34c753fSRafael Auler BC.setSymbolToFunctionMap(EntrySymbol, this); 3399a34c753fSRafael Auler 3400a34c753fSRafael Auler return EntrySymbol; 3401a34c753fSRafael Auler } 3402a34c753fSRafael Auler 3403a34c753fSRafael Auler MCSymbol *BinaryFunction::addEntryPoint(const BinaryBasicBlock &BB) { 3404a34c753fSRafael Auler assert(CurrentState == State::CFG && 3405a34c753fSRafael Auler "basic block can be added as an entry only in a function with CFG"); 3406a34c753fSRafael Auler 3407a34c753fSRafael Auler if (&BB == BasicBlocks.front()) 3408a34c753fSRafael Auler return getSymbol(); 3409a34c753fSRafael Auler 3410a34c753fSRafael Auler MCSymbol *EntrySymbol = getSecondaryEntryPointSymbol(BB); 3411a34c753fSRafael Auler if (EntrySymbol) 3412a34c753fSRafael Auler return EntrySymbol; 3413a34c753fSRafael Auler 3414a34c753fSRafael Auler EntrySymbol = 3415a34c753fSRafael Auler BC.Ctx->getOrCreateSymbol("__ENTRY_" + BB.getLabel()->getName()); 3416a34c753fSRafael Auler 3417a34c753fSRafael Auler SecondaryEntryPoints[BB.getLabel()] = EntrySymbol; 3418a34c753fSRafael Auler 3419a34c753fSRafael Auler BC.setSymbolToFunctionMap(EntrySymbol, this); 3420a34c753fSRafael Auler 3421a34c753fSRafael Auler return EntrySymbol; 3422a34c753fSRafael Auler } 3423a34c753fSRafael Auler 3424a34c753fSRafael Auler MCSymbol *BinaryFunction::getSymbolForEntryID(uint64_t EntryID) { 3425a34c753fSRafael Auler if (EntryID == 0) 3426a34c753fSRafael Auler return getSymbol(); 3427a34c753fSRafael Auler 3428a34c753fSRafael Auler if (!isMultiEntry()) 3429a34c753fSRafael Auler return nullptr; 3430a34c753fSRafael Auler 3431a34c753fSRafael Auler uint64_t NumEntries = 0; 3432a34c753fSRafael Auler if (hasCFG()) { 3433a34c753fSRafael Auler for (BinaryBasicBlock *BB : BasicBlocks) { 3434a34c753fSRafael Auler MCSymbol *EntrySymbol = getSecondaryEntryPointSymbol(*BB); 3435a34c753fSRafael Auler if (!EntrySymbol) 3436a34c753fSRafael Auler continue; 3437a34c753fSRafael Auler if (NumEntries == EntryID) 3438a34c753fSRafael Auler return EntrySymbol; 3439a34c753fSRafael Auler ++NumEntries; 3440a34c753fSRafael Auler } 3441a34c753fSRafael Auler } else { 3442a34c753fSRafael Auler for (std::pair<const uint32_t, MCSymbol *> &KV : Labels) { 3443a34c753fSRafael Auler MCSymbol *EntrySymbol = getSecondaryEntryPointSymbol(KV.second); 3444a34c753fSRafael Auler if (!EntrySymbol) 3445a34c753fSRafael Auler continue; 3446a34c753fSRafael Auler if (NumEntries == EntryID) 3447a34c753fSRafael Auler return EntrySymbol; 3448a34c753fSRafael Auler ++NumEntries; 3449a34c753fSRafael Auler } 3450a34c753fSRafael Auler } 3451a34c753fSRafael Auler 3452a34c753fSRafael Auler return nullptr; 3453a34c753fSRafael Auler } 3454a34c753fSRafael Auler 3455a34c753fSRafael Auler uint64_t BinaryFunction::getEntryIDForSymbol(const MCSymbol *Symbol) const { 3456a34c753fSRafael Auler if (!isMultiEntry()) 3457a34c753fSRafael Auler return 0; 3458a34c753fSRafael Auler 3459a34c753fSRafael Auler for (const MCSymbol *FunctionSymbol : getSymbols()) 3460a34c753fSRafael Auler if (FunctionSymbol == Symbol) 3461a34c753fSRafael Auler return 0; 3462a34c753fSRafael Auler 3463a34c753fSRafael Auler // Check all secondary entries available as either basic blocks or lables. 3464a34c753fSRafael Auler uint64_t NumEntries = 0; 3465a34c753fSRafael Auler for (const BinaryBasicBlock *BB : BasicBlocks) { 3466a34c753fSRafael Auler MCSymbol *EntrySymbol = getSecondaryEntryPointSymbol(*BB); 3467a34c753fSRafael Auler if (!EntrySymbol) 3468a34c753fSRafael Auler continue; 3469a34c753fSRafael Auler if (EntrySymbol == Symbol) 3470a34c753fSRafael Auler return NumEntries; 3471a34c753fSRafael Auler ++NumEntries; 3472a34c753fSRafael Auler } 3473a34c753fSRafael Auler NumEntries = 0; 3474a34c753fSRafael Auler for (const std::pair<const uint32_t, MCSymbol *> &KV : Labels) { 3475a34c753fSRafael Auler MCSymbol *EntrySymbol = getSecondaryEntryPointSymbol(KV.second); 3476a34c753fSRafael Auler if (!EntrySymbol) 3477a34c753fSRafael Auler continue; 3478a34c753fSRafael Auler if (EntrySymbol == Symbol) 3479a34c753fSRafael Auler return NumEntries; 3480a34c753fSRafael Auler ++NumEntries; 3481a34c753fSRafael Auler } 3482a34c753fSRafael Auler 3483a34c753fSRafael Auler llvm_unreachable("symbol not found"); 3484a34c753fSRafael Auler } 3485a34c753fSRafael Auler 3486a34c753fSRafael Auler bool BinaryFunction::forEachEntryPoint(EntryPointCallbackTy Callback) const { 3487a34c753fSRafael Auler bool Status = Callback(0, getSymbol()); 3488a34c753fSRafael Auler if (!isMultiEntry()) 3489a34c753fSRafael Auler return Status; 3490a34c753fSRafael Auler 3491a34c753fSRafael Auler for (const std::pair<const uint32_t, MCSymbol *> &KV : Labels) { 3492a34c753fSRafael Auler if (!Status) 3493a34c753fSRafael Auler break; 3494a34c753fSRafael Auler 3495a34c753fSRafael Auler MCSymbol *EntrySymbol = getSecondaryEntryPointSymbol(KV.second); 3496a34c753fSRafael Auler if (!EntrySymbol) 3497a34c753fSRafael Auler continue; 3498a34c753fSRafael Auler 3499a34c753fSRafael Auler Status = Callback(KV.first, EntrySymbol); 3500a34c753fSRafael Auler } 3501a34c753fSRafael Auler 3502a34c753fSRafael Auler return Status; 3503a34c753fSRafael Auler } 3504a34c753fSRafael Auler 3505a34c753fSRafael Auler BinaryFunction::BasicBlockOrderType BinaryFunction::dfs() const { 3506a34c753fSRafael Auler BasicBlockOrderType DFS; 3507a34c753fSRafael Auler unsigned Index = 0; 3508a34c753fSRafael Auler std::stack<BinaryBasicBlock *> Stack; 3509a34c753fSRafael Auler 3510a34c753fSRafael Auler // Push entry points to the stack in reverse order. 3511a34c753fSRafael Auler // 3512a34c753fSRafael Auler // NB: we rely on the original order of entries to match. 3513a34c753fSRafael Auler for (auto BBI = layout_rbegin(); BBI != layout_rend(); ++BBI) { 3514a34c753fSRafael Auler BinaryBasicBlock *BB = *BBI; 3515a34c753fSRafael Auler if (isEntryPoint(*BB)) 3516a34c753fSRafael Auler Stack.push(BB); 3517a34c753fSRafael Auler BB->setLayoutIndex(BinaryBasicBlock::InvalidIndex); 3518a34c753fSRafael Auler } 3519a34c753fSRafael Auler 3520a34c753fSRafael Auler while (!Stack.empty()) { 3521a34c753fSRafael Auler BinaryBasicBlock *BB = Stack.top(); 3522a34c753fSRafael Auler Stack.pop(); 3523a34c753fSRafael Auler 3524a34c753fSRafael Auler if (BB->getLayoutIndex() != BinaryBasicBlock::InvalidIndex) 3525a34c753fSRafael Auler continue; 3526a34c753fSRafael Auler 3527a34c753fSRafael Auler BB->setLayoutIndex(Index++); 3528a34c753fSRafael Auler DFS.push_back(BB); 3529a34c753fSRafael Auler 3530a34c753fSRafael Auler for (BinaryBasicBlock *SuccBB : BB->landing_pads()) { 3531a34c753fSRafael Auler Stack.push(SuccBB); 3532a34c753fSRafael Auler } 3533a34c753fSRafael Auler 3534a34c753fSRafael Auler const MCSymbol *TBB = nullptr; 3535a34c753fSRafael Auler const MCSymbol *FBB = nullptr; 3536a34c753fSRafael Auler MCInst *CondBranch = nullptr; 3537a34c753fSRafael Auler MCInst *UncondBranch = nullptr; 353840c2e0faSMaksim Panchenko if (BB->analyzeBranch(TBB, FBB, CondBranch, UncondBranch) && CondBranch && 353940c2e0faSMaksim Panchenko BB->succ_size() == 2) { 3540a34c753fSRafael Auler if (BC.MIB->getCanonicalBranchCondCode(BC.MIB->getCondCode( 3541a34c753fSRafael Auler *CondBranch)) == BC.MIB->getCondCode(*CondBranch)) { 3542a34c753fSRafael Auler Stack.push(BB->getConditionalSuccessor(true)); 3543a34c753fSRafael Auler Stack.push(BB->getConditionalSuccessor(false)); 3544a34c753fSRafael Auler } else { 3545a34c753fSRafael Auler Stack.push(BB->getConditionalSuccessor(false)); 3546a34c753fSRafael Auler Stack.push(BB->getConditionalSuccessor(true)); 3547a34c753fSRafael Auler } 3548a34c753fSRafael Auler } else { 3549a34c753fSRafael Auler for (BinaryBasicBlock *SuccBB : BB->successors()) { 3550a34c753fSRafael Auler Stack.push(SuccBB); 3551a34c753fSRafael Auler } 3552a34c753fSRafael Auler } 3553a34c753fSRafael Auler } 3554a34c753fSRafael Auler 3555a34c753fSRafael Auler return DFS; 3556a34c753fSRafael Auler } 3557a34c753fSRafael Auler 3558a34c753fSRafael Auler size_t BinaryFunction::computeHash(bool UseDFS, 3559a34c753fSRafael Auler OperandHashFuncTy OperandHashFunc) const { 3560a34c753fSRafael Auler if (size() == 0) 3561a34c753fSRafael Auler return 0; 3562a34c753fSRafael Auler 3563a34c753fSRafael Auler assert(hasCFG() && "function is expected to have CFG"); 3564a34c753fSRafael Auler 3565ebe51c4dSMaksim Panchenko const BasicBlockOrderType &Order = UseDFS ? dfs() : BasicBlocksLayout; 3566a34c753fSRafael Auler 3567a34c753fSRafael Auler // The hash is computed by creating a string of all instruction opcodes and 3568a34c753fSRafael Auler // possibly their operands and then hashing that string with std::hash. 3569a34c753fSRafael Auler std::string HashString; 3570a34c753fSRafael Auler for (const BinaryBasicBlock *BB : Order) { 3571a34c753fSRafael Auler for (const MCInst &Inst : *BB) { 3572a34c753fSRafael Auler unsigned Opcode = Inst.getOpcode(); 3573a34c753fSRafael Auler 3574a34c753fSRafael Auler if (BC.MIB->isPseudo(Inst)) 3575a34c753fSRafael Auler continue; 3576a34c753fSRafael Auler 3577a34c753fSRafael Auler // Ignore unconditional jumps since we check CFG consistency by processing 3578a34c753fSRafael Auler // basic blocks in order and do not rely on branches to be in-sync with 3579a34c753fSRafael Auler // CFG. Note that we still use condition code of conditional jumps. 3580a34c753fSRafael Auler if (BC.MIB->isUnconditionalBranch(Inst)) 3581a34c753fSRafael Auler continue; 3582a34c753fSRafael Auler 3583a34c753fSRafael Auler if (Opcode == 0) 3584a34c753fSRafael Auler HashString.push_back(0); 3585a34c753fSRafael Auler 3586a34c753fSRafael Auler while (Opcode) { 3587a34c753fSRafael Auler uint8_t LSB = Opcode & 0xff; 3588a34c753fSRafael Auler HashString.push_back(LSB); 3589a34c753fSRafael Auler Opcode = Opcode >> 8; 3590a34c753fSRafael Auler } 3591a34c753fSRafael Auler 35928cb7a873SAmir Ayupov for (const MCOperand &Op : MCPlus::primeOperands(Inst)) 35938cb7a873SAmir Ayupov HashString.append(OperandHashFunc(Op)); 3594a34c753fSRafael Auler } 3595a34c753fSRafael Auler } 3596a34c753fSRafael Auler 3597a34c753fSRafael Auler return Hash = std::hash<std::string>{}(HashString); 3598a34c753fSRafael Auler } 3599a34c753fSRafael Auler 3600a34c753fSRafael Auler void BinaryFunction::insertBasicBlocks( 3601a34c753fSRafael Auler BinaryBasicBlock *Start, 3602a34c753fSRafael Auler std::vector<std::unique_ptr<BinaryBasicBlock>> &&NewBBs, 360340c2e0faSMaksim Panchenko const bool UpdateLayout, const bool UpdateCFIState, 3604a34c753fSRafael Auler const bool RecomputeLandingPads) { 3605f18fcdabSAmir Ayupov const int64_t StartIndex = Start ? getIndex(Start) : -1LL; 3606a34c753fSRafael Auler const size_t NumNewBlocks = NewBBs.size(); 3607a34c753fSRafael Auler 360840c2e0faSMaksim Panchenko BasicBlocks.insert(BasicBlocks.begin() + (StartIndex + 1), NumNewBlocks, 3609a34c753fSRafael Auler nullptr); 3610a34c753fSRafael Auler 3611f18fcdabSAmir Ayupov int64_t I = StartIndex + 1; 3612a34c753fSRafael Auler for (std::unique_ptr<BinaryBasicBlock> &BB : NewBBs) { 3613a34c753fSRafael Auler assert(!BasicBlocks[I]); 3614a34c753fSRafael Auler BasicBlocks[I++] = BB.release(); 3615a34c753fSRafael Auler } 3616a34c753fSRafael Auler 36173652483cSRafael Auler if (RecomputeLandingPads) 3618a34c753fSRafael Auler recomputeLandingPads(); 36193652483cSRafael Auler else 3620a34c753fSRafael Auler updateBBIndices(0); 3621a34c753fSRafael Auler 36223652483cSRafael Auler if (UpdateLayout) 3623a34c753fSRafael Auler updateLayout(Start, NumNewBlocks); 3624a34c753fSRafael Auler 36253652483cSRafael Auler if (UpdateCFIState) 3626a34c753fSRafael Auler updateCFIState(Start, NumNewBlocks); 3627a34c753fSRafael Auler } 3628a34c753fSRafael Auler 3629a34c753fSRafael Auler BinaryFunction::iterator BinaryFunction::insertBasicBlocks( 3630a34c753fSRafael Auler BinaryFunction::iterator StartBB, 3631a34c753fSRafael Auler std::vector<std::unique_ptr<BinaryBasicBlock>> &&NewBBs, 363240c2e0faSMaksim Panchenko const bool UpdateLayout, const bool UpdateCFIState, 3633a34c753fSRafael Auler const bool RecomputeLandingPads) { 3634a34c753fSRafael Auler const unsigned StartIndex = getIndex(&*StartBB); 3635a34c753fSRafael Auler const size_t NumNewBlocks = NewBBs.size(); 3636a34c753fSRafael Auler 3637a34c753fSRafael Auler BasicBlocks.insert(BasicBlocks.begin() + StartIndex + 1, NumNewBlocks, 3638a34c753fSRafael Auler nullptr); 3639a34c753fSRafael Auler auto RetIter = BasicBlocks.begin() + StartIndex + 1; 3640a34c753fSRafael Auler 3641a34c753fSRafael Auler unsigned I = StartIndex + 1; 3642a34c753fSRafael Auler for (std::unique_ptr<BinaryBasicBlock> &BB : NewBBs) { 3643a34c753fSRafael Auler assert(!BasicBlocks[I]); 3644a34c753fSRafael Auler BasicBlocks[I++] = BB.release(); 3645a34c753fSRafael Auler } 3646a34c753fSRafael Auler 36473652483cSRafael Auler if (RecomputeLandingPads) 3648a34c753fSRafael Auler recomputeLandingPads(); 36493652483cSRafael Auler else 3650a34c753fSRafael Auler updateBBIndices(0); 3651a34c753fSRafael Auler 36523652483cSRafael Auler if (UpdateLayout) 3653a34c753fSRafael Auler updateLayout(*std::prev(RetIter), NumNewBlocks); 3654a34c753fSRafael Auler 36553652483cSRafael Auler if (UpdateCFIState) 3656a34c753fSRafael Auler updateCFIState(*std::prev(RetIter), NumNewBlocks); 3657a34c753fSRafael Auler 3658a34c753fSRafael Auler return RetIter; 3659a34c753fSRafael Auler } 3660a34c753fSRafael Auler 3661a34c753fSRafael Auler void BinaryFunction::updateBBIndices(const unsigned StartIndex) { 36623652483cSRafael Auler for (unsigned I = StartIndex; I < BasicBlocks.size(); ++I) 3663a34c753fSRafael Auler BasicBlocks[I]->Index = I; 3664a34c753fSRafael Auler } 3665a34c753fSRafael Auler 3666a34c753fSRafael Auler void BinaryFunction::updateCFIState(BinaryBasicBlock *Start, 3667a34c753fSRafael Auler const unsigned NumNewBlocks) { 3668a34c753fSRafael Auler const int32_t CFIState = Start->getCFIStateAtExit(); 3669a34c753fSRafael Auler const unsigned StartIndex = getIndex(Start) + 1; 36703652483cSRafael Auler for (unsigned I = 0; I < NumNewBlocks; ++I) 3671a34c753fSRafael Auler BasicBlocks[StartIndex + I]->setCFIState(CFIState); 3672a34c753fSRafael Auler } 3673a34c753fSRafael Auler 3674a34c753fSRafael Auler void BinaryFunction::updateLayout(BinaryBasicBlock *Start, 3675a34c753fSRafael Auler const unsigned NumNewBlocks) { 3676a34c753fSRafael Auler // If start not provided insert new blocks at the beginning 3677a34c753fSRafael Auler if (!Start) { 3678a34c753fSRafael Auler BasicBlocksLayout.insert(layout_begin(), BasicBlocks.begin(), 3679a34c753fSRafael Auler BasicBlocks.begin() + NumNewBlocks); 3680a34c753fSRafael Auler updateLayoutIndices(); 3681a34c753fSRafael Auler return; 3682a34c753fSRafael Auler } 3683a34c753fSRafael Auler 3684a34c753fSRafael Auler // Insert new blocks in the layout immediately after Start. 3685d2c87699SAmir Ayupov auto Pos = llvm::find(layout(), Start); 3686a34c753fSRafael Auler assert(Pos != layout_end()); 3687ae585be1SRafael Auler BasicBlockListType::iterator Begin = 3688ae585be1SRafael Auler std::next(BasicBlocks.begin(), getIndex(Start) + 1); 3689ae585be1SRafael Auler BasicBlockListType::iterator End = 3690ae585be1SRafael Auler std::next(BasicBlocks.begin(), getIndex(Start) + NumNewBlocks + 1); 3691a34c753fSRafael Auler BasicBlocksLayout.insert(Pos + 1, Begin, End); 3692a34c753fSRafael Auler updateLayoutIndices(); 3693a34c753fSRafael Auler } 3694a34c753fSRafael Auler 3695a34c753fSRafael Auler bool BinaryFunction::checkForAmbiguousJumpTables() { 3696a34c753fSRafael Auler SmallSet<uint64_t, 4> JumpTables; 3697a34c753fSRafael Auler for (BinaryBasicBlock *&BB : BasicBlocks) { 3698a34c753fSRafael Auler for (MCInst &Inst : *BB) { 3699a34c753fSRafael Auler if (!BC.MIB->isIndirectBranch(Inst)) 3700a34c753fSRafael Auler continue; 3701a34c753fSRafael Auler uint64_t JTAddress = BC.MIB->getJumpTable(Inst); 3702a34c753fSRafael Auler if (!JTAddress) 3703a34c753fSRafael Auler continue; 3704a34c753fSRafael Auler // This address can be inside another jump table, but we only consider 3705a34c753fSRafael Auler // it ambiguous when the same start address is used, not the same JT 3706a34c753fSRafael Auler // object. 3707a34c753fSRafael Auler if (!JumpTables.count(JTAddress)) { 3708a34c753fSRafael Auler JumpTables.insert(JTAddress); 3709a34c753fSRafael Auler continue; 3710a34c753fSRafael Auler } 3711a34c753fSRafael Auler return true; 3712a34c753fSRafael Auler } 3713a34c753fSRafael Auler } 3714a34c753fSRafael Auler return false; 3715a34c753fSRafael Auler } 3716a34c753fSRafael Auler 3717a34c753fSRafael Auler void BinaryFunction::disambiguateJumpTables( 3718a34c753fSRafael Auler MCPlusBuilder::AllocatorIdTy AllocId) { 3719a34c753fSRafael Auler assert((opts::JumpTables != JTS_BASIC && isSimple()) || !BC.HasRelocations); 3720a34c753fSRafael Auler SmallPtrSet<JumpTable *, 4> JumpTables; 3721a34c753fSRafael Auler for (BinaryBasicBlock *&BB : BasicBlocks) { 3722a34c753fSRafael Auler for (MCInst &Inst : *BB) { 3723a34c753fSRafael Auler if (!BC.MIB->isIndirectBranch(Inst)) 3724a34c753fSRafael Auler continue; 3725a34c753fSRafael Auler JumpTable *JT = getJumpTable(Inst); 3726a34c753fSRafael Auler if (!JT) 3727a34c753fSRafael Auler continue; 3728a34c753fSRafael Auler auto Iter = JumpTables.find(JT); 3729a34c753fSRafael Auler if (Iter == JumpTables.end()) { 3730a34c753fSRafael Auler JumpTables.insert(JT); 3731a34c753fSRafael Auler continue; 3732a34c753fSRafael Auler } 3733a34c753fSRafael Auler // This instruction is an indirect jump using a jump table, but it is 3734a34c753fSRafael Auler // using the same jump table of another jump. Try all our tricks to 3735a34c753fSRafael Auler // extract the jump table symbol and make it point to a new, duplicated JT 3736a34c753fSRafael Auler MCPhysReg BaseReg1; 3737a34c753fSRafael Auler uint64_t Scale; 3738a34c753fSRafael Auler const MCSymbol *Target; 3739a34c753fSRafael Auler // In case we match if our first matcher, first instruction is the one to 3740a34c753fSRafael Auler // patch 3741a34c753fSRafael Auler MCInst *JTLoadInst = &Inst; 3742a34c753fSRafael Auler // Try a standard indirect jump matcher, scale 8 3743a34c753fSRafael Auler std::unique_ptr<MCPlusBuilder::MCInstMatcher> IndJmpMatcher = 3744a34c753fSRafael Auler BC.MIB->matchIndJmp(BC.MIB->matchReg(BaseReg1), 3745a34c753fSRafael Auler BC.MIB->matchImm(Scale), BC.MIB->matchReg(), 3746a34c753fSRafael Auler /*Offset=*/BC.MIB->matchSymbol(Target)); 3747a34c753fSRafael Auler if (!IndJmpMatcher->match( 3748a34c753fSRafael Auler *BC.MRI, *BC.MIB, 3749a34c753fSRafael Auler MutableArrayRef<MCInst>(&*BB->begin(), &Inst + 1), -1) || 375040c2e0faSMaksim Panchenko BaseReg1 != BC.MIB->getNoRegister() || Scale != 8) { 3751a34c753fSRafael Auler MCPhysReg BaseReg2; 3752a34c753fSRafael Auler uint64_t Offset; 3753a34c753fSRafael Auler // Standard JT matching failed. Trying now: 3754a34c753fSRafael Auler // movq "jt.2397/1"(,%rax,8), %rax 3755a34c753fSRafael Auler // jmpq *%rax 3756a34c753fSRafael Auler std::unique_ptr<MCPlusBuilder::MCInstMatcher> LoadMatcherOwner = 3757a34c753fSRafael Auler BC.MIB->matchLoad(BC.MIB->matchReg(BaseReg1), 3758a34c753fSRafael Auler BC.MIB->matchImm(Scale), BC.MIB->matchReg(), 3759a34c753fSRafael Auler /*Offset=*/BC.MIB->matchSymbol(Target)); 3760a34c753fSRafael Auler MCPlusBuilder::MCInstMatcher *LoadMatcher = LoadMatcherOwner.get(); 3761a34c753fSRafael Auler std::unique_ptr<MCPlusBuilder::MCInstMatcher> IndJmpMatcher2 = 3762a34c753fSRafael Auler BC.MIB->matchIndJmp(std::move(LoadMatcherOwner)); 3763a34c753fSRafael Auler if (!IndJmpMatcher2->match( 3764a34c753fSRafael Auler *BC.MRI, *BC.MIB, 3765a34c753fSRafael Auler MutableArrayRef<MCInst>(&*BB->begin(), &Inst + 1), -1) || 3766a34c753fSRafael Auler BaseReg1 != BC.MIB->getNoRegister() || Scale != 8) { 3767a34c753fSRafael Auler // JT matching failed. Trying now: 3768a34c753fSRafael Auler // PIC-style matcher, scale 4 3769a34c753fSRafael Auler // addq %rdx, %rsi 3770a34c753fSRafael Auler // addq %rdx, %rdi 3771a34c753fSRafael Auler // leaq DATAat0x402450(%rip), %r11 3772a34c753fSRafael Auler // movslq (%r11,%rdx,4), %rcx 3773a34c753fSRafael Auler // addq %r11, %rcx 3774a34c753fSRafael Auler // jmpq *%rcx # JUMPTABLE @0x402450 3775a34c753fSRafael Auler std::unique_ptr<MCPlusBuilder::MCInstMatcher> PICIndJmpMatcher = 3776a34c753fSRafael Auler BC.MIB->matchIndJmp(BC.MIB->matchAdd( 3777a34c753fSRafael Auler BC.MIB->matchReg(BaseReg1), 3778a34c753fSRafael Auler BC.MIB->matchLoad(BC.MIB->matchReg(BaseReg2), 3779a34c753fSRafael Auler BC.MIB->matchImm(Scale), BC.MIB->matchReg(), 3780a34c753fSRafael Auler BC.MIB->matchImm(Offset)))); 3781a34c753fSRafael Auler std::unique_ptr<MCPlusBuilder::MCInstMatcher> LEAMatcherOwner = 3782a34c753fSRafael Auler BC.MIB->matchLoadAddr(BC.MIB->matchSymbol(Target)); 3783a34c753fSRafael Auler MCPlusBuilder::MCInstMatcher *LEAMatcher = LEAMatcherOwner.get(); 3784a34c753fSRafael Auler std::unique_ptr<MCPlusBuilder::MCInstMatcher> PICBaseAddrMatcher = 3785a34c753fSRafael Auler BC.MIB->matchIndJmp(BC.MIB->matchAdd(std::move(LEAMatcherOwner), 3786a34c753fSRafael Auler BC.MIB->matchAnyOperand())); 3787a34c753fSRafael Auler if (!PICIndJmpMatcher->match( 3788a34c753fSRafael Auler *BC.MRI, *BC.MIB, 3789a34c753fSRafael Auler MutableArrayRef<MCInst>(&*BB->begin(), &Inst + 1), -1) || 3790a34c753fSRafael Auler Scale != 4 || BaseReg1 != BaseReg2 || Offset != 0 || 3791a34c753fSRafael Auler !PICBaseAddrMatcher->match( 3792a34c753fSRafael Auler *BC.MRI, *BC.MIB, 3793a34c753fSRafael Auler MutableArrayRef<MCInst>(&*BB->begin(), &Inst + 1), -1)) { 3794a34c753fSRafael Auler llvm_unreachable("Failed to extract jump table base"); 3795a34c753fSRafael Auler continue; 3796a34c753fSRafael Auler } 3797a34c753fSRafael Auler // Matched PIC, identify the instruction with the reference to the JT 3798a34c753fSRafael Auler JTLoadInst = LEAMatcher->CurInst; 3799a34c753fSRafael Auler } else { 3800a34c753fSRafael Auler // Matched non-PIC 3801a34c753fSRafael Auler JTLoadInst = LoadMatcher->CurInst; 3802a34c753fSRafael Auler } 3803a34c753fSRafael Auler } 3804a34c753fSRafael Auler 3805a34c753fSRafael Auler uint64_t NewJumpTableID = 0; 3806a34c753fSRafael Auler const MCSymbol *NewJTLabel; 3807a34c753fSRafael Auler std::tie(NewJumpTableID, NewJTLabel) = 3808a34c753fSRafael Auler BC.duplicateJumpTable(*this, JT, Target); 3809a34c753fSRafael Auler { 3810a34c753fSRafael Auler auto L = BC.scopeLock(); 3811a34c753fSRafael Auler BC.MIB->replaceMemOperandDisp(*JTLoadInst, NewJTLabel, BC.Ctx.get()); 3812a34c753fSRafael Auler } 3813a34c753fSRafael Auler // We use a unique ID with the high bit set as address for this "injected" 3814a34c753fSRafael Auler // jump table (not originally in the input binary). 3815a34c753fSRafael Auler BC.MIB->setJumpTable(Inst, NewJumpTableID, 0, AllocId); 3816a34c753fSRafael Auler } 3817a34c753fSRafael Auler } 3818a34c753fSRafael Auler } 3819a34c753fSRafael Auler 3820a34c753fSRafael Auler bool BinaryFunction::replaceJumpTableEntryIn(BinaryBasicBlock *BB, 3821a34c753fSRafael Auler BinaryBasicBlock *OldDest, 3822a34c753fSRafael Auler BinaryBasicBlock *NewDest) { 3823a34c753fSRafael Auler MCInst *Instr = BB->getLastNonPseudoInstr(); 3824a34c753fSRafael Auler if (!Instr || !BC.MIB->isIndirectBranch(*Instr)) 3825a34c753fSRafael Auler return false; 3826a34c753fSRafael Auler uint64_t JTAddress = BC.MIB->getJumpTable(*Instr); 3827a34c753fSRafael Auler assert(JTAddress && "Invalid jump table address"); 3828a34c753fSRafael Auler JumpTable *JT = getJumpTableContainingAddress(JTAddress); 3829a34c753fSRafael Auler assert(JT && "No jump table structure for this indirect branch"); 3830a34c753fSRafael Auler bool Patched = JT->replaceDestination(JTAddress, OldDest->getLabel(), 3831a34c753fSRafael Auler NewDest->getLabel()); 3832a34c753fSRafael Auler (void)Patched; 3833a34c753fSRafael Auler assert(Patched && "Invalid entry to be replaced in jump table"); 3834a34c753fSRafael Auler return true; 3835a34c753fSRafael Auler } 3836a34c753fSRafael Auler 3837a34c753fSRafael Auler BinaryBasicBlock *BinaryFunction::splitEdge(BinaryBasicBlock *From, 3838a34c753fSRafael Auler BinaryBasicBlock *To) { 3839a34c753fSRafael Auler // Create intermediate BB 3840a34c753fSRafael Auler MCSymbol *Tmp; 3841a34c753fSRafael Auler { 3842a34c753fSRafael Auler auto L = BC.scopeLock(); 3843a34c753fSRafael Auler Tmp = BC.Ctx->createNamedTempSymbol("SplitEdge"); 3844a34c753fSRafael Auler } 3845a34c753fSRafael Auler // Link new BBs to the original input offset of the From BB, so we can map 3846a34c753fSRafael Auler // samples recorded in new BBs back to the original BB seem in the input 3847a34c753fSRafael Auler // binary (if using BAT) 38488228c703SMaksim Panchenko std::unique_ptr<BinaryBasicBlock> NewBB = createBasicBlock(Tmp); 38498228c703SMaksim Panchenko NewBB->setOffset(From->getInputOffset()); 3850a34c753fSRafael Auler BinaryBasicBlock *NewBBPtr = NewBB.get(); 3851a34c753fSRafael Auler 3852a34c753fSRafael Auler // Update "From" BB 3853a34c753fSRafael Auler auto I = From->succ_begin(); 3854a34c753fSRafael Auler auto BI = From->branch_info_begin(); 3855a34c753fSRafael Auler for (; I != From->succ_end(); ++I) { 3856a34c753fSRafael Auler if (*I == To) 3857a34c753fSRafael Auler break; 3858a34c753fSRafael Auler ++BI; 3859a34c753fSRafael Auler } 3860a34c753fSRafael Auler assert(I != From->succ_end() && "Invalid CFG edge in splitEdge!"); 3861a34c753fSRafael Auler uint64_t OrigCount = BI->Count; 3862a34c753fSRafael Auler uint64_t OrigMispreds = BI->MispredictedCount; 3863a34c753fSRafael Auler replaceJumpTableEntryIn(From, To, NewBBPtr); 3864a34c753fSRafael Auler From->replaceSuccessor(To, NewBBPtr, OrigCount, OrigMispreds); 3865a34c753fSRafael Auler 3866a34c753fSRafael Auler NewBB->addSuccessor(To, OrigCount, OrigMispreds); 3867a34c753fSRafael Auler NewBB->setExecutionCount(OrigCount); 3868a34c753fSRafael Auler NewBB->setIsCold(From->isCold()); 3869a34c753fSRafael Auler 3870a34c753fSRafael Auler // Update CFI and BB layout with new intermediate BB 3871a34c753fSRafael Auler std::vector<std::unique_ptr<BinaryBasicBlock>> NewBBs; 3872a34c753fSRafael Auler NewBBs.emplace_back(std::move(NewBB)); 3873a34c753fSRafael Auler insertBasicBlocks(From, std::move(NewBBs), true, true, 3874a34c753fSRafael Auler /*RecomputeLandingPads=*/false); 3875a34c753fSRafael Auler return NewBBPtr; 3876a34c753fSRafael Auler } 3877a34c753fSRafael Auler 3878a34c753fSRafael Auler void BinaryFunction::deleteConservativeEdges() { 3879a34c753fSRafael Auler // Our goal is to aggressively remove edges from the CFG that we believe are 3880a34c753fSRafael Auler // wrong. This is used for instrumentation, where it is safe to remove 3881a34c753fSRafael Auler // fallthrough edges because we won't reorder blocks. 3882a34c753fSRafael Auler for (auto I = BasicBlocks.begin(), E = BasicBlocks.end(); I != E; ++I) { 3883a34c753fSRafael Auler BinaryBasicBlock *BB = *I; 3884a34c753fSRafael Auler if (BB->succ_size() != 1 || BB->size() == 0) 3885a34c753fSRafael Auler continue; 3886a34c753fSRafael Auler 3887a34c753fSRafael Auler auto NextBB = std::next(I); 3888a34c753fSRafael Auler MCInst *Last = BB->getLastNonPseudoInstr(); 3889a34c753fSRafael Auler // Fallthrough is a landing pad? Delete this edge (as long as we don't 3890a34c753fSRafael Auler // have a direct jump to it) 3891a34c753fSRafael Auler if ((*BB->succ_begin())->isLandingPad() && NextBB != E && 3892a34c753fSRafael Auler *BB->succ_begin() == *NextBB && Last && !BC.MIB->isBranch(*Last)) { 3893a34c753fSRafael Auler BB->removeAllSuccessors(); 3894a34c753fSRafael Auler continue; 3895a34c753fSRafael Auler } 3896a34c753fSRafael Auler 3897a34c753fSRafael Auler // Look for suspicious calls at the end of BB where gcc may optimize it and 3898a34c753fSRafael Auler // remove the jump to the epilogue when it knows the call won't return. 3899a34c753fSRafael Auler if (!Last || !BC.MIB->isCall(*Last)) 3900a34c753fSRafael Auler continue; 3901a34c753fSRafael Auler 3902a34c753fSRafael Auler const MCSymbol *CalleeSymbol = BC.MIB->getTargetSymbol(*Last); 3903a34c753fSRafael Auler if (!CalleeSymbol) 3904a34c753fSRafael Auler continue; 3905a34c753fSRafael Auler 3906a34c753fSRafael Auler StringRef CalleeName = CalleeSymbol->getName(); 390740c2e0faSMaksim Panchenko if (CalleeName != "__cxa_throw@PLT" && CalleeName != "_Unwind_Resume@PLT" && 390840c2e0faSMaksim Panchenko CalleeName != "__cxa_rethrow@PLT" && CalleeName != "exit@PLT" && 3909a34c753fSRafael Auler CalleeName != "abort@PLT") 3910a34c753fSRafael Auler continue; 3911a34c753fSRafael Auler 3912a34c753fSRafael Auler BB->removeAllSuccessors(); 3913a34c753fSRafael Auler } 3914a34c753fSRafael Auler } 3915a34c753fSRafael Auler 3916a34c753fSRafael Auler bool BinaryFunction::isSymbolValidInScope(const SymbolRef &Symbol, 3917a34c753fSRafael Auler uint64_t SymbolSize) const { 3918a34c753fSRafael Auler // If this symbol is in a different section from the one where the 3919a34c753fSRafael Auler // function symbol is, don't consider it as valid. 3920a34c753fSRafael Auler if (!getOriginSection()->containsAddress( 3921a34c753fSRafael Auler cantFail(Symbol.getAddress(), "cannot get symbol address"))) 3922a34c753fSRafael Auler return false; 3923a34c753fSRafael Auler 3924a34c753fSRafael Auler // Some symbols are tolerated inside function bodies, others are not. 3925a34c753fSRafael Auler // The real function boundaries may not be known at this point. 39268579db96SDenis Revunov if (BC.isMarker(Symbol)) 3927a34c753fSRafael Auler return true; 3928a34c753fSRafael Auler 3929a34c753fSRafael Auler // It's okay to have a zero-sized symbol in the middle of non-zero-sized 3930a34c753fSRafael Auler // function. 3931a34c753fSRafael Auler if (SymbolSize == 0 && containsAddress(cantFail(Symbol.getAddress()))) 3932a34c753fSRafael Auler return true; 3933a34c753fSRafael Auler 3934a34c753fSRafael Auler if (cantFail(Symbol.getType()) != SymbolRef::ST_Unknown) 3935a34c753fSRafael Auler return false; 3936a34c753fSRafael Auler 3937a34c753fSRafael Auler if (cantFail(Symbol.getFlags()) & SymbolRef::SF_Global) 3938a34c753fSRafael Auler return false; 3939a34c753fSRafael Auler 3940a34c753fSRafael Auler return true; 3941a34c753fSRafael Auler } 3942a34c753fSRafael Auler 3943a34c753fSRafael Auler void BinaryFunction::adjustExecutionCount(uint64_t Count) { 3944a34c753fSRafael Auler if (getKnownExecutionCount() == 0 || Count == 0) 3945a34c753fSRafael Auler return; 3946a34c753fSRafael Auler 3947a34c753fSRafael Auler if (ExecutionCount < Count) 3948a34c753fSRafael Auler Count = ExecutionCount; 3949a34c753fSRafael Auler 3950a34c753fSRafael Auler double AdjustmentRatio = ((double)ExecutionCount - Count) / ExecutionCount; 3951a34c753fSRafael Auler if (AdjustmentRatio < 0.0) 3952a34c753fSRafael Auler AdjustmentRatio = 0.0; 3953a34c753fSRafael Auler 3954a34c753fSRafael Auler for (BinaryBasicBlock *&BB : layout()) 3955a34c753fSRafael Auler BB->adjustExecutionCount(AdjustmentRatio); 3956a34c753fSRafael Auler 3957a34c753fSRafael Auler ExecutionCount -= Count; 3958a34c753fSRafael Auler } 3959a34c753fSRafael Auler 3960a34c753fSRafael Auler BinaryFunction::~BinaryFunction() { 39613652483cSRafael Auler for (BinaryBasicBlock *BB : BasicBlocks) 3962a34c753fSRafael Auler delete BB; 39633652483cSRafael Auler for (BinaryBasicBlock *BB : DeletedBasicBlocks) 3964a34c753fSRafael Auler delete BB; 3965a34c753fSRafael Auler } 3966a34c753fSRafael Auler 3967a34c753fSRafael Auler void BinaryFunction::calculateLoopInfo() { 3968a34c753fSRafael Auler // Discover loops. 3969a34c753fSRafael Auler BinaryDominatorTree DomTree; 3970a34c753fSRafael Auler DomTree.recalculate(*this); 3971a34c753fSRafael Auler BLI.reset(new BinaryLoopInfo()); 3972a34c753fSRafael Auler BLI->analyze(DomTree); 3973a34c753fSRafael Auler 3974a34c753fSRafael Auler // Traverse discovered loops and add depth and profile information. 3975a34c753fSRafael Auler std::stack<BinaryLoop *> St; 3976a34c753fSRafael Auler for (auto I = BLI->begin(), E = BLI->end(); I != E; ++I) { 3977a34c753fSRafael Auler St.push(*I); 3978a34c753fSRafael Auler ++BLI->OuterLoops; 3979a34c753fSRafael Auler } 3980a34c753fSRafael Auler 3981a34c753fSRafael Auler while (!St.empty()) { 3982a34c753fSRafael Auler BinaryLoop *L = St.top(); 3983a34c753fSRafael Auler St.pop(); 3984a34c753fSRafael Auler ++BLI->TotalLoops; 3985a34c753fSRafael Auler BLI->MaximumDepth = std::max(L->getLoopDepth(), BLI->MaximumDepth); 3986a34c753fSRafael Auler 3987a34c753fSRafael Auler // Add nested loops in the stack. 39883652483cSRafael Auler for (BinaryLoop::iterator I = L->begin(), E = L->end(); I != E; ++I) 3989a34c753fSRafael Auler St.push(*I); 3990a34c753fSRafael Auler 3991a34c753fSRafael Auler // Skip if no valid profile is found. 3992a34c753fSRafael Auler if (!hasValidProfile()) { 3993a34c753fSRafael Auler L->EntryCount = COUNT_NO_PROFILE; 3994a34c753fSRafael Auler L->ExitCount = COUNT_NO_PROFILE; 3995a34c753fSRafael Auler L->TotalBackEdgeCount = COUNT_NO_PROFILE; 3996a34c753fSRafael Auler continue; 3997a34c753fSRafael Auler } 3998a34c753fSRafael Auler 3999a34c753fSRafael Auler // Compute back edge count. 4000a34c753fSRafael Auler SmallVector<BinaryBasicBlock *, 1> Latches; 4001a34c753fSRafael Auler L->getLoopLatches(Latches); 4002a34c753fSRafael Auler 4003a34c753fSRafael Auler for (BinaryBasicBlock *Latch : Latches) { 4004a34c753fSRafael Auler auto BI = Latch->branch_info_begin(); 4005a34c753fSRafael Auler for (BinaryBasicBlock *Succ : Latch->successors()) { 4006a34c753fSRafael Auler if (Succ == L->getHeader()) { 4007a34c753fSRafael Auler assert(BI->Count != BinaryBasicBlock::COUNT_NO_PROFILE && 4008a34c753fSRafael Auler "profile data not found"); 4009a34c753fSRafael Auler L->TotalBackEdgeCount += BI->Count; 4010a34c753fSRafael Auler } 4011a34c753fSRafael Auler ++BI; 4012a34c753fSRafael Auler } 4013a34c753fSRafael Auler } 4014a34c753fSRafael Auler 4015a34c753fSRafael Auler // Compute entry count. 4016a34c753fSRafael Auler L->EntryCount = L->getHeader()->getExecutionCount() - L->TotalBackEdgeCount; 4017a34c753fSRafael Auler 4018a34c753fSRafael Auler // Compute exit count. 4019a34c753fSRafael Auler SmallVector<BinaryLoop::Edge, 1> ExitEdges; 4020a34c753fSRafael Auler L->getExitEdges(ExitEdges); 4021a34c753fSRafael Auler for (BinaryLoop::Edge &Exit : ExitEdges) { 4022a34c753fSRafael Auler const BinaryBasicBlock *Exiting = Exit.first; 4023a34c753fSRafael Auler const BinaryBasicBlock *ExitTarget = Exit.second; 4024a34c753fSRafael Auler auto BI = Exiting->branch_info_begin(); 4025a34c753fSRafael Auler for (BinaryBasicBlock *Succ : Exiting->successors()) { 4026a34c753fSRafael Auler if (Succ == ExitTarget) { 4027a34c753fSRafael Auler assert(BI->Count != BinaryBasicBlock::COUNT_NO_PROFILE && 4028a34c753fSRafael Auler "profile data not found"); 4029a34c753fSRafael Auler L->ExitCount += BI->Count; 4030a34c753fSRafael Auler } 4031a34c753fSRafael Auler ++BI; 4032a34c753fSRafael Auler } 4033a34c753fSRafael Auler } 4034a34c753fSRafael Auler } 4035a34c753fSRafael Auler } 4036a34c753fSRafael Auler 4037a34c753fSRafael Auler void BinaryFunction::updateOutputValues(const MCAsmLayout &Layout) { 4038a34c753fSRafael Auler if (!isEmitted()) { 4039a34c753fSRafael Auler assert(!isInjected() && "injected function should be emitted"); 4040a34c753fSRafael Auler setOutputAddress(getAddress()); 4041a34c753fSRafael Auler setOutputSize(getSize()); 4042a34c753fSRafael Auler return; 4043a34c753fSRafael Auler } 4044a34c753fSRafael Auler 4045a34c753fSRafael Auler const uint64_t BaseAddress = getCodeSection()->getOutputAddress(); 4046a34c753fSRafael Auler ErrorOr<BinarySection &> ColdSection = getColdCodeSection(); 4047a34c753fSRafael Auler const uint64_t ColdBaseAddress = 4048a34c753fSRafael Auler isSplit() ? ColdSection->getOutputAddress() : 0; 4049a34c753fSRafael Auler if (BC.HasRelocations || isInjected()) { 4050a34c753fSRafael Auler const uint64_t StartOffset = Layout.getSymbolOffset(*getSymbol()); 4051a34c753fSRafael Auler const uint64_t EndOffset = Layout.getSymbolOffset(*getFunctionEndLabel()); 4052a34c753fSRafael Auler setOutputAddress(BaseAddress + StartOffset); 4053a34c753fSRafael Auler setOutputSize(EndOffset - StartOffset); 4054a34c753fSRafael Auler if (hasConstantIsland()) { 4055a34c753fSRafael Auler const uint64_t DataOffset = 4056a34c753fSRafael Auler Layout.getSymbolOffset(*getFunctionConstantIslandLabel()); 4057a34c753fSRafael Auler setOutputDataAddress(BaseAddress + DataOffset); 4058a34c753fSRafael Auler } 4059a34c753fSRafael Auler if (isSplit()) { 4060a34c753fSRafael Auler const MCSymbol *ColdStartSymbol = getColdSymbol(); 4061a34c753fSRafael Auler assert(ColdStartSymbol && ColdStartSymbol->isDefined() && 4062a34c753fSRafael Auler "split function should have defined cold symbol"); 4063a34c753fSRafael Auler const MCSymbol *ColdEndSymbol = getFunctionColdEndLabel(); 4064a34c753fSRafael Auler assert(ColdEndSymbol && ColdEndSymbol->isDefined() && 4065a34c753fSRafael Auler "split function should have defined cold end symbol"); 4066a34c753fSRafael Auler const uint64_t ColdStartOffset = Layout.getSymbolOffset(*ColdStartSymbol); 4067a34c753fSRafael Auler const uint64_t ColdEndOffset = Layout.getSymbolOffset(*ColdEndSymbol); 4068a34c753fSRafael Auler cold().setAddress(ColdBaseAddress + ColdStartOffset); 4069a34c753fSRafael Auler cold().setImageSize(ColdEndOffset - ColdStartOffset); 4070a34c753fSRafael Auler if (hasConstantIsland()) { 4071a34c753fSRafael Auler const uint64_t DataOffset = 4072a34c753fSRafael Auler Layout.getSymbolOffset(*getFunctionColdConstantIslandLabel()); 4073a34c753fSRafael Auler setOutputColdDataAddress(ColdBaseAddress + DataOffset); 4074a34c753fSRafael Auler } 4075a34c753fSRafael Auler } 4076a34c753fSRafael Auler } else { 4077a34c753fSRafael Auler setOutputAddress(getAddress()); 407840c2e0faSMaksim Panchenko setOutputSize(Layout.getSymbolOffset(*getFunctionEndLabel())); 4079a34c753fSRafael Auler } 4080a34c753fSRafael Auler 4081a34c753fSRafael Auler // Update basic block output ranges for the debug info, if we have 4082a34c753fSRafael Auler // secondary entry points in the symbol table to update or if writing BAT. 4083a34c753fSRafael Auler if (!opts::UpdateDebugSections && !isMultiEntry() && 4084a34c753fSRafael Auler !requiresAddressTranslation()) 4085a34c753fSRafael Auler return; 4086a34c753fSRafael Auler 4087a34c753fSRafael Auler // Output ranges should match the input if the body hasn't changed. 4088a34c753fSRafael Auler if (!isSimple() && !BC.HasRelocations) 4089a34c753fSRafael Auler return; 4090a34c753fSRafael Auler 4091a34c753fSRafael Auler // AArch64 may have functions that only contains a constant island (no code). 4092a34c753fSRafael Auler if (layout_begin() == layout_end()) 4093a34c753fSRafael Auler return; 4094a34c753fSRafael Auler 4095a34c753fSRafael Auler BinaryBasicBlock *PrevBB = nullptr; 4096a34c753fSRafael Auler for (auto BBI = layout_begin(), BBE = layout_end(); BBI != BBE; ++BBI) { 4097a34c753fSRafael Auler BinaryBasicBlock *BB = *BBI; 4098a34c753fSRafael Auler assert(BB->getLabel()->isDefined() && "symbol should be defined"); 4099a34c753fSRafael Auler const uint64_t BBBaseAddress = BB->isCold() ? ColdBaseAddress : BaseAddress; 4100a34c753fSRafael Auler if (!BC.HasRelocations) { 4101a34c753fSRafael Auler if (BB->isCold()) { 4102a34c753fSRafael Auler assert(BBBaseAddress == cold().getAddress()); 4103a34c753fSRafael Auler } else { 4104a34c753fSRafael Auler assert(BBBaseAddress == getOutputAddress()); 4105a34c753fSRafael Auler } 4106a34c753fSRafael Auler } 4107a34c753fSRafael Auler const uint64_t BBOffset = Layout.getSymbolOffset(*BB->getLabel()); 4108a34c753fSRafael Auler const uint64_t BBAddress = BBBaseAddress + BBOffset; 4109a34c753fSRafael Auler BB->setOutputStartAddress(BBAddress); 4110a34c753fSRafael Auler 4111a34c753fSRafael Auler if (PrevBB) { 4112a34c753fSRafael Auler uint64_t PrevBBEndAddress = BBAddress; 41133652483cSRafael Auler if (BB->isCold() != PrevBB->isCold()) 411440c2e0faSMaksim Panchenko PrevBBEndAddress = getOutputAddress() + getOutputSize(); 4115a34c753fSRafael Auler PrevBB->setOutputEndAddress(PrevBBEndAddress); 4116a34c753fSRafael Auler } 4117a34c753fSRafael Auler PrevBB = BB; 4118a34c753fSRafael Auler 4119a34c753fSRafael Auler BB->updateOutputValues(Layout); 4120a34c753fSRafael Auler } 412140c2e0faSMaksim Panchenko PrevBB->setOutputEndAddress(PrevBB->isCold() 412240c2e0faSMaksim Panchenko ? cold().getAddress() + cold().getImageSize() 412340c2e0faSMaksim Panchenko : getOutputAddress() + getOutputSize()); 4124a34c753fSRafael Auler } 4125a34c753fSRafael Auler 4126a34c753fSRafael Auler DebugAddressRangesVector BinaryFunction::getOutputAddressRanges() const { 4127a34c753fSRafael Auler DebugAddressRangesVector OutputRanges; 4128a34c753fSRafael Auler 4129a34c753fSRafael Auler if (isFolded()) 4130a34c753fSRafael Auler return OutputRanges; 4131a34c753fSRafael Auler 4132a34c753fSRafael Auler if (IsFragment) 4133a34c753fSRafael Auler return OutputRanges; 4134a34c753fSRafael Auler 4135a34c753fSRafael Auler OutputRanges.emplace_back(getOutputAddress(), 4136a34c753fSRafael Auler getOutputAddress() + getOutputSize()); 4137a34c753fSRafael Auler if (isSplit()) { 4138a34c753fSRafael Auler assert(isEmitted() && "split function should be emitted"); 4139a34c753fSRafael Auler OutputRanges.emplace_back(cold().getAddress(), 4140a34c753fSRafael Auler cold().getAddress() + cold().getImageSize()); 4141a34c753fSRafael Auler } 4142a34c753fSRafael Auler 4143a34c753fSRafael Auler if (isSimple()) 4144a34c753fSRafael Auler return OutputRanges; 4145a34c753fSRafael Auler 4146a34c753fSRafael Auler for (BinaryFunction *Frag : Fragments) { 4147a34c753fSRafael Auler assert(!Frag->isSimple() && 4148a34c753fSRafael Auler "fragment of non-simple function should also be non-simple"); 4149a34c753fSRafael Auler OutputRanges.emplace_back(Frag->getOutputAddress(), 4150a34c753fSRafael Auler Frag->getOutputAddress() + Frag->getOutputSize()); 4151a34c753fSRafael Auler } 4152a34c753fSRafael Auler 4153a34c753fSRafael Auler return OutputRanges; 4154a34c753fSRafael Auler } 4155a34c753fSRafael Auler 4156a34c753fSRafael Auler uint64_t BinaryFunction::translateInputToOutputAddress(uint64_t Address) const { 4157a34c753fSRafael Auler if (isFolded()) 4158a34c753fSRafael Auler return 0; 4159a34c753fSRafael Auler 4160a34c753fSRafael Auler // If the function hasn't changed return the same address. 4161a34c753fSRafael Auler if (!isEmitted()) 4162a34c753fSRafael Auler return Address; 4163a34c753fSRafael Auler 4164a34c753fSRafael Auler if (Address < getAddress()) 4165a34c753fSRafael Auler return 0; 4166a34c753fSRafael Auler 4167a34c753fSRafael Auler // Check if the address is associated with an instruction that is tracked 4168a34c753fSRafael Auler // by address translation. 4169a34c753fSRafael Auler auto KV = InputOffsetToAddressMap.find(Address - getAddress()); 41703652483cSRafael Auler if (KV != InputOffsetToAddressMap.end()) 4171a34c753fSRafael Auler return KV->second; 4172a34c753fSRafael Auler 4173a34c753fSRafael Auler // FIXME: #18950828 - we rely on relative offsets inside basic blocks to stay 4174a34c753fSRafael Auler // intact. Instead we can use pseudo instructions and/or annotations. 4175a34c753fSRafael Auler const uint64_t Offset = Address - getAddress(); 4176a34c753fSRafael Auler const BinaryBasicBlock *BB = getBasicBlockContainingOffset(Offset); 4177a34c753fSRafael Auler if (!BB) { 4178a34c753fSRafael Auler // Special case for address immediately past the end of the function. 4179a34c753fSRafael Auler if (Offset == getSize()) 4180a34c753fSRafael Auler return getOutputAddress() + getOutputSize(); 4181a34c753fSRafael Auler 4182a34c753fSRafael Auler return 0; 4183a34c753fSRafael Auler } 4184a34c753fSRafael Auler 4185a34c753fSRafael Auler return std::min(BB->getOutputAddressRange().first + Offset - BB->getOffset(), 4186a34c753fSRafael Auler BB->getOutputAddressRange().second); 4187a34c753fSRafael Auler } 4188a34c753fSRafael Auler 4189a34c753fSRafael Auler DebugAddressRangesVector BinaryFunction::translateInputToOutputRanges( 4190a34c753fSRafael Auler const DWARFAddressRangesVector &InputRanges) const { 4191a34c753fSRafael Auler DebugAddressRangesVector OutputRanges; 4192a34c753fSRafael Auler 4193a34c753fSRafael Auler if (isFolded()) 4194a34c753fSRafael Auler return OutputRanges; 4195a34c753fSRafael Auler 4196a34c753fSRafael Auler // If the function hasn't changed return the same ranges. 4197a34c753fSRafael Auler if (!isEmitted()) { 4198a34c753fSRafael Auler OutputRanges.resize(InputRanges.size()); 4199d2c87699SAmir Ayupov llvm::transform(InputRanges, OutputRanges.begin(), 4200a34c753fSRafael Auler [](const DWARFAddressRange &Range) { 4201a34c753fSRafael Auler return DebugAddressRange(Range.LowPC, Range.HighPC); 4202a34c753fSRafael Auler }); 4203a34c753fSRafael Auler return OutputRanges; 4204a34c753fSRafael Auler } 4205a34c753fSRafael Auler 4206a34c753fSRafael Auler // Even though we will merge ranges in a post-processing pass, we attempt to 4207a34c753fSRafael Auler // merge them in a main processing loop as it improves the processing time. 4208a34c753fSRafael Auler uint64_t PrevEndAddress = 0; 4209a34c753fSRafael Auler for (const DWARFAddressRange &Range : InputRanges) { 4210a34c753fSRafael Auler if (!containsAddress(Range.LowPC)) { 4211a34c753fSRafael Auler LLVM_DEBUG( 4212a34c753fSRafael Auler dbgs() << "BOLT-DEBUG: invalid debug address range detected for " 4213a34c753fSRafael Auler << *this << " : [0x" << Twine::utohexstr(Range.LowPC) << ", 0x" 4214a34c753fSRafael Auler << Twine::utohexstr(Range.HighPC) << "]\n"); 4215a34c753fSRafael Auler PrevEndAddress = 0; 4216a34c753fSRafael Auler continue; 4217a34c753fSRafael Auler } 4218a34c753fSRafael Auler uint64_t InputOffset = Range.LowPC - getAddress(); 4219a34c753fSRafael Auler const uint64_t InputEndOffset = 4220a34c753fSRafael Auler std::min(Range.HighPC - getAddress(), getSize()); 4221a34c753fSRafael Auler 4222d2c87699SAmir Ayupov auto BBI = llvm::upper_bound(BasicBlockOffsets, 4223d2c87699SAmir Ayupov BasicBlockOffset(InputOffset, nullptr), 4224d2c87699SAmir Ayupov CompareBasicBlockOffsets()); 4225a34c753fSRafael Auler --BBI; 4226a34c753fSRafael Auler do { 4227a34c753fSRafael Auler const BinaryBasicBlock *BB = BBI->second; 4228a34c753fSRafael Auler if (InputOffset < BB->getOffset() || InputOffset >= BB->getEndOffset()) { 4229a34c753fSRafael Auler LLVM_DEBUG( 4230a34c753fSRafael Auler dbgs() << "BOLT-DEBUG: invalid debug address range detected for " 4231a34c753fSRafael Auler << *this << " : [0x" << Twine::utohexstr(Range.LowPC) 4232a34c753fSRafael Auler << ", 0x" << Twine::utohexstr(Range.HighPC) << "]\n"); 4233a34c753fSRafael Auler PrevEndAddress = 0; 4234a34c753fSRafael Auler break; 4235a34c753fSRafael Auler } 4236a34c753fSRafael Auler 4237a34c753fSRafael Auler // Skip the range if the block was deleted. 4238a34c753fSRafael Auler if (const uint64_t OutputStart = BB->getOutputAddressRange().first) { 4239a34c753fSRafael Auler const uint64_t StartAddress = 4240a34c753fSRafael Auler OutputStart + InputOffset - BB->getOffset(); 4241a34c753fSRafael Auler uint64_t EndAddress = BB->getOutputAddressRange().second; 4242a34c753fSRafael Auler if (InputEndOffset < BB->getEndOffset()) 4243a34c753fSRafael Auler EndAddress = StartAddress + InputEndOffset - InputOffset; 4244a34c753fSRafael Auler 4245a34c753fSRafael Auler if (StartAddress == PrevEndAddress) { 424640c2e0faSMaksim Panchenko OutputRanges.back().HighPC = 424740c2e0faSMaksim Panchenko std::max(OutputRanges.back().HighPC, EndAddress); 4248a34c753fSRafael Auler } else { 4249a34c753fSRafael Auler OutputRanges.emplace_back(StartAddress, 4250a34c753fSRafael Auler std::max(StartAddress, EndAddress)); 4251a34c753fSRafael Auler } 4252a34c753fSRafael Auler PrevEndAddress = OutputRanges.back().HighPC; 4253a34c753fSRafael Auler } 4254a34c753fSRafael Auler 4255a34c753fSRafael Auler InputOffset = BB->getEndOffset(); 4256a34c753fSRafael Auler ++BBI; 4257a34c753fSRafael Auler } while (InputOffset < InputEndOffset); 4258a34c753fSRafael Auler } 4259a34c753fSRafael Auler 4260a34c753fSRafael Auler // Post-processing pass to sort and merge ranges. 4261d2c87699SAmir Ayupov llvm::sort(OutputRanges); 4262a34c753fSRafael Auler DebugAddressRangesVector MergedRanges; 4263a34c753fSRafael Auler PrevEndAddress = 0; 4264a34c753fSRafael Auler for (const DebugAddressRange &Range : OutputRanges) { 4265a34c753fSRafael Auler if (Range.LowPC <= PrevEndAddress) { 426640c2e0faSMaksim Panchenko MergedRanges.back().HighPC = 426740c2e0faSMaksim Panchenko std::max(MergedRanges.back().HighPC, Range.HighPC); 4268a34c753fSRafael Auler } else { 4269a34c753fSRafael Auler MergedRanges.emplace_back(Range.LowPC, Range.HighPC); 4270a34c753fSRafael Auler } 4271a34c753fSRafael Auler PrevEndAddress = MergedRanges.back().HighPC; 4272a34c753fSRafael Auler } 4273a34c753fSRafael Auler 4274a34c753fSRafael Auler return MergedRanges; 4275a34c753fSRafael Auler } 4276a34c753fSRafael Auler 4277a34c753fSRafael Auler MCInst *BinaryFunction::getInstructionAtOffset(uint64_t Offset) { 4278a34c753fSRafael Auler if (CurrentState == State::Disassembled) { 4279a34c753fSRafael Auler auto II = Instructions.find(Offset); 4280a34c753fSRafael Auler return (II == Instructions.end()) ? nullptr : &II->second; 4281a34c753fSRafael Auler } else if (CurrentState == State::CFG) { 4282a34c753fSRafael Auler BinaryBasicBlock *BB = getBasicBlockContainingOffset(Offset); 4283a34c753fSRafael Auler if (!BB) 4284a34c753fSRafael Auler return nullptr; 4285a34c753fSRafael Auler 4286a34c753fSRafael Auler for (MCInst &Inst : *BB) { 4287a34c753fSRafael Auler constexpr uint32_t InvalidOffset = std::numeric_limits<uint32_t>::max(); 4288a9cd49d5SAmir Ayupov if (Offset == BC.MIB->getOffsetWithDefault(Inst, InvalidOffset)) 4289a34c753fSRafael Auler return &Inst; 4290a34c753fSRafael Auler } 4291a34c753fSRafael Auler 4292ccb99dd1SMaksim Panchenko if (MCInst *LastInstr = BB->getLastNonPseudoInstr()) { 4293ccb99dd1SMaksim Panchenko const uint32_t Size = 4294ccb99dd1SMaksim Panchenko BC.MIB->getAnnotationWithDefault<uint32_t>(*LastInstr, "Size"); 4295ccb99dd1SMaksim Panchenko if (BB->getEndOffset() - Offset == Size) 4296ccb99dd1SMaksim Panchenko return LastInstr; 4297ccb99dd1SMaksim Panchenko } 4298ccb99dd1SMaksim Panchenko 4299a34c753fSRafael Auler return nullptr; 4300a34c753fSRafael Auler } else { 4301a34c753fSRafael Auler llvm_unreachable("invalid CFG state to use getInstructionAtOffset()"); 4302a34c753fSRafael Auler } 4303a34c753fSRafael Auler } 4304a34c753fSRafael Auler 4305a34c753fSRafael Auler DebugLocationsVector BinaryFunction::translateInputToOutputLocationList( 4306a34c753fSRafael Auler const DebugLocationsVector &InputLL) const { 4307a34c753fSRafael Auler DebugLocationsVector OutputLL; 4308a34c753fSRafael Auler 43093652483cSRafael Auler if (isFolded()) 4310a34c753fSRafael Auler return OutputLL; 4311a34c753fSRafael Auler 4312a34c753fSRafael Auler // If the function hasn't changed - there's nothing to update. 43133652483cSRafael Auler if (!isEmitted()) 4314a34c753fSRafael Auler return InputLL; 4315a34c753fSRafael Auler 4316a34c753fSRafael Auler uint64_t PrevEndAddress = 0; 4317a34c753fSRafael Auler SmallVectorImpl<uint8_t> *PrevExpr = nullptr; 4318a34c753fSRafael Auler for (const DebugLocationEntry &Entry : InputLL) { 4319a34c753fSRafael Auler const uint64_t Start = Entry.LowPC; 4320a34c753fSRafael Auler const uint64_t End = Entry.HighPC; 4321a34c753fSRafael Auler if (!containsAddress(Start)) { 4322a34c753fSRafael Auler LLVM_DEBUG(dbgs() << "BOLT-DEBUG: invalid debug address range detected " 4323a34c753fSRafael Auler "for " 4324a34c753fSRafael Auler << *this << " : [0x" << Twine::utohexstr(Start) 4325a34c753fSRafael Auler << ", 0x" << Twine::utohexstr(End) << "]\n"); 4326a34c753fSRafael Auler continue; 4327a34c753fSRafael Auler } 4328a34c753fSRafael Auler uint64_t InputOffset = Start - getAddress(); 4329a34c753fSRafael Auler const uint64_t InputEndOffset = std::min(End - getAddress(), getSize()); 4330d2c87699SAmir Ayupov auto BBI = llvm::upper_bound(BasicBlockOffsets, 4331d2c87699SAmir Ayupov BasicBlockOffset(InputOffset, nullptr), 4332d2c87699SAmir Ayupov CompareBasicBlockOffsets()); 4333a34c753fSRafael Auler --BBI; 4334a34c753fSRafael Auler do { 4335a34c753fSRafael Auler const BinaryBasicBlock *BB = BBI->second; 4336a34c753fSRafael Auler if (InputOffset < BB->getOffset() || InputOffset >= BB->getEndOffset()) { 4337a34c753fSRafael Auler LLVM_DEBUG(dbgs() << "BOLT-DEBUG: invalid debug address range detected " 4338a34c753fSRafael Auler "for " 4339a34c753fSRafael Auler << *this << " : [0x" << Twine::utohexstr(Start) 4340a34c753fSRafael Auler << ", 0x" << Twine::utohexstr(End) << "]\n"); 4341a34c753fSRafael Auler PrevEndAddress = 0; 4342a34c753fSRafael Auler break; 4343a34c753fSRafael Auler } 4344a34c753fSRafael Auler 4345a34c753fSRafael Auler // Skip the range if the block was deleted. 4346a34c753fSRafael Auler if (const uint64_t OutputStart = BB->getOutputAddressRange().first) { 4347a34c753fSRafael Auler const uint64_t StartAddress = 4348a34c753fSRafael Auler OutputStart + InputOffset - BB->getOffset(); 4349a34c753fSRafael Auler uint64_t EndAddress = BB->getOutputAddressRange().second; 4350a34c753fSRafael Auler if (InputEndOffset < BB->getEndOffset()) 4351a34c753fSRafael Auler EndAddress = StartAddress + InputEndOffset - InputOffset; 4352a34c753fSRafael Auler 4353a34c753fSRafael Auler if (StartAddress == PrevEndAddress && Entry.Expr == *PrevExpr) { 4354a34c753fSRafael Auler OutputLL.back().HighPC = std::max(OutputLL.back().HighPC, EndAddress); 4355a34c753fSRafael Auler } else { 435640c2e0faSMaksim Panchenko OutputLL.emplace_back(DebugLocationEntry{ 435740c2e0faSMaksim Panchenko StartAddress, std::max(StartAddress, EndAddress), Entry.Expr}); 4358a34c753fSRafael Auler } 4359a34c753fSRafael Auler PrevEndAddress = OutputLL.back().HighPC; 4360a34c753fSRafael Auler PrevExpr = &OutputLL.back().Expr; 4361a34c753fSRafael Auler } 4362a34c753fSRafael Auler 4363a34c753fSRafael Auler ++BBI; 4364a34c753fSRafael Auler InputOffset = BB->getEndOffset(); 4365a34c753fSRafael Auler } while (InputOffset < InputEndOffset); 4366a34c753fSRafael Auler } 4367a34c753fSRafael Auler 4368a34c753fSRafael Auler // Sort and merge adjacent entries with identical location. 4369d2c87699SAmir Ayupov llvm::stable_sort( 4370d2c87699SAmir Ayupov OutputLL, [](const DebugLocationEntry &A, const DebugLocationEntry &B) { 4371a34c753fSRafael Auler return A.LowPC < B.LowPC; 4372a34c753fSRafael Auler }); 4373a34c753fSRafael Auler DebugLocationsVector MergedLL; 4374a34c753fSRafael Auler PrevEndAddress = 0; 4375a34c753fSRafael Auler PrevExpr = nullptr; 4376a34c753fSRafael Auler for (const DebugLocationEntry &Entry : OutputLL) { 4377a34c753fSRafael Auler if (Entry.LowPC <= PrevEndAddress && *PrevExpr == Entry.Expr) { 4378a34c753fSRafael Auler MergedLL.back().HighPC = std::max(Entry.HighPC, MergedLL.back().HighPC); 4379a34c753fSRafael Auler } else { 4380a34c753fSRafael Auler const uint64_t Begin = std::max(Entry.LowPC, PrevEndAddress); 4381a34c753fSRafael Auler const uint64_t End = std::max(Begin, Entry.HighPC); 4382a34c753fSRafael Auler MergedLL.emplace_back(DebugLocationEntry{Begin, End, Entry.Expr}); 4383a34c753fSRafael Auler } 4384a34c753fSRafael Auler PrevEndAddress = MergedLL.back().HighPC; 4385a34c753fSRafael Auler PrevExpr = &MergedLL.back().Expr; 4386a34c753fSRafael Auler } 4387a34c753fSRafael Auler 4388a34c753fSRafael Auler return MergedLL; 4389a34c753fSRafael Auler } 4390a34c753fSRafael Auler 4391a34c753fSRafael Auler void BinaryFunction::printLoopInfo(raw_ostream &OS) const { 4392798e92c6SAmir Ayupov if (!opts::shouldPrint(*this)) 4393798e92c6SAmir Ayupov return; 4394798e92c6SAmir Ayupov 4395a34c753fSRafael Auler OS << "Loop Info for Function \"" << *this << "\""; 43963652483cSRafael Auler if (hasValidProfile()) 4397a34c753fSRafael Auler OS << " (count: " << getExecutionCount() << ")"; 4398a34c753fSRafael Auler OS << "\n"; 4399a34c753fSRafael Auler 4400a34c753fSRafael Auler std::stack<BinaryLoop *> St; 44015d8247d4SAmir Ayupov for_each(*BLI, [&](BinaryLoop *L) { St.push(L); }); 4402a34c753fSRafael Auler while (!St.empty()) { 4403a34c753fSRafael Auler BinaryLoop *L = St.top(); 4404a34c753fSRafael Auler St.pop(); 4405a34c753fSRafael Auler 44065d8247d4SAmir Ayupov for_each(*L, [&](BinaryLoop *Inner) { St.push(Inner); }); 4407a34c753fSRafael Auler 4408a34c753fSRafael Auler if (!hasValidProfile()) 4409a34c753fSRafael Auler continue; 4410a34c753fSRafael Auler 441140c2e0faSMaksim Panchenko OS << (L->getLoopDepth() > 1 ? "Nested" : "Outer") 441240c2e0faSMaksim Panchenko << " loop header: " << L->getHeader()->getName(); 4413a34c753fSRafael Auler OS << "\n"; 4414a34c753fSRafael Auler OS << "Loop basic blocks: "; 4415f7581a39SAmir Ayupov ListSeparator LS; 4416f7581a39SAmir Ayupov for (BinaryBasicBlock *BB : L->blocks()) 4417f7581a39SAmir Ayupov OS << LS << BB->getName(); 4418a34c753fSRafael Auler OS << "\n"; 4419a34c753fSRafael Auler if (hasValidProfile()) { 4420a34c753fSRafael Auler OS << "Total back edge count: " << L->TotalBackEdgeCount << "\n"; 4421a34c753fSRafael Auler OS << "Loop entry count: " << L->EntryCount << "\n"; 4422a34c753fSRafael Auler OS << "Loop exit count: " << L->ExitCount << "\n"; 4423a34c753fSRafael Auler if (L->EntryCount > 0) { 4424a34c753fSRafael Auler OS << "Average iters per entry: " 4425a34c753fSRafael Auler << format("%.4lf", (double)L->TotalBackEdgeCount / L->EntryCount) 4426a34c753fSRafael Auler << "\n"; 4427a34c753fSRafael Auler } 4428a34c753fSRafael Auler } 4429a34c753fSRafael Auler OS << "----\n"; 4430a34c753fSRafael Auler } 4431a34c753fSRafael Auler 4432a34c753fSRafael Auler OS << "Total number of loops: " << BLI->TotalLoops << "\n"; 4433a34c753fSRafael Auler OS << "Number of outer loops: " << BLI->OuterLoops << "\n"; 4434a34c753fSRafael Auler OS << "Maximum nested loop depth: " << BLI->MaximumDepth << "\n\n"; 4435a34c753fSRafael Auler } 4436a34c753fSRafael Auler 4437a34c753fSRafael Auler bool BinaryFunction::isAArch64Veneer() const { 443835efe1d8SVladislav Khmelevsky if (empty()) 4439a34c753fSRafael Auler return false; 4440a34c753fSRafael Auler 4441a34c753fSRafael Auler BinaryBasicBlock &BB = **BasicBlocks.begin(); 44423652483cSRafael Auler for (MCInst &Inst : BB) 4443a34c753fSRafael Auler if (!BC.MIB->hasAnnotation(Inst, "AArch64Veneer")) 4444a34c753fSRafael Auler return false; 4445a34c753fSRafael Auler 444635efe1d8SVladislav Khmelevsky for (auto I = BasicBlocks.begin() + 1, E = BasicBlocks.end(); I != E; ++I) { 444735efe1d8SVladislav Khmelevsky for (MCInst &Inst : **I) 444835efe1d8SVladislav Khmelevsky if (!BC.MIB->isNoop(Inst)) 444935efe1d8SVladislav Khmelevsky return false; 445035efe1d8SVladislav Khmelevsky } 445135efe1d8SVladislav Khmelevsky 4452a34c753fSRafael Auler return true; 4453a34c753fSRafael Auler } 4454a34c753fSRafael Auler 4455a34c753fSRafael Auler } // namespace bolt 4456a34c753fSRafael Auler } // namespace llvm 4457