xref: /llvm-project/bolt/lib/Core/BinaryFunction.cpp (revision 05523dc32d8ca81d9a92ff955194a9e80cf79dc0)
12f09f445SMaksim Panchenko //===- bolt/Core/BinaryFunction.cpp - Low-level function ------------------===//
2a34c753fSRafael Auler //
3a34c753fSRafael Auler // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4a34c753fSRafael Auler // See https://llvm.org/LICENSE.txt for license information.
5a34c753fSRafael Auler // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6a34c753fSRafael Auler //
7a34c753fSRafael Auler //===----------------------------------------------------------------------===//
8a34c753fSRafael Auler //
92f09f445SMaksim Panchenko // This file implements the BinaryFunction class.
102f09f445SMaksim Panchenko //
11a34c753fSRafael Auler //===----------------------------------------------------------------------===//
12a34c753fSRafael Auler 
13a34c753fSRafael Auler #include "bolt/Core/BinaryFunction.h"
14a34c753fSRafael Auler #include "bolt/Core/BinaryBasicBlock.h"
15a7b69dbdSAmir Ayupov #include "bolt/Core/BinaryDomTree.h"
16a34c753fSRafael Auler #include "bolt/Core/DynoStats.h"
17a34c753fSRafael Auler #include "bolt/Core/MCPlusBuilder.h"
18a34c753fSRafael Auler #include "bolt/Utils/NameResolver.h"
19a34c753fSRafael Auler #include "bolt/Utils/NameShortener.h"
20a34c753fSRafael Auler #include "bolt/Utils/Utils.h"
215d8247d4SAmir Ayupov #include "llvm/ADT/STLExtras.h"
22a34c753fSRafael Auler #include "llvm/ADT/SmallSet.h"
23f7581a39SAmir Ayupov #include "llvm/ADT/StringExtras.h"
24a34c753fSRafael Auler #include "llvm/ADT/StringRef.h"
25a34c753fSRafael Auler #include "llvm/ADT/edit_distance.h"
26ae585be1SRafael Auler #include "llvm/Demangle/Demangle.h"
27a34c753fSRafael Auler #include "llvm/MC/MCAsmInfo.h"
28a34c753fSRafael Auler #include "llvm/MC/MCAsmLayout.h"
29a34c753fSRafael Auler #include "llvm/MC/MCContext.h"
30a34c753fSRafael Auler #include "llvm/MC/MCDisassembler/MCDisassembler.h"
31a34c753fSRafael Auler #include "llvm/MC/MCExpr.h"
32a34c753fSRafael Auler #include "llvm/MC/MCInst.h"
33a34c753fSRafael Auler #include "llvm/MC/MCInstPrinter.h"
3457f7c7d9Sserge-sans-paille #include "llvm/MC/MCRegisterInfo.h"
35a34c753fSRafael Auler #include "llvm/Object/ObjectFile.h"
36a34c753fSRafael Auler #include "llvm/Support/CommandLine.h"
37a34c753fSRafael Auler #include "llvm/Support/Debug.h"
38a34c753fSRafael Auler #include "llvm/Support/GraphWriter.h"
39a34c753fSRafael Auler #include "llvm/Support/LEB128.h"
40a34c753fSRafael Auler #include "llvm/Support/Regex.h"
41a34c753fSRafael Auler #include "llvm/Support/Timer.h"
42a34c753fSRafael Auler #include "llvm/Support/raw_ostream.h"
43a34c753fSRafael Auler #include <functional>
44a34c753fSRafael Auler #include <limits>
45a34c753fSRafael Auler #include <numeric>
46a34c753fSRafael Auler #include <string>
47a34c753fSRafael Auler 
48a34c753fSRafael Auler #define DEBUG_TYPE "bolt"
49a34c753fSRafael Auler 
50a34c753fSRafael Auler using namespace llvm;
51a34c753fSRafael Auler using namespace bolt;
52a34c753fSRafael Auler 
53a34c753fSRafael Auler namespace opts {
54a34c753fSRafael Auler 
55a34c753fSRafael Auler extern cl::OptionCategory BoltCategory;
56a34c753fSRafael Auler extern cl::OptionCategory BoltOptCategory;
57a34c753fSRafael Auler extern cl::OptionCategory BoltRelocCategory;
58a34c753fSRafael Auler 
59a34c753fSRafael Auler extern cl::opt<bool> EnableBAT;
60a34c753fSRafael Auler extern cl::opt<bool> Instrument;
61a34c753fSRafael Auler extern cl::opt<bool> StrictMode;
62a34c753fSRafael Auler extern cl::opt<bool> UpdateDebugSections;
63a34c753fSRafael Auler extern cl::opt<unsigned> Verbosity;
64a34c753fSRafael Auler 
65a34c753fSRafael Auler extern bool processAllFunctions();
66a34c753fSRafael Auler 
67b92436efSFangrui Song cl::opt<bool> CheckEncoding(
68b92436efSFangrui Song     "check-encoding",
69a34c753fSRafael Auler     cl::desc("perform verification of LLVM instruction encoding/decoding. "
70a34c753fSRafael Auler              "Every instruction in the input is decoded and re-encoded. "
71a34c753fSRafael Auler              "If the resulting bytes do not match the input, a warning message "
72a34c753fSRafael Auler              "is printed."),
73b92436efSFangrui Song     cl::Hidden, cl::cat(BoltCategory));
74a34c753fSRafael Auler 
75b92436efSFangrui Song static cl::opt<bool> DotToolTipCode(
76b92436efSFangrui Song     "dot-tooltip-code",
77b92436efSFangrui Song     cl::desc("add basic block instructions as tool tips on nodes"), cl::Hidden,
78a34c753fSRafael Auler     cl::cat(BoltCategory));
79a34c753fSRafael Auler 
80a34c753fSRafael Auler cl::opt<JumpTableSupportLevel>
81a34c753fSRafael Auler JumpTables("jump-tables",
82a34c753fSRafael Auler   cl::desc("jump tables support (default=basic)"),
83a34c753fSRafael Auler   cl::init(JTS_BASIC),
84a34c753fSRafael Auler   cl::values(
85a34c753fSRafael Auler       clEnumValN(JTS_NONE, "none",
86a34c753fSRafael Auler                  "do not optimize functions with jump tables"),
87a34c753fSRafael Auler       clEnumValN(JTS_BASIC, "basic",
88a34c753fSRafael Auler                  "optimize functions with jump tables"),
89a34c753fSRafael Auler       clEnumValN(JTS_MOVE, "move",
90a34c753fSRafael Auler                  "move jump tables to a separate section"),
91a34c753fSRafael Auler       clEnumValN(JTS_SPLIT, "split",
92a34c753fSRafael Auler                  "split jump tables section into hot and cold based on "
93a34c753fSRafael Auler                  "function execution frequency"),
94a34c753fSRafael Auler       clEnumValN(JTS_AGGRESSIVE, "aggressive",
95a34c753fSRafael Auler                  "aggressively split jump tables section based on usage "
96a34c753fSRafael Auler                  "of the tables")),
97a34c753fSRafael Auler   cl::ZeroOrMore,
98a34c753fSRafael Auler   cl::cat(BoltOptCategory));
99a34c753fSRafael Auler 
100b92436efSFangrui Song static cl::opt<bool> NoScan(
101b92436efSFangrui Song     "no-scan",
102b92436efSFangrui Song     cl::desc(
103b92436efSFangrui Song         "do not scan cold functions for external references (may result in "
104a34c753fSRafael Auler         "slower binary)"),
105b92436efSFangrui Song     cl::Hidden, cl::cat(BoltOptCategory));
106a34c753fSRafael Auler 
107a34c753fSRafael Auler cl::opt<bool>
108a34c753fSRafael Auler     PreserveBlocksAlignment("preserve-blocks-alignment",
109a34c753fSRafael Auler                             cl::desc("try to preserve basic block alignment"),
110a34c753fSRafael Auler                             cl::cat(BoltOptCategory));
111a34c753fSRafael Auler 
112a34c753fSRafael Auler cl::opt<bool>
113a34c753fSRafael Auler PrintDynoStats("dyno-stats",
114a34c753fSRafael Auler   cl::desc("print execution info based on profile"),
115a34c753fSRafael Auler   cl::cat(BoltCategory));
116a34c753fSRafael Auler 
117a34c753fSRafael Auler static cl::opt<bool>
118a34c753fSRafael Auler PrintDynoStatsOnly("print-dyno-stats-only",
119a34c753fSRafael Auler   cl::desc("while printing functions output dyno-stats and skip instructions"),
120a34c753fSRafael Auler   cl::init(false),
121a34c753fSRafael Auler   cl::Hidden,
122a34c753fSRafael Auler   cl::cat(BoltCategory));
123a34c753fSRafael Auler 
124a34c753fSRafael Auler static cl::list<std::string>
125a34c753fSRafael Auler PrintOnly("print-only",
126a34c753fSRafael Auler   cl::CommaSeparated,
127a34c753fSRafael Auler   cl::desc("list of functions to print"),
128a34c753fSRafael Auler   cl::value_desc("func1,func2,func3,..."),
129a34c753fSRafael Auler   cl::Hidden,
130a34c753fSRafael Auler   cl::cat(BoltCategory));
131a34c753fSRafael Auler 
132a34c753fSRafael Auler cl::opt<bool>
133a34c753fSRafael Auler     TimeBuild("time-build",
134a34c753fSRafael Auler               cl::desc("print time spent constructing binary functions"),
135b92436efSFangrui Song               cl::Hidden, cl::cat(BoltCategory));
136a34c753fSRafael Auler 
137a34c753fSRafael Auler cl::opt<bool>
138a34c753fSRafael Auler TrapOnAVX512("trap-avx512",
139a34c753fSRafael Auler   cl::desc("in relocation mode trap upon entry to any function that uses "
140a34c753fSRafael Auler             "AVX-512 instructions"),
141a34c753fSRafael Auler   cl::init(false),
142a34c753fSRafael Auler   cl::ZeroOrMore,
143a34c753fSRafael Auler   cl::Hidden,
144a34c753fSRafael Auler   cl::cat(BoltCategory));
145a34c753fSRafael Auler 
146a34c753fSRafael Auler bool shouldPrint(const BinaryFunction &Function) {
147a34c753fSRafael Auler   if (Function.isIgnored())
148a34c753fSRafael Auler     return false;
149a34c753fSRafael Auler 
150a34c753fSRafael Auler   if (PrintOnly.empty())
151a34c753fSRafael Auler     return true;
152a34c753fSRafael Auler 
153a34c753fSRafael Auler   for (std::string &Name : opts::PrintOnly) {
154a34c753fSRafael Auler     if (Function.hasNameRegex(Name)) {
155a34c753fSRafael Auler       return true;
156a34c753fSRafael Auler     }
157a34c753fSRafael Auler   }
158a34c753fSRafael Auler 
159a34c753fSRafael Auler   return false;
160a34c753fSRafael Auler }
161a34c753fSRafael Auler 
162a34c753fSRafael Auler } // namespace opts
163a34c753fSRafael Auler 
164a34c753fSRafael Auler namespace llvm {
165a34c753fSRafael Auler namespace bolt {
166a34c753fSRafael Auler 
167a34c753fSRafael Auler constexpr unsigned BinaryFunction::MinAlign;
168a34c753fSRafael Auler 
169a34c753fSRafael Auler namespace {
170a34c753fSRafael Auler 
17140c2e0faSMaksim Panchenko template <typename R> bool emptyRange(const R &Range) {
172a34c753fSRafael Auler   return Range.begin() == Range.end();
173a34c753fSRafael Auler }
174a34c753fSRafael Auler 
175a34c753fSRafael Auler /// Gets debug line information for the instruction located at the given
176a34c753fSRafael Auler /// address in the original binary. The SMLoc's pointer is used
177a34c753fSRafael Auler /// to point to this information, which is represented by a
178a34c753fSRafael Auler /// DebugLineTableRowRef. The returned pointer is null if no debug line
179a34c753fSRafael Auler /// information for this instruction was found.
18040c2e0faSMaksim Panchenko SMLoc findDebugLineInformationForInstructionAt(
18140c2e0faSMaksim Panchenko     uint64_t Address, DWARFUnit *Unit,
18240c2e0faSMaksim Panchenko     const DWARFDebugLine::LineTable *LineTable) {
183a34c753fSRafael Auler   // We use the pointer in SMLoc to store an instance of DebugLineTableRowRef,
184a34c753fSRafael Auler   // which occupies 64 bits. Thus, we can only proceed if the struct fits into
185a34c753fSRafael Auler   // the pointer itself.
18640c2e0faSMaksim Panchenko   assert(sizeof(decltype(SMLoc().getPointer())) >=
18740c2e0faSMaksim Panchenko              sizeof(DebugLineTableRowRef) &&
188a34c753fSRafael Auler          "Cannot fit instruction debug line information into SMLoc's pointer");
189a34c753fSRafael Auler 
190a34c753fSRafael Auler   SMLoc NullResult = DebugLineTableRowRef::NULL_ROW.toSMLoc();
191a34c753fSRafael Auler   uint32_t RowIndex = LineTable->lookupAddress(
192a34c753fSRafael Auler       {Address, object::SectionedAddress::UndefSection});
193a34c753fSRafael Auler   if (RowIndex == LineTable->UnknownRowIndex)
194a34c753fSRafael Auler     return NullResult;
195a34c753fSRafael Auler 
196a34c753fSRafael Auler   assert(RowIndex < LineTable->Rows.size() &&
197a34c753fSRafael Auler          "Line Table lookup returned invalid index.");
198a34c753fSRafael Auler 
199a34c753fSRafael Auler   decltype(SMLoc().getPointer()) Ptr;
200a34c753fSRafael Auler   DebugLineTableRowRef *InstructionLocation =
201a34c753fSRafael Auler       reinterpret_cast<DebugLineTableRowRef *>(&Ptr);
202a34c753fSRafael Auler 
203a34c753fSRafael Auler   InstructionLocation->DwCompileUnitIndex = Unit->getOffset();
204a34c753fSRafael Auler   InstructionLocation->RowIndex = RowIndex + 1;
205a34c753fSRafael Auler 
206a34c753fSRafael Auler   return SMLoc::getFromPointer(Ptr);
207a34c753fSRafael Auler }
208a34c753fSRafael Auler 
209a34c753fSRafael Auler std::string buildSectionName(StringRef Prefix, StringRef Name,
210a34c753fSRafael Auler                              const BinaryContext &BC) {
211a34c753fSRafael Auler   if (BC.isELF())
212a34c753fSRafael Auler     return (Prefix + Name).str();
213a34c753fSRafael Auler   static NameShortener NS;
214a34c753fSRafael Auler   return (Prefix + Twine(NS.getID(Name))).str();
215a34c753fSRafael Auler }
216a34c753fSRafael Auler 
21740c2e0faSMaksim Panchenko raw_ostream &operator<<(raw_ostream &OS, const BinaryFunction::State State) {
21840c2e0faSMaksim Panchenko   switch (State) {
21940c2e0faSMaksim Panchenko   case BinaryFunction::State::Empty:         OS << "empty"; break;
22040c2e0faSMaksim Panchenko   case BinaryFunction::State::Disassembled:  OS << "disassembled"; break;
22140c2e0faSMaksim Panchenko   case BinaryFunction::State::CFG:           OS << "CFG constructed"; break;
22240c2e0faSMaksim Panchenko   case BinaryFunction::State::CFG_Finalized: OS << "CFG finalized"; break;
22340c2e0faSMaksim Panchenko   case BinaryFunction::State::EmittedCFG:    OS << "emitted with CFG"; break;
22440c2e0faSMaksim Panchenko   case BinaryFunction::State::Emitted:       OS << "emitted"; break;
22540c2e0faSMaksim Panchenko   }
22640c2e0faSMaksim Panchenko 
22740c2e0faSMaksim Panchenko   return OS;
22840c2e0faSMaksim Panchenko }
22940c2e0faSMaksim Panchenko 
230a34c753fSRafael Auler } // namespace
231a34c753fSRafael Auler 
232a34c753fSRafael Auler std::string BinaryFunction::buildCodeSectionName(StringRef Name,
233a34c753fSRafael Auler                                                  const BinaryContext &BC) {
234a34c753fSRafael Auler   return buildSectionName(BC.isELF() ? ".local.text." : ".l.text.", Name, BC);
235a34c753fSRafael Auler }
236a34c753fSRafael Auler 
237a34c753fSRafael Auler std::string BinaryFunction::buildColdCodeSectionName(StringRef Name,
238a34c753fSRafael Auler                                                      const BinaryContext &BC) {
239a34c753fSRafael Auler   return buildSectionName(BC.isELF() ? ".local.cold.text." : ".l.c.text.", Name,
240a34c753fSRafael Auler                           BC);
241a34c753fSRafael Auler }
242a34c753fSRafael Auler 
243a34c753fSRafael Auler uint64_t BinaryFunction::Count = 0;
244a34c753fSRafael Auler 
24540c2e0faSMaksim Panchenko Optional<StringRef> BinaryFunction::hasNameRegex(const StringRef Name) const {
246a34c753fSRafael Auler   const std::string RegexName = (Twine("^") + StringRef(Name) + "$").str();
247a34c753fSRafael Auler   Regex MatchName(RegexName);
248a34c753fSRafael Auler   Optional<StringRef> Match = forEachName(
249a34c753fSRafael Auler       [&MatchName](StringRef Name) { return MatchName.match(Name); });
250a34c753fSRafael Auler 
251a34c753fSRafael Auler   return Match;
252a34c753fSRafael Auler }
253a34c753fSRafael Auler 
254a34c753fSRafael Auler Optional<StringRef>
255a34c753fSRafael Auler BinaryFunction::hasRestoredNameRegex(const StringRef Name) const {
256a34c753fSRafael Auler   const std::string RegexName = (Twine("^") + StringRef(Name) + "$").str();
257a34c753fSRafael Auler   Regex MatchName(RegexName);
258a34c753fSRafael Auler   Optional<StringRef> Match = forEachName([&MatchName](StringRef Name) {
259a34c753fSRafael Auler     return MatchName.match(NameResolver::restore(Name));
260a34c753fSRafael Auler   });
261a34c753fSRafael Auler 
262a34c753fSRafael Auler   return Match;
263a34c753fSRafael Auler }
264a34c753fSRafael Auler 
265a34c753fSRafael Auler std::string BinaryFunction::getDemangledName() const {
266a34c753fSRafael Auler   StringRef MangledName = NameResolver::restore(getOneName());
267ae585be1SRafael Auler   return demangle(MangledName.str());
268a34c753fSRafael Auler }
269a34c753fSRafael Auler 
270a34c753fSRafael Auler BinaryBasicBlock *
271a34c753fSRafael Auler BinaryFunction::getBasicBlockContainingOffset(uint64_t Offset) {
272a34c753fSRafael Auler   if (Offset > Size)
273a34c753fSRafael Auler     return nullptr;
274a34c753fSRafael Auler 
275a34c753fSRafael Auler   if (BasicBlockOffsets.empty())
276a34c753fSRafael Auler     return nullptr;
277a34c753fSRafael Auler 
278a34c753fSRafael Auler   /*
279a34c753fSRafael Auler    * This is commented out because it makes BOLT too slow.
280a34c753fSRafael Auler    * assert(std::is_sorted(BasicBlockOffsets.begin(),
281a34c753fSRafael Auler    *                       BasicBlockOffsets.end(),
282a34c753fSRafael Auler    *                       CompareBasicBlockOffsets())));
283a34c753fSRafael Auler    */
284d2c87699SAmir Ayupov   auto I =
285d2c87699SAmir Ayupov       llvm::upper_bound(BasicBlockOffsets, BasicBlockOffset(Offset, nullptr),
286a34c753fSRafael Auler                         CompareBasicBlockOffsets());
287a34c753fSRafael Auler   assert(I != BasicBlockOffsets.begin() && "first basic block not at offset 0");
288a34c753fSRafael Auler   --I;
289a34c753fSRafael Auler   BinaryBasicBlock *BB = I->second;
290a34c753fSRafael Auler   return (Offset < BB->getOffset() + BB->getOriginalSize()) ? BB : nullptr;
291a34c753fSRafael Auler }
292a34c753fSRafael Auler 
293a34c753fSRafael Auler void BinaryFunction::markUnreachableBlocks() {
294a34c753fSRafael Auler   std::stack<BinaryBasicBlock *> Stack;
295a34c753fSRafael Auler 
2963652483cSRafael Auler   for (BinaryBasicBlock *BB : layout())
297a34c753fSRafael Auler     BB->markValid(false);
298a34c753fSRafael Auler 
299a34c753fSRafael Auler   // Add all entries and landing pads as roots.
300a34c753fSRafael Auler   for (BinaryBasicBlock *BB : BasicBlocks) {
301a34c753fSRafael Auler     if (isEntryPoint(*BB) || BB->isLandingPad()) {
302a34c753fSRafael Auler       Stack.push(BB);
303a34c753fSRafael Auler       BB->markValid(true);
304a34c753fSRafael Auler       continue;
305a34c753fSRafael Auler     }
306a34c753fSRafael Auler     // FIXME:
307a34c753fSRafael Auler     // Also mark BBs with indirect jumps as reachable, since we do not
308933df2a4SMaksim Panchenko     // support removing unused jump tables yet (GH-issue20).
309a34c753fSRafael Auler     for (const MCInst &Inst : *BB) {
310a34c753fSRafael Auler       if (BC.MIB->getJumpTable(Inst)) {
311a34c753fSRafael Auler         Stack.push(BB);
312a34c753fSRafael Auler         BB->markValid(true);
313a34c753fSRafael Auler         break;
314a34c753fSRafael Auler       }
315a34c753fSRafael Auler     }
316a34c753fSRafael Auler   }
317a34c753fSRafael Auler 
318a34c753fSRafael Auler   // Determine reachable BBs from the entry point
319a34c753fSRafael Auler   while (!Stack.empty()) {
320a34c753fSRafael Auler     BinaryBasicBlock *BB = Stack.top();
321a34c753fSRafael Auler     Stack.pop();
322a34c753fSRafael Auler     for (BinaryBasicBlock *Succ : BB->successors()) {
323a34c753fSRafael Auler       if (Succ->isValid())
324a34c753fSRafael Auler         continue;
325a34c753fSRafael Auler       Succ->markValid(true);
326a34c753fSRafael Auler       Stack.push(Succ);
327a34c753fSRafael Auler     }
328a34c753fSRafael Auler   }
329a34c753fSRafael Auler }
330a34c753fSRafael Auler 
331a34c753fSRafael Auler // Any unnecessary fallthrough jumps revealed after calling eraseInvalidBBs
332a34c753fSRafael Auler // will be cleaned up by fixBranches().
333a34c753fSRafael Auler std::pair<unsigned, uint64_t> BinaryFunction::eraseInvalidBBs() {
334a34c753fSRafael Auler   BasicBlockOrderType NewLayout;
335a34c753fSRafael Auler   unsigned Count = 0;
336a34c753fSRafael Auler   uint64_t Bytes = 0;
337a34c753fSRafael Auler   for (BinaryBasicBlock *BB : layout()) {
338a34c753fSRafael Auler     if (BB->isValid()) {
339a34c753fSRafael Auler       NewLayout.push_back(BB);
340a34c753fSRafael Auler     } else {
341a34c753fSRafael Auler       assert(!isEntryPoint(*BB) && "all entry blocks must be valid");
342a34c753fSRafael Auler       ++Count;
343a34c753fSRafael Auler       Bytes += BC.computeCodeSize(BB->begin(), BB->end());
344a34c753fSRafael Auler     }
345a34c753fSRafael Auler   }
346a34c753fSRafael Auler   BasicBlocksLayout = std::move(NewLayout);
347a34c753fSRafael Auler 
348a34c753fSRafael Auler   BasicBlockListType NewBasicBlocks;
349a34c753fSRafael Auler   for (auto I = BasicBlocks.begin(), E = BasicBlocks.end(); I != E; ++I) {
350a34c753fSRafael Auler     BinaryBasicBlock *BB = *I;
351a34c753fSRafael Auler     if (BB->isValid()) {
352a34c753fSRafael Auler       NewBasicBlocks.push_back(BB);
353a34c753fSRafael Auler     } else {
354a34c753fSRafael Auler       // Make sure the block is removed from the list of predecessors.
355a34c753fSRafael Auler       BB->removeAllSuccessors();
356a34c753fSRafael Auler       DeletedBasicBlocks.push_back(BB);
357a34c753fSRafael Auler     }
358a34c753fSRafael Auler   }
359a34c753fSRafael Auler   BasicBlocks = std::move(NewBasicBlocks);
360a34c753fSRafael Auler 
361a34c753fSRafael Auler   assert(BasicBlocks.size() == BasicBlocksLayout.size());
362a34c753fSRafael Auler 
363a34c753fSRafael Auler   // Update CFG state if needed
364a34c753fSRafael Auler   if (Count > 0)
365a34c753fSRafael Auler     recomputeLandingPads();
366a34c753fSRafael Auler 
367a34c753fSRafael Auler   return std::make_pair(Count, Bytes);
368a34c753fSRafael Auler }
369a34c753fSRafael Auler 
370a34c753fSRafael Auler bool BinaryFunction::isForwardCall(const MCSymbol *CalleeSymbol) const {
371a34c753fSRafael Auler   // This function should work properly before and after function reordering.
372a34c753fSRafael Auler   // In order to accomplish this, we use the function index (if it is valid).
373a34c753fSRafael Auler   // If the function indices are not valid, we fall back to the original
374a34c753fSRafael Auler   // addresses.  This should be ok because the functions without valid indices
375a34c753fSRafael Auler   // should have been ordered with a stable sort.
376a34c753fSRafael Auler   const BinaryFunction *CalleeBF = BC.getFunctionForSymbol(CalleeSymbol);
377a34c753fSRafael Auler   if (CalleeBF) {
378a34c753fSRafael Auler     if (CalleeBF->isInjected())
379a34c753fSRafael Auler       return true;
380a34c753fSRafael Auler 
381a34c753fSRafael Auler     if (hasValidIndex() && CalleeBF->hasValidIndex()) {
382a34c753fSRafael Auler       return getIndex() < CalleeBF->getIndex();
383a34c753fSRafael Auler     } else if (hasValidIndex() && !CalleeBF->hasValidIndex()) {
384a34c753fSRafael Auler       return true;
385a34c753fSRafael Auler     } else if (!hasValidIndex() && CalleeBF->hasValidIndex()) {
386a34c753fSRafael Auler       return false;
387a34c753fSRafael Auler     } else {
388a34c753fSRafael Auler       return getAddress() < CalleeBF->getAddress();
389a34c753fSRafael Auler     }
390a34c753fSRafael Auler   } else {
391a34c753fSRafael Auler     // Absolute symbol.
392a34c753fSRafael Auler     ErrorOr<uint64_t> CalleeAddressOrError = BC.getSymbolValue(*CalleeSymbol);
393a34c753fSRafael Auler     assert(CalleeAddressOrError && "unregistered symbol found");
394a34c753fSRafael Auler     return *CalleeAddressOrError > getAddress();
395a34c753fSRafael Auler   }
396a34c753fSRafael Auler }
397a34c753fSRafael Auler 
398a34c753fSRafael Auler void BinaryFunction::dump(bool PrintInstructions) const {
399a34c753fSRafael Auler   print(dbgs(), "", PrintInstructions);
400a34c753fSRafael Auler }
401a34c753fSRafael Auler 
402a34c753fSRafael Auler void BinaryFunction::print(raw_ostream &OS, std::string Annotation,
403a34c753fSRafael Auler                            bool PrintInstructions) const {
404a34c753fSRafael Auler   if (!opts::shouldPrint(*this))
405a34c753fSRafael Auler     return;
406a34c753fSRafael Auler 
407a34c753fSRafael Auler   StringRef SectionName =
408a34c753fSRafael Auler       OriginSection ? OriginSection->getName() : "<no origin section>";
409a34c753fSRafael Auler   OS << "Binary Function \"" << *this << "\" " << Annotation << " {";
410a34c753fSRafael Auler   std::vector<StringRef> AllNames = getNames();
411a34c753fSRafael Auler   if (AllNames.size() > 1) {
412a34c753fSRafael Auler     OS << "\n  All names   : ";
413a34c753fSRafael Auler     const char *Sep = "";
414253b8f0aSAmir Ayupov     for (const StringRef &Name : AllNames) {
415a34c753fSRafael Auler       OS << Sep << Name;
416a34c753fSRafael Auler       Sep = "\n                ";
417a34c753fSRafael Auler     }
418a34c753fSRafael Auler   }
419a34c753fSRafael Auler   OS << "\n  Number      : "   << FunctionNumber
420a34c753fSRafael Auler      << "\n  State       : "   << CurrentState
421a34c753fSRafael Auler      << "\n  Address     : 0x" << Twine::utohexstr(Address)
422a34c753fSRafael Auler      << "\n  Size        : 0x" << Twine::utohexstr(Size)
423a34c753fSRafael Auler      << "\n  MaxSize     : 0x" << Twine::utohexstr(MaxSize)
424a34c753fSRafael Auler      << "\n  Offset      : 0x" << Twine::utohexstr(FileOffset)
425a34c753fSRafael Auler      << "\n  Section     : "   << SectionName
426a34c753fSRafael Auler      << "\n  Orc Section : "   << getCodeSectionName()
427a34c753fSRafael Auler      << "\n  LSDA        : 0x" << Twine::utohexstr(getLSDAAddress())
428a34c753fSRafael Auler      << "\n  IsSimple    : "   << IsSimple
429a34c753fSRafael Auler      << "\n  IsMultiEntry: "   << isMultiEntry()
430a34c753fSRafael Auler      << "\n  IsSplit     : "   << isSplit()
431a34c753fSRafael Auler      << "\n  BB Count    : "   << size();
432a34c753fSRafael Auler 
43340c2e0faSMaksim Panchenko   if (HasFixedIndirectBranch)
434a34c753fSRafael Auler     OS << "\n  HasFixedIndirectBranch : true";
43540c2e0faSMaksim Panchenko   if (HasUnknownControlFlow)
436a34c753fSRafael Auler     OS << "\n  Unknown CF  : true";
43740c2e0faSMaksim Panchenko   if (getPersonalityFunction())
438a34c753fSRafael Auler     OS << "\n  Personality : " << getPersonalityFunction()->getName();
43940c2e0faSMaksim Panchenko   if (IsFragment)
440a34c753fSRafael Auler     OS << "\n  IsFragment  : true";
44140c2e0faSMaksim Panchenko   if (isFolded())
442a34c753fSRafael Auler     OS << "\n  FoldedInto  : " << *getFoldedIntoFunction();
44340c2e0faSMaksim Panchenko   for (BinaryFunction *ParentFragment : ParentFragments)
444a34c753fSRafael Auler     OS << "\n  Parent      : " << *ParentFragment;
445a34c753fSRafael Auler   if (!Fragments.empty()) {
446a34c753fSRafael Auler     OS << "\n  Fragments   : ";
447f7581a39SAmir Ayupov     ListSeparator LS;
448f7581a39SAmir Ayupov     for (BinaryFunction *Frag : Fragments)
449f7581a39SAmir Ayupov       OS << LS << *Frag;
450a34c753fSRafael Auler   }
45140c2e0faSMaksim Panchenko   if (hasCFG())
452a34c753fSRafael Auler     OS << "\n  Hash        : " << Twine::utohexstr(computeHash());
453a34c753fSRafael Auler   if (isMultiEntry()) {
454a34c753fSRafael Auler     OS << "\n  Secondary Entry Points : ";
455f7581a39SAmir Ayupov     ListSeparator LS;
456f7581a39SAmir Ayupov     for (const auto &KV : SecondaryEntryPoints)
457f7581a39SAmir Ayupov       OS << LS << KV.second->getName();
458a34c753fSRafael Auler   }
45940c2e0faSMaksim Panchenko   if (FrameInstructions.size())
460a34c753fSRafael Auler     OS << "\n  CFI Instrs  : " << FrameInstructions.size();
461a34c753fSRafael Auler   if (BasicBlocksLayout.size()) {
462a34c753fSRafael Auler     OS << "\n  BB Layout   : ";
463f7581a39SAmir Ayupov     ListSeparator LS;
464f7581a39SAmir Ayupov     for (BinaryBasicBlock *BB : BasicBlocksLayout)
465f7581a39SAmir Ayupov       OS << LS << BB->getName();
466a34c753fSRafael Auler   }
467a34c753fSRafael Auler   if (ImageAddress)
468a34c753fSRafael Auler     OS << "\n  Image       : 0x" << Twine::utohexstr(ImageAddress);
469a34c753fSRafael Auler   if (ExecutionCount != COUNT_NO_PROFILE) {
470a34c753fSRafael Auler     OS << "\n  Exec Count  : " << ExecutionCount;
471a34c753fSRafael Auler     OS << "\n  Profile Acc : " << format("%.1f%%", ProfileMatchRatio * 100.0f);
472a34c753fSRafael Auler   }
473a34c753fSRafael Auler 
474a34c753fSRafael Auler   if (opts::PrintDynoStats && !BasicBlocksLayout.empty()) {
475a34c753fSRafael Auler     OS << '\n';
476a34c753fSRafael Auler     DynoStats dynoStats = getDynoStats(*this);
477a34c753fSRafael Auler     OS << dynoStats;
478a34c753fSRafael Auler   }
479a34c753fSRafael Auler 
480a34c753fSRafael Auler   OS << "\n}\n";
481a34c753fSRafael Auler 
482a34c753fSRafael Auler   if (opts::PrintDynoStatsOnly || !PrintInstructions || !BC.InstPrinter)
483a34c753fSRafael Auler     return;
484a34c753fSRafael Auler 
485a34c753fSRafael Auler   // Offset of the instruction in function.
486a34c753fSRafael Auler   uint64_t Offset = 0;
487a34c753fSRafael Auler 
488a34c753fSRafael Auler   if (BasicBlocks.empty() && !Instructions.empty()) {
489a34c753fSRafael Auler     // Print before CFG was built.
490a34c753fSRafael Auler     for (const std::pair<const uint32_t, MCInst> &II : Instructions) {
491a34c753fSRafael Auler       Offset = II.first;
492a34c753fSRafael Auler 
493a34c753fSRafael Auler       // Print label if exists at this offset.
494a34c753fSRafael Auler       auto LI = Labels.find(Offset);
495a34c753fSRafael Auler       if (LI != Labels.end()) {
496a34c753fSRafael Auler         if (const MCSymbol *EntrySymbol =
497a34c753fSRafael Auler                 getSecondaryEntryPointSymbol(LI->second))
498a34c753fSRafael Auler           OS << EntrySymbol->getName() << " (Entry Point):\n";
499a34c753fSRafael Auler         OS << LI->second->getName() << ":\n";
500a34c753fSRafael Auler       }
501a34c753fSRafael Auler 
502a34c753fSRafael Auler       BC.printInstruction(OS, II.second, Offset, this);
503a34c753fSRafael Auler     }
504a34c753fSRafael Auler   }
505a34c753fSRafael Auler 
506a34c753fSRafael Auler   for (uint32_t I = 0, E = BasicBlocksLayout.size(); I != E; ++I) {
507a34c753fSRafael Auler     BinaryBasicBlock *BB = BasicBlocksLayout[I];
50840c2e0faSMaksim Panchenko     if (I != 0 && BB->isCold() != BasicBlocksLayout[I - 1]->isCold())
509a34c753fSRafael Auler       OS << "-------   HOT-COLD SPLIT POINT   -------\n\n";
510a34c753fSRafael Auler 
51140c2e0faSMaksim Panchenko     OS << BB->getName() << " (" << BB->size()
51240c2e0faSMaksim Panchenko        << " instructions, align : " << BB->getAlignment() << ")\n";
513a34c753fSRafael Auler 
514a34c753fSRafael Auler     if (isEntryPoint(*BB)) {
515a34c753fSRafael Auler       if (MCSymbol *EntrySymbol = getSecondaryEntryPointSymbol(*BB))
516a34c753fSRafael Auler         OS << "  Secondary Entry Point: " << EntrySymbol->getName() << '\n';
517a34c753fSRafael Auler       else
518a34c753fSRafael Auler         OS << "  Entry Point\n";
519a34c753fSRafael Auler     }
520a34c753fSRafael Auler 
521a34c753fSRafael Auler     if (BB->isLandingPad())
522a34c753fSRafael Auler       OS << "  Landing Pad\n";
523a34c753fSRafael Auler 
524a34c753fSRafael Auler     uint64_t BBExecCount = BB->getExecutionCount();
525a34c753fSRafael Auler     if (hasValidProfile()) {
526a34c753fSRafael Auler       OS << "  Exec Count : ";
527a34c753fSRafael Auler       if (BB->getExecutionCount() != BinaryBasicBlock::COUNT_NO_PROFILE)
528a34c753fSRafael Auler         OS << BBExecCount << '\n';
529a34c753fSRafael Auler       else
530a34c753fSRafael Auler         OS << "<unknown>\n";
531a34c753fSRafael Auler     }
5323652483cSRafael Auler     if (BB->getCFIState() >= 0)
533a34c753fSRafael Auler       OS << "  CFI State : " << BB->getCFIState() << '\n';
534a34c753fSRafael Auler     if (opts::EnableBAT) {
535a34c753fSRafael Auler       OS << "  Input offset: " << Twine::utohexstr(BB->getInputOffset())
536a34c753fSRafael Auler          << "\n";
537a34c753fSRafael Auler     }
538a34c753fSRafael Auler     if (!BB->pred_empty()) {
539a34c753fSRafael Auler       OS << "  Predecessors: ";
540f7581a39SAmir Ayupov       ListSeparator LS;
541f7581a39SAmir Ayupov       for (BinaryBasicBlock *Pred : BB->predecessors())
542f7581a39SAmir Ayupov         OS << LS << Pred->getName();
543a34c753fSRafael Auler       OS << '\n';
544a34c753fSRafael Auler     }
545a34c753fSRafael Auler     if (!BB->throw_empty()) {
546a34c753fSRafael Auler       OS << "  Throwers: ";
547f7581a39SAmir Ayupov       ListSeparator LS;
548f7581a39SAmir Ayupov       for (BinaryBasicBlock *Throw : BB->throwers())
549f7581a39SAmir Ayupov         OS << LS << Throw->getName();
550a34c753fSRafael Auler       OS << '\n';
551a34c753fSRafael Auler     }
552a34c753fSRafael Auler 
553a34c753fSRafael Auler     Offset = alignTo(Offset, BB->getAlignment());
554a34c753fSRafael Auler 
555a34c753fSRafael Auler     // Note: offsets are imprecise since this is happening prior to relaxation.
556a34c753fSRafael Auler     Offset = BC.printInstructions(OS, BB->begin(), BB->end(), Offset, this);
557a34c753fSRafael Auler 
558a34c753fSRafael Auler     if (!BB->succ_empty()) {
559a34c753fSRafael Auler       OS << "  Successors: ";
560a34c753fSRafael Auler       // For more than 2 successors, sort them based on frequency.
561a34c753fSRafael Auler       std::vector<uint64_t> Indices(BB->succ_size());
562a34c753fSRafael Auler       std::iota(Indices.begin(), Indices.end(), 0);
563a34c753fSRafael Auler       if (BB->succ_size() > 2 && BB->getKnownExecutionCount()) {
564d2c87699SAmir Ayupov         llvm::stable_sort(Indices, [&](const uint64_t A, const uint64_t B) {
565a34c753fSRafael Auler           return BB->BranchInfo[B] < BB->BranchInfo[A];
566a34c753fSRafael Auler         });
567a34c753fSRafael Auler       }
568f7581a39SAmir Ayupov       ListSeparator LS;
569a34c753fSRafael Auler       for (unsigned I = 0; I < Indices.size(); ++I) {
570a34c753fSRafael Auler         BinaryBasicBlock *Succ = BB->Successors[Indices[I]];
571a34c753fSRafael Auler         BinaryBasicBlock::BinaryBranchInfo &BI = BB->BranchInfo[Indices[I]];
572f7581a39SAmir Ayupov         OS << LS << Succ->getName();
573a34c753fSRafael Auler         if (ExecutionCount != COUNT_NO_PROFILE &&
574a34c753fSRafael Auler             BI.MispredictedCount != BinaryBasicBlock::COUNT_INFERRED) {
575a34c753fSRafael Auler           OS << " (mispreds: " << BI.MispredictedCount
576a34c753fSRafael Auler              << ", count: " << BI.Count << ")";
577a34c753fSRafael Auler         } else if (ExecutionCount != COUNT_NO_PROFILE &&
578a34c753fSRafael Auler                    BI.Count != BinaryBasicBlock::COUNT_NO_PROFILE) {
579a34c753fSRafael Auler           OS << " (inferred count: " << BI.Count << ")";
580a34c753fSRafael Auler         }
581a34c753fSRafael Auler       }
582a34c753fSRafael Auler       OS << '\n';
583a34c753fSRafael Auler     }
584a34c753fSRafael Auler 
585a34c753fSRafael Auler     if (!BB->lp_empty()) {
586a34c753fSRafael Auler       OS << "  Landing Pads: ";
587f7581a39SAmir Ayupov       ListSeparator LS;
588a34c753fSRafael Auler       for (BinaryBasicBlock *LP : BB->landing_pads()) {
589f7581a39SAmir Ayupov         OS << LS << LP->getName();
590a34c753fSRafael Auler         if (ExecutionCount != COUNT_NO_PROFILE) {
591a34c753fSRafael Auler           OS << " (count: " << LP->getExecutionCount() << ")";
592a34c753fSRafael Auler         }
593a34c753fSRafael Auler       }
594a34c753fSRafael Auler       OS << '\n';
595a34c753fSRafael Auler     }
596a34c753fSRafael Auler 
597a34c753fSRafael Auler     // In CFG_Finalized state we can miscalculate CFI state at exit.
598a34c753fSRafael Auler     if (CurrentState == State::CFG) {
599a34c753fSRafael Auler       const int32_t CFIStateAtExit = BB->getCFIStateAtExit();
600a34c753fSRafael Auler       if (CFIStateAtExit >= 0)
601a34c753fSRafael Auler         OS << "  CFI State: " << CFIStateAtExit << '\n';
602a34c753fSRafael Auler     }
603a34c753fSRafael Auler 
604a34c753fSRafael Auler     OS << '\n';
605a34c753fSRafael Auler   }
606a34c753fSRafael Auler 
607a34c753fSRafael Auler   // Dump new exception ranges for the function.
608a34c753fSRafael Auler   if (!CallSites.empty()) {
609a34c753fSRafael Auler     OS << "EH table:\n";
610a34c753fSRafael Auler     for (const CallSite &CSI : CallSites) {
611a34c753fSRafael Auler       OS << "  [" << *CSI.Start << ", " << *CSI.End << ") landing pad : ";
612a34c753fSRafael Auler       if (CSI.LP)
613a34c753fSRafael Auler         OS << *CSI.LP;
614a34c753fSRafael Auler       else
615a34c753fSRafael Auler         OS << "0";
616a34c753fSRafael Auler       OS << ", action : " << CSI.Action << '\n';
617a34c753fSRafael Auler     }
618a34c753fSRafael Auler     OS << '\n';
619a34c753fSRafael Auler   }
620a34c753fSRafael Auler 
621a34c753fSRafael Auler   // Print all jump tables.
6223652483cSRafael Auler   for (const std::pair<const uint64_t, JumpTable *> &JTI : JumpTables)
623a34c753fSRafael Auler     JTI.second->print(OS);
624a34c753fSRafael Auler 
625a34c753fSRafael Auler   OS << "DWARF CFI Instructions:\n";
626a34c753fSRafael Auler   if (OffsetToCFI.size()) {
627a34c753fSRafael Auler     // Pre-buildCFG information
628a34c753fSRafael Auler     for (const std::pair<const uint32_t, uint32_t> &Elmt : OffsetToCFI) {
629a34c753fSRafael Auler       OS << format("    %08x:\t", Elmt.first);
630a34c753fSRafael Auler       assert(Elmt.second < FrameInstructions.size() && "Incorrect CFI offset");
631a34c753fSRafael Auler       BinaryContext::printCFI(OS, FrameInstructions[Elmt.second]);
632a34c753fSRafael Auler       OS << "\n";
633a34c753fSRafael Auler     }
634a34c753fSRafael Auler   } else {
635a34c753fSRafael Auler     // Post-buildCFG information
636a34c753fSRafael Auler     for (uint32_t I = 0, E = FrameInstructions.size(); I != E; ++I) {
637a34c753fSRafael Auler       const MCCFIInstruction &CFI = FrameInstructions[I];
638a34c753fSRafael Auler       OS << format("    %d:\t", I);
639a34c753fSRafael Auler       BinaryContext::printCFI(OS, CFI);
640a34c753fSRafael Auler       OS << "\n";
641a34c753fSRafael Auler     }
642a34c753fSRafael Auler   }
643a34c753fSRafael Auler   if (FrameInstructions.empty())
644a34c753fSRafael Auler     OS << "    <empty>\n";
645a34c753fSRafael Auler 
646a34c753fSRafael Auler   OS << "End of Function \"" << *this << "\"\n\n";
647a34c753fSRafael Auler }
648a34c753fSRafael Auler 
64940c2e0faSMaksim Panchenko void BinaryFunction::printRelocations(raw_ostream &OS, uint64_t Offset,
650a34c753fSRafael Auler                                       uint64_t Size) const {
651a34c753fSRafael Auler   const char *Sep = " # Relocs: ";
652a34c753fSRafael Auler 
653a34c753fSRafael Auler   auto RI = Relocations.lower_bound(Offset);
654a34c753fSRafael Auler   while (RI != Relocations.end() && RI->first < Offset + Size) {
655a34c753fSRafael Auler     OS << Sep << "(R: " << RI->second << ")";
656a34c753fSRafael Auler     Sep = ", ";
657a34c753fSRafael Auler     ++RI;
658a34c753fSRafael Auler   }
659a34c753fSRafael Auler }
660a34c753fSRafael Auler 
661a34c753fSRafael Auler namespace {
662a34c753fSRafael Auler std::string mutateDWARFExpressionTargetReg(const MCCFIInstruction &Instr,
663a34c753fSRafael Auler                                            MCPhysReg NewReg) {
664a34c753fSRafael Auler   StringRef ExprBytes = Instr.getValues();
665a34c753fSRafael Auler   assert(ExprBytes.size() > 1 && "DWARF expression CFI is too short");
666a34c753fSRafael Auler   uint8_t Opcode = ExprBytes[0];
667a34c753fSRafael Auler   assert((Opcode == dwarf::DW_CFA_expression ||
668a34c753fSRafael Auler           Opcode == dwarf::DW_CFA_val_expression) &&
669a34c753fSRafael Auler          "invalid DWARF expression CFI");
670139744acSAmir Ayupov   (void)Opcode;
671a34c753fSRafael Auler   const uint8_t *const Start =
672a34c753fSRafael Auler       reinterpret_cast<const uint8_t *>(ExprBytes.drop_front(1).data());
673a34c753fSRafael Auler   const uint8_t *const End =
674a34c753fSRafael Auler       reinterpret_cast<const uint8_t *>(Start + ExprBytes.size() - 1);
675a34c753fSRafael Auler   unsigned Size = 0;
676a34c753fSRafael Auler   decodeULEB128(Start, &Size, End);
677a34c753fSRafael Auler   assert(Size > 0 && "Invalid reg encoding for DWARF expression CFI");
678a34c753fSRafael Auler   SmallString<8> Tmp;
679a34c753fSRafael Auler   raw_svector_ostream OSE(Tmp);
680a34c753fSRafael Auler   encodeULEB128(NewReg, OSE);
681a34c753fSRafael Auler   return Twine(ExprBytes.slice(0, 1))
682a34c753fSRafael Auler       .concat(OSE.str())
683a34c753fSRafael Auler       .concat(ExprBytes.drop_front(1 + Size))
684a34c753fSRafael Auler       .str();
685a34c753fSRafael Auler }
686a34c753fSRafael Auler } // namespace
687a34c753fSRafael Auler 
688a34c753fSRafael Auler void BinaryFunction::mutateCFIRegisterFor(const MCInst &Instr,
689a34c753fSRafael Auler                                           MCPhysReg NewReg) {
690a34c753fSRafael Auler   const MCCFIInstruction *OldCFI = getCFIFor(Instr);
691a34c753fSRafael Auler   assert(OldCFI && "invalid CFI instr");
692a34c753fSRafael Auler   switch (OldCFI->getOperation()) {
693a34c753fSRafael Auler   default:
694a34c753fSRafael Auler     llvm_unreachable("Unexpected instruction");
695a34c753fSRafael Auler   case MCCFIInstruction::OpDefCfa:
696a34c753fSRafael Auler     setCFIFor(Instr, MCCFIInstruction::cfiDefCfa(nullptr, NewReg,
697a34c753fSRafael Auler                                                  OldCFI->getOffset()));
698a34c753fSRafael Auler     break;
699a34c753fSRafael Auler   case MCCFIInstruction::OpDefCfaRegister:
700a34c753fSRafael Auler     setCFIFor(Instr, MCCFIInstruction::createDefCfaRegister(nullptr, NewReg));
701a34c753fSRafael Auler     break;
702a34c753fSRafael Auler   case MCCFIInstruction::OpOffset:
703a34c753fSRafael Auler     setCFIFor(Instr, MCCFIInstruction::createOffset(nullptr, NewReg,
704a34c753fSRafael Auler                                                     OldCFI->getOffset()));
705a34c753fSRafael Auler     break;
706a34c753fSRafael Auler   case MCCFIInstruction::OpRegister:
707a34c753fSRafael Auler     setCFIFor(Instr, MCCFIInstruction::createRegister(nullptr, NewReg,
708a34c753fSRafael Auler                                                       OldCFI->getRegister2()));
709a34c753fSRafael Auler     break;
710a34c753fSRafael Auler   case MCCFIInstruction::OpSameValue:
711a34c753fSRafael Auler     setCFIFor(Instr, MCCFIInstruction::createSameValue(nullptr, NewReg));
712a34c753fSRafael Auler     break;
713a34c753fSRafael Auler   case MCCFIInstruction::OpEscape:
714a34c753fSRafael Auler     setCFIFor(Instr,
715a34c753fSRafael Auler               MCCFIInstruction::createEscape(
716a34c753fSRafael Auler                   nullptr,
717a34c753fSRafael Auler                   StringRef(mutateDWARFExpressionTargetReg(*OldCFI, NewReg))));
718a34c753fSRafael Auler     break;
719a34c753fSRafael Auler   case MCCFIInstruction::OpRestore:
720a34c753fSRafael Auler     setCFIFor(Instr, MCCFIInstruction::createRestore(nullptr, NewReg));
721a34c753fSRafael Auler     break;
722a34c753fSRafael Auler   case MCCFIInstruction::OpUndefined:
723a34c753fSRafael Auler     setCFIFor(Instr, MCCFIInstruction::createUndefined(nullptr, NewReg));
724a34c753fSRafael Auler     break;
725a34c753fSRafael Auler   }
726a34c753fSRafael Auler }
727a34c753fSRafael Auler 
728a34c753fSRafael Auler const MCCFIInstruction *BinaryFunction::mutateCFIOffsetFor(const MCInst &Instr,
729a34c753fSRafael Auler                                                            int64_t NewOffset) {
730a34c753fSRafael Auler   const MCCFIInstruction *OldCFI = getCFIFor(Instr);
731a34c753fSRafael Auler   assert(OldCFI && "invalid CFI instr");
732a34c753fSRafael Auler   switch (OldCFI->getOperation()) {
733a34c753fSRafael Auler   default:
734a34c753fSRafael Auler     llvm_unreachable("Unexpected instruction");
735a34c753fSRafael Auler   case MCCFIInstruction::OpDefCfaOffset:
736a34c753fSRafael Auler     setCFIFor(Instr, MCCFIInstruction::cfiDefCfaOffset(nullptr, NewOffset));
737a34c753fSRafael Auler     break;
738a34c753fSRafael Auler   case MCCFIInstruction::OpAdjustCfaOffset:
739a34c753fSRafael Auler     setCFIFor(Instr,
740a34c753fSRafael Auler               MCCFIInstruction::createAdjustCfaOffset(nullptr, NewOffset));
741a34c753fSRafael Auler     break;
742a34c753fSRafael Auler   case MCCFIInstruction::OpDefCfa:
743a34c753fSRafael Auler     setCFIFor(Instr, MCCFIInstruction::cfiDefCfa(nullptr, OldCFI->getRegister(),
744a34c753fSRafael Auler                                                  NewOffset));
745a34c753fSRafael Auler     break;
746a34c753fSRafael Auler   case MCCFIInstruction::OpOffset:
747a34c753fSRafael Auler     setCFIFor(Instr, MCCFIInstruction::createOffset(
748a34c753fSRafael Auler                          nullptr, OldCFI->getRegister(), NewOffset));
749a34c753fSRafael Auler     break;
750a34c753fSRafael Auler   }
751a34c753fSRafael Auler   return getCFIFor(Instr);
752a34c753fSRafael Auler }
753a34c753fSRafael Auler 
754a34c753fSRafael Auler IndirectBranchType
75540c2e0faSMaksim Panchenko BinaryFunction::processIndirectBranch(MCInst &Instruction, unsigned Size,
756a34c753fSRafael Auler                                       uint64_t Offset,
757a34c753fSRafael Auler                                       uint64_t &TargetAddress) {
758a34c753fSRafael Auler   const unsigned PtrSize = BC.AsmInfo->getCodePointerSize();
759a34c753fSRafael Auler 
760a34c753fSRafael Auler   // The instruction referencing memory used by the branch instruction.
761a34c753fSRafael Auler   // It could be the branch instruction itself or one of the instructions
762a34c753fSRafael Auler   // setting the value of the register used by the branch.
763a34c753fSRafael Auler   MCInst *MemLocInstr;
764a34c753fSRafael Auler 
765a34c753fSRafael Auler   // Address of the table referenced by MemLocInstr. Could be either an
766a34c753fSRafael Auler   // array of function pointers, or a jump table.
767a34c753fSRafael Auler   uint64_t ArrayStart = 0;
768a34c753fSRafael Auler 
769a34c753fSRafael Auler   unsigned BaseRegNum, IndexRegNum;
770a34c753fSRafael Auler   int64_t DispValue;
771a34c753fSRafael Auler   const MCExpr *DispExpr;
772a34c753fSRafael Auler 
773a34c753fSRafael Auler   // In AArch, identify the instruction adding the PC-relative offset to
774a34c753fSRafael Auler   // jump table entries to correctly decode it.
775a34c753fSRafael Auler   MCInst *PCRelBaseInstr;
776a34c753fSRafael Auler   uint64_t PCRelAddr = 0;
777a34c753fSRafael Auler 
778a34c753fSRafael Auler   auto Begin = Instructions.begin();
779a34c753fSRafael Auler   if (BC.isAArch64()) {
780a34c753fSRafael Auler     PreserveNops = BC.HasRelocations;
781a34c753fSRafael Auler     // Start at the last label as an approximation of the current basic block.
782a34c753fSRafael Auler     // This is a heuristic, since the full set of labels have yet to be
783a34c753fSRafael Auler     // determined
784a34c753fSRafael Auler     for (auto LI = Labels.rbegin(); LI != Labels.rend(); ++LI) {
785a34c753fSRafael Auler       auto II = Instructions.find(LI->first);
786a34c753fSRafael Auler       if (II != Instructions.end()) {
787a34c753fSRafael Auler         Begin = II;
788a34c753fSRafael Auler         break;
789a34c753fSRafael Auler       }
790a34c753fSRafael Auler     }
791a34c753fSRafael Auler   }
792a34c753fSRafael Auler 
79340c2e0faSMaksim Panchenko   IndirectBranchType BranchType = BC.MIB->analyzeIndirectBranch(
79440c2e0faSMaksim Panchenko       Instruction, Begin, Instructions.end(), PtrSize, MemLocInstr, BaseRegNum,
79540c2e0faSMaksim Panchenko       IndexRegNum, DispValue, DispExpr, PCRelBaseInstr);
796a34c753fSRafael Auler 
797a34c753fSRafael Auler   if (BranchType == IndirectBranchType::UNKNOWN && !MemLocInstr)
798a34c753fSRafael Auler     return BranchType;
799a34c753fSRafael Auler 
800a34c753fSRafael Auler   if (MemLocInstr != &Instruction)
801a34c753fSRafael Auler     IndexRegNum = BC.MIB->getNoRegister();
802a34c753fSRafael Auler 
803a34c753fSRafael Auler   if (BC.isAArch64()) {
804a34c753fSRafael Auler     const MCSymbol *Sym = BC.MIB->getTargetSymbol(*PCRelBaseInstr, 1);
805a34c753fSRafael Auler     assert(Sym && "Symbol extraction failed");
806a34c753fSRafael Auler     ErrorOr<uint64_t> SymValueOrError = BC.getSymbolValue(*Sym);
807a34c753fSRafael Auler     if (SymValueOrError) {
808a34c753fSRafael Auler       PCRelAddr = *SymValueOrError;
809a34c753fSRafael Auler     } else {
810a34c753fSRafael Auler       for (std::pair<const uint32_t, MCSymbol *> &Elmt : Labels) {
811a34c753fSRafael Auler         if (Elmt.second == Sym) {
812a34c753fSRafael Auler           PCRelAddr = Elmt.first + getAddress();
813a34c753fSRafael Auler           break;
814a34c753fSRafael Auler         }
815a34c753fSRafael Auler       }
816a34c753fSRafael Auler     }
817a34c753fSRafael Auler     uint64_t InstrAddr = 0;
818a34c753fSRafael Auler     for (auto II = Instructions.rbegin(); II != Instructions.rend(); ++II) {
819a34c753fSRafael Auler       if (&II->second == PCRelBaseInstr) {
820a34c753fSRafael Auler         InstrAddr = II->first + getAddress();
821a34c753fSRafael Auler         break;
822a34c753fSRafael Auler       }
823a34c753fSRafael Auler     }
824a34c753fSRafael Auler     assert(InstrAddr != 0 && "instruction not found");
825a34c753fSRafael Auler     // We do this to avoid spurious references to code locations outside this
826a34c753fSRafael Auler     // function (for example, if the indirect jump lives in the last basic
827a34c753fSRafael Auler     // block of the function, it will create a reference to the next function).
828a34c753fSRafael Auler     // This replaces a symbol reference with an immediate.
829a34c753fSRafael Auler     BC.MIB->replaceMemOperandDisp(*PCRelBaseInstr,
830a34c753fSRafael Auler                                   MCOperand::createImm(PCRelAddr - InstrAddr));
831a34c753fSRafael Auler     // FIXME: Disable full jump table processing for AArch64 until we have a
832a34c753fSRafael Auler     // proper way of determining the jump table limits.
833a34c753fSRafael Auler     return IndirectBranchType::UNKNOWN;
834a34c753fSRafael Auler   }
835a34c753fSRafael Auler 
836a34c753fSRafael Auler   // RIP-relative addressing should be converted to symbol form by now
837a34c753fSRafael Auler   // in processed instructions (but not in jump).
838a34c753fSRafael Auler   if (DispExpr) {
839a34c753fSRafael Auler     const MCSymbol *TargetSym;
840a34c753fSRafael Auler     uint64_t TargetOffset;
841a34c753fSRafael Auler     std::tie(TargetSym, TargetOffset) = BC.MIB->getTargetSymbolInfo(DispExpr);
842a34c753fSRafael Auler     ErrorOr<uint64_t> SymValueOrError = BC.getSymbolValue(*TargetSym);
843a34c753fSRafael Auler     assert(SymValueOrError && "global symbol needs a value");
844a34c753fSRafael Auler     ArrayStart = *SymValueOrError + TargetOffset;
845a34c753fSRafael Auler     BaseRegNum = BC.MIB->getNoRegister();
846a34c753fSRafael Auler     if (BC.isAArch64()) {
847a34c753fSRafael Auler       ArrayStart &= ~0xFFFULL;
848a34c753fSRafael Auler       ArrayStart += DispValue & 0xFFFULL;
849a34c753fSRafael Auler     }
850a34c753fSRafael Auler   } else {
851a34c753fSRafael Auler     ArrayStart = static_cast<uint64_t>(DispValue);
852a34c753fSRafael Auler   }
853a34c753fSRafael Auler 
854a34c753fSRafael Auler   if (BaseRegNum == BC.MRI->getProgramCounter())
855a34c753fSRafael Auler     ArrayStart += getAddress() + Offset + Size;
856a34c753fSRafael Auler 
857a34c753fSRafael Auler   LLVM_DEBUG(dbgs() << "BOLT-DEBUG: addressed memory is 0x"
858a34c753fSRafael Auler                     << Twine::utohexstr(ArrayStart) << '\n');
859a34c753fSRafael Auler 
860a34c753fSRafael Auler   ErrorOr<BinarySection &> Section = BC.getSectionForAddress(ArrayStart);
861a34c753fSRafael Auler   if (!Section) {
862a34c753fSRafael Auler     // No section - possibly an absolute address. Since we don't allow
863a34c753fSRafael Auler     // internal function addresses to escape the function scope - we
864a34c753fSRafael Auler     // consider it a tail call.
865a34c753fSRafael Auler     if (opts::Verbosity >= 1) {
866a34c753fSRafael Auler       errs() << "BOLT-WARNING: no section for address 0x"
867a34c753fSRafael Auler              << Twine::utohexstr(ArrayStart) << " referenced from function "
868a34c753fSRafael Auler              << *this << '\n';
869a34c753fSRafael Auler     }
870a34c753fSRafael Auler     return IndirectBranchType::POSSIBLE_TAIL_CALL;
871a34c753fSRafael Auler   }
872a34c753fSRafael Auler   if (Section->isVirtual()) {
873a34c753fSRafael Auler     // The contents are filled at runtime.
874a34c753fSRafael Auler     return IndirectBranchType::POSSIBLE_TAIL_CALL;
875a34c753fSRafael Auler   }
876a34c753fSRafael Auler 
877a34c753fSRafael Auler   if (BranchType == IndirectBranchType::POSSIBLE_FIXED_BRANCH) {
878a34c753fSRafael Auler     ErrorOr<uint64_t> Value = BC.getPointerAtAddress(ArrayStart);
879a34c753fSRafael Auler     if (!Value)
880a34c753fSRafael Auler       return IndirectBranchType::UNKNOWN;
881a34c753fSRafael Auler 
882a34c753fSRafael Auler     if (!BC.getSectionForAddress(ArrayStart)->isReadOnly())
883a34c753fSRafael Auler       return IndirectBranchType::UNKNOWN;
884a34c753fSRafael Auler 
885a34c753fSRafael Auler     outs() << "BOLT-INFO: fixed indirect branch detected in " << *this
886a34c753fSRafael Auler            << " at 0x" << Twine::utohexstr(getAddress() + Offset)
887a34c753fSRafael Auler            << " referencing data at 0x" << Twine::utohexstr(ArrayStart)
888a34c753fSRafael Auler            << " the destination value is 0x" << Twine::utohexstr(*Value)
889a34c753fSRafael Auler            << '\n';
890a34c753fSRafael Auler 
891a34c753fSRafael Auler     TargetAddress = *Value;
892a34c753fSRafael Auler     return BranchType;
893a34c753fSRafael Auler   }
894a34c753fSRafael Auler 
895a34c753fSRafael Auler   // Check if there's already a jump table registered at this address.
896a34c753fSRafael Auler   MemoryContentsType MemType;
897a34c753fSRafael Auler   if (JumpTable *JT = BC.getJumpTableContainingAddress(ArrayStart)) {
898a34c753fSRafael Auler     switch (JT->Type) {
899a34c753fSRafael Auler     case JumpTable::JTT_NORMAL:
900a34c753fSRafael Auler       MemType = MemoryContentsType::POSSIBLE_JUMP_TABLE;
901a34c753fSRafael Auler       break;
902a34c753fSRafael Auler     case JumpTable::JTT_PIC:
903a34c753fSRafael Auler       MemType = MemoryContentsType::POSSIBLE_PIC_JUMP_TABLE;
904a34c753fSRafael Auler       break;
905a34c753fSRafael Auler     }
906a34c753fSRafael Auler   } else {
907a34c753fSRafael Auler     MemType = BC.analyzeMemoryAt(ArrayStart, *this);
908a34c753fSRafael Auler   }
909a34c753fSRafael Auler 
910a34c753fSRafael Auler   // Check that jump table type in instruction pattern matches memory contents.
911a34c753fSRafael Auler   JumpTable::JumpTableType JTType;
912a34c753fSRafael Auler   if (BranchType == IndirectBranchType::POSSIBLE_PIC_JUMP_TABLE) {
913a34c753fSRafael Auler     if (MemType != MemoryContentsType::POSSIBLE_PIC_JUMP_TABLE)
914a34c753fSRafael Auler       return IndirectBranchType::UNKNOWN;
915a34c753fSRafael Auler     JTType = JumpTable::JTT_PIC;
916a34c753fSRafael Auler   } else {
917a34c753fSRafael Auler     if (MemType == MemoryContentsType::POSSIBLE_PIC_JUMP_TABLE)
918a34c753fSRafael Auler       return IndirectBranchType::UNKNOWN;
919a34c753fSRafael Auler 
920a34c753fSRafael Auler     if (MemType == MemoryContentsType::UNKNOWN)
921a34c753fSRafael Auler       return IndirectBranchType::POSSIBLE_TAIL_CALL;
922a34c753fSRafael Auler 
923a34c753fSRafael Auler     BranchType = IndirectBranchType::POSSIBLE_JUMP_TABLE;
924a34c753fSRafael Auler     JTType = JumpTable::JTT_NORMAL;
925a34c753fSRafael Auler   }
926a34c753fSRafael Auler 
927a34c753fSRafael Auler   // Convert the instruction into jump table branch.
928a34c753fSRafael Auler   const MCSymbol *JTLabel = BC.getOrCreateJumpTable(*this, ArrayStart, JTType);
929a34c753fSRafael Auler   BC.MIB->replaceMemOperandDisp(*MemLocInstr, JTLabel, BC.Ctx.get());
930a34c753fSRafael Auler   BC.MIB->setJumpTable(Instruction, ArrayStart, IndexRegNum);
931a34c753fSRafael Auler 
932a34c753fSRafael Auler   JTSites.emplace_back(Offset, ArrayStart);
933a34c753fSRafael Auler 
934a34c753fSRafael Auler   return BranchType;
935a34c753fSRafael Auler }
936a34c753fSRafael Auler 
937a34c753fSRafael Auler MCSymbol *BinaryFunction::getOrCreateLocalLabel(uint64_t Address,
938a34c753fSRafael Auler                                                 bool CreatePastEnd) {
939a34c753fSRafael Auler   const uint64_t Offset = Address - getAddress();
940a34c753fSRafael Auler 
941a34c753fSRafael Auler   if ((Offset == getSize()) && CreatePastEnd)
942a34c753fSRafael Auler     return getFunctionEndLabel();
943a34c753fSRafael Auler 
944a34c753fSRafael Auler   auto LI = Labels.find(Offset);
945a34c753fSRafael Auler   if (LI != Labels.end())
946a34c753fSRafael Auler     return LI->second;
947a34c753fSRafael Auler 
948a34c753fSRafael Auler   // For AArch64, check if this address is part of a constant island.
949a34c753fSRafael Auler   if (BC.isAArch64()) {
9503652483cSRafael Auler     if (MCSymbol *IslandSym = getOrCreateIslandAccess(Address))
951a34c753fSRafael Auler       return IslandSym;
952a34c753fSRafael Auler   }
953a34c753fSRafael Auler 
954a34c753fSRafael Auler   MCSymbol *Label = BC.Ctx->createNamedTempSymbol();
955a34c753fSRafael Auler   Labels[Offset] = Label;
956a34c753fSRafael Auler 
957a34c753fSRafael Auler   return Label;
958a34c753fSRafael Auler }
959a34c753fSRafael Auler 
960a34c753fSRafael Auler ErrorOr<ArrayRef<uint8_t>> BinaryFunction::getData() const {
961a34c753fSRafael Auler   BinarySection &Section = *getOriginSection();
962a34c753fSRafael Auler   assert(Section.containsRange(getAddress(), getMaxSize()) &&
963a34c753fSRafael Auler          "wrong section for function");
964a34c753fSRafael Auler 
9653652483cSRafael Auler   if (!Section.isText() || Section.isVirtual() || !Section.getSize())
966a34c753fSRafael Auler     return std::make_error_code(std::errc::bad_address);
967a34c753fSRafael Auler 
968a34c753fSRafael Auler   StringRef SectionContents = Section.getContents();
969a34c753fSRafael Auler 
970a34c753fSRafael Auler   assert(SectionContents.size() == Section.getSize() &&
971a34c753fSRafael Auler          "section size mismatch");
972a34c753fSRafael Auler 
973a34c753fSRafael Auler   // Function offset from the section start.
974a34c753fSRafael Auler   uint64_t Offset = getAddress() - Section.getAddress();
975a34c753fSRafael Auler   auto *Bytes = reinterpret_cast<const uint8_t *>(SectionContents.data());
976a34c753fSRafael Auler   return ArrayRef<uint8_t>(Bytes + Offset, getMaxSize());
977a34c753fSRafael Auler }
978a34c753fSRafael Auler 
979a34c753fSRafael Auler size_t BinaryFunction::getSizeOfDataInCodeAt(uint64_t Offset) const {
980a34c753fSRafael Auler   if (!Islands)
981a34c753fSRafael Auler     return 0;
982a34c753fSRafael Auler 
983a34c753fSRafael Auler   if (Islands->DataOffsets.find(Offset) == Islands->DataOffsets.end())
984a34c753fSRafael Auler     return 0;
985a34c753fSRafael Auler 
986a34c753fSRafael Auler   auto Iter = Islands->CodeOffsets.upper_bound(Offset);
9873652483cSRafael Auler   if (Iter != Islands->CodeOffsets.end())
988a34c753fSRafael Auler     return *Iter - Offset;
989a34c753fSRafael Auler   return getSize() - Offset;
990a34c753fSRafael Auler }
991a34c753fSRafael Auler 
992a34c753fSRafael Auler bool BinaryFunction::isZeroPaddingAt(uint64_t Offset) const {
993a34c753fSRafael Auler   ArrayRef<uint8_t> FunctionData = *getData();
994a34c753fSRafael Auler   uint64_t EndOfCode = getSize();
995a34c753fSRafael Auler   if (Islands) {
996a34c753fSRafael Auler     auto Iter = Islands->DataOffsets.upper_bound(Offset);
997a34c753fSRafael Auler     if (Iter != Islands->DataOffsets.end())
998a34c753fSRafael Auler       EndOfCode = *Iter;
999a34c753fSRafael Auler   }
10003652483cSRafael Auler   for (uint64_t I = Offset; I < EndOfCode; ++I)
10013652483cSRafael Auler     if (FunctionData[I] != 0)
1002a34c753fSRafael Auler       return false;
1003a34c753fSRafael Auler 
1004a34c753fSRafael Auler   return true;
1005a34c753fSRafael Auler }
1006a34c753fSRafael Auler 
1007a34c753fSRafael Auler bool BinaryFunction::disassemble() {
1008a34c753fSRafael Auler   NamedRegionTimer T("disassemble", "Disassemble function", "buildfuncs",
1009a34c753fSRafael Auler                      "Build Binary Functions", opts::TimeBuild);
1010a34c753fSRafael Auler   ErrorOr<ArrayRef<uint8_t>> ErrorOrFunctionData = getData();
1011a34c753fSRafael Auler   assert(ErrorOrFunctionData && "function data is not available");
1012a34c753fSRafael Auler   ArrayRef<uint8_t> FunctionData = *ErrorOrFunctionData;
1013a34c753fSRafael Auler   assert(FunctionData.size() == getMaxSize() &&
1014a34c753fSRafael Auler          "function size does not match raw data size");
1015a34c753fSRafael Auler 
1016a34c753fSRafael Auler   auto &Ctx = BC.Ctx;
1017a34c753fSRafael Auler   auto &MIB = BC.MIB;
1018a34c753fSRafael Auler 
1019e290133cSMaksim Panchenko   BC.SymbolicDisAsm->setSymbolizer(MIB->createTargetSymbolizer(*this));
1020e290133cSMaksim Panchenko 
1021a34c753fSRafael Auler   // Insert a label at the beginning of the function. This will be our first
1022a34c753fSRafael Auler   // basic block.
1023a34c753fSRafael Auler   Labels[0] = Ctx->createNamedTempSymbol("BB0");
1024a34c753fSRafael Auler 
102540c2e0faSMaksim Panchenko   auto handlePCRelOperand = [&](MCInst &Instruction, uint64_t Address,
102640c2e0faSMaksim Panchenko                                 uint64_t Size) {
1027a34c753fSRafael Auler     uint64_t TargetAddress = 0;
1028a34c753fSRafael Auler     if (!MIB->evaluateMemOperandTarget(Instruction, TargetAddress, Address,
1029a34c753fSRafael Auler                                        Size)) {
1030a34c753fSRafael Auler       errs() << "BOLT-ERROR: PC-relative operand can't be evaluated:\n";
1031a34c753fSRafael Auler       BC.InstPrinter->printInst(&Instruction, 0, "", *BC.STI, errs());
1032a34c753fSRafael Auler       errs() << '\n';
1033a34c753fSRafael Auler       Instruction.dump_pretty(errs(), BC.InstPrinter.get());
1034a34c753fSRafael Auler       errs() << '\n';
1035a34c753fSRafael Auler       errs() << "BOLT-ERROR: cannot handle PC-relative operand at 0x"
1036a34c753fSRafael Auler              << Twine::utohexstr(Address) << ". Skipping function " << *this
1037a34c753fSRafael Auler              << ".\n";
1038a34c753fSRafael Auler       if (BC.HasRelocations)
1039a34c753fSRafael Auler         exit(1);
1040a34c753fSRafael Auler       IsSimple = false;
1041a34c753fSRafael Auler       return;
1042a34c753fSRafael Auler     }
1043a34c753fSRafael Auler     if (TargetAddress == 0 && opts::Verbosity >= 1) {
1044a34c753fSRafael Auler       outs() << "BOLT-INFO: PC-relative operand is zero in function " << *this
1045a34c753fSRafael Auler              << '\n';
1046a34c753fSRafael Auler     }
1047a34c753fSRafael Auler 
1048a34c753fSRafael Auler     const MCSymbol *TargetSymbol;
1049a34c753fSRafael Auler     uint64_t TargetOffset;
1050a34c753fSRafael Auler     std::tie(TargetSymbol, TargetOffset) =
1051a34c753fSRafael Auler         BC.handleAddressRef(TargetAddress, *this, /*IsPCRel*/ true);
105240c2e0faSMaksim Panchenko     const MCExpr *Expr = MCSymbolRefExpr::create(
105340c2e0faSMaksim Panchenko         TargetSymbol, MCSymbolRefExpr::VK_None, *BC.Ctx);
1054a34c753fSRafael Auler     if (TargetOffset) {
1055a34c753fSRafael Auler       const MCConstantExpr *Offset =
1056a34c753fSRafael Auler           MCConstantExpr::create(TargetOffset, *BC.Ctx);
1057a34c753fSRafael Auler       Expr = MCBinaryExpr::createAdd(Expr, Offset, *BC.Ctx);
1058a34c753fSRafael Auler     }
105940c2e0faSMaksim Panchenko     MIB->replaceMemOperandDisp(Instruction,
106040c2e0faSMaksim Panchenko                                MCOperand::createExpr(BC.MIB->getTargetExprFor(
106140c2e0faSMaksim Panchenko                                    Instruction, Expr, *BC.Ctx, 0)));
1062a34c753fSRafael Auler   };
1063a34c753fSRafael Auler 
1064a34c753fSRafael Auler   auto handleExternalReference = [&](MCInst &Instruction, uint64_t Size,
1065a34c753fSRafael Auler                                      uint64_t Offset, uint64_t TargetAddress,
1066a34c753fSRafael Auler                                      bool &IsCall) -> MCSymbol * {
1067a34c753fSRafael Auler     const uint64_t AbsoluteInstrAddr = getAddress() + Offset;
1068a34c753fSRafael Auler     MCSymbol *TargetSymbol = nullptr;
106935efe1d8SVladislav Khmelevsky     BC.addInterproceduralReference(this, TargetAddress);
1070a34c753fSRafael Auler     if (opts::Verbosity >= 2 && !IsCall && Size == 2 && !BC.HasRelocations) {
1071a34c753fSRafael Auler       errs() << "BOLT-WARNING: relaxed tail call detected at 0x"
1072a34c753fSRafael Auler              << Twine::utohexstr(AbsoluteInstrAddr) << " in function " << *this
1073a34c753fSRafael Auler              << ". Code size will be increased.\n";
1074a34c753fSRafael Auler     }
1075a34c753fSRafael Auler 
1076a34c753fSRafael Auler     assert(!MIB->isTailCall(Instruction) &&
1077a34c753fSRafael Auler            "synthetic tail call instruction found");
1078a34c753fSRafael Auler 
1079a34c753fSRafael Auler     // This is a call regardless of the opcode.
1080a34c753fSRafael Auler     // Assign proper opcode for tail calls, so that they could be
1081a34c753fSRafael Auler     // treated as calls.
1082a34c753fSRafael Auler     if (!IsCall) {
1083a34c753fSRafael Auler       if (!MIB->convertJmpToTailCall(Instruction)) {
1084139744acSAmir Ayupov         assert(MIB->isConditionalBranch(Instruction) &&
1085139744acSAmir Ayupov                "unknown tail call instruction");
1086a34c753fSRafael Auler         if (opts::Verbosity >= 2) {
1087a34c753fSRafael Auler           errs() << "BOLT-WARNING: conditional tail call detected in "
1088a34c753fSRafael Auler                  << "function " << *this << " at 0x"
1089a34c753fSRafael Auler                  << Twine::utohexstr(AbsoluteInstrAddr) << ".\n";
1090a34c753fSRafael Auler         }
1091a34c753fSRafael Auler       }
1092a34c753fSRafael Auler       IsCall = true;
1093a34c753fSRafael Auler     }
1094a34c753fSRafael Auler 
1095a34c753fSRafael Auler     TargetSymbol = BC.getOrCreateGlobalSymbol(TargetAddress, "FUNCat");
1096a34c753fSRafael Auler     if (opts::Verbosity >= 2 && TargetAddress == 0) {
1097a34c753fSRafael Auler       // We actually see calls to address 0 in presence of weak
1098a34c753fSRafael Auler       // symbols originating from libraries. This code is never meant
1099a34c753fSRafael Auler       // to be executed.
1100a34c753fSRafael Auler       outs() << "BOLT-INFO: Function " << *this
1101a34c753fSRafael Auler              << " has a call to address zero.\n";
1102a34c753fSRafael Auler     }
1103a34c753fSRafael Auler 
1104a34c753fSRafael Auler     return TargetSymbol;
1105a34c753fSRafael Auler   };
1106a34c753fSRafael Auler 
1107a34c753fSRafael Auler   auto handleIndirectBranch = [&](MCInst &Instruction, uint64_t Size,
1108a34c753fSRafael Auler                                   uint64_t Offset) {
1109a34c753fSRafael Auler     uint64_t IndirectTarget = 0;
1110a34c753fSRafael Auler     IndirectBranchType Result =
1111a34c753fSRafael Auler         processIndirectBranch(Instruction, Size, Offset, IndirectTarget);
1112a34c753fSRafael Auler     switch (Result) {
1113a34c753fSRafael Auler     default:
1114a34c753fSRafael Auler       llvm_unreachable("unexpected result");
1115a34c753fSRafael Auler     case IndirectBranchType::POSSIBLE_TAIL_CALL: {
1116a34c753fSRafael Auler       bool Result = MIB->convertJmpToTailCall(Instruction);
1117a34c753fSRafael Auler       (void)Result;
1118a34c753fSRafael Auler       assert(Result);
1119a34c753fSRafael Auler       break;
1120a34c753fSRafael Auler     }
1121a34c753fSRafael Auler     case IndirectBranchType::POSSIBLE_JUMP_TABLE:
1122a34c753fSRafael Auler     case IndirectBranchType::POSSIBLE_PIC_JUMP_TABLE:
1123a34c753fSRafael Auler       if (opts::JumpTables == JTS_NONE)
1124a34c753fSRafael Auler         IsSimple = false;
1125a34c753fSRafael Auler       break;
1126a34c753fSRafael Auler     case IndirectBranchType::POSSIBLE_FIXED_BRANCH: {
1127a34c753fSRafael Auler       if (containsAddress(IndirectTarget)) {
1128a34c753fSRafael Auler         const MCSymbol *TargetSymbol = getOrCreateLocalLabel(IndirectTarget);
1129a34c753fSRafael Auler         Instruction.clear();
1130a34c753fSRafael Auler         MIB->createUncondBranch(Instruction, TargetSymbol, BC.Ctx.get());
1131a34c753fSRafael Auler         TakenBranches.emplace_back(Offset, IndirectTarget - getAddress());
1132a34c753fSRafael Auler         HasFixedIndirectBranch = true;
1133a34c753fSRafael Auler       } else {
1134a34c753fSRafael Auler         MIB->convertJmpToTailCall(Instruction);
113535efe1d8SVladislav Khmelevsky         BC.addInterproceduralReference(this, IndirectTarget);
1136a34c753fSRafael Auler       }
1137a34c753fSRafael Auler       break;
1138a34c753fSRafael Auler     }
1139a34c753fSRafael Auler     case IndirectBranchType::UNKNOWN:
1140a34c753fSRafael Auler       // Keep processing. We'll do more checks and fixes in
1141a34c753fSRafael Auler       // postProcessIndirectBranches().
1142a34c753fSRafael Auler       UnknownIndirectBranchOffsets.emplace(Offset);
1143a34c753fSRafael Auler       break;
1144a34c753fSRafael Auler     }
1145a34c753fSRafael Auler   };
1146a34c753fSRafael Auler 
1147a34c753fSRafael Auler   // Check for linker veneers, which lack relocations and need manual
1148a34c753fSRafael Auler   // adjustments.
1149a34c753fSRafael Auler   auto handleAArch64IndirectCall = [&](MCInst &Instruction, uint64_t Offset) {
1150a34c753fSRafael Auler     const uint64_t AbsoluteInstrAddr = getAddress() + Offset;
1151a34c753fSRafael Auler     MCInst *TargetHiBits, *TargetLowBits;
115235efe1d8SVladislav Khmelevsky     uint64_t TargetAddress, Count;
115335efe1d8SVladislav Khmelevsky     Count = MIB->matchLinkerVeneer(Instructions.begin(), Instructions.end(),
1154a34c753fSRafael Auler                                    AbsoluteInstrAddr, Instruction, TargetHiBits,
115535efe1d8SVladislav Khmelevsky                                    TargetLowBits, TargetAddress);
115635efe1d8SVladislav Khmelevsky     if (Count) {
1157a34c753fSRafael Auler       MIB->addAnnotation(Instruction, "AArch64Veneer", true);
115835efe1d8SVladislav Khmelevsky       --Count;
115935efe1d8SVladislav Khmelevsky       for (auto It = std::prev(Instructions.end()); Count != 0;
116035efe1d8SVladislav Khmelevsky            It = std::prev(It), --Count) {
1161a34c753fSRafael Auler         MIB->addAnnotation(It->second, "AArch64Veneer", true);
1162a34c753fSRafael Auler       }
1163a34c753fSRafael Auler 
116435efe1d8SVladislav Khmelevsky       BC.addAdrpAddRelocAArch64(*this, *TargetLowBits, *TargetHiBits,
116535efe1d8SVladislav Khmelevsky                                 TargetAddress);
1166a34c753fSRafael Auler     }
1167a34c753fSRafael Auler   };
1168a34c753fSRafael Auler 
1169a34c753fSRafael Auler   uint64_t Size = 0; // instruction size
1170a34c753fSRafael Auler   for (uint64_t Offset = 0; Offset < getSize(); Offset += Size) {
1171a34c753fSRafael Auler     MCInst Instruction;
1172a34c753fSRafael Auler     const uint64_t AbsoluteInstrAddr = getAddress() + Offset;
1173a34c753fSRafael Auler 
1174a34c753fSRafael Auler     // Check for data inside code and ignore it
1175a34c753fSRafael Auler     if (const size_t DataInCodeSize = getSizeOfDataInCodeAt(Offset)) {
1176a34c753fSRafael Auler       Size = DataInCodeSize;
1177a34c753fSRafael Auler       continue;
1178a34c753fSRafael Auler     }
1179a34c753fSRafael Auler 
1180e290133cSMaksim Panchenko     if (!BC.SymbolicDisAsm->getInstruction(Instruction, Size,
1181a34c753fSRafael Auler                                            FunctionData.slice(Offset),
118240c2e0faSMaksim Panchenko                                            AbsoluteInstrAddr, nulls())) {
1183a34c753fSRafael Auler       // Functions with "soft" boundaries, e.g. coming from assembly source,
1184a34c753fSRafael Auler       // can have 0-byte padding at the end.
1185a34c753fSRafael Auler       if (isZeroPaddingAt(Offset))
1186a34c753fSRafael Auler         break;
1187a34c753fSRafael Auler 
1188a34c753fSRafael Auler       errs() << "BOLT-WARNING: unable to disassemble instruction at offset 0x"
1189a34c753fSRafael Auler              << Twine::utohexstr(Offset) << " (address 0x"
119040c2e0faSMaksim Panchenko              << Twine::utohexstr(AbsoluteInstrAddr) << ") in function " << *this
119140c2e0faSMaksim Panchenko              << '\n';
1192a34c753fSRafael Auler       // Some AVX-512 instructions could not be disassembled at all.
1193a34c753fSRafael Auler       if (BC.HasRelocations && opts::TrapOnAVX512 && BC.isX86()) {
1194a34c753fSRafael Auler         setTrapOnEntry();
1195a34c753fSRafael Auler         BC.TrappedFunctions.push_back(this);
1196a34c753fSRafael Auler       } else {
1197a34c753fSRafael Auler         setIgnored();
1198a34c753fSRafael Auler       }
1199a34c753fSRafael Auler 
1200a34c753fSRafael Auler       break;
1201a34c753fSRafael Auler     }
1202a34c753fSRafael Auler 
1203a34c753fSRafael Auler     // Check integrity of LLVM assembler/disassembler.
1204a34c753fSRafael Auler     if (opts::CheckEncoding && !BC.MIB->isBranch(Instruction) &&
1205a34c753fSRafael Auler         !BC.MIB->isCall(Instruction) && !BC.MIB->isNoop(Instruction)) {
1206a34c753fSRafael Auler       if (!BC.validateEncoding(Instruction, FunctionData.slice(Offset, Size))) {
1207a34c753fSRafael Auler         errs() << "BOLT-WARNING: mismatching LLVM encoding detected in "
1208a34c753fSRafael Auler                << "function " << *this << " for instruction :\n";
1209a34c753fSRafael Auler         BC.printInstruction(errs(), Instruction, AbsoluteInstrAddr);
1210a34c753fSRafael Auler         errs() << '\n';
1211a34c753fSRafael Auler       }
1212a34c753fSRafael Auler     }
1213a34c753fSRafael Auler 
1214a34c753fSRafael Auler     // Special handling for AVX-512 instructions.
1215a34c753fSRafael Auler     if (MIB->hasEVEXEncoding(Instruction)) {
1216a34c753fSRafael Auler       if (BC.HasRelocations && opts::TrapOnAVX512) {
1217a34c753fSRafael Auler         setTrapOnEntry();
1218a34c753fSRafael Auler         BC.TrappedFunctions.push_back(this);
1219a34c753fSRafael Auler         break;
1220a34c753fSRafael Auler       }
1221a34c753fSRafael Auler 
1222e290133cSMaksim Panchenko       // Disassemble again without the symbolizer and check that the disassembly
1223e290133cSMaksim Panchenko       // matches the assembler output.
1224e290133cSMaksim Panchenko       MCInst TempInst;
1225e290133cSMaksim Panchenko       BC.DisAsm->getInstruction(TempInst, Size, FunctionData.slice(Offset),
1226e290133cSMaksim Panchenko                                 AbsoluteInstrAddr, nulls());
1227e290133cSMaksim Panchenko       if (!BC.validateEncoding(TempInst, FunctionData.slice(Offset, Size))) {
1228e290133cSMaksim Panchenko         if (opts::Verbosity >= 0) {
1229a34c753fSRafael Auler           errs() << "BOLT-WARNING: internal assembler/disassembler error "
1230a34c753fSRafael Auler                     "detected for AVX512 instruction:\n";
1231e290133cSMaksim Panchenko           BC.printInstruction(errs(), TempInst, AbsoluteInstrAddr);
1232a34c753fSRafael Auler           errs() << " in function " << *this << '\n';
1233a34c753fSRafael Auler         }
1234a34c753fSRafael Auler 
1235a34c753fSRafael Auler         setIgnored();
1236a34c753fSRafael Auler         break;
1237a34c753fSRafael Auler       }
1238a34c753fSRafael Auler     }
1239a34c753fSRafael Auler 
1240a34c753fSRafael Auler     if (MIB->isBranch(Instruction) || MIB->isCall(Instruction)) {
1241a34c753fSRafael Auler       uint64_t TargetAddress = 0;
1242a34c753fSRafael Auler       if (MIB->evaluateBranch(Instruction, AbsoluteInstrAddr, Size,
1243a34c753fSRafael Auler                               TargetAddress)) {
1244a34c753fSRafael Auler         // Check if the target is within the same function. Otherwise it's
1245a34c753fSRafael Auler         // a call, possibly a tail call.
1246a34c753fSRafael Auler         //
1247a34c753fSRafael Auler         // If the target *is* the function address it could be either a branch
1248a34c753fSRafael Auler         // or a recursive call.
1249a34c753fSRafael Auler         bool IsCall = MIB->isCall(Instruction);
1250a34c753fSRafael Auler         const bool IsCondBranch = MIB->isConditionalBranch(Instruction);
1251a34c753fSRafael Auler         MCSymbol *TargetSymbol = nullptr;
1252a34c753fSRafael Auler 
1253a34c753fSRafael Auler         if (BC.MIB->isUnsupportedBranch(Instruction.getOpcode())) {
1254a34c753fSRafael Auler           setIgnored();
1255a34c753fSRafael Auler           if (BinaryFunction *TargetFunc =
1256a34c753fSRafael Auler                   BC.getBinaryFunctionContainingAddress(TargetAddress))
1257a34c753fSRafael Auler             TargetFunc->setIgnored();
1258a34c753fSRafael Auler         }
1259a34c753fSRafael Auler 
1260a34c753fSRafael Auler         if (IsCall && containsAddress(TargetAddress)) {
1261a34c753fSRafael Auler           if (TargetAddress == getAddress()) {
1262a34c753fSRafael Auler             // Recursive call.
1263a34c753fSRafael Auler             TargetSymbol = getSymbol();
1264a34c753fSRafael Auler           } else {
1265a34c753fSRafael Auler             if (BC.isX86()) {
1266a34c753fSRafael Auler               // Dangerous old-style x86 PIC code. We may need to freeze this
1267a34c753fSRafael Auler               // function, so preserve the function as is for now.
1268a34c753fSRafael Auler               PreserveNops = true;
1269a34c753fSRafael Auler             } else {
1270a34c753fSRafael Auler               errs() << "BOLT-WARNING: internal call detected at 0x"
1271a34c753fSRafael Auler                      << Twine::utohexstr(AbsoluteInstrAddr) << " in function "
1272a34c753fSRafael Auler                      << *this << ". Skipping.\n";
1273a34c753fSRafael Auler               IsSimple = false;
1274a34c753fSRafael Auler             }
1275a34c753fSRafael Auler           }
1276a34c753fSRafael Auler         }
1277a34c753fSRafael Auler 
1278a34c753fSRafael Auler         if (!TargetSymbol) {
1279a34c753fSRafael Auler           // Create either local label or external symbol.
1280a34c753fSRafael Auler           if (containsAddress(TargetAddress)) {
1281a34c753fSRafael Auler             TargetSymbol = getOrCreateLocalLabel(TargetAddress);
1282a34c753fSRafael Auler           } else {
1283a34c753fSRafael Auler             if (TargetAddress == getAddress() + getSize() &&
128435efe1d8SVladislav Khmelevsky                 TargetAddress < getAddress() + getMaxSize() &&
128535efe1d8SVladislav Khmelevsky                 !(BC.isAArch64() &&
128635efe1d8SVladislav Khmelevsky                   BC.handleAArch64Veneer(TargetAddress, /*MatchOnly*/ true))) {
1287a34c753fSRafael Auler               // Result of __builtin_unreachable().
1288a34c753fSRafael Auler               LLVM_DEBUG(dbgs() << "BOLT-DEBUG: jump past end detected at 0x"
1289a34c753fSRafael Auler                                 << Twine::utohexstr(AbsoluteInstrAddr)
1290a34c753fSRafael Auler                                 << " in function " << *this
1291a34c753fSRafael Auler                                 << " : replacing with nop.\n");
1292a34c753fSRafael Auler               BC.MIB->createNoop(Instruction);
1293a34c753fSRafael Auler               if (IsCondBranch) {
1294a34c753fSRafael Auler                 // Register branch offset for profile validation.
1295a34c753fSRafael Auler                 IgnoredBranches.emplace_back(Offset, Offset + Size);
1296a34c753fSRafael Auler               }
1297a34c753fSRafael Auler               goto add_instruction;
1298a34c753fSRafael Auler             }
1299a34c753fSRafael Auler             // May update Instruction and IsCall
1300a34c753fSRafael Auler             TargetSymbol = handleExternalReference(Instruction, Size, Offset,
1301a34c753fSRafael Auler                                                    TargetAddress, IsCall);
1302a34c753fSRafael Auler           }
1303a34c753fSRafael Auler         }
1304a34c753fSRafael Auler 
1305a34c753fSRafael Auler         if (!IsCall) {
1306a34c753fSRafael Auler           // Add taken branch info.
1307a34c753fSRafael Auler           TakenBranches.emplace_back(Offset, TargetAddress - getAddress());
1308a34c753fSRafael Auler         }
1309a34c753fSRafael Auler         BC.MIB->replaceBranchTarget(Instruction, TargetSymbol, &*Ctx);
1310a34c753fSRafael Auler 
1311a34c753fSRafael Auler         // Mark CTC.
13123652483cSRafael Auler         if (IsCondBranch && IsCall)
1313a34c753fSRafael Auler           MIB->setConditionalTailCall(Instruction, TargetAddress);
1314a34c753fSRafael Auler       } else {
1315a34c753fSRafael Auler         // Could not evaluate branch. Should be an indirect call or an
1316a34c753fSRafael Auler         // indirect branch. Bail out on the latter case.
1317a34c753fSRafael Auler         if (MIB->isIndirectBranch(Instruction))
1318a34c753fSRafael Auler           handleIndirectBranch(Instruction, Size, Offset);
1319a34c753fSRafael Auler         // Indirect call. We only need to fix it if the operand is RIP-relative.
1320a34c753fSRafael Auler         if (IsSimple && MIB->hasPCRelOperand(Instruction))
1321a34c753fSRafael Auler           handlePCRelOperand(Instruction, AbsoluteInstrAddr, Size);
1322a34c753fSRafael Auler 
1323a34c753fSRafael Auler         if (BC.isAArch64())
1324a34c753fSRafael Auler           handleAArch64IndirectCall(Instruction, Offset);
1325a34c753fSRafael Auler       }
1326e290133cSMaksim Panchenko     } else if (BC.isAArch64()) {
13274101aa13SMaksim Panchenko       // Check if there's a relocation associated with this instruction.
13284101aa13SMaksim Panchenko       bool UsedReloc = false;
13294101aa13SMaksim Panchenko       for (auto Itr = Relocations.lower_bound(Offset),
13304101aa13SMaksim Panchenko                 ItrE = Relocations.lower_bound(Offset + Size);
13314101aa13SMaksim Panchenko            Itr != ItrE; ++Itr) {
13324101aa13SMaksim Panchenko         const Relocation &Relocation = Itr->second;
13334101aa13SMaksim Panchenko         int64_t Value = Relocation.Value;
13344101aa13SMaksim Panchenko         const bool Result = BC.MIB->replaceImmWithSymbolRef(
1335e290133cSMaksim Panchenko             Instruction, Relocation.Symbol, Relocation.Addend, Ctx.get(), Value,
1336e290133cSMaksim Panchenko             Relocation.Type);
13374101aa13SMaksim Panchenko         (void)Result;
13384101aa13SMaksim Panchenko         assert(Result && "cannot replace immediate with relocation");
13394101aa13SMaksim Panchenko 
1340e290133cSMaksim Panchenko         // For aarch64, if we replaced an immediate with a symbol from a
13414101aa13SMaksim Panchenko         // relocation, we mark it so we do not try to further process a
13424101aa13SMaksim Panchenko         // pc-relative operand. All we need is the symbol.
13434101aa13SMaksim Panchenko         UsedReloc = true;
13444101aa13SMaksim Panchenko       }
13454101aa13SMaksim Panchenko 
13464101aa13SMaksim Panchenko       if (MIB->hasPCRelOperand(Instruction) && !UsedReloc)
1347a34c753fSRafael Auler         handlePCRelOperand(Instruction, AbsoluteInstrAddr, Size);
13483652483cSRafael Auler     }
1349a34c753fSRafael Auler 
1350a34c753fSRafael Auler add_instruction:
1351a34c753fSRafael Auler     if (getDWARFLineTable()) {
135240c2e0faSMaksim Panchenko       Instruction.setLoc(findDebugLineInformationForInstructionAt(
135340c2e0faSMaksim Panchenko           AbsoluteInstrAddr, getDWARFUnit(), getDWARFLineTable()));
1354a34c753fSRafael Auler     }
1355a34c753fSRafael Auler 
1356a34c753fSRafael Auler     // Record offset of the instruction for profile matching.
13573652483cSRafael Auler     if (BC.keepOffsetForInstruction(Instruction))
1358a9cd49d5SAmir Ayupov       MIB->setOffset(Instruction, static_cast<uint32_t>(Offset));
1359a34c753fSRafael Auler 
136008f56926SVladislav Khmelevsky     if (BC.MIB->isNoop(Instruction)) {
136108f56926SVladislav Khmelevsky       // NOTE: disassembly loses the correct size information for noops.
136208f56926SVladislav Khmelevsky       //       E.g. nopw 0x0(%rax,%rax,1) is 9 bytes, but re-encoded it's only
136308f56926SVladislav Khmelevsky       //       5 bytes. Preserve the size info using annotations.
136408f56926SVladislav Khmelevsky       MIB->addAnnotation(Instruction, "Size", static_cast<uint32_t>(Size));
136508f56926SVladislav Khmelevsky     }
136608f56926SVladislav Khmelevsky 
1367a34c753fSRafael Auler     addInstruction(Offset, std::move(Instruction));
1368a34c753fSRafael Auler   }
1369a34c753fSRafael Auler 
1370e290133cSMaksim Panchenko   // Reset symbolizer for the disassembler.
1371e290133cSMaksim Panchenko   BC.SymbolicDisAsm->setSymbolizer(nullptr);
1372e290133cSMaksim Panchenko 
13730b7e8bafSDenis Revunov   if (uint64_t Offset = getFirstInstructionOffset())
13740b7e8bafSDenis Revunov     Labels[Offset] = BC.Ctx->createNamedTempSymbol();
13750b7e8bafSDenis Revunov 
1376a34c753fSRafael Auler   clearList(Relocations);
1377a34c753fSRafael Auler 
1378a34c753fSRafael Auler   if (!IsSimple) {
1379a34c753fSRafael Auler     clearList(Instructions);
1380a34c753fSRafael Auler     return false;
1381a34c753fSRafael Auler   }
1382a34c753fSRafael Auler 
1383a34c753fSRafael Auler   updateState(State::Disassembled);
1384a34c753fSRafael Auler 
1385a34c753fSRafael Auler   return true;
1386a34c753fSRafael Auler }
1387a34c753fSRafael Auler 
1388a34c753fSRafael Auler bool BinaryFunction::scanExternalRefs() {
1389a34c753fSRafael Auler   bool Success = true;
1390a34c753fSRafael Auler   bool DisassemblyFailed = false;
1391a34c753fSRafael Auler 
1392a34c753fSRafael Auler   // Ignore pseudo functions.
1393a34c753fSRafael Auler   if (isPseudo())
1394a34c753fSRafael Auler     return Success;
1395a34c753fSRafael Auler 
1396a34c753fSRafael Auler   if (opts::NoScan) {
1397a34c753fSRafael Auler     clearList(Relocations);
1398a34c753fSRafael Auler     clearList(ExternallyReferencedOffsets);
1399a34c753fSRafael Auler 
1400a34c753fSRafael Auler     return false;
1401a34c753fSRafael Auler   }
1402a34c753fSRafael Auler 
1403a34c753fSRafael Auler   // List of external references for this function.
1404a34c753fSRafael Auler   std::vector<Relocation> FunctionRelocations;
1405a34c753fSRafael Auler 
1406a34c753fSRafael Auler   static BinaryContext::IndependentCodeEmitter Emitter =
1407a34c753fSRafael Auler       BC.createIndependentMCCodeEmitter();
1408a34c753fSRafael Auler 
1409a34c753fSRafael Auler   ErrorOr<ArrayRef<uint8_t>> ErrorOrFunctionData = getData();
1410a34c753fSRafael Auler   assert(ErrorOrFunctionData && "function data is not available");
1411a34c753fSRafael Auler   ArrayRef<uint8_t> FunctionData = *ErrorOrFunctionData;
1412a34c753fSRafael Auler   assert(FunctionData.size() == getMaxSize() &&
1413a34c753fSRafael Auler          "function size does not match raw data size");
1414a34c753fSRafael Auler 
1415a34c753fSRafael Auler   uint64_t Size = 0; // instruction size
1416a34c753fSRafael Auler   for (uint64_t Offset = 0; Offset < getSize(); Offset += Size) {
1417a34c753fSRafael Auler     // Check for data inside code and ignore it
1418a34c753fSRafael Auler     if (const size_t DataInCodeSize = getSizeOfDataInCodeAt(Offset)) {
1419a34c753fSRafael Auler       Size = DataInCodeSize;
1420a34c753fSRafael Auler       continue;
1421a34c753fSRafael Auler     }
1422a34c753fSRafael Auler 
1423a34c753fSRafael Auler     const uint64_t AbsoluteInstrAddr = getAddress() + Offset;
1424a34c753fSRafael Auler     MCInst Instruction;
142540c2e0faSMaksim Panchenko     if (!BC.DisAsm->getInstruction(Instruction, Size,
1426a34c753fSRafael Auler                                    FunctionData.slice(Offset),
142740c2e0faSMaksim Panchenko                                    AbsoluteInstrAddr, nulls())) {
1428a34c753fSRafael Auler       if (opts::Verbosity >= 1 && !isZeroPaddingAt(Offset)) {
1429a34c753fSRafael Auler         errs() << "BOLT-WARNING: unable to disassemble instruction at offset 0x"
1430a34c753fSRafael Auler                << Twine::utohexstr(Offset) << " (address 0x"
1431a34c753fSRafael Auler                << Twine::utohexstr(AbsoluteInstrAddr) << ") in function "
1432a34c753fSRafael Auler                << *this << '\n';
1433a34c753fSRafael Auler       }
1434a34c753fSRafael Auler       Success = false;
1435a34c753fSRafael Auler       DisassemblyFailed = true;
1436a34c753fSRafael Auler       break;
1437a34c753fSRafael Auler     }
1438a34c753fSRafael Auler 
1439a34c753fSRafael Auler     // Return true if we can skip handling the Target function reference.
1440a34c753fSRafael Auler     auto ignoreFunctionRef = [&](const BinaryFunction &Target) {
1441a34c753fSRafael Auler       if (&Target == this)
1442a34c753fSRafael Auler         return true;
1443a34c753fSRafael Auler 
1444a34c753fSRafael Auler       // Note that later we may decide not to emit Target function. In that
1445a34c753fSRafael Auler       // case, we conservatively create references that will be ignored or
1446a34c753fSRafael Auler       // resolved to the same function.
1447a34c753fSRafael Auler       if (!BC.shouldEmit(Target))
1448a34c753fSRafael Auler         return true;
1449a34c753fSRafael Auler 
1450a34c753fSRafael Auler       return false;
1451a34c753fSRafael Auler     };
1452a34c753fSRafael Auler 
1453a34c753fSRafael Auler     // Return true if we can ignore reference to the symbol.
1454a34c753fSRafael Auler     auto ignoreReference = [&](const MCSymbol *TargetSymbol) {
1455a34c753fSRafael Auler       if (!TargetSymbol)
1456a34c753fSRafael Auler         return true;
1457a34c753fSRafael Auler 
1458a34c753fSRafael Auler       if (BC.forceSymbolRelocations(TargetSymbol->getName()))
1459a34c753fSRafael Auler         return false;
1460a34c753fSRafael Auler 
1461a34c753fSRafael Auler       BinaryFunction *TargetFunction = BC.getFunctionForSymbol(TargetSymbol);
1462a34c753fSRafael Auler       if (!TargetFunction)
1463a34c753fSRafael Auler         return true;
1464a34c753fSRafael Auler 
1465a34c753fSRafael Auler       return ignoreFunctionRef(*TargetFunction);
1466a34c753fSRafael Auler     };
1467a34c753fSRafael Auler 
1468a34c753fSRafael Auler     // Detect if the instruction references an address.
1469a34c753fSRafael Auler     // Without relocations, we can only trust PC-relative address modes.
1470a34c753fSRafael Auler     uint64_t TargetAddress = 0;
1471a34c753fSRafael Auler     bool IsPCRel = false;
1472a34c753fSRafael Auler     bool IsBranch = false;
1473a34c753fSRafael Auler     if (BC.MIB->hasPCRelOperand(Instruction)) {
1474a34c753fSRafael Auler       if (BC.MIB->evaluateMemOperandTarget(Instruction, TargetAddress,
1475a34c753fSRafael Auler                                            AbsoluteInstrAddr, Size)) {
1476a34c753fSRafael Auler         IsPCRel = true;
1477a34c753fSRafael Auler       }
1478a34c753fSRafael Auler     } else if (BC.MIB->isCall(Instruction) || BC.MIB->isBranch(Instruction)) {
1479a34c753fSRafael Auler       if (BC.MIB->evaluateBranch(Instruction, AbsoluteInstrAddr, Size,
1480a34c753fSRafael Auler                                  TargetAddress)) {
1481a34c753fSRafael Auler         IsBranch = true;
1482a34c753fSRafael Auler       }
1483a34c753fSRafael Auler     }
1484a34c753fSRafael Auler 
1485a34c753fSRafael Auler     MCSymbol *TargetSymbol = nullptr;
1486a34c753fSRafael Auler 
1487a34c753fSRafael Auler     // Create an entry point at reference address if needed.
1488a34c753fSRafael Auler     BinaryFunction *TargetFunction =
1489a34c753fSRafael Auler         BC.getBinaryFunctionContainingAddress(TargetAddress);
1490a34c753fSRafael Auler     if (TargetFunction && !ignoreFunctionRef(*TargetFunction)) {
1491a34c753fSRafael Auler       const uint64_t FunctionOffset =
1492a34c753fSRafael Auler           TargetAddress - TargetFunction->getAddress();
1493a34c753fSRafael Auler       TargetSymbol = FunctionOffset
1494a34c753fSRafael Auler                          ? TargetFunction->addEntryPointAtOffset(FunctionOffset)
1495a34c753fSRafael Auler                          : TargetFunction->getSymbol();
1496a34c753fSRafael Auler     }
1497a34c753fSRafael Auler 
1498a34c753fSRafael Auler     // Can't find more references and not creating relocations.
1499a34c753fSRafael Auler     if (!BC.HasRelocations)
1500a34c753fSRafael Auler       continue;
1501a34c753fSRafael Auler 
1502a34c753fSRafael Auler     // Create a relocation against the TargetSymbol as the symbol might get
1503a34c753fSRafael Auler     // moved.
1504a34c753fSRafael Auler     if (TargetSymbol) {
1505a34c753fSRafael Auler       if (IsBranch) {
1506a34c753fSRafael Auler         BC.MIB->replaceBranchTarget(Instruction, TargetSymbol,
1507a34c753fSRafael Auler                                     Emitter.LocalCtx.get());
1508a34c753fSRafael Auler       } else if (IsPCRel) {
150940c2e0faSMaksim Panchenko         const MCExpr *Expr = MCSymbolRefExpr::create(
151040c2e0faSMaksim Panchenko             TargetSymbol, MCSymbolRefExpr::VK_None, *Emitter.LocalCtx.get());
1511a34c753fSRafael Auler         BC.MIB->replaceMemOperandDisp(
1512a34c753fSRafael Auler             Instruction, MCOperand::createExpr(BC.MIB->getTargetExprFor(
151340c2e0faSMaksim Panchenko                              Instruction, Expr, *Emitter.LocalCtx.get(), 0)));
1514a34c753fSRafael Auler       }
1515a34c753fSRafael Auler     }
1516a34c753fSRafael Auler 
1517a34c753fSRafael Auler     // Create more relocations based on input file relocations.
1518a34c753fSRafael Auler     bool HasRel = false;
1519a34c753fSRafael Auler     for (auto Itr = Relocations.lower_bound(Offset),
152040c2e0faSMaksim Panchenko               ItrE = Relocations.lower_bound(Offset + Size);
152140c2e0faSMaksim Panchenko          Itr != ItrE; ++Itr) {
1522a34c753fSRafael Auler       Relocation &Relocation = Itr->second;
15234101aa13SMaksim Panchenko       if (Relocation.isPCRelative() && BC.isX86())
15244101aa13SMaksim Panchenko         continue;
1525a34c753fSRafael Auler       if (ignoreReference(Relocation.Symbol))
1526a34c753fSRafael Auler         continue;
1527a34c753fSRafael Auler 
1528a34c753fSRafael Auler       int64_t Value = Relocation.Value;
152940c2e0faSMaksim Panchenko       const bool Result = BC.MIB->replaceImmWithSymbolRef(
153040c2e0faSMaksim Panchenko           Instruction, Relocation.Symbol, Relocation.Addend,
153140c2e0faSMaksim Panchenko           Emitter.LocalCtx.get(), Value, Relocation.Type);
1532a34c753fSRafael Auler       (void)Result;
1533a34c753fSRafael Auler       assert(Result && "cannot replace immediate with relocation");
1534a34c753fSRafael Auler 
1535a34c753fSRafael Auler       HasRel = true;
1536a34c753fSRafael Auler     }
1537a34c753fSRafael Auler 
1538a34c753fSRafael Auler     if (!TargetSymbol && !HasRel)
1539a34c753fSRafael Auler       continue;
1540a34c753fSRafael Auler 
1541a34c753fSRafael Auler     // Emit the instruction using temp emitter and generate relocations.
1542a34c753fSRafael Auler     SmallString<256> Code;
1543a34c753fSRafael Auler     SmallVector<MCFixup, 4> Fixups;
1544a34c753fSRafael Auler     raw_svector_ostream VecOS(Code);
1545a34c753fSRafael Auler     Emitter.MCE->encodeInstruction(Instruction, VecOS, Fixups, *BC.STI);
1546a34c753fSRafael Auler 
1547a34c753fSRafael Auler     // Create relocation for every fixup.
1548a34c753fSRafael Auler     for (const MCFixup &Fixup : Fixups) {
1549a34c753fSRafael Auler       Optional<Relocation> Rel = BC.MIB->createRelocation(Fixup, *BC.MAB);
1550a34c753fSRafael Auler       if (!Rel) {
1551a34c753fSRafael Auler         Success = false;
1552a34c753fSRafael Auler         continue;
1553a34c753fSRafael Auler       }
1554a34c753fSRafael Auler 
1555a34c753fSRafael Auler       if (Relocation::getSizeForType(Rel->Type) < 4) {
1556a34c753fSRafael Auler         // If the instruction uses a short form, then we might not be able
1557a34c753fSRafael Auler         // to handle the rewrite without relaxation, and hence cannot reliably
1558a34c753fSRafael Auler         // create an external reference relocation.
1559a34c753fSRafael Auler         Success = false;
1560a34c753fSRafael Auler         continue;
1561a34c753fSRafael Auler       }
1562a34c753fSRafael Auler       Rel->Offset += getAddress() - getOriginSection()->getAddress() + Offset;
1563a34c753fSRafael Auler       FunctionRelocations.push_back(*Rel);
1564a34c753fSRafael Auler     }
1565a34c753fSRafael Auler 
1566a34c753fSRafael Auler     if (!Success)
1567a34c753fSRafael Auler       break;
1568a34c753fSRafael Auler   }
1569a34c753fSRafael Auler 
1570a34c753fSRafael Auler   // Add relocations unless disassembly failed for this function.
15713652483cSRafael Auler   if (!DisassemblyFailed)
15723652483cSRafael Auler     for (Relocation &Rel : FunctionRelocations)
1573a34c753fSRafael Auler       getOriginSection()->addPendingRelocation(Rel);
1574a34c753fSRafael Auler 
1575a34c753fSRafael Auler   // Inform BinaryContext that this function symbols will not be defined and
1576a34c753fSRafael Auler   // relocations should not be created against them.
1577a34c753fSRafael Auler   if (BC.HasRelocations) {
15783652483cSRafael Auler     for (std::pair<const uint32_t, MCSymbol *> &LI : Labels)
1579a34c753fSRafael Auler       BC.UndefinedSymbols.insert(LI.second);
15803652483cSRafael Auler     if (FunctionEndLabel)
1581a34c753fSRafael Auler       BC.UndefinedSymbols.insert(FunctionEndLabel);
1582a34c753fSRafael Auler   }
1583a34c753fSRafael Auler 
1584a34c753fSRafael Auler   clearList(Relocations);
1585a34c753fSRafael Auler   clearList(ExternallyReferencedOffsets);
1586a34c753fSRafael Auler 
15873652483cSRafael Auler   if (Success && BC.HasRelocations)
1588a34c753fSRafael Auler     HasExternalRefRelocations = true;
1589a34c753fSRafael Auler 
15903652483cSRafael Auler   if (opts::Verbosity >= 1 && !Success)
1591a34c753fSRafael Auler     outs() << "BOLT-INFO: failed to scan refs for  " << *this << '\n';
1592a34c753fSRafael Auler 
1593a34c753fSRafael Auler   return Success;
1594a34c753fSRafael Auler }
1595a34c753fSRafael Auler 
1596a34c753fSRafael Auler void BinaryFunction::postProcessEntryPoints() {
1597a34c753fSRafael Auler   if (!isSimple())
1598a34c753fSRafael Auler     return;
1599a34c753fSRafael Auler 
1600a34c753fSRafael Auler   for (auto &KV : Labels) {
1601a34c753fSRafael Auler     MCSymbol *Label = KV.second;
1602a34c753fSRafael Auler     if (!getSecondaryEntryPointSymbol(Label))
1603a34c753fSRafael Auler       continue;
1604a34c753fSRafael Auler 
1605a34c753fSRafael Auler     // In non-relocation mode there's potentially an external undetectable
1606a34c753fSRafael Auler     // reference to the entry point and hence we cannot move this entry
1607a34c753fSRafael Auler     // point. Optimizing without moving could be difficult.
1608a34c753fSRafael Auler     if (!BC.HasRelocations)
1609a34c753fSRafael Auler       setSimple(false);
1610a34c753fSRafael Auler 
1611a34c753fSRafael Auler     const uint32_t Offset = KV.first;
1612a34c753fSRafael Auler 
1613a34c753fSRafael Auler     // If we are at Offset 0 and there is no instruction associated with it,
1614a34c753fSRafael Auler     // this means this is an empty function. Just ignore. If we find an
1615a34c753fSRafael Auler     // instruction at this offset, this entry point is valid.
16163652483cSRafael Auler     if (!Offset || getInstructionAtOffset(Offset))
1617a34c753fSRafael Auler       continue;
1618a34c753fSRafael Auler 
1619a34c753fSRafael Auler     // On AArch64 there are legitimate reasons to have references past the
1620a34c753fSRafael Auler     // end of the function, e.g. jump tables.
16213652483cSRafael Auler     if (BC.isAArch64() && Offset == getSize())
1622a34c753fSRafael Auler       continue;
1623a34c753fSRafael Auler 
1624a34c753fSRafael Auler     errs() << "BOLT-WARNING: reference in the middle of instruction "
162540c2e0faSMaksim Panchenko               "detected in function "
162640c2e0faSMaksim Panchenko            << *this << " at offset 0x" << Twine::utohexstr(Offset) << '\n';
16273652483cSRafael Auler     if (BC.HasRelocations)
1628a34c753fSRafael Auler       setIgnored();
1629a34c753fSRafael Auler     setSimple(false);
1630a34c753fSRafael Auler     return;
1631a34c753fSRafael Auler   }
1632a34c753fSRafael Auler }
1633a34c753fSRafael Auler 
1634a34c753fSRafael Auler void BinaryFunction::postProcessJumpTables() {
1635a34c753fSRafael Auler   // Create labels for all entries.
1636a34c753fSRafael Auler   for (auto &JTI : JumpTables) {
1637a34c753fSRafael Auler     JumpTable &JT = *JTI.second;
1638a34c753fSRafael Auler     if (JT.Type == JumpTable::JTT_PIC && opts::JumpTables == JTS_BASIC) {
1639a34c753fSRafael Auler       opts::JumpTables = JTS_MOVE;
1640a34c753fSRafael Auler       outs() << "BOLT-INFO: forcing -jump-tables=move as PIC jump table was "
164140c2e0faSMaksim Panchenko                 "detected in function "
164240c2e0faSMaksim Panchenko              << *this << '\n';
1643a34c753fSRafael Auler     }
164428b1dcb1SHuan Nguyen     if (JT.Entries.empty()) {
1645*05523dc3SHuan Nguyen       bool HasOneParent = (JT.Parents.size() == 1);
1646*05523dc3SHuan Nguyen       for (unsigned I = 0; I < JT.EntriesAsAddress.size(); ++I) {
1647*05523dc3SHuan Nguyen         uint64_t EntryAddress = JT.EntriesAsAddress[I];
1648*05523dc3SHuan Nguyen         // builtin_unreachable does not belong to any function
1649*05523dc3SHuan Nguyen         // Need to handle separately
1650*05523dc3SHuan Nguyen         bool IsBuiltIn = false;
1651*05523dc3SHuan Nguyen         for (BinaryFunction *Parent : JT.Parents) {
1652*05523dc3SHuan Nguyen           if (EntryAddress == Parent->getAddress() + Parent->getSize()) {
1653*05523dc3SHuan Nguyen             IsBuiltIn = true;
1654*05523dc3SHuan Nguyen             // Specify second parameter as true to accept builtin_unreachable
1655*05523dc3SHuan Nguyen             MCSymbol *Label = getOrCreateLocalLabel(EntryAddress, true);
1656a34c753fSRafael Auler             JT.Entries.push_back(Label);
1657*05523dc3SHuan Nguyen             break;
1658*05523dc3SHuan Nguyen           }
1659*05523dc3SHuan Nguyen         }
1660*05523dc3SHuan Nguyen         if (IsBuiltIn)
1661*05523dc3SHuan Nguyen           continue;
1662*05523dc3SHuan Nguyen         // Create local label for targets cannot be reached by other fragments
1663*05523dc3SHuan Nguyen         // Otherwise, secondary entry point to target function
1664*05523dc3SHuan Nguyen         BinaryFunction *TargetBF =
1665*05523dc3SHuan Nguyen             BC.getBinaryFunctionContainingAddress(EntryAddress);
1666*05523dc3SHuan Nguyen         if (TargetBF->getAddress() != EntryAddress) {
1667*05523dc3SHuan Nguyen           MCSymbol *Label =
1668*05523dc3SHuan Nguyen               (HasOneParent && TargetBF == this)
1669*05523dc3SHuan Nguyen                   ? getOrCreateLocalLabel(JT.EntriesAsAddress[I], true)
1670*05523dc3SHuan Nguyen                   : TargetBF->addEntryPointAtOffset(EntryAddress -
1671*05523dc3SHuan Nguyen                                                     TargetBF->getAddress());
1672*05523dc3SHuan Nguyen           JT.Entries.push_back(Label);
1673*05523dc3SHuan Nguyen         }
1674a34c753fSRafael Auler       }
167528b1dcb1SHuan Nguyen     }
1676a34c753fSRafael Auler 
1677a34c753fSRafael Auler     const uint64_t BDSize =
1678a34c753fSRafael Auler         BC.getBinaryDataAtAddress(JT.getAddress())->getSize();
1679a34c753fSRafael Auler     if (!BDSize) {
1680a34c753fSRafael Auler       BC.setBinaryDataSize(JT.getAddress(), JT.getSize());
1681a34c753fSRafael Auler     } else {
1682a34c753fSRafael Auler       assert(BDSize >= JT.getSize() &&
1683a34c753fSRafael Auler              "jump table cannot be larger than the containing object");
1684a34c753fSRafael Auler     }
1685a34c753fSRafael Auler   }
1686a34c753fSRafael Auler 
1687a34c753fSRafael Auler   // Add TakenBranches from JumpTables.
1688a34c753fSRafael Auler   //
1689a34c753fSRafael Auler   // We want to do it after initial processing since we don't know jump tables'
1690a34c753fSRafael Auler   // boundaries until we process them all.
1691a34c753fSRafael Auler   for (auto &JTSite : JTSites) {
1692a34c753fSRafael Auler     const uint64_t JTSiteOffset = JTSite.first;
1693a34c753fSRafael Auler     const uint64_t JTAddress = JTSite.second;
1694a34c753fSRafael Auler     const JumpTable *JT = getJumpTableContainingAddress(JTAddress);
1695a34c753fSRafael Auler     assert(JT && "cannot find jump table for address");
1696a34c753fSRafael Auler 
1697a34c753fSRafael Auler     uint64_t EntryOffset = JTAddress - JT->getAddress();
1698a34c753fSRafael Auler     while (EntryOffset < JT->getSize()) {
1699*05523dc3SHuan Nguyen       uint64_t EntryAddress = JT->EntriesAsAddress[EntryOffset / JT->EntrySize];
1700*05523dc3SHuan Nguyen       uint64_t TargetOffset = EntryAddress - getAddress();
1701a34c753fSRafael Auler       if (TargetOffset < getSize()) {
1702a34c753fSRafael Auler         TakenBranches.emplace_back(JTSiteOffset, TargetOffset);
1703a34c753fSRafael Auler 
1704a34c753fSRafael Auler         if (opts::StrictMode)
1705a34c753fSRafael Auler           registerReferencedOffset(TargetOffset);
1706a34c753fSRafael Auler       }
1707a34c753fSRafael Auler 
1708a34c753fSRafael Auler       EntryOffset += JT->EntrySize;
1709a34c753fSRafael Auler 
1710a34c753fSRafael Auler       // A label at the next entry means the end of this jump table.
1711a34c753fSRafael Auler       if (JT->Labels.count(EntryOffset))
1712a34c753fSRafael Auler         break;
1713a34c753fSRafael Auler     }
1714a34c753fSRafael Auler   }
1715a34c753fSRafael Auler   clearList(JTSites);
1716a34c753fSRafael Auler 
1717a34c753fSRafael Auler   // Conservatively populate all possible destinations for unknown indirect
1718a34c753fSRafael Auler   // branches.
1719a34c753fSRafael Auler   if (opts::StrictMode && hasInternalReference()) {
1720a34c753fSRafael Auler     for (uint64_t Offset : UnknownIndirectBranchOffsets) {
1721a34c753fSRafael Auler       for (uint64_t PossibleDestination : ExternallyReferencedOffsets) {
1722a34c753fSRafael Auler         // Ignore __builtin_unreachable().
1723a34c753fSRafael Auler         if (PossibleDestination == getSize())
1724a34c753fSRafael Auler           continue;
1725a34c753fSRafael Auler         TakenBranches.emplace_back(Offset, PossibleDestination);
1726a34c753fSRafael Auler       }
1727a34c753fSRafael Auler     }
1728a34c753fSRafael Auler   }
1729a34c753fSRafael Auler 
1730a34c753fSRafael Auler   // Remove duplicates branches. We can get a bunch of them from jump tables.
1731a34c753fSRafael Auler   // Without doing jump table value profiling we don't have use for extra
1732a34c753fSRafael Auler   // (duplicate) branches.
1733d2c87699SAmir Ayupov   llvm::sort(TakenBranches);
1734a34c753fSRafael Auler   auto NewEnd = std::unique(TakenBranches.begin(), TakenBranches.end());
1735a34c753fSRafael Auler   TakenBranches.erase(NewEnd, TakenBranches.end());
1736a34c753fSRafael Auler }
1737a34c753fSRafael Auler 
1738a34c753fSRafael Auler bool BinaryFunction::postProcessIndirectBranches(
1739a34c753fSRafael Auler     MCPlusBuilder::AllocatorIdTy AllocId) {
1740a34c753fSRafael Auler   auto addUnknownControlFlow = [&](BinaryBasicBlock &BB) {
1741a34c753fSRafael Auler     HasUnknownControlFlow = true;
1742a34c753fSRafael Auler     BB.removeAllSuccessors();
17433652483cSRafael Auler     for (uint64_t PossibleDestination : ExternallyReferencedOffsets)
1744a34c753fSRafael Auler       if (BinaryBasicBlock *SuccBB = getBasicBlockAtOffset(PossibleDestination))
1745a34c753fSRafael Auler         BB.addSuccessor(SuccBB);
1746a34c753fSRafael Auler   };
1747a34c753fSRafael Auler 
1748a34c753fSRafael Auler   uint64_t NumIndirectJumps = 0;
1749a34c753fSRafael Auler   MCInst *LastIndirectJump = nullptr;
1750a34c753fSRafael Auler   BinaryBasicBlock *LastIndirectJumpBB = nullptr;
1751a34c753fSRafael Auler   uint64_t LastJT = 0;
1752a34c753fSRafael Auler   uint16_t LastJTIndexReg = BC.MIB->getNoRegister();
1753a34c753fSRafael Auler   for (BinaryBasicBlock *BB : layout()) {
1754a34c753fSRafael Auler     for (MCInst &Instr : *BB) {
1755a34c753fSRafael Auler       if (!BC.MIB->isIndirectBranch(Instr))
1756a34c753fSRafael Auler         continue;
1757a34c753fSRafael Auler 
1758a34c753fSRafael Auler       // If there's an indirect branch in a single-block function -
1759a34c753fSRafael Auler       // it must be a tail call.
1760a34c753fSRafael Auler       if (layout_size() == 1) {
1761a34c753fSRafael Auler         BC.MIB->convertJmpToTailCall(Instr);
1762a34c753fSRafael Auler         return true;
1763a34c753fSRafael Auler       }
1764a34c753fSRafael Auler 
1765a34c753fSRafael Auler       ++NumIndirectJumps;
1766a34c753fSRafael Auler 
1767a34c753fSRafael Auler       if (opts::StrictMode && !hasInternalReference()) {
1768a34c753fSRafael Auler         BC.MIB->convertJmpToTailCall(Instr);
1769a34c753fSRafael Auler         break;
1770a34c753fSRafael Auler       }
1771a34c753fSRafael Auler 
1772a34c753fSRafael Auler       // Validate the tail call or jump table assumptions now that we know
1773a34c753fSRafael Auler       // basic block boundaries.
1774a34c753fSRafael Auler       if (BC.MIB->isTailCall(Instr) || BC.MIB->getJumpTable(Instr)) {
1775a34c753fSRafael Auler         const unsigned PtrSize = BC.AsmInfo->getCodePointerSize();
1776a34c753fSRafael Auler         MCInst *MemLocInstr;
1777a34c753fSRafael Auler         unsigned BaseRegNum, IndexRegNum;
1778a34c753fSRafael Auler         int64_t DispValue;
1779a34c753fSRafael Auler         const MCExpr *DispExpr;
1780a34c753fSRafael Auler         MCInst *PCRelBaseInstr;
1781a34c753fSRafael Auler         IndirectBranchType Type = BC.MIB->analyzeIndirectBranch(
1782a34c753fSRafael Auler             Instr, BB->begin(), BB->end(), PtrSize, MemLocInstr, BaseRegNum,
1783a34c753fSRafael Auler             IndexRegNum, DispValue, DispExpr, PCRelBaseInstr);
1784a34c753fSRafael Auler         if (Type != IndirectBranchType::UNKNOWN || MemLocInstr != nullptr)
1785a34c753fSRafael Auler           continue;
1786a34c753fSRafael Auler 
1787a34c753fSRafael Auler         if (!opts::StrictMode)
1788a34c753fSRafael Auler           return false;
1789a34c753fSRafael Auler 
1790a34c753fSRafael Auler         if (BC.MIB->isTailCall(Instr)) {
1791a34c753fSRafael Auler           BC.MIB->convertTailCallToJmp(Instr);
1792a34c753fSRafael Auler         } else {
1793a34c753fSRafael Auler           LastIndirectJump = &Instr;
1794a34c753fSRafael Auler           LastIndirectJumpBB = BB;
1795a34c753fSRafael Auler           LastJT = BC.MIB->getJumpTable(Instr);
1796a34c753fSRafael Auler           LastJTIndexReg = BC.MIB->getJumpTableIndexReg(Instr);
1797a34c753fSRafael Auler           BC.MIB->unsetJumpTable(Instr);
1798a34c753fSRafael Auler 
1799a34c753fSRafael Auler           JumpTable *JT = BC.getJumpTableContainingAddress(LastJT);
1800a34c753fSRafael Auler           if (JT->Type == JumpTable::JTT_NORMAL) {
1801a34c753fSRafael Auler             // Invalidating the jump table may also invalidate other jump table
1802a34c753fSRafael Auler             // boundaries. Until we have/need a support for this, mark the
1803a34c753fSRafael Auler             // function as non-simple.
1804a34c753fSRafael Auler             LLVM_DEBUG(dbgs() << "BOLT-DEBUG: rejected jump table reference"
1805a34c753fSRafael Auler                               << JT->getName() << " in " << *this << '\n');
1806a34c753fSRafael Auler             return false;
1807a34c753fSRafael Auler           }
1808a34c753fSRafael Auler         }
1809a34c753fSRafael Auler 
1810a34c753fSRafael Auler         addUnknownControlFlow(*BB);
1811a34c753fSRafael Auler         continue;
1812a34c753fSRafael Auler       }
1813a34c753fSRafael Auler 
1814a34c753fSRafael Auler       // If this block contains an epilogue code and has an indirect branch,
1815a34c753fSRafael Auler       // then most likely it's a tail call. Otherwise, we cannot tell for sure
1816a34c753fSRafael Auler       // what it is and conservatively reject the function's CFG.
1817a34c753fSRafael Auler       bool IsEpilogue = false;
1818a34c753fSRafael Auler       for (const MCInst &Instr : *BB) {
1819a34c753fSRafael Auler         if (BC.MIB->isLeave(Instr) || BC.MIB->isPop(Instr)) {
1820a34c753fSRafael Auler           IsEpilogue = true;
1821a34c753fSRafael Auler           break;
1822a34c753fSRafael Auler         }
1823a34c753fSRafael Auler       }
1824a34c753fSRafael Auler       if (IsEpilogue) {
1825a34c753fSRafael Auler         BC.MIB->convertJmpToTailCall(Instr);
1826a34c753fSRafael Auler         BB->removeAllSuccessors();
1827a34c753fSRafael Auler         continue;
1828a34c753fSRafael Auler       }
1829a34c753fSRafael Auler 
1830a34c753fSRafael Auler       if (opts::Verbosity >= 2) {
1831a34c753fSRafael Auler         outs() << "BOLT-INFO: rejected potential indirect tail call in "
183240c2e0faSMaksim Panchenko                << "function " << *this << " in basic block " << BB->getName()
183340c2e0faSMaksim Panchenko                << ".\n";
1834a34c753fSRafael Auler         LLVM_DEBUG(BC.printInstructions(dbgs(), BB->begin(), BB->end(),
1835a34c753fSRafael Auler                                         BB->getOffset(), this, true));
1836a34c753fSRafael Auler       }
1837a34c753fSRafael Auler 
1838a34c753fSRafael Auler       if (!opts::StrictMode)
1839a34c753fSRafael Auler         return false;
1840a34c753fSRafael Auler 
1841a34c753fSRafael Auler       addUnknownControlFlow(*BB);
1842a34c753fSRafael Auler     }
1843a34c753fSRafael Auler   }
1844a34c753fSRafael Auler 
1845a34c753fSRafael Auler   if (HasInternalLabelReference)
1846a34c753fSRafael Auler     return false;
1847a34c753fSRafael Auler 
1848a34c753fSRafael Auler   // If there's only one jump table, and one indirect jump, and no other
1849a34c753fSRafael Auler   // references, then we should be able to derive the jump table even if we
1850a34c753fSRafael Auler   // fail to match the pattern.
1851a34c753fSRafael Auler   if (HasUnknownControlFlow && NumIndirectJumps == 1 &&
1852a34c753fSRafael Auler       JumpTables.size() == 1 && LastIndirectJump) {
1853a34c753fSRafael Auler     BC.MIB->setJumpTable(*LastIndirectJump, LastJT, LastJTIndexReg, AllocId);
1854a34c753fSRafael Auler     HasUnknownControlFlow = false;
1855a34c753fSRafael Auler 
18565a343994SMaksim Panchenko     LastIndirectJumpBB->updateJumpTableSuccessors();
1857a34c753fSRafael Auler   }
1858a34c753fSRafael Auler 
1859a34c753fSRafael Auler   if (HasFixedIndirectBranch)
1860a34c753fSRafael Auler     return false;
1861a34c753fSRafael Auler 
1862a34c753fSRafael Auler   if (HasUnknownControlFlow && !BC.HasRelocations)
1863a34c753fSRafael Auler     return false;
1864a34c753fSRafael Auler 
1865a34c753fSRafael Auler   return true;
1866a34c753fSRafael Auler }
1867a34c753fSRafael Auler 
1868a34c753fSRafael Auler void BinaryFunction::recomputeLandingPads() {
1869a34c753fSRafael Auler   updateBBIndices(0);
1870a34c753fSRafael Auler 
1871a34c753fSRafael Auler   for (BinaryBasicBlock *BB : BasicBlocks) {
1872a34c753fSRafael Auler     BB->LandingPads.clear();
1873a34c753fSRafael Auler     BB->Throwers.clear();
1874a34c753fSRafael Auler   }
1875a34c753fSRafael Auler 
1876a34c753fSRafael Auler   for (BinaryBasicBlock *BB : BasicBlocks) {
1877a34c753fSRafael Auler     std::unordered_set<const BinaryBasicBlock *> BBLandingPads;
1878a34c753fSRafael Auler     for (MCInst &Instr : *BB) {
1879a34c753fSRafael Auler       if (!BC.MIB->isInvoke(Instr))
1880a34c753fSRafael Auler         continue;
1881a34c753fSRafael Auler 
1882a34c753fSRafael Auler       const Optional<MCPlus::MCLandingPad> EHInfo = BC.MIB->getEHInfo(Instr);
1883a34c753fSRafael Auler       if (!EHInfo || !EHInfo->first)
1884a34c753fSRafael Auler         continue;
1885a34c753fSRafael Auler 
1886a34c753fSRafael Auler       BinaryBasicBlock *LPBlock = getBasicBlockForLabel(EHInfo->first);
1887a34c753fSRafael Auler       if (!BBLandingPads.count(LPBlock)) {
1888a34c753fSRafael Auler         BBLandingPads.insert(LPBlock);
1889a34c753fSRafael Auler         BB->LandingPads.emplace_back(LPBlock);
1890a34c753fSRafael Auler         LPBlock->Throwers.emplace_back(BB);
1891a34c753fSRafael Auler       }
1892a34c753fSRafael Auler     }
1893a34c753fSRafael Auler   }
1894a34c753fSRafael Auler }
1895a34c753fSRafael Auler 
1896a34c753fSRafael Auler bool BinaryFunction::buildCFG(MCPlusBuilder::AllocatorIdTy AllocatorId) {
1897a34c753fSRafael Auler   auto &MIB = BC.MIB;
1898a34c753fSRafael Auler 
1899a34c753fSRafael Auler   if (!isSimple()) {
1900a34c753fSRafael Auler     assert(!BC.HasRelocations &&
1901a34c753fSRafael Auler            "cannot process file with non-simple function in relocs mode");
1902a34c753fSRafael Auler     return false;
1903a34c753fSRafael Auler   }
1904a34c753fSRafael Auler 
1905a34c753fSRafael Auler   if (CurrentState != State::Disassembled)
1906a34c753fSRafael Auler     return false;
1907a34c753fSRafael Auler 
1908a34c753fSRafael Auler   assert(BasicBlocks.empty() && "basic block list should be empty");
19090b7e8bafSDenis Revunov   assert((Labels.find(getFirstInstructionOffset()) != Labels.end()) &&
1910a34c753fSRafael Auler          "first instruction should always have a label");
1911a34c753fSRafael Auler 
1912a34c753fSRafael Auler   // Create basic blocks in the original layout order:
1913a34c753fSRafael Auler   //
1914a34c753fSRafael Auler   //  * Every instruction with associated label marks
1915a34c753fSRafael Auler   //    the beginning of a basic block.
1916a34c753fSRafael Auler   //  * Conditional instruction marks the end of a basic block,
1917a34c753fSRafael Auler   //    except when the following instruction is an
1918a34c753fSRafael Auler   //    unconditional branch, and the unconditional branch is not
1919a34c753fSRafael Auler   //    a destination of another branch. In the latter case, the
1920a34c753fSRafael Auler   //    basic block will consist of a single unconditional branch
1921a34c753fSRafael Auler   //    (missed "double-jump" optimization).
1922a34c753fSRafael Auler   //
1923a34c753fSRafael Auler   // Created basic blocks are sorted in layout order since they are
1924a34c753fSRafael Auler   // created in the same order as instructions, and instructions are
1925a34c753fSRafael Auler   // sorted by offsets.
1926a34c753fSRafael Auler   BinaryBasicBlock *InsertBB = nullptr;
1927a34c753fSRafael Auler   BinaryBasicBlock *PrevBB = nullptr;
1928a34c753fSRafael Auler   bool IsLastInstrNop = false;
1929ccb99dd1SMaksim Panchenko   // Offset of the last non-nop instruction.
1930a34c753fSRafael Auler   uint64_t LastInstrOffset = 0;
1931a34c753fSRafael Auler 
193240c2e0faSMaksim Panchenko   auto addCFIPlaceholders = [this](uint64_t CFIOffset,
193340c2e0faSMaksim Panchenko                                    BinaryBasicBlock *InsertBB) {
1934a34c753fSRafael Auler     for (auto FI = OffsetToCFI.lower_bound(CFIOffset),
1935a34c753fSRafael Auler               FE = OffsetToCFI.upper_bound(CFIOffset);
1936a34c753fSRafael Auler          FI != FE; ++FI) {
1937a34c753fSRafael Auler       addCFIPseudo(InsertBB, InsertBB->end(), FI->second);
1938a34c753fSRafael Auler     }
1939a34c753fSRafael Auler   };
1940a34c753fSRafael Auler 
1941a34c753fSRafael Auler   // For profiling purposes we need to save the offset of the last instruction
1942ccb99dd1SMaksim Panchenko   // in the basic block.
1943ccb99dd1SMaksim Panchenko   // NOTE: nops always have an Offset annotation. Annotate the last non-nop as
1944ccb99dd1SMaksim Panchenko   //       older profiles ignored nops.
1945a34c753fSRafael Auler   auto updateOffset = [&](uint64_t Offset) {
1946a34c753fSRafael Auler     assert(PrevBB && PrevBB != InsertBB && "invalid previous block");
1947ccb99dd1SMaksim Panchenko     MCInst *LastNonNop = nullptr;
1948ccb99dd1SMaksim Panchenko     for (BinaryBasicBlock::reverse_iterator RII = PrevBB->getLastNonPseudo(),
1949ccb99dd1SMaksim Panchenko                                             E = PrevBB->rend();
1950ccb99dd1SMaksim Panchenko          RII != E; ++RII) {
1951ccb99dd1SMaksim Panchenko       if (!BC.MIB->isPseudo(*RII) && !BC.MIB->isNoop(*RII)) {
1952ccb99dd1SMaksim Panchenko         LastNonNop = &*RII;
1953ccb99dd1SMaksim Panchenko         break;
1954ccb99dd1SMaksim Panchenko       }
1955ccb99dd1SMaksim Panchenko     }
1956a9cd49d5SAmir Ayupov     if (LastNonNop && !MIB->getOffset(*LastNonNop))
1957a9cd49d5SAmir Ayupov       MIB->setOffset(*LastNonNop, static_cast<uint32_t>(Offset), AllocatorId);
1958a34c753fSRafael Auler   };
1959a34c753fSRafael Auler 
1960a34c753fSRafael Auler   for (auto I = Instructions.begin(), E = Instructions.end(); I != E; ++I) {
1961a34c753fSRafael Auler     const uint32_t Offset = I->first;
1962a34c753fSRafael Auler     MCInst &Instr = I->second;
1963a34c753fSRafael Auler 
1964a34c753fSRafael Auler     auto LI = Labels.find(Offset);
1965a34c753fSRafael Auler     if (LI != Labels.end()) {
1966a34c753fSRafael Auler       // Always create new BB at branch destination.
1967ccb99dd1SMaksim Panchenko       PrevBB = InsertBB ? InsertBB : PrevBB;
19688228c703SMaksim Panchenko       InsertBB = addBasicBlockAt(LI->first, LI->second);
19698228c703SMaksim Panchenko       if (opts::PreserveBlocksAlignment && IsLastInstrNop)
19708228c703SMaksim Panchenko         InsertBB->setDerivedAlignment();
19718228c703SMaksim Panchenko 
1972a34c753fSRafael Auler       if (PrevBB)
1973a34c753fSRafael Auler         updateOffset(LastInstrOffset);
1974a34c753fSRafael Auler     }
1975a34c753fSRafael Auler 
1976a34c753fSRafael Auler     const uint64_t InstrInputAddr = I->first + Address;
1977a34c753fSRafael Auler     bool IsSDTMarker =
1978a34c753fSRafael Auler         MIB->isNoop(Instr) && BC.SDTMarkers.count(InstrInputAddr);
1979a34c753fSRafael Auler     bool IsLKMarker = BC.LKMarkers.count(InstrInputAddr);
1980ccb99dd1SMaksim Panchenko     // Mark all nops with Offset for profile tracking purposes.
1981ccb99dd1SMaksim Panchenko     if (MIB->isNoop(Instr) || IsLKMarker) {
1982a9cd49d5SAmir Ayupov       if (!MIB->getOffset(Instr))
1983a9cd49d5SAmir Ayupov         MIB->setOffset(Instr, static_cast<uint32_t>(Offset), AllocatorId);
1984ccb99dd1SMaksim Panchenko       if (IsSDTMarker || IsLKMarker)
1985ccb99dd1SMaksim Panchenko         HasSDTMarker = true;
1986ccb99dd1SMaksim Panchenko       else
1987ccb99dd1SMaksim Panchenko         // Annotate ordinary nops, so we can safely delete them if required.
1988ccb99dd1SMaksim Panchenko         MIB->addAnnotation(Instr, "NOP", static_cast<uint32_t>(1), AllocatorId);
1989a34c753fSRafael Auler     }
1990a34c753fSRafael Auler 
1991a34c753fSRafael Auler     if (!InsertBB) {
1992a34c753fSRafael Auler       // It must be a fallthrough or unreachable code. Create a new block unless
1993a34c753fSRafael Auler       // we see an unconditional branch following a conditional one. The latter
1994a34c753fSRafael Auler       // should not be a conditional tail call.
1995a34c753fSRafael Auler       assert(PrevBB && "no previous basic block for a fall through");
1996a34c753fSRafael Auler       MCInst *PrevInstr = PrevBB->getLastNonPseudoInstr();
1997a34c753fSRafael Auler       assert(PrevInstr && "no previous instruction for a fall through");
1998a34c753fSRafael Auler       if (MIB->isUnconditionalBranch(Instr) &&
1999a34c753fSRafael Auler           !MIB->isUnconditionalBranch(*PrevInstr) &&
2000bb8e7ebaSVladislav Khmelevsky           !MIB->getConditionalTailCall(*PrevInstr) &&
2001bb8e7ebaSVladislav Khmelevsky           !MIB->isReturn(*PrevInstr)) {
2002a34c753fSRafael Auler         // Temporarily restore inserter basic block.
2003a34c753fSRafael Auler         InsertBB = PrevBB;
2004a34c753fSRafael Auler       } else {
2005a34c753fSRafael Auler         MCSymbol *Label;
2006a34c753fSRafael Auler         {
2007a34c753fSRafael Auler           auto L = BC.scopeLock();
2008a34c753fSRafael Auler           Label = BC.Ctx->createNamedTempSymbol("FT");
2009a34c753fSRafael Auler         }
20108228c703SMaksim Panchenko         InsertBB = addBasicBlockAt(Offset, Label);
20118228c703SMaksim Panchenko         if (opts::PreserveBlocksAlignment && IsLastInstrNop)
20128228c703SMaksim Panchenko           InsertBB->setDerivedAlignment();
2013a34c753fSRafael Auler         updateOffset(LastInstrOffset);
2014a34c753fSRafael Auler       }
2015a34c753fSRafael Auler     }
20160b7e8bafSDenis Revunov     if (Offset == getFirstInstructionOffset()) {
20170b7e8bafSDenis Revunov       // Add associated CFI pseudos in the first offset
20180b7e8bafSDenis Revunov       addCFIPlaceholders(Offset, InsertBB);
2019a34c753fSRafael Auler     }
2020a34c753fSRafael Auler 
2021a34c753fSRafael Auler     const bool IsBlockEnd = MIB->isTerminator(Instr);
2022a34c753fSRafael Auler     IsLastInstrNop = MIB->isNoop(Instr);
2023ccb99dd1SMaksim Panchenko     if (!IsLastInstrNop)
2024a34c753fSRafael Auler       LastInstrOffset = Offset;
2025a34c753fSRafael Auler     InsertBB->addInstruction(std::move(Instr));
2026a34c753fSRafael Auler 
2027a34c753fSRafael Auler     // Add associated CFI instrs. We always add the CFI instruction that is
2028a34c753fSRafael Auler     // located immediately after this instruction, since the next CFI
2029a34c753fSRafael Auler     // instruction reflects the change in state caused by this instruction.
2030a34c753fSRafael Auler     auto NextInstr = std::next(I);
2031a34c753fSRafael Auler     uint64_t CFIOffset;
2032a34c753fSRafael Auler     if (NextInstr != E)
2033a34c753fSRafael Auler       CFIOffset = NextInstr->first;
2034a34c753fSRafael Auler     else
2035a34c753fSRafael Auler       CFIOffset = getSize();
2036a34c753fSRafael Auler 
2037a34c753fSRafael Auler     // Note: this potentially invalidates instruction pointers/iterators.
2038a34c753fSRafael Auler     addCFIPlaceholders(CFIOffset, InsertBB);
2039a34c753fSRafael Auler 
2040a34c753fSRafael Auler     if (IsBlockEnd) {
2041a34c753fSRafael Auler       PrevBB = InsertBB;
2042a34c753fSRafael Auler       InsertBB = nullptr;
2043a34c753fSRafael Auler     }
2044a34c753fSRafael Auler   }
2045a34c753fSRafael Auler 
2046a34c753fSRafael Auler   if (BasicBlocks.empty()) {
2047a34c753fSRafael Auler     setSimple(false);
2048a34c753fSRafael Auler     return false;
2049a34c753fSRafael Auler   }
2050a34c753fSRafael Auler 
2051a34c753fSRafael Auler   // Intermediate dump.
2052a34c753fSRafael Auler   LLVM_DEBUG(print(dbgs(), "after creating basic blocks"));
2053a34c753fSRafael Auler 
2054a34c753fSRafael Auler   // TODO: handle properly calls to no-return functions,
2055a34c753fSRafael Auler   // e.g. exit(3), etc. Otherwise we'll see a false fall-through
2056a34c753fSRafael Auler   // blocks.
2057a34c753fSRafael Auler 
2058a34c753fSRafael Auler   for (std::pair<uint32_t, uint32_t> &Branch : TakenBranches) {
2059a34c753fSRafael Auler     LLVM_DEBUG(dbgs() << "registering branch [0x"
2060a34c753fSRafael Auler                       << Twine::utohexstr(Branch.first) << "] -> [0x"
2061a34c753fSRafael Auler                       << Twine::utohexstr(Branch.second) << "]\n");
2062a34c753fSRafael Auler     BinaryBasicBlock *FromBB = getBasicBlockContainingOffset(Branch.first);
2063a34c753fSRafael Auler     BinaryBasicBlock *ToBB = getBasicBlockAtOffset(Branch.second);
2064a34c753fSRafael Auler     if (!FromBB || !ToBB) {
2065a34c753fSRafael Auler       if (!FromBB)
2066a34c753fSRafael Auler         errs() << "BOLT-ERROR: cannot find BB containing the branch.\n";
2067a34c753fSRafael Auler       if (!ToBB)
2068a34c753fSRafael Auler         errs() << "BOLT-ERROR: cannot find BB containing branch destination.\n";
2069a34c753fSRafael Auler       BC.exitWithBugReport("disassembly failed - inconsistent branch found.",
2070a34c753fSRafael Auler                            *this);
2071a34c753fSRafael Auler     }
2072a34c753fSRafael Auler 
2073a34c753fSRafael Auler     FromBB->addSuccessor(ToBB);
2074a34c753fSRafael Auler   }
2075a34c753fSRafael Auler 
2076a34c753fSRafael Auler   // Add fall-through branches.
2077a34c753fSRafael Auler   PrevBB = nullptr;
2078a34c753fSRafael Auler   bool IsPrevFT = false; // Is previous block a fall-through.
2079a34c753fSRafael Auler   for (BinaryBasicBlock *BB : BasicBlocks) {
20803652483cSRafael Auler     if (IsPrevFT)
2081a34c753fSRafael Auler       PrevBB->addSuccessor(BB);
20823652483cSRafael Auler 
2083a34c753fSRafael Auler     if (BB->empty()) {
2084a34c753fSRafael Auler       IsPrevFT = true;
2085a34c753fSRafael Auler       PrevBB = BB;
2086a34c753fSRafael Auler       continue;
2087a34c753fSRafael Auler     }
2088a34c753fSRafael Auler 
2089a34c753fSRafael Auler     MCInst *LastInstr = BB->getLastNonPseudoInstr();
2090a34c753fSRafael Auler     assert(LastInstr &&
2091a34c753fSRafael Auler            "should have non-pseudo instruction in non-empty block");
2092a34c753fSRafael Auler 
2093a34c753fSRafael Auler     if (BB->succ_size() == 0) {
2094a34c753fSRafael Auler       // Since there's no existing successors, we know the last instruction is
2095a34c753fSRafael Auler       // not a conditional branch. Thus if it's a terminator, it shouldn't be a
2096a34c753fSRafael Auler       // fall-through.
2097a34c753fSRafael Auler       //
2098a34c753fSRafael Auler       // Conditional tail call is a special case since we don't add a taken
2099a34c753fSRafael Auler       // branch successor for it.
2100a34c753fSRafael Auler       IsPrevFT = !MIB->isTerminator(*LastInstr) ||
2101a34c753fSRafael Auler                  MIB->getConditionalTailCall(*LastInstr);
2102a34c753fSRafael Auler     } else if (BB->succ_size() == 1) {
2103a34c753fSRafael Auler       IsPrevFT = MIB->isConditionalBranch(*LastInstr);
2104a34c753fSRafael Auler     } else {
2105a34c753fSRafael Auler       IsPrevFT = false;
2106a34c753fSRafael Auler     }
2107a34c753fSRafael Auler 
2108a34c753fSRafael Auler     PrevBB = BB;
2109a34c753fSRafael Auler   }
2110a34c753fSRafael Auler 
2111a34c753fSRafael Auler   // Assign landing pads and throwers info.
2112a34c753fSRafael Auler   recomputeLandingPads();
2113a34c753fSRafael Auler 
2114a34c753fSRafael Auler   // Assign CFI information to each BB entry.
2115a34c753fSRafael Auler   annotateCFIState();
2116a34c753fSRafael Auler 
2117a34c753fSRafael Auler   // Annotate invoke instructions with GNU_args_size data.
2118a34c753fSRafael Auler   propagateGnuArgsSizeInfo(AllocatorId);
2119a34c753fSRafael Auler 
2120a34c753fSRafael Auler   // Set the basic block layout to the original order and set end offsets.
2121a34c753fSRafael Auler   PrevBB = nullptr;
2122a34c753fSRafael Auler   for (BinaryBasicBlock *BB : BasicBlocks) {
2123a34c753fSRafael Auler     BasicBlocksLayout.emplace_back(BB);
2124a34c753fSRafael Auler     if (PrevBB)
2125a34c753fSRafael Auler       PrevBB->setEndOffset(BB->getOffset());
2126a34c753fSRafael Auler     PrevBB = BB;
2127a34c753fSRafael Auler   }
2128a34c753fSRafael Auler   PrevBB->setEndOffset(getSize());
2129a34c753fSRafael Auler 
2130a34c753fSRafael Auler   updateLayoutIndices();
2131a34c753fSRafael Auler 
2132a34c753fSRafael Auler   normalizeCFIState();
2133a34c753fSRafael Auler 
2134a34c753fSRafael Auler   // Clean-up memory taken by intermediate structures.
2135a34c753fSRafael Auler   //
2136a34c753fSRafael Auler   // NB: don't clear Labels list as we may need them if we mark the function
2137a34c753fSRafael Auler   //     as non-simple later in the process of discovering extra entry points.
2138a34c753fSRafael Auler   clearList(Instructions);
2139a34c753fSRafael Auler   clearList(OffsetToCFI);
2140a34c753fSRafael Auler   clearList(TakenBranches);
2141a34c753fSRafael Auler 
2142a34c753fSRafael Auler   // Update the state.
2143a34c753fSRafael Auler   CurrentState = State::CFG;
2144a34c753fSRafael Auler 
2145a34c753fSRafael Auler   // Make any necessary adjustments for indirect branches.
2146a34c753fSRafael Auler   if (!postProcessIndirectBranches(AllocatorId)) {
2147a34c753fSRafael Auler     if (opts::Verbosity) {
2148a34c753fSRafael Auler       errs() << "BOLT-WARNING: failed to post-process indirect branches for "
2149a34c753fSRafael Auler              << *this << '\n';
2150a34c753fSRafael Auler     }
2151a34c753fSRafael Auler     // In relocation mode we want to keep processing the function but avoid
2152a34c753fSRafael Auler     // optimizing it.
2153a34c753fSRafael Auler     setSimple(false);
2154a34c753fSRafael Auler   }
2155a34c753fSRafael Auler 
2156a34c753fSRafael Auler   clearList(ExternallyReferencedOffsets);
2157a34c753fSRafael Auler   clearList(UnknownIndirectBranchOffsets);
2158a34c753fSRafael Auler 
2159a34c753fSRafael Auler   return true;
2160a34c753fSRafael Auler }
2161a34c753fSRafael Auler 
2162a34c753fSRafael Auler void BinaryFunction::postProcessCFG() {
2163a34c753fSRafael Auler   if (isSimple() && !BasicBlocks.empty()) {
2164a34c753fSRafael Auler     // Convert conditional tail call branches to conditional branches that jump
2165a34c753fSRafael Auler     // to a tail call.
2166a34c753fSRafael Auler     removeConditionalTailCalls();
2167a34c753fSRafael Auler 
2168a34c753fSRafael Auler     postProcessProfile();
2169a34c753fSRafael Auler 
2170a34c753fSRafael Auler     // Eliminate inconsistencies between branch instructions and CFG.
2171a34c753fSRafael Auler     postProcessBranches();
2172a34c753fSRafael Auler   }
2173a34c753fSRafael Auler 
2174a34c753fSRafael Auler   calculateMacroOpFusionStats();
2175a34c753fSRafael Auler 
2176a34c753fSRafael Auler   // The final cleanup of intermediate structures.
2177a34c753fSRafael Auler   clearList(IgnoredBranches);
2178a34c753fSRafael Auler 
2179a34c753fSRafael Auler   // Remove "Offset" annotations, unless we need an address-translation table
2180a34c753fSRafael Auler   // later. This has no cost, since annotations are allocated by a bumpptr
2181a34c753fSRafael Auler   // allocator and won't be released anyway until late in the pipeline.
21823652483cSRafael Auler   if (!requiresAddressTranslation() && !opts::Instrument) {
2183a34c753fSRafael Auler     for (BinaryBasicBlock *BB : layout())
2184a34c753fSRafael Auler       for (MCInst &Inst : *BB)
2185a9cd49d5SAmir Ayupov         BC.MIB->clearOffset(Inst);
21863652483cSRafael Auler   }
2187a34c753fSRafael Auler 
2188a34c753fSRafael Auler   assert((!isSimple() || validateCFG()) &&
2189a34c753fSRafael Auler          "invalid CFG detected after post-processing");
2190a34c753fSRafael Auler }
2191a34c753fSRafael Auler 
2192a34c753fSRafael Auler void BinaryFunction::calculateMacroOpFusionStats() {
2193a34c753fSRafael Auler   if (!getBinaryContext().isX86())
2194a34c753fSRafael Auler     return;
2195a34c753fSRafael Auler   for (BinaryBasicBlock *BB : layout()) {
2196a34c753fSRafael Auler     auto II = BB->getMacroOpFusionPair();
2197a34c753fSRafael Auler     if (II == BB->end())
2198a34c753fSRafael Auler       continue;
2199a34c753fSRafael Auler 
2200a34c753fSRafael Auler     // Check offset of the second instruction.
2201a34c753fSRafael Auler     // FIXME: arch-specific.
2202a9cd49d5SAmir Ayupov     const uint32_t Offset = BC.MIB->getOffsetWithDefault(*std::next(II), 0);
2203a34c753fSRafael Auler     if (!Offset || (getAddress() + Offset) % 64)
2204a34c753fSRafael Auler       continue;
2205a34c753fSRafael Auler 
2206a34c753fSRafael Auler     LLVM_DEBUG(dbgs() << "\nmissed macro-op fusion at address 0x"
2207a34c753fSRafael Auler                       << Twine::utohexstr(getAddress() + Offset)
2208a34c753fSRafael Auler                       << " in function " << *this << "; executed "
2209a34c753fSRafael Auler                       << BB->getKnownExecutionCount() << " times.\n");
2210a34c753fSRafael Auler     ++BC.MissedMacroFusionPairs;
2211a34c753fSRafael Auler     BC.MissedMacroFusionExecCount += BB->getKnownExecutionCount();
2212a34c753fSRafael Auler   }
2213a34c753fSRafael Auler }
2214a34c753fSRafael Auler 
2215a34c753fSRafael Auler void BinaryFunction::removeTagsFromProfile() {
2216a34c753fSRafael Auler   for (BinaryBasicBlock *BB : BasicBlocks) {
2217a34c753fSRafael Auler     if (BB->ExecutionCount == BinaryBasicBlock::COUNT_NO_PROFILE)
2218a34c753fSRafael Auler       BB->ExecutionCount = 0;
2219a34c753fSRafael Auler     for (BinaryBasicBlock::BinaryBranchInfo &BI : BB->branch_info()) {
2220a34c753fSRafael Auler       if (BI.Count != BinaryBasicBlock::COUNT_NO_PROFILE &&
2221a34c753fSRafael Auler           BI.MispredictedCount != BinaryBasicBlock::COUNT_NO_PROFILE)
2222a34c753fSRafael Auler         continue;
2223a34c753fSRafael Auler       BI.Count = 0;
2224a34c753fSRafael Auler       BI.MispredictedCount = 0;
2225a34c753fSRafael Auler     }
2226a34c753fSRafael Auler   }
2227a34c753fSRafael Auler }
2228a34c753fSRafael Auler 
2229a34c753fSRafael Auler void BinaryFunction::removeConditionalTailCalls() {
2230a34c753fSRafael Auler   // Blocks to be appended at the end.
2231a34c753fSRafael Auler   std::vector<std::unique_ptr<BinaryBasicBlock>> NewBlocks;
2232a34c753fSRafael Auler 
2233a34c753fSRafael Auler   for (auto BBI = begin(); BBI != end(); ++BBI) {
2234a34c753fSRafael Auler     BinaryBasicBlock &BB = *BBI;
2235a34c753fSRafael Auler     MCInst *CTCInstr = BB.getLastNonPseudoInstr();
2236a34c753fSRafael Auler     if (!CTCInstr)
2237a34c753fSRafael Auler       continue;
2238a34c753fSRafael Auler 
2239a34c753fSRafael Auler     Optional<uint64_t> TargetAddressOrNone =
2240a34c753fSRafael Auler         BC.MIB->getConditionalTailCall(*CTCInstr);
2241a34c753fSRafael Auler     if (!TargetAddressOrNone)
2242a34c753fSRafael Auler       continue;
2243a34c753fSRafael Auler 
2244a34c753fSRafael Auler     // Gather all necessary information about CTC instruction before
2245a34c753fSRafael Auler     // annotations are destroyed.
2246a34c753fSRafael Auler     const int32_t CFIStateBeforeCTC = BB.getCFIStateAtInstr(CTCInstr);
2247a34c753fSRafael Auler     uint64_t CTCTakenCount = BinaryBasicBlock::COUNT_NO_PROFILE;
2248a34c753fSRafael Auler     uint64_t CTCMispredCount = BinaryBasicBlock::COUNT_NO_PROFILE;
2249a34c753fSRafael Auler     if (hasValidProfile()) {
225040c2e0faSMaksim Panchenko       CTCTakenCount = BC.MIB->getAnnotationWithDefault<uint64_t>(
225140c2e0faSMaksim Panchenko           *CTCInstr, "CTCTakenCount");
225240c2e0faSMaksim Panchenko       CTCMispredCount = BC.MIB->getAnnotationWithDefault<uint64_t>(
225340c2e0faSMaksim Panchenko           *CTCInstr, "CTCMispredCount");
2254a34c753fSRafael Auler     }
2255a34c753fSRafael Auler 
2256a34c753fSRafael Auler     // Assert that the tail call does not throw.
2257a34c753fSRafael Auler     assert(!BC.MIB->getEHInfo(*CTCInstr) &&
2258a34c753fSRafael Auler            "found tail call with associated landing pad");
2259a34c753fSRafael Auler 
2260a34c753fSRafael Auler     // Create a basic block with an unconditional tail call instruction using
2261a34c753fSRafael Auler     // the same destination.
2262a34c753fSRafael Auler     const MCSymbol *CTCTargetLabel = BC.MIB->getTargetSymbol(*CTCInstr);
2263a34c753fSRafael Auler     assert(CTCTargetLabel && "symbol expected for conditional tail call");
2264a34c753fSRafael Auler     MCInst TailCallInstr;
2265a34c753fSRafael Auler     BC.MIB->createTailCall(TailCallInstr, CTCTargetLabel, BC.Ctx.get());
2266a34c753fSRafael Auler     // Link new BBs to the original input offset of the BB where the CTC
2267a34c753fSRafael Auler     // is, so we can map samples recorded in new BBs back to the original BB
2268a34c753fSRafael Auler     // seem in the input binary (if using BAT)
22698228c703SMaksim Panchenko     std::unique_ptr<BinaryBasicBlock> TailCallBB =
22708228c703SMaksim Panchenko         createBasicBlock(BC.Ctx->createNamedTempSymbol("TC"));
22718228c703SMaksim Panchenko     TailCallBB->setOffset(BB.getInputOffset());
2272a34c753fSRafael Auler     TailCallBB->addInstruction(TailCallInstr);
2273a34c753fSRafael Auler     TailCallBB->setCFIState(CFIStateBeforeCTC);
2274a34c753fSRafael Auler 
2275a34c753fSRafael Auler     // Add CFG edge with profile info from BB to TailCallBB.
2276a34c753fSRafael Auler     BB.addSuccessor(TailCallBB.get(), CTCTakenCount, CTCMispredCount);
2277a34c753fSRafael Auler 
2278a34c753fSRafael Auler     // Add execution count for the block.
2279a34c753fSRafael Auler     TailCallBB->setExecutionCount(CTCTakenCount);
2280a34c753fSRafael Auler 
2281a34c753fSRafael Auler     BC.MIB->convertTailCallToJmp(*CTCInstr);
2282a34c753fSRafael Auler 
2283a34c753fSRafael Auler     BC.MIB->replaceBranchTarget(*CTCInstr, TailCallBB->getLabel(),
2284a34c753fSRafael Auler                                 BC.Ctx.get());
2285a34c753fSRafael Auler 
2286a34c753fSRafael Auler     // Add basic block to the list that will be added to the end.
2287a34c753fSRafael Auler     NewBlocks.emplace_back(std::move(TailCallBB));
2288a34c753fSRafael Auler 
2289a34c753fSRafael Auler     // Swap edges as the TailCallBB corresponds to the taken branch.
2290a34c753fSRafael Auler     BB.swapConditionalSuccessors();
2291a34c753fSRafael Auler 
2292a34c753fSRafael Auler     // This branch is no longer a conditional tail call.
2293a34c753fSRafael Auler     BC.MIB->unsetConditionalTailCall(*CTCInstr);
2294a34c753fSRafael Auler   }
2295a34c753fSRafael Auler 
229640c2e0faSMaksim Panchenko   insertBasicBlocks(std::prev(end()), std::move(NewBlocks),
2297a34c753fSRafael Auler                     /* UpdateLayout */ true,
2298a34c753fSRafael Auler                     /* UpdateCFIState */ false);
2299a34c753fSRafael Auler }
2300a34c753fSRafael Auler 
2301a34c753fSRafael Auler uint64_t BinaryFunction::getFunctionScore() const {
2302a34c753fSRafael Auler   if (FunctionScore != -1)
2303a34c753fSRafael Auler     return FunctionScore;
2304a34c753fSRafael Auler 
2305a34c753fSRafael Auler   if (!isSimple() || !hasValidProfile()) {
2306a34c753fSRafael Auler     FunctionScore = 0;
2307a34c753fSRafael Auler     return FunctionScore;
2308a34c753fSRafael Auler   }
2309a34c753fSRafael Auler 
2310a34c753fSRafael Auler   uint64_t TotalScore = 0ULL;
2311a34c753fSRafael Auler   for (BinaryBasicBlock *BB : layout()) {
2312a34c753fSRafael Auler     uint64_t BBExecCount = BB->getExecutionCount();
2313a34c753fSRafael Auler     if (BBExecCount == BinaryBasicBlock::COUNT_NO_PROFILE)
2314a34c753fSRafael Auler       continue;
231542465efdSRafael Auler     TotalScore += BBExecCount * BB->getNumNonPseudos();
2316a34c753fSRafael Auler   }
2317a34c753fSRafael Auler   FunctionScore = TotalScore;
2318a34c753fSRafael Auler   return FunctionScore;
2319a34c753fSRafael Auler }
2320a34c753fSRafael Auler 
2321a34c753fSRafael Auler void BinaryFunction::annotateCFIState() {
2322a34c753fSRafael Auler   assert(CurrentState == State::Disassembled && "unexpected function state");
2323a34c753fSRafael Auler   assert(!BasicBlocks.empty() && "basic block list should not be empty");
2324a34c753fSRafael Auler 
2325a34c753fSRafael Auler   // This is an index of the last processed CFI in FDE CFI program.
2326a34c753fSRafael Auler   uint32_t State = 0;
2327a34c753fSRafael Auler 
2328a34c753fSRafael Auler   // This is an index of RememberState CFI reflecting effective state right
2329a34c753fSRafael Auler   // after execution of RestoreState CFI.
2330a34c753fSRafael Auler   //
2331a34c753fSRafael Auler   // It differs from State iff the CFI at (State-1)
2332a34c753fSRafael Auler   // was RestoreState (modulo GNU_args_size CFIs, which are ignored).
2333a34c753fSRafael Auler   //
2334a34c753fSRafael Auler   // This allows us to generate shorter replay sequences when producing new
2335a34c753fSRafael Auler   // CFI programs.
2336a34c753fSRafael Auler   uint32_t EffectiveState = 0;
2337a34c753fSRafael Auler 
2338a34c753fSRafael Auler   // For tracking RememberState/RestoreState sequences.
2339a34c753fSRafael Auler   std::stack<uint32_t> StateStack;
2340a34c753fSRafael Auler 
2341a34c753fSRafael Auler   for (BinaryBasicBlock *BB : BasicBlocks) {
2342a34c753fSRafael Auler     BB->setCFIState(EffectiveState);
2343a34c753fSRafael Auler 
2344a34c753fSRafael Auler     for (const MCInst &Instr : *BB) {
2345a34c753fSRafael Auler       const MCCFIInstruction *CFI = getCFIFor(Instr);
2346a34c753fSRafael Auler       if (!CFI)
2347a34c753fSRafael Auler         continue;
2348a34c753fSRafael Auler 
2349a34c753fSRafael Auler       ++State;
2350a34c753fSRafael Auler 
2351a34c753fSRafael Auler       switch (CFI->getOperation()) {
2352a34c753fSRafael Auler       case MCCFIInstruction::OpRememberState:
2353a34c753fSRafael Auler         StateStack.push(EffectiveState);
2354a34c753fSRafael Auler         EffectiveState = State;
2355a34c753fSRafael Auler         break;
2356a34c753fSRafael Auler       case MCCFIInstruction::OpRestoreState:
2357a34c753fSRafael Auler         assert(!StateStack.empty() && "corrupt CFI stack");
2358a34c753fSRafael Auler         EffectiveState = StateStack.top();
2359a34c753fSRafael Auler         StateStack.pop();
2360a34c753fSRafael Auler         break;
2361a34c753fSRafael Auler       case MCCFIInstruction::OpGnuArgsSize:
2362a34c753fSRafael Auler         // OpGnuArgsSize CFIs do not affect the CFI state.
2363a34c753fSRafael Auler         break;
2364a34c753fSRafael Auler       default:
2365a34c753fSRafael Auler         // Any other CFI updates the state.
2366a34c753fSRafael Auler         EffectiveState = State;
2367a34c753fSRafael Auler         break;
2368a34c753fSRafael Auler       }
2369a34c753fSRafael Auler     }
2370a34c753fSRafael Auler   }
2371a34c753fSRafael Auler 
2372a34c753fSRafael Auler   assert(StateStack.empty() && "corrupt CFI stack");
2373a34c753fSRafael Auler }
2374a34c753fSRafael Auler 
2375a34c753fSRafael Auler namespace {
2376a34c753fSRafael Auler 
2377a34c753fSRafael Auler /// Our full interpretation of a DWARF CFI machine state at a given point
2378a34c753fSRafael Auler struct CFISnapshot {
2379a34c753fSRafael Auler   /// CFA register number and offset defining the canonical frame at this
2380a34c753fSRafael Auler   /// point, or the number of a rule (CFI state) that computes it with a
2381a34c753fSRafael Auler   /// DWARF expression. This number will be negative if it refers to a CFI
2382a34c753fSRafael Auler   /// located in the CIE instead of the FDE.
2383a34c753fSRafael Auler   uint32_t CFAReg;
2384a34c753fSRafael Auler   int32_t CFAOffset;
2385a34c753fSRafael Auler   int32_t CFARule;
2386a34c753fSRafael Auler   /// Mapping of rules (CFI states) that define the location of each
2387a34c753fSRafael Auler   /// register. If absent, no rule defining the location of such register
2388a34c753fSRafael Auler   /// was ever read. This number will be negative if it refers to a CFI
2389a34c753fSRafael Auler   /// located in the CIE instead of the FDE.
2390a34c753fSRafael Auler   DenseMap<int32_t, int32_t> RegRule;
2391a34c753fSRafael Auler 
2392a34c753fSRafael Auler   /// References to CIE, FDE and expanded instructions after a restore state
2393ebe51c4dSMaksim Panchenko   const BinaryFunction::CFIInstrMapType &CIE;
2394ebe51c4dSMaksim Panchenko   const BinaryFunction::CFIInstrMapType &FDE;
2395a34c753fSRafael Auler   const DenseMap<int32_t, SmallVector<int32_t, 4>> &FrameRestoreEquivalents;
2396a34c753fSRafael Auler 
2397a34c753fSRafael Auler   /// Current FDE CFI number representing the state where the snapshot is at
2398a34c753fSRafael Auler   int32_t CurState;
2399a34c753fSRafael Auler 
2400a34c753fSRafael Auler   /// Used when we don't have information about which state/rule to apply
2401a34c753fSRafael Auler   /// to recover the location of either the CFA or a specific register
2402a34c753fSRafael Auler   constexpr static int32_t UNKNOWN = std::numeric_limits<int32_t>::min();
2403a34c753fSRafael Auler 
2404a34c753fSRafael Auler private:
2405a34c753fSRafael Auler   /// Update our snapshot by executing a single CFI
2406a34c753fSRafael Auler   void update(const MCCFIInstruction &Instr, int32_t RuleNumber) {
2407a34c753fSRafael Auler     switch (Instr.getOperation()) {
2408a34c753fSRafael Auler     case MCCFIInstruction::OpSameValue:
2409a34c753fSRafael Auler     case MCCFIInstruction::OpRelOffset:
2410a34c753fSRafael Auler     case MCCFIInstruction::OpOffset:
2411a34c753fSRafael Auler     case MCCFIInstruction::OpRestore:
2412a34c753fSRafael Auler     case MCCFIInstruction::OpUndefined:
2413a34c753fSRafael Auler     case MCCFIInstruction::OpRegister:
2414a34c753fSRafael Auler       RegRule[Instr.getRegister()] = RuleNumber;
2415a34c753fSRafael Auler       break;
2416a34c753fSRafael Auler     case MCCFIInstruction::OpDefCfaRegister:
2417a34c753fSRafael Auler       CFAReg = Instr.getRegister();
2418a34c753fSRafael Auler       CFARule = UNKNOWN;
2419a34c753fSRafael Auler       break;
2420a34c753fSRafael Auler     case MCCFIInstruction::OpDefCfaOffset:
2421a34c753fSRafael Auler       CFAOffset = Instr.getOffset();
2422a34c753fSRafael Auler       CFARule = UNKNOWN;
2423a34c753fSRafael Auler       break;
2424a34c753fSRafael Auler     case MCCFIInstruction::OpDefCfa:
2425a34c753fSRafael Auler       CFAReg = Instr.getRegister();
2426a34c753fSRafael Auler       CFAOffset = Instr.getOffset();
2427a34c753fSRafael Auler       CFARule = UNKNOWN;
2428a34c753fSRafael Auler       break;
2429a34c753fSRafael Auler     case MCCFIInstruction::OpEscape: {
2430a34c753fSRafael Auler       Optional<uint8_t> Reg = readDWARFExpressionTargetReg(Instr.getValues());
2431a34c753fSRafael Auler       // Handle DW_CFA_def_cfa_expression
2432a34c753fSRafael Auler       if (!Reg) {
2433a34c753fSRafael Auler         CFARule = RuleNumber;
2434a34c753fSRafael Auler         break;
2435a34c753fSRafael Auler       }
2436a34c753fSRafael Auler       RegRule[*Reg] = RuleNumber;
2437a34c753fSRafael Auler       break;
2438a34c753fSRafael Auler     }
2439a34c753fSRafael Auler     case MCCFIInstruction::OpAdjustCfaOffset:
2440a34c753fSRafael Auler     case MCCFIInstruction::OpWindowSave:
2441a34c753fSRafael Auler     case MCCFIInstruction::OpNegateRAState:
2442a34c753fSRafael Auler     case MCCFIInstruction::OpLLVMDefAspaceCfa:
2443a34c753fSRafael Auler       llvm_unreachable("unsupported CFI opcode");
2444a34c753fSRafael Auler       break;
2445a34c753fSRafael Auler     case MCCFIInstruction::OpRememberState:
2446a34c753fSRafael Auler     case MCCFIInstruction::OpRestoreState:
2447a34c753fSRafael Auler     case MCCFIInstruction::OpGnuArgsSize:
2448a34c753fSRafael Auler       // do not affect CFI state
2449a34c753fSRafael Auler       break;
2450a34c753fSRafael Auler     }
2451a34c753fSRafael Auler   }
2452a34c753fSRafael Auler 
2453a34c753fSRafael Auler public:
2454a34c753fSRafael Auler   /// Advance state reading FDE CFI instructions up to State number
2455a34c753fSRafael Auler   void advanceTo(int32_t State) {
2456a34c753fSRafael Auler     for (int32_t I = CurState, E = State; I != E; ++I) {
2457a34c753fSRafael Auler       const MCCFIInstruction &Instr = FDE[I];
2458a34c753fSRafael Auler       if (Instr.getOperation() != MCCFIInstruction::OpRestoreState) {
2459a34c753fSRafael Auler         update(Instr, I);
2460a34c753fSRafael Auler         continue;
2461a34c753fSRafael Auler       }
2462a34c753fSRafael Auler       // If restore state instruction, fetch the equivalent CFIs that have
2463a34c753fSRafael Auler       // the same effect of this restore. This is used to ensure remember-
2464a34c753fSRafael Auler       // restore pairs are completely removed.
2465a34c753fSRafael Auler       auto Iter = FrameRestoreEquivalents.find(I);
2466a34c753fSRafael Auler       if (Iter == FrameRestoreEquivalents.end())
2467a34c753fSRafael Auler         continue;
24683652483cSRafael Auler       for (int32_t RuleNumber : Iter->second)
2469a34c753fSRafael Auler         update(FDE[RuleNumber], RuleNumber);
2470a34c753fSRafael Auler     }
2471a34c753fSRafael Auler 
2472a34c753fSRafael Auler     assert(((CFAReg != (uint32_t)UNKNOWN && CFAOffset != UNKNOWN) ||
2473a34c753fSRafael Auler             CFARule != UNKNOWN) &&
2474a34c753fSRafael Auler            "CIE did not define default CFA?");
2475a34c753fSRafael Auler 
2476a34c753fSRafael Auler     CurState = State;
2477a34c753fSRafael Auler   }
2478a34c753fSRafael Auler 
2479a34c753fSRafael Auler   /// Interpret all CIE and FDE instructions up until CFI State number and
2480a34c753fSRafael Auler   /// populate this snapshot
2481a34c753fSRafael Auler   CFISnapshot(
2482ebe51c4dSMaksim Panchenko       const BinaryFunction::CFIInstrMapType &CIE,
2483ebe51c4dSMaksim Panchenko       const BinaryFunction::CFIInstrMapType &FDE,
2484a34c753fSRafael Auler       const DenseMap<int32_t, SmallVector<int32_t, 4>> &FrameRestoreEquivalents,
2485a34c753fSRafael Auler       int32_t State)
2486a34c753fSRafael Auler       : CIE(CIE), FDE(FDE), FrameRestoreEquivalents(FrameRestoreEquivalents) {
2487a34c753fSRafael Auler     CFAReg = UNKNOWN;
2488a34c753fSRafael Auler     CFAOffset = UNKNOWN;
2489a34c753fSRafael Auler     CFARule = UNKNOWN;
2490a34c753fSRafael Auler     CurState = 0;
2491a34c753fSRafael Auler 
2492a34c753fSRafael Auler     for (int32_t I = 0, E = CIE.size(); I != E; ++I) {
2493a34c753fSRafael Auler       const MCCFIInstruction &Instr = CIE[I];
2494a34c753fSRafael Auler       update(Instr, -I);
2495a34c753fSRafael Auler     }
2496a34c753fSRafael Auler 
2497a34c753fSRafael Auler     advanceTo(State);
2498a34c753fSRafael Auler   }
2499a34c753fSRafael Auler };
2500a34c753fSRafael Auler 
2501a34c753fSRafael Auler /// A CFI snapshot with the capability of checking if incremental additions to
2502a34c753fSRafael Auler /// it are redundant. This is used to ensure we do not emit two CFI instructions
2503a34c753fSRafael Auler /// back-to-back that are doing the same state change, or to avoid emitting a
2504a34c753fSRafael Auler /// CFI at all when the state at that point would not be modified after that CFI
2505a34c753fSRafael Auler struct CFISnapshotDiff : public CFISnapshot {
2506a34c753fSRafael Auler   bool RestoredCFAReg{false};
2507a34c753fSRafael Auler   bool RestoredCFAOffset{false};
2508a34c753fSRafael Auler   DenseMap<int32_t, bool> RestoredRegs;
2509a34c753fSRafael Auler 
2510a34c753fSRafael Auler   CFISnapshotDiff(const CFISnapshot &S) : CFISnapshot(S) {}
2511a34c753fSRafael Auler 
2512a34c753fSRafael Auler   CFISnapshotDiff(
2513ebe51c4dSMaksim Panchenko       const BinaryFunction::CFIInstrMapType &CIE,
2514ebe51c4dSMaksim Panchenko       const BinaryFunction::CFIInstrMapType &FDE,
2515a34c753fSRafael Auler       const DenseMap<int32_t, SmallVector<int32_t, 4>> &FrameRestoreEquivalents,
2516a34c753fSRafael Auler       int32_t State)
2517a34c753fSRafael Auler       : CFISnapshot(CIE, FDE, FrameRestoreEquivalents, State) {}
2518a34c753fSRafael Auler 
2519a34c753fSRafael Auler   /// Return true if applying Instr to this state is redundant and can be
2520a34c753fSRafael Auler   /// dismissed.
2521a34c753fSRafael Auler   bool isRedundant(const MCCFIInstruction &Instr) {
2522a34c753fSRafael Auler     switch (Instr.getOperation()) {
2523a34c753fSRafael Auler     case MCCFIInstruction::OpSameValue:
2524a34c753fSRafael Auler     case MCCFIInstruction::OpRelOffset:
2525a34c753fSRafael Auler     case MCCFIInstruction::OpOffset:
2526a34c753fSRafael Auler     case MCCFIInstruction::OpRestore:
2527a34c753fSRafael Auler     case MCCFIInstruction::OpUndefined:
2528a34c753fSRafael Auler     case MCCFIInstruction::OpRegister:
2529a34c753fSRafael Auler     case MCCFIInstruction::OpEscape: {
2530a34c753fSRafael Auler       uint32_t Reg;
2531a34c753fSRafael Auler       if (Instr.getOperation() != MCCFIInstruction::OpEscape) {
2532a34c753fSRafael Auler         Reg = Instr.getRegister();
2533a34c753fSRafael Auler       } else {
2534a34c753fSRafael Auler         Optional<uint8_t> R = readDWARFExpressionTargetReg(Instr.getValues());
2535a34c753fSRafael Auler         // Handle DW_CFA_def_cfa_expression
2536a34c753fSRafael Auler         if (!R) {
2537a34c753fSRafael Auler           if (RestoredCFAReg && RestoredCFAOffset)
2538a34c753fSRafael Auler             return true;
2539a34c753fSRafael Auler           RestoredCFAReg = true;
2540a34c753fSRafael Auler           RestoredCFAOffset = true;
2541a34c753fSRafael Auler           return false;
2542a34c753fSRafael Auler         }
2543a34c753fSRafael Auler         Reg = *R;
2544a34c753fSRafael Auler       }
2545a34c753fSRafael Auler       if (RestoredRegs[Reg])
2546a34c753fSRafael Auler         return true;
2547a34c753fSRafael Auler       RestoredRegs[Reg] = true;
2548a34c753fSRafael Auler       const int32_t CurRegRule =
2549a34c753fSRafael Auler           RegRule.find(Reg) != RegRule.end() ? RegRule[Reg] : UNKNOWN;
2550a34c753fSRafael Auler       if (CurRegRule == UNKNOWN) {
2551a34c753fSRafael Auler         if (Instr.getOperation() == MCCFIInstruction::OpRestore ||
2552a34c753fSRafael Auler             Instr.getOperation() == MCCFIInstruction::OpSameValue)
2553a34c753fSRafael Auler           return true;
2554a34c753fSRafael Auler         return false;
2555a34c753fSRafael Auler       }
2556a34c753fSRafael Auler       const MCCFIInstruction &LastDef =
2557a34c753fSRafael Auler           CurRegRule < 0 ? CIE[-CurRegRule] : FDE[CurRegRule];
2558a34c753fSRafael Auler       return LastDef == Instr;
2559a34c753fSRafael Auler     }
2560a34c753fSRafael Auler     case MCCFIInstruction::OpDefCfaRegister:
2561a34c753fSRafael Auler       if (RestoredCFAReg)
2562a34c753fSRafael Auler         return true;
2563a34c753fSRafael Auler       RestoredCFAReg = true;
2564a34c753fSRafael Auler       return CFAReg == Instr.getRegister();
2565a34c753fSRafael Auler     case MCCFIInstruction::OpDefCfaOffset:
2566a34c753fSRafael Auler       if (RestoredCFAOffset)
2567a34c753fSRafael Auler         return true;
2568a34c753fSRafael Auler       RestoredCFAOffset = true;
2569a34c753fSRafael Auler       return CFAOffset == Instr.getOffset();
2570a34c753fSRafael Auler     case MCCFIInstruction::OpDefCfa:
2571a34c753fSRafael Auler       if (RestoredCFAReg && RestoredCFAOffset)
2572a34c753fSRafael Auler         return true;
2573a34c753fSRafael Auler       RestoredCFAReg = true;
2574a34c753fSRafael Auler       RestoredCFAOffset = true;
2575a34c753fSRafael Auler       return CFAReg == Instr.getRegister() && CFAOffset == Instr.getOffset();
2576a34c753fSRafael Auler     case MCCFIInstruction::OpAdjustCfaOffset:
2577a34c753fSRafael Auler     case MCCFIInstruction::OpWindowSave:
2578a34c753fSRafael Auler     case MCCFIInstruction::OpNegateRAState:
2579a34c753fSRafael Auler     case MCCFIInstruction::OpLLVMDefAspaceCfa:
2580a34c753fSRafael Auler       llvm_unreachable("unsupported CFI opcode");
2581a34c753fSRafael Auler       return false;
2582a34c753fSRafael Auler     case MCCFIInstruction::OpRememberState:
2583a34c753fSRafael Auler     case MCCFIInstruction::OpRestoreState:
2584a34c753fSRafael Auler     case MCCFIInstruction::OpGnuArgsSize:
2585a34c753fSRafael Auler       // do not affect CFI state
2586a34c753fSRafael Auler       return true;
2587a34c753fSRafael Auler     }
2588a34c753fSRafael Auler     return false;
2589a34c753fSRafael Auler   }
2590a34c753fSRafael Auler };
2591a34c753fSRafael Auler 
2592a34c753fSRafael Auler } // end anonymous namespace
2593a34c753fSRafael Auler 
2594a34c753fSRafael Auler bool BinaryFunction::replayCFIInstrs(int32_t FromState, int32_t ToState,
2595a34c753fSRafael Auler                                      BinaryBasicBlock *InBB,
2596a34c753fSRafael Auler                                      BinaryBasicBlock::iterator InsertIt) {
2597a34c753fSRafael Auler   if (FromState == ToState)
2598a34c753fSRafael Auler     return true;
2599a34c753fSRafael Auler   assert(FromState < ToState && "can only replay CFIs forward");
2600a34c753fSRafael Auler 
2601a34c753fSRafael Auler   CFISnapshotDiff CFIDiff(CIEFrameInstructions, FrameInstructions,
2602a34c753fSRafael Auler                           FrameRestoreEquivalents, FromState);
2603a34c753fSRafael Auler 
2604a34c753fSRafael Auler   std::vector<uint32_t> NewCFIs;
2605a34c753fSRafael Auler   for (int32_t CurState = FromState; CurState < ToState; ++CurState) {
2606a34c753fSRafael Auler     MCCFIInstruction *Instr = &FrameInstructions[CurState];
2607a34c753fSRafael Auler     if (Instr->getOperation() == MCCFIInstruction::OpRestoreState) {
2608a34c753fSRafael Auler       auto Iter = FrameRestoreEquivalents.find(CurState);
2609a34c753fSRafael Auler       assert(Iter != FrameRestoreEquivalents.end());
261040c2e0faSMaksim Panchenko       NewCFIs.insert(NewCFIs.end(), Iter->second.begin(), Iter->second.end());
2611a34c753fSRafael Auler       // RestoreState / Remember will be filtered out later by CFISnapshotDiff,
2612a34c753fSRafael Auler       // so we might as well fall-through here.
2613a34c753fSRafael Auler     }
2614a34c753fSRafael Auler     NewCFIs.push_back(CurState);
2615a34c753fSRafael Auler     continue;
2616a34c753fSRafael Auler   }
2617a34c753fSRafael Auler 
2618a34c753fSRafael Auler   // Replay instructions while avoiding duplicates
2619a34c753fSRafael Auler   for (auto I = NewCFIs.rbegin(), E = NewCFIs.rend(); I != E; ++I) {
2620a34c753fSRafael Auler     if (CFIDiff.isRedundant(FrameInstructions[*I]))
2621a34c753fSRafael Auler       continue;
2622a34c753fSRafael Auler     InsertIt = addCFIPseudo(InBB, InsertIt, *I);
2623a34c753fSRafael Auler   }
2624a34c753fSRafael Auler 
2625a34c753fSRafael Auler   return true;
2626a34c753fSRafael Auler }
2627a34c753fSRafael Auler 
2628a34c753fSRafael Auler SmallVector<int32_t, 4>
2629a34c753fSRafael Auler BinaryFunction::unwindCFIState(int32_t FromState, int32_t ToState,
2630a34c753fSRafael Auler                                BinaryBasicBlock *InBB,
2631a34c753fSRafael Auler                                BinaryBasicBlock::iterator &InsertIt) {
2632a34c753fSRafael Auler   SmallVector<int32_t, 4> NewStates;
2633a34c753fSRafael Auler 
2634a34c753fSRafael Auler   CFISnapshot ToCFITable(CIEFrameInstructions, FrameInstructions,
2635a34c753fSRafael Auler                          FrameRestoreEquivalents, ToState);
2636a34c753fSRafael Auler   CFISnapshotDiff FromCFITable(ToCFITable);
2637a34c753fSRafael Auler   FromCFITable.advanceTo(FromState);
2638a34c753fSRafael Auler 
2639a34c753fSRafael Auler   auto undoStateDefCfa = [&]() {
2640a34c753fSRafael Auler     if (ToCFITable.CFARule == CFISnapshot::UNKNOWN) {
2641a34c753fSRafael Auler       FrameInstructions.emplace_back(MCCFIInstruction::cfiDefCfa(
2642a34c753fSRafael Auler           nullptr, ToCFITable.CFAReg, ToCFITable.CFAOffset));
2643a34c753fSRafael Auler       if (FromCFITable.isRedundant(FrameInstructions.back())) {
2644a34c753fSRafael Auler         FrameInstructions.pop_back();
2645a34c753fSRafael Auler         return;
2646a34c753fSRafael Auler       }
2647a34c753fSRafael Auler       NewStates.push_back(FrameInstructions.size() - 1);
2648a34c753fSRafael Auler       InsertIt = addCFIPseudo(InBB, InsertIt, FrameInstructions.size() - 1);
2649a34c753fSRafael Auler       ++InsertIt;
2650a34c753fSRafael Auler     } else if (ToCFITable.CFARule < 0) {
2651a34c753fSRafael Auler       if (FromCFITable.isRedundant(CIEFrameInstructions[-ToCFITable.CFARule]))
2652a34c753fSRafael Auler         return;
2653a34c753fSRafael Auler       NewStates.push_back(FrameInstructions.size());
2654a34c753fSRafael Auler       InsertIt = addCFIPseudo(InBB, InsertIt, FrameInstructions.size());
2655a34c753fSRafael Auler       ++InsertIt;
2656a34c753fSRafael Auler       FrameInstructions.emplace_back(CIEFrameInstructions[-ToCFITable.CFARule]);
2657a34c753fSRafael Auler     } else if (!FromCFITable.isRedundant(
2658a34c753fSRafael Auler                    FrameInstructions[ToCFITable.CFARule])) {
2659a34c753fSRafael Auler       NewStates.push_back(ToCFITable.CFARule);
2660a34c753fSRafael Auler       InsertIt = addCFIPseudo(InBB, InsertIt, ToCFITable.CFARule);
2661a34c753fSRafael Auler       ++InsertIt;
2662a34c753fSRafael Auler     }
2663a34c753fSRafael Auler   };
2664a34c753fSRafael Auler 
2665a34c753fSRafael Auler   auto undoState = [&](const MCCFIInstruction &Instr) {
2666a34c753fSRafael Auler     switch (Instr.getOperation()) {
2667a34c753fSRafael Auler     case MCCFIInstruction::OpRememberState:
2668a34c753fSRafael Auler     case MCCFIInstruction::OpRestoreState:
2669a34c753fSRafael Auler       break;
2670a34c753fSRafael Auler     case MCCFIInstruction::OpSameValue:
2671a34c753fSRafael Auler     case MCCFIInstruction::OpRelOffset:
2672a34c753fSRafael Auler     case MCCFIInstruction::OpOffset:
2673a34c753fSRafael Auler     case MCCFIInstruction::OpRestore:
2674a34c753fSRafael Auler     case MCCFIInstruction::OpUndefined:
2675a34c753fSRafael Auler     case MCCFIInstruction::OpEscape:
2676a34c753fSRafael Auler     case MCCFIInstruction::OpRegister: {
2677a34c753fSRafael Auler       uint32_t Reg;
2678a34c753fSRafael Auler       if (Instr.getOperation() != MCCFIInstruction::OpEscape) {
2679a34c753fSRafael Auler         Reg = Instr.getRegister();
2680a34c753fSRafael Auler       } else {
2681a34c753fSRafael Auler         Optional<uint8_t> R = readDWARFExpressionTargetReg(Instr.getValues());
2682a34c753fSRafael Auler         // Handle DW_CFA_def_cfa_expression
2683a34c753fSRafael Auler         if (!R) {
2684a34c753fSRafael Auler           undoStateDefCfa();
2685a34c753fSRafael Auler           return;
2686a34c753fSRafael Auler         }
2687a34c753fSRafael Auler         Reg = *R;
2688a34c753fSRafael Auler       }
2689a34c753fSRafael Auler 
2690a34c753fSRafael Auler       if (ToCFITable.RegRule.find(Reg) == ToCFITable.RegRule.end()) {
2691a34c753fSRafael Auler         FrameInstructions.emplace_back(
2692a34c753fSRafael Auler             MCCFIInstruction::createRestore(nullptr, Reg));
2693a34c753fSRafael Auler         if (FromCFITable.isRedundant(FrameInstructions.back())) {
2694a34c753fSRafael Auler           FrameInstructions.pop_back();
2695a34c753fSRafael Auler           break;
2696a34c753fSRafael Auler         }
2697a34c753fSRafael Auler         NewStates.push_back(FrameInstructions.size() - 1);
2698a34c753fSRafael Auler         InsertIt = addCFIPseudo(InBB, InsertIt, FrameInstructions.size() - 1);
2699a34c753fSRafael Auler         ++InsertIt;
2700a34c753fSRafael Auler         break;
2701a34c753fSRafael Auler       }
2702a34c753fSRafael Auler       const int32_t Rule = ToCFITable.RegRule[Reg];
2703a34c753fSRafael Auler       if (Rule < 0) {
2704a34c753fSRafael Auler         if (FromCFITable.isRedundant(CIEFrameInstructions[-Rule]))
2705a34c753fSRafael Auler           break;
2706a34c753fSRafael Auler         NewStates.push_back(FrameInstructions.size());
2707a34c753fSRafael Auler         InsertIt = addCFIPseudo(InBB, InsertIt, FrameInstructions.size());
2708a34c753fSRafael Auler         ++InsertIt;
2709a34c753fSRafael Auler         FrameInstructions.emplace_back(CIEFrameInstructions[-Rule]);
2710a34c753fSRafael Auler         break;
2711a34c753fSRafael Auler       }
2712a34c753fSRafael Auler       if (FromCFITable.isRedundant(FrameInstructions[Rule]))
2713a34c753fSRafael Auler         break;
2714a34c753fSRafael Auler       NewStates.push_back(Rule);
2715a34c753fSRafael Auler       InsertIt = addCFIPseudo(InBB, InsertIt, Rule);
2716a34c753fSRafael Auler       ++InsertIt;
2717a34c753fSRafael Auler       break;
2718a34c753fSRafael Auler     }
2719a34c753fSRafael Auler     case MCCFIInstruction::OpDefCfaRegister:
2720a34c753fSRafael Auler     case MCCFIInstruction::OpDefCfaOffset:
2721a34c753fSRafael Auler     case MCCFIInstruction::OpDefCfa:
2722a34c753fSRafael Auler       undoStateDefCfa();
2723a34c753fSRafael Auler       break;
2724a34c753fSRafael Auler     case MCCFIInstruction::OpAdjustCfaOffset:
2725a34c753fSRafael Auler     case MCCFIInstruction::OpWindowSave:
2726a34c753fSRafael Auler     case MCCFIInstruction::OpNegateRAState:
2727a34c753fSRafael Auler     case MCCFIInstruction::OpLLVMDefAspaceCfa:
2728a34c753fSRafael Auler       llvm_unreachable("unsupported CFI opcode");
2729a34c753fSRafael Auler       break;
2730a34c753fSRafael Auler     case MCCFIInstruction::OpGnuArgsSize:
2731a34c753fSRafael Auler       // do not affect CFI state
2732a34c753fSRafael Auler       break;
2733a34c753fSRafael Auler     }
2734a34c753fSRafael Auler   };
2735a34c753fSRafael Auler 
2736a34c753fSRafael Auler   // Undo all modifications from ToState to FromState
2737a34c753fSRafael Auler   for (int32_t I = ToState, E = FromState; I != E; ++I) {
2738a34c753fSRafael Auler     const MCCFIInstruction &Instr = FrameInstructions[I];
2739a34c753fSRafael Auler     if (Instr.getOperation() != MCCFIInstruction::OpRestoreState) {
2740a34c753fSRafael Auler       undoState(Instr);
2741a34c753fSRafael Auler       continue;
2742a34c753fSRafael Auler     }
2743a34c753fSRafael Auler     auto Iter = FrameRestoreEquivalents.find(I);
2744a34c753fSRafael Auler     if (Iter == FrameRestoreEquivalents.end())
2745a34c753fSRafael Auler       continue;
2746a34c753fSRafael Auler     for (int32_t State : Iter->second)
2747a34c753fSRafael Auler       undoState(FrameInstructions[State]);
2748a34c753fSRafael Auler   }
2749a34c753fSRafael Auler 
2750a34c753fSRafael Auler   return NewStates;
2751a34c753fSRafael Auler }
2752a34c753fSRafael Auler 
2753a34c753fSRafael Auler void BinaryFunction::normalizeCFIState() {
2754a34c753fSRafael Auler   // Reordering blocks with remember-restore state instructions can be specially
2755a34c753fSRafael Auler   // tricky. When rewriting the CFI, we omit remember-restore state instructions
2756a34c753fSRafael Auler   // entirely. For restore state, we build a map expanding each restore to the
2757a34c753fSRafael Auler   // equivalent unwindCFIState sequence required at that point to achieve the
2758a34c753fSRafael Auler   // same effect of the restore. All remember state are then just ignored.
2759a34c753fSRafael Auler   std::stack<int32_t> Stack;
2760a34c753fSRafael Auler   for (BinaryBasicBlock *CurBB : BasicBlocksLayout) {
2761a34c753fSRafael Auler     for (auto II = CurBB->begin(); II != CurBB->end(); ++II) {
2762a34c753fSRafael Auler       if (const MCCFIInstruction *CFI = getCFIFor(*II)) {
2763a34c753fSRafael Auler         if (CFI->getOperation() == MCCFIInstruction::OpRememberState) {
2764a34c753fSRafael Auler           Stack.push(II->getOperand(0).getImm());
2765a34c753fSRafael Auler           continue;
2766a34c753fSRafael Auler         }
2767a34c753fSRafael Auler         if (CFI->getOperation() == MCCFIInstruction::OpRestoreState) {
2768a34c753fSRafael Auler           const int32_t RememberState = Stack.top();
2769a34c753fSRafael Auler           const int32_t CurState = II->getOperand(0).getImm();
2770a34c753fSRafael Auler           FrameRestoreEquivalents[CurState] =
2771a34c753fSRafael Auler               unwindCFIState(CurState, RememberState, CurBB, II);
2772a34c753fSRafael Auler           Stack.pop();
2773a34c753fSRafael Auler         }
2774a34c753fSRafael Auler       }
2775a34c753fSRafael Auler     }
2776a34c753fSRafael Auler   }
2777a34c753fSRafael Auler }
2778a34c753fSRafael Auler 
2779a34c753fSRafael Auler bool BinaryFunction::finalizeCFIState() {
2780a34c753fSRafael Auler   LLVM_DEBUG(
2781a34c753fSRafael Auler       dbgs() << "Trying to fix CFI states for each BB after reordering.\n");
2782a34c753fSRafael Auler   LLVM_DEBUG(dbgs() << "This is the list of CFI states for each BB of " << *this
2783a34c753fSRafael Auler                     << ": ");
2784a34c753fSRafael Auler 
2785a34c753fSRafael Auler   int32_t State = 0;
2786a34c753fSRafael Auler   bool SeenCold = false;
2787a34c753fSRafael Auler   const char *Sep = "";
2788a34c753fSRafael Auler   (void)Sep;
2789a34c753fSRafael Auler   for (BinaryBasicBlock *BB : BasicBlocksLayout) {
2790a34c753fSRafael Auler     const int32_t CFIStateAtExit = BB->getCFIStateAtExit();
2791a34c753fSRafael Auler 
2792a34c753fSRafael Auler     // Hot-cold border: check if this is the first BB to be allocated in a cold
2793a34c753fSRafael Auler     // region (with a different FDE). If yes, we need to reset the CFI state.
2794a34c753fSRafael Auler     if (!SeenCold && BB->isCold()) {
2795a34c753fSRafael Auler       State = 0;
2796a34c753fSRafael Auler       SeenCold = true;
2797a34c753fSRafael Auler     }
2798a34c753fSRafael Auler 
2799a34c753fSRafael Auler     // We need to recover the correct state if it doesn't match expected
2800a34c753fSRafael Auler     // state at BB entry point.
2801a34c753fSRafael Auler     if (BB->getCFIState() < State) {
2802a34c753fSRafael Auler       // In this case, State is currently higher than what this BB expect it
2803a34c753fSRafael Auler       // to be. To solve this, we need to insert CFI instructions to undo
2804a34c753fSRafael Auler       // the effect of all CFI from BB's state to current State.
2805a34c753fSRafael Auler       auto InsertIt = BB->begin();
2806a34c753fSRafael Auler       unwindCFIState(State, BB->getCFIState(), BB, InsertIt);
2807a34c753fSRafael Auler     } else if (BB->getCFIState() > State) {
2808a34c753fSRafael Auler       // If BB's CFI state is greater than State, it means we are behind in the
2809a34c753fSRafael Auler       // state. Just emit all instructions to reach this state at the
2810a34c753fSRafael Auler       // beginning of this BB. If this sequence of instructions involve
2811a34c753fSRafael Auler       // remember state or restore state, bail out.
2812a34c753fSRafael Auler       if (!replayCFIInstrs(State, BB->getCFIState(), BB, BB->begin()))
2813a34c753fSRafael Auler         return false;
2814a34c753fSRafael Auler     }
2815a34c753fSRafael Auler 
2816a34c753fSRafael Auler     State = CFIStateAtExit;
2817a34c753fSRafael Auler     LLVM_DEBUG(dbgs() << Sep << State; Sep = ", ");
2818a34c753fSRafael Auler   }
2819a34c753fSRafael Auler   LLVM_DEBUG(dbgs() << "\n");
2820a34c753fSRafael Auler 
2821a34c753fSRafael Auler   for (BinaryBasicBlock *BB : BasicBlocksLayout) {
2822a34c753fSRafael Auler     for (auto II = BB->begin(); II != BB->end();) {
2823a34c753fSRafael Auler       const MCCFIInstruction *CFI = getCFIFor(*II);
282440c2e0faSMaksim Panchenko       if (CFI && (CFI->getOperation() == MCCFIInstruction::OpRememberState ||
2825a34c753fSRafael Auler                   CFI->getOperation() == MCCFIInstruction::OpRestoreState)) {
2826a34c753fSRafael Auler         II = BB->eraseInstruction(II);
2827a34c753fSRafael Auler       } else {
2828a34c753fSRafael Auler         ++II;
2829a34c753fSRafael Auler       }
2830a34c753fSRafael Auler     }
2831a34c753fSRafael Auler   }
2832a34c753fSRafael Auler 
2833a34c753fSRafael Auler   return true;
2834a34c753fSRafael Auler }
2835a34c753fSRafael Auler 
2836a34c753fSRafael Auler bool BinaryFunction::requiresAddressTranslation() const {
2837a34c753fSRafael Auler   return opts::EnableBAT || hasSDTMarker() || hasPseudoProbe();
2838a34c753fSRafael Auler }
2839a34c753fSRafael Auler 
2840a34c753fSRafael Auler uint64_t BinaryFunction::getInstructionCount() const {
2841a34c753fSRafael Auler   uint64_t Count = 0;
28423652483cSRafael Auler   for (BinaryBasicBlock *const &Block : BasicBlocksLayout)
2843a34c753fSRafael Auler     Count += Block->getNumNonPseudos();
2844a34c753fSRafael Auler   return Count;
2845a34c753fSRafael Auler }
2846a34c753fSRafael Auler 
284740c2e0faSMaksim Panchenko bool BinaryFunction::hasLayoutChanged() const { return ModifiedLayout; }
2848a34c753fSRafael Auler 
2849a34c753fSRafael Auler uint64_t BinaryFunction::getEditDistance() const {
2850a34c753fSRafael Auler   return ComputeEditDistance<BinaryBasicBlock *>(BasicBlocksPreviousLayout,
2851a34c753fSRafael Auler                                                  BasicBlocksLayout);
2852a34c753fSRafael Auler }
2853a34c753fSRafael Auler 
2854a34c753fSRafael Auler void BinaryFunction::clearDisasmState() {
2855a34c753fSRafael Auler   clearList(Instructions);
2856a34c753fSRafael Auler   clearList(IgnoredBranches);
2857a34c753fSRafael Auler   clearList(TakenBranches);
2858a34c753fSRafael Auler 
2859a34c753fSRafael Auler   if (BC.HasRelocations) {
28603652483cSRafael Auler     for (std::pair<const uint32_t, MCSymbol *> &LI : Labels)
2861a34c753fSRafael Auler       BC.UndefinedSymbols.insert(LI.second);
28623652483cSRafael Auler     if (FunctionEndLabel)
2863a34c753fSRafael Auler       BC.UndefinedSymbols.insert(FunctionEndLabel);
2864a34c753fSRafael Auler   }
2865a34c753fSRafael Auler }
2866a34c753fSRafael Auler 
2867a34c753fSRafael Auler void BinaryFunction::setTrapOnEntry() {
2868a34c753fSRafael Auler   clearDisasmState();
2869a34c753fSRafael Auler 
2870a34c753fSRafael Auler   auto addTrapAtOffset = [&](uint64_t Offset) {
2871a34c753fSRafael Auler     MCInst TrapInstr;
2872a34c753fSRafael Auler     BC.MIB->createTrap(TrapInstr);
2873a34c753fSRafael Auler     addInstruction(Offset, std::move(TrapInstr));
2874a34c753fSRafael Auler   };
2875a34c753fSRafael Auler 
2876a34c753fSRafael Auler   addTrapAtOffset(0);
28773652483cSRafael Auler   for (const std::pair<const uint32_t, MCSymbol *> &KV : getLabels())
28783652483cSRafael Auler     if (getSecondaryEntryPointSymbol(KV.second))
2879a34c753fSRafael Auler       addTrapAtOffset(KV.first);
2880a34c753fSRafael Auler 
2881a34c753fSRafael Auler   TrapsOnEntry = true;
2882a34c753fSRafael Auler }
2883a34c753fSRafael Auler 
2884a34c753fSRafael Auler void BinaryFunction::setIgnored() {
2885a34c753fSRafael Auler   if (opts::processAllFunctions()) {
2886a34c753fSRafael Auler     // We can accept ignored functions before they've been disassembled.
2887a34c753fSRafael Auler     // In that case, they would still get disassembled and emited, but not
2888a34c753fSRafael Auler     // optimized.
2889a34c753fSRafael Auler     assert(CurrentState == State::Empty &&
2890a34c753fSRafael Auler            "cannot ignore non-empty functions in current mode");
2891a34c753fSRafael Auler     IsIgnored = true;
2892a34c753fSRafael Auler     return;
2893a34c753fSRafael Auler   }
2894a34c753fSRafael Auler 
2895a34c753fSRafael Auler   clearDisasmState();
2896a34c753fSRafael Auler 
2897a34c753fSRafael Auler   // Clear CFG state too.
2898a34c753fSRafael Auler   if (hasCFG()) {
2899a34c753fSRafael Auler     releaseCFG();
2900a34c753fSRafael Auler 
29013652483cSRafael Auler     for (BinaryBasicBlock *BB : BasicBlocks)
2902a34c753fSRafael Auler       delete BB;
2903a34c753fSRafael Auler     clearList(BasicBlocks);
2904a34c753fSRafael Auler 
29053652483cSRafael Auler     for (BinaryBasicBlock *BB : DeletedBasicBlocks)
2906a34c753fSRafael Auler       delete BB;
2907a34c753fSRafael Auler     clearList(DeletedBasicBlocks);
2908a34c753fSRafael Auler 
2909a34c753fSRafael Auler     clearList(BasicBlocksLayout);
2910a34c753fSRafael Auler     clearList(BasicBlocksPreviousLayout);
2911a34c753fSRafael Auler   }
2912a34c753fSRafael Auler 
2913a34c753fSRafael Auler   CurrentState = State::Empty;
2914a34c753fSRafael Auler 
2915a34c753fSRafael Auler   IsIgnored = true;
2916a34c753fSRafael Auler   IsSimple = false;
2917a34c753fSRafael Auler   LLVM_DEBUG(dbgs() << "Ignoring " << getPrintName() << '\n');
2918a34c753fSRafael Auler }
2919a34c753fSRafael Auler 
2920a34c753fSRafael Auler void BinaryFunction::duplicateConstantIslands() {
2921a34c753fSRafael Auler   assert(Islands && "function expected to have constant islands");
2922a34c753fSRafael Auler 
2923a34c753fSRafael Auler   for (BinaryBasicBlock *BB : layout()) {
2924a34c753fSRafael Auler     if (!BB->isCold())
2925a34c753fSRafael Auler       continue;
2926a34c753fSRafael Auler 
2927a34c753fSRafael Auler     for (MCInst &Inst : *BB) {
2928a34c753fSRafael Auler       int OpNum = 0;
2929a34c753fSRafael Auler       for (MCOperand &Operand : Inst) {
2930a34c753fSRafael Auler         if (!Operand.isExpr()) {
2931a34c753fSRafael Auler           ++OpNum;
2932a34c753fSRafael Auler           continue;
2933a34c753fSRafael Auler         }
2934a34c753fSRafael Auler         const MCSymbol *Symbol = BC.MIB->getTargetSymbol(Inst, OpNum);
2935a34c753fSRafael Auler         // Check if this is an island symbol
2936a34c753fSRafael Auler         if (!Islands->Symbols.count(Symbol) &&
2937a34c753fSRafael Auler             !Islands->ProxySymbols.count(Symbol))
2938a34c753fSRafael Auler           continue;
2939a34c753fSRafael Auler 
2940a34c753fSRafael Auler         // Create cold symbol, if missing
2941a34c753fSRafael Auler         auto ISym = Islands->ColdSymbols.find(Symbol);
2942a34c753fSRafael Auler         MCSymbol *ColdSymbol;
2943a34c753fSRafael Auler         if (ISym != Islands->ColdSymbols.end()) {
2944a34c753fSRafael Auler           ColdSymbol = ISym->second;
2945a34c753fSRafael Auler         } else {
2946a34c753fSRafael Auler           ColdSymbol = BC.Ctx->getOrCreateSymbol(Symbol->getName() + ".cold");
2947a34c753fSRafael Auler           Islands->ColdSymbols[Symbol] = ColdSymbol;
2948a34c753fSRafael Auler           // Check if this is a proxy island symbol and update owner proxy map
2949a34c753fSRafael Auler           if (Islands->ProxySymbols.count(Symbol)) {
2950a34c753fSRafael Auler             BinaryFunction *Owner = Islands->ProxySymbols[Symbol];
2951a34c753fSRafael Auler             auto IProxiedSym = Owner->Islands->Proxies[this].find(Symbol);
2952a34c753fSRafael Auler             Owner->Islands->ColdProxies[this][IProxiedSym->second] = ColdSymbol;
2953a34c753fSRafael Auler           }
2954a34c753fSRafael Auler         }
2955a34c753fSRafael Auler 
2956a34c753fSRafael Auler         // Update instruction reference
2957a34c753fSRafael Auler         Operand = MCOperand::createExpr(BC.MIB->getTargetExprFor(
2958a34c753fSRafael Auler             Inst,
2959a34c753fSRafael Auler             MCSymbolRefExpr::create(ColdSymbol, MCSymbolRefExpr::VK_None,
2960a34c753fSRafael Auler                                     *BC.Ctx),
2961a34c753fSRafael Auler             *BC.Ctx, 0));
2962a34c753fSRafael Auler         ++OpNum;
2963a34c753fSRafael Auler       }
2964a34c753fSRafael Auler     }
2965a34c753fSRafael Auler   }
2966a34c753fSRafael Auler }
2967a34c753fSRafael Auler 
2968a34c753fSRafael Auler namespace {
2969a34c753fSRafael Auler 
2970a34c753fSRafael Auler #ifndef MAX_PATH
2971a34c753fSRafael Auler #define MAX_PATH 255
2972a34c753fSRafael Auler #endif
2973a34c753fSRafael Auler 
297440c2e0faSMaksim Panchenko std::string constructFilename(std::string Filename, std::string Annotation,
2975a34c753fSRafael Auler                               std::string Suffix) {
2976a34c753fSRafael Auler   std::replace(Filename.begin(), Filename.end(), '/', '-');
29773652483cSRafael Auler   if (!Annotation.empty())
2978a34c753fSRafael Auler     Annotation.insert(0, "-");
2979a34c753fSRafael Auler   if (Filename.size() + Annotation.size() + Suffix.size() > MAX_PATH) {
2980a34c753fSRafael Auler     assert(Suffix.size() + Annotation.size() <= MAX_PATH);
2981a34c753fSRafael Auler     if (opts::Verbosity >= 1) {
2982a34c753fSRafael Auler       errs() << "BOLT-WARNING: Filename \"" << Filename << Annotation << Suffix
2983a34c753fSRafael Auler              << "\" exceeds the " << MAX_PATH << " size limit, truncating.\n";
2984a34c753fSRafael Auler     }
2985a34c753fSRafael Auler     Filename.resize(MAX_PATH - (Suffix.size() + Annotation.size()));
2986a34c753fSRafael Auler   }
2987a34c753fSRafael Auler   Filename += Annotation;
2988a34c753fSRafael Auler   Filename += Suffix;
2989a34c753fSRafael Auler   return Filename;
2990a34c753fSRafael Auler }
2991a34c753fSRafael Auler 
2992a34c753fSRafael Auler std::string formatEscapes(const std::string &Str) {
2993a34c753fSRafael Auler   std::string Result;
2994a34c753fSRafael Auler   for (unsigned I = 0; I < Str.size(); ++I) {
2995a34c753fSRafael Auler     char C = Str[I];
2996a34c753fSRafael Auler     switch (C) {
2997a34c753fSRafael Auler     case '\n':
2998a34c753fSRafael Auler       Result += "&#13;";
2999a34c753fSRafael Auler       break;
3000a34c753fSRafael Auler     case '"':
3001a34c753fSRafael Auler       break;
3002a34c753fSRafael Auler     default:
3003a34c753fSRafael Auler       Result += C;
3004a34c753fSRafael Auler       break;
3005a34c753fSRafael Auler     }
3006a34c753fSRafael Auler   }
3007a34c753fSRafael Auler   return Result;
3008a34c753fSRafael Auler }
3009a34c753fSRafael Auler 
301040c2e0faSMaksim Panchenko } // namespace
3011a34c753fSRafael Auler 
3012a34c753fSRafael Auler void BinaryFunction::dumpGraph(raw_ostream &OS) const {
30136333e5ddSAmir Ayupov   OS << "digraph \"" << getPrintName() << "\" {\n"
30146333e5ddSAmir Ayupov      << "node [fontname=courier, shape=box, style=filled, colorscheme=brbg9]\n";
3015a34c753fSRafael Auler   uint64_t Offset = Address;
3016a34c753fSRafael Auler   for (BinaryBasicBlock *BB : BasicBlocks) {
3017d2c87699SAmir Ayupov     auto LayoutPos = llvm::find(BasicBlocksLayout, BB);
3018a34c753fSRafael Auler     unsigned Layout = LayoutPos - BasicBlocksLayout.begin();
3019a34c753fSRafael Auler     const char *ColdStr = BB->isCold() ? " (cold)" : "";
30206333e5ddSAmir Ayupov     std::vector<std::string> Attrs;
30216333e5ddSAmir Ayupov     // Bold box for entry points
30226333e5ddSAmir Ayupov     if (isEntryPoint(*BB))
30236333e5ddSAmir Ayupov       Attrs.push_back("penwidth=2");
30246333e5ddSAmir Ayupov     if (BLI && BLI->getLoopFor(BB)) {
30256333e5ddSAmir Ayupov       // Distinguish innermost loops
30266333e5ddSAmir Ayupov       const BinaryLoop *Loop = BLI->getLoopFor(BB);
30276333e5ddSAmir Ayupov       if (Loop->isInnermost())
30286333e5ddSAmir Ayupov         Attrs.push_back("fillcolor=6");
30296333e5ddSAmir Ayupov       else // some outer loop
30306333e5ddSAmir Ayupov         Attrs.push_back("fillcolor=4");
30316333e5ddSAmir Ayupov     } else { // non-loopy code
30326333e5ddSAmir Ayupov       Attrs.push_back("fillcolor=5");
30336333e5ddSAmir Ayupov     }
30346333e5ddSAmir Ayupov     ListSeparator LS;
30356333e5ddSAmir Ayupov     OS << "\"" << BB->getName() << "\" [";
30366333e5ddSAmir Ayupov     for (StringRef Attr : Attrs)
30376333e5ddSAmir Ayupov       OS << LS << Attr;
30386333e5ddSAmir Ayupov     OS << "]\n";
3039cc23c64fSAmir Ayupov     OS << format("\"%s\" [label=\"%s%s\\n(C:%lu,O:%lu,I:%u,L:%u,CFI:%u)\\n",
3040cc23c64fSAmir Ayupov                  BB->getName().data(), BB->getName().data(), ColdStr,
3041cc23c64fSAmir Ayupov                  BB->getKnownExecutionCount(), BB->getOffset(), getIndex(BB),
3042cc23c64fSAmir Ayupov                  Layout, BB->getCFIState());
3043cc23c64fSAmir Ayupov 
3044a34c753fSRafael Auler     if (opts::DotToolTipCode) {
3045a34c753fSRafael Auler       std::string Str;
3046a34c753fSRafael Auler       raw_string_ostream CS(Str);
3047cc23c64fSAmir Ayupov       Offset = BC.printInstructions(CS, BB->begin(), BB->end(), Offset, this,
3048cc23c64fSAmir Ayupov                                     /* PrintMCInst = */ false,
3049cc23c64fSAmir Ayupov                                     /* PrintMemData = */ false,
3050cc23c64fSAmir Ayupov                                     /* PrintRelocations = */ false,
3051cc23c64fSAmir Ayupov                                     /* Endl = */ R"(\\l)");
3052cc23c64fSAmir Ayupov       OS << formatEscapes(CS.str()) << '\n';
3053a34c753fSRafael Auler     }
3054cc23c64fSAmir Ayupov     OS << "\"]\n";
3055a34c753fSRafael Auler 
3056a34c753fSRafael Auler     // analyzeBranch is just used to get the names of the branch
3057a34c753fSRafael Auler     // opcodes.
3058a34c753fSRafael Auler     const MCSymbol *TBB = nullptr;
3059a34c753fSRafael Auler     const MCSymbol *FBB = nullptr;
3060a34c753fSRafael Auler     MCInst *CondBranch = nullptr;
3061a34c753fSRafael Auler     MCInst *UncondBranch = nullptr;
306240c2e0faSMaksim Panchenko     const bool Success = BB->analyzeBranch(TBB, FBB, CondBranch, UncondBranch);
3063a34c753fSRafael Auler 
3064a34c753fSRafael Auler     const MCInst *LastInstr = BB->getLastNonPseudoInstr();
3065a34c753fSRafael Auler     const bool IsJumpTable = LastInstr && BC.MIB->getJumpTable(*LastInstr);
3066a34c753fSRafael Auler 
3067a34c753fSRafael Auler     auto BI = BB->branch_info_begin();
3068a34c753fSRafael Auler     for (BinaryBasicBlock *Succ : BB->successors()) {
3069a34c753fSRafael Auler       std::string Branch;
3070a34c753fSRafael Auler       if (Success) {
3071a34c753fSRafael Auler         if (Succ == BB->getConditionalSuccessor(true)) {
3072a34c753fSRafael Auler           Branch = CondBranch ? std::string(BC.InstPrinter->getOpcodeName(
3073a34c753fSRafael Auler                                     CondBranch->getOpcode()))
3074a34c753fSRafael Auler                               : "TB";
3075a34c753fSRafael Auler         } else if (Succ == BB->getConditionalSuccessor(false)) {
3076a34c753fSRafael Auler           Branch = UncondBranch ? std::string(BC.InstPrinter->getOpcodeName(
3077a34c753fSRafael Auler                                       UncondBranch->getOpcode()))
3078a34c753fSRafael Auler                                 : "FB";
3079a34c753fSRafael Auler         } else {
3080a34c753fSRafael Auler           Branch = "FT";
3081a34c753fSRafael Auler         }
3082a34c753fSRafael Auler       }
30833652483cSRafael Auler       if (IsJumpTable)
3084a34c753fSRafael Auler         Branch = "JT";
308540c2e0faSMaksim Panchenko       OS << format("\"%s\" -> \"%s\" [label=\"%s", BB->getName().data(),
308640c2e0faSMaksim Panchenko                    Succ->getName().data(), Branch.c_str());
3087a34c753fSRafael Auler 
3088a34c753fSRafael Auler       if (BB->getExecutionCount() != COUNT_NO_PROFILE &&
3089a34c753fSRafael Auler           BI->MispredictedCount != BinaryBasicBlock::COUNT_INFERRED) {
3090a34c753fSRafael Auler         OS << "\\n(C:" << BI->Count << ",M:" << BI->MispredictedCount << ")";
3091a34c753fSRafael Auler       } else if (ExecutionCount != COUNT_NO_PROFILE &&
3092a34c753fSRafael Auler                  BI->Count != BinaryBasicBlock::COUNT_NO_PROFILE) {
3093a34c753fSRafael Auler         OS << "\\n(IC:" << BI->Count << ")";
3094a34c753fSRafael Auler       }
3095a34c753fSRafael Auler       OS << "\"]\n";
3096a34c753fSRafael Auler 
3097a34c753fSRafael Auler       ++BI;
3098a34c753fSRafael Auler     }
3099a34c753fSRafael Auler     for (BinaryBasicBlock *LP : BB->landing_pads()) {
3100a34c753fSRafael Auler       OS << format("\"%s\" -> \"%s\" [constraint=false style=dashed]\n",
310140c2e0faSMaksim Panchenko                    BB->getName().data(), LP->getName().data());
3102a34c753fSRafael Auler     }
3103a34c753fSRafael Auler   }
3104a34c753fSRafael Auler   OS << "}\n";
3105a34c753fSRafael Auler }
3106a34c753fSRafael Auler 
3107a34c753fSRafael Auler void BinaryFunction::viewGraph() const {
3108a34c753fSRafael Auler   SmallString<MAX_PATH> Filename;
3109a34c753fSRafael Auler   if (std::error_code EC =
3110a34c753fSRafael Auler           sys::fs::createTemporaryFile("bolt-cfg", "dot", Filename)) {
3111a34c753fSRafael Auler     errs() << "BOLT-ERROR: " << EC.message() << ", unable to create "
3112a34c753fSRafael Auler            << " bolt-cfg-XXXXX.dot temporary file.\n";
3113a34c753fSRafael Auler     return;
3114a34c753fSRafael Auler   }
3115a34c753fSRafael Auler   dumpGraphToFile(std::string(Filename));
31163652483cSRafael Auler   if (DisplayGraph(Filename))
3117a34c753fSRafael Auler     errs() << "BOLT-ERROR: Can't display " << Filename << " with graphviz.\n";
3118a34c753fSRafael Auler   if (std::error_code EC = sys::fs::remove(Filename)) {
3119a34c753fSRafael Auler     errs() << "BOLT-WARNING: " << EC.message() << ", failed to remove "
3120a34c753fSRafael Auler            << Filename << "\n";
3121a34c753fSRafael Auler   }
3122a34c753fSRafael Auler }
3123a34c753fSRafael Auler 
3124a34c753fSRafael Auler void BinaryFunction::dumpGraphForPass(std::string Annotation) const {
3125798e92c6SAmir Ayupov   if (!opts::shouldPrint(*this))
3126798e92c6SAmir Ayupov     return;
3127798e92c6SAmir Ayupov 
3128a34c753fSRafael Auler   std::string Filename = constructFilename(getPrintName(), Annotation, ".dot");
3129798e92c6SAmir Ayupov   if (opts::Verbosity >= 1)
3130798e92c6SAmir Ayupov     outs() << "BOLT-INFO: dumping CFG to " << Filename << "\n";
3131a34c753fSRafael Auler   dumpGraphToFile(Filename);
3132a34c753fSRafael Auler }
3133a34c753fSRafael Auler 
3134a34c753fSRafael Auler void BinaryFunction::dumpGraphToFile(std::string Filename) const {
3135a34c753fSRafael Auler   std::error_code EC;
3136a34c753fSRafael Auler   raw_fd_ostream of(Filename, EC, sys::fs::OF_None);
3137a34c753fSRafael Auler   if (EC) {
3138a34c753fSRafael Auler     if (opts::Verbosity >= 1) {
3139a34c753fSRafael Auler       errs() << "BOLT-WARNING: " << EC.message() << ", unable to open "
3140a34c753fSRafael Auler              << Filename << " for output.\n";
3141a34c753fSRafael Auler     }
3142a34c753fSRafael Auler     return;
3143a34c753fSRafael Auler   }
3144a34c753fSRafael Auler   dumpGraph(of);
3145a34c753fSRafael Auler }
3146a34c753fSRafael Auler 
3147a34c753fSRafael Auler bool BinaryFunction::validateCFG() const {
3148a34c753fSRafael Auler   bool Valid = true;
31493652483cSRafael Auler   for (BinaryBasicBlock *BB : BasicBlocks)
3150a34c753fSRafael Auler     Valid &= BB->validateSuccessorInvariants();
3151a34c753fSRafael Auler 
3152a34c753fSRafael Auler   if (!Valid)
3153a34c753fSRafael Auler     return Valid;
3154a34c753fSRafael Auler 
3155a34c753fSRafael Auler   // Make sure all blocks in CFG are valid.
3156a34c753fSRafael Auler   auto validateBlock = [this](const BinaryBasicBlock *BB, StringRef Desc) {
3157a34c753fSRafael Auler     if (!BB->isValid()) {
3158a34c753fSRafael Auler       errs() << "BOLT-ERROR: deleted " << Desc << " " << BB->getName()
3159a34c753fSRafael Auler              << " detected in:\n";
3160a34c753fSRafael Auler       this->dump();
3161a34c753fSRafael Auler       return false;
3162a34c753fSRafael Auler     }
3163a34c753fSRafael Auler     return true;
3164a34c753fSRafael Auler   };
3165a34c753fSRafael Auler   for (const BinaryBasicBlock *BB : BasicBlocks) {
3166a34c753fSRafael Auler     if (!validateBlock(BB, "block"))
3167a34c753fSRafael Auler       return false;
3168a34c753fSRafael Auler     for (const BinaryBasicBlock *PredBB : BB->predecessors())
3169a34c753fSRafael Auler       if (!validateBlock(PredBB, "predecessor"))
3170a34c753fSRafael Auler         return false;
3171a34c753fSRafael Auler     for (const BinaryBasicBlock *SuccBB : BB->successors())
3172a34c753fSRafael Auler       if (!validateBlock(SuccBB, "successor"))
3173a34c753fSRafael Auler         return false;
3174a34c753fSRafael Auler     for (const BinaryBasicBlock *LP : BB->landing_pads())
3175a34c753fSRafael Auler       if (!validateBlock(LP, "landing pad"))
3176a34c753fSRafael Auler         return false;
3177a34c753fSRafael Auler     for (const BinaryBasicBlock *Thrower : BB->throwers())
3178a34c753fSRafael Auler       if (!validateBlock(Thrower, "thrower"))
3179a34c753fSRafael Auler         return false;
3180a34c753fSRafael Auler   }
3181a34c753fSRafael Auler 
3182a34c753fSRafael Auler   for (const BinaryBasicBlock *BB : BasicBlocks) {
3183a34c753fSRafael Auler     std::unordered_set<const BinaryBasicBlock *> BBLandingPads;
3184a34c753fSRafael Auler     for (const BinaryBasicBlock *LP : BB->landing_pads()) {
3185a34c753fSRafael Auler       if (BBLandingPads.count(LP)) {
3186a34c753fSRafael Auler         errs() << "BOLT-ERROR: duplicate landing pad detected in"
3187a34c753fSRafael Auler                << BB->getName() << " in function " << *this << '\n';
3188a34c753fSRafael Auler         return false;
3189a34c753fSRafael Auler       }
3190a34c753fSRafael Auler       BBLandingPads.insert(LP);
3191a34c753fSRafael Auler     }
3192a34c753fSRafael Auler 
3193a34c753fSRafael Auler     std::unordered_set<const BinaryBasicBlock *> BBThrowers;
3194a34c753fSRafael Auler     for (const BinaryBasicBlock *Thrower : BB->throwers()) {
3195a34c753fSRafael Auler       if (BBThrowers.count(Thrower)) {
319640c2e0faSMaksim Panchenko         errs() << "BOLT-ERROR: duplicate thrower detected in" << BB->getName()
319740c2e0faSMaksim Panchenko                << " in function " << *this << '\n';
3198a34c753fSRafael Auler         return false;
3199a34c753fSRafael Auler       }
3200a34c753fSRafael Auler       BBThrowers.insert(Thrower);
3201a34c753fSRafael Auler     }
3202a34c753fSRafael Auler 
3203a34c753fSRafael Auler     for (const BinaryBasicBlock *LPBlock : BB->landing_pads()) {
3204d2c87699SAmir Ayupov       if (!llvm::is_contained(LPBlock->throwers(), BB)) {
320540c2e0faSMaksim Panchenko         errs() << "BOLT-ERROR: inconsistent landing pad detected in " << *this
320640c2e0faSMaksim Panchenko                << ": " << BB->getName() << " is in LandingPads but not in "
320740c2e0faSMaksim Panchenko                << LPBlock->getName() << " Throwers\n";
3208a34c753fSRafael Auler         return false;
3209a34c753fSRafael Auler       }
3210a34c753fSRafael Auler     }
3211a34c753fSRafael Auler     for (const BinaryBasicBlock *Thrower : BB->throwers()) {
3212d2c87699SAmir Ayupov       if (!llvm::is_contained(Thrower->landing_pads(), BB)) {
321340c2e0faSMaksim Panchenko         errs() << "BOLT-ERROR: inconsistent thrower detected in " << *this
321440c2e0faSMaksim Panchenko                << ": " << BB->getName() << " is in Throwers list but not in "
321540c2e0faSMaksim Panchenko                << Thrower->getName() << " LandingPads\n";
3216a34c753fSRafael Auler         return false;
3217a34c753fSRafael Auler       }
3218a34c753fSRafael Auler     }
3219a34c753fSRafael Auler   }
3220a34c753fSRafael Auler 
3221a34c753fSRafael Auler   return Valid;
3222a34c753fSRafael Auler }
3223a34c753fSRafael Auler 
3224a34c753fSRafael Auler void BinaryFunction::fixBranches() {
3225a34c753fSRafael Auler   auto &MIB = BC.MIB;
3226a34c753fSRafael Auler   MCContext *Ctx = BC.Ctx.get();
3227a34c753fSRafael Auler 
3228a34c753fSRafael Auler   for (unsigned I = 0, E = BasicBlocksLayout.size(); I != E; ++I) {
3229a34c753fSRafael Auler     BinaryBasicBlock *BB = BasicBlocksLayout[I];
3230a34c753fSRafael Auler     const MCSymbol *TBB = nullptr;
3231a34c753fSRafael Auler     const MCSymbol *FBB = nullptr;
3232a34c753fSRafael Auler     MCInst *CondBranch = nullptr;
3233a34c753fSRafael Auler     MCInst *UncondBranch = nullptr;
3234a34c753fSRafael Auler     if (!BB->analyzeBranch(TBB, FBB, CondBranch, UncondBranch))
3235a34c753fSRafael Auler       continue;
3236a34c753fSRafael Auler 
3237a34c753fSRafael Auler     // We will create unconditional branch with correct destination if needed.
3238a34c753fSRafael Auler     if (UncondBranch)
3239a34c753fSRafael Auler       BB->eraseInstruction(BB->findInstruction(UncondBranch));
3240a34c753fSRafael Auler 
3241a34c753fSRafael Auler     // Basic block that follows the current one in the final layout.
3242a34c753fSRafael Auler     const BinaryBasicBlock *NextBB = nullptr;
3243a34c753fSRafael Auler     if (I + 1 != E && BB->isCold() == BasicBlocksLayout[I + 1]->isCold())
3244a34c753fSRafael Auler       NextBB = BasicBlocksLayout[I + 1];
3245a34c753fSRafael Auler 
3246a34c753fSRafael Auler     if (BB->succ_size() == 1) {
3247a34c753fSRafael Auler       // __builtin_unreachable() could create a conditional branch that
3248a34c753fSRafael Auler       // falls-through into the next function - hence the block will have only
3249a34c753fSRafael Auler       // one valid successor. Since behaviour is undefined - we replace
3250a34c753fSRafael Auler       // the conditional branch with an unconditional if required.
3251a34c753fSRafael Auler       if (CondBranch)
3252a34c753fSRafael Auler         BB->eraseInstruction(BB->findInstruction(CondBranch));
3253a34c753fSRafael Auler       if (BB->getSuccessor() == NextBB)
3254a34c753fSRafael Auler         continue;
3255a34c753fSRafael Auler       BB->addBranchInstruction(BB->getSuccessor());
3256a34c753fSRafael Auler     } else if (BB->succ_size() == 2) {
3257a34c753fSRafael Auler       assert(CondBranch && "conditional branch expected");
3258a34c753fSRafael Auler       const BinaryBasicBlock *TSuccessor = BB->getConditionalSuccessor(true);
3259a34c753fSRafael Auler       const BinaryBasicBlock *FSuccessor = BB->getConditionalSuccessor(false);
3260a34c753fSRafael Auler       // Check whether we support reversing this branch direction
3261a34c753fSRafael Auler       const bool IsSupported =
3262a34c753fSRafael Auler           !MIB->isUnsupportedBranch(CondBranch->getOpcode());
3263a34c753fSRafael Auler       if (NextBB && NextBB == TSuccessor && IsSupported) {
3264a34c753fSRafael Auler         std::swap(TSuccessor, FSuccessor);
3265a34c753fSRafael Auler         {
3266a34c753fSRafael Auler           auto L = BC.scopeLock();
3267a34c753fSRafael Auler           MIB->reverseBranchCondition(*CondBranch, TSuccessor->getLabel(), Ctx);
3268a34c753fSRafael Auler         }
3269a34c753fSRafael Auler         BB->swapConditionalSuccessors();
3270a34c753fSRafael Auler       } else {
3271a34c753fSRafael Auler         auto L = BC.scopeLock();
3272a34c753fSRafael Auler         MIB->replaceBranchTarget(*CondBranch, TSuccessor->getLabel(), Ctx);
3273a34c753fSRafael Auler       }
32743652483cSRafael Auler       if (TSuccessor == FSuccessor)
3275a34c753fSRafael Auler         BB->removeDuplicateConditionalSuccessor(CondBranch);
3276a34c753fSRafael Auler       if (!NextBB ||
3277a34c753fSRafael Auler           ((NextBB != TSuccessor || !IsSupported) && NextBB != FSuccessor)) {
3278a34c753fSRafael Auler         // If one of the branches is guaranteed to be "long" while the other
3279a34c753fSRafael Auler         // could be "short", then prioritize short for "taken". This will
3280a34c753fSRafael Auler         // generate a sequence 1 byte shorter on x86.
3281a34c753fSRafael Auler         if (IsSupported && BC.isX86() &&
3282a34c753fSRafael Auler             TSuccessor->isCold() != FSuccessor->isCold() &&
3283a34c753fSRafael Auler             BB->isCold() != TSuccessor->isCold()) {
3284a34c753fSRafael Auler           std::swap(TSuccessor, FSuccessor);
3285a34c753fSRafael Auler           {
3286a34c753fSRafael Auler             auto L = BC.scopeLock();
3287a34c753fSRafael Auler             MIB->reverseBranchCondition(*CondBranch, TSuccessor->getLabel(),
3288a34c753fSRafael Auler                                         Ctx);
3289a34c753fSRafael Auler           }
3290a34c753fSRafael Auler           BB->swapConditionalSuccessors();
3291a34c753fSRafael Auler         }
3292a34c753fSRafael Auler         BB->addBranchInstruction(FSuccessor);
3293a34c753fSRafael Auler       }
3294a34c753fSRafael Auler     }
3295a34c753fSRafael Auler     // Cases where the number of successors is 0 (block ends with a
3296a34c753fSRafael Auler     // terminator) or more than 2 (switch table) don't require branch
3297a34c753fSRafael Auler     // instruction adjustments.
3298a34c753fSRafael Auler   }
329940c2e0faSMaksim Panchenko   assert((!isSimple() || validateCFG()) &&
330040c2e0faSMaksim Panchenko          "Invalid CFG detected after fixing branches");
3301a34c753fSRafael Auler }
3302a34c753fSRafael Auler 
3303a34c753fSRafael Auler void BinaryFunction::propagateGnuArgsSizeInfo(
3304a34c753fSRafael Auler     MCPlusBuilder::AllocatorIdTy AllocId) {
3305a34c753fSRafael Auler   assert(CurrentState == State::Disassembled && "unexpected function state");
3306a34c753fSRafael Auler 
3307a34c753fSRafael Auler   if (!hasEHRanges() || !usesGnuArgsSize())
3308a34c753fSRafael Auler     return;
3309a34c753fSRafael Auler 
3310a34c753fSRafael Auler   // The current value of DW_CFA_GNU_args_size affects all following
3311a34c753fSRafael Auler   // invoke instructions until the next CFI overrides it.
3312a34c753fSRafael Auler   // It is important to iterate basic blocks in the original order when
3313a34c753fSRafael Auler   // assigning the value.
3314a34c753fSRafael Auler   uint64_t CurrentGnuArgsSize = 0;
3315a34c753fSRafael Auler   for (BinaryBasicBlock *BB : BasicBlocks) {
3316a34c753fSRafael Auler     for (auto II = BB->begin(); II != BB->end();) {
3317a34c753fSRafael Auler       MCInst &Instr = *II;
3318a34c753fSRafael Auler       if (BC.MIB->isCFI(Instr)) {
3319a34c753fSRafael Auler         const MCCFIInstruction *CFI = getCFIFor(Instr);
3320a34c753fSRafael Auler         if (CFI->getOperation() == MCCFIInstruction::OpGnuArgsSize) {
3321a34c753fSRafael Auler           CurrentGnuArgsSize = CFI->getOffset();
3322a34c753fSRafael Auler           // Delete DW_CFA_GNU_args_size instructions and only regenerate
3323a34c753fSRafael Auler           // during the final code emission. The information is embedded
3324a34c753fSRafael Auler           // inside call instructions.
3325a34c753fSRafael Auler           II = BB->erasePseudoInstruction(II);
3326a34c753fSRafael Auler           continue;
3327a34c753fSRafael Auler         }
3328a34c753fSRafael Auler       } else if (BC.MIB->isInvoke(Instr)) {
3329a34c753fSRafael Auler         // Add the value of GNU_args_size as an extra operand to invokes.
3330a34c753fSRafael Auler         BC.MIB->addGnuArgsSize(Instr, CurrentGnuArgsSize, AllocId);
3331a34c753fSRafael Auler       }
3332a34c753fSRafael Auler       ++II;
3333a34c753fSRafael Auler     }
3334a34c753fSRafael Auler   }
3335a34c753fSRafael Auler }
3336a34c753fSRafael Auler 
3337a34c753fSRafael Auler void BinaryFunction::postProcessBranches() {
3338a34c753fSRafael Auler   if (!isSimple())
3339a34c753fSRafael Auler     return;
3340a34c753fSRafael Auler   for (BinaryBasicBlock *BB : BasicBlocksLayout) {
3341a34c753fSRafael Auler     auto LastInstrRI = BB->getLastNonPseudo();
3342a34c753fSRafael Auler     if (BB->succ_size() == 1) {
3343a34c753fSRafael Auler       if (LastInstrRI != BB->rend() &&
3344a34c753fSRafael Auler           BC.MIB->isConditionalBranch(*LastInstrRI)) {
3345a34c753fSRafael Auler         // __builtin_unreachable() could create a conditional branch that
3346a34c753fSRafael Auler         // falls-through into the next function - hence the block will have only
3347a34c753fSRafael Auler         // one valid successor. Such behaviour is undefined and thus we remove
3348a34c753fSRafael Auler         // the conditional branch while leaving a valid successor.
3349a34c753fSRafael Auler         BB->eraseInstruction(std::prev(LastInstrRI.base()));
3350a34c753fSRafael Auler         LLVM_DEBUG(dbgs() << "BOLT-DEBUG: erasing conditional branch in "
3351a34c753fSRafael Auler                           << BB->getName() << " in function " << *this << '\n');
3352a34c753fSRafael Auler       }
3353a34c753fSRafael Auler     } else if (BB->succ_size() == 0) {
3354a34c753fSRafael Auler       // Ignore unreachable basic blocks.
3355a34c753fSRafael Auler       if (BB->pred_size() == 0 || BB->isLandingPad())
3356a34c753fSRafael Auler         continue;
3357a34c753fSRafael Auler 
3358a34c753fSRafael Auler       // If it's the basic block that does not end up with a terminator - we
3359a34c753fSRafael Auler       // insert a return instruction unless it's a call instruction.
3360a34c753fSRafael Auler       if (LastInstrRI == BB->rend()) {
3361a34c753fSRafael Auler         LLVM_DEBUG(
3362a34c753fSRafael Auler             dbgs() << "BOLT-DEBUG: at least one instruction expected in BB "
3363a34c753fSRafael Auler                    << BB->getName() << " in function " << *this << '\n');
3364a34c753fSRafael Auler         continue;
3365a34c753fSRafael Auler       }
3366a34c753fSRafael Auler       if (!BC.MIB->isTerminator(*LastInstrRI) &&
3367a34c753fSRafael Auler           !BC.MIB->isCall(*LastInstrRI)) {
3368a34c753fSRafael Auler         LLVM_DEBUG(dbgs() << "BOLT-DEBUG: adding return to basic block "
3369a34c753fSRafael Auler                           << BB->getName() << " in function " << *this << '\n');
3370a34c753fSRafael Auler         MCInst ReturnInstr;
3371a34c753fSRafael Auler         BC.MIB->createReturn(ReturnInstr);
3372a34c753fSRafael Auler         BB->addInstruction(ReturnInstr);
3373a34c753fSRafael Auler       }
3374a34c753fSRafael Auler     }
3375a34c753fSRafael Auler   }
3376a34c753fSRafael Auler   assert(validateCFG() && "invalid CFG");
3377a34c753fSRafael Auler }
3378a34c753fSRafael Auler 
3379a34c753fSRafael Auler MCSymbol *BinaryFunction::addEntryPointAtOffset(uint64_t Offset) {
3380a34c753fSRafael Auler   assert(Offset && "cannot add primary entry point");
3381a34c753fSRafael Auler   assert(CurrentState == State::Empty || CurrentState == State::Disassembled);
3382a34c753fSRafael Auler 
3383a34c753fSRafael Auler   const uint64_t EntryPointAddress = getAddress() + Offset;
3384a34c753fSRafael Auler   MCSymbol *LocalSymbol = getOrCreateLocalLabel(EntryPointAddress);
3385a34c753fSRafael Auler 
3386a34c753fSRafael Auler   MCSymbol *EntrySymbol = getSecondaryEntryPointSymbol(LocalSymbol);
3387a34c753fSRafael Auler   if (EntrySymbol)
3388a34c753fSRafael Auler     return EntrySymbol;
3389a34c753fSRafael Auler 
3390a34c753fSRafael Auler   if (BinaryData *EntryBD = BC.getBinaryDataAtAddress(EntryPointAddress)) {
3391a34c753fSRafael Auler     EntrySymbol = EntryBD->getSymbol();
3392a34c753fSRafael Auler   } else {
339340c2e0faSMaksim Panchenko     EntrySymbol = BC.getOrCreateGlobalSymbol(
339440c2e0faSMaksim Panchenko         EntryPointAddress, Twine("__ENTRY_") + getOneName() + "@");
3395a34c753fSRafael Auler   }
3396a34c753fSRafael Auler   SecondaryEntryPoints[LocalSymbol] = EntrySymbol;
3397a34c753fSRafael Auler 
3398a34c753fSRafael Auler   BC.setSymbolToFunctionMap(EntrySymbol, this);
3399a34c753fSRafael Auler 
3400a34c753fSRafael Auler   return EntrySymbol;
3401a34c753fSRafael Auler }
3402a34c753fSRafael Auler 
3403a34c753fSRafael Auler MCSymbol *BinaryFunction::addEntryPoint(const BinaryBasicBlock &BB) {
3404a34c753fSRafael Auler   assert(CurrentState == State::CFG &&
3405a34c753fSRafael Auler          "basic block can be added as an entry only in a function with CFG");
3406a34c753fSRafael Auler 
3407a34c753fSRafael Auler   if (&BB == BasicBlocks.front())
3408a34c753fSRafael Auler     return getSymbol();
3409a34c753fSRafael Auler 
3410a34c753fSRafael Auler   MCSymbol *EntrySymbol = getSecondaryEntryPointSymbol(BB);
3411a34c753fSRafael Auler   if (EntrySymbol)
3412a34c753fSRafael Auler     return EntrySymbol;
3413a34c753fSRafael Auler 
3414a34c753fSRafael Auler   EntrySymbol =
3415a34c753fSRafael Auler       BC.Ctx->getOrCreateSymbol("__ENTRY_" + BB.getLabel()->getName());
3416a34c753fSRafael Auler 
3417a34c753fSRafael Auler   SecondaryEntryPoints[BB.getLabel()] = EntrySymbol;
3418a34c753fSRafael Auler 
3419a34c753fSRafael Auler   BC.setSymbolToFunctionMap(EntrySymbol, this);
3420a34c753fSRafael Auler 
3421a34c753fSRafael Auler   return EntrySymbol;
3422a34c753fSRafael Auler }
3423a34c753fSRafael Auler 
3424a34c753fSRafael Auler MCSymbol *BinaryFunction::getSymbolForEntryID(uint64_t EntryID) {
3425a34c753fSRafael Auler   if (EntryID == 0)
3426a34c753fSRafael Auler     return getSymbol();
3427a34c753fSRafael Auler 
3428a34c753fSRafael Auler   if (!isMultiEntry())
3429a34c753fSRafael Auler     return nullptr;
3430a34c753fSRafael Auler 
3431a34c753fSRafael Auler   uint64_t NumEntries = 0;
3432a34c753fSRafael Auler   if (hasCFG()) {
3433a34c753fSRafael Auler     for (BinaryBasicBlock *BB : BasicBlocks) {
3434a34c753fSRafael Auler       MCSymbol *EntrySymbol = getSecondaryEntryPointSymbol(*BB);
3435a34c753fSRafael Auler       if (!EntrySymbol)
3436a34c753fSRafael Auler         continue;
3437a34c753fSRafael Auler       if (NumEntries == EntryID)
3438a34c753fSRafael Auler         return EntrySymbol;
3439a34c753fSRafael Auler       ++NumEntries;
3440a34c753fSRafael Auler     }
3441a34c753fSRafael Auler   } else {
3442a34c753fSRafael Auler     for (std::pair<const uint32_t, MCSymbol *> &KV : Labels) {
3443a34c753fSRafael Auler       MCSymbol *EntrySymbol = getSecondaryEntryPointSymbol(KV.second);
3444a34c753fSRafael Auler       if (!EntrySymbol)
3445a34c753fSRafael Auler         continue;
3446a34c753fSRafael Auler       if (NumEntries == EntryID)
3447a34c753fSRafael Auler         return EntrySymbol;
3448a34c753fSRafael Auler       ++NumEntries;
3449a34c753fSRafael Auler     }
3450a34c753fSRafael Auler   }
3451a34c753fSRafael Auler 
3452a34c753fSRafael Auler   return nullptr;
3453a34c753fSRafael Auler }
3454a34c753fSRafael Auler 
3455a34c753fSRafael Auler uint64_t BinaryFunction::getEntryIDForSymbol(const MCSymbol *Symbol) const {
3456a34c753fSRafael Auler   if (!isMultiEntry())
3457a34c753fSRafael Auler     return 0;
3458a34c753fSRafael Auler 
3459a34c753fSRafael Auler   for (const MCSymbol *FunctionSymbol : getSymbols())
3460a34c753fSRafael Auler     if (FunctionSymbol == Symbol)
3461a34c753fSRafael Auler       return 0;
3462a34c753fSRafael Auler 
3463a34c753fSRafael Auler   // Check all secondary entries available as either basic blocks or lables.
3464a34c753fSRafael Auler   uint64_t NumEntries = 0;
3465a34c753fSRafael Auler   for (const BinaryBasicBlock *BB : BasicBlocks) {
3466a34c753fSRafael Auler     MCSymbol *EntrySymbol = getSecondaryEntryPointSymbol(*BB);
3467a34c753fSRafael Auler     if (!EntrySymbol)
3468a34c753fSRafael Auler       continue;
3469a34c753fSRafael Auler     if (EntrySymbol == Symbol)
3470a34c753fSRafael Auler       return NumEntries;
3471a34c753fSRafael Auler     ++NumEntries;
3472a34c753fSRafael Auler   }
3473a34c753fSRafael Auler   NumEntries = 0;
3474a34c753fSRafael Auler   for (const std::pair<const uint32_t, MCSymbol *> &KV : Labels) {
3475a34c753fSRafael Auler     MCSymbol *EntrySymbol = getSecondaryEntryPointSymbol(KV.second);
3476a34c753fSRafael Auler     if (!EntrySymbol)
3477a34c753fSRafael Auler       continue;
3478a34c753fSRafael Auler     if (EntrySymbol == Symbol)
3479a34c753fSRafael Auler       return NumEntries;
3480a34c753fSRafael Auler     ++NumEntries;
3481a34c753fSRafael Auler   }
3482a34c753fSRafael Auler 
3483a34c753fSRafael Auler   llvm_unreachable("symbol not found");
3484a34c753fSRafael Auler }
3485a34c753fSRafael Auler 
3486a34c753fSRafael Auler bool BinaryFunction::forEachEntryPoint(EntryPointCallbackTy Callback) const {
3487a34c753fSRafael Auler   bool Status = Callback(0, getSymbol());
3488a34c753fSRafael Auler   if (!isMultiEntry())
3489a34c753fSRafael Auler     return Status;
3490a34c753fSRafael Auler 
3491a34c753fSRafael Auler   for (const std::pair<const uint32_t, MCSymbol *> &KV : Labels) {
3492a34c753fSRafael Auler     if (!Status)
3493a34c753fSRafael Auler       break;
3494a34c753fSRafael Auler 
3495a34c753fSRafael Auler     MCSymbol *EntrySymbol = getSecondaryEntryPointSymbol(KV.second);
3496a34c753fSRafael Auler     if (!EntrySymbol)
3497a34c753fSRafael Auler       continue;
3498a34c753fSRafael Auler 
3499a34c753fSRafael Auler     Status = Callback(KV.first, EntrySymbol);
3500a34c753fSRafael Auler   }
3501a34c753fSRafael Auler 
3502a34c753fSRafael Auler   return Status;
3503a34c753fSRafael Auler }
3504a34c753fSRafael Auler 
3505a34c753fSRafael Auler BinaryFunction::BasicBlockOrderType BinaryFunction::dfs() const {
3506a34c753fSRafael Auler   BasicBlockOrderType DFS;
3507a34c753fSRafael Auler   unsigned Index = 0;
3508a34c753fSRafael Auler   std::stack<BinaryBasicBlock *> Stack;
3509a34c753fSRafael Auler 
3510a34c753fSRafael Auler   // Push entry points to the stack in reverse order.
3511a34c753fSRafael Auler   //
3512a34c753fSRafael Auler   // NB: we rely on the original order of entries to match.
3513a34c753fSRafael Auler   for (auto BBI = layout_rbegin(); BBI != layout_rend(); ++BBI) {
3514a34c753fSRafael Auler     BinaryBasicBlock *BB = *BBI;
3515a34c753fSRafael Auler     if (isEntryPoint(*BB))
3516a34c753fSRafael Auler       Stack.push(BB);
3517a34c753fSRafael Auler     BB->setLayoutIndex(BinaryBasicBlock::InvalidIndex);
3518a34c753fSRafael Auler   }
3519a34c753fSRafael Auler 
3520a34c753fSRafael Auler   while (!Stack.empty()) {
3521a34c753fSRafael Auler     BinaryBasicBlock *BB = Stack.top();
3522a34c753fSRafael Auler     Stack.pop();
3523a34c753fSRafael Auler 
3524a34c753fSRafael Auler     if (BB->getLayoutIndex() != BinaryBasicBlock::InvalidIndex)
3525a34c753fSRafael Auler       continue;
3526a34c753fSRafael Auler 
3527a34c753fSRafael Auler     BB->setLayoutIndex(Index++);
3528a34c753fSRafael Auler     DFS.push_back(BB);
3529a34c753fSRafael Auler 
3530a34c753fSRafael Auler     for (BinaryBasicBlock *SuccBB : BB->landing_pads()) {
3531a34c753fSRafael Auler       Stack.push(SuccBB);
3532a34c753fSRafael Auler     }
3533a34c753fSRafael Auler 
3534a34c753fSRafael Auler     const MCSymbol *TBB = nullptr;
3535a34c753fSRafael Auler     const MCSymbol *FBB = nullptr;
3536a34c753fSRafael Auler     MCInst *CondBranch = nullptr;
3537a34c753fSRafael Auler     MCInst *UncondBranch = nullptr;
353840c2e0faSMaksim Panchenko     if (BB->analyzeBranch(TBB, FBB, CondBranch, UncondBranch) && CondBranch &&
353940c2e0faSMaksim Panchenko         BB->succ_size() == 2) {
3540a34c753fSRafael Auler       if (BC.MIB->getCanonicalBranchCondCode(BC.MIB->getCondCode(
3541a34c753fSRafael Auler               *CondBranch)) == BC.MIB->getCondCode(*CondBranch)) {
3542a34c753fSRafael Auler         Stack.push(BB->getConditionalSuccessor(true));
3543a34c753fSRafael Auler         Stack.push(BB->getConditionalSuccessor(false));
3544a34c753fSRafael Auler       } else {
3545a34c753fSRafael Auler         Stack.push(BB->getConditionalSuccessor(false));
3546a34c753fSRafael Auler         Stack.push(BB->getConditionalSuccessor(true));
3547a34c753fSRafael Auler       }
3548a34c753fSRafael Auler     } else {
3549a34c753fSRafael Auler       for (BinaryBasicBlock *SuccBB : BB->successors()) {
3550a34c753fSRafael Auler         Stack.push(SuccBB);
3551a34c753fSRafael Auler       }
3552a34c753fSRafael Auler     }
3553a34c753fSRafael Auler   }
3554a34c753fSRafael Auler 
3555a34c753fSRafael Auler   return DFS;
3556a34c753fSRafael Auler }
3557a34c753fSRafael Auler 
3558a34c753fSRafael Auler size_t BinaryFunction::computeHash(bool UseDFS,
3559a34c753fSRafael Auler                                    OperandHashFuncTy OperandHashFunc) const {
3560a34c753fSRafael Auler   if (size() == 0)
3561a34c753fSRafael Auler     return 0;
3562a34c753fSRafael Auler 
3563a34c753fSRafael Auler   assert(hasCFG() && "function is expected to have CFG");
3564a34c753fSRafael Auler 
3565ebe51c4dSMaksim Panchenko   const BasicBlockOrderType &Order = UseDFS ? dfs() : BasicBlocksLayout;
3566a34c753fSRafael Auler 
3567a34c753fSRafael Auler   // The hash is computed by creating a string of all instruction opcodes and
3568a34c753fSRafael Auler   // possibly their operands and then hashing that string with std::hash.
3569a34c753fSRafael Auler   std::string HashString;
3570a34c753fSRafael Auler   for (const BinaryBasicBlock *BB : Order) {
3571a34c753fSRafael Auler     for (const MCInst &Inst : *BB) {
3572a34c753fSRafael Auler       unsigned Opcode = Inst.getOpcode();
3573a34c753fSRafael Auler 
3574a34c753fSRafael Auler       if (BC.MIB->isPseudo(Inst))
3575a34c753fSRafael Auler         continue;
3576a34c753fSRafael Auler 
3577a34c753fSRafael Auler       // Ignore unconditional jumps since we check CFG consistency by processing
3578a34c753fSRafael Auler       // basic blocks in order and do not rely on branches to be in-sync with
3579a34c753fSRafael Auler       // CFG. Note that we still use condition code of conditional jumps.
3580a34c753fSRafael Auler       if (BC.MIB->isUnconditionalBranch(Inst))
3581a34c753fSRafael Auler         continue;
3582a34c753fSRafael Auler 
3583a34c753fSRafael Auler       if (Opcode == 0)
3584a34c753fSRafael Auler         HashString.push_back(0);
3585a34c753fSRafael Auler 
3586a34c753fSRafael Auler       while (Opcode) {
3587a34c753fSRafael Auler         uint8_t LSB = Opcode & 0xff;
3588a34c753fSRafael Auler         HashString.push_back(LSB);
3589a34c753fSRafael Auler         Opcode = Opcode >> 8;
3590a34c753fSRafael Auler       }
3591a34c753fSRafael Auler 
35928cb7a873SAmir Ayupov       for (const MCOperand &Op : MCPlus::primeOperands(Inst))
35938cb7a873SAmir Ayupov         HashString.append(OperandHashFunc(Op));
3594a34c753fSRafael Auler     }
3595a34c753fSRafael Auler   }
3596a34c753fSRafael Auler 
3597a34c753fSRafael Auler   return Hash = std::hash<std::string>{}(HashString);
3598a34c753fSRafael Auler }
3599a34c753fSRafael Auler 
3600a34c753fSRafael Auler void BinaryFunction::insertBasicBlocks(
3601a34c753fSRafael Auler     BinaryBasicBlock *Start,
3602a34c753fSRafael Auler     std::vector<std::unique_ptr<BinaryBasicBlock>> &&NewBBs,
360340c2e0faSMaksim Panchenko     const bool UpdateLayout, const bool UpdateCFIState,
3604a34c753fSRafael Auler     const bool RecomputeLandingPads) {
3605f18fcdabSAmir Ayupov   const int64_t StartIndex = Start ? getIndex(Start) : -1LL;
3606a34c753fSRafael Auler   const size_t NumNewBlocks = NewBBs.size();
3607a34c753fSRafael Auler 
360840c2e0faSMaksim Panchenko   BasicBlocks.insert(BasicBlocks.begin() + (StartIndex + 1), NumNewBlocks,
3609a34c753fSRafael Auler                      nullptr);
3610a34c753fSRafael Auler 
3611f18fcdabSAmir Ayupov   int64_t I = StartIndex + 1;
3612a34c753fSRafael Auler   for (std::unique_ptr<BinaryBasicBlock> &BB : NewBBs) {
3613a34c753fSRafael Auler     assert(!BasicBlocks[I]);
3614a34c753fSRafael Auler     BasicBlocks[I++] = BB.release();
3615a34c753fSRafael Auler   }
3616a34c753fSRafael Auler 
36173652483cSRafael Auler   if (RecomputeLandingPads)
3618a34c753fSRafael Auler     recomputeLandingPads();
36193652483cSRafael Auler   else
3620a34c753fSRafael Auler     updateBBIndices(0);
3621a34c753fSRafael Auler 
36223652483cSRafael Auler   if (UpdateLayout)
3623a34c753fSRafael Auler     updateLayout(Start, NumNewBlocks);
3624a34c753fSRafael Auler 
36253652483cSRafael Auler   if (UpdateCFIState)
3626a34c753fSRafael Auler     updateCFIState(Start, NumNewBlocks);
3627a34c753fSRafael Auler }
3628a34c753fSRafael Auler 
3629a34c753fSRafael Auler BinaryFunction::iterator BinaryFunction::insertBasicBlocks(
3630a34c753fSRafael Auler     BinaryFunction::iterator StartBB,
3631a34c753fSRafael Auler     std::vector<std::unique_ptr<BinaryBasicBlock>> &&NewBBs,
363240c2e0faSMaksim Panchenko     const bool UpdateLayout, const bool UpdateCFIState,
3633a34c753fSRafael Auler     const bool RecomputeLandingPads) {
3634a34c753fSRafael Auler   const unsigned StartIndex = getIndex(&*StartBB);
3635a34c753fSRafael Auler   const size_t NumNewBlocks = NewBBs.size();
3636a34c753fSRafael Auler 
3637a34c753fSRafael Auler   BasicBlocks.insert(BasicBlocks.begin() + StartIndex + 1, NumNewBlocks,
3638a34c753fSRafael Auler                      nullptr);
3639a34c753fSRafael Auler   auto RetIter = BasicBlocks.begin() + StartIndex + 1;
3640a34c753fSRafael Auler 
3641a34c753fSRafael Auler   unsigned I = StartIndex + 1;
3642a34c753fSRafael Auler   for (std::unique_ptr<BinaryBasicBlock> &BB : NewBBs) {
3643a34c753fSRafael Auler     assert(!BasicBlocks[I]);
3644a34c753fSRafael Auler     BasicBlocks[I++] = BB.release();
3645a34c753fSRafael Auler   }
3646a34c753fSRafael Auler 
36473652483cSRafael Auler   if (RecomputeLandingPads)
3648a34c753fSRafael Auler     recomputeLandingPads();
36493652483cSRafael Auler   else
3650a34c753fSRafael Auler     updateBBIndices(0);
3651a34c753fSRafael Auler 
36523652483cSRafael Auler   if (UpdateLayout)
3653a34c753fSRafael Auler     updateLayout(*std::prev(RetIter), NumNewBlocks);
3654a34c753fSRafael Auler 
36553652483cSRafael Auler   if (UpdateCFIState)
3656a34c753fSRafael Auler     updateCFIState(*std::prev(RetIter), NumNewBlocks);
3657a34c753fSRafael Auler 
3658a34c753fSRafael Auler   return RetIter;
3659a34c753fSRafael Auler }
3660a34c753fSRafael Auler 
3661a34c753fSRafael Auler void BinaryFunction::updateBBIndices(const unsigned StartIndex) {
36623652483cSRafael Auler   for (unsigned I = StartIndex; I < BasicBlocks.size(); ++I)
3663a34c753fSRafael Auler     BasicBlocks[I]->Index = I;
3664a34c753fSRafael Auler }
3665a34c753fSRafael Auler 
3666a34c753fSRafael Auler void BinaryFunction::updateCFIState(BinaryBasicBlock *Start,
3667a34c753fSRafael Auler                                     const unsigned NumNewBlocks) {
3668a34c753fSRafael Auler   const int32_t CFIState = Start->getCFIStateAtExit();
3669a34c753fSRafael Auler   const unsigned StartIndex = getIndex(Start) + 1;
36703652483cSRafael Auler   for (unsigned I = 0; I < NumNewBlocks; ++I)
3671a34c753fSRafael Auler     BasicBlocks[StartIndex + I]->setCFIState(CFIState);
3672a34c753fSRafael Auler }
3673a34c753fSRafael Auler 
3674a34c753fSRafael Auler void BinaryFunction::updateLayout(BinaryBasicBlock *Start,
3675a34c753fSRafael Auler                                   const unsigned NumNewBlocks) {
3676a34c753fSRafael Auler   // If start not provided insert new blocks at the beginning
3677a34c753fSRafael Auler   if (!Start) {
3678a34c753fSRafael Auler     BasicBlocksLayout.insert(layout_begin(), BasicBlocks.begin(),
3679a34c753fSRafael Auler                              BasicBlocks.begin() + NumNewBlocks);
3680a34c753fSRafael Auler     updateLayoutIndices();
3681a34c753fSRafael Auler     return;
3682a34c753fSRafael Auler   }
3683a34c753fSRafael Auler 
3684a34c753fSRafael Auler   // Insert new blocks in the layout immediately after Start.
3685d2c87699SAmir Ayupov   auto Pos = llvm::find(layout(), Start);
3686a34c753fSRafael Auler   assert(Pos != layout_end());
3687ae585be1SRafael Auler   BasicBlockListType::iterator Begin =
3688ae585be1SRafael Auler       std::next(BasicBlocks.begin(), getIndex(Start) + 1);
3689ae585be1SRafael Auler   BasicBlockListType::iterator End =
3690ae585be1SRafael Auler       std::next(BasicBlocks.begin(), getIndex(Start) + NumNewBlocks + 1);
3691a34c753fSRafael Auler   BasicBlocksLayout.insert(Pos + 1, Begin, End);
3692a34c753fSRafael Auler   updateLayoutIndices();
3693a34c753fSRafael Auler }
3694a34c753fSRafael Auler 
3695a34c753fSRafael Auler bool BinaryFunction::checkForAmbiguousJumpTables() {
3696a34c753fSRafael Auler   SmallSet<uint64_t, 4> JumpTables;
3697a34c753fSRafael Auler   for (BinaryBasicBlock *&BB : BasicBlocks) {
3698a34c753fSRafael Auler     for (MCInst &Inst : *BB) {
3699a34c753fSRafael Auler       if (!BC.MIB->isIndirectBranch(Inst))
3700a34c753fSRafael Auler         continue;
3701a34c753fSRafael Auler       uint64_t JTAddress = BC.MIB->getJumpTable(Inst);
3702a34c753fSRafael Auler       if (!JTAddress)
3703a34c753fSRafael Auler         continue;
3704a34c753fSRafael Auler       // This address can be inside another jump table, but we only consider
3705a34c753fSRafael Auler       // it ambiguous when the same start address is used, not the same JT
3706a34c753fSRafael Auler       // object.
3707a34c753fSRafael Auler       if (!JumpTables.count(JTAddress)) {
3708a34c753fSRafael Auler         JumpTables.insert(JTAddress);
3709a34c753fSRafael Auler         continue;
3710a34c753fSRafael Auler       }
3711a34c753fSRafael Auler       return true;
3712a34c753fSRafael Auler     }
3713a34c753fSRafael Auler   }
3714a34c753fSRafael Auler   return false;
3715a34c753fSRafael Auler }
3716a34c753fSRafael Auler 
3717a34c753fSRafael Auler void BinaryFunction::disambiguateJumpTables(
3718a34c753fSRafael Auler     MCPlusBuilder::AllocatorIdTy AllocId) {
3719a34c753fSRafael Auler   assert((opts::JumpTables != JTS_BASIC && isSimple()) || !BC.HasRelocations);
3720a34c753fSRafael Auler   SmallPtrSet<JumpTable *, 4> JumpTables;
3721a34c753fSRafael Auler   for (BinaryBasicBlock *&BB : BasicBlocks) {
3722a34c753fSRafael Auler     for (MCInst &Inst : *BB) {
3723a34c753fSRafael Auler       if (!BC.MIB->isIndirectBranch(Inst))
3724a34c753fSRafael Auler         continue;
3725a34c753fSRafael Auler       JumpTable *JT = getJumpTable(Inst);
3726a34c753fSRafael Auler       if (!JT)
3727a34c753fSRafael Auler         continue;
3728a34c753fSRafael Auler       auto Iter = JumpTables.find(JT);
3729a34c753fSRafael Auler       if (Iter == JumpTables.end()) {
3730a34c753fSRafael Auler         JumpTables.insert(JT);
3731a34c753fSRafael Auler         continue;
3732a34c753fSRafael Auler       }
3733a34c753fSRafael Auler       // This instruction is an indirect jump using a jump table, but it is
3734a34c753fSRafael Auler       // using the same jump table of another jump. Try all our tricks to
3735a34c753fSRafael Auler       // extract the jump table symbol and make it point to a new, duplicated JT
3736a34c753fSRafael Auler       MCPhysReg BaseReg1;
3737a34c753fSRafael Auler       uint64_t Scale;
3738a34c753fSRafael Auler       const MCSymbol *Target;
3739a34c753fSRafael Auler       // In case we match if our first matcher, first instruction is the one to
3740a34c753fSRafael Auler       // patch
3741a34c753fSRafael Auler       MCInst *JTLoadInst = &Inst;
3742a34c753fSRafael Auler       // Try a standard indirect jump matcher, scale 8
3743a34c753fSRafael Auler       std::unique_ptr<MCPlusBuilder::MCInstMatcher> IndJmpMatcher =
3744a34c753fSRafael Auler           BC.MIB->matchIndJmp(BC.MIB->matchReg(BaseReg1),
3745a34c753fSRafael Auler                               BC.MIB->matchImm(Scale), BC.MIB->matchReg(),
3746a34c753fSRafael Auler                               /*Offset=*/BC.MIB->matchSymbol(Target));
3747a34c753fSRafael Auler       if (!IndJmpMatcher->match(
3748a34c753fSRafael Auler               *BC.MRI, *BC.MIB,
3749a34c753fSRafael Auler               MutableArrayRef<MCInst>(&*BB->begin(), &Inst + 1), -1) ||
375040c2e0faSMaksim Panchenko           BaseReg1 != BC.MIB->getNoRegister() || Scale != 8) {
3751a34c753fSRafael Auler         MCPhysReg BaseReg2;
3752a34c753fSRafael Auler         uint64_t Offset;
3753a34c753fSRafael Auler         // Standard JT matching failed. Trying now:
3754a34c753fSRafael Auler         //     movq  "jt.2397/1"(,%rax,8), %rax
3755a34c753fSRafael Auler         //     jmpq  *%rax
3756a34c753fSRafael Auler         std::unique_ptr<MCPlusBuilder::MCInstMatcher> LoadMatcherOwner =
3757a34c753fSRafael Auler             BC.MIB->matchLoad(BC.MIB->matchReg(BaseReg1),
3758a34c753fSRafael Auler                               BC.MIB->matchImm(Scale), BC.MIB->matchReg(),
3759a34c753fSRafael Auler                               /*Offset=*/BC.MIB->matchSymbol(Target));
3760a34c753fSRafael Auler         MCPlusBuilder::MCInstMatcher *LoadMatcher = LoadMatcherOwner.get();
3761a34c753fSRafael Auler         std::unique_ptr<MCPlusBuilder::MCInstMatcher> IndJmpMatcher2 =
3762a34c753fSRafael Auler             BC.MIB->matchIndJmp(std::move(LoadMatcherOwner));
3763a34c753fSRafael Auler         if (!IndJmpMatcher2->match(
3764a34c753fSRafael Auler                 *BC.MRI, *BC.MIB,
3765a34c753fSRafael Auler                 MutableArrayRef<MCInst>(&*BB->begin(), &Inst + 1), -1) ||
3766a34c753fSRafael Auler             BaseReg1 != BC.MIB->getNoRegister() || Scale != 8) {
3767a34c753fSRafael Auler           // JT matching failed. Trying now:
3768a34c753fSRafael Auler           // PIC-style matcher, scale 4
3769a34c753fSRafael Auler           //    addq    %rdx, %rsi
3770a34c753fSRafael Auler           //    addq    %rdx, %rdi
3771a34c753fSRafael Auler           //    leaq    DATAat0x402450(%rip), %r11
3772a34c753fSRafael Auler           //    movslq  (%r11,%rdx,4), %rcx
3773a34c753fSRafael Auler           //    addq    %r11, %rcx
3774a34c753fSRafael Auler           //    jmpq    *%rcx # JUMPTABLE @0x402450
3775a34c753fSRafael Auler           std::unique_ptr<MCPlusBuilder::MCInstMatcher> PICIndJmpMatcher =
3776a34c753fSRafael Auler               BC.MIB->matchIndJmp(BC.MIB->matchAdd(
3777a34c753fSRafael Auler                   BC.MIB->matchReg(BaseReg1),
3778a34c753fSRafael Auler                   BC.MIB->matchLoad(BC.MIB->matchReg(BaseReg2),
3779a34c753fSRafael Auler                                     BC.MIB->matchImm(Scale), BC.MIB->matchReg(),
3780a34c753fSRafael Auler                                     BC.MIB->matchImm(Offset))));
3781a34c753fSRafael Auler           std::unique_ptr<MCPlusBuilder::MCInstMatcher> LEAMatcherOwner =
3782a34c753fSRafael Auler               BC.MIB->matchLoadAddr(BC.MIB->matchSymbol(Target));
3783a34c753fSRafael Auler           MCPlusBuilder::MCInstMatcher *LEAMatcher = LEAMatcherOwner.get();
3784a34c753fSRafael Auler           std::unique_ptr<MCPlusBuilder::MCInstMatcher> PICBaseAddrMatcher =
3785a34c753fSRafael Auler               BC.MIB->matchIndJmp(BC.MIB->matchAdd(std::move(LEAMatcherOwner),
3786a34c753fSRafael Auler                                                    BC.MIB->matchAnyOperand()));
3787a34c753fSRafael Auler           if (!PICIndJmpMatcher->match(
3788a34c753fSRafael Auler                   *BC.MRI, *BC.MIB,
3789a34c753fSRafael Auler                   MutableArrayRef<MCInst>(&*BB->begin(), &Inst + 1), -1) ||
3790a34c753fSRafael Auler               Scale != 4 || BaseReg1 != BaseReg2 || Offset != 0 ||
3791a34c753fSRafael Auler               !PICBaseAddrMatcher->match(
3792a34c753fSRafael Auler                   *BC.MRI, *BC.MIB,
3793a34c753fSRafael Auler                   MutableArrayRef<MCInst>(&*BB->begin(), &Inst + 1), -1)) {
3794a34c753fSRafael Auler             llvm_unreachable("Failed to extract jump table base");
3795a34c753fSRafael Auler             continue;
3796a34c753fSRafael Auler           }
3797a34c753fSRafael Auler           // Matched PIC, identify the instruction with the reference to the JT
3798a34c753fSRafael Auler           JTLoadInst = LEAMatcher->CurInst;
3799a34c753fSRafael Auler         } else {
3800a34c753fSRafael Auler           // Matched non-PIC
3801a34c753fSRafael Auler           JTLoadInst = LoadMatcher->CurInst;
3802a34c753fSRafael Auler         }
3803a34c753fSRafael Auler       }
3804a34c753fSRafael Auler 
3805a34c753fSRafael Auler       uint64_t NewJumpTableID = 0;
3806a34c753fSRafael Auler       const MCSymbol *NewJTLabel;
3807a34c753fSRafael Auler       std::tie(NewJumpTableID, NewJTLabel) =
3808a34c753fSRafael Auler           BC.duplicateJumpTable(*this, JT, Target);
3809a34c753fSRafael Auler       {
3810a34c753fSRafael Auler         auto L = BC.scopeLock();
3811a34c753fSRafael Auler         BC.MIB->replaceMemOperandDisp(*JTLoadInst, NewJTLabel, BC.Ctx.get());
3812a34c753fSRafael Auler       }
3813a34c753fSRafael Auler       // We use a unique ID with the high bit set as address for this "injected"
3814a34c753fSRafael Auler       // jump table (not originally in the input binary).
3815a34c753fSRafael Auler       BC.MIB->setJumpTable(Inst, NewJumpTableID, 0, AllocId);
3816a34c753fSRafael Auler     }
3817a34c753fSRafael Auler   }
3818a34c753fSRafael Auler }
3819a34c753fSRafael Auler 
3820a34c753fSRafael Auler bool BinaryFunction::replaceJumpTableEntryIn(BinaryBasicBlock *BB,
3821a34c753fSRafael Auler                                              BinaryBasicBlock *OldDest,
3822a34c753fSRafael Auler                                              BinaryBasicBlock *NewDest) {
3823a34c753fSRafael Auler   MCInst *Instr = BB->getLastNonPseudoInstr();
3824a34c753fSRafael Auler   if (!Instr || !BC.MIB->isIndirectBranch(*Instr))
3825a34c753fSRafael Auler     return false;
3826a34c753fSRafael Auler   uint64_t JTAddress = BC.MIB->getJumpTable(*Instr);
3827a34c753fSRafael Auler   assert(JTAddress && "Invalid jump table address");
3828a34c753fSRafael Auler   JumpTable *JT = getJumpTableContainingAddress(JTAddress);
3829a34c753fSRafael Auler   assert(JT && "No jump table structure for this indirect branch");
3830a34c753fSRafael Auler   bool Patched = JT->replaceDestination(JTAddress, OldDest->getLabel(),
3831a34c753fSRafael Auler                                         NewDest->getLabel());
3832a34c753fSRafael Auler   (void)Patched;
3833a34c753fSRafael Auler   assert(Patched && "Invalid entry to be replaced in jump table");
3834a34c753fSRafael Auler   return true;
3835a34c753fSRafael Auler }
3836a34c753fSRafael Auler 
3837a34c753fSRafael Auler BinaryBasicBlock *BinaryFunction::splitEdge(BinaryBasicBlock *From,
3838a34c753fSRafael Auler                                             BinaryBasicBlock *To) {
3839a34c753fSRafael Auler   // Create intermediate BB
3840a34c753fSRafael Auler   MCSymbol *Tmp;
3841a34c753fSRafael Auler   {
3842a34c753fSRafael Auler     auto L = BC.scopeLock();
3843a34c753fSRafael Auler     Tmp = BC.Ctx->createNamedTempSymbol("SplitEdge");
3844a34c753fSRafael Auler   }
3845a34c753fSRafael Auler   // Link new BBs to the original input offset of the From BB, so we can map
3846a34c753fSRafael Auler   // samples recorded in new BBs back to the original BB seem in the input
3847a34c753fSRafael Auler   // binary (if using BAT)
38488228c703SMaksim Panchenko   std::unique_ptr<BinaryBasicBlock> NewBB = createBasicBlock(Tmp);
38498228c703SMaksim Panchenko   NewBB->setOffset(From->getInputOffset());
3850a34c753fSRafael Auler   BinaryBasicBlock *NewBBPtr = NewBB.get();
3851a34c753fSRafael Auler 
3852a34c753fSRafael Auler   // Update "From" BB
3853a34c753fSRafael Auler   auto I = From->succ_begin();
3854a34c753fSRafael Auler   auto BI = From->branch_info_begin();
3855a34c753fSRafael Auler   for (; I != From->succ_end(); ++I) {
3856a34c753fSRafael Auler     if (*I == To)
3857a34c753fSRafael Auler       break;
3858a34c753fSRafael Auler     ++BI;
3859a34c753fSRafael Auler   }
3860a34c753fSRafael Auler   assert(I != From->succ_end() && "Invalid CFG edge in splitEdge!");
3861a34c753fSRafael Auler   uint64_t OrigCount = BI->Count;
3862a34c753fSRafael Auler   uint64_t OrigMispreds = BI->MispredictedCount;
3863a34c753fSRafael Auler   replaceJumpTableEntryIn(From, To, NewBBPtr);
3864a34c753fSRafael Auler   From->replaceSuccessor(To, NewBBPtr, OrigCount, OrigMispreds);
3865a34c753fSRafael Auler 
3866a34c753fSRafael Auler   NewBB->addSuccessor(To, OrigCount, OrigMispreds);
3867a34c753fSRafael Auler   NewBB->setExecutionCount(OrigCount);
3868a34c753fSRafael Auler   NewBB->setIsCold(From->isCold());
3869a34c753fSRafael Auler 
3870a34c753fSRafael Auler   // Update CFI and BB layout with new intermediate BB
3871a34c753fSRafael Auler   std::vector<std::unique_ptr<BinaryBasicBlock>> NewBBs;
3872a34c753fSRafael Auler   NewBBs.emplace_back(std::move(NewBB));
3873a34c753fSRafael Auler   insertBasicBlocks(From, std::move(NewBBs), true, true,
3874a34c753fSRafael Auler                     /*RecomputeLandingPads=*/false);
3875a34c753fSRafael Auler   return NewBBPtr;
3876a34c753fSRafael Auler }
3877a34c753fSRafael Auler 
3878a34c753fSRafael Auler void BinaryFunction::deleteConservativeEdges() {
3879a34c753fSRafael Auler   // Our goal is to aggressively remove edges from the CFG that we believe are
3880a34c753fSRafael Auler   // wrong. This is used for instrumentation, where it is safe to remove
3881a34c753fSRafael Auler   // fallthrough edges because we won't reorder blocks.
3882a34c753fSRafael Auler   for (auto I = BasicBlocks.begin(), E = BasicBlocks.end(); I != E; ++I) {
3883a34c753fSRafael Auler     BinaryBasicBlock *BB = *I;
3884a34c753fSRafael Auler     if (BB->succ_size() != 1 || BB->size() == 0)
3885a34c753fSRafael Auler       continue;
3886a34c753fSRafael Auler 
3887a34c753fSRafael Auler     auto NextBB = std::next(I);
3888a34c753fSRafael Auler     MCInst *Last = BB->getLastNonPseudoInstr();
3889a34c753fSRafael Auler     // Fallthrough is a landing pad? Delete this edge (as long as we don't
3890a34c753fSRafael Auler     // have a direct jump to it)
3891a34c753fSRafael Auler     if ((*BB->succ_begin())->isLandingPad() && NextBB != E &&
3892a34c753fSRafael Auler         *BB->succ_begin() == *NextBB && Last && !BC.MIB->isBranch(*Last)) {
3893a34c753fSRafael Auler       BB->removeAllSuccessors();
3894a34c753fSRafael Auler       continue;
3895a34c753fSRafael Auler     }
3896a34c753fSRafael Auler 
3897a34c753fSRafael Auler     // Look for suspicious calls at the end of BB where gcc may optimize it and
3898a34c753fSRafael Auler     // remove the jump to the epilogue when it knows the call won't return.
3899a34c753fSRafael Auler     if (!Last || !BC.MIB->isCall(*Last))
3900a34c753fSRafael Auler       continue;
3901a34c753fSRafael Auler 
3902a34c753fSRafael Auler     const MCSymbol *CalleeSymbol = BC.MIB->getTargetSymbol(*Last);
3903a34c753fSRafael Auler     if (!CalleeSymbol)
3904a34c753fSRafael Auler       continue;
3905a34c753fSRafael Auler 
3906a34c753fSRafael Auler     StringRef CalleeName = CalleeSymbol->getName();
390740c2e0faSMaksim Panchenko     if (CalleeName != "__cxa_throw@PLT" && CalleeName != "_Unwind_Resume@PLT" &&
390840c2e0faSMaksim Panchenko         CalleeName != "__cxa_rethrow@PLT" && CalleeName != "exit@PLT" &&
3909a34c753fSRafael Auler         CalleeName != "abort@PLT")
3910a34c753fSRafael Auler       continue;
3911a34c753fSRafael Auler 
3912a34c753fSRafael Auler     BB->removeAllSuccessors();
3913a34c753fSRafael Auler   }
3914a34c753fSRafael Auler }
3915a34c753fSRafael Auler 
3916a34c753fSRafael Auler bool BinaryFunction::isSymbolValidInScope(const SymbolRef &Symbol,
3917a34c753fSRafael Auler                                           uint64_t SymbolSize) const {
3918a34c753fSRafael Auler   // If this symbol is in a different section from the one where the
3919a34c753fSRafael Auler   // function symbol is, don't consider it as valid.
3920a34c753fSRafael Auler   if (!getOriginSection()->containsAddress(
3921a34c753fSRafael Auler           cantFail(Symbol.getAddress(), "cannot get symbol address")))
3922a34c753fSRafael Auler     return false;
3923a34c753fSRafael Auler 
3924a34c753fSRafael Auler   // Some symbols are tolerated inside function bodies, others are not.
3925a34c753fSRafael Auler   // The real function boundaries may not be known at this point.
39268579db96SDenis Revunov   if (BC.isMarker(Symbol))
3927a34c753fSRafael Auler     return true;
3928a34c753fSRafael Auler 
3929a34c753fSRafael Auler   // It's okay to have a zero-sized symbol in the middle of non-zero-sized
3930a34c753fSRafael Auler   // function.
3931a34c753fSRafael Auler   if (SymbolSize == 0 && containsAddress(cantFail(Symbol.getAddress())))
3932a34c753fSRafael Auler     return true;
3933a34c753fSRafael Auler 
3934a34c753fSRafael Auler   if (cantFail(Symbol.getType()) != SymbolRef::ST_Unknown)
3935a34c753fSRafael Auler     return false;
3936a34c753fSRafael Auler 
3937a34c753fSRafael Auler   if (cantFail(Symbol.getFlags()) & SymbolRef::SF_Global)
3938a34c753fSRafael Auler     return false;
3939a34c753fSRafael Auler 
3940a34c753fSRafael Auler   return true;
3941a34c753fSRafael Auler }
3942a34c753fSRafael Auler 
3943a34c753fSRafael Auler void BinaryFunction::adjustExecutionCount(uint64_t Count) {
3944a34c753fSRafael Auler   if (getKnownExecutionCount() == 0 || Count == 0)
3945a34c753fSRafael Auler     return;
3946a34c753fSRafael Auler 
3947a34c753fSRafael Auler   if (ExecutionCount < Count)
3948a34c753fSRafael Auler     Count = ExecutionCount;
3949a34c753fSRafael Auler 
3950a34c753fSRafael Auler   double AdjustmentRatio = ((double)ExecutionCount - Count) / ExecutionCount;
3951a34c753fSRafael Auler   if (AdjustmentRatio < 0.0)
3952a34c753fSRafael Auler     AdjustmentRatio = 0.0;
3953a34c753fSRafael Auler 
3954a34c753fSRafael Auler   for (BinaryBasicBlock *&BB : layout())
3955a34c753fSRafael Auler     BB->adjustExecutionCount(AdjustmentRatio);
3956a34c753fSRafael Auler 
3957a34c753fSRafael Auler   ExecutionCount -= Count;
3958a34c753fSRafael Auler }
3959a34c753fSRafael Auler 
3960a34c753fSRafael Auler BinaryFunction::~BinaryFunction() {
39613652483cSRafael Auler   for (BinaryBasicBlock *BB : BasicBlocks)
3962a34c753fSRafael Auler     delete BB;
39633652483cSRafael Auler   for (BinaryBasicBlock *BB : DeletedBasicBlocks)
3964a34c753fSRafael Auler     delete BB;
3965a34c753fSRafael Auler }
3966a34c753fSRafael Auler 
3967a34c753fSRafael Auler void BinaryFunction::calculateLoopInfo() {
3968a34c753fSRafael Auler   // Discover loops.
3969a34c753fSRafael Auler   BinaryDominatorTree DomTree;
3970a34c753fSRafael Auler   DomTree.recalculate(*this);
3971a34c753fSRafael Auler   BLI.reset(new BinaryLoopInfo());
3972a34c753fSRafael Auler   BLI->analyze(DomTree);
3973a34c753fSRafael Auler 
3974a34c753fSRafael Auler   // Traverse discovered loops and add depth and profile information.
3975a34c753fSRafael Auler   std::stack<BinaryLoop *> St;
3976a34c753fSRafael Auler   for (auto I = BLI->begin(), E = BLI->end(); I != E; ++I) {
3977a34c753fSRafael Auler     St.push(*I);
3978a34c753fSRafael Auler     ++BLI->OuterLoops;
3979a34c753fSRafael Auler   }
3980a34c753fSRafael Auler 
3981a34c753fSRafael Auler   while (!St.empty()) {
3982a34c753fSRafael Auler     BinaryLoop *L = St.top();
3983a34c753fSRafael Auler     St.pop();
3984a34c753fSRafael Auler     ++BLI->TotalLoops;
3985a34c753fSRafael Auler     BLI->MaximumDepth = std::max(L->getLoopDepth(), BLI->MaximumDepth);
3986a34c753fSRafael Auler 
3987a34c753fSRafael Auler     // Add nested loops in the stack.
39883652483cSRafael Auler     for (BinaryLoop::iterator I = L->begin(), E = L->end(); I != E; ++I)
3989a34c753fSRafael Auler       St.push(*I);
3990a34c753fSRafael Auler 
3991a34c753fSRafael Auler     // Skip if no valid profile is found.
3992a34c753fSRafael Auler     if (!hasValidProfile()) {
3993a34c753fSRafael Auler       L->EntryCount = COUNT_NO_PROFILE;
3994a34c753fSRafael Auler       L->ExitCount = COUNT_NO_PROFILE;
3995a34c753fSRafael Auler       L->TotalBackEdgeCount = COUNT_NO_PROFILE;
3996a34c753fSRafael Auler       continue;
3997a34c753fSRafael Auler     }
3998a34c753fSRafael Auler 
3999a34c753fSRafael Auler     // Compute back edge count.
4000a34c753fSRafael Auler     SmallVector<BinaryBasicBlock *, 1> Latches;
4001a34c753fSRafael Auler     L->getLoopLatches(Latches);
4002a34c753fSRafael Auler 
4003a34c753fSRafael Auler     for (BinaryBasicBlock *Latch : Latches) {
4004a34c753fSRafael Auler       auto BI = Latch->branch_info_begin();
4005a34c753fSRafael Auler       for (BinaryBasicBlock *Succ : Latch->successors()) {
4006a34c753fSRafael Auler         if (Succ == L->getHeader()) {
4007a34c753fSRafael Auler           assert(BI->Count != BinaryBasicBlock::COUNT_NO_PROFILE &&
4008a34c753fSRafael Auler                  "profile data not found");
4009a34c753fSRafael Auler           L->TotalBackEdgeCount += BI->Count;
4010a34c753fSRafael Auler         }
4011a34c753fSRafael Auler         ++BI;
4012a34c753fSRafael Auler       }
4013a34c753fSRafael Auler     }
4014a34c753fSRafael Auler 
4015a34c753fSRafael Auler     // Compute entry count.
4016a34c753fSRafael Auler     L->EntryCount = L->getHeader()->getExecutionCount() - L->TotalBackEdgeCount;
4017a34c753fSRafael Auler 
4018a34c753fSRafael Auler     // Compute exit count.
4019a34c753fSRafael Auler     SmallVector<BinaryLoop::Edge, 1> ExitEdges;
4020a34c753fSRafael Auler     L->getExitEdges(ExitEdges);
4021a34c753fSRafael Auler     for (BinaryLoop::Edge &Exit : ExitEdges) {
4022a34c753fSRafael Auler       const BinaryBasicBlock *Exiting = Exit.first;
4023a34c753fSRafael Auler       const BinaryBasicBlock *ExitTarget = Exit.second;
4024a34c753fSRafael Auler       auto BI = Exiting->branch_info_begin();
4025a34c753fSRafael Auler       for (BinaryBasicBlock *Succ : Exiting->successors()) {
4026a34c753fSRafael Auler         if (Succ == ExitTarget) {
4027a34c753fSRafael Auler           assert(BI->Count != BinaryBasicBlock::COUNT_NO_PROFILE &&
4028a34c753fSRafael Auler                  "profile data not found");
4029a34c753fSRafael Auler           L->ExitCount += BI->Count;
4030a34c753fSRafael Auler         }
4031a34c753fSRafael Auler         ++BI;
4032a34c753fSRafael Auler       }
4033a34c753fSRafael Auler     }
4034a34c753fSRafael Auler   }
4035a34c753fSRafael Auler }
4036a34c753fSRafael Auler 
4037a34c753fSRafael Auler void BinaryFunction::updateOutputValues(const MCAsmLayout &Layout) {
4038a34c753fSRafael Auler   if (!isEmitted()) {
4039a34c753fSRafael Auler     assert(!isInjected() && "injected function should be emitted");
4040a34c753fSRafael Auler     setOutputAddress(getAddress());
4041a34c753fSRafael Auler     setOutputSize(getSize());
4042a34c753fSRafael Auler     return;
4043a34c753fSRafael Auler   }
4044a34c753fSRafael Auler 
4045a34c753fSRafael Auler   const uint64_t BaseAddress = getCodeSection()->getOutputAddress();
4046a34c753fSRafael Auler   ErrorOr<BinarySection &> ColdSection = getColdCodeSection();
4047a34c753fSRafael Auler   const uint64_t ColdBaseAddress =
4048a34c753fSRafael Auler       isSplit() ? ColdSection->getOutputAddress() : 0;
4049a34c753fSRafael Auler   if (BC.HasRelocations || isInjected()) {
4050a34c753fSRafael Auler     const uint64_t StartOffset = Layout.getSymbolOffset(*getSymbol());
4051a34c753fSRafael Auler     const uint64_t EndOffset = Layout.getSymbolOffset(*getFunctionEndLabel());
4052a34c753fSRafael Auler     setOutputAddress(BaseAddress + StartOffset);
4053a34c753fSRafael Auler     setOutputSize(EndOffset - StartOffset);
4054a34c753fSRafael Auler     if (hasConstantIsland()) {
4055a34c753fSRafael Auler       const uint64_t DataOffset =
4056a34c753fSRafael Auler           Layout.getSymbolOffset(*getFunctionConstantIslandLabel());
4057a34c753fSRafael Auler       setOutputDataAddress(BaseAddress + DataOffset);
4058a34c753fSRafael Auler     }
4059a34c753fSRafael Auler     if (isSplit()) {
4060a34c753fSRafael Auler       const MCSymbol *ColdStartSymbol = getColdSymbol();
4061a34c753fSRafael Auler       assert(ColdStartSymbol && ColdStartSymbol->isDefined() &&
4062a34c753fSRafael Auler              "split function should have defined cold symbol");
4063a34c753fSRafael Auler       const MCSymbol *ColdEndSymbol = getFunctionColdEndLabel();
4064a34c753fSRafael Auler       assert(ColdEndSymbol && ColdEndSymbol->isDefined() &&
4065a34c753fSRafael Auler              "split function should have defined cold end symbol");
4066a34c753fSRafael Auler       const uint64_t ColdStartOffset = Layout.getSymbolOffset(*ColdStartSymbol);
4067a34c753fSRafael Auler       const uint64_t ColdEndOffset = Layout.getSymbolOffset(*ColdEndSymbol);
4068a34c753fSRafael Auler       cold().setAddress(ColdBaseAddress + ColdStartOffset);
4069a34c753fSRafael Auler       cold().setImageSize(ColdEndOffset - ColdStartOffset);
4070a34c753fSRafael Auler       if (hasConstantIsland()) {
4071a34c753fSRafael Auler         const uint64_t DataOffset =
4072a34c753fSRafael Auler             Layout.getSymbolOffset(*getFunctionColdConstantIslandLabel());
4073a34c753fSRafael Auler         setOutputColdDataAddress(ColdBaseAddress + DataOffset);
4074a34c753fSRafael Auler       }
4075a34c753fSRafael Auler     }
4076a34c753fSRafael Auler   } else {
4077a34c753fSRafael Auler     setOutputAddress(getAddress());
407840c2e0faSMaksim Panchenko     setOutputSize(Layout.getSymbolOffset(*getFunctionEndLabel()));
4079a34c753fSRafael Auler   }
4080a34c753fSRafael Auler 
4081a34c753fSRafael Auler   // Update basic block output ranges for the debug info, if we have
4082a34c753fSRafael Auler   // secondary entry points in the symbol table to update or if writing BAT.
4083a34c753fSRafael Auler   if (!opts::UpdateDebugSections && !isMultiEntry() &&
4084a34c753fSRafael Auler       !requiresAddressTranslation())
4085a34c753fSRafael Auler     return;
4086a34c753fSRafael Auler 
4087a34c753fSRafael Auler   // Output ranges should match the input if the body hasn't changed.
4088a34c753fSRafael Auler   if (!isSimple() && !BC.HasRelocations)
4089a34c753fSRafael Auler     return;
4090a34c753fSRafael Auler 
4091a34c753fSRafael Auler   // AArch64 may have functions that only contains a constant island (no code).
4092a34c753fSRafael Auler   if (layout_begin() == layout_end())
4093a34c753fSRafael Auler     return;
4094a34c753fSRafael Auler 
4095a34c753fSRafael Auler   BinaryBasicBlock *PrevBB = nullptr;
4096a34c753fSRafael Auler   for (auto BBI = layout_begin(), BBE = layout_end(); BBI != BBE; ++BBI) {
4097a34c753fSRafael Auler     BinaryBasicBlock *BB = *BBI;
4098a34c753fSRafael Auler     assert(BB->getLabel()->isDefined() && "symbol should be defined");
4099a34c753fSRafael Auler     const uint64_t BBBaseAddress = BB->isCold() ? ColdBaseAddress : BaseAddress;
4100a34c753fSRafael Auler     if (!BC.HasRelocations) {
4101a34c753fSRafael Auler       if (BB->isCold()) {
4102a34c753fSRafael Auler         assert(BBBaseAddress == cold().getAddress());
4103a34c753fSRafael Auler       } else {
4104a34c753fSRafael Auler         assert(BBBaseAddress == getOutputAddress());
4105a34c753fSRafael Auler       }
4106a34c753fSRafael Auler     }
4107a34c753fSRafael Auler     const uint64_t BBOffset = Layout.getSymbolOffset(*BB->getLabel());
4108a34c753fSRafael Auler     const uint64_t BBAddress = BBBaseAddress + BBOffset;
4109a34c753fSRafael Auler     BB->setOutputStartAddress(BBAddress);
4110a34c753fSRafael Auler 
4111a34c753fSRafael Auler     if (PrevBB) {
4112a34c753fSRafael Auler       uint64_t PrevBBEndAddress = BBAddress;
41133652483cSRafael Auler       if (BB->isCold() != PrevBB->isCold())
411440c2e0faSMaksim Panchenko         PrevBBEndAddress = getOutputAddress() + getOutputSize();
4115a34c753fSRafael Auler       PrevBB->setOutputEndAddress(PrevBBEndAddress);
4116a34c753fSRafael Auler     }
4117a34c753fSRafael Auler     PrevBB = BB;
4118a34c753fSRafael Auler 
4119a34c753fSRafael Auler     BB->updateOutputValues(Layout);
4120a34c753fSRafael Auler   }
412140c2e0faSMaksim Panchenko   PrevBB->setOutputEndAddress(PrevBB->isCold()
412240c2e0faSMaksim Panchenko                                   ? cold().getAddress() + cold().getImageSize()
412340c2e0faSMaksim Panchenko                                   : getOutputAddress() + getOutputSize());
4124a34c753fSRafael Auler }
4125a34c753fSRafael Auler 
4126a34c753fSRafael Auler DebugAddressRangesVector BinaryFunction::getOutputAddressRanges() const {
4127a34c753fSRafael Auler   DebugAddressRangesVector OutputRanges;
4128a34c753fSRafael Auler 
4129a34c753fSRafael Auler   if (isFolded())
4130a34c753fSRafael Auler     return OutputRanges;
4131a34c753fSRafael Auler 
4132a34c753fSRafael Auler   if (IsFragment)
4133a34c753fSRafael Auler     return OutputRanges;
4134a34c753fSRafael Auler 
4135a34c753fSRafael Auler   OutputRanges.emplace_back(getOutputAddress(),
4136a34c753fSRafael Auler                             getOutputAddress() + getOutputSize());
4137a34c753fSRafael Auler   if (isSplit()) {
4138a34c753fSRafael Auler     assert(isEmitted() && "split function should be emitted");
4139a34c753fSRafael Auler     OutputRanges.emplace_back(cold().getAddress(),
4140a34c753fSRafael Auler                               cold().getAddress() + cold().getImageSize());
4141a34c753fSRafael Auler   }
4142a34c753fSRafael Auler 
4143a34c753fSRafael Auler   if (isSimple())
4144a34c753fSRafael Auler     return OutputRanges;
4145a34c753fSRafael Auler 
4146a34c753fSRafael Auler   for (BinaryFunction *Frag : Fragments) {
4147a34c753fSRafael Auler     assert(!Frag->isSimple() &&
4148a34c753fSRafael Auler            "fragment of non-simple function should also be non-simple");
4149a34c753fSRafael Auler     OutputRanges.emplace_back(Frag->getOutputAddress(),
4150a34c753fSRafael Auler                               Frag->getOutputAddress() + Frag->getOutputSize());
4151a34c753fSRafael Auler   }
4152a34c753fSRafael Auler 
4153a34c753fSRafael Auler   return OutputRanges;
4154a34c753fSRafael Auler }
4155a34c753fSRafael Auler 
4156a34c753fSRafael Auler uint64_t BinaryFunction::translateInputToOutputAddress(uint64_t Address) const {
4157a34c753fSRafael Auler   if (isFolded())
4158a34c753fSRafael Auler     return 0;
4159a34c753fSRafael Auler 
4160a34c753fSRafael Auler   // If the function hasn't changed return the same address.
4161a34c753fSRafael Auler   if (!isEmitted())
4162a34c753fSRafael Auler     return Address;
4163a34c753fSRafael Auler 
4164a34c753fSRafael Auler   if (Address < getAddress())
4165a34c753fSRafael Auler     return 0;
4166a34c753fSRafael Auler 
4167a34c753fSRafael Auler   // Check if the address is associated with an instruction that is tracked
4168a34c753fSRafael Auler   // by address translation.
4169a34c753fSRafael Auler   auto KV = InputOffsetToAddressMap.find(Address - getAddress());
41703652483cSRafael Auler   if (KV != InputOffsetToAddressMap.end())
4171a34c753fSRafael Auler     return KV->second;
4172a34c753fSRafael Auler 
4173a34c753fSRafael Auler   // FIXME: #18950828 - we rely on relative offsets inside basic blocks to stay
4174a34c753fSRafael Auler   //        intact. Instead we can use pseudo instructions and/or annotations.
4175a34c753fSRafael Auler   const uint64_t Offset = Address - getAddress();
4176a34c753fSRafael Auler   const BinaryBasicBlock *BB = getBasicBlockContainingOffset(Offset);
4177a34c753fSRafael Auler   if (!BB) {
4178a34c753fSRafael Auler     // Special case for address immediately past the end of the function.
4179a34c753fSRafael Auler     if (Offset == getSize())
4180a34c753fSRafael Auler       return getOutputAddress() + getOutputSize();
4181a34c753fSRafael Auler 
4182a34c753fSRafael Auler     return 0;
4183a34c753fSRafael Auler   }
4184a34c753fSRafael Auler 
4185a34c753fSRafael Auler   return std::min(BB->getOutputAddressRange().first + Offset - BB->getOffset(),
4186a34c753fSRafael Auler                   BB->getOutputAddressRange().second);
4187a34c753fSRafael Auler }
4188a34c753fSRafael Auler 
4189a34c753fSRafael Auler DebugAddressRangesVector BinaryFunction::translateInputToOutputRanges(
4190a34c753fSRafael Auler     const DWARFAddressRangesVector &InputRanges) const {
4191a34c753fSRafael Auler   DebugAddressRangesVector OutputRanges;
4192a34c753fSRafael Auler 
4193a34c753fSRafael Auler   if (isFolded())
4194a34c753fSRafael Auler     return OutputRanges;
4195a34c753fSRafael Auler 
4196a34c753fSRafael Auler   // If the function hasn't changed return the same ranges.
4197a34c753fSRafael Auler   if (!isEmitted()) {
4198a34c753fSRafael Auler     OutputRanges.resize(InputRanges.size());
4199d2c87699SAmir Ayupov     llvm::transform(InputRanges, OutputRanges.begin(),
4200a34c753fSRafael Auler                     [](const DWARFAddressRange &Range) {
4201a34c753fSRafael Auler                       return DebugAddressRange(Range.LowPC, Range.HighPC);
4202a34c753fSRafael Auler                     });
4203a34c753fSRafael Auler     return OutputRanges;
4204a34c753fSRafael Auler   }
4205a34c753fSRafael Auler 
4206a34c753fSRafael Auler   // Even though we will merge ranges in a post-processing pass, we attempt to
4207a34c753fSRafael Auler   // merge them in a main processing loop as it improves the processing time.
4208a34c753fSRafael Auler   uint64_t PrevEndAddress = 0;
4209a34c753fSRafael Auler   for (const DWARFAddressRange &Range : InputRanges) {
4210a34c753fSRafael Auler     if (!containsAddress(Range.LowPC)) {
4211a34c753fSRafael Auler       LLVM_DEBUG(
4212a34c753fSRafael Auler           dbgs() << "BOLT-DEBUG: invalid debug address range detected for "
4213a34c753fSRafael Auler                  << *this << " : [0x" << Twine::utohexstr(Range.LowPC) << ", 0x"
4214a34c753fSRafael Auler                  << Twine::utohexstr(Range.HighPC) << "]\n");
4215a34c753fSRafael Auler       PrevEndAddress = 0;
4216a34c753fSRafael Auler       continue;
4217a34c753fSRafael Auler     }
4218a34c753fSRafael Auler     uint64_t InputOffset = Range.LowPC - getAddress();
4219a34c753fSRafael Auler     const uint64_t InputEndOffset =
4220a34c753fSRafael Auler         std::min(Range.HighPC - getAddress(), getSize());
4221a34c753fSRafael Auler 
4222d2c87699SAmir Ayupov     auto BBI = llvm::upper_bound(BasicBlockOffsets,
4223d2c87699SAmir Ayupov                                  BasicBlockOffset(InputOffset, nullptr),
4224d2c87699SAmir Ayupov                                  CompareBasicBlockOffsets());
4225a34c753fSRafael Auler     --BBI;
4226a34c753fSRafael Auler     do {
4227a34c753fSRafael Auler       const BinaryBasicBlock *BB = BBI->second;
4228a34c753fSRafael Auler       if (InputOffset < BB->getOffset() || InputOffset >= BB->getEndOffset()) {
4229a34c753fSRafael Auler         LLVM_DEBUG(
4230a34c753fSRafael Auler             dbgs() << "BOLT-DEBUG: invalid debug address range detected for "
4231a34c753fSRafael Auler                    << *this << " : [0x" << Twine::utohexstr(Range.LowPC)
4232a34c753fSRafael Auler                    << ", 0x" << Twine::utohexstr(Range.HighPC) << "]\n");
4233a34c753fSRafael Auler         PrevEndAddress = 0;
4234a34c753fSRafael Auler         break;
4235a34c753fSRafael Auler       }
4236a34c753fSRafael Auler 
4237a34c753fSRafael Auler       // Skip the range if the block was deleted.
4238a34c753fSRafael Auler       if (const uint64_t OutputStart = BB->getOutputAddressRange().first) {
4239a34c753fSRafael Auler         const uint64_t StartAddress =
4240a34c753fSRafael Auler             OutputStart + InputOffset - BB->getOffset();
4241a34c753fSRafael Auler         uint64_t EndAddress = BB->getOutputAddressRange().second;
4242a34c753fSRafael Auler         if (InputEndOffset < BB->getEndOffset())
4243a34c753fSRafael Auler           EndAddress = StartAddress + InputEndOffset - InputOffset;
4244a34c753fSRafael Auler 
4245a34c753fSRafael Auler         if (StartAddress == PrevEndAddress) {
424640c2e0faSMaksim Panchenko           OutputRanges.back().HighPC =
424740c2e0faSMaksim Panchenko               std::max(OutputRanges.back().HighPC, EndAddress);
4248a34c753fSRafael Auler         } else {
4249a34c753fSRafael Auler           OutputRanges.emplace_back(StartAddress,
4250a34c753fSRafael Auler                                     std::max(StartAddress, EndAddress));
4251a34c753fSRafael Auler         }
4252a34c753fSRafael Auler         PrevEndAddress = OutputRanges.back().HighPC;
4253a34c753fSRafael Auler       }
4254a34c753fSRafael Auler 
4255a34c753fSRafael Auler       InputOffset = BB->getEndOffset();
4256a34c753fSRafael Auler       ++BBI;
4257a34c753fSRafael Auler     } while (InputOffset < InputEndOffset);
4258a34c753fSRafael Auler   }
4259a34c753fSRafael Auler 
4260a34c753fSRafael Auler   // Post-processing pass to sort and merge ranges.
4261d2c87699SAmir Ayupov   llvm::sort(OutputRanges);
4262a34c753fSRafael Auler   DebugAddressRangesVector MergedRanges;
4263a34c753fSRafael Auler   PrevEndAddress = 0;
4264a34c753fSRafael Auler   for (const DebugAddressRange &Range : OutputRanges) {
4265a34c753fSRafael Auler     if (Range.LowPC <= PrevEndAddress) {
426640c2e0faSMaksim Panchenko       MergedRanges.back().HighPC =
426740c2e0faSMaksim Panchenko           std::max(MergedRanges.back().HighPC, Range.HighPC);
4268a34c753fSRafael Auler     } else {
4269a34c753fSRafael Auler       MergedRanges.emplace_back(Range.LowPC, Range.HighPC);
4270a34c753fSRafael Auler     }
4271a34c753fSRafael Auler     PrevEndAddress = MergedRanges.back().HighPC;
4272a34c753fSRafael Auler   }
4273a34c753fSRafael Auler 
4274a34c753fSRafael Auler   return MergedRanges;
4275a34c753fSRafael Auler }
4276a34c753fSRafael Auler 
4277a34c753fSRafael Auler MCInst *BinaryFunction::getInstructionAtOffset(uint64_t Offset) {
4278a34c753fSRafael Auler   if (CurrentState == State::Disassembled) {
4279a34c753fSRafael Auler     auto II = Instructions.find(Offset);
4280a34c753fSRafael Auler     return (II == Instructions.end()) ? nullptr : &II->second;
4281a34c753fSRafael Auler   } else if (CurrentState == State::CFG) {
4282a34c753fSRafael Auler     BinaryBasicBlock *BB = getBasicBlockContainingOffset(Offset);
4283a34c753fSRafael Auler     if (!BB)
4284a34c753fSRafael Auler       return nullptr;
4285a34c753fSRafael Auler 
4286a34c753fSRafael Auler     for (MCInst &Inst : *BB) {
4287a34c753fSRafael Auler       constexpr uint32_t InvalidOffset = std::numeric_limits<uint32_t>::max();
4288a9cd49d5SAmir Ayupov       if (Offset == BC.MIB->getOffsetWithDefault(Inst, InvalidOffset))
4289a34c753fSRafael Auler         return &Inst;
4290a34c753fSRafael Auler     }
4291a34c753fSRafael Auler 
4292ccb99dd1SMaksim Panchenko     if (MCInst *LastInstr = BB->getLastNonPseudoInstr()) {
4293ccb99dd1SMaksim Panchenko       const uint32_t Size =
4294ccb99dd1SMaksim Panchenko           BC.MIB->getAnnotationWithDefault<uint32_t>(*LastInstr, "Size");
4295ccb99dd1SMaksim Panchenko       if (BB->getEndOffset() - Offset == Size)
4296ccb99dd1SMaksim Panchenko         return LastInstr;
4297ccb99dd1SMaksim Panchenko     }
4298ccb99dd1SMaksim Panchenko 
4299a34c753fSRafael Auler     return nullptr;
4300a34c753fSRafael Auler   } else {
4301a34c753fSRafael Auler     llvm_unreachable("invalid CFG state to use getInstructionAtOffset()");
4302a34c753fSRafael Auler   }
4303a34c753fSRafael Auler }
4304a34c753fSRafael Auler 
4305a34c753fSRafael Auler DebugLocationsVector BinaryFunction::translateInputToOutputLocationList(
4306a34c753fSRafael Auler     const DebugLocationsVector &InputLL) const {
4307a34c753fSRafael Auler   DebugLocationsVector OutputLL;
4308a34c753fSRafael Auler 
43093652483cSRafael Auler   if (isFolded())
4310a34c753fSRafael Auler     return OutputLL;
4311a34c753fSRafael Auler 
4312a34c753fSRafael Auler   // If the function hasn't changed - there's nothing to update.
43133652483cSRafael Auler   if (!isEmitted())
4314a34c753fSRafael Auler     return InputLL;
4315a34c753fSRafael Auler 
4316a34c753fSRafael Auler   uint64_t PrevEndAddress = 0;
4317a34c753fSRafael Auler   SmallVectorImpl<uint8_t> *PrevExpr = nullptr;
4318a34c753fSRafael Auler   for (const DebugLocationEntry &Entry : InputLL) {
4319a34c753fSRafael Auler     const uint64_t Start = Entry.LowPC;
4320a34c753fSRafael Auler     const uint64_t End = Entry.HighPC;
4321a34c753fSRafael Auler     if (!containsAddress(Start)) {
4322a34c753fSRafael Auler       LLVM_DEBUG(dbgs() << "BOLT-DEBUG: invalid debug address range detected "
4323a34c753fSRafael Auler                            "for "
4324a34c753fSRafael Auler                         << *this << " : [0x" << Twine::utohexstr(Start)
4325a34c753fSRafael Auler                         << ", 0x" << Twine::utohexstr(End) << "]\n");
4326a34c753fSRafael Auler       continue;
4327a34c753fSRafael Auler     }
4328a34c753fSRafael Auler     uint64_t InputOffset = Start - getAddress();
4329a34c753fSRafael Auler     const uint64_t InputEndOffset = std::min(End - getAddress(), getSize());
4330d2c87699SAmir Ayupov     auto BBI = llvm::upper_bound(BasicBlockOffsets,
4331d2c87699SAmir Ayupov                                  BasicBlockOffset(InputOffset, nullptr),
4332d2c87699SAmir Ayupov                                  CompareBasicBlockOffsets());
4333a34c753fSRafael Auler     --BBI;
4334a34c753fSRafael Auler     do {
4335a34c753fSRafael Auler       const BinaryBasicBlock *BB = BBI->second;
4336a34c753fSRafael Auler       if (InputOffset < BB->getOffset() || InputOffset >= BB->getEndOffset()) {
4337a34c753fSRafael Auler         LLVM_DEBUG(dbgs() << "BOLT-DEBUG: invalid debug address range detected "
4338a34c753fSRafael Auler                              "for "
4339a34c753fSRafael Auler                           << *this << " : [0x" << Twine::utohexstr(Start)
4340a34c753fSRafael Auler                           << ", 0x" << Twine::utohexstr(End) << "]\n");
4341a34c753fSRafael Auler         PrevEndAddress = 0;
4342a34c753fSRafael Auler         break;
4343a34c753fSRafael Auler       }
4344a34c753fSRafael Auler 
4345a34c753fSRafael Auler       // Skip the range if the block was deleted.
4346a34c753fSRafael Auler       if (const uint64_t OutputStart = BB->getOutputAddressRange().first) {
4347a34c753fSRafael Auler         const uint64_t StartAddress =
4348a34c753fSRafael Auler             OutputStart + InputOffset - BB->getOffset();
4349a34c753fSRafael Auler         uint64_t EndAddress = BB->getOutputAddressRange().second;
4350a34c753fSRafael Auler         if (InputEndOffset < BB->getEndOffset())
4351a34c753fSRafael Auler           EndAddress = StartAddress + InputEndOffset - InputOffset;
4352a34c753fSRafael Auler 
4353a34c753fSRafael Auler         if (StartAddress == PrevEndAddress && Entry.Expr == *PrevExpr) {
4354a34c753fSRafael Auler           OutputLL.back().HighPC = std::max(OutputLL.back().HighPC, EndAddress);
4355a34c753fSRafael Auler         } else {
435640c2e0faSMaksim Panchenko           OutputLL.emplace_back(DebugLocationEntry{
435740c2e0faSMaksim Panchenko               StartAddress, std::max(StartAddress, EndAddress), Entry.Expr});
4358a34c753fSRafael Auler         }
4359a34c753fSRafael Auler         PrevEndAddress = OutputLL.back().HighPC;
4360a34c753fSRafael Auler         PrevExpr = &OutputLL.back().Expr;
4361a34c753fSRafael Auler       }
4362a34c753fSRafael Auler 
4363a34c753fSRafael Auler       ++BBI;
4364a34c753fSRafael Auler       InputOffset = BB->getEndOffset();
4365a34c753fSRafael Auler     } while (InputOffset < InputEndOffset);
4366a34c753fSRafael Auler   }
4367a34c753fSRafael Auler 
4368a34c753fSRafael Auler   // Sort and merge adjacent entries with identical location.
4369d2c87699SAmir Ayupov   llvm::stable_sort(
4370d2c87699SAmir Ayupov       OutputLL, [](const DebugLocationEntry &A, const DebugLocationEntry &B) {
4371a34c753fSRafael Auler         return A.LowPC < B.LowPC;
4372a34c753fSRafael Auler       });
4373a34c753fSRafael Auler   DebugLocationsVector MergedLL;
4374a34c753fSRafael Auler   PrevEndAddress = 0;
4375a34c753fSRafael Auler   PrevExpr = nullptr;
4376a34c753fSRafael Auler   for (const DebugLocationEntry &Entry : OutputLL) {
4377a34c753fSRafael Auler     if (Entry.LowPC <= PrevEndAddress && *PrevExpr == Entry.Expr) {
4378a34c753fSRafael Auler       MergedLL.back().HighPC = std::max(Entry.HighPC, MergedLL.back().HighPC);
4379a34c753fSRafael Auler     } else {
4380a34c753fSRafael Auler       const uint64_t Begin = std::max(Entry.LowPC, PrevEndAddress);
4381a34c753fSRafael Auler       const uint64_t End = std::max(Begin, Entry.HighPC);
4382a34c753fSRafael Auler       MergedLL.emplace_back(DebugLocationEntry{Begin, End, Entry.Expr});
4383a34c753fSRafael Auler     }
4384a34c753fSRafael Auler     PrevEndAddress = MergedLL.back().HighPC;
4385a34c753fSRafael Auler     PrevExpr = &MergedLL.back().Expr;
4386a34c753fSRafael Auler   }
4387a34c753fSRafael Auler 
4388a34c753fSRafael Auler   return MergedLL;
4389a34c753fSRafael Auler }
4390a34c753fSRafael Auler 
4391a34c753fSRafael Auler void BinaryFunction::printLoopInfo(raw_ostream &OS) const {
4392798e92c6SAmir Ayupov   if (!opts::shouldPrint(*this))
4393798e92c6SAmir Ayupov     return;
4394798e92c6SAmir Ayupov 
4395a34c753fSRafael Auler   OS << "Loop Info for Function \"" << *this << "\"";
43963652483cSRafael Auler   if (hasValidProfile())
4397a34c753fSRafael Auler     OS << " (count: " << getExecutionCount() << ")";
4398a34c753fSRafael Auler   OS << "\n";
4399a34c753fSRafael Auler 
4400a34c753fSRafael Auler   std::stack<BinaryLoop *> St;
44015d8247d4SAmir Ayupov   for_each(*BLI, [&](BinaryLoop *L) { St.push(L); });
4402a34c753fSRafael Auler   while (!St.empty()) {
4403a34c753fSRafael Auler     BinaryLoop *L = St.top();
4404a34c753fSRafael Auler     St.pop();
4405a34c753fSRafael Auler 
44065d8247d4SAmir Ayupov     for_each(*L, [&](BinaryLoop *Inner) { St.push(Inner); });
4407a34c753fSRafael Auler 
4408a34c753fSRafael Auler     if (!hasValidProfile())
4409a34c753fSRafael Auler       continue;
4410a34c753fSRafael Auler 
441140c2e0faSMaksim Panchenko     OS << (L->getLoopDepth() > 1 ? "Nested" : "Outer")
441240c2e0faSMaksim Panchenko        << " loop header: " << L->getHeader()->getName();
4413a34c753fSRafael Auler     OS << "\n";
4414a34c753fSRafael Auler     OS << "Loop basic blocks: ";
4415f7581a39SAmir Ayupov     ListSeparator LS;
4416f7581a39SAmir Ayupov     for (BinaryBasicBlock *BB : L->blocks())
4417f7581a39SAmir Ayupov       OS << LS << BB->getName();
4418a34c753fSRafael Auler     OS << "\n";
4419a34c753fSRafael Auler     if (hasValidProfile()) {
4420a34c753fSRafael Auler       OS << "Total back edge count: " << L->TotalBackEdgeCount << "\n";
4421a34c753fSRafael Auler       OS << "Loop entry count: " << L->EntryCount << "\n";
4422a34c753fSRafael Auler       OS << "Loop exit count: " << L->ExitCount << "\n";
4423a34c753fSRafael Auler       if (L->EntryCount > 0) {
4424a34c753fSRafael Auler         OS << "Average iters per entry: "
4425a34c753fSRafael Auler            << format("%.4lf", (double)L->TotalBackEdgeCount / L->EntryCount)
4426a34c753fSRafael Auler            << "\n";
4427a34c753fSRafael Auler       }
4428a34c753fSRafael Auler     }
4429a34c753fSRafael Auler     OS << "----\n";
4430a34c753fSRafael Auler   }
4431a34c753fSRafael Auler 
4432a34c753fSRafael Auler   OS << "Total number of loops: " << BLI->TotalLoops << "\n";
4433a34c753fSRafael Auler   OS << "Number of outer loops: " << BLI->OuterLoops << "\n";
4434a34c753fSRafael Auler   OS << "Maximum nested loop depth: " << BLI->MaximumDepth << "\n\n";
4435a34c753fSRafael Auler }
4436a34c753fSRafael Auler 
4437a34c753fSRafael Auler bool BinaryFunction::isAArch64Veneer() const {
443835efe1d8SVladislav Khmelevsky   if (empty())
4439a34c753fSRafael Auler     return false;
4440a34c753fSRafael Auler 
4441a34c753fSRafael Auler   BinaryBasicBlock &BB = **BasicBlocks.begin();
44423652483cSRafael Auler   for (MCInst &Inst : BB)
4443a34c753fSRafael Auler     if (!BC.MIB->hasAnnotation(Inst, "AArch64Veneer"))
4444a34c753fSRafael Auler       return false;
4445a34c753fSRafael Auler 
444635efe1d8SVladislav Khmelevsky   for (auto I = BasicBlocks.begin() + 1, E = BasicBlocks.end(); I != E; ++I) {
444735efe1d8SVladislav Khmelevsky     for (MCInst &Inst : **I)
444835efe1d8SVladislav Khmelevsky       if (!BC.MIB->isNoop(Inst))
444935efe1d8SVladislav Khmelevsky         return false;
445035efe1d8SVladislav Khmelevsky   }
445135efe1d8SVladislav Khmelevsky 
4452a34c753fSRafael Auler   return true;
4453a34c753fSRafael Auler }
4454a34c753fSRafael Auler 
4455a34c753fSRafael Auler } // namespace bolt
4456a34c753fSRafael Auler } // namespace llvm
4457