1 //===- bolt/Passes/PatchEntries.cpp - Pass for patching function entries --===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements the PatchEntries class that is used for patching
10 // the original function entry points.
11 //
12 //===----------------------------------------------------------------------===//
13
14 #include "bolt/Passes/PatchEntries.h"
15 #include "bolt/Utils/NameResolver.h"
16 #include "llvm/ADT/STLExtras.h"
17 #include "llvm/Support/CommandLine.h"
18
19 namespace opts {
20
21 extern llvm::cl::OptionCategory BoltCategory;
22
23 extern llvm::cl::opt<unsigned> Verbosity;
24
25 llvm::cl::opt<bool>
26 ForcePatch("force-patch",
27 llvm::cl::desc("force patching of original entry points"),
28 llvm::cl::Hidden, llvm::cl::cat(BoltCategory));
29 }
30
31 namespace llvm {
32 namespace bolt {
33
runOnFunctions(BinaryContext & BC)34 Error PatchEntries::runOnFunctions(BinaryContext &BC) {
35 if (!opts::ForcePatch) {
36 // Mark the binary for patching if we did not create external references
37 // for original code in any of functions we are not going to emit.
38 bool NeedsPatching = llvm::any_of(
39 llvm::make_second_range(BC.getBinaryFunctions()),
40 [&](BinaryFunction &BF) {
41 return !BC.shouldEmit(BF) && !BF.hasExternalRefRelocations();
42 });
43
44 if (!NeedsPatching)
45 return Error::success();
46 }
47
48 if (opts::Verbosity >= 1)
49 BC.outs() << "BOLT-INFO: patching entries in original code\n";
50
51 // Calculate the size of the patch.
52 static size_t PatchSize = 0;
53 if (!PatchSize) {
54 InstructionListType Seq;
55 BC.MIB->createLongTailCall(Seq, BC.Ctx->createTempSymbol(), BC.Ctx.get());
56 PatchSize = BC.computeCodeSize(Seq.begin(), Seq.end());
57 }
58
59 for (auto &BFI : BC.getBinaryFunctions()) {
60 BinaryFunction &Function = BFI.second;
61
62 // Patch original code only for functions that will be emitted.
63 if (!BC.shouldEmit(Function))
64 continue;
65
66 // Check if we can skip patching the function.
67 if (!opts::ForcePatch && !Function.hasEHRanges() &&
68 Function.getSize() < PatchThreshold)
69 continue;
70
71 // List of patches for function entries. We either successfully patch
72 // all entries or, if we cannot patch one or more, do no patch any and
73 // mark the function as ignorable.
74 std::vector<Patch> PendingPatches;
75
76 uint64_t NextValidByte = 0; // offset of the byte past the last patch
77 bool Success = Function.forEachEntryPoint([&](uint64_t Offset,
78 const MCSymbol *Symbol) {
79 if (Offset < NextValidByte) {
80 if (opts::Verbosity >= 1)
81 BC.outs() << "BOLT-INFO: unable to patch entry point in " << Function
82 << " at offset 0x" << Twine::utohexstr(Offset) << '\n';
83 return false;
84 }
85
86 PendingPatches.emplace_back(Patch{Symbol, Function.getAddress() + Offset,
87 Function.getFileOffset() + Offset,
88 Function.getOriginSection()});
89 NextValidByte = Offset + PatchSize;
90 if (NextValidByte > Function.getMaxSize()) {
91 if (opts::Verbosity >= 1)
92 BC.outs() << "BOLT-INFO: function " << Function
93 << " too small to patch its entry point\n";
94 return false;
95 }
96
97 return true;
98 });
99
100 if (!Success) {
101 // We can't change output layout for AArch64 due to LongJmp pass
102 if (BC.isAArch64()) {
103 if (opts::ForcePatch) {
104 BC.errs() << "BOLT-ERROR: unable to patch entries in " << Function
105 << "\n";
106 return createFatalBOLTError("");
107 }
108
109 continue;
110 }
111
112 // If the original function entries cannot be patched, then we cannot
113 // safely emit new function body.
114 BC.errs() << "BOLT-WARNING: failed to patch entries in " << Function
115 << ". The function will not be optimized.\n";
116 Function.setIgnored();
117 continue;
118 }
119
120 for (Patch &Patch : PendingPatches) {
121 BinaryFunction *PatchFunction = BC.createInjectedBinaryFunction(
122 NameResolver::append(Patch.Symbol->getName(), ".org.0"));
123 // Force the function to be emitted at the given address.
124 PatchFunction->setOutputAddress(Patch.Address);
125 PatchFunction->setFileOffset(Patch.FileOffset);
126 PatchFunction->setOriginSection(Patch.Section);
127
128 InstructionListType Seq;
129 BC.MIB->createLongTailCall(Seq, Patch.Symbol, BC.Ctx.get());
130 PatchFunction->addBasicBlock()->addInstructions(Seq);
131
132 // Verify the size requirements.
133 uint64_t HotSize, ColdSize;
134 std::tie(HotSize, ColdSize) = BC.calculateEmittedSize(*PatchFunction);
135 assert(!ColdSize && "unexpected cold code");
136 assert(HotSize <= PatchSize && "max patch size exceeded");
137 }
138
139 Function.setIsPatched(true);
140 }
141 return Error::success();
142 }
143
144 } // end namespace bolt
145 } // end namespace llvm
146