1 //===---- IndirectionUtils.cpp - Utilities for call indirection in Orc ----===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8
9 #include "llvm/ExecutionEngine/Orc/IndirectionUtils.h"
10 #include "llvm/ADT/STLExtras.h"
11 #include "llvm/ADT/Triple.h"
12 #include "llvm/ExecutionEngine/JITLink/x86_64.h"
13 #include "llvm/ExecutionEngine/Orc/OrcABISupport.h"
14 #include "llvm/IR/IRBuilder.h"
15 #include "llvm/MC/MCDisassembler/MCDisassembler.h"
16 #include "llvm/MC/MCInstrAnalysis.h"
17 #include "llvm/Support/Format.h"
18 #include "llvm/Transforms/Utils/Cloning.h"
19 #include <sstream>
20
21 #define DEBUG_TYPE "orc"
22
23 using namespace llvm;
24 using namespace llvm::orc;
25
26 namespace {
27
28 class CompileCallbackMaterializationUnit : public orc::MaterializationUnit {
29 public:
30 using CompileFunction = JITCompileCallbackManager::CompileFunction;
31
CompileCallbackMaterializationUnit(SymbolStringPtr Name,CompileFunction Compile)32 CompileCallbackMaterializationUnit(SymbolStringPtr Name,
33 CompileFunction Compile)
34 : MaterializationUnit(Interface(
35 SymbolFlagsMap({{Name, JITSymbolFlags::Exported}}), nullptr)),
36 Name(std::move(Name)), Compile(std::move(Compile)) {}
37
getName() const38 StringRef getName() const override { return "<Compile Callbacks>"; }
39
40 private:
materialize(std::unique_ptr<MaterializationResponsibility> R)41 void materialize(std::unique_ptr<MaterializationResponsibility> R) override {
42 SymbolMap Result;
43 Result[Name] = JITEvaluatedSymbol(Compile(), JITSymbolFlags::Exported);
44 // No dependencies, so these calls cannot fail.
45 cantFail(R->notifyResolved(Result));
46 cantFail(R->notifyEmitted());
47 }
48
discard(const JITDylib & JD,const SymbolStringPtr & Name)49 void discard(const JITDylib &JD, const SymbolStringPtr &Name) override {
50 llvm_unreachable("Discard should never occur on a LMU?");
51 }
52
53 SymbolStringPtr Name;
54 CompileFunction Compile;
55 };
56
57 } // namespace
58
59 namespace llvm {
60 namespace orc {
61
62 TrampolinePool::~TrampolinePool() = default;
anchor()63 void IndirectStubsManager::anchor() {}
64
65 Expected<JITTargetAddress>
getCompileCallback(CompileFunction Compile)66 JITCompileCallbackManager::getCompileCallback(CompileFunction Compile) {
67 if (auto TrampolineAddr = TP->getTrampoline()) {
68 auto CallbackName =
69 ES.intern(std::string("cc") + std::to_string(++NextCallbackId));
70
71 std::lock_guard<std::mutex> Lock(CCMgrMutex);
72 AddrToSymbol[*TrampolineAddr] = CallbackName;
73 cantFail(
74 CallbacksJD.define(std::make_unique<CompileCallbackMaterializationUnit>(
75 std::move(CallbackName), std::move(Compile))));
76 return *TrampolineAddr;
77 } else
78 return TrampolineAddr.takeError();
79 }
80
executeCompileCallback(JITTargetAddress TrampolineAddr)81 JITTargetAddress JITCompileCallbackManager::executeCompileCallback(
82 JITTargetAddress TrampolineAddr) {
83 SymbolStringPtr Name;
84
85 {
86 std::unique_lock<std::mutex> Lock(CCMgrMutex);
87 auto I = AddrToSymbol.find(TrampolineAddr);
88
89 // If this address is not associated with a compile callback then report an
90 // error to the execution session and return ErrorHandlerAddress to the
91 // callee.
92 if (I == AddrToSymbol.end()) {
93 Lock.unlock();
94 std::string ErrMsg;
95 {
96 raw_string_ostream ErrMsgStream(ErrMsg);
97 ErrMsgStream << "No compile callback for trampoline at "
98 << format("0x%016" PRIx64, TrampolineAddr);
99 }
100 ES.reportError(
101 make_error<StringError>(std::move(ErrMsg), inconvertibleErrorCode()));
102 return ErrorHandlerAddress;
103 } else
104 Name = I->second;
105 }
106
107 if (auto Sym =
108 ES.lookup(makeJITDylibSearchOrder(
109 &CallbacksJD, JITDylibLookupFlags::MatchAllSymbols),
110 Name))
111 return Sym->getAddress();
112 else {
113 llvm::dbgs() << "Didn't find callback.\n";
114 // If anything goes wrong materializing Sym then report it to the session
115 // and return the ErrorHandlerAddress;
116 ES.reportError(Sym.takeError());
117 return ErrorHandlerAddress;
118 }
119 }
120
121 Expected<std::unique_ptr<JITCompileCallbackManager>>
createLocalCompileCallbackManager(const Triple & T,ExecutionSession & ES,JITTargetAddress ErrorHandlerAddress)122 createLocalCompileCallbackManager(const Triple &T, ExecutionSession &ES,
123 JITTargetAddress ErrorHandlerAddress) {
124 switch (T.getArch()) {
125 default:
126 return make_error<StringError>(
127 std::string("No callback manager available for ") + T.str(),
128 inconvertibleErrorCode());
129 case Triple::aarch64:
130 case Triple::aarch64_32: {
131 typedef orc::LocalJITCompileCallbackManager<orc::OrcAArch64> CCMgrT;
132 return CCMgrT::Create(ES, ErrorHandlerAddress);
133 }
134
135 case Triple::x86: {
136 typedef orc::LocalJITCompileCallbackManager<orc::OrcI386> CCMgrT;
137 return CCMgrT::Create(ES, ErrorHandlerAddress);
138 }
139
140 case Triple::loongarch64: {
141 typedef orc::LocalJITCompileCallbackManager<orc::OrcLoongArch64> CCMgrT;
142 return CCMgrT::Create(ES, ErrorHandlerAddress);
143 }
144
145 case Triple::mips: {
146 typedef orc::LocalJITCompileCallbackManager<orc::OrcMips32Be> CCMgrT;
147 return CCMgrT::Create(ES, ErrorHandlerAddress);
148 }
149 case Triple::mipsel: {
150 typedef orc::LocalJITCompileCallbackManager<orc::OrcMips32Le> CCMgrT;
151 return CCMgrT::Create(ES, ErrorHandlerAddress);
152 }
153
154 case Triple::mips64:
155 case Triple::mips64el: {
156 typedef orc::LocalJITCompileCallbackManager<orc::OrcMips64> CCMgrT;
157 return CCMgrT::Create(ES, ErrorHandlerAddress);
158 }
159
160 case Triple::riscv64: {
161 typedef orc::LocalJITCompileCallbackManager<orc::OrcRiscv64> CCMgrT;
162 return CCMgrT::Create(ES, ErrorHandlerAddress);
163 }
164
165 case Triple::x86_64: {
166 if (T.getOS() == Triple::OSType::Win32) {
167 typedef orc::LocalJITCompileCallbackManager<orc::OrcX86_64_Win32> CCMgrT;
168 return CCMgrT::Create(ES, ErrorHandlerAddress);
169 } else {
170 typedef orc::LocalJITCompileCallbackManager<orc::OrcX86_64_SysV> CCMgrT;
171 return CCMgrT::Create(ES, ErrorHandlerAddress);
172 }
173 }
174
175 }
176 }
177
178 std::function<std::unique_ptr<IndirectStubsManager>()>
createLocalIndirectStubsManagerBuilder(const Triple & T)179 createLocalIndirectStubsManagerBuilder(const Triple &T) {
180 switch (T.getArch()) {
181 default:
182 return [](){
183 return std::make_unique<
184 orc::LocalIndirectStubsManager<orc::OrcGenericABI>>();
185 };
186
187 case Triple::aarch64:
188 case Triple::aarch64_32:
189 return [](){
190 return std::make_unique<
191 orc::LocalIndirectStubsManager<orc::OrcAArch64>>();
192 };
193
194 case Triple::x86:
195 return [](){
196 return std::make_unique<
197 orc::LocalIndirectStubsManager<orc::OrcI386>>();
198 };
199
200 case Triple::loongarch64:
201 return []() {
202 return std::make_unique<
203 orc::LocalIndirectStubsManager<orc::OrcLoongArch64>>();
204 };
205
206 case Triple::mips:
207 return [](){
208 return std::make_unique<
209 orc::LocalIndirectStubsManager<orc::OrcMips32Be>>();
210 };
211
212 case Triple::mipsel:
213 return [](){
214 return std::make_unique<
215 orc::LocalIndirectStubsManager<orc::OrcMips32Le>>();
216 };
217
218 case Triple::mips64:
219 case Triple::mips64el:
220 return [](){
221 return std::make_unique<
222 orc::LocalIndirectStubsManager<orc::OrcMips64>>();
223 };
224
225 case Triple::riscv64:
226 return []() {
227 return std::make_unique<
228 orc::LocalIndirectStubsManager<orc::OrcRiscv64>>();
229 };
230
231 case Triple::x86_64:
232 if (T.getOS() == Triple::OSType::Win32) {
233 return [](){
234 return std::make_unique<
235 orc::LocalIndirectStubsManager<orc::OrcX86_64_Win32>>();
236 };
237 } else {
238 return [](){
239 return std::make_unique<
240 orc::LocalIndirectStubsManager<orc::OrcX86_64_SysV>>();
241 };
242 }
243
244 }
245 }
246
createIRTypedAddress(FunctionType & FT,JITTargetAddress Addr)247 Constant* createIRTypedAddress(FunctionType &FT, JITTargetAddress Addr) {
248 Constant *AddrIntVal =
249 ConstantInt::get(Type::getInt64Ty(FT.getContext()), Addr);
250 Constant *AddrPtrVal =
251 ConstantExpr::getCast(Instruction::IntToPtr, AddrIntVal,
252 PointerType::get(&FT, 0));
253 return AddrPtrVal;
254 }
255
createImplPointer(PointerType & PT,Module & M,const Twine & Name,Constant * Initializer)256 GlobalVariable* createImplPointer(PointerType &PT, Module &M,
257 const Twine &Name, Constant *Initializer) {
258 auto IP = new GlobalVariable(M, &PT, false, GlobalValue::ExternalLinkage,
259 Initializer, Name, nullptr,
260 GlobalValue::NotThreadLocal, 0, true);
261 IP->setVisibility(GlobalValue::HiddenVisibility);
262 return IP;
263 }
264
makeStub(Function & F,Value & ImplPointer)265 void makeStub(Function &F, Value &ImplPointer) {
266 assert(F.isDeclaration() && "Can't turn a definition into a stub.");
267 assert(F.getParent() && "Function isn't in a module.");
268 Module &M = *F.getParent();
269 BasicBlock *EntryBlock = BasicBlock::Create(M.getContext(), "entry", &F);
270 IRBuilder<> Builder(EntryBlock);
271 LoadInst *ImplAddr = Builder.CreateLoad(F.getType(), &ImplPointer);
272 std::vector<Value*> CallArgs;
273 for (auto &A : F.args())
274 CallArgs.push_back(&A);
275 CallInst *Call = Builder.CreateCall(F.getFunctionType(), ImplAddr, CallArgs);
276 Call->setTailCall();
277 Call->setAttributes(F.getAttributes());
278 if (F.getReturnType()->isVoidTy())
279 Builder.CreateRetVoid();
280 else
281 Builder.CreateRet(Call);
282 }
283
operator ()(Module & M)284 std::vector<GlobalValue *> SymbolLinkagePromoter::operator()(Module &M) {
285 std::vector<GlobalValue *> PromotedGlobals;
286
287 for (auto &GV : M.global_values()) {
288 bool Promoted = true;
289
290 // Rename if necessary.
291 if (!GV.hasName())
292 GV.setName("__orc_anon." + Twine(NextId++));
293 else if (GV.getName().startswith("\01L"))
294 GV.setName("__" + GV.getName().substr(1) + "." + Twine(NextId++));
295 else if (GV.hasLocalLinkage())
296 GV.setName("__orc_lcl." + GV.getName() + "." + Twine(NextId++));
297 else
298 Promoted = false;
299
300 if (GV.hasLocalLinkage()) {
301 GV.setLinkage(GlobalValue::ExternalLinkage);
302 GV.setVisibility(GlobalValue::HiddenVisibility);
303 Promoted = true;
304 }
305 GV.setUnnamedAddr(GlobalValue::UnnamedAddr::None);
306
307 if (Promoted)
308 PromotedGlobals.push_back(&GV);
309 }
310
311 return PromotedGlobals;
312 }
313
cloneFunctionDecl(Module & Dst,const Function & F,ValueToValueMapTy * VMap)314 Function* cloneFunctionDecl(Module &Dst, const Function &F,
315 ValueToValueMapTy *VMap) {
316 Function *NewF =
317 Function::Create(cast<FunctionType>(F.getValueType()),
318 F.getLinkage(), F.getName(), &Dst);
319 NewF->copyAttributesFrom(&F);
320
321 if (VMap) {
322 (*VMap)[&F] = NewF;
323 auto NewArgI = NewF->arg_begin();
324 for (auto ArgI = F.arg_begin(), ArgE = F.arg_end(); ArgI != ArgE;
325 ++ArgI, ++NewArgI)
326 (*VMap)[&*ArgI] = &*NewArgI;
327 }
328
329 return NewF;
330 }
331
moveFunctionBody(Function & OrigF,ValueToValueMapTy & VMap,ValueMaterializer * Materializer,Function * NewF)332 void moveFunctionBody(Function &OrigF, ValueToValueMapTy &VMap,
333 ValueMaterializer *Materializer,
334 Function *NewF) {
335 assert(!OrigF.isDeclaration() && "Nothing to move");
336 if (!NewF)
337 NewF = cast<Function>(VMap[&OrigF]);
338 else
339 assert(VMap[&OrigF] == NewF && "Incorrect function mapping in VMap.");
340 assert(NewF && "Function mapping missing from VMap.");
341 assert(NewF->getParent() != OrigF.getParent() &&
342 "moveFunctionBody should only be used to move bodies between "
343 "modules.");
344
345 SmallVector<ReturnInst *, 8> Returns; // Ignore returns cloned.
346 CloneFunctionInto(NewF, &OrigF, VMap,
347 CloneFunctionChangeType::DifferentModule, Returns, "",
348 nullptr, nullptr, Materializer);
349 OrigF.deleteBody();
350 }
351
cloneGlobalVariableDecl(Module & Dst,const GlobalVariable & GV,ValueToValueMapTy * VMap)352 GlobalVariable* cloneGlobalVariableDecl(Module &Dst, const GlobalVariable &GV,
353 ValueToValueMapTy *VMap) {
354 GlobalVariable *NewGV = new GlobalVariable(
355 Dst, GV.getValueType(), GV.isConstant(),
356 GV.getLinkage(), nullptr, GV.getName(), nullptr,
357 GV.getThreadLocalMode(), GV.getType()->getAddressSpace());
358 NewGV->copyAttributesFrom(&GV);
359 if (VMap)
360 (*VMap)[&GV] = NewGV;
361 return NewGV;
362 }
363
moveGlobalVariableInitializer(GlobalVariable & OrigGV,ValueToValueMapTy & VMap,ValueMaterializer * Materializer,GlobalVariable * NewGV)364 void moveGlobalVariableInitializer(GlobalVariable &OrigGV,
365 ValueToValueMapTy &VMap,
366 ValueMaterializer *Materializer,
367 GlobalVariable *NewGV) {
368 assert(OrigGV.hasInitializer() && "Nothing to move");
369 if (!NewGV)
370 NewGV = cast<GlobalVariable>(VMap[&OrigGV]);
371 else
372 assert(VMap[&OrigGV] == NewGV &&
373 "Incorrect global variable mapping in VMap.");
374 assert(NewGV->getParent() != OrigGV.getParent() &&
375 "moveGlobalVariableInitializer should only be used to move "
376 "initializers between modules");
377
378 NewGV->setInitializer(MapValue(OrigGV.getInitializer(), VMap, RF_None,
379 nullptr, Materializer));
380 }
381
cloneGlobalAliasDecl(Module & Dst,const GlobalAlias & OrigA,ValueToValueMapTy & VMap)382 GlobalAlias* cloneGlobalAliasDecl(Module &Dst, const GlobalAlias &OrigA,
383 ValueToValueMapTy &VMap) {
384 assert(OrigA.getAliasee() && "Original alias doesn't have an aliasee?");
385 auto *NewA = GlobalAlias::create(OrigA.getValueType(),
386 OrigA.getType()->getPointerAddressSpace(),
387 OrigA.getLinkage(), OrigA.getName(), &Dst);
388 NewA->copyAttributesFrom(&OrigA);
389 VMap[&OrigA] = NewA;
390 return NewA;
391 }
392
cloneModuleFlagsMetadata(Module & Dst,const Module & Src,ValueToValueMapTy & VMap)393 void cloneModuleFlagsMetadata(Module &Dst, const Module &Src,
394 ValueToValueMapTy &VMap) {
395 auto *MFs = Src.getModuleFlagsMetadata();
396 if (!MFs)
397 return;
398 for (auto *MF : MFs->operands())
399 Dst.addModuleFlag(MapMetadata(MF, VMap));
400 }
401
addFunctionPointerRelocationsToCurrentSymbol(jitlink::Symbol & Sym,jitlink::LinkGraph & G,MCDisassembler & Disassembler,MCInstrAnalysis & MIA)402 Error addFunctionPointerRelocationsToCurrentSymbol(jitlink::Symbol &Sym,
403 jitlink::LinkGraph &G,
404 MCDisassembler &Disassembler,
405 MCInstrAnalysis &MIA) {
406 // AArch64 appears to already come with the necessary relocations. Among other
407 // architectures, only x86_64 is currently implemented here.
408 if (G.getTargetTriple().getArch() != Triple::x86_64)
409 return Error::success();
410
411 raw_null_ostream CommentStream;
412 auto &STI = Disassembler.getSubtargetInfo();
413
414 // Determine the function bounds
415 auto &B = Sym.getBlock();
416 assert(!B.isZeroFill() && "expected content block");
417 auto SymAddress = Sym.getAddress();
418 auto SymStartInBlock =
419 (const uint8_t *)B.getContent().data() + Sym.getOffset();
420 auto SymSize = Sym.getSize() ? Sym.getSize() : B.getSize() - Sym.getOffset();
421 auto Content = ArrayRef(SymStartInBlock, SymSize);
422
423 LLVM_DEBUG(dbgs() << "Adding self-relocations to " << Sym.getName() << "\n");
424
425 SmallDenseSet<uintptr_t, 8> ExistingRelocations;
426 for (auto &E : B.edges()) {
427 if (E.isRelocation())
428 ExistingRelocations.insert(E.getOffset());
429 }
430
431 size_t I = 0;
432 while (I < Content.size()) {
433 MCInst Instr;
434 uint64_t InstrSize = 0;
435 uint64_t InstrStart = SymAddress.getValue() + I;
436 auto DecodeStatus = Disassembler.getInstruction(
437 Instr, InstrSize, Content.drop_front(I), InstrStart, CommentStream);
438 if (DecodeStatus != MCDisassembler::Success) {
439 LLVM_DEBUG(dbgs() << "Aborting due to disassembly failure at address "
440 << InstrStart);
441 return make_error<StringError>(
442 formatv("failed to disassemble at address {0:x16}", InstrStart),
443 inconvertibleErrorCode());
444 }
445 // Advance to the next instruction.
446 I += InstrSize;
447
448 // Check for a PC-relative address equal to the symbol itself.
449 auto PCRelAddr =
450 MIA.evaluateMemoryOperandAddress(Instr, &STI, InstrStart, InstrSize);
451 if (!PCRelAddr || *PCRelAddr != SymAddress.getValue())
452 continue;
453
454 auto RelocOffInInstr =
455 MIA.getMemoryOperandRelocationOffset(Instr, InstrSize);
456 if (!RelocOffInInstr || InstrSize - *RelocOffInInstr != 4) {
457 LLVM_DEBUG(dbgs() << "Skipping unknown self-relocation at "
458 << InstrStart);
459 continue;
460 }
461
462 auto RelocOffInBlock = orc::ExecutorAddr(InstrStart) + *RelocOffInInstr -
463 SymAddress + Sym.getOffset();
464 if (ExistingRelocations.contains(RelocOffInBlock))
465 continue;
466
467 LLVM_DEBUG(dbgs() << "Adding delta32 self-relocation at " << InstrStart);
468 B.addEdge(jitlink::x86_64::Delta32, RelocOffInBlock, Sym, /*Addend=*/-4);
469 }
470 return Error::success();
471 }
472
473 } // End namespace orc.
474 } // End namespace llvm.
475