xref: /openbsd-src/gnu/llvm/llvm/lib/ExecutionEngine/Orc/LLJIT.cpp (revision d415bd752c734aee168c4ee86ff32e8cc249eb16) 
1 //===--------- LLJIT.cpp - An ORC-based JIT for compiling LLVM IR ---------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
9 #include "llvm/ExecutionEngine/Orc/LLJIT.h"
10 #include "llvm/ExecutionEngine/JITLink/EHFrameSupport.h"
11 #include "llvm/ExecutionEngine/JITLink/JITLinkMemoryManager.h"
12 #include "llvm/ExecutionEngine/Orc/EPCEHFrameRegistrar.h"
13 #include "llvm/ExecutionEngine/Orc/ExecutorProcessControl.h"
14 #include "llvm/ExecutionEngine/Orc/MachOPlatform.h"
15 #include "llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h"
16 #include "llvm/ExecutionEngine/Orc/ObjectTransformLayer.h"
17 #include "llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h"
18 #include "llvm/ExecutionEngine/Orc/Shared/OrcError.h"
19 #include "llvm/ExecutionEngine/Orc/TargetProcess/RegisterEHFrames.h"
20 #include "llvm/ExecutionEngine/SectionMemoryManager.h"
21 #include "llvm/IR/GlobalVariable.h"
22 #include "llvm/IR/IRBuilder.h"
23 #include "llvm/IR/Mangler.h"
24 #include "llvm/IR/Module.h"
25 #include "llvm/Support/DynamicLibrary.h"
26 
27 #include <map>
28 
29 #define DEBUG_TYPE "orc"
30 
31 using namespace llvm;
32 using namespace llvm::orc;
33 
34 namespace {
35 
36 /// Adds helper function decls and wrapper functions that call the helper with
37 /// some additional prefix arguments.
38 ///
39 /// E.g. For wrapper "foo" with type i8(i8, i64), helper "bar", and prefix
40 /// args i32 4 and i16 12345, this function will add:
41 ///
42 /// declare i8 @bar(i32, i16, i8, i64)
43 ///
44 /// define i8 @foo(i8, i64) {
45 /// entry:
46 ///   %2 = call i8 @bar(i32 4, i16 12345, i8 %0, i64 %1)
47 ///   ret i8 %2
48 /// }
49 ///
addHelperAndWrapper(Module & M,StringRef WrapperName,FunctionType * WrapperFnType,GlobalValue::VisibilityTypes WrapperVisibility,StringRef HelperName,ArrayRef<Value * > HelperPrefixArgs)50 Function *addHelperAndWrapper(Module &M, StringRef WrapperName,
51                               FunctionType *WrapperFnType,
52                               GlobalValue::VisibilityTypes WrapperVisibility,
53                               StringRef HelperName,
54                               ArrayRef<Value *> HelperPrefixArgs) {
55   std::vector<Type *> HelperArgTypes;
56   for (auto *Arg : HelperPrefixArgs)
57     HelperArgTypes.push_back(Arg->getType());
58   for (auto *T : WrapperFnType->params())
59     HelperArgTypes.push_back(T);
60   auto *HelperFnType =
61       FunctionType::get(WrapperFnType->getReturnType(), HelperArgTypes, false);
62   auto *HelperFn = Function::Create(HelperFnType, GlobalValue::ExternalLinkage,
63                                     HelperName, M);
64 
65   auto *WrapperFn = Function::Create(
66       WrapperFnType, GlobalValue::ExternalLinkage, WrapperName, M);
67   WrapperFn->setVisibility(WrapperVisibility);
68 
69   auto *EntryBlock = BasicBlock::Create(M.getContext(), "entry", WrapperFn);
70   IRBuilder<> IB(EntryBlock);
71 
72   std::vector<Value *> HelperArgs;
73   for (auto *Arg : HelperPrefixArgs)
74     HelperArgs.push_back(Arg);
75   for (auto &Arg : WrapperFn->args())
76     HelperArgs.push_back(&Arg);
77   auto *HelperResult = IB.CreateCall(HelperFn, HelperArgs);
78   if (HelperFn->getReturnType()->isVoidTy())
79     IB.CreateRetVoid();
80   else
81     IB.CreateRet(HelperResult);
82 
83   return WrapperFn;
84 }
85 
86 class ORCPlatformSupport : public LLJIT::PlatformSupport {
87 public:
ORCPlatformSupport(orc::LLJIT & J)88   ORCPlatformSupport(orc::LLJIT &J) : J(J) {}
89 
initialize(orc::JITDylib & JD)90   Error initialize(orc::JITDylib &JD) override {
91     using llvm::orc::shared::SPSExecutorAddr;
92     using llvm::orc::shared::SPSString;
93     using SPSDLOpenSig = SPSExecutorAddr(SPSString, int32_t);
94     enum dlopen_mode : int32_t {
95       ORC_RT_RTLD_LAZY = 0x1,
96       ORC_RT_RTLD_NOW = 0x2,
97       ORC_RT_RTLD_LOCAL = 0x4,
98       ORC_RT_RTLD_GLOBAL = 0x8
99     };
100 
101     if (auto WrapperAddr = J.lookup("__orc_rt_jit_dlopen_wrapper")) {
102       return J.getExecutionSession().callSPSWrapper<SPSDLOpenSig>(
103           *WrapperAddr, DSOHandles[&JD], JD.getName(),
104           int32_t(ORC_RT_RTLD_LAZY));
105     } else
106       return WrapperAddr.takeError();
107   }
108 
deinitialize(orc::JITDylib & JD)109   Error deinitialize(orc::JITDylib &JD) override {
110     using llvm::orc::shared::SPSExecutorAddr;
111     using SPSDLCloseSig = int32_t(SPSExecutorAddr);
112 
113     if (auto WrapperAddr = J.lookup("__orc_rt_jit_dlclose_wrapper")) {
114       int32_t result;
115       auto E = J.getExecutionSession().callSPSWrapper<SPSDLCloseSig>(
116           *WrapperAddr, result, DSOHandles[&JD]);
117       if (E)
118         return E;
119       else if (result)
120         return make_error<StringError>("dlclose failed",
121                                        inconvertibleErrorCode());
122       DSOHandles.erase(&JD);
123     } else
124       return WrapperAddr.takeError();
125     return Error::success();
126   }
127 
128 private:
129   orc::LLJIT &J;
130   DenseMap<orc::JITDylib *, orc::ExecutorAddr> DSOHandles;
131 };
132 
133 class GenericLLVMIRPlatformSupport;
134 
135 /// orc::Platform component of Generic LLVM IR Platform support.
136 /// Just forwards calls to the GenericLLVMIRPlatformSupport class below.
137 class GenericLLVMIRPlatform : public Platform {
138 public:
GenericLLVMIRPlatform(GenericLLVMIRPlatformSupport & S)139   GenericLLVMIRPlatform(GenericLLVMIRPlatformSupport &S) : S(S) {}
140   Error setupJITDylib(JITDylib &JD) override;
141   Error teardownJITDylib(JITDylib &JD) override;
142   Error notifyAdding(ResourceTracker &RT,
143                      const MaterializationUnit &MU) override;
notifyRemoving(ResourceTracker & RT)144   Error notifyRemoving(ResourceTracker &RT) override {
145     // Noop -- Nothing to do (yet).
146     return Error::success();
147   }
148 
149 private:
150   GenericLLVMIRPlatformSupport &S;
151 };
152 
153 /// This transform parses llvm.global_ctors to produce a single initialization
154 /// function for the module, records the function, then deletes
155 /// llvm.global_ctors.
156 class GlobalCtorDtorScraper {
157 public:
GlobalCtorDtorScraper(GenericLLVMIRPlatformSupport & PS,StringRef InitFunctionPrefix,StringRef DeInitFunctionPrefix)158   GlobalCtorDtorScraper(GenericLLVMIRPlatformSupport &PS,
159                         StringRef InitFunctionPrefix,
160                         StringRef DeInitFunctionPrefix)
161       : PS(PS), InitFunctionPrefix(InitFunctionPrefix),
162         DeInitFunctionPrefix(DeInitFunctionPrefix) {}
163   Expected<ThreadSafeModule> operator()(ThreadSafeModule TSM,
164                                         MaterializationResponsibility &R);
165 
166 private:
167   GenericLLVMIRPlatformSupport &PS;
168   StringRef InitFunctionPrefix;
169   StringRef DeInitFunctionPrefix;
170 };
171 
172 /// Generic IR Platform Support
173 ///
174 /// Scrapes llvm.global_ctors and llvm.global_dtors and replaces them with
175 /// specially named 'init' and 'deinit'. Injects definitions / interposes for
176 /// some runtime API, including __cxa_atexit, dlopen, and dlclose.
177 class GenericLLVMIRPlatformSupport : public LLJIT::PlatformSupport {
178 public:
GenericLLVMIRPlatformSupport(LLJIT & J)179   GenericLLVMIRPlatformSupport(LLJIT &J)
180       : J(J), InitFunctionPrefix(J.mangle("__orc_init_func.")),
181         DeInitFunctionPrefix(J.mangle("__orc_deinit_func.")) {
182 
183     getExecutionSession().setPlatform(
184         std::make_unique<GenericLLVMIRPlatform>(*this));
185 
186     setInitTransform(J, GlobalCtorDtorScraper(*this, InitFunctionPrefix,
187                                               DeInitFunctionPrefix));
188 
189     SymbolMap StdInterposes;
190 
191     StdInterposes[J.mangleAndIntern("__lljit.platform_support_instance")] =
192         JITEvaluatedSymbol(pointerToJITTargetAddress(this),
193                            JITSymbolFlags::Exported);
194     StdInterposes[J.mangleAndIntern("__lljit.cxa_atexit_helper")] =
195         JITEvaluatedSymbol(pointerToJITTargetAddress(registerCxaAtExitHelper),
196                            JITSymbolFlags());
197 
198     cantFail(
199         J.getMainJITDylib().define(absoluteSymbols(std::move(StdInterposes))));
200     cantFail(setupJITDylib(J.getMainJITDylib()));
201     cantFail(J.addIRModule(J.getMainJITDylib(), createPlatformRuntimeModule()));
202   }
203 
getExecutionSession()204   ExecutionSession &getExecutionSession() { return J.getExecutionSession(); }
205 
206   /// Adds a module that defines the __dso_handle global.
setupJITDylib(JITDylib & JD)207   Error setupJITDylib(JITDylib &JD) {
208 
209     // Add per-jitdylib standard interposes.
210     SymbolMap PerJDInterposes;
211     PerJDInterposes[J.mangleAndIntern("__lljit.run_atexits_helper")] =
212         JITEvaluatedSymbol(pointerToJITTargetAddress(runAtExitsHelper),
213                            JITSymbolFlags());
214     PerJDInterposes[J.mangleAndIntern("__lljit.atexit_helper")] =
215         JITEvaluatedSymbol(pointerToJITTargetAddress(registerAtExitHelper),
216                            JITSymbolFlags());
217     cantFail(JD.define(absoluteSymbols(std::move(PerJDInterposes))));
218 
219     auto Ctx = std::make_unique<LLVMContext>();
220     auto M = std::make_unique<Module>("__standard_lib", *Ctx);
221     M->setDataLayout(J.getDataLayout());
222 
223     auto *Int64Ty = Type::getInt64Ty(*Ctx);
224     auto *DSOHandle = new GlobalVariable(
225         *M, Int64Ty, true, GlobalValue::ExternalLinkage,
226         ConstantInt::get(Int64Ty, reinterpret_cast<uintptr_t>(&JD)),
227         "__dso_handle");
228     DSOHandle->setVisibility(GlobalValue::DefaultVisibility);
229     DSOHandle->setInitializer(
230         ConstantInt::get(Int64Ty, pointerToJITTargetAddress(&JD)));
231 
232     auto *GenericIRPlatformSupportTy =
233         StructType::create(*Ctx, "lljit.GenericLLJITIRPlatformSupport");
234 
235     auto *PlatformInstanceDecl = new GlobalVariable(
236         *M, GenericIRPlatformSupportTy, true, GlobalValue::ExternalLinkage,
237         nullptr, "__lljit.platform_support_instance");
238 
239     auto *VoidTy = Type::getVoidTy(*Ctx);
240     addHelperAndWrapper(
241         *M, "__lljit_run_atexits", FunctionType::get(VoidTy, {}, false),
242         GlobalValue::HiddenVisibility, "__lljit.run_atexits_helper",
243         {PlatformInstanceDecl, DSOHandle});
244 
245     auto *IntTy = Type::getIntNTy(*Ctx, sizeof(int) * CHAR_BIT);
246     auto *AtExitCallbackTy = FunctionType::get(VoidTy, {}, false);
247     auto *AtExitCallbackPtrTy = PointerType::getUnqual(AtExitCallbackTy);
248     addHelperAndWrapper(*M, "atexit",
249                         FunctionType::get(IntTy, {AtExitCallbackPtrTy}, false),
250                         GlobalValue::HiddenVisibility, "__lljit.atexit_helper",
251                         {PlatformInstanceDecl, DSOHandle});
252 
253     return J.addIRModule(JD, ThreadSafeModule(std::move(M), std::move(Ctx)));
254   }
255 
notifyAdding(ResourceTracker & RT,const MaterializationUnit & MU)256   Error notifyAdding(ResourceTracker &RT, const MaterializationUnit &MU) {
257     auto &JD = RT.getJITDylib();
258     if (auto &InitSym = MU.getInitializerSymbol())
259       InitSymbols[&JD].add(InitSym, SymbolLookupFlags::WeaklyReferencedSymbol);
260     else {
261       // If there's no identified init symbol attached, but there is a symbol
262       // with the GenericIRPlatform::InitFunctionPrefix, then treat that as
263       // an init function. Add the symbol to both the InitSymbols map (which
264       // will trigger a lookup to materialize the module) and the InitFunctions
265       // map (which holds the names of the symbols to execute).
266       for (auto &KV : MU.getSymbols())
267         if ((*KV.first).startswith(InitFunctionPrefix)) {
268           InitSymbols[&JD].add(KV.first,
269                                SymbolLookupFlags::WeaklyReferencedSymbol);
270           InitFunctions[&JD].add(KV.first);
271         } else if ((*KV.first).startswith(DeInitFunctionPrefix)) {
272           DeInitFunctions[&JD].add(KV.first);
273         }
274     }
275     return Error::success();
276   }
277 
initialize(JITDylib & JD)278   Error initialize(JITDylib &JD) override {
279     LLVM_DEBUG({
280       dbgs() << "GenericLLVMIRPlatformSupport getting initializers to run\n";
281     });
282     if (auto Initializers = getInitializers(JD)) {
283       LLVM_DEBUG(
284           { dbgs() << "GenericLLVMIRPlatformSupport running initializers\n"; });
285       for (auto InitFnAddr : *Initializers) {
286         LLVM_DEBUG({
287           dbgs() << "  Running init " << formatv("{0:x16}", InitFnAddr)
288                  << "...\n";
289         });
290         auto *InitFn = jitTargetAddressToFunction<void (*)()>(InitFnAddr);
291         InitFn();
292       }
293     } else
294       return Initializers.takeError();
295     return Error::success();
296   }
297 
deinitialize(JITDylib & JD)298   Error deinitialize(JITDylib &JD) override {
299     LLVM_DEBUG({
300       dbgs() << "GenericLLVMIRPlatformSupport getting deinitializers to run\n";
301     });
302     if (auto Deinitializers = getDeinitializers(JD)) {
303       LLVM_DEBUG({
304         dbgs() << "GenericLLVMIRPlatformSupport running deinitializers\n";
305       });
306       for (auto DeinitFnAddr : *Deinitializers) {
307         LLVM_DEBUG({
308           dbgs() << "  Running deinit " << formatv("{0:x16}", DeinitFnAddr)
309                  << "...\n";
310         });
311         auto *DeinitFn = jitTargetAddressToFunction<void (*)()>(DeinitFnAddr);
312         DeinitFn();
313       }
314     } else
315       return Deinitializers.takeError();
316 
317     return Error::success();
318   }
319 
registerInitFunc(JITDylib & JD,SymbolStringPtr InitName)320   void registerInitFunc(JITDylib &JD, SymbolStringPtr InitName) {
321     getExecutionSession().runSessionLocked([&]() {
322         InitFunctions[&JD].add(InitName);
323       });
324   }
325 
registerDeInitFunc(JITDylib & JD,SymbolStringPtr DeInitName)326   void registerDeInitFunc(JITDylib &JD, SymbolStringPtr DeInitName) {
327     getExecutionSession().runSessionLocked(
328         [&]() { DeInitFunctions[&JD].add(DeInitName); });
329   }
330 
331 private:
332 
getInitializers(JITDylib & JD)333   Expected<std::vector<JITTargetAddress>> getInitializers(JITDylib &JD) {
334     if (auto Err = issueInitLookups(JD))
335       return std::move(Err);
336 
337     DenseMap<JITDylib *, SymbolLookupSet> LookupSymbols;
338     std::vector<JITDylibSP> DFSLinkOrder;
339 
340     if (auto Err = getExecutionSession().runSessionLocked([&]() -> Error {
341           if (auto DFSLinkOrderOrErr = JD.getDFSLinkOrder())
342             DFSLinkOrder = std::move(*DFSLinkOrderOrErr);
343           else
344             return DFSLinkOrderOrErr.takeError();
345 
346           for (auto &NextJD : DFSLinkOrder) {
347             auto IFItr = InitFunctions.find(NextJD.get());
348             if (IFItr != InitFunctions.end()) {
349               LookupSymbols[NextJD.get()] = std::move(IFItr->second);
350               InitFunctions.erase(IFItr);
351             }
352           }
353           return Error::success();
354         }))
355       return std::move(Err);
356 
357     LLVM_DEBUG({
358       dbgs() << "JITDylib init order is [ ";
359       for (auto &JD : llvm::reverse(DFSLinkOrder))
360         dbgs() << "\"" << JD->getName() << "\" ";
361       dbgs() << "]\n";
362       dbgs() << "Looking up init functions:\n";
363       for (auto &KV : LookupSymbols)
364         dbgs() << "  \"" << KV.first->getName() << "\": " << KV.second << "\n";
365     });
366 
367     auto &ES = getExecutionSession();
368     auto LookupResult = Platform::lookupInitSymbols(ES, LookupSymbols);
369 
370     if (!LookupResult)
371       return LookupResult.takeError();
372 
373     std::vector<JITTargetAddress> Initializers;
374     while (!DFSLinkOrder.empty()) {
375       auto &NextJD = *DFSLinkOrder.back();
376       DFSLinkOrder.pop_back();
377       auto InitsItr = LookupResult->find(&NextJD);
378       if (InitsItr == LookupResult->end())
379         continue;
380       for (auto &KV : InitsItr->second)
381         Initializers.push_back(KV.second.getAddress());
382     }
383 
384     return Initializers;
385   }
386 
getDeinitializers(JITDylib & JD)387   Expected<std::vector<JITTargetAddress>> getDeinitializers(JITDylib &JD) {
388     auto &ES = getExecutionSession();
389 
390     auto LLJITRunAtExits = J.mangleAndIntern("__lljit_run_atexits");
391 
392     DenseMap<JITDylib *, SymbolLookupSet> LookupSymbols;
393     std::vector<JITDylibSP> DFSLinkOrder;
394 
395     if (auto Err = ES.runSessionLocked([&]() -> Error {
396           if (auto DFSLinkOrderOrErr = JD.getDFSLinkOrder())
397             DFSLinkOrder = std::move(*DFSLinkOrderOrErr);
398           else
399             return DFSLinkOrderOrErr.takeError();
400 
401           for (auto &NextJD : DFSLinkOrder) {
402             auto &JDLookupSymbols = LookupSymbols[NextJD.get()];
403             auto DIFItr = DeInitFunctions.find(NextJD.get());
404             if (DIFItr != DeInitFunctions.end()) {
405               LookupSymbols[NextJD.get()] = std::move(DIFItr->second);
406               DeInitFunctions.erase(DIFItr);
407             }
408             JDLookupSymbols.add(LLJITRunAtExits,
409                                 SymbolLookupFlags::WeaklyReferencedSymbol);
410           }
411           return Error::success();
412         }))
413       return std::move(Err);
414 
415     LLVM_DEBUG({
416       dbgs() << "JITDylib deinit order is [ ";
417       for (auto &JD : DFSLinkOrder)
418         dbgs() << "\"" << JD->getName() << "\" ";
419       dbgs() << "]\n";
420       dbgs() << "Looking up deinit functions:\n";
421       for (auto &KV : LookupSymbols)
422         dbgs() << "  \"" << KV.first->getName() << "\": " << KV.second << "\n";
423     });
424 
425     auto LookupResult = Platform::lookupInitSymbols(ES, LookupSymbols);
426 
427     if (!LookupResult)
428       return LookupResult.takeError();
429 
430     std::vector<JITTargetAddress> DeInitializers;
431     for (auto &NextJD : DFSLinkOrder) {
432       auto DeInitsItr = LookupResult->find(NextJD.get());
433       assert(DeInitsItr != LookupResult->end() &&
434              "Every JD should have at least __lljit_run_atexits");
435 
436       auto RunAtExitsItr = DeInitsItr->second.find(LLJITRunAtExits);
437       if (RunAtExitsItr != DeInitsItr->second.end())
438         DeInitializers.push_back(RunAtExitsItr->second.getAddress());
439 
440       for (auto &KV : DeInitsItr->second)
441         if (KV.first != LLJITRunAtExits)
442           DeInitializers.push_back(KV.second.getAddress());
443     }
444 
445     return DeInitializers;
446   }
447 
448   /// Issue lookups for all init symbols required to initialize JD (and any
449   /// JITDylibs that it depends on).
issueInitLookups(JITDylib & JD)450   Error issueInitLookups(JITDylib &JD) {
451     DenseMap<JITDylib *, SymbolLookupSet> RequiredInitSymbols;
452     std::vector<JITDylibSP> DFSLinkOrder;
453 
454     if (auto Err = getExecutionSession().runSessionLocked([&]() -> Error {
455           if (auto DFSLinkOrderOrErr = JD.getDFSLinkOrder())
456             DFSLinkOrder = std::move(*DFSLinkOrderOrErr);
457           else
458             return DFSLinkOrderOrErr.takeError();
459 
460           for (auto &NextJD : DFSLinkOrder) {
461             auto ISItr = InitSymbols.find(NextJD.get());
462             if (ISItr != InitSymbols.end()) {
463               RequiredInitSymbols[NextJD.get()] = std::move(ISItr->second);
464               InitSymbols.erase(ISItr);
465             }
466           }
467           return Error::success();
468         }))
469       return Err;
470 
471     return Platform::lookupInitSymbols(getExecutionSession(),
472                                        RequiredInitSymbols)
473         .takeError();
474   }
475 
registerCxaAtExitHelper(void * Self,void (* F)(void *),void * Ctx,void * DSOHandle)476   static void registerCxaAtExitHelper(void *Self, void (*F)(void *), void *Ctx,
477                                       void *DSOHandle) {
478     LLVM_DEBUG({
479       dbgs() << "Registering cxa atexit function " << (void *)F << " for JD "
480              << (*static_cast<JITDylib **>(DSOHandle))->getName() << "\n";
481     });
482     static_cast<GenericLLVMIRPlatformSupport *>(Self)->AtExitMgr.registerAtExit(
483         F, Ctx, DSOHandle);
484   }
485 
registerAtExitHelper(void * Self,void * DSOHandle,void (* F)())486   static void registerAtExitHelper(void *Self, void *DSOHandle, void (*F)()) {
487     LLVM_DEBUG({
488       dbgs() << "Registering atexit function " << (void *)F << " for JD "
489              << (*static_cast<JITDylib **>(DSOHandle))->getName() << "\n";
490     });
491     static_cast<GenericLLVMIRPlatformSupport *>(Self)->AtExitMgr.registerAtExit(
492         reinterpret_cast<void (*)(void *)>(F), nullptr, DSOHandle);
493   }
494 
runAtExitsHelper(void * Self,void * DSOHandle)495   static void runAtExitsHelper(void *Self, void *DSOHandle) {
496     LLVM_DEBUG({
497       dbgs() << "Running atexit functions for JD "
498              << (*static_cast<JITDylib **>(DSOHandle))->getName() << "\n";
499     });
500     static_cast<GenericLLVMIRPlatformSupport *>(Self)->AtExitMgr.runAtExits(
501         DSOHandle);
502   }
503 
504   // Constructs an LLVM IR module containing platform runtime globals,
505   // functions, and interposes.
createPlatformRuntimeModule()506   ThreadSafeModule createPlatformRuntimeModule() {
507     auto Ctx = std::make_unique<LLVMContext>();
508     auto M = std::make_unique<Module>("__standard_lib", *Ctx);
509     M->setDataLayout(J.getDataLayout());
510 
511     auto *GenericIRPlatformSupportTy =
512         StructType::create(*Ctx, "lljit.GenericLLJITIRPlatformSupport");
513 
514     auto *PlatformInstanceDecl = new GlobalVariable(
515         *M, GenericIRPlatformSupportTy, true, GlobalValue::ExternalLinkage,
516         nullptr, "__lljit.platform_support_instance");
517 
518     auto *Int8Ty = Type::getInt8Ty(*Ctx);
519     auto *IntTy = Type::getIntNTy(*Ctx, sizeof(int) * CHAR_BIT);
520     auto *VoidTy = Type::getVoidTy(*Ctx);
521     auto *BytePtrTy = PointerType::getUnqual(Int8Ty);
522     auto *CxaAtExitCallbackTy = FunctionType::get(VoidTy, {BytePtrTy}, false);
523     auto *CxaAtExitCallbackPtrTy = PointerType::getUnqual(CxaAtExitCallbackTy);
524 
525     addHelperAndWrapper(
526         *M, "__cxa_atexit",
527         FunctionType::get(IntTy, {CxaAtExitCallbackPtrTy, BytePtrTy, BytePtrTy},
528                           false),
529         GlobalValue::DefaultVisibility, "__lljit.cxa_atexit_helper",
530         {PlatformInstanceDecl});
531 
532     return ThreadSafeModule(std::move(M), std::move(Ctx));
533   }
534 
535   LLJIT &J;
536   std::string InitFunctionPrefix;
537   std::string DeInitFunctionPrefix;
538   DenseMap<JITDylib *, SymbolLookupSet> InitSymbols;
539   DenseMap<JITDylib *, SymbolLookupSet> InitFunctions;
540   DenseMap<JITDylib *, SymbolLookupSet> DeInitFunctions;
541   ItaniumCXAAtExitSupport AtExitMgr;
542 };
543 
setupJITDylib(JITDylib & JD)544 Error GenericLLVMIRPlatform::setupJITDylib(JITDylib &JD) {
545   return S.setupJITDylib(JD);
546 }
547 
teardownJITDylib(JITDylib & JD)548 Error GenericLLVMIRPlatform::teardownJITDylib(JITDylib &JD) {
549   return Error::success();
550 }
551 
notifyAdding(ResourceTracker & RT,const MaterializationUnit & MU)552 Error GenericLLVMIRPlatform::notifyAdding(ResourceTracker &RT,
553                                           const MaterializationUnit &MU) {
554   return S.notifyAdding(RT, MU);
555 }
556 
557 Expected<ThreadSafeModule>
operator ()(ThreadSafeModule TSM,MaterializationResponsibility & R)558 GlobalCtorDtorScraper::operator()(ThreadSafeModule TSM,
559                                   MaterializationResponsibility &R) {
560   auto Err = TSM.withModuleDo([&](Module &M) -> Error {
561     auto &Ctx = M.getContext();
562     auto *GlobalCtors = M.getNamedGlobal("llvm.global_ctors");
563     auto *GlobalDtors = M.getNamedGlobal("llvm.global_dtors");
564 
565     auto RegisterCOrDtors = [&](GlobalVariable *GlobalCOrDtors,
566                                 bool isCtor) -> Error {
567       // If there's no llvm.global_c/dtor or it's just a decl then skip.
568       if (!GlobalCOrDtors || GlobalCOrDtors->isDeclaration())
569         return Error::success();
570       std::string InitOrDeInitFunctionName;
571       if (isCtor)
572         raw_string_ostream(InitOrDeInitFunctionName)
573             << InitFunctionPrefix << M.getModuleIdentifier();
574       else
575         raw_string_ostream(InitOrDeInitFunctionName)
576             << DeInitFunctionPrefix << M.getModuleIdentifier();
577 
578       MangleAndInterner Mangle(PS.getExecutionSession(), M.getDataLayout());
579       auto InternedInitOrDeInitName = Mangle(InitOrDeInitFunctionName);
580       if (auto Err = R.defineMaterializing(
581               {{InternedInitOrDeInitName, JITSymbolFlags::Callable}}))
582         return Err;
583 
584       auto *InitOrDeInitFunc = Function::Create(
585           FunctionType::get(Type::getVoidTy(Ctx), {}, false),
586           GlobalValue::ExternalLinkage, InitOrDeInitFunctionName, &M);
587       InitOrDeInitFunc->setVisibility(GlobalValue::HiddenVisibility);
588       std::vector<std::pair<Function *, unsigned>> InitsOrDeInits;
589       auto COrDtors = isCtor ? getConstructors(M) : getDestructors(M);
590 
591       for (auto E : COrDtors)
592         InitsOrDeInits.push_back(std::make_pair(E.Func, E.Priority));
593       llvm::sort(InitsOrDeInits, llvm::less_second());
594 
595       auto *InitOrDeInitFuncEntryBlock =
596           BasicBlock::Create(Ctx, "entry", InitOrDeInitFunc);
597       IRBuilder<> IB(InitOrDeInitFuncEntryBlock);
598       for (auto &KV : InitsOrDeInits)
599         IB.CreateCall(KV.first);
600       IB.CreateRetVoid();
601 
602       if (isCtor)
603         PS.registerInitFunc(R.getTargetJITDylib(), InternedInitOrDeInitName);
604       else
605         PS.registerDeInitFunc(R.getTargetJITDylib(), InternedInitOrDeInitName);
606 
607       GlobalCOrDtors->eraseFromParent();
608       return Error::success();
609     };
610 
611     if (auto Err = RegisterCOrDtors(GlobalCtors, true))
612       return Err;
613     if (auto Err = RegisterCOrDtors(GlobalDtors, false))
614       return Err;
615 
616     return Error::success();
617   });
618 
619   if (Err)
620     return std::move(Err);
621 
622   return std::move(TSM);
623 }
624 
625 /// Inactive Platform Support
626 ///
627 /// Explicitly disables platform support. JITDylibs are not scanned for special
628 /// init/deinit symbols. No runtime API interposes are injected.
629 class InactivePlatformSupport : public LLJIT::PlatformSupport {
630 public:
631   InactivePlatformSupport() = default;
632 
initialize(JITDylib & JD)633   Error initialize(JITDylib &JD) override {
634     LLVM_DEBUG(dbgs() << "InactivePlatformSupport: no initializers running for "
635                       << JD.getName() << "\n");
636     return Error::success();
637   }
638 
deinitialize(JITDylib & JD)639   Error deinitialize(JITDylib &JD) override {
640     LLVM_DEBUG(
641         dbgs() << "InactivePlatformSupport: no deinitializers running for "
642                << JD.getName() << "\n");
643     return Error::success();
644   }
645 };
646 
647 } // end anonymous namespace
648 
649 namespace llvm {
650 namespace orc {
651 
setInitTransform(LLJIT & J,IRTransformLayer::TransformFunction T)652 void LLJIT::PlatformSupport::setInitTransform(
653     LLJIT &J, IRTransformLayer::TransformFunction T) {
654   J.InitHelperTransformLayer->setTransform(std::move(T));
655 }
656 
657 LLJIT::PlatformSupport::~PlatformSupport() = default;
658 
prepareForConstruction()659 Error LLJITBuilderState::prepareForConstruction() {
660 
661   LLVM_DEBUG(dbgs() << "Preparing to create LLJIT instance...\n");
662 
663   if (!JTMB) {
664     LLVM_DEBUG({
665       dbgs() << "  No explicitly set JITTargetMachineBuilder. "
666                 "Detecting host...\n";
667     });
668     if (auto JTMBOrErr = JITTargetMachineBuilder::detectHost())
669       JTMB = std::move(*JTMBOrErr);
670     else
671       return JTMBOrErr.takeError();
672   }
673 
674   LLVM_DEBUG({
675     dbgs() << "  JITTargetMachineBuilder is "
676            << JITTargetMachineBuilderPrinter(*JTMB, "  ")
677            << "  Pre-constructed ExecutionSession: " << (ES ? "Yes" : "No")
678            << "\n"
679            << "  DataLayout: ";
680     if (DL)
681       dbgs() << DL->getStringRepresentation() << "\n";
682     else
683       dbgs() << "None (will be created by JITTargetMachineBuilder)\n";
684 
685     dbgs() << "  Custom object-linking-layer creator: "
686            << (CreateObjectLinkingLayer ? "Yes" : "No") << "\n"
687            << "  Custom compile-function creator: "
688            << (CreateCompileFunction ? "Yes" : "No") << "\n"
689            << "  Custom platform-setup function: "
690            << (SetUpPlatform ? "Yes" : "No") << "\n"
691            << "  Number of compile threads: " << NumCompileThreads;
692     if (!NumCompileThreads)
693       dbgs() << " (code will be compiled on the execution thread)\n";
694     else
695       dbgs() << "\n";
696   });
697 
698   // If neither ES nor EPC has been set then create an EPC instance.
699   if (!ES && !EPC) {
700     LLVM_DEBUG({
701       dbgs() << "ExecutorProcessControl not specified, "
702                 "Creating SelfExecutorProcessControl instance\n";
703     });
704     if (auto EPCOrErr = SelfExecutorProcessControl::Create())
705       EPC = std::move(*EPCOrErr);
706     else
707       return EPCOrErr.takeError();
708   } else
709     LLVM_DEBUG({
710       dbgs() << "Using explicitly specified ExecutorProcessControl instance "
711              << EPC.get() << "\n";
712     });
713 
714   // If the client didn't configure any linker options then auto-configure the
715   // JIT linker.
716   if (!CreateObjectLinkingLayer) {
717     auto &TT = JTMB->getTargetTriple();
718     if (TT.getArch() == Triple::riscv64 ||
719         TT.getArch() == Triple::loongarch64 ||
720         (TT.isOSBinFormatMachO() &&
721          (TT.getArch() == Triple::aarch64 || TT.getArch() == Triple::x86_64))) {
722 
723       JTMB->setRelocationModel(Reloc::PIC_);
724       JTMB->setCodeModel(CodeModel::Small);
725       CreateObjectLinkingLayer =
726           [](ExecutionSession &ES,
727              const Triple &) -> Expected<std::unique_ptr<ObjectLayer>> {
728         auto ObjLinkingLayer = std::make_unique<ObjectLinkingLayer>(ES);
729         if (auto EHFrameRegistrar = EPCEHFrameRegistrar::Create(ES))
730           ObjLinkingLayer->addPlugin(
731               std::make_unique<EHFrameRegistrationPlugin>(
732                   ES, std::move(*EHFrameRegistrar)));
733         else
734           return EHFrameRegistrar.takeError();
735         return std::move(ObjLinkingLayer);
736       };
737     }
738   }
739 
740   return Error::success();
741 }
742 
~LLJIT()743 LLJIT::~LLJIT() {
744   if (CompileThreads)
745     CompileThreads->wait();
746   if (auto Err = ES->endSession())
747     ES->reportError(std::move(Err));
748 }
749 
addIRModule(ResourceTrackerSP RT,ThreadSafeModule TSM)750 Error LLJIT::addIRModule(ResourceTrackerSP RT, ThreadSafeModule TSM) {
751   assert(TSM && "Can not add null module");
752 
753   if (auto Err =
754           TSM.withModuleDo([&](Module &M) { return applyDataLayout(M); }))
755     return Err;
756 
757   return InitHelperTransformLayer->add(std::move(RT), std::move(TSM));
758 }
759 
addIRModule(JITDylib & JD,ThreadSafeModule TSM)760 Error LLJIT::addIRModule(JITDylib &JD, ThreadSafeModule TSM) {
761   return addIRModule(JD.getDefaultResourceTracker(), std::move(TSM));
762 }
763 
addObjectFile(ResourceTrackerSP RT,std::unique_ptr<MemoryBuffer> Obj)764 Error LLJIT::addObjectFile(ResourceTrackerSP RT,
765                            std::unique_ptr<MemoryBuffer> Obj) {
766   assert(Obj && "Can not add null object");
767 
768   return ObjTransformLayer->add(std::move(RT), std::move(Obj));
769 }
770 
addObjectFile(JITDylib & JD,std::unique_ptr<MemoryBuffer> Obj)771 Error LLJIT::addObjectFile(JITDylib &JD, std::unique_ptr<MemoryBuffer> Obj) {
772   return addObjectFile(JD.getDefaultResourceTracker(), std::move(Obj));
773 }
774 
lookupLinkerMangled(JITDylib & JD,SymbolStringPtr Name)775 Expected<ExecutorAddr> LLJIT::lookupLinkerMangled(JITDylib &JD,
776                                                   SymbolStringPtr Name) {
777   if (auto Sym = ES->lookup(
778         makeJITDylibSearchOrder(&JD, JITDylibLookupFlags::MatchAllSymbols),
779         Name))
780     return ExecutorAddr(Sym->getAddress());
781   else
782     return Sym.takeError();
783 }
784 
785 Expected<std::unique_ptr<ObjectLayer>>
createObjectLinkingLayer(LLJITBuilderState & S,ExecutionSession & ES)786 LLJIT::createObjectLinkingLayer(LLJITBuilderState &S, ExecutionSession &ES) {
787 
788   // If the config state provided an ObjectLinkingLayer factory then use it.
789   if (S.CreateObjectLinkingLayer)
790     return S.CreateObjectLinkingLayer(ES, S.JTMB->getTargetTriple());
791 
792   // Otherwise default to creating an RTDyldObjectLinkingLayer that constructs
793   // a new SectionMemoryManager for each object.
794   auto GetMemMgr = []() { return std::make_unique<SectionMemoryManager>(); };
795   auto Layer =
796       std::make_unique<RTDyldObjectLinkingLayer>(ES, std::move(GetMemMgr));
797 
798   if (S.JTMB->getTargetTriple().isOSBinFormatCOFF()) {
799     Layer->setOverrideObjectFlagsWithResponsibilityFlags(true);
800     Layer->setAutoClaimResponsibilityForObjectSymbols(true);
801   }
802 
803   if (S.JTMB->getTargetTriple().isOSBinFormatELF() &&
804       (S.JTMB->getTargetTriple().getArch() == Triple::ArchType::ppc64 ||
805        S.JTMB->getTargetTriple().getArch() == Triple::ArchType::ppc64le))
806     Layer->setAutoClaimResponsibilityForObjectSymbols(true);
807 
808   // FIXME: Explicit conversion to std::unique_ptr<ObjectLayer> added to silence
809   //        errors from some GCC / libstdc++ bots. Remove this conversion (i.e.
810   //        just return ObjLinkingLayer) once those bots are upgraded.
811   return std::unique_ptr<ObjectLayer>(std::move(Layer));
812 }
813 
814 Expected<std::unique_ptr<IRCompileLayer::IRCompiler>>
createCompileFunction(LLJITBuilderState & S,JITTargetMachineBuilder JTMB)815 LLJIT::createCompileFunction(LLJITBuilderState &S,
816                              JITTargetMachineBuilder JTMB) {
817 
818   /// If there is a custom compile function creator set then use it.
819   if (S.CreateCompileFunction)
820     return S.CreateCompileFunction(std::move(JTMB));
821 
822   // Otherwise default to creating a SimpleCompiler, or ConcurrentIRCompiler,
823   // depending on the number of threads requested.
824   if (S.NumCompileThreads > 0)
825     return std::make_unique<ConcurrentIRCompiler>(std::move(JTMB));
826 
827   auto TM = JTMB.createTargetMachine();
828   if (!TM)
829     return TM.takeError();
830 
831   return std::make_unique<TMOwningSimpleCompiler>(std::move(*TM));
832 }
833 
LLJIT(LLJITBuilderState & S,Error & Err)834 LLJIT::LLJIT(LLJITBuilderState &S, Error &Err)
835     : DL(""), TT(S.JTMB->getTargetTriple()) {
836 
837   ErrorAsOutParameter _(&Err);
838 
839   assert(!(S.EPC && S.ES) && "EPC and ES should not both be set");
840 
841   if (S.EPC) {
842     ES = std::make_unique<ExecutionSession>(std::move(S.EPC));
843   } else if (S.ES)
844     ES = std::move(S.ES);
845   else {
846     if (auto EPC = SelfExecutorProcessControl::Create()) {
847       ES = std::make_unique<ExecutionSession>(std::move(*EPC));
848     } else {
849       Err = EPC.takeError();
850       return;
851     }
852   }
853 
854   if (auto MainOrErr = this->ES->createJITDylib("main"))
855     Main = &*MainOrErr;
856   else {
857     Err = MainOrErr.takeError();
858     return;
859   }
860 
861   if (S.DL)
862     DL = std::move(*S.DL);
863   else if (auto DLOrErr = S.JTMB->getDefaultDataLayoutForTarget())
864     DL = std::move(*DLOrErr);
865   else {
866     Err = DLOrErr.takeError();
867     return;
868   }
869 
870   auto ObjLayer = createObjectLinkingLayer(S, *ES);
871   if (!ObjLayer) {
872     Err = ObjLayer.takeError();
873     return;
874   }
875   ObjLinkingLayer = std::move(*ObjLayer);
876   ObjTransformLayer =
877       std::make_unique<ObjectTransformLayer>(*ES, *ObjLinkingLayer);
878 
879   {
880     auto CompileFunction = createCompileFunction(S, std::move(*S.JTMB));
881     if (!CompileFunction) {
882       Err = CompileFunction.takeError();
883       return;
884     }
885     CompileLayer = std::make_unique<IRCompileLayer>(
886         *ES, *ObjTransformLayer, std::move(*CompileFunction));
887     TransformLayer = std::make_unique<IRTransformLayer>(*ES, *CompileLayer);
888     InitHelperTransformLayer =
889         std::make_unique<IRTransformLayer>(*ES, *TransformLayer);
890   }
891 
892   if (S.NumCompileThreads > 0) {
893     InitHelperTransformLayer->setCloneToNewContextOnEmit(true);
894     CompileThreads =
895         std::make_unique<ThreadPool>(hardware_concurrency(S.NumCompileThreads));
896     ES->setDispatchTask([this](std::unique_ptr<Task> T) {
897       // FIXME: We should be able to use move-capture here, but ThreadPool's
898       // AsyncTaskTys are std::functions rather than unique_functions
899       // (because MSVC's std::packaged_tasks don't support move-only types).
900       // Fix this when all the above gets sorted out.
901       CompileThreads->async([UnownedT = T.release()]() mutable {
902         std::unique_ptr<Task> T(UnownedT);
903         T->run();
904       });
905     });
906   }
907 
908   if (S.SetUpPlatform)
909     Err = S.SetUpPlatform(*this);
910   else
911     setUpGenericLLVMIRPlatform(*this);
912 }
913 
mangle(StringRef UnmangledName) const914 std::string LLJIT::mangle(StringRef UnmangledName) const {
915   std::string MangledName;
916   {
917     raw_string_ostream MangledNameStream(MangledName);
918     Mangler::getNameWithPrefix(MangledNameStream, UnmangledName, DL);
919   }
920   return MangledName;
921 }
922 
applyDataLayout(Module & M)923 Error LLJIT::applyDataLayout(Module &M) {
924   if (M.getDataLayout().isDefault())
925     M.setDataLayout(DL);
926 
927   if (M.getDataLayout() != DL)
928     return make_error<StringError>(
929         "Added modules have incompatible data layouts: " +
930             M.getDataLayout().getStringRepresentation() + " (module) vs " +
931             DL.getStringRepresentation() + " (jit)",
932         inconvertibleErrorCode());
933 
934   return Error::success();
935 }
936 
setUpOrcPlatform(LLJIT & J)937 Error setUpOrcPlatform(LLJIT& J) {
938     LLVM_DEBUG(
939         { dbgs() << "Setting up orc platform support for LLJIT\n"; });
940     J.setPlatformSupport(std::make_unique<ORCPlatformSupport>(J));
941     return Error::success();
942 }
943 
setUpGenericLLVMIRPlatform(LLJIT & J)944 void setUpGenericLLVMIRPlatform(LLJIT &J) {
945   LLVM_DEBUG(
946       { dbgs() << "Setting up GenericLLVMIRPlatform support for LLJIT\n"; });
947   J.setPlatformSupport(std::make_unique<GenericLLVMIRPlatformSupport>(J));
948 }
949 
setUpInactivePlatform(LLJIT & J)950 Error setUpInactivePlatform(LLJIT &J) {
951   LLVM_DEBUG(
952       { dbgs() << "Explicitly deactivated platform support for LLJIT\n"; });
953   J.setPlatformSupport(std::make_unique<InactivePlatformSupport>());
954   return Error::success();
955 }
956 
prepareForConstruction()957 Error LLLazyJITBuilderState::prepareForConstruction() {
958   if (auto Err = LLJITBuilderState::prepareForConstruction())
959     return Err;
960   TT = JTMB->getTargetTriple();
961   return Error::success();
962 }
963 
addLazyIRModule(JITDylib & JD,ThreadSafeModule TSM)964 Error LLLazyJIT::addLazyIRModule(JITDylib &JD, ThreadSafeModule TSM) {
965   assert(TSM && "Can not add null module");
966 
967   if (auto Err = TSM.withModuleDo(
968           [&](Module &M) -> Error { return applyDataLayout(M); }))
969     return Err;
970 
971   return CODLayer->add(JD, std::move(TSM));
972 }
973 
LLLazyJIT(LLLazyJITBuilderState & S,Error & Err)974 LLLazyJIT::LLLazyJIT(LLLazyJITBuilderState &S, Error &Err) : LLJIT(S, Err) {
975 
976   // If LLJIT construction failed then bail out.
977   if (Err)
978     return;
979 
980   ErrorAsOutParameter _(&Err);
981 
982   /// Take/Create the lazy-compile callthrough manager.
983   if (S.LCTMgr)
984     LCTMgr = std::move(S.LCTMgr);
985   else {
986     if (auto LCTMgrOrErr = createLocalLazyCallThroughManager(
987         S.TT, *ES, S.LazyCompileFailureAddr.getValue()))
988       LCTMgr = std::move(*LCTMgrOrErr);
989     else {
990       Err = LCTMgrOrErr.takeError();
991       return;
992     }
993   }
994 
995   // Take/Create the indirect stubs manager builder.
996   auto ISMBuilder = std::move(S.ISMBuilder);
997 
998   // If none was provided, try to build one.
999   if (!ISMBuilder)
1000     ISMBuilder = createLocalIndirectStubsManagerBuilder(S.TT);
1001 
1002   // No luck. Bail out.
1003   if (!ISMBuilder) {
1004     Err = make_error<StringError>("Could not construct "
1005                                   "IndirectStubsManagerBuilder for target " +
1006                                       S.TT.str(),
1007                                   inconvertibleErrorCode());
1008     return;
1009   }
1010 
1011   // Create the COD layer.
1012   CODLayer = std::make_unique<CompileOnDemandLayer>(
1013       *ES, *InitHelperTransformLayer, *LCTMgr, std::move(ISMBuilder));
1014 
1015   if (S.NumCompileThreads > 0)
1016     CODLayer->setCloneToNewContextOnEmit(true);
1017 }
1018 
1019 // In-process LLJIT uses eh-frame section wrappers via EPC, so we need to force
1020 // them to be linked in.
linkComponents()1021 LLVM_ATTRIBUTE_USED void linkComponents() {
1022   errs() << (void *)&llvm_orc_registerEHFrameSectionWrapper
1023          << (void *)&llvm_orc_deregisterEHFrameSectionWrapper;
1024 }
1025 
1026 } // End namespace orc.
1027 } // End namespace llvm.
1028