xref: /llvm-project/llvm/lib/ExecutionEngine/Orc/ELFNixPlatform.cpp (revision 79231a86846b7dff09497fc58ea1e82e892052bd) 
1 //===------ ELFNixPlatform.cpp - Utilities for executing ELFNix in Orc
2 //-----===//
3 //
4 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
5 // See https://llvm.org/LICENSE.txt for license information.
6 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 //
8 //===----------------------------------------------------------------------===//
9 
10 #include "llvm/ExecutionEngine/Orc/ELFNixPlatform.h"
11 
12 #include "llvm/ExecutionEngine/JITLink/aarch64.h"
13 #include "llvm/ExecutionEngine/JITLink/loongarch.h"
14 #include "llvm/ExecutionEngine/JITLink/ppc64.h"
15 #include "llvm/ExecutionEngine/JITLink/x86_64.h"
16 #include "llvm/ExecutionEngine/Orc/AbsoluteSymbols.h"
17 #include "llvm/ExecutionEngine/Orc/ExecutionUtils.h"
18 #include "llvm/ExecutionEngine/Orc/Shared/ObjectFormats.h"
19 #include "llvm/Support/Debug.h"
20 #include <optional>
21 
22 #define DEBUG_TYPE "orc"
23 
24 using namespace llvm;
25 using namespace llvm::orc;
26 using namespace llvm::orc::shared;
27 
28 namespace {
29 
30 template <typename SPSSerializer, typename... ArgTs>
31 shared::WrapperFunctionCall::ArgDataBufferType
32 getArgDataBufferType(const ArgTs &...Args) {
33   shared::WrapperFunctionCall::ArgDataBufferType ArgData;
34   ArgData.resize(SPSSerializer::size(Args...));
35   SPSOutputBuffer OB(ArgData.empty() ? nullptr : ArgData.data(),
36                      ArgData.size());
37   if (SPSSerializer::serialize(OB, Args...))
38     return ArgData;
39   return {};
40 }
41 
42 std::unique_ptr<jitlink::LinkGraph> createPlatformGraph(ELFNixPlatform &MOP,
43                                                         std::string Name) {
44   auto &ES = MOP.getExecutionSession();
45   return std::make_unique<jitlink::LinkGraph>(
46       std::move(Name), ES.getSymbolStringPool(), ES.getTargetTriple(),
47       SubtargetFeatures(), jitlink::getGenericEdgeKindName);
48 }
49 
50 // Creates a Bootstrap-Complete LinkGraph to run deferred actions.
51 class ELFNixPlatformCompleteBootstrapMaterializationUnit
52     : public MaterializationUnit {
53 public:
54   ELFNixPlatformCompleteBootstrapMaterializationUnit(
55       ELFNixPlatform &MOP, StringRef PlatformJDName,
56       SymbolStringPtr CompleteBootstrapSymbol, DeferredRuntimeFnMap DeferredAAs,
57       ExecutorAddr ELFNixHeaderAddr, ExecutorAddr PlatformBootstrap,
58       ExecutorAddr PlatformShutdown, ExecutorAddr RegisterJITDylib,
59       ExecutorAddr DeregisterJITDylib)
60       : MaterializationUnit(
61             {{{CompleteBootstrapSymbol, JITSymbolFlags::None}}, nullptr}),
62         MOP(MOP), PlatformJDName(PlatformJDName),
63         CompleteBootstrapSymbol(std::move(CompleteBootstrapSymbol)),
64         DeferredAAsMap(std::move(DeferredAAs)),
65         ELFNixHeaderAddr(ELFNixHeaderAddr),
66         PlatformBootstrap(PlatformBootstrap),
67         PlatformShutdown(PlatformShutdown), RegisterJITDylib(RegisterJITDylib),
68         DeregisterJITDylib(DeregisterJITDylib) {}
69 
70   StringRef getName() const override {
71     return "ELFNixPlatformCompleteBootstrap";
72   }
73 
74   void materialize(std::unique_ptr<MaterializationResponsibility> R) override {
75     using namespace jitlink;
76     auto G = createPlatformGraph(MOP, "<OrcRTCompleteBootstrap>");
77     auto &PlaceholderSection =
78         G->createSection("__orc_rt_cplt_bs", MemProt::Read);
79     auto &PlaceholderBlock =
80         G->createZeroFillBlock(PlaceholderSection, 1, ExecutorAddr(), 1, 0);
81     G->addDefinedSymbol(PlaceholderBlock, 0, *CompleteBootstrapSymbol, 1,
82                         Linkage::Strong, Scope::Hidden, false, true);
83 
84     // 1. Bootstrap the platform support code.
85     G->allocActions().push_back(
86         {cantFail(WrapperFunctionCall::Create<SPSArgList<SPSExecutorAddr>>(
87              PlatformBootstrap, ELFNixHeaderAddr)),
88          cantFail(
89              WrapperFunctionCall::Create<SPSArgList<>>(PlatformShutdown))});
90 
91     // 2. Register the platform JITDylib.
92     G->allocActions().push_back(
93         {cantFail(WrapperFunctionCall::Create<
94                   SPSArgList<SPSString, SPSExecutorAddr>>(
95              RegisterJITDylib, PlatformJDName, ELFNixHeaderAddr)),
96          cantFail(WrapperFunctionCall::Create<SPSArgList<SPSExecutorAddr>>(
97              DeregisterJITDylib, ELFNixHeaderAddr))});
98 
99     // 4. Add the deferred actions to the graph.
100     for (auto &[Fn, CallDatas] : DeferredAAsMap) {
101       for (auto &CallData : CallDatas) {
102         G->allocActions().push_back(
103             {WrapperFunctionCall(Fn.first->Addr, std::move(CallData.first)),
104              WrapperFunctionCall(Fn.second->Addr, std::move(CallData.second))});
105       }
106     }
107 
108     MOP.getObjectLinkingLayer().emit(std::move(R), std::move(G));
109   }
110 
111   void discard(const JITDylib &JD, const SymbolStringPtr &Sym) override {}
112 
113 private:
114   ELFNixPlatform &MOP;
115   StringRef PlatformJDName;
116   SymbolStringPtr CompleteBootstrapSymbol;
117   DeferredRuntimeFnMap DeferredAAsMap;
118   ExecutorAddr ELFNixHeaderAddr;
119   ExecutorAddr PlatformBootstrap;
120   ExecutorAddr PlatformShutdown;
121   ExecutorAddr RegisterJITDylib;
122   ExecutorAddr DeregisterJITDylib;
123 };
124 
125 class DSOHandleMaterializationUnit : public MaterializationUnit {
126 public:
127   DSOHandleMaterializationUnit(ELFNixPlatform &ENP,
128                                const SymbolStringPtr &DSOHandleSymbol)
129       : MaterializationUnit(
130             createDSOHandleSectionInterface(ENP, DSOHandleSymbol)),
131         ENP(ENP) {}
132 
133   StringRef getName() const override { return "DSOHandleMU"; }
134 
135   void materialize(std::unique_ptr<MaterializationResponsibility> R) override {
136 
137     auto &ES = ENP.getExecutionSession();
138 
139     jitlink::Edge::Kind EdgeKind;
140 
141     switch (ES.getTargetTriple().getArch()) {
142     case Triple::x86_64:
143       EdgeKind = jitlink::x86_64::Pointer64;
144       break;
145     case Triple::aarch64:
146       EdgeKind = jitlink::aarch64::Pointer64;
147       break;
148     case Triple::ppc64:
149       EdgeKind = jitlink::ppc64::Pointer64;
150       break;
151     case Triple::ppc64le:
152       EdgeKind = jitlink::ppc64::Pointer64;
153       break;
154     case Triple::loongarch64:
155       EdgeKind = jitlink::loongarch::Pointer64;
156       break;
157     default:
158       llvm_unreachable("Unrecognized architecture");
159     }
160 
161     // void *__dso_handle = &__dso_handle;
162     auto G = std::make_unique<jitlink::LinkGraph>(
163         "<DSOHandleMU>", ES.getSymbolStringPool(), ES.getTargetTriple(),
164         SubtargetFeatures(), jitlink::getGenericEdgeKindName);
165     auto &DSOHandleSection =
166         G->createSection(".data.__dso_handle", MemProt::Read);
167     auto &DSOHandleBlock = G->createContentBlock(
168         DSOHandleSection, getDSOHandleContent(G->getPointerSize()),
169         orc::ExecutorAddr(), 8, 0);
170     auto &DSOHandleSymbol = G->addDefinedSymbol(
171         DSOHandleBlock, 0, *R->getInitializerSymbol(), DSOHandleBlock.getSize(),
172         jitlink::Linkage::Strong, jitlink::Scope::Default, false, true);
173     DSOHandleBlock.addEdge(EdgeKind, 0, DSOHandleSymbol, 0);
174 
175     ENP.getObjectLinkingLayer().emit(std::move(R), std::move(G));
176   }
177 
178   void discard(const JITDylib &JD, const SymbolStringPtr &Sym) override {}
179 
180 private:
181   static MaterializationUnit::Interface
182   createDSOHandleSectionInterface(ELFNixPlatform &ENP,
183                                   const SymbolStringPtr &DSOHandleSymbol) {
184     SymbolFlagsMap SymbolFlags;
185     SymbolFlags[DSOHandleSymbol] = JITSymbolFlags::Exported;
186     return MaterializationUnit::Interface(std::move(SymbolFlags),
187                                           DSOHandleSymbol);
188   }
189 
190   ArrayRef<char> getDSOHandleContent(size_t PointerSize) {
191     static const char Content[8] = {0};
192     assert(PointerSize <= sizeof Content);
193     return {Content, PointerSize};
194   }
195 
196   ELFNixPlatform &ENP;
197 };
198 
199 } // end anonymous namespace
200 
201 namespace llvm {
202 namespace orc {
203 
204 Expected<std::unique_ptr<ELFNixPlatform>>
205 ELFNixPlatform::Create(ObjectLinkingLayer &ObjLinkingLayer,
206                        JITDylib &PlatformJD,
207                        std::unique_ptr<DefinitionGenerator> OrcRuntime,
208                        std::optional<SymbolAliasMap> RuntimeAliases) {
209 
210   auto &ES = ObjLinkingLayer.getExecutionSession();
211 
212   // If the target is not supported then bail out immediately.
213   if (!supportedTarget(ES.getTargetTriple()))
214     return make_error<StringError>("Unsupported ELFNixPlatform triple: " +
215                                        ES.getTargetTriple().str(),
216                                    inconvertibleErrorCode());
217 
218   auto &EPC = ES.getExecutorProcessControl();
219 
220   // Create default aliases if the caller didn't supply any.
221   if (!RuntimeAliases) {
222     auto StandardRuntimeAliases = standardPlatformAliases(ES, PlatformJD);
223     if (!StandardRuntimeAliases)
224       return StandardRuntimeAliases.takeError();
225     RuntimeAliases = std::move(*StandardRuntimeAliases);
226   }
227 
228   // Define the aliases.
229   if (auto Err = PlatformJD.define(symbolAliases(std::move(*RuntimeAliases))))
230     return std::move(Err);
231 
232   // Add JIT-dispatch function support symbols.
233   if (auto Err = PlatformJD.define(
234           absoluteSymbols({{ES.intern("__orc_rt_jit_dispatch"),
235                             {EPC.getJITDispatchInfo().JITDispatchFunction,
236                              JITSymbolFlags::Exported}},
237                            {ES.intern("__orc_rt_jit_dispatch_ctx"),
238                             {EPC.getJITDispatchInfo().JITDispatchContext,
239                              JITSymbolFlags::Exported}}})))
240     return std::move(Err);
241 
242   // Create the instance.
243   Error Err = Error::success();
244   auto P = std::unique_ptr<ELFNixPlatform>(new ELFNixPlatform(
245       ObjLinkingLayer, PlatformJD, std::move(OrcRuntime), Err));
246   if (Err)
247     return std::move(Err);
248   return std::move(P);
249 }
250 
251 Expected<std::unique_ptr<ELFNixPlatform>>
252 ELFNixPlatform::Create(ObjectLinkingLayer &ObjLinkingLayer,
253                        JITDylib &PlatformJD, const char *OrcRuntimePath,
254                        std::optional<SymbolAliasMap> RuntimeAliases) {
255 
256   // Create a generator for the ORC runtime archive.
257   auto OrcRuntimeArchiveGenerator =
258       StaticLibraryDefinitionGenerator::Load(ObjLinkingLayer, OrcRuntimePath);
259   if (!OrcRuntimeArchiveGenerator)
260     return OrcRuntimeArchiveGenerator.takeError();
261 
262   return Create(ObjLinkingLayer, PlatformJD,
263                 std::move(*OrcRuntimeArchiveGenerator),
264                 std::move(RuntimeAliases));
265 }
266 
267 Error ELFNixPlatform::setupJITDylib(JITDylib &JD) {
268   if (auto Err = JD.define(std::make_unique<DSOHandleMaterializationUnit>(
269           *this, DSOHandleSymbol)))
270     return Err;
271 
272   return ES.lookup({&JD}, DSOHandleSymbol).takeError();
273 }
274 
275 Error ELFNixPlatform::teardownJITDylib(JITDylib &JD) {
276   std::lock_guard<std::mutex> Lock(PlatformMutex);
277   auto I = JITDylibToHandleAddr.find(&JD);
278   if (I != JITDylibToHandleAddr.end()) {
279     assert(HandleAddrToJITDylib.count(I->second) &&
280            "HandleAddrToJITDylib missing entry");
281     HandleAddrToJITDylib.erase(I->second);
282     JITDylibToHandleAddr.erase(I);
283   }
284   return Error::success();
285 }
286 
287 Error ELFNixPlatform::notifyAdding(ResourceTracker &RT,
288                                    const MaterializationUnit &MU) {
289 
290   auto &JD = RT.getJITDylib();
291   const auto &InitSym = MU.getInitializerSymbol();
292   if (!InitSym)
293     return Error::success();
294 
295   RegisteredInitSymbols[&JD].add(InitSym,
296                                  SymbolLookupFlags::WeaklyReferencedSymbol);
297   LLVM_DEBUG({
298     dbgs() << "ELFNixPlatform: Registered init symbol " << *InitSym
299            << " for MU " << MU.getName() << "\n";
300   });
301   return Error::success();
302 }
303 
304 Error ELFNixPlatform::notifyRemoving(ResourceTracker &RT) {
305   llvm_unreachable("Not supported yet");
306 }
307 
308 static void addAliases(ExecutionSession &ES, SymbolAliasMap &Aliases,
309                        ArrayRef<std::pair<const char *, const char *>> AL) {
310   for (auto &KV : AL) {
311     auto AliasName = ES.intern(KV.first);
312     assert(!Aliases.count(AliasName) && "Duplicate symbol name in alias map");
313     Aliases[std::move(AliasName)] = {ES.intern(KV.second),
314                                      JITSymbolFlags::Exported};
315   }
316 }
317 
318 Expected<SymbolAliasMap>
319 ELFNixPlatform::standardPlatformAliases(ExecutionSession &ES,
320                                         JITDylib &PlatformJD) {
321   SymbolAliasMap Aliases;
322   addAliases(ES, Aliases, requiredCXXAliases());
323   addAliases(ES, Aliases, standardRuntimeUtilityAliases());
324   addAliases(ES, Aliases, standardLazyCompilationAliases());
325   return Aliases;
326 }
327 
328 ArrayRef<std::pair<const char *, const char *>>
329 ELFNixPlatform::requiredCXXAliases() {
330   static const std::pair<const char *, const char *> RequiredCXXAliases[] = {
331       {"__cxa_atexit", "__orc_rt_elfnix_cxa_atexit"},
332       {"atexit", "__orc_rt_elfnix_atexit"}};
333 
334   return ArrayRef<std::pair<const char *, const char *>>(RequiredCXXAliases);
335 }
336 
337 ArrayRef<std::pair<const char *, const char *>>
338 ELFNixPlatform::standardRuntimeUtilityAliases() {
339   static const std::pair<const char *, const char *>
340       StandardRuntimeUtilityAliases[] = {
341           {"__orc_rt_run_program", "__orc_rt_elfnix_run_program"},
342           {"__orc_rt_jit_dlerror", "__orc_rt_elfnix_jit_dlerror"},
343           {"__orc_rt_jit_dlopen", "__orc_rt_elfnix_jit_dlopen"},
344           {"__orc_rt_jit_dlupdate", "__orc_rt_elfnix_jit_dlupdate"},
345           {"__orc_rt_jit_dlclose", "__orc_rt_elfnix_jit_dlclose"},
346           {"__orc_rt_jit_dlsym", "__orc_rt_elfnix_jit_dlsym"},
347           {"__orc_rt_log_error", "__orc_rt_log_error_to_stderr"}};
348 
349   return ArrayRef<std::pair<const char *, const char *>>(
350       StandardRuntimeUtilityAliases);
351 }
352 
353 ArrayRef<std::pair<const char *, const char *>>
354 ELFNixPlatform::standardLazyCompilationAliases() {
355   static const std::pair<const char *, const char *>
356       StandardLazyCompilationAliases[] = {
357           {"__orc_rt_reenter", "__orc_rt_sysv_reenter"}};
358 
359   return ArrayRef<std::pair<const char *, const char *>>(
360       StandardLazyCompilationAliases);
361 }
362 
363 bool ELFNixPlatform::supportedTarget(const Triple &TT) {
364   switch (TT.getArch()) {
365   case Triple::x86_64:
366   case Triple::aarch64:
367   // FIXME: jitlink for ppc64 hasn't been well tested, leave it unsupported
368   // right now.
369   case Triple::ppc64le:
370   case Triple::loongarch64:
371     return true;
372   default:
373     return false;
374   }
375 }
376 
377 ELFNixPlatform::ELFNixPlatform(
378     ObjectLinkingLayer &ObjLinkingLayer, JITDylib &PlatformJD,
379     std::unique_ptr<DefinitionGenerator> OrcRuntimeGenerator, Error &Err)
380     : ES(ObjLinkingLayer.getExecutionSession()), PlatformJD(PlatformJD),
381       ObjLinkingLayer(ObjLinkingLayer),
382       DSOHandleSymbol(ES.intern("__dso_handle")) {
383   ErrorAsOutParameter _(Err);
384   ObjLinkingLayer.addPlugin(std::make_unique<ELFNixPlatformPlugin>(*this));
385 
386   PlatformJD.addGenerator(std::move(OrcRuntimeGenerator));
387 
388   BootstrapInfo BI;
389   Bootstrap = &BI;
390 
391   // PlatformJD hasn't been 'set-up' by the platform yet (since we're creating
392   // the platform now), so set it up.
393   if (auto E2 = setupJITDylib(PlatformJD)) {
394     Err = std::move(E2);
395     return;
396   }
397 
398   // Step (2) Request runtime registration functions to trigger
399   // materialization..
400   if ((Err = ES.lookup(
401                    makeJITDylibSearchOrder(&PlatformJD),
402                    SymbolLookupSet(
403                        {PlatformBootstrap.Name, PlatformShutdown.Name,
404                         RegisterJITDylib.Name, DeregisterJITDylib.Name,
405                         RegisterInitSections.Name, DeregisterInitSections.Name,
406                         RegisterObjectSections.Name,
407                         DeregisterObjectSections.Name, CreatePThreadKey.Name}))
408                  .takeError()))
409     return;
410 
411   // Step (3) Wait for any incidental linker work to complete.
412   {
413     std::unique_lock<std::mutex> Lock(BI.Mutex);
414     BI.CV.wait(Lock, [&]() { return BI.ActiveGraphs == 0; });
415     Bootstrap = nullptr;
416   }
417 
418   // Step (4) Add complete-bootstrap materialization unit and request.
419   auto BootstrapCompleteSymbol =
420       ES.intern("__orc_rt_elfnix_complete_bootstrap");
421   if ((Err = PlatformJD.define(
422            std::make_unique<ELFNixPlatformCompleteBootstrapMaterializationUnit>(
423                *this, PlatformJD.getName(), BootstrapCompleteSymbol,
424                std::move(BI.DeferredRTFnMap), BI.ELFNixHeaderAddr,
425                PlatformBootstrap.Addr, PlatformShutdown.Addr,
426                RegisterJITDylib.Addr, DeregisterJITDylib.Addr))))
427     return;
428   if ((Err = ES.lookup(makeJITDylibSearchOrder(
429                            &PlatformJD, JITDylibLookupFlags::MatchAllSymbols),
430                        std::move(BootstrapCompleteSymbol))
431                  .takeError()))
432     return;
433 
434   // Associate wrapper function tags with JIT-side function implementations.
435   if (auto E2 = associateRuntimeSupportFunctions(PlatformJD)) {
436     Err = std::move(E2);
437     return;
438   }
439 }
440 
441 Error ELFNixPlatform::associateRuntimeSupportFunctions(JITDylib &PlatformJD) {
442   ExecutionSession::JITDispatchHandlerAssociationMap WFs;
443 
444   using RecordInitializersSPSSig =
445       SPSExpected<SPSELFNixJITDylibDepInfoMap>(SPSExecutorAddr);
446   WFs[ES.intern("__orc_rt_elfnix_push_initializers_tag")] =
447       ES.wrapAsyncWithSPS<RecordInitializersSPSSig>(
448           this, &ELFNixPlatform::rt_recordInitializers);
449 
450   using LookupSymbolSPSSig =
451       SPSExpected<SPSExecutorAddr>(SPSExecutorAddr, SPSString);
452   WFs[ES.intern("__orc_rt_elfnix_symbol_lookup_tag")] =
453       ES.wrapAsyncWithSPS<LookupSymbolSPSSig>(this,
454                                               &ELFNixPlatform::rt_lookupSymbol);
455 
456   return ES.registerJITDispatchHandlers(PlatformJD, std::move(WFs));
457 }
458 
459 void ELFNixPlatform::pushInitializersLoop(
460     PushInitializersSendResultFn SendResult, JITDylibSP JD) {
461   DenseMap<JITDylib *, SymbolLookupSet> NewInitSymbols;
462   DenseMap<JITDylib *, SmallVector<JITDylib *>> JDDepMap;
463   SmallVector<JITDylib *, 16> Worklist({JD.get()});
464 
465   ES.runSessionLocked([&]() {
466     while (!Worklist.empty()) {
467       // FIXME: Check for defunct dylibs.
468 
469       auto DepJD = Worklist.back();
470       Worklist.pop_back();
471 
472       // If we've already visited this JITDylib on this iteration then continue.
473       if (JDDepMap.count(DepJD))
474         continue;
475 
476       // Add dep info.
477       auto &DM = JDDepMap[DepJD];
478       DepJD->withLinkOrderDo([&](const JITDylibSearchOrder &O) {
479         for (auto &KV : O) {
480           if (KV.first == DepJD)
481             continue;
482           DM.push_back(KV.first);
483           Worklist.push_back(KV.first);
484         }
485       });
486 
487       // Add any registered init symbols.
488       auto RISItr = RegisteredInitSymbols.find(DepJD);
489       if (RISItr != RegisteredInitSymbols.end()) {
490         NewInitSymbols[DepJD] = std::move(RISItr->second);
491         RegisteredInitSymbols.erase(RISItr);
492       }
493     }
494   });
495 
496   // If there are no further init symbols to look up then send the link order
497   // (as a list of header addresses) to the caller.
498   if (NewInitSymbols.empty()) {
499 
500     // To make the list intelligible to the runtime we need to convert all
501     // JITDylib pointers to their header addresses. Only include JITDylibs
502     // that appear in the JITDylibToHandleAddr map (i.e. those that have been
503     // through setupJITDylib) -- bare JITDylibs aren't managed by the platform.
504     DenseMap<JITDylib *, ExecutorAddr> HeaderAddrs;
505     HeaderAddrs.reserve(JDDepMap.size());
506     {
507       std::lock_guard<std::mutex> Lock(PlatformMutex);
508       for (auto &KV : JDDepMap) {
509         auto I = JITDylibToHandleAddr.find(KV.first);
510         if (I != JITDylibToHandleAddr.end())
511           HeaderAddrs[KV.first] = I->second;
512       }
513     }
514 
515     // Build the dep info map to return.
516     ELFNixJITDylibDepInfoMap DIM;
517     DIM.reserve(JDDepMap.size());
518     for (auto &KV : JDDepMap) {
519       auto HI = HeaderAddrs.find(KV.first);
520       // Skip unmanaged JITDylibs.
521       if (HI == HeaderAddrs.end())
522         continue;
523       auto H = HI->second;
524       ELFNixJITDylibDepInfo DepInfo;
525       for (auto &Dep : KV.second) {
526         auto HJ = HeaderAddrs.find(Dep);
527         if (HJ != HeaderAddrs.end())
528           DepInfo.push_back(HJ->second);
529       }
530       DIM.push_back(std::make_pair(H, std::move(DepInfo)));
531     }
532     SendResult(DIM);
533     return;
534   }
535 
536   // Otherwise issue a lookup and re-run this phase when it completes.
537   lookupInitSymbolsAsync(
538       [this, SendResult = std::move(SendResult), JD](Error Err) mutable {
539         if (Err)
540           SendResult(std::move(Err));
541         else
542           pushInitializersLoop(std::move(SendResult), JD);
543       },
544       ES, std::move(NewInitSymbols));
545 }
546 
547 void ELFNixPlatform::rt_recordInitializers(
548     PushInitializersSendResultFn SendResult, ExecutorAddr JDHeaderAddr) {
549   JITDylibSP JD;
550   {
551     std::lock_guard<std::mutex> Lock(PlatformMutex);
552     auto I = HandleAddrToJITDylib.find(JDHeaderAddr);
553     if (I != HandleAddrToJITDylib.end())
554       JD = I->second;
555   }
556 
557   LLVM_DEBUG({
558     dbgs() << "ELFNixPlatform::rt_recordInitializers(" << JDHeaderAddr << ") ";
559     if (JD)
560       dbgs() << "pushing initializers for " << JD->getName() << "\n";
561     else
562       dbgs() << "No JITDylib for header address.\n";
563   });
564 
565   if (!JD) {
566     SendResult(make_error<StringError>("No JITDylib with header addr " +
567                                            formatv("{0:x}", JDHeaderAddr),
568                                        inconvertibleErrorCode()));
569     return;
570   }
571 
572   pushInitializersLoop(std::move(SendResult), JD);
573 }
574 
575 void ELFNixPlatform::rt_lookupSymbol(SendSymbolAddressFn SendResult,
576                                      ExecutorAddr Handle,
577                                      StringRef SymbolName) {
578   LLVM_DEBUG({
579     dbgs() << "ELFNixPlatform::rt_lookupSymbol(\"" << Handle << "\")\n";
580   });
581 
582   JITDylib *JD = nullptr;
583 
584   {
585     std::lock_guard<std::mutex> Lock(PlatformMutex);
586     auto I = HandleAddrToJITDylib.find(Handle);
587     if (I != HandleAddrToJITDylib.end())
588       JD = I->second;
589   }
590 
591   if (!JD) {
592     LLVM_DEBUG(dbgs() << "  No JITDylib for handle " << Handle << "\n");
593     SendResult(make_error<StringError>("No JITDylib associated with handle " +
594                                            formatv("{0:x}", Handle),
595                                        inconvertibleErrorCode()));
596     return;
597   }
598 
599   // Use functor class to work around XL build compiler issue on AIX.
600   class RtLookupNotifyComplete {
601   public:
602     RtLookupNotifyComplete(SendSymbolAddressFn &&SendResult)
603         : SendResult(std::move(SendResult)) {}
604     void operator()(Expected<SymbolMap> Result) {
605       if (Result) {
606         assert(Result->size() == 1 && "Unexpected result map count");
607         SendResult(Result->begin()->second.getAddress());
608       } else {
609         SendResult(Result.takeError());
610       }
611     }
612 
613   private:
614     SendSymbolAddressFn SendResult;
615   };
616 
617   ES.lookup(
618       LookupKind::DLSym, {{JD, JITDylibLookupFlags::MatchExportedSymbolsOnly}},
619       SymbolLookupSet(ES.intern(SymbolName)), SymbolState::Ready,
620       RtLookupNotifyComplete(std::move(SendResult)), NoDependenciesToRegister);
621 }
622 
623 Error ELFNixPlatform::ELFNixPlatformPlugin::bootstrapPipelineStart(
624     jitlink::LinkGraph &G) {
625   // Increment the active graphs count in BootstrapInfo.
626   std::lock_guard<std::mutex> Lock(MP.Bootstrap.load()->Mutex);
627   ++MP.Bootstrap.load()->ActiveGraphs;
628   return Error::success();
629 }
630 
631 Error ELFNixPlatform::ELFNixPlatformPlugin::
632     bootstrapPipelineRecordRuntimeFunctions(jitlink::LinkGraph &G) {
633   // Record bootstrap function names.
634   std::pair<StringRef, ExecutorAddr *> RuntimeSymbols[] = {
635       {*MP.DSOHandleSymbol, &MP.Bootstrap.load()->ELFNixHeaderAddr},
636       {*MP.PlatformBootstrap.Name, &MP.PlatformBootstrap.Addr},
637       {*MP.PlatformShutdown.Name, &MP.PlatformShutdown.Addr},
638       {*MP.RegisterJITDylib.Name, &MP.RegisterJITDylib.Addr},
639       {*MP.DeregisterJITDylib.Name, &MP.DeregisterJITDylib.Addr},
640       {*MP.RegisterObjectSections.Name, &MP.RegisterObjectSections.Addr},
641       {*MP.DeregisterObjectSections.Name, &MP.DeregisterObjectSections.Addr},
642       {*MP.RegisterInitSections.Name, &MP.RegisterInitSections.Addr},
643       {*MP.DeregisterInitSections.Name, &MP.DeregisterInitSections.Addr},
644       {*MP.CreatePThreadKey.Name, &MP.CreatePThreadKey.Addr}};
645 
646   bool RegisterELFNixHeader = false;
647 
648   for (auto *Sym : G.defined_symbols()) {
649     for (auto &RTSym : RuntimeSymbols) {
650       if (Sym->hasName() && *Sym->getName() == RTSym.first) {
651         if (*RTSym.second)
652           return make_error<StringError>(
653               "Duplicate " + RTSym.first +
654                   " detected during ELFNixPlatform bootstrap",
655               inconvertibleErrorCode());
656 
657         if (*Sym->getName() == *MP.DSOHandleSymbol)
658           RegisterELFNixHeader = true;
659 
660         *RTSym.second = Sym->getAddress();
661       }
662     }
663   }
664 
665   if (RegisterELFNixHeader) {
666     // If this graph defines the elfnix header symbol then create the internal
667     // mapping between it and PlatformJD.
668     std::lock_guard<std::mutex> Lock(MP.PlatformMutex);
669     MP.JITDylibToHandleAddr[&MP.PlatformJD] =
670         MP.Bootstrap.load()->ELFNixHeaderAddr;
671     MP.HandleAddrToJITDylib[MP.Bootstrap.load()->ELFNixHeaderAddr] =
672         &MP.PlatformJD;
673   }
674 
675   return Error::success();
676 }
677 
678 Error ELFNixPlatform::ELFNixPlatformPlugin::bootstrapPipelineEnd(
679     jitlink::LinkGraph &G) {
680   std::lock_guard<std::mutex> Lock(MP.Bootstrap.load()->Mutex);
681   assert(MP.Bootstrap && "DeferredAAs reset before bootstrap completed");
682   --MP.Bootstrap.load()->ActiveGraphs;
683   // Notify Bootstrap->CV while holding the mutex because the mutex is
684   // also keeping Bootstrap->CV alive.
685   if (MP.Bootstrap.load()->ActiveGraphs == 0)
686     MP.Bootstrap.load()->CV.notify_all();
687   return Error::success();
688 }
689 
690 Error ELFNixPlatform::registerPerObjectSections(
691     jitlink::LinkGraph &G, const ELFPerObjectSectionsToRegister &POSR,
692     bool IsBootstrapping) {
693   using SPSRegisterPerObjSectionsArgs =
694       SPSArgList<SPSELFPerObjectSectionsToRegister>;
695 
696   if (LLVM_UNLIKELY(IsBootstrapping)) {
697     Bootstrap.load()->addArgumentsToRTFnMap(
698         &RegisterObjectSections, &DeregisterObjectSections,
699         getArgDataBufferType<SPSRegisterPerObjSectionsArgs>(POSR),
700         getArgDataBufferType<SPSRegisterPerObjSectionsArgs>(POSR));
701     return Error::success();
702   }
703 
704   G.allocActions().push_back(
705       {cantFail(WrapperFunctionCall::Create<SPSRegisterPerObjSectionsArgs>(
706            RegisterObjectSections.Addr, POSR)),
707        cantFail(WrapperFunctionCall::Create<SPSRegisterPerObjSectionsArgs>(
708            DeregisterObjectSections.Addr, POSR))});
709 
710   return Error::success();
711 }
712 
713 Expected<uint64_t> ELFNixPlatform::createPThreadKey() {
714   if (!CreatePThreadKey.Addr)
715     return make_error<StringError>(
716         "Attempting to create pthread key in target, but runtime support has "
717         "not been loaded yet",
718         inconvertibleErrorCode());
719 
720   Expected<uint64_t> Result(0);
721   if (auto Err = ES.callSPSWrapper<SPSExpected<uint64_t>(void)>(
722           CreatePThreadKey.Addr, Result))
723     return std::move(Err);
724   return Result;
725 }
726 
727 void ELFNixPlatform::ELFNixPlatformPlugin::modifyPassConfig(
728     MaterializationResponsibility &MR, jitlink::LinkGraph &LG,
729     jitlink::PassConfiguration &Config) {
730   using namespace jitlink;
731 
732   bool InBootstrapPhase =
733       &MR.getTargetJITDylib() == &MP.PlatformJD && MP.Bootstrap;
734 
735   // If we're in the bootstrap phase then increment the active graphs.
736   if (InBootstrapPhase) {
737     Config.PrePrunePasses.push_back(
738         [this](LinkGraph &G) { return bootstrapPipelineStart(G); });
739     Config.PostAllocationPasses.push_back([this](LinkGraph &G) {
740       return bootstrapPipelineRecordRuntimeFunctions(G);
741     });
742   }
743 
744   // If the initializer symbol is the __dso_handle symbol then just add
745   // the DSO handle support passes.
746   if (auto InitSymbol = MR.getInitializerSymbol()) {
747     if (InitSymbol == MP.DSOHandleSymbol && !InBootstrapPhase) {
748       addDSOHandleSupportPasses(MR, Config);
749       // The DSOHandle materialization unit doesn't require any other
750       // support, so we can bail out early.
751       return;
752     }
753 
754     /// Preserve init sections.
755     Config.PrePrunePasses.push_back(
756         [this, &MR](jitlink::LinkGraph &G) -> Error {
757           if (auto Err = preserveInitSections(G, MR))
758             return Err;
759           return Error::success();
760         });
761   }
762 
763   // Add passes for eh-frame and TLV support.
764   addEHAndTLVSupportPasses(MR, Config, InBootstrapPhase);
765 
766   // If the object contains initializers then add passes to record them.
767   Config.PostFixupPasses.push_back([this, &JD = MR.getTargetJITDylib(),
768                                     InBootstrapPhase](jitlink::LinkGraph &G) {
769     return registerInitSections(G, JD, InBootstrapPhase);
770   });
771 
772   // If we're in the bootstrap phase then steal allocation actions and then
773   // decrement the active graphs.
774   if (InBootstrapPhase)
775     Config.PostFixupPasses.push_back(
776         [this](LinkGraph &G) { return bootstrapPipelineEnd(G); });
777 }
778 
779 void ELFNixPlatform::ELFNixPlatformPlugin::addDSOHandleSupportPasses(
780     MaterializationResponsibility &MR, jitlink::PassConfiguration &Config) {
781 
782   Config.PostAllocationPasses.push_back([this, &JD = MR.getTargetJITDylib()](
783                                             jitlink::LinkGraph &G) -> Error {
784     auto I = llvm::find_if(G.defined_symbols(), [this](jitlink::Symbol *Sym) {
785       return Sym->getName() == MP.DSOHandleSymbol;
786     });
787     assert(I != G.defined_symbols().end() && "Missing DSO handle symbol");
788     {
789       std::lock_guard<std::mutex> Lock(MP.PlatformMutex);
790       auto HandleAddr = (*I)->getAddress();
791       MP.HandleAddrToJITDylib[HandleAddr] = &JD;
792       MP.JITDylibToHandleAddr[&JD] = HandleAddr;
793 
794       G.allocActions().push_back(
795           {cantFail(WrapperFunctionCall::Create<
796                     SPSArgList<SPSString, SPSExecutorAddr>>(
797                MP.RegisterJITDylib.Addr, JD.getName(), HandleAddr)),
798            cantFail(WrapperFunctionCall::Create<SPSArgList<SPSExecutorAddr>>(
799                MP.DeregisterJITDylib.Addr, HandleAddr))});
800     }
801     return Error::success();
802   });
803 }
804 
805 void ELFNixPlatform::ELFNixPlatformPlugin::addEHAndTLVSupportPasses(
806     MaterializationResponsibility &MR, jitlink::PassConfiguration &Config,
807     bool IsBootstrapping) {
808 
809   // Insert TLV lowering at the start of the PostPrunePasses, since we want
810   // it to run before GOT/PLT lowering.
811 
812   // TODO: Check that before the fixTLVSectionsAndEdges pass, the GOT/PLT build
813   // pass has done. Because the TLS descriptor need to be allocate in GOT.
814   Config.PostPrunePasses.push_back(
815       [this, &JD = MR.getTargetJITDylib()](jitlink::LinkGraph &G) {
816         return fixTLVSectionsAndEdges(G, JD);
817       });
818 
819   // Add a pass to register the final addresses of the eh-frame and TLV sections
820   // with the runtime.
821   Config.PostFixupPasses.push_back([this, IsBootstrapping](
822                                        jitlink::LinkGraph &G) -> Error {
823     ELFPerObjectSectionsToRegister POSR;
824 
825     if (auto *EHFrameSection = G.findSectionByName(ELFEHFrameSectionName)) {
826       jitlink::SectionRange R(*EHFrameSection);
827       if (!R.empty())
828         POSR.EHFrameSection = R.getRange();
829     }
830 
831     // Get a pointer to the thread data section if there is one. It will be used
832     // below.
833     jitlink::Section *ThreadDataSection =
834         G.findSectionByName(ELFThreadDataSectionName);
835 
836     // Handle thread BSS section if there is one.
837     if (auto *ThreadBSSSection = G.findSectionByName(ELFThreadBSSSectionName)) {
838       // If there's already a thread data section in this graph then merge the
839       // thread BSS section content into it, otherwise just treat the thread
840       // BSS section as the thread data section.
841       if (ThreadDataSection)
842         G.mergeSections(*ThreadDataSection, *ThreadBSSSection);
843       else
844         ThreadDataSection = ThreadBSSSection;
845     }
846 
847     // Having merged thread BSS (if present) and thread data (if present),
848     // record the resulting section range.
849     if (ThreadDataSection) {
850       jitlink::SectionRange R(*ThreadDataSection);
851       if (!R.empty())
852         POSR.ThreadDataSection = R.getRange();
853     }
854 
855     if (POSR.EHFrameSection.Start || POSR.ThreadDataSection.Start) {
856       if (auto Err = MP.registerPerObjectSections(G, POSR, IsBootstrapping))
857         return Err;
858     }
859 
860     return Error::success();
861   });
862 }
863 
864 Error ELFNixPlatform::ELFNixPlatformPlugin::preserveInitSections(
865     jitlink::LinkGraph &G, MaterializationResponsibility &MR) {
866 
867   if (const auto &InitSymName = MR.getInitializerSymbol()) {
868 
869     jitlink::Symbol *InitSym = nullptr;
870 
871     for (auto &InitSection : G.sections()) {
872       // Skip non-init sections.
873       if (!isELFInitializerSection(InitSection.getName()) ||
874           InitSection.empty())
875         continue;
876 
877       // Create the init symbol if it has not been created already and attach it
878       // to the first block.
879       if (!InitSym) {
880         auto &B = **InitSection.blocks().begin();
881         InitSym = &G.addDefinedSymbol(
882             B, 0, *InitSymName, B.getSize(), jitlink::Linkage::Strong,
883             jitlink::Scope::SideEffectsOnly, false, true);
884       }
885 
886       // Add keep-alive edges to anonymous symbols in all other init blocks.
887       for (auto *B : InitSection.blocks()) {
888         if (B == &InitSym->getBlock())
889           continue;
890 
891         auto &S = G.addAnonymousSymbol(*B, 0, B->getSize(), false, true);
892         InitSym->getBlock().addEdge(jitlink::Edge::KeepAlive, 0, S, 0);
893       }
894     }
895   }
896 
897   return Error::success();
898 }
899 
900 Error ELFNixPlatform::ELFNixPlatformPlugin::registerInitSections(
901     jitlink::LinkGraph &G, JITDylib &JD, bool IsBootstrapping) {
902   SmallVector<ExecutorAddrRange> ELFNixPlatformSecs;
903   LLVM_DEBUG(dbgs() << "ELFNixPlatform::registerInitSections\n");
904 
905   SmallVector<jitlink::Section *> OrderedInitSections;
906   for (auto &Sec : G.sections())
907     if (isELFInitializerSection(Sec.getName()))
908       OrderedInitSections.push_back(&Sec);
909 
910   // FIXME: This handles priority order within the current graph, but we'll need
911   //        to include priority information in the initializer allocation
912   //        actions in order to respect the ordering across multiple graphs.
913   llvm::sort(OrderedInitSections, [](const jitlink::Section *LHS,
914                                      const jitlink::Section *RHS) {
915     if (LHS->getName().starts_with(".init_array")) {
916       if (RHS->getName().starts_with(".init_array")) {
917         StringRef LHSPrioStr(LHS->getName());
918         StringRef RHSPrioStr(RHS->getName());
919         uint64_t LHSPriority;
920         bool LHSHasPriority = LHSPrioStr.consume_front(".init_array.") &&
921                               !LHSPrioStr.getAsInteger(10, LHSPriority);
922         uint64_t RHSPriority;
923         bool RHSHasPriority = RHSPrioStr.consume_front(".init_array.") &&
924                               !RHSPrioStr.getAsInteger(10, RHSPriority);
925         if (LHSHasPriority)
926           return RHSHasPriority ? LHSPriority < RHSPriority : true;
927         else if (RHSHasPriority)
928           return false;
929         // If we get here we'll fall through to the
930         // LHS->getName() < RHS->getName() test below.
931       } else {
932         // .init_array[.N] comes before any non-.init_array[.N] section.
933         return true;
934       }
935     }
936     return LHS->getName() < RHS->getName();
937   });
938 
939   for (auto &Sec : OrderedInitSections)
940     ELFNixPlatformSecs.push_back(jitlink::SectionRange(*Sec).getRange());
941 
942   // Dump the scraped inits.
943   LLVM_DEBUG({
944     dbgs() << "ELFNixPlatform: Scraped " << G.getName() << " init sections:\n";
945     for (auto &Sec : G.sections()) {
946       jitlink::SectionRange R(Sec);
947       dbgs() << "  " << Sec.getName() << ": " << R.getRange() << "\n";
948     }
949   });
950 
951   ExecutorAddr HeaderAddr;
952   {
953     std::lock_guard<std::mutex> Lock(MP.PlatformMutex);
954     auto I = MP.JITDylibToHandleAddr.find(&JD);
955     assert(I != MP.JITDylibToHandleAddr.end() && "No header registered for JD");
956     assert(I->second && "Null header registered for JD");
957     HeaderAddr = I->second;
958   }
959 
960   using SPSRegisterInitSectionsArgs =
961       SPSArgList<SPSExecutorAddr, SPSSequence<SPSExecutorAddrRange>>;
962 
963   if (LLVM_UNLIKELY(IsBootstrapping)) {
964     MP.Bootstrap.load()->addArgumentsToRTFnMap(
965         &MP.RegisterInitSections, &MP.DeregisterInitSections,
966         getArgDataBufferType<SPSRegisterInitSectionsArgs>(HeaderAddr,
967                                                           ELFNixPlatformSecs),
968         getArgDataBufferType<SPSRegisterInitSectionsArgs>(HeaderAddr,
969                                                           ELFNixPlatformSecs));
970     return Error::success();
971   }
972 
973   G.allocActions().push_back(
974       {cantFail(WrapperFunctionCall::Create<SPSRegisterInitSectionsArgs>(
975            MP.RegisterInitSections.Addr, HeaderAddr, ELFNixPlatformSecs)),
976        cantFail(WrapperFunctionCall::Create<SPSRegisterInitSectionsArgs>(
977            MP.DeregisterInitSections.Addr, HeaderAddr, ELFNixPlatformSecs))});
978 
979   return Error::success();
980 }
981 
982 Error ELFNixPlatform::ELFNixPlatformPlugin::fixTLVSectionsAndEdges(
983     jitlink::LinkGraph &G, JITDylib &JD) {
984   auto TLSGetAddrSymbolName = G.intern("__tls_get_addr");
985   auto TLSDescResolveSymbolName = G.intern("__tlsdesc_resolver");
986   for (auto *Sym : G.external_symbols()) {
987     if (Sym->getName() == TLSGetAddrSymbolName) {
988       auto TLSGetAddr =
989           MP.getExecutionSession().intern("___orc_rt_elfnix_tls_get_addr");
990       Sym->setName(std::move(TLSGetAddr));
991     } else if (Sym->getName() == TLSDescResolveSymbolName) {
992       auto TLSGetAddr =
993           MP.getExecutionSession().intern("___orc_rt_elfnix_tlsdesc_resolver");
994       Sym->setName(std::move(TLSGetAddr));
995     }
996   }
997 
998   auto *TLSInfoEntrySection = G.findSectionByName("$__TLSINFO");
999 
1000   if (TLSInfoEntrySection) {
1001     std::optional<uint64_t> Key;
1002     {
1003       std::lock_guard<std::mutex> Lock(MP.PlatformMutex);
1004       auto I = MP.JITDylibToPThreadKey.find(&JD);
1005       if (I != MP.JITDylibToPThreadKey.end())
1006         Key = I->second;
1007     }
1008     if (!Key) {
1009       if (auto KeyOrErr = MP.createPThreadKey())
1010         Key = *KeyOrErr;
1011       else
1012         return KeyOrErr.takeError();
1013     }
1014 
1015     uint64_t PlatformKeyBits =
1016         support::endian::byte_swap(*Key, G.getEndianness());
1017 
1018     for (auto *B : TLSInfoEntrySection->blocks()) {
1019       // FIXME: The TLS descriptor byte length may different with different
1020       // ISA
1021       assert(B->getSize() == (G.getPointerSize() * 2) &&
1022              "TLS descriptor must be 2 words length");
1023       auto TLSInfoEntryContent = B->getMutableContent(G);
1024       memcpy(TLSInfoEntryContent.data(), &PlatformKeyBits, G.getPointerSize());
1025     }
1026   }
1027 
1028   return Error::success();
1029 }
1030 
1031 } // End namespace orc.
1032 } // End namespace llvm.
1033