xref: /llvm-project/compiler-rt/lib/orc/elfnix_platform.cpp (revision f363f9d61eaff7090a19d226ea8786b2987d4fcc) 
1 //===- elfnix_platform.cpp ------------------------------------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file contains code required to load the rest of the ELF-on-*IX runtime.
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "elfnix_platform.h"
14 #include "common.h"
15 #include "compiler.h"
16 #include "error.h"
17 #include "jit_dispatch.h"
18 #include "record_section_tracker.h"
19 #include "wrapper_function_utils.h"
20 
21 #include <algorithm>
22 #include <map>
23 #include <mutex>
24 #include <sstream>
25 #include <string_view>
26 #include <unordered_map>
27 #include <vector>
28 
29 using namespace orc_rt;
30 using namespace orc_rt::elfnix;
31 
32 // Declare function tags for functions in the JIT process.
33 ORC_RT_JIT_DISPATCH_TAG(__orc_rt_reoptimize_tag)
34 ORC_RT_JIT_DISPATCH_TAG(__orc_rt_elfnix_push_initializers_tag)
35 ORC_RT_JIT_DISPATCH_TAG(__orc_rt_elfnix_symbol_lookup_tag)
36 
37 // eh-frame registration functions, made available via aliases
38 // installed by the Platform
39 extern "C" void __register_frame(const void *);
40 extern "C" void __deregister_frame(const void *);
41 
42 extern "C" void
43 __unw_add_dynamic_eh_frame_section(const void *) ORC_RT_WEAK_IMPORT;
44 extern "C" void
45 __unw_remove_dynamic_eh_frame_section(const void *) ORC_RT_WEAK_IMPORT;
46 
47 namespace {
48 
49 struct TLSInfoEntry {
50   unsigned long Key = 0;
51   unsigned long DataAddress = 0;
52 };
53 
54 struct TLSDescriptor {
55   void (*Resolver)(void *);
56   TLSInfoEntry *InfoEntry;
57 };
58 
59 class ELFNixPlatformRuntimeState {
60 private:
61   struct AtExitEntry {
62     void (*Func)(void *);
63     void *Arg;
64   };
65 
66   using AtExitsVector = std::vector<AtExitEntry>;
67 
68   struct PerJITDylibState {
69     std::string Name;
70     void *Header = nullptr;
71     size_t RefCount = 0;
72     size_t LinkedAgainstRefCount = 0;
73     bool AllowReinitialization = false;
74     AtExitsVector AtExits;
75     std::vector<PerJITDylibState *> Deps;
76     RecordSectionsTracker<void (*)()> RecordedInits;
77 
78     bool referenced() const {
79       return LinkedAgainstRefCount != 0 || RefCount != 0;
80     }
81   };
82 
83 public:
84   static void initialize(void *DSOHandle);
85   static ELFNixPlatformRuntimeState &get();
86   static void destroy();
87 
88   ELFNixPlatformRuntimeState(void *DSOHandle);
89 
90   // Delete copy and move constructors.
91   ELFNixPlatformRuntimeState(const ELFNixPlatformRuntimeState &) = delete;
92   ELFNixPlatformRuntimeState &
93   operator=(const ELFNixPlatformRuntimeState &) = delete;
94   ELFNixPlatformRuntimeState(ELFNixPlatformRuntimeState &&) = delete;
95   ELFNixPlatformRuntimeState &operator=(ELFNixPlatformRuntimeState &&) = delete;
96 
97   Error registerObjectSections(ELFNixPerObjectSectionsToRegister POSR);
98   Error registerJITDylib(std::string &Name, void *Handle);
99   Error deregisterJITDylib(void *Handle);
100   Error registerInits(ExecutorAddr HeaderAddr,
101                       std::vector<ExecutorAddrRange> Inits);
102   Error deregisterInits(ExecutorAddr HeaderAddr,
103                         std::vector<ExecutorAddrRange> Inits);
104   Error deregisterObjectSections(ELFNixPerObjectSectionsToRegister POSR);
105 
106   const char *dlerror();
107   void *dlopen(std::string_view Name, int Mode);
108   int dlupdate(void *DSOHandle);
109   int dlclose(void *DSOHandle);
110   void *dlsym(void *DSOHandle, std::string_view Symbol);
111 
112   int registerAtExit(void (*F)(void *), void *Arg, void *DSOHandle);
113   void runAtExits(void *DSOHandle);
114   void runAtExits(std::unique_lock<std::recursive_mutex> &JDStateLock,
115                   PerJITDylibState &JDS);
116 
117   /// Returns the base address of the section containing ThreadData.
118   Expected<std::pair<const char *, size_t>>
119   getThreadDataSectionFor(const char *ThreadData);
120 
121   void *getPlatformJDDSOHandle() { return PlatformJDDSOHandle; }
122 
123 private:
124   PerJITDylibState *getJITDylibStateByHeaderAddr(void *DSOHandle);
125   PerJITDylibState *getJITDylibStateByName(std::string_view Path);
126 
127   Error registerThreadDataSection(span<const char> ThreadDataSection);
128 
129   Expected<ExecutorAddr> lookupSymbolInJITDylib(void *DSOHandle,
130                                                 std::string_view Symbol);
131 
132   Error runInits(std::unique_lock<std::recursive_mutex> &JDStatesLock,
133                  PerJITDylibState &JDS);
134   Expected<void *> dlopenImpl(std::string_view Path, int Mode);
135   Error dlopenFull(std::unique_lock<std::recursive_mutex> &JDStatesLock,
136                    PerJITDylibState &JDS);
137   Error dlopenInitialize(std::unique_lock<std::recursive_mutex> &JDStatesLock,
138                          PerJITDylibState &JDS,
139                          ELFNixJITDylibDepInfoMap &DepInfo);
140   Error dlupdateImpl(void *DSOHandle);
141   Error dlupdateFull(std::unique_lock<std::recursive_mutex> &JDStatesLock,
142                      PerJITDylibState &JDS);
143 
144   Error dlcloseImpl(void *DSOHandle);
145   Error dlcloseInitialize(std::unique_lock<std::recursive_mutex> &JDStatesLock,
146                           PerJITDylibState &JDS);
147 
148   static ELFNixPlatformRuntimeState *MOPS;
149 
150   void *PlatformJDDSOHandle;
151 
152   // Frame registration functions:
153   void (*registerEHFrameSection)(const void *) = nullptr;
154   void (*deregisterEHFrameSection)(const void *) = nullptr;
155 
156   // FIXME: Move to thread-state.
157   std::string DLFcnError;
158 
159   std::recursive_mutex JDStatesMutex;
160   std::unordered_map<void *, PerJITDylibState> JDStates;
161   std::unordered_map<std::string, void *> JDNameToHeader;
162 
163   std::mutex ThreadDataSectionsMutex;
164   std::map<const char *, size_t> ThreadDataSections;
165 };
166 
167 ELFNixPlatformRuntimeState *ELFNixPlatformRuntimeState::MOPS = nullptr;
168 
169 void ELFNixPlatformRuntimeState::initialize(void *DSOHandle) {
170   assert(!MOPS && "ELFNixPlatformRuntimeState should be null");
171   MOPS = new ELFNixPlatformRuntimeState(DSOHandle);
172 }
173 
174 ELFNixPlatformRuntimeState &ELFNixPlatformRuntimeState::get() {
175   assert(MOPS && "ELFNixPlatformRuntimeState not initialized");
176   return *MOPS;
177 }
178 
179 void ELFNixPlatformRuntimeState::destroy() {
180   assert(MOPS && "ELFNixPlatformRuntimeState not initialized");
181   delete MOPS;
182 }
183 
184 ELFNixPlatformRuntimeState::ELFNixPlatformRuntimeState(void *DSOHandle)
185     : PlatformJDDSOHandle(DSOHandle) {
186   if (__unw_add_dynamic_eh_frame_section &&
187       __unw_remove_dynamic_eh_frame_section) {
188     registerEHFrameSection = __unw_add_dynamic_eh_frame_section;
189     deregisterEHFrameSection = __unw_remove_dynamic_eh_frame_section;
190   } else {
191     registerEHFrameSection = __register_frame;
192     deregisterEHFrameSection = __deregister_frame;
193   }
194 }
195 
196 Error ELFNixPlatformRuntimeState::registerObjectSections(
197     ELFNixPerObjectSectionsToRegister POSR) {
198   if (POSR.EHFrameSection.Start)
199     registerEHFrameSection(POSR.EHFrameSection.Start.toPtr<const char *>());
200 
201   if (POSR.ThreadDataSection.Start) {
202     if (auto Err = registerThreadDataSection(
203             POSR.ThreadDataSection.toSpan<const char>()))
204       return Err;
205   }
206 
207   return Error::success();
208 }
209 
210 Error ELFNixPlatformRuntimeState::deregisterObjectSections(
211     ELFNixPerObjectSectionsToRegister POSR) {
212   if (POSR.EHFrameSection.Start)
213     deregisterEHFrameSection(POSR.EHFrameSection.Start.toPtr<const char *>());
214 
215   return Error::success();
216 }
217 
218 Error ELFNixPlatformRuntimeState::registerJITDylib(std::string &Name,
219                                                    void *Handle) {
220   std::lock_guard<std::recursive_mutex> Lock(JDStatesMutex);
221 
222   if (JDStates.count(Handle)) {
223     std::ostringstream ErrStream;
224     ErrStream << "Duplicate JITDylib registration for header " << Handle
225               << " (name = " << Name << ")";
226     return make_error<StringError>(ErrStream.str());
227   }
228 
229   if (JDNameToHeader.count(Name)) {
230     std::ostringstream ErrStream;
231     ErrStream << "Duplicate JITDylib registration for header " << Handle
232               << " (header = " << Handle << ")";
233     return make_error<StringError>(ErrStream.str());
234   }
235 
236   auto &JD = JDStates[Handle];
237   JD.Header = Handle;
238   JD.Name = std::move(Name);
239   JDNameToHeader[JD.Name] = Handle;
240   return Error::success();
241 }
242 
243 Error ELFNixPlatformRuntimeState::deregisterJITDylib(void *Handle) {
244   std::lock_guard<std::recursive_mutex> Lock(JDStatesMutex);
245 
246   auto I = JDStates.find(Handle);
247   if (I == JDStates.end()) {
248     std::ostringstream ErrStream;
249     ErrStream << "Attempted to deregister unrecognized header " << Handle;
250     return make_error<StringError>(ErrStream.str());
251   }
252 
253   auto J = JDNameToHeader.find(
254       std::string(I->second.Name.data(), I->second.Name.size()));
255   assert(J != JDNameToHeader.end() &&
256          "Missing JDNameToHeader entry for JITDylib");
257   JDNameToHeader.erase(J);
258   JDStates.erase(I);
259   return Error::success();
260 }
261 
262 Error ELFNixPlatformRuntimeState::registerInits(
263     ExecutorAddr HeaderAddr, std::vector<ExecutorAddrRange> Inits) {
264   std::lock_guard<std::recursive_mutex> Lock(JDStatesMutex);
265   PerJITDylibState *JDS =
266       getJITDylibStateByHeaderAddr(HeaderAddr.toPtr<void *>());
267 
268   if (!JDS) {
269     std::ostringstream ErrStream;
270     ErrStream << "Could not register object platform sections for "
271                  "unrecognized header "
272               << HeaderAddr.toPtr<void *>();
273     return make_error<StringError>(ErrStream.str());
274   }
275 
276   for (auto &I : Inits) {
277     JDS->RecordedInits.add(I.toSpan<void (*)()>());
278   }
279 
280   return Error::success();
281 }
282 
283 Error ELFNixPlatformRuntimeState::deregisterInits(
284     ExecutorAddr HeaderAddr, std::vector<ExecutorAddrRange> Inits) {
285 
286   std::lock_guard<std::recursive_mutex> Lock(JDStatesMutex);
287   PerJITDylibState *JDS =
288       getJITDylibStateByHeaderAddr(HeaderAddr.toPtr<void *>());
289 
290   if (!JDS) {
291     std::ostringstream ErrStream;
292     ErrStream << "Could not register object platform sections for unrecognized "
293                  "header "
294               << HeaderAddr.toPtr<void *>();
295     return make_error<StringError>(ErrStream.str());
296   }
297 
298   for (auto &I : Inits) {
299     JDS->RecordedInits.removeIfPresent(I);
300   }
301 
302   return Error::success();
303 }
304 
305 const char *ELFNixPlatformRuntimeState::dlerror() { return DLFcnError.c_str(); }
306 
307 void *ELFNixPlatformRuntimeState::dlopen(std::string_view Path, int Mode) {
308   if (auto H = dlopenImpl(Path, Mode))
309     return *H;
310   else {
311     // FIXME: Make dlerror thread safe.
312     DLFcnError = toString(H.takeError());
313     return nullptr;
314   }
315 }
316 
317 int ELFNixPlatformRuntimeState::dlupdate(void *DSOHandle) {
318   if (auto Err = dlupdateImpl(DSOHandle)) {
319     // FIXME: Make dlerror thread safe.
320     DLFcnError = toString(std::move(Err));
321     return -1;
322   }
323   return 0;
324 }
325 
326 int ELFNixPlatformRuntimeState::dlclose(void *DSOHandle) {
327   if (auto Err = dlcloseImpl(DSOHandle)) {
328     DLFcnError = toString(std::move(Err));
329     return -1;
330   }
331   return 0;
332 }
333 
334 void *ELFNixPlatformRuntimeState::dlsym(void *DSOHandle,
335                                         std::string_view Symbol) {
336   auto Addr = lookupSymbolInJITDylib(DSOHandle, Symbol);
337   if (!Addr) {
338     DLFcnError = toString(Addr.takeError());
339     return 0;
340   }
341 
342   return Addr->toPtr<void *>();
343 }
344 
345 int ELFNixPlatformRuntimeState::registerAtExit(void (*F)(void *), void *Arg,
346                                                void *DSOHandle) {
347   // FIXME: Handle out-of-memory errors, returning -1 if OOM.
348   std::lock_guard<std::recursive_mutex> Lock(JDStatesMutex);
349   auto *JDS = getJITDylibStateByHeaderAddr(DSOHandle);
350   assert(JDS && "JITDylib state not initialized");
351   JDS->AtExits.push_back({F, Arg});
352   return 0;
353 }
354 
355 void ELFNixPlatformRuntimeState::runAtExits(void *DSOHandle) {
356   std::unique_lock<std::recursive_mutex> Lock(JDStatesMutex);
357   PerJITDylibState *JDS = getJITDylibStateByHeaderAddr(DSOHandle);
358 
359   if (JDS)
360     runAtExits(Lock, *JDS);
361 }
362 
363 void ELFNixPlatformRuntimeState::runAtExits(
364     std::unique_lock<std::recursive_mutex> &JDStateLock,
365     PerJITDylibState &JDS) {
366   AtExitsVector V = std::move(JDS.AtExits);
367 
368   while (!V.empty()) {
369     auto &AE = V.back();
370     AE.Func(AE.Arg);
371     V.pop_back();
372   }
373 }
374 
375 Expected<std::pair<const char *, size_t>>
376 ELFNixPlatformRuntimeState::getThreadDataSectionFor(const char *ThreadData) {
377   std::lock_guard<std::mutex> Lock(ThreadDataSectionsMutex);
378   auto I = ThreadDataSections.upper_bound(ThreadData);
379   // Check that we have a valid entry conovering this address.
380   if (I == ThreadDataSections.begin())
381     return make_error<StringError>("No thread local data section for key");
382   I = std::prev(I);
383   if (ThreadData >= I->first + I->second)
384     return make_error<StringError>("No thread local data section for key");
385   return *I;
386 }
387 
388 ELFNixPlatformRuntimeState::PerJITDylibState *
389 ELFNixPlatformRuntimeState::getJITDylibStateByHeaderAddr(void *DSOHandle) {
390   auto I = JDStates.find(DSOHandle);
391   if (I == JDStates.end())
392     return nullptr;
393 
394   return &I->second;
395 }
396 
397 ELFNixPlatformRuntimeState::PerJITDylibState *
398 ELFNixPlatformRuntimeState::getJITDylibStateByName(std::string_view Name) {
399   // FIXME: Avoid creating string copy here.
400   auto I = JDNameToHeader.find(std::string(Name.data(), Name.size()));
401   if (I == JDNameToHeader.end())
402     return nullptr;
403   void *H = I->second;
404   auto J = JDStates.find(H);
405   assert(J != JDStates.end() &&
406          "JITDylib has name map entry but no header map entry");
407   return &J->second;
408 }
409 
410 Error ELFNixPlatformRuntimeState::registerThreadDataSection(
411     span<const char> ThreadDataSection) {
412   std::lock_guard<std::mutex> Lock(ThreadDataSectionsMutex);
413   auto I = ThreadDataSections.upper_bound(ThreadDataSection.data());
414   if (I != ThreadDataSections.begin()) {
415     auto J = std::prev(I);
416     if (J->first + J->second > ThreadDataSection.data())
417       return make_error<StringError>("Overlapping .tdata sections");
418   }
419   ThreadDataSections.insert(
420       I, std::make_pair(ThreadDataSection.data(), ThreadDataSection.size()));
421   return Error::success();
422 }
423 
424 Expected<ExecutorAddr>
425 ELFNixPlatformRuntimeState::lookupSymbolInJITDylib(void *DSOHandle,
426                                                    std::string_view Sym) {
427   Expected<ExecutorAddr> Result((ExecutorAddr()));
428   if (auto Err = WrapperFunction<SPSExpected<SPSExecutorAddr>(
429           SPSExecutorAddr,
430           SPSString)>::call(JITDispatch(&__orc_rt_elfnix_symbol_lookup_tag),
431                             Result, ExecutorAddr::fromPtr(DSOHandle), Sym))
432     return std::move(Err);
433   return Result;
434 }
435 
436 Error ELFNixPlatformRuntimeState::runInits(
437     std::unique_lock<std::recursive_mutex> &JDStatesLock,
438     PerJITDylibState &JDS) {
439   std::vector<span<void (*)()>> InitSections;
440   InitSections.reserve(JDS.RecordedInits.numNewSections());
441 
442   JDS.RecordedInits.processNewSections(
443       [&](span<void (*)()> Inits) { InitSections.push_back(Inits); });
444 
445   JDStatesLock.unlock();
446   for (auto Sec : InitSections)
447     for (auto *Init : Sec)
448       Init();
449 
450   JDStatesLock.lock();
451 
452   return Error::success();
453 }
454 
455 Expected<void *> ELFNixPlatformRuntimeState::dlopenImpl(std::string_view Path,
456                                                         int Mode) {
457   std::unique_lock<std::recursive_mutex> Lock(JDStatesMutex);
458   PerJITDylibState *JDS = getJITDylibStateByName(Path);
459 
460   if (!JDS)
461     return make_error<StringError>("No registered JTIDylib for path " +
462                                    std::string(Path.data(), Path.size()));
463 
464   if (auto Err = dlopenFull(Lock, *JDS))
465     return std::move(Err);
466 
467   ++JDS->RefCount;
468 
469   return JDS->Header;
470 }
471 
472 Error ELFNixPlatformRuntimeState::dlopenFull(
473     std::unique_lock<std::recursive_mutex> &JDStateLock,
474     PerJITDylibState &JDS) {
475   Expected<ELFNixJITDylibDepInfoMap> DepInfo((ELFNixJITDylibDepInfoMap()));
476   JDStateLock.unlock();
477   if (auto Err = WrapperFunction<SPSExpected<SPSELFNixJITDylibDepInfoMap>(
478           SPSExecutorAddr)>::
479           call(JITDispatch(&__orc_rt_elfnix_push_initializers_tag), DepInfo,
480                ExecutorAddr::fromPtr(JDS.Header)))
481     return Err;
482   JDStateLock.lock();
483 
484   if (!DepInfo)
485     return DepInfo.takeError();
486 
487   if (auto Err = dlopenInitialize(JDStateLock, JDS, *DepInfo))
488     return Err;
489 
490   if (!DepInfo->empty()) {
491     std::ostringstream ErrStream;
492     ErrStream << "Encountered unrecognized dep-info key headers "
493                  "while processing dlopen of "
494               << JDS.Name;
495     return make_error<StringError>(ErrStream.str());
496   }
497 
498   return Error::success();
499 }
500 
501 Error ELFNixPlatformRuntimeState::dlopenInitialize(
502     std::unique_lock<std::recursive_mutex> &JDStatesLock, PerJITDylibState &JDS,
503     ELFNixJITDylibDepInfoMap &DepInfo) {
504 
505   auto I = DepInfo.find(ExecutorAddr::fromPtr(JDS.Header));
506   if (I == DepInfo.end())
507     return Error::success();
508 
509   auto Deps = std::move(I->second);
510   DepInfo.erase(I);
511 
512   std::vector<PerJITDylibState *> OldDeps;
513   std::swap(JDS.Deps, OldDeps);
514   JDS.Deps.reserve(Deps.size());
515   for (auto H : Deps) {
516     PerJITDylibState *DepJDS = getJITDylibStateByHeaderAddr(H.toPtr<void *>());
517     if (!DepJDS) {
518       std::ostringstream ErrStream;
519       ErrStream << "Encountered unrecognized dep header " << H.toPtr<void *>()
520                 << " while initializing " << JDS.Name;
521       return make_error<StringError>(ErrStream.str());
522     }
523     ++DepJDS->LinkedAgainstRefCount;
524     if (auto Err = dlopenInitialize(JDStatesLock, *DepJDS, DepInfo))
525       return Err;
526   }
527 
528   if (auto Err = runInits(JDStatesLock, JDS))
529     return Err;
530 
531   for (auto *DepJDS : OldDeps) {
532     --DepJDS->LinkedAgainstRefCount;
533     if (!DepJDS->referenced())
534       if (auto Err = dlcloseInitialize(JDStatesLock, *DepJDS))
535         return Err;
536   }
537   return Error::success();
538 }
539 
540 Error ELFNixPlatformRuntimeState::dlupdateImpl(void *DSOHandle) {
541   std::unique_lock<std::recursive_mutex> Lock(JDStatesMutex);
542 
543   // Try to find JITDylib state by name.
544   auto *JDS = getJITDylibStateByHeaderAddr(DSOHandle);
545 
546   if (!JDS) {
547     std::ostringstream ErrStream;
548     ErrStream << "No registered JITDylib for " << DSOHandle;
549     return make_error<StringError>(ErrStream.str());
550   }
551 
552   if (!JDS->referenced())
553     return make_error<StringError>("dlupdate failed, JITDylib must be open.");
554 
555   if (auto Err = dlupdateFull(Lock, *JDS))
556     return Err;
557 
558   return Error::success();
559 }
560 
561 Error ELFNixPlatformRuntimeState::dlupdateFull(
562     std::unique_lock<std::recursive_mutex> &JDStatesLock,
563     PerJITDylibState &JDS) {
564   // Call back to the JIT to push the initializers.
565   Expected<ELFNixJITDylibDepInfoMap> DepInfo((ELFNixJITDylibDepInfoMap()));
566   // Unlock so that we can accept the initializer update.
567   JDStatesLock.unlock();
568   if (auto Err = WrapperFunction<SPSExpected<SPSELFNixJITDylibDepInfoMap>(
569           SPSExecutorAddr)>::
570           call(JITDispatch(&__orc_rt_elfnix_push_initializers_tag), DepInfo,
571                ExecutorAddr::fromPtr(JDS.Header)))
572     return Err;
573   JDStatesLock.lock();
574 
575   if (!DepInfo)
576     return DepInfo.takeError();
577 
578   if (auto Err = runInits(JDStatesLock, JDS))
579     return Err;
580 
581   return Error::success();
582 }
583 
584 Error ELFNixPlatformRuntimeState::dlcloseImpl(void *DSOHandle) {
585 
586   std::unique_lock<std::recursive_mutex> Lock(JDStatesMutex);
587   PerJITDylibState *JDS = getJITDylibStateByHeaderAddr(DSOHandle);
588 
589   if (!JDS) {
590     std::ostringstream ErrStream;
591     ErrStream << "No registered JITDylib for " << DSOHandle;
592     return make_error<StringError>(ErrStream.str());
593   }
594 
595   --JDS->RefCount;
596 
597   if (!JDS->referenced())
598     return dlcloseInitialize(Lock, *JDS);
599 
600   return Error::success();
601 }
602 
603 Error ELFNixPlatformRuntimeState::dlcloseInitialize(
604     std::unique_lock<std::recursive_mutex> &JDStatesLock,
605     PerJITDylibState &JDS) {
606   runAtExits(JDStatesLock, JDS);
607   JDS.RecordedInits.reset();
608   for (auto *DepJDS : JDS.Deps)
609     if (!JDS.referenced())
610       if (auto Err = dlcloseInitialize(JDStatesLock, *DepJDS))
611         return Err;
612 
613   return Error::success();
614 }
615 
616 class ELFNixPlatformRuntimeTLVManager {
617 public:
618   void *getInstance(const char *ThreadData);
619 
620 private:
621   std::unordered_map<const char *, char *> Instances;
622   std::unordered_map<const char *, std::unique_ptr<char[]>> AllocatedSections;
623 };
624 
625 void *ELFNixPlatformRuntimeTLVManager::getInstance(const char *ThreadData) {
626   auto I = Instances.find(ThreadData);
627   if (I != Instances.end())
628     return I->second;
629   auto TDS =
630       ELFNixPlatformRuntimeState::get().getThreadDataSectionFor(ThreadData);
631   if (!TDS) {
632     __orc_rt_log_error(toString(TDS.takeError()).c_str());
633     return nullptr;
634   }
635 
636   auto &Allocated = AllocatedSections[TDS->first];
637   if (!Allocated) {
638     Allocated = std::make_unique<char[]>(TDS->second);
639     memcpy(Allocated.get(), TDS->first, TDS->second);
640   }
641   size_t ThreadDataDelta = ThreadData - TDS->first;
642   assert(ThreadDataDelta <= TDS->second && "ThreadData outside section bounds");
643 
644   char *Instance = Allocated.get() + ThreadDataDelta;
645   Instances[ThreadData] = Instance;
646   return Instance;
647 }
648 
649 void destroyELFNixTLVMgr(void *ELFNixTLVMgr) {
650   delete static_cast<ELFNixPlatformRuntimeTLVManager *>(ELFNixTLVMgr);
651 }
652 
653 } // end anonymous namespace
654 
655 //------------------------------------------------------------------------------
656 //                             JIT entry points
657 //------------------------------------------------------------------------------
658 
659 ORC_RT_INTERFACE orc_rt_CWrapperFunctionResult
660 __orc_rt_elfnix_platform_bootstrap(char *ArgData, size_t ArgSize) {
661   return WrapperFunction<SPSError(SPSExecutorAddr)>::handle(
662              ArgData, ArgSize,
663              [](ExecutorAddr DSOHandle) {
664                ELFNixPlatformRuntimeState::initialize(
665                    DSOHandle.toPtr<void *>());
666                return Error::success();
667              })
668       .release();
669 }
670 
671 ORC_RT_INTERFACE orc_rt_CWrapperFunctionResult
672 __orc_rt_elfnix_platform_shutdown(char *ArgData, size_t ArgSize) {
673   return WrapperFunction<SPSError()>::handle(
674              ArgData, ArgSize,
675              []() {
676                ELFNixPlatformRuntimeState::destroy();
677                return Error::success();
678              })
679       .release();
680 }
681 
682 ORC_RT_INTERFACE orc_rt_CWrapperFunctionResult
683 __orc_rt_elfnix_register_jitdylib(char *ArgData, size_t ArgSize) {
684   return WrapperFunction<SPSError(SPSString, SPSExecutorAddr)>::handle(
685              ArgData, ArgSize,
686              [](std::string &JDName, ExecutorAddr HeaderAddr) {
687                return ELFNixPlatformRuntimeState::get().registerJITDylib(
688                    JDName, HeaderAddr.toPtr<void *>());
689              })
690       .release();
691 }
692 
693 ORC_RT_INTERFACE orc_rt_CWrapperFunctionResult
694 __orc_rt_elfnix_deregister_jitdylib(char *ArgData, size_t ArgSize) {
695   return WrapperFunction<SPSError(SPSExecutorAddr)>::handle(
696              ArgData, ArgSize,
697              [](ExecutorAddr HeaderAddr) {
698                return ELFNixPlatformRuntimeState::get().deregisterJITDylib(
699                    HeaderAddr.toPtr<void *>());
700              })
701       .release();
702 }
703 
704 ORC_RT_INTERFACE orc_rt_CWrapperFunctionResult
705 __orc_rt_elfnix_register_init_sections(char *ArgData, size_t ArgSize) {
706   return WrapperFunction<SPSError(SPSExecutorAddr,
707                                   SPSSequence<SPSExecutorAddrRange>)>::
708       handle(ArgData, ArgSize,
709              [](ExecutorAddr HeaderAddr,
710                 std::vector<ExecutorAddrRange> &Inits) {
711                return ELFNixPlatformRuntimeState::get().registerInits(
712                    HeaderAddr, std::move(Inits));
713              })
714           .release();
715 }
716 
717 ORC_RT_INTERFACE orc_rt_CWrapperFunctionResult
718 __orc_rt_elfnix_deregister_init_sections(char *ArgData, size_t ArgSize) {
719   return WrapperFunction<SPSError(SPSExecutorAddr,
720                                   SPSSequence<SPSExecutorAddrRange>)>::
721       handle(ArgData, ArgSize,
722              [](ExecutorAddr HeaderAddr,
723                 std::vector<ExecutorAddrRange> &Inits) {
724                return ELFNixPlatformRuntimeState::get().deregisterInits(
725                    HeaderAddr, std::move(Inits));
726              })
727           .release();
728 }
729 
730 /// Wrapper function for registering metadata on a per-object basis.
731 ORC_RT_INTERFACE orc_rt_CWrapperFunctionResult
732 __orc_rt_elfnix_register_object_sections(char *ArgData, size_t ArgSize) {
733   return WrapperFunction<SPSError(SPSELFNixPerObjectSectionsToRegister)>::
734       handle(ArgData, ArgSize,
735              [](ELFNixPerObjectSectionsToRegister &POSR) {
736                return ELFNixPlatformRuntimeState::get().registerObjectSections(
737                    std::move(POSR));
738              })
739           .release();
740 }
741 
742 /// Wrapper for releasing per-object metadat.
743 ORC_RT_INTERFACE orc_rt_CWrapperFunctionResult
744 __orc_rt_elfnix_deregister_object_sections(char *ArgData, size_t ArgSize) {
745   return WrapperFunction<SPSError(SPSELFNixPerObjectSectionsToRegister)>::
746       handle(ArgData, ArgSize,
747              [](ELFNixPerObjectSectionsToRegister &POSR) {
748                return ELFNixPlatformRuntimeState::get()
749                    .deregisterObjectSections(std::move(POSR));
750              })
751           .release();
752 }
753 
754 //------------------------------------------------------------------------------
755 //                           TLV support
756 //------------------------------------------------------------------------------
757 
758 ORC_RT_INTERFACE void *__orc_rt_elfnix_tls_get_addr_impl(TLSInfoEntry *D) {
759   auto *TLVMgr = static_cast<ELFNixPlatformRuntimeTLVManager *>(
760       pthread_getspecific(D->Key));
761   if (!TLVMgr)
762     TLVMgr = new ELFNixPlatformRuntimeTLVManager();
763   if (pthread_setspecific(D->Key, TLVMgr)) {
764     __orc_rt_log_error("Call to pthread_setspecific failed");
765     return nullptr;
766   }
767 
768   return TLVMgr->getInstance(
769       reinterpret_cast<char *>(static_cast<uintptr_t>(D->DataAddress)));
770 }
771 
772 ORC_RT_INTERFACE ptrdiff_t ___orc_rt_elfnix_tlsdesc_resolver_impl(
773     TLSDescriptor *D, const char *ThreadPointer) {
774   const char *TLVPtr = reinterpret_cast<const char *>(
775       __orc_rt_elfnix_tls_get_addr_impl(D->InfoEntry));
776   return TLVPtr - ThreadPointer;
777 }
778 
779 ORC_RT_INTERFACE orc_rt_CWrapperFunctionResult
780 __orc_rt_elfnix_create_pthread_key(char *ArgData, size_t ArgSize) {
781   return WrapperFunction<SPSExpected<uint64_t>(void)>::handle(
782              ArgData, ArgSize,
783              []() -> Expected<uint64_t> {
784                pthread_key_t Key;
785                if (int Err = pthread_key_create(&Key, destroyELFNixTLVMgr)) {
786                  __orc_rt_log_error("Call to pthread_key_create failed");
787                  return make_error<StringError>(strerror(Err));
788                }
789                return static_cast<uint64_t>(Key);
790              })
791       .release();
792 }
793 
794 //------------------------------------------------------------------------------
795 //                           cxa_atexit support
796 //------------------------------------------------------------------------------
797 
798 int __orc_rt_elfnix_cxa_atexit(void (*func)(void *), void *arg,
799                                void *dso_handle) {
800   return ELFNixPlatformRuntimeState::get().registerAtExit(func, arg,
801                                                           dso_handle);
802 }
803 
804 int __orc_rt_elfnix_atexit(void (*func)(void *)) {
805   auto &PlatformRTState = ELFNixPlatformRuntimeState::get();
806   return ELFNixPlatformRuntimeState::get().registerAtExit(
807       func, NULL, PlatformRTState.getPlatformJDDSOHandle());
808 }
809 
810 void __orc_rt_elfnix_cxa_finalize(void *dso_handle) {
811   ELFNixPlatformRuntimeState::get().runAtExits(dso_handle);
812 }
813 
814 //------------------------------------------------------------------------------
815 //                        JIT'd dlfcn alternatives.
816 //------------------------------------------------------------------------------
817 
818 const char *__orc_rt_elfnix_jit_dlerror() {
819   return ELFNixPlatformRuntimeState::get().dlerror();
820 }
821 
822 void *__orc_rt_elfnix_jit_dlopen(const char *path, int mode) {
823   return ELFNixPlatformRuntimeState::get().dlopen(path, mode);
824 }
825 
826 int __orc_rt_elfnix_jit_dlupdate(void *dso_handle) {
827   return ELFNixPlatformRuntimeState::get().dlupdate(dso_handle);
828 }
829 
830 int __orc_rt_elfnix_jit_dlclose(void *dso_handle) {
831   return ELFNixPlatformRuntimeState::get().dlclose(dso_handle);
832 }
833 
834 void *__orc_rt_elfnix_jit_dlsym(void *dso_handle, const char *symbol) {
835   return ELFNixPlatformRuntimeState::get().dlsym(dso_handle, symbol);
836 }
837 
838 //------------------------------------------------------------------------------
839 //                             ELFNix Run Program
840 //------------------------------------------------------------------------------
841 
842 ORC_RT_INTERFACE int64_t __orc_rt_elfnix_run_program(
843     const char *JITDylibName, const char *EntrySymbolName, int argc,
844     char *argv[]) {
845   using MainTy = int (*)(int, char *[]);
846 
847   void *H = __orc_rt_elfnix_jit_dlopen(JITDylibName,
848                                        orc_rt::elfnix::ORC_RT_RTLD_LAZY);
849   if (!H) {
850     __orc_rt_log_error(__orc_rt_elfnix_jit_dlerror());
851     return -1;
852   }
853 
854   auto *Main =
855       reinterpret_cast<MainTy>(__orc_rt_elfnix_jit_dlsym(H, EntrySymbolName));
856 
857   if (!Main) {
858     __orc_rt_log_error(__orc_rt_elfnix_jit_dlerror());
859     return -1;
860   }
861 
862   int Result = Main(argc, argv);
863 
864   if (__orc_rt_elfnix_jit_dlclose(H) == -1)
865     __orc_rt_log_error(__orc_rt_elfnix_jit_dlerror());
866 
867   return Result;
868 }
869