1 //===- macho_platform.cpp -------------------------------------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file contains code required to load the rest of the MachO runtime.
10 //
11 //===----------------------------------------------------------------------===//
12
13 #include "macho_platform.h"
14 #include "bitmask_enum.h"
15 #include "common.h"
16 #include "debug.h"
17 #include "error.h"
18 #include "interval_map.h"
19 #include "wrapper_function_utils.h"
20
21 #include <algorithm>
22 #include <ios>
23 #include <map>
24 #include <mutex>
25 #include <sstream>
26 #include <string_view>
27 #include <unordered_map>
28 #include <unordered_set>
29 #include <vector>
30
31 #define DEBUG_TYPE "macho_platform"
32
33 using namespace __orc_rt;
34 using namespace __orc_rt::macho;
35
36 // Declare function tags for functions in the JIT process.
37 ORC_RT_JIT_DISPATCH_TAG(__orc_rt_macho_push_initializers_tag)
38 ORC_RT_JIT_DISPATCH_TAG(__orc_rt_macho_push_symbols_tag)
39
40 struct objc_image_info;
41 struct mach_header;
42
43 // Objective-C registration functions.
44 // These are weakly imported. If the Objective-C runtime has not been loaded
45 // then code containing Objective-C sections will generate an error.
46 extern "C" void
47 _objc_map_images(unsigned count, const char *const paths[],
48 const mach_header *const mhdrs[]) ORC_RT_WEAK_IMPORT;
49
50 extern "C" void _objc_load_image(const char *path,
51 const mach_header *mh) ORC_RT_WEAK_IMPORT;
52
53 // Libunwind prototypes.
54 struct unw_dynamic_unwind_sections {
55 uintptr_t dso_base;
56 uintptr_t dwarf_section;
57 size_t dwarf_section_length;
58 uintptr_t compact_unwind_section;
59 size_t compact_unwind_section_length;
60 };
61
62 typedef int (*unw_find_dynamic_unwind_sections)(
63 uintptr_t addr, struct unw_dynamic_unwind_sections *info);
64
65 extern "C" int __unw_add_find_dynamic_unwind_sections(
66 unw_find_dynamic_unwind_sections find_dynamic_unwind_sections)
67 ORC_RT_WEAK_IMPORT;
68
69 extern "C" int __unw_remove_find_dynamic_unwind_sections(
70 unw_find_dynamic_unwind_sections find_dynamic_unwind_sections)
71 ORC_RT_WEAK_IMPORT;
72
73 namespace {
74
75 struct MachOJITDylibDepInfo {
76 bool Sealed = false;
77 std::vector<ExecutorAddr> DepHeaders;
78 };
79
80 using MachOJITDylibDepInfoMap =
81 std::unordered_map<ExecutorAddr, MachOJITDylibDepInfo>;
82
83 } // anonymous namespace
84
85 namespace __orc_rt {
86
87 using SPSMachOObjectPlatformSectionsMap =
88 SPSSequence<SPSTuple<SPSString, SPSExecutorAddrRange>>;
89
90 using SPSMachOJITDylibDepInfo = SPSTuple<bool, SPSSequence<SPSExecutorAddr>>;
91
92 using SPSMachOJITDylibDepInfoMap =
93 SPSSequence<SPSTuple<SPSExecutorAddr, SPSMachOJITDylibDepInfo>>;
94
95 template <>
96 class SPSSerializationTraits<SPSMachOJITDylibDepInfo, MachOJITDylibDepInfo> {
97 public:
size(const MachOJITDylibDepInfo & JDI)98 static size_t size(const MachOJITDylibDepInfo &JDI) {
99 return SPSMachOJITDylibDepInfo::AsArgList::size(JDI.Sealed, JDI.DepHeaders);
100 }
101
serialize(SPSOutputBuffer & OB,const MachOJITDylibDepInfo & JDI)102 static bool serialize(SPSOutputBuffer &OB, const MachOJITDylibDepInfo &JDI) {
103 return SPSMachOJITDylibDepInfo::AsArgList::serialize(OB, JDI.Sealed,
104 JDI.DepHeaders);
105 }
106
deserialize(SPSInputBuffer & IB,MachOJITDylibDepInfo & JDI)107 static bool deserialize(SPSInputBuffer &IB, MachOJITDylibDepInfo &JDI) {
108 return SPSMachOJITDylibDepInfo::AsArgList::deserialize(IB, JDI.Sealed,
109 JDI.DepHeaders);
110 }
111 };
112
113 struct UnwindSectionInfo {
114 std::vector<ExecutorAddrRange> CodeRanges;
115 ExecutorAddrRange DwarfSection;
116 ExecutorAddrRange CompactUnwindSection;
117 };
118
119 using SPSUnwindSectionInfo =
120 SPSTuple<SPSSequence<SPSExecutorAddrRange>, SPSExecutorAddrRange,
121 SPSExecutorAddrRange>;
122
123 template <>
124 class SPSSerializationTraits<SPSUnwindSectionInfo, UnwindSectionInfo> {
125 public:
size(const UnwindSectionInfo & USI)126 static size_t size(const UnwindSectionInfo &USI) {
127 return SPSUnwindSectionInfo::AsArgList::size(
128 USI.CodeRanges, USI.DwarfSection, USI.CompactUnwindSection);
129 }
130
serialize(SPSOutputBuffer & OB,const UnwindSectionInfo & USI)131 static bool serialize(SPSOutputBuffer &OB, const UnwindSectionInfo &USI) {
132 return SPSUnwindSectionInfo::AsArgList::serialize(
133 OB, USI.CodeRanges, USI.DwarfSection, USI.CompactUnwindSection);
134 }
135
deserialize(SPSInputBuffer & IB,UnwindSectionInfo & USI)136 static bool deserialize(SPSInputBuffer &IB, UnwindSectionInfo &USI) {
137 return SPSUnwindSectionInfo::AsArgList::deserialize(
138 IB, USI.CodeRanges, USI.DwarfSection, USI.CompactUnwindSection);
139 }
140 };
141
142 } // namespace __orc_rt
143
144 namespace {
145 struct TLVDescriptor {
146 void *(*Thunk)(TLVDescriptor *) = nullptr;
147 unsigned long Key = 0;
148 unsigned long DataAddress = 0;
149 };
150
151 class MachOPlatformRuntimeState {
152 public:
153 // Used internally by MachOPlatformRuntimeState, but made public to enable
154 // serialization.
155 enum class MachOExecutorSymbolFlags : uint8_t {
156 None = 0,
157 Weak = 1U << 0,
158 Callable = 1U << 1,
159 ORC_RT_MARK_AS_BITMASK_ENUM(/* LargestValue = */ Callable)
160 };
161
162 private:
163 struct AtExitEntry {
164 void (*Func)(void *);
165 void *Arg;
166 };
167
168 using AtExitsVector = std::vector<AtExitEntry>;
169
170 /// Used to manage sections of fixed-sized metadata records (e.g. pointer
171 /// sections, selector refs, etc.)
172 template <typename RecordElement> class RecordSectionsTracker {
173 public:
174 /// Add a section to the "new" list.
add(span<RecordElement> Sec)175 void add(span<RecordElement> Sec) { New.push_back(std::move(Sec)); }
176
177 /// Returns true if there are new sections to process.
hasNewSections() const178 bool hasNewSections() const { return !New.empty(); }
179
180 /// Returns the number of new sections to process.
numNewSections() const181 size_t numNewSections() const { return New.size(); }
182
183 /// Process all new sections.
184 template <typename ProcessSectionFunc>
185 std::enable_if_t<std::is_void_v<
186 std::invoke_result_t<ProcessSectionFunc, span<RecordElement>>>>
processNewSections(ProcessSectionFunc && ProcessSection)187 processNewSections(ProcessSectionFunc &&ProcessSection) {
188 for (auto &Sec : New)
189 ProcessSection(Sec);
190 moveNewToProcessed();
191 }
192
193 /// Proces all new sections with a fallible handler.
194 ///
195 /// Successfully handled sections will be moved to the Processed
196 /// list.
197 template <typename ProcessSectionFunc>
198 std::enable_if_t<
199 std::is_same_v<Error, std::invoke_result_t<ProcessSectionFunc,
200 span<RecordElement>>>,
201 Error>
processNewSections(ProcessSectionFunc && ProcessSection)202 processNewSections(ProcessSectionFunc &&ProcessSection) {
203 for (size_t I = 0; I != New.size(); ++I) {
204 if (auto Err = ProcessSection(New[I])) {
205 for (size_t J = 0; J != I; ++J)
206 Processed.push_back(New[J]);
207 New.erase(New.begin(), New.begin() + I);
208 return Err;
209 }
210 }
211 moveNewToProcessed();
212 return Error::success();
213 }
214
215 /// Move all sections back to New for reprocessing.
reset()216 void reset() {
217 moveNewToProcessed();
218 New = std::move(Processed);
219 }
220
221 /// Remove the section with the given range.
removeIfPresent(ExecutorAddrRange R)222 bool removeIfPresent(ExecutorAddrRange R) {
223 if (removeIfPresent(New, R))
224 return true;
225 return removeIfPresent(Processed, R);
226 }
227
228 private:
moveNewToProcessed()229 void moveNewToProcessed() {
230 if (Processed.empty())
231 Processed = std::move(New);
232 else {
233 Processed.reserve(Processed.size() + New.size());
234 std::copy(New.begin(), New.end(), std::back_inserter(Processed));
235 New.clear();
236 }
237 }
238
removeIfPresent(std::vector<span<RecordElement>> & V,ExecutorAddrRange R)239 bool removeIfPresent(std::vector<span<RecordElement>> &V,
240 ExecutorAddrRange R) {
241 auto RI = std::find_if(
242 V.rbegin(), V.rend(),
243 [RS = R.toSpan<RecordElement>()](const span<RecordElement> &E) {
244 return E.data() == RS.data();
245 });
246 if (RI != V.rend()) {
247 V.erase(std::next(RI).base());
248 return true;
249 }
250 return false;
251 }
252
253 std::vector<span<RecordElement>> Processed;
254 std::vector<span<RecordElement>> New;
255 };
256
257 struct UnwindSections {
UnwindSections__anon068503140211::MachOPlatformRuntimeState::UnwindSections258 UnwindSections(const UnwindSectionInfo &USI)
259 : DwarfSection(USI.DwarfSection.toSpan<char>()),
260 CompactUnwindSection(USI.CompactUnwindSection.toSpan<char>()) {}
261
262 span<char> DwarfSection;
263 span<char> CompactUnwindSection;
264 };
265
266 using UnwindSectionsMap =
267 IntervalMap<char *, UnwindSections, IntervalCoalescing::Disabled>;
268
269 struct JITDylibState {
270
271 using SymbolTableMap =
272 std::unordered_map<std::string_view,
273 std::pair<ExecutorAddr, MachOExecutorSymbolFlags>>;
274
275 std::string Name;
276 void *Header = nullptr;
277 bool Sealed = false;
278 size_t LinkedAgainstRefCount = 0;
279 size_t DlRefCount = 0;
280 SymbolTableMap SymbolTable;
281 std::vector<JITDylibState *> Deps;
282 AtExitsVector AtExits;
283 const objc_image_info *ObjCImageInfo = nullptr;
284 std::unordered_map<void *, std::vector<char>> DataSectionContent;
285 std::unordered_map<void *, size_t> ZeroInitRanges;
286 UnwindSectionsMap UnwindSections;
287 RecordSectionsTracker<void (*)()> ModInitsSections;
288 RecordSectionsTracker<char> ObjCRuntimeRegistrationObjects;
289
referenced__anon068503140211::MachOPlatformRuntimeState::JITDylibState290 bool referenced() const {
291 return LinkedAgainstRefCount != 0 || DlRefCount != 0;
292 }
293 };
294
295 public:
296 static Error create();
297 static MachOPlatformRuntimeState &get();
298 static Error destroy();
299
300 MachOPlatformRuntimeState() = default;
301
302 // Delete copy and move constructors.
303 MachOPlatformRuntimeState(const MachOPlatformRuntimeState &) = delete;
304 MachOPlatformRuntimeState &
305 operator=(const MachOPlatformRuntimeState &) = delete;
306 MachOPlatformRuntimeState(MachOPlatformRuntimeState &&) = delete;
307 MachOPlatformRuntimeState &operator=(MachOPlatformRuntimeState &&) = delete;
308
309 Error initialize();
310 Error shutdown();
311
312 Error registerJITDylib(std::string Name, void *Header);
313 Error deregisterJITDylib(void *Header);
314 Error registerThreadDataSection(span<const char> ThreadDataSection);
315 Error deregisterThreadDataSection(span<const char> ThreadDataSection);
316 Error registerObjectSymbolTable(
317 ExecutorAddr HeaderAddr,
318 const std::vector<std::tuple<ExecutorAddr, ExecutorAddr,
319 MachOExecutorSymbolFlags>> &Entries);
320 Error deregisterObjectSymbolTable(
321 ExecutorAddr HeaderAddr,
322 const std::vector<std::tuple<ExecutorAddr, ExecutorAddr,
323 MachOExecutorSymbolFlags>> &Entries);
324 Error registerObjectPlatformSections(
325 ExecutorAddr HeaderAddr, std::optional<UnwindSectionInfo> UnwindSections,
326 std::vector<std::pair<std::string_view, ExecutorAddrRange>> Secs);
327 Error deregisterObjectPlatformSections(
328 ExecutorAddr HeaderAddr, std::optional<UnwindSectionInfo> UnwindSections,
329 std::vector<std::pair<std::string_view, ExecutorAddrRange>> Secs);
330
331 const char *dlerror();
332 void *dlopen(std::string_view Name, int Mode);
333 int dlclose(void *DSOHandle);
334 void *dlsym(void *DSOHandle, const char *Symbol);
335
336 int registerAtExit(void (*F)(void *), void *Arg, void *DSOHandle);
337 void runAtExits(std::unique_lock<std::mutex> &JDStatesLock,
338 JITDylibState &JDS);
339 void runAtExits(void *DSOHandle);
340
341 /// Returns the base address of the section containing ThreadData.
342 Expected<std::pair<const char *, size_t>>
343 getThreadDataSectionFor(const char *ThreadData);
344
345 private:
346 JITDylibState *getJITDylibStateByHeader(void *DSOHandle);
347 JITDylibState *getJITDylibStateByName(std::string_view Path);
348
349 /// Requests materialization of the given symbols. For each pair, the bool
350 /// element indicates whether the symbol is required (true) or weakly
351 /// referenced (false).
352 Error requestPushSymbols(JITDylibState &JDS,
353 span<std::pair<std::string_view, bool>> Symbols);
354
355 /// Attempts to look up the given symbols locally, requesting a push from the
356 /// remote if they're not found. Results are written to the Result span, which
357 /// must have the same size as the Symbols span.
358 Error
359 lookupSymbols(JITDylibState &JDS, std::unique_lock<std::mutex> &JDStatesLock,
360 span<std::pair<ExecutorAddr, MachOExecutorSymbolFlags>> Result,
361 span<std::pair<std::string_view, bool>> Symbols);
362
363 bool lookupUnwindSections(void *Addr, unw_dynamic_unwind_sections &Info);
364
365 static int findDynamicUnwindSections(uintptr_t addr,
366 unw_dynamic_unwind_sections *info);
367 static Error registerEHFrames(span<const char> EHFrameSection);
368 static Error deregisterEHFrames(span<const char> EHFrameSection);
369
370 static Error registerObjCRegistrationObjects(JITDylibState &JDS);
371 static Error runModInits(std::unique_lock<std::mutex> &JDStatesLock,
372 JITDylibState &JDS);
373
374 Expected<void *> dlopenImpl(std::string_view Path, int Mode);
375 Error dlopenFull(std::unique_lock<std::mutex> &JDStatesLock,
376 JITDylibState &JDS);
377 Error dlopenInitialize(std::unique_lock<std::mutex> &JDStatesLock,
378 JITDylibState &JDS, MachOJITDylibDepInfoMap &DepInfo);
379
380 Error dlcloseImpl(void *DSOHandle);
381 Error dlcloseDeinitialize(std::unique_lock<std::mutex> &JDStatesLock,
382 JITDylibState &JDS);
383
384 static MachOPlatformRuntimeState *MOPS;
385
386 bool UseCallbackStyleUnwindInfo = false;
387
388 // FIXME: Move to thread-state.
389 std::string DLFcnError;
390
391 // APIMutex guards against concurrent entry into key "dyld" API functions
392 // (e.g. dlopen, dlclose).
393 std::recursive_mutex DyldAPIMutex;
394
395 // JDStatesMutex guards the data structures that hold JITDylib state.
396 std::mutex JDStatesMutex;
397 std::unordered_map<void *, JITDylibState> JDStates;
398 std::unordered_map<std::string_view, void *> JDNameToHeader;
399
400 // ThreadDataSectionsMutex guards thread local data section state.
401 std::mutex ThreadDataSectionsMutex;
402 std::map<const char *, size_t> ThreadDataSections;
403 };
404
405 } // anonymous namespace
406
407 namespace __orc_rt {
408
409 class SPSMachOExecutorSymbolFlags;
410
411 template <>
412 class SPSSerializationTraits<
413 SPSMachOExecutorSymbolFlags,
414 MachOPlatformRuntimeState::MachOExecutorSymbolFlags> {
415 private:
416 using UT = std::underlying_type_t<
417 MachOPlatformRuntimeState::MachOExecutorSymbolFlags>;
418
419 public:
420 static size_t
size(const MachOPlatformRuntimeState::MachOExecutorSymbolFlags & SF)421 size(const MachOPlatformRuntimeState::MachOExecutorSymbolFlags &SF) {
422 return sizeof(UT);
423 }
424
425 static bool
serialize(SPSOutputBuffer & OB,const MachOPlatformRuntimeState::MachOExecutorSymbolFlags & SF)426 serialize(SPSOutputBuffer &OB,
427 const MachOPlatformRuntimeState::MachOExecutorSymbolFlags &SF) {
428 return SPSArgList<UT>::serialize(OB, static_cast<UT>(SF));
429 }
430
431 static bool
deserialize(SPSInputBuffer & IB,MachOPlatformRuntimeState::MachOExecutorSymbolFlags & SF)432 deserialize(SPSInputBuffer &IB,
433 MachOPlatformRuntimeState::MachOExecutorSymbolFlags &SF) {
434 UT Tmp;
435 if (!SPSArgList<UT>::deserialize(IB, Tmp))
436 return false;
437 SF = static_cast<MachOPlatformRuntimeState::MachOExecutorSymbolFlags>(Tmp);
438 return true;
439 }
440 };
441
442 } // namespace __orc_rt
443
444 namespace {
445
446 MachOPlatformRuntimeState *MachOPlatformRuntimeState::MOPS = nullptr;
447
create()448 Error MachOPlatformRuntimeState::create() {
449 assert(!MOPS && "MachOPlatformRuntimeState should be null");
450 MOPS = new MachOPlatformRuntimeState();
451 return MOPS->initialize();
452 }
453
get()454 MachOPlatformRuntimeState &MachOPlatformRuntimeState::get() {
455 assert(MOPS && "MachOPlatformRuntimeState not initialized");
456 return *MOPS;
457 }
458
destroy()459 Error MachOPlatformRuntimeState::destroy() {
460 assert(MOPS && "MachOPlatformRuntimeState not initialized");
461 auto Err = MOPS->shutdown();
462 delete MOPS;
463 return Err;
464 }
465
initialize()466 Error MachOPlatformRuntimeState::initialize() {
467 UseCallbackStyleUnwindInfo = __unw_add_find_dynamic_unwind_sections &&
468 __unw_remove_find_dynamic_unwind_sections;
469 if (UseCallbackStyleUnwindInfo) {
470 ORC_RT_DEBUG({
471 printdbg("__unw_add/remove_find_dynamic_unwind_sections available."
472 " Using callback-based frame info lookup.\n");
473 });
474 if (__unw_add_find_dynamic_unwind_sections(&findDynamicUnwindSections))
475 return make_error<StringError>(
476 "Could not register findDynamicUnwindSections");
477 } else {
478 ORC_RT_DEBUG({
479 printdbg("__unw_add/remove_find_dynamic_unwind_sections not available."
480 " Using classic frame info registration.\n");
481 });
482 }
483 return Error::success();
484 }
485
shutdown()486 Error MachOPlatformRuntimeState::shutdown() {
487 if (UseCallbackStyleUnwindInfo) {
488 if (__unw_remove_find_dynamic_unwind_sections(&findDynamicUnwindSections)) {
489 ORC_RT_DEBUG(
490 { printdbg("__unw_remove_find_dynamic_unwind_sections failed.\n"); });
491 }
492 }
493 return Error::success();
494 }
495
registerJITDylib(std::string Name,void * Header)496 Error MachOPlatformRuntimeState::registerJITDylib(std::string Name,
497 void *Header) {
498 ORC_RT_DEBUG({
499 printdbg("Registering JITDylib %s: Header = %p\n", Name.c_str(), Header);
500 });
501 std::lock_guard<std::mutex> Lock(JDStatesMutex);
502 if (JDStates.count(Header)) {
503 std::ostringstream ErrStream;
504 ErrStream << "Duplicate JITDylib registration for header " << Header
505 << " (name = " << Name << ")";
506 return make_error<StringError>(ErrStream.str());
507 }
508 if (JDNameToHeader.count(Name)) {
509 std::ostringstream ErrStream;
510 ErrStream << "Duplicate JITDylib registration for header " << Header
511 << " (header = " << Header << ")";
512 return make_error<StringError>(ErrStream.str());
513 }
514
515 auto &JDS = JDStates[Header];
516 JDS.Name = std::move(Name);
517 JDS.Header = Header;
518 JDNameToHeader[JDS.Name] = Header;
519 return Error::success();
520 }
521
deregisterJITDylib(void * Header)522 Error MachOPlatformRuntimeState::deregisterJITDylib(void *Header) {
523 std::lock_guard<std::mutex> Lock(JDStatesMutex);
524 auto I = JDStates.find(Header);
525 if (I == JDStates.end()) {
526 std::ostringstream ErrStream;
527 ErrStream << "Attempted to deregister unrecognized header " << Header;
528 return make_error<StringError>(ErrStream.str());
529 }
530
531 // Remove std::string construction once we can use C++20.
532 auto J = JDNameToHeader.find(
533 std::string(I->second.Name.data(), I->second.Name.size()));
534 assert(J != JDNameToHeader.end() &&
535 "Missing JDNameToHeader entry for JITDylib");
536
537 ORC_RT_DEBUG({
538 printdbg("Deregistering JITDylib %s: Header = %p\n", I->second.Name.c_str(),
539 Header);
540 });
541
542 JDNameToHeader.erase(J);
543 JDStates.erase(I);
544 return Error::success();
545 }
546
registerThreadDataSection(span<const char> ThreadDataSection)547 Error MachOPlatformRuntimeState::registerThreadDataSection(
548 span<const char> ThreadDataSection) {
549 std::lock_guard<std::mutex> Lock(ThreadDataSectionsMutex);
550 auto I = ThreadDataSections.upper_bound(ThreadDataSection.data());
551 if (I != ThreadDataSections.begin()) {
552 auto J = std::prev(I);
553 if (J->first + J->second > ThreadDataSection.data())
554 return make_error<StringError>("Overlapping __thread_data sections");
555 }
556 ThreadDataSections.insert(
557 I, std::make_pair(ThreadDataSection.data(), ThreadDataSection.size()));
558 return Error::success();
559 }
560
deregisterThreadDataSection(span<const char> ThreadDataSection)561 Error MachOPlatformRuntimeState::deregisterThreadDataSection(
562 span<const char> ThreadDataSection) {
563 std::lock_guard<std::mutex> Lock(ThreadDataSectionsMutex);
564 auto I = ThreadDataSections.find(ThreadDataSection.data());
565 if (I == ThreadDataSections.end())
566 return make_error<StringError>("Attempt to deregister unknown thread data "
567 "section");
568 ThreadDataSections.erase(I);
569 return Error::success();
570 }
571
registerObjectSymbolTable(ExecutorAddr HeaderAddr,const std::vector<std::tuple<ExecutorAddr,ExecutorAddr,MachOExecutorSymbolFlags>> & Entries)572 Error MachOPlatformRuntimeState::registerObjectSymbolTable(
573 ExecutorAddr HeaderAddr,
574 const std::vector<std::tuple<ExecutorAddr, ExecutorAddr,
575 MachOExecutorSymbolFlags>> &Entries) {
576
577 std::lock_guard<std::mutex> Lock(JDStatesMutex);
578 auto *JDS = getJITDylibStateByHeader(HeaderAddr.toPtr<void *>());
579 if (!JDS) {
580 std::ostringstream ErrStream;
581 ErrStream << "Could not register object platform sections for "
582 "unrecognized header "
583 << HeaderAddr.toPtr<void *>();
584 return make_error<StringError>(ErrStream.str());
585 }
586
587 for (auto &[NameAddr, SymAddr, Flags] : Entries)
588 JDS->SymbolTable[NameAddr.toPtr<const char *>()] = {SymAddr, Flags};
589
590 return Error::success();
591 }
592
deregisterObjectSymbolTable(ExecutorAddr HeaderAddr,const std::vector<std::tuple<ExecutorAddr,ExecutorAddr,MachOExecutorSymbolFlags>> & Entries)593 Error MachOPlatformRuntimeState::deregisterObjectSymbolTable(
594 ExecutorAddr HeaderAddr,
595 const std::vector<std::tuple<ExecutorAddr, ExecutorAddr,
596 MachOExecutorSymbolFlags>> &Entries) {
597
598 std::lock_guard<std::mutex> Lock(JDStatesMutex);
599 auto *JDS = getJITDylibStateByHeader(HeaderAddr.toPtr<void *>());
600 if (!JDS) {
601 std::ostringstream ErrStream;
602 ErrStream << "Could not register object platform sections for "
603 "unrecognized header "
604 << HeaderAddr.toPtr<void *>();
605 return make_error<StringError>(ErrStream.str());
606 }
607
608 for (auto &[NameAddr, SymAddr, Flags] : Entries)
609 JDS->SymbolTable.erase(NameAddr.toPtr<const char *>());
610
611 return Error::success();
612 }
613
registerObjectPlatformSections(ExecutorAddr HeaderAddr,std::optional<UnwindSectionInfo> UnwindInfo,std::vector<std::pair<std::string_view,ExecutorAddrRange>> Secs)614 Error MachOPlatformRuntimeState::registerObjectPlatformSections(
615 ExecutorAddr HeaderAddr, std::optional<UnwindSectionInfo> UnwindInfo,
616 std::vector<std::pair<std::string_view, ExecutorAddrRange>> Secs) {
617
618 // FIXME: Reject platform section registration after the JITDylib is
619 // sealed?
620
621 ORC_RT_DEBUG({
622 printdbg("MachOPlatform: Registering object sections for %p.\n",
623 HeaderAddr.toPtr<void *>());
624 });
625
626 std::lock_guard<std::mutex> Lock(JDStatesMutex);
627 auto *JDS = getJITDylibStateByHeader(HeaderAddr.toPtr<void *>());
628 if (!JDS) {
629 std::ostringstream ErrStream;
630 ErrStream << "Could not register object platform sections for "
631 "unrecognized header "
632 << HeaderAddr.toPtr<void *>();
633 return make_error<StringError>(ErrStream.str());
634 }
635
636 if (UnwindInfo && UseCallbackStyleUnwindInfo) {
637 ORC_RT_DEBUG({
638 printdbg(" Registering new-style unwind info for:\n"
639 " DWARF: %p -- %p\n"
640 " Compact-unwind: %p -- %p\n"
641 " for:\n",
642 UnwindInfo->DwarfSection.Start.toPtr<void *>(),
643 UnwindInfo->DwarfSection.End.toPtr<void *>(),
644 UnwindInfo->CompactUnwindSection.Start.toPtr<void *>(),
645 UnwindInfo->CompactUnwindSection.End.toPtr<void *>());
646 });
647 for (auto &CodeRange : UnwindInfo->CodeRanges) {
648 JDS->UnwindSections.insert(CodeRange.Start.toPtr<char *>(),
649 CodeRange.End.toPtr<char *>(), *UnwindInfo);
650 ORC_RT_DEBUG({
651 printdbg(" [ %p -- %p ]\n", CodeRange.Start.toPtr<void *>(),
652 CodeRange.End.toPtr<void *>());
653 });
654 }
655 }
656
657 for (auto &KV : Secs) {
658 // FIXME: Validate section ranges?
659 if (KV.first == "__TEXT,__eh_frame") {
660 if (!UseCallbackStyleUnwindInfo) {
661 // Use classic libunwind registration.
662 if (auto Err = registerEHFrames(KV.second.toSpan<const char>()))
663 return Err;
664 }
665 } else if (KV.first == "__DATA,__data") {
666 assert(!JDS->DataSectionContent.count(KV.second.Start.toPtr<char *>()) &&
667 "Address already registered.");
668 auto S = KV.second.toSpan<char>();
669 JDS->DataSectionContent[KV.second.Start.toPtr<char *>()] =
670 std::vector<char>(S.begin(), S.end());
671 } else if (KV.first == "__DATA,__common") {
672 JDS->ZeroInitRanges[KV.second.Start.toPtr<char *>()] = KV.second.size();
673 } else if (KV.first == "__DATA,__thread_data") {
674 if (auto Err = registerThreadDataSection(KV.second.toSpan<const char>()))
675 return Err;
676 } else if (KV.first == "__llvm_jitlink_ObjCRuntimeRegistrationObject")
677 JDS->ObjCRuntimeRegistrationObjects.add(KV.second.toSpan<char>());
678 else if (KV.first == "__DATA,__mod_init_func")
679 JDS->ModInitsSections.add(KV.second.toSpan<void (*)()>());
680 else {
681 // Should this be a warning instead?
682 return make_error<StringError>(
683 "Encountered unexpected section " +
684 std::string(KV.first.data(), KV.first.size()) +
685 " while registering object platform sections");
686 }
687 }
688
689 return Error::success();
690 }
691
deregisterObjectPlatformSections(ExecutorAddr HeaderAddr,std::optional<UnwindSectionInfo> UnwindInfo,std::vector<std::pair<std::string_view,ExecutorAddrRange>> Secs)692 Error MachOPlatformRuntimeState::deregisterObjectPlatformSections(
693 ExecutorAddr HeaderAddr, std::optional<UnwindSectionInfo> UnwindInfo,
694 std::vector<std::pair<std::string_view, ExecutorAddrRange>> Secs) {
695 // TODO: Make this more efficient? (maybe unnecessary if removal is rare?)
696 // TODO: Add a JITDylib prepare-for-teardown operation that clears all
697 // registered sections, causing this function to take the fast-path.
698 ORC_RT_DEBUG({
699 printdbg("MachOPlatform: Deregistering object sections for %p.\n",
700 HeaderAddr.toPtr<void *>());
701 });
702
703 std::lock_guard<std::mutex> Lock(JDStatesMutex);
704 auto *JDS = getJITDylibStateByHeader(HeaderAddr.toPtr<void *>());
705 if (!JDS) {
706 std::ostringstream ErrStream;
707 ErrStream << "Could not register object platform sections for unrecognized "
708 "header "
709 << HeaderAddr.toPtr<void *>();
710 return make_error<StringError>(ErrStream.str());
711 }
712
713 // FIXME: Implement faster-path by returning immediately if JDS is being
714 // torn down entirely?
715
716 // TODO: Make library permanent (i.e. not able to be dlclosed) if it contains
717 // any Swift or ObjC. Once this happens we can clear (and no longer record)
718 // data section content, as the library could never be re-initialized.
719
720 if (UnwindInfo && UseCallbackStyleUnwindInfo) {
721 ORC_RT_DEBUG({
722 printdbg(" Deregistering new-style unwind info for:\n"
723 " DWARF: %p -- %p\n"
724 " Compact-unwind: %p -- %p\n"
725 " for:\n",
726 UnwindInfo->DwarfSection.Start.toPtr<void *>(),
727 UnwindInfo->DwarfSection.End.toPtr<void *>(),
728 UnwindInfo->CompactUnwindSection.Start.toPtr<void *>(),
729 UnwindInfo->CompactUnwindSection.End.toPtr<void *>());
730 });
731 for (auto &CodeRange : UnwindInfo->CodeRanges) {
732 JDS->UnwindSections.erase(CodeRange.Start.toPtr<char *>(),
733 CodeRange.End.toPtr<char *>());
734 ORC_RT_DEBUG({
735 printdbg(" [ %p -- %p ]\n", CodeRange.Start.toPtr<void *>(),
736 CodeRange.End.toPtr<void *>());
737 });
738 }
739 }
740
741 for (auto &KV : Secs) {
742 // FIXME: Validate section ranges?
743 if (KV.first == "__TEXT,__eh_frame") {
744 if (!UseCallbackStyleUnwindInfo) {
745 // Use classic libunwind registration.
746 if (auto Err = deregisterEHFrames(KV.second.toSpan<const char>()))
747 return Err;
748 }
749 } else if (KV.first == "__DATA,__data") {
750 JDS->DataSectionContent.erase(KV.second.Start.toPtr<char *>());
751 } else if (KV.first == "__DATA,__common") {
752 JDS->ZeroInitRanges.erase(KV.second.Start.toPtr<char *>());
753 } else if (KV.first == "__DATA,__thread_data") {
754 if (auto Err =
755 deregisterThreadDataSection(KV.second.toSpan<const char>()))
756 return Err;
757 } else if (KV.first == "__llvm_jitlink_ObjCRuntimeRegistrationObject")
758 JDS->ObjCRuntimeRegistrationObjects.removeIfPresent(KV.second);
759 else if (KV.first == "__DATA,__mod_init_func")
760 JDS->ModInitsSections.removeIfPresent(KV.second);
761 else {
762 // Should this be a warning instead?
763 return make_error<StringError>(
764 "Encountered unexpected section " +
765 std::string(KV.first.data(), KV.first.size()) +
766 " while deregistering object platform sections");
767 }
768 }
769 return Error::success();
770 }
771
dlerror()772 const char *MachOPlatformRuntimeState::dlerror() { return DLFcnError.c_str(); }
773
dlopen(std::string_view Path,int Mode)774 void *MachOPlatformRuntimeState::dlopen(std::string_view Path, int Mode) {
775 ORC_RT_DEBUG({
776 std::string S(Path.data(), Path.size());
777 printdbg("MachOPlatform::dlopen(\"%s\")\n", S.c_str());
778 });
779 std::lock_guard<std::recursive_mutex> Lock(DyldAPIMutex);
780 if (auto H = dlopenImpl(Path, Mode))
781 return *H;
782 else {
783 // FIXME: Make dlerror thread safe.
784 DLFcnError = toString(H.takeError());
785 return nullptr;
786 }
787 }
788
dlclose(void * DSOHandle)789 int MachOPlatformRuntimeState::dlclose(void *DSOHandle) {
790 ORC_RT_DEBUG({
791 auto *JDS = getJITDylibStateByHeader(DSOHandle);
792 std::string DylibName;
793 if (JDS) {
794 std::string S;
795 printdbg("MachOPlatform::dlclose(%p) (%s)\n", DSOHandle, S.c_str());
796 } else
797 printdbg("MachOPlatform::dlclose(%p) (%s)\n", DSOHandle,
798 "invalid handle");
799 });
800 std::lock_guard<std::recursive_mutex> Lock(DyldAPIMutex);
801 if (auto Err = dlcloseImpl(DSOHandle)) {
802 // FIXME: Make dlerror thread safe.
803 DLFcnError = toString(std::move(Err));
804 return -1;
805 }
806 return 0;
807 }
808
dlsym(void * DSOHandle,const char * Symbol)809 void *MachOPlatformRuntimeState::dlsym(void *DSOHandle, const char *Symbol) {
810 std::unique_lock<std::mutex> Lock(JDStatesMutex);
811 auto *JDS = getJITDylibStateByHeader(DSOHandle);
812 if (!JDS) {
813 std::ostringstream ErrStream;
814 ErrStream << "In call to dlsym, unrecognized header address " << DSOHandle;
815 DLFcnError = ErrStream.str();
816 return nullptr;
817 }
818
819 std::string MangledName = std::string("_") + Symbol;
820 std::pair<std::string_view, bool> Lookup(MangledName, false);
821 std::pair<ExecutorAddr, MachOExecutorSymbolFlags> Result;
822
823 if (auto Err = lookupSymbols(*JDS, Lock, {&Result, 1}, {&Lookup, 1})) {
824 DLFcnError = toString(std::move(Err));
825 return nullptr;
826 }
827
828 // Sign callable symbols as functions, to match dyld.
829 if ((Result.second & MachOExecutorSymbolFlags::Callable) ==
830 MachOExecutorSymbolFlags::Callable)
831 return reinterpret_cast<void *>(Result.first.toPtr<void(void)>());
832 return Result.first.toPtr<void *>();
833 }
834
registerAtExit(void (* F)(void *),void * Arg,void * DSOHandle)835 int MachOPlatformRuntimeState::registerAtExit(void (*F)(void *), void *Arg,
836 void *DSOHandle) {
837 // FIXME: Handle out-of-memory errors, returning -1 if OOM.
838 std::lock_guard<std::mutex> Lock(JDStatesMutex);
839 auto *JDS = getJITDylibStateByHeader(DSOHandle);
840 if (!JDS) {
841 ORC_RT_DEBUG({
842 printdbg("MachOPlatformRuntimeState::registerAtExit called with "
843 "unrecognized dso handle %p\n",
844 DSOHandle);
845 });
846 return -1;
847 }
848 JDS->AtExits.push_back({F, Arg});
849 return 0;
850 }
851
runAtExits(std::unique_lock<std::mutex> & JDStatesLock,JITDylibState & JDS)852 void MachOPlatformRuntimeState::runAtExits(
853 std::unique_lock<std::mutex> &JDStatesLock, JITDylibState &JDS) {
854 auto AtExits = std::move(JDS.AtExits);
855
856 // Unlock while running atexits, as they may trigger operations that modify
857 // JDStates.
858 JDStatesLock.unlock();
859 while (!AtExits.empty()) {
860 auto &AE = AtExits.back();
861 AE.Func(AE.Arg);
862 AtExits.pop_back();
863 }
864 JDStatesLock.lock();
865 }
866
runAtExits(void * DSOHandle)867 void MachOPlatformRuntimeState::runAtExits(void *DSOHandle) {
868 std::unique_lock<std::mutex> Lock(JDStatesMutex);
869 auto *JDS = getJITDylibStateByHeader(DSOHandle);
870 ORC_RT_DEBUG({
871 printdbg("MachOPlatformRuntimeState::runAtExits called on unrecognized "
872 "dso_handle %p\n",
873 DSOHandle);
874 });
875 if (JDS)
876 runAtExits(Lock, *JDS);
877 }
878
879 Expected<std::pair<const char *, size_t>>
getThreadDataSectionFor(const char * ThreadData)880 MachOPlatformRuntimeState::getThreadDataSectionFor(const char *ThreadData) {
881 std::lock_guard<std::mutex> Lock(ThreadDataSectionsMutex);
882 auto I = ThreadDataSections.upper_bound(ThreadData);
883 // Check that we have a valid entry covering this address.
884 if (I == ThreadDataSections.begin())
885 return make_error<StringError>("No thread local data section for key");
886 I = std::prev(I);
887 if (ThreadData >= I->first + I->second)
888 return make_error<StringError>("No thread local data section for key");
889 return *I;
890 }
891
892 MachOPlatformRuntimeState::JITDylibState *
getJITDylibStateByHeader(void * DSOHandle)893 MachOPlatformRuntimeState::getJITDylibStateByHeader(void *DSOHandle) {
894 auto I = JDStates.find(DSOHandle);
895 if (I == JDStates.end()) {
896 I = JDStates.insert(std::make_pair(DSOHandle, JITDylibState())).first;
897 I->second.Header = DSOHandle;
898 }
899 return &I->second;
900 }
901
902 MachOPlatformRuntimeState::JITDylibState *
getJITDylibStateByName(std::string_view Name)903 MachOPlatformRuntimeState::getJITDylibStateByName(std::string_view Name) {
904 // FIXME: Avoid creating string once we have C++20.
905 auto I = JDNameToHeader.find(std::string(Name.data(), Name.size()));
906 if (I != JDNameToHeader.end())
907 return getJITDylibStateByHeader(I->second);
908 return nullptr;
909 }
910
requestPushSymbols(JITDylibState & JDS,span<std::pair<std::string_view,bool>> Symbols)911 Error MachOPlatformRuntimeState::requestPushSymbols(
912 JITDylibState &JDS, span<std::pair<std::string_view, bool>> Symbols) {
913 Error OpErr = Error::success();
914 if (auto Err = WrapperFunction<SPSError(
915 SPSExecutorAddr, SPSSequence<SPSTuple<SPSString, bool>>)>::
916 call(&__orc_rt_macho_push_symbols_tag, OpErr,
917 ExecutorAddr::fromPtr(JDS.Header), Symbols)) {
918 cantFail(std::move(OpErr));
919 return std::move(Err);
920 }
921 return OpErr;
922 }
923
lookupSymbols(JITDylibState & JDS,std::unique_lock<std::mutex> & JDStatesLock,span<std::pair<ExecutorAddr,MachOExecutorSymbolFlags>> Result,span<std::pair<std::string_view,bool>> Symbols)924 Error MachOPlatformRuntimeState::lookupSymbols(
925 JITDylibState &JDS, std::unique_lock<std::mutex> &JDStatesLock,
926 span<std::pair<ExecutorAddr, MachOExecutorSymbolFlags>> Result,
927 span<std::pair<std::string_view, bool>> Symbols) {
928 assert(JDStatesLock.owns_lock() &&
929 "JDStatesLock should be locked at call-site");
930 assert(Result.size() == Symbols.size() &&
931 "Results and Symbols span sizes should match");
932
933 // Make an initial pass over the local symbol table.
934 std::vector<size_t> MissingSymbolIndexes;
935 for (size_t Idx = 0; Idx != Symbols.size(); ++Idx) {
936 auto I = JDS.SymbolTable.find(Symbols[Idx].first);
937 if (I != JDS.SymbolTable.end())
938 Result[Idx] = I->second;
939 else
940 MissingSymbolIndexes.push_back(Idx);
941 }
942
943 // If everything has been resolved already then bail out early.
944 if (MissingSymbolIndexes.empty())
945 return Error::success();
946
947 // Otherwise call back to the controller to try to request that the symbol
948 // be materialized.
949 std::vector<std::pair<std::string_view, bool>> MissingSymbols;
950 MissingSymbols.reserve(MissingSymbolIndexes.size());
951 ORC_RT_DEBUG({
952 printdbg("requesting push of %i missing symbols...\n",
953 MissingSymbolIndexes.size());
954 });
955 for (auto MissingIdx : MissingSymbolIndexes)
956 MissingSymbols.push_back(Symbols[MissingIdx]);
957
958 JDStatesLock.unlock();
959 if (auto Err = requestPushSymbols(
960 JDS, {MissingSymbols.data(), MissingSymbols.size()}))
961 return Err;
962 JDStatesLock.lock();
963
964 // Try to resolve the previously missing symbols locally.
965 std::vector<size_t> MissingRequiredSymbols;
966 for (auto MissingIdx : MissingSymbolIndexes) {
967 auto I = JDS.SymbolTable.find(Symbols[MissingIdx].first);
968 if (I != JDS.SymbolTable.end())
969 Result[MissingIdx] = I->second;
970 else {
971 if (Symbols[MissingIdx].second)
972 MissingRequiredSymbols.push_back(MissingIdx);
973 else
974 Result[MissingIdx] = {ExecutorAddr(), {}};
975 }
976 }
977
978 // Error out if any missing symbols could not be resolved.
979 if (!MissingRequiredSymbols.empty()) {
980 std::ostringstream ErrStream;
981 ErrStream << "Lookup could not find required symbols: [ ";
982 for (auto MissingIdx : MissingRequiredSymbols)
983 ErrStream << "\"" << Symbols[MissingIdx].first << "\" ";
984 ErrStream << "]";
985 return make_error<StringError>(ErrStream.str());
986 }
987
988 return Error::success();
989 }
990
991 // eh-frame registration functions.
992 // We expect these to be available for all processes.
993 extern "C" void __register_frame(const void *);
994 extern "C" void __deregister_frame(const void *);
995
996 template <typename HandleFDEFn>
walkEHFrameSection(span<const char> EHFrameSection,HandleFDEFn HandleFDE)997 void walkEHFrameSection(span<const char> EHFrameSection,
998 HandleFDEFn HandleFDE) {
999 const char *CurCFIRecord = EHFrameSection.data();
1000 uint64_t Size = *reinterpret_cast<const uint32_t *>(CurCFIRecord);
1001
1002 while (CurCFIRecord != EHFrameSection.end() && Size != 0) {
1003 const char *OffsetField = CurCFIRecord + (Size == 0xffffffff ? 12 : 4);
1004 if (Size == 0xffffffff)
1005 Size = *reinterpret_cast<const uint64_t *>(CurCFIRecord + 4) + 12;
1006 else
1007 Size += 4;
1008 uint32_t Offset = *reinterpret_cast<const uint32_t *>(OffsetField);
1009
1010 if (Offset != 0)
1011 HandleFDE(CurCFIRecord);
1012
1013 CurCFIRecord += Size;
1014 Size = *reinterpret_cast<const uint32_t *>(CurCFIRecord);
1015 }
1016 }
1017
lookupUnwindSections(void * Addr,unw_dynamic_unwind_sections & Info)1018 bool MachOPlatformRuntimeState::lookupUnwindSections(
1019 void *Addr, unw_dynamic_unwind_sections &Info) {
1020 ORC_RT_DEBUG(
1021 { printdbg("Tried to lookup unwind-info via new lookup call.\n"); });
1022 std::lock_guard<std::mutex> Lock(JDStatesMutex);
1023 for (auto &KV : JDStates) {
1024 auto &JD = KV.second;
1025 auto I = JD.UnwindSections.find(reinterpret_cast<char *>(Addr));
1026 if (I != JD.UnwindSections.end()) {
1027 Info.dso_base = reinterpret_cast<uintptr_t>(JD.Header);
1028 Info.dwarf_section =
1029 reinterpret_cast<uintptr_t>(I->second.DwarfSection.data());
1030 Info.dwarf_section_length = I->second.DwarfSection.size();
1031 Info.compact_unwind_section =
1032 reinterpret_cast<uintptr_t>(I->second.CompactUnwindSection.data());
1033 Info.compact_unwind_section_length =
1034 I->second.CompactUnwindSection.size();
1035 return true;
1036 }
1037 }
1038 return false;
1039 }
1040
findDynamicUnwindSections(uintptr_t addr,unw_dynamic_unwind_sections * info)1041 int MachOPlatformRuntimeState::findDynamicUnwindSections(
1042 uintptr_t addr, unw_dynamic_unwind_sections *info) {
1043 if (!info)
1044 return 0;
1045 return MachOPlatformRuntimeState::get().lookupUnwindSections((void *)addr,
1046 *info);
1047 }
1048
registerEHFrames(span<const char> EHFrameSection)1049 Error MachOPlatformRuntimeState::registerEHFrames(
1050 span<const char> EHFrameSection) {
1051 walkEHFrameSection(EHFrameSection, __register_frame);
1052 return Error::success();
1053 }
1054
deregisterEHFrames(span<const char> EHFrameSection)1055 Error MachOPlatformRuntimeState::deregisterEHFrames(
1056 span<const char> EHFrameSection) {
1057 walkEHFrameSection(EHFrameSection, __deregister_frame);
1058 return Error::success();
1059 }
1060
registerObjCRegistrationObjects(JITDylibState & JDS)1061 Error MachOPlatformRuntimeState::registerObjCRegistrationObjects(
1062 JITDylibState &JDS) {
1063 ORC_RT_DEBUG(printdbg("Registering Objective-C / Swift metadata.\n"));
1064
1065 std::vector<char *> RegObjBases;
1066 JDS.ObjCRuntimeRegistrationObjects.processNewSections(
1067 [&](span<char> RegObj) { RegObjBases.push_back(RegObj.data()); });
1068
1069 if (RegObjBases.empty())
1070 return Error::success();
1071
1072 if (!_objc_map_images || !_objc_load_image)
1073 return make_error<StringError>(
1074 "Could not register Objective-C / Swift metadata: _objc_map_images / "
1075 "_objc_load_image not found");
1076
1077 std::vector<char *> Paths;
1078 Paths.resize(RegObjBases.size());
1079 _objc_map_images(RegObjBases.size(), Paths.data(),
1080 reinterpret_cast<mach_header **>(RegObjBases.data()));
1081
1082 for (void *RegObjBase : RegObjBases)
1083 _objc_load_image(nullptr, reinterpret_cast<mach_header *>(RegObjBase));
1084
1085 return Error::success();
1086 }
1087
runModInits(std::unique_lock<std::mutex> & JDStatesLock,JITDylibState & JDS)1088 Error MachOPlatformRuntimeState::runModInits(
1089 std::unique_lock<std::mutex> &JDStatesLock, JITDylibState &JDS) {
1090 std::vector<span<void (*)()>> InitSections;
1091 InitSections.reserve(JDS.ModInitsSections.numNewSections());
1092
1093 // Copy initializer sections: If the JITDylib is unsealed then the
1094 // initializers could reach back into the JIT and cause more initializers to
1095 // be added.
1096 // FIXME: Skip unlock and run in-place on sealed JITDylibs?
1097 JDS.ModInitsSections.processNewSections(
1098 [&](span<void (*)()> Inits) { InitSections.push_back(Inits); });
1099
1100 JDStatesLock.unlock();
1101 for (auto InitSec : InitSections)
1102 for (auto *Init : InitSec)
1103 Init();
1104 JDStatesLock.lock();
1105
1106 return Error::success();
1107 }
1108
dlopenImpl(std::string_view Path,int Mode)1109 Expected<void *> MachOPlatformRuntimeState::dlopenImpl(std::string_view Path,
1110 int Mode) {
1111 std::unique_lock<std::mutex> Lock(JDStatesMutex);
1112
1113 // Try to find JITDylib state by name.
1114 auto *JDS = getJITDylibStateByName(Path);
1115
1116 if (!JDS)
1117 return make_error<StringError>("No registered JTIDylib for path " +
1118 std::string(Path.data(), Path.size()));
1119
1120 // If this JITDylib is unsealed, or this is the first dlopen then run
1121 // full dlopen path (update deps, push and run initializers, update ref
1122 // counts on all JITDylibs in the dep tree).
1123 if (!JDS->referenced() || !JDS->Sealed) {
1124 if (auto Err = dlopenFull(Lock, *JDS))
1125 return std::move(Err);
1126 }
1127
1128 // Bump the ref-count on this dylib.
1129 ++JDS->DlRefCount;
1130
1131 // Return the header address.
1132 return JDS->Header;
1133 }
1134
dlopenFull(std::unique_lock<std::mutex> & JDStatesLock,JITDylibState & JDS)1135 Error MachOPlatformRuntimeState::dlopenFull(
1136 std::unique_lock<std::mutex> &JDStatesLock, JITDylibState &JDS) {
1137 // Call back to the JIT to push the initializers.
1138 Expected<MachOJITDylibDepInfoMap> DepInfo((MachOJITDylibDepInfoMap()));
1139 // Unlock so that we can accept the initializer update.
1140 JDStatesLock.unlock();
1141 if (auto Err = WrapperFunction<SPSExpected<SPSMachOJITDylibDepInfoMap>(
1142 SPSExecutorAddr)>::call(&__orc_rt_macho_push_initializers_tag,
1143 DepInfo, ExecutorAddr::fromPtr(JDS.Header)))
1144 return Err;
1145 JDStatesLock.lock();
1146
1147 if (!DepInfo)
1148 return DepInfo.takeError();
1149
1150 if (auto Err = dlopenInitialize(JDStatesLock, JDS, *DepInfo))
1151 return Err;
1152
1153 if (!DepInfo->empty()) {
1154 ORC_RT_DEBUG({
1155 printdbg("Unrecognized dep-info key headers in dlopen of %s\n",
1156 JDS.Name.c_str());
1157 });
1158 std::ostringstream ErrStream;
1159 ErrStream << "Encountered unrecognized dep-info key headers "
1160 "while processing dlopen of "
1161 << JDS.Name;
1162 return make_error<StringError>(ErrStream.str());
1163 }
1164
1165 return Error::success();
1166 }
1167
dlopenInitialize(std::unique_lock<std::mutex> & JDStatesLock,JITDylibState & JDS,MachOJITDylibDepInfoMap & DepInfo)1168 Error MachOPlatformRuntimeState::dlopenInitialize(
1169 std::unique_lock<std::mutex> &JDStatesLock, JITDylibState &JDS,
1170 MachOJITDylibDepInfoMap &DepInfo) {
1171 ORC_RT_DEBUG({
1172 printdbg("MachOPlatformRuntimeState::dlopenInitialize(\"%s\")\n",
1173 JDS.Name.c_str());
1174 });
1175
1176 // If the header is not present in the dep map then assume that we
1177 // already processed it earlier in the dlopenInitialize traversal and
1178 // return.
1179 // TODO: Keep a visited set instead so that we can error out on missing
1180 // entries?
1181 auto I = DepInfo.find(ExecutorAddr::fromPtr(JDS.Header));
1182 if (I == DepInfo.end())
1183 return Error::success();
1184
1185 auto DI = std::move(I->second);
1186 DepInfo.erase(I);
1187
1188 // We don't need to re-initialize sealed JITDylibs that have already been
1189 // initialized. Just check that their dep-map entry is empty as expected.
1190 if (JDS.Sealed) {
1191 if (!DI.DepHeaders.empty()) {
1192 std::ostringstream ErrStream;
1193 ErrStream << "Sealed JITDylib " << JDS.Header
1194 << " already has registered dependencies";
1195 return make_error<StringError>(ErrStream.str());
1196 }
1197 if (JDS.referenced())
1198 return Error::success();
1199 } else
1200 JDS.Sealed = DI.Sealed;
1201
1202 // This is an unsealed or newly sealed JITDylib. Run initializers.
1203 std::vector<JITDylibState *> OldDeps;
1204 std::swap(JDS.Deps, OldDeps);
1205 JDS.Deps.reserve(DI.DepHeaders.size());
1206 for (auto DepHeaderAddr : DI.DepHeaders) {
1207 auto *DepJDS = getJITDylibStateByHeader(DepHeaderAddr.toPtr<void *>());
1208 if (!DepJDS) {
1209 std::ostringstream ErrStream;
1210 ErrStream << "Encountered unrecognized dep header "
1211 << DepHeaderAddr.toPtr<void *>() << " while initializing "
1212 << JDS.Name;
1213 return make_error<StringError>(ErrStream.str());
1214 }
1215 ++DepJDS->LinkedAgainstRefCount;
1216 if (auto Err = dlopenInitialize(JDStatesLock, *DepJDS, DepInfo))
1217 return Err;
1218 }
1219
1220 // Initialize this JITDylib.
1221 if (auto Err = registerObjCRegistrationObjects(JDS))
1222 return Err;
1223 if (auto Err = runModInits(JDStatesLock, JDS))
1224 return Err;
1225
1226 // Decrement old deps.
1227 // FIXME: We should probably continue and just report deinitialize errors
1228 // here.
1229 for (auto *DepJDS : OldDeps) {
1230 --DepJDS->LinkedAgainstRefCount;
1231 if (!DepJDS->referenced())
1232 if (auto Err = dlcloseDeinitialize(JDStatesLock, *DepJDS))
1233 return Err;
1234 }
1235
1236 return Error::success();
1237 }
1238
dlcloseImpl(void * DSOHandle)1239 Error MachOPlatformRuntimeState::dlcloseImpl(void *DSOHandle) {
1240 std::unique_lock<std::mutex> Lock(JDStatesMutex);
1241
1242 // Try to find JITDylib state by header.
1243 auto *JDS = getJITDylibStateByHeader(DSOHandle);
1244
1245 if (!JDS) {
1246 std::ostringstream ErrStream;
1247 ErrStream << "No registered JITDylib for " << DSOHandle;
1248 return make_error<StringError>(ErrStream.str());
1249 }
1250
1251 // Bump the ref-count.
1252 --JDS->DlRefCount;
1253
1254 if (!JDS->referenced())
1255 return dlcloseDeinitialize(Lock, *JDS);
1256
1257 return Error::success();
1258 }
1259
dlcloseDeinitialize(std::unique_lock<std::mutex> & JDStatesLock,JITDylibState & JDS)1260 Error MachOPlatformRuntimeState::dlcloseDeinitialize(
1261 std::unique_lock<std::mutex> &JDStatesLock, JITDylibState &JDS) {
1262
1263 ORC_RT_DEBUG({
1264 printdbg("MachOPlatformRuntimeState::dlcloseDeinitialize(\"%s\")\n",
1265 JDS.Name.c_str());
1266 });
1267
1268 runAtExits(JDStatesLock, JDS);
1269
1270 // Reset mod-inits
1271 JDS.ModInitsSections.reset();
1272
1273 // Reset data section contents.
1274 for (auto &KV : JDS.DataSectionContent)
1275 memcpy(KV.first, KV.second.data(), KV.second.size());
1276 for (auto &KV : JDS.ZeroInitRanges)
1277 memset(KV.first, 0, KV.second);
1278
1279 // Deinitialize any dependencies.
1280 for (auto *DepJDS : JDS.Deps) {
1281 --DepJDS->LinkedAgainstRefCount;
1282 if (!DepJDS->referenced())
1283 if (auto Err = dlcloseDeinitialize(JDStatesLock, *DepJDS))
1284 return Err;
1285 }
1286
1287 return Error::success();
1288 }
1289
1290 class MachOPlatformRuntimeTLVManager {
1291 public:
1292 void *getInstance(const char *ThreadData);
1293
1294 private:
1295 std::unordered_map<const char *, char *> Instances;
1296 std::unordered_map<const char *, std::unique_ptr<char[]>> AllocatedSections;
1297 };
1298
getInstance(const char * ThreadData)1299 void *MachOPlatformRuntimeTLVManager::getInstance(const char *ThreadData) {
1300 auto I = Instances.find(ThreadData);
1301 if (I != Instances.end())
1302 return I->second;
1303
1304 auto TDS =
1305 MachOPlatformRuntimeState::get().getThreadDataSectionFor(ThreadData);
1306 if (!TDS) {
1307 __orc_rt_log_error(toString(TDS.takeError()).c_str());
1308 return nullptr;
1309 }
1310
1311 auto &Allocated = AllocatedSections[TDS->first];
1312 if (!Allocated) {
1313 Allocated = std::make_unique<char[]>(TDS->second);
1314 memcpy(Allocated.get(), TDS->first, TDS->second);
1315 }
1316
1317 size_t ThreadDataDelta = ThreadData - TDS->first;
1318 assert(ThreadDataDelta <= TDS->second && "ThreadData outside section bounds");
1319
1320 char *Instance = Allocated.get() + ThreadDataDelta;
1321 Instances[ThreadData] = Instance;
1322 return Instance;
1323 }
1324
destroyMachOTLVMgr(void * MachOTLVMgr)1325 void destroyMachOTLVMgr(void *MachOTLVMgr) {
1326 delete static_cast<MachOPlatformRuntimeTLVManager *>(MachOTLVMgr);
1327 }
1328
runWrapperFunctionCalls(std::vector<WrapperFunctionCall> WFCs)1329 Error runWrapperFunctionCalls(std::vector<WrapperFunctionCall> WFCs) {
1330 for (auto &WFC : WFCs)
1331 if (auto Err = WFC.runWithSPSRet<void>())
1332 return Err;
1333 return Error::success();
1334 }
1335
1336 } // end anonymous namespace
1337
1338 //------------------------------------------------------------------------------
1339 // JIT entry points
1340 //------------------------------------------------------------------------------
1341
1342 ORC_RT_INTERFACE orc_rt_CWrapperFunctionResult
__orc_rt_macho_platform_bootstrap(char * ArgData,size_t ArgSize)1343 __orc_rt_macho_platform_bootstrap(char *ArgData, size_t ArgSize) {
1344 return WrapperFunction<SPSError()>::handle(
1345 ArgData, ArgSize,
1346 []() { return MachOPlatformRuntimeState::create(); })
1347 .release();
1348 }
1349
1350 ORC_RT_INTERFACE orc_rt_CWrapperFunctionResult
__orc_rt_macho_platform_shutdown(char * ArgData,size_t ArgSize)1351 __orc_rt_macho_platform_shutdown(char *ArgData, size_t ArgSize) {
1352 return WrapperFunction<SPSError()>::handle(
1353 ArgData, ArgSize,
1354 []() { return MachOPlatformRuntimeState::destroy(); })
1355 .release();
1356 }
1357
1358 ORC_RT_INTERFACE orc_rt_CWrapperFunctionResult
__orc_rt_macho_register_jitdylib(char * ArgData,size_t ArgSize)1359 __orc_rt_macho_register_jitdylib(char *ArgData, size_t ArgSize) {
1360 return WrapperFunction<SPSError(SPSString, SPSExecutorAddr)>::handle(
1361 ArgData, ArgSize,
1362 [](std::string &Name, ExecutorAddr HeaderAddr) {
1363 return MachOPlatformRuntimeState::get().registerJITDylib(
1364 std::move(Name), HeaderAddr.toPtr<void *>());
1365 })
1366 .release();
1367 }
1368
1369 ORC_RT_INTERFACE orc_rt_CWrapperFunctionResult
__orc_rt_macho_deregister_jitdylib(char * ArgData,size_t ArgSize)1370 __orc_rt_macho_deregister_jitdylib(char *ArgData, size_t ArgSize) {
1371 return WrapperFunction<SPSError(SPSExecutorAddr)>::handle(
1372 ArgData, ArgSize,
1373 [](ExecutorAddr HeaderAddr) {
1374 return MachOPlatformRuntimeState::get().deregisterJITDylib(
1375 HeaderAddr.toPtr<void *>());
1376 })
1377 .release();
1378 }
1379
1380 ORC_RT_INTERFACE orc_rt_CWrapperFunctionResult
__orc_rt_macho_register_object_platform_sections(char * ArgData,size_t ArgSize)1381 __orc_rt_macho_register_object_platform_sections(char *ArgData,
1382 size_t ArgSize) {
1383 return WrapperFunction<SPSError(SPSExecutorAddr,
1384 SPSOptional<SPSUnwindSectionInfo>,
1385 SPSMachOObjectPlatformSectionsMap)>::
1386 handle(ArgData, ArgSize,
1387 [](ExecutorAddr HeaderAddr, std::optional<UnwindSectionInfo> USI,
1388 std::vector<std::pair<std::string_view, ExecutorAddrRange>>
1389 &Secs) {
1390 return MachOPlatformRuntimeState::get()
1391 .registerObjectPlatformSections(HeaderAddr, std::move(USI),
1392 std::move(Secs));
1393 })
1394 .release();
1395 }
1396
1397 ORC_RT_INTERFACE orc_rt_CWrapperFunctionResult
__orc_rt_macho_register_object_symbol_table(char * ArgData,size_t ArgSize)1398 __orc_rt_macho_register_object_symbol_table(char *ArgData, size_t ArgSize) {
1399 using SymtabContainer = std::vector<
1400 std::tuple<ExecutorAddr, ExecutorAddr,
1401 MachOPlatformRuntimeState::MachOExecutorSymbolFlags>>;
1402 return WrapperFunction<SPSError(
1403 SPSExecutorAddr, SPSSequence<SPSTuple<SPSExecutorAddr, SPSExecutorAddr,
1404 SPSMachOExecutorSymbolFlags>>)>::
1405 handle(ArgData, ArgSize,
1406 [](ExecutorAddr HeaderAddr, SymtabContainer &Symbols) {
1407 return MachOPlatformRuntimeState::get()
1408 .registerObjectSymbolTable(HeaderAddr, Symbols);
1409 })
1410 .release();
1411 }
1412
1413 ORC_RT_INTERFACE orc_rt_CWrapperFunctionResult
__orc_rt_macho_deregister_object_symbol_table(char * ArgData,size_t ArgSize)1414 __orc_rt_macho_deregister_object_symbol_table(char *ArgData, size_t ArgSize) {
1415 using SymtabContainer = std::vector<
1416 std::tuple<ExecutorAddr, ExecutorAddr,
1417 MachOPlatformRuntimeState::MachOExecutorSymbolFlags>>;
1418 return WrapperFunction<SPSError(
1419 SPSExecutorAddr, SPSSequence<SPSTuple<SPSExecutorAddr, SPSExecutorAddr,
1420 SPSMachOExecutorSymbolFlags>>)>::
1421 handle(ArgData, ArgSize,
1422 [](ExecutorAddr HeaderAddr, SymtabContainer &Symbols) {
1423 return MachOPlatformRuntimeState::get()
1424 .deregisterObjectSymbolTable(HeaderAddr, Symbols);
1425 })
1426 .release();
1427 }
1428
1429 ORC_RT_INTERFACE orc_rt_CWrapperFunctionResult
__orc_rt_macho_deregister_object_platform_sections(char * ArgData,size_t ArgSize)1430 __orc_rt_macho_deregister_object_platform_sections(char *ArgData,
1431 size_t ArgSize) {
1432 return WrapperFunction<SPSError(SPSExecutorAddr,
1433 SPSOptional<SPSUnwindSectionInfo>,
1434 SPSMachOObjectPlatformSectionsMap)>::
1435 handle(ArgData, ArgSize,
1436 [](ExecutorAddr HeaderAddr, std::optional<UnwindSectionInfo> USI,
1437 std::vector<std::pair<std::string_view, ExecutorAddrRange>>
1438 &Secs) {
1439 return MachOPlatformRuntimeState::get()
1440 .deregisterObjectPlatformSections(HeaderAddr, std::move(USI),
1441 std::move(Secs));
1442 })
1443 .release();
1444 }
1445
1446 ORC_RT_INTERFACE orc_rt_CWrapperFunctionResult
__orc_rt_macho_run_wrapper_function_calls(char * ArgData,size_t ArgSize)1447 __orc_rt_macho_run_wrapper_function_calls(char *ArgData, size_t ArgSize) {
1448 return WrapperFunction<SPSError(SPSSequence<SPSWrapperFunctionCall>)>::handle(
1449 ArgData, ArgSize, runWrapperFunctionCalls)
1450 .release();
1451 }
1452
1453 //------------------------------------------------------------------------------
1454 // TLV support
1455 //------------------------------------------------------------------------------
1456
__orc_rt_macho_tlv_get_addr_impl(TLVDescriptor * D)1457 ORC_RT_INTERFACE void *__orc_rt_macho_tlv_get_addr_impl(TLVDescriptor *D) {
1458 auto *TLVMgr = static_cast<MachOPlatformRuntimeTLVManager *>(
1459 pthread_getspecific(D->Key));
1460 if (!TLVMgr) {
1461 TLVMgr = new MachOPlatformRuntimeTLVManager();
1462 if (pthread_setspecific(D->Key, TLVMgr)) {
1463 __orc_rt_log_error("Call to pthread_setspecific failed");
1464 return nullptr;
1465 }
1466 }
1467
1468 return TLVMgr->getInstance(
1469 reinterpret_cast<char *>(static_cast<uintptr_t>(D->DataAddress)));
1470 }
1471
1472 ORC_RT_INTERFACE orc_rt_CWrapperFunctionResult
__orc_rt_macho_create_pthread_key(char * ArgData,size_t ArgSize)1473 __orc_rt_macho_create_pthread_key(char *ArgData, size_t ArgSize) {
1474 return WrapperFunction<SPSExpected<uint64_t>(void)>::handle(
1475 ArgData, ArgSize,
1476 []() -> Expected<uint64_t> {
1477 pthread_key_t Key;
1478 if (int Err = pthread_key_create(&Key, destroyMachOTLVMgr)) {
1479 __orc_rt_log_error("Call to pthread_key_create failed");
1480 return make_error<StringError>(strerror(Err));
1481 }
1482 return static_cast<uint64_t>(Key);
1483 })
1484 .release();
1485 }
1486
1487 //------------------------------------------------------------------------------
1488 // cxa_atexit support
1489 //------------------------------------------------------------------------------
1490
__orc_rt_macho_cxa_atexit(void (* func)(void *),void * arg,void * dso_handle)1491 int __orc_rt_macho_cxa_atexit(void (*func)(void *), void *arg,
1492 void *dso_handle) {
1493 return MachOPlatformRuntimeState::get().registerAtExit(func, arg, dso_handle);
1494 }
1495
__orc_rt_macho_cxa_finalize(void * dso_handle)1496 void __orc_rt_macho_cxa_finalize(void *dso_handle) {
1497 MachOPlatformRuntimeState::get().runAtExits(dso_handle);
1498 }
1499
1500 //------------------------------------------------------------------------------
1501 // JIT'd dlfcn alternatives.
1502 //------------------------------------------------------------------------------
1503
__orc_rt_macho_jit_dlerror()1504 const char *__orc_rt_macho_jit_dlerror() {
1505 return MachOPlatformRuntimeState::get().dlerror();
1506 }
1507
__orc_rt_macho_jit_dlopen(const char * path,int mode)1508 void *__orc_rt_macho_jit_dlopen(const char *path, int mode) {
1509 return MachOPlatformRuntimeState::get().dlopen(path, mode);
1510 }
1511
__orc_rt_macho_jit_dlclose(void * dso_handle)1512 int __orc_rt_macho_jit_dlclose(void *dso_handle) {
1513 return MachOPlatformRuntimeState::get().dlclose(dso_handle);
1514 }
1515
__orc_rt_macho_jit_dlsym(void * dso_handle,const char * symbol)1516 void *__orc_rt_macho_jit_dlsym(void *dso_handle, const char *symbol) {
1517 return MachOPlatformRuntimeState::get().dlsym(dso_handle, symbol);
1518 }
1519
1520 //------------------------------------------------------------------------------
1521 // MachO Run Program
1522 //------------------------------------------------------------------------------
1523
__orc_rt_macho_run_program(const char * JITDylibName,const char * EntrySymbolName,int argc,char * argv[])1524 ORC_RT_INTERFACE int64_t __orc_rt_macho_run_program(const char *JITDylibName,
1525 const char *EntrySymbolName,
1526 int argc, char *argv[]) {
1527 using MainTy = int (*)(int, char *[]);
1528
1529 void *H = __orc_rt_macho_jit_dlopen(JITDylibName,
1530 __orc_rt::macho::ORC_RT_RTLD_LAZY);
1531 if (!H) {
1532 __orc_rt_log_error(__orc_rt_macho_jit_dlerror());
1533 return -1;
1534 }
1535
1536 auto *Main =
1537 reinterpret_cast<MainTy>(__orc_rt_macho_jit_dlsym(H, EntrySymbolName));
1538
1539 if (!Main) {
1540 __orc_rt_log_error(__orc_rt_macho_jit_dlerror());
1541 return -1;
1542 }
1543
1544 int Result = Main(argc, argv);
1545
1546 if (__orc_rt_macho_jit_dlclose(H) == -1)
1547 __orc_rt_log_error(__orc_rt_macho_jit_dlerror());
1548
1549 return Result;
1550 }
1551