xref: /llvm-project/llvm/lib/ExecutionEngine/JITLink/JITLink.cpp (revision 4eaff6c58ae2f130ac8d63cf2c87bbb483114876) 
1 //===------------- JITLink.cpp - Core Run-time JIT linker APIs ------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
9 #include "llvm/ExecutionEngine/JITLink/JITLink.h"
10 
11 #include "llvm/ADT/StringExtras.h"
12 #include "llvm/BinaryFormat/Magic.h"
13 #include "llvm/ExecutionEngine/JITLink/COFF.h"
14 #include "llvm/ExecutionEngine/JITLink/ELF.h"
15 #include "llvm/ExecutionEngine/JITLink/MachO.h"
16 #include "llvm/ExecutionEngine/JITLink/aarch64.h"
17 #include "llvm/ExecutionEngine/JITLink/i386.h"
18 #include "llvm/ExecutionEngine/JITLink/loongarch.h"
19 #include "llvm/ExecutionEngine/JITLink/x86_64.h"
20 #include "llvm/Support/raw_ostream.h"
21 
22 using namespace llvm;
23 using namespace llvm::object;
24 
25 #define DEBUG_TYPE "jitlink"
26 
27 namespace {
28 
29 enum JITLinkErrorCode { GenericJITLinkError = 1 };
30 
31 // FIXME: This class is only here to support the transition to llvm::Error. It
32 // will be removed once this transition is complete. Clients should prefer to
33 // deal with the Error value directly, rather than converting to error_code.
34 class JITLinkerErrorCategory : public std::error_category {
35 public:
36   const char *name() const noexcept override { return "runtimedyld"; }
37 
38   std::string message(int Condition) const override {
39     switch (static_cast<JITLinkErrorCode>(Condition)) {
40     case GenericJITLinkError:
41       return "Generic JITLink error";
42     }
43     llvm_unreachable("Unrecognized JITLinkErrorCode");
44   }
45 };
46 
47 } // namespace
48 
49 namespace llvm {
50 namespace jitlink {
51 
52 char JITLinkError::ID = 0;
53 
54 void JITLinkError::log(raw_ostream &OS) const { OS << ErrMsg; }
55 
56 std::error_code JITLinkError::convertToErrorCode() const {
57   static JITLinkerErrorCategory TheJITLinkerErrorCategory;
58   return std::error_code(GenericJITLinkError, TheJITLinkerErrorCategory);
59 }
60 
61 const char *getGenericEdgeKindName(Edge::Kind K) {
62   switch (K) {
63   case Edge::Invalid:
64     return "INVALID RELOCATION";
65   case Edge::KeepAlive:
66     return "Keep-Alive";
67   default:
68     return "<Unrecognized edge kind>";
69   }
70 }
71 
72 const char *getLinkageName(Linkage L) {
73   switch (L) {
74   case Linkage::Strong:
75     return "strong";
76   case Linkage::Weak:
77     return "weak";
78   }
79   llvm_unreachable("Unrecognized llvm.jitlink.Linkage enum");
80 }
81 
82 const char *getScopeName(Scope S) {
83   switch (S) {
84   case Scope::Default:
85     return "default";
86   case Scope::Hidden:
87     return "hidden";
88   case Scope::SideEffectsOnly:
89     return "side-effects-only";
90   case Scope::Local:
91     return "local";
92   }
93   llvm_unreachable("Unrecognized llvm.jitlink.Scope enum");
94 }
95 
96 bool isCStringBlock(Block &B) {
97   if (B.getSize() == 0) // Empty blocks are not valid C-strings.
98     return false;
99 
100   // Zero-fill blocks of size one are valid empty strings.
101   if (B.isZeroFill())
102     return B.getSize() == 1;
103 
104   for (size_t I = 0; I != B.getSize() - 1; ++I)
105     if (B.getContent()[I] == '\0')
106       return false;
107 
108   return B.getContent()[B.getSize() - 1] == '\0';
109 }
110 
111 raw_ostream &operator<<(raw_ostream &OS, const Block &B) {
112   return OS << B.getAddress() << " -- " << (B.getAddress() + B.getSize())
113             << ": "
114             << "size = " << formatv("{0:x8}", B.getSize()) << ", "
115             << (B.isZeroFill() ? "zero-fill" : "content")
116             << ", align = " << B.getAlignment()
117             << ", align-ofs = " << B.getAlignmentOffset()
118             << ", section = " << B.getSection().getName();
119 }
120 
121 raw_ostream &operator<<(raw_ostream &OS, const Symbol &Sym) {
122   OS << Sym.getAddress() << " (" << (Sym.isDefined() ? "block" : "addressable")
123      << " + " << formatv("{0:x8}", Sym.getOffset())
124      << "): size: " << formatv("{0:x8}", Sym.getSize())
125      << ", linkage: " << formatv("{0:6}", getLinkageName(Sym.getLinkage()))
126      << ", scope: " << formatv("{0:8}", getScopeName(Sym.getScope())) << ", "
127      << (Sym.isLive() ? "live" : "dead") << "  -   "
128      << (Sym.hasName() ? *Sym.getName() : "<anonymous symbol>");
129   return OS;
130 }
131 
132 void printEdge(raw_ostream &OS, const Block &B, const Edge &E,
133                StringRef EdgeKindName) {
134   OS << "edge@" << B.getAddress() + E.getOffset() << ": " << B.getAddress()
135      << " + " << formatv("{0:x}", E.getOffset()) << " -- " << EdgeKindName
136      << " -> ";
137 
138   auto &TargetSym = E.getTarget();
139   if (TargetSym.hasName())
140     OS << TargetSym.getName();
141   else {
142     auto &TargetBlock = TargetSym.getBlock();
143     auto &TargetSec = TargetBlock.getSection();
144     orc::ExecutorAddr SecAddress(~uint64_t(0));
145     for (auto *B : TargetSec.blocks())
146       if (B->getAddress() < SecAddress)
147         SecAddress = B->getAddress();
148 
149     orc::ExecutorAddrDiff SecDelta = TargetSym.getAddress() - SecAddress;
150     OS << TargetSym.getAddress() << " (section " << TargetSec.getName();
151     if (SecDelta)
152       OS << " + " << formatv("{0:x}", SecDelta);
153     OS << " / block " << TargetBlock.getAddress();
154     if (TargetSym.getOffset())
155       OS << " + " << formatv("{0:x}", TargetSym.getOffset());
156     OS << ")";
157   }
158 
159   if (E.getAddend() != 0)
160     OS << " + " << E.getAddend();
161 }
162 
163 Section::~Section() {
164   for (auto *Sym : Symbols)
165     Sym->~Symbol();
166   for (auto *B : Blocks)
167     B->~Block();
168 }
169 
170 LinkGraph::~LinkGraph() {
171   for (auto *Sym : AbsoluteSymbols) {
172     Sym->~Symbol();
173   }
174   for (auto *Sym : external_symbols()) {
175     Sym->~Symbol();
176   }
177   ExternalSymbols.clear();
178 }
179 
180 std::vector<Block *> LinkGraph::splitBlockImpl(std::vector<Block *> Blocks,
181                                                SplitBlockCache *Cache) {
182   assert(!Blocks.empty() && "Blocks must at least contain the original block");
183 
184   // Fix up content of all blocks.
185   ArrayRef<char> Content = Blocks.front()->getContent();
186   for (size_t I = 0; I != Blocks.size() - 1; ++I) {
187     Blocks[I]->setContent(
188         Content.slice(Blocks[I]->getAddress() - Blocks[0]->getAddress(),
189                       Blocks[I + 1]->getAddress() - Blocks[I]->getAddress()));
190   }
191   Blocks.back()->setContent(
192       Content.slice(Blocks.back()->getAddress() - Blocks[0]->getAddress()));
193   bool IsMutable = Blocks[0]->ContentMutable;
194   for (auto *B : Blocks)
195     B->ContentMutable = IsMutable;
196 
197   // Transfer symbols.
198   {
199     SplitBlockCache LocalBlockSymbolsCache;
200     if (!Cache)
201       Cache = &LocalBlockSymbolsCache;
202 
203     // Build cache if required.
204     if (*Cache == std::nullopt) {
205       *Cache = SplitBlockCache::value_type();
206 
207       for (auto *Sym : Blocks[0]->getSection().symbols())
208         if (&Sym->getBlock() == Blocks[0])
209           (*Cache)->push_back(Sym);
210       llvm::sort(**Cache, [](const Symbol *LHS, const Symbol *RHS) {
211         return LHS->getAddress() > RHS->getAddress();
212       });
213     }
214 
215     auto TransferSymbol = [](Symbol &Sym, Block &B) {
216       Sym.setOffset(Sym.getAddress() - B.getAddress());
217       Sym.setBlock(B);
218       if (Sym.getSize() > B.getSize())
219         Sym.setSize(B.getSize() - Sym.getOffset());
220     };
221 
222     // Transfer symbols to all blocks except the last one.
223     for (size_t I = 0; I != Blocks.size() - 1; ++I) {
224       if ((*Cache)->empty())
225         break;
226       while (!(*Cache)->empty() &&
227              (*Cache)->back()->getAddress() < Blocks[I + 1]->getAddress()) {
228         TransferSymbol(*(*Cache)->back(), *Blocks[I]);
229         (*Cache)->pop_back();
230       }
231     }
232     // Transfer symbols to the last block, checking that all are in-range.
233     while (!(*Cache)->empty()) {
234       auto &Sym = *(*Cache)->back();
235       (*Cache)->pop_back();
236       assert(Sym.getAddress() >= Blocks.back()->getAddress() &&
237              "Symbol address preceeds block");
238       assert(Sym.getAddress() <= Blocks.back()->getRange().End &&
239              "Symbol address starts past end of block");
240       TransferSymbol(Sym, *Blocks.back());
241     }
242   }
243 
244   // Transfer edges.
245   auto &Edges = Blocks[0]->Edges;
246   llvm::sort(Edges, [](const Edge &LHS, const Edge &RHS) {
247     return LHS.getOffset() < RHS.getOffset();
248   });
249 
250   for (size_t I = Blocks.size() - 1; I != 0; --I) {
251 
252     // If all edges have been transferred then bail out.
253     if (Edges.empty())
254       break;
255 
256     Edge::OffsetT Delta = Blocks[I]->getAddress() - Blocks[0]->getAddress();
257 
258     // If no edges to move for this block then move to the next one.
259     if (Edges.back().getOffset() < Delta)
260       continue;
261 
262     size_t EI = Edges.size() - 1;
263     while (EI != 0 && Edges[EI - 1].getOffset() >= Delta)
264       --EI;
265 
266     for (size_t J = EI; J != Edges.size(); ++J) {
267       Blocks[I]->Edges.push_back(std::move(Edges[J]));
268       Blocks[I]->Edges.back().setOffset(Blocks[I]->Edges.back().getOffset() -
269                                         Delta);
270     }
271 
272     while (Edges.size() > EI)
273       Edges.pop_back();
274   }
275 
276   return Blocks;
277 }
278 
279 void LinkGraph::dump(raw_ostream &OS) {
280   DenseMap<Block *, std::vector<Symbol *>> BlockSymbols;
281 
282   // Map from blocks to the symbols pointing at them.
283   for (auto *Sym : defined_symbols())
284     BlockSymbols[&Sym->getBlock()].push_back(Sym);
285 
286   // For each block, sort its symbols by something approximating
287   // relevance.
288   for (auto &KV : BlockSymbols)
289     llvm::sort(KV.second, [](const Symbol *LHS, const Symbol *RHS) {
290       if (LHS->getOffset() != RHS->getOffset())
291         return LHS->getOffset() < RHS->getOffset();
292       if (LHS->getLinkage() != RHS->getLinkage())
293         return LHS->getLinkage() < RHS->getLinkage();
294       if (LHS->getScope() != RHS->getScope())
295         return LHS->getScope() < RHS->getScope();
296       if (LHS->hasName()) {
297         if (!RHS->hasName())
298           return true;
299         return LHS->getName() < RHS->getName();
300       }
301       return false;
302     });
303 
304   std::vector<Section *> SortedSections;
305   for (auto &Sec : sections())
306     SortedSections.push_back(&Sec);
307   llvm::sort(SortedSections, [](const Section *LHS, const Section *RHS) {
308     return LHS->getName() < RHS->getName();
309   });
310 
311   for (auto *Sec : SortedSections) {
312     OS << "section " << Sec->getName() << ":\n\n";
313 
314     std::vector<Block *> SortedBlocks;
315     llvm::copy(Sec->blocks(), std::back_inserter(SortedBlocks));
316     llvm::sort(SortedBlocks, [](const Block *LHS, const Block *RHS) {
317       return LHS->getAddress() < RHS->getAddress();
318     });
319 
320     for (auto *B : SortedBlocks) {
321       OS << "  block " << B->getAddress()
322          << " size = " << formatv("{0:x8}", B->getSize())
323          << ", align = " << B->getAlignment()
324          << ", alignment-offset = " << B->getAlignmentOffset();
325       if (B->isZeroFill())
326         OS << ", zero-fill";
327       OS << "\n";
328 
329       auto BlockSymsI = BlockSymbols.find(B);
330       if (BlockSymsI != BlockSymbols.end()) {
331         OS << "    symbols:\n";
332         auto &Syms = BlockSymsI->second;
333         for (auto *Sym : Syms)
334           OS << "      " << *Sym << "\n";
335       } else
336         OS << "    no symbols\n";
337 
338       if (!B->edges_empty()) {
339         OS << "    edges:\n";
340         std::vector<Edge> SortedEdges;
341         llvm::copy(B->edges(), std::back_inserter(SortedEdges));
342         llvm::sort(SortedEdges, [](const Edge &LHS, const Edge &RHS) {
343           return LHS.getOffset() < RHS.getOffset();
344         });
345         for (auto &E : SortedEdges) {
346           OS << "      " << B->getFixupAddress(E) << " (block + "
347              << formatv("{0:x8}", E.getOffset()) << "), addend = ";
348           if (E.getAddend() >= 0)
349             OS << formatv("+{0:x8}", E.getAddend());
350           else
351             OS << formatv("-{0:x8}", -E.getAddend());
352           OS << ", kind = " << getEdgeKindName(E.getKind()) << ", target = ";
353           if (E.getTarget().hasName())
354             OS << E.getTarget().getName();
355           else
356             OS << "addressable@"
357                << formatv("{0:x16}", E.getTarget().getAddress()) << "+"
358                << formatv("{0:x8}", E.getTarget().getOffset());
359           OS << "\n";
360         }
361       } else
362         OS << "    no edges\n";
363       OS << "\n";
364     }
365   }
366 
367   OS << "Absolute symbols:\n";
368   if (!absolute_symbols().empty()) {
369     for (auto *Sym : absolute_symbols())
370       OS << "  " << Sym->getAddress() << ": " << *Sym << "\n";
371   } else
372     OS << "  none\n";
373 
374   OS << "\nExternal symbols:\n";
375   if (!external_symbols().empty()) {
376     for (auto *Sym : external_symbols())
377       OS << "  " << Sym->getAddress() << ": " << *Sym
378          << (Sym->isWeaklyReferenced() ? " (weakly referenced)" : "") << "\n";
379   } else
380     OS << "  none\n";
381 }
382 
383 raw_ostream &operator<<(raw_ostream &OS, const SymbolLookupFlags &LF) {
384   switch (LF) {
385   case SymbolLookupFlags::RequiredSymbol:
386     return OS << "RequiredSymbol";
387   case SymbolLookupFlags::WeaklyReferencedSymbol:
388     return OS << "WeaklyReferencedSymbol";
389   }
390   llvm_unreachable("Unrecognized lookup flags");
391 }
392 
393 void JITLinkAsyncLookupContinuation::anchor() {}
394 
395 JITLinkContext::~JITLinkContext() = default;
396 
397 bool JITLinkContext::shouldAddDefaultTargetPasses(const Triple &TT) const {
398   return true;
399 }
400 
401 LinkGraphPassFunction JITLinkContext::getMarkLivePass(const Triple &TT) const {
402   return LinkGraphPassFunction();
403 }
404 
405 Error JITLinkContext::modifyPassConfig(LinkGraph &G,
406                                        PassConfiguration &Config) {
407   return Error::success();
408 }
409 
410 Error markAllSymbolsLive(LinkGraph &G) {
411   for (auto *Sym : G.defined_symbols())
412     Sym->setLive(true);
413   return Error::success();
414 }
415 
416 Error makeTargetOutOfRangeError(const LinkGraph &G, const Block &B,
417                                 const Edge &E) {
418   std::string ErrMsg;
419   {
420     raw_string_ostream ErrStream(ErrMsg);
421     Section &Sec = B.getSection();
422     ErrStream << "In graph " << G.getName() << ", section " << Sec.getName()
423               << ": relocation target ";
424     if (E.getTarget().hasName()) {
425       ErrStream << "\"" << E.getTarget().getName() << "\"";
426     } else
427       ErrStream << E.getTarget().getBlock().getSection().getName() << " + "
428                 << formatv("{0:x}", E.getOffset());
429     ErrStream << " at address " << formatv("{0:x}", E.getTarget().getAddress())
430               << " is out of range of " << G.getEdgeKindName(E.getKind())
431               << " fixup at " << formatv("{0:x}", B.getFixupAddress(E)) << " (";
432 
433     Symbol *BestSymbolForBlock = nullptr;
434     for (auto *Sym : Sec.symbols())
435       if (&Sym->getBlock() == &B && Sym->hasName() && Sym->getOffset() == 0 &&
436           (!BestSymbolForBlock ||
437            Sym->getScope() < BestSymbolForBlock->getScope() ||
438            Sym->getLinkage() < BestSymbolForBlock->getLinkage()))
439         BestSymbolForBlock = Sym;
440 
441     if (BestSymbolForBlock)
442       ErrStream << BestSymbolForBlock->getName() << ", ";
443     else
444       ErrStream << "<anonymous block> @ ";
445 
446     ErrStream << formatv("{0:x}", B.getAddress()) << " + "
447               << formatv("{0:x}", E.getOffset()) << ")";
448   }
449   return make_error<JITLinkError>(std::move(ErrMsg));
450 }
451 
452 Error makeAlignmentError(llvm::orc::ExecutorAddr Loc, uint64_t Value, int N,
453                          const Edge &E) {
454   return make_error<JITLinkError>("0x" + llvm::utohexstr(Loc.getValue()) +
455                                   " improper alignment for relocation " +
456                                   formatv("{0:d}", E.getKind()) + ": 0x" +
457                                   llvm::utohexstr(Value) +
458                                   " is not aligned to " + Twine(N) + " bytes");
459 }
460 
461 AnonymousPointerCreator getAnonymousPointerCreator(const Triple &TT) {
462   switch (TT.getArch()) {
463   case Triple::aarch64:
464     return aarch64::createAnonymousPointer;
465   case Triple::x86_64:
466     return x86_64::createAnonymousPointer;
467   case Triple::x86:
468     return i386::createAnonymousPointer;
469   case Triple::loongarch32:
470   case Triple::loongarch64:
471     return loongarch::createAnonymousPointer;
472   default:
473     return nullptr;
474   }
475 }
476 
477 PointerJumpStubCreator getPointerJumpStubCreator(const Triple &TT) {
478   switch (TT.getArch()) {
479   case Triple::aarch64:
480     return aarch64::createAnonymousPointerJumpStub;
481   case Triple::x86_64:
482     return x86_64::createAnonymousPointerJumpStub;
483   case Triple::x86:
484     return i386::createAnonymousPointerJumpStub;
485   case Triple::loongarch32:
486   case Triple::loongarch64:
487     return loongarch::createAnonymousPointerJumpStub;
488   default:
489     return nullptr;
490   }
491 }
492 
493 Expected<std::unique_ptr<LinkGraph>>
494 createLinkGraphFromObject(MemoryBufferRef ObjectBuffer,
495                           std::shared_ptr<orc::SymbolStringPool> SSP) {
496   auto Magic = identify_magic(ObjectBuffer.getBuffer());
497   switch (Magic) {
498   case file_magic::macho_object:
499     return createLinkGraphFromMachOObject(ObjectBuffer, std::move(SSP));
500   case file_magic::elf_relocatable:
501     return createLinkGraphFromELFObject(ObjectBuffer, std::move(SSP));
502   case file_magic::coff_object:
503     return createLinkGraphFromCOFFObject(ObjectBuffer, std::move(SSP));
504   default:
505     return make_error<JITLinkError>("Unsupported file format");
506   };
507 }
508 
509 std::unique_ptr<LinkGraph>
510 absoluteSymbolsLinkGraph(Triple TT, std::shared_ptr<orc::SymbolStringPool> SSP,
511                          orc::SymbolMap Symbols) {
512   static std::atomic<uint64_t> Counter = {0};
513   auto Index = Counter.fetch_add(1, std::memory_order_relaxed);
514   auto G = std::make_unique<LinkGraph>(
515       "<Absolute Symbols " + std::to_string(Index) + ">", std::move(SSP),
516       std::move(TT), SubtargetFeatures(), getGenericEdgeKindName);
517   for (auto &[Name, Def] : Symbols) {
518     auto &Sym =
519         G->addAbsoluteSymbol(*Name, Def.getAddress(), /*Size=*/0,
520                              Linkage::Strong, Scope::Default, /*IsLive=*/true);
521     Sym.setCallable(Def.getFlags().isCallable());
522   }
523 
524   return G;
525 }
526 
527 void link(std::unique_ptr<LinkGraph> G, std::unique_ptr<JITLinkContext> Ctx) {
528   switch (G->getTargetTriple().getObjectFormat()) {
529   case Triple::MachO:
530     return link_MachO(std::move(G), std::move(Ctx));
531   case Triple::ELF:
532     return link_ELF(std::move(G), std::move(Ctx));
533   case Triple::COFF:
534     return link_COFF(std::move(G), std::move(Ctx));
535   default:
536     Ctx->notifyFailed(make_error<JITLinkError>("Unsupported object format"));
537   };
538 }
539 
540 } // end namespace jitlink
541 } // end namespace llvm
542