xref: /llvm-project/llvm/lib/ExecutionEngine/JITLink/JITLink.cpp (revision 521c99627690e09ba25383c83232f94ff457f00c) 
1 //===------------- JITLink.cpp - Core Run-time JIT linker APIs ------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
9 #include "llvm/ExecutionEngine/JITLink/JITLink.h"
10 
11 #include "llvm/ADT/StringExtras.h"
12 #include "llvm/BinaryFormat/Magic.h"
13 #include "llvm/ExecutionEngine/JITLink/COFF.h"
14 #include "llvm/ExecutionEngine/JITLink/ELF.h"
15 #include "llvm/ExecutionEngine/JITLink/MachO.h"
16 #include "llvm/ExecutionEngine/JITLink/aarch64.h"
17 #include "llvm/ExecutionEngine/JITLink/i386.h"
18 #include "llvm/ExecutionEngine/JITLink/loongarch.h"
19 #include "llvm/ExecutionEngine/JITLink/x86_64.h"
20 #include "llvm/Support/Format.h"
21 #include "llvm/Support/MemoryBuffer.h"
22 #include "llvm/Support/raw_ostream.h"
23 
24 using namespace llvm;
25 using namespace llvm::object;
26 
27 #define DEBUG_TYPE "jitlink"
28 
29 namespace {
30 
31 enum JITLinkErrorCode { GenericJITLinkError = 1 };
32 
33 // FIXME: This class is only here to support the transition to llvm::Error. It
34 // will be removed once this transition is complete. Clients should prefer to
35 // deal with the Error value directly, rather than converting to error_code.
36 class JITLinkerErrorCategory : public std::error_category {
37 public:
38   const char *name() const noexcept override { return "runtimedyld"; }
39 
40   std::string message(int Condition) const override {
41     switch (static_cast<JITLinkErrorCode>(Condition)) {
42     case GenericJITLinkError:
43       return "Generic JITLink error";
44     }
45     llvm_unreachable("Unrecognized JITLinkErrorCode");
46   }
47 };
48 
49 } // namespace
50 
51 namespace llvm {
52 namespace jitlink {
53 
54 char JITLinkError::ID = 0;
55 
56 void JITLinkError::log(raw_ostream &OS) const { OS << ErrMsg; }
57 
58 std::error_code JITLinkError::convertToErrorCode() const {
59   static JITLinkerErrorCategory TheJITLinkerErrorCategory;
60   return std::error_code(GenericJITLinkError, TheJITLinkerErrorCategory);
61 }
62 
63 const char *getGenericEdgeKindName(Edge::Kind K) {
64   switch (K) {
65   case Edge::Invalid:
66     return "INVALID RELOCATION";
67   case Edge::KeepAlive:
68     return "Keep-Alive";
69   default:
70     return "<Unrecognized edge kind>";
71   }
72 }
73 
74 const char *getLinkageName(Linkage L) {
75   switch (L) {
76   case Linkage::Strong:
77     return "strong";
78   case Linkage::Weak:
79     return "weak";
80   }
81   llvm_unreachable("Unrecognized llvm.jitlink.Linkage enum");
82 }
83 
84 const char *getScopeName(Scope S) {
85   switch (S) {
86   case Scope::Default:
87     return "default";
88   case Scope::Hidden:
89     return "hidden";
90   case Scope::Local:
91     return "local";
92   }
93   llvm_unreachable("Unrecognized llvm.jitlink.Scope enum");
94 }
95 
96 bool isCStringBlock(Block &B) {
97   if (B.getSize() == 0) // Empty blocks are not valid C-strings.
98     return false;
99 
100   // Zero-fill blocks of size one are valid empty strings.
101   if (B.isZeroFill())
102     return B.getSize() == 1;
103 
104   for (size_t I = 0; I != B.getSize() - 1; ++I)
105     if (B.getContent()[I] == '\0')
106       return false;
107 
108   return B.getContent()[B.getSize() - 1] == '\0';
109 }
110 
111 raw_ostream &operator<<(raw_ostream &OS, const Block &B) {
112   return OS << B.getAddress() << " -- " << (B.getAddress() + B.getSize())
113             << ": "
114             << "size = " << formatv("{0:x8}", B.getSize()) << ", "
115             << (B.isZeroFill() ? "zero-fill" : "content")
116             << ", align = " << B.getAlignment()
117             << ", align-ofs = " << B.getAlignmentOffset()
118             << ", section = " << B.getSection().getName();
119 }
120 
121 raw_ostream &operator<<(raw_ostream &OS, const Symbol &Sym) {
122   OS << Sym.getAddress() << " (" << (Sym.isDefined() ? "block" : "addressable")
123      << " + " << formatv("{0:x8}", Sym.getOffset())
124      << "): size: " << formatv("{0:x8}", Sym.getSize())
125      << ", linkage: " << formatv("{0:6}", getLinkageName(Sym.getLinkage()))
126      << ", scope: " << formatv("{0:8}", getScopeName(Sym.getScope())) << ", "
127      << (Sym.isLive() ? "live" : "dead") << "  -   "
128      << (Sym.hasName() ? Sym.getName() : "<anonymous symbol>");
129   return OS;
130 }
131 
132 void printEdge(raw_ostream &OS, const Block &B, const Edge &E,
133                StringRef EdgeKindName) {
134   OS << "edge@" << B.getAddress() + E.getOffset() << ": " << B.getAddress()
135      << " + " << formatv("{0:x}", E.getOffset()) << " -- " << EdgeKindName
136      << " -> ";
137 
138   auto &TargetSym = E.getTarget();
139   if (TargetSym.hasName())
140     OS << TargetSym.getName();
141   else {
142     auto &TargetBlock = TargetSym.getBlock();
143     auto &TargetSec = TargetBlock.getSection();
144     orc::ExecutorAddr SecAddress(~uint64_t(0));
145     for (auto *B : TargetSec.blocks())
146       if (B->getAddress() < SecAddress)
147         SecAddress = B->getAddress();
148 
149     orc::ExecutorAddrDiff SecDelta = TargetSym.getAddress() - SecAddress;
150     OS << TargetSym.getAddress() << " (section " << TargetSec.getName();
151     if (SecDelta)
152       OS << " + " << formatv("{0:x}", SecDelta);
153     OS << " / block " << TargetBlock.getAddress();
154     if (TargetSym.getOffset())
155       OS << " + " << formatv("{0:x}", TargetSym.getOffset());
156     OS << ")";
157   }
158 
159   if (E.getAddend() != 0)
160     OS << " + " << E.getAddend();
161 }
162 
163 Section::~Section() {
164   for (auto *Sym : Symbols)
165     Sym->~Symbol();
166   for (auto *B : Blocks)
167     B->~Block();
168 }
169 
170 std::vector<Block *> LinkGraph::splitBlockImpl(std::vector<Block *> Blocks,
171                                                SplitBlockCache *Cache) {
172   assert(!Blocks.empty() && "Blocks must at least contain the original block");
173 
174   // Fix up content of all blocks.
175   ArrayRef<char> Content = Blocks.front()->getContent();
176   for (size_t I = 0; I != Blocks.size() - 1; ++I) {
177     Blocks[I]->setContent(
178         Content.slice(Blocks[I]->getAddress() - Blocks[0]->getAddress(),
179                       Blocks[I + 1]->getAddress() - Blocks[I]->getAddress()));
180   }
181   Blocks.back()->setContent(
182       Content.slice(Blocks.back()->getAddress() - Blocks[0]->getAddress()));
183   bool IsMutable = Blocks[0]->ContentMutable;
184   for (auto *B : Blocks)
185     B->ContentMutable = IsMutable;
186 
187   // Transfer symbols.
188   {
189     SplitBlockCache LocalBlockSymbolsCache;
190     if (!Cache)
191       Cache = &LocalBlockSymbolsCache;
192 
193     // Build cache if required.
194     if (*Cache == std::nullopt) {
195       *Cache = SplitBlockCache::value_type();
196 
197       for (auto *Sym : Blocks[0]->getSection().symbols())
198         if (&Sym->getBlock() == Blocks[0])
199           (*Cache)->push_back(Sym);
200       llvm::sort(**Cache, [](const Symbol *LHS, const Symbol *RHS) {
201         return LHS->getAddress() > RHS->getAddress();
202       });
203     }
204 
205     auto TransferSymbol = [](Symbol &Sym, Block &B) {
206       Sym.setOffset(Sym.getAddress() - B.getAddress());
207       Sym.setBlock(B);
208       if (Sym.getSize() > B.getSize())
209         Sym.setSize(B.getSize() - Sym.getOffset());
210     };
211 
212     // Transfer symbols to all blocks except the last one.
213     for (size_t I = 0; I != Blocks.size() - 1; ++I) {
214       if ((*Cache)->empty())
215         break;
216       while (!(*Cache)->empty() &&
217              (*Cache)->back()->getAddress() < Blocks[I + 1]->getAddress()) {
218         TransferSymbol(*(*Cache)->back(), *Blocks[I]);
219         (*Cache)->pop_back();
220       }
221     }
222     // Transfer symbols to the last block, checking that all are in-range.
223     while (!(*Cache)->empty()) {
224       auto &Sym = *(*Cache)->back();
225       (*Cache)->pop_back();
226       assert(Sym.getAddress() >= Blocks.back()->getAddress() &&
227              "Symbol address preceeds block");
228       assert(Sym.getAddress() <= Blocks.back()->getRange().End &&
229              "Symbol address starts past end of block");
230       TransferSymbol(Sym, *Blocks.back());
231     }
232   }
233 
234   // Transfer edges.
235   auto &Edges = Blocks[0]->Edges;
236   llvm::sort(Edges, [](const Edge &LHS, const Edge &RHS) {
237     return LHS.getOffset() < RHS.getOffset();
238   });
239 
240   for (size_t I = Blocks.size() - 1; I != 0; --I) {
241 
242     // If all edges have been transferred then bail out.
243     if (Edges.empty())
244       break;
245 
246     Edge::OffsetT Delta = Blocks[I]->getAddress() - Blocks[0]->getAddress();
247 
248     // If no edges to move for this block then move to the next one.
249     if (Edges.back().getOffset() < Delta)
250       continue;
251 
252     size_t EI = Edges.size() - 1;
253     while (EI != 0 && Edges[EI - 1].getOffset() >= Delta)
254       --EI;
255 
256     for (size_t J = EI; J != Edges.size(); ++J) {
257       Blocks[I]->Edges.push_back(std::move(Edges[J]));
258       Blocks[I]->Edges.back().setOffset(Blocks[I]->Edges.back().getOffset() -
259                                         Delta);
260     }
261 
262     while (Edges.size() > EI)
263       Edges.pop_back();
264   }
265 
266   return Blocks;
267 }
268 
269 void LinkGraph::dump(raw_ostream &OS) {
270   DenseMap<Block *, std::vector<Symbol *>> BlockSymbols;
271 
272   // Map from blocks to the symbols pointing at them.
273   for (auto *Sym : defined_symbols())
274     BlockSymbols[&Sym->getBlock()].push_back(Sym);
275 
276   // For each block, sort its symbols by something approximating
277   // relevance.
278   for (auto &KV : BlockSymbols)
279     llvm::sort(KV.second, [](const Symbol *LHS, const Symbol *RHS) {
280       if (LHS->getOffset() != RHS->getOffset())
281         return LHS->getOffset() < RHS->getOffset();
282       if (LHS->getLinkage() != RHS->getLinkage())
283         return LHS->getLinkage() < RHS->getLinkage();
284       if (LHS->getScope() != RHS->getScope())
285         return LHS->getScope() < RHS->getScope();
286       if (LHS->hasName()) {
287         if (!RHS->hasName())
288           return true;
289         return LHS->getName() < RHS->getName();
290       }
291       return false;
292     });
293 
294   std::vector<Section *> SortedSections;
295   for (auto &Sec : sections())
296     SortedSections.push_back(&Sec);
297   llvm::sort(SortedSections, [](const Section *LHS, const Section *RHS) {
298     return LHS->getName() < RHS->getName();
299   });
300 
301   for (auto *Sec : SortedSections) {
302     OS << "section " << Sec->getName() << ":\n\n";
303 
304     std::vector<Block *> SortedBlocks;
305     llvm::copy(Sec->blocks(), std::back_inserter(SortedBlocks));
306     llvm::sort(SortedBlocks, [](const Block *LHS, const Block *RHS) {
307       return LHS->getAddress() < RHS->getAddress();
308     });
309 
310     for (auto *B : SortedBlocks) {
311       OS << "  block " << B->getAddress()
312          << " size = " << formatv("{0:x8}", B->getSize())
313          << ", align = " << B->getAlignment()
314          << ", alignment-offset = " << B->getAlignmentOffset();
315       if (B->isZeroFill())
316         OS << ", zero-fill";
317       OS << "\n";
318 
319       auto BlockSymsI = BlockSymbols.find(B);
320       if (BlockSymsI != BlockSymbols.end()) {
321         OS << "    symbols:\n";
322         auto &Syms = BlockSymsI->second;
323         for (auto *Sym : Syms)
324           OS << "      " << *Sym << "\n";
325       } else
326         OS << "    no symbols\n";
327 
328       if (!B->edges_empty()) {
329         OS << "    edges:\n";
330         std::vector<Edge> SortedEdges;
331         llvm::copy(B->edges(), std::back_inserter(SortedEdges));
332         llvm::sort(SortedEdges, [](const Edge &LHS, const Edge &RHS) {
333           return LHS.getOffset() < RHS.getOffset();
334         });
335         for (auto &E : SortedEdges) {
336           OS << "      " << B->getFixupAddress(E) << " (block + "
337              << formatv("{0:x8}", E.getOffset()) << "), addend = ";
338           if (E.getAddend() >= 0)
339             OS << formatv("+{0:x8}", E.getAddend());
340           else
341             OS << formatv("-{0:x8}", -E.getAddend());
342           OS << ", kind = " << getEdgeKindName(E.getKind()) << ", target = ";
343           if (E.getTarget().hasName())
344             OS << E.getTarget().getName();
345           else
346             OS << "addressable@"
347                << formatv("{0:x16}", E.getTarget().getAddress()) << "+"
348                << formatv("{0:x8}", E.getTarget().getOffset());
349           OS << "\n";
350         }
351       } else
352         OS << "    no edges\n";
353       OS << "\n";
354     }
355   }
356 
357   OS << "Absolute symbols:\n";
358   if (!absolute_symbols().empty()) {
359     for (auto *Sym : absolute_symbols())
360       OS << "  " << Sym->getAddress() << ": " << *Sym << "\n";
361   } else
362     OS << "  none\n";
363 
364   OS << "\nExternal symbols:\n";
365   if (!external_symbols().empty()) {
366     for (auto *Sym : external_symbols())
367       OS << "  " << Sym->getAddress() << ": " << *Sym
368          << (Sym->isWeaklyReferenced() ? " (weakly referenced)" : "") << "\n";
369   } else
370     OS << "  none\n";
371 }
372 
373 raw_ostream &operator<<(raw_ostream &OS, const SymbolLookupFlags &LF) {
374   switch (LF) {
375   case SymbolLookupFlags::RequiredSymbol:
376     return OS << "RequiredSymbol";
377   case SymbolLookupFlags::WeaklyReferencedSymbol:
378     return OS << "WeaklyReferencedSymbol";
379   }
380   llvm_unreachable("Unrecognized lookup flags");
381 }
382 
383 void JITLinkAsyncLookupContinuation::anchor() {}
384 
385 JITLinkContext::~JITLinkContext() = default;
386 
387 bool JITLinkContext::shouldAddDefaultTargetPasses(const Triple &TT) const {
388   return true;
389 }
390 
391 LinkGraphPassFunction JITLinkContext::getMarkLivePass(const Triple &TT) const {
392   return LinkGraphPassFunction();
393 }
394 
395 Error JITLinkContext::modifyPassConfig(LinkGraph &G,
396                                        PassConfiguration &Config) {
397   return Error::success();
398 }
399 
400 Error markAllSymbolsLive(LinkGraph &G) {
401   for (auto *Sym : G.defined_symbols())
402     Sym->setLive(true);
403   return Error::success();
404 }
405 
406 Error makeTargetOutOfRangeError(const LinkGraph &G, const Block &B,
407                                 const Edge &E) {
408   std::string ErrMsg;
409   {
410     raw_string_ostream ErrStream(ErrMsg);
411     Section &Sec = B.getSection();
412     ErrStream << "In graph " << G.getName() << ", section " << Sec.getName()
413               << ": relocation target ";
414     if (E.getTarget().hasName()) {
415       ErrStream << "\"" << E.getTarget().getName() << "\"";
416     } else
417       ErrStream << E.getTarget().getBlock().getSection().getName() << " + "
418                 << formatv("{0:x}", E.getOffset());
419     ErrStream << " at address " << formatv("{0:x}", E.getTarget().getAddress())
420               << " is out of range of " << G.getEdgeKindName(E.getKind())
421               << " fixup at " << formatv("{0:x}", B.getFixupAddress(E)) << " (";
422 
423     Symbol *BestSymbolForBlock = nullptr;
424     for (auto *Sym : Sec.symbols())
425       if (&Sym->getBlock() == &B && Sym->hasName() && Sym->getOffset() == 0 &&
426           (!BestSymbolForBlock ||
427            Sym->getScope() < BestSymbolForBlock->getScope() ||
428            Sym->getLinkage() < BestSymbolForBlock->getLinkage()))
429         BestSymbolForBlock = Sym;
430 
431     if (BestSymbolForBlock)
432       ErrStream << BestSymbolForBlock->getName() << ", ";
433     else
434       ErrStream << "<anonymous block> @ ";
435 
436     ErrStream << formatv("{0:x}", B.getAddress()) << " + "
437               << formatv("{0:x}", E.getOffset()) << ")";
438   }
439   return make_error<JITLinkError>(std::move(ErrMsg));
440 }
441 
442 Error makeAlignmentError(llvm::orc::ExecutorAddr Loc, uint64_t Value, int N,
443                          const Edge &E) {
444   return make_error<JITLinkError>("0x" + llvm::utohexstr(Loc.getValue()) +
445                                   " improper alignment for relocation " +
446                                   formatv("{0:d}", E.getKind()) + ": 0x" +
447                                   llvm::utohexstr(Value) +
448                                   " is not aligned to " + Twine(N) + " bytes");
449 }
450 
451 AnonymousPointerCreator getAnonymousPointerCreator(const Triple &TT) {
452   switch (TT.getArch()) {
453   case Triple::aarch64:
454     return aarch64::createAnonymousPointer;
455   case Triple::x86_64:
456     return x86_64::createAnonymousPointer;
457   case Triple::x86:
458     return i386::createAnonymousPointer;
459   case Triple::loongarch32:
460   case Triple::loongarch64:
461     return loongarch::createAnonymousPointer;
462   default:
463     return nullptr;
464   }
465 }
466 
467 PointerJumpStubCreator getPointerJumpStubCreator(const Triple &TT) {
468   switch (TT.getArch()) {
469   case Triple::aarch64:
470     return aarch64::createAnonymousPointerJumpStub;
471   case Triple::x86_64:
472     return x86_64::createAnonymousPointerJumpStub;
473   case Triple::x86:
474     return i386::createAnonymousPointerJumpStub;
475   case Triple::loongarch32:
476   case Triple::loongarch64:
477     return loongarch::createAnonymousPointerJumpStub;
478   default:
479     return nullptr;
480   }
481 }
482 
483 Expected<std::unique_ptr<LinkGraph>>
484 createLinkGraphFromObject(MemoryBufferRef ObjectBuffer) {
485   auto Magic = identify_magic(ObjectBuffer.getBuffer());
486   switch (Magic) {
487   case file_magic::macho_object:
488     return createLinkGraphFromMachOObject(ObjectBuffer);
489   case file_magic::elf_relocatable:
490     return createLinkGraphFromELFObject(ObjectBuffer);
491   case file_magic::coff_object:
492     return createLinkGraphFromCOFFObject(ObjectBuffer);
493   default:
494     return make_error<JITLinkError>("Unsupported file format");
495   };
496 }
497 
498 std::unique_ptr<LinkGraph> absoluteSymbolsLinkGraph(const Triple &TT,
499                                                     orc::SymbolMap Symbols) {
500   unsigned PointerSize;
501   endianness Endianness =
502       TT.isLittleEndian() ? endianness::little : endianness::big;
503   switch (TT.getArch()) {
504   case Triple::aarch64:
505   case llvm::Triple::riscv64:
506   case Triple::x86_64:
507     PointerSize = 8;
508     break;
509   case llvm::Triple::arm:
510   case llvm::Triple::riscv32:
511   case llvm::Triple::x86:
512     PointerSize = 4;
513     break;
514   default:
515     llvm::report_fatal_error("unhandled target architecture");
516   }
517 
518   static std::atomic<uint64_t> Counter = {0};
519   auto Index = Counter.fetch_add(1, std::memory_order_relaxed);
520   auto G = std::make_unique<LinkGraph>(
521       "<Absolute Symbols " + std::to_string(Index) + ">", TT, PointerSize,
522       Endianness, /*GetEdgeKindName=*/nullptr);
523   for (auto &[Name, Def] : Symbols) {
524     auto &Sym =
525         G->addAbsoluteSymbol(*Name, Def.getAddress(), /*Size=*/0,
526                              Linkage::Strong, Scope::Default, /*IsLive=*/true);
527     Sym.setCallable(Def.getFlags().isCallable());
528   }
529 
530   return G;
531 }
532 
533 void link(std::unique_ptr<LinkGraph> G, std::unique_ptr<JITLinkContext> Ctx) {
534   switch (G->getTargetTriple().getObjectFormat()) {
535   case Triple::MachO:
536     return link_MachO(std::move(G), std::move(Ctx));
537   case Triple::ELF:
538     return link_ELF(std::move(G), std::move(Ctx));
539   case Triple::COFF:
540     return link_COFF(std::move(G), std::move(Ctx));
541   default:
542     Ctx->notifyFailed(make_error<JITLinkError>("Unsupported object format"));
543   };
544 }
545 
546 } // end namespace jitlink
547 } // end namespace llvm
548