1 //===---- ELF_x86_64.cpp -JIT linker implementation for ELF/x86-64 ----===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // ELF/x86-64 jit-link implementation.
10 //
11 //===----------------------------------------------------------------------===//
12
13 #include "llvm/ExecutionEngine/JITLink/ELF_x86_64.h"
14 #include "llvm/ExecutionEngine/JITLink/JITLink.h"
15 #include "llvm/ExecutionEngine/JITLink/x86_64.h"
16 #include "llvm/Object/ELFObjectFile.h"
17 #include "llvm/Support/Endian.h"
18
19 #include "DefineExternalSectionStartAndEndSymbols.h"
20 #include "EHFrameSupportImpl.h"
21 #include "JITLinkGeneric.h"
22 #include "PerGraphGOTAndPLTStubsBuilder.h"
23
24 #define DEBUG_TYPE "jitlink"
25
26 using namespace llvm;
27 using namespace llvm::jitlink;
28 using namespace llvm::jitlink::ELF_x86_64_Edges;
29
30 namespace {
31
32 constexpr StringRef ELFGOTSectionName = "$__GOT";
33 constexpr StringRef ELFGOTSymbolName = "_GLOBAL_OFFSET_TABLE_";
34
35 class PerGraphGOTAndPLTStubsBuilder_ELF_x86_64
36 : public PerGraphGOTAndPLTStubsBuilder<
37 PerGraphGOTAndPLTStubsBuilder_ELF_x86_64> {
38 public:
39 static const uint8_t NullGOTEntryContent[8];
40 static const uint8_t StubContent[6];
41
42 using PerGraphGOTAndPLTStubsBuilder<
43 PerGraphGOTAndPLTStubsBuilder_ELF_x86_64>::PerGraphGOTAndPLTStubsBuilder;
44
isGOTEdgeToFix(Edge & E) const45 bool isGOTEdgeToFix(Edge &E) const {
46 if (E.getKind() == GOTOFF64) {
47 // We need to make sure that the GOT section exists, but don't otherwise
48 // need to fix up this edge.
49 getGOTSection();
50 return false;
51 }
52
53 return E.getKind() == PCRel32GOT || E.getKind() == PCRel32GOTLoad ||
54 E.getKind() == PCRel64GOT || E.getKind() == GOT64;
55 }
56
createGOTEntry(Symbol & Target)57 Symbol &createGOTEntry(Symbol &Target) {
58 auto &GOTEntryBlock = G.createContentBlock(
59 getGOTSection(), getGOTEntryBlockContent(), 0, 8, 0);
60 GOTEntryBlock.addEdge(Pointer64, 0, Target, 0);
61 return G.addAnonymousSymbol(GOTEntryBlock, 0, 8, false, false);
62 }
63
fixGOTEdge(Edge & E,Symbol & GOTEntry)64 void fixGOTEdge(Edge &E, Symbol &GOTEntry) {
65 // If this is a PCRel32GOT/PCRel64GOT then change it to an ordinary
66 // PCRel32/PCRel64. If it is a PCRel32GOTLoad then leave it as-is for now:
67 // We will use the kind to check for GOT optimization opportunities in the
68 // optimizeMachO_x86_64_GOTAndStubs pass below.
69 // If it's a GOT64 leave it as is.
70 switch (E.getKind()) {
71 case PCRel32GOT:
72 E.setKind(PCRel32);
73 break;
74 case PCRel64GOT:
75 E.setKind(PCRel64);
76 break;
77 case GOT64:
78 break;
79 case PCRel32GOTLoad:
80 break;
81 default:
82 llvm_unreachable("Unexpected GOT edge kind");
83 }
84
85 E.setTarget(GOTEntry);
86 // Leave the edge addend as-is.
87 }
88
isExternalBranchEdge(Edge & E)89 bool isExternalBranchEdge(Edge &E) {
90 return E.getKind() == Branch32 && !E.getTarget().isDefined();
91 }
92
createPLTStub(Symbol & Target)93 Symbol &createPLTStub(Symbol &Target) {
94 auto &StubContentBlock =
95 G.createContentBlock(getStubsSection(), getStubBlockContent(), 0, 1, 0);
96 // Re-use GOT entries for stub targets.
97 auto &GOTEntrySymbol = getGOTEntry(Target);
98 StubContentBlock.addEdge(PCRel32, 2, GOTEntrySymbol, -4);
99 return G.addAnonymousSymbol(StubContentBlock, 0, 6, true, false);
100 }
101
fixPLTEdge(Edge & E,Symbol & Stub)102 void fixPLTEdge(Edge &E, Symbol &Stub) {
103 assert(E.getKind() == Branch32 && "Not a Branch32 edge?");
104
105 // Set the edge kind to Branch32ToStub. We will use this to check for stub
106 // optimization opportunities in the optimize ELF_x86_64_GOTAndStubs pass
107 // below.
108 E.setKind(Branch32ToStub);
109 E.setTarget(Stub);
110 }
111
112 private:
getGOTSection() const113 Section &getGOTSection() const {
114 if (!GOTSection)
115 GOTSection = &G.createSection(ELFGOTSectionName, sys::Memory::MF_READ);
116 return *GOTSection;
117 }
118
getStubsSection() const119 Section &getStubsSection() const {
120 if (!StubsSection) {
121 auto StubsProt = static_cast<sys::Memory::ProtectionFlags>(
122 sys::Memory::MF_READ | sys::Memory::MF_EXEC);
123 StubsSection = &G.createSection("$__STUBS", StubsProt);
124 }
125 return *StubsSection;
126 }
127
getGOTEntryBlockContent()128 ArrayRef<char> getGOTEntryBlockContent() {
129 return {reinterpret_cast<const char *>(NullGOTEntryContent),
130 sizeof(NullGOTEntryContent)};
131 }
132
getStubBlockContent()133 ArrayRef<char> getStubBlockContent() {
134 return {reinterpret_cast<const char *>(StubContent), sizeof(StubContent)};
135 }
136
137 mutable Section *GOTSection = nullptr;
138 mutable Section *StubsSection = nullptr;
139 };
140
141 const char *const DwarfSectionNames[] = {
142 #define HANDLE_DWARF_SECTION(ENUM_NAME, ELF_NAME, CMDLINE_NAME, OPTION) \
143 ELF_NAME,
144 #include "llvm/BinaryFormat/Dwarf.def"
145 #undef HANDLE_DWARF_SECTION
146 };
147
148 } // namespace
149
150 const uint8_t PerGraphGOTAndPLTStubsBuilder_ELF_x86_64::NullGOTEntryContent[8] =
151 {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00};
152 const uint8_t PerGraphGOTAndPLTStubsBuilder_ELF_x86_64::StubContent[6] = {
153 0xFF, 0x25, 0x00, 0x00, 0x00, 0x00};
154
155 static const char *CommonSectionName = "__common";
optimizeELF_x86_64_GOTAndStubs(LinkGraph & G)156 static Error optimizeELF_x86_64_GOTAndStubs(LinkGraph &G) {
157 LLVM_DEBUG(dbgs() << "Optimizing GOT entries and stubs:\n");
158
159 for (auto *B : G.blocks())
160 for (auto &E : B->edges())
161 if (E.getKind() == PCRel32GOTLoad) {
162 // Replace GOT load with LEA only for MOVQ instructions.
163 constexpr uint8_t MOVQRIPRel[] = {0x48, 0x8b};
164 if (E.getOffset() < 3 ||
165 strncmp(B->getContent().data() + E.getOffset() - 3,
166 reinterpret_cast<const char *>(MOVQRIPRel), 2) != 0)
167 continue;
168
169 auto &GOTBlock = E.getTarget().getBlock();
170 assert(GOTBlock.getSize() == G.getPointerSize() &&
171 "GOT entry block should be pointer sized");
172 assert(GOTBlock.edges_size() == 1 &&
173 "GOT entry should only have one outgoing edge");
174
175 auto &GOTTarget = GOTBlock.edges().begin()->getTarget();
176 JITTargetAddress EdgeAddr = B->getAddress() + E.getOffset();
177 JITTargetAddress TargetAddr = GOTTarget.getAddress();
178
179 int64_t Displacement = TargetAddr - EdgeAddr + 4;
180 if (Displacement >= std::numeric_limits<int32_t>::min() &&
181 Displacement <= std::numeric_limits<int32_t>::max()) {
182 // Change the edge kind as we don't go through GOT anymore. This is
183 // for formal correctness only. Technically, the two relocation kinds
184 // are resolved the same way.
185 E.setKind(PCRel32);
186 E.setTarget(GOTTarget);
187 auto *BlockData = reinterpret_cast<uint8_t *>(
188 const_cast<char *>(B->getContent().data()));
189 BlockData[E.getOffset() - 2] = 0x8d;
190 LLVM_DEBUG({
191 dbgs() << " Replaced GOT load wih LEA:\n ";
192 printEdge(dbgs(), *B, E, getELFX86RelocationKindName(E.getKind()));
193 dbgs() << "\n";
194 });
195 }
196 } else if (E.getKind() == Branch32ToStub) {
197 auto &StubBlock = E.getTarget().getBlock();
198 assert(
199 StubBlock.getSize() ==
200 sizeof(PerGraphGOTAndPLTStubsBuilder_ELF_x86_64::StubContent) &&
201 "Stub block should be stub sized");
202 assert(StubBlock.edges_size() == 1 &&
203 "Stub block should only have one outgoing edge");
204
205 auto &GOTBlock = StubBlock.edges().begin()->getTarget().getBlock();
206 assert(GOTBlock.getSize() == G.getPointerSize() &&
207 "GOT block should be pointer sized");
208 assert(GOTBlock.edges_size() == 1 &&
209 "GOT block should only have one outgoing edge");
210
211 auto &GOTTarget = GOTBlock.edges().begin()->getTarget();
212 JITTargetAddress EdgeAddr = B->getAddress() + E.getOffset();
213 JITTargetAddress TargetAddr = GOTTarget.getAddress();
214
215 int64_t Displacement = TargetAddr - EdgeAddr + 4;
216 if (Displacement >= std::numeric_limits<int32_t>::min() &&
217 Displacement <= std::numeric_limits<int32_t>::max()) {
218 E.setKind(Branch32);
219 E.setTarget(GOTTarget);
220 LLVM_DEBUG({
221 dbgs() << " Replaced stub branch with direct branch:\n ";
222 printEdge(dbgs(), *B, E, getELFX86RelocationKindName(E.getKind()));
223 dbgs() << "\n";
224 });
225 }
226 }
227
228 return Error::success();
229 }
230
isDwarfSection(StringRef SectionName)231 static bool isDwarfSection(StringRef SectionName) {
232 return llvm::is_contained(DwarfSectionNames, SectionName);
233 }
234
235 namespace llvm {
236 namespace jitlink {
237
238 // This should become a template as the ELFFile is so a lot of this could become
239 // generic
240 class ELFLinkGraphBuilder_x86_64 {
241
242 private:
243 Section *CommonSection = nullptr;
244
245 // TODO hack to get this working
246 // Find a better way
247 using SymbolTable = object::ELFFile<object::ELF64LE>::Elf_Shdr;
248 // For now we just assume
249 using SymbolMap = std::map<int32_t, Symbol *>;
250 SymbolMap JITSymbolTable;
251
getCommonSection()252 Section &getCommonSection() {
253 if (!CommonSection) {
254 auto Prot = static_cast<sys::Memory::ProtectionFlags>(
255 sys::Memory::MF_READ | sys::Memory::MF_WRITE);
256 CommonSection = &G->createSection(CommonSectionName, Prot);
257 }
258 return *CommonSection;
259 }
260
261 static Expected<ELF_x86_64_Edges::ELFX86RelocationKind>
getRelocationKind(const uint32_t Type)262 getRelocationKind(const uint32_t Type) {
263 switch (Type) {
264 case ELF::R_X86_64_PC32:
265 return ELF_x86_64_Edges::ELFX86RelocationKind::PCRel32;
266 case ELF::R_X86_64_PC64:
267 case ELF::R_X86_64_GOTPC64:
268 return ELF_x86_64_Edges::ELFX86RelocationKind::Delta64;
269 case ELF::R_X86_64_64:
270 return ELF_x86_64_Edges::ELFX86RelocationKind::Pointer64;
271 case ELF::R_X86_64_GOTPCREL:
272 case ELF::R_X86_64_GOTPCRELX:
273 case ELF::R_X86_64_REX_GOTPCRELX:
274 return ELF_x86_64_Edges::ELFX86RelocationKind::PCRel32GOTLoad;
275 case ELF::R_X86_64_GOTPCREL64:
276 return ELF_x86_64_Edges::ELFX86RelocationKind::PCRel64GOT;
277 case ELF::R_X86_64_GOT64:
278 return ELF_x86_64_Edges::ELFX86RelocationKind::GOT64;
279 case ELF::R_X86_64_GOTOFF64:
280 return ELF_x86_64_Edges::ELFX86RelocationKind::GOTOFF64;
281 case ELF::R_X86_64_PLT32:
282 return ELF_x86_64_Edges::ELFX86RelocationKind::Branch32;
283 }
284 return make_error<JITLinkError>("Unsupported x86-64 relocation:" +
285 formatv("{0:d}", Type));
286 }
287
288 std::unique_ptr<LinkGraph> G;
289 // This could be a template
290 const object::ELFFile<object::ELF64LE> &Obj;
291 object::ELFFile<object::ELF64LE>::Elf_Shdr_Range sections;
292 SymbolTable SymTab;
293
isRelocatable()294 bool isRelocatable() { return Obj.getHeader().e_type == llvm::ELF::ET_REL; }
295
296 support::endianness
getEndianness(const object::ELFFile<object::ELF64LE> & Obj)297 getEndianness(const object::ELFFile<object::ELF64LE> &Obj) {
298 return Obj.isLE() ? support::little : support::big;
299 }
300
301 // This could also just become part of a template
getPointerSize(const object::ELFFile<object::ELF64LE> & Obj)302 unsigned getPointerSize(const object::ELFFile<object::ELF64LE> &Obj) {
303 return Obj.getHeader().getFileClass() == ELF::ELFCLASS64 ? 8 : 4;
304 }
305
306 // We don't technically need this right now
307 // But for now going to keep it as it helps me to debug things
308
createNormalizedSymbols()309 Error createNormalizedSymbols() {
310 LLVM_DEBUG(dbgs() << "Creating normalized symbols...\n");
311
312 for (auto SecRef : sections) {
313 if (SecRef.sh_type != ELF::SHT_SYMTAB &&
314 SecRef.sh_type != ELF::SHT_DYNSYM)
315 continue;
316
317 auto Symbols = Obj.symbols(&SecRef);
318 // TODO: Currently I use this function to test things
319 // I also want to leave it to see if its common between MACH and elf
320 // so for now I just want to continue even if there is an error
321 if (errorToBool(Symbols.takeError()))
322 continue;
323
324 auto StrTabSec = Obj.getSection(SecRef.sh_link);
325 if (!StrTabSec)
326 return StrTabSec.takeError();
327 auto StringTable = Obj.getStringTable(**StrTabSec);
328 if (!StringTable)
329 return StringTable.takeError();
330
331 for (auto SymRef : *Symbols) {
332 Optional<StringRef> Name;
333
334 if (auto NameOrErr = SymRef.getName(*StringTable))
335 Name = *NameOrErr;
336 else
337 return NameOrErr.takeError();
338
339 LLVM_DEBUG({
340 dbgs() << " value = " << formatv("{0:x16}", SymRef.getValue())
341 << ", type = " << formatv("{0:x2}", SymRef.getType())
342 << ", binding = " << formatv("{0:x2}", SymRef.getBinding())
343 << ", size = "
344 << formatv("{0:x16}", static_cast<uint64_t>(SymRef.st_size))
345 << ", info = " << formatv("{0:x2}", SymRef.st_info)
346 << " :" << (Name ? *Name : "<anonymous symbol>") << "\n";
347 });
348 }
349 }
350 return Error::success();
351 }
352
createNormalizedSections()353 Error createNormalizedSections() {
354 LLVM_DEBUG(dbgs() << "Creating normalized sections...\n");
355 for (auto &SecRef : sections) {
356 auto Name = Obj.getSectionName(SecRef);
357 if (!Name)
358 return Name.takeError();
359
360 // Skip Dwarf sections.
361 if (isDwarfSection(*Name)) {
362 LLVM_DEBUG({
363 dbgs() << *Name
364 << " is a debug section: No graph section will be created.\n";
365 });
366 continue;
367 }
368
369 sys::Memory::ProtectionFlags Prot;
370 if (SecRef.sh_flags & ELF::SHF_EXECINSTR) {
371 Prot = static_cast<sys::Memory::ProtectionFlags>(sys::Memory::MF_READ |
372 sys::Memory::MF_EXEC);
373 } else {
374 Prot = static_cast<sys::Memory::ProtectionFlags>(sys::Memory::MF_READ |
375 sys::Memory::MF_WRITE);
376 }
377 uint64_t Address = SecRef.sh_addr;
378 uint64_t Size = SecRef.sh_size;
379 uint64_t Flags = SecRef.sh_flags;
380 uint64_t Alignment = SecRef.sh_addralign;
381 const char *Data = nullptr;
382 // for now we just use this to skip the "undefined" section, probably need
383 // to revist
384 if (Size == 0)
385 continue;
386
387 // FIXME: Use flags.
388 (void)Flags;
389
390 LLVM_DEBUG({
391 dbgs() << " " << *Name << ": " << formatv("{0:x16}", Address) << " -- "
392 << formatv("{0:x16}", Address + Size) << ", align: " << Alignment
393 << " Flags: " << formatv("{0:x}", Flags) << "\n";
394 });
395
396 if (SecRef.sh_type != ELF::SHT_NOBITS) {
397 // .sections() already checks that the data is not beyond the end of
398 // file
399 auto contents = Obj.getSectionContentsAsArray<char>(SecRef);
400 if (!contents)
401 return contents.takeError();
402
403 Data = contents->data();
404 // TODO protection flags.
405 // for now everything is
406 auto §ion = G->createSection(*Name, Prot);
407 // Do this here because we have it, but move it into graphify later
408 G->createContentBlock(section, ArrayRef<char>(Data, Size), Address,
409 Alignment, 0);
410 if (SecRef.sh_type == ELF::SHT_SYMTAB)
411 // TODO: Dynamic?
412 SymTab = SecRef;
413 } else {
414 auto &Section = G->createSection(*Name, Prot);
415 G->createZeroFillBlock(Section, Size, Address, Alignment, 0);
416 }
417 }
418
419 return Error::success();
420 }
421
addRelocations()422 Error addRelocations() {
423 LLVM_DEBUG(dbgs() << "Adding relocations\n");
424 // TODO a partern is forming of iterate some sections but only give me
425 // ones I am interested, i should abstract that concept some where
426 for (auto &SecRef : sections) {
427 if (SecRef.sh_type != ELF::SHT_RELA && SecRef.sh_type != ELF::SHT_REL)
428 continue;
429 // TODO can the elf obj file do this for me?
430 if (SecRef.sh_type == ELF::SHT_REL)
431 return make_error<llvm::StringError>("Shouldn't have REL in x64",
432 llvm::inconvertibleErrorCode());
433
434 auto RelSectName = Obj.getSectionName(SecRef);
435 if (!RelSectName)
436 return RelSectName.takeError();
437
438 LLVM_DEBUG({
439 dbgs() << "Adding relocations from section " << *RelSectName << "\n";
440 });
441
442 auto UpdateSection = Obj.getSection(SecRef.sh_info);
443 if (!UpdateSection)
444 return UpdateSection.takeError();
445
446 auto UpdateSectionName = Obj.getSectionName(**UpdateSection);
447 if (!UpdateSectionName)
448 return UpdateSectionName.takeError();
449
450 // Don't process relocations for debug sections.
451 if (isDwarfSection(*UpdateSectionName)) {
452 LLVM_DEBUG({
453 dbgs() << " Target is dwarf section " << *UpdateSectionName
454 << ". Skipping.\n";
455 });
456 continue;
457 } else
458 LLVM_DEBUG({
459 dbgs() << " For target section " << *UpdateSectionName << "\n";
460 });
461
462 auto JITSection = G->findSectionByName(*UpdateSectionName);
463 if (!JITSection)
464 return make_error<llvm::StringError>(
465 "Refencing a a section that wasn't added to graph" +
466 *UpdateSectionName,
467 llvm::inconvertibleErrorCode());
468
469 auto Relocations = Obj.relas(SecRef);
470 if (!Relocations)
471 return Relocations.takeError();
472
473 for (const auto &Rela : *Relocations) {
474 auto Type = Rela.getType(false);
475
476 LLVM_DEBUG({
477 dbgs() << "Relocation Type: " << Type << "\n"
478 << "Name: " << Obj.getRelocationTypeName(Type) << "\n";
479 });
480 auto SymbolIndex = Rela.getSymbol(false);
481 auto Symbol = Obj.getRelocationSymbol(Rela, &SymTab);
482 if (!Symbol)
483 return Symbol.takeError();
484
485 auto BlockToFix = *(JITSection->blocks().begin());
486 auto *TargetSymbol = JITSymbolTable[SymbolIndex];
487
488 if (!TargetSymbol) {
489 return make_error<llvm::StringError>(
490 "Could not find symbol at given index, did you add it to "
491 "JITSymbolTable? index: " + std::to_string(SymbolIndex)
492 + ", shndx: " + std::to_string((*Symbol)->st_shndx) +
493 " Size of table: " + std::to_string(JITSymbolTable.size()),
494 llvm::inconvertibleErrorCode());
495 }
496 uint64_t Addend = Rela.r_addend;
497 JITTargetAddress FixupAddress =
498 (*UpdateSection)->sh_addr + Rela.r_offset;
499
500 LLVM_DEBUG({
501 dbgs() << "Processing relocation at "
502 << format("0x%016" PRIx64, FixupAddress) << "\n";
503 });
504 auto Kind = getRelocationKind(Type);
505 if (!Kind)
506 return Kind.takeError();
507
508 LLVM_DEBUG({
509 Edge GE(*Kind, FixupAddress - BlockToFix->getAddress(), *TargetSymbol,
510 Addend);
511 printEdge(dbgs(), *BlockToFix, GE,
512 getELFX86RelocationKindName(*Kind));
513 dbgs() << "\n";
514 });
515 BlockToFix->addEdge(*Kind, FixupAddress - BlockToFix->getAddress(),
516 *TargetSymbol, Addend);
517 }
518 }
519 return Error::success();
520 }
521
graphifyRegularSymbols()522 Error graphifyRegularSymbols() {
523
524 // TODO: ELF supports beyond SHN_LORESERVE,
525 // need to perf test how a vector vs map handles those cases
526
527 std::vector<std::vector<object::ELFFile<object::ELF64LE>::Elf_Shdr_Range *>>
528 SecIndexToSymbols;
529
530 LLVM_DEBUG(dbgs() << "Creating graph symbols...\n");
531
532 for (auto SecRef : sections) {
533
534 if (SecRef.sh_type != ELF::SHT_SYMTAB &&
535 SecRef.sh_type != ELF::SHT_DYNSYM)
536 continue;
537 auto Symbols = Obj.symbols(&SecRef);
538 if (!Symbols)
539 return Symbols.takeError();
540
541 auto StrTabSec = Obj.getSection(SecRef.sh_link);
542 if (!StrTabSec)
543 return StrTabSec.takeError();
544 auto StringTable = Obj.getStringTable(**StrTabSec);
545 if (!StringTable)
546 return StringTable.takeError();
547 auto Name = Obj.getSectionName(SecRef);
548 if (!Name)
549 return Name.takeError();
550
551 LLVM_DEBUG(dbgs() << "Processing symbol section " << *Name << ":\n");
552
553 auto Section = G->findSectionByName(*Name);
554 if (!Section)
555 return make_error<llvm::StringError>("Could not find a section " +
556 *Name,
557 llvm::inconvertibleErrorCode());
558 // we only have one for now
559 auto blocks = Section->blocks();
560 if (blocks.empty())
561 return make_error<llvm::StringError>("Section has no block",
562 llvm::inconvertibleErrorCode());
563 int SymbolIndex = -1;
564 for (auto SymRef : *Symbols) {
565 ++SymbolIndex;
566 auto Type = SymRef.getType();
567
568 if (Type == ELF::STT_FILE || SymbolIndex == 0)
569 continue;
570 // these should do it for now
571 // if(Type != ELF::STT_NOTYPE &&
572 // Type != ELF::STT_OBJECT &&
573 // Type != ELF::STT_FUNC &&
574 // Type != ELF::STT_SECTION &&
575 // Type != ELF::STT_COMMON) {
576 // continue;
577 // }
578 auto Name = SymRef.getName(*StringTable);
579 // I am not sure on If this is going to hold as an invariant. Revisit.
580 if (!Name)
581 return Name.takeError();
582
583 if (SymRef.isCommon()) {
584 // Symbols in SHN_COMMON refer to uninitialized data. The st_value
585 // field holds alignment constraints.
586 Symbol &S =
587 G->addCommonSymbol(*Name, Scope::Default, getCommonSection(), 0,
588 SymRef.st_size, SymRef.getValue(), false);
589 JITSymbolTable[SymbolIndex] = &S;
590 continue;
591 }
592
593 // Map Visibility and Binding to Scope and Linkage:
594 Linkage L = Linkage::Strong;
595 Scope S = Scope::Default;
596
597 switch (SymRef.getBinding()) {
598 case ELF::STB_LOCAL:
599 S = Scope::Local;
600 break;
601 case ELF::STB_GLOBAL:
602 // Nothing to do here.
603 break;
604 case ELF::STB_WEAK:
605 L = Linkage::Weak;
606 break;
607 default:
608 return make_error<StringError>("Unrecognized symbol binding for " +
609 *Name,
610 inconvertibleErrorCode());
611 }
612
613 switch (SymRef.getVisibility()) {
614 case ELF::STV_DEFAULT:
615 case ELF::STV_PROTECTED:
616 // FIXME: Make STV_DEFAULT symbols pre-emptible? This probably needs
617 // Orc support.
618 // Otherwise nothing to do here.
619 break;
620 case ELF::STV_HIDDEN:
621 // Default scope -> Hidden scope. No effect on local scope.
622 if (S == Scope::Default)
623 S = Scope::Hidden;
624 break;
625 case ELF::STV_INTERNAL:
626 return make_error<StringError>("Unrecognized symbol visibility for " +
627 *Name,
628 inconvertibleErrorCode());
629 }
630
631 if (SymRef.isDefined() &&
632 (Type == ELF::STT_NOTYPE || Type == ELF::STT_FUNC ||
633 Type == ELF::STT_OBJECT || Type == ELF::STT_SECTION)) {
634
635 auto DefinedSection = Obj.getSection(SymRef.st_shndx);
636 if (!DefinedSection)
637 return DefinedSection.takeError();
638 auto sectName = Obj.getSectionName(**DefinedSection);
639 if (!sectName)
640 return Name.takeError();
641
642 // Skip debug section symbols.
643 if (isDwarfSection(*sectName))
644 continue;
645
646 auto JitSection = G->findSectionByName(*sectName);
647 if (!JitSection)
648 return make_error<llvm::StringError>(
649 "Could not find the JitSection " + *sectName,
650 llvm::inconvertibleErrorCode());
651 auto bs = JitSection->blocks();
652 if (bs.empty())
653 return make_error<llvm::StringError>(
654 "Section has no block", llvm::inconvertibleErrorCode());
655
656 auto *B = *bs.begin();
657 LLVM_DEBUG({ dbgs() << " " << *Name << " at index " << SymbolIndex << "\n"; });
658 if (SymRef.getType() == ELF::STT_SECTION)
659 *Name = *sectName;
660 auto &Sym = G->addDefinedSymbol(
661 *B, SymRef.getValue(), *Name, SymRef.st_size, L, S,
662 SymRef.getType() == ELF::STT_FUNC, false);
663 JITSymbolTable[SymbolIndex] = &Sym;
664 } else if (SymRef.isUndefined() && SymRef.isExternal()) {
665 auto &Sym = G->addExternalSymbol(*Name, SymRef.st_size, L);
666 JITSymbolTable[SymbolIndex] = &Sym;
667 } else
668 LLVM_DEBUG({
669 dbgs()
670 << "Not creating graph symbol for normalized symbol at index "
671 << SymbolIndex << ", \"" << *Name << "\"\n";
672 });
673
674 // TODO: The following has to be implmented.
675 // leaving commented out to save time for future patchs
676 /*
677 G->addAbsoluteSymbol(*Name, SymRef.getValue(), SymRef.st_size,
678 Linkage::Strong, Scope::Default, false);
679 */
680 }
681 }
682 return Error::success();
683 }
684
685 public:
ELFLinkGraphBuilder_x86_64(StringRef FileName,const object::ELFFile<object::ELF64LE> & Obj)686 ELFLinkGraphBuilder_x86_64(StringRef FileName,
687 const object::ELFFile<object::ELF64LE> &Obj)
688 : G(std::make_unique<LinkGraph>(
689 FileName.str(), Triple("x86_64-unknown-linux"), getPointerSize(Obj),
690 getEndianness(Obj), getELFX86RelocationKindName)),
691 Obj(Obj) {}
692
buildGraph()693 Expected<std::unique_ptr<LinkGraph>> buildGraph() {
694 // Sanity check: we only operate on relocatable objects.
695 if (!isRelocatable())
696 return make_error<JITLinkError>("Object is not a relocatable ELF");
697
698 auto Secs = Obj.sections();
699
700 if (!Secs) {
701 return Secs.takeError();
702 }
703 sections = *Secs;
704
705 if (auto Err = createNormalizedSections())
706 return std::move(Err);
707
708 if (auto Err = createNormalizedSymbols())
709 return std::move(Err);
710
711 if (auto Err = graphifyRegularSymbols())
712 return std::move(Err);
713
714 if (auto Err = addRelocations())
715 return std::move(Err);
716
717 return std::move(G);
718 }
719 };
720
721 class ELFJITLinker_x86_64 : public JITLinker<ELFJITLinker_x86_64> {
722 friend class JITLinker<ELFJITLinker_x86_64>;
723
724 public:
ELFJITLinker_x86_64(std::unique_ptr<JITLinkContext> Ctx,std::unique_ptr<LinkGraph> G,PassConfiguration PassConfig)725 ELFJITLinker_x86_64(std::unique_ptr<JITLinkContext> Ctx,
726 std::unique_ptr<LinkGraph> G,
727 PassConfiguration PassConfig)
728 : JITLinker(std::move(Ctx), std::move(G), std::move(PassConfig)) {
729 getPassConfig().PostAllocationPasses.push_back(
730 [this](LinkGraph &G) { return getOrCreateGOTSymbol(G); });
731 }
732
733 private:
734 Symbol *GOTSymbol = nullptr;
735
getOrCreateGOTSymbol(LinkGraph & G)736 Error getOrCreateGOTSymbol(LinkGraph &G) {
737 auto DefineExternalGOTSymbolIfPresent =
738 createDefineExternalSectionStartAndEndSymbolsPass(
739 [&](LinkGraph &LG, Symbol &Sym) -> SectionRangeSymbolDesc {
740 if (Sym.getName() == ELFGOTSymbolName)
741 if (auto *GOTSection = G.findSectionByName(ELFGOTSectionName)) {
742 GOTSymbol = &Sym;
743 return {*GOTSection, true};
744 }
745 return {};
746 });
747
748 // Try to attach _GLOBAL_OFFSET_TABLE_ to the GOT if it's defined as an
749 // external.
750 if (auto Err = DefineExternalGOTSymbolIfPresent(G))
751 return Err;
752
753 // If we succeeded then we're done.
754 if (GOTSymbol)
755 return Error::success();
756
757 // Otherwise look for a GOT section: If it already has a start symbol we'll
758 // record it, otherwise we'll create our own.
759 // If there's a GOT section but we didn't find an external GOT symbol...
760 if (auto *GOTSection = G.findSectionByName(ELFGOTSectionName)) {
761
762 // Check for an existing defined symbol.
763 for (auto *Sym : GOTSection->symbols())
764 if (Sym->getName() == ELFGOTSymbolName) {
765 GOTSymbol = Sym;
766 return Error::success();
767 }
768
769 // If there's no defined symbol then create one.
770 SectionRange SR(*GOTSection);
771 if (SR.empty())
772 GOTSymbol = &G.addAbsoluteSymbol(ELFGOTSymbolName, 0, 0,
773 Linkage::Strong, Scope::Local, true);
774 else
775 GOTSymbol =
776 &G.addDefinedSymbol(*SR.getFirstBlock(), 0, ELFGOTSymbolName, 0,
777 Linkage::Strong, Scope::Local, false, true);
778 }
779
780 return Error::success();
781 }
782
applyFixup(LinkGraph & G,Block & B,const Edge & E,char * BlockWorkingMem) const783 Error applyFixup(LinkGraph &G, Block &B, const Edge &E,
784 char *BlockWorkingMem) const {
785 using namespace ELF_x86_64_Edges;
786 using namespace llvm::support;
787 char *FixupPtr = BlockWorkingMem + E.getOffset();
788 JITTargetAddress FixupAddress = B.getAddress() + E.getOffset();
789 switch (E.getKind()) {
790 case ELFX86RelocationKind::Branch32:
791 case ELFX86RelocationKind::Branch32ToStub:
792 case ELFX86RelocationKind::PCRel32:
793 case ELFX86RelocationKind::PCRel32GOTLoad: {
794 int64_t Value = E.getTarget().getAddress() + E.getAddend() - FixupAddress;
795 if (LLVM_LIKELY(x86_64::isInRangeForImmS32(Value)))
796 *(little32_t *)FixupPtr = Value;
797 else
798 return makeTargetOutOfRangeError(G, B, E);
799 break;
800 }
801 case ELFX86RelocationKind::PCRel64: {
802 int64_t Value = E.getTarget().getAddress() + E.getAddend() - FixupAddress;
803 *(little64_t *)FixupPtr = Value;
804 break;
805 }
806 case ELFX86RelocationKind::Pointer64: {
807 int64_t Value = E.getTarget().getAddress() + E.getAddend();
808 *(ulittle64_t *)FixupPtr = Value;
809 break;
810 }
811 case ELFX86RelocationKind::Delta32: {
812 int64_t Value = E.getTarget().getAddress() + E.getAddend() - FixupAddress;
813 if (LLVM_LIKELY(x86_64::isInRangeForImmS32(Value)))
814 *(little32_t *)FixupPtr = Value;
815 else
816 return makeTargetOutOfRangeError(G, B, E);
817 break;
818 }
819 case ELFX86RelocationKind::Delta64: {
820 int64_t Value = E.getTarget().getAddress() + E.getAddend() - FixupAddress;
821 *(little64_t *)FixupPtr = Value;
822 break;
823 }
824 case ELFX86RelocationKind::NegDelta32: {
825 int64_t Value = FixupAddress - E.getTarget().getAddress() + E.getAddend();
826 if (LLVM_LIKELY(x86_64::isInRangeForImmS32(Value)))
827 *(little32_t *)FixupPtr = Value;
828 else
829 return makeTargetOutOfRangeError(G, B, E);
830 break;
831 }
832 case ELFX86RelocationKind::NegDelta64: {
833 int64_t Value = FixupAddress - E.getTarget().getAddress() + E.getAddend();
834 *(little64_t *)FixupPtr = Value;
835 break;
836 }
837 case ELFX86RelocationKind::GOT64:
838 case ELFX86RelocationKind::GOTOFF64: {
839 // GOT64: Offset of GOT entry within GOT.
840 // GOTOFF64: Offset from GOT base to target.
841 // The expressions are the same in both cases, but in the GOT64 case the
842 // edge will have been fixed to point at the GOT entry, and in the
843 // GOTOFF64 case it will still point at the original target.
844 assert(GOTSymbol && "No GOT section symbol");
845 int64_t Value =
846 E.getTarget().getAddress() - GOTSymbol->getAddress() + E.getAddend();
847 *(little64_t *)FixupPtr = Value;
848 break;
849 }
850 default:
851 LLVM_DEBUG({
852 dbgs() << "Bad edge: " << getELFX86RelocationKindName(E.getKind())
853 << "\n";
854 });
855 llvm_unreachable("Unsupported relocation");
856 }
857 return Error::success();
858 }
859 };
860
861 Expected<std::unique_ptr<LinkGraph>>
createLinkGraphFromELFObject_x86_64(MemoryBufferRef ObjectBuffer)862 createLinkGraphFromELFObject_x86_64(MemoryBufferRef ObjectBuffer) {
863 LLVM_DEBUG({
864 dbgs() << "Building jitlink graph for new input "
865 << ObjectBuffer.getBufferIdentifier() << "...\n";
866 });
867
868 auto ELFObj = object::ObjectFile::createELFObjectFile(ObjectBuffer);
869 if (!ELFObj)
870 return ELFObj.takeError();
871
872 auto &ELFObjFile = cast<object::ELFObjectFile<object::ELF64LE>>(**ELFObj);
873 return ELFLinkGraphBuilder_x86_64((*ELFObj)->getFileName(),
874 ELFObjFile.getELFFile())
875 .buildGraph();
876 }
877
878 static SectionRangeSymbolDesc
identifyELFSectionStartAndEndSymbols(LinkGraph & G,Symbol & Sym)879 identifyELFSectionStartAndEndSymbols(LinkGraph &G, Symbol &Sym) {
880 constexpr StringRef StartSymbolPrefix = "__start";
881 constexpr StringRef EndSymbolPrefix = "__end";
882
883 auto SymName = Sym.getName();
884 if (SymName.startswith(StartSymbolPrefix)) {
885 if (auto *Sec =
886 G.findSectionByName(SymName.drop_front(StartSymbolPrefix.size())))
887 return {*Sec, true};
888 } else if (SymName.startswith(EndSymbolPrefix)) {
889 if (auto *Sec =
890 G.findSectionByName(SymName.drop_front(EndSymbolPrefix.size())))
891 return {*Sec, false};
892 }
893 return {};
894 }
895
link_ELF_x86_64(std::unique_ptr<LinkGraph> G,std::unique_ptr<JITLinkContext> Ctx)896 void link_ELF_x86_64(std::unique_ptr<LinkGraph> G,
897 std::unique_ptr<JITLinkContext> Ctx) {
898 PassConfiguration Config;
899
900 if (Ctx->shouldAddDefaultTargetPasses(G->getTargetTriple())) {
901
902 Config.PrePrunePasses.push_back(EHFrameSplitter(".eh_frame"));
903 Config.PrePrunePasses.push_back(EHFrameEdgeFixer(
904 ".eh_frame", G->getPointerSize(), Delta64, Delta32, NegDelta32));
905 Config.PrePrunePasses.push_back(EHFrameNullTerminator(".eh_frame"));
906
907 // Construct a JITLinker and run the link function.
908 // Add a mark-live pass.
909 if (auto MarkLive = Ctx->getMarkLivePass(G->getTargetTriple()))
910 Config.PrePrunePasses.push_back(std::move(MarkLive));
911 else
912 Config.PrePrunePasses.push_back(markAllSymbolsLive);
913
914 // Add an in-place GOT/Stubs pass.
915 Config.PostPrunePasses.push_back(
916 PerGraphGOTAndPLTStubsBuilder_ELF_x86_64::asPass);
917
918 // Resolve any external section start / end symbols.
919 Config.PostAllocationPasses.push_back(
920 createDefineExternalSectionStartAndEndSymbolsPass(
921 identifyELFSectionStartAndEndSymbols));
922
923 // Add GOT/Stubs optimizer pass.
924 Config.PreFixupPasses.push_back(optimizeELF_x86_64_GOTAndStubs);
925 }
926
927 if (auto Err = Ctx->modifyPassConfig(*G, Config))
928 return Ctx->notifyFailed(std::move(Err));
929
930 ELFJITLinker_x86_64::link(std::move(Ctx), std::move(G), std::move(Config));
931 }
getELFX86RelocationKindName(Edge::Kind R)932 const char *getELFX86RelocationKindName(Edge::Kind R) {
933 switch (R) {
934 case Branch32:
935 return "Branch32";
936 case Branch32ToStub:
937 return "Branch32ToStub";
938 case Pointer32:
939 return "Pointer32";
940 case Pointer64:
941 return "Pointer64";
942 case Pointer64Anon:
943 return "Pointer64Anon";
944 case PCRel32:
945 return "PCRel32";
946 case PCRel32Minus1:
947 return "PCRel32Minus1";
948 case PCRel32Minus2:
949 return "PCRel32Minus2";
950 case PCRel32Minus4:
951 return "PCRel32Minus4";
952 case PCRel32Anon:
953 return "PCRel32Anon";
954 case PCRel32Minus1Anon:
955 return "PCRel32Minus1Anon";
956 case PCRel32Minus2Anon:
957 return "PCRel32Minus2Anon";
958 case PCRel32Minus4Anon:
959 return "PCRel32Minus4Anon";
960 case PCRel32GOTLoad:
961 return "PCRel32GOTLoad";
962 case PCRel32GOT:
963 return "PCRel32GOT";
964 case PCRel32TLV:
965 return "PCRel32TLV";
966 case Delta32:
967 return "Delta32";
968 case Delta64:
969 return "Delta64";
970 case NegDelta32:
971 return "NegDelta32";
972 case NegDelta64:
973 return "NegDelta64";
974 }
975 return getGenericEdgeKindName(static_cast<Edge::Kind>(R));
976 }
977 } // end namespace jitlink
978 } // end namespace llvm
979