1 //===------- ObjectLinkingLayer.cpp - JITLink backed ORC ObjectLayer ------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 9 #include "llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h" 10 #include "llvm/ExecutionEngine/JITLink/EHFrameSupport.h" 11 #include "llvm/ExecutionEngine/JITLink/aarch32.h" 12 #include "llvm/ExecutionEngine/Orc/DebugObjectManagerPlugin.h" 13 #include "llvm/ExecutionEngine/Orc/DebugUtils.h" 14 #include "llvm/ExecutionEngine/Orc/ObjectFileInterface.h" 15 #include "llvm/ExecutionEngine/Orc/Shared/ObjectFormats.h" 16 #include "llvm/Support/MemoryBuffer.h" 17 18 #include <string> 19 20 #define DEBUG_TYPE "orc" 21 22 using namespace llvm; 23 using namespace llvm::jitlink; 24 using namespace llvm::orc; 25 26 namespace { 27 28 bool hasInitializerSection(jitlink::LinkGraph &G) { 29 bool IsMachO = G.getTargetTriple().isOSBinFormatMachO(); 30 bool IsElf = G.getTargetTriple().isOSBinFormatELF(); 31 if (!IsMachO && !IsElf) 32 return false; 33 34 for (auto &Sec : G.sections()) { 35 if (IsMachO && isMachOInitializerSection(Sec.getName())) 36 return true; 37 if (IsElf && isELFInitializerSection(Sec.getName())) 38 return true; 39 } 40 41 return false; 42 } 43 44 ExecutorAddr getJITSymbolPtrForSymbol(Symbol &Sym, const Triple &TT) { 45 switch (TT.getArch()) { 46 case Triple::arm: 47 case Triple::armeb: 48 case Triple::thumb: 49 case Triple::thumbeb: 50 if (hasTargetFlags(Sym, aarch32::ThumbSymbol)) { 51 // Set LSB to indicate thumb target 52 assert(Sym.isCallable() && "Only callable symbols can have thumb flag"); 53 assert((Sym.getAddress().getValue() & 0x01) == 0 && "LSB is clear"); 54 return Sym.getAddress() + 0x01; 55 } 56 return Sym.getAddress(); 57 default: 58 return Sym.getAddress(); 59 } 60 } 61 62 JITSymbolFlags getJITSymbolFlagsForSymbol(Symbol &Sym) { 63 JITSymbolFlags Flags; 64 65 if (Sym.getLinkage() == Linkage::Weak) 66 Flags |= JITSymbolFlags::Weak; 67 68 if (Sym.getScope() == Scope::Default) 69 Flags |= JITSymbolFlags::Exported; 70 71 if (Sym.isCallable()) 72 Flags |= JITSymbolFlags::Callable; 73 74 return Flags; 75 } 76 77 class LinkGraphMaterializationUnit : public MaterializationUnit { 78 public: 79 static std::unique_ptr<LinkGraphMaterializationUnit> 80 Create(ObjectLinkingLayer &ObjLinkingLayer, std::unique_ptr<LinkGraph> G) { 81 auto LGI = scanLinkGraph(ObjLinkingLayer.getExecutionSession(), *G); 82 return std::unique_ptr<LinkGraphMaterializationUnit>( 83 new LinkGraphMaterializationUnit(ObjLinkingLayer, std::move(G), 84 std::move(LGI))); 85 } 86 87 StringRef getName() const override { return G->getName(); } 88 void materialize(std::unique_ptr<MaterializationResponsibility> MR) override { 89 ObjLinkingLayer.emit(std::move(MR), std::move(G)); 90 } 91 92 private: 93 static Interface scanLinkGraph(ExecutionSession &ES, LinkGraph &G) { 94 95 Interface LGI; 96 97 auto AddSymbol = [&](Symbol *Sym) { 98 // Skip local symbols. 99 if (Sym->getScope() == Scope::Local) 100 return; 101 assert(Sym->hasName() && "Anonymous non-local symbol?"); 102 103 LGI.SymbolFlags[ES.intern(Sym->getName())] = 104 getJITSymbolFlagsForSymbol(*Sym); 105 }; 106 107 for (auto *Sym : G.defined_symbols()) 108 AddSymbol(Sym); 109 for (auto *Sym : G.absolute_symbols()) 110 AddSymbol(Sym); 111 112 if (hasInitializerSection(G)) 113 LGI.InitSymbol = makeInitSymbol(ES, G); 114 115 return LGI; 116 } 117 118 static SymbolStringPtr makeInitSymbol(ExecutionSession &ES, LinkGraph &G) { 119 std::string InitSymString; 120 raw_string_ostream(InitSymString) 121 << "$." << G.getName() << ".__inits" << Counter++; 122 return ES.intern(InitSymString); 123 } 124 125 LinkGraphMaterializationUnit(ObjectLinkingLayer &ObjLinkingLayer, 126 std::unique_ptr<LinkGraph> G, Interface LGI) 127 : MaterializationUnit(std::move(LGI)), ObjLinkingLayer(ObjLinkingLayer), 128 G(std::move(G)) {} 129 130 void discard(const JITDylib &JD, const SymbolStringPtr &Name) override { 131 for (auto *Sym : G->defined_symbols()) 132 if (Sym->getName() == *Name) { 133 assert(Sym->getLinkage() == Linkage::Weak && 134 "Discarding non-weak definition"); 135 G->makeExternal(*Sym); 136 break; 137 } 138 } 139 140 ObjectLinkingLayer &ObjLinkingLayer; 141 std::unique_ptr<LinkGraph> G; 142 static std::atomic<uint64_t> Counter; 143 }; 144 145 std::atomic<uint64_t> LinkGraphMaterializationUnit::Counter{0}; 146 147 } // end anonymous namespace 148 149 namespace llvm { 150 namespace orc { 151 152 class ObjectLinkingLayerJITLinkContext final : public JITLinkContext { 153 public: 154 ObjectLinkingLayerJITLinkContext( 155 ObjectLinkingLayer &Layer, 156 std::unique_ptr<MaterializationResponsibility> MR, 157 std::unique_ptr<MemoryBuffer> ObjBuffer) 158 : JITLinkContext(&MR->getTargetJITDylib()), Layer(Layer), 159 MR(std::move(MR)), ObjBuffer(std::move(ObjBuffer)) { 160 std::lock_guard<std::mutex> Lock(Layer.LayerMutex); 161 Plugins = Layer.Plugins; 162 } 163 164 ~ObjectLinkingLayerJITLinkContext() { 165 // If there is an object buffer return function then use it to 166 // return ownership of the buffer. 167 if (Layer.ReturnObjectBuffer && ObjBuffer) 168 Layer.ReturnObjectBuffer(std::move(ObjBuffer)); 169 } 170 171 JITLinkMemoryManager &getMemoryManager() override { return Layer.MemMgr; } 172 173 void notifyMaterializing(LinkGraph &G) { 174 for (auto &P : Plugins) 175 P->notifyMaterializing(*MR, G, *this, 176 ObjBuffer ? ObjBuffer->getMemBufferRef() 177 : MemoryBufferRef()); 178 } 179 180 void notifyFailed(Error Err) override { 181 for (auto &P : Plugins) 182 Err = joinErrors(std::move(Err), P->notifyFailed(*MR)); 183 Layer.getExecutionSession().reportError(std::move(Err)); 184 MR->failMaterialization(); 185 } 186 187 void lookup(const LookupMap &Symbols, 188 std::unique_ptr<JITLinkAsyncLookupContinuation> LC) override { 189 190 JITDylibSearchOrder LinkOrder; 191 MR->getTargetJITDylib().withLinkOrderDo( 192 [&](const JITDylibSearchOrder &LO) { LinkOrder = LO; }); 193 194 auto &ES = Layer.getExecutionSession(); 195 196 SymbolLookupSet LookupSet; 197 for (auto &KV : Symbols) { 198 orc::SymbolLookupFlags LookupFlags; 199 switch (KV.second) { 200 case jitlink::SymbolLookupFlags::RequiredSymbol: 201 LookupFlags = orc::SymbolLookupFlags::RequiredSymbol; 202 break; 203 case jitlink::SymbolLookupFlags::WeaklyReferencedSymbol: 204 LookupFlags = orc::SymbolLookupFlags::WeaklyReferencedSymbol; 205 break; 206 } 207 LookupSet.add(ES.intern(KV.first), LookupFlags); 208 } 209 210 // OnResolve -- De-intern the symbols and pass the result to the linker. 211 auto OnResolve = [LookupContinuation = 212 std::move(LC)](Expected<SymbolMap> Result) mutable { 213 if (!Result) 214 LookupContinuation->run(Result.takeError()); 215 else { 216 AsyncLookupResult LR; 217 for (auto &KV : *Result) 218 LR[*KV.first] = KV.second; 219 LookupContinuation->run(std::move(LR)); 220 } 221 }; 222 223 ES.lookup(LookupKind::Static, LinkOrder, std::move(LookupSet), 224 SymbolState::Resolved, std::move(OnResolve), 225 [this](const SymbolDependenceMap &Deps) { 226 // Translate LookupDeps map to SymbolSourceJD. 227 for (auto &[DepJD, Deps] : Deps) 228 for (auto &DepSym : Deps) 229 SymbolSourceJDs[NonOwningSymbolStringPtr(DepSym)] = DepJD; 230 }); 231 } 232 233 Error notifyResolved(LinkGraph &G) override { 234 auto &ES = Layer.getExecutionSession(); 235 236 SymbolFlagsMap ExtraSymbolsToClaim; 237 bool AutoClaim = Layer.AutoClaimObjectSymbols; 238 239 SymbolMap InternedResult; 240 for (auto *Sym : G.defined_symbols()) 241 if (Sym->hasName() && Sym->getScope() != Scope::Local) { 242 auto InternedName = ES.intern(Sym->getName()); 243 auto Ptr = getJITSymbolPtrForSymbol(*Sym, G.getTargetTriple()); 244 auto Flags = getJITSymbolFlagsForSymbol(*Sym); 245 InternedResult[InternedName] = {Ptr, Flags}; 246 if (AutoClaim && !MR->getSymbols().count(InternedName)) { 247 assert(!ExtraSymbolsToClaim.count(InternedName) && 248 "Duplicate symbol to claim?"); 249 ExtraSymbolsToClaim[InternedName] = Flags; 250 } 251 } 252 253 for (auto *Sym : G.absolute_symbols()) 254 if (Sym->hasName() && Sym->getScope() != Scope::Local) { 255 auto InternedName = ES.intern(Sym->getName()); 256 auto Ptr = getJITSymbolPtrForSymbol(*Sym, G.getTargetTriple()); 257 auto Flags = getJITSymbolFlagsForSymbol(*Sym); 258 InternedResult[InternedName] = {Ptr, Flags}; 259 if (AutoClaim && !MR->getSymbols().count(InternedName)) { 260 assert(!ExtraSymbolsToClaim.count(InternedName) && 261 "Duplicate symbol to claim?"); 262 ExtraSymbolsToClaim[InternedName] = Flags; 263 } 264 } 265 266 if (!ExtraSymbolsToClaim.empty()) 267 if (auto Err = MR->defineMaterializing(ExtraSymbolsToClaim)) 268 return Err; 269 270 { 271 272 // Check that InternedResult matches up with MR->getSymbols(), overriding 273 // flags if requested. 274 // This guards against faulty transformations / compilers / object caches. 275 276 // First check that there aren't any missing symbols. 277 size_t NumMaterializationSideEffectsOnlySymbols = 0; 278 SymbolNameVector MissingSymbols; 279 for (auto &[Sym, Flags] : MR->getSymbols()) { 280 281 auto I = InternedResult.find(Sym); 282 283 // If this is a materialization-side-effects only symbol then bump 284 // the counter and remove in from the result, otherwise make sure that 285 // it's defined. 286 if (Flags.hasMaterializationSideEffectsOnly()) { 287 ++NumMaterializationSideEffectsOnlySymbols; 288 InternedResult.erase(Sym); 289 continue; 290 } else if (I == InternedResult.end()) 291 MissingSymbols.push_back(Sym); 292 else if (Layer.OverrideObjectFlags) 293 I->second.setFlags(Flags); 294 } 295 296 // If there were missing symbols then report the error. 297 if (!MissingSymbols.empty()) 298 return make_error<MissingSymbolDefinitions>( 299 Layer.getExecutionSession().getSymbolStringPool(), G.getName(), 300 std::move(MissingSymbols)); 301 302 // If there are more definitions than expected, add them to the 303 // ExtraSymbols vector. 304 SymbolNameVector ExtraSymbols; 305 if (InternedResult.size() > 306 MR->getSymbols().size() - NumMaterializationSideEffectsOnlySymbols) { 307 for (auto &KV : InternedResult) 308 if (!MR->getSymbols().count(KV.first)) 309 ExtraSymbols.push_back(KV.first); 310 } 311 312 // If there were extra definitions then report the error. 313 if (!ExtraSymbols.empty()) 314 return make_error<UnexpectedSymbolDefinitions>( 315 Layer.getExecutionSession().getSymbolStringPool(), G.getName(), 316 std::move(ExtraSymbols)); 317 } 318 319 if (auto Err = MR->notifyResolved(InternedResult)) 320 return Err; 321 322 notifyLoaded(); 323 return Error::success(); 324 } 325 326 void notifyFinalized(JITLinkMemoryManager::FinalizedAlloc A) override { 327 if (auto Err = notifyEmitted(std::move(A))) { 328 Layer.getExecutionSession().reportError(std::move(Err)); 329 MR->failMaterialization(); 330 return; 331 } 332 333 if (auto Err = MR->notifyEmitted(SymbolDepGroups)) { 334 Layer.getExecutionSession().reportError(std::move(Err)); 335 MR->failMaterialization(); 336 } 337 } 338 339 LinkGraphPassFunction getMarkLivePass(const Triple &TT) const override { 340 return [this](LinkGraph &G) { return markResponsibilitySymbolsLive(G); }; 341 } 342 343 Error modifyPassConfig(LinkGraph &LG, PassConfiguration &Config) override { 344 // Add passes to mark duplicate defs as should-discard, and to walk the 345 // link graph to build the symbol dependence graph. 346 Config.PrePrunePasses.push_back([this](LinkGraph &G) { 347 return claimOrExternalizeWeakAndCommonSymbols(G); 348 }); 349 350 for (auto &P : Plugins) 351 P->modifyPassConfig(*MR, LG, Config); 352 353 Config.PreFixupPasses.push_back( 354 [this](LinkGraph &G) { return registerDependencies(G); }); 355 356 return Error::success(); 357 } 358 359 void notifyLoaded() { 360 for (auto &P : Plugins) 361 P->notifyLoaded(*MR); 362 } 363 364 Error notifyEmitted(jitlink::JITLinkMemoryManager::FinalizedAlloc FA) { 365 Error Err = Error::success(); 366 for (auto &P : Plugins) 367 Err = joinErrors(std::move(Err), P->notifyEmitted(*MR)); 368 369 if (Err) { 370 if (FA) 371 Err = 372 joinErrors(std::move(Err), Layer.MemMgr.deallocate(std::move(FA))); 373 return Err; 374 } 375 376 if (FA) 377 return Layer.recordFinalizedAlloc(*MR, std::move(FA)); 378 379 return Error::success(); 380 } 381 382 private: 383 Error claimOrExternalizeWeakAndCommonSymbols(LinkGraph &G) { 384 auto &ES = Layer.getExecutionSession(); 385 386 SymbolFlagsMap NewSymbolsToClaim; 387 std::vector<std::pair<SymbolStringPtr, Symbol *>> NameToSym; 388 389 auto ProcessSymbol = [&](Symbol *Sym) { 390 if (Sym->hasName() && Sym->getLinkage() == Linkage::Weak && 391 Sym->getScope() != Scope::Local) { 392 auto Name = ES.intern(Sym->getName()); 393 if (!MR->getSymbols().count(ES.intern(Sym->getName()))) { 394 NewSymbolsToClaim[Name] = 395 getJITSymbolFlagsForSymbol(*Sym) | JITSymbolFlags::Weak; 396 NameToSym.push_back(std::make_pair(std::move(Name), Sym)); 397 } 398 } 399 }; 400 401 for (auto *Sym : G.defined_symbols()) 402 ProcessSymbol(Sym); 403 for (auto *Sym : G.absolute_symbols()) 404 ProcessSymbol(Sym); 405 406 // Attempt to claim all weak defs that we're not already responsible for. 407 // This may fail if the resource tracker has become defunct, but should 408 // always succeed otherwise. 409 if (auto Err = MR->defineMaterializing(std::move(NewSymbolsToClaim))) 410 return Err; 411 412 // Walk the list of symbols that we just tried to claim. Symbols that we're 413 // responsible for are marked live. Symbols that we're not responsible for 414 // are turned into external references. 415 for (auto &KV : NameToSym) { 416 if (MR->getSymbols().count(KV.first)) 417 KV.second->setLive(true); 418 else 419 G.makeExternal(*KV.second); 420 } 421 422 return Error::success(); 423 } 424 425 Error markResponsibilitySymbolsLive(LinkGraph &G) const { 426 auto &ES = Layer.getExecutionSession(); 427 for (auto *Sym : G.defined_symbols()) 428 if (Sym->hasName() && MR->getSymbols().count(ES.intern(Sym->getName()))) 429 Sym->setLive(true); 430 return Error::success(); 431 } 432 433 Error registerDependencies(LinkGraph &G) { 434 435 struct BlockInfo { 436 bool InWorklist = false; 437 DenseSet<Symbol *> Defs; 438 DenseSet<Symbol *> SymbolDeps; 439 DenseSet<Block *> AnonEdges, AnonBackEdges; 440 }; 441 442 DenseMap<Block *, BlockInfo> BlockInfos; 443 444 // Reserve space so that BlockInfos doesn't need to resize. This is 445 // essential to avoid invalidating pointers to entries below. 446 { 447 size_t NumBlocks = 0; 448 for (auto &Sec : G.sections()) 449 NumBlocks += Sec.blocks_size(); 450 BlockInfos.reserve(NumBlocks); 451 } 452 453 // Identify non-locally-scoped symbols defined by each block. 454 for (auto *Sym : G.defined_symbols()) { 455 if (Sym->getScope() != Scope::Local) 456 BlockInfos[&Sym->getBlock()].Defs.insert(Sym); 457 } 458 459 // Identify the symbolic and anonymous-block dependencies for each block. 460 for (auto *B : G.blocks()) { 461 auto &BI = BlockInfos[B]; 462 463 for (auto &E : B->edges()) { 464 465 // External symbols are trivially depended on. 466 if (E.getTarget().isExternal()) { 467 BI.SymbolDeps.insert(&E.getTarget()); 468 continue; 469 } 470 471 // Anonymous symbols aren't depended on at all (they're assumed to be 472 // already available). 473 if (E.getTarget().isAbsolute()) 474 continue; 475 476 // If we get here then we depend on a symbol defined by some other 477 // block. 478 auto &TgtBI = BlockInfos[&E.getTarget().getBlock()]; 479 480 // If that block has any definitions then use the first one as the 481 // "effective" dependence here (all symbols in TgtBI will become 482 // ready at the same time, and chosing a single symbol to represent 483 // the block keeps the SymbolDepGroup size small). 484 if (!TgtBI.Defs.empty()) { 485 BI.SymbolDeps.insert(*TgtBI.Defs.begin()); 486 continue; 487 } 488 489 // Otherwise we've got a dependence on an anonymous block. Record it 490 // here for back-propagating symbol dependencies below. 491 BI.AnonEdges.insert(&E.getTarget().getBlock()); 492 TgtBI.AnonBackEdges.insert(B); 493 } 494 } 495 496 // Prune anonymous blocks. 497 { 498 std::vector<Block *> BlocksToRemove; 499 for (auto &[B, BI] : BlockInfos) { 500 // Skip blocks with defs. We only care about anonyous blocks. 501 if (!BI.Defs.empty()) 502 continue; 503 504 BlocksToRemove.push_back(B); 505 506 for (auto *FB : BI.AnonEdges) 507 BlockInfos[FB].AnonBackEdges.erase(B); 508 509 for (auto *BB : BI.AnonBackEdges) 510 BlockInfos[BB].AnonEdges.erase(B); 511 512 for (auto *FB : BI.AnonEdges) { 513 auto &FBI = BlockInfos[FB]; 514 for (auto *BB : BI.AnonBackEdges) 515 FBI.AnonBackEdges.insert(BB); 516 } 517 518 for (auto *BB : BI.AnonBackEdges) { 519 auto &BBI = BlockInfos[BB]; 520 for (auto *SD : BI.SymbolDeps) 521 BBI.SymbolDeps.insert(SD); 522 for (auto *FB : BI.AnonEdges) 523 BBI.AnonEdges.insert(FB); 524 } 525 } 526 527 for (auto *B : BlocksToRemove) 528 BlockInfos.erase(B); 529 } 530 531 // Build the initial dependence propagation worklist. 532 std::deque<Block *> Worklist; 533 for (auto &[B, BI] : BlockInfos) { 534 if (!BI.SymbolDeps.empty() && !BI.AnonBackEdges.empty()) { 535 Worklist.push_back(B); 536 BI.InWorklist = true; 537 } 538 } 539 540 // Propagate symbol deps through the graph. 541 while (!Worklist.empty()) { 542 auto *B = Worklist.front(); 543 Worklist.pop_front(); 544 545 auto &BI = BlockInfos[B]; 546 BI.InWorklist = false; 547 548 for (auto *DB : BI.AnonBackEdges) { 549 auto &DBI = BlockInfos[DB]; 550 for (auto *Sym : BI.SymbolDeps) { 551 if (DBI.SymbolDeps.insert(Sym).second && !DBI.InWorklist) { 552 Worklist.push_back(DB); 553 DBI.InWorklist = true; 554 } 555 } 556 } 557 } 558 559 // Transform our local dependence information into a list of 560 // SymbolDependenceGroups (in the SymbolDepGroups member), ready for use in 561 // the upcoming notifyFinalized call. 562 auto &TargetJD = MR->getTargetJITDylib(); 563 auto &ES = TargetJD.getExecutionSession(); 564 565 DenseMap<Symbol *, SymbolStringPtr> InternedNames; 566 auto GetInternedName = [&](Symbol *S) { 567 auto &Name = InternedNames[S]; 568 if (!Name) 569 Name = ES.intern(S->getName()); 570 return Name; 571 }; 572 573 for (auto &[B, BI] : BlockInfos) { 574 if (!BI.Defs.empty()) { 575 SymbolDepGroups.push_back(SymbolDependenceGroup()); 576 auto &SDG = SymbolDepGroups.back(); 577 578 for (auto *Def : BI.Defs) 579 SDG.Symbols.insert(GetInternedName(Def)); 580 581 for (auto *Dep : BI.SymbolDeps) { 582 auto DepName = GetInternedName(Dep); 583 if (Dep->isDefined()) 584 SDG.Dependencies[&TargetJD].insert(std::move(DepName)); 585 else { 586 auto SourceJDItr = 587 SymbolSourceJDs.find(NonOwningSymbolStringPtr(DepName)); 588 if (SourceJDItr != SymbolSourceJDs.end()) 589 SDG.Dependencies[SourceJDItr->second].insert(std::move(DepName)); 590 } 591 } 592 } 593 } 594 595 return Error::success(); 596 } 597 598 ObjectLinkingLayer &Layer; 599 std::vector<std::shared_ptr<ObjectLinkingLayer::Plugin>> Plugins; 600 std::unique_ptr<MaterializationResponsibility> MR; 601 std::unique_ptr<MemoryBuffer> ObjBuffer; 602 DenseMap<NonOwningSymbolStringPtr, JITDylib *> SymbolSourceJDs; 603 std::vector<SymbolDependenceGroup> SymbolDepGroups; 604 }; 605 606 ObjectLinkingLayer::Plugin::~Plugin() = default; 607 608 char ObjectLinkingLayer::ID; 609 610 using BaseT = RTTIExtends<ObjectLinkingLayer, ObjectLayer>; 611 612 ObjectLinkingLayer::ObjectLinkingLayer(ExecutionSession &ES) 613 : BaseT(ES), MemMgr(ES.getExecutorProcessControl().getMemMgr()) { 614 ES.registerResourceManager(*this); 615 } 616 617 ObjectLinkingLayer::ObjectLinkingLayer(ExecutionSession &ES, 618 JITLinkMemoryManager &MemMgr) 619 : BaseT(ES), MemMgr(MemMgr) { 620 ES.registerResourceManager(*this); 621 } 622 623 ObjectLinkingLayer::ObjectLinkingLayer( 624 ExecutionSession &ES, std::unique_ptr<JITLinkMemoryManager> MemMgr) 625 : BaseT(ES), MemMgr(*MemMgr), MemMgrOwnership(std::move(MemMgr)) { 626 ES.registerResourceManager(*this); 627 } 628 629 ObjectLinkingLayer::~ObjectLinkingLayer() { 630 assert(Allocs.empty() && "Layer destroyed with resources still attached"); 631 getExecutionSession().deregisterResourceManager(*this); 632 } 633 634 Error ObjectLinkingLayer::add(ResourceTrackerSP RT, 635 std::unique_ptr<LinkGraph> G) { 636 auto &JD = RT->getJITDylib(); 637 return JD.define(LinkGraphMaterializationUnit::Create(*this, std::move(G)), 638 std::move(RT)); 639 } 640 641 void ObjectLinkingLayer::emit(std::unique_ptr<MaterializationResponsibility> R, 642 std::unique_ptr<MemoryBuffer> O) { 643 assert(O && "Object must not be null"); 644 MemoryBufferRef ObjBuffer = O->getMemBufferRef(); 645 646 auto Ctx = std::make_unique<ObjectLinkingLayerJITLinkContext>( 647 *this, std::move(R), std::move(O)); 648 649 if (auto G = createLinkGraphFromObject(ObjBuffer)) { 650 Ctx->notifyMaterializing(**G); 651 link(std::move(*G), std::move(Ctx)); 652 } else { 653 Ctx->notifyFailed(G.takeError()); 654 } 655 } 656 657 void ObjectLinkingLayer::emit(std::unique_ptr<MaterializationResponsibility> R, 658 std::unique_ptr<LinkGraph> G) { 659 auto Ctx = std::make_unique<ObjectLinkingLayerJITLinkContext>( 660 *this, std::move(R), nullptr); 661 Ctx->notifyMaterializing(*G); 662 link(std::move(G), std::move(Ctx)); 663 } 664 665 Error ObjectLinkingLayer::recordFinalizedAlloc( 666 MaterializationResponsibility &MR, FinalizedAlloc FA) { 667 auto Err = MR.withResourceKeyDo( 668 [&](ResourceKey K) { Allocs[K].push_back(std::move(FA)); }); 669 670 if (Err) 671 Err = joinErrors(std::move(Err), MemMgr.deallocate(std::move(FA))); 672 673 return Err; 674 } 675 676 Error ObjectLinkingLayer::handleRemoveResources(JITDylib &JD, ResourceKey K) { 677 678 { 679 Error Err = Error::success(); 680 for (auto &P : Plugins) 681 Err = joinErrors(std::move(Err), P->notifyRemovingResources(JD, K)); 682 if (Err) 683 return Err; 684 } 685 686 std::vector<FinalizedAlloc> AllocsToRemove; 687 getExecutionSession().runSessionLocked([&] { 688 auto I = Allocs.find(K); 689 if (I != Allocs.end()) { 690 std::swap(AllocsToRemove, I->second); 691 Allocs.erase(I); 692 } 693 }); 694 695 if (AllocsToRemove.empty()) 696 return Error::success(); 697 698 return MemMgr.deallocate(std::move(AllocsToRemove)); 699 } 700 701 void ObjectLinkingLayer::handleTransferResources(JITDylib &JD, 702 ResourceKey DstKey, 703 ResourceKey SrcKey) { 704 auto I = Allocs.find(SrcKey); 705 if (I != Allocs.end()) { 706 auto &SrcAllocs = I->second; 707 auto &DstAllocs = Allocs[DstKey]; 708 DstAllocs.reserve(DstAllocs.size() + SrcAllocs.size()); 709 for (auto &Alloc : SrcAllocs) 710 DstAllocs.push_back(std::move(Alloc)); 711 712 // Erase SrcKey entry using value rather than iterator I: I may have been 713 // invalidated when we looked up DstKey. 714 Allocs.erase(SrcKey); 715 } 716 717 for (auto &P : Plugins) 718 P->notifyTransferringResources(JD, DstKey, SrcKey); 719 } 720 721 EHFrameRegistrationPlugin::EHFrameRegistrationPlugin( 722 ExecutionSession &ES, std::unique_ptr<EHFrameRegistrar> Registrar) 723 : ES(ES), Registrar(std::move(Registrar)) {} 724 725 void EHFrameRegistrationPlugin::modifyPassConfig( 726 MaterializationResponsibility &MR, LinkGraph &G, 727 PassConfiguration &PassConfig) { 728 729 PassConfig.PostFixupPasses.push_back(createEHFrameRecorderPass( 730 G.getTargetTriple(), [this, &MR](ExecutorAddr Addr, size_t Size) { 731 if (Addr) { 732 std::lock_guard<std::mutex> Lock(EHFramePluginMutex); 733 assert(!InProcessLinks.count(&MR) && 734 "Link for MR already being tracked?"); 735 InProcessLinks[&MR] = {Addr, Size}; 736 } 737 })); 738 } 739 740 Error EHFrameRegistrationPlugin::notifyEmitted( 741 MaterializationResponsibility &MR) { 742 743 ExecutorAddrRange EmittedRange; 744 { 745 std::lock_guard<std::mutex> Lock(EHFramePluginMutex); 746 747 auto EHFrameRangeItr = InProcessLinks.find(&MR); 748 if (EHFrameRangeItr == InProcessLinks.end()) 749 return Error::success(); 750 751 EmittedRange = EHFrameRangeItr->second; 752 assert(EmittedRange.Start && "eh-frame addr to register can not be null"); 753 InProcessLinks.erase(EHFrameRangeItr); 754 } 755 756 if (auto Err = MR.withResourceKeyDo( 757 [&](ResourceKey K) { EHFrameRanges[K].push_back(EmittedRange); })) 758 return Err; 759 760 return Registrar->registerEHFrames(EmittedRange); 761 } 762 763 Error EHFrameRegistrationPlugin::notifyFailed( 764 MaterializationResponsibility &MR) { 765 std::lock_guard<std::mutex> Lock(EHFramePluginMutex); 766 InProcessLinks.erase(&MR); 767 return Error::success(); 768 } 769 770 Error EHFrameRegistrationPlugin::notifyRemovingResources(JITDylib &JD, 771 ResourceKey K) { 772 std::vector<ExecutorAddrRange> RangesToRemove; 773 774 ES.runSessionLocked([&] { 775 auto I = EHFrameRanges.find(K); 776 if (I != EHFrameRanges.end()) { 777 RangesToRemove = std::move(I->second); 778 EHFrameRanges.erase(I); 779 } 780 }); 781 782 Error Err = Error::success(); 783 while (!RangesToRemove.empty()) { 784 auto RangeToRemove = RangesToRemove.back(); 785 RangesToRemove.pop_back(); 786 assert(RangeToRemove.Start && "Untracked eh-frame range must not be null"); 787 Err = joinErrors(std::move(Err), 788 Registrar->deregisterEHFrames(RangeToRemove)); 789 } 790 791 return Err; 792 } 793 794 void EHFrameRegistrationPlugin::notifyTransferringResources( 795 JITDylib &JD, ResourceKey DstKey, ResourceKey SrcKey) { 796 auto SI = EHFrameRanges.find(SrcKey); 797 if (SI == EHFrameRanges.end()) 798 return; 799 800 auto DI = EHFrameRanges.find(DstKey); 801 if (DI != EHFrameRanges.end()) { 802 auto &SrcRanges = SI->second; 803 auto &DstRanges = DI->second; 804 DstRanges.reserve(DstRanges.size() + SrcRanges.size()); 805 for (auto &SrcRange : SrcRanges) 806 DstRanges.push_back(std::move(SrcRange)); 807 EHFrameRanges.erase(SI); 808 } else { 809 // We need to move SrcKey's ranges over without invalidating the SI 810 // iterator. 811 auto Tmp = std::move(SI->second); 812 EHFrameRanges.erase(SI); 813 EHFrameRanges[DstKey] = std::move(Tmp); 814 } 815 } 816 817 } // End namespace orc. 818 } // End namespace llvm. 819