1 //===------- ObjectLinkingLayer.cpp - JITLink backed ORC ObjectLayer ------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 9 #include "llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h" 10 #include "llvm/ExecutionEngine/JITLink/EHFrameSupport.h" 11 #include "llvm/ExecutionEngine/JITLink/aarch32.h" 12 #include "llvm/ExecutionEngine/Orc/DebugObjectManagerPlugin.h" 13 #include "llvm/ExecutionEngine/Orc/DebugUtils.h" 14 #include "llvm/ExecutionEngine/Orc/ObjectFileInterface.h" 15 #include "llvm/ExecutionEngine/Orc/Shared/ObjectFormats.h" 16 #include "llvm/Support/MemoryBuffer.h" 17 #include <string> 18 #include <vector> 19 20 #define DEBUG_TYPE "orc" 21 22 using namespace llvm; 23 using namespace llvm::jitlink; 24 using namespace llvm::orc; 25 26 namespace { 27 28 bool hasInitializerSection(jitlink::LinkGraph &G) { 29 bool IsMachO = G.getTargetTriple().isOSBinFormatMachO(); 30 bool IsElf = G.getTargetTriple().isOSBinFormatELF(); 31 if (!IsMachO && !IsElf) 32 return false; 33 34 for (auto &Sec : G.sections()) { 35 if (IsMachO && isMachOInitializerSection(Sec.getName())) 36 return true; 37 if (IsElf && isELFInitializerSection(Sec.getName())) 38 return true; 39 } 40 41 return false; 42 } 43 44 ExecutorAddr getJITSymbolPtrForSymbol(Symbol &Sym, const Triple &TT) { 45 switch (TT.getArch()) { 46 case Triple::arm: 47 case Triple::armeb: 48 case Triple::thumb: 49 case Triple::thumbeb: 50 if (hasTargetFlags(Sym, aarch32::ThumbSymbol)) { 51 // Set LSB to indicate thumb target 52 assert(Sym.isCallable() && "Only callable symbols can have thumb flag"); 53 assert((Sym.getAddress().getValue() & 0x01) == 0 && "LSB is clear"); 54 return Sym.getAddress() + 0x01; 55 } 56 return Sym.getAddress(); 57 default: 58 return Sym.getAddress(); 59 } 60 } 61 62 JITSymbolFlags getJITSymbolFlagsForSymbol(Symbol &Sym) { 63 JITSymbolFlags Flags; 64 65 if (Sym.getLinkage() == Linkage::Weak) 66 Flags |= JITSymbolFlags::Weak; 67 68 if (Sym.getScope() == Scope::Default) 69 Flags |= JITSymbolFlags::Exported; 70 71 if (Sym.isCallable()) 72 Flags |= JITSymbolFlags::Callable; 73 74 return Flags; 75 } 76 77 class LinkGraphMaterializationUnit : public MaterializationUnit { 78 public: 79 static std::unique_ptr<LinkGraphMaterializationUnit> 80 Create(ObjectLinkingLayer &ObjLinkingLayer, std::unique_ptr<LinkGraph> G) { 81 auto LGI = scanLinkGraph(ObjLinkingLayer.getExecutionSession(), *G); 82 return std::unique_ptr<LinkGraphMaterializationUnit>( 83 new LinkGraphMaterializationUnit(ObjLinkingLayer, std::move(G), 84 std::move(LGI))); 85 } 86 87 StringRef getName() const override { return G->getName(); } 88 void materialize(std::unique_ptr<MaterializationResponsibility> MR) override { 89 ObjLinkingLayer.emit(std::move(MR), std::move(G)); 90 } 91 92 private: 93 static Interface scanLinkGraph(ExecutionSession &ES, LinkGraph &G) { 94 95 Interface LGI; 96 97 auto AddSymbol = [&](Symbol *Sym) { 98 // Skip local symbols. 99 if (Sym->getScope() == Scope::Local) 100 return; 101 assert(Sym->hasName() && "Anonymous non-local symbol?"); 102 103 LGI.SymbolFlags[ES.intern(Sym->getName())] = 104 getJITSymbolFlagsForSymbol(*Sym); 105 }; 106 107 for (auto *Sym : G.defined_symbols()) 108 AddSymbol(Sym); 109 for (auto *Sym : G.absolute_symbols()) 110 AddSymbol(Sym); 111 112 if (hasInitializerSection(G)) 113 LGI.InitSymbol = makeInitSymbol(ES, G); 114 115 return LGI; 116 } 117 118 static SymbolStringPtr makeInitSymbol(ExecutionSession &ES, LinkGraph &G) { 119 std::string InitSymString; 120 raw_string_ostream(InitSymString) 121 << "$." << G.getName() << ".__inits" << Counter++; 122 return ES.intern(InitSymString); 123 } 124 125 LinkGraphMaterializationUnit(ObjectLinkingLayer &ObjLinkingLayer, 126 std::unique_ptr<LinkGraph> G, Interface LGI) 127 : MaterializationUnit(std::move(LGI)), ObjLinkingLayer(ObjLinkingLayer), 128 G(std::move(G)) {} 129 130 void discard(const JITDylib &JD, const SymbolStringPtr &Name) override { 131 for (auto *Sym : G->defined_symbols()) 132 if (Sym->getName() == *Name) { 133 assert(Sym->getLinkage() == Linkage::Weak && 134 "Discarding non-weak definition"); 135 G->makeExternal(*Sym); 136 break; 137 } 138 } 139 140 ObjectLinkingLayer &ObjLinkingLayer; 141 std::unique_ptr<LinkGraph> G; 142 static std::atomic<uint64_t> Counter; 143 }; 144 145 std::atomic<uint64_t> LinkGraphMaterializationUnit::Counter{0}; 146 147 } // end anonymous namespace 148 149 namespace llvm { 150 namespace orc { 151 152 class ObjectLinkingLayerJITLinkContext final : public JITLinkContext { 153 public: 154 ObjectLinkingLayerJITLinkContext( 155 ObjectLinkingLayer &Layer, 156 std::unique_ptr<MaterializationResponsibility> MR, 157 std::unique_ptr<MemoryBuffer> ObjBuffer) 158 : JITLinkContext(&MR->getTargetJITDylib()), Layer(Layer), 159 MR(std::move(MR)), ObjBuffer(std::move(ObjBuffer)) {} 160 161 ~ObjectLinkingLayerJITLinkContext() { 162 // If there is an object buffer return function then use it to 163 // return ownership of the buffer. 164 if (Layer.ReturnObjectBuffer && ObjBuffer) 165 Layer.ReturnObjectBuffer(std::move(ObjBuffer)); 166 } 167 168 JITLinkMemoryManager &getMemoryManager() override { return Layer.MemMgr; } 169 170 void notifyMaterializing(LinkGraph &G) { 171 for (auto &P : Layer.Plugins) 172 P->notifyMaterializing(*MR, G, *this, 173 ObjBuffer ? ObjBuffer->getMemBufferRef() 174 : MemoryBufferRef()); 175 } 176 177 void notifyFailed(Error Err) override { 178 for (auto &P : Layer.Plugins) 179 Err = joinErrors(std::move(Err), P->notifyFailed(*MR)); 180 Layer.getExecutionSession().reportError(std::move(Err)); 181 MR->failMaterialization(); 182 } 183 184 void lookup(const LookupMap &Symbols, 185 std::unique_ptr<JITLinkAsyncLookupContinuation> LC) override { 186 187 JITDylibSearchOrder LinkOrder; 188 MR->getTargetJITDylib().withLinkOrderDo( 189 [&](const JITDylibSearchOrder &LO) { LinkOrder = LO; }); 190 191 auto &ES = Layer.getExecutionSession(); 192 193 SymbolLookupSet LookupSet; 194 for (auto &KV : Symbols) { 195 orc::SymbolLookupFlags LookupFlags; 196 switch (KV.second) { 197 case jitlink::SymbolLookupFlags::RequiredSymbol: 198 LookupFlags = orc::SymbolLookupFlags::RequiredSymbol; 199 break; 200 case jitlink::SymbolLookupFlags::WeaklyReferencedSymbol: 201 LookupFlags = orc::SymbolLookupFlags::WeaklyReferencedSymbol; 202 break; 203 } 204 LookupSet.add(ES.intern(KV.first), LookupFlags); 205 } 206 207 // OnResolve -- De-intern the symbols and pass the result to the linker. 208 auto OnResolve = [LookupContinuation = 209 std::move(LC)](Expected<SymbolMap> Result) mutable { 210 if (!Result) 211 LookupContinuation->run(Result.takeError()); 212 else { 213 AsyncLookupResult LR; 214 for (auto &KV : *Result) 215 LR[*KV.first] = KV.second; 216 LookupContinuation->run(std::move(LR)); 217 } 218 }; 219 220 ES.lookup(LookupKind::Static, LinkOrder, std::move(LookupSet), 221 SymbolState::Resolved, std::move(OnResolve), 222 [this](const SymbolDependenceMap &Deps) { 223 // Translate LookupDeps map to SymbolSourceJD. 224 for (auto &[DepJD, Deps] : Deps) 225 for (auto &DepSym : Deps) 226 SymbolSourceJDs[NonOwningSymbolStringPtr(DepSym)] = DepJD; 227 }); 228 } 229 230 Error notifyResolved(LinkGraph &G) override { 231 auto &ES = Layer.getExecutionSession(); 232 233 SymbolFlagsMap ExtraSymbolsToClaim; 234 bool AutoClaim = Layer.AutoClaimObjectSymbols; 235 236 SymbolMap InternedResult; 237 for (auto *Sym : G.defined_symbols()) 238 if (Sym->hasName() && Sym->getScope() != Scope::Local) { 239 auto InternedName = ES.intern(Sym->getName()); 240 auto Ptr = getJITSymbolPtrForSymbol(*Sym, G.getTargetTriple()); 241 auto Flags = getJITSymbolFlagsForSymbol(*Sym); 242 InternedResult[InternedName] = {Ptr, Flags}; 243 if (AutoClaim && !MR->getSymbols().count(InternedName)) { 244 assert(!ExtraSymbolsToClaim.count(InternedName) && 245 "Duplicate symbol to claim?"); 246 ExtraSymbolsToClaim[InternedName] = Flags; 247 } 248 } 249 250 for (auto *Sym : G.absolute_symbols()) 251 if (Sym->hasName() && Sym->getScope() != Scope::Local) { 252 auto InternedName = ES.intern(Sym->getName()); 253 auto Ptr = getJITSymbolPtrForSymbol(*Sym, G.getTargetTriple()); 254 auto Flags = getJITSymbolFlagsForSymbol(*Sym); 255 InternedResult[InternedName] = {Ptr, Flags}; 256 if (AutoClaim && !MR->getSymbols().count(InternedName)) { 257 assert(!ExtraSymbolsToClaim.count(InternedName) && 258 "Duplicate symbol to claim?"); 259 ExtraSymbolsToClaim[InternedName] = Flags; 260 } 261 } 262 263 if (!ExtraSymbolsToClaim.empty()) 264 if (auto Err = MR->defineMaterializing(ExtraSymbolsToClaim)) 265 return Err; 266 267 { 268 269 // Check that InternedResult matches up with MR->getSymbols(), overriding 270 // flags if requested. 271 // This guards against faulty transformations / compilers / object caches. 272 273 // First check that there aren't any missing symbols. 274 size_t NumMaterializationSideEffectsOnlySymbols = 0; 275 SymbolNameVector ExtraSymbols; 276 SymbolNameVector MissingSymbols; 277 for (auto &KV : MR->getSymbols()) { 278 279 auto I = InternedResult.find(KV.first); 280 281 // If this is a materialization-side-effects only symbol then bump 282 // the counter and make sure it's *not* defined, otherwise make 283 // sure that it is defined. 284 if (KV.second.hasMaterializationSideEffectsOnly()) { 285 ++NumMaterializationSideEffectsOnlySymbols; 286 if (I != InternedResult.end()) 287 ExtraSymbols.push_back(KV.first); 288 continue; 289 } else if (I == InternedResult.end()) 290 MissingSymbols.push_back(KV.first); 291 else if (Layer.OverrideObjectFlags) 292 I->second.setFlags(KV.second); 293 } 294 295 // If there were missing symbols then report the error. 296 if (!MissingSymbols.empty()) 297 return make_error<MissingSymbolDefinitions>( 298 Layer.getExecutionSession().getSymbolStringPool(), G.getName(), 299 std::move(MissingSymbols)); 300 301 // If there are more definitions than expected, add them to the 302 // ExtraSymbols vector. 303 if (InternedResult.size() > 304 MR->getSymbols().size() - NumMaterializationSideEffectsOnlySymbols) { 305 for (auto &KV : InternedResult) 306 if (!MR->getSymbols().count(KV.first)) 307 ExtraSymbols.push_back(KV.first); 308 } 309 310 // If there were extra definitions then report the error. 311 if (!ExtraSymbols.empty()) 312 return make_error<UnexpectedSymbolDefinitions>( 313 Layer.getExecutionSession().getSymbolStringPool(), G.getName(), 314 std::move(ExtraSymbols)); 315 } 316 317 if (auto Err = MR->notifyResolved(InternedResult)) 318 return Err; 319 320 Layer.notifyLoaded(*MR); 321 return Error::success(); 322 } 323 324 void notifyFinalized(JITLinkMemoryManager::FinalizedAlloc A) override { 325 if (auto Err = Layer.notifyEmitted(*MR, std::move(A))) { 326 Layer.getExecutionSession().reportError(std::move(Err)); 327 MR->failMaterialization(); 328 return; 329 } 330 if (auto Err = MR->notifyEmitted(SymbolDepGroups)) { 331 Layer.getExecutionSession().reportError(std::move(Err)); 332 MR->failMaterialization(); 333 } 334 } 335 336 LinkGraphPassFunction getMarkLivePass(const Triple &TT) const override { 337 return [this](LinkGraph &G) { return markResponsibilitySymbolsLive(G); }; 338 } 339 340 Error modifyPassConfig(LinkGraph &LG, PassConfiguration &Config) override { 341 // Add passes to mark duplicate defs as should-discard, and to walk the 342 // link graph to build the symbol dependence graph. 343 Config.PrePrunePasses.push_back([this](LinkGraph &G) { 344 return claimOrExternalizeWeakAndCommonSymbols(G); 345 }); 346 347 Layer.modifyPassConfig(*MR, LG, Config); 348 349 Config.PreFixupPasses.push_back( 350 [this](LinkGraph &G) { return registerDependencies(G); }); 351 352 return Error::success(); 353 } 354 355 private: 356 // Symbol name dependencies: 357 // Internal: Defined in this graph. 358 // External: Defined externally. 359 struct BlockSymbolDependencies { 360 SymbolNameSet Internal, External; 361 }; 362 363 // Lazily populated map of blocks to BlockSymbolDependencies values. 364 class BlockDependenciesMap { 365 public: 366 BlockDependenciesMap(ExecutionSession &ES, 367 DenseMap<const Block *, DenseSet<Block *>> BlockDeps) 368 : ES(ES), BlockDeps(std::move(BlockDeps)) {} 369 370 const BlockSymbolDependencies &operator[](const Block &B) { 371 // Check the cache first. 372 auto I = BlockTransitiveDepsCache.find(&B); 373 if (I != BlockTransitiveDepsCache.end()) 374 return I->second; 375 376 // No value. Populate the cache. 377 BlockSymbolDependencies BTDCacheVal; 378 auto BDI = BlockDeps.find(&B); 379 assert(BDI != BlockDeps.end() && "No block dependencies"); 380 381 for (auto *BDep : BDI->second) { 382 auto &BID = getBlockImmediateDeps(*BDep); 383 for (auto &ExternalDep : BID.External) 384 BTDCacheVal.External.insert(ExternalDep); 385 for (auto &InternalDep : BID.Internal) 386 BTDCacheVal.Internal.insert(InternalDep); 387 } 388 389 return BlockTransitiveDepsCache 390 .insert(std::make_pair(&B, std::move(BTDCacheVal))) 391 .first->second; 392 } 393 394 SymbolStringPtr &getInternedName(Symbol &Sym) { 395 auto I = NameCache.find(&Sym); 396 if (I != NameCache.end()) 397 return I->second; 398 399 return NameCache.insert(std::make_pair(&Sym, ES.intern(Sym.getName()))) 400 .first->second; 401 } 402 403 private: 404 BlockSymbolDependencies &getBlockImmediateDeps(Block &B) { 405 // Check the cache first. 406 auto I = BlockImmediateDepsCache.find(&B); 407 if (I != BlockImmediateDepsCache.end()) 408 return I->second; 409 410 BlockSymbolDependencies BIDCacheVal; 411 for (auto &E : B.edges()) { 412 auto &Tgt = E.getTarget(); 413 if (Tgt.getScope() != Scope::Local) { 414 if (Tgt.isExternal()) { 415 if (Tgt.getAddress() || !Tgt.isWeaklyReferenced()) 416 BIDCacheVal.External.insert(getInternedName(Tgt)); 417 } else 418 BIDCacheVal.Internal.insert(getInternedName(Tgt)); 419 } 420 } 421 422 return BlockImmediateDepsCache 423 .insert(std::make_pair(&B, std::move(BIDCacheVal))) 424 .first->second; 425 } 426 427 ExecutionSession &ES; 428 DenseMap<const Block *, DenseSet<Block *>> BlockDeps; 429 DenseMap<const Symbol *, SymbolStringPtr> NameCache; 430 DenseMap<const Block *, BlockSymbolDependencies> BlockImmediateDepsCache; 431 DenseMap<const Block *, BlockSymbolDependencies> BlockTransitiveDepsCache; 432 }; 433 434 Error claimOrExternalizeWeakAndCommonSymbols(LinkGraph &G) { 435 auto &ES = Layer.getExecutionSession(); 436 437 SymbolFlagsMap NewSymbolsToClaim; 438 std::vector<std::pair<SymbolStringPtr, Symbol *>> NameToSym; 439 440 auto ProcessSymbol = [&](Symbol *Sym) { 441 if (Sym->hasName() && Sym->getLinkage() == Linkage::Weak && 442 Sym->getScope() != Scope::Local) { 443 auto Name = ES.intern(Sym->getName()); 444 if (!MR->getSymbols().count(ES.intern(Sym->getName()))) { 445 NewSymbolsToClaim[Name] = 446 getJITSymbolFlagsForSymbol(*Sym) | JITSymbolFlags::Weak; 447 NameToSym.push_back(std::make_pair(std::move(Name), Sym)); 448 } 449 } 450 }; 451 452 for (auto *Sym : G.defined_symbols()) 453 ProcessSymbol(Sym); 454 for (auto *Sym : G.absolute_symbols()) 455 ProcessSymbol(Sym); 456 457 // Attempt to claim all weak defs that we're not already responsible for. 458 // This may fail if the resource tracker has become defunct, but should 459 // always succeed otherwise. 460 if (auto Err = MR->defineMaterializing(std::move(NewSymbolsToClaim))) 461 return Err; 462 463 // Walk the list of symbols that we just tried to claim. Symbols that we're 464 // responsible for are marked live. Symbols that we're not responsible for 465 // are turned into external references. 466 for (auto &KV : NameToSym) { 467 if (MR->getSymbols().count(KV.first)) 468 KV.second->setLive(true); 469 else 470 G.makeExternal(*KV.second); 471 } 472 473 return Error::success(); 474 } 475 476 Error markResponsibilitySymbolsLive(LinkGraph &G) const { 477 auto &ES = Layer.getExecutionSession(); 478 for (auto *Sym : G.defined_symbols()) 479 if (Sym->hasName() && MR->getSymbols().count(ES.intern(Sym->getName()))) 480 Sym->setLive(true); 481 return Error::success(); 482 } 483 484 Error registerDependencies(LinkGraph &G) { 485 auto &TargetJD = MR->getTargetJITDylib(); 486 auto &ES = TargetJD.getExecutionSession(); 487 auto BlockDeps = computeBlockNonLocalDeps(G); 488 489 DenseSet<Block *> BlockDepsProcessed; 490 DenseMap<Block *, SymbolDependenceGroup> DepGroupForBlock; 491 492 // Compute dependencies for symbols defined in the JITLink graph. 493 for (auto *Sym : G.defined_symbols()) { 494 495 // Skip local symbols. 496 if (Sym->getScope() == Scope::Local) 497 continue; 498 assert(Sym->hasName() && 499 "Defined non-local jitlink::Symbol should have a name"); 500 501 auto &BDeps = BlockDeps[Sym->getBlock()]; 502 503 // Skip symbols in blocks that don't depend on anything. 504 if (BDeps.Internal.empty() && BDeps.External.empty()) 505 continue; 506 507 SymbolDependenceGroup &SDG = DepGroupForBlock[&Sym->getBlock()]; 508 SDG.Symbols.insert(ES.intern(Sym->getName())); 509 510 if (!BlockDepsProcessed.count(&Sym->getBlock())) { 511 BlockDepsProcessed.insert(&Sym->getBlock()); 512 513 if (!BDeps.Internal.empty()) 514 SDG.Dependencies[&TargetJD] = BDeps.Internal; 515 for (auto &Dep : BDeps.External) { 516 auto DepSrcItr = SymbolSourceJDs.find(NonOwningSymbolStringPtr(Dep)); 517 if (DepSrcItr != SymbolSourceJDs.end()) 518 SDG.Dependencies[DepSrcItr->second].insert(Dep); 519 } 520 } 521 } 522 523 SymbolDependenceGroup SynthSDG; 524 525 for (auto &P : Layer.Plugins) { 526 auto SynthDeps = P->getSyntheticSymbolDependencies(*MR); 527 if (SynthDeps.empty()) 528 continue; 529 530 DenseSet<Block *> BlockVisited; 531 for (auto &[Name, DepSyms] : SynthDeps) { 532 SynthSDG.Symbols.insert(Name); 533 for (auto *Sym : DepSyms) { 534 if (Sym->getScope() == Scope::Local) { 535 auto &BDeps = BlockDeps[Sym->getBlock()]; 536 for (auto &S : BDeps.Internal) 537 SynthSDG.Dependencies[&TargetJD].insert(S); 538 for (auto &S : BDeps.External) { 539 auto DepSrcItr = 540 SymbolSourceJDs.find(NonOwningSymbolStringPtr(S)); 541 if (DepSrcItr != SymbolSourceJDs.end()) 542 SynthSDG.Dependencies[DepSrcItr->second].insert(S); 543 } 544 } else { 545 auto SymName = ES.intern(Sym->getName()); 546 if (Sym->isExternal()) { 547 assert(SymbolSourceJDs.count(NonOwningSymbolStringPtr(SymName)) && 548 "External symbol source entry missing"); 549 SynthSDG 550 .Dependencies[SymbolSourceJDs[NonOwningSymbolStringPtr( 551 SymName)]] 552 .insert(SymName); 553 } else 554 SynthSDG.Dependencies[&TargetJD].insert(SymName); 555 } 556 } 557 } 558 } 559 560 // Transfer SDGs to SymbolDepGroups. 561 DepGroupForBlock.reserve(DepGroupForBlock.size() + 1); 562 for (auto &[B, SDG] : DepGroupForBlock) { 563 assert(!SDG.Symbols.empty() && "SymbolDependenceGroup covers no symbols"); 564 if (!SDG.Dependencies.empty()) 565 SymbolDepGroups.push_back(std::move(SDG)); 566 } 567 if (!SynthSDG.Symbols.empty() && !SynthSDG.Dependencies.empty()) 568 SymbolDepGroups.push_back(std::move(SynthSDG)); 569 570 return Error::success(); 571 } 572 573 BlockDependenciesMap computeBlockNonLocalDeps(LinkGraph &G) { 574 // First calculate the reachable-via-non-local-symbol blocks for each block. 575 struct BlockInfo { 576 DenseSet<Block *> Dependencies; 577 DenseSet<Block *> Dependants; 578 bool DependenciesChanged = true; 579 }; 580 DenseMap<Block *, BlockInfo> BlockInfos; 581 SmallVector<Block *> WorkList; 582 583 // Pre-allocate map entries. This prevents any iterator/reference 584 // invalidation in the next loop. 585 for (auto *B : G.blocks()) 586 (void)BlockInfos[B]; 587 588 // Build initial worklist, record block dependencies/dependants and 589 // non-local symbol dependencies. 590 for (auto *B : G.blocks()) { 591 auto &BI = BlockInfos[B]; 592 for (auto &E : B->edges()) { 593 if (E.getTarget().getScope() == Scope::Local && 594 !E.getTarget().isAbsolute()) { 595 auto &TgtB = E.getTarget().getBlock(); 596 if (&TgtB != B) { 597 BI.Dependencies.insert(&TgtB); 598 BlockInfos[&TgtB].Dependants.insert(B); 599 } 600 } 601 } 602 603 // If this node has both dependants and dependencies then add it to the 604 // worklist to propagate the dependencies to the dependants. 605 if (!BI.Dependants.empty() && !BI.Dependencies.empty()) 606 WorkList.push_back(B); 607 } 608 609 // Propagate block-level dependencies through the block-dependence graph. 610 while (!WorkList.empty()) { 611 auto *B = WorkList.pop_back_val(); 612 613 auto &BI = BlockInfos[B]; 614 assert(BI.DependenciesChanged && 615 "Block in worklist has unchanged dependencies"); 616 BI.DependenciesChanged = false; 617 for (auto *Dependant : BI.Dependants) { 618 auto &DependantBI = BlockInfos[Dependant]; 619 for (auto *Dependency : BI.Dependencies) { 620 if (Dependant != Dependency && 621 DependantBI.Dependencies.insert(Dependency).second) 622 if (!DependantBI.DependenciesChanged) { 623 DependantBI.DependenciesChanged = true; 624 WorkList.push_back(Dependant); 625 } 626 } 627 } 628 } 629 630 DenseMap<const Block *, DenseSet<Block *>> BlockDeps; 631 for (auto &KV : BlockInfos) 632 BlockDeps[KV.first] = std::move(KV.second.Dependencies); 633 634 return BlockDependenciesMap(Layer.getExecutionSession(), 635 std::move(BlockDeps)); 636 } 637 638 ObjectLinkingLayer &Layer; 639 std::unique_ptr<MaterializationResponsibility> MR; 640 std::unique_ptr<MemoryBuffer> ObjBuffer; 641 DenseMap<Block *, SymbolNameSet> ExternalBlockDeps; 642 DenseMap<Block *, SymbolNameSet> InternalBlockDeps; 643 DenseMap<NonOwningSymbolStringPtr, JITDylib *> SymbolSourceJDs; 644 std::vector<SymbolDependenceGroup> SymbolDepGroups; 645 }; 646 647 ObjectLinkingLayer::Plugin::~Plugin() = default; 648 649 char ObjectLinkingLayer::ID; 650 651 using BaseT = RTTIExtends<ObjectLinkingLayer, ObjectLayer>; 652 653 ObjectLinkingLayer::ObjectLinkingLayer(ExecutionSession &ES) 654 : BaseT(ES), MemMgr(ES.getExecutorProcessControl().getMemMgr()) { 655 ES.registerResourceManager(*this); 656 } 657 658 ObjectLinkingLayer::ObjectLinkingLayer(ExecutionSession &ES, 659 JITLinkMemoryManager &MemMgr) 660 : BaseT(ES), MemMgr(MemMgr) { 661 ES.registerResourceManager(*this); 662 } 663 664 ObjectLinkingLayer::ObjectLinkingLayer( 665 ExecutionSession &ES, std::unique_ptr<JITLinkMemoryManager> MemMgr) 666 : BaseT(ES), MemMgr(*MemMgr), MemMgrOwnership(std::move(MemMgr)) { 667 ES.registerResourceManager(*this); 668 } 669 670 ObjectLinkingLayer::~ObjectLinkingLayer() { 671 assert(Allocs.empty() && "Layer destroyed with resources still attached"); 672 getExecutionSession().deregisterResourceManager(*this); 673 } 674 675 Error ObjectLinkingLayer::add(ResourceTrackerSP RT, 676 std::unique_ptr<LinkGraph> G) { 677 auto &JD = RT->getJITDylib(); 678 return JD.define(LinkGraphMaterializationUnit::Create(*this, std::move(G)), 679 std::move(RT)); 680 } 681 682 void ObjectLinkingLayer::emit(std::unique_ptr<MaterializationResponsibility> R, 683 std::unique_ptr<MemoryBuffer> O) { 684 assert(O && "Object must not be null"); 685 MemoryBufferRef ObjBuffer = O->getMemBufferRef(); 686 687 auto Ctx = std::make_unique<ObjectLinkingLayerJITLinkContext>( 688 *this, std::move(R), std::move(O)); 689 if (auto G = createLinkGraphFromObject(ObjBuffer)) { 690 Ctx->notifyMaterializing(**G); 691 link(std::move(*G), std::move(Ctx)); 692 } else { 693 Ctx->notifyFailed(G.takeError()); 694 } 695 } 696 697 void ObjectLinkingLayer::emit(std::unique_ptr<MaterializationResponsibility> R, 698 std::unique_ptr<LinkGraph> G) { 699 auto Ctx = std::make_unique<ObjectLinkingLayerJITLinkContext>( 700 *this, std::move(R), nullptr); 701 Ctx->notifyMaterializing(*G); 702 link(std::move(G), std::move(Ctx)); 703 } 704 705 void ObjectLinkingLayer::modifyPassConfig(MaterializationResponsibility &MR, 706 LinkGraph &G, 707 PassConfiguration &PassConfig) { 708 for (auto &P : Plugins) 709 P->modifyPassConfig(MR, G, PassConfig); 710 } 711 712 void ObjectLinkingLayer::notifyLoaded(MaterializationResponsibility &MR) { 713 for (auto &P : Plugins) 714 P->notifyLoaded(MR); 715 } 716 717 Error ObjectLinkingLayer::notifyEmitted(MaterializationResponsibility &MR, 718 FinalizedAlloc FA) { 719 Error Err = Error::success(); 720 for (auto &P : Plugins) 721 Err = joinErrors(std::move(Err), P->notifyEmitted(MR)); 722 723 if (Err) 724 return Err; 725 726 if (!FA) 727 return Error::success(); 728 729 return MR.withResourceKeyDo( 730 [&](ResourceKey K) { Allocs[K].push_back(std::move(FA)); }); 731 } 732 733 Error ObjectLinkingLayer::handleRemoveResources(JITDylib &JD, ResourceKey K) { 734 735 { 736 Error Err = Error::success(); 737 for (auto &P : Plugins) 738 Err = joinErrors(std::move(Err), P->notifyRemovingResources(JD, K)); 739 if (Err) 740 return Err; 741 } 742 743 std::vector<FinalizedAlloc> AllocsToRemove; 744 getExecutionSession().runSessionLocked([&] { 745 auto I = Allocs.find(K); 746 if (I != Allocs.end()) { 747 std::swap(AllocsToRemove, I->second); 748 Allocs.erase(I); 749 } 750 }); 751 752 if (AllocsToRemove.empty()) 753 return Error::success(); 754 755 return MemMgr.deallocate(std::move(AllocsToRemove)); 756 } 757 758 void ObjectLinkingLayer::handleTransferResources(JITDylib &JD, 759 ResourceKey DstKey, 760 ResourceKey SrcKey) { 761 auto I = Allocs.find(SrcKey); 762 if (I != Allocs.end()) { 763 auto &SrcAllocs = I->second; 764 auto &DstAllocs = Allocs[DstKey]; 765 DstAllocs.reserve(DstAllocs.size() + SrcAllocs.size()); 766 for (auto &Alloc : SrcAllocs) 767 DstAllocs.push_back(std::move(Alloc)); 768 769 // Erase SrcKey entry using value rather than iterator I: I may have been 770 // invalidated when we looked up DstKey. 771 Allocs.erase(SrcKey); 772 } 773 774 for (auto &P : Plugins) 775 P->notifyTransferringResources(JD, DstKey, SrcKey); 776 } 777 778 EHFrameRegistrationPlugin::EHFrameRegistrationPlugin( 779 ExecutionSession &ES, std::unique_ptr<EHFrameRegistrar> Registrar) 780 : ES(ES), Registrar(std::move(Registrar)) {} 781 782 void EHFrameRegistrationPlugin::modifyPassConfig( 783 MaterializationResponsibility &MR, LinkGraph &G, 784 PassConfiguration &PassConfig) { 785 786 PassConfig.PostFixupPasses.push_back(createEHFrameRecorderPass( 787 G.getTargetTriple(), [this, &MR](ExecutorAddr Addr, size_t Size) { 788 if (Addr) { 789 std::lock_guard<std::mutex> Lock(EHFramePluginMutex); 790 assert(!InProcessLinks.count(&MR) && 791 "Link for MR already being tracked?"); 792 InProcessLinks[&MR] = {Addr, Size}; 793 } 794 })); 795 } 796 797 Error EHFrameRegistrationPlugin::notifyEmitted( 798 MaterializationResponsibility &MR) { 799 800 ExecutorAddrRange EmittedRange; 801 { 802 std::lock_guard<std::mutex> Lock(EHFramePluginMutex); 803 804 auto EHFrameRangeItr = InProcessLinks.find(&MR); 805 if (EHFrameRangeItr == InProcessLinks.end()) 806 return Error::success(); 807 808 EmittedRange = EHFrameRangeItr->second; 809 assert(EmittedRange.Start && "eh-frame addr to register can not be null"); 810 InProcessLinks.erase(EHFrameRangeItr); 811 } 812 813 if (auto Err = MR.withResourceKeyDo( 814 [&](ResourceKey K) { EHFrameRanges[K].push_back(EmittedRange); })) 815 return Err; 816 817 return Registrar->registerEHFrames(EmittedRange); 818 } 819 820 Error EHFrameRegistrationPlugin::notifyFailed( 821 MaterializationResponsibility &MR) { 822 std::lock_guard<std::mutex> Lock(EHFramePluginMutex); 823 InProcessLinks.erase(&MR); 824 return Error::success(); 825 } 826 827 Error EHFrameRegistrationPlugin::notifyRemovingResources(JITDylib &JD, 828 ResourceKey K) { 829 std::vector<ExecutorAddrRange> RangesToRemove; 830 831 ES.runSessionLocked([&] { 832 auto I = EHFrameRanges.find(K); 833 if (I != EHFrameRanges.end()) { 834 RangesToRemove = std::move(I->second); 835 EHFrameRanges.erase(I); 836 } 837 }); 838 839 Error Err = Error::success(); 840 while (!RangesToRemove.empty()) { 841 auto RangeToRemove = RangesToRemove.back(); 842 RangesToRemove.pop_back(); 843 assert(RangeToRemove.Start && "Untracked eh-frame range must not be null"); 844 Err = joinErrors(std::move(Err), 845 Registrar->deregisterEHFrames(RangeToRemove)); 846 } 847 848 return Err; 849 } 850 851 void EHFrameRegistrationPlugin::notifyTransferringResources( 852 JITDylib &JD, ResourceKey DstKey, ResourceKey SrcKey) { 853 auto SI = EHFrameRanges.find(SrcKey); 854 if (SI == EHFrameRanges.end()) 855 return; 856 857 auto DI = EHFrameRanges.find(DstKey); 858 if (DI != EHFrameRanges.end()) { 859 auto &SrcRanges = SI->second; 860 auto &DstRanges = DI->second; 861 DstRanges.reserve(DstRanges.size() + SrcRanges.size()); 862 for (auto &SrcRange : SrcRanges) 863 DstRanges.push_back(std::move(SrcRange)); 864 EHFrameRanges.erase(SI); 865 } else { 866 // We need to move SrcKey's ranges over without invalidating the SI 867 // iterator. 868 auto Tmp = std::move(SI->second); 869 EHFrameRanges.erase(SI); 870 EHFrameRanges[DstKey] = std::move(Tmp); 871 } 872 } 873 874 } // End namespace orc. 875 } // End namespace llvm. 876