1 //===------- ObjectLinkingLayer.cpp - JITLink backed ORC ObjectLayer ------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 9 #include "llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h" 10 #include "llvm/ExecutionEngine/JITLink/EHFrameSupport.h" 11 #include "llvm/ExecutionEngine/JITLink/aarch32.h" 12 #include "llvm/ExecutionEngine/Orc/DebugObjectManagerPlugin.h" 13 #include "llvm/ExecutionEngine/Orc/DebugUtils.h" 14 #include "llvm/ExecutionEngine/Orc/ObjectFileInterface.h" 15 #include "llvm/ExecutionEngine/Orc/Shared/ObjectFormats.h" 16 #include "llvm/Support/MemoryBuffer.h" 17 #include <string> 18 #include <vector> 19 20 #define DEBUG_TYPE "orc" 21 22 using namespace llvm; 23 using namespace llvm::jitlink; 24 using namespace llvm::orc; 25 26 namespace { 27 28 bool hasInitializerSection(jitlink::LinkGraph &G) { 29 bool IsMachO = G.getTargetTriple().isOSBinFormatMachO(); 30 bool IsElf = G.getTargetTriple().isOSBinFormatELF(); 31 if (!IsMachO && !IsElf) 32 return false; 33 34 for (auto &Sec : G.sections()) { 35 if (IsMachO && isMachOInitializerSection(Sec.getName())) 36 return true; 37 if (IsElf && isELFInitializerSection(Sec.getName())) 38 return true; 39 } 40 41 return false; 42 } 43 44 ExecutorAddr getJITSymbolPtrForSymbol(Symbol &Sym, const Triple &TT) { 45 switch (TT.getArch()) { 46 case Triple::arm: 47 case Triple::armeb: 48 case Triple::thumb: 49 case Triple::thumbeb: 50 if (hasTargetFlags(Sym, aarch32::ThumbSymbol)) { 51 // Set LSB to indicate thumb target 52 assert(Sym.isCallable() && "Only callable symbols can have thumb flag"); 53 assert((Sym.getAddress().getValue() & 0x01) == 0 && "LSB is clear"); 54 return Sym.getAddress() + 0x01; 55 } 56 return Sym.getAddress(); 57 default: 58 return Sym.getAddress(); 59 } 60 } 61 62 JITSymbolFlags getJITSymbolFlagsForSymbol(Symbol &Sym) { 63 JITSymbolFlags Flags; 64 65 if (Sym.getLinkage() == Linkage::Weak) 66 Flags |= JITSymbolFlags::Weak; 67 68 if (Sym.getScope() == Scope::Default) 69 Flags |= JITSymbolFlags::Exported; 70 71 if (Sym.isCallable()) 72 Flags |= JITSymbolFlags::Callable; 73 74 return Flags; 75 } 76 77 class LinkGraphMaterializationUnit : public MaterializationUnit { 78 public: 79 static std::unique_ptr<LinkGraphMaterializationUnit> 80 Create(ObjectLinkingLayer &ObjLinkingLayer, std::unique_ptr<LinkGraph> G) { 81 auto LGI = scanLinkGraph(ObjLinkingLayer.getExecutionSession(), *G); 82 return std::unique_ptr<LinkGraphMaterializationUnit>( 83 new LinkGraphMaterializationUnit(ObjLinkingLayer, std::move(G), 84 std::move(LGI))); 85 } 86 87 StringRef getName() const override { return G->getName(); } 88 void materialize(std::unique_ptr<MaterializationResponsibility> MR) override { 89 ObjLinkingLayer.emit(std::move(MR), std::move(G)); 90 } 91 92 private: 93 static Interface scanLinkGraph(ExecutionSession &ES, LinkGraph &G) { 94 95 Interface LGI; 96 97 auto AddSymbol = [&](Symbol *Sym) { 98 // Skip local symbols. 99 if (Sym->getScope() == Scope::Local) 100 return; 101 assert(Sym->hasName() && "Anonymous non-local symbol?"); 102 103 LGI.SymbolFlags[ES.intern(Sym->getName())] = 104 getJITSymbolFlagsForSymbol(*Sym); 105 }; 106 107 for (auto *Sym : G.defined_symbols()) 108 AddSymbol(Sym); 109 for (auto *Sym : G.absolute_symbols()) 110 AddSymbol(Sym); 111 112 if (hasInitializerSection(G)) 113 LGI.InitSymbol = makeInitSymbol(ES, G); 114 115 return LGI; 116 } 117 118 static SymbolStringPtr makeInitSymbol(ExecutionSession &ES, LinkGraph &G) { 119 std::string InitSymString; 120 raw_string_ostream(InitSymString) 121 << "$." << G.getName() << ".__inits" << Counter++; 122 return ES.intern(InitSymString); 123 } 124 125 LinkGraphMaterializationUnit(ObjectLinkingLayer &ObjLinkingLayer, 126 std::unique_ptr<LinkGraph> G, Interface LGI) 127 : MaterializationUnit(std::move(LGI)), ObjLinkingLayer(ObjLinkingLayer), 128 G(std::move(G)) {} 129 130 void discard(const JITDylib &JD, const SymbolStringPtr &Name) override { 131 for (auto *Sym : G->defined_symbols()) 132 if (Sym->getName() == *Name) { 133 assert(Sym->getLinkage() == Linkage::Weak && 134 "Discarding non-weak definition"); 135 G->makeExternal(*Sym); 136 break; 137 } 138 } 139 140 ObjectLinkingLayer &ObjLinkingLayer; 141 std::unique_ptr<LinkGraph> G; 142 static std::atomic<uint64_t> Counter; 143 }; 144 145 std::atomic<uint64_t> LinkGraphMaterializationUnit::Counter{0}; 146 147 } // end anonymous namespace 148 149 namespace llvm { 150 namespace orc { 151 152 class ObjectLinkingLayerJITLinkContext final : public JITLinkContext { 153 public: 154 ObjectLinkingLayerJITLinkContext( 155 ObjectLinkingLayer &Layer, 156 std::unique_ptr<MaterializationResponsibility> MR, 157 std::unique_ptr<MemoryBuffer> ObjBuffer) 158 : JITLinkContext(&MR->getTargetJITDylib()), Layer(Layer), 159 MR(std::move(MR)), ObjBuffer(std::move(ObjBuffer)) {} 160 161 ~ObjectLinkingLayerJITLinkContext() { 162 // If there is an object buffer return function then use it to 163 // return ownership of the buffer. 164 if (Layer.ReturnObjectBuffer && ObjBuffer) 165 Layer.ReturnObjectBuffer(std::move(ObjBuffer)); 166 } 167 168 JITLinkMemoryManager &getMemoryManager() override { return Layer.MemMgr; } 169 170 void notifyMaterializing(LinkGraph &G) { 171 for (auto &P : Layer.Plugins) 172 P->notifyMaterializing(*MR, G, *this, 173 ObjBuffer ? ObjBuffer->getMemBufferRef() 174 : MemoryBufferRef()); 175 } 176 177 void notifyFailed(Error Err) override { 178 for (auto &P : Layer.Plugins) 179 Err = joinErrors(std::move(Err), P->notifyFailed(*MR)); 180 Layer.getExecutionSession().reportError(std::move(Err)); 181 MR->failMaterialization(); 182 } 183 184 void lookup(const LookupMap &Symbols, 185 std::unique_ptr<JITLinkAsyncLookupContinuation> LC) override { 186 187 JITDylibSearchOrder LinkOrder; 188 MR->getTargetJITDylib().withLinkOrderDo( 189 [&](const JITDylibSearchOrder &LO) { LinkOrder = LO; }); 190 191 auto &ES = Layer.getExecutionSession(); 192 193 SymbolLookupSet LookupSet; 194 for (auto &KV : Symbols) { 195 orc::SymbolLookupFlags LookupFlags; 196 switch (KV.second) { 197 case jitlink::SymbolLookupFlags::RequiredSymbol: 198 LookupFlags = orc::SymbolLookupFlags::RequiredSymbol; 199 break; 200 case jitlink::SymbolLookupFlags::WeaklyReferencedSymbol: 201 LookupFlags = orc::SymbolLookupFlags::WeaklyReferencedSymbol; 202 break; 203 } 204 LookupSet.add(ES.intern(KV.first), LookupFlags); 205 } 206 207 // OnResolve -- De-intern the symbols and pass the result to the linker. 208 auto OnResolve = [LookupContinuation = 209 std::move(LC)](Expected<SymbolMap> Result) mutable { 210 if (!Result) 211 LookupContinuation->run(Result.takeError()); 212 else { 213 AsyncLookupResult LR; 214 for (auto &KV : *Result) 215 LR[*KV.first] = KV.second; 216 LookupContinuation->run(std::move(LR)); 217 } 218 }; 219 220 ES.lookup(LookupKind::Static, LinkOrder, std::move(LookupSet), 221 SymbolState::Resolved, std::move(OnResolve), 222 [this](const SymbolDependenceMap &Deps) { 223 // Translate LookupDeps map to SymbolSourceJD. 224 for (auto &[DepJD, Deps] : Deps) 225 for (auto &DepSym : Deps) 226 SymbolSourceJDs[NonOwningSymbolStringPtr(DepSym)] = DepJD; 227 }); 228 } 229 230 Error notifyResolved(LinkGraph &G) override { 231 auto &ES = Layer.getExecutionSession(); 232 233 SymbolFlagsMap ExtraSymbolsToClaim; 234 bool AutoClaim = Layer.AutoClaimObjectSymbols; 235 236 SymbolMap InternedResult; 237 for (auto *Sym : G.defined_symbols()) 238 if (Sym->hasName() && Sym->getScope() != Scope::Local) { 239 auto InternedName = ES.intern(Sym->getName()); 240 auto Ptr = getJITSymbolPtrForSymbol(*Sym, G.getTargetTriple()); 241 auto Flags = getJITSymbolFlagsForSymbol(*Sym); 242 InternedResult[InternedName] = {Ptr, Flags}; 243 if (AutoClaim && !MR->getSymbols().count(InternedName)) { 244 assert(!ExtraSymbolsToClaim.count(InternedName) && 245 "Duplicate symbol to claim?"); 246 ExtraSymbolsToClaim[InternedName] = Flags; 247 } 248 } 249 250 for (auto *Sym : G.absolute_symbols()) 251 if (Sym->hasName() && Sym->getScope() != Scope::Local) { 252 auto InternedName = ES.intern(Sym->getName()); 253 auto Ptr = getJITSymbolPtrForSymbol(*Sym, G.getTargetTriple()); 254 auto Flags = getJITSymbolFlagsForSymbol(*Sym); 255 InternedResult[InternedName] = {Ptr, Flags}; 256 if (AutoClaim && !MR->getSymbols().count(InternedName)) { 257 assert(!ExtraSymbolsToClaim.count(InternedName) && 258 "Duplicate symbol to claim?"); 259 ExtraSymbolsToClaim[InternedName] = Flags; 260 } 261 } 262 263 if (!ExtraSymbolsToClaim.empty()) 264 if (auto Err = MR->defineMaterializing(ExtraSymbolsToClaim)) 265 return Err; 266 267 { 268 269 // Check that InternedResult matches up with MR->getSymbols(), overriding 270 // flags if requested. 271 // This guards against faulty transformations / compilers / object caches. 272 273 // First check that there aren't any missing symbols. 274 size_t NumMaterializationSideEffectsOnlySymbols = 0; 275 SymbolNameVector ExtraSymbols; 276 SymbolNameVector MissingSymbols; 277 for (auto &KV : MR->getSymbols()) { 278 279 auto I = InternedResult.find(KV.first); 280 281 // If this is a materialization-side-effects only symbol then bump 282 // the counter and make sure it's *not* defined, otherwise make 283 // sure that it is defined. 284 if (KV.second.hasMaterializationSideEffectsOnly()) { 285 ++NumMaterializationSideEffectsOnlySymbols; 286 if (I != InternedResult.end()) 287 ExtraSymbols.push_back(KV.first); 288 continue; 289 } else if (I == InternedResult.end()) 290 MissingSymbols.push_back(KV.first); 291 else if (Layer.OverrideObjectFlags) 292 I->second.setFlags(KV.second); 293 } 294 295 // If there were missing symbols then report the error. 296 if (!MissingSymbols.empty()) 297 return make_error<MissingSymbolDefinitions>( 298 Layer.getExecutionSession().getSymbolStringPool(), G.getName(), 299 std::move(MissingSymbols)); 300 301 // If there are more definitions than expected, add them to the 302 // ExtraSymbols vector. 303 if (InternedResult.size() > 304 MR->getSymbols().size() - NumMaterializationSideEffectsOnlySymbols) { 305 for (auto &KV : InternedResult) 306 if (!MR->getSymbols().count(KV.first)) 307 ExtraSymbols.push_back(KV.first); 308 } 309 310 // If there were extra definitions then report the error. 311 if (!ExtraSymbols.empty()) 312 return make_error<UnexpectedSymbolDefinitions>( 313 Layer.getExecutionSession().getSymbolStringPool(), G.getName(), 314 std::move(ExtraSymbols)); 315 } 316 317 if (auto Err = MR->notifyResolved(InternedResult)) 318 return Err; 319 320 Layer.notifyLoaded(*MR); 321 return Error::success(); 322 } 323 324 void notifyFinalized(JITLinkMemoryManager::FinalizedAlloc A) override { 325 if (auto Err = Layer.notifyEmitted(*MR, std::move(A))) { 326 Layer.getExecutionSession().reportError(std::move(Err)); 327 MR->failMaterialization(); 328 return; 329 } 330 if (auto Err = MR->notifyEmitted(SymbolDepGroups)) { 331 Layer.getExecutionSession().reportError(std::move(Err)); 332 MR->failMaterialization(); 333 } 334 } 335 336 LinkGraphPassFunction getMarkLivePass(const Triple &TT) const override { 337 return [this](LinkGraph &G) { return markResponsibilitySymbolsLive(G); }; 338 } 339 340 Error modifyPassConfig(LinkGraph &LG, PassConfiguration &Config) override { 341 // Add passes to mark duplicate defs as should-discard, and to walk the 342 // link graph to build the symbol dependence graph. 343 Config.PrePrunePasses.push_back([this](LinkGraph &G) { 344 return claimOrExternalizeWeakAndCommonSymbols(G); 345 }); 346 347 Layer.modifyPassConfig(*MR, LG, Config); 348 349 Config.PreFixupPasses.push_back( 350 [this](LinkGraph &G) { return registerDependencies(G); }); 351 352 return Error::success(); 353 } 354 355 private: 356 // Symbol name dependencies: 357 // Internal: Defined in this graph. 358 // External: Defined externally. 359 struct BlockSymbolDependencies { 360 SymbolNameSet Internal, External; 361 }; 362 363 // Lazily populated map of blocks to BlockSymbolDependencies values. 364 class BlockDependenciesMap { 365 public: 366 BlockDependenciesMap(ExecutionSession &ES, 367 DenseMap<const Block *, DenseSet<Block *>> BlockDeps) 368 : ES(ES), BlockDeps(std::move(BlockDeps)) {} 369 370 const BlockSymbolDependencies &operator[](const Block &B) { 371 // Check the cache first. 372 auto I = BlockTransitiveDepsCache.find(&B); 373 if (I != BlockTransitiveDepsCache.end()) 374 return I->second; 375 376 // No value. Populate the cache. 377 BlockSymbolDependencies BTDCacheVal; 378 auto BDI = BlockDeps.find(&B); 379 assert(BDI != BlockDeps.end() && "No block dependencies"); 380 381 for (auto *BDep : BDI->second) { 382 auto &BID = getBlockImmediateDeps(*BDep); 383 for (auto &ExternalDep : BID.External) 384 BTDCacheVal.External.insert(ExternalDep); 385 for (auto &InternalDep : BID.Internal) 386 BTDCacheVal.Internal.insert(InternalDep); 387 } 388 389 return BlockTransitiveDepsCache 390 .insert(std::make_pair(&B, std::move(BTDCacheVal))) 391 .first->second; 392 } 393 394 SymbolStringPtr &getInternedName(Symbol &Sym) { 395 auto I = NameCache.find(&Sym); 396 if (I != NameCache.end()) 397 return I->second; 398 399 return NameCache.insert(std::make_pair(&Sym, ES.intern(Sym.getName()))) 400 .first->second; 401 } 402 403 private: 404 BlockSymbolDependencies &getBlockImmediateDeps(Block &B) { 405 // Check the cache first. 406 auto I = BlockImmediateDepsCache.find(&B); 407 if (I != BlockImmediateDepsCache.end()) 408 return I->second; 409 410 BlockSymbolDependencies BIDCacheVal; 411 for (auto &E : B.edges()) { 412 auto &Tgt = E.getTarget(); 413 if (Tgt.getScope() != Scope::Local) { 414 if (Tgt.isExternal()) { 415 if (Tgt.getAddress() || !Tgt.isWeaklyReferenced()) 416 BIDCacheVal.External.insert(getInternedName(Tgt)); 417 } else 418 BIDCacheVal.Internal.insert(getInternedName(Tgt)); 419 } 420 } 421 422 return BlockImmediateDepsCache 423 .insert(std::make_pair(&B, std::move(BIDCacheVal))) 424 .first->second; 425 } 426 427 ExecutionSession &ES; 428 DenseMap<const Block *, DenseSet<Block *>> BlockDeps; 429 DenseMap<const Symbol *, SymbolStringPtr> NameCache; 430 DenseMap<const Block *, BlockSymbolDependencies> BlockImmediateDepsCache; 431 DenseMap<const Block *, BlockSymbolDependencies> BlockTransitiveDepsCache; 432 }; 433 434 Error claimOrExternalizeWeakAndCommonSymbols(LinkGraph &G) { 435 auto &ES = Layer.getExecutionSession(); 436 437 SymbolFlagsMap NewSymbolsToClaim; 438 std::vector<std::pair<SymbolStringPtr, Symbol *>> NameToSym; 439 440 auto ProcessSymbol = [&](Symbol *Sym) { 441 if (Sym->hasName() && Sym->getLinkage() == Linkage::Weak && 442 Sym->getScope() != Scope::Local) { 443 auto Name = ES.intern(Sym->getName()); 444 if (!MR->getSymbols().count(ES.intern(Sym->getName()))) { 445 NewSymbolsToClaim[Name] = 446 getJITSymbolFlagsForSymbol(*Sym) | JITSymbolFlags::Weak; 447 NameToSym.push_back(std::make_pair(std::move(Name), Sym)); 448 } 449 } 450 }; 451 452 for (auto *Sym : G.defined_symbols()) 453 ProcessSymbol(Sym); 454 for (auto *Sym : G.absolute_symbols()) 455 ProcessSymbol(Sym); 456 457 // Attempt to claim all weak defs that we're not already responsible for. 458 // This cannot fail -- any clashes will just result in rejection of our 459 // claim, at which point we'll externalize that symbol. 460 cantFail(MR->defineMaterializing(std::move(NewSymbolsToClaim))); 461 462 // Walk the list of symbols that we just tried to claim. Symbols that we're 463 // responsible for are marked live. Symbols that we're not responsible for 464 // are turned into external references. 465 for (auto &KV : NameToSym) { 466 if (MR->getSymbols().count(KV.first)) 467 KV.second->setLive(true); 468 else 469 G.makeExternal(*KV.second); 470 } 471 472 return Error::success(); 473 } 474 475 Error markResponsibilitySymbolsLive(LinkGraph &G) const { 476 auto &ES = Layer.getExecutionSession(); 477 for (auto *Sym : G.defined_symbols()) 478 if (Sym->hasName() && MR->getSymbols().count(ES.intern(Sym->getName()))) 479 Sym->setLive(true); 480 return Error::success(); 481 } 482 483 Error registerDependencies(LinkGraph &G) { 484 auto &TargetJD = MR->getTargetJITDylib(); 485 auto &ES = TargetJD.getExecutionSession(); 486 auto BlockDeps = computeBlockNonLocalDeps(G); 487 488 DenseSet<Block *> BlockDepsProcessed; 489 DenseMap<Block *, SymbolDependenceGroup> DepGroupForBlock; 490 491 // Compute dependencies for symbols defined in the JITLink graph. 492 for (auto *Sym : G.defined_symbols()) { 493 494 // Skip local symbols. 495 if (Sym->getScope() == Scope::Local) 496 continue; 497 assert(Sym->hasName() && 498 "Defined non-local jitlink::Symbol should have a name"); 499 500 auto &BDeps = BlockDeps[Sym->getBlock()]; 501 502 // Skip symbols in blocks that don't depend on anything. 503 if (BDeps.Internal.empty() && BDeps.External.empty()) 504 continue; 505 506 SymbolDependenceGroup &SDG = DepGroupForBlock[&Sym->getBlock()]; 507 SDG.Symbols.insert(ES.intern(Sym->getName())); 508 509 if (!BlockDepsProcessed.count(&Sym->getBlock())) { 510 BlockDepsProcessed.insert(&Sym->getBlock()); 511 512 if (!BDeps.Internal.empty()) 513 SDG.Dependencies[&TargetJD] = BDeps.Internal; 514 for (auto &Dep : BDeps.External) { 515 auto DepSrcItr = SymbolSourceJDs.find(NonOwningSymbolStringPtr(Dep)); 516 if (DepSrcItr != SymbolSourceJDs.end()) 517 SDG.Dependencies[DepSrcItr->second].insert(Dep); 518 } 519 } 520 } 521 522 SymbolDependenceGroup SynthSDG; 523 524 for (auto &P : Layer.Plugins) { 525 auto SynthDeps = P->getSyntheticSymbolDependencies(*MR); 526 if (SynthDeps.empty()) 527 continue; 528 529 DenseSet<Block *> BlockVisited; 530 for (auto &[Name, DepSyms] : SynthDeps) { 531 SynthSDG.Symbols.insert(Name); 532 for (auto *Sym : DepSyms) { 533 if (Sym->getScope() == Scope::Local) { 534 auto &BDeps = BlockDeps[Sym->getBlock()]; 535 for (auto &S : BDeps.Internal) 536 SynthSDG.Dependencies[&TargetJD].insert(S); 537 for (auto &S : BDeps.External) { 538 auto DepSrcItr = 539 SymbolSourceJDs.find(NonOwningSymbolStringPtr(S)); 540 if (DepSrcItr != SymbolSourceJDs.end()) 541 SynthSDG.Dependencies[DepSrcItr->second].insert(S); 542 } 543 } else { 544 auto SymName = ES.intern(Sym->getName()); 545 if (Sym->isExternal()) { 546 assert(SymbolSourceJDs.count(NonOwningSymbolStringPtr(SymName)) && 547 "External symbol source entry missing"); 548 SynthSDG 549 .Dependencies[SymbolSourceJDs[NonOwningSymbolStringPtr( 550 SymName)]] 551 .insert(SymName); 552 } else 553 SynthSDG.Dependencies[&TargetJD].insert(SymName); 554 } 555 } 556 } 557 } 558 559 // Transfer SDGs to SymbolDepGroups. 560 DepGroupForBlock.reserve(DepGroupForBlock.size() + 1); 561 for (auto &[B, SDG] : DepGroupForBlock) { 562 assert(!SDG.Symbols.empty() && "SymbolDependenceGroup covers no symbols"); 563 if (!SDG.Dependencies.empty()) 564 SymbolDepGroups.push_back(std::move(SDG)); 565 } 566 if (!SynthSDG.Symbols.empty() && !SynthSDG.Dependencies.empty()) 567 SymbolDepGroups.push_back(std::move(SynthSDG)); 568 569 return Error::success(); 570 } 571 572 BlockDependenciesMap computeBlockNonLocalDeps(LinkGraph &G) { 573 // First calculate the reachable-via-non-local-symbol blocks for each block. 574 struct BlockInfo { 575 DenseSet<Block *> Dependencies; 576 DenseSet<Block *> Dependants; 577 bool DependenciesChanged = true; 578 }; 579 DenseMap<Block *, BlockInfo> BlockInfos; 580 SmallVector<Block *> WorkList; 581 582 // Pre-allocate map entries. This prevents any iterator/reference 583 // invalidation in the next loop. 584 for (auto *B : G.blocks()) 585 (void)BlockInfos[B]; 586 587 // Build initial worklist, record block dependencies/dependants and 588 // non-local symbol dependencies. 589 for (auto *B : G.blocks()) { 590 auto &BI = BlockInfos[B]; 591 for (auto &E : B->edges()) { 592 if (E.getTarget().getScope() == Scope::Local && 593 !E.getTarget().isAbsolute()) { 594 auto &TgtB = E.getTarget().getBlock(); 595 if (&TgtB != B) { 596 BI.Dependencies.insert(&TgtB); 597 BlockInfos[&TgtB].Dependants.insert(B); 598 } 599 } 600 } 601 602 // If this node has both dependants and dependencies then add it to the 603 // worklist to propagate the dependencies to the dependants. 604 if (!BI.Dependants.empty() && !BI.Dependencies.empty()) 605 WorkList.push_back(B); 606 } 607 608 // Propagate block-level dependencies through the block-dependence graph. 609 while (!WorkList.empty()) { 610 auto *B = WorkList.pop_back_val(); 611 612 auto &BI = BlockInfos[B]; 613 assert(BI.DependenciesChanged && 614 "Block in worklist has unchanged dependencies"); 615 BI.DependenciesChanged = false; 616 for (auto *Dependant : BI.Dependants) { 617 auto &DependantBI = BlockInfos[Dependant]; 618 for (auto *Dependency : BI.Dependencies) { 619 if (Dependant != Dependency && 620 DependantBI.Dependencies.insert(Dependency).second) 621 if (!DependantBI.DependenciesChanged) { 622 DependantBI.DependenciesChanged = true; 623 WorkList.push_back(Dependant); 624 } 625 } 626 } 627 } 628 629 DenseMap<const Block *, DenseSet<Block *>> BlockDeps; 630 for (auto &KV : BlockInfos) 631 BlockDeps[KV.first] = std::move(KV.second.Dependencies); 632 633 return BlockDependenciesMap(Layer.getExecutionSession(), 634 std::move(BlockDeps)); 635 } 636 637 ObjectLinkingLayer &Layer; 638 std::unique_ptr<MaterializationResponsibility> MR; 639 std::unique_ptr<MemoryBuffer> ObjBuffer; 640 DenseMap<Block *, SymbolNameSet> ExternalBlockDeps; 641 DenseMap<Block *, SymbolNameSet> InternalBlockDeps; 642 DenseMap<NonOwningSymbolStringPtr, JITDylib *> SymbolSourceJDs; 643 std::vector<SymbolDependenceGroup> SymbolDepGroups; 644 }; 645 646 ObjectLinkingLayer::Plugin::~Plugin() = default; 647 648 char ObjectLinkingLayer::ID; 649 650 using BaseT = RTTIExtends<ObjectLinkingLayer, ObjectLayer>; 651 652 ObjectLinkingLayer::ObjectLinkingLayer(ExecutionSession &ES) 653 : BaseT(ES), MemMgr(ES.getExecutorProcessControl().getMemMgr()) { 654 ES.registerResourceManager(*this); 655 } 656 657 ObjectLinkingLayer::ObjectLinkingLayer(ExecutionSession &ES, 658 JITLinkMemoryManager &MemMgr) 659 : BaseT(ES), MemMgr(MemMgr) { 660 ES.registerResourceManager(*this); 661 } 662 663 ObjectLinkingLayer::ObjectLinkingLayer( 664 ExecutionSession &ES, std::unique_ptr<JITLinkMemoryManager> MemMgr) 665 : BaseT(ES), MemMgr(*MemMgr), MemMgrOwnership(std::move(MemMgr)) { 666 ES.registerResourceManager(*this); 667 } 668 669 ObjectLinkingLayer::~ObjectLinkingLayer() { 670 assert(Allocs.empty() && "Layer destroyed with resources still attached"); 671 getExecutionSession().deregisterResourceManager(*this); 672 } 673 674 Error ObjectLinkingLayer::add(ResourceTrackerSP RT, 675 std::unique_ptr<LinkGraph> G) { 676 auto &JD = RT->getJITDylib(); 677 return JD.define(LinkGraphMaterializationUnit::Create(*this, std::move(G)), 678 std::move(RT)); 679 } 680 681 void ObjectLinkingLayer::emit(std::unique_ptr<MaterializationResponsibility> R, 682 std::unique_ptr<MemoryBuffer> O) { 683 assert(O && "Object must not be null"); 684 MemoryBufferRef ObjBuffer = O->getMemBufferRef(); 685 686 auto Ctx = std::make_unique<ObjectLinkingLayerJITLinkContext>( 687 *this, std::move(R), std::move(O)); 688 if (auto G = createLinkGraphFromObject(ObjBuffer)) { 689 Ctx->notifyMaterializing(**G); 690 link(std::move(*G), std::move(Ctx)); 691 } else { 692 Ctx->notifyFailed(G.takeError()); 693 } 694 } 695 696 void ObjectLinkingLayer::emit(std::unique_ptr<MaterializationResponsibility> R, 697 std::unique_ptr<LinkGraph> G) { 698 auto Ctx = std::make_unique<ObjectLinkingLayerJITLinkContext>( 699 *this, std::move(R), nullptr); 700 Ctx->notifyMaterializing(*G); 701 link(std::move(G), std::move(Ctx)); 702 } 703 704 void ObjectLinkingLayer::modifyPassConfig(MaterializationResponsibility &MR, 705 LinkGraph &G, 706 PassConfiguration &PassConfig) { 707 for (auto &P : Plugins) 708 P->modifyPassConfig(MR, G, PassConfig); 709 } 710 711 void ObjectLinkingLayer::notifyLoaded(MaterializationResponsibility &MR) { 712 for (auto &P : Plugins) 713 P->notifyLoaded(MR); 714 } 715 716 Error ObjectLinkingLayer::notifyEmitted(MaterializationResponsibility &MR, 717 FinalizedAlloc FA) { 718 Error Err = Error::success(); 719 for (auto &P : Plugins) 720 Err = joinErrors(std::move(Err), P->notifyEmitted(MR)); 721 722 if (Err) 723 return Err; 724 725 if (!FA) 726 return Error::success(); 727 728 return MR.withResourceKeyDo( 729 [&](ResourceKey K) { Allocs[K].push_back(std::move(FA)); }); 730 } 731 732 Error ObjectLinkingLayer::handleRemoveResources(JITDylib &JD, ResourceKey K) { 733 734 { 735 Error Err = Error::success(); 736 for (auto &P : Plugins) 737 Err = joinErrors(std::move(Err), P->notifyRemovingResources(JD, K)); 738 if (Err) 739 return Err; 740 } 741 742 std::vector<FinalizedAlloc> AllocsToRemove; 743 getExecutionSession().runSessionLocked([&] { 744 auto I = Allocs.find(K); 745 if (I != Allocs.end()) { 746 std::swap(AllocsToRemove, I->second); 747 Allocs.erase(I); 748 } 749 }); 750 751 if (AllocsToRemove.empty()) 752 return Error::success(); 753 754 return MemMgr.deallocate(std::move(AllocsToRemove)); 755 } 756 757 void ObjectLinkingLayer::handleTransferResources(JITDylib &JD, 758 ResourceKey DstKey, 759 ResourceKey SrcKey) { 760 auto I = Allocs.find(SrcKey); 761 if (I != Allocs.end()) { 762 auto &SrcAllocs = I->second; 763 auto &DstAllocs = Allocs[DstKey]; 764 DstAllocs.reserve(DstAllocs.size() + SrcAllocs.size()); 765 for (auto &Alloc : SrcAllocs) 766 DstAllocs.push_back(std::move(Alloc)); 767 768 // Erase SrcKey entry using value rather than iterator I: I may have been 769 // invalidated when we looked up DstKey. 770 Allocs.erase(SrcKey); 771 } 772 773 for (auto &P : Plugins) 774 P->notifyTransferringResources(JD, DstKey, SrcKey); 775 } 776 777 EHFrameRegistrationPlugin::EHFrameRegistrationPlugin( 778 ExecutionSession &ES, std::unique_ptr<EHFrameRegistrar> Registrar) 779 : ES(ES), Registrar(std::move(Registrar)) {} 780 781 void EHFrameRegistrationPlugin::modifyPassConfig( 782 MaterializationResponsibility &MR, LinkGraph &G, 783 PassConfiguration &PassConfig) { 784 785 PassConfig.PostFixupPasses.push_back(createEHFrameRecorderPass( 786 G.getTargetTriple(), [this, &MR](ExecutorAddr Addr, size_t Size) { 787 if (Addr) { 788 std::lock_guard<std::mutex> Lock(EHFramePluginMutex); 789 assert(!InProcessLinks.count(&MR) && 790 "Link for MR already being tracked?"); 791 InProcessLinks[&MR] = {Addr, Size}; 792 } 793 })); 794 } 795 796 Error EHFrameRegistrationPlugin::notifyEmitted( 797 MaterializationResponsibility &MR) { 798 799 ExecutorAddrRange EmittedRange; 800 { 801 std::lock_guard<std::mutex> Lock(EHFramePluginMutex); 802 803 auto EHFrameRangeItr = InProcessLinks.find(&MR); 804 if (EHFrameRangeItr == InProcessLinks.end()) 805 return Error::success(); 806 807 EmittedRange = EHFrameRangeItr->second; 808 assert(EmittedRange.Start && "eh-frame addr to register can not be null"); 809 InProcessLinks.erase(EHFrameRangeItr); 810 } 811 812 if (auto Err = MR.withResourceKeyDo( 813 [&](ResourceKey K) { EHFrameRanges[K].push_back(EmittedRange); })) 814 return Err; 815 816 return Registrar->registerEHFrames(EmittedRange); 817 } 818 819 Error EHFrameRegistrationPlugin::notifyFailed( 820 MaterializationResponsibility &MR) { 821 std::lock_guard<std::mutex> Lock(EHFramePluginMutex); 822 InProcessLinks.erase(&MR); 823 return Error::success(); 824 } 825 826 Error EHFrameRegistrationPlugin::notifyRemovingResources(JITDylib &JD, 827 ResourceKey K) { 828 std::vector<ExecutorAddrRange> RangesToRemove; 829 830 ES.runSessionLocked([&] { 831 auto I = EHFrameRanges.find(K); 832 if (I != EHFrameRanges.end()) { 833 RangesToRemove = std::move(I->second); 834 EHFrameRanges.erase(I); 835 } 836 }); 837 838 Error Err = Error::success(); 839 while (!RangesToRemove.empty()) { 840 auto RangeToRemove = RangesToRemove.back(); 841 RangesToRemove.pop_back(); 842 assert(RangeToRemove.Start && "Untracked eh-frame range must not be null"); 843 Err = joinErrors(std::move(Err), 844 Registrar->deregisterEHFrames(RangeToRemove)); 845 } 846 847 return Err; 848 } 849 850 void EHFrameRegistrationPlugin::notifyTransferringResources( 851 JITDylib &JD, ResourceKey DstKey, ResourceKey SrcKey) { 852 auto SI = EHFrameRanges.find(SrcKey); 853 if (SI == EHFrameRanges.end()) 854 return; 855 856 auto DI = EHFrameRanges.find(DstKey); 857 if (DI != EHFrameRanges.end()) { 858 auto &SrcRanges = SI->second; 859 auto &DstRanges = DI->second; 860 DstRanges.reserve(DstRanges.size() + SrcRanges.size()); 861 for (auto &SrcRange : SrcRanges) 862 DstRanges.push_back(std::move(SrcRange)); 863 EHFrameRanges.erase(SI); 864 } else { 865 // We need to move SrcKey's ranges over without invalidating the SI 866 // iterator. 867 auto Tmp = std::move(SI->second); 868 EHFrameRanges.erase(SI); 869 EHFrameRanges[DstKey] = std::move(Tmp); 870 } 871 } 872 873 } // End namespace orc. 874 } // End namespace llvm. 875