1 //===------- ObjectLinkingLayer.cpp - JITLink backed ORC ObjectLayer ------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 9 #include "llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h" 10 #include "llvm/ExecutionEngine/JITLink/EHFrameSupport.h" 11 #include "llvm/ExecutionEngine/JITLink/aarch32.h" 12 #include "llvm/ExecutionEngine/Orc/DebugObjectManagerPlugin.h" 13 #include "llvm/ExecutionEngine/Orc/DebugUtils.h" 14 #include "llvm/ExecutionEngine/Orc/ObjectFileInterface.h" 15 #include "llvm/ExecutionEngine/Orc/Shared/ObjectFormats.h" 16 #include "llvm/Support/MemoryBuffer.h" 17 #include <string> 18 #include <vector> 19 20 #define DEBUG_TYPE "orc" 21 22 using namespace llvm; 23 using namespace llvm::jitlink; 24 using namespace llvm::orc; 25 26 namespace { 27 28 bool hasInitializerSection(jitlink::LinkGraph &G) { 29 bool IsMachO = G.getTargetTriple().isOSBinFormatMachO(); 30 bool IsElf = G.getTargetTriple().isOSBinFormatELF(); 31 if (!IsMachO && !IsElf) 32 return false; 33 34 for (auto &Sec : G.sections()) { 35 if (IsMachO && isMachOInitializerSection(Sec.getName())) 36 return true; 37 if (IsElf && isELFInitializerSection(Sec.getName())) 38 return true; 39 } 40 41 return false; 42 } 43 44 ExecutorAddr getJITSymbolPtrForSymbol(Symbol &Sym, const Triple &TT) { 45 switch (TT.getArch()) { 46 case Triple::arm: 47 case Triple::armeb: 48 case Triple::thumb: 49 case Triple::thumbeb: 50 if (hasTargetFlags(Sym, aarch32::ThumbSymbol)) { 51 // Set LSB to indicate thumb target 52 assert(Sym.isCallable() && "Only callable symbols can have thumb flag"); 53 assert((Sym.getAddress().getValue() & 0x01) == 0 && "LSB is clear"); 54 return Sym.getAddress() + 0x01; 55 } 56 return Sym.getAddress(); 57 default: 58 return Sym.getAddress(); 59 } 60 } 61 62 JITSymbolFlags getJITSymbolFlagsForSymbol(Symbol &Sym) { 63 JITSymbolFlags Flags; 64 65 if (Sym.getLinkage() == Linkage::Weak) 66 Flags |= JITSymbolFlags::Weak; 67 68 if (Sym.getScope() == Scope::Default) 69 Flags |= JITSymbolFlags::Exported; 70 71 if (Sym.isCallable()) 72 Flags |= JITSymbolFlags::Callable; 73 74 return Flags; 75 } 76 77 class LinkGraphMaterializationUnit : public MaterializationUnit { 78 public: 79 static std::unique_ptr<LinkGraphMaterializationUnit> 80 Create(ObjectLinkingLayer &ObjLinkingLayer, std::unique_ptr<LinkGraph> G) { 81 auto LGI = scanLinkGraph(ObjLinkingLayer.getExecutionSession(), *G); 82 return std::unique_ptr<LinkGraphMaterializationUnit>( 83 new LinkGraphMaterializationUnit(ObjLinkingLayer, std::move(G), 84 std::move(LGI))); 85 } 86 87 StringRef getName() const override { return G->getName(); } 88 void materialize(std::unique_ptr<MaterializationResponsibility> MR) override { 89 ObjLinkingLayer.emit(std::move(MR), std::move(G)); 90 } 91 92 private: 93 static Interface scanLinkGraph(ExecutionSession &ES, LinkGraph &G) { 94 95 Interface LGI; 96 97 auto AddSymbol = [&](Symbol *Sym) { 98 // Skip local symbols. 99 if (Sym->getScope() == Scope::Local) 100 return; 101 assert(Sym->hasName() && "Anonymous non-local symbol?"); 102 103 LGI.SymbolFlags[ES.intern(Sym->getName())] = 104 getJITSymbolFlagsForSymbol(*Sym); 105 }; 106 107 for (auto *Sym : G.defined_symbols()) 108 AddSymbol(Sym); 109 for (auto *Sym : G.absolute_symbols()) 110 AddSymbol(Sym); 111 112 if (hasInitializerSection(G)) 113 LGI.InitSymbol = makeInitSymbol(ES, G); 114 115 return LGI; 116 } 117 118 static SymbolStringPtr makeInitSymbol(ExecutionSession &ES, LinkGraph &G) { 119 std::string InitSymString; 120 raw_string_ostream(InitSymString) 121 << "$." << G.getName() << ".__inits" << Counter++; 122 return ES.intern(InitSymString); 123 } 124 125 LinkGraphMaterializationUnit(ObjectLinkingLayer &ObjLinkingLayer, 126 std::unique_ptr<LinkGraph> G, Interface LGI) 127 : MaterializationUnit(std::move(LGI)), ObjLinkingLayer(ObjLinkingLayer), 128 G(std::move(G)) {} 129 130 void discard(const JITDylib &JD, const SymbolStringPtr &Name) override { 131 for (auto *Sym : G->defined_symbols()) 132 if (Sym->getName() == *Name) { 133 assert(Sym->getLinkage() == Linkage::Weak && 134 "Discarding non-weak definition"); 135 G->makeExternal(*Sym); 136 break; 137 } 138 } 139 140 ObjectLinkingLayer &ObjLinkingLayer; 141 std::unique_ptr<LinkGraph> G; 142 static std::atomic<uint64_t> Counter; 143 }; 144 145 std::atomic<uint64_t> LinkGraphMaterializationUnit::Counter{0}; 146 147 } // end anonymous namespace 148 149 namespace llvm { 150 namespace orc { 151 152 class ObjectLinkingLayerJITLinkContext final : public JITLinkContext { 153 public: 154 ObjectLinkingLayerJITLinkContext( 155 ObjectLinkingLayer &Layer, 156 std::unique_ptr<MaterializationResponsibility> MR, 157 std::unique_ptr<MemoryBuffer> ObjBuffer) 158 : JITLinkContext(&MR->getTargetJITDylib()), Layer(Layer), 159 MR(std::move(MR)), ObjBuffer(std::move(ObjBuffer)) { 160 std::lock_guard<std::mutex> Lock(Layer.LayerMutex); 161 Plugins = Layer.Plugins; 162 } 163 164 ~ObjectLinkingLayerJITLinkContext() { 165 // If there is an object buffer return function then use it to 166 // return ownership of the buffer. 167 if (Layer.ReturnObjectBuffer && ObjBuffer) 168 Layer.ReturnObjectBuffer(std::move(ObjBuffer)); 169 } 170 171 JITLinkMemoryManager &getMemoryManager() override { return Layer.MemMgr; } 172 173 void notifyMaterializing(LinkGraph &G) { 174 for (auto &P : Plugins) 175 P->notifyMaterializing(*MR, G, *this, 176 ObjBuffer ? ObjBuffer->getMemBufferRef() 177 : MemoryBufferRef()); 178 } 179 180 void notifyFailed(Error Err) override { 181 for (auto &P : Plugins) 182 Err = joinErrors(std::move(Err), P->notifyFailed(*MR)); 183 Layer.getExecutionSession().reportError(std::move(Err)); 184 MR->failMaterialization(); 185 } 186 187 void lookup(const LookupMap &Symbols, 188 std::unique_ptr<JITLinkAsyncLookupContinuation> LC) override { 189 190 JITDylibSearchOrder LinkOrder; 191 MR->getTargetJITDylib().withLinkOrderDo( 192 [&](const JITDylibSearchOrder &LO) { LinkOrder = LO; }); 193 194 auto &ES = Layer.getExecutionSession(); 195 196 SymbolLookupSet LookupSet; 197 for (auto &KV : Symbols) { 198 orc::SymbolLookupFlags LookupFlags; 199 switch (KV.second) { 200 case jitlink::SymbolLookupFlags::RequiredSymbol: 201 LookupFlags = orc::SymbolLookupFlags::RequiredSymbol; 202 break; 203 case jitlink::SymbolLookupFlags::WeaklyReferencedSymbol: 204 LookupFlags = orc::SymbolLookupFlags::WeaklyReferencedSymbol; 205 break; 206 } 207 LookupSet.add(ES.intern(KV.first), LookupFlags); 208 } 209 210 // OnResolve -- De-intern the symbols and pass the result to the linker. 211 auto OnResolve = [LookupContinuation = 212 std::move(LC)](Expected<SymbolMap> Result) mutable { 213 if (!Result) 214 LookupContinuation->run(Result.takeError()); 215 else { 216 AsyncLookupResult LR; 217 for (auto &KV : *Result) 218 LR[*KV.first] = KV.second; 219 LookupContinuation->run(std::move(LR)); 220 } 221 }; 222 223 ES.lookup(LookupKind::Static, LinkOrder, std::move(LookupSet), 224 SymbolState::Resolved, std::move(OnResolve), 225 [this](const SymbolDependenceMap &Deps) { 226 // Translate LookupDeps map to SymbolSourceJD. 227 for (auto &[DepJD, Deps] : Deps) 228 for (auto &DepSym : Deps) 229 SymbolSourceJDs[NonOwningSymbolStringPtr(DepSym)] = DepJD; 230 }); 231 } 232 233 Error notifyResolved(LinkGraph &G) override { 234 auto &ES = Layer.getExecutionSession(); 235 236 SymbolFlagsMap ExtraSymbolsToClaim; 237 bool AutoClaim = Layer.AutoClaimObjectSymbols; 238 239 SymbolMap InternedResult; 240 for (auto *Sym : G.defined_symbols()) 241 if (Sym->hasName() && Sym->getScope() != Scope::Local) { 242 auto InternedName = ES.intern(Sym->getName()); 243 auto Ptr = getJITSymbolPtrForSymbol(*Sym, G.getTargetTriple()); 244 auto Flags = getJITSymbolFlagsForSymbol(*Sym); 245 InternedResult[InternedName] = {Ptr, Flags}; 246 if (AutoClaim && !MR->getSymbols().count(InternedName)) { 247 assert(!ExtraSymbolsToClaim.count(InternedName) && 248 "Duplicate symbol to claim?"); 249 ExtraSymbolsToClaim[InternedName] = Flags; 250 } 251 } 252 253 for (auto *Sym : G.absolute_symbols()) 254 if (Sym->hasName() && Sym->getScope() != Scope::Local) { 255 auto InternedName = ES.intern(Sym->getName()); 256 auto Ptr = getJITSymbolPtrForSymbol(*Sym, G.getTargetTriple()); 257 auto Flags = getJITSymbolFlagsForSymbol(*Sym); 258 InternedResult[InternedName] = {Ptr, Flags}; 259 if (AutoClaim && !MR->getSymbols().count(InternedName)) { 260 assert(!ExtraSymbolsToClaim.count(InternedName) && 261 "Duplicate symbol to claim?"); 262 ExtraSymbolsToClaim[InternedName] = Flags; 263 } 264 } 265 266 if (!ExtraSymbolsToClaim.empty()) 267 if (auto Err = MR->defineMaterializing(ExtraSymbolsToClaim)) 268 return Err; 269 270 { 271 272 // Check that InternedResult matches up with MR->getSymbols(), overriding 273 // flags if requested. 274 // This guards against faulty transformations / compilers / object caches. 275 276 // First check that there aren't any missing symbols. 277 size_t NumMaterializationSideEffectsOnlySymbols = 0; 278 SymbolNameVector ExtraSymbols; 279 SymbolNameVector MissingSymbols; 280 for (auto &KV : MR->getSymbols()) { 281 282 auto I = InternedResult.find(KV.first); 283 284 // If this is a materialization-side-effects only symbol then bump 285 // the counter and make sure it's *not* defined, otherwise make 286 // sure that it is defined. 287 if (KV.second.hasMaterializationSideEffectsOnly()) { 288 ++NumMaterializationSideEffectsOnlySymbols; 289 if (I != InternedResult.end()) 290 ExtraSymbols.push_back(KV.first); 291 continue; 292 } else if (I == InternedResult.end()) 293 MissingSymbols.push_back(KV.first); 294 else if (Layer.OverrideObjectFlags) 295 I->second.setFlags(KV.second); 296 } 297 298 // If there were missing symbols then report the error. 299 if (!MissingSymbols.empty()) 300 return make_error<MissingSymbolDefinitions>( 301 Layer.getExecutionSession().getSymbolStringPool(), G.getName(), 302 std::move(MissingSymbols)); 303 304 // If there are more definitions than expected, add them to the 305 // ExtraSymbols vector. 306 if (InternedResult.size() > 307 MR->getSymbols().size() - NumMaterializationSideEffectsOnlySymbols) { 308 for (auto &KV : InternedResult) 309 if (!MR->getSymbols().count(KV.first)) 310 ExtraSymbols.push_back(KV.first); 311 } 312 313 // If there were extra definitions then report the error. 314 if (!ExtraSymbols.empty()) 315 return make_error<UnexpectedSymbolDefinitions>( 316 Layer.getExecutionSession().getSymbolStringPool(), G.getName(), 317 std::move(ExtraSymbols)); 318 } 319 320 if (auto Err = MR->notifyResolved(InternedResult)) 321 return Err; 322 323 notifyLoaded(); 324 return Error::success(); 325 } 326 327 void notifyFinalized(JITLinkMemoryManager::FinalizedAlloc A) override { 328 if (auto Err = notifyEmitted(std::move(A))) { 329 Layer.getExecutionSession().reportError(std::move(Err)); 330 MR->failMaterialization(); 331 return; 332 } 333 if (auto Err = MR->notifyEmitted(SymbolDepGroups)) { 334 Layer.getExecutionSession().reportError(std::move(Err)); 335 MR->failMaterialization(); 336 } 337 } 338 339 LinkGraphPassFunction getMarkLivePass(const Triple &TT) const override { 340 return [this](LinkGraph &G) { return markResponsibilitySymbolsLive(G); }; 341 } 342 343 Error modifyPassConfig(LinkGraph &LG, PassConfiguration &Config) override { 344 // Add passes to mark duplicate defs as should-discard, and to walk the 345 // link graph to build the symbol dependence graph. 346 Config.PrePrunePasses.push_back([this](LinkGraph &G) { 347 return claimOrExternalizeWeakAndCommonSymbols(G); 348 }); 349 350 for (auto &P : Plugins) 351 P->modifyPassConfig(*MR, LG, Config); 352 353 Config.PreFixupPasses.push_back( 354 [this](LinkGraph &G) { return registerDependencies(G); }); 355 356 return Error::success(); 357 } 358 359 void notifyLoaded() { 360 for (auto &P : Plugins) 361 P->notifyLoaded(*MR); 362 } 363 364 Error notifyEmitted(jitlink::JITLinkMemoryManager::FinalizedAlloc FA) { 365 Error Err = Error::success(); 366 for (auto &P : Plugins) 367 Err = joinErrors(std::move(Err), P->notifyEmitted(*MR)); 368 369 if (Err) { 370 if (FA) 371 Err = 372 joinErrors(std::move(Err), Layer.MemMgr.deallocate(std::move(FA))); 373 return Err; 374 } 375 376 if (FA) 377 return Layer.recordFinalizedAlloc(*MR, std::move(FA)); 378 379 return Error::success(); 380 } 381 382 private: 383 // Symbol name dependencies: 384 // Internal: Defined in this graph. 385 // External: Defined externally. 386 struct BlockSymbolDependencies { 387 SymbolNameSet Internal, External; 388 }; 389 390 // Lazily populated map of blocks to BlockSymbolDependencies values. 391 class BlockDependenciesMap { 392 public: 393 BlockDependenciesMap(ExecutionSession &ES, 394 DenseMap<const Block *, DenseSet<Block *>> BlockDeps) 395 : ES(ES), BlockDeps(std::move(BlockDeps)) {} 396 397 const BlockSymbolDependencies &operator[](const Block &B) { 398 // Check the cache first. 399 auto I = BlockTransitiveDepsCache.find(&B); 400 if (I != BlockTransitiveDepsCache.end()) 401 return I->second; 402 403 // No value. Populate the cache. 404 BlockSymbolDependencies BTDCacheVal; 405 auto BDI = BlockDeps.find(&B); 406 assert(BDI != BlockDeps.end() && "No block dependencies"); 407 408 for (auto *BDep : BDI->second) { 409 auto &BID = getBlockImmediateDeps(*BDep); 410 for (auto &ExternalDep : BID.External) 411 BTDCacheVal.External.insert(ExternalDep); 412 for (auto &InternalDep : BID.Internal) 413 BTDCacheVal.Internal.insert(InternalDep); 414 } 415 416 return BlockTransitiveDepsCache 417 .insert(std::make_pair(&B, std::move(BTDCacheVal))) 418 .first->second; 419 } 420 421 SymbolStringPtr &getInternedName(Symbol &Sym) { 422 auto I = NameCache.find(&Sym); 423 if (I != NameCache.end()) 424 return I->second; 425 426 return NameCache.insert(std::make_pair(&Sym, ES.intern(Sym.getName()))) 427 .first->second; 428 } 429 430 private: 431 BlockSymbolDependencies &getBlockImmediateDeps(Block &B) { 432 // Check the cache first. 433 auto I = BlockImmediateDepsCache.find(&B); 434 if (I != BlockImmediateDepsCache.end()) 435 return I->second; 436 437 BlockSymbolDependencies BIDCacheVal; 438 for (auto &E : B.edges()) { 439 auto &Tgt = E.getTarget(); 440 if (Tgt.getScope() != Scope::Local) { 441 if (Tgt.isExternal()) { 442 if (Tgt.getAddress() || !Tgt.isWeaklyReferenced()) 443 BIDCacheVal.External.insert(getInternedName(Tgt)); 444 } else 445 BIDCacheVal.Internal.insert(getInternedName(Tgt)); 446 } 447 } 448 449 return BlockImmediateDepsCache 450 .insert(std::make_pair(&B, std::move(BIDCacheVal))) 451 .first->second; 452 } 453 454 ExecutionSession &ES; 455 DenseMap<const Block *, DenseSet<Block *>> BlockDeps; 456 DenseMap<const Symbol *, SymbolStringPtr> NameCache; 457 DenseMap<const Block *, BlockSymbolDependencies> BlockImmediateDepsCache; 458 DenseMap<const Block *, BlockSymbolDependencies> BlockTransitiveDepsCache; 459 }; 460 461 Error claimOrExternalizeWeakAndCommonSymbols(LinkGraph &G) { 462 auto &ES = Layer.getExecutionSession(); 463 464 SymbolFlagsMap NewSymbolsToClaim; 465 std::vector<std::pair<SymbolStringPtr, Symbol *>> NameToSym; 466 467 auto ProcessSymbol = [&](Symbol *Sym) { 468 if (Sym->hasName() && Sym->getLinkage() == Linkage::Weak && 469 Sym->getScope() != Scope::Local) { 470 auto Name = ES.intern(Sym->getName()); 471 if (!MR->getSymbols().count(ES.intern(Sym->getName()))) { 472 NewSymbolsToClaim[Name] = 473 getJITSymbolFlagsForSymbol(*Sym) | JITSymbolFlags::Weak; 474 NameToSym.push_back(std::make_pair(std::move(Name), Sym)); 475 } 476 } 477 }; 478 479 for (auto *Sym : G.defined_symbols()) 480 ProcessSymbol(Sym); 481 for (auto *Sym : G.absolute_symbols()) 482 ProcessSymbol(Sym); 483 484 // Attempt to claim all weak defs that we're not already responsible for. 485 // This may fail if the resource tracker has become defunct, but should 486 // always succeed otherwise. 487 if (auto Err = MR->defineMaterializing(std::move(NewSymbolsToClaim))) 488 return Err; 489 490 // Walk the list of symbols that we just tried to claim. Symbols that we're 491 // responsible for are marked live. Symbols that we're not responsible for 492 // are turned into external references. 493 for (auto &KV : NameToSym) { 494 if (MR->getSymbols().count(KV.first)) 495 KV.second->setLive(true); 496 else 497 G.makeExternal(*KV.second); 498 } 499 500 return Error::success(); 501 } 502 503 Error markResponsibilitySymbolsLive(LinkGraph &G) const { 504 auto &ES = Layer.getExecutionSession(); 505 for (auto *Sym : G.defined_symbols()) 506 if (Sym->hasName() && MR->getSymbols().count(ES.intern(Sym->getName()))) 507 Sym->setLive(true); 508 return Error::success(); 509 } 510 511 Error registerDependencies(LinkGraph &G) { 512 auto &TargetJD = MR->getTargetJITDylib(); 513 auto &ES = TargetJD.getExecutionSession(); 514 auto BlockDeps = computeBlockNonLocalDeps(G); 515 516 DenseSet<Block *> BlockDepsProcessed; 517 DenseMap<Block *, SymbolDependenceGroup> DepGroupForBlock; 518 519 // Compute dependencies for symbols defined in the JITLink graph. 520 for (auto *Sym : G.defined_symbols()) { 521 522 // Skip local symbols. 523 if (Sym->getScope() == Scope::Local) 524 continue; 525 assert(Sym->hasName() && 526 "Defined non-local jitlink::Symbol should have a name"); 527 528 auto &BDeps = BlockDeps[Sym->getBlock()]; 529 530 // Skip symbols in blocks that don't depend on anything. 531 if (BDeps.Internal.empty() && BDeps.External.empty()) 532 continue; 533 534 SymbolDependenceGroup &SDG = DepGroupForBlock[&Sym->getBlock()]; 535 SDG.Symbols.insert(ES.intern(Sym->getName())); 536 537 if (!BlockDepsProcessed.count(&Sym->getBlock())) { 538 BlockDepsProcessed.insert(&Sym->getBlock()); 539 540 if (!BDeps.Internal.empty()) 541 SDG.Dependencies[&TargetJD] = BDeps.Internal; 542 for (auto &Dep : BDeps.External) { 543 auto DepSrcItr = SymbolSourceJDs.find(NonOwningSymbolStringPtr(Dep)); 544 if (DepSrcItr != SymbolSourceJDs.end()) 545 SDG.Dependencies[DepSrcItr->second].insert(Dep); 546 } 547 } 548 } 549 550 SymbolDependenceGroup SynthSDG; 551 552 for (auto &P : Plugins) { 553 auto SynthDeps = P->getSyntheticSymbolDependencies(*MR); 554 if (SynthDeps.empty()) 555 continue; 556 557 DenseSet<Block *> BlockVisited; 558 for (auto &[Name, DepSyms] : SynthDeps) { 559 SynthSDG.Symbols.insert(Name); 560 for (auto *Sym : DepSyms) { 561 if (Sym->getScope() == Scope::Local) { 562 auto &BDeps = BlockDeps[Sym->getBlock()]; 563 for (auto &S : BDeps.Internal) 564 SynthSDG.Dependencies[&TargetJD].insert(S); 565 for (auto &S : BDeps.External) { 566 auto DepSrcItr = 567 SymbolSourceJDs.find(NonOwningSymbolStringPtr(S)); 568 if (DepSrcItr != SymbolSourceJDs.end()) 569 SynthSDG.Dependencies[DepSrcItr->second].insert(S); 570 } 571 } else { 572 auto SymName = ES.intern(Sym->getName()); 573 if (Sym->isExternal()) { 574 assert(SymbolSourceJDs.count(NonOwningSymbolStringPtr(SymName)) && 575 "External symbol source entry missing"); 576 SynthSDG 577 .Dependencies[SymbolSourceJDs[NonOwningSymbolStringPtr( 578 SymName)]] 579 .insert(SymName); 580 } else 581 SynthSDG.Dependencies[&TargetJD].insert(SymName); 582 } 583 } 584 } 585 } 586 587 // Transfer SDGs to SymbolDepGroups. 588 DepGroupForBlock.reserve(DepGroupForBlock.size() + 1); 589 for (auto &[B, SDG] : DepGroupForBlock) { 590 assert(!SDG.Symbols.empty() && "SymbolDependenceGroup covers no symbols"); 591 if (!SDG.Dependencies.empty()) 592 SymbolDepGroups.push_back(std::move(SDG)); 593 } 594 if (!SynthSDG.Symbols.empty() && !SynthSDG.Dependencies.empty()) 595 SymbolDepGroups.push_back(std::move(SynthSDG)); 596 597 return Error::success(); 598 } 599 600 BlockDependenciesMap computeBlockNonLocalDeps(LinkGraph &G) { 601 // First calculate the reachable-via-non-local-symbol blocks for each block. 602 struct BlockInfo { 603 DenseSet<Block *> Dependencies; 604 DenseSet<Block *> Dependants; 605 bool DependenciesChanged = true; 606 }; 607 DenseMap<Block *, BlockInfo> BlockInfos; 608 SmallVector<Block *> WorkList; 609 610 // Pre-allocate map entries. This prevents any iterator/reference 611 // invalidation in the next loop. 612 for (auto *B : G.blocks()) 613 (void)BlockInfos[B]; 614 615 // Build initial worklist, record block dependencies/dependants and 616 // non-local symbol dependencies. 617 for (auto *B : G.blocks()) { 618 auto &BI = BlockInfos[B]; 619 for (auto &E : B->edges()) { 620 if (E.getTarget().getScope() == Scope::Local && 621 !E.getTarget().isAbsolute()) { 622 auto &TgtB = E.getTarget().getBlock(); 623 if (&TgtB != B) { 624 BI.Dependencies.insert(&TgtB); 625 BlockInfos[&TgtB].Dependants.insert(B); 626 } 627 } 628 } 629 630 // If this node has both dependants and dependencies then add it to the 631 // worklist to propagate the dependencies to the dependants. 632 if (!BI.Dependants.empty() && !BI.Dependencies.empty()) 633 WorkList.push_back(B); 634 } 635 636 // Propagate block-level dependencies through the block-dependence graph. 637 while (!WorkList.empty()) { 638 auto *B = WorkList.pop_back_val(); 639 640 auto &BI = BlockInfos[B]; 641 assert(BI.DependenciesChanged && 642 "Block in worklist has unchanged dependencies"); 643 BI.DependenciesChanged = false; 644 for (auto *Dependant : BI.Dependants) { 645 auto &DependantBI = BlockInfos[Dependant]; 646 for (auto *Dependency : BI.Dependencies) { 647 if (Dependant != Dependency && 648 DependantBI.Dependencies.insert(Dependency).second) 649 if (!DependantBI.DependenciesChanged) { 650 DependantBI.DependenciesChanged = true; 651 WorkList.push_back(Dependant); 652 } 653 } 654 } 655 } 656 657 DenseMap<const Block *, DenseSet<Block *>> BlockDeps; 658 for (auto &KV : BlockInfos) 659 BlockDeps[KV.first] = std::move(KV.second.Dependencies); 660 661 return BlockDependenciesMap(Layer.getExecutionSession(), 662 std::move(BlockDeps)); 663 } 664 665 ObjectLinkingLayer &Layer; 666 std::vector<std::shared_ptr<ObjectLinkingLayer::Plugin>> Plugins; 667 std::unique_ptr<MaterializationResponsibility> MR; 668 std::unique_ptr<MemoryBuffer> ObjBuffer; 669 DenseMap<Block *, SymbolNameSet> ExternalBlockDeps; 670 DenseMap<Block *, SymbolNameSet> InternalBlockDeps; 671 DenseMap<NonOwningSymbolStringPtr, JITDylib *> SymbolSourceJDs; 672 std::vector<SymbolDependenceGroup> SymbolDepGroups; 673 }; 674 675 ObjectLinkingLayer::Plugin::~Plugin() = default; 676 677 char ObjectLinkingLayer::ID; 678 679 using BaseT = RTTIExtends<ObjectLinkingLayer, ObjectLayer>; 680 681 ObjectLinkingLayer::ObjectLinkingLayer(ExecutionSession &ES) 682 : BaseT(ES), MemMgr(ES.getExecutorProcessControl().getMemMgr()) { 683 ES.registerResourceManager(*this); 684 } 685 686 ObjectLinkingLayer::ObjectLinkingLayer(ExecutionSession &ES, 687 JITLinkMemoryManager &MemMgr) 688 : BaseT(ES), MemMgr(MemMgr) { 689 ES.registerResourceManager(*this); 690 } 691 692 ObjectLinkingLayer::ObjectLinkingLayer( 693 ExecutionSession &ES, std::unique_ptr<JITLinkMemoryManager> MemMgr) 694 : BaseT(ES), MemMgr(*MemMgr), MemMgrOwnership(std::move(MemMgr)) { 695 ES.registerResourceManager(*this); 696 } 697 698 ObjectLinkingLayer::~ObjectLinkingLayer() { 699 assert(Allocs.empty() && "Layer destroyed with resources still attached"); 700 getExecutionSession().deregisterResourceManager(*this); 701 } 702 703 Error ObjectLinkingLayer::add(ResourceTrackerSP RT, 704 std::unique_ptr<LinkGraph> G) { 705 auto &JD = RT->getJITDylib(); 706 return JD.define(LinkGraphMaterializationUnit::Create(*this, std::move(G)), 707 std::move(RT)); 708 } 709 710 void ObjectLinkingLayer::emit(std::unique_ptr<MaterializationResponsibility> R, 711 std::unique_ptr<MemoryBuffer> O) { 712 assert(O && "Object must not be null"); 713 MemoryBufferRef ObjBuffer = O->getMemBufferRef(); 714 715 auto Ctx = std::make_unique<ObjectLinkingLayerJITLinkContext>( 716 *this, std::move(R), std::move(O)); 717 if (auto G = createLinkGraphFromObject(ObjBuffer)) { 718 Ctx->notifyMaterializing(**G); 719 link(std::move(*G), std::move(Ctx)); 720 } else { 721 Ctx->notifyFailed(G.takeError()); 722 } 723 } 724 725 void ObjectLinkingLayer::emit(std::unique_ptr<MaterializationResponsibility> R, 726 std::unique_ptr<LinkGraph> G) { 727 auto Ctx = std::make_unique<ObjectLinkingLayerJITLinkContext>( 728 *this, std::move(R), nullptr); 729 Ctx->notifyMaterializing(*G); 730 link(std::move(G), std::move(Ctx)); 731 } 732 733 Error ObjectLinkingLayer::recordFinalizedAlloc( 734 MaterializationResponsibility &MR, FinalizedAlloc FA) { 735 auto Err = MR.withResourceKeyDo( 736 [&](ResourceKey K) { Allocs[K].push_back(std::move(FA)); }); 737 738 if (Err) 739 Err = joinErrors(std::move(Err), MemMgr.deallocate(std::move(FA))); 740 741 return Err; 742 } 743 744 Error ObjectLinkingLayer::handleRemoveResources(JITDylib &JD, ResourceKey K) { 745 746 { 747 Error Err = Error::success(); 748 for (auto &P : Plugins) 749 Err = joinErrors(std::move(Err), P->notifyRemovingResources(JD, K)); 750 if (Err) 751 return Err; 752 } 753 754 std::vector<FinalizedAlloc> AllocsToRemove; 755 getExecutionSession().runSessionLocked([&] { 756 auto I = Allocs.find(K); 757 if (I != Allocs.end()) { 758 std::swap(AllocsToRemove, I->second); 759 Allocs.erase(I); 760 } 761 }); 762 763 if (AllocsToRemove.empty()) 764 return Error::success(); 765 766 return MemMgr.deallocate(std::move(AllocsToRemove)); 767 } 768 769 void ObjectLinkingLayer::handleTransferResources(JITDylib &JD, 770 ResourceKey DstKey, 771 ResourceKey SrcKey) { 772 auto I = Allocs.find(SrcKey); 773 if (I != Allocs.end()) { 774 auto &SrcAllocs = I->second; 775 auto &DstAllocs = Allocs[DstKey]; 776 DstAllocs.reserve(DstAllocs.size() + SrcAllocs.size()); 777 for (auto &Alloc : SrcAllocs) 778 DstAllocs.push_back(std::move(Alloc)); 779 780 // Erase SrcKey entry using value rather than iterator I: I may have been 781 // invalidated when we looked up DstKey. 782 Allocs.erase(SrcKey); 783 } 784 785 for (auto &P : Plugins) 786 P->notifyTransferringResources(JD, DstKey, SrcKey); 787 } 788 789 EHFrameRegistrationPlugin::EHFrameRegistrationPlugin( 790 ExecutionSession &ES, std::unique_ptr<EHFrameRegistrar> Registrar) 791 : ES(ES), Registrar(std::move(Registrar)) {} 792 793 void EHFrameRegistrationPlugin::modifyPassConfig( 794 MaterializationResponsibility &MR, LinkGraph &G, 795 PassConfiguration &PassConfig) { 796 797 PassConfig.PostFixupPasses.push_back(createEHFrameRecorderPass( 798 G.getTargetTriple(), [this, &MR](ExecutorAddr Addr, size_t Size) { 799 if (Addr) { 800 std::lock_guard<std::mutex> Lock(EHFramePluginMutex); 801 assert(!InProcessLinks.count(&MR) && 802 "Link for MR already being tracked?"); 803 InProcessLinks[&MR] = {Addr, Size}; 804 } 805 })); 806 } 807 808 Error EHFrameRegistrationPlugin::notifyEmitted( 809 MaterializationResponsibility &MR) { 810 811 ExecutorAddrRange EmittedRange; 812 { 813 std::lock_guard<std::mutex> Lock(EHFramePluginMutex); 814 815 auto EHFrameRangeItr = InProcessLinks.find(&MR); 816 if (EHFrameRangeItr == InProcessLinks.end()) 817 return Error::success(); 818 819 EmittedRange = EHFrameRangeItr->second; 820 assert(EmittedRange.Start && "eh-frame addr to register can not be null"); 821 InProcessLinks.erase(EHFrameRangeItr); 822 } 823 824 if (auto Err = MR.withResourceKeyDo( 825 [&](ResourceKey K) { EHFrameRanges[K].push_back(EmittedRange); })) 826 return Err; 827 828 return Registrar->registerEHFrames(EmittedRange); 829 } 830 831 Error EHFrameRegistrationPlugin::notifyFailed( 832 MaterializationResponsibility &MR) { 833 std::lock_guard<std::mutex> Lock(EHFramePluginMutex); 834 InProcessLinks.erase(&MR); 835 return Error::success(); 836 } 837 838 Error EHFrameRegistrationPlugin::notifyRemovingResources(JITDylib &JD, 839 ResourceKey K) { 840 std::vector<ExecutorAddrRange> RangesToRemove; 841 842 ES.runSessionLocked([&] { 843 auto I = EHFrameRanges.find(K); 844 if (I != EHFrameRanges.end()) { 845 RangesToRemove = std::move(I->second); 846 EHFrameRanges.erase(I); 847 } 848 }); 849 850 Error Err = Error::success(); 851 while (!RangesToRemove.empty()) { 852 auto RangeToRemove = RangesToRemove.back(); 853 RangesToRemove.pop_back(); 854 assert(RangeToRemove.Start && "Untracked eh-frame range must not be null"); 855 Err = joinErrors(std::move(Err), 856 Registrar->deregisterEHFrames(RangeToRemove)); 857 } 858 859 return Err; 860 } 861 862 void EHFrameRegistrationPlugin::notifyTransferringResources( 863 JITDylib &JD, ResourceKey DstKey, ResourceKey SrcKey) { 864 auto SI = EHFrameRanges.find(SrcKey); 865 if (SI == EHFrameRanges.end()) 866 return; 867 868 auto DI = EHFrameRanges.find(DstKey); 869 if (DI != EHFrameRanges.end()) { 870 auto &SrcRanges = SI->second; 871 auto &DstRanges = DI->second; 872 DstRanges.reserve(DstRanges.size() + SrcRanges.size()); 873 for (auto &SrcRange : SrcRanges) 874 DstRanges.push_back(std::move(SrcRange)); 875 EHFrameRanges.erase(SI); 876 } else { 877 // We need to move SrcKey's ranges over without invalidating the SI 878 // iterator. 879 auto Tmp = std::move(SI->second); 880 EHFrameRanges.erase(SI); 881 EHFrameRanges[DstKey] = std::move(Tmp); 882 } 883 } 884 885 } // End namespace orc. 886 } // End namespace llvm. 887