1 //===------- EPCIndirectionUtils.cpp -- EPC based indirection APIs --------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 9 #include "llvm/ExecutionEngine/Orc/EPCIndirectionUtils.h" 10 11 #include "llvm/ExecutionEngine/Orc/ExecutorProcessControl.h" 12 #include "llvm/Support/MathExtras.h" 13 14 #include <future> 15 16 using namespace llvm; 17 using namespace llvm::orc; 18 19 namespace llvm { 20 namespace orc { 21 22 class EPCIndirectionUtilsAccess { 23 public: 24 using IndirectStubInfo = EPCIndirectionUtils::IndirectStubInfo; 25 using IndirectStubInfoVector = EPCIndirectionUtils::IndirectStubInfoVector; 26 27 static Expected<IndirectStubInfoVector> 28 getIndirectStubs(EPCIndirectionUtils &EPCIU, unsigned NumStubs) { 29 return EPCIU.getIndirectStubs(NumStubs); 30 }; 31 }; 32 33 } // end namespace orc 34 } // end namespace llvm 35 36 namespace { 37 38 class EPCTrampolinePool : public TrampolinePool { 39 public: 40 EPCTrampolinePool(EPCIndirectionUtils &EPCIU); 41 Error deallocatePool(); 42 43 protected: 44 Error grow() override; 45 46 using FinalizedAlloc = jitlink::JITLinkMemoryManager::FinalizedAlloc; 47 48 EPCIndirectionUtils &EPCIU; 49 unsigned TrampolineSize = 0; 50 unsigned TrampolinesPerPage = 0; 51 std::vector<FinalizedAlloc> TrampolineBlocks; 52 }; 53 54 class EPCIndirectStubsManager : public IndirectStubsManager, 55 private EPCIndirectionUtilsAccess { 56 public: 57 EPCIndirectStubsManager(EPCIndirectionUtils &EPCIU) : EPCIU(EPCIU) {} 58 59 Error deallocateStubs(); 60 61 Error createStub(StringRef StubName, JITTargetAddress StubAddr, 62 JITSymbolFlags StubFlags) override; 63 64 Error createStubs(const StubInitsMap &StubInits) override; 65 66 JITEvaluatedSymbol findStub(StringRef Name, bool ExportedStubsOnly) override; 67 68 JITEvaluatedSymbol findPointer(StringRef Name) override; 69 70 Error updatePointer(StringRef Name, JITTargetAddress NewAddr) override; 71 72 private: 73 using StubInfo = std::pair<IndirectStubInfo, JITSymbolFlags>; 74 75 std::mutex ISMMutex; 76 EPCIndirectionUtils &EPCIU; 77 StringMap<StubInfo> StubInfos; 78 }; 79 80 EPCTrampolinePool::EPCTrampolinePool(EPCIndirectionUtils &EPCIU) 81 : EPCIU(EPCIU) { 82 auto &EPC = EPCIU.getExecutorProcessControl(); 83 auto &ABI = EPCIU.getABISupport(); 84 85 TrampolineSize = ABI.getTrampolineSize(); 86 TrampolinesPerPage = 87 (EPC.getPageSize() - ABI.getPointerSize()) / TrampolineSize; 88 } 89 90 Error EPCTrampolinePool::deallocatePool() { 91 Error Err = Error::success(); 92 std::promise<MSVCPError> DeallocResultP; 93 auto DeallocResultF = DeallocResultP.get_future(); 94 95 EPCIU.getExecutorProcessControl().getMemMgr().deallocate( 96 std::move(TrampolineBlocks), 97 [&](Error Err) { DeallocResultP.set_value(std::move(Err)); }); 98 99 return DeallocResultF.get(); 100 } 101 102 Error EPCTrampolinePool::grow() { 103 using namespace jitlink; 104 105 assert(AvailableTrampolines.empty() && 106 "Grow called with trampolines still available"); 107 108 auto ResolverAddress = EPCIU.getResolverBlockAddress(); 109 assert(ResolverAddress && "Resolver address can not be null"); 110 111 auto &EPC = EPCIU.getExecutorProcessControl(); 112 auto PageSize = EPC.getPageSize(); 113 auto Alloc = SimpleSegmentAlloc::Create( 114 EPC.getMemMgr(), nullptr, 115 {{MemProt::Read | MemProt::Exec, {PageSize, Align(PageSize)}}}); 116 if (!Alloc) 117 return Alloc.takeError(); 118 119 unsigned NumTrampolines = TrampolinesPerPage; 120 121 auto SegInfo = Alloc->getSegInfo(MemProt::Read | MemProt::Exec); 122 EPCIU.getABISupport().writeTrampolines(SegInfo.WorkingMem.data(), 123 SegInfo.Addr.getValue(), 124 ResolverAddress, NumTrampolines); 125 for (unsigned I = 0; I < NumTrampolines; ++I) 126 AvailableTrampolines.push_back(SegInfo.Addr.getValue() + 127 (I * TrampolineSize)); 128 129 auto FA = Alloc->finalize(); 130 if (!FA) 131 return FA.takeError(); 132 133 TrampolineBlocks.push_back(std::move(*FA)); 134 135 return Error::success(); 136 } 137 138 Error EPCIndirectStubsManager::createStub(StringRef StubName, 139 JITTargetAddress StubAddr, 140 JITSymbolFlags StubFlags) { 141 StubInitsMap SIM; 142 SIM[StubName] = std::make_pair(StubAddr, StubFlags); 143 return createStubs(SIM); 144 } 145 146 Error EPCIndirectStubsManager::createStubs(const StubInitsMap &StubInits) { 147 auto AvailableStubInfos = getIndirectStubs(EPCIU, StubInits.size()); 148 if (!AvailableStubInfos) 149 return AvailableStubInfos.takeError(); 150 151 { 152 std::lock_guard<std::mutex> Lock(ISMMutex); 153 unsigned ASIdx = 0; 154 for (auto &SI : StubInits) { 155 auto &A = (*AvailableStubInfos)[ASIdx++]; 156 StubInfos[SI.first()] = std::make_pair(A, SI.second.second); 157 } 158 } 159 160 auto &MemAccess = EPCIU.getExecutorProcessControl().getMemoryAccess(); 161 switch (EPCIU.getABISupport().getPointerSize()) { 162 case 4: { 163 unsigned ASIdx = 0; 164 std::vector<tpctypes::UInt32Write> PtrUpdates; 165 for (auto &SI : StubInits) 166 PtrUpdates.push_back( 167 {ExecutorAddr((*AvailableStubInfos)[ASIdx++].PointerAddress), 168 static_cast<uint32_t>(SI.second.first)}); 169 return MemAccess.writeUInt32s(PtrUpdates); 170 } 171 case 8: { 172 unsigned ASIdx = 0; 173 std::vector<tpctypes::UInt64Write> PtrUpdates; 174 for (auto &SI : StubInits) 175 PtrUpdates.push_back( 176 {ExecutorAddr((*AvailableStubInfos)[ASIdx++].PointerAddress), 177 static_cast<uint64_t>(SI.second.first)}); 178 return MemAccess.writeUInt64s(PtrUpdates); 179 } 180 default: 181 return make_error<StringError>("Unsupported pointer size", 182 inconvertibleErrorCode()); 183 } 184 } 185 186 JITEvaluatedSymbol EPCIndirectStubsManager::findStub(StringRef Name, 187 bool ExportedStubsOnly) { 188 std::lock_guard<std::mutex> Lock(ISMMutex); 189 auto I = StubInfos.find(Name); 190 if (I == StubInfos.end()) 191 return nullptr; 192 return {I->second.first.StubAddress, I->second.second}; 193 } 194 195 JITEvaluatedSymbol EPCIndirectStubsManager::findPointer(StringRef Name) { 196 std::lock_guard<std::mutex> Lock(ISMMutex); 197 auto I = StubInfos.find(Name); 198 if (I == StubInfos.end()) 199 return nullptr; 200 return {I->second.first.PointerAddress, I->second.second}; 201 } 202 203 Error EPCIndirectStubsManager::updatePointer(StringRef Name, 204 JITTargetAddress NewAddr) { 205 206 JITTargetAddress PtrAddr = 0; 207 { 208 std::lock_guard<std::mutex> Lock(ISMMutex); 209 auto I = StubInfos.find(Name); 210 if (I == StubInfos.end()) 211 return make_error<StringError>("Unknown stub name", 212 inconvertibleErrorCode()); 213 PtrAddr = I->second.first.PointerAddress; 214 } 215 216 auto &MemAccess = EPCIU.getExecutorProcessControl().getMemoryAccess(); 217 switch (EPCIU.getABISupport().getPointerSize()) { 218 case 4: { 219 tpctypes::UInt32Write PUpdate(ExecutorAddr(PtrAddr), NewAddr); 220 return MemAccess.writeUInt32s(PUpdate); 221 } 222 case 8: { 223 tpctypes::UInt64Write PUpdate(ExecutorAddr(PtrAddr), NewAddr); 224 return MemAccess.writeUInt64s(PUpdate); 225 } 226 default: 227 return make_error<StringError>("Unsupported pointer size", 228 inconvertibleErrorCode()); 229 } 230 } 231 232 } // end anonymous namespace. 233 234 namespace llvm { 235 namespace orc { 236 237 EPCIndirectionUtils::ABISupport::~ABISupport() = default; 238 239 Expected<std::unique_ptr<EPCIndirectionUtils>> 240 EPCIndirectionUtils::Create(ExecutorProcessControl &EPC) { 241 const auto &TT = EPC.getTargetTriple(); 242 switch (TT.getArch()) { 243 default: 244 return make_error<StringError>( 245 std::string("No EPCIndirectionUtils available for ") + TT.str(), 246 inconvertibleErrorCode()); 247 case Triple::aarch64: 248 case Triple::aarch64_32: 249 return CreateWithABI<OrcAArch64>(EPC); 250 251 case Triple::x86: 252 return CreateWithABI<OrcI386>(EPC); 253 254 case Triple::mips: 255 return CreateWithABI<OrcMips32Be>(EPC); 256 257 case Triple::mipsel: 258 return CreateWithABI<OrcMips32Le>(EPC); 259 260 case Triple::mips64: 261 case Triple::mips64el: 262 return CreateWithABI<OrcMips64>(EPC); 263 264 case Triple::riscv64: 265 return CreateWithABI<OrcRiscv64>(EPC); 266 267 case Triple::x86_64: 268 if (TT.getOS() == Triple::OSType::Win32) 269 return CreateWithABI<OrcX86_64_Win32>(EPC); 270 else 271 return CreateWithABI<OrcX86_64_SysV>(EPC); 272 } 273 } 274 275 Error EPCIndirectionUtils::cleanup() { 276 277 auto &MemMgr = EPC.getMemMgr(); 278 auto Err = MemMgr.deallocate(std::move(IndirectStubAllocs)); 279 280 if (TP) 281 Err = joinErrors(std::move(Err), 282 static_cast<EPCTrampolinePool &>(*TP).deallocatePool()); 283 284 if (ResolverBlock) 285 Err = 286 joinErrors(std::move(Err), MemMgr.deallocate(std::move(ResolverBlock))); 287 288 return Err; 289 } 290 291 Expected<JITTargetAddress> 292 EPCIndirectionUtils::writeResolverBlock(JITTargetAddress ReentryFnAddr, 293 JITTargetAddress ReentryCtxAddr) { 294 using namespace jitlink; 295 296 assert(ABI && "ABI can not be null"); 297 auto ResolverSize = ABI->getResolverCodeSize(); 298 299 auto Alloc = 300 SimpleSegmentAlloc::Create(EPC.getMemMgr(), nullptr, 301 {{MemProt::Read | MemProt::Exec, 302 {ResolverSize, Align(EPC.getPageSize())}}}); 303 304 if (!Alloc) 305 return Alloc.takeError(); 306 307 auto SegInfo = Alloc->getSegInfo(MemProt::Read | MemProt::Exec); 308 ResolverBlockAddr = SegInfo.Addr.getValue(); 309 ABI->writeResolverCode(SegInfo.WorkingMem.data(), ResolverBlockAddr, 310 ReentryFnAddr, ReentryCtxAddr); 311 312 auto FA = Alloc->finalize(); 313 if (!FA) 314 return FA.takeError(); 315 316 ResolverBlock = std::move(*FA); 317 return ResolverBlockAddr; 318 } 319 320 std::unique_ptr<IndirectStubsManager> 321 EPCIndirectionUtils::createIndirectStubsManager() { 322 return std::make_unique<EPCIndirectStubsManager>(*this); 323 } 324 325 TrampolinePool &EPCIndirectionUtils::getTrampolinePool() { 326 if (!TP) 327 TP = std::make_unique<EPCTrampolinePool>(*this); 328 return *TP; 329 } 330 331 LazyCallThroughManager &EPCIndirectionUtils::createLazyCallThroughManager( 332 ExecutionSession &ES, JITTargetAddress ErrorHandlerAddr) { 333 assert(!LCTM && 334 "createLazyCallThroughManager can not have been called before"); 335 LCTM = std::make_unique<LazyCallThroughManager>(ES, ErrorHandlerAddr, 336 &getTrampolinePool()); 337 return *LCTM; 338 } 339 340 EPCIndirectionUtils::EPCIndirectionUtils(ExecutorProcessControl &EPC, 341 std::unique_ptr<ABISupport> ABI) 342 : EPC(EPC), ABI(std::move(ABI)) { 343 assert(this->ABI && "ABI can not be null"); 344 345 assert(EPC.getPageSize() > getABISupport().getStubSize() && 346 "Stubs larger than one page are not supported"); 347 } 348 349 Expected<EPCIndirectionUtils::IndirectStubInfoVector> 350 EPCIndirectionUtils::getIndirectStubs(unsigned NumStubs) { 351 using namespace jitlink; 352 353 std::lock_guard<std::mutex> Lock(EPCUIMutex); 354 355 // If there aren't enough stubs available then allocate some more. 356 if (NumStubs > AvailableIndirectStubs.size()) { 357 auto NumStubsToAllocate = NumStubs; 358 auto PageSize = EPC.getPageSize(); 359 auto StubBytes = alignTo(NumStubsToAllocate * ABI->getStubSize(), PageSize); 360 NumStubsToAllocate = StubBytes / ABI->getStubSize(); 361 auto PtrBytes = 362 alignTo(NumStubsToAllocate * ABI->getPointerSize(), PageSize); 363 364 auto StubProt = MemProt::Read | MemProt::Exec; 365 auto PtrProt = MemProt::Read | MemProt::Write; 366 367 auto Alloc = SimpleSegmentAlloc::Create( 368 EPC.getMemMgr(), nullptr, 369 {{StubProt, {static_cast<size_t>(StubBytes), Align(PageSize)}}, 370 {PtrProt, {static_cast<size_t>(PtrBytes), Align(PageSize)}}}); 371 372 if (!Alloc) 373 return Alloc.takeError(); 374 375 auto StubSeg = Alloc->getSegInfo(StubProt); 376 auto PtrSeg = Alloc->getSegInfo(PtrProt); 377 378 ABI->writeIndirectStubsBlock(StubSeg.WorkingMem.data(), 379 StubSeg.Addr.getValue(), 380 PtrSeg.Addr.getValue(), NumStubsToAllocate); 381 382 auto FA = Alloc->finalize(); 383 if (!FA) 384 return FA.takeError(); 385 386 IndirectStubAllocs.push_back(std::move(*FA)); 387 388 auto StubExecutorAddr = StubSeg.Addr; 389 auto PtrExecutorAddr = PtrSeg.Addr; 390 for (unsigned I = 0; I != NumStubsToAllocate; ++I) { 391 AvailableIndirectStubs.push_back(IndirectStubInfo( 392 StubExecutorAddr.getValue(), PtrExecutorAddr.getValue())); 393 StubExecutorAddr += ABI->getStubSize(); 394 PtrExecutorAddr += ABI->getPointerSize(); 395 } 396 } 397 398 assert(NumStubs <= AvailableIndirectStubs.size() && 399 "Sufficient stubs should have been allocated above"); 400 401 IndirectStubInfoVector Result; 402 while (NumStubs--) { 403 Result.push_back(AvailableIndirectStubs.back()); 404 AvailableIndirectStubs.pop_back(); 405 } 406 407 return std::move(Result); 408 } 409 410 static JITTargetAddress reentry(JITTargetAddress LCTMAddr, 411 JITTargetAddress TrampolineAddr) { 412 auto &LCTM = *jitTargetAddressToPointer<LazyCallThroughManager *>(LCTMAddr); 413 std::promise<JITTargetAddress> LandingAddrP; 414 auto LandingAddrF = LandingAddrP.get_future(); 415 LCTM.resolveTrampolineLandingAddress( 416 TrampolineAddr, 417 [&](JITTargetAddress Addr) { LandingAddrP.set_value(Addr); }); 418 return LandingAddrF.get(); 419 } 420 421 Error setUpInProcessLCTMReentryViaEPCIU(EPCIndirectionUtils &EPCIU) { 422 auto &LCTM = EPCIU.getLazyCallThroughManager(); 423 return EPCIU 424 .writeResolverBlock(pointerToJITTargetAddress(&reentry), 425 pointerToJITTargetAddress(&LCTM)) 426 .takeError(); 427 } 428 429 } // end namespace orc 430 } // end namespace llvm 431