1 //===------- EPCIndirectionUtils.cpp -- EPC based indirection APIs --------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 9 #include "llvm/ExecutionEngine/Orc/EPCIndirectionUtils.h" 10 11 #include "llvm/ExecutionEngine/Orc/ExecutorProcessControl.h" 12 #include "llvm/Support/MathExtras.h" 13 14 #include <future> 15 16 using namespace llvm; 17 using namespace llvm::orc; 18 19 namespace llvm { 20 namespace orc { 21 22 class EPCIndirectionUtilsAccess { 23 public: 24 using IndirectStubInfo = EPCIndirectionUtils::IndirectStubInfo; 25 using IndirectStubInfoVector = EPCIndirectionUtils::IndirectStubInfoVector; 26 27 static Expected<IndirectStubInfoVector> 28 getIndirectStubs(EPCIndirectionUtils &EPCIU, unsigned NumStubs) { 29 return EPCIU.getIndirectStubs(NumStubs); 30 }; 31 }; 32 33 } // end namespace orc 34 } // end namespace llvm 35 36 namespace { 37 38 class EPCTrampolinePool : public TrampolinePool { 39 public: 40 EPCTrampolinePool(EPCIndirectionUtils &EPCIU); 41 Error deallocatePool(); 42 43 protected: 44 Error grow() override; 45 46 using FinalizedAlloc = jitlink::JITLinkMemoryManager::FinalizedAlloc; 47 48 EPCIndirectionUtils &EPCIU; 49 unsigned TrampolineSize = 0; 50 unsigned TrampolinesPerPage = 0; 51 std::vector<FinalizedAlloc> TrampolineBlocks; 52 }; 53 54 class EPCIndirectStubsManager : public IndirectStubsManager, 55 private EPCIndirectionUtilsAccess { 56 public: 57 EPCIndirectStubsManager(EPCIndirectionUtils &EPCIU) : EPCIU(EPCIU) {} 58 59 Error deallocateStubs(); 60 61 Error createStub(StringRef StubName, JITTargetAddress StubAddr, 62 JITSymbolFlags StubFlags) override; 63 64 Error createStubs(const StubInitsMap &StubInits) override; 65 66 JITEvaluatedSymbol findStub(StringRef Name, bool ExportedStubsOnly) override; 67 68 JITEvaluatedSymbol findPointer(StringRef Name) override; 69 70 Error updatePointer(StringRef Name, JITTargetAddress NewAddr) override; 71 72 private: 73 using StubInfo = std::pair<IndirectStubInfo, JITSymbolFlags>; 74 75 std::mutex ISMMutex; 76 EPCIndirectionUtils &EPCIU; 77 StringMap<StubInfo> StubInfos; 78 }; 79 80 EPCTrampolinePool::EPCTrampolinePool(EPCIndirectionUtils &EPCIU) 81 : EPCIU(EPCIU) { 82 auto &EPC = EPCIU.getExecutorProcessControl(); 83 auto &ABI = EPCIU.getABISupport(); 84 85 TrampolineSize = ABI.getTrampolineSize(); 86 TrampolinesPerPage = 87 (EPC.getPageSize() - ABI.getPointerSize()) / TrampolineSize; 88 } 89 90 Error EPCTrampolinePool::deallocatePool() { 91 Error Err = Error::success(); 92 std::promise<MSVCPError> DeallocResultP; 93 auto DeallocResultF = DeallocResultP.get_future(); 94 95 EPCIU.getExecutorProcessControl().getMemMgr().deallocate( 96 std::move(TrampolineBlocks), 97 [&](Error Err) { DeallocResultP.set_value(std::move(Err)); }); 98 99 return DeallocResultF.get(); 100 } 101 102 Error EPCTrampolinePool::grow() { 103 using namespace jitlink; 104 105 assert(AvailableTrampolines.empty() && 106 "Grow called with trampolines still available"); 107 108 auto ResolverAddress = EPCIU.getResolverBlockAddress(); 109 assert(ResolverAddress && "Resolver address can not be null"); 110 111 auto &EPC = EPCIU.getExecutorProcessControl(); 112 auto PageSize = EPC.getPageSize(); 113 auto Alloc = SimpleSegmentAlloc::Create( 114 EPC.getMemMgr(), nullptr, 115 {{MemProt::Read | MemProt::Exec, {PageSize, Align(PageSize)}}}); 116 if (!Alloc) 117 return Alloc.takeError(); 118 119 unsigned NumTrampolines = TrampolinesPerPage; 120 121 auto SegInfo = Alloc->getSegInfo(MemProt::Read | MemProt::Exec); 122 EPCIU.getABISupport().writeTrampolines( 123 SegInfo.WorkingMem.data(), SegInfo.Addr, ResolverAddress, NumTrampolines); 124 for (unsigned I = 0; I < NumTrampolines; ++I) 125 AvailableTrampolines.push_back(SegInfo.Addr + (I * TrampolineSize)); 126 127 auto FA = Alloc->finalize(); 128 if (!FA) 129 return FA.takeError(); 130 131 TrampolineBlocks.push_back(std::move(*FA)); 132 133 return Error::success(); 134 } 135 136 Error EPCIndirectStubsManager::createStub(StringRef StubName, 137 JITTargetAddress StubAddr, 138 JITSymbolFlags StubFlags) { 139 StubInitsMap SIM; 140 SIM[StubName] = std::make_pair(StubAddr, StubFlags); 141 return createStubs(SIM); 142 } 143 144 Error EPCIndirectStubsManager::createStubs(const StubInitsMap &StubInits) { 145 auto AvailableStubInfos = getIndirectStubs(EPCIU, StubInits.size()); 146 if (!AvailableStubInfos) 147 return AvailableStubInfos.takeError(); 148 149 { 150 std::lock_guard<std::mutex> Lock(ISMMutex); 151 unsigned ASIdx = 0; 152 for (auto &SI : StubInits) { 153 auto &A = (*AvailableStubInfos)[ASIdx++]; 154 StubInfos[SI.first()] = std::make_pair(A, SI.second.second); 155 } 156 } 157 158 auto &MemAccess = EPCIU.getExecutorProcessControl().getMemoryAccess(); 159 switch (EPCIU.getABISupport().getPointerSize()) { 160 case 4: { 161 unsigned ASIdx = 0; 162 std::vector<tpctypes::UInt32Write> PtrUpdates; 163 for (auto &SI : StubInits) 164 PtrUpdates.push_back({(*AvailableStubInfos)[ASIdx++].PointerAddress, 165 static_cast<uint32_t>(SI.second.first)}); 166 return MemAccess.writeUInt32s(PtrUpdates); 167 } 168 case 8: { 169 unsigned ASIdx = 0; 170 std::vector<tpctypes::UInt64Write> PtrUpdates; 171 for (auto &SI : StubInits) 172 PtrUpdates.push_back({(*AvailableStubInfos)[ASIdx++].PointerAddress, 173 static_cast<uint64_t>(SI.second.first)}); 174 return MemAccess.writeUInt64s(PtrUpdates); 175 } 176 default: 177 return make_error<StringError>("Unsupported pointer size", 178 inconvertibleErrorCode()); 179 } 180 } 181 182 JITEvaluatedSymbol EPCIndirectStubsManager::findStub(StringRef Name, 183 bool ExportedStubsOnly) { 184 std::lock_guard<std::mutex> Lock(ISMMutex); 185 auto I = StubInfos.find(Name); 186 if (I == StubInfos.end()) 187 return nullptr; 188 return {I->second.first.StubAddress, I->second.second}; 189 } 190 191 JITEvaluatedSymbol EPCIndirectStubsManager::findPointer(StringRef Name) { 192 std::lock_guard<std::mutex> Lock(ISMMutex); 193 auto I = StubInfos.find(Name); 194 if (I == StubInfos.end()) 195 return nullptr; 196 return {I->second.first.PointerAddress, I->second.second}; 197 } 198 199 Error EPCIndirectStubsManager::updatePointer(StringRef Name, 200 JITTargetAddress NewAddr) { 201 202 JITTargetAddress PtrAddr = 0; 203 { 204 std::lock_guard<std::mutex> Lock(ISMMutex); 205 auto I = StubInfos.find(Name); 206 if (I == StubInfos.end()) 207 return make_error<StringError>("Unknown stub name", 208 inconvertibleErrorCode()); 209 PtrAddr = I->second.first.PointerAddress; 210 } 211 212 auto &MemAccess = EPCIU.getExecutorProcessControl().getMemoryAccess(); 213 switch (EPCIU.getABISupport().getPointerSize()) { 214 case 4: { 215 tpctypes::UInt32Write PUpdate(PtrAddr, NewAddr); 216 return MemAccess.writeUInt32s(PUpdate); 217 } 218 case 8: { 219 tpctypes::UInt64Write PUpdate(PtrAddr, NewAddr); 220 return MemAccess.writeUInt64s(PUpdate); 221 } 222 default: 223 return make_error<StringError>("Unsupported pointer size", 224 inconvertibleErrorCode()); 225 } 226 } 227 228 } // end anonymous namespace. 229 230 namespace llvm { 231 namespace orc { 232 233 EPCIndirectionUtils::ABISupport::~ABISupport() {} 234 235 Expected<std::unique_ptr<EPCIndirectionUtils>> 236 EPCIndirectionUtils::Create(ExecutorProcessControl &EPC) { 237 const auto &TT = EPC.getTargetTriple(); 238 switch (TT.getArch()) { 239 default: 240 return make_error<StringError>( 241 std::string("No EPCIndirectionUtils available for ") + TT.str(), 242 inconvertibleErrorCode()); 243 case Triple::aarch64: 244 case Triple::aarch64_32: 245 return CreateWithABI<OrcAArch64>(EPC); 246 247 case Triple::x86: 248 return CreateWithABI<OrcI386>(EPC); 249 250 case Triple::mips: 251 return CreateWithABI<OrcMips32Be>(EPC); 252 253 case Triple::mipsel: 254 return CreateWithABI<OrcMips32Le>(EPC); 255 256 case Triple::mips64: 257 case Triple::mips64el: 258 return CreateWithABI<OrcMips64>(EPC); 259 260 case Triple::x86_64: 261 if (TT.getOS() == Triple::OSType::Win32) 262 return CreateWithABI<OrcX86_64_Win32>(EPC); 263 else 264 return CreateWithABI<OrcX86_64_SysV>(EPC); 265 } 266 } 267 268 Error EPCIndirectionUtils::cleanup() { 269 270 auto &MemMgr = EPC.getMemMgr(); 271 auto Err = MemMgr.deallocate(std::move(IndirectStubAllocs)); 272 273 if (TP) 274 Err = joinErrors(std::move(Err), 275 static_cast<EPCTrampolinePool &>(*TP).deallocatePool()); 276 277 if (ResolverBlock) 278 Err = 279 joinErrors(std::move(Err), MemMgr.deallocate(std::move(ResolverBlock))); 280 281 return Err; 282 } 283 284 Expected<JITTargetAddress> 285 EPCIndirectionUtils::writeResolverBlock(JITTargetAddress ReentryFnAddr, 286 JITTargetAddress ReentryCtxAddr) { 287 using namespace jitlink; 288 289 assert(ABI && "ABI can not be null"); 290 auto ResolverSize = ABI->getResolverCodeSize(); 291 292 auto Alloc = 293 SimpleSegmentAlloc::Create(EPC.getMemMgr(), nullptr, 294 {{MemProt::Read | MemProt::Exec, 295 {ResolverSize, Align(EPC.getPageSize())}}}); 296 297 if (!Alloc) 298 return Alloc.takeError(); 299 300 auto SegInfo = Alloc->getSegInfo(MemProt::Read | MemProt::Exec); 301 ABI->writeResolverCode(SegInfo.WorkingMem.data(), SegInfo.Addr, ReentryFnAddr, 302 ReentryCtxAddr); 303 304 auto FA = Alloc->finalize(); 305 if (!FA) 306 return FA.takeError(); 307 308 ResolverBlock = std::move(*FA); 309 return SegInfo.Addr; 310 } 311 312 std::unique_ptr<IndirectStubsManager> 313 EPCIndirectionUtils::createIndirectStubsManager() { 314 return std::make_unique<EPCIndirectStubsManager>(*this); 315 } 316 317 TrampolinePool &EPCIndirectionUtils::getTrampolinePool() { 318 if (!TP) 319 TP = std::make_unique<EPCTrampolinePool>(*this); 320 return *TP; 321 } 322 323 LazyCallThroughManager &EPCIndirectionUtils::createLazyCallThroughManager( 324 ExecutionSession &ES, JITTargetAddress ErrorHandlerAddr) { 325 assert(!LCTM && 326 "createLazyCallThroughManager can not have been called before"); 327 LCTM = std::make_unique<LazyCallThroughManager>(ES, ErrorHandlerAddr, 328 &getTrampolinePool()); 329 return *LCTM; 330 } 331 332 EPCIndirectionUtils::EPCIndirectionUtils(ExecutorProcessControl &EPC, 333 std::unique_ptr<ABISupport> ABI) 334 : EPC(EPC), ABI(std::move(ABI)) { 335 assert(this->ABI && "ABI can not be null"); 336 337 assert(EPC.getPageSize() > getABISupport().getStubSize() && 338 "Stubs larger than one page are not supported"); 339 } 340 341 Expected<EPCIndirectionUtils::IndirectStubInfoVector> 342 EPCIndirectionUtils::getIndirectStubs(unsigned NumStubs) { 343 using namespace jitlink; 344 345 std::lock_guard<std::mutex> Lock(EPCUIMutex); 346 347 // If there aren't enough stubs available then allocate some more. 348 if (NumStubs > AvailableIndirectStubs.size()) { 349 auto NumStubsToAllocate = NumStubs; 350 auto PageSize = EPC.getPageSize(); 351 auto StubBytes = alignTo(NumStubsToAllocate * ABI->getStubSize(), PageSize); 352 NumStubsToAllocate = StubBytes / ABI->getStubSize(); 353 auto PtrBytes = 354 alignTo(NumStubsToAllocate * ABI->getPointerSize(), PageSize); 355 356 auto StubProt = MemProt::Read | MemProt::Exec; 357 auto PtrProt = MemProt::Read | MemProt::Write; 358 359 auto Alloc = SimpleSegmentAlloc::Create( 360 EPC.getMemMgr(), nullptr, 361 {{StubProt, {static_cast<size_t>(StubBytes), Align(PageSize)}}, 362 {PtrProt, {PtrBytes, Align(PageSize)}}}); 363 364 if (!Alloc) 365 return Alloc.takeError(); 366 367 auto StubSeg = Alloc->getSegInfo(StubProt); 368 auto PtrSeg = Alloc->getSegInfo(PtrProt); 369 370 ABI->writeIndirectStubsBlock(StubSeg.WorkingMem.data(), StubSeg.Addr, 371 PtrSeg.Addr, NumStubsToAllocate); 372 373 auto FA = Alloc->finalize(); 374 if (!FA) 375 return FA.takeError(); 376 377 IndirectStubAllocs.push_back(std::move(*FA)); 378 379 auto StubExecutorAddr = StubSeg.Addr; 380 auto PtrExecutorAddr = PtrSeg.Addr; 381 for (unsigned I = 0; I != NumStubsToAllocate; ++I) { 382 AvailableIndirectStubs.push_back( 383 IndirectStubInfo(StubExecutorAddr, PtrExecutorAddr)); 384 StubExecutorAddr += ABI->getStubSize(); 385 PtrExecutorAddr += ABI->getPointerSize(); 386 } 387 } 388 389 assert(NumStubs <= AvailableIndirectStubs.size() && 390 "Sufficient stubs should have been allocated above"); 391 392 IndirectStubInfoVector Result; 393 while (NumStubs--) { 394 Result.push_back(AvailableIndirectStubs.back()); 395 AvailableIndirectStubs.pop_back(); 396 } 397 398 return std::move(Result); 399 } 400 401 static JITTargetAddress reentry(JITTargetAddress LCTMAddr, 402 JITTargetAddress TrampolineAddr) { 403 auto &LCTM = *jitTargetAddressToPointer<LazyCallThroughManager *>(LCTMAddr); 404 std::promise<JITTargetAddress> LandingAddrP; 405 auto LandingAddrF = LandingAddrP.get_future(); 406 LCTM.resolveTrampolineLandingAddress( 407 TrampolineAddr, 408 [&](JITTargetAddress Addr) { LandingAddrP.set_value(Addr); }); 409 return LandingAddrF.get(); 410 } 411 412 Error setUpInProcessLCTMReentryViaEPCIU(EPCIndirectionUtils &EPCIU) { 413 auto &LCTM = EPCIU.getLazyCallThroughManager(); 414 return EPCIU 415 .writeResolverBlock(pointerToJITTargetAddress(&reentry), 416 pointerToJITTargetAddress(&LCTM)) 417 .takeError(); 418 } 419 420 } // end namespace orc 421 } // end namespace llvm 422