1 //===---- IndirectionUtils.cpp - Utilities for call indirection in Orc ----===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 9 #include "llvm/ExecutionEngine/Orc/IndirectionUtils.h" 10 #include "llvm/ExecutionEngine/JITLink/x86_64.h" 11 #include "llvm/ExecutionEngine/Orc/OrcABISupport.h" 12 #include "llvm/IR/IRBuilder.h" 13 #include "llvm/IR/Module.h" 14 #include "llvm/MC/MCDisassembler/MCDisassembler.h" 15 #include "llvm/MC/MCInstrAnalysis.h" 16 #include "llvm/TargetParser/Triple.h" 17 #include "llvm/Transforms/Utils/Cloning.h" 18 19 #define DEBUG_TYPE "orc" 20 21 using namespace llvm; 22 using namespace llvm::orc; 23 24 namespace { 25 26 class CompileCallbackMaterializationUnit : public orc::MaterializationUnit { 27 public: 28 using CompileFunction = JITCompileCallbackManager::CompileFunction; 29 30 CompileCallbackMaterializationUnit(SymbolStringPtr Name, 31 CompileFunction Compile) 32 : MaterializationUnit(Interface( 33 SymbolFlagsMap({{Name, JITSymbolFlags::Exported}}), nullptr)), 34 Name(std::move(Name)), Compile(std::move(Compile)) {} 35 36 StringRef getName() const override { return "<Compile Callbacks>"; } 37 38 private: 39 void materialize(std::unique_ptr<MaterializationResponsibility> R) override { 40 SymbolMap Result; 41 Result[Name] = {Compile(), JITSymbolFlags::Exported}; 42 // No dependencies, so these calls cannot fail. 43 cantFail(R->notifyResolved(Result)); 44 cantFail(R->notifyEmitted({})); 45 } 46 47 void discard(const JITDylib &JD, const SymbolStringPtr &Name) override { 48 llvm_unreachable("Discard should never occur on a LMU?"); 49 } 50 51 SymbolStringPtr Name; 52 CompileFunction Compile; 53 }; 54 55 } // namespace 56 57 namespace llvm { 58 namespace orc { 59 60 TrampolinePool::~TrampolinePool() = default; 61 void IndirectStubsManager::anchor() {} 62 63 Expected<ExecutorAddr> 64 JITCompileCallbackManager::getCompileCallback(CompileFunction Compile) { 65 if (auto TrampolineAddr = TP->getTrampoline()) { 66 auto CallbackName = 67 ES.intern(std::string("cc") + std::to_string(++NextCallbackId)); 68 69 std::lock_guard<std::mutex> Lock(CCMgrMutex); 70 AddrToSymbol[*TrampolineAddr] = CallbackName; 71 cantFail( 72 CallbacksJD.define(std::make_unique<CompileCallbackMaterializationUnit>( 73 std::move(CallbackName), std::move(Compile)))); 74 return *TrampolineAddr; 75 } else 76 return TrampolineAddr.takeError(); 77 } 78 79 ExecutorAddr 80 JITCompileCallbackManager::executeCompileCallback(ExecutorAddr TrampolineAddr) { 81 SymbolStringPtr Name; 82 83 { 84 std::unique_lock<std::mutex> Lock(CCMgrMutex); 85 auto I = AddrToSymbol.find(TrampolineAddr); 86 87 // If this address is not associated with a compile callback then report an 88 // error to the execution session and return ErrorHandlerAddress to the 89 // callee. 90 if (I == AddrToSymbol.end()) { 91 Lock.unlock(); 92 ES.reportError( 93 make_error<StringError>("No compile callback for trampoline at " + 94 formatv("{0:x}", TrampolineAddr), 95 inconvertibleErrorCode())); 96 return ErrorHandlerAddress; 97 } else 98 Name = I->second; 99 } 100 101 if (auto Sym = 102 ES.lookup(makeJITDylibSearchOrder( 103 &CallbacksJD, JITDylibLookupFlags::MatchAllSymbols), 104 Name)) 105 return Sym->getAddress(); 106 else { 107 llvm::dbgs() << "Didn't find callback.\n"; 108 // If anything goes wrong materializing Sym then report it to the session 109 // and return the ErrorHandlerAddress; 110 ES.reportError(Sym.takeError()); 111 return ErrorHandlerAddress; 112 } 113 } 114 115 Error IndirectStubsManager::redirect(JITDylib &JD, const SymbolMap &NewDests) { 116 for (auto &[Name, Dest] : NewDests) 117 if (auto Err = updatePointer(*Name, Dest.getAddress())) 118 return Err; 119 return Error::success(); 120 } 121 122 void IndirectStubsManager::emitRedirectableSymbols( 123 std::unique_ptr<MaterializationResponsibility> MR, SymbolMap InitialDests) { 124 StubInitsMap StubInits; 125 for (auto &[Name, Dest] : InitialDests) 126 StubInits[*Name] = {Dest.getAddress(), Dest.getFlags()}; 127 if (auto Err = createStubs(StubInits)) { 128 MR->getExecutionSession().reportError(std::move(Err)); 129 return MR->failMaterialization(); 130 } 131 SymbolMap Stubs; 132 for (auto &[Name, Dest] : InitialDests) { 133 auto StubSym = findStub(*Name, false); 134 assert(StubSym.getAddress() && "Stub symbol should be present"); 135 Stubs[Name] = StubSym; 136 } 137 if (auto Err = MR->notifyResolved(Stubs)) { 138 MR->getExecutionSession().reportError(std::move(Err)); 139 return MR->failMaterialization(); 140 } 141 if (auto Err = MR->notifyEmitted({})) { 142 MR->getExecutionSession().reportError(std::move(Err)); 143 return MR->failMaterialization(); 144 } 145 } 146 147 Expected<std::unique_ptr<JITCompileCallbackManager>> 148 createLocalCompileCallbackManager(const Triple &T, ExecutionSession &ES, 149 ExecutorAddr ErrorHandlerAddress) { 150 switch (T.getArch()) { 151 default: 152 return make_error<StringError>( 153 std::string("No callback manager available for ") + T.str(), 154 inconvertibleErrorCode()); 155 case Triple::aarch64: 156 case Triple::aarch64_32: { 157 typedef orc::LocalJITCompileCallbackManager<orc::OrcAArch64> CCMgrT; 158 return CCMgrT::Create(ES, ErrorHandlerAddress); 159 } 160 161 case Triple::x86: { 162 typedef orc::LocalJITCompileCallbackManager<orc::OrcI386> CCMgrT; 163 return CCMgrT::Create(ES, ErrorHandlerAddress); 164 } 165 166 case Triple::loongarch64: { 167 typedef orc::LocalJITCompileCallbackManager<orc::OrcLoongArch64> CCMgrT; 168 return CCMgrT::Create(ES, ErrorHandlerAddress); 169 } 170 171 case Triple::mips: { 172 typedef orc::LocalJITCompileCallbackManager<orc::OrcMips32Be> CCMgrT; 173 return CCMgrT::Create(ES, ErrorHandlerAddress); 174 } 175 case Triple::mipsel: { 176 typedef orc::LocalJITCompileCallbackManager<orc::OrcMips32Le> CCMgrT; 177 return CCMgrT::Create(ES, ErrorHandlerAddress); 178 } 179 180 case Triple::mips64: 181 case Triple::mips64el: { 182 typedef orc::LocalJITCompileCallbackManager<orc::OrcMips64> CCMgrT; 183 return CCMgrT::Create(ES, ErrorHandlerAddress); 184 } 185 186 case Triple::riscv64: { 187 typedef orc::LocalJITCompileCallbackManager<orc::OrcRiscv64> CCMgrT; 188 return CCMgrT::Create(ES, ErrorHandlerAddress); 189 } 190 191 case Triple::x86_64: { 192 if (T.getOS() == Triple::OSType::Win32) { 193 typedef orc::LocalJITCompileCallbackManager<orc::OrcX86_64_Win32> CCMgrT; 194 return CCMgrT::Create(ES, ErrorHandlerAddress); 195 } else { 196 typedef orc::LocalJITCompileCallbackManager<orc::OrcX86_64_SysV> CCMgrT; 197 return CCMgrT::Create(ES, ErrorHandlerAddress); 198 } 199 } 200 201 } 202 } 203 204 std::function<std::unique_ptr<IndirectStubsManager>()> 205 createLocalIndirectStubsManagerBuilder(const Triple &T) { 206 switch (T.getArch()) { 207 default: 208 return [](){ 209 return std::make_unique< 210 orc::LocalIndirectStubsManager<orc::OrcGenericABI>>(); 211 }; 212 213 case Triple::aarch64: 214 case Triple::aarch64_32: 215 return [](){ 216 return std::make_unique< 217 orc::LocalIndirectStubsManager<orc::OrcAArch64>>(); 218 }; 219 220 case Triple::x86: 221 return [](){ 222 return std::make_unique< 223 orc::LocalIndirectStubsManager<orc::OrcI386>>(); 224 }; 225 226 case Triple::loongarch64: 227 return []() { 228 return std::make_unique< 229 orc::LocalIndirectStubsManager<orc::OrcLoongArch64>>(); 230 }; 231 232 case Triple::mips: 233 return [](){ 234 return std::make_unique< 235 orc::LocalIndirectStubsManager<orc::OrcMips32Be>>(); 236 }; 237 238 case Triple::mipsel: 239 return [](){ 240 return std::make_unique< 241 orc::LocalIndirectStubsManager<orc::OrcMips32Le>>(); 242 }; 243 244 case Triple::mips64: 245 case Triple::mips64el: 246 return [](){ 247 return std::make_unique< 248 orc::LocalIndirectStubsManager<orc::OrcMips64>>(); 249 }; 250 251 case Triple::riscv64: 252 return []() { 253 return std::make_unique< 254 orc::LocalIndirectStubsManager<orc::OrcRiscv64>>(); 255 }; 256 257 case Triple::x86_64: 258 if (T.getOS() == Triple::OSType::Win32) { 259 return [](){ 260 return std::make_unique< 261 orc::LocalIndirectStubsManager<orc::OrcX86_64_Win32>>(); 262 }; 263 } else { 264 return [](){ 265 return std::make_unique< 266 orc::LocalIndirectStubsManager<orc::OrcX86_64_SysV>>(); 267 }; 268 } 269 270 } 271 } 272 273 Constant* createIRTypedAddress(FunctionType &FT, ExecutorAddr Addr) { 274 Constant *AddrIntVal = 275 ConstantInt::get(Type::getInt64Ty(FT.getContext()), Addr.getValue()); 276 Constant *AddrPtrVal = ConstantExpr::getIntToPtr( 277 AddrIntVal, PointerType::get(FT.getContext(), 0)); 278 return AddrPtrVal; 279 } 280 281 GlobalVariable* createImplPointer(PointerType &PT, Module &M, 282 const Twine &Name, Constant *Initializer) { 283 auto IP = new GlobalVariable(M, &PT, false, GlobalValue::ExternalLinkage, 284 Initializer, Name, nullptr, 285 GlobalValue::NotThreadLocal, 0, true); 286 IP->setVisibility(GlobalValue::HiddenVisibility); 287 return IP; 288 } 289 290 void makeStub(Function &F, Value &ImplPointer) { 291 assert(F.isDeclaration() && "Can't turn a definition into a stub."); 292 assert(F.getParent() && "Function isn't in a module."); 293 Module &M = *F.getParent(); 294 BasicBlock *EntryBlock = BasicBlock::Create(M.getContext(), "entry", &F); 295 IRBuilder<> Builder(EntryBlock); 296 LoadInst *ImplAddr = Builder.CreateLoad(F.getType(), &ImplPointer); 297 std::vector<Value*> CallArgs; 298 for (auto &A : F.args()) 299 CallArgs.push_back(&A); 300 CallInst *Call = Builder.CreateCall(F.getFunctionType(), ImplAddr, CallArgs); 301 Call->setTailCall(); 302 Call->setAttributes(F.getAttributes()); 303 if (F.getReturnType()->isVoidTy()) 304 Builder.CreateRetVoid(); 305 else 306 Builder.CreateRet(Call); 307 } 308 309 std::vector<GlobalValue *> SymbolLinkagePromoter::operator()(Module &M) { 310 std::vector<GlobalValue *> PromotedGlobals; 311 312 for (auto &GV : M.global_values()) { 313 bool Promoted = true; 314 315 // Rename if necessary. 316 if (!GV.hasName()) 317 GV.setName("__orc_anon." + Twine(NextId++)); 318 else if (GV.getName().starts_with("\01L")) 319 GV.setName("__" + GV.getName().substr(1) + "." + Twine(NextId++)); 320 else if (GV.hasLocalLinkage()) 321 GV.setName("__orc_lcl." + GV.getName() + "." + Twine(NextId++)); 322 else 323 Promoted = false; 324 325 if (GV.hasLocalLinkage()) { 326 GV.setLinkage(GlobalValue::ExternalLinkage); 327 GV.setVisibility(GlobalValue::HiddenVisibility); 328 Promoted = true; 329 } 330 GV.setUnnamedAddr(GlobalValue::UnnamedAddr::None); 331 332 if (Promoted) 333 PromotedGlobals.push_back(&GV); 334 } 335 336 return PromotedGlobals; 337 } 338 339 Function* cloneFunctionDecl(Module &Dst, const Function &F, 340 ValueToValueMapTy *VMap) { 341 Function *NewF = 342 Function::Create(cast<FunctionType>(F.getValueType()), 343 F.getLinkage(), F.getName(), &Dst); 344 NewF->copyAttributesFrom(&F); 345 346 if (VMap) { 347 (*VMap)[&F] = NewF; 348 auto NewArgI = NewF->arg_begin(); 349 for (auto ArgI = F.arg_begin(), ArgE = F.arg_end(); ArgI != ArgE; 350 ++ArgI, ++NewArgI) 351 (*VMap)[&*ArgI] = &*NewArgI; 352 } 353 354 return NewF; 355 } 356 357 GlobalVariable* cloneGlobalVariableDecl(Module &Dst, const GlobalVariable &GV, 358 ValueToValueMapTy *VMap) { 359 GlobalVariable *NewGV = new GlobalVariable( 360 Dst, GV.getValueType(), GV.isConstant(), 361 GV.getLinkage(), nullptr, GV.getName(), nullptr, 362 GV.getThreadLocalMode(), GV.getType()->getAddressSpace()); 363 NewGV->copyAttributesFrom(&GV); 364 if (VMap) 365 (*VMap)[&GV] = NewGV; 366 return NewGV; 367 } 368 369 GlobalAlias* cloneGlobalAliasDecl(Module &Dst, const GlobalAlias &OrigA, 370 ValueToValueMapTy &VMap) { 371 assert(OrigA.getAliasee() && "Original alias doesn't have an aliasee?"); 372 auto *NewA = GlobalAlias::create(OrigA.getValueType(), 373 OrigA.getType()->getPointerAddressSpace(), 374 OrigA.getLinkage(), OrigA.getName(), &Dst); 375 NewA->copyAttributesFrom(&OrigA); 376 VMap[&OrigA] = NewA; 377 return NewA; 378 } 379 380 Error addFunctionPointerRelocationsToCurrentSymbol(jitlink::Symbol &Sym, 381 jitlink::LinkGraph &G, 382 MCDisassembler &Disassembler, 383 MCInstrAnalysis &MIA) { 384 // AArch64 appears to already come with the necessary relocations. Among other 385 // architectures, only x86_64 is currently implemented here. 386 if (G.getTargetTriple().getArch() != Triple::x86_64) 387 return Error::success(); 388 389 raw_null_ostream CommentStream; 390 auto &STI = Disassembler.getSubtargetInfo(); 391 392 // Determine the function bounds 393 auto &B = Sym.getBlock(); 394 assert(!B.isZeroFill() && "expected content block"); 395 auto SymAddress = Sym.getAddress(); 396 auto SymStartInBlock = 397 (const uint8_t *)B.getContent().data() + Sym.getOffset(); 398 auto SymSize = Sym.getSize() ? Sym.getSize() : B.getSize() - Sym.getOffset(); 399 auto Content = ArrayRef(SymStartInBlock, SymSize); 400 401 LLVM_DEBUG(dbgs() << "Adding self-relocations to " << Sym.getName() << "\n"); 402 403 SmallDenseSet<uintptr_t, 8> ExistingRelocations; 404 for (auto &E : B.edges()) { 405 if (E.isRelocation()) 406 ExistingRelocations.insert(E.getOffset()); 407 } 408 409 size_t I = 0; 410 while (I < Content.size()) { 411 MCInst Instr; 412 uint64_t InstrSize = 0; 413 uint64_t InstrStart = SymAddress.getValue() + I; 414 auto DecodeStatus = Disassembler.getInstruction( 415 Instr, InstrSize, Content.drop_front(I), InstrStart, CommentStream); 416 if (DecodeStatus != MCDisassembler::Success) { 417 LLVM_DEBUG(dbgs() << "Aborting due to disassembly failure at address " 418 << InstrStart); 419 return make_error<StringError>( 420 formatv("failed to disassemble at address {0:x16}", InstrStart), 421 inconvertibleErrorCode()); 422 } 423 // Advance to the next instruction. 424 I += InstrSize; 425 426 // Check for a PC-relative address equal to the symbol itself. 427 auto PCRelAddr = 428 MIA.evaluateMemoryOperandAddress(Instr, &STI, InstrStart, InstrSize); 429 if (!PCRelAddr || *PCRelAddr != SymAddress.getValue()) 430 continue; 431 432 auto RelocOffInInstr = 433 MIA.getMemoryOperandRelocationOffset(Instr, InstrSize); 434 if (!RelocOffInInstr || InstrSize - *RelocOffInInstr != 4) { 435 LLVM_DEBUG(dbgs() << "Skipping unknown self-relocation at " 436 << InstrStart); 437 continue; 438 } 439 440 auto RelocOffInBlock = orc::ExecutorAddr(InstrStart) + *RelocOffInInstr - 441 SymAddress + Sym.getOffset(); 442 if (ExistingRelocations.contains(RelocOffInBlock)) 443 continue; 444 445 LLVM_DEBUG(dbgs() << "Adding delta32 self-relocation at " << InstrStart); 446 B.addEdge(jitlink::x86_64::Delta32, RelocOffInBlock, Sym, /*Addend=*/-4); 447 } 448 return Error::success(); 449 } 450 451 } // End namespace orc. 452 } // End namespace llvm. 453