1 //===---- IndirectionUtils.cpp - Utilities for call indirection in Orc ----===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 9 #include "llvm/ExecutionEngine/Orc/IndirectionUtils.h" 10 #include "llvm/ADT/STLExtras.h" 11 #include "llvm/ExecutionEngine/JITLink/x86_64.h" 12 #include "llvm/ExecutionEngine/Orc/OrcABISupport.h" 13 #include "llvm/IR/IRBuilder.h" 14 #include "llvm/IR/Module.h" 15 #include "llvm/MC/MCDisassembler/MCDisassembler.h" 16 #include "llvm/MC/MCInstrAnalysis.h" 17 #include "llvm/Support/Format.h" 18 #include "llvm/TargetParser/Triple.h" 19 #include "llvm/Transforms/Utils/Cloning.h" 20 #include <sstream> 21 22 #define DEBUG_TYPE "orc" 23 24 using namespace llvm; 25 using namespace llvm::orc; 26 27 namespace { 28 29 class CompileCallbackMaterializationUnit : public orc::MaterializationUnit { 30 public: 31 using CompileFunction = JITCompileCallbackManager::CompileFunction; 32 33 CompileCallbackMaterializationUnit(SymbolStringPtr Name, 34 CompileFunction Compile) 35 : MaterializationUnit(Interface( 36 SymbolFlagsMap({{Name, JITSymbolFlags::Exported}}), nullptr)), 37 Name(std::move(Name)), Compile(std::move(Compile)) {} 38 39 StringRef getName() const override { return "<Compile Callbacks>"; } 40 41 private: 42 void materialize(std::unique_ptr<MaterializationResponsibility> R) override { 43 SymbolMap Result; 44 Result[Name] = {Compile(), JITSymbolFlags::Exported}; 45 // No dependencies, so these calls cannot fail. 46 cantFail(R->notifyResolved(Result)); 47 cantFail(R->notifyEmitted({})); 48 } 49 50 void discard(const JITDylib &JD, const SymbolStringPtr &Name) override { 51 llvm_unreachable("Discard should never occur on a LMU?"); 52 } 53 54 SymbolStringPtr Name; 55 CompileFunction Compile; 56 }; 57 58 } // namespace 59 60 namespace llvm { 61 namespace orc { 62 63 TrampolinePool::~TrampolinePool() = default; 64 void IndirectStubsManager::anchor() {} 65 66 Expected<ExecutorAddr> 67 JITCompileCallbackManager::getCompileCallback(CompileFunction Compile) { 68 if (auto TrampolineAddr = TP->getTrampoline()) { 69 auto CallbackName = 70 ES.intern(std::string("cc") + std::to_string(++NextCallbackId)); 71 72 std::lock_guard<std::mutex> Lock(CCMgrMutex); 73 AddrToSymbol[*TrampolineAddr] = CallbackName; 74 cantFail( 75 CallbacksJD.define(std::make_unique<CompileCallbackMaterializationUnit>( 76 std::move(CallbackName), std::move(Compile)))); 77 return *TrampolineAddr; 78 } else 79 return TrampolineAddr.takeError(); 80 } 81 82 ExecutorAddr 83 JITCompileCallbackManager::executeCompileCallback(ExecutorAddr TrampolineAddr) { 84 SymbolStringPtr Name; 85 86 { 87 std::unique_lock<std::mutex> Lock(CCMgrMutex); 88 auto I = AddrToSymbol.find(TrampolineAddr); 89 90 // If this address is not associated with a compile callback then report an 91 // error to the execution session and return ErrorHandlerAddress to the 92 // callee. 93 if (I == AddrToSymbol.end()) { 94 Lock.unlock(); 95 ES.reportError( 96 make_error<StringError>("No compile callback for trampoline at " + 97 formatv("{0:x}", TrampolineAddr), 98 inconvertibleErrorCode())); 99 return ErrorHandlerAddress; 100 } else 101 Name = I->second; 102 } 103 104 if (auto Sym = 105 ES.lookup(makeJITDylibSearchOrder( 106 &CallbacksJD, JITDylibLookupFlags::MatchAllSymbols), 107 Name)) 108 return Sym->getAddress(); 109 else { 110 llvm::dbgs() << "Didn't find callback.\n"; 111 // If anything goes wrong materializing Sym then report it to the session 112 // and return the ErrorHandlerAddress; 113 ES.reportError(Sym.takeError()); 114 return ErrorHandlerAddress; 115 } 116 } 117 118 Error IndirectStubsManager::redirect(JITDylib &JD, const SymbolMap &NewDests) { 119 for (auto &[Name, Dest] : NewDests) 120 if (auto Err = updatePointer(*Name, Dest.getAddress())) 121 return Err; 122 return Error::success(); 123 } 124 125 void IndirectStubsManager::emitRedirectableSymbols( 126 std::unique_ptr<MaterializationResponsibility> MR, SymbolMap InitialDests) { 127 StubInitsMap StubInits; 128 for (auto &[Name, Dest] : InitialDests) 129 StubInits[*Name] = {Dest.getAddress(), Dest.getFlags()}; 130 if (auto Err = createStubs(StubInits)) { 131 MR->getExecutionSession().reportError(std::move(Err)); 132 return MR->failMaterialization(); 133 } 134 SymbolMap Stubs; 135 for (auto &[Name, Dest] : InitialDests) { 136 auto StubSym = findStub(*Name, false); 137 assert(StubSym.getAddress() && "Stub symbol should be present"); 138 Stubs[Name] = StubSym; 139 } 140 if (auto Err = MR->notifyResolved(Stubs)) { 141 MR->getExecutionSession().reportError(std::move(Err)); 142 return MR->failMaterialization(); 143 } 144 if (auto Err = MR->notifyEmitted({})) { 145 MR->getExecutionSession().reportError(std::move(Err)); 146 return MR->failMaterialization(); 147 } 148 } 149 150 Expected<std::unique_ptr<JITCompileCallbackManager>> 151 createLocalCompileCallbackManager(const Triple &T, ExecutionSession &ES, 152 ExecutorAddr ErrorHandlerAddress) { 153 switch (T.getArch()) { 154 default: 155 return make_error<StringError>( 156 std::string("No callback manager available for ") + T.str(), 157 inconvertibleErrorCode()); 158 case Triple::aarch64: 159 case Triple::aarch64_32: { 160 typedef orc::LocalJITCompileCallbackManager<orc::OrcAArch64> CCMgrT; 161 return CCMgrT::Create(ES, ErrorHandlerAddress); 162 } 163 164 case Triple::x86: { 165 typedef orc::LocalJITCompileCallbackManager<orc::OrcI386> CCMgrT; 166 return CCMgrT::Create(ES, ErrorHandlerAddress); 167 } 168 169 case Triple::loongarch64: { 170 typedef orc::LocalJITCompileCallbackManager<orc::OrcLoongArch64> CCMgrT; 171 return CCMgrT::Create(ES, ErrorHandlerAddress); 172 } 173 174 case Triple::mips: { 175 typedef orc::LocalJITCompileCallbackManager<orc::OrcMips32Be> CCMgrT; 176 return CCMgrT::Create(ES, ErrorHandlerAddress); 177 } 178 case Triple::mipsel: { 179 typedef orc::LocalJITCompileCallbackManager<orc::OrcMips32Le> CCMgrT; 180 return CCMgrT::Create(ES, ErrorHandlerAddress); 181 } 182 183 case Triple::mips64: 184 case Triple::mips64el: { 185 typedef orc::LocalJITCompileCallbackManager<orc::OrcMips64> CCMgrT; 186 return CCMgrT::Create(ES, ErrorHandlerAddress); 187 } 188 189 case Triple::riscv64: { 190 typedef orc::LocalJITCompileCallbackManager<orc::OrcRiscv64> CCMgrT; 191 return CCMgrT::Create(ES, ErrorHandlerAddress); 192 } 193 194 case Triple::x86_64: { 195 if (T.getOS() == Triple::OSType::Win32) { 196 typedef orc::LocalJITCompileCallbackManager<orc::OrcX86_64_Win32> CCMgrT; 197 return CCMgrT::Create(ES, ErrorHandlerAddress); 198 } else { 199 typedef orc::LocalJITCompileCallbackManager<orc::OrcX86_64_SysV> CCMgrT; 200 return CCMgrT::Create(ES, ErrorHandlerAddress); 201 } 202 } 203 204 } 205 } 206 207 std::function<std::unique_ptr<IndirectStubsManager>()> 208 createLocalIndirectStubsManagerBuilder(const Triple &T) { 209 switch (T.getArch()) { 210 default: 211 return [](){ 212 return std::make_unique< 213 orc::LocalIndirectStubsManager<orc::OrcGenericABI>>(); 214 }; 215 216 case Triple::aarch64: 217 case Triple::aarch64_32: 218 return [](){ 219 return std::make_unique< 220 orc::LocalIndirectStubsManager<orc::OrcAArch64>>(); 221 }; 222 223 case Triple::x86: 224 return [](){ 225 return std::make_unique< 226 orc::LocalIndirectStubsManager<orc::OrcI386>>(); 227 }; 228 229 case Triple::loongarch64: 230 return []() { 231 return std::make_unique< 232 orc::LocalIndirectStubsManager<orc::OrcLoongArch64>>(); 233 }; 234 235 case Triple::mips: 236 return [](){ 237 return std::make_unique< 238 orc::LocalIndirectStubsManager<orc::OrcMips32Be>>(); 239 }; 240 241 case Triple::mipsel: 242 return [](){ 243 return std::make_unique< 244 orc::LocalIndirectStubsManager<orc::OrcMips32Le>>(); 245 }; 246 247 case Triple::mips64: 248 case Triple::mips64el: 249 return [](){ 250 return std::make_unique< 251 orc::LocalIndirectStubsManager<orc::OrcMips64>>(); 252 }; 253 254 case Triple::riscv64: 255 return []() { 256 return std::make_unique< 257 orc::LocalIndirectStubsManager<orc::OrcRiscv64>>(); 258 }; 259 260 case Triple::x86_64: 261 if (T.getOS() == Triple::OSType::Win32) { 262 return [](){ 263 return std::make_unique< 264 orc::LocalIndirectStubsManager<orc::OrcX86_64_Win32>>(); 265 }; 266 } else { 267 return [](){ 268 return std::make_unique< 269 orc::LocalIndirectStubsManager<orc::OrcX86_64_SysV>>(); 270 }; 271 } 272 273 } 274 } 275 276 Constant* createIRTypedAddress(FunctionType &FT, ExecutorAddr Addr) { 277 Constant *AddrIntVal = 278 ConstantInt::get(Type::getInt64Ty(FT.getContext()), Addr.getValue()); 279 Constant *AddrPtrVal = 280 ConstantExpr::getIntToPtr(AddrIntVal, PointerType::get(&FT, 0)); 281 return AddrPtrVal; 282 } 283 284 GlobalVariable* createImplPointer(PointerType &PT, Module &M, 285 const Twine &Name, Constant *Initializer) { 286 auto IP = new GlobalVariable(M, &PT, false, GlobalValue::ExternalLinkage, 287 Initializer, Name, nullptr, 288 GlobalValue::NotThreadLocal, 0, true); 289 IP->setVisibility(GlobalValue::HiddenVisibility); 290 return IP; 291 } 292 293 void makeStub(Function &F, Value &ImplPointer) { 294 assert(F.isDeclaration() && "Can't turn a definition into a stub."); 295 assert(F.getParent() && "Function isn't in a module."); 296 Module &M = *F.getParent(); 297 BasicBlock *EntryBlock = BasicBlock::Create(M.getContext(), "entry", &F); 298 IRBuilder<> Builder(EntryBlock); 299 LoadInst *ImplAddr = Builder.CreateLoad(F.getType(), &ImplPointer); 300 std::vector<Value*> CallArgs; 301 for (auto &A : F.args()) 302 CallArgs.push_back(&A); 303 CallInst *Call = Builder.CreateCall(F.getFunctionType(), ImplAddr, CallArgs); 304 Call->setTailCall(); 305 Call->setAttributes(F.getAttributes()); 306 if (F.getReturnType()->isVoidTy()) 307 Builder.CreateRetVoid(); 308 else 309 Builder.CreateRet(Call); 310 } 311 312 std::vector<GlobalValue *> SymbolLinkagePromoter::operator()(Module &M) { 313 std::vector<GlobalValue *> PromotedGlobals; 314 315 for (auto &GV : M.global_values()) { 316 bool Promoted = true; 317 318 // Rename if necessary. 319 if (!GV.hasName()) 320 GV.setName("__orc_anon." + Twine(NextId++)); 321 else if (GV.getName().starts_with("\01L")) 322 GV.setName("__" + GV.getName().substr(1) + "." + Twine(NextId++)); 323 else if (GV.hasLocalLinkage()) 324 GV.setName("__orc_lcl." + GV.getName() + "." + Twine(NextId++)); 325 else 326 Promoted = false; 327 328 if (GV.hasLocalLinkage()) { 329 GV.setLinkage(GlobalValue::ExternalLinkage); 330 GV.setVisibility(GlobalValue::HiddenVisibility); 331 Promoted = true; 332 } 333 GV.setUnnamedAddr(GlobalValue::UnnamedAddr::None); 334 335 if (Promoted) 336 PromotedGlobals.push_back(&GV); 337 } 338 339 return PromotedGlobals; 340 } 341 342 Function* cloneFunctionDecl(Module &Dst, const Function &F, 343 ValueToValueMapTy *VMap) { 344 Function *NewF = 345 Function::Create(cast<FunctionType>(F.getValueType()), 346 F.getLinkage(), F.getName(), &Dst); 347 NewF->copyAttributesFrom(&F); 348 349 if (VMap) { 350 (*VMap)[&F] = NewF; 351 auto NewArgI = NewF->arg_begin(); 352 for (auto ArgI = F.arg_begin(), ArgE = F.arg_end(); ArgI != ArgE; 353 ++ArgI, ++NewArgI) 354 (*VMap)[&*ArgI] = &*NewArgI; 355 } 356 357 return NewF; 358 } 359 360 GlobalVariable* cloneGlobalVariableDecl(Module &Dst, const GlobalVariable &GV, 361 ValueToValueMapTy *VMap) { 362 GlobalVariable *NewGV = new GlobalVariable( 363 Dst, GV.getValueType(), GV.isConstant(), 364 GV.getLinkage(), nullptr, GV.getName(), nullptr, 365 GV.getThreadLocalMode(), GV.getType()->getAddressSpace()); 366 NewGV->copyAttributesFrom(&GV); 367 if (VMap) 368 (*VMap)[&GV] = NewGV; 369 return NewGV; 370 } 371 372 GlobalAlias* cloneGlobalAliasDecl(Module &Dst, const GlobalAlias &OrigA, 373 ValueToValueMapTy &VMap) { 374 assert(OrigA.getAliasee() && "Original alias doesn't have an aliasee?"); 375 auto *NewA = GlobalAlias::create(OrigA.getValueType(), 376 OrigA.getType()->getPointerAddressSpace(), 377 OrigA.getLinkage(), OrigA.getName(), &Dst); 378 NewA->copyAttributesFrom(&OrigA); 379 VMap[&OrigA] = NewA; 380 return NewA; 381 } 382 383 Error addFunctionPointerRelocationsToCurrentSymbol(jitlink::Symbol &Sym, 384 jitlink::LinkGraph &G, 385 MCDisassembler &Disassembler, 386 MCInstrAnalysis &MIA) { 387 // AArch64 appears to already come with the necessary relocations. Among other 388 // architectures, only x86_64 is currently implemented here. 389 if (G.getTargetTriple().getArch() != Triple::x86_64) 390 return Error::success(); 391 392 raw_null_ostream CommentStream; 393 auto &STI = Disassembler.getSubtargetInfo(); 394 395 // Determine the function bounds 396 auto &B = Sym.getBlock(); 397 assert(!B.isZeroFill() && "expected content block"); 398 auto SymAddress = Sym.getAddress(); 399 auto SymStartInBlock = 400 (const uint8_t *)B.getContent().data() + Sym.getOffset(); 401 auto SymSize = Sym.getSize() ? Sym.getSize() : B.getSize() - Sym.getOffset(); 402 auto Content = ArrayRef(SymStartInBlock, SymSize); 403 404 LLVM_DEBUG(dbgs() << "Adding self-relocations to " << Sym.getName() << "\n"); 405 406 SmallDenseSet<uintptr_t, 8> ExistingRelocations; 407 for (auto &E : B.edges()) { 408 if (E.isRelocation()) 409 ExistingRelocations.insert(E.getOffset()); 410 } 411 412 size_t I = 0; 413 while (I < Content.size()) { 414 MCInst Instr; 415 uint64_t InstrSize = 0; 416 uint64_t InstrStart = SymAddress.getValue() + I; 417 auto DecodeStatus = Disassembler.getInstruction( 418 Instr, InstrSize, Content.drop_front(I), InstrStart, CommentStream); 419 if (DecodeStatus != MCDisassembler::Success) { 420 LLVM_DEBUG(dbgs() << "Aborting due to disassembly failure at address " 421 << InstrStart); 422 return make_error<StringError>( 423 formatv("failed to disassemble at address {0:x16}", InstrStart), 424 inconvertibleErrorCode()); 425 } 426 // Advance to the next instruction. 427 I += InstrSize; 428 429 // Check for a PC-relative address equal to the symbol itself. 430 auto PCRelAddr = 431 MIA.evaluateMemoryOperandAddress(Instr, &STI, InstrStart, InstrSize); 432 if (!PCRelAddr || *PCRelAddr != SymAddress.getValue()) 433 continue; 434 435 auto RelocOffInInstr = 436 MIA.getMemoryOperandRelocationOffset(Instr, InstrSize); 437 if (!RelocOffInInstr || InstrSize - *RelocOffInInstr != 4) { 438 LLVM_DEBUG(dbgs() << "Skipping unknown self-relocation at " 439 << InstrStart); 440 continue; 441 } 442 443 auto RelocOffInBlock = orc::ExecutorAddr(InstrStart) + *RelocOffInInstr - 444 SymAddress + Sym.getOffset(); 445 if (ExistingRelocations.contains(RelocOffInBlock)) 446 continue; 447 448 LLVM_DEBUG(dbgs() << "Adding delta32 self-relocation at " << InstrStart); 449 B.addEdge(jitlink::x86_64::Delta32, RelocOffInBlock, Sym, /*Addend=*/-4); 450 } 451 return Error::success(); 452 } 453 454 } // End namespace orc. 455 } // End namespace llvm. 456