1 //===-- Assembler.cpp -------------------------------------------*- C++ -*-===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 9 #include "Assembler.h" 10 11 #include "SnippetRepetitor.h" 12 #include "SubprocessMemory.h" 13 #include "Target.h" 14 #include "llvm/Analysis/TargetLibraryInfo.h" 15 #include "llvm/CodeGen/FunctionLoweringInfo.h" 16 #include "llvm/CodeGen/GlobalISel/CallLowering.h" 17 #include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h" 18 #include "llvm/CodeGen/MachineInstrBuilder.h" 19 #include "llvm/CodeGen/MachineModuleInfo.h" 20 #include "llvm/CodeGen/MachineRegisterInfo.h" 21 #include "llvm/CodeGen/TargetInstrInfo.h" 22 #include "llvm/CodeGen/TargetLowering.h" 23 #include "llvm/CodeGen/TargetPassConfig.h" 24 #include "llvm/CodeGen/TargetSubtargetInfo.h" 25 #include "llvm/ExecutionEngine/Orc/LLJIT.h" 26 #include "llvm/IR/BasicBlock.h" 27 #include "llvm/IR/Instructions.h" 28 #include "llvm/IR/LegacyPassManager.h" 29 #include "llvm/MC/MCInstrInfo.h" 30 #include "llvm/Object/SymbolSize.h" 31 #include "llvm/Support/Alignment.h" 32 #include "llvm/Support/MemoryBuffer.h" 33 #include "llvm/Support/raw_ostream.h" 34 35 #ifdef HAVE_LIBPFM 36 #include "perfmon/perf_event.h" 37 #endif // HAVE_LIBPFM 38 39 #ifdef __linux__ 40 #include <unistd.h> 41 #endif 42 43 namespace llvm { 44 namespace exegesis { 45 46 static constexpr const char ModuleID[] = "ExegesisInfoTest"; 47 static constexpr const char FunctionID[] = "foo"; 48 static const Align kFunctionAlignment(4096); 49 50 // Fills the given basic block with register setup code, and returns true if 51 // all registers could be setup correctly. 52 static bool generateSnippetSetupCode(const ExegesisTarget &ET, 53 const MCSubtargetInfo *const MSI, 54 BasicBlockFiller &BBF, 55 const BenchmarkKey &Key, 56 bool GenerateMemoryInstructions) { 57 bool IsSnippetSetupComplete = true; 58 if (GenerateMemoryInstructions) { 59 BBF.addInstructions(ET.generateMemoryInitialSetup()); 60 for (const MemoryMapping &MM : Key.MemoryMappings) { 61 #ifdef __linux__ 62 // The frontend that generates that parses the memory mapping information 63 // from the user should validate that the requested address is a multiple 64 // of the page size. Assert that this is true here. 65 assert(MM.Address % getpagesize() == 0 && 66 "Memory mappings need to be aligned to page boundaries."); 67 #endif 68 BBF.addInstructions(ET.generateMmap( 69 MM.Address, Key.MemoryValues.at(MM.MemoryValueName).SizeBytes, 70 ET.getAuxiliaryMemoryStartAddress() + 71 sizeof(int) * (Key.MemoryValues.at(MM.MemoryValueName).Index + 72 SubprocessMemory::AuxiliaryMemoryOffset))); 73 } 74 BBF.addInstructions(ET.setStackRegisterToAuxMem()); 75 } 76 Register StackPointerRegister = BBF.MF.getSubtarget() 77 .getTargetLowering() 78 ->getStackPointerRegisterToSaveRestore(); 79 for (const RegisterValue &RV : Key.RegisterInitialValues) { 80 if (GenerateMemoryInstructions) { 81 // If we're generating memory instructions, don't load in the value for 82 // the register with the stack pointer as it will be used later to finish 83 // the setup. 84 if (Register(RV.Register) == StackPointerRegister) 85 continue; 86 } 87 // Load a constant in the register. 88 const auto SetRegisterCode = ET.setRegTo(*MSI, RV.Register, RV.Value); 89 if (SetRegisterCode.empty()) 90 IsSnippetSetupComplete = false; 91 BBF.addInstructions(SetRegisterCode); 92 } 93 if (GenerateMemoryInstructions) { 94 #ifdef HAVE_LIBPFM 95 BBF.addInstructions(ET.configurePerfCounter(PERF_EVENT_IOC_RESET, true)); 96 #endif // HAVE_LIBPFM 97 for (const RegisterValue &RV : Key.RegisterInitialValues) { 98 // Load in the stack register now as we're done using it elsewhere 99 // and need to set the value in preparation for executing the 100 // snippet. 101 if (Register(RV.Register) != StackPointerRegister) 102 continue; 103 const auto SetRegisterCode = ET.setRegTo(*MSI, RV.Register, RV.Value); 104 if (SetRegisterCode.empty()) 105 IsSnippetSetupComplete = false; 106 BBF.addInstructions(SetRegisterCode); 107 break; 108 } 109 } 110 return IsSnippetSetupComplete; 111 } 112 113 // Small utility function to add named passes. 114 static bool addPass(PassManagerBase &PM, StringRef PassName, 115 TargetPassConfig &TPC) { 116 const PassRegistry *PR = PassRegistry::getPassRegistry(); 117 const PassInfo *PI = PR->getPassInfo(PassName); 118 if (!PI) { 119 errs() << " run-pass " << PassName << " is not registered.\n"; 120 return true; 121 } 122 123 if (!PI->getNormalCtor()) { 124 errs() << " cannot create pass: " << PI->getPassName() << "\n"; 125 return true; 126 } 127 Pass *P = PI->getNormalCtor()(); 128 std::string Banner = std::string("After ") + std::string(P->getPassName()); 129 PM.add(P); 130 TPC.printAndVerify(Banner); 131 132 return false; 133 } 134 135 MachineFunction &createVoidVoidPtrMachineFunction(StringRef FunctionName, 136 Module *Module, 137 MachineModuleInfo *MMI) { 138 Type *const ReturnType = Type::getInt32Ty(Module->getContext()); 139 Type *const MemParamType = 140 PointerType::get(Module->getContext(), 0 /*default address space*/); 141 FunctionType *FunctionType = 142 FunctionType::get(ReturnType, {MemParamType}, false); 143 Function *const F = Function::Create( 144 FunctionType, GlobalValue::ExternalLinkage, FunctionName, Module); 145 BasicBlock *BB = BasicBlock::Create(Module->getContext(), "", F); 146 new UnreachableInst(Module->getContext(), BB); 147 return MMI->getOrCreateMachineFunction(*F); 148 } 149 150 BasicBlockFiller::BasicBlockFiller(MachineFunction &MF, MachineBasicBlock *MBB, 151 const MCInstrInfo *MCII) 152 : MF(MF), MBB(MBB), MCII(MCII) {} 153 154 void BasicBlockFiller::addInstruction(const MCInst &Inst, const DebugLoc &DL) { 155 const unsigned Opcode = Inst.getOpcode(); 156 const MCInstrDesc &MCID = MCII->get(Opcode); 157 MachineInstrBuilder Builder = BuildMI(MBB, DL, MCID); 158 for (unsigned OpIndex = 0, E = Inst.getNumOperands(); OpIndex < E; 159 ++OpIndex) { 160 const MCOperand &Op = Inst.getOperand(OpIndex); 161 if (Op.isReg()) { 162 const bool IsDef = OpIndex < MCID.getNumDefs(); 163 unsigned Flags = 0; 164 const MCOperandInfo &OpInfo = MCID.operands().begin()[OpIndex]; 165 if (IsDef && !OpInfo.isOptionalDef()) 166 Flags |= RegState::Define; 167 Builder.addReg(Op.getReg(), Flags); 168 } else if (Op.isImm()) { 169 Builder.addImm(Op.getImm()); 170 } else if (!Op.isValid()) { 171 llvm_unreachable("Operand is not set"); 172 } else { 173 llvm_unreachable("Not yet implemented"); 174 } 175 } 176 } 177 178 void BasicBlockFiller::addInstructions(ArrayRef<MCInst> Insts, 179 const DebugLoc &DL) { 180 for (const MCInst &Inst : Insts) 181 addInstruction(Inst, DL); 182 } 183 184 void BasicBlockFiller::addReturn(const ExegesisTarget &ET, 185 bool SubprocessCleanup, const DebugLoc &DL) { 186 // Insert cleanup code 187 if (SubprocessCleanup) { 188 #ifdef HAVE_LIBPFM 189 addInstructions(ET.configurePerfCounter(PERF_EVENT_IOC_DISABLE, false)); 190 #endif // HAVE_LIBPFM 191 #ifdef __linux__ 192 addInstructions(ET.generateExitSyscall(0)); 193 #endif // __linux__ 194 } 195 // Insert the return code. 196 const TargetInstrInfo *TII = MF.getSubtarget().getInstrInfo(); 197 if (TII->getReturnOpcode() < TII->getNumOpcodes()) { 198 BuildMI(MBB, DL, TII->get(TII->getReturnOpcode())); 199 } else { 200 MachineIRBuilder MIB(MF); 201 MIB.setMBB(*MBB); 202 203 FunctionLoweringInfo FuncInfo; 204 FuncInfo.CanLowerReturn = true; 205 MF.getSubtarget().getCallLowering()->lowerReturn(MIB, nullptr, {}, FuncInfo, 206 0); 207 } 208 } 209 210 FunctionFiller::FunctionFiller(MachineFunction &MF, 211 std::vector<MCRegister> RegistersSetUp) 212 : MF(MF), MCII(MF.getTarget().getMCInstrInfo()), Entry(addBasicBlock()), 213 RegistersSetUp(std::move(RegistersSetUp)) {} 214 215 BasicBlockFiller FunctionFiller::addBasicBlock() { 216 MachineBasicBlock *MBB = MF.CreateMachineBasicBlock(); 217 MF.push_back(MBB); 218 return BasicBlockFiller(MF, MBB, MCII); 219 } 220 221 ArrayRef<MCRegister> FunctionFiller::getRegistersSetUp() const { 222 return RegistersSetUp; 223 } 224 225 static std::unique_ptr<Module> 226 createModule(const std::unique_ptr<LLVMContext> &Context, const DataLayout &DL) { 227 auto Mod = std::make_unique<Module>(ModuleID, *Context); 228 Mod->setDataLayout(DL); 229 return Mod; 230 } 231 232 BitVector getFunctionReservedRegs(const TargetMachine &TM) { 233 std::unique_ptr<LLVMContext> Context = std::make_unique<LLVMContext>(); 234 std::unique_ptr<Module> Module = createModule(Context, TM.createDataLayout()); 235 auto MMIWP = std::make_unique<MachineModuleInfoWrapperPass>(&TM); 236 MachineFunction &MF = createVoidVoidPtrMachineFunction( 237 FunctionID, Module.get(), &MMIWP->getMMI()); 238 // Saving reserved registers for client. 239 return MF.getSubtarget().getRegisterInfo()->getReservedRegs(MF); 240 } 241 242 Error assembleToStream(const ExegesisTarget &ET, 243 std::unique_ptr<TargetMachine> TM, 244 ArrayRef<MCRegister> LiveIns, const FillFunction &Fill, 245 raw_pwrite_stream &AsmStream, const BenchmarkKey &Key, 246 bool GenerateMemoryInstructions) { 247 auto Context = std::make_unique<LLVMContext>(); 248 std::unique_ptr<Module> Module = 249 createModule(Context, TM->createDataLayout()); 250 auto MMIWP = std::make_unique<MachineModuleInfoWrapperPass>(TM.get()); 251 MachineFunction &MF = createVoidVoidPtrMachineFunction( 252 FunctionID, Module.get(), &MMIWP.get()->getMMI()); 253 MF.ensureAlignment(kFunctionAlignment); 254 255 // We need to instruct the passes that we're done with SSA and virtual 256 // registers. 257 auto &Properties = MF.getProperties(); 258 Properties.set(MachineFunctionProperties::Property::NoVRegs); 259 Properties.reset(MachineFunctionProperties::Property::IsSSA); 260 Properties.set(MachineFunctionProperties::Property::NoPHIs); 261 262 for (const MCRegister Reg : LiveIns) 263 MF.getRegInfo().addLiveIn(Reg); 264 265 if (GenerateMemoryInstructions) { 266 for (const MCRegister Reg : ET.getArgumentRegisters()) 267 MF.getRegInfo().addLiveIn(Reg); 268 // Add a live in for registers that need saving so that the machine verifier 269 // doesn't fail if the register is never defined. 270 for (const MCRegister Reg : ET.getRegistersNeedSaving()) 271 MF.getRegInfo().addLiveIn(Reg); 272 } 273 274 std::vector<MCRegister> RegistersSetUp; 275 RegistersSetUp.reserve(Key.RegisterInitialValues.size()); 276 for (const auto &InitValue : Key.RegisterInitialValues) { 277 RegistersSetUp.push_back(InitValue.Register); 278 } 279 FunctionFiller Sink(MF, std::move(RegistersSetUp)); 280 auto Entry = Sink.getEntry(); 281 282 for (const MCRegister Reg : LiveIns) 283 Entry.MBB->addLiveIn(Reg); 284 285 if (GenerateMemoryInstructions) { 286 for (const MCRegister Reg : ET.getArgumentRegisters()) 287 Entry.MBB->addLiveIn(Reg); 288 // Add a live in for registers that need saving so that the machine verifier 289 // doesn't fail if the register is never defined. 290 for (const MCRegister Reg : ET.getRegistersNeedSaving()) 291 Entry.MBB->addLiveIn(Reg); 292 } 293 294 const bool IsSnippetSetupComplete = generateSnippetSetupCode( 295 ET, TM->getMCSubtargetInfo(), Entry, Key, GenerateMemoryInstructions); 296 297 // If the snippet setup is not complete, we disable liveliness tracking. This 298 // means that we won't know what values are in the registers. 299 // FIXME: this should probably be an assertion. 300 if (!IsSnippetSetupComplete) 301 Properties.reset(MachineFunctionProperties::Property::TracksLiveness); 302 303 Fill(Sink); 304 305 // prologue/epilogue pass needs the reserved registers to be frozen, this 306 // is usually done by the SelectionDAGISel pass. 307 MF.getRegInfo().freezeReservedRegs(); 308 309 // We create the pass manager, run the passes to populate AsmBuffer. 310 MCContext &MCContext = MMIWP->getMMI().getContext(); 311 legacy::PassManager PM; 312 313 TargetLibraryInfoImpl TLII(Triple(Module->getTargetTriple())); 314 PM.add(new TargetLibraryInfoWrapperPass(TLII)); 315 316 TargetPassConfig *TPC = TM->createPassConfig(PM); 317 PM.add(TPC); 318 PM.add(MMIWP.release()); 319 TPC->printAndVerify("MachineFunctionGenerator::assemble"); 320 // Add target-specific passes. 321 ET.addTargetSpecificPasses(PM); 322 TPC->printAndVerify("After ExegesisTarget::addTargetSpecificPasses"); 323 // Adding the following passes: 324 // - postrapseudos: expands pseudo return instructions used on some targets. 325 // - machineverifier: checks that the MachineFunction is well formed. 326 // - prologepilog: saves and restore callee saved registers. 327 for (const char *PassName : 328 {"postrapseudos", "machineverifier", "prologepilog"}) 329 if (addPass(PM, PassName, *TPC)) 330 return make_error<Failure>("Unable to add a mandatory pass"); 331 TPC->setInitialized(); 332 333 // AsmPrinter is responsible for generating the assembly into AsmBuffer. 334 if (TM->addAsmPrinter(PM, AsmStream, nullptr, CodeGenFileType::ObjectFile, 335 MCContext)) 336 return make_error<Failure>("Cannot add AsmPrinter passes"); 337 338 PM.run(*Module); // Run all the passes 339 return Error::success(); 340 } 341 342 object::OwningBinary<object::ObjectFile> 343 getObjectFromBuffer(StringRef InputData) { 344 // Storing the generated assembly into a MemoryBuffer that owns the memory. 345 std::unique_ptr<MemoryBuffer> Buffer = 346 MemoryBuffer::getMemBufferCopy(InputData); 347 // Create the ObjectFile from the MemoryBuffer. 348 std::unique_ptr<object::ObjectFile> Obj = 349 cantFail(object::ObjectFile::createObjectFile(Buffer->getMemBufferRef())); 350 // Returning both the MemoryBuffer and the ObjectFile. 351 return object::OwningBinary<object::ObjectFile>(std::move(Obj), 352 std::move(Buffer)); 353 } 354 355 object::OwningBinary<object::ObjectFile> getObjectFromFile(StringRef Filename) { 356 return cantFail(object::ObjectFile::createObjectFile(Filename)); 357 } 358 359 Expected<ExecutableFunction> ExecutableFunction::create( 360 std::unique_ptr<TargetMachine> TM, 361 object::OwningBinary<object::ObjectFile> &&ObjectFileHolder) { 362 assert(ObjectFileHolder.getBinary() && "cannot create object file"); 363 std::unique_ptr<LLVMContext> Ctx = std::make_unique<LLVMContext>(); 364 365 auto SymbolSizes = object::computeSymbolSizes(*ObjectFileHolder.getBinary()); 366 // Get the size of the function that we want to call into (with the name of 367 // FunctionID). 368 auto SymbolIt = find_if(SymbolSizes, [&](const auto &Pair) { 369 auto SymbolName = Pair.first.getName(); 370 if (SymbolName) 371 return *SymbolName == FunctionID; 372 // We should always succeed in finding the FunctionID, hence we suppress 373 // the error here and assert later on the search result, rather than 374 // propagating the Expected<> error back to the caller. 375 consumeError(SymbolName.takeError()); 376 return false; 377 }); 378 assert(SymbolIt != SymbolSizes.end() && 379 "Cannot find the symbol for FunctionID"); 380 uintptr_t CodeSize = SymbolIt->second; 381 382 auto EJITOrErr = orc::LLJITBuilder().create(); 383 if (!EJITOrErr) 384 return EJITOrErr.takeError(); 385 386 auto EJIT = std::move(*EJITOrErr); 387 388 if (auto ObjErr = 389 EJIT->addObjectFile(std::get<1>(ObjectFileHolder.takeBinary()))) 390 return std::move(ObjErr); 391 392 auto FunctionAddressOrErr = EJIT->lookup(FunctionID); 393 if (!FunctionAddressOrErr) 394 return FunctionAddressOrErr.takeError(); 395 396 const uint64_t FunctionAddress = FunctionAddressOrErr->getValue(); 397 398 assert(isAligned(kFunctionAlignment, FunctionAddress) && 399 "function is not properly aligned"); 400 401 StringRef FBytes = 402 StringRef(reinterpret_cast<const char *>(FunctionAddress), CodeSize); 403 return ExecutableFunction(std::move(Ctx), std::move(EJIT), FBytes); 404 } 405 406 ExecutableFunction::ExecutableFunction(std::unique_ptr<LLVMContext> Ctx, 407 std::unique_ptr<orc::LLJIT> EJIT, 408 StringRef FB) 409 : FunctionBytes(FB), Context(std::move(Ctx)), ExecJIT(std::move(EJIT)) {} 410 411 Error getBenchmarkFunctionBytes(const StringRef InputData, 412 std::vector<uint8_t> &Bytes) { 413 const auto Holder = getObjectFromBuffer(InputData); 414 const auto *Obj = Holder.getBinary(); 415 // See RuntimeDyldImpl::loadObjectImpl(Obj) for much more complete 416 // implementation. 417 418 // Find the only function in the object file. 419 SmallVector<object::SymbolRef, 1> Functions; 420 for (auto &Sym : Obj->symbols()) { 421 auto SymType = Sym.getType(); 422 if (SymType && *SymType == object::SymbolRef::Type::ST_Function) 423 Functions.push_back(Sym); 424 } 425 if (Functions.size() != 1) 426 return make_error<Failure>("Exactly one function expected"); 427 428 // Find the containing section - it is assumed to contain only this function. 429 auto SectionOrErr = Functions.front().getSection(); 430 if (!SectionOrErr || *SectionOrErr == Obj->section_end()) 431 return make_error<Failure>("Section not found"); 432 433 auto Address = Functions.front().getAddress(); 434 if (!Address || *Address != SectionOrErr.get()->getAddress()) 435 return make_error<Failure>("Unexpected layout"); 436 437 auto ContentsOrErr = SectionOrErr.get()->getContents(); 438 if (!ContentsOrErr) 439 return ContentsOrErr.takeError(); 440 Bytes.assign(ContentsOrErr->begin(), ContentsOrErr->end()); 441 return Error::success(); 442 } 443 444 } // namespace exegesis 445 } // namespace llvm 446