Lines Matching refs:llvm
38 llvm::IntegerType *IntTy, *SizeTy;
39 llvm::Type *VoidTy;
40 llvm::PointerType *CharPtrTy, *VoidPtrTy, *VoidPtrPtrTy;
43 llvm::LLVMContext &Context;
45 llvm::Module &TheModule;
48 llvm::Function *Kernel; // stub function to help launch kernel
51 llvm::SmallVector<KernelInfo, 16> EmittedKernels;
55 llvm::DenseMap<llvm::Function *, llvm::GlobalValue *> KernelHandles;
57 llvm::DenseMap<llvm::GlobalValue *, llvm::Function *> KernelStubs;
59 llvm::GlobalVariable *Var;
63 llvm::SmallVector<VarInfo, 16> DeviceVars;
67 llvm::GlobalVariable *GpuBinaryHandle = nullptr;
73 llvm::FunctionCallee getSetupArgumentFn() const;
74 llvm::FunctionCallee getLaunchFn() const;
76 llvm::FunctionType *getRegisterGlobalsFnTy() const;
77 llvm::FunctionType *getCallbackFnTy() const;
78 llvm::FunctionType *getRegisterLinkedBinaryFnTy() const;
83 llvm::Function *makeRegisterGlobalsFn();
88 llvm::Constant *makeConstantString(const std::string &Str, in makeConstantString()
92 llvm::Constant *Zeros[] = {llvm::ConstantInt::get(SizeTy, 0), in makeConstantString()
93 llvm::ConstantInt::get(SizeTy, 0)}; in makeConstantString()
95 llvm::GlobalVariable *GV = in makeConstantString()
96 cast<llvm::GlobalVariable>(ConstStr.getPointer()); in makeConstantString()
101 GV->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::None); in makeConstantString()
104 GV->setAlignment(llvm::Align(Alignment)); in makeConstantString()
106 return llvm::ConstantExpr::getGetElementPtr(ConstStr.getElementType(), in makeConstantString()
111 llvm::Function *makeDummyFunction(llvm::FunctionType *FnTy) { in makeDummyFunction()
114 llvm::Function *DummyFunc = llvm::Function::Create( in makeDummyFunction()
115 FnTy, llvm::GlobalValue::InternalLinkage, "dummy", &TheModule); in makeDummyFunction()
117 llvm::BasicBlock *DummyBlock = in makeDummyFunction()
118 llvm::BasicBlock::Create(Context, "", DummyFunc); in makeDummyFunction()
130 void registerDeviceVar(const VarDecl *VD, llvm::GlobalVariable &Var, in registerDeviceVar()
138 void registerDeviceSurf(const VarDecl *VD, llvm::GlobalVariable &Var, in registerDeviceSurf()
146 void registerDeviceTex(const VarDecl *VD, llvm::GlobalVariable &Var, in registerDeviceTex()
155 llvm::Function *makeModuleCtorFunction();
157 llvm::Function *makeModuleDtorFunction();
164 llvm::GlobalValue *getKernelHandle(llvm::Function *F, GlobalDecl GD) override;
165 llvm::Function *getKernelStub(llvm::GlobalValue *Handle) override { in getKernelStub()
172 llvm::GlobalVariable &Var) override;
175 llvm::GlobalValue::LinkageTypes &Linkage) override;
177 llvm::Function *finalizeModule() override;
207 CharPtrTy = llvm::PointerType::getUnqual(Types.ConvertType(Ctx.CharTy)); in CGNVCUDARuntime()
208 VoidPtrTy = cast<llvm::PointerType>(Types.ConvertType(Ctx.VoidPtrTy)); in CGNVCUDARuntime()
220 llvm::FunctionCallee CGNVCUDARuntime::getSetupArgumentFn() const { in getSetupArgumentFn()
222 llvm::Type *Params[] = {VoidPtrTy, SizeTy, SizeTy}; in getSetupArgumentFn()
224 llvm::FunctionType::get(IntTy, Params, false), in getSetupArgumentFn()
228 llvm::FunctionCallee CGNVCUDARuntime::getLaunchFn() const { in getLaunchFn()
232 llvm::FunctionType::get(IntTy, CharPtrTy, false), "hipLaunchByPtr"); in getLaunchFn()
236 llvm::FunctionType::get(IntTy, CharPtrTy, false), "cudaLaunch"); in getLaunchFn()
240 llvm::FunctionType *CGNVCUDARuntime::getRegisterGlobalsFnTy() const { in getRegisterGlobalsFnTy()
241 return llvm::FunctionType::get(VoidTy, VoidPtrPtrTy, false); in getRegisterGlobalsFnTy()
244 llvm::FunctionType *CGNVCUDARuntime::getCallbackFnTy() const { in getCallbackFnTy()
245 return llvm::FunctionType::get(VoidTy, VoidPtrTy, false); in getCallbackFnTy()
248 llvm::FunctionType *CGNVCUDARuntime::getRegisterLinkedBinaryFnTy() const { in getRegisterLinkedBinaryFnTy()
251 llvm::Type *Params[] = {RegisterGlobalsFnTy->getPointerTo(), VoidPtrTy, in getRegisterLinkedBinaryFnTy()
253 return llvm::FunctionType::get(VoidTy, Params, false); in getRegisterLinkedBinaryFnTy()
271 llvm::raw_svector_ostream Out(Buffer); in getDeviceSideName()
282 llvm::raw_svector_ostream Out(Buffer); in getDeviceSideName()
293 if (auto *GV = dyn_cast<llvm::GlobalVariable>(KernelHandles[CGF.CurFn])) { in emitDeviceStub()
316 llvm::ConstantInt::get(SizeTy, std::max<size_t>(1, Args.size()))); in emitDeviceStubBodyNew()
319 llvm::Value* VarPtr = CGF.GetAddrOfLocalVar(Args[i]).getPointer(); in emitDeviceStubBodyNew()
320 llvm::Value *VoidVarPtr = CGF.Builder.CreatePointerCast(VarPtr, VoidPtrTy); in emitDeviceStubBodyNew()
325 llvm::BasicBlock *EndBlock = CGF.createBasicBlock("setup.end"); in emitDeviceStubBodyNew()
361 llvm::FunctionCallee cudaPopConfigFn = CGM.CreateRuntimeFunction( in emitDeviceStubBodyNew()
362 llvm::FunctionType::get(IntTy, in emitDeviceStubBodyNew()
375 llvm::Value *Kernel = in emitDeviceStubBodyNew()
391 llvm::Type *Ty = CGM.getTypes().ConvertType(CQT); in emitDeviceStubBodyNew()
392 llvm::FunctionType *FTy = dyn_cast<llvm::FunctionType>(Ty); in emitDeviceStubBodyNew()
396 llvm::FunctionCallee cudaLaunchKernelFn = in emitDeviceStubBodyNew()
408 llvm::FunctionCallee cudaSetupArgFn = getSetupArgumentFn(); in emitDeviceStubBodyLegacy()
409 llvm::BasicBlock *EndBlock = CGF.createBasicBlock("setup.end"); in emitDeviceStubBodyLegacy()
414 llvm::Value *Args[] = { in emitDeviceStubBodyLegacy()
417 llvm::ConstantInt::get(SizeTy, TInfo.Width.getQuantity()), in emitDeviceStubBodyLegacy()
418 llvm::ConstantInt::get(SizeTy, Offset.getQuantity()), in emitDeviceStubBodyLegacy()
420 llvm::CallBase *CB = CGF.EmitRuntimeCallOrInvoke(cudaSetupArgFn, Args); in emitDeviceStubBodyLegacy()
421 llvm::Constant *Zero = llvm::ConstantInt::get(IntTy, 0); in emitDeviceStubBodyLegacy()
422 llvm::Value *CBZero = CGF.Builder.CreateICmpEQ(CB, Zero); in emitDeviceStubBodyLegacy()
423 llvm::BasicBlock *NextBlock = CGF.createBasicBlock("setup.next"); in emitDeviceStubBodyLegacy()
430 llvm::FunctionCallee cudaLaunchFn = getLaunchFn(); in emitDeviceStubBodyLegacy()
431 llvm::Value *Arg = in emitDeviceStubBodyLegacy()
441 static void replaceManagedVar(llvm::GlobalVariable *Var, in replaceManagedVar()
442 llvm::GlobalVariable *ManagedVar) { in replaceManagedVar()
443 SmallVector<SmallVector<llvm::User *, 8>, 8> WorkList; in replaceManagedVar()
450 if (isa<llvm::ConstantExpr>(U)) { in replaceManagedVar()
458 if (auto *I = dyn_cast<llvm::Instruction>(U)) { in replaceManagedVar()
459 llvm::Value *OldV = Var; in replaceManagedVar()
460 llvm::Instruction *NewV = in replaceManagedVar()
461 new llvm::LoadInst(Var->getType(), ManagedVar, "ld.managed", false, in replaceManagedVar()
462 llvm::Align(Var->getAlignment()), I); in replaceManagedVar()
467 auto *CE = cast<llvm::ConstantExpr>(Op); in replaceManagedVar()
468 auto *NewInst = llvm::createReplacementInstr(CE, I); in replaceManagedVar()
494 llvm::Function *CGNVCUDARuntime::makeRegisterGlobalsFn() { in makeRegisterGlobalsFn()
499 llvm::Function *RegisterKernelsFunc = llvm::Function::Create( in makeRegisterGlobalsFn()
500 getRegisterGlobalsFnTy(), llvm::GlobalValue::InternalLinkage, in makeRegisterGlobalsFn()
502 llvm::BasicBlock *EntryBB = in makeRegisterGlobalsFn()
503 llvm::BasicBlock::Create(Context, "entry", RegisterKernelsFunc); in makeRegisterGlobalsFn()
509 llvm::Type *RegisterFuncParams[] = { in makeRegisterGlobalsFn()
512 llvm::FunctionCallee RegisterFunc = CGM.CreateRuntimeFunction( in makeRegisterGlobalsFn()
513 llvm::FunctionType::get(IntTy, RegisterFuncParams, false), in makeRegisterGlobalsFn()
519 llvm::Argument &GpuBinaryHandlePtr = *RegisterKernelsFunc->arg_begin(); in makeRegisterGlobalsFn()
521 llvm::Constant *KernelName = in makeRegisterGlobalsFn()
523 llvm::Constant *NullPtr = llvm::ConstantPointerNull::get(VoidPtrTy); in makeRegisterGlobalsFn()
524 llvm::Value *Args[] = { in makeRegisterGlobalsFn()
529 llvm::ConstantInt::get(IntTy, -1), in makeRegisterGlobalsFn()
534 llvm::ConstantPointerNull::get(IntTy->getPointerTo())}; in makeRegisterGlobalsFn()
538 llvm::Type *VarSizeTy = IntTy; in makeRegisterGlobalsFn()
546 llvm::Type *RegisterVarParams[] = {VoidPtrPtrTy, CharPtrTy, CharPtrTy, in makeRegisterGlobalsFn()
549 llvm::FunctionCallee RegisterVar = CGM.CreateRuntimeFunction( in makeRegisterGlobalsFn()
550 llvm::FunctionType::get(VoidTy, RegisterVarParams, false), in makeRegisterGlobalsFn()
554 llvm::Type *RegisterManagedVarParams[] = {VoidPtrPtrTy, CharPtrTy, CharPtrTy, in makeRegisterGlobalsFn()
556 llvm::FunctionCallee RegisterManagedVar = CGM.CreateRuntimeFunction( in makeRegisterGlobalsFn()
557 llvm::FunctionType::get(VoidTy, RegisterManagedVarParams, false), in makeRegisterGlobalsFn()
561 llvm::FunctionCallee RegisterSurf = CGM.CreateRuntimeFunction( in makeRegisterGlobalsFn()
562 llvm::FunctionType::get( in makeRegisterGlobalsFn()
568 llvm::FunctionCallee RegisterTex = CGM.CreateRuntimeFunction( in makeRegisterGlobalsFn()
569 llvm::FunctionType::get( in makeRegisterGlobalsFn()
575 llvm::GlobalVariable *Var = Info.Var; in makeRegisterGlobalsFn()
579 llvm::Constant *VarName = makeConstantString(getDeviceSideName(Info.D)); in makeRegisterGlobalsFn()
585 auto ManagedVar = new llvm::GlobalVariable( in makeRegisterGlobalsFn()
590 : llvm::ConstantPointerNull::get(Var->getType()), in makeRegisterGlobalsFn()
592 llvm::GlobalVariable::NotThreadLocal); in makeRegisterGlobalsFn()
599 llvm::Value *Args[] = { in makeRegisterGlobalsFn()
604 llvm::ConstantInt::get(VarSizeTy, VarSize), in makeRegisterGlobalsFn()
605 llvm::ConstantInt::get(IntTy, Var->getAlignment())}; in makeRegisterGlobalsFn()
609 llvm::Value *Args[] = { in makeRegisterGlobalsFn()
614 llvm::ConstantInt::get(IntTy, Info.Flags.isExtern()), in makeRegisterGlobalsFn()
615 llvm::ConstantInt::get(VarSizeTy, VarSize), in makeRegisterGlobalsFn()
616 llvm::ConstantInt::get(IntTy, Info.Flags.isConstant()), in makeRegisterGlobalsFn()
617 llvm::ConstantInt::get(IntTy, 0)}; in makeRegisterGlobalsFn()
626 VarName, llvm::ConstantInt::get(IntTy, Info.Flags.getSurfTexType()), in makeRegisterGlobalsFn()
627 llvm::ConstantInt::get(IntTy, Info.Flags.isExtern())}); in makeRegisterGlobalsFn()
633 VarName, llvm::ConstantInt::get(IntTy, Info.Flags.getSurfTexType()), in makeRegisterGlobalsFn()
634 llvm::ConstantInt::get(IntTy, Info.Flags.isNormalized()), in makeRegisterGlobalsFn()
635 llvm::ConstantInt::get(IntTy, Info.Flags.isExtern())}); in makeRegisterGlobalsFn()
663 llvm::Function *CGNVCUDARuntime::makeModuleCtorFunction() { in makeModuleCtorFunction()
675 llvm::Function *RegisterGlobalsFunc = makeRegisterGlobalsFn(); in makeModuleCtorFunction()
682 llvm::FunctionCallee RegisterFatbinFunc = CGM.CreateRuntimeFunction( in makeModuleCtorFunction()
683 llvm::FunctionType::get(VoidPtrPtrTy, VoidPtrTy, false), in makeModuleCtorFunction()
686 llvm::StructType *FatbinWrapperTy = in makeModuleCtorFunction()
687 llvm::StructType::get(IntTy, IntTy, VoidPtrTy, VoidPtrTy); in makeModuleCtorFunction()
693 std::unique_ptr<llvm::MemoryBuffer> CudaGpuBinary = nullptr; in makeModuleCtorFunction()
695 llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> CudaGpuBinaryOrErr = in makeModuleCtorFunction()
696 llvm::MemoryBuffer::getFileOrSTDIN(CudaGpuBinaryFileName); in makeModuleCtorFunction()
705 llvm::Function *ModuleCtorFunc = llvm::Function::Create( in makeModuleCtorFunction()
706 llvm::FunctionType::get(VoidTy, VoidPtrTy, false), in makeModuleCtorFunction()
707 llvm::GlobalValue::InternalLinkage, in makeModuleCtorFunction()
709 llvm::BasicBlock *CtorEntryBB = in makeModuleCtorFunction()
710 llvm::BasicBlock::Create(Context, "entry", ModuleCtorFunc); in makeModuleCtorFunction()
719 llvm::Constant *FatBinStr; in makeModuleCtorFunction()
740 FatBinStr = new llvm::GlobalVariable( in makeModuleCtorFunction()
742 /*isConstant=*/true, llvm::GlobalValue::ExternalLinkage, nullptr, in makeModuleCtorFunction()
744 llvm::GlobalVariable::NotThreadLocal); in makeModuleCtorFunction()
745 cast<llvm::GlobalVariable>(FatBinStr)->setSection(FatbinConstantName); in makeModuleCtorFunction()
783 Values.add(llvm::ConstantPointerNull::get(VoidPtrTy)); in makeModuleCtorFunction()
784 llvm::GlobalVariable *FatbinWrapper = Values.finishAndCreateGlobal( in makeModuleCtorFunction()
797 auto Linkage = CudaGpuBinary ? llvm::GlobalValue::InternalLinkage : in makeModuleCtorFunction()
798 llvm::GlobalValue::LinkOnceAnyLinkage; in makeModuleCtorFunction()
799 llvm::BasicBlock *IfBlock = in makeModuleCtorFunction()
800 llvm::BasicBlock::Create(Context, "if", ModuleCtorFunc); in makeModuleCtorFunction()
801 llvm::BasicBlock *ExitBlock = in makeModuleCtorFunction()
802 llvm::BasicBlock::Create(Context, "exit", ModuleCtorFunc); in makeModuleCtorFunction()
805 GpuBinaryHandle = new llvm::GlobalVariable( in makeModuleCtorFunction()
808 /*Initializer=*/llvm::ConstantPointerNull::get(VoidPtrPtrTy), in makeModuleCtorFunction()
812 if (Linkage != llvm::GlobalValue::InternalLinkage) in makeModuleCtorFunction()
813 GpuBinaryHandle->setVisibility(llvm::GlobalValue::HiddenVisibility); in makeModuleCtorFunction()
819 llvm::Constant *Zero = in makeModuleCtorFunction()
820 llvm::Constant::getNullValue(HandleValue->getType()); in makeModuleCtorFunction()
821 llvm::Value *EQZero = CtorBuilder.CreateICmpEQ(HandleValue, Zero); in makeModuleCtorFunction()
827 llvm::CallInst *RegisterFatbinCall = CtorBuilder.CreateCall( in makeModuleCtorFunction()
845 llvm::CallInst *RegisterFatbinCall = CtorBuilder.CreateCall( in makeModuleCtorFunction()
848 GpuBinaryHandle = new llvm::GlobalVariable( in makeModuleCtorFunction()
849 TheModule, VoidPtrPtrTy, false, llvm::GlobalValue::InternalLinkage, in makeModuleCtorFunction()
850 llvm::ConstantPointerNull::get(VoidPtrPtrTy), "__cuda_gpubin_handle"); in makeModuleCtorFunction()
863 llvm::FunctionCallee RegisterFatbinEndFunc = CGM.CreateRuntimeFunction( in makeModuleCtorFunction()
864 llvm::FunctionType::get(VoidTy, VoidPtrPtrTy, false), in makeModuleCtorFunction()
871 llvm::raw_svector_ostream OS(ModuleID); in makeModuleCtorFunction()
872 OS << ModuleIDPrefix << llvm::format("%" PRIx64, FatbinWrapper->getGUID()); in makeModuleCtorFunction()
873 llvm::Constant *ModuleIDConstant = makeConstantString( in makeModuleCtorFunction()
877 llvm::GlobalAlias::create(llvm::GlobalValue::ExternalLinkage, in makeModuleCtorFunction()
884 llvm::FunctionCallee RegisterLinkedBinaryFunc = CGM.CreateRuntimeFunction( in makeModuleCtorFunction()
888 llvm::Value *Args[] = {RegisterGlobalsFunc, in makeModuleCtorFunction()
898 if (llvm::Function *CleanupFn = makeModuleDtorFunction()) { in makeModuleCtorFunction()
900 llvm::FunctionType *AtExitTy = in makeModuleCtorFunction()
901 llvm::FunctionType::get(IntTy, CleanupFn->getType(), false); in makeModuleCtorFunction()
902 llvm::FunctionCallee AtExitFunc = in makeModuleCtorFunction()
903 CGM.CreateRuntimeFunction(AtExitTy, "atexit", llvm::AttributeList(), in makeModuleCtorFunction()
931 llvm::Function *CGNVCUDARuntime::makeModuleDtorFunction() { in makeModuleDtorFunction()
937 llvm::FunctionCallee UnregisterFatbinFunc = CGM.CreateRuntimeFunction( in makeModuleDtorFunction()
938 llvm::FunctionType::get(VoidTy, VoidPtrPtrTy, false), in makeModuleDtorFunction()
941 llvm::Function *ModuleDtorFunc = llvm::Function::Create( in makeModuleDtorFunction()
942 llvm::FunctionType::get(VoidTy, VoidPtrTy, false), in makeModuleDtorFunction()
943 llvm::GlobalValue::InternalLinkage, in makeModuleDtorFunction()
946 llvm::BasicBlock *DtorEntryBB = in makeModuleDtorFunction()
947 llvm::BasicBlock::Create(Context, "entry", ModuleDtorFunc); in makeModuleDtorFunction()
958 llvm::BasicBlock *IfBlock = in makeModuleDtorFunction()
959 llvm::BasicBlock::Create(Context, "if", ModuleDtorFunc); in makeModuleDtorFunction()
960 llvm::BasicBlock *ExitBlock = in makeModuleDtorFunction()
961 llvm::BasicBlock::Create(Context, "exit", ModuleDtorFunc); in makeModuleDtorFunction()
962 llvm::Constant *Zero = llvm::Constant::getNullValue(HandleValue->getType()); in makeModuleDtorFunction()
963 llvm::Value *NEZero = DtorBuilder.CreateICmpNE(HandleValue, Zero); in makeModuleDtorFunction()
984 const VarDecl *D, llvm::GlobalValue::LinkageTypes &Linkage) { in internalizeDeviceSideVar()
1004 Linkage = llvm::GlobalValue::InternalLinkage; in internalizeDeviceSideVar()
1009 llvm::GlobalVariable &GV) { in handleVarRegistration()
1063 llvm::GlobalVariable *Var = Info.Var; in transformManagedVars()
1066 auto ManagedVar = new llvm::GlobalVariable( in transformManagedVars()
1071 : llvm::ConstantPointerNull::get(Var->getType()), in transformManagedVars()
1073 llvm::GlobalVariable::NotThreadLocal, in transformManagedVars()
1093 llvm::Function *CGNVCUDARuntime::finalizeModule() { in finalizeModule()
1110 !llvm::GlobalValue::isLocalLinkage(Info.Var->getLinkage()) && in finalizeModule()
1123 llvm::GlobalValue *CGNVCUDARuntime::getKernelHandle(llvm::Function *F, in getKernelHandle()
1135 auto *Var = new llvm::GlobalVariable( in getKernelHandle()