Lines Matching refs:CGF
47 void Enter(CodeGenFunction &CGF) override { in Enter() argument
48 llvm::Value *EnterRes = CGF.EmitRuntimeCall(EnterCallee, EnterArgs); in Enter()
50 llvm::Value *CallBool = CGF.Builder.CreateIsNotNull(EnterRes); in Enter()
51 auto *ThenBlock = CGF.createBasicBlock("omp_if.then"); in Enter()
52 ContBlock = CGF.createBasicBlock("omp_if.end"); in Enter()
54 CGF.Builder.CreateCondBr(CallBool, ThenBlock, ContBlock); in Enter()
55 CGF.EmitBlock(ThenBlock); in Enter()
58 void Done(CodeGenFunction &CGF) { in Done() argument
60 CGF.EmitBranch(ContBlock); in Done()
61 CGF.EmitBlock(ContBlock, true); in Done()
63 void Exit(CodeGenFunction &CGF) override { in Exit() argument
64 CGF.EmitRuntimeCall(ExitCallee, ExitArgs); in Exit()
205 CodeGenFunction &CGF; member in __anonf5421e1e0111::CheckVarsEscapingDeclContext
224 if (auto *CSI = CGF.CapturedStmtInfo) { in markAsEscaped()
249 if ((!CGF.CapturedStmtInfo || in markAsEscaped()
250 (IsForCombinedParallelRegion && CGF.CapturedStmtInfo)) && in markAsEscaped()
323 unsigned WarpSize = CGF.getTarget().getGridValue().GV_Warp_Size; in buildRecordForGlobalizedVars()
329 CGF.getContext(), EscapedDeclsForParallel, EscapedDeclsForTeams, in buildRecordForGlobalizedVars()
334 CheckVarsEscapingDeclContext(CodeGenFunction &CGF, in CheckVarsEscapingDeclContext() argument
336 : CGF(CGF), EscapedDecls(TeamsReductions.begin(), TeamsReductions.end()) { in CheckVarsEscapingDeclContext()
516 static llvm::Value *getNVPTXWarpID(CodeGenFunction &CGF) { in getNVPTXWarpID() argument
517 CGBuilderTy &Bld = CGF.Builder; in getNVPTXWarpID()
519 llvm::Log2_32(CGF.getTarget().getGridValue().GV_Warp_Size); in getNVPTXWarpID()
520 auto &RT = static_cast<CGOpenMPRuntimeGPU &>(CGF.CGM.getOpenMPRuntime()); in getNVPTXWarpID()
521 return Bld.CreateAShr(RT.getGPUThreadID(CGF), LaneIDBits, "nvptx_warp_id"); in getNVPTXWarpID()
527 static llvm::Value *getNVPTXLaneID(CodeGenFunction &CGF) { in getNVPTXLaneID() argument
528 CGBuilderTy &Bld = CGF.Builder; in getNVPTXLaneID()
530 llvm::Log2_32(CGF.getTarget().getGridValue().GV_Warp_Size); in getNVPTXLaneID()
532 auto &RT = static_cast<CGOpenMPRuntimeGPU &>(CGF.CGM.getOpenMPRuntime()); in getNVPTXLaneID()
533 return Bld.CreateAnd(RT.getGPUThreadID(CGF), Bld.getInt32(LaneIDMask), in getNVPTXLaneID()
748 void Enter(CodeGenFunction &CGF) override { in emitNonSPMDKernel() argument
750 static_cast<CGOpenMPRuntimeGPU &>(CGF.CGM.getOpenMPRuntime()); in emitNonSPMDKernel()
751 RT.emitKernelInit(CGF, EST, /* IsSPMD */ false); in emitNonSPMDKernel()
753 RT.setLocThreadIdInsertPt(CGF, /*AtCurrentPoint=*/true); in emitNonSPMDKernel()
755 void Exit(CodeGenFunction &CGF) override { in emitNonSPMDKernel() argument
757 static_cast<CGOpenMPRuntimeGPU &>(CGF.CGM.getOpenMPRuntime()); in emitNonSPMDKernel()
758 RT.clearLocThreadIdInsertPt(CGF); in emitNonSPMDKernel()
759 RT.emitKernelDeinit(CGF, EST, /* IsSPMD */ false); in emitNonSPMDKernel()
769 void CGOpenMPRuntimeGPU::emitKernelInit(CodeGenFunction &CGF, in emitKernelInit() argument
771 CGBuilderTy &Bld = CGF.Builder; in emitKernelInit()
774 emitGenericVarsProlog(CGF, EST.Loc); in emitKernelInit()
777 void CGOpenMPRuntimeGPU::emitKernelDeinit(CodeGenFunction &CGF, in emitKernelDeinit() argument
781 emitGenericVarsEpilog(CGF); in emitKernelDeinit()
783 CGBuilderTy &Bld = CGF.Builder; in emitKernelDeinit()
805 void Enter(CodeGenFunction &CGF) override { in emitSPMDKernel() argument
806 RT.emitKernelInit(CGF, EST, /* IsSPMD */ true); in emitSPMDKernel()
808 RT.setLocThreadIdInsertPt(CGF, /*AtCurrentPoint=*/true); in emitSPMDKernel()
810 void Exit(CodeGenFunction &CGF) override { in emitSPMDKernel() argument
811 RT.clearLocThreadIdInsertPt(CGF); in emitSPMDKernel()
812 RT.emitKernelDeinit(CGF, EST, /* IsSPMD */ true); in emitSPMDKernel()
887 void CGOpenMPRuntimeGPU::emitProcBindClause(CodeGenFunction &CGF, in emitProcBindClause() argument
894 CGOpenMPRuntime::emitProcBindClause(CGF, ProcBind, Loc); in emitProcBindClause()
897 void CGOpenMPRuntimeGPU::emitNumThreadsClause(CodeGenFunction &CGF, in emitNumThreadsClause() argument
903 void CGOpenMPRuntimeGPU::emitNumTeamsClause(CodeGenFunction &CGF, in emitNumTeamsClause() argument
1005 void Enter(CodeGenFunction &CGF) override { in emitTeamsOutlinedFunction() argument
1007 static_cast<CGOpenMPRuntimeGPU &>(CGF.CGM.getOpenMPRuntime()); in emitTeamsOutlinedFunction()
1009 auto I = Rt.FunctionGlobalizedDecls.try_emplace(CGF.CurFn).first; in emitTeamsOutlinedFunction()
1019 Rt.emitGenericVarsProlog(CGF, Loc); in emitTeamsOutlinedFunction()
1021 void Exit(CodeGenFunction &CGF) override { in emitTeamsOutlinedFunction() argument
1022 static_cast<CGOpenMPRuntimeGPU &>(CGF.CGM.getOpenMPRuntime()) in emitTeamsOutlinedFunction()
1023 .emitGenericVarsEpilog(CGF); in emitTeamsOutlinedFunction()
1033 void CGOpenMPRuntimeGPU::emitGenericVarsProlog(CodeGenFunction &CGF, in emitGenericVarsProlog() argument
1040 CGBuilderTy &Bld = CGF.Builder; in emitGenericVarsProlog()
1042 const auto I = FunctionGlobalizedDecls.find(CGF.CurFn); in emitGenericVarsProlog()
1055 CGF.MakeAddrLValue(CGF.GetAddrOfLocalVar(VD), VD->getType()); in emitGenericVarsProlog()
1056 ParValue = CGF.EmitLoadOfScalar(ParLVal, Loc); in emitGenericVarsProlog()
1060 llvm::Value *AllocArgs[] = {CGF.getTypeSize(VD->getType())}; in emitGenericVarsProlog()
1062 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in emitGenericVarsProlog()
1071 llvm::PointerType *VarPtrTy = CGF.ConvertTypeForMem(VarTy)->getPointerTo(); in emitGenericVarsProlog()
1074 LValue VarAddr = CGF.MakeNaturalAlignAddrLValue(CastedVoidPtr, VarTy); in emitGenericVarsProlog()
1075 Rec.second.PrivateAddr = VarAddr.getAddress(CGF); in emitGenericVarsProlog()
1080 CGF.EmitStoreOfScalar(ParValue, VarAddr); in emitGenericVarsProlog()
1081 I->getSecond().MappedParams->setVarAddr(CGF, VD, VarAddr.getAddress(CGF)); in emitGenericVarsProlog()
1083 if (auto *DI = CGF.getDebugInfo()) in emitGenericVarsProlog()
1089 llvm::Value *Size = CGF.getTypeSize(VD->getType()); in emitGenericVarsProlog()
1092 Size, llvm::ConstantInt::get(CGF.SizeTy, Align.getQuantity() - 1)); in emitGenericVarsProlog()
1094 llvm::ConstantInt::get(CGF.SizeTy, Align.getQuantity()); in emitGenericVarsProlog()
1100 llvm::Value *AllocArgs[] = {CGF.getTypeSize(VD->getType())}; in emitGenericVarsProlog()
1102 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in emitGenericVarsProlog()
1111 {VoidPtr, CGF.getTypeSize(VD->getType())})); in emitGenericVarsProlog()
1112 LValue Base = CGF.MakeAddrLValue(VoidPtr, VD->getType(), in emitGenericVarsProlog()
1115 I->getSecond().MappedParams->setVarAddr(CGF, cast<VarDecl>(VD), in emitGenericVarsProlog()
1116 Base.getAddress(CGF)); in emitGenericVarsProlog()
1118 I->getSecond().MappedParams->apply(CGF); in emitGenericVarsProlog()
1121 void CGOpenMPRuntimeGPU::emitGenericVarsEpilog(CodeGenFunction &CGF, in emitGenericVarsEpilog() argument
1127 const auto I = FunctionGlobalizedDecls.find(CGF.CurFn); in emitGenericVarsEpilog()
1132 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in emitGenericVarsEpilog()
1139 I->getSecond().MappedParams->restore(CGF); in emitGenericVarsEpilog()
1142 CGF.getTypeSize(VD->getType())}; in emitGenericVarsEpilog()
1143 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in emitGenericVarsEpilog()
1150 void CGOpenMPRuntimeGPU::emitTeamsCall(CodeGenFunction &CGF, in emitTeamsCall() argument
1155 if (!CGF.HaveInsertPoint()) in emitTeamsCall()
1158 Address ZeroAddr = CGF.CreateDefaultAlignTempAlloca(CGF.Int32Ty, in emitTeamsCall()
1160 CGF.Builder.CreateStore(CGF.Builder.getInt32(/*C*/ 0), ZeroAddr); in emitTeamsCall()
1162 OutlinedFnArgs.push_back(emitThreadIDAddress(CGF, Loc).getPointer()); in emitTeamsCall()
1165 emitOutlinedFunctionCall(CGF, Loc, OutlinedFn, OutlinedFnArgs); in emitTeamsCall()
1168 void CGOpenMPRuntimeGPU::emitParallelCall(CodeGenFunction &CGF, in emitParallelCall() argument
1174 if (!CGF.HaveInsertPoint()) in emitParallelCall()
1178 NumThreads](CodeGenFunction &CGF, in emitParallelCall()
1180 CGBuilderTy &Bld = CGF.Builder; in emitParallelCall()
1191 CodeGenFunction::OMPPrivateScope PrivateArgScope(CGF); in emitParallelCall()
1193 Address CapturedVarsAddrs = CGF.CreateDefaultAlignTempAlloca( in emitParallelCall()
1199 ASTContext &Ctx = CGF.getContext(); in emitParallelCall()
1205 PtrV = Bld.CreateIntToPtr(V, CGF.VoidPtrTy); in emitParallelCall()
1207 PtrV = Bld.CreatePointerBitCastOrAddrSpaceCast(V, CGF.VoidPtrTy); in emitParallelCall()
1208 CGF.EmitStoreOfScalar(PtrV, Dst, /*Volatile=*/false, in emitParallelCall()
1216 IfCondVal = Bld.CreateIntCast(CGF.EvaluateExprAsBool(IfCond), CGF.Int32Ty, in emitParallelCall()
1219 IfCondVal = llvm::ConstantInt::get(CGF.Int32Ty, 1); in emitParallelCall()
1222 NumThreadsVal = llvm::ConstantInt::get(CGF.Int32Ty, -1); in emitParallelCall()
1224 NumThreadsVal = Bld.CreateZExtOrTrunc(NumThreadsVal, CGF.Int32Ty), in emitParallelCall()
1227 llvm::Value *RTLoc = emitUpdateLocation(CGF, Loc); in emitParallelCall()
1230 getThreadID(CGF, Loc), in emitParallelCall()
1233 llvm::ConstantInt::get(CGF.Int32Ty, -1), in emitParallelCall()
1237 CGF.VoidPtrPtrTy), in emitParallelCall()
1239 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in emitParallelCall()
1245 RCG(CGF); in emitParallelCall()
1248 void CGOpenMPRuntimeGPU::syncCTAThreads(CodeGenFunction &CGF) { in syncCTAThreads() argument
1250 if (!CGF.HaveInsertPoint()) in syncCTAThreads()
1257 llvm::ConstantInt::get(CGF.Int32Ty, /*V=*/0, /*isSigned=*/true)}; in syncCTAThreads()
1258 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in syncCTAThreads()
1263 void CGOpenMPRuntimeGPU::emitBarrierCall(CodeGenFunction &CGF, in emitBarrierCall() argument
1268 if (!CGF.HaveInsertPoint()) in emitBarrierCall()
1272 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc, Flags), in emitBarrierCall()
1273 getThreadID(CGF, Loc)}; in emitBarrierCall()
1275 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in emitBarrierCall()
1281 CodeGenFunction &CGF, StringRef CriticalName, in emitCriticalRegion() argument
1284 llvm::BasicBlock *LoopBB = CGF.createBasicBlock("omp.critical.loop"); in emitCriticalRegion()
1285 llvm::BasicBlock *TestBB = CGF.createBasicBlock("omp.critical.test"); in emitCriticalRegion()
1286 llvm::BasicBlock *SyncBB = CGF.createBasicBlock("omp.critical.sync"); in emitCriticalRegion()
1287 llvm::BasicBlock *BodyBB = CGF.createBasicBlock("omp.critical.body"); in emitCriticalRegion()
1288 llvm::BasicBlock *ExitBB = CGF.createBasicBlock("omp.critical.exit"); in emitCriticalRegion()
1290 auto &RT = static_cast<CGOpenMPRuntimeGPU &>(CGF.CGM.getOpenMPRuntime()); in emitCriticalRegion()
1293 llvm::Value *Mask = CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in emitCriticalRegion()
1296 llvm::Value *ThreadID = RT.getGPUThreadID(CGF); in emitCriticalRegion()
1299 llvm::Value *TeamWidth = RT.getGPUNumThreads(CGF); in emitCriticalRegion()
1303 CGF.getContext().getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/0); in emitCriticalRegion()
1304 Address Counter = CGF.CreateMemTemp(Int32Ty, "critical_counter"); in emitCriticalRegion()
1305 LValue CounterLVal = CGF.MakeAddrLValue(Counter, Int32Ty); in emitCriticalRegion()
1306 CGF.EmitStoreOfScalar(llvm::Constant::getNullValue(CGM.Int32Ty), CounterLVal, in emitCriticalRegion()
1310 CGF.EmitBlock(LoopBB); in emitCriticalRegion()
1311 llvm::Value *CounterVal = CGF.EmitLoadOfScalar(CounterLVal, Loc); in emitCriticalRegion()
1312 llvm::Value *CmpLoopBound = CGF.Builder.CreateICmpSLT(CounterVal, TeamWidth); in emitCriticalRegion()
1313 CGF.Builder.CreateCondBr(CmpLoopBound, TestBB, ExitBB); in emitCriticalRegion()
1317 CGF.EmitBlock(TestBB); in emitCriticalRegion()
1318 CounterVal = CGF.EmitLoadOfScalar(CounterLVal, Loc); in emitCriticalRegion()
1320 CGF.Builder.CreateICmpEQ(ThreadID, CounterVal); in emitCriticalRegion()
1321 CGF.Builder.CreateCondBr(CmpThreadToCounter, BodyBB, SyncBB); in emitCriticalRegion()
1324 CGF.EmitBlock(BodyBB); in emitCriticalRegion()
1327 CGOpenMPRuntime::emitCriticalRegion(CGF, CriticalName, CriticalOpGen, Loc, in emitCriticalRegion()
1334 CGF.EmitBlock(SyncBB); in emitCriticalRegion()
1336 (void)CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in emitCriticalRegion()
1341 CGF.Builder.CreateNSWAdd(CounterVal, CGF.Builder.getInt32(1)); in emitCriticalRegion()
1342 CGF.EmitStoreOfScalar(IncCounterVal, CounterLVal); in emitCriticalRegion()
1343 CGF.EmitBranch(LoopBB); in emitCriticalRegion()
1346 CGF.EmitBlock(ExitBB, /*IsFinished=*/true); in emitCriticalRegion()
1350 static llvm::Value *castValueToType(CodeGenFunction &CGF, llvm::Value *Val, in castValueToType() argument
1353 assert(!CGF.getContext().getTypeSizeInChars(CastTy).isZero() && in castValueToType()
1355 assert(!CGF.getContext().getTypeSizeInChars(ValTy).isZero() && in castValueToType()
1357 llvm::Type *LLVMCastTy = CGF.ConvertTypeForMem(CastTy); in castValueToType()
1360 if (CGF.getContext().getTypeSizeInChars(ValTy) == in castValueToType()
1361 CGF.getContext().getTypeSizeInChars(CastTy)) in castValueToType()
1362 return CGF.Builder.CreateBitCast(Val, LLVMCastTy); in castValueToType()
1364 return CGF.Builder.CreateIntCast(Val, LLVMCastTy, in castValueToType()
1366 Address CastItem = CGF.CreateMemTemp(CastTy); in castValueToType()
1367 Address ValCastItem = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in castValueToType()
1370 CGF.EmitStoreOfScalar(Val, ValCastItem, /*Volatile=*/false, ValTy, in castValueToType()
1373 return CGF.EmitLoadOfScalar(CastItem, /*Volatile=*/false, CastTy, Loc, in castValueToType()
1380 static llvm::Value *createRuntimeShuffleFunction(CodeGenFunction &CGF, in createRuntimeShuffleFunction() argument
1385 CodeGenModule &CGM = CGF.CGM; in createRuntimeShuffleFunction()
1386 CGBuilderTy &Bld = CGF.Builder; in createRuntimeShuffleFunction()
1391 CharUnits Size = CGF.getContext().getTypeSizeInChars(ElemType); in createRuntimeShuffleFunction()
1400 QualType CastTy = CGF.getContext().getIntTypeForBitwidth( in createRuntimeShuffleFunction()
1402 llvm::Value *ElemCast = castValueToType(CGF, Elem, ElemType, CastTy, Loc); in createRuntimeShuffleFunction()
1404 Bld.CreateIntCast(RT.getGPUWarpSize(CGF), CGM.Int16Ty, /*isSigned=*/true); in createRuntimeShuffleFunction()
1406 llvm::Value *ShuffledVal = CGF.EmitRuntimeCall( in createRuntimeShuffleFunction()
1410 return castValueToType(CGF, ShuffledVal, CastTy, ElemType, Loc); in createRuntimeShuffleFunction()
1413 static void shuffleAndStore(CodeGenFunction &CGF, Address SrcAddr, in shuffleAndStore() argument
1416 CGBuilderTy &Bld = CGF.Builder; in shuffleAndStore()
1418 CharUnits Size = CGF.getContext().getTypeSizeInChars(ElemType); in shuffleAndStore()
1432 Bld.CreateConstGEP(SrcAddr, 1), CGF.VoidPtrTy, CGF.Int8Ty); in shuffleAndStore()
1436 QualType IntType = CGF.getContext().getIntTypeForBitwidth( in shuffleAndStore()
1437 CGF.getContext().toBits(CharUnits::fromQuantity(IntSize)), in shuffleAndStore()
1439 llvm::Type *IntTy = CGF.ConvertTypeForMem(IntType); in shuffleAndStore()
1445 llvm::BasicBlock *PreCondBB = CGF.createBasicBlock(".shuffle.pre_cond"); in shuffleAndStore()
1446 llvm::BasicBlock *ThenBB = CGF.createBasicBlock(".shuffle.then"); in shuffleAndStore()
1447 llvm::BasicBlock *ExitBB = CGF.createBasicBlock(".shuffle.exit"); in shuffleAndStore()
1449 CGF.EmitBlock(PreCondBB); in shuffleAndStore()
1460 CGF.Int8Ty, PtrEnd.getPointer(), in shuffleAndStore()
1462 CGF.VoidPtrTy)); in shuffleAndStore()
1465 CGF.EmitBlock(ThenBB); in shuffleAndStore()
1467 CGF, in shuffleAndStore()
1468 CGF.EmitLoadOfScalar(Ptr, /*Volatile=*/false, IntType, Loc, in shuffleAndStore()
1472 CGF.EmitStoreOfScalar(Res, ElemPtr, /*Volatile=*/false, IntType, in shuffleAndStore()
1479 CGF.EmitBranch(PreCondBB); in shuffleAndStore()
1480 CGF.EmitBlock(ExitBB); in shuffleAndStore()
1483 CGF, in shuffleAndStore()
1484 CGF.EmitLoadOfScalar(Ptr, /*Volatile=*/false, IntType, Loc, in shuffleAndStore()
1488 CGF.EmitStoreOfScalar(Res, ElemPtr, /*Volatile=*/false, IntType, in shuffleAndStore()
1522 CopyAction Action, CodeGenFunction &CGF, QualType ReductionArrayTy, in emitReductionListCopy() argument
1526 CodeGenModule &CGM = CGF.CGM;
1528 CGBuilderTy &Bld = CGF.Builder;
1552 llvm::Type *PrivateLlvmPtrType = CGF.ConvertType(PrivatePtrType);
1559 CGF.EmitLoadOfPointer(CGF.Builder.CreateElementBitCast(
1567 CGF.CreateMemTemp(Private->getType(), ".omp.reduction.element");
1576 CGF.EmitLoadOfPointer(CGF.Builder.CreateElementBitCast(
1584 CGF.EmitLoadOfPointer(CGF.Builder.CreateElementBitCast(
1593 CGF.EmitLoadOfPointer(CGF.Builder.CreateElementBitCast(
1599 llvm::Value *ElementSizeInChars = CGF.getTypeSize(Private->getType());
1605 Bld.CreateIntToPtr(ScratchPadElemAbsolutePtrVal, CGF.VoidPtrTy);
1606 DestElementAddr = Address(ScratchPadElemAbsolutePtrVal, CGF.Int8Ty,
1614 llvm::Value *ElementSizeInChars = CGF.getTypeSize(Private->getType());
1620 Bld.CreateIntToPtr(ScratchPadElemAbsolutePtrVal, CGF.VoidPtrTy);
1621 SrcElementAddr = Address(ScratchPadElemAbsolutePtrVal, CGF.Int8Ty,
1629 CGF.CreateMemTemp(Private->getType(), ".omp.reduction.element");
1638 SrcElementAddr, CGF.ConvertTypeForMem(Private->getType()));
1645 shuffleAndStore(CGF, SrcElementAddr, DestElementAddr, Private->getType(),
1648 switch (CGF.getEvaluationKind(Private->getType())) {
1650 llvm::Value *Elem = CGF.EmitLoadOfScalar(
1655 CGF.EmitStoreOfScalar(
1661 CodeGenFunction::ComplexPairTy Elem = CGF.EmitLoadOfComplex(
1662 CGF.MakeAddrLValue(SrcElementAddr, Private->getType()),
1664 CGF.EmitStoreOfComplex(
1665 Elem, CGF.MakeAddrLValue(DestElementAddr, Private->getType()),
1670 CGF.EmitAggregateCopy(
1671 CGF.MakeAddrLValue(DestElementAddr, Private->getType()),
1672 CGF.MakeAddrLValue(SrcElementAddr, Private->getType()),
1684 CGF.EmitStoreOfScalar(Bld.CreatePointerBitCastOrAddrSpaceCast(
1685 DestElementAddr.getPointer(), CGF.VoidPtrTy),
1698 llvm::Value *ElementSizeInChars = CGF.getTypeSize(Private->getType());
1717 Address(ScratchpadBasePtr, CGF.VoidPtrTy, CGF.getPointerAlign());
1720 Address(ScratchpadBasePtr, CGF.VoidPtrTy, CGF.getPointerAlign());
1767 CodeGenFunction CGF(CGM); in emitInterWarpCopyFunction() local
1768 CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, CGFI, Args, Loc, Loc); in emitInterWarpCopyFunction()
1770 CGBuilderTy &Bld = CGF.Builder; in emitInterWarpCopyFunction()
1783 unsigned WarpSize = CGF.getTarget().getGridValue().GV_Warp_Size; in emitInterWarpCopyFunction()
1795 auto &RT = static_cast<CGOpenMPRuntimeGPU &>(CGF.CGM.getOpenMPRuntime()); in emitInterWarpCopyFunction()
1797 llvm::Value *ThreadID = RT.getGPUThreadID(CGF); in emitInterWarpCopyFunction()
1799 llvm::Value *LaneID = getNVPTXLaneID(CGF); in emitInterWarpCopyFunction()
1801 llvm::Value *WarpID = getNVPTXWarpID(CGF); in emitInterWarpCopyFunction()
1803 Address AddrReduceListArg = CGF.GetAddrOfLocalVar(&ReduceListArg); in emitInterWarpCopyFunction()
1804 llvm::Type *ElemTy = CGF.ConvertTypeForMem(ReductionArrayTy); in emitInterWarpCopyFunction()
1807 CGF.EmitLoadOfScalar( in emitInterWarpCopyFunction()
1811 ElemTy, CGF.getPointerAlign()); in emitInterWarpCopyFunction()
1829 llvm::Type *CopyType = CGF.ConvertTypeForMem(CType); in emitInterWarpCopyFunction()
1836 CntAddr = CGF.CreateMemTemp(C.IntTy, ".cnt.addr"); in emitInterWarpCopyFunction()
1837 CGF.EmitStoreOfScalar(llvm::Constant::getNullValue(CGM.IntTy), CntAddr, in emitInterWarpCopyFunction()
1839 PrecondBB = CGF.createBasicBlock("precond"); in emitInterWarpCopyFunction()
1840 ExitBB = CGF.createBasicBlock("exit"); in emitInterWarpCopyFunction()
1841 llvm::BasicBlock *BodyBB = CGF.createBasicBlock("body"); in emitInterWarpCopyFunction()
1843 (void)ApplyDebugLocation::CreateEmpty(CGF); in emitInterWarpCopyFunction()
1844 CGF.EmitBlock(PrecondBB); in emitInterWarpCopyFunction()
1845 Cnt = CGF.EmitLoadOfScalar(CntAddr, /*Volatile=*/false, C.IntTy, Loc); in emitInterWarpCopyFunction()
1849 CGF.EmitBlock(BodyBB); in emitInterWarpCopyFunction()
1852 CGM.getOpenMPRuntime().emitBarrierCall(CGF, Loc, OMPD_unknown, in emitInterWarpCopyFunction()
1855 llvm::BasicBlock *ThenBB = CGF.createBasicBlock("then"); in emitInterWarpCopyFunction()
1856 llvm::BasicBlock *ElseBB = CGF.createBasicBlock("else"); in emitInterWarpCopyFunction()
1857 llvm::BasicBlock *MergeBB = CGF.createBasicBlock("ifcont"); in emitInterWarpCopyFunction()
1862 CGF.EmitBlock(ThenBB); in emitInterWarpCopyFunction()
1866 llvm::Value *ElemPtrPtr = CGF.EmitLoadOfScalar( in emitInterWarpCopyFunction()
1869 Address ElemPtr(ElemPtrPtr, CGF.Int8Ty, Align); in emitInterWarpCopyFunction()
1890 llvm::Value *Elem = CGF.EmitLoadOfScalar( in emitInterWarpCopyFunction()
1894 CGF.EmitStoreOfScalar(Elem, MediumPtr, /*Volatile=*/true, CType, in emitInterWarpCopyFunction()
1900 CGF.EmitBlock(ElseBB); in emitInterWarpCopyFunction()
1903 CGF.EmitBlock(MergeBB); in emitInterWarpCopyFunction()
1906 CGM.getOpenMPRuntime().emitBarrierCall(CGF, Loc, OMPD_unknown, in emitInterWarpCopyFunction()
1913 llvm::BasicBlock *W0ThenBB = CGF.createBasicBlock("then"); in emitInterWarpCopyFunction()
1914 llvm::BasicBlock *W0ElseBB = CGF.createBasicBlock("else"); in emitInterWarpCopyFunction()
1915 llvm::BasicBlock *W0MergeBB = CGF.createBasicBlock("ifcont"); in emitInterWarpCopyFunction()
1917 Address AddrNumWarpsArg = CGF.GetAddrOfLocalVar(&NumWarpsArg); in emitInterWarpCopyFunction()
1918 llvm::Value *NumWarpsVal = CGF.EmitLoadOfScalar( in emitInterWarpCopyFunction()
1926 CGF.EmitBlock(W0ThenBB); in emitInterWarpCopyFunction()
1942 llvm::Value *TargetElemPtrVal = CGF.EmitLoadOfScalar( in emitInterWarpCopyFunction()
1944 Address TargetElemPtr(TargetElemPtrVal, CGF.Int8Ty, Align); in emitInterWarpCopyFunction()
1951 CGF.EmitLoadOfScalar(SrcMediumPtr, /*Volatile=*/true, CType, Loc); in emitInterWarpCopyFunction()
1952 CGF.EmitStoreOfScalar(SrcMediumValue, TargetElemPtr, /*Volatile=*/false, in emitInterWarpCopyFunction()
1956 CGF.EmitBlock(W0ElseBB); in emitInterWarpCopyFunction()
1959 CGF.EmitBlock(W0MergeBB); in emitInterWarpCopyFunction()
1963 CGF.EmitStoreOfScalar(Cnt, CntAddr, /*Volatile=*/false, C.IntTy); in emitInterWarpCopyFunction()
1964 CGF.EmitBranch(PrecondBB); in emitInterWarpCopyFunction()
1965 (void)ApplyDebugLocation::CreateEmpty(CGF); in emitInterWarpCopyFunction()
1966 CGF.EmitBlock(ExitBB); in emitInterWarpCopyFunction()
1973 CGF.FinishFunction(); in emitInterWarpCopyFunction()
2074 CodeGenFunction CGF(CGM); in emitShuffleAndReduceFunction() local
2075 CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, CGFI, Args, Loc, Loc); in emitShuffleAndReduceFunction()
2077 CGBuilderTy &Bld = CGF.Builder; in emitShuffleAndReduceFunction()
2079 Address AddrReduceListArg = CGF.GetAddrOfLocalVar(&ReduceListArg); in emitShuffleAndReduceFunction()
2080 llvm::Type *ElemTy = CGF.ConvertTypeForMem(ReductionArrayTy); in emitShuffleAndReduceFunction()
2083 CGF.EmitLoadOfScalar(AddrReduceListArg, /*Volatile=*/false, in emitShuffleAndReduceFunction()
2086 ElemTy, CGF.getPointerAlign()); in emitShuffleAndReduceFunction()
2088 Address AddrLaneIDArg = CGF.GetAddrOfLocalVar(&LaneIDArg); in emitShuffleAndReduceFunction()
2089 llvm::Value *LaneIDArgVal = CGF.EmitLoadOfScalar( in emitShuffleAndReduceFunction()
2092 Address AddrRemoteLaneOffsetArg = CGF.GetAddrOfLocalVar(&RemoteLaneOffsetArg); in emitShuffleAndReduceFunction()
2093 llvm::Value *RemoteLaneOffsetArgVal = CGF.EmitLoadOfScalar( in emitShuffleAndReduceFunction()
2096 Address AddrAlgoVerArg = CGF.GetAddrOfLocalVar(&AlgoVerArg); in emitShuffleAndReduceFunction()
2097 llvm::Value *AlgoVerArgVal = CGF.EmitLoadOfScalar( in emitShuffleAndReduceFunction()
2103 CGF.CreateMemTemp(ReductionArrayTy, ".omp.reduction.remote_reduce_list"); in emitShuffleAndReduceFunction()
2108 emitReductionListCopy(RemoteLaneToThread, CGF, ReductionArrayTy, Privates, in emitShuffleAndReduceFunction()
2150 llvm::BasicBlock *ThenBB = CGF.createBasicBlock("then"); in emitShuffleAndReduceFunction()
2151 llvm::BasicBlock *ElseBB = CGF.createBasicBlock("else"); in emitShuffleAndReduceFunction()
2152 llvm::BasicBlock *MergeBB = CGF.createBasicBlock("ifcont"); in emitShuffleAndReduceFunction()
2155 CGF.EmitBlock(ThenBB); in emitShuffleAndReduceFunction()
2158 LocalReduceList.getPointer(), CGF.VoidPtrTy); in emitShuffleAndReduceFunction()
2160 RemoteReduceList.getPointer(), CGF.VoidPtrTy); in emitShuffleAndReduceFunction()
2162 CGF, Loc, ReduceFn, {LocalReduceListPtr, RemoteReduceListPtr}); in emitShuffleAndReduceFunction()
2165 CGF.EmitBlock(ElseBB); in emitShuffleAndReduceFunction()
2168 CGF.EmitBlock(MergeBB); in emitShuffleAndReduceFunction()
2176 llvm::BasicBlock *CpyThenBB = CGF.createBasicBlock("then"); in emitShuffleAndReduceFunction()
2177 llvm::BasicBlock *CpyElseBB = CGF.createBasicBlock("else"); in emitShuffleAndReduceFunction()
2178 llvm::BasicBlock *CpyMergeBB = CGF.createBasicBlock("ifcont"); in emitShuffleAndReduceFunction()
2181 CGF.EmitBlock(CpyThenBB); in emitShuffleAndReduceFunction()
2182 emitReductionListCopy(ThreadCopy, CGF, ReductionArrayTy, Privates, in emitShuffleAndReduceFunction()
2186 CGF.EmitBlock(CpyElseBB); in emitShuffleAndReduceFunction()
2189 CGF.EmitBlock(CpyMergeBB); in emitShuffleAndReduceFunction()
2191 CGF.FinishFunction(); in emitShuffleAndReduceFunction()
2230 CodeGenFunction CGF(CGM); in emitListToGlobalCopyFunction() local
2231 CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, CGFI, Args, Loc, Loc); in emitListToGlobalCopyFunction()
2233 CGBuilderTy &Bld = CGF.Builder; in emitListToGlobalCopyFunction()
2235 Address AddrReduceListArg = CGF.GetAddrOfLocalVar(&ReduceListArg); in emitListToGlobalCopyFunction()
2236 Address AddrBufferArg = CGF.GetAddrOfLocalVar(&BufferArg); in emitListToGlobalCopyFunction()
2237 llvm::Type *ElemTy = CGF.ConvertTypeForMem(ReductionArrayTy); in emitListToGlobalCopyFunction()
2240 CGF.EmitLoadOfScalar(AddrReduceListArg, /*Volatile=*/false, in emitListToGlobalCopyFunction()
2243 ElemTy, CGF.getPointerAlign()); in emitListToGlobalCopyFunction()
2248 CGF.EmitLoadOfScalar(AddrBufferArg, /*Volatile=*/false, C.VoidPtrTy, Loc), in emitListToGlobalCopyFunction()
2250 llvm::Value *Idxs[] = {llvm::ConstantInt::getNullValue(CGF.Int32Ty), in emitListToGlobalCopyFunction()
2251 CGF.EmitLoadOfScalar(CGF.GetAddrOfLocalVar(&IdxArg), in emitListToGlobalCopyFunction()
2258 llvm::Value *ElemPtrPtr = CGF.EmitLoadOfScalar( in emitListToGlobalCopyFunction()
2261 ElemTy = CGF.ConvertTypeForMem(Private->getType()); in emitListToGlobalCopyFunction()
2269 LValue GlobLVal = CGF.EmitLValueForField( in emitListToGlobalCopyFunction()
2270 CGF.MakeNaturalAlignAddrLValue(BufferArrPtr, StaticTy), FD); in emitListToGlobalCopyFunction()
2271 Address GlobAddr = GlobLVal.getAddress(CGF); in emitListToGlobalCopyFunction()
2275 CGF.ConvertTypeForMem(Private->getType()), in emitListToGlobalCopyFunction()
2277 switch (CGF.getEvaluationKind(Private->getType())) { in emitListToGlobalCopyFunction()
2279 llvm::Value *V = CGF.EmitLoadOfScalar( in emitListToGlobalCopyFunction()
2282 CGF.EmitStoreOfScalar(V, GlobLVal); in emitListToGlobalCopyFunction()
2286 CodeGenFunction::ComplexPairTy V = CGF.EmitLoadOfComplex( in emitListToGlobalCopyFunction()
2287 CGF.MakeAddrLValue(ElemPtr, Private->getType()), Loc); in emitListToGlobalCopyFunction()
2288 CGF.EmitStoreOfComplex(V, GlobLVal, /*isInit=*/false); in emitListToGlobalCopyFunction()
2292 CGF.EmitAggregateCopy(GlobLVal, in emitListToGlobalCopyFunction()
2293 CGF.MakeAddrLValue(ElemPtr, Private->getType()), in emitListToGlobalCopyFunction()
2300 CGF.FinishFunction(); in emitListToGlobalCopyFunction()
2343 CodeGenFunction CGF(CGM); in emitListToGlobalReduceFunction() local
2344 CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, CGFI, Args, Loc, Loc); in emitListToGlobalReduceFunction()
2346 CGBuilderTy &Bld = CGF.Builder; in emitListToGlobalReduceFunction()
2348 Address AddrBufferArg = CGF.GetAddrOfLocalVar(&BufferArg); in emitListToGlobalReduceFunction()
2353 CGF.EmitLoadOfScalar(AddrBufferArg, /*Volatile=*/false, C.VoidPtrTy, Loc), in emitListToGlobalReduceFunction()
2359 CGF.CreateMemTemp(ReductionArrayTy, ".omp.reduction.red_list"); in emitListToGlobalReduceFunction()
2361 llvm::Value *Idxs[] = {llvm::ConstantInt::getNullValue(CGF.Int32Ty), in emitListToGlobalReduceFunction()
2362 CGF.EmitLoadOfScalar(CGF.GetAddrOfLocalVar(&IdxArg), in emitListToGlobalReduceFunction()
2367 Address Elem = CGF.Builder.CreateConstArrayGEP(ReductionList, Idx); in emitListToGlobalReduceFunction()
2371 LValue GlobLVal = CGF.EmitLValueForField( in emitListToGlobalReduceFunction()
2372 CGF.MakeNaturalAlignAddrLValue(BufferArrPtr, StaticTy), FD); in emitListToGlobalReduceFunction()
2373 Address GlobAddr = GlobLVal.getAddress(CGF); in emitListToGlobalReduceFunction()
2376 llvm::Value *Ptr = CGF.EmitCastToVoidPtr(BufferPtr); in emitListToGlobalReduceFunction()
2377 CGF.EmitStoreOfScalar(Ptr, Elem, /*Volatile=*/false, C.VoidPtrTy); in emitListToGlobalReduceFunction()
2381 Elem = CGF.Builder.CreateConstArrayGEP(ReductionList, Idx); in emitListToGlobalReduceFunction()
2382 llvm::Value *Size = CGF.Builder.CreateIntCast( in emitListToGlobalReduceFunction()
2383 CGF.getVLASize( in emitListToGlobalReduceFunction()
2384 CGF.getContext().getAsVariableArrayType((*IPriv)->getType())) in emitListToGlobalReduceFunction()
2386 CGF.SizeTy, /*isSigned=*/false); in emitListToGlobalReduceFunction()
2387 CGF.Builder.CreateStore(CGF.Builder.CreateIntToPtr(Size, CGF.VoidPtrTy), in emitListToGlobalReduceFunction()
2394 CGF.EmitCastToVoidPtr(ReductionList.getPointer()); in emitListToGlobalReduceFunction()
2395 Address AddrReduceListArg = CGF.GetAddrOfLocalVar(&ReduceListArg); in emitListToGlobalReduceFunction()
2396 llvm::Value *ReducedPtr = CGF.EmitLoadOfScalar( in emitListToGlobalReduceFunction()
2399 CGF, Loc, ReduceFn, {GlobalReduceList, ReducedPtr}); in emitListToGlobalReduceFunction()
2400 CGF.FinishFunction(); in emitListToGlobalReduceFunction()
2439 CodeGenFunction CGF(CGM); in emitGlobalToListCopyFunction() local
2440 CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, CGFI, Args, Loc, Loc); in emitGlobalToListCopyFunction()
2442 CGBuilderTy &Bld = CGF.Builder; in emitGlobalToListCopyFunction()
2444 Address AddrReduceListArg = CGF.GetAddrOfLocalVar(&ReduceListArg); in emitGlobalToListCopyFunction()
2445 Address AddrBufferArg = CGF.GetAddrOfLocalVar(&BufferArg); in emitGlobalToListCopyFunction()
2446 llvm::Type *ElemTy = CGF.ConvertTypeForMem(ReductionArrayTy); in emitGlobalToListCopyFunction()
2449 CGF.EmitLoadOfScalar(AddrReduceListArg, /*Volatile=*/false, in emitGlobalToListCopyFunction()
2452 ElemTy, CGF.getPointerAlign()); in emitGlobalToListCopyFunction()
2457 CGF.EmitLoadOfScalar(AddrBufferArg, /*Volatile=*/false, C.VoidPtrTy, Loc), in emitGlobalToListCopyFunction()
2460 llvm::Value *Idxs[] = {llvm::ConstantInt::getNullValue(CGF.Int32Ty), in emitGlobalToListCopyFunction()
2461 CGF.EmitLoadOfScalar(CGF.GetAddrOfLocalVar(&IdxArg), in emitGlobalToListCopyFunction()
2468 llvm::Value *ElemPtrPtr = CGF.EmitLoadOfScalar( in emitGlobalToListCopyFunction()
2471 ElemTy = CGF.ConvertTypeForMem(Private->getType()); in emitGlobalToListCopyFunction()
2479 LValue GlobLVal = CGF.EmitLValueForField( in emitGlobalToListCopyFunction()
2480 CGF.MakeNaturalAlignAddrLValue(BufferArrPtr, StaticTy), FD); in emitGlobalToListCopyFunction()
2481 Address GlobAddr = GlobLVal.getAddress(CGF); in emitGlobalToListCopyFunction()
2485 CGF.ConvertTypeForMem(Private->getType()), in emitGlobalToListCopyFunction()
2487 switch (CGF.getEvaluationKind(Private->getType())) { in emitGlobalToListCopyFunction()
2489 llvm::Value *V = CGF.EmitLoadOfScalar(GlobLVal, Loc); in emitGlobalToListCopyFunction()
2490 CGF.EmitStoreOfScalar(V, ElemPtr, /*Volatile=*/false, Private->getType(), in emitGlobalToListCopyFunction()
2496 CodeGenFunction::ComplexPairTy V = CGF.EmitLoadOfComplex(GlobLVal, Loc); in emitGlobalToListCopyFunction()
2497 CGF.EmitStoreOfComplex(V, CGF.MakeAddrLValue(ElemPtr, Private->getType()), in emitGlobalToListCopyFunction()
2502 CGF.EmitAggregateCopy(CGF.MakeAddrLValue(ElemPtr, Private->getType()), in emitGlobalToListCopyFunction()
2510 CGF.FinishFunction(); in emitGlobalToListCopyFunction()
2553 CodeGenFunction CGF(CGM); in emitGlobalToListReduceFunction() local
2554 CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, CGFI, Args, Loc, Loc); in emitGlobalToListReduceFunction()
2556 CGBuilderTy &Bld = CGF.Builder; in emitGlobalToListReduceFunction()
2558 Address AddrBufferArg = CGF.GetAddrOfLocalVar(&BufferArg); in emitGlobalToListReduceFunction()
2563 CGF.EmitLoadOfScalar(AddrBufferArg, /*Volatile=*/false, C.VoidPtrTy, Loc), in emitGlobalToListReduceFunction()
2569 CGF.CreateMemTemp(ReductionArrayTy, ".omp.reduction.red_list"); in emitGlobalToListReduceFunction()
2571 llvm::Value *Idxs[] = {llvm::ConstantInt::getNullValue(CGF.Int32Ty), in emitGlobalToListReduceFunction()
2572 CGF.EmitLoadOfScalar(CGF.GetAddrOfLocalVar(&IdxArg), in emitGlobalToListReduceFunction()
2577 Address Elem = CGF.Builder.CreateConstArrayGEP(ReductionList, Idx); in emitGlobalToListReduceFunction()
2581 LValue GlobLVal = CGF.EmitLValueForField( in emitGlobalToListReduceFunction()
2582 CGF.MakeNaturalAlignAddrLValue(BufferArrPtr, StaticTy), FD); in emitGlobalToListReduceFunction()
2583 Address GlobAddr = GlobLVal.getAddress(CGF); in emitGlobalToListReduceFunction()
2586 llvm::Value *Ptr = CGF.EmitCastToVoidPtr(BufferPtr); in emitGlobalToListReduceFunction()
2587 CGF.EmitStoreOfScalar(Ptr, Elem, /*Volatile=*/false, C.VoidPtrTy); in emitGlobalToListReduceFunction()
2591 Elem = CGF.Builder.CreateConstArrayGEP(ReductionList, Idx); in emitGlobalToListReduceFunction()
2592 llvm::Value *Size = CGF.Builder.CreateIntCast( in emitGlobalToListReduceFunction()
2593 CGF.getVLASize( in emitGlobalToListReduceFunction()
2594 CGF.getContext().getAsVariableArrayType((*IPriv)->getType())) in emitGlobalToListReduceFunction()
2596 CGF.SizeTy, /*isSigned=*/false); in emitGlobalToListReduceFunction()
2597 CGF.Builder.CreateStore(CGF.Builder.CreateIntToPtr(Size, CGF.VoidPtrTy), in emitGlobalToListReduceFunction()
2604 CGF.EmitCastToVoidPtr(ReductionList.getPointer()); in emitGlobalToListReduceFunction()
2605 Address AddrReduceListArg = CGF.GetAddrOfLocalVar(&ReduceListArg); in emitGlobalToListReduceFunction()
2606 llvm::Value *ReducedPtr = CGF.EmitLoadOfScalar( in emitGlobalToListReduceFunction()
2609 CGF, Loc, ReduceFn, {ReducedPtr, GlobalReduceList}); in emitGlobalToListReduceFunction()
2610 CGF.FinishFunction(); in emitGlobalToListReduceFunction()
2857 CodeGenFunction &CGF, SourceLocation Loc, ArrayRef<const Expr *> Privates, in emitReduction() argument
2860 if (!CGF.HaveInsertPoint()) in emitReduction()
2871 CGOpenMPRuntime::emitReduction(CGF, Loc, Privates, LHSExprs, RHSExprs, in emitReduction()
2883 llvm::Value *RTLoc = emitUpdateLocation(CGF, Loc); in emitReduction()
2884 llvm::Value *ThreadId = getThreadID(CGF, Loc); in emitReduction()
2901 CGF.CreateMemTemp(ReductionArrayTy, ".omp.reduction.red_list"); in emitReduction()
2905 Address Elem = CGF.Builder.CreateConstArrayGEP(ReductionList, Idx); in emitReduction()
2906 CGF.Builder.CreateStore( in emitReduction()
2907 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitReduction()
2908 CGF.EmitLValue(RHSExprs[I]).getPointer(CGF), CGF.VoidPtrTy), in emitReduction()
2913 Elem = CGF.Builder.CreateConstArrayGEP(ReductionList, Idx); in emitReduction()
2914 llvm::Value *Size = CGF.Builder.CreateIntCast( in emitReduction()
2915 CGF.getVLASize( in emitReduction()
2916 CGF.getContext().getAsVariableArrayType((*IPriv)->getType())) in emitReduction()
2918 CGF.SizeTy, /*isSigned=*/false); in emitReduction()
2919 CGF.Builder.CreateStore(CGF.Builder.CreateIntToPtr(Size, CGF.VoidPtrTy), in emitReduction()
2924 llvm::Value *RL = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitReduction()
2925 ReductionList.getPointer(), CGF.VoidPtrTy); in emitReduction()
2927 emitReductionFunction(Loc, CGF.ConvertTypeForMem(ReductionArrayTy), in emitReduction()
2929 llvm::Value *ReductionArrayTySize = CGF.getTypeSize(ReductionArrayTy); in emitReduction()
2938 CGF.Builder.getInt32(RHSExprs.size()), in emitReduction()
2944 Res = CGF.EmitRuntimeCall( in emitReduction()
2967 llvm::Value *GlobalBufferPtr = CGF.EmitLoadOfScalar( in emitReduction()
2968 Address(KernelTeamsReductionPtr, CGF.VoidPtrTy, CGM.getPointerAlign()), in emitReduction()
2985 CGF.Builder.getInt32(C.getLangOpts().OpenMPCUDAReductionBufNum), in emitReduction()
2994 Res = CGF.EmitRuntimeCall( in emitReduction()
3001 llvm::BasicBlock *ExitBB = CGF.createBasicBlock(".omp.reduction.done"); in emitReduction()
3002 llvm::BasicBlock *ThenBB = CGF.createBasicBlock(".omp.reduction.then"); in emitReduction()
3003 llvm::Value *Cond = CGF.Builder.CreateICmpEQ( in emitReduction()
3005 CGF.Builder.CreateCondBr(Cond, ThenBB, ExitBB); in emitReduction()
3011 CGF.EmitBlock(ThenBB); in emitReduction()
3015 this](CodeGenFunction &CGF, PrePostActionTy &Action) { in emitReduction() argument
3020 emitSingleReductionCombiner(CGF, E, *IPriv, cast<DeclRefExpr>(*ILHS), in emitReduction()
3035 RCG(CGF); in emitReduction()
3037 (void)ApplyDebugLocation::CreateEmpty(CGF); in emitReduction()
3038 CGF.EmitBlock(ExitBB, /*IsFinished=*/true); in emitReduction()
3074 CGOpenMPRuntimeGPU::getParameterAddress(CodeGenFunction &CGF, in getParameterAddress() argument
3080 Address LocalAddr = CGF.GetAddrOfLocalVar(TargetParam); in getParameterAddress()
3086 CGF.getTypes().getTargetAddressSpace(NativePointeeTy); in getParameterAddress()
3088 llvm::Value *TargetAddr = CGF.EmitLoadOfScalar( in getParameterAddress()
3091 TargetAddr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in getParameterAddress()
3095 TargetAddr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in getParameterAddress()
3099 Address NativeParamAddr = CGF.CreateMemTemp(NativeParamType); in getParameterAddress()
3100 CGF.EmitStoreOfScalar(TargetAddr, NativeParamAddr, /*Volatile=*/false, in getParameterAddress()
3106 CodeGenFunction &CGF, SourceLocation Loc, llvm::FunctionCallee OutlinedFn, in emitOutlinedFunctionCall() argument
3122 llvm::Value *TargetArg = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitOutlinedFunctionCall()
3126 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(TargetArg, TargetType)); in emitOutlinedFunctionCall()
3128 CGOpenMPRuntime::emitOutlinedFunctionCall(CGF, Loc, OutlinedFn, TargetArgs); in emitOutlinedFunctionCall()
3174 CodeGenFunction CGF(CGM, /*suppressNewContext=*/true); in createParallelDataSharingWrapper() local
3175 CGF.StartFunction(GlobalDecl(), Ctx.VoidTy, Fn, CGFI, WrapperArgs, in createParallelDataSharingWrapper()
3181 Address ZeroAddr = CGF.CreateDefaultAlignTempAlloca(CGF.Int32Ty, in createParallelDataSharingWrapper()
3183 CGF.Builder.CreateStore(CGF.Builder.getInt32(/*C*/ 0), ZeroAddr); in createParallelDataSharingWrapper()
3187 Args.emplace_back(CGF.GetAddrOfLocalVar(&WrapperArg).getPointer()); in createParallelDataSharingWrapper()
3190 CGBuilderTy &Bld = CGF.Builder; in createParallelDataSharingWrapper()
3196 CGF.CreateDefaultAlignTempAlloca(CGF.VoidPtrPtrTy, "global_args"); in createParallelDataSharingWrapper()
3199 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in createParallelDataSharingWrapper()
3208 SharedArgListAddress = CGF.EmitLoadOfPointer( in createParallelDataSharingWrapper()
3209 GlobalArgs, CGF.getContext() in createParallelDataSharingWrapper()
3210 .getPointerType(CGF.getContext().VoidPtrTy) in createParallelDataSharingWrapper()
3217 Src, CGF.SizeTy->getPointerTo(), CGF.SizeTy); in createParallelDataSharingWrapper()
3218 llvm::Value *LB = CGF.EmitLoadOfScalar( in createParallelDataSharingWrapper()
3221 CGF.getContext().getPointerType(CGF.getContext().getSizeType()), in createParallelDataSharingWrapper()
3227 Src, CGF.SizeTy->getPointerTo(), CGF.SizeTy); in createParallelDataSharingWrapper()
3228 llvm::Value *UB = CGF.EmitLoadOfScalar( in createParallelDataSharingWrapper()
3231 CGF.getContext().getPointerType(CGF.getContext().getSizeType()), in createParallelDataSharingWrapper()
3237 ASTContext &CGFContext = CGF.getContext(); in createParallelDataSharingWrapper()
3242 Src, CGF.ConvertTypeForMem(CGFContext.getPointerType(ElemTy)), in createParallelDataSharingWrapper()
3243 CGF.ConvertTypeForMem(ElemTy)); in createParallelDataSharingWrapper()
3244 llvm::Value *Arg = CGF.EmitLoadOfScalar(TypedAddress, in createParallelDataSharingWrapper()
3250 Arg = castValueToType(CGF, Arg, ElemTy, CGFContext.getUIntPtrType(), in createParallelDataSharingWrapper()
3257 emitOutlinedFunctionCall(CGF, D.getBeginLoc(), OutlinedParallelFn, Args); in createParallelDataSharingWrapper()
3258 CGF.FinishFunction(); in createParallelDataSharingWrapper()
3262 void CGOpenMPRuntimeGPU::emitFunctionProlog(CodeGenFunction &CGF, in emitFunctionProlog() argument
3268 assert(FunctionGlobalizedDecls.count(CGF.CurFn) == 0 && in emitFunctionProlog()
3280 NeedToDelayGlobalization = CGF.CapturedStmtInfo->getKind() == CR_OpenMP; in emitFunctionProlog()
3287 CheckVarsEscapingDeclContext VarChecker(CGF, TeamAndReductions.second); in emitFunctionProlog()
3297 auto I = FunctionGlobalizedDecls.try_emplace(CGF.CurFn).first; in emitFunctionProlog()
3311 emitGenericVarsProlog(CGF, D->getBeginLoc(), /*WithSPMDCheck=*/true); in emitFunctionProlog()
3315 void Emit(CodeGenFunction &CGF, Flags flags) override { in emitFunctionProlog()
3316 static_cast<CGOpenMPRuntimeGPU &>(CGF.CGM.getOpenMPRuntime()) in emitFunctionProlog()
3317 .emitGenericVarsEpilog(CGF, /*WithSPMDCheck=*/true); in emitFunctionProlog()
3320 CGF.EHStack.pushCleanup<GlobalizationScope>(NormalAndEHCleanup); in emitFunctionProlog()
3324 Address CGOpenMPRuntimeGPU::getAddressOfLocalVariable(CodeGenFunction &CGF, in getAddressOfLocalVariable() argument
3352 llvm::Type *VarTy = CGF.ConvertTypeForMem(VD->getType()); in getAddressOfLocalVariable()
3362 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in getAddressOfLocalVariable()
3372 auto I = FunctionGlobalizedDecls.find(CGF.CurFn); in getAddressOfLocalVariable()
3393 void CGOpenMPRuntimeGPU::functionFinished(CodeGenFunction &CGF) { in functionFinished() argument
3394 FunctionGlobalizedDecls.erase(CGF.CurFn); in functionFinished()
3395 CGOpenMPRuntime::functionFinished(CGF); in functionFinished()
3399 CodeGenFunction &CGF, const OMPLoopDirective &S, in getDefaultDistScheduleAndChunk() argument
3402 auto &RT = static_cast<CGOpenMPRuntimeGPU &>(CGF.CGM.getOpenMPRuntime()); in getDefaultDistScheduleAndChunk()
3405 Chunk = CGF.EmitScalarConversion( in getDefaultDistScheduleAndChunk()
3406 RT.getGPUNumThreads(CGF), in getDefaultDistScheduleAndChunk()
3407 CGF.getContext().getIntTypeForBitwidth(32, /*Signed=*/0), in getDefaultDistScheduleAndChunk()
3412 CGF, S, ScheduleKind, Chunk); in getDefaultDistScheduleAndChunk()
3416 CodeGenFunction &CGF, const OMPLoopDirective &S, in getDefaultScheduleAndChunk() argument
3422 ChunkExpr = IntegerLiteral::Create(CGF.getContext(), ChunkSize, in getDefaultScheduleAndChunk()
3423 CGF.getContext().getIntTypeForBitwidth(32, /*Signed=*/0), in getDefaultScheduleAndChunk()
3428 CodeGenFunction &CGF, const OMPExecutableDirective &D) const { in adjustTargetSpecificDataForLambdas() argument
3444 Address VDAddr = CGF.GetAddrOfLocalVar(VD); in adjustTargetSpecificDataForLambdas()
3447 VDLVal = CGF.EmitLoadOfReferenceLValue(VDAddr, VD->getType()); in adjustTargetSpecificDataForLambdas()
3449 VDLVal = CGF.MakeAddrLValue( in adjustTargetSpecificDataForLambdas()
3454 if (ThisCapture && CGF.CapturedStmtInfo->isCXXThisExprCaptured()) { in adjustTargetSpecificDataForLambdas()
3456 CGF.EmitLValueForFieldInitialization(VDLVal, ThisCapture); in adjustTargetSpecificDataForLambdas()
3457 llvm::Value *CXXThis = CGF.LoadCXXThis(); in adjustTargetSpecificDataForLambdas()
3458 CGF.EmitStoreOfScalar(CXXThis, ThisLVal); in adjustTargetSpecificDataForLambdas()
3470 LValue VarLVal = CGF.EmitLValueForFieldInitialization(VDLVal, It->second); in adjustTargetSpecificDataForLambdas()
3471 Address VDAddr = CGF.GetAddrOfLocalVar(cast<VarDecl>(VD)); in adjustTargetSpecificDataForLambdas()
3473 VDAddr = CGF.EmitLoadOfReferenceLValue(VDAddr, in adjustTargetSpecificDataForLambdas()
3475 .getAddress(CGF); in adjustTargetSpecificDataForLambdas()
3476 CGF.EmitStoreOfScalar(VDAddr.getPointer(), VarLVal); in adjustTargetSpecificDataForLambdas()
3645 llvm::Value *CGOpenMPRuntimeGPU::getGPUNumThreads(CodeGenFunction &CGF) { in getGPUNumThreads() argument
3646 CGBuilderTy &Bld = CGF.Builder; in getGPUNumThreads()
3647 llvm::Module *M = &CGF.CGM.getModule(); in getGPUNumThreads()
3652 llvm::FunctionType::get(CGF.Int32Ty, std::nullopt, false), in getGPUNumThreads()
3653 llvm::GlobalVariable::ExternalLinkage, LocSize, &CGF.CGM.getModule()); in getGPUNumThreads()
3658 llvm::Value *CGOpenMPRuntimeGPU::getGPUThreadID(CodeGenFunction &CGF) { in getGPUThreadID() argument
3660 return CGF.EmitRuntimeCall( in getGPUThreadID()
3666 llvm::Value *CGOpenMPRuntimeGPU::getGPUWarpSize(CodeGenFunction &CGF) { in getGPUWarpSize() argument
3668 return CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in getGPUWarpSize()