Lines Matching refs:CGF
47 void Enter(CodeGenFunction &CGF) override { in Enter() argument
48 llvm::Value *EnterRes = CGF.EmitRuntimeCall(EnterCallee, EnterArgs); in Enter()
50 llvm::Value *CallBool = CGF.Builder.CreateIsNotNull(EnterRes); in Enter()
51 auto *ThenBlock = CGF.createBasicBlock("omp_if.then"); in Enter()
52 ContBlock = CGF.createBasicBlock("omp_if.end"); in Enter()
54 CGF.Builder.CreateCondBr(CallBool, ThenBlock, ContBlock); in Enter()
55 CGF.EmitBlock(ThenBlock); in Enter()
58 void Done(CodeGenFunction &CGF) { in Done() argument
60 CGF.EmitBranch(ContBlock); in Done()
61 CGF.EmitBlock(ContBlock, true); in Done()
63 void Exit(CodeGenFunction &CGF) override { in Exit() argument
64 CGF.EmitRuntimeCall(ExitCallee, ExitArgs); in Exit()
224 CodeGenFunction &CGF; member in __anonacc9eed20111::CheckVarsEscapingDeclContext
243 if (auto *CSI = CGF.CapturedStmtInfo) { in markAsEscaped()
268 if ((!CGF.CapturedStmtInfo || in markAsEscaped()
269 (IsForCombinedParallelRegion && CGF.CapturedStmtInfo)) && in markAsEscaped()
342 unsigned WarpSize = CGF.getTarget().getGridValue(llvm::omp::GV_Warp_Size); in buildRecordForGlobalizedVars()
348 CGF.getContext(), EscapedDeclsForParallel, EscapedDeclsForTeams, in buildRecordForGlobalizedVars()
353 CheckVarsEscapingDeclContext(CodeGenFunction &CGF, in CheckVarsEscapingDeclContext() argument
355 : CGF(CGF), EscapedDecls(TeamsReductions.begin(), TeamsReductions.end()) { in CheckVarsEscapingDeclContext()
536 static llvm::Value *getNVPTXWarpID(CodeGenFunction &CGF) { in getNVPTXWarpID() argument
537 CGBuilderTy &Bld = CGF.Builder; in getNVPTXWarpID()
539 CGF.getTarget().getGridValue(llvm::omp::GV_Warp_Size_Log2); in getNVPTXWarpID()
540 auto &RT = static_cast<CGOpenMPRuntimeGPU &>(CGF.CGM.getOpenMPRuntime()); in getNVPTXWarpID()
541 return Bld.CreateAShr(RT.getGPUThreadID(CGF), LaneIDBits, "nvptx_warp_id"); in getNVPTXWarpID()
547 static llvm::Value *getNVPTXLaneID(CodeGenFunction &CGF) { in getNVPTXLaneID() argument
548 CGBuilderTy &Bld = CGF.Builder; in getNVPTXLaneID()
549 unsigned LaneIDMask = CGF.getContext().getTargetInfo().getGridValue( in getNVPTXLaneID()
551 auto &RT = static_cast<CGOpenMPRuntimeGPU &>(CGF.CGM.getOpenMPRuntime()); in getNVPTXLaneID()
552 return Bld.CreateAnd(RT.getGPUThreadID(CGF), Bld.getInt32(LaneIDMask), in getNVPTXLaneID()
561 static llvm::Value *getThreadLimit(CodeGenFunction &CGF, in getThreadLimit() argument
563 CGBuilderTy &Bld = CGF.Builder; in getThreadLimit()
564 auto &RT = static_cast<CGOpenMPRuntimeGPU &>(CGF.CGM.getOpenMPRuntime()); in getThreadLimit()
567 ThreadLimit = RT.getGPUNumThreads(CGF); in getThreadLimit()
569 llvm::Value *GPUNumThreads = RT.getGPUNumThreads(CGF); in getThreadLimit()
570 llvm::Value *GPUWarpSize = RT.getGPUWarpSize(CGF); in getThreadLimit()
584 static llvm::Value *getMasterThreadID(CodeGenFunction &CGF) { in getMasterThreadID() argument
585 CGBuilderTy &Bld = CGF.Builder; in getMasterThreadID()
586 auto &RT = static_cast<CGOpenMPRuntimeGPU &>(CGF.CGM.getOpenMPRuntime()); in getMasterThreadID()
587 llvm::Value *NumThreads = RT.getGPUNumThreads(CGF); in getMasterThreadID()
589 llvm::Value *Mask = Bld.CreateNUWSub(RT.getGPUWarpSize(CGF), Bld.getInt32(1)); in getMasterThreadID()
1089 void Enter(CodeGenFunction &CGF) override { in emitNonSPMDKernel() argument
1091 static_cast<CGOpenMPRuntimeGPU &>(CGF.CGM.getOpenMPRuntime()); in emitNonSPMDKernel()
1092 RT.emitNonSPMDEntryHeader(CGF, EST, WST); in emitNonSPMDKernel()
1094 RT.setLocThreadIdInsertPt(CGF, /*AtCurrentPoint=*/true); in emitNonSPMDKernel()
1096 void Exit(CodeGenFunction &CGF) override { in emitNonSPMDKernel() argument
1098 static_cast<CGOpenMPRuntimeGPU &>(CGF.CGM.getOpenMPRuntime()); in emitNonSPMDKernel()
1099 RT.clearLocThreadIdInsertPt(CGF); in emitNonSPMDKernel()
1100 RT.emitNonSPMDEntryFooter(CGF, EST); in emitNonSPMDKernel()
1129 void CGOpenMPRuntimeGPU::emitNonSPMDEntryHeader(CodeGenFunction &CGF, in emitNonSPMDEntryHeader() argument
1132 CGBuilderTy &Bld = CGF.Builder; in emitNonSPMDEntryHeader()
1134 llvm::BasicBlock *WorkerBB = CGF.createBasicBlock(".worker"); in emitNonSPMDEntryHeader()
1135 llvm::BasicBlock *MasterCheckBB = CGF.createBasicBlock(".mastercheck"); in emitNonSPMDEntryHeader()
1136 llvm::BasicBlock *MasterBB = CGF.createBasicBlock(".master"); in emitNonSPMDEntryHeader()
1137 EST.ExitBB = CGF.createBasicBlock(".exit"); in emitNonSPMDEntryHeader()
1139 auto &RT = static_cast<CGOpenMPRuntimeGPU &>(CGF.CGM.getOpenMPRuntime()); in emitNonSPMDEntryHeader()
1140 llvm::Value *GPUThreadID = RT.getGPUThreadID(CGF); in emitNonSPMDEntryHeader()
1141 llvm::Value *ThreadLimit = getThreadLimit(CGF); in emitNonSPMDEntryHeader()
1145 CGF.EmitBlock(WorkerBB); in emitNonSPMDEntryHeader()
1146 emitCall(CGF, WST.Loc, WST.WorkerFn); in emitNonSPMDEntryHeader()
1147 CGF.EmitBranch(EST.ExitBB); in emitNonSPMDEntryHeader()
1149 CGF.EmitBlock(MasterCheckBB); in emitNonSPMDEntryHeader()
1150 GPUThreadID = RT.getGPUThreadID(CGF); in emitNonSPMDEntryHeader()
1151 llvm::Value *MasterThreadID = getMasterThreadID(CGF); in emitNonSPMDEntryHeader()
1155 CGF.EmitBlock(MasterBB); in emitNonSPMDEntryHeader()
1161 llvm::Value *Args[] = {getThreadLimit(CGF), in emitNonSPMDEntryHeader()
1163 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in emitNonSPMDEntryHeader()
1168 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in emitNonSPMDEntryHeader()
1171 emitGenericVarsProlog(CGF, WST.Loc); in emitNonSPMDEntryHeader()
1174 void CGOpenMPRuntimeGPU::emitNonSPMDEntryFooter(CodeGenFunction &CGF, in emitNonSPMDEntryFooter() argument
1177 if (!CGF.HaveInsertPoint()) in emitNonSPMDEntryFooter()
1180 emitGenericVarsEpilog(CGF); in emitNonSPMDEntryFooter()
1183 EST.ExitBB = CGF.createBasicBlock(".exit"); in emitNonSPMDEntryFooter()
1185 llvm::BasicBlock *TerminateBB = CGF.createBasicBlock(".termination.notifier"); in emitNonSPMDEntryFooter()
1186 CGF.EmitBranch(TerminateBB); in emitNonSPMDEntryFooter()
1188 CGF.EmitBlock(TerminateBB); in emitNonSPMDEntryFooter()
1191 llvm::Value *Args[] = {CGF.Builder.getInt16(/*IsOMPRuntimeInitialized=*/1)}; in emitNonSPMDEntryFooter()
1192 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in emitNonSPMDEntryFooter()
1196 syncCTAThreads(CGF); in emitNonSPMDEntryFooter()
1198 CGF.EmitBranch(EST.ExitBB); in emitNonSPMDEntryFooter()
1200 CGF.EmitBlock(EST.ExitBB); in emitNonSPMDEntryFooter()
1227 void Enter(CodeGenFunction &CGF) override { in emitSPMDKernel() argument
1228 RT.emitSPMDEntryHeader(CGF, EST, D); in emitSPMDKernel()
1230 RT.setLocThreadIdInsertPt(CGF, /*AtCurrentPoint=*/true); in emitSPMDKernel()
1232 void Exit(CodeGenFunction &CGF) override { in emitSPMDKernel() argument
1233 RT.clearLocThreadIdInsertPt(CGF); in emitSPMDKernel()
1234 RT.emitSPMDEntryFooter(CGF, EST); in emitSPMDKernel()
1256 CodeGenFunction &CGF, EntryFunctionState &EST, in emitSPMDEntryHeader() argument
1258 CGBuilderTy &Bld = CGF.Builder; in emitSPMDEntryHeader()
1261 llvm::BasicBlock *ExecuteBB = CGF.createBasicBlock(".execute"); in emitSPMDEntryHeader()
1262 EST.ExitBB = CGF.createBasicBlock(".exit"); in emitSPMDEntryHeader()
1264 llvm::Value *Args[] = {getThreadLimit(CGF, /*IsInSPMDExecutionMode=*/true), in emitSPMDEntryHeader()
1267 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in emitSPMDEntryHeader()
1273 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in emitSPMDEntryHeader()
1277 CGF.EmitBranch(ExecuteBB); in emitSPMDEntryHeader()
1279 CGF.EmitBlock(ExecuteBB); in emitSPMDEntryHeader()
1284 void CGOpenMPRuntimeGPU::emitSPMDEntryFooter(CodeGenFunction &CGF, in emitSPMDEntryFooter() argument
1287 if (!CGF.HaveInsertPoint()) in emitSPMDEntryFooter()
1291 EST.ExitBB = CGF.createBasicBlock(".exit"); in emitSPMDEntryFooter()
1293 llvm::BasicBlock *OMPDeInitBB = CGF.createBasicBlock(".omp.deinit"); in emitSPMDEntryFooter()
1294 CGF.EmitBranch(OMPDeInitBB); in emitSPMDEntryFooter()
1296 CGF.EmitBlock(OMPDeInitBB); in emitSPMDEntryFooter()
1299 CGF.Builder.getInt16(RequiresFullRuntime ? 1 : 0)}; in emitSPMDEntryFooter()
1300 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in emitSPMDEntryFooter()
1303 CGF.EmitBranch(EST.ExitBB); in emitSPMDEntryFooter()
1305 CGF.EmitBlock(EST.ExitBB); in emitSPMDEntryFooter()
1328 CodeGenFunction CGF(CGM, /*suppressNewContext=*/true); in emitWorkerFunction() local
1329 CGF.StartFunction(GlobalDecl(), Ctx.VoidTy, WST.WorkerFn, WST.CGFI, {}, in emitWorkerFunction()
1331 emitWorkerLoop(CGF, WST); in emitWorkerFunction()
1332 CGF.FinishFunction(); in emitWorkerFunction()
1335 void CGOpenMPRuntimeGPU::emitWorkerLoop(CodeGenFunction &CGF, in emitWorkerLoop() argument
1346 CGBuilderTy &Bld = CGF.Builder; in emitWorkerLoop()
1348 llvm::BasicBlock *AwaitBB = CGF.createBasicBlock(".await.work"); in emitWorkerLoop()
1349 llvm::BasicBlock *SelectWorkersBB = CGF.createBasicBlock(".select.workers"); in emitWorkerLoop()
1350 llvm::BasicBlock *ExecuteBB = CGF.createBasicBlock(".execute.parallel"); in emitWorkerLoop()
1351 llvm::BasicBlock *TerminateBB = CGF.createBasicBlock(".terminate.parallel"); in emitWorkerLoop()
1352 llvm::BasicBlock *BarrierBB = CGF.createBasicBlock(".barrier.parallel"); in emitWorkerLoop()
1353 llvm::BasicBlock *ExitBB = CGF.createBasicBlock(".exit"); in emitWorkerLoop()
1355 CGF.EmitBranch(AwaitBB); in emitWorkerLoop()
1358 CGF.EmitBlock(AwaitBB); in emitWorkerLoop()
1360 syncCTAThreads(CGF); in emitWorkerLoop()
1363 CGF.CreateDefaultAlignTempAlloca(CGF.Int8PtrTy, /*Name=*/"work_fn"); in emitWorkerLoop()
1365 CGF.CreateDefaultAlignTempAlloca(CGF.Int8Ty, /*Name=*/"exec_status"); in emitWorkerLoop()
1366 CGF.InitTempAlloca(ExecStatus, Bld.getInt8(/*C=*/0)); in emitWorkerLoop()
1367 CGF.InitTempAlloca(WorkFn, llvm::Constant::getNullValue(CGF.Int8PtrTy)); in emitWorkerLoop()
1372 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in emitWorkerLoop()
1375 Bld.CreateStore(Bld.CreateZExt(Ret, CGF.Int8Ty), ExecStatus); in emitWorkerLoop()
1383 CGF.EmitBlock(SelectWorkersBB); in emitWorkerLoop()
1389 CGF.EmitBlock(ExecuteBB); in emitWorkerLoop()
1391 setLocThreadIdInsertPt(CGF, /*AtCurrentPoint=*/true); in emitWorkerLoop()
1401 llvm::BasicBlock *ExecuteFNBB = CGF.createBasicBlock(".execute.fn"); in emitWorkerLoop()
1402 llvm::BasicBlock *CheckNextBB = CGF.createBasicBlock(".check.next"); in emitWorkerLoop()
1406 CGF.EmitBlock(ExecuteFNBB); in emitWorkerLoop()
1412 emitCall(CGF, WST.Loc, W, in emitWorkerLoop()
1413 {Bld.getInt16(/*ParallelLevel=*/0), getThreadID(CGF, WST.Loc)}); in emitWorkerLoop()
1416 CGF.EmitBranch(TerminateBB); in emitWorkerLoop()
1418 CGF.EmitBlock(CheckNextBB); in emitWorkerLoop()
1432 emitCall(CGF, WST.Loc, {ParallelFnTy, WorkFnCast}, in emitWorkerLoop()
1433 {Bld.getInt16(/*ParallelLevel=*/0), getThreadID(CGF, WST.Loc)}); in emitWorkerLoop()
1435 CGF.EmitBranch(TerminateBB); in emitWorkerLoop()
1438 CGF.EmitBlock(TerminateBB); in emitWorkerLoop()
1439 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in emitWorkerLoop()
1442 CGF.EmitBranch(BarrierBB); in emitWorkerLoop()
1445 CGF.EmitBlock(BarrierBB); in emitWorkerLoop()
1447 syncCTAThreads(CGF); in emitWorkerLoop()
1448 CGF.EmitBranch(AwaitBB); in emitWorkerLoop()
1451 CGF.EmitBlock(ExitBB); in emitWorkerLoop()
1453 clearLocThreadIdInsertPt(CGF); in emitWorkerLoop()
1535 void CGOpenMPRuntimeGPU::emitProcBindClause(CodeGenFunction &CGF, in emitProcBindClause() argument
1542 CGOpenMPRuntime::emitProcBindClause(CGF, ProcBind, Loc); in emitProcBindClause()
1545 void CGOpenMPRuntimeGPU::emitNumThreadsClause(CodeGenFunction &CGF, in emitNumThreadsClause() argument
1552 CGOpenMPRuntime::emitNumThreadsClause(CGF, NumThreads, Loc); in emitNumThreadsClause()
1555 void CGOpenMPRuntimeGPU::emitNumTeamsClause(CodeGenFunction &CGF, in emitNumTeamsClause() argument
1571 void Enter(CodeGenFunction &CGF) override { in emitParallelOutlinedFunction() argument
1575 void Exit(CodeGenFunction &CGF) override { in emitParallelOutlinedFunction() argument
1677 void Enter(CodeGenFunction &CGF) override { in emitTeamsOutlinedFunction() argument
1679 static_cast<CGOpenMPRuntimeGPU &>(CGF.CGM.getOpenMPRuntime()); in emitTeamsOutlinedFunction()
1681 auto I = Rt.FunctionGlobalizedDecls.try_emplace(CGF.CurFn).first; in emitTeamsOutlinedFunction()
1694 Rt.emitGenericVarsProlog(CGF, Loc); in emitTeamsOutlinedFunction()
1696 void Exit(CodeGenFunction &CGF) override { in emitTeamsOutlinedFunction() argument
1697 static_cast<CGOpenMPRuntimeGPU &>(CGF.CGM.getOpenMPRuntime()) in emitTeamsOutlinedFunction()
1698 .emitGenericVarsEpilog(CGF); in emitTeamsOutlinedFunction()
1708 void CGOpenMPRuntimeGPU::emitGenericVarsProlog(CodeGenFunction &CGF, in emitGenericVarsProlog() argument
1715 CGBuilderTy &Bld = CGF.Builder; in emitGenericVarsProlog()
1717 const auto I = FunctionGlobalizedDecls.find(CGF.CurFn); in emitGenericVarsProlog()
1735 CGF.ConvertTypeForMem(GlobalRecTy)->getPointerTo(); in emitGenericVarsProlog()
1741 llvm::BasicBlock *ExitBB = CGF.createBasicBlock(".exit"); in emitGenericVarsProlog()
1742 llvm::BasicBlock *SPMDBB = CGF.createBasicBlock(".spmd"); in emitGenericVarsProlog()
1743 llvm::BasicBlock *NonSPMDBB = CGF.createBasicBlock(".non-spmd"); in emitGenericVarsProlog()
1745 llvm::Value *RTLoc = emitUpdateLocation(CGF, Loc); in emitGenericVarsProlog()
1746 llvm::Value *ThreadID = getThreadID(CGF, Loc); in emitGenericVarsProlog()
1747 llvm::Value *PL = CGF.EmitRuntimeCall( in emitGenericVarsProlog()
1754 CGF.EmitNounwindRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in emitGenericVarsProlog()
1758 (void)ApplyDebugLocation::CreateEmpty(CGF); in emitGenericVarsProlog()
1759 CGF.EmitBlock(SPMDBB); in emitGenericVarsProlog()
1762 CGF.EmitBranch(ExitBB); in emitGenericVarsProlog()
1764 (void)ApplyDebugLocation::CreateEmpty(CGF); in emitGenericVarsProlog()
1765 CGF.EmitBlock(NonSPMDBB); in emitGenericVarsProlog()
1787 Size, CGF.Builder.getInt16(/*UseSharedMemory=*/0)}; in emitGenericVarsProlog()
1788 llvm::Value *GlobalRecValue = CGF.EmitRuntimeCall( in emitGenericVarsProlog()
1794 CGF.EmitBlock(ExitBB); in emitGenericVarsProlog()
1830 llvm::Value *IsInSharedMemory = CGF.EmitLoadOfScalar( in emitGenericVarsProlog()
1842 llvm::Value *Ld = CGF.EmitLoadOfScalar( in emitGenericVarsProlog()
1852 CGF.EmitRuntimeCall( in emitGenericVarsProlog()
1862 Address FrameAddr = CGF.EmitLoadOfPointer( in emitGenericVarsProlog()
1872 GlobalRecValue, CGF.ConvertTypeForMem(GlobalRecTy)->getPointerTo()); in emitGenericVarsProlog()
1880 CGF.Builder.getInt16(UseSharedMemory ? 1 : 0)}; in emitGenericVarsProlog()
1881 llvm::Value *GlobalRecValue = CGF.EmitRuntimeCall( in emitGenericVarsProlog()
1893 CGF.MakeNaturalAlignPointeeAddrLValue(GlobalRecCastAddr, GlobalRecTy); in emitGenericVarsProlog()
1902 CGF.ConvertTypeForMem(SecGlobalRecTy)->getPointerTo(); in emitGenericVarsProlog()
1903 SecBase = CGF.MakeNaturalAlignPointeeAddrLValue( in emitGenericVarsProlog()
1914 CGF.MakeAddrLValue(CGF.GetAddrOfLocalVar(VD), VD->getType()); in emitGenericVarsProlog()
1915 ParValue = CGF.EmitLoadOfScalar(ParLVal, Loc); in emitGenericVarsProlog()
1917 LValue VarAddr = CGF.EmitLValueForField(Base, Rec.second.FD); in emitGenericVarsProlog()
1923 Address Addr = VarAddr.getAddress(CGF); in emitGenericVarsProlog()
1924 llvm::Value *Ptr = CGF.Builder.CreateInBoundsGEP( in emitGenericVarsProlog()
1926 {Bld.getInt32(0), getNVPTXLaneID(CGF)}); in emitGenericVarsProlog()
1929 VarAddr = CGF.MakeAddrLValue( in emitGenericVarsProlog()
1933 Rec.second.PrivateAddr = VarAddr.getAddress(CGF); in emitGenericVarsProlog()
1942 LValue SecVarAddr = CGF.EmitLValueForField(SecBase, SecIt->second.FD); in emitGenericVarsProlog()
1944 Address(Bld.CreateSelect(IsTTD, SecVarAddr.getPointer(CGF), in emitGenericVarsProlog()
1945 VarAddr.getPointer(CGF)), in emitGenericVarsProlog()
1947 Rec.second.PrivateAddr = VarAddr.getAddress(CGF); in emitGenericVarsProlog()
1950 Address LocalAddr = CGF.CreateMemTemp(VarTy, Rec.second.FD->getName()); in emitGenericVarsProlog()
1958 CGF.EmitStoreOfScalar(ParValue, VarAddr); in emitGenericVarsProlog()
1959 I->getSecond().MappedParams->setVarAddr(CGF, VD, in emitGenericVarsProlog()
1960 VarAddr.getAddress(CGF)); in emitGenericVarsProlog()
1971 CGBuilderTy &Bld = CGF.Builder; in emitGenericVarsProlog()
1972 llvm::Value *Size = CGF.getTypeSize(VD->getType()); in emitGenericVarsProlog()
1975 Size, llvm::ConstantInt::get(CGF.SizeTy, Align.getQuantity() - 1)); in emitGenericVarsProlog()
1977 llvm::ConstantInt::get(CGF.SizeTy, Align.getQuantity()); in emitGenericVarsProlog()
1983 Size, CGF.Builder.getInt16(/*UseSharedMemory=*/0)}; in emitGenericVarsProlog()
1984 llvm::Value *GlobalRecValue = CGF.EmitRuntimeCall( in emitGenericVarsProlog()
1989 GlobalRecValue, CGF.ConvertTypeForMem(VD->getType())->getPointerTo()); in emitGenericVarsProlog()
1990 LValue Base = CGF.MakeAddrLValue(GlobalRecCastAddr, VD->getType(), in emitGenericVarsProlog()
1993 I->getSecond().MappedParams->setVarAddr(CGF, cast<VarDecl>(VD), in emitGenericVarsProlog()
1994 Base.getAddress(CGF)); in emitGenericVarsProlog()
1997 I->getSecond().MappedParams->apply(CGF); in emitGenericVarsProlog()
2000 void CGOpenMPRuntimeGPU::emitGenericVarsEpilog(CodeGenFunction &CGF, in emitGenericVarsEpilog() argument
2006 const auto I = FunctionGlobalizedDecls.find(CGF.CurFn); in emitGenericVarsEpilog()
2008 I->getSecond().MappedParams->restore(CGF); in emitGenericVarsEpilog()
2009 if (!CGF.HaveInsertPoint()) in emitGenericVarsEpilog()
2013 CGF.EmitRuntimeCall( in emitGenericVarsEpilog()
2022 CGBuilderTy &Bld = CGF.Builder; in emitGenericVarsEpilog()
2023 llvm::BasicBlock *ExitBB = CGF.createBasicBlock(".exit"); in emitGenericVarsEpilog()
2024 llvm::BasicBlock *NonSPMDBB = CGF.createBasicBlock(".non-spmd"); in emitGenericVarsEpilog()
2027 (void)ApplyDebugLocation::CreateEmpty(CGF); in emitGenericVarsEpilog()
2028 CGF.EmitBlock(NonSPMDBB); in emitGenericVarsEpilog()
2029 CGF.EmitRuntimeCall( in emitGenericVarsEpilog()
2032 CGF.EmitCastToVoidPtr(I->getSecond().GlobalRecordAddr)); in emitGenericVarsEpilog()
2033 CGF.EmitBlock(ExitBB); in emitGenericVarsEpilog()
2042 llvm::Value *IsInSharedMemory = CGF.EmitLoadOfScalar( in emitGenericVarsEpilog()
2051 CGF.EmitRuntimeCall( in emitGenericVarsEpilog()
2057 CGF.EmitRuntimeCall( in emitGenericVarsEpilog()
2066 void CGOpenMPRuntimeGPU::emitTeamsCall(CodeGenFunction &CGF, in emitTeamsCall() argument
2071 if (!CGF.HaveInsertPoint()) in emitTeamsCall()
2074 Address ZeroAddr = CGF.CreateDefaultAlignTempAlloca(CGF.Int32Ty, in emitTeamsCall()
2076 CGF.InitTempAlloca(ZeroAddr, CGF.Builder.getInt32(/*C*/ 0)); in emitTeamsCall()
2078 OutlinedFnArgs.push_back(emitThreadIDAddress(CGF, Loc).getPointer()); in emitTeamsCall()
2081 emitOutlinedFunctionCall(CGF, Loc, OutlinedFn, OutlinedFnArgs); in emitTeamsCall()
2084 void CGOpenMPRuntimeGPU::emitParallelCall(CodeGenFunction &CGF, in emitParallelCall() argument
2089 if (!CGF.HaveInsertPoint()) in emitParallelCall()
2093 IfCond](CodeGenFunction &CGF, PrePostActionTy &Action) { in emitParallelCall() argument
2094 CGBuilderTy &Bld = CGF.Builder; in emitParallelCall()
2107 CodeGenFunction::OMPPrivateScope PrivateArgScope(CGF); in emitParallelCall()
2109 Address CapturedVarsAddrs = CGF.CreateDefaultAlignTempAlloca( in emitParallelCall()
2115 ASTContext &Ctx = CGF.getContext(); in emitParallelCall()
2121 PtrV = Bld.CreateIntToPtr(V, CGF.VoidPtrTy); in emitParallelCall()
2123 PtrV = Bld.CreatePointerBitCastOrAddrSpaceCast(V, CGF.VoidPtrTy); in emitParallelCall()
2124 CGF.EmitStoreOfScalar(PtrV, Dst, /*Volatile=*/false, in emitParallelCall()
2132 IfCondVal = Bld.CreateIntCast(CGF.EvaluateExprAsBool(IfCond), CGF.Int32Ty, in emitParallelCall()
2135 IfCondVal = llvm::ConstantInt::get(CGF.Int32Ty, 1); in emitParallelCall()
2138 llvm::Value *RTLoc = emitUpdateLocation(CGF, Loc); in emitParallelCall()
2141 getThreadID(CGF, Loc), in emitParallelCall()
2143 llvm::ConstantInt::get(CGF.Int32Ty, -1), in emitParallelCall()
2144 llvm::ConstantInt::get(CGF.Int32Ty, -1), in emitParallelCall()
2148 CGF.VoidPtrPtrTy), in emitParallelCall()
2150 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in emitParallelCall()
2156 RCG(CGF); in emitParallelCall()
2159 void CGOpenMPRuntimeGPU::syncCTAThreads(CodeGenFunction &CGF) { in syncCTAThreads() argument
2161 if (!CGF.HaveInsertPoint()) in syncCTAThreads()
2168 llvm::ConstantInt::get(CGF.Int32Ty, /*V=*/0, /*isSigned=*/true)}; in syncCTAThreads()
2169 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in syncCTAThreads()
2174 void CGOpenMPRuntimeGPU::emitBarrierCall(CodeGenFunction &CGF, in emitBarrierCall() argument
2179 if (!CGF.HaveInsertPoint()) in emitBarrierCall()
2183 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc, Flags), in emitBarrierCall()
2184 getThreadID(CGF, Loc)}; in emitBarrierCall()
2186 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in emitBarrierCall()
2192 CodeGenFunction &CGF, StringRef CriticalName, in emitCriticalRegion() argument
2195 llvm::BasicBlock *LoopBB = CGF.createBasicBlock("omp.critical.loop"); in emitCriticalRegion()
2196 llvm::BasicBlock *TestBB = CGF.createBasicBlock("omp.critical.test"); in emitCriticalRegion()
2197 llvm::BasicBlock *SyncBB = CGF.createBasicBlock("omp.critical.sync"); in emitCriticalRegion()
2198 llvm::BasicBlock *BodyBB = CGF.createBasicBlock("omp.critical.body"); in emitCriticalRegion()
2199 llvm::BasicBlock *ExitBB = CGF.createBasicBlock("omp.critical.exit"); in emitCriticalRegion()
2201 auto &RT = static_cast<CGOpenMPRuntimeGPU &>(CGF.CGM.getOpenMPRuntime()); in emitCriticalRegion()
2204 llvm::Value *Mask = CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in emitCriticalRegion()
2207 llvm::Value *ThreadID = RT.getGPUThreadID(CGF); in emitCriticalRegion()
2210 llvm::Value *TeamWidth = RT.getGPUNumThreads(CGF); in emitCriticalRegion()
2214 CGF.getContext().getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/0); in emitCriticalRegion()
2215 Address Counter = CGF.CreateMemTemp(Int32Ty, "critical_counter"); in emitCriticalRegion()
2216 LValue CounterLVal = CGF.MakeAddrLValue(Counter, Int32Ty); in emitCriticalRegion()
2217 CGF.EmitStoreOfScalar(llvm::Constant::getNullValue(CGM.Int32Ty), CounterLVal, in emitCriticalRegion()
2221 CGF.EmitBlock(LoopBB); in emitCriticalRegion()
2222 llvm::Value *CounterVal = CGF.EmitLoadOfScalar(CounterLVal, Loc); in emitCriticalRegion()
2223 llvm::Value *CmpLoopBound = CGF.Builder.CreateICmpSLT(CounterVal, TeamWidth); in emitCriticalRegion()
2224 CGF.Builder.CreateCondBr(CmpLoopBound, TestBB, ExitBB); in emitCriticalRegion()
2228 CGF.EmitBlock(TestBB); in emitCriticalRegion()
2229 CounterVal = CGF.EmitLoadOfScalar(CounterLVal, Loc); in emitCriticalRegion()
2231 CGF.Builder.CreateICmpEQ(ThreadID, CounterVal); in emitCriticalRegion()
2232 CGF.Builder.CreateCondBr(CmpThreadToCounter, BodyBB, SyncBB); in emitCriticalRegion()
2235 CGF.EmitBlock(BodyBB); in emitCriticalRegion()
2238 CGOpenMPRuntime::emitCriticalRegion(CGF, CriticalName, CriticalOpGen, Loc, in emitCriticalRegion()
2245 CGF.EmitBlock(SyncBB); in emitCriticalRegion()
2247 (void)CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in emitCriticalRegion()
2252 CGF.Builder.CreateNSWAdd(CounterVal, CGF.Builder.getInt32(1)); in emitCriticalRegion()
2253 CGF.EmitStoreOfScalar(IncCounterVal, CounterLVal); in emitCriticalRegion()
2254 CGF.EmitBranch(LoopBB); in emitCriticalRegion()
2257 CGF.EmitBlock(ExitBB, /*IsFinished=*/true); in emitCriticalRegion()
2261 static llvm::Value *castValueToType(CodeGenFunction &CGF, llvm::Value *Val, in castValueToType() argument
2264 assert(!CGF.getContext().getTypeSizeInChars(CastTy).isZero() && in castValueToType()
2266 assert(!CGF.getContext().getTypeSizeInChars(ValTy).isZero() && in castValueToType()
2268 llvm::Type *LLVMCastTy = CGF.ConvertTypeForMem(CastTy); in castValueToType()
2271 if (CGF.getContext().getTypeSizeInChars(ValTy) == in castValueToType()
2272 CGF.getContext().getTypeSizeInChars(CastTy)) in castValueToType()
2273 return CGF.Builder.CreateBitCast(Val, LLVMCastTy); in castValueToType()
2275 return CGF.Builder.CreateIntCast(Val, LLVMCastTy, in castValueToType()
2277 Address CastItem = CGF.CreateMemTemp(CastTy); in castValueToType()
2278 Address ValCastItem = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in castValueToType()
2280 CGF.EmitStoreOfScalar(Val, ValCastItem, /*Volatile=*/false, ValTy, in castValueToType()
2283 return CGF.EmitLoadOfScalar(CastItem, /*Volatile=*/false, CastTy, Loc, in castValueToType()
2290 static llvm::Value *createRuntimeShuffleFunction(CodeGenFunction &CGF, in createRuntimeShuffleFunction() argument
2295 CodeGenModule &CGM = CGF.CGM; in createRuntimeShuffleFunction()
2296 CGBuilderTy &Bld = CGF.Builder; in createRuntimeShuffleFunction()
2301 CharUnits Size = CGF.getContext().getTypeSizeInChars(ElemType); in createRuntimeShuffleFunction()
2310 QualType CastTy = CGF.getContext().getIntTypeForBitwidth( in createRuntimeShuffleFunction()
2312 llvm::Value *ElemCast = castValueToType(CGF, Elem, ElemType, CastTy, Loc); in createRuntimeShuffleFunction()
2314 Bld.CreateIntCast(RT.getGPUWarpSize(CGF), CGM.Int16Ty, /*isSigned=*/true); in createRuntimeShuffleFunction()
2316 llvm::Value *ShuffledVal = CGF.EmitRuntimeCall( in createRuntimeShuffleFunction()
2320 return castValueToType(CGF, ShuffledVal, CastTy, ElemType, Loc); in createRuntimeShuffleFunction()
2323 static void shuffleAndStore(CodeGenFunction &CGF, Address SrcAddr, in shuffleAndStore() argument
2326 CGBuilderTy &Bld = CGF.Builder; in shuffleAndStore()
2328 CharUnits Size = CGF.getContext().getTypeSizeInChars(ElemType); in shuffleAndStore()
2342 Bld.CreateConstGEP(SrcAddr, 1), CGF.VoidPtrTy); in shuffleAndStore()
2346 QualType IntType = CGF.getContext().getIntTypeForBitwidth( in shuffleAndStore()
2347 CGF.getContext().toBits(CharUnits::fromQuantity(IntSize)), in shuffleAndStore()
2349 llvm::Type *IntTy = CGF.ConvertTypeForMem(IntType); in shuffleAndStore()
2354 llvm::BasicBlock *PreCondBB = CGF.createBasicBlock(".shuffle.pre_cond"); in shuffleAndStore()
2355 llvm::BasicBlock *ThenBB = CGF.createBasicBlock(".shuffle.then"); in shuffleAndStore()
2356 llvm::BasicBlock *ExitBB = CGF.createBasicBlock(".shuffle.exit"); in shuffleAndStore()
2358 CGF.EmitBlock(PreCondBB); in shuffleAndStore()
2369 Ptr.getPointer(), CGF.VoidPtrTy)); in shuffleAndStore()
2372 CGF.EmitBlock(ThenBB); in shuffleAndStore()
2374 CGF, in shuffleAndStore()
2375 CGF.EmitLoadOfScalar(Ptr, /*Volatile=*/false, IntType, Loc, in shuffleAndStore()
2379 CGF.EmitStoreOfScalar(Res, ElemPtr, /*Volatile=*/false, IntType, in shuffleAndStore()
2386 CGF.EmitBranch(PreCondBB); in shuffleAndStore()
2387 CGF.EmitBlock(ExitBB); in shuffleAndStore()
2390 CGF, in shuffleAndStore()
2391 CGF.EmitLoadOfScalar(Ptr, /*Volatile=*/false, IntType, Loc, in shuffleAndStore()
2395 CGF.EmitStoreOfScalar(Res, ElemPtr, /*Volatile=*/false, IntType, in shuffleAndStore()
2429 CopyAction Action, CodeGenFunction &CGF, QualType ReductionArrayTy, in emitReductionListCopy() argument
2433 CodeGenModule &CGM = CGF.CGM;
2435 CGBuilderTy &Bld = CGF.Builder;
2463 SrcElementAddr = CGF.EmitLoadOfPointer(
2471 CGF.CreateMemTemp(Private->getType(), ".omp.reduction.element");
2479 SrcElementAddr = CGF.EmitLoadOfPointer(
2486 DestElementAddr = CGF.EmitLoadOfPointer(
2494 SrcElementAddr = CGF.EmitLoadOfPointer(
2500 llvm::Value *ElementSizeInChars = CGF.getTypeSize(Private->getType());
2506 Bld.CreateIntToPtr(ScratchPadElemAbsolutePtrVal, CGF.VoidPtrTy);
2515 llvm::Value *ElementSizeInChars = CGF.getTypeSize(Private->getType());
2521 Bld.CreateIntToPtr(ScratchPadElemAbsolutePtrVal, CGF.VoidPtrTy);
2530 CGF.CreateMemTemp(Private->getType(), ".omp.reduction.element");
2539 SrcElementAddr, CGF.ConvertTypeForMem(Private->getType()));
2546 shuffleAndStore(CGF, SrcElementAddr, DestElementAddr, Private->getType(),
2549 switch (CGF.getEvaluationKind(Private->getType())) {
2551 llvm::Value *Elem = CGF.EmitLoadOfScalar(
2556 CGF.EmitStoreOfScalar(
2562 CodeGenFunction::ComplexPairTy Elem = CGF.EmitLoadOfComplex(
2563 CGF.MakeAddrLValue(SrcElementAddr, Private->getType()),
2565 CGF.EmitStoreOfComplex(
2566 Elem, CGF.MakeAddrLValue(DestElementAddr, Private->getType()),
2571 CGF.EmitAggregateCopy(
2572 CGF.MakeAddrLValue(DestElementAddr, Private->getType()),
2573 CGF.MakeAddrLValue(SrcElementAddr, Private->getType()),
2585 CGF.EmitStoreOfScalar(Bld.CreatePointerBitCastOrAddrSpaceCast(
2586 DestElementAddr.getPointer(), CGF.VoidPtrTy),
2597 llvm::Value *ElementSizeInChars = CGF.getTypeSize(Private->getType());
2615 DestBase = Address(ScratchpadBasePtr, CGF.getPointerAlign());
2617 SrcBase = Address(ScratchpadBasePtr, CGF.getPointerAlign());
2664 CodeGenFunction CGF(CGM); in emitInterWarpCopyFunction() local
2665 CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, CGFI, Args, Loc, Loc); in emitInterWarpCopyFunction()
2667 CGBuilderTy &Bld = CGF.Builder; in emitInterWarpCopyFunction()
2680 unsigned WarpSize = CGF.getTarget().getGridValue(llvm::omp::GV_Warp_Size); in emitInterWarpCopyFunction()
2692 auto &RT = static_cast<CGOpenMPRuntimeGPU &>(CGF.CGM.getOpenMPRuntime()); in emitInterWarpCopyFunction()
2694 llvm::Value *ThreadID = RT.getGPUThreadID(CGF); in emitInterWarpCopyFunction()
2696 llvm::Value *LaneID = getNVPTXLaneID(CGF); in emitInterWarpCopyFunction()
2698 llvm::Value *WarpID = getNVPTXWarpID(CGF); in emitInterWarpCopyFunction()
2700 Address AddrReduceListArg = CGF.GetAddrOfLocalVar(&ReduceListArg); in emitInterWarpCopyFunction()
2703 CGF.EmitLoadOfScalar( in emitInterWarpCopyFunction()
2706 CGF.ConvertTypeForMem(ReductionArrayTy)->getPointerTo()), in emitInterWarpCopyFunction()
2707 CGF.getPointerAlign()); in emitInterWarpCopyFunction()
2725 llvm::Type *CopyType = CGF.ConvertTypeForMem(CType); in emitInterWarpCopyFunction()
2732 CntAddr = CGF.CreateMemTemp(C.IntTy, ".cnt.addr"); in emitInterWarpCopyFunction()
2733 CGF.EmitStoreOfScalar(llvm::Constant::getNullValue(CGM.IntTy), CntAddr, in emitInterWarpCopyFunction()
2735 PrecondBB = CGF.createBasicBlock("precond"); in emitInterWarpCopyFunction()
2736 ExitBB = CGF.createBasicBlock("exit"); in emitInterWarpCopyFunction()
2737 llvm::BasicBlock *BodyBB = CGF.createBasicBlock("body"); in emitInterWarpCopyFunction()
2739 (void)ApplyDebugLocation::CreateEmpty(CGF); in emitInterWarpCopyFunction()
2740 CGF.EmitBlock(PrecondBB); in emitInterWarpCopyFunction()
2741 Cnt = CGF.EmitLoadOfScalar(CntAddr, /*Volatile=*/false, C.IntTy, Loc); in emitInterWarpCopyFunction()
2745 CGF.EmitBlock(BodyBB); in emitInterWarpCopyFunction()
2748 CGM.getOpenMPRuntime().emitBarrierCall(CGF, Loc, OMPD_unknown, in emitInterWarpCopyFunction()
2751 llvm::BasicBlock *ThenBB = CGF.createBasicBlock("then"); in emitInterWarpCopyFunction()
2752 llvm::BasicBlock *ElseBB = CGF.createBasicBlock("else"); in emitInterWarpCopyFunction()
2753 llvm::BasicBlock *MergeBB = CGF.createBasicBlock("ifcont"); in emitInterWarpCopyFunction()
2758 CGF.EmitBlock(ThenBB); in emitInterWarpCopyFunction()
2762 llvm::Value *ElemPtrPtr = CGF.EmitLoadOfScalar( in emitInterWarpCopyFunction()
2784 llvm::Value *Elem = CGF.EmitLoadOfScalar( in emitInterWarpCopyFunction()
2788 CGF.EmitStoreOfScalar(Elem, MediumPtr, /*Volatile=*/true, CType, in emitInterWarpCopyFunction()
2794 CGF.EmitBlock(ElseBB); in emitInterWarpCopyFunction()
2797 CGF.EmitBlock(MergeBB); in emitInterWarpCopyFunction()
2800 CGM.getOpenMPRuntime().emitBarrierCall(CGF, Loc, OMPD_unknown, in emitInterWarpCopyFunction()
2807 llvm::BasicBlock *W0ThenBB = CGF.createBasicBlock("then"); in emitInterWarpCopyFunction()
2808 llvm::BasicBlock *W0ElseBB = CGF.createBasicBlock("else"); in emitInterWarpCopyFunction()
2809 llvm::BasicBlock *W0MergeBB = CGF.createBasicBlock("ifcont"); in emitInterWarpCopyFunction()
2811 Address AddrNumWarpsArg = CGF.GetAddrOfLocalVar(&NumWarpsArg); in emitInterWarpCopyFunction()
2812 llvm::Value *NumWarpsVal = CGF.EmitLoadOfScalar( in emitInterWarpCopyFunction()
2820 CGF.EmitBlock(W0ThenBB); in emitInterWarpCopyFunction()
2832 llvm::Value *TargetElemPtrVal = CGF.EmitLoadOfScalar( in emitInterWarpCopyFunction()
2843 CGF.EmitLoadOfScalar(SrcMediumPtr, /*Volatile=*/true, CType, Loc); in emitInterWarpCopyFunction()
2844 CGF.EmitStoreOfScalar(SrcMediumValue, TargetElemPtr, /*Volatile=*/false, in emitInterWarpCopyFunction()
2848 CGF.EmitBlock(W0ElseBB); in emitInterWarpCopyFunction()
2851 CGF.EmitBlock(W0MergeBB); in emitInterWarpCopyFunction()
2855 CGF.EmitStoreOfScalar(Cnt, CntAddr, /*Volatile=*/false, C.IntTy); in emitInterWarpCopyFunction()
2856 CGF.EmitBranch(PrecondBB); in emitInterWarpCopyFunction()
2857 (void)ApplyDebugLocation::CreateEmpty(CGF); in emitInterWarpCopyFunction()
2858 CGF.EmitBlock(ExitBB); in emitInterWarpCopyFunction()
2865 CGF.FinishFunction(); in emitInterWarpCopyFunction()
2966 CodeGenFunction CGF(CGM); in emitShuffleAndReduceFunction() local
2967 CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, CGFI, Args, Loc, Loc); in emitShuffleAndReduceFunction()
2969 CGBuilderTy &Bld = CGF.Builder; in emitShuffleAndReduceFunction()
2971 Address AddrReduceListArg = CGF.GetAddrOfLocalVar(&ReduceListArg); in emitShuffleAndReduceFunction()
2974 CGF.EmitLoadOfScalar(AddrReduceListArg, /*Volatile=*/false, in emitShuffleAndReduceFunction()
2976 CGF.ConvertTypeForMem(ReductionArrayTy)->getPointerTo()), in emitShuffleAndReduceFunction()
2977 CGF.getPointerAlign()); in emitShuffleAndReduceFunction()
2979 Address AddrLaneIDArg = CGF.GetAddrOfLocalVar(&LaneIDArg); in emitShuffleAndReduceFunction()
2980 llvm::Value *LaneIDArgVal = CGF.EmitLoadOfScalar( in emitShuffleAndReduceFunction()
2983 Address AddrRemoteLaneOffsetArg = CGF.GetAddrOfLocalVar(&RemoteLaneOffsetArg); in emitShuffleAndReduceFunction()
2984 llvm::Value *RemoteLaneOffsetArgVal = CGF.EmitLoadOfScalar( in emitShuffleAndReduceFunction()
2987 Address AddrAlgoVerArg = CGF.GetAddrOfLocalVar(&AlgoVerArg); in emitShuffleAndReduceFunction()
2988 llvm::Value *AlgoVerArgVal = CGF.EmitLoadOfScalar( in emitShuffleAndReduceFunction()
2994 CGF.CreateMemTemp(ReductionArrayTy, ".omp.reduction.remote_reduce_list"); in emitShuffleAndReduceFunction()
2999 emitReductionListCopy(RemoteLaneToThread, CGF, ReductionArrayTy, Privates, in emitShuffleAndReduceFunction()
3041 llvm::BasicBlock *ThenBB = CGF.createBasicBlock("then"); in emitShuffleAndReduceFunction()
3042 llvm::BasicBlock *ElseBB = CGF.createBasicBlock("else"); in emitShuffleAndReduceFunction()
3043 llvm::BasicBlock *MergeBB = CGF.createBasicBlock("ifcont"); in emitShuffleAndReduceFunction()
3046 CGF.EmitBlock(ThenBB); in emitShuffleAndReduceFunction()
3049 LocalReduceList.getPointer(), CGF.VoidPtrTy); in emitShuffleAndReduceFunction()
3051 RemoteReduceList.getPointer(), CGF.VoidPtrTy); in emitShuffleAndReduceFunction()
3053 CGF, Loc, ReduceFn, {LocalReduceListPtr, RemoteReduceListPtr}); in emitShuffleAndReduceFunction()
3056 CGF.EmitBlock(ElseBB); in emitShuffleAndReduceFunction()
3059 CGF.EmitBlock(MergeBB); in emitShuffleAndReduceFunction()
3067 llvm::BasicBlock *CpyThenBB = CGF.createBasicBlock("then"); in emitShuffleAndReduceFunction()
3068 llvm::BasicBlock *CpyElseBB = CGF.createBasicBlock("else"); in emitShuffleAndReduceFunction()
3069 llvm::BasicBlock *CpyMergeBB = CGF.createBasicBlock("ifcont"); in emitShuffleAndReduceFunction()
3072 CGF.EmitBlock(CpyThenBB); in emitShuffleAndReduceFunction()
3073 emitReductionListCopy(ThreadCopy, CGF, ReductionArrayTy, Privates, in emitShuffleAndReduceFunction()
3077 CGF.EmitBlock(CpyElseBB); in emitShuffleAndReduceFunction()
3080 CGF.EmitBlock(CpyMergeBB); in emitShuffleAndReduceFunction()
3082 CGF.FinishFunction(); in emitShuffleAndReduceFunction()
3121 CodeGenFunction CGF(CGM); in emitListToGlobalCopyFunction() local
3122 CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, CGFI, Args, Loc, Loc); in emitListToGlobalCopyFunction()
3124 CGBuilderTy &Bld = CGF.Builder; in emitListToGlobalCopyFunction()
3126 Address AddrReduceListArg = CGF.GetAddrOfLocalVar(&ReduceListArg); in emitListToGlobalCopyFunction()
3127 Address AddrBufferArg = CGF.GetAddrOfLocalVar(&BufferArg); in emitListToGlobalCopyFunction()
3130 CGF.EmitLoadOfScalar(AddrReduceListArg, /*Volatile=*/false, in emitListToGlobalCopyFunction()
3132 CGF.ConvertTypeForMem(ReductionArrayTy)->getPointerTo()), in emitListToGlobalCopyFunction()
3133 CGF.getPointerAlign()); in emitListToGlobalCopyFunction()
3138 CGF.EmitLoadOfScalar(AddrBufferArg, /*Volatile=*/false, C.VoidPtrTy, Loc), in emitListToGlobalCopyFunction()
3140 llvm::Value *Idxs[] = {llvm::ConstantInt::getNullValue(CGF.Int32Ty), in emitListToGlobalCopyFunction()
3141 CGF.EmitLoadOfScalar(CGF.GetAddrOfLocalVar(&IdxArg), in emitListToGlobalCopyFunction()
3148 llvm::Value *ElemPtrPtr = CGF.EmitLoadOfScalar( in emitListToGlobalCopyFunction()
3152 ElemPtrPtr, CGF.ConvertTypeForMem(Private->getType())->getPointerTo()); in emitListToGlobalCopyFunction()
3158 LValue GlobLVal = CGF.EmitLValueForField( in emitListToGlobalCopyFunction()
3159 CGF.MakeNaturalAlignAddrLValue(BufferArrPtr, StaticTy), FD); in emitListToGlobalCopyFunction()
3160 Address GlobAddr = GlobLVal.getAddress(CGF); in emitListToGlobalCopyFunction()
3164 switch (CGF.getEvaluationKind(Private->getType())) { in emitListToGlobalCopyFunction()
3166 llvm::Value *V = CGF.EmitLoadOfScalar( in emitListToGlobalCopyFunction()
3169 CGF.EmitStoreOfScalar(V, GlobLVal); in emitListToGlobalCopyFunction()
3173 CodeGenFunction::ComplexPairTy V = CGF.EmitLoadOfComplex( in emitListToGlobalCopyFunction()
3174 CGF.MakeAddrLValue(ElemPtr, Private->getType()), Loc); in emitListToGlobalCopyFunction()
3175 CGF.EmitStoreOfComplex(V, GlobLVal, /*isInit=*/false); in emitListToGlobalCopyFunction()
3179 CGF.EmitAggregateCopy(GlobLVal, in emitListToGlobalCopyFunction()
3180 CGF.MakeAddrLValue(ElemPtr, Private->getType()), in emitListToGlobalCopyFunction()
3187 CGF.FinishFunction(); in emitListToGlobalCopyFunction()
3230 CodeGenFunction CGF(CGM); in emitListToGlobalReduceFunction() local
3231 CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, CGFI, Args, Loc, Loc); in emitListToGlobalReduceFunction()
3233 CGBuilderTy &Bld = CGF.Builder; in emitListToGlobalReduceFunction()
3235 Address AddrBufferArg = CGF.GetAddrOfLocalVar(&BufferArg); in emitListToGlobalReduceFunction()
3240 CGF.EmitLoadOfScalar(AddrBufferArg, /*Volatile=*/false, C.VoidPtrTy, Loc), in emitListToGlobalReduceFunction()
3246 CGF.CreateMemTemp(ReductionArrayTy, ".omp.reduction.red_list"); in emitListToGlobalReduceFunction()
3248 llvm::Value *Idxs[] = {llvm::ConstantInt::getNullValue(CGF.Int32Ty), in emitListToGlobalReduceFunction()
3249 CGF.EmitLoadOfScalar(CGF.GetAddrOfLocalVar(&IdxArg), in emitListToGlobalReduceFunction()
3254 Address Elem = CGF.Builder.CreateConstArrayGEP(ReductionList, Idx); in emitListToGlobalReduceFunction()
3258 LValue GlobLVal = CGF.EmitLValueForField( in emitListToGlobalReduceFunction()
3259 CGF.MakeNaturalAlignAddrLValue(BufferArrPtr, StaticTy), FD); in emitListToGlobalReduceFunction()
3260 Address GlobAddr = GlobLVal.getAddress(CGF); in emitListToGlobalReduceFunction()
3263 llvm::Value *Ptr = CGF.EmitCastToVoidPtr(BufferPtr); in emitListToGlobalReduceFunction()
3264 CGF.EmitStoreOfScalar(Ptr, Elem, /*Volatile=*/false, C.VoidPtrTy); in emitListToGlobalReduceFunction()
3268 Elem = CGF.Builder.CreateConstArrayGEP(ReductionList, Idx); in emitListToGlobalReduceFunction()
3269 llvm::Value *Size = CGF.Builder.CreateIntCast( in emitListToGlobalReduceFunction()
3270 CGF.getVLASize( in emitListToGlobalReduceFunction()
3271 CGF.getContext().getAsVariableArrayType((*IPriv)->getType())) in emitListToGlobalReduceFunction()
3273 CGF.SizeTy, /*isSigned=*/false); in emitListToGlobalReduceFunction()
3274 CGF.Builder.CreateStore(CGF.Builder.CreateIntToPtr(Size, CGF.VoidPtrTy), in emitListToGlobalReduceFunction()
3281 CGF.EmitCastToVoidPtr(ReductionList.getPointer()); in emitListToGlobalReduceFunction()
3282 Address AddrReduceListArg = CGF.GetAddrOfLocalVar(&ReduceListArg); in emitListToGlobalReduceFunction()
3283 llvm::Value *ReducedPtr = CGF.EmitLoadOfScalar( in emitListToGlobalReduceFunction()
3286 CGF, Loc, ReduceFn, {GlobalReduceList, ReducedPtr}); in emitListToGlobalReduceFunction()
3287 CGF.FinishFunction(); in emitListToGlobalReduceFunction()
3326 CodeGenFunction CGF(CGM); in emitGlobalToListCopyFunction() local
3327 CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, CGFI, Args, Loc, Loc); in emitGlobalToListCopyFunction()
3329 CGBuilderTy &Bld = CGF.Builder; in emitGlobalToListCopyFunction()
3331 Address AddrReduceListArg = CGF.GetAddrOfLocalVar(&ReduceListArg); in emitGlobalToListCopyFunction()
3332 Address AddrBufferArg = CGF.GetAddrOfLocalVar(&BufferArg); in emitGlobalToListCopyFunction()
3335 CGF.EmitLoadOfScalar(AddrReduceListArg, /*Volatile=*/false, in emitGlobalToListCopyFunction()
3337 CGF.ConvertTypeForMem(ReductionArrayTy)->getPointerTo()), in emitGlobalToListCopyFunction()
3338 CGF.getPointerAlign()); in emitGlobalToListCopyFunction()
3343 CGF.EmitLoadOfScalar(AddrBufferArg, /*Volatile=*/false, C.VoidPtrTy, Loc), in emitGlobalToListCopyFunction()
3346 llvm::Value *Idxs[] = {llvm::ConstantInt::getNullValue(CGF.Int32Ty), in emitGlobalToListCopyFunction()
3347 CGF.EmitLoadOfScalar(CGF.GetAddrOfLocalVar(&IdxArg), in emitGlobalToListCopyFunction()
3354 llvm::Value *ElemPtrPtr = CGF.EmitLoadOfScalar( in emitGlobalToListCopyFunction()
3358 ElemPtrPtr, CGF.ConvertTypeForMem(Private->getType())->getPointerTo()); in emitGlobalToListCopyFunction()
3364 LValue GlobLVal = CGF.EmitLValueForField( in emitGlobalToListCopyFunction()
3365 CGF.MakeNaturalAlignAddrLValue(BufferArrPtr, StaticTy), FD); in emitGlobalToListCopyFunction()
3366 Address GlobAddr = GlobLVal.getAddress(CGF); in emitGlobalToListCopyFunction()
3370 switch (CGF.getEvaluationKind(Private->getType())) { in emitGlobalToListCopyFunction()
3372 llvm::Value *V = CGF.EmitLoadOfScalar(GlobLVal, Loc); in emitGlobalToListCopyFunction()
3373 CGF.EmitStoreOfScalar(V, ElemPtr, /*Volatile=*/false, Private->getType(), in emitGlobalToListCopyFunction()
3379 CodeGenFunction::ComplexPairTy V = CGF.EmitLoadOfComplex(GlobLVal, Loc); in emitGlobalToListCopyFunction()
3380 CGF.EmitStoreOfComplex(V, CGF.MakeAddrLValue(ElemPtr, Private->getType()), in emitGlobalToListCopyFunction()
3385 CGF.EmitAggregateCopy(CGF.MakeAddrLValue(ElemPtr, Private->getType()), in emitGlobalToListCopyFunction()
3393 CGF.FinishFunction(); in emitGlobalToListCopyFunction()
3436 CodeGenFunction CGF(CGM); in emitGlobalToListReduceFunction() local
3437 CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, CGFI, Args, Loc, Loc); in emitGlobalToListReduceFunction()
3439 CGBuilderTy &Bld = CGF.Builder; in emitGlobalToListReduceFunction()
3441 Address AddrBufferArg = CGF.GetAddrOfLocalVar(&BufferArg); in emitGlobalToListReduceFunction()
3446 CGF.EmitLoadOfScalar(AddrBufferArg, /*Volatile=*/false, C.VoidPtrTy, Loc), in emitGlobalToListReduceFunction()
3452 CGF.CreateMemTemp(ReductionArrayTy, ".omp.reduction.red_list"); in emitGlobalToListReduceFunction()
3454 llvm::Value *Idxs[] = {llvm::ConstantInt::getNullValue(CGF.Int32Ty), in emitGlobalToListReduceFunction()
3455 CGF.EmitLoadOfScalar(CGF.GetAddrOfLocalVar(&IdxArg), in emitGlobalToListReduceFunction()
3460 Address Elem = CGF.Builder.CreateConstArrayGEP(ReductionList, Idx); in emitGlobalToListReduceFunction()
3464 LValue GlobLVal = CGF.EmitLValueForField( in emitGlobalToListReduceFunction()
3465 CGF.MakeNaturalAlignAddrLValue(BufferArrPtr, StaticTy), FD); in emitGlobalToListReduceFunction()
3466 Address GlobAddr = GlobLVal.getAddress(CGF); in emitGlobalToListReduceFunction()
3469 llvm::Value *Ptr = CGF.EmitCastToVoidPtr(BufferPtr); in emitGlobalToListReduceFunction()
3470 CGF.EmitStoreOfScalar(Ptr, Elem, /*Volatile=*/false, C.VoidPtrTy); in emitGlobalToListReduceFunction()
3474 Elem = CGF.Builder.CreateConstArrayGEP(ReductionList, Idx); in emitGlobalToListReduceFunction()
3475 llvm::Value *Size = CGF.Builder.CreateIntCast( in emitGlobalToListReduceFunction()
3476 CGF.getVLASize( in emitGlobalToListReduceFunction()
3477 CGF.getContext().getAsVariableArrayType((*IPriv)->getType())) in emitGlobalToListReduceFunction()
3479 CGF.SizeTy, /*isSigned=*/false); in emitGlobalToListReduceFunction()
3480 CGF.Builder.CreateStore(CGF.Builder.CreateIntToPtr(Size, CGF.VoidPtrTy), in emitGlobalToListReduceFunction()
3487 CGF.EmitCastToVoidPtr(ReductionList.getPointer()); in emitGlobalToListReduceFunction()
3488 Address AddrReduceListArg = CGF.GetAddrOfLocalVar(&ReduceListArg); in emitGlobalToListReduceFunction()
3489 llvm::Value *ReducedPtr = CGF.EmitLoadOfScalar( in emitGlobalToListReduceFunction()
3492 CGF, Loc, ReduceFn, {ReducedPtr, GlobalReduceList}); in emitGlobalToListReduceFunction()
3493 CGF.FinishFunction(); in emitGlobalToListReduceFunction()
3740 CodeGenFunction &CGF, SourceLocation Loc, ArrayRef<const Expr *> Privates, in emitReduction() argument
3743 if (!CGF.HaveInsertPoint()) in emitReduction()
3754 CGOpenMPRuntime::emitReduction(CGF, Loc, Privates, LHSExprs, RHSExprs, in emitReduction()
3766 llvm::Value *RTLoc = emitUpdateLocation(CGF, Loc); in emitReduction()
3767 llvm::Value *ThreadId = getThreadID(CGF, Loc); in emitReduction()
3784 CGF.CreateMemTemp(ReductionArrayTy, ".omp.reduction.red_list"); in emitReduction()
3788 Address Elem = CGF.Builder.CreateConstArrayGEP(ReductionList, Idx); in emitReduction()
3789 CGF.Builder.CreateStore( in emitReduction()
3790 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitReduction()
3791 CGF.EmitLValue(RHSExprs[I]).getPointer(CGF), CGF.VoidPtrTy), in emitReduction()
3796 Elem = CGF.Builder.CreateConstArrayGEP(ReductionList, Idx); in emitReduction()
3797 llvm::Value *Size = CGF.Builder.CreateIntCast( in emitReduction()
3798 CGF.getVLASize( in emitReduction()
3799 CGF.getContext().getAsVariableArrayType((*IPriv)->getType())) in emitReduction()
3801 CGF.SizeTy, /*isSigned=*/false); in emitReduction()
3802 CGF.Builder.CreateStore(CGF.Builder.CreateIntToPtr(Size, CGF.VoidPtrTy), in emitReduction()
3807 llvm::Value *RL = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitReduction()
3808 ReductionList.getPointer(), CGF.VoidPtrTy); in emitReduction()
3810 Loc, CGF.ConvertTypeForMem(ReductionArrayTy)->getPointerTo(), Privates, in emitReduction()
3812 llvm::Value *ReductionArrayTySize = CGF.getTypeSize(ReductionArrayTy); in emitReduction()
3821 CGF.Builder.getInt32(RHSExprs.size()), in emitReduction()
3827 Res = CGF.EmitRuntimeCall( in emitReduction()
3850 llvm::Value *GlobalBufferPtr = CGF.EmitLoadOfScalar( in emitReduction()
3868 CGF.Builder.getInt32(C.getLangOpts().OpenMPCUDAReductionBufNum), in emitReduction()
3877 Res = CGF.EmitRuntimeCall( in emitReduction()
3884 llvm::BasicBlock *ExitBB = CGF.createBasicBlock(".omp.reduction.done"); in emitReduction()
3885 llvm::BasicBlock *ThenBB = CGF.createBasicBlock(".omp.reduction.then"); in emitReduction()
3886 llvm::Value *Cond = CGF.Builder.CreateICmpEQ( in emitReduction()
3888 CGF.Builder.CreateCondBr(Cond, ThenBB, ExitBB); in emitReduction()
3894 CGF.EmitBlock(ThenBB); in emitReduction()
3898 this](CodeGenFunction &CGF, PrePostActionTy &Action) { in emitReduction() argument
3903 emitSingleReductionCombiner(CGF, E, *IPriv, cast<DeclRefExpr>(*ILHS), in emitReduction()
3918 RCG(CGF); in emitReduction()
3920 (void)ApplyDebugLocation::CreateEmpty(CGF); in emitReduction()
3921 CGF.EmitBlock(ExitBB, /*IsFinished=*/true); in emitReduction()
3961 CGOpenMPRuntimeGPU::getParameterAddress(CodeGenFunction &CGF, in getParameterAddress() argument
3967 Address LocalAddr = CGF.GetAddrOfLocalVar(TargetParam); in getParameterAddress()
3973 CGF.getContext().getTargetAddressSpace(NativePointeeTy); in getParameterAddress()
3975 llvm::Value *TargetAddr = CGF.EmitLoadOfScalar( in getParameterAddress()
3978 TargetAddr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in getParameterAddress()
3982 TargetAddr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in getParameterAddress()
3985 Address NativeParamAddr = CGF.CreateMemTemp(NativeParamType); in getParameterAddress()
3986 CGF.EmitStoreOfScalar(TargetAddr, NativeParamAddr, /*Volatile=*/false, in getParameterAddress()
3992 CodeGenFunction &CGF, SourceLocation Loc, llvm::FunctionCallee OutlinedFn, in emitOutlinedFunctionCall() argument
4008 llvm::Value *TargetArg = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitOutlinedFunctionCall()
4012 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(TargetArg, TargetType)); in emitOutlinedFunctionCall()
4014 CGOpenMPRuntime::emitOutlinedFunctionCall(CGF, Loc, OutlinedFn, TargetArgs); in emitOutlinedFunctionCall()
4060 CodeGenFunction CGF(CGM, /*suppressNewContext=*/true); in createParallelDataSharingWrapper() local
4061 CGF.StartFunction(GlobalDecl(), Ctx.VoidTy, Fn, CGFI, WrapperArgs, in createParallelDataSharingWrapper()
4067 Address ZeroAddr = CGF.CreateDefaultAlignTempAlloca(CGF.Int32Ty, in createParallelDataSharingWrapper()
4069 CGF.InitTempAlloca(ZeroAddr, CGF.Builder.getInt32(/*C*/ 0)); in createParallelDataSharingWrapper()
4073 Args.emplace_back(CGF.GetAddrOfLocalVar(&WrapperArg).getPointer()); in createParallelDataSharingWrapper()
4076 CGBuilderTy &Bld = CGF.Builder; in createParallelDataSharingWrapper()
4082 CGF.CreateDefaultAlignTempAlloca(CGF.VoidPtrPtrTy, "global_args"); in createParallelDataSharingWrapper()
4085 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in createParallelDataSharingWrapper()
4094 SharedArgListAddress = CGF.EmitLoadOfPointer( in createParallelDataSharingWrapper()
4095 GlobalArgs, CGF.getContext() in createParallelDataSharingWrapper()
4096 .getPointerType(CGF.getContext().getPointerType( in createParallelDataSharingWrapper()
4097 CGF.getContext().VoidPtrTy)) in createParallelDataSharingWrapper()
4104 Src, CGF.SizeTy->getPointerTo()); in createParallelDataSharingWrapper()
4105 llvm::Value *LB = CGF.EmitLoadOfScalar( in createParallelDataSharingWrapper()
4108 CGF.getContext().getPointerType(CGF.getContext().getSizeType()), in createParallelDataSharingWrapper()
4114 Src, CGF.SizeTy->getPointerTo()); in createParallelDataSharingWrapper()
4115 llvm::Value *UB = CGF.EmitLoadOfScalar( in createParallelDataSharingWrapper()
4118 CGF.getContext().getPointerType(CGF.getContext().getSizeType()), in createParallelDataSharingWrapper()
4124 ASTContext &CGFContext = CGF.getContext(); in createParallelDataSharingWrapper()
4129 Src, CGF.ConvertTypeForMem(CGFContext.getPointerType(ElemTy))); in createParallelDataSharingWrapper()
4130 llvm::Value *Arg = CGF.EmitLoadOfScalar(TypedAddress, in createParallelDataSharingWrapper()
4136 Arg = castValueToType(CGF, Arg, ElemTy, CGFContext.getUIntPtrType(), in createParallelDataSharingWrapper()
4143 emitOutlinedFunctionCall(CGF, D.getBeginLoc(), OutlinedParallelFn, Args); in createParallelDataSharingWrapper()
4144 CGF.FinishFunction(); in createParallelDataSharingWrapper()
4148 void CGOpenMPRuntimeGPU::emitFunctionProlog(CodeGenFunction &CGF, in emitFunctionProlog() argument
4154 assert(FunctionGlobalizedDecls.count(CGF.CurFn) == 0 && in emitFunctionProlog()
4166 NeedToDelayGlobalization = CGF.CapturedStmtInfo->getKind() == CR_OpenMP; in emitFunctionProlog()
4173 CheckVarsEscapingDeclContext VarChecker(CGF, TeamAndReductions.second); in emitFunctionProlog()
4183 auto I = FunctionGlobalizedDecls.try_emplace(CGF.CurFn).first; in emitFunctionProlog()
4199 CheckVarsEscapingDeclContext VarChecker(CGF, llvm::None); in emitFunctionProlog()
4213 emitGenericVarsProlog(CGF, D->getBeginLoc(), /*WithSPMDCheck=*/true); in emitFunctionProlog()
4217 void Emit(CodeGenFunction &CGF, Flags flags) override { in emitFunctionProlog()
4218 static_cast<CGOpenMPRuntimeGPU &>(CGF.CGM.getOpenMPRuntime()) in emitFunctionProlog()
4219 .emitGenericVarsEpilog(CGF, /*WithSPMDCheck=*/true); in emitFunctionProlog()
4222 CGF.EHStack.pushCleanup<GlobalizationScope>(NormalAndEHCleanup); in emitFunctionProlog()
4226 Address CGOpenMPRuntimeGPU::getAddressOfLocalVariable(CodeGenFunction &CGF, in getAddressOfLocalVariable() argument
4254 llvm::Type *VarTy = CGF.ConvertTypeForMem(VD->getType()); in getAddressOfLocalVariable()
4264 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in getAddressOfLocalVariable()
4274 auto I = FunctionGlobalizedDecls.find(CGF.CurFn); in getAddressOfLocalVariable()
4295 void CGOpenMPRuntimeGPU::functionFinished(CodeGenFunction &CGF) { in functionFinished() argument
4296 FunctionGlobalizedDecls.erase(CGF.CurFn); in functionFinished()
4297 CGOpenMPRuntime::functionFinished(CGF); in functionFinished()
4301 CodeGenFunction &CGF, const OMPLoopDirective &S, in getDefaultDistScheduleAndChunk() argument
4304 auto &RT = static_cast<CGOpenMPRuntimeGPU &>(CGF.CGM.getOpenMPRuntime()); in getDefaultDistScheduleAndChunk()
4307 Chunk = CGF.EmitScalarConversion( in getDefaultDistScheduleAndChunk()
4308 RT.getGPUNumThreads(CGF), in getDefaultDistScheduleAndChunk()
4309 CGF.getContext().getIntTypeForBitwidth(32, /*Signed=*/0), in getDefaultDistScheduleAndChunk()
4314 CGF, S, ScheduleKind, Chunk); in getDefaultDistScheduleAndChunk()
4318 CodeGenFunction &CGF, const OMPLoopDirective &S, in getDefaultScheduleAndChunk() argument
4324 ChunkExpr = IntegerLiteral::Create(CGF.getContext(), ChunkSize, in getDefaultScheduleAndChunk()
4325 CGF.getContext().getIntTypeForBitwidth(32, /*Signed=*/0), in getDefaultScheduleAndChunk()
4330 CodeGenFunction &CGF, const OMPExecutableDirective &D) const { in adjustTargetSpecificDataForLambdas() argument
4346 Address VDAddr = CGF.GetAddrOfLocalVar(VD); in adjustTargetSpecificDataForLambdas()
4349 VDLVal = CGF.EmitLoadOfReferenceLValue(VDAddr, VD->getType()); in adjustTargetSpecificDataForLambdas()
4351 VDLVal = CGF.MakeAddrLValue( in adjustTargetSpecificDataForLambdas()
4356 if (ThisCapture && CGF.CapturedStmtInfo->isCXXThisExprCaptured()) { in adjustTargetSpecificDataForLambdas()
4358 CGF.EmitLValueForFieldInitialization(VDLVal, ThisCapture); in adjustTargetSpecificDataForLambdas()
4359 llvm::Value *CXXThis = CGF.LoadCXXThis(); in adjustTargetSpecificDataForLambdas()
4360 CGF.EmitStoreOfScalar(CXXThis, ThisLVal); in adjustTargetSpecificDataForLambdas()
4370 LValue VarLVal = CGF.EmitLValueForFieldInitialization(VDLVal, It->second); in adjustTargetSpecificDataForLambdas()
4371 Address VDAddr = CGF.GetAddrOfLocalVar(VD); in adjustTargetSpecificDataForLambdas()
4373 VDAddr = CGF.EmitLoadOfReferenceLValue(VDAddr, in adjustTargetSpecificDataForLambdas()
4375 .getAddress(CGF); in adjustTargetSpecificDataForLambdas()
4376 CGF.EmitStoreOfScalar(VDAddr.getPointer(), VarLVal); in adjustTargetSpecificDataForLambdas()