Lines Matching refs:CGF
42 void emitPreInitStmt(CodeGenFunction &CGF, const OMPExecutableDirective &S) { in emitPreInitStmt() argument
49 CGF.EmitVarDecl(cast<VarDecl>(*I)); in emitPreInitStmt()
52 CGF.EmitAutoVarAlloca(cast<VarDecl>(*I)); in emitPreInitStmt()
53 CGF.EmitAutoVarCleanups(Emission); in emitPreInitStmt()
62 static bool isCapturedVar(CodeGenFunction &CGF, const VarDecl *VD) { in isCapturedVar() argument
63 return CGF.LambdaCaptureFields.lookup(VD) || in isCapturedVar()
64 (CGF.CapturedStmtInfo && CGF.CapturedStmtInfo->lookup(VD)) || in isCapturedVar()
65 (CGF.CurCodeDecl && isa<BlockDecl>(CGF.CurCodeDecl) && in isCapturedVar()
66 cast<BlockDecl>(CGF.CurCodeDecl)->capturesVariable(VD)); in isCapturedVar()
71 CodeGenFunction &CGF, const OMPExecutableDirective &S, in OMPLexicalScope() argument
74 : CodeGenFunction::LexicalScope(CGF, S.getSourceRange()), in OMPLexicalScope()
75 InlinedShareds(CGF) { in OMPLexicalScope()
77 emitPreInitStmt(CGF, S); in OMPLexicalScope()
89 CGF.getContext(), const_cast<VarDecl *>(VD), in OMPLexicalScope()
90 isCapturedVar(CGF, VD) || (CGF.CapturedStmtInfo && in OMPLexicalScope()
93 InlinedShareds.addPrivate(VD, [&CGF, &DRE]() -> Address { in OMPLexicalScope()
94 return CGF.EmitLValue(&DRE).getAddress(CGF); in OMPLexicalScope()
113 OMPParallelScope(CodeGenFunction &CGF, const OMPExecutableDirective &S) in OMPParallelScope() argument
114 : OMPLexicalScope(CGF, S, /*CapturedRegion=*/llvm::None, in OMPParallelScope()
128 OMPTeamsScope(CodeGenFunction &CGF, const OMPExecutableDirective &S) in OMPTeamsScope() argument
129 : OMPLexicalScope(CGF, S, /*CapturedRegion=*/llvm::None, in OMPTeamsScope()
136 void emitPreInitStmt(CodeGenFunction &CGF, const OMPLoopBasedDirective &S) { in emitPreInitStmt() argument
145 CGF, VD, CGF.CreateMemTemp(VD->getType().getNonReferenceType())); in emitPreInitStmt()
154 CGF, OrigVD, in emitPreInitStmt()
155 Address(llvm::UndefValue::get(CGF.ConvertTypeForMem( in emitPreInitStmt()
156 CGF.getContext().getPointerType( in emitPreInitStmt()
158 CGF.getContext().getDeclAlign(OrigVD))); in emitPreInitStmt()
162 (void)PreCondVars.apply(CGF); in emitPreInitStmt()
167 [&CGF](unsigned Cnt, const Stmt *CurStmt) { in emitPreInitStmt()
170 CGF.EmitStmt(Init); in emitPreInitStmt()
171 CGF.EmitStmt(CXXFor->getRangeStmt()); in emitPreInitStmt()
172 CGF.EmitStmt(CXXFor->getEndStmt()); in emitPreInitStmt()
184 CGF.EmitVarDecl(cast<VarDecl>(*I)); in emitPreInitStmt()
186 PreCondVars.restore(CGF); in emitPreInitStmt()
190 OMPLoopScope(CodeGenFunction &CGF, const OMPLoopBasedDirective &S) in OMPLoopScope() argument
191 : CodeGenFunction::RunCleanupsScope(CGF) { in OMPLoopScope()
192 emitPreInitStmt(CGF, S); in OMPLoopScope()
199 static bool isCapturedVar(CodeGenFunction &CGF, const VarDecl *VD) { in isCapturedVar() argument
200 return CGF.LambdaCaptureFields.lookup(VD) || in isCapturedVar()
201 (CGF.CapturedStmtInfo && CGF.CapturedStmtInfo->lookup(VD)) || in isCapturedVar()
202 (CGF.CurCodeDecl && isa<BlockDecl>(CGF.CurCodeDecl) && in isCapturedVar()
203 cast<BlockDecl>(CGF.CurCodeDecl)->capturesVariable(VD)); in isCapturedVar()
207 OMPSimdLexicalScope(CodeGenFunction &CGF, const OMPExecutableDirective &S) in OMPSimdLexicalScope() argument
208 : CodeGenFunction::LexicalScope(CGF, S.getSourceRange()), in OMPSimdLexicalScope()
209 InlinedShareds(CGF) { in OMPSimdLexicalScope()
216 CGF.EmitVarDecl(cast<VarDecl>(*I)); in OMPSimdLexicalScope()
219 CGF.EmitAutoVarAlloca(cast<VarDecl>(*I)); in OMPSimdLexicalScope()
220 CGF.EmitAutoVarCleanups(Emission); in OMPSimdLexicalScope()
228 CGF.EmitVarDecl(*OED); in OMPSimdLexicalScope()
234 CGF.EmitVarDecl(*OED); in OMPSimdLexicalScope()
239 CGF.EmitOMPPrivateClause(S, InlinedShareds); in OMPSimdLexicalScope()
242 CGF.EmitVarDecl(*cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl())); in OMPSimdLexicalScope()
262 DeclRefExpr DRE(CGF.getContext(), const_cast<VarDecl *>(VD), in OMPSimdLexicalScope()
263 isCapturedVar(CGF, VD) || in OMPSimdLexicalScope()
264 (CGF.CapturedStmtInfo && in OMPSimdLexicalScope()
268 InlinedShareds.addPrivate(VD, [&CGF, &DRE]() -> Address { in OMPSimdLexicalScope()
269 return CGF.EmitLValue(&DRE).getAddress(CGF); in OMPSimdLexicalScope()
281 static void emitCommonOMPTargetDirective(CodeGenFunction &CGF,
367 static Address castValueFromUintptr(CodeGenFunction &CGF, SourceLocation Loc, in castValueFromUintptr() argument
370 ASTContext &Ctx = CGF.getContext(); in castValueFromUintptr()
372 llvm::Value *CastedPtr = CGF.EmitScalarConversion( in castValueFromUintptr()
373 AddrLV.getAddress(CGF).getPointer(), Ctx.getUIntPtrType(), in castValueFromUintptr()
376 CGF.MakeNaturalAlignAddrLValue(CastedPtr, Ctx.getPointerType(DstType)) in castValueFromUintptr()
377 .getAddress(CGF); in castValueFromUintptr()
422 CodeGenFunction &CGF, FunctionArgList &Args, in emitOutlinedFunctionPrologue() argument
434 CodeGenModule &CGM = CGF.CGM; in emitOutlinedFunctionPrologue()
519 CGF.StartFunction(CD, Ctx.VoidTy, F, FuncInfo, TargetArgs, in emitOutlinedFunctionPrologue()
529 LocalAddr = CGM.getOpenMPRuntime().getParameterAddress(CGF, Args[Cnt], in emitOutlinedFunctionPrologue()
532 LocalAddr = CGF.GetAddrOfLocalVar(Args[Cnt]); in emitOutlinedFunctionPrologue()
545 LValue ArgLVal = CGF.MakeAddrLValue(LocalAddr, Args[Cnt]->getType(), in emitOutlinedFunctionPrologue()
549 ArgLVal = CGF.MakeAddrLValue( in emitOutlinedFunctionPrologue()
550 castValueFromUintptr(CGF, I->getLocation(), FD->getType(), in emitOutlinedFunctionPrologue()
554 llvm::Value *ExprArg = CGF.EmitLoadOfScalar(ArgLVal, I->getLocation()); in emitOutlinedFunctionPrologue()
560 Address ArgAddr = ArgLVal.getAddress(CGF); in emitOutlinedFunctionPrologue()
562 ArgAddr = CGF.EmitLoadOfReference(ArgLVal); in emitOutlinedFunctionPrologue()
565 ArgAddr = CGF.EmitLoadOfPointer( in emitOutlinedFunctionPrologue()
580 CGF, I->getLocation(), FD->getType(), in emitOutlinedFunctionPrologue()
582 : ArgLVal.getAddress(CGF)}}); in emitOutlinedFunctionPrologue()
586 CXXThisValue = CGF.EmitLoadOfScalar(ArgLVal, I->getLocation()); in emitOutlinedFunctionPrologue()
587 LocalAddrs.insert({Args[Cnt], {nullptr, ArgLVal.getAddress(CGF)}}); in emitOutlinedFunctionPrologue()
1239 [&Emission](CodeGenFunction &CGF) { in EmitOMPReductionClauseInit() argument
1240 CGF.EmitAutoVarInit(Emission); in EmitOMPReductionClauseInit()
1456 CodeGenFunction &CGF, const OMPExecutableDirective &D, in emitPostUpdateForReductionClause() argument
1458 if (!CGF.HaveInsertPoint()) in emitPostUpdateForReductionClause()
1464 if (llvm::Value *Cond = CondGen(CGF)) { in emitPostUpdateForReductionClause()
1467 llvm::BasicBlock *ThenBB = CGF.createBasicBlock(".omp.reduction.pu"); in emitPostUpdateForReductionClause()
1468 DoneBB = CGF.createBasicBlock(".omp.reduction.pu.done"); in emitPostUpdateForReductionClause()
1469 CGF.Builder.CreateCondBr(Cond, ThenBB, DoneBB); in emitPostUpdateForReductionClause()
1470 CGF.EmitBlock(ThenBB); in emitPostUpdateForReductionClause()
1473 CGF.EmitIgnoredExpr(PostUpdate); in emitPostUpdateForReductionClause()
1477 CGF.EmitBlock(DoneBB, /*IsFinished=*/true); in emitPostUpdateForReductionClause()
1491 checkForLastprivateConditionalUpdate(CodeGenFunction &CGF, in checkForLastprivateConditionalUpdate() argument
1493 if (CGF.getLangOpts().OpenMP < 50) in checkForLastprivateConditionalUpdate()
1504 CGF.CGM.getOpenMPRuntime().checkAndEmitLastprivateConditional(CGF, Ref); in checkForLastprivateConditionalUpdate()
1515 CGF.CGM.getOpenMPRuntime().checkAndEmitLastprivateConditional(CGF, Ref); in checkForLastprivateConditionalUpdate()
1526 CGF.CGM.getOpenMPRuntime().checkAndEmitLastprivateConditional(CGF, Ref); in checkForLastprivateConditionalUpdate()
1543 CGF.CGM.getOpenMPRuntime().checkAndEmitSharedLastprivateConditional( in checkForLastprivateConditionalUpdate()
1544 CGF, S, PrivateDecls); in checkForLastprivateConditionalUpdate()
1548 CodeGenFunction &CGF, const OMPExecutableDirective &S, in emitCommonOMPParallelDirective() argument
1553 CGF.CGM.getOpenMPRuntime().emitParallelOutlinedFunction( in emitCommonOMPParallelDirective()
1556 CodeGenFunction::RunCleanupsScope NumThreadsScope(CGF); in emitCommonOMPParallelDirective()
1558 CGF.EmitScalarExpr(NumThreadsClause->getNumThreads(), in emitCommonOMPParallelDirective()
1560 CGF.CGM.getOpenMPRuntime().emitNumThreadsClause( in emitCommonOMPParallelDirective()
1561 CGF, NumThreads, NumThreadsClause->getBeginLoc()); in emitCommonOMPParallelDirective()
1564 CodeGenFunction::RunCleanupsScope ProcBindScope(CGF); in emitCommonOMPParallelDirective()
1565 CGF.CGM.getOpenMPRuntime().emitProcBindClause( in emitCommonOMPParallelDirective()
1566 CGF, ProcBindClause->getProcBindKind(), ProcBindClause->getBeginLoc()); in emitCommonOMPParallelDirective()
1577 OMPParallelScope Scope(CGF, S); in emitCommonOMPParallelDirective()
1583 CodeGenBoundParameters(CGF, S, CapturedVars); in emitCommonOMPParallelDirective()
1584 CGF.GenerateOpenMPCapturedVars(*CS, CapturedVars); in emitCommonOMPParallelDirective()
1585 CGF.CGM.getOpenMPRuntime().emitParallelCall(CGF, S.getBeginLoc(), OutlinedFn, in emitCommonOMPParallelDirective()
1605 CodeGenFunction &CGF, const VarDecl *VD) { in getAddressOfLocalVariable() argument
1606 CodeGenModule &CGM = CGF.CGM; in getAddressOfLocalVariable()
1617 Size = CGF.getTypeSize(CVD->getType()); in getAddressOfLocalVariable()
1619 Size = CGF.Builder.CreateNUWAdd( in getAddressOfLocalVariable()
1621 Size = CGF.Builder.CreateUDiv(Size, CGM.getSize(Align)); in getAddressOfLocalVariable()
1622 Size = CGF.Builder.CreateNUWMul(Size, CGM.getSize(Align)); in getAddressOfLocalVariable()
1631 llvm::Value *Allocator = CGF.EmitScalarExpr(AA->getAllocator()); in getAddressOfLocalVariable()
1635 Allocator = CGF.Builder.CreateIntToPtr(Allocator, CGM.VoidPtrTy); in getAddressOfLocalVariable()
1637 Allocator = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(Allocator, in getAddressOfLocalVariable()
1641 CGF.Builder, Size, Allocator, in getAddressOfLocalVariable()
1644 OMPBuilder.createOMPFree(CGF.Builder, Addr, Allocator); in getAddressOfLocalVariable()
1646 CGF.EHStack.pushCleanup<OMPAllocateCleanupTy>(NormalAndEHCleanup, FreeCI); in getAddressOfLocalVariable()
1647 Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in getAddressOfLocalVariable()
1649 CGF.ConvertTypeForMem(CGM.getContext().getPointerType(CVD->getType())), in getAddressOfLocalVariable()
1655 CodeGenFunction &CGF, const VarDecl *VD, Address VDAddr, in getAddrOfThreadPrivate() argument
1657 CodeGenModule &CGM = CGF.CGM; in getAddrOfThreadPrivate()
1666 CGF.Builder.CreatePointerCast(VDAddr.getPointer(), CGM.Int8PtrTy); in getAddrOfThreadPrivate()
1672 OMPBuilder.createCachedThreadPrivate(CGF.Builder, Data, Size, CacheName); in getAddrOfThreadPrivate()
1750 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPParallelDirective() argument
1751 Action.Enter(CGF); in EmitOMPParallelDirective()
1752 OMPPrivateScope PrivateScope(CGF); in EmitOMPParallelDirective()
1753 bool Copyins = CGF.EmitOMPCopyinClause(S); in EmitOMPParallelDirective()
1754 (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope); in EmitOMPParallelDirective()
1759 CGF.CGM.getOpenMPRuntime().emitBarrierCall( in EmitOMPParallelDirective()
1760 CGF, S.getBeginLoc(), OMPD_unknown, /*EmitChecks=*/false, in EmitOMPParallelDirective()
1763 CGF.EmitOMPPrivateClause(S, PrivateScope); in EmitOMPParallelDirective()
1764 CGF.EmitOMPReductionClauseInit(S, PrivateScope); in EmitOMPParallelDirective()
1766 CGF.EmitStmt(S.getCapturedStmt(OMPD_parallel)->getCapturedStmt()); in EmitOMPParallelDirective()
1767 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_parallel); in EmitOMPParallelDirective()
1789 OMPTransformDirectiveScopeRAII(CodeGenFunction &CGF, const Stmt *S) { in OMPTransformDirectiveScopeRAII() argument
1791 Scope = new OMPLoopScope(CGF, *Dir); in OMPTransformDirectiveScopeRAII()
1793 CapInfoRAII = new CodeGenFunction::CGCapturedStmtRAII(CGF, CGSI); in OMPTransformDirectiveScopeRAII()
1806 static void emitBody(CodeGenFunction &CGF, const Stmt *S, const Stmt *NextLoop, in emitBody() argument
1812 CGF.getContext().getSourceManager(), CS->getLBracLoc(), in emitBody()
1816 CodeGenFunction::LexicalScope Scope(CGF, S->getSourceRange()); in emitBody()
1818 emitBody(CGF, CurStmt, NextLoop, MaxLevel, Level); in emitBody()
1822 OMPTransformDirectiveScopeRAII PossiblyTransformDirectiveScope(CGF, in emitBody()
1834 CGF.EmitStmt(CXXFor->getLoopVarStmt()); in emitBody()
1840 emitBody(CGF, S, NextLoop, MaxLevel, Level + 1); in emitBody()
1844 CGF.EmitStmt(S); in emitBody()
1922 CodeGenFunction CGF(ParentCGF.CGM, /*suppressNewContext=*/true); in emitCapturedStmtFunc() local
1925 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, CSI.get()); in emitCapturedStmtFunc()
1926 llvm::Function *F = CGF.GenerateCapturedStmtFunction(*S); in emitCapturedStmtFunc()
2166 static void emitAlignedClause(CodeGenFunction &CGF, in emitAlignedClause() argument
2168 if (!CGF.HaveInsertPoint()) in emitAlignedClause()
2174 cast<llvm::ConstantInt>(CGF.EmitScalarExpr(AlignmentExpr)); in emitAlignedClause()
2184 CGF.getContext() in emitAlignedClause()
2185 .toCharUnitsFromBits(CGF.getContext().getOpenMPDefaultSimdAlign( in emitAlignedClause()
2192 llvm::Value *PtrValue = CGF.EmitScalarExpr(E); in emitAlignedClause()
2193 CGF.emitAlignmentAssumption( in emitAlignedClause()
2195 llvm::ConstantInt::get(CGF.getLLVMContext(), Alignment)); in emitAlignedClause()
2250 static void emitPreCond(CodeGenFunction &CGF, const OMPLoopDirective &S, in emitPreCond() argument
2253 if (!CGF.HaveInsertPoint()) in emitPreCond()
2256 CodeGenFunction::OMPPrivateScope PreCondScope(CGF); in emitPreCond()
2257 CGF.EmitOMPPrivateLoopCounters(S, PreCondScope); in emitPreCond()
2261 CGF.EmitIgnoredExpr(I); in emitPreCond()
2274 CGF.CreateMemTemp(VD->getType().getNonReferenceType()); in emitPreCond()
2275 (void)PreCondVars.setVarAddr(CGF, VD, CounterAddr); in emitPreCond()
2277 (void)PreCondVars.apply(CGF); in emitPreCond()
2281 CGF.EmitIgnoredExpr(E); in emitPreCond()
2284 CGF.EmitBranchOnBoolExpr(Cond, TrueBlock, FalseBlock, TrueCount); in emitPreCond()
2285 PreCondVars.restore(CGF); in emitPreCond()
2323 static void emitSimdlenSafelenClause(CodeGenFunction &CGF, in emitSimdlenSafelenClause() argument
2326 if (!CGF.HaveInsertPoint()) in emitSimdlenSafelenClause()
2329 RValue Len = CGF.EmitAnyExpr(C->getSimdlen(), AggValueSlot::ignored(), in emitSimdlenSafelenClause()
2332 CGF.LoopStack.setVectorizeWidth(Val->getZExtValue()); in emitSimdlenSafelenClause()
2337 CGF.LoopStack.setParallel(!D.getSingleClause<OMPSafelenClause>()); in emitSimdlenSafelenClause()
2339 RValue Len = CGF.EmitAnyExpr(C->getSafelen(), AggValueSlot::ignored(), in emitSimdlenSafelenClause()
2342 CGF.LoopStack.setVectorizeWidth(Val->getZExtValue()); in emitSimdlenSafelenClause()
2346 CGF.LoopStack.setParallel(/*Enable=*/false); in emitSimdlenSafelenClause()
2416 static void emitOMPLoopBodyWithStopPoint(CodeGenFunction &CGF, in emitOMPLoopBodyWithStopPoint() argument
2419 CGF.EmitOMPLoopBody(S, LoopExit); in emitOMPLoopBodyWithStopPoint()
2420 CGF.EmitStopPoint(&S); in emitOMPLoopBodyWithStopPoint()
2424 static LValue EmitOMPHelperVar(CodeGenFunction &CGF, in EmitOMPHelperVar() argument
2427 CGF.EmitVarDecl(*VDecl); in EmitOMPHelperVar()
2428 return CGF.EmitLValue(Helper); in EmitOMPHelperVar()
2431 static void emitCommonSimdLoop(CodeGenFunction &CGF, const OMPLoopDirective &S, in emitCommonSimdLoop() argument
2434 auto &&ThenGen = [&S, &SimdInitGen, &BodyCodeGen](CodeGenFunction &CGF, in emitCommonSimdLoop()
2436 CGOpenMPRuntime::NontemporalDeclsRAII NontemporalsRegion(CGF.CGM, S); in emitCommonSimdLoop()
2437 CodeGenFunction::OMPLocalDeclMapRAII Scope(CGF); in emitCommonSimdLoop()
2438 SimdInitGen(CGF); in emitCommonSimdLoop()
2440 BodyCodeGen(CGF); in emitCommonSimdLoop()
2442 auto &&ElseGen = [&BodyCodeGen](CodeGenFunction &CGF, PrePostActionTy &) { in emitCommonSimdLoop() argument
2443 CodeGenFunction::OMPLocalDeclMapRAII Scope(CGF); in emitCommonSimdLoop()
2444 CGF.LoopStack.setVectorizeEnable(/*Enable=*/false); in emitCommonSimdLoop()
2446 BodyCodeGen(CGF); in emitCommonSimdLoop()
2451 if (CGF.getLangOpts().OpenMP >= 50 && in emitCommonSimdLoop()
2460 CGF.CGM.getOpenMPRuntime().emitIfClause(CGF, IfCond, ThenGen, ElseGen); in emitCommonSimdLoop()
2463 ThenRCG(CGF); in emitCommonSimdLoop()
2467 static void emitOMPSimdRegion(CodeGenFunction &CGF, const OMPLoopDirective &S, in emitOMPSimdRegion() argument
2469 Action.Enter(CGF); in emitOMPSimdRegion()
2472 OMPLoopScope PreInitScope(CGF, S); in emitOMPSimdRegion()
2481 (void)EmitOMPHelperVar(CGF, cast<DeclRefExpr>(S.getLowerBoundVariable())); in emitOMPSimdRegion()
2482 (void)EmitOMPHelperVar(CGF, cast<DeclRefExpr>(S.getUpperBoundVariable())); in emitOMPSimdRegion()
2490 if (CGF.ConstantFoldsToSimpleInteger(S.getPreCond(), CondConstant)) { in emitOMPSimdRegion()
2494 llvm::BasicBlock *ThenBlock = CGF.createBasicBlock("simd.if.then"); in emitOMPSimdRegion()
2495 ContBlock = CGF.createBasicBlock("simd.if.end"); in emitOMPSimdRegion()
2496 emitPreCond(CGF, S, S.getPreCond(), ThenBlock, ContBlock, in emitOMPSimdRegion()
2497 CGF.getProfileCount(&S)); in emitOMPSimdRegion()
2498 CGF.EmitBlock(ThenBlock); in emitOMPSimdRegion()
2499 CGF.incrementProfileCounter(&S); in emitOMPSimdRegion()
2505 CGF.EmitVarDecl(*IVDecl); in emitOMPSimdRegion()
2506 CGF.EmitIgnoredExpr(S.getInit()); in emitOMPSimdRegion()
2512 CGF.EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl())); in emitOMPSimdRegion()
2514 CGF.EmitIgnoredExpr(S.getCalcLastIteration()); in emitOMPSimdRegion()
2517 emitAlignedClause(CGF, S); in emitOMPSimdRegion()
2518 (void)CGF.EmitOMPLinearClauseInit(S); in emitOMPSimdRegion()
2520 CodeGenFunction::OMPPrivateScope LoopScope(CGF); in emitOMPSimdRegion()
2521 CGF.EmitOMPPrivateLoopCounters(S, LoopScope); in emitOMPSimdRegion()
2522 CGF.EmitOMPLinearClause(S, LoopScope); in emitOMPSimdRegion()
2523 CGF.EmitOMPPrivateClause(S, LoopScope); in emitOMPSimdRegion()
2524 CGF.EmitOMPReductionClauseInit(S, LoopScope); in emitOMPSimdRegion()
2526 CGF, S, CGF.EmitLValue(S.getIterationVariable())); in emitOMPSimdRegion()
2527 bool HasLastprivateClause = CGF.EmitOMPLastprivateClauseInit(S, LoopScope); in emitOMPSimdRegion()
2530 CGF.CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(CGF, S); in emitOMPSimdRegion()
2533 CGF, S, in emitOMPSimdRegion()
2534 [&S](CodeGenFunction &CGF, PrePostActionTy &) { in emitOMPSimdRegion() argument
2535 CGF.EmitOMPSimdInit(S); in emitOMPSimdRegion()
2537 [&S, &LoopScope](CodeGenFunction &CGF, PrePostActionTy &) { in emitOMPSimdRegion() argument
2538 CGF.EmitOMPInnerLoop( in emitOMPSimdRegion()
2540 [&S](CodeGenFunction &CGF) { in emitOMPSimdRegion() argument
2541 emitOMPLoopBodyWithStopPoint(CGF, S, in emitOMPSimdRegion()
2546 CGF.EmitOMPSimdFinal(S, [](CodeGenFunction &) { return nullptr; }); in emitOMPSimdRegion()
2549 CGF.EmitOMPLastprivateClauseFinal(S, /*NoFinals=*/true); in emitOMPSimdRegion()
2550 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_simd); in emitOMPSimdRegion()
2551 emitPostUpdateForReductionClause(CGF, S, in emitOMPSimdRegion()
2554 CGF.EmitOMPLinearClauseFinal(S, [](CodeGenFunction &) { return nullptr; }); in emitOMPSimdRegion()
2557 CGF.EmitBranch(ContBlock); in emitOMPSimdRegion()
2558 CGF.EmitBlock(ContBlock, true); in emitOMPSimdRegion()
2565 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPSimdDirective() argument
2566 emitOMPSimdRegion(CGF, S, Action); in EmitOMPSimdDirective()
2647 [&S, IsMonotonic](CodeGenFunction &CGF, PrePostActionTy &) { in EmitOMPOuterLoop() argument
2651 CGF.LoopStack.setParallel(!IsMonotonic); in EmitOMPOuterLoop()
2654 CGF.LoopStack.setParallel(/*Enable=*/true); in EmitOMPOuterLoop()
2656 CGF.EmitOMPSimdInit(S, IsMonotonic); in EmitOMPOuterLoop()
2660 &LoopScope](CodeGenFunction &CGF, PrePostActionTy &) { in EmitOMPOuterLoop() argument
2667 CGF.EmitOMPInnerLoop( in EmitOMPOuterLoop()
2669 [&S, LoopExit, &CodeGenLoop](CodeGenFunction &CGF) { in EmitOMPOuterLoop() argument
2670 CodeGenLoop(CGF, S, LoopExit); in EmitOMPOuterLoop()
2672 [IVSize, IVSigned, Loc, &CodeGenOrdered](CodeGenFunction &CGF) { in EmitOMPOuterLoop() argument
2673 CodeGenOrdered(CGF, Loc, IVSize, IVSigned); in EmitOMPOuterLoop()
2692 auto &&CodeGen = [DynamicOrOrdered, &S](CodeGenFunction &CGF) { in EmitOMPOuterLoop() argument
2694 CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getEndLoc(), in EmitOMPOuterLoop()
2787 auto &&CodeGenOrdered = [Ordered](CodeGenFunction &CGF, SourceLocation Loc, in EmitOMPForOuterLoop()
2791 CGF.CGM.getOpenMPRuntime().emitForOrderedIterationEnd(CGF, Loc, IVSize, in EmitOMPForOuterLoop()
2871 emitDistributeParallelForInnerBounds(CodeGenFunction &CGF, in emitDistributeParallelForInnerBounds() argument
2875 EmitOMPHelperVar(CGF, cast<DeclRefExpr>(LS.getLowerBoundVariable())); in emitDistributeParallelForInnerBounds()
2877 EmitOMPHelperVar(CGF, cast<DeclRefExpr>(LS.getUpperBoundVariable())); in emitDistributeParallelForInnerBounds()
2885 LValue PrevLB = CGF.EmitLValue(LS.getPrevLowerBoundVariable()); in emitDistributeParallelForInnerBounds()
2886 LValue PrevUB = CGF.EmitLValue(LS.getPrevUpperBoundVariable()); in emitDistributeParallelForInnerBounds()
2887 llvm::Value *PrevLBVal = CGF.EmitLoadOfScalar( in emitDistributeParallelForInnerBounds()
2889 PrevLBVal = CGF.EmitScalarConversion( in emitDistributeParallelForInnerBounds()
2893 llvm::Value *PrevUBVal = CGF.EmitLoadOfScalar( in emitDistributeParallelForInnerBounds()
2895 PrevUBVal = CGF.EmitScalarConversion( in emitDistributeParallelForInnerBounds()
2900 CGF.EmitStoreOfScalar(PrevLBVal, LB); in emitDistributeParallelForInnerBounds()
2901 CGF.EmitStoreOfScalar(PrevUBVal, UB); in emitDistributeParallelForInnerBounds()
2914 emitDistributeParallelForDispatchBounds(CodeGenFunction &CGF, in emitDistributeParallelForDispatchBounds() argument
2925 CGF.EmitLoadOfScalar(LB, /*Volatile=*/false, IteratorTy, S.getBeginLoc()); in emitDistributeParallelForDispatchBounds()
2927 CGF.EmitLoadOfScalar(UB, /*Volatile=*/false, IteratorTy, S.getBeginLoc()); in emitDistributeParallelForDispatchBounds()
2932 CodeGenFunction &CGF, const OMPExecutableDirective &S, in emitDistributeParallelForDistributeInnerBoundParams() argument
2936 CGF.EmitLValue(cast<DeclRefExpr>(Dir.getCombinedLowerBoundVariable())); in emitDistributeParallelForDistributeInnerBoundParams()
2938 CGF.Builder.CreateIntCast(CGF.Builder.CreateLoad(LB.getAddress(CGF)), in emitDistributeParallelForDistributeInnerBoundParams()
2939 CGF.SizeTy, /*isSigned=*/false); in emitDistributeParallelForDistributeInnerBoundParams()
2942 CGF.EmitLValue(cast<DeclRefExpr>(Dir.getCombinedUpperBoundVariable())); in emitDistributeParallelForDistributeInnerBoundParams()
2945 CGF.Builder.CreateIntCast(CGF.Builder.CreateLoad(UB.getAddress(CGF)), in emitDistributeParallelForDistributeInnerBoundParams()
2946 CGF.SizeTy, /*isSigned=*/false); in emitDistributeParallelForDistributeInnerBoundParams()
2951 emitInnerParallelForWhenCombined(CodeGenFunction &CGF, in emitInnerParallelForWhenCombined() argument
2954 auto &&CGInlinedWorksharingLoop = [&S](CodeGenFunction &CGF, in emitInnerParallelForWhenCombined()
2956 Action.Enter(CGF); in emitInnerParallelForWhenCombined()
2967 CodeGenFunction::OMPCancelStackRAII CancelRegion(CGF, S.getDirectiveKind(), in emitInnerParallelForWhenCombined()
2969 CGF.EmitOMPWorksharingLoop(S, S.getPrevEnsureUpperBound(), in emitInnerParallelForWhenCombined()
2975 CGF, S, in emitInnerParallelForWhenCombined()
2983 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) { in EmitOMPDistributeParallelForDirective() argument
2984 CGF.EmitOMPDistributeLoop(S, emitInnerParallelForWhenCombined, in EmitOMPDistributeParallelForDirective()
2993 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) { in EmitOMPDistributeParallelForSimdDirective() argument
2994 CGF.EmitOMPDistributeLoop(S, emitInnerParallelForWhenCombined, in EmitOMPDistributeParallelForSimdDirective()
3003 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) { in EmitOMPDistributeSimdDirective() argument
3004 CGF.EmitOMPDistributeLoop(S, emitOMPLoopBodyWithStopPoint, S.getInc()); in EmitOMPDistributeSimdDirective()
3013 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTargetSimdDeviceFunction() argument
3014 emitOMPSimdRegion(CGF, S, Action); in EmitOMPTargetSimdDeviceFunction()
3026 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTargetSimdDirective() argument
3027 emitOMPSimdRegion(CGF, S, Action); in EmitOMPTargetSimdDirective()
3180 [&S, IsMonotonic](CodeGenFunction &CGF, PrePostActionTy &) { in EmitOMPWorksharingLoop() argument
3182 CGF.EmitOMPSimdInit(S, IsMonotonic); in EmitOMPWorksharingLoop()
3185 CGF.LoopStack.setParallel(/*Enable=*/true); in EmitOMPWorksharingLoop()
3190 &LoopScope](CodeGenFunction &CGF, PrePostActionTy &) { in EmitOMPWorksharingLoop() argument
3197 IVSize, IVSigned, Ordered, IL.getAddress(CGF), in EmitOMPWorksharingLoop()
3198 LB.getAddress(CGF), UB.getAddress(CGF), ST.getAddress(CGF), in EmitOMPWorksharingLoop()
3200 CGF.CGM.getOpenMPRuntime().emitForStaticInit( in EmitOMPWorksharingLoop()
3201 CGF, S.getBeginLoc(), S.getDirectiveKind(), ScheduleKind, in EmitOMPWorksharingLoop()
3205 CGF.EmitIgnoredExpr(S.getEnsureUpperBound()); in EmitOMPWorksharingLoop()
3207 CGF.EmitIgnoredExpr(S.getInit()); in EmitOMPWorksharingLoop()
3221 CGF.EmitOMPInnerLoop( in EmitOMPWorksharingLoop()
3226 [&S, LoopExit](CodeGenFunction &CGF) { in EmitOMPWorksharingLoop() argument
3227 emitOMPLoopBodyWithStopPoint(CGF, S, LoopExit); in EmitOMPWorksharingLoop()
3233 auto &&CodeGen = [&S](CodeGenFunction &CGF) { in EmitOMPWorksharingLoop() argument
3234 CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getEndLoc(), in EmitOMPWorksharingLoop()
3248 EmitOMPSimdFinal(S, [IL, &S](CodeGenFunction &CGF) { in EmitOMPWorksharingLoop() argument
3249 return CGF.Builder.CreateIsNotNull( in EmitOMPWorksharingLoop()
3250 CGF.EmitLoadOfScalar(IL, S.getBeginLoc())); in EmitOMPWorksharingLoop()
3259 *this, S, [IL, &S](CodeGenFunction &CGF) { in EmitOMPWorksharingLoop() argument
3260 return CGF.Builder.CreateIsNotNull( in EmitOMPWorksharingLoop()
3261 CGF.EmitLoadOfScalar(IL, S.getBeginLoc())); in EmitOMPWorksharingLoop()
3269 EmitOMPLinearClauseFinal(S, [IL, &S](CodeGenFunction &CGF) { in EmitOMPWorksharingLoop() argument
3270 return CGF.Builder.CreateIsNotNull( in EmitOMPWorksharingLoop()
3271 CGF.EmitLoadOfScalar(IL, S.getBeginLoc())); in EmitOMPWorksharingLoop()
3287 emitForLoopBounds(CodeGenFunction &CGF, const OMPExecutableDirective &S) { in emitForLoopBounds() argument
3290 EmitOMPHelperVar(CGF, cast<DeclRefExpr>(LS.getLowerBoundVariable())); in emitForLoopBounds()
3292 EmitOMPHelperVar(CGF, cast<DeclRefExpr>(LS.getUpperBoundVariable())); in emitForLoopBounds()
3301 emitDispatchForLoopBounds(CodeGenFunction &CGF, const OMPExecutableDirective &S, in emitDispatchForLoopBounds() argument
3305 const unsigned IVSize = CGF.getContext().getTypeSize(IVExpr->getType()); in emitDispatchForLoopBounds()
3306 llvm::Value *LBVal = CGF.Builder.getIntN(IVSize, 0); in emitDispatchForLoopBounds()
3307 llvm::Value *UBVal = CGF.EmitScalarExpr(LS.getLastIteration()); in emitDispatchForLoopBounds()
3319 CodeGenFunction &CGF, const OMPLoopDirective &S, in emitScanBasedDirectiveDecls() argument
3321 llvm::Value *OMPScanNumIterations = CGF.Builder.CreateIntCast( in emitScanBasedDirectiveDecls()
3322 NumIteratorsGen(CGF), CGF.SizeTy, /*isSigned=*/false); in emitScanBasedDirectiveDecls()
3348 RedCG.emitSharedOrigLValue(CGF, Count); in emitScanBasedDirectiveDecls()
3349 RedCG.emitAggregateType(CGF, Count); in emitScanBasedDirectiveDecls()
3352 CGF, in emitScanBasedDirectiveDecls()
3358 CGF.EmitVarDecl(*cast<VarDecl>(cast<DeclRefExpr>(*ITA)->getDecl())); in emitScanBasedDirectiveDecls()
3385 CodeGenFunction &CGF, const OMPLoopDirective &S, in emitScanBasedDirective() argument
3389 llvm::Value *OMPScanNumIterations = CGF.Builder.CreateIntCast( in emitScanBasedDirective()
3390 NumIteratorsGen(CGF), CGF.SizeTy, /*isSigned=*/false); in emitScanBasedDirective()
3406 CodeGenFunction::ParentLoopDirectiveForScanRegion ScanRegion(CGF, S); in emitScanBasedDirective()
3414 CGF.OMPFirstScanLoop = true; in emitScanBasedDirective()
3415 CodeGenFunction::OMPLocalDeclMapRAII Scope(CGF); in emitScanBasedDirective()
3416 FirstGen(CGF); in emitScanBasedDirective()
3421 &Privates](CodeGenFunction &CGF, PrePostActionTy &Action) { in emitScanBasedDirective() argument
3422 Action.Enter(CGF); in emitScanBasedDirective()
3426 llvm::BasicBlock *InputBB = CGF.Builder.GetInsertBlock(); in emitScanBasedDirective()
3427 llvm::BasicBlock *LoopBB = CGF.createBasicBlock("omp.outer.log.scan.body"); in emitScanBasedDirective()
3428 llvm::BasicBlock *ExitBB = CGF.createBasicBlock("omp.outer.log.scan.exit"); in emitScanBasedDirective()
3430 CGF.CGM.getIntrinsic(llvm::Intrinsic::log2, CGF.DoubleTy); in emitScanBasedDirective()
3432 CGF.Builder.CreateUIToFP(OMPScanNumIterations, CGF.DoubleTy); in emitScanBasedDirective()
3433 llvm::Value *LogVal = CGF.EmitNounwindRuntimeCall(F, Arg); in emitScanBasedDirective()
3434 F = CGF.CGM.getIntrinsic(llvm::Intrinsic::ceil, CGF.DoubleTy); in emitScanBasedDirective()
3435 LogVal = CGF.EmitNounwindRuntimeCall(F, LogVal); in emitScanBasedDirective()
3436 LogVal = CGF.Builder.CreateFPToUI(LogVal, CGF.IntTy); in emitScanBasedDirective()
3437 llvm::Value *NMin1 = CGF.Builder.CreateNUWSub( in emitScanBasedDirective()
3438 OMPScanNumIterations, llvm::ConstantInt::get(CGF.SizeTy, 1)); in emitScanBasedDirective()
3439 auto DL = ApplyDebugLocation::CreateDefaultArtificial(CGF, S.getBeginLoc()); in emitScanBasedDirective()
3440 CGF.EmitBlock(LoopBB); in emitScanBasedDirective()
3441 auto *Counter = CGF.Builder.CreatePHI(CGF.IntTy, 2); in emitScanBasedDirective()
3443 auto *Pow2K = CGF.Builder.CreatePHI(CGF.SizeTy, 2); in emitScanBasedDirective()
3444 Counter->addIncoming(llvm::ConstantInt::get(CGF.IntTy, 0), InputBB); in emitScanBasedDirective()
3445 Pow2K->addIncoming(llvm::ConstantInt::get(CGF.SizeTy, 1), InputBB); in emitScanBasedDirective()
3449 CGF.createBasicBlock("omp.inner.log.scan.body"); in emitScanBasedDirective()
3451 CGF.createBasicBlock("omp.inner.log.scan.exit"); in emitScanBasedDirective()
3452 llvm::Value *CmpI = CGF.Builder.CreateICmpUGE(NMin1, Pow2K); in emitScanBasedDirective()
3453 CGF.Builder.CreateCondBr(CmpI, InnerLoopBB, InnerExitBB); in emitScanBasedDirective()
3454 CGF.EmitBlock(InnerLoopBB); in emitScanBasedDirective()
3455 auto *IVal = CGF.Builder.CreatePHI(CGF.SizeTy, 2); in emitScanBasedDirective()
3458 CodeGenFunction::OMPPrivateScope PrivScope(CGF); in emitScanBasedDirective()
3467 CGF, in emitScanBasedDirective()
3471 LHSAddr = CGF.EmitLValue(CopyArrayElem).getAddress(CGF); in emitScanBasedDirective()
3476 llvm::Value *OffsetIVal = CGF.Builder.CreateNUWSub(IVal, Pow2K); in emitScanBasedDirective()
3478 CGF, in emitScanBasedDirective()
3482 RHSAddr = CGF.EmitLValue(CopyArrayElem).getAddress(CGF); in emitScanBasedDirective()
3489 CGF.CGM.getOpenMPRuntime().emitReduction( in emitScanBasedDirective()
3490 CGF, S.getEndLoc(), Privates, LHSs, RHSs, ReductionOps, in emitScanBasedDirective()
3494 CGF.Builder.CreateNUWSub(IVal, llvm::ConstantInt::get(CGF.SizeTy, 1)); in emitScanBasedDirective()
3495 IVal->addIncoming(NextIVal, CGF.Builder.GetInsertBlock()); in emitScanBasedDirective()
3496 CmpI = CGF.Builder.CreateICmpUGE(NextIVal, Pow2K); in emitScanBasedDirective()
3497 CGF.Builder.CreateCondBr(CmpI, InnerLoopBB, InnerExitBB); in emitScanBasedDirective()
3498 CGF.EmitBlock(InnerExitBB); in emitScanBasedDirective()
3500 CGF.Builder.CreateNUWAdd(Counter, llvm::ConstantInt::get(CGF.IntTy, 1)); in emitScanBasedDirective()
3501 Counter->addIncoming(Next, CGF.Builder.GetInsertBlock()); in emitScanBasedDirective()
3504 CGF.Builder.CreateShl(Pow2K, 1, "", /*HasNUW=*/true); in emitScanBasedDirective()
3505 Pow2K->addIncoming(NextPow2K, CGF.Builder.GetInsertBlock()); in emitScanBasedDirective()
3506 llvm::Value *Cmp = CGF.Builder.CreateICmpNE(Next, LogVal); in emitScanBasedDirective()
3507 CGF.Builder.CreateCondBr(Cmp, LoopBB, ExitBB); in emitScanBasedDirective()
3508 auto DL1 = ApplyDebugLocation::CreateDefaultArtificial(CGF, S.getEndLoc()); in emitScanBasedDirective()
3509 CGF.EmitBlock(ExitBB); in emitScanBasedDirective()
3512 CGF.CGM.getOpenMPRuntime().emitMasterRegion(CGF, CodeGen, S.getBeginLoc()); in emitScanBasedDirective()
3513 CGF.CGM.getOpenMPRuntime().emitBarrierCall( in emitScanBasedDirective()
3514 CGF, S.getBeginLoc(), OMPD_unknown, /*EmitChecks=*/false, in emitScanBasedDirective()
3518 RCG(CGF); in emitScanBasedDirective()
3521 CGF.OMPFirstScanLoop = false; in emitScanBasedDirective()
3522 SecondGen(CGF); in emitScanBasedDirective()
3525 static bool emitWorksharingDirective(CodeGenFunction &CGF, in emitWorksharingDirective() argument
3533 const auto &&NumIteratorsGen = [&S](CodeGenFunction &CGF) { in emitWorksharingDirective() argument
3534 CodeGenFunction::OMPLocalDeclMapRAII Scope(CGF); in emitWorksharingDirective()
3535 OMPLoopScope LoopScope(CGF, S); in emitWorksharingDirective()
3536 return CGF.EmitScalarExpr(S.getNumIterations()); in emitWorksharingDirective()
3538 const auto &&FirstGen = [&S, HasCancel](CodeGenFunction &CGF) { in emitWorksharingDirective() argument
3540 CGF, S.getDirectiveKind(), HasCancel); in emitWorksharingDirective()
3541 (void)CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(), in emitWorksharingDirective()
3545 CGF.CGM.getOpenMPRuntime().emitBarrierCall(CGF, S.getBeginLoc(), in emitWorksharingDirective()
3549 &HasLastprivates](CodeGenFunction &CGF) { in emitWorksharingDirective() argument
3551 CGF, S.getDirectiveKind(), HasCancel); in emitWorksharingDirective()
3552 HasLastprivates = CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(), in emitWorksharingDirective()
3557 emitScanBasedDirectiveDecls(CGF, S, NumIteratorsGen); in emitWorksharingDirective()
3558 emitScanBasedDirective(CGF, S, NumIteratorsGen, FirstGen, SecondGen); in emitWorksharingDirective()
3560 CodeGenFunction::OMPCancelStackRAII CancelRegion(CGF, S.getDirectiveKind(), in emitWorksharingDirective()
3562 HasLastprivates = CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(), in emitWorksharingDirective()
3584 UseOMPIRBuilder](CodeGenFunction &CGF, PrePostActionTy &) { in EmitOMPForDirective() argument
3601 HasLastprivates = emitWorksharingDirective(CGF, S, S.hasCancel()); in EmitOMPForDirective()
3622 auto &&CodeGen = [&S, &HasLastprivates](CodeGenFunction &CGF, in EmitOMPForSimdDirective()
3624 HasLastprivates = emitWorksharingDirective(CGF, S, /*HasCancel=*/false); in EmitOMPForSimdDirective()
3640 static LValue createSectionLVal(CodeGenFunction &CGF, QualType Ty, in createSectionLVal() argument
3643 LValue LVal = CGF.MakeAddrLValue(CGF.CreateMemTemp(Ty, Name), Ty); in createSectionLVal()
3645 CGF.EmitStoreThroughLValue(RValue::get(Init), LVal, /*isInit*/ true); in createSectionLVal()
3654 &HasLastprivates](CodeGenFunction &CGF, PrePostActionTy &) { in EmitSections() argument
3655 const ASTContext &C = CGF.getContext(); in EmitSections()
3659 LValue LB = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.lb.", in EmitSections()
3660 CGF.Builder.getInt32(0)); in EmitSections()
3662 ? CGF.Builder.getInt32(CS->size() - 1) in EmitSections()
3663 : CGF.Builder.getInt32(0); in EmitSections()
3665 createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.ub.", GlobalUBVal); in EmitSections()
3666 LValue ST = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.st.", in EmitSections()
3667 CGF.Builder.getInt32(1)); in EmitSections()
3668 LValue IL = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.il.", in EmitSections()
3669 CGF.Builder.getInt32(0)); in EmitSections()
3671 LValue IV = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.iv."); in EmitSections()
3673 CodeGenFunction::OpaqueValueMapping OpaqueIV(CGF, &IVRefExpr, IV); in EmitSections()
3675 CodeGenFunction::OpaqueValueMapping OpaqueUB(CGF, &UBRefExpr, UB); in EmitSections()
3684 auto &&BodyGen = [CapturedStmt, CS, &S, &IV](CodeGenFunction &CGF) { in EmitSections() argument
3696 llvm::BasicBlock *ExitBB = CGF.createBasicBlock(".omp.sections.exit"); in EmitSections()
3698 CGF.Builder.CreateSwitch(CGF.EmitLoadOfScalar(IV, S.getBeginLoc()), in EmitSections()
3703 auto CaseBB = CGF.createBasicBlock(".omp.sections.case"); in EmitSections()
3704 CGF.EmitBlock(CaseBB); in EmitSections()
3705 SwitchStmt->addCase(CGF.Builder.getInt32(CaseNumber), CaseBB); in EmitSections()
3706 CGF.EmitStmt(SubStmt); in EmitSections()
3707 CGF.EmitBranch(ExitBB); in EmitSections()
3711 llvm::BasicBlock *CaseBB = CGF.createBasicBlock(".omp.sections.case"); in EmitSections()
3712 CGF.EmitBlock(CaseBB); in EmitSections()
3713 SwitchStmt->addCase(CGF.Builder.getInt32(0), CaseBB); in EmitSections()
3714 CGF.EmitStmt(CapturedStmt); in EmitSections()
3715 CGF.EmitBranch(ExitBB); in EmitSections()
3717 CGF.EmitBlock(ExitBB, /*IsFinished=*/true); in EmitSections()
3720 CodeGenFunction::OMPPrivateScope LoopScope(CGF); in EmitSections()
3721 if (CGF.EmitOMPFirstprivateClause(S, LoopScope)) { in EmitSections()
3725 CGF.CGM.getOpenMPRuntime().emitBarrierCall( in EmitSections()
3726 CGF, S.getBeginLoc(), OMPD_unknown, /*EmitChecks=*/false, in EmitSections()
3729 CGF.EmitOMPPrivateClause(S, LoopScope); in EmitSections()
3730 CGOpenMPRuntime::LastprivateConditionalRAII LPCRegion(CGF, S, IV); in EmitSections()
3731 HasLastprivates = CGF.EmitOMPLastprivateClauseInit(S, LoopScope); in EmitSections()
3732 CGF.EmitOMPReductionClauseInit(S, LoopScope); in EmitSections()
3735 CGF.CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(CGF, S); in EmitSections()
3741 /*IVSize=*/32, /*IVSigned=*/true, /*Ordered=*/false, IL.getAddress(CGF), in EmitSections()
3742 LB.getAddress(CGF), UB.getAddress(CGF), ST.getAddress(CGF)); in EmitSections()
3743 CGF.CGM.getOpenMPRuntime().emitForStaticInit( in EmitSections()
3744 CGF, S.getBeginLoc(), S.getDirectiveKind(), ScheduleKind, StaticInit); in EmitSections()
3746 llvm::Value *UBVal = CGF.EmitLoadOfScalar(UB, S.getBeginLoc()); in EmitSections()
3747 llvm::Value *MinUBGlobalUB = CGF.Builder.CreateSelect( in EmitSections()
3748 CGF.Builder.CreateICmpSLT(UBVal, GlobalUBVal), UBVal, GlobalUBVal); in EmitSections()
3749 CGF.EmitStoreOfScalar(MinUBGlobalUB, UB); in EmitSections()
3751 CGF.EmitStoreOfScalar(CGF.EmitLoadOfScalar(LB, S.getBeginLoc()), IV); in EmitSections()
3753 CGF.EmitOMPInnerLoop(S, /*RequiresCleanup=*/false, Cond, Inc, BodyGen, in EmitSections()
3756 auto &&CodeGen = [&S](CodeGenFunction &CGF) { in EmitSections() argument
3757 CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getEndLoc(), in EmitSections()
3760 CGF.OMPCancelStack.emitExit(CGF, S.getDirectiveKind(), CodeGen); in EmitSections()
3761 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_parallel); in EmitSections()
3763 emitPostUpdateForReductionClause(CGF, S, [IL, &S](CodeGenFunction &CGF) { in EmitSections() argument
3764 return CGF.Builder.CreateIsNotNull( in EmitSections()
3765 CGF.EmitLoadOfScalar(IL, S.getBeginLoc())); in EmitSections()
3770 CGF.EmitOMPLastprivateClauseFinal( in EmitSections()
3772 CGF.Builder.CreateIsNotNull( in EmitSections()
3773 CGF.EmitLoadOfScalar(IL, S.getBeginLoc()))); in EmitSections()
3916 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPSingleDirective() argument
3917 Action.Enter(CGF); in EmitOMPSingleDirective()
3918 OMPPrivateScope SingleScope(CGF); in EmitOMPSingleDirective()
3919 (void)CGF.EmitOMPFirstprivateClause(S, SingleScope); in EmitOMPSingleDirective()
3920 CGF.EmitOMPPrivateClause(S, SingleScope); in EmitOMPSingleDirective()
3922 CGF.EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt()); in EmitOMPSingleDirective()
3943 static void emitMaster(CodeGenFunction &CGF, const OMPExecutableDirective &S) { in emitMaster() argument
3944 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in emitMaster() argument
3945 Action.Enter(CGF); in emitMaster()
3946 CGF.EmitStmt(S.getRawStmt()); in emitMaster()
3948 CGF.CGM.getOpenMPRuntime().emitMasterRegion(CGF, CodeGen, S.getBeginLoc()); in emitMaster()
3981 static void emitMasked(CodeGenFunction &CGF, const OMPExecutableDirective &S) { in emitMasked() argument
3982 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in emitMasked() argument
3983 Action.Enter(CGF); in emitMasked()
3984 CGF.EmitStmt(S.getRawStmt()); in emitMasked()
3989 CGF.CGM.getOpenMPRuntime().emitMaskedRegion(CGF, CodeGen, S.getBeginLoc(), in emitMasked()
4069 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPCriticalDirective() argument
4070 Action.Enter(CGF); in EmitOMPCriticalDirective()
4071 CGF.EmitStmt(S.getAssociatedStmt()); in EmitOMPCriticalDirective()
4087 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPParallelForDirective() argument
4088 Action.Enter(CGF); in EmitOMPParallelForDirective()
4089 (void)emitWorksharingDirective(CGF, S, S.hasCancel()); in EmitOMPParallelForDirective()
4096 const auto &&NumIteratorsGen = [&S](CodeGenFunction &CGF) { in EmitOMPParallelForDirective() argument
4097 CodeGenFunction::OMPLocalDeclMapRAII Scope(CGF); in EmitOMPParallelForDirective()
4099 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGSI); in EmitOMPParallelForDirective()
4100 OMPLoopScope LoopScope(CGF, S); in EmitOMPParallelForDirective()
4101 return CGF.EmitScalarExpr(S.getNumIterations()); in EmitOMPParallelForDirective()
4118 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPParallelForSimdDirective() argument
4119 Action.Enter(CGF); in EmitOMPParallelForSimdDirective()
4120 (void)emitWorksharingDirective(CGF, S, /*HasCancel=*/false); in EmitOMPParallelForSimdDirective()
4127 const auto &&NumIteratorsGen = [&S](CodeGenFunction &CGF) { in EmitOMPParallelForSimdDirective() argument
4128 CodeGenFunction::OMPLocalDeclMapRAII Scope(CGF); in EmitOMPParallelForSimdDirective()
4130 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGSI); in EmitOMPParallelForSimdDirective()
4131 OMPLoopScope LoopScope(CGF, S); in EmitOMPParallelForSimdDirective()
4132 return CGF.EmitScalarExpr(S.getNumIterations()); in EmitOMPParallelForSimdDirective()
4149 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPParallelMasterDirective() argument
4150 Action.Enter(CGF); in EmitOMPParallelMasterDirective()
4151 OMPPrivateScope PrivateScope(CGF); in EmitOMPParallelMasterDirective()
4152 bool Copyins = CGF.EmitOMPCopyinClause(S); in EmitOMPParallelMasterDirective()
4153 (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope); in EmitOMPParallelMasterDirective()
4158 CGF.CGM.getOpenMPRuntime().emitBarrierCall( in EmitOMPParallelMasterDirective()
4159 CGF, S.getBeginLoc(), OMPD_unknown, /*EmitChecks=*/false, in EmitOMPParallelMasterDirective()
4162 CGF.EmitOMPPrivateClause(S, PrivateScope); in EmitOMPParallelMasterDirective()
4163 CGF.EmitOMPReductionClauseInit(S, PrivateScope); in EmitOMPParallelMasterDirective()
4165 emitMaster(CGF, S); in EmitOMPParallelMasterDirective()
4166 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_parallel); in EmitOMPParallelMasterDirective()
4184 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPParallelSectionsDirective() argument
4185 Action.Enter(CGF); in EmitOMPParallelSectionsDirective()
4186 CGF.EmitSections(S); in EmitOMPParallelSectionsDirective()
4342 CapturedRegion](CodeGenFunction &CGF, in EmitOMPTaskBasedDirective()
4348 OMPPrivateScope Scope(CGF); in EmitOMPTaskBasedDirective()
4353 llvm::Value *CopyFn = CGF.Builder.CreateLoad( in EmitOMPTaskBasedDirective()
4354 CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(CopyFnParam))); in EmitOMPTaskBasedDirective()
4355 llvm::Value *PrivatesPtr = CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar( in EmitOMPTaskBasedDirective()
4365 Address PrivatePtr = CGF.CreateMemTemp( in EmitOMPTaskBasedDirective()
4366 CGF.getContext().getPointerType(E->getType()), ".priv.ptr.addr"); in EmitOMPTaskBasedDirective()
4374 CGF.CreateMemTemp(CGF.getContext().getPointerType(E->getType()), in EmitOMPTaskBasedDirective()
4384 CGF.CreateMemTemp(CGF.getContext().getPointerType(E->getType()), in EmitOMPTaskBasedDirective()
4393 Ty = CGF.getContext().getPointerType(Ty); in EmitOMPTaskBasedDirective()
4395 Ty = CGF.getContext().getPointerType(Ty); in EmitOMPTaskBasedDirective()
4396 Address PrivatePtr = CGF.CreateMemTemp( in EmitOMPTaskBasedDirective()
4397 CGF.getContext().getPointerType(Ty), ".local.ptr.addr"); in EmitOMPTaskBasedDirective()
4407 auto *CopyFnTy = llvm::FunctionType::get(CGF.Builder.getVoidTy(), in EmitOMPTaskBasedDirective()
4409 CopyFn = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in EmitOMPTaskBasedDirective()
4411 CGF.CGM.getOpenMPRuntime().emitOutlinedFunctionCall( in EmitOMPTaskBasedDirective()
4412 CGF, S.getBeginLoc(), {CopyFnTy, CopyFn}, CallArgs); in EmitOMPTaskBasedDirective()
4415 DeclRefExpr DRE(CGF.getContext(), const_cast<VarDecl *>(OrigVD), in EmitOMPTaskBasedDirective()
4417 CGF.CapturedStmtInfo->lookup(OrigVD) != nullptr, in EmitOMPTaskBasedDirective()
4420 Scope.addPrivate(Pair.first, [&CGF, &DRE]() { in EmitOMPTaskBasedDirective()
4421 return CGF.EmitLValue(&DRE).getAddress(CGF); in EmitOMPTaskBasedDirective()
4425 Address Replacement(CGF.Builder.CreateLoad(Pair.second), in EmitOMPTaskBasedDirective()
4426 CGF.getContext().getDeclAlign(Pair.first)); in EmitOMPTaskBasedDirective()
4433 llvm::Value *Ptr = CGF.Builder.CreateLoad(Pair.second.first); in EmitOMPTaskBasedDirective()
4434 Address Replacement(Ptr, CGF.getPointerAlign()); in EmitOMPTaskBasedDirective()
4436 Ptr = CGF.Builder.CreateLoad(Replacement); in EmitOMPTaskBasedDirective()
4437 Replacement = Address(Ptr, CGF.getContext().getDeclAlign(Pair.first)); in EmitOMPTaskBasedDirective()
4440 llvm::Value *Ptr = CGF.Builder.CreateLoad(Pair.second.first); in EmitOMPTaskBasedDirective()
4441 Address Replacement(Ptr, CGF.getContext().getDeclAlign(Pair.first)); in EmitOMPTaskBasedDirective()
4447 OMPPrivateScope FirstprivateScope(CGF); in EmitOMPTaskBasedDirective()
4449 Address Replacement(CGF.Builder.CreateLoad(Pair.second), in EmitOMPTaskBasedDirective()
4450 CGF.getContext().getDeclAlign(Pair.first)); in EmitOMPTaskBasedDirective()
4455 OMPLexicalScope LexScope(CGF, S, CapturedRegion); in EmitOMPTaskBasedDirective()
4458 llvm::Value *ReductionsPtr = CGF.Builder.CreateLoad( in EmitOMPTaskBasedDirective()
4459 CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(9))); in EmitOMPTaskBasedDirective()
4461 RedCG.emitSharedOrigLValue(CGF, Cnt); in EmitOMPTaskBasedDirective()
4462 RedCG.emitAggregateType(CGF, Cnt); in EmitOMPTaskBasedDirective()
4466 CGF.CGM.getOpenMPRuntime().emitTaskReductionFixups(CGF, S.getBeginLoc(), in EmitOMPTaskBasedDirective()
4468 Address Replacement = CGF.CGM.getOpenMPRuntime().getTaskReductionItem( in EmitOMPTaskBasedDirective()
4469 CGF, S.getBeginLoc(), ReductionsPtr, RedCG.getSharedLValue(Cnt)); in EmitOMPTaskBasedDirective()
4471 Address(CGF.EmitScalarConversion( in EmitOMPTaskBasedDirective()
4472 Replacement.getPointer(), CGF.getContext().VoidPtrTy, in EmitOMPTaskBasedDirective()
4473 CGF.getContext().getPointerType( in EmitOMPTaskBasedDirective()
4477 Replacement = RedCG.adjustPrivateAddress(CGF, Cnt, Replacement); in EmitOMPTaskBasedDirective()
4504 OMPPrivateScope InRedScope(CGF); in EmitOMPTaskBasedDirective()
4508 RedCG.emitSharedOrigLValue(CGF, Cnt); in EmitOMPTaskBasedDirective()
4509 RedCG.emitAggregateType(CGF, Cnt); in EmitOMPTaskBasedDirective()
4515 CGF.CGM.getOpenMPRuntime().emitTaskReductionFixups(CGF, S.getBeginLoc(), in EmitOMPTaskBasedDirective()
4519 ReductionsPtr = CGF.EmitLoadOfScalar(CGF.EmitLValue(TRExpr), in EmitOMPTaskBasedDirective()
4522 ReductionsPtr = llvm::ConstantPointerNull::get(CGF.VoidPtrTy); in EmitOMPTaskBasedDirective()
4524 Address Replacement = CGF.CGM.getOpenMPRuntime().getTaskReductionItem( in EmitOMPTaskBasedDirective()
4525 CGF, S.getBeginLoc(), ReductionsPtr, RedCG.getSharedLValue(Cnt)); in EmitOMPTaskBasedDirective()
4527 CGF.EmitScalarConversion( in EmitOMPTaskBasedDirective()
4528 Replacement.getPointer(), CGF.getContext().VoidPtrTy, in EmitOMPTaskBasedDirective()
4529 CGF.getContext().getPointerType(InRedPrivs[Cnt]->getType()), in EmitOMPTaskBasedDirective()
4532 Replacement = RedCG.adjustPrivateAddress(CGF, Cnt, Replacement); in EmitOMPTaskBasedDirective()
4539 CGOpenMPRuntime::UntiedTaskLocalDeclsRAII LocalVarsScope(CGF, in EmitOMPTaskBasedDirective()
4541 Action.Enter(CGF); in EmitOMPTaskBasedDirective()
4542 BodyGen(CGF); in EmitOMPTaskBasedDirective()
4654 &InputInfo](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTargetTaskBasedDirective() argument
4656 OMPPrivateScope Scope(CGF); in EmitOMPTargetTaskBasedDirective()
4659 llvm::Value *CopyFn = CGF.Builder.CreateLoad( in EmitOMPTargetTaskBasedDirective()
4660 CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(CopyFnParam))); in EmitOMPTargetTaskBasedDirective()
4661 llvm::Value *PrivatesPtr = CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar( in EmitOMPTargetTaskBasedDirective()
4672 CGF.CreateMemTemp(CGF.getContext().getPointerType(E->getType()), in EmitOMPTargetTaskBasedDirective()
4678 auto *CopyFnTy = llvm::FunctionType::get(CGF.Builder.getVoidTy(), in EmitOMPTargetTaskBasedDirective()
4680 CopyFn = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in EmitOMPTargetTaskBasedDirective()
4682 CGF.CGM.getOpenMPRuntime().emitOutlinedFunctionCall( in EmitOMPTargetTaskBasedDirective()
4683 CGF, S.getBeginLoc(), {CopyFnTy, CopyFn}, CallArgs); in EmitOMPTargetTaskBasedDirective()
4685 Address Replacement(CGF.Builder.CreateLoad(Pair.second), in EmitOMPTargetTaskBasedDirective()
4686 CGF.getContext().getDeclAlign(Pair.first)); in EmitOMPTargetTaskBasedDirective()
4693 InputInfo.BasePointersArray = CGF.Builder.CreateConstArrayGEP( in EmitOMPTargetTaskBasedDirective()
4694 CGF.GetAddrOfLocalVar(BPVD), /*Index=*/0); in EmitOMPTargetTaskBasedDirective()
4695 InputInfo.PointersArray = CGF.Builder.CreateConstArrayGEP( in EmitOMPTargetTaskBasedDirective()
4696 CGF.GetAddrOfLocalVar(PVD), /*Index=*/0); in EmitOMPTargetTaskBasedDirective()
4697 InputInfo.SizesArray = CGF.Builder.CreateConstArrayGEP( in EmitOMPTargetTaskBasedDirective()
4698 CGF.GetAddrOfLocalVar(SVD), /*Index=*/0); in EmitOMPTargetTaskBasedDirective()
4701 InputInfo.MappersArray = CGF.Builder.CreateConstArrayGEP( in EmitOMPTargetTaskBasedDirective()
4702 CGF.GetAddrOfLocalVar(MVD), /*Index=*/0); in EmitOMPTargetTaskBasedDirective()
4705 Action.Enter(CGF); in EmitOMPTargetTaskBasedDirective()
4706 OMPLexicalScope LexScope(CGF, S, OMPD_task, /*EmitPreInitStmt=*/false); in EmitOMPTargetTaskBasedDirective()
4707 BodyGen(CGF); in EmitOMPTargetTaskBasedDirective()
4738 auto &&BodyGen = [CS](CodeGenFunction &CGF, PrePostActionTy &) { in EmitOMPTaskDirective() argument
4739 CGF.EmitStmt(CS->getCapturedStmt()); in EmitOMPTaskDirective()
4742 IfCond](CodeGenFunction &CGF, llvm::Function *OutlinedFn, in EmitOMPTaskDirective()
4744 CGF.CGM.getOpenMPRuntime().emitTaskCall(CGF, S.getBeginLoc(), S, OutlinedFn, in EmitOMPTaskDirective()
4768 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTaskgroupDirective() argument
4769 Action.Enter(CGF); in EmitOMPTaskgroupDirective()
4784 CGF.CGM.getOpenMPRuntime().emitTaskReductionInit(CGF, S.getBeginLoc(), in EmitOMPTaskgroupDirective()
4787 CGF.EmitVarDecl(*VD); in EmitOMPTaskgroupDirective()
4788 CGF.EmitStoreOfScalar(ReductionDesc, CGF.GetAddrOfLocalVar(VD), in EmitOMPTaskgroupDirective()
4791 CGF.EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt()); in EmitOMPTaskgroupDirective()
5202 [&S](CodeGenFunction &CGF, PrePostActionTy &) { in EmitOMPDistributeLoop() argument
5204 CGF.EmitOMPSimdInit(S, /*IsMonotonic=*/true); in EmitOMPDistributeLoop()
5207 StaticChunked](CodeGenFunction &CGF, PrePostActionTy &) { in EmitOMPDistributeLoop() argument
5208 CGF.EmitOMPInnerLoop( in EmitOMPDistributeLoop()
5210 [&S, LoopExit, &CodeGenLoop](CodeGenFunction &CGF) { in EmitOMPDistributeLoop() argument
5211 CodeGenLoop(CGF, S, LoopExit); in EmitOMPDistributeLoop()
5213 [&S, StaticChunked](CodeGenFunction &CGF) { in EmitOMPDistributeLoop() argument
5215 CGF.EmitIgnoredExpr(S.getCombinedNextLowerBound()); in EmitOMPDistributeLoop()
5216 CGF.EmitIgnoredExpr(S.getCombinedNextUpperBound()); in EmitOMPDistributeLoop()
5217 CGF.EmitIgnoredExpr(S.getCombinedEnsureUpperBound()); in EmitOMPDistributeLoop()
5218 CGF.EmitIgnoredExpr(S.getCombinedInit()); in EmitOMPDistributeLoop()
5235 EmitOMPSimdFinal(S, [IL, &S](CodeGenFunction &CGF) { in EmitOMPDistributeLoop() argument
5236 return CGF.Builder.CreateIsNotNull( in EmitOMPDistributeLoop()
5237 CGF.EmitLoadOfScalar(IL, S.getBeginLoc())); in EmitOMPDistributeLoop()
5246 *this, S, [IL, &S](CodeGenFunction &CGF) { in EmitOMPDistributeLoop() argument
5247 return CGF.Builder.CreateIsNotNull( in EmitOMPDistributeLoop()
5248 CGF.EmitLoadOfScalar(IL, S.getBeginLoc())); in EmitOMPDistributeLoop()
5269 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) { in EmitOMPDistributeDirective() argument
5270 CGF.EmitOMPDistributeLoop(S, emitOMPLoopBodyWithStopPoint, S.getInc()); in EmitOMPDistributeDirective()
5279 CodeGenFunction CGF(CGM, /*suppressNewContext=*/true); in emitOutlinedOrderedFunction() local
5281 CGF.CapturedStmtInfo = &CapStmtInfo; in emitOutlinedOrderedFunction()
5282 llvm::Function *Fn = CGF.GenerateOpenMPCapturedStmtFunction(*S, Loc); in emitOutlinedOrderedFunction()
5296 auto &&CodeGen = [&S, C, this](CodeGenFunction &CGF, in EmitOMPOrderedDirective()
5301 CGF.GenerateOpenMPCapturedVars(*CS, CapturedVars); in EmitOMPOrderedDirective()
5304 CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF, S.getBeginLoc(), in EmitOMPOrderedDirective()
5307 Action.Enter(CGF); in EmitOMPOrderedDirective()
5308 CGF.EmitStmt(CS->getCapturedStmt()); in EmitOMPOrderedDirective()
5315 static llvm::Value *convertToScalarValue(CodeGenFunction &CGF, RValue Val, in convertToScalarValue() argument
5318 assert(CGF.hasScalarEvaluationKind(DestType) && in convertToScalarValue()
5321 return Val.isScalar() ? CGF.EmitScalarConversion(Val.getScalarVal(), SrcType, in convertToScalarValue()
5323 : CGF.EmitComplexToScalarConversion( in convertToScalarValue()
5328 convertToComplexValue(CodeGenFunction &CGF, RValue Val, QualType SrcType, in convertToComplexValue() argument
5330 assert(CGF.getEvaluationKind(DestType) == TEK_Complex && in convertToComplexValue()
5337 llvm::Value *ScalarVal = CGF.EmitScalarConversion( in convertToComplexValue()
5346 ComplexVal.first = CGF.EmitScalarConversion( in convertToComplexValue()
5348 ComplexVal.second = CGF.EmitScalarConversion( in convertToComplexValue()
5354 static void emitSimpleAtomicStore(CodeGenFunction &CGF, llvm::AtomicOrdering AO, in emitSimpleAtomicStore() argument
5357 CGF.EmitStoreThroughGlobalRegLValue(RVal, LVal); in emitSimpleAtomicStore()
5359 CGF.EmitAtomicStore(RVal, LVal, AO, LVal.isVolatile(), /*isInit=*/false); in emitSimpleAtomicStore()
5362 static RValue emitSimpleAtomicLoad(CodeGenFunction &CGF, in emitSimpleAtomicLoad() argument
5366 return CGF.EmitLoadOfLValue(LVal, Loc); in emitSimpleAtomicLoad()
5367 return CGF.EmitAtomicLoad( in emitSimpleAtomicLoad()
5390 static void emitOMPAtomicReadExpr(CodeGenFunction &CGF, llvm::AtomicOrdering AO, in emitOMPAtomicReadExpr() argument
5396 LValue XLValue = CGF.EmitLValue(X); in emitOMPAtomicReadExpr()
5397 LValue VLValue = CGF.EmitLValue(V); in emitOMPAtomicReadExpr()
5398 RValue Res = emitSimpleAtomicLoad(CGF, AO, XLValue, Loc); in emitOMPAtomicReadExpr()
5407 CGF.CGM.getOpenMPRuntime().emitFlush(CGF, llvm::None, Loc, in emitOMPAtomicReadExpr()
5417 CGF.emitOMPSimpleStore(VLValue, Res, X->getType().getNonReferenceType(), Loc); in emitOMPAtomicReadExpr()
5418 CGF.CGM.getOpenMPRuntime().checkAndEmitLastprivateConditional(CGF, V); in emitOMPAtomicReadExpr()
5421 static void emitOMPAtomicWriteExpr(CodeGenFunction &CGF, in emitOMPAtomicWriteExpr() argument
5426 emitSimpleAtomicStore(CGF, AO, CGF.EmitLValue(X), CGF.EmitAnyExpr(E)); in emitOMPAtomicWriteExpr()
5427 CGF.CGM.getOpenMPRuntime().checkAndEmitLastprivateConditional(CGF, X); in emitOMPAtomicWriteExpr()
5436 CGF.CGM.getOpenMPRuntime().emitFlush(CGF, llvm::None, Loc, in emitOMPAtomicWriteExpr()
5448 static std::pair<bool, RValue> emitOMPAtomicRMW(CodeGenFunction &CGF, LValue X, in emitOMPAtomicRMW() argument
5453 ASTContext &Context = CGF.getContext(); in emitOMPAtomicRMW()
5461 X.getAddress(CGF).getElementType())) || in emitOMPAtomicRMW()
5462 !X.getAddress(CGF).getElementType()->isIntegerTy() || in emitOMPAtomicRMW()
5533 UpdateVal = CGF.Builder.CreateIntCast( in emitOMPAtomicRMW()
5534 IC, X.getAddress(CGF).getElementType(), in emitOMPAtomicRMW()
5538 CGF.Builder.CreateAtomicRMW(RMWOp, X.getPointer(CGF), UpdateVal, AO); in emitOMPAtomicRMW()
5566 static void emitOMPAtomicUpdateExpr(CodeGenFunction &CGF, in emitOMPAtomicUpdateExpr() argument
5580 LValue XLValue = CGF.EmitLValue(X); in emitOMPAtomicUpdateExpr()
5581 RValue ExprRValue = CGF.EmitAnyExpr(E); in emitOMPAtomicUpdateExpr()
5586 auto &&Gen = [&CGF, UE, ExprRValue, XRValExpr, ERValExpr](RValue XRValue) { in emitOMPAtomicUpdateExpr()
5587 CodeGenFunction::OpaqueValueMapping MapExpr(CGF, ERValExpr, ExprRValue); in emitOMPAtomicUpdateExpr()
5588 CodeGenFunction::OpaqueValueMapping MapX(CGF, XRValExpr, XRValue); in emitOMPAtomicUpdateExpr()
5589 return CGF.EmitAnyExpr(UE); in emitOMPAtomicUpdateExpr()
5591 (void)CGF.EmitOMPAtomicSimpleUpdateExpr( in emitOMPAtomicUpdateExpr()
5593 CGF.CGM.getOpenMPRuntime().checkAndEmitLastprivateConditional(CGF, X); in emitOMPAtomicUpdateExpr()
5602 CGF.CGM.getOpenMPRuntime().emitFlush(CGF, llvm::None, Loc, in emitOMPAtomicUpdateExpr()
5614 static RValue convertToType(CodeGenFunction &CGF, RValue Value, in convertToType() argument
5617 switch (CGF.getEvaluationKind(ResType)) { in convertToType()
5620 convertToScalarValue(CGF, Value, SourceType, ResType, Loc)); in convertToType()
5622 auto Res = convertToComplexValue(CGF, Value, SourceType, ResType, Loc); in convertToType()
5631 static void emitOMPAtomicCaptureExpr(CodeGenFunction &CGF, in emitOMPAtomicCaptureExpr() argument
5640 LValue VLValue = CGF.EmitLValue(V); in emitOMPAtomicCaptureExpr()
5641 LValue XLValue = CGF.EmitLValue(X); in emitOMPAtomicCaptureExpr()
5642 RValue ExprRValue = CGF.EmitAnyExpr(E); in emitOMPAtomicCaptureExpr()
5660 auto &&Gen = [&CGF, &NewVVal, UE, ExprRValue, XRValExpr, ERValExpr, in emitOMPAtomicCaptureExpr()
5662 CodeGenFunction::OpaqueValueMapping MapExpr(CGF, ERValExpr, ExprRValue); in emitOMPAtomicCaptureExpr()
5663 CodeGenFunction::OpaqueValueMapping MapX(CGF, XRValExpr, XRValue); in emitOMPAtomicCaptureExpr()
5664 RValue Res = CGF.EmitAnyExpr(UE); in emitOMPAtomicCaptureExpr()
5668 auto Res = CGF.EmitOMPAtomicSimpleUpdateExpr( in emitOMPAtomicCaptureExpr()
5670 CGF.CGM.getOpenMPRuntime().checkAndEmitLastprivateConditional(CGF, X); in emitOMPAtomicCaptureExpr()
5679 CodeGenFunction::OpaqueValueMapping MapExpr(CGF, ERValExpr, ExprRValue); in emitOMPAtomicCaptureExpr()
5680 CodeGenFunction::OpaqueValueMapping MapX(CGF, XRValExpr, Res.second); in emitOMPAtomicCaptureExpr()
5681 NewVVal = CGF.EmitAnyExpr(UE); in emitOMPAtomicCaptureExpr()
5687 ExprRValue = convertToType(CGF, ExprRValue, E->getType(), in emitOMPAtomicCaptureExpr()
5694 auto Res = CGF.EmitOMPAtomicSimpleUpdateExpr( in emitOMPAtomicCaptureExpr()
5697 CGF.CGM.getOpenMPRuntime().checkAndEmitLastprivateConditional(CGF, X); in emitOMPAtomicCaptureExpr()
5704 CGF.emitOMPSimpleStore(VLValue, NewVVal, NewVValType, Loc); in emitOMPAtomicCaptureExpr()
5705 CGF.CGM.getOpenMPRuntime().checkAndEmitLastprivateConditional(CGF, V); in emitOMPAtomicCaptureExpr()
5715 CGF.CGM.getOpenMPRuntime().emitFlush(CGF, llvm::None, Loc, in emitOMPAtomicCaptureExpr()
5719 CGF.CGM.getOpenMPRuntime().emitFlush(CGF, llvm::None, Loc, in emitOMPAtomicCaptureExpr()
5724 CGF.CGM.getOpenMPRuntime().emitFlush(CGF, llvm::None, Loc, in emitOMPAtomicCaptureExpr()
5735 static void emitOMPAtomicExpr(CodeGenFunction &CGF, OpenMPClauseKind Kind, in emitOMPAtomicExpr() argument
5742 emitOMPAtomicReadExpr(CGF, AO, X, V, Loc); in emitOMPAtomicExpr()
5745 emitOMPAtomicWriteExpr(CGF, AO, X, E, Loc); in emitOMPAtomicExpr()
5749 emitOMPAtomicUpdateExpr(CGF, AO, X, E, UE, IsXLHSInRHSPart, Loc); in emitOMPAtomicExpr()
5752 emitOMPAtomicCaptureExpr(CGF, AO, IsPostfixUpdate, V, X, E, UE, in emitOMPAtomicExpr()
5894 static void emitCommonOMPTargetDirective(CodeGenFunction &CGF, in emitCommonOMPTargetDirective() argument
5898 CodeGenModule &CGM = CGF.CGM; in emitCommonOMPTargetDirective()
5902 OMPLexicalScope Scope(CGF, S, OMPD_target); in emitCommonOMPTargetDirective()
5904 CGF, OMPD_target, [&S](CodeGenFunction &CGF, PrePostActionTy &) { in emitCommonOMPTargetDirective() argument
5905 CGF.EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt()); in emitCommonOMPTargetDirective()
5911 CGOpenMPRuntime::LastprivateConditionalRAII::disable(CGF, S); in emitCommonOMPTargetDirective()
5937 if (CGF.ConstantFoldsToSimpleInteger(IfCond, Val) && !Val) in emitCommonOMPTargetDirective()
5943 assert(CGF.CurFuncDecl && "No parent declaration for target region!"); in emitCommonOMPTargetDirective()
5947 if (const auto *D = dyn_cast<CXXConstructorDecl>(CGF.CurFuncDecl)) in emitCommonOMPTargetDirective()
5949 else if (const auto *D = dyn_cast<CXXDestructorDecl>(CGF.CurFuncDecl)) in emitCommonOMPTargetDirective()
5953 CGM.getMangledName(GlobalDecl(cast<FunctionDecl>(CGF.CurFuncDecl))); in emitCommonOMPTargetDirective()
5958 OMPLexicalScope Scope(CGF, S, OMPD_task); in emitCommonOMPTargetDirective()
5960 [IsOffloadEntry](CodeGenFunction &CGF, in emitCommonOMPTargetDirective()
5963 OMPLoopScope(CGF, D); in emitCommonOMPTargetDirective()
5965 llvm::Value *NumIterations = CGF.EmitScalarExpr(D.getNumIterations()); in emitCommonOMPTargetDirective()
5966 NumIterations = CGF.Builder.CreateIntCast(NumIterations, CGF.Int64Ty, in emitCommonOMPTargetDirective()
5972 CGM.getOpenMPRuntime().emitTargetCall(CGF, S, Fn, FnID, IfCond, Device, in emitCommonOMPTargetDirective()
5976 static void emitTargetRegion(CodeGenFunction &CGF, const OMPTargetDirective &S, in emitTargetRegion() argument
5978 Action.Enter(CGF); in emitTargetRegion()
5979 CodeGenFunction::OMPPrivateScope PrivateScope(CGF); in emitTargetRegion()
5980 (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope); in emitTargetRegion()
5981 CGF.EmitOMPPrivateClause(S, PrivateScope); in emitTargetRegion()
5984 CGF.CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(CGF, S); in emitTargetRegion()
5986 CGF.EmitStmt(S.getCapturedStmt(OMPD_target)->getCapturedStmt()); in emitTargetRegion()
5987 CGF.EnsureInsertPoint(); in emitTargetRegion()
5993 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTargetDeviceFunction() argument
5994 emitTargetRegion(CGF, S, Action); in EmitOMPTargetDeviceFunction()
6005 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTargetDirective() argument
6006 emitTargetRegion(CGF, S, Action); in EmitOMPTargetDirective()
6011 static void emitCommonOMPTeamsDirective(CodeGenFunction &CGF, in emitCommonOMPTeamsDirective() argument
6017 CGF.CGM.getOpenMPRuntime().emitTeamsOutlinedFunction( in emitCommonOMPTeamsDirective()
6026 CGF.CGM.getOpenMPRuntime().emitNumTeamsClause(CGF, NumTeams, ThreadLimit, in emitCommonOMPTeamsDirective()
6030 OMPTeamsScope Scope(CGF, S); in emitCommonOMPTeamsDirective()
6032 CGF.GenerateOpenMPCapturedVars(*CS, CapturedVars); in emitCommonOMPTeamsDirective()
6033 CGF.CGM.getOpenMPRuntime().emitTeamsCall(CGF, S, S.getBeginLoc(), OutlinedFn, in emitCommonOMPTeamsDirective()
6039 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTeamsDirective() argument
6040 Action.Enter(CGF); in EmitOMPTeamsDirective()
6041 OMPPrivateScope PrivateScope(CGF); in EmitOMPTeamsDirective()
6042 (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope); in EmitOMPTeamsDirective()
6043 CGF.EmitOMPPrivateClause(S, PrivateScope); in EmitOMPTeamsDirective()
6044 CGF.EmitOMPReductionClauseInit(S, PrivateScope); in EmitOMPTeamsDirective()
6046 CGF.EmitStmt(S.getCapturedStmt(OMPD_teams)->getCapturedStmt()); in EmitOMPTeamsDirective()
6047 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams); in EmitOMPTeamsDirective()
6054 static void emitTargetTeamsRegion(CodeGenFunction &CGF, PrePostActionTy &Action, in emitTargetTeamsRegion() argument
6057 Action.Enter(CGF); in emitTargetTeamsRegion()
6059 auto &&CodeGen = [&S, CS](CodeGenFunction &CGF, PrePostActionTy &Action) { in emitTargetTeamsRegion() argument
6060 Action.Enter(CGF); in emitTargetTeamsRegion()
6061 CodeGenFunction::OMPPrivateScope PrivateScope(CGF); in emitTargetTeamsRegion()
6062 (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope); in emitTargetTeamsRegion()
6063 CGF.EmitOMPPrivateClause(S, PrivateScope); in emitTargetTeamsRegion()
6064 CGF.EmitOMPReductionClauseInit(S, PrivateScope); in emitTargetTeamsRegion()
6067 CGF.CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(CGF, S); in emitTargetTeamsRegion()
6068 CGF.EmitStmt(CS->getCapturedStmt()); in emitTargetTeamsRegion()
6069 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams); in emitTargetTeamsRegion()
6071 emitCommonOMPTeamsDirective(CGF, S, OMPD_teams, CodeGen); in emitTargetTeamsRegion()
6072 emitPostUpdateForReductionClause(CGF, S, in emitTargetTeamsRegion()
6079 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTargetTeamsDeviceFunction() argument
6080 emitTargetTeamsRegion(CGF, Action, S); in EmitOMPTargetTeamsDeviceFunction()
6092 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTargetTeamsDirective() argument
6093 emitTargetTeamsRegion(CGF, Action, S); in EmitOMPTargetTeamsDirective()
6099 emitTargetTeamsDistributeRegion(CodeGenFunction &CGF, PrePostActionTy &Action, in emitTargetTeamsDistributeRegion() argument
6101 Action.Enter(CGF); in emitTargetTeamsDistributeRegion()
6102 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) { in emitTargetTeamsDistributeRegion() argument
6103 CGF.EmitOMPDistributeLoop(S, emitOMPLoopBodyWithStopPoint, S.getInc()); in emitTargetTeamsDistributeRegion()
6107 auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF, in emitTargetTeamsDistributeRegion()
6109 Action.Enter(CGF); in emitTargetTeamsDistributeRegion()
6110 CodeGenFunction::OMPPrivateScope PrivateScope(CGF); in emitTargetTeamsDistributeRegion()
6111 CGF.EmitOMPReductionClauseInit(S, PrivateScope); in emitTargetTeamsDistributeRegion()
6113 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute, in emitTargetTeamsDistributeRegion()
6115 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams); in emitTargetTeamsDistributeRegion()
6117 emitCommonOMPTeamsDirective(CGF, S, OMPD_distribute, CodeGen); in emitTargetTeamsDistributeRegion()
6118 emitPostUpdateForReductionClause(CGF, S, in emitTargetTeamsDistributeRegion()
6125 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTargetTeamsDistributeDeviceFunction() argument
6126 emitTargetTeamsDistributeRegion(CGF, Action, S); in EmitOMPTargetTeamsDistributeDeviceFunction()
6138 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTargetTeamsDistributeDirective() argument
6139 emitTargetTeamsDistributeRegion(CGF, Action, S); in EmitOMPTargetTeamsDistributeDirective()
6145 CodeGenFunction &CGF, PrePostActionTy &Action, in emitTargetTeamsDistributeSimdRegion() argument
6147 Action.Enter(CGF); in emitTargetTeamsDistributeSimdRegion()
6148 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) { in emitTargetTeamsDistributeSimdRegion() argument
6149 CGF.EmitOMPDistributeLoop(S, emitOMPLoopBodyWithStopPoint, S.getInc()); in emitTargetTeamsDistributeSimdRegion()
6153 auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF, in emitTargetTeamsDistributeSimdRegion()
6155 Action.Enter(CGF); in emitTargetTeamsDistributeSimdRegion()
6156 CodeGenFunction::OMPPrivateScope PrivateScope(CGF); in emitTargetTeamsDistributeSimdRegion()
6157 CGF.EmitOMPReductionClauseInit(S, PrivateScope); in emitTargetTeamsDistributeSimdRegion()
6159 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute, in emitTargetTeamsDistributeSimdRegion()
6161 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams); in emitTargetTeamsDistributeSimdRegion()
6163 emitCommonOMPTeamsDirective(CGF, S, OMPD_distribute_simd, CodeGen); in emitTargetTeamsDistributeSimdRegion()
6164 emitPostUpdateForReductionClause(CGF, S, in emitTargetTeamsDistributeSimdRegion()
6171 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTargetTeamsDistributeSimdDeviceFunction() argument
6172 emitTargetTeamsDistributeSimdRegion(CGF, Action, S); in EmitOMPTargetTeamsDistributeSimdDeviceFunction()
6184 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTargetTeamsDistributeSimdDirective() argument
6185 emitTargetTeamsDistributeSimdRegion(CGF, Action, S); in EmitOMPTargetTeamsDistributeSimdDirective()
6193 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) { in EmitOMPTeamsDistributeDirective() argument
6194 CGF.EmitOMPDistributeLoop(S, emitOMPLoopBodyWithStopPoint, S.getInc()); in EmitOMPTeamsDistributeDirective()
6198 auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF, in EmitOMPTeamsDistributeDirective()
6200 Action.Enter(CGF); in EmitOMPTeamsDistributeDirective()
6201 OMPPrivateScope PrivateScope(CGF); in EmitOMPTeamsDistributeDirective()
6202 CGF.EmitOMPReductionClauseInit(S, PrivateScope); in EmitOMPTeamsDistributeDirective()
6204 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute, in EmitOMPTeamsDistributeDirective()
6206 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams); in EmitOMPTeamsDistributeDirective()
6215 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) { in EmitOMPTeamsDistributeSimdDirective() argument
6216 CGF.EmitOMPDistributeLoop(S, emitOMPLoopBodyWithStopPoint, S.getInc()); in EmitOMPTeamsDistributeSimdDirective()
6220 auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF, in EmitOMPTeamsDistributeSimdDirective()
6222 Action.Enter(CGF); in EmitOMPTeamsDistributeSimdDirective()
6223 OMPPrivateScope PrivateScope(CGF); in EmitOMPTeamsDistributeSimdDirective()
6224 CGF.EmitOMPReductionClauseInit(S, PrivateScope); in EmitOMPTeamsDistributeSimdDirective()
6226 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_simd, in EmitOMPTeamsDistributeSimdDirective()
6228 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams); in EmitOMPTeamsDistributeSimdDirective()
6237 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) { in EmitOMPTeamsDistributeParallelForDirective() argument
6238 CGF.EmitOMPDistributeLoop(S, emitInnerParallelForWhenCombined, in EmitOMPTeamsDistributeParallelForDirective()
6243 auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF, in EmitOMPTeamsDistributeParallelForDirective()
6245 Action.Enter(CGF); in EmitOMPTeamsDistributeParallelForDirective()
6246 OMPPrivateScope PrivateScope(CGF); in EmitOMPTeamsDistributeParallelForDirective()
6247 CGF.EmitOMPReductionClauseInit(S, PrivateScope); in EmitOMPTeamsDistributeParallelForDirective()
6249 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute, in EmitOMPTeamsDistributeParallelForDirective()
6251 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams); in EmitOMPTeamsDistributeParallelForDirective()
6260 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) { in EmitOMPTeamsDistributeParallelForSimdDirective() argument
6261 CGF.EmitOMPDistributeLoop(S, emitInnerParallelForWhenCombined, in EmitOMPTeamsDistributeParallelForSimdDirective()
6266 auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF, in EmitOMPTeamsDistributeParallelForSimdDirective()
6268 Action.Enter(CGF); in EmitOMPTeamsDistributeParallelForSimdDirective()
6269 OMPPrivateScope PrivateScope(CGF); in EmitOMPTeamsDistributeParallelForSimdDirective()
6270 CGF.EmitOMPReductionClauseInit(S, PrivateScope); in EmitOMPTeamsDistributeParallelForSimdDirective()
6272 CGF.CGM.getOpenMPRuntime().emitInlinedDirective( in EmitOMPTeamsDistributeParallelForSimdDirective()
6273 CGF, OMPD_distribute, CodeGenDistribute, /*HasCancel=*/false); in EmitOMPTeamsDistributeParallelForSimdDirective()
6274 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams); in EmitOMPTeamsDistributeParallelForSimdDirective()
6283 CodeGenFunction &CGF, const OMPTargetTeamsDistributeParallelForDirective &S, in emitTargetTeamsDistributeParallelForRegion() argument
6285 Action.Enter(CGF); in emitTargetTeamsDistributeParallelForRegion()
6286 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) { in emitTargetTeamsDistributeParallelForRegion() argument
6287 CGF.EmitOMPDistributeLoop(S, emitInnerParallelForWhenCombined, in emitTargetTeamsDistributeParallelForRegion()
6292 auto &&CodeGenTeams = [&S, &CodeGenDistribute](CodeGenFunction &CGF, in emitTargetTeamsDistributeParallelForRegion()
6294 Action.Enter(CGF); in emitTargetTeamsDistributeParallelForRegion()
6295 CodeGenFunction::OMPPrivateScope PrivateScope(CGF); in emitTargetTeamsDistributeParallelForRegion()
6296 CGF.EmitOMPReductionClauseInit(S, PrivateScope); in emitTargetTeamsDistributeParallelForRegion()
6298 CGF.CGM.getOpenMPRuntime().emitInlinedDirective( in emitTargetTeamsDistributeParallelForRegion()
6299 CGF, OMPD_distribute, CodeGenDistribute, /*HasCancel=*/false); in emitTargetTeamsDistributeParallelForRegion()
6300 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams); in emitTargetTeamsDistributeParallelForRegion()
6303 emitCommonOMPTeamsDirective(CGF, S, OMPD_distribute_parallel_for, in emitTargetTeamsDistributeParallelForRegion()
6305 emitPostUpdateForReductionClause(CGF, S, in emitTargetTeamsDistributeParallelForRegion()
6314 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTargetTeamsDistributeParallelForDeviceFunction() argument
6315 emitTargetTeamsDistributeParallelForRegion(CGF, S, Action); in EmitOMPTargetTeamsDistributeParallelForDeviceFunction()
6327 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTargetTeamsDistributeParallelForDirective() argument
6328 emitTargetTeamsDistributeParallelForRegion(CGF, S, Action); in EmitOMPTargetTeamsDistributeParallelForDirective()
6334 CodeGenFunction &CGF, in emitTargetTeamsDistributeParallelForSimdRegion() argument
6337 Action.Enter(CGF); in emitTargetTeamsDistributeParallelForSimdRegion()
6338 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) { in emitTargetTeamsDistributeParallelForSimdRegion() argument
6339 CGF.EmitOMPDistributeLoop(S, emitInnerParallelForWhenCombined, in emitTargetTeamsDistributeParallelForSimdRegion()
6344 auto &&CodeGenTeams = [&S, &CodeGenDistribute](CodeGenFunction &CGF, in emitTargetTeamsDistributeParallelForSimdRegion()
6346 Action.Enter(CGF); in emitTargetTeamsDistributeParallelForSimdRegion()
6347 CodeGenFunction::OMPPrivateScope PrivateScope(CGF); in emitTargetTeamsDistributeParallelForSimdRegion()
6348 CGF.EmitOMPReductionClauseInit(S, PrivateScope); in emitTargetTeamsDistributeParallelForSimdRegion()
6350 CGF.CGM.getOpenMPRuntime().emitInlinedDirective( in emitTargetTeamsDistributeParallelForSimdRegion()
6351 CGF, OMPD_distribute, CodeGenDistribute, /*HasCancel=*/false); in emitTargetTeamsDistributeParallelForSimdRegion()
6352 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams); in emitTargetTeamsDistributeParallelForSimdRegion()
6355 emitCommonOMPTeamsDirective(CGF, S, OMPD_distribute_parallel_for_simd, in emitTargetTeamsDistributeParallelForSimdRegion()
6357 emitPostUpdateForReductionClause(CGF, S, in emitTargetTeamsDistributeParallelForSimdRegion()
6366 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTargetTeamsDistributeParallelForSimdDeviceFunction() argument
6367 emitTargetTeamsDistributeParallelForSimdRegion(CGF, S, Action); in EmitOMPTargetTeamsDistributeParallelForSimdDeviceFunction()
6379 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTargetTeamsDistributeParallelForSimdDirective() argument
6380 emitTargetTeamsDistributeParallelForSimdRegion(CGF, S, Action); in EmitOMPTargetTeamsDistributeParallelForSimdDirective()
6568 void Enter(CodeGenFunction &CGF) override { in EmitOMPTargetDataDirective() argument
6575 CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTargetDataDirective() argument
6576 auto &&InnermostCodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) { in EmitOMPTargetDataDirective() argument
6577 CGF.EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt()); in EmitOMPTargetDataDirective()
6582 &InnermostCodeGen](CodeGenFunction &CGF, in EmitOMPTargetDataDirective()
6589 Action.Enter(CGF); in EmitOMPTargetDataDirective()
6592 OMPPrivateScope PrivateScope(CGF); in EmitOMPTargetDataDirective()
6595 CGF.EmitOMPUseDevicePtrClause(*C, PrivateScope, in EmitOMPTargetDataDirective()
6598 CGF.EmitOMPUseDeviceAddrClause(*C, PrivateScope, in EmitOMPTargetDataDirective()
6601 RCG(CGF); in EmitOMPTargetDataDirective()
6603 OMPLexicalScope Scope(CGF, S, OMPD_unknown); in EmitOMPTargetDataDirective()
6604 RCG(CGF); in EmitOMPTargetDataDirective()
6615 OMPLexicalScope Scope(CGF, S); in EmitOMPTargetDataDirective()
6616 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_target_data, in EmitOMPTargetDataDirective()
6689 static void emitTargetParallelRegion(CodeGenFunction &CGF, in emitTargetParallelRegion() argument
6694 Action.Enter(CGF); in emitTargetParallelRegion()
6695 auto &&CodeGen = [&S, CS](CodeGenFunction &CGF, PrePostActionTy &Action) { in emitTargetParallelRegion() argument
6696 Action.Enter(CGF); in emitTargetParallelRegion()
6697 CodeGenFunction::OMPPrivateScope PrivateScope(CGF); in emitTargetParallelRegion()
6698 (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope); in emitTargetParallelRegion()
6699 CGF.EmitOMPPrivateClause(S, PrivateScope); in emitTargetParallelRegion()
6700 CGF.EmitOMPReductionClauseInit(S, PrivateScope); in emitTargetParallelRegion()
6703 CGF.CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(CGF, S); in emitTargetParallelRegion()
6705 CGF.EmitStmt(CS->getCapturedStmt()); in emitTargetParallelRegion()
6706 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_parallel); in emitTargetParallelRegion()
6708 emitCommonOMPParallelDirective(CGF, S, OMPD_parallel, CodeGen, in emitTargetParallelRegion()
6710 emitPostUpdateForReductionClause(CGF, S, in emitTargetParallelRegion()
6717 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTargetParallelDeviceFunction() argument
6718 emitTargetParallelRegion(CGF, S, Action); in EmitOMPTargetParallelDeviceFunction()
6730 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTargetParallelDirective() argument
6731 emitTargetParallelRegion(CGF, S, Action); in EmitOMPTargetParallelDirective()
6736 static void emitTargetParallelForRegion(CodeGenFunction &CGF, in emitTargetParallelForRegion() argument
6739 Action.Enter(CGF); in emitTargetParallelForRegion()
6742 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in emitTargetParallelForRegion() argument
6743 Action.Enter(CGF); in emitTargetParallelForRegion()
6745 CGF, OMPD_target_parallel_for, S.hasCancel()); in emitTargetParallelForRegion()
6746 CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(), emitForLoopBounds, in emitTargetParallelForRegion()
6749 emitCommonOMPParallelDirective(CGF, S, OMPD_for, CodeGen, in emitTargetParallelForRegion()
6757 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTargetParallelForDeviceFunction() argument
6758 emitTargetParallelForRegion(CGF, S, Action); in EmitOMPTargetParallelForDeviceFunction()
6770 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTargetParallelForDirective() argument
6771 emitTargetParallelForRegion(CGF, S, Action); in EmitOMPTargetParallelForDirective()
6777 emitTargetParallelForSimdRegion(CodeGenFunction &CGF, in emitTargetParallelForSimdRegion() argument
6780 Action.Enter(CGF); in emitTargetParallelForSimdRegion()
6783 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in emitTargetParallelForSimdRegion() argument
6784 Action.Enter(CGF); in emitTargetParallelForSimdRegion()
6785 CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(), emitForLoopBounds, in emitTargetParallelForSimdRegion()
6788 emitCommonOMPParallelDirective(CGF, S, OMPD_simd, CodeGen, in emitTargetParallelForSimdRegion()
6796 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTargetParallelForSimdDeviceFunction() argument
6797 emitTargetParallelForSimdRegion(CGF, S, Action); in EmitOMPTargetParallelForSimdDeviceFunction()
6809 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTargetParallelForSimdDirective() argument
6810 emitTargetParallelForSimdRegion(CGF, S, Action); in EmitOMPTargetParallelForSimdDirective()
6816 static void mapParam(CodeGenFunction &CGF, const DeclRefExpr *Helper, in mapParam() argument
6821 [&CGF, PVD]() { return CGF.GetAddrOfLocalVar(PVD); }); in mapParam()
6859 auto &&BodyGen = [CS, &S](CodeGenFunction &CGF, PrePostActionTy &) { in EmitOMPTaskLoopBasedDirective() argument
6871 OMPLoopScope PreInitScope(CGF, S); in EmitOMPTaskLoopBasedDirective()
6872 if (CGF.ConstantFoldsToSimpleInteger(S.getPreCond(), CondConstant)) { in EmitOMPTaskLoopBasedDirective()
6876 llvm::BasicBlock *ThenBlock = CGF.createBasicBlock("taskloop.if.then"); in EmitOMPTaskLoopBasedDirective()
6877 ContBlock = CGF.createBasicBlock("taskloop.if.end"); in EmitOMPTaskLoopBasedDirective()
6878 emitPreCond(CGF, S, S.getPreCond(), ThenBlock, ContBlock, in EmitOMPTaskLoopBasedDirective()
6879 CGF.getProfileCount(&S)); in EmitOMPTaskLoopBasedDirective()
6880 CGF.EmitBlock(ThenBlock); in EmitOMPTaskLoopBasedDirective()
6881 CGF.incrementProfileCounter(&S); in EmitOMPTaskLoopBasedDirective()
6884 (void)CGF.EmitOMPLinearClauseInit(S); in EmitOMPTaskLoopBasedDirective()
6886 OMPPrivateScope LoopScope(CGF); in EmitOMPTaskLoopBasedDirective()
6894 mapParam(CGF, cast<DeclRefExpr>(S.getLowerBoundVariable()), *LBP, in EmitOMPTaskLoopBasedDirective()
6896 mapParam(CGF, cast<DeclRefExpr>(S.getUpperBoundVariable()), *UBP, in EmitOMPTaskLoopBasedDirective()
6898 mapParam(CGF, cast<DeclRefExpr>(S.getStrideVariable()), *STP, LoopScope); in EmitOMPTaskLoopBasedDirective()
6899 mapParam(CGF, cast<DeclRefExpr>(S.getIsLastIterVariable()), *LIP, in EmitOMPTaskLoopBasedDirective()
6901 CGF.EmitOMPPrivateLoopCounters(S, LoopScope); in EmitOMPTaskLoopBasedDirective()
6902 CGF.EmitOMPLinearClause(S, LoopScope); in EmitOMPTaskLoopBasedDirective()
6903 bool HasLastprivateClause = CGF.EmitOMPLastprivateClauseInit(S, LoopScope); in EmitOMPTaskLoopBasedDirective()
6908 CGF.EmitVarDecl(*IVDecl); in EmitOMPTaskLoopBasedDirective()
6909 CGF.EmitIgnoredExpr(S.getInit()); in EmitOMPTaskLoopBasedDirective()
6915 CGF.EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl())); in EmitOMPTaskLoopBasedDirective()
6917 CGF.EmitIgnoredExpr(S.getCalcLastIteration()); in EmitOMPTaskLoopBasedDirective()
6921 OMPLexicalScope Scope(CGF, S, OMPD_taskloop, /*EmitPreInitStmt=*/false); in EmitOMPTaskLoopBasedDirective()
6923 CGF, S, in EmitOMPTaskLoopBasedDirective()
6924 [&S](CodeGenFunction &CGF, PrePostActionTy &) { in EmitOMPTaskLoopBasedDirective() argument
6926 CGF.EmitOMPSimdInit(S); in EmitOMPTaskLoopBasedDirective()
6928 [&S, &LoopScope](CodeGenFunction &CGF, PrePostActionTy &) { in EmitOMPTaskLoopBasedDirective() argument
6929 CGF.EmitOMPInnerLoop( in EmitOMPTaskLoopBasedDirective()
6931 [&S](CodeGenFunction &CGF) { in EmitOMPTaskLoopBasedDirective() argument
6932 emitOMPLoopBodyWithStopPoint(CGF, S, in EmitOMPTaskLoopBasedDirective()
6940 CGF.EmitBranch(ContBlock); in EmitOMPTaskLoopBasedDirective()
6941 CGF.EmitBlock(ContBlock, true); in EmitOMPTaskLoopBasedDirective()
6945 CGF.EmitOMPLastprivateClauseFinal( in EmitOMPTaskLoopBasedDirective()
6947 CGF.Builder.CreateIsNotNull(CGF.EmitLoadOfScalar( in EmitOMPTaskLoopBasedDirective()
6948 CGF.GetAddrOfLocalVar(*LIP), /*Volatile=*/false, in EmitOMPTaskLoopBasedDirective()
6951 CGF.EmitOMPLinearClauseFinal(S, [LIP, &S](CodeGenFunction &CGF) { in EmitOMPTaskLoopBasedDirective() argument
6952 return CGF.Builder.CreateIsNotNull( in EmitOMPTaskLoopBasedDirective()
6953 CGF.EmitLoadOfScalar(CGF.GetAddrOfLocalVar(*LIP), /*Volatile=*/false, in EmitOMPTaskLoopBasedDirective()
6958 IfCond](CodeGenFunction &CGF, llvm::Function *OutlinedFn, in EmitOMPTaskLoopBasedDirective()
6961 &Data](CodeGenFunction &CGF, PrePostActionTy &) { in EmitOMPTaskLoopBasedDirective() argument
6962 OMPLoopScope PreInitScope(CGF, S); in EmitOMPTaskLoopBasedDirective()
6963 CGF.CGM.getOpenMPRuntime().emitTaskLoopCall(CGF, S.getBeginLoc(), S, in EmitOMPTaskLoopBasedDirective()
6967 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_taskloop, in EmitOMPTaskLoopBasedDirective()
6975 [&S, &BodyGen, &TaskGen, &Data](CodeGenFunction &CGF, in EmitOMPTaskLoopBasedDirective()
6977 Action.Enter(CGF); in EmitOMPTaskLoopBasedDirective()
6978 CGF.EmitOMPTaskBasedDirective(S, OMPD_taskloop, BodyGen, TaskGen, in EmitOMPTaskLoopBasedDirective()
7001 auto &&CodeGen = [this, &S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPMasterTaskLoopDirective() argument
7002 Action.Enter(CGF); in EmitOMPMasterTaskLoopDirective()
7013 auto &&CodeGen = [this, &S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPMasterTaskLoopSimdDirective() argument
7014 Action.Enter(CGF); in EmitOMPMasterTaskLoopSimdDirective()
7025 auto &&CodeGen = [this, &S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPParallelMasterTaskLoopDirective() argument
7026 auto &&TaskLoopCodeGen = [&S](CodeGenFunction &CGF, in EmitOMPParallelMasterTaskLoopDirective()
7028 Action.Enter(CGF); in EmitOMPParallelMasterTaskLoopDirective()
7029 CGF.EmitOMPTaskLoopBasedDirective(S); in EmitOMPParallelMasterTaskLoopDirective()
7031 OMPLexicalScope Scope(CGF, S, OMPD_parallel, /*EmitPreInitStmt=*/false); in EmitOMPParallelMasterTaskLoopDirective()
7032 CGM.getOpenMPRuntime().emitMasterRegion(CGF, TaskLoopCodeGen, in EmitOMPParallelMasterTaskLoopDirective()
7043 auto &&CodeGen = [this, &S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPParallelMasterTaskLoopSimdDirective() argument
7044 auto &&TaskLoopCodeGen = [&S](CodeGenFunction &CGF, in EmitOMPParallelMasterTaskLoopSimdDirective()
7046 Action.Enter(CGF); in EmitOMPParallelMasterTaskLoopSimdDirective()
7047 CGF.EmitOMPTaskLoopBasedDirective(S); in EmitOMPParallelMasterTaskLoopSimdDirective()
7049 OMPLexicalScope Scope(CGF, S, OMPD_parallel, /*EmitPreInitStmt=*/false); in EmitOMPParallelMasterTaskLoopSimdDirective()
7050 CGM.getOpenMPRuntime().emitMasterRegion(CGF, TaskLoopCodeGen, in EmitOMPParallelMasterTaskLoopSimdDirective()
7089 auto &&CodeGen = [&D](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitSimpleOMPExecutableDirective() argument
7090 OMPPrivateScope GlobalsScope(CGF); in EmitSimpleOMPExecutableDirective()
7101 if (!CGF.LocalDeclMap.count(VD)) { in EmitSimpleOMPExecutableDirective()
7102 LValue GlobLVal = CGF.EmitLValue(Ref); in EmitSimpleOMPExecutableDirective()
7104 VD, [&GlobLVal, &CGF]() { return GlobLVal.getAddress(CGF); }); in EmitSimpleOMPExecutableDirective()
7111 ParentLoopDirectiveForScanRegion ScanRegion(CGF, D); in EmitSimpleOMPExecutableDirective()
7112 emitOMPSimdRegion(CGF, cast<OMPLoopDirective>(D), Action); in EmitSimpleOMPExecutableDirective()
7117 if (!VD->hasLocalStorage() && !CGF.LocalDeclMap.count(VD)) { in EmitSimpleOMPExecutableDirective()
7118 LValue GlobLVal = CGF.EmitLValue(E); in EmitSimpleOMPExecutableDirective()
7120 VD, [&GlobLVal, &CGF]() { return GlobLVal.getAddress(CGF); }); in EmitSimpleOMPExecutableDirective()
7124 if (!CGF.LocalDeclMap.count(VD)) in EmitSimpleOMPExecutableDirective()
7125 CGF.EmitVarDecl(*VD); in EmitSimpleOMPExecutableDirective()
7137 if (!CGF.LocalDeclMap.count(VD)) in EmitSimpleOMPExecutableDirective()
7138 CGF.EmitVarDecl(*VD); in EmitSimpleOMPExecutableDirective()
7144 CGF.EmitStmt(D.getInnermostCapturedStmt()->getCapturedStmt()); in EmitSimpleOMPExecutableDirective()