Lines Matching defs:CGF
54 void emitPreInitStmt(CodeGenFunction &CGF, const OMPExecutableDirective &S) {
61 CGF.EmitVarDecl(cast<VarDecl>(*I));
64 CGF.EmitAutoVarAlloca(cast<VarDecl>(*I));
65 CGF.EmitAutoVarCleanups(Emission);
74 static bool isCapturedVar(CodeGenFunction &CGF, const VarDecl *VD) {
75 return CGF.LambdaCaptureFields.lookup(VD) ||
76 (CGF.CapturedStmtInfo && CGF.CapturedStmtInfo->lookup(VD)) ||
77 (isa_and_nonnull<BlockDecl>(CGF.CurCodeDecl) &&
78 cast<BlockDecl>(CGF.CurCodeDecl)->capturesVariable(VD));
83 CodeGenFunction &CGF, const OMPExecutableDirective &S,
86 : CodeGenFunction::LexicalScope(CGF, S.getSourceRange()),
87 InlinedShareds(CGF) {
89 emitPreInitStmt(CGF, S);
101 CGF.getContext(), const_cast<VarDecl *>(VD),
102 isCapturedVar(CGF, VD) || (CGF.CapturedStmtInfo &&
105 InlinedShareds.addPrivate(VD, CGF.EmitLValue(&DRE).getAddress());
123 OMPParallelScope(CodeGenFunction &CGF, const OMPExecutableDirective &S)
124 : OMPLexicalScope(CGF, S, /*CapturedRegion=*/std::nullopt,
138 OMPTeamsScope(CodeGenFunction &CGF, const OMPExecutableDirective &S)
139 : OMPLexicalScope(CGF, S, /*CapturedRegion=*/std::nullopt,
146 void emitPreInitStmt(CodeGenFunction &CGF, const OMPLoopBasedDirective &S) {
155 CGF, VD, CGF.CreateMemTemp(VD->getType().getNonReferenceType()));
165 CGF, OrigVD,
166 Address(llvm::UndefValue::get(CGF.ConvertTypeForMem(
167 CGF.getContext().getPointerType(OrigVDTy))),
168 CGF.ConvertTypeForMem(OrigVDTy),
169 CGF.getContext().getDeclAlign(OrigVD)));
173 (void)PreCondVars.apply(CGF);
178 [&CGF](unsigned Cnt, const Stmt *CurStmt) {
181 CGF.EmitStmt(Init);
182 CGF.EmitStmt(CXXFor->getRangeStmt());
183 CGF.EmitStmt(CXXFor->getEndStmt());
215 CGF.EmitVarDecl(cast<VarDecl>(*I));
218 CGF.EmitStmt(S);
221 PreCondVars.restore(CGF);
225 OMPLoopScope(CodeGenFunction &CGF, const OMPLoopBasedDirective &S)
226 : CodeGenFunction::RunCleanupsScope(CGF) {
227 emitPreInitStmt(CGF, S);
234 static bool isCapturedVar(CodeGenFunction &CGF, const VarDecl *VD) {
235 return CGF.LambdaCaptureFields.lookup(VD) ||
236 (CGF.CapturedStmtInfo && CGF.CapturedStmtInfo->lookup(VD)) ||
237 (isa_and_nonnull<BlockDecl>(CGF.CurCodeDecl) &&
238 cast<BlockDecl>(CGF.CurCodeDecl)->capturesVariable(VD));
242 OMPSimdLexicalScope(CodeGenFunction &CGF, const OMPExecutableDirective &S)
243 : CodeGenFunction::LexicalScope(CGF, S.getSourceRange()),
244 InlinedShareds(CGF) {
251 CGF.EmitVarDecl(cast<VarDecl>(*I));
254 CGF.EmitAutoVarAlloca(cast<VarDecl>(*I));
255 CGF.EmitAutoVarCleanups(Emission);
263 CGF.EmitVarDecl(*OED);
269 CGF.EmitVarDecl(*OED);
274 CGF.EmitOMPPrivateClause(S, InlinedShareds);
277 CGF.EmitVarDecl(*cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl()));
297 DeclRefExpr DRE(CGF.getContext(), const_cast<VarDecl *>(VD),
298 isCapturedVar(CGF, VD) ||
299 (CGF.CapturedStmtInfo &&
303 InlinedShareds.addPrivate(VD, CGF.EmitLValue(&DRE).getAddress());
338 static void emitCommonOMPTargetDirective(CodeGenFunction &CGF,
425 static Address castValueFromUintptr(CodeGenFunction &CGF, SourceLocation Loc,
428 ASTContext &Ctx = CGF.getContext();
430 llvm::Value *CastedPtr = CGF.EmitScalarConversion(
431 AddrLV.getAddress().emitRawPointer(CGF), Ctx.getUIntPtrType(),
435 CGF.MakeNaturalAlignAddrLValue(CastedPtr, DstType).getAddress();
480 CodeGenFunction &CGF, FunctionArgList &Args,
492 CodeGenModule &CGM = CGF.CGM;
587 CGF.StartFunction(CD, Ctx.VoidTy, F, FuncInfo, TargetArgs,
597 LocalAddr = CGM.getOpenMPRuntime().getParameterAddress(CGF, Args[Cnt],
600 LocalAddr = CGF.GetAddrOfLocalVar(Args[Cnt]);
613 LValue ArgLVal = CGF.MakeAddrLValue(LocalAddr, Args[Cnt]->getType(),
617 ArgLVal = CGF.MakeAddrLValue(
618 castValueFromUintptr(CGF, I->getLocation(), FD->getType(),
622 llvm::Value *ExprArg = CGF.EmitLoadOfScalar(ArgLVal, I->getLocation());
630 ArgAddr = CGF.EmitLoadOfReference(ArgLVal);
633 ArgAddr = CGF.EmitLoadOfPointer(
647 CGF, I->getLocation(), FD->getType(),
653 CXXThisValue = CGF.EmitLoadOfScalar(ArgLVal, I->getLocation());
1297 [&Emission](CodeGenFunction &CGF) {
1298 CGF.EmitAutoVarInit(Emission);
1507 CodeGenFunction &CGF, const OMPExecutableDirective &D,
1509 if (!CGF.HaveInsertPoint())
1515 if (llvm::Value *Cond = CondGen(CGF)) {
1518 llvm::BasicBlock *ThenBB = CGF.createBasicBlock(".omp.reduction.pu");
1519 DoneBB = CGF.createBasicBlock(".omp.reduction.pu.done");
1520 CGF.Builder.CreateCondBr(Cond, ThenBB, DoneBB);
1521 CGF.EmitBlock(ThenBB);
1524 CGF.EmitIgnoredExpr(PostUpdate);
1528 CGF.EmitBlock(DoneBB, /*IsFinished=*/true);
1542 checkForLastprivateConditionalUpdate(CodeGenFunction &CGF,
1544 if (CGF.getLangOpts().OpenMP < 50)
1555 CGF.CGM.getOpenMPRuntime().checkAndEmitLastprivateConditional(CGF, Ref);
1566 CGF.CGM.getOpenMPRuntime().checkAndEmitLastprivateConditional(CGF, Ref);
1577 CGF.CGM.getOpenMPRuntime().checkAndEmitLastprivateConditional(CGF, Ref);
1594 CGF.CGM.getOpenMPRuntime().checkAndEmitSharedLastprivateConditional(
1595 CGF, S, PrivateDecls);
1599 CodeGenFunction &CGF, const OMPExecutableDirective &S,
1605 CGF.CGM.getOpenMPRuntime().emitParallelOutlinedFunction(
1606 CGF, S, *CS->getCapturedDecl()->param_begin(), InnermostKind,
1609 CodeGenFunction::RunCleanupsScope NumThreadsScope(CGF);
1610 NumThreads = CGF.EmitScalarExpr(NumThreadsClause->getNumThreads(),
1612 CGF.CGM.getOpenMPRuntime().emitNumThreadsClause(
1613 CGF, NumThreads, NumThreadsClause->getBeginLoc());
1616 CodeGenFunction::RunCleanupsScope ProcBindScope(CGF);
1617 CGF.CGM.getOpenMPRuntime().emitProcBindClause(
1618 CGF, ProcBindClause->getProcBindKind(), ProcBindClause->getBeginLoc());
1629 OMPParallelScope Scope(CGF, S);
1635 CodeGenBoundParameters(CGF, S, CapturedVars);
1636 CGF.GenerateOpenMPCapturedVars(*CS, CapturedVars);
1637 CGF.CGM.getOpenMPRuntime().emitParallelCall(CGF, S.getBeginLoc(), OutlinedFn,
1656 static void emitOMPCopyinClause(CodeGenFunction &CGF,
1658 bool Copyins = CGF.EmitOMPCopyinClause(S);
1663 CGF.CGM.getOpenMPRuntime().emitBarrierCall(
1664 CGF, S.getBeginLoc(), OMPD_unknown, /*EmitChecks=*/false,
1670 CodeGenFunction &CGF, const VarDecl *VD) {
1671 CodeGenModule &CGM = CGF.CGM;
1682 Size = CGF.getTypeSize(CVD->getType());
1684 Size = CGF.Builder.CreateNUWAdd(
1686 Size = CGF.Builder.CreateUDiv(Size, CGM.getSize(Align));
1687 Size = CGF.Builder.CreateNUWMul(Size, CGM.getSize(Align));
1696 llvm::Value *Allocator = CGF.EmitScalarExpr(AA->getAllocator());
1700 Allocator = CGF.Builder.CreateIntToPtr(Allocator, CGM.VoidPtrTy);
1702 Allocator = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(Allocator,
1706 CGF.Builder, Size, Allocator,
1709 OMPBuilder.createOMPFree(CGF.Builder, Addr, Allocator);
1711 CGF.EHStack.pushCleanup<OMPAllocateCleanupTy>(NormalAndEHCleanup, FreeCI);
1712 Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
1714 CGF.ConvertTypeForMem(CGM.getContext().getPointerType(CVD->getType())),
1716 return Address(Addr, CGF.ConvertTypeForMem(CVD->getType()), Align);
1720 CodeGenFunction &CGF, const VarDecl *VD, Address VDAddr,
1722 CodeGenModule &CGM = CGF.CGM;
1731 CGF.Builder.CreatePointerCast(VDAddr.emitRawPointer(CGF), CGM.Int8PtrTy);
1737 OMPBuilder.createCachedThreadPrivate(CGF.Builder, Data, Size, CacheName);
1755 CodeGenFunction &CGF, const Stmt *RegionBodyStmt, InsertPointTy AllocaIP,
1757 CGBuilderTy &Builder = CGF.Builder;
1763 OMPBuilderCBHelpers::InlinedRegionBodyRAII IRB(CGF, AllocaIP, *FiniBB);
1764 CGF.EmitStmt(RegionBodyStmt);
1772 CodeGenFunction &CGF, const Stmt *RegionBodyStmt, InsertPointTy AllocaIP,
1774 CGBuilderTy &Builder = CGF.Builder;
1780 OMPBuilderCBHelpers::OutlinedRegionBodyRAII IRB(CGF, AllocaIP, *FiniBB);
1781 CGF.EmitStmt(RegionBodyStmt);
1850 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
1851 Action.Enter(CGF);
1852 OMPPrivateScope PrivateScope(CGF);
1853 emitOMPCopyinClause(CGF, S);
1854 (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);
1855 CGF.EmitOMPPrivateClause(S, PrivateScope);
1856 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
1858 CGF.EmitStmt(S.getCapturedStmt(OMPD_parallel)->getCapturedStmt());
1859 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_parallel);
1890 OMPTransformDirectiveScopeRAII(CodeGenFunction &CGF, const Stmt *S) {
1892 Scope = new OMPLoopScope(CGF, *Dir);
1894 CapInfoRAII = new CodeGenFunction::CGCapturedStmtRAII(CGF, CGSI);
1907 static void emitBody(CodeGenFunction &CGF, const Stmt *S, const Stmt *NextLoop,
1913 CGF.getContext().getSourceManager(), CS->getLBracLoc(),
1917 CodeGenFunction::LexicalScope Scope(CGF, S->getSourceRange());
1919 emitBody(CGF, CurStmt, NextLoop, MaxLevel, Level);
1933 CGF.EmitStmt(CXXFor->getLoopVarStmt());
1939 emitBody(CGF, S, NextLoop, MaxLevel, Level + 1);
1943 CGF.EmitStmt(S);
2022 CodeGenFunction CGF(ParentCGF.CGM, /*suppressNewContext=*/true);
2025 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, CSI.get());
2026 llvm::Function *F = CGF.GenerateCapturedStmtFunction(*S);
2285 static void emitAlignedClause(CodeGenFunction &CGF,
2287 if (!CGF.HaveInsertPoint())
2293 cast<llvm::ConstantInt>(CGF.EmitScalarExpr(AlignmentExpr));
2303 CGF.getContext()
2304 .toCharUnitsFromBits(CGF.getContext().getOpenMPDefaultSimdAlign(
2311 llvm::Value *PtrValue = CGF.EmitScalarExpr(E);
2312 CGF.emitAlignmentAssumption(
2314 llvm::ConstantInt::get(CGF.getLLVMContext(), Alignment));
2362 static void emitPreCond(CodeGenFunction &CGF, const OMPLoopDirective &S,
2365 if (!CGF.HaveInsertPoint())
2368 CodeGenFunction::OMPPrivateScope PreCondScope(CGF);
2369 CGF.EmitOMPPrivateLoopCounters(S, PreCondScope);
2373 CGF.EmitIgnoredExpr(I);
2386 CGF.CreateMemTemp(VD->getType().getNonReferenceType());
2387 (void)PreCondVars.setVarAddr(CGF, VD, CounterAddr);
2389 (void)PreCondVars.apply(CGF);
2393 CGF.EmitIgnoredExpr(E);
2396 CGF.EmitBranchOnBoolExpr(Cond, TrueBlock, FalseBlock, TrueCount);
2397 PreCondVars.restore(CGF);
2435 static void emitSimdlenSafelenClause(CodeGenFunction &CGF,
2437 if (!CGF.HaveInsertPoint())
2440 RValue Len = CGF.EmitAnyExpr(C->getSimdlen(), AggValueSlot::ignored(),
2443 CGF.LoopStack.setVectorizeWidth(Val->getZExtValue());
2447 CGF.LoopStack.setParallel(!D.getSingleClause<OMPSafelenClause>());
2449 RValue Len = CGF.EmitAnyExpr(C->getSafelen(), AggValueSlot::ignored(),
2452 CGF.LoopStack.setVectorizeWidth(Val->getZExtValue());
2456 CGF.LoopStack.setParallel(/*Enable=*/false);
2524 static void emitOMPLoopBodyWithStopPoint(CodeGenFunction &CGF,
2527 CGF.EmitOMPLoopBody(S, LoopExit);
2528 CGF.EmitStopPoint(&S);
2532 static LValue EmitOMPHelperVar(CodeGenFunction &CGF,
2535 CGF.EmitVarDecl(*VDecl);
2536 return CGF.EmitLValue(Helper);
2539 static void emitCommonSimdLoop(CodeGenFunction &CGF, const OMPLoopDirective &S,
2542 auto &&ThenGen = [&S, &SimdInitGen, &BodyCodeGen](CodeGenFunction &CGF,
2544 CGOpenMPRuntime::NontemporalDeclsRAII NontemporalsRegion(CGF.CGM, S);
2545 CodeGenFunction::OMPLocalDeclMapRAII Scope(CGF);
2546 SimdInitGen(CGF);
2548 BodyCodeGen(CGF);
2550 auto &&ElseGen = [&BodyCodeGen](CodeGenFunction &CGF, PrePostActionTy &) {
2551 CodeGenFunction::OMPLocalDeclMapRAII Scope(CGF);
2552 CGF.LoopStack.setVectorizeEnable(/*Enable=*/false);
2554 BodyCodeGen(CGF);
2560 if (CGF.getLangOpts().OpenMP >= 50 &&
2569 CGF.CGM.getOpenMPRuntime().emitIfClause(CGF, IfCond, ThenGen, ElseGen);
2572 ThenRCG(CGF);
2576 static void emitOMPSimdRegion(CodeGenFunction &CGF, const OMPLoopDirective &S,
2578 Action.Enter(CGF);
2579 OMPLoopScope PreInitScope(CGF, S);
2592 (void)EmitOMPHelperVar(CGF, cast<DeclRefExpr>(S.getLowerBoundVariable()));
2593 (void)EmitOMPHelperVar(CGF, cast<DeclRefExpr>(S.getUpperBoundVariable()));
2602 if (CGF.ConstantFoldsToSimpleInteger(S.getPreCond(), CondConstant)) {
2606 llvm::BasicBlock *ThenBlock = CGF.createBasicBlock("simd.if.then");
2607 ContBlock = CGF.createBasicBlock("simd.if.end");
2608 emitPreCond(CGF, S, S.getPreCond(), ThenBlock, ContBlock,
2609 CGF.getProfileCount(&S));
2610 CGF.EmitBlock(ThenBlock);
2611 CGF.incrementProfileCounter(&S);
2617 CGF.EmitVarDecl(*IVDecl);
2618 CGF.EmitIgnoredExpr(S.getInit());
2624 CGF.EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl()));
2626 CGF.EmitIgnoredExpr(S.getCalcLastIteration());
2629 emitAlignedClause(CGF, S);
2630 (void)CGF.EmitOMPLinearClauseInit(S);
2632 CodeGenFunction::OMPPrivateScope LoopScope(CGF);
2633 CGF.EmitOMPPrivateClause(S, LoopScope);
2634 CGF.EmitOMPPrivateLoopCounters(S, LoopScope);
2635 CGF.EmitOMPLinearClause(S, LoopScope);
2636 CGF.EmitOMPReductionClauseInit(S, LoopScope);
2638 CGF, S, CGF.EmitLValue(S.getIterationVariable()));
2639 bool HasLastprivateClause = CGF.EmitOMPLastprivateClauseInit(S, LoopScope);
2642 CGF.CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(CGF, S);
2645 CGF, S,
2646 [&S](CodeGenFunction &CGF, PrePostActionTy &) {
2647 CGF.EmitOMPSimdInit(S);
2649 [&S, &LoopScope](CodeGenFunction &CGF, PrePostActionTy &) {
2650 CGF.EmitOMPInnerLoop(
2652 [&S](CodeGenFunction &CGF) {
2653 emitOMPLoopBodyWithStopPoint(CGF, S,
2658 CGF.EmitOMPSimdFinal(S, [](CodeGenFunction &) { return nullptr; });
2661 CGF.EmitOMPLastprivateClauseFinal(S, /*NoFinals=*/true);
2662 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_simd);
2663 emitPostUpdateForReductionClause(CGF, S,
2666 CGF.EmitOMPLinearClauseFinal(S, [](CodeGenFunction &) { return nullptr; });
2670 CGF.EmitBranch(ContBlock);
2671 CGF.EmitBlock(ContBlock, true);
2710 GetAlignedMapping(const OMPLoopDirective &S, CodeGenFunction &CGF) {
2716 cast<llvm::ConstantInt>(CGF.EmitScalarExpr(AlignmentExpr));
2726 CGF.getContext()
2727 .toCharUnitsFromBits(CGF.getContext().getOpenMPDefaultSimdAlign(
2733 llvm::Value *PtrValue = CGF.EmitScalarExpr(E);
2734 AlignedVars[PtrValue] = CGF.Builder.getInt64(Alignment.getSExtValue());
2743 CodeGenFunction &CGF, CodeGenModule &CGM) {
2747 auto &&CodeGenIRBuilder = [&S, &CGM, UseOMPIRBuilder](CodeGenFunction &CGF,
2752 GetAlignedMapping(S, CGF);
2756 CGF.EmitOMPCollapsedCanonicalLoopNest(Inner, 1);
2763 RValue Len = CGF.EmitAnyExpr(C->getSimdlen(), AggValueSlot::ignored(),
2770 RValue Len = CGF.EmitAnyExpr(C->getSafelen(), AggValueSlot::ignored(),
2790 CGOpenMPRuntime::LastprivateConditionalRAII::disable(CGF, S);
2791 OMPLexicalScope Scope(CGF, S, OMPD_unknown);
2792 CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_simd,
2798 CodeGenFunction::ParentLoopDirectiveForScanRegion ScanRegion(CGF, S);
2799 CGF.OMPFirstScanLoop = true;
2800 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
2801 emitOMPSimdRegion(CGF, S, Action);
2805 CGOpenMPRuntime::LastprivateConditionalRAII::disable(CGF, S);
2806 OMPLexicalScope Scope(CGF, S, OMPD_unknown);
2807 CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_simd, CodeGen);
2810 checkForLastprivateConditionalUpdate(CGF, S);
2962 [&S, IsMonotonic, EKind](CodeGenFunction &CGF, PrePostActionTy &) {
2966 CGF.LoopStack.setParallel(!IsMonotonic);
2969 CGF.LoopStack.setParallel(/*Enable=*/true);
2971 CGF.EmitOMPSimdInit(S);
2975 &LoopScope](CodeGenFunction &CGF, PrePostActionTy &) {
2982 CGF.EmitOMPInnerLoop(
2984 [&S, LoopExit, &CodeGenLoop](CodeGenFunction &CGF) {
2985 CodeGenLoop(CGF, S, LoopExit);
2987 [IVSize, IVSigned, Loc, &CodeGenOrdered](CodeGenFunction &CGF) {
2988 CodeGenOrdered(CGF, Loc, IVSize, IVSigned);
3007 auto &&CodeGen = [DynamicOrOrdered, &S, &LoopArgs](CodeGenFunction &CGF) {
3009 CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getEndLoc(),
3103 auto &&CodeGenOrdered = [Ordered](CodeGenFunction &CGF, SourceLocation Loc,
3107 CGF.CGM.getOpenMPRuntime().emitForOrderedIterationEnd(CGF, Loc, IVSize,
3193 emitDistributeParallelForInnerBounds(CodeGenFunction &CGF,
3197 EmitOMPHelperVar(CGF, cast<DeclRefExpr>(LS.getLowerBoundVariable()));
3199 EmitOMPHelperVar(CGF, cast<DeclRefExpr>(LS.getUpperBoundVariable()));
3207 LValue PrevLB = CGF.EmitLValue(LS.getPrevLowerBoundVariable());
3208 LValue PrevUB = CGF.EmitLValue(LS.getPrevUpperBoundVariable());
3209 llvm::Value *PrevLBVal = CGF.EmitLoadOfScalar(
3211 PrevLBVal = CGF.EmitScalarConversion(
3215 llvm::Value *PrevUBVal = CGF.EmitLoadOfScalar(
3217 PrevUBVal = CGF.EmitScalarConversion(
3222 CGF.EmitStoreOfScalar(PrevLBVal, LB);
3223 CGF.EmitStoreOfScalar(PrevUBVal, UB);
3236 emitDistributeParallelForDispatchBounds(CodeGenFunction &CGF,
3247 CGF.EmitLoadOfScalar(LB, /*Volatile=*/false, IteratorTy, S.getBeginLoc());
3249 CGF.EmitLoadOfScalar(UB, /*Volatile=*/false, IteratorTy, S.getBeginLoc());
3254 CodeGenFunction &CGF, const OMPExecutableDirective &S,
3258 CGF.EmitLValue(cast<DeclRefExpr>(Dir.getCombinedLowerBoundVariable()));
3259 llvm::Value *LBCast = CGF.Builder.CreateIntCast(
3260 CGF.Builder.CreateLoad(LB.getAddress()), CGF.SizeTy, /*isSigned=*/false);
3263 CGF.EmitLValue(cast<DeclRefExpr>(Dir.getCombinedUpperBoundVariable()));
3265 llvm::Value *UBCast = CGF.Builder.CreateIntCast(
3266 CGF.Builder.CreateLoad(UB.getAddress()), CGF.SizeTy, /*isSigned=*/false);
3271 emitInnerParallelForWhenCombined(CodeGenFunction &CGF,
3275 auto &&CGInlinedWorksharingLoop = [&S, EKind](CodeGenFunction &CGF,
3277 Action.Enter(CGF);
3288 CodeGenFunction::OMPCancelStackRAII CancelRegion(CGF, EKind, HasCancel);
3289 CGF.EmitOMPWorksharingLoop(S, S.getPrevEnsureUpperBound(),
3295 CGF, S, isOpenMPSimdDirective(EKind) ? OMPD_for_simd : OMPD_for,
3302 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
3303 CGF.EmitOMPDistributeLoop(S, emitInnerParallelForWhenCombined,
3312 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
3313 CGF.EmitOMPDistributeLoop(S, emitInnerParallelForWhenCombined,
3322 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
3323 CGF.EmitOMPDistributeLoop(S, emitOMPLoopBodyWithStopPoint, S.getInc());
3332 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
3333 emitOMPSimdRegion(CGF, S, Action);
3345 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
3346 emitOMPSimdRegion(CGF, S, Action);
3500 [&S, EKind](CodeGenFunction &CGF, PrePostActionTy &) {
3502 CGF.EmitOMPSimdInit(S);
3505 CGF.LoopStack.setParallel(/*Enable=*/true);
3510 &LoopScope](CodeGenFunction &CGF, PrePostActionTy &) {
3520 CGF.CGM.getOpenMPRuntime().emitForStaticInit(
3521 CGF, S.getBeginLoc(), EKind, ScheduleKind, StaticInit);
3524 CGF.EmitIgnoredExpr(S.getEnsureUpperBound());
3526 CGF.EmitIgnoredExpr(S.getInit());
3540 CGF.EmitOMPInnerLoop(
3545 [&S, LoopExit](CodeGenFunction &CGF) {
3546 emitOMPLoopBodyWithStopPoint(CGF, S, LoopExit);
3552 auto &&CodeGen = [&S](CodeGenFunction &CGF) {
3553 CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getEndLoc(),
3568 EmitOMPSimdFinal(S, [IL, &S](CodeGenFunction &CGF) {
3569 return CGF.Builder.CreateIsNotNull(
3570 CGF.EmitLoadOfScalar(IL, S.getBeginLoc()));
3579 *this, S, [IL, &S](CodeGenFunction &CGF) {
3580 return CGF.Builder.CreateIsNotNull(
3581 CGF.EmitLoadOfScalar(IL, S.getBeginLoc()));
3589 EmitOMPLinearClauseFinal(S, [IL, &S](CodeGenFunction &CGF) {
3590 return CGF.Builder.CreateIsNotNull(
3591 CGF.EmitLoadOfScalar(IL, S.getBeginLoc()));
3608 emitForLoopBounds(CodeGenFunction &CGF, const OMPExecutableDirective &S) {
3611 EmitOMPHelperVar(CGF, cast<DeclRefExpr>(LS.getLowerBoundVariable()));
3613 EmitOMPHelperVar(CGF, cast<DeclRefExpr>(LS.getUpperBoundVariable()));
3622 emitDispatchForLoopBounds(CodeGenFunction &CGF, const OMPExecutableDirective &S,
3626 const unsigned IVSize = CGF.getContext().getTypeSize(IVExpr->getType());
3627 llvm::Value *LBVal = CGF.Builder.getIntN(IVSize, 0);
3628 llvm::Value *UBVal = CGF.EmitScalarExpr(LS.getLastIteration());
3640 CodeGenFunction &CGF, const OMPLoopDirective &S,
3642 llvm::Value *OMPScanNumIterations = CGF.Builder.CreateIntCast(
3643 NumIteratorsGen(CGF), CGF.SizeTy, /*isSigned=*/false);
3669 RedCG.emitSharedOrigLValue(CGF, Count);
3670 RedCG.emitAggregateType(CGF, Count);
3673 CGF,
3679 CGF.EmitVarDecl(*cast<VarDecl>(cast<DeclRefExpr>(*ITA)->getDecl()));
3692 CodeGenFunction &CGF, const OMPLoopDirective &S,
3694 llvm::Value *OMPScanNumIterations = CGF.Builder.CreateIntCast(
3695 NumIteratorsGen(CGF), CGF.SizeTy, /*isSigned=*/false);
3715 llvm::Value *OMPLast = CGF.Builder.CreateNSWSub(
3717 llvm::ConstantInt::get(CGF.SizeTy, 1, /*isSigned=*/false));
3723 CGF,
3727 LValue DestLVal = CGF.EmitLValue(OrigExpr);
3728 LValue SrcLVal = CGF.EmitLValue(CopyArrayElem);
3729 CGF.EmitOMPCopy(
3756 CodeGenFunction &CGF, const OMPLoopDirective &S,
3760 llvm::Value *OMPScanNumIterations = CGF.Builder.CreateIntCast(
3761 NumIteratorsGen(CGF), CGF.SizeTy, /*isSigned=*/false);
3777 CodeGenFunction::ParentLoopDirectiveForScanRegion ScanRegion(CGF, S);
3785 CGF.OMPFirstScanLoop = true;
3786 CodeGenFunction::OMPLocalDeclMapRAII Scope(CGF);
3787 FirstGen(CGF);
3792 &Privates](CodeGenFunction &CGF, PrePostActionTy &Action) {
3793 Action.Enter(CGF);
3797 llvm::BasicBlock *InputBB = CGF.Builder.GetInsertBlock();
3798 llvm::BasicBlock *LoopBB = CGF.createBasicBlock("omp.outer.log.scan.body");
3799 llvm::BasicBlock *ExitBB = CGF.createBasicBlock("omp.outer.log.scan.exit");
3801 CGF.CGM.getIntrinsic(llvm::Intrinsic::log2, CGF.DoubleTy);
3803 CGF.Builder.CreateUIToFP(OMPScanNumIterations, CGF.DoubleTy);
3804 llvm::Value *LogVal = CGF.EmitNounwindRuntimeCall(F, Arg);
3805 F = CGF.CGM.getIntrinsic(llvm::Intrinsic::ceil, CGF.DoubleTy);
3806 LogVal = CGF.EmitNounwindRuntimeCall(F, LogVal);
3807 LogVal = CGF.Builder.CreateFPToUI(LogVal, CGF.IntTy);
3808 llvm::Value *NMin1 = CGF.Builder.CreateNUWSub(
3809 OMPScanNumIterations, llvm::ConstantInt::get(CGF.SizeTy, 1));
3810 auto DL = ApplyDebugLocation::CreateDefaultArtificial(CGF, S.getBeginLoc());
3811 CGF.EmitBlock(LoopBB);
3812 auto *Counter = CGF.Builder.CreatePHI(CGF.IntTy, 2);
3814 auto *Pow2K = CGF.Builder.CreatePHI(CGF.SizeTy, 2);
3815 Counter->addIncoming(llvm::ConstantInt::get(CGF.IntTy, 0), InputBB);
3816 Pow2K->addIncoming(llvm::ConstantInt::get(CGF.SizeTy, 1), InputBB);
3820 CGF.createBasicBlock("omp.inner.log.scan.body");
3822 CGF.createBasicBlock("omp.inner.log.scan.exit");
3823 llvm::Value *CmpI = CGF.Builder.CreateICmpUGE(NMin1, Pow2K);
3824 CGF.Builder.CreateCondBr(CmpI, InnerLoopBB, InnerExitBB);
3825 CGF.EmitBlock(InnerLoopBB);
3826 auto *IVal = CGF.Builder.CreatePHI(CGF.SizeTy, 2);
3829 CodeGenFunction::OMPPrivateScope PrivScope(CGF);
3838 CGF,
3842 LHSAddr = CGF.EmitLValue(CopyArrayElem).getAddress();
3847 llvm::Value *OffsetIVal = CGF.Builder.CreateNUWSub(IVal, Pow2K);
3849 CGF,
3853 RHSAddr = CGF.EmitLValue(CopyArrayElem).getAddress();
3860 CGF.CGM.getOpenMPRuntime().emitReduction(
3861 CGF, S.getEndLoc(), Privates, LHSs, RHSs, ReductionOps,
3865 CGF.Builder.CreateNUWSub(IVal, llvm::ConstantInt::get(CGF.SizeTy, 1));
3866 IVal->addIncoming(NextIVal, CGF.Builder.GetInsertBlock());
3867 CmpI = CGF.Builder.CreateICmpUGE(NextIVal, Pow2K);
3868 CGF.Builder.CreateCondBr(CmpI, InnerLoopBB, InnerExitBB);
3869 CGF.EmitBlock(InnerExitBB);
3871 CGF.Builder.CreateNUWAdd(Counter, llvm::ConstantInt::get(CGF.IntTy, 1));
3872 Counter->addIncoming(Next, CGF.Builder.GetInsertBlock());
3875 CGF.Builder.CreateShl(Pow2K, 1, "", /*HasNUW=*/true);
3876 Pow2K->addIncoming(NextPow2K, CGF.Builder.GetInsertBlock());
3877 llvm::Value *Cmp = CGF.Builder.CreateICmpNE(Next, LogVal);
3878 CGF.Builder.CreateCondBr(Cmp, LoopBB, ExitBB);
3879 auto DL1 = ApplyDebugLocation::CreateDefaultArtificial(CGF, S.getEndLoc());
3880 CGF.EmitBlock(ExitBB);
3884 CGF.CGM.getOpenMPRuntime().emitMasterRegion(CGF, CodeGen, S.getBeginLoc());
3885 CGF.CGM.getOpenMPRuntime().emitBarrierCall(
3886 CGF, S.getBeginLoc(), OMPD_unknown, /*EmitChecks=*/false,
3890 RCG(CGF);
3893 CGF.OMPFirstScanLoop = false;
3894 SecondGen(CGF);
3897 static bool emitWorksharingDirective(CodeGenFunction &CGF,
3906 const auto &&NumIteratorsGen = [&S](CodeGenFunction &CGF) {
3907 CodeGenFunction::OMPLocalDeclMapRAII Scope(CGF);
3908 OMPLoopScope LoopScope(CGF, S);
3909 return CGF.EmitScalarExpr(S.getNumIterations());
3911 const auto &&FirstGen = [&S, HasCancel, EKind](CodeGenFunction &CGF) {
3912 CodeGenFunction::OMPCancelStackRAII CancelRegion(CGF, EKind, HasCancel);
3913 (void)CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(),
3917 CGF.CGM.getOpenMPRuntime().emitBarrierCall(CGF, S.getBeginLoc(),
3921 &HasLastprivates](CodeGenFunction &CGF) {
3922 CodeGenFunction::OMPCancelStackRAII CancelRegion(CGF, EKind, HasCancel);
3923 HasLastprivates = CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(),
3928 emitScanBasedDirectiveDecls(CGF, S, NumIteratorsGen);
3929 emitScanBasedDirective(CGF, S, NumIteratorsGen, FirstGen, SecondGen);
3931 emitScanBasedDirectiveFinals(CGF, S, NumIteratorsGen);
3933 CodeGenFunction::OMPCancelStackRAII CancelRegion(CGF, EKind, HasCancel);
3934 HasLastprivates = CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(),
3995 static void emitOMPForDirective(const OMPLoopDirective &S, CodeGenFunction &CGF,
4001 UseOMPIRBuilder](CodeGenFunction &CGF, PrePostActionTy &) {
4012 ChunkSize = CGF.EmitScalarExpr(ChunkSizeExpr);
4018 CGF.EmitOMPCollapsedCanonicalLoopNest(Inner, 1);
4023 CGF.AllocaInsertPt->getParent(), CGF.AllocaInsertPt->getIterator());
4025 CGF.Builder.getCurrentDebugLocation(), CLI, AllocaIP, NeedsBarrier,
4032 HasLastprivates = emitWorksharingDirective(CGF, S, HasCancel);
4036 CGOpenMPRuntime::LastprivateConditionalRAII::disable(CGF, S);
4037 OMPLexicalScope Scope(CGF, S, OMPD_unknown);
4038 CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_for, CodeGen,
4045 CGM.getOpenMPRuntime().emitBarrierCall(CGF, S.getBeginLoc(), OMPD_for);
4048 checkForLastprivateConditionalUpdate(CGF, S);
4057 auto &&CodeGen = [&S, &HasLastprivates](CodeGenFunction &CGF,
4059 HasLastprivates = emitWorksharingDirective(CGF, S, /*HasCancel=*/false);
4075 static LValue createSectionLVal(CodeGenFunction &CGF, QualType Ty,
4078 LValue LVal = CGF.MakeAddrLValue(CGF.CreateMemTemp(Ty, Name), Ty);
4080 CGF.EmitStoreThroughLValue(RValue::get(Init), LVal, /*isInit*/ true);
4090 &HasLastprivates](CodeGenFunction &CGF, PrePostActionTy &) {
4091 const ASTContext &C = CGF.getContext();
4095 LValue LB = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.lb.",
4096 CGF.Builder.getInt32(0));
4098 ? CGF.Builder.getInt32(CS->size() - 1)
4099 : CGF.Builder.getInt32(0);
4101 createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.ub.", GlobalUBVal);
4102 LValue ST = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.st.",
4103 CGF.Builder.getInt32(1));
4104 LValue IL = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.il.",
4105 CGF.Builder.getInt32(0));
4107 LValue IV = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.iv.");
4109 CodeGenFunction::OpaqueValueMapping OpaqueIV(CGF, &IVRefExpr, IV);
4111 CodeGenFunction::OpaqueValueMapping OpaqueUB(CGF, &UBRefExpr, UB);
4120 auto &&BodyGen = [CapturedStmt, CS, &S, &IV](CodeGenFunction &CGF) {
4132 llvm::BasicBlock *ExitBB = CGF.createBasicBlock(".omp.sections.exit");
4134 CGF.Builder.CreateSwitch(CGF.EmitLoadOfScalar(IV, S.getBeginLoc()),
4139 auto CaseBB = CGF.createBasicBlock(".omp.sections.case");
4140 CGF.EmitBlock(CaseBB);
4141 SwitchStmt->addCase(CGF.Builder.getInt32(CaseNumber), CaseBB);
4142 CGF.EmitStmt(SubStmt);
4143 CGF.EmitBranch(ExitBB);
4147 llvm::BasicBlock *CaseBB = CGF.createBasicBlock(".omp.sections.case");
4148 CGF.EmitBlock(CaseBB);
4149 SwitchStmt->addCase(CGF.Builder.getInt32(0), CaseBB);
4150 CGF.EmitStmt(CapturedStmt);
4151 CGF.EmitBranch(ExitBB);
4153 CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
4156 CodeGenFunction::OMPPrivateScope LoopScope(CGF);
4157 if (CGF.EmitOMPFirstprivateClause(S, LoopScope)) {
4161 CGF.CGM.getOpenMPRuntime().emitBarrierCall(
4162 CGF, S.getBeginLoc(), OMPD_unknown, /*EmitChecks=*/false,
4165 CGF.EmitOMPPrivateClause(S, LoopScope);
4166 CGOpenMPRuntime::LastprivateConditionalRAII LPCRegion(CGF, S, IV);
4167 HasLastprivates = CGF.EmitOMPLastprivateClauseInit(S, LoopScope);
4168 CGF.EmitOMPReductionClauseInit(S, LoopScope);
4171 CGF.CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(CGF, S);
4179 CGF.CGM.getOpenMPRuntime().emitForStaticInit(CGF, S.getBeginLoc(), EKind,
4182 llvm::Value *UBVal = CGF.EmitLoadOfScalar(UB, S.getBeginLoc());
4183 llvm::Value *MinUBGlobalUB = CGF.Builder.CreateSelect(
4184 CGF.Builder.CreateICmpSLT(UBVal, GlobalUBVal), UBVal, GlobalUBVal);
4185 CGF.EmitStoreOfScalar(MinUBGlobalUB, UB);
4187 CGF.EmitStoreOfScalar(CGF.EmitLoadOfScalar(LB, S.getBeginLoc()), IV);
4189 CGF.EmitOMPInnerLoop(S, /*RequiresCleanup=*/false, Cond, Inc, BodyGen,
4192 auto &&CodeGen = [&S](CodeGenFunction &CGF) {
4193 CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getEndLoc(),
4196 CGF.OMPCancelStack.emitExit(CGF, EKind, CodeGen);
4197 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_parallel);
4199 emitPostUpdateForReductionClause(CGF, S, [IL, &S](CodeGenFunction &CGF) {
4200 return CGF.Builder.CreateIsNotNull(
4201 CGF.EmitLoadOfScalar(IL, S.getBeginLoc()));
4206 CGF.EmitOMPLastprivateClauseFinal(
4208 CGF.Builder.CreateIsNotNull(
4209 CGF.EmitLoadOfScalar(IL, S.getBeginLoc())));
4234 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4235 Action.Enter(CGF);
4236 OMPPrivateScope PrivateScope(CGF);
4237 (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);
4238 CGF.EmitOMPPrivateClause(S, PrivateScope);
4239 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4241 CGF.EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt());
4242 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_parallel);
4380 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4381 Action.Enter(CGF);
4382 OMPPrivateScope SingleScope(CGF);
4383 (void)CGF.EmitOMPFirstprivateClause(S, SingleScope);
4384 CGF.EmitOMPPrivateClause(S, SingleScope);
4386 CGF.EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt());
4407 static void emitMaster(CodeGenFunction &CGF, const OMPExecutableDirective &S) {
4408 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4409 Action.Enter(CGF);
4410 CGF.EmitStmt(S.getRawStmt());
4412 CGF.CGM.getOpenMPRuntime().emitMasterRegion(CGF, CodeGen, S.getBeginLoc());
4447 static void emitMasked(CodeGenFunction &CGF, const OMPExecutableDirective &S) {
4448 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4449 Action.Enter(CGF);
4450 CGF.EmitStmt(S.getRawStmt());
4455 CGF.CGM.getOpenMPRuntime().emitMaskedRegion(CGF, CodeGen, S.getBeginLoc(),
4538 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4539 Action.Enter(CGF);
4540 CGF.EmitStmt(S.getAssociatedStmt());
4556 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4557 Action.Enter(CGF);
4558 emitOMPCopyinClause(CGF, S);
4559 (void)emitWorksharingDirective(CGF, S, S.hasCancel());
4562 const auto &&NumIteratorsGen = [&S](CodeGenFunction &CGF) {
4563 CodeGenFunction::OMPLocalDeclMapRAII Scope(CGF);
4565 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGSI);
4566 OMPLoopScope LoopScope(CGF, S);
4567 return CGF.EmitScalarExpr(S.getNumIterations());
4590 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4591 Action.Enter(CGF);
4592 emitOMPCopyinClause(CGF, S);
4593 (void)emitWorksharingDirective(CGF, S, /*HasCancel=*/false);
4596 const auto &&NumIteratorsGen = [&S](CodeGenFunction &CGF) {
4597 CodeGenFunction::OMPLocalDeclMapRAII Scope(CGF);
4599 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGSI);
4600 OMPLoopScope LoopScope(CGF, S);
4601 return CGF.EmitScalarExpr(S.getNumIterations());
4624 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4625 Action.Enter(CGF);
4626 OMPPrivateScope PrivateScope(CGF);
4627 emitOMPCopyinClause(CGF, S);
4628 (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);
4629 CGF.EmitOMPPrivateClause(S, PrivateScope);
4630 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4632 emitMaster(CGF, S);
4633 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_parallel);
4651 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4652 Action.Enter(CGF);
4653 OMPPrivateScope PrivateScope(CGF);
4654 emitOMPCopyinClause(CGF, S);
4655 (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);
4656 CGF.EmitOMPPrivateClause(S, PrivateScope);
4657 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4659 emitMasked(CGF, S);
4660 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_parallel);
4678 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4679 Action.Enter(CGF);
4680 emitOMPCopyinClause(CGF, S);
4681 CGF.EmitSections(S);
4867 CapturedRegion](CodeGenFunction &CGF,
4873 OMPPrivateScope Scope(CGF);
4875 if (auto *DI = CGF.getDebugInfo()) {
4877 CGF.CapturedStmtInfo->getCaptureFields();
4878 llvm::Value *ContextValue = CGF.CapturedStmtInfo->getContextValue();
4880 unsigned CharWidth = CGF.getContext().getCharWidth();
4898 CGF.getContext().getASTRecordLayout(CaptureRecord);
4901 if (CGF.CGM.getCodeGenOpts().hasReducedDebugInfo())
4903 CGF.Builder, false);
4917 llvm::Instruction &Last = CGF.Builder.GetInsertBlock()->back();
4924 CGF.Builder.GetInsertBlock()->getTrailingDbgRecords()) {
4938 llvm::Value *CopyFn = CGF.Builder.CreateLoad(
4939 CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(CopyFnParam)));
4940 llvm::Value *PrivatesPtr = CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(
4950 RawAddress PrivatePtr = CGF.CreateMemTemp(
4951 CGF.getContext().getPointerType(E->getType()), ".priv.ptr.addr");
4959 CGF.CreateMemTemp(CGF.getContext().getPointerType(E->getType()),
4969 CGF.CreateMemTemp(CGF.getContext().getPointerType(E->getType()),
4978 Ty = CGF.getContext().getPointerType(Ty);
4980 Ty = CGF.getContext().getPointerType(Ty);
4981 RawAddress PrivatePtr = CGF.CreateMemTemp(
4982 CGF.getContext().getPointerType(Ty), ".local.ptr.addr");
4992 auto *CopyFnTy = llvm::FunctionType::get(CGF.Builder.getVoidTy(),
4994 CGF.CGM.getOpenMPRuntime().emitOutlinedFunctionCall(
4995 CGF, S.getBeginLoc(), {CopyFnTy, CopyFn}, CallArgs);
4998 DeclRefExpr DRE(CGF.getContext(), const_cast<VarDecl *>(OrigVD),
5000 CGF.CapturedStmtInfo->lookup(OrigVD) != nullptr,
5003 Scope.addPrivate(Pair.first, CGF.EmitLValue(&DRE).getAddress());
5007 CGF.Builder.CreateLoad(Pair.second),
5008 CGF.ConvertTypeForMem(Pair.first->getType().getNonReferenceType()),
5009 CGF.getContext().getDeclAlign(Pair.first));
5011 if (auto *DI = CGF.getDebugInfo())
5012 if (CGF.CGM.getCodeGenOpts().hasReducedDebugInfo())
5014 Pair.first, Pair.second.getBasePointer(), CGF.Builder,
5022 VDType = CGF.getContext().getPointerType(VDType);
5024 llvm::Value *Ptr = CGF.Builder.CreateLoad(Pair.second.first);
5027 CGF.ConvertTypeForMem(CGF.getContext().getPointerType(VDType)),
5028 CGF.getPointerAlign());
5030 Ptr = CGF.Builder.CreateLoad(Replacement);
5031 Replacement = Address(Ptr, CGF.ConvertTypeForMem(VDType),
5032 CGF.getContext().getDeclAlign(Pair.first));
5035 llvm::Value *Ptr = CGF.Builder.CreateLoad(Pair.second.first);
5036 Address Replacement(Ptr, CGF.ConvertTypeForMem(VDType),
5037 CGF.getContext().getDeclAlign(Pair.first));
5043 OMPPrivateScope FirstprivateScope(CGF);
5046 CGF.Builder.CreateLoad(Pair.second),
5047 CGF.ConvertTypeForMem(Pair.first->getType().getNonReferenceType()),
5048 CGF.getContext().getDeclAlign(Pair.first));
5052 OMPLexicalScope LexScope(CGF, S, CapturedRegion);
5055 llvm::Value *ReductionsPtr = CGF.Builder.CreateLoad(
5056 CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(9)));
5058 RedCG.emitSharedOrigLValue(CGF, Cnt);
5059 RedCG.emitAggregateType(CGF, Cnt);
5063 CGF.CGM.getOpenMPRuntime().emitTaskReductionFixups(CGF, S.getBeginLoc(),
5065 Address Replacement = CGF.CGM.getOpenMPRuntime().getTaskReductionItem(
5066 CGF, S.getBeginLoc(), ReductionsPtr, RedCG.getSharedLValue(Cnt));
5068 CGF.EmitScalarConversion(Replacement.emitRawPointer(CGF),
5069 CGF.getContext().VoidPtrTy,
5070 CGF.getContext().getPointerType(
5073 CGF.ConvertTypeForMem(Data.ReductionCopies[Cnt]->getType()),
5075 Replacement = RedCG.adjustPrivateAddress(CGF, Cnt, Replacement);
5101 OMPPrivateScope InRedScope(CGF);
5105 RedCG.emitSharedOrigLValue(CGF, Cnt);
5106 RedCG.emitAggregateType(CGF, Cnt);
5112 CGF.CGM.getOpenMPRuntime().emitTaskReductionFixups(CGF, S.getBeginLoc(),
5116 ReductionsPtr = CGF.EmitLoadOfScalar(CGF.EmitLValue(TRExpr),
5119 ReductionsPtr = llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
5121 Address Replacement = CGF.CGM.getOpenMPRuntime().getTaskReductionItem(
5122 CGF, S.getBeginLoc(), ReductionsPtr, RedCG.getSharedLValue(Cnt));
5124 CGF.EmitScalarConversion(
5125 Replacement.emitRawPointer(CGF), CGF.getContext().VoidPtrTy,
5126 CGF.getContext().getPointerType(InRedPrivs[Cnt]->getType()),
5128 CGF.ConvertTypeForMem(InRedPrivs[Cnt]->getType()),
5130 Replacement = RedCG.adjustPrivateAddress(CGF, Cnt, Replacement);
5136 CGOpenMPRuntime::UntiedTaskLocalDeclsRAII LocalVarsScope(CGF,
5138 Action.Enter(CGF);
5139 BodyGen(CGF);
5254 &InputInfo](CodeGenFunction &CGF, PrePostActionTy &Action) {
5256 OMPPrivateScope Scope(CGF);
5259 llvm::Value *CopyFn = CGF.Builder.CreateLoad(
5260 CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(CopyFnParam)));
5261 llvm::Value *PrivatesPtr = CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(
5272 CGF.CreateMemTemp(CGF.getContext().getPointerType(E->getType()),
5278 auto *CopyFnTy = llvm::FunctionType::get(CGF.Builder.getVoidTy(),
5280 CGF.CGM.getOpenMPRuntime().emitOutlinedFunctionCall(
5281 CGF, S.getBeginLoc(), {CopyFnTy, CopyFn}, CallArgs);
5284 CGF.Builder.CreateLoad(Pair.second),
5285 CGF.ConvertTypeForMem(Pair.first->getType().getNonReferenceType()),
5286 CGF.getContext().getDeclAlign(Pair.first));
5290 CGF.processInReduction(S, Data, CGF, CS, Scope);
5292 InputInfo.BasePointersArray = CGF.Builder.CreateConstArrayGEP(
5293 CGF.GetAddrOfLocalVar(BPVD), /*Index=*/0);
5294 InputInfo.PointersArray = CGF.Builder.CreateConstArrayGEP(
5295 CGF.GetAddrOfLocalVar(PVD), /*Index=*/0);
5296 InputInfo.SizesArray = CGF.Builder.CreateConstArrayGEP(
5297 CGF.GetAddrOfLocalVar(SVD), /*Index=*/0);
5300 InputInfo.MappersArray = CGF.Builder.CreateConstArrayGEP(
5301 CGF.GetAddrOfLocalVar(MVD), /*Index=*/0);
5304 Action.Enter(CGF);
5305 OMPLexicalScope LexScope(CGF, S, OMPD_task, /*EmitPreInitStmt=*/false);
5307 if (CGF.CGM.getLangOpts().OpenMP >= 51 &&
5312 CGF.CGM.getOpenMPRuntime().emitThreadLimitClause(
5313 CGF, TL->getThreadLimit().front(), S.getBeginLoc());
5315 BodyGen(CGF);
5330 CodeGenFunction &CGF,
5336 OMPLexicalScope LexScope(CGF, S, CapturedRegion);
5339 llvm::Value *ReductionsPtr = CGF.Builder.CreateLoad(
5340 CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(4)));
5342 RedCG.emitSharedOrigLValue(CGF, Cnt);
5343 RedCG.emitAggregateType(CGF, Cnt);
5347 CGF.CGM.getOpenMPRuntime().emitTaskReductionFixups(CGF, S.getBeginLoc(),
5349 Address Replacement = CGF.CGM.getOpenMPRuntime().getTaskReductionItem(
5350 CGF, S.getBeginLoc(), ReductionsPtr, RedCG.getSharedLValue(Cnt));
5352 CGF.EmitScalarConversion(Replacement.emitRawPointer(CGF),
5353 CGF.getContext().VoidPtrTy,
5354 CGF.getContext().getPointerType(
5357 CGF.ConvertTypeForMem(Data.ReductionCopies[Cnt]->getType()),
5359 Replacement = RedCG.adjustPrivateAddress(CGF, Cnt, Replacement);
5382 OMPPrivateScope InRedScope(CGF);
5386 RedCG.emitSharedOrigLValue(CGF, Cnt);
5387 RedCG.emitAggregateType(CGF, Cnt);
5391 CGF.CGM.getOpenMPRuntime().emitTaskReductionFixups(CGF, S.getBeginLoc(),
5396 CGF.EmitLoadOfScalar(CGF.EmitLValue(TRExpr), TRExpr->getExprLoc());
5398 ReductionsPtr = llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
5400 Address Replacement = CGF.CGM.getOpenMPRuntime().getTaskReductionItem(
5401 CGF, S.getBeginLoc(), ReductionsPtr, RedCG.getSharedLValue(Cnt));
5403 CGF.EmitScalarConversion(
5404 Replacement.emitRawPointer(CGF), CGF.getContext().VoidPtrTy,
5405 CGF.getContext().getPointerType(InRedPrivs[Cnt]->getType()),
5407 CGF.ConvertTypeForMem(InRedPrivs[Cnt]->getType()),
5409 Replacement = RedCG.adjustPrivateAddress(CGF, Cnt, Replacement);
5433 auto &&BodyGen = [CS](CodeGenFunction &CGF, PrePostActionTy &) {
5434 CGF.EmitStmt(CS->getCapturedStmt());
5437 IfCond](CodeGenFunction &CGF, llvm::Function *OutlinedFn,
5439 CGF.CGM.getOpenMPRuntime().emitTaskCall(CGF, S.getBeginLoc(), S, OutlinedFn,
5502 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
5503 Action.Enter(CGF);
5518 CGF.CGM.getOpenMPRuntime().emitTaskReductionInit(CGF, S.getBeginLoc(),
5521 CGF.EmitVarDecl(*VD);
5522 CGF.EmitStoreOfScalar(ReductionDesc, CGF.GetAddrOfLocalVar(VD),
5525 CGF.EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt());
5934 [&S](CodeGenFunction &CGF, PrePostActionTy &) {
5936 CGF.EmitOMPSimdInit(S);
5939 StaticChunked](CodeGenFunction &CGF, PrePostActionTy &) {
5940 CGF.EmitOMPInnerLoop(
5942 [&S, LoopExit, &CodeGenLoop](CodeGenFunction &CGF) {
5943 CodeGenLoop(CGF, S, LoopExit);
5945 [&S, StaticChunked](CodeGenFunction &CGF) {
5947 CGF.EmitIgnoredExpr(S.getCombinedNextLowerBound());
5948 CGF.EmitIgnoredExpr(S.getCombinedNextUpperBound());
5949 CGF.EmitIgnoredExpr(S.getCombinedEnsureUpperBound());
5950 CGF.EmitIgnoredExpr(S.getCombinedInit());
5967 EmitOMPSimdFinal(S, [IL, &S](CodeGenFunction &CGF) {
5968 return CGF.Builder.CreateIsNotNull(
5969 CGF.EmitLoadOfScalar(IL, S.getBeginLoc()));
5978 *this, S, [IL, &S](CodeGenFunction &CGF) {
5979 return CGF.Builder.CreateIsNotNull(
5980 CGF.EmitLoadOfScalar(IL, S.getBeginLoc()));
6002 CodeGenFunction &CGF,
6004 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
6005 CGF.EmitOMPDistributeLoop(S, emitOMPLoopBodyWithStopPoint, S.getInc());
6007 OMPLexicalScope Scope(CGF, S, OMPD_unknown);
6008 CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute, CodeGen);
6019 CodeGenFunction CGF(CGM, /*suppressNewContext=*/true);
6021 CGF.CapturedStmtInfo = &CapStmtInfo;
6022 llvm::Function *Fn = CGF.GenerateOpenMPCapturedStmtFunction(*S, Loc);
6028 static void emitRestoreIP(CodeGenFunction &CGF, const T *C,
6033 QualType Int64Ty = CGF.CGM.getContext().getIntTypeForBitwidth(
6039 llvm::Value *StoreValue = CGF.EmitScalarConversion(
6040 CGF.EmitScalarExpr(CounterVal), CounterVal->getType(), Int64Ty,
6046 CGF.Builder.restoreIP(
6047 OMPBuilder.createOrderedDepend(CGF.Builder, AllocaIP, NumLoops,
6124 auto &&CodeGen = [&S, C, this](CodeGenFunction &CGF,
6129 CGF.GenerateOpenMPCapturedVars(*CS, CapturedVars);
6132 CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF, S.getBeginLoc(),
6135 Action.Enter(CGF);
6136 CGF.EmitStmt(CS->getCapturedStmt());
6143 static llvm::Value *convertToScalarValue(CodeGenFunction &CGF, RValue Val,
6146 assert(CGF.hasScalarEvaluationKind(DestType) &&
6149 return Val.isScalar() ? CGF.EmitScalarConversion(Val.getScalarVal(), SrcType,
6151 : CGF.EmitComplexToScalarConversion(
6156 convertToComplexValue(CodeGenFunction &CGF, RValue Val, QualType SrcType,
6158 assert(CGF.getEvaluationKind(DestType) == TEK_Complex &&
6165 llvm::Value *ScalarVal = CGF.EmitScalarConversion(
6174 ComplexVal.first = CGF.EmitScalarConversion(
6176 ComplexVal.second = CGF.EmitScalarConversion(
6182 static void emitSimpleAtomicStore(CodeGenFunction &CGF, llvm::AtomicOrdering AO,
6185 CGF.EmitStoreThroughGlobalRegLValue(RVal, LVal);
6187 CGF.EmitAtomicStore(RVal, LVal, AO, LVal.isVolatile(), /*isInit=*/false);
6190 static RValue emitSimpleAtomicLoad(CodeGenFunction &CGF,
6194 return CGF.EmitLoadOfLValue(LVal, Loc);
6195 return CGF.EmitAtomicLoad(
6218 static void emitOMPAtomicReadExpr(CodeGenFunction &CGF, llvm::AtomicOrdering AO,
6224 LValue XLValue = CGF.EmitLValue(X);
6225 LValue VLValue = CGF.EmitLValue(V);
6226 RValue Res = emitSimpleAtomicLoad(CGF, AO, XLValue, Loc);
6235 CGF.CGM.getOpenMPRuntime().emitFlush(CGF, {}, Loc,
6245 CGF.emitOMPSimpleStore(VLValue, Res, X->getType().getNonReferenceType(), Loc);
6246 CGF.CGM.getOpenMPRuntime().checkAndEmitLastprivateConditional(CGF, V);
6249 static void emitOMPAtomicWriteExpr(CodeGenFunction &CGF,
6254 emitSimpleAtomicStore(CGF, AO, CGF.EmitLValue(X), CGF.EmitAnyExpr(E));
6255 CGF.CGM.getOpenMPRuntime().checkAndEmitLastprivateConditional(CGF, X);
6264 CGF.CGM.getOpenMPRuntime().emitFlush(CGF, {}, Loc,
6276 static std::pair<bool, RValue> emitOMPAtomicRMW(CodeGenFunction &CGF, LValue X,
6281 ASTContext &Context = CGF.getContext();
6292 auto &&CheckAtomicSupport = [&CGF](llvm::Type *T, BinaryOperatorKind BO) {
6297 return llvm::isPowerOf2_64(CGF.CGM.getDataLayout().getTypeStoreSize(T));
6382 UpdateVal = CGF.Builder.CreateIntCast(
6386 UpdateVal = CGF.Builder.CreateCast(llvm::Instruction::CastOps::UIToFP, IC,
6390 CGF.emitAtomicRMWInst(RMWOp, X.getAddress(), UpdateVal, AO);
6418 static void emitOMPAtomicUpdateExpr(CodeGenFunction &CGF,
6432 LValue XLValue = CGF.EmitLValue(X);
6433 RValue ExprRValue = CGF.EmitAnyExpr(E);
6438 auto &&Gen = [&CGF, UE, ExprRValue, XRValExpr, ERValExpr](RValue XRValue) {
6439 CodeGenFunction::OpaqueValueMapping MapExpr(CGF, ERValExpr, ExprRValue);
6440 CodeGenFunction::OpaqueValueMapping MapX(CGF, XRValExpr, XRValue);
6441 return CGF.EmitAnyExpr(UE);
6443 (void)CGF.EmitOMPAtomicSimpleUpdateExpr(
6445 CGF.CGM.getOpenMPRuntime().checkAndEmitLastprivateConditional(CGF, X);
6454 CGF.CGM.getOpenMPRuntime().emitFlush(CGF, {}, Loc,
6466 static RValue convertToType(CodeGenFunction &CGF, RValue Value,
6469 switch (CGF.getEvaluationKind(ResType)) {
6472 convertToScalarValue(CGF, Value, SourceType, ResType, Loc));
6474 auto Res = convertToComplexValue(CGF, Value, SourceType, ResType, Loc);
6483 static void emitOMPAtomicCaptureExpr(CodeGenFunction &CGF,
6492 LValue VLValue = CGF.EmitLValue(V);
6493 LValue XLValue = CGF.EmitLValue(X);
6494 RValue ExprRValue = CGF.EmitAnyExpr(E);
6512 auto &&Gen = [&CGF, &NewVVal, UE, ExprRValue, XRValExpr, ERValExpr,
6514 CodeGenFunction::OpaqueValueMapping MapExpr(CGF, ERValExpr, ExprRValue);
6515 CodeGenFunction::OpaqueValueMapping MapX(CGF, XRValExpr, XRValue);
6516 RValue Res = CGF.EmitAnyExpr(UE);
6520 auto Res = CGF.EmitOMPAtomicSimpleUpdateExpr(
6522 CGF.CGM.getOpenMPRuntime().checkAndEmitLastprivateConditional(CGF, X);
6531 CodeGenFunction::OpaqueValueMapping MapExpr(CGF, ERValExpr, ExprRValue);
6532 CodeGenFunction::OpaqueValueMapping MapX(CGF, XRValExpr, Res.second);
6533 NewVVal = CGF.EmitAnyExpr(UE);
6539 ExprRValue = convertToType(CGF, ExprRValue, E->getType(),
6546 auto Res = CGF.EmitOMPAtomicSimpleUpdateExpr(
6549 CGF.CGM.getOpenMPRuntime().checkAndEmitLastprivateConditional(CGF, X);
6556 CGF.emitOMPSimpleStore(VLValue, NewVVal, NewVValType, Loc);
6557 CGF.CGM.getOpenMPRuntime().checkAndEmitLastprivateConditional(CGF, V);
6559 if (CGF.CGM.getLangOpts().OpenMP < 51) {
6569 CGF.CGM.getOpenMPRuntime().emitFlush(CGF, {}, Loc,
6573 CGF.CGM.getOpenMPRuntime().emitFlush(CGF, {}, Loc,
6578 CGF.CGM.getOpenMPRuntime().emitFlush(
6579 CGF, {}, Loc, llvm::AtomicOrdering::AcquireRelease);
6591 CodeGenFunction &CGF, llvm::AtomicOrdering AO, llvm::AtomicOrdering FailAO,
6596 CGF.CGM.getOpenMPRuntime().getOMPBuilder();
6614 LValue XLVal = CGF.EmitLValue(X);
6617 auto EmitRValueWithCastIfNeeded = [&CGF, Loc](const Expr *X, const Expr *E) {
6619 return CGF.EmitScalarExpr(E);
6621 llvm::Value *V = CGF.EmitScalarExpr(NewE);
6624 return CGF.EmitScalarConversion(V, NewE->getType(), X->getType(), Loc);
6630 EVal = CGF.Builder.CreateIntCast(
6635 DVal = CGF.Builder.CreateIntCast(
6640 XAddr.emitRawPointer(CGF), XAddr.getElementType(),
6645 LValue LV = CGF.EmitLValue(V);
6647 VOpVal = {Addr.emitRawPointer(CGF), Addr.getElementType(),
6652 LValue LV = CGF.EmitLValue(R);
6654 ROpVal = {Addr.emitRawPointer(CGF), Addr.getElementType(),
6662 CGF.Builder.restoreIP(OMPBuilder.createAtomicCompare(
6663 CGF.Builder, XOpVal, VOpVal, ROpVal, EVal, DVal, AO, Op, IsXBinopExpr,
6666 CGF.Builder.restoreIP(OMPBuilder.createAtomicCompare(
6667 CGF.Builder, XOpVal, VOpVal, ROpVal, EVal, DVal, AO, Op, IsXBinopExpr,
6671 static void emitOMPAtomicExpr(CodeGenFunction &CGF, OpenMPClauseKind Kind,
6680 emitOMPAtomicReadExpr(CGF, AO, X, V, Loc);
6683 emitOMPAtomicWriteExpr(CGF, AO, X, E, Loc);
6687 emitOMPAtomicUpdateExpr(CGF, AO, X, E, UE, IsXLHSInRHSPart, Loc);
6690 emitOMPAtomicCaptureExpr(CGF, AO, IsPostfixUpdate, V, X, E, UE,
6694 emitOMPAtomicCompareExpr(CGF, AO, FailAO, X, V, R, E, D, CE,
6786 static void emitCommonOMPTargetDirective(CodeGenFunction &CGF,
6790 CodeGenModule &CGM = CGF.CGM;
6794 OMPLexicalScope Scope(CGF, S, OMPD_target);
6796 CGF, OMPD_target, [&S](CodeGenFunction &CGF, PrePostActionTy &) {
6797 CGF.EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt());
6802 auto LPCRegion = CGOpenMPRuntime::LastprivateConditionalRAII::disable(CGF, S);
6828 if (CGF.ConstantFoldsToSimpleInteger(IfCond, Val) && !Val)
6841 assert(CGF.CurFuncDecl && "No parent declaration for target region!");
6845 if (const auto *D = dyn_cast<CXXConstructorDecl>(CGF.CurFuncDecl))
6847 else if (const auto *D = dyn_cast<CXXDestructorDecl>(CGF.CurFuncDecl))
6851 CGM.getMangledName(GlobalDecl(cast<FunctionDecl>(CGF.CurFuncDecl)));
6856 OMPLexicalScope Scope(CGF, S, OMPD_task);
6858 [IsOffloadEntry](CodeGenFunction &CGF,
6861 OMPLoopScope(CGF, D);
6863 llvm::Value *NumIterations = CGF.EmitScalarExpr(D.getNumIterations());
6864 NumIterations = CGF.Builder.CreateIntCast(NumIterations, CGF.Int64Ty,
6870 CGM.getOpenMPRuntime().emitTargetCall(CGF, S, Fn, FnID, IfCond, Device,
6874 static void emitTargetRegion(CodeGenFunction &CGF, const OMPTargetDirective &S,
6876 Action.Enter(CGF);
6877 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
6878 (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);
6879 CGF.EmitOMPPrivateClause(S, PrivateScope);
6882 CGF.CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(CGF, S);
6884 CGF.EmitStmt(S.getCapturedStmt(OMPD_target)->getCapturedStmt());
6885 CGF.EnsureInsertPoint();
6891 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
6892 emitTargetRegion(CGF, S, Action);
6903 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
6904 emitTargetRegion(CGF, S, Action);
6909 static void emitCommonOMPTeamsDirective(CodeGenFunction &CGF,
6915 CGF.CGM.getOpenMPRuntime().emitTeamsOutlinedFunction(
6916 CGF, S, *CS->getCapturedDecl()->param_begin(), InnermostKind,
6925 CGF.CGM.getOpenMPRuntime().emitNumTeamsClause(CGF, NumTeams, ThreadLimit,
6929 OMPTeamsScope Scope(CGF, S);
6931 CGF.GenerateOpenMPCapturedVars(*CS, CapturedVars);
6932 CGF.CGM.getOpenMPRuntime().emitTeamsCall(CGF, S, S.getBeginLoc(), OutlinedFn,
6938 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
6939 Action.Enter(CGF);
6940 OMPPrivateScope PrivateScope(CGF);
6941 (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);
6942 CGF.EmitOMPPrivateClause(S, PrivateScope);
6943 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
6945 CGF.EmitStmt(S.getCapturedStmt(OMPD_teams)->getCapturedStmt());
6946 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
6953 static void emitTargetTeamsRegion(CodeGenFunction &CGF, PrePostActionTy &Action,
6956 Action.Enter(CGF);
6958 auto &&CodeGen = [&S, CS](CodeGenFunction &CGF, PrePostActionTy &Action) {
6959 Action.Enter(CGF);
6960 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
6961 (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);
6962 CGF.EmitOMPPrivateClause(S, PrivateScope);
6963 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
6966 CGF.CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(CGF, S);
6967 CGF.EmitStmt(CS->getCapturedStmt());
6968 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
6970 emitCommonOMPTeamsDirective(CGF, S, OMPD_teams, CodeGen);
6971 emitPostUpdateForReductionClause(CGF, S,
6978 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
6979 emitTargetTeamsRegion(CGF, Action, S);
6991 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
6992 emitTargetTeamsRegion(CGF, Action, S);
6998 emitTargetTeamsDistributeRegion(CodeGenFunction &CGF, PrePostActionTy &Action,
7000 Action.Enter(CGF);
7001 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
7002 CGF.EmitOMPDistributeLoop(S, emitOMPLoopBodyWithStopPoint, S.getInc());
7006 auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
7008 Action.Enter(CGF);
7009 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
7010 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
7012 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute,
7014 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
7016 emitCommonOMPTeamsDirective(CGF, S, OMPD_distribute, CodeGen);
7017 emitPostUpdateForReductionClause(CGF, S,
7024 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
7025 emitTargetTeamsDistributeRegion(CGF, Action, S);
7037 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
7038 emitTargetTeamsDistributeRegion(CGF, Action, S);
7044 CodeGenFunction &CGF, PrePostActionTy &Action,
7046 Action.Enter(CGF);
7047 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
7048 CGF.EmitOMPDistributeLoop(S, emitOMPLoopBodyWithStopPoint, S.getInc());
7052 auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
7054 Action.Enter(CGF);
7055 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
7056 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
7058 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute,
7060 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
7062 emitCommonOMPTeamsDirective(CGF, S, OMPD_distribute_simd, CodeGen);
7063 emitPostUpdateForReductionClause(CGF, S,
7070 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
7071 emitTargetTeamsDistributeSimdRegion(CGF, Action, S);
7083 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
7084 emitTargetTeamsDistributeSimdRegion(CGF, Action, S);
7092 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
7093 CGF.EmitOMPDistributeLoop(S, emitOMPLoopBodyWithStopPoint, S.getInc());
7097 auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
7099 Action.Enter(CGF);
7100 OMPPrivateScope PrivateScope(CGF);
7101 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
7103 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute,
7105 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
7114 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
7115 CGF.EmitOMPDistributeLoop(S, emitOMPLoopBodyWithStopPoint, S.getInc());
7119 auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
7121 Action.Enter(CGF);
7122 OMPPrivateScope PrivateScope(CGF);
7123 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
7125 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_simd,
7127 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
7136 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
7137 CGF.EmitOMPDistributeLoop(S, emitInnerParallelForWhenCombined,
7142 auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
7144 Action.Enter(CGF);
7145 OMPPrivateScope PrivateScope(CGF);
7146 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
7148 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute,
7150 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
7159 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
7160 CGF.EmitOMPDistributeLoop(S, emitInnerParallelForWhenCombined,
7165 auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
7167 Action.Enter(CGF);
7168 OMPPrivateScope PrivateScope(CGF);
7169 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
7171 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(
7172 CGF, OMPD_distribute, CodeGenDistribute, /*HasCancel=*/false);
7173 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
7252 CodeGenFunction &CGF, const OMPTargetTeamsDistributeParallelForDirective &S,
7254 Action.Enter(CGF);
7255 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
7256 CGF.EmitOMPDistributeLoop(S, emitInnerParallelForWhenCombined,
7261 auto &&CodeGenTeams = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
7263 Action.Enter(CGF);
7264 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
7265 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
7267 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(
7268 CGF, OMPD_distribute, CodeGenDistribute, /*HasCancel=*/false);
7269 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
7272 emitCommonOMPTeamsDirective(CGF, S, OMPD_distribute_parallel_for,
7274 emitPostUpdateForReductionClause(CGF, S,
7283 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
7284 emitTargetTeamsDistributeParallelForRegion(CGF, S, Action);
7296 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
7297 emitTargetTeamsDistributeParallelForRegion(CGF, S, Action);
7303 CodeGenFunction &CGF,
7306 Action.Enter(CGF);
7307 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
7308 CGF.EmitOMPDistributeLoop(S, emitInnerParallelForWhenCombined,
7313 auto &&CodeGenTeams = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
7315 Action.Enter(CGF);
7316 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
7317 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
7319 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(
7320 CGF, OMPD_distribute, CodeGenDistribute, /*HasCancel=*/false);
7321 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
7324 emitCommonOMPTeamsDirective(CGF, S, OMPD_distribute_parallel_for_simd,
7326 emitPostUpdateForReductionClause(CGF, S,
7335 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
7336 emitTargetTeamsDistributeParallelForSimdRegion(CGF, S, Action);
7348 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
7349 emitTargetTeamsDistributeParallelForSimdRegion(CGF, S, Action);
7520 void Enter(CodeGenFunction &CGF) override {
7526 auto &&CodeGen = [&](CodeGenFunction &CGF, PrePostActionTy &Action) {
7527 auto &&InnermostCodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
7528 CGF.EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt());
7532 auto &&PrivCodeGen = [&](CodeGenFunction &CGF, PrePostActionTy &Action) {
7538 Action.Enter(CGF);
7541 OMPPrivateScope PrivateScope(CGF);
7544 CGF.EmitOMPUseDevicePtrClause(*C, PrivateScope,
7547 CGF.EmitOMPUseDeviceAddrClause(*C, PrivateScope,
7550 RCG(CGF);
7561 CGF.EmitVarDecl(*OED);
7567 CGF.EmitVarDecl(*OED);
7573 OMPLexicalScope Scope(CGF, S, CaptureRegion);
7574 RCG(CGF);
7585 OMPLexicalScope Scope(CGF, S);
7586 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_target_data,
7659 static void emitTargetParallelRegion(CodeGenFunction &CGF,
7664 Action.Enter(CGF);
7665 auto &&CodeGen = [&S, CS](CodeGenFunction &CGF, PrePostActionTy &Action) {
7666 Action.Enter(CGF);
7667 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
7668 (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);
7669 CGF.EmitOMPPrivateClause(S, PrivateScope);
7670 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
7673 CGF.CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(CGF, S);
7675 CGF.EmitStmt(CS->getCapturedStmt());
7676 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_parallel);
7678 emitCommonOMPParallelDirective(CGF, S, OMPD_parallel, CodeGen,
7680 emitPostUpdateForReductionClause(CGF, S,
7687 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
7688 emitTargetParallelRegion(CGF, S, Action);
7700 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
7701 emitTargetParallelRegion(CGF, S, Action);
7706 static void emitTargetParallelForRegion(CodeGenFunction &CGF,
7709 Action.Enter(CGF);
7712 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
7713 Action.Enter(CGF);
7715 CGF, OMPD_target_parallel_for, S.hasCancel());
7716 CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(), emitForLoopBounds,
7719 emitCommonOMPParallelDirective(CGF, S, OMPD_for, CodeGen,
7727 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
7728 emitTargetParallelForRegion(CGF, S, Action);
7740 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
7741 emitTargetParallelForRegion(CGF, S, Action);
7747 emitTargetParallelForSimdRegion(CodeGenFunction &CGF,
7750 Action.Enter(CGF);
7753 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
7754 Action.Enter(CGF);
7755 CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(), emitForLoopBounds,
7758 emitCommonOMPParallelDirective(CGF, S, OMPD_simd, CodeGen,
7766 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
7767 emitTargetParallelForSimdRegion(CGF, S, Action);
7779 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
7780 emitTargetParallelForSimdRegion(CGF, S, Action);
7786 static void mapParam(CodeGenFunction &CGF, const DeclRefExpr *Helper,
7790 Privates.addPrivate(VDecl, CGF.GetAddrOfLocalVar(PVD));
7832 auto &&BodyGen = [CS, &S](CodeGenFunction &CGF, PrePostActionTy &) {
7844 OMPLoopScope PreInitScope(CGF, S);
7845 if (CGF.ConstantFoldsToSimpleInteger(S.getPreCond(), CondConstant)) {
7849 llvm::BasicBlock *ThenBlock = CGF.createBasicBlock("taskloop.if.then");
7850 ContBlock = CGF.createBasicBlock("taskloop.if.end");
7851 emitPreCond(CGF, S, S.getPreCond(), ThenBlock, ContBlock,
7852 CGF.getProfileCount(&S));
7853 CGF.EmitBlock(ThenBlock);
7854 CGF.incrementProfileCounter(&S);
7857 (void)CGF.EmitOMPLinearClauseInit(S);
7859 OMPPrivateScope LoopScope(CGF);
7867 mapParam(CGF, cast<DeclRefExpr>(S.getLowerBoundVariable()), *LBP,
7869 mapParam(CGF, cast<DeclRefExpr>(S.getUpperBoundVariable()), *UBP,
7871 mapParam(CGF, cast<DeclRefExpr>(S.getStrideVariable()), *STP, LoopScope);
7872 mapParam(CGF, cast<DeclRefExpr>(S.getIsLastIterVariable()), *LIP,
7874 CGF.EmitOMPPrivateLoopCounters(S, LoopScope);
7875 CGF.EmitOMPLinearClause(S, LoopScope);
7876 bool HasLastprivateClause = CGF.EmitOMPLastprivateClauseInit(S, LoopScope);
7881 CGF.EmitVarDecl(*IVDecl);
7882 CGF.EmitIgnoredExpr(S.getInit());
7888 CGF.EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl()));
7890 CGF.EmitIgnoredExpr(S.getCalcLastIteration());
7894 OMPLexicalScope Scope(CGF, S, OMPD_taskloop, /*EmitPreInitStmt=*/false);
7896 CGF, S,
7897 [&S](CodeGenFunction &CGF, PrePostActionTy &) {
7899 CGF.EmitOMPSimdInit(S);
7901 [&S, &LoopScope](CodeGenFunction &CGF, PrePostActionTy &) {
7902 CGF.EmitOMPInnerLoop(
7904 [&S](CodeGenFunction &CGF) {
7905 emitOMPLoopBodyWithStopPoint(CGF, S,
7913 CGF.EmitBranch(ContBlock);
7914 CGF.EmitBlock(ContBlock, true);
7918 CGF.EmitOMPLastprivateClauseFinal(
7920 CGF.Builder.CreateIsNotNull(CGF.EmitLoadOfScalar(
7921 CGF.GetAddrOfLocalVar(*LIP), /*Volatile=*/false,
7925 CGF.EmitOMPLinearClauseFinal(S, [LIP, &S](CodeGenFunction &CGF) {
7926 return CGF.Builder.CreateIsNotNull(
7927 CGF.EmitLoadOfScalar(CGF.GetAddrOfLocalVar(*LIP), /*Volatile=*/false,
7932 IfCond](CodeGenFunction &CGF, llvm::Function *OutlinedFn,
7935 &Data](CodeGenFunction &CGF, PrePostActionTy &) {
7936 OMPLoopScope PreInitScope(CGF, S);
7937 CGF.CGM.getOpenMPRuntime().emitTaskLoopCall(CGF, S.getBeginLoc(), S,
7941 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_taskloop,
7949 [&S, &BodyGen, &TaskGen, &Data](CodeGenFunction &CGF,
7951 Action.Enter(CGF);
7952 CGF.EmitOMPTaskBasedDirective(S, OMPD_taskloop, BodyGen, TaskGen,
7975 auto &&CodeGen = [this, &S](CodeGenFunction &CGF, PrePostActionTy &Action) {
7976 Action.Enter(CGF);
7987 auto &&CodeGen = [this, &S](CodeGenFunction &CGF, PrePostActionTy &Action) {
7988 Action.Enter(CGF);
7999 auto &&CodeGen = [this, &S](CodeGenFunction &CGF, PrePostActionTy &Action) {
8000 Action.Enter(CGF);
8011 auto &&CodeGen = [this, &S](CodeGenFunction &CGF, PrePostActionTy &Action) {
8012 Action.Enter(CGF);
8023 auto &&CodeGen = [this, &S](CodeGenFunction &CGF, PrePostActionTy &Action) {
8024 auto &&TaskLoopCodeGen = [&S](CodeGenFunction &CGF,
8026 Action.Enter(CGF);
8027 CGF.EmitOMPTaskLoopBasedDirective(S);
8029 OMPLexicalScope Scope(CGF, S, OMPD_parallel, /*EmitPreInitStmt=*/false);
8030 CGM.getOpenMPRuntime().emitMasterRegion(CGF, TaskLoopCodeGen,
8041 auto &&CodeGen = [this, &S](CodeGenFunction &CGF, PrePostActionTy &Action) {
8042 auto &&TaskLoopCodeGen = [&S](CodeGenFunction &CGF,
8044 Action.Enter(CGF);
8045 CGF.EmitOMPTaskLoopBasedDirective(S);
8047 OMPLexicalScope Scope(CGF, S, OMPD_parallel, /*EmitPreInitStmt=*/false);
8048 CGM.getOpenMPRuntime().emitMaskedRegion(CGF, TaskLoopCodeGen,
8059 auto &&CodeGen = [this, &S](CodeGenFunction &CGF, PrePostActionTy &Action) {
8060 auto &&TaskLoopCodeGen = [&S](CodeGenFunction &CGF,
8062 Action.Enter(CGF);
8063 CGF.EmitOMPTaskLoopBasedDirective(S);
8065 OMPLexicalScope Scope(CGF, S, OMPD_parallel, /*EmitPreInitStmt=*/false);
8066 CGM.getOpenMPRuntime().emitMasterRegion(CGF, TaskLoopCodeGen,
8077 auto &&CodeGen = [this, &S](CodeGenFunction &CGF, PrePostActionTy &Action) {
8078 auto &&TaskLoopCodeGen = [&S](CodeGenFunction &CGF,
8080 Action.Enter(CGF);
8081 CGF.EmitOMPTaskLoopBasedDirective(S);
8083 OMPLexicalScope Scope(CGF, S, OMPD_parallel, /*EmitPreInitStmt=*/false);
8084 CGM.getOpenMPRuntime().emitMaskedRegion(CGF, TaskLoopCodeGen,
8136 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
8142 OMPPrivateScope LoopScope(CGF);
8143 CGF.EmitOMPPrivateLoopCounters(S, LoopScope);
8145 CGF.EmitStmt(CS);
8148 CGF.EmitStmt(CS);
8159 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
8160 Action.Enter(CGF);
8161 emitOMPCopyinClause(CGF, S);
8162 (void)emitWorksharingDirective(CGF, S, /*HasCancel=*/false);
8178 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
8179 CGF.EmitOMPDistributeLoop(S, emitOMPLoopBodyWithStopPoint, S.getInc());
8183 auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
8185 Action.Enter(CGF);
8186 OMPPrivateScope PrivateScope(CGF);
8187 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
8189 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute,
8191 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
8199 static void emitTargetTeamsLoopCodegenStatus(CodeGenFunction &CGF,
8202 bool IsDevice = CGF.CGM.getLangOpts().OpenMPIsTargetDevice;
8208 auto &SM = CGF.getContext().getSourceManager();
8218 CodeGenFunction &CGF, PrePostActionTy &Action,
8220 Action.Enter(CGF);
8223 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
8224 CGF.EmitOMPDistributeLoop(S, emitInnerParallelForWhenCombined,
8229 auto &&CodeGenTeams = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
8231 Action.Enter(CGF);
8232 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
8233 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
8235 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(
8236 CGF, OMPD_distribute, CodeGenDistribute, /*HasCancel=*/false);
8237 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
8241 CGF, TTL_CODEGEN_TYPE " as parallel for", S));
8242 emitCommonOMPTeamsDirective(CGF, S, OMPD_distribute_parallel_for,
8244 emitPostUpdateForReductionClause(CGF, S,
8249 CodeGenFunction &CGF, PrePostActionTy &Action,
8251 Action.Enter(CGF);
8253 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
8254 CGF.EmitOMPDistributeLoop(S, emitOMPLoopBodyWithStopPoint, S.getInc());
8258 auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
8260 Action.Enter(CGF);
8261 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
8262 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
8264 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(
8265 CGF, OMPD_distribute, CodeGenDistribute, /*HasCancel=*/false);
8266 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
8270 CGF, TTL_CODEGEN_TYPE " as distribute", S));
8271 emitCommonOMPTeamsDirective(CGF, S, OMPD_distribute, CodeGen);
8272 emitPostUpdateForReductionClause(CGF, S,
8278 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
8280 emitTargetTeamsGenericLoopRegionAsParallel(CGF, Action, S);
8282 emitTargetTeamsGenericLoopRegionAsDistribute(CGF, Action, S);
8291 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
8293 emitTargetTeamsGenericLoopRegionAsParallel(CGF, Action, S);
8295 emitTargetTeamsGenericLoopRegionAsDistribute(CGF, Action, S);
8307 CodeGenFunction &CGF, const OMPTargetParallelGenericLoopDirective &S,
8309 Action.Enter(CGF);
8311 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
8312 Action.Enter(CGF);
8314 CGF, OMPD_target_parallel_loop, /*hasCancel=*/false);
8315 CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(), emitForLoopBounds,
8318 emitCommonOMPParallelDirective(CGF, S, OMPD_for, CodeGen,
8326 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
8327 emitTargetParallelGenericLoopRegion(CGF, S, Action);
8341 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
8342 emitTargetParallelGenericLoopRegion(CGF, S, Action);
8355 auto &&CodeGen = [&D](CodeGenFunction &CGF, PrePostActionTy &Action) {
8356 OMPPrivateScope GlobalsScope(CGF);
8367 if (!CGF.LocalDeclMap.count(VD)) {
8368 LValue GlobLVal = CGF.EmitLValue(Ref);
8376 ParentLoopDirectiveForScanRegion ScanRegion(CGF, D);
8377 emitOMPSimdRegion(CGF, cast<OMPLoopDirective>(D), Action);
8382 if (!VD->hasLocalStorage() && !CGF.LocalDeclMap.count(VD)) {
8383 LValue GlobLVal = CGF.EmitLValue(E);
8388 if (!CGF.LocalDeclMap.count(VD))
8389 CGF.EmitVarDecl(*VD);
8401 if (!CGF.LocalDeclMap.count(VD))
8402 CGF.EmitVarDecl(*VD);
8408 CGF.EmitStmt(D.getInnermostCapturedStmt()->getCapturedStmt());