Lines Matching defs:CGF
52 void emitPreInitStmt(CodeGenFunction &CGF, const OMPExecutableDirective &S) {
59 CGF.EmitVarDecl(cast<VarDecl>(*I));
62 CGF.EmitAutoVarAlloca(cast<VarDecl>(*I));
63 CGF.EmitAutoVarCleanups(Emission);
72 static bool isCapturedVar(CodeGenFunction &CGF, const VarDecl *VD) {
73 return CGF.LambdaCaptureFields.lookup(VD) ||
74 (CGF.CapturedStmtInfo && CGF.CapturedStmtInfo->lookup(VD)) ||
75 (isa_and_nonnull<BlockDecl>(CGF.CurCodeDecl) &&
76 cast<BlockDecl>(CGF.CurCodeDecl)->capturesVariable(VD));
81 CodeGenFunction &CGF, const OMPExecutableDirective &S,
84 : CodeGenFunction::LexicalScope(CGF, S.getSourceRange()),
85 InlinedShareds(CGF) {
87 emitPreInitStmt(CGF, S);
99 CGF.getContext(), const_cast<VarDecl *>(VD),
100 isCapturedVar(CGF, VD) || (CGF.CapturedStmtInfo &&
103 InlinedShareds.addPrivate(VD, CGF.EmitLValue(&DRE).getAddress());
121 OMPParallelScope(CodeGenFunction &CGF, const OMPExecutableDirective &S)
122 : OMPLexicalScope(CGF, S, /*CapturedRegion=*/std::nullopt,
136 OMPTeamsScope(CodeGenFunction &CGF, const OMPExecutableDirective &S)
137 : OMPLexicalScope(CGF, S, /*CapturedRegion=*/std::nullopt,
144 void emitPreInitStmt(CodeGenFunction &CGF, const OMPLoopBasedDirective &S) {
153 CGF, VD, CGF.CreateMemTemp(VD->getType().getNonReferenceType()));
163 CGF, OrigVD,
164 Address(llvm::UndefValue::get(CGF.ConvertTypeForMem(
165 CGF.getContext().getPointerType(OrigVDTy))),
166 CGF.ConvertTypeForMem(OrigVDTy),
167 CGF.getContext().getDeclAlign(OrigVD)));
171 (void)PreCondVars.apply(CGF);
176 [&CGF](unsigned Cnt, const Stmt *CurStmt) {
179 CGF.EmitStmt(Init);
180 CGF.EmitStmt(CXXFor->getRangeStmt());
181 CGF.EmitStmt(CXXFor->getEndStmt());
213 CGF.EmitVarDecl(cast<VarDecl>(*I));
216 CGF.EmitStmt(S);
219 PreCondVars.restore(CGF);
223 OMPLoopScope(CodeGenFunction &CGF, const OMPLoopBasedDirective &S)
224 : CodeGenFunction::RunCleanupsScope(CGF) {
225 emitPreInitStmt(CGF, S);
232 static bool isCapturedVar(CodeGenFunction &CGF, const VarDecl *VD) {
233 return CGF.LambdaCaptureFields.lookup(VD) ||
234 (CGF.CapturedStmtInfo && CGF.CapturedStmtInfo->lookup(VD)) ||
235 (isa_and_nonnull<BlockDecl>(CGF.CurCodeDecl) &&
236 cast<BlockDecl>(CGF.CurCodeDecl)->capturesVariable(VD));
240 OMPSimdLexicalScope(CodeGenFunction &CGF, const OMPExecutableDirective &S)
241 : CodeGenFunction::LexicalScope(CGF, S.getSourceRange()),
242 InlinedShareds(CGF) {
249 CGF.EmitVarDecl(cast<VarDecl>(*I));
252 CGF.EmitAutoVarAlloca(cast<VarDecl>(*I));
253 CGF.EmitAutoVarCleanups(Emission);
261 CGF.EmitVarDecl(*OED);
267 CGF.EmitVarDecl(*OED);
272 CGF.EmitOMPPrivateClause(S, InlinedShareds);
275 CGF.EmitVarDecl(*cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl()));
295 DeclRefExpr DRE(CGF.getContext(), const_cast<VarDecl *>(VD),
296 isCapturedVar(CGF, VD) ||
297 (CGF.CapturedStmtInfo &&
301 InlinedShareds.addPrivate(VD, CGF.EmitLValue(&DRE).getAddress());
312 static void emitCommonOMPTargetDirective(CodeGenFunction &CGF,
399 static Address castValueFromUintptr(CodeGenFunction &CGF, SourceLocation Loc,
402 ASTContext &Ctx = CGF.getContext();
404 llvm::Value *CastedPtr = CGF.EmitScalarConversion(
405 AddrLV.getAddress().emitRawPointer(CGF), Ctx.getUIntPtrType(),
409 CGF.MakeNaturalAlignAddrLValue(CastedPtr, DstType).getAddress();
454 CodeGenFunction &CGF, FunctionArgList &Args,
466 CodeGenModule &CGM = CGF.CGM;
561 CGF.StartFunction(CD, Ctx.VoidTy, F, FuncInfo, TargetArgs,
571 LocalAddr = CGM.getOpenMPRuntime().getParameterAddress(CGF, Args[Cnt],
574 LocalAddr = CGF.GetAddrOfLocalVar(Args[Cnt]);
587 LValue ArgLVal = CGF.MakeAddrLValue(LocalAddr, Args[Cnt]->getType(),
591 ArgLVal = CGF.MakeAddrLValue(
592 castValueFromUintptr(CGF, I->getLocation(), FD->getType(),
596 llvm::Value *ExprArg = CGF.EmitLoadOfScalar(ArgLVal, I->getLocation());
604 ArgAddr = CGF.EmitLoadOfReference(ArgLVal);
607 ArgAddr = CGF.EmitLoadOfPointer(
621 CGF, I->getLocation(), FD->getType(),
627 CXXThisValue = CGF.EmitLoadOfScalar(ArgLVal, I->getLocation());
1263 [&Emission](CodeGenFunction &CGF) {
1264 CGF.EmitAutoVarInit(Emission);
1473 CodeGenFunction &CGF, const OMPExecutableDirective &D,
1475 if (!CGF.HaveInsertPoint())
1481 if (llvm::Value *Cond = CondGen(CGF)) {
1484 llvm::BasicBlock *ThenBB = CGF.createBasicBlock(".omp.reduction.pu");
1485 DoneBB = CGF.createBasicBlock(".omp.reduction.pu.done");
1486 CGF.Builder.CreateCondBr(Cond, ThenBB, DoneBB);
1487 CGF.EmitBlock(ThenBB);
1490 CGF.EmitIgnoredExpr(PostUpdate);
1494 CGF.EmitBlock(DoneBB, /*IsFinished=*/true);
1508 checkForLastprivateConditionalUpdate(CodeGenFunction &CGF,
1510 if (CGF.getLangOpts().OpenMP < 50)
1521 CGF.CGM.getOpenMPRuntime().checkAndEmitLastprivateConditional(CGF, Ref);
1532 CGF.CGM.getOpenMPRuntime().checkAndEmitLastprivateConditional(CGF, Ref);
1543 CGF.CGM.getOpenMPRuntime().checkAndEmitLastprivateConditional(CGF, Ref);
1560 CGF.CGM.getOpenMPRuntime().checkAndEmitSharedLastprivateConditional(
1561 CGF, S, PrivateDecls);
1565 CodeGenFunction &CGF, const OMPExecutableDirective &S,
1571 CGF.CGM.getOpenMPRuntime().emitParallelOutlinedFunction(
1572 CGF, S, *CS->getCapturedDecl()->param_begin(), InnermostKind,
1575 CodeGenFunction::RunCleanupsScope NumThreadsScope(CGF);
1576 NumThreads = CGF.EmitScalarExpr(NumThreadsClause->getNumThreads(),
1578 CGF.CGM.getOpenMPRuntime().emitNumThreadsClause(
1579 CGF, NumThreads, NumThreadsClause->getBeginLoc());
1582 CodeGenFunction::RunCleanupsScope ProcBindScope(CGF);
1583 CGF.CGM.getOpenMPRuntime().emitProcBindClause(
1584 CGF, ProcBindClause->getProcBindKind(), ProcBindClause->getBeginLoc());
1595 OMPParallelScope Scope(CGF, S);
1601 CodeGenBoundParameters(CGF, S, CapturedVars);
1602 CGF.GenerateOpenMPCapturedVars(*CS, CapturedVars);
1603 CGF.CGM.getOpenMPRuntime().emitParallelCall(CGF, S.getBeginLoc(), OutlinedFn,
1622 static void emitOMPCopyinClause(CodeGenFunction &CGF,
1624 bool Copyins = CGF.EmitOMPCopyinClause(S);
1629 CGF.CGM.getOpenMPRuntime().emitBarrierCall(
1630 CGF, S.getBeginLoc(), OMPD_unknown, /*EmitChecks=*/false,
1636 CodeGenFunction &CGF, const VarDecl *VD) {
1637 CodeGenModule &CGM = CGF.CGM;
1648 Size = CGF.getTypeSize(CVD->getType());
1650 Size = CGF.Builder.CreateNUWAdd(
1652 Size = CGF.Builder.CreateUDiv(Size, CGM.getSize(Align));
1653 Size = CGF.Builder.CreateNUWMul(Size, CGM.getSize(Align));
1662 llvm::Value *Allocator = CGF.EmitScalarExpr(AA->getAllocator());
1666 Allocator = CGF.Builder.CreateIntToPtr(Allocator, CGM.VoidPtrTy);
1668 Allocator = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(Allocator,
1672 CGF.Builder, Size, Allocator,
1675 OMPBuilder.createOMPFree(CGF.Builder, Addr, Allocator);
1677 CGF.EHStack.pushCleanup<OMPAllocateCleanupTy>(NormalAndEHCleanup, FreeCI);
1678 Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
1680 CGF.ConvertTypeForMem(CGM.getContext().getPointerType(CVD->getType())),
1682 return Address(Addr, CGF.ConvertTypeForMem(CVD->getType()), Align);
1686 CodeGenFunction &CGF, const VarDecl *VD, Address VDAddr,
1688 CodeGenModule &CGM = CGF.CGM;
1697 CGF.Builder.CreatePointerCast(VDAddr.emitRawPointer(CGF), CGM.Int8PtrTy);
1703 OMPBuilder.createCachedThreadPrivate(CGF.Builder, Data, Size, CacheName);
1721 CodeGenFunction &CGF, const Stmt *RegionBodyStmt, InsertPointTy AllocaIP,
1723 CGBuilderTy &Builder = CGF.Builder;
1729 OMPBuilderCBHelpers::InlinedRegionBodyRAII IRB(CGF, AllocaIP, *FiniBB);
1730 CGF.EmitStmt(RegionBodyStmt);
1738 CodeGenFunction &CGF, const Stmt *RegionBodyStmt, InsertPointTy AllocaIP,
1740 CGBuilderTy &Builder = CGF.Builder;
1746 OMPBuilderCBHelpers::OutlinedRegionBodyRAII IRB(CGF, AllocaIP, *FiniBB);
1747 CGF.EmitStmt(RegionBodyStmt);
1813 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
1814 Action.Enter(CGF);
1815 OMPPrivateScope PrivateScope(CGF);
1816 emitOMPCopyinClause(CGF, S);
1817 (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);
1818 CGF.EmitOMPPrivateClause(S, PrivateScope);
1819 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
1821 CGF.EmitStmt(S.getCapturedStmt(OMPD_parallel)->getCapturedStmt());
1822 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_parallel);
1853 OMPTransformDirectiveScopeRAII(CodeGenFunction &CGF, const Stmt *S) {
1855 Scope = new OMPLoopScope(CGF, *Dir);
1857 CapInfoRAII = new CodeGenFunction::CGCapturedStmtRAII(CGF, CGSI);
1870 static void emitBody(CodeGenFunction &CGF, const Stmt *S, const Stmt *NextLoop,
1876 CGF.getContext().getSourceManager(), CS->getLBracLoc(),
1880 CodeGenFunction::LexicalScope Scope(CGF, S->getSourceRange());
1882 emitBody(CGF, CurStmt, NextLoop, MaxLevel, Level);
1896 CGF.EmitStmt(CXXFor->getLoopVarStmt());
1902 emitBody(CGF, S, NextLoop, MaxLevel, Level + 1);
1906 CGF.EmitStmt(S);
1984 CodeGenFunction CGF(ParentCGF.CGM, /*suppressNewContext=*/true);
1987 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, CSI.get());
1988 llvm::Function *F = CGF.GenerateCapturedStmtFunction(*S);
2245 static void emitAlignedClause(CodeGenFunction &CGF,
2247 if (!CGF.HaveInsertPoint())
2253 cast<llvm::ConstantInt>(CGF.EmitScalarExpr(AlignmentExpr));
2263 CGF.getContext()
2264 .toCharUnitsFromBits(CGF.getContext().getOpenMPDefaultSimdAlign(
2271 llvm::Value *PtrValue = CGF.EmitScalarExpr(E);
2272 CGF.emitAlignmentAssumption(
2274 llvm::ConstantInt::get(CGF.getLLVMContext(), Alignment));
2322 static void emitPreCond(CodeGenFunction &CGF, const OMPLoopDirective &S,
2325 if (!CGF.HaveInsertPoint())
2328 CodeGenFunction::OMPPrivateScope PreCondScope(CGF);
2329 CGF.EmitOMPPrivateLoopCounters(S, PreCondScope);
2333 CGF.EmitIgnoredExpr(I);
2346 CGF.CreateMemTemp(VD->getType().getNonReferenceType());
2347 (void)PreCondVars.setVarAddr(CGF, VD, CounterAddr);
2349 (void)PreCondVars.apply(CGF);
2353 CGF.EmitIgnoredExpr(E);
2356 CGF.EmitBranchOnBoolExpr(Cond, TrueBlock, FalseBlock, TrueCount);
2357 PreCondVars.restore(CGF);
2394 static void emitSimdlenSafelenClause(CodeGenFunction &CGF,
2396 if (!CGF.HaveInsertPoint())
2399 RValue Len = CGF.EmitAnyExpr(C->getSimdlen(), AggValueSlot::ignored(),
2402 CGF.LoopStack.setVectorizeWidth(Val->getZExtValue());
2406 CGF.LoopStack.setParallel(!D.getSingleClause<OMPSafelenClause>());
2408 RValue Len = CGF.EmitAnyExpr(C->getSafelen(), AggValueSlot::ignored(),
2411 CGF.LoopStack.setVectorizeWidth(Val->getZExtValue());
2415 CGF.LoopStack.setParallel(/*Enable=*/false);
2483 static void emitOMPLoopBodyWithStopPoint(CodeGenFunction &CGF,
2486 CGF.EmitOMPLoopBody(S, LoopExit);
2487 CGF.EmitStopPoint(&S);
2491 static LValue EmitOMPHelperVar(CodeGenFunction &CGF,
2494 CGF.EmitVarDecl(*VDecl);
2495 return CGF.EmitLValue(Helper);
2498 static void emitCommonSimdLoop(CodeGenFunction &CGF, const OMPLoopDirective &S,
2501 auto &&ThenGen = [&S, &SimdInitGen, &BodyCodeGen](CodeGenFunction &CGF,
2503 CGOpenMPRuntime::NontemporalDeclsRAII NontemporalsRegion(CGF.CGM, S);
2504 CodeGenFunction::OMPLocalDeclMapRAII Scope(CGF);
2505 SimdInitGen(CGF);
2507 BodyCodeGen(CGF);
2509 auto &&ElseGen = [&BodyCodeGen](CodeGenFunction &CGF, PrePostActionTy &) {
2510 CodeGenFunction::OMPLocalDeclMapRAII Scope(CGF);
2511 CGF.LoopStack.setVectorizeEnable(/*Enable=*/false);
2513 BodyCodeGen(CGF);
2518 if (CGF.getLangOpts().OpenMP >= 50 &&
2527 CGF.CGM.getOpenMPRuntime().emitIfClause(CGF, IfCond, ThenGen, ElseGen);
2530 ThenRCG(CGF);
2534 static void emitOMPSimdRegion(CodeGenFunction &CGF, const OMPLoopDirective &S,
2536 Action.Enter(CGF);
2539 OMPLoopScope PreInitScope(CGF, S);
2548 (void)EmitOMPHelperVar(CGF, cast<DeclRefExpr>(S.getLowerBoundVariable()));
2549 (void)EmitOMPHelperVar(CGF, cast<DeclRefExpr>(S.getUpperBoundVariable()));
2557 if (CGF.ConstantFoldsToSimpleInteger(S.getPreCond(), CondConstant)) {
2561 llvm::BasicBlock *ThenBlock = CGF.createBasicBlock("simd.if.then");
2562 ContBlock = CGF.createBasicBlock("simd.if.end");
2563 emitPreCond(CGF, S, S.getPreCond(), ThenBlock, ContBlock,
2564 CGF.getProfileCount(&S));
2565 CGF.EmitBlock(ThenBlock);
2566 CGF.incrementProfileCounter(&S);
2572 CGF.EmitVarDecl(*IVDecl);
2573 CGF.EmitIgnoredExpr(S.getInit());
2579 CGF.EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl()));
2581 CGF.EmitIgnoredExpr(S.getCalcLastIteration());
2584 emitAlignedClause(CGF, S);
2585 (void)CGF.EmitOMPLinearClauseInit(S);
2587 CodeGenFunction::OMPPrivateScope LoopScope(CGF);
2588 CGF.EmitOMPPrivateClause(S, LoopScope);
2589 CGF.EmitOMPPrivateLoopCounters(S, LoopScope);
2590 CGF.EmitOMPLinearClause(S, LoopScope);
2591 CGF.EmitOMPReductionClauseInit(S, LoopScope);
2593 CGF, S, CGF.EmitLValue(S.getIterationVariable()));
2594 bool HasLastprivateClause = CGF.EmitOMPLastprivateClauseInit(S, LoopScope);
2597 CGF.CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(CGF, S);
2600 CGF, S,
2601 [&S](CodeGenFunction &CGF, PrePostActionTy &) {
2602 CGF.EmitOMPSimdInit(S);
2604 [&S, &LoopScope](CodeGenFunction &CGF, PrePostActionTy &) {
2605 CGF.EmitOMPInnerLoop(
2607 [&S](CodeGenFunction &CGF) {
2608 emitOMPLoopBodyWithStopPoint(CGF, S,
2613 CGF.EmitOMPSimdFinal(S, [](CodeGenFunction &) { return nullptr; });
2616 CGF.EmitOMPLastprivateClauseFinal(S, /*NoFinals=*/true);
2617 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_simd);
2618 emitPostUpdateForReductionClause(CGF, S,
2621 CGF.EmitOMPLinearClauseFinal(S, [](CodeGenFunction &) { return nullptr; });
2625 CGF.EmitBranch(ContBlock);
2626 CGF.EmitBlock(ContBlock, true);
2662 GetAlignedMapping(const OMPSimdDirective &S, CodeGenFunction &CGF) {
2668 cast<llvm::ConstantInt>(CGF.EmitScalarExpr(AlignmentExpr));
2678 CGF.getContext()
2679 .toCharUnitsFromBits(CGF.getContext().getOpenMPDefaultSimdAlign(
2685 llvm::Value *PtrValue = CGF.EmitScalarExpr(E);
2686 AlignedVars[PtrValue] = CGF.Builder.getInt64(Alignment.getSExtValue());
2696 auto &&CodeGenIRBuilder = [this, &S, UseOMPIRBuilder](CodeGenFunction &CGF,
2701 GetAlignedMapping(S, CGF);
2751 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
2752 emitOMPSimdRegion(CGF, S, Action);
2908 [&S, IsMonotonic](CodeGenFunction &CGF, PrePostActionTy &) {
2912 CGF.LoopStack.setParallel(!IsMonotonic);
2915 CGF.LoopStack.setParallel(/*Enable=*/true);
2917 CGF.EmitOMPSimdInit(S);
2921 &LoopScope](CodeGenFunction &CGF, PrePostActionTy &) {
2928 CGF.EmitOMPInnerLoop(
2930 [&S, LoopExit, &CodeGenLoop](CodeGenFunction &CGF) {
2931 CodeGenLoop(CGF, S, LoopExit);
2933 [IVSize, IVSigned, Loc, &CodeGenOrdered](CodeGenFunction &CGF) {
2934 CodeGenOrdered(CGF, Loc, IVSize, IVSigned);
2953 auto &&CodeGen = [DynamicOrOrdered, &S, &LoopArgs](CodeGenFunction &CGF) {
2955 CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getEndLoc(),
3048 auto &&CodeGenOrdered = [Ordered](CodeGenFunction &CGF, SourceLocation Loc,
3052 CGF.CGM.getOpenMPRuntime().emitForOrderedIterationEnd(CGF, Loc, IVSize,
3137 emitDistributeParallelForInnerBounds(CodeGenFunction &CGF,
3141 EmitOMPHelperVar(CGF, cast<DeclRefExpr>(LS.getLowerBoundVariable()));
3143 EmitOMPHelperVar(CGF, cast<DeclRefExpr>(LS.getUpperBoundVariable()));
3151 LValue PrevLB = CGF.EmitLValue(LS.getPrevLowerBoundVariable());
3152 LValue PrevUB = CGF.EmitLValue(LS.getPrevUpperBoundVariable());
3153 llvm::Value *PrevLBVal = CGF.EmitLoadOfScalar(
3155 PrevLBVal = CGF.EmitScalarConversion(
3159 llvm::Value *PrevUBVal = CGF.EmitLoadOfScalar(
3161 PrevUBVal = CGF.EmitScalarConversion(
3166 CGF.EmitStoreOfScalar(PrevLBVal, LB);
3167 CGF.EmitStoreOfScalar(PrevUBVal, UB);
3180 emitDistributeParallelForDispatchBounds(CodeGenFunction &CGF,
3191 CGF.EmitLoadOfScalar(LB, /*Volatile=*/false, IteratorTy, S.getBeginLoc());
3193 CGF.EmitLoadOfScalar(UB, /*Volatile=*/false, IteratorTy, S.getBeginLoc());
3198 CodeGenFunction &CGF, const OMPExecutableDirective &S,
3202 CGF.EmitLValue(cast<DeclRefExpr>(Dir.getCombinedLowerBoundVariable()));
3203 llvm::Value *LBCast = CGF.Builder.CreateIntCast(
3204 CGF.Builder.CreateLoad(LB.getAddress()), CGF.SizeTy, /*isSigned=*/false);
3207 CGF.EmitLValue(cast<DeclRefExpr>(Dir.getCombinedUpperBoundVariable()));
3209 llvm::Value *UBCast = CGF.Builder.CreateIntCast(
3210 CGF.Builder.CreateLoad(UB.getAddress()), CGF.SizeTy, /*isSigned=*/false);
3215 emitInnerParallelForWhenCombined(CodeGenFunction &CGF,
3218 auto &&CGInlinedWorksharingLoop = [&S](CodeGenFunction &CGF,
3220 Action.Enter(CGF);
3231 CodeGenFunction::OMPCancelStackRAII CancelRegion(CGF, S.getDirectiveKind(),
3233 CGF.EmitOMPWorksharingLoop(S, S.getPrevEnsureUpperBound(),
3239 CGF, S,
3247 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
3248 CGF.EmitOMPDistributeLoop(S, emitInnerParallelForWhenCombined,
3257 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
3258 CGF.EmitOMPDistributeLoop(S, emitInnerParallelForWhenCombined,
3267 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
3268 CGF.EmitOMPDistributeLoop(S, emitOMPLoopBodyWithStopPoint, S.getInc());
3277 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
3278 emitOMPSimdRegion(CGF, S, Action);
3290 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
3291 emitOMPSimdRegion(CGF, S, Action);
3445 [&S](CodeGenFunction &CGF, PrePostActionTy &) {
3447 CGF.EmitOMPSimdInit(S);
3450 CGF.LoopStack.setParallel(/*Enable=*/true);
3455 &LoopScope](CodeGenFunction &CGF, PrePostActionTy &) {
3465 CGF.CGM.getOpenMPRuntime().emitForStaticInit(
3466 CGF, S.getBeginLoc(), S.getDirectiveKind(), ScheduleKind,
3470 CGF.EmitIgnoredExpr(S.getEnsureUpperBound());
3472 CGF.EmitIgnoredExpr(S.getInit());
3486 CGF.EmitOMPInnerLoop(
3491 [&S, LoopExit](CodeGenFunction &CGF) {
3492 emitOMPLoopBodyWithStopPoint(CGF, S, LoopExit);
3498 auto &&CodeGen = [&S](CodeGenFunction &CGF) {
3499 CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getEndLoc(),
3514 EmitOMPSimdFinal(S, [IL, &S](CodeGenFunction &CGF) {
3515 return CGF.Builder.CreateIsNotNull(
3516 CGF.EmitLoadOfScalar(IL, S.getBeginLoc()));
3525 *this, S, [IL, &S](CodeGenFunction &CGF) {
3526 return CGF.Builder.CreateIsNotNull(
3527 CGF.EmitLoadOfScalar(IL, S.getBeginLoc()));
3535 EmitOMPLinearClauseFinal(S, [IL, &S](CodeGenFunction &CGF) {
3536 return CGF.Builder.CreateIsNotNull(
3537 CGF.EmitLoadOfScalar(IL, S.getBeginLoc()));
3554 emitForLoopBounds(CodeGenFunction &CGF, const OMPExecutableDirective &S) {
3557 EmitOMPHelperVar(CGF, cast<DeclRefExpr>(LS.getLowerBoundVariable()));
3559 EmitOMPHelperVar(CGF, cast<DeclRefExpr>(LS.getUpperBoundVariable()));
3568 emitDispatchForLoopBounds(CodeGenFunction &CGF, const OMPExecutableDirective &S,
3572 const unsigned IVSize = CGF.getContext().getTypeSize(IVExpr->getType());
3573 llvm::Value *LBVal = CGF.Builder.getIntN(IVSize, 0);
3574 llvm::Value *UBVal = CGF.EmitScalarExpr(LS.getLastIteration());
3586 CodeGenFunction &CGF, const OMPLoopDirective &S,
3588 llvm::Value *OMPScanNumIterations = CGF.Builder.CreateIntCast(
3589 NumIteratorsGen(CGF), CGF.SizeTy, /*isSigned=*/false);
3615 RedCG.emitSharedOrigLValue(CGF, Count);
3616 RedCG.emitAggregateType(CGF, Count);
3619 CGF,
3625 CGF.EmitVarDecl(*cast<VarDecl>(cast<DeclRefExpr>(*ITA)->getDecl()));
3638 CodeGenFunction &CGF, const OMPLoopDirective &S,
3640 llvm::Value *OMPScanNumIterations = CGF.Builder.CreateIntCast(
3641 NumIteratorsGen(CGF), CGF.SizeTy, /*isSigned=*/false);
3661 llvm::Value *OMPLast = CGF.Builder.CreateNSWSub(
3663 llvm::ConstantInt::get(CGF.SizeTy, 1, /*isSigned=*/false));
3669 CGF,
3673 LValue DestLVal = CGF.EmitLValue(OrigExpr);
3674 LValue SrcLVal = CGF.EmitLValue(CopyArrayElem);
3675 CGF.EmitOMPCopy(
3702 CodeGenFunction &CGF, const OMPLoopDirective &S,
3706 llvm::Value *OMPScanNumIterations = CGF.Builder.CreateIntCast(
3707 NumIteratorsGen(CGF), CGF.SizeTy, /*isSigned=*/false);
3723 CodeGenFunction::ParentLoopDirectiveForScanRegion ScanRegion(CGF, S);
3731 CGF.OMPFirstScanLoop = true;
3732 CodeGenFunction::OMPLocalDeclMapRAII Scope(CGF);
3733 FirstGen(CGF);
3738 &Privates](CodeGenFunction &CGF, PrePostActionTy &Action) {
3739 Action.Enter(CGF);
3743 llvm::BasicBlock *InputBB = CGF.Builder.GetInsertBlock();
3744 llvm::BasicBlock *LoopBB = CGF.createBasicBlock("omp.outer.log.scan.body");
3745 llvm::BasicBlock *ExitBB = CGF.createBasicBlock("omp.outer.log.scan.exit");
3747 CGF.CGM.getIntrinsic(llvm::Intrinsic::log2, CGF.DoubleTy);
3749 CGF.Builder.CreateUIToFP(OMPScanNumIterations, CGF.DoubleTy);
3750 llvm::Value *LogVal = CGF.EmitNounwindRuntimeCall(F, Arg);
3751 F = CGF.CGM.getIntrinsic(llvm::Intrinsic::ceil, CGF.DoubleTy);
3752 LogVal = CGF.EmitNounwindRuntimeCall(F, LogVal);
3753 LogVal = CGF.Builder.CreateFPToUI(LogVal, CGF.IntTy);
3754 llvm::Value *NMin1 = CGF.Builder.CreateNUWSub(
3755 OMPScanNumIterations, llvm::ConstantInt::get(CGF.SizeTy, 1));
3756 auto DL = ApplyDebugLocation::CreateDefaultArtificial(CGF, S.getBeginLoc());
3757 CGF.EmitBlock(LoopBB);
3758 auto *Counter = CGF.Builder.CreatePHI(CGF.IntTy, 2);
3760 auto *Pow2K = CGF.Builder.CreatePHI(CGF.SizeTy, 2);
3761 Counter->addIncoming(llvm::ConstantInt::get(CGF.IntTy, 0), InputBB);
3762 Pow2K->addIncoming(llvm::ConstantInt::get(CGF.SizeTy, 1), InputBB);
3766 CGF.createBasicBlock("omp.inner.log.scan.body");
3768 CGF.createBasicBlock("omp.inner.log.scan.exit");
3769 llvm::Value *CmpI = CGF.Builder.CreateICmpUGE(NMin1, Pow2K);
3770 CGF.Builder.CreateCondBr(CmpI, InnerLoopBB, InnerExitBB);
3771 CGF.EmitBlock(InnerLoopBB);
3772 auto *IVal = CGF.Builder.CreatePHI(CGF.SizeTy, 2);
3775 CodeGenFunction::OMPPrivateScope PrivScope(CGF);
3784 CGF,
3788 LHSAddr = CGF.EmitLValue(CopyArrayElem).getAddress();
3793 llvm::Value *OffsetIVal = CGF.Builder.CreateNUWSub(IVal, Pow2K);
3795 CGF,
3799 RHSAddr = CGF.EmitLValue(CopyArrayElem).getAddress();
3806 CGF.CGM.getOpenMPRuntime().emitReduction(
3807 CGF, S.getEndLoc(), Privates, LHSs, RHSs, ReductionOps,
3811 CGF.Builder.CreateNUWSub(IVal, llvm::ConstantInt::get(CGF.SizeTy, 1));
3812 IVal->addIncoming(NextIVal, CGF.Builder.GetInsertBlock());
3813 CmpI = CGF.Builder.CreateICmpUGE(NextIVal, Pow2K);
3814 CGF.Builder.CreateCondBr(CmpI, InnerLoopBB, InnerExitBB);
3815 CGF.EmitBlock(InnerExitBB);
3817 CGF.Builder.CreateNUWAdd(Counter, llvm::ConstantInt::get(CGF.IntTy, 1));
3818 Counter->addIncoming(Next, CGF.Builder.GetInsertBlock());
3821 CGF.Builder.CreateShl(Pow2K, 1, "", /*HasNUW=*/true);
3822 Pow2K->addIncoming(NextPow2K, CGF.Builder.GetInsertBlock());
3823 llvm::Value *Cmp = CGF.Builder.CreateICmpNE(Next, LogVal);
3824 CGF.Builder.CreateCondBr(Cmp, LoopBB, ExitBB);
3825 auto DL1 = ApplyDebugLocation::CreateDefaultArtificial(CGF, S.getEndLoc());
3826 CGF.EmitBlock(ExitBB);
3829 CGF.CGM.getOpenMPRuntime().emitMasterRegion(CGF, CodeGen, S.getBeginLoc());
3830 CGF.CGM.getOpenMPRuntime().emitBarrierCall(
3831 CGF, S.getBeginLoc(), OMPD_unknown, /*EmitChecks=*/false,
3835 RCG(CGF);
3838 CGF.OMPFirstScanLoop = false;
3839 SecondGen(CGF);
3842 static bool emitWorksharingDirective(CodeGenFunction &CGF,
3850 const auto &&NumIteratorsGen = [&S](CodeGenFunction &CGF) {
3851 CodeGenFunction::OMPLocalDeclMapRAII Scope(CGF);
3852 OMPLoopScope LoopScope(CGF, S);
3853 return CGF.EmitScalarExpr(S.getNumIterations());
3855 const auto &&FirstGen = [&S, HasCancel](CodeGenFunction &CGF) {
3857 CGF, S.getDirectiveKind(), HasCancel);
3858 (void)CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(),
3862 CGF.CGM.getOpenMPRuntime().emitBarrierCall(CGF, S.getBeginLoc(),
3866 &HasLastprivates](CodeGenFunction &CGF) {
3868 CGF, S.getDirectiveKind(), HasCancel);
3869 HasLastprivates = CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(),
3874 emitScanBasedDirectiveDecls(CGF, S, NumIteratorsGen);
3875 emitScanBasedDirective(CGF, S, NumIteratorsGen, FirstGen, SecondGen);
3877 emitScanBasedDirectiveFinals(CGF, S, NumIteratorsGen);
3879 CodeGenFunction::OMPCancelStackRAII CancelRegion(CGF, S.getDirectiveKind(),
3881 HasLastprivates = CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(),
3942 UseOMPIRBuilder](CodeGenFunction &CGF, PrePostActionTy &) {
3973 HasLastprivates = emitWorksharingDirective(CGF, S, S.hasCancel());
3994 auto &&CodeGen = [&S, &HasLastprivates](CodeGenFunction &CGF,
3996 HasLastprivates = emitWorksharingDirective(CGF, S, /*HasCancel=*/false);
4012 static LValue createSectionLVal(CodeGenFunction &CGF, QualType Ty,
4015 LValue LVal = CGF.MakeAddrLValue(CGF.CreateMemTemp(Ty, Name), Ty);
4017 CGF.EmitStoreThroughLValue(RValue::get(Init), LVal, /*isInit*/ true);
4026 &HasLastprivates](CodeGenFunction &CGF, PrePostActionTy &) {
4027 const ASTContext &C = CGF.getContext();
4031 LValue LB = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.lb.",
4032 CGF.Builder.getInt32(0));
4034 ? CGF.Builder.getInt32(CS->size() - 1)
4035 : CGF.Builder.getInt32(0);
4037 createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.ub.", GlobalUBVal);
4038 LValue ST = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.st.",
4039 CGF.Builder.getInt32(1));
4040 LValue IL = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.il.",
4041 CGF.Builder.getInt32(0));
4043 LValue IV = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.iv.");
4045 CodeGenFunction::OpaqueValueMapping OpaqueIV(CGF, &IVRefExpr, IV);
4047 CodeGenFunction::OpaqueValueMapping OpaqueUB(CGF, &UBRefExpr, UB);
4056 auto &&BodyGen = [CapturedStmt, CS, &S, &IV](CodeGenFunction &CGF) {
4068 llvm::BasicBlock *ExitBB = CGF.createBasicBlock(".omp.sections.exit");
4070 CGF.Builder.CreateSwitch(CGF.EmitLoadOfScalar(IV, S.getBeginLoc()),
4075 auto CaseBB = CGF.createBasicBlock(".omp.sections.case");
4076 CGF.EmitBlock(CaseBB);
4077 SwitchStmt->addCase(CGF.Builder.getInt32(CaseNumber), CaseBB);
4078 CGF.EmitStmt(SubStmt);
4079 CGF.EmitBranch(ExitBB);
4083 llvm::BasicBlock *CaseBB = CGF.createBasicBlock(".omp.sections.case");
4084 CGF.EmitBlock(CaseBB);
4085 SwitchStmt->addCase(CGF.Builder.getInt32(0), CaseBB);
4086 CGF.EmitStmt(CapturedStmt);
4087 CGF.EmitBranch(ExitBB);
4089 CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
4092 CodeGenFunction::OMPPrivateScope LoopScope(CGF);
4093 if (CGF.EmitOMPFirstprivateClause(S, LoopScope)) {
4097 CGF.CGM.getOpenMPRuntime().emitBarrierCall(
4098 CGF, S.getBeginLoc(), OMPD_unknown, /*EmitChecks=*/false,
4101 CGF.EmitOMPPrivateClause(S, LoopScope);
4102 CGOpenMPRuntime::LastprivateConditionalRAII LPCRegion(CGF, S, IV);
4103 HasLastprivates = CGF.EmitOMPLastprivateClauseInit(S, LoopScope);
4104 CGF.EmitOMPReductionClauseInit(S, LoopScope);
4107 CGF.CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(CGF, S);
4115 CGF.CGM.getOpenMPRuntime().emitForStaticInit(
4116 CGF, S.getBeginLoc(), S.getDirectiveKind(), ScheduleKind, StaticInit);
4118 llvm::Value *UBVal = CGF.EmitLoadOfScalar(UB, S.getBeginLoc());
4119 llvm::Value *MinUBGlobalUB = CGF.Builder.CreateSelect(
4120 CGF.Builder.CreateICmpSLT(UBVal, GlobalUBVal), UBVal, GlobalUBVal);
4121 CGF.EmitStoreOfScalar(MinUBGlobalUB, UB);
4123 CGF.EmitStoreOfScalar(CGF.EmitLoadOfScalar(LB, S.getBeginLoc()), IV);
4125 CGF.EmitOMPInnerLoop(S, /*RequiresCleanup=*/false, Cond, Inc, BodyGen,
4128 auto &&CodeGen = [&S](CodeGenFunction &CGF) {
4129 CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getEndLoc(),
4132 CGF.OMPCancelStack.emitExit(CGF, S.getDirectiveKind(), CodeGen);
4133 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_parallel);
4135 emitPostUpdateForReductionClause(CGF, S, [IL, &S](CodeGenFunction &CGF) {
4136 return CGF.Builder.CreateIsNotNull(
4137 CGF.EmitLoadOfScalar(IL, S.getBeginLoc()));
4142 CGF.EmitOMPLastprivateClauseFinal(
4144 CGF.Builder.CreateIsNotNull(
4145 CGF.EmitLoadOfScalar(IL, S.getBeginLoc())));
4281 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4282 Action.Enter(CGF);
4283 OMPPrivateScope SingleScope(CGF);
4284 (void)CGF.EmitOMPFirstprivateClause(S, SingleScope);
4285 CGF.EmitOMPPrivateClause(S, SingleScope);
4287 CGF.EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt());
4308 static void emitMaster(CodeGenFunction &CGF, const OMPExecutableDirective &S) {
4309 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4310 Action.Enter(CGF);
4311 CGF.EmitStmt(S.getRawStmt());
4313 CGF.CGM.getOpenMPRuntime().emitMasterRegion(CGF, CodeGen, S.getBeginLoc());
4344 static void emitMasked(CodeGenFunction &CGF, const OMPExecutableDirective &S) {
4345 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4346 Action.Enter(CGF);
4347 CGF.EmitStmt(S.getRawStmt());
4352 CGF.CGM.getOpenMPRuntime().emitMaskedRegion(CGF, CodeGen, S.getBeginLoc(),
4428 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4429 Action.Enter(CGF);
4430 CGF.EmitStmt(S.getAssociatedStmt());
4446 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4447 Action.Enter(CGF);
4448 emitOMPCopyinClause(CGF, S);
4449 (void)emitWorksharingDirective(CGF, S, S.hasCancel());
4452 const auto &&NumIteratorsGen = [&S](CodeGenFunction &CGF) {
4453 CodeGenFunction::OMPLocalDeclMapRAII Scope(CGF);
4455 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGSI);
4456 OMPLoopScope LoopScope(CGF, S);
4457 return CGF.EmitScalarExpr(S.getNumIterations());
4480 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4481 Action.Enter(CGF);
4482 emitOMPCopyinClause(CGF, S);
4483 (void)emitWorksharingDirective(CGF, S, /*HasCancel=*/false);
4486 const auto &&NumIteratorsGen = [&S](CodeGenFunction &CGF) {
4487 CodeGenFunction::OMPLocalDeclMapRAII Scope(CGF);
4489 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGSI);
4490 OMPLoopScope LoopScope(CGF, S);
4491 return CGF.EmitScalarExpr(S.getNumIterations());
4514 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4515 Action.Enter(CGF);
4516 OMPPrivateScope PrivateScope(CGF);
4517 emitOMPCopyinClause(CGF, S);
4518 (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);
4519 CGF.EmitOMPPrivateClause(S, PrivateScope);
4520 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4522 emitMaster(CGF, S);
4523 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_parallel);
4541 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4542 Action.Enter(CGF);
4543 OMPPrivateScope PrivateScope(CGF);
4544 emitOMPCopyinClause(CGF, S);
4545 (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);
4546 CGF.EmitOMPPrivateClause(S, PrivateScope);
4547 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4549 emitMasked(CGF, S);
4550 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_parallel);
4568 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4569 Action.Enter(CGF);
4570 emitOMPCopyinClause(CGF, S);
4571 CGF.EmitSections(S);
4757 CapturedRegion](CodeGenFunction &CGF,
4763 OMPPrivateScope Scope(CGF);
4765 if (auto *DI = CGF.getDebugInfo()) {
4767 CGF.CapturedStmtInfo->getCaptureFields();
4768 llvm::Value *ContextValue = CGF.CapturedStmtInfo->getContextValue();
4770 unsigned CharWidth = CGF.getContext().getCharWidth();
4788 CGF.getContext().getASTRecordLayout(CaptureRecord);
4791 if (CGF.CGM.getCodeGenOpts().hasReducedDebugInfo())
4793 CGF.Builder, false);
4807 llvm::Instruction &Last = CGF.Builder.GetInsertBlock()->back();
4814 CGF.Builder.GetInsertBlock()->getTrailingDbgRecords()) {
4828 llvm::Value *CopyFn = CGF.Builder.CreateLoad(
4829 CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(CopyFnParam)));
4830 llvm::Value *PrivatesPtr = CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(
4840 RawAddress PrivatePtr = CGF.CreateMemTemp(
4841 CGF.getContext().getPointerType(E->getType()), ".priv.ptr.addr");
4849 CGF.CreateMemTemp(CGF.getContext().getPointerType(E->getType()),
4859 CGF.CreateMemTemp(CGF.getContext().getPointerType(E->getType()),
4868 Ty = CGF.getContext().getPointerType(Ty);
4870 Ty = CGF.getContext().getPointerType(Ty);
4871 RawAddress PrivatePtr = CGF.CreateMemTemp(
4872 CGF.getContext().getPointerType(Ty), ".local.ptr.addr");
4882 auto *CopyFnTy = llvm::FunctionType::get(CGF.Builder.getVoidTy(),
4884 CGF.CGM.getOpenMPRuntime().emitOutlinedFunctionCall(
4885 CGF, S.getBeginLoc(), {CopyFnTy, CopyFn}, CallArgs);
4888 DeclRefExpr DRE(CGF.getContext(), const_cast<VarDecl *>(OrigVD),
4890 CGF.CapturedStmtInfo->lookup(OrigVD) != nullptr,
4893 Scope.addPrivate(Pair.first, CGF.EmitLValue(&DRE).getAddress());
4897 CGF.Builder.CreateLoad(Pair.second),
4898 CGF.ConvertTypeForMem(Pair.first->getType().getNonReferenceType()),
4899 CGF.getContext().getDeclAlign(Pair.first));
4901 if (auto *DI = CGF.getDebugInfo())
4902 if (CGF.CGM.getCodeGenOpts().hasReducedDebugInfo())
4904 Pair.first, Pair.second.getBasePointer(), CGF.Builder,
4912 VDType = CGF.getContext().getPointerType(VDType);
4914 llvm::Value *Ptr = CGF.Builder.CreateLoad(Pair.second.first);
4917 CGF.ConvertTypeForMem(CGF.getContext().getPointerType(VDType)),
4918 CGF.getPointerAlign());
4920 Ptr = CGF.Builder.CreateLoad(Replacement);
4921 Replacement = Address(Ptr, CGF.ConvertTypeForMem(VDType),
4922 CGF.getContext().getDeclAlign(Pair.first));
4925 llvm::Value *Ptr = CGF.Builder.CreateLoad(Pair.second.first);
4926 Address Replacement(Ptr, CGF.ConvertTypeForMem(VDType),
4927 CGF.getContext().getDeclAlign(Pair.first));
4933 OMPPrivateScope FirstprivateScope(CGF);
4936 CGF.Builder.CreateLoad(Pair.second),
4937 CGF.ConvertTypeForMem(Pair.first->getType().getNonReferenceType()),
4938 CGF.getContext().getDeclAlign(Pair.first));
4942 OMPLexicalScope LexScope(CGF, S, CapturedRegion);
4945 llvm::Value *ReductionsPtr = CGF.Builder.CreateLoad(
4946 CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(9)));
4948 RedCG.emitSharedOrigLValue(CGF, Cnt);
4949 RedCG.emitAggregateType(CGF, Cnt);
4953 CGF.CGM.getOpenMPRuntime().emitTaskReductionFixups(CGF, S.getBeginLoc(),
4955 Address Replacement = CGF.CGM.getOpenMPRuntime().getTaskReductionItem(
4956 CGF, S.getBeginLoc(), ReductionsPtr, RedCG.getSharedLValue(Cnt));
4958 CGF.EmitScalarConversion(Replacement.emitRawPointer(CGF),
4959 CGF.getContext().VoidPtrTy,
4960 CGF.getContext().getPointerType(
4963 CGF.ConvertTypeForMem(Data.ReductionCopies[Cnt]->getType()),
4965 Replacement = RedCG.adjustPrivateAddress(CGF, Cnt, Replacement);
4991 OMPPrivateScope InRedScope(CGF);
4995 RedCG.emitSharedOrigLValue(CGF, Cnt);
4996 RedCG.emitAggregateType(CGF, Cnt);
5002 CGF.CGM.getOpenMPRuntime().emitTaskReductionFixups(CGF, S.getBeginLoc(),
5006 ReductionsPtr = CGF.EmitLoadOfScalar(CGF.EmitLValue(TRExpr),
5009 ReductionsPtr = llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
5011 Address Replacement = CGF.CGM.getOpenMPRuntime().getTaskReductionItem(
5012 CGF, S.getBeginLoc(), ReductionsPtr, RedCG.getSharedLValue(Cnt));
5014 CGF.EmitScalarConversion(
5015 Replacement.emitRawPointer(CGF), CGF.getContext().VoidPtrTy,
5016 CGF.getContext().getPointerType(InRedPrivs[Cnt]->getType()),
5018 CGF.ConvertTypeForMem(InRedPrivs[Cnt]->getType()),
5020 Replacement = RedCG.adjustPrivateAddress(CGF, Cnt, Replacement);
5026 CGOpenMPRuntime::UntiedTaskLocalDeclsRAII LocalVarsScope(CGF,
5028 Action.Enter(CGF);
5029 BodyGen(CGF);
5143 &InputInfo](CodeGenFunction &CGF, PrePostActionTy &Action) {
5145 OMPPrivateScope Scope(CGF);
5148 llvm::Value *CopyFn = CGF.Builder.CreateLoad(
5149 CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(CopyFnParam)));
5150 llvm::Value *PrivatesPtr = CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(
5161 CGF.CreateMemTemp(CGF.getContext().getPointerType(E->getType()),
5167 auto *CopyFnTy = llvm::FunctionType::get(CGF.Builder.getVoidTy(),
5169 CGF.CGM.getOpenMPRuntime().emitOutlinedFunctionCall(
5170 CGF, S.getBeginLoc(), {CopyFnTy, CopyFn}, CallArgs);
5173 CGF.Builder.CreateLoad(Pair.second),
5174 CGF.ConvertTypeForMem(Pair.first->getType().getNonReferenceType()),
5175 CGF.getContext().getDeclAlign(Pair.first));
5179 CGF.processInReduction(S, Data, CGF, CS, Scope);
5181 InputInfo.BasePointersArray = CGF.Builder.CreateConstArrayGEP(
5182 CGF.GetAddrOfLocalVar(BPVD), /*Index=*/0);
5183 InputInfo.PointersArray = CGF.Builder.CreateConstArrayGEP(
5184 CGF.GetAddrOfLocalVar(PVD), /*Index=*/0);
5185 InputInfo.SizesArray = CGF.Builder.CreateConstArrayGEP(
5186 CGF.GetAddrOfLocalVar(SVD), /*Index=*/0);
5189 InputInfo.MappersArray = CGF.Builder.CreateConstArrayGEP(
5190 CGF.GetAddrOfLocalVar(MVD), /*Index=*/0);
5193 Action.Enter(CGF);
5194 OMPLexicalScope LexScope(CGF, S, OMPD_task, /*EmitPreInitStmt=*/false);
5196 if (CGF.CGM.getLangOpts().OpenMP >= 51 &&
5201 CGF.CGM.getOpenMPRuntime().emitThreadLimitClause(
5202 CGF, TL->getThreadLimit(), S.getBeginLoc());
5204 BodyGen(CGF);
5219 CodeGenFunction &CGF,
5224 OMPLexicalScope LexScope(CGF, S, CapturedRegion);
5227 llvm::Value *ReductionsPtr = CGF.Builder.CreateLoad(
5228 CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(4)));
5230 RedCG.emitSharedOrigLValue(CGF, Cnt);
5231 RedCG.emitAggregateType(CGF, Cnt);
5235 CGF.CGM.getOpenMPRuntime().emitTaskReductionFixups(CGF, S.getBeginLoc(),
5237 Address Replacement = CGF.CGM.getOpenMPRuntime().getTaskReductionItem(
5238 CGF, S.getBeginLoc(), ReductionsPtr, RedCG.getSharedLValue(Cnt));
5240 CGF.EmitScalarConversion(Replacement.emitRawPointer(CGF),
5241 CGF.getContext().VoidPtrTy,
5242 CGF.getContext().getPointerType(
5245 CGF.ConvertTypeForMem(Data.ReductionCopies[Cnt]->getType()),
5247 Replacement = RedCG.adjustPrivateAddress(CGF, Cnt, Replacement);
5270 OMPPrivateScope InRedScope(CGF);
5274 RedCG.emitSharedOrigLValue(CGF, Cnt);
5275 RedCG.emitAggregateType(CGF, Cnt);
5279 CGF.CGM.getOpenMPRuntime().emitTaskReductionFixups(CGF, S.getBeginLoc(),
5284 CGF.EmitLoadOfScalar(CGF.EmitLValue(TRExpr), TRExpr->getExprLoc());
5286 ReductionsPtr = llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
5288 Address Replacement = CGF.CGM.getOpenMPRuntime().getTaskReductionItem(
5289 CGF, S.getBeginLoc(), ReductionsPtr, RedCG.getSharedLValue(Cnt));
5291 CGF.EmitScalarConversion(
5292 Replacement.emitRawPointer(CGF), CGF.getContext().VoidPtrTy,
5293 CGF.getContext().getPointerType(InRedPrivs[Cnt]->getType()),
5295 CGF.ConvertTypeForMem(InRedPrivs[Cnt]->getType()),
5297 Replacement = RedCG.adjustPrivateAddress(CGF, Cnt, Replacement);
5321 auto &&BodyGen = [CS](CodeGenFunction &CGF, PrePostActionTy &) {
5322 CGF.EmitStmt(CS->getCapturedStmt());
5325 IfCond](CodeGenFunction &CGF, llvm::Function *OutlinedFn,
5327 CGF.CGM.getOpenMPRuntime().emitTaskCall(CGF, S.getBeginLoc(), S, OutlinedFn,
5387 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
5388 Action.Enter(CGF);
5403 CGF.CGM.getOpenMPRuntime().emitTaskReductionInit(CGF, S.getBeginLoc(),
5406 CGF.EmitVarDecl(*VD);
5407 CGF.EmitStoreOfScalar(ReductionDesc, CGF.GetAddrOfLocalVar(VD),
5410 CGF.EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt());
5817 [&S](CodeGenFunction &CGF, PrePostActionTy &) {
5819 CGF.EmitOMPSimdInit(S);
5822 StaticChunked](CodeGenFunction &CGF, PrePostActionTy &) {
5823 CGF.EmitOMPInnerLoop(
5825 [&S, LoopExit, &CodeGenLoop](CodeGenFunction &CGF) {
5826 CodeGenLoop(CGF, S, LoopExit);
5828 [&S, StaticChunked](CodeGenFunction &CGF) {
5830 CGF.EmitIgnoredExpr(S.getCombinedNextLowerBound());
5831 CGF.EmitIgnoredExpr(S.getCombinedNextUpperBound());
5832 CGF.EmitIgnoredExpr(S.getCombinedEnsureUpperBound());
5833 CGF.EmitIgnoredExpr(S.getCombinedInit());
5850 EmitOMPSimdFinal(S, [IL, &S](CodeGenFunction &CGF) {
5851 return CGF.Builder.CreateIsNotNull(
5852 CGF.EmitLoadOfScalar(IL, S.getBeginLoc()));
5861 *this, S, [IL, &S](CodeGenFunction &CGF) {
5862 return CGF.Builder.CreateIsNotNull(
5863 CGF.EmitLoadOfScalar(IL, S.getBeginLoc()));
5884 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
5885 CGF.EmitOMPDistributeLoop(S, emitOMPLoopBodyWithStopPoint, S.getInc());
5894 CodeGenFunction CGF(CGM, /*suppressNewContext=*/true);
5896 CGF.CapturedStmtInfo = &CapStmtInfo;
5897 llvm::Function *Fn = CGF.GenerateOpenMPCapturedStmtFunction(*S, Loc);
5903 static void emitRestoreIP(CodeGenFunction &CGF, const T *C,
5908 QualType Int64Ty = CGF.CGM.getContext().getIntTypeForBitwidth(
5914 llvm::Value *StoreValue = CGF.EmitScalarConversion(
5915 CGF.EmitScalarExpr(CounterVal), CounterVal->getType(), Int64Ty,
5921 CGF.Builder.restoreIP(
5922 OMPBuilder.createOrderedDepend(CGF.Builder, AllocaIP, NumLoops,
5996 auto &&CodeGen = [&S, C, this](CodeGenFunction &CGF,
6001 CGF.GenerateOpenMPCapturedVars(*CS, CapturedVars);
6004 CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF, S.getBeginLoc(),
6007 Action.Enter(CGF);
6008 CGF.EmitStmt(CS->getCapturedStmt());
6015 static llvm::Value *convertToScalarValue(CodeGenFunction &CGF, RValue Val,
6018 assert(CGF.hasScalarEvaluationKind(DestType) &&
6021 return Val.isScalar() ? CGF.EmitScalarConversion(Val.getScalarVal(), SrcType,
6023 : CGF.EmitComplexToScalarConversion(
6028 convertToComplexValue(CodeGenFunction &CGF, RValue Val, QualType SrcType,
6030 assert(CGF.getEvaluationKind(DestType) == TEK_Complex &&
6037 llvm::Value *ScalarVal = CGF.EmitScalarConversion(
6046 ComplexVal.first = CGF.EmitScalarConversion(
6048 ComplexVal.second = CGF.EmitScalarConversion(
6054 static void emitSimpleAtomicStore(CodeGenFunction &CGF, llvm::AtomicOrdering AO,
6057 CGF.EmitStoreThroughGlobalRegLValue(RVal, LVal);
6059 CGF.EmitAtomicStore(RVal, LVal, AO, LVal.isVolatile(), /*isInit=*/false);
6062 static RValue emitSimpleAtomicLoad(CodeGenFunction &CGF,
6066 return CGF.EmitLoadOfLValue(LVal, Loc);
6067 return CGF.EmitAtomicLoad(
6090 static void emitOMPAtomicReadExpr(CodeGenFunction &CGF, llvm::AtomicOrdering AO,
6096 LValue XLValue = CGF.EmitLValue(X);
6097 LValue VLValue = CGF.EmitLValue(V);
6098 RValue Res = emitSimpleAtomicLoad(CGF, AO, XLValue, Loc);
6107 CGF.CGM.getOpenMPRuntime().emitFlush(CGF, std::nullopt, Loc,
6117 CGF.emitOMPSimpleStore(VLValue, Res, X->getType().getNonReferenceType(), Loc);
6118 CGF.CGM.getOpenMPRuntime().checkAndEmitLastprivateConditional(CGF, V);
6121 static void emitOMPAtomicWriteExpr(CodeGenFunction &CGF,
6126 emitSimpleAtomicStore(CGF, AO, CGF.EmitLValue(X), CGF.EmitAnyExpr(E));
6127 CGF.CGM.getOpenMPRuntime().checkAndEmitLastprivateConditional(CGF, X);
6136 CGF.CGM.getOpenMPRuntime().emitFlush(CGF, std::nullopt, Loc,
6148 static std::pair<bool, RValue> emitOMPAtomicRMW(CodeGenFunction &CGF, LValue X,
6153 ASTContext &Context = CGF.getContext();
6164 auto &&CheckAtomicSupport = [&CGF](llvm::Type *T, BinaryOperatorKind BO) {
6169 return llvm::isPowerOf2_64(CGF.CGM.getDataLayout().getTypeStoreSize(T));
6254 UpdateVal = CGF.Builder.CreateIntCast(
6258 UpdateVal = CGF.Builder.CreateCast(llvm::Instruction::CastOps::UIToFP, IC,
6262 CGF.Builder.CreateAtomicRMW(RMWOp, X.getAddress(), UpdateVal, AO);
6290 static void emitOMPAtomicUpdateExpr(CodeGenFunction &CGF,
6304 LValue XLValue = CGF.EmitLValue(X);
6305 RValue ExprRValue = CGF.EmitAnyExpr(E);
6310 auto &&Gen = [&CGF, UE, ExprRValue, XRValExpr, ERValExpr](RValue XRValue) {
6311 CodeGenFunction::OpaqueValueMapping MapExpr(CGF, ERValExpr, ExprRValue);
6312 CodeGenFunction::OpaqueValueMapping MapX(CGF, XRValExpr, XRValue);
6313 return CGF.EmitAnyExpr(UE);
6315 (void)CGF.EmitOMPAtomicSimpleUpdateExpr(
6317 CGF.CGM.getOpenMPRuntime().checkAndEmitLastprivateConditional(CGF, X);
6326 CGF.CGM.getOpenMPRuntime().emitFlush(CGF, std::nullopt, Loc,
6338 static RValue convertToType(CodeGenFunction &CGF, RValue Value,
6341 switch (CGF.getEvaluationKind(ResType)) {
6344 convertToScalarValue(CGF, Value, SourceType, ResType, Loc));
6346 auto Res = convertToComplexValue(CGF, Value, SourceType, ResType, Loc);
6355 static void emitOMPAtomicCaptureExpr(CodeGenFunction &CGF,
6364 LValue VLValue = CGF.EmitLValue(V);
6365 LValue XLValue = CGF.EmitLValue(X);
6366 RValue ExprRValue = CGF.EmitAnyExpr(E);
6384 auto &&Gen = [&CGF, &NewVVal, UE, ExprRValue, XRValExpr, ERValExpr,
6386 CodeGenFunction::OpaqueValueMapping MapExpr(CGF, ERValExpr, ExprRValue);
6387 CodeGenFunction::OpaqueValueMapping MapX(CGF, XRValExpr, XRValue);
6388 RValue Res = CGF.EmitAnyExpr(UE);
6392 auto Res = CGF.EmitOMPAtomicSimpleUpdateExpr(
6394 CGF.CGM.getOpenMPRuntime().checkAndEmitLastprivateConditional(CGF, X);
6403 CodeGenFunction::OpaqueValueMapping MapExpr(CGF, ERValExpr, ExprRValue);
6404 CodeGenFunction::OpaqueValueMapping MapX(CGF, XRValExpr, Res.second);
6405 NewVVal = CGF.EmitAnyExpr(UE);
6411 ExprRValue = convertToType(CGF, ExprRValue, E->getType(),
6418 auto Res = CGF.EmitOMPAtomicSimpleUpdateExpr(
6421 CGF.CGM.getOpenMPRuntime().checkAndEmitLastprivateConditional(CGF, X);
6428 CGF.emitOMPSimpleStore(VLValue, NewVVal, NewVValType, Loc);
6429 CGF.CGM.getOpenMPRuntime().checkAndEmitLastprivateConditional(CGF, V);
6431 if (CGF.CGM.getLangOpts().OpenMP < 51) {
6441 CGF.CGM.getOpenMPRuntime().emitFlush(CGF, std::nullopt, Loc,
6445 CGF.CGM.getOpenMPRuntime().emitFlush(CGF, std::nullopt, Loc,
6450 CGF.CGM.getOpenMPRuntime().emitFlush(
6451 CGF, std::nullopt, Loc, llvm::AtomicOrdering::AcquireRelease);
6463 CodeGenFunction &CGF, llvm::AtomicOrdering AO, llvm::AtomicOrdering FailAO,
6468 CGF.CGM.getOpenMPRuntime().getOMPBuilder();
6486 LValue XLVal = CGF.EmitLValue(X);
6489 auto EmitRValueWithCastIfNeeded = [&CGF, Loc](const Expr *X, const Expr *E) {
6491 return CGF.EmitScalarExpr(E);
6493 llvm::Value *V = CGF.EmitScalarExpr(NewE);
6496 return CGF.EmitScalarConversion(V, NewE->getType(), X->getType(), Loc);
6502 EVal = CGF.Builder.CreateIntCast(
6507 DVal = CGF.Builder.CreateIntCast(
6512 XAddr.emitRawPointer(CGF), XAddr.getElementType(),
6517 LValue LV = CGF.EmitLValue(V);
6519 VOpVal = {Addr.emitRawPointer(CGF), Addr.getElementType(),
6524 LValue LV = CGF.EmitLValue(R);
6526 ROpVal = {Addr.emitRawPointer(CGF), Addr.getElementType(),
6534 CGF.Builder.restoreIP(OMPBuilder.createAtomicCompare(
6535 CGF.Builder, XOpVal, VOpVal, ROpVal, EVal, DVal, AO, Op, IsXBinopExpr,
6538 CGF.Builder.restoreIP(OMPBuilder.createAtomicCompare(
6539 CGF.Builder, XOpVal, VOpVal, ROpVal, EVal, DVal, AO, Op, IsXBinopExpr,
6543 static void emitOMPAtomicExpr(CodeGenFunction &CGF, OpenMPClauseKind Kind,
6552 emitOMPAtomicReadExpr(CGF, AO, X, V, Loc);
6555 emitOMPAtomicWriteExpr(CGF, AO, X, E, Loc);
6559 emitOMPAtomicUpdateExpr(CGF, AO, X, E, UE, IsXLHSInRHSPart, Loc);
6562 emitOMPAtomicCaptureExpr(CGF, AO, IsPostfixUpdate, V, X, E, UE,
6566 emitOMPAtomicCompareExpr(CGF, AO, FailAO, X, V, R, E, D, CE,
6658 static void emitCommonOMPTargetDirective(CodeGenFunction &CGF,
6662 CodeGenModule &CGM = CGF.CGM;
6666 OMPLexicalScope Scope(CGF, S, OMPD_target);
6668 CGF, OMPD_target, [&S](CodeGenFunction &CGF, PrePostActionTy &) {
6669 CGF.EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt());
6674 auto LPCRegion = CGOpenMPRuntime::LastprivateConditionalRAII::disable(CGF, S);
6700 if (CGF.ConstantFoldsToSimpleInteger(IfCond, Val) && !Val)
6713 assert(CGF.CurFuncDecl && "No parent declaration for target region!");
6717 if (const auto *D = dyn_cast<CXXConstructorDecl>(CGF.CurFuncDecl))
6719 else if (const auto *D = dyn_cast<CXXDestructorDecl>(CGF.CurFuncDecl))
6723 CGM.getMangledName(GlobalDecl(cast<FunctionDecl>(CGF.CurFuncDecl)));
6728 OMPLexicalScope Scope(CGF, S, OMPD_task);
6730 [IsOffloadEntry](CodeGenFunction &CGF,
6733 OMPLoopScope(CGF, D);
6735 llvm::Value *NumIterations = CGF.EmitScalarExpr(D.getNumIterations());
6736 NumIterations = CGF.Builder.CreateIntCast(NumIterations, CGF.Int64Ty,
6742 CGM.getOpenMPRuntime().emitTargetCall(CGF, S, Fn, FnID, IfCond, Device,
6746 static void emitTargetRegion(CodeGenFunction &CGF, const OMPTargetDirective &S,
6748 Action.Enter(CGF);
6749 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
6750 (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);
6751 CGF.EmitOMPPrivateClause(S, PrivateScope);
6754 CGF.CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(CGF, S);
6756 CGF.EmitStmt(S.getCapturedStmt(OMPD_target)->getCapturedStmt());
6757 CGF.EnsureInsertPoint();
6763 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
6764 emitTargetRegion(CGF, S, Action);
6775 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
6776 emitTargetRegion(CGF, S, Action);
6781 static void emitCommonOMPTeamsDirective(CodeGenFunction &CGF,
6787 CGF.CGM.getOpenMPRuntime().emitTeamsOutlinedFunction(
6788 CGF, S, *CS->getCapturedDecl()->param_begin(), InnermostKind,
6797 CGF.CGM.getOpenMPRuntime().emitNumTeamsClause(CGF, NumTeams, ThreadLimit,
6801 OMPTeamsScope Scope(CGF, S);
6803 CGF.GenerateOpenMPCapturedVars(*CS, CapturedVars);
6804 CGF.CGM.getOpenMPRuntime().emitTeamsCall(CGF, S, S.getBeginLoc(), OutlinedFn,
6810 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
6811 Action.Enter(CGF);
6812 OMPPrivateScope PrivateScope(CGF);
6813 (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);
6814 CGF.EmitOMPPrivateClause(S, PrivateScope);
6815 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
6817 CGF.EmitStmt(S.getCapturedStmt(OMPD_teams)->getCapturedStmt());
6818 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
6825 static void emitTargetTeamsRegion(CodeGenFunction &CGF, PrePostActionTy &Action,
6828 Action.Enter(CGF);
6830 auto &&CodeGen = [&S, CS](CodeGenFunction &CGF, PrePostActionTy &Action) {
6831 Action.Enter(CGF);
6832 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
6833 (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);
6834 CGF.EmitOMPPrivateClause(S, PrivateScope);
6835 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
6838 CGF.CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(CGF, S);
6839 CGF.EmitStmt(CS->getCapturedStmt());
6840 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
6842 emitCommonOMPTeamsDirective(CGF, S, OMPD_teams, CodeGen);
6843 emitPostUpdateForReductionClause(CGF, S,
6850 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
6851 emitTargetTeamsRegion(CGF, Action, S);
6863 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
6864 emitTargetTeamsRegion(CGF, Action, S);
6870 emitTargetTeamsDistributeRegion(CodeGenFunction &CGF, PrePostActionTy &Action,
6872 Action.Enter(CGF);
6873 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
6874 CGF.EmitOMPDistributeLoop(S, emitOMPLoopBodyWithStopPoint, S.getInc());
6878 auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
6880 Action.Enter(CGF);
6881 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
6882 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
6884 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute,
6886 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
6888 emitCommonOMPTeamsDirective(CGF, S, OMPD_distribute, CodeGen);
6889 emitPostUpdateForReductionClause(CGF, S,
6896 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
6897 emitTargetTeamsDistributeRegion(CGF, Action, S);
6909 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
6910 emitTargetTeamsDistributeRegion(CGF, Action, S);
6916 CodeGenFunction &CGF, PrePostActionTy &Action,
6918 Action.Enter(CGF);
6919 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
6920 CGF.EmitOMPDistributeLoop(S, emitOMPLoopBodyWithStopPoint, S.getInc());
6924 auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
6926 Action.Enter(CGF);
6927 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
6928 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
6930 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute,
6932 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
6934 emitCommonOMPTeamsDirective(CGF, S, OMPD_distribute_simd, CodeGen);
6935 emitPostUpdateForReductionClause(CGF, S,
6942 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
6943 emitTargetTeamsDistributeSimdRegion(CGF, Action, S);
6955 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
6956 emitTargetTeamsDistributeSimdRegion(CGF, Action, S);
6964 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
6965 CGF.EmitOMPDistributeLoop(S, emitOMPLoopBodyWithStopPoint, S.getInc());
6969 auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
6971 Action.Enter(CGF);
6972 OMPPrivateScope PrivateScope(CGF);
6973 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
6975 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute,
6977 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
6986 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
6987 CGF.EmitOMPDistributeLoop(S, emitOMPLoopBodyWithStopPoint, S.getInc());
6991 auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
6993 Action.Enter(CGF);
6994 OMPPrivateScope PrivateScope(CGF);
6995 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
6997 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_simd,
6999 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
7008 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
7009 CGF.EmitOMPDistributeLoop(S, emitInnerParallelForWhenCombined,
7014 auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
7016 Action.Enter(CGF);
7017 OMPPrivateScope PrivateScope(CGF);
7018 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
7020 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute,
7022 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
7031 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
7032 CGF.EmitOMPDistributeLoop(S, emitInnerParallelForWhenCombined,
7037 auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
7039 Action.Enter(CGF);
7040 OMPPrivateScope PrivateScope(CGF);
7041 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
7043 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(
7044 CGF, OMPD_distribute, CodeGenDistribute, /*HasCancel=*/false);
7045 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
7124 CodeGenFunction &CGF, const OMPTargetTeamsDistributeParallelForDirective &S,
7126 Action.Enter(CGF);
7127 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
7128 CGF.EmitOMPDistributeLoop(S, emitInnerParallelForWhenCombined,
7133 auto &&CodeGenTeams = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
7135 Action.Enter(CGF);
7136 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
7137 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
7139 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(
7140 CGF, OMPD_distribute, CodeGenDistribute, /*HasCancel=*/false);
7141 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
7144 emitCommonOMPTeamsDirective(CGF, S, OMPD_distribute_parallel_for,
7146 emitPostUpdateForReductionClause(CGF, S,
7155 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
7156 emitTargetTeamsDistributeParallelForRegion(CGF, S, Action);
7168 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
7169 emitTargetTeamsDistributeParallelForRegion(CGF, S, Action);
7175 CodeGenFunction &CGF,
7178 Action.Enter(CGF);
7179 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
7180 CGF.EmitOMPDistributeLoop(S, emitInnerParallelForWhenCombined,
7185 auto &&CodeGenTeams = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
7187 Action.Enter(CGF);
7188 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
7189 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
7191 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(
7192 CGF, OMPD_distribute, CodeGenDistribute, /*HasCancel=*/false);
7193 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
7196 emitCommonOMPTeamsDirective(CGF, S, OMPD_distribute_parallel_for_simd,
7198 emitPostUpdateForReductionClause(CGF, S,
7207 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
7208 emitTargetTeamsDistributeParallelForSimdRegion(CGF, S, Action);
7220 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
7221 emitTargetTeamsDistributeParallelForSimdRegion(CGF, S, Action);
7391 void Enter(CodeGenFunction &CGF) override {
7397 auto &&CodeGen = [&](CodeGenFunction &CGF, PrePostActionTy &Action) {
7398 auto &&InnermostCodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
7399 CGF.EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt());
7403 auto &&PrivCodeGen = [&](CodeGenFunction &CGF, PrePostActionTy &Action) {
7409 Action.Enter(CGF);
7412 OMPPrivateScope PrivateScope(CGF);
7415 CGF.EmitOMPUseDevicePtrClause(*C, PrivateScope,
7418 CGF.EmitOMPUseDeviceAddrClause(*C, PrivateScope,
7421 RCG(CGF);
7432 CGF.EmitVarDecl(*OED);
7438 CGF.EmitVarDecl(*OED);
7444 OMPLexicalScope Scope(CGF, S, CaptureRegion);
7445 RCG(CGF);
7456 OMPLexicalScope Scope(CGF, S);
7457 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_target_data,
7530 static void emitTargetParallelRegion(CodeGenFunction &CGF,
7535 Action.Enter(CGF);
7536 auto &&CodeGen = [&S, CS](CodeGenFunction &CGF, PrePostActionTy &Action) {
7537 Action.Enter(CGF);
7538 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
7539 (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);
7540 CGF.EmitOMPPrivateClause(S, PrivateScope);
7541 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
7544 CGF.CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(CGF, S);
7546 CGF.EmitStmt(CS->getCapturedStmt());
7547 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_parallel);
7549 emitCommonOMPParallelDirective(CGF, S, OMPD_parallel, CodeGen,
7551 emitPostUpdateForReductionClause(CGF, S,
7558 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
7559 emitTargetParallelRegion(CGF, S, Action);
7571 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
7572 emitTargetParallelRegion(CGF, S, Action);
7577 static void emitTargetParallelForRegion(CodeGenFunction &CGF,
7580 Action.Enter(CGF);
7583 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
7584 Action.Enter(CGF);
7586 CGF, OMPD_target_parallel_for, S.hasCancel());
7587 CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(), emitForLoopBounds,
7590 emitCommonOMPParallelDirective(CGF, S, OMPD_for, CodeGen,
7598 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
7599 emitTargetParallelForRegion(CGF, S, Action);
7611 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
7612 emitTargetParallelForRegion(CGF, S, Action);
7618 emitTargetParallelForSimdRegion(CodeGenFunction &CGF,
7621 Action.Enter(CGF);
7624 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
7625 Action.Enter(CGF);
7626 CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(), emitForLoopBounds,
7629 emitCommonOMPParallelDirective(CGF, S, OMPD_simd, CodeGen,
7637 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
7638 emitTargetParallelForSimdRegion(CGF, S, Action);
7650 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
7651 emitTargetParallelForSimdRegion(CGF, S, Action);
7657 static void mapParam(CodeGenFunction &CGF, const DeclRefExpr *Helper,
7661 Privates.addPrivate(VDecl, CGF.GetAddrOfLocalVar(PVD));
7699 auto &&BodyGen = [CS, &S](CodeGenFunction &CGF, PrePostActionTy &) {
7711 OMPLoopScope PreInitScope(CGF, S);
7712 if (CGF.ConstantFoldsToSimpleInteger(S.getPreCond(), CondConstant)) {
7716 llvm::BasicBlock *ThenBlock = CGF.createBasicBlock("taskloop.if.then");
7717 ContBlock = CGF.createBasicBlock("taskloop.if.end");
7718 emitPreCond(CGF, S, S.getPreCond(), ThenBlock, ContBlock,
7719 CGF.getProfileCount(&S));
7720 CGF.EmitBlock(ThenBlock);
7721 CGF.incrementProfileCounter(&S);
7724 (void)CGF.EmitOMPLinearClauseInit(S);
7726 OMPPrivateScope LoopScope(CGF);
7734 mapParam(CGF, cast<DeclRefExpr>(S.getLowerBoundVariable()), *LBP,
7736 mapParam(CGF, cast<DeclRefExpr>(S.getUpperBoundVariable()), *UBP,
7738 mapParam(CGF, cast<DeclRefExpr>(S.getStrideVariable()), *STP, LoopScope);
7739 mapParam(CGF, cast<DeclRefExpr>(S.getIsLastIterVariable()), *LIP,
7741 CGF.EmitOMPPrivateLoopCounters(S, LoopScope);
7742 CGF.EmitOMPLinearClause(S, LoopScope);
7743 bool HasLastprivateClause = CGF.EmitOMPLastprivateClauseInit(S, LoopScope);
7748 CGF.EmitVarDecl(*IVDecl);
7749 CGF.EmitIgnoredExpr(S.getInit());
7755 CGF.EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl()));
7757 CGF.EmitIgnoredExpr(S.getCalcLastIteration());
7761 OMPLexicalScope Scope(CGF, S, OMPD_taskloop, /*EmitPreInitStmt=*/false);
7763 CGF, S,
7764 [&S](CodeGenFunction &CGF, PrePostActionTy &) {
7766 CGF.EmitOMPSimdInit(S);
7768 [&S, &LoopScope](CodeGenFunction &CGF, PrePostActionTy &) {
7769 CGF.EmitOMPInnerLoop(
7771 [&S](CodeGenFunction &CGF) {
7772 emitOMPLoopBodyWithStopPoint(CGF, S,
7780 CGF.EmitBranch(ContBlock);
7781 CGF.EmitBlock(ContBlock, true);
7785 CGF.EmitOMPLastprivateClauseFinal(
7787 CGF.Builder.CreateIsNotNull(CGF.EmitLoadOfScalar(
7788 CGF.GetAddrOfLocalVar(*LIP), /*Volatile=*/false,
7792 CGF.EmitOMPLinearClauseFinal(S, [LIP, &S](CodeGenFunction &CGF) {
7793 return CGF.Builder.CreateIsNotNull(
7794 CGF.EmitLoadOfScalar(CGF.GetAddrOfLocalVar(*LIP), /*Volatile=*/false,
7799 IfCond](CodeGenFunction &CGF, llvm::Function *OutlinedFn,
7802 &Data](CodeGenFunction &CGF, PrePostActionTy &) {
7803 OMPLoopScope PreInitScope(CGF, S);
7804 CGF.CGM.getOpenMPRuntime().emitTaskLoopCall(CGF, S.getBeginLoc(), S,
7808 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_taskloop,
7816 [&S, &BodyGen, &TaskGen, &Data](CodeGenFunction &CGF,
7818 Action.Enter(CGF);
7819 CGF.EmitOMPTaskBasedDirective(S, OMPD_taskloop, BodyGen, TaskGen,
7842 auto &&CodeGen = [this, &S](CodeGenFunction &CGF, PrePostActionTy &Action) {
7843 Action.Enter(CGF);
7854 auto &&CodeGen = [this, &S](CodeGenFunction &CGF, PrePostActionTy &Action) {
7855 Action.Enter(CGF);
7866 auto &&CodeGen = [this, &S](CodeGenFunction &CGF, PrePostActionTy &Action) {
7867 auto &&TaskLoopCodeGen = [&S](CodeGenFunction &CGF,
7869 Action.Enter(CGF);
7870 CGF.EmitOMPTaskLoopBasedDirective(S);
7872 OMPLexicalScope Scope(CGF, S, OMPD_parallel, /*EmitPreInitStmt=*/false);
7873 CGM.getOpenMPRuntime().emitMasterRegion(CGF, TaskLoopCodeGen,
7884 auto &&CodeGen = [this, &S](CodeGenFunction &CGF, PrePostActionTy &Action) {
7885 auto &&TaskLoopCodeGen = [&S](CodeGenFunction &CGF,
7887 Action.Enter(CGF);
7888 CGF.EmitOMPTaskLoopBasedDirective(S);
7890 OMPLexicalScope Scope(CGF, S, OMPD_parallel, /*EmitPreInitStmt=*/false);
7891 CGM.getOpenMPRuntime().emitMasterRegion(CGF, TaskLoopCodeGen,
7925 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
7931 OMPPrivateScope LoopScope(CGF);
7932 CGF.EmitOMPPrivateLoopCounters(S, LoopScope);
7934 CGF.EmitStmt(CS);
7937 CGF.EmitStmt(CS);
7948 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
7949 Action.Enter(CGF);
7950 emitOMPCopyinClause(CGF, S);
7951 (void)emitWorksharingDirective(CGF, S, /*HasCancel=*/false);
7967 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
7968 CGF.EmitOMPDistributeLoop(S, emitOMPLoopBodyWithStopPoint, S.getInc());
7972 auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
7974 Action.Enter(CGF);
7975 OMPPrivateScope PrivateScope(CGF);
7976 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
7978 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute,
7980 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
7988 static void emitTargetTeamsLoopCodegenStatus(CodeGenFunction &CGF,
7991 bool IsDevice = CGF.CGM.getLangOpts().OpenMPIsTargetDevice;
7997 auto &SM = CGF.getContext().getSourceManager();
8007 CodeGenFunction &CGF, PrePostActionTy &Action,
8009 Action.Enter(CGF);
8012 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
8013 CGF.EmitOMPDistributeLoop(S, emitInnerParallelForWhenCombined,
8018 auto &&CodeGenTeams = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
8020 Action.Enter(CGF);
8021 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
8022 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
8024 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(
8025 CGF, OMPD_distribute, CodeGenDistribute, /*HasCancel=*/false);
8026 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
8030 CGF, TTL_CODEGEN_TYPE " as parallel for", S));
8031 emitCommonOMPTeamsDirective(CGF, S, OMPD_distribute_parallel_for,
8033 emitPostUpdateForReductionClause(CGF, S,
8038 CodeGenFunction &CGF, PrePostActionTy &Action,
8040 Action.Enter(CGF);
8042 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
8043 CGF.EmitOMPDistributeLoop(S, emitOMPLoopBodyWithStopPoint, S.getInc());
8047 auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
8049 Action.Enter(CGF);
8050 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
8051 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
8053 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(
8054 CGF, OMPD_distribute, CodeGenDistribute, /*HasCancel=*/false);
8055 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
8059 CGF, TTL_CODEGEN_TYPE " as distribute", S));
8060 emitCommonOMPTeamsDirective(CGF, S, OMPD_distribute, CodeGen);
8061 emitPostUpdateForReductionClause(CGF, S,
8067 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
8069 emitTargetTeamsGenericLoopRegionAsParallel(CGF, Action, S);
8071 emitTargetTeamsGenericLoopRegionAsDistribute(CGF, Action, S);
8080 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
8082 emitTargetTeamsGenericLoopRegionAsParallel(CGF, Action, S);
8084 emitTargetTeamsGenericLoopRegionAsDistribute(CGF, Action, S);
8096 CodeGenFunction &CGF, const OMPTargetParallelGenericLoopDirective &S,
8098 Action.Enter(CGF);
8100 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
8101 Action.Enter(CGF);
8103 CGF, OMPD_target_parallel_loop, /*hasCancel=*/false);
8104 CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(), emitForLoopBounds,
8107 emitCommonOMPParallelDirective(CGF, S, OMPD_for, CodeGen,
8115 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
8116 emitTargetParallelGenericLoopRegion(CGF, S, Action);
8130 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
8131 emitTargetParallelGenericLoopRegion(CGF, S, Action);
8144 auto &&CodeGen = [&D](CodeGenFunction &CGF, PrePostActionTy &Action) {
8145 OMPPrivateScope GlobalsScope(CGF);
8156 if (!CGF.LocalDeclMap.count(VD)) {
8157 LValue GlobLVal = CGF.EmitLValue(Ref);
8165 ParentLoopDirectiveForScanRegion ScanRegion(CGF, D);
8166 emitOMPSimdRegion(CGF, cast<OMPLoopDirective>(D), Action);
8171 if (!VD->hasLocalStorage() && !CGF.LocalDeclMap.count(VD)) {
8172 LValue GlobLVal = CGF.EmitLValue(E);
8177 if (!CGF.LocalDeclMap.count(VD))
8178 CGF.EmitVarDecl(*VD);
8190 if (!CGF.LocalDeclMap.count(VD))
8191 CGF.EmitVarDecl(*VD);
8197 CGF.EmitStmt(D.getInnermostCapturedStmt()->getCapturedStmt());