Lines Matching +full:dc +full:- +full:dc +full:- +full:phase
1 //===--- CGStmtOpenMP.cpp - Emit LLVM Code from Statements ----------------===//
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 //===----------------------------------------------------------------------===//
11 //===----------------------------------------------------------------------===//
44 #define TTL_CODEGEN_TYPE "target-teams-loop-codegen"
56 cast_or_null<DeclStmt>(CPI->getPreInitStmt())) {
57 for (const auto *I : PreInit->decls()) {
58 if (!I->hasAttr<OMPCaptureNoInitAttr>()) {
74 (CGF.CapturedStmtInfo && CGF.CapturedStmtInfo->lookup(VD)) ||
76 cast<BlockDecl>(CGF.CurCodeDecl)->capturesVariable(VD));
93 for (const auto &C : CS->captures()) {
96 assert(VD == VD->getCanonicalDecl() &&
102 VD->getType().getNonReferenceType(), VK_LValue, C.getLocation());
141 /// Private scope for OpenMP loop-based directives, that supports capturing
149 for (const auto *E : LD->counters()) {
150 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
151 EmittedAsPrivate.insert(VD->getCanonicalDecl());
153 CGF, VD, CGF.CreateMemTemp(VD->getType().getNonReferenceType()));
156 for (const auto *C : LD->getClausesOfKind<OMPPrivateClause>()) {
157 for (const Expr *IRef : C->varlists()) {
159 cast<VarDecl>(cast<DeclRefExpr>(IRef)->getDecl());
160 if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
161 QualType OrigVDTy = OrigVD->getType().getNonReferenceType();
174 LD->getInnermostCapturedStmt()->getCapturedStmt(),
175 /*TryImperfectlyNestedLoops=*/true, LD->getLoopsNumber(),
178 if (const Stmt *Init = CXXFor->getInit())
180 CGF.EmitStmt(CXXFor->getRangeStmt());
181 CGF.EmitStmt(CXXFor->getEndStmt());
185 PreInits = LD->getPreInits();
187 PreInits = Tile->getPreInits();
189 PreInits = Unroll->getPreInits();
191 PreInits = Reverse->getPreInits();
194 PreInits = Interchange->getPreInits();
196 llvm_unreachable("Unknown loop-based directive kind.");
204 llvm::append_range(PreInitStmts, PreInitCompound->body());
212 for (Decl *I : PreInitDecl->decls())
234 (CGF.CapturedStmtInfo && CGF.CapturedStmtInfo->lookup(VD)) ||
236 cast<BlockDecl>(CGF.CurCodeDecl)->capturesVariable(VD));
246 cast_or_null<DeclStmt>(CPI->getPreInitStmt())) {
247 for (const auto *I : PreInit->decls()) {
248 if (!I->hasAttr<OMPCaptureNoInitAttr>()) {
258 for (const Expr *E : UDP->varlists()) {
259 const Decl *D = cast<DeclRefExpr>(E)->getDecl();
264 for (const Expr *E : UDP->varlists()) {
274 if (const Expr *E = TG->getReductionRef())
275 CGF.EmitVarDecl(*cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl()));
281 if (C->getModifier() != OMPC_REDUCTION_inscan)
283 for (const Expr *E : C->copy_array_temps())
284 CopyArrayTemps.insert(cast<DeclRefExpr>(E)->getDecl());
288 for (auto &C : CS->captures()) {
293 assert(VD == VD->getCanonicalDecl() &&
299 VD->getType().getNonReferenceType(), VK_LValue,
304 CS = dyn_cast<CapturedStmt>(CS->getCapturedStmt());
318 if (const auto *OrigVD = dyn_cast<VarDecl>(OrigDRE->getDecl())) {
319 OrigVD = OrigVD->getCanonicalDecl();
322 (CapturedStmtInfo && CapturedStmtInfo->lookup(OrigVD)) ||
325 OrigDRE->getType(), VK_LValue, OrigDRE->getExprLoc());
355 auto CurField = RD->field_begin();
360 if (CurField->hasCapturedVLAType()) {
361 const VariableArrayType *VAT = CurField->getCapturedVLAType();
362 llvm::Value *Val = VLASizeMap[VAT->getSizeExpr()];
364 } else if (CurCap->capturesThis()) {
366 } else if (CurCap->capturesVariableByCopy()) {
367 llvm::Value *CV = EmitLoadOfScalar(EmitLValue(*I), CurCap->getLocation());
371 if (!CurField->getType()->isAnyPointerType()) {
375 Twine(CurCap->getCapturedVar()->getName(), ".casted"));
381 Ctx.getPointerType(CurField->getType()), CurCap->getLocation());
383 MakeNaturalAlignAddrLValue(SrcAddrVal, CurField->getType());
389 CV = EmitLoadOfScalar(DstLV, CurCap->getLocation());
393 assert(CurCap->capturesVariable() && "Expected capture by reference.");
414 if (T->isLValueReferenceType())
418 if (T->isPointerType())
419 return C.getPointerType(getCanonicalParamType(C, T->getPointeeType()));
420 if (const ArrayType *A = T->getAsArrayTypeUnsafe()) {
422 return getCanonicalParamType(C, VLA->getElementType());
423 if (!A->isVariablyModifiedType())
442 /// Location of the non-debug version of the outlined function.
460 const CapturedDecl *CD = FO.S->getCapturedDecl();
461 const RecordDecl *RD = FO.S->getCapturedRecordDecl();
462 assert(CD->hasBody() && "missing CapturedDecl body");
469 Args.append(CD->param_begin(),
470 std::next(CD->param_begin(), CD->getContextParamPosition()));
472 CD->param_begin(),
473 std::next(CD->param_begin(), CD->getContextParamPosition()));
474 auto I = FO.S->captures().begin();
480 Ctx, Ctx.getTranslationUnitDecl(), FO.S->getBeginLoc(),
486 for (const FieldDecl *FD : RD->fields()) {
487 QualType ArgType = FD->getType();
497 ((I->capturesVariableByCopy() && !ArgType->isAnyPointerType()) ||
498 I->capturesVariableArrayType()))
501 if (I->capturesVariable() || I->capturesVariableByCopy()) {
502 CapVar = I->getCapturedVar();
503 II = CapVar->getIdentifier();
504 } else if (I->capturesThis()) {
507 assert(I->capturesVariableArrayType());
510 if (ArgType->isVariablyModifiedType())
513 if (CapVar && (CapVar->getTLSKind() != clang::VarDecl::TLS_None)) {
514 Arg = ImplicitParamDecl::Create(Ctx, /*DC=*/nullptr, FD->getLocation(),
517 } else if (DebugFunctionDecl && (CapVar || I->capturesThis())) {
520 CapVar ? CapVar->getBeginLoc() : FD->getBeginLoc(),
521 CapVar ? CapVar->getLocation() : FD->getLocation(), II, ArgType,
524 Arg = ImplicitParamDecl::Create(Ctx, /*DC=*/nullptr, FD->getLocation(),
528 // Do not cast arguments if we emit function with non-original types.
535 Args.append(std::next(CD->param_begin(), CD->getContextParamPosition() + 1),
536 CD->param_end());
538 std::next(CD->param_begin(), CD->getContextParamPosition() + 1),
539 CD->param_end());
550 if (CD->isNothrow())
551 F->setDoesNotThrow();
552 F->setDoesNotRecurse();
556 F->removeFnAttr(llvm::Attribute::NoInline);
557 F->addFnAttr(llvm::Attribute::AlwaysInline);
562 FO.UIntPtrCastRequired ? FO.Loc : FO.S->getBeginLoc(),
564 : CD->getBody()->getBeginLoc());
565 unsigned Cnt = CD->getContextParamPosition();
566 I = FO.S->captures().begin();
567 for (const FieldDecl *FD : RD->fields()) {
568 // Do not map arguments if we emit function with non-original types.
578 if (I->capturesVariableByCopy() && FD->getType()->isAnyPointerType()) {
579 const VarDecl *CurVD = I->getCapturedVar();
587 LValue ArgLVal = CGF.MakeAddrLValue(LocalAddr, Args[Cnt]->getType(),
589 if (FD->hasCapturedVLAType()) {
592 castValueFromUintptr(CGF, I->getLocation(), FD->getType(),
593 Args[Cnt]->getName(), ArgLVal),
594 FD->getType(), AlignmentSource::Decl);
596 llvm::Value *ExprArg = CGF.EmitLoadOfScalar(ArgLVal, I->getLocation());
597 const VariableArrayType *VAT = FD->getCapturedVLAType();
598 VLASizes.try_emplace(Args[Cnt], VAT->getSizeExpr(), ExprArg);
599 } else if (I->capturesVariable()) {
600 const VarDecl *Var = I->getCapturedVar();
601 QualType VarTy = Var->getType();
603 if (ArgLVal.getType()->isLValueReferenceType()) {
605 } else if (!VarTy->isVariablyModifiedType() || !VarTy->isPointerType()) {
606 assert(ArgLVal.getType()->isPointerType());
608 ArgAddr, ArgLVal.getType()->castAs<PointerType>());
614 } else if (I->capturesVariableByCopy()) {
615 assert(!FD->getType()->isAnyPointerType() &&
617 const VarDecl *Var = I->getCapturedVar();
621 CGF, I->getLocation(), FD->getType(),
622 Args[Cnt]->getName(), ArgLVal)
626 assert(I->capturesThis());
627 CXXThisValue = CGF.EmitLoadOfScalar(ArgLVal, I->getLocation());
652 Out << CapturedStmtInfo->getHelperName();
670 CapturedStmtInfo->EmitBody(*this, CD->getBody());
672 FinishFunction(CD->getBodyRBrace());
678 CapturedStmtInfo->getHelperName(), Loc);
688 auto *PI = F->arg_begin();
694 I->second.second,
695 I->second.first ? I->second.first->getType() : Arg->getType(),
697 if (LV.getType()->isAnyComplexType())
698 LV.setAddress(LV.getAddress().withElementType(PI->getType()));
703 CallArg = EI->second.second;
707 Arg->getType(), AlignmentSource::Decl);
711 CallArgs.emplace_back(WrapperCGF.EmitFromMemory(CallArg, Arg->getType()));
719 //===----------------------------------------------------------------------===//
721 //===----------------------------------------------------------------------===//
725 // Perform element-by-element initialization.
729 const ArrayType *ArrayTy = OriginalType->getAsArrayTypeUnsafe();
739 // The basic structure here is a while-do loop.
753 Builder.CreatePHI(SrcBegin->getType(), 2, "omp.arraycpy.srcElementPast");
754 SrcElementPHI->addIncoming(SrcBegin, EntryBB);
760 DestBegin->getType(), 2, "omp.arraycpy.destElementPast");
761 DestElementPHI->addIncoming(DestBegin, EntryBB);
780 DestElementPHI->addIncoming(DestElementNext, Builder.GetInsertBlock());
781 SrcElementPHI->addIncoming(SrcElementNext, Builder.GetInsertBlock());
790 if (OriginalType->isArrayType()) {
792 if (BO && BO->getOpcode() == BO_Assign) {
834 for (const auto *D : C->varlists())
836 cast<VarDecl>(cast<DeclRefExpr>(D)->getDecl())->getCanonicalDecl(),
837 C->getKind());
847 const auto *IRef = C->varlist_begin();
848 const auto *InitsRef = C->inits().begin();
849 for (const Expr *IInit : C->private_copies()) {
850 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
852 Lastprivates.count(OrigVD->getCanonicalDecl()) > 0;
853 const FieldDecl *FD = CapturedStmtInfo->lookup(OrigVD);
854 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(IInit)->getDecl());
856 !FD->getType()->isReferenceType() &&
857 (!VD || !VD->hasAttr<OMPAllocateDeclAttr>())) {
858 EmittedAsFirstprivate.insert(OrigVD->getCanonicalDecl());
865 if (DeviceConstTarget && OrigVD->getType().isConstant(getContext()) &&
866 FD && FD->getType()->isReferenceType() &&
867 (!VD || !VD->hasAttr<OMPAllocateDeclAttr>())) {
868 EmittedAsFirstprivate.insert(OrigVD->getCanonicalDecl());
875 if (EmittedAsFirstprivate.insert(OrigVD->getCanonicalDecl()).second) {
877 cast<VarDecl>(cast<DeclRefExpr>(*InitsRef)->getDecl());
881 (*IRef)->getType(), VK_LValue, (*IRef)->getExprLoc());
895 assert(!CE && "Expected non-constant firstprivate.");
901 QualType Type = VD->getType();
902 if (Type->isArrayType()) {
907 const Expr *Init = VD->getInit();
922 Init->getType().getQualifiers(),
940 Lastprivates[OrigVD->getCanonicalDecl()] ==
944 EmitLoadOfScalar(MakeAddrLValue(VDAddr, (*IRef)->getType(),
946 (*IRef)->getExprLoc());
949 EmitStoreOfScalar(V, MakeAddrLValue(VDAddr, (*IRef)->getType(),
975 auto IRef = C->varlist_begin();
976 for (const Expr *IInit : C->private_copies()) {
977 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
978 if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
979 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(IInit)->getDecl());
1003 auto IRef = C->varlist_begin();
1004 auto ISrcRef = C->source_exprs().begin();
1005 auto IDestRef = C->destination_exprs().begin();
1006 for (const Expr *AssignOp : C->assignment_ops()) {
1007 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
1008 QualType Type = VD->getType();
1009 if (CopiedVars.insert(VD->getCanonicalDecl()).second) {
1016 assert(CapturedStmtInfo->lookup(VD) &&
1019 (*IRef)->getType(), VK_LValue, (*IRef)->getExprLoc());
1024 Address(VD->isStaticLocal() ? CGM.getStaticLocalDeclAddress(VD)
1026 CGM.getTypes().ConvertTypeForMem(VD->getType()),
1047 cast<VarDecl>(cast<DeclRefExpr>(*ISrcRef)->getDecl());
1049 cast<VarDecl>(cast<DeclRefExpr>(*IDestRef)->getDecl());
1058 // Exit out of copying procedure for non-master thread.
1073 for (const Expr *C : LoopDirective->counters()) {
1075 cast<VarDecl>(cast<DeclRefExpr>(C)->getDecl())->getCanonicalDecl());
1084 const auto *IRef = C->varlist_begin();
1085 const auto *IDestRef = C->destination_exprs().begin();
1086 for (const Expr *IInit : C->private_copies()) {
1089 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
1092 if (AlreadyEmittedVars.insert(OrigVD->getCanonicalDecl()).second) {
1094 cast<VarDecl>(cast<DeclRefExpr>(*IDestRef)->getDecl());
1097 CapturedStmtInfo->lookup(OrigVD) != nullptr,
1098 (*IRef)->getType(), VK_LValue, (*IRef)->getExprLoc());
1103 if (IInit && !SIMDLCVs.count(OrigVD->getCanonicalDecl())) {
1104 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(IInit)->getDecl());
1106 if (C->getKind() == OMPC_LASTPRIVATE_conditional) {
1147 return C->getKind() == OMPC_LASTPRIVATE_conditional;
1162 auto IC = LoopDirective->counters().begin();
1163 for (const Expr *F : LoopDirective->finals()) {
1165 cast<VarDecl>(cast<DeclRefExpr>(*IC)->getDecl())->getCanonicalDecl();
1174 auto IRef = C->varlist_begin();
1175 auto ISrcRef = C->source_exprs().begin();
1176 auto IDestRef = C->destination_exprs().begin();
1177 for (const Expr *AssignOp : C->assignment_ops()) {
1179 cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
1180 QualType Type = PrivateVD->getType();
1181 const auto *CanonicalVD = PrivateVD->getCanonicalDecl();
1183 // If lastprivate variable is a loop control variable for loop-based
1189 cast<VarDecl>(cast<DeclRefExpr>(*ISrcRef)->getDecl());
1191 cast<VarDecl>(cast<DeclRefExpr>(*IDestRef)->getDecl());
1194 if (const auto *RefTy = PrivateVD->getType()->getAs<ReferenceType>())
1197 CGM.getTypes().ConvertTypeForMem(RefTy->getPointeeType()),
1198 CGM.getNaturalTypeAlignment(RefTy->getPointeeType()));
1200 if (C->getKind() == OMPC_LASTPRIVATE_conditional)
1202 *this, MakeAddrLValue(PrivateAddr, (*IRef)->getType()), PrivateVD,
1203 (*IRef)->getExprLoc());
1212 if (const Expr *PostUpdate = C->getPostUpdateExpr())
1233 if (ForInscan != (C->getModifier() == OMPC_REDUCTION_inscan))
1235 Shareds.append(C->varlist_begin(), C->varlist_end());
1236 Privates.append(C->privates().begin(), C->privates().end());
1237 ReductionOps.append(C->reduction_ops().begin(), C->reduction_ops().end());
1238 LHSs.append(C->lhs_exprs().begin(), C->lhs_exprs().end());
1239 RHSs.append(C->rhs_exprs().begin(), C->rhs_exprs().end());
1240 if (C->getModifier() == OMPC_REDUCTION_task) {
1241 Data.ReductionVars.append(C->privates().begin(), C->privates().end());
1242 Data.ReductionOrigs.append(C->varlist_begin(), C->varlist_end());
1243 Data.ReductionCopies.append(C->privates().begin(), C->privates().end());
1244 Data.ReductionOps.append(C->reduction_ops().begin(),
1245 C->reduction_ops().end());
1246 TaskLHSs.append(C->lhs_exprs().begin(), C->lhs_exprs().end());
1247 TaskRHSs.append(C->rhs_exprs().begin(), C->rhs_exprs().end());
1256 const auto *PrivateVD = cast<VarDecl>(cast<DeclRefExpr>(*IPriv)->getDecl());
1276 const auto *LHSVD = cast<VarDecl>(cast<DeclRefExpr>(*ILHS)->getDecl());
1277 const auto *RHSVD = cast<VarDecl>(cast<DeclRefExpr>(*IRHS)->getDecl());
1278 QualType Type = PrivateVD->getType();
1280 if (isaOMPArraySectionExpr && Type->isVariablyModifiedType()) {
1285 } else if ((isaOMPArraySectionExpr && Type->isScalarType()) ||
1292 ConvertTypeForMem(RHSVD->getType())));
1294 QualType Type = PrivateVD->getType();
1301 OriginalAddr.withElementType(ConvertTypeForMem(LHSVD->getType()));
1306 ConvertTypeForMem(RHSVD->getType()))
1422 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(TaskRedRef)->getDecl());
1425 /*Volatile=*/false, TaskRedRef->getType());
1441 if (C->getModifier() == OMPC_REDUCTION_inscan)
1444 Privates.append(C->privates().begin(), C->privates().end());
1445 LHSExprs.append(C->lhs_exprs().begin(), C->lhs_exprs().end());
1446 RHSExprs.append(C->rhs_exprs().begin(), C->rhs_exprs().end());
1447 ReductionOps.append(C->reduction_ops().begin(), C->reduction_ops().end());
1449 IsReductionWithTaskMod || C->getModifier() == OMPC_REDUCTION_task;
1459 TeamsLoopCanBeParallel = TTLD->canBeParallelFor();
1479 if (const Expr *PostUpdate = C->getPostUpdateExpr()) {
1482 // If the first post-update expression is found, emit conditional
1514 for (const Expr *Ref : C->varlists()) {
1515 if (!Ref->getType()->isScalarType())
1517 const auto *DRE = dyn_cast<DeclRefExpr>(Ref->IgnoreParenImpCasts());
1520 PrivateDecls.insert(cast<VarDecl>(DRE->getDecl()));
1525 for (const Expr *Ref : C->varlists()) {
1526 if (!Ref->getType()->isScalarType())
1528 const auto *DRE = dyn_cast<DeclRefExpr>(Ref->IgnoreParenImpCasts());
1531 PrivateDecls.insert(cast<VarDecl>(DRE->getDecl()));
1536 for (const Expr *Ref : C->varlists()) {
1537 if (!Ref->getType()->isScalarType())
1539 const auto *DRE = dyn_cast<DeclRefExpr>(Ref->IgnoreParenImpCasts());
1542 PrivateDecls.insert(cast<VarDecl>(DRE->getDecl()));
1547 // Task reductions may be skipped - tasks are ignored.
1548 // Firstprivates do not return value but may be passed by reference - no need
1551 for (const Expr *Ref : C->varlists()) {
1552 if (!Ref->getType()->isScalarType())
1554 const auto *DRE = dyn_cast<DeclRefExpr>(Ref->IgnoreParenImpCasts());
1557 PrivateDecls.insert(cast<VarDecl>(DRE->getDecl()));
1572 CGF, S, *CS->getCapturedDecl()->param_begin(), InnermostKind,
1576 NumThreads = CGF.EmitScalarExpr(NumThreadsClause->getNumThreads(),
1579 CGF, NumThreads, NumThreadsClause->getBeginLoc());
1584 CGF, ProcBindClause->getProcBindKind(), ProcBindClause->getBeginLoc());
1588 if (C->getNameModifier() == OMPD_unknown ||
1589 C->getNameModifier() == OMPD_parallel) {
1590 IfCond = C->getCondition();
1608 const VarDecl *CVD = VD->getCanonicalDecl();
1609 if (!CVD->hasAttr<OMPAllocateDeclAttr>())
1611 const auto *AA = CVD->getAttr<OMPAllocateDeclAttr>();
1613 return !((AA->getAllocatorType() == OMPAllocateDeclAttr::OMPDefaultMemAlloc ||
1614 AA->getAllocatorType() == OMPAllocateDeclAttr::OMPNullMemAlloc) &&
1615 !AA->getAllocator());
1642 const VarDecl *CVD = VD->getCanonicalDecl();
1647 if (CVD->getType()->isVariablyModifiedType()) {
1648 Size = CGF.getTypeSize(CVD->getType());
1649 // Align the size: ((size + align - 1) / align) * align
1651 Size, CGM.getSize(Align - CharUnits::fromQuantity(1)));
1655 CharUnits Sz = CGM.getContext().getTypeSizeInChars(CVD->getType());
1659 const auto *AA = CVD->getAttr<OMPAllocateDeclAttr>();
1660 assert(AA->getAllocator() &&
1661 "Expected allocator expression for non-default allocator.");
1662 llvm::Value *Allocator = CGF.EmitScalarExpr(AA->getAllocator());
1665 if (Allocator->getType()->isIntegerTy())
1667 else if (Allocator->getType()->isPointerTy())
1673 getNameWithSeparators({CVD->getName(), ".void.addr"}, ".", "."));
1680 CGF.ConvertTypeForMem(CGM.getContext().getPointerType(CVD->getType())),
1681 getNameWithSeparators({CVD->getName(), ".addr"}, ".", "."));
1682 return Address(Addr, CGF.ConvertTypeForMem(CVD->getType()), Align);
1760 IfCond = EmitScalarExpr(C->getCondition(),
1765 NumThreads = EmitScalarExpr(NumThreadsClause->getNumThreads(),
1770 ProcBind = ProcBindClause->getProcBindKind();
1787 // data-sharing attribute "shared".
1794 const Stmt *ParallelRegionBodyStmt = CS->getCapturedStmt();
1805 AllocaInsertPt->getParent(), AllocaInsertPt->getIterator());
1821 CGF.EmitStmt(S.getCapturedStmt(OMPD_parallel)->getCapturedStmt());
1873 const Stmt *SimplifiedS = S->IgnoreContainers();
1876 CGF.getContext().getSourceManager(), CS->getLBracLoc(),
1880 CodeGenFunction::LexicalScope Scope(CGF, S->getSourceRange());
1881 for (const Stmt *CurStmt : CS->body())
1887 SimplifiedS = Dir->getTransformedStmt();
1889 SimplifiedS = CanonLoop->getLoopStmt();
1891 S = For->getBody();
1894 "Expected canonical for loop or range-based for loop.");
1896 CGF.EmitStmt(CXXFor->getLoopVarStmt());
1897 S = CXXFor->getBody();
1920 for (const Expr *UE : C->updates())
1931 // Check that loop counter in non-rectangular nest fits into the iteration
1961 D.getInnermostCapturedStmt()->getCapturedStmt()->IgnoreContainers();
2010 // The caller is processing the loop-associated directive processing the \p
2011 // Depth loops nested in \p S. Put the previous pending loop-associated
2012 // directive to the stack. If the current loop-associated directive is a loop
2015 // nest for the parent loop-associated directive.
2034 const Stmt *SyntacticalLoop = S->getLoopStmt();
2041 LexicalScope ForScope(*this, S->getSourceRange());
2047 if (const Stmt *InitStmt = For->getInit())
2049 BodyStmt = For->getBody();
2052 if (const DeclStmt *RangeStmt = RangeFor->getRangeStmt())
2054 if (const DeclStmt *BeginStmt = RangeFor->getBeginStmt())
2056 if (const DeclStmt *EndStmt = RangeFor->getEndStmt())
2058 if (const DeclStmt *LoopVarStmt = RangeFor->getLoopVarStmt())
2060 BodyStmt = RangeFor->getBody();
2062 llvm_unreachable("Expected for-stmt or range-based for-stmt");
2064 // Emit closure for later use. By-value captures will be captured here.
2065 const CapturedStmt *DistanceFunc = S->getDistanceFunc();
2067 const CapturedStmt *LoopVarFunc = S->getLoopVarFunc();
2072 QualType LogicalTy = DistanceFunc->getCapturedDecl()
2073 ->getParam(0)
2074 ->getType()
2088 const DeclRefExpr *LoopVarRef = S->getLoopVarRef();
2101 Builder.restoreIP(CL->getAfterIP());
2123 const Stmt *SS = ICS->getCapturedStmt();
2128 AS->getAttrs(), SourceLocToDebugLoc(R.getBegin()),
2134 // If there are any cleanups between here and the loop-exit scope,
2158 // Emit "IV = IV + 1" and a back-edge to the condition block.
2165 // Emit the fall-through block.
2175 for (const Expr *Init : C->inits()) {
2177 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(Init)->getDecl());
2179 dyn_cast<DeclRefExpr>(VD->getInit()->IgnoreImpCasts())) {
2181 const auto *OrigVD = cast<VarDecl>(Ref->getDecl());
2183 CapturedStmtInfo->lookup(OrigVD) != nullptr,
2184 VD->getInit()->getType(), VK_LValue,
2185 VD->getInit()->getExprLoc());
2188 MakeAddrLValue(Emission.getAllocatedAddress(), VD->getType()),
2196 // If a step is not constant, it is pre-calculated before the loop.
2197 if (const auto *CS = cast_or_null<BinaryOperator>(C->getCalcStep()))
2198 if (const auto *SaveRef = cast<DeclRefExpr>(CS->getLHS())) {
2199 EmitVarDecl(*cast<VarDecl>(SaveRef->getDecl()));
2215 auto IC = C->varlist_begin();
2216 for (const Expr *F : C->finals()) {
2219 // If the first post-update expression is found, emit conditional
2227 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IC)->getDecl());
2229 CapturedStmtInfo->lookup(OrigVD) != nullptr,
2230 (*IC)->getType(), VK_LValue, (*IC)->getExprLoc());
2238 if (const Expr *PostUpdate = C->getPostUpdateExpr())
2251 if (const Expr *AlignmentExpr = Clause->getAlignment()) {
2254 ClauseAlignment = AlignmentCI->getValue();
2256 for (const Expr *E : Clause->varlists()) {
2260 // If no optional parameter is specified, implementation-defined default
2265 E->getType()->getPointeeType()))
2286 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
2287 const auto *PrivateVD = cast<VarDecl>(cast<DeclRefExpr>(*I)->getDecl());
2293 if (LocalDeclMap.count(VD) || CapturedStmtInfo->lookup(VD) ||
2294 VD->hasGlobalStorage()) {
2296 LocalDeclMap.count(VD) || CapturedStmtInfo->lookup(VD),
2297 E->getType(), VK_LValue, E->getExprLoc());
2306 if (!C->getNumForLoops())
2308 for (unsigned I = S.getLoopsNumber(), E = C->getLoopNumIterations().size();
2310 const auto *DRE = cast<DeclRefExpr>(C->getLoopCounter(I));
2311 const auto *VD = cast<VarDecl>(DRE->getDecl());
2312 // Override only those variables that can be captured to avoid re-emission
2314 if (DRE->refersToEnclosingVariableOrCapture()) {
2316 VD, CreateMemTemp(DRE->getType(), VD->getName()));
2337 // non-rectangular loops.
2342 assert(!E->getType().getNonReferenceType()->isRecordType() &&
2344 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
2346 CGF.CreateMemTemp(VD->getType().getNonReferenceType());
2367 for (const Expr *C : LoopDirective->counters()) {
2369 cast<VarDecl>(cast<DeclRefExpr>(C)->getDecl())->getCanonicalDecl());
2373 auto CurPrivate = C->privates().begin();
2374 for (const Expr *E : C->varlists()) {
2375 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
2377 cast<VarDecl>(cast<DeclRefExpr>(*CurPrivate)->getDecl());
2378 if (!SIMDLCVs.count(VD->getCanonicalDecl())) {
2399 RValue Len = CGF.EmitAnyExpr(C->getSimdlen(), AggValueSlot::ignored(),
2402 CGF.LoopStack.setVectorizeWidth(Val->getZExtValue());
2404 // the memory instructions parallel, because loop-carried
2408 RValue Len = CGF.EmitAnyExpr(C->getSafelen(), AggValueSlot::ignored(),
2411 CGF.LoopStack.setVectorizeWidth(Val->getZExtValue());
2413 // the memory instructions parallel, because loop-carried
2425 if (C->getKind() == OMPC_ORDER_concurrent)
2432 return C->getModifier() == OMPC_REDUCTION_inscan;
2447 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>((*IC))->getDecl());
2448 const auto *PrivateVD = cast<VarDecl>(cast<DeclRefExpr>((*IPC))->getDecl());
2450 if (LocalDeclMap.count(OrigVD) || CapturedStmtInfo->lookup(OrigVD) ||
2451 OrigVD->hasGlobalStorage() || CED) {
2454 // If the first post-update expression is found, emit conditional
2464 OrigAddr = EmitLValue(CED->getInit()->IgnoreImpCasts()).getAddress();
2468 (*IPC)->getType(), VK_LValue, (*IPC)->getExprLoc());
2493 auto VDecl = cast<VarDecl>(Helper->getDecl());
2519 (C->getNameModifier() == OMPD_unknown ||
2520 C->getNameModifier() == OMPD_simd)) {
2521 IfCond = C->getCondition();
2552 // Emit: if (PreCond) - begin.
2571 const auto *IVDecl = cast<VarDecl>(cast<DeclRefExpr>(IVExpr)->getDecl());
2579 CGF.EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl()));
2623 // Emit: if (PreCond) - end.
2643 if (const Stmt *SyntacticalLoop = CanonLoop->getLoopStmt()) {
2644 for (const Stmt *SubStmt : SyntacticalLoop->children()) {
2648 for (const Stmt *CSSubStmt : CS->children()) {
2666 if (const Expr *AlignmentExpr = Clause->getAlignment()) {
2669 ClauseAlignment = AlignmentCI->getValue();
2671 for (const Expr *E : Clause->varlists()) {
2675 // If no optional parameter is specified, implementation-defined default
2680 E->getType()->getPointeeType()))
2713 this->EmitAnyExpr(C->getSimdlen(), AggValueSlot::ignored(),
2721 this->EmitAnyExpr(C->getSafelen(), AggValueSlot::ignored(),
2728 if (C->getKind() == OpenMPOrderClauseKind ::OMPC_ORDER_concurrent) {
2765 // Emit the de-sugared statement.
2771 // Emit the de-sugared statement.
2778 // Emit the de-sugared statement.
2791 // fully unrolled loop is non-transformable. For partial unrolling the
2806 if (Expr *FactorExpr = PartialClause->getFactor()) {
2807 Factor = FactorExpr->EvaluateKnownConstInt(getContext()).getZExtValue();
2825 // other loop-associated construct. Such a loop-associated construct will have
2834 if (Expr *FactorExpr = PartialClause->getFactor()) {
2836 FactorExpr->EvaluateKnownConstInt(getContext()).getZExtValue();
2854 const unsigned IVSize = getContext().getTypeSize(IVExpr->getType());
2855 const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();
2883 // If there are any cleanups between here and the loop-exit scope,
2914 if (C->getKind() == OMPC_ORDER_concurrent)
2949 // Emit the fall-through block.
2973 "static non-chunked schedule does not need outer loop");
2977 // OpenMP [2.7.1, Loop Construct, Description, table 2-1]
3003 // run-sched-var ICV. If the ICV is set to auto, the schedule is
3015 // OpenMP [2.7.1, Loop Construct, Description, table 2-1]
3018 // the team in a round-robin fashion in the order of the thread number.
3028 const unsigned IVSize = getContext().getTypeSize(IVExpr->getType());
3029 const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();
3088 const unsigned IVSize = getContext().getTypeSize(IVExpr->getType());
3089 const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();
3154 PrevLB, LS.getPrevLowerBoundVariable()->getExprLoc());
3156 PrevLBVal, LS.getPrevLowerBoundVariable()->getType(),
3157 LS.getIterationVariable()->getType(),
3158 LS.getPrevLowerBoundVariable()->getExprLoc());
3160 PrevUB, LS.getPrevUpperBoundVariable()->getExprLoc());
3162 PrevUBVal, LS.getPrevUpperBoundVariable()->getType(),
3163 LS.getIterationVariable()->getType(),
3164 LS.getPrevUpperBoundVariable()->getExprLoc());
3189 QualType IteratorTy = IVExpr->getType();
3224 HasCancel = D->hasCancel();
3226 HasCancel = D->hasCancel();
3229 HasCancel = D->hasCancel();
3314 const auto *IVDecl = cast<VarDecl>(IVExpr->getDecl());
3321 EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl()));
3329 // Check pre-condition.
3352 if (OrderedClause->getNumForLoops())
3353 RT.emitDoacrossInit(*this, S, OrderedClause->getLoopNumIterations());
3376 // initialization of firstprivate variables and post-update of
3397 ScheduleKind.Schedule = C->getScheduleKind();
3398 ScheduleKind.M1 = C->getFirstScheduleModifier();
3399 ScheduleKind.M2 = C->getSecondScheduleModifier();
3400 ChunkExpr = C->getChunkSize();
3410 Chunk = EmitScalarConversion(Chunk, ChunkExpr->getType(),
3411 S.getIterationVariable()->getType(),
3414 if (ChunkExpr->EvaluateAsInt(Result, getContext())) {
3419 const unsigned IVSize = getContext().getTypeSize(IVExpr->getType());
3420 const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();
3449 if (C->getKind() == OMPC_ORDER_concurrent)
3456 // OpenMP [2.7.1, Loop Construct, Description, table 2-1]
3523 // Emit post-update of the reduction variables if IsLastIter != 0.
3572 const unsigned IVSize = CGF.getContext().getTypeSize(IVExpr->getType());
3595 assert(C->getModifier() == OMPC_REDUCTION_inscan &&
3597 Shareds.append(C->varlist_begin(), C->varlist_end());
3598 Privates.append(C->privates().begin(), C->privates().end());
3599 ReductionOps.append(C->reduction_ops().begin(), C->reduction_ops().end());
3600 CopyArrayTemps.append(C->copy_array_temps().begin(),
3601 C->copy_array_temps().end());
3611 const auto *PrivateVD = cast<VarDecl>(cast<DeclRefExpr>(IRef)->getDecl());
3614 if (PrivateVD->getType()->isVariablyModifiedType()) {
3621 cast<VariableArrayType>((*ITA)->getType()->getAsArrayTypeUnsafe())
3622 ->getSizeExpr()),
3625 CGF.EmitVarDecl(*cast<VarDecl>(cast<DeclRefExpr>(*ITA)->getDecl()));
3635 /// <orig_var> = buffer[num_iters-1];
3649 assert(C->getModifier() == OMPC_REDUCTION_inscan &&
3651 Shareds.append(C->varlist_begin(), C->varlist_end());
3652 LHSs.append(C->lhs_exprs().begin(), C->lhs_exprs().end());
3653 RHSs.append(C->rhs_exprs().begin(), C->rhs_exprs().end());
3654 Privates.append(C->privates().begin(), C->privates().end());
3655 CopyOps.append(C->copy_ops().begin(), C->copy_ops().end());
3656 CopyArrayElems.append(C->copy_array_elems().begin(),
3657 C->copy_array_elems().end());
3671 cast<ArraySubscriptExpr>(CopyArrayElem)->getIdx()),
3676 PrivateExpr->getType(), DestLVal.getAddress(), SrcLVal.getAddress(),
3677 cast<VarDecl>(cast<DeclRefExpr>(LHSs[I])->getDecl()),
3678 cast<VarDecl>(cast<DeclRefExpr>(RHSs[I])->getDecl()), CopyOps[I]);
3687 /// <input phase>;
3692 /// for (size cnt = last_iter; cnt >= pow(2, k); --k)
3693 /// buffer[i] op= buffer[i-pow(2,k)];
3697 /// red = InclusiveScan ? buffer[i] : buffer[i-1];
3698 /// <scan phase>;
3714 assert(C->getModifier() == OMPC_REDUCTION_inscan &&
3716 Privates.append(C->privates().begin(), C->privates().end());
3717 ReductionOps.append(C->reduction_ops().begin(), C->reduction_ops().end());
3718 LHSs.append(C->lhs_exprs().begin(), C->lhs_exprs().end());
3719 RHSs.append(C->rhs_exprs().begin(), C->rhs_exprs().end());
3720 CopyArrayElems.append(C->copy_array_elems().begin(),
3721 C->copy_array_elems().end());
3725 // Emit loop with input phase:
3728 // <input phase>;
3761 Counter->addIncoming(llvm::ConstantInt::get(CGF.IntTy, 0), InputBB);
3762 Pow2K->addIncoming(llvm::ConstantInt::get(CGF.SizeTy, 1), InputBB);
3763 // for (size i = n - 1; i >= 2 ^ k; --i)
3764 // tmp[i] op= tmp[i-pow2k];
3773 IVal->addIncoming(NMin1, LoopBB);
3779 const auto *LHSVD = cast<VarDecl>(cast<DeclRefExpr>(*ILHS)->getDecl());
3780 const auto *RHSVD = cast<VarDecl>(cast<DeclRefExpr>(*IRHS)->getDecl());
3786 cast<ArraySubscriptExpr>(CopyArrayElem)->getIdx()),
3797 cast<ArraySubscriptExpr>(CopyArrayElem)->getIdx()),
3812 IVal->addIncoming(NextIVal, CGF.Builder.GetInsertBlock());
3818 Counter->addIncoming(Next, CGF.Builder.GetInsertBlock());
3822 Pow2K->addIncoming(NextPow2K, CGF.Builder.GetInsertBlock());
3848 return C->getModifier() == OMPC_REDUCTION_inscan;
3896 if (SC->getFirstScheduleModifier() != OMPC_SCHEDULE_MODIFIER_unknown)
3898 if (SC->getSecondScheduleModifier() != OMPC_SCHEDULE_MODIFIER_unknown)
3900 switch (SC->getScheduleKind()) {
3951 convertClauseKindToSchedKind(SchedClause->getScheduleKind());
3952 if (const Expr *ChunkSizeExpr = SchedClause->getChunkSize())
3964 AllocaInsertPt->getParent(), AllocaInsertPt->getIterator());
4022 const Stmt *CapturedStmt = S.getInnermostCapturedStmt()->getCapturedStmt();
4034 ? CGF.Builder.getInt32(CS->size() - 1)
4063 // case <NumSection> - 1:
4064 // <SectionStmt[<NumSection> - 1]>;
4071 ExitBB, CS == nullptr ? 1 : CS->size());
4074 for (const Stmt *SubStmt : CS->children()) {
4077 SwitchStmt->addCase(CGF.Builder.getInt32(CaseNumber), CaseBB);
4085 SwitchStmt->addCase(CGF.Builder.getInt32(0), CaseBB);
4095 // initialization of firstprivate variables and post-update of lastprivate
4109 // Emit static non-chunked loop.
4134 // Emit post-update of the reduction variables if IsLastIter != 0.
4150 HasCancel = OSD->hasCancel();
4152 HasCancel = OPSD->hasCancel();
4178 const Stmt *CapturedStmt = S.getInnermostCapturedStmt()->getCapturedStmt();
4182 for (const Stmt *SubStmt : CS->children()) {
4206 // data-sharing attribute "shared".
4215 AllocaInsertPt->getParent(), AllocaInsertPt->getIterator());
4273 CopyprivateVars.append(C->varlists().begin(), C->varlists().end());
4274 DestExprs.append(C->destination_exprs().begin(),
4275 C->destination_exprs().end());
4276 SrcExprs.append(C->source_exprs().begin(), C->source_exprs().end());
4277 AssignmentOps.append(C->assignment_ops().begin(),
4278 C->assignment_ops().end());
4287 CGF.EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt());
4351 Filter = FilterClause->getThreadID();
4364 Filter = FilterClause->getThreadID();
4399 Hint = HintClause->getHint();
4434 Hint = HintClause->getHint();
4461 return C->getModifier() == OMPC_REDUCTION_inscan;
4495 return C->getModifier() == OMPC_REDUCTION_inscan;
4596 for (const Decl *D : S->decls()) {
4598 if (VD->hasLocalStorage())
4609 for (const Stmt *Child : S->children())
4626 return C->getDependencyKind() == OMPC_DEPEND_outallmemory ||
4627 C->getDependencyKind() == OMPC_DEPEND_inoutallmemory;
4642 OpenMPDependClauseKind Kind = C->getDependencyKind();
4648 Data.Dependences.emplace_back(C->getDependencyKind(), C->getModifier());
4649 DD.DepExprs.append(C->varlist_begin(), C->varlist_end());
4659 auto I = CS->getCapturedDecl()->param_begin();
4666 const Expr *Cond = Clause->getCondition();
4678 const Expr *Prio = Clause->getPriority();
4681 EmitScalarExpr(Prio), Prio->getType(),
4683 Prio->getExprLoc()));
4690 auto IRef = C->varlist_begin();
4691 for (const Expr *IInit : C->private_copies()) {
4692 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
4693 if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
4703 auto IRef = C->varlist_begin();
4704 auto IElemInitRef = C->inits().begin();
4705 for (const Expr *IInit : C->private_copies()) {
4706 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
4707 if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
4719 auto IRef = C->varlist_begin();
4720 auto ID = C->destination_exprs().begin();
4721 for (const Expr *IInit : C->private_copies()) {
4722 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
4723 if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
4728 std::make_pair(cast<VarDecl>(cast<DeclRefExpr>(*ID)->getDecl()),
4737 Data.ReductionVars.append(C->varlist_begin(), C->varlist_end());
4738 Data.ReductionOrigs.append(C->varlist_begin(), C->varlist_end());
4739 Data.ReductionCopies.append(C->privates().begin(), C->privates().end());
4740 Data.ReductionOps.append(C->reduction_ops().begin(),
4741 C->reduction_ops().end());
4742 LHSs.append(C->lhs_exprs().begin(), C->lhs_exprs().end());
4743 RHSs.append(C->rhs_exprs().begin(), C->rhs_exprs().end());
4752 Checker.Visit(S.getInnermostCapturedStmt()->getCapturedStmt());
4767 CGF.CapturedStmtInfo->getCaptureFields();
4768 llvm::Value *ContextValue = CGF.CapturedStmtInfo->getContextValue();
4785 const VarDecl *SharedVar = It->first;
4786 RecordDecl *CaptureRecord = It->second->getParent();
4790 Layout.getFieldOffset(It->second->getFieldIndex()) / CharWidth;
4792 (void)DI->EmitDeclareOfAutoVariable(SharedVar, ContextValue,
4805 Declare->setExpression(llvm::DIExpression::get(Ctx, Ops));
4807 llvm::Instruction &Last = CGF.Builder.GetInsertBlock()->back();
4809 UpdateExpr(DDI->getContext(), DDI, Offset);
4814 CGF.Builder.GetInsertBlock()->getTrailingDbgRecords()) {
4816 llvm::filterDbgVars(Marker->getDbgRecordRange()))) {
4829 CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(CopyFnParam)));
4831 CS->getCapturedDecl()->getParam(PrivatesParam)));
4837 ParamTypes.push_back(PrivatesPtr->getType());
4839 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
4841 CGF.getContext().getPointerType(E->getType()), ".priv.ptr.addr");
4847 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
4849 CGF.CreateMemTemp(CGF.getContext().getPointerType(E->getType()),
4857 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
4859 CGF.CreateMemTemp(CGF.getContext().getPointerType(E->getType()),
4866 QualType Ty = VD->getType().getNonReferenceType();
4867 if (VD->getType()->isLValueReferenceType())
4887 const auto *OrigVD = cast<VarDecl>(Pair.second->getDecl());
4890 CGF.CapturedStmtInfo->lookup(OrigVD) != nullptr,
4891 Pair.second->getType(), VK_LValue,
4892 Pair.second->getExprLoc());
4898 CGF.ConvertTypeForMem(Pair.first->getType().getNonReferenceType()),
4903 (void)DI->EmitDeclareOfAutoVariable(
4910 QualType VDType = Pair.first->getType().getNonReferenceType();
4911 if (Pair.first->getType()->isLValueReferenceType())
4937 CGF.ConvertTypeForMem(Pair.first->getType().getNonReferenceType()),
4946 CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(9)));
4961 Data.ReductionCopies[Cnt]->getType()),
4962 Data.ReductionCopies[Cnt]->getExprLoc()),
4963 CGF.ConvertTypeForMem(Data.ReductionCopies[Cnt]->getType()),
4976 auto IPriv = C->privates().begin();
4977 auto IRed = C->reduction_ops().begin();
4978 auto ITD = C->taskgroup_descriptors().begin();
4979 for (const Expr *Ref : C->varlists()) {
5007 TRExpr->getExprLoc());
5016 CGF.getContext().getPointerType(InRedPrivs[Cnt]->getType()),
5017 InRedPrivs[Cnt]->getExprLoc()),
5018 CGF.ConvertTypeForMem(InRedPrivs[Cnt]->getType()),
5060 PrivateVD->setInitStyle(VarDecl::CInit);
5061 PrivateVD->setInit(ImplicitCastExpr::Create(C, ElemType, CK_LValueToRValue,
5076 QualType SharedsTy = getContext().getRecordType(CS->getCapturedRecordDecl());
5077 auto I = CS->getCapturedDecl()->param_begin();
5085 auto IRef = C->varlist_begin();
5086 auto IElemInitRef = C->inits().begin();
5087 for (auto *IInit : C->private_copies()) {
5098 Data.ReductionVars.append(C->varlist_begin(), C->varlist_end());
5099 Data.ReductionOrigs.append(C->varlist_begin(), C->varlist_end());
5100 Data.ReductionCopies.append(C->privates().begin(), C->privates().end());
5101 Data.ReductionOps.append(C->reduction_ops().begin(),
5102 C->reduction_ops().end());
5103 LHSs.append(C->lhs_exprs().begin(), C->lhs_exprs().end());
5104 RHSs.append(C->rhs_exprs().begin(), C->rhs_exprs().end());
5131 // If there is no user-defined mapper, the mapper array will be nullptr. In
5149 CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(CopyFnParam)));
5151 CS->getCapturedDecl()->getParam(PrivatesParam)));
5157 ParamTypes.push_back(PrivatesPtr->getType());
5159 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
5161 CGF.CreateMemTemp(CGF.getContext().getPointerType(E->getType()),
5174 CGF.ConvertTypeForMem(Pair.first->getType().getNonReferenceType()),
5202 CGF, TL->getThreadLimit(), S.getBeginLoc());
5228 CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(4)));
5243 Data.ReductionCopies[Cnt]->getType()),
5244 Data.ReductionCopies[Cnt]->getExprLoc()),
5245 CGF.ConvertTypeForMem(Data.ReductionCopies[Cnt]->getType()),
5257 auto IPriv = C->privates().begin();
5258 auto IRed = C->reduction_ops().begin();
5259 auto ITD = C->taskgroup_descriptors().begin();
5260 for (const Expr *Ref : C->varlists()) {
5284 CGF.EmitLoadOfScalar(CGF.EmitLValue(TRExpr), TRExpr->getExprLoc());
5293 CGF.getContext().getPointerType(InRedPrivs[Cnt]->getType()),
5294 InRedPrivs[Cnt]->getExprLoc()),
5295 CGF.ConvertTypeForMem(InRedPrivs[Cnt]->getType()),
5308 QualType SharedsTy = getContext().getRecordType(CS->getCapturedRecordDecl());
5311 if (C->getNameModifier() == OMPD_unknown ||
5312 C->getNameModifier() == OMPD_task) {
5313 IfCond = C->getCondition();
5322 CGF.EmitStmt(CS->getCapturedStmt());
5343 Expr *ME = MC ? MC->getMessageString() : nullptr;
5346 if (!SC || SC->getSeverityKind() == OMPC_SEVERITY_fatal)
5373 InsertPointTy AllocaIP(AllocaInsertPt->getParent(),
5374 AllocaInsertPt->getIterator());
5379 EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt());
5394 Data.ReductionVars.append(C->varlist_begin(), C->varlist_end());
5395 Data.ReductionOrigs.append(C->varlist_begin(), C->varlist_end());
5396 Data.ReductionCopies.append(C->privates().begin(), C->privates().end());
5397 Data.ReductionOps.append(C->reduction_ops().begin(),
5398 C->reduction_ops().end());
5399 LHSs.append(C->lhs_exprs().begin(), C->lhs_exprs().end());
5400 RHSs.append(C->rhs_exprs().begin(), C->rhs_exprs().end());
5405 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
5408 /*Volatile=*/false, E->getType());
5410 CGF.EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt());
5421 [&S]() -> ArrayRef<const Expr *> {
5423 return llvm::ArrayRef(FlushClause->varlist_begin(),
5424 FlushClause->varlist_end());
5432 LValue DOLVal = EmitLValue(DO->getDepobj());
5433 if (const auto *DC = S.getSingleClause<OMPDependClause>()) {
5434 OMPTaskDataTy::DependData Dependencies(DC->getDependencyKind(),
5435 DC->getModifier());
5436 Dependencies.DepExprs.append(DC->varlist_begin(), DC->varlist_end());
5438 *this, Dependencies, DC->getBeginLoc());
5442 if (const auto *DC = S.getSingleClause<OMPDestroyClause>()) {
5443 CGM.getOpenMPRuntime().emitDestroyClause(*this, DOLVal, DC->getBeginLoc());
5448 *this, DOLVal, UC->getDependencyKind(), UC->getBeginLoc());
5467 if (C->getModifier() != OMPC_REDUCTION_inscan)
5469 Shareds.append(C->varlist_begin(), C->varlist_end());
5470 Privates.append(C->privates().begin(), C->privates().end());
5471 LHSs.append(C->lhs_exprs().begin(), C->lhs_exprs().end());
5472 RHSs.append(C->rhs_exprs().begin(), C->rhs_exprs().end());
5473 ReductionOps.append(C->reduction_ops().begin(), C->reduction_ops().end());
5474 CopyOps.append(C->copy_ops().begin(), C->copy_ops().end());
5475 CopyArrayTemps.append(C->copy_array_temps().begin(),
5476 C->copy_array_temps().end());
5477 CopyArrayElems.append(C->copy_array_elems().begin(),
5478 C->copy_array_elems().end());
5483 // For simd directive and simd-based directives in simd only mode, use the
5537 *cast<VarDecl>(cast<DeclRefExpr>(TempExpr)->getDecl()));
5540 EmitOMPCopy(PrivateExpr->getType(), DestLVal.getAddress(),
5542 cast<VarDecl>(cast<DeclRefExpr>(LHSs[I])->getDecl()),
5543 cast<VarDecl>(cast<DeclRefExpr>(RHSs[I])->getDecl()),
5563 PrivateExpr->getType(), DestLVal.getAddress(), SrcLVal.getAddress(),
5564 cast<VarDecl>(cast<DeclRefExpr>(LHSs[I])->getDecl()),
5565 cast<VarDecl>(cast<DeclRefExpr>(RHSs[I])->getDecl()), CopyOps[I]);
5580 // Emit buffer[i] = red; at the end of the input phase.
5583 ->IgnoreParenImpCasts();
5585 llvm::Value *IdxVal = EmitLoadOfScalar(IdxLVal, IVExpr->getExprLoc());
5594 cast<ArraySubscriptExpr>(CopyArrayElem)->getIdx()),
5599 PrivateExpr->getType(), DestLVal.getAddress(), SrcLVal.getAddress(),
5600 cast<VarDecl>(cast<DeclRefExpr>(LHSs[I])->getDecl()),
5601 cast<VarDecl>(cast<DeclRefExpr>(RHSs[I])->getDecl()), CopyOps[I]);
5611 // Emit red = buffer[i]; at the entrance to the scan phase.
5614 ->IgnoreParenImpCasts();
5616 llvm::Value *IdxVal = EmitLoadOfScalar(IdxLVal, IVExpr->getExprLoc());
5625 // Use idx - 1 iteration for exclusive scan.
5635 cast<ArraySubscriptExpr>(CopyArrayElem)->getIdx()),
5640 PrivateExpr->getType(), DestLVal.getAddress(), SrcLVal.getAddress(),
5641 cast<VarDecl>(cast<DeclRefExpr>(LHSs[I])->getDecl()),
5642 cast<VarDecl>(cast<DeclRefExpr>(RHSs[I])->getDecl()), CopyOps[I]);
5658 const auto *IVDecl = cast<VarDecl>(IVExpr->getDecl());
5665 EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl()));
5673 // Check pre-condition.
5716 // on initialization of firstprivate variables and post-update of
5737 ScheduleKind = C->getDistScheduleKind();
5738 if (const Expr *Ch = C->getChunkSize()) {
5740 Chunk = EmitScalarConversion(Chunk, Ch->getType(),
5741 S.getIterationVariable()->getType(),
5749 const unsigned IVSize = getContext().getTypeSize(IVExpr->getType());
5750 const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();
5755 // assigned to the teams of the league in a round-robin fashion in the
5859 // Emit post-update of the reduction variables if IsLastIter != 0.
5898 Fn->setDoesNotRecurse();
5907 unsigned NumLoops = C->getNumLoops();
5912 const Expr *CounterVal = C->getLoopData(I);
5915 CGF.EmitScalarExpr(CounterVal), CounterVal->getType(), Int64Ty,
5916 CounterVal->getExprLoc());
5936 InsertPointTy AllocaIP(AllocaInsertPt->getParent(),
5937 AllocaInsertPt->getIterator());
5938 for (const auto *DC : S.getClausesOfKind<OMPDependClause>())
5939 emitRestoreIP(*this, DC, AllocaIP, OMPBuilder);
5940 for (const auto *DC : S.getClausesOfKind<OMPDoacrossClause>())
5941 emitRestoreIP(*this, DC, AllocaIP, OMPBuilder);
5970 *this, CS->getCapturedStmt(), AllocaIP, CodeGenIP, "ordered");
5984 for (const auto *DC : S.getClausesOfKind<OMPDependClause>())
5985 CGM.getOpenMPRuntime().emitDoacrossOrdered(*this, DC);
5991 for (const auto *DC : S.getClausesOfKind<OMPDoacrossClause>())
5992 CGM.getOpenMPRuntime().emitDoacrossOrdered(*this, DC);
6008 CGF.EmitStmt(CS->getCapturedStmt());
6036 DestType->castAs<ComplexType>()->getElementType();
6040 ScalarVal, llvm::Constant::getNullValue(ScalarVal->getType()));
6043 QualType SrcElementType = SrcType->castAs<ComplexType>()->getElementType();
6045 DestType->castAs<ComplexType>()->getElementType();
6094 assert(V->isLValue() && "V of 'omp atomic read' is not lvalue");
6095 assert(X->isLValue() && "X of 'omp atomic read' is not lvalue");
6117 CGF.emitOMPSimpleStore(VLValue, Res, X->getType().getNonReferenceType(), Loc);
6125 assert(X->isLValue() && "X of 'omp atomic write' is not lvalue");
6159 (Update.getScalarVal()->getType() != X.getAddress().getElementType())) ||
6165 if (T->isIntegerTy())
6168 if (T->isFloatingPointTy() && (BO == BO_Add || BO == BO_Sub))
6174 if (!CheckAtomicSupport(Update.getScalarVal()->getType(), BO) ||
6178 bool IsInteger = X.getAddress().getElementType()->isIntegerTy();
6200 RMWOp = X.getType()->hasSignedIntegerRepresentation()
6211 RMWOp = X.getType()->hasSignedIntegerRepresentation()
6256 X.getType()->hasSignedIntegerRepresentation());
6271 // x binop= expr; -> xrval + expr;
6272 // x++, ++x -> xrval + 1;
6273 // x--, --x -> xrval - 1;
6274 // x = x binop expr; -> xrval binop expr
6275 // x = expr Op x; - > expr binop xrval;
6283 // Perform compare-and-swap procedure.
6294 assert(isa<BinaryOperator>(UE->IgnoreImpCasts()) &&
6296 const auto *BOUE = cast<BinaryOperator>(UE->IgnoreImpCasts());
6298 // x binop= expr; -> xrval + expr;
6299 // x++, ++x -> xrval + 1;
6300 // x--, --x -> xrval - 1;
6301 // x = x binop expr; -> xrval binop expr
6302 // x = expr Op x; - > expr binop xrval;
6303 assert(X->isLValue() && "X of 'omp atomic update' is not lvalue");
6306 const auto *LHS = cast<OpaqueValueExpr>(BOUE->getLHS()->IgnoreImpCasts());
6307 const auto *RHS = cast<OpaqueValueExpr>(BOUE->getRHS()->IgnoreImpCasts());
6316 XLValue, ExprRValue, BOUE->getOpcode(), IsXLHSInRHSPart, AO, Loc, Gen);
6361 assert(X->isLValue() && "X of 'omp atomic capture' is not lvalue");
6362 assert(V->isLValue() && "V of 'omp atomic capture' is not lvalue");
6370 assert(isa<BinaryOperator>(UE->IgnoreImpCasts()) &&
6372 const auto *BOUE = cast<BinaryOperator>(UE->IgnoreImpCasts());
6374 // x binop= expr; -> xrval + expr;
6375 // x++, ++x -> xrval + 1;
6376 // x--, --x -> xrval - 1;
6377 // x = x binop expr; -> xrval binop expr
6378 // x = expr Op x; - > expr binop xrval;
6379 const auto *LHS = cast<OpaqueValueExpr>(BOUE->getLHS()->IgnoreImpCasts());
6380 const auto *RHS = cast<OpaqueValueExpr>(BOUE->getRHS()->IgnoreImpCasts());
6382 NewVValType = XRValExpr->getType();
6393 XLValue, ExprRValue, BOUE->getOpcode(), IsXLHSInRHSPart, AO, Loc, Gen);
6410 NewVValType = X->getType().getNonReferenceType();
6411 ExprRValue = convertToType(CGF, ExprRValue, E->getType(),
6412 X->getType().getNonReferenceType(), Loc);
6427 // Emit post-update store to 'v' of old/new 'x' value.
6472 switch (cast<BinaryOperator>(CE)->getOpcode()) {
6490 if (X->getType() == E->getType())
6492 const Expr *NewE = E->IgnoreImplicitAsWritten();
6494 if (NewE->getType() == X->getType())
6496 return CGF.EmitScalarConversion(V, NewE->getType(), X->getType(), Loc);
6504 E->getType()->hasSignedIntegerRepresentation());
6509 D->getType()->hasSignedIntegerRepresentation());
6513 X->getType()->hasSignedIntegerRepresentation(),
6514 X->getType().isVolatileQualified()};
6520 V->getType()->hasSignedIntegerRepresentation(),
6521 V->getType().isVolatileQualified()};
6527 R->getType()->hasSignedIntegerRepresentation(),
6528 R->getType().isVolatileQualified()};
6601 OpenMPClauseKind K = C->getClauseKind();
6640 OpenMPClauseKind FailParameter = FailClause->getFailParameter();
6669 CGF.EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt());
6681 if (C->getNameModifier() == OMPD_unknown ||
6682 C->getNameModifier() == OMPD_target) {
6683 IfCond = C->getCondition();
6692 Device.setPointerAndInt(C->getDevice(), C->getModifier());
6731 const OMPLoopDirective &D) -> llvm::Value * {
6756 CGF.EmitStmt(S.getCapturedStmt(OMPD_target)->getCapturedStmt());
6788 CGF, S, *CS->getCapturedDecl()->param_begin(), InnermostKind,
6794 const Expr *NumTeams = NT ? NT->getNumTeams() : nullptr;
6795 const Expr *ThreadLimit = TL ? TL->getThreadLimit() : nullptr;
6817 CGF.EmitStmt(S.getCapturedStmt(OMPD_teams)->getCapturedStmt());
6839 CGF.EmitStmt(CS->getCapturedStmt());
7060 Device = EmitScalarExpr(C->getDevice());
7084 EmitLValue(C->getInteropVar()).getPointer(*this);
7087 if (C->getIsTarget()) {
7090 assert(C->getIsTargetSync() &&
7091 "Expected interop-type target/targetsync");
7104 EmitLValue(C->getInteropVar()).getPointer(*this);
7115 EmitLValue(C->getInteropVar()).getPointer(*this);
7235 if (C->getNameModifier() == OMPD_unknown ||
7236 C->getNameModifier() == OMPD_cancel) {
7237 IfCond = C->getCondition();
7282 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(OrigVarIt)->getDecl());
7293 const auto *ME = cast<MemberExpr>(OED->getInit());
7294 assert(isa<CXXThisExpr>(ME->getBase()->IgnoreImpCasts()) &&
7296 MatchingVD = ME->getMemberDecl();
7305 llvm::Type *Ty = ConvertTypeForMem(OrigVD->getType().getNonReferenceType());
7310 Address(InitAddrIt->second, Ty,
7319 const Expr *Base = Ref->IgnoreParenImpCasts();
7321 Base = OASE->getBase()->IgnoreParenImpCasts();
7323 Base = ASE->getBase()->IgnoreParenImpCasts();
7324 return cast<VarDecl>(cast<DeclRefExpr>(Base)->getDecl());
7343 const auto *ME = cast<MemberExpr>(OED->getInit());
7344 assert(isa<CXXThisExpr>(ME->getBase()) &&
7346 MatchingVD = ME->getMemberDecl();
7355 llvm::Type *Ty = ConvertTypeForMem(OrigVD->getType().getNonReferenceType());
7358 Address(InitAddrIt->second, Ty,
7362 if (isa<DeclRefExpr>(Ref->IgnoreParenImpCasts()) ||
7363 MatchingVD->getType()->isArrayType()) {
7365 OrigVD->getType().getNonReferenceType());
7368 PtrTy->castAs<PointerType>());
7399 CGF.EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt());
7407 // Call the pre-action to change the status of PrivatizeDevicePointers if
7429 for (const Expr *E : C->varlists()) {
7430 const Decl *D = cast<DeclRefExpr>(E)->getDecl();
7435 for (const Expr *E : C->varlists()) {
7473 IfCond = C->getCondition();
7478 Device = C->getDevice();
7498 IfCond = C->getCondition();
7503 Device = C->getDevice();
7519 IfCond = C->getCondition();
7524 Device = C->getDevice();
7546 CGF.EmitStmt(CS->getCapturedStmt());
7660 const auto *VDecl = cast<VarDecl>(Helper->getDecl());
7673 QualType SharedsTy = getContext().getRecordType(CS->getCapturedRecordDecl());
7676 if (C->getNameModifier() == OMPD_unknown ||
7677 C->getNameModifier() == OMPD_taskloop) {
7678 IfCond = C->getCondition();
7692 Data.Schedule.setPointer(EmitScalarExpr(Clause->getGrainsize()));
7696 Data.Schedule.setPointer(EmitScalarExpr(Clause->getNumTasks()));
7706 // Emit: if (PreCond) - begin.
7729 auto *I = CS->getCapturedDecl()->param_begin();
7747 const auto *IVDecl = cast<VarDecl>(cast<DeclRefExpr>(IVExpr)->getDecl());
7755 CGF.EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl()));
7778 // Emit: if (PreCond) - end.
7789 (*LIP)->getType(), S.getBeginLoc())));
7795 (*LIP)->getType(), S.getBeginLoc()));
7911 IfCond = C->getCondition();
7916 Device = C->getDevice();
7928 cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt();
7930 if (ForS && !isa<DeclStmt>(ForS->getInit())) {
8149 for (const Expr *Ref : C->varlists()) {
8150 const auto *DRE = cast<DeclRefExpr>(Ref->IgnoreParenImpCasts());
8153 const auto *VD = dyn_cast<VarDecl>(DRE->getDecl());
8154 if (!VD || VD->hasLocalStorage())
8169 for (const Expr *E : LD->counters()) {
8170 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
8171 if (!VD->hasLocalStorage() && !CGF.LocalDeclMap.count(VD)) {
8182 if (!C->getNumForLoops())
8184 for (unsigned I = LD->getLoopsNumber(),
8185 E = C->getLoopNumIterations().size();
8188 cast<DeclRefExpr>(C->getLoopCounter(I))->getDecl())) {
8197 CGF.EmitStmt(D.getInnermostCapturedStmt()->getCapturedStmt());