Lines Matching defs:CGF
85 void EmitBody(CodeGenFunction &CGF, const Stmt *S) override;
89 virtual LValue getThreadIDVariableLValue(CodeGenFunction &CGF);
91 virtual void emitUntiedSwitch(CodeGenFunction & /*CGF*/) {}
158 void Enter(CodeGenFunction &CGF) override {
161 LValue PartIdLVal = CGF.EmitLoadOfPointerLValue(
162 CGF.GetAddrOfLocalVar(PartIDVar),
165 CGF.EmitLoadOfScalar(PartIdLVal, PartIDVar->getLocation());
166 llvm::BasicBlock *DoneBB = CGF.createBasicBlock(".untied.done.");
167 UntiedSwitch = CGF.Builder.CreateSwitch(Res, DoneBB);
168 CGF.EmitBlock(DoneBB);
169 CGF.EmitBranchThroughCleanup(CGF.ReturnBlock);
170 CGF.EmitBlock(CGF.createBasicBlock(".untied.jmp."));
171 UntiedSwitch->addCase(CGF.Builder.getInt32(0),
172 CGF.Builder.GetInsertBlock());
173 emitUntiedSwitch(CGF);
176 void emitUntiedSwitch(CodeGenFunction &CGF) const {
178 LValue PartIdLVal = CGF.EmitLoadOfPointerLValue(
179 CGF.GetAddrOfLocalVar(PartIDVar),
181 CGF.EmitStoreOfScalar(CGF.Builder.getInt32(UntiedSwitch->getNumCases()),
183 UntiedCodeGen(CGF);
185 CGF.getJumpDestInCurrentScope(".untied.next.");
186 CGF.EmitBranch(CGF.ReturnBlock.getBlock());
187 CGF.EmitBlock(CGF.createBasicBlock(".untied.jmp."));
188 UntiedSwitch->addCase(CGF.Builder.getInt32(UntiedSwitch->getNumCases()),
189 CGF.Builder.GetInsertBlock());
190 CGF.EmitBranchThroughCleanup(CurPoint);
191 CGF.EmitBlock(CurPoint.getBlock());
211 LValue getThreadIDVariableLValue(CodeGenFunction &CGF) override;
216 void emitUntiedSwitch(CodeGenFunction &CGF) override {
217 Action.emitUntiedSwitch(CGF);
284 LValue getThreadIDVariableLValue(CodeGenFunction &CGF) override {
286 return OuterRegionInfo->getThreadIDVariableLValue(CGF);
297 void emitUntiedSwitch(CodeGenFunction &CGF) override {
299 OuterRegionInfo->emitUntiedSwitch(CGF);
353 CGOpenMPInnerExprInfo(CodeGenFunction &CGF, const CapturedStmt &CS)
354 : CGOpenMPInlinedRegionInfo(CGF.CapturedStmtInfo, EmptyCodeGen,
357 PrivScope(CGF) {
369 DeclRefExpr DRE(CGF.getContext(), const_cast<VarDecl *>(VD),
373 PrivScope.addPrivate(VD, CGF.EmitLValue(&DRE).getAddress());
386 void EmitBody(CodeGenFunction &CGF, const Stmt *S) override {
410 CodeGenFunction &CGF;
421 InlinedOpenMPRegionRAII(CodeGenFunction &CGF, const RegionCodeGenTy &CodeGen,
424 : CGF(CGF), NoInheritance(NoInheritance) {
426 CGF.CapturedStmtInfo = new CGOpenMPInlinedRegionInfo(
427 CGF.CapturedStmtInfo, CodeGen, Kind, HasCancel);
429 std::swap(CGF.LambdaCaptureFields, LambdaCaptureFields);
430 LambdaThisCaptureField = CGF.LambdaThisCaptureField;
431 CGF.LambdaThisCaptureField = nullptr;
432 BlockInfo = CGF.BlockInfo;
433 CGF.BlockInfo = nullptr;
440 cast<CGOpenMPInlinedRegionInfo>(CGF.CapturedStmtInfo)->getOldCSI();
441 delete CGF.CapturedStmtInfo;
442 CGF.CapturedStmtInfo = OldCSI;
444 std::swap(CGF.LambdaCaptureFields, LambdaCaptureFields);
445 CGF.LambdaThisCaptureField = LambdaThisCaptureField;
446 CGF.BlockInfo = BlockInfo;
560 void Emit(CodeGenFunction &CGF, Flags /*flags*/) override {
561 if (!CGF.HaveInsertPoint())
563 Action->Exit(CGF);
569 void RegionCodeGenTy::operator()(CodeGenFunction &CGF) const {
570 CodeGenFunction::RunCleanupsScope Scope(CGF);
572 CGF.EHStack.pushCleanup<CleanupTy>(NormalAndEHCleanup, PrePostAction);
573 Callback(CodeGen, CGF, *PrePostAction);
576 Callback(CodeGen, CGF, Action);
593 static void emitInitWithReductionInitializer(CodeGenFunction &CGF,
600 CGF.CGM.getOpenMPRuntime().getUserDefinedReduction(DRD);
609 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
614 CodeGenFunction::OpaqueValueMapping Map(CGF, OVE, Func);
615 CGF.EmitIgnoredExpr(InitOp);
617 llvm::Constant *Init = CGF.CGM.EmitNullConstant(Ty);
618 std::string Name = CGF.CGM.getOpenMPRuntime().getName({"init"});
620 CGF.CGM.getModule(), Init->getType(), /*isConstant=*/true,
622 LValue LV = CGF.MakeNaturalAlignRawAddrLValue(GV, Ty);
624 switch (CGF.getEvaluationKind(Ty)) {
626 InitRVal = CGF.EmitLoadOfLValue(LV, DRD->getLocation());
630 RValue::getComplex(CGF.EmitLoadOfComplex(LV, DRD->getLocation()));
634 CodeGenFunction::OpaqueValueMapping OpaqueMap(CGF, &OVE, LV);
635 CGF.EmitAnyExprToMem(&OVE, Private, Ty.getQualifiers(),
641 CodeGenFunction::OpaqueValueMapping OpaqueMap(CGF, &OVE, InitRVal);
642 CGF.EmitAnyExprToMem(&OVE, Private, Ty.getQualifiers(),
652 static void EmitOMPAggregateInit(CodeGenFunction &CGF, Address DestAddr,
662 llvm::Value *NumElements = CGF.emitArrayLength(ArrayTy, ElementTy, DestAddr);
668 SrcBegin = SrcAddr.emitRawPointer(CGF);
669 llvm::Value *DestBegin = DestAddr.emitRawPointer(CGF);
672 CGF.Builder.CreateGEP(DestAddr.getElementType(), DestBegin, NumElements);
674 llvm::BasicBlock *BodyBB = CGF.createBasicBlock("omp.arrayinit.body");
675 llvm::BasicBlock *DoneBB = CGF.createBasicBlock("omp.arrayinit.done");
677 CGF.Builder.CreateICmpEQ(DestBegin, DestEnd, "omp.arrayinit.isempty");
678 CGF.Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB);
681 llvm::BasicBlock *EntryBB = CGF.Builder.GetInsertBlock();
682 CGF.EmitBlock(BodyBB);
684 CharUnits ElementSize = CGF.getContext().getTypeSizeInChars(ElementTy);
689 SrcElementPHI = CGF.Builder.CreatePHI(SrcBegin->getType(), 2,
696 llvm::PHINode *DestElementPHI = CGF.Builder.CreatePHI(
705 CodeGenFunction::RunCleanupsScope InitScope(CGF);
707 emitInitWithReductionInitializer(CGF, DRD, Init, DestElementCurrent,
710 CGF.EmitAnyExprToMem(Init, DestElementCurrent, ElementTy.getQualifiers(),
716 llvm::Value *SrcElementNext = CGF.Builder.CreateConstGEP1_32(
719 SrcElementPHI->addIncoming(SrcElementNext, CGF.Builder.GetInsertBlock());
723 llvm::Value *DestElementNext = CGF.Builder.CreateConstGEP1_32(
728 CGF.Builder.CreateICmpEQ(DestElementNext, DestEnd, "omp.arraycpy.done");
729 CGF.Builder.CreateCondBr(Done, DoneBB, BodyBB);
730 DestElementPHI->addIncoming(DestElementNext, CGF.Builder.GetInsertBlock());
733 CGF.EmitBlock(DoneBB, /*IsFinished=*/true);
736 LValue ReductionCodeGen::emitSharedLValue(CodeGenFunction &CGF, const Expr *E) {
737 return CGF.EmitOMPSharedLValue(E);
740 LValue ReductionCodeGen::emitSharedLValueUB(CodeGenFunction &CGF,
743 return CGF.EmitArraySectionExpr(OASE, /*IsLowerBound=*/false);
748 CodeGenFunction &CGF, unsigned N, Address PrivateAddr, Address SharedAddr,
757 EmitOMPAggregateInit(CGF, PrivateAddr, PrivateVD->getType(),
783 void ReductionCodeGen::emitSharedOrigLValue(CodeGenFunction &CGF, unsigned N) {
786 LValue First = emitSharedLValue(CGF, ClausesData[N].Shared);
787 LValue Second = emitSharedLValueUB(CGF, ClausesData[N].Shared);
792 LValue First = emitSharedLValue(CGF, ClausesData[N].Ref);
793 LValue Second = emitSharedLValueUB(CGF, ClausesData[N].Ref);
798 void ReductionCodeGen::emitAggregateType(CodeGenFunction &CGF, unsigned N) {
803 CGF.getTypeSize(OrigAddresses[N].first.getType().getNonReferenceType()),
812 Size = CGF.Builder.CreatePtrDiff(ElemType,
813 OrigAddresses[N].second.getPointer(CGF),
814 OrigAddresses[N].first.getPointer(CGF));
815 Size = CGF.Builder.CreateNUWAdd(
817 SizeInChars = CGF.Builder.CreateNUWMul(Size, ElemSizeOf);
820 CGF.getTypeSize(OrigAddresses[N].first.getType().getNonReferenceType());
821 Size = CGF.Builder.CreateExactUDiv(SizeInChars, ElemSizeOf);
825 CGF,
827 CGF.getContext().getAsVariableArrayType(PrivateType)->getSizeExpr()),
829 CGF.EmitVariablyModifiedType(PrivateType);
832 void ReductionCodeGen::emitAggregateType(CodeGenFunction &CGF, unsigned N,
842 CGF,
844 CGF.getContext().getAsVariableArrayType(PrivateType)->getSizeExpr()),
846 CGF.EmitVariablyModifiedType(PrivateType);
850 CodeGenFunction &CGF, unsigned N, Address PrivateAddr, Address SharedAddr,
857 if (CGF.getContext().getAsArrayType(PrivateVD->getType())) {
859 (void)DefaultInit(CGF);
860 emitAggregateInitialization(CGF, N, PrivateAddr, SharedAddr, DRD);
862 (void)DefaultInit(CGF);
864 emitInitWithReductionInitializer(CGF, DRD, ClausesData[N].ReductionOp,
866 } else if (!DefaultInit(CGF) && PrivateVD->hasInit() &&
867 !CGF.isTrivialInitializer(PrivateVD->getInit())) {
868 CGF.EmitAnyExprToMem(PrivateVD->getInit(), PrivateAddr,
880 void ReductionCodeGen::emitCleanups(CodeGenFunction &CGF, unsigned N,
886 PrivateAddr.withElementType(CGF.ConvertTypeForMem(PrivateType));
887 CGF.pushDestroy(DTorKind, PrivateAddr, PrivateType);
891 static LValue loadToBegin(CodeGenFunction &CGF, QualType BaseTy, QualType ElTy,
895 !CGF.getContext().hasSameType(BaseTy, ElTy)) {
897 BaseLV = CGF.EmitLoadOfPointerLValue(BaseLV.getAddress(), PtrTy);
899 LValue RefLVal = CGF.MakeAddrLValue(BaseLV.getAddress(), BaseTy);
900 BaseLV = CGF.EmitLoadOfReferenceLValue(RefLVal);
904 return CGF.MakeAddrLValue(
905 BaseLV.getAddress().withElementType(CGF.ConvertTypeForMem(ElTy)),
907 CGF.CGM.getTBAAInfoForSubobject(BaseLV, BaseLV.getType()));
910 static Address castToBase(CodeGenFunction &CGF, QualType BaseTy, QualType ElTy,
917 !CGF.getContext().hasSameType(BaseTy, ElTy)) {
918 Tmp = CGF.CreateMemTemp(BaseTy);
920 CGF.Builder.CreateStore(Tmp.getPointer(), TopTmp);
928 Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
930 CGF.Builder.CreateStore(Addr, Tmp);
934 Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
959 Address ReductionCodeGen::adjustPrivateAddress(CodeGenFunction &CGF, unsigned N,
964 LValue OriginalBaseLValue = CGF.EmitLValue(DE);
966 loadToBegin(CGF, OrigVD->getType(), SharedAddresses[N].first.getType(),
969 llvm::Value *Adjustment = CGF.Builder.CreatePtrDiff(
970 SharedAddr.getElementType(), BaseLValue.getPointer(CGF),
971 SharedAddr.emitRawPointer(CGF));
973 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
974 PrivateAddr.emitRawPointer(CGF), SharedAddr.getType());
975 llvm::Value *Ptr = CGF.Builder.CreateGEP(
977 return castToBase(CGF, OrigVD->getType(),
992 LValue CGOpenMPRegionInfo::getThreadIDVariableLValue(CodeGenFunction &CGF) {
993 return CGF.EmitLoadOfPointerLValue(
994 CGF.GetAddrOfLocalVar(getThreadIDVariable()),
998 void CGOpenMPRegionInfo::EmitBody(CodeGenFunction &CGF, const Stmt *S) {
999 if (!CGF.HaveInsertPoint())
1006 CGF.EHStack.pushTerminate();
1008 CGF.incrementProfileCounter(S);
1009 CodeGen(CGF);
1010 CGF.EHStack.popTerminate();
1014 CodeGenFunction &CGF) {
1015 return CGF.MakeAddrLValue(CGF.GetAddrOfLocalVar(getThreadIDVariable()),
1099 CodeGenFunction CGF(CGM);
1102 CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, FnInfo, Args, In->getLocation(),
1104 CodeGenFunction::OMPPrivateScope Scope(CGF);
1105 Address AddrIn = CGF.GetAddrOfLocalVar(&OmpInParm);
1107 In, CGF.EmitLoadOfPointerLValue(AddrIn, PtrTy->castAs<PointerType>())
1109 Address AddrOut = CGF.GetAddrOfLocalVar(&OmpOutParm);
1111 Out, CGF.EmitLoadOfPointerLValue(AddrOut, PtrTy->castAs<PointerType>())
1115 !CGF.isTrivialInitializer(Out->getInit())) {
1116 CGF.EmitAnyExprToMem(Out->getInit(), CGF.GetAddrOfLocalVar(Out),
1121 CGF.EmitIgnoredExpr(CombinerInitializer);
1123 CGF.FinishFunction();
1128 CodeGenFunction *CGF, const OMPDeclareReductionDecl *D) {
1147 if (CGF)
1148 FunctionUDRMap[CGF->CurFn].push_back(D);
1156 emitUserDefinedReduction(/*CGF=*/nullptr, D);
1164 PushAndPopStackRAII(llvm::OpenMPIRBuilder *OMPBuilder, CodeGenFunction &CGF,
1182 auto FiniCB = [&CGF](llvm::OpenMPIRBuilder::InsertPointTy IP) {
1185 CGBuilderTy::InsertPointGuard IPG(CGF.Builder);
1186 CGF.Builder.restoreIP(IP);
1188 CGF.getOMPCancelDestination(OMPD_parallel);
1189 CGF.EmitBranchThroughCleanup(Dest);
1212 CodeGenFunction CGF(CGM, true);
1236 PushAndPopStackRAII PSR(&OMPBuilder, CGF, HasCancel, InnermostKind);
1239 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo);
1240 return CGF.GenerateOpenMPCapturedStmtFunction(*CS, D.getBeginLoc());
1248 std::string CGOpenMPRuntime::getOutlinedHelperName(CodeGenFunction &CGF) const {
1249 return getOutlinedHelperName(CGF.CurFn->getName());
1258 CodeGenFunction &CGF, const OMPExecutableDirective &D,
1263 CGM, D, CS, ThreadIDVar, InnermostKind, getOutlinedHelperName(CGF),
1268 CodeGenFunction &CGF, const OMPExecutableDirective &D,
1273 CGM, D, CS, ThreadIDVar, InnermostKind, getOutlinedHelperName(CGF),
1282 auto &&UntiedCodeGen = [this, &D, TaskTVar](CodeGenFunction &CGF,
1284 llvm::Value *ThreadID = getThreadID(CGF, D.getBeginLoc());
1285 llvm::Value *UpLoc = emitUpdateLocation(CGF, D.getBeginLoc());
1288 CGF.EmitLoadOfPointerLValue(CGF.GetAddrOfLocalVar(TaskTVar),
1290 .getPointer(CGF)};
1291 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
1314 CodeGenFunction CGF(CGM, true);
1317 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo);
1318 llvm::Function *Res = CGF.GenerateCapturedStmtFunction(*CS);
1324 void CGOpenMPRuntime::setLocThreadIdInsertPt(CodeGenFunction &CGF,
1326 auto &Elem = OpenMPLocThreadIDMap[CGF.CurFn];
1329 llvm::Value *Undef = llvm::UndefValue::get(CGF.Int32Ty);
1331 Elem.ServiceInsertPt = new llvm::BitCastInst(Undef, CGF.Int32Ty, "svcpt",
1332 CGF.Builder.GetInsertBlock());
1334 Elem.ServiceInsertPt = new llvm::BitCastInst(Undef, CGF.Int32Ty, "svcpt");
1335 Elem.ServiceInsertPt->insertAfter(CGF.AllocaInsertPt->getIterator());
1339 void CGOpenMPRuntime::clearLocThreadIdInsertPt(CodeGenFunction &CGF) {
1340 auto &Elem = OpenMPLocThreadIDMap[CGF.CurFn];
1348 static StringRef getIdentStringFromSourceLocation(CodeGenFunction &CGF,
1353 PresumedLoc PLoc = CGF.getContext().getSourceManager().getPresumedLoc(Loc);
1355 if (const auto *FD = dyn_cast_or_null<FunctionDecl>(CGF.CurFuncDecl))
1361 llvm::Value *CGOpenMPRuntime::emitUpdateLocation(CodeGenFunction &CGF,
1372 if (const auto *FD = dyn_cast_or_null<FunctionDecl>(CGF.CurFuncDecl))
1374 PresumedLoc PLoc = CGF.getContext().getSourceManager().getPresumedLoc(Loc);
1386 llvm::Value *CGOpenMPRuntime::getThreadID(CodeGenFunction &CGF,
1388 assert(CGF.CurFn && "No function in current CodeGenFunction.");
1393 OMPBuilder.updateToLocation(CGF.Builder.saveIP());
1396 getIdentStringFromSourceLocation(CGF, Loc, Buffer), SrcLocStrSize);
1404 auto I = OpenMPLocThreadIDMap.find(CGF.CurFn);
1412 dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo)) {
1415 LValue LVal = OMPRegionInfo->getThreadIDVariableLValue(CGF);
1416 llvm::BasicBlock *TopBlock = CGF.AllocaInsertPt->getParent();
1417 if (!CGF.EHStack.requiresLandingPad() || !CGF.getLangOpts().Exceptions ||
1418 !CGF.getLangOpts().CXXExceptions ||
1419 CGF.Builder.GetInsertBlock() == TopBlock ||
1420 !isa<llvm::Instruction>(LVal.getPointer(CGF)) ||
1421 cast<llvm::Instruction>(LVal.getPointer(CGF))->getParent() ==
1423 cast<llvm::Instruction>(LVal.getPointer(CGF))->getParent() ==
1424 CGF.Builder.GetInsertBlock()) {
1425 ThreadID = CGF.EmitLoadOfScalar(LVal, Loc);
1428 if (CGF.Builder.GetInsertBlock() == TopBlock)
1429 OpenMPLocThreadIDMap[CGF.CurFn].ThreadID = ThreadID;
1439 auto &Elem = OpenMPLocThreadIDMap[CGF.CurFn];
1441 setLocThreadIdInsertPt(CGF);
1442 CGBuilderTy::InsertPointGuard IPG(CGF.Builder);
1443 CGF.Builder.SetInsertPoint(Elem.ServiceInsertPt);
1444 auto DL = ApplyDebugLocation::CreateDefaultArtificial(CGF, Loc);
1445 llvm::CallInst *Call = CGF.Builder.CreateCall(
1448 emitUpdateLocation(CGF, Loc));
1449 Call->setCallingConv(CGF.getRuntimeCC());
1454 void CGOpenMPRuntime::functionFinished(CodeGenFunction &CGF) {
1455 assert(CGF.CurFn && "No function in current CodeGenFunction.");
1456 if (OpenMPLocThreadIDMap.count(CGF.CurFn)) {
1457 clearLocThreadIdInsertPt(CGF);
1458 OpenMPLocThreadIDMap.erase(CGF.CurFn);
1460 if (FunctionUDRMap.count(CGF.CurFn) > 0) {
1461 for(const auto *D : FunctionUDRMap[CGF.CurFn])
1463 FunctionUDRMap.erase(CGF.CurFn);
1465 auto I = FunctionUDMMap.find(CGF.CurFn);
1471 LastprivateConditionalToTypes.erase(CGF.CurFn);
1472 FunctionToUntiedTaskStackMap.erase(CGF.CurFn);
1589 Address CGOpenMPRuntime::getAddrOfThreadPrivate(CodeGenFunction &CGF,
1599 emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
1600 CGF.Builder.CreatePointerCast(VDAddr.emitRawPointer(CGF), CGM.Int8PtrTy),
1604 CGF.EmitRuntimeCall(
1608 CGF.Int8Ty, VDAddr.getAlignment());
1612 CodeGenFunction &CGF, Address VDAddr, llvm::Value *Ctor,
1616 llvm::Value *OMPLoc = emitUpdateLocation(CGF, Loc);
1617 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
1624 CGF.Builder.CreatePointerCast(VDAddr.emitRawPointer(CGF), CGM.VoidPtrTy),
1626 CGF.EmitRuntimeCall(
1634 bool PerformInit, CodeGenFunction *CGF) {
1722 if (!CGF) {
1737 emitThreadPrivateVarInit(*CGF, VDAddr, Ctor, CopyCtor, Dtor, Loc);
1777 Address CGOpenMPRuntime::getAddrOfArtificialThreadPrivate(CodeGenFunction &CGF,
1781 llvm::Type *VarLVType = CGF.ConvertTypeForMem(VarType);
1792 emitUpdateLocation(CGF, SourceLocation()),
1793 getThreadID(CGF, SourceLocation()),
1794 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(GAddr, CGM.VoidPtrTy),
1795 CGF.Builder.CreateIntCast(CGF.getTypeSize(VarType), CGM.SizeTy,
1801 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
1802 CGF.EmitRuntimeCall(
1806 CGF.Builder.getPtrTy(0)),
1810 void CGOpenMPRuntime::emitIfClause(CodeGenFunction &CGF, const Expr *Cond,
1813 CodeGenFunction::LexicalScope ConditionScope(CGF, Cond->getSourceRange());
1818 if (CGF.ConstantFoldsToSimpleInteger(Cond, CondConstant)) {
1820 ThenGen(CGF);
1822 ElseGen(CGF);
1828 llvm::BasicBlock *ThenBlock = CGF.createBasicBlock("omp_if.then");
1829 llvm::BasicBlock *ElseBlock = CGF.createBasicBlock("omp_if.else");
1830 llvm::BasicBlock *ContBlock = CGF.createBasicBlock("omp_if.end");
1831 CGF.EmitBranchOnBoolExpr(Cond, ThenBlock, ElseBlock, /*TrueCount=*/0);
1834 CGF.EmitBlock(ThenBlock);
1835 ThenGen(CGF);
1836 CGF.EmitBranch(ContBlock);
1839 (void)ApplyDebugLocation::CreateEmpty(CGF);
1840 CGF.EmitBlock(ElseBlock);
1841 ElseGen(CGF);
1843 (void)ApplyDebugLocation::CreateEmpty(CGF);
1844 CGF.EmitBranch(ContBlock);
1846 CGF.EmitBlock(ContBlock, /*IsFinished=*/true);
1849 void CGOpenMPRuntime::emitParallelCall(CodeGenFunction &CGF, SourceLocation Loc,
1854 if (!CGF.HaveInsertPoint())
1856 llvm::Value *RTLoc = emitUpdateLocation(CGF, Loc);
1859 this](CodeGenFunction &CGF, PrePostActionTy &) {
1861 CGOpenMPRuntime &RT = CGF.CGM.getOpenMPRuntime();
1864 CGF.Builder.getInt32(CapturedVars.size()), // Number of captured vars
1865 CGF.Builder.CreateBitCast(OutlinedFn, RT.getKmpc_MicroPointerTy())};
1872 CGF.EmitRuntimeCall(RTLFn, RealArgs);
1875 this](CodeGenFunction &CGF, PrePostActionTy &) {
1876 CGOpenMPRuntime &RT = CGF.CGM.getOpenMPRuntime();
1877 llvm::Value *ThreadID = RT.getThreadID(CGF, Loc);
1881 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
1886 Address ThreadIDAddr = RT.emitThreadIDAddress(CGF, Loc);
1888 CGF.CreateDefaultAlignTempAlloca(CGF.Int32Ty,
1890 CGF.Builder.CreateStore(CGF.Builder.getInt32(/*C*/ 0), ZeroAddrBound);
1893 OutlinedFnArgs.push_back(ThreadIDAddr.emitRawPointer(CGF));
1905 RT.emitOutlinedFunctionCall(CGF, Loc, OutlinedFn, OutlinedFnArgs);
1908 llvm::Value *EndArgs[] = {RT.emitUpdateLocation(CGF, Loc), ThreadID};
1909 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
1914 emitIfClause(CGF, IfCond, ThenGen, ElseGen);
1917 ThenRCG(CGF);
1927 Address CGOpenMPRuntime::emitThreadIDAddress(CodeGenFunction &CGF,
1930 dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo))
1932 return OMPRegionInfo->getThreadIDVariableLValue(CGF).getAddress();
1934 llvm::Value *ThreadID = getThreadID(CGF, Loc);
1936 CGF.getContext().getIntTypeForBitwidth(/*DestWidth*/ 32, /*Signed*/ true);
1937 Address ThreadIDTemp = CGF.CreateMemTemp(Int32Ty, /*Name*/ ".threadid_temp.");
1938 CGF.EmitStoreOfScalar(ThreadID,
1939 CGF.MakeAddrLValue(ThreadIDTemp, Int32Ty));
1967 void Enter(CodeGenFunction &CGF) override {
1968 llvm::Value *EnterRes = CGF.EmitRuntimeCall(EnterCallee, EnterArgs);
1970 llvm::Value *CallBool = CGF.Builder.CreateIsNotNull(EnterRes);
1971 auto *ThenBlock = CGF.createBasicBlock("omp_if.then");
1972 ContBlock = CGF.createBasicBlock("omp_if.end");
1974 CGF.Builder.CreateCondBr(CallBool, ThenBlock, ContBlock);
1975 CGF.EmitBlock(ThenBlock);
1978 void Done(CodeGenFunction &CGF) {
1980 CGF.EmitBranch(ContBlock);
1981 CGF.EmitBlock(ContBlock, true);
1983 void Exit(CodeGenFunction &CGF) override {
1984 CGF.EmitRuntimeCall(ExitCallee, ExitArgs);
1989 void CGOpenMPRuntime::emitCriticalRegion(CodeGenFunction &CGF,
1997 if (!CGF.HaveInsertPoint())
1999 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
2004 EnterArgs.push_back(CGF.Builder.CreateIntCast(
2005 CGF.EmitScalarExpr(Hint), CGM.Int32Ty, /*isSigned=*/false));
2016 emitInlinedDirective(CGF, OMPD_critical, CriticalOpGen);
2019 void CGOpenMPRuntime::emitMasterRegion(CodeGenFunction &CGF,
2022 if (!CGF.HaveInsertPoint())
2029 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
2038 emitInlinedDirective(CGF, OMPD_master, MasterOpGen);
2039 Action.Done(CGF);
2042 void CGOpenMPRuntime::emitMaskedRegion(CodeGenFunction &CGF,
2045 if (!CGF.HaveInsertPoint())
2053 ? CGF.EmitScalarExpr(Filter, CGF.Int32Ty)
2055 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
2057 llvm::Value *ArgsEnd[] = {emitUpdateLocation(CGF, Loc),
2058 getThreadID(CGF, Loc)};
2067 emitInlinedDirective(CGF, OMPD_masked, MaskedOpGen);
2068 Action.Done(CGF);
2071 void CGOpenMPRuntime::emitTaskyieldCall(CodeGenFunction &CGF,
2073 if (!CGF.HaveInsertPoint())
2075 if (CGF.CGM.getLangOpts().OpenMPIRBuilder) {
2076 OMPBuilder.createTaskyield(CGF.Builder);
2080 emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
2082 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
2087 if (auto *Region = dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo))
2088 Region->emitUntiedSwitch(CGF);
2091 void CGOpenMPRuntime::emitTaskgroupRegion(CodeGenFunction &CGF,
2094 if (!CGF.HaveInsertPoint())
2100 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
2108 emitInlinedDirective(CGF, OMPD_taskgroup, TaskgroupOpGen);
2113 static Address emitAddrOfVarFromArray(CodeGenFunction &CGF, Address Array,
2116 Address PtrAddr = CGF.Builder.CreateConstArrayGEP(Array, Index);
2117 llvm::Value *Ptr = CGF.Builder.CreateLoad(PtrAddr);
2119 llvm::Type *ElemTy = CGF.ConvertTypeForMem(Var->getType());
2120 return Address(Ptr, ElemTy, CGF.getContext().getDeclAlign(Var));
2146 CodeGenFunction CGF(CGM);
2147 CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, CGFI, Args, Loc, Loc);
2150 Address LHS(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
2151 CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(&LHSArg)),
2152 CGF.Builder.getPtrTy(0)),
2153 ArgsElemType, CGF.getPointerAlign());
2154 Address RHS(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
2155 CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(&RHSArg)),
2156 CGF.Builder.getPtrTy(0)),
2157 ArgsElemType, CGF.getPointerAlign());
2165 Address DestAddr = emitAddrOfVarFromArray(CGF, LHS, I, DestVar);
2169 Address SrcAddr = emitAddrOfVarFromArray(CGF, RHS, I, SrcVar);
2173 CGF.EmitOMPCopy(Type, DestAddr, SrcAddr, DestVar, SrcVar, AssignmentOps[I]);
2175 CGF.FinishFunction();
2179 void CGOpenMPRuntime::emitSingleRegion(CodeGenFunction &CGF,
2186 if (!CGF.HaveInsertPoint())
2206 DidIt = CGF.CreateMemTemp(KmpInt32Ty, ".omp.copyprivate.did_it");
2207 CGF.Builder.CreateStore(CGF.Builder.getInt32(0), DidIt);
2210 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
2219 emitInlinedDirective(CGF, OMPD_single, SingleOpGen);
2222 CGF.Builder.CreateStore(CGF.Builder.getInt32(1), DidIt);
2224 Action.Done(CGF);
2234 CGF.CreateMemTemp(CopyprivateArrayTy, ".omp.copyprivate.cpr_list");
2236 Address Elem = CGF.Builder.CreateConstArrayGEP(CopyprivateList, I);
2237 CGF.Builder.CreateStore(
2238 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
2239 CGF.EmitLValue(CopyprivateVars[I]).getPointer(CGF),
2240 CGF.VoidPtrTy),
2246 CGM, CGF.ConvertTypeForMem(CopyprivateArrayTy), CopyprivateVars,
2248 llvm::Value *BufSize = CGF.getTypeSize(CopyprivateArrayTy);
2249 Address CL = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
2250 CopyprivateList, CGF.VoidPtrTy, CGF.Int8Ty);
2251 llvm::Value *DidItVal = CGF.Builder.CreateLoad(DidIt);
2253 emitUpdateLocation(CGF, Loc), // ident_t *<loc>
2254 getThreadID(CGF, Loc), // i32 <gtid>
2256 CL.emitRawPointer(CGF), // void *<copyprivate list>
2260 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
2266 void CGOpenMPRuntime::emitOrderedRegion(CodeGenFunction &CGF,
2269 if (!CGF.HaveInsertPoint())
2276 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
2284 emitInlinedDirective(CGF, OMPD_ordered, OrderedOpGen);
2287 emitInlinedDirective(CGF, OMPD_ordered, OrderedOpGen);
2306 CodeGenFunction &CGF, const OMPLoopDirective &S,
2317 CGF.getContext(), ChunkSize,
2318 CGF.getContext().getIntTypeForBitwidth(32, /*Signed=*/0),
2323 void CGOpenMPRuntime::emitBarrierCall(CodeGenFunction &CGF, SourceLocation Loc,
2328 dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo);
2329 if (CGF.CGM.getLangOpts().OpenMPIRBuilder) {
2331 cantFail(OMPBuilder.createBarrier(CGF.Builder, Kind, ForceSimpleCall,
2333 CGF.Builder.restoreIP(AfterIP);
2337 if (!CGF.HaveInsertPoint())
2344 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc, Flags),
2345 getThreadID(CGF, Loc)};
2348 llvm::Value *Result = CGF.EmitRuntimeCall(
2356 llvm::BasicBlock *ExitBB = CGF.createBasicBlock(".cancel.exit");
2357 llvm::BasicBlock *ContBB = CGF.createBasicBlock(".cancel.continue");
2358 llvm::Value *Cmp = CGF.Builder.CreateIsNotNull(Result);
2359 CGF.Builder.CreateCondBr(Cmp, ExitBB, ContBB);
2360 CGF.EmitBlock(ExitBB);
2363 CGF.getOMPCancelDestination(OMPRegionInfo->getDirectiveKind());
2364 CGF.EmitBranchThroughCleanup(CancelDestination);
2365 CGF.EmitBlock(ContBB, /*IsFinished=*/true);
2370 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
2375 void CGOpenMPRuntime::emitErrorCall(CodeGenFunction &CGF, SourceLocation Loc,
2378 ME ? CGF.EmitStringLiteralLValue(cast<StringLiteral>(ME)).getPointer(CGF)
2379 : llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
2383 emitUpdateLocation(CGF, Loc, /*Flags=*/0, /*GenLoc=*/true),
2385 CGF.Builder.CreatePointerCast(MVL, CGM.Int8PtrTy)};
2386 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
2505 CodeGenFunction &CGF, SourceLocation Loc,
2508 if (!CGF.HaveInsertPoint())
2523 : CGF.Builder.getIntN(IVSize, 1);
2525 emitUpdateLocation(CGF, Loc),
2526 getThreadID(CGF, Loc),
2527 CGF.Builder.getInt32(addMonoNonMonoModifier(
2531 CGF.Builder.getIntN(IVSize, 1), // Stride
2534 CGF.EmitRuntimeCall(OMPBuilder.createDispatchInitFunction(IVSize, IVSigned),
2538 void CGOpenMPRuntime::emitForDispatchDeinit(CodeGenFunction &CGF,
2540 if (!CGF.HaveInsertPoint())
2543 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
2544 CGF.EmitRuntimeCall(OMPBuilder.createDispatchDeinitFunction(), Args);
2548 CodeGenFunction &CGF, llvm::Value *UpdateLocation, llvm::Value *ThreadId,
2552 if (!CGF.HaveInsertPoint())
2573 Chunk = CGF.Builder.getIntN(Values.IVSize, 1);
2584 CGF.Builder.getInt32(addMonoNonMonoModifier(CGF.CGM, Schedule, M1,
2586 Values.IL.emitRawPointer(CGF), // &isLastIter
2587 Values.LB.emitRawPointer(CGF), // &LB
2588 Values.UB.emitRawPointer(CGF), // &UB
2589 Values.ST.emitRawPointer(CGF), // &Stride
2590 CGF.Builder.getIntN(Values.IVSize, 1), // Incr
2593 CGF.EmitRuntimeCall(ForStaticInitFunction, Args);
2596 void CGOpenMPRuntime::emitForStaticInit(CodeGenFunction &CGF,
2605 llvm::Value *UpdatedLocation = emitUpdateLocation(CGF, Loc,
2609 llvm::Value *ThreadId = getThreadID(CGF, Loc);
2613 auto DL = ApplyDebugLocation::CreateDefaultArtificial(CGF, Loc);
2614 emitForStaticInitCall(CGF, UpdatedLocation, ThreadId, StaticInitFunction,
2619 CodeGenFunction &CGF, SourceLocation Loc,
2625 emitUpdateLocation(CGF, Loc, OMP_IDENT_WORK_DISTRIBUTE);
2626 llvm::Value *ThreadId = getThreadID(CGF, Loc);
2634 emitForStaticInitCall(CGF, UpdatedLocation, ThreadId, StaticInitFunction,
2639 void CGOpenMPRuntime::emitForStaticFinish(CodeGenFunction &CGF,
2645 if (!CGF.HaveInsertPoint())
2649 emitUpdateLocation(CGF, Loc,
2656 getThreadID(CGF, Loc)};
2657 auto DL = ApplyDebugLocation::CreateDefaultArtificial(CGF, Loc);
2661 CGF.EmitRuntimeCall(
2666 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
2671 void CGOpenMPRuntime::emitForOrderedIterationEnd(CodeGenFunction &CGF,
2675 if (!CGF.HaveInsertPoint())
2678 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
2679 CGF.EmitRuntimeCall(OMPBuilder.createDispatchFiniFunction(IVSize, IVSigned),
2683 llvm::Value *CGOpenMPRuntime::emitForNext(CodeGenFunction &CGF,
2693 emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
2694 IL.emitRawPointer(CGF), // &isLastIter
2695 LB.emitRawPointer(CGF), // &Lower
2696 UB.emitRawPointer(CGF), // &Upper
2697 ST.emitRawPointer(CGF) // &Stride
2699 llvm::Value *Call = CGF.EmitRuntimeCall(
2701 return CGF.EmitScalarConversion(
2702 Call, CGF.getContext().getIntTypeForBitwidth(32, /*Signed=*/1),
2703 CGF.getContext().BoolTy, Loc);
2706 void CGOpenMPRuntime::emitNumThreadsClause(CodeGenFunction &CGF,
2709 if (!CGF.HaveInsertPoint())
2713 emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
2714 CGF.Builder.CreateIntCast(NumThreads, CGF.Int32Ty, /*isSigned*/ true)};
2715 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
2720 void CGOpenMPRuntime::emitProcBindClause(CodeGenFunction &CGF,
2723 if (!CGF.HaveInsertPoint())
2728 emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
2730 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
2735 void CGOpenMPRuntime::emitFlush(CodeGenFunction &CGF, ArrayRef<const Expr *>,
2737 if (CGF.CGM.getLangOpts().OpenMPIRBuilder) {
2738 OMPBuilder.createFlush(CGF.Builder);
2740 if (!CGF.HaveInsertPoint())
2743 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
2745 emitUpdateLocation(CGF, Loc));
2998 CodeGenFunction CGF(CGM);
2999 CGF.StartFunction(GlobalDecl(), KmpInt32Ty, TaskEntry, TaskEntryFnInfo, Args,
3007 llvm::Value *GtidParam = CGF.EmitLoadOfScalar(
3008 CGF.GetAddrOfLocalVar(&GtidArg), /*Volatile=*/false, KmpInt32Ty, Loc);
3009 LValue TDBase = CGF.EmitLoadOfPointerLValue(
3010 CGF.GetAddrOfLocalVar(&TaskTypeArg),
3015 CGF.EmitLValueForField(TDBase, *KmpTaskTWithPrivatesQTyRD->field_begin());
3018 LValue PartIdLVal = CGF.EmitLValueForField(Base, *PartIdFI);
3019 llvm::Value *PartidParam = PartIdLVal.getPointer(CGF);
3022 LValue SharedsLVal = CGF.EmitLValueForField(Base, *SharedsFI);
3023 llvm::Value *SharedsParam = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
3024 CGF.EmitLoadOfScalar(SharedsLVal, Loc),
3025 CGF.ConvertTypeForMem(SharedsPtrTy));
3030 LValue PrivatesLVal = CGF.EmitLValueForField(TDBase, *PrivatesFI);
3031 PrivatesParam = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
3032 PrivatesLVal.getPointer(CGF), CGF.VoidPtrTy);
3034 PrivatesParam = llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
3039 CGF.Builder
3041 CGF.VoidPtrTy, CGF.Int8Ty)
3042 .emitRawPointer(CGF)};
3047 LValue LBLVal = CGF.EmitLValueForField(Base, *LBFI);
3048 llvm::Value *LBParam = CGF.EmitLoadOfScalar(LBLVal, Loc);
3050 LValue UBLVal = CGF.EmitLValueForField(Base, *UBFI);
3051 llvm::Value *UBParam = CGF.EmitLoadOfScalar(UBLVal, Loc);
3053 LValue StLVal = CGF.EmitLValueForField(Base, *StFI);
3054 llvm::Value *StParam = CGF.EmitLoadOfScalar(StLVal, Loc);
3056 LValue LILVal = CGF.EmitLValueForField(Base, *LIFI);
3057 llvm::Value *LIParam = CGF.EmitLoadOfScalar(LILVal, Loc);
3059 LValue RLVal = CGF.EmitLValueForField(Base, *RFI);
3060 llvm::Value *RParam = CGF.EmitLoadOfScalar(RLVal, Loc);
3069 CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF, Loc, TaskFunction,
3071 CGF.EmitStoreThroughLValue(RValue::get(CGF.Builder.getInt32(/*C=*/0)),
3072 CGF.MakeAddrLValue(CGF.ReturnValue, KmpInt32Ty));
3073 CGF.FinishFunction();
3103 CodeGenFunction CGF(CGM);
3104 CGF.StartFunction(GlobalDecl(), KmpInt32Ty, DestructorFn, DestructorFnInfo,
3107 LValue Base = CGF.EmitLoadOfPointerLValue(
3108 CGF.GetAddrOfLocalVar(&TaskTypeArg),
3113 Base = CGF.EmitLValueForField(Base, *FI);
3118 LValue FieldLValue = CGF.EmitLValueForField(Base, Field);
3119 CGF.pushDestroy(DtorKind, FieldLValue.getAddress(), Field->getType());
3122 CGF.FinishFunction();
3211 CodeGenFunction CGF(CGM);
3212 CGF.StartFunction(GlobalDecl(), C.VoidTy, TaskPrivatesMap,
3216 LValue Base = CGF.EmitLoadOfPointerLValue(
3217 CGF.GetAddrOfLocalVar(&TaskPrivatesArg),
3222 LValue FieldLVal = CGF.EmitLValueForField(Base, Field);
3225 CGF.MakeAddrLValue(CGF.GetAddrOfLocalVar(VD), VD->getType());
3226 LValue RefLoadLVal = CGF.EmitLoadOfPointerLValue(
3228 CGF.EmitStoreOfScalar(FieldLVal.getPointer(CGF), RefLoadLVal);
3231 CGF.FinishFunction();
3236 static void emitPrivatesInit(CodeGenFunction &CGF,
3243 ASTContext &C = CGF.getContext();
3245 LValue PrivatesBase = CGF.EmitLValueForField(TDBase, *FI);
3260 SrcBase = CGF.MakeAddrLValue(
3261 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
3262 KmpTaskSharedsPtr, CGF.ConvertTypeForMem(SharedsPtrTy),
3263 CGF.ConvertTypeForMem(SharedsTy)),
3276 !CGF.isTrivialInitializer(Init)))) {
3277 LValue PrivateLValue = CGF.EmitLValueForField(PrivatesBase, *FI);
3295 CGF.MakeAddrLValue(CGF.GetAddrOfLocalVar(OriginalVD), Type);
3297 SharedRefLValue = CGF.EmitLValueForField(SrcBase, SharedField);
3298 SharedRefLValue = CGF.MakeAddrLValue(
3303 } else if (CGF.LambdaCaptureFields.count(
3305 isa_and_nonnull<BlockDecl>(CGF.CurCodeDecl)) {
3306 SharedRefLValue = CGF.EmitLValue(Pair.second.OriginalRef);
3310 CGF, [](CodeGenFunction &, PrePostActionTy &) {}, OMPD_unknown,
3312 SharedRefLValue = CGF.EmitLValue(Pair.second.OriginalRef);
3316 if (!isa<CXXConstructExpr>(Init) || CGF.isTrivialInitializer(Init)) {
3318 CGF.EmitAggregateAssign(PrivateLValue, SharedRefLValue, Type);
3322 CGF.EmitOMPAggregateAssign(
3324 [&CGF, Elem, Init, &CapturesInfo](Address DestElement,
3327 CodeGenFunction::OMPPrivateScope InitScope(CGF);
3332 CGF, &CapturesInfo);
3333 CGF.EmitAnyExprToMem(Init, DestElement,
3339 CodeGenFunction::OMPPrivateScope InitScope(CGF);
3342 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CapturesInfo);
3343 CGF.EmitExprAsInit(Init, VD, PrivateLValue,
3347 CGF.EmitExprAsInit(Init, VD, PrivateLValue, /*capturedByInit=*/false);
3355 static bool checkInitIsRequired(CodeGenFunction &CGF,
3364 !CGF.isTrivialInitializer(Init));
3411 CodeGenFunction CGF(CGM);
3412 CGF.StartFunction(GlobalDecl(), C.VoidTy, TaskDup, TaskDupFnInfo, Args, Loc,
3415 LValue TDBase = CGF.EmitLoadOfPointerLValue(
3416 CGF.GetAddrOfLocalVar(&DstArg),
3421 LValue Base = CGF.EmitLValueForField(
3423 LValue LILVal = CGF.EmitLValueForField(Base, *LIFI);
3424 llvm::Value *Lastpriv = CGF.EmitLoadOfScalar(
3425 CGF.GetAddrOfLocalVar(&LastprivArg), /*Volatile=*/false, C.IntTy, Loc);
3426 CGF.EmitStoreOfScalar(Lastpriv, LILVal);
3433 LValue TDBase = CGF.EmitLoadOfPointerLValue(
3434 CGF.GetAddrOfLocalVar(&SrcArg),
3436 LValue Base = CGF.EmitLValueForField(
3439 CGF.EmitLoadOfScalar(CGF.EmitLValueForField(
3443 CGF.Int8Ty, CGM.getNaturalTypeAlignment(SharedsTy));
3445 emitPrivatesInit(CGF, D, KmpTaskSharedsPtr, TDBase, KmpTaskTWithPrivatesQTyRD,
3447 CGF.FinishFunction();
3470 CodeGenFunction &CGF;
3478 OMPIteratorGeneratorScope(CodeGenFunction &CGF, const OMPIteratorExpr *E)
3479 : CodeGenFunction::OMPPrivateScope(CGF), CGF(CGF), E(E) {
3484 Uppers.push_back(CGF.EmitScalarExpr(E->getHelper(I).Upper));
3486 addPrivate(VD, CGF.CreateMemTemp(VD->getType(), VD->getName()));
3490 CGF.CreateMemTemp(HelperData.CounterVD->getType(), "counter.addr"));
3497 CGF.MakeAddrLValue(CGF.GetAddrOfLocalVar(HelperData.CounterVD),
3500 CGF.EmitStoreOfScalar(
3504 ContDests.emplace_back(CGF.getJumpDestInCurrentScope("iter.cont"));
3506 ExitDests.emplace_back(CGF.getJumpDestInCurrentScope("iter.exit"));
3511 CGF.EmitBlock(ContDest.getBlock());
3513 CGF.EmitLoadOfScalar(CLVal, HelperData.CounterVD->getLocation());
3516 ? CGF.Builder.CreateICmpSLT(CVal, N)
3517 : CGF.Builder.CreateICmpULT(CVal, N);
3518 llvm::BasicBlock *BodyBB = CGF.createBasicBlock("iter.body");
3519 CGF.Builder.CreateCondBr(Cmp, BodyBB, ExitDest.getBlock());
3521 CGF.EmitBlock(BodyBB);
3523 CGF.EmitIgnoredExpr(HelperData.Update);
3532 CGF.EmitIgnoredExpr(HelperData.CounterUpdate);
3534 CGF.EmitBranchThroughCleanup(ContDests[I - 1]);
3536 CGF.EmitBlock(ExitDests[I - 1].getBlock(), /*IsFinished=*/I == 1);
3543 getPointerAndSize(CodeGenFunction &CGF, const Expr *E) {
3548 Addr = CGF.EmitScalarExpr(Base);
3550 Addr = CGF.EmitLValue(E).getPointer(CGF);
3555 SizeVal = CGF.getTypeSize(OASE->getBase()->getType()->getPointeeType());
3557 llvm::Value *Sz = CGF.EmitScalarExpr(SE);
3558 Sz = CGF.EmitScalarConversion(
3559 Sz, SE->getType(), CGF.getContext().getSizeType(), SE->getExprLoc());
3560 SizeVal = CGF.Builder.CreateNUWMul(SizeVal, Sz);
3564 LValue UpAddrLVal = CGF.EmitArraySectionExpr(ASE, /*IsLowerBound=*/false);
3566 llvm::Value *UpAddr = CGF.Builder.CreateConstGEP1_32(
3567 UpAddrAddress.getElementType(), UpAddrAddress.emitRawPointer(CGF),
3569 llvm::Value *LowIntPtr = CGF.Builder.CreatePtrToInt(Addr, CGF.SizeTy);
3570 llvm::Value *UpIntPtr = CGF.Builder.CreatePtrToInt(UpAddr, CGF.SizeTy);
3571 SizeVal = CGF.Builder.CreateNUWSub(UpIntPtr, LowIntPtr);
3573 SizeVal = CGF.getTypeSize(Ty);
3594 CGOpenMPRuntime::emitTaskInit(CodeGenFunction &CGF, SourceLocation Loc,
3669 llvm::Type *KmpTaskTWithPrivatesPtrTy = CGF.Builder.getPtrTy(0);
3671 CGF.getTypeSize(KmpTaskTWithPrivatesQTy);
3682 TaskPrivatesMap = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
3722 ? CGF.Builder.CreateSelect(Data.Final.getPointer(),
3723 CGF.Builder.getInt32(FinalFlag),
3724 CGF.Builder.getInt32(/*C=*/0))
3725 : CGF.Builder.getInt32(Data.Final.getInt() ? FinalFlag : 0);
3726 TaskFlags = CGF.Builder.CreateOr(TaskFlags, CGF.Builder.getInt32(Flags));
3728 SmallVector<llvm::Value *, 8> AllocArgs = {emitUpdateLocation(CGF, Loc),
3729 getThreadID(CGF, Loc), TaskFlags, KmpTaskTWithPrivatesTySize,
3730 SharedsSize, CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
3741 DeviceID = CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(Device),
3742 CGF.Int64Ty, /*isSigned=*/true);
3744 DeviceID = CGF.Builder.getInt64(OMP_DEVICEID_UNDEF);
3746 NewTask = CGF.EmitRuntimeCall(
3752 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
3761 LValue EvtLVal = CGF.EmitLValue(Evt);
3765 llvm::Value *Loc = emitUpdateLocation(CGF, DC->getBeginLoc());
3766 llvm::Value *Tid = getThreadID(CGF, DC->getBeginLoc());
3767 Tid = CGF.Builder.CreateIntCast(Tid, CGF.IntTy, /*isSigned=*/false);
3768 llvm::Value *EvtVal = CGF.EmitRuntimeCall(
3772 EvtVal = CGF.EmitScalarConversion(EvtVal, C.VoidPtrTy, Evt->getType(),
3774 CGF.EmitStoreOfScalar(EvtVal, EvtLVal);
3788 llvm::Value *Sz = CGF.EmitScalarExpr(IE->getHelper(I).Upper);
3789 Sz = CGF.Builder.CreateIntCast(Sz, CGF.SizeTy, /*isSigned=*/false);
3791 NumOfElements ? CGF.Builder.CreateNUWMul(NumOfElements, Sz) : Sz;
3803 NumOfElements = CGF.Builder.CreateNUWAdd(
3804 llvm::ConstantInt::get(CGF.SizeTy, NumAffinities), NumOfElements);
3809 CodeGenFunction::OpaqueValueMapping OpaqueMap(CGF, OVE,
3817 CGF.EmitVarDecl(*PD);
3818 AffinitiesArray = CGF.GetAddrOfLocalVar(PD);
3819 NumOfElements = CGF.Builder.CreateIntCast(NumOfElements, CGF.Int32Ty,
3827 CGF.CreateMemTemp(KmpTaskAffinityInfoArrayTy, ".affs.arr.addr");
3828 AffinitiesArray = CGF.Builder.CreateConstArrayGEP(AffinitiesArray, 0);
3845 std::tie(Addr, Size) = getPointerAndSize(CGF, E);
3847 CGF.MakeAddrLValue(CGF.Builder.CreateConstGEP(AffinitiesArray, Pos),
3850 LValue BaseAddrLVal = CGF.EmitLValueForField(
3852 CGF.EmitStoreOfScalar(CGF.Builder.CreatePtrToInt(Addr, CGF.IntPtrTy),
3855 LValue LenLVal = CGF.EmitLValueForField(
3857 CGF.EmitStoreOfScalar(Size, LenLVal);
3863 PosLVal = CGF.MakeAddrLValue(
3864 CGF.CreateMemTemp(C.getSizeType(), "affs.counter.addr"),
3866 CGF.EmitStoreOfScalar(llvm::ConstantInt::get(CGF.SizeTy, Pos), PosLVal);
3874 CGF, cast_or_null<OMPIteratorExpr>(Modifier->IgnoreParenImpCasts()));
3878 std::tie(Addr, Size) = getPointerAndSize(CGF, E);
3879 llvm::Value *Idx = CGF.EmitLoadOfScalar(PosLVal, E->getExprLoc());
3881 CGF.MakeAddrLValue(CGF.Builder.CreateGEP(CGF, AffinitiesArray, Idx),
3884 LValue BaseAddrLVal = CGF.EmitLValueForField(
3886 CGF.EmitStoreOfScalar(CGF.Builder.CreatePtrToInt(Addr, CGF.IntPtrTy),
3889 LValue LenLVal = CGF.EmitLValueForField(
3891 CGF.EmitStoreOfScalar(Size, LenLVal);
3892 Idx = CGF.Builder.CreateNUWAdd(
3894 CGF.EmitStoreOfScalar(Idx, PosLVal);
3900 llvm::Value *LocRef = emitUpdateLocation(CGF, Loc);
3901 llvm::Value *GTid = getThreadID(CGF, Loc);
3902 llvm::Value *AffinListPtr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
3903 AffinitiesArray.emitRawPointer(CGF), CGM.VoidPtrTy);
3906 (void)CGF.EmitRuntimeCall(
3912 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
3914 LValue Base = CGF.MakeNaturalAlignRawAddrLValue(NewTaskNewTaskTTy,
3917 CGF.EmitLValueForField(Base, *KmpTaskTWithPrivatesQTyRD->field_begin());
3923 CGF.EmitLoadOfScalar(
3924 CGF.EmitLValueForField(
3928 CGF.Int8Ty, CGM.getNaturalTypeAlignment(SharedsTy));
3929 LValue Dest = CGF.MakeAddrLValue(KmpTaskSharedsPtr, SharedsTy);
3930 LValue Src = CGF.MakeAddrLValue(Shareds, SharedsTy);
3931 CGF.EmitAggregateCopy(Dest, Src, SharedsTy, AggValueSlot::DoesNotOverlap);
3936 emitPrivatesInit(CGF, D, KmpTaskSharedsPtr, Base, KmpTaskTWithPrivatesQTyRD,
3940 (!Data.LastprivateVars.empty() || checkInitIsRequired(CGF, Privates))) {
3957 LValue Data1LV = CGF.EmitLValueForField(TDBase, *FI);
3958 LValue DestructorsLV = CGF.EmitLValueForField(
3960 CGF.EmitStoreOfScalar(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
3966 LValue Data2LV = CGF.EmitLValueForField(
3968 LValue PriorityLV = CGF.EmitLValueForField(
3970 CGF.EmitStoreOfScalar(Data.Priority.getPointer(), PriorityLV);
4027 CGOpenMPRuntime::getDepobjElements(CodeGenFunction &CGF, LValue DepobjLVal,
4035 LValue Base = CGF.EmitLoadOfPointerLValue(
4037 CGF.ConvertTypeForMem(KmpDependInfoPtrTy)),
4039 Address DepObjAddr = CGF.Builder.CreateGEP(
4040 CGF, Base.getAddress(),
4041 llvm::ConstantInt::get(CGF.IntPtrTy, -1, /*isSigned=*/true));
4042 LValue NumDepsBase = CGF.MakeAddrLValue(
4045 LValue BaseAddrLVal = CGF.EmitLValueForField(
4049 llvm::Value *NumDeps = CGF.EmitLoadOfScalar(BaseAddrLVal, Loc);
4053 static void emitDependData(CodeGenFunction &CGF, QualType &KmpDependInfoTy,
4057 CodeGenModule &CGM = CGF.CGM;
4063 llvm::Type *LLVMFlagsTy = CGF.ConvertTypeForMem(FlagsTy);
4066 CGF, cast_or_null<OMPIteratorExpr>(
4075 std::tie(Addr, Size) = getPointerAndSize(CGF, E);
4076 Addr = CGF.Builder.CreatePtrToInt(Addr, CGF.IntPtrTy);
4078 Addr = llvm::ConstantInt::get(CGF.IntPtrTy, 0);
4079 Size = llvm::ConstantInt::get(CGF.SizeTy, 0);
4083 Base = CGF.MakeAddrLValue(
4084 CGF.Builder.CreateConstGEP(DependenciesArray, *P), KmpDependInfoTy);
4088 llvm::Value *Idx = CGF.EmitLoadOfScalar(PosLVal, E->getExprLoc());
4089 Base = CGF.MakeAddrLValue(
4090 CGF.Builder.CreateGEP(CGF, DependenciesArray, Idx), KmpDependInfoTy);
4093 LValue BaseAddrLVal = CGF.EmitLValueForField(
4097 CGF.EmitStoreOfScalar(Addr, BaseAddrLVal);
4099 LValue LenLVal = CGF.EmitLValueForField(
4102 CGF.EmitStoreOfScalar(Size, LenLVal);
4105 LValue FlagsLVal = CGF.EmitLValueForField(
4109 CGF.EmitStoreOfScalar(
4116 llvm::Value *Idx = CGF.EmitLoadOfScalar(PosLVal, E->getExprLoc());
4117 Idx = CGF.Builder.CreateNUWAdd(Idx,
4119 CGF.EmitStoreOfScalar(Idx, PosLVal);
4125 CodeGenFunction &CGF, QualType &KmpDependInfoTy,
4131 ASTContext &C = CGF.getContext();
4134 CGF, cast_or_null<OMPIteratorExpr>(
4140 LValue DepobjLVal = CGF.EmitLValue(E->IgnoreParenImpCasts());
4142 getDepobjElements(CGF, DepobjLVal, E->getExprLoc());
4143 LValue NumLVal = CGF.MakeAddrLValue(
4144 CGF.CreateMemTemp(C.getUIntPtrType(), "depobj.size.addr"),
4146 CGF.Builder.CreateStore(llvm::ConstantInt::get(CGF.IntPtrTy, 0),
4148 llvm::Value *PrevVal = CGF.EmitLoadOfScalar(NumLVal, E->getExprLoc());
4149 llvm::Value *Add = CGF.Builder.CreateNUWAdd(PrevVal, NumDeps);
4150 CGF.EmitStoreOfScalar(Add, NumLVal);
4156 CGF.EmitLoadOfScalar(SizeLVals[I], Data.DepExprs[I]->getExprLoc());
4162 void CGOpenMPRuntime::emitDepobjElements(CodeGenFunction &CGF,
4169 llvm::Value *ElSize = CGF.getTypeSize(KmpDependInfoTy);
4172 CGF, cast_or_null<OMPIteratorExpr>(
4179 LValue DepobjLVal = CGF.EmitLValue(E->IgnoreParenImpCasts());
4181 getDepobjElements(CGF, DepobjLVal, E->getExprLoc());
4184 llvm::Value *Size = CGF.Builder.CreateNUWMul(
4186 CGF.Builder.CreateIntCast(NumDeps, CGF.SizeTy, /*isSigned=*/false));
4187 llvm::Value *Pos = CGF.EmitLoadOfScalar(PosLVal, E->getExprLoc());
4188 Address DepAddr = CGF.Builder.CreateGEP(CGF, DependenciesArray, Pos);
4189 CGF.Builder.CreateMemCpy(DepAddr, Base.getAddress(), Size);
4193 llvm::Value *Add = CGF.Builder.CreateNUWAdd(Pos, NumDeps);
4194 CGF.EmitStoreOfScalar(Add, PosLVal);
4200 CodeGenFunction &CGF, ArrayRef<OMPTaskDataTy::DependData> Dependencies,
4221 llvm::Value *NumOfDepobjElements = llvm::ConstantInt::get(CGF.IntPtrTy, 0);
4223 llvm::ConstantInt::get(CGF.IntPtrTy, 0);
4229 emitDepobjElementsSizes(CGF, KmpDependInfoTy, D);
4232 CGF.Builder.CreateNUWAdd(NumOfDepobjElements, Size);
4241 llvm::ConstantInt::get(CGF.IntPtrTy, 1);
4243 llvm::Value *Sz = CGF.EmitScalarExpr(IE->getHelper(I).Upper);
4244 Sz = CGF.Builder.CreateIntCast(Sz, CGF.IntPtrTy, /*isSigned=*/false);
4245 ClauseIteratorSpace = CGF.Builder.CreateNUWMul(Sz, ClauseIteratorSpace);
4247 llvm::Value *NumClauseDeps = CGF.Builder.CreateNUWMul(
4249 llvm::ConstantInt::get(CGF.IntPtrTy, D.DepExprs.size()));
4251 CGF.Builder.CreateNUWAdd(NumOfRegularWithIterators, NumClauseDeps);
4263 CGF.Builder.CreateNUWAdd(NumOfDepobjElements, NumOfElements);
4267 CGF.Builder.CreateNUWAdd(NumOfRegularWithIterators, NumOfElements);
4272 CodeGenFunction::OpaqueValueMapping OpaqueMap(CGF, OVE,
4277 // CGF.EmitVariablyModifiedType(KmpDependInfoArrayTy);
4281 CGF.EmitVarDecl(*PD);
4282 DependenciesArray = CGF.GetAddrOfLocalVar(PD);
4283 NumOfElements = CGF.Builder.CreateIntCast(NumOfElements, CGF.Int32Ty,
4290 CGF.CreateMemTemp(KmpDependInfoArrayTy, ".dep.arr.addr");
4291 DependenciesArray = CGF.Builder.CreateConstArrayGEP(DependenciesArray, 0);
4300 emitDependData(CGF, KmpDependInfoTy, &Pos, Dependencies[I],
4304 LValue PosLVal = CGF.MakeAddrLValue(
4305 CGF.CreateMemTemp(C.getSizeType(), "dep.counter.addr"), C.getSizeType());
4306 CGF.EmitStoreOfScalar(llvm::ConstantInt::get(CGF.SizeTy, Pos), PosLVal);
4311 emitDependData(CGF, KmpDependInfoTy, &PosLVal, Dependencies[I],
4319 emitDepobjElements(CGF, KmpDependInfoTy, PosLVal, Dependencies[I],
4323 DependenciesArray = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4324 DependenciesArray, CGF.VoidPtrTy, CGF.Int8Ty);
4329 CodeGenFunction &CGF, const OMPTaskDataTy::DependData &Dependencies,
4351 NumDepsVal = llvm::ConstantInt::get(CGF.SizeTy, 1);
4353 llvm::Value *Sz = CGF.EmitScalarExpr(IE->getHelper(I).Upper);
4354 Sz = CGF.Builder.CreateIntCast(Sz, CGF.SizeTy, /*isSigned=*/false);
4355 NumDepsVal = CGF.Builder.CreateNUWMul(NumDepsVal, Sz);
4357 Size = CGF.Builder.CreateNUWAdd(llvm::ConstantInt::get(CGF.SizeTy, 1),
4362 Size = CGF.Builder.CreateNUWMul(Size, RecSize);
4364 CGF.Builder.CreateIntCast(NumDepsVal, CGF.IntPtrTy, /*isSigned=*/false);
4371 NumDepsVal = llvm::ConstantInt::get(CGF.IntPtrTy, NumDependencies);
4374 llvm::Value *ThreadID = getThreadID(CGF, Loc);
4376 llvm::Value *Allocator = llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
4380 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
4383 llvm::Type *KmpDependInfoLlvmTy = CGF.ConvertTypeForMem(KmpDependInfoTy);
4384 Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4385 Addr, CGF.Builder.getPtrTy(0));
4388 LValue Base = CGF.MakeAddrLValue(DependenciesArray, KmpDependInfoTy);
4390 LValue BaseAddrLVal = CGF.EmitLValueForField(
4394 CGF.EmitStoreOfScalar(NumDepsVal, BaseAddrLVal);
4399 PosLVal = CGF.MakeAddrLValue(
4400 CGF.CreateMemTemp(C.getSizeType(), "iterator.counter.addr"),
4402 CGF.EmitStoreOfScalar(llvm::ConstantInt::get(CGF.SizeTy, Idx), PosLVal,
4408 emitDependData(CGF, KmpDependInfoTy, Pos, Dependencies, DependenciesArray);
4409 DependenciesArray = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4410 CGF.Builder.CreateConstGEP(DependenciesArray, 1), CGF.VoidPtrTy,
4411 CGF.Int8Ty);
4415 void CGOpenMPRuntime::emitDestroyClause(CodeGenFunction &CGF, LValue DepobjLVal,
4420 LValue Base = CGF.EmitLoadOfPointerLValue(DepobjLVal.getAddress(),
4423 Address Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4424 Base.getAddress(), CGF.ConvertTypeForMem(KmpDependInfoPtrTy),
4425 CGF.ConvertTypeForMem(KmpDependInfoTy));
4426 llvm::Value *DepObjAddr = CGF.Builder.CreateGEP(
4427 Addr.getElementType(), Addr.emitRawPointer(CGF),
4428 llvm::ConstantInt::get(CGF.IntPtrTy, -1, /*isSigned=*/true));
4429 DepObjAddr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(DepObjAddr,
4430 CGF.VoidPtrTy);
4431 llvm::Value *ThreadID = getThreadID(CGF, Loc);
4433 llvm::Value *Allocator = llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
4437 (void)CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
4442 void CGOpenMPRuntime::emitUpdateClause(CodeGenFunction &CGF, LValue DepobjLVal,
4450 llvm::Type *LLVMFlagsTy = CGF.ConvertTypeForMem(FlagsTy);
4453 std::tie(NumDeps, Base) = getDepobjElements(CGF, DepobjLVal, Loc);
4457 llvm::Value *End = CGF.Builder.CreateGEP(Begin.getElementType(),
4458 Begin.emitRawPointer(CGF), NumDeps);
4460 llvm::BasicBlock *BodyBB = CGF.createBasicBlock("omp.body");
4461 llvm::BasicBlock *DoneBB = CGF.createBasicBlock("omp.done");
4462 llvm::BasicBlock *EntryBB = CGF.Builder.GetInsertBlock();
4463 CGF.EmitBlock(BodyBB);
4465 CGF.Builder.CreatePHI(Begin.getType(), 2, "omp.elementPast");
4466 ElementPHI->addIncoming(Begin.emitRawPointer(CGF), EntryBB);
4468 Base = CGF.MakeAddrLValue(Begin, KmpDependInfoTy, Base.getBaseInfo(),
4472 LValue FlagsLVal = CGF.EmitLValueForField(
4475 CGF.EmitStoreOfScalar(
4481 CGF.Builder.CreateConstGEP(Begin, /*Index=*/1, "omp.elementNext")
4482 .emitRawPointer(CGF);
4483 ElementPHI->addIncoming(ElementNext, CGF.Builder.GetInsertBlock());
4485 CGF.Builder.CreateICmpEQ(ElementNext, End, "omp.isempty");
4486 CGF.Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB);
4488 CGF.EmitBlock(DoneBB, /*IsFinished=*/true);
4491 void CGOpenMPRuntime::emitTaskCall(CodeGenFunction &CGF, SourceLocation Loc,
4497 if (!CGF.HaveInsertPoint())
4501 emitTaskInit(CGF, Loc, D, TaskFunction, SharedsTy, Shareds, Data);
4511 emitDependClause(CGF, Data.Dependences, Loc);
4519 llvm::Value *ThreadID = getThreadID(CGF, Loc);
4520 llvm::Value *UpLoc = emitUpdateLocation(CGF, Loc);
4528 DepTaskArgs[4] = DependenciesArray.emitRawPointer(CGF);
4529 DepTaskArgs[5] = CGF.Builder.getInt32(0);
4530 DepTaskArgs[6] = llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
4533 &DepTaskArgs](CodeGenFunction &CGF, PrePostActionTy &) {
4536 LValue PartIdLVal = CGF.EmitLValueForField(TDBase, *PartIdFI);
4537 CGF.EmitStoreOfScalar(CGF.Builder.getInt32(0), PartIdLVal);
4540 CGF.EmitRuntimeCall(
4545 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
4551 dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo))
4552 Region->emitUntiedSwitch(CGF);
4560 DepWaitTaskArgs[3] = DependenciesArray.emitRawPointer(CGF);
4561 DepWaitTaskArgs[4] = CGF.Builder.getInt32(0);
4562 DepWaitTaskArgs[5] = llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
4564 llvm::ConstantInt::get(CGF.Int32Ty, Data.HasNowaitClause);
4569 Loc](CodeGenFunction &CGF, PrePostActionTy &) {
4570 CodeGenFunction::RunCleanupsScope LocalScope(CGF);
4576 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
4581 Loc](CodeGenFunction &CGF, PrePostActionTy &Action) {
4582 Action.Enter(CGF);
4584 CGF.CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF, Loc, TaskEntry,
4600 RCG(CGF);
4604 emitIfClause(CGF, IfCond, ThenCodeGen, ElseCodeGen);
4607 ThenRCG(CGF);
4611 void CGOpenMPRuntime::emitTaskLoopCall(CodeGenFunction &CGF, SourceLocation Loc,
4617 if (!CGF.HaveInsertPoint())
4620 emitTaskInit(CGF, Loc, D, TaskFunction, SharedsTy, Shareds, Data);
4626 llvm::Value *ThreadID = getThreadID(CGF, Loc);
4627 llvm::Value *UpLoc = emitUpdateLocation(CGF, Loc);
4630 IfVal = CGF.Builder.CreateIntCast(CGF.EvaluateExprAsBool(IfCond), CGF.IntTy,
4633 IfVal = llvm::ConstantInt::getSigned(CGF.IntTy, /*V=*/1);
4636 LValue LBLVal = CGF.EmitLValueForField(
4641 CGF.EmitAnyExprToMem(LBVar->getInit(), LBLVal.getAddress(), LBLVal.getQuals(),
4643 LValue UBLVal = CGF.EmitLValueForField(
4648 CGF.EmitAnyExprToMem(UBVar->getInit(), UBLVal.getAddress(), UBLVal.getQuals(),
4650 LValue StLVal = CGF.EmitLValueForField(
4655 CGF.EmitAnyExprToMem(StVar->getInit(), StLVal.getAddress(), StLVal.getQuals(),
4658 LValue RedLVal = CGF.EmitLValueForField(
4662 CGF.EmitStoreOfScalar(Data.Reductions, RedLVal);
4664 CGF.EmitNullInitialization(RedLVal.getAddress(),
4665 CGF.getContext().VoidPtrTy);
4673 LBLVal.getPointer(CGF),
4674 UBLVal.getPointer(CGF),
4675 CGF.EmitLoadOfScalar(StLVal, Loc),
4677 CGF.IntTy, 1), // Always 1 because taskgroup emitted by the compiler
4679 CGF.IntTy, Data.Schedule.getPointer()
4683 ? CGF.Builder.CreateIntCast(Data.Schedule.getPointer(), CGF.Int64Ty,
4685 : llvm::ConstantInt::get(CGF.Int64Ty, /*V=*/0)};
4687 TaskArgs.push_back(llvm::ConstantInt::get(CGF.Int32Ty, 1));
4690 ? CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4691 Result.TaskDupFn, CGF.VoidPtrTy)
4692 : llvm::ConstantPointerNull::get(CGF.VoidPtrTy));
4693 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
4710 CodeGenFunction &CGF, QualType Type, const VarDecl *LHSVar,
4712 const llvm::function_ref<void(CodeGenFunction &CGF, const Expr *,
4718 Address LHSAddr = CGF.GetAddrOfLocalVar(LHSVar);
4719 Address RHSAddr = CGF.GetAddrOfLocalVar(RHSVar);
4723 llvm::Value *NumElements = CGF.emitArrayLength(ArrayTy, ElementTy, LHSAddr);
4725 llvm::Value *RHSBegin = RHSAddr.emitRawPointer(CGF);
4726 llvm::Value *LHSBegin = LHSAddr.emitRawPointer(CGF);
4729 CGF.Builder.CreateGEP(LHSAddr.getElementType(), LHSBegin, NumElements);
4731 llvm::BasicBlock *BodyBB = CGF.createBasicBlock("omp.arraycpy.body");
4732 llvm::BasicBlock *DoneBB = CGF.createBasicBlock("omp.arraycpy.done");
4734 CGF.Builder.CreateICmpEQ(LHSBegin, LHSEnd, "omp.arraycpy.isempty");
4735 CGF.Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB);
4738 llvm::BasicBlock *EntryBB = CGF.Builder.GetInsertBlock();
4739 CGF.EmitBlock(BodyBB);
4741 CharUnits ElementSize = CGF.getContext().getTypeSizeInChars(ElementTy);
4743 llvm::PHINode *RHSElementPHI = CGF.Builder.CreatePHI(
4750 llvm::PHINode *LHSElementPHI = CGF.Builder.CreatePHI(
4758 CodeGenFunction::OMPPrivateScope Scope(CGF);
4762 RedOpGen(CGF, XExpr, EExpr, UpExpr);
4766 llvm::Value *LHSElementNext = CGF.Builder.CreateConstGEP1_32(
4769 llvm::Value *RHSElementNext = CGF.Builder.CreateConstGEP1_32(
4774 CGF.Builder.CreateICmpEQ(LHSElementNext, LHSEnd, "omp.arraycpy.done");
4775 CGF.Builder.CreateCondBr(Done, DoneBB, BodyBB);
4776 LHSElementPHI->addIncoming(LHSElementNext, CGF.Builder.GetInsertBlock());
4777 RHSElementPHI->addIncoming(RHSElementNext, CGF.Builder.GetInsertBlock());
4780 CGF.EmitBlock(DoneBB, /*IsFinished=*/true);
4786 static void emitReductionCombiner(CodeGenFunction &CGF,
4795 CGF.CGM.getOpenMPRuntime().getUserDefinedReduction(DRD);
4797 CodeGenFunction::OpaqueValueMapping Map(CGF, OVE, Func);
4798 CGF.EmitIgnoredExpr(ReductionOp);
4801 CGF.EmitIgnoredExpr(ReductionOp);
4826 CodeGenFunction CGF(CGM);
4827 CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, CGFI, Args, Loc, Loc);
4831 Address LHS(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4832 CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(&LHSArg)),
4833 CGF.Builder.getPtrTy(0)),
4834 ArgsElemType, CGF.getPointerAlign());
4835 Address RHS(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4836 CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(&RHSArg)),
4837 CGF.Builder.getPtrTy(0)),
4838 ArgsElemType, CGF.getPointerAlign());
4843 CodeGenFunction::OMPPrivateScope Scope(CGF);
4849 Scope.addPrivate(RHSVar, emitAddrOfVarFromArray(CGF, RHS, Idx, RHSVar));
4852 Scope.addPrivate(LHSVar, emitAddrOfVarFromArray(CGF, LHS, Idx, LHSVar));
4857 Address Elem = CGF.Builder.CreateConstArrayGEP(LHS, Idx);
4858 llvm::Value *Ptr = CGF.Builder.CreateLoad(Elem);
4860 CGF.getContext().getAsVariableArrayType(PrivTy);
4863 CGF, OVE, RValue::get(CGF.Builder.CreatePtrToInt(Ptr, CGF.SizeTy)));
4864 CGF.EmitVariablyModifiedType(PrivTy);
4877 CGF, (*IPriv)->getType(), LHSVar, RHSVar,
4878 [=](CodeGenFunction &CGF, const Expr *, const Expr *, const Expr *) {
4879 emitReductionCombiner(CGF, E);
4883 emitReductionCombiner(CGF, E);
4890 CGF.FinishFunction();
4894 void CGOpenMPRuntime::emitSingleReductionCombiner(CodeGenFunction &CGF,
4904 CGF, PrivateRef->getType(), LHSVar, RHSVar,
4905 [=](CodeGenFunction &CGF, const Expr *, const Expr *, const Expr *) {
4906 emitReductionCombiner(CGF, ReductionOp);
4910 emitReductionCombiner(CGF, ReductionOp);
4914 void CGOpenMPRuntime::emitReduction(CodeGenFunction &CGF, SourceLocation Loc,
4920 if (!CGF.HaveInsertPoint())
4964 CodeGenFunction::RunCleanupsScope Scope(CGF);
4969 emitSingleReductionCombiner(CGF, E, *IPriv, cast<DeclRefExpr>(*ILHS),
4991 CGF.CreateMemTemp(ReductionArrayTy, ".omp.reduction.red_list");
4995 Address Elem = CGF.Builder.CreateConstArrayGEP(ReductionList, Idx);
4996 CGF.Builder.CreateStore(
4997 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4998 CGF.EmitLValue(RHSExprs[I]).getPointer(CGF), CGF.VoidPtrTy),
5003 Elem = CGF.Builder.CreateConstArrayGEP(ReductionList, Idx);
5004 llvm::Value *Size = CGF.Builder.CreateIntCast(
5005 CGF.getVLASize(
5006 CGF.getContext().getAsVariableArrayType((*IPriv)->getType()))
5008 CGF.SizeTy, /*isSigned=*/false);
5009 CGF.Builder.CreateStore(CGF.Builder.CreateIntToPtr(Size, CGF.VoidPtrTy),
5016 CGF.CurFn->getName(), Loc, CGF.ConvertTypeForMem(ReductionArrayTy),
5025 llvm::Value *IdentTLoc = emitUpdateLocation(CGF, Loc, OMP_ATOMIC_REDUCE);
5026 llvm::Value *ThreadId = getThreadID(CGF, Loc);
5027 llvm::Value *ReductionArrayTySize = CGF.getTypeSize(ReductionArrayTy);
5028 llvm::Value *RL = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
5029 ReductionList.getPointer(), CGF.VoidPtrTy);
5033 CGF.Builder.getInt32(RHSExprs.size()), // i32 <n>
5039 llvm::Value *Res = CGF.EmitRuntimeCall(
5046 llvm::BasicBlock *DefaultBB = CGF.createBasicBlock(".omp.reduction.default");
5048 CGF.Builder.CreateSwitch(Res, DefaultBB, /*NumCases=*/2);
5056 llvm::BasicBlock *Case1BB = CGF.createBasicBlock(".omp.reduction.case1");
5057 SwInst->addCase(CGF.Builder.getInt32(1), Case1BB);
5058 CGF.EmitBlock(Case1BB);
5067 CodeGenFunction &CGF, PrePostActionTy &Action) {
5068 CGOpenMPRuntime &RT = CGF.CGM.getOpenMPRuntime();
5073 RT.emitSingleReductionCombiner(CGF, E, *IPriv, cast<DeclRefExpr>(*ILHS),
5088 RCG(CGF);
5090 CGF.EmitBranch(DefaultBB);
5097 llvm::BasicBlock *Case2BB = CGF.createBasicBlock(".omp.reduction.case2");
5098 SwInst->addCase(CGF.Builder.getInt32(2), Case2BB);
5099 CGF.EmitBlock(Case2BB);
5102 CodeGenFunction &CGF, PrePostActionTy &Action) {
5136 Loc](CodeGenFunction &CGF, const Expr *XExpr,
5138 LValue X = CGF.EmitLValue(XExpr);
5141 E = CGF.EmitAnyExpr(EExpr);
5142 CGF.EmitOMPAtomicSimpleUpdateExpr(
5145 [&CGF, UpExpr, VD, Loc](RValue XRValue) {
5146 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
5147 Address LHSTemp = CGF.CreateMemTemp(VD->getType());
5148 CGF.emitOMPSimpleStore(
5149 CGF.MakeAddrLValue(LHSTemp, VD->getType()), XRValue,
5153 return CGF.EmitAnyExpr(UpExpr);
5160 EmitOMPAggregateReduction(CGF, (*IPriv)->getType(), VD, RHSVar,
5164 AtomicRedGen(CGF, XExpr, EExpr, UpExpr);
5168 auto &&CritRedGen = [E, Loc](CodeGenFunction &CGF, const Expr *,
5170 CGOpenMPRuntime &RT = CGF.CGM.getOpenMPRuntime();
5173 CGF, Name,
5174 [=](CodeGenFunction &CGF, PrePostActionTy &Action) {
5175 Action.Enter(CGF);
5176 emitReductionCombiner(CGF, E);
5185 EmitOMPAggregateReduction(CGF, (*IPriv)->getType(), LHSVar, RHSVar,
5188 CritRedGen(CGF, nullptr, nullptr, nullptr);
5209 AtomicRCG(CGF);
5211 AtomicRCG(CGF);
5214 CGF.EmitBranch(DefaultBB);
5215 CGF.EmitBlock(DefaultBB, /*IsFinished=*/true);
5265 CodeGenFunction CGF(CGM);
5266 CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, FnInfo, Args, Loc, Loc);
5268 Address PrivateAddr = CGF.EmitLoadOfPointer(
5269 CGF.GetAddrOfLocalVar(&Param).withElementType(CGF.Builder.getPtrTy(0)),
5276 CGF, CGM.getContext().getSizeType(),
5278 Size = CGF.EmitLoadOfScalar(SizeAddr, /*Volatile=*/false,
5281 RCG.emitAggregateType(CGF, N, Size);
5287 Address SharedAddr = CGF.GetAddrOfLocalVar(&ParamOrig);
5288 OrigAddr = CGF.EmitLoadOfPointer(
5295 RCG.emitInitialization(CGF, N, PrivateAddr, OrigAddr,
5297 CGF.FinishFunction();
5335 CodeGenFunction CGF(CGM);
5336 CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, FnInfo, Args, Loc, Loc);
5342 CGF, CGM.getContext().getSizeType(),
5344 Size = CGF.EmitLoadOfScalar(SizeAddr, /*Volatile=*/false,
5347 RCG.emitAggregateType(CGF, N, Size);
5351 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
5355 CGF.EmitLoadOfPointer(
5356 CGF.GetAddrOfLocalVar(&ParamInOut)
5357 .withElementType(CGF.Builder.getPtrTy(0)),
5362 CGF.EmitLoadOfPointer(
5363 CGF.GetAddrOfLocalVar(&ParamIn).withElementType(
5364 CGF.Builder.getPtrTy(0)),
5371 CGF, ReductionOp, PrivateRef, cast<DeclRefExpr>(LHS),
5373 CGF.FinishFunction();
5403 CodeGenFunction CGF(CGM);
5404 CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, FnInfo, Args, Loc, Loc);
5405 Address PrivateAddr = CGF.EmitLoadOfPointer(
5406 CGF.GetAddrOfLocalVar(&Param), C.VoidPtrTy.castAs<PointerType>());
5412 CGF, CGM.getContext().getSizeType(),
5414 Size = CGF.EmitLoadOfScalar(SizeAddr, /*Volatile=*/false,
5417 RCG.emitAggregateType(CGF, N, Size);
5420 RCG.emitCleanups(CGF, N, PrivateAddr);
5421 CGF.FinishFunction(Loc);
5426 CodeGenFunction &CGF, SourceLocation Loc, ArrayRef<const Expr *> LHSExprs,
5428 if (!CGF.HaveInsertPoint() || Data.ReductionVars.empty())
5460 RawAddress TaskRedInput = CGF.CreateMemTemp(ArrayRDType, ".rd_input.");
5467 llvm::Value *GEP = CGF.EmitCheckedInBoundsGEP(
5471 LValue ElemLVal = CGF.MakeNaturalAlignRawAddrLValue(GEP, RDType);
5473 LValue SharedLVal = CGF.EmitLValueForField(ElemLVal, SharedFD);
5474 RCG.emitSharedOrigLValue(CGF, Cnt);
5475 llvm::Value *Shared = RCG.getSharedLValue(Cnt).getPointer(CGF);
5476 CGF.EmitStoreOfScalar(Shared, SharedLVal);
5478 LValue OrigLVal = CGF.EmitLValueForField(ElemLVal, OrigFD);
5479 llvm::Value *Orig = RCG.getOrigLValue(Cnt).getPointer(CGF);
5480 CGF.EmitStoreOfScalar(Orig, OrigLVal);
5481 RCG.emitAggregateType(CGF, Cnt);
5491 SizeValInChars = CGF.Builder.CreateIntCast(SizeValInChars, CGM.SizeTy,
5493 LValue SizeLVal = CGF.EmitLValueForField(ElemLVal, SizeFD);
5494 CGF.EmitStoreOfScalar(SizeValInChars, SizeLVal);
5496 LValue InitLVal = CGF.EmitLValueForField(ElemLVal, InitFD);
5498 CGF.EmitStoreOfScalar(InitAddr, InitLVal);
5500 LValue FiniLVal = CGF.EmitLValueForField(ElemLVal, FiniFD);
5504 CGF.EmitStoreOfScalar(FiniAddr, FiniLVal);
5506 LValue CombLVal = CGF.EmitLValueForField(ElemLVal, CombFD);
5510 CGF.EmitStoreOfScalar(CombAddr, CombLVal);
5512 LValue FlagsLVal = CGF.EmitLValueForField(ElemLVal, FlagsFD);
5514 CGF.EmitStoreOfScalar(
5518 CGF.EmitNullInitialization(FlagsLVal.getAddress(), FlagsLVal.getType());
5523 llvm::Value *IdentTLoc = emitUpdateLocation(CGF, Loc);
5524 llvm::Value *GTid = CGF.Builder.CreateIntCast(getThreadID(CGF, Loc),
5531 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
5533 return CGF.EmitRuntimeCall(
5540 CGF.Builder.CreateIntCast(getThreadID(CGF, Loc), CGM.IntTy,
5543 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(TaskRedInput.getPointer(),
5545 return CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
5550 void CGOpenMPRuntime::emitTaskReductionFini(CodeGenFunction &CGF,
5555 llvm::Value *IdentTLoc = emitUpdateLocation(CGF, Loc);
5556 llvm::Value *GTid = CGF.Builder.CreateIntCast(getThreadID(CGF, Loc),
5562 (void)CGF.EmitRuntimeCall(
5568 void CGOpenMPRuntime::emitTaskReductionFixups(CodeGenFunction &CGF,
5576 llvm::Value *SizeVal = CGF.Builder.CreateIntCast(Sizes.second, CGM.SizeTy,
5579 CGF, CGM.getContext().getSizeType(),
5581 CGF.Builder.CreateStore(SizeVal, SizeAddr, /*IsVolatile=*/false);
5585 Address CGOpenMPRuntime::getTaskReductionItem(CodeGenFunction &CGF,
5591 llvm::Value *Args[] = {CGF.Builder.CreateIntCast(getThreadID(CGF, Loc),
5595 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
5596 SharedLVal.getPointer(CGF), CGM.VoidPtrTy)};
5598 CGF.EmitRuntimeCall(
5602 CGF.Int8Ty, SharedLVal.getAlignment());
5605 void CGOpenMPRuntime::emitTaskwaitCall(CodeGenFunction &CGF, SourceLocation Loc,
5607 if (!CGF.HaveInsertPoint())
5610 if (CGF.CGM.getLangOpts().OpenMPIRBuilder && Data.Dependences.empty()) {
5612 OMPBuilder.createTaskwait(CGF.Builder);
5614 llvm::Value *ThreadID = getThreadID(CGF, Loc);
5615 llvm::Value *UpLoc = emitUpdateLocation(CGF, Loc);
5620 emitDependClause(CGF, Data.Dependences, Loc);
5626 DepWaitTaskArgs[3] = DependenciesArray.emitRawPointer(CGF);
5627 DepWaitTaskArgs[4] = CGF.Builder.getInt32(0);
5628 DepWaitTaskArgs[5] = llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
5630 llvm::ConstantInt::get(CGF.Int32Ty, Data.HasNowaitClause);
5632 CodeGenFunction::RunCleanupsScope LocalScope(CGF);
5638 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
5648 CGF.EmitRuntimeCall(
5654 if (auto *Region = dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo))
5655 Region->emitUntiedSwitch(CGF);
5658 void CGOpenMPRuntime::emitInlinedDirective(CodeGenFunction &CGF,
5662 if (!CGF.HaveInsertPoint())
5664 InlinedOpenMPRegionRAII Region(CGF, CodeGen, InnerKind, HasCancel,
5668 CGF.CapturedStmtInfo->EmitBody(CGF, /*S=*/nullptr);
5697 CodeGenFunction &CGF, SourceLocation Loc,
5699 if (!CGF.HaveInsertPoint())
5704 dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo)) {
5709 emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
5710 CGF.Builder.getInt32(getCancellationKind(CancelRegion))};
5712 llvm::Value *Result = CGF.EmitRuntimeCall(
5720 llvm::BasicBlock *ExitBB = CGF.createBasicBlock(".cancel.exit");
5721 llvm::BasicBlock *ContBB = CGF.createBasicBlock(".cancel.continue");
5722 llvm::Value *Cmp = CGF.Builder.CreateIsNotNull(Result);
5723 CGF.Builder.CreateCondBr(Cmp, ExitBB, ContBB);
5724 CGF.EmitBlock(ExitBB);
5726 emitBarrierCall(CGF, Loc, OMPD_unknown, /*EmitChecks=*/false);
5729 CGF.getOMPCancelDestination(OMPRegionInfo->getDirectiveKind());
5730 CGF.EmitBranchThroughCleanup(CancelDest);
5731 CGF.EmitBlock(ContBB, /*IsFinished=*/true);
5736 void CGOpenMPRuntime::emitCancelCall(CodeGenFunction &CGF, SourceLocation Loc,
5739 if (!CGF.HaveInsertPoint())
5745 dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo)) {
5747 OMPRegionInfo](CodeGenFunction &CGF, PrePostActionTy &) {
5748 CGOpenMPRuntime &RT = CGF.CGM.getOpenMPRuntime();
5750 RT.emitUpdateLocation(CGF, Loc), RT.getThreadID(CGF, Loc),
5751 CGF.Builder.getInt32(getCancellationKind(CancelRegion))};
5753 llvm::Value *Result = CGF.EmitRuntimeCall(
5759 llvm::BasicBlock *ExitBB = CGF.createBasicBlock(".cancel.exit");
5760 llvm::BasicBlock *ContBB = CGF.createBasicBlock(".cancel.continue");
5761 llvm::Value *Cmp = CGF.Builder.CreateIsNotNull(Result);
5762 CGF.Builder.CreateCondBr(Cmp, ExitBB, ContBB);
5763 CGF.EmitBlock(ExitBB);
5765 RT.emitBarrierCall(CGF, Loc, OMPD_unknown, /*EmitChecks=*/false);
5768 CGF.getOMPCancelDestination(OMPRegionInfo->getDirectiveKind());
5769 CGF.EmitBranchThroughCleanup(CancelDest);
5770 CGF.EmitBlock(ContBB, /*IsFinished=*/true);
5773 emitIfClause(CGF, IfCond, ThenGen,
5777 ThenRCG(CGF);
5791 void Enter(CodeGenFunction &CGF) override {
5792 if (!CGF.HaveInsertPoint())
5795 CGF.CGM.getOpenMPRuntime().emitUsesAllocatorsInit(
5796 CGF, AllocatorData.first, AllocatorData.second);
5799 void Exit(CodeGenFunction &CGF) override {
5800 if (!CGF.HaveInsertPoint())
5803 CGF.CGM.getOpenMPRuntime().emitUsesAllocatorsFini(CGF,
5831 void CGOpenMPRuntime::emitUsesAllocatorsInit(CodeGenFunction &CGF,
5834 llvm::Value *ThreadId = getThreadID(CGF, Allocator->getExprLoc());
5835 ThreadId = CGF.Builder.CreateIntCast(ThreadId, CGF.IntTy, /*isSigned=*/true);
5837 llvm::Value *MemSpaceHandle = llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
5839 CGF.IntTy, cast<ConstantArrayType>(
5843 LValue AllocatorTraitsLVal = CGF.EmitLValue(AllocatorTraits);
5844 Address Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
5845 AllocatorTraitsLVal.getAddress(), CGF.VoidPtrPtrTy, CGF.VoidPtrTy);
5846 AllocatorTraitsLVal = CGF.MakeAddrLValue(Addr, CGF.getContext().VoidPtrTy,
5849 llvm::Value *Traits = Addr.emitRawPointer(CGF);
5852 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
5856 CGF.EmitAutoVarAlloca(*cast<VarDecl>(
5858 LValue AllocatorLVal = CGF.EmitLValue(Allocator->IgnoreParenImpCasts());
5860 CGF.EmitScalarConversion(AllocatorVal, CGF.getContext().VoidPtrTy,
5862 CGF.EmitStoreOfScalar(AllocatorVal, AllocatorLVal);
5865 void CGOpenMPRuntime::emitUsesAllocatorsFini(CodeGenFunction &CGF,
5867 llvm::Value *ThreadId = getThreadID(CGF, Allocator->getExprLoc());
5868 ThreadId = CGF.Builder.CreateIntCast(ThreadId, CGF.IntTy, /*isSigned=*/true);
5869 LValue AllocatorLVal = CGF.EmitLValue(Allocator->IgnoreParenImpCasts());
5871 CGF.EmitLoadOfScalar(AllocatorLVal, Allocator->getExprLoc());
5872 AllocatorVal = CGF.EmitScalarConversion(AllocatorVal, Allocator->getType(),
5873 CGF.getContext().VoidPtrTy,
5875 (void)CGF.EmitRuntimeCall(
5882 const OMPExecutableDirective &D, CodeGenFunction &CGF,
5889 getNumTeamsExprForTargetDirective(CGF, D, Attrs.MinTeams, MaxTeamsVal);
5890 getNumThreadsExprForTargetDirective(CGF, D, MaxThreadsVal,
5928 CodeGenFunction CGF(CGM, true);
5930 [&CGF, &D, &CodeGen](StringRef EntryFnName) {
5934 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo);
5935 return CGF.GenerateOpenMPCapturedStmtFunction(CS, D.getBeginLoc());
6008 CodeGenFunction &CGF, const OMPExecutableDirective &D, int32_t &MinTeamsVal,
6020 CGOpenMPRuntime::getSingleCompoundChild(CGF.getContext(), Body);
6028 if (NumTeams->isIntegerConstantExpr(CGF.getContext()))
6030 NumTeams->getIntegerConstantExpr(CGF.getContext()))
6053 if (NumTeams->isIntegerConstantExpr(CGF.getContext()))
6054 if (auto Constant = NumTeams->getIntegerConstantExpr(CGF.getContext()))
6136 CodeGenFunction &CGF, const OMPExecutableDirective &D) {
6137 assert(!CGF.getLangOpts().OpenMPIsTargetDevice &&
6140 CGBuilderTy &Bld = CGF.Builder;
6143 getNumTeamsExprForTargetDirective(CGF, D, MinNT, MaxNT);
6150 CGOpenMPInnerExprInfo CGInfo(CGF, *CS);
6151 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo);
6152 llvm::Value *NumTeamsVal = CGF.EmitScalarExpr(NumTeams,
6154 return Bld.CreateIntCast(NumTeamsVal, CGF.Int32Ty,
6162 CodeGenFunction::RunCleanupsScope NumTeamsScope(CGF);
6163 llvm::Value *NumTeamsVal = CGF.EmitScalarExpr(NumTeams,
6165 return Bld.CreateIntCast(NumTeamsVal, CGF.Int32Ty,
6174 return llvm::ConstantInt::get(CGF.Int32Ty, MinNT);
6181 static void getNumThreads(CodeGenFunction &CGF, const CapturedStmt *CS,
6185 CGF.getContext(), CS->getCapturedStmt());
6194 CGOpenMPInnerExprInfo CGInfo(CGF, *CS);
6195 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo);
6207 if (CondExpr->EvaluateAsBooleanCondition(Result, CGF.getContext())) {
6213 CodeGenFunction::LexicalScope Scope(CGF, CondExpr->getSourceRange());
6218 CGF.EmitVarDecl(cast<VarDecl>(*I));
6221 CGF.EmitAutoVarAlloca(cast<VarDecl>(*I));
6222 CGF.EmitAutoVarCleanups(Emission);
6225 *CondVal = CGF.EvaluateExprAsBool(CondExpr);
6233 CGOpenMPInnerExprInfo CGInfo(CGF, *CS);
6234 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo);
6238 if (NTExpr->isIntegerConstantExpr(CGF.getContext()))
6239 if (auto Constant = NTExpr->getIntegerConstantExpr(CGF.getContext()))
6251 CodeGenFunction::LexicalScope Scope(CGF, NTExpr->getSourceRange());
6256 CGF.EmitVarDecl(cast<VarDecl>(*I));
6259 CGF.EmitAutoVarAlloca(cast<VarDecl>(*I));
6260 CGF.EmitAutoVarCleanups(Emission);
6273 CodeGenFunction &CGF, const OMPExecutableDirective &D, int32_t &UpperBound,
6275 assert((!CGF.getLangOpts().OpenMPIsTargetDevice || UpperBoundOnly) &&
6286 if (E->isIntegerConstantExpr(CGF.getContext())) {
6287 if (auto Constant = E->getIntegerConstantExpr(CGF.getContext()))
6308 getNumThreads(CGF, CS, NTPtr, UpperBound, UpperBoundOnly, CondVal);
6310 CGF.getContext(), CS->getCapturedStmt());
6318 CGOpenMPInnerExprInfo CGInfo(CGF, *CS);
6319 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo);
6321 CGF,
6327 CGF.EmitVarDecl(cast<VarDecl>(*I));
6330 CGF.EmitAutoVarAlloca(cast<VarDecl>(*I));
6331 CGF.EmitAutoVarCleanups(Emission);
6346 CGF.getContext(), CS->getCapturedStmt());
6351 getNumThreads(CGF, CS, NTPtr, UpperBound, UpperBoundOnly, CondVal);
6359 CodeGenFunction::RunCleanupsScope ThreadLimitScope(CGF);
6365 getNumThreads(CGF, CS, NTPtr, UpperBound, UpperBoundOnly, CondVal);
6367 CGF.getContext(), CS->getCapturedStmt());
6371 getNumThreads(CGF, CS, NTPtr, UpperBound, UpperBoundOnly, CondVal);
6378 CodeGenFunction::RunCleanupsScope ThreadLimitScope(CGF);
6383 getNumThreads(CGF, D.getInnermostCapturedStmt(), NTPtr, UpperBound,
6405 if (Cond->EvaluateAsBooleanCondition(Result, CGF.getContext())) {
6409 CodeGenFunction::RunCleanupsScope Scope(CGF);
6410 *CondVal = CGF.EvaluateExprAsBool(Cond);
6415 CodeGenFunction::RunCleanupsScope ThreadLimitScope(CGF);
6421 CodeGenFunction::RunCleanupsScope NumThreadsScope(CGF);
6438 CodeGenFunction &CGF, const OMPExecutableDirective &D) {
6446 CGF, D, UpperBound, /* UpperBoundOnly */ false, &CondVal,
6452 CGF.EmitScalarExpr(ThreadLimitExpr, /*IgnoreResultAssign=*/true);
6453 ThreadLimitVal = CGF.Builder.CreateIntCast(ThreadLimitVal, CGF.Int32Ty,
6459 NumThreadsVal = CGF.Builder.getInt32(UpperBound);
6461 NumThreadsVal = CGF.EmitScalarExpr(NT, /*IgnoreResultAssign=*/true);
6462 NumThreadsVal = CGF.Builder.CreateIntCast(NumThreadsVal, CGF.Int32Ty,
6472 NumThreadsVal = CGF.Builder.getInt32(0);
6478 CodeGenFunction::RunCleanupsScope Scope(CGF);
6479 NumThreadsVal = CGF.Builder.CreateSelect(CondVal, NumThreadsVal,
6480 CGF.Builder.getInt32(1));
6486 NumThreadsVal = CGF.Builder.CreateSelect(
6487 CGF.Builder.CreateICmpULT(ThreadLimitVal, NumThreadsVal),
6623 CodeGenFunction &CGF;
6653 CGF.getTypeSize(OAE->getBase()->getType()->getPointeeType());
6655 llvm::Value *Sz = CGF.EmitScalarExpr(SE);
6656 Sz = CGF.EmitScalarConversion(Sz, SE->getType(),
6657 CGF.getContext().getSizeType(),
6659 Size = CGF.Builder.CreateNUWMul(Size, Sz);
6670 // on CGF.getTypeSize(E->getType()).
6681 return CGF.getTypeSize(BaseTy);
6685 ElemSize = CGF.getTypeSize(PTy->getPointeeType().getCanonicalType());
6689 ElemSize = CGF.getTypeSize(ATy->getElementType().getCanonicalType());
6698 llvm::Value *LengthVal = CGF.EmitScalarExpr(LenExpr);
6699 LengthVal = CGF.EmitScalarConversion(LengthVal, LenExpr->getType(),
6700 CGF.getContext().getSizeType(),
6702 return CGF.Builder.CreateNUWMul(LengthVal, ElemSize);
6707 llvm::Value *LengthVal = CGF.getTypeSize(BaseTy);
6708 llvm::Value *LBVal = CGF.EmitScalarExpr(OAE->getLowerBound());
6709 LBVal = CGF.EmitScalarConversion(LBVal, OAE->getLowerBound()->getType(),
6710 CGF.getContext().getSizeType(),
6712 LBVal = CGF.Builder.CreateNUWMul(LBVal, ElemSize);
6713 llvm::Value *Cmp = CGF.Builder.CreateICmpUGT(LengthVal, LBVal);
6714 llvm::Value *TrueVal = CGF.Builder.CreateNUWSub(LengthVal, LBVal);
6715 LengthVal = CGF.Builder.CreateSelect(
6716 Cmp, TrueVal, llvm::ConstantInt::get(CGF.SizeTy, 0));
6719 return CGF.getTypeSize(ExprTy);
6807 if (!Length->EvaluateAsInt(Result, CGF.getContext()))
7039 BP = CGF.LoadCXXThisAddress();
7043 BP = CGF.EmitOMPSharedLValue(AssocExpr).getAddress();
7047 CGF.EmitScalarExpr(OAShE->getBase()),
7048 CGF.ConvertTypeForMem(OAShE->getBase()->getType()->getPointeeType()),
7049 CGF.getContext().getTypeAlignInChars(OAShE->getBase()->getType()));
7053 BP = CGF.EmitOMPSharedLValue(AssocExpr).getAddress();
7061 CGF.CGM.getOpenMPRuntime().hasRequiresUnifiedSharedMemory())) {
7063 BP = CGF.CGM.getOpenMPRuntime().getAddrOfDeclareTargetVar(VD);
7079 (CGF.CGM.getOpenMPRuntime().hasRequiresUnifiedSharedMemory() ||
7081 BP = CGF.EmitLoadOfPointer(BP, Ty->castAs<PointerType>());
7157 BP = CGF.EmitLoadOfPointer(BP, Ty->castAs<PointerType>());
7225 auto &&EmitMemberExprBase = [](CodeGenFunction &CGF,
7235 CGF.EmitPointerWithAlignment(BaseExpr, &BaseInfo, &TBAAInfo);
7237 BaseLV = CGF.MakeAddrLValue(Addr, PtrTy, BaseInfo, TBAAInfo);
7239 BaseLV = CGF.EmitOMPSharedLValue(BaseExpr);
7245 Address(CGF.EmitScalarExpr(OAShE->getBase()),
7246 CGF.ConvertTypeForMem(
7248 CGF.getContext().getTypeAlignInChars(
7252 LValue BaseLVal = EmitMemberExprBase(CGF, ME);
7253 LowestElem = CGF.EmitLValueForFieldInitialization(
7256 LB = CGF.EmitLoadOfReferenceLValue(LowestElem, MapDecl->getType())
7260 CGF.EmitOMPSharedLValue(I->getAssociatedExpression())
7282 CharUnits TypeSize = CGF.getContext().getTypeSizeInChars(
7284 Address HB = CGF.Builder.CreateConstGEP(
7285 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
7286 LowestElem, CGF.VoidPtrTy, CGF.Int8Ty),
7316 LValue BaseLVal = EmitMemberExprBase(CGF, ME);
7318 CGF.EmitLValueForFieldInitialization(BaseLVal, FD)
7322 CGF.EmitOMPSharedLValue(MC.getAssociatedExpression())
7325 llvm::Value *ComponentLBPtr = ComponentLB.emitRawPointer(CGF);
7326 llvm::Value *LBPtr = LB.emitRawPointer(CGF);
7327 Size = CGF.Builder.CreatePtrDiff(CGF.Int8Ty, ComponentLBPtr,
7334 CombinedInfo.BasePointers.push_back(BP.emitRawPointer(CGF));
7337 CombinedInfo.Pointers.push_back(LB.emitRawPointer(CGF));
7338 CombinedInfo.Sizes.push_back(CGF.Builder.CreateIntCast(
7339 Size, CGF.Int64Ty, /*isSigned=*/true));
7344 LB = CGF.Builder.CreateConstGEP(ComponentLB, 1);
7347 CombinedInfo.BasePointers.push_back(BP.emitRawPointer(CGF));
7350 CombinedInfo.Pointers.push_back(LB.emitRawPointer(CGF));
7351 llvm::Value *LBPtr = LB.emitRawPointer(CGF);
7352 Size = CGF.Builder.CreatePtrDiff(
7353 CGF.Int8Ty, CGF.Builder.CreateConstGEP(HB, 1).emitRawPointer(CGF),
7356 CGF.Builder.CreateIntCast(Size, CGF.Int64Ty, /*isSigned=*/true));
7374 CombinedInfo.BasePointers.push_back(BP.emitRawPointer(CGF));
7377 CombinedInfo.Pointers.push_back(LB.emitRawPointer(CGF));
7378 CombinedInfo.Sizes.push_back(CGF.Builder.CreateIntCast(
7379 Size, CGF.Int64Ty, /*isSigned=*/true));
7385 BP.emitRawPointer(CGF));
7388 StructBaseCombinedInfo.Pointers.push_back(LB.emitRawPointer(CGF));
7389 StructBaseCombinedInfo.Sizes.push_back(CGF.Builder.CreateIntCast(
7390 Size, CGF.Int64Ty, /*isSigned=*/true));
7453 CGF.EmitArraySectionExpr(OASE, /*IsLowerBound=*/false)
7466 CGF.EmitArraySectionExpr(OASE, /*IsLowerBound=*/false)
7496 BP = CGF.EmitLoadOfPointer(BP, Ty->castAs<PointerType>());
7508 const ASTContext &Context = CGF.getContext();
7512 MapValuesArrayTy CurOffsets = {llvm::ConstantInt::get(CGF.CGM.Int64Ty, 0)};
7513 MapValuesArrayTy CurCounts = {llvm::ConstantInt::get(CGF.CGM.Int64Ty, 1)};
7515 MapValuesArrayTy DimSizes{llvm::ConstantInt::get(CGF.CGM.Int64Ty, 1)};
7560 llvm::ConstantInt::get(CGF.Int64Ty, ElementTypeSize));
7568 llvm::ConstantInt::get(CGF.Int64Ty, CAT->getZExtSize()));
7570 DimSizes.push_back(CGF.Builder.CreateIntCast(
7571 CGF.EmitScalarExpr(VAT->getSizeExpr()), CGF.Int64Ty,
7580 llvm::ConstantInt::get(CGF.CGM.Int64Ty, ElementTypeSize);
7594 llvm::Value *Offset = CGF.Builder.CreateIntCast(
7595 CGF.EmitScalarExpr(AE->getIdx()), CGF.Int64Ty,
7598 CurCounts.push_back(llvm::ConstantInt::get(CGF.Int64Ty, /*V=*/1));
7613 Offset = llvm::ConstantInt::get(CGF.Int64Ty, 0);
7615 Offset = CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(OffsetExpr),
7616 CGF.Int64Ty,
7631 Count = llvm::ConstantInt::get(CGF.Int64Ty, 1);
7640 ? CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(StrideExpr),
7641 CGF.Int64Ty, /*isSigned=*/false)
7644 Count = CGF.Builder.CreateUDiv(
7645 CGF.Builder.CreateNUWSub(*DI, Offset), Stride);
7647 Count = CGF.Builder.CreateNUWSub(*DI, Offset);
7650 Count = CGF.EmitScalarExpr(CountExpr);
7652 Count = CGF.Builder.CreateIntCast(Count, CGF.Int64Ty, /*isSigned=*/false);
7665 ? CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(StrideExpr),
7666 CGF.Int64Ty, /*isSigned=*/false)
7668 DimProd = CGF.Builder.CreateNUWMul(DimProd, *(DI - 1));
7670 CurStrides.push_back(CGF.Builder.CreateNUWMul(DimProd, Stride));
7715 const CGRecordLayout &RL = CGF.getTypes().getCGRecordLayout(RD);
7733 if (isEmptyRecordForLayout(CGF.getContext(), BaseTy) ||
7734 CGF.getContext()
7747 if (isEmptyRecordForLayout(CGF.getContext(), BaseTy))
7762 !isEmptyFieldForLayout(CGF.getContext(), Field)) {
7883 CodeGenFunction &CGF, bool IsDevAddr) {
7891 llvm::Constant::getNullValue(CGF.Int64Ty));
7899 &InfoGen](CodeGenFunction &CGF, const Expr *IE, const ValueDecl *VD,
7922 Ptr = CGF.EmitLValue(IE).getPointer(CGF);
7924 Ptr = CGF.EmitScalarExpr(IE);
7926 Ptr = CGF.EmitLoadOfScalar(CGF.EmitLValue(IE), IE->getExprLoc());
7928 UseDeviceDataCombinedInfoGen(VD, Ptr, CGF, IsDevAddr);
7932 auto &&IsMapInfoExist = [&Info](CodeGenFunction &CGF, const ValueDecl *VD,
7962 if (CGF.CGM.getOpenMPRuntime().hasRequiresUnifiedSharedMemory() ||
7999 if (IsMapInfoExist(CGF, VD, IE, /*IsDevAddr=*/false))
8001 MapInfoGen(CGF, IE, VD, Components, C->isImplicit(),
8021 if (IsMapInfoExist(CGF, VD, IE, /*IsDevAddr=*/true))
8023 MapInfoGen(CGF, IE, VD, Components, C->isImplicit(),
8121 Ptr = this->CGF.EmitLValue(L.IE).getPointer(CGF);
8123 Ptr = this->CGF.EmitScalarExpr(L.IE);
8132 BasePtr = this->CGF.EmitLValue(L.IE).getPointer(CGF);
8133 Ptr = this->CGF.EmitLoadOfScalar(this->CGF.EmitLValue(L.IE),
8150 llvm::Constant::getNullValue(this->CGF.Int64Ty));
8178 MappableExprsHandler(const OMPExecutableDirective &Dir, CodeGenFunction &CGF)
8179 : CurDir(&Dir), CGF(CGF) {
8224 MappableExprsHandler(const OMPDeclareMapperDecl &Dir, CodeGenFunction &CGF)
8225 : CurDir(&Dir), CGF(CGF) {}
8249 CombinedInfo.BasePointers.push_back(PartialStruct.Base.emitRawPointer(CGF));
8253 llvm::Value *LB = LBAddr.emitRawPointer(CGF);
8255 CGF.CurFuncDecl ? dyn_cast<CXXMethodDecl>(CGF.CurFuncDecl) : nullptr;
8267 CombinedInfo.Pointers.push_back(PartialStruct.Base.emitRawPointer(CGF));
8270 CGF.Builder.CreateIntCast(CGF.getTypeSize(Ty), CGF.Int64Ty,
8276 llvm::Value *HB = HBAddr.emitRawPointer(CGF);
8277 llvm::Value *HAddr = CGF.Builder.CreateConstGEP1_32(
8279 llvm::Value *CLAddr = CGF.Builder.CreatePointerCast(LB, CGF.VoidPtrTy);
8280 llvm::Value *CHAddr = CGF.Builder.CreatePointerCast(HAddr, CGF.VoidPtrTy);
8281 llvm::Value *Diff = CGF.Builder.CreatePtrDiff(CGF.Int8Ty, CHAddr, CLAddr);
8282 llvm::Value *Size = CGF.Builder.CreateIntCast(Diff, CGF.Int64Ty,
8364 Address VDAddr(Arg, CGF.ConvertTypeForMem(VDType),
8365 CGF.getContext().getDeclAlign(VD));
8366 LValue VDLVal = CGF.MakeAddrLValue(VDAddr, VDType);
8372 CGF.EmitLValueForFieldInitialization(VDLVal, ThisCapture);
8373 LValue ThisLValVal = CGF.EmitLValueForField(VDLVal, ThisCapture);
8374 LambdaPointers.try_emplace(ThisLVal.getPointer(CGF),
8375 VDLVal.getPointer(CGF));
8377 CombinedInfo.BasePointers.push_back(ThisLVal.getPointer(CGF));
8380 CombinedInfo.Pointers.push_back(ThisLValVal.getPointer(CGF));
8382 CGF.Builder.CreateIntCast(CGF.getTypeSize(CGF.getContext().VoidPtrTy),
8383 CGF.Int64Ty, /*isSigned=*/true));
8399 LValue VarLVal = CGF.EmitLValueForFieldInitialization(VDLVal, It->second);
8401 LValue VarLValVal = CGF.EmitLValueForField(VDLVal, It->second);
8402 LambdaPointers.try_emplace(VarLVal.getPointer(CGF),
8403 VDLVal.getPointer(CGF));
8405 CombinedInfo.BasePointers.push_back(VarLVal.getPointer(CGF));
8408 CombinedInfo.Pointers.push_back(VarLValVal.getPointer(CGF));
8409 CombinedInfo.Sizes.push_back(CGF.Builder.CreateIntCast(
8410 CGF.getTypeSize(
8412 CGF.Int64Ty, /*isSigned=*/true));
8414 RValue VarRVal = CGF.EmitLoadOfLValue(VarLVal, RD->getLocation());
8415 LambdaPointers.try_emplace(VarLVal.getPointer(CGF),
8416 VDLVal.getPointer(CGF));
8418 CombinedInfo.BasePointers.push_back(VarLVal.getPointer(CGF));
8422 CombinedInfo.Sizes.push_back(llvm::ConstantInt::get(CGF.Int64Ty, 0));
8493 CombinedInfo.Sizes.push_back(CGF.Builder.CreateIntCast(
8494 CGF.getTypeSize(CGF.getContext().VoidPtrTy), CGF.Int64Ty,
8746 CGF.Builder.CreateIntCast(CGF.getTypeSize(PtrTy->getPointeeType()),
8747 CGF.Int64Ty, /*isSigned=*/true));
8763 CombinedInfo.Sizes.push_back(CGF.Builder.CreateIntCast(
8764 CGF.getTypeSize(RI.getType()), CGF.Int64Ty, /*isSigned=*/true));
8769 CombinedInfo.Sizes.push_back(llvm::Constant::getNullValue(CGF.Int64Ty));
8778 CombinedInfo.Sizes.push_back(CGF.Builder.CreateIntCast(
8779 CGF.getTypeSize(ElementType), CGF.Int64Ty, /*isSigned=*/true));
8791 Address PtrAddr = CGF.EmitLoadOfReference(CGF.MakeAddrLValue(
8792 CV, ElementType, CGF.getContext().getDeclAlign(VD),
8794 CombinedInfo.Pointers.push_back(PtrAddr.emitRawPointer(CGF));
8830 emitMappingInformation(CodeGenFunction &CGF, llvm::OpenMPIRBuilder &OMPBuilder,
8849 PrintingPolicy P(CGF.getContext().getLangOpts());
8856 PresumedLoc PLoc = CGF.getContext().getSourceManager().getPresumedLoc(Loc);
8865 CodeGenFunction &CGF, MappableExprsHandler::MapCombinedInfoTy &CombinedInfo,
8868 CodeGenModule &CGM = CGF.CGM;
8871 InsertPointTy AllocaIP(CGF.AllocaInsertPt->getParent(),
8872 CGF.AllocaInsertPt->getIterator());
8873 InsertPointTy CodeGenIP(CGF.Builder.GetInsertBlock(),
8874 CGF.Builder.GetInsertPoint());
9041 CodeGenFunction *CGF) {
9105 if (CGF)
9106 FunctionUDMMap[CGF->CurFn].push_back(D);
9119 CodeGenFunction &CGF, const OMPExecutableDirective &D,
9120 llvm::function_ref<llvm::Value *(CodeGenFunction &CGF,
9131 return llvm::ConstantInt::get(CGF.Int64Ty, 0);
9134 if (llvm::Value *NumIterations = SizeEmitter(CGF, *LD))
9136 return llvm::ConstantInt::get(CGF.Int64Ty, 0);
9144 bool OffloadingMandatory, CodeGenFunction &CGF) {
9146 CGF.Builder.CreateUnreachable();
9150 CGF.GenerateOpenMPCapturedVars(CS, CapturedVars);
9152 OMPRuntime->emitOutlinedFunctionCall(CGF, D.getBeginLoc(), OutlinedFn,
9159 CodeGenFunction &CGF) {
9166 llvm::Value *DevVal = CGF.EmitScalarExpr(Device.getPointer());
9168 CGF.Builder.CreateIntCast(DevVal, CGF.Int64Ty, /*isSigned=*/true);
9170 DeviceID = CGF.Builder.getInt64(OMP_DEVICEID_UNDEF);
9176 CodeGenFunction &CGF) {
9177 llvm::Value *DynCGroupMem = CGF.Builder.getInt32(0);
9180 CodeGenFunction::RunCleanupsScope DynCGroupMemScope(CGF);
9181 llvm::Value *DynCGroupMemVal = CGF.EmitScalarExpr(
9183 DynCGroupMem = CGF.Builder.CreateIntCast(DynCGroupMemVal, CGF.Int32Ty,
9189 MappableExprsHandler &MEHandler, CodeGenFunction &CGF,
9213 CurInfo.Sizes.push_back(CGF.Builder.CreateIntCast(
9214 CGF.getTypeSize(RI->getType()), CGF.Int64Ty, /*isSigned=*/true));
9263 genMapInfo(MappableExprsHandler &MEHandler, CodeGenFunction &CGF,
9269 CodeGenModule &CGM = CGF.CGM;
9275 return emitMappingInformation(CGF, OMPBuilder, MapExpr);
9285 static void genMapInfo(const OMPExecutableDirective &D, CodeGenFunction &CGF,
9291 MappableExprsHandler MEHandler(D, CGF);
9294 genMapInfoForCaptures(MEHandler, CGF, CS, CapturedVars, OMPBuilder,
9296 genMapInfo(MEHandler, CGF, CombinedInfo, OMPBuilder, MappedVarSet);
9301 emitClauseForBareTargetDirective(CodeGenFunction &CGF,
9307 CodeGenFunction::RunCleanupsScope Scope(CGF);
9309 llvm::Value *V = CGF.EmitScalarExpr(E);
9311 CGF.Builder.CreateIntCast(V, CGF.Int32Ty, /*isSigned=*/true));
9323 llvm::function_ref<llvm::Value *(CodeGenFunction &CGF,
9326 CodeGenFunction &CGF, CodeGenModule &CGM) {
9332 genMapInfo(D, CGF, CS, CapturedVars, OMPBuilder, CombinedInfo);
9334 emitOffloadingArraysAndArgs(CGF, CombinedInfo, Info, OMPBuilder,
9339 CGF.VoidPtrTy, CGM.getPointerAlign());
9341 Address(Info.RTArgs.PointersArray, CGF.VoidPtrTy, CGM.getPointerAlign());
9343 Address(Info.RTArgs.SizesArray, CGF.Int64Ty, CGM.getPointerAlign());
9345 Address(Info.RTArgs.MappersArray, CGF.VoidPtrTy, CGM.getPointerAlign());
9352 SizeEmitter](CodeGenFunction &CGF, PrePostActionTy &) {
9361 RequiresOuterTask, CS, OffloadingMandatory, CGF);
9369 InputInfo.BasePointersArray.emitRawPointer(CGF);
9370 llvm::Value *PointersArray = InputInfo.PointersArray.emitRawPointer(CGF);
9371 llvm::Value *SizesArray = InputInfo.SizesArray.emitRawPointer(CGF);
9372 llvm::Value *MappersArray = InputInfo.MappersArray.emitRawPointer(CGF);
9376 OffloadingMandatory, &CGF](llvm::OpenMPIRBuilder::InsertPointTy IP)
9378 CGF.Builder.restoreIP(IP);
9380 RequiresOuterTask, CS, OffloadingMandatory, CGF);
9381 return CGF.Builder.saveIP();
9388 emitClauseForBareTargetDirective<OMPNumTeamsClause>(CGF, D, NumTeams);
9389 emitClauseForBareTargetDirective<OMPThreadLimitClause>(CGF, D,
9392 NumTeams.push_back(OMPRuntime->emitNumTeamsForTargetDirective(CGF, D));
9394 OMPRuntime->emitNumThreadsForTargetDirective(CGF, D));
9397 llvm::Value *DeviceID = emitDeviceID(Device, CGF);
9398 llvm::Value *RTLoc = OMPRuntime->emitUpdateLocation(CGF, D.getBeginLoc());
9400 OMPRuntime->emitTargetNumIterationsCall(CGF, D, SizeEmitter);
9401 llvm::Value *DynCGGroupMem = emitDynCGGroupMem(D, CGF);
9403 CGF.AllocaInsertPt->getParent(), CGF.AllocaInsertPt->getIterator());
9415 CGF.Builder, OutlinedFnID, EmitTargetCallFallbackCB, Args, DeviceID,
9417 CGF.Builder.restoreIP(AfterIP);
9421 CGF.EmitOMPTargetTaskBasedDirective(D, ThenGen, InputInfo);
9423 OMPRuntime->emitInlinedDirective(CGF, D.getDirectiveKind(), ThenGen);
9431 bool OffloadingMandatory, CodeGenFunction &CGF) {
9436 OffloadingMandatory](CodeGenFunction &CGF, PrePostActionTy &) {
9438 RequiresOuterTask, CS, OffloadingMandatory, CGF);
9443 CGF.EmitOMPTargetTaskBasedDirective(D, ElseGen, InputInfo);
9445 OMPRuntime->emitInlinedDirective(CGF, D.getDirectiveKind(), ElseGen);
9450 CodeGenFunction &CGF, const OMPExecutableDirective &D,
9453 llvm::function_ref<llvm::Value *(CodeGenFunction &CGF,
9456 if (!CGF.HaveInsertPoint())
9473 auto &&ArgsCodegen = [&CS, &CapturedVars](CodeGenFunction &CGF,
9475 CGF.GenerateOpenMPCapturedVars(CS, CapturedVars);
9477 emitInlinedDirective(CGF, OMPD_unknown, ArgsCodegen);
9486 &MapNamesArray, SizeEmitter](CodeGenFunction &CGF,
9491 MapNamesArray, SizeEmitter, CGF, CGM);
9496 OffloadingMandatory](CodeGenFunction &CGF, PrePostActionTy &) {
9498 CS, OffloadingMandatory, CGF);
9507 emitIfClause(CGF, IfCond, TargetThenGen, TargetElseGen);
9510 ThenRCG(CGF);
9514 ElseRCG(CGF);
9830 CodeGenFunction &CGF, const OMPExecutableDirective &D) const {
9926 void CGOpenMPRuntime::emitTeamsCall(CodeGenFunction &CGF,
9931 if (!CGF.HaveInsertPoint())
9934 llvm::Value *RTLoc = emitUpdateLocation(CGF, Loc);
9935 CodeGenFunction::RunCleanupsScope Scope(CGF);
9940 CGF.Builder.getInt32(CapturedVars.size()), // Number of captured vars
9941 CGF.Builder.CreateBitCast(OutlinedFn, getKmpc_MicroPointerTy())};
9948 CGF.EmitRuntimeCall(RTLFn, RealArgs);
9951 void CGOpenMPRuntime::emitNumTeamsClause(CodeGenFunction &CGF,
9955 if (!CGF.HaveInsertPoint())
9958 llvm::Value *RTLoc = emitUpdateLocation(CGF, Loc);
9962 ? CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(NumTeams),
9963 CGF.CGM.Int32Ty, /* isSigned = */ true)
9964 : CGF.Builder.getInt32(0);
9968 ? CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(ThreadLimit),
9969 CGF.CGM.Int32Ty, /* isSigned = */ true)
9970 : CGF.Builder.getInt32(0);
9973 llvm::Value *PushNumTeamsArgs[] = {RTLoc, getThreadID(CGF, Loc), NumTeamsVal,
9975 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
9980 void CGOpenMPRuntime::emitThreadLimitClause(CodeGenFunction &CGF,
9983 llvm::Value *RTLoc = emitUpdateLocation(CGF, Loc);
9986 ? CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(ThreadLimit),
9987 CGF.CGM.Int32Ty, /* isSigned = */ true)
9988 : CGF.Builder.getInt32(0);
9991 llvm::Value *ThreadLimitArgs[] = {RTLoc, getThreadID(CGF, Loc),
9993 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
9999 CodeGenFunction &CGF, const OMPExecutableDirective &D, const Expr *IfCond,
10002 if (!CGF.HaveInsertPoint())
10013 IfCondVal = CGF.EvaluateExprAsBool(IfCond);
10018 DeviceID = CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(Device),
10019 CGF.Int64Ty, /*isSigned=*/true);
10021 DeviceID = CGF.Builder.getInt64(OMP_DEVICEID_UNDEF);
10028 CGF.Builder.restoreIP(CodeGenIP);
10030 MappableExprsHandler MEHandler(D, CGF);
10034 return emitMappingInformation(CGF, OMPBuilder, MapExpr);
10047 CGF.Builder.restoreIP(CodeGenIP);
10051 CodeGen(CGF);
10056 CodeGen(CGF);
10062 CodeGen(CGF);
10066 return InsertPointTy(CGF.Builder.GetInsertBlock(),
10067 CGF.Builder.GetInsertPoint());
10080 MFunc = CGF.CGM.getOpenMPRuntime().getOrCreateUserDefinedMapperFunc(
10087 llvm::Value *RTLoc = emitUpdateLocation(CGF, D.getBeginLoc());
10089 InsertPointTy AllocaIP(CGF.AllocaInsertPt->getParent(),
10090 CGF.AllocaInsertPt->getIterator());
10091 InsertPointTy CodeGenIP(CGF.Builder.GetInsertBlock(),
10092 CGF.Builder.GetInsertPoint());
10098 CGF.Builder.restoreIP(AfterIP);
10102 CodeGenFunction &CGF, const OMPExecutableDirective &D, const Expr *IfCond,
10104 if (!CGF.HaveInsertPoint())
10117 &MapNamesArray](CodeGenFunction &CGF, PrePostActionTy &) {
10121 DeviceID = CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(Device),
10122 CGF.Int64Ty, /*isSigned=*/true);
10124 DeviceID = CGF.Builder.getInt64(OMP_DEVICEID_UNDEF);
10129 CGF.Builder.getInt32(InputInfo.NumberOfTargetItems);
10132 llvm::Value *RTLoc = emitUpdateLocation(CGF, D.getBeginLoc());
10136 InputInfo.BasePointersArray.emitRawPointer(CGF),
10137 InputInfo.PointersArray.emitRawPointer(CGF),
10138 InputInfo.SizesArray.emitRawPointer(CGF), MapTypesArray, MapNamesArray,
10139 InputInfo.MappersArray.emitRawPointer(CGF)});
10229 OffloadingArgs.push_back(llvm::Constant::getNullValue(CGF.Int32Ty));
10230 OffloadingArgs.push_back(llvm::Constant::getNullValue(CGF.VoidPtrTy));
10231 OffloadingArgs.push_back(llvm::Constant::getNullValue(CGF.Int32Ty));
10232 OffloadingArgs.push_back(llvm::Constant::getNullValue(CGF.VoidPtrTy));
10234 CGF.EmitRuntimeCall(
10240 &MapNamesArray](CodeGenFunction &CGF,
10245 MappableExprsHandler MEHandler(D, CGF);
10246 genMapInfo(MEHandler, CGF, CombinedInfo, OMPBuilder);
10247 emitOffloadingArraysAndArgs(CGF, CombinedInfo, Info, OMPBuilder,
10255 CGF.VoidPtrTy, CGM.getPointerAlign());
10256 InputInfo.PointersArray = Address(Info.RTArgs.PointersArray, CGF.VoidPtrTy,
10259 Address(Info.RTArgs.SizesArray, CGF.Int64Ty, CGM.getPointerAlign());
10261 Address(Info.RTArgs.MappersArray, CGF.VoidPtrTy, CGM.getPointerAlign());
10265 CGF.EmitOMPTargetTaskBasedDirective(D, ThenGen, InputInfo);
10267 emitInlinedDirective(CGF, D.getDirectiveKind(), ThenGen);
10271 emitIfClause(CGF, IfCond, TargetThenGen,
10272 [](CodeGenFunction &CGF, PrePostActionTy &) {});
10275 ThenRCG(CGF);
10884 void Emit(CodeGenFunction &CGF, Flags /*flags*/) override {
10885 if (!CGF.HaveInsertPoint())
10887 CGF.EmitRuntimeCall(RTLFn, Args);
10892 void CGOpenMPRuntime::emitDoacrossInit(CodeGenFunction &CGF,
10895 if (!CGF.HaveInsertPoint())
10921 Address DimsAddr = CGF.CreateMemTemp(ArrayTy, "dims");
10922 CGF.EmitNullInitialization(DimsAddr, ArrayTy);
10926 LValue DimsLVal = CGF.MakeAddrLValue(
10927 CGF.Builder.CreateConstArrayGEP(DimsAddr, I), KmpDimTy);
10929 LValue UpperLVal = CGF.EmitLValueForField(
10931 llvm::Value *NumIterVal = CGF.EmitScalarConversion(
10932 CGF.EmitScalarExpr(NumIterations[I]), NumIterations[I]->getType(),
10934 CGF.EmitStoreOfScalar(NumIterVal, UpperLVal);
10936 LValue StrideLVal = CGF.EmitLValueForField(
10938 CGF.EmitStoreOfScalar(llvm::ConstantInt::getSigned(CGM.Int64Ty, /*V=*/1),
10945 emitUpdateLocation(CGF, D.getBeginLoc()),
10946 getThreadID(CGF, D.getBeginLoc()),
10948 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
10949 CGF.Builder.CreateConstArrayGEP(DimsAddr, 0).emitRawPointer(CGF),
10954 CGF.EmitRuntimeCall(RTLFn, Args);
10956 emitUpdateLocation(CGF, D.getEndLoc()), getThreadID(CGF, D.getEndLoc())};
10959 CGF.EHStack.pushCleanup<DoacrossCleanupTy>(NormalAndEHCleanup, FiniRTLFn,
10964 static void EmitDoacrossOrdered(CodeGenFunction &CGF, CodeGenModule &CGM,
10972 Address CntAddr = CGF.CreateMemTemp(ArrayTy, ".cnt.addr");
10976 llvm::Value *CntVal = CGF.EmitScalarConversion(
10977 CGF.EmitScalarExpr(CounterVal), CounterVal->getType(), Int64Ty,
10979 CGF.EmitStoreOfScalar(CntVal, CGF.Builder.CreateConstArrayGEP(CntAddr, I),
10984 CGF.Builder.CreateConstArrayGEP(CntAddr, 0).emitRawPointer(CGF)};
10996 CGF.EmitRuntimeCall(RTLFn, Args);
10999 void CGOpenMPRuntime::emitDoacrossOrdered(CodeGenFunction &CGF,
11002 CGF, CGM, C, emitUpdateLocation(CGF, C->getBeginLoc()),
11003 getThreadID(CGF, C->getBeginLoc()));
11006 void CGOpenMPRuntime::emitDoacrossOrdered(CodeGenFunction &CGF,
11009 CGF, CGM, C, emitUpdateLocation(CGF, C->getBeginLoc()),
11010 getThreadID(CGF, C->getBeginLoc()));
11013 void CGOpenMPRuntime::emitCall(CodeGenFunction &CGF, SourceLocation Loc,
11017 auto DL = ApplyDebugLocation::CreateDefaultArtificial(CGF, Loc);
11021 CGF.EmitNounwindRuntimeCall(Fn, Args);
11025 CGF.EmitRuntimeCall(Callee, Args);
11029 CodeGenFunction &CGF, SourceLocation Loc, llvm::FunctionCallee OutlinedFn,
11031 emitCall(CGF, Loc, OutlinedFn, Args);
11034 void CGOpenMPRuntime::emitFunctionProlog(CodeGenFunction &CGF, const Decl *D) {
11040 Address CGOpenMPRuntime::getParameterAddress(CodeGenFunction &CGF,
11043 return CGF.GetAddrOfLocalVar(NativeParam);
11048 static llvm::Value *getAllocatorVal(CodeGenFunction &CGF,
11052 AllocVal = CGF.EmitScalarExpr(Allocator);
11055 AllocVal = CGF.EmitScalarConversion(AllocVal, Allocator->getType(),
11056 CGF.getContext().VoidPtrTy,
11061 CGF.CGM.getTypes().ConvertType(CGF.getContext().VoidPtrTy));
11076 Address CGOpenMPRuntime::getAddressOfLocalVariable(CodeGenFunction &CGF,
11082 auto It = FunctionToUntiedTaskStackMap.find(CGF.CurFn);
11100 Size = CGF.getTypeSize(CVD->getType());
11102 Size = CGF.Builder.CreateNUWAdd(
11104 Size = CGF.Builder.CreateUDiv(Size, CGM.getSize(Align));
11105 Size = CGF.Builder.CreateNUWMul(Size, CGM.getSize(Align));
11110 llvm::Value *ThreadID = getThreadID(CGF, CVD->getBeginLoc());
11113 llvm::Value *AllocVal = getAllocatorVal(CGF, Allocator);
11123 llvm::Value *Addr = CGF.EmitRuntimeCall(
11129 Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
11130 Addr, CGF.ConvertTypeForMem(Ty), getName({CVD->getName(), ".addr"}));
11132 CGF.EmitStoreOfScalar(Addr, UntiedAddr, /*Volatile=*/false, Ty);
11147 void Emit(CodeGenFunction &CGF, Flags /*flags*/) override {
11148 if (!CGF.HaveInsertPoint())
11151 Args[0] = CGF.CGM.getOpenMPRuntime().getThreadID(
11152 CGF, SourceLocation::getFromRawEncoding(LocEncoding));
11153 Args[1] = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
11154 Addr.emitRawPointer(CGF), CGF.VoidPtrTy);
11155 llvm::Value *AllocVal = getAllocatorVal(CGF, AllocExpr);
11157 CGF.EmitRuntimeCall(RTLFn, Args);
11163 : Address(Addr, CGF.ConvertTypeForMem(CVD->getType()), Align);
11164 CGF.EHStack.pushCleanup<OMPAllocateCleanupTy>(
11169 dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo))
11170 Region->emitUntiedSwitch(CGF);
11176 bool CGOpenMPRuntime::isLocalVarInUntiedTask(CodeGenFunction &CGF,
11178 auto It = FunctionToUntiedTaskStackMap.find(CGF.CurFn);
11217 CodeGenFunction &CGF,
11220 : CGM(CGF.CGM), NeedToPush(!LocalVars.empty()) {
11224 CGF.CurFn, CGM.getOpenMPRuntime().UntiedLocalVarsStack.size());
11322 CodeGenFunction &CGF, const OMPExecutableDirective &S, LValue IVLVal)
11323 : CGM(CGF.CGM),
11350 Data.Fn = CGF.CurFn;
11354 CodeGenFunction &CGF, const OMPExecutableDirective &S)
11355 : CGM(CGF.CGM), Action(ActionToDo::DoNotPush) {
11367 Data.Fn = CGF.CurFn;
11374 CodeGenFunction &CGF, const OMPExecutableDirective &S) {
11375 return LastprivateConditionalRAII(CGF, S);
11394 Address CGOpenMPRuntime::emitLastprivateConditionalInit(CodeGenFunction &CGF,
11397 auto I = LastprivateConditionalToTypes.try_emplace(CGF.CurFn).first;
11410 Address Addr = CGF.CreateMemTemp(NewType, C.getDeclAlign(VD), VD->getName());
11411 BaseLVal = CGF.MakeAddrLValue(Addr, NewType, AlignmentSource::Decl);
11420 CGF.EmitLValueForField(BaseLVal, FiredField);
11421 CGF.EmitStoreOfScalar(
11422 llvm::ConstantInt::getNullValue(CGF.ConvertTypeForMem(C.CharTy)),
11424 return CGF.EmitLValueForField(BaseLVal, VDField).getAddress();
11498 void CGOpenMPRuntime::emitLastprivateConditionalUpdate(CodeGenFunction &CGF,
11505 llvm::Type *LLIVTy = CGF.ConvertTypeForMem(IVLVal.getType());
11511 CGF.MakeNaturalAlignRawAddrLValue(LastIV, IVLVal.getType());
11516 CGF.ConvertTypeForMem(LVal.getType()), UniqueDeclName);
11520 CGF.MakeRawAddrLValue(Last, LVal.getType(), LVal.getAlignment());
11524 llvm::Value *IVVal = CGF.EmitLoadOfScalar(IVLVal, Loc);
11532 Loc](CodeGenFunction &CGF, PrePostActionTy &Action) {
11533 Action.Enter(CGF);
11534 llvm::Value *LastIVVal = CGF.EmitLoadOfScalar(LastIVLVal, Loc);
11539 CmpRes = CGF.Builder.CreateICmpSLE(LastIVVal, IVVal);
11543 CmpRes = CGF.Builder.CreateICmpULE(LastIVVal, IVVal);
11545 llvm::BasicBlock *ThenBB = CGF.createBasicBlock("lp_cond_then");
11546 llvm::BasicBlock *ExitBB = CGF.createBasicBlock("lp_cond_exit");
11547 CGF.Builder.CreateCondBr(CmpRes, ThenBB, ExitBB);
11549 CGF.EmitBlock(ThenBB);
11552 CGF.EmitStoreOfScalar(IVVal, LastIVLVal);
11555 switch (CGF.getEvaluationKind(LVal.getType())) {
11557 llvm::Value *PrivVal = CGF.EmitLoadOfScalar(LVal, Loc);
11558 CGF.EmitStoreOfScalar(PrivVal, LastLVal);
11562 CodeGenFunction::ComplexPairTy PrivVal = CGF.EmitLoadOfComplex(LVal, Loc);
11563 CGF.EmitStoreOfComplex(PrivVal, LastLVal, /*isInit=*/false);
11571 CGF.EmitBranch(ExitBB);
11573 (void)ApplyDebugLocation::CreateEmpty(CGF);
11574 CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
11580 ThenRCG(CGF);
11582 emitCriticalRegion(CGF, UniqueDeclName, CodeGen, Loc);
11586 void CGOpenMPRuntime::checkAndEmitLastprivateConditional(CodeGenFunction &CGF,
11588 if (CGF.getLangOpts().OpenMP < 50 || LastprivateConditionalStack.empty())
11600 if (FoundFn != CGF.CurFn) {
11608 LValue PrivLVal = CGF.EmitLValue(FoundE);
11609 Address StructAddr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
11611 CGF.ConvertTypeForMem(CGF.getContext().getPointerType(StructTy)),
11612 CGF.ConvertTypeForMem(StructTy));
11614 CGF.MakeAddrLValue(StructAddr, StructTy, AlignmentSource::Decl);
11615 LValue FiredLVal = CGF.EmitLValueForField(BaseLVal, FiredDecl);
11616 CGF.EmitAtomicStore(RValue::get(llvm::ConstantInt::get(
11617 CGF.ConvertTypeForMem(FiredDecl->getType()), 1)),
11625 LValue LVal = CGF.EmitLValue(FoundE);
11626 emitLastprivateConditionalUpdate(CGF, IVLVal, UniqueDeclName, LVal,
11631 CodeGenFunction &CGF, const OMPExecutableDirective &D,
11633 if (CGF.getLangOpts().OpenMP < 50 || LastprivateConditionalStack.empty())
11638 if (It == Range.end() || It->Fn != CGF.CurFn)
11656 CGF.EmitLValueForField(BaseLVal, std::get<2>(I->getSecond()));
11657 llvm::Value *Res = CGF.EmitLoadOfScalar(FiredLVal, D.getBeginLoc());
11658 llvm::Value *Cmp = CGF.Builder.CreateIsNotNull(Res);
11659 llvm::BasicBlock *ThenBB = CGF.createBasicBlock("lpc.then");
11660 llvm::BasicBlock *DoneBB = CGF.createBasicBlock("lpc.done");
11662 CGF.Builder.CreateCondBr(Cmp, ThenBB, DoneBB);
11663 CGF.EmitBlock(ThenBB);
11664 Address Addr = CGF.GetAddrOfLocalVar(VD);
11667 LVal = CGF.EmitLoadOfReferenceLValue(Addr, VD->getType(),
11670 LVal = CGF.MakeAddrLValue(Addr, VD->getType().getNonReferenceType(),
11672 emitLastprivateConditionalUpdate(CGF, It->IVLVal, Pair.second, LVal,
11674 auto AL = ApplyDebugLocation::CreateArtificial(CGF);
11675 CGF.EmitBlock(DoneBB, /*IsFinal=*/true);
11681 CodeGenFunction &CGF, LValue PrivLVal, const VarDecl *VD,
11683 if (CGF.getLangOpts().OpenMP < 50)
11693 LValue LPLVal = CGF.MakeRawAddrLValue(
11695 llvm::Value *Res = CGF.EmitLoadOfScalar(LPLVal, Loc);
11696 CGF.EmitStoreOfScalar(Res, PrivLVal);
11700 CodeGenFunction &CGF, const OMPExecutableDirective &D,
11707 CodeGenFunction &CGF, const OMPExecutableDirective &D,
11721 void CGOpenMPSIMDRuntime::emitParallelCall(CodeGenFunction &CGF,
11731 CodeGenFunction &CGF, StringRef CriticalName,
11737 void CGOpenMPSIMDRuntime::emitMasterRegion(CodeGenFunction &CGF,
11743 void CGOpenMPSIMDRuntime::emitMaskedRegion(CodeGenFunction &CGF,
11750 void CGOpenMPSIMDRuntime::emitTaskyieldCall(CodeGenFunction &CGF,
11756 CodeGenFunction &CGF, const RegionCodeGenTy &TaskgroupOpGen,
11762 CodeGenFunction &CGF, const RegionCodeGenTy &SingleOpGen,
11769 void CGOpenMPSIMDRuntime::emitOrderedRegion(CodeGenFunction &CGF,
11776 void CGOpenMPSIMDRuntime::emitBarrierCall(CodeGenFunction &CGF,
11785 CodeGenFunction &CGF, SourceLocation Loc,
11791 void CGOpenMPSIMDRuntime::emitForDispatchDeinit(CodeGenFunction &CGF,
11797 CodeGenFunction &CGF, SourceLocation Loc, OpenMPDirectiveKind DKind,
11803 CodeGenFunction &CGF, SourceLocation Loc,
11808 void CGOpenMPSIMDRuntime::emitForOrderedIterationEnd(CodeGenFunction &CGF,
11815 void CGOpenMPSIMDRuntime::emitForStaticFinish(CodeGenFunction &CGF,
11821 llvm::Value *CGOpenMPSIMDRuntime::emitForNext(CodeGenFunction &CGF,
11829 void CGOpenMPSIMDRuntime::emitNumThreadsClause(CodeGenFunction &CGF,
11835 void CGOpenMPSIMDRuntime::emitProcBindClause(CodeGenFunction &CGF,
11841 Address CGOpenMPSIMDRuntime::getAddrOfThreadPrivate(CodeGenFunction &CGF,
11850 CodeGenFunction *CGF) {
11855 CodeGenFunction &CGF, QualType VarType, StringRef Name) {
11859 void CGOpenMPSIMDRuntime::emitFlush(CodeGenFunction &CGF,
11866 void CGOpenMPSIMDRuntime::emitTaskCall(CodeGenFunction &CGF, SourceLocation Loc,
11876 CodeGenFunction &CGF, SourceLocation Loc, const OMPLoopDirective &D,
11883 CodeGenFunction &CGF, SourceLocation Loc, ArrayRef<const Expr *> Privates,
11887 CGOpenMPRuntime::emitReduction(CGF, Loc, Privates, LHSExprs, RHSExprs,
11892 CodeGenFunction &CGF, SourceLocation Loc, ArrayRef<const Expr *> LHSExprs,
11897 void CGOpenMPSIMDRuntime::emitTaskReductionFini(CodeGenFunction &CGF,
11903 void CGOpenMPSIMDRuntime::emitTaskReductionFixups(CodeGenFunction &CGF,
11910 Address CGOpenMPSIMDRuntime::getTaskReductionItem(CodeGenFunction &CGF,
11917 void CGOpenMPSIMDRuntime::emitTaskwaitCall(CodeGenFunction &CGF,
11924 CodeGenFunction &CGF, SourceLocation Loc,
11929 void CGOpenMPSIMDRuntime::emitCancelCall(CodeGenFunction &CGF,
11943 CodeGenFunction &CGF, const OMPExecutableDirective &D,
11946 llvm::function_ref<llvm::Value *(CodeGenFunction &CGF,
11964 void CGOpenMPSIMDRuntime::emitTeamsCall(CodeGenFunction &CGF,
11972 void CGOpenMPSIMDRuntime::emitNumTeamsClause(CodeGenFunction &CGF,
11980 CodeGenFunction &CGF, const OMPExecutableDirective &D, const Expr *IfCond,
11987 CodeGenFunction &CGF, const OMPExecutableDirective &D, const Expr *IfCond,
11992 void CGOpenMPSIMDRuntime::emitDoacrossInit(CodeGenFunction &CGF,
11998 void CGOpenMPSIMDRuntime::emitDoacrossOrdered(CodeGenFunction &CGF,
12003 void CGOpenMPSIMDRuntime::emitDoacrossOrdered(CodeGenFunction &CGF,
12015 CGOpenMPSIMDRuntime::getParameterAddress(CodeGenFunction &CGF,