Lines Matching defs:CGF
90 void EmitBody(CodeGenFunction &CGF, const Stmt *S) override;
94 virtual LValue getThreadIDVariableLValue(CodeGenFunction &CGF);
96 virtual void emitUntiedSwitch(CodeGenFunction & /*CGF*/) {}
163 void Enter(CodeGenFunction &CGF) override {
166 LValue PartIdLVal = CGF.EmitLoadOfPointerLValue(
167 CGF.GetAddrOfLocalVar(PartIDVar),
170 CGF.EmitLoadOfScalar(PartIdLVal, PartIDVar->getLocation());
171 llvm::BasicBlock *DoneBB = CGF.createBasicBlock(".untied.done.");
172 UntiedSwitch = CGF.Builder.CreateSwitch(Res, DoneBB);
173 CGF.EmitBlock(DoneBB);
174 CGF.EmitBranchThroughCleanup(CGF.ReturnBlock);
175 CGF.EmitBlock(CGF.createBasicBlock(".untied.jmp."));
176 UntiedSwitch->addCase(CGF.Builder.getInt32(0),
177 CGF.Builder.GetInsertBlock());
178 emitUntiedSwitch(CGF);
181 void emitUntiedSwitch(CodeGenFunction &CGF) const {
183 LValue PartIdLVal = CGF.EmitLoadOfPointerLValue(
184 CGF.GetAddrOfLocalVar(PartIDVar),
186 CGF.EmitStoreOfScalar(CGF.Builder.getInt32(UntiedSwitch->getNumCases()),
188 UntiedCodeGen(CGF);
190 CGF.getJumpDestInCurrentScope(".untied.next.");
191 CGF.EmitBranch(CGF.ReturnBlock.getBlock());
192 CGF.EmitBlock(CGF.createBasicBlock(".untied.jmp."));
193 UntiedSwitch->addCase(CGF.Builder.getInt32(UntiedSwitch->getNumCases()),
194 CGF.Builder.GetInsertBlock());
195 CGF.EmitBranchThroughCleanup(CurPoint);
196 CGF.EmitBlock(CurPoint.getBlock());
216 LValue getThreadIDVariableLValue(CodeGenFunction &CGF) override;
221 void emitUntiedSwitch(CodeGenFunction &CGF) override {
222 Action.emitUntiedSwitch(CGF);
289 LValue getThreadIDVariableLValue(CodeGenFunction &CGF) override {
291 return OuterRegionInfo->getThreadIDVariableLValue(CGF);
302 void emitUntiedSwitch(CodeGenFunction &CGF) override {
304 OuterRegionInfo->emitUntiedSwitch(CGF);
358 CGOpenMPInnerExprInfo(CodeGenFunction &CGF, const CapturedStmt &CS)
359 : CGOpenMPInlinedRegionInfo(CGF.CapturedStmtInfo, EmptyCodeGen,
362 PrivScope(CGF) {
374 DeclRefExpr DRE(CGF.getContext(), const_cast<VarDecl *>(VD),
378 PrivScope.addPrivate(VD, CGF.EmitLValue(&DRE).getAddress());
391 void EmitBody(CodeGenFunction &CGF, const Stmt *S) override {
415 CodeGenFunction &CGF;
426 InlinedOpenMPRegionRAII(CodeGenFunction &CGF, const RegionCodeGenTy &CodeGen,
429 : CGF(CGF), NoInheritance(NoInheritance) {
431 CGF.CapturedStmtInfo = new CGOpenMPInlinedRegionInfo(
432 CGF.CapturedStmtInfo, CodeGen, Kind, HasCancel);
434 std::swap(CGF.LambdaCaptureFields, LambdaCaptureFields);
435 LambdaThisCaptureField = CGF.LambdaThisCaptureField;
436 CGF.LambdaThisCaptureField = nullptr;
437 BlockInfo = CGF.BlockInfo;
438 CGF.BlockInfo = nullptr;
445 cast<CGOpenMPInlinedRegionInfo>(CGF.CapturedStmtInfo)->getOldCSI();
446 delete CGF.CapturedStmtInfo;
447 CGF.CapturedStmtInfo = OldCSI;
449 std::swap(CGF.LambdaCaptureFields, LambdaCaptureFields);
450 CGF.LambdaThisCaptureField = LambdaThisCaptureField;
451 CGF.BlockInfo = BlockInfo;
565 void Emit(CodeGenFunction &CGF, Flags /*flags*/) override {
566 if (!CGF.HaveInsertPoint())
568 Action->Exit(CGF);
574 void RegionCodeGenTy::operator()(CodeGenFunction &CGF) const {
575 CodeGenFunction::RunCleanupsScope Scope(CGF);
577 CGF.EHStack.pushCleanup<CleanupTy>(NormalAndEHCleanup, PrePostAction);
578 Callback(CodeGen, CGF, *PrePostAction);
581 Callback(CodeGen, CGF, Action);
598 static void emitInitWithReductionInitializer(CodeGenFunction &CGF,
605 CGF.CGM.getOpenMPRuntime().getUserDefinedReduction(DRD);
614 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
619 CodeGenFunction::OpaqueValueMapping Map(CGF, OVE, Func);
620 CGF.EmitIgnoredExpr(InitOp);
622 llvm::Constant *Init = CGF.CGM.EmitNullConstant(Ty);
623 std::string Name = CGF.CGM.getOpenMPRuntime().getName({"init"});
625 CGF.CGM.getModule(), Init->getType(), /*isConstant=*/true,
627 LValue LV = CGF.MakeNaturalAlignRawAddrLValue(GV, Ty);
629 switch (CGF.getEvaluationKind(Ty)) {
631 InitRVal = CGF.EmitLoadOfLValue(LV, DRD->getLocation());
635 RValue::getComplex(CGF.EmitLoadOfComplex(LV, DRD->getLocation()));
639 CodeGenFunction::OpaqueValueMapping OpaqueMap(CGF, &OVE, LV);
640 CGF.EmitAnyExprToMem(&OVE, Private, Ty.getQualifiers(),
646 CodeGenFunction::OpaqueValueMapping OpaqueMap(CGF, &OVE, InitRVal);
647 CGF.EmitAnyExprToMem(&OVE, Private, Ty.getQualifiers(),
657 static void EmitOMPAggregateInit(CodeGenFunction &CGF, Address DestAddr,
667 llvm::Value *NumElements = CGF.emitArrayLength(ArrayTy, ElementTy, DestAddr);
673 SrcBegin = SrcAddr.emitRawPointer(CGF);
674 llvm::Value *DestBegin = DestAddr.emitRawPointer(CGF);
677 CGF.Builder.CreateGEP(DestAddr.getElementType(), DestBegin, NumElements);
679 llvm::BasicBlock *BodyBB = CGF.createBasicBlock("omp.arrayinit.body");
680 llvm::BasicBlock *DoneBB = CGF.createBasicBlock("omp.arrayinit.done");
682 CGF.Builder.CreateICmpEQ(DestBegin, DestEnd, "omp.arrayinit.isempty");
683 CGF.Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB);
686 llvm::BasicBlock *EntryBB = CGF.Builder.GetInsertBlock();
687 CGF.EmitBlock(BodyBB);
689 CharUnits ElementSize = CGF.getContext().getTypeSizeInChars(ElementTy);
694 SrcElementPHI = CGF.Builder.CreatePHI(SrcBegin->getType(), 2,
701 llvm::PHINode *DestElementPHI = CGF.Builder.CreatePHI(
710 CodeGenFunction::RunCleanupsScope InitScope(CGF);
712 emitInitWithReductionInitializer(CGF, DRD, Init, DestElementCurrent,
715 CGF.EmitAnyExprToMem(Init, DestElementCurrent, ElementTy.getQualifiers(),
721 llvm::Value *SrcElementNext = CGF.Builder.CreateConstGEP1_32(
724 SrcElementPHI->addIncoming(SrcElementNext, CGF.Builder.GetInsertBlock());
728 llvm::Value *DestElementNext = CGF.Builder.CreateConstGEP1_32(
733 CGF.Builder.CreateICmpEQ(DestElementNext, DestEnd, "omp.arraycpy.done");
734 CGF.Builder.CreateCondBr(Done, DoneBB, BodyBB);
735 DestElementPHI->addIncoming(DestElementNext, CGF.Builder.GetInsertBlock());
738 CGF.EmitBlock(DoneBB, /*IsFinished=*/true);
741 LValue ReductionCodeGen::emitSharedLValue(CodeGenFunction &CGF, const Expr *E) {
742 return CGF.EmitOMPSharedLValue(E);
745 LValue ReductionCodeGen::emitSharedLValueUB(CodeGenFunction &CGF,
748 return CGF.EmitArraySectionExpr(OASE, /*IsLowerBound=*/false);
753 CodeGenFunction &CGF, unsigned N, Address PrivateAddr, Address SharedAddr,
762 EmitOMPAggregateInit(CGF, PrivateAddr, PrivateVD->getType(),
788 void ReductionCodeGen::emitSharedOrigLValue(CodeGenFunction &CGF, unsigned N) {
791 LValue First = emitSharedLValue(CGF, ClausesData[N].Shared);
792 LValue Second = emitSharedLValueUB(CGF, ClausesData[N].Shared);
797 LValue First = emitSharedLValue(CGF, ClausesData[N].Ref);
798 LValue Second = emitSharedLValueUB(CGF, ClausesData[N].Ref);
803 void ReductionCodeGen::emitAggregateType(CodeGenFunction &CGF, unsigned N) {
808 CGF.getTypeSize(OrigAddresses[N].first.getType().getNonReferenceType()),
817 Size = CGF.Builder.CreatePtrDiff(ElemType,
818 OrigAddresses[N].second.getPointer(CGF),
819 OrigAddresses[N].first.getPointer(CGF));
820 Size = CGF.Builder.CreateNUWAdd(
822 SizeInChars = CGF.Builder.CreateNUWMul(Size, ElemSizeOf);
825 CGF.getTypeSize(OrigAddresses[N].first.getType().getNonReferenceType());
826 Size = CGF.Builder.CreateExactUDiv(SizeInChars, ElemSizeOf);
830 CGF,
832 CGF.getContext().getAsVariableArrayType(PrivateType)->getSizeExpr()),
834 CGF.EmitVariablyModifiedType(PrivateType);
837 void ReductionCodeGen::emitAggregateType(CodeGenFunction &CGF, unsigned N,
847 CGF,
849 CGF.getContext().getAsVariableArrayType(PrivateType)->getSizeExpr()),
851 CGF.EmitVariablyModifiedType(PrivateType);
855 CodeGenFunction &CGF, unsigned N, Address PrivateAddr, Address SharedAddr,
862 if (CGF.getContext().getAsArrayType(PrivateVD->getType())) {
864 (void)DefaultInit(CGF);
865 emitAggregateInitialization(CGF, N, PrivateAddr, SharedAddr, DRD);
867 (void)DefaultInit(CGF);
869 emitInitWithReductionInitializer(CGF, DRD, ClausesData[N].ReductionOp,
871 } else if (!DefaultInit(CGF) && PrivateVD->hasInit() &&
872 !CGF.isTrivialInitializer(PrivateVD->getInit())) {
873 CGF.EmitAnyExprToMem(PrivateVD->getInit(), PrivateAddr,
885 void ReductionCodeGen::emitCleanups(CodeGenFunction &CGF, unsigned N,
891 PrivateAddr.withElementType(CGF.ConvertTypeForMem(PrivateType));
892 CGF.pushDestroy(DTorKind, PrivateAddr, PrivateType);
896 static LValue loadToBegin(CodeGenFunction &CGF, QualType BaseTy, QualType ElTy,
900 !CGF.getContext().hasSameType(BaseTy, ElTy)) {
902 BaseLV = CGF.EmitLoadOfPointerLValue(BaseLV.getAddress(), PtrTy);
904 LValue RefLVal = CGF.MakeAddrLValue(BaseLV.getAddress(), BaseTy);
905 BaseLV = CGF.EmitLoadOfReferenceLValue(RefLVal);
909 return CGF.MakeAddrLValue(
910 BaseLV.getAddress().withElementType(CGF.ConvertTypeForMem(ElTy)),
912 CGF.CGM.getTBAAInfoForSubobject(BaseLV, BaseLV.getType()));
915 static Address castToBase(CodeGenFunction &CGF, QualType BaseTy, QualType ElTy,
922 !CGF.getContext().hasSameType(BaseTy, ElTy)) {
923 Tmp = CGF.CreateMemTemp(BaseTy);
925 CGF.Builder.CreateStore(Tmp.getPointer(), TopTmp);
933 Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
935 CGF.Builder.CreateStore(Addr, Tmp);
939 Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
964 Address ReductionCodeGen::adjustPrivateAddress(CodeGenFunction &CGF, unsigned N,
969 LValue OriginalBaseLValue = CGF.EmitLValue(DE);
971 loadToBegin(CGF, OrigVD->getType(), SharedAddresses[N].first.getType(),
974 llvm::Value *Adjustment = CGF.Builder.CreatePtrDiff(
975 SharedAddr.getElementType(), BaseLValue.getPointer(CGF),
976 SharedAddr.emitRawPointer(CGF));
978 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
979 PrivateAddr.emitRawPointer(CGF), SharedAddr.getType());
980 llvm::Value *Ptr = CGF.Builder.CreateGEP(
982 return castToBase(CGF, OrigVD->getType(),
997 LValue CGOpenMPRegionInfo::getThreadIDVariableLValue(CodeGenFunction &CGF) {
998 return CGF.EmitLoadOfPointerLValue(
999 CGF.GetAddrOfLocalVar(getThreadIDVariable()),
1003 void CGOpenMPRegionInfo::EmitBody(CodeGenFunction &CGF, const Stmt *S) {
1004 if (!CGF.HaveInsertPoint())
1011 CGF.EHStack.pushTerminate();
1013 CGF.incrementProfileCounter(S);
1014 CodeGen(CGF);
1015 CGF.EHStack.popTerminate();
1019 CodeGenFunction &CGF) {
1020 return CGF.MakeAddrLValue(CGF.GetAddrOfLocalVar(getThreadIDVariable()),
1104 CodeGenFunction CGF(CGM);
1107 CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, FnInfo, Args, In->getLocation(),
1109 CodeGenFunction::OMPPrivateScope Scope(CGF);
1110 Address AddrIn = CGF.GetAddrOfLocalVar(&OmpInParm);
1112 In, CGF.EmitLoadOfPointerLValue(AddrIn, PtrTy->castAs<PointerType>())
1114 Address AddrOut = CGF.GetAddrOfLocalVar(&OmpOutParm);
1116 Out, CGF.EmitLoadOfPointerLValue(AddrOut, PtrTy->castAs<PointerType>())
1120 !CGF.isTrivialInitializer(Out->getInit())) {
1121 CGF.EmitAnyExprToMem(Out->getInit(), CGF.GetAddrOfLocalVar(Out),
1126 CGF.EmitIgnoredExpr(CombinerInitializer);
1128 CGF.FinishFunction();
1133 CodeGenFunction *CGF, const OMPDeclareReductionDecl *D) {
1152 if (CGF) {
1153 auto &Decls = FunctionUDRMap.FindAndConstruct(CGF->CurFn);
1163 emitUserDefinedReduction(/*CGF=*/nullptr, D);
1171 PushAndPopStackRAII(llvm::OpenMPIRBuilder *OMPBuilder, CodeGenFunction &CGF,
1189 auto FiniCB = [&CGF](llvm::OpenMPIRBuilder::InsertPointTy IP) {
1192 CGBuilderTy::InsertPointGuard IPG(CGF.Builder);
1193 CGF.Builder.restoreIP(IP);
1195 CGF.getOMPCancelDestination(OMPD_parallel);
1196 CGF.EmitBranchThroughCleanup(Dest);
1218 CodeGenFunction CGF(CGM, true);
1242 PushAndPopStackRAII PSR(&OMPBuilder, CGF, HasCancel, InnermostKind);
1245 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo);
1246 return CGF.GenerateOpenMPCapturedStmtFunction(*CS, D.getBeginLoc());
1254 std::string CGOpenMPRuntime::getOutlinedHelperName(CodeGenFunction &CGF) const {
1255 return getOutlinedHelperName(CGF.CurFn->getName());
1264 CodeGenFunction &CGF, const OMPExecutableDirective &D,
1269 CGM, D, CS, ThreadIDVar, InnermostKind, getOutlinedHelperName(CGF),
1274 CodeGenFunction &CGF, const OMPExecutableDirective &D,
1279 CGM, D, CS, ThreadIDVar, InnermostKind, getOutlinedHelperName(CGF),
1288 auto &&UntiedCodeGen = [this, &D, TaskTVar](CodeGenFunction &CGF,
1290 llvm::Value *ThreadID = getThreadID(CGF, D.getBeginLoc());
1291 llvm::Value *UpLoc = emitUpdateLocation(CGF, D.getBeginLoc());
1294 CGF.EmitLoadOfPointerLValue(CGF.GetAddrOfLocalVar(TaskTVar),
1296 .getPointer(CGF)};
1297 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
1320 CodeGenFunction CGF(CGM, true);
1323 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo);
1324 llvm::Function *Res = CGF.GenerateCapturedStmtFunction(*CS);
1330 void CGOpenMPRuntime::setLocThreadIdInsertPt(CodeGenFunction &CGF,
1332 auto &Elem = OpenMPLocThreadIDMap.FindAndConstruct(CGF.CurFn);
1335 llvm::Value *Undef = llvm::UndefValue::get(CGF.Int32Ty);
1338 Undef, CGF.Int32Ty, "svcpt", CGF.Builder.GetInsertBlock());
1341 new llvm::BitCastInst(Undef, CGF.Int32Ty, "svcpt");
1342 Elem.second.ServiceInsertPt->insertAfter(CGF.AllocaInsertPt);
1346 void CGOpenMPRuntime::clearLocThreadIdInsertPt(CodeGenFunction &CGF) {
1347 auto &Elem = OpenMPLocThreadIDMap.FindAndConstruct(CGF.CurFn);
1355 static StringRef getIdentStringFromSourceLocation(CodeGenFunction &CGF,
1360 PresumedLoc PLoc = CGF.getContext().getSourceManager().getPresumedLoc(Loc);
1362 if (const auto *FD = dyn_cast_or_null<FunctionDecl>(CGF.CurFuncDecl))
1368 llvm::Value *CGOpenMPRuntime::emitUpdateLocation(CodeGenFunction &CGF,
1379 if (const auto *FD = dyn_cast_or_null<FunctionDecl>(CGF.CurFuncDecl))
1381 PresumedLoc PLoc = CGF.getContext().getSourceManager().getPresumedLoc(Loc);
1393 llvm::Value *CGOpenMPRuntime::getThreadID(CodeGenFunction &CGF,
1395 assert(CGF.CurFn && "No function in current CodeGenFunction.");
1400 OMPBuilder.updateToLocation(CGF.Builder.saveIP());
1403 getIdentStringFromSourceLocation(CGF, Loc, Buffer), SrcLocStrSize);
1411 auto I = OpenMPLocThreadIDMap.find(CGF.CurFn);
1419 dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo)) {
1422 LValue LVal = OMPRegionInfo->getThreadIDVariableLValue(CGF);
1423 llvm::BasicBlock *TopBlock = CGF.AllocaInsertPt->getParent();
1424 if (!CGF.EHStack.requiresLandingPad() || !CGF.getLangOpts().Exceptions ||
1425 !CGF.getLangOpts().CXXExceptions ||
1426 CGF.Builder.GetInsertBlock() == TopBlock ||
1427 !isa<llvm::Instruction>(LVal.getPointer(CGF)) ||
1428 cast<llvm::Instruction>(LVal.getPointer(CGF))->getParent() ==
1430 cast<llvm::Instruction>(LVal.getPointer(CGF))->getParent() ==
1431 CGF.Builder.GetInsertBlock()) {
1432 ThreadID = CGF.EmitLoadOfScalar(LVal, Loc);
1435 if (CGF.Builder.GetInsertBlock() == TopBlock) {
1436 auto &Elem = OpenMPLocThreadIDMap.FindAndConstruct(CGF.CurFn);
1448 auto &Elem = OpenMPLocThreadIDMap.FindAndConstruct(CGF.CurFn);
1450 setLocThreadIdInsertPt(CGF);
1451 CGBuilderTy::InsertPointGuard IPG(CGF.Builder);
1452 CGF.Builder.SetInsertPoint(Elem.second.ServiceInsertPt);
1453 auto DL = ApplyDebugLocation::CreateDefaultArtificial(CGF, Loc);
1454 llvm::CallInst *Call = CGF.Builder.CreateCall(
1457 emitUpdateLocation(CGF, Loc));
1458 Call->setCallingConv(CGF.getRuntimeCC());
1463 void CGOpenMPRuntime::functionFinished(CodeGenFunction &CGF) {
1464 assert(CGF.CurFn && "No function in current CodeGenFunction.");
1465 if (OpenMPLocThreadIDMap.count(CGF.CurFn)) {
1466 clearLocThreadIdInsertPt(CGF);
1467 OpenMPLocThreadIDMap.erase(CGF.CurFn);
1469 if (FunctionUDRMap.count(CGF.CurFn) > 0) {
1470 for(const auto *D : FunctionUDRMap[CGF.CurFn])
1472 FunctionUDRMap.erase(CGF.CurFn);
1474 auto I = FunctionUDMMap.find(CGF.CurFn);
1480 LastprivateConditionalToTypes.erase(CGF.CurFn);
1481 FunctionToUntiedTaskStackMap.erase(CGF.CurFn);
1598 Address CGOpenMPRuntime::getAddrOfThreadPrivate(CodeGenFunction &CGF,
1608 emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
1609 CGF.Builder.CreatePointerCast(VDAddr.emitRawPointer(CGF), CGM.Int8PtrTy),
1613 CGF.EmitRuntimeCall(
1617 CGF.Int8Ty, VDAddr.getAlignment());
1621 CodeGenFunction &CGF, Address VDAddr, llvm::Value *Ctor,
1625 llvm::Value *OMPLoc = emitUpdateLocation(CGF, Loc);
1626 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
1633 CGF.Builder.CreatePointerCast(VDAddr.emitRawPointer(CGF), CGM.VoidPtrTy),
1635 CGF.EmitRuntimeCall(
1643 bool PerformInit, CodeGenFunction *CGF) {
1741 if (!CGF) {
1756 emitThreadPrivateVarInit(*CGF, VDAddr, Ctor, CopyCtor, Dtor, Loc);
1796 Address CGOpenMPRuntime::getAddrOfArtificialThreadPrivate(CodeGenFunction &CGF,
1800 llvm::Type *VarLVType = CGF.ConvertTypeForMem(VarType);
1811 emitUpdateLocation(CGF, SourceLocation()),
1812 getThreadID(CGF, SourceLocation()),
1813 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(GAddr, CGM.VoidPtrTy),
1814 CGF.Builder.CreateIntCast(CGF.getTypeSize(VarType), CGM.SizeTy,
1820 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
1821 CGF.EmitRuntimeCall(
1829 void CGOpenMPRuntime::emitIfClause(CodeGenFunction &CGF, const Expr *Cond,
1832 CodeGenFunction::LexicalScope ConditionScope(CGF, Cond->getSourceRange());
1837 if (CGF.ConstantFoldsToSimpleInteger(Cond, CondConstant)) {
1839 ThenGen(CGF);
1841 ElseGen(CGF);
1847 llvm::BasicBlock *ThenBlock = CGF.createBasicBlock("omp_if.then");
1848 llvm::BasicBlock *ElseBlock = CGF.createBasicBlock("omp_if.else");
1849 llvm::BasicBlock *ContBlock = CGF.createBasicBlock("omp_if.end");
1850 CGF.EmitBranchOnBoolExpr(Cond, ThenBlock, ElseBlock, /*TrueCount=*/0);
1853 CGF.EmitBlock(ThenBlock);
1854 ThenGen(CGF);
1855 CGF.EmitBranch(ContBlock);
1858 (void)ApplyDebugLocation::CreateEmpty(CGF);
1859 CGF.EmitBlock(ElseBlock);
1860 ElseGen(CGF);
1862 (void)ApplyDebugLocation::CreateEmpty(CGF);
1863 CGF.EmitBranch(ContBlock);
1865 CGF.EmitBlock(ContBlock, /*IsFinished=*/true);
1868 void CGOpenMPRuntime::emitParallelCall(CodeGenFunction &CGF, SourceLocation Loc,
1873 if (!CGF.HaveInsertPoint())
1875 llvm::Value *RTLoc = emitUpdateLocation(CGF, Loc);
1878 this](CodeGenFunction &CGF, PrePostActionTy &) {
1880 CGOpenMPRuntime &RT = CGF.CGM.getOpenMPRuntime();
1883 CGF.Builder.getInt32(CapturedVars.size()), // Number of captured vars
1884 CGF.Builder.CreateBitCast(OutlinedFn, RT.getKmpc_MicroPointerTy())};
1891 CGF.EmitRuntimeCall(RTLFn, RealArgs);
1894 this](CodeGenFunction &CGF, PrePostActionTy &) {
1895 CGOpenMPRuntime &RT = CGF.CGM.getOpenMPRuntime();
1896 llvm::Value *ThreadID = RT.getThreadID(CGF, Loc);
1900 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
1905 Address ThreadIDAddr = RT.emitThreadIDAddress(CGF, Loc);
1907 CGF.CreateDefaultAlignTempAlloca(CGF.Int32Ty,
1909 CGF.Builder.CreateStore(CGF.Builder.getInt32(/*C*/ 0), ZeroAddrBound);
1912 OutlinedFnArgs.push_back(ThreadIDAddr.emitRawPointer(CGF));
1924 RT.emitOutlinedFunctionCall(CGF, Loc, OutlinedFn, OutlinedFnArgs);
1927 llvm::Value *EndArgs[] = {RT.emitUpdateLocation(CGF, Loc), ThreadID};
1928 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
1933 emitIfClause(CGF, IfCond, ThenGen, ElseGen);
1936 ThenRCG(CGF);
1946 Address CGOpenMPRuntime::emitThreadIDAddress(CodeGenFunction &CGF,
1949 dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo))
1951 return OMPRegionInfo->getThreadIDVariableLValue(CGF).getAddress();
1953 llvm::Value *ThreadID = getThreadID(CGF, Loc);
1955 CGF.getContext().getIntTypeForBitwidth(/*DestWidth*/ 32, /*Signed*/ true);
1956 Address ThreadIDTemp = CGF.CreateMemTemp(Int32Ty, /*Name*/ ".threadid_temp.");
1957 CGF.EmitStoreOfScalar(ThreadID,
1958 CGF.MakeAddrLValue(ThreadIDTemp, Int32Ty));
1986 void Enter(CodeGenFunction &CGF) override {
1987 llvm::Value *EnterRes = CGF.EmitRuntimeCall(EnterCallee, EnterArgs);
1989 llvm::Value *CallBool = CGF.Builder.CreateIsNotNull(EnterRes);
1990 auto *ThenBlock = CGF.createBasicBlock("omp_if.then");
1991 ContBlock = CGF.createBasicBlock("omp_if.end");
1993 CGF.Builder.CreateCondBr(CallBool, ThenBlock, ContBlock);
1994 CGF.EmitBlock(ThenBlock);
1997 void Done(CodeGenFunction &CGF) {
1999 CGF.EmitBranch(ContBlock);
2000 CGF.EmitBlock(ContBlock, true);
2002 void Exit(CodeGenFunction &CGF) override {
2003 CGF.EmitRuntimeCall(ExitCallee, ExitArgs);
2008 void CGOpenMPRuntime::emitCriticalRegion(CodeGenFunction &CGF,
2016 if (!CGF.HaveInsertPoint())
2018 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
2023 EnterArgs.push_back(CGF.Builder.CreateIntCast(
2024 CGF.EmitScalarExpr(Hint), CGM.Int32Ty, /*isSigned=*/false));
2035 emitInlinedDirective(CGF, OMPD_critical, CriticalOpGen);
2038 void CGOpenMPRuntime::emitMasterRegion(CodeGenFunction &CGF,
2041 if (!CGF.HaveInsertPoint())
2048 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
2057 emitInlinedDirective(CGF, OMPD_master, MasterOpGen);
2058 Action.Done(CGF);
2061 void CGOpenMPRuntime::emitMaskedRegion(CodeGenFunction &CGF,
2064 if (!CGF.HaveInsertPoint())
2072 ? CGF.EmitScalarExpr(Filter, CGF.Int32Ty)
2074 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
2076 llvm::Value *ArgsEnd[] = {emitUpdateLocation(CGF, Loc),
2077 getThreadID(CGF, Loc)};
2086 emitInlinedDirective(CGF, OMPD_masked, MaskedOpGen);
2087 Action.Done(CGF);
2090 void CGOpenMPRuntime::emitTaskyieldCall(CodeGenFunction &CGF,
2092 if (!CGF.HaveInsertPoint())
2094 if (CGF.CGM.getLangOpts().OpenMPIRBuilder) {
2095 OMPBuilder.createTaskyield(CGF.Builder);
2099 emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
2101 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
2106 if (auto *Region = dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo))
2107 Region->emitUntiedSwitch(CGF);
2110 void CGOpenMPRuntime::emitTaskgroupRegion(CodeGenFunction &CGF,
2113 if (!CGF.HaveInsertPoint())
2119 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
2127 emitInlinedDirective(CGF, OMPD_taskgroup, TaskgroupOpGen);
2132 static Address emitAddrOfVarFromArray(CodeGenFunction &CGF, Address Array,
2135 Address PtrAddr = CGF.Builder.CreateConstArrayGEP(Array, Index);
2136 llvm::Value *Ptr = CGF.Builder.CreateLoad(PtrAddr);
2138 llvm::Type *ElemTy = CGF.ConvertTypeForMem(Var->getType());
2140 CGF.Builder.CreateBitCast(
2142 ElemTy, CGF.getContext().getDeclAlign(Var));
2168 CodeGenFunction CGF(CGM);
2169 CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, CGFI, Args, Loc, Loc);
2172 Address LHS(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
2173 CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(&LHSArg)),
2175 ArgsElemType, CGF.getPointerAlign());
2176 Address RHS(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
2177 CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(&RHSArg)),
2179 ArgsElemType, CGF.getPointerAlign());
2187 Address DestAddr = emitAddrOfVarFromArray(CGF, LHS, I, DestVar);
2191 Address SrcAddr = emitAddrOfVarFromArray(CGF, RHS, I, SrcVar);
2195 CGF.EmitOMPCopy(Type, DestAddr, SrcAddr, DestVar, SrcVar, AssignmentOps[I]);
2197 CGF.FinishFunction();
2201 void CGOpenMPRuntime::emitSingleRegion(CodeGenFunction &CGF,
2208 if (!CGF.HaveInsertPoint())
2228 DidIt = CGF.CreateMemTemp(KmpInt32Ty, ".omp.copyprivate.did_it");
2229 CGF.Builder.CreateStore(CGF.Builder.getInt32(0), DidIt);
2232 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
2241 emitInlinedDirective(CGF, OMPD_single, SingleOpGen);
2244 CGF.Builder.CreateStore(CGF.Builder.getInt32(1), DidIt);
2246 Action.Done(CGF);
2256 CGF.CreateMemTemp(CopyprivateArrayTy, ".omp.copyprivate.cpr_list");
2258 Address Elem = CGF.Builder.CreateConstArrayGEP(CopyprivateList, I);
2259 CGF.Builder.CreateStore(
2260 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
2261 CGF.EmitLValue(CopyprivateVars[I]).getPointer(CGF),
2262 CGF.VoidPtrTy),
2268 CGM, CGF.ConvertTypeForMem(CopyprivateArrayTy), CopyprivateVars,
2270 llvm::Value *BufSize = CGF.getTypeSize(CopyprivateArrayTy);
2271 Address CL = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
2272 CopyprivateList, CGF.VoidPtrTy, CGF.Int8Ty);
2273 llvm::Value *DidItVal = CGF.Builder.CreateLoad(DidIt);
2275 emitUpdateLocation(CGF, Loc), // ident_t *<loc>
2276 getThreadID(CGF, Loc), // i32 <gtid>
2278 CL.emitRawPointer(CGF), // void *<copyprivate list>
2282 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
2288 void CGOpenMPRuntime::emitOrderedRegion(CodeGenFunction &CGF,
2291 if (!CGF.HaveInsertPoint())
2298 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
2306 emitInlinedDirective(CGF, OMPD_ordered, OrderedOpGen);
2309 emitInlinedDirective(CGF, OMPD_ordered, OrderedOpGen);
2328 CodeGenFunction &CGF, const OMPLoopDirective &S,
2339 CGF.getContext(), ChunkSize,
2340 CGF.getContext().getIntTypeForBitwidth(32, /*Signed=*/0),
2345 void CGOpenMPRuntime::emitBarrierCall(CodeGenFunction &CGF, SourceLocation Loc,
2350 dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo);
2351 if (CGF.CGM.getLangOpts().OpenMPIRBuilder) {
2352 CGF.Builder.restoreIP(OMPBuilder.createBarrier(
2353 CGF.Builder, Kind, ForceSimpleCall, EmitChecks));
2357 if (!CGF.HaveInsertPoint())
2364 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc, Flags),
2365 getThreadID(CGF, Loc)};
2368 llvm::Value *Result = CGF.EmitRuntimeCall(
2376 llvm::BasicBlock *ExitBB = CGF.createBasicBlock(".cancel.exit");
2377 llvm::BasicBlock *ContBB = CGF.createBasicBlock(".cancel.continue");
2378 llvm::Value *Cmp = CGF.Builder.CreateIsNotNull(Result);
2379 CGF.Builder.CreateCondBr(Cmp, ExitBB, ContBB);
2380 CGF.EmitBlock(ExitBB);
2383 CGF.getOMPCancelDestination(OMPRegionInfo->getDirectiveKind());
2384 CGF.EmitBranchThroughCleanup(CancelDestination);
2385 CGF.EmitBlock(ContBB, /*IsFinished=*/true);
2390 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
2395 void CGOpenMPRuntime::emitErrorCall(CodeGenFunction &CGF, SourceLocation Loc,
2398 ME ? CGF.EmitStringLiteralLValue(cast<StringLiteral>(ME)).getPointer(CGF)
2399 : llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
2403 emitUpdateLocation(CGF, Loc, /*Flags=*/0, /*GenLoc=*/true),
2405 CGF.Builder.CreatePointerCast(MVL, CGM.Int8PtrTy)};
2406 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
2525 CodeGenFunction &CGF, SourceLocation Loc,
2528 if (!CGF.HaveInsertPoint())
2543 : CGF.Builder.getIntN(IVSize, 1);
2545 emitUpdateLocation(CGF, Loc),
2546 getThreadID(CGF, Loc),
2547 CGF.Builder.getInt32(addMonoNonMonoModifier(
2551 CGF.Builder.getIntN(IVSize, 1), // Stride
2554 CGF.EmitRuntimeCall(OMPBuilder.createDispatchInitFunction(IVSize, IVSigned),
2558 void CGOpenMPRuntime::emitForDispatchDeinit(CodeGenFunction &CGF,
2560 if (!CGF.HaveInsertPoint())
2563 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
2564 CGF.EmitRuntimeCall(OMPBuilder.createDispatchDeinitFunction(), Args);
2568 CodeGenFunction &CGF, llvm::Value *UpdateLocation, llvm::Value *ThreadId,
2572 if (!CGF.HaveInsertPoint())
2593 Chunk = CGF.Builder.getIntN(Values.IVSize, 1);
2604 CGF.Builder.getInt32(addMonoNonMonoModifier(CGF.CGM, Schedule, M1,
2606 Values.IL.emitRawPointer(CGF), // &isLastIter
2607 Values.LB.emitRawPointer(CGF), // &LB
2608 Values.UB.emitRawPointer(CGF), // &UB
2609 Values.ST.emitRawPointer(CGF), // &Stride
2610 CGF.Builder.getIntN(Values.IVSize, 1), // Incr
2613 CGF.EmitRuntimeCall(ForStaticInitFunction, Args);
2616 void CGOpenMPRuntime::emitForStaticInit(CodeGenFunction &CGF,
2625 llvm::Value *UpdatedLocation = emitUpdateLocation(CGF, Loc,
2629 llvm::Value *ThreadId = getThreadID(CGF, Loc);
2633 auto DL = ApplyDebugLocation::CreateDefaultArtificial(CGF, Loc);
2634 emitForStaticInitCall(CGF, UpdatedLocation, ThreadId, StaticInitFunction,
2639 CodeGenFunction &CGF, SourceLocation Loc,
2645 emitUpdateLocation(CGF, Loc, OMP_IDENT_WORK_DISTRIBUTE);
2646 llvm::Value *ThreadId = getThreadID(CGF, Loc);
2654 emitForStaticInitCall(CGF, UpdatedLocation, ThreadId, StaticInitFunction,
2659 void CGOpenMPRuntime::emitForStaticFinish(CodeGenFunction &CGF,
2665 if (!CGF.HaveInsertPoint())
2669 emitUpdateLocation(CGF, Loc,
2676 getThreadID(CGF, Loc)};
2677 auto DL = ApplyDebugLocation::CreateDefaultArtificial(CGF, Loc);
2681 CGF.EmitRuntimeCall(
2686 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
2691 void CGOpenMPRuntime::emitForOrderedIterationEnd(CodeGenFunction &CGF,
2695 if (!CGF.HaveInsertPoint())
2698 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
2699 CGF.EmitRuntimeCall(OMPBuilder.createDispatchFiniFunction(IVSize, IVSigned),
2703 llvm::Value *CGOpenMPRuntime::emitForNext(CodeGenFunction &CGF,
2713 emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
2714 IL.emitRawPointer(CGF), // &isLastIter
2715 LB.emitRawPointer(CGF), // &Lower
2716 UB.emitRawPointer(CGF), // &Upper
2717 ST.emitRawPointer(CGF) // &Stride
2719 llvm::Value *Call = CGF.EmitRuntimeCall(
2721 return CGF.EmitScalarConversion(
2722 Call, CGF.getContext().getIntTypeForBitwidth(32, /*Signed=*/1),
2723 CGF.getContext().BoolTy, Loc);
2726 void CGOpenMPRuntime::emitNumThreadsClause(CodeGenFunction &CGF,
2729 if (!CGF.HaveInsertPoint())
2733 emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
2734 CGF.Builder.CreateIntCast(NumThreads, CGF.Int32Ty, /*isSigned*/ true)};
2735 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
2740 void CGOpenMPRuntime::emitProcBindClause(CodeGenFunction &CGF,
2743 if (!CGF.HaveInsertPoint())
2748 emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
2750 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
2755 void CGOpenMPRuntime::emitFlush(CodeGenFunction &CGF, ArrayRef<const Expr *>,
2757 if (CGF.CGM.getLangOpts().OpenMPIRBuilder) {
2758 OMPBuilder.createFlush(CGF.Builder);
2760 if (!CGF.HaveInsertPoint())
2763 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
2765 emitUpdateLocation(CGF, Loc));
3018 CodeGenFunction CGF(CGM);
3019 CGF.StartFunction(GlobalDecl(), KmpInt32Ty, TaskEntry, TaskEntryFnInfo, Args,
3027 llvm::Value *GtidParam = CGF.EmitLoadOfScalar(
3028 CGF.GetAddrOfLocalVar(&GtidArg), /*Volatile=*/false, KmpInt32Ty, Loc);
3029 LValue TDBase = CGF.EmitLoadOfPointerLValue(
3030 CGF.GetAddrOfLocalVar(&TaskTypeArg),
3035 CGF.EmitLValueForField(TDBase, *KmpTaskTWithPrivatesQTyRD->field_begin());
3038 LValue PartIdLVal = CGF.EmitLValueForField(Base, *PartIdFI);
3039 llvm::Value *PartidParam = PartIdLVal.getPointer(CGF);
3042 LValue SharedsLVal = CGF.EmitLValueForField(Base, *SharedsFI);
3043 llvm::Value *SharedsParam = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
3044 CGF.EmitLoadOfScalar(SharedsLVal, Loc),
3045 CGF.ConvertTypeForMem(SharedsPtrTy));
3050 LValue PrivatesLVal = CGF.EmitLValueForField(TDBase, *PrivatesFI);
3051 PrivatesParam = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
3052 PrivatesLVal.getPointer(CGF), CGF.VoidPtrTy);
3054 PrivatesParam = llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
3059 CGF.Builder
3061 CGF.VoidPtrTy, CGF.Int8Ty)
3062 .emitRawPointer(CGF)};
3067 LValue LBLVal = CGF.EmitLValueForField(Base, *LBFI);
3068 llvm::Value *LBParam = CGF.EmitLoadOfScalar(LBLVal, Loc);
3070 LValue UBLVal = CGF.EmitLValueForField(Base, *UBFI);
3071 llvm::Value *UBParam = CGF.EmitLoadOfScalar(UBLVal, Loc);
3073 LValue StLVal = CGF.EmitLValueForField(Base, *StFI);
3074 llvm::Value *StParam = CGF.EmitLoadOfScalar(StLVal, Loc);
3076 LValue LILVal = CGF.EmitLValueForField(Base, *LIFI);
3077 llvm::Value *LIParam = CGF.EmitLoadOfScalar(LILVal, Loc);
3079 LValue RLVal = CGF.EmitLValueForField(Base, *RFI);
3080 llvm::Value *RParam = CGF.EmitLoadOfScalar(RLVal, Loc);
3089 CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF, Loc, TaskFunction,
3091 CGF.EmitStoreThroughLValue(RValue::get(CGF.Builder.getInt32(/*C=*/0)),
3092 CGF.MakeAddrLValue(CGF.ReturnValue, KmpInt32Ty));
3093 CGF.FinishFunction();
3123 CodeGenFunction CGF(CGM);
3124 CGF.StartFunction(GlobalDecl(), KmpInt32Ty, DestructorFn, DestructorFnInfo,
3127 LValue Base = CGF.EmitLoadOfPointerLValue(
3128 CGF.GetAddrOfLocalVar(&TaskTypeArg),
3133 Base = CGF.EmitLValueForField(Base, *FI);
3138 LValue FieldLValue = CGF.EmitLValueForField(Base, Field);
3139 CGF.pushDestroy(DtorKind, FieldLValue.getAddress(), Field->getType());
3142 CGF.FinishFunction();
3231 CodeGenFunction CGF(CGM);
3232 CGF.StartFunction(GlobalDecl(), C.VoidTy, TaskPrivatesMap,
3236 LValue Base = CGF.EmitLoadOfPointerLValue(
3237 CGF.GetAddrOfLocalVar(&TaskPrivatesArg),
3242 LValue FieldLVal = CGF.EmitLValueForField(Base, Field);
3245 CGF.MakeAddrLValue(CGF.GetAddrOfLocalVar(VD), VD->getType());
3246 LValue RefLoadLVal = CGF.EmitLoadOfPointerLValue(
3248 CGF.EmitStoreOfScalar(FieldLVal.getPointer(CGF), RefLoadLVal);
3251 CGF.FinishFunction();
3256 static void emitPrivatesInit(CodeGenFunction &CGF,
3263 ASTContext &C = CGF.getContext();
3265 LValue PrivatesBase = CGF.EmitLValueForField(TDBase, *FI);
3280 SrcBase = CGF.MakeAddrLValue(
3281 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
3282 KmpTaskSharedsPtr, CGF.ConvertTypeForMem(SharedsPtrTy),
3283 CGF.ConvertTypeForMem(SharedsTy)),
3296 !CGF.isTrivialInitializer(Init)))) {
3297 LValue PrivateLValue = CGF.EmitLValueForField(PrivatesBase, *FI);
3315 CGF.MakeAddrLValue(CGF.GetAddrOfLocalVar(OriginalVD), Type);
3317 SharedRefLValue = CGF.EmitLValueForField(SrcBase, SharedField);
3318 SharedRefLValue = CGF.MakeAddrLValue(
3323 } else if (CGF.LambdaCaptureFields.count(
3325 isa_and_nonnull<BlockDecl>(CGF.CurCodeDecl)) {
3326 SharedRefLValue = CGF.EmitLValue(Pair.second.OriginalRef);
3330 CGF, [](CodeGenFunction &, PrePostActionTy &) {}, OMPD_unknown,
3332 SharedRefLValue = CGF.EmitLValue(Pair.second.OriginalRef);
3336 if (!isa<CXXConstructExpr>(Init) || CGF.isTrivialInitializer(Init)) {
3338 CGF.EmitAggregateAssign(PrivateLValue, SharedRefLValue, Type);
3342 CGF.EmitOMPAggregateAssign(
3344 [&CGF, Elem, Init, &CapturesInfo](Address DestElement,
3347 CodeGenFunction::OMPPrivateScope InitScope(CGF);
3352 CGF, &CapturesInfo);
3353 CGF.EmitAnyExprToMem(Init, DestElement,
3359 CodeGenFunction::OMPPrivateScope InitScope(CGF);
3362 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CapturesInfo);
3363 CGF.EmitExprAsInit(Init, VD, PrivateLValue,
3367 CGF.EmitExprAsInit(Init, VD, PrivateLValue, /*capturedByInit=*/false);
3375 static bool checkInitIsRequired(CodeGenFunction &CGF,
3384 !CGF.isTrivialInitializer(Init));
3431 CodeGenFunction CGF(CGM);
3432 CGF.StartFunction(GlobalDecl(), C.VoidTy, TaskDup, TaskDupFnInfo, Args, Loc,
3435 LValue TDBase = CGF.EmitLoadOfPointerLValue(
3436 CGF.GetAddrOfLocalVar(&DstArg),
3441 LValue Base = CGF.EmitLValueForField(
3443 LValue LILVal = CGF.EmitLValueForField(Base, *LIFI);
3444 llvm::Value *Lastpriv = CGF.EmitLoadOfScalar(
3445 CGF.GetAddrOfLocalVar(&LastprivArg), /*Volatile=*/false, C.IntTy, Loc);
3446 CGF.EmitStoreOfScalar(Lastpriv, LILVal);
3453 LValue TDBase = CGF.EmitLoadOfPointerLValue(
3454 CGF.GetAddrOfLocalVar(&SrcArg),
3456 LValue Base = CGF.EmitLValueForField(
3459 CGF.EmitLoadOfScalar(CGF.EmitLValueForField(
3463 CGF.Int8Ty, CGM.getNaturalTypeAlignment(SharedsTy));
3465 emitPrivatesInit(CGF, D, KmpTaskSharedsPtr, TDBase, KmpTaskTWithPrivatesQTyRD,
3467 CGF.FinishFunction();
3490 CodeGenFunction &CGF;
3498 OMPIteratorGeneratorScope(CodeGenFunction &CGF, const OMPIteratorExpr *E)
3499 : CodeGenFunction::OMPPrivateScope(CGF), CGF(CGF), E(E) {
3504 Uppers.push_back(CGF.EmitScalarExpr(E->getHelper(I).Upper));
3506 addPrivate(VD, CGF.CreateMemTemp(VD->getType(), VD->getName()));
3510 CGF.CreateMemTemp(HelperData.CounterVD->getType(), "counter.addr"));
3517 CGF.MakeAddrLValue(CGF.GetAddrOfLocalVar(HelperData.CounterVD),
3520 CGF.EmitStoreOfScalar(
3524 ContDests.emplace_back(CGF.getJumpDestInCurrentScope("iter.cont"));
3526 ExitDests.emplace_back(CGF.getJumpDestInCurrentScope("iter.exit"));
3531 CGF.EmitBlock(ContDest.getBlock());
3533 CGF.EmitLoadOfScalar(CLVal, HelperData.CounterVD->getLocation());
3536 ? CGF.Builder.CreateICmpSLT(CVal, N)
3537 : CGF.Builder.CreateICmpULT(CVal, N);
3538 llvm::BasicBlock *BodyBB = CGF.createBasicBlock("iter.body");
3539 CGF.Builder.CreateCondBr(Cmp, BodyBB, ExitDest.getBlock());
3541 CGF.EmitBlock(BodyBB);
3543 CGF.EmitIgnoredExpr(HelperData.Update);
3552 CGF.EmitIgnoredExpr(HelperData.CounterUpdate);
3554 CGF.EmitBranchThroughCleanup(ContDests[I - 1]);
3556 CGF.EmitBlock(ExitDests[I - 1].getBlock(), /*IsFinished=*/I == 1);
3563 getPointerAndSize(CodeGenFunction &CGF, const Expr *E) {
3568 Addr = CGF.EmitScalarExpr(Base);
3570 Addr = CGF.EmitLValue(E).getPointer(CGF);
3575 SizeVal = CGF.getTypeSize(OASE->getBase()->getType()->getPointeeType());
3577 llvm::Value *Sz = CGF.EmitScalarExpr(SE);
3578 Sz = CGF.EmitScalarConversion(
3579 Sz, SE->getType(), CGF.getContext().getSizeType(), SE->getExprLoc());
3580 SizeVal = CGF.Builder.CreateNUWMul(SizeVal, Sz);
3584 LValue UpAddrLVal = CGF.EmitArraySectionExpr(ASE, /*IsLowerBound=*/false);
3586 llvm::Value *UpAddr = CGF.Builder.CreateConstGEP1_32(
3587 UpAddrAddress.getElementType(), UpAddrAddress.emitRawPointer(CGF),
3589 llvm::Value *LowIntPtr = CGF.Builder.CreatePtrToInt(Addr, CGF.SizeTy);
3590 llvm::Value *UpIntPtr = CGF.Builder.CreatePtrToInt(UpAddr, CGF.SizeTy);
3591 SizeVal = CGF.Builder.CreateNUWSub(UpIntPtr, LowIntPtr);
3593 SizeVal = CGF.getTypeSize(Ty);
3614 CGOpenMPRuntime::emitTaskInit(CodeGenFunction &CGF, SourceLocation Loc,
3689 llvm::Type *KmpTaskTWithPrivatesTy = CGF.ConvertType(KmpTaskTWithPrivatesQTy);
3693 CGF.getTypeSize(KmpTaskTWithPrivatesQTy);
3704 TaskPrivatesMap = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
3744 ? CGF.Builder.CreateSelect(Data.Final.getPointer(),
3745 CGF.Builder.getInt32(FinalFlag),
3746 CGF.Builder.getInt32(/*C=*/0))
3747 : CGF.Builder.getInt32(Data.Final.getInt() ? FinalFlag : 0);
3748 TaskFlags = CGF.Builder.CreateOr(TaskFlags, CGF.Builder.getInt32(Flags));
3750 SmallVector<llvm::Value *, 8> AllocArgs = {emitUpdateLocation(CGF, Loc),
3751 getThreadID(CGF, Loc), TaskFlags, KmpTaskTWithPrivatesTySize,
3752 SharedsSize, CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
3763 DeviceID = CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(Device),
3764 CGF.Int64Ty, /*isSigned=*/true);
3766 DeviceID = CGF.Builder.getInt64(OMP_DEVICEID_UNDEF);
3768 NewTask = CGF.EmitRuntimeCall(
3774 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
3783 LValue EvtLVal = CGF.EmitLValue(Evt);
3787 llvm::Value *Loc = emitUpdateLocation(CGF, DC->getBeginLoc());
3788 llvm::Value *Tid = getThreadID(CGF, DC->getBeginLoc());
3789 Tid = CGF.Builder.CreateIntCast(Tid, CGF.IntTy, /*isSigned=*/false);
3790 llvm::Value *EvtVal = CGF.EmitRuntimeCall(
3794 EvtVal = CGF.EmitScalarConversion(EvtVal, C.VoidPtrTy, Evt->getType(),
3796 CGF.EmitStoreOfScalar(EvtVal, EvtLVal);
3810 llvm::Value *Sz = CGF.EmitScalarExpr(IE->getHelper(I).Upper);
3811 Sz = CGF.Builder.CreateIntCast(Sz, CGF.SizeTy, /*isSigned=*/false);
3813 NumOfElements ? CGF.Builder.CreateNUWMul(NumOfElements, Sz) : Sz;
3825 NumOfElements = CGF.Builder.CreateNUWAdd(
3826 llvm::ConstantInt::get(CGF.SizeTy, NumAffinities), NumOfElements);
3831 CodeGenFunction::OpaqueValueMapping OpaqueMap(CGF, OVE,
3839 CGF.EmitVarDecl(*PD);
3840 AffinitiesArray = CGF.GetAddrOfLocalVar(PD);
3841 NumOfElements = CGF.Builder.CreateIntCast(NumOfElements, CGF.Int32Ty,
3849 CGF.CreateMemTemp(KmpTaskAffinityInfoArrayTy, ".affs.arr.addr");
3850 AffinitiesArray = CGF.Builder.CreateConstArrayGEP(AffinitiesArray, 0);
3867 std::tie(Addr, Size) = getPointerAndSize(CGF, E);
3869 CGF.MakeAddrLValue(CGF.Builder.CreateConstGEP(AffinitiesArray, Pos),
3872 LValue BaseAddrLVal = CGF.EmitLValueForField(
3874 CGF.EmitStoreOfScalar(CGF.Builder.CreatePtrToInt(Addr, CGF.IntPtrTy),
3877 LValue LenLVal = CGF.EmitLValueForField(
3879 CGF.EmitStoreOfScalar(Size, LenLVal);
3885 PosLVal = CGF.MakeAddrLValue(
3886 CGF.CreateMemTemp(C.getSizeType(), "affs.counter.addr"),
3888 CGF.EmitStoreOfScalar(llvm::ConstantInt::get(CGF.SizeTy, Pos), PosLVal);
3896 CGF, cast_or_null<OMPIteratorExpr>(Modifier->IgnoreParenImpCasts()));
3900 std::tie(Addr, Size) = getPointerAndSize(CGF, E);
3901 llvm::Value *Idx = CGF.EmitLoadOfScalar(PosLVal, E->getExprLoc());
3903 CGF.MakeAddrLValue(CGF.Builder.CreateGEP(CGF, AffinitiesArray, Idx),
3906 LValue BaseAddrLVal = CGF.EmitLValueForField(
3908 CGF.EmitStoreOfScalar(CGF.Builder.CreatePtrToInt(Addr, CGF.IntPtrTy),
3911 LValue LenLVal = CGF.EmitLValueForField(
3913 CGF.EmitStoreOfScalar(Size, LenLVal);
3914 Idx = CGF.Builder.CreateNUWAdd(
3916 CGF.EmitStoreOfScalar(Idx, PosLVal);
3922 llvm::Value *LocRef = emitUpdateLocation(CGF, Loc);
3923 llvm::Value *GTid = getThreadID(CGF, Loc);
3924 llvm::Value *AffinListPtr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
3925 AffinitiesArray.emitRawPointer(CGF), CGM.VoidPtrTy);
3928 (void)CGF.EmitRuntimeCall(
3934 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
3936 LValue Base = CGF.MakeNaturalAlignRawAddrLValue(NewTaskNewTaskTTy,
3939 CGF.EmitLValueForField(Base, *KmpTaskTWithPrivatesQTyRD->field_begin());
3945 CGF.EmitLoadOfScalar(
3946 CGF.EmitLValueForField(
3950 CGF.Int8Ty, CGM.getNaturalTypeAlignment(SharedsTy));
3951 LValue Dest = CGF.MakeAddrLValue(KmpTaskSharedsPtr, SharedsTy);
3952 LValue Src = CGF.MakeAddrLValue(Shareds, SharedsTy);
3953 CGF.EmitAggregateCopy(Dest, Src, SharedsTy, AggValueSlot::DoesNotOverlap);
3958 emitPrivatesInit(CGF, D, KmpTaskSharedsPtr, Base, KmpTaskTWithPrivatesQTyRD,
3962 (!Data.LastprivateVars.empty() || checkInitIsRequired(CGF, Privates))) {
3979 LValue Data1LV = CGF.EmitLValueForField(TDBase, *FI);
3980 LValue DestructorsLV = CGF.EmitLValueForField(
3982 CGF.EmitStoreOfScalar(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
3988 LValue Data2LV = CGF.EmitLValueForField(
3990 LValue PriorityLV = CGF.EmitLValueForField(
3992 CGF.EmitStoreOfScalar(Data.Priority.getPointer(), PriorityLV);
4049 CGOpenMPRuntime::getDepobjElements(CodeGenFunction &CGF, LValue DepobjLVal,
4057 LValue Base = CGF.EmitLoadOfPointerLValue(
4059 CGF.ConvertTypeForMem(KmpDependInfoPtrTy)),
4061 Address DepObjAddr = CGF.Builder.CreateGEP(
4062 CGF, Base.getAddress(),
4063 llvm::ConstantInt::get(CGF.IntPtrTy, -1, /*isSigned=*/true));
4064 LValue NumDepsBase = CGF.MakeAddrLValue(
4067 LValue BaseAddrLVal = CGF.EmitLValueForField(
4071 llvm::Value *NumDeps = CGF.EmitLoadOfScalar(BaseAddrLVal, Loc);
4075 static void emitDependData(CodeGenFunction &CGF, QualType &KmpDependInfoTy,
4079 CodeGenModule &CGM = CGF.CGM;
4085 llvm::Type *LLVMFlagsTy = CGF.ConvertTypeForMem(FlagsTy);
4088 CGF, cast_or_null<OMPIteratorExpr>(
4097 std::tie(Addr, Size) = getPointerAndSize(CGF, E);
4098 Addr = CGF.Builder.CreatePtrToInt(Addr, CGF.IntPtrTy);
4100 Addr = llvm::ConstantInt::get(CGF.IntPtrTy, 0);
4101 Size = llvm::ConstantInt::get(CGF.SizeTy, 0);
4105 Base = CGF.MakeAddrLValue(
4106 CGF.Builder.CreateConstGEP(DependenciesArray, *P), KmpDependInfoTy);
4110 llvm::Value *Idx = CGF.EmitLoadOfScalar(PosLVal, E->getExprLoc());
4111 Base = CGF.MakeAddrLValue(
4112 CGF.Builder.CreateGEP(CGF, DependenciesArray, Idx), KmpDependInfoTy);
4115 LValue BaseAddrLVal = CGF.EmitLValueForField(
4119 CGF.EmitStoreOfScalar(Addr, BaseAddrLVal);
4121 LValue LenLVal = CGF.EmitLValueForField(
4124 CGF.EmitStoreOfScalar(Size, LenLVal);
4127 LValue FlagsLVal = CGF.EmitLValueForField(
4131 CGF.EmitStoreOfScalar(
4138 llvm::Value *Idx = CGF.EmitLoadOfScalar(PosLVal, E->getExprLoc());
4139 Idx = CGF.Builder.CreateNUWAdd(Idx,
4141 CGF.EmitStoreOfScalar(Idx, PosLVal);
4147 CodeGenFunction &CGF, QualType &KmpDependInfoTy,
4153 ASTContext &C = CGF.getContext();
4156 CGF, cast_or_null<OMPIteratorExpr>(
4162 LValue DepobjLVal = CGF.EmitLValue(E->IgnoreParenImpCasts());
4164 getDepobjElements(CGF, DepobjLVal, E->getExprLoc());
4165 LValue NumLVal = CGF.MakeAddrLValue(
4166 CGF.CreateMemTemp(C.getUIntPtrType(), "depobj.size.addr"),
4168 CGF.Builder.CreateStore(llvm::ConstantInt::get(CGF.IntPtrTy, 0),
4170 llvm::Value *PrevVal = CGF.EmitLoadOfScalar(NumLVal, E->getExprLoc());
4171 llvm::Value *Add = CGF.Builder.CreateNUWAdd(PrevVal, NumDeps);
4172 CGF.EmitStoreOfScalar(Add, NumLVal);
4178 CGF.EmitLoadOfScalar(SizeLVals[I], Data.DepExprs[I]->getExprLoc());
4184 void CGOpenMPRuntime::emitDepobjElements(CodeGenFunction &CGF,
4191 llvm::Value *ElSize = CGF.getTypeSize(KmpDependInfoTy);
4194 CGF, cast_or_null<OMPIteratorExpr>(
4201 LValue DepobjLVal = CGF.EmitLValue(E->IgnoreParenImpCasts());
4203 getDepobjElements(CGF, DepobjLVal, E->getExprLoc());
4206 llvm::Value *Size = CGF.Builder.CreateNUWMul(
4208 CGF.Builder.CreateIntCast(NumDeps, CGF.SizeTy, /*isSigned=*/false));
4209 llvm::Value *Pos = CGF.EmitLoadOfScalar(PosLVal, E->getExprLoc());
4210 Address DepAddr = CGF.Builder.CreateGEP(CGF, DependenciesArray, Pos);
4211 CGF.Builder.CreateMemCpy(DepAddr, Base.getAddress(), Size);
4215 llvm::Value *Add = CGF.Builder.CreateNUWAdd(Pos, NumDeps);
4216 CGF.EmitStoreOfScalar(Add, PosLVal);
4222 CodeGenFunction &CGF, ArrayRef<OMPTaskDataTy::DependData> Dependencies,
4243 llvm::Value *NumOfDepobjElements = llvm::ConstantInt::get(CGF.IntPtrTy, 0);
4245 llvm::ConstantInt::get(CGF.IntPtrTy, 0);
4251 emitDepobjElementsSizes(CGF, KmpDependInfoTy, D);
4254 CGF.Builder.CreateNUWAdd(NumOfDepobjElements, Size);
4263 llvm::ConstantInt::get(CGF.IntPtrTy, 1);
4265 llvm::Value *Sz = CGF.EmitScalarExpr(IE->getHelper(I).Upper);
4266 Sz = CGF.Builder.CreateIntCast(Sz, CGF.IntPtrTy, /*isSigned=*/false);
4267 ClauseIteratorSpace = CGF.Builder.CreateNUWMul(Sz, ClauseIteratorSpace);
4269 llvm::Value *NumClauseDeps = CGF.Builder.CreateNUWMul(
4271 llvm::ConstantInt::get(CGF.IntPtrTy, D.DepExprs.size()));
4273 CGF.Builder.CreateNUWAdd(NumOfRegularWithIterators, NumClauseDeps);
4285 CGF.Builder.CreateNUWAdd(NumOfDepobjElements, NumOfElements);
4289 CGF.Builder.CreateNUWAdd(NumOfRegularWithIterators, NumOfElements);
4294 CodeGenFunction::OpaqueValueMapping OpaqueMap(CGF, OVE,
4299 // CGF.EmitVariablyModifiedType(KmpDependInfoArrayTy);
4303 CGF.EmitVarDecl(*PD);
4304 DependenciesArray = CGF.GetAddrOfLocalVar(PD);
4305 NumOfElements = CGF.Builder.CreateIntCast(NumOfElements, CGF.Int32Ty,
4312 CGF.CreateMemTemp(KmpDependInfoArrayTy, ".dep.arr.addr");
4313 DependenciesArray = CGF.Builder.CreateConstArrayGEP(DependenciesArray, 0);
4322 emitDependData(CGF, KmpDependInfoTy, &Pos, Dependencies[I],
4326 LValue PosLVal = CGF.MakeAddrLValue(
4327 CGF.CreateMemTemp(C.getSizeType(), "dep.counter.addr"), C.getSizeType());
4328 CGF.EmitStoreOfScalar(llvm::ConstantInt::get(CGF.SizeTy, Pos), PosLVal);
4333 emitDependData(CGF, KmpDependInfoTy, &PosLVal, Dependencies[I],
4341 emitDepobjElements(CGF, KmpDependInfoTy, PosLVal, Dependencies[I],
4345 DependenciesArray = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4346 DependenciesArray, CGF.VoidPtrTy, CGF.Int8Ty);
4351 CodeGenFunction &CGF, const OMPTaskDataTy::DependData &Dependencies,
4373 NumDepsVal = llvm::ConstantInt::get(CGF.SizeTy, 1);
4375 llvm::Value *Sz = CGF.EmitScalarExpr(IE->getHelper(I).Upper);
4376 Sz = CGF.Builder.CreateIntCast(Sz, CGF.SizeTy, /*isSigned=*/false);
4377 NumDepsVal = CGF.Builder.CreateNUWMul(NumDepsVal, Sz);
4379 Size = CGF.Builder.CreateNUWAdd(llvm::ConstantInt::get(CGF.SizeTy, 1),
4384 Size = CGF.Builder.CreateNUWMul(Size, RecSize);
4386 CGF.Builder.CreateIntCast(NumDepsVal, CGF.IntPtrTy, /*isSigned=*/false);
4393 NumDepsVal = llvm::ConstantInt::get(CGF.IntPtrTy, NumDependencies);
4396 llvm::Value *ThreadID = getThreadID(CGF, Loc);
4398 llvm::Value *Allocator = llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
4402 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
4405 llvm::Type *KmpDependInfoLlvmTy = CGF.ConvertTypeForMem(KmpDependInfoTy);
4406 Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4410 LValue Base = CGF.MakeAddrLValue(DependenciesArray, KmpDependInfoTy);
4412 LValue BaseAddrLVal = CGF.EmitLValueForField(
4416 CGF.EmitStoreOfScalar(NumDepsVal, BaseAddrLVal);
4421 PosLVal = CGF.MakeAddrLValue(
4422 CGF.CreateMemTemp(C.getSizeType(), "iterator.counter.addr"),
4424 CGF.EmitStoreOfScalar(llvm::ConstantInt::get(CGF.SizeTy, Idx), PosLVal,
4430 emitDependData(CGF, KmpDependInfoTy, Pos, Dependencies, DependenciesArray);
4431 DependenciesArray = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4432 CGF.Builder.CreateConstGEP(DependenciesArray, 1), CGF.VoidPtrTy,
4433 CGF.Int8Ty);
4437 void CGOpenMPRuntime::emitDestroyClause(CodeGenFunction &CGF, LValue DepobjLVal,
4442 LValue Base = CGF.EmitLoadOfPointerLValue(DepobjLVal.getAddress(),
4445 Address Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4446 Base.getAddress(), CGF.ConvertTypeForMem(KmpDependInfoPtrTy),
4447 CGF.ConvertTypeForMem(KmpDependInfoTy));
4448 llvm::Value *DepObjAddr = CGF.Builder.CreateGEP(
4449 Addr.getElementType(), Addr.emitRawPointer(CGF),
4450 llvm::ConstantInt::get(CGF.IntPtrTy, -1, /*isSigned=*/true));
4451 DepObjAddr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(DepObjAddr,
4452 CGF.VoidPtrTy);
4453 llvm::Value *ThreadID = getThreadID(CGF, Loc);
4455 llvm::Value *Allocator = llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
4459 (void)CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
4464 void CGOpenMPRuntime::emitUpdateClause(CodeGenFunction &CGF, LValue DepobjLVal,
4472 llvm::Type *LLVMFlagsTy = CGF.ConvertTypeForMem(FlagsTy);
4475 std::tie(NumDeps, Base) = getDepobjElements(CGF, DepobjLVal, Loc);
4479 llvm::Value *End = CGF.Builder.CreateGEP(Begin.getElementType(),
4480 Begin.emitRawPointer(CGF), NumDeps);
4482 llvm::BasicBlock *BodyBB = CGF.createBasicBlock("omp.body");
4483 llvm::BasicBlock *DoneBB = CGF.createBasicBlock("omp.done");
4484 llvm::BasicBlock *EntryBB = CGF.Builder.GetInsertBlock();
4485 CGF.EmitBlock(BodyBB);
4487 CGF.Builder.CreatePHI(Begin.getType(), 2, "omp.elementPast");
4488 ElementPHI->addIncoming(Begin.emitRawPointer(CGF), EntryBB);
4490 Base = CGF.MakeAddrLValue(Begin, KmpDependInfoTy, Base.getBaseInfo(),
4494 LValue FlagsLVal = CGF.EmitLValueForField(
4497 CGF.EmitStoreOfScalar(
4503 CGF.Builder.CreateConstGEP(Begin, /*Index=*/1, "omp.elementNext")
4504 .emitRawPointer(CGF);
4505 ElementPHI->addIncoming(ElementNext, CGF.Builder.GetInsertBlock());
4507 CGF.Builder.CreateICmpEQ(ElementNext, End, "omp.isempty");
4508 CGF.Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB);
4510 CGF.EmitBlock(DoneBB, /*IsFinished=*/true);
4513 void CGOpenMPRuntime::emitTaskCall(CodeGenFunction &CGF, SourceLocation Loc,
4519 if (!CGF.HaveInsertPoint())
4523 emitTaskInit(CGF, Loc, D, TaskFunction, SharedsTy, Shareds, Data);
4533 emitDependClause(CGF, Data.Dependences, Loc);
4541 llvm::Value *ThreadID = getThreadID(CGF, Loc);
4542 llvm::Value *UpLoc = emitUpdateLocation(CGF, Loc);
4550 DepTaskArgs[4] = DependenciesArray.emitRawPointer(CGF);
4551 DepTaskArgs[5] = CGF.Builder.getInt32(0);
4552 DepTaskArgs[6] = llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
4555 &DepTaskArgs](CodeGenFunction &CGF, PrePostActionTy &) {
4558 LValue PartIdLVal = CGF.EmitLValueForField(TDBase, *PartIdFI);
4559 CGF.EmitStoreOfScalar(CGF.Builder.getInt32(0), PartIdLVal);
4562 CGF.EmitRuntimeCall(
4567 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
4573 dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo))
4574 Region->emitUntiedSwitch(CGF);
4582 DepWaitTaskArgs[3] = DependenciesArray.emitRawPointer(CGF);
4583 DepWaitTaskArgs[4] = CGF.Builder.getInt32(0);
4584 DepWaitTaskArgs[5] = llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
4586 llvm::ConstantInt::get(CGF.Int32Ty, Data.HasNowaitClause);
4591 Loc](CodeGenFunction &CGF, PrePostActionTy &) {
4592 CodeGenFunction::RunCleanupsScope LocalScope(CGF);
4598 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
4603 Loc](CodeGenFunction &CGF, PrePostActionTy &Action) {
4604 Action.Enter(CGF);
4606 CGF.CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF, Loc, TaskEntry,
4622 RCG(CGF);
4626 emitIfClause(CGF, IfCond, ThenCodeGen, ElseCodeGen);
4629 ThenRCG(CGF);
4633 void CGOpenMPRuntime::emitTaskLoopCall(CodeGenFunction &CGF, SourceLocation Loc,
4639 if (!CGF.HaveInsertPoint())
4642 emitTaskInit(CGF, Loc, D, TaskFunction, SharedsTy, Shareds, Data);
4648 llvm::Value *ThreadID = getThreadID(CGF, Loc);
4649 llvm::Value *UpLoc = emitUpdateLocation(CGF, Loc);
4652 IfVal = CGF.Builder.CreateIntCast(CGF.EvaluateExprAsBool(IfCond), CGF.IntTy,
4655 IfVal = llvm::ConstantInt::getSigned(CGF.IntTy, /*V=*/1);
4658 LValue LBLVal = CGF.EmitLValueForField(
4663 CGF.EmitAnyExprToMem(LBVar->getInit(), LBLVal.getAddress(), LBLVal.getQuals(),
4665 LValue UBLVal = CGF.EmitLValueForField(
4670 CGF.EmitAnyExprToMem(UBVar->getInit(), UBLVal.getAddress(), UBLVal.getQuals(),
4672 LValue StLVal = CGF.EmitLValueForField(
4677 CGF.EmitAnyExprToMem(StVar->getInit(), StLVal.getAddress(), StLVal.getQuals(),
4680 LValue RedLVal = CGF.EmitLValueForField(
4684 CGF.EmitStoreOfScalar(Data.Reductions, RedLVal);
4686 CGF.EmitNullInitialization(RedLVal.getAddress(),
4687 CGF.getContext().VoidPtrTy);
4695 LBLVal.getPointer(CGF),
4696 UBLVal.getPointer(CGF),
4697 CGF.EmitLoadOfScalar(StLVal, Loc),
4699 CGF.IntTy, 1), // Always 1 because taskgroup emitted by the compiler
4701 CGF.IntTy, Data.Schedule.getPointer()
4705 ? CGF.Builder.CreateIntCast(Data.Schedule.getPointer(), CGF.Int64Ty,
4707 : llvm::ConstantInt::get(CGF.Int64Ty, /*V=*/0),
4708 Result.TaskDupFn ? CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4709 Result.TaskDupFn, CGF.VoidPtrTy)
4710 : llvm::ConstantPointerNull::get(CGF.VoidPtrTy)};
4711 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
4726 CodeGenFunction &CGF, QualType Type, const VarDecl *LHSVar,
4728 const llvm::function_ref<void(CodeGenFunction &CGF, const Expr *,
4734 Address LHSAddr = CGF.GetAddrOfLocalVar(LHSVar);
4735 Address RHSAddr = CGF.GetAddrOfLocalVar(RHSVar);
4739 llvm::Value *NumElements = CGF.emitArrayLength(ArrayTy, ElementTy, LHSAddr);
4741 llvm::Value *RHSBegin = RHSAddr.emitRawPointer(CGF);
4742 llvm::Value *LHSBegin = LHSAddr.emitRawPointer(CGF);
4745 CGF.Builder.CreateGEP(LHSAddr.getElementType(), LHSBegin, NumElements);
4747 llvm::BasicBlock *BodyBB = CGF.createBasicBlock("omp.arraycpy.body");
4748 llvm::BasicBlock *DoneBB = CGF.createBasicBlock("omp.arraycpy.done");
4750 CGF.Builder.CreateICmpEQ(LHSBegin, LHSEnd, "omp.arraycpy.isempty");
4751 CGF.Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB);
4754 llvm::BasicBlock *EntryBB = CGF.Builder.GetInsertBlock();
4755 CGF.EmitBlock(BodyBB);
4757 CharUnits ElementSize = CGF.getContext().getTypeSizeInChars(ElementTy);
4759 llvm::PHINode *RHSElementPHI = CGF.Builder.CreatePHI(
4766 llvm::PHINode *LHSElementPHI = CGF.Builder.CreatePHI(
4774 CodeGenFunction::OMPPrivateScope Scope(CGF);
4778 RedOpGen(CGF, XExpr, EExpr, UpExpr);
4782 llvm::Value *LHSElementNext = CGF.Builder.CreateConstGEP1_32(
4785 llvm::Value *RHSElementNext = CGF.Builder.CreateConstGEP1_32(
4790 CGF.Builder.CreateICmpEQ(LHSElementNext, LHSEnd, "omp.arraycpy.done");
4791 CGF.Builder.CreateCondBr(Done, DoneBB, BodyBB);
4792 LHSElementPHI->addIncoming(LHSElementNext, CGF.Builder.GetInsertBlock());
4793 RHSElementPHI->addIncoming(RHSElementNext, CGF.Builder.GetInsertBlock());
4796 CGF.EmitBlock(DoneBB, /*IsFinished=*/true);
4802 static void emitReductionCombiner(CodeGenFunction &CGF,
4811 CGF.CGM.getOpenMPRuntime().getUserDefinedReduction(DRD);
4813 CodeGenFunction::OpaqueValueMapping Map(CGF, OVE, Func);
4814 CGF.EmitIgnoredExpr(ReductionOp);
4817 CGF.EmitIgnoredExpr(ReductionOp);
4842 CodeGenFunction CGF(CGM);
4843 CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, CGFI, Args, Loc, Loc);
4847 Address LHS(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4848 CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(&LHSArg)),
4850 ArgsElemType, CGF.getPointerAlign());
4851 Address RHS(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4852 CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(&RHSArg)),
4854 ArgsElemType, CGF.getPointerAlign());
4859 CodeGenFunction::OMPPrivateScope Scope(CGF);
4865 Scope.addPrivate(RHSVar, emitAddrOfVarFromArray(CGF, RHS, Idx, RHSVar));
4868 Scope.addPrivate(LHSVar, emitAddrOfVarFromArray(CGF, LHS, Idx, LHSVar));
4873 Address Elem = CGF.Builder.CreateConstArrayGEP(LHS, Idx);
4874 llvm::Value *Ptr = CGF.Builder.CreateLoad(Elem);
4876 CGF.getContext().getAsVariableArrayType(PrivTy);
4879 CGF, OVE, RValue::get(CGF.Builder.CreatePtrToInt(Ptr, CGF.SizeTy)));
4880 CGF.EmitVariablyModifiedType(PrivTy);
4893 CGF, (*IPriv)->getType(), LHSVar, RHSVar,
4894 [=](CodeGenFunction &CGF, const Expr *, const Expr *, const Expr *) {
4895 emitReductionCombiner(CGF, E);
4899 emitReductionCombiner(CGF, E);
4906 CGF.FinishFunction();
4910 void CGOpenMPRuntime::emitSingleReductionCombiner(CodeGenFunction &CGF,
4920 CGF, PrivateRef->getType(), LHSVar, RHSVar,
4921 [=](CodeGenFunction &CGF, const Expr *, const Expr *, const Expr *) {
4922 emitReductionCombiner(CGF, ReductionOp);
4926 emitReductionCombiner(CGF, ReductionOp);
4930 void CGOpenMPRuntime::emitReduction(CodeGenFunction &CGF, SourceLocation Loc,
4936 if (!CGF.HaveInsertPoint())
4980 CodeGenFunction::RunCleanupsScope Scope(CGF);
4985 emitSingleReductionCombiner(CGF, E, *IPriv, cast<DeclRefExpr>(*ILHS),
5007 CGF.CreateMemTemp(ReductionArrayTy, ".omp.reduction.red_list");
5011 Address Elem = CGF.Builder.CreateConstArrayGEP(ReductionList, Idx);
5012 CGF.Builder.CreateStore(
5013 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
5014 CGF.EmitLValue(RHSExprs[I]).getPointer(CGF), CGF.VoidPtrTy),
5019 Elem = CGF.Builder.CreateConstArrayGEP(ReductionList, Idx);
5020 llvm::Value *Size = CGF.Builder.CreateIntCast(
5021 CGF.getVLASize(
5022 CGF.getContext().getAsVariableArrayType((*IPriv)->getType()))
5024 CGF.SizeTy, /*isSigned=*/false);
5025 CGF.Builder.CreateStore(CGF.Builder.CreateIntToPtr(Size, CGF.VoidPtrTy),
5032 CGF.CurFn->getName(), Loc, CGF.ConvertTypeForMem(ReductionArrayTy),
5041 llvm::Value *IdentTLoc = emitUpdateLocation(CGF, Loc, OMP_ATOMIC_REDUCE);
5042 llvm::Value *ThreadId = getThreadID(CGF, Loc);
5043 llvm::Value *ReductionArrayTySize = CGF.getTypeSize(ReductionArrayTy);
5044 llvm::Value *RL = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
5045 ReductionList.getPointer(), CGF.VoidPtrTy);
5049 CGF.Builder.getInt32(RHSExprs.size()), // i32 <n>
5055 llvm::Value *Res = CGF.EmitRuntimeCall(
5062 llvm::BasicBlock *DefaultBB = CGF.createBasicBlock(".omp.reduction.default");
5064 CGF.Builder.CreateSwitch(Res, DefaultBB, /*NumCases=*/2);
5072 llvm::BasicBlock *Case1BB = CGF.createBasicBlock(".omp.reduction.case1");
5073 SwInst->addCase(CGF.Builder.getInt32(1), Case1BB);
5074 CGF.EmitBlock(Case1BB);
5083 CodeGenFunction &CGF, PrePostActionTy &Action) {
5084 CGOpenMPRuntime &RT = CGF.CGM.getOpenMPRuntime();
5089 RT.emitSingleReductionCombiner(CGF, E, *IPriv, cast<DeclRefExpr>(*ILHS),
5104 RCG(CGF);
5106 CGF.EmitBranch(DefaultBB);
5113 llvm::BasicBlock *Case2BB = CGF.createBasicBlock(".omp.reduction.case2");
5114 SwInst->addCase(CGF.Builder.getInt32(2), Case2BB);
5115 CGF.EmitBlock(Case2BB);
5118 CodeGenFunction &CGF, PrePostActionTy &Action) {
5152 Loc](CodeGenFunction &CGF, const Expr *XExpr,
5154 LValue X = CGF.EmitLValue(XExpr);
5157 E = CGF.EmitAnyExpr(EExpr);
5158 CGF.EmitOMPAtomicSimpleUpdateExpr(
5161 [&CGF, UpExpr, VD, Loc](RValue XRValue) {
5162 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
5163 Address LHSTemp = CGF.CreateMemTemp(VD->getType());
5164 CGF.emitOMPSimpleStore(
5165 CGF.MakeAddrLValue(LHSTemp, VD->getType()), XRValue,
5169 return CGF.EmitAnyExpr(UpExpr);
5176 EmitOMPAggregateReduction(CGF, (*IPriv)->getType(), VD, RHSVar,
5180 AtomicRedGen(CGF, XExpr, EExpr, UpExpr);
5184 auto &&CritRedGen = [E, Loc](CodeGenFunction &CGF, const Expr *,
5186 CGOpenMPRuntime &RT = CGF.CGM.getOpenMPRuntime();
5189 CGF, Name,
5190 [=](CodeGenFunction &CGF, PrePostActionTy &Action) {
5191 Action.Enter(CGF);
5192 emitReductionCombiner(CGF, E);
5201 EmitOMPAggregateReduction(CGF, (*IPriv)->getType(), LHSVar, RHSVar,
5204 CritRedGen(CGF, nullptr, nullptr, nullptr);
5225 AtomicRCG(CGF);
5227 AtomicRCG(CGF);
5230 CGF.EmitBranch(DefaultBB);
5231 CGF.EmitBlock(DefaultBB, /*IsFinished=*/true);
5281 CodeGenFunction CGF(CGM);
5282 CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, FnInfo, Args, Loc, Loc);
5284 Address PrivateAddr = CGF.EmitLoadOfPointer(
5285 CGF.GetAddrOfLocalVar(&Param).withElementType(
5286 CGF.ConvertTypeForMem(PrivateType)->getPointerTo()),
5293 CGF, CGM.getContext().getSizeType(),
5295 Size = CGF.EmitLoadOfScalar(SizeAddr, /*Volatile=*/false,
5298 RCG.emitAggregateType(CGF, N, Size);
5304 Address SharedAddr = CGF.GetAddrOfLocalVar(&ParamOrig);
5305 OrigAddr = CGF.EmitLoadOfPointer(
5312 RCG.emitInitialization(CGF, N, PrivateAddr, OrigAddr,
5314 CGF.FinishFunction();
5352 CodeGenFunction CGF(CGM);
5353 CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, FnInfo, Args, Loc, Loc);
5359 CGF, CGM.getContext().getSizeType(),
5361 Size = CGF.EmitLoadOfScalar(SizeAddr, /*Volatile=*/false,
5364 RCG.emitAggregateType(CGF, N, Size);
5368 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
5372 CGF.EmitLoadOfPointer(
5373 CGF.GetAddrOfLocalVar(&ParamInOut)
5375 CGF.ConvertTypeForMem(LHSVD->getType())->getPointerTo()),
5380 CGF.EmitLoadOfPointer(
5381 CGF.GetAddrOfLocalVar(&ParamIn).withElementType(
5382 CGF.ConvertTypeForMem(RHSVD->getType())->getPointerTo()),
5389 CGF, ReductionOp, PrivateRef, cast<DeclRefExpr>(LHS),
5391 CGF.FinishFunction();
5421 CodeGenFunction CGF(CGM);
5422 CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, FnInfo, Args, Loc, Loc);
5423 Address PrivateAddr = CGF.EmitLoadOfPointer(
5424 CGF.GetAddrOfLocalVar(&Param), C.VoidPtrTy.castAs<PointerType>());
5430 CGF, CGM.getContext().getSizeType(),
5432 Size = CGF.EmitLoadOfScalar(SizeAddr, /*Volatile=*/false,
5435 RCG.emitAggregateType(CGF, N, Size);
5438 RCG.emitCleanups(CGF, N, PrivateAddr);
5439 CGF.FinishFunction(Loc);
5444 CodeGenFunction &CGF, SourceLocation Loc, ArrayRef<const Expr *> LHSExprs,
5446 if (!CGF.HaveInsertPoint() || Data.ReductionVars.empty())
5478 RawAddress TaskRedInput = CGF.CreateMemTemp(ArrayRDType, ".rd_input.");
5485 llvm::Value *GEP = CGF.EmitCheckedInBoundsGEP(
5489 LValue ElemLVal = CGF.MakeNaturalAlignRawAddrLValue(GEP, RDType);
5491 LValue SharedLVal = CGF.EmitLValueForField(ElemLVal, SharedFD);
5492 RCG.emitSharedOrigLValue(CGF, Cnt);
5493 llvm::Value *Shared = RCG.getSharedLValue(Cnt).getPointer(CGF);
5494 CGF.EmitStoreOfScalar(Shared, SharedLVal);
5496 LValue OrigLVal = CGF.EmitLValueForField(ElemLVal, OrigFD);
5497 llvm::Value *Orig = RCG.getOrigLValue(Cnt).getPointer(CGF);
5498 CGF.EmitStoreOfScalar(Orig, OrigLVal);
5499 RCG.emitAggregateType(CGF, Cnt);
5509 SizeValInChars = CGF.Builder.CreateIntCast(SizeValInChars, CGM.SizeTy,
5511 LValue SizeLVal = CGF.EmitLValueForField(ElemLVal, SizeFD);
5512 CGF.EmitStoreOfScalar(SizeValInChars, SizeLVal);
5514 LValue InitLVal = CGF.EmitLValueForField(ElemLVal, InitFD);
5516 CGF.EmitStoreOfScalar(InitAddr, InitLVal);
5518 LValue FiniLVal = CGF.EmitLValueForField(ElemLVal, FiniFD);
5522 CGF.EmitStoreOfScalar(FiniAddr, FiniLVal);
5524 LValue CombLVal = CGF.EmitLValueForField(ElemLVal, CombFD);
5528 CGF.EmitStoreOfScalar(CombAddr, CombLVal);
5530 LValue FlagsLVal = CGF.EmitLValueForField(ElemLVal, FlagsFD);
5532 CGF.EmitStoreOfScalar(
5536 CGF.EmitNullInitialization(FlagsLVal.getAddress(), FlagsLVal.getType());
5541 llvm::Value *IdentTLoc = emitUpdateLocation(CGF, Loc);
5542 llvm::Value *GTid = CGF.Builder.CreateIntCast(getThreadID(CGF, Loc),
5549 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
5551 return CGF.EmitRuntimeCall(
5558 CGF.Builder.CreateIntCast(getThreadID(CGF, Loc), CGM.IntTy,
5561 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(TaskRedInput.getPointer(),
5563 return CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
5568 void CGOpenMPRuntime::emitTaskReductionFini(CodeGenFunction &CGF,
5573 llvm::Value *IdentTLoc = emitUpdateLocation(CGF, Loc);
5574 llvm::Value *GTid = CGF.Builder.CreateIntCast(getThreadID(CGF, Loc),
5580 (void)CGF.EmitRuntimeCall(
5586 void CGOpenMPRuntime::emitTaskReductionFixups(CodeGenFunction &CGF,
5594 llvm::Value *SizeVal = CGF.Builder.CreateIntCast(Sizes.second, CGM.SizeTy,
5597 CGF, CGM.getContext().getSizeType(),
5599 CGF.Builder.CreateStore(SizeVal, SizeAddr, /*IsVolatile=*/false);
5603 Address CGOpenMPRuntime::getTaskReductionItem(CodeGenFunction &CGF,
5609 llvm::Value *Args[] = {CGF.Builder.CreateIntCast(getThreadID(CGF, Loc),
5613 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
5614 SharedLVal.getPointer(CGF), CGM.VoidPtrTy)};
5616 CGF.EmitRuntimeCall(
5620 CGF.Int8Ty, SharedLVal.getAlignment());
5623 void CGOpenMPRuntime::emitTaskwaitCall(CodeGenFunction &CGF, SourceLocation Loc,
5625 if (!CGF.HaveInsertPoint())
5628 if (CGF.CGM.getLangOpts().OpenMPIRBuilder && Data.Dependences.empty()) {
5630 OMPBuilder.createTaskwait(CGF.Builder);
5632 llvm::Value *ThreadID = getThreadID(CGF, Loc);
5633 llvm::Value *UpLoc = emitUpdateLocation(CGF, Loc);
5638 emitDependClause(CGF, Data.Dependences, Loc);
5644 DepWaitTaskArgs[3] = DependenciesArray.emitRawPointer(CGF);
5645 DepWaitTaskArgs[4] = CGF.Builder.getInt32(0);
5646 DepWaitTaskArgs[5] = llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
5648 llvm::ConstantInt::get(CGF.Int32Ty, Data.HasNowaitClause);
5650 CodeGenFunction::RunCleanupsScope LocalScope(CGF);
5656 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
5666 CGF.EmitRuntimeCall(
5672 if (auto *Region = dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo))
5673 Region->emitUntiedSwitch(CGF);
5676 void CGOpenMPRuntime::emitInlinedDirective(CodeGenFunction &CGF,
5680 if (!CGF.HaveInsertPoint())
5682 InlinedOpenMPRegionRAII Region(CGF, CodeGen, InnerKind, HasCancel,
5686 CGF.CapturedStmtInfo->EmitBody(CGF, /*S=*/nullptr);
5715 CodeGenFunction &CGF, SourceLocation Loc,
5717 if (!CGF.HaveInsertPoint())
5722 dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo)) {
5727 emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
5728 CGF.Builder.getInt32(getCancellationKind(CancelRegion))};
5730 llvm::Value *Result = CGF.EmitRuntimeCall(
5738 llvm::BasicBlock *ExitBB = CGF.createBasicBlock(".cancel.exit");
5739 llvm::BasicBlock *ContBB = CGF.createBasicBlock(".cancel.continue");
5740 llvm::Value *Cmp = CGF.Builder.CreateIsNotNull(Result);
5741 CGF.Builder.CreateCondBr(Cmp, ExitBB, ContBB);
5742 CGF.EmitBlock(ExitBB);
5744 emitBarrierCall(CGF, Loc, OMPD_unknown, /*EmitChecks=*/false);
5747 CGF.getOMPCancelDestination(OMPRegionInfo->getDirectiveKind());
5748 CGF.EmitBranchThroughCleanup(CancelDest);
5749 CGF.EmitBlock(ContBB, /*IsFinished=*/true);
5754 void CGOpenMPRuntime::emitCancelCall(CodeGenFunction &CGF, SourceLocation Loc,
5757 if (!CGF.HaveInsertPoint())
5763 dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo)) {
5765 OMPRegionInfo](CodeGenFunction &CGF, PrePostActionTy &) {
5766 CGOpenMPRuntime &RT = CGF.CGM.getOpenMPRuntime();
5768 RT.emitUpdateLocation(CGF, Loc), RT.getThreadID(CGF, Loc),
5769 CGF.Builder.getInt32(getCancellationKind(CancelRegion))};
5771 llvm::Value *Result = CGF.EmitRuntimeCall(
5777 llvm::BasicBlock *ExitBB = CGF.createBasicBlock(".cancel.exit");
5778 llvm::BasicBlock *ContBB = CGF.createBasicBlock(".cancel.continue");
5779 llvm::Value *Cmp = CGF.Builder.CreateIsNotNull(Result);
5780 CGF.Builder.CreateCondBr(Cmp, ExitBB, ContBB);
5781 CGF.EmitBlock(ExitBB);
5783 RT.emitBarrierCall(CGF, Loc, OMPD_unknown, /*EmitChecks=*/false);
5786 CGF.getOMPCancelDestination(OMPRegionInfo->getDirectiveKind());
5787 CGF.EmitBranchThroughCleanup(CancelDest);
5788 CGF.EmitBlock(ContBB, /*IsFinished=*/true);
5791 emitIfClause(CGF, IfCond, ThenGen,
5795 ThenRCG(CGF);
5809 void Enter(CodeGenFunction &CGF) override {
5810 if (!CGF.HaveInsertPoint())
5813 CGF.CGM.getOpenMPRuntime().emitUsesAllocatorsInit(
5814 CGF, AllocatorData.first, AllocatorData.second);
5817 void Exit(CodeGenFunction &CGF) override {
5818 if (!CGF.HaveInsertPoint())
5821 CGF.CGM.getOpenMPRuntime().emitUsesAllocatorsFini(CGF,
5849 void CGOpenMPRuntime::emitUsesAllocatorsInit(CodeGenFunction &CGF,
5852 llvm::Value *ThreadId = getThreadID(CGF, Allocator->getExprLoc());
5853 ThreadId = CGF.Builder.CreateIntCast(ThreadId, CGF.IntTy, /*isSigned=*/true);
5855 llvm::Value *MemSpaceHandle = llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
5857 CGF.IntTy, cast<ConstantArrayType>(
5861 LValue AllocatorTraitsLVal = CGF.EmitLValue(AllocatorTraits);
5862 Address Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
5863 AllocatorTraitsLVal.getAddress(), CGF.VoidPtrPtrTy, CGF.VoidPtrTy);
5864 AllocatorTraitsLVal = CGF.MakeAddrLValue(Addr, CGF.getContext().VoidPtrTy,
5867 llvm::Value *Traits = Addr.emitRawPointer(CGF);
5870 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
5874 CGF.EmitAutoVarAlloca(*cast<VarDecl>(
5876 LValue AllocatorLVal = CGF.EmitLValue(Allocator->IgnoreParenImpCasts());
5878 CGF.EmitScalarConversion(AllocatorVal, CGF.getContext().VoidPtrTy,
5880 CGF.EmitStoreOfScalar(AllocatorVal, AllocatorLVal);
5883 void CGOpenMPRuntime::emitUsesAllocatorsFini(CodeGenFunction &CGF,
5885 llvm::Value *ThreadId = getThreadID(CGF, Allocator->getExprLoc());
5886 ThreadId = CGF.Builder.CreateIntCast(ThreadId, CGF.IntTy, /*isSigned=*/true);
5887 LValue AllocatorLVal = CGF.EmitLValue(Allocator->IgnoreParenImpCasts());
5889 CGF.EmitLoadOfScalar(AllocatorLVal, Allocator->getExprLoc());
5890 AllocatorVal = CGF.EmitScalarConversion(AllocatorVal, Allocator->getType(),
5891 CGF.getContext().VoidPtrTy,
5893 (void)CGF.EmitRuntimeCall(
5900 const OMPExecutableDirective &D, CodeGenFunction &CGF,
5904 getNumTeamsExprForTargetDirective(CGF, D, MinTeamsVal, MaxTeamsVal);
5905 getNumThreadsExprForTargetDirective(CGF, D, MaxThreadsVal,
5943 CodeGenFunction CGF(CGM, true);
5945 [&CGF, &D, &CodeGen](StringRef EntryFnName) {
5949 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo);
5950 return CGF.GenerateOpenMPCapturedStmtFunction(CS, D.getBeginLoc());
6022 CodeGenFunction &CGF, const OMPExecutableDirective &D, int32_t &MinTeamsVal,
6034 CGOpenMPRuntime::getSingleCompoundChild(CGF.getContext(), Body);
6041 if (NumTeams->isIntegerConstantExpr(CGF.getContext()))
6043 NumTeams->getIntegerConstantExpr(CGF.getContext()))
6071 if (NumTeams->isIntegerConstantExpr(CGF.getContext()))
6072 if (auto Constant = NumTeams->getIntegerConstantExpr(CGF.getContext()))
6154 CodeGenFunction &CGF, const OMPExecutableDirective &D) {
6155 assert(!CGF.getLangOpts().OpenMPIsTargetDevice &&
6158 CGBuilderTy &Bld = CGF.Builder;
6161 getNumTeamsExprForTargetDirective(CGF, D, MinNT, MaxNT);
6168 CGOpenMPInnerExprInfo CGInfo(CGF, *CS);
6169 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo);
6170 llvm::Value *NumTeamsVal = CGF.EmitScalarExpr(NumTeams,
6172 return Bld.CreateIntCast(NumTeamsVal, CGF.Int32Ty,
6180 CodeGenFunction::RunCleanupsScope NumTeamsScope(CGF);
6181 llvm::Value *NumTeamsVal = CGF.EmitScalarExpr(NumTeams,
6183 return Bld.CreateIntCast(NumTeamsVal, CGF.Int32Ty,
6192 return llvm::ConstantInt::get(CGF.Int32Ty, MinNT);
6199 static void getNumThreads(CodeGenFunction &CGF, const CapturedStmt *CS,
6203 CGF.getContext(), CS->getCapturedStmt());
6212 CGOpenMPInnerExprInfo CGInfo(CGF, *CS);
6213 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo);
6225 if (CondExpr->EvaluateAsBooleanCondition(Result, CGF.getContext())) {
6231 CodeGenFunction::LexicalScope Scope(CGF, CondExpr->getSourceRange());
6236 CGF.EmitVarDecl(cast<VarDecl>(*I));
6239 CGF.EmitAutoVarAlloca(cast<VarDecl>(*I));
6240 CGF.EmitAutoVarCleanups(Emission);
6243 *CondVal = CGF.EvaluateExprAsBool(CondExpr);
6251 CGOpenMPInnerExprInfo CGInfo(CGF, *CS);
6252 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo);
6256 if (NTExpr->isIntegerConstantExpr(CGF.getContext()))
6257 if (auto Constant = NTExpr->getIntegerConstantExpr(CGF.getContext()))
6269 CodeGenFunction::LexicalScope Scope(CGF, NTExpr->getSourceRange());
6274 CGF.EmitVarDecl(cast<VarDecl>(*I));
6277 CGF.EmitAutoVarAlloca(cast<VarDecl>(*I));
6278 CGF.EmitAutoVarCleanups(Emission);
6291 CodeGenFunction &CGF, const OMPExecutableDirective &D, int32_t &UpperBound,
6293 assert((!CGF.getLangOpts().OpenMPIsTargetDevice || UpperBoundOnly) &&
6304 if (E->isIntegerConstantExpr(CGF.getContext())) {
6305 if (auto Constant = E->getIntegerConstantExpr(CGF.getContext()))
6326 getNumThreads(CGF, CS, NTPtr, UpperBound, UpperBoundOnly, CondVal);
6328 CGF.getContext(), CS->getCapturedStmt());
6336 CGOpenMPInnerExprInfo CGInfo(CGF, *CS);
6337 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo);
6339 CGF, ThreadLimitClause->getThreadLimit()->getSourceRange());
6344 CGF.EmitVarDecl(cast<VarDecl>(*I));
6347 CGF.EmitAutoVarAlloca(cast<VarDecl>(*I));
6348 CGF.EmitAutoVarCleanups(Emission);
6362 CGF.getContext(), CS->getCapturedStmt());
6367 getNumThreads(CGF, CS, NTPtr, UpperBound, UpperBoundOnly, CondVal);
6375 CodeGenFunction::RunCleanupsScope ThreadLimitScope(CGF);
6380 getNumThreads(CGF, CS, NTPtr, UpperBound, UpperBoundOnly, CondVal);
6382 CGF.getContext(), CS->getCapturedStmt());
6386 getNumThreads(CGF, CS, NTPtr, UpperBound, UpperBoundOnly, CondVal);
6393 CodeGenFunction::RunCleanupsScope ThreadLimitScope(CGF);
6397 getNumThreads(CGF, D.getInnermostCapturedStmt(), NTPtr, UpperBound,
6419 if (Cond->EvaluateAsBooleanCondition(Result, CGF.getContext())) {
6423 CodeGenFunction::RunCleanupsScope Scope(CGF);
6424 *CondVal = CGF.EvaluateExprAsBool(Cond);
6429 CodeGenFunction::RunCleanupsScope ThreadLimitScope(CGF);
6434 CodeGenFunction::RunCleanupsScope NumThreadsScope(CGF);
6451 CodeGenFunction &CGF, const OMPExecutableDirective &D) {
6459 CGF, D, UpperBound, /* UpperBoundOnly */ false, &CondVal,
6465 CGF.EmitScalarExpr(ThreadLimitExpr, /*IgnoreResultAssign=*/true);
6466 ThreadLimitVal = CGF.Builder.CreateIntCast(ThreadLimitVal, CGF.Int32Ty,
6472 NumThreadsVal = CGF.Builder.getInt32(UpperBound);
6474 NumThreadsVal = CGF.EmitScalarExpr(NT, /*IgnoreResultAssign=*/true);
6475 NumThreadsVal = CGF.Builder.CreateIntCast(NumThreadsVal, CGF.Int32Ty,
6485 NumThreadsVal = CGF.Builder.getInt32(0);
6491 CodeGenFunction::RunCleanupsScope Scope(CGF);
6492 NumThreadsVal = CGF.Builder.CreateSelect(CondVal, NumThreadsVal,
6493 CGF.Builder.getInt32(1));
6499 NumThreadsVal = CGF.Builder.CreateSelect(
6500 CGF.Builder.CreateICmpULT(ThreadLimitVal, NumThreadsVal),
6636 CodeGenFunction &CGF;
6666 CGF.getTypeSize(OAE->getBase()->getType()->getPointeeType());
6668 llvm::Value *Sz = CGF.EmitScalarExpr(SE);
6669 Sz = CGF.EmitScalarConversion(Sz, SE->getType(),
6670 CGF.getContext().getSizeType(),
6672 Size = CGF.Builder.CreateNUWMul(Size, Sz);
6683 // on CGF.getTypeSize(E->getType()).
6694 return CGF.getTypeSize(BaseTy);
6698 ElemSize = CGF.getTypeSize(PTy->getPointeeType().getCanonicalType());
6702 ElemSize = CGF.getTypeSize(ATy->getElementType().getCanonicalType());
6711 llvm::Value *LengthVal = CGF.EmitScalarExpr(LenExpr);
6712 LengthVal = CGF.EmitScalarConversion(LengthVal, LenExpr->getType(),
6713 CGF.getContext().getSizeType(),
6715 return CGF.Builder.CreateNUWMul(LengthVal, ElemSize);
6720 llvm::Value *LengthVal = CGF.getTypeSize(BaseTy);
6721 llvm::Value *LBVal = CGF.EmitScalarExpr(OAE->getLowerBound());
6722 LBVal = CGF.EmitScalarConversion(LBVal, OAE->getLowerBound()->getType(),
6723 CGF.getContext().getSizeType(),
6725 LBVal = CGF.Builder.CreateNUWMul(LBVal, ElemSize);
6726 llvm::Value *Cmp = CGF.Builder.CreateICmpUGT(LengthVal, LBVal);
6727 llvm::Value *TrueVal = CGF.Builder.CreateNUWSub(LengthVal, LBVal);
6728 LengthVal = CGF.Builder.CreateSelect(
6729 Cmp, TrueVal, llvm::ConstantInt::get(CGF.SizeTy, 0));
6732 return CGF.getTypeSize(ExprTy);
6820 if (!Length->EvaluateAsInt(Result, CGF.getContext()))
7052 BP = CGF.LoadCXXThisAddress();
7056 BP = CGF.EmitOMPSharedLValue(AssocExpr).getAddress();
7060 CGF.EmitScalarExpr(OAShE->getBase()),
7061 CGF.ConvertTypeForMem(OAShE->getBase()->getType()->getPointeeType()),
7062 CGF.getContext().getTypeAlignInChars(OAShE->getBase()->getType()));
7066 BP = CGF.EmitOMPSharedLValue(AssocExpr).getAddress();
7074 CGF.CGM.getOpenMPRuntime().hasRequiresUnifiedSharedMemory())) {
7076 BP = CGF.CGM.getOpenMPRuntime().getAddrOfDeclareTargetVar(VD);
7092 (CGF.CGM.getOpenMPRuntime().hasRequiresUnifiedSharedMemory() ||
7094 BP = CGF.EmitLoadOfPointer(BP, Ty->castAs<PointerType>());
7167 BP = CGF.EmitLoadOfPointer(BP, Ty->castAs<PointerType>());
7235 auto &&EmitMemberExprBase = [](CodeGenFunction &CGF,
7245 CGF.EmitPointerWithAlignment(BaseExpr, &BaseInfo, &TBAAInfo);
7247 BaseLV = CGF.MakeAddrLValue(Addr, PtrTy, BaseInfo, TBAAInfo);
7249 BaseLV = CGF.EmitOMPSharedLValue(BaseExpr);
7255 Address(CGF.EmitScalarExpr(OAShE->getBase()),
7256 CGF.ConvertTypeForMem(
7258 CGF.getContext().getTypeAlignInChars(
7262 LValue BaseLVal = EmitMemberExprBase(CGF, ME);
7263 LowestElem = CGF.EmitLValueForFieldInitialization(
7266 LB = CGF.EmitLoadOfReferenceLValue(LowestElem, MapDecl->getType())
7270 CGF.EmitOMPSharedLValue(I->getAssociatedExpression())
7292 CharUnits TypeSize = CGF.getContext().getTypeSizeInChars(
7294 Address HB = CGF.Builder.CreateConstGEP(
7295 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
7296 LowestElem, CGF.VoidPtrTy, CGF.Int8Ty),
7326 LValue BaseLVal = EmitMemberExprBase(CGF, ME);
7328 CGF.EmitLValueForFieldInitialization(BaseLVal, FD)
7332 CGF.EmitOMPSharedLValue(MC.getAssociatedExpression())
7335 llvm::Value *ComponentLBPtr = ComponentLB.emitRawPointer(CGF);
7336 llvm::Value *LBPtr = LB.emitRawPointer(CGF);
7337 Size = CGF.Builder.CreatePtrDiff(CGF.Int8Ty, ComponentLBPtr,
7344 CombinedInfo.BasePointers.push_back(BP.emitRawPointer(CGF));
7347 CombinedInfo.Pointers.push_back(LB.emitRawPointer(CGF));
7348 CombinedInfo.Sizes.push_back(CGF.Builder.CreateIntCast(
7349 Size, CGF.Int64Ty, /*isSigned=*/true));
7354 LB = CGF.Builder.CreateConstGEP(ComponentLB, 1);
7357 CombinedInfo.BasePointers.push_back(BP.emitRawPointer(CGF));
7360 CombinedInfo.Pointers.push_back(LB.emitRawPointer(CGF));
7361 llvm::Value *LBPtr = LB.emitRawPointer(CGF);
7362 Size = CGF.Builder.CreatePtrDiff(
7363 CGF.Int8Ty, CGF.Builder.CreateConstGEP(HB, 1).emitRawPointer(CGF),
7366 CGF.Builder.CreateIntCast(Size, CGF.Int64Ty, /*isSigned=*/true));
7382 CombinedInfo.BasePointers.push_back(BP.emitRawPointer(CGF));
7385 CombinedInfo.Pointers.push_back(LB.emitRawPointer(CGF));
7386 CombinedInfo.Sizes.push_back(CGF.Builder.CreateIntCast(
7387 Size, CGF.Int64Ty, /*isSigned=*/true));
7393 BP.emitRawPointer(CGF));
7396 StructBaseCombinedInfo.Pointers.push_back(LB.emitRawPointer(CGF));
7397 StructBaseCombinedInfo.Sizes.push_back(CGF.Builder.CreateIntCast(
7398 Size, CGF.Int64Ty, /*isSigned=*/true));
7461 CGF.EmitArraySectionExpr(OASE, /*IsLowerBound=*/false)
7474 CGF.EmitArraySectionExpr(OASE, /*IsLowerBound=*/false)
7504 BP = CGF.EmitLoadOfPointer(BP, Ty->castAs<PointerType>());
7516 const ASTContext &Context = CGF.getContext();
7520 MapValuesArrayTy CurOffsets = {llvm::ConstantInt::get(CGF.CGM.Int64Ty, 0)};
7521 MapValuesArrayTy CurCounts = {llvm::ConstantInt::get(CGF.CGM.Int64Ty, 1)};
7523 MapValuesArrayTy DimSizes{llvm::ConstantInt::get(CGF.CGM.Int64Ty, 1)};
7568 llvm::ConstantInt::get(CGF.Int64Ty, ElementTypeSize));
7576 llvm::ConstantInt::get(CGF.Int64Ty, CAT->getZExtSize()));
7578 DimSizes.push_back(CGF.Builder.CreateIntCast(
7579 CGF.EmitScalarExpr(VAT->getSizeExpr()), CGF.Int64Ty,
7588 llvm::ConstantInt::get(CGF.CGM.Int64Ty, ElementTypeSize);
7602 llvm::Value *Offset = CGF.Builder.CreateIntCast(
7603 CGF.EmitScalarExpr(AE->getIdx()), CGF.Int64Ty,
7606 CurCounts.push_back(llvm::ConstantInt::get(CGF.Int64Ty, /*V=*/1));
7621 Offset = llvm::ConstantInt::get(CGF.Int64Ty, 0);
7623 Offset = CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(OffsetExpr),
7624 CGF.Int64Ty,
7639 Count = llvm::ConstantInt::get(CGF.Int64Ty, 1);
7648 ? CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(StrideExpr),
7649 CGF.Int64Ty, /*isSigned=*/false)
7652 Count = CGF.Builder.CreateUDiv(
7653 CGF.Builder.CreateNUWSub(*DI, Offset), Stride);
7655 Count = CGF.Builder.CreateNUWSub(*DI, Offset);
7658 Count = CGF.EmitScalarExpr(CountExpr);
7660 Count = CGF.Builder.CreateIntCast(Count, CGF.Int64Ty, /*isSigned=*/false);
7673 ? CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(StrideExpr),
7674 CGF.Int64Ty, /*isSigned=*/false)
7676 DimProd = CGF.Builder.CreateNUWMul(DimProd, *(DI - 1));
7678 CurStrides.push_back(CGF.Builder.CreateNUWMul(DimProd, Stride));
7723 const CGRecordLayout &RL = CGF.getTypes().getCGRecordLayout(RD);
7741 if (isEmptyRecordForLayout(CGF.getContext(), BaseTy) ||
7742 CGF.getContext()
7755 if (isEmptyRecordForLayout(CGF.getContext(), BaseTy))
7770 !isEmptyFieldForLayout(CGF.getContext(), Field)) {
7896 CodeGenFunction &CGF, bool IsDevAddr) {
7904 llvm::Constant::getNullValue(CGF.Int64Ty));
7912 &InfoGen](CodeGenFunction &CGF, const Expr *IE, const ValueDecl *VD,
7935 Ptr = CGF.EmitLValue(IE).getPointer(CGF);
7937 Ptr = CGF.EmitScalarExpr(IE);
7939 Ptr = CGF.EmitLoadOfScalar(CGF.EmitLValue(IE), IE->getExprLoc());
7941 UseDeviceDataCombinedInfoGen(VD, Ptr, CGF, IsDevAddr);
7945 auto &&IsMapInfoExist = [&Info](CodeGenFunction &CGF, const ValueDecl *VD,
7975 if (CGF.CGM.getOpenMPRuntime().hasRequiresUnifiedSharedMemory() ||
8012 if (IsMapInfoExist(CGF, VD, IE, /*IsDevAddr=*/false))
8014 MapInfoGen(CGF, IE, VD, Components, C->isImplicit(),
8034 if (IsMapInfoExist(CGF, VD, IE, /*IsDevAddr=*/true))
8036 MapInfoGen(CGF, IE, VD, Components, C->isImplicit(),
8134 Ptr = this->CGF.EmitLValue(L.IE).getPointer(CGF);
8136 Ptr = this->CGF.EmitScalarExpr(L.IE);
8145 BasePtr = this->CGF.EmitLValue(L.IE).getPointer(CGF);
8146 Ptr = this->CGF.EmitLoadOfScalar(this->CGF.EmitLValue(L.IE),
8163 llvm::Constant::getNullValue(this->CGF.Int64Ty));
8191 MappableExprsHandler(const OMPExecutableDirective &Dir, CodeGenFunction &CGF)
8192 : CurDir(&Dir), CGF(CGF) {
8237 MappableExprsHandler(const OMPDeclareMapperDecl &Dir, CodeGenFunction &CGF)
8238 : CurDir(&Dir), CGF(CGF) {}
8262 CombinedInfo.BasePointers.push_back(PartialStruct.Base.emitRawPointer(CGF));
8266 llvm::Value *LB = LBAddr.emitRawPointer(CGF);
8268 CGF.CurFuncDecl ? dyn_cast<CXXMethodDecl>(CGF.CurFuncDecl) : nullptr;
8280 CombinedInfo.Pointers.push_back(PartialStruct.Base.emitRawPointer(CGF));
8283 CGF.Builder.CreateIntCast(CGF.getTypeSize(Ty), CGF.Int64Ty,
8289 llvm::Value *HB = HBAddr.emitRawPointer(CGF);
8290 llvm::Value *HAddr = CGF.Builder.CreateConstGEP1_32(
8292 llvm::Value *CLAddr = CGF.Builder.CreatePointerCast(LB, CGF.VoidPtrTy);
8293 llvm::Value *CHAddr = CGF.Builder.CreatePointerCast(HAddr, CGF.VoidPtrTy);
8294 llvm::Value *Diff = CGF.Builder.CreatePtrDiff(CGF.Int8Ty, CHAddr, CLAddr);
8295 llvm::Value *Size = CGF.Builder.CreateIntCast(Diff, CGF.Int64Ty,
8375 Address VDAddr(Arg, CGF.ConvertTypeForMem(VDType),
8376 CGF.getContext().getDeclAlign(VD));
8377 LValue VDLVal = CGF.MakeAddrLValue(VDAddr, VDType);
8383 CGF.EmitLValueForFieldInitialization(VDLVal, ThisCapture);
8384 LValue ThisLValVal = CGF.EmitLValueForField(VDLVal, ThisCapture);
8385 LambdaPointers.try_emplace(ThisLVal.getPointer(CGF),
8386 VDLVal.getPointer(CGF));
8388 CombinedInfo.BasePointers.push_back(ThisLVal.getPointer(CGF));
8391 CombinedInfo.Pointers.push_back(ThisLValVal.getPointer(CGF));
8393 CGF.Builder.CreateIntCast(CGF.getTypeSize(CGF.getContext().VoidPtrTy),
8394 CGF.Int64Ty, /*isSigned=*/true));
8410 LValue VarLVal = CGF.EmitLValueForFieldInitialization(VDLVal, It->second);
8412 LValue VarLValVal = CGF.EmitLValueForField(VDLVal, It->second);
8413 LambdaPointers.try_emplace(VarLVal.getPointer(CGF),
8414 VDLVal.getPointer(CGF));
8416 CombinedInfo.BasePointers.push_back(VarLVal.getPointer(CGF));
8419 CombinedInfo.Pointers.push_back(VarLValVal.getPointer(CGF));
8420 CombinedInfo.Sizes.push_back(CGF.Builder.CreateIntCast(
8421 CGF.getTypeSize(
8423 CGF.Int64Ty, /*isSigned=*/true));
8425 RValue VarRVal = CGF.EmitLoadOfLValue(VarLVal, RD->getLocation());
8426 LambdaPointers.try_emplace(VarLVal.getPointer(CGF),
8427 VDLVal.getPointer(CGF));
8429 CombinedInfo.BasePointers.push_back(VarLVal.getPointer(CGF));
8433 CombinedInfo.Sizes.push_back(llvm::ConstantInt::get(CGF.Int64Ty, 0));
8504 CombinedInfo.Sizes.push_back(CGF.Builder.CreateIntCast(
8505 CGF.getTypeSize(CGF.getContext().VoidPtrTy), CGF.Int64Ty,
8759 CGF.Builder.CreateIntCast(CGF.getTypeSize(PtrTy->getPointeeType()),
8760 CGF.Int64Ty, /*isSigned=*/true));
8776 CombinedInfo.Sizes.push_back(CGF.Builder.CreateIntCast(
8777 CGF.getTypeSize(RI.getType()), CGF.Int64Ty, /*isSigned=*/true));
8782 CombinedInfo.Sizes.push_back(llvm::Constant::getNullValue(CGF.Int64Ty));
8791 CombinedInfo.Sizes.push_back(CGF.Builder.CreateIntCast(
8792 CGF.getTypeSize(ElementType), CGF.Int64Ty, /*isSigned=*/true));
8804 Address PtrAddr = CGF.EmitLoadOfReference(CGF.MakeAddrLValue(
8805 CV, ElementType, CGF.getContext().getDeclAlign(VD),
8807 CombinedInfo.Pointers.push_back(PtrAddr.emitRawPointer(CGF));
8843 emitMappingInformation(CodeGenFunction &CGF, llvm::OpenMPIRBuilder &OMPBuilder,
8862 PrintingPolicy P(CGF.getContext().getLangOpts());
8870 PresumedLoc PLoc = CGF.getContext().getSourceManager().getPresumedLoc(Loc);
8880 CodeGenFunction &CGF, MappableExprsHandler::MapCombinedInfoTy &CombinedInfo,
8883 CodeGenModule &CGM = CGF.CGM;
8890 InsertPointTy AllocaIP(CGF.AllocaInsertPt->getParent(),
8891 CGF.AllocaInsertPt->getIterator());
8892 InsertPointTy CodeGenIP(CGF.Builder.GetInsertBlock(),
8893 CGF.Builder.GetInsertPoint());
8896 return emitMappingInformation(CGF, OMPBuilder, MapExpr);
8915 MFunc = CGF.CGM.getOpenMPRuntime().getOrCreateUserDefinedMapperFunc(
9070 CodeGenFunction *CGF) {
9326 if (CGF) {
9327 auto &Decls = FunctionUDMMap.FindAndConstruct(CGF->CurFn);
9418 CodeGenFunction &CGF, const OMPExecutableDirective &D,
9419 llvm::function_ref<llvm::Value *(CodeGenFunction &CGF,
9430 return llvm::ConstantInt::get(CGF.Int64Ty, 0);
9433 if (llvm::Value *NumIterations = SizeEmitter(CGF, *LD))
9435 return llvm::ConstantInt::get(CGF.Int64Ty, 0);
9443 bool OffloadingMandatory, CodeGenFunction &CGF) {
9445 CGF.Builder.CreateUnreachable();
9449 CGF.GenerateOpenMPCapturedVars(CS, CapturedVars);
9451 OMPRuntime->emitOutlinedFunctionCall(CGF, D.getBeginLoc(), OutlinedFn,
9458 CodeGenFunction &CGF) {
9465 llvm::Value *DevVal = CGF.EmitScalarExpr(Device.getPointer());
9467 CGF.Builder.CreateIntCast(DevVal, CGF.Int64Ty, /*isSigned=*/true);
9469 DeviceID = CGF.Builder.getInt64(OMP_DEVICEID_UNDEF);
9475 CodeGenFunction &CGF) {
9476 llvm::Value *DynCGroupMem = CGF.Builder.getInt32(0);
9479 CodeGenFunction::RunCleanupsScope DynCGroupMemScope(CGF);
9480 llvm::Value *DynCGroupMemVal = CGF.EmitScalarExpr(
9482 DynCGroupMem = CGF.Builder.CreateIntCast(DynCGroupMemVal, CGF.Int32Ty,
9496 llvm::function_ref<llvm::Value *(CodeGenFunction &CGF,
9499 CodeGenFunction &CGF, CodeGenModule &CGM) {
9506 MappableExprsHandler MEHandler(D, CGF);
9527 CurInfo.Sizes.push_back(CGF.Builder.CreateIntCast(
9528 CGF.getTypeSize(RI->getType()), CGF.Int64Ty, /*isSigned=*/true));
9582 emitOffloadingArrays(CGF, CombinedInfo, Info, OMPBuilder);
9583 bool EmitDebug = CGF.CGM.getCodeGenOpts().getDebugInfo() !=
9585 OMPBuilder.emitOffloadingArraysArgument(CGF.Builder, Info.RTArgs, Info,
9591 CGF.VoidPtrTy, CGM.getPointerAlign());
9593 Address(Info.RTArgs.PointersArray, CGF.VoidPtrTy, CGM.getPointerAlign());
9595 Address(Info.RTArgs.SizesArray, CGF.Int64Ty, CGM.getPointerAlign());
9597 Address(Info.RTArgs.MappersArray, CGF.VoidPtrTy, CGM.getPointerAlign());
9604 SizeEmitter](CodeGenFunction &CGF, PrePostActionTy &) {
9613 RequiresOuterTask, CS, OffloadingMandatory, CGF);
9621 InputInfo.BasePointersArray.emitRawPointer(CGF);
9622 llvm::Value *PointersArray = InputInfo.PointersArray.emitRawPointer(CGF);
9623 llvm::Value *SizesArray = InputInfo.SizesArray.emitRawPointer(CGF);
9624 llvm::Value *MappersArray = InputInfo.MappersArray.emitRawPointer(CGF);
9628 OffloadingMandatory, &CGF](llvm::OpenMPIRBuilder::InsertPointTy IP)
9630 CGF.Builder.restoreIP(IP);
9632 RequiresOuterTask, CS, OffloadingMandatory, CGF);
9633 return CGF.Builder.saveIP();
9636 llvm::Value *DeviceID = emitDeviceID(Device, CGF);
9637 llvm::Value *NumTeams = OMPRuntime->emitNumTeamsForTargetDirective(CGF, D);
9639 OMPRuntime->emitNumThreadsForTargetDirective(CGF, D);
9640 llvm::Value *RTLoc = OMPRuntime->emitUpdateLocation(CGF, D.getBeginLoc());
9642 OMPRuntime->emitTargetNumIterationsCall(CGF, D, SizeEmitter);
9643 llvm::Value *DynCGGroupMem = emitDynCGGroupMem(D, CGF);
9645 CGF.AllocaInsertPt->getParent(), CGF.AllocaInsertPt->getIterator());
9655 CGF.Builder.restoreIP(OMPRuntime->getOMPBuilder().emitKernelLaunch(
9656 CGF.Builder, OutlinedFn, OutlinedFnID, EmitTargetCallFallbackCB, Args,
9661 CGF.EmitOMPTargetTaskBasedDirective(D, ThenGen, InputInfo);
9663 OMPRuntime->emitInlinedDirective(CGF, D.getDirectiveKind(), ThenGen);
9671 bool OffloadingMandatory, CodeGenFunction &CGF) {
9676 OffloadingMandatory](CodeGenFunction &CGF, PrePostActionTy &) {
9678 RequiresOuterTask, CS, OffloadingMandatory, CGF);
9683 CGF.EmitOMPTargetTaskBasedDirective(D, ElseGen, InputInfo);
9685 OMPRuntime->emitInlinedDirective(CGF, D.getDirectiveKind(), ElseGen);
9690 CodeGenFunction &CGF, const OMPExecutableDirective &D,
9693 llvm::function_ref<llvm::Value *(CodeGenFunction &CGF,
9696 if (!CGF.HaveInsertPoint())
9713 auto &&ArgsCodegen = [&CS, &CapturedVars](CodeGenFunction &CGF,
9715 CGF.GenerateOpenMPCapturedVars(CS, CapturedVars);
9717 emitInlinedDirective(CGF, OMPD_unknown, ArgsCodegen);
9726 &MapNamesArray, SizeEmitter](CodeGenFunction &CGF,
9731 MapNamesArray, SizeEmitter, CGF, CGM);
9736 OffloadingMandatory](CodeGenFunction &CGF, PrePostActionTy &) {
9738 CS, OffloadingMandatory, CGF);
9747 emitIfClause(CGF, IfCond, TargetThenGen, TargetElseGen);
9750 ThenRCG(CGF);
9754 ElseRCG(CGF);
10070 CodeGenFunction &CGF, const OMPExecutableDirective &D) const {
10166 void CGOpenMPRuntime::emitTeamsCall(CodeGenFunction &CGF,
10171 if (!CGF.HaveInsertPoint())
10174 llvm::Value *RTLoc = emitUpdateLocation(CGF, Loc);
10175 CodeGenFunction::RunCleanupsScope Scope(CGF);
10180 CGF.Builder.getInt32(CapturedVars.size()), // Number of captured vars
10181 CGF.Builder.CreateBitCast(OutlinedFn, getKmpc_MicroPointerTy())};
10188 CGF.EmitRuntimeCall(RTLFn, RealArgs);
10191 void CGOpenMPRuntime::emitNumTeamsClause(CodeGenFunction &CGF,
10195 if (!CGF.HaveInsertPoint())
10198 llvm::Value *RTLoc = emitUpdateLocation(CGF, Loc);
10202 ? CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(NumTeams),
10203 CGF.CGM.Int32Ty, /* isSigned = */ true)
10204 : CGF.Builder.getInt32(0);
10208 ? CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(ThreadLimit),
10209 CGF.CGM.Int32Ty, /* isSigned = */ true)
10210 : CGF.Builder.getInt32(0);
10213 llvm::Value *PushNumTeamsArgs[] = {RTLoc, getThreadID(CGF, Loc), NumTeamsVal,
10215 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
10220 void CGOpenMPRuntime::emitThreadLimitClause(CodeGenFunction &CGF,
10223 llvm::Value *RTLoc = emitUpdateLocation(CGF, Loc);
10226 ? CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(ThreadLimit),
10227 CGF.CGM.Int32Ty, /* isSigned = */ true)
10228 : CGF.Builder.getInt32(0);
10231 llvm::Value *ThreadLimitArgs[] = {RTLoc, getThreadID(CGF, Loc),
10233 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
10239 CodeGenFunction &CGF, const OMPExecutableDirective &D, const Expr *IfCond,
10242 if (!CGF.HaveInsertPoint())
10253 IfCondVal = CGF.EvaluateExprAsBool(IfCond);
10258 DeviceID = CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(Device),
10259 CGF.Int64Ty, /*isSigned=*/true);
10261 DeviceID = CGF.Builder.getInt64(OMP_DEVICEID_UNDEF);
10268 CGF.Builder.restoreIP(CodeGenIP);
10270 MappableExprsHandler MEHandler(D, CGF);
10274 return emitMappingInformation(CGF, OMPBuilder, MapExpr);
10287 CGF.Builder.restoreIP(CodeGenIP);
10291 CodeGen(CGF);
10296 CodeGen(CGF);
10302 CodeGen(CGF);
10306 return InsertPointTy(CGF.Builder.GetInsertBlock(),
10307 CGF.Builder.GetInsertPoint());
10320 MFunc = CGF.CGM.getOpenMPRuntime().getOrCreateUserDefinedMapperFunc(
10327 llvm::Value *RTLoc = emitUpdateLocation(CGF, D.getBeginLoc());
10329 InsertPointTy AllocaIP(CGF.AllocaInsertPt->getParent(),
10330 CGF.AllocaInsertPt->getIterator());
10331 InsertPointTy CodeGenIP(CGF.Builder.GetInsertBlock(),
10332 CGF.Builder.GetInsertPoint());
10334 CGF.Builder.restoreIP(OMPBuilder.createTargetData(
10340 CodeGenFunction &CGF, const OMPExecutableDirective &D, const Expr *IfCond,
10342 if (!CGF.HaveInsertPoint())
10355 &MapNamesArray](CodeGenFunction &CGF, PrePostActionTy &) {
10359 DeviceID = CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(Device),
10360 CGF.Int64Ty, /*isSigned=*/true);
10362 DeviceID = CGF.Builder.getInt64(OMP_DEVICEID_UNDEF);
10367 CGF.Builder.getInt32(InputInfo.NumberOfTargetItems);
10370 llvm::Value *RTLoc = emitUpdateLocation(CGF, D.getBeginLoc());
10374 InputInfo.BasePointersArray.emitRawPointer(CGF),
10375 InputInfo.PointersArray.emitRawPointer(CGF),
10376 InputInfo.SizesArray.emitRawPointer(CGF), MapTypesArray, MapNamesArray,
10377 InputInfo.MappersArray.emitRawPointer(CGF)});
10467 OffloadingArgs.push_back(llvm::Constant::getNullValue(CGF.Int32Ty));
10468 OffloadingArgs.push_back(llvm::Constant::getNullValue(CGF.VoidPtrTy));
10469 OffloadingArgs.push_back(llvm::Constant::getNullValue(CGF.Int32Ty));
10470 OffloadingArgs.push_back(llvm::Constant::getNullValue(CGF.VoidPtrTy));
10472 CGF.EmitRuntimeCall(
10478 &MapNamesArray](CodeGenFunction &CGF,
10484 MappableExprsHandler MEHandler(D, CGF);
10489 emitOffloadingArrays(CGF, CombinedInfo, Info, OMPBuilder,
10493 bool EmitDebug = CGF.CGM.getCodeGenOpts().getDebugInfo() !=
10495 OMPBuilder.emitOffloadingArraysArgument(CGF.Builder, Info.RTArgs, Info,
10500 CGF.VoidPtrTy, CGM.getPointerAlign());
10501 InputInfo.PointersArray = Address(Info.RTArgs.PointersArray, CGF.VoidPtrTy,
10504 Address(Info.RTArgs.SizesArray, CGF.Int64Ty, CGM.getPointerAlign());
10506 Address(Info.RTArgs.MappersArray, CGF.VoidPtrTy, CGM.getPointerAlign());
10510 CGF.EmitOMPTargetTaskBasedDirective(D, ThenGen, InputInfo);
10512 emitInlinedDirective(CGF, D.getDirectiveKind(), ThenGen);
10516 emitIfClause(CGF, IfCond, TargetThenGen,
10517 [](CodeGenFunction &CGF, PrePostActionTy &) {});
10520 ThenRCG(CGF);
11129 void Emit(CodeGenFunction &CGF, Flags /*flags*/) override {
11130 if (!CGF.HaveInsertPoint())
11132 CGF.EmitRuntimeCall(RTLFn, Args);
11137 void CGOpenMPRuntime::emitDoacrossInit(CodeGenFunction &CGF,
11140 if (!CGF.HaveInsertPoint())
11166 Address DimsAddr = CGF.CreateMemTemp(ArrayTy, "dims");
11167 CGF.EmitNullInitialization(DimsAddr, ArrayTy);
11171 LValue DimsLVal = CGF.MakeAddrLValue(
11172 CGF.Builder.CreateConstArrayGEP(DimsAddr, I), KmpDimTy);
11174 LValue UpperLVal = CGF.EmitLValueForField(
11176 llvm::Value *NumIterVal = CGF.EmitScalarConversion(
11177 CGF.EmitScalarExpr(NumIterations[I]), NumIterations[I]->getType(),
11179 CGF.EmitStoreOfScalar(NumIterVal, UpperLVal);
11181 LValue StrideLVal = CGF.EmitLValueForField(
11183 CGF.EmitStoreOfScalar(llvm::ConstantInt::getSigned(CGM.Int64Ty, /*V=*/1),
11190 emitUpdateLocation(CGF, D.getBeginLoc()),
11191 getThreadID(CGF, D.getBeginLoc()),
11193 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
11194 CGF.Builder.CreateConstArrayGEP(DimsAddr, 0).emitRawPointer(CGF),
11199 CGF.EmitRuntimeCall(RTLFn, Args);
11201 emitUpdateLocation(CGF, D.getEndLoc()), getThreadID(CGF, D.getEndLoc())};
11204 CGF.EHStack.pushCleanup<DoacrossCleanupTy>(NormalAndEHCleanup, FiniRTLFn,
11209 static void EmitDoacrossOrdered(CodeGenFunction &CGF, CodeGenModule &CGM,
11217 Address CntAddr = CGF.CreateMemTemp(ArrayTy, ".cnt.addr");
11221 llvm::Value *CntVal = CGF.EmitScalarConversion(
11222 CGF.EmitScalarExpr(CounterVal), CounterVal->getType(), Int64Ty,
11224 CGF.EmitStoreOfScalar(CntVal, CGF.Builder.CreateConstArrayGEP(CntAddr, I),
11229 CGF.Builder.CreateConstArrayGEP(CntAddr, 0).emitRawPointer(CGF)};
11241 CGF.EmitRuntimeCall(RTLFn, Args);
11244 void CGOpenMPRuntime::emitDoacrossOrdered(CodeGenFunction &CGF,
11247 CGF, CGM, C, emitUpdateLocation(CGF, C->getBeginLoc()),
11248 getThreadID(CGF, C->getBeginLoc()));
11251 void CGOpenMPRuntime::emitDoacrossOrdered(CodeGenFunction &CGF,
11254 CGF, CGM, C, emitUpdateLocation(CGF, C->getBeginLoc()),
11255 getThreadID(CGF, C->getBeginLoc()));
11258 void CGOpenMPRuntime::emitCall(CodeGenFunction &CGF, SourceLocation Loc,
11262 auto DL = ApplyDebugLocation::CreateDefaultArtificial(CGF, Loc);
11266 CGF.EmitNounwindRuntimeCall(Fn, Args);
11270 CGF.EmitRuntimeCall(Callee, Args);
11274 CodeGenFunction &CGF, SourceLocation Loc, llvm::FunctionCallee OutlinedFn,
11276 emitCall(CGF, Loc, OutlinedFn, Args);
11279 void CGOpenMPRuntime::emitFunctionProlog(CodeGenFunction &CGF, const Decl *D) {
11285 Address CGOpenMPRuntime::getParameterAddress(CodeGenFunction &CGF,
11288 return CGF.GetAddrOfLocalVar(NativeParam);
11293 static llvm::Value *getAllocatorVal(CodeGenFunction &CGF,
11297 AllocVal = CGF.EmitScalarExpr(Allocator);
11300 AllocVal = CGF.EmitScalarConversion(AllocVal, Allocator->getType(),
11301 CGF.getContext().VoidPtrTy,
11306 CGF.CGM.getTypes().ConvertType(CGF.getContext().VoidPtrTy));
11321 Address CGOpenMPRuntime::getAddressOfLocalVariable(CodeGenFunction &CGF,
11327 auto It = FunctionToUntiedTaskStackMap.find(CGF.CurFn);
11345 Size = CGF.getTypeSize(CVD->getType());
11347 Size = CGF.Builder.CreateNUWAdd(
11349 Size = CGF.Builder.CreateUDiv(Size, CGM.getSize(Align));
11350 Size = CGF.Builder.CreateNUWMul(Size, CGM.getSize(Align));
11355 llvm::Value *ThreadID = getThreadID(CGF, CVD->getBeginLoc());
11358 llvm::Value *AllocVal = getAllocatorVal(CGF, Allocator);
11368 llvm::Value *Addr = CGF.EmitRuntimeCall(
11374 Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
11375 Addr, CGF.ConvertTypeForMem(Ty), getName({CVD->getName(), ".addr"}));
11377 CGF.EmitStoreOfScalar(Addr, UntiedAddr, /*Volatile=*/false, Ty);
11392 void Emit(CodeGenFunction &CGF, Flags /*flags*/) override {
11393 if (!CGF.HaveInsertPoint())
11396 Args[0] = CGF.CGM.getOpenMPRuntime().getThreadID(
11397 CGF, SourceLocation::getFromRawEncoding(LocEncoding));
11398 Args[1] = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
11399 Addr.emitRawPointer(CGF), CGF.VoidPtrTy);
11400 llvm::Value *AllocVal = getAllocatorVal(CGF, AllocExpr);
11402 CGF.EmitRuntimeCall(RTLFn, Args);
11408 : Address(Addr, CGF.ConvertTypeForMem(CVD->getType()), Align);
11409 CGF.EHStack.pushCleanup<OMPAllocateCleanupTy>(
11414 dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo))
11415 Region->emitUntiedSwitch(CGF);
11421 bool CGOpenMPRuntime::isLocalVarInUntiedTask(CodeGenFunction &CGF,
11423 auto It = FunctionToUntiedTaskStackMap.find(CGF.CurFn);
11462 CodeGenFunction &CGF,
11465 : CGM(CGF.CGM), NeedToPush(!LocalVars.empty()) {
11469 CGF.CurFn, CGM.getOpenMPRuntime().UntiedLocalVarsStack.size());
11567 CodeGenFunction &CGF, const OMPExecutableDirective &S, LValue IVLVal)
11568 : CGM(CGF.CGM),
11595 Data.Fn = CGF.CurFn;
11599 CodeGenFunction &CGF, const OMPExecutableDirective &S)
11600 : CGM(CGF.CGM), Action(ActionToDo::DoNotPush) {
11612 Data.Fn = CGF.CurFn;
11619 CodeGenFunction &CGF, const OMPExecutableDirective &S) {
11620 return LastprivateConditionalRAII(CGF, S);
11639 Address CGOpenMPRuntime::emitLastprivateConditionalInit(CodeGenFunction &CGF,
11642 auto I = LastprivateConditionalToTypes.find(CGF.CurFn);
11644 I = LastprivateConditionalToTypes.try_emplace(CGF.CurFn).first;
11657 Address Addr = CGF.CreateMemTemp(NewType, C.getDeclAlign(VD), VD->getName());
11658 BaseLVal = CGF.MakeAddrLValue(Addr, NewType, AlignmentSource::Decl);
11667 CGF.EmitLValueForField(BaseLVal, FiredField);
11668 CGF.EmitStoreOfScalar(
11669 llvm::ConstantInt::getNullValue(CGF.ConvertTypeForMem(C.CharTy)),
11671 return CGF.EmitLValueForField(BaseLVal, VDField).getAddress();
11745 void CGOpenMPRuntime::emitLastprivateConditionalUpdate(CodeGenFunction &CGF,
11752 llvm::Type *LLIVTy = CGF.ConvertTypeForMem(IVLVal.getType());
11758 CGF.MakeNaturalAlignRawAddrLValue(LastIV, IVLVal.getType());
11763 CGF.ConvertTypeForMem(LVal.getType()), UniqueDeclName);
11767 CGF.MakeRawAddrLValue(Last, LVal.getType(), LVal.getAlignment());
11771 llvm::Value *IVVal = CGF.EmitLoadOfScalar(IVLVal, Loc);
11779 Loc](CodeGenFunction &CGF, PrePostActionTy &Action) {
11780 Action.Enter(CGF);
11781 llvm::Value *LastIVVal = CGF.EmitLoadOfScalar(LastIVLVal, Loc);
11786 CmpRes = CGF.Builder.CreateICmpSLE(LastIVVal, IVVal);
11790 CmpRes = CGF.Builder.CreateICmpULE(LastIVVal, IVVal);
11792 llvm::BasicBlock *ThenBB = CGF.createBasicBlock("lp_cond_then");
11793 llvm::BasicBlock *ExitBB = CGF.createBasicBlock("lp_cond_exit");
11794 CGF.Builder.CreateCondBr(CmpRes, ThenBB, ExitBB);
11796 CGF.EmitBlock(ThenBB);
11799 CGF.EmitStoreOfScalar(IVVal, LastIVLVal);
11802 switch (CGF.getEvaluationKind(LVal.getType())) {
11804 llvm::Value *PrivVal = CGF.EmitLoadOfScalar(LVal, Loc);
11805 CGF.EmitStoreOfScalar(PrivVal, LastLVal);
11809 CodeGenFunction::ComplexPairTy PrivVal = CGF.EmitLoadOfComplex(LVal, Loc);
11810 CGF.EmitStoreOfComplex(PrivVal, LastLVal, /*isInit=*/false);
11818 CGF.EmitBranch(ExitBB);
11820 (void)ApplyDebugLocation::CreateEmpty(CGF);
11821 CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
11827 ThenRCG(CGF);
11829 emitCriticalRegion(CGF, UniqueDeclName, CodeGen, Loc);
11833 void CGOpenMPRuntime::checkAndEmitLastprivateConditional(CodeGenFunction &CGF,
11835 if (CGF.getLangOpts().OpenMP < 50 || LastprivateConditionalStack.empty())
11847 if (FoundFn != CGF.CurFn) {
11855 LValue PrivLVal = CGF.EmitLValue(FoundE);
11856 Address StructAddr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
11858 CGF.ConvertTypeForMem(CGF.getContext().getPointerType(StructTy)),
11859 CGF.ConvertTypeForMem(StructTy));
11861 CGF.MakeAddrLValue(StructAddr, StructTy, AlignmentSource::Decl);
11862 LValue FiredLVal = CGF.EmitLValueForField(BaseLVal, FiredDecl);
11863 CGF.EmitAtomicStore(RValue::get(llvm::ConstantInt::get(
11864 CGF.ConvertTypeForMem(FiredDecl->getType()), 1)),
11872 LValue LVal = CGF.EmitLValue(FoundE);
11873 emitLastprivateConditionalUpdate(CGF, IVLVal, UniqueDeclName, LVal,
11878 CodeGenFunction &CGF, const OMPExecutableDirective &D,
11880 if (CGF.getLangOpts().OpenMP < 50 || LastprivateConditionalStack.empty())
11885 if (It == Range.end() || It->Fn != CGF.CurFn)
11903 CGF.EmitLValueForField(BaseLVal, std::get<2>(I->getSecond()));
11904 llvm::Value *Res = CGF.EmitLoadOfScalar(FiredLVal, D.getBeginLoc());
11905 llvm::Value *Cmp = CGF.Builder.CreateIsNotNull(Res);
11906 llvm::BasicBlock *ThenBB = CGF.createBasicBlock("lpc.then");
11907 llvm::BasicBlock *DoneBB = CGF.createBasicBlock("lpc.done");
11909 CGF.Builder.CreateCondBr(Cmp, ThenBB, DoneBB);
11910 CGF.EmitBlock(ThenBB);
11911 Address Addr = CGF.GetAddrOfLocalVar(VD);
11914 LVal = CGF.EmitLoadOfReferenceLValue(Addr, VD->getType(),
11917 LVal = CGF.MakeAddrLValue(Addr, VD->getType().getNonReferenceType(),
11919 emitLastprivateConditionalUpdate(CGF, It->IVLVal, Pair.second, LVal,
11921 auto AL = ApplyDebugLocation::CreateArtificial(CGF);
11922 CGF.EmitBlock(DoneBB, /*IsFinal=*/true);
11928 CodeGenFunction &CGF, LValue PrivLVal, const VarDecl *VD,
11930 if (CGF.getLangOpts().OpenMP < 50)
11940 LValue LPLVal = CGF.MakeRawAddrLValue(
11942 llvm::Value *Res = CGF.EmitLoadOfScalar(LPLVal, Loc);
11943 CGF.EmitStoreOfScalar(Res, PrivLVal);
11947 CodeGenFunction &CGF, const OMPExecutableDirective &D,
11954 CodeGenFunction &CGF, const OMPExecutableDirective &D,
11968 void CGOpenMPSIMDRuntime::emitParallelCall(CodeGenFunction &CGF,
11978 CodeGenFunction &CGF, StringRef CriticalName,
11984 void CGOpenMPSIMDRuntime::emitMasterRegion(CodeGenFunction &CGF,
11990 void CGOpenMPSIMDRuntime::emitMaskedRegion(CodeGenFunction &CGF,
11997 void CGOpenMPSIMDRuntime::emitTaskyieldCall(CodeGenFunction &CGF,
12003 CodeGenFunction &CGF, const RegionCodeGenTy &TaskgroupOpGen,
12009 CodeGenFunction &CGF, const RegionCodeGenTy &SingleOpGen,
12016 void CGOpenMPSIMDRuntime::emitOrderedRegion(CodeGenFunction &CGF,
12023 void CGOpenMPSIMDRuntime::emitBarrierCall(CodeGenFunction &CGF,
12032 CodeGenFunction &CGF, SourceLocation Loc,
12038 void CGOpenMPSIMDRuntime::emitForDispatchDeinit(CodeGenFunction &CGF,
12044 CodeGenFunction &CGF, SourceLocation Loc, OpenMPDirectiveKind DKind,
12050 CodeGenFunction &CGF, SourceLocation Loc,
12055 void CGOpenMPSIMDRuntime::emitForOrderedIterationEnd(CodeGenFunction &CGF,
12062 void CGOpenMPSIMDRuntime::emitForStaticFinish(CodeGenFunction &CGF,
12068 llvm::Value *CGOpenMPSIMDRuntime::emitForNext(CodeGenFunction &CGF,
12076 void CGOpenMPSIMDRuntime::emitNumThreadsClause(CodeGenFunction &CGF,
12082 void CGOpenMPSIMDRuntime::emitProcBindClause(CodeGenFunction &CGF,
12088 Address CGOpenMPSIMDRuntime::getAddrOfThreadPrivate(CodeGenFunction &CGF,
12097 CodeGenFunction *CGF) {
12102 CodeGenFunction &CGF, QualType VarType, StringRef Name) {
12106 void CGOpenMPSIMDRuntime::emitFlush(CodeGenFunction &CGF,
12113 void CGOpenMPSIMDRuntime::emitTaskCall(CodeGenFunction &CGF, SourceLocation Loc,
12123 CodeGenFunction &CGF, SourceLocation Loc, const OMPLoopDirective &D,
12130 CodeGenFunction &CGF, SourceLocation Loc, ArrayRef<const Expr *> Privates,
12134 CGOpenMPRuntime::emitReduction(CGF, Loc, Privates, LHSExprs, RHSExprs,
12139 CodeGenFunction &CGF, SourceLocation Loc, ArrayRef<const Expr *> LHSExprs,
12144 void CGOpenMPSIMDRuntime::emitTaskReductionFini(CodeGenFunction &CGF,
12150 void CGOpenMPSIMDRuntime::emitTaskReductionFixups(CodeGenFunction &CGF,
12157 Address CGOpenMPSIMDRuntime::getTaskReductionItem(CodeGenFunction &CGF,
12164 void CGOpenMPSIMDRuntime::emitTaskwaitCall(CodeGenFunction &CGF,
12171 CodeGenFunction &CGF, SourceLocation Loc,
12176 void CGOpenMPSIMDRuntime::emitCancelCall(CodeGenFunction &CGF,
12190 CodeGenFunction &CGF, const OMPExecutableDirective &D,
12193 llvm::function_ref<llvm::Value *(CodeGenFunction &CGF,
12211 void CGOpenMPSIMDRuntime::emitTeamsCall(CodeGenFunction &CGF,
12219 void CGOpenMPSIMDRuntime::emitNumTeamsClause(CodeGenFunction &CGF,
12227 CodeGenFunction &CGF, const OMPExecutableDirective &D, const Expr *IfCond,
12234 CodeGenFunction &CGF, const OMPExecutableDirective &D, const Expr *IfCond,
12239 void CGOpenMPSIMDRuntime::emitDoacrossInit(CodeGenFunction &CGF,
12245 void CGOpenMPSIMDRuntime::emitDoacrossOrdered(CodeGenFunction &CGF,
12250 void CGOpenMPSIMDRuntime::emitDoacrossOrdered(CodeGenFunction &CGF,
12262 CGOpenMPSIMDRuntime::getParameterAddress(CodeGenFunction &CGF,