Lines Matching refs:CGF
83 void EmitBody(CodeGenFunction &CGF, const Stmt *S) override;
87 virtual LValue getThreadIDVariableLValue(CodeGenFunction &CGF);
156 void Enter(CodeGenFunction &CGF) override { in Enter() argument
159 LValue PartIdLVal = CGF.EmitLoadOfPointerLValue( in Enter()
160 CGF.GetAddrOfLocalVar(PartIDVar), in Enter()
163 CGF.EmitLoadOfScalar(PartIdLVal, PartIDVar->getLocation()); in Enter()
164 llvm::BasicBlock *DoneBB = CGF.createBasicBlock(".untied.done."); in Enter()
165 UntiedSwitch = CGF.Builder.CreateSwitch(Res, DoneBB); in Enter()
166 CGF.EmitBlock(DoneBB); in Enter()
167 CGF.EmitBranchThroughCleanup(CGF.ReturnBlock); in Enter()
168 CGF.EmitBlock(CGF.createBasicBlock(".untied.jmp.")); in Enter()
169 UntiedSwitch->addCase(CGF.Builder.getInt32(0), in Enter()
170 CGF.Builder.GetInsertBlock()); in Enter()
171 emitUntiedSwitch(CGF); in Enter()
174 void emitUntiedSwitch(CodeGenFunction &CGF) const { in emitUntiedSwitch()
176 LValue PartIdLVal = CGF.EmitLoadOfPointerLValue( in emitUntiedSwitch()
177 CGF.GetAddrOfLocalVar(PartIDVar), in emitUntiedSwitch()
179 CGF.EmitStoreOfScalar(CGF.Builder.getInt32(UntiedSwitch->getNumCases()), in emitUntiedSwitch()
181 UntiedCodeGen(CGF); in emitUntiedSwitch()
183 CGF.getJumpDestInCurrentScope(".untied.next."); in emitUntiedSwitch()
184 CGF.EmitBranch(CGF.ReturnBlock.getBlock()); in emitUntiedSwitch()
185 CGF.EmitBlock(CGF.createBasicBlock(".untied.jmp.")); in emitUntiedSwitch()
186 UntiedSwitch->addCase(CGF.Builder.getInt32(UntiedSwitch->getNumCases()), in emitUntiedSwitch()
187 CGF.Builder.GetInsertBlock()); in emitUntiedSwitch()
188 CGF.EmitBranchThroughCleanup(CurPoint); in emitUntiedSwitch()
189 CGF.EmitBlock(CurPoint.getBlock()); in emitUntiedSwitch()
209 LValue getThreadIDVariableLValue(CodeGenFunction &CGF) override;
214 void emitUntiedSwitch(CodeGenFunction &CGF) override { in emitUntiedSwitch() argument
215 Action.emitUntiedSwitch(CGF); in emitUntiedSwitch()
282 LValue getThreadIDVariableLValue(CodeGenFunction &CGF) override { in getThreadIDVariableLValue() argument
284 return OuterRegionInfo->getThreadIDVariableLValue(CGF); in getThreadIDVariableLValue()
295 void emitUntiedSwitch(CodeGenFunction &CGF) override { in emitUntiedSwitch() argument
297 OuterRegionInfo->emitUntiedSwitch(CGF); in emitUntiedSwitch()
351 CGOpenMPInnerExprInfo(CodeGenFunction &CGF, const CapturedStmt &CS) in CGOpenMPInnerExprInfo() argument
352 : CGOpenMPInlinedRegionInfo(CGF.CapturedStmtInfo, EmptyCodeGen, in CGOpenMPInnerExprInfo()
355 PrivScope(CGF) { in CGOpenMPInnerExprInfo()
367 DeclRefExpr DRE(CGF.getContext(), const_cast<VarDecl *>(VD), in CGOpenMPInnerExprInfo()
372 VD, [&CGF, &DRE]() { return CGF.EmitLValue(&DRE).getAddress(CGF); }); in CGOpenMPInnerExprInfo()
385 void EmitBody(CodeGenFunction &CGF, const Stmt *S) override { in EmitBody() argument
409 CodeGenFunction &CGF; member in __anonad2d34460111::InlinedOpenMPRegionRAII
420 InlinedOpenMPRegionRAII(CodeGenFunction &CGF, const RegionCodeGenTy &CodeGen, in InlinedOpenMPRegionRAII() argument
423 : CGF(CGF), NoInheritance(NoInheritance) { in InlinedOpenMPRegionRAII()
425 CGF.CapturedStmtInfo = new CGOpenMPInlinedRegionInfo( in InlinedOpenMPRegionRAII()
426 CGF.CapturedStmtInfo, CodeGen, Kind, HasCancel); in InlinedOpenMPRegionRAII()
428 std::swap(CGF.LambdaCaptureFields, LambdaCaptureFields); in InlinedOpenMPRegionRAII()
429 LambdaThisCaptureField = CGF.LambdaThisCaptureField; in InlinedOpenMPRegionRAII()
430 CGF.LambdaThisCaptureField = nullptr; in InlinedOpenMPRegionRAII()
431 BlockInfo = CGF.BlockInfo; in InlinedOpenMPRegionRAII()
432 CGF.BlockInfo = nullptr; in InlinedOpenMPRegionRAII()
439 cast<CGOpenMPInlinedRegionInfo>(CGF.CapturedStmtInfo)->getOldCSI(); in ~InlinedOpenMPRegionRAII()
440 delete CGF.CapturedStmtInfo; in ~InlinedOpenMPRegionRAII()
441 CGF.CapturedStmtInfo = OldCSI; in ~InlinedOpenMPRegionRAII()
443 std::swap(CGF.LambdaCaptureFields, LambdaCaptureFields); in ~InlinedOpenMPRegionRAII()
444 CGF.LambdaThisCaptureField = LambdaThisCaptureField; in ~InlinedOpenMPRegionRAII()
445 CGF.BlockInfo = BlockInfo; in ~InlinedOpenMPRegionRAII()
585 void Emit(CodeGenFunction &CGF, Flags /*flags*/) override { in Emit() argument
586 if (!CGF.HaveInsertPoint()) in Emit()
588 Action->Exit(CGF); in Emit()
594 void RegionCodeGenTy::operator()(CodeGenFunction &CGF) const { in operator ()()
595 CodeGenFunction::RunCleanupsScope Scope(CGF); in operator ()()
597 CGF.EHStack.pushCleanup<CleanupTy>(NormalAndEHCleanup, PrePostAction); in operator ()()
598 Callback(CodeGen, CGF, *PrePostAction); in operator ()()
601 Callback(CodeGen, CGF, Action); in operator ()()
618 static void emitInitWithReductionInitializer(CodeGenFunction &CGF, in emitInitWithReductionInitializer() argument
625 CGF.CGM.getOpenMPRuntime().getUserDefinedReduction(DRD); in emitInitWithReductionInitializer()
634 CodeGenFunction::OMPPrivateScope PrivateScope(CGF); in emitInitWithReductionInitializer()
641 CodeGenFunction::OpaqueValueMapping Map(CGF, OVE, Func); in emitInitWithReductionInitializer()
642 CGF.EmitIgnoredExpr(InitOp); in emitInitWithReductionInitializer()
644 llvm::Constant *Init = CGF.CGM.EmitNullConstant(Ty); in emitInitWithReductionInitializer()
645 std::string Name = CGF.CGM.getOpenMPRuntime().getName({"init"}); in emitInitWithReductionInitializer()
647 CGF.CGM.getModule(), Init->getType(), /*isConstant=*/true, in emitInitWithReductionInitializer()
649 LValue LV = CGF.MakeNaturalAlignAddrLValue(GV, Ty); in emitInitWithReductionInitializer()
651 switch (CGF.getEvaluationKind(Ty)) { in emitInitWithReductionInitializer()
653 InitRVal = CGF.EmitLoadOfLValue(LV, DRD->getLocation()); in emitInitWithReductionInitializer()
657 RValue::getComplex(CGF.EmitLoadOfComplex(LV, DRD->getLocation())); in emitInitWithReductionInitializer()
661 CodeGenFunction::OpaqueValueMapping OpaqueMap(CGF, &OVE, LV); in emitInitWithReductionInitializer()
662 CGF.EmitAnyExprToMem(&OVE, Private, Ty.getQualifiers(), in emitInitWithReductionInitializer()
668 CodeGenFunction::OpaqueValueMapping OpaqueMap(CGF, &OVE, InitRVal); in emitInitWithReductionInitializer()
669 CGF.EmitAnyExprToMem(&OVE, Private, Ty.getQualifiers(), in emitInitWithReductionInitializer()
679 static void EmitOMPAggregateInit(CodeGenFunction &CGF, Address DestAddr, in EmitOMPAggregateInit() argument
689 llvm::Value *NumElements = CGF.emitArrayLength(ArrayTy, ElementTy, DestAddr); in EmitOMPAggregateInit()
691 CGF.Builder.CreateElementBitCast(DestAddr, DestAddr.getElementType()); in EmitOMPAggregateInit()
694 CGF.Builder.CreateElementBitCast(SrcAddr, DestAddr.getElementType()); in EmitOMPAggregateInit()
701 llvm::Value *DestEnd = CGF.Builder.CreateGEP(DestBegin, NumElements); in EmitOMPAggregateInit()
703 llvm::BasicBlock *BodyBB = CGF.createBasicBlock("omp.arrayinit.body"); in EmitOMPAggregateInit()
704 llvm::BasicBlock *DoneBB = CGF.createBasicBlock("omp.arrayinit.done"); in EmitOMPAggregateInit()
706 CGF.Builder.CreateICmpEQ(DestBegin, DestEnd, "omp.arrayinit.isempty"); in EmitOMPAggregateInit()
707 CGF.Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB); in EmitOMPAggregateInit()
710 llvm::BasicBlock *EntryBB = CGF.Builder.GetInsertBlock(); in EmitOMPAggregateInit()
711 CGF.EmitBlock(BodyBB); in EmitOMPAggregateInit()
713 CharUnits ElementSize = CGF.getContext().getTypeSizeInChars(ElementTy); in EmitOMPAggregateInit()
718 SrcElementPHI = CGF.Builder.CreatePHI(SrcBegin->getType(), 2, in EmitOMPAggregateInit()
725 llvm::PHINode *DestElementPHI = CGF.Builder.CreatePHI( in EmitOMPAggregateInit()
734 CodeGenFunction::RunCleanupsScope InitScope(CGF); in EmitOMPAggregateInit()
736 emitInitWithReductionInitializer(CGF, DRD, Init, DestElementCurrent, in EmitOMPAggregateInit()
739 CGF.EmitAnyExprToMem(Init, DestElementCurrent, ElementTy.getQualifiers(), in EmitOMPAggregateInit()
745 llvm::Value *SrcElementNext = CGF.Builder.CreateConstGEP1_32( in EmitOMPAggregateInit()
747 SrcElementPHI->addIncoming(SrcElementNext, CGF.Builder.GetInsertBlock()); in EmitOMPAggregateInit()
751 llvm::Value *DestElementNext = CGF.Builder.CreateConstGEP1_32( in EmitOMPAggregateInit()
755 CGF.Builder.CreateICmpEQ(DestElementNext, DestEnd, "omp.arraycpy.done"); in EmitOMPAggregateInit()
756 CGF.Builder.CreateCondBr(Done, DoneBB, BodyBB); in EmitOMPAggregateInit()
757 DestElementPHI->addIncoming(DestElementNext, CGF.Builder.GetInsertBlock()); in EmitOMPAggregateInit()
760 CGF.EmitBlock(DoneBB, /*IsFinished=*/true); in EmitOMPAggregateInit()
763 LValue ReductionCodeGen::emitSharedLValue(CodeGenFunction &CGF, const Expr *E) { in emitSharedLValue() argument
764 return CGF.EmitOMPSharedLValue(E); in emitSharedLValue()
767 LValue ReductionCodeGen::emitSharedLValueUB(CodeGenFunction &CGF, in emitSharedLValueUB() argument
770 return CGF.EmitOMPArraySectionExpr(OASE, /*IsLowerBound=*/false); in emitSharedLValueUB()
775 CodeGenFunction &CGF, unsigned N, Address PrivateAddr, LValue SharedLVal, in emitAggregateInitialization() argument
784 EmitOMPAggregateInit(CGF, PrivateAddr, PrivateVD->getType(), in emitAggregateInitialization()
788 DRD, SharedLVal.getAddress(CGF)); in emitAggregateInitialization()
810 void ReductionCodeGen::emitSharedOrigLValue(CodeGenFunction &CGF, unsigned N) { in emitSharedOrigLValue() argument
813 LValue First = emitSharedLValue(CGF, ClausesData[N].Shared); in emitSharedOrigLValue()
814 LValue Second = emitSharedLValueUB(CGF, ClausesData[N].Shared); in emitSharedOrigLValue()
819 LValue First = emitSharedLValue(CGF, ClausesData[N].Ref); in emitSharedOrigLValue()
820 LValue Second = emitSharedLValueUB(CGF, ClausesData[N].Ref); in emitSharedOrigLValue()
825 void ReductionCodeGen::emitAggregateType(CodeGenFunction &CGF, unsigned N) { in emitAggregateType() argument
832 CGF.getTypeSize(OrigAddresses[N].first.getType().getNonReferenceType()), in emitAggregateType()
839 cast<llvm::PointerType>(OrigAddresses[N].first.getPointer(CGF)->getType()) in emitAggregateType()
843 Size = CGF.Builder.CreatePtrDiff(OrigAddresses[N].second.getPointer(CGF), in emitAggregateType()
844 OrigAddresses[N].first.getPointer(CGF)); in emitAggregateType()
845 Size = CGF.Builder.CreateNUWAdd( in emitAggregateType()
847 SizeInChars = CGF.Builder.CreateNUWMul(Size, ElemSizeOf); in emitAggregateType()
850 CGF.getTypeSize(OrigAddresses[N].first.getType().getNonReferenceType()); in emitAggregateType()
851 Size = CGF.Builder.CreateExactUDiv(SizeInChars, ElemSizeOf); in emitAggregateType()
855 CGF, in emitAggregateType()
857 CGF.getContext().getAsVariableArrayType(PrivateType)->getSizeExpr()), in emitAggregateType()
859 CGF.EmitVariablyModifiedType(PrivateType); in emitAggregateType()
862 void ReductionCodeGen::emitAggregateType(CodeGenFunction &CGF, unsigned N, in emitAggregateType() argument
874 CGF, in emitAggregateType()
876 CGF.getContext().getAsVariableArrayType(PrivateType)->getSizeExpr()), in emitAggregateType()
878 CGF.EmitVariablyModifiedType(PrivateType); in emitAggregateType()
882 CodeGenFunction &CGF, unsigned N, Address PrivateAddr, LValue SharedLVal, in emitInitialization() argument
890 PrivateAddr = CGF.Builder.CreateElementBitCast( in emitInitialization()
891 PrivateAddr, CGF.ConvertTypeForMem(PrivateType)); in emitInitialization()
893 SharedLVal = CGF.MakeAddrLValue( in emitInitialization()
894 CGF.Builder.CreateElementBitCast(SharedLVal.getAddress(CGF), in emitInitialization()
895 CGF.ConvertTypeForMem(SharedType)), in emitInitialization()
897 CGF.CGM.getTBAAInfoForSubobject(SharedAddresses[N].first, SharedType)); in emitInitialization()
898 if (CGF.getContext().getAsArrayType(PrivateVD->getType())) { in emitInitialization()
900 (void)DefaultInit(CGF); in emitInitialization()
901 emitAggregateInitialization(CGF, N, PrivateAddr, SharedLVal, DRD); in emitInitialization()
903 (void)DefaultInit(CGF); in emitInitialization()
904 emitInitWithReductionInitializer(CGF, DRD, ClausesData[N].ReductionOp, in emitInitialization()
905 PrivateAddr, SharedLVal.getAddress(CGF), in emitInitialization()
907 } else if (!DefaultInit(CGF) && PrivateVD->hasInit() && in emitInitialization()
908 !CGF.isTrivialInitializer(PrivateVD->getInit())) { in emitInitialization()
909 CGF.EmitAnyExprToMem(PrivateVD->getInit(), PrivateAddr, in emitInitialization()
923 void ReductionCodeGen::emitCleanups(CodeGenFunction &CGF, unsigned N, in emitCleanups() argument
930 PrivateAddr = CGF.Builder.CreateElementBitCast( in emitCleanups()
931 PrivateAddr, CGF.ConvertTypeForMem(PrivateType)); in emitCleanups()
932 CGF.pushDestroy(DTorKind, PrivateAddr, PrivateType); in emitCleanups()
936 static LValue loadToBegin(CodeGenFunction &CGF, QualType BaseTy, QualType ElTy, in loadToBegin() argument
940 !CGF.getContext().hasSameType(BaseTy, ElTy)) { in loadToBegin()
942 BaseLV = CGF.EmitLoadOfPointerLValue(BaseLV.getAddress(CGF), PtrTy); in loadToBegin()
944 LValue RefLVal = CGF.MakeAddrLValue(BaseLV.getAddress(CGF), BaseTy); in loadToBegin()
945 BaseLV = CGF.EmitLoadOfReferenceLValue(RefLVal); in loadToBegin()
949 return CGF.MakeAddrLValue( in loadToBegin()
950 CGF.Builder.CreateElementBitCast(BaseLV.getAddress(CGF), in loadToBegin()
951 CGF.ConvertTypeForMem(ElTy)), in loadToBegin()
953 CGF.CGM.getTBAAInfoForSubobject(BaseLV, BaseLV.getType())); in loadToBegin()
956 static Address castToBase(CodeGenFunction &CGF, QualType BaseTy, QualType ElTy, in castToBase() argument
964 !CGF.getContext().hasSameType(BaseTy, ElTy)) { in castToBase()
965 Tmp = CGF.CreateMemTemp(BaseTy); in castToBase()
967 CGF.Builder.CreateStore(Tmp.getPointer(), TopTmp); in castToBase()
976 Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(Addr, Ty); in castToBase()
978 CGF.Builder.CreateStore(Addr, Tmp); in castToBase()
1004 Address ReductionCodeGen::adjustPrivateAddress(CodeGenFunction &CGF, unsigned N, in adjustPrivateAddress() argument
1009 LValue OriginalBaseLValue = CGF.EmitLValue(DE); in adjustPrivateAddress()
1011 loadToBegin(CGF, OrigVD->getType(), SharedAddresses[N].first.getType(), in adjustPrivateAddress()
1013 llvm::Value *Adjustment = CGF.Builder.CreatePtrDiff( in adjustPrivateAddress()
1014 BaseLValue.getPointer(CGF), SharedAddresses[N].first.getPointer(CGF)); in adjustPrivateAddress()
1016 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in adjustPrivateAddress()
1018 SharedAddresses[N].first.getAddress(CGF).getType()); in adjustPrivateAddress()
1019 llvm::Value *Ptr = CGF.Builder.CreateGEP(PrivatePointer, Adjustment); in adjustPrivateAddress()
1020 return castToBase(CGF, OrigVD->getType(), in adjustPrivateAddress()
1022 OriginalBaseLValue.getAddress(CGF).getType(), in adjustPrivateAddress()
1036 LValue CGOpenMPRegionInfo::getThreadIDVariableLValue(CodeGenFunction &CGF) { in getThreadIDVariableLValue() argument
1037 return CGF.EmitLoadOfPointerLValue( in getThreadIDVariableLValue()
1038 CGF.GetAddrOfLocalVar(getThreadIDVariable()), in getThreadIDVariableLValue()
1042 void CGOpenMPRegionInfo::EmitBody(CodeGenFunction &CGF, const Stmt *S) { in EmitBody() argument
1043 if (!CGF.HaveInsertPoint()) in EmitBody()
1050 CGF.EHStack.pushTerminate(); in EmitBody()
1052 CGF.incrementProfileCounter(S); in EmitBody()
1053 CodeGen(CGF); in EmitBody()
1054 CGF.EHStack.popTerminate(); in EmitBody()
1058 CodeGenFunction &CGF) { in getThreadIDVariableLValue() argument
1059 return CGF.MakeAddrLValue(CGF.GetAddrOfLocalVar(getThreadIDVariable()), in getThreadIDVariableLValue()
1139 CodeGenFunction CGF(CGM); in emitCombinerOrInitializer() local
1142 CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, FnInfo, Args, In->getLocation(), in emitCombinerOrInitializer()
1144 CodeGenFunction::OMPPrivateScope Scope(CGF); in emitCombinerOrInitializer()
1145 Address AddrIn = CGF.GetAddrOfLocalVar(&OmpInParm); in emitCombinerOrInitializer()
1146 Scope.addPrivate(In, [&CGF, AddrIn, PtrTy]() { in emitCombinerOrInitializer()
1147 return CGF.EmitLoadOfPointerLValue(AddrIn, PtrTy->castAs<PointerType>()) in emitCombinerOrInitializer()
1148 .getAddress(CGF); in emitCombinerOrInitializer()
1150 Address AddrOut = CGF.GetAddrOfLocalVar(&OmpOutParm); in emitCombinerOrInitializer()
1151 Scope.addPrivate(Out, [&CGF, AddrOut, PtrTy]() { in emitCombinerOrInitializer()
1152 return CGF.EmitLoadOfPointerLValue(AddrOut, PtrTy->castAs<PointerType>()) in emitCombinerOrInitializer()
1153 .getAddress(CGF); in emitCombinerOrInitializer()
1157 !CGF.isTrivialInitializer(Out->getInit())) { in emitCombinerOrInitializer()
1158 CGF.EmitAnyExprToMem(Out->getInit(), CGF.GetAddrOfLocalVar(Out), in emitCombinerOrInitializer()
1163 CGF.EmitIgnoredExpr(CombinerInitializer); in emitCombinerOrInitializer()
1165 CGF.FinishFunction(); in emitCombinerOrInitializer()
1170 CodeGenFunction *CGF, const OMPDeclareReductionDecl *D) { in emitUserDefinedReduction() argument
1189 if (CGF) { in emitUserDefinedReduction()
1190 auto &Decls = FunctionUDRMap.FindAndConstruct(CGF->CurFn); in emitUserDefinedReduction()
1208 PushAndPopStackRAII(llvm::OpenMPIRBuilder *OMPBuilder, CodeGenFunction &CGF, in PushAndPopStackRAII()
1226 auto FiniCB = [&CGF](llvm::OpenMPIRBuilder::InsertPointTy IP) { in PushAndPopStackRAII()
1229 CGBuilderTy::InsertPointGuard IPG(CGF.Builder); in PushAndPopStackRAII()
1230 CGF.Builder.restoreIP(IP); in PushAndPopStackRAII()
1232 CGF.getOMPCancelDestination(OMPD_parallel); in PushAndPopStackRAII()
1233 CGF.EmitBranchThroughCleanup(Dest); in PushAndPopStackRAII()
1255 CodeGenFunction CGF(CGM, true); in emitParallelOrTeamsOutlinedFunction() local
1279 PushAndPopStackRAII PSR(&OMPBuilder, CGF, HasCancel, InnermostKind); in emitParallelOrTeamsOutlinedFunction()
1282 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo); in emitParallelOrTeamsOutlinedFunction()
1283 return CGF.GenerateOpenMPCapturedStmtFunction(*CS, D.getBeginLoc()); in emitParallelOrTeamsOutlinedFunction()
1307 auto &&UntiedCodeGen = [this, &D, TaskTVar](CodeGenFunction &CGF, in emitTaskOutlinedFunction()
1309 llvm::Value *ThreadID = getThreadID(CGF, D.getBeginLoc()); in emitTaskOutlinedFunction()
1310 llvm::Value *UpLoc = emitUpdateLocation(CGF, D.getBeginLoc()); in emitTaskOutlinedFunction()
1313 CGF.EmitLoadOfPointerLValue(CGF.GetAddrOfLocalVar(TaskTVar), in emitTaskOutlinedFunction()
1315 .getPointer(CGF)}; in emitTaskOutlinedFunction()
1316 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in emitTaskOutlinedFunction()
1339 CodeGenFunction CGF(CGM, true); in emitTaskOutlinedFunction() local
1342 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo); in emitTaskOutlinedFunction()
1343 llvm::Function *Res = CGF.GenerateCapturedStmtFunction(*CS); in emitTaskOutlinedFunction()
1394 void CGOpenMPRuntime::setLocThreadIdInsertPt(CodeGenFunction &CGF, in setLocThreadIdInsertPt() argument
1396 auto &Elem = OpenMPLocThreadIDMap.FindAndConstruct(CGF.CurFn); in setLocThreadIdInsertPt()
1399 llvm::Value *Undef = llvm::UndefValue::get(CGF.Int32Ty); in setLocThreadIdInsertPt()
1402 Undef, CGF.Int32Ty, "svcpt", CGF.Builder.GetInsertBlock()); in setLocThreadIdInsertPt()
1405 new llvm::BitCastInst(Undef, CGF.Int32Ty, "svcpt"); in setLocThreadIdInsertPt()
1406 Elem.second.ServiceInsertPt->insertAfter(CGF.AllocaInsertPt); in setLocThreadIdInsertPt()
1410 void CGOpenMPRuntime::clearLocThreadIdInsertPt(CodeGenFunction &CGF) { in clearLocThreadIdInsertPt() argument
1411 auto &Elem = OpenMPLocThreadIDMap.FindAndConstruct(CGF.CurFn); in clearLocThreadIdInsertPt()
1419 static StringRef getIdentStringFromSourceLocation(CodeGenFunction &CGF, in getIdentStringFromSourceLocation() argument
1424 PresumedLoc PLoc = CGF.getContext().getSourceManager().getPresumedLoc(Loc); in getIdentStringFromSourceLocation()
1426 if (const auto *FD = dyn_cast_or_null<FunctionDecl>(CGF.CurFuncDecl)) in getIdentStringFromSourceLocation()
1432 llvm::Value *CGOpenMPRuntime::emitUpdateLocation(CodeGenFunction &CGF, in emitUpdateLocation() argument
1441 if (const auto *FD = dyn_cast_or_null<FunctionDecl>(CGF.CurFuncDecl)) in emitUpdateLocation()
1443 PresumedLoc PLoc = CGF.getContext().getSourceManager().getPresumedLoc(Loc); in emitUpdateLocation()
1455 llvm::Value *CGOpenMPRuntime::getThreadID(CodeGenFunction &CGF, in getThreadID() argument
1457 assert(CGF.CurFn && "No function in current CodeGenFunction."); in getThreadID()
1462 OMPBuilder.updateToLocation(CGF.Builder.saveIP()); in getThreadID()
1464 getIdentStringFromSourceLocation(CGF, Loc, Buffer)); in getThreadID()
1472 auto I = OpenMPLocThreadIDMap.find(CGF.CurFn); in getThreadID()
1480 dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo)) { in getThreadID()
1483 LValue LVal = OMPRegionInfo->getThreadIDVariableLValue(CGF); in getThreadID()
1484 llvm::BasicBlock *TopBlock = CGF.AllocaInsertPt->getParent(); in getThreadID()
1485 if (!CGF.EHStack.requiresLandingPad() || !CGF.getLangOpts().Exceptions || in getThreadID()
1486 !CGF.getLangOpts().CXXExceptions || in getThreadID()
1487 CGF.Builder.GetInsertBlock() == TopBlock || in getThreadID()
1488 !isa<llvm::Instruction>(LVal.getPointer(CGF)) || in getThreadID()
1489 cast<llvm::Instruction>(LVal.getPointer(CGF))->getParent() == in getThreadID()
1491 cast<llvm::Instruction>(LVal.getPointer(CGF))->getParent() == in getThreadID()
1492 CGF.Builder.GetInsertBlock()) { in getThreadID()
1493 ThreadID = CGF.EmitLoadOfScalar(LVal, Loc); in getThreadID()
1496 if (CGF.Builder.GetInsertBlock() == TopBlock) { in getThreadID()
1497 auto &Elem = OpenMPLocThreadIDMap.FindAndConstruct(CGF.CurFn); in getThreadID()
1509 auto &Elem = OpenMPLocThreadIDMap.FindAndConstruct(CGF.CurFn); in getThreadID()
1511 setLocThreadIdInsertPt(CGF); in getThreadID()
1512 CGBuilderTy::InsertPointGuard IPG(CGF.Builder); in getThreadID()
1513 CGF.Builder.SetInsertPoint(Elem.second.ServiceInsertPt); in getThreadID()
1514 llvm::CallInst *Call = CGF.Builder.CreateCall( in getThreadID()
1517 emitUpdateLocation(CGF, Loc)); in getThreadID()
1518 Call->setCallingConv(CGF.getRuntimeCC()); in getThreadID()
1523 void CGOpenMPRuntime::functionFinished(CodeGenFunction &CGF) { in functionFinished() argument
1524 assert(CGF.CurFn && "No function in current CodeGenFunction."); in functionFinished()
1525 if (OpenMPLocThreadIDMap.count(CGF.CurFn)) { in functionFinished()
1526 clearLocThreadIdInsertPt(CGF); in functionFinished()
1527 OpenMPLocThreadIDMap.erase(CGF.CurFn); in functionFinished()
1529 if (FunctionUDRMap.count(CGF.CurFn) > 0) { in functionFinished()
1530 for(const auto *D : FunctionUDRMap[CGF.CurFn]) in functionFinished()
1532 FunctionUDRMap.erase(CGF.CurFn); in functionFinished()
1534 auto I = FunctionUDMMap.find(CGF.CurFn); in functionFinished()
1540 LastprivateConditionalToTypes.erase(CGF.CurFn); in functionFinished()
1541 FunctionToUntiedTaskStackMap.erase(CGF.CurFn); in functionFinished()
1725 Address CGOpenMPRuntime::getAddrOfThreadPrivate(CodeGenFunction &CGF, in getAddrOfThreadPrivate() argument
1734 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc), in getAddrOfThreadPrivate()
1735 CGF.Builder.CreatePointerCast(VDAddr.getPointer(), in getAddrOfThreadPrivate()
1739 return Address(CGF.EmitRuntimeCall( in getAddrOfThreadPrivate()
1747 CodeGenFunction &CGF, Address VDAddr, llvm::Value *Ctor, in emitThreadPrivateVarInit() argument
1751 llvm::Value *OMPLoc = emitUpdateLocation(CGF, Loc); in emitThreadPrivateVarInit()
1752 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in emitThreadPrivateVarInit()
1758 OMPLoc, CGF.Builder.CreatePointerCast(VDAddr.getPointer(), CGM.VoidPtrTy), in emitThreadPrivateVarInit()
1760 CGF.EmitRuntimeCall( in emitThreadPrivateVarInit()
1768 bool PerformInit, CodeGenFunction *CGF) { in emitThreadPrivateVarDefinition() argument
1866 if (!CGF) { in emitThreadPrivateVarDefinition()
1881 emitThreadPrivateVarInit(*CGF, VDAddr, Ctor, CopyCtor, Dtor, Loc); in emitThreadPrivateVarDefinition()
2001 Address CGOpenMPRuntime::getAddrOfArtificialThreadPrivate(CodeGenFunction &CGF, in getAddrOfArtificialThreadPrivate() argument
2005 llvm::Type *VarLVType = CGF.ConvertTypeForMem(VarType); in getAddrOfArtificialThreadPrivate()
2015 emitUpdateLocation(CGF, SourceLocation()), in getAddrOfArtificialThreadPrivate()
2016 getThreadID(CGF, SourceLocation()), in getAddrOfArtificialThreadPrivate()
2017 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(GAddr, CGM.VoidPtrTy), in getAddrOfArtificialThreadPrivate()
2018 CGF.Builder.CreateIntCast(CGF.getTypeSize(VarType), CGM.SizeTy, in getAddrOfArtificialThreadPrivate()
2023 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in getAddrOfArtificialThreadPrivate()
2024 CGF.EmitRuntimeCall( in getAddrOfArtificialThreadPrivate()
2032 void CGOpenMPRuntime::emitIfClause(CodeGenFunction &CGF, const Expr *Cond, in emitIfClause() argument
2035 CodeGenFunction::LexicalScope ConditionScope(CGF, Cond->getSourceRange()); in emitIfClause()
2040 if (CGF.ConstantFoldsToSimpleInteger(Cond, CondConstant)) { in emitIfClause()
2042 ThenGen(CGF); in emitIfClause()
2044 ElseGen(CGF); in emitIfClause()
2050 llvm::BasicBlock *ThenBlock = CGF.createBasicBlock("omp_if.then"); in emitIfClause()
2051 llvm::BasicBlock *ElseBlock = CGF.createBasicBlock("omp_if.else"); in emitIfClause()
2052 llvm::BasicBlock *ContBlock = CGF.createBasicBlock("omp_if.end"); in emitIfClause()
2053 CGF.EmitBranchOnBoolExpr(Cond, ThenBlock, ElseBlock, /*TrueCount=*/0); in emitIfClause()
2056 CGF.EmitBlock(ThenBlock); in emitIfClause()
2057 ThenGen(CGF); in emitIfClause()
2058 CGF.EmitBranch(ContBlock); in emitIfClause()
2061 (void)ApplyDebugLocation::CreateEmpty(CGF); in emitIfClause()
2062 CGF.EmitBlock(ElseBlock); in emitIfClause()
2063 ElseGen(CGF); in emitIfClause()
2065 (void)ApplyDebugLocation::CreateEmpty(CGF); in emitIfClause()
2066 CGF.EmitBranch(ContBlock); in emitIfClause()
2068 CGF.EmitBlock(ContBlock, /*IsFinished=*/true); in emitIfClause()
2071 void CGOpenMPRuntime::emitParallelCall(CodeGenFunction &CGF, SourceLocation Loc, in emitParallelCall() argument
2075 if (!CGF.HaveInsertPoint()) in emitParallelCall()
2077 llvm::Value *RTLoc = emitUpdateLocation(CGF, Loc); in emitParallelCall()
2080 this](CodeGenFunction &CGF, PrePostActionTy &) { in emitParallelCall() argument
2082 CGOpenMPRuntime &RT = CGF.CGM.getOpenMPRuntime(); in emitParallelCall()
2085 CGF.Builder.getInt32(CapturedVars.size()), // Number of captured vars in emitParallelCall()
2086 CGF.Builder.CreateBitCast(OutlinedFn, RT.getKmpc_MicroPointerTy())}; in emitParallelCall()
2093 CGF.EmitRuntimeCall(RTLFn, RealArgs); in emitParallelCall()
2096 this](CodeGenFunction &CGF, PrePostActionTy &) { in emitParallelCall() argument
2097 CGOpenMPRuntime &RT = CGF.CGM.getOpenMPRuntime(); in emitParallelCall()
2098 llvm::Value *ThreadID = RT.getThreadID(CGF, Loc); in emitParallelCall()
2102 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in emitParallelCall()
2107 Address ThreadIDAddr = RT.emitThreadIDAddress(CGF, Loc); in emitParallelCall()
2109 CGF.CreateDefaultAlignTempAlloca(CGF.Int32Ty, in emitParallelCall()
2111 CGF.InitTempAlloca(ZeroAddrBound, CGF.Builder.getInt32(/*C*/ 0)); in emitParallelCall()
2125 RT.emitOutlinedFunctionCall(CGF, Loc, OutlinedFn, OutlinedFnArgs); in emitParallelCall()
2128 llvm::Value *EndArgs[] = {RT.emitUpdateLocation(CGF, Loc), ThreadID}; in emitParallelCall()
2129 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in emitParallelCall()
2134 emitIfClause(CGF, IfCond, ThenGen, ElseGen); in emitParallelCall()
2137 ThenRCG(CGF); in emitParallelCall()
2147 Address CGOpenMPRuntime::emitThreadIDAddress(CodeGenFunction &CGF, in emitThreadIDAddress() argument
2150 dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo)) in emitThreadIDAddress()
2152 return OMPRegionInfo->getThreadIDVariableLValue(CGF).getAddress(CGF); in emitThreadIDAddress()
2154 llvm::Value *ThreadID = getThreadID(CGF, Loc); in emitThreadIDAddress()
2156 CGF.getContext().getIntTypeForBitwidth(/*DestWidth*/ 32, /*Signed*/ true); in emitThreadIDAddress()
2157 Address ThreadIDTemp = CGF.CreateMemTemp(Int32Ty, /*Name*/ ".threadid_temp."); in emitThreadIDAddress()
2158 CGF.EmitStoreOfScalar(ThreadID, in emitThreadIDAddress()
2159 CGF.MakeAddrLValue(ThreadIDTemp, Int32Ty)); in emitThreadIDAddress()
2207 void Enter(CodeGenFunction &CGF) override { in Enter() argument
2208 llvm::Value *EnterRes = CGF.EmitRuntimeCall(EnterCallee, EnterArgs); in Enter()
2210 llvm::Value *CallBool = CGF.Builder.CreateIsNotNull(EnterRes); in Enter()
2211 auto *ThenBlock = CGF.createBasicBlock("omp_if.then"); in Enter()
2212 ContBlock = CGF.createBasicBlock("omp_if.end"); in Enter()
2214 CGF.Builder.CreateCondBr(CallBool, ThenBlock, ContBlock); in Enter()
2215 CGF.EmitBlock(ThenBlock); in Enter()
2218 void Done(CodeGenFunction &CGF) { in Done() argument
2220 CGF.EmitBranch(ContBlock); in Done()
2221 CGF.EmitBlock(ContBlock, true); in Done()
2223 void Exit(CodeGenFunction &CGF) override { in Exit() argument
2224 CGF.EmitRuntimeCall(ExitCallee, ExitArgs); in Exit()
2229 void CGOpenMPRuntime::emitCriticalRegion(CodeGenFunction &CGF, in emitCriticalRegion() argument
2237 if (!CGF.HaveInsertPoint()) in emitCriticalRegion()
2239 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc), in emitCriticalRegion()
2244 EnterArgs.push_back(CGF.Builder.CreateIntCast( in emitCriticalRegion()
2245 CGF.EmitScalarExpr(Hint), CGM.Int32Ty, /*isSigned=*/false)); in emitCriticalRegion()
2256 emitInlinedDirective(CGF, OMPD_critical, CriticalOpGen); in emitCriticalRegion()
2259 void CGOpenMPRuntime::emitMasterRegion(CodeGenFunction &CGF, in emitMasterRegion() argument
2262 if (!CGF.HaveInsertPoint()) in emitMasterRegion()
2269 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)}; in emitMasterRegion()
2278 emitInlinedDirective(CGF, OMPD_master, MasterOpGen); in emitMasterRegion()
2279 Action.Done(CGF); in emitMasterRegion()
2282 void CGOpenMPRuntime::emitMaskedRegion(CodeGenFunction &CGF, in emitMaskedRegion() argument
2285 if (!CGF.HaveInsertPoint()) in emitMaskedRegion()
2293 ? CGF.EmitScalarExpr(Filter, CGF.Int32Ty) in emitMaskedRegion()
2295 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc), in emitMaskedRegion()
2297 llvm::Value *ArgsEnd[] = {emitUpdateLocation(CGF, Loc), in emitMaskedRegion()
2298 getThreadID(CGF, Loc)}; in emitMaskedRegion()
2307 emitInlinedDirective(CGF, OMPD_masked, MaskedOpGen); in emitMaskedRegion()
2308 Action.Done(CGF); in emitMaskedRegion()
2311 void CGOpenMPRuntime::emitTaskyieldCall(CodeGenFunction &CGF, in emitTaskyieldCall() argument
2313 if (!CGF.HaveInsertPoint()) in emitTaskyieldCall()
2315 if (CGF.CGM.getLangOpts().OpenMPIRBuilder) { in emitTaskyieldCall()
2316 OMPBuilder.createTaskyield(CGF.Builder); in emitTaskyieldCall()
2320 emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc), in emitTaskyieldCall()
2322 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in emitTaskyieldCall()
2327 if (auto *Region = dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo)) in emitTaskyieldCall()
2328 Region->emitUntiedSwitch(CGF); in emitTaskyieldCall()
2331 void CGOpenMPRuntime::emitTaskgroupRegion(CodeGenFunction &CGF, in emitTaskgroupRegion() argument
2334 if (!CGF.HaveInsertPoint()) in emitTaskgroupRegion()
2340 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)}; in emitTaskgroupRegion()
2348 emitInlinedDirective(CGF, OMPD_taskgroup, TaskgroupOpGen); in emitTaskgroupRegion()
2353 static Address emitAddrOfVarFromArray(CodeGenFunction &CGF, Address Array, in emitAddrOfVarFromArray() argument
2356 Address PtrAddr = CGF.Builder.CreateConstArrayGEP(Array, Index); in emitAddrOfVarFromArray()
2357 llvm::Value *Ptr = CGF.Builder.CreateLoad(PtrAddr); in emitAddrOfVarFromArray()
2359 Address Addr = Address(Ptr, CGF.getContext().getDeclAlign(Var)); in emitAddrOfVarFromArray()
2360 Addr = CGF.Builder.CreateElementBitCast( in emitAddrOfVarFromArray()
2361 Addr, CGF.ConvertTypeForMem(Var->getType())); in emitAddrOfVarFromArray()
2388 CodeGenFunction CGF(CGM); in emitCopyprivateCopyFunction() local
2389 CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, CGFI, Args, Loc, Loc); in emitCopyprivateCopyFunction()
2392 Address LHS(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitCopyprivateCopyFunction()
2393 CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(&LHSArg)), in emitCopyprivateCopyFunction()
2394 ArgsType), CGF.getPointerAlign()); in emitCopyprivateCopyFunction()
2395 Address RHS(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitCopyprivateCopyFunction()
2396 CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(&RHSArg)), in emitCopyprivateCopyFunction()
2397 ArgsType), CGF.getPointerAlign()); in emitCopyprivateCopyFunction()
2405 Address DestAddr = emitAddrOfVarFromArray(CGF, LHS, I, DestVar); in emitCopyprivateCopyFunction()
2409 Address SrcAddr = emitAddrOfVarFromArray(CGF, RHS, I, SrcVar); in emitCopyprivateCopyFunction()
2413 CGF.EmitOMPCopy(Type, DestAddr, SrcAddr, DestVar, SrcVar, AssignmentOps[I]); in emitCopyprivateCopyFunction()
2415 CGF.FinishFunction(); in emitCopyprivateCopyFunction()
2419 void CGOpenMPRuntime::emitSingleRegion(CodeGenFunction &CGF, in emitSingleRegion() argument
2426 if (!CGF.HaveInsertPoint()) in emitSingleRegion()
2446 DidIt = CGF.CreateMemTemp(KmpInt32Ty, ".omp.copyprivate.did_it"); in emitSingleRegion()
2447 CGF.Builder.CreateStore(CGF.Builder.getInt32(0), DidIt); in emitSingleRegion()
2450 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)}; in emitSingleRegion()
2459 emitInlinedDirective(CGF, OMPD_single, SingleOpGen); in emitSingleRegion()
2462 CGF.Builder.CreateStore(CGF.Builder.getInt32(1), DidIt); in emitSingleRegion()
2464 Action.Done(CGF); in emitSingleRegion()
2474 CGF.CreateMemTemp(CopyprivateArrayTy, ".omp.copyprivate.cpr_list"); in emitSingleRegion()
2476 Address Elem = CGF.Builder.CreateConstArrayGEP(CopyprivateList, I); in emitSingleRegion()
2477 CGF.Builder.CreateStore( in emitSingleRegion()
2478 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitSingleRegion()
2479 CGF.EmitLValue(CopyprivateVars[I]).getPointer(CGF), in emitSingleRegion()
2480 CGF.VoidPtrTy), in emitSingleRegion()
2486 CGM, CGF.ConvertTypeForMem(CopyprivateArrayTy)->getPointerTo(), in emitSingleRegion()
2488 llvm::Value *BufSize = CGF.getTypeSize(CopyprivateArrayTy); in emitSingleRegion()
2490 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(CopyprivateList, in emitSingleRegion()
2491 CGF.VoidPtrTy); in emitSingleRegion()
2492 llvm::Value *DidItVal = CGF.Builder.CreateLoad(DidIt); in emitSingleRegion()
2494 emitUpdateLocation(CGF, Loc), // ident_t *<loc> in emitSingleRegion()
2495 getThreadID(CGF, Loc), // i32 <gtid> in emitSingleRegion()
2501 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in emitSingleRegion()
2507 void CGOpenMPRuntime::emitOrderedRegion(CodeGenFunction &CGF, in emitOrderedRegion() argument
2510 if (!CGF.HaveInsertPoint()) in emitOrderedRegion()
2517 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)}; in emitOrderedRegion()
2525 emitInlinedDirective(CGF, OMPD_ordered, OrderedOpGen); in emitOrderedRegion()
2528 emitInlinedDirective(CGF, OMPD_ordered, OrderedOpGen); in emitOrderedRegion()
2547 CodeGenFunction &CGF, const OMPLoopDirective &S, in getDefaultScheduleAndChunk() argument
2558 CGF.getContext(), ChunkSize, in getDefaultScheduleAndChunk()
2559 CGF.getContext().getIntTypeForBitwidth(32, /*Signed=*/0), in getDefaultScheduleAndChunk()
2564 void CGOpenMPRuntime::emitBarrierCall(CodeGenFunction &CGF, SourceLocation Loc, in emitBarrierCall() argument
2569 dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo); in emitBarrierCall()
2570 if (CGF.CGM.getLangOpts().OpenMPIRBuilder) { in emitBarrierCall()
2571 CGF.Builder.restoreIP(OMPBuilder.createBarrier( in emitBarrierCall()
2572 CGF.Builder, Kind, ForceSimpleCall, EmitChecks)); in emitBarrierCall()
2576 if (!CGF.HaveInsertPoint()) in emitBarrierCall()
2583 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc, Flags), in emitBarrierCall()
2584 getThreadID(CGF, Loc)}; in emitBarrierCall()
2587 llvm::Value *Result = CGF.EmitRuntimeCall( in emitBarrierCall()
2595 llvm::BasicBlock *ExitBB = CGF.createBasicBlock(".cancel.exit"); in emitBarrierCall()
2596 llvm::BasicBlock *ContBB = CGF.createBasicBlock(".cancel.continue"); in emitBarrierCall()
2597 llvm::Value *Cmp = CGF.Builder.CreateIsNotNull(Result); in emitBarrierCall()
2598 CGF.Builder.CreateCondBr(Cmp, ExitBB, ContBB); in emitBarrierCall()
2599 CGF.EmitBlock(ExitBB); in emitBarrierCall()
2602 CGF.getOMPCancelDestination(OMPRegionInfo->getDirectiveKind()); in emitBarrierCall()
2603 CGF.EmitBranchThroughCleanup(CancelDestination); in emitBarrierCall()
2604 CGF.EmitBlock(ContBB, /*IsFinished=*/true); in emitBarrierCall()
2609 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in emitBarrierCall()
2728 CodeGenFunction &CGF, SourceLocation Loc, in emitForDispatchInit() argument
2731 if (!CGF.HaveInsertPoint()) in emitForDispatchInit()
2746 : CGF.Builder.getIntN(IVSize, 1); in emitForDispatchInit()
2748 emitUpdateLocation(CGF, Loc), in emitForDispatchInit()
2749 getThreadID(CGF, Loc), in emitForDispatchInit()
2750 CGF.Builder.getInt32(addMonoNonMonoModifier( in emitForDispatchInit()
2754 CGF.Builder.getIntN(IVSize, 1), // Stride in emitForDispatchInit()
2757 CGF.EmitRuntimeCall(createDispatchInitFunction(IVSize, IVSigned), Args); in emitForDispatchInit()
2761 CodeGenFunction &CGF, llvm::Value *UpdateLocation, llvm::Value *ThreadId, in emitForStaticInitCall() argument
2765 if (!CGF.HaveInsertPoint()) in emitForStaticInitCall()
2786 Chunk = CGF.Builder.getIntN(Values.IVSize, 1); in emitForStaticInitCall()
2797 CGF.Builder.getInt32(addMonoNonMonoModifier(CGF.CGM, Schedule, M1, in emitForStaticInitCall()
2803 CGF.Builder.getIntN(Values.IVSize, 1), // Incr in emitForStaticInitCall()
2806 CGF.EmitRuntimeCall(ForStaticInitFunction, Args); in emitForStaticInitCall()
2809 void CGOpenMPRuntime::emitForStaticInit(CodeGenFunction &CGF, in emitForStaticInit() argument
2818 llvm::Value *UpdatedLocation = emitUpdateLocation(CGF, Loc, in emitForStaticInit()
2822 llvm::Value *ThreadId = getThreadID(CGF, Loc); in emitForStaticInit()
2825 auto DL = ApplyDebugLocation::CreateDefaultArtificial(CGF, Loc); in emitForStaticInit()
2826 emitForStaticInitCall(CGF, UpdatedLocation, ThreadId, StaticInitFunction, in emitForStaticInit()
2831 CodeGenFunction &CGF, SourceLocation Loc, in emitDistributeStaticInit() argument
2837 emitUpdateLocation(CGF, Loc, OMP_IDENT_WORK_DISTRIBUTE); in emitDistributeStaticInit()
2838 llvm::Value *ThreadId = getThreadID(CGF, Loc); in emitDistributeStaticInit()
2841 emitForStaticInitCall(CGF, UpdatedLocation, ThreadId, StaticInitFunction, in emitDistributeStaticInit()
2846 void CGOpenMPRuntime::emitForStaticFinish(CodeGenFunction &CGF, in emitForStaticFinish() argument
2849 if (!CGF.HaveInsertPoint()) in emitForStaticFinish()
2853 emitUpdateLocation(CGF, Loc, in emitForStaticFinish()
2859 getThreadID(CGF, Loc)}; in emitForStaticFinish()
2860 auto DL = ApplyDebugLocation::CreateDefaultArtificial(CGF, Loc); in emitForStaticFinish()
2861 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in emitForStaticFinish()
2866 void CGOpenMPRuntime::emitForOrderedIterationEnd(CodeGenFunction &CGF, in emitForOrderedIterationEnd() argument
2870 if (!CGF.HaveInsertPoint()) in emitForOrderedIterationEnd()
2873 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)}; in emitForOrderedIterationEnd()
2874 CGF.EmitRuntimeCall(createDispatchFiniFunction(IVSize, IVSigned), Args); in emitForOrderedIterationEnd()
2877 llvm::Value *CGOpenMPRuntime::emitForNext(CodeGenFunction &CGF, in emitForNext() argument
2887 emitUpdateLocation(CGF, Loc), in emitForNext()
2888 getThreadID(CGF, Loc), in emitForNext()
2895 CGF.EmitRuntimeCall(createDispatchNextFunction(IVSize, IVSigned), Args); in emitForNext()
2896 return CGF.EmitScalarConversion( in emitForNext()
2897 Call, CGF.getContext().getIntTypeForBitwidth(32, /*Signed=*/1), in emitForNext()
2898 CGF.getContext().BoolTy, Loc); in emitForNext()
2901 void CGOpenMPRuntime::emitNumThreadsClause(CodeGenFunction &CGF, in emitNumThreadsClause() argument
2904 if (!CGF.HaveInsertPoint()) in emitNumThreadsClause()
2908 emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc), in emitNumThreadsClause()
2909 CGF.Builder.CreateIntCast(NumThreads, CGF.Int32Ty, /*isSigned*/ true)}; in emitNumThreadsClause()
2910 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in emitNumThreadsClause()
2915 void CGOpenMPRuntime::emitProcBindClause(CodeGenFunction &CGF, in emitProcBindClause() argument
2918 if (!CGF.HaveInsertPoint()) in emitProcBindClause()
2923 emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc), in emitProcBindClause()
2925 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in emitProcBindClause()
2930 void CGOpenMPRuntime::emitFlush(CodeGenFunction &CGF, ArrayRef<const Expr *>, in emitFlush() argument
2932 if (CGF.CGM.getLangOpts().OpenMPIRBuilder) { in emitFlush()
2933 OMPBuilder.createFlush(CGF.Builder); in emitFlush()
2935 if (!CGF.HaveInsertPoint()) in emitFlush()
2938 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in emitFlush()
2940 emitUpdateLocation(CGF, Loc)); in emitFlush()
3584 CodeGenFunction CGF(CGM); in emitProxyTaskFunction() local
3585 CGF.StartFunction(GlobalDecl(), KmpInt32Ty, TaskEntry, TaskEntryFnInfo, Args, in emitProxyTaskFunction()
3593 llvm::Value *GtidParam = CGF.EmitLoadOfScalar( in emitProxyTaskFunction()
3594 CGF.GetAddrOfLocalVar(&GtidArg), /*Volatile=*/false, KmpInt32Ty, Loc); in emitProxyTaskFunction()
3595 LValue TDBase = CGF.EmitLoadOfPointerLValue( in emitProxyTaskFunction()
3596 CGF.GetAddrOfLocalVar(&TaskTypeArg), in emitProxyTaskFunction()
3601 CGF.EmitLValueForField(TDBase, *KmpTaskTWithPrivatesQTyRD->field_begin()); in emitProxyTaskFunction()
3604 LValue PartIdLVal = CGF.EmitLValueForField(Base, *PartIdFI); in emitProxyTaskFunction()
3605 llvm::Value *PartidParam = PartIdLVal.getPointer(CGF); in emitProxyTaskFunction()
3608 LValue SharedsLVal = CGF.EmitLValueForField(Base, *SharedsFI); in emitProxyTaskFunction()
3609 llvm::Value *SharedsParam = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitProxyTaskFunction()
3610 CGF.EmitLoadOfScalar(SharedsLVal, Loc), in emitProxyTaskFunction()
3611 CGF.ConvertTypeForMem(SharedsPtrTy)); in emitProxyTaskFunction()
3616 LValue PrivatesLVal = CGF.EmitLValueForField(TDBase, *PrivatesFI); in emitProxyTaskFunction()
3617 PrivatesParam = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitProxyTaskFunction()
3618 PrivatesLVal.getPointer(CGF), CGF.VoidPtrTy); in emitProxyTaskFunction()
3620 PrivatesParam = llvm::ConstantPointerNull::get(CGF.VoidPtrTy); in emitProxyTaskFunction()
3625 CGF.Builder in emitProxyTaskFunction()
3627 TDBase.getAddress(CGF), CGF.VoidPtrTy) in emitProxyTaskFunction()
3633 LValue LBLVal = CGF.EmitLValueForField(Base, *LBFI); in emitProxyTaskFunction()
3634 llvm::Value *LBParam = CGF.EmitLoadOfScalar(LBLVal, Loc); in emitProxyTaskFunction()
3636 LValue UBLVal = CGF.EmitLValueForField(Base, *UBFI); in emitProxyTaskFunction()
3637 llvm::Value *UBParam = CGF.EmitLoadOfScalar(UBLVal, Loc); in emitProxyTaskFunction()
3639 LValue StLVal = CGF.EmitLValueForField(Base, *StFI); in emitProxyTaskFunction()
3640 llvm::Value *StParam = CGF.EmitLoadOfScalar(StLVal, Loc); in emitProxyTaskFunction()
3642 LValue LILVal = CGF.EmitLValueForField(Base, *LIFI); in emitProxyTaskFunction()
3643 llvm::Value *LIParam = CGF.EmitLoadOfScalar(LILVal, Loc); in emitProxyTaskFunction()
3645 LValue RLVal = CGF.EmitLValueForField(Base, *RFI); in emitProxyTaskFunction()
3646 llvm::Value *RParam = CGF.EmitLoadOfScalar(RLVal, Loc); in emitProxyTaskFunction()
3655 CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF, Loc, TaskFunction, in emitProxyTaskFunction()
3657 CGF.EmitStoreThroughLValue(RValue::get(CGF.Builder.getInt32(/*C=*/0)), in emitProxyTaskFunction()
3658 CGF.MakeAddrLValue(CGF.ReturnValue, KmpInt32Ty)); in emitProxyTaskFunction()
3659 CGF.FinishFunction(); in emitProxyTaskFunction()
3689 CodeGenFunction CGF(CGM); in emitDestructorsFunction() local
3690 CGF.StartFunction(GlobalDecl(), KmpInt32Ty, DestructorFn, DestructorFnInfo, in emitDestructorsFunction()
3693 LValue Base = CGF.EmitLoadOfPointerLValue( in emitDestructorsFunction()
3694 CGF.GetAddrOfLocalVar(&TaskTypeArg), in emitDestructorsFunction()
3699 Base = CGF.EmitLValueForField(Base, *FI); in emitDestructorsFunction()
3704 LValue FieldLValue = CGF.EmitLValueForField(Base, Field); in emitDestructorsFunction()
3705 CGF.pushDestroy(DtorKind, FieldLValue.getAddress(CGF), Field->getType()); in emitDestructorsFunction()
3708 CGF.FinishFunction(); in emitDestructorsFunction()
3797 CodeGenFunction CGF(CGM); in emitTaskPrivateMappingFunction() local
3798 CGF.StartFunction(GlobalDecl(), C.VoidTy, TaskPrivatesMap, in emitTaskPrivateMappingFunction()
3802 LValue Base = CGF.EmitLoadOfPointerLValue( in emitTaskPrivateMappingFunction()
3803 CGF.GetAddrOfLocalVar(&TaskPrivatesArg), in emitTaskPrivateMappingFunction()
3808 LValue FieldLVal = CGF.EmitLValueForField(Base, Field); in emitTaskPrivateMappingFunction()
3811 CGF.MakeAddrLValue(CGF.GetAddrOfLocalVar(VD), VD->getType()); in emitTaskPrivateMappingFunction()
3812 LValue RefLoadLVal = CGF.EmitLoadOfPointerLValue( in emitTaskPrivateMappingFunction()
3813 RefLVal.getAddress(CGF), RefLVal.getType()->castAs<PointerType>()); in emitTaskPrivateMappingFunction()
3814 CGF.EmitStoreOfScalar(FieldLVal.getPointer(CGF), RefLoadLVal); in emitTaskPrivateMappingFunction()
3817 CGF.FinishFunction(); in emitTaskPrivateMappingFunction()
3822 static void emitPrivatesInit(CodeGenFunction &CGF, in emitPrivatesInit() argument
3829 ASTContext &C = CGF.getContext(); in emitPrivatesInit()
3831 LValue PrivatesBase = CGF.EmitLValueForField(TDBase, *FI); in emitPrivatesInit()
3846 SrcBase = CGF.MakeAddrLValue( in emitPrivatesInit()
3847 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitPrivatesInit()
3848 KmpTaskSharedsPtr, CGF.ConvertTypeForMem(SharedsPtrTy)), in emitPrivatesInit()
3861 !CGF.isTrivialInitializer(Init)))) { in emitPrivatesInit()
3862 LValue PrivateLValue = CGF.EmitLValueForField(PrivatesBase, *FI); in emitPrivatesInit()
3880 CGF.MakeAddrLValue(CGF.GetAddrOfLocalVar(OriginalVD), Type); in emitPrivatesInit()
3882 SharedRefLValue = CGF.EmitLValueForField(SrcBase, SharedField); in emitPrivatesInit()
3883 SharedRefLValue = CGF.MakeAddrLValue( in emitPrivatesInit()
3884 Address(SharedRefLValue.getPointer(CGF), in emitPrivatesInit()
3888 } else if (CGF.LambdaCaptureFields.count( in emitPrivatesInit()
3890 dyn_cast_or_null<BlockDecl>(CGF.CurCodeDecl)) { in emitPrivatesInit()
3891 SharedRefLValue = CGF.EmitLValue(Pair.second.OriginalRef); in emitPrivatesInit()
3895 CGF, [](CodeGenFunction &, PrePostActionTy &) {}, OMPD_unknown, in emitPrivatesInit()
3897 SharedRefLValue = CGF.EmitLValue(Pair.second.OriginalRef); in emitPrivatesInit()
3901 if (!isa<CXXConstructExpr>(Init) || CGF.isTrivialInitializer(Init)) { in emitPrivatesInit()
3903 CGF.EmitAggregateAssign(PrivateLValue, SharedRefLValue, Type); in emitPrivatesInit()
3907 CGF.EmitOMPAggregateAssign( in emitPrivatesInit()
3908 PrivateLValue.getAddress(CGF), SharedRefLValue.getAddress(CGF), in emitPrivatesInit()
3910 [&CGF, Elem, Init, &CapturesInfo](Address DestElement, in emitPrivatesInit()
3913 CodeGenFunction::OMPPrivateScope InitScope(CGF); in emitPrivatesInit()
3919 CGF, &CapturesInfo); in emitPrivatesInit()
3920 CGF.EmitAnyExprToMem(Init, DestElement, in emitPrivatesInit()
3926 CodeGenFunction::OMPPrivateScope InitScope(CGF); in emitPrivatesInit()
3927 InitScope.addPrivate(Elem, [SharedRefLValue, &CGF]() -> Address { in emitPrivatesInit()
3928 return SharedRefLValue.getAddress(CGF); in emitPrivatesInit()
3931 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CapturesInfo); in emitPrivatesInit()
3932 CGF.EmitExprAsInit(Init, VD, PrivateLValue, in emitPrivatesInit()
3936 CGF.EmitExprAsInit(Init, VD, PrivateLValue, /*capturedByInit=*/false); in emitPrivatesInit()
3944 static bool checkInitIsRequired(CodeGenFunction &CGF, in checkInitIsRequired() argument
3953 !CGF.isTrivialInitializer(Init)); in checkInitIsRequired()
4000 CodeGenFunction CGF(CGM); in emitTaskDupFunction() local
4001 CGF.StartFunction(GlobalDecl(), C.VoidTy, TaskDup, TaskDupFnInfo, Args, Loc, in emitTaskDupFunction()
4004 LValue TDBase = CGF.EmitLoadOfPointerLValue( in emitTaskDupFunction()
4005 CGF.GetAddrOfLocalVar(&DstArg), in emitTaskDupFunction()
4010 LValue Base = CGF.EmitLValueForField( in emitTaskDupFunction()
4012 LValue LILVal = CGF.EmitLValueForField(Base, *LIFI); in emitTaskDupFunction()
4013 llvm::Value *Lastpriv = CGF.EmitLoadOfScalar( in emitTaskDupFunction()
4014 CGF.GetAddrOfLocalVar(&LastprivArg), /*Volatile=*/false, C.IntTy, Loc); in emitTaskDupFunction()
4015 CGF.EmitStoreOfScalar(Lastpriv, LILVal); in emitTaskDupFunction()
4022 LValue TDBase = CGF.EmitLoadOfPointerLValue( in emitTaskDupFunction()
4023 CGF.GetAddrOfLocalVar(&SrcArg), in emitTaskDupFunction()
4025 LValue Base = CGF.EmitLValueForField( in emitTaskDupFunction()
4028 CGF.EmitLoadOfScalar(CGF.EmitLValueForField( in emitTaskDupFunction()
4034 emitPrivatesInit(CGF, D, KmpTaskSharedsPtr, TDBase, KmpTaskTWithPrivatesQTyRD, in emitTaskDupFunction()
4036 CGF.FinishFunction(); in emitTaskDupFunction()
4059 CodeGenFunction &CGF; member in __anonad2d34461b11::OMPIteratorGeneratorScope
4067 OMPIteratorGeneratorScope(CodeGenFunction &CGF, const OMPIteratorExpr *E) in OMPIteratorGeneratorScope() argument
4068 : CodeGenFunction::OMPPrivateScope(CGF), CGF(CGF), E(E) { in OMPIteratorGeneratorScope()
4073 Uppers.push_back(CGF.EmitScalarExpr(E->getHelper(I).Upper)); in OMPIteratorGeneratorScope()
4075 addPrivate(VD, [&CGF, VD]() { in OMPIteratorGeneratorScope()
4076 return CGF.CreateMemTemp(VD->getType(), VD->getName()); in OMPIteratorGeneratorScope()
4079 addPrivate(HelperData.CounterVD, [&CGF, &HelperData]() { in OMPIteratorGeneratorScope()
4080 return CGF.CreateMemTemp(HelperData.CounterVD->getType(), in OMPIteratorGeneratorScope()
4089 CGF.MakeAddrLValue(CGF.GetAddrOfLocalVar(HelperData.CounterVD), in OMPIteratorGeneratorScope()
4092 CGF.EmitStoreOfScalar( in OMPIteratorGeneratorScope()
4093 llvm::ConstantInt::get(CLVal.getAddress(CGF).getElementType(), 0), in OMPIteratorGeneratorScope()
4096 ContDests.emplace_back(CGF.getJumpDestInCurrentScope("iter.cont")); in OMPIteratorGeneratorScope()
4098 ExitDests.emplace_back(CGF.getJumpDestInCurrentScope("iter.exit")); in OMPIteratorGeneratorScope()
4103 CGF.EmitBlock(ContDest.getBlock()); in OMPIteratorGeneratorScope()
4105 CGF.EmitLoadOfScalar(CLVal, HelperData.CounterVD->getLocation()); in OMPIteratorGeneratorScope()
4108 ? CGF.Builder.CreateICmpSLT(CVal, N) in OMPIteratorGeneratorScope()
4109 : CGF.Builder.CreateICmpULT(CVal, N); in OMPIteratorGeneratorScope()
4110 llvm::BasicBlock *BodyBB = CGF.createBasicBlock("iter.body"); in OMPIteratorGeneratorScope()
4111 CGF.Builder.CreateCondBr(Cmp, BodyBB, ExitDest.getBlock()); in OMPIteratorGeneratorScope()
4113 CGF.EmitBlock(BodyBB); in OMPIteratorGeneratorScope()
4115 CGF.EmitIgnoredExpr(HelperData.Update); in OMPIteratorGeneratorScope()
4124 CGF.EmitIgnoredExpr(HelperData.CounterUpdate); in ~OMPIteratorGeneratorScope()
4126 CGF.EmitBranchThroughCleanup(ContDests[I - 1]); in ~OMPIteratorGeneratorScope()
4128 CGF.EmitBlock(ExitDests[I - 1].getBlock(), /*IsFinished=*/I == 1); in ~OMPIteratorGeneratorScope()
4135 getPointerAndSize(CodeGenFunction &CGF, const Expr *E) { in getPointerAndSize() argument
4140 Addr = CGF.EmitScalarExpr(Base); in getPointerAndSize()
4142 Addr = CGF.EmitLValue(E).getPointer(CGF); in getPointerAndSize()
4147 SizeVal = CGF.getTypeSize(OASE->getBase()->getType()->getPointeeType()); in getPointerAndSize()
4149 llvm::Value *Sz = CGF.EmitScalarExpr(SE); in getPointerAndSize()
4150 Sz = CGF.EmitScalarConversion( in getPointerAndSize()
4151 Sz, SE->getType(), CGF.getContext().getSizeType(), SE->getExprLoc()); in getPointerAndSize()
4152 SizeVal = CGF.Builder.CreateNUWMul(SizeVal, Sz); in getPointerAndSize()
4157 CGF.EmitOMPArraySectionExpr(ASE, /*IsLowerBound=*/false); in getPointerAndSize()
4159 CGF.Builder.CreateConstGEP1_32(UpAddrLVal.getPointer(CGF), /*Idx0=*/1); in getPointerAndSize()
4160 llvm::Value *LowIntPtr = CGF.Builder.CreatePtrToInt(Addr, CGF.SizeTy); in getPointerAndSize()
4161 llvm::Value *UpIntPtr = CGF.Builder.CreatePtrToInt(UpAddr, CGF.SizeTy); in getPointerAndSize()
4162 SizeVal = CGF.Builder.CreateNUWSub(UpIntPtr, LowIntPtr); in getPointerAndSize()
4164 SizeVal = CGF.getTypeSize(Ty); in getPointerAndSize()
4185 CGOpenMPRuntime::emitTaskInit(CodeGenFunction &CGF, SourceLocation Loc, in emitTaskInit() argument
4260 llvm::Type *KmpTaskTWithPrivatesTy = CGF.ConvertType(KmpTaskTWithPrivatesQTy); in emitTaskInit()
4264 CGF.getTypeSize(KmpTaskTWithPrivatesQTy); in emitTaskInit()
4275 TaskPrivatesMap = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitTaskInit()
4315 ? CGF.Builder.CreateSelect(Data.Final.getPointer(), in emitTaskInit()
4316 CGF.Builder.getInt32(FinalFlag), in emitTaskInit()
4317 CGF.Builder.getInt32(/*C=*/0)) in emitTaskInit()
4318 : CGF.Builder.getInt32(Data.Final.getInt() ? FinalFlag : 0); in emitTaskInit()
4319 TaskFlags = CGF.Builder.CreateOr(TaskFlags, CGF.Builder.getInt32(Flags)); in emitTaskInit()
4321 SmallVector<llvm::Value *, 8> AllocArgs = {emitUpdateLocation(CGF, Loc), in emitTaskInit()
4322 getThreadID(CGF, Loc), TaskFlags, KmpTaskTWithPrivatesTySize, in emitTaskInit()
4323 SharedsSize, CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitTaskInit()
4334 DeviceID = CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(Device), in emitTaskInit()
4335 CGF.Int64Ty, /*isSigned=*/true); in emitTaskInit()
4337 DeviceID = CGF.Builder.getInt64(OMP_DEVICEID_UNDEF); in emitTaskInit()
4339 NewTask = CGF.EmitRuntimeCall( in emitTaskInit()
4345 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in emitTaskInit()
4354 LValue EvtLVal = CGF.EmitLValue(Evt); in emitTaskInit()
4358 llvm::Value *Loc = emitUpdateLocation(CGF, DC->getBeginLoc()); in emitTaskInit()
4359 llvm::Value *Tid = getThreadID(CGF, DC->getBeginLoc()); in emitTaskInit()
4360 Tid = CGF.Builder.CreateIntCast(Tid, CGF.IntTy, /*isSigned=*/false); in emitTaskInit()
4361 llvm::Value *EvtVal = CGF.EmitRuntimeCall( in emitTaskInit()
4365 EvtVal = CGF.EmitScalarConversion(EvtVal, C.VoidPtrTy, Evt->getType(), in emitTaskInit()
4367 CGF.EmitStoreOfScalar(EvtVal, EvtLVal); in emitTaskInit()
4381 llvm::Value *Sz = CGF.EmitScalarExpr(IE->getHelper(I).Upper); in emitTaskInit()
4382 Sz = CGF.Builder.CreateIntCast(Sz, CGF.SizeTy, /*isSigned=*/false); in emitTaskInit()
4384 NumOfElements ? CGF.Builder.CreateNUWMul(NumOfElements, Sz) : Sz; in emitTaskInit()
4396 NumOfElements = CGF.Builder.CreateNUWAdd( in emitTaskInit()
4397 llvm::ConstantInt::get(CGF.SizeTy, NumAffinities), NumOfElements); in emitTaskInit()
4402 CodeGenFunction::OpaqueValueMapping OpaqueMap(CGF, &OVE, in emitTaskInit()
4410 CGF.EmitVarDecl(*PD); in emitTaskInit()
4411 AffinitiesArray = CGF.GetAddrOfLocalVar(PD); in emitTaskInit()
4412 NumOfElements = CGF.Builder.CreateIntCast(NumOfElements, CGF.Int32Ty, in emitTaskInit()
4420 CGF.CreateMemTemp(KmpTaskAffinityInfoArrayTy, ".affs.arr.addr"); in emitTaskInit()
4421 AffinitiesArray = CGF.Builder.CreateConstArrayGEP(AffinitiesArray, 0); in emitTaskInit()
4438 std::tie(Addr, Size) = getPointerAndSize(CGF, E); in emitTaskInit()
4440 CGF.MakeAddrLValue(CGF.Builder.CreateConstGEP(AffinitiesArray, Pos), in emitTaskInit()
4443 LValue BaseAddrLVal = CGF.EmitLValueForField( in emitTaskInit()
4445 CGF.EmitStoreOfScalar(CGF.Builder.CreatePtrToInt(Addr, CGF.IntPtrTy), in emitTaskInit()
4448 LValue LenLVal = CGF.EmitLValueForField( in emitTaskInit()
4450 CGF.EmitStoreOfScalar(Size, LenLVal); in emitTaskInit()
4456 PosLVal = CGF.MakeAddrLValue( in emitTaskInit()
4457 CGF.CreateMemTemp(C.getSizeType(), "affs.counter.addr"), in emitTaskInit()
4459 CGF.EmitStoreOfScalar(llvm::ConstantInt::get(CGF.SizeTy, Pos), PosLVal); in emitTaskInit()
4467 CGF, cast_or_null<OMPIteratorExpr>(Modifier->IgnoreParenImpCasts())); in emitTaskInit()
4471 std::tie(Addr, Size) = getPointerAndSize(CGF, E); in emitTaskInit()
4472 llvm::Value *Idx = CGF.EmitLoadOfScalar(PosLVal, E->getExprLoc()); in emitTaskInit()
4473 LValue Base = CGF.MakeAddrLValue( in emitTaskInit()
4474 Address(CGF.Builder.CreateGEP(AffinitiesArray.getPointer(), Idx), in emitTaskInit()
4478 LValue BaseAddrLVal = CGF.EmitLValueForField( in emitTaskInit()
4480 CGF.EmitStoreOfScalar(CGF.Builder.CreatePtrToInt(Addr, CGF.IntPtrTy), in emitTaskInit()
4483 LValue LenLVal = CGF.EmitLValueForField( in emitTaskInit()
4485 CGF.EmitStoreOfScalar(Size, LenLVal); in emitTaskInit()
4486 Idx = CGF.Builder.CreateNUWAdd( in emitTaskInit()
4488 CGF.EmitStoreOfScalar(Idx, PosLVal); in emitTaskInit()
4494 llvm::Value *LocRef = emitUpdateLocation(CGF, Loc); in emitTaskInit()
4495 llvm::Value *GTid = getThreadID(CGF, Loc); in emitTaskInit()
4496 llvm::Value *AffinListPtr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitTaskInit()
4500 (void)CGF.EmitRuntimeCall( in emitTaskInit()
4506 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitTaskInit()
4508 LValue Base = CGF.MakeNaturalAlignAddrLValue(NewTaskNewTaskTTy, in emitTaskInit()
4511 CGF.EmitLValueForField(Base, *KmpTaskTWithPrivatesQTyRD->field_begin()); in emitTaskInit()
4517 Address(CGF.EmitLoadOfScalar( in emitTaskInit()
4518 CGF.EmitLValueForField( in emitTaskInit()
4523 LValue Dest = CGF.MakeAddrLValue(KmpTaskSharedsPtr, SharedsTy); in emitTaskInit()
4524 LValue Src = CGF.MakeAddrLValue(Shareds, SharedsTy); in emitTaskInit()
4525 CGF.EmitAggregateCopy(Dest, Src, SharedsTy, AggValueSlot::DoesNotOverlap); in emitTaskInit()
4530 emitPrivatesInit(CGF, D, KmpTaskSharedsPtr, Base, KmpTaskTWithPrivatesQTyRD, in emitTaskInit()
4534 (!Data.LastprivateVars.empty() || checkInitIsRequired(CGF, Privates))) { in emitTaskInit()
4551 LValue Data1LV = CGF.EmitLValueForField(TDBase, *FI); in emitTaskInit()
4552 LValue DestructorsLV = CGF.EmitLValueForField( in emitTaskInit()
4554 CGF.EmitStoreOfScalar(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitTaskInit()
4560 LValue Data2LV = CGF.EmitLValueForField( in emitTaskInit()
4562 LValue PriorityLV = CGF.EmitLValueForField( in emitTaskInit()
4564 CGF.EmitStoreOfScalar(Data.Priority.getPointer(), PriorityLV); in emitTaskInit()
4625 CGOpenMPRuntime::getDepobjElements(CodeGenFunction &CGF, LValue DepobjLVal, in getDepobjElements() argument
4632 LValue Base = CGF.EmitLoadOfPointerLValue( in getDepobjElements()
4633 DepobjLVal.getAddress(CGF), in getDepobjElements()
4636 Address Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in getDepobjElements()
4637 Base.getAddress(CGF), CGF.ConvertTypeForMem(KmpDependInfoPtrTy)); in getDepobjElements()
4638 Base = CGF.MakeAddrLValue(Addr, KmpDependInfoTy, Base.getBaseInfo(), in getDepobjElements()
4640 llvm::Value *DepObjAddr = CGF.Builder.CreateGEP( in getDepobjElements()
4642 llvm::ConstantInt::get(CGF.IntPtrTy, -1, /*isSigned=*/true)); in getDepobjElements()
4643 LValue NumDepsBase = CGF.MakeAddrLValue( in getDepobjElements()
4647 LValue BaseAddrLVal = CGF.EmitLValueForField( in getDepobjElements()
4649 llvm::Value *NumDeps = CGF.EmitLoadOfScalar(BaseAddrLVal, Loc); in getDepobjElements()
4653 static void emitDependData(CodeGenFunction &CGF, QualType &KmpDependInfoTy, in emitDependData() argument
4657 CodeGenModule &CGM = CGF.CGM; in emitDependData()
4663 llvm::Type *LLVMFlagsTy = CGF.ConvertTypeForMem(FlagsTy); in emitDependData()
4666 CGF, cast_or_null<OMPIteratorExpr>( in emitDependData()
4672 std::tie(Addr, Size) = getPointerAndSize(CGF, E); in emitDependData()
4675 Base = CGF.MakeAddrLValue( in emitDependData()
4676 CGF.Builder.CreateConstGEP(DependenciesArray, *P), KmpDependInfoTy); in emitDependData()
4679 llvm::Value *Idx = CGF.EmitLoadOfScalar(PosLVal, E->getExprLoc()); in emitDependData()
4680 Base = CGF.MakeAddrLValue( in emitDependData()
4681 Address(CGF.Builder.CreateGEP(DependenciesArray.getPointer(), Idx), in emitDependData()
4686 LValue BaseAddrLVal = CGF.EmitLValueForField( in emitDependData()
4688 CGF.EmitStoreOfScalar(CGF.Builder.CreatePtrToInt(Addr, CGF.IntPtrTy), in emitDependData()
4691 LValue LenLVal = CGF.EmitLValueForField( in emitDependData()
4693 CGF.EmitStoreOfScalar(Size, LenLVal); in emitDependData()
4696 LValue FlagsLVal = CGF.EmitLValueForField( in emitDependData()
4698 CGF.EmitStoreOfScalar(llvm::ConstantInt::get(LLVMFlagsTy, DepKind), in emitDependData()
4704 llvm::Value *Idx = CGF.EmitLoadOfScalar(PosLVal, E->getExprLoc()); in emitDependData()
4705 Idx = CGF.Builder.CreateNUWAdd(Idx, in emitDependData()
4707 CGF.EmitStoreOfScalar(Idx, PosLVal); in emitDependData()
4713 emitDepobjElementsSizes(CodeGenFunction &CGF, QualType &KmpDependInfoTy, in emitDepobjElementsSizes() argument
4719 ASTContext &C = CGF.getContext(); in emitDepobjElementsSizes()
4725 llvm::Type *KmpDependInfoPtrT = CGF.ConvertTypeForMem(KmpDependInfoPtrTy); in emitDepobjElementsSizes()
4728 CGF, cast_or_null<OMPIteratorExpr>( in emitDepobjElementsSizes()
4732 LValue DepobjLVal = CGF.EmitLValue(E->IgnoreParenImpCasts()); in emitDepobjElementsSizes()
4733 LValue Base = CGF.EmitLoadOfPointerLValue( in emitDepobjElementsSizes()
4734 DepobjLVal.getAddress(CGF), in emitDepobjElementsSizes()
4736 Address Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitDepobjElementsSizes()
4737 Base.getAddress(CGF), KmpDependInfoPtrT); in emitDepobjElementsSizes()
4738 Base = CGF.MakeAddrLValue(Addr, KmpDependInfoTy, Base.getBaseInfo(), in emitDepobjElementsSizes()
4740 llvm::Value *DepObjAddr = CGF.Builder.CreateGEP( in emitDepobjElementsSizes()
4742 llvm::ConstantInt::get(CGF.IntPtrTy, -1, /*isSigned=*/true)); in emitDepobjElementsSizes()
4743 LValue NumDepsBase = CGF.MakeAddrLValue( in emitDepobjElementsSizes()
4747 LValue BaseAddrLVal = CGF.EmitLValueForField( in emitDepobjElementsSizes()
4750 CGF.EmitLoadOfScalar(BaseAddrLVal, E->getExprLoc()); in emitDepobjElementsSizes()
4751 LValue NumLVal = CGF.MakeAddrLValue( in emitDepobjElementsSizes()
4752 CGF.CreateMemTemp(C.getUIntPtrType(), "depobj.size.addr"), in emitDepobjElementsSizes()
4754 CGF.InitTempAlloca(NumLVal.getAddress(CGF), in emitDepobjElementsSizes()
4755 llvm::ConstantInt::get(CGF.IntPtrTy, 0)); in emitDepobjElementsSizes()
4756 llvm::Value *PrevVal = CGF.EmitLoadOfScalar(NumLVal, E->getExprLoc()); in emitDepobjElementsSizes()
4757 llvm::Value *Add = CGF.Builder.CreateNUWAdd(PrevVal, NumDeps); in emitDepobjElementsSizes()
4758 CGF.EmitStoreOfScalar(Add, NumLVal); in emitDepobjElementsSizes()
4764 CGF.EmitLoadOfScalar(SizeLVals[I], Data.DepExprs[I]->getExprLoc()); in emitDepobjElementsSizes()
4770 static void emitDepobjElements(CodeGenFunction &CGF, QualType &KmpDependInfoTy, in emitDepobjElements() argument
4776 ASTContext &C = CGF.getContext(); in emitDepobjElements()
4782 llvm::Type *KmpDependInfoPtrT = CGF.ConvertTypeForMem(KmpDependInfoPtrTy); in emitDepobjElements()
4783 llvm::Value *ElSize = CGF.getTypeSize(KmpDependInfoTy); in emitDepobjElements()
4786 CGF, cast_or_null<OMPIteratorExpr>( in emitDepobjElements()
4791 LValue DepobjLVal = CGF.EmitLValue(E->IgnoreParenImpCasts()); in emitDepobjElements()
4792 LValue Base = CGF.EmitLoadOfPointerLValue( in emitDepobjElements()
4793 DepobjLVal.getAddress(CGF), in emitDepobjElements()
4795 Address Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitDepobjElements()
4796 Base.getAddress(CGF), KmpDependInfoPtrT); in emitDepobjElements()
4797 Base = CGF.MakeAddrLValue(Addr, KmpDependInfoTy, Base.getBaseInfo(), in emitDepobjElements()
4801 llvm::Value *DepObjAddr = CGF.Builder.CreateGEP( in emitDepobjElements()
4803 llvm::ConstantInt::get(CGF.IntPtrTy, -1, /*isSigned=*/true)); in emitDepobjElements()
4804 LValue NumDepsBase = CGF.MakeAddrLValue( in emitDepobjElements()
4808 LValue BaseAddrLVal = CGF.EmitLValueForField( in emitDepobjElements()
4811 CGF.EmitLoadOfScalar(BaseAddrLVal, E->getExprLoc()); in emitDepobjElements()
4814 llvm::Value *Size = CGF.Builder.CreateNUWMul( in emitDepobjElements()
4816 CGF.Builder.CreateIntCast(NumDeps, CGF.SizeTy, /*isSigned=*/false)); in emitDepobjElements()
4817 llvm::Value *Pos = CGF.EmitLoadOfScalar(PosLVal, E->getExprLoc()); in emitDepobjElements()
4819 Address(CGF.Builder.CreateGEP(DependenciesArray.getPointer(), Pos), in emitDepobjElements()
4821 CGF.Builder.CreateMemCpy(DepAddr, Base.getAddress(CGF), Size); in emitDepobjElements()
4825 llvm::Value *Add = CGF.Builder.CreateNUWAdd(Pos, NumDeps); in emitDepobjElements()
4826 CGF.EmitStoreOfScalar(Add, PosLVal); in emitDepobjElements()
4832 CodeGenFunction &CGF, ArrayRef<OMPTaskDataTy::DependData> Dependencies, in emitDependClause() argument
4853 llvm::Value *NumOfDepobjElements = llvm::ConstantInt::get(CGF.IntPtrTy, 0); in emitDependClause()
4855 llvm::ConstantInt::get(CGF.IntPtrTy, 1); in emitDependClause()
4860 emitDepobjElementsSizes(CGF, KmpDependInfoTy, D); in emitDependClause()
4863 CGF.Builder.CreateNUWAdd(NumOfDepobjElements, Size); in emitDependClause()
4871 llvm::Value *Sz = CGF.EmitScalarExpr(IE->getHelper(I).Upper); in emitDependClause()
4872 Sz = CGF.Builder.CreateIntCast(Sz, CGF.IntPtrTy, /*isSigned=*/false); in emitDependClause()
4874 CGF.Builder.CreateNUWMul(NumOfRegularWithIterators, Sz); in emitDependClause()
4887 CGF.Builder.CreateNUWAdd(NumOfDepobjElements, NumOfElements); in emitDependClause()
4891 CGF.Builder.CreateNUWAdd(NumOfRegularWithIterators, NumOfElements); in emitDependClause()
4896 CodeGenFunction::OpaqueValueMapping OpaqueMap(CGF, &OVE, in emitDependClause()
4905 CGF.EmitVarDecl(*PD); in emitDependClause()
4906 DependenciesArray = CGF.GetAddrOfLocalVar(PD); in emitDependClause()
4907 NumOfElements = CGF.Builder.CreateIntCast(NumOfElements, CGF.Int32Ty, in emitDependClause()
4914 CGF.CreateMemTemp(KmpDependInfoArrayTy, ".dep.arr.addr"); in emitDependClause()
4915 DependenciesArray = CGF.Builder.CreateConstArrayGEP(DependenciesArray, 0); in emitDependClause()
4924 emitDependData(CGF, KmpDependInfoTy, &Pos, Dependencies[I], in emitDependClause()
4928 LValue PosLVal = CGF.MakeAddrLValue( in emitDependClause()
4929 CGF.CreateMemTemp(C.getSizeType(), "dep.counter.addr"), C.getSizeType()); in emitDependClause()
4930 CGF.EmitStoreOfScalar(llvm::ConstantInt::get(CGF.SizeTy, Pos), PosLVal); in emitDependClause()
4935 emitDependData(CGF, KmpDependInfoTy, &PosLVal, Dependencies[I], in emitDependClause()
4943 emitDepobjElements(CGF, KmpDependInfoTy, PosLVal, Dependencies[I], in emitDependClause()
4947 DependenciesArray = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitDependClause()
4948 DependenciesArray, CGF.VoidPtrTy); in emitDependClause()
4953 CodeGenFunction &CGF, const OMPTaskDataTy::DependData &Dependencies, in emitDepobjDependClause() argument
4975 NumDepsVal = llvm::ConstantInt::get(CGF.SizeTy, 1); in emitDepobjDependClause()
4977 llvm::Value *Sz = CGF.EmitScalarExpr(IE->getHelper(I).Upper); in emitDepobjDependClause()
4978 Sz = CGF.Builder.CreateIntCast(Sz, CGF.SizeTy, /*isSigned=*/false); in emitDepobjDependClause()
4979 NumDepsVal = CGF.Builder.CreateNUWMul(NumDepsVal, Sz); in emitDepobjDependClause()
4981 Size = CGF.Builder.CreateNUWAdd(llvm::ConstantInt::get(CGF.SizeTy, 1), in emitDepobjDependClause()
4986 Size = CGF.Builder.CreateNUWMul(Size, RecSize); in emitDepobjDependClause()
4988 CGF.Builder.CreateIntCast(NumDepsVal, CGF.IntPtrTy, /*isSigned=*/false); in emitDepobjDependClause()
4995 NumDepsVal = llvm::ConstantInt::get(CGF.IntPtrTy, NumDependencies); in emitDepobjDependClause()
4998 llvm::Value *ThreadID = getThreadID(CGF, Loc); in emitDepobjDependClause()
5000 llvm::Value *Allocator = llvm::ConstantPointerNull::get(CGF.VoidPtrTy); in emitDepobjDependClause()
5004 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in emitDepobjDependClause()
5007 Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitDepobjDependClause()
5008 Addr, CGF.ConvertTypeForMem(KmpDependInfoTy)->getPointerTo()); in emitDepobjDependClause()
5011 LValue Base = CGF.MakeAddrLValue(DependenciesArray, KmpDependInfoTy); in emitDepobjDependClause()
5013 LValue BaseAddrLVal = CGF.EmitLValueForField( in emitDepobjDependClause()
5015 CGF.EmitStoreOfScalar(NumDepsVal, BaseAddrLVal); in emitDepobjDependClause()
5020 PosLVal = CGF.MakeAddrLValue( in emitDepobjDependClause()
5021 CGF.CreateMemTemp(C.getSizeType(), "iterator.counter.addr"), in emitDepobjDependClause()
5023 CGF.EmitStoreOfScalar(llvm::ConstantInt::get(CGF.SizeTy, Idx), PosLVal, in emitDepobjDependClause()
5029 emitDependData(CGF, KmpDependInfoTy, Pos, Dependencies, DependenciesArray); in emitDepobjDependClause()
5030 DependenciesArray = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitDepobjDependClause()
5031 CGF.Builder.CreateConstGEP(DependenciesArray, 1), CGF.VoidPtrTy); in emitDepobjDependClause()
5035 void CGOpenMPRuntime::emitDestroyClause(CodeGenFunction &CGF, LValue DepobjLVal, in emitDestroyClause() argument
5040 LValue Base = CGF.EmitLoadOfPointerLValue( in emitDestroyClause()
5041 DepobjLVal.getAddress(CGF), in emitDestroyClause()
5044 Address Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitDestroyClause()
5045 Base.getAddress(CGF), CGF.ConvertTypeForMem(KmpDependInfoPtrTy)); in emitDestroyClause()
5046 llvm::Value *DepObjAddr = CGF.Builder.CreateGEP( in emitDestroyClause()
5048 llvm::ConstantInt::get(CGF.IntPtrTy, -1, /*isSigned=*/true)); in emitDestroyClause()
5049 DepObjAddr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(DepObjAddr, in emitDestroyClause()
5050 CGF.VoidPtrTy); in emitDestroyClause()
5051 llvm::Value *ThreadID = getThreadID(CGF, Loc); in emitDestroyClause()
5053 llvm::Value *Allocator = llvm::ConstantPointerNull::get(CGF.VoidPtrTy); in emitDestroyClause()
5057 (void)CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in emitDestroyClause()
5062 void CGOpenMPRuntime::emitUpdateClause(CodeGenFunction &CGF, LValue DepobjLVal, in emitUpdateClause() argument
5070 llvm::Type *LLVMFlagsTy = CGF.ConvertTypeForMem(FlagsTy); in emitUpdateClause()
5073 std::tie(NumDeps, Base) = getDepobjElements(CGF, DepobjLVal, Loc); in emitUpdateClause()
5075 Address Begin = Base.getAddress(CGF); in emitUpdateClause()
5077 llvm::Value *End = CGF.Builder.CreateGEP(Begin.getPointer(), NumDeps); in emitUpdateClause()
5079 llvm::BasicBlock *BodyBB = CGF.createBasicBlock("omp.body"); in emitUpdateClause()
5080 llvm::BasicBlock *DoneBB = CGF.createBasicBlock("omp.done"); in emitUpdateClause()
5081 llvm::BasicBlock *EntryBB = CGF.Builder.GetInsertBlock(); in emitUpdateClause()
5082 CGF.EmitBlock(BodyBB); in emitUpdateClause()
5084 CGF.Builder.CreatePHI(Begin.getType(), 2, "omp.elementPast"); in emitUpdateClause()
5087 Base = CGF.MakeAddrLValue(Begin, KmpDependInfoTy, Base.getBaseInfo(), in emitUpdateClause()
5091 LValue FlagsLVal = CGF.EmitLValueForField( in emitUpdateClause()
5093 CGF.EmitStoreOfScalar(llvm::ConstantInt::get(LLVMFlagsTy, DepKind), in emitUpdateClause()
5098 CGF.Builder.CreateConstGEP(Begin, /*Index=*/1, "omp.elementNext"); in emitUpdateClause()
5100 CGF.Builder.GetInsertBlock()); in emitUpdateClause()
5102 CGF.Builder.CreateICmpEQ(ElementNext.getPointer(), End, "omp.isempty"); in emitUpdateClause()
5103 CGF.Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB); in emitUpdateClause()
5105 CGF.EmitBlock(DoneBB, /*IsFinished=*/true); in emitUpdateClause()
5108 void CGOpenMPRuntime::emitTaskCall(CodeGenFunction &CGF, SourceLocation Loc, in emitTaskCall() argument
5114 if (!CGF.HaveInsertPoint()) in emitTaskCall()
5118 emitTaskInit(CGF, Loc, D, TaskFunction, SharedsTy, Shareds, Data); in emitTaskCall()
5128 emitDependClause(CGF, Data.Dependences, Loc); in emitTaskCall()
5136 llvm::Value *ThreadID = getThreadID(CGF, Loc); in emitTaskCall()
5137 llvm::Value *UpLoc = emitUpdateLocation(CGF, Loc); in emitTaskCall()
5146 DepTaskArgs[5] = CGF.Builder.getInt32(0); in emitTaskCall()
5147 DepTaskArgs[6] = llvm::ConstantPointerNull::get(CGF.VoidPtrTy); in emitTaskCall()
5150 &DepTaskArgs](CodeGenFunction &CGF, PrePostActionTy &) { in emitTaskCall() argument
5153 LValue PartIdLVal = CGF.EmitLValueForField(TDBase, *PartIdFI); in emitTaskCall()
5154 CGF.EmitStoreOfScalar(CGF.Builder.getInt32(0), PartIdLVal); in emitTaskCall()
5157 CGF.EmitRuntimeCall( in emitTaskCall()
5162 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in emitTaskCall()
5168 dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo)) in emitTaskCall()
5169 Region->emitUntiedSwitch(CGF); in emitTaskCall()
5178 DepWaitTaskArgs[4] = CGF.Builder.getInt32(0); in emitTaskCall()
5179 DepWaitTaskArgs[5] = llvm::ConstantPointerNull::get(CGF.VoidPtrTy); in emitTaskCall()
5184 Loc](CodeGenFunction &CGF, PrePostActionTy &) { in emitTaskCall() argument
5185 CodeGenFunction::RunCleanupsScope LocalScope(CGF); in emitTaskCall()
5191 CGF.EmitRuntimeCall( in emitTaskCall()
5196 Loc](CodeGenFunction &CGF, PrePostActionTy &Action) { in emitTaskCall() argument
5197 Action.Enter(CGF); in emitTaskCall()
5199 CGF.CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF, Loc, TaskEntry, in emitTaskCall()
5215 RCG(CGF); in emitTaskCall()
5219 emitIfClause(CGF, IfCond, ThenCodeGen, ElseCodeGen); in emitTaskCall()
5222 ThenRCG(CGF); in emitTaskCall()
5226 void CGOpenMPRuntime::emitTaskLoopCall(CodeGenFunction &CGF, SourceLocation Loc, in emitTaskLoopCall() argument
5232 if (!CGF.HaveInsertPoint()) in emitTaskLoopCall()
5235 emitTaskInit(CGF, Loc, D, TaskFunction, SharedsTy, Shareds, Data); in emitTaskLoopCall()
5241 llvm::Value *ThreadID = getThreadID(CGF, Loc); in emitTaskLoopCall()
5242 llvm::Value *UpLoc = emitUpdateLocation(CGF, Loc); in emitTaskLoopCall()
5245 IfVal = CGF.Builder.CreateIntCast(CGF.EvaluateExprAsBool(IfCond), CGF.IntTy, in emitTaskLoopCall()
5248 IfVal = llvm::ConstantInt::getSigned(CGF.IntTy, /*V=*/1); in emitTaskLoopCall()
5251 LValue LBLVal = CGF.EmitLValueForField( in emitTaskLoopCall()
5256 CGF.EmitAnyExprToMem(LBVar->getInit(), LBLVal.getAddress(CGF), in emitTaskLoopCall()
5259 LValue UBLVal = CGF.EmitLValueForField( in emitTaskLoopCall()
5264 CGF.EmitAnyExprToMem(UBVar->getInit(), UBLVal.getAddress(CGF), in emitTaskLoopCall()
5267 LValue StLVal = CGF.EmitLValueForField( in emitTaskLoopCall()
5272 CGF.EmitAnyExprToMem(StVar->getInit(), StLVal.getAddress(CGF), in emitTaskLoopCall()
5276 LValue RedLVal = CGF.EmitLValueForField( in emitTaskLoopCall()
5280 CGF.EmitStoreOfScalar(Data.Reductions, RedLVal); in emitTaskLoopCall()
5282 CGF.EmitNullInitialization(RedLVal.getAddress(CGF), in emitTaskLoopCall()
5283 CGF.getContext().VoidPtrTy); in emitTaskLoopCall()
5291 LBLVal.getPointer(CGF), in emitTaskLoopCall()
5292 UBLVal.getPointer(CGF), in emitTaskLoopCall()
5293 CGF.EmitLoadOfScalar(StLVal, Loc), in emitTaskLoopCall()
5295 CGF.IntTy, 1), // Always 1 because taskgroup emitted by the compiler in emitTaskLoopCall()
5297 CGF.IntTy, Data.Schedule.getPointer() in emitTaskLoopCall()
5301 ? CGF.Builder.CreateIntCast(Data.Schedule.getPointer(), CGF.Int64Ty, in emitTaskLoopCall()
5303 : llvm::ConstantInt::get(CGF.Int64Ty, /*V=*/0), in emitTaskLoopCall()
5304 Result.TaskDupFn ? CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitTaskLoopCall()
5305 Result.TaskDupFn, CGF.VoidPtrTy) in emitTaskLoopCall()
5306 : llvm::ConstantPointerNull::get(CGF.VoidPtrTy)}; in emitTaskLoopCall()
5307 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in emitTaskLoopCall()
5322 CodeGenFunction &CGF, QualType Type, const VarDecl *LHSVar, in EmitOMPAggregateReduction() argument
5324 const llvm::function_ref<void(CodeGenFunction &CGF, const Expr *, in EmitOMPAggregateReduction() argument
5330 Address LHSAddr = CGF.GetAddrOfLocalVar(LHSVar); in EmitOMPAggregateReduction()
5331 Address RHSAddr = CGF.GetAddrOfLocalVar(RHSVar); in EmitOMPAggregateReduction()
5335 llvm::Value *NumElements = CGF.emitArrayLength(ArrayTy, ElementTy, LHSAddr); in EmitOMPAggregateReduction()
5340 llvm::Value *LHSEnd = CGF.Builder.CreateGEP(LHSBegin, NumElements); in EmitOMPAggregateReduction()
5342 llvm::BasicBlock *BodyBB = CGF.createBasicBlock("omp.arraycpy.body"); in EmitOMPAggregateReduction()
5343 llvm::BasicBlock *DoneBB = CGF.createBasicBlock("omp.arraycpy.done"); in EmitOMPAggregateReduction()
5345 CGF.Builder.CreateICmpEQ(LHSBegin, LHSEnd, "omp.arraycpy.isempty"); in EmitOMPAggregateReduction()
5346 CGF.Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB); in EmitOMPAggregateReduction()
5349 llvm::BasicBlock *EntryBB = CGF.Builder.GetInsertBlock(); in EmitOMPAggregateReduction()
5350 CGF.EmitBlock(BodyBB); in EmitOMPAggregateReduction()
5352 CharUnits ElementSize = CGF.getContext().getTypeSizeInChars(ElementTy); in EmitOMPAggregateReduction()
5354 llvm::PHINode *RHSElementPHI = CGF.Builder.CreatePHI( in EmitOMPAggregateReduction()
5361 llvm::PHINode *LHSElementPHI = CGF.Builder.CreatePHI( in EmitOMPAggregateReduction()
5369 CodeGenFunction::OMPPrivateScope Scope(CGF); in EmitOMPAggregateReduction()
5373 RedOpGen(CGF, XExpr, EExpr, UpExpr); in EmitOMPAggregateReduction()
5377 llvm::Value *LHSElementNext = CGF.Builder.CreateConstGEP1_32( in EmitOMPAggregateReduction()
5379 llvm::Value *RHSElementNext = CGF.Builder.CreateConstGEP1_32( in EmitOMPAggregateReduction()
5383 CGF.Builder.CreateICmpEQ(LHSElementNext, LHSEnd, "omp.arraycpy.done"); in EmitOMPAggregateReduction()
5384 CGF.Builder.CreateCondBr(Done, DoneBB, BodyBB); in EmitOMPAggregateReduction()
5385 LHSElementPHI->addIncoming(LHSElementNext, CGF.Builder.GetInsertBlock()); in EmitOMPAggregateReduction()
5386 RHSElementPHI->addIncoming(RHSElementNext, CGF.Builder.GetInsertBlock()); in EmitOMPAggregateReduction()
5389 CGF.EmitBlock(DoneBB, /*IsFinished=*/true); in EmitOMPAggregateReduction()
5395 static void emitReductionCombiner(CodeGenFunction &CGF, in emitReductionCombiner() argument
5404 CGF.CGM.getOpenMPRuntime().getUserDefinedReduction(DRD); in emitReductionCombiner()
5406 CodeGenFunction::OpaqueValueMapping Map(CGF, OVE, Func); in emitReductionCombiner()
5407 CGF.EmitIgnoredExpr(ReductionOp); in emitReductionCombiner()
5410 CGF.EmitIgnoredExpr(ReductionOp); in emitReductionCombiner()
5435 CodeGenFunction CGF(CGM); in emitReductionFunction() local
5436 CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, CGFI, Args, Loc, Loc); in emitReductionFunction()
5440 Address LHS(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitReductionFunction()
5441 CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(&LHSArg)), in emitReductionFunction()
5442 ArgsType), CGF.getPointerAlign()); in emitReductionFunction()
5443 Address RHS(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitReductionFunction()
5444 CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(&RHSArg)), in emitReductionFunction()
5445 ArgsType), CGF.getPointerAlign()); in emitReductionFunction()
5450 CodeGenFunction::OMPPrivateScope Scope(CGF); in emitReductionFunction()
5456 Scope.addPrivate(RHSVar, [&CGF, RHS, Idx, RHSVar]() { in emitReductionFunction()
5457 return emitAddrOfVarFromArray(CGF, RHS, Idx, RHSVar); in emitReductionFunction()
5461 Scope.addPrivate(LHSVar, [&CGF, LHS, Idx, LHSVar]() { in emitReductionFunction()
5462 return emitAddrOfVarFromArray(CGF, LHS, Idx, LHSVar); in emitReductionFunction()
5468 Address Elem = CGF.Builder.CreateConstArrayGEP(LHS, Idx); in emitReductionFunction()
5469 llvm::Value *Ptr = CGF.Builder.CreateLoad(Elem); in emitReductionFunction()
5471 CGF.getContext().getAsVariableArrayType(PrivTy); in emitReductionFunction()
5474 CGF, OVE, RValue::get(CGF.Builder.CreatePtrToInt(Ptr, CGF.SizeTy))); in emitReductionFunction()
5475 CGF.EmitVariablyModifiedType(PrivTy); in emitReductionFunction()
5488 CGF, (*IPriv)->getType(), LHSVar, RHSVar, in emitReductionFunction()
5489 [=](CodeGenFunction &CGF, const Expr *, const Expr *, const Expr *) { in emitReductionFunction() argument
5490 emitReductionCombiner(CGF, E); in emitReductionFunction()
5494 emitReductionCombiner(CGF, E); in emitReductionFunction()
5501 CGF.FinishFunction(); in emitReductionFunction()
5505 void CGOpenMPRuntime::emitSingleReductionCombiner(CodeGenFunction &CGF, in emitSingleReductionCombiner() argument
5515 CGF, PrivateRef->getType(), LHSVar, RHSVar, in emitSingleReductionCombiner()
5516 [=](CodeGenFunction &CGF, const Expr *, const Expr *, const Expr *) { in emitSingleReductionCombiner() argument
5517 emitReductionCombiner(CGF, ReductionOp); in emitSingleReductionCombiner()
5521 emitReductionCombiner(CGF, ReductionOp); in emitSingleReductionCombiner()
5525 void CGOpenMPRuntime::emitReduction(CodeGenFunction &CGF, SourceLocation Loc, in emitReduction() argument
5531 if (!CGF.HaveInsertPoint()) in emitReduction()
5575 CodeGenFunction::RunCleanupsScope Scope(CGF); in emitReduction()
5580 emitSingleReductionCombiner(CGF, E, *IPriv, cast<DeclRefExpr>(*ILHS), in emitReduction()
5602 CGF.CreateMemTemp(ReductionArrayTy, ".omp.reduction.red_list"); in emitReduction()
5606 Address Elem = CGF.Builder.CreateConstArrayGEP(ReductionList, Idx); in emitReduction()
5607 CGF.Builder.CreateStore( in emitReduction()
5608 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitReduction()
5609 CGF.EmitLValue(RHSExprs[I]).getPointer(CGF), CGF.VoidPtrTy), in emitReduction()
5614 Elem = CGF.Builder.CreateConstArrayGEP(ReductionList, Idx); in emitReduction()
5615 llvm::Value *Size = CGF.Builder.CreateIntCast( in emitReduction()
5616 CGF.getVLASize( in emitReduction()
5617 CGF.getContext().getAsVariableArrayType((*IPriv)->getType())) in emitReduction()
5619 CGF.SizeTy, /*isSigned=*/false); in emitReduction()
5620 CGF.Builder.CreateStore(CGF.Builder.CreateIntToPtr(Size, CGF.VoidPtrTy), in emitReduction()
5627 Loc, CGF.ConvertTypeForMem(ReductionArrayTy)->getPointerTo(), Privates, in emitReduction()
5636 llvm::Value *IdentTLoc = emitUpdateLocation(CGF, Loc, OMP_ATOMIC_REDUCE); in emitReduction()
5637 llvm::Value *ThreadId = getThreadID(CGF, Loc); in emitReduction()
5638 llvm::Value *ReductionArrayTySize = CGF.getTypeSize(ReductionArrayTy); in emitReduction()
5639 llvm::Value *RL = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitReduction()
5640 ReductionList.getPointer(), CGF.VoidPtrTy); in emitReduction()
5644 CGF.Builder.getInt32(RHSExprs.size()), // i32 <n> in emitReduction()
5650 llvm::Value *Res = CGF.EmitRuntimeCall( in emitReduction()
5657 llvm::BasicBlock *DefaultBB = CGF.createBasicBlock(".omp.reduction.default"); in emitReduction()
5659 CGF.Builder.CreateSwitch(Res, DefaultBB, /*NumCases=*/2); in emitReduction()
5667 llvm::BasicBlock *Case1BB = CGF.createBasicBlock(".omp.reduction.case1"); in emitReduction()
5668 SwInst->addCase(CGF.Builder.getInt32(1), Case1BB); in emitReduction()
5669 CGF.EmitBlock(Case1BB); in emitReduction()
5678 CodeGenFunction &CGF, PrePostActionTy &Action) { in emitReduction() argument
5679 CGOpenMPRuntime &RT = CGF.CGM.getOpenMPRuntime(); in emitReduction()
5684 RT.emitSingleReductionCombiner(CGF, E, *IPriv, cast<DeclRefExpr>(*ILHS), in emitReduction()
5699 RCG(CGF); in emitReduction()
5701 CGF.EmitBranch(DefaultBB); in emitReduction()
5708 llvm::BasicBlock *Case2BB = CGF.createBasicBlock(".omp.reduction.case2"); in emitReduction()
5709 SwInst->addCase(CGF.Builder.getInt32(2), Case2BB); in emitReduction()
5710 CGF.EmitBlock(Case2BB); in emitReduction()
5713 CodeGenFunction &CGF, PrePostActionTy &Action) { in emitReduction() argument
5747 Loc](CodeGenFunction &CGF, const Expr *XExpr, in emitReduction()
5749 LValue X = CGF.EmitLValue(XExpr); in emitReduction()
5752 E = CGF.EmitAnyExpr(EExpr); in emitReduction()
5753 CGF.EmitOMPAtomicSimpleUpdateExpr( in emitReduction()
5756 [&CGF, UpExpr, VD, Loc](RValue XRValue) { in emitReduction()
5757 CodeGenFunction::OMPPrivateScope PrivateScope(CGF); in emitReduction()
5759 VD, [&CGF, VD, XRValue, Loc]() { in emitReduction()
5760 Address LHSTemp = CGF.CreateMemTemp(VD->getType()); in emitReduction()
5761 CGF.emitOMPSimpleStore( in emitReduction()
5762 CGF.MakeAddrLValue(LHSTemp, VD->getType()), XRValue, in emitReduction()
5767 return CGF.EmitAnyExpr(UpExpr); in emitReduction()
5774 EmitOMPAggregateReduction(CGF, (*IPriv)->getType(), VD, RHSVar, in emitReduction()
5778 AtomicRedGen(CGF, XExpr, EExpr, UpExpr); in emitReduction()
5782 auto &&CritRedGen = [E, Loc](CodeGenFunction &CGF, const Expr *, in emitReduction()
5784 CGOpenMPRuntime &RT = CGF.CGM.getOpenMPRuntime(); in emitReduction()
5787 CGF, Name, in emitReduction()
5788 [=](CodeGenFunction &CGF, PrePostActionTy &Action) { in emitReduction() argument
5789 Action.Enter(CGF); in emitReduction()
5790 emitReductionCombiner(CGF, E); in emitReduction()
5799 EmitOMPAggregateReduction(CGF, (*IPriv)->getType(), LHSVar, RHSVar, in emitReduction()
5802 CritRedGen(CGF, nullptr, nullptr, nullptr); in emitReduction()
5823 AtomicRCG(CGF); in emitReduction()
5825 AtomicRCG(CGF); in emitReduction()
5828 CGF.EmitBranch(DefaultBB); in emitReduction()
5829 CGF.EmitBlock(DefaultBB, /*IsFinished=*/true); in emitReduction()
5879 CodeGenFunction CGF(CGM); in emitReduceInitFunction() local
5880 CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, FnInfo, Args, Loc, Loc); in emitReduceInitFunction()
5881 Address PrivateAddr = CGF.EmitLoadOfPointer( in emitReduceInitFunction()
5882 CGF.GetAddrOfLocalVar(&Param), in emitReduceInitFunction()
5889 CGF, CGM.getContext().getSizeType(), in emitReduceInitFunction()
5891 Size = CGF.EmitLoadOfScalar(SizeAddr, /*Volatile=*/false, in emitReduceInitFunction()
5894 RCG.emitAggregateType(CGF, N, Size); in emitReduceInitFunction()
5900 Address SharedAddr = CGF.GetAddrOfLocalVar(&ParamOrig); in emitReduceInitFunction()
5901 SharedAddr = CGF.EmitLoadOfPointer( in emitReduceInitFunction()
5904 OrigLVal = CGF.MakeAddrLValue(SharedAddr, CGM.getContext().VoidPtrTy); in emitReduceInitFunction()
5906 OrigLVal = CGF.MakeNaturalAlignAddrLValue( in emitReduceInitFunction()
5913 RCG.emitInitialization(CGF, N, PrivateAddr, OrigLVal, in emitReduceInitFunction()
5915 CGF.FinishFunction(); in emitReduceInitFunction()
5953 CodeGenFunction CGF(CGM); in emitReduceCombFunction() local
5954 CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, FnInfo, Args, Loc, Loc); in emitReduceCombFunction()
5960 CGF, CGM.getContext().getSizeType(), in emitReduceCombFunction()
5962 Size = CGF.EmitLoadOfScalar(SizeAddr, /*Volatile=*/false, in emitReduceCombFunction()
5965 RCG.emitAggregateType(CGF, N, Size); in emitReduceCombFunction()
5969 CodeGenFunction::OMPPrivateScope PrivateScope(CGF); in emitReduceCombFunction()
5970 PrivateScope.addPrivate(LHSVD, [&C, &CGF, &ParamInOut, LHSVD]() { in emitReduceCombFunction()
5972 Address PtrAddr = CGF.EmitLoadOfPointer( in emitReduceCombFunction()
5973 CGF.GetAddrOfLocalVar(&ParamInOut), in emitReduceCombFunction()
5975 return CGF.Builder.CreateElementBitCast( in emitReduceCombFunction()
5976 PtrAddr, CGF.ConvertTypeForMem(LHSVD->getType())); in emitReduceCombFunction()
5978 PrivateScope.addPrivate(RHSVD, [&C, &CGF, &ParamIn, RHSVD]() { in emitReduceCombFunction()
5980 Address PtrAddr = CGF.EmitLoadOfPointer( in emitReduceCombFunction()
5981 CGF.GetAddrOfLocalVar(&ParamIn), in emitReduceCombFunction()
5983 return CGF.Builder.CreateElementBitCast( in emitReduceCombFunction()
5984 PtrAddr, CGF.ConvertTypeForMem(RHSVD->getType())); in emitReduceCombFunction()
5991 CGF, ReductionOp, PrivateRef, cast<DeclRefExpr>(LHS), in emitReduceCombFunction()
5993 CGF.FinishFunction(); in emitReduceCombFunction()
6023 CodeGenFunction CGF(CGM); in emitReduceFiniFunction() local
6024 CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, FnInfo, Args, Loc, Loc); in emitReduceFiniFunction()
6025 Address PrivateAddr = CGF.EmitLoadOfPointer( in emitReduceFiniFunction()
6026 CGF.GetAddrOfLocalVar(&Param), in emitReduceFiniFunction()
6033 CGF, CGM.getContext().getSizeType(), in emitReduceFiniFunction()
6035 Size = CGF.EmitLoadOfScalar(SizeAddr, /*Volatile=*/false, in emitReduceFiniFunction()
6038 RCG.emitAggregateType(CGF, N, Size); in emitReduceFiniFunction()
6041 RCG.emitCleanups(CGF, N, PrivateAddr); in emitReduceFiniFunction()
6042 CGF.FinishFunction(Loc); in emitReduceFiniFunction()
6047 CodeGenFunction &CGF, SourceLocation Loc, ArrayRef<const Expr *> LHSExprs, in emitTaskReductionInit() argument
6049 if (!CGF.HaveInsertPoint() || Data.ReductionVars.empty()) in emitTaskReductionInit()
6080 Address TaskRedInput = CGF.CreateMemTemp(ArrayRDType, ".rd_input."); in emitTaskReductionInit()
6087 llvm::Value *GEP = CGF.EmitCheckedInBoundsGEP( in emitTaskReductionInit()
6091 LValue ElemLVal = CGF.MakeNaturalAlignAddrLValue(GEP, RDType); in emitTaskReductionInit()
6093 LValue SharedLVal = CGF.EmitLValueForField(ElemLVal, SharedFD); in emitTaskReductionInit()
6094 RCG.emitSharedOrigLValue(CGF, Cnt); in emitTaskReductionInit()
6096 CGF.EmitCastToVoidPtr(RCG.getSharedLValue(Cnt).getPointer(CGF)); in emitTaskReductionInit()
6097 CGF.EmitStoreOfScalar(CastedShared, SharedLVal); in emitTaskReductionInit()
6099 LValue OrigLVal = CGF.EmitLValueForField(ElemLVal, OrigFD); in emitTaskReductionInit()
6101 CGF.EmitCastToVoidPtr(RCG.getOrigLValue(Cnt).getPointer(CGF)); in emitTaskReductionInit()
6102 CGF.EmitStoreOfScalar(CastedOrig, OrigLVal); in emitTaskReductionInit()
6103 RCG.emitAggregateType(CGF, Cnt); in emitTaskReductionInit()
6113 SizeValInChars = CGF.Builder.CreateIntCast(SizeValInChars, CGM.SizeTy, in emitTaskReductionInit()
6115 LValue SizeLVal = CGF.EmitLValueForField(ElemLVal, SizeFD); in emitTaskReductionInit()
6116 CGF.EmitStoreOfScalar(SizeValInChars, SizeLVal); in emitTaskReductionInit()
6118 LValue InitLVal = CGF.EmitLValueForField(ElemLVal, InitFD); in emitTaskReductionInit()
6120 CGF.EmitCastToVoidPtr(emitReduceInitFunction(CGM, Loc, RCG, Cnt)); in emitTaskReductionInit()
6121 CGF.EmitStoreOfScalar(InitAddr, InitLVal); in emitTaskReductionInit()
6123 LValue FiniLVal = CGF.EmitLValueForField(ElemLVal, FiniFD); in emitTaskReductionInit()
6126 ? CGF.EmitCastToVoidPtr(Fini) in emitTaskReductionInit()
6128 CGF.EmitStoreOfScalar(FiniAddr, FiniLVal); in emitTaskReductionInit()
6130 LValue CombLVal = CGF.EmitLValueForField(ElemLVal, CombFD); in emitTaskReductionInit()
6131 llvm::Value *CombAddr = CGF.EmitCastToVoidPtr(emitReduceCombFunction( in emitTaskReductionInit()
6134 CGF.EmitStoreOfScalar(CombAddr, CombLVal); in emitTaskReductionInit()
6136 LValue FlagsLVal = CGF.EmitLValueForField(ElemLVal, FlagsFD); in emitTaskReductionInit()
6138 CGF.EmitStoreOfScalar( in emitTaskReductionInit()
6142 CGF.EmitNullInitialization(FlagsLVal.getAddress(CGF), in emitTaskReductionInit()
6148 llvm::Value *IdentTLoc = emitUpdateLocation(CGF, Loc); in emitTaskReductionInit()
6149 llvm::Value *GTid = CGF.Builder.CreateIntCast(getThreadID(CGF, Loc), in emitTaskReductionInit()
6156 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitTaskReductionInit()
6158 return CGF.EmitRuntimeCall( in emitTaskReductionInit()
6165 CGF.Builder.CreateIntCast(getThreadID(CGF, Loc), CGM.IntTy, in emitTaskReductionInit()
6168 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(TaskRedInput.getPointer(), in emitTaskReductionInit()
6170 return CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in emitTaskReductionInit()
6175 void CGOpenMPRuntime::emitTaskReductionFini(CodeGenFunction &CGF, in emitTaskReductionFini() argument
6180 llvm::Value *IdentTLoc = emitUpdateLocation(CGF, Loc); in emitTaskReductionFini()
6181 llvm::Value *GTid = CGF.Builder.CreateIntCast(getThreadID(CGF, Loc), in emitTaskReductionFini()
6187 (void)CGF.EmitRuntimeCall( in emitTaskReductionFini()
6193 void CGOpenMPRuntime::emitTaskReductionFixups(CodeGenFunction &CGF, in emitTaskReductionFixups() argument
6201 llvm::Value *SizeVal = CGF.Builder.CreateIntCast(Sizes.second, CGM.SizeTy, in emitTaskReductionFixups()
6204 CGF, CGM.getContext().getSizeType(), in emitTaskReductionFixups()
6206 CGF.Builder.CreateStore(SizeVal, SizeAddr, /*IsVolatile=*/false); in emitTaskReductionFixups()
6210 Address CGOpenMPRuntime::getTaskReductionItem(CodeGenFunction &CGF, in getTaskReductionItem() argument
6216 llvm::Value *Args[] = {CGF.Builder.CreateIntCast(getThreadID(CGF, Loc), in getTaskReductionItem()
6220 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in getTaskReductionItem()
6221 SharedLVal.getPointer(CGF), CGM.VoidPtrTy)}; in getTaskReductionItem()
6223 CGF.EmitRuntimeCall( in getTaskReductionItem()
6230 void CGOpenMPRuntime::emitTaskwaitCall(CodeGenFunction &CGF, in emitTaskwaitCall() argument
6232 if (!CGF.HaveInsertPoint()) in emitTaskwaitCall()
6235 if (CGF.CGM.getLangOpts().OpenMPIRBuilder) { in emitTaskwaitCall()
6236 OMPBuilder.createTaskwait(CGF.Builder); in emitTaskwaitCall()
6240 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)}; in emitTaskwaitCall()
6242 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in emitTaskwaitCall()
6247 if (auto *Region = dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo)) in emitTaskwaitCall()
6248 Region->emitUntiedSwitch(CGF); in emitTaskwaitCall()
6251 void CGOpenMPRuntime::emitInlinedDirective(CodeGenFunction &CGF, in emitInlinedDirective() argument
6255 if (!CGF.HaveInsertPoint()) in emitInlinedDirective()
6257 InlinedOpenMPRegionRAII Region(CGF, CodeGen, InnerKind, HasCancel, in emitInlinedDirective()
6261 CGF.CapturedStmtInfo->EmitBody(CGF, /*S=*/nullptr); in emitInlinedDirective()
6290 CodeGenFunction &CGF, SourceLocation Loc, in emitCancellationPointCall() argument
6292 if (!CGF.HaveInsertPoint()) in emitCancellationPointCall()
6297 dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo)) { in emitCancellationPointCall()
6302 emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc), in emitCancellationPointCall()
6303 CGF.Builder.getInt32(getCancellationKind(CancelRegion))}; in emitCancellationPointCall()
6305 llvm::Value *Result = CGF.EmitRuntimeCall( in emitCancellationPointCall()
6312 llvm::BasicBlock *ExitBB = CGF.createBasicBlock(".cancel.exit"); in emitCancellationPointCall()
6313 llvm::BasicBlock *ContBB = CGF.createBasicBlock(".cancel.continue"); in emitCancellationPointCall()
6314 llvm::Value *Cmp = CGF.Builder.CreateIsNotNull(Result); in emitCancellationPointCall()
6315 CGF.Builder.CreateCondBr(Cmp, ExitBB, ContBB); in emitCancellationPointCall()
6316 CGF.EmitBlock(ExitBB); in emitCancellationPointCall()
6319 CGF.getOMPCancelDestination(OMPRegionInfo->getDirectiveKind()); in emitCancellationPointCall()
6320 CGF.EmitBranchThroughCleanup(CancelDest); in emitCancellationPointCall()
6321 CGF.EmitBlock(ContBB, /*IsFinished=*/true); in emitCancellationPointCall()
6326 void CGOpenMPRuntime::emitCancelCall(CodeGenFunction &CGF, SourceLocation Loc, in emitCancelCall() argument
6329 if (!CGF.HaveInsertPoint()) in emitCancelCall()
6335 dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo)) { in emitCancelCall()
6337 OMPRegionInfo](CodeGenFunction &CGF, PrePostActionTy &) { in emitCancelCall() argument
6338 CGOpenMPRuntime &RT = CGF.CGM.getOpenMPRuntime(); in emitCancelCall()
6340 RT.emitUpdateLocation(CGF, Loc), RT.getThreadID(CGF, Loc), in emitCancelCall()
6341 CGF.Builder.getInt32(getCancellationKind(CancelRegion))}; in emitCancelCall()
6343 llvm::Value *Result = CGF.EmitRuntimeCall( in emitCancelCall()
6348 llvm::BasicBlock *ExitBB = CGF.createBasicBlock(".cancel.exit"); in emitCancelCall()
6349 llvm::BasicBlock *ContBB = CGF.createBasicBlock(".cancel.continue"); in emitCancelCall()
6350 llvm::Value *Cmp = CGF.Builder.CreateIsNotNull(Result); in emitCancelCall()
6351 CGF.Builder.CreateCondBr(Cmp, ExitBB, ContBB); in emitCancelCall()
6352 CGF.EmitBlock(ExitBB); in emitCancelCall()
6355 CGF.getOMPCancelDestination(OMPRegionInfo->getDirectiveKind()); in emitCancelCall()
6356 CGF.EmitBranchThroughCleanup(CancelDest); in emitCancelCall()
6357 CGF.EmitBlock(ContBB, /*IsFinished=*/true); in emitCancelCall()
6360 emitIfClause(CGF, IfCond, ThenGen, in emitCancelCall()
6364 ThenRCG(CGF); in emitCancelCall()
6378 void Enter(CodeGenFunction &CGF) override { in Enter() argument
6379 if (!CGF.HaveInsertPoint()) in Enter()
6382 CGF.CGM.getOpenMPRuntime().emitUsesAllocatorsInit( in Enter()
6383 CGF, AllocatorData.first, AllocatorData.second); in Enter()
6386 void Exit(CodeGenFunction &CGF) override { in Exit() argument
6387 if (!CGF.HaveInsertPoint()) in Exit()
6390 CGF.CGM.getOpenMPRuntime().emitUsesAllocatorsFini(CGF, in Exit()
6418 void CGOpenMPRuntime::emitUsesAllocatorsInit(CodeGenFunction &CGF, in emitUsesAllocatorsInit() argument
6421 llvm::Value *ThreadId = getThreadID(CGF, Allocator->getExprLoc()); in emitUsesAllocatorsInit()
6422 ThreadId = CGF.Builder.CreateIntCast(ThreadId, CGF.IntTy, /*isSigned=*/true); in emitUsesAllocatorsInit()
6424 llvm::Value *MemSpaceHandle = llvm::ConstantPointerNull::get(CGF.VoidPtrTy); in emitUsesAllocatorsInit()
6426 CGF.IntTy, cast<ConstantArrayType>( in emitUsesAllocatorsInit()
6430 LValue AllocatorTraitsLVal = CGF.EmitLValue(AllocatorTraits); in emitUsesAllocatorsInit()
6431 Address Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitUsesAllocatorsInit()
6432 AllocatorTraitsLVal.getAddress(CGF), CGF.VoidPtrPtrTy); in emitUsesAllocatorsInit()
6433 AllocatorTraitsLVal = CGF.MakeAddrLValue(Addr, CGF.getContext().VoidPtrTy, in emitUsesAllocatorsInit()
6437 CGF.EmitLoadOfScalar(AllocatorTraitsLVal, AllocatorTraits->getExprLoc()); in emitUsesAllocatorsInit()
6440 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in emitUsesAllocatorsInit()
6444 CGF.EmitVarDecl(*cast<VarDecl>( in emitUsesAllocatorsInit()
6446 LValue AllocatorLVal = CGF.EmitLValue(Allocator->IgnoreParenImpCasts()); in emitUsesAllocatorsInit()
6448 CGF.EmitScalarConversion(AllocatorVal, CGF.getContext().VoidPtrTy, in emitUsesAllocatorsInit()
6450 CGF.EmitStoreOfScalar(AllocatorVal, AllocatorLVal); in emitUsesAllocatorsInit()
6453 void CGOpenMPRuntime::emitUsesAllocatorsFini(CodeGenFunction &CGF, in emitUsesAllocatorsFini() argument
6455 llvm::Value *ThreadId = getThreadID(CGF, Allocator->getExprLoc()); in emitUsesAllocatorsFini()
6456 ThreadId = CGF.Builder.CreateIntCast(ThreadId, CGF.IntTy, /*isSigned=*/true); in emitUsesAllocatorsFini()
6457 LValue AllocatorLVal = CGF.EmitLValue(Allocator->IgnoreParenImpCasts()); in emitUsesAllocatorsFini()
6459 CGF.EmitLoadOfScalar(AllocatorLVal, Allocator->getExprLoc()); in emitUsesAllocatorsFini()
6460 AllocatorVal = CGF.EmitScalarConversion(AllocatorVal, Allocator->getType(), in emitUsesAllocatorsFini()
6461 CGF.getContext().VoidPtrTy, in emitUsesAllocatorsFini()
6463 (void)CGF.EmitRuntimeCall( in emitUsesAllocatorsFini()
6496 CodeGenFunction CGF(CGM, true); in emitTargetOutlinedFunctionHelper() local
6498 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo); in emitTargetOutlinedFunctionHelper()
6500 OutlinedFn = CGF.GenerateOpenMPCapturedStmtFunction(CS, D.getBeginLoc()); in emitTargetOutlinedFunctionHelper()
6599 emitNumTeamsForTargetDirective(CodeGenFunction &CGF, in emitNumTeamsForTargetDirective() argument
6601 assert(!CGF.getLangOpts().OpenMPIsDevice && in emitNumTeamsForTargetDirective()
6607 CGBuilderTy &Bld = CGF.Builder; in emitNumTeamsForTargetDirective()
6614 CGOpenMPRuntime::getSingleCompoundChild(CGF.getContext(), Body); in emitNumTeamsForTargetDirective()
6619 CGOpenMPInnerExprInfo CGInfo(CGF, *CS); in emitNumTeamsForTargetDirective()
6620 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo); in emitNumTeamsForTargetDirective()
6624 CGF.EmitScalarExpr(NumTeams, in emitNumTeamsForTargetDirective()
6626 return Bld.CreateIntCast(NumTeamsVal, CGF.Int32Ty, in emitNumTeamsForTargetDirective()
6644 CodeGenFunction::RunCleanupsScope NumTeamsScope(CGF); in emitNumTeamsForTargetDirective()
6648 CGF.EmitScalarExpr(NumTeams, in emitNumTeamsForTargetDirective()
6650 return Bld.CreateIntCast(NumTeamsVal, CGF.Int32Ty, in emitNumTeamsForTargetDirective()
6724 static llvm::Value *getNumThreads(CodeGenFunction &CGF, const CapturedStmt *CS, in getNumThreads() argument
6727 CGF.getContext(), CS->getCapturedStmt()); in getNumThreads()
6735 CGOpenMPInnerExprInfo CGInfo(CGF, *CS); in getNumThreads()
6736 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo); in getNumThreads()
6748 if (Cond->EvaluateAsBooleanCondition(Result, CGF.getContext())) { in getNumThreads()
6750 return CGF.Builder.getInt32(1); in getNumThreads()
6752 CodeGenFunction::LexicalScope Scope(CGF, Cond->getSourceRange()); in getNumThreads()
6757 CGF.EmitVarDecl(cast<VarDecl>(*I)); in getNumThreads()
6760 CGF.EmitAutoVarAlloca(cast<VarDecl>(*I)); in getNumThreads()
6761 CGF.EmitAutoVarCleanups(Emission); in getNumThreads()
6765 CondVal = CGF.EvaluateExprAsBool(Cond); in getNumThreads()
6772 CGOpenMPInnerExprInfo CGInfo(CGF, *CS); in getNumThreads()
6773 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo); in getNumThreads()
6777 CGF, NumThreadsClause->getNumThreads()->getSourceRange()); in getNumThreads()
6782 CGF.EmitVarDecl(cast<VarDecl>(*I)); in getNumThreads()
6785 CGF.EmitAutoVarAlloca(cast<VarDecl>(*I)); in getNumThreads()
6786 CGF.EmitAutoVarCleanups(Emission); in getNumThreads()
6790 NumThreads = CGF.EmitScalarExpr(NumThreadsClause->getNumThreads()); in getNumThreads()
6791 NumThreads = CGF.Builder.CreateIntCast(NumThreads, CGF.Int32Ty, in getNumThreads()
6794 NumThreads = CGF.Builder.CreateSelect( in getNumThreads()
6795 CGF.Builder.CreateICmpULT(DefaultThreadLimitVal, NumThreads), in getNumThreads()
6799 : CGF.Builder.getInt32(0); in getNumThreads()
6803 NumThreads = CGF.Builder.CreateSelect(CondVal, NumThreads, in getNumThreads()
6804 CGF.Builder.getInt32(1)); in getNumThreads()
6809 return CGF.Builder.getInt32(1); in getNumThreads()
6813 : CGF.Builder.getInt32(0); in getNumThreads()
6825 emitNumThreadsForTargetDirective(CodeGenFunction &CGF, in emitNumThreadsForTargetDirective() argument
6827 assert(!CGF.getLangOpts().OpenMPIsDevice && in emitNumThreadsForTargetDirective()
6833 CGBuilderTy &Bld = CGF.Builder; in emitNumThreadsForTargetDirective()
6839 if (llvm::Value *NumThreads = getNumThreads(CGF, CS, ThreadLimitVal)) in emitNumThreadsForTargetDirective()
6842 CGF.getContext(), CS->getCapturedStmt()); in emitNumThreadsForTargetDirective()
6845 CGOpenMPInnerExprInfo CGInfo(CGF, *CS); in emitNumThreadsForTargetDirective()
6846 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo); in emitNumThreadsForTargetDirective()
6850 CGF, ThreadLimitClause->getThreadLimit()->getSourceRange()); in emitNumThreadsForTargetDirective()
6855 CGF.EmitVarDecl(cast<VarDecl>(*I)); in emitNumThreadsForTargetDirective()
6858 CGF.EmitAutoVarAlloca(cast<VarDecl>(*I)); in emitNumThreadsForTargetDirective()
6859 CGF.EmitAutoVarCleanups(Emission); in emitNumThreadsForTargetDirective()
6863 llvm::Value *ThreadLimit = CGF.EmitScalarExpr( in emitNumThreadsForTargetDirective()
6866 Bld.CreateIntCast(ThreadLimit, CGF.Int32Ty, /*isSigned=*/false); in emitNumThreadsForTargetDirective()
6872 CGF.getContext(), CS->getCapturedStmt()); in emitNumThreadsForTargetDirective()
6878 if (llvm::Value *NumThreads = getNumThreads(CGF, CS, ThreadLimitVal)) in emitNumThreadsForTargetDirective()
6888 CodeGenFunction::RunCleanupsScope ThreadLimitScope(CGF); in emitNumThreadsForTargetDirective()
6890 llvm::Value *ThreadLimit = CGF.EmitScalarExpr( in emitNumThreadsForTargetDirective()
6893 Bld.CreateIntCast(ThreadLimit, CGF.Int32Ty, /*isSigned=*/false); in emitNumThreadsForTargetDirective()
6896 if (llvm::Value *NumThreads = getNumThreads(CGF, CS, ThreadLimitVal)) in emitNumThreadsForTargetDirective()
6899 CGF.getContext(), CS->getCapturedStmt()); in emitNumThreadsForTargetDirective()
6903 if (llvm::Value *NumThreads = getNumThreads(CGF, CS, ThreadLimitVal)) in emitNumThreadsForTargetDirective()
6911 CodeGenFunction::RunCleanupsScope ThreadLimitScope(CGF); in emitNumThreadsForTargetDirective()
6913 llvm::Value *ThreadLimit = CGF.EmitScalarExpr( in emitNumThreadsForTargetDirective()
6916 Bld.CreateIntCast(ThreadLimit, CGF.Int32Ty, /*isSigned=*/false); in emitNumThreadsForTargetDirective()
6918 return getNumThreads(CGF, D.getInnermostCapturedStmt(), ThreadLimitVal); in emitNumThreadsForTargetDirective()
6939 if (Cond->EvaluateAsBooleanCondition(Result, CGF.getContext())) { in emitNumThreadsForTargetDirective()
6943 CodeGenFunction::RunCleanupsScope Scope(CGF); in emitNumThreadsForTargetDirective()
6944 CondVal = CGF.EvaluateExprAsBool(Cond); in emitNumThreadsForTargetDirective()
6949 CodeGenFunction::RunCleanupsScope ThreadLimitScope(CGF); in emitNumThreadsForTargetDirective()
6951 llvm::Value *ThreadLimit = CGF.EmitScalarExpr( in emitNumThreadsForTargetDirective()
6954 Bld.CreateIntCast(ThreadLimit, CGF.Int32Ty, /*isSigned=*/false); in emitNumThreadsForTargetDirective()
6957 CodeGenFunction::RunCleanupsScope NumThreadsScope(CGF); in emitNumThreadsForTargetDirective()
6959 llvm::Value *NumThreads = CGF.EmitScalarExpr( in emitNumThreadsForTargetDirective()
6962 Bld.CreateIntCast(NumThreads, CGF.Int32Ty, /*isSigned=*/false); in emitNumThreadsForTargetDirective()
7253 CodeGenFunction &CGF; member in __anonad2d34463d11::MappableExprsHandler
7273 CGF.getTypeSize(OAE->getBase()->getType()->getPointeeType()); in getExprTypeSize()
7275 llvm::Value *Sz = CGF.EmitScalarExpr(SE); in getExprTypeSize()
7276 Sz = CGF.EmitScalarConversion(Sz, SE->getType(), in getExprTypeSize()
7277 CGF.getContext().getSizeType(), in getExprTypeSize()
7279 Size = CGF.Builder.CreateNUWMul(Size, Sz); in getExprTypeSize()
7301 return CGF.getTypeSize(BaseTy); in getExprTypeSize()
7305 ElemSize = CGF.getTypeSize(PTy->getPointeeType().getCanonicalType()); in getExprTypeSize()
7309 ElemSize = CGF.getTypeSize(ATy->getElementType().getCanonicalType()); in getExprTypeSize()
7318 llvm::Value *LengthVal = CGF.EmitScalarExpr(LenExpr); in getExprTypeSize()
7319 LengthVal = CGF.EmitScalarConversion(LengthVal, LenExpr->getType(), in getExprTypeSize()
7320 CGF.getContext().getSizeType(), in getExprTypeSize()
7322 return CGF.Builder.CreateNUWMul(LengthVal, ElemSize); in getExprTypeSize()
7327 llvm::Value *LengthVal = CGF.getTypeSize(BaseTy); in getExprTypeSize()
7328 llvm::Value *LBVal = CGF.EmitScalarExpr(OAE->getLowerBound()); in getExprTypeSize()
7329 LBVal = CGF.EmitScalarConversion(LBVal, OAE->getLowerBound()->getType(), in getExprTypeSize()
7330 CGF.getContext().getSizeType(), in getExprTypeSize()
7332 LBVal = CGF.Builder.CreateNUWMul(LBVal, ElemSize); in getExprTypeSize()
7333 llvm::Value *Cmp = CGF.Builder.CreateICmpUGT(LengthVal, LBVal); in getExprTypeSize()
7334 llvm::Value *TrueVal = CGF.Builder.CreateNUWSub(LengthVal, LBVal); in getExprTypeSize()
7335 LengthVal = CGF.Builder.CreateSelect( in getExprTypeSize()
7336 Cmp, TrueVal, llvm::ConstantInt::get(CGF.SizeTy, 0)); in getExprTypeSize()
7339 return CGF.getTypeSize(ExprTy); in getExprTypeSize()
7427 if (!Length->EvaluateAsInt(Result, CGF.getContext())) in isFinalArraySectionExpression()
7641 BP = CGF.LoadCXXThisAddress(); in generateInfoForComponentList()
7645 BP = CGF.EmitOMPSharedLValue(AssocExpr).getAddress(CGF); in generateInfoForComponentList()
7649 CGF.EmitScalarExpr(OAShE->getBase()), in generateInfoForComponentList()
7650 CGF.getContext().getTypeAlignInChars(OAShE->getBase()->getType())); in generateInfoForComponentList()
7654 BP = CGF.EmitOMPSharedLValue(AssocExpr).getAddress(CGF); in generateInfoForComponentList()
7661 CGF.CGM.getOpenMPRuntime().hasRequiresUnifiedSharedMemory())) { in generateInfoForComponentList()
7663 BP = CGF.CGM.getOpenMPRuntime().getAddrOfDeclareTargetVar(VD); in generateInfoForComponentList()
7678 if (CGF.CGM.getOpenMPRuntime().hasRequiresUnifiedSharedMemory() || in generateInfoForComponentList()
7680 BP = CGF.EmitLoadOfPointer(BP, Ty->castAs<PointerType>()); in generateInfoForComponentList()
7734 BP = CGF.EmitLoadOfPointer(BP, Ty->castAs<PointerType>()); in generateInfoForComponentList()
7800 auto &&EmitMemberExprBase = [](CodeGenFunction &CGF, in generateInfoForComponentList()
7810 CGF.EmitPointerWithAlignment(BaseExpr, &BaseInfo, &TBAAInfo); in generateInfoForComponentList()
7812 BaseLV = CGF.MakeAddrLValue(Addr, PtrTy, BaseInfo, TBAAInfo); in generateInfoForComponentList()
7814 BaseLV = CGF.EmitOMPSharedLValue(BaseExpr); in generateInfoForComponentList()
7819 LowestElem = LB = Address(CGF.EmitScalarExpr(OAShE->getBase()), in generateInfoForComponentList()
7820 CGF.getContext().getTypeAlignInChars( in generateInfoForComponentList()
7824 LValue BaseLVal = EmitMemberExprBase(CGF, ME); in generateInfoForComponentList()
7825 LowestElem = CGF.EmitLValueForFieldInitialization( in generateInfoForComponentList()
7827 .getAddress(CGF); in generateInfoForComponentList()
7828 LB = CGF.EmitLoadOfReferenceLValue(LowestElem, MapDecl->getType()) in generateInfoForComponentList()
7829 .getAddress(CGF); in generateInfoForComponentList()
7832 CGF.EmitOMPSharedLValue(I->getAssociatedExpression()) in generateInfoForComponentList()
7833 .getAddress(CGF); in generateInfoForComponentList()
7854 CharUnits TypeSize = CGF.getContext().getTypeSizeInChars( in generateInfoForComponentList()
7856 Address HB = CGF.Builder.CreateConstGEP( in generateInfoForComponentList()
7857 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(LowestElem, in generateInfoForComponentList()
7858 CGF.VoidPtrTy), in generateInfoForComponentList()
7888 LValue BaseLVal = EmitMemberExprBase(CGF, ME); in generateInfoForComponentList()
7890 CGF.EmitLValueForFieldInitialization(BaseLVal, FD) in generateInfoForComponentList()
7891 .getAddress(CGF); in generateInfoForComponentList()
7894 CGF.EmitOMPSharedLValue(MC.getAssociatedExpression()) in generateInfoForComponentList()
7895 .getAddress(CGF); in generateInfoForComponentList()
7897 Size = CGF.Builder.CreatePtrDiff( in generateInfoForComponentList()
7898 CGF.EmitCastToVoidPtr(ComponentLB.getPointer()), in generateInfoForComponentList()
7899 CGF.EmitCastToVoidPtr(LB.getPointer())); in generateInfoForComponentList()
7907 CombinedInfo.Sizes.push_back(CGF.Builder.CreateIntCast( in generateInfoForComponentList()
7908 Size, CGF.Int64Ty, /*isSigned=*/true)); in generateInfoForComponentList()
7913 LB = CGF.Builder.CreateConstGEP(ComponentLB, 1); in generateInfoForComponentList()
7918 Size = CGF.Builder.CreatePtrDiff( in generateInfoForComponentList()
7919 CGF.Builder.CreateConstGEP(HB, 1).getPointer(), in generateInfoForComponentList()
7920 CGF.EmitCastToVoidPtr(LB.getPointer())); in generateInfoForComponentList()
7922 CGF.Builder.CreateIntCast(Size, CGF.Int64Ty, /*isSigned=*/true)); in generateInfoForComponentList()
7936 CGF.Builder.CreateIntCast(Size, CGF.Int64Ty, /*isSigned=*/true)); in generateInfoForComponentList()
7986 CGF.EmitOMPArraySectionExpr(OASE, /*IsLowerBound=*/false) in generateInfoForComponentList()
7987 .getAddress(CGF); in generateInfoForComponentList()
8022 BP = CGF.EmitLoadOfPointer(BP, Ty->castAs<PointerType>()); in generateInfoForComponentList()
8034 const ASTContext &Context = CGF.getContext(); in generateInfoForComponentList()
8038 MapValuesArrayTy CurOffsets = {llvm::ConstantInt::get(CGF.CGM.Int64Ty, 0)}; in generateInfoForComponentList()
8039 MapValuesArrayTy CurCounts = {llvm::ConstantInt::get(CGF.CGM.Int64Ty, 1)}; in generateInfoForComponentList()
8041 MapValuesArrayTy DimSizes{llvm::ConstantInt::get(CGF.CGM.Int64Ty, 1)}; in generateInfoForComponentList()
8086 llvm::ConstantInt::get(CGF.Int64Ty, ElementTypeSize)); in generateInfoForComponentList()
8094 CGF.Int64Ty, CAT->getSize().getZExtValue())); in generateInfoForComponentList()
8096 DimSizes.push_back(CGF.Builder.CreateIntCast( in generateInfoForComponentList()
8097 CGF.EmitScalarExpr(VAT->getSizeExpr()), CGF.Int64Ty, in generateInfoForComponentList()
8106 llvm::ConstantInt::get(CGF.CGM.Int64Ty, ElementTypeSize); in generateInfoForComponentList()
8120 llvm::Value *Offset = CGF.Builder.CreateIntCast( in generateInfoForComponentList()
8121 CGF.EmitScalarExpr(AE->getIdx()), CGF.Int64Ty, in generateInfoForComponentList()
8124 CurCounts.push_back(llvm::ConstantInt::get(CGF.Int64Ty, /*V=*/1)); in generateInfoForComponentList()
8139 Offset = llvm::ConstantInt::get(CGF.Int64Ty, 0); in generateInfoForComponentList()
8141 Offset = CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(OffsetExpr), in generateInfoForComponentList()
8142 CGF.Int64Ty, in generateInfoForComponentList()
8157 Count = llvm::ConstantInt::get(CGF.Int64Ty, 1); in generateInfoForComponentList()
8166 ? CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(StrideExpr), in generateInfoForComponentList()
8167 CGF.Int64Ty, /*isSigned=*/false) in generateInfoForComponentList()
8170 Count = CGF.Builder.CreateUDiv( in generateInfoForComponentList()
8171 CGF.Builder.CreateNUWSub(*DI, Offset), Stride); in generateInfoForComponentList()
8173 Count = CGF.Builder.CreateNUWSub(*DI, Offset); in generateInfoForComponentList()
8176 Count = CGF.EmitScalarExpr(CountExpr); in generateInfoForComponentList()
8178 Count = CGF.Builder.CreateIntCast(Count, CGF.Int64Ty, /*isSigned=*/false); in generateInfoForComponentList()
8191 ? CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(StrideExpr), in generateInfoForComponentList()
8192 CGF.Int64Ty, /*isSigned=*/false) in generateInfoForComponentList()
8194 DimProd = CGF.Builder.CreateNUWMul(DimProd, *(DI - 1)); in generateInfoForComponentList()
8196 CurStrides.push_back(CGF.Builder.CreateNUWMul(DimProd, Stride)); in generateInfoForComponentList()
8219 if (Cap.getCapturedVar()->getType().isConstant(CGF.getContext()) && in getMapModifiersForPrivateClauses()
8257 const CGRecordLayout &RL = CGF.getTypes().getCGRecordLayout(RD); in getPlainLayout()
8273 if (Base->isEmpty() || CGF.getContext() in getPlainLayout()
8298 if (!Field->isBitField() && !Field->isZeroSize(CGF.getContext())) { in getPlainLayout()
8459 if (CGF.CGM.getOpenMPRuntime().hasRequiresUnifiedSharedMemory() || in generateAllInfoForClauses()
8492 CGF.EmitLoadOfScalar(CGF.EmitLValue(IE), IE->getExprLoc()); in generateAllInfoForClauses()
8497 llvm::Constant::getNullValue(CGF.Int64Ty)); in generateAllInfoForClauses()
8566 Ptr = CGF.EmitLValue(IE).getPointer(CGF); in generateAllInfoForClauses()
8568 Ptr = CGF.EmitScalarExpr(IE); in generateAllInfoForClauses()
8573 llvm::Constant::getNullValue(CGF.Int64Ty)); in generateAllInfoForClauses()
8627 Ptr = this->CGF.EmitLValue(L.IE).getPointer(CGF); in generateAllInfoForClauses()
8629 Ptr = this->CGF.EmitScalarExpr(L.IE); in generateAllInfoForClauses()
8636 BasePtr = this->CGF.EmitLValue(L.IE).getPointer(CGF); in generateAllInfoForClauses()
8637 Ptr = this->CGF.EmitLoadOfScalar(this->CGF.EmitLValue(L.IE), in generateAllInfoForClauses()
8649 llvm::Constant::getNullValue(this->CGF.Int64Ty)); in generateAllInfoForClauses()
8669 MappableExprsHandler(const OMPExecutableDirective &Dir, CodeGenFunction &CGF) in MappableExprsHandler() argument
8670 : CurDir(&Dir), CGF(CGF) { in MappableExprsHandler()
8696 MappableExprsHandler(const OMPDeclareMapperDecl &Dir, CodeGenFunction &CGF) in MappableExprsHandler() argument
8697 : CurDir(&Dir), CGF(CGF) {} in MappableExprsHandler()
8727 llvm::Value *HAddr = CGF.Builder.CreateConstGEP1_32(HB, /*Idx0=*/1); in emitCombinedEntry()
8728 llvm::Value *CLAddr = CGF.Builder.CreatePointerCast(LB, CGF.VoidPtrTy); in emitCombinedEntry()
8729 llvm::Value *CHAddr = CGF.Builder.CreatePointerCast(HAddr, CGF.VoidPtrTy); in emitCombinedEntry()
8730 llvm::Value *Diff = CGF.Builder.CreatePtrDiff(CHAddr, CLAddr); in emitCombinedEntry()
8731 llvm::Value *Size = CGF.Builder.CreateIntCast(Diff, CGF.Int64Ty, in emitCombinedEntry()
8791 Address VDAddr = Address(Arg, CGF.getContext().getDeclAlign(VD)); in generateInfoForLambdaCaptures()
8792 LValue VDLVal = CGF.MakeAddrLValue( in generateInfoForLambdaCaptures()
8799 CGF.EmitLValueForFieldInitialization(VDLVal, ThisCapture); in generateInfoForLambdaCaptures()
8800 LValue ThisLValVal = CGF.EmitLValueForField(VDLVal, ThisCapture); in generateInfoForLambdaCaptures()
8801 LambdaPointers.try_emplace(ThisLVal.getPointer(CGF), in generateInfoForLambdaCaptures()
8802 VDLVal.getPointer(CGF)); in generateInfoForLambdaCaptures()
8804 CombinedInfo.BasePointers.push_back(ThisLVal.getPointer(CGF)); in generateInfoForLambdaCaptures()
8805 CombinedInfo.Pointers.push_back(ThisLValVal.getPointer(CGF)); in generateInfoForLambdaCaptures()
8807 CGF.Builder.CreateIntCast(CGF.getTypeSize(CGF.getContext().VoidPtrTy), in generateInfoForLambdaCaptures()
8808 CGF.Int64Ty, /*isSigned=*/true)); in generateInfoForLambdaCaptures()
8821 LValue VarLVal = CGF.EmitLValueForFieldInitialization(VDLVal, It->second); in generateInfoForLambdaCaptures()
8823 LValue VarLValVal = CGF.EmitLValueForField(VDLVal, It->second); in generateInfoForLambdaCaptures()
8824 LambdaPointers.try_emplace(VarLVal.getPointer(CGF), in generateInfoForLambdaCaptures()
8825 VDLVal.getPointer(CGF)); in generateInfoForLambdaCaptures()
8827 CombinedInfo.BasePointers.push_back(VarLVal.getPointer(CGF)); in generateInfoForLambdaCaptures()
8828 CombinedInfo.Pointers.push_back(VarLValVal.getPointer(CGF)); in generateInfoForLambdaCaptures()
8829 CombinedInfo.Sizes.push_back(CGF.Builder.CreateIntCast( in generateInfoForLambdaCaptures()
8830 CGF.getTypeSize( in generateInfoForLambdaCaptures()
8832 CGF.Int64Ty, /*isSigned=*/true)); in generateInfoForLambdaCaptures()
8834 RValue VarRVal = CGF.EmitLoadOfLValue(VarLVal, RD->getLocation()); in generateInfoForLambdaCaptures()
8835 LambdaPointers.try_emplace(VarLVal.getPointer(CGF), in generateInfoForLambdaCaptures()
8836 VDLVal.getPointer(CGF)); in generateInfoForLambdaCaptures()
8838 CombinedInfo.BasePointers.push_back(VarLVal.getPointer(CGF)); in generateInfoForLambdaCaptures()
8840 CombinedInfo.Sizes.push_back(llvm::ConstantInt::get(CGF.Int64Ty, 0)); in generateInfoForLambdaCaptures()
8897 CombinedInfo.Sizes.push_back(CGF.Builder.CreateIntCast( in generateInfoForCapture()
8898 CGF.getTypeSize(CGF.getContext().VoidPtrTy), CGF.Int64Ty, in generateInfoForCapture()
9122 CGF.Builder.CreateIntCast(CGF.getTypeSize(PtrTy->getPointeeType()), in generateDefaultMapInfo()
9123 CGF.Int64Ty, /*isSigned=*/true)); in generateDefaultMapInfo()
9135 CombinedInfo.Sizes.push_back(CGF.Builder.CreateIntCast( in generateDefaultMapInfo()
9136 CGF.getTypeSize(RI.getType()), CGF.Int64Ty, /*isSigned=*/true)); in generateDefaultMapInfo()
9141 CombinedInfo.Sizes.push_back(llvm::Constant::getNullValue(CGF.Int64Ty)); in generateDefaultMapInfo()
9150 CombinedInfo.Sizes.push_back(CGF.Builder.CreateIntCast( in generateDefaultMapInfo()
9151 CGF.getTypeSize(ElementType), CGF.Int64Ty, /*isSigned=*/true)); in generateDefaultMapInfo()
9159 VD->getType().isConstant(CGF.getContext())) { in generateDefaultMapInfo()
9161 CGF.CGM.getOpenMPRuntime().registerTargetFirstprivateCopy(CGF, VD); in generateDefaultMapInfo()
9163 CGF.Builder.CreateMemCpy( in generateDefaultMapInfo()
9164 CGF.MakeNaturalAlignAddrLValue(Addr, ElementType).getAddress(CGF), in generateDefaultMapInfo()
9165 Address(CV, CGF.getContext().getTypeAlignInChars(ElementType)), in generateDefaultMapInfo()
9175 Address PtrAddr = CGF.EmitLoadOfReference(CGF.MakeAddrLValue( in generateDefaultMapInfo()
9176 CV, ElementType, CGF.getContext().getDeclAlign(VD), in generateDefaultMapInfo()
9200 CodeGenFunction &CGF, MappableExprsHandler::MapCombinedInfoTy &CombinedInfo, in emitNonContiguousDescriptor() argument
9202 CodeGenModule &CGM = CGF.CGM; in emitNonContiguousDescriptor()
9214 ASTContext &C = CGF.getContext(); in emitNonContiguousDescriptor()
9237 Address DimsAddr = CGF.CreateMemTemp(ArrayTy, "dims"); in emitNonContiguousDescriptor()
9240 LValue DimsLVal = CGF.MakeAddrLValue( in emitNonContiguousDescriptor()
9241 CGF.Builder.CreateConstArrayGEP(DimsAddr, II), DimTy); in emitNonContiguousDescriptor()
9243 LValue OffsetLVal = CGF.EmitLValueForField( in emitNonContiguousDescriptor()
9245 CGF.EmitStoreOfScalar(NonContigInfo.Offsets[L][RevIdx], OffsetLVal); in emitNonContiguousDescriptor()
9247 LValue CountLVal = CGF.EmitLValueForField( in emitNonContiguousDescriptor()
9249 CGF.EmitStoreOfScalar(NonContigInfo.Counts[L][RevIdx], CountLVal); in emitNonContiguousDescriptor()
9251 LValue StrideLVal = CGF.EmitLValueForField( in emitNonContiguousDescriptor()
9253 CGF.EmitStoreOfScalar(NonContigInfo.Strides[L][RevIdx], StrideLVal); in emitNonContiguousDescriptor()
9256 Address DAddr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitNonContiguousDescriptor()
9258 llvm::Value *P = CGF.Builder.CreateConstInBoundsGEP2_32( in emitNonContiguousDescriptor()
9261 Address PAddr(P, CGF.getPointerAlign()); in emitNonContiguousDescriptor()
9262 CGF.Builder.CreateStore(DAddr.getPointer(), PAddr); in emitNonContiguousDescriptor()
9270 emitMappingInformation(CodeGenFunction &CGF, llvm::OpenMPIRBuilder &OMPBuilder, in emitMappingInformation() argument
9278 PrintingPolicy P(CGF.getContext().getLangOpts()); in emitMappingInformation()
9287 PresumedLoc PLoc = CGF.getContext().getSourceManager().getPresumedLoc(Loc); in emitMappingInformation()
9301 CodeGenFunction &CGF, MappableExprsHandler::MapCombinedInfoTy &CombinedInfo, in emitOffloadingArrays() argument
9304 CodeGenModule &CGM = CGF.CGM; in emitOffloadingArrays()
9305 ASTContext &Ctx = CGF.getContext(); in emitOffloadingArrays()
9327 CGF.CreateMemTemp(PointerArrayType, ".offload_baseptrs").getPointer(); in emitOffloadingArrays()
9329 CGF.CreateMemTemp(PointerArrayType, ".offload_ptrs").getPointer(); in emitOffloadingArrays()
9331 CGF.CreateMemTemp(PointerArrayType, ".offload_mappers"); in emitOffloadingArrays()
9344 CGF.CreateMemTemp(SizeArrayType, ".offload_sizes").getPointer(); in emitOffloadingArrays()
9353 CGF.Int64Ty, CombinedInfo.NonContigInfo.Dims[I])); in emitOffloadingArrays()
9384 llvm::Type::getInt8Ty(CGF.Builder.getContext())->getPointerTo()); in emitOffloadingArrays()
9387 return emitMappingInformation(CGF, OMPBuilder, MapExpr); in emitOffloadingArrays()
9417 llvm::Value *BP = CGF.Builder.CreateConstInBoundsGEP2_32( in emitOffloadingArrays()
9420 BP = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitOffloadingArrays()
9423 CGF.Builder.CreateStore(BPVal, BPAddr); in emitOffloadingArrays()
9431 llvm::Value *P = CGF.Builder.CreateConstInBoundsGEP2_32( in emitOffloadingArrays()
9434 P = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitOffloadingArrays()
9437 CGF.Builder.CreateStore(PVal, PAddr); in emitOffloadingArrays()
9440 llvm::Value *S = CGF.Builder.CreateConstInBoundsGEP2_32( in emitOffloadingArrays()
9446 CGF.Builder.CreateStore(CGF.Builder.CreateIntCast(CombinedInfo.Sizes[I], in emitOffloadingArrays()
9457 MFunc = CGF.Builder.CreatePointerCast(MFunc, CGM.VoidPtrTy); in emitOffloadingArrays()
9460 Address MAddr = CGF.Builder.CreateConstArrayGEP(MappersArray, I); in emitOffloadingArrays()
9461 CGF.Builder.CreateStore(MFunc, MAddr); in emitOffloadingArrays()
9469 emitNonContiguousDescriptor(CGF, CombinedInfo, Info); in emitOffloadingArrays()
9486 CodeGenFunction &CGF, llvm::Value *&BasePointersArrayArg, in emitOffloadingArraysArgument() argument
9493 CodeGenModule &CGM = CGF.CGM; in emitOffloadingArraysArgument()
9495 BasePointersArrayArg = CGF.Builder.CreateConstInBoundsGEP2_32( in emitOffloadingArraysArgument()
9499 PointersArrayArg = CGF.Builder.CreateConstInBoundsGEP2_32( in emitOffloadingArraysArgument()
9504 SizesArrayArg = CGF.Builder.CreateConstInBoundsGEP2_32( in emitOffloadingArraysArgument()
9507 MapTypesArrayArg = CGF.Builder.CreateConstInBoundsGEP2_32( in emitOffloadingArraysArgument()
9516 if (CGF.CGM.getCodeGenOpts().getDebugInfo() == codegenoptions::NoDebugInfo) in emitOffloadingArraysArgument()
9519 MapNamesArrayArg = CGF.Builder.CreateConstInBoundsGEP2_32( in emitOffloadingArraysArgument()
9530 CGF.Builder.CreatePointerCast(Info.MappersArray, CGM.VoidPtrPtrTy); in emitOffloadingArraysArgument()
9683 CodeGenFunction *CGF) { in emitUserDefinedMapper() argument
9923 if (CGF) { in emitUserDefinedMapper()
9924 auto &Decls = FunctionUDMMap.FindAndConstruct(CGF->CurFn); in emitUserDefinedMapper()
10008 CodeGenFunction &CGF, const OMPExecutableDirective &D, in emitTargetNumIterationsCall() argument
10010 llvm::function_ref<llvm::Value *(CodeGenFunction &CGF, in emitTargetNumIterationsCall() argument
10021 auto &&CodeGen = [LD, DeviceID, SizeEmitter, &D, this](CodeGenFunction &CGF, in emitTargetNumIterationsCall()
10023 if (llvm::Value *NumIterations = SizeEmitter(CGF, *LD)) { in emitTargetNumIterationsCall()
10024 llvm::Value *RTLoc = emitUpdateLocation(CGF, D.getBeginLoc()); in emitTargetNumIterationsCall()
10026 CGF.EmitRuntimeCall( in emitTargetNumIterationsCall()
10032 emitInlinedDirective(CGF, OMPD_unknown, CodeGen); in emitTargetNumIterationsCall()
10036 CodeGenFunction &CGF, const OMPExecutableDirective &D, in emitTargetCall() argument
10039 llvm::function_ref<llvm::Value *(CodeGenFunction &CGF, in emitTargetCall() argument
10042 if (!CGF.HaveInsertPoint()) in emitTargetCall()
10051 auto &&ArgsCodegen = [&CS, &CapturedVars](CodeGenFunction &CGF, in emitTargetCall()
10053 CGF.GenerateOpenMPCapturedVars(CS, CapturedVars); in emitTargetCall()
10055 emitInlinedDirective(CGF, OMPD_unknown, ArgsCodegen); in emitTargetCall()
10064 SizeEmitter](CodeGenFunction &CGF, PrePostActionTy &) { in emitTargetCall() argument
10069 CGF.GenerateOpenMPCapturedVars(CS, CapturedVars); in emitTargetCall()
10071 emitOutlinedFunctionCall(CGF, D.getBeginLoc(), OutlinedFn, CapturedVars); in emitTargetCall()
10093 llvm::Value *DevVal = CGF.EmitScalarExpr(Device.getPointer()); in emitTargetCall()
10095 CGF.Builder.CreateIntCast(DevVal, CGF.Int64Ty, /*isSigned=*/true); in emitTargetCall()
10097 DeviceID = CGF.Builder.getInt64(OMP_DEVICEID_UNDEF); in emitTargetCall()
10102 CGF.Builder.getInt32(InputInfo.NumberOfTargetItems); in emitTargetCall()
10107 llvm::Value *NumTeams = emitNumTeamsForTargetDirective(CGF, D); in emitTargetCall()
10108 llvm::Value *NumThreads = emitNumThreadsForTargetDirective(CGF, D); in emitTargetCall()
10111 llvm::Value *RTLoc = emitUpdateLocation(CGF, D.getBeginLoc()); in emitTargetCall()
10114 emitTargetNumIterationsCall(CGF, D, DeviceID, SizeEmitter); in emitTargetCall()
10164 Return = CGF.EmitRuntimeCall( in emitTargetCall()
10181 Return = CGF.EmitRuntimeCall( in emitTargetCall()
10190 CGF.createBasicBlock("omp_offload.failed"); in emitTargetCall()
10192 CGF.createBasicBlock("omp_offload.cont"); in emitTargetCall()
10193 llvm::Value *Failed = CGF.Builder.CreateIsNotNull(Return); in emitTargetCall()
10194 CGF.Builder.CreateCondBr(Failed, OffloadFailedBlock, OffloadContBlock); in emitTargetCall()
10196 CGF.EmitBlock(OffloadFailedBlock); in emitTargetCall()
10199 CGF.GenerateOpenMPCapturedVars(CS, CapturedVars); in emitTargetCall()
10201 emitOutlinedFunctionCall(CGF, D.getBeginLoc(), OutlinedFn, CapturedVars); in emitTargetCall()
10202 CGF.EmitBranch(OffloadContBlock); in emitTargetCall()
10204 CGF.EmitBlock(OffloadContBlock, /*IsFinished=*/true); in emitTargetCall()
10209 RequiresOuterTask](CodeGenFunction &CGF, in emitTargetCall()
10213 CGF.GenerateOpenMPCapturedVars(CS, CapturedVars); in emitTargetCall()
10215 emitOutlinedFunctionCall(CGF, D.getBeginLoc(), OutlinedFn, CapturedVars); in emitTargetCall()
10220 &CS](CodeGenFunction &CGF, PrePostActionTy &) { in emitTargetCall() argument
10225 MappableExprsHandler MEHandler(D, CGF); in emitTargetCall()
10243 CurInfo.Sizes.push_back(CGF.Builder.CreateIntCast( in emitTargetCall()
10244 CGF.getTypeSize(RI->getType()), CGF.Int64Ty, /*isSigned=*/true)); in emitTargetCall()
10297 emitOffloadingArrays(CGF, CombinedInfo, Info, OMPBuilder); in emitTargetCall()
10299 CGF, Info.BasePointersArray, Info.PointersArray, Info.SizesArray, in emitTargetCall()
10313 CGF.EmitOMPTargetTaskBasedDirective(D, ThenGen, InputInfo); in emitTargetCall()
10315 emitInlinedDirective(CGF, D.getDirectiveKind(), ThenGen); in emitTargetCall()
10319 CodeGenFunction &CGF, PrePostActionTy &) { in emitTargetCall() argument
10322 CGF.EmitOMPTargetTaskBasedDirective(D, ElseGen, InputInfo); in emitTargetCall()
10324 emitInlinedDirective(CGF, D.getDirectiveKind(), ElseGen); in emitTargetCall()
10334 emitIfClause(CGF, IfCond, TargetThenGen, TargetElseGen); in emitTargetCall()
10337 ThenRCG(CGF); in emitTargetCall()
10341 ElseRCG(CGF); in emitTargetCall()
10573 CGOpenMPRuntime::registerTargetFirstprivateCopy(CodeGenFunction &CGF, in registerTargetFirstprivateCopy() argument
10716 CodeGenFunction &CGF, const OMPExecutableDirective &D) const { in adjustTargetSpecificDataForLambdas() argument
10826 CodeGenFunction CGF(CGM); in emitRequiresDirectiveRegFun() local
10831 CGF.StartFunction(GlobalDecl(), C.VoidTy, RequiresRegFn, FI, {}); in emitRequiresDirectiveRegFun()
10845 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in emitRequiresDirectiveRegFun()
10848 CGF.FinishFunction(); in emitRequiresDirectiveRegFun()
10853 void CGOpenMPRuntime::emitTeamsCall(CodeGenFunction &CGF, in emitTeamsCall() argument
10858 if (!CGF.HaveInsertPoint()) in emitTeamsCall()
10861 llvm::Value *RTLoc = emitUpdateLocation(CGF, Loc); in emitTeamsCall()
10862 CodeGenFunction::RunCleanupsScope Scope(CGF); in emitTeamsCall()
10867 CGF.Builder.getInt32(CapturedVars.size()), // Number of captured vars in emitTeamsCall()
10868 CGF.Builder.CreateBitCast(OutlinedFn, getKmpc_MicroPointerTy())}; in emitTeamsCall()
10875 CGF.EmitRuntimeCall(RTLFn, RealArgs); in emitTeamsCall()
10878 void CGOpenMPRuntime::emitNumTeamsClause(CodeGenFunction &CGF, in emitNumTeamsClause() argument
10882 if (!CGF.HaveInsertPoint()) in emitNumTeamsClause()
10885 llvm::Value *RTLoc = emitUpdateLocation(CGF, Loc); in emitNumTeamsClause()
10889 ? CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(NumTeams), in emitNumTeamsClause()
10890 CGF.CGM.Int32Ty, /* isSigned = */ true) in emitNumTeamsClause()
10891 : CGF.Builder.getInt32(0); in emitNumTeamsClause()
10895 ? CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(ThreadLimit), in emitNumTeamsClause()
10896 CGF.CGM.Int32Ty, /* isSigned = */ true) in emitNumTeamsClause()
10897 : CGF.Builder.getInt32(0); in emitNumTeamsClause()
10900 llvm::Value *PushNumTeamsArgs[] = {RTLoc, getThreadID(CGF, Loc), NumTeamsVal, in emitNumTeamsClause()
10902 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in emitNumTeamsClause()
10908 CodeGenFunction &CGF, const OMPExecutableDirective &D, const Expr *IfCond, in emitTargetDataCalls() argument
10910 if (!CGF.HaveInsertPoint()) in emitTargetDataCalls()
10921 &CodeGen](CodeGenFunction &CGF, PrePostActionTy &) { in emitTargetDataCalls() argument
10926 MappableExprsHandler MEHandler(D, CGF); in emitTargetDataCalls()
10930 emitOffloadingArrays(CGF, CombinedInfo, Info, OMPBuilder, in emitTargetDataCalls()
10939 emitOffloadingArraysArgument(CGF, BasePointersArrayArg, PointersArrayArg, in emitTargetDataCalls()
10946 DeviceID = CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(Device), in emitTargetDataCalls()
10947 CGF.Int64Ty, /*isSigned=*/true); in emitTargetDataCalls()
10949 DeviceID = CGF.Builder.getInt64(OMP_DEVICEID_UNDEF); in emitTargetDataCalls()
10953 llvm::Value *PointerNum = CGF.Builder.getInt32(Info.NumberOfPtrs); in emitTargetDataCalls()
10956 llvm::Value *RTLoc = emitUpdateLocation(CGF, D.getBeginLoc()); in emitTargetDataCalls()
10967 CGF.EmitRuntimeCall( in emitTargetDataCalls()
10975 CodeGen(CGF); in emitTargetDataCalls()
10979 auto &&EndThenGen = [this, Device, &Info, &D](CodeGenFunction &CGF, in emitTargetDataCalls()
10989 emitOffloadingArraysArgument(CGF, BasePointersArrayArg, PointersArrayArg, in emitTargetDataCalls()
10997 DeviceID = CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(Device), in emitTargetDataCalls()
10998 CGF.Int64Ty, /*isSigned=*/true); in emitTargetDataCalls()
11000 DeviceID = CGF.Builder.getInt64(OMP_DEVICEID_UNDEF); in emitTargetDataCalls()
11004 llvm::Value *PointerNum = CGF.Builder.getInt32(Info.NumberOfPtrs); in emitTargetDataCalls()
11007 llvm::Value *RTLoc = emitUpdateLocation(CGF, D.getBeginLoc()); in emitTargetDataCalls()
11018 CGF.EmitRuntimeCall( in emitTargetDataCalls()
11027 auto &&BeginElseGen = [&Info, &CodeGen, &NoPrivAction](CodeGenFunction &CGF, in emitTargetDataCalls()
11031 CodeGen(CGF); in emitTargetDataCalls()
11037 auto &&EndElseGen = [](CodeGenFunction &CGF, PrePostActionTy &) {}; in emitTargetDataCalls() argument
11040 emitIfClause(CGF, IfCond, BeginThenGen, BeginElseGen); in emitTargetDataCalls()
11043 RCG(CGF); in emitTargetDataCalls()
11050 CodeGen(CGF); in emitTargetDataCalls()
11054 emitIfClause(CGF, IfCond, EndThenGen, EndElseGen); in emitTargetDataCalls()
11057 RCG(CGF); in emitTargetDataCalls()
11062 CodeGenFunction &CGF, const OMPExecutableDirective &D, const Expr *IfCond, in emitTargetDataStandAloneCall() argument
11064 if (!CGF.HaveInsertPoint()) in emitTargetDataStandAloneCall()
11077 &MapNamesArray](CodeGenFunction &CGF, PrePostActionTy &) { in emitTargetDataStandAloneCall() argument
11081 DeviceID = CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(Device), in emitTargetDataStandAloneCall()
11082 CGF.Int64Ty, /*isSigned=*/true); in emitTargetDataStandAloneCall()
11084 DeviceID = CGF.Builder.getInt64(OMP_DEVICEID_UNDEF); in emitTargetDataStandAloneCall()
11089 CGF.Builder.getInt32(InputInfo.NumberOfTargetItems); in emitTargetDataStandAloneCall()
11092 llvm::Value *RTLoc = emitUpdateLocation(CGF, D.getBeginLoc()); in emitTargetDataStandAloneCall()
11189 CGF.EmitRuntimeCall( in emitTargetDataStandAloneCall()
11195 &MapNamesArray](CodeGenFunction &CGF, in emitTargetDataStandAloneCall()
11201 MappableExprsHandler MEHandler(D, CGF); in emitTargetDataStandAloneCall()
11206 emitOffloadingArrays(CGF, CombinedInfo, Info, OMPBuilder, in emitTargetDataStandAloneCall()
11211 CGF, Info.BasePointersArray, Info.PointersArray, Info.SizesArray, in emitTargetDataStandAloneCall()
11225 CGF.EmitOMPTargetTaskBasedDirective(D, ThenGen, InputInfo); in emitTargetDataStandAloneCall()
11227 emitInlinedDirective(CGF, D.getDirectiveKind(), ThenGen); in emitTargetDataStandAloneCall()
11231 emitIfClause(CGF, IfCond, TargetThenGen, in emitTargetDataStandAloneCall()
11232 [](CodeGenFunction &CGF, PrePostActionTy &) {}); in emitTargetDataStandAloneCall() argument
11235 ThenRCG(CGF); in emitTargetDataStandAloneCall()
11818 void Emit(CodeGenFunction &CGF, Flags /*flags*/) override { in Emit() argument
11819 if (!CGF.HaveInsertPoint()) in Emit()
11821 CGF.EmitRuntimeCall(RTLFn, Args); in Emit()
11826 void CGOpenMPRuntime::emitDoacrossInit(CodeGenFunction &CGF, in emitDoacrossInit() argument
11829 if (!CGF.HaveInsertPoint()) in emitDoacrossInit()
11855 Address DimsAddr = CGF.CreateMemTemp(ArrayTy, "dims"); in emitDoacrossInit()
11856 CGF.EmitNullInitialization(DimsAddr, ArrayTy); in emitDoacrossInit()
11860 LValue DimsLVal = CGF.MakeAddrLValue( in emitDoacrossInit()
11861 CGF.Builder.CreateConstArrayGEP(DimsAddr, I), KmpDimTy); in emitDoacrossInit()
11863 LValue UpperLVal = CGF.EmitLValueForField( in emitDoacrossInit()
11865 llvm::Value *NumIterVal = CGF.EmitScalarConversion( in emitDoacrossInit()
11866 CGF.EmitScalarExpr(NumIterations[I]), NumIterations[I]->getType(), in emitDoacrossInit()
11868 CGF.EmitStoreOfScalar(NumIterVal, UpperLVal); in emitDoacrossInit()
11870 LValue StrideLVal = CGF.EmitLValueForField( in emitDoacrossInit()
11872 CGF.EmitStoreOfScalar(llvm::ConstantInt::getSigned(CGM.Int64Ty, /*V=*/1), in emitDoacrossInit()
11879 emitUpdateLocation(CGF, D.getBeginLoc()), in emitDoacrossInit()
11880 getThreadID(CGF, D.getBeginLoc()), in emitDoacrossInit()
11882 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitDoacrossInit()
11883 CGF.Builder.CreateConstArrayGEP(DimsAddr, 0).getPointer(), in emitDoacrossInit()
11888 CGF.EmitRuntimeCall(RTLFn, Args); in emitDoacrossInit()
11890 emitUpdateLocation(CGF, D.getEndLoc()), getThreadID(CGF, D.getEndLoc())}; in emitDoacrossInit()
11893 CGF.EHStack.pushCleanup<DoacrossCleanupTy>(NormalAndEHCleanup, FiniRTLFn, in emitDoacrossInit()
11897 void CGOpenMPRuntime::emitDoacrossOrdered(CodeGenFunction &CGF, in emitDoacrossOrdered() argument
11904 Address CntAddr = CGF.CreateMemTemp(ArrayTy, ".cnt.addr"); in emitDoacrossOrdered()
11908 llvm::Value *CntVal = CGF.EmitScalarConversion( in emitDoacrossOrdered()
11909 CGF.EmitScalarExpr(CounterVal), CounterVal->getType(), Int64Ty, in emitDoacrossOrdered()
11911 CGF.EmitStoreOfScalar(CntVal, CGF.Builder.CreateConstArrayGEP(CntAddr, I), in emitDoacrossOrdered()
11915 emitUpdateLocation(CGF, C->getBeginLoc()), in emitDoacrossOrdered()
11916 getThreadID(CGF, C->getBeginLoc()), in emitDoacrossOrdered()
11917 CGF.Builder.CreateConstArrayGEP(CntAddr, 0).getPointer()}; in emitDoacrossOrdered()
11927 CGF.EmitRuntimeCall(RTLFn, Args); in emitDoacrossOrdered()
11930 void CGOpenMPRuntime::emitCall(CodeGenFunction &CGF, SourceLocation Loc, in emitCall() argument
11934 auto DL = ApplyDebugLocation::CreateDefaultArtificial(CGF, Loc); in emitCall()
11938 CGF.EmitNounwindRuntimeCall(Fn, Args); in emitCall()
11942 CGF.EmitRuntimeCall(Callee, Args); in emitCall()
11946 CodeGenFunction &CGF, SourceLocation Loc, llvm::FunctionCallee OutlinedFn, in emitOutlinedFunctionCall() argument
11948 emitCall(CGF, Loc, OutlinedFn, Args); in emitOutlinedFunctionCall()
11951 void CGOpenMPRuntime::emitFunctionProlog(CodeGenFunction &CGF, const Decl *D) { in emitFunctionProlog() argument
11957 Address CGOpenMPRuntime::getParameterAddress(CodeGenFunction &CGF, in getParameterAddress() argument
11960 return CGF.GetAddrOfLocalVar(NativeParam); in getParameterAddress()
11963 Address CGOpenMPRuntime::getAddressOfLocalVariable(CodeGenFunction &CGF, in getAddressOfLocalVariable() argument
11969 auto It = FunctionToUntiedTaskStackMap.find(CGF.CurFn); in getAddressOfLocalVariable()
11987 Size = CGF.getTypeSize(CVD->getType()); in getAddressOfLocalVariable()
11989 Size = CGF.Builder.CreateNUWAdd( in getAddressOfLocalVariable()
11991 Size = CGF.Builder.CreateUDiv(Size, CGM.getSize(Align)); in getAddressOfLocalVariable()
11992 Size = CGF.Builder.CreateNUWMul(Size, CGM.getSize(Align)); in getAddressOfLocalVariable()
11997 llvm::Value *ThreadID = getThreadID(CGF, CVD->getBeginLoc()); in getAddressOfLocalVariable()
12001 llvm::Value *Allocator = CGF.EmitScalarExpr(AA->getAllocator()); in getAddressOfLocalVariable()
12004 Allocator = CGF.EmitScalarConversion( in getAddressOfLocalVariable()
12005 Allocator, AA->getAllocator()->getType(), CGF.getContext().VoidPtrTy, in getAddressOfLocalVariable()
12010 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in getAddressOfLocalVariable()
12016 Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in getAddressOfLocalVariable()
12017 Addr, CGF.ConvertTypeForMem(Ty), getName({CVD->getName(), ".addr"})); in getAddressOfLocalVariable()
12019 CGF.EmitStoreOfScalar(Addr, UntiedAddr, /*Volatile=*/false, Ty); in getAddressOfLocalVariable()
12033 void Emit(CodeGenFunction &CGF, Flags /*flags*/) override { in getAddressOfLocalVariable() argument
12034 if (!CGF.HaveInsertPoint()) in getAddressOfLocalVariable()
12037 Args[0] = CGF.CGM.getOpenMPRuntime().getThreadID( in getAddressOfLocalVariable()
12038 CGF, SourceLocation::getFromRawEncoding(LocEncoding)); in getAddressOfLocalVariable()
12039 Args[1] = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in getAddressOfLocalVariable()
12040 Addr.getPointer(), CGF.VoidPtrTy); in getAddressOfLocalVariable()
12041 llvm::Value *AllocVal = CGF.EmitScalarExpr(Allocator); in getAddressOfLocalVariable()
12044 AllocVal = CGF.EmitScalarConversion(AllocVal, Allocator->getType(), in getAddressOfLocalVariable()
12045 CGF.getContext().VoidPtrTy, in getAddressOfLocalVariable()
12049 CGF.EmitRuntimeCall(RTLFn, Args); in getAddressOfLocalVariable()
12054 CGF.EHStack.pushCleanup<OMPAllocateCleanupTy>( in getAddressOfLocalVariable()
12059 dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo)) in getAddressOfLocalVariable()
12060 Region->emitUntiedSwitch(CGF); in getAddressOfLocalVariable()
12066 bool CGOpenMPRuntime::isLocalVarInUntiedTask(CodeGenFunction &CGF, in isLocalVarInUntiedTask() argument
12068 auto It = FunctionToUntiedTaskStackMap.find(CGF.CurFn); in isLocalVarInUntiedTask()
12107 CodeGenFunction &CGF, in UntiedTaskLocalDeclsRAII() argument
12110 : CGM(CGF.CGM), NeedToPush(!LocalVars.empty()) { in UntiedTaskLocalDeclsRAII()
12114 CGF.CurFn, CGM.getOpenMPRuntime().UntiedLocalVarsStack.size()); in UntiedTaskLocalDeclsRAII()
12212 CodeGenFunction &CGF, const OMPExecutableDirective &S, LValue IVLVal) in LastprivateConditionalRAII() argument
12213 : CGM(CGF.CGM), in LastprivateConditionalRAII()
12240 Data.Fn = CGF.CurFn;
12244 CodeGenFunction &CGF, const OMPExecutableDirective &S) in LastprivateConditionalRAII() argument
12245 : CGM(CGF.CGM), Action(ActionToDo::DoNotPush) { in LastprivateConditionalRAII()
12257 Data.Fn = CGF.CurFn; in LastprivateConditionalRAII()
12264 CodeGenFunction &CGF, const OMPExecutableDirective &S) { in disable() argument
12265 return LastprivateConditionalRAII(CGF, S); in disable()
12284 Address CGOpenMPRuntime::emitLastprivateConditionalInit(CodeGenFunction &CGF, in emitLastprivateConditionalInit() argument
12287 auto I = LastprivateConditionalToTypes.find(CGF.CurFn); in emitLastprivateConditionalInit()
12289 I = LastprivateConditionalToTypes.try_emplace(CGF.CurFn).first; in emitLastprivateConditionalInit()
12302 Address Addr = CGF.CreateMemTemp(NewType, C.getDeclAlign(VD), VD->getName()); in emitLastprivateConditionalInit()
12303 BaseLVal = CGF.MakeAddrLValue(Addr, NewType, AlignmentSource::Decl); in emitLastprivateConditionalInit()
12312 CGF.EmitLValueForField(BaseLVal, FiredField); in emitLastprivateConditionalInit()
12313 CGF.EmitStoreOfScalar( in emitLastprivateConditionalInit()
12314 llvm::ConstantInt::getNullValue(CGF.ConvertTypeForMem(C.CharTy)), in emitLastprivateConditionalInit()
12316 return CGF.EmitLValueForField(BaseLVal, VDField).getAddress(CGF); in emitLastprivateConditionalInit()
12390 void CGOpenMPRuntime::emitLastprivateConditionalUpdate(CodeGenFunction &CGF, in emitLastprivateConditionalUpdate() argument
12397 llvm::Type *LLIVTy = CGF.ConvertTypeForMem(IVLVal.getType()); in emitLastprivateConditionalUpdate()
12402 LValue LastIVLVal = CGF.MakeNaturalAlignAddrLValue(LastIV, IVLVal.getType()); in emitLastprivateConditionalUpdate()
12407 CGF.ConvertTypeForMem(LVal.getType()), UniqueDeclName); in emitLastprivateConditionalUpdate()
12411 CGF.MakeAddrLValue(Last, LVal.getType(), LVal.getAlignment()); in emitLastprivateConditionalUpdate()
12415 llvm::Value *IVVal = CGF.EmitLoadOfScalar(IVLVal, Loc); in emitLastprivateConditionalUpdate()
12423 Loc](CodeGenFunction &CGF, PrePostActionTy &Action) { in emitLastprivateConditionalUpdate() argument
12424 Action.Enter(CGF); in emitLastprivateConditionalUpdate()
12425 llvm::Value *LastIVVal = CGF.EmitLoadOfScalar(LastIVLVal, Loc); in emitLastprivateConditionalUpdate()
12430 CmpRes = CGF.Builder.CreateICmpSLE(LastIVVal, IVVal); in emitLastprivateConditionalUpdate()
12434 CmpRes = CGF.Builder.CreateICmpULE(LastIVVal, IVVal); in emitLastprivateConditionalUpdate()
12436 llvm::BasicBlock *ThenBB = CGF.createBasicBlock("lp_cond_then"); in emitLastprivateConditionalUpdate()
12437 llvm::BasicBlock *ExitBB = CGF.createBasicBlock("lp_cond_exit"); in emitLastprivateConditionalUpdate()
12438 CGF.Builder.CreateCondBr(CmpRes, ThenBB, ExitBB); in emitLastprivateConditionalUpdate()
12440 CGF.EmitBlock(ThenBB); in emitLastprivateConditionalUpdate()
12443 CGF.EmitStoreOfScalar(IVVal, LastIVLVal); in emitLastprivateConditionalUpdate()
12446 switch (CGF.getEvaluationKind(LVal.getType())) { in emitLastprivateConditionalUpdate()
12448 llvm::Value *PrivVal = CGF.EmitLoadOfScalar(LVal, Loc); in emitLastprivateConditionalUpdate()
12449 CGF.EmitStoreOfScalar(PrivVal, LastLVal); in emitLastprivateConditionalUpdate()
12453 CodeGenFunction::ComplexPairTy PrivVal = CGF.EmitLoadOfComplex(LVal, Loc); in emitLastprivateConditionalUpdate()
12454 CGF.EmitStoreOfComplex(PrivVal, LastLVal, /*isInit=*/false); in emitLastprivateConditionalUpdate()
12462 CGF.EmitBranch(ExitBB); in emitLastprivateConditionalUpdate()
12464 (void)ApplyDebugLocation::CreateEmpty(CGF); in emitLastprivateConditionalUpdate()
12465 CGF.EmitBlock(ExitBB, /*IsFinished=*/true); in emitLastprivateConditionalUpdate()
12471 ThenRCG(CGF); in emitLastprivateConditionalUpdate()
12473 emitCriticalRegion(CGF, UniqueDeclName, CodeGen, Loc); in emitLastprivateConditionalUpdate()
12477 void CGOpenMPRuntime::checkAndEmitLastprivateConditional(CodeGenFunction &CGF, in checkAndEmitLastprivateConditional() argument
12479 if (CGF.getLangOpts().OpenMP < 50 || LastprivateConditionalStack.empty()) in checkAndEmitLastprivateConditional()
12491 if (FoundFn != CGF.CurFn) { in checkAndEmitLastprivateConditional()
12499 LValue PrivLVal = CGF.EmitLValue(FoundE); in checkAndEmitLastprivateConditional()
12500 Address StructAddr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in checkAndEmitLastprivateConditional()
12501 PrivLVal.getAddress(CGF), in checkAndEmitLastprivateConditional()
12502 CGF.ConvertTypeForMem(CGF.getContext().getPointerType(StructTy))); in checkAndEmitLastprivateConditional()
12504 CGF.MakeAddrLValue(StructAddr, StructTy, AlignmentSource::Decl); in checkAndEmitLastprivateConditional()
12505 LValue FiredLVal = CGF.EmitLValueForField(BaseLVal, FiredDecl); in checkAndEmitLastprivateConditional()
12506 CGF.EmitAtomicStore(RValue::get(llvm::ConstantInt::get( in checkAndEmitLastprivateConditional()
12507 CGF.ConvertTypeForMem(FiredDecl->getType()), 1)), in checkAndEmitLastprivateConditional()
12515 LValue LVal = CGF.EmitLValue(FoundE); in checkAndEmitLastprivateConditional()
12516 emitLastprivateConditionalUpdate(CGF, IVLVal, UniqueDeclName, LVal, in checkAndEmitLastprivateConditional()
12521 CodeGenFunction &CGF, const OMPExecutableDirective &D, in checkAndEmitSharedLastprivateConditional() argument
12523 if (CGF.getLangOpts().OpenMP < 50 || LastprivateConditionalStack.empty()) in checkAndEmitSharedLastprivateConditional()
12528 if (It == Range.end() || It->Fn != CGF.CurFn) in checkAndEmitSharedLastprivateConditional()
12546 CGF.EmitLValueForField(BaseLVal, std::get<2>(I->getSecond())); in checkAndEmitSharedLastprivateConditional()
12547 llvm::Value *Res = CGF.EmitLoadOfScalar(FiredLVal, D.getBeginLoc()); in checkAndEmitSharedLastprivateConditional()
12548 llvm::Value *Cmp = CGF.Builder.CreateIsNotNull(Res); in checkAndEmitSharedLastprivateConditional()
12549 llvm::BasicBlock *ThenBB = CGF.createBasicBlock("lpc.then"); in checkAndEmitSharedLastprivateConditional()
12550 llvm::BasicBlock *DoneBB = CGF.createBasicBlock("lpc.done"); in checkAndEmitSharedLastprivateConditional()
12552 CGF.Builder.CreateCondBr(Cmp, ThenBB, DoneBB); in checkAndEmitSharedLastprivateConditional()
12553 CGF.EmitBlock(ThenBB); in checkAndEmitSharedLastprivateConditional()
12554 Address Addr = CGF.GetAddrOfLocalVar(VD); in checkAndEmitSharedLastprivateConditional()
12557 LVal = CGF.EmitLoadOfReferenceLValue(Addr, VD->getType(), in checkAndEmitSharedLastprivateConditional()
12560 LVal = CGF.MakeAddrLValue(Addr, VD->getType().getNonReferenceType(), in checkAndEmitSharedLastprivateConditional()
12562 emitLastprivateConditionalUpdate(CGF, It->IVLVal, Pair.second, LVal, in checkAndEmitSharedLastprivateConditional()
12564 auto AL = ApplyDebugLocation::CreateArtificial(CGF); in checkAndEmitSharedLastprivateConditional()
12565 CGF.EmitBlock(DoneBB, /*IsFinal=*/true); in checkAndEmitSharedLastprivateConditional()
12571 CodeGenFunction &CGF, LValue PrivLVal, const VarDecl *VD, in emitLastprivateConditionalFinalUpdate() argument
12573 if (CGF.getLangOpts().OpenMP < 50) in emitLastprivateConditionalFinalUpdate()
12583 LValue LPLVal = CGF.MakeAddrLValue( in emitLastprivateConditionalFinalUpdate()
12585 llvm::Value *Res = CGF.EmitLoadOfScalar(LPLVal, Loc); in emitLastprivateConditionalFinalUpdate()
12586 CGF.EmitStoreOfScalar(Res, PrivLVal); in emitLastprivateConditionalFinalUpdate()
12609 void CGOpenMPSIMDRuntime::emitParallelCall(CodeGenFunction &CGF, in emitParallelCall() argument
12618 CodeGenFunction &CGF, StringRef CriticalName, in emitCriticalRegion() argument
12624 void CGOpenMPSIMDRuntime::emitMasterRegion(CodeGenFunction &CGF, in emitMasterRegion() argument
12630 void CGOpenMPSIMDRuntime::emitMaskedRegion(CodeGenFunction &CGF, in emitMaskedRegion() argument
12637 void CGOpenMPSIMDRuntime::emitTaskyieldCall(CodeGenFunction &CGF, in emitTaskyieldCall() argument
12643 CodeGenFunction &CGF, const RegionCodeGenTy &TaskgroupOpGen, in emitTaskgroupRegion() argument
12649 CodeGenFunction &CGF, const RegionCodeGenTy &SingleOpGen, in emitSingleRegion() argument
12656 void CGOpenMPSIMDRuntime::emitOrderedRegion(CodeGenFunction &CGF, in emitOrderedRegion() argument
12663 void CGOpenMPSIMDRuntime::emitBarrierCall(CodeGenFunction &CGF, in emitBarrierCall() argument
12672 CodeGenFunction &CGF, SourceLocation Loc, in emitForDispatchInit() argument
12679 CodeGenFunction &CGF, SourceLocation Loc, OpenMPDirectiveKind DKind, in emitForStaticInit() argument
12685 CodeGenFunction &CGF, SourceLocation Loc, in emitDistributeStaticInit() argument
12690 void CGOpenMPSIMDRuntime::emitForOrderedIterationEnd(CodeGenFunction &CGF, in emitForOrderedIterationEnd() argument
12697 void CGOpenMPSIMDRuntime::emitForStaticFinish(CodeGenFunction &CGF, in emitForStaticFinish() argument
12703 llvm::Value *CGOpenMPSIMDRuntime::emitForNext(CodeGenFunction &CGF, in emitForNext() argument
12711 void CGOpenMPSIMDRuntime::emitNumThreadsClause(CodeGenFunction &CGF, in emitNumThreadsClause() argument
12717 void CGOpenMPSIMDRuntime::emitProcBindClause(CodeGenFunction &CGF, in emitProcBindClause() argument
12723 Address CGOpenMPSIMDRuntime::getAddrOfThreadPrivate(CodeGenFunction &CGF, in getAddrOfThreadPrivate() argument
12732 CodeGenFunction *CGF) { in emitThreadPrivateVarDefinition() argument
12737 CodeGenFunction &CGF, QualType VarType, StringRef Name) { in getAddrOfArtificialThreadPrivate() argument
12741 void CGOpenMPSIMDRuntime::emitFlush(CodeGenFunction &CGF, in emitFlush() argument
12748 void CGOpenMPSIMDRuntime::emitTaskCall(CodeGenFunction &CGF, SourceLocation Loc, in emitTaskCall() argument
12758 CodeGenFunction &CGF, SourceLocation Loc, const OMPLoopDirective &D, in emitTaskLoopCall() argument
12765 CodeGenFunction &CGF, SourceLocation Loc, ArrayRef<const Expr *> Privates, in emitReduction() argument
12769 CGOpenMPRuntime::emitReduction(CGF, Loc, Privates, LHSExprs, RHSExprs, in emitReduction()
12774 CodeGenFunction &CGF, SourceLocation Loc, ArrayRef<const Expr *> LHSExprs, in emitTaskReductionInit() argument
12779 void CGOpenMPSIMDRuntime::emitTaskReductionFini(CodeGenFunction &CGF, in emitTaskReductionFini() argument
12785 void CGOpenMPSIMDRuntime::emitTaskReductionFixups(CodeGenFunction &CGF, in emitTaskReductionFixups() argument
12792 Address CGOpenMPSIMDRuntime::getTaskReductionItem(CodeGenFunction &CGF, in getTaskReductionItem() argument
12799 void CGOpenMPSIMDRuntime::emitTaskwaitCall(CodeGenFunction &CGF, in emitTaskwaitCall() argument
12805 CodeGenFunction &CGF, SourceLocation Loc, in emitCancellationPointCall() argument
12810 void CGOpenMPSIMDRuntime::emitCancelCall(CodeGenFunction &CGF, in emitCancelCall() argument
12824 CodeGenFunction &CGF, const OMPExecutableDirective &D, in emitTargetCall() argument
12827 llvm::function_ref<llvm::Value *(CodeGenFunction &CGF, in emitTargetCall() argument
12845 void CGOpenMPSIMDRuntime::emitTeamsCall(CodeGenFunction &CGF, in emitTeamsCall() argument
12853 void CGOpenMPSIMDRuntime::emitNumTeamsClause(CodeGenFunction &CGF, in emitNumTeamsClause() argument
12861 CodeGenFunction &CGF, const OMPExecutableDirective &D, const Expr *IfCond, in emitTargetDataCalls() argument
12867 CodeGenFunction &CGF, const OMPExecutableDirective &D, const Expr *IfCond, in emitTargetDataStandAloneCall() argument
12872 void CGOpenMPSIMDRuntime::emitDoacrossInit(CodeGenFunction &CGF, in emitDoacrossInit() argument
12878 void CGOpenMPSIMDRuntime::emitDoacrossOrdered(CodeGenFunction &CGF, in emitDoacrossOrdered() argument
12890 CGOpenMPSIMDRuntime::getParameterAddress(CodeGenFunction &CGF, in getParameterAddress() argument