Lines Matching refs:CGF
87 void EmitBody(CodeGenFunction &CGF, const Stmt *S) override;
91 virtual LValue getThreadIDVariableLValue(CodeGenFunction &CGF);
160 void Enter(CodeGenFunction &CGF) override { in Enter() argument
163 LValue PartIdLVal = CGF.EmitLoadOfPointerLValue( in Enter()
164 CGF.GetAddrOfLocalVar(PartIDVar), in Enter()
167 CGF.EmitLoadOfScalar(PartIdLVal, PartIDVar->getLocation()); in Enter()
168 llvm::BasicBlock *DoneBB = CGF.createBasicBlock(".untied.done."); in Enter()
169 UntiedSwitch = CGF.Builder.CreateSwitch(Res, DoneBB); in Enter()
170 CGF.EmitBlock(DoneBB); in Enter()
171 CGF.EmitBranchThroughCleanup(CGF.ReturnBlock); in Enter()
172 CGF.EmitBlock(CGF.createBasicBlock(".untied.jmp.")); in Enter()
173 UntiedSwitch->addCase(CGF.Builder.getInt32(0), in Enter()
174 CGF.Builder.GetInsertBlock()); in Enter()
175 emitUntiedSwitch(CGF); in Enter()
178 void emitUntiedSwitch(CodeGenFunction &CGF) const { in emitUntiedSwitch()
180 LValue PartIdLVal = CGF.EmitLoadOfPointerLValue( in emitUntiedSwitch()
181 CGF.GetAddrOfLocalVar(PartIDVar), in emitUntiedSwitch()
183 CGF.EmitStoreOfScalar(CGF.Builder.getInt32(UntiedSwitch->getNumCases()), in emitUntiedSwitch()
185 UntiedCodeGen(CGF); in emitUntiedSwitch()
187 CGF.getJumpDestInCurrentScope(".untied.next."); in emitUntiedSwitch()
188 CGF.EmitBranch(CGF.ReturnBlock.getBlock()); in emitUntiedSwitch()
189 CGF.EmitBlock(CGF.createBasicBlock(".untied.jmp.")); in emitUntiedSwitch()
190 UntiedSwitch->addCase(CGF.Builder.getInt32(UntiedSwitch->getNumCases()), in emitUntiedSwitch()
191 CGF.Builder.GetInsertBlock()); in emitUntiedSwitch()
192 CGF.EmitBranchThroughCleanup(CurPoint); in emitUntiedSwitch()
193 CGF.EmitBlock(CurPoint.getBlock()); in emitUntiedSwitch()
213 LValue getThreadIDVariableLValue(CodeGenFunction &CGF) override;
218 void emitUntiedSwitch(CodeGenFunction &CGF) override { in emitUntiedSwitch() argument
219 Action.emitUntiedSwitch(CGF); in emitUntiedSwitch()
286 LValue getThreadIDVariableLValue(CodeGenFunction &CGF) override { in getThreadIDVariableLValue() argument
288 return OuterRegionInfo->getThreadIDVariableLValue(CGF); in getThreadIDVariableLValue()
299 void emitUntiedSwitch(CodeGenFunction &CGF) override { in emitUntiedSwitch() argument
301 OuterRegionInfo->emitUntiedSwitch(CGF); in emitUntiedSwitch()
355 CGOpenMPInnerExprInfo(CodeGenFunction &CGF, const CapturedStmt &CS) in CGOpenMPInnerExprInfo() argument
356 : CGOpenMPInlinedRegionInfo(CGF.CapturedStmtInfo, EmptyCodeGen, in CGOpenMPInnerExprInfo()
359 PrivScope(CGF) { in CGOpenMPInnerExprInfo()
371 DeclRefExpr DRE(CGF.getContext(), const_cast<VarDecl *>(VD), in CGOpenMPInnerExprInfo()
375 PrivScope.addPrivate(VD, CGF.EmitLValue(&DRE).getAddress(CGF)); in CGOpenMPInnerExprInfo()
388 void EmitBody(CodeGenFunction &CGF, const Stmt *S) override { in EmitBody() argument
412 CodeGenFunction &CGF; member in __anonffc6c7120111::InlinedOpenMPRegionRAII
423 InlinedOpenMPRegionRAII(CodeGenFunction &CGF, const RegionCodeGenTy &CodeGen, in InlinedOpenMPRegionRAII() argument
426 : CGF(CGF), NoInheritance(NoInheritance) { in InlinedOpenMPRegionRAII()
428 CGF.CapturedStmtInfo = new CGOpenMPInlinedRegionInfo( in InlinedOpenMPRegionRAII()
429 CGF.CapturedStmtInfo, CodeGen, Kind, HasCancel); in InlinedOpenMPRegionRAII()
431 std::swap(CGF.LambdaCaptureFields, LambdaCaptureFields); in InlinedOpenMPRegionRAII()
432 LambdaThisCaptureField = CGF.LambdaThisCaptureField; in InlinedOpenMPRegionRAII()
433 CGF.LambdaThisCaptureField = nullptr; in InlinedOpenMPRegionRAII()
434 BlockInfo = CGF.BlockInfo; in InlinedOpenMPRegionRAII()
435 CGF.BlockInfo = nullptr; in InlinedOpenMPRegionRAII()
442 cast<CGOpenMPInlinedRegionInfo>(CGF.CapturedStmtInfo)->getOldCSI(); in ~InlinedOpenMPRegionRAII()
443 delete CGF.CapturedStmtInfo; in ~InlinedOpenMPRegionRAII()
444 CGF.CapturedStmtInfo = OldCSI; in ~InlinedOpenMPRegionRAII()
446 std::swap(CGF.LambdaCaptureFields, LambdaCaptureFields); in ~InlinedOpenMPRegionRAII()
447 CGF.LambdaThisCaptureField = LambdaThisCaptureField; in ~InlinedOpenMPRegionRAII()
448 CGF.BlockInfo = BlockInfo; in ~InlinedOpenMPRegionRAII()
588 void Emit(CodeGenFunction &CGF, Flags /*flags*/) override { in Emit() argument
589 if (!CGF.HaveInsertPoint()) in Emit()
591 Action->Exit(CGF); in Emit()
597 void RegionCodeGenTy::operator()(CodeGenFunction &CGF) const { in operator ()()
598 CodeGenFunction::RunCleanupsScope Scope(CGF); in operator ()()
600 CGF.EHStack.pushCleanup<CleanupTy>(NormalAndEHCleanup, PrePostAction); in operator ()()
601 Callback(CodeGen, CGF, *PrePostAction); in operator ()()
604 Callback(CodeGen, CGF, Action); in operator ()()
621 static void emitInitWithReductionInitializer(CodeGenFunction &CGF, in emitInitWithReductionInitializer() argument
628 CGF.CGM.getOpenMPRuntime().getUserDefinedReduction(DRD); in emitInitWithReductionInitializer()
637 CodeGenFunction::OMPPrivateScope PrivateScope(CGF); in emitInitWithReductionInitializer()
642 CodeGenFunction::OpaqueValueMapping Map(CGF, OVE, Func); in emitInitWithReductionInitializer()
643 CGF.EmitIgnoredExpr(InitOp); in emitInitWithReductionInitializer()
645 llvm::Constant *Init = CGF.CGM.EmitNullConstant(Ty); in emitInitWithReductionInitializer()
646 std::string Name = CGF.CGM.getOpenMPRuntime().getName({"init"}); in emitInitWithReductionInitializer()
648 CGF.CGM.getModule(), Init->getType(), /*isConstant=*/true, in emitInitWithReductionInitializer()
650 LValue LV = CGF.MakeNaturalAlignAddrLValue(GV, Ty); in emitInitWithReductionInitializer()
652 switch (CGF.getEvaluationKind(Ty)) { in emitInitWithReductionInitializer()
654 InitRVal = CGF.EmitLoadOfLValue(LV, DRD->getLocation()); in emitInitWithReductionInitializer()
658 RValue::getComplex(CGF.EmitLoadOfComplex(LV, DRD->getLocation())); in emitInitWithReductionInitializer()
662 CodeGenFunction::OpaqueValueMapping OpaqueMap(CGF, &OVE, LV); in emitInitWithReductionInitializer()
663 CGF.EmitAnyExprToMem(&OVE, Private, Ty.getQualifiers(), in emitInitWithReductionInitializer()
669 CodeGenFunction::OpaqueValueMapping OpaqueMap(CGF, &OVE, InitRVal); in emitInitWithReductionInitializer()
670 CGF.EmitAnyExprToMem(&OVE, Private, Ty.getQualifiers(), in emitInitWithReductionInitializer()
680 static void EmitOMPAggregateInit(CodeGenFunction &CGF, Address DestAddr, in EmitOMPAggregateInit() argument
690 llvm::Value *NumElements = CGF.emitArrayLength(ArrayTy, ElementTy, DestAddr); in EmitOMPAggregateInit()
693 CGF.Builder.CreateElementBitCast(SrcAddr, DestAddr.getElementType()); in EmitOMPAggregateInit()
701 CGF.Builder.CreateGEP(DestAddr.getElementType(), DestBegin, NumElements); in EmitOMPAggregateInit()
703 llvm::BasicBlock *BodyBB = CGF.createBasicBlock("omp.arrayinit.body"); in EmitOMPAggregateInit()
704 llvm::BasicBlock *DoneBB = CGF.createBasicBlock("omp.arrayinit.done"); in EmitOMPAggregateInit()
706 CGF.Builder.CreateICmpEQ(DestBegin, DestEnd, "omp.arrayinit.isempty"); in EmitOMPAggregateInit()
707 CGF.Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB); in EmitOMPAggregateInit()
710 llvm::BasicBlock *EntryBB = CGF.Builder.GetInsertBlock(); in EmitOMPAggregateInit()
711 CGF.EmitBlock(BodyBB); in EmitOMPAggregateInit()
713 CharUnits ElementSize = CGF.getContext().getTypeSizeInChars(ElementTy); in EmitOMPAggregateInit()
718 SrcElementPHI = CGF.Builder.CreatePHI(SrcBegin->getType(), 2, in EmitOMPAggregateInit()
725 llvm::PHINode *DestElementPHI = CGF.Builder.CreatePHI( in EmitOMPAggregateInit()
734 CodeGenFunction::RunCleanupsScope InitScope(CGF); in EmitOMPAggregateInit()
736 emitInitWithReductionInitializer(CGF, DRD, Init, DestElementCurrent, in EmitOMPAggregateInit()
739 CGF.EmitAnyExprToMem(Init, DestElementCurrent, ElementTy.getQualifiers(), in EmitOMPAggregateInit()
745 llvm::Value *SrcElementNext = CGF.Builder.CreateConstGEP1_32( in EmitOMPAggregateInit()
748 SrcElementPHI->addIncoming(SrcElementNext, CGF.Builder.GetInsertBlock()); in EmitOMPAggregateInit()
752 llvm::Value *DestElementNext = CGF.Builder.CreateConstGEP1_32( in EmitOMPAggregateInit()
757 CGF.Builder.CreateICmpEQ(DestElementNext, DestEnd, "omp.arraycpy.done"); in EmitOMPAggregateInit()
758 CGF.Builder.CreateCondBr(Done, DoneBB, BodyBB); in EmitOMPAggregateInit()
759 DestElementPHI->addIncoming(DestElementNext, CGF.Builder.GetInsertBlock()); in EmitOMPAggregateInit()
762 CGF.EmitBlock(DoneBB, /*IsFinished=*/true); in EmitOMPAggregateInit()
765 LValue ReductionCodeGen::emitSharedLValue(CodeGenFunction &CGF, const Expr *E) { in emitSharedLValue() argument
766 return CGF.EmitOMPSharedLValue(E); in emitSharedLValue()
769 LValue ReductionCodeGen::emitSharedLValueUB(CodeGenFunction &CGF, in emitSharedLValueUB() argument
772 return CGF.EmitOMPArraySectionExpr(OASE, /*IsLowerBound=*/false); in emitSharedLValueUB()
777 CodeGenFunction &CGF, unsigned N, Address PrivateAddr, Address SharedAddr, in emitAggregateInitialization() argument
786 EmitOMPAggregateInit(CGF, PrivateAddr, PrivateVD->getType(), in emitAggregateInitialization()
812 void ReductionCodeGen::emitSharedOrigLValue(CodeGenFunction &CGF, unsigned N) { in emitSharedOrigLValue() argument
815 LValue First = emitSharedLValue(CGF, ClausesData[N].Shared); in emitSharedOrigLValue()
816 LValue Second = emitSharedLValueUB(CGF, ClausesData[N].Shared); in emitSharedOrigLValue()
821 LValue First = emitSharedLValue(CGF, ClausesData[N].Ref); in emitSharedOrigLValue()
822 LValue Second = emitSharedLValueUB(CGF, ClausesData[N].Ref); in emitSharedOrigLValue()
827 void ReductionCodeGen::emitAggregateType(CodeGenFunction &CGF, unsigned N) { in emitAggregateType() argument
832 CGF.getTypeSize(OrigAddresses[N].first.getType().getNonReferenceType()), in emitAggregateType()
838 auto *ElemType = OrigAddresses[N].first.getAddress(CGF).getElementType(); in emitAggregateType()
841 Size = CGF.Builder.CreatePtrDiff(ElemType, in emitAggregateType()
842 OrigAddresses[N].second.getPointer(CGF), in emitAggregateType()
843 OrigAddresses[N].first.getPointer(CGF)); in emitAggregateType()
844 Size = CGF.Builder.CreateNUWAdd( in emitAggregateType()
846 SizeInChars = CGF.Builder.CreateNUWMul(Size, ElemSizeOf); in emitAggregateType()
849 CGF.getTypeSize(OrigAddresses[N].first.getType().getNonReferenceType()); in emitAggregateType()
850 Size = CGF.Builder.CreateExactUDiv(SizeInChars, ElemSizeOf); in emitAggregateType()
854 CGF, in emitAggregateType()
856 CGF.getContext().getAsVariableArrayType(PrivateType)->getSizeExpr()), in emitAggregateType()
858 CGF.EmitVariablyModifiedType(PrivateType); in emitAggregateType()
861 void ReductionCodeGen::emitAggregateType(CodeGenFunction &CGF, unsigned N, in emitAggregateType() argument
871 CGF, in emitAggregateType()
873 CGF.getContext().getAsVariableArrayType(PrivateType)->getSizeExpr()), in emitAggregateType()
875 CGF.EmitVariablyModifiedType(PrivateType); in emitAggregateType()
879 CodeGenFunction &CGF, unsigned N, Address PrivateAddr, Address SharedAddr, in emitInitialization() argument
886 if (CGF.getContext().getAsArrayType(PrivateVD->getType())) { in emitInitialization()
888 (void)DefaultInit(CGF); in emitInitialization()
889 emitAggregateInitialization(CGF, N, PrivateAddr, SharedAddr, DRD); in emitInitialization()
891 (void)DefaultInit(CGF); in emitInitialization()
893 emitInitWithReductionInitializer(CGF, DRD, ClausesData[N].ReductionOp, in emitInitialization()
895 } else if (!DefaultInit(CGF) && PrivateVD->hasInit() && in emitInitialization()
896 !CGF.isTrivialInitializer(PrivateVD->getInit())) { in emitInitialization()
897 CGF.EmitAnyExprToMem(PrivateVD->getInit(), PrivateAddr, in emitInitialization()
909 void ReductionCodeGen::emitCleanups(CodeGenFunction &CGF, unsigned N, in emitCleanups() argument
914 PrivateAddr = CGF.Builder.CreateElementBitCast( in emitCleanups()
915 PrivateAddr, CGF.ConvertTypeForMem(PrivateType)); in emitCleanups()
916 CGF.pushDestroy(DTorKind, PrivateAddr, PrivateType); in emitCleanups()
920 static LValue loadToBegin(CodeGenFunction &CGF, QualType BaseTy, QualType ElTy, in loadToBegin() argument
924 !CGF.getContext().hasSameType(BaseTy, ElTy)) { in loadToBegin()
926 BaseLV = CGF.EmitLoadOfPointerLValue(BaseLV.getAddress(CGF), PtrTy); in loadToBegin()
928 LValue RefLVal = CGF.MakeAddrLValue(BaseLV.getAddress(CGF), BaseTy); in loadToBegin()
929 BaseLV = CGF.EmitLoadOfReferenceLValue(RefLVal); in loadToBegin()
933 return CGF.MakeAddrLValue( in loadToBegin()
934 CGF.Builder.CreateElementBitCast(BaseLV.getAddress(CGF), in loadToBegin()
935 CGF.ConvertTypeForMem(ElTy)), in loadToBegin()
937 CGF.CGM.getTBAAInfoForSubobject(BaseLV, BaseLV.getType())); in loadToBegin()
940 static Address castToBase(CodeGenFunction &CGF, QualType BaseTy, QualType ElTy, in castToBase() argument
947 !CGF.getContext().hasSameType(BaseTy, ElTy)) { in castToBase()
948 Tmp = CGF.CreateMemTemp(BaseTy); in castToBase()
950 CGF.Builder.CreateStore(Tmp.getPointer(), TopTmp); in castToBase()
958 Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in castToBase()
960 CGF.Builder.CreateStore(Addr, Tmp); in castToBase()
964 Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in castToBase()
989 Address ReductionCodeGen::adjustPrivateAddress(CodeGenFunction &CGF, unsigned N, in adjustPrivateAddress() argument
994 LValue OriginalBaseLValue = CGF.EmitLValue(DE); in adjustPrivateAddress()
996 loadToBegin(CGF, OrigVD->getType(), SharedAddresses[N].first.getType(), in adjustPrivateAddress()
998 Address SharedAddr = SharedAddresses[N].first.getAddress(CGF); in adjustPrivateAddress()
999 llvm::Value *Adjustment = CGF.Builder.CreatePtrDiff( in adjustPrivateAddress()
1000 SharedAddr.getElementType(), BaseLValue.getPointer(CGF), in adjustPrivateAddress()
1003 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in adjustPrivateAddress()
1005 llvm::Value *Ptr = CGF.Builder.CreateGEP( in adjustPrivateAddress()
1007 return castToBase(CGF, OrigVD->getType(), in adjustPrivateAddress()
1009 OriginalBaseLValue.getAddress(CGF), Ptr); in adjustPrivateAddress()
1022 LValue CGOpenMPRegionInfo::getThreadIDVariableLValue(CodeGenFunction &CGF) { in getThreadIDVariableLValue() argument
1023 return CGF.EmitLoadOfPointerLValue( in getThreadIDVariableLValue()
1024 CGF.GetAddrOfLocalVar(getThreadIDVariable()), in getThreadIDVariableLValue()
1028 void CGOpenMPRegionInfo::EmitBody(CodeGenFunction &CGF, const Stmt *S) { in EmitBody() argument
1029 if (!CGF.HaveInsertPoint()) in EmitBody()
1036 CGF.EHStack.pushTerminate(); in EmitBody()
1038 CGF.incrementProfileCounter(S); in EmitBody()
1039 CodeGen(CGF); in EmitBody()
1040 CGF.EHStack.popTerminate(); in EmitBody()
1044 CodeGenFunction &CGF) { in getThreadIDVariableLValue() argument
1045 return CGF.MakeAddrLValue(CGF.GetAddrOfLocalVar(getThreadIDVariable()), in getThreadIDVariableLValue()
1120 CodeGenFunction CGF(CGM); in emitCombinerOrInitializer() local
1123 CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, FnInfo, Args, In->getLocation(), in emitCombinerOrInitializer()
1125 CodeGenFunction::OMPPrivateScope Scope(CGF); in emitCombinerOrInitializer()
1126 Address AddrIn = CGF.GetAddrOfLocalVar(&OmpInParm); in emitCombinerOrInitializer()
1128 In, CGF.EmitLoadOfPointerLValue(AddrIn, PtrTy->castAs<PointerType>()) in emitCombinerOrInitializer()
1129 .getAddress(CGF)); in emitCombinerOrInitializer()
1130 Address AddrOut = CGF.GetAddrOfLocalVar(&OmpOutParm); in emitCombinerOrInitializer()
1132 Out, CGF.EmitLoadOfPointerLValue(AddrOut, PtrTy->castAs<PointerType>()) in emitCombinerOrInitializer()
1133 .getAddress(CGF)); in emitCombinerOrInitializer()
1136 !CGF.isTrivialInitializer(Out->getInit())) { in emitCombinerOrInitializer()
1137 CGF.EmitAnyExprToMem(Out->getInit(), CGF.GetAddrOfLocalVar(Out), in emitCombinerOrInitializer()
1142 CGF.EmitIgnoredExpr(CombinerInitializer); in emitCombinerOrInitializer()
1144 CGF.FinishFunction(); in emitCombinerOrInitializer()
1149 CodeGenFunction *CGF, const OMPDeclareReductionDecl *D) { in emitUserDefinedReduction() argument
1168 if (CGF) { in emitUserDefinedReduction()
1169 auto &Decls = FunctionUDRMap.FindAndConstruct(CGF->CurFn); in emitUserDefinedReduction()
1187 PushAndPopStackRAII(llvm::OpenMPIRBuilder *OMPBuilder, CodeGenFunction &CGF, in PushAndPopStackRAII()
1205 auto FiniCB = [&CGF](llvm::OpenMPIRBuilder::InsertPointTy IP) { in PushAndPopStackRAII()
1208 CGBuilderTy::InsertPointGuard IPG(CGF.Builder); in PushAndPopStackRAII()
1209 CGF.Builder.restoreIP(IP); in PushAndPopStackRAII()
1211 CGF.getOMPCancelDestination(OMPD_parallel); in PushAndPopStackRAII()
1212 CGF.EmitBranchThroughCleanup(Dest); in PushAndPopStackRAII()
1234 CodeGenFunction CGF(CGM, true); in emitParallelOrTeamsOutlinedFunction() local
1258 PushAndPopStackRAII PSR(&OMPBuilder, CGF, HasCancel, InnermostKind); in emitParallelOrTeamsOutlinedFunction()
1261 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo); in emitParallelOrTeamsOutlinedFunction()
1262 return CGF.GenerateOpenMPCapturedStmtFunction(*CS, D.getBeginLoc()); in emitParallelOrTeamsOutlinedFunction()
1286 auto &&UntiedCodeGen = [this, &D, TaskTVar](CodeGenFunction &CGF, in emitTaskOutlinedFunction()
1288 llvm::Value *ThreadID = getThreadID(CGF, D.getBeginLoc()); in emitTaskOutlinedFunction()
1289 llvm::Value *UpLoc = emitUpdateLocation(CGF, D.getBeginLoc()); in emitTaskOutlinedFunction()
1292 CGF.EmitLoadOfPointerLValue(CGF.GetAddrOfLocalVar(TaskTVar), in emitTaskOutlinedFunction()
1294 .getPointer(CGF)}; in emitTaskOutlinedFunction()
1295 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in emitTaskOutlinedFunction()
1318 CodeGenFunction CGF(CGM, true); in emitTaskOutlinedFunction() local
1321 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo); in emitTaskOutlinedFunction()
1322 llvm::Function *Res = CGF.GenerateCapturedStmtFunction(*CS); in emitTaskOutlinedFunction()
1328 void CGOpenMPRuntime::setLocThreadIdInsertPt(CodeGenFunction &CGF, in setLocThreadIdInsertPt() argument
1330 auto &Elem = OpenMPLocThreadIDMap.FindAndConstruct(CGF.CurFn); in setLocThreadIdInsertPt()
1333 llvm::Value *Undef = llvm::UndefValue::get(CGF.Int32Ty); in setLocThreadIdInsertPt()
1336 Undef, CGF.Int32Ty, "svcpt", CGF.Builder.GetInsertBlock()); in setLocThreadIdInsertPt()
1339 new llvm::BitCastInst(Undef, CGF.Int32Ty, "svcpt"); in setLocThreadIdInsertPt()
1340 Elem.second.ServiceInsertPt->insertAfter(CGF.AllocaInsertPt); in setLocThreadIdInsertPt()
1344 void CGOpenMPRuntime::clearLocThreadIdInsertPt(CodeGenFunction &CGF) { in clearLocThreadIdInsertPt() argument
1345 auto &Elem = OpenMPLocThreadIDMap.FindAndConstruct(CGF.CurFn); in clearLocThreadIdInsertPt()
1353 static StringRef getIdentStringFromSourceLocation(CodeGenFunction &CGF, in getIdentStringFromSourceLocation() argument
1358 PresumedLoc PLoc = CGF.getContext().getSourceManager().getPresumedLoc(Loc); in getIdentStringFromSourceLocation()
1360 if (const auto *FD = dyn_cast_or_null<FunctionDecl>(CGF.CurFuncDecl)) in getIdentStringFromSourceLocation()
1366 llvm::Value *CGOpenMPRuntime::emitUpdateLocation(CodeGenFunction &CGF, in emitUpdateLocation() argument
1377 if (const auto *FD = dyn_cast_or_null<FunctionDecl>(CGF.CurFuncDecl)) in emitUpdateLocation()
1379 PresumedLoc PLoc = CGF.getContext().getSourceManager().getPresumedLoc(Loc); in emitUpdateLocation()
1391 llvm::Value *CGOpenMPRuntime::getThreadID(CodeGenFunction &CGF, in getThreadID() argument
1393 assert(CGF.CurFn && "No function in current CodeGenFunction."); in getThreadID()
1398 OMPBuilder.updateToLocation(CGF.Builder.saveIP()); in getThreadID()
1401 getIdentStringFromSourceLocation(CGF, Loc, Buffer), SrcLocStrSize); in getThreadID()
1409 auto I = OpenMPLocThreadIDMap.find(CGF.CurFn); in getThreadID()
1417 dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo)) { in getThreadID()
1420 LValue LVal = OMPRegionInfo->getThreadIDVariableLValue(CGF); in getThreadID()
1421 llvm::BasicBlock *TopBlock = CGF.AllocaInsertPt->getParent(); in getThreadID()
1422 if (!CGF.EHStack.requiresLandingPad() || !CGF.getLangOpts().Exceptions || in getThreadID()
1423 !CGF.getLangOpts().CXXExceptions || in getThreadID()
1424 CGF.Builder.GetInsertBlock() == TopBlock || in getThreadID()
1425 !isa<llvm::Instruction>(LVal.getPointer(CGF)) || in getThreadID()
1426 cast<llvm::Instruction>(LVal.getPointer(CGF))->getParent() == in getThreadID()
1428 cast<llvm::Instruction>(LVal.getPointer(CGF))->getParent() == in getThreadID()
1429 CGF.Builder.GetInsertBlock()) { in getThreadID()
1430 ThreadID = CGF.EmitLoadOfScalar(LVal, Loc); in getThreadID()
1433 if (CGF.Builder.GetInsertBlock() == TopBlock) { in getThreadID()
1434 auto &Elem = OpenMPLocThreadIDMap.FindAndConstruct(CGF.CurFn); in getThreadID()
1446 auto &Elem = OpenMPLocThreadIDMap.FindAndConstruct(CGF.CurFn); in getThreadID()
1448 setLocThreadIdInsertPt(CGF); in getThreadID()
1449 CGBuilderTy::InsertPointGuard IPG(CGF.Builder); in getThreadID()
1450 CGF.Builder.SetInsertPoint(Elem.second.ServiceInsertPt); in getThreadID()
1451 llvm::CallInst *Call = CGF.Builder.CreateCall( in getThreadID()
1454 emitUpdateLocation(CGF, Loc)); in getThreadID()
1455 Call->setCallingConv(CGF.getRuntimeCC()); in getThreadID()
1460 void CGOpenMPRuntime::functionFinished(CodeGenFunction &CGF) { in functionFinished() argument
1461 assert(CGF.CurFn && "No function in current CodeGenFunction."); in functionFinished()
1462 if (OpenMPLocThreadIDMap.count(CGF.CurFn)) { in functionFinished()
1463 clearLocThreadIdInsertPt(CGF); in functionFinished()
1464 OpenMPLocThreadIDMap.erase(CGF.CurFn); in functionFinished()
1466 if (FunctionUDRMap.count(CGF.CurFn) > 0) { in functionFinished()
1467 for(const auto *D : FunctionUDRMap[CGF.CurFn]) in functionFinished()
1469 FunctionUDRMap.erase(CGF.CurFn); in functionFinished()
1471 auto I = FunctionUDMMap.find(CGF.CurFn); in functionFinished()
1477 LastprivateConditionalToTypes.erase(CGF.CurFn); in functionFinished()
1478 FunctionToUntiedTaskStackMap.erase(CGF.CurFn); in functionFinished()
1669 Address CGOpenMPRuntime::getAddrOfThreadPrivate(CodeGenFunction &CGF, in getAddrOfThreadPrivate() argument
1679 emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc), in getAddrOfThreadPrivate()
1680 CGF.Builder.CreatePointerCast(VDAddr.getPointer(), CGM.Int8PtrTy), in getAddrOfThreadPrivate()
1684 CGF.EmitRuntimeCall( in getAddrOfThreadPrivate()
1688 CGF.Int8Ty, VDAddr.getAlignment()); in getAddrOfThreadPrivate()
1692 CodeGenFunction &CGF, Address VDAddr, llvm::Value *Ctor, in emitThreadPrivateVarInit() argument
1696 llvm::Value *OMPLoc = emitUpdateLocation(CGF, Loc); in emitThreadPrivateVarInit()
1697 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in emitThreadPrivateVarInit()
1703 OMPLoc, CGF.Builder.CreatePointerCast(VDAddr.getPointer(), CGM.VoidPtrTy), in emitThreadPrivateVarInit()
1705 CGF.EmitRuntimeCall( in emitThreadPrivateVarInit()
1713 bool PerformInit, CodeGenFunction *CGF) { in emitThreadPrivateVarDefinition() argument
1812 if (!CGF) { in emitThreadPrivateVarDefinition()
1827 emitThreadPrivateVarInit(*CGF, VDAddr, Ctor, CopyCtor, Dtor, Loc); in emitThreadPrivateVarDefinition()
1964 Address CGOpenMPRuntime::getAddrOfArtificialThreadPrivate(CodeGenFunction &CGF, in getAddrOfArtificialThreadPrivate() argument
1968 llvm::Type *VarLVType = CGF.ConvertTypeForMem(VarType); in getAddrOfArtificialThreadPrivate()
1979 emitUpdateLocation(CGF, SourceLocation()), in getAddrOfArtificialThreadPrivate()
1980 getThreadID(CGF, SourceLocation()), in getAddrOfArtificialThreadPrivate()
1981 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(GAddr, CGM.VoidPtrTy), in getAddrOfArtificialThreadPrivate()
1982 CGF.Builder.CreateIntCast(CGF.getTypeSize(VarType), CGM.SizeTy, in getAddrOfArtificialThreadPrivate()
1988 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in getAddrOfArtificialThreadPrivate()
1989 CGF.EmitRuntimeCall( in getAddrOfArtificialThreadPrivate()
1997 void CGOpenMPRuntime::emitIfClause(CodeGenFunction &CGF, const Expr *Cond, in emitIfClause() argument
2000 CodeGenFunction::LexicalScope ConditionScope(CGF, Cond->getSourceRange()); in emitIfClause()
2005 if (CGF.ConstantFoldsToSimpleInteger(Cond, CondConstant)) { in emitIfClause()
2007 ThenGen(CGF); in emitIfClause()
2009 ElseGen(CGF); in emitIfClause()
2015 llvm::BasicBlock *ThenBlock = CGF.createBasicBlock("omp_if.then"); in emitIfClause()
2016 llvm::BasicBlock *ElseBlock = CGF.createBasicBlock("omp_if.else"); in emitIfClause()
2017 llvm::BasicBlock *ContBlock = CGF.createBasicBlock("omp_if.end"); in emitIfClause()
2018 CGF.EmitBranchOnBoolExpr(Cond, ThenBlock, ElseBlock, /*TrueCount=*/0); in emitIfClause()
2021 CGF.EmitBlock(ThenBlock); in emitIfClause()
2022 ThenGen(CGF); in emitIfClause()
2023 CGF.EmitBranch(ContBlock); in emitIfClause()
2026 (void)ApplyDebugLocation::CreateEmpty(CGF); in emitIfClause()
2027 CGF.EmitBlock(ElseBlock); in emitIfClause()
2028 ElseGen(CGF); in emitIfClause()
2030 (void)ApplyDebugLocation::CreateEmpty(CGF); in emitIfClause()
2031 CGF.EmitBranch(ContBlock); in emitIfClause()
2033 CGF.EmitBlock(ContBlock, /*IsFinished=*/true); in emitIfClause()
2036 void CGOpenMPRuntime::emitParallelCall(CodeGenFunction &CGF, SourceLocation Loc, in emitParallelCall() argument
2041 if (!CGF.HaveInsertPoint()) in emitParallelCall()
2043 llvm::Value *RTLoc = emitUpdateLocation(CGF, Loc); in emitParallelCall()
2046 this](CodeGenFunction &CGF, PrePostActionTy &) { in emitParallelCall() argument
2048 CGOpenMPRuntime &RT = CGF.CGM.getOpenMPRuntime(); in emitParallelCall()
2051 CGF.Builder.getInt32(CapturedVars.size()), // Number of captured vars in emitParallelCall()
2052 CGF.Builder.CreateBitCast(OutlinedFn, RT.getKmpc_MicroPointerTy())}; in emitParallelCall()
2059 CGF.EmitRuntimeCall(RTLFn, RealArgs); in emitParallelCall()
2062 this](CodeGenFunction &CGF, PrePostActionTy &) { in emitParallelCall() argument
2063 CGOpenMPRuntime &RT = CGF.CGM.getOpenMPRuntime(); in emitParallelCall()
2064 llvm::Value *ThreadID = RT.getThreadID(CGF, Loc); in emitParallelCall()
2068 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in emitParallelCall()
2073 Address ThreadIDAddr = RT.emitThreadIDAddress(CGF, Loc); in emitParallelCall()
2075 CGF.CreateDefaultAlignTempAlloca(CGF.Int32Ty, in emitParallelCall()
2077 CGF.Builder.CreateStore(CGF.Builder.getInt32(/*C*/ 0), ZeroAddrBound); in emitParallelCall()
2092 RT.emitOutlinedFunctionCall(CGF, Loc, OutlinedFn, OutlinedFnArgs); in emitParallelCall()
2095 llvm::Value *EndArgs[] = {RT.emitUpdateLocation(CGF, Loc), ThreadID}; in emitParallelCall()
2096 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in emitParallelCall()
2101 emitIfClause(CGF, IfCond, ThenGen, ElseGen); in emitParallelCall()
2104 ThenRCG(CGF); in emitParallelCall()
2114 Address CGOpenMPRuntime::emitThreadIDAddress(CodeGenFunction &CGF, in emitThreadIDAddress() argument
2117 dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo)) in emitThreadIDAddress()
2119 return OMPRegionInfo->getThreadIDVariableLValue(CGF).getAddress(CGF); in emitThreadIDAddress()
2121 llvm::Value *ThreadID = getThreadID(CGF, Loc); in emitThreadIDAddress()
2123 CGF.getContext().getIntTypeForBitwidth(/*DestWidth*/ 32, /*Signed*/ true); in emitThreadIDAddress()
2124 Address ThreadIDTemp = CGF.CreateMemTemp(Int32Ty, /*Name*/ ".threadid_temp."); in emitThreadIDAddress()
2125 CGF.EmitStoreOfScalar(ThreadID, in emitThreadIDAddress()
2126 CGF.MakeAddrLValue(ThreadIDTemp, Int32Ty)); in emitThreadIDAddress()
2154 void Enter(CodeGenFunction &CGF) override { in Enter() argument
2155 llvm::Value *EnterRes = CGF.EmitRuntimeCall(EnterCallee, EnterArgs); in Enter()
2157 llvm::Value *CallBool = CGF.Builder.CreateIsNotNull(EnterRes); in Enter()
2158 auto *ThenBlock = CGF.createBasicBlock("omp_if.then"); in Enter()
2159 ContBlock = CGF.createBasicBlock("omp_if.end"); in Enter()
2161 CGF.Builder.CreateCondBr(CallBool, ThenBlock, ContBlock); in Enter()
2162 CGF.EmitBlock(ThenBlock); in Enter()
2165 void Done(CodeGenFunction &CGF) { in Done() argument
2167 CGF.EmitBranch(ContBlock); in Done()
2168 CGF.EmitBlock(ContBlock, true); in Done()
2170 void Exit(CodeGenFunction &CGF) override { in Exit() argument
2171 CGF.EmitRuntimeCall(ExitCallee, ExitArgs); in Exit()
2176 void CGOpenMPRuntime::emitCriticalRegion(CodeGenFunction &CGF, in emitCriticalRegion() argument
2184 if (!CGF.HaveInsertPoint()) in emitCriticalRegion()
2186 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc), in emitCriticalRegion()
2191 EnterArgs.push_back(CGF.Builder.CreateIntCast( in emitCriticalRegion()
2192 CGF.EmitScalarExpr(Hint), CGM.Int32Ty, /*isSigned=*/false)); in emitCriticalRegion()
2203 emitInlinedDirective(CGF, OMPD_critical, CriticalOpGen); in emitCriticalRegion()
2206 void CGOpenMPRuntime::emitMasterRegion(CodeGenFunction &CGF, in emitMasterRegion() argument
2209 if (!CGF.HaveInsertPoint()) in emitMasterRegion()
2216 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)}; in emitMasterRegion()
2225 emitInlinedDirective(CGF, OMPD_master, MasterOpGen); in emitMasterRegion()
2226 Action.Done(CGF); in emitMasterRegion()
2229 void CGOpenMPRuntime::emitMaskedRegion(CodeGenFunction &CGF, in emitMaskedRegion() argument
2232 if (!CGF.HaveInsertPoint()) in emitMaskedRegion()
2240 ? CGF.EmitScalarExpr(Filter, CGF.Int32Ty) in emitMaskedRegion()
2242 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc), in emitMaskedRegion()
2244 llvm::Value *ArgsEnd[] = {emitUpdateLocation(CGF, Loc), in emitMaskedRegion()
2245 getThreadID(CGF, Loc)}; in emitMaskedRegion()
2254 emitInlinedDirective(CGF, OMPD_masked, MaskedOpGen); in emitMaskedRegion()
2255 Action.Done(CGF); in emitMaskedRegion()
2258 void CGOpenMPRuntime::emitTaskyieldCall(CodeGenFunction &CGF, in emitTaskyieldCall() argument
2260 if (!CGF.HaveInsertPoint()) in emitTaskyieldCall()
2262 if (CGF.CGM.getLangOpts().OpenMPIRBuilder) { in emitTaskyieldCall()
2263 OMPBuilder.createTaskyield(CGF.Builder); in emitTaskyieldCall()
2267 emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc), in emitTaskyieldCall()
2269 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in emitTaskyieldCall()
2274 if (auto *Region = dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo)) in emitTaskyieldCall()
2275 Region->emitUntiedSwitch(CGF); in emitTaskyieldCall()
2278 void CGOpenMPRuntime::emitTaskgroupRegion(CodeGenFunction &CGF, in emitTaskgroupRegion() argument
2281 if (!CGF.HaveInsertPoint()) in emitTaskgroupRegion()
2287 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)}; in emitTaskgroupRegion()
2295 emitInlinedDirective(CGF, OMPD_taskgroup, TaskgroupOpGen); in emitTaskgroupRegion()
2300 static Address emitAddrOfVarFromArray(CodeGenFunction &CGF, Address Array, in emitAddrOfVarFromArray() argument
2303 Address PtrAddr = CGF.Builder.CreateConstArrayGEP(Array, Index); in emitAddrOfVarFromArray()
2304 llvm::Value *Ptr = CGF.Builder.CreateLoad(PtrAddr); in emitAddrOfVarFromArray()
2306 llvm::Type *ElemTy = CGF.ConvertTypeForMem(Var->getType()); in emitAddrOfVarFromArray()
2308 CGF.Builder.CreateBitCast( in emitAddrOfVarFromArray()
2310 ElemTy, CGF.getContext().getDeclAlign(Var)); in emitAddrOfVarFromArray()
2336 CodeGenFunction CGF(CGM); in emitCopyprivateCopyFunction() local
2337 CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, CGFI, Args, Loc, Loc); in emitCopyprivateCopyFunction()
2340 Address LHS(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitCopyprivateCopyFunction()
2341 CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(&LHSArg)), in emitCopyprivateCopyFunction()
2343 ArgsElemType, CGF.getPointerAlign()); in emitCopyprivateCopyFunction()
2344 Address RHS(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitCopyprivateCopyFunction()
2345 CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(&RHSArg)), in emitCopyprivateCopyFunction()
2347 ArgsElemType, CGF.getPointerAlign()); in emitCopyprivateCopyFunction()
2355 Address DestAddr = emitAddrOfVarFromArray(CGF, LHS, I, DestVar); in emitCopyprivateCopyFunction()
2359 Address SrcAddr = emitAddrOfVarFromArray(CGF, RHS, I, SrcVar); in emitCopyprivateCopyFunction()
2363 CGF.EmitOMPCopy(Type, DestAddr, SrcAddr, DestVar, SrcVar, AssignmentOps[I]); in emitCopyprivateCopyFunction()
2365 CGF.FinishFunction(); in emitCopyprivateCopyFunction()
2369 void CGOpenMPRuntime::emitSingleRegion(CodeGenFunction &CGF, in emitSingleRegion() argument
2376 if (!CGF.HaveInsertPoint()) in emitSingleRegion()
2396 DidIt = CGF.CreateMemTemp(KmpInt32Ty, ".omp.copyprivate.did_it"); in emitSingleRegion()
2397 CGF.Builder.CreateStore(CGF.Builder.getInt32(0), DidIt); in emitSingleRegion()
2400 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)}; in emitSingleRegion()
2409 emitInlinedDirective(CGF, OMPD_single, SingleOpGen); in emitSingleRegion()
2412 CGF.Builder.CreateStore(CGF.Builder.getInt32(1), DidIt); in emitSingleRegion()
2414 Action.Done(CGF); in emitSingleRegion()
2424 CGF.CreateMemTemp(CopyprivateArrayTy, ".omp.copyprivate.cpr_list"); in emitSingleRegion()
2426 Address Elem = CGF.Builder.CreateConstArrayGEP(CopyprivateList, I); in emitSingleRegion()
2427 CGF.Builder.CreateStore( in emitSingleRegion()
2428 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitSingleRegion()
2429 CGF.EmitLValue(CopyprivateVars[I]).getPointer(CGF), in emitSingleRegion()
2430 CGF.VoidPtrTy), in emitSingleRegion()
2436 CGM, CGF.ConvertTypeForMem(CopyprivateArrayTy), CopyprivateVars, in emitSingleRegion()
2438 llvm::Value *BufSize = CGF.getTypeSize(CopyprivateArrayTy); in emitSingleRegion()
2439 Address CL = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitSingleRegion()
2440 CopyprivateList, CGF.VoidPtrTy, CGF.Int8Ty); in emitSingleRegion()
2441 llvm::Value *DidItVal = CGF.Builder.CreateLoad(DidIt); in emitSingleRegion()
2443 emitUpdateLocation(CGF, Loc), // ident_t *<loc> in emitSingleRegion()
2444 getThreadID(CGF, Loc), // i32 <gtid> in emitSingleRegion()
2450 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in emitSingleRegion()
2456 void CGOpenMPRuntime::emitOrderedRegion(CodeGenFunction &CGF, in emitOrderedRegion() argument
2459 if (!CGF.HaveInsertPoint()) in emitOrderedRegion()
2466 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)}; in emitOrderedRegion()
2474 emitInlinedDirective(CGF, OMPD_ordered, OrderedOpGen); in emitOrderedRegion()
2477 emitInlinedDirective(CGF, OMPD_ordered, OrderedOpGen); in emitOrderedRegion()
2496 CodeGenFunction &CGF, const OMPLoopDirective &S, in getDefaultScheduleAndChunk() argument
2507 CGF.getContext(), ChunkSize, in getDefaultScheduleAndChunk()
2508 CGF.getContext().getIntTypeForBitwidth(32, /*Signed=*/0), in getDefaultScheduleAndChunk()
2513 void CGOpenMPRuntime::emitBarrierCall(CodeGenFunction &CGF, SourceLocation Loc, in emitBarrierCall() argument
2518 dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo); in emitBarrierCall()
2519 if (CGF.CGM.getLangOpts().OpenMPIRBuilder) { in emitBarrierCall()
2520 CGF.Builder.restoreIP(OMPBuilder.createBarrier( in emitBarrierCall()
2521 CGF.Builder, Kind, ForceSimpleCall, EmitChecks)); in emitBarrierCall()
2525 if (!CGF.HaveInsertPoint()) in emitBarrierCall()
2532 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc, Flags), in emitBarrierCall()
2533 getThreadID(CGF, Loc)}; in emitBarrierCall()
2536 llvm::Value *Result = CGF.EmitRuntimeCall( in emitBarrierCall()
2544 llvm::BasicBlock *ExitBB = CGF.createBasicBlock(".cancel.exit"); in emitBarrierCall()
2545 llvm::BasicBlock *ContBB = CGF.createBasicBlock(".cancel.continue"); in emitBarrierCall()
2546 llvm::Value *Cmp = CGF.Builder.CreateIsNotNull(Result); in emitBarrierCall()
2547 CGF.Builder.CreateCondBr(Cmp, ExitBB, ContBB); in emitBarrierCall()
2548 CGF.EmitBlock(ExitBB); in emitBarrierCall()
2551 CGF.getOMPCancelDestination(OMPRegionInfo->getDirectiveKind()); in emitBarrierCall()
2552 CGF.EmitBranchThroughCleanup(CancelDestination); in emitBarrierCall()
2553 CGF.EmitBlock(ContBB, /*IsFinished=*/true); in emitBarrierCall()
2558 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in emitBarrierCall()
2563 void CGOpenMPRuntime::emitErrorCall(CodeGenFunction &CGF, SourceLocation Loc, in emitErrorCall() argument
2566 ME ? CGF.EmitStringLiteralLValue(cast<StringLiteral>(ME)).getPointer(CGF) in emitErrorCall()
2567 : llvm::ConstantPointerNull::get(CGF.VoidPtrTy); in emitErrorCall()
2571 emitUpdateLocation(CGF, Loc, /*Flags=*/0, /*GenLoc=*/true), in emitErrorCall()
2573 CGF.Builder.CreatePointerCast(MVL, CGM.Int8PtrTy)}; in emitErrorCall()
2574 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in emitErrorCall()
2693 CodeGenFunction &CGF, SourceLocation Loc, in emitForDispatchInit() argument
2696 if (!CGF.HaveInsertPoint()) in emitForDispatchInit()
2711 : CGF.Builder.getIntN(IVSize, 1); in emitForDispatchInit()
2713 emitUpdateLocation(CGF, Loc), in emitForDispatchInit()
2714 getThreadID(CGF, Loc), in emitForDispatchInit()
2715 CGF.Builder.getInt32(addMonoNonMonoModifier( in emitForDispatchInit()
2719 CGF.Builder.getIntN(IVSize, 1), // Stride in emitForDispatchInit()
2722 CGF.EmitRuntimeCall(createDispatchInitFunction(IVSize, IVSigned), Args); in emitForDispatchInit()
2726 CodeGenFunction &CGF, llvm::Value *UpdateLocation, llvm::Value *ThreadId, in emitForStaticInitCall() argument
2730 if (!CGF.HaveInsertPoint()) in emitForStaticInitCall()
2751 Chunk = CGF.Builder.getIntN(Values.IVSize, 1); in emitForStaticInitCall()
2762 CGF.Builder.getInt32(addMonoNonMonoModifier(CGF.CGM, Schedule, M1, in emitForStaticInitCall()
2768 CGF.Builder.getIntN(Values.IVSize, 1), // Incr in emitForStaticInitCall()
2771 CGF.EmitRuntimeCall(ForStaticInitFunction, Args); in emitForStaticInitCall()
2774 void CGOpenMPRuntime::emitForStaticInit(CodeGenFunction &CGF, in emitForStaticInit() argument
2783 llvm::Value *UpdatedLocation = emitUpdateLocation(CGF, Loc, in emitForStaticInit()
2787 llvm::Value *ThreadId = getThreadID(CGF, Loc); in emitForStaticInit()
2790 auto DL = ApplyDebugLocation::CreateDefaultArtificial(CGF, Loc); in emitForStaticInit()
2791 emitForStaticInitCall(CGF, UpdatedLocation, ThreadId, StaticInitFunction, in emitForStaticInit()
2796 CodeGenFunction &CGF, SourceLocation Loc, in emitDistributeStaticInit() argument
2802 emitUpdateLocation(CGF, Loc, OMP_IDENT_WORK_DISTRIBUTE); in emitDistributeStaticInit()
2803 llvm::Value *ThreadId = getThreadID(CGF, Loc); in emitDistributeStaticInit()
2811 emitForStaticInitCall(CGF, UpdatedLocation, ThreadId, StaticInitFunction, in emitDistributeStaticInit()
2816 void CGOpenMPRuntime::emitForStaticFinish(CodeGenFunction &CGF, in emitForStaticFinish() argument
2819 if (!CGF.HaveInsertPoint()) in emitForStaticFinish()
2823 emitUpdateLocation(CGF, Loc, in emitForStaticFinish()
2829 getThreadID(CGF, Loc)}; in emitForStaticFinish()
2830 auto DL = ApplyDebugLocation::CreateDefaultArtificial(CGF, Loc); in emitForStaticFinish()
2833 CGF.EmitRuntimeCall( in emitForStaticFinish()
2838 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in emitForStaticFinish()
2843 void CGOpenMPRuntime::emitForOrderedIterationEnd(CodeGenFunction &CGF, in emitForOrderedIterationEnd() argument
2847 if (!CGF.HaveInsertPoint()) in emitForOrderedIterationEnd()
2850 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)}; in emitForOrderedIterationEnd()
2851 CGF.EmitRuntimeCall(createDispatchFiniFunction(IVSize, IVSigned), Args); in emitForOrderedIterationEnd()
2854 llvm::Value *CGOpenMPRuntime::emitForNext(CodeGenFunction &CGF, in emitForNext() argument
2864 emitUpdateLocation(CGF, Loc), in emitForNext()
2865 getThreadID(CGF, Loc), in emitForNext()
2872 CGF.EmitRuntimeCall(createDispatchNextFunction(IVSize, IVSigned), Args); in emitForNext()
2873 return CGF.EmitScalarConversion( in emitForNext()
2874 Call, CGF.getContext().getIntTypeForBitwidth(32, /*Signed=*/1), in emitForNext()
2875 CGF.getContext().BoolTy, Loc); in emitForNext()
2878 void CGOpenMPRuntime::emitNumThreadsClause(CodeGenFunction &CGF, in emitNumThreadsClause() argument
2881 if (!CGF.HaveInsertPoint()) in emitNumThreadsClause()
2885 emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc), in emitNumThreadsClause()
2886 CGF.Builder.CreateIntCast(NumThreads, CGF.Int32Ty, /*isSigned*/ true)}; in emitNumThreadsClause()
2887 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in emitNumThreadsClause()
2892 void CGOpenMPRuntime::emitProcBindClause(CodeGenFunction &CGF, in emitProcBindClause() argument
2895 if (!CGF.HaveInsertPoint()) in emitProcBindClause()
2900 emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc), in emitProcBindClause()
2902 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in emitProcBindClause()
2907 void CGOpenMPRuntime::emitFlush(CodeGenFunction &CGF, ArrayRef<const Expr *>, in emitFlush() argument
2909 if (CGF.CGM.getLangOpts().OpenMPIRBuilder) { in emitFlush()
2910 OMPBuilder.createFlush(CGF.Builder); in emitFlush()
2912 if (!CGF.HaveInsertPoint()) in emitFlush()
2915 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in emitFlush()
2917 emitUpdateLocation(CGF, Loc)); in emitFlush()
3205 CodeGenFunction CGF(CGM); in emitProxyTaskFunction() local
3206 CGF.StartFunction(GlobalDecl(), KmpInt32Ty, TaskEntry, TaskEntryFnInfo, Args, in emitProxyTaskFunction()
3214 llvm::Value *GtidParam = CGF.EmitLoadOfScalar( in emitProxyTaskFunction()
3215 CGF.GetAddrOfLocalVar(&GtidArg), /*Volatile=*/false, KmpInt32Ty, Loc); in emitProxyTaskFunction()
3216 LValue TDBase = CGF.EmitLoadOfPointerLValue( in emitProxyTaskFunction()
3217 CGF.GetAddrOfLocalVar(&TaskTypeArg), in emitProxyTaskFunction()
3222 CGF.EmitLValueForField(TDBase, *KmpTaskTWithPrivatesQTyRD->field_begin()); in emitProxyTaskFunction()
3225 LValue PartIdLVal = CGF.EmitLValueForField(Base, *PartIdFI); in emitProxyTaskFunction()
3226 llvm::Value *PartidParam = PartIdLVal.getPointer(CGF); in emitProxyTaskFunction()
3229 LValue SharedsLVal = CGF.EmitLValueForField(Base, *SharedsFI); in emitProxyTaskFunction()
3230 llvm::Value *SharedsParam = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitProxyTaskFunction()
3231 CGF.EmitLoadOfScalar(SharedsLVal, Loc), in emitProxyTaskFunction()
3232 CGF.ConvertTypeForMem(SharedsPtrTy)); in emitProxyTaskFunction()
3237 LValue PrivatesLVal = CGF.EmitLValueForField(TDBase, *PrivatesFI); in emitProxyTaskFunction()
3238 PrivatesParam = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitProxyTaskFunction()
3239 PrivatesLVal.getPointer(CGF), CGF.VoidPtrTy); in emitProxyTaskFunction()
3241 PrivatesParam = llvm::ConstantPointerNull::get(CGF.VoidPtrTy); in emitProxyTaskFunction()
3246 CGF.Builder in emitProxyTaskFunction()
3247 .CreatePointerBitCastOrAddrSpaceCast(TDBase.getAddress(CGF), in emitProxyTaskFunction()
3248 CGF.VoidPtrTy, CGF.Int8Ty) in emitProxyTaskFunction()
3254 LValue LBLVal = CGF.EmitLValueForField(Base, *LBFI); in emitProxyTaskFunction()
3255 llvm::Value *LBParam = CGF.EmitLoadOfScalar(LBLVal, Loc); in emitProxyTaskFunction()
3257 LValue UBLVal = CGF.EmitLValueForField(Base, *UBFI); in emitProxyTaskFunction()
3258 llvm::Value *UBParam = CGF.EmitLoadOfScalar(UBLVal, Loc); in emitProxyTaskFunction()
3260 LValue StLVal = CGF.EmitLValueForField(Base, *StFI); in emitProxyTaskFunction()
3261 llvm::Value *StParam = CGF.EmitLoadOfScalar(StLVal, Loc); in emitProxyTaskFunction()
3263 LValue LILVal = CGF.EmitLValueForField(Base, *LIFI); in emitProxyTaskFunction()
3264 llvm::Value *LIParam = CGF.EmitLoadOfScalar(LILVal, Loc); in emitProxyTaskFunction()
3266 LValue RLVal = CGF.EmitLValueForField(Base, *RFI); in emitProxyTaskFunction()
3267 llvm::Value *RParam = CGF.EmitLoadOfScalar(RLVal, Loc); in emitProxyTaskFunction()
3276 CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF, Loc, TaskFunction, in emitProxyTaskFunction()
3278 CGF.EmitStoreThroughLValue(RValue::get(CGF.Builder.getInt32(/*C=*/0)), in emitProxyTaskFunction()
3279 CGF.MakeAddrLValue(CGF.ReturnValue, KmpInt32Ty)); in emitProxyTaskFunction()
3280 CGF.FinishFunction(); in emitProxyTaskFunction()
3310 CodeGenFunction CGF(CGM); in emitDestructorsFunction() local
3311 CGF.StartFunction(GlobalDecl(), KmpInt32Ty, DestructorFn, DestructorFnInfo, in emitDestructorsFunction()
3314 LValue Base = CGF.EmitLoadOfPointerLValue( in emitDestructorsFunction()
3315 CGF.GetAddrOfLocalVar(&TaskTypeArg), in emitDestructorsFunction()
3320 Base = CGF.EmitLValueForField(Base, *FI); in emitDestructorsFunction()
3325 LValue FieldLValue = CGF.EmitLValueForField(Base, Field); in emitDestructorsFunction()
3326 CGF.pushDestroy(DtorKind, FieldLValue.getAddress(CGF), Field->getType()); in emitDestructorsFunction()
3329 CGF.FinishFunction(); in emitDestructorsFunction()
3418 CodeGenFunction CGF(CGM); in emitTaskPrivateMappingFunction() local
3419 CGF.StartFunction(GlobalDecl(), C.VoidTy, TaskPrivatesMap, in emitTaskPrivateMappingFunction()
3423 LValue Base = CGF.EmitLoadOfPointerLValue( in emitTaskPrivateMappingFunction()
3424 CGF.GetAddrOfLocalVar(&TaskPrivatesArg), in emitTaskPrivateMappingFunction()
3429 LValue FieldLVal = CGF.EmitLValueForField(Base, Field); in emitTaskPrivateMappingFunction()
3432 CGF.MakeAddrLValue(CGF.GetAddrOfLocalVar(VD), VD->getType()); in emitTaskPrivateMappingFunction()
3433 LValue RefLoadLVal = CGF.EmitLoadOfPointerLValue( in emitTaskPrivateMappingFunction()
3434 RefLVal.getAddress(CGF), RefLVal.getType()->castAs<PointerType>()); in emitTaskPrivateMappingFunction()
3435 CGF.EmitStoreOfScalar(FieldLVal.getPointer(CGF), RefLoadLVal); in emitTaskPrivateMappingFunction()
3438 CGF.FinishFunction(); in emitTaskPrivateMappingFunction()
3443 static void emitPrivatesInit(CodeGenFunction &CGF, in emitPrivatesInit() argument
3450 ASTContext &C = CGF.getContext(); in emitPrivatesInit()
3452 LValue PrivatesBase = CGF.EmitLValueForField(TDBase, *FI); in emitPrivatesInit()
3467 SrcBase = CGF.MakeAddrLValue( in emitPrivatesInit()
3468 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitPrivatesInit()
3469 KmpTaskSharedsPtr, CGF.ConvertTypeForMem(SharedsPtrTy), in emitPrivatesInit()
3470 CGF.ConvertTypeForMem(SharedsTy)), in emitPrivatesInit()
3483 !CGF.isTrivialInitializer(Init)))) { in emitPrivatesInit()
3484 LValue PrivateLValue = CGF.EmitLValueForField(PrivatesBase, *FI); in emitPrivatesInit()
3502 CGF.MakeAddrLValue(CGF.GetAddrOfLocalVar(OriginalVD), Type); in emitPrivatesInit()
3504 SharedRefLValue = CGF.EmitLValueForField(SrcBase, SharedField); in emitPrivatesInit()
3505 SharedRefLValue = CGF.MakeAddrLValue( in emitPrivatesInit()
3506 SharedRefLValue.getAddress(CGF).withAlignment( in emitPrivatesInit()
3510 } else if (CGF.LambdaCaptureFields.count( in emitPrivatesInit()
3512 isa_and_nonnull<BlockDecl>(CGF.CurCodeDecl)) { in emitPrivatesInit()
3513 SharedRefLValue = CGF.EmitLValue(Pair.second.OriginalRef); in emitPrivatesInit()
3517 CGF, [](CodeGenFunction &, PrePostActionTy &) {}, OMPD_unknown, in emitPrivatesInit()
3519 SharedRefLValue = CGF.EmitLValue(Pair.second.OriginalRef); in emitPrivatesInit()
3523 if (!isa<CXXConstructExpr>(Init) || CGF.isTrivialInitializer(Init)) { in emitPrivatesInit()
3525 CGF.EmitAggregateAssign(PrivateLValue, SharedRefLValue, Type); in emitPrivatesInit()
3529 CGF.EmitOMPAggregateAssign( in emitPrivatesInit()
3530 PrivateLValue.getAddress(CGF), SharedRefLValue.getAddress(CGF), in emitPrivatesInit()
3532 [&CGF, Elem, Init, &CapturesInfo](Address DestElement, in emitPrivatesInit()
3535 CodeGenFunction::OMPPrivateScope InitScope(CGF); in emitPrivatesInit()
3540 CGF, &CapturesInfo); in emitPrivatesInit()
3541 CGF.EmitAnyExprToMem(Init, DestElement, in emitPrivatesInit()
3547 CodeGenFunction::OMPPrivateScope InitScope(CGF); in emitPrivatesInit()
3548 InitScope.addPrivate(Elem, SharedRefLValue.getAddress(CGF)); in emitPrivatesInit()
3550 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CapturesInfo); in emitPrivatesInit()
3551 CGF.EmitExprAsInit(Init, VD, PrivateLValue, in emitPrivatesInit()
3555 CGF.EmitExprAsInit(Init, VD, PrivateLValue, /*capturedByInit=*/false); in emitPrivatesInit()
3563 static bool checkInitIsRequired(CodeGenFunction &CGF, in checkInitIsRequired() argument
3572 !CGF.isTrivialInitializer(Init)); in checkInitIsRequired()
3619 CodeGenFunction CGF(CGM); in emitTaskDupFunction() local
3620 CGF.StartFunction(GlobalDecl(), C.VoidTy, TaskDup, TaskDupFnInfo, Args, Loc, in emitTaskDupFunction()
3623 LValue TDBase = CGF.EmitLoadOfPointerLValue( in emitTaskDupFunction()
3624 CGF.GetAddrOfLocalVar(&DstArg), in emitTaskDupFunction()
3629 LValue Base = CGF.EmitLValueForField( in emitTaskDupFunction()
3631 LValue LILVal = CGF.EmitLValueForField(Base, *LIFI); in emitTaskDupFunction()
3632 llvm::Value *Lastpriv = CGF.EmitLoadOfScalar( in emitTaskDupFunction()
3633 CGF.GetAddrOfLocalVar(&LastprivArg), /*Volatile=*/false, C.IntTy, Loc); in emitTaskDupFunction()
3634 CGF.EmitStoreOfScalar(Lastpriv, LILVal); in emitTaskDupFunction()
3641 LValue TDBase = CGF.EmitLoadOfPointerLValue( in emitTaskDupFunction()
3642 CGF.GetAddrOfLocalVar(&SrcArg), in emitTaskDupFunction()
3644 LValue Base = CGF.EmitLValueForField( in emitTaskDupFunction()
3647 CGF.EmitLoadOfScalar(CGF.EmitLValueForField( in emitTaskDupFunction()
3651 CGF.Int8Ty, CGM.getNaturalTypeAlignment(SharedsTy)); in emitTaskDupFunction()
3653 emitPrivatesInit(CGF, D, KmpTaskSharedsPtr, TDBase, KmpTaskTWithPrivatesQTyRD, in emitTaskDupFunction()
3655 CGF.FinishFunction(); in emitTaskDupFunction()
3678 CodeGenFunction &CGF; member in __anonffc6c7120f11::OMPIteratorGeneratorScope
3686 OMPIteratorGeneratorScope(CodeGenFunction &CGF, const OMPIteratorExpr *E) in OMPIteratorGeneratorScope() argument
3687 : CodeGenFunction::OMPPrivateScope(CGF), CGF(CGF), E(E) { in OMPIteratorGeneratorScope()
3692 Uppers.push_back(CGF.EmitScalarExpr(E->getHelper(I).Upper)); in OMPIteratorGeneratorScope()
3694 addPrivate(VD, CGF.CreateMemTemp(VD->getType(), VD->getName())); in OMPIteratorGeneratorScope()
3698 CGF.CreateMemTemp(HelperData.CounterVD->getType(), "counter.addr")); in OMPIteratorGeneratorScope()
3705 CGF.MakeAddrLValue(CGF.GetAddrOfLocalVar(HelperData.CounterVD), in OMPIteratorGeneratorScope()
3708 CGF.EmitStoreOfScalar( in OMPIteratorGeneratorScope()
3709 llvm::ConstantInt::get(CLVal.getAddress(CGF).getElementType(), 0), in OMPIteratorGeneratorScope()
3712 ContDests.emplace_back(CGF.getJumpDestInCurrentScope("iter.cont")); in OMPIteratorGeneratorScope()
3714 ExitDests.emplace_back(CGF.getJumpDestInCurrentScope("iter.exit")); in OMPIteratorGeneratorScope()
3719 CGF.EmitBlock(ContDest.getBlock()); in OMPIteratorGeneratorScope()
3721 CGF.EmitLoadOfScalar(CLVal, HelperData.CounterVD->getLocation()); in OMPIteratorGeneratorScope()
3724 ? CGF.Builder.CreateICmpSLT(CVal, N) in OMPIteratorGeneratorScope()
3725 : CGF.Builder.CreateICmpULT(CVal, N); in OMPIteratorGeneratorScope()
3726 llvm::BasicBlock *BodyBB = CGF.createBasicBlock("iter.body"); in OMPIteratorGeneratorScope()
3727 CGF.Builder.CreateCondBr(Cmp, BodyBB, ExitDest.getBlock()); in OMPIteratorGeneratorScope()
3729 CGF.EmitBlock(BodyBB); in OMPIteratorGeneratorScope()
3731 CGF.EmitIgnoredExpr(HelperData.Update); in OMPIteratorGeneratorScope()
3740 CGF.EmitIgnoredExpr(HelperData.CounterUpdate); in ~OMPIteratorGeneratorScope()
3742 CGF.EmitBranchThroughCleanup(ContDests[I - 1]); in ~OMPIteratorGeneratorScope()
3744 CGF.EmitBlock(ExitDests[I - 1].getBlock(), /*IsFinished=*/I == 1); in ~OMPIteratorGeneratorScope()
3751 getPointerAndSize(CodeGenFunction &CGF, const Expr *E) { in getPointerAndSize() argument
3756 Addr = CGF.EmitScalarExpr(Base); in getPointerAndSize()
3758 Addr = CGF.EmitLValue(E).getPointer(CGF); in getPointerAndSize()
3763 SizeVal = CGF.getTypeSize(OASE->getBase()->getType()->getPointeeType()); in getPointerAndSize()
3765 llvm::Value *Sz = CGF.EmitScalarExpr(SE); in getPointerAndSize()
3766 Sz = CGF.EmitScalarConversion( in getPointerAndSize()
3767 Sz, SE->getType(), CGF.getContext().getSizeType(), SE->getExprLoc()); in getPointerAndSize()
3768 SizeVal = CGF.Builder.CreateNUWMul(SizeVal, Sz); in getPointerAndSize()
3773 CGF.EmitOMPArraySectionExpr(ASE, /*IsLowerBound=*/false); in getPointerAndSize()
3774 Address UpAddrAddress = UpAddrLVal.getAddress(CGF); in getPointerAndSize()
3775 llvm::Value *UpAddr = CGF.Builder.CreateConstGEP1_32( in getPointerAndSize()
3777 llvm::Value *LowIntPtr = CGF.Builder.CreatePtrToInt(Addr, CGF.SizeTy); in getPointerAndSize()
3778 llvm::Value *UpIntPtr = CGF.Builder.CreatePtrToInt(UpAddr, CGF.SizeTy); in getPointerAndSize()
3779 SizeVal = CGF.Builder.CreateNUWSub(UpIntPtr, LowIntPtr); in getPointerAndSize()
3781 SizeVal = CGF.getTypeSize(Ty); in getPointerAndSize()
3802 CGOpenMPRuntime::emitTaskInit(CodeGenFunction &CGF, SourceLocation Loc, in emitTaskInit() argument
3877 llvm::Type *KmpTaskTWithPrivatesTy = CGF.ConvertType(KmpTaskTWithPrivatesQTy); in emitTaskInit()
3881 CGF.getTypeSize(KmpTaskTWithPrivatesQTy); in emitTaskInit()
3892 TaskPrivatesMap = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitTaskInit()
3932 ? CGF.Builder.CreateSelect(Data.Final.getPointer(), in emitTaskInit()
3933 CGF.Builder.getInt32(FinalFlag), in emitTaskInit()
3934 CGF.Builder.getInt32(/*C=*/0)) in emitTaskInit()
3935 : CGF.Builder.getInt32(Data.Final.getInt() ? FinalFlag : 0); in emitTaskInit()
3936 TaskFlags = CGF.Builder.CreateOr(TaskFlags, CGF.Builder.getInt32(Flags)); in emitTaskInit()
3938 SmallVector<llvm::Value *, 8> AllocArgs = {emitUpdateLocation(CGF, Loc), in emitTaskInit()
3939 getThreadID(CGF, Loc), TaskFlags, KmpTaskTWithPrivatesTySize, in emitTaskInit()
3940 SharedsSize, CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitTaskInit()
3951 DeviceID = CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(Device), in emitTaskInit()
3952 CGF.Int64Ty, /*isSigned=*/true); in emitTaskInit()
3954 DeviceID = CGF.Builder.getInt64(OMP_DEVICEID_UNDEF); in emitTaskInit()
3956 NewTask = CGF.EmitRuntimeCall( in emitTaskInit()
3962 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in emitTaskInit()
3971 LValue EvtLVal = CGF.EmitLValue(Evt); in emitTaskInit()
3975 llvm::Value *Loc = emitUpdateLocation(CGF, DC->getBeginLoc()); in emitTaskInit()
3976 llvm::Value *Tid = getThreadID(CGF, DC->getBeginLoc()); in emitTaskInit()
3977 Tid = CGF.Builder.CreateIntCast(Tid, CGF.IntTy, /*isSigned=*/false); in emitTaskInit()
3978 llvm::Value *EvtVal = CGF.EmitRuntimeCall( in emitTaskInit()
3982 EvtVal = CGF.EmitScalarConversion(EvtVal, C.VoidPtrTy, Evt->getType(), in emitTaskInit()
3984 CGF.EmitStoreOfScalar(EvtVal, EvtLVal); in emitTaskInit()
3998 llvm::Value *Sz = CGF.EmitScalarExpr(IE->getHelper(I).Upper); in emitTaskInit()
3999 Sz = CGF.Builder.CreateIntCast(Sz, CGF.SizeTy, /*isSigned=*/false); in emitTaskInit()
4001 NumOfElements ? CGF.Builder.CreateNUWMul(NumOfElements, Sz) : Sz; in emitTaskInit()
4013 NumOfElements = CGF.Builder.CreateNUWAdd( in emitTaskInit()
4014 llvm::ConstantInt::get(CGF.SizeTy, NumAffinities), NumOfElements); in emitTaskInit()
4019 CodeGenFunction::OpaqueValueMapping OpaqueMap(CGF, OVE, in emitTaskInit()
4027 CGF.EmitVarDecl(*PD); in emitTaskInit()
4028 AffinitiesArray = CGF.GetAddrOfLocalVar(PD); in emitTaskInit()
4029 NumOfElements = CGF.Builder.CreateIntCast(NumOfElements, CGF.Int32Ty, in emitTaskInit()
4037 CGF.CreateMemTemp(KmpTaskAffinityInfoArrayTy, ".affs.arr.addr"); in emitTaskInit()
4038 AffinitiesArray = CGF.Builder.CreateConstArrayGEP(AffinitiesArray, 0); in emitTaskInit()
4055 std::tie(Addr, Size) = getPointerAndSize(CGF, E); in emitTaskInit()
4057 CGF.MakeAddrLValue(CGF.Builder.CreateConstGEP(AffinitiesArray, Pos), in emitTaskInit()
4060 LValue BaseAddrLVal = CGF.EmitLValueForField( in emitTaskInit()
4062 CGF.EmitStoreOfScalar(CGF.Builder.CreatePtrToInt(Addr, CGF.IntPtrTy), in emitTaskInit()
4065 LValue LenLVal = CGF.EmitLValueForField( in emitTaskInit()
4067 CGF.EmitStoreOfScalar(Size, LenLVal); in emitTaskInit()
4073 PosLVal = CGF.MakeAddrLValue( in emitTaskInit()
4074 CGF.CreateMemTemp(C.getSizeType(), "affs.counter.addr"), in emitTaskInit()
4076 CGF.EmitStoreOfScalar(llvm::ConstantInt::get(CGF.SizeTy, Pos), PosLVal); in emitTaskInit()
4084 CGF, cast_or_null<OMPIteratorExpr>(Modifier->IgnoreParenImpCasts())); in emitTaskInit()
4088 std::tie(Addr, Size) = getPointerAndSize(CGF, E); in emitTaskInit()
4089 llvm::Value *Idx = CGF.EmitLoadOfScalar(PosLVal, E->getExprLoc()); in emitTaskInit()
4090 LValue Base = CGF.MakeAddrLValue( in emitTaskInit()
4091 CGF.Builder.CreateGEP(AffinitiesArray, Idx), KmpTaskAffinityInfoTy); in emitTaskInit()
4093 LValue BaseAddrLVal = CGF.EmitLValueForField( in emitTaskInit()
4095 CGF.EmitStoreOfScalar(CGF.Builder.CreatePtrToInt(Addr, CGF.IntPtrTy), in emitTaskInit()
4098 LValue LenLVal = CGF.EmitLValueForField( in emitTaskInit()
4100 CGF.EmitStoreOfScalar(Size, LenLVal); in emitTaskInit()
4101 Idx = CGF.Builder.CreateNUWAdd( in emitTaskInit()
4103 CGF.EmitStoreOfScalar(Idx, PosLVal); in emitTaskInit()
4109 llvm::Value *LocRef = emitUpdateLocation(CGF, Loc); in emitTaskInit()
4110 llvm::Value *GTid = getThreadID(CGF, Loc); in emitTaskInit()
4111 llvm::Value *AffinListPtr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitTaskInit()
4115 (void)CGF.EmitRuntimeCall( in emitTaskInit()
4121 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitTaskInit()
4123 LValue Base = CGF.MakeNaturalAlignAddrLValue(NewTaskNewTaskTTy, in emitTaskInit()
4126 CGF.EmitLValueForField(Base, *KmpTaskTWithPrivatesQTyRD->field_begin()); in emitTaskInit()
4132 CGF.EmitLoadOfScalar( in emitTaskInit()
4133 CGF.EmitLValueForField( in emitTaskInit()
4137 CGF.Int8Ty, CGM.getNaturalTypeAlignment(SharedsTy)); in emitTaskInit()
4138 LValue Dest = CGF.MakeAddrLValue(KmpTaskSharedsPtr, SharedsTy); in emitTaskInit()
4139 LValue Src = CGF.MakeAddrLValue(Shareds, SharedsTy); in emitTaskInit()
4140 CGF.EmitAggregateCopy(Dest, Src, SharedsTy, AggValueSlot::DoesNotOverlap); in emitTaskInit()
4145 emitPrivatesInit(CGF, D, KmpTaskSharedsPtr, Base, KmpTaskTWithPrivatesQTyRD, in emitTaskInit()
4149 (!Data.LastprivateVars.empty() || checkInitIsRequired(CGF, Privates))) { in emitTaskInit()
4166 LValue Data1LV = CGF.EmitLValueForField(TDBase, *FI); in emitTaskInit()
4167 LValue DestructorsLV = CGF.EmitLValueForField( in emitTaskInit()
4169 CGF.EmitStoreOfScalar(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitTaskInit()
4175 LValue Data2LV = CGF.EmitLValueForField( in emitTaskInit()
4177 LValue PriorityLV = CGF.EmitLValueForField( in emitTaskInit()
4179 CGF.EmitStoreOfScalar(Data.Priority.getPointer(), PriorityLV); in emitTaskInit()
4236 CGOpenMPRuntime::getDepobjElements(CodeGenFunction &CGF, LValue DepobjLVal, in getDepobjElements() argument
4244 LValue Base = CGF.EmitLoadOfPointerLValue( in getDepobjElements()
4245 CGF.Builder.CreateElementBitCast( in getDepobjElements()
4246 DepobjLVal.getAddress(CGF), in getDepobjElements()
4247 CGF.ConvertTypeForMem(KmpDependInfoPtrTy)), in getDepobjElements()
4249 Address DepObjAddr = CGF.Builder.CreateGEP( in getDepobjElements()
4250 Base.getAddress(CGF), in getDepobjElements()
4251 llvm::ConstantInt::get(CGF.IntPtrTy, -1, /*isSigned=*/true)); in getDepobjElements()
4252 LValue NumDepsBase = CGF.MakeAddrLValue( in getDepobjElements()
4255 LValue BaseAddrLVal = CGF.EmitLValueForField( in getDepobjElements()
4259 llvm::Value *NumDeps = CGF.EmitLoadOfScalar(BaseAddrLVal, Loc); in getDepobjElements()
4263 static void emitDependData(CodeGenFunction &CGF, QualType &KmpDependInfoTy, in emitDependData() argument
4267 CodeGenModule &CGM = CGF.CGM; in emitDependData()
4273 llvm::Type *LLVMFlagsTy = CGF.ConvertTypeForMem(FlagsTy); in emitDependData()
4276 CGF, cast_or_null<OMPIteratorExpr>( in emitDependData()
4285 std::tie(Addr, Size) = getPointerAndSize(CGF, E); in emitDependData()
4286 Addr = CGF.Builder.CreatePtrToInt(Addr, CGF.IntPtrTy); in emitDependData()
4288 Addr = llvm::ConstantInt::get(CGF.IntPtrTy, 0); in emitDependData()
4289 Size = llvm::ConstantInt::get(CGF.SizeTy, 0); in emitDependData()
4293 Base = CGF.MakeAddrLValue( in emitDependData()
4294 CGF.Builder.CreateConstGEP(DependenciesArray, *P), KmpDependInfoTy); in emitDependData()
4298 llvm::Value *Idx = CGF.EmitLoadOfScalar(PosLVal, E->getExprLoc()); in emitDependData()
4299 Base = CGF.MakeAddrLValue( in emitDependData()
4300 CGF.Builder.CreateGEP(DependenciesArray, Idx), KmpDependInfoTy); in emitDependData()
4303 LValue BaseAddrLVal = CGF.EmitLValueForField( in emitDependData()
4307 CGF.EmitStoreOfScalar(Addr, BaseAddrLVal); in emitDependData()
4309 LValue LenLVal = CGF.EmitLValueForField( in emitDependData()
4312 CGF.EmitStoreOfScalar(Size, LenLVal); in emitDependData()
4315 LValue FlagsLVal = CGF.EmitLValueForField( in emitDependData()
4319 CGF.EmitStoreOfScalar( in emitDependData()
4326 llvm::Value *Idx = CGF.EmitLoadOfScalar(PosLVal, E->getExprLoc()); in emitDependData()
4327 Idx = CGF.Builder.CreateNUWAdd(Idx, in emitDependData()
4329 CGF.EmitStoreOfScalar(Idx, PosLVal); in emitDependData()
4335 CodeGenFunction &CGF, QualType &KmpDependInfoTy, in emitDepobjElementsSizes() argument
4341 ASTContext &C = CGF.getContext(); in emitDepobjElementsSizes()
4344 CGF, cast_or_null<OMPIteratorExpr>( in emitDepobjElementsSizes()
4350 LValue DepobjLVal = CGF.EmitLValue(E->IgnoreParenImpCasts()); in emitDepobjElementsSizes()
4352 getDepobjElements(CGF, DepobjLVal, E->getExprLoc()); in emitDepobjElementsSizes()
4353 LValue NumLVal = CGF.MakeAddrLValue( in emitDepobjElementsSizes()
4354 CGF.CreateMemTemp(C.getUIntPtrType(), "depobj.size.addr"), in emitDepobjElementsSizes()
4356 CGF.Builder.CreateStore(llvm::ConstantInt::get(CGF.IntPtrTy, 0), in emitDepobjElementsSizes()
4357 NumLVal.getAddress(CGF)); in emitDepobjElementsSizes()
4358 llvm::Value *PrevVal = CGF.EmitLoadOfScalar(NumLVal, E->getExprLoc()); in emitDepobjElementsSizes()
4359 llvm::Value *Add = CGF.Builder.CreateNUWAdd(PrevVal, NumDeps); in emitDepobjElementsSizes()
4360 CGF.EmitStoreOfScalar(Add, NumLVal); in emitDepobjElementsSizes()
4366 CGF.EmitLoadOfScalar(SizeLVals[I], Data.DepExprs[I]->getExprLoc()); in emitDepobjElementsSizes()
4372 void CGOpenMPRuntime::emitDepobjElements(CodeGenFunction &CGF, in emitDepobjElements() argument
4379 llvm::Value *ElSize = CGF.getTypeSize(KmpDependInfoTy); in emitDepobjElements()
4382 CGF, cast_or_null<OMPIteratorExpr>( in emitDepobjElements()
4389 LValue DepobjLVal = CGF.EmitLValue(E->IgnoreParenImpCasts()); in emitDepobjElements()
4391 getDepobjElements(CGF, DepobjLVal, E->getExprLoc()); in emitDepobjElements()
4394 llvm::Value *Size = CGF.Builder.CreateNUWMul( in emitDepobjElements()
4396 CGF.Builder.CreateIntCast(NumDeps, CGF.SizeTy, /*isSigned=*/false)); in emitDepobjElements()
4397 llvm::Value *Pos = CGF.EmitLoadOfScalar(PosLVal, E->getExprLoc()); in emitDepobjElements()
4398 Address DepAddr = CGF.Builder.CreateGEP(DependenciesArray, Pos); in emitDepobjElements()
4399 CGF.Builder.CreateMemCpy(DepAddr, Base.getAddress(CGF), Size); in emitDepobjElements()
4403 llvm::Value *Add = CGF.Builder.CreateNUWAdd(Pos, NumDeps); in emitDepobjElements()
4404 CGF.EmitStoreOfScalar(Add, PosLVal); in emitDepobjElements()
4410 CodeGenFunction &CGF, ArrayRef<OMPTaskDataTy::DependData> Dependencies, in emitDependClause() argument
4431 llvm::Value *NumOfDepobjElements = llvm::ConstantInt::get(CGF.IntPtrTy, 0); in emitDependClause()
4433 llvm::ConstantInt::get(CGF.IntPtrTy, 0); in emitDependClause()
4439 emitDepobjElementsSizes(CGF, KmpDependInfoTy, D); in emitDependClause()
4442 CGF.Builder.CreateNUWAdd(NumOfDepobjElements, Size); in emitDependClause()
4451 llvm::Value *Sz = CGF.EmitScalarExpr(IE->getHelper(I).Upper); in emitDependClause()
4452 Sz = CGF.Builder.CreateIntCast(Sz, CGF.IntPtrTy, /*isSigned=*/false); in emitDependClause()
4453 llvm::Value *NumClauseDeps = CGF.Builder.CreateNUWMul( in emitDependClause()
4454 Sz, llvm::ConstantInt::get(CGF.IntPtrTy, D.DepExprs.size())); in emitDependClause()
4456 CGF.Builder.CreateNUWAdd(NumOfRegularWithIterators, NumClauseDeps); in emitDependClause()
4469 CGF.Builder.CreateNUWAdd(NumOfDepobjElements, NumOfElements); in emitDependClause()
4473 CGF.Builder.CreateNUWAdd(NumOfRegularWithIterators, NumOfElements); in emitDependClause()
4478 CodeGenFunction::OpaqueValueMapping OpaqueMap(CGF, OVE, in emitDependClause()
4487 CGF.EmitVarDecl(*PD); in emitDependClause()
4488 DependenciesArray = CGF.GetAddrOfLocalVar(PD); in emitDependClause()
4489 NumOfElements = CGF.Builder.CreateIntCast(NumOfElements, CGF.Int32Ty, in emitDependClause()
4496 CGF.CreateMemTemp(KmpDependInfoArrayTy, ".dep.arr.addr"); in emitDependClause()
4497 DependenciesArray = CGF.Builder.CreateConstArrayGEP(DependenciesArray, 0); in emitDependClause()
4506 emitDependData(CGF, KmpDependInfoTy, &Pos, Dependencies[I], in emitDependClause()
4510 LValue PosLVal = CGF.MakeAddrLValue( in emitDependClause()
4511 CGF.CreateMemTemp(C.getSizeType(), "dep.counter.addr"), C.getSizeType()); in emitDependClause()
4512 CGF.EmitStoreOfScalar(llvm::ConstantInt::get(CGF.SizeTy, Pos), PosLVal); in emitDependClause()
4517 emitDependData(CGF, KmpDependInfoTy, &PosLVal, Dependencies[I], in emitDependClause()
4525 emitDepobjElements(CGF, KmpDependInfoTy, PosLVal, Dependencies[I], in emitDependClause()
4529 DependenciesArray = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitDependClause()
4530 DependenciesArray, CGF.VoidPtrTy, CGF.Int8Ty); in emitDependClause()
4535 CodeGenFunction &CGF, const OMPTaskDataTy::DependData &Dependencies, in emitDepobjDependClause() argument
4557 NumDepsVal = llvm::ConstantInt::get(CGF.SizeTy, 1); in emitDepobjDependClause()
4559 llvm::Value *Sz = CGF.EmitScalarExpr(IE->getHelper(I).Upper); in emitDepobjDependClause()
4560 Sz = CGF.Builder.CreateIntCast(Sz, CGF.SizeTy, /*isSigned=*/false); in emitDepobjDependClause()
4561 NumDepsVal = CGF.Builder.CreateNUWMul(NumDepsVal, Sz); in emitDepobjDependClause()
4563 Size = CGF.Builder.CreateNUWAdd(llvm::ConstantInt::get(CGF.SizeTy, 1), in emitDepobjDependClause()
4568 Size = CGF.Builder.CreateNUWMul(Size, RecSize); in emitDepobjDependClause()
4570 CGF.Builder.CreateIntCast(NumDepsVal, CGF.IntPtrTy, /*isSigned=*/false); in emitDepobjDependClause()
4577 NumDepsVal = llvm::ConstantInt::get(CGF.IntPtrTy, NumDependencies); in emitDepobjDependClause()
4580 llvm::Value *ThreadID = getThreadID(CGF, Loc); in emitDepobjDependClause()
4582 llvm::Value *Allocator = llvm::ConstantPointerNull::get(CGF.VoidPtrTy); in emitDepobjDependClause()
4586 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in emitDepobjDependClause()
4589 llvm::Type *KmpDependInfoLlvmTy = CGF.ConvertTypeForMem(KmpDependInfoTy); in emitDepobjDependClause()
4590 Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitDepobjDependClause()
4594 LValue Base = CGF.MakeAddrLValue(DependenciesArray, KmpDependInfoTy); in emitDepobjDependClause()
4596 LValue BaseAddrLVal = CGF.EmitLValueForField( in emitDepobjDependClause()
4600 CGF.EmitStoreOfScalar(NumDepsVal, BaseAddrLVal); in emitDepobjDependClause()
4605 PosLVal = CGF.MakeAddrLValue( in emitDepobjDependClause()
4606 CGF.CreateMemTemp(C.getSizeType(), "iterator.counter.addr"), in emitDepobjDependClause()
4608 CGF.EmitStoreOfScalar(llvm::ConstantInt::get(CGF.SizeTy, Idx), PosLVal, in emitDepobjDependClause()
4614 emitDependData(CGF, KmpDependInfoTy, Pos, Dependencies, DependenciesArray); in emitDepobjDependClause()
4615 DependenciesArray = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitDepobjDependClause()
4616 CGF.Builder.CreateConstGEP(DependenciesArray, 1), CGF.VoidPtrTy, in emitDepobjDependClause()
4617 CGF.Int8Ty); in emitDepobjDependClause()
4621 void CGOpenMPRuntime::emitDestroyClause(CodeGenFunction &CGF, LValue DepobjLVal, in emitDestroyClause() argument
4626 LValue Base = CGF.EmitLoadOfPointerLValue( in emitDestroyClause()
4627 DepobjLVal.getAddress(CGF), C.VoidPtrTy.castAs<PointerType>()); in emitDestroyClause()
4629 Address Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitDestroyClause()
4630 Base.getAddress(CGF), CGF.ConvertTypeForMem(KmpDependInfoPtrTy), in emitDestroyClause()
4631 CGF.ConvertTypeForMem(KmpDependInfoTy)); in emitDestroyClause()
4632 llvm::Value *DepObjAddr = CGF.Builder.CreateGEP( in emitDestroyClause()
4634 llvm::ConstantInt::get(CGF.IntPtrTy, -1, /*isSigned=*/true)); in emitDestroyClause()
4635 DepObjAddr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(DepObjAddr, in emitDestroyClause()
4636 CGF.VoidPtrTy); in emitDestroyClause()
4637 llvm::Value *ThreadID = getThreadID(CGF, Loc); in emitDestroyClause()
4639 llvm::Value *Allocator = llvm::ConstantPointerNull::get(CGF.VoidPtrTy); in emitDestroyClause()
4643 (void)CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in emitDestroyClause()
4648 void CGOpenMPRuntime::emitUpdateClause(CodeGenFunction &CGF, LValue DepobjLVal, in emitUpdateClause() argument
4656 llvm::Type *LLVMFlagsTy = CGF.ConvertTypeForMem(FlagsTy); in emitUpdateClause()
4659 std::tie(NumDeps, Base) = getDepobjElements(CGF, DepobjLVal, Loc); in emitUpdateClause()
4661 Address Begin = Base.getAddress(CGF); in emitUpdateClause()
4663 llvm::Value *End = CGF.Builder.CreateGEP( in emitUpdateClause()
4666 llvm::BasicBlock *BodyBB = CGF.createBasicBlock("omp.body"); in emitUpdateClause()
4667 llvm::BasicBlock *DoneBB = CGF.createBasicBlock("omp.done"); in emitUpdateClause()
4668 llvm::BasicBlock *EntryBB = CGF.Builder.GetInsertBlock(); in emitUpdateClause()
4669 CGF.EmitBlock(BodyBB); in emitUpdateClause()
4671 CGF.Builder.CreatePHI(Begin.getType(), 2, "omp.elementPast"); in emitUpdateClause()
4674 Base = CGF.MakeAddrLValue(Begin, KmpDependInfoTy, Base.getBaseInfo(), in emitUpdateClause()
4678 LValue FlagsLVal = CGF.EmitLValueForField( in emitUpdateClause()
4681 CGF.EmitStoreOfScalar( in emitUpdateClause()
4687 CGF.Builder.CreateConstGEP(Begin, /*Index=*/1, "omp.elementNext"); in emitUpdateClause()
4689 CGF.Builder.GetInsertBlock()); in emitUpdateClause()
4691 CGF.Builder.CreateICmpEQ(ElementNext.getPointer(), End, "omp.isempty"); in emitUpdateClause()
4692 CGF.Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB); in emitUpdateClause()
4694 CGF.EmitBlock(DoneBB, /*IsFinished=*/true); in emitUpdateClause()
4697 void CGOpenMPRuntime::emitTaskCall(CodeGenFunction &CGF, SourceLocation Loc, in emitTaskCall() argument
4703 if (!CGF.HaveInsertPoint()) in emitTaskCall()
4707 emitTaskInit(CGF, Loc, D, TaskFunction, SharedsTy, Shareds, Data); in emitTaskCall()
4717 emitDependClause(CGF, Data.Dependences, Loc); in emitTaskCall()
4725 llvm::Value *ThreadID = getThreadID(CGF, Loc); in emitTaskCall()
4726 llvm::Value *UpLoc = emitUpdateLocation(CGF, Loc); in emitTaskCall()
4735 DepTaskArgs[5] = CGF.Builder.getInt32(0); in emitTaskCall()
4736 DepTaskArgs[6] = llvm::ConstantPointerNull::get(CGF.VoidPtrTy); in emitTaskCall()
4739 &DepTaskArgs](CodeGenFunction &CGF, PrePostActionTy &) { in emitTaskCall() argument
4742 LValue PartIdLVal = CGF.EmitLValueForField(TDBase, *PartIdFI); in emitTaskCall()
4743 CGF.EmitStoreOfScalar(CGF.Builder.getInt32(0), PartIdLVal); in emitTaskCall()
4746 CGF.EmitRuntimeCall( in emitTaskCall()
4751 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in emitTaskCall()
4757 dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo)) in emitTaskCall()
4758 Region->emitUntiedSwitch(CGF); in emitTaskCall()
4767 DepWaitTaskArgs[4] = CGF.Builder.getInt32(0); in emitTaskCall()
4768 DepWaitTaskArgs[5] = llvm::ConstantPointerNull::get(CGF.VoidPtrTy); in emitTaskCall()
4770 llvm::ConstantInt::get(CGF.Int32Ty, Data.HasNowaitClause); in emitTaskCall()
4775 Loc](CodeGenFunction &CGF, PrePostActionTy &) { in emitTaskCall() argument
4776 CodeGenFunction::RunCleanupsScope LocalScope(CGF); in emitTaskCall()
4782 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in emitTaskCall()
4787 Loc](CodeGenFunction &CGF, PrePostActionTy &Action) { in emitTaskCall() argument
4788 Action.Enter(CGF); in emitTaskCall()
4790 CGF.CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF, Loc, TaskEntry, in emitTaskCall()
4806 RCG(CGF); in emitTaskCall()
4810 emitIfClause(CGF, IfCond, ThenCodeGen, ElseCodeGen); in emitTaskCall()
4813 ThenRCG(CGF); in emitTaskCall()
4817 void CGOpenMPRuntime::emitTaskLoopCall(CodeGenFunction &CGF, SourceLocation Loc, in emitTaskLoopCall() argument
4823 if (!CGF.HaveInsertPoint()) in emitTaskLoopCall()
4826 emitTaskInit(CGF, Loc, D, TaskFunction, SharedsTy, Shareds, Data); in emitTaskLoopCall()
4832 llvm::Value *ThreadID = getThreadID(CGF, Loc); in emitTaskLoopCall()
4833 llvm::Value *UpLoc = emitUpdateLocation(CGF, Loc); in emitTaskLoopCall()
4836 IfVal = CGF.Builder.CreateIntCast(CGF.EvaluateExprAsBool(IfCond), CGF.IntTy, in emitTaskLoopCall()
4839 IfVal = llvm::ConstantInt::getSigned(CGF.IntTy, /*V=*/1); in emitTaskLoopCall()
4842 LValue LBLVal = CGF.EmitLValueForField( in emitTaskLoopCall()
4847 CGF.EmitAnyExprToMem(LBVar->getInit(), LBLVal.getAddress(CGF), in emitTaskLoopCall()
4850 LValue UBLVal = CGF.EmitLValueForField( in emitTaskLoopCall()
4855 CGF.EmitAnyExprToMem(UBVar->getInit(), UBLVal.getAddress(CGF), in emitTaskLoopCall()
4858 LValue StLVal = CGF.EmitLValueForField( in emitTaskLoopCall()
4863 CGF.EmitAnyExprToMem(StVar->getInit(), StLVal.getAddress(CGF), in emitTaskLoopCall()
4867 LValue RedLVal = CGF.EmitLValueForField( in emitTaskLoopCall()
4871 CGF.EmitStoreOfScalar(Data.Reductions, RedLVal); in emitTaskLoopCall()
4873 CGF.EmitNullInitialization(RedLVal.getAddress(CGF), in emitTaskLoopCall()
4874 CGF.getContext().VoidPtrTy); in emitTaskLoopCall()
4882 LBLVal.getPointer(CGF), in emitTaskLoopCall()
4883 UBLVal.getPointer(CGF), in emitTaskLoopCall()
4884 CGF.EmitLoadOfScalar(StLVal, Loc), in emitTaskLoopCall()
4886 CGF.IntTy, 1), // Always 1 because taskgroup emitted by the compiler in emitTaskLoopCall()
4888 CGF.IntTy, Data.Schedule.getPointer() in emitTaskLoopCall()
4892 ? CGF.Builder.CreateIntCast(Data.Schedule.getPointer(), CGF.Int64Ty, in emitTaskLoopCall()
4894 : llvm::ConstantInt::get(CGF.Int64Ty, /*V=*/0), in emitTaskLoopCall()
4895 Result.TaskDupFn ? CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitTaskLoopCall()
4896 Result.TaskDupFn, CGF.VoidPtrTy) in emitTaskLoopCall()
4897 : llvm::ConstantPointerNull::get(CGF.VoidPtrTy)}; in emitTaskLoopCall()
4898 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in emitTaskLoopCall()
4913 CodeGenFunction &CGF, QualType Type, const VarDecl *LHSVar, in EmitOMPAggregateReduction() argument
4915 const llvm::function_ref<void(CodeGenFunction &CGF, const Expr *, in EmitOMPAggregateReduction() argument
4921 Address LHSAddr = CGF.GetAddrOfLocalVar(LHSVar); in EmitOMPAggregateReduction()
4922 Address RHSAddr = CGF.GetAddrOfLocalVar(RHSVar); in EmitOMPAggregateReduction()
4926 llvm::Value *NumElements = CGF.emitArrayLength(ArrayTy, ElementTy, LHSAddr); in EmitOMPAggregateReduction()
4932 CGF.Builder.CreateGEP(LHSAddr.getElementType(), LHSBegin, NumElements); in EmitOMPAggregateReduction()
4934 llvm::BasicBlock *BodyBB = CGF.createBasicBlock("omp.arraycpy.body"); in EmitOMPAggregateReduction()
4935 llvm::BasicBlock *DoneBB = CGF.createBasicBlock("omp.arraycpy.done"); in EmitOMPAggregateReduction()
4937 CGF.Builder.CreateICmpEQ(LHSBegin, LHSEnd, "omp.arraycpy.isempty"); in EmitOMPAggregateReduction()
4938 CGF.Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB); in EmitOMPAggregateReduction()
4941 llvm::BasicBlock *EntryBB = CGF.Builder.GetInsertBlock(); in EmitOMPAggregateReduction()
4942 CGF.EmitBlock(BodyBB); in EmitOMPAggregateReduction()
4944 CharUnits ElementSize = CGF.getContext().getTypeSizeInChars(ElementTy); in EmitOMPAggregateReduction()
4946 llvm::PHINode *RHSElementPHI = CGF.Builder.CreatePHI( in EmitOMPAggregateReduction()
4953 llvm::PHINode *LHSElementPHI = CGF.Builder.CreatePHI( in EmitOMPAggregateReduction()
4961 CodeGenFunction::OMPPrivateScope Scope(CGF); in EmitOMPAggregateReduction()
4965 RedOpGen(CGF, XExpr, EExpr, UpExpr); in EmitOMPAggregateReduction()
4969 llvm::Value *LHSElementNext = CGF.Builder.CreateConstGEP1_32( in EmitOMPAggregateReduction()
4972 llvm::Value *RHSElementNext = CGF.Builder.CreateConstGEP1_32( in EmitOMPAggregateReduction()
4977 CGF.Builder.CreateICmpEQ(LHSElementNext, LHSEnd, "omp.arraycpy.done"); in EmitOMPAggregateReduction()
4978 CGF.Builder.CreateCondBr(Done, DoneBB, BodyBB); in EmitOMPAggregateReduction()
4979 LHSElementPHI->addIncoming(LHSElementNext, CGF.Builder.GetInsertBlock()); in EmitOMPAggregateReduction()
4980 RHSElementPHI->addIncoming(RHSElementNext, CGF.Builder.GetInsertBlock()); in EmitOMPAggregateReduction()
4983 CGF.EmitBlock(DoneBB, /*IsFinished=*/true); in EmitOMPAggregateReduction()
4989 static void emitReductionCombiner(CodeGenFunction &CGF, in emitReductionCombiner() argument
4998 CGF.CGM.getOpenMPRuntime().getUserDefinedReduction(DRD); in emitReductionCombiner()
5000 CodeGenFunction::OpaqueValueMapping Map(CGF, OVE, Func); in emitReductionCombiner()
5001 CGF.EmitIgnoredExpr(ReductionOp); in emitReductionCombiner()
5004 CGF.EmitIgnoredExpr(ReductionOp); in emitReductionCombiner()
5029 CodeGenFunction CGF(CGM); in emitReductionFunction() local
5030 CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, CGFI, Args, Loc, Loc); in emitReductionFunction()
5034 Address LHS(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitReductionFunction()
5035 CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(&LHSArg)), in emitReductionFunction()
5037 ArgsElemType, CGF.getPointerAlign()); in emitReductionFunction()
5038 Address RHS(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitReductionFunction()
5039 CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(&RHSArg)), in emitReductionFunction()
5041 ArgsElemType, CGF.getPointerAlign()); in emitReductionFunction()
5046 CodeGenFunction::OMPPrivateScope Scope(CGF); in emitReductionFunction()
5052 Scope.addPrivate(RHSVar, emitAddrOfVarFromArray(CGF, RHS, Idx, RHSVar)); in emitReductionFunction()
5055 Scope.addPrivate(LHSVar, emitAddrOfVarFromArray(CGF, LHS, Idx, LHSVar)); in emitReductionFunction()
5060 Address Elem = CGF.Builder.CreateConstArrayGEP(LHS, Idx); in emitReductionFunction()
5061 llvm::Value *Ptr = CGF.Builder.CreateLoad(Elem); in emitReductionFunction()
5063 CGF.getContext().getAsVariableArrayType(PrivTy); in emitReductionFunction()
5066 CGF, OVE, RValue::get(CGF.Builder.CreatePtrToInt(Ptr, CGF.SizeTy))); in emitReductionFunction()
5067 CGF.EmitVariablyModifiedType(PrivTy); in emitReductionFunction()
5080 CGF, (*IPriv)->getType(), LHSVar, RHSVar, in emitReductionFunction()
5081 [=](CodeGenFunction &CGF, const Expr *, const Expr *, const Expr *) { in emitReductionFunction() argument
5082 emitReductionCombiner(CGF, E); in emitReductionFunction()
5086 emitReductionCombiner(CGF, E); in emitReductionFunction()
5093 CGF.FinishFunction(); in emitReductionFunction()
5097 void CGOpenMPRuntime::emitSingleReductionCombiner(CodeGenFunction &CGF, in emitSingleReductionCombiner() argument
5107 CGF, PrivateRef->getType(), LHSVar, RHSVar, in emitSingleReductionCombiner()
5108 [=](CodeGenFunction &CGF, const Expr *, const Expr *, const Expr *) { in emitSingleReductionCombiner() argument
5109 emitReductionCombiner(CGF, ReductionOp); in emitSingleReductionCombiner()
5113 emitReductionCombiner(CGF, ReductionOp); in emitSingleReductionCombiner()
5117 void CGOpenMPRuntime::emitReduction(CodeGenFunction &CGF, SourceLocation Loc, in emitReduction() argument
5123 if (!CGF.HaveInsertPoint()) in emitReduction()
5167 CodeGenFunction::RunCleanupsScope Scope(CGF); in emitReduction()
5172 emitSingleReductionCombiner(CGF, E, *IPriv, cast<DeclRefExpr>(*ILHS), in emitReduction()
5194 CGF.CreateMemTemp(ReductionArrayTy, ".omp.reduction.red_list"); in emitReduction()
5198 Address Elem = CGF.Builder.CreateConstArrayGEP(ReductionList, Idx); in emitReduction()
5199 CGF.Builder.CreateStore( in emitReduction()
5200 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitReduction()
5201 CGF.EmitLValue(RHSExprs[I]).getPointer(CGF), CGF.VoidPtrTy), in emitReduction()
5206 Elem = CGF.Builder.CreateConstArrayGEP(ReductionList, Idx); in emitReduction()
5207 llvm::Value *Size = CGF.Builder.CreateIntCast( in emitReduction()
5208 CGF.getVLASize( in emitReduction()
5209 CGF.getContext().getAsVariableArrayType((*IPriv)->getType())) in emitReduction()
5211 CGF.SizeTy, /*isSigned=*/false); in emitReduction()
5212 CGF.Builder.CreateStore(CGF.Builder.CreateIntToPtr(Size, CGF.VoidPtrTy), in emitReduction()
5219 emitReductionFunction(Loc, CGF.ConvertTypeForMem(ReductionArrayTy), in emitReduction()
5228 llvm::Value *IdentTLoc = emitUpdateLocation(CGF, Loc, OMP_ATOMIC_REDUCE); in emitReduction()
5229 llvm::Value *ThreadId = getThreadID(CGF, Loc); in emitReduction()
5230 llvm::Value *ReductionArrayTySize = CGF.getTypeSize(ReductionArrayTy); in emitReduction()
5231 llvm::Value *RL = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitReduction()
5232 ReductionList.getPointer(), CGF.VoidPtrTy); in emitReduction()
5236 CGF.Builder.getInt32(RHSExprs.size()), // i32 <n> in emitReduction()
5242 llvm::Value *Res = CGF.EmitRuntimeCall( in emitReduction()
5249 llvm::BasicBlock *DefaultBB = CGF.createBasicBlock(".omp.reduction.default"); in emitReduction()
5251 CGF.Builder.CreateSwitch(Res, DefaultBB, /*NumCases=*/2); in emitReduction()
5259 llvm::BasicBlock *Case1BB = CGF.createBasicBlock(".omp.reduction.case1"); in emitReduction()
5260 SwInst->addCase(CGF.Builder.getInt32(1), Case1BB); in emitReduction()
5261 CGF.EmitBlock(Case1BB); in emitReduction()
5270 CodeGenFunction &CGF, PrePostActionTy &Action) { in emitReduction() argument
5271 CGOpenMPRuntime &RT = CGF.CGM.getOpenMPRuntime(); in emitReduction()
5276 RT.emitSingleReductionCombiner(CGF, E, *IPriv, cast<DeclRefExpr>(*ILHS), in emitReduction()
5291 RCG(CGF); in emitReduction()
5293 CGF.EmitBranch(DefaultBB); in emitReduction()
5300 llvm::BasicBlock *Case2BB = CGF.createBasicBlock(".omp.reduction.case2"); in emitReduction()
5301 SwInst->addCase(CGF.Builder.getInt32(2), Case2BB); in emitReduction()
5302 CGF.EmitBlock(Case2BB); in emitReduction()
5305 CodeGenFunction &CGF, PrePostActionTy &Action) { in emitReduction() argument
5339 Loc](CodeGenFunction &CGF, const Expr *XExpr, in emitReduction()
5341 LValue X = CGF.EmitLValue(XExpr); in emitReduction()
5344 E = CGF.EmitAnyExpr(EExpr); in emitReduction()
5345 CGF.EmitOMPAtomicSimpleUpdateExpr( in emitReduction()
5348 [&CGF, UpExpr, VD, Loc](RValue XRValue) { in emitReduction()
5349 CodeGenFunction::OMPPrivateScope PrivateScope(CGF); in emitReduction()
5350 Address LHSTemp = CGF.CreateMemTemp(VD->getType()); in emitReduction()
5351 CGF.emitOMPSimpleStore( in emitReduction()
5352 CGF.MakeAddrLValue(LHSTemp, VD->getType()), XRValue, in emitReduction()
5356 return CGF.EmitAnyExpr(UpExpr); in emitReduction()
5363 EmitOMPAggregateReduction(CGF, (*IPriv)->getType(), VD, RHSVar, in emitReduction()
5367 AtomicRedGen(CGF, XExpr, EExpr, UpExpr); in emitReduction()
5371 auto &&CritRedGen = [E, Loc](CodeGenFunction &CGF, const Expr *, in emitReduction()
5373 CGOpenMPRuntime &RT = CGF.CGM.getOpenMPRuntime(); in emitReduction()
5376 CGF, Name, in emitReduction()
5377 [=](CodeGenFunction &CGF, PrePostActionTy &Action) { in emitReduction() argument
5378 Action.Enter(CGF); in emitReduction()
5379 emitReductionCombiner(CGF, E); in emitReduction()
5388 EmitOMPAggregateReduction(CGF, (*IPriv)->getType(), LHSVar, RHSVar, in emitReduction()
5391 CritRedGen(CGF, nullptr, nullptr, nullptr); in emitReduction()
5412 AtomicRCG(CGF); in emitReduction()
5414 AtomicRCG(CGF); in emitReduction()
5417 CGF.EmitBranch(DefaultBB); in emitReduction()
5418 CGF.EmitBlock(DefaultBB, /*IsFinished=*/true); in emitReduction()
5468 CodeGenFunction CGF(CGM); in emitReduceInitFunction() local
5469 CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, FnInfo, Args, Loc, Loc); in emitReduceInitFunction()
5471 Address PrivateAddr = CGF.EmitLoadOfPointer( in emitReduceInitFunction()
5472 CGF.Builder.CreateElementBitCast( in emitReduceInitFunction()
5473 CGF.GetAddrOfLocalVar(&Param), in emitReduceInitFunction()
5474 CGF.ConvertTypeForMem(PrivateType)->getPointerTo()), in emitReduceInitFunction()
5481 CGF, CGM.getContext().getSizeType(), in emitReduceInitFunction()
5483 Size = CGF.EmitLoadOfScalar(SizeAddr, /*Volatile=*/false, in emitReduceInitFunction()
5486 RCG.emitAggregateType(CGF, N, Size); in emitReduceInitFunction()
5492 Address SharedAddr = CGF.GetAddrOfLocalVar(&ParamOrig); in emitReduceInitFunction()
5493 OrigAddr = CGF.EmitLoadOfPointer( in emitReduceInitFunction()
5500 RCG.emitInitialization(CGF, N, PrivateAddr, OrigAddr, in emitReduceInitFunction()
5502 CGF.FinishFunction(); in emitReduceInitFunction()
5540 CodeGenFunction CGF(CGM); in emitReduceCombFunction() local
5541 CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, FnInfo, Args, Loc, Loc); in emitReduceCombFunction()
5547 CGF, CGM.getContext().getSizeType(), in emitReduceCombFunction()
5549 Size = CGF.EmitLoadOfScalar(SizeAddr, /*Volatile=*/false, in emitReduceCombFunction()
5552 RCG.emitAggregateType(CGF, N, Size); in emitReduceCombFunction()
5556 CodeGenFunction::OMPPrivateScope PrivateScope(CGF); in emitReduceCombFunction()
5560 CGF.EmitLoadOfPointer( in emitReduceCombFunction()
5561 CGF.Builder.CreateElementBitCast( in emitReduceCombFunction()
5562 CGF.GetAddrOfLocalVar(&ParamInOut), in emitReduceCombFunction()
5563 CGF.ConvertTypeForMem(LHSVD->getType())->getPointerTo()), in emitReduceCombFunction()
5568 CGF.EmitLoadOfPointer( in emitReduceCombFunction()
5569 CGF.Builder.CreateElementBitCast( in emitReduceCombFunction()
5570 CGF.GetAddrOfLocalVar(&ParamIn), in emitReduceCombFunction()
5571 CGF.ConvertTypeForMem(RHSVD->getType())->getPointerTo()), in emitReduceCombFunction()
5578 CGF, ReductionOp, PrivateRef, cast<DeclRefExpr>(LHS), in emitReduceCombFunction()
5580 CGF.FinishFunction(); in emitReduceCombFunction()
5610 CodeGenFunction CGF(CGM); in emitReduceFiniFunction() local
5611 CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, FnInfo, Args, Loc, Loc); in emitReduceFiniFunction()
5612 Address PrivateAddr = CGF.EmitLoadOfPointer( in emitReduceFiniFunction()
5613 CGF.GetAddrOfLocalVar(&Param), C.VoidPtrTy.castAs<PointerType>()); in emitReduceFiniFunction()
5619 CGF, CGM.getContext().getSizeType(), in emitReduceFiniFunction()
5621 Size = CGF.EmitLoadOfScalar(SizeAddr, /*Volatile=*/false, in emitReduceFiniFunction()
5624 RCG.emitAggregateType(CGF, N, Size); in emitReduceFiniFunction()
5627 RCG.emitCleanups(CGF, N, PrivateAddr); in emitReduceFiniFunction()
5628 CGF.FinishFunction(Loc); in emitReduceFiniFunction()
5633 CodeGenFunction &CGF, SourceLocation Loc, ArrayRef<const Expr *> LHSExprs, in emitTaskReductionInit() argument
5635 if (!CGF.HaveInsertPoint() || Data.ReductionVars.empty()) in emitTaskReductionInit()
5666 Address TaskRedInput = CGF.CreateMemTemp(ArrayRDType, ".rd_input."); in emitTaskReductionInit()
5673 llvm::Value *GEP = CGF.EmitCheckedInBoundsGEP( in emitTaskReductionInit()
5677 LValue ElemLVal = CGF.MakeNaturalAlignAddrLValue(GEP, RDType); in emitTaskReductionInit()
5679 LValue SharedLVal = CGF.EmitLValueForField(ElemLVal, SharedFD); in emitTaskReductionInit()
5680 RCG.emitSharedOrigLValue(CGF, Cnt); in emitTaskReductionInit()
5682 CGF.EmitCastToVoidPtr(RCG.getSharedLValue(Cnt).getPointer(CGF)); in emitTaskReductionInit()
5683 CGF.EmitStoreOfScalar(CastedShared, SharedLVal); in emitTaskReductionInit()
5685 LValue OrigLVal = CGF.EmitLValueForField(ElemLVal, OrigFD); in emitTaskReductionInit()
5687 CGF.EmitCastToVoidPtr(RCG.getOrigLValue(Cnt).getPointer(CGF)); in emitTaskReductionInit()
5688 CGF.EmitStoreOfScalar(CastedOrig, OrigLVal); in emitTaskReductionInit()
5689 RCG.emitAggregateType(CGF, Cnt); in emitTaskReductionInit()
5699 SizeValInChars = CGF.Builder.CreateIntCast(SizeValInChars, CGM.SizeTy, in emitTaskReductionInit()
5701 LValue SizeLVal = CGF.EmitLValueForField(ElemLVal, SizeFD); in emitTaskReductionInit()
5702 CGF.EmitStoreOfScalar(SizeValInChars, SizeLVal); in emitTaskReductionInit()
5704 LValue InitLVal = CGF.EmitLValueForField(ElemLVal, InitFD); in emitTaskReductionInit()
5706 CGF.EmitCastToVoidPtr(emitReduceInitFunction(CGM, Loc, RCG, Cnt)); in emitTaskReductionInit()
5707 CGF.EmitStoreOfScalar(InitAddr, InitLVal); in emitTaskReductionInit()
5709 LValue FiniLVal = CGF.EmitLValueForField(ElemLVal, FiniFD); in emitTaskReductionInit()
5712 ? CGF.EmitCastToVoidPtr(Fini) in emitTaskReductionInit()
5714 CGF.EmitStoreOfScalar(FiniAddr, FiniLVal); in emitTaskReductionInit()
5716 LValue CombLVal = CGF.EmitLValueForField(ElemLVal, CombFD); in emitTaskReductionInit()
5717 llvm::Value *CombAddr = CGF.EmitCastToVoidPtr(emitReduceCombFunction( in emitTaskReductionInit()
5720 CGF.EmitStoreOfScalar(CombAddr, CombLVal); in emitTaskReductionInit()
5722 LValue FlagsLVal = CGF.EmitLValueForField(ElemLVal, FlagsFD); in emitTaskReductionInit()
5724 CGF.EmitStoreOfScalar( in emitTaskReductionInit()
5728 CGF.EmitNullInitialization(FlagsLVal.getAddress(CGF), in emitTaskReductionInit()
5734 llvm::Value *IdentTLoc = emitUpdateLocation(CGF, Loc); in emitTaskReductionInit()
5735 llvm::Value *GTid = CGF.Builder.CreateIntCast(getThreadID(CGF, Loc), in emitTaskReductionInit()
5742 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitTaskReductionInit()
5744 return CGF.EmitRuntimeCall( in emitTaskReductionInit()
5751 CGF.Builder.CreateIntCast(getThreadID(CGF, Loc), CGM.IntTy, in emitTaskReductionInit()
5754 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(TaskRedInput.getPointer(), in emitTaskReductionInit()
5756 return CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in emitTaskReductionInit()
5761 void CGOpenMPRuntime::emitTaskReductionFini(CodeGenFunction &CGF, in emitTaskReductionFini() argument
5766 llvm::Value *IdentTLoc = emitUpdateLocation(CGF, Loc); in emitTaskReductionFini()
5767 llvm::Value *GTid = CGF.Builder.CreateIntCast(getThreadID(CGF, Loc), in emitTaskReductionFini()
5773 (void)CGF.EmitRuntimeCall( in emitTaskReductionFini()
5779 void CGOpenMPRuntime::emitTaskReductionFixups(CodeGenFunction &CGF, in emitTaskReductionFixups() argument
5787 llvm::Value *SizeVal = CGF.Builder.CreateIntCast(Sizes.second, CGM.SizeTy, in emitTaskReductionFixups()
5790 CGF, CGM.getContext().getSizeType(), in emitTaskReductionFixups()
5792 CGF.Builder.CreateStore(SizeVal, SizeAddr, /*IsVolatile=*/false); in emitTaskReductionFixups()
5796 Address CGOpenMPRuntime::getTaskReductionItem(CodeGenFunction &CGF, in getTaskReductionItem() argument
5802 llvm::Value *Args[] = {CGF.Builder.CreateIntCast(getThreadID(CGF, Loc), in getTaskReductionItem()
5806 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in getTaskReductionItem()
5807 SharedLVal.getPointer(CGF), CGM.VoidPtrTy)}; in getTaskReductionItem()
5809 CGF.EmitRuntimeCall( in getTaskReductionItem()
5813 CGF.Int8Ty, SharedLVal.getAlignment()); in getTaskReductionItem()
5816 void CGOpenMPRuntime::emitTaskwaitCall(CodeGenFunction &CGF, SourceLocation Loc, in emitTaskwaitCall() argument
5818 if (!CGF.HaveInsertPoint()) in emitTaskwaitCall()
5821 if (CGF.CGM.getLangOpts().OpenMPIRBuilder && Data.Dependences.empty()) { in emitTaskwaitCall()
5823 OMPBuilder.createTaskwait(CGF.Builder); in emitTaskwaitCall()
5825 llvm::Value *ThreadID = getThreadID(CGF, Loc); in emitTaskwaitCall()
5826 llvm::Value *UpLoc = emitUpdateLocation(CGF, Loc); in emitTaskwaitCall()
5831 emitDependClause(CGF, Data.Dependences, Loc); in emitTaskwaitCall()
5838 DepWaitTaskArgs[4] = CGF.Builder.getInt32(0); in emitTaskwaitCall()
5839 DepWaitTaskArgs[5] = llvm::ConstantPointerNull::get(CGF.VoidPtrTy); in emitTaskwaitCall()
5841 llvm::ConstantInt::get(CGF.Int32Ty, Data.HasNowaitClause); in emitTaskwaitCall()
5843 CodeGenFunction::RunCleanupsScope LocalScope(CGF); in emitTaskwaitCall()
5849 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in emitTaskwaitCall()
5859 CGF.EmitRuntimeCall( in emitTaskwaitCall()
5865 if (auto *Region = dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo)) in emitTaskwaitCall()
5866 Region->emitUntiedSwitch(CGF); in emitTaskwaitCall()
5869 void CGOpenMPRuntime::emitInlinedDirective(CodeGenFunction &CGF, in emitInlinedDirective() argument
5873 if (!CGF.HaveInsertPoint()) in emitInlinedDirective()
5875 InlinedOpenMPRegionRAII Region(CGF, CodeGen, InnerKind, HasCancel, in emitInlinedDirective()
5879 CGF.CapturedStmtInfo->EmitBody(CGF, /*S=*/nullptr); in emitInlinedDirective()
5908 CodeGenFunction &CGF, SourceLocation Loc, in emitCancellationPointCall() argument
5910 if (!CGF.HaveInsertPoint()) in emitCancellationPointCall()
5915 dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo)) { in emitCancellationPointCall()
5920 emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc), in emitCancellationPointCall()
5921 CGF.Builder.getInt32(getCancellationKind(CancelRegion))}; in emitCancellationPointCall()
5923 llvm::Value *Result = CGF.EmitRuntimeCall( in emitCancellationPointCall()
5931 llvm::BasicBlock *ExitBB = CGF.createBasicBlock(".cancel.exit"); in emitCancellationPointCall()
5932 llvm::BasicBlock *ContBB = CGF.createBasicBlock(".cancel.continue"); in emitCancellationPointCall()
5933 llvm::Value *Cmp = CGF.Builder.CreateIsNotNull(Result); in emitCancellationPointCall()
5934 CGF.Builder.CreateCondBr(Cmp, ExitBB, ContBB); in emitCancellationPointCall()
5935 CGF.EmitBlock(ExitBB); in emitCancellationPointCall()
5937 emitBarrierCall(CGF, Loc, OMPD_unknown, /*EmitChecks=*/false); in emitCancellationPointCall()
5940 CGF.getOMPCancelDestination(OMPRegionInfo->getDirectiveKind()); in emitCancellationPointCall()
5941 CGF.EmitBranchThroughCleanup(CancelDest); in emitCancellationPointCall()
5942 CGF.EmitBlock(ContBB, /*IsFinished=*/true); in emitCancellationPointCall()
5947 void CGOpenMPRuntime::emitCancelCall(CodeGenFunction &CGF, SourceLocation Loc, in emitCancelCall() argument
5950 if (!CGF.HaveInsertPoint()) in emitCancelCall()
5956 dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo)) { in emitCancelCall()
5958 OMPRegionInfo](CodeGenFunction &CGF, PrePostActionTy &) { in emitCancelCall() argument
5959 CGOpenMPRuntime &RT = CGF.CGM.getOpenMPRuntime(); in emitCancelCall()
5961 RT.emitUpdateLocation(CGF, Loc), RT.getThreadID(CGF, Loc), in emitCancelCall()
5962 CGF.Builder.getInt32(getCancellationKind(CancelRegion))}; in emitCancelCall()
5964 llvm::Value *Result = CGF.EmitRuntimeCall( in emitCancelCall()
5970 llvm::BasicBlock *ExitBB = CGF.createBasicBlock(".cancel.exit"); in emitCancelCall()
5971 llvm::BasicBlock *ContBB = CGF.createBasicBlock(".cancel.continue"); in emitCancelCall()
5972 llvm::Value *Cmp = CGF.Builder.CreateIsNotNull(Result); in emitCancelCall()
5973 CGF.Builder.CreateCondBr(Cmp, ExitBB, ContBB); in emitCancelCall()
5974 CGF.EmitBlock(ExitBB); in emitCancelCall()
5976 RT.emitBarrierCall(CGF, Loc, OMPD_unknown, /*EmitChecks=*/false); in emitCancelCall()
5979 CGF.getOMPCancelDestination(OMPRegionInfo->getDirectiveKind()); in emitCancelCall()
5980 CGF.EmitBranchThroughCleanup(CancelDest); in emitCancelCall()
5981 CGF.EmitBlock(ContBB, /*IsFinished=*/true); in emitCancelCall()
5984 emitIfClause(CGF, IfCond, ThenGen, in emitCancelCall()
5988 ThenRCG(CGF); in emitCancelCall()
6002 void Enter(CodeGenFunction &CGF) override { in Enter() argument
6003 if (!CGF.HaveInsertPoint()) in Enter()
6006 CGF.CGM.getOpenMPRuntime().emitUsesAllocatorsInit( in Enter()
6007 CGF, AllocatorData.first, AllocatorData.second); in Enter()
6010 void Exit(CodeGenFunction &CGF) override { in Exit() argument
6011 if (!CGF.HaveInsertPoint()) in Exit()
6014 CGF.CGM.getOpenMPRuntime().emitUsesAllocatorsFini(CGF, in Exit()
6042 void CGOpenMPRuntime::emitUsesAllocatorsInit(CodeGenFunction &CGF, in emitUsesAllocatorsInit() argument
6045 llvm::Value *ThreadId = getThreadID(CGF, Allocator->getExprLoc()); in emitUsesAllocatorsInit()
6046 ThreadId = CGF.Builder.CreateIntCast(ThreadId, CGF.IntTy, /*isSigned=*/true); in emitUsesAllocatorsInit()
6048 llvm::Value *MemSpaceHandle = llvm::ConstantPointerNull::get(CGF.VoidPtrTy); in emitUsesAllocatorsInit()
6050 CGF.IntTy, cast<ConstantArrayType>( in emitUsesAllocatorsInit()
6054 LValue AllocatorTraitsLVal = CGF.EmitLValue(AllocatorTraits); in emitUsesAllocatorsInit()
6055 Address Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitUsesAllocatorsInit()
6056 AllocatorTraitsLVal.getAddress(CGF), CGF.VoidPtrPtrTy, CGF.VoidPtrTy); in emitUsesAllocatorsInit()
6057 AllocatorTraitsLVal = CGF.MakeAddrLValue(Addr, CGF.getContext().VoidPtrTy, in emitUsesAllocatorsInit()
6061 CGF.EmitLoadOfScalar(AllocatorTraitsLVal, AllocatorTraits->getExprLoc()); in emitUsesAllocatorsInit()
6064 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in emitUsesAllocatorsInit()
6068 CGF.EmitVarDecl(*cast<VarDecl>( in emitUsesAllocatorsInit()
6070 LValue AllocatorLVal = CGF.EmitLValue(Allocator->IgnoreParenImpCasts()); in emitUsesAllocatorsInit()
6072 CGF.EmitScalarConversion(AllocatorVal, CGF.getContext().VoidPtrTy, in emitUsesAllocatorsInit()
6074 CGF.EmitStoreOfScalar(AllocatorVal, AllocatorLVal); in emitUsesAllocatorsInit()
6077 void CGOpenMPRuntime::emitUsesAllocatorsFini(CodeGenFunction &CGF, in emitUsesAllocatorsFini() argument
6079 llvm::Value *ThreadId = getThreadID(CGF, Allocator->getExprLoc()); in emitUsesAllocatorsFini()
6080 ThreadId = CGF.Builder.CreateIntCast(ThreadId, CGF.IntTy, /*isSigned=*/true); in emitUsesAllocatorsFini()
6081 LValue AllocatorLVal = CGF.EmitLValue(Allocator->IgnoreParenImpCasts()); in emitUsesAllocatorsFini()
6083 CGF.EmitLoadOfScalar(AllocatorLVal, Allocator->getExprLoc()); in emitUsesAllocatorsFini()
6084 AllocatorVal = CGF.EmitScalarConversion(AllocatorVal, Allocator->getType(), in emitUsesAllocatorsFini()
6085 CGF.getContext().VoidPtrTy, in emitUsesAllocatorsFini()
6087 (void)CGF.EmitRuntimeCall( in emitUsesAllocatorsFini()
6101 CodeGenFunction CGF(CGM, true); in emitTargetOutlinedFunctionHelper() local
6103 [&CGF, &D, &CodeGen](StringRef EntryFnName) { in emitTargetOutlinedFunctionHelper()
6107 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo); in emitTargetOutlinedFunctionHelper()
6108 return CGF.GenerateOpenMPCapturedStmtFunction(CS, D.getBeginLoc()); in emitTargetOutlinedFunctionHelper()
6114 getNumTeamsExprForTargetDirective(CGF, D, DefaultValTeams); in emitTargetOutlinedFunctionHelper()
6115 getNumThreadsExprForTargetDirective(CGF, D, DefaultValThreads); in emitTargetOutlinedFunctionHelper()
6179 CodeGenFunction &CGF, const OMPExecutableDirective &D, in getNumTeamsExprForTargetDirective() argument
6191 CGOpenMPRuntime::getSingleCompoundChild(CGF.getContext(), Body); in getNumTeamsExprForTargetDirective()
6198 if (NumTeams->isIntegerConstantExpr(CGF.getContext())) in getNumTeamsExprForTargetDirective()
6200 NumTeams->getIntegerConstantExpr(CGF.getContext())) in getNumTeamsExprForTargetDirective()
6227 if (NumTeams->isIntegerConstantExpr(CGF.getContext())) in getNumTeamsExprForTargetDirective()
6228 if (auto Constant = NumTeams->getIntegerConstantExpr(CGF.getContext())) in getNumTeamsExprForTargetDirective()
6308 CodeGenFunction &CGF, const OMPExecutableDirective &D) { in emitNumTeamsForTargetDirective() argument
6309 assert(!CGF.getLangOpts().OpenMPIsDevice && in emitNumTeamsForTargetDirective()
6312 CGBuilderTy &Bld = CGF.Builder; in emitNumTeamsForTargetDirective()
6314 const Expr *NumTeams = getNumTeamsExprForTargetDirective(CGF, D, DefaultNT); in emitNumTeamsForTargetDirective()
6321 CGOpenMPInnerExprInfo CGInfo(CGF, *CS); in emitNumTeamsForTargetDirective()
6322 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo); in emitNumTeamsForTargetDirective()
6323 llvm::Value *NumTeamsVal = CGF.EmitScalarExpr(NumTeams, in emitNumTeamsForTargetDirective()
6325 return Bld.CreateIntCast(NumTeamsVal, CGF.Int32Ty, in emitNumTeamsForTargetDirective()
6333 CodeGenFunction::RunCleanupsScope NumTeamsScope(CGF); in emitNumTeamsForTargetDirective()
6334 llvm::Value *NumTeamsVal = CGF.EmitScalarExpr(NumTeams, in emitNumTeamsForTargetDirective()
6336 return Bld.CreateIntCast(NumTeamsVal, CGF.Int32Ty, in emitNumTeamsForTargetDirective()
6344 return llvm::ConstantInt::get(CGF.Int32Ty, DefaultNT); in emitNumTeamsForTargetDirective()
6347 static llvm::Value *getNumThreads(CodeGenFunction &CGF, const CapturedStmt *CS, in getNumThreads() argument
6350 CGF.getContext(), CS->getCapturedStmt()); in getNumThreads()
6358 CGOpenMPInnerExprInfo CGInfo(CGF, *CS); in getNumThreads()
6359 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo); in getNumThreads()
6371 if (Cond->EvaluateAsBooleanCondition(Result, CGF.getContext())) { in getNumThreads()
6373 return CGF.Builder.getInt32(1); in getNumThreads()
6375 CodeGenFunction::LexicalScope Scope(CGF, Cond->getSourceRange()); in getNumThreads()
6380 CGF.EmitVarDecl(cast<VarDecl>(*I)); in getNumThreads()
6383 CGF.EmitAutoVarAlloca(cast<VarDecl>(*I)); in getNumThreads()
6384 CGF.EmitAutoVarCleanups(Emission); in getNumThreads()
6388 CondVal = CGF.EvaluateExprAsBool(Cond); in getNumThreads()
6395 CGOpenMPInnerExprInfo CGInfo(CGF, *CS); in getNumThreads()
6396 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo); in getNumThreads()
6400 CGF, NumThreadsClause->getNumThreads()->getSourceRange()); in getNumThreads()
6405 CGF.EmitVarDecl(cast<VarDecl>(*I)); in getNumThreads()
6408 CGF.EmitAutoVarAlloca(cast<VarDecl>(*I)); in getNumThreads()
6409 CGF.EmitAutoVarCleanups(Emission); in getNumThreads()
6413 NumThreads = CGF.EmitScalarExpr(NumThreadsClause->getNumThreads()); in getNumThreads()
6414 NumThreads = CGF.Builder.CreateIntCast(NumThreads, CGF.Int32Ty, in getNumThreads()
6417 NumThreads = CGF.Builder.CreateSelect( in getNumThreads()
6418 CGF.Builder.CreateICmpULT(DefaultThreadLimitVal, NumThreads), in getNumThreads()
6422 : CGF.Builder.getInt32(0); in getNumThreads()
6426 NumThreads = CGF.Builder.CreateSelect(CondVal, NumThreads, in getNumThreads()
6427 CGF.Builder.getInt32(1)); in getNumThreads()
6432 return CGF.Builder.getInt32(1); in getNumThreads()
6438 CodeGenFunction &CGF, const OMPExecutableDirective &D, in getNumThreadsExprForTargetDirective() argument
6453 if (ThreadLimit->isIntegerConstantExpr(CGF.getContext())) in getNumThreadsExprForTargetDirective()
6455 ThreadLimit->getIntegerConstantExpr(CGF.getContext())) in getNumThreadsExprForTargetDirective()
6470 if (ThreadLimit->isIntegerConstantExpr(CGF.getContext())) in getNumThreadsExprForTargetDirective()
6472 ThreadLimit->getIntegerConstantExpr(CGF.getContext())) in getNumThreadsExprForTargetDirective()
6478 if (NumThreads->isIntegerConstantExpr(CGF.getContext())) { in getNumThreadsExprForTargetDirective()
6480 NumThreads->getIntegerConstantExpr(CGF.getContext())) { in getNumThreadsExprForTargetDirective()
6560 CodeGenFunction &CGF, const OMPExecutableDirective &D) { in emitNumThreadsForTargetDirective() argument
6561 assert(!CGF.getLangOpts().OpenMPIsDevice && in emitNumThreadsForTargetDirective()
6567 CGBuilderTy &Bld = CGF.Builder; in emitNumThreadsForTargetDirective()
6573 if (llvm::Value *NumThreads = getNumThreads(CGF, CS, ThreadLimitVal)) in emitNumThreadsForTargetDirective()
6576 CGF.getContext(), CS->getCapturedStmt()); in emitNumThreadsForTargetDirective()
6583 CGOpenMPInnerExprInfo CGInfo(CGF, *CS); in emitNumThreadsForTargetDirective()
6584 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo); in emitNumThreadsForTargetDirective()
6586 CGF, ThreadLimitClause->getThreadLimit()->getSourceRange()); in emitNumThreadsForTargetDirective()
6591 CGF.EmitVarDecl(cast<VarDecl>(*I)); in emitNumThreadsForTargetDirective()
6594 CGF.EmitAutoVarAlloca(cast<VarDecl>(*I)); in emitNumThreadsForTargetDirective()
6595 CGF.EmitAutoVarCleanups(Emission); in emitNumThreadsForTargetDirective()
6602 llvm::Value *ThreadLimit = CGF.EmitScalarExpr( in emitNumThreadsForTargetDirective()
6605 Bld.CreateIntCast(ThreadLimit, CGF.Int32Ty, /*isSigned=*/false); in emitNumThreadsForTargetDirective()
6612 CGF.getContext(), CS->getCapturedStmt()); in emitNumThreadsForTargetDirective()
6618 if (llvm::Value *NumThreads = getNumThreads(CGF, CS, ThreadLimitVal)) in emitNumThreadsForTargetDirective()
6628 CodeGenFunction::RunCleanupsScope ThreadLimitScope(CGF); in emitNumThreadsForTargetDirective()
6630 llvm::Value *ThreadLimit = CGF.EmitScalarExpr( in emitNumThreadsForTargetDirective()
6633 Bld.CreateIntCast(ThreadLimit, CGF.Int32Ty, /*isSigned=*/false); in emitNumThreadsForTargetDirective()
6636 if (llvm::Value *NumThreads = getNumThreads(CGF, CS, ThreadLimitVal)) in emitNumThreadsForTargetDirective()
6639 CGF.getContext(), CS->getCapturedStmt()); in emitNumThreadsForTargetDirective()
6643 if (llvm::Value *NumThreads = getNumThreads(CGF, CS, ThreadLimitVal)) in emitNumThreadsForTargetDirective()
6651 CodeGenFunction::RunCleanupsScope ThreadLimitScope(CGF); in emitNumThreadsForTargetDirective()
6653 llvm::Value *ThreadLimit = CGF.EmitScalarExpr( in emitNumThreadsForTargetDirective()
6656 Bld.CreateIntCast(ThreadLimit, CGF.Int32Ty, /*isSigned=*/false); in emitNumThreadsForTargetDirective()
6659 getNumThreads(CGF, D.getInnermostCapturedStmt(), ThreadLimitVal)) in emitNumThreadsForTargetDirective()
6682 if (Cond->EvaluateAsBooleanCondition(Result, CGF.getContext())) { in emitNumThreadsForTargetDirective()
6686 CodeGenFunction::RunCleanupsScope Scope(CGF); in emitNumThreadsForTargetDirective()
6687 CondVal = CGF.EvaluateExprAsBool(Cond); in emitNumThreadsForTargetDirective()
6692 CodeGenFunction::RunCleanupsScope ThreadLimitScope(CGF); in emitNumThreadsForTargetDirective()
6694 llvm::Value *ThreadLimit = CGF.EmitScalarExpr( in emitNumThreadsForTargetDirective()
6697 Bld.CreateIntCast(ThreadLimit, CGF.Int32Ty, /*isSigned=*/false); in emitNumThreadsForTargetDirective()
6700 CodeGenFunction::RunCleanupsScope NumThreadsScope(CGF); in emitNumThreadsForTargetDirective()
6702 llvm::Value *NumThreads = CGF.EmitScalarExpr( in emitNumThreadsForTargetDirective()
6705 Bld.CreateIntCast(NumThreads, CGF.Int32Ty, /*isSigned=*/false); in emitNumThreadsForTargetDirective()
6952 CodeGenFunction &CGF; member in __anonffc6c7122811::MappableExprsHandler
6982 CGF.getTypeSize(OAE->getBase()->getType()->getPointeeType()); in getExprTypeSize()
6984 llvm::Value *Sz = CGF.EmitScalarExpr(SE); in getExprTypeSize()
6985 Sz = CGF.EmitScalarConversion(Sz, SE->getType(), in getExprTypeSize()
6986 CGF.getContext().getSizeType(), in getExprTypeSize()
6988 Size = CGF.Builder.CreateNUWMul(Size, Sz); in getExprTypeSize()
7010 return CGF.getTypeSize(BaseTy); in getExprTypeSize()
7014 ElemSize = CGF.getTypeSize(PTy->getPointeeType().getCanonicalType()); in getExprTypeSize()
7018 ElemSize = CGF.getTypeSize(ATy->getElementType().getCanonicalType()); in getExprTypeSize()
7027 llvm::Value *LengthVal = CGF.EmitScalarExpr(LenExpr); in getExprTypeSize()
7028 LengthVal = CGF.EmitScalarConversion(LengthVal, LenExpr->getType(), in getExprTypeSize()
7029 CGF.getContext().getSizeType(), in getExprTypeSize()
7031 return CGF.Builder.CreateNUWMul(LengthVal, ElemSize); in getExprTypeSize()
7036 llvm::Value *LengthVal = CGF.getTypeSize(BaseTy); in getExprTypeSize()
7037 llvm::Value *LBVal = CGF.EmitScalarExpr(OAE->getLowerBound()); in getExprTypeSize()
7038 LBVal = CGF.EmitScalarConversion(LBVal, OAE->getLowerBound()->getType(), in getExprTypeSize()
7039 CGF.getContext().getSizeType(), in getExprTypeSize()
7041 LBVal = CGF.Builder.CreateNUWMul(LBVal, ElemSize); in getExprTypeSize()
7042 llvm::Value *Cmp = CGF.Builder.CreateICmpUGT(LengthVal, LBVal); in getExprTypeSize()
7043 llvm::Value *TrueVal = CGF.Builder.CreateNUWSub(LengthVal, LBVal); in getExprTypeSize()
7044 LengthVal = CGF.Builder.CreateSelect( in getExprTypeSize()
7045 Cmp, TrueVal, llvm::ConstantInt::get(CGF.SizeTy, 0)); in getExprTypeSize()
7048 return CGF.getTypeSize(ExprTy); in getExprTypeSize()
7136 if (!Length->EvaluateAsInt(Result, CGF.getContext())) in isFinalArraySectionExpression()
7350 BP = CGF.LoadCXXThisAddress(); in generateInfoForComponentList()
7354 BP = CGF.EmitOMPSharedLValue(AssocExpr).getAddress(CGF); in generateInfoForComponentList()
7358 CGF.EmitScalarExpr(OAShE->getBase()), in generateInfoForComponentList()
7359 CGF.ConvertTypeForMem(OAShE->getBase()->getType()->getPointeeType()), in generateInfoForComponentList()
7360 CGF.getContext().getTypeAlignInChars(OAShE->getBase()->getType())); in generateInfoForComponentList()
7364 BP = CGF.EmitOMPSharedLValue(AssocExpr).getAddress(CGF); in generateInfoForComponentList()
7372 CGF.CGM.getOpenMPRuntime().hasRequiresUnifiedSharedMemory())) { in generateInfoForComponentList()
7374 BP = CGF.CGM.getOpenMPRuntime().getAddrOfDeclareTargetVar(VD); in generateInfoForComponentList()
7389 if (CGF.CGM.getOpenMPRuntime().hasRequiresUnifiedSharedMemory() || in generateInfoForComponentList()
7391 BP = CGF.EmitLoadOfPointer(BP, Ty->castAs<PointerType>()); in generateInfoForComponentList()
7445 BP = CGF.EmitLoadOfPointer(BP, Ty->castAs<PointerType>()); in generateInfoForComponentList()
7511 auto &&EmitMemberExprBase = [](CodeGenFunction &CGF, in generateInfoForComponentList()
7521 CGF.EmitPointerWithAlignment(BaseExpr, &BaseInfo, &TBAAInfo); in generateInfoForComponentList()
7523 BaseLV = CGF.MakeAddrLValue(Addr, PtrTy, BaseInfo, TBAAInfo); in generateInfoForComponentList()
7525 BaseLV = CGF.EmitOMPSharedLValue(BaseExpr); in generateInfoForComponentList()
7531 Address(CGF.EmitScalarExpr(OAShE->getBase()), in generateInfoForComponentList()
7532 CGF.ConvertTypeForMem( in generateInfoForComponentList()
7534 CGF.getContext().getTypeAlignInChars( in generateInfoForComponentList()
7538 LValue BaseLVal = EmitMemberExprBase(CGF, ME); in generateInfoForComponentList()
7539 LowestElem = CGF.EmitLValueForFieldInitialization( in generateInfoForComponentList()
7541 .getAddress(CGF); in generateInfoForComponentList()
7542 LB = CGF.EmitLoadOfReferenceLValue(LowestElem, MapDecl->getType()) in generateInfoForComponentList()
7543 .getAddress(CGF); in generateInfoForComponentList()
7546 CGF.EmitOMPSharedLValue(I->getAssociatedExpression()) in generateInfoForComponentList()
7547 .getAddress(CGF); in generateInfoForComponentList()
7568 CharUnits TypeSize = CGF.getContext().getTypeSizeInChars( in generateInfoForComponentList()
7570 Address HB = CGF.Builder.CreateConstGEP( in generateInfoForComponentList()
7571 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in generateInfoForComponentList()
7572 LowestElem, CGF.VoidPtrTy, CGF.Int8Ty), in generateInfoForComponentList()
7602 LValue BaseLVal = EmitMemberExprBase(CGF, ME); in generateInfoForComponentList()
7604 CGF.EmitLValueForFieldInitialization(BaseLVal, FD) in generateInfoForComponentList()
7605 .getAddress(CGF); in generateInfoForComponentList()
7608 CGF.EmitOMPSharedLValue(MC.getAssociatedExpression()) in generateInfoForComponentList()
7609 .getAddress(CGF); in generateInfoForComponentList()
7611 Size = CGF.Builder.CreatePtrDiff( in generateInfoForComponentList()
7612 CGF.Int8Ty, CGF.EmitCastToVoidPtr(ComponentLB.getPointer()), in generateInfoForComponentList()
7613 CGF.EmitCastToVoidPtr(LB.getPointer())); in generateInfoForComponentList()
7621 CombinedInfo.Sizes.push_back(CGF.Builder.CreateIntCast( in generateInfoForComponentList()
7622 Size, CGF.Int64Ty, /*isSigned=*/true)); in generateInfoForComponentList()
7627 LB = CGF.Builder.CreateConstGEP(ComponentLB, 1); in generateInfoForComponentList()
7632 Size = CGF.Builder.CreatePtrDiff( in generateInfoForComponentList()
7633 CGF.Int8Ty, CGF.Builder.CreateConstGEP(HB, 1).getPointer(), in generateInfoForComponentList()
7634 CGF.EmitCastToVoidPtr(LB.getPointer())); in generateInfoForComponentList()
7636 CGF.Builder.CreateIntCast(Size, CGF.Int64Ty, /*isSigned=*/true)); in generateInfoForComponentList()
7650 CGF.Builder.CreateIntCast(Size, CGF.Int64Ty, /*isSigned=*/true)); in generateInfoForComponentList()
7703 CGF.EmitOMPArraySectionExpr(OASE, /*IsLowerBound=*/false) in generateInfoForComponentList()
7704 .getAddress(CGF); in generateInfoForComponentList()
7739 BP = CGF.EmitLoadOfPointer(BP, Ty->castAs<PointerType>()); in generateInfoForComponentList()
7751 const ASTContext &Context = CGF.getContext(); in generateInfoForComponentList()
7755 MapValuesArrayTy CurOffsets = {llvm::ConstantInt::get(CGF.CGM.Int64Ty, 0)}; in generateInfoForComponentList()
7756 MapValuesArrayTy CurCounts = {llvm::ConstantInt::get(CGF.CGM.Int64Ty, 1)}; in generateInfoForComponentList()
7758 MapValuesArrayTy DimSizes{llvm::ConstantInt::get(CGF.CGM.Int64Ty, 1)}; in generateInfoForComponentList()
7803 llvm::ConstantInt::get(CGF.Int64Ty, ElementTypeSize)); in generateInfoForComponentList()
7811 CGF.Int64Ty, CAT->getSize().getZExtValue())); in generateInfoForComponentList()
7813 DimSizes.push_back(CGF.Builder.CreateIntCast( in generateInfoForComponentList()
7814 CGF.EmitScalarExpr(VAT->getSizeExpr()), CGF.Int64Ty, in generateInfoForComponentList()
7823 llvm::ConstantInt::get(CGF.CGM.Int64Ty, ElementTypeSize); in generateInfoForComponentList()
7837 llvm::Value *Offset = CGF.Builder.CreateIntCast( in generateInfoForComponentList()
7838 CGF.EmitScalarExpr(AE->getIdx()), CGF.Int64Ty, in generateInfoForComponentList()
7841 CurCounts.push_back(llvm::ConstantInt::get(CGF.Int64Ty, /*V=*/1)); in generateInfoForComponentList()
7856 Offset = llvm::ConstantInt::get(CGF.Int64Ty, 0); in generateInfoForComponentList()
7858 Offset = CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(OffsetExpr), in generateInfoForComponentList()
7859 CGF.Int64Ty, in generateInfoForComponentList()
7874 Count = llvm::ConstantInt::get(CGF.Int64Ty, 1); in generateInfoForComponentList()
7883 ? CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(StrideExpr), in generateInfoForComponentList()
7884 CGF.Int64Ty, /*isSigned=*/false) in generateInfoForComponentList()
7887 Count = CGF.Builder.CreateUDiv( in generateInfoForComponentList()
7888 CGF.Builder.CreateNUWSub(*DI, Offset), Stride); in generateInfoForComponentList()
7890 Count = CGF.Builder.CreateNUWSub(*DI, Offset); in generateInfoForComponentList()
7893 Count = CGF.EmitScalarExpr(CountExpr); in generateInfoForComponentList()
7895 Count = CGF.Builder.CreateIntCast(Count, CGF.Int64Ty, /*isSigned=*/false); in generateInfoForComponentList()
7908 ? CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(StrideExpr), in generateInfoForComponentList()
7909 CGF.Int64Ty, /*isSigned=*/false) in generateInfoForComponentList()
7911 DimProd = CGF.Builder.CreateNUWMul(DimProd, *(DI - 1)); in generateInfoForComponentList()
7913 CurStrides.push_back(CGF.Builder.CreateNUWMul(DimProd, Stride)); in generateInfoForComponentList()
7982 const CGRecordLayout &RL = CGF.getTypes().getCGRecordLayout(RD); in getPlainLayout()
7998 if (Base->isEmpty() || CGF.getContext() in getPlainLayout()
8023 if (!Field->isBitField() && !Field->isZeroSize(CGF.getContext())) { in getPlainLayout()
8148 CodeGenFunction &CGF) { in generateAllInfoForClauses() argument
8153 llvm::Constant::getNullValue(CGF.Int64Ty)); in generateAllInfoForClauses()
8161 &InfoGen](CodeGenFunction &CGF, const Expr *IE, const ValueDecl *VD, in generateAllInfoForClauses()
8184 Ptr = CGF.EmitLValue(IE).getPointer(CGF); in generateAllInfoForClauses()
8186 Ptr = CGF.EmitScalarExpr(IE); in generateAllInfoForClauses()
8188 Ptr = CGF.EmitLoadOfScalar(CGF.EmitLValue(IE), IE->getExprLoc()); in generateAllInfoForClauses()
8190 UseDeviceDataCombinedInfoGen(VD, Ptr, CGF); in generateAllInfoForClauses()
8194 auto &&IsMapInfoExist = [&Info](CodeGenFunction &CGF, const ValueDecl *VD, in generateAllInfoForClauses()
8223 if (CGF.CGM.getOpenMPRuntime().hasRequiresUnifiedSharedMemory() || in generateAllInfoForClauses()
8259 if (IsMapInfoExist(CGF, VD, IE, /*IsDevAddr=*/false)) in generateAllInfoForClauses()
8261 MapInfoGen(CGF, IE, VD, Components, C->isImplicit(), in generateAllInfoForClauses()
8281 if (IsMapInfoExist(CGF, VD, IE, /*IsDevAddr=*/true)) in generateAllInfoForClauses()
8283 MapInfoGen(CGF, IE, VD, Components, C->isImplicit(), in generateAllInfoForClauses()
8336 Ptr = this->CGF.EmitLValue(L.IE).getPointer(CGF); in generateAllInfoForClauses()
8338 Ptr = this->CGF.EmitScalarExpr(L.IE); in generateAllInfoForClauses()
8347 BasePtr = this->CGF.EmitLValue(L.IE).getPointer(CGF); in generateAllInfoForClauses()
8348 Ptr = this->CGF.EmitLoadOfScalar(this->CGF.EmitLValue(L.IE), in generateAllInfoForClauses()
8362 llvm::Constant::getNullValue(this->CGF.Int64Ty)); in generateAllInfoForClauses()
8382 MappableExprsHandler(const OMPExecutableDirective &Dir, CodeGenFunction &CGF) in MappableExprsHandler() argument
8383 : CurDir(&Dir), CGF(CGF) { in MappableExprsHandler()
8428 MappableExprsHandler(const OMPDeclareMapperDecl &Dir, CodeGenFunction &CGF) in MappableExprsHandler() argument
8429 : CurDir(&Dir), CGF(CGF) {} in MappableExprsHandler()
8456 CGF.CurFuncDecl ? dyn_cast<CXXMethodDecl>(CGF.CurFuncDecl) : nullptr; in emitCombinedEntry()
8471 CGF.Builder.CreateIntCast(CGF.getTypeSize(Ty), CGF.Int64Ty, in emitCombinedEntry()
8478 llvm::Value *HAddr = CGF.Builder.CreateConstGEP1_32( in emitCombinedEntry()
8480 llvm::Value *CLAddr = CGF.Builder.CreatePointerCast(LB, CGF.VoidPtrTy); in emitCombinedEntry()
8481 llvm::Value *CHAddr = CGF.Builder.CreatePointerCast(HAddr, CGF.VoidPtrTy); in emitCombinedEntry()
8482 llvm::Value *Diff = CGF.Builder.CreatePtrDiff(CGF.Int8Ty, CHAddr, CLAddr); in emitCombinedEntry()
8483 llvm::Value *Size = CGF.Builder.CreateIntCast(Diff, CGF.Int64Ty, in emitCombinedEntry()
8560 Address VDAddr(Arg, CGF.ConvertTypeForMem(VDType), in generateInfoForLambdaCaptures()
8561 CGF.getContext().getDeclAlign(VD)); in generateInfoForLambdaCaptures()
8562 LValue VDLVal = CGF.MakeAddrLValue(VDAddr, VDType); in generateInfoForLambdaCaptures()
8568 CGF.EmitLValueForFieldInitialization(VDLVal, ThisCapture); in generateInfoForLambdaCaptures()
8569 LValue ThisLValVal = CGF.EmitLValueForField(VDLVal, ThisCapture); in generateInfoForLambdaCaptures()
8570 LambdaPointers.try_emplace(ThisLVal.getPointer(CGF), in generateInfoForLambdaCaptures()
8571 VDLVal.getPointer(CGF)); in generateInfoForLambdaCaptures()
8573 CombinedInfo.BasePointers.push_back(ThisLVal.getPointer(CGF)); in generateInfoForLambdaCaptures()
8574 CombinedInfo.Pointers.push_back(ThisLValVal.getPointer(CGF)); in generateInfoForLambdaCaptures()
8576 CGF.Builder.CreateIntCast(CGF.getTypeSize(CGF.getContext().VoidPtrTy), in generateInfoForLambdaCaptures()
8577 CGF.Int64Ty, /*isSigned=*/true)); in generateInfoForLambdaCaptures()
8593 LValue VarLVal = CGF.EmitLValueForFieldInitialization(VDLVal, It->second); in generateInfoForLambdaCaptures()
8595 LValue VarLValVal = CGF.EmitLValueForField(VDLVal, It->second); in generateInfoForLambdaCaptures()
8596 LambdaPointers.try_emplace(VarLVal.getPointer(CGF), in generateInfoForLambdaCaptures()
8597 VDLVal.getPointer(CGF)); in generateInfoForLambdaCaptures()
8599 CombinedInfo.BasePointers.push_back(VarLVal.getPointer(CGF)); in generateInfoForLambdaCaptures()
8600 CombinedInfo.Pointers.push_back(VarLValVal.getPointer(CGF)); in generateInfoForLambdaCaptures()
8601 CombinedInfo.Sizes.push_back(CGF.Builder.CreateIntCast( in generateInfoForLambdaCaptures()
8602 CGF.getTypeSize( in generateInfoForLambdaCaptures()
8604 CGF.Int64Ty, /*isSigned=*/true)); in generateInfoForLambdaCaptures()
8606 RValue VarRVal = CGF.EmitLoadOfLValue(VarLVal, RD->getLocation()); in generateInfoForLambdaCaptures()
8607 LambdaPointers.try_emplace(VarLVal.getPointer(CGF), in generateInfoForLambdaCaptures()
8608 VDLVal.getPointer(CGF)); in generateInfoForLambdaCaptures()
8610 CombinedInfo.BasePointers.push_back(VarLVal.getPointer(CGF)); in generateInfoForLambdaCaptures()
8612 CombinedInfo.Sizes.push_back(llvm::ConstantInt::get(CGF.Int64Ty, 0)); in generateInfoForLambdaCaptures()
8679 CombinedInfo.Sizes.push_back(CGF.Builder.CreateIntCast( in generateInfoForCapture()
8680 CGF.getTypeSize(CGF.getContext().VoidPtrTy), CGF.Int64Ty, in generateInfoForCapture()
8922 CGF.Builder.CreateIntCast(CGF.getTypeSize(PtrTy->getPointeeType()), in generateDefaultMapInfo()
8923 CGF.Int64Ty, /*isSigned=*/true)); in generateDefaultMapInfo()
8937 CombinedInfo.Sizes.push_back(CGF.Builder.CreateIntCast( in generateDefaultMapInfo()
8938 CGF.getTypeSize(RI.getType()), CGF.Int64Ty, /*isSigned=*/true)); in generateDefaultMapInfo()
8943 CombinedInfo.Sizes.push_back(llvm::Constant::getNullValue(CGF.Int64Ty)); in generateDefaultMapInfo()
8952 CombinedInfo.Sizes.push_back(CGF.Builder.CreateIntCast( in generateDefaultMapInfo()
8953 CGF.getTypeSize(ElementType), CGF.Int64Ty, /*isSigned=*/true)); in generateDefaultMapInfo()
8963 Address PtrAddr = CGF.EmitLoadOfReference(CGF.MakeAddrLValue( in generateDefaultMapInfo()
8964 CV, ElementType, CGF.getContext().getDeclAlign(VD), in generateDefaultMapInfo()
8988 CodeGenFunction &CGF, MappableExprsHandler::MapCombinedInfoTy &CombinedInfo, in emitNonContiguousDescriptor() argument
8990 CodeGenModule &CGM = CGF.CGM; in emitNonContiguousDescriptor()
9002 ASTContext &C = CGF.getContext(); in emitNonContiguousDescriptor()
9025 Address DimsAddr = CGF.CreateMemTemp(ArrayTy, "dims"); in emitNonContiguousDescriptor()
9028 LValue DimsLVal = CGF.MakeAddrLValue( in emitNonContiguousDescriptor()
9029 CGF.Builder.CreateConstArrayGEP(DimsAddr, II), DimTy); in emitNonContiguousDescriptor()
9031 LValue OffsetLVal = CGF.EmitLValueForField( in emitNonContiguousDescriptor()
9033 CGF.EmitStoreOfScalar(NonContigInfo.Offsets[L][RevIdx], OffsetLVal); in emitNonContiguousDescriptor()
9035 LValue CountLVal = CGF.EmitLValueForField( in emitNonContiguousDescriptor()
9037 CGF.EmitStoreOfScalar(NonContigInfo.Counts[L][RevIdx], CountLVal); in emitNonContiguousDescriptor()
9039 LValue StrideLVal = CGF.EmitLValueForField( in emitNonContiguousDescriptor()
9041 CGF.EmitStoreOfScalar(NonContigInfo.Strides[L][RevIdx], StrideLVal); in emitNonContiguousDescriptor()
9044 Address DAddr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitNonContiguousDescriptor()
9046 llvm::Value *P = CGF.Builder.CreateConstInBoundsGEP2_32( in emitNonContiguousDescriptor()
9049 Address PAddr(P, CGM.VoidPtrTy, CGF.getPointerAlign()); in emitNonContiguousDescriptor()
9050 CGF.Builder.CreateStore(DAddr.getPointer(), PAddr); in emitNonContiguousDescriptor()
9070 emitMappingInformation(CodeGenFunction &CGF, llvm::OpenMPIRBuilder &OMPBuilder, in emitMappingInformation() argument
9089 PrintingPolicy P(CGF.getContext().getLangOpts()); in emitMappingInformation()
9097 PresumedLoc PLoc = CGF.getContext().getSourceManager().getPresumedLoc(Loc); in emitMappingInformation()
9107 CodeGenFunction &CGF, MappableExprsHandler::MapCombinedInfoTy &CombinedInfo, in emitOffloadingArrays() argument
9110 CodeGenModule &CGM = CGF.CGM; in emitOffloadingArrays()
9111 ASTContext &Ctx = CGF.getContext(); in emitOffloadingArrays()
9127 CGF.CreateMemTemp(PointerArrayType, ".offload_baseptrs").getPointer(); in emitOffloadingArrays()
9129 CGF.CreateMemTemp(PointerArrayType, ".offload_ptrs").getPointer(); in emitOffloadingArrays()
9131 CGF.CreateMemTemp(PointerArrayType, ".offload_mappers"); in emitOffloadingArrays()
9140 CombinedInfo.Sizes.size(), llvm::ConstantInt::get(CGF.Int64Ty, 0)); in emitOffloadingArrays()
9150 CGF.Int64Ty, CombinedInfo.NonContigInfo.Dims[I]); in emitOffloadingArrays()
9164 CGF.CreateMemTemp(SizeArrayType, ".offload_sizes").getPointer(); in emitOffloadingArrays()
9177 Address Buffer = CGF.CreateMemTemp(SizeArrayType, ".offload_sizes"); in emitOffloadingArrays()
9179 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitOffloadingArrays()
9181 CGF.Builder.CreateMemCpy( in emitOffloadingArrays()
9186 CGF.getTypeSize(SizeArrayType)); in emitOffloadingArrays()
9210 llvm::Type::getInt8Ty(CGF.Builder.getContext())->getPointerTo()); in emitOffloadingArrays()
9213 return emitMappingInformation(CGF, OMPBuilder, MapExpr); in emitOffloadingArrays()
9247 llvm::Value *BP = CGF.Builder.CreateConstInBoundsGEP2_32( in emitOffloadingArrays()
9250 BP = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitOffloadingArrays()
9254 CGF.Builder.CreateStore(BPVal, BPAddr); in emitOffloadingArrays()
9262 llvm::Value *P = CGF.Builder.CreateConstInBoundsGEP2_32( in emitOffloadingArrays()
9265 P = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitOffloadingArrays()
9268 CGF.Builder.CreateStore(PVal, PAddr); in emitOffloadingArrays()
9271 llvm::Value *S = CGF.Builder.CreateConstInBoundsGEP2_32( in emitOffloadingArrays()
9277 CGF.Builder.CreateStore(CGF.Builder.CreateIntCast(CombinedInfo.Sizes[I], in emitOffloadingArrays()
9288 MFunc = CGF.Builder.CreatePointerCast(MFunc, CGM.VoidPtrTy); in emitOffloadingArrays()
9291 Address MAddr = CGF.Builder.CreateConstArrayGEP(MappersArray, I); in emitOffloadingArrays()
9292 CGF.Builder.CreateStore(MFunc, MAddr); in emitOffloadingArrays()
9300 emitNonContiguousDescriptor(CGF, CombinedInfo, Info); in emitOffloadingArrays()
9446 CodeGenFunction *CGF) { in emitUserDefinedMapper() argument
9701 if (CGF) { in emitUserDefinedMapper()
9702 auto &Decls = FunctionUDMMap.FindAndConstruct(CGF->CurFn); in emitUserDefinedMapper()
9793 CodeGenFunction &CGF, const OMPExecutableDirective &D, in emitTargetNumIterationsCall() argument
9794 llvm::function_ref<llvm::Value *(CodeGenFunction &CGF, in emitTargetNumIterationsCall() argument
9803 return llvm::ConstantInt::get(CGF.Int64Ty, 0); in emitTargetNumIterationsCall()
9806 if (llvm::Value *NumIterations = SizeEmitter(CGF, *LD)) in emitTargetNumIterationsCall()
9808 return llvm::ConstantInt::get(CGF.Int64Ty, 0); in emitTargetNumIterationsCall()
9812 CodeGenFunction &CGF, const OMPExecutableDirective &D, in emitTargetCall() argument
9815 llvm::function_ref<llvm::Value *(CodeGenFunction &CGF, in emitTargetCall() argument
9818 if (!CGF.HaveInsertPoint()) in emitTargetCall()
9831 auto &&ArgsCodegen = [&CS, &CapturedVars](CodeGenFunction &CGF, in emitTargetCall()
9833 CGF.GenerateOpenMPCapturedVars(CS, CapturedVars); in emitTargetCall()
9835 emitInlinedDirective(CGF, OMPD_unknown, ArgsCodegen); in emitTargetCall()
9842 &CS, OffloadingMandatory](CodeGenFunction &CGF) { in emitTargetCall() argument
9844 CGF.Builder.CreateUnreachable(); in emitTargetCall()
9848 CGF.GenerateOpenMPCapturedVars(CS, CapturedVars); in emitTargetCall()
9850 emitOutlinedFunctionCall(CGF, D.getBeginLoc(), OutlinedFn, CapturedVars); in emitTargetCall()
9856 FallbackGen](CodeGenFunction &CGF, PrePostActionTy &) { in emitTargetCall() argument
9859 FallbackGen(CGF); in emitTargetCall()
9882 llvm::Value *DevVal = CGF.EmitScalarExpr(Device.getPointer()); in emitTargetCall()
9884 CGF.Builder.CreateIntCast(DevVal, CGF.Int64Ty, /*isSigned=*/true); in emitTargetCall()
9886 DeviceID = CGF.Builder.getInt64(OMP_DEVICEID_UNDEF); in emitTargetCall()
9891 CGF.Builder.getInt32(InputInfo.NumberOfTargetItems); in emitTargetCall()
9896 llvm::Value *NumTeams = emitNumTeamsForTargetDirective(CGF, D); in emitTargetCall()
9897 llvm::Value *NumThreads = emitNumThreadsForTargetDirective(CGF, D); in emitTargetCall()
9900 llvm::Value *RTLoc = emitUpdateLocation(CGF, D.getBeginLoc()); in emitTargetCall()
9904 emitTargetNumIterationsCall(CGF, D, SizeEmitter); in emitTargetCall()
9906 llvm::Value *DynCGroupMem = CGF.Builder.getInt32(0); in emitTargetCall()
9908 CodeGenFunction::RunCleanupsScope DynCGroupMemScope(CGF); in emitTargetCall()
9909 llvm::Value *DynCGroupMemVal = CGF.EmitScalarExpr( in emitTargetCall()
9911 DynCGroupMem = CGF.Builder.CreateIntCast(DynCGroupMemVal, CGF.Int32Ty, in emitTargetCall()
9916 llvm::Constant::getNullValue(llvm::ArrayType::get(CGF.CGM.Int32Ty, 3)); in emitTargetCall()
9919 llvm::Value *Flags = CGF.Builder.getInt64(HasNoWait); in emitTargetCall()
9922 CGF.Builder.CreateInsertValue(ZeroArray, NumTeams, {0}); in emitTargetCall()
9924 CGF.Builder.CreateInsertValue(ZeroArray, NumThreads, {0}); in emitTargetCall()
9928 CGF.Builder.getInt32(/* Version */ 2), in emitTargetCall()
9956 CGF.Builder.restoreIP(OMPBuilder.emitTargetKernel( in emitTargetCall()
9957 CGF.Builder, Return, RTLoc, DeviceID, NumTeams, NumThreads, in emitTargetCall()
9961 CGF.createBasicBlock("omp_offload.failed"); in emitTargetCall()
9963 CGF.createBasicBlock("omp_offload.cont"); in emitTargetCall()
9964 llvm::Value *Failed = CGF.Builder.CreateIsNotNull(Return); in emitTargetCall()
9965 CGF.Builder.CreateCondBr(Failed, OffloadFailedBlock, OffloadContBlock); in emitTargetCall()
9967 CGF.EmitBlock(OffloadFailedBlock); in emitTargetCall()
9968 FallbackGen(CGF); in emitTargetCall()
9970 CGF.EmitBranch(OffloadContBlock); in emitTargetCall()
9972 CGF.EmitBlock(OffloadContBlock, /*IsFinished=*/true); in emitTargetCall()
9976 auto &&ElseGen = [FallbackGen](CodeGenFunction &CGF, PrePostActionTy &) { in emitTargetCall() argument
9977 FallbackGen(CGF); in emitTargetCall()
9982 &CS](CodeGenFunction &CGF, PrePostActionTy &) { in emitTargetCall() argument
9987 MappableExprsHandler MEHandler(D, CGF); in emitTargetCall()
10005 CurInfo.Sizes.push_back(CGF.Builder.CreateIntCast( in emitTargetCall()
10006 CGF.getTypeSize(RI->getType()), CGF.Int64Ty, /*isSigned=*/true)); in emitTargetCall()
10060 emitOffloadingArrays(CGF, CombinedInfo, Info, OMPBuilder); in emitTargetCall()
10062 CGF.CGM.getCodeGenOpts().getDebugInfo() != codegenoptions::NoDebugInfo; in emitTargetCall()
10063 OMPBuilder.emitOffloadingArraysArgument(CGF.Builder, Info.RTArgs, Info, in emitTargetCall()
10069 CGF.VoidPtrTy, CGM.getPointerAlign()); in emitTargetCall()
10070 InputInfo.PointersArray = Address(Info.RTArgs.PointersArray, CGF.VoidPtrTy, in emitTargetCall()
10073 Address(Info.RTArgs.SizesArray, CGF.Int64Ty, CGM.getPointerAlign()); in emitTargetCall()
10075 Address(Info.RTArgs.MappersArray, CGF.VoidPtrTy, CGM.getPointerAlign()); in emitTargetCall()
10079 CGF.EmitOMPTargetTaskBasedDirective(D, ThenGen, InputInfo); in emitTargetCall()
10081 emitInlinedDirective(CGF, D.getDirectiveKind(), ThenGen); in emitTargetCall()
10085 CodeGenFunction &CGF, PrePostActionTy &) { in emitTargetCall() argument
10088 CGF.EmitOMPTargetTaskBasedDirective(D, ElseGen, InputInfo); in emitTargetCall()
10090 emitInlinedDirective(CGF, D.getDirectiveKind(), ElseGen); in emitTargetCall()
10100 emitIfClause(CGF, IfCond, TargetThenGen, TargetElseGen); in emitTargetCall()
10103 ThenRCG(CGF); in emitTargetCall()
10107 ElseRCG(CGF); in emitTargetCall()
10452 CodeGenFunction &CGF, const OMPExecutableDirective &D) const { in adjustTargetSpecificDataForLambdas() argument
10563 CodeGenFunction CGF(CGM); in emitRequiresDirectiveRegFun() local
10568 CGF.StartFunction(GlobalDecl(), C.VoidTy, RequiresRegFn, FI, {}); in emitRequiresDirectiveRegFun()
10582 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in emitRequiresDirectiveRegFun()
10585 CGF.FinishFunction(); in emitRequiresDirectiveRegFun()
10590 void CGOpenMPRuntime::emitTeamsCall(CodeGenFunction &CGF, in emitTeamsCall() argument
10595 if (!CGF.HaveInsertPoint()) in emitTeamsCall()
10598 llvm::Value *RTLoc = emitUpdateLocation(CGF, Loc); in emitTeamsCall()
10599 CodeGenFunction::RunCleanupsScope Scope(CGF); in emitTeamsCall()
10604 CGF.Builder.getInt32(CapturedVars.size()), // Number of captured vars in emitTeamsCall()
10605 CGF.Builder.CreateBitCast(OutlinedFn, getKmpc_MicroPointerTy())}; in emitTeamsCall()
10612 CGF.EmitRuntimeCall(RTLFn, RealArgs); in emitTeamsCall()
10615 void CGOpenMPRuntime::emitNumTeamsClause(CodeGenFunction &CGF, in emitNumTeamsClause() argument
10619 if (!CGF.HaveInsertPoint()) in emitNumTeamsClause()
10622 llvm::Value *RTLoc = emitUpdateLocation(CGF, Loc); in emitNumTeamsClause()
10626 ? CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(NumTeams), in emitNumTeamsClause()
10627 CGF.CGM.Int32Ty, /* isSigned = */ true) in emitNumTeamsClause()
10628 : CGF.Builder.getInt32(0); in emitNumTeamsClause()
10632 ? CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(ThreadLimit), in emitNumTeamsClause()
10633 CGF.CGM.Int32Ty, /* isSigned = */ true) in emitNumTeamsClause()
10634 : CGF.Builder.getInt32(0); in emitNumTeamsClause()
10637 llvm::Value *PushNumTeamsArgs[] = {RTLoc, getThreadID(CGF, Loc), NumTeamsVal, in emitNumTeamsClause()
10639 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in emitNumTeamsClause()
10645 CodeGenFunction &CGF, const OMPExecutableDirective &D, const Expr *IfCond, in emitTargetDataCalls() argument
10648 if (!CGF.HaveInsertPoint()) in emitTargetDataCalls()
10659 &CodeGen](CodeGenFunction &CGF, PrePostActionTy &) { in emitTargetDataCalls() argument
10664 MappableExprsHandler MEHandler(D, CGF); in emitTargetDataCalls()
10668 emitOffloadingArrays(CGF, CombinedInfo, Info, OMPBuilder, in emitTargetDataCalls()
10673 CGF.CGM.getCodeGenOpts().getDebugInfo() != codegenoptions::NoDebugInfo; in emitTargetDataCalls()
10674 OMPBuilder.emitOffloadingArraysArgument(CGF.Builder, RTArgs, Info, in emitTargetDataCalls()
10680 DeviceID = CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(Device), in emitTargetDataCalls()
10681 CGF.Int64Ty, /*isSigned=*/true); in emitTargetDataCalls()
10683 DeviceID = CGF.Builder.getInt64(OMP_DEVICEID_UNDEF); in emitTargetDataCalls()
10687 llvm::Value *PointerNum = CGF.Builder.getInt32(Info.NumberOfPtrs); in emitTargetDataCalls()
10690 llvm::Value *RTLoc = emitUpdateLocation(CGF, D.getBeginLoc()); in emitTargetDataCalls()
10701 CGF.EmitRuntimeCall( in emitTargetDataCalls()
10709 CodeGen(CGF); in emitTargetDataCalls()
10713 auto &&EndThenGen = [this, Device, &Info, &D](CodeGenFunction &CGF, in emitTargetDataCalls()
10719 CGF.CGM.getCodeGenOpts().getDebugInfo() != codegenoptions::NoDebugInfo; in emitTargetDataCalls()
10720 OMPBuilder.emitOffloadingArraysArgument(CGF.Builder, RTArgs, Info, in emitTargetDataCalls()
10727 DeviceID = CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(Device), in emitTargetDataCalls()
10728 CGF.Int64Ty, /*isSigned=*/true); in emitTargetDataCalls()
10730 DeviceID = CGF.Builder.getInt64(OMP_DEVICEID_UNDEF); in emitTargetDataCalls()
10734 llvm::Value *PointerNum = CGF.Builder.getInt32(Info.NumberOfPtrs); in emitTargetDataCalls()
10737 llvm::Value *RTLoc = emitUpdateLocation(CGF, D.getBeginLoc()); in emitTargetDataCalls()
10748 CGF.EmitRuntimeCall( in emitTargetDataCalls()
10757 auto &&BeginElseGen = [&Info, &CodeGen, &NoPrivAction](CodeGenFunction &CGF, in emitTargetDataCalls()
10761 CodeGen(CGF); in emitTargetDataCalls()
10767 auto &&EndElseGen = [](CodeGenFunction &CGF, PrePostActionTy &) {}; in emitTargetDataCalls() argument
10770 emitIfClause(CGF, IfCond, BeginThenGen, BeginElseGen); in emitTargetDataCalls()
10773 RCG(CGF); in emitTargetDataCalls()
10780 CodeGen(CGF); in emitTargetDataCalls()
10784 emitIfClause(CGF, IfCond, EndThenGen, EndElseGen); in emitTargetDataCalls()
10787 RCG(CGF); in emitTargetDataCalls()
10792 CodeGenFunction &CGF, const OMPExecutableDirective &D, const Expr *IfCond, in emitTargetDataStandAloneCall() argument
10794 if (!CGF.HaveInsertPoint()) in emitTargetDataStandAloneCall()
10807 &MapNamesArray](CodeGenFunction &CGF, PrePostActionTy &) { in emitTargetDataStandAloneCall() argument
10811 DeviceID = CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(Device), in emitTargetDataStandAloneCall()
10812 CGF.Int64Ty, /*isSigned=*/true); in emitTargetDataStandAloneCall()
10814 DeviceID = CGF.Builder.getInt64(OMP_DEVICEID_UNDEF); in emitTargetDataStandAloneCall()
10819 CGF.Builder.getInt32(InputInfo.NumberOfTargetItems); in emitTargetDataStandAloneCall()
10822 llvm::Value *RTLoc = emitUpdateLocation(CGF, D.getBeginLoc()); in emitTargetDataStandAloneCall()
10921 CGF.EmitRuntimeCall( in emitTargetDataStandAloneCall()
10927 &MapNamesArray](CodeGenFunction &CGF, in emitTargetDataStandAloneCall()
10933 MappableExprsHandler MEHandler(D, CGF); in emitTargetDataStandAloneCall()
10938 emitOffloadingArrays(CGF, CombinedInfo, Info, OMPBuilder, in emitTargetDataStandAloneCall()
10943 CGF.CGM.getCodeGenOpts().getDebugInfo() != codegenoptions::NoDebugInfo; in emitTargetDataStandAloneCall()
10944 OMPBuilder.emitOffloadingArraysArgument(CGF.Builder, Info.RTArgs, Info, in emitTargetDataStandAloneCall()
10949 CGF.VoidPtrTy, CGM.getPointerAlign()); in emitTargetDataStandAloneCall()
10950 InputInfo.PointersArray = Address(Info.RTArgs.PointersArray, CGF.VoidPtrTy, in emitTargetDataStandAloneCall()
10953 Address(Info.RTArgs.SizesArray, CGF.Int64Ty, CGM.getPointerAlign()); in emitTargetDataStandAloneCall()
10955 Address(Info.RTArgs.MappersArray, CGF.VoidPtrTy, CGM.getPointerAlign()); in emitTargetDataStandAloneCall()
10959 CGF.EmitOMPTargetTaskBasedDirective(D, ThenGen, InputInfo); in emitTargetDataStandAloneCall()
10961 emitInlinedDirective(CGF, D.getDirectiveKind(), ThenGen); in emitTargetDataStandAloneCall()
10965 emitIfClause(CGF, IfCond, TargetThenGen, in emitTargetDataStandAloneCall()
10966 [](CodeGenFunction &CGF, PrePostActionTy &) {}); in emitTargetDataStandAloneCall() argument
10969 ThenRCG(CGF); in emitTargetDataStandAloneCall()
11578 void Emit(CodeGenFunction &CGF, Flags /*flags*/) override { in Emit() argument
11579 if (!CGF.HaveInsertPoint()) in Emit()
11581 CGF.EmitRuntimeCall(RTLFn, Args); in Emit()
11586 void CGOpenMPRuntime::emitDoacrossInit(CodeGenFunction &CGF, in emitDoacrossInit() argument
11589 if (!CGF.HaveInsertPoint()) in emitDoacrossInit()
11615 Address DimsAddr = CGF.CreateMemTemp(ArrayTy, "dims"); in emitDoacrossInit()
11616 CGF.EmitNullInitialization(DimsAddr, ArrayTy); in emitDoacrossInit()
11620 LValue DimsLVal = CGF.MakeAddrLValue( in emitDoacrossInit()
11621 CGF.Builder.CreateConstArrayGEP(DimsAddr, I), KmpDimTy); in emitDoacrossInit()
11623 LValue UpperLVal = CGF.EmitLValueForField( in emitDoacrossInit()
11625 llvm::Value *NumIterVal = CGF.EmitScalarConversion( in emitDoacrossInit()
11626 CGF.EmitScalarExpr(NumIterations[I]), NumIterations[I]->getType(), in emitDoacrossInit()
11628 CGF.EmitStoreOfScalar(NumIterVal, UpperLVal); in emitDoacrossInit()
11630 LValue StrideLVal = CGF.EmitLValueForField( in emitDoacrossInit()
11632 CGF.EmitStoreOfScalar(llvm::ConstantInt::getSigned(CGM.Int64Ty, /*V=*/1), in emitDoacrossInit()
11639 emitUpdateLocation(CGF, D.getBeginLoc()), in emitDoacrossInit()
11640 getThreadID(CGF, D.getBeginLoc()), in emitDoacrossInit()
11642 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitDoacrossInit()
11643 CGF.Builder.CreateConstArrayGEP(DimsAddr, 0).getPointer(), in emitDoacrossInit()
11648 CGF.EmitRuntimeCall(RTLFn, Args); in emitDoacrossInit()
11650 emitUpdateLocation(CGF, D.getEndLoc()), getThreadID(CGF, D.getEndLoc())}; in emitDoacrossInit()
11653 CGF.EHStack.pushCleanup<DoacrossCleanupTy>(NormalAndEHCleanup, FiniRTLFn, in emitDoacrossInit()
11657 void CGOpenMPRuntime::emitDoacrossOrdered(CodeGenFunction &CGF, in emitDoacrossOrdered() argument
11664 Address CntAddr = CGF.CreateMemTemp(ArrayTy, ".cnt.addr"); in emitDoacrossOrdered()
11668 llvm::Value *CntVal = CGF.EmitScalarConversion( in emitDoacrossOrdered()
11669 CGF.EmitScalarExpr(CounterVal), CounterVal->getType(), Int64Ty, in emitDoacrossOrdered()
11671 CGF.EmitStoreOfScalar(CntVal, CGF.Builder.CreateConstArrayGEP(CntAddr, I), in emitDoacrossOrdered()
11675 emitUpdateLocation(CGF, C->getBeginLoc()), in emitDoacrossOrdered()
11676 getThreadID(CGF, C->getBeginLoc()), in emitDoacrossOrdered()
11677 CGF.Builder.CreateConstArrayGEP(CntAddr, 0).getPointer()}; in emitDoacrossOrdered()
11687 CGF.EmitRuntimeCall(RTLFn, Args); in emitDoacrossOrdered()
11690 void CGOpenMPRuntime::emitCall(CodeGenFunction &CGF, SourceLocation Loc, in emitCall() argument
11694 auto DL = ApplyDebugLocation::CreateDefaultArtificial(CGF, Loc); in emitCall()
11698 CGF.EmitNounwindRuntimeCall(Fn, Args); in emitCall()
11702 CGF.EmitRuntimeCall(Callee, Args); in emitCall()
11706 CodeGenFunction &CGF, SourceLocation Loc, llvm::FunctionCallee OutlinedFn, in emitOutlinedFunctionCall() argument
11708 emitCall(CGF, Loc, OutlinedFn, Args); in emitOutlinedFunctionCall()
11711 void CGOpenMPRuntime::emitFunctionProlog(CodeGenFunction &CGF, const Decl *D) { in emitFunctionProlog() argument
11717 Address CGOpenMPRuntime::getParameterAddress(CodeGenFunction &CGF, in getParameterAddress() argument
11720 return CGF.GetAddrOfLocalVar(NativeParam); in getParameterAddress()
11725 static llvm::Value *getAllocatorVal(CodeGenFunction &CGF, in getAllocatorVal() argument
11729 AllocVal = CGF.EmitScalarExpr(Allocator); in getAllocatorVal()
11732 AllocVal = CGF.EmitScalarConversion(AllocVal, Allocator->getType(), in getAllocatorVal()
11733 CGF.getContext().VoidPtrTy, in getAllocatorVal()
11738 CGF.CGM.getTypes().ConvertType(CGF.getContext().VoidPtrTy)); in getAllocatorVal()
11753 Address CGOpenMPRuntime::getAddressOfLocalVariable(CodeGenFunction &CGF, in getAddressOfLocalVariable() argument
11759 auto It = FunctionToUntiedTaskStackMap.find(CGF.CurFn); in getAddressOfLocalVariable()
11777 Size = CGF.getTypeSize(CVD->getType()); in getAddressOfLocalVariable()
11779 Size = CGF.Builder.CreateNUWAdd( in getAddressOfLocalVariable()
11781 Size = CGF.Builder.CreateUDiv(Size, CGM.getSize(Align)); in getAddressOfLocalVariable()
11782 Size = CGF.Builder.CreateNUWMul(Size, CGM.getSize(Align)); in getAddressOfLocalVariable()
11787 llvm::Value *ThreadID = getThreadID(CGF, CVD->getBeginLoc()); in getAddressOfLocalVariable()
11790 llvm::Value *AllocVal = getAllocatorVal(CGF, Allocator); in getAddressOfLocalVariable()
11800 llvm::Value *Addr = CGF.EmitRuntimeCall( in getAddressOfLocalVariable()
11806 Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in getAddressOfLocalVariable()
11807 Addr, CGF.ConvertTypeForMem(Ty), getName({CVD->getName(), ".addr"})); in getAddressOfLocalVariable()
11809 CGF.EmitStoreOfScalar(Addr, UntiedAddr, /*Volatile=*/false, Ty); in getAddressOfLocalVariable()
11824 void Emit(CodeGenFunction &CGF, Flags /*flags*/) override { in getAddressOfLocalVariable() argument
11825 if (!CGF.HaveInsertPoint()) in getAddressOfLocalVariable()
11828 Args[0] = CGF.CGM.getOpenMPRuntime().getThreadID( in getAddressOfLocalVariable()
11829 CGF, SourceLocation::getFromRawEncoding(LocEncoding)); in getAddressOfLocalVariable()
11830 Args[1] = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in getAddressOfLocalVariable()
11831 Addr.getPointer(), CGF.VoidPtrTy); in getAddressOfLocalVariable()
11832 llvm::Value *AllocVal = getAllocatorVal(CGF, AllocExpr); in getAddressOfLocalVariable()
11834 CGF.EmitRuntimeCall(RTLFn, Args); in getAddressOfLocalVariable()
11840 : Address(Addr, CGF.ConvertTypeForMem(CVD->getType()), Align); in getAddressOfLocalVariable()
11841 CGF.EHStack.pushCleanup<OMPAllocateCleanupTy>( in getAddressOfLocalVariable()
11846 dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo)) in getAddressOfLocalVariable()
11847 Region->emitUntiedSwitch(CGF); in getAddressOfLocalVariable()
11853 bool CGOpenMPRuntime::isLocalVarInUntiedTask(CodeGenFunction &CGF, in isLocalVarInUntiedTask() argument
11855 auto It = FunctionToUntiedTaskStackMap.find(CGF.CurFn); in isLocalVarInUntiedTask()
11894 CodeGenFunction &CGF, in UntiedTaskLocalDeclsRAII() argument
11897 : CGM(CGF.CGM), NeedToPush(!LocalVars.empty()) { in UntiedTaskLocalDeclsRAII()
11901 CGF.CurFn, CGM.getOpenMPRuntime().UntiedLocalVarsStack.size()); in UntiedTaskLocalDeclsRAII()
11999 CodeGenFunction &CGF, const OMPExecutableDirective &S, LValue IVLVal) in LastprivateConditionalRAII() argument
12000 : CGM(CGF.CGM), in LastprivateConditionalRAII()
12027 Data.Fn = CGF.CurFn;
12031 CodeGenFunction &CGF, const OMPExecutableDirective &S) in LastprivateConditionalRAII() argument
12032 : CGM(CGF.CGM), Action(ActionToDo::DoNotPush) { in LastprivateConditionalRAII()
12044 Data.Fn = CGF.CurFn; in LastprivateConditionalRAII()
12051 CodeGenFunction &CGF, const OMPExecutableDirective &S) { in disable() argument
12052 return LastprivateConditionalRAII(CGF, S); in disable()
12071 Address CGOpenMPRuntime::emitLastprivateConditionalInit(CodeGenFunction &CGF, in emitLastprivateConditionalInit() argument
12074 auto I = LastprivateConditionalToTypes.find(CGF.CurFn); in emitLastprivateConditionalInit()
12076 I = LastprivateConditionalToTypes.try_emplace(CGF.CurFn).first; in emitLastprivateConditionalInit()
12089 Address Addr = CGF.CreateMemTemp(NewType, C.getDeclAlign(VD), VD->getName()); in emitLastprivateConditionalInit()
12090 BaseLVal = CGF.MakeAddrLValue(Addr, NewType, AlignmentSource::Decl); in emitLastprivateConditionalInit()
12099 CGF.EmitLValueForField(BaseLVal, FiredField); in emitLastprivateConditionalInit()
12100 CGF.EmitStoreOfScalar( in emitLastprivateConditionalInit()
12101 llvm::ConstantInt::getNullValue(CGF.ConvertTypeForMem(C.CharTy)), in emitLastprivateConditionalInit()
12103 return CGF.EmitLValueForField(BaseLVal, VDField).getAddress(CGF); in emitLastprivateConditionalInit()
12177 void CGOpenMPRuntime::emitLastprivateConditionalUpdate(CodeGenFunction &CGF, in emitLastprivateConditionalUpdate() argument
12184 llvm::Type *LLIVTy = CGF.ConvertTypeForMem(IVLVal.getType()); in emitLastprivateConditionalUpdate()
12189 LValue LastIVLVal = CGF.MakeNaturalAlignAddrLValue(LastIV, IVLVal.getType()); in emitLastprivateConditionalUpdate()
12194 CGF.ConvertTypeForMem(LVal.getType()), UniqueDeclName); in emitLastprivateConditionalUpdate()
12196 LValue LastLVal = CGF.MakeAddrLValue( in emitLastprivateConditionalUpdate()
12201 llvm::Value *IVVal = CGF.EmitLoadOfScalar(IVLVal, Loc); in emitLastprivateConditionalUpdate()
12209 Loc](CodeGenFunction &CGF, PrePostActionTy &Action) { in emitLastprivateConditionalUpdate() argument
12210 Action.Enter(CGF); in emitLastprivateConditionalUpdate()
12211 llvm::Value *LastIVVal = CGF.EmitLoadOfScalar(LastIVLVal, Loc); in emitLastprivateConditionalUpdate()
12216 CmpRes = CGF.Builder.CreateICmpSLE(LastIVVal, IVVal); in emitLastprivateConditionalUpdate()
12220 CmpRes = CGF.Builder.CreateICmpULE(LastIVVal, IVVal); in emitLastprivateConditionalUpdate()
12222 llvm::BasicBlock *ThenBB = CGF.createBasicBlock("lp_cond_then"); in emitLastprivateConditionalUpdate()
12223 llvm::BasicBlock *ExitBB = CGF.createBasicBlock("lp_cond_exit"); in emitLastprivateConditionalUpdate()
12224 CGF.Builder.CreateCondBr(CmpRes, ThenBB, ExitBB); in emitLastprivateConditionalUpdate()
12226 CGF.EmitBlock(ThenBB); in emitLastprivateConditionalUpdate()
12229 CGF.EmitStoreOfScalar(IVVal, LastIVLVal); in emitLastprivateConditionalUpdate()
12232 switch (CGF.getEvaluationKind(LVal.getType())) { in emitLastprivateConditionalUpdate()
12234 llvm::Value *PrivVal = CGF.EmitLoadOfScalar(LVal, Loc); in emitLastprivateConditionalUpdate()
12235 CGF.EmitStoreOfScalar(PrivVal, LastLVal); in emitLastprivateConditionalUpdate()
12239 CodeGenFunction::ComplexPairTy PrivVal = CGF.EmitLoadOfComplex(LVal, Loc); in emitLastprivateConditionalUpdate()
12240 CGF.EmitStoreOfComplex(PrivVal, LastLVal, /*isInit=*/false); in emitLastprivateConditionalUpdate()
12248 CGF.EmitBranch(ExitBB); in emitLastprivateConditionalUpdate()
12250 (void)ApplyDebugLocation::CreateEmpty(CGF); in emitLastprivateConditionalUpdate()
12251 CGF.EmitBlock(ExitBB, /*IsFinished=*/true); in emitLastprivateConditionalUpdate()
12257 ThenRCG(CGF); in emitLastprivateConditionalUpdate()
12259 emitCriticalRegion(CGF, UniqueDeclName, CodeGen, Loc); in emitLastprivateConditionalUpdate()
12263 void CGOpenMPRuntime::checkAndEmitLastprivateConditional(CodeGenFunction &CGF, in checkAndEmitLastprivateConditional() argument
12265 if (CGF.getLangOpts().OpenMP < 50 || LastprivateConditionalStack.empty()) in checkAndEmitLastprivateConditional()
12277 if (FoundFn != CGF.CurFn) { in checkAndEmitLastprivateConditional()
12285 LValue PrivLVal = CGF.EmitLValue(FoundE); in checkAndEmitLastprivateConditional()
12286 Address StructAddr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in checkAndEmitLastprivateConditional()
12287 PrivLVal.getAddress(CGF), in checkAndEmitLastprivateConditional()
12288 CGF.ConvertTypeForMem(CGF.getContext().getPointerType(StructTy)), in checkAndEmitLastprivateConditional()
12289 CGF.ConvertTypeForMem(StructTy)); in checkAndEmitLastprivateConditional()
12291 CGF.MakeAddrLValue(StructAddr, StructTy, AlignmentSource::Decl); in checkAndEmitLastprivateConditional()
12292 LValue FiredLVal = CGF.EmitLValueForField(BaseLVal, FiredDecl); in checkAndEmitLastprivateConditional()
12293 CGF.EmitAtomicStore(RValue::get(llvm::ConstantInt::get( in checkAndEmitLastprivateConditional()
12294 CGF.ConvertTypeForMem(FiredDecl->getType()), 1)), in checkAndEmitLastprivateConditional()
12302 LValue LVal = CGF.EmitLValue(FoundE); in checkAndEmitLastprivateConditional()
12303 emitLastprivateConditionalUpdate(CGF, IVLVal, UniqueDeclName, LVal, in checkAndEmitLastprivateConditional()
12308 CodeGenFunction &CGF, const OMPExecutableDirective &D, in checkAndEmitSharedLastprivateConditional() argument
12310 if (CGF.getLangOpts().OpenMP < 50 || LastprivateConditionalStack.empty()) in checkAndEmitSharedLastprivateConditional()
12315 if (It == Range.end() || It->Fn != CGF.CurFn) in checkAndEmitSharedLastprivateConditional()
12333 CGF.EmitLValueForField(BaseLVal, std::get<2>(I->getSecond())); in checkAndEmitSharedLastprivateConditional()
12334 llvm::Value *Res = CGF.EmitLoadOfScalar(FiredLVal, D.getBeginLoc()); in checkAndEmitSharedLastprivateConditional()
12335 llvm::Value *Cmp = CGF.Builder.CreateIsNotNull(Res); in checkAndEmitSharedLastprivateConditional()
12336 llvm::BasicBlock *ThenBB = CGF.createBasicBlock("lpc.then"); in checkAndEmitSharedLastprivateConditional()
12337 llvm::BasicBlock *DoneBB = CGF.createBasicBlock("lpc.done"); in checkAndEmitSharedLastprivateConditional()
12339 CGF.Builder.CreateCondBr(Cmp, ThenBB, DoneBB); in checkAndEmitSharedLastprivateConditional()
12340 CGF.EmitBlock(ThenBB); in checkAndEmitSharedLastprivateConditional()
12341 Address Addr = CGF.GetAddrOfLocalVar(VD); in checkAndEmitSharedLastprivateConditional()
12344 LVal = CGF.EmitLoadOfReferenceLValue(Addr, VD->getType(), in checkAndEmitSharedLastprivateConditional()
12347 LVal = CGF.MakeAddrLValue(Addr, VD->getType().getNonReferenceType(), in checkAndEmitSharedLastprivateConditional()
12349 emitLastprivateConditionalUpdate(CGF, It->IVLVal, Pair.second, LVal, in checkAndEmitSharedLastprivateConditional()
12351 auto AL = ApplyDebugLocation::CreateArtificial(CGF); in checkAndEmitSharedLastprivateConditional()
12352 CGF.EmitBlock(DoneBB, /*IsFinal=*/true); in checkAndEmitSharedLastprivateConditional()
12358 CodeGenFunction &CGF, LValue PrivLVal, const VarDecl *VD, in emitLastprivateConditionalFinalUpdate() argument
12360 if (CGF.getLangOpts().OpenMP < 50) in emitLastprivateConditionalFinalUpdate()
12370 LValue LPLVal = CGF.MakeAddrLValue( in emitLastprivateConditionalFinalUpdate()
12373 llvm::Value *Res = CGF.EmitLoadOfScalar(LPLVal, Loc); in emitLastprivateConditionalFinalUpdate()
12374 CGF.EmitStoreOfScalar(Res, PrivLVal); in emitLastprivateConditionalFinalUpdate()
12397 void CGOpenMPSIMDRuntime::emitParallelCall(CodeGenFunction &CGF, in emitParallelCall() argument
12407 CodeGenFunction &CGF, StringRef CriticalName, in emitCriticalRegion() argument
12413 void CGOpenMPSIMDRuntime::emitMasterRegion(CodeGenFunction &CGF, in emitMasterRegion() argument
12419 void CGOpenMPSIMDRuntime::emitMaskedRegion(CodeGenFunction &CGF, in emitMaskedRegion() argument
12426 void CGOpenMPSIMDRuntime::emitTaskyieldCall(CodeGenFunction &CGF, in emitTaskyieldCall() argument
12432 CodeGenFunction &CGF, const RegionCodeGenTy &TaskgroupOpGen, in emitTaskgroupRegion() argument
12438 CodeGenFunction &CGF, const RegionCodeGenTy &SingleOpGen, in emitSingleRegion() argument
12445 void CGOpenMPSIMDRuntime::emitOrderedRegion(CodeGenFunction &CGF, in emitOrderedRegion() argument
12452 void CGOpenMPSIMDRuntime::emitBarrierCall(CodeGenFunction &CGF, in emitBarrierCall() argument
12461 CodeGenFunction &CGF, SourceLocation Loc, in emitForDispatchInit() argument
12468 CodeGenFunction &CGF, SourceLocation Loc, OpenMPDirectiveKind DKind, in emitForStaticInit() argument
12474 CodeGenFunction &CGF, SourceLocation Loc, in emitDistributeStaticInit() argument
12479 void CGOpenMPSIMDRuntime::emitForOrderedIterationEnd(CodeGenFunction &CGF, in emitForOrderedIterationEnd() argument
12486 void CGOpenMPSIMDRuntime::emitForStaticFinish(CodeGenFunction &CGF, in emitForStaticFinish() argument
12492 llvm::Value *CGOpenMPSIMDRuntime::emitForNext(CodeGenFunction &CGF, in emitForNext() argument
12500 void CGOpenMPSIMDRuntime::emitNumThreadsClause(CodeGenFunction &CGF, in emitNumThreadsClause() argument
12506 void CGOpenMPSIMDRuntime::emitProcBindClause(CodeGenFunction &CGF, in emitProcBindClause() argument
12512 Address CGOpenMPSIMDRuntime::getAddrOfThreadPrivate(CodeGenFunction &CGF, in getAddrOfThreadPrivate() argument
12521 CodeGenFunction *CGF) { in emitThreadPrivateVarDefinition() argument
12526 CodeGenFunction &CGF, QualType VarType, StringRef Name) { in getAddrOfArtificialThreadPrivate() argument
12530 void CGOpenMPSIMDRuntime::emitFlush(CodeGenFunction &CGF, in emitFlush() argument
12537 void CGOpenMPSIMDRuntime::emitTaskCall(CodeGenFunction &CGF, SourceLocation Loc, in emitTaskCall() argument
12547 CodeGenFunction &CGF, SourceLocation Loc, const OMPLoopDirective &D, in emitTaskLoopCall() argument
12554 CodeGenFunction &CGF, SourceLocation Loc, ArrayRef<const Expr *> Privates, in emitReduction() argument
12558 CGOpenMPRuntime::emitReduction(CGF, Loc, Privates, LHSExprs, RHSExprs, in emitReduction()
12563 CodeGenFunction &CGF, SourceLocation Loc, ArrayRef<const Expr *> LHSExprs, in emitTaskReductionInit() argument
12568 void CGOpenMPSIMDRuntime::emitTaskReductionFini(CodeGenFunction &CGF, in emitTaskReductionFini() argument
12574 void CGOpenMPSIMDRuntime::emitTaskReductionFixups(CodeGenFunction &CGF, in emitTaskReductionFixups() argument
12581 Address CGOpenMPSIMDRuntime::getTaskReductionItem(CodeGenFunction &CGF, in getTaskReductionItem() argument
12588 void CGOpenMPSIMDRuntime::emitTaskwaitCall(CodeGenFunction &CGF, in emitTaskwaitCall() argument
12595 CodeGenFunction &CGF, SourceLocation Loc, in emitCancellationPointCall() argument
12600 void CGOpenMPSIMDRuntime::emitCancelCall(CodeGenFunction &CGF, in emitCancelCall() argument
12614 CodeGenFunction &CGF, const OMPExecutableDirective &D, in emitTargetCall() argument
12617 llvm::function_ref<llvm::Value *(CodeGenFunction &CGF, in emitTargetCall() argument
12635 void CGOpenMPSIMDRuntime::emitTeamsCall(CodeGenFunction &CGF, in emitTeamsCall() argument
12643 void CGOpenMPSIMDRuntime::emitNumTeamsClause(CodeGenFunction &CGF, in emitNumTeamsClause() argument
12651 CodeGenFunction &CGF, const OMPExecutableDirective &D, const Expr *IfCond, in emitTargetDataCalls() argument
12658 CodeGenFunction &CGF, const OMPExecutableDirective &D, const Expr *IfCond, in emitTargetDataStandAloneCall() argument
12663 void CGOpenMPSIMDRuntime::emitDoacrossInit(CodeGenFunction &CGF, in emitDoacrossInit() argument
12669 void CGOpenMPSIMDRuntime::emitDoacrossOrdered(CodeGenFunction &CGF, in emitDoacrossOrdered() argument
12681 CGOpenMPSIMDRuntime::getParameterAddress(CodeGenFunction &CGF, in getParameterAddress() argument