Lines Matching refs:Bld
537 CGBuilderTy &Bld = CGF.Builder; in getNVPTXWarpID() local
541 return Bld.CreateAShr(RT.getGPUThreadID(CGF), LaneIDBits, "nvptx_warp_id"); in getNVPTXWarpID()
548 CGBuilderTy &Bld = CGF.Builder; in getNVPTXLaneID() local
552 return Bld.CreateAnd(RT.getGPUThreadID(CGF), Bld.getInt32(LaneIDMask), in getNVPTXLaneID()
563 CGBuilderTy &Bld = CGF.Builder; in getThreadLimit() local
571 ThreadLimit = Bld.CreateNUWSub(GPUNumThreads, GPUWarpSize, "thread_limit"); in getThreadLimit()
585 CGBuilderTy &Bld = CGF.Builder; in getMasterThreadID() local
589 llvm::Value *Mask = Bld.CreateNUWSub(RT.getGPUWarpSize(CGF), Bld.getInt32(1)); in getMasterThreadID()
591 llvm::Value *NumThreadsSubOne = Bld.CreateNUWSub(NumThreads, Bld.getInt32(1)); in getMasterThreadID()
592 return Bld.CreateAnd(NumThreadsSubOne, Bld.CreateNot(Mask), "master_tid"); in getMasterThreadID()
1132 CGBuilderTy &Bld = CGF.Builder; in emitNonSPMDEntryHeader() local
1142 llvm::Value *IsWorker = Bld.CreateICmpULT(GPUThreadID, ThreadLimit); in emitNonSPMDEntryHeader()
1143 Bld.CreateCondBr(IsWorker, WorkerBB, MasterCheckBB); in emitNonSPMDEntryHeader()
1152 llvm::Value *IsMaster = Bld.CreateICmpEQ(GPUThreadID, MasterThreadID); in emitNonSPMDEntryHeader()
1153 Bld.CreateCondBr(IsMaster, MasterBB, EST.ExitBB); in emitNonSPMDEntryHeader()
1162 Bld.getInt16(/*RequiresOMPRuntime=*/1)}; in emitNonSPMDEntryHeader()
1258 CGBuilderTy &Bld = CGF.Builder; in emitSPMDEntryHeader() local
1266 Bld.getInt16(RequiresFullRuntime ? 1 : 0)}; in emitSPMDEntryHeader()
1346 CGBuilderTy &Bld = CGF.Builder; in emitWorkerLoop() local
1366 CGF.InitTempAlloca(ExecStatus, Bld.getInt8(/*C=*/0)); in emitWorkerLoop()
1375 Bld.CreateStore(Bld.CreateZExt(Ret, CGF.Int8Ty), ExecStatus); in emitWorkerLoop()
1378 llvm::Value *WorkID = Bld.CreateLoad(WorkFn); in emitWorkerLoop()
1379 llvm::Value *ShouldTerminate = Bld.CreateIsNull(WorkID, "should_terminate"); in emitWorkerLoop()
1380 Bld.CreateCondBr(ShouldTerminate, ExitBB, SelectWorkersBB); in emitWorkerLoop()
1385 Bld.CreateIsNotNull(Bld.CreateLoad(ExecStatus), "is_active"); in emitWorkerLoop()
1386 Bld.CreateCondBr(IsActive, ExecuteBB, BarrierBB); in emitWorkerLoop()
1396 llvm::Value *ID = Bld.CreatePointerBitCastOrAddrSpaceCast(W, CGM.Int8PtrTy); in emitWorkerLoop()
1399 Bld.CreateICmpEQ(Bld.CreateLoad(WorkFn), ID, "work_match"); in emitWorkerLoop()
1403 Bld.CreateCondBr(WorkFnMatch, ExecuteFNBB, CheckNextBB); in emitWorkerLoop()
1413 {Bld.getInt16(/*ParallelLevel=*/0), getThreadID(CGF, WST.Loc)}); in emitWorkerLoop()
1427 Bld.CreateBitCast(WorkID, ParallelFnTy->getPointerTo()); in emitWorkerLoop()
1433 {Bld.getInt16(/*ParallelLevel=*/0), getThreadID(CGF, WST.Loc)}); in emitWorkerLoop()
1715 CGBuilderTy &Bld = CGF.Builder; in emitGenericVarsProlog() local
1751 IsTTD = Bld.CreateIsNull(PL); in emitGenericVarsProlog()
1753 llvm::Value *IsSPMD = Bld.CreateIsNotNull( in emitGenericVarsProlog()
1756 Bld.CreateCondBr(IsSPMD, SPMDBB, NonSPMDBB); in emitGenericVarsProlog()
1781 Size = Bld.CreateSelect( in emitGenericVarsProlog()
1792 GlobalRecCastAddr = Bld.CreatePointerBitCastOrAddrSpaceCast( in emitGenericVarsProlog()
1795 auto *Phi = Bld.CreatePHI(GlobalRecPtrTy, in emitGenericVarsProlog()
1845 llvm::Value *ResAddr = Bld.CreatePointerBitCastOrAddrSpaceCast( in emitGenericVarsProlog()
1868 Bld.CreateConstInBoundsGEP(FrameAddr, Offset).getPointer(); in emitGenericVarsProlog()
1871 GlobalRecCastAddr = Bld.CreatePointerBitCastOrAddrSpaceCast( in emitGenericVarsProlog()
1887 GlobalRecCastAddr = Bld.CreatePointerBitCastOrAddrSpaceCast( in emitGenericVarsProlog()
1904 Bld.CreatePointerBitCastOrAddrSpaceCast( in emitGenericVarsProlog()
1926 {Bld.getInt32(0), getNVPTXLaneID(CGF)}); in emitGenericVarsProlog()
1944 Address(Bld.CreateSelect(IsTTD, SecVarAddr.getPointer(CGF), in emitGenericVarsProlog()
1952 Bld.CreateSelect(I->getSecond().IsInSPMDModeFlag, in emitGenericVarsProlog()
1971 CGBuilderTy &Bld = CGF.Builder; in emitGenericVarsProlog() local
1974 Size = Bld.CreateNUWAdd( in emitGenericVarsProlog()
1978 Size = Bld.CreateUDiv(Size, AlignVal); in emitGenericVarsProlog()
1979 Size = Bld.CreateNUWMul(Size, AlignVal); in emitGenericVarsProlog()
1988 llvm::Value *GlobalRecCastAddr = Bld.CreatePointerBitCastOrAddrSpaceCast( in emitGenericVarsProlog()
2022 CGBuilderTy &Bld = CGF.Builder; in emitGenericVarsEpilog() local
2025 Bld.CreateCondBr(I->getSecond().IsInSPMDModeFlag, ExitBB, NonSPMDBB); in emitGenericVarsEpilog()
2094 CGBuilderTy &Bld = CGF.Builder; in emitParallelCall() local
2098 ID = Bld.CreateBitOrPointerCast(WFn, CGM.Int8PtrTy); in emitParallelCall()
2102 llvm::Value *FnPtr = Bld.CreateBitOrPointerCast(OutlinedFn, CGM.Int8PtrTy); in emitParallelCall()
2118 Address Dst = Bld.CreateConstArrayGEP(CapturedVarsAddrs, Idx); in emitParallelCall()
2121 PtrV = Bld.CreateIntToPtr(V, CGF.VoidPtrTy); in emitParallelCall()
2123 PtrV = Bld.CreatePointerBitCastOrAddrSpaceCast(V, CGF.VoidPtrTy); in emitParallelCall()
2132 IfCondVal = Bld.CreateIntCast(CGF.EvaluateExprAsBool(IfCond), CGF.Int32Ty, in emitParallelCall()
2147 Bld.CreateBitOrPointerCast(CapturedVarsAddrs.getPointer(), in emitParallelCall()
2296 CGBuilderTy &Bld = CGF.Builder; in createRuntimeShuffleFunction() local
2314 Bld.CreateIntCast(RT.getGPUWarpSize(CGF), CGM.Int16Ty, /*isSigned=*/true); in createRuntimeShuffleFunction()
2326 CGBuilderTy &Bld = CGF.Builder; in shuffleAndStore() local
2341 Address PtrEnd = Bld.CreatePointerBitCastOrAddrSpaceCast( in shuffleAndStore()
2342 Bld.CreateConstGEP(SrcAddr, 1), CGF.VoidPtrTy); in shuffleAndStore()
2350 Ptr = Bld.CreatePointerBitCastOrAddrSpaceCast(Ptr, IntTy->getPointerTo()); in shuffleAndStore()
2352 Bld.CreatePointerBitCastOrAddrSpaceCast(ElemPtr, IntTy->getPointerTo()); in shuffleAndStore()
2357 llvm::BasicBlock *CurrentBB = Bld.GetInsertBlock(); in shuffleAndStore()
2360 Bld.CreatePHI(Ptr.getType(), /*NumReservedValues=*/2); in shuffleAndStore()
2363 Bld.CreatePHI(ElemPtr.getType(), /*NumReservedValues=*/2); in shuffleAndStore()
2367 llvm::Value *PtrDiff = Bld.CreatePtrDiff( in shuffleAndStore()
2368 PtrEnd.getPointer(), Bld.CreatePointerBitCastOrAddrSpaceCast( in shuffleAndStore()
2370 Bld.CreateCondBr(Bld.CreateICmpSGT(PtrDiff, Bld.getInt64(IntSize - 1)), in shuffleAndStore()
2382 Address LocalPtr = Bld.CreateConstGEP(Ptr, 1); in shuffleAndStore()
2383 Address LocalElemPtr = Bld.CreateConstGEP(ElemPtr, 1); in shuffleAndStore()
2398 Ptr = Bld.CreateConstGEP(Ptr, 1); in shuffleAndStore()
2399 ElemPtr = Bld.CreateConstGEP(ElemPtr, 1); in shuffleAndStore()
2435 CGBuilderTy &Bld = CGF.Builder; local
2462 Address SrcElementPtrAddr = Bld.CreateConstArrayGEP(SrcBase, Idx);
2469 DestElementPtrAddr = Bld.CreateConstArrayGEP(DestBase, Idx);
2478 Address SrcElementPtrAddr = Bld.CreateConstArrayGEP(SrcBase, Idx);
2485 DestElementPtrAddr = Bld.CreateConstArrayGEP(DestBase, Idx);
2493 Address SrcElementPtrAddr = Bld.CreateConstArrayGEP(SrcBase, Idx);
2502 Bld.CreateNUWMul(ElementSizeInChars, ScratchpadIndex);
2504 Bld.CreateNUWAdd(DestBase.getPointer(), CurrentOffset);
2506 Bld.CreateIntToPtr(ScratchPadElemAbsolutePtrVal, CGF.VoidPtrTy);
2517 Bld.CreateNUWMul(ElementSizeInChars, ScratchpadIndex);
2519 Bld.CreateNUWAdd(SrcBase.getPointer(), CurrentOffset);
2521 Bld.CreateIntToPtr(ScratchPadElemAbsolutePtrVal, CGF.VoidPtrTy);
2528 DestElementPtrAddr = Bld.CreateConstArrayGEP(DestBase, Idx);
2538 SrcElementAddr = Bld.CreateElementBitCast(
2540 DestElementAddr = Bld.CreateElementBitCast(DestElementAddr,
2585 CGF.EmitStoreOfScalar(Bld.CreatePointerBitCastOrAddrSpaceCast(
2598 ScratchpadBasePtr = Bld.CreateNUWAdd(
2600 Bld.CreateNUWMul(ScratchpadWidth, ElementSizeInChars));
2603 ScratchpadBasePtr = Bld.CreateNUWSub(
2605 ScratchpadBasePtr = Bld.CreateUDiv(
2608 ScratchpadBasePtr = Bld.CreateNUWAdd(
2610 ScratchpadBasePtr = Bld.CreateNUWMul(
2667 CGBuilderTy &Bld = CGF.Builder; in emitInterWarpCopyFunction() local
2702 Bld.CreatePointerBitCastOrAddrSpaceCast( in emitInterWarpCopyFunction()
2743 Bld.CreateICmpULT(Cnt, llvm::ConstantInt::get(CGM.IntTy, NumIters)); in emitInterWarpCopyFunction()
2744 Bld.CreateCondBr(Cmp, BodyBB, ExitBB); in emitInterWarpCopyFunction()
2756 llvm::Value *IsWarpMaster = Bld.CreateIsNull(LaneID, "warp_master"); in emitInterWarpCopyFunction()
2757 Bld.CreateCondBr(IsWarpMaster, ThenBB, ElseBB); in emitInterWarpCopyFunction()
2761 Address ElemPtrPtrAddr = Bld.CreateConstArrayGEP(LocalReduceList, Idx); in emitInterWarpCopyFunction()
2766 ElemPtr = Bld.CreateElementBitCast(ElemPtr, CopyType); in emitInterWarpCopyFunction()
2768 ElemPtr = Address(Bld.CreateGEP(ElemPtr.getPointer(), Cnt), in emitInterWarpCopyFunction()
2774 llvm::Value *MediumPtrVal = Bld.CreateInBoundsGEP( in emitInterWarpCopyFunction()
2780 MediumPtr = Bld.CreateElementBitCast(MediumPtr, CopyType); in emitInterWarpCopyFunction()
2792 Bld.CreateBr(MergeBB); in emitInterWarpCopyFunction()
2795 Bld.CreateBr(MergeBB); in emitInterWarpCopyFunction()
2817 Bld.CreateICmpULT(ThreadID, NumWarpsVal, "is_active_thread"); in emitInterWarpCopyFunction()
2818 Bld.CreateCondBr(IsActiveThread, W0ThenBB, W0ElseBB); in emitInterWarpCopyFunction()
2823 llvm::Value *SrcMediumPtrVal = Bld.CreateInBoundsGEP( in emitInterWarpCopyFunction()
2828 SrcMediumPtr = Bld.CreateElementBitCast(SrcMediumPtr, CopyType); in emitInterWarpCopyFunction()
2831 Address TargetElemPtrPtr = Bld.CreateConstArrayGEP(LocalReduceList, Idx); in emitInterWarpCopyFunction()
2835 TargetElemPtr = Bld.CreateElementBitCast(TargetElemPtr, CopyType); in emitInterWarpCopyFunction()
2837 TargetElemPtr = Address(Bld.CreateGEP(TargetElemPtr.getPointer(), Cnt), in emitInterWarpCopyFunction()
2846 Bld.CreateBr(W0MergeBB); in emitInterWarpCopyFunction()
2849 Bld.CreateBr(W0MergeBB); in emitInterWarpCopyFunction()
2854 Cnt = Bld.CreateNSWAdd(Cnt, llvm::ConstantInt::get(CGM.IntTy, /*V=*/1)); in emitInterWarpCopyFunction()
2969 CGBuilderTy &Bld = CGF.Builder; in emitShuffleAndReduceFunction() local
2973 Bld.CreatePointerBitCastOrAddrSpaceCast( in emitShuffleAndReduceFunction()
3026 llvm::Value *CondAlgo0 = Bld.CreateIsNull(AlgoVerArgVal); in emitShuffleAndReduceFunction()
3028 llvm::Value *Algo1 = Bld.CreateICmpEQ(AlgoVerArgVal, Bld.getInt16(1)); in emitShuffleAndReduceFunction()
3029 llvm::Value *CondAlgo1 = Bld.CreateAnd( in emitShuffleAndReduceFunction()
3030 Algo1, Bld.CreateICmpULT(LaneIDArgVal, RemoteLaneOffsetArgVal)); in emitShuffleAndReduceFunction()
3032 llvm::Value *Algo2 = Bld.CreateICmpEQ(AlgoVerArgVal, Bld.getInt16(2)); in emitShuffleAndReduceFunction()
3033 llvm::Value *CondAlgo2 = Bld.CreateAnd( in emitShuffleAndReduceFunction()
3034 Algo2, Bld.CreateIsNull(Bld.CreateAnd(LaneIDArgVal, Bld.getInt16(1)))); in emitShuffleAndReduceFunction()
3035 CondAlgo2 = Bld.CreateAnd( in emitShuffleAndReduceFunction()
3036 CondAlgo2, Bld.CreateICmpSGT(RemoteLaneOffsetArgVal, Bld.getInt16(0))); in emitShuffleAndReduceFunction()
3038 llvm::Value *CondReduce = Bld.CreateOr(CondAlgo0, CondAlgo1); in emitShuffleAndReduceFunction()
3039 CondReduce = Bld.CreateOr(CondReduce, CondAlgo2); in emitShuffleAndReduceFunction()
3044 Bld.CreateCondBr(CondReduce, ThenBB, ElseBB); in emitShuffleAndReduceFunction()
3048 llvm::Value *LocalReduceListPtr = Bld.CreatePointerBitCastOrAddrSpaceCast( in emitShuffleAndReduceFunction()
3050 llvm::Value *RemoteReduceListPtr = Bld.CreatePointerBitCastOrAddrSpaceCast( in emitShuffleAndReduceFunction()
3054 Bld.CreateBr(MergeBB); in emitShuffleAndReduceFunction()
3057 Bld.CreateBr(MergeBB); in emitShuffleAndReduceFunction()
3063 Algo1 = Bld.CreateICmpEQ(AlgoVerArgVal, Bld.getInt16(1)); in emitShuffleAndReduceFunction()
3064 llvm::Value *CondCopy = Bld.CreateAnd( in emitShuffleAndReduceFunction()
3065 Algo1, Bld.CreateICmpUGE(LaneIDArgVal, RemoteLaneOffsetArgVal)); in emitShuffleAndReduceFunction()
3070 Bld.CreateCondBr(CondCopy, CpyThenBB, CpyElseBB); in emitShuffleAndReduceFunction()
3075 Bld.CreateBr(CpyMergeBB); in emitShuffleAndReduceFunction()
3078 Bld.CreateBr(CpyMergeBB); in emitShuffleAndReduceFunction()
3124 CGBuilderTy &Bld = CGF.Builder; in emitListToGlobalCopyFunction() local
3129 Bld.CreatePointerBitCastOrAddrSpaceCast( in emitListToGlobalCopyFunction()
3137 llvm::Value *BufferArrPtr = Bld.CreatePointerBitCastOrAddrSpaceCast( in emitListToGlobalCopyFunction()
3147 Address ElemPtrPtrAddr = Bld.CreateConstArrayGEP(LocalReduceList, Idx); in emitListToGlobalCopyFunction()
3151 ElemPtrPtr = Bld.CreatePointerBitCastOrAddrSpaceCast( in emitListToGlobalCopyFunction()
3161 llvm::Value *BufferPtr = Bld.CreateInBoundsGEP( in emitListToGlobalCopyFunction()
3233 CGBuilderTy &Bld = CGF.Builder; in emitListToGlobalReduceFunction() local
3239 llvm::Value *BufferArrPtr = Bld.CreatePointerBitCastOrAddrSpaceCast( in emitListToGlobalReduceFunction()
3261 llvm::Value *BufferPtr = Bld.CreateInBoundsGEP( in emitListToGlobalReduceFunction()
3329 CGBuilderTy &Bld = CGF.Builder; in emitGlobalToListCopyFunction() local
3334 Bld.CreatePointerBitCastOrAddrSpaceCast( in emitGlobalToListCopyFunction()
3342 llvm::Value *BufferArrPtr = Bld.CreatePointerBitCastOrAddrSpaceCast( in emitGlobalToListCopyFunction()
3353 Address ElemPtrPtrAddr = Bld.CreateConstArrayGEP(LocalReduceList, Idx); in emitGlobalToListCopyFunction()
3357 ElemPtrPtr = Bld.CreatePointerBitCastOrAddrSpaceCast( in emitGlobalToListCopyFunction()
3367 llvm::Value *BufferPtr = Bld.CreateInBoundsGEP( in emitGlobalToListCopyFunction()
3439 CGBuilderTy &Bld = CGF.Builder; in emitGlobalToListReduceFunction() local
3445 llvm::Value *BufferArrPtr = Bld.CreatePointerBitCastOrAddrSpaceCast( in emitGlobalToListReduceFunction()
3467 llvm::Value *BufferPtr = Bld.CreateInBoundsGEP( in emitGlobalToListReduceFunction()
4076 CGBuilderTy &Bld = CGF.Builder; in createParallelDataSharingWrapper() local
4102 Address Src = Bld.CreateConstInBoundsGEP(SharedArgListAddress, Idx); in createParallelDataSharingWrapper()
4103 Address TypedAddress = Bld.CreatePointerBitCastOrAddrSpaceCast( in createParallelDataSharingWrapper()
4112 Src = Bld.CreateConstInBoundsGEP(SharedArgListAddress, Idx); in createParallelDataSharingWrapper()
4113 TypedAddress = Bld.CreatePointerBitCastOrAddrSpaceCast( in createParallelDataSharingWrapper()
4127 Address Src = Bld.CreateConstInBoundsGEP(SharedArgListAddress, I + Idx); in createParallelDataSharingWrapper()
4128 Address TypedAddress = Bld.CreatePointerBitCastOrAddrSpaceCast( in createParallelDataSharingWrapper()