Lines Matching full:builder
1 //===- OpenMPIRBuilder.cpp - Builder for LLVM-IR for OpenMP directives ----===//
62 #define DEBUG_TYPE "openmp-ir-builder"
68 OptimisticAttributes("openmp-ir-builder-optimistic-attributes", cl::Hidden,
74 "openmp-ir-builder-unroll-threshold-factor", cl::Hidden,
300 void llvm::spliceBB(IRBuilder<> &Builder, BasicBlock *New, bool CreateBranch) {
301 DebugLoc DebugLoc = Builder.getCurrentDebugLocation();
302 BasicBlock *Old = Builder.GetInsertBlock();
304 spliceBB(Builder.saveIP(), New, CreateBranch);
306 Builder.SetInsertPoint(Old->getTerminator());
308 Builder.SetInsertPoint(Old);
310 // SetInsertPoint also updates the Builder's debug location, but we want to
311 // keep the one the Builder was configured to use.
312 Builder.SetCurrentDebugLocation(DebugLoc);
326 BasicBlock *llvm::splitBB(IRBuilderBase &Builder, bool CreateBranch,
328 DebugLoc DebugLoc = Builder.getCurrentDebugLocation();
329 BasicBlock *New = splitBB(Builder.saveIP(), CreateBranch, Name);
331 Builder.SetInsertPoint(Builder.GetInsertBlock()->getTerminator());
333 Builder.SetInsertPoint(Builder.GetInsertBlock());
334 // SetInsertPoint also updates the Builder's debug location, but we want to
335 // keep the one the Builder was configured to use.
336 Builder.SetCurrentDebugLocation(DebugLoc);
340 BasicBlock *llvm::splitBB(IRBuilder<> &Builder, bool CreateBranch,
342 DebugLoc DebugLoc = Builder.getCurrentDebugLocation();
343 BasicBlock *New = splitBB(Builder.saveIP(), CreateBranch, Name);
345 Builder.SetInsertPoint(Builder.GetInsertBlock()->getTerminator());
347 Builder.SetInsertPoint(Builder.GetInsertBlock());
348 // SetInsertPoint also updates the Builder's debug location, but we want to
349 // keep the one the Builder was configured to use.
350 Builder.SetCurrentDebugLocation(DebugLoc);
354 BasicBlock *llvm::splitBBWithSuffix(IRBuilderBase &Builder, bool CreateBranch,
356 BasicBlock *Old = Builder.GetInsertBlock();
357 return splitBB(Builder, CreateBranch, Old->getName() + Suffix);
363 Value *createFakeIntVal(IRBuilderBase &Builder,
368 Builder.restoreIP(OuterAllocaIP);
371 Builder.CreateAlloca(Builder.getInt32Ty(), nullptr, Name + ".addr");
378 Builder.CreateLoad(Builder.getInt32Ty(), FakeValAddr, Name + ".val");
383 Builder.restoreIP(InnerAllocaIP);
387 Builder.CreateLoad(Builder.getInt32Ty(), FakeVal, Name + ".use");
390 cast<BinaryOperator>(Builder.CreateAdd(FakeVal, Builder.getInt32(10)));
495 IRBuilderBase &Builder,
497 Value *Version = Builder.getInt32(OMP_KERNEL_ARG_VERSION);
498 Value *PointerNum = Builder.getInt32(KernelArgs.NumTargetItems);
499 auto Int32Ty = Type::getInt32Ty(Builder.getContext());
501 Value *Flags = Builder.getInt64(KernelArgs.HasNoWait);
504 Builder.CreateInsertValue(ZeroArray, KernelArgs.NumTeams, {0});
506 Builder.CreateInsertValue(ZeroArray, KernelArgs.NumThreads, {0});
639 static void raiseUserConstantDataAllocasToEntryBlock(IRBuilderBase &Builder,
786 raiseUserConstantDataAllocasToEntryBlock(Builder, F);
876 SrcLocStr = Builder.CreateGlobalStringPtr(LocStr, /* Name */ "",
930 return Builder.CreateCall(
976 Builder.CreateCall(getOrCreateRuntimeFunctionPtr(
984 return Builder.saveIP();
995 auto *UI = Builder.CreateUnreachable();
1000 Builder.SetInsertPoint(ThenTI);
1006 CancelKind = Builder.getInt32(Value); \
1017 Value *Result = Builder.CreateCall(
1021 IRBuilder<>::InsertPointGuard IPG(Builder);
1022 Builder.restoreIP(IP);
1023 createBarrier(LocationDescription(Builder.saveIP(), Loc.DL),
1033 Builder.SetInsertPoint(UI->getParent());
1036 return Builder.saveIP();
1046 Builder.restoreIP(AllocaIP);
1048 Builder.CreateAlloca(OpenMPIRBuilder::KernelArgs, nullptr, "kernel_args");
1049 Builder.restoreIP(Loc.IP);
1053 Builder.CreateStructGEP(OpenMPIRBuilder::KernelArgs, KernelArgsPtr, I);
1054 Builder.CreateAlignedStore(
1062 Return = Builder.CreateCall(
1066 return Builder.saveIP();
1077 Builder.restoreIP(Loc.IP);
1096 getKernelArgsVector(Args, Builder, ArgsVector);
1111 Builder.restoreIP(emitTargetKernel(Builder, AllocaIP, Return, RTLoc, DeviceID,
1116 BasicBlock::Create(Builder.getContext(), "omp_offload.failed");
1118 BasicBlock::Create(Builder.getContext(), "omp_offload.cont");
1119 Value *Failed = Builder.CreateIsNotNull(Return);
1120 Builder.CreateCondBr(Failed, OffloadFailedBlock, OffloadContBlock);
1122 auto CurFn = Builder.GetInsertBlock()->getParent();
1124 Builder.restoreIP(emitTargetCallFallbackCB(Builder.saveIP()));
1127 return Builder.saveIP();
1137 BasicBlock *BB = Builder.GetInsertBlock();
1139 if (Builder.GetInsertPoint() == BB->end()) {
1145 NonCancellationBlock = SplitBlock(BB, &*Builder.GetInsertPoint());
1147 Builder.SetInsertPoint(BB);
1153 Value *Cmp = Builder.CreateIsNull(CancelFlag);
1154 Builder.CreateCondBr(Cmp, NonCancellationBlock, CancellationBlock,
1159 Builder.SetInsertPoint(CancellationBlock);
1161 ExitCB(Builder.saveIP());
1163 FI.FiniCB(Builder.saveIP());
1166 Builder.SetInsertPoint(NonCancellationBlock, NonCancellationBlock->begin());
1179 IRBuilder<> &Builder = OMPIRBuilder->Builder;
1194 Builder.SetInsertPoint(CI);
1199 OpenMPIRBuilder ::InsertPointTy CurrentIP = Builder.saveIP();
1200 Builder.SetInsertPoint(OuterAllocaBB, OuterAllocaBB->getFirstInsertionPt());
1202 Builder.CreateAlloca(ArrayType::get(PtrTy, NumCapturedVars));
1207 Args = Builder.CreatePointerCast(ArgsAlloca, PtrTy);
1208 Builder.restoreIP(CurrentIP);
1213 Value *StoreAddress = Builder.CreateConstInBoundsGEP2_64(
1215 Builder.CreateStore(V, StoreAddress);
1219 IfCondition ? Builder.CreateSExtOrTrunc(IfCondition, OMPIRBuilder->Int32)
1220 : Builder.getInt32(1);
1227 /* number of threads */ NumThreads ? NumThreads : Builder.getInt32(-1),
1228 /* Proc bind */ Builder.getInt32(-1),
1230 Builder.CreateBitCast(&OutlinedFn, OMPIRBuilder->ParallelTaskPtr),
1233 /* number of arguments */ Builder.getInt64(NumCapturedVars)};
1238 Builder.CreateCall(RTLFn, Parallel51CallArgs);
1241 << *Builder.GetInsertBlock()->getParent() << "\n");
1244 Builder.SetInsertPoint(PrivTID);
1246 Builder.CreateStore(Builder.CreateLoad(OMPIRBuilder->Int32, OutlinedAI),
1266 IRBuilder<> &Builder = OMPIRBuilder->Builder;
1301 Builder.SetInsertPoint(CI);
1305 Ident, Builder.getInt32(NumCapturedVars),
1306 Builder.CreateBitCast(&OutlinedFn, OMPIRBuilder->ParallelTaskPtr)};
1311 Value *Cond = Builder.CreateSExtOrTrunc(IfCondition, OMPIRBuilder->Int32);
1324 RealArgs.back() = Builder.CreateBitCast(RealArgs.back(), PtrTy);
1326 Builder.CreateCall(RTLFn, RealArgs);
1329 << *Builder.GetInsertBlock()->getParent() << "\n");
1332 Builder.SetInsertPoint(PrivTID);
1334 Builder.CreateStore(Builder.CreateLoad(OMPIRBuilder->Int32, OutlinedAI),
1371 Builder.CreateIntCast(NumThreads, Int32, /*isSigned*/ false)};
1372 Builder.CreateCall(
1381 Builder.CreateCall(
1385 BasicBlock *InsertBB = Builder.GetInsertBlock();
1399 Builder.restoreIP(NewOuter);
1400 AllocaInst *TIDAddrAlloca = Builder.CreateAlloca(Int32, nullptr, "tid.addr");
1402 Builder.CreateAlloca(Int32, nullptr, "zero.addr");
1425 auto *UI = new UnreachableInst(Builder.getContext(), InsertBB);
1438 IRBuilder<>::InsertPointGuard IPG(Builder);
1439 Builder.restoreIP(IP);
1440 Instruction *I = Builder.CreateBr(PRegExitBB);
1453 Builder.SetInsertPoint(PRegEntryBB->getTerminator());
1454 InsertPointTy InnerAllocaIP = Builder.saveIP();
1457 Builder.CreateAlloca(Int32, nullptr, "tid.addr.local");
1458 Instruction *PrivTID = Builder.CreateLoad(Int32, PrivTIDAddr, "tid");
1461 ToBeDeleted.push_back(Builder.CreateLoad(Int32, TIDAddr, "tid.addr.use"));
1463 Builder.CreateLoad(Int32, ZeroAddr, "zero.addr.use");
1566 IRBuilder<>::InsertPointGuard Guard(Builder);
1569 Builder.restoreIP(OuterAllocaIP);
1571 Builder.CreateAlloca(V.getType(), nullptr, V.getName() + ".reloaded");
1575 Builder.SetInsertPoint(InsertBB,
1577 Builder.CreateStore(&V, Ptr);
1580 Builder.restoreIP(InnerAllocaIP);
1581 Inner = Builder.CreateLoad(V.getType(), Ptr);
1589 Builder.restoreIP(
1590 PrivCB(InnerAllocaIP, Builder.saveIP(), V, *Inner, ReplacementValue));
1664 Builder.CreateCall(getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_flush), Args);
1682 Builder.CreateCall(getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_omp_taskwait),
1700 Builder.CreateCall(getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_omp_taskyield),
1738 IRBuilderBase &Builder = OMPBuilder.Builder;
1743 OpenMPIRBuilder::InsertPointTy OldIP = Builder.saveIP();
1744 Builder.SetInsertPoint(
1748 DepArray = Builder.CreateAlloca(DepArrayTy, nullptr, ".dep.arr.addr");
1752 Builder.CreateConstInBoundsGEP2_64(DepArrayTy, DepArray, 0, DepIdx);
1754 Value *Addr = Builder.CreateStructGEP(
1757 Value *DepValPtr = Builder.CreatePtrToInt(Dep.DepVal, Builder.getInt64Ty());
1758 Builder.CreateStore(DepValPtr, Addr);
1760 Value *Size = Builder.CreateStructGEP(
1762 Builder.CreateStore(
1763 Builder.getInt64(M.getDataLayout().getTypeStoreSize(Dep.DepValueType)),
1766 Value *Flags = Builder.CreateStructGEP(
1769 Builder.CreateStore(
1770 ConstantInt::get(Builder.getInt8Ty(),
1774 Builder.restoreIP(OldIP);
1806 BasicBlock *TaskExitBB = splitBB(Builder, /*CreateBranch=*/true, "task.exit");
1807 BasicBlock *TaskBodyBB = splitBB(Builder, /*CreateBranch=*/true, "task.body");
1809 splitBB(Builder, /*CreateBranch=*/true, "task.alloca");
1824 Builder, AllocaIP, ToBeDeleted, TaskAllocaIP, "global.tid", false));
1836 Builder.SetInsertPoint(StaleCI);
1853 Value *Flags = Builder.getInt32(Tied);
1856 Builder.CreateSelect(Final, Builder.getInt32(2), Builder.getInt32(0));
1857 Flags = Builder.CreateOr(FinalFlag, Flags);
1864 Value *TaskSize = Builder.getInt64(
1870 Value *SharedsSize = Builder.getInt64(0);
1882 Builder.getInt64(M.getDataLayout().getTypeStoreSize(ArgStructType));
1887 CallInst *TaskData = Builder.CreateCall(
1896 Value *TaskShareds = Builder.CreateLoad(VoidPtr, TaskData);
1897 Builder.CreateMemCpy(TaskShareds, Alignment, Shareds, Alignment,
1903 InsertPointTy OldIP = Builder.saveIP();
1904 Builder.SetInsertPoint(
1908 DepArray = Builder.CreateAlloca(DepArrayTy, nullptr, ".dep.arr.addr");
1913 Builder.CreateConstInBoundsGEP2_64(DepArrayTy, DepArray, 0, P);
1915 Value *Addr = Builder.CreateStructGEP(
1919 Builder.CreatePtrToInt(Dep.DepVal, Builder.getInt64Ty());
1920 Builder.CreateStore(DepValPtr, Addr);
1922 Value *Size = Builder.CreateStructGEP(
1925 Builder.CreateStore(Builder.getInt64(M.getDataLayout().getTypeStoreSize(
1929 Value *Flags = Builder.CreateStructGEP(
1932 Builder.CreateStore(
1933 ConstantInt::get(Builder.getInt8Ty(),
1939 Builder.restoreIP(OldIP);
1962 splitBB(Builder, /*CreateBranch=*/true, "if.end");
1964 Builder.GetInsertPoint()->getParent()->getTerminator();
1966 Builder.SetInsertPoint(IfTerminator);
1969 Builder.SetInsertPoint(ElseTI);
1974 Builder.CreateCall(
1976 {Ident, ThreadID, Builder.getInt32(Dependencies.size()), DepArray,
1977 ConstantInt::get(Builder.getInt32Ty(), 0),
1984 Builder.CreateCall(TaskBeginFn, {Ident, ThreadID, TaskData});
1987 CI = Builder.CreateCall(&OutlinedFn, {ThreadID, TaskData});
1989 CI = Builder.CreateCall(&OutlinedFn, {ThreadID});
1991 Builder.CreateCall(TaskCompleteFn, {Ident, ThreadID, TaskData});
1992 Builder.SetInsertPoint(ThenTI);
1998 Builder.CreateCall(
2000 {Ident, ThreadID, TaskData, Builder.getInt32(Dependencies.size()),
2001 DepArray, ConstantInt::get(Builder.getInt32Ty(), 0),
2007 Builder.CreateCall(TaskFn, {Ident, ThreadID, TaskData});
2012 Builder.SetInsertPoint(TaskAllocaBB, TaskAllocaBB->begin());
2014 LoadInst *Shareds = Builder.CreateLoad(VoidPtr, OutlinedFn.getArg(1));
2024 Builder.SetInsertPoint(TaskExitBB, TaskExitBB->begin());
2026 return Builder.saveIP();
2044 Builder.CreateCall(TaskgroupFn, {Ident, ThreadID});
2046 BasicBlock *TaskgroupExitBB = splitBB(Builder, true, "taskgroup.exit");
2047 BodyGenCB(AllocaIP, Builder.saveIP());
2049 Builder.SetInsertPoint(TaskgroupExitBB);
2053 Builder.CreateCall(EndTaskgroupFn, {Ident, ThreadID});
2055 return Builder.saveIP();
2077 IRBuilder<>::InsertPointGuard IPG(Builder);
2078 Builder.restoreIP(IP);
2082 Instruction *I = Builder.CreateBr(ExitBB);
2106 Builder.restoreIP(CodeGenIP);
2108 splitBBWithSuffix(Builder, /*CreateBranch=*/false, ".sections.after");
2110 SwitchInst *SwitchStmt = Builder.CreateSwitch(IndVar, Continue);
2116 SwitchStmt->addCase(Builder.getInt32(CaseNumber), CaseBB);
2117 Builder.SetInsertPoint(CaseBB);
2118 BranchInst *CaseEndBr = Builder.CreateBr(Continue);
2142 Builder.restoreIP(AfterIP);
2144 splitBBWithSuffix(Builder, /*CreateBranch=*/true, "sections.fini");
2145 CB(Builder.saveIP());
2169 IRBuilder<>::InsertPointGuard IPG(Builder);
2170 Builder.restoreIP(IP);
2174 Instruction *I = Builder.CreateBr(ExitBB);
2203 cast<Constant>(&*List[I]), Builder.getPtrTy());
2207 ArrayType *ATy = ArrayType::get(Builder.getPtrTy(), UsedArray.size());
2216 return Builder.CreateCall(
2223 return Builder.CreateCall(
2229 return Builder.CreateAShr(getGPUThreadID(), LaneIDBits, "nvptx_warp_id");
2236 return Builder.CreateAnd(getGPUThreadID(), Builder.getInt32(LaneIDMask),
2250 return Builder.CreateBitCast(From, ToType);
2252 return Builder.CreateIntCast(From, ToType, /*isSigned*/ true);
2253 InsertPointTy SaveIP = Builder.saveIP();
2254 Builder.restoreIP(AllocaIP);
2255 Value *CastItem = Builder.CreateAlloca(ToType);
2256 Builder.restoreIP(SaveIP);
2258 Value *ValCastItem = Builder.CreatePointerBitCastOrAddrSpaceCast(
2260 Builder.CreateStore(From, ValCastItem);
2261 return Builder.CreateLoad(ToType, CastItem);
2272 Type *CastTy = Builder.getIntNTy(Size <= 4 ? 32 : 64);
2275 Builder.CreateIntCast(getGPUWarpSize(), Builder.getInt16Ty(), true);
2280 Builder.CreateIntCast(WarpSize, Builder.getInt16Ty(), /*isSigned=*/true);
2282 Builder.CreateCall(ShuffleFunc, {ElemCast, Offset, WarpSizeCast});
2300 Type *IndexTy = Builder.getIndexTy(
2307 Type *IntType = Builder.getIntNTy(IntSize * 8);
2308 Ptr = Builder.CreatePointerBitCastOrAddrSpaceCast(
2311 Builder.CreateGEP(ElemType, SrcAddr, {ConstantInt::get(IndexTy, 1)});
2312 ElemPtr = Builder.CreatePointerBitCastOrAddrSpaceCast(
2315 Function *CurFunc = Builder.GetInsertBlock()->getParent();
2317 Value *PtrEnd = Builder.CreatePointerBitCastOrAddrSpaceCast(
2318 SrcAddrGEP, Builder.getPtrTy());
2323 BasicBlock *CurrentBB = Builder.GetInsertBlock();
2326 Builder.CreatePHI(Ptr->getType(), /*NumReservedValues=*/2);
2329 Builder.CreatePHI(ElemPtr->getType(), /*NumReservedValues=*/2);
2333 Value *PtrDiff = Builder.CreatePtrDiff(
2334 Builder.getInt8Ty(), PtrEnd,
2335 Builder.CreatePointerBitCastOrAddrSpaceCast(Ptr, Builder.getPtrTy()));
2336 Builder.CreateCondBr(
2337 Builder.CreateICmpSGT(PtrDiff, Builder.getInt64(IntSize - 1)), ThenBB,
2342 Builder.CreateAlignedLoad(
2345 Builder.CreateAlignedStore(Res, ElemPtr,
2348 Builder.CreateGEP(IntType, Ptr, {ConstantInt::get(IndexTy, 1)});
2350 Builder.CreateGEP(IntType, ElemPtr, {ConstantInt::get(IndexTy, 1)});
2357 AllocaIP, Builder.CreateLoad(IntType, Ptr), IntType, Offset);
2360 Res = Builder.CreateTrunc(Res, ElemType);
2361 Builder.CreateStore(Res, ElemPtr);
2362 Ptr = Builder.CreateGEP(IntType, Ptr, {ConstantInt::get(IndexTy, 1)});
2364 Builder.CreateGEP(IntType, ElemPtr, {ConstantInt::get(IndexTy, 1)});
2374 Type *IndexTy = Builder.getIndexTy(
2392 Value *SrcElementPtrAddr = Builder.CreateInBoundsGEP(
2395 SrcElementAddr = Builder.CreateLoad(Builder.getPtrTy(), SrcElementPtrAddr);
2399 DestElementPtrAddr = Builder.CreateInBoundsGEP(
2404 InsertPointTy CurIP = Builder.saveIP();
2405 Builder.restoreIP(AllocaIP);
2406 AllocaInst *DestAlloca = Builder.CreateAlloca(RI.ElementType, nullptr,
2412 Builder.CreateAddrSpaceCast(DestElementAddr, Builder.getPtrTy(),
2414 Builder.restoreIP(CurIP);
2421 Builder.CreateLoad(Builder.getPtrTy(), DestElementPtrAddr);
2434 Value *Elem = Builder.CreateLoad(RI.ElementType, SrcElementAddr);
2436 Builder.CreateStore(Elem, DestElementAddr);
2440 Value *SrcRealPtr = Builder.CreateConstInBoundsGEP2_32(
2442 Value *SrcReal = Builder.CreateLoad(
2444 Value *SrcImgPtr = Builder.CreateConstInBoundsGEP2_32(
2446 Value *SrcImg = Builder.CreateLoad(
2449 Value *DestRealPtr = Builder.CreateConstInBoundsGEP2_32(
2451 Value *DestImgPtr = Builder.CreateConstInBoundsGEP2_32(
2453 Builder.CreateStore(SrcReal, DestRealPtr);
2454 Builder.CreateStore(SrcImg, DestImgPtr);
2458 Value *SizeVal = Builder.getInt64(
2460 Builder.CreateMemCpy(
2475 Value *CastDestAddr = Builder.CreatePointerBitCastOrAddrSpaceCast(
2476 DestElementAddr, Builder.getPtrTy(),
2478 Builder.CreateStore(CastDestAddr, DestElementPtrAddr);
2486 InsertPointTy SavedIP = Builder.saveIP();
2489 Builder.getVoidTy(), {Builder.getPtrTy(), Builder.getInt32Ty()},
2498 Builder.SetInsertPoint(EntryBB);
2519 ArrayType *ArrayTy = ArrayType::get(Builder.getInt32Ty(), WarpSize);
2536 InsertPointTy(Builder.GetInsertBlock(),
2537 Builder.GetInsertBlock()->getFirstInsertionPt());
2540 Builder.restoreIP(AllocaIP);
2541 AllocaInst *ReduceListAlloca = Builder.CreateAlloca(
2544 Builder.CreateAlloca(Arg1Type, nullptr, NumWarpsArg->getName() + ".addr");
2545 Value *ReduceListAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
2547 Value *NumWarpsAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
2550 Builder.CreateStore(ReduceListArg, ReduceListAddrCast);
2551 Builder.CreateStore(NumWarpsArg, NumWarpsAddrCast);
2554 getInsertPointAfterInstr(&Builder.GetInsertBlock()->back());
2555 Builder.restoreIP(CodeGenIP);
2558 Builder.CreateLoad(Builder.getPtrTy(), ReduceListAddrCast);
2568 Type *CType = Builder.getIntNTy(TySize * 8);
2578 CodeGenIP = Builder.saveIP();
2579 Builder.restoreIP(AllocaIP);
2581 Builder.CreateAlloca(Builder.getInt32Ty(), nullptr, ".cnt.addr");
2583 CntAddr = Builder.CreateAddrSpaceCast(CntAddr, Builder.getPtrTy(),
2585 Builder.restoreIP(CodeGenIP);
2586 Builder.CreateStore(Constant::getNullValue(Builder.getInt32Ty()),
2592 emitBlock(PrecondBB, Builder.GetInsertBlock()->getParent());
2593 Cnt = Builder.CreateLoad(Builder.getInt32Ty(), CntAddr,
2595 Value *Cmp = Builder.CreateICmpULT(
2596 Cnt, ConstantInt::get(Builder.getInt32Ty(), NumIters));
2597 Builder.CreateCondBr(Cmp, BodyBB, ExitBB);
2598 emitBlock(BodyBB, Builder.GetInsertBlock()->getParent());
2602 createBarrier(LocationDescription(Builder.saveIP(), Loc.DL),
2611 Value *IsWarpMaster = Builder.CreateIsNull(LaneID, "warp_master");
2612 Builder.CreateCondBr(IsWarpMaster, ThenBB, ElseBB);
2613 emitBlock(ThenBB, Builder.GetInsertBlock()->getParent());
2617 ArrayType::get(Builder.getPtrTy(), ReductionInfos.size());
2618 Type *IndexTy = Builder.getIndexTy(
2621 Builder.CreateInBoundsGEP(RedListArrayTy, ReduceList,
2625 Value *ElemPtr = Builder.CreateLoad(Builder.getPtrTy(), ElemPtrPtr);
2627 ElemPtr = Builder.CreateGEP(Builder.getInt32Ty(), ElemPtr, Cnt);
2631 Value *MediumPtr = Builder.CreateInBoundsGEP(
2632 ArrayTy, TransferMedium, {Builder.getInt64(0), WarpID});
2635 Value *Elem = Builder.CreateLoad(CType, ElemPtr);
2637 Builder.CreateStore(Elem, MediumPtr,
2639 Builder.CreateBr(MergeBB);
2642 emitBlock(ElseBB, Builder.GetInsertBlock()->getParent());
2643 Builder.CreateBr(MergeBB);
2646 emitBlock(MergeBB, Builder.GetInsertBlock()->getParent());
2647 createBarrier(LocationDescription(Builder.saveIP(), Loc.DL),
2658 Builder.CreateLoad(Builder.getInt32Ty(), NumWarpsAddrCast);
2661 Builder.CreateICmpULT(GPUThreadID, NumWarpsVal, "is_active_thread");
2662 Builder.CreateCondBr(IsActiveThread, W0ThenBB, W0ElseBB);
2664 emitBlock(W0ThenBB, Builder.GetInsertBlock()->getParent());
2668 Value *SrcMediumPtrVal = Builder.CreateInBoundsGEP(
2669 ArrayTy, TransferMedium, {Builder.getInt64(0), GPUThreadID});
2672 Builder.CreateInBoundsGEP(RedListArrayTy, ReduceList,
2676 Builder.CreateLoad(Builder.getPtrTy(), TargetElemPtrPtr);
2680 Builder.CreateGEP(Builder.getInt32Ty(), TargetElemPtr, Cnt);
2684 Builder.CreateLoad(CType, SrcMediumPtrVal, /*IsVolatile*/ true);
2685 Builder.CreateStore(SrcMediumValue, TargetElemPtr);
2686 Builder.CreateBr(W0MergeBB);
2688 emitBlock(W0ElseBB, Builder.GetInsertBlock()->getParent());
2689 Builder.CreateBr(W0MergeBB);
2691 emitBlock(W0MergeBB, Builder.GetInsertBlock()->getParent());
2694 Cnt = Builder.CreateNSWAdd(
2695 Cnt, ConstantInt::get(Builder.getInt32Ty(), /*V=*/1));
2696 Builder.CreateStore(Cnt, CntAddr, /*Volatile=*/false);
2698 auto *CurFn = Builder.GetInsertBlock()->getParent();
2706 Builder.CreateRetVoid();
2707 Builder.restoreIP(SavedIP);
2717 FunctionType::get(Builder.getVoidTy(),
2718 {Builder.getPtrTy(), Builder.getInt16Ty(),
2719 Builder.getInt16Ty(), Builder.getInt16Ty()},
2733 Builder.SetInsertPoint(EntryBB);
2747 Value *ReduceListAlloca = Builder.CreateAlloca(
2749 Value *LaneIdAlloca = Builder.CreateAlloca(LaneIDArgType, nullptr,
2751 Value *RemoteLaneOffsetAlloca = Builder.CreateAlloca(
2753 Value *AlgoVerAlloca = Builder.CreateAlloca(LaneIDArgType, nullptr,
2756 ArrayType::get(Builder.getPtrTy(), ReductionInfos.size());
2760 Instruction *RemoteReductionListAlloca = Builder.CreateAlloca(
2763 Value *ReduceListAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
2766 Value *LaneIdAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
2768 Value *RemoteLaneOffsetAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
2771 Value *AlgoVerAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
2773 Value *RemoteListAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
2774 RemoteReductionListAlloca, Builder.getPtrTy(),
2777 Builder.CreateStore(ReduceListArg, ReduceListAddrCast);
2778 Builder.CreateStore(LaneIDArg, LaneIdAddrCast);
2779 Builder.CreateStore(RemoteLaneOffsetArg, RemoteLaneOffsetAddrCast);
2780 Builder.CreateStore(AlgoVerArg, AlgoVerAddrCast);
2782 Value *ReduceList = Builder.CreateLoad(ReduceListArgType, ReduceListAddrCast);
2783 Value *LaneId = Builder.CreateLoad(LaneIDArgType, LaneIdAddrCast);
2785 Builder.CreateLoad(LaneIDArgType, RemoteLaneOffsetAddrCast);
2786 Value *AlgoVer = Builder.CreateLoad(LaneIDArgType, AlgoVerAddrCast);
2818 Value *CondAlgo0 = Builder.CreateIsNull(AlgoVer);
2819 Value *Algo1 = Builder.CreateICmpEQ(AlgoVer, Builder.getInt16(1));
2820 Value *LaneComp = Builder.CreateICmpULT(LaneId, RemoteLaneOffset);
2821 Value *CondAlgo1 = Builder.CreateAnd(Algo1, LaneComp);
2822 Value *Algo2 = Builder.CreateICmpEQ(AlgoVer, Builder.getInt16(2));
2823 Value *LaneIdAnd1 = Builder.CreateAnd(LaneId, Builder.getInt16(1));
2824 Value *LaneIdComp = Builder.CreateIsNull(LaneIdAnd1);
2825 Value *Algo2AndLaneIdComp = Builder.CreateAnd(Algo2, LaneIdComp);
2827 Builder.CreateICmpSGT(RemoteLaneOffset, Builder.getInt16(0));
2828 Value *CondAlgo2 = Builder.CreateAnd(Algo2AndLaneIdComp, RemoteOffsetComp);
2829 Value *CA0OrCA1 = Builder.CreateOr(CondAlgo0, CondAlgo1);
2830 Value *CondReduce = Builder.CreateOr(CA0OrCA1, CondAlgo2);
2836 Builder.CreateCondBr(CondReduce, ThenBB, ElseBB);
2837 emitBlock(ThenBB, Builder.GetInsertBlock()->getParent());
2838 Value *LocalReduceListPtr = Builder.CreatePointerBitCastOrAddrSpaceCast(
2839 ReduceList, Builder.getPtrTy());
2840 Value *RemoteReduceListPtr = Builder.CreatePointerBitCastOrAddrSpaceCast(
2841 RemoteListAddrCast, Builder.getPtrTy());
2842 Builder.CreateCall(ReduceFn, {LocalReduceListPtr, RemoteReduceListPtr})
2844 Builder.CreateBr(MergeBB);
2846 emitBlock(ElseBB, Builder.GetInsertBlock()->getParent());
2847 Builder.CreateBr(MergeBB);
2849 emitBlock(MergeBB, Builder.GetInsertBlock()->getParent());
2853 Algo1 = Builder.CreateICmpEQ(AlgoVer, Builder.getInt16(1));
2854 Value *LaneIdGtOffset = Builder.CreateICmpUGE(LaneId, RemoteLaneOffset);
2855 Value *CondCopy = Builder.CreateAnd(Algo1, LaneIdGtOffset);
2860 Builder.CreateCondBr(CondCopy, CpyThenBB, CpyElseBB);
2862 emitBlock(CpyThenBB, Builder.GetInsertBlock()->getParent());
2865 Builder.CreateBr(CpyMergeBB);
2867 emitBlock(CpyElseBB, Builder.GetInsertBlock()->getParent());
2868 Builder.CreateBr(CpyMergeBB);
2870 emitBlock(CpyMergeBB, Builder.GetInsertBlock()->getParent());
2872 Builder.CreateRetVoid();
2880 OpenMPIRBuilder::InsertPointTy OldIP = Builder.saveIP();
2883 Builder.getVoidTy(),
2884 {Builder.getPtrTy(), Builder.getInt32Ty(), Builder.getPtrTy()},
2895 Builder.SetInsertPoint(EntryBlock);
2904 Value *BufferArgAlloca = Builder.CreateAlloca(Builder.getPtrTy(), nullptr,
2906 Value *IdxArgAlloca = Builder.CreateAlloca(Builder.getInt32Ty(), nullptr,
2908 Value *ReduceListArgAlloca = Builder.CreateAlloca(
2909 Builder.getPtrTy(), nullptr, ReduceListArg->getName() + ".addr");
2910 Value *BufferArgAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
2911 BufferArgAlloca, Builder.getPtrTy(),
2913 Value *IdxArgAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
2914 IdxArgAlloca, Builder.getPtrTy(), IdxArgAlloca->getName() + ".ascast");
2915 Value *ReduceListArgAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
2916 ReduceListArgAlloca, Builder.getPtrTy(),
2919 Builder.CreateStore(BufferArg, BufferArgAddrCast);
2920 Builder.CreateStore(IdxArg, IdxArgAddrCast);
2921 Builder.CreateStore(ReduceListArg, ReduceListArgAddrCast);
2924 Builder.CreateLoad(Builder.getPtrTy(), ReduceListArgAddrCast);
2926 Builder.CreateLoad(Builder.getPtrTy(), BufferArgAddrCast);
2927 Value *Idxs[] = {Builder.CreateLoad(Builder.getInt32Ty(), IdxArgAddrCast)};
2928 Type *IndexTy = Builder.getIndexTy(
2933 ArrayType::get(Builder.getPtrTy(), ReductionInfos.size());
2935 Value *ElemPtrPtr = Builder.CreateInBoundsGEP(
2939 Value *ElemPtr = Builder.CreateLoad(Builder.getPtrTy(), ElemPtrPtr);
2943 Builder.CreateInBoundsGEP(ReductionsBufferTy, BufferArgVal, Idxs);
2944 Value *GlobVal = Builder.CreateConstInBoundsGEP2_32(
2949 Value *TargetElement = Builder.CreateLoad(RI.ElementType, ElemPtr);
2950 Builder.CreateStore(TargetElement, GlobVal);
2954 Value *SrcRealPtr = Builder.CreateConstInBoundsGEP2_32(
2956 Value *SrcReal = Builder.CreateLoad(
2958 Value *SrcImgPtr = Builder.CreateConstInBoundsGEP2_32(
2960 Value *SrcImg = Builder.CreateLoad(
2963 Value *DestRealPtr = Builder.CreateConstInBoundsGEP2_32(
2965 Value *DestImgPtr = Builder.CreateConstInBoundsGEP2_32(
2967 Builder.CreateStore(SrcReal, DestRealPtr);
2968 Builder.CreateStore(SrcImg, DestImgPtr);
2973 Builder.getInt64(M.getDataLayout().getTypeStoreSize(RI.ElementType));
2974 Builder.CreateMemCpy(
2982 Builder.CreateRetVoid();
2983 Builder.restoreIP(OldIP);
2990 OpenMPIRBuilder::InsertPointTy OldIP = Builder.saveIP();
2993 Builder.getVoidTy(),
2994 {Builder.getPtrTy(), Builder.getInt32Ty(), Builder.getPtrTy()},
3005 Builder.SetInsertPoint(EntryBlock);
3014 Value *BufferArgAlloca = Builder.CreateAlloca(Builder.getPtrTy(), nullptr,
3016 Value *IdxArgAlloca = Builder.CreateAlloca(Builder.getInt32Ty(), nullptr,
3018 Value *ReduceListArgAlloca = Builder.CreateAlloca(
3019 Builder.getPtrTy(), nullptr, ReduceListArg->getName() + ".addr");
3021 ArrayType::get(Builder.getPtrTy(), ReductionInfos.size());
3026 Builder.CreateAlloca(RedListArrayTy, nullptr, ".omp.reduction.red_list");
3028 Value *BufferArgAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
3029 BufferArgAlloca, Builder.getPtrTy(),
3031 Value *IdxArgAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
3032 IdxArgAlloca, Builder.getPtrTy(), IdxArgAlloca->getName() + ".ascast");
3033 Value *ReduceListArgAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
3034 ReduceListArgAlloca, Builder.getPtrTy(),
3036 Value *LocalReduceListAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
3037 LocalReduceList, Builder.getPtrTy(),
3040 Builder.CreateStore(BufferArg, BufferArgAddrCast);
3041 Builder.CreateStore(IdxArg, IdxArgAddrCast);
3042 Builder.CreateStore(ReduceListArg, ReduceListArgAddrCast);
3044 Value *BufferVal = Builder.CreateLoad(Builder.getPtrTy(), BufferArgAddrCast);
3045 Value *Idxs[] = {Builder.CreateLoad(Builder.getInt32Ty(), IdxArgAddrCast)};
3046 Type *IndexTy = Builder.getIndexTy(
3049 Value *TargetElementPtrPtr = Builder.CreateInBoundsGEP(
3053 Builder.CreateInBoundsGEP(ReductionsBufferTy, BufferVal, Idxs);
3055 Value *GlobValPtr = Builder.CreateConstInBoundsGEP2_32(
3057 Builder.CreateStore(GlobValPtr, TargetElementPtrPtr);
3062 Builder.CreateLoad(Builder.getPtrTy(), ReduceListArgAddrCast);
3063 Builder.CreateCall(ReduceFn, {LocalReduceListAddrCast, ReduceList})
3065 Builder.CreateRetVoid();
3066 Builder.restoreIP(OldIP);
3073 OpenMPIRBuilder::InsertPointTy OldIP = Builder.saveIP();
3076 Builder.getVoidTy(),
3077 {Builder.getPtrTy(), Builder.getInt32Ty(), Builder.getPtrTy()},
3088 Builder.SetInsertPoint(EntryBlock);
3097 Value *BufferArgAlloca = Builder.CreateAlloca(Builder.getPtrTy(), nullptr,
3099 Value *IdxArgAlloca = Builder.CreateAlloca(Builder.getInt32Ty(), nullptr,
3101 Value *ReduceListArgAlloca = Builder.CreateAlloca(
3102 Builder.getPtrTy(), nullptr, ReduceListArg->getName() + ".addr");
3103 Value *BufferArgAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
3104 BufferArgAlloca, Builder.getPtrTy(),
3106 Value *IdxArgAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
3107 IdxArgAlloca, Builder.getPtrTy(), IdxArgAlloca->getName() + ".ascast");
3108 Value *ReduceListArgAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
3109 ReduceListArgAlloca, Builder.getPtrTy(),
3111 Builder.CreateStore(BufferArg, BufferArgAddrCast);
3112 Builder.CreateStore(IdxArg, IdxArgAddrCast);
3113 Builder.CreateStore(ReduceListArg, ReduceListArgAddrCast);
3116 Builder.CreateLoad(Builder.getPtrTy(), ReduceListArgAddrCast);
3117 Value *BufferVal = Builder.CreateLoad(Builder.getPtrTy(), BufferArgAddrCast);
3118 Value *Idxs[] = {Builder.CreateLoad(Builder.getInt32Ty(), IdxArgAddrCast)};
3119 Type *IndexTy = Builder.getIndexTy(
3124 ArrayType::get(Builder.getPtrTy(), ReductionInfos.size());
3126 Value *ElemPtrPtr = Builder.CreateInBoundsGEP(
3130 Value *ElemPtr = Builder.CreateLoad(Builder.getPtrTy(), ElemPtrPtr);
3133 Builder.CreateInBoundsGEP(ReductionsBufferTy, BufferVal, Idxs);
3134 Value *GlobValPtr = Builder.CreateConstInBoundsGEP2_32(
3139 Value *TargetElement = Builder.CreateLoad(RI.ElementType, GlobValPtr);
3140 Builder.CreateStore(TargetElement, ElemPtr);
3144 Value *SrcRealPtr = Builder.CreateConstInBoundsGEP2_32(
3146 Value *SrcReal = Builder.CreateLoad(
3148 Value *SrcImgPtr = Builder.CreateConstInBoundsGEP2_32(
3150 Value *SrcImg = Builder.CreateLoad(
3153 Value *DestRealPtr = Builder.CreateConstInBoundsGEP2_32(
3155 Value *DestImgPtr = Builder.CreateConstInBoundsGEP2_32(
3157 Builder.CreateStore(SrcReal, DestRealPtr);
3158 Builder.CreateStore(SrcImg, DestImgPtr);
3163 Builder.getInt64(M.getDataLayout().getTypeStoreSize(RI.ElementType));
3164 Builder.CreateMemCpy(
3173 Builder.CreateRetVoid();
3174 Builder.restoreIP(OldIP);
3181 OpenMPIRBuilder::InsertPointTy OldIP = Builder.saveIP();
3184 Builder.getVoidTy(),
3185 {Builder.getPtrTy(), Builder.getInt32Ty(), Builder.getPtrTy()},
3196 Builder.SetInsertPoint(EntryBlock);
3205 Value *BufferArgAlloca = Builder.CreateAlloca(Builder.getPtrTy(), nullptr,
3207 Value *IdxArgAlloca = Builder.CreateAlloca(Builder.getInt32Ty(), nullptr,
3209 Value *ReduceListArgAlloca = Builder.CreateAlloca(
3210 Builder.getPtrTy(), nullptr, ReduceListArg->getName() + ".addr");
3212 ArrayType::get(Builder.getPtrTy(), ReductionInfos.size());
3217 Builder.CreateAlloca(RedListArrayTy, nullptr, ".omp.reduction.red_list");
3219 Value *BufferArgAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
3220 BufferArgAlloca, Builder.getPtrTy(),
3222 Value *IdxArgAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
3223 IdxArgAlloca, Builder.getPtrTy(), IdxArgAlloca->getName() + ".ascast");
3224 Value *ReduceListArgAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
3225 ReduceListArgAlloca, Builder.getPtrTy(),
3227 Value *ReductionList = Builder.CreatePointerBitCastOrAddrSpaceCast(
3228 LocalReduceList, Builder.getPtrTy(),
3231 Builder.CreateStore(BufferArg, BufferArgAddrCast);
3232 Builder.CreateStore(IdxArg, IdxArgAddrCast);
3233 Builder.CreateStore(ReduceListArg, ReduceListArgAddrCast);
3235 Value *BufferVal = Builder.CreateLoad(Builder.getPtrTy(), BufferArgAddrCast);
3236 Value *Idxs[] = {Builder.CreateLoad(Builder.getInt32Ty(), IdxArgAddrCast)};
3237 Type *IndexTy = Builder.getIndexTy(
3240 Value *TargetElementPtrPtr = Builder.CreateInBoundsGEP(
3245 Builder.CreateInBoundsGEP(ReductionsBufferTy, BufferVal, Idxs);
3246 Value *GlobValPtr = Builder.CreateConstInBoundsGEP2_32(
3248 Builder.CreateStore(GlobValPtr, TargetElementPtrPtr);
3253 Builder.CreateLoad(Builder.getPtrTy(), ReduceListArgAddrCast);
3254 Builder.CreateCall(ReduceFn, {ReduceList, ReductionList})
3256 Builder.CreateRetVoid();
3257 Builder.restoreIP(OldIP);
3270 auto *FuncTy = FunctionType::get(Builder.getVoidTy(),
3271 {Builder.getPtrTy(), Builder.getPtrTy()},
3281 Builder.SetInsertPoint(EntryBB);
3293 Builder.CreateAlloca(Arg0Type, nullptr, Arg0->getName() + ".addr");
3295 Builder.CreateAlloca(Arg1Type, nullptr, Arg1->getName() + ".addr");
3296 Value *LHSAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
3298 Value *RHSAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
3300 Builder.CreateStore(Arg0, LHSAddrCast);
3301 Builder.CreateStore(Arg1, RHSAddrCast);
3302 LHSArrayPtr = Builder.CreateLoad(Arg0Type, LHSAddrCast);
3303 RHSArrayPtr = Builder.CreateLoad(Arg1Type, RHSAddrCast);
3305 Type *RedArrayTy = ArrayType::get(Builder.getPtrTy(), ReductionInfos.size());
3306 Type *IndexTy = Builder.getIndexTy(
3311 Value *RHSI8PtrPtr = Builder.CreateInBoundsGEP(
3314 Value *RHSI8Ptr = Builder.CreateLoad(Builder.getPtrTy(), RHSI8PtrPtr);
3315 Value *RHSPtr = Builder.CreatePointerBitCastOrAddrSpaceCast(
3319 Value *LHSI8PtrPtr = Builder.CreateInBoundsGEP(
3322 Value *LHSI8Ptr = Builder.CreateLoad(Builder.getPtrTy(), LHSI8PtrPtr);
3323 Value *LHSPtr = Builder.CreatePointerBitCastOrAddrSpaceCast(
3330 Value *LHS = Builder.CreateLoad(RI.ElementType, LHSPtr);
3331 Value *RHS = Builder.CreateLoad(RI.ElementType, RHSPtr);
3333 RI.ReductionGen(Builder.saveIP(), LHS, RHS, Reduced);
3334 if (!Builder.GetInsertBlock())
3336 Builder.CreateStore(Reduced, LHSPtr);
3345 Builder.restoreIP(RI.ReductionGenClang(
3346 Builder.saveIP(), Index, &LHSFixupPtr, &RHSFixupPtr, ReductionFunc));
3362 Builder.CreateRetVoid();
3394 Builder.restoreIP(CodeGenIP);
3406 return Builder.saveIP();
3408 Function *CurFunc = Builder.GetInsertBlock()->getParent();
3417 CodeGenIP = Builder.saveIP();
3419 createReductionFunction(Builder.GetInsertBlock()->getParent()->getName(),
3421 Builder.restoreIP(CodeGenIP);
3445 CodeGenIP = Builder.saveIP();
3446 Builder.restoreIP(AllocaIP);
3448 Builder.CreateAlloca(RedArrayTy, nullptr, ".omp.reduction.red_list");
3449 Value *ReductionList = Builder.CreatePointerBitCastOrAddrSpaceCast(
3451 Builder.restoreIP(CodeGenIP);
3452 Type *IndexTy = Builder.getIndexTy(
3456 Value *ElemPtr = Builder.CreateInBoundsGEP(
3460 Builder.CreatePointerBitCastOrAddrSpaceCast(RI.PrivateVariable, PtrTy);
3461 Builder.CreateStore(CastElem, ElemPtr);
3463 CodeGenIP = Builder.saveIP();
3467 Builder.restoreIP(CodeGenIP);
3469 Value *RL = Builder.CreatePointerBitCastOrAddrSpaceCast(ReductionList, PtrTy);
3480 Builder.getInt64(MaxDataSize * ReductionInfos.size());
3483 Builder.CreatePointerBitCastOrAddrSpaceCast(SarFunc, PtrTy);
3485 Builder.CreatePointerBitCastOrAddrSpaceCast(WcFunc, PtrTy);
3489 Res = Builder.CreateCall(Pv2Ptr, Args);
3491 CodeGenIP = Builder.saveIP();
3504 Builder.restoreIP(CodeGenIP);
3506 Value *KernelTeamsReductionPtr = Builder.CreateCall(
3511 Builder.getInt32(ReductionBufNum),
3523 Res = Builder.CreateCall(TeamsReduceFn, Args3);
3529 Value *Cond = Builder.CreateICmpEQ(Res, Builder.getInt32(1));
3530 Builder.CreateCondBr(Cond, ThenBB, ExitBB);
3543 Builder.CreatePointerBitCastOrAddrSpaceCast(RI.PrivateVariable, PtrTy);
3547 Builder.restoreIP(RI.ReductionGenClang(Builder.saveIP(), En.index(),
3568 return Builder.saveIP();
3609 Type *RedArrayTy = ArrayType::get(Builder.getPtrTy(), NumReductions);
3610 Builder.SetInsertPoint(AllocaIP.getBlock()->getTerminator());
3611 Value *RedArray = Builder.CreateAlloca(RedArrayTy, nullptr, "red.array");
3613 Builder.SetInsertPoint(InsertBlock, InsertBlock->end());
3618 Value *RedArrayElemPtr = Builder.CreateConstInBoundsGEP2_64(
3620 Builder.CreateStore(RI.PrivateVariable, RedArrayElemPtr);
3625 Function *Func = Builder.GetInsertBlock()->getParent();
3637 Constant *NumVariables = Builder.getInt32(NumReductions);
3640 Constant *RedArraySize = Builder.getInt64(RedArrayByteSize);
3647 Builder.CreateCall(ReduceFunc,
3660 Builder.CreateSwitch(ReduceCall, ContinuationBlock, /* NumCases */ 2);
3661 Switch->addCase(Builder.getInt32(1), NonAtomicRedBlock);
3662 Switch->addCase(Builder.getInt32(2), AtomicRedBlock);
3667 Builder.SetInsertPoint(NonAtomicRedBlock);
3675 RedValue = Builder.CreateLoad(ValueType, RI.Variable,
3679 Builder.CreateLoad(ValueType, RI.PrivateVariable,
3683 Builder.restoreIP(RI.ReductionGen(Builder.saveIP(), RI.Variable,
3686 Builder.restoreIP(RI.ReductionGen(Builder.saveIP(), RedValue,
3689 if (!Builder.GetInsertBlock())
3693 Builder.CreateStore(Reduced, RI.Variable);
3698 Builder.CreateCall(EndReduceFunc, {Ident, ThreadId, Lock});
3699 Builder.CreateBr(ContinuationBlock);
3704 Builder.SetInsertPoint(AtomicRedBlock);
3707 Builder.restoreIP(RI.AtomicReductionGen(Builder.saveIP(), RI.ElementType,
3709 if (!Builder.GetInsertBlock())
3712 Builder.CreateBr(ContinuationBlock);
3714 Builder.CreateUnreachable();
3722 Builder.SetInsertPoint(ReductionFuncBlock);
3728 Value *LHSI8PtrPtr = Builder.CreateConstInBoundsGEP2_64(
3730 Value *LHSI8Ptr = Builder.CreateLoad(Builder.getPtrTy(), LHSI8PtrPtr);
3731 Value *LHSPtr = Builder.CreateBitCast(LHSI8Ptr, RI.Variable->getType());
3732 Value *LHS = Builder.CreateLoad(RI.ElementType, LHSPtr);
3733 Value *RHSI8PtrPtr = Builder.CreateConstInBoundsGEP2_64(
3735 Value *RHSI8Ptr = Builder.CreateLoad(Builder.getPtrTy(), RHSI8PtrPtr);
3737 Builder.CreateBitCast(RHSI8Ptr, RI.PrivateVariable->getType());
3738 Value *RHS = Builder.CreateLoad(RI.ElementType, RHSPtr);
3740 Builder.restoreIP(RI.ReductionGen(Builder.saveIP(), LHS, RHS, Reduced));
3741 if (!Builder.GetInsertBlock())
3745 Builder.CreateStore(Reduced, LHSPtr);
3747 Builder.CreateRetVoid();
3749 Builder.SetInsertPoint(ContinuationBlock);
3750 return Builder.saveIP();
3769 Instruction *EntryCall = Builder.CreateCall(EntryRTLFn, Args);
3772 Instruction *ExitCall = Builder.CreateCall(ExitRTLFn, Args);
3794 Instruction *EntryCall = Builder.CreateCall(EntryRTLFn, Args);
3797 Instruction *ExitCall = Builder.CreateCall(ExitRTLFn, ArgsEnd);
3827 Builder.SetCurrentDebugLocation(DL);
3829 Builder.SetInsertPoint(Preheader);
3830 Builder.CreateBr(Header);
3832 Builder.SetInsertPoint(Header);
3833 PHINode *IndVarPHI = Builder.CreatePHI(IndVarTy, 2, "omp_" + Name + ".iv");
3835 Builder.CreateBr(Cond);
3837 Builder.SetInsertPoint(Cond);
3839 Builder.CreateICmpULT(IndVarPHI, TripCount, "omp_" + Name + ".cmp");
3840 Builder.CreateCondBr(Cmp, Body, Exit);
3842 Builder.SetInsertPoint(Body);
3843 Builder.CreateBr(Latch);
3845 Builder.SetInsertPoint(Latch);
3846 Value *Next = Builder.CreateAdd(IndVarPHI, ConstantInt::get(IndVarTy, 1),
3848 Builder.CreateBr(Header);
3851 Builder.SetInsertPoint(Exit);
3852 Builder.CreateBr(After);
3885 spliceBB(Builder, After, /*CreateBranch=*/false);
3886 Builder.CreateBr(CL->getPreheader());
3934 Value *IsNeg = Builder.CreateICmpSLT(Step, Zero);
3935 Incr = Builder.CreateSelect(IsNeg, Builder.CreateNeg(Step), Step);
3936 Value *LB = Builder.CreateSelect(IsNeg, Stop, Start);
3937 Value *UB = Builder.CreateSelect(IsNeg, Start, Stop);
3938 Span = Builder.CreateSub(UB, LB, "", false, true);
3939 ZeroCmp = Builder.CreateICmp(
3942 Span = Builder.CreateSub(Stop, Start, "", true);
3943 ZeroCmp = Builder.CreateICmp(
3949 CountIfLooping = Builder.CreateAdd(Builder.CreateUDiv(Span, Incr), One);
3952 Value *CountIfTwo = Builder.CreateAdd(
3953 Builder.CreateUDiv(Builder.CreateSub(Span, One), Incr), One);
3954 Value *OneCmp = Builder.CreateICmp(CmpInst::ICMP_ULE, Span, Incr);
3955 CountIfLooping = Builder.CreateSelect(OneCmp, One, CountIfTwo);
3957 Value *TripCount = Builder.CreateSelect(ZeroCmp, Zero, CountIfLooping,
3961 Builder.restoreIP(CodeGenIP);
3962 Value *Span = Builder.CreateMul(IV, Step);
3963 Value *IndVar = Builder.CreateAdd(Span, Start);
3964 BodyGenCB(Builder.saveIP(), IndVar);
3966 LocationDescription LoopLoc = ComputeIP.isSet() ? Loc.IP : Builder.saveIP();
3995 Builder.restoreIP(CLI->getPreheaderIP());
3996 Builder.SetCurrentDebugLocation(DL);
4010 Builder.SetInsertPoint(AllocaIP.getBlock()->getFirstNonPHIOrDbgOrAlloca());
4013 Value *PLastIter = Builder.CreateAlloca(I32Type, nullptr, "p.lastiter");
4014 Value *PLowerBound = Builder.CreateAlloca(IVTy, nullptr, "p.lowerbound");
4015 Value *PUpperBound = Builder.CreateAlloca(IVTy, nullptr, "p.upperbound");
4016 Value *PStride = Builder.CreateAlloca(IVTy, nullptr, "p.stride");
4022 Builder.SetInsertPoint(CLI->getPreheader()->getTerminator());
4025 Builder.CreateStore(Zero, PLowerBound);
4026 Value *UpperBound = Builder.CreateSub(CLI->getTripCount(), One);
4027 Builder.CreateStore(UpperBound, PUpperBound);
4028 Builder.CreateStore(One, PStride);
4037 Builder.CreateCall(StaticInit,
4040 Value *LowerBound = Builder.CreateLoad(IVTy, PLowerBound);
4041 Value *InclusiveUpperBound = Builder.CreateLoad(IVTy, PUpperBound);
4042 Value *TripCountMinusOne = Builder.CreateSub(InclusiveUpperBound, LowerBound);
4043 Value *TripCount = Builder.CreateAdd(TripCountMinusOne, One);
4051 Builder.SetInsertPoint(CLI->getBody(),
4053 Builder.SetCurrentDebugLocation(DL);
4054 return Builder.CreateAdd(OldIV, LowerBound);
4058 Builder.SetInsertPoint(CLI->getExit(),
4060 Builder.CreateCall(StaticFini, {SrcLoc, ThreadNum});
4064 createBarrier(LocationDescription(Builder.saveIP(), DL),
4099 Builder.restoreIP(AllocaIP);
4100 Builder.SetCurrentDebugLocation(DL);
4101 Value *PLastIter = Builder.CreateAlloca(I32Type, nullptr, "p.lastiter");
4103 Builder.CreateAlloca(InternalIVTy, nullptr, "p.lowerbound");
4105 Builder.CreateAlloca(InternalIVTy, nullptr, "p.upperbound");
4106 Value *PStride = Builder.CreateAlloca(InternalIVTy, nullptr, "p.stride");
4109 Builder.restoreIP(CLI->getPreheaderIP());
4110 Builder.SetCurrentDebugLocation(DL);
4114 Builder.CreateZExtOrTrunc(ChunkSize, InternalIVTy, "chunksize");
4116 Builder.CreateZExt(OrigTripCount, InternalIVTy, "tripcount");
4120 Builder.CreateStore(Zero, PLowerBound);
4121 Value *OrigUpperBound = Builder.CreateSub(CastedTripCount, One);
4122 Builder.CreateStore(OrigUpperBound, PUpperBound);
4123 Builder.CreateStore(One, PStride);
4131 Builder.CreateCall(StaticInit,
4140 Builder.CreateLoad(InternalIVTy, PLowerBound, "omp_firstchunk.lb");
4142 Builder.CreateLoad(InternalIVTy, PUpperBound, "omp_firstchunk.ub");
4143 Value *FirstChunkEnd = Builder.CreateAdd(FirstChunkStop, One);
4145 Builder.CreateSub(FirstChunkEnd, FirstChunkStart, "omp_chunk.range");
4147 Builder.CreateLoad(InternalIVTy, PStride, "omp_dispatch.stride");
4150 BasicBlock *DispatchEnter = splitBB(Builder, true);
4153 {Builder.saveIP(), DL},
4173 Builder.restoreIP(CLI->getPreheaderIP());
4174 Builder.SetCurrentDebugLocation(DL);
4177 Builder.SetInsertPoint(CLI->getPreheader()->getTerminator());
4178 Value *ChunkEnd = Builder.CreateAdd(DispatchCounter, ChunkRange);
4180 Builder.CreateICmpUGE(ChunkEnd, CastedTripCount, "omp_chunk.is_last");
4182 Builder.CreateSub(CastedTripCount, DispatchCounter);
4183 Value *ChunkTripCount = Builder.CreateSelect(
4186 Builder.CreateTrunc(ChunkTripCount, IVTy, "omp_chunk.tripcount.trunc");
4193 Builder.CreateTrunc(DispatchCounter, IVTy, "omp_dispatch.iv.trunc");
4195 Builder.restoreIP(CLI->getBodyIP());
4196 return Builder.CreateAdd(IV, BackcastedDispatchCounter);
4200 Builder.SetInsertPoint(DispatchExit, DispatchExit->getFirstInsertionPt());
4201 Builder.CreateCall(StaticFini, {SrcLoc, ThreadNum});
4205 createBarrier(LocationDescription(Builder.saveIP(), DL), OMPD_for,
4265 IRBuilder<> &Builder = OMPBuilder->Builder;
4270 RealArgs.push_back(Builder.CreateBitCast(&LoopBodyFn, ParallelTaskPtr));
4275 Builder.CreateCall(RTLFn, RealArgs);
4280 Builder.restoreIP({InsertBlock, std::prev(InsertBlock->end())});
4281 Value *NumThreads = Builder.CreateCall(RTLNumThreads, {});
4284 Builder.CreateZExtOrTrunc(NumThreads, TripCountTy, "num.threads.cast"));
4290 Builder.CreateCall(RTLFn, RealArgs);
4299 IRBuilder<> &Builder = OMPIRBuilder->Builder;
4313 Builder.restoreIP({Preheader, Preheader->end()});
4315 Builder.CreateBr(CLI->getExit());
4340 LoopBodyArg = Constant::getNullValue(Builder.getPtrTy());
4375 Builder.restoreIP({CLI->getPreheader(), CLI->getPreheader()->begin()});
4379 AllocaInst *NewLoopCnt = Builder.CreateAlloca(CLI->getIndVarType(), 0, "");
4381 Builder.CreateLoad(CLI->getIndVarType(), NewLoopCnt);
4560 Builder.SetCurrentDebugLocation(DL);
4573 Builder.SetInsertPoint(AllocaIP.getBlock()->getFirstNonPHIOrDbgOrAlloca());
4575 Value *PLastIter = Builder.CreateAlloca(I32Type, nullptr, "p.lastiter");
4576 Value *PLowerBound = Builder.CreateAlloca(IVTy, nullptr, "p.lowerbound");
4577 Value *PUpperBound = Builder.CreateAlloca(IVTy, nullptr, "p.upperbound");
4578 Value *PStride = Builder.CreateAlloca(IVTy, nullptr, "p.stride");
4585 Builder.SetInsertPoint(PreHeader->getTerminator());
4587 Builder.CreateStore(One, PLowerBound);
4589 Builder.CreateStore(UpperBound, PUpperBound);
4590 Builder.CreateStore(One, PStride);
4610 Builder.CreateCall(DynamicInit,
4619 Builder.SetInsertPoint(OuterCond, OuterCond->getFirstInsertionPt());
4621 Builder.CreateCall(DynamicNext, {SrcLoc, ThreadNum, PLastIter,
4624 Value *MoreWork = Builder.CreateCmp(CmpInst::ICMP_NE, Res, Zero32);
4626 Builder.CreateSub(Builder.CreateLoad(IVTy, PLowerBound), One, "lb");
4627 Builder.CreateCondBr(MoreWork, Header, Exit);
4644 Builder.SetInsertPoint(Cond, Cond->getFirstInsertionPt());
4645 UpperBound = Builder.CreateLoad(IVTy, PUpperBound, "ub");
4646 Instruction *Comp = &*Builder.GetInsertPoint();
4657 Builder.SetInsertPoint(&Latch->back());
4659 Builder.CreateCall(DynamicFini, {SrcLoc, ThreadNum});
4664 Builder.SetInsertPoint(&Exit->back());
4665 createBarrier(LocationDescription(Builder.saveIP(), DL),
4729 Builder.SetCurrentDebugLocation(DL);
4731 Builder.restoreIP(ComputeIP);
4733 Builder.restoreIP(Outermost->getPreheaderIP());
4748 CollapsedTripCount = Builder.CreateMul(CollapsedTripCount, OrigTripCount,
4761 Builder.restoreIP(Result->getBodyIP());
4769 Value *NewIndVar = Builder.CreateURem(Leftover, OrigTripCount);
4772 Leftover = Builder.CreateUDiv(Leftover, OrigTripCount);
4882 Builder.SetCurrentDebugLocation(DL);
4883 Builder.restoreIP(OutermostLoop->getPreheaderIP());
4890 Value *FloorTripCount = Builder.CreateUDiv(OrigTripCount, TileSize);
4891 Value *FloorTripRem = Builder.CreateURem(OrigTripCount, TileSize);
4901 Builder.CreateICmpNE(FloorTripRem, ConstantInt::get(IVType, 0));
4903 FloorTripOverflow = Builder.CreateZExt(FloorTripOverflow, IVType);
4905 Builder.CreateAdd(FloorTripCount, FloorTripOverflow,
4956 Builder.SetInsertPoint(Enter->getTerminator());
4963 Builder.CreateICmpEQ(FloorLoop->getIndVar(), FloorCount[i]);
4965 Builder.CreateSelect(FloorIsEpilogue, FloorRems[i], TileSize);
4998 Builder.restoreIP(Result.back()->getBodyIP());
5006 Builder.CreateMul(Size, FloorLoop->getIndVar(), {}, /*HasNUW=*/true);
5008 Builder.CreateAdd(Scale, TileLoop->getIndVar(), {}, /*HasNUW=*/true);
5075 LLVMContext &Ctx = Builder.getContext();
5082 LLVMContext &Ctx = Builder.getContext();
5126 Builder.SetInsertPoint(HeadOldTerm);
5128 Builder.CreateCondBr(IfCond, ThenBlock, /*ifFalse*/ ElseBlock);
5134 Builder.SetInsertPoint(ElseBlock);
5147 Builder.CreateBr(NewBlocks.front());
5171 LLVMContext &Ctx = Builder.getContext();
5189 InsertPointTy IP = Builder.saveIP();
5190 Builder.SetInsertPoint(CanonicalLoop->getPreheader()->getTerminator());
5194 Builder.CreateAlignmentAssumption(F->getDataLayout(),
5197 Builder.restoreIP(IP);
5516 llvm::Value *DidItLD = Builder.CreateLoad(Builder.getInt32Ty(), DidIt);
5521 Builder.CreateCall(Fn, Args);
5523 return Builder.saveIP();
5538 DidIt = Builder.CreateAlloca(llvm::Type::getInt32Ty(Builder.getContext()));
5539 Builder.CreateStore(Builder.getInt32(0), DidIt);
5550 Instruction *EntryCall = Builder.CreateCall(EntryRTLFn, Args);
5553 Instruction *ExitCall = Builder.CreateCall(ExitRTLFn, Args);
5562 Builder.CreateStore(Builder.getInt32(1), DidIt);
5580 createCopyPrivate(LocationDescription(Builder.saveIP(), Loc.DL),
5585 createBarrier(LocationDescription(Builder.saveIP(), Loc.DL),
5588 return Builder.saveIP();
5615 Instruction *EntryCall = Builder.CreateCall(RTFn, EnterArgs);
5619 Instruction *ExitCall = Builder.CreateCall(ExitRTLFn, Args);
5640 Builder.restoreIP(AllocaIP);
5641 AllocaInst *ArgsBase = Builder.CreateAlloca(ArrI64Ty, nullptr, Name);
5643 Builder.restoreIP(Loc.IP);
5647 Value *DependAddrGEPIter = Builder.CreateInBoundsGEP(
5648 ArrI64Ty, ArgsBase, {Builder.getInt64(0), Builder.getInt64(I)});
5649 StoreInst *STInst = Builder.CreateStore(StoreValues[I], DependAddrGEPIter);
5653 Value *DependBaseAddrGEP = Builder.CreateInBoundsGEP(
5654 ArrI64Ty, ArgsBase, {Builder.getInt64(0), Builder.getInt64(0)});
5667 Builder.CreateCall(RTLFn, Args);
5669 return Builder.saveIP();
5690 EntryCall = Builder.CreateCall(EntryRTLFn, Args);
5694 ExitCall = Builder.CreateCall(ExitRTLFn, Args);
5711 BasicBlock *EntryBB = Builder.GetInsertBlock();
5714 SplitPos = new UnreachableInst(Builder.getContext(), EntryBB);
5719 Builder.SetInsertPoint(EntryBB->getTerminator());
5724 /* CodeGenIP */ Builder.saveIP());
5737 // block, and clear the builder's insertion point.
5745 Builder.SetInsertPoint(InsertBB);
5747 return Builder.saveIP();
5754 return Builder.saveIP();
5756 BasicBlock *EntryBB = Builder.GetInsertBlock();
5757 Value *CallBool = Builder.CreateIsNotNull(EntryCall);
5759 auto *UI = new UnreachableInst(Builder.getContext(), ThenBB);
5761 // Emit thenBB and set the Builder's insertion point there for
5769 Builder.CreateCondBr(CallBool, ThenBB, ExitBB);
5771 Builder.SetInsertPoint(UI);
5772 Builder.Insert(EntryBBTI);
5774 Builder.SetInsertPoint(ThenBB->getTerminator());
5784 Builder.restoreIP(FinIP);
5799 // set Builder IP for call creation
5800 Builder.SetInsertPoint(FiniBBTI);
5804 return Builder.saveIP();
5808 Builder.Insert(ExitCall);
5820 IRBuilder<>::InsertPointGuard IPG(Builder);
5851 Builder.SetInsertPoint(OMP_Entry);
5852 Value *MasterPtr = Builder.CreatePtrToInt(MasterAddr, IntPtrTy);
5853 Value *PrivatePtr = Builder.CreatePtrToInt(PrivateAddr, IntPtrTy);
5854 Value *cmp = Builder.CreateICmpNE(MasterPtr, PrivatePtr);
5855 Builder.CreateCondBr(cmp, CopyBegin, CopyEnd);
5857 Builder.SetInsertPoint(CopyBegin);
5859 Builder.SetInsertPoint(Builder.CreateBr(CopyEnd));
5861 return Builder.saveIP();
5867 IRBuilder<>::InsertPointGuard IPG(Builder);
5878 return Builder.CreateCall(Fn, Args, Name);
5884 IRBuilder<>::InsertPointGuard IPG(Builder);
5893 return Builder.CreateCall(Fn, Args, Name);
5900 IRBuilder<>::InsertPointGuard IPG(Builder);
5922 return Builder.CreateCall(Fn, Args);
5928 IRBuilder<>::InsertPointGuard IPG(Builder);
5949 return Builder.CreateCall(Fn, Args);
5957 IRBuilder<>::InsertPointGuard IPG(Builder);
5977 return Builder.CreateCall(Fn, Args);
5983 IRBuilder<>::InsertPointGuard IPG(Builder);
5997 return Builder.CreateCall(Fn, Args);
6016 Function *Kernel = Builder.GetInsertBlock()->getParent();
6098 Builder.CreateCall(Fn, {KernelEnvironment, KernelLaunchEnvironment});
6100 Value *ExecUserCode = Builder.CreateICmpEQ(
6110 auto *UI = Builder.CreateUnreachable();
6116 Builder.SetInsertPoint(WorkerExitBB);
6117 Builder.CreateRetVoid();
6120 Builder.SetInsertPoint(CheckBBTI);
6121 Builder.CreateCondBr(ExecUserCode, UI->getParent(), WorkerExitBB);
6140 Builder.CreateCall(Fn, {});
6145 Function *Kernel = Builder.GetInsertBlock()->getParent();
6283 M, Builder.getInt8Ty(), /*isConstant=*/true, GlobalValue::WeakAnyLinkage,
6284 Constant::getNullValue(Builder.getInt8Ty()), EntryFnIDName);
6295 M, Builder.getInt8Ty(), /*isConstant=*/true, GlobalValue::InternalLinkage,
6296 Constant::getNullValue(Builder.getInt8Ty()), EntryFnName);
6354 Builder.restoreIP(BodyGenCB(Builder.saveIP(), BodyGenTy::NoPriv));
6355 return Builder.saveIP();
6358 Builder.restoreIP(CodeGenIP);
6365 MapInfo = &GenMapInfoCB(Builder.saveIP());
6366 emitOffloadingArrays(AllocaIP, Builder.saveIP(), *MapInfo, Info,
6371 emitOffloadingArraysArgument(Builder, RTArgs, Info,
6375 Value *PointerNum = Builder.getInt32(Info.NumberOfPtrs);
6392 Builder.CreateCall(getOrCreateRuntimeFunctionPtr(*MapperFunc),
6398 Builder.CreateCall(BeginMapperFunc, OffloadingArgs);
6403 Builder.CreateLoad(Builder.getPtrTy(), DeviceMap.second.first);
6404 Builder.CreateStore(LI, DeviceMap.second.second);
6411 Builder.restoreIP(BodyGenCB(Builder.saveIP(), BodyGenTy::Priv));
6419 Builder.restoreIP(BodyGenCB(Builder.saveIP(), BodyGenTy::DupNoPriv));
6425 emitOffloadingArraysArgument(Builder, RTArgs, Info, !MapInfo->Names.empty(),
6429 Value *PointerNum = Builder.getInt32(Info.NumberOfPtrs);
6446 Builder.CreateCall(EndMapperFunc, OffloadingArgs);
6457 BeginThenGen(AllocaIP, Builder.saveIP());
6462 Builder.restoreIP(BodyGenCB(Builder.saveIP(), BodyGenTy::NoPriv));
6467 EndThenGen(AllocaIP, Builder.saveIP());
6473 BeginThenGen(AllocaIP, Builder.saveIP());
6477 return Builder.saveIP();
6545 OpenMPIRBuilder &OMPBuilder, IRBuilderBase &Builder, StringRef FuncName,
6553 auto *Int8PtrTy = PointerType::getUnqual(Builder.getContext());
6561 : Type::getInt64Ty(Builder.getContext()));
6567 auto FuncType = FunctionType::get(Builder.getVoidTy(), ParameterTypes,
6570 Builder.GetInsertBlock()->getModule());
6573 auto OldInsertPoint = Builder.saveIP();
6576 BasicBlock *EntryBB = BasicBlock::Create(Builder.getContext(), "entry", Func);
6577 Builder.SetInsertPoint(EntryBB);
6581 Builder.restoreIP(OMPBuilder.createTargetInit(Builder, /*IsSPMD*/ false));
6583 BasicBlock *UserCodeEntryBB = Builder.GetInsertBlock();
6592 Builder.restoreIP(CBFunc(Builder.saveIP(), Builder.saveIP()));
6594 OMPBuilder.createTargetDeinit(Builder);
6597 Builder.CreateRetVoid();
6600 Builder.SetInsertPoint(EntryBB->getFirstNonPHI());
6601 auto AllocaIP = Builder.saveIP();
6603 Builder.SetInsertPoint(UserCodeEntryBB->getFirstNonPHIOrDbg());
6647 Builder.restoreIP(
6648 ArgAccessorFuncCB(Arg, Input, InputCopy, AllocaIP, Builder.saveIP()));
6681 Builder.restoreIP(OldInsertPoint);
6692 IRBuilderBase &Builder,
6724 FunctionType::get(Builder.getVoidTy(), {ThreadIDTy, TaskPtrTy},
6728 Builder.GetInsertBlock()->getModule());
6733 BasicBlock::Create(Builder.getContext(), "entry", ProxyFn);
6734 Builder.SetInsertPoint(EntryBB);
6752 Builder.CreateAlloca(ArgStructType, nullptr, "structArg");
6756 Builder.getInt64(M.getDataLayout().getTypeStoreSize(ArgStructType));
6758 Value *Shareds = Builder.CreateStructGEP(TaskTy, TaskT, 0);
6760 Builder.CreateLoad(PointerType::getUnqual(Ctx), Shareds);
6762 Builder.CreateMemCpy(
6766 Builder.CreateCall(KernelLaunchFunction, {ThreadId, NewArgStructAlloca});
6768 Builder.CreateRetVoid();
6772 OpenMPIRBuilder &OMPBuilder, IRBuilderBase &Builder,
6779 [&OMPBuilder, &Builder, &Inputs, &CBFunc,
6781 return createOutlinedFunction(OMPBuilder, Builder, EntryFnName, Inputs,
6883 splitBB(Builder, /*CreateBranch=*/true, "target.task.body");
6885 splitBB(Builder, /*CreateBranch=*/true, "target.task.alloca");
6898 Builder, AllocaIP, ToBeDeleted, TargetTaskAllocaIP, "global.tid", false));
6900 Builder.restoreIP(TargetTaskBodyIP);
6908 Builder.restoreIP(emitKernelLaunch(Builder, OutlinedFn, OutlinedFnID,
6912 OI.ExitBB = Builder.saveIP().getBlock();
6921 Function *ProxyFn = emitTargetTaskProxyFunction(*this, Builder, StaleCI);
6926 Builder.SetInsertPoint(StaleCI);
6931 getOrCreateSrcLocStr(LocationDescription(Builder), SrcLocStrSize);
6947 Builder.getInt64(M.getDataLayout().getTypeStoreSize(Task));
6952 Value *SharedsSize = Builder.getInt64(0);
6963 Builder.getInt64(M.getDataLayout().getTypeStoreSize(ArgStructType));
6972 Value *Flags = Builder.getInt32(0);
6977 CallInst *TaskData = Builder.CreateCall(
6985 Value *TaskShareds = Builder.CreateLoad(VoidPtr, TaskData);
6986 Builder.CreateMemCpy(TaskShareds, Alignment, Shareds, Alignment,
7004 Builder.CreateCall(
7007 /*ndeps=*/Builder.getInt32(Dependencies.size()),
7009 /*ndeps_noalias=*/ConstantInt::get(Builder.getInt32Ty(), 0),
7018 Builder.CreateCall(TaskBeginFn, {Ident, ThreadID, TaskData});
7021 CI = Builder.CreateCall(ProxyFn, {ThreadID, TaskData});
7023 CI = Builder.CreateCall(ProxyFn, {ThreadID});
7025 Builder.CreateCall(TaskCompleteFn, {Ident, ThreadID, TaskData});
7032 Builder.CreateCall(
7034 {Ident, ThreadID, TaskData, Builder.getInt32(Dependencies.size()),
7035 DepArray, ConstantInt::get(Builder.getInt32Ty(), 0),
7040 Builder.CreateCall(TaskFn, {Ident, ThreadID, TaskData});
7050 << *(Builder.GetInsertBlock()) << "\n");
7052 << *(Builder.GetInsertBlock()->getParent()->getParent())
7054 return Builder.saveIP();
7057 OpenMPIRBuilder &OMPBuilder, IRBuilderBase &Builder,
7068 OpenMPIRBuilder::MapInfosTy &MapInfo = GenMapInfoCB(Builder.saveIP());
7069 OMPBuilder.emitOffloadingArrays(AllocaIP, Builder.saveIP(), MapInfo, Info,
7073 OMPBuilder.emitOffloadingArraysArgument(Builder, RTArgs, Info,
7079 Builder.restoreIP(IP);
7080 Builder.CreateCall(OutlinedFn, Args);
7081 return Builder.saveIP();
7086 Value *DeviceID = Builder.getInt64(OMP_DEVICEID_UNDEF);
7087 Value *NumTeamsVal = Builder.getInt32(NumTeams);
7088 Value *NumThreadsVal = Builder.getInt32(NumThreads);
7094 Value *NumIterations = Builder.getInt64(0);
7096 Value *DynCGGroupMem = Builder.getInt32(0);
7109 Builder.restoreIP(OMPBuilder.emitTargetTask(
7113 Builder.restoreIP(OMPBuilder.emitKernelLaunch(
7114 Builder, OutlinedFn, OutlinedFnID, EmitTargetCallFallbackCB, KArgs,
7130 Builder.restoreIP(CodeGenIP);
7137 emitTargetOutlinedFunction(*this, Builder, EntryInfo, OutlinedFn,
7144 emitTargetCall(*this, Builder, AllocaIP, OutlinedFn, OutlinedFnID, NumTeams,
7146 return Builder.saveIP();
7204 LLVMContext &Ctx = Builder.getContext();
7208 Builder.CreateGEP(BasePtr->getType(), Null, Builder.getInt32(1));
7209 Value *SizePtrToInt = Builder.CreatePtrToInt(SizeGep, Type::getInt64Ty(Ctx));
7235 Builder.restoreIP(AllocaIP);
7236 AllocaInst *ArgsBase = Builder.CreateAlloca(
7238 AllocaInst *Args = Builder.CreateAlloca(ArrI8PtrTy, /* ArraySize = */ nullptr,
7240 AllocaInst *ArgSizes = Builder.CreateAlloca(
7242 Builder.restoreIP(Loc.IP);
7259 Builder.CreateInBoundsGEP(ArrI8PtrTy, MapperAllocas.ArgsBase,
7260 {Builder.getInt32(0), Builder.getInt32(0)});
7262 Builder.CreateInBoundsGEP(ArrI8PtrTy, MapperAllocas.Args,
7263 {Builder.getInt32(0), Builder.getInt32(0)});
7265 Builder.CreateInBoundsGEP(ArrI64Ty, MapperAllocas.ArgSizes,
7266 {Builder.getInt32(0), Builder.getInt32(0)});
7269 Builder.CreateCall(MapperFunc,
7270 {SrcLocInfo, Builder.getInt64(DeviceID),
7271 Builder.getInt32(NumOperands), ArgsBaseGEP, ArgsGEP,
7275 void OpenMPIRBuilder::emitOffloadingArraysArgument(IRBuilderBase &Builder,
7298 RTArgs.BasePointersArray = Builder.CreateConstInBoundsGEP2_32(
7302 RTArgs.PointersArray = Builder.CreateConstInBoundsGEP2_32(
7306 RTArgs.SizesArray = Builder.CreateConstInBoundsGEP2_32(
7309 RTArgs.MapTypesArray = Builder.CreateConstInBoundsGEP2_32(
7321 RTArgs.MapNamesArray = Builder.CreateConstInBoundsGEP2_32(
7331 Builder.CreatePointerCast(Info.RTArgs.MappersArray, VoidPtrPtrTy);
7349 Type *Int64Ty = Builder.getInt64Ty();
7363 Builder.restoreIP(AllocaIP);
7366 Builder.CreateAlloca(ArrayTy, /* ArraySize = */ nullptr, "dims");
7367 Builder.restoreIP(CodeGenIP);
7370 Value *DimsLVal = Builder.CreateInBoundsGEP(
7372 {Builder.getInt64(0), Builder.getInt64(II)});
7374 Value *OffsetLVal = Builder.CreateStructGEP(DimTy, DimsLVal, OffsetFD);
7375 Builder.CreateAlignedStore(
7379 Value *CountLVal = Builder.CreateStructGEP(DimTy, DimsLVal, CountFD);
7380 Builder.CreateAlignedStore(
7384 Value *StrideLVal = Builder.CreateStructGEP(DimTy, DimsLVal, StrideFD);
7385 Builder.CreateAlignedStore(
7390 Builder.restoreIP(CodeGenIP);
7391 Value *DAddr = Builder.CreatePointerBitCastOrAddrSpaceCast(
7392 DimsAddr, Builder.getPtrTy());
7393 Value *P = Builder.CreateConstInBoundsGEP2_32(
7394 ArrayType::get(Builder.getPtrTy(), Info.NumberOfPtrs),
7396 Builder.CreateAlignedStore(
7397 DAddr, P, M.getDataLayout().getPrefTypeAlign(Builder.getPtrTy()));
7415 Builder.restoreIP(AllocaIP);
7419 ArrayType::get(Builder.getPtrTy(), Info.NumberOfPtrs);
7421 Info.RTArgs.BasePointersArray = Builder.CreateAlloca(
7424 Info.RTArgs.PointersArray = Builder.CreateAlloca(
7426 AllocaInst *MappersArray = Builder.CreateAlloca(
7433 Type *Int64Ty = Builder.getInt64Ty();
7456 Info.RTArgs.SizesArray = Builder.CreateAlloca(
7458 Builder.restoreIP(CodeGenIP);
7474 AllocaInst *Buffer = Builder.CreateAlloca(
7477 Builder.restoreIP(CodeGenIP);
7478 Builder.CreateMemCpy(
7481 Builder.getIntN(
7487 Builder.restoreIP(CodeGenIP);
7509 Constant::getNullValue(PointerType::getUnqual(Builder.getContext()));
7530 PointerType *PtrTy = Builder.getPtrTy();
7533 Value *BP = Builder.CreateConstInBoundsGEP2_32(
7536 Builder.CreateAlignedStore(BPVal, BP,
7541 CodeGenIP = Builder.saveIP();
7542 Builder.restoreIP(AllocaIP);
7543 Info.DevicePtrInfoMap[BPVal] = {BP, Builder.CreateAlloca(PtrTy)};
7544 Builder.restoreIP(CodeGenIP);
7555 Value *P = Builder.CreateConstInBoundsGEP2_32(
7559 Builder.CreateAlignedStore(PVal, P,
7563 Value *S = Builder.CreateConstInBoundsGEP2_32(
7567 Builder.CreateAlignedStore(Builder.CreateIntCast(CombinedInfo.Sizes[I],
7577 MFunc = Builder.CreatePointerCast(CustomMFunc, PtrTy);
7578 Value *MAddr = Builder.CreateInBoundsGEP(
7580 {Builder.getIntN(IndexSize, 0), Builder.getIntN(IndexSize, I)});
7581 Builder.CreateAlignedStore(
7592 BasicBlock *CurBB = Builder.GetInsertBlock();
7599 Builder.CreateBr(Target);
7602 Builder.ClearInsertionPoint();
7607 BasicBlock *CurBB = Builder.GetInsertBlock();
7623 Builder.SetInsertPoint(BB);
7634 ThenGen(AllocaIP, Builder.saveIP());
7636 ElseGen(AllocaIP, Builder.saveIP());
7640 Function *CurFn = Builder.GetInsertBlock()->getParent();
7647 Builder.CreateCondBr(Cond, ThenBlock, ElseBlock);
7650 ThenGen(AllocaIP, Builder.saveIP());
7655 ElseGen(AllocaIP, Builder.saveIP());
7741 Builder.CreateLoad(XElemTy, X.Var, X.IsVolatile, "omp.atomic.read");
7749 Builder.CreateLoad(IntCastTy, X.Var, X.IsVolatile, "omp.atomic.load");
7752 XRead = Builder.CreateBitCast(XLoad, XElemTy, "atomic.flt.cast");
7754 XRead = Builder.CreateIntToPtr(XLoad, XElemTy, "atomic.ptr.cast");
7758 Builder.CreateStore(XRead, V.Var, V.IsVolatile);
7759 return Builder.saveIP();
7777 StoreInst *XSt = Builder.CreateStore(Expr, X.Var, X.IsVolatile);
7784 Builder.CreateBitCast(Expr, IntCastTy, "atomic.src.int.cast");
7785 StoreInst *XSt = Builder.CreateStore(ExprCast, X.Var, X.IsVolatile);
7790 return Builder.saveIP();
7817 return Builder.saveIP();
7825 return Builder.CreateAdd(Src1, Src2);
7827 return Builder.CreateSub(Src1, Src2);
7829 return Builder.CreateAnd(Src1, Src2);
7831 return Builder.CreateNeg(Builder.CreateAnd(Src1, Src2));
7833 return Builder.CreateOr(Src1, Src2);
7835 return Builder.CreateXor(Src1, Src2);
7879 Res.first = Builder.CreateAtomicRMW(RMWOp, X, Expr, llvm::MaybeAlign(), AO);
7891 Builder.CreateLoad(IntCastTy, X, X->getName() + ".atomic.load");
7898 BasicBlock *CurBB = Builder.GetInsertBlock();
7900 CurBBTI = CurBBTI ? CurBBTI : Builder.CreateUnreachable();
7906 Builder.restoreIP(AllocaIP);
7907 AllocaInst *NewAtomicAddr = Builder.CreateAlloca(XElemTy);
7909 Builder.SetInsertPoint(ContBB);
7910 llvm::PHINode *PHI = Builder.CreatePHI(OldVal->getType(), 2);
7916 OldExprVal = Builder.CreateBitCast(PHI, XElemTy,
7919 OldExprVal = Builder.CreateIntToPtr(PHI, XElemTy,
7924 Value *Upd = UpdateOp(OldExprVal, Builder);
7925 Builder.CreateStore(Upd, NewAtomicAddr);
7926 LoadInst *DesiredVal = Builder.CreateLoad(IntCastTy, NewAtomicAddr);
7929 AtomicCmpXchgInst *Result = Builder.CreateAtomicCmpXchg(
7932 Value *PreviousVal = Builder.CreateExtractValue(Result, /*Idxs=*/0);
7933 Value *SuccessFailureVal = Builder.CreateExtractValue(Result, /*Idxs=*/1);
7934 PHI->addIncoming(PreviousVal, Builder.GetInsertBlock());
7935 Builder.CreateCondBr(SuccessFailureVal, ExitBB, ContBB);
7944 Builder.SetInsertPoint(ExitBB);
7946 Builder.SetInsertPoint(ExitTI);
7981 Builder.CreateStore(CapturedVal, V.Var, V.IsVolatile);
7984 return Builder.saveIP();
8022 Value *EBCast = Builder.CreateBitCast(E, IntCastTy);
8023 Value *DBCast = Builder.CreateBitCast(D, IntCastTy);
8024 Result = Builder.CreateAtomicCmpXchg(X.Var, EBCast, DBCast, MaybeAlign(),
8028 Builder.CreateAtomicCmpXchg(X.Var, E, D, MaybeAlign(), AO, Failure);
8032 Value *OldValue = Builder.CreateExtractValue(Result, /*Idxs=*/0);
8034 OldValue = Builder.CreateBitCast(OldValue, X.ElemTy);
8038 Builder.CreateStore(OldValue, V.Var, V.IsVolatile);
8040 Value *SuccessOrFail = Builder.CreateExtractValue(Result, /*Idxs=*/1);
8051 BasicBlock *CurBB = Builder.GetInsertBlock();
8053 CurBBTI = CurBBTI ? CurBBTI : Builder.CreateUnreachable();
8061 Builder.CreateCondBr(SuccessOrFail, ExitBB, ContBB);
8063 Builder.SetInsertPoint(ContBB);
8064 Builder.CreateStore(OldValue, V.Var);
8065 Builder.CreateBr(ExitBB);
8070 Builder.SetInsertPoint(ExitBB);
8072 Builder.SetInsertPoint(ExitTI);
8076 Builder.CreateSelect(SuccessOrFail, E, OldValue);
8077 Builder.CreateStore(CapturedValue, V.Var, V.IsVolatile);
8087 Value *SuccessFailureVal = Builder.CreateExtractValue(Result, /*Idxs=*/1);
8089 ? Builder.CreateSExt(SuccessFailureVal, R.ElemTy)
8090 : Builder.CreateZExt(SuccessFailureVal, R.ElemTy);
8091 Builder.CreateStore(ResultCast, R.Var, R.IsVolatile);
8134 Builder.CreateAtomicRMW(NewOp, X.Var, E, MaybeAlign(), AO);
8163 Value *NonAtomicCmp = Builder.CreateCmp(Pred, OldValue, E);
8164 CapturedValue = Builder.CreateSelect(NonAtomicCmp, E, OldValue);
8166 Builder.CreateStore(CapturedValue, V.Var, V.IsVolatile);
8172 return Builder.saveIP();
8186 Function *CurrentFunction = Builder.GetInsertBlock()->getParent();
8190 if (&OuterAllocaBB == Builder.GetInsertBlock()) {
8191 BasicBlock *BodyBB = splitBB(Builder, /*CreateBranch=*/true, "teams.entry");
8192 Builder.SetInsertPoint(BodyBB, BodyBB->begin());
8212 BasicBlock *ExitBB = splitBB(Builder, /*CreateBranch=*/true, "teams.exit");
8213 BasicBlock *BodyBB = splitBB(Builder, /*CreateBranch=*/true, "teams.body");
8215 splitBB(Builder, /*CreateBranch=*/true, "teams.alloca");
8226 NumTeamsUpper = Builder.getInt32(0);
8237 IfExpr = Builder.CreateICmpNE(IfExpr,
8239 NumTeamsUpper = Builder.CreateSelect(
8240 IfExpr, NumTeamsUpper, Builder.getInt32(1), "numTeamsUpper");
8243 NumTeamsLower = Builder.CreateSelect(
8244 IfExpr, NumTeamsLower, Builder.getInt32(1), "numTeamsLower");
8248 ThreadLimit = Builder.getInt32(0);
8251 Builder.CreateCall(
8269 Builder, OuterAllocaIP, ToBeDeleted, AllocaIP, "gid", true));
8271 Builder, OuterAllocaIP, ToBeDeleted, AllocaIP, "tid", true));
8296 Builder.SetInsertPoint(StaleCI);
8298 Ident, Builder.getInt32(StaleCI->arg_size() - 2), &OutlinedFn};
8301 Builder.CreateCall(getOrCreateRuntimeFunctionPtr(
8315 Builder.SetInsertPoint(ExitBB, ExitBB->begin());
8317 return Builder.saveIP();
8423 return ConstantAsMetadata::get(ConstantInt::get(Builder.getInt32Ty(), V));