Lines Matching full:state
317 void VPPartialReductionRecipe::execute(VPTransformState &State) {
318 State.setDebugLocFrom(getDebugLoc());
319 auto &Builder = State.Builder;
324 Value *BinOpVal = State.get(getOperand(0));
325 Value *PhiVal = State.get(getOperand(1));
334 State.set(this, V);
424 Value *VPInstruction::generatePerLane(VPTransformState &State,
426 IRBuilderBase &Builder = State.Builder;
430 return Builder.CreatePtrAdd(State.get(getOperand(0), Lane),
431 State.get(getOperand(1), Lane), Name);
434 Value *VPInstruction::generate(VPTransformState &State) {
435 IRBuilderBase &Builder = State.Builder;
439 Value *A = State.get(getOperand(0), OnlyFirstLaneUsed);
440 Value *B = State.get(getOperand(1), OnlyFirstLaneUsed);
450 Value *A = State.get(getOperand(0));
455 Value *A = State.get(getOperand(0), OnlyFirstLaneUsed);
456 Value *B = State.get(getOperand(1), OnlyFirstLaneUsed);
461 Value *Cond = State.get(getOperand(0), OnlyFirstLaneUsed);
462 Value *Op1 = State.get(getOperand(1), OnlyFirstLaneUsed);
463 Value *Op2 = State.get(getOperand(2), OnlyFirstLaneUsed);
468 Value *VIVElem0 = State.get(getOperand(0), VPLane(0));
470 Value *ScalarTC = State.get(getOperand(1), VPLane(0));
474 if (State.VF.isScalar())
479 auto *PredTy = VectorType::get(Int1Ty, State.VF);
497 auto *V1 = State.get(getOperand(0));
500 Value *V2 = State.get(getOperand(1));
505 Value *ScalarTC = State.get(getOperand(0), VPLane(0));
506 Value *Step = createStepForVF(Builder, ScalarTC->getType(), State.VF, UF);
515 Value *AVL = State.get(getOperand(0), /*IsScalar*/ true);
520 assert(State.VF.isScalable() && "Expected scalable vector factor.");
521 Value *VFArg = State.Builder.getInt32(State.VF.getKnownMinValue());
523 Value *EVL = State.Builder.CreateIntrinsic(
524 State.Builder.getInt32Ty(), Intrinsic::experimental_get_vector_length,
525 {AVL, VFArg, State.Builder.getTrue()});
530 auto *IV = State.get(getOperand(0), VPLane(0));
534 Value *Step = createStepForVF(Builder, IV->getType(), State.VF, Part);
539 Value *Cond = State.get(getOperand(0), VPLane(0));
553 CondBr->setSuccessor(1, State.CFG.VPBB2IRBB[Header]);
558 Value *IV = State.get(getOperand(0), /*IsScalar*/ true);
559 Value *TC = State.get(getOperand(1), /*IsScalar*/ true);
573 State.CFG.VPBB2IRBB[Header]);
594 RdxParts[Part] = State.get(getOperand(1 + Part), PhiR->isInLoop());
600 if (State.VF.isVector() && PhiTy != RdxDesc.getRecurrenceType()) {
601 Type *RdxVecTy = VectorType::get(RdxDesc.getRecurrenceType(), State.VF);
632 if ((State.VF.isVector() ||
653 if (State.VF.isVector()) {
654 assert(Offset <= State.VF.getKnownMinValue() &&
657 Res = State.get(getOperand(0), VPLane::getLaneFromEnd(State.VF, Offset));
660 Res = State.get(getOperand(0));
667 Value *A = State.get(getOperand(0));
668 Value *B = State.get(getOperand(1));
674 Value *Ptr = State.get(getOperand(0), VPLane(0));
675 Value *Addend = State.get(getOperand(1), VPLane(0));
680 State.get(getOperand(0), /* IsScalar */ true);
682 State.get(getOperand(1), /* IsScalar */ true);
684 Builder.CreatePHI(State.TypeAnalysis.inferScalarType(this), 2, Name);
686 State.CFG
697 Value *A = State.get(getOperand(0));
727 void VPInstruction::execute(VPTransformState &State) {
728 assert(!State.Lane && "VPInstruction executing an Lane");
729 IRBuilderBase::FastMathFlagGuard FMFGuard(State.Builder);
734 State.Builder.setFastMathFlags(getFastMathFlags());
735 State.setDebugLocFrom(getDebugLoc());
741 for (unsigned Lane = 0, NumLanes = State.VF.getKnownMinValue();
743 Value *GeneratedValue = generatePerLane(State, VPLane(Lane));
745 State.set(this, GeneratedValue, VPLane(Lane));
750 Value *GeneratedValue = generate(State);
756 State.VF.isScalar()) &&
758 State.set(this, GeneratedValue,
905 void VPIRInstruction::execute(VPTransformState &State) {
912 : VPLane::getLastLaneForVF(State.VF);
915 BasicBlock *PredBB = State.CFG.VPBB2IRBB[PredVPBB];
918 State.Builder.SetInsertPoint(PredBB, PredBB->getFirstNonPHIIt());
919 Value *V = State.get(ExitValue, VPLane(Lane));
931 State.Builder.SetInsertPoint(I.getParent(), std::next(I.getIterator()));
975 void VPWidenCallRecipe::execute(VPTransformState &State) {
976 assert(State.VF.isVector() && "not widening");
977 State.setDebugLocFrom(getDebugLoc());
988 Arg = State.get(I.value(), VPLane(0));
990 Arg = State.get(I.value(), onlyFirstLaneUsed(I.value()));
1001 CallInst *V = State.Builder.CreateCall(Variant, Args, OpBundles);
1005 State.set(this, V);
1006 State.addMetadata(V, CI);
1044 void VPWidenIntrinsicRecipe::execute(VPTransformState &State) {
1045 assert(State.VF.isVector() && "not widening");
1046 State.setDebugLocFrom(getDebugLoc());
1050 if (isVectorIntrinsicWithOverloadTypeAtArg(VectorIntrinsicID, -1, State.TTI))
1051 TysForDecl.push_back(VectorType::get(getResultType(), State.VF));
1058 State.TTI))
1059 Arg = State.get(I.value(), VPLane(0));
1061 Arg = State.get(I.value(), onlyFirstLaneUsed(I.value()));
1063 State.TTI))
1069 Module *M = State.Builder.GetInsertBlock()->getModule();
1080 CallInst *V = State.Builder.CreateCall(VectorF, Args, OpBundles);
1085 State.set(this, V);
1086 State.addMetadata(V, CI);
1167 void VPHistogramRecipe::execute(VPTransformState &State) {
1168 State.setDebugLocFrom(getDebugLoc());
1169 IRBuilderBase &Builder = State.Builder;
1171 Value *Address = State.get(getOperand(0));
1172 Value *IncAmt = State.get(getOperand(1), /*IsScalar=*/true);
1180 Mask = State.get(VPMask);
1192 State.Builder.CreateIntrinsic(Intrinsic::experimental_vector_histogram_add,
1268 void VPWidenSelectRecipe::execute(VPTransformState &State) {
1269 State.setDebugLocFrom(getDebugLoc());
1276 isInvariantCond() ? State.get(getCond(), VPLane(0)) : nullptr;
1278 Value *Cond = InvarCond ? InvarCond : State.get(getCond());
1279 Value *Op0 = State.get(getOperand(1));
1280 Value *Op1 = State.get(getOperand(2));
1281 Value *Sel = State.Builder.CreateSelect(Cond, Op0, Op1);
1282 State.set(this, Sel);
1285 State.addMetadata(Sel, dyn_cast_or_null<Instruction>(getUnderlyingValue()));
1381 void VPWidenRecipe::execute(VPTransformState &State) {
1382 State.setDebugLocFrom(getDebugLoc());
1383 auto &Builder = State.Builder;
1413 Ops.push_back(State.get(VPOp));
1421 State.set(this, V);
1422 State.addMetadata(V, dyn_cast_or_null<Instruction>(getUnderlyingValue()));
1426 Value *Op = State.get(getOperand(0));
1429 State.set(this, Freeze);
1436 Value *A = State.get(getOperand(0));
1437 Value *B = State.get(getOperand(1));
1447 State.set(this, C);
1448 State.addMetadata(C, dyn_cast_or_null<Instruction>(getUnderlyingValue()));
1461 assert(VectorType::get(State.TypeAnalysis.inferScalarType(this), State.VF) ==
1462 State.get(this)->getType() &&
1540 void VPWidenEVLRecipe::execute(VPTransformState &State) {
1546 State.setDebugLocFrom(getDebugLoc());
1548 assert(State.get(getOperand(0))->getType()->isVectorTy() &&
1552 Value *EVLArg = State.get(EVL, /*NeedsScalar=*/true);
1553 IRBuilderBase &BuilderIR = State.Builder;
1555 Value *Mask = BuilderIR.CreateVectorSplat(State.VF, BuilderIR.getTrue());
1560 Ops.push_back(State.get(VPOp));
1571 State.set(this, VPInst);
1572 State.addMetadata(VPInst,
1596 void VPWidenCastRecipe::execute(VPTransformState &State) {
1597 State.setDebugLocFrom(getDebugLoc());
1598 auto &Builder = State.Builder;
1600 assert(State.VF.isVector() && "Not vectorizing?");
1601 Type *DestTy = VectorType::get(getResultType(), State.VF);
1603 Value *A = State.get(Op);
1605 State.set(this, Cast);
1606 State.addMetadata(Cast, cast_or_null<Instruction>(getUnderlyingValue()));
1737 void VPWidenIntOrFpInductionRecipe::execute(VPTransformState &State) {
1738 assert(!State.Lane && "Int or FP induction being replicated.");
1743 IRBuilderBase &Builder = State.Builder;
1746 assert(State.VF.isVector() && "must have vector VF");
1758 Value *Step = State.get(getStepValue(), VPLane(0));
1765 BasicBlock *VectorPH = State.CFG.getPreheaderBBFor(this);
1775 Value *SplatStart = Builder.CreateVectorSplat(State.VF, Start);
1777 State.VF, State.Builder);
1795 SplatVF = State.get(SplatVFOperand);
1800 Value *RuntimeVF = State.get(getVFValue(), VPLane(0));
1808 SplatVF = Builder.CreateVectorSplat(State.VF, Mul);
1816 VecInd->insertBefore(State.CFG.PrevBB->getFirstInsertionPt());
1818 State.set(this, VecInd);
1823 State.addMetadata(LastInduction, EntryVal);
1875 void VPScalarIVStepsRecipe::execute(VPTransformState &State) {
1877 IRBuilder<>::FastMathFlagGuard FMFG(State.Builder);
1879 State.Builder.setFastMathFlags(getFastMathFlags());
1884 Value *BaseIV = State.get(getOperand(0), VPLane(0));
1885 Value *Step = State.get(getStepValue(), VPLane(0));
1886 IRBuilderBase &Builder = State.Builder;
1907 // Compute the scalar steps and save the results in State.
1912 if (!FirstLaneOnly && State.VF.isScalable()) {
1913 VecIVTy = VectorType::get(BaseIVTy, State.VF);
1915 Builder.CreateStepVector(VectorType::get(IntStepTy, State.VF));
1916 SplatStep = Builder.CreateVectorSplat(State.VF, Step);
1917 SplatIV = Builder.CreateVectorSplat(State.VF, BaseIV);
1921 unsigned EndLane = FirstLaneOnly ? 1 : State.VF.getKnownMinValue();
1922 if (State.Lane) {
1923 StartLane = State.Lane->getKnownLane();
1927 createStepForVF(Builder, IntStepTy, State.VF, getUnrollPart(*this));
1929 if (!FirstLaneOnly && State.VF.isScalable()) {
1930 auto *SplatStartIdx = Builder.CreateVectorSplat(State.VF, StartIdx0);
1936 State.set(this, Add);
1950 assert((State.VF.isScalable() || isa<Constant>(StartIdx)) &&
1955 State.set(this, Add, VPLane(Lane));
1969 void VPWidenGEPRecipe::execute(VPTransformState &State) {
1970 assert(State.VF.isVector() && "not widening");
1993 Ops.push_back(State.get(getOperand(I), VPLane(0)));
1995 auto *NewGEP = State.Builder.CreateGEP(GEP->getSourceElementType(), Ops[0],
1998 Value *Splat = State.Builder.CreateVectorSplat(State.VF, NewGEP);
1999 State.set(this, Splat);
2000 State.addMetadata(Splat, GEP);
2006 auto *Ptr = isPointerLoopInvariant() ? State.get(getOperand(0), VPLane(0))
2007 : State.get(getOperand(0));
2015 Indices.push_back(State.get(Operand, VPLane(0)));
2017 Indices.push_back(State.get(Operand));
2022 auto *NewGEP = State.Builder.CreateGEP(GEP->getSourceElementType(), Ptr,
2024 assert((State.VF.isScalar() || NewGEP->getType()->isVectorTy()) &&
2026 State.set(this, NewGEP);
2027 State.addMetadata(NewGEP, GEP);
2057 void VPReverseVectorPointerRecipe::execute(VPTransformState &State) {
2058 auto &Builder = State.Builder;
2059 State.setDebugLocFrom(getDebugLoc());
2061 Type *IndexTy = getGEPIndexTy(State.VF.isScalable(), /*IsReverse*/ true,
2065 Value *RunTimeVF = State.get(getVFValue(), VPLane(0));
2073 Value *Ptr = State.get(getOperand(0), VPLane(0));
2079 State.set(this, ResultPtr, /*IsScalar*/ true);
2093 void VPVectorPointerRecipe::execute(VPTransformState &State) {
2094 auto &Builder = State.Builder;
2095 State.setDebugLocFrom(getDebugLoc());
2097 Type *IndexTy = getGEPIndexTy(State.VF.isScalable(), /*IsReverse*/ false,
2099 Value *Ptr = State.get(getOperand(0), VPLane(0));
2101 Value *Increment = createStepForVF(Builder, IndexTy, State.VF, CurrentPart);
2105 State.set(this, ResultPtr, /*IsScalar*/ true);
2119 void VPBlendRecipe::execute(VPTransformState &State) {
2121 State.setDebugLocFrom(getDebugLoc());
2143 Value *In0 = State.get(getIncomingValue(In), OnlyFirstLaneUsed);
2149 Value *Cond = State.get(getMask(In), OnlyFirstLaneUsed);
2150 Result = State.Builder.CreateSelect(Cond, In0, Result, "predphi");
2153 State.set(this, Result, OnlyFirstLaneUsed);
2194 void VPReductionRecipe::execute(VPTransformState &State) {
2195 assert(!State.Lane && "Reduction being replicated.");
2196 Value *PrevInChain = State.get(getChainOp(), /*IsScalar*/ true);
2199 IRBuilderBase::FastMathFlagGuard FMFGuard(State.Builder);
2200 State.Builder.setFastMathFlags(RdxDesc.getFastMathFlags());
2201 State.setDebugLocFrom(getDebugLoc());
2202 Value *NewVecOp = State.get(getVecOp());
2204 Value *NewCond = State.get(Cond, State.VF.isScalar());
2214 if (State.VF.isVector())
2215 Start = State.Builder.CreateVectorSplat(VecTy->getElementCount(), Start);
2217 Value *Select = State.Builder.CreateSelect(NewCond, NewVecOp, Start);
2223 if (State.VF.isVector())
2225 createOrderedReduction(State.Builder, RdxDesc, NewVecOp, PrevInChain);
2227 NewRed = State.Builder.CreateBinOp(
2232 PrevInChain = State.get(getChainOp(), /*IsScalar*/ true);
2233 NewRed = createReduction(State.Builder, RdxDesc, NewVecOp);
2235 NextInChain = createMinMaxOp(State.Builder, RdxDesc.getRecurrenceKind(),
2238 NextInChain = State.Builder.CreateBinOp(
2241 State.set(this, NextInChain, /*IsScalar*/ true);
2244 void VPReductionEVLRecipe::execute(VPTransformState &State) {
2245 assert(!State.Lane && "Reduction being replicated.");
2247 auto &Builder = State.Builder;
2254 Value *Prev = State.get(getChainOp(), /*IsScalar*/ true);
2255 Value *VecOp = State.get(getVecOp());
2256 Value *EVL = State.get(getEVL(), VPLane(0));
2263 Mask = State.get(CondOp);
2265 Mask = Builder.CreateVectorSplat(State.VF, Builder.getTrue());
2279 State.set(this, NewRed, /*IsScalar*/ true);
2412 Value *VPScalarCastRecipe ::generate(VPTransformState &State) {
2413 State.setDebugLocFrom(getDebugLoc());
2421 Value *Op = State.get(getOperand(0), VPLane(0));
2422 return State.Builder.CreateCast(Instruction::CastOps(Opcode), Op, ResultTy);
2429 void VPScalarCastRecipe ::execute(VPTransformState &State) {
2430 State.set(this, generate(State), VPLane(0));
2444 void VPBranchOnMaskRecipe::execute(VPTransformState &State) {
2445 assert(State.Lane && "Branch on Mask works only on single instance.");
2451 ConditionBit = State.get(BlockInMask, *State.Lane);
2453 ConditionBit = State.Builder.getTrue();
2457 auto *CurrentTerminator = State.CFG.PrevBB->getTerminator();
2460 auto *CondBr = BranchInst::Create(State.CFG.PrevBB, nullptr, ConditionBit);
2473 void VPPredInstPHIRecipe::execute(VPTransformState &State) {
2474 State.setDebugLocFrom(getDebugLoc());
2475 assert(State.Lane && "Predicated instruction PHI works per instance.");
2477 cast<Instruction>(State.get(getOperand(0), *State.Lane));
2490 if (State.hasVectorValue(getOperand(0))) {
2491 Value *VectorValue = State.get(getOperand(0));
2493 PHINode *VPhi = State.Builder.CreatePHI(IEI->getType(), 2);
2496 if (State.hasVectorValue(this))
2497 State.reset(this, VPhi);
2499 State.set(this, VPhi);
2502 State.reset(getOperand(0), VPhi);
2504 if (vputils::onlyFirstLaneUsed(this) && !State.Lane->isFirstLane())
2508 PHINode *Phi = State.Builder.CreatePHI(PredInstType, 2);
2512 if (State.hasScalarValue(this, *State.Lane))
2513 State.reset(this, Phi, *State.Lane);
2515 State.set(this, Phi, *State.Lane);
2518 State.reset(getOperand(0), Phi, *State.Lane);
2571 void VPWidenLoadRecipe::execute(VPTransformState &State) {
2575 auto *DataTy = VectorType::get(ScalarDataTy, State.VF);
2579 auto &Builder = State.Builder;
2580 State.setDebugLocFrom(getDebugLoc());
2585 Mask = State.get(VPMask);
2590 Value *Addr = State.get(getAddr(), /*IsScalar*/ !CreateGather);
2603 State.addMetadata(NewLI, LI);
2606 State.set(this, NewLI);
2630 void VPWidenLoadEVLRecipe::execute(VPTransformState &State) {
2634 auto *DataTy = VectorType::get(ScalarDataTy, State.VF);
2638 auto &Builder = State.Builder;
2639 State.setDebugLocFrom(getDebugLoc());
2641 Value *EVL = State.get(getEVL(), VPLane(0));
2642 Value *Addr = State.get(getAddr(), !CreateGather);
2645 Mask = State.get(VPMask);
2649 Mask = Builder.CreateVectorSplat(State.VF, Builder.getTrue());
2664 State.addMetadata(NewLI, LI);
2668 State.set(this, Res);
2706 void VPWidenStoreRecipe::execute(VPTransformState &State) {
2713 auto &Builder = State.Builder;
2714 State.setDebugLocFrom(getDebugLoc());
2720 Mask = State.get(VPMask);
2725 Value *StoredVal = State.get(StoredVPValue);
2733 Value *Addr = State.get(getAddr(), /*IsScalar*/ !CreateScatter);
2741 State.addMetadata(NewSI, SI);
2752 void VPWidenStoreEVLRecipe::execute(VPTransformState &State) {
2759 auto &Builder = State.Builder;
2760 State.setDebugLocFrom(getDebugLoc());
2763 Value *StoredVal = State.get(StoredValue);
2764 Value *EVL = State.get(getEVL(), VPLane(0));
2769 Mask = State.get(VPMask);
2773 Mask = Builder.CreateVectorSplat(State.VF, Builder.getTrue());
2775 Value *Addr = State.get(getAddr(), !CreateScatter);
2789 State.addMetadata(NewSI, SI);
2914 void VPInterleaveRecipe::execute(VPTransformState &State) {
2915 assert(!State.Lane && "Interleave group being replicated.");
2922 auto *VecTy = VectorType::get(ScalarTy, State.VF * InterleaveFactor);
2930 Value *ResAddr = State.get(Addr, VPLane(0));
2932 State.setDebugLocFrom(I->getDebugLoc());
2940 getRuntimeVF(State.Builder, State.Builder.getInt32Ty(), State.VF);
2942 State.Builder.CreateSub(RuntimeVF, State.Builder.getInt32(1));
2943 Index = State.Builder.CreateMul(Index,
2944 State.Builder.getInt32(Group->getFactor()));
2945 Index = State.Builder.CreateNeg(Index);
2950 ResAddr = State.Builder.CreateGEP(ScalarTy, ResAddr, Index, "", InBounds);
2953 State.setDebugLocFrom(Instr->getDebugLoc());
2956 auto CreateGroupMask = [&BlockInMask, &State,
2958 if (State.VF.isScalable()) {
2962 auto *ResBlockInMask = State.get(BlockInMask);
2964 auto *MaskTy = VectorType::get(State.Builder.getInt1Ty(),
2965 State.VF.getKnownMinValue() * 2, true);
2966 return State.Builder.CreateIntrinsic(
2974 Value *ResBlockInMask = State.get(BlockInMask);
2975 Value *ShuffledMask = State.Builder.CreateShuffleVector(
2977 createReplicatedMask(InterleaveFactor, State.VF.getKnownMinValue()),
2979 return MaskForGaps ? State.Builder.CreateBinOp(Instruction::And,
2989 MaskForGaps = createBitMaskForGaps(State.Builder,
2990 State.VF.getKnownMinValue(), *Group);
2997 NewLoad = State.Builder.CreateMaskedLoad(VecTy, ResAddr,
3001 NewLoad = State.Builder.CreateAlignedLoad(VecTy, ResAddr,
3006 const DataLayout &DL = State.CFG.PrevBB->getDataLayout();
3013 Value *DI = State.Builder.CreateIntrinsic(
3023 Value *StridedVec = State.Builder.CreateExtractValue(DI, I);
3026 VectorType *OtherVTy = VectorType::get(Member->getType(), State.VF);
3028 createBitOrPointerCast(State.Builder, StridedVec, OtherVTy, DL);
3032 StridedVec = State.Builder.CreateVectorReverse(StridedVec, "reverse");
3034 State.set(VPDefs[J], StridedVec);
3052 createStrideMask(I, InterleaveFactor, State.VF.getKnownMinValue());
3054 State.Builder.CreateShuffleVector(NewLoad, StrideMask, "strided.vec");
3058 assert(!State.VF.isScalable() && "VF is assumed to be non scalable.");
3059 VectorType *OtherVTy = VectorType::get(Member->getType(), State.VF);
3061 createBitOrPointerCast(State.Builder, StridedVec, OtherVTy, DL);
3065 StridedVec = State.Builder.CreateVectorReverse(StridedVec, "reverse");
3067 State.set(VPDefs[J], StridedVec);
3074 auto *SubVT = VectorType::get(ScalarTy, State.VF);
3078 createBitMaskForGaps(State.Builder, State.VF.getKnownMinValue(), *Group);
3079 assert((!MaskForGaps || !State.VF.isScalable()) &&
3097 Value *StoredVec = State.get(StoredValues[StoredIdx]);
3101 StoredVec = State.Builder.CreateVectorReverse(StoredVec, "reverse");
3106 StoredVec = createBitOrPointerCast(State.Builder, StoredVec, SubVT, DL);
3112 Value *IVec = interleaveVectors(State.Builder, StoredVecs, "interleaved.vec");
3116 NewStoreInstr = State.Builder.CreateMaskedStore(
3120 State.Builder.CreateAlignedStore(IVec, ResAddr, Group->getAlign());
3211 void VPWidenPointerInductionRecipe::execute(VPTransformState &State) {
3217 assert(!onlyScalarsGenerated(State.VF.isScalable()) &&
3226 BasicBlock *VectorPH = State.CFG.getPreheaderBBFor(this);
3234 PHINode *CanonicalIV = cast<PHINode>(State.get(IVR, /*IsScalar*/ true));
3243 cast<GetElementPtrInst>(State.get(getFirstUnrolledPartOperand()));
3248 BasicBlock::iterator InductionLoc = State.Builder.GetInsertPoint();
3249 Value *ScalarStepValue = State.get(getStepValue(), VPLane(0));
3250 Type *PhiType = State.TypeAnalysis.inferScalarType(getStepValue());
3251 Value *RuntimeVF = getRuntimeVF(State.Builder, PhiType, State.VF);
3262 State.Builder.CreateMul(RuntimeVF, ConstantInt::get(PhiType, UF));
3265 State.Builder.getInt8Ty(), NewPointerPhi,
3266 State.Builder.CreateMul(ScalarStepValue, NumUnrolledElems), "ptr.ind",
3274 Type *VecPhiType = VectorType::get(PhiType, State.VF);
3275 Value *StartOffsetScalar = State.Builder.CreateMul(
3278 State.Builder.CreateVectorSplat(State.VF, StartOffsetScalar);
3280 StartOffset = State.Builder.CreateAdd(
3281 StartOffset, State.Builder.CreateStepVector(VecPhiType));
3283 assert(ScalarStepValue == State.get(getOperand(1), VPLane(0)) &&
3285 Value *GEP = State.Builder.CreateGEP(
3286 State.Builder.getInt8Ty(), NewPointerPhi,
3287 State.Builder.CreateMul(StartOffset, State.Builder.CreateVectorSplat(
3288 State.VF, ScalarStepValue)),
3290 State.set(this, GEP);
3313 void VPExpandSCEVRecipe::execute(VPTransformState &State) {
3314 assert(!State.Lane && "cannot be used in per-lane");
3315 if (State.ExpandedSCEVs.contains(Expr)) {
3320 State.Builder.SetInsertPoint(State.CFG.VPBB2IRBB[getParent()]);
3321 assert(State.get(this, VPLane(0)) == State.ExpandedSCEVs[Expr] &&
3326 const DataLayout &DL = State.CFG.PrevBB->getDataLayout();
3330 &*State.Builder.GetInsertPoint());
3331 State.ExpandedSCEVs[Expr] = Res;
3332 State.set(this, Res, VPLane(0));
3344 void VPWidenCanonicalIVRecipe::execute(VPTransformState &State) {
3345 Value *CanonicalIV = State.get(getOperand(0), /*IsScalar*/ true);
3347 IRBuilder<> Builder(State.CFG.PrevBB->getTerminator());
3348 ElementCount VF = State.VF;
3359 State.set(this, CanonicalVectorIV);
3372 void VPFirstOrderRecurrencePHIRecipe::execute(VPTransformState &State) {
3373 auto &Builder = State.Builder;
3377 Type *VecTy = State.VF.isScalar()
3379 : VectorType::get(VectorInit->getType(), State.VF);
3381 BasicBlock *VectorPH = State.CFG.getPreheaderBBFor(this);
3382 if (State.VF.isVector()) {
3387 auto *RuntimeVF = getRuntimeVF(Builder, IdxTy, State.VF);
3395 Phi->insertBefore(State.CFG.PrevBB->getFirstInsertionPt());
3397 State.set(this, Phi);
3429 void VPReductionPHIRecipe::execute(VPTransformState &State) {
3430 auto &Builder = State.Builder;
3434 ElementCount VF = State.VF.divideCoefficientBy(VFScaleFactor);
3445 bool ScalarPHI = State.VF.isScalar() || IsInLoop;
3449 BasicBlock *HeaderBB = State.CFG.PrevBB;
3450 assert(State.CurrentParentLoop->getHeader() == HeaderBB &&
3454 State.set(this, Phi, IsInLoop);
3456 BasicBlock *VectorPH = State.CFG.getPreheaderBBFor(this);
3470 StartV = Iden = State.get(StartVPV);
3486 StartV = Iden = Builder.CreateVectorSplat(State.VF, Iden);
3508 Phi = cast<PHINode>(State.get(this, IsInLoop));
3526 void VPWidenPHIRecipe::execute(VPTransformState &State) {
3530 State.setDebugLocFrom(getDebugLoc());
3531 Value *Op0 = State.get(getOperand(0));
3533 Value *VecPhi = State.Builder.CreatePHI(VecTy, 2, "vec.phi");
3534 State.set(this, VecPhi);
3560 void VPActiveLaneMaskPHIRecipe::execute(VPTransformState &State) {
3561 BasicBlock *VectorPH = State.CFG.getPreheaderBBFor(this);
3562 Value *StartMask = State.get(getOperand(0));
3564 State.Builder.CreatePHI(StartMask->getType(), 2, "active.lane.mask");
3567 State.set(this, Phi);
3592 void VPScalarPHIRecipe::execute(VPTransformState &State) {
3593 BasicBlock *VectorPH = State.CFG.getPreheaderBBFor(this);
3594 Value *Start = State.get(getStartValue(), VPLane(0));
3595 PHINode *Phi = State.Builder.CreatePHI(Start->getType(), 2, Name);
3598 State.set(this, Phi, /*IsScalar=*/true);