Lines Matching defs:Plan
487 VPlan &Plan)
492 PSI(PSI), RTChecks(RTChecks), Plan(Plan),
493 VectorPHVPB(Plan.getEntry()->getSingleSuccessor()) {
528 /// Fix the non-induction PHIs in \p Plan.
591 /// Introduces a new VPIRBasicBlock for \p CheckIRBB to Plan between the
689 VPlan &Plan;
691 /// The vector preheader block of \p Plan, used as target for check blocks
738 GeneratedRTChecks &Checks, VPlan &Plan)
741 CM, BFI, PSI, Checks, Plan),
778 GeneratedRTChecks &Check, VPlan &Plan)
780 EPI, LVL, CM, BFI, PSI, Check, Plan) {}
807 GeneratedRTChecks &Checks, VPlan &Plan)
809 EPI, LVL, CM, BFI, PSI, Checks, Plan) {
2468 VPBlockBase *ScalarPH = Plan.getScalarPreheader();
2474 VPIRBasicBlock *CheckVPIRBB = Plan.createVPIRBasicBlock(CheckIRBB);
2648 replaceVPBBWithIRVPBB(Plan.getMiddleBlock(), LoopMiddleBlock);
2652 replaceVPBBWithIRVPBB(Plan.getScalarPreheader(), LoopScalarPreHeader);
2735 [ ] <-- old preheader - loop iteration number check and SCEVs in Plan's
2916 if (!State.Plan->getVectorLoopRegion())
2922 VPRegionBlock *VectorRegion = State.Plan->getVectorLoopRegion();
3024 auto Iter = vp_depth_first_deep(Plan.getEntry());
4340 for (const auto &Plan : VPlans) {
4341 for (ElementCount VF : Plan->vectorFactors()) {
4344 precomputeCosts(*Plan, VF, CostCtx);
4345 auto Iter = vp_depth_first_deep(Plan->getVectorLoopRegion()->getEntry());
4447 /// Check if any recipe of \p Plan will generate a vector value, which will be
4449 static bool willGenerateVectors(VPlan &Plan, ElementCount VF,
4452 VPTypeAnalysis TypeInfo(Plan.getCanonicalIV()->getScalarType());
4454 collectEphemeralRecipesForVPlan(Plan, EphemeralRecipes);
4458 vp_depth_first_shallow(Plan.getVectorLoopRegion()->getEntry()))) {
7195 LoopVectorizationPlanner::precomputeCosts(VPlan &Plan, ElementCount VF,
7392 InstructionCost LoopVectorizationPlanner::cost(VPlan &Plan,
7396 InstructionCost Cost = precomputeCosts(Plan, VF, CostCtx);
7399 Cost += Plan.cost(VF, CostCtx);
7416 /// not have corresponding recipes in \p Plan and are not marked to be ignored
7419 static bool planContainsAdditionalSimplifications(VPlan &Plan,
7422 // First collect all instructions for the recipes in Plan.
7432 auto Iter = vp_depth_first_deep(Plan.getVectorLoopRegion()->getEntry());
8046 VPIRBasicBlock *NewEntry = Plan.createVPIRBasicBlock(Insert);
8047 VPBasicBlock *OldEntry = Plan.getEntry();
8049 Plan.setEntry(NewEntry);
8207 VPBasicBlock *HeaderVPBB = Plan.getVectorLoopRegion()->getEntryBasicBlock();
8209 auto *IV = new VPWidenCanonicalIVRecipe(Plan.getCanonicalIV());
8215 VPValue *BTC = Plan.getOrCreateBackedgeTakenCount();
8305 Ptr, &Plan.getVF(), getLoadStoreType(I), Flags, I->getDebugLoc());
8329 VPlan &Plan, ScalarEvolution &SE, Loop &OrigLoop) {
8336 vputils::getOrCreateVPValueForSCEVExpr(Plan, IndDesc.getStep(), SE);
8338 return new VPWidenIntOrFpInductionRecipe(Phi, Start, Step, &Plan.getVF(),
8343 return new VPWidenIntOrFpInductionRecipe(Phi, Start, Step, &Plan.getVF(),
8353 return createWidenInductionRecipes(Phi, Phi, Operands[0], *II, Plan,
8358 VPValue *Step = vputils::getOrCreateVPValueForSCEVExpr(Plan, II->getStep(),
8393 VPValue *Start = Plan.getOrAddLiveIn(II.getStartValue());
8394 return createWidenInductionRecipes(Phi, I, Start, II, Plan, *PSE.getSE(),
8503 Mask = Plan.getOrAddLiveIn(
8546 Plan.getOrAddLiveIn(ConstantInt::get(I->getType(), 1u, false));
8585 return Plan.getOrAddLiveIn(C->getValue());
8918 if (auto Plan = tryToBuildVPlanWithVPRecipes(SubRange)) {
8920 if (!Plan->hasVF(ElementCount::getFixed(1)))
8922 *Plan, CM.getMinimalBitwidths());
8923 VPlanTransforms::optimize(*Plan);
8928 *Plan, CM.getMaxSafeElements()))
8930 assert(verifyVPlanIsValid(*Plan) && "VPlan is invalid");
8931 VPlans.push_back(std::move(Plan));
8939 static void addCanonicalIVRecipes(VPlan &Plan, Type *IdxTy, bool HasNUW,
8942 auto *StartV = Plan.getOrAddLiveIn(StartIdx);
8946 VPRegionBlock *TopRegion = Plan.getVectorLoopRegion();
8953 Instruction::Add, {CanonicalIVPHI, &Plan.getVFxUF()}, {HasNUW, false}, DL,
8959 {CanonicalIVIncrement, &Plan.getVectorTripCount()}, DL);
9003 static void addScalarResumePhis(VPRecipeBuilder &Builder, VPlan &Plan,
9005 VPTypeAnalysis TypeInfo(Plan.getCanonicalIV()->getScalarType());
9006 auto *ScalarPH = Plan.getScalarPreheader();
9008 VPRegionBlock *VectorRegion = Plan.getVectorLoopRegion();
9013 VPValue *OneVPV = Plan.getOrAddLiveIn(
9014 ConstantInt::get(Plan.getCanonicalIV()->getScalarType(), 1));
9015 for (VPRecipeBase &ScalarPhiR : *Plan.getScalarHeader()) {
9027 &Plan.getVectorTripCount())) {
9047 assert(VectorRegion->getSingleSuccessor() == Plan.getMiddleBlock() &&
9065 VPlan &Plan) {
9067 for (VPIRBasicBlock *ExitVPBB : Plan.getExitBlocks()) {
9075 if (ExitVPBB->getSinglePredecessor() != Plan.getMiddleBlock()) {
9095 // Add exit values to \p Plan. Extracts are added for each entry in \p
9098 addUsersInExitBlocks(VPlan &Plan,
9103 auto *MiddleVPBB = Plan.getMiddleBlock();
9122 VPlan &Plan, SetVector<VPIRInstruction *> &ExitUsersToFix) {
9123 VPRegionBlock *VectorRegion = Plan.getVectorLoopRegion();
9124 auto *ScalarPHVPBB = Plan.getScalarPreheader();
9125 auto *MiddleVPBB = Plan.getMiddleBlock();
9128 VPValue *TwoVPV = Plan.getOrAddLiveIn(
9129 ConstantInt::get(Plan.getCanonicalIV()->getScalarType(), 2));
9136 assert(VectorRegion->getSingleSuccessor() == Plan.getMiddleBlock() &&
9243 VPlanPtr Plan = VPlan::createInitialVPlan(Legal->getWidestInductionType(),
9262 addCanonicalIVRecipes(*Plan, Legal->getWidestInductionType(), HasNUW, DL);
9264 VPRecipeBuilder RecipeBuilder(*Plan, OrigLoop, TLI, &TTI, Legal, CM, PSE,
9302 VPBasicBlock *HeaderVPBB = Plan->getVectorLoopRegion()->getEntryBasicBlock();
9314 auto *MiddleVPBB = Plan->getMiddleBlock();
9335 Operands.push_back(Plan->getOrAddLiveIn(
9381 VPBlockUtils::insertBlockAfter(Plan->createVPBasicBlock(""), VPBB);
9388 assert(isa<VPRegionBlock>(Plan->getVectorLoopRegion()) &&
9389 !Plan->getVectorLoopRegion()->getEntryBasicBlock()->empty() &&
9411 *Plan, *PSE.getSE(), OrigLoop, UncountableExitingBlock,
9420 addScalarResumePhis(RecipeBuilder, *Plan, IVEndValues);
9422 collectUsersInExitBlocks(OrigLoop, RecipeBuilder, *Plan);
9423 addExitUsersForFirstOrderRecurrences(*Plan, ExitUsersToFix);
9424 addUsersInExitBlocks(*Plan, ExitUsersToFix);
9432 adjustRecipesForReductions(Plan, RecipeBuilder, Range.Start);
9437 VPlanTransforms::runPass(VPlanTransforms::createInterleaveGroups, *Plan,
9442 Plan->addVF(VF);
9443 Plan->setName("Initial VPlan");
9447 auto CanUseVersionedStride = [&Plan](VPUser &U, unsigned) {
9451 Plan->getVectorLoopRegion()->getSinglePredecessor();
9460 auto *CI = Plan->getOrAddLiveIn(
9462 if (VPValue *StrideVPV = Plan->getLiveIn(StrideV))
9470 VPValue *StrideVPV = Plan->getLiveIn(U);
9476 VPValue *CI = Plan->getOrAddLiveIn(ConstantInt::get(U->getType(), C));
9484 VPlanTransforms::runPass(VPlanTransforms::dropPoisonGeneratingRecipes, *Plan,
9490 *Plan, Builder))
9499 VPlanTransforms::addActiveLaneMask(*Plan, ForControlFlow,
9502 VPlanTransforms::optimizeInductionExitUsers(*Plan, IVEndValues);
9504 assert(verifyVPlanIsValid(*Plan) && "VPlan is invalid");
9505 return Plan;
9517 auto Plan = VPlan::createInitialVPlan(Legal->getWidestInductionType(), PSE,
9521 VPlanHCFGBuilder HCFGBuilder(OrigLoop, LI, *Plan);
9525 Plan->addVF(VF);
9528 Plan,
9535 addCanonicalIVRecipes(*Plan, Legal->getWidestInductionType(), HasNUW,
9540 VPRecipeBuilder RecipeBuilder(*Plan, OrigLoop, TLI, &TTI, Legal, CM, PSE,
9542 for (auto &R : Plan->getVectorLoopRegion()->getEntryBasicBlock()->phis()) {
9551 addScalarResumePhis(RecipeBuilder, *Plan, IVEndValues);
9553 assert(verifyVPlanIsValid(*Plan) && "VPlan is invalid");
9554 return Plan;
9571 VPlanPtr &Plan, VPRecipeBuilder &RecipeBuilder, ElementCount MinVF) {
9573 VPRegionBlock *VectorLoopRegion = Plan->getVectorLoopRegion();
9575 VPBasicBlock *MiddleVPBB = Plan->getMiddleBlock();
9599 UserRecipe->getParent() == Plan->getScalarPreheader()) &&
9709 Plan->getVectorLoopRegion()->getEntryBasicBlock()->phis()) {
9820 PhiR->setOperand(0, Plan->getOrAddLiveIn(ConstantInt::getFalse(
9830 PhiR->setOperand(0, Plan->getOrAddLiveIn(RdxDesc.getSentinelValue()));
9836 VPlanTransforms::runPass(VPlanTransforms::clearReductionWrapFlags, *Plan);
9991 // Plan how to best vectorize, return the best VF and its cost.
10220 /// Prepare \p Plan for vectorizing the epilogue loop. That is, re-use expanded
10223 preparePlanForEpilogueVectorLoop(VPlan &Plan, Loop *L,
10226 VPRegionBlock *VectorLoop = Plan.getVectorLoopRegion();
10236 for (auto &R : make_early_inc_range(*Plan.getEntry())) {
10241 Plan.getOrAddLiveIn(ExpandedSCEVs.find(ExpandR->getSCEV())->second);
10243 if (Plan.getTripCount() == ExpandR)
10244 Plan.resetTripCount(ExpandedVal);
10281 VPValue *VPV = Plan.getOrAddLiveIn(EPResumeVal);
10335 VPValue *StartVal = Plan.getOrAddLiveIn(ResumeV);
10532 // Plan how to best vectorize.