Lines Matching defs:Plan
35 VPlanPtr &Plan,
41 Plan->getVectorLoopRegion());
62 VPValue *Start = Plan->getOrAddLiveIn(II->getStartValue());
64 vputils::getOrCreateVPValueForSCEVExpr(*Plan, II->getStep(), SE);
66 Phi, Start, Step, &Plan->getVF(), *II, Ingredient.getDebugLoc());
110 static bool sinkScalarOperands(VPlan &Plan) {
111 auto Iter = vp_depth_first_deep(Plan.getEntry());
131 bool ScalarVFOnly = Plan.hasScalarVFOnly();
220 static bool mergeReplicateRegionsIntoSuccessors(VPlan &Plan) {
228 vp_depth_first_deep(Plan.getEntry()))) {
305 VPlan &Plan) {
313 Plan.createVPBasicBlock(Twine(RegionName) + ".entry", BOMRecipe);
322 Plan.createVPBasicBlock(Twine(RegionName) + ".if", RecipeWithoutMask);
333 Plan.createVPBasicBlock(Twine(RegionName) + ".continue", PHIRecipe);
335 Plan.createVPRegionBlock(Entry, Exiting, RegionName, true);
345 static void addReplicateRegions(VPlan &Plan) {
348 vp_depth_first_deep(Plan.getEntry()))) {
365 VPBlockBase *Region = createReplicateRegion(RepR, Plan);
373 static bool mergeBlocksIntoPredecessors(VPlan &Plan) {
376 vp_depth_first_deep(Plan.getEntry()))) {
377 // Don't fold the blocks in the skeleton of the Plan into their single
407 void VPlanTransforms::createAndOptimizeReplicateRegions(VPlan &Plan) {
409 addReplicateRegions(Plan);
413 ShouldSimplify = sinkScalarOperands(Plan);
414 ShouldSimplify |= mergeReplicateRegionsIntoSuccessors(Plan);
415 ShouldSimplify |= mergeBlocksIntoPredecessors(Plan);
425 static void removeRedundantInductionCasts(VPlan &Plan) {
426 for (auto &Phi : Plan.getVectorLoopRegion()->getEntryBasicBlock()->phis()) {
458 static void removeRedundantCanonicalIVs(VPlan &Plan) {
459 VPCanonicalIVPHIRecipe *CanonicalIV = Plan.getCanonicalIV();
470 VPBasicBlock *HeaderVPBB = Plan.getVectorLoopRegion()->getEntryBasicBlock();
513 void VPlanTransforms::removeDeadRecipes(VPlan &Plan) {
515 Plan.getEntry());
528 createScalarIVSteps(VPlan &Plan, InductionDescriptor::InductionKind Kind,
533 VPBasicBlock *HeaderVPBB = Plan.getVectorLoopRegion()->getEntryBasicBlock();
534 VPCanonicalIVPHIRecipe *CanonicalIV = Plan.getCanonicalIV();
590 static void legalizeAndOptimizeInductions(VPlan &Plan) {
592 VPBasicBlock *HeaderVPBB = Plan.getVectorLoopRegion()->getEntryBasicBlock();
593 bool HasOnlyVectorVFs = !Plan.hasVF(ElementCount::getFixed(1));
628 if (!PtrIV->onlyScalarsGenerated(Plan.hasScalableVF()))
633 Plan.getOrAddLiveIn(ConstantInt::get(ID.getStep()->getType(), 0));
636 Plan, InductionDescriptor::IK_IntInduction, Instruction::Add, nullptr,
656 Plan, ID.getKind(), ID.getInductionOpcode(),
733 VPlan &Plan, DenseMap<VPValue *, VPValue *> &EndValues) {
735 SmallVector<VPIRBasicBlock *> ExitVPBBs(Plan.getExitBlocks());
743 assert(PredVPBB == Plan.getMiddleBlock() &&
746 VPTypeAnalysis TypeInfo(Plan.getCanonicalIV()->getScalarType());
747 VPBuilder B(Plan.getMiddleBlock()->getTerminator());
777 auto *Zero = Plan.getOrAddLiveIn(
796 /// Remove redundant EpxandSCEVRecipes in \p Plan's entry block by replacing
798 static void removeRedundantExpandSCEVRecipes(VPlan &Plan) {
802 make_early_inc_range(*Plan.getEntry()->getEntryBasicBlock())) {
966 /// Try to simplify the recipes in \p Plan. Use \p CanonicalIVTy as type for all
968 static void simplifyRecipes(VPlan &Plan, Type &CanonicalIVTy) {
970 Plan.getEntry());
979 void VPlanTransforms::optimizeForVFAndUF(VPlan &Plan, ElementCount BestVF,
982 assert(Plan.hasVF(BestVF) && "BestVF is not available in Plan");
983 assert(Plan.hasUF(BestUF) && "BestUF is not available in Plan");
984 VPRegionBlock *VectorRegion = Plan.getVectorLoopRegion();
1000 vputils::getSCEVExprForVPValue(Plan.getTripCount(), SE);
1013 auto *CanIVTy = Plan.getCanonicalIV()->getScalarType();
1033 simplifyRecipes(Plan, *CanIVTy);
1040 {Plan.getOrAddLiveIn(ConstantInt::getTrue(Ctx))}, Term->getDebugLoc());
1045 VPlanTransforms::removeDeadRecipes(Plan);
1047 Plan.setVF(BestVF);
1048 Plan.setUF(BestUF);
1209 bool VPlanTransforms::adjustFixedOrderRecurrences(VPlan &Plan,
1212 VPDT.recalculate(Plan);
1216 Plan.getVectorLoopRegion()->getEntry()->getEntryBasicBlock()->phis())
1257 void VPlanTransforms::clearReductionWrapFlags(VPlan &Plan) {
1259 Plan.getVectorLoopRegion()->getEntryBasicBlock()->phis()) {
1275 /// Move loop-invariant recipes out of the vector loop region in \p Plan.
1276 static void licm(VPlan &Plan) {
1277 VPBasicBlock *Preheader = Plan.getVectorPreheader();
1291 VPRegionBlock *LoopRegion = Plan.getVectorLoopRegion();
1310 VPlan &Plan, const MapVector<Instruction *, uint64_t> &MinBWs) {
1321 Type *CanonicalIVType = Plan.getCanonicalIV()->getScalarType();
1323 VPBasicBlock *PH = Plan.getVectorPreheader();
1325 vp_depth_first_deep(Plan.getVectorLoopRegion()))) {
1442 void VPlanTransforms::optimize(VPlan &Plan) {
1443 runPass(removeRedundantCanonicalIVs, Plan);
1444 runPass(removeRedundantInductionCasts, Plan);
1446 runPass(simplifyRecipes, Plan, *Plan.getCanonicalIV()->getScalarType());
1447 runPass(removeDeadRecipes, Plan);
1448 runPass(legalizeAndOptimizeInductions, Plan);
1449 runPass(removeRedundantExpandSCEVRecipes, Plan);
1450 runPass(simplifyRecipes, Plan, *Plan.getCanonicalIV()->getScalarType());
1451 runPass(removeDeadRecipes, Plan);
1453 runPass(createAndOptimizeReplicateRegions, Plan);
1454 runPass(mergeBlocksIntoPredecessors, Plan);
1455 runPass(licm, Plan);
1458 // Add a VPActiveLaneMaskPHIRecipe and related recipes to \p Plan and replace
1492 VPlan &Plan, bool DataAndControlFlowWithoutRuntimeCheck) {
1493 VPRegionBlock *TopRegion = Plan.getVectorLoopRegion();
1495 auto *CanonicalIVPHI = Plan.getCanonicalIV();
1507 auto *VecPreheader = Plan.getVectorPreheader();
1511 VPValue *TC = Plan.getTripCount();
1566 static SmallVector<VPValue *> collectAllHeaderMasks(VPlan &Plan) {
1569 find_if(Plan.getCanonicalIV()->users(),
1571 assert(count_if(Plan.getCanonicalIV()->users(),
1575 if (FoundWidenCanonicalIVUser != Plan.getCanonicalIV()->users().end()) {
1583 VPBasicBlock *HeaderVPBB = Plan.getVectorLoopRegion()->getEntryBasicBlock();
1596 if (!HeaderMask || !vputils::isHeaderMask(HeaderMask, Plan))
1608 VPlan &Plan, bool UseActiveLaneMaskForControlFlow,
1616 find_if(Plan.getCanonicalIV()->users(),
1625 Plan, DataAndControlFlowWithoutRuntimeCheck);
1629 {WideCanonicalIV, Plan.getTripCount()}, nullptr,
1636 for (VPValue *HeaderMask : collectAllHeaderMasks(Plan))
1727 static void transformRecipestoEVLRecipes(VPlan &Plan, VPValue &EVL) {
1728 Type *CanonicalIVType = Plan.getCanonicalIV()->getScalarType();
1731 VPValue *AllOneMask = Plan.getOrAddLiveIn(ConstantInt::getTrue(Ctx));
1733 for (VPUser *U : to_vector(Plan.getVF().users())) {
1740 for (VPValue *HeaderMask : collectAllHeaderMasks(Plan)) {
1774 /// Add a VPEVLBasedIVPHIRecipe and related recipes to \p Plan and
1814 VPlan &Plan, const std::optional<unsigned> &MaxSafeElements) {
1815 VPBasicBlock *Header = Plan.getVectorLoopRegion()->getEntryBasicBlock();
1825 auto *CanonicalIVPHI = Plan.getCanonicalIV();
1834 Instruction::Sub, {Plan.getTripCount(), EVLPhi}, DebugLoc(), "avl");
1837 VPValue *AVLSafe = Plan.getOrAddLiveIn(
1863 transformRecipestoEVLRecipes(Plan, *VPEVL);
1870 Plan.setUF(1);
1875 VPlan &Plan,
1941 auto Iter = vp_depth_first_deep(Plan.getEntry());
1973 VPlan &Plan,
1984 VPDT.recalculate(Plan);
2024 VPValue *OffsetVPV = Plan.getOrAddLiveIn(
2048 void VPlanTransforms::convertToConcreteRecipes(VPlan &Plan) {
2050 vp_depth_first_deep(Plan.getEntry()))) {
2068 VPlan &Plan, ScalarEvolution &SE, Loop *OrigLoop,
2070 VPRegionBlock *LoopRegion = Plan.getVectorLoopRegion();
2073 auto *MiddleVPBB = Plan.getMiddleBlock();
2093 VPEarlyExitBlock = Plan.createVPIRBasicBlock(
2103 VPBasicBlock *NewMiddle = Plan.createVPBasicBlock("middle.split");