1 //===- VPlanTransforms.h - Utility VPlan to VPlan transforms --------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 /// 9 /// \file 10 /// This file provides utility VPlan to VPlan transformations. 11 //===----------------------------------------------------------------------===// 12 13 #ifndef LLVM_TRANSFORMS_VECTORIZE_VPLANTRANSFORMS_H 14 #define LLVM_TRANSFORMS_VECTORIZE_VPLANTRANSFORMS_H 15 16 #include "VPlan.h" 17 #include "llvm/ADT/STLFunctionalExtras.h" 18 19 namespace llvm { 20 21 class InductionDescriptor; 22 class Instruction; 23 class PHINode; 24 class ScalarEvolution; 25 class PredicatedScalarEvolution; 26 class TargetLibraryInfo; 27 class VPBuilder; 28 class VPRecipeBuilder; 29 30 struct VPlanTransforms { 31 /// Replaces the VPInstructions in \p Plan with corresponding 32 /// widen recipes. 33 static void 34 VPInstructionsToVPRecipes(VPlanPtr &Plan, 35 function_ref<const InductionDescriptor *(PHINode *)> 36 GetIntOrFpInductionDescriptor, 37 ScalarEvolution &SE, const TargetLibraryInfo &TLI); 38 39 /// Try to have all users of fixed-order recurrences appear after the recipe 40 /// defining their previous value, by either sinking users or hoisting recipes 41 /// defining their previous value (and its operands). Then introduce 42 /// FirstOrderRecurrenceSplice VPInstructions to combine the value from the 43 /// recurrence phis and previous values. 44 /// \returns true if all users of fixed-order recurrences could be re-arranged 45 /// as needed or false if it is not possible. In the latter case, \p Plan is 46 /// not valid. 47 static bool adjustFixedOrderRecurrences(VPlan &Plan, VPBuilder &Builder); 48 49 /// Clear NSW/NUW flags from reduction instructions if necessary. 50 static void clearReductionWrapFlags(VPlan &Plan); 51 52 /// Explicitly unroll \p Plan by \p UF. 53 static void unrollByUF(VPlan &Plan, unsigned UF, LLVMContext &Ctx); 54 55 /// Optimize \p Plan based on \p BestVF and \p BestUF. This may restrict the 56 /// resulting plan to \p BestVF and \p BestUF. 57 static void optimizeForVFAndUF(VPlan &Plan, ElementCount BestVF, 58 unsigned BestUF, 59 PredicatedScalarEvolution &PSE); 60 61 /// Apply VPlan-to-VPlan optimizations to \p Plan, including induction recipe 62 /// optimizations, dead recipe removal, replicate region optimizations and 63 /// block merging. 64 static void optimize(VPlan &Plan); 65 66 /// Wrap predicated VPReplicateRecipes with a mask operand in an if-then 67 /// region block and remove the mask operand. Optimize the created regions by 68 /// iteratively sinking scalar operands into the region, followed by merging 69 /// regions until no improvements are remaining. 70 static void createAndOptimizeReplicateRegions(VPlan &Plan); 71 72 /// Replace (ICMP_ULE, wide canonical IV, backedge-taken-count) checks with an 73 /// (active-lane-mask recipe, wide canonical IV, trip-count). If \p 74 /// UseActiveLaneMaskForControlFlow is true, introduce an 75 /// VPActiveLaneMaskPHIRecipe. If \p DataAndControlFlowWithoutRuntimeCheck is 76 /// true, no minimum-iteration runtime check will be created (during skeleton 77 /// creation) and instead it is handled using active-lane-mask. \p 78 /// DataAndControlFlowWithoutRuntimeCheck implies \p 79 /// UseActiveLaneMaskForControlFlow. 80 static void addActiveLaneMask(VPlan &Plan, 81 bool UseActiveLaneMaskForControlFlow, 82 bool DataAndControlFlowWithoutRuntimeCheck); 83 84 /// Insert truncates and extends for any truncated recipe. Redundant casts 85 /// will be folded later. 86 static void 87 truncateToMinimalBitwidths(VPlan &Plan, 88 const MapVector<Instruction *, uint64_t> &MinBWs); 89 90 /// Drop poison flags from recipes that may generate a poison value that is 91 /// used after vectorization, even when their operands are not poison. Those 92 /// recipes meet the following conditions: 93 /// * Contribute to the address computation of a recipe generating a widen 94 /// memory load/store (VPWidenMemoryInstructionRecipe or 95 /// VPInterleaveRecipe). 96 /// * Such a widen memory load/store has at least one underlying Instruction 97 /// that is in a basic block that needs predication and after vectorization 98 /// the generated instruction won't be predicated. 99 /// Uses \p BlockNeedsPredication to check if a block needs predicating. 100 /// TODO: Replace BlockNeedsPredication callback with retrieving info from 101 /// VPlan directly. 102 static void dropPoisonGeneratingRecipes( 103 VPlan &Plan, function_ref<bool(BasicBlock *)> BlockNeedsPredication); 104 105 /// Add a VPEVLBasedIVPHIRecipe and related recipes to \p Plan and 106 /// replaces all uses except the canonical IV increment of 107 /// VPCanonicalIVPHIRecipe with a VPEVLBasedIVPHIRecipe. 108 /// VPCanonicalIVPHIRecipe is only used to control the loop after 109 /// this transformation. 110 /// \returns true if the transformation succeeds, or false if it doesn't. 111 static bool 112 tryAddExplicitVectorLength(VPlan &Plan, 113 const std::optional<unsigned> &MaxEVLSafeElements); 114 115 // For each Interleave Group in \p InterleaveGroups replace the Recipes 116 // widening its memory instructions with a single VPInterleaveRecipe at its 117 // insertion point. 118 static void createInterleaveGroups( 119 VPlan &Plan, 120 const SmallPtrSetImpl<const InterleaveGroup<Instruction> *> 121 &InterleaveGroups, 122 VPRecipeBuilder &RecipeBuilder, bool ScalarEpilogueAllowed); 123 124 /// Remove dead recipes from \p Plan. 125 static void removeDeadRecipes(VPlan &Plan); 126 127 /// Update \p Plan to account for the uncountable early exit block in \p 128 /// UncountableExitingBlock by 129 /// * updating the condition exiting the vector loop to include the early 130 /// exit conditions 131 /// * splitting the original middle block to branch to the early exit block 132 /// if taken. 133 static void handleUncountableEarlyExit(VPlan &Plan, ScalarEvolution &SE, 134 Loop *OrigLoop, 135 BasicBlock *UncountableExitingBlock, 136 VPRecipeBuilder &RecipeBuilder); 137 138 /// Lower abstract recipes to concrete ones, that can be codegen'd. 139 static void convertToConcreteRecipes(VPlan &Plan); 140 141 /// If there's a single exit block, optimize its phi recipes that use exiting 142 /// IV values by feeding them precomputed end values instead, possibly taken 143 /// one step backwards. 144 static void 145 optimizeInductionExitUsers(VPlan &Plan, 146 DenseMap<VPValue *, VPValue *> &EndValues); 147 }; 148 149 } // namespace llvm 150 151 #endif // LLVM_TRANSFORMS_VECTORIZE_VPLANTRANSFORMS_H 152