Lines Matching defs:VPValue
16 /// also inherit from VPValue.
275 DenseMap<VPValue *, PerPartValuesTy> PerPartOutput;
278 DenseMap<VPValue *, ScalarsPerPartValuesTy> PerPartScalars;
281 /// Get the generated vector Value for a given VPValue \p Def and a given \p
284 Value *get(VPValue *Def, unsigned Part, bool IsScalar = false);
286 /// Get the generated Value for a given VPValue and given Part and Lane.
287 Value *get(VPValue *Def, const VPIteration &Instance);
289 bool hasVectorValue(VPValue *Def, unsigned Part) {
295 bool hasScalarValue(VPValue *Def, VPIteration Instance) {
305 /// Set the generated vector Value for a given VPValue and a given Part, if \p
307 void set(VPValue *Def, Value *V, unsigned Part, bool IsScalar = false) {
322 void reset(VPValue *Def, Value *V, unsigned Part) {
330 void set(VPValue *Def, Value *V, const VPIteration &Instance) {
344 void reset(VPValue *Def, Value *V, const VPIteration &Instance) {
374 void packScalarIntoVectorValue(VPValue *Def, const VPIteration &Instance);
663 virtual void dropAllReferences(VPValue *NewValue) = 0;
671 /// with \p Indent. \p SlotTracker is used to print unnamed VPValue's using
701 /// determined by where the VPValue is defined: if it is defined by a recipe
708 VPLiveOut(PHINode *Phi, VPValue *Op)
722 bool usesScalars(const VPValue *Op) const override {
761 /// VPRecipeBase and VPValue separately.
775 VPRecipeBase(const unsigned char SC, ArrayRef<VPValue *> Operands,
873 static inline bool classof(const VPValue *V) { \
889 /// more output IR that define a single result VPValue.
890 /// Note that VPRecipeBase must be inherited from before VPValue.
891 class VPSingleDefRecipe : public VPRecipeBase, public VPValue {
895 : VPRecipeBase(SC, Operands, DL), VPValue(this) {}
897 VPSingleDefRecipe(const unsigned char SC, ArrayRef<VPValue *> Operands,
899 : VPRecipeBase(SC, Operands, DL), VPValue(this) {}
904 : VPRecipeBase(SC, Operands, DL), VPValue(this, UV) {}
1254 // Takes the VPValue to extract from as first operand and the lane or part
1304 VPInstruction(unsigned Opcode, ArrayRef<VPValue *> Operands, DebugLoc DL,
1309 VPInstruction(unsigned Opcode, std::initializer_list<VPValue *> Operands,
1311 : VPInstruction(Opcode, ArrayRef<VPValue *>(Operands), DL, Name) {}
1313 VPInstruction(unsigned Opcode, CmpInst::Predicate Pred, VPValue *A,
1314 VPValue *B, DebugLoc DL = {}, const Twine &Name = "");
1316 VPInstruction(unsigned Opcode, std::initializer_list<VPValue *> Operands,
1321 VPInstruction(unsigned Opcode, std::initializer_list<VPValue *> Operands,
1329 VPInstruction(unsigned Opcode, std::initializer_list<VPValue *> Operands,
1335 SmallVector<VPValue *, 2> Operands(operands());
1388 bool onlyFirstLaneUsed(const VPValue *Op) const override;
1391 bool onlyFirstPartUsed(const VPValue *Op) const override;
1447 VPWidenCastRecipe(Instruction::CastOps Opcode, VPValue *Op, Type *ResultTy,
1455 VPWidenCastRecipe(Instruction::CastOps Opcode, VPValue *Op, Type *ResultTy)
1495 VPScalarCastRecipe(Instruction::CastOps Opcode, VPValue *Op, Type *ResultTy)
1517 bool onlyFirstLaneUsed(const VPValue *Op) const override {
1603 VPValue *getCond() const {
1623 return all_of(operands(), [](VPValue *Op) {
1660 VPVectorPointerRecipe(VPValue *Ptr, Type *IndexedTy, bool IsReverse,
1662 : VPRecipeWithIRFlags(VPDef::VPVectorPointerSC, ArrayRef<VPValue *>(Ptr),
1670 bool onlyFirstLaneUsed(const VPValue *Op) const override {
1714 VPValue *Start = nullptr, DebugLoc DL = {})
1715 : VPSingleDefRecipe(VPDefID, ArrayRef<VPValue *>(), UnderlyingInstr, DL) {
1728 static inline bool classof(const VPValue *V) {
1744 VPValue *getStartValue() {
1747 VPValue *getStartValue() const {
1752 void setStartValue(VPValue *V) { setOperand(0, V); }
1755 virtual VPValue *getBackedgeValue() {
1774 VPWidenIntOrFpInductionRecipe(PHINode *IV, VPValue *Start, VPValue *Step,
1781 VPWidenIntOrFpInductionRecipe(PHINode *IV, VPValue *Start, VPValue *Step,
1808 VPValue *getBackedgeValue() override {
1823 VPValue *getStepValue() { return getOperand(1); }
1824 const VPValue *getStepValue() const { return getOperand(1); }
1855 VPWidenPointerInductionRecipe(PHINode *Phi, VPValue *Start, VPValue *Step,
1900 VPWidenPHIRecipe(PHINode *Phi, VPValue *Start = nullptr)
1901 : VPSingleDefRecipe(VPDef::VPWidenPHISC, ArrayRef<VPValue *>(), Phi) {
1924 void addIncoming(VPValue *IncomingV, VPBasicBlock *IncomingBlock) {
1932 /// Returns the \p I th incoming VPValue.
1933 VPValue *getIncomingValue(unsigned I) { return getOperand(I); }
1940 VPFirstOrderRecurrencePHIRecipe(PHINode *Phi, VPValue &Start)
1980 VPValue &Start, bool IsInLoop = false,
2030 VPBlendRecipe(PHINode *Phi, ArrayRef<VPValue *> Operands)
2037 SmallVector<VPValue *> Ops(operands());
2048 VPValue *getIncomingValue(unsigned Idx) const {
2053 VPValue *getMask(unsigned Idx) const {
2068 bool onlyFirstLaneUsed(const VPValue *Op) const override {
2094 VPInterleaveRecipe(const InterleaveGroup<Instruction> *IG, VPValue *Addr,
2095 ArrayRef<VPValue *> StoredValues, VPValue *Mask,
2103 new VPValue(I, this);
2123 VPValue *getAddr() const {
2129 VPValue *getMask() const {
2136 ArrayRef<VPValue *> getStoredValues() const {
2139 return ArrayRef<VPValue *>(op_begin(), getNumOperands())
2161 bool onlyFirstLaneUsed(const VPValue *Op) const override {
2182 Instruction *I, ArrayRef<VPValue *> Operands,
2183 VPValue *CondOp, bool IsOrdered)
2193 VPValue *ChainOp, VPValue *VecOp, VPValue *CondOp,
2196 ArrayRef<VPValue *>({ChainOp, VecOp}), CondOp,
2233 /// The VPValue of the scalar Chain being accumulated.
2234 VPValue *getChainOp() const { return getOperand(0); }
2235 /// The VPValue of the vector value to be reduced.
2236 VPValue *getVecOp() const { return getOperand(1); }
2237 /// The VPValue of the condition for the block.
2238 VPValue *getCondOp() const {
2249 VPReductionEVLRecipe(VPReductionRecipe *R, VPValue *EVL, VPValue *CondOp)
2253 ArrayRef<VPValue *>({R->getChainOp(), R->getVecOp(), EVL}), CondOp,
2273 /// The VPValue of the explicit vector length.
2274 VPValue *getEVL() const { return getOperand(2); }
2277 bool onlyFirstLaneUsed(const VPValue *Op) const override {
2298 bool IsUniform, VPValue *Mask = nullptr)
2333 bool onlyFirstLaneUsed(const VPValue *Op) const override {
2340 bool usesScalars(const VPValue *Op) const override {
2352 VPValue *getMask() {
2363 VPBranchOnMaskRecipe(VPValue *BlockInMask)
2384 if (VPValue *Mask = getMask())
2393 VPValue *getMask() const {
2400 bool usesScalars(const VPValue *Op) const override {
2416 VPPredInstPHIRecipe(VPValue *PredV)
2436 bool usesScalars(const VPValue *Op) const override {
2458 void setMask(VPValue *Mask) {
2467 std::initializer_list<VPValue *> Operands,
2499 VPValue *getAddr() const { return getOperand(0); }
2506 VPValue *getMask() const {
2521 struct VPWidenLoadRecipe final : public VPWidenMemoryRecipe, public VPValue {
2522 VPWidenLoadRecipe(LoadInst &Load, VPValue *Addr, VPValue *Mask,
2526 VPValue(this, &Load) {
2548 bool onlyFirstLaneUsed(const VPValue *Op) const override {
2560 struct VPWidenLoadEVLRecipe final : public VPWidenMemoryRecipe, public VPValue {
2561 VPWidenLoadEVLRecipe(VPWidenLoadRecipe *L, VPValue *EVL, VPValue *Mask)
2565 VPValue(this, &getIngredient()) {
2572 VPValue *getEVL() const { return getOperand(1); }
2584 bool onlyFirstLaneUsed(const VPValue *Op) const override {
2596 VPWidenStoreRecipe(StoreInst &Store, VPValue *Addr, VPValue *StoredVal,
2597 VPValue *Mask, bool Consecutive, bool Reverse, DebugLoc DL)
2612 VPValue *getStoredValue() const { return getOperand(1); }
2624 bool onlyFirstLaneUsed(const VPValue *Op) const override {
2637 VPWidenStoreEVLRecipe(VPWidenStoreRecipe *S, VPValue *EVL, VPValue *Mask)
2648 VPValue *getStoredValue() const { return getOperand(1); }
2651 VPValue *getEVL() const { return getOperand(2); }
2663 bool onlyFirstLaneUsed(const VPValue *Op) const override {
2712 VPCanonicalIVPHIRecipe(VPValue *StartV, DebugLoc DL)
2744 bool onlyFirstLaneUsed(const VPValue *Op) const override {
2751 bool onlyFirstPartUsed(const VPValue *Op) const override {
2759 bool isCanonical(InductionDescriptor::InductionKind Kind, VPValue *Start,
2760 VPValue *Step) const;
2769 VPActiveLaneMaskPHIRecipe(VPValue *StartMask, DebugLoc DL)
2801 VPEVLBasedIVPHIRecipe(VPValue *StartIV, DebugLoc DL)
2821 bool onlyFirstLaneUsed(const VPValue *Op) const override {
2872 VPDerivedIVRecipe(const InductionDescriptor &IndDesc, VPValue *Start,
2873 VPCanonicalIVPHIRecipe *CanonicalIV, VPValue *Step)
2880 const FPMathOperator *FPBinOp, VPValue *Start, VPValue *IV,
2881 VPValue *Step)
2908 VPValue *getStartValue() const { return getOperand(0); }
2909 VPValue *getStepValue() const { return getOperand(2); }
2912 bool onlyFirstLaneUsed(const VPValue *Op) const override {
2925 VPScalarIVStepsRecipe(VPValue *IV, VPValue *Step,
2928 ArrayRef<VPValue *>({IV, Step}), FMFs),
2931 VPScalarIVStepsRecipe(const InductionDescriptor &IndDesc, VPValue *IV,
2932 VPValue *Step)
2958 VPValue *getStepValue() const { return getOperand(1); }
2961 bool onlyFirstLaneUsed(const VPValue *Op) const override {
3060 void dropAllReferences(VPValue *NewValue) override;
3071 /// SlotTracker is used to print unnamed VPValue's using consequtive numbers.
3177 VPValue DummyValue;
3229 void dropAllReferences(VPValue *NewValue) override;
3233 /// \p Indent. \p SlotTracker is used to print unnamed VPValue's using
3278 VPValue *TripCount = nullptr;
3282 VPValue *BackedgeTakenCount = nullptr;
3285 VPValue VectorTripCount;
3288 VPValue VFxUF;
3290 /// Holds a mapping between Values and their corresponding VPValue inside
3296 SmallVector<VPValue *, 16> VPLiveInsToFree;
3306 DenseMap<const SCEV *, VPValue *> SCEVToExpansion;
3313 VPlan(VPBasicBlock *Preheader, VPValue *TC, VPBasicBlock *Entry)
3359 VPValue *getTripCount() const {
3366 void resetTripCount(VPValue *NewTripCount) {
3373 VPValue *getOrCreateBackedgeTakenCount() {
3375 BackedgeTakenCount = new VPValue();
3380 VPValue &getVectorTripCount() { return VectorTripCount; }
3383 VPValue &getVFxUF() { return VFxUF; }
3419 /// Gets the live-in VPValue for \p V or adds a new live-in (if none exists
3421 VPValue *getOrAddLiveIn(Value *V) {
3422 assert(V && "Trying to get or add the VPValue of a null Value");
3424 VPValue *VPV = new VPValue(V);
3437 /// Return the live-in VPValue for \p V, if there is one or nullptr otherwise.
3438 VPValue *getLiveIn(Value *V) const { return Value2VPValue.lookup(V); }
3472 void addLiveOut(PHINode *PN, VPValue *V);
3483 VPValue *getSCEVExpansion(const SCEV *S) const {
3487 void addSCEVExpansion(const SCEV *S, VPValue *V) {
3701 /// A DenseMapInfo implementation for using SmallVector<VPValue *, 4> as
3704 static SmallVector<VPValue *, 4> getEmptyKey() {
3705 return {reinterpret_cast<VPValue *>(-1)};
3708 static SmallVector<VPValue *, 4> getTombstoneKey() {
3709 return {reinterpret_cast<VPValue *>(-2)};
3712 static unsigned getHashValue(const SmallVector<VPValue *, 4> &V) {
3716 static bool isEqual(const SmallVector<VPValue *, 4> &LHS,
3717 const SmallVector<VPValue *, 4> &RHS) {
3723 DenseMap<SmallVector<VPValue *, 4>, VPInstruction *, BundleDenseMapInfo>
3739 typename std::pair<VPInstruction *, SmallVector<VPValue *, 4>>;
3751 bool areVectorizable(ArrayRef<VPValue *> Operands) const;
3754 void addCombined(ArrayRef<VPValue *> Operands, VPInstruction *New);
3766 std::pair<OpMode, VPValue *> getBest(OpMode Mode, VPValue *Last,
3767 SmallPtrSetImpl<VPValue *> &Candidates,
3772 void dumpBundle(ArrayRef<VPValue *> Values);
3782 VPInstruction *buildGraph(ArrayRef<VPValue *> Operands);
3794 bool onlyFirstLaneUsed(const VPValue *Def);
3797 bool onlyFirstPartUsed(const VPValue *Def);
3799 /// Get or create a VPValue that corresponds to the expansion of \p Expr. If \p
3800 /// Expr is a SCEVConstant or SCEVUnknown, return a VPValue wrapping the live-in
3804 VPValue *getOrCreateVPValueForSCEVExpr(VPlan &Plan, const SCEV *Expr,
3808 inline bool isUniformAfterVectorization(VPValue *VPV) {
3825 bool isHeaderMask(VPValue *V, VPlan &Plan);