Lines Matching defs:VPValue
16 /// also inherit from VPValue.
250 DenseMap<VPValue *, Value *> VPV2Vector;
252 DenseMap<VPValue *, SmallVector<Value *, 4>> VPV2Scalars;
255 /// Get the generated vector Value for a given VPValue \p Def if \p IsScalar
257 Value *get(VPValue *Def, bool IsScalar = false);
259 /// Get the generated Value for a given VPValue and given Part and Lane.
260 Value *get(VPValue *Def, const VPLane &Lane);
262 bool hasVectorValue(VPValue *Def) { return Data.VPV2Vector.contains(Def); }
264 bool hasScalarValue(VPValue *Def, VPLane Lane) {
272 /// Set the generated vector Value for a given VPValue, if \p
274 void set(VPValue *Def, Value *V, bool IsScalar = false) {
285 void reset(VPValue *Def, Value *V) {
291 void set(VPValue *Def, Value *V, const VPLane &Lane) {
301 void reset(VPValue *Def, Value *V, const VPLane &Lane) {
329 void packScalarIntoVectorValue(VPValue *Def, const VPLane &Lane);
648 /// with \p Indent. \p SlotTracker is used to print unnamed VPValue's using
701 TargetTransformInfo::OperandValueInfo getOperandInfo(VPValue *V) const;
708 /// VPRecipeBase and VPValue separately.
722 VPRecipeBase(const unsigned char SC, ArrayRef<VPValue *> Operands,
820 static inline bool classof(const VPValue *V) { \
836 /// more output IR that define a single result VPValue.
837 /// Note that VPRecipeBase must be inherited from before VPValue.
838 class VPSingleDefRecipe : public VPRecipeBase, public VPValue {
842 : VPRecipeBase(SC, Operands, DL), VPValue(this) {}
844 VPSingleDefRecipe(const unsigned char SC, ArrayRef<VPValue *> Operands,
846 : VPRecipeBase(SC, Operands, DL), VPValue(this) {}
851 : VPRecipeBase(SC, Operands, DL), VPValue(this, UV) {}
1176 /// Return the VPValue operand containing the unroll part or null if there is
1178 VPValue *getUnrollPartOperand(VPUser &U) const;
1214 // Takes the VPValue to extract from as first operand and the lane or part
1264 VPInstruction(unsigned Opcode, ArrayRef<VPValue *> Operands, DebugLoc DL,
1269 VPInstruction(unsigned Opcode, std::initializer_list<VPValue *> Operands,
1271 : VPInstruction(Opcode, ArrayRef<VPValue *>(Operands), DL, Name) {}
1273 VPInstruction(unsigned Opcode, CmpInst::Predicate Pred, VPValue *A,
1274 VPValue *B, DebugLoc DL = {}, const Twine &Name = "");
1276 VPInstruction(unsigned Opcode, std::initializer_list<VPValue *> Operands,
1281 VPInstruction(unsigned Opcode, std::initializer_list<VPValue *> Operands,
1289 VPInstruction(VPValue *Ptr, VPValue *Offset, GEPNoWrapFlags Flags,
1292 ArrayRef<VPValue *>({Ptr, Offset}), Flags, DL),
1295 VPInstruction(unsigned Opcode, std::initializer_list<VPValue *> Operands,
1301 SmallVector<VPValue *, 2> Operands(operands());
1356 bool onlyFirstLaneUsed(const VPValue *Op) const override;
1359 bool onlyFirstPartUsed(const VPValue *Op) const override;
1374 /// execution, execept for PHIs. For PHIs, a single VPValue operand is allowed,
1382 : VPRecipeBase(VPDef::VPIRInstructionSC, ArrayRef<VPValue *>()), I(I) {}
1409 bool usesScalars(const VPValue *Op) const override {
1415 bool onlyFirstPartUsed(const VPValue *Op) const override {
1421 bool onlyFirstLaneUsed(const VPValue *Op) const override {
1493 VPWidenEVLRecipe(Instruction &I, iterator_range<IterT> Operands, VPValue &EVL)
1497 VPWidenEVLRecipe(VPWidenRecipe &W, VPValue &EVL)
1511 VPValue *getEVL() { return getOperand(getNumOperands() - 1); }
1512 const VPValue *getEVL() const { return getOperand(getNumOperands() - 1); }
1519 bool onlyFirstLaneUsed(const VPValue *Op) const override {
1523 // the VPValue should be vectorized.
1543 VPWidenCastRecipe(Instruction::CastOps Opcode, VPValue *Op, Type *ResultTy,
1551 VPWidenCastRecipe(Instruction::CastOps Opcode, VPValue *Op, Type *ResultTy)
1595 VPScalarCastRecipe(Instruction::CastOps Opcode, VPValue *Op, Type *ResultTy,
1626 bool onlyFirstLaneUsed(const VPValue *Op) const override {
1653 ArrayRef<VPValue *> CallArguments, Type *Ty,
1662 ArrayRef<VPValue *> CallArguments, Type *Ty,
1677 std::initializer_list<VPValue *> CallArguments,
1680 ArrayRef<VPValue *>(CallArguments), Ty, DL) {}
1723 bool onlyFirstLaneUsed(const VPValue *Op) const override;
1735 ArrayRef<VPValue *> CallArguments, DebugLoc DL = {})
1812 VPValue *getMask() const {
1851 VPValue *getCond() const {
1871 return all_of(operands(), [](VPValue *Op) {
1914 VPReverseVectorPointerRecipe(VPValue *Ptr, VPValue *VF, Type *IndexedTy,
1917 ArrayRef<VPValue *>({Ptr, VF}), GEPFlags, DL),
1922 VPValue *getVFValue() { return getOperand(1); }
1923 const VPValue *getVFValue() const { return getOperand(1); }
1927 bool onlyFirstLaneUsed(const VPValue *Op) const override {
1941 bool onlyFirstPartUsed(const VPValue *Op) const override {
1967 VPVectorPointerRecipe(VPValue *Ptr, Type *IndexedTy, GEPNoWrapFlags GEPFlags,
1969 : VPRecipeWithIRFlags(VPDef::VPVectorPointerSC, ArrayRef<VPValue *>(Ptr),
1977 bool onlyFirstLaneUsed(const VPValue *Op) const override {
1984 bool onlyFirstPartUsed(const VPValue *Op) const override {
2036 VPValue *Start = nullptr, DebugLoc DL = {})
2037 : VPSingleDefRecipe(VPDefID, ArrayRef<VPValue *>(), UnderlyingInstr, DL) {
2050 static inline bool classof(const VPValue *V) {
2070 VPValue *getStartValue() {
2073 VPValue *getStartValue() const {
2078 void setStartValue(VPValue *V) { setOperand(0, V); }
2081 virtual VPValue *getBackedgeValue() {
2099 VPWidenInductionRecipe(unsigned char Kind, PHINode *IV, VPValue *Start,
2100 VPValue *Step, const InductionDescriptor &IndDesc,
2111 static inline bool classof(const VPValue *V) {
2123 VPValue *getStepValue() { return getOperand(1); }
2124 const VPValue *getStepValue() const { return getOperand(1); }
2131 VPValue *getBackedgeValue() override {
2152 VPWidenIntOrFpInductionRecipe(PHINode *IV, VPValue *Start, VPValue *Step,
2153 VPValue *VF, const InductionDescriptor &IndDesc,
2161 VPWidenIntOrFpInductionRecipe(PHINode *IV, VPValue *Start, VPValue *Step,
2162 VPValue *VF, const InductionDescriptor &IndDesc,
2190 VPValue *getVFValue() { return getOperand(2); }
2191 const VPValue *getVFValue() const { return getOperand(2); }
2193 VPValue *getSplatVFValue() {
2194 // If the recipe has been unrolled (4 operands), return the VPValue for the
2214 /// Returns the VPValue representing the value of this induction at
2217 VPValue *getLastUnrolledPartOperand() {
2229 VPWidenPointerInductionRecipe(PHINode *Phi, VPValue *Start, VPValue *Step,
2252 /// Returns the VPValue representing the value of this induction at
2255 VPValue *getFirstUnrolledPartOperand() {
2273 VPScalarPHIRecipe(VPValue *Start, VPValue *BackedgeValue, DebugLoc DL,
2292 bool onlyFirstLaneUsed(const VPValue *Op) const override {
2315 VPWidenPHIRecipe(PHINode *Phi, VPValue *Start = nullptr, DebugLoc DL = {})
2316 : VPSingleDefRecipe(VPDef::VPWidenPHISC, ArrayRef<VPValue *>(), Phi, DL) {
2339 void addIncoming(VPValue *IncomingV, VPBasicBlock *IncomingBlock) {
2347 /// Returns the \p I th incoming VPValue.
2348 VPValue *getIncomingValue(unsigned I) { return getOperand(I); }
2355 VPFirstOrderRecurrencePHIRecipe(PHINode *Phi, VPValue &Start)
2404 VPValue &Start, bool IsInLoop = false,
2456 VPPartialReductionRecipe(Instruction *ReductionInst, VPValue *Op0,
2457 VPValue *Op1)
2460 VPPartialReductionRecipe(unsigned Opcode, VPValue *Op0, VPValue *Op1,
2463 ArrayRef<VPValue *>({Op0, Op1}), ReductionInst),
2505 VPBlendRecipe(PHINode *Phi, ArrayRef<VPValue *> Operands)
2511 SmallVector<VPValue *> Ops(operands());
2528 VPValue *getIncomingValue(unsigned Idx) const {
2533 VPValue *getMask(unsigned Idx) const {
2552 bool onlyFirstLaneUsed(const VPValue *Op) const override {
2578 VPInterleaveRecipe(const InterleaveGroup<Instruction> *IG, VPValue *Addr,
2579 ArrayRef<VPValue *> StoredValues, VPValue *Mask,
2587 new VPValue(I, this);
2607 VPValue *getAddr() const {
2613 VPValue *getMask() const {
2620 ArrayRef<VPValue *> getStoredValues() const {
2623 return ArrayRef<VPValue *>(op_begin(), getNumOperands())
2649 bool onlyFirstLaneUsed(const VPValue *Op) const override {
2670 Instruction *I, ArrayRef<VPValue *> Operands,
2671 VPValue *CondOp, bool IsOrdered, DebugLoc DL)
2682 VPValue *ChainOp, VPValue *VecOp, VPValue *CondOp,
2685 ArrayRef<VPValue *>({ChainOp, VecOp}), CondOp,
2727 /// The VPValue of the scalar Chain being accumulated.
2728 VPValue *getChainOp() const { return getOperand(0); }
2729 /// The VPValue of the vector value to be reduced.
2730 VPValue *getVecOp() const { return getOperand(1); }
2731 /// The VPValue of the condition for the block.
2732 VPValue *getCondOp() const {
2743 VPReductionEVLRecipe(VPReductionRecipe &R, VPValue &EVL, VPValue *CondOp)
2747 ArrayRef<VPValue *>({R.getChainOp(), R.getVecOp(), &EVL}), CondOp,
2767 /// The VPValue of the explicit vector length.
2768 VPValue *getEVL() const { return getOperand(2); }
2771 bool onlyFirstLaneUsed(const VPValue *Op) const override {
2792 bool IsUniform, VPValue *Mask = nullptr)
2831 bool onlyFirstLaneUsed(const VPValue *Op) const override {
2838 bool usesScalars(const VPValue *Op) const override {
2850 VPValue *getMask() {
2861 VPBranchOnMaskRecipe(VPValue *BlockInMask)
2886 if (VPValue *Mask = getMask())
2895 VPValue *getMask() const {
2902 bool usesScalars(const VPValue *Op) const override {
2918 VPPredInstPHIRecipe(VPValue *PredV, DebugLoc DL)
2946 bool usesScalars(const VPValue *Op) const override {
2968 void setMask(VPValue *Mask) {
2977 std::initializer_list<VPValue *> Operands,
3009 VPValue *getAddr() const { return getOperand(0); }
3016 VPValue *getMask() const {
3035 struct VPWidenLoadRecipe final : public VPWidenMemoryRecipe, public VPValue {
3036 VPWidenLoadRecipe(LoadInst &Load, VPValue *Addr, VPValue *Mask,
3040 VPValue(this, &Load) {
3062 bool onlyFirstLaneUsed(const VPValue *Op) const override {
3074 struct VPWidenLoadEVLRecipe final : public VPWidenMemoryRecipe, public VPValue {
3075 VPWidenLoadEVLRecipe(VPWidenLoadRecipe &L, VPValue &EVL, VPValue *Mask)
3079 VPValue(this, &getIngredient()) {
3086 VPValue *getEVL() const { return getOperand(1); }
3102 bool onlyFirstLaneUsed(const VPValue *Op) const override {
3114 VPWidenStoreRecipe(StoreInst &Store, VPValue *Addr, VPValue *StoredVal,
3115 VPValue *Mask, bool Consecutive, bool Reverse, DebugLoc DL)
3130 VPValue *getStoredValue() const { return getOperand(1); }
3142 bool onlyFirstLaneUsed(const VPValue *Op) const override {
3155 VPWidenStoreEVLRecipe(VPWidenStoreRecipe &S, VPValue &EVL, VPValue *Mask)
3165 VPValue *getStoredValue() const { return getOperand(1); }
3168 VPValue *getEVL() const { return getOperand(2); }
3184 bool onlyFirstLaneUsed(const VPValue *Op) const override {
3240 VPCanonicalIVPHIRecipe(VPValue *StartV, DebugLoc DL)
3274 bool onlyFirstLaneUsed(const VPValue *Op) const override {
3281 bool onlyFirstPartUsed(const VPValue *Op) const override {
3301 VPActiveLaneMaskPHIRecipe(VPValue *StartMask, DebugLoc DL)
3336 VPEVLBasedIVPHIRecipe(VPValue *StartIV, DebugLoc DL)
3364 bool onlyFirstLaneUsed(const VPValue *Op) const override {
3426 VPDerivedIVRecipe(const InductionDescriptor &IndDesc, VPValue *Start,
3427 VPCanonicalIVPHIRecipe *CanonicalIV, VPValue *Step,
3435 const FPMathOperator *FPBinOp, VPValue *Start, VPValue *IV,
3436 VPValue *Step, const Twine &Name = "")
3470 VPValue *getStartValue() const { return getOperand(0); }
3471 VPValue *getStepValue() const { return getOperand(2); }
3474 bool onlyFirstLaneUsed(const VPValue *Op) const override {
3488 VPScalarIVStepsRecipe(VPValue *IV, VPValue *Step,
3491 ArrayRef<VPValue *>({IV, Step}), FMFs),
3494 VPScalarIVStepsRecipe(const InductionDescriptor &IndDesc, VPValue *IV,
3495 VPValue *Step)
3528 VPValue *getStepValue() const { return getOperand(1); }
3531 bool onlyFirstLaneUsed(const VPValue *Op) const override {
3643 /// SlotTracker is used to print unnamed VPValue's using consequtive numbers.
3797 /// \p Indent. \p SlotTracker is used to print unnamed VPValue's using
3843 VPValue *TripCount = nullptr;
3847 VPValue *BackedgeTakenCount = nullptr;
3850 VPValue VectorTripCount;
3853 VPValue VF;
3856 VPValue VFxUF;
3858 /// Holds a mapping between Values and their corresponding VPValue inside
3864 SmallVector<VPValue *, 16> VPLiveInsToFree;
3869 DenseMap<const SCEV *, VPValue *> SCEVToExpansion;
3892 VPlan(BasicBlock *ScalarHeaderBB, VPValue *TC) {
3971 VPValue *getTripCount() const {
3978 void resetTripCount(VPValue *NewTripCount) {
3985 VPValue *getOrCreateBackedgeTakenCount() {
3987 BackedgeTakenCount = new VPValue();
3992 VPValue &getVectorTripCount() { return VectorTripCount; }
3995 VPValue &getVF() { return VF; };
3998 VPValue &getVFxUF() { return VFxUF; }
4039 /// Gets the live-in VPValue for \p V or adds a new live-in (if none exists
4041 VPValue *getOrAddLiveIn(Value *V) {
4042 assert(V && "Trying to get or add the VPValue of a null Value");
4044 VPValue *VPV = new VPValue(V);
4057 /// Return the live-in VPValue for \p V, if there is one or nullptr otherwise.
4058 VPValue *getLiveIn(Value *V) const { return Value2VPValue.lookup(V); }
4084 VPValue *getSCEVExpansion(const SCEV *S) const {
4088 void addSCEVExpansion(const SCEV *S, VPValue *V) {
4254 /// A DenseMapInfo implementation for using SmallVector<VPValue *, 4> as
4257 static SmallVector<VPValue *, 4> getEmptyKey() {
4258 return {reinterpret_cast<VPValue *>(-1)};
4261 static SmallVector<VPValue *, 4> getTombstoneKey() {
4262 return {reinterpret_cast<VPValue *>(-2)};
4265 static unsigned getHashValue(const SmallVector<VPValue *, 4> &V) {
4269 static bool isEqual(const SmallVector<VPValue *, 4> &LHS,
4270 const SmallVector<VPValue *, 4> &RHS) {
4276 DenseMap<SmallVector<VPValue *, 4>, VPInstruction *, BundleDenseMapInfo>
4292 typename std::pair<VPInstruction *, SmallVector<VPValue *, 4>>;
4304 bool areVectorizable(ArrayRef<VPValue *> Operands) const;
4307 void addCombined(ArrayRef<VPValue *> Operands, VPInstruction *New);
4319 std::pair<OpMode, VPValue *> getBest(OpMode Mode, VPValue *Last,
4320 SmallPtrSetImpl<VPValue *> &Candidates,
4325 void dumpBundle(ArrayRef<VPValue *> Values);
4335 VPInstruction *buildGraph(ArrayRef<VPValue *> Operands);