Lines Matching defs:VF
263 static FixedVectorType *getWidenedType(Type *ScalarTy, unsigned VF) {
265 VF * getNumElements(ScalarTy));
603 ///< indices are in range [0:VF)).
606 ///< are in range [VF:2*VF))
616 static SmallBitVector buildUseMask(int VF, ArrayRef<int> Mask,
618 SmallBitVector UseMask(VF, true);
625 if (MaskArg == UseMask::FirstArg && Value < VF)
627 else if (MaskArg == UseMask::SecondArg && Value >= VF)
628 UseMask.reset(Value - VF);
1077 Mappings.front().Shape.VF != BaseMappings.front().Shape.VF ||
1545 /// parts of subgraph with smaller VF and they are reordered independently. We
1546 /// can make it because we still need to extend smaller nodes to the wider VF
4687 const int VF = GetVF(I);
4688 if (VF == 0)
4711 if (Idx < VF) {
4755 unsigned VF = 0;
4771 VF = std::max(VF, cast<VectorType>(EI->getVectorOperandType())
4775 return VF;
5230 for (unsigned VF =
5232 VF >= MinVF;
5233 VF = getFloorFullVectorNumberOfElements(TTI, ScalarTy, VF - 1)) {
5235 for (unsigned Cnt = 0, End = VL.size(); Cnt + VF <= End; Cnt += VF) {
5236 ArrayRef<Value *> Slice = VL.slice(Cnt, VF);
5248 DemandedElts.setBits(Cnt, Cnt + VF);
5259 // All loads gathered - try smaller VF.
5274 auto *SubVecTy = getWidenedType(ScalarTy, VF);
5276 auto *LI0 = cast<LoadInst>(VL[I * VF]);
5280 : getGEPCosts(TTI, ArrayRef(PointerOps).slice(I * VF, VF),
5294 SubVecTy, APInt::getAllOnes(VF),
5299 SubVecTy, APInt::getOneBitSet(ScalarTyNumElements * VF, 0),
5332 ShuffleMask[Idx] = Idx / VF == I ? VL.size() + Idx % VF : Idx;
5336 CostKind, I * VF, SubVecTy);
5344 *BestVF = VF;
5607 unsigned VF = ReorderMask.size();
5608 OrdersType ResOrder(VF, VF);
5609 unsigned NumParts = divideCeil(VF, Sz);
5611 for (unsigned I = 0; I < VF; I += Sz) {
5614 unsigned Limit = std::min(Sz, VF - I);
5629 if (Idx < VF)
5635 unsigned VF = TE.getVectorFactor();
5653 for (unsigned I = 0; I < VF; ++I) {
5662 // Build the order of the VF size, need to reorder reuses shuffles, they are
5663 // always of VF size.
5664 OrdersType ResOrder(VF);
5667 for (unsigned K = 0; K < VF; K += Sz) {
5947 // Maps VF to the graph nodes.
5963 // Find all reorderable nodes with the given VF.
6028 for (unsigned VF = VectorizableTree.front()->getVectorFactor();
6029 !VFToOrderedEntries.empty() && VF > 1; VF -= 2 - (VF & 1U)) {
6030 auto It = VFToOrderedEntries.find(VF);
6037 // Delete VF entry upon exit.
6114 OrdersType IdentityOrder(VF, VF);
6153 // Just do the reordering for the nodes with the given VF.
6154 if (TE->Scalars.size() != VF) {
6155 if (TE->ReuseShuffleIndices.size() == VF) {
6156 // Need to reorder the reuses masks of the operands with smaller VF to
6160 [VF, &TE](const EdgeInfo &EI) {
6161 return EI.UserTE->Scalars.size() == VF ||
6165 "All users must be of VF size.");
6183 // Update ordering of the operands with the smaller VF than the given
6281 // Find all reorderable leaf nodes with the given VF.
6443 unsigned VF = Data.second.front().second->getVectorFactor();
6444 OrdersType IdentityOrder(VF, VF);
7214 // Select maximum VF as a maximum of user gathered nodes and
7274 unsigned VF = *CommonVF;
7280 getWidenedType(Slice.front()->getType(), VF),
7287 UserMaxVF = InterleaveFactor * VF;
7330 for (unsigned VF = MaxVF; VF >= 2; VF /= 2) {
7332 for (unsigned I = 0, E = Slice.size(); I < E; I += VF) {
7334 Slice.slice(I, std::min(VF, E - I));
7354 VF = 2 * (MaxVF / InterleaveFactor);
9157 /// When REVEC is disabled, there is no difference between VF and
9159 /// When REVEC is enabled, VF is VNumElements / ScalarTyNumElements.
9183 int VF = VecTy->getNumElements();
9185 if (VF == Limit && ShuffleVectorInst::isIdentityMask(Mask, Limit))
9189 if (ShuffleVectorInst::isExtractSubvectorMask(Mask, VF, Index) &&
9192 // All VF-size submasks are identity (e.g.
9193 // <poison,poison,poison,poison,0,1,2,poison,poison,1,2,3> etc. for VF 4).
9194 if (Limit % VF == 0 && all_of(seq<int>(0, Limit / VF), [=](int Idx) {
9195 ArrayRef<int> Slice = Mask.slice(Idx * VF, VF);
9197 ShuffleVectorInst::isIdentityMask(Slice, VF);
9210 unsigned VF = Mask.size();
9215 int MaskedIdx = Mask[ExtMask[I] % VF];
9382 int VF = Mask.size();
9384 VF = FTy->getNumElements();
9386 V2, buildUseMask(VF, Mask, UseMask::SecondArg))
9391 int VF =
9396 if (Mask[I] < VF)
9399 CombinedMask2[I] = Mask[I] - VF;
9455 VF = std::max(cast<VectorType>(Op1->getType())
9465 CombinedMask1[I] = CombinedMask2[I] + (Op1 == Op2 ? 0 : VF);
9469 (ShuffleVectorInst::isIdentityMask(CombinedMask1, VF) ||
9470 (ShuffleVectorInst::isZeroEltSplatMask(CombinedMask1, VF) &&
9818 for (unsigned VF = getFloorFullVectorNumberOfElements(
9820 VF >= MinVF; VF = getFloorFullVectorNumberOfElements(
9821 *TTI, VL.front()->getType(), VF - 1)) {
9822 if (StartIdx + VF > End)
9825 for (unsigned Cnt = StartIdx; Cnt + VF <= End; Cnt += VF) {
9826 ArrayRef<Value *> Slice = VL.slice(Cnt, VF);
9839 if (IsSplat && VF == 2) {
9841 *TTI, getWidenedType(Slice.front()->getType(), 2 * VF));
9846 static_cast<long>(isa<UndefValue>(Slice.front()) ? VF - 1
9855 !hasFullVectorsOrPowerOf2(*TTI, Slice.front()->getType(), VF)))
9857 if (VF == 2) {
9930 if (UserIgnoreList && E.Idx == 0 && VF == 2)
10055 // Small graph with small VF - exit.
10356 unsigned VF = E1.getVectorFactor();
10358 VF = std::max(VF,
10362 VF = std::max(VF, E->getVectorFactor());
10366 CommonMask[Idx] = Mask[Idx] + VF;
10372 unsigned VF = Mask.size();
10374 VF = std::max(VF,
10378 VF = std::max(VF, E->getVectorFactor());
10382 CommonMask[Idx] = Idx + (InVectors.empty() ? 0 : VF);
10391 static bool isEmptyOrIdentity(ArrayRef<int> Mask, unsigned VF) {
10394 (VF == Mask.size() &&
10395 ShuffleVectorInst::isIdentityMask(Mask, VF)) ||
10396 (ShuffleVectorInst::isExtractSubvectorMask(Mask, VF, Index) &&
10406 unsigned VF =
10408 if (isEmptyOrIdentity(Mask, VF))
10415 unsigned VF =
10417 if (isEmptyOrIdentity(Mask, VF))
10423 InstructionCost createPoison(Type *Ty, unsigned VF) const {
10442 unsigned VF) -> InstructionCost {
10457 return TTI.getCastInstrCost(CastOpcode, getWidenedType(ScalarTy, VF),
10458 getWidenedType(EScalarTy, VF),
10484 unsigned VF = E->getVectorFactor();
10486 CommonVF = std::max(VF, E2->getVectorFactor());
10518 unsigned VF = E->getVectorFactor();
10519 CommonVF = VF;
10524 if (E->Scalars.size() == Mask.size() && VF != Mask.size()) {
10565 unsigned VF = getVF(V1);
10567 CommonVF = std::max(VF, E2->getVectorFactor());
10573 if (E2->Scalars.size() == VF && VF != CommonVF) {
10580 Idx = E2Mask[Idx - CommonVF] + VF;
10582 CommonVF = VF;
10591 unsigned VF = getVF(V2);
10593 CommonVF = std::max(VF, E1->getVectorFactor());
10599 if (E1->Scalars.size() == VF && VF != CommonVF) {
10606 Idx = E1Mask[Idx - CommonVF] + VF;
10610 CommonVF = VF;
10619 unsigned VF = getVF(V1);
10620 CommonVF = std::max(VF, getVF(V2));
10852 unsigned VF = getVF(V1);
10856 VF = std::max<unsigned>(VF, CommonMask.size());
10859 VF = std::max(VF, InTE->getVectorFactor());
10861 VF = std::max(
10862 VF, cast<FixedVectorType>(cast<Value *>(InVectors.front())->getType())
10868 CommonMask[Idx] = Mask[Idx] + VF;
10876 unsigned VF = VL.size();
10878 VF = std::min(VF, MaskVF);
10879 for (Value *V : VL.take_front(VF)) {
10891 SmallVector<Constant *> NewVals(VF * VecTyNumElements, nullptr);
10918 ArrayRef<int> SubVectorsMask, unsigned VF = 0,
10928 assert(VF > 0 &&
11052 const unsigned VF, unsigned MinBW,
11063 getWidenedType(IntegerType::get(CI->getContext(), MinBW), VF));
11067 ArgTys.push_back(getWidenedType(Arg->getType(), VF));
12328 /// have common VF and perform action for 2 input vectors (including non-undef
12354 for (unsigned Idx = 0, VF = Mask.size(); Idx < VF; ++Idx) {
12358 Mask[Idx] = (Res.second ? Idx : Mask[Idx]) + VF;
12362 "Expected base vector of VF number of elements.");
12383 for (unsigned I = 0, VF = Mask.size(); I < VF; ++I) {
12397 for (unsigned I = 0, VF = Mask.size(); I < VF; ++I) {
12404 Mask[I] = (Res2.second ? I : SecMask[I]) + VF;
12418 for (unsigned I = 0, VF = Mask.size(); I < VF; ++I) {
12422 Mask[I] = (Res.second ? I : SecMask[I]) + VF;
12787 unsigned VF = Mask.size();
12789 if (VF != VecVF &&
12790 (any_of(Mask, [VF](int Idx) { return Idx >= static_cast<int>(VF); }) ||
12791 !ShuffleVectorInst::isIdentityMask(Mask, VF))) {
12793 std::copy(Mask.begin(), std::next(Mask.begin(), std::min(VF, VecVF)),
12811 unsigned VF = 0;
12817 if (VF == 0)
12818 VF = TEs.front()->getVectorFactor();
12819 auto *FTy = getWidenedType(TEs.back()->Scalars.front()->getType(), VF);
12820 if (!ShuffleVectorInst::isIdentityMask(Mask, VF) &&
12823 (Data.index() < VF &&
12836 if (VF == 0) {
12839 VF = TEs.front()->getVectorFactor();
12841 VF = Mask.size();
12843 auto *FTy = getWidenedType(TEs.back()->Scalars.front()->getType(), VF);
12853 VF = Mask.size();
13240 unsigned VF = 0;
13274 VF = FirstEntries.front()->getVectorFactor();
13281 unsigned VF = TE->getVectorFactor();
13282 auto It = VFToTE.find(VF);
13288 VFToTE.try_emplace(VF, TE);
13299 VF = It->first;
13313 VF = std::max(Entries.front()->getVectorFactor(),
13316 VF = Entries.front()->getVectorFactor();
13421 Pair.first * VF +
13450 if (MinElement == PoisonMaskElem || MinElement % VF > Idx % VF)
13452 if (MaxElement == PoisonMaskElem || MaxElement % VF < Idx % VF)
13456 MaxElement % VF >= MinElement % VF &&
13460 (MaxElement % VF) -
13461 (MinElement % VF) + 1));
13462 if (NewVF < VF) {
13466 Idx = ((Idx % VF) - (((MinElement % VF) / NewVF) * NewVF)) % NewVF +
13467 (Idx >= static_cast<int>(VF) ? NewVF : 0);
13470 NewVF = VF;
13550 if (Idx >= static_cast<int>(VF))
13560 if (Idx < static_cast<int>(VF))
13563 Idx -= VF;
14137 unsigned VF = Mask.size();
14139 if (VF == LocalVF && ShuffleVectorInst::isIdentityMask(Mask, VF))
14149 Value *createPoison(Type *Ty, unsigned VF) {
14150 return PoisonValue::get(getWidenedType(Ty, VF));
14159 int VF = std::max(V1VF, V2VF);
14161 SmallVector<int> IdentityMask(VF, PoisonMaskElem);
14275 const unsigned VF = std::accumulate(
14297 Bases[I / VF] = VecOp;
14419 unsigned VF = std::max(getVF(V1), getVF(Vec));
14422 CommonMask[Idx] = Idx + VF;
14452 unsigned VF = std::max(CommonMask.size(), Mask.size());
14457 ? Idx + VF
14477 unsigned VF = 0;
14479 VF = std::max(VF, getVF(V));
14482 CommonMask[Idx] = Mask[Idx] + (It == InVectors.begin() ? 0 : VF);
14504 ArrayRef<int> SubVectorsMask, unsigned VF = 0,
14516 assert(VF > 0 &&
14519 if (VecVF < VF) {
14520 SmallVector<int> ResizeMask(VF, PoisonMaskElem);
14640 const unsigned VF = VL.size();
14666 if (VF * getNumElements(VL[0]->getType()) !=
14687 SmallVector<int> Mask(VF, PoisonMaskElem);
14695 assert(VF < cast<FixedVectorType>(V->getType())->getNumElements() &&
14698 SmallVector<int> UniformMask(VF, 0);
14738 unsigned VF = E->getVectorFactor();
14868 if (VF == VecBaseTy->getNumElements() &&
14869 GatheredScalars.size() != VF) {
14871 GatheredScalars.append(VF - GatheredScalars.size(),
14874 ::getNumberOfParts(*TTI, getWidenedType(OrigScalarTy, VF), VF);
14933 if (GatheredScalars.size() != VF &&
14936 return TE->getVectorFactor() == VF;
14939 GatheredScalars.append(VF - GatheredScalars.size(),
14956 Scalars.append(VF - Scalars.size(), PoisonValue::get(OrigScalarTy));
14991 ReuseMask.assign(VF, PoisonMaskElem);
15917 unsigned VF = VecTyNumElements / ScalarTyNumElements;
15925 VecPtr, createReplicatedMask(ScalarTyNumElements, VF)),
16713 int VF = cast<FixedVectorType>(V1->getType())->getNumElements();
16715 if (Mask[I] < VF)
16718 CombinedMask2[I] = Mask[I] - VF;
16730 unsigned VF = Mask.size();
16732 if (VF != VecVF) {
16733 if (any_of(Mask, [VF](int Idx) { return Idx >= static_cast<int>(VF); })) {
16738 SmallVector<int> ResizeMask(VF, PoisonMaskElem);
16739 for (unsigned I = 0; I < VF; ++I) {
17795 const unsigned VF = E.Scalars.size();
17798 ::getNumberOfParts(*TTI, getWidenedType(OrigScalarTy, VF)) ==
17803 VF)))
18078 unsigned VF = E.Scalars.size();
18083 buildIntrinsicArgTypes(IC, ID, VF, MinBW, TTI);
18085 IC, getWidenedType(IntegerType::get(IC->getContext(), MinBW), VF),
18205 unsigned VF = E.getVectorFactor();
18217 *TTI, getWidenedType(TreeRootIT, VF * ScalarTyNumElements));
18280 VF)))
18569 unsigned VF = Chain.size();
18574 VF) ||
18575 VF < 2 || VF < MinVF) {
18576 // Check if vectorizing with a non-power-of-2 VF should be considered. At
18577 // the moment, only consider cases where VF + 1 is a power-of-2, i.e. almost
18579 if (!VectorizeNonPowerOf2 || (VF < MinVF && VF + 1 != MinVF))
18583 LLVM_DEBUG(dbgs() << "SLP: Analyzing " << VF << " stores at offset " << Idx
18635 LLVM_DEBUG(dbgs() << "SLP: Found cost = " << Cost << " for VF=" << VF << "\n");
18755 // First try vectorizing with a non-power-of-2 VF. At the moment, only
18756 // consider cases where VF + 1 is a power-of-2, i.e. almost all vector
18762 "Non-power-of-2 VF should not be equal to MaxVF");
18778 for_each(reverse(CandidateVFs), [&](unsigned &VF) {
18779 VF = Size > MaxVF ? NonPowerOf2VF : Size;
18949 unsigned VF = bit_ceil(CandidateVFs.front()) * 2;
18953 if (bit_floor(Limit) == VF)
18955 if (VF > MaxTotalNum || VF >= StoresLimit)
18963 CandidateVFs.push_back(VF);
19175 for (unsigned VF = MaxVF; NextInst + 1 < MaxInst && VF >= MinVF;
19176 VF = getFloorFullVectorNumberOfElements(*TTI, I0->getType(), VF - 1)) {
19180 auto *VecTy = getWidenedType(ScalarTy, VF);
19181 if (TTI->getNumberOfParts(VecTy) == VF)
19184 unsigned ActualVF = std::min(MaxInst - I, VF);
19191 if ((VF > MinVF && ActualVF < VF) || (VF == MinVF && ActualVF < 2))
19230 << " for VF=" << ActualVF << "\n");
19241 I += VF - 1;
20235 for (unsigned VF = getFloorFullVectorNumberOfElements(
20237 VF >= ReductionLimit;
20238 VF = getFloorFullVectorNumberOfElements(
20239 *TTI, VL.front()->getType(), VF - 1)) {
20240 if (has_single_bit(VF) &&
20243 for (unsigned Idx : seq<unsigned>(ReduxWidth - VF))
20244 IgnoredCandidates.insert(std::make_pair(Offset + Idx, VF));
20752 for (unsigned I = 0, VF = VL.size(); I < VF; ++I) {
20756 Mask[I] = VF;