Lines Matching defs:VL
339 static unsigned getShufflevectorNumGroups(ArrayRef<Value *> VL) {
340 if (VL.empty())
342 if (!all_of(VL, IsaPred<ShuffleVectorInst>))
344 auto *SV = cast<ShuffleVectorInst>(VL.front());
351 if (GroupSize == 0 || (VL.size() % GroupSize) != 0)
354 for (size_t I = 0, E = VL.size(); I != E; I += GroupSize) {
355 auto *SV = cast<ShuffleVectorInst>(VL[I]);
357 ArrayRef<Value *> Group = VL.slice(I, GroupSize);
375 assert(NumGroup == (VL.size() / GroupSize) && "Unexpected number of groups");
391 static SmallVector<int> calculateShufflevectorMask(ArrayRef<Value *> VL) {
392 assert(getShufflevectorNumGroups(VL) && "Not supported shufflevector usage.");
393 auto *SV = cast<ShuffleVectorInst>(VL.front());
398 for (Value *V : VL) {
449 static std::string shortBundleName(ArrayRef<Value *> VL, int Idx = -1) {
454 OS << "n=" << VL.size() << " [" << *VL.front() << ", ..]";
459 /// \returns true if all of the instructions in \p VL are in the same block or
461 static bool allSameBlock(ArrayRef<Value *> VL) {
462 auto *It = find_if(VL, IsaPred<Instruction>);
463 if (It == VL.end())
466 if (all_of(VL, isVectorLikeInstWithConstOps))
470 for (Value *V : iterator_range(It, VL.end())) {
483 /// \returns True if all of the values in \p VL are constants (but not
485 static bool allConstant(ArrayRef<Value *> VL) {
488 return all_of(VL, isConstant);
491 /// \returns True if all of the values in \p VL are identical or some of them
493 static bool isSplat(ArrayRef<Value *> VL) {
495 for (Value *V : VL) {
706 isFixedVectorShuffle(ArrayRef<Value *> VL, SmallVectorImpl<int> &Mask,
708 const auto *It = find_if(VL, IsaPred<ExtractElementInst>);
709 if (It == VL.end())
712 std::accumulate(VL.begin(), VL.end(), 0u, [](unsigned S, Value *V) {
724 bool HasNonUndefVec = any_of(VL, [&](Value *V) {
735 Mask.assign(VL.size(), PoisonMaskElem);
736 for (unsigned I = 0, E = VL.size(); I < E; ++I) {
738 if (isa<UndefValue>(VL[I]))
740 auto *EI = cast<ExtractElementInst>(VL[I]);
868 static InstructionsState getSameOpcode(ArrayRef<Value *> VL,
906 /// \returns analysis of the Instructions in \p VL described in
909 static InstructionsState getSameOpcode(ArrayRef<Value *> VL,
912 if (!all_of(VL, IsaPred<Instruction, PoisonValue>))
915 auto *It = find_if(VL, IsaPred<Instruction>);
916 if (It == VL.end())
920 unsigned InstCnt = std::count_if(It, VL.end(), IsaPred<Instruction>);
921 if ((VL.size() > 2 && !isa<PHINode>(MainOp) && InstCnt < VL.size() / 2) ||
922 (VL.size() == 2 && InstCnt < 2))
938 for (Value *V : VL) {
965 bool AnyPoison = InstCnt != VL.size();
968 for (Value *V : iterator_range(It, VL.end())) {
1020 if ((VL.size() == 2 || SwappedPredsCompatible) &&
1091 /// \returns true if all of the values in \p VL have the same type or false
1093 static bool allSameType(ArrayRef<Value *> VL) {
1094 Type *Ty = VL.front()->getType();
1095 return all_of(VL.drop_front(), [&](Value *V) { return V->getType() == Ty; });
1212 static SmallBitVector getAltInstrMask(ArrayRef<Value *> VL, unsigned Opcode0,
1214 Type *ScalarTy = VL[0]->getType();
1216 SmallBitVector OpcodeMask(VL.size() * ScalarTyNumElements, false);
1217 for (unsigned Lane : seq<unsigned>(VL.size())) {
1218 if (isa<PoisonValue>(VL[Lane]))
1220 if (cast<Instruction>(VL[Lane])->getOpcode() == Opcode1)
1296 static bool doesNotNeedToSchedule(ArrayRef<Value *> VL) {
1297 return !VL.empty() &&
1298 (all_of(VL, isUsedOutsideBlock) || all_of(VL, areAllOperandsNonInsts));
1385 /// Vectorize the tree that starts with the elements in \p VL.
1400 /// \returns the vectorization cost of the subtree that starts at \p VL.
1628 /// \param VL list of loads.
1636 LoadsState canVectorizeLoads(ArrayRef<Value *> VL, const Value *VL0,
1643 template <typename T> void registerNonVectorizableLoads(ArrayRef<T *> VL) {
1644 ListOfKnonwnNonVectorizableLoads.insert(hash_value(VL));
1649 bool areKnownNonVectorizableLoads(ArrayRef<T *> VL) const {
1650 return ListOfKnonwnNonVectorizableLoads.contains(hash_value(VL));
2051 /// When VL[0] is IntrinsicInst, ArgSize is CallBase::arg_size. When VL[0]
2436 /// Go through the instructions in VL and append their operands.
2437 void appendOperandsOfVL(ArrayRef<Value *> VL, const InstructionsState &S) {
2438 assert(!VL.empty() && "Bad VL");
2439 assert((empty() || VL.size() == getNumLanes()) &&
2449 unsigned NumLanes = VL.size();
2453 assert((isa<Instruction>(VL[Lane]) || isa<PoisonValue>(VL[Lane])) &&
2457 // opcode of VL[Lane] and whether the operand at OpIdx is the LHS or
2460 // is false. The RHS is true only if VL[Lane] is an inverse operation.
2465 if (isa<PoisonValue>(VL[Lane])) {
2482 bool IsInverseOperation = !isCommutative(cast<Instruction>(VL[Lane]));
2484 OpsVec[OpIdx][Lane] = {cast<Instruction>(VL[Lane])->getOperand(OpIdx),
2933 bool areAnalyzedReductionVals(ArrayRef<Value *> VL) const {
2934 return AnalyzedReductionVals.contains(hash_value(VL));
2938 void analyzedReductionVals(ArrayRef<Value *> VL) {
2939 AnalyzedReductionVals.insert(hash_value(VL));
2999 ArrayRef<Value *> VL = UserTE->getOperand(OpIdx);
3001 const auto *It = find_if(VL, [&](Value *V) {
3010 if (It != VL.end()) {
3011 assert(TE->isSame(VL) && "Expected same scalars.");
3055 /// \returns true if the ExtractElement/ExtractValue instructions in \p VL can
3062 bool canReuseExtract(ArrayRef<Value *> VL,
3106 /// was successful, the matched scalars are replaced by poison values in \p VL
3109 tryToGatherSingleRegisterExtractElements(MutableArrayRef<Value *> VL,
3115 /// was successful, the matched scalars are replaced by poison values in \p VL
3118 tryToGatherExtractElements(SmallVectorImpl<Value *> &VL,
3122 /// Checks if the gathered \p VL can be represented as a single register
3125 /// \param VL List of scalars (a subset of the TE scalar), checked for
3134 const TreeEntry *TE, ArrayRef<Value *> VL, MutableArrayRef<int> Mask,
3138 /// Checks if the gathered \p VL can be represented as multi-register
3141 /// \param VL List of scalars (a subset of the TE scalar), checked for
3151 const TreeEntry *TE, ArrayRef<Value *> VL, SmallVectorImpl<int> &Mask,
3155 /// \returns the cost of gathering (inserting) the values in \p VL into a
3158 InstructionCost getGatherCost(ArrayRef<Value *> VL, bool ForPoisonSrc,
3165 /// \returns a vector from a collection of scalars in \p VL. if \p Root is not
3168 gather(ArrayRef<Value *> VL, Value *Root, Type *ScalarTy,
3221 /// \returns true if the scalars in VL are equal to this entry.
3222 bool isSame(ArrayRef<Value *> VL) const {
3223 auto &&IsSame = [VL](ArrayRef<Value *> Scalars, ArrayRef<int> Mask) {
3224 if (Mask.size() != VL.size() && VL.size() == Scalars.size())
3225 return std::equal(VL.begin(), VL.end(), Scalars.begin());
3226 return VL.size() == Mask.size() &&
3227 std::equal(VL.begin(), VL.end(), Mask.begin(),
3236 // treat the vector as the same if the list of scalars matches VL
3240 if (VL.size() == Scalars.size())
3242 if (VL.size() == ReuseShuffleIndices.size()) {
3593 TreeEntry *newTreeEntry(ArrayRef<Value *> VL,
3602 TreeEntry *E = newTreeEntry(VL, EntryState, Bundle, S, UserTreeIdx,
3609 TreeEntry *newTreeEntry(ArrayRef<Value *> VL,
3632 (hasFullVectorsOrPowerOf2(*TTI, getValueType(VL.front()), VL.size()) ||
3638 Last->Scalars.assign(VL.begin(), VL.end());
3643 Last->Scalars.assign(VL.size(), nullptr);
3645 [VL](unsigned Idx) -> Value * {
3646 if (Idx >= VL.size())
3647 return UndefValue::get(VL.front()->getType());
3648 return VL[Idx];
3657 for (Value *V : VL) {
3679 doesNotNeedToSchedule(VL)) &&
3680 "Bundle and VL out of sync");
3682 for (Value *V : VL) {
3691 assert(!BundleMember && "Bundle and VL out of sync");
3695 for (Value *V : VL)
3706 MustGather.insert(VL.begin(), VL.end());
3737 /// Returns first vector node for value \p V, matching values \p VL.
3738 TreeEntry *getSameValuesTreeEntry(Value *V, ArrayRef<Value *> VL,
3742 if ((!SameVF || TE->getVectorFactor() == VL.size()) && TE->isSame(VL))
3752 /// \param VL list of the instructions with alternate opcodes.
3754 ArrayRef<Value *> VL) const;
3759 getScalarsVectorizationState(const InstructionsState &S, ArrayRef<Value *> VL,
4274 ScheduleData *buildBundle(ArrayRef<Value *> VL);
4280 /// std::nullopt if \p VL is allowed to be scheduled.
4282 tryScheduleBundle(ArrayRef<Value *> VL, BoUpSLP *SLP,
4286 void cancelScheduling(ArrayRef<Value *> VL, Value *OpValue);
4817 static Align computeCommonAlignment(ArrayRef<Value *> VL) {
4818 Align CommonAlignment = cast<T>(VL.front())->getAlign();
4819 for (Value *V : VL.drop_front())
5033 BoUpSLP::canVectorizeLoads(ArrayRef<Value *> VL, const Value *VL0,
5045 if (areKnownNonVectorizableLoads(VL))
5055 const unsigned Sz = VL.size();
5058 for (Value *V : VL) {
5071 Align CommonAlignment = computeCommonAlignment<LoadInst>(VL);
5138 cast<LoadInst>(Order.empty() ? VL.front() : VL[Order.front()])
5197 std::accumulate(VL.begin(), VL.end(), InstructionCost(),
5216 if (!TryRecursiveCheck || VL.size() < ListLimit)
5222 if (!hasFullVectorsOrPowerOf2(TTI, ScalarTy, VL.size()))
5231 getFloorFullVectorNumberOfElements(TTI, ScalarTy, VL.size() - 1);
5235 for (unsigned Cnt = 0, End = VL.size(); Cnt + VF <= End; Cnt += VF) {
5236 ArrayRef<Value *> Slice = VL.slice(Cnt, VF);
5268 for (unsigned Idx : seq<unsigned>(VL.size()))
5271 TTI.getInstructionCost(cast<Instruction>(VL[Idx]), CostKind);
5276 auto *LI0 = cast<LoadInst>(VL[I * VF]);
5330 SmallVector<int> ShuffleMask(VL.size());
5331 for (int Idx : seq<int>(0, VL.size()))
5332 ShuffleMask[Idx] = Idx / VF == I ? VL.size() + Idx % VF : Idx;
5377 static bool clusterSortPtrAccesses(ArrayRef<Value *> VL,
5382 all_of(VL, [](const Value *V) { return V->getType()->isPointerTy(); }) &&
5392 BBs.front(), getUnderlyingObject(VL.front(), RecursionMaxDepth)))
5393 .first->second.emplace_back().emplace_back(VL.front(), 0U, 0U);
5396 for (auto [Cnt, Ptr] : enumerate(VL.drop_front())) {
5414 if (Bases.size() > VL.size() / 2 - 1)
5422 if (Bases.size() == VL.size())
5426 Bases.front().second.size() == VL.size()))
5478 assert(SortedIndices.size() == VL.size() &&
5479 "Expected SortedIndices to be the size of VL");
6806 const BoUpSLP &R, ArrayRef<Value *> VL, const DataLayout &DL,
6810 if (VL.empty())
6812 Type *ScalarTy = getValueType(VL.front());
6817 for (Value *V : VL) {
7303 ArrayRef<Value *> VL =
7305 ConsecutiveNodesSize += VL.size();
7308 return Sz < VL.size() ||
7310 VL.size()) != VL;
7415 static bool needToScheduleSingleInstruction(ArrayRef<Value *> VL) {
7417 for (Value *V : VL) {
7525 ArrayRef<Value *> VL) const {
7528 SmallBitVector OpcodeMask(getAltInstrMask(VL, Opcode0, Opcode1));
7530 if (TTI->isLegalAltInstr(getWidenedType(S.getMainOp()->getType(), VL.size()),
7537 for (Value *V : VL) {
7548 for (unsigned I : seq<unsigned>(0, VL.size() - 1)) {
7632 (UndefCnt < (VL.size() - 1) * S.getMainOp()->getNumOperands() &&
7634 NumAltInsts) < S.getMainOp()->getNumOperands() * VL.size());
7638 const InstructionsState &S, ArrayRef<Value *> VL,
7653 for (Value *V : VL) {
7671 bool Reuse = canReuseExtract(VL, CurrentOrder);
7674 if (!hasFullVectorsOrPowerOf2(*TTI, VL0->getType(), VL.size()))
7685 for (Value *V : VL) {
7691 if (count_if(VL, [&SourceVectors](Value *V) {
7700 if (any_of(VL, [&SourceVectors](Value *V) {
7719 switch (canVectorizeLoads(VL, VL0, CurrentOrder, PointerOps)) {
7742 else if (any_of(VL, [](Value *V) {
7750 registerNonVectorizableLoads(VL);
7768 for (Value *V : VL) {
7786 for (Value *V : VL) {
7820 TTI->isFPVectorizationPotentiallyUnsafe() && any_of(VL, [](Value *V) {
7828 for (Value *V : VL) {
7841 for (Value *V : VL) {
7854 for (Value *V : VL) {
7885 for (Value *V : VL) {
7908 if (static_cast<unsigned>(*Dist) == VL.size() - 1)
7917 TTI->isFPVectorizationPotentiallyUnsafe() && any_of(VL, [](Value *V) {
7929 ElementCount::getFixed(static_cast<unsigned int>(VL.size())),
7943 for (Value *V : VL) {
7983 if (SLPReVec && getShufflevectorNumGroups(VL))
7990 if (!SLPSkipEarlyProfitabilityCheck && !areAltOperandsProfitable(S, VL)) {
8096 void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
8099 assert((allConstant(VL) || allSameType(VL)) && "Invalid types!");
8107 SmallDenseMap<Value *, unsigned, 16> UniquePositions(VL.size());
8108 for (Value *V : VL) {
8123 if (NumUniqueScalarValues == VL.size() &&
8130 !hasFullVectorsOrPowerOf2(*TTI, VL.front()->getType(), VL.size())) {
8133 newTreeEntry(VL, std::nullopt /*not vectorized*/, S, UserTreeIdx);
8147 if (PWSz == VL.size()) {
8158 newTreeEntry(VL, std::nullopt /*not vectorized*/, S, UserTreeIdx);
8161 VL = NonUniqueValueVL;
8166 newTreeEntry(VL, std::nullopt /*not vectorized*/, S, UserTreeIdx);
8169 VL = UniqueValues;
8174 InstructionsState S = getSameOpcode(VL, *TLI);
8181 newTreeEntry(VL, std::nullopt /*not vectorized*/, S, UserTreeIdx);
8189 if (E->isSame(VL)) {
8197 if (all_of(VL, [&](Value *V) {
8202 newTreeEntry(VL, std::nullopt /*not vectorized*/, S, UserTreeIdx,
8213 !(S && !S.isAltShuffle() && VL.size() >= 4 &&
8215 all_of(VL, [&S](const Value *I) {
8222 newTreeEntry(VL, std::nullopt /*not vectorized*/, S, UserTreeIdx,
8233 newTreeEntry(VL, std::nullopt /*not vectorized*/, S, UserTreeIdx,
8239 if (!SLPReVec && getValueType(VL.front())->isVectorTy()) {
8241 newTreeEntry(VL, std::nullopt /*not vectorized*/, S, UserTreeIdx);
8251 Depth](ArrayRef<Value *> VL) {
8252 if (!S || !S.isAltShuffle() || VL.size() > 2)
8261 for (Value *V : VL) {
8274 assert(VL.size() == 2 && "Expected only 2 alternate op instructions.");
8276 auto *I1 = cast<Instruction>(VL.front());
8277 auto *I2 = cast<Instruction>(VL.back());
8307 bool AreAllSameBlock = S && allSameBlock(VL);
8309 (IsScatterVectorizeUserTE && VL.front()->getType()->isPointerTy() &&
8310 VL.size() > 2 &&
8311 all_of(VL,
8321 sortPtrAccesses(VL, UserTreeIdx.UserTE->getMainOp()->getType(), *DL, *SE,
8324 if (!AreAllSameInsts || (!S && allConstant(VL)) || isSplat(VL) ||
8328 !all_of(VL, isVectorLikeInstWithConstOps)) ||
8329 NotProfitableForVectorization(VL)) {
8332 newTreeEntry(VL, std::nullopt /*not vectorized*/, S, UserTreeIdx,
8339 for (Value *V : VL) {
8343 newTreeEntry(VL, std::nullopt /*not vectorized*/, S, UserTreeIdx);
8353 for (Value *V : VL) {
8361 newTreeEntry(VL, std::nullopt /*not vectorized*/, S, UserTreeIdx,
8369 for (Value *V : VL) {
8373 newTreeEntry(VL, std::nullopt /*not vectorized*/, S, UserTreeIdx,
8383 assert(VL.front()->getType()->isPointerTy() &&
8384 count_if(VL, IsaPred<GetElementPtrInst>) >= 2 &&
8387 const auto *It = find_if(VL, IsaPred<GetElementPtrInst>);
8388 assert(It != VL.end() && "Expected at least one GEP.");
8405 newTreeEntry(VL, std::nullopt /*not vectorized*/, S, UserTreeIdx);
8417 S, VL, IsScatterVectorizeUserTE, CurrentOrder, PointerOps);
8419 newTreeEntry(VL, std::nullopt /*not vectorized*/, S, UserTreeIdx,
8441 newTreeEntry(VL, std::nullopt /*not vectorized*/, S, UserTreeIdx,
8443 NonScheduledFirst.insert(VL.front());
8446 registerNonVectorizableLoads(VL);
8474 newTreeEntry(VL, Bundle, S, UserTreeIdx, ReuseShuffleIndices);
8479 PHIHandler Handler(*DT, PH, VL);
8505 TreeEntry *TE = newTreeEntry(VL, Bundle /*vectorized*/, S, UserTreeIdx,
8525 for (int I = 0, E = VL.size(); I < E; ++I) {
8526 unsigned Idx = *getElementIndex(VL[I]);
8529 OrdersType CurrentOrder(VL.size(), VL.size());
8531 for (int I = 0, E = VL.size(); I < E; ++I) {
8538 TreeEntry *TE = newTreeEntry(VL, Bundle /*vectorized*/, S, UserTreeIdx,
8558 TE = newTreeEntry(VL, Bundle /*vectorized*/, S, UserTreeIdx,
8570 TE = newTreeEntry(VL, TreeEntry::StridedVectorize, Bundle, S,
8577 TE = newTreeEntry(VL, TreeEntry::ScatterVectorize, Bundle, S,
8624 TreeEntry *TE = newTreeEntry(VL, Bundle /*vectorized*/, S, UserTreeIdx,
8652 TreeEntry *TE = newTreeEntry(VL, Bundle /*vectorized*/, S, UserTreeIdx,
8658 VLOperands Ops(VL, S, *this);
8669 for (Value *V : VL) {
8723 TreeEntry *TE = newTreeEntry(VL, Bundle /*vectorized*/, S, UserTreeIdx,
8736 TreeEntry *TE = newTreeEntry(VL, Bundle /*vectorized*/, S, UserTreeIdx,
8742 for (Value *V : VL) {
8758 Type *Ty = all_of(VL,
8770 for (Value *V : VL) {
8795 TreeEntry *TE = newTreeEntry(VL, Bundle /*vectorized*/, S, UserTreeIdx,
8814 TreeEntry *TE = newTreeEntry(VL, Bundle /*vectorized*/, S, UserTreeIdx,
8829 TreeEntry *TE = newTreeEntry(VL, Bundle /*vectorized*/, S, UserTreeIdx,
8843 if (CI && any_of(VL, [](Value *V) {
8855 for (Value *V : VL) {
8926 bool BoUpSLP::canReuseExtract(ArrayRef<Value *> VL,
8929 const auto *It = find_if(VL, IsaPred<ExtractElementInst, ExtractValueInst>);
8930 assert(It != VL.end() && "Expected at least one extract instruction.");
8933 all_of(VL, IsaPred<UndefValue, ExtractElementInst, ExtractValueInst>) &&
8949 if (!LI || !LI->isSimple() || !LI->hasNUses(VL.size()))
8955 unsigned E = VL.size();
8960 for (auto [I, V] : enumerate(VL)) {
9804 ArrayRef<Value *> VL = E.Scalars;
9805 const unsigned Sz = getVectorElementSize(VL.front());
9809 if (VL.size() <= 2 || LoadEntriesToVectorize.contains(Idx) ||
9811 E.isAltShuffle() || !allSameBlock(VL)) ||
9812 allConstant(VL) || isSplat(VL))
9817 unsigned End = VL.size();
9819 *TTI, VL.front()->getType(), VL.size() - 1);
9821 *TTI, VL.front()->getType(), VF - 1)) {
9826 ArrayRef<Value *> Slice = VL.slice(Cnt, VF);
9912 ArrayRef<Value *> Slice = VL.slice(Cnt, Sz);
10148 InstructionCost getBuildVectorCost(ArrayRef<Value *> VL, Value *Root) {
10149 if ((!Root && allConstant(VL)) || all_of(VL, IsaPred<UndefValue>))
10151 auto *VecTy = getWidenedType(ScalarTy, VL.size());
10153 SmallVector<Value *> Gathers(VL);
10154 if (!Root && isSplat(VL)) {
10157 const auto *It = find_if_not(VL, IsaPred<UndefValue>);
10158 assert(It != VL.end() && "Expected at least one non-undef value.");
10161 count(VL, *It) > 1 &&
10162 (VL.front() != *It || !all_of(VL.drop_front(), IsaPred<UndefValue>));
10168 std::distance(VL.begin(), It) * getNumElements(ScalarTy),
10172 CostKind, std::distance(VL.begin(), It),
10176 SmallVector<int> ShuffleMask(VL.size(), PoisonMaskElem);
10177 transform(VL, ShuffleMask.begin(), [](Value *V) {
10192 : R.getGatherCost(Gathers, !Root && VL.equals(Gathers),
10197 /// \p VL.
10199 computeExtractCost(ArrayRef<Value *> VL, ArrayRef<int> Mask,
10202 assert(VL.size() > NumParts && "Unexpected scalarized shuffle.");
10204 std::accumulate(VL.begin(), VL.end(), 0, [](unsigned Sz, Value *V) {
10214 unsigned EltsPerVector = getPartNumElems(VL.size(), NumParts);
10301 *R.TTI, VL.front()->getType(), alignTo(NumElts, EltsPerVector));
10660 SmallVector<Value *> VL(E->Scalars.begin(), E->Scalars.end());
10664 reorderScalars(VL, ReorderMask);
10675 return VL.size() > Data.index() &&
10677 isa<UndefValue>(VL[Data.index()]) ||
10678 Data.value() == VL[Data.index()]);
10682 unsigned SliceSize = getPartNumElems(VL.size(), NumParts);
10684 unsigned Limit = getNumElems(VL.size(), SliceSize, Part);
10687 enumerate(ArrayRef(VL).slice(Part * SliceSize, Limit))) {
10744 Cost += computeExtractCost(VL, Mask, ShuffleKinds, NumParts);
10870 Value *gather(ArrayRef<Value *> VL, unsigned MaskVF = 0,
10872 Cost += getBuildVectorCost(VL, Root);
10876 unsigned VF = VL.size();
10879 for (Value *V : VL.take_front(VF)) {
11075 ArrayRef<Value *> VL = E->Scalars;
11077 Type *ScalarTy = getValueType(VL[0]);
11092 auto *VecTy = getWidenedType(ScalarTy, VL.size());
11097 if (allConstant(VL))
11099 if (isa<InsertElementInst>(VL[0]))
11101 if (isa<CmpInst>(VL.front()))
11102 ScalarTy = VL.front()->getType();
11130 ((allSameType(VL) && allSameBlock(VL)) ||
11133 "Invalid VL");
11139 SmallSetVector<Value *, 16> UniqueValues(VL.begin(), VL.end());
11231 auto [MinMaxID, SelectOnly] = canConvertToMinOrMaxIntrinsic(VI ? VI : VL);
11329 unsigned const NumScalars = VL.size();
11334 unsigned OffsetBeg = *getElementIndex(VL.front());
11337 for (auto [I, V] : enumerate(VL.drop_front())) {
11379 unsigned InsertIdx = *getElementIndex(VL[PrevMask[I]]);
11444 auto *SrcVecTy = getWidenedType(SrcScalarTy, VL.size());
11456 getWidenedType(SrcScalarTy, VL.size() * SrcScalarTyNumElements);
11548 auto *MaskTy = getWidenedType(Builder.getInt1Ty(), VL.size());
11556 getWidenedType(SI->getCondition()->getType(), VL.size());
11638 return CommonCost + GetGEPCostDiff(VL, VL0);
11693 SmallVector<Value *> PointerOps(VL.size());
11694 for (auto [I, V] : enumerate(VL))
11701 auto *VI = cast<StoreInst>(VL[Idx]);
11708 cast<StoreInst>(IsReorder ? VL[E->ReorderIndices.front()] : VL0);
11737 SmallVector<Value *> PointerOps(VL.size());
11738 for (auto [I, V] : enumerate(VL)) {
11822 auto *MaskTy = getWidenedType(Builder.getInt1Ty(), VL.size());
11834 auto *SrcTy = getWidenedType(SrcSclTy, VL.size());
11843 SrcTy = getWidenedType(SrcSclTy, VL.size());
11896 assert(isa<ShuffleVectorInst>(VL.front()) &&
11898 auto *SV = cast<ShuffleVectorInst>(VL.front());
11903 for (size_t I = 0, End = VL.size(); I != End; I += GroupSize) {
11904 ArrayRef<Value *> Group = VL.slice(I, GroupSize);
12941 /// successful, the matched scalars are replaced by poison values in \p VL for
12945 MutableArrayRef<Value *> VL, SmallVectorImpl<int> &Mask) const {
12950 for (int I = 0, E = VL.size(); I < E; ++I) {
12951 auto *EI = dyn_cast<ExtractElementInst>(VL[I]);
12953 if (isa<UndefValue>(VL[I]))
12999 SmallVector<Value *> SavedVL(VL.begin(), VL.end());
13001 VL.size(), PoisonValue::get(VL.front()->getType()));
13004 std::swap(GatheredExtracts[Idx], VL[Idx]);
13008 std::swap(GatheredExtracts[Idx], VL[Idx]);
13012 std::swap(GatheredExtracts[Idx], VL[Idx]);
13019 // Restore the original VL if attempt was not successful.
13020 copy(SavedVL, VL.begin());
13028 std::swap(VL[I], GatheredExtracts[I]);
13031 auto *EI = dyn_cast<ExtractElementInst>(VL[I]);
13043 /// successful, the matched scalars are replaced by poison values in \p VL for
13046 BoUpSLP::tryToGatherExtractElements(SmallVectorImpl<Value *> &VL,
13051 Mask.assign(VL.size(), PoisonMaskElem);
13052 unsigned SliceSize = getPartNumElems(VL.size(), NumParts);
13056 MutableArrayRef<Value *> SubVL = MutableArrayRef(VL).slice(
13057 Part * SliceSize, getNumElems(VL.size(), SliceSize, Part));
13073 const TreeEntry *TE, ArrayRef<Value *> VL, MutableArrayRef<int> Mask,
13095 SmallPtrSet<Value *, 4> GatheredScalars(VL.begin(), VL.end());
13134 for (Value *V : VL) {
13250 return EntryPtr->isSame(VL) || EntryPtr->isSame(TE->Scalars);
13253 ((*It)->getVectorFactor() == VL.size() ||
13255 TE->ReuseShuffleIndices.size() == VL.size() &&
13258 if ((*It)->getVectorFactor() == VL.size()) {
13259 std::iota(std::next(Mask.begin(), Part * VL.size()),
13260 std::next(Mask.begin(), (Part + 1) * VL.size()), 0);
13266 for (unsigned I : seq<unsigned>(VL.size()))
13267 if (isa<PoisonValue>(VL[I]))
13268 Mask[Part * VL.size() + I] = PoisonMaskElem;
13320 bool IsSplatOrUndefs = isSplat(VL) || all_of(VL, IsaPred<UndefValue>);
13357 Value *V1 = VL[Idx];
13371 for (int I = 0, E = VL.size(); I < E; ++I) {
13372 Value *V = VL[I];
13404 !VL.equals(ArrayRef(TE->Scalars)
13405 .slice(Part * VL.size(),
13406 std::min<int>(VL.size(), TE->Scalars.size())))) {
13409 // profitable. Since VL is not the same as TE->Scalars, it means we already
13419 unsigned Idx = Part * VL.size() + Pair.second;
13424 find(Entries[Pair.first]->Scalars, VL[Pair.second]))
13425 : Entries[Pair.first]->findLaneForValue(VL[Pair.second]));
13431 if (IsIdentity || EntryLanes.size() > 1 || VL.size() <= 2)
13435 if (EntryLanes.size() > 2 || VL.size() <= 2)
13441 } else if (!isa<VectorType>(VL.front()->getType()) &&
13442 (EntryLanes.size() > Entries.size() || VL.size() <= 2)) {
13444 SmallVector<int> SubMask(std::next(Mask.begin(), Part * VL.size()),
13445 std::next(Mask.begin(), (Part + 1) * VL.size()));
13459 VL.size(), getFullVectorNumberOfElements(*TTI, VL.front()->getType(),
13474 auto *VecTy = getWidenedType(VL.front()->getType(), NewVF);
13475 auto *MaskVecTy = getWidenedType(VL.front()->getType(), SubMask.size());
13547 std::for_each(std::next(Mask.begin(), Part * VL.size()),
13548 std::next(Mask.begin(), (Part + 1) * VL.size()),
13557 std::for_each(std::next(Mask.begin(), Part * VL.size()),
13558 std::next(Mask.begin(), (Part + 1) * VL.size()),
13579 std::fill(std::next(Mask.begin(), Part * VL.size()),
13580 std::next(Mask.begin(), (Part + 1) * VL.size()), PoisonMaskElem);
13586 const TreeEntry *TE, ArrayRef<Value *> VL, SmallVectorImpl<int> &Mask,
13589 assert(NumParts > 0 && NumParts < VL.size() &&
13604 Mask.assign(VL.size(), PoisonMaskElem);
13608 assert(VL.size() % NumParts == 0 &&
13620 unsigned SliceSize = getPartNumElems(VL.size(), NumParts);
13624 VL.slice(Part * SliceSize, getNumElems(VL.size(), SliceSize, Part));
13633 SubEntries.front()->getVectorFactor() == VL.size() &&
13635 SubEntries.front()->isSame(VL))) {
13642 for (int I = 0, Sz = VL.size(); I < Sz; ++I)
13643 if (isa<PoisonValue>(VL[I]))
13658 InstructionCost BoUpSLP::getGatherCost(ArrayRef<Value *> VL, bool ForPoisonSrc,
13660 auto *VecTy = getWidenedType(ScalarTy, VL.size());
13665 APInt ShuffledElements = APInt::getZero(VL.size());
13680 SmallVector<int> ShuffleMask(VL.size(), PoisonMaskElem);
13681 for (unsigned I = 0, E = VL.size(); I < E; ++I) {
13682 Value *V = VL[I];
13708 for (unsigned I : seq<unsigned>(VL.size()))
13717 /*Extract*/ false, CostKind, VL);
13858 // scheduled, and the last instruction is VL.back(). So we start with
13859 // VL.back() and iterate over schedule data until we reach the end of the
13873 // VL.back(). This can be the case if buildTree_rec aborts for various
13917 ArrayRef<Value *> VL, Value *Root, Type *ScalarTy,
13932 for (int I = 0, E = VL.size(); I < E; ++I) {
13933 if (auto *Inst = dyn_cast<Instruction>(VL[I]))
13995 auto *VecTy = getWidenedType(ScalarTy, VL.size());
13998 SmallVector<int> Mask(VL.size());
14008 for (int I = 0, E = VL.size(); I < E; ++I) {
14011 if (!isConstant(VL[I])) {
14015 if (isa<PoisonValue>(VL[I]))
14017 Vec = CreateInsertElement(Vec, VL[I], I, ScalarTy);
14035 Vec = CreateInsertElement(Vec, VL[I], I, ScalarTy);
14212 SmallVector<Value *> VL(E->Scalars.begin(), E->Scalars.end());
14216 reorderScalars(VL, ReorderMask);
14222 auto *EI = cast<ExtractElementInst>(VL[I]);
14231 (NumParts != 1 && count(VL, EI) > 1) ||
14245 is_contained(VL, EI);
14267 unsigned SliceSize = getPartNumElems(VL.size(), NumParts);
14269 unsigned Limit = getNumElems(VL.size(), SliceSize, Part);
14270 ArrayRef<Value *> SubVL = ArrayRef(VL).slice(Part * SliceSize, Limit);
14490 Value *gather(ArrayRef<Value *> VL, unsigned MaskVF = 0,
14492 return R.gather(VL, Root, ScalarTy,
14609 ArrayRef<Value *> VL = E->getOperand(NodeIdx);
14610 InstructionsState S = getSameOpcode(VL, *TLI);
14612 if (!S && VL.front()->getType()->isPointerTy()) {
14613 const auto *It = find_if(VL, IsaPred<GetElementPtrInst>);
14614 if (It != VL.end())
14631 TreeEntry *VE = getSameValuesTreeEntry(S.getMainOp(), VL);
14639 ValueList &VL = E->getOperand(NodeIdx);
14640 const unsigned VF = VL.size();
14647 unsigned NumElements = getNumElements(VL.front()->getType());
14666 if (VF * getNumElements(VL[0]->getType()) !=
14688 for (auto [I, V] : enumerate(VL)) {
14730 assert(I->get()->isSame(VL) && "Expected same list of scalars.");
15233 /// \returns \p I after propagating metadata from \p VL only for instructions in
15234 /// \p VL.
15235 static Instruction *propagateMetadata(Instruction *Inst, ArrayRef<Value *> VL) {
15237 for (Value *V : VL)
17080 BoUpSLP::BlockScheduling::buildBundle(ArrayRef<Value *> VL) {
17083 for (Value *V : VL) {
17109 BoUpSLP::BlockScheduling::tryScheduleBundle(ArrayRef<Value *> VL, BoUpSLP *SLP,
17114 isVectorLikeInstWithConstOps(S.getMainOp()) || doesNotNeedToSchedule(VL))
17160 for (Value *V : VL) {
17176 for (Value *V : VL) {
17197 auto *Bundle = buildBundle(VL);
17200 cancelScheduling(VL, S.getMainOp());
17206 void BoUpSLP::BlockScheduling::cancelScheduling(ArrayRef<Value *> VL,
17209 doesNotNeedToSchedule(VL))
17213 OpValue = *find_if_not(VL, doesNotNeedToBeScheduled);
17219 (Bundle->isPartOfBundle() || needToScheduleSingleInstruction(VL)) &&
19121 bool SLPVectorizerPass::tryToVectorizeList(ArrayRef<Value *> VL, BoUpSLP &R,
19123 if (VL.size() < 2)
19127 << VL.size() << ".\n");
19131 InstructionsState S = getSameOpcode(VL, *TLI);
19138 for (Value *V : VL) {
19155 Type *ScalarTy = getValueType(VL[0]);
19159 getFloorFullVectorNumberOfElements(*TTI, ScalarTy, VL.size()), MinVF);
19174 unsigned NextInst = 0, MaxInst = VL.size();
19196 for (Value *V : VL.drop_front(I)) {
20115 ArrayRef<Value *> VL(std::next(Candidates.begin(), Pos), ReduxWidth);
20124 V.areAnalyzedReductionVals(VL)) {
20130 if (any_of(VL, [&V](Value *RedVal) {
20137 V.buildTree(VL, IgnoreList);
20140 V.analyzedReductionVals(VL);
20145 V.analyzedReductionVals(VL);
20179 SmallPtrSet<Value *, 4> VLScalars(VL.begin(), VL.end());
20205 for (Value *RdxVal : VL)
20213 InstructionCost TreeCost = V.getTreeCost(VL);
20215 getReductionCost(TTI, VL, IsCmpSelMinMax, RdxFMF, V);
20224 ReducedValsToOps.at(VL[0]).front())
20231 V.analyzedReductionVals(VL);
20234 // Add subvectors of VL to the list of the analyzed values.
20236 *TTI, VL.front()->getType(), ReduxWidth - 1);
20239 *TTI, VL.front()->getType(), VF - 1)) {
20255 ReducedValsToOps.at(VL[0]).front())
20297 Type *ScalarTy = VL.front()->getType();
20306 // VL[0] = <4 x Ty> <a, b, c, d>
20307 // VL[1] = <4 x Ty> <e, f, g, h>
20317 createStrideMask(I, ScalarTyNumElements, VL.size());
20327 if (ReducedSubTree->getType() != VL.front()->getType()) {
20328 assert(ReducedSubTree->getType() != VL.front()->getType() &&
20331 Builder.CreateIntCast(ReducedSubTree, VL.front()->getType(),
20344 for (Value *RdxVal : VL) {
20703 ArrayRef<Value *> VL = R.getRootNodeScalars();
20705 if (VTy->getElementType() != VL.front()->getType()) {
20708 getWidenedType(VL.front()->getType(), VTy->getNumElements()),
20715 for (Value *V : VL) {
20752 for (unsigned I = 0, VF = VL.size(); I < VF; ++I) {
20753 Value *V = VL[I];
20773 for (Value *V : VL) {
21182 SmallVector<T *> VL;
21184 VL.clear()) {
21199 VL.push_back(cast<T>(I));
21203 unsigned NumElts = VL.size();
21215 if (NumElts > 1 && TryToVectorizeHelper(ArrayRef(VL), MaxVFOnly)) {
21218 VL.swap(Candidates);
21220 for (T *V : VL) {
21234 for (T *V : VL) {
21248 SmallVector<T *> VL;
21250 VL.clear()) {
21264 VL.push_back(cast<T>(I));
21266 unsigned NumElts = VL.size();
21267 if (NumElts > 1 && TryToVectorizeHelper(ArrayRef(VL),