Lines Matching defs:VectorizableTree
1414 return !VectorizableTree.empty() &&
1415 !VectorizableTree.front()->UserTreeIndices.empty();
1420 assert(!VectorizableTree.empty() && "No graph to get the first node from");
1421 return VectorizableTree.front()->Scalars;
1427 const TreeEntry &Root = *VectorizableTree.front().get();
1446 return MinBWs.at(VectorizableTree.front().get()).second;
1452 !VectorizableTree.front()->Scalars.front()->getType()->isIntegerTy() ||
1455 VectorizableTree.front()->Scalars.front()->getType()))
1457 VectorizableTree.front()->Scalars.front()->getType(),
1458 VectorizableTree.front()->getVectorFactor());
1461 VectorizableTree.front()->Scalars.front()->getContext(),
1463 VectorizableTree.front()->getVectorFactor());
1478 VectorizableTree.clear();
1503 unsigned getTreeSize() const { return VectorizableTree.size(); }
1598 /// \returns True if the VectorizableTree is both tiny and not fully
3171 /// \returns whether the VectorizableTree is fully vectorizable and will
3328 /// Points back to the VectorizableTree.
3340 /// The index of this treeEntry in VectorizableTree.
3592 /// Create a new VectorizableTree entry.
3625 VectorizableTree.push_back(std::make_unique<TreeEntry>(VectorizableTree));
3626 TreeEntry *Last = VectorizableTree.back().get();
3627 Last->Idx = VectorizableTree.size() - 1;
3716 TreeEntry::VecTreeTy VectorizableTree;
3721 for (unsigned Id = 0, IdE = VectorizableTree.size(); Id != IdE; ++Id) {
3722 VectorizableTree[Id]->dump();
4447 /// Add the VectorizableTree to the index iterator to be able to return
4452 ContainerTy &VectorizableTree;
4456 : ChildIteratorType::iterator_adaptor_base(W), VectorizableTree(VT) {}
4462 return R.VectorizableTree[0].get();
4490 return nodes_iterator(R->VectorizableTree.begin());
4494 return nodes_iterator(R->VectorizableTree.end());
4497 static unsigned size(BoUpSLP *R) { return R->VectorizableTree.size(); }
5966 for_each(VectorizableTree, [&, &TTIRef = *TTI](
6028 for (unsigned VF = VectorizableTree.front()->getVectorFactor();
6152 for (std::unique_ptr<TreeEntry> &TE : VectorizableTree) {
6285 for (const std::unique_ptr<TreeEntry> &TE : VectorizableTree) {
6500 if (IgnoreReorder && TE == VectorizableTree.front().get())
6541 if (IgnoreReorder && !VectorizableTree.front()->ReorderIndices.empty() &&
6542 VectorizableTree.front()->ReuseShuffleIndices.empty())
6543 VectorizableTree.front()->ReorderIndices.clear();
6559 for (auto &TEPtr : VectorizableTree) {
6962 GatheredLoadsEntriesFirst = VectorizableTree.size();
6967 Set.insert(VectorizableTree[Idx]->Scalars.begin(),
6968 VectorizableTree[Idx]->Scalars.end());
7304 VectorizableTree[std::get<0>(P)]->Scalars;
7342 VectorizableTree[std::get<0>(P)]
7347 unsigned Sz = VectorizableTree.size();
7349 if (Sz == VectorizableTree.size()) {
7393 const TreeEntry &E = *VectorizableTree[Idx];
7408 VectorizableTree.size())
7723 if (!IsGraphTransformMode && !VectorizableTree.empty()) {
7725 LoadEntriesToVectorize.insert(VectorizableTree.size());
7730 if (!IsGraphTransformMode && VectorizableTree.size() > 1) {
7732 LoadEntriesToVectorize.insert(VectorizableTree.size());
8254 if (VectorizableTree.size() < MinTreeSize)
9596 return VectorizableTree[Idx]->isSame(TE.Scalars);
9757 BaseGraphSize = VectorizableTree.size();
9795 TreeEntry &E = *VectorizableTree[Idx];
9802 TreeEntry &E = *VectorizableTree[Idx];
9920 unsigned PrevSize = VectorizableTree.size();
9924 if (PrevSize + 1 == VectorizableTree.size() &&
9925 VectorizableTree[PrevSize]->isGather() &&
9926 VectorizableTree[PrevSize]->hasState() &&
9927 VectorizableTree[PrevSize]->getOpcode() !=
9932 VectorizableTree.pop_back();
10052 if (VectorizableTree.size() <= 1 && VectorizableTree.front()->hasState() &&
10053 VectorizableTree.front()->getOpcode() == Instruction::Load)
10058 if ((VectorizableTree.size() <= SmallTree &&
10059 VectorizableTree.front()->Scalars.size() == SmallVF) ||
10060 (VectorizableTree.size() <= 2 && UserIgnoreList))
10063 if (VectorizableTree.front()->isNonPowOf2Vec() &&
10066 count_if(ArrayRef(VectorizableTree).drop_front(getCanonicalGraphSize()),
10081 for (std::unique_ptr<TreeEntry> &TE : VectorizableTree) {
10669 ArrayRef(R.VectorizableTree).take_front(E->Idx),
11022 find_if(VectorizableTree, [&](const std::unique_ptr<TreeEntry> &TE) {
11028 assert(It != VectorizableTree.end() && "Expected vectorizable entry.");
11781 for (const std::unique_ptr<TreeEntry> &TE : VectorizableTree) {
11937 << VectorizableTree.size() << " is fully vectorizable .\n");
11956 if (VectorizableTree.size() == 1 &&
11957 (VectorizableTree[0]->State == TreeEntry::Vectorize ||
11958 VectorizableTree[0]->State == TreeEntry::StridedVectorize ||
11960 AreVectorizableGathers(VectorizableTree[0].get(),
11961 VectorizableTree[0]->Scalars.size()) &&
11962 VectorizableTree[0]->getVectorFactor() > 2)))
11965 if (VectorizableTree.size() != 2)
11973 if (VectorizableTree[0]->State == TreeEntry::Vectorize &&
11974 AreVectorizableGathers(VectorizableTree[1].get(),
11975 VectorizableTree[0]->Scalars.size()))
11979 if (VectorizableTree[0]->isGather() ||
11980 (VectorizableTree[1]->isGather() &&
11981 VectorizableTree[0]->State != TreeEntry::ScatterVectorize &&
11982 VectorizableTree[0]->State != TreeEntry::StridedVectorize))
12032 unsigned NumElts = VectorizableTree[0]->Scalars.size();
12033 Value *FirstReduced = VectorizableTree[0]->Scalars[0];
12056 if (VectorizableTree.empty()) {
12063 if (VectorizableTree.size() == 2 &&
12064 isa<InsertElementInst>(VectorizableTree[0]->Scalars[0]) &&
12065 VectorizableTree[1]->isGather() &&
12066 (VectorizableTree[1]->getVectorFactor() <= 2 ||
12067 !(isSplat(VectorizableTree[1]->Scalars) ||
12068 allConstant(VectorizableTree[1]->Scalars))))
12077 !VectorizableTree.empty() &&
12078 all_of(VectorizableTree, [&](const std::unique_ptr<TreeEntry> &TE) {
12089 if (VectorizableTree.size() >= MinTreeSize)
12100 VectorizableTree.size() > 1 ||
12101 (VectorizableTree.size() == 1 && VectorizableTree.front()->hasState() &&
12102 !VectorizableTree.front()->isAltShuffle() &&
12103 VectorizableTree.front()->getOpcode() != Instruction::PHI &&
12104 VectorizableTree.front()->getOpcode() != Instruction::GetElementPtr &&
12105 allSameBlock(VectorizableTree.front()->Scalars));
12106 if (any_of(VectorizableTree, [&](const std::unique_ptr<TreeEntry> &TE) {
12116 if (VectorizableTree.back()->isGather() &&
12117 VectorizableTree.back()->hasState() &&
12118 VectorizableTree.back()->isAltShuffle() &&
12119 VectorizableTree.back()->getVectorFactor() > 2 &&
12120 allSameBlock(VectorizableTree.back()->Scalars) &&
12121 !VectorizableTree.back()->Scalars.front()->getType()->isVectorTy() &&
12123 getWidenedType(VectorizableTree.back()->Scalars.front()->getType(),
12124 VectorizableTree.back()->getVectorFactor()),
12125 APInt::getAllOnes(VectorizableTree.back()->getVectorFactor()),
12138 if (VectorizableTree.front()->isNonPowOf2Vec() &&
12140 count_if(ArrayRef(VectorizableTree).drop_front(getCanonicalGraphSize()),
12151 TreeEntry &E = *VectorizableTree[Idx];
12168 unsigned BundleWidth = VectorizableTree.front()->Scalars.size();
12174 // The entries in VectorizableTree are not necessarily ordered by their
12181 for (const auto &TEPtr : VectorizableTree) {
12446 << VectorizableTree.size() << ".\n");
12448 unsigned BundleWidth = VectorizableTree[0]->Scalars.size();
12451 for (unsigned I = 0, E = VectorizableTree.size(); I < E; ++I) {
12452 TreeEntry &TE = *VectorizableTree[I];
12679 VectorizableTree.front()->Scalars.size() > 2)) &&
12680 VectorizableTree.front()->getOpcode() == Instruction::PHI &&
12688 VectorizableTree.front()->getMainOp())
12759 const TreeEntry &Root = *VectorizableTree.front();
12871 const TreeEntry &E = *VectorizableTree.front();
13078 const EdgeInfo &TEUseEI = TE == VectorizableTree.front().get()
13593 if (TE == VectorizableTree.front().get() &&
13595 none_of(ArrayRef(VectorizableTree).drop_front(),
13606 TE == VectorizableTree.front().get()) &&
14027 none_of(VectorizableTree, [&](const std::unique_ptr<TreeEntry> &TE) {
14238 count_if(R.VectorizableTree,
14624 any_of(VectorizableTree,
14657 return std::make_pair(VectorizableTree[P.first].get(),
14710 find_if(VectorizableTree, [&](const std::unique_ptr<TreeEntry> &TE) {
14714 assert(It != VectorizableTree.end() && "Expected gather node operand.");
14723 auto *I = find_if(VectorizableTree,
14727 assert(I != VectorizableTree.end() && "Gather node is not in the graph.");
14747 .slice(Idx, VectorizableTree[EIdx]->getVectorFactor()),
14753 return std::make_pair(VectorizableTree[P.first].get(), P.second);
14786 find_if(VectorizableTree, [=](const std::unique_ptr<TreeEntry> &TE) {
14791 if (It == VectorizableTree.end())
15228 (void)vectorizeTree(VectorizableTree[EIdx].get(), PostponedPHIs);
15293 return std::make_pair(VectorizableTree[P.first].get(), P.second);
15322 E != VectorizableTree.front().get() ||
16299 for (const std::unique_ptr<TreeEntry> &TE : VectorizableTree) {
16310 (void)vectorizeTree(VectorizableTree[0].get(), /*PostponedPHIs=*/true);
16311 for (const std::unique_ptr<TreeEntry> &TE : VectorizableTree)
16818 for (auto &TEPtr : VectorizableTree) {
16862 if (auto *V = dyn_cast<Instruction>(VectorizableTree[0]->VectorizedValue))
16870 !(VectorizableTree.front()->isGather() &&
16873 VectorizableTree.front().get()) ||
16876 return EI.UserTE == VectorizableTree.front().get() &&
16881 VectorizableTree.front()->isGather() &&
16882 is_contained(VectorizableTree.front()->Scalars, I)))
16912 const TreeEntry &RootTE = *VectorizableTree.front();
18113 VectorizableTree.front()->hasState() &&
18114 (VectorizableTree.front()->getOpcode() == Instruction::Store ||
18115 VectorizableTree.front()->getOpcode() == Instruction::InsertElement);
18123 if (IsStoreOrInsertElt && !VectorizableTree.front()->isGather())
18127 if (VectorizableTree[NodeIdx]->isGather() ||
18128 (NodeIdx == 0 && !VectorizableTree[NodeIdx]->UserTreeIndices.empty()) ||
18129 (NodeIdx != 0 && any_of(VectorizableTree[NodeIdx]->UserTreeIndices,
18142 VectorizableTree[NodeIdx]->State == TreeEntry::Vectorize &&
18143 VectorizableTree[NodeIdx]->getOpcode() == Instruction::Trunc) {
18152 if (AnalyzedMinBWVals.contains(VectorizableTree[NodeIdx]->Scalars.front()))
18316 isa<IntegerType>(VectorizableTree.front()->Scalars.front()->getType())) {
18324 VectorizableTree.front()->State == TreeEntry::Vectorize &&
18325 VectorizableTree.front()->getOpcode() == Instruction::ZExt &&
18326 cast<CastInst>(VectorizableTree.front()->getMainOp())->getSrcTy() ==
18353 while (NodeIdx < VectorizableTree.size() &&
18354 VectorizableTree[NodeIdx]->State == TreeEntry::Vectorize &&
18355 VectorizableTree[NodeIdx]->getOpcode() == Instruction::Trunc) {
18361 while (NodeIdx < VectorizableTree.size()) {
18362 ArrayRef<Value *> TreeRoot = VectorizableTree[NodeIdx]->Scalars;
18367 VectorizableTree.front()->Scalars.front()->getType()))
18370 *VectorizableTree[NodeIdx], IsTopRoot, IsProfitableToDemoteRoot, Limit,
18380 if (all_of(VectorizableTree[Idx]->Scalars, [&](Value *V) {
18396 NodeIdx = VectorizableTree.size();
18405 NodeIdx < VectorizableTree.size() &&
18406 any_of(VectorizableTree[NodeIdx]->UserTreeIndices,
18413 NodeIdx < VectorizableTree.size() &&
18415 VectorizableTree[NodeIdx]->UserTreeIndices,
18446 TreeEntry *TE = VectorizableTree[Idx].get();