Lines Matching defs:SLP
1 //===- SLPVectorizer.cpp - A bottom up SLP Vectorizer ---------------------===//
9 // This pass implements the Bottom Up SLP vectorizer. It detects consecutive
12 // was found, the SLP vectorizer performs vectorization on the tree.
15 // "Loop-Aware SLP in GCC" by Ira Rosen, Dorit Nuzman, Ayal Zaks.
110 #define DEBUG_TYPE "SLP"
115 "Controls which SLP graphs should be vectorized.");
119 cl::desc("Run the SLP vectorization passes"));
132 cl::desc("When true, SLP vectorizer bypasses profitability checks based on "
150 cl::desc("Maximum SLP vectorization factor (0=unlimited)"));
158 cl::desc("Limit the size of the SLP scheduling region per block"));
198 cl::desc("Display the SLP trees with Graphviz"));
224 /// Predicate for the element types that the SLP vectorizer supports.
303 // But the element have different meaning for SLP (scalar) and REVEC
857 /// instruction for SLP vectorization.
1335 /// Bottom Up SLP Vectorizer.
1919 /// Look-ahead SLP: Auto-vectorization in the presence of commutative
2092 /// after removing duplicates. Currently the SLP vectorizer supports only
3197 /// potentially vectorized in a future SLP-tree. If found, it keeps track of
3581 dbgs() << "SLP: " << Banner << ":\n";
3583 dbgs() << "SLP: Costs:\n";
3584 dbgs() << "SLP: ReuseShuffleCost = " << ReuseShuffleCost << "\n";
3585 dbgs() << "SLP: VectorCost = " << VecCost << "\n";
3586 dbgs() << "SLP: ScalarCost = " << ScalarCost << "\n";
3587 dbgs() << "SLP: ReuseShuffleCost + VecCost - ScalarCost = "
3850 // globally through SLP because we don't perform any action which
4145 LLVM_DEBUG(dbgs() << "SLP: schedule " << *SD << "\n");
4165 << "SLP: gets ready (def): " << *DepBundle << "\n");
4212 << "SLP: gets ready (mem): " << *DepBundle << "\n");
4225 << "SLP: gets ready (ctl): " << *DepBundle << "\n");
4267 << "SLP: initially in ready list: " << *SD << "\n");
4282 tryScheduleBundle(ArrayRef<Value *> VL, BoUpSLP *SLP,
4304 BoUpSLP *SLP);
4960 /// type \p Tp and optional \p Mask. Adds SLP-specifc cost estimation for insert
6580 LLVM_DEBUG(dbgs() << "SLP: Need to extract: Extra arg from lane "
6587 LLVM_DEBUG(dbgs() << "SLP: Checking user:" << *U << ".\n");
6609 LLVM_DEBUG(dbgs() << "SLP: \tInternal user will be removed:" << *U
6628 LLVM_DEBUG(dbgs() << "SLP: Need to extract:" << *UserInst
7205 LLVM_DEBUG(dbgs() << "SLP: Trying to vectorize gathered loads ("
7661 << "SLP: Need to swizzle PHINodes (terminator use).\n");
7678 LLVM_DEBUG(dbgs() << "SLP: Gather extract sequence.\n");
7695 LLVM_DEBUG(dbgs() << "SLP: Gather of insertelement vectors with "
7705 LLVM_DEBUG(dbgs() << "SLP: Gather of insertelement vectors with "
7741 LLVM_DEBUG(dbgs() << "SLP: Gathering loads of non-packed type.\n");
7746 LLVM_DEBUG(dbgs() << "SLP: Gathering non-simple loads.\n");
7748 LLVM_DEBUG(dbgs() << "SLP: Gathering non-consecutive loads.\n");
7774 dbgs() << "SLP: Gathering casts with different src types.\n");
7792 LLVM_DEBUG(dbgs() << "SLP: Gathering cmp with different predicate.\n");
7833 LLVM_DEBUG(dbgs() << "SLP: not-vectorizable GEP (nested indexes).\n");
7847 LLVM_DEBUG(dbgs() << "SLP: not-vectorizable GEP (different types).\n");
7866 dbgs() << "SLP: not-vectorizable GEP (non-constant indexes).\n");
7880 LLVM_DEBUG(dbgs() << "SLP: Gathering stores of non-packed type.\n");
7888 LLVM_DEBUG(dbgs() << "SLP: Gathering non-simple stores.\n");
7912 LLVM_DEBUG(dbgs() << "SLP: Non-consecutive store.\n");
7934 LLVM_DEBUG(dbgs() << "SLP: Non-vectorizable call.\n");
7950 LLVM_DEBUG(dbgs() << "SLP: mismatched calls:" << *CI << "!=" << *V
7961 << "SLP: mismatched arguments in call:" << *CI
7972 LLVM_DEBUG(dbgs() << "SLP: mismatched bundle operands in calls:" << *CI
7987 LLVM_DEBUG(dbgs() << "SLP: ShuffleVector are not vectorized.\n");
7993 << "SLP: ShuffleVector not vectorized, operands are buildvector and "
8001 LLVM_DEBUG(dbgs() << "SLP: Gathering unknown instruction.\n");
8131 LLVM_DEBUG(dbgs() << "SLP: Reshuffling scalars not yet supported "
8136 LLVM_DEBUG(dbgs() << "SLP: Shuffle for reused scalars.\n");
8157 LLVM_DEBUG(dbgs() << "SLP: Scalar used twice in bundle.\n");
8165 LLVM_DEBUG(dbgs() << "SLP: Scalar used twice in bundle.\n");
8180 LLVM_DEBUG(dbgs() << "SLP: bundle in catchswitch block.\n");
8187 LLVM_DEBUG(dbgs() << "SLP: \tChecking bundle: " << *S.getMainOp() << ".\n");
8192 LLVM_DEBUG(dbgs() << "SLP: Perfect diamond merge at " << *S.getMainOp()
8200 LLVM_DEBUG(dbgs() << "SLP: Gathering due to full overlap.\n");
8220 LLVM_DEBUG(dbgs() << "SLP: Gathering due to max recursion depth.\n");
8231 LLVM_DEBUG(dbgs() << "SLP: Gathering due to scalable vector type.\n");
8240 LLVM_DEBUG(dbgs() << "SLP: Gathering due to vector type.\n");
8330 LLVM_DEBUG(dbgs() << "SLP: Gathering due to C,S,B,O, small shuffle. \n");
8341 LLVM_DEBUG(dbgs() << "SLP: The instruction (" << *V
8358 LLVM_DEBUG(dbgs() << "SLP: The instruction (" << *V
8371 LLVM_DEBUG(dbgs() << "SLP: Gathering due to gathered scalar.\n");
8404 LLVM_DEBUG(dbgs() << "SLP: bundle in unreachable block.\n");
8437 LLVM_DEBUG(dbgs() << "SLP: We are not able to schedule this bundle!\n");
8449 LLVM_DEBUG(dbgs() << "SLP: We are able to schedule this bundle.\n");
8475 LLVM_DEBUG(dbgs() << "SLP: added a new TreeEntry (PHINode).\n";
8492 LLVM_DEBUG(dbgs() << "SLP: Reusing or shuffling extract sequence.\n");
8495 dbgs() << "SLP: Reusing or shuffling of reordered extract sequence "
8507 LLVM_DEBUG(dbgs() << "SLP: added a new TreeEntry "
8540 LLVM_DEBUG(dbgs() << "SLP: added a new TreeEntry (InsertElementInst).\n";
8561 LLVM_DEBUG(dbgs() << "SLP: added a new TreeEntry (LoadInst).\n";
8565 << "SLP: added a new TreeEntry (jumbled LoadInst).\n";
8572 LLVM_DEBUG(dbgs() << "SLP: added a new TreeEntry (strided LoadInst).\n";
8581 << "SLP: added a new TreeEntry (non-consecutive LoadInst).\n";
8626 LLVM_DEBUG(dbgs() << "SLP: added a new TreeEntry (CastInst).\n";
8654 LLVM_DEBUG(dbgs() << "SLP: added a new TreeEntry (CmpInst).\n";
8726 dbgs() << "SLP: added a new TreeEntry "
8738 LLVM_DEBUG(dbgs() << "SLP: added a new TreeEntry (GetElementPtrInst).\n";
8798 LLVM_DEBUG(dbgs() << "SLP: added a new TreeEntry (StoreInst).\n";
8802 dbgs() << "SLP: added a new TreeEntry (jumbled StoreInst).\n";
8816 LLVM_DEBUG(dbgs() << "SLP: added a new TreeEntry (CallInst).\n";
8832 LLVM_DEBUG(dbgs() << "SLP: added a new TreeEntry (isAltShuffle).\n";
8837 dbgs() << "SLP: added a new TreeEntry (ShuffleVectorInst).\n";
11811 dbgs() << "SLP: diamond match for alternate node found.\n";
11852 << "SLP: alternate extension, which should be truncated.\n";
11936 LLVM_DEBUG(dbgs() << "SLP: Check whether the tree with height "
12022 LLVM_DEBUG(dbgs() << "SLP: Assume load combining for tree starting at "
12215 dbgs() << "SLP: #LV: " << LiveValues.size();
12445 LLVM_DEBUG(dbgs() << "SLP: Calculating cost for tree of size "
12457 dbgs() << "SLP: Skipping cost for combined node that starts with "
12459 TE.dump(); dbgs() << "SLP: Current total cost = " << Cost << "\n");
12468 LLVM_DEBUG(dbgs() << "SLP: Adding cost 0 for bundle "
12470 << "SLP: Current total cost = " << Cost << "\n");
12482 LLVM_DEBUG(dbgs() << "SLP: Adding cost " << C << " for bundle "
12484 << "SLP: Current total cost = " << Cost << "\n");
12576 LLVM_DEBUG(dbgs() << "SLP: Adding cost " << C
12799 dbgs() << "SLP: Adding cost " << C
12801 TE->dump(); dbgs() << "SLP: Current total cost = " << Cost << "\n");
12828 LLVM_DEBUG(dbgs() << "SLP: Adding cost " << C
12832 dbgs() << "SLP: Current total cost = " << Cost << "\n");
12846 LLVM_DEBUG(dbgs() << "SLP: Adding cost " << C
12850 dbgs() << "SLP: Current total cost = " << Cost << "\n");
12914 LLVM_DEBUG(dbgs() << "SLP: Adding cost " << CastCost
12917 dbgs() << "SLP: Current total cost = " << Cost << "\n");
12926 OS << "SLP: Spill Cost = " << SpillCost << ".\n"
12927 << "SLP: Extract Cost = " << ExtractCost << ".\n"
12928 << "SLP: Total Cost = " << Cost << ".\n";
12932 ViewGraph(this, "SLP" + F->getName(), false, Str);
14908 LLVM_DEBUG(dbgs() << "SLP: perfect diamond match for gather bundle "
15249 LLVM_DEBUG(dbgs() << "SLP: Diamond merged for " << *E->Scalars[0] << ".\n");
15362 LLVM_DEBUG(dbgs() << "SLP: Diamond merged for " << *VL0 << ".\n");
15585 LLVM_DEBUG(dbgs() << "SLP: Diamond merged for " << *VL0 << ".\n");
15632 LLVM_DEBUG(dbgs() << "SLP: Diamond merged for " << *VL0 << ".\n");
15637 LLVM_DEBUG(dbgs() << "SLP: Diamond merged for " << *VL0 << ".\n");
15677 LLVM_DEBUG(dbgs() << "SLP: Diamond merged for " << *VL0 << ".\n");
15682 LLVM_DEBUG(dbgs() << "SLP: Diamond merged for " << *VL0 << ".\n");
15687 LLVM_DEBUG(dbgs() << "SLP: Diamond merged for " << *VL0 << ".\n");
15731 LLVM_DEBUG(dbgs() << "SLP: Diamond merged for " << *VL0 << ".\n");
15754 LLVM_DEBUG(dbgs() << "SLP: Diamond merged for " << *VL0 << ".\n");
15794 LLVM_DEBUG(dbgs() << "SLP: Diamond merged for " << *VL0 << ".\n");
15799 LLVM_DEBUG(dbgs() << "SLP: Diamond merged for " << *VL0 << ".\n");
15904 LLVM_DEBUG(dbgs() << "SLP: Diamond merged for " << *VL0 << ".\n");
15989 LLVM_DEBUG(dbgs() << "SLP: Diamond merged for " << *VL0 << ".\n");
15997 LLVM_DEBUG(dbgs() << "SLP: Diamond merged for " << *VL0 << ".\n");
16059 LLVM_DEBUG(dbgs() << "SLP: Diamond merged for " << *VL0 << ".\n");
16072 LLVM_DEBUG(dbgs() << "SLP: OpVec[" << I << "]: " << *OpVec << "\n");
16107 LLVM_DEBUG(dbgs() << "SLP: Diamond merged for " << *VL0 << ".\n");
16140 LLVM_DEBUG(dbgs() << "SLP: Diamond merged for " << *VL0 << ".\n");
16149 LLVM_DEBUG(dbgs() << "SLP: Diamond merged for " << *VL0 << ".\n");
16424 LLVM_DEBUG(dbgs() << "SLP: Extracting " << ExternalUses.size()
16707 LLVM_DEBUG(dbgs() << "SLP: Replaced:" << *User << ".\n");
16843 LLVM_DEBUG(dbgs() << "SLP: \tvalidating user:" << *U << ".\n");
16854 LLVM_DEBUG(dbgs() << "SLP: \tErasing scalar:" << *Scalar << ".\n");
16930 LLVM_DEBUG(dbgs() << "SLP: Optimizing " << GatherShuffleExtractSeq.size()
17109 BoUpSLP::BlockScheduling::tryScheduleBundle(ArrayRef<Value *> VL, BoUpSLP *SLP,
17119 LLVM_DEBUG(dbgs() << "SLP: bundle: " << *S.getMainOp() << "\n");
17121 auto TryScheduleBundleImpl = [this, OldScheduleEnd, SLP](bool ReSchedule,
17135 LLVM_DEBUG(dbgs() << "SLP: try schedule bundle " << *Bundle
17137 calculateDependencies(Bundle, /*InsertInReadyList=*/true, SLP);
17192 LLVM_DEBUG(dbgs() << "SLP: reset schedule because " << *BundleMember
17215 LLVM_DEBUG(dbgs() << "SLP: cancel scheduling of " << *Bundle << "\n");
17266 LLVM_DEBUG(dbgs() << "SLP: initialize schedule region to " << *I << "\n");
17288 LLVM_DEBUG(dbgs() << "SLP: exceeded schedule region size limit\n");
17303 LLVM_DEBUG(dbgs() << "SLP: extend schedule region start to " << *I
17316 LLVM_DEBUG(dbgs() << "SLP: extend schedule region end to " << *I << "\n");
17366 BoUpSLP *SLP) {
17380 LLVM_DEBUG(dbgs() << "SLP: update deps of " << *BundleMember
17415 if (isSafeToSpeculativelyExecute(I, &*BB->begin(), SLP->AC))
17486 // SLP->isAliased (which is the expensive part in this loop).
17494 SLP->isAliased(SrcLoc, SrcInst, DepDest->Inst)))) {
17532 LLVM_DEBUG(dbgs() << "SLP: gets ready on update: " << *SD->Inst
17556 LLVM_DEBUG(dbgs() << "SLP: schedule block " << BS->BB->getName() << "\n");
18505 dbgs() << "SLP: Didn't find any vector registers for target, abort.\n");
18513 LLVM_DEBUG(dbgs() << "SLP: Analyzing blocks in " << F.getName() << ".\n");
18536 LLVM_DEBUG(dbgs() << "SLP: Found stores for " << Stores.size()
18548 LLVM_DEBUG(dbgs() << "SLP: Found GEPs for " << GEPs.size()
18556 LLVM_DEBUG(dbgs() << "SLP: vectorized \"" << F.getName() << "\"\n");
18566 LLVM_DEBUG(dbgs() << "SLP: Analyzing a store chain of length " << Chain.size()
18583 LLVM_DEBUG(dbgs() << "SLP: Analyzing " << VF << " stores at offset " << Idx
18635 LLVM_DEBUG(dbgs() << "SLP: Found cost = " << Cost << " for VF=" << VF << "\n");
18637 LLVM_DEBUG(dbgs() << "SLP: Decided to vectorize cost = " << Cost << "\n");
18643 << "Stores SLP vectorized with cost " << NV("Cost", Cost)
18747 LLVM_DEBUG(dbgs() << "SLP: Vectorization infeasible as MaxVF (" << MaxVF
18769 LLVM_DEBUG(dbgs() << "SLP: Vectorization infeasible as MaxVF (" << MaxVF
19126 LLVM_DEBUG(dbgs() << "SLP: Trying to vectorize a list of length = "
19148 << "Cannot SLP vectorize list: type "
19164 << "Cannot SLP vectorize list: vectorization factor "
19211 LLVM_DEBUG(dbgs() << "SLP: Analyzing " << ActualVF << " operations "
19229 LLVM_DEBUG(dbgs() << "SLP: Found cost = " << Cost
19232 LLVM_DEBUG(dbgs() << "SLP: Vectorizing list at cost:" << Cost << ".\n");
19235 << "SLP vectorized with cost " << ore::NV("Cost", Cost)
19258 << "Cannot SLP vectorize list: vectorization was impossible"
19514 // This matches either cmp+select or intrinsics. SLP is expected to handle
19530 // During the intermediate stages of SLP, it's very common to have
20217 LLVM_DEBUG(dbgs() << "SLP: Found cost = " << Cost
20251 LLVM_DEBUG(dbgs() << "SLP: Vectorizing horizontal reduction at cost:"
20610 LLVM_DEBUG(dbgs() << "SLP: Adding cost " << VectorCost - ScalarCost
20650 LLVM_DEBUG(dbgs() << "SLP: Add (to-mul) " << Cnt << "of "
20656 LLVM_DEBUG(dbgs() << "SLP: Xor " << Cnt << "of " << VectorizedValue
20665 LLVM_DEBUG(dbgs() << "SLP: FAdd (to-fmul) " << Cnt << "of "
20720 LLVM_DEBUG(dbgs() << "SLP: Add (to-mul) " << Scale << "of "
20727 LLVM_DEBUG(dbgs() << "SLP: And/or of same " << VectorizedValue
20739 LLVM_DEBUG(dbgs() << "SLP: Max/min of same " << VectorizedValue
20760 LLVM_DEBUG(dbgs() << "SLP: Xor <"; for (int I
21137 << "Cannot SLP vectorize list: only 2 elements of buildvalue, "
21142 LLVM_DEBUG(dbgs() << "SLP: array mappable to vector: " << *IVI << "\n");
21161 << "Cannot SLP vectorize list: only 2 elements of buildvector, "
21166 LLVM_DEBUG(dbgs() << "SLP: array mappable to vector: " << *IEI << "\n");
21204 LLVM_DEBUG(dbgs() << "SLP: Trying to vectorize starting at nodes ("
21795 LLVM_DEBUG(dbgs() << "SLP: Analyzing a getelementptr list of length "
21961 LLVM_DEBUG(dbgs() << "SLP: Analyzing a store chain of length "