Lines Matching defs:loops
9 // This file implements vectorization of loops, operations and data types to
74 /// 1. the associated loops have dependency semantics that do not prevent
76 /// 2. the associate loops have been sliced in chunks of static sizes that are
78 /// 3. the inner loops, in the unroll-and-jam analogy of 2, are captured by
118 /// 1. at the MLIR level into a combination of loops, unrolling, DmaStartOp +
158 /// references along fastest varying dimensions and loops with recursive nested
250 /// loops that have a. conformable loop annotations attached (e.g. parallel,
257 /// a. applying iterative rewriting of the loops and all their nested
259 /// coarsening the loops and converting operations and operands to their
270 /// * Uniform operands (only induction variables of loops not mapped to
276 /// TODO: Support more complex loops with divergent lbs and/or ubs.
292 /// The choice of loop transformation to apply for coarsening vectorized loops
325 /// consequences on dependence analysis and fusability of loops: fusable
326 /// loops probably need to have the same step (because we don't want to
510 /// Vectorizing reduction loops along the reduction dimension is supported if:
516 /// during vectorization of such loops:
523 /// passing a map from loops to reductions to utility functions, or by passing
728 /// vectorized reduction loops with iter_args.
755 // Maps results of reduction loops to their new scalar counterparts.
758 // Maps the newly created vector loops to their vector dimension.
761 // Maps the new vectorized loops to the corresponding vector masks if it is
865 /// vectorized reduction loops with iter_args.
1019 /// of unaligned loops. If a mask is not required then `nullptr` is returned.
1029 "Creating a mask for loops with non-unit original step size is not "
1095 // TODO: For now, only values that are induction variables of loops not in
1096 // `loopToVectorDim` or invariants to all the loops in the vectorization
1216 // Compute permutation map using the information of new vector loops.
1261 // Compute permutation map using the information of new vector loops.
1305 // TODO: Vectorization of reduction loops is not supported for non-unit steps.
1344 // For reduction loops we need to pass a vector of neutral elements as an
1529 /// Recursive implementation to convert all the nested loops in 'match' to a 2D
1535 std::vector<SmallVector<AffineForOp, 2>> &loops) {
1537 assert(currentLevel <= loops.size() && "Unexpected currentLevel");
1538 if (currentLevel == loops.size())
1539 loops.emplace_back();
1542 loops[currentLevel].push_back(cast<AffineForOp>(match.getMatchedOperation()));
1544 getMatchedAffineLoopsRec(childMatch, currentLevel + 1, loops);
1548 /// Converts all the nested loops in 'match' to a 2D vector container that
1550 /// in 'match'. This means that every loop in 'loops[i]' will have a parent loop
1551 /// in 'loops[i-1]'. A loop in 'loops[i]' may or may not have a child loop in
1552 /// 'loops[i+1]'.
1555 std::vector<SmallVector<AffineForOp, 2>> &loops) {
1556 getMatchedAffineLoopsRec(match, /*currLoopDepth=*/0, loops);
1559 /// Internal implementation to vectorize affine loops from a single loop nest
1562 vectorizeLoopNest(std::vector<SmallVector<AffineForOp, 2>> &loops,
1564 assert(loops[0].size() == 1 && "Expected single root loop");
1565 AffineForOp rootLoop = loops[0][0];
1614 // Replace results of reduction loops with the scalar values computed using
1630 /// Extracts the matched loops and vectorizes them following a topological
1682 /// Internal implementation to vectorize affine loops in 'loops' using the n-D
1684 /// factor is applied inner-to-outer to the loops of each loop nest.
1686 /// vectorization order. `reductionLoops` can be provided to specify loops which
1688 static void vectorizeLoops(Operation *parentOp, DenseSet<Operation *> &loops,
1695 // Compute 1-D, 2-D or 3-D loop pattern to be matched on the target loops.
1697 makePattern(loops, vectorSizes.size(), fastestVaryingPattern);
1793 /// Verify that affine loops in 'loops' meet the nesting criteria expected by
1800 verifyLoopNesting(const std::vector<SmallVector<AffineForOp, 2>> &loops) {
1802 if (loops.empty())
1806 if (loops[0].size() != 1)
1809 // Traverse loops outer-to-inner to check some invariants.
1810 for (int i = 1, end = loops.size(); i < end; ++i) {
1811 for (AffineForOp loop : loops[i]) {
1812 // Check that each loop at this level is nested in one of the loops from
1814 if (none_of(loops[i - 1], [&](AffineForOp maybeParent) {
1821 for (AffineForOp sibling : loops[i]) {
1832 /// External utility to vectorize affine loops in 'loops' using the n-D
1834 /// factor is applied inner-to-outer to the loops of each loop nest.
1837 /// If `reductionLoops` is not empty, the given reduction loops may be
1841 Operation *parentOp, DenseSet<Operation *> &loops,
1846 vectorizeLoops(parentOp, loops, vectorSizes, fastestVaryingPattern,
1850 /// External utility to vectorize affine loops from a single loop nest using an
1853 /// the nesting level relative to the loops to be vectorized. The second
1854 /// dimension contains the loops. This means that:
1855 /// a) every loop in 'loops[i]' must have a parent loop in 'loops[i-1]',
1856 /// b) a loop in 'loops[i]' may or may not have a child loop in 'loops[i+1]'.
1878 /// loops = {{%i0}, {%i2, %i3}}, to vectorize the outermost and the two
1879 /// innermost loops;
1880 /// loops = {{%i1}, {%i2, %i3}}, to vectorize the middle and the two innermost
1881 /// loops;
1882 /// loops = {{%i2}}, to vectorize only the first innermost loop;
1883 /// loops = {{%i3}}, to vectorize only the second innermost loop;
1884 /// loops = {{%i1}}, to vectorize only the middle loop.
1886 std::vector<SmallVector<AffineForOp, 2>> &loops,
1890 if (failed(verifyLoopNesting(loops)))
1892 return vectorizeLoopNest(loops, strategy);