Lines Matching defs:rewriter
82 TrivialPatternRewriter rewriter(operation->getContext());
83 rewriter.setInsertionPoint(operation);
84 auto result = pattern.returningMatchAndRewrite(op, rewriter);
332 transform::TransformRewriter &rewriter,
335 OpBuilder::Listener *previousListener = rewriter.getListener();
337 llvm::make_scope_exit([&]() { rewriter.setListener(previousListener); });
339 rewriter.setListener(&newOpsListener);
372 linalg::bufferizeToAllocation(rewriter, options, op, memorySpace);
415 transform::DecomposeOp::applyToOne(transform::TransformRewriter &rewriter,
456 transform::TransformRewriter &rewriter, Operation *target,
461 failed(rewriter.notifyMatchFailure(target,
467 decomposableOp.decomposeOperation(rewriter);
471 rewriter.replaceOp(decomposableOp, *maybeNewResults);
492 transform::TransformRewriter &rewriter, TransformResults &transformResults,
503 rewriter, target, state)))
518 RewriterBase &rewriter, Operation *transformOp, Range &&payloadOps,
530 rewriter.setInsertionPoint(target);
542 rewriter.replaceAllUsesWith(res, replacement);
544 rewriter.eraseOp(toReplace);
565 transform::FuseOp::apply(transform::TransformRewriter &rewriter,
576 getAsIndexOpFoldResult(rewriter.getContext(), tileSizes);
582 MLIRContext *context = rewriter.getContext();
590 rewriter, getOperation(), state.getPayloadOps(getTarget()),
594 return tileConsumerAndFuseProducersUsingSCF(rewriter, tilingInterfaceOp,
636 RewriterBase &rewriter, Diagnostic &diag, Operation *producerOp,
655 OpBuilder::InsertionGuard g(rewriter);
656 rewriter.setInsertionPoint(forallOp);
668 auto newforallOp = rewriter.create<scf::ForallOp>(
671 rewriter.eraseBlock(newforallOp.getBody());
680 rewriter.replaceUsesWithIf(newOuts.back(), bbArgs.back(),
691 rewriter.setInsertionPoint(firstYieldOp);
694 SmallVector<OpFoldResult> strides(offsets.size(), rewriter.getIndexAttr(1));
695 rewriter.create<tensor::ParallelInsertSliceOp>(firstYieldOp->getLoc(), src,
699 rewriter.replaceAllUsesWith(result.value(),
702 rewriter.replaceUsesWithIf(producerOp->getResult(resultNumber),
718 tileAndFuseFirstExtractUse(RewriterBase &rewriter, Diagnostic &diag,
745 OpBuilder::InsertionGuard guard(rewriter);
746 rewriter.setInsertionPoint(sliceOpToTile);
757 tileableProducer.generateResultTileValue(rewriter, resultNumber, offsets,
774 rewriter, sliceOpToTile->getLoc(), tileAndFuseResult->tiledValues[0],
783 rewriter.replaceOp(sliceOpToTile, *maybeRankReduced);
787 rewriter, diag, producerOp, containingOp, *tileAndFuseResult,
801 RewriterBase &rewriter, Diagnostic &diag, Operation *producerOp,
850 OpBuilder::InsertionGuard guard(rewriter);
851 rewriter.setInsertionPoint(sliceOpToTile);
861 rewriter, tileableProducer->getLoc(), tileableProducer,
871 cast<TilingInterface>(rewriter.clone(*tileableProducer, bvm));
873 llvm::make_scope_exit([&]() { rewriter.eraseOp(tileableProducerClone); });
878 rewriter, resultNumber, sliceOpToTile.getMixedOffsets(),
888 rewriter, sliceOpToTile->getLoc(), tileAndFuseResult->tiledValues[0],
891 rewriter.replaceOp(sliceOpToTile, *maybeRankReduced);
894 rewriter.modifyOpInPlace(containingOp, [&]() {
902 static Operation *cloneAndFuseFirstUse(RewriterBase &rewriter, Diagnostic &diag,
941 OpBuilder::InsertionGuard guard(rewriter);
942 rewriter.setInsertionPoint(use->getOwner());
943 fusedOp = rewriter.clone(*producerOp);
944 rewriter.modifyOpInPlace(
956 transform::FuseIntoContainingOp::apply(transform::TransformRewriter &rewriter,
1021 tileAndFuseFirstExtractUse(rewriter, diag, producerOp, containingOp);
1034 rewriter.notifyPayloadOperationReplaced(containingOp,
1039 rewriter.eraseOp(containingOp);
1047 rewriter, diag, producerOp, containingOp);
1056 cloneAndFuseFirstUse(rewriter, diag, producerOp, containingOp);
1083 transform::GeneralizeOp::applyToOne(transform::TransformRewriter &rewriter,
1092 rewriter.setInsertionPoint(target);
1093 FailureOr<LinalgOp> generic = generalizeNamedOp(rewriter, target);
1106 transform::SpecializeOp::applyToOne(transform::TransformRewriter &rewriter,
1115 rewriter.setInsertionPoint(target);
1117 specializeGenericOp(rewriter, cast<GenericOp>(target));
1130 transform::InterchangeOp::applyToOne(transform::TransformRewriter &rewriter,
1150 rewriter, target, SmallVector<unsigned>(interchangeVector));
1174 transform::TransformRewriter &rewriter, tensor::PackOp target,
1177 rewriter.setInsertionPoint(target);
1180 lowerPack(rewriter, target, lowerPadLikeWithInsertSlice);
1196 transform::TransformRewriter &rewriter, tensor::UnPackOp target,
1199 rewriter.setInsertionPoint(target);
1202 lowerUnPack(rewriter, target, lowerUnpadLikeWithExtractSlice);
1239 transform::MatchOp::apply(transform::TransformRewriter &rewriter,
1368 transform::TransformRewriter &rewriter, LinalgOp target,
1470 transform::PackOp::apply(transform::TransformRewriter &rewriter,
1500 rewriter.setInsertionPoint(linalgOp);
1501 FailureOr<PackResult> maybeResult = pack(rewriter, linalgOp, packedSizes);
1545 PackGreedilyOp::apply(transform::TransformRewriter &rewriter,
1555 rewriter.setInsertionPointAfter(linalgOp);
1559 /*rewriter=*/rewriter,
1644 transform::PackTransposeOp::apply(transform::TransformRewriter &rewriter,
1724 rewriter, packOp, linalgOp, unPackOp, getOuterPerm(), getInnerPerm());
1807 transform::PadOp::apply(transform::TransformRewriter &rewriter,
1898 if (failed(rewriteAsPaddedOp(rewriter, linalgTarget, options, paddedOp,
1910 rewriter.replaceOp(linalgTarget, replacements);
1976 transform::TransformRewriter &rewriter,
1994 linalg::detail::buildPackingLoopNest(rewriter, padOp, loopOp,
2031 transform::HoistPadOp::applyToOne(transform::TransformRewriter &rewriter,
2038 hoistPaddingOnTensors(rewriter, target, getNumLoops(), getTranspose(),
2046 rewriter.replaceOp(target, *result);
2069 transform::PromoteOp::applyToOne(transform::TransformRewriter &rewriter,
2122 rewriter.setInsertionPoint(target);
2123 FailureOr<LinalgOp> res = promoteSubViews(rewriter, target, promotionOptions);
2135 transform::ReplaceOp::apply(transform::TransformRewriter &rewriter,
2156 rewriter.setInsertionPoint(target);
2157 Operation *replacement = rewriter.clone(*pattern);
2158 rewriter.replaceOp(target, replacement->getResults());
2194 transform::ScalarizeOp::applyToOne(transform::TransformRewriter &rewriter,
2208 affine::makeComposedFoldedMultiResultAffineApply(rewriter, loc, map,
2219 rewriter.setInsertionPoint(target);
2221 rewriter, cast<TilingInterface>(target.getOperation()), tilingOptions);
2226 rewriter.replaceOp(target, maybeTilingResult->mergeResult.replacements);
2228 rewriter.eraseOp(target);
2241 transform::ConvertToLoopsOp::apply(transform::TransformRewriter &rewriter,
2254 rewriter.setInsertionPoint(target);
2256 scf::lowerToLoopsUsingSCFForOp(rewriter, tilingOp);
2262 rewriter.eraseOp(target);
2274 transform::TransformRewriter &rewriter, Operation *target,
2278 rewriter.setInsertionPoint(target);
2282 [&rewriter](auto op) {
2283 return rewriteInDestinationPassingStyle(rewriter, op);
2296 SplitOp::apply(transform::TransformRewriter &rewriter,
2349 rewriter.getIndexAttr(getStaticChunkSizes()));
2405 rewriter.setInsertionPoint(linalgOp);
2407 rewriter, cast<TilingInterface>(linalgOp.getOperation()),
2437 rewriter.setInsertionPoint(linalgOp);
2439 rewriter, cast<TilingInterface>(linalgOp.getOperation()),
2584 transform::TransformRewriter &rewriter, LinalgOp target,
2592 rewriter.setInsertionPoint(target);
2595 ? splitReductionByScaling(rewriter, target, splitFn, getUseAlloc())
2596 : splitReduction(rewriter, target, splitFn, getUseAlloc());
2629 transform::TransformRewriter &rewriter, Operation *target,
2632 rewriter.setInsertionPoint(target);
2641 rewriter, partialReductionOp,
2642 getAsOpFoldResult(rewriter.getI64ArrayAttr(getTileSizes())));
2646 rewriter.replaceOp(target, result->mergeResult.replacements);
2683 transform::TransformRewriter &rewriter, LinalgOp target,
2686 rewriter.setInsertionPoint(target);
2688 getAsOpFoldResult(rewriter.getI64ArrayAttr(getNumThreads()));
2690 getAsOpFoldResult(rewriter.getI64ArrayAttr(getTileSizes()));
2693 rewriter, cast<PartialReductionOpInterface>(target.getOperation()),
2716 transform::ContinuousTileSizesOp::apply(transform::TransformRewriter &rewriter,
2943 transform::TileUsingForOp::apply(transform::TransformRewriter &rewriter,
3071 tileUsingSCF(rewriter, tilingInterface, tilingOptions);
3075 rewriter.replaceOp(op, maybeTilingResult->mergeResult.replacements);
3197 normalizeUpperBounds(RewriterBase &rewriter, Location loc,
3201 bindSymbols(rewriter.getContext(), s0, s1, s2);
3206 rewriter, loc, normalizedUbExpr, {lb, ub, step});
3214 static SmallVector<Value> denormalizeIndVar(RewriterBase &rewriter,
3220 bindSymbols(rewriter.getContext(), s0, s1);
3221 bindDims(rewriter.getContext(), d0);
3227 rewriter, loc, denormExpr, ArrayRef<OpFoldResult>{iv, lb, step});
3229 getValueOrCreateConstantIndexOp(rewriter, loc, denormValue));
3241 static scf::ForallOp normalizeForallLoopOp(RewriterBase &rewriter,
3256 normalizeUpperBounds(rewriter, loc, lbs, ubs, steps);
3258 rewriter.getIndexAttr(0));
3260 rewriter.getIndexAttr(1));
3262 auto normalizedForallOp = rewriter.create<scf::ForallOp>(
3267 OpBuilder::InsertionGuard g(rewriter);
3269 rewriter.setInsertionPointToStart(normalizedLoopBlock);
3272 denormalizeIndVar(rewriter, loc, normalizedLoopIvs, lbs, steps);
3276 rewriter.mergeBlocks(origLoopBlock, normalizedLoopBlock, argValues);
3278 rewriter.replaceOp(loop, normalizedForallOp);
3283 RewriterBase &rewriter, transform::TransformState &state,
3297 rewriter.setInsertionPoint(tileableOp);
3309 scf::tileUsingSCF(rewriter, tileableOp, options);
3314 rewriter.replaceOp(tileableOp, maybeTilingResult->mergeResult.replacements);
3320 OpBuilder::InsertionGuard g(rewriter);
3321 rewriter.setInsertionPoint(generatedForallOp);
3323 normalizeForallLoopOp(rewriter, generatedForallOp);
3331 transform::TransformRewriter &rewriter,
3362 rewriter, state, transformOp, target, mixedNumThreads, mixedTileSizes,
3454 PatternRewriter &rewriter) const override {
3456 return rewriter.notifyMatchFailure(op,
3458 return vectorize(rewriter, op, /*inputVectorSizes=*/{},
3476 transform::TransformRewriter &rewriter, Operation *target,
3533 transform::TransformRewriter &rewriter,
3553 if (failed(linalg::vectorize(rewriter, target, vectorSizes,
3590 transform::TransformRewriter &rewriter, func::FuncOp target,
3607 transform::TransformRewriter &rewriter, mlir::Operation *target,
3610 rewriter.setInsertionPoint(target);
3611 linalg::hoistRedundantVectorBroadcasts(rewriter, target);
3621 transform::TransformRewriter &rewriter, linalg::LinalgOp target,
3624 rewriter.setInsertionPoint(target);
3629 return rewriteInIm2Col(rewriter, op);
3632 return rewriteInIm2Col(rewriter, op);
3635 return rewriteInIm2Col(rewriter, op);
3638 return rewriteInIm2Col(rewriter, op);
3641 return rewriter.notifyMatchFailure(op, "not supported");
3657 transform::TransformRewriter &rewriter, linalg::LinalgOp target,
3660 rewriter.setInsertionPoint(target);
3675 collapseOpIterationDims(target, reassociation, rewriter);
3680 rewriter.replaceOp(target, maybeFlattened->results);
3689 transform::TransformRewriter &rewriter, linalg::LinalgOp target,
3692 rewriter.setInsertionPoint(target);
3696 return transposeConv2D(rewriter, op);
3699 return transposeConv2D(rewriter, op);
3702 return rewriter.notifyMatchFailure(op, "not supported");
3716 transform::TransformRewriter &rewriter, linalg::LinalgOp target,
3719 rewriter.setInsertionPoint(target);
3724 return transposeMatmul(rewriter, op, transposeLHS);
3727 return transposeBatchMatmul(rewriter, op, transposeLHS);
3741 DiagnosedSilenceableFailure doit(RewriterBase &rewriter, OpTy target,
3757 rewriter.setInsertionPoint(
3761 Value extracted = rewriter.create<tensor::ExtractSliceOp>(
3764 Value copied = rewriter
3769 rewriter.setInsertionPoint(target);
3770 rewriter.replaceOpWithNewOp<OpTy>(
3779 transform::TransformRewriter &rewriter, Operation *targetOp,
3783 rewriter.setInsertionPoint(targetOp);
3785 return doit(rewriter, target, results, state);
3787 return doit(rewriter, target, results, state);
3801 transform::TransformRewriter &rewriter, Operation *target,
3852 /*rewriter=*/rewriter,
3874 transform::TransformRewriter &rewriter, linalg::LinalgOp target,
3877 rewriter.setInsertionPoint(target);
3882 winogradConv2D(rewriter, op, getM(), getR());
3901 transform::TransformRewriter &rewriter, Operation *target,
3904 rewriter.setInsertionPoint(target);
3909 maybeTransformed = decomposeWinogradFilterTransformOp(rewriter, op);
3913 maybeTransformed = decomposeWinogradInputTransformOp(rewriter, op);
3917 maybeTransformed = decomposeWinogradOutputTransformOp(rewriter, op);