Lines Matching defs:rewriter
34 static Value createInserts(RewriterBase &rewriter, Location loc, int dim,
42 destination = rewriter.create<tensor::InsertOp>(loc, *elementIt,
50 destination = createInserts(rewriter, loc, dim + 1, destination, shape,
95 static Operation *movePaddingToFillOrGenericOp(RewriterBase &rewriter,
98 OpBuilder::InsertionGuard g(rewriter);
119 rewriter.getContext()->getLoadedDialect<arith::ArithDialect>();
122 ->materializeConstant(rewriter, constYieldedValue,
125 auto fillOp = rewriter.create<linalg::FillOp>(loc, ValueRange(fillValue),
132 auto fillOp = rewriter.create<linalg::FillOp>(loc, ValueRange(yieldedValue),
141 1, rewriter.getMultiDimIdentityMap(resultType.getRank()));
142 auto genericOp = rewriter.create<linalg::GenericOp>(
146 Block *body = rewriter.createBlock(&genericOp->getRegion(0), {},
148 rewriter.setInsertionPointToStart(body);
151 bbArgReplacements.push_back(rewriter.create<linalg::IndexOp>(loc, i));
152 rewriter.mergeBlocks(padOp.getBody(), body, bbArgReplacements);
156 rewriter.replaceOpWithNewOp<linalg::YieldOp>(yieldOp, yieldOp.getValue());
191 createAllocationForTensor(RewriterBase &rewriter, Location loc, Value value,
194 OpBuilder::InsertionGuard g(rewriter);
201 SmallVector<Value> dynamicSizes = reifyOrComputeDynamicSizes(rewriter, value);
206 alloc = rewriter.create<memref::AllocOp>(loc, memrefType, dynamicSizes);
209 rewriter.setInsertionPoint(rewriter.getInsertionBlock()->getTerminator());
210 rewriter.create<memref::DeallocOp>(loc, alloc);
214 alloc = rewriter.create<memref::AllocaOp>(loc, memrefType, dynamicSizes);
222 RewriterBase &rewriter, const linalg::BufferizeToAllocationOptions &options,
227 OpBuilder::InsertionGuard g(rewriter);
228 rewriter.setInsertionPoint(insertionPoint ? insertionPoint : padOp);
232 Value alloc = createAllocationForTensor(rewriter, loc, padOp.getResult(),
234 rewriter.setInsertionPoint(padOp);
239 movePaddingToFillOrGenericOp(rewriter, loc, padOp, alloc);
240 rewriter.setInsertionPointAfter(fillOp);
245 getMixedSizes(rewriter, loc, padOp.getSource());
247 rewriter.getIndexAttr(1));
248 Value subview = rewriter.create<memref::SubViewOp>(
250 createMemcpy(rewriter, loc, padOp.getSource(), subview, options);
254 Value toTensorOp = rewriter.create<bufferization::ToTensorOp>(
256 rewriter.replaceOp(padOp, toTensorOp);
261 RewriterBase &rewriter, const linalg::BufferizeToAllocationOptions &options,
265 OpBuilder::InsertionGuard g(rewriter);
273 rewriter, options, maskOp.getMaskableOp(), memorySpace,
280 rewriter.setInsertionPoint(yieldOp);
282 rewriter, bufferizationOptions)))
294 rewriter.eraseOp(op);
302 rewriter.setInsertionPoint(maskOp);
304 .bufferize(rewriter, bufferizationOptions)))
313 rewriter.modifyOpInPlace(toTensorOp, [&]() {
323 RewriterBase &rewriter, const linalg::BufferizeToAllocationOptions &options,
327 OpBuilder::InsertionGuard g(rewriter);
328 rewriter.setInsertionPoint(insertionPoint ? insertionPoint : allocTensorOp);
333 rewriter, loc, allocTensorOp.getResult(), options, memorySpace);
337 Value toTensorOp = rewriter.create<bufferization::ToTensorOp>(
339 rewriter.replaceOp(allocTensorOp, toTensorOp);
345 RewriterBase &rewriter, tensor::FromElementsOp fromElementsOp) {
352 auto emptyOp = rewriter.create<EmptyOp>(loc, tensorType, ValueRange());
356 Operation *res = rewriter.replaceOpWithNewOp<tensor::InsertOp>(
367 constants.push_back(rewriter.create<arith::ConstantIndexOp>(loc, i));
372 Value result = createInserts(rewriter, loc, /*dim=*/0, emptyOp.getResult(),
376 rewriter.replaceOp(fromElementsOp, result);
382 mlir::linalg::rewriteInDestinationPassingStyle(RewriterBase &rewriter,
393 rewriter.create<EmptyOp>(loc, tensorType, generateOp.getDynamicExtents());
399 1, rewriter.getMultiDimIdentityMap(tensorType.getRank()));
400 auto genericOp = rewriter.create<linalg::GenericOp>(
404 Block *body = rewriter.createBlock(&genericOp->getRegion(0), {},
406 rewriter.setInsertionPointToStart(body);
409 bbArgReplacements.push_back(rewriter.create<linalg::IndexOp>(loc, i));
410 rewriter.mergeBlocks(&generateOp.getBody().front(), body, bbArgReplacements);
414 rewriter.replaceOpWithNewOp<linalg::YieldOp>(yieldOp, yieldOp.getValue());
417 rewriter.replaceOp(generateOp, genericOp->getResult(0));
423 mlir::linalg::rewriteInDestinationPassingStyle(RewriterBase &rewriter,
433 if (failed(reifyResultShapes(rewriter, padOp, reifiedShape)))
434 return rewriter.notifyMatchFailure(
448 rewriter.create<AllocTensorOp>(loc, resultType, dynamicSizes);
449 auto copyOp = rewriter.replaceOpWithNewOp<linalg::CopyOp>(
454 Value empty = rewriter.create<EmptyOp>(loc, resultType, dynamicSizes);
456 Operation *fillOp = movePaddingToFillOrGenericOp(rewriter, loc, padOp, empty);
457 rewriter.setInsertionPointAfter(fillOp);
461 getMixedSizes(rewriter, loc, padOp.getSource());
463 rewriter.getIndexAttr(1));
464 auto insertSliceOp = rewriter.replaceOpWithNewOp<tensor::InsertSliceOp>(
471 RewriterBase &rewriter, const linalg::BufferizeToAllocationOptions &options,
477 return bufferizeToAllocation(rewriter, options, padOp, memorySpace);
479 return bufferizeToAllocation(rewriter, options, maskOp, memorySpace);
481 return bufferizeToAllocation(rewriter, options, allocTensorOp, memorySpace);
549 OpBuilder::InsertionGuard g(rewriter);
550 rewriter.setInsertionPoint(insertionPoint ? insertionPoint : op);
554 rewriter, op->getLoc(), operand->get(), options, memorySpace);
559 createMemcpy(rewriter, op->getLoc(), operand->get(), alloc, options);
561 rewriter.modifyOpInPlace(op, [&]() {
562 auto toTensorOp = rewriter.create<ToTensorOp>(op->getLoc(), alloc);
565 rewriter.modifyOpInPlace(toTensorOp, [&]() {
577 rewriter.setInsertionPoint(op);
578 if (failed(bufferizableOp.bufferize(rewriter, bufferizationOptions)))
586 rewriter.modifyOpInPlace(toTensorOp, [&]() {
598 PatternRewriter &rewriter) {
599 return linalg::rewriteInDestinationPassingStyle(rewriter, op);