xref: /llvm-project/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp (revision e84f6b6a88c1222d512edf0644c8f869dd12b8ef)
15c0c51a9SNicolas Vasilache //===- VectorToLLVM.cpp - Conversion from Vector to the LLVM dialect ------===//
25c0c51a9SNicolas Vasilache //
330857107SMehdi Amini // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
456222a06SMehdi Amini // See https://llvm.org/LICENSE.txt for license information.
556222a06SMehdi Amini // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
65c0c51a9SNicolas Vasilache //
756222a06SMehdi Amini //===----------------------------------------------------------------------===//
85c0c51a9SNicolas Vasilache 
965678d93SNicolas Vasilache #include "mlir/Conversion/VectorToLLVM/ConvertVectorToLLVM.h"
10870c1fd4SAlex Zinenko 
111b8b5564SNicolas Vasilache #include "mlir/Conversion/ArithCommon/AttrToLLVMConverter.h"
123be3883eSBenjamin Maxwell #include "mlir/Conversion/LLVMCommon/PrintCallHelper.h"
13499abb24SKrzysztof Drewniak #include "mlir/Conversion/LLVMCommon/TypeConverter.h"
1475e5f0aaSAlex Zinenko #include "mlir/Conversion/LLVMCommon/VectorPattern.h"
15abc362a1SJakub Kuderski #include "mlir/Dialect/Arith/IR/Arith.h"
16abc362a1SJakub Kuderski #include "mlir/Dialect/Arith/Utils/Utils.h"
17e332c22cSNicolas Vasilache #include "mlir/Dialect/LLVMIR/FunctionCallUtils.h"
185c0c51a9SNicolas Vasilache #include "mlir/Dialect/LLVMIR/LLVMDialect.h"
19e2310704SJulian Gross #include "mlir/Dialect/MemRef/IR/MemRef.h"
208f5d5194SDaniil Dudkin #include "mlir/Dialect/Vector/IR/VectorOps.h"
21e9b82a5cSDiego Caballero #include "mlir/Dialect/Vector/Interfaces/MaskableOpInterface.h"
222bc4c3e9SNicolas Vasilache #include "mlir/Dialect/Vector/Transforms/LoweringPatterns.h"
2399ef9eebSMatthias Springer #include "mlir/Dialect/Vector/Transforms/VectorTransforms.h"
248f5d5194SDaniil Dudkin #include "mlir/IR/BuiltinAttributes.h"
258f5d5194SDaniil Dudkin #include "mlir/IR/BuiltinTypeInterfaces.h"
2609f7a55fSRiver Riddle #include "mlir/IR/BuiltinTypes.h"
278fe076ffSThomas Raoux #include "mlir/IR/TypeUtilities.h"
28929189a4SWilliam S. Moses #include "mlir/Target/LLVMIR/TypeToLLVM.h"
295c0c51a9SNicolas Vasilache #include "mlir/Transforms/DialectConversion.h"
308f5d5194SDaniil Dudkin #include "llvm/ADT/APFloat.h"
3169049988SMehdi Amini #include "llvm/Support/Casting.h"
32a9a1b45bSKazu Hirata #include <optional>
335c0c51a9SNicolas Vasilache 
345c0c51a9SNicolas Vasilache using namespace mlir;
3565678d93SNicolas Vasilache using namespace mlir::vector;
365c0c51a9SNicolas Vasilache 
379826fe5cSAart Bik // Helper to reduce vector type by *all* but one rank at back.
389826fe5cSAart Bik static VectorType reducedVectorTypeBack(VectorType tp) {
399826fe5cSAart Bik   assert((tp.getRank() > 1) && "unlowerable vector type");
40a4830d14SJavier Setoain   return VectorType::get(tp.getShape().take_back(), tp.getElementType(),
41f22af204SAndrzej Warzynski                          tp.getScalableDims().take_back());
429826fe5cSAart Bik }
439826fe5cSAart Bik 
441c81adf3SAart Bik // Helper that picks the proper sequence for inserting.
45e62a6956SRiver Riddle static Value insertOne(ConversionPatternRewriter &rewriter,
46ce254598SMatthias Springer                        const LLVMTypeConverter &typeConverter, Location loc,
470f04384dSAlex Zinenko                        Value val1, Value val2, Type llvmType, int64_t rank,
480f04384dSAlex Zinenko                        int64_t pos) {
49e7026abaSNicolas Vasilache   assert(rank > 0 && "0-D vector corner case should have been handled already");
501c81adf3SAart Bik   if (rank == 1) {
511c81adf3SAart Bik     auto idxType = rewriter.getIndexType();
521c81adf3SAart Bik     auto constant = rewriter.create<LLVM::ConstantOp>(
530f04384dSAlex Zinenko         loc, typeConverter.convertType(idxType),
541c81adf3SAart Bik         rewriter.getIntegerAttr(idxType, pos));
551c81adf3SAart Bik     return rewriter.create<LLVM::InsertElementOp>(loc, llvmType, val1, val2,
561c81adf3SAart Bik                                                   constant);
571c81adf3SAart Bik   }
585c5af910SJeff Niu   return rewriter.create<LLVM::InsertValueOp>(loc, val1, val2, pos);
591c81adf3SAart Bik }
601c81adf3SAart Bik 
611c81adf3SAart Bik // Helper that picks the proper sequence for extracting.
62e62a6956SRiver Riddle static Value extractOne(ConversionPatternRewriter &rewriter,
63ce254598SMatthias Springer                         const LLVMTypeConverter &typeConverter, Location loc,
640f04384dSAlex Zinenko                         Value val, Type llvmType, int64_t rank, int64_t pos) {
65cc311a15SMichal Terepeta   if (rank <= 1) {
661c81adf3SAart Bik     auto idxType = rewriter.getIndexType();
671c81adf3SAart Bik     auto constant = rewriter.create<LLVM::ConstantOp>(
680f04384dSAlex Zinenko         loc, typeConverter.convertType(idxType),
691c81adf3SAart Bik         rewriter.getIntegerAttr(idxType, pos));
701c81adf3SAart Bik     return rewriter.create<LLVM::ExtractElementOp>(loc, llvmType, val,
711c81adf3SAart Bik                                                    constant);
721c81adf3SAart Bik   }
735c5af910SJeff Niu   return rewriter.create<LLVM::ExtractValueOp>(loc, val, pos);
741c81adf3SAart Bik }
751c81adf3SAart Bik 
7626c8f908SThomas Raoux // Helper that returns data layout alignment of a memref.
77ce254598SMatthias Springer LogicalResult getMemRefAlignment(const LLVMTypeConverter &typeConverter,
7826c8f908SThomas Raoux                                  MemRefType memrefType, unsigned &align) {
7926c8f908SThomas Raoux   Type elementTy = typeConverter.convertType(memrefType.getElementType());
805f9e0466SNicolas Vasilache   if (!elementTy)
815f9e0466SNicolas Vasilache     return failure();
825f9e0466SNicolas Vasilache 
83b2ab375dSAlex Zinenko   // TODO: this should use the MLIR data layout when it becomes available and
84b2ab375dSAlex Zinenko   // stop depending on translation.
8587a89e0fSAlex Zinenko   llvm::LLVMContext llvmContext;
8687a89e0fSAlex Zinenko   align = LLVM::TypeToLLVMIRTranslator(llvmContext)
87c69c9e0fSAlex Zinenko               .getPreferredAlignment(elementTy, typeConverter.getDataLayout());
885f9e0466SNicolas Vasilache   return success();
895f9e0466SNicolas Vasilache }
905f9e0466SNicolas Vasilache 
9178c49743SQuinn Dawkins // Check if the last stride is non-unit and has a valid memory space.
92499abb24SKrzysztof Drewniak static LogicalResult isMemRefTypeSupported(MemRefType memRefType,
93ce254598SMatthias Springer                                            const LLVMTypeConverter &converter) {
946aaa8f25SMatthias Springer   if (!memRefType.isLastDimUnitStride())
95b1d26875SMatthias Springer     return failure();
9678c49743SQuinn Dawkins   if (failed(converter.getMemRefAddressSpace(memRefType)))
97e8dcf5f8Saartbik     return failure();
9819dbb230Saartbik   return success();
9919dbb230Saartbik }
10019dbb230Saartbik 
101f250b972SChe-Yu Wu // Add an index vector component to a base pointer.
102f250b972SChe-Yu Wu static Value getIndexedPtrs(ConversionPatternRewriter &rewriter, Location loc,
103ce254598SMatthias Springer                             const LLVMTypeConverter &typeConverter,
104f250b972SChe-Yu Wu                             MemRefType memRefType, Value llvmMemref, Value base,
1051e7d6d34SCullen Rhodes                             Value index, VectorType vectorType) {
106499abb24SKrzysztof Drewniak   assert(succeeded(isMemRefTypeSupported(memRefType, typeConverter)) &&
107f250b972SChe-Yu Wu          "unsupported memref type");
1081e7d6d34SCullen Rhodes   assert(vectorType.getRank() == 1 && "expected a 1-d vector type");
109f250b972SChe-Yu Wu   auto pType = MemRefDescriptor(llvmMemref).getElementPtrType();
1101e7d6d34SCullen Rhodes   auto ptrsType =
1111e7d6d34SCullen Rhodes       LLVM::getVectorType(pType, vectorType.getDimSize(0),
1121e7d6d34SCullen Rhodes                           /*isScalable=*/vectorType.getScalableDims()[0]);
1134a2d4588SMarkus Böck   return rewriter.create<LLVM::GEPOp>(
1144a2d4588SMarkus Böck       loc, ptrsType, typeConverter.convertType(memRefType.getElementType()),
1154a2d4588SMarkus Böck       base, index);
116f250b972SChe-Yu Wu }
117f250b972SChe-Yu Wu 
11898f6289aSDiego Caballero /// Convert `foldResult` into a Value. Integer attribute is converted to
11998f6289aSDiego Caballero /// an LLVM constant op.
12098f6289aSDiego Caballero static Value getAsLLVMValue(OpBuilder &builder, Location loc,
12198f6289aSDiego Caballero                             OpFoldResult foldResult) {
12298f6289aSDiego Caballero   if (auto attr = foldResult.dyn_cast<Attribute>()) {
12398f6289aSDiego Caballero     auto intAttr = cast<IntegerAttr>(attr);
12498f6289aSDiego Caballero     return builder.create<LLVM::ConstantOp>(loc, intAttr).getResult();
12598f6289aSDiego Caballero   }
12698f6289aSDiego Caballero 
127129ec845SKazu Hirata   return cast<Value>(foldResult);
12898f6289aSDiego Caballero }
12998f6289aSDiego Caballero 
13090c01357SBenjamin Kramer namespace {
131e83b7b99Saartbik 
132a4830d14SJavier Setoain /// Trivial Vector to LLVM conversions
133a4830d14SJavier Setoain using VectorScaleOpConversion =
134a4830d14SJavier Setoain     OneToOneConvertToLLVMPattern<vector::VectorScaleOp, LLVM::vscale>;
135a4830d14SJavier Setoain 
136cf5c517cSDiego Caballero /// Conversion pattern for a vector.bitcast.
137cf5c517cSDiego Caballero class VectorBitCastOpConversion
138cf5c517cSDiego Caballero     : public ConvertOpToLLVMPattern<vector::BitCastOp> {
139cf5c517cSDiego Caballero public:
140cf5c517cSDiego Caballero   using ConvertOpToLLVMPattern<vector::BitCastOp>::ConvertOpToLLVMPattern;
141cf5c517cSDiego Caballero 
142cf5c517cSDiego Caballero   LogicalResult
143ef976337SRiver Riddle   matchAndRewrite(vector::BitCastOp bitCastOp, OpAdaptor adaptor,
144cf5c517cSDiego Caballero                   ConversionPatternRewriter &rewriter) const override {
1451423e8bfSMichal Terepeta     // Only 0-D and 1-D vectors can be lowered to LLVM.
1461423e8bfSMichal Terepeta     VectorType resultTy = bitCastOp.getResultVectorType();
1471423e8bfSMichal Terepeta     if (resultTy.getRank() > 1)
148cf5c517cSDiego Caballero       return failure();
149cf5c517cSDiego Caballero     Type newResultTy = typeConverter->convertType(resultTy);
150cf5c517cSDiego Caballero     rewriter.replaceOpWithNewOp<LLVM::BitcastOp>(bitCastOp, newResultTy,
151ef976337SRiver Riddle                                                  adaptor.getOperands()[0]);
152cf5c517cSDiego Caballero     return success();
153cf5c517cSDiego Caballero   }
154cf5c517cSDiego Caballero };
155cf5c517cSDiego Caballero 
15663b683a8SNicolas Vasilache /// Conversion pattern for a vector.matrix_multiply.
15763b683a8SNicolas Vasilache /// This is lowered directly to the proper llvm.intr.matrix.multiply.
158563879b6SRahul Joshi class VectorMatmulOpConversion
159563879b6SRahul Joshi     : public ConvertOpToLLVMPattern<vector::MatmulOp> {
16063b683a8SNicolas Vasilache public:
161563879b6SRahul Joshi   using ConvertOpToLLVMPattern<vector::MatmulOp>::ConvertOpToLLVMPattern;
16263b683a8SNicolas Vasilache 
1633145427dSRiver Riddle   LogicalResult
164ef976337SRiver Riddle   matchAndRewrite(vector::MatmulOp matmulOp, OpAdaptor adaptor,
16563b683a8SNicolas Vasilache                   ConversionPatternRewriter &rewriter) const override {
16663b683a8SNicolas Vasilache     rewriter.replaceOpWithNewOp<LLVM::MatrixMultiplyOp>(
1677c38fd60SJacques Pienaar         matmulOp, typeConverter->convertType(matmulOp.getRes().getType()),
1687c38fd60SJacques Pienaar         adaptor.getLhs(), adaptor.getRhs(), matmulOp.getLhsRows(),
1697c38fd60SJacques Pienaar         matmulOp.getLhsColumns(), matmulOp.getRhsColumns());
1703145427dSRiver Riddle     return success();
17163b683a8SNicolas Vasilache   }
17263b683a8SNicolas Vasilache };
17363b683a8SNicolas Vasilache 
174c295a65dSaartbik /// Conversion pattern for a vector.flat_transpose.
175c295a65dSaartbik /// This is lowered directly to the proper llvm.intr.matrix.transpose.
176563879b6SRahul Joshi class VectorFlatTransposeOpConversion
177563879b6SRahul Joshi     : public ConvertOpToLLVMPattern<vector::FlatTransposeOp> {
178c295a65dSaartbik public:
179563879b6SRahul Joshi   using ConvertOpToLLVMPattern<vector::FlatTransposeOp>::ConvertOpToLLVMPattern;
180c295a65dSaartbik 
181c295a65dSaartbik   LogicalResult
182ef976337SRiver Riddle   matchAndRewrite(vector::FlatTransposeOp transOp, OpAdaptor adaptor,
183c295a65dSaartbik                   ConversionPatternRewriter &rewriter) const override {
184c295a65dSaartbik     rewriter.replaceOpWithNewOp<LLVM::MatrixTransposeOp>(
1857c38fd60SJacques Pienaar         transOp, typeConverter->convertType(transOp.getRes().getType()),
1867c38fd60SJacques Pienaar         adaptor.getMatrix(), transOp.getRows(), transOp.getColumns());
187c295a65dSaartbik     return success();
188c295a65dSaartbik   }
189c295a65dSaartbik };
190c295a65dSaartbik 
191ee66e43aSDiego Caballero /// Overloaded utility that replaces a vector.load, vector.store,
192ee66e43aSDiego Caballero /// vector.maskedload and vector.maskedstore with their respective LLVM
193ee66e43aSDiego Caballero /// couterparts.
194ee66e43aSDiego Caballero static void replaceLoadOrStoreOp(vector::LoadOp loadOp,
195ee66e43aSDiego Caballero                                  vector::LoadOpAdaptor adaptor,
196ee66e43aSDiego Caballero                                  VectorType vectorTy, Value ptr, unsigned align,
197ee66e43aSDiego Caballero                                  ConversionPatternRewriter &rewriter) {
1985cfe24eeSKrzysztof Drewniak   rewriter.replaceOpWithNewOp<LLVM::LoadOp>(loadOp, vectorTy, ptr, align,
1995cfe24eeSKrzysztof Drewniak                                             /*volatile_=*/false,
2005cfe24eeSKrzysztof Drewniak                                             loadOp.getNontemporal());
20139379916Saartbik }
20239379916Saartbik 
203ee66e43aSDiego Caballero static void replaceLoadOrStoreOp(vector::MaskedLoadOp loadOp,
204ee66e43aSDiego Caballero                                  vector::MaskedLoadOpAdaptor adaptor,
205ee66e43aSDiego Caballero                                  VectorType vectorTy, Value ptr, unsigned align,
206ee66e43aSDiego Caballero                                  ConversionPatternRewriter &rewriter) {
207ee66e43aSDiego Caballero   rewriter.replaceOpWithNewOp<LLVM::MaskedLoadOp>(
2087c38fd60SJacques Pienaar       loadOp, vectorTy, ptr, adaptor.getMask(), adaptor.getPassThru(), align);
209ee66e43aSDiego Caballero }
210ee66e43aSDiego Caballero 
211ee66e43aSDiego Caballero static void replaceLoadOrStoreOp(vector::StoreOp storeOp,
212ee66e43aSDiego Caballero                                  vector::StoreOpAdaptor adaptor,
213ee66e43aSDiego Caballero                                  VectorType vectorTy, Value ptr, unsigned align,
214ee66e43aSDiego Caballero                                  ConversionPatternRewriter &rewriter) {
2157c38fd60SJacques Pienaar   rewriter.replaceOpWithNewOp<LLVM::StoreOp>(storeOp, adaptor.getValueToStore(),
2165cfe24eeSKrzysztof Drewniak                                              ptr, align, /*volatile_=*/false,
2175cfe24eeSKrzysztof Drewniak                                              storeOp.getNontemporal());
218ee66e43aSDiego Caballero }
219ee66e43aSDiego Caballero 
220ee66e43aSDiego Caballero static void replaceLoadOrStoreOp(vector::MaskedStoreOp storeOp,
221ee66e43aSDiego Caballero                                  vector::MaskedStoreOpAdaptor adaptor,
222ee66e43aSDiego Caballero                                  VectorType vectorTy, Value ptr, unsigned align,
223ee66e43aSDiego Caballero                                  ConversionPatternRewriter &rewriter) {
224ee66e43aSDiego Caballero   rewriter.replaceOpWithNewOp<LLVM::MaskedStoreOp>(
2257c38fd60SJacques Pienaar       storeOp, adaptor.getValueToStore(), ptr, adaptor.getMask(), align);
226ee66e43aSDiego Caballero }
227ee66e43aSDiego Caballero 
228ee66e43aSDiego Caballero /// Conversion pattern for a vector.load, vector.store, vector.maskedload, and
229ee66e43aSDiego Caballero /// vector.maskedstore.
2304db0bd28SCullen Rhodes template <class LoadOrStoreOp>
231ee66e43aSDiego Caballero class VectorLoadStoreConversion : public ConvertOpToLLVMPattern<LoadOrStoreOp> {
23239379916Saartbik public:
233ee66e43aSDiego Caballero   using ConvertOpToLLVMPattern<LoadOrStoreOp>::ConvertOpToLLVMPattern;
23439379916Saartbik 
23539379916Saartbik   LogicalResult
236ef976337SRiver Riddle   matchAndRewrite(LoadOrStoreOp loadOrStoreOp,
237ef976337SRiver Riddle                   typename LoadOrStoreOp::Adaptor adaptor,
23839379916Saartbik                   ConversionPatternRewriter &rewriter) const override {
239ee66e43aSDiego Caballero     // Only 1-D vectors can be lowered to LLVM.
240ee66e43aSDiego Caballero     VectorType vectorTy = loadOrStoreOp.getVectorType();
241ee66e43aSDiego Caballero     if (vectorTy.getRank() > 1)
242ee66e43aSDiego Caballero       return failure();
243ee66e43aSDiego Caballero 
244ee66e43aSDiego Caballero     auto loc = loadOrStoreOp->getLoc();
245ee66e43aSDiego Caballero     MemRefType memRefTy = loadOrStoreOp.getMemRefType();
24639379916Saartbik 
24739379916Saartbik     // Resolve alignment.
24839379916Saartbik     unsigned align;
24973863648SStephen Neuendorffer     if (failed(getMemRefAlignment(*this->getTypeConverter(), memRefTy, align)))
25039379916Saartbik       return failure();
25139379916Saartbik 
252a57def30SAart Bik     // Resolve address.
2535550c821STres Popp     auto vtype = cast<VectorType>(
2545550c821STres Popp         this->typeConverter->convertType(loadOrStoreOp.getVectorType()));
2557c38fd60SJacques Pienaar     Value dataPtr = this->getStridedElementPtr(loc, memRefTy, adaptor.getBase(),
2567c38fd60SJacques Pienaar                                                adaptor.getIndices(), rewriter);
257ceb4dc44SChristian Ulmann     replaceLoadOrStoreOp(loadOrStoreOp, adaptor, vtype, dataPtr, align,
258ceb4dc44SChristian Ulmann                          rewriter);
25939379916Saartbik     return success();
26039379916Saartbik   }
26139379916Saartbik };
26239379916Saartbik 
26319dbb230Saartbik /// Conversion pattern for a vector.gather.
264563879b6SRahul Joshi class VectorGatherOpConversion
265563879b6SRahul Joshi     : public ConvertOpToLLVMPattern<vector::GatherOp> {
26619dbb230Saartbik public:
267563879b6SRahul Joshi   using ConvertOpToLLVMPattern<vector::GatherOp>::ConvertOpToLLVMPattern;
26819dbb230Saartbik 
26919dbb230Saartbik   LogicalResult
270ef976337SRiver Riddle   matchAndRewrite(vector::GatherOp gather, OpAdaptor adaptor,
27119dbb230Saartbik                   ConversionPatternRewriter &rewriter) const override {
2725550c821STres Popp     MemRefType memRefType = dyn_cast<MemRefType>(gather.getBaseType());
27366c2b768SJerry Wu     assert(memRefType && "The base should be bufferized");
27419dbb230Saartbik 
275499abb24SKrzysztof Drewniak     if (failed(isMemRefTypeSupported(memRefType, *this->getTypeConverter())))
276f250b972SChe-Yu Wu       return failure();
277f250b972SChe-Yu Wu 
278f250b972SChe-Yu Wu     auto loc = gather->getLoc();
279f250b972SChe-Yu Wu 
28019dbb230Saartbik     // Resolve alignment.
28119dbb230Saartbik     unsigned align;
28273863648SStephen Neuendorffer     if (failed(getMemRefAlignment(*getTypeConverter(), memRefType, align)))
28319dbb230Saartbik       return failure();
28419dbb230Saartbik 
2857c38fd60SJacques Pienaar     Value ptr = getStridedElementPtr(loc, memRefType, adaptor.getBase(),
2867c38fd60SJacques Pienaar                                      adaptor.getIndices(), rewriter);
287f250b972SChe-Yu Wu     Value base = adaptor.getBase();
28819dbb230Saartbik 
289f250b972SChe-Yu Wu     auto llvmNDVectorTy = adaptor.getIndexVec().getType();
290f250b972SChe-Yu Wu     // Handle the simple case of 1-D vector.
2915550c821STres Popp     if (!isa<LLVM::LLVMArrayType>(llvmNDVectorTy)) {
292f250b972SChe-Yu Wu       auto vType = gather.getVectorType();
293f250b972SChe-Yu Wu       // Resolve address.
2941e7d6d34SCullen Rhodes       Value ptrs =
2951e7d6d34SCullen Rhodes           getIndexedPtrs(rewriter, loc, *this->getTypeConverter(), memRefType,
2961e7d6d34SCullen Rhodes                          base, ptr, adaptor.getIndexVec(), vType);
29719dbb230Saartbik       // Replace with the gather intrinsic.
29819dbb230Saartbik       rewriter.replaceOpWithNewOp<LLVM::masked_gather>(
2997c38fd60SJacques Pienaar           gather, typeConverter->convertType(vType), ptrs, adaptor.getMask(),
3007c38fd60SJacques Pienaar           adaptor.getPassThru(), rewriter.getI32IntegerAttr(align));
30119dbb230Saartbik       return success();
30219dbb230Saartbik     }
303f250b972SChe-Yu Wu 
304ce254598SMatthias Springer     const LLVMTypeConverter &typeConverter = *this->getTypeConverter();
305499abb24SKrzysztof Drewniak     auto callback = [align, memRefType, base, ptr, loc, &rewriter,
306499abb24SKrzysztof Drewniak                      &typeConverter](Type llvm1DVectorTy,
307499abb24SKrzysztof Drewniak                                      ValueRange vectorOperands) {
308f250b972SChe-Yu Wu       // Resolve address.
309f250b972SChe-Yu Wu       Value ptrs = getIndexedPtrs(
310499abb24SKrzysztof Drewniak           rewriter, loc, typeConverter, memRefType, base, ptr,
3111e7d6d34SCullen Rhodes           /*index=*/vectorOperands[0], cast<VectorType>(llvm1DVectorTy));
312f250b972SChe-Yu Wu       // Create the gather intrinsic.
313f250b972SChe-Yu Wu       return rewriter.create<LLVM::masked_gather>(
314f250b972SChe-Yu Wu           loc, llvm1DVectorTy, ptrs, /*mask=*/vectorOperands[1],
315f250b972SChe-Yu Wu           /*passThru=*/vectorOperands[2], rewriter.getI32IntegerAttr(align));
316f250b972SChe-Yu Wu     };
317f250b972SChe-Yu Wu     SmallVector<Value> vectorOperands = {
318f250b972SChe-Yu Wu         adaptor.getIndexVec(), adaptor.getMask(), adaptor.getPassThru()};
319f250b972SChe-Yu Wu     return LLVM::detail::handleMultidimensionalVectors(
320f250b972SChe-Yu Wu         gather, vectorOperands, *getTypeConverter(), callback, rewriter);
321f250b972SChe-Yu Wu   }
32219dbb230Saartbik };
32319dbb230Saartbik 
32419dbb230Saartbik /// Conversion pattern for a vector.scatter.
325563879b6SRahul Joshi class VectorScatterOpConversion
326563879b6SRahul Joshi     : public ConvertOpToLLVMPattern<vector::ScatterOp> {
32719dbb230Saartbik public:
328563879b6SRahul Joshi   using ConvertOpToLLVMPattern<vector::ScatterOp>::ConvertOpToLLVMPattern;
32919dbb230Saartbik 
33019dbb230Saartbik   LogicalResult
331ef976337SRiver Riddle   matchAndRewrite(vector::ScatterOp scatter, OpAdaptor adaptor,
33219dbb230Saartbik                   ConversionPatternRewriter &rewriter) const override {
333563879b6SRahul Joshi     auto loc = scatter->getLoc();
334df5ccf5aSAart Bik     MemRefType memRefType = scatter.getMemRefType();
33519dbb230Saartbik 
336499abb24SKrzysztof Drewniak     if (failed(isMemRefTypeSupported(memRefType, *this->getTypeConverter())))
337f250b972SChe-Yu Wu       return failure();
338f250b972SChe-Yu Wu 
33919dbb230Saartbik     // Resolve alignment.
34019dbb230Saartbik     unsigned align;
34173863648SStephen Neuendorffer     if (failed(getMemRefAlignment(*getTypeConverter(), memRefType, align)))
34219dbb230Saartbik       return failure();
34319dbb230Saartbik 
344df5ccf5aSAart Bik     // Resolve address.
345df5ccf5aSAart Bik     VectorType vType = scatter.getVectorType();
3467c38fd60SJacques Pienaar     Value ptr = getStridedElementPtr(loc, memRefType, adaptor.getBase(),
3477c38fd60SJacques Pienaar                                      adaptor.getIndices(), rewriter);
3481e7d6d34SCullen Rhodes     Value ptrs =
3491e7d6d34SCullen Rhodes         getIndexedPtrs(rewriter, loc, *this->getTypeConverter(), memRefType,
3501e7d6d34SCullen Rhodes                        adaptor.getBase(), ptr, adaptor.getIndexVec(), vType);
35119dbb230Saartbik 
35219dbb230Saartbik     // Replace with the scatter intrinsic.
35319dbb230Saartbik     rewriter.replaceOpWithNewOp<LLVM::masked_scatter>(
3547c38fd60SJacques Pienaar         scatter, adaptor.getValueToStore(), ptrs, adaptor.getMask(),
35519dbb230Saartbik         rewriter.getI32IntegerAttr(align));
35619dbb230Saartbik     return success();
35719dbb230Saartbik   }
35819dbb230Saartbik };
35919dbb230Saartbik 
360e8dcf5f8Saartbik /// Conversion pattern for a vector.expandload.
361563879b6SRahul Joshi class VectorExpandLoadOpConversion
362563879b6SRahul Joshi     : public ConvertOpToLLVMPattern<vector::ExpandLoadOp> {
363e8dcf5f8Saartbik public:
364563879b6SRahul Joshi   using ConvertOpToLLVMPattern<vector::ExpandLoadOp>::ConvertOpToLLVMPattern;
365e8dcf5f8Saartbik 
366e8dcf5f8Saartbik   LogicalResult
367ef976337SRiver Riddle   matchAndRewrite(vector::ExpandLoadOp expand, OpAdaptor adaptor,
368e8dcf5f8Saartbik                   ConversionPatternRewriter &rewriter) const override {
369563879b6SRahul Joshi     auto loc = expand->getLoc();
370a57def30SAart Bik     MemRefType memRefType = expand.getMemRefType();
371e8dcf5f8Saartbik 
372a57def30SAart Bik     // Resolve address.
373656674a7SDiego Caballero     auto vtype = typeConverter->convertType(expand.getVectorType());
3747c38fd60SJacques Pienaar     Value ptr = getStridedElementPtr(loc, memRefType, adaptor.getBase(),
3757c38fd60SJacques Pienaar                                      adaptor.getIndices(), rewriter);
376e8dcf5f8Saartbik 
377e8dcf5f8Saartbik     rewriter.replaceOpWithNewOp<LLVM::masked_expandload>(
3787c38fd60SJacques Pienaar         expand, vtype, ptr, adaptor.getMask(), adaptor.getPassThru());
379e8dcf5f8Saartbik     return success();
380e8dcf5f8Saartbik   }
381e8dcf5f8Saartbik };
382e8dcf5f8Saartbik 
383e8dcf5f8Saartbik /// Conversion pattern for a vector.compressstore.
384563879b6SRahul Joshi class VectorCompressStoreOpConversion
385563879b6SRahul Joshi     : public ConvertOpToLLVMPattern<vector::CompressStoreOp> {
386e8dcf5f8Saartbik public:
387563879b6SRahul Joshi   using ConvertOpToLLVMPattern<vector::CompressStoreOp>::ConvertOpToLLVMPattern;
388e8dcf5f8Saartbik 
389e8dcf5f8Saartbik   LogicalResult
390ef976337SRiver Riddle   matchAndRewrite(vector::CompressStoreOp compress, OpAdaptor adaptor,
391e8dcf5f8Saartbik                   ConversionPatternRewriter &rewriter) const override {
392563879b6SRahul Joshi     auto loc = compress->getLoc();
393a57def30SAart Bik     MemRefType memRefType = compress.getMemRefType();
394e8dcf5f8Saartbik 
395a57def30SAart Bik     // Resolve address.
3967c38fd60SJacques Pienaar     Value ptr = getStridedElementPtr(loc, memRefType, adaptor.getBase(),
3977c38fd60SJacques Pienaar                                      adaptor.getIndices(), rewriter);
398e8dcf5f8Saartbik 
399e8dcf5f8Saartbik     rewriter.replaceOpWithNewOp<LLVM::masked_compressstore>(
4007c38fd60SJacques Pienaar         compress, adaptor.getValueToStore(), ptr, adaptor.getMask());
401e8dcf5f8Saartbik     return success();
402e8dcf5f8Saartbik   }
403e8dcf5f8Saartbik };
404e8dcf5f8Saartbik 
405e9b82a5cSDiego Caballero /// Reduction neutral classes for overloading.
406e9b82a5cSDiego Caballero class ReductionNeutralZero {};
407e9b82a5cSDiego Caballero class ReductionNeutralIntOne {};
408e9b82a5cSDiego Caballero class ReductionNeutralFPOne {};
409e9b82a5cSDiego Caballero class ReductionNeutralAllOnes {};
410e9b82a5cSDiego Caballero class ReductionNeutralSIntMin {};
411e9b82a5cSDiego Caballero class ReductionNeutralUIntMin {};
412e9b82a5cSDiego Caballero class ReductionNeutralSIntMax {};
413e9b82a5cSDiego Caballero class ReductionNeutralUIntMax {};
414e9b82a5cSDiego Caballero class ReductionNeutralFPMin {};
415e9b82a5cSDiego Caballero class ReductionNeutralFPMax {};
416e9b82a5cSDiego Caballero 
417e9b82a5cSDiego Caballero /// Create the reduction neutral zero value.
418e9b82a5cSDiego Caballero static Value createReductionNeutralValue(ReductionNeutralZero neutral,
419e9b82a5cSDiego Caballero                                          ConversionPatternRewriter &rewriter,
420e9b82a5cSDiego Caballero                                          Location loc, Type llvmType) {
421e9b82a5cSDiego Caballero   return rewriter.create<LLVM::ConstantOp>(loc, llvmType,
422e9b82a5cSDiego Caballero                                            rewriter.getZeroAttr(llvmType));
423e9b82a5cSDiego Caballero }
424e9b82a5cSDiego Caballero 
425e9b82a5cSDiego Caballero /// Create the reduction neutral integer one value.
426e9b82a5cSDiego Caballero static Value createReductionNeutralValue(ReductionNeutralIntOne neutral,
427e9b82a5cSDiego Caballero                                          ConversionPatternRewriter &rewriter,
428e9b82a5cSDiego Caballero                                          Location loc, Type llvmType) {
429e9b82a5cSDiego Caballero   return rewriter.create<LLVM::ConstantOp>(
430e9b82a5cSDiego Caballero       loc, llvmType, rewriter.getIntegerAttr(llvmType, 1));
431e9b82a5cSDiego Caballero }
432e9b82a5cSDiego Caballero 
433e9b82a5cSDiego Caballero /// Create the reduction neutral fp one value.
434e9b82a5cSDiego Caballero static Value createReductionNeutralValue(ReductionNeutralFPOne neutral,
435e9b82a5cSDiego Caballero                                          ConversionPatternRewriter &rewriter,
436e9b82a5cSDiego Caballero                                          Location loc, Type llvmType) {
437e9b82a5cSDiego Caballero   return rewriter.create<LLVM::ConstantOp>(
438e9b82a5cSDiego Caballero       loc, llvmType, rewriter.getFloatAttr(llvmType, 1.0));
439e9b82a5cSDiego Caballero }
440e9b82a5cSDiego Caballero 
441e9b82a5cSDiego Caballero /// Create the reduction neutral all-ones value.
442e9b82a5cSDiego Caballero static Value createReductionNeutralValue(ReductionNeutralAllOnes neutral,
443e9b82a5cSDiego Caballero                                          ConversionPatternRewriter &rewriter,
444e9b82a5cSDiego Caballero                                          Location loc, Type llvmType) {
445e9b82a5cSDiego Caballero   return rewriter.create<LLVM::ConstantOp>(
446e9b82a5cSDiego Caballero       loc, llvmType,
447e9b82a5cSDiego Caballero       rewriter.getIntegerAttr(
448e9b82a5cSDiego Caballero           llvmType, llvm::APInt::getAllOnes(llvmType.getIntOrFloatBitWidth())));
449e9b82a5cSDiego Caballero }
450e9b82a5cSDiego Caballero 
451e9b82a5cSDiego Caballero /// Create the reduction neutral signed int minimum value.
452e9b82a5cSDiego Caballero static Value createReductionNeutralValue(ReductionNeutralSIntMin neutral,
453e9b82a5cSDiego Caballero                                          ConversionPatternRewriter &rewriter,
454e9b82a5cSDiego Caballero                                          Location loc, Type llvmType) {
455e9b82a5cSDiego Caballero   return rewriter.create<LLVM::ConstantOp>(
456e9b82a5cSDiego Caballero       loc, llvmType,
457e9b82a5cSDiego Caballero       rewriter.getIntegerAttr(llvmType, llvm::APInt::getSignedMinValue(
458e9b82a5cSDiego Caballero                                             llvmType.getIntOrFloatBitWidth())));
459e9b82a5cSDiego Caballero }
460e9b82a5cSDiego Caballero 
461e9b82a5cSDiego Caballero /// Create the reduction neutral unsigned int minimum value.
462e9b82a5cSDiego Caballero static Value createReductionNeutralValue(ReductionNeutralUIntMin neutral,
463e9b82a5cSDiego Caballero                                          ConversionPatternRewriter &rewriter,
464e9b82a5cSDiego Caballero                                          Location loc, Type llvmType) {
465e9b82a5cSDiego Caballero   return rewriter.create<LLVM::ConstantOp>(
466e9b82a5cSDiego Caballero       loc, llvmType,
467e9b82a5cSDiego Caballero       rewriter.getIntegerAttr(llvmType, llvm::APInt::getMinValue(
468e9b82a5cSDiego Caballero                                             llvmType.getIntOrFloatBitWidth())));
469e9b82a5cSDiego Caballero }
470e9b82a5cSDiego Caballero 
471e9b82a5cSDiego Caballero /// Create the reduction neutral signed int maximum value.
472e9b82a5cSDiego Caballero static Value createReductionNeutralValue(ReductionNeutralSIntMax neutral,
473e9b82a5cSDiego Caballero                                          ConversionPatternRewriter &rewriter,
474e9b82a5cSDiego Caballero                                          Location loc, Type llvmType) {
475e9b82a5cSDiego Caballero   return rewriter.create<LLVM::ConstantOp>(
476e9b82a5cSDiego Caballero       loc, llvmType,
477e9b82a5cSDiego Caballero       rewriter.getIntegerAttr(llvmType, llvm::APInt::getSignedMaxValue(
478e9b82a5cSDiego Caballero                                             llvmType.getIntOrFloatBitWidth())));
479e9b82a5cSDiego Caballero }
480e9b82a5cSDiego Caballero 
481e9b82a5cSDiego Caballero /// Create the reduction neutral unsigned int maximum value.
482e9b82a5cSDiego Caballero static Value createReductionNeutralValue(ReductionNeutralUIntMax neutral,
483e9b82a5cSDiego Caballero                                          ConversionPatternRewriter &rewriter,
484e9b82a5cSDiego Caballero                                          Location loc, Type llvmType) {
485e9b82a5cSDiego Caballero   return rewriter.create<LLVM::ConstantOp>(
486e9b82a5cSDiego Caballero       loc, llvmType,
487e9b82a5cSDiego Caballero       rewriter.getIntegerAttr(llvmType, llvm::APInt::getMaxValue(
488e9b82a5cSDiego Caballero                                             llvmType.getIntOrFloatBitWidth())));
489e9b82a5cSDiego Caballero }
490e9b82a5cSDiego Caballero 
491e9b82a5cSDiego Caballero /// Create the reduction neutral fp minimum value.
492e9b82a5cSDiego Caballero static Value createReductionNeutralValue(ReductionNeutralFPMin neutral,
493e9b82a5cSDiego Caballero                                          ConversionPatternRewriter &rewriter,
494e9b82a5cSDiego Caballero                                          Location loc, Type llvmType) {
4955550c821STres Popp   auto floatType = cast<FloatType>(llvmType);
496e9b82a5cSDiego Caballero   return rewriter.create<LLVM::ConstantOp>(
497e9b82a5cSDiego Caballero       loc, llvmType,
498e9b82a5cSDiego Caballero       rewriter.getFloatAttr(
499e9b82a5cSDiego Caballero           llvmType, llvm::APFloat::getQNaN(floatType.getFloatSemantics(),
500e9b82a5cSDiego Caballero                                            /*Negative=*/false)));
501e9b82a5cSDiego Caballero }
502e9b82a5cSDiego Caballero 
503e9b82a5cSDiego Caballero /// Create the reduction neutral fp maximum value.
504e9b82a5cSDiego Caballero static Value createReductionNeutralValue(ReductionNeutralFPMax neutral,
505e9b82a5cSDiego Caballero                                          ConversionPatternRewriter &rewriter,
506e9b82a5cSDiego Caballero                                          Location loc, Type llvmType) {
5075550c821STres Popp   auto floatType = cast<FloatType>(llvmType);
508e9b82a5cSDiego Caballero   return rewriter.create<LLVM::ConstantOp>(
509e9b82a5cSDiego Caballero       loc, llvmType,
510e9b82a5cSDiego Caballero       rewriter.getFloatAttr(
511e9b82a5cSDiego Caballero           llvmType, llvm::APFloat::getQNaN(floatType.getFloatSemantics(),
512e9b82a5cSDiego Caballero                                            /*Negative=*/true)));
513e9b82a5cSDiego Caballero }
514e9b82a5cSDiego Caballero 
515e9b82a5cSDiego Caballero /// Returns `accumulator` if it has a valid value. Otherwise, creates and
516e9b82a5cSDiego Caballero /// returns a new accumulator value using `ReductionNeutral`.
517e9b82a5cSDiego Caballero template <class ReductionNeutral>
518e9b82a5cSDiego Caballero static Value getOrCreateAccumulator(ConversionPatternRewriter &rewriter,
519e9b82a5cSDiego Caballero                                     Location loc, Type llvmType,
520e9b82a5cSDiego Caballero                                     Value accumulator) {
521e9b82a5cSDiego Caballero   if (accumulator)
522e9b82a5cSDiego Caballero     return accumulator;
523e9b82a5cSDiego Caballero 
524e9b82a5cSDiego Caballero   return createReductionNeutralValue(ReductionNeutral(), rewriter, loc,
525e9b82a5cSDiego Caballero                                      llvmType);
526e9b82a5cSDiego Caballero }
527e9b82a5cSDiego Caballero 
528abcbbe71SZhaoshi Zheng /// Creates a value with the 1-D vector shape provided in `llvmType`.
529e9b82a5cSDiego Caballero /// This is used as effective vector length by some intrinsics supporting
530e9b82a5cSDiego Caballero /// dynamic vector lengths at runtime.
531e9b82a5cSDiego Caballero static Value createVectorLengthValue(ConversionPatternRewriter &rewriter,
532e9b82a5cSDiego Caballero                                      Location loc, Type llvmType) {
533e9b82a5cSDiego Caballero   VectorType vType = cast<VectorType>(llvmType);
534e9b82a5cSDiego Caballero   auto vShape = vType.getShape();
535e9b82a5cSDiego Caballero   assert(vShape.size() == 1 && "Unexpected multi-dim vector type");
536e9b82a5cSDiego Caballero 
537abcbbe71SZhaoshi Zheng   Value baseVecLength = rewriter.create<LLVM::ConstantOp>(
538e9b82a5cSDiego Caballero       loc, rewriter.getI32Type(),
539e9b82a5cSDiego Caballero       rewriter.getIntegerAttr(rewriter.getI32Type(), vShape[0]));
540abcbbe71SZhaoshi Zheng 
541abcbbe71SZhaoshi Zheng   if (!vType.getScalableDims()[0])
542abcbbe71SZhaoshi Zheng     return baseVecLength;
543abcbbe71SZhaoshi Zheng 
544abcbbe71SZhaoshi Zheng   // For a scalable vector type, create and return `vScale * baseVecLength`.
545abcbbe71SZhaoshi Zheng   Value vScale = rewriter.create<vector::VectorScaleOp>(loc);
546abcbbe71SZhaoshi Zheng   vScale =
547abcbbe71SZhaoshi Zheng       rewriter.create<arith::IndexCastOp>(loc, rewriter.getI32Type(), vScale);
548abcbbe71SZhaoshi Zheng   Value scalableVecLength =
549abcbbe71SZhaoshi Zheng       rewriter.create<arith::MulIOp>(loc, baseVecLength, vScale);
550abcbbe71SZhaoshi Zheng   return scalableVecLength;
551e9b82a5cSDiego Caballero }
552e9b82a5cSDiego Caballero 
553fa596c69SMahesh Ravishankar /// Helper method to lower a `vector.reduction` op that performs an arithmetic
554fa596c69SMahesh Ravishankar /// operation like add,mul, etc.. `VectorOp` is the LLVM vector intrinsic to use
555fa596c69SMahesh Ravishankar /// and `ScalarOp` is the scalar operation used to add the accumulation value if
556fa596c69SMahesh Ravishankar /// non-null.
557e9b82a5cSDiego Caballero template <class LLVMRedIntrinOp, class ScalarOp>
558fa596c69SMahesh Ravishankar static Value createIntegerReductionArithmeticOpLowering(
559fa596c69SMahesh Ravishankar     ConversionPatternRewriter &rewriter, Location loc, Type llvmType,
560fa596c69SMahesh Ravishankar     Value vectorOperand, Value accumulator) {
561e9b82a5cSDiego Caballero 
562e9b82a5cSDiego Caballero   Value result = rewriter.create<LLVMRedIntrinOp>(loc, llvmType, vectorOperand);
563e9b82a5cSDiego Caballero 
564fa596c69SMahesh Ravishankar   if (accumulator)
565fa596c69SMahesh Ravishankar     result = rewriter.create<ScalarOp>(loc, accumulator, result);
566fa596c69SMahesh Ravishankar   return result;
567fa596c69SMahesh Ravishankar }
568fa596c69SMahesh Ravishankar 
569fa596c69SMahesh Ravishankar /// Helper method to lower a `vector.reduction` operation that performs
570fa596c69SMahesh Ravishankar /// a comparison operation like `min`/`max`. `VectorOp` is the LLVM vector
571fa596c69SMahesh Ravishankar /// intrinsic to use and `predicate` is the predicate to use to compare+combine
572fa596c69SMahesh Ravishankar /// the accumulator value if non-null.
573e9b82a5cSDiego Caballero template <class LLVMRedIntrinOp>
574fa596c69SMahesh Ravishankar static Value createIntegerReductionComparisonOpLowering(
575fa596c69SMahesh Ravishankar     ConversionPatternRewriter &rewriter, Location loc, Type llvmType,
576fa596c69SMahesh Ravishankar     Value vectorOperand, Value accumulator, LLVM::ICmpPredicate predicate) {
577e9b82a5cSDiego Caballero   Value result = rewriter.create<LLVMRedIntrinOp>(loc, llvmType, vectorOperand);
578fa596c69SMahesh Ravishankar   if (accumulator) {
579fa596c69SMahesh Ravishankar     Value cmp =
580fa596c69SMahesh Ravishankar         rewriter.create<LLVM::ICmpOp>(loc, predicate, accumulator, result);
581fa596c69SMahesh Ravishankar     result = rewriter.create<LLVM::SelectOp>(loc, cmp, accumulator, result);
582fa596c69SMahesh Ravishankar   }
583fa596c69SMahesh Ravishankar   return result;
584fa596c69SMahesh Ravishankar }
585fa596c69SMahesh Ravishankar 
586dad9de0aSDaniil Dudkin namespace {
587dad9de0aSDaniil Dudkin template <typename Source>
588dad9de0aSDaniil Dudkin struct VectorToScalarMapper;
589dad9de0aSDaniil Dudkin template <>
590dad9de0aSDaniil Dudkin struct VectorToScalarMapper<LLVM::vector_reduce_fmaximum> {
591dad9de0aSDaniil Dudkin   using Type = LLVM::MaximumOp;
592dad9de0aSDaniil Dudkin };
593dad9de0aSDaniil Dudkin template <>
594dad9de0aSDaniil Dudkin struct VectorToScalarMapper<LLVM::vector_reduce_fminimum> {
595dad9de0aSDaniil Dudkin   using Type = LLVM::MinimumOp;
596dad9de0aSDaniil Dudkin };
597709b2742SDaniil Dudkin template <>
598709b2742SDaniil Dudkin struct VectorToScalarMapper<LLVM::vector_reduce_fmax> {
599709b2742SDaniil Dudkin   using Type = LLVM::MaxNumOp;
600709b2742SDaniil Dudkin };
601709b2742SDaniil Dudkin template <>
602709b2742SDaniil Dudkin struct VectorToScalarMapper<LLVM::vector_reduce_fmin> {
603709b2742SDaniil Dudkin   using Type = LLVM::MinNumOp;
604709b2742SDaniil Dudkin };
605dad9de0aSDaniil Dudkin } // namespace
6068fe076ffSThomas Raoux 
607e9b82a5cSDiego Caballero template <class LLVMRedIntrinOp>
6081b8b5564SNicolas Vasilache static Value createFPReductionComparisonOpLowering(
6091b8b5564SNicolas Vasilache     ConversionPatternRewriter &rewriter, Location loc, Type llvmType,
6101b8b5564SNicolas Vasilache     Value vectorOperand, Value accumulator, LLVM::FastmathFlagsAttr fmf) {
6111b8b5564SNicolas Vasilache   Value result =
6121b8b5564SNicolas Vasilache       rewriter.create<LLVMRedIntrinOp>(loc, llvmType, vectorOperand, fmf);
613e9b82a5cSDiego Caballero 
614dad9de0aSDaniil Dudkin   if (accumulator) {
615dad9de0aSDaniil Dudkin     result =
616dad9de0aSDaniil Dudkin         rewriter.create<typename VectorToScalarMapper<LLVMRedIntrinOp>::Type>(
617dad9de0aSDaniil Dudkin             loc, result, accumulator);
618dad9de0aSDaniil Dudkin   }
619e9b82a5cSDiego Caballero 
620e9b82a5cSDiego Caballero   return result;
621e9b82a5cSDiego Caballero }
622e9b82a5cSDiego Caballero 
6238f5d5194SDaniil Dudkin /// Reduction neutral classes for overloading
6248f5d5194SDaniil Dudkin class MaskNeutralFMaximum {};
6258f5d5194SDaniil Dudkin class MaskNeutralFMinimum {};
6268f5d5194SDaniil Dudkin 
6278f5d5194SDaniil Dudkin /// Get the mask neutral floating point maximum value
6288f5d5194SDaniil Dudkin static llvm::APFloat
6298f5d5194SDaniil Dudkin getMaskNeutralValue(MaskNeutralFMaximum,
6308f5d5194SDaniil Dudkin                     const llvm::fltSemantics &floatSemantics) {
6318f5d5194SDaniil Dudkin   return llvm::APFloat::getSmallest(floatSemantics, /*Negative=*/true);
6328f5d5194SDaniil Dudkin }
6338f5d5194SDaniil Dudkin /// Get the mask neutral floating point minimum value
6348f5d5194SDaniil Dudkin static llvm::APFloat
6358f5d5194SDaniil Dudkin getMaskNeutralValue(MaskNeutralFMinimum,
6368f5d5194SDaniil Dudkin                     const llvm::fltSemantics &floatSemantics) {
6378f5d5194SDaniil Dudkin   return llvm::APFloat::getLargest(floatSemantics, /*Negative=*/false);
6388f5d5194SDaniil Dudkin }
6398f5d5194SDaniil Dudkin 
6408f5d5194SDaniil Dudkin /// Create the mask neutral floating point MLIR vector constant
6418f5d5194SDaniil Dudkin template <typename MaskNeutral>
6428f5d5194SDaniil Dudkin static Value createMaskNeutralValue(ConversionPatternRewriter &rewriter,
6438f5d5194SDaniil Dudkin                                     Location loc, Type llvmType,
6448f5d5194SDaniil Dudkin                                     Type vectorType) {
6458f5d5194SDaniil Dudkin   const auto &floatSemantics = cast<FloatType>(llvmType).getFloatSemantics();
6468f5d5194SDaniil Dudkin   auto value = getMaskNeutralValue(MaskNeutral{}, floatSemantics);
647a5757c5bSChristian Sigg   auto denseValue = DenseElementsAttr::get(cast<ShapedType>(vectorType), value);
6488f5d5194SDaniil Dudkin   return rewriter.create<LLVM::ConstantOp>(loc, vectorType, denseValue);
6498f5d5194SDaniil Dudkin }
6508f5d5194SDaniil Dudkin 
6518f5d5194SDaniil Dudkin /// Lowers masked `fmaximum` and `fminimum` reductions using the non-masked
6528f5d5194SDaniil Dudkin /// intrinsics. It is a workaround to overcome the lack of masked intrinsics for
6538f5d5194SDaniil Dudkin /// `fmaximum`/`fminimum`.
6548f5d5194SDaniil Dudkin /// More information: https://github.com/llvm/llvm-project/issues/64940
6558f5d5194SDaniil Dudkin template <class LLVMRedIntrinOp, class MaskNeutral>
6561b8b5564SNicolas Vasilache static Value
6571b8b5564SNicolas Vasilache lowerMaskedReductionWithRegular(ConversionPatternRewriter &rewriter,
6581b8b5564SNicolas Vasilache                                 Location loc, Type llvmType,
6591b8b5564SNicolas Vasilache                                 Value vectorOperand, Value accumulator,
6601b8b5564SNicolas Vasilache                                 Value mask, LLVM::FastmathFlagsAttr fmf) {
6618f5d5194SDaniil Dudkin   const Value vectorMaskNeutral = createMaskNeutralValue<MaskNeutral>(
6628f5d5194SDaniil Dudkin       rewriter, loc, llvmType, vectorOperand.getType());
6638f5d5194SDaniil Dudkin   const Value selectedVectorByMask = rewriter.create<LLVM::SelectOp>(
6648f5d5194SDaniil Dudkin       loc, mask, vectorOperand, vectorMaskNeutral);
6658f5d5194SDaniil Dudkin   return createFPReductionComparisonOpLowering<LLVMRedIntrinOp>(
6661b8b5564SNicolas Vasilache       rewriter, loc, llvmType, selectedVectorByMask, accumulator, fmf);
6678f5d5194SDaniil Dudkin }
6688f5d5194SDaniil Dudkin 
6691b8b5564SNicolas Vasilache template <class LLVMRedIntrinOp, class ReductionNeutral>
6701b8b5564SNicolas Vasilache static Value
6711b8b5564SNicolas Vasilache lowerReductionWithStartValue(ConversionPatternRewriter &rewriter, Location loc,
6721b8b5564SNicolas Vasilache                              Type llvmType, Value vectorOperand,
6731b8b5564SNicolas Vasilache                              Value accumulator, LLVM::FastmathFlagsAttr fmf) {
6741b8b5564SNicolas Vasilache   accumulator = getOrCreateAccumulator<ReductionNeutral>(rewriter, loc,
6751b8b5564SNicolas Vasilache                                                          llvmType, accumulator);
6761b8b5564SNicolas Vasilache   return rewriter.create<LLVMRedIntrinOp>(loc, llvmType,
6771b8b5564SNicolas Vasilache                                           /*startValue=*/accumulator,
6781b8b5564SNicolas Vasilache                                           vectorOperand, fmf);
6791b8b5564SNicolas Vasilache }
6801b8b5564SNicolas Vasilache 
681922992a2SJay Foad /// Overloaded methods to lower a *predicated* reduction to an llvm intrinsic
6821b8b5564SNicolas Vasilache /// that requires a start value. This start value format spans across fp
6831b8b5564SNicolas Vasilache /// reductions without mask and all the masked reduction intrinsics.
684e9b82a5cSDiego Caballero template <class LLVMVPRedIntrinOp, class ReductionNeutral>
6851b8b5564SNicolas Vasilache static Value
6861b8b5564SNicolas Vasilache lowerPredicatedReductionWithStartValue(ConversionPatternRewriter &rewriter,
687e9b82a5cSDiego Caballero                                        Location loc, Type llvmType,
6881b8b5564SNicolas Vasilache                                        Value vectorOperand, Value accumulator) {
689e9b82a5cSDiego Caballero   accumulator = getOrCreateAccumulator<ReductionNeutral>(rewriter, loc,
690e9b82a5cSDiego Caballero                                                          llvmType, accumulator);
691e9b82a5cSDiego Caballero   return rewriter.create<LLVMVPRedIntrinOp>(loc, llvmType,
692e9b82a5cSDiego Caballero                                             /*startValue=*/accumulator,
693e9b82a5cSDiego Caballero                                             vectorOperand);
694e9b82a5cSDiego Caballero }
695e9b82a5cSDiego Caballero 
696e9b82a5cSDiego Caballero template <class LLVMVPRedIntrinOp, class ReductionNeutral>
6971b8b5564SNicolas Vasilache static Value lowerPredicatedReductionWithStartValue(
6981b8b5564SNicolas Vasilache     ConversionPatternRewriter &rewriter, Location loc, Type llvmType,
6991b8b5564SNicolas Vasilache     Value vectorOperand, Value accumulator, Value mask) {
700e9b82a5cSDiego Caballero   accumulator = getOrCreateAccumulator<ReductionNeutral>(rewriter, loc,
701e9b82a5cSDiego Caballero                                                          llvmType, accumulator);
702e9b82a5cSDiego Caballero   Value vectorLength =
703e9b82a5cSDiego Caballero       createVectorLengthValue(rewriter, loc, vectorOperand.getType());
704e9b82a5cSDiego Caballero   return rewriter.create<LLVMVPRedIntrinOp>(loc, llvmType,
705e9b82a5cSDiego Caballero                                             /*startValue=*/accumulator,
706e9b82a5cSDiego Caballero                                             vectorOperand, mask, vectorLength);
707e9b82a5cSDiego Caballero }
708e9b82a5cSDiego Caballero 
709e9b82a5cSDiego Caballero template <class LLVMIntVPRedIntrinOp, class IntReductionNeutral,
710e9b82a5cSDiego Caballero           class LLVMFPVPRedIntrinOp, class FPReductionNeutral>
7111b8b5564SNicolas Vasilache static Value lowerPredicatedReductionWithStartValue(
7121b8b5564SNicolas Vasilache     ConversionPatternRewriter &rewriter, Location loc, Type llvmType,
7131b8b5564SNicolas Vasilache     Value vectorOperand, Value accumulator, Value mask) {
714e9b82a5cSDiego Caballero   if (llvmType.isIntOrIndex())
7151b8b5564SNicolas Vasilache     return lowerPredicatedReductionWithStartValue<LLVMIntVPRedIntrinOp,
716e9b82a5cSDiego Caballero                                                   IntReductionNeutral>(
717e9b82a5cSDiego Caballero         rewriter, loc, llvmType, vectorOperand, accumulator, mask);
718e9b82a5cSDiego Caballero 
719e9b82a5cSDiego Caballero   // FP dispatch.
7201b8b5564SNicolas Vasilache   return lowerPredicatedReductionWithStartValue<LLVMFPVPRedIntrinOp,
7211b8b5564SNicolas Vasilache                                                 FPReductionNeutral>(
722e9b82a5cSDiego Caballero       rewriter, loc, llvmType, vectorOperand, accumulator, mask);
723e9b82a5cSDiego Caballero }
724e9b82a5cSDiego Caballero 
72519dbb230Saartbik /// Conversion pattern for all vector reductions.
726563879b6SRahul Joshi class VectorReductionOpConversion
727563879b6SRahul Joshi     : public ConvertOpToLLVMPattern<vector::ReductionOp> {
728e83b7b99Saartbik public:
729ce254598SMatthias Springer   explicit VectorReductionOpConversion(const LLVMTypeConverter &typeConv,
730060c9dd1Saartbik                                        bool reassociateFPRed)
731563879b6SRahul Joshi       : ConvertOpToLLVMPattern<vector::ReductionOp>(typeConv),
732060c9dd1Saartbik         reassociateFPReductions(reassociateFPRed) {}
733e83b7b99Saartbik 
7343145427dSRiver Riddle   LogicalResult
735ef976337SRiver Riddle   matchAndRewrite(vector::ReductionOp reductionOp, OpAdaptor adaptor,
736e83b7b99Saartbik                   ConversionPatternRewriter &rewriter) const override {
7377c38fd60SJacques Pienaar     auto kind = reductionOp.getKind();
7387c38fd60SJacques Pienaar     Type eltType = reductionOp.getDest().getType();
739dcec2ca5SChristian Sigg     Type llvmType = typeConverter->convertType(eltType);
740fa596c69SMahesh Ravishankar     Value operand = adaptor.getVector();
741fa596c69SMahesh Ravishankar     Value acc = adaptor.getAcc();
742fa596c69SMahesh Ravishankar     Location loc = reductionOp.getLoc();
743e9b82a5cSDiego Caballero 
744e9628955SAart Bik     if (eltType.isIntOrIndex()) {
745e83b7b99Saartbik       // Integer reductions: add/mul/min/max/and/or/xor.
746fa596c69SMahesh Ravishankar       Value result;
747fa596c69SMahesh Ravishankar       switch (kind) {
748fa596c69SMahesh Ravishankar       case vector::CombiningKind::ADD:
749fa596c69SMahesh Ravishankar         result =
750fa596c69SMahesh Ravishankar             createIntegerReductionArithmeticOpLowering<LLVM::vector_reduce_add,
751fa596c69SMahesh Ravishankar                                                        LLVM::AddOp>(
752fa596c69SMahesh Ravishankar                 rewriter, loc, llvmType, operand, acc);
753fa596c69SMahesh Ravishankar         break;
754fa596c69SMahesh Ravishankar       case vector::CombiningKind::MUL:
755fa596c69SMahesh Ravishankar         result =
756fa596c69SMahesh Ravishankar             createIntegerReductionArithmeticOpLowering<LLVM::vector_reduce_mul,
757fa596c69SMahesh Ravishankar                                                        LLVM::MulOp>(
758fa596c69SMahesh Ravishankar                 rewriter, loc, llvmType, operand, acc);
759fa596c69SMahesh Ravishankar         break;
760fa596c69SMahesh Ravishankar       case vector::CombiningKind::MINUI:
761fa596c69SMahesh Ravishankar         result = createIntegerReductionComparisonOpLowering<
762fa596c69SMahesh Ravishankar             LLVM::vector_reduce_umin>(rewriter, loc, llvmType, operand, acc,
763fa596c69SMahesh Ravishankar                                       LLVM::ICmpPredicate::ule);
764fa596c69SMahesh Ravishankar         break;
765fa596c69SMahesh Ravishankar       case vector::CombiningKind::MINSI:
766fa596c69SMahesh Ravishankar         result = createIntegerReductionComparisonOpLowering<
767fa596c69SMahesh Ravishankar             LLVM::vector_reduce_smin>(rewriter, loc, llvmType, operand, acc,
768fa596c69SMahesh Ravishankar                                       LLVM::ICmpPredicate::sle);
769fa596c69SMahesh Ravishankar         break;
770fa596c69SMahesh Ravishankar       case vector::CombiningKind::MAXUI:
771fa596c69SMahesh Ravishankar         result = createIntegerReductionComparisonOpLowering<
772fa596c69SMahesh Ravishankar             LLVM::vector_reduce_umax>(rewriter, loc, llvmType, operand, acc,
773fa596c69SMahesh Ravishankar                                       LLVM::ICmpPredicate::uge);
774fa596c69SMahesh Ravishankar         break;
775fa596c69SMahesh Ravishankar       case vector::CombiningKind::MAXSI:
776fa596c69SMahesh Ravishankar         result = createIntegerReductionComparisonOpLowering<
777fa596c69SMahesh Ravishankar             LLVM::vector_reduce_smax>(rewriter, loc, llvmType, operand, acc,
778fa596c69SMahesh Ravishankar                                       LLVM::ICmpPredicate::sge);
779fa596c69SMahesh Ravishankar         break;
780fa596c69SMahesh Ravishankar       case vector::CombiningKind::AND:
781fa596c69SMahesh Ravishankar         result =
782fa596c69SMahesh Ravishankar             createIntegerReductionArithmeticOpLowering<LLVM::vector_reduce_and,
783fa596c69SMahesh Ravishankar                                                        LLVM::AndOp>(
784fa596c69SMahesh Ravishankar                 rewriter, loc, llvmType, operand, acc);
785fa596c69SMahesh Ravishankar         break;
786fa596c69SMahesh Ravishankar       case vector::CombiningKind::OR:
787fa596c69SMahesh Ravishankar         result =
788fa596c69SMahesh Ravishankar             createIntegerReductionArithmeticOpLowering<LLVM::vector_reduce_or,
789fa596c69SMahesh Ravishankar                                                        LLVM::OrOp>(
790fa596c69SMahesh Ravishankar                 rewriter, loc, llvmType, operand, acc);
791fa596c69SMahesh Ravishankar         break;
792fa596c69SMahesh Ravishankar       case vector::CombiningKind::XOR:
793fa596c69SMahesh Ravishankar         result =
794fa596c69SMahesh Ravishankar             createIntegerReductionArithmeticOpLowering<LLVM::vector_reduce_xor,
795fa596c69SMahesh Ravishankar                                                        LLVM::XOrOp>(
796fa596c69SMahesh Ravishankar                 rewriter, loc, llvmType, operand, acc);
797fa596c69SMahesh Ravishankar         break;
798fa596c69SMahesh Ravishankar       default:
7993145427dSRiver Riddle         return failure();
800fa596c69SMahesh Ravishankar       }
801fa596c69SMahesh Ravishankar       rewriter.replaceOp(reductionOp, result);
802fa596c69SMahesh Ravishankar 
8033145427dSRiver Riddle       return success();
804dcec2ca5SChristian Sigg     }
805e83b7b99Saartbik 
8065550c821STres Popp     if (!isa<FloatType>(eltType))
807dcec2ca5SChristian Sigg       return failure();
808dcec2ca5SChristian Sigg 
8091b8b5564SNicolas Vasilache     arith::FastMathFlagsAttr fMFAttr = reductionOp.getFastMathFlagsAttr();
8101b8b5564SNicolas Vasilache     LLVM::FastmathFlagsAttr fmf = LLVM::FastmathFlagsAttr::get(
8111b8b5564SNicolas Vasilache         reductionOp.getContext(),
8121b8b5564SNicolas Vasilache         convertArithFastMathFlagsToLLVM(fMFAttr.getValue()));
8131b8b5564SNicolas Vasilache     fmf = LLVM::FastmathFlagsAttr::get(
8141b8b5564SNicolas Vasilache         reductionOp.getContext(),
8151b8b5564SNicolas Vasilache         fmf.getValue() | (reassociateFPReductions ? LLVM::FastmathFlags::reassoc
8161b8b5564SNicolas Vasilache                                                   : LLVM::FastmathFlags::none));
8171b8b5564SNicolas Vasilache 
818e83b7b99Saartbik     // Floating-point reductions: add/mul/min/max
819e9b82a5cSDiego Caballero     Value result;
820fe0bf7d4SMatthias Springer     if (kind == vector::CombiningKind::ADD) {
821e9b82a5cSDiego Caballero       result = lowerReductionWithStartValue<LLVM::vector_reduce_fadd,
822e9b82a5cSDiego Caballero                                             ReductionNeutralZero>(
8231b8b5564SNicolas Vasilache           rewriter, loc, llvmType, operand, acc, fmf);
824fe0bf7d4SMatthias Springer     } else if (kind == vector::CombiningKind::MUL) {
825e9b82a5cSDiego Caballero       result = lowerReductionWithStartValue<LLVM::vector_reduce_fmul,
826e9b82a5cSDiego Caballero                                             ReductionNeutralFPOne>(
8271b8b5564SNicolas Vasilache           rewriter, loc, llvmType, operand, acc, fmf);
8284a831250SDaniil Dudkin     } else if (kind == vector::CombiningKind::MINIMUMF) {
829dad9de0aSDaniil Dudkin       result =
830dad9de0aSDaniil Dudkin           createFPReductionComparisonOpLowering<LLVM::vector_reduce_fminimum>(
8311b8b5564SNicolas Vasilache               rewriter, loc, llvmType, operand, acc, fmf);
8324a831250SDaniil Dudkin     } else if (kind == vector::CombiningKind::MAXIMUMF) {
833dad9de0aSDaniil Dudkin       result =
834dad9de0aSDaniil Dudkin           createFPReductionComparisonOpLowering<LLVM::vector_reduce_fmaximum>(
8351b8b5564SNicolas Vasilache               rewriter, loc, llvmType, operand, acc, fmf);
836560564f5SJakub Kuderski     } else if (kind == vector::CombiningKind::MINNUMF) {
837709b2742SDaniil Dudkin       result = createFPReductionComparisonOpLowering<LLVM::vector_reduce_fmin>(
8381b8b5564SNicolas Vasilache           rewriter, loc, llvmType, operand, acc, fmf);
839560564f5SJakub Kuderski     } else if (kind == vector::CombiningKind::MAXNUMF) {
840709b2742SDaniil Dudkin       result = createFPReductionComparisonOpLowering<LLVM::vector_reduce_fmax>(
8411b8b5564SNicolas Vasilache           rewriter, loc, llvmType, operand, acc, fmf);
8428fe076ffSThomas Raoux     } else
8433145427dSRiver Riddle       return failure();
8448fe076ffSThomas Raoux 
845e9b82a5cSDiego Caballero     rewriter.replaceOp(reductionOp, result);
8463145427dSRiver Riddle     return success();
847e83b7b99Saartbik   }
848ceb1b327Saartbik 
849ceb1b327Saartbik private:
850ceb1b327Saartbik   const bool reassociateFPReductions;
851e83b7b99Saartbik };
852e83b7b99Saartbik 
853e9b82a5cSDiego Caballero /// Base class to convert a `vector.mask` operation while matching traits
854e9b82a5cSDiego Caballero /// of the maskable operation nested inside. A `VectorMaskOpConversionBase`
855e9b82a5cSDiego Caballero /// instance matches against a `vector.mask` operation. The `matchAndRewrite`
856e9b82a5cSDiego Caballero /// method performs a second match against the maskable operation `MaskedOp`.
857e9b82a5cSDiego Caballero /// Finally, it invokes the virtual method `matchAndRewriteMaskableOp` to be
858e9b82a5cSDiego Caballero /// implemented by the concrete conversion classes. This method can match
859e9b82a5cSDiego Caballero /// against specific traits of the `vector.mask` and the maskable operation. It
860e9b82a5cSDiego Caballero /// must replace the `vector.mask` operation.
861e9b82a5cSDiego Caballero template <class MaskedOp>
862e9b82a5cSDiego Caballero class VectorMaskOpConversionBase
863e9b82a5cSDiego Caballero     : public ConvertOpToLLVMPattern<vector::MaskOp> {
864e9b82a5cSDiego Caballero public:
865e9b82a5cSDiego Caballero   using ConvertOpToLLVMPattern<vector::MaskOp>::ConvertOpToLLVMPattern;
866e9b82a5cSDiego Caballero 
867e9b82a5cSDiego Caballero   LogicalResult
868e9b82a5cSDiego Caballero   matchAndRewrite(vector::MaskOp maskOp, OpAdaptor adaptor,
8694eb1f1faSAdrian Kuegel                   ConversionPatternRewriter &rewriter) const final {
870e9b82a5cSDiego Caballero     // Match against the maskable operation kind.
87169049988SMehdi Amini     auto maskedOp = llvm::dyn_cast_or_null<MaskedOp>(maskOp.getMaskableOp());
87269049988SMehdi Amini     if (!maskedOp)
873e9b82a5cSDiego Caballero       return failure();
87469049988SMehdi Amini     return matchAndRewriteMaskableOp(maskOp, maskedOp, rewriter);
875e9b82a5cSDiego Caballero   }
876e9b82a5cSDiego Caballero 
877e9b82a5cSDiego Caballero protected:
878e9b82a5cSDiego Caballero   virtual LogicalResult
879e9b82a5cSDiego Caballero   matchAndRewriteMaskableOp(vector::MaskOp maskOp,
880e9b82a5cSDiego Caballero                             vector::MaskableOpInterface maskableOp,
881e9b82a5cSDiego Caballero                             ConversionPatternRewriter &rewriter) const = 0;
882e9b82a5cSDiego Caballero };
883e9b82a5cSDiego Caballero 
884e9b82a5cSDiego Caballero class MaskedReductionOpConversion
885e9b82a5cSDiego Caballero     : public VectorMaskOpConversionBase<vector::ReductionOp> {
886e9b82a5cSDiego Caballero 
887e9b82a5cSDiego Caballero public:
888e9b82a5cSDiego Caballero   using VectorMaskOpConversionBase<
889e9b82a5cSDiego Caballero       vector::ReductionOp>::VectorMaskOpConversionBase;
890e9b82a5cSDiego Caballero 
8914eb1f1faSAdrian Kuegel   LogicalResult matchAndRewriteMaskableOp(
892e9b82a5cSDiego Caballero       vector::MaskOp maskOp, MaskableOpInterface maskableOp,
893e9b82a5cSDiego Caballero       ConversionPatternRewriter &rewriter) const override {
894e9b82a5cSDiego Caballero     auto reductionOp = cast<ReductionOp>(maskableOp.getOperation());
895e9b82a5cSDiego Caballero     auto kind = reductionOp.getKind();
896e9b82a5cSDiego Caballero     Type eltType = reductionOp.getDest().getType();
897e9b82a5cSDiego Caballero     Type llvmType = typeConverter->convertType(eltType);
898e9b82a5cSDiego Caballero     Value operand = reductionOp.getVector();
899e9b82a5cSDiego Caballero     Value acc = reductionOp.getAcc();
900e9b82a5cSDiego Caballero     Location loc = reductionOp.getLoc();
901e9b82a5cSDiego Caballero 
9021b8b5564SNicolas Vasilache     arith::FastMathFlagsAttr fMFAttr = reductionOp.getFastMathFlagsAttr();
9031b8b5564SNicolas Vasilache     LLVM::FastmathFlagsAttr fmf = LLVM::FastmathFlagsAttr::get(
9041b8b5564SNicolas Vasilache         reductionOp.getContext(),
9051b8b5564SNicolas Vasilache         convertArithFastMathFlagsToLLVM(fMFAttr.getValue()));
9061b8b5564SNicolas Vasilache 
907e9b82a5cSDiego Caballero     Value result;
908e9b82a5cSDiego Caballero     switch (kind) {
909e9b82a5cSDiego Caballero     case vector::CombiningKind::ADD:
9101b8b5564SNicolas Vasilache       result = lowerPredicatedReductionWithStartValue<
911e9b82a5cSDiego Caballero           LLVM::VPReduceAddOp, ReductionNeutralZero, LLVM::VPReduceFAddOp,
912e9b82a5cSDiego Caballero           ReductionNeutralZero>(rewriter, loc, llvmType, operand, acc,
913e9b82a5cSDiego Caballero                                 maskOp.getMask());
914e9b82a5cSDiego Caballero       break;
915e9b82a5cSDiego Caballero     case vector::CombiningKind::MUL:
9161b8b5564SNicolas Vasilache       result = lowerPredicatedReductionWithStartValue<
917e9b82a5cSDiego Caballero           LLVM::VPReduceMulOp, ReductionNeutralIntOne, LLVM::VPReduceFMulOp,
918e9b82a5cSDiego Caballero           ReductionNeutralFPOne>(rewriter, loc, llvmType, operand, acc,
919e9b82a5cSDiego Caballero                                  maskOp.getMask());
920e9b82a5cSDiego Caballero       break;
921e9b82a5cSDiego Caballero     case vector::CombiningKind::MINUI:
9221b8b5564SNicolas Vasilache       result = lowerPredicatedReductionWithStartValue<LLVM::VPReduceUMinOp,
923e9b82a5cSDiego Caballero                                                       ReductionNeutralUIntMax>(
924e9b82a5cSDiego Caballero           rewriter, loc, llvmType, operand, acc, maskOp.getMask());
925e9b82a5cSDiego Caballero       break;
926e9b82a5cSDiego Caballero     case vector::CombiningKind::MINSI:
9271b8b5564SNicolas Vasilache       result = lowerPredicatedReductionWithStartValue<LLVM::VPReduceSMinOp,
928e9b82a5cSDiego Caballero                                                       ReductionNeutralSIntMax>(
929e9b82a5cSDiego Caballero           rewriter, loc, llvmType, operand, acc, maskOp.getMask());
930e9b82a5cSDiego Caballero       break;
931e9b82a5cSDiego Caballero     case vector::CombiningKind::MAXUI:
9321b8b5564SNicolas Vasilache       result = lowerPredicatedReductionWithStartValue<LLVM::VPReduceUMaxOp,
933e9b82a5cSDiego Caballero                                                       ReductionNeutralUIntMin>(
934e9b82a5cSDiego Caballero           rewriter, loc, llvmType, operand, acc, maskOp.getMask());
935e9b82a5cSDiego Caballero       break;
936e9b82a5cSDiego Caballero     case vector::CombiningKind::MAXSI:
9371b8b5564SNicolas Vasilache       result = lowerPredicatedReductionWithStartValue<LLVM::VPReduceSMaxOp,
938e9b82a5cSDiego Caballero                                                       ReductionNeutralSIntMin>(
939e9b82a5cSDiego Caballero           rewriter, loc, llvmType, operand, acc, maskOp.getMask());
940e9b82a5cSDiego Caballero       break;
941e9b82a5cSDiego Caballero     case vector::CombiningKind::AND:
9421b8b5564SNicolas Vasilache       result = lowerPredicatedReductionWithStartValue<LLVM::VPReduceAndOp,
943e9b82a5cSDiego Caballero                                                       ReductionNeutralAllOnes>(
944e9b82a5cSDiego Caballero           rewriter, loc, llvmType, operand, acc, maskOp.getMask());
945e9b82a5cSDiego Caballero       break;
946e9b82a5cSDiego Caballero     case vector::CombiningKind::OR:
9471b8b5564SNicolas Vasilache       result = lowerPredicatedReductionWithStartValue<LLVM::VPReduceOrOp,
948e9b82a5cSDiego Caballero                                                       ReductionNeutralZero>(
949e9b82a5cSDiego Caballero           rewriter, loc, llvmType, operand, acc, maskOp.getMask());
950e9b82a5cSDiego Caballero       break;
951e9b82a5cSDiego Caballero     case vector::CombiningKind::XOR:
9521b8b5564SNicolas Vasilache       result = lowerPredicatedReductionWithStartValue<LLVM::VPReduceXorOp,
953e9b82a5cSDiego Caballero                                                       ReductionNeutralZero>(
954e9b82a5cSDiego Caballero           rewriter, loc, llvmType, operand, acc, maskOp.getMask());
955e9b82a5cSDiego Caballero       break;
956560564f5SJakub Kuderski     case vector::CombiningKind::MINNUMF:
9571b8b5564SNicolas Vasilache       result = lowerPredicatedReductionWithStartValue<LLVM::VPReduceFMinOp,
958e9b82a5cSDiego Caballero                                                       ReductionNeutralFPMax>(
959e9b82a5cSDiego Caballero           rewriter, loc, llvmType, operand, acc, maskOp.getMask());
960e9b82a5cSDiego Caballero       break;
961560564f5SJakub Kuderski     case vector::CombiningKind::MAXNUMF:
9621b8b5564SNicolas Vasilache       result = lowerPredicatedReductionWithStartValue<LLVM::VPReduceFMaxOp,
963e9b82a5cSDiego Caballero                                                       ReductionNeutralFPMin>(
964e9b82a5cSDiego Caballero           rewriter, loc, llvmType, operand, acc, maskOp.getMask());
965e9b82a5cSDiego Caballero       break;
9668f5d5194SDaniil Dudkin     case CombiningKind::MAXIMUMF:
9678f5d5194SDaniil Dudkin       result = lowerMaskedReductionWithRegular<LLVM::vector_reduce_fmaximum,
9688f5d5194SDaniil Dudkin                                                MaskNeutralFMaximum>(
9691b8b5564SNicolas Vasilache           rewriter, loc, llvmType, operand, acc, maskOp.getMask(), fmf);
9708f5d5194SDaniil Dudkin       break;
9718f5d5194SDaniil Dudkin     case CombiningKind::MINIMUMF:
9728f5d5194SDaniil Dudkin       result = lowerMaskedReductionWithRegular<LLVM::vector_reduce_fminimum,
9738f5d5194SDaniil Dudkin                                                MaskNeutralFMinimum>(
9741b8b5564SNicolas Vasilache           rewriter, loc, llvmType, operand, acc, maskOp.getMask(), fmf);
9758f5d5194SDaniil Dudkin       break;
976e9b82a5cSDiego Caballero     }
977e9b82a5cSDiego Caballero 
978e9b82a5cSDiego Caballero     // Replace `vector.mask` operation altogether.
979e9b82a5cSDiego Caballero     rewriter.replaceOp(maskOp, result);
980e9b82a5cSDiego Caballero     return success();
981e9b82a5cSDiego Caballero   }
982e9b82a5cSDiego Caballero };
983e9b82a5cSDiego Caballero 
984563879b6SRahul Joshi class VectorShuffleOpConversion
985563879b6SRahul Joshi     : public ConvertOpToLLVMPattern<vector::ShuffleOp> {
9861c81adf3SAart Bik public:
987563879b6SRahul Joshi   using ConvertOpToLLVMPattern<vector::ShuffleOp>::ConvertOpToLLVMPattern;
9881c81adf3SAart Bik 
9893145427dSRiver Riddle   LogicalResult
990ef976337SRiver Riddle   matchAndRewrite(vector::ShuffleOp shuffleOp, OpAdaptor adaptor,
9911c81adf3SAart Bik                   ConversionPatternRewriter &rewriter) const override {
992563879b6SRahul Joshi     auto loc = shuffleOp->getLoc();
9931c81adf3SAart Bik     auto v1Type = shuffleOp.getV1VectorType();
9941c81adf3SAart Bik     auto v2Type = shuffleOp.getV2VectorType();
995a1aad28dSLei Zhang     auto vectorType = shuffleOp.getResultVectorType();
996dcec2ca5SChristian Sigg     Type llvmType = typeConverter->convertType(vectorType);
997b4444dcaSBenjamin Maxwell     ArrayRef<int64_t> mask = shuffleOp.getMask();
9981c81adf3SAart Bik 
9991c81adf3SAart Bik     // Bail if result type cannot be lowered.
10001c81adf3SAart Bik     if (!llvmType)
10013145427dSRiver Riddle       return failure();
10021c81adf3SAart Bik 
10031c81adf3SAart Bik     // Get rank and dimension sizes.
10041c81adf3SAart Bik     int64_t rank = vectorType.getRank();
1005fe09cee3SBenjamin Kramer #ifndef NDEBUG
1006db6f8ebeSNicolas Vasilache     bool wellFormed0DCase =
1007db6f8ebeSNicolas Vasilache         v1Type.getRank() == 0 && v2Type.getRank() == 0 && rank == 1;
1008db6f8ebeSNicolas Vasilache     bool wellFormedNDCase =
1009db6f8ebeSNicolas Vasilache         v1Type.getRank() == rank && v2Type.getRank() == rank;
1010db6f8ebeSNicolas Vasilache     assert((wellFormed0DCase || wellFormedNDCase) && "op is not well-formed");
1011fe09cee3SBenjamin Kramer #endif
10121c81adf3SAart Bik 
1013db6f8ebeSNicolas Vasilache     // For rank 0 and 1, where both operands have *exactly* the same vector
1014db6f8ebeSNicolas Vasilache     // type, there is direct shuffle support in LLVM. Use it!
1015db6f8ebeSNicolas Vasilache     if (rank <= 1 && v1Type == v2Type) {
1016563879b6SRahul Joshi       Value llvmShuffleOp = rewriter.create<LLVM::ShuffleVectorOp>(
1017b2ccfb4dSJeff Niu           loc, adaptor.getV1(), adaptor.getV2(),
1018b4444dcaSBenjamin Maxwell           llvm::to_vector_of<int32_t>(mask));
1019563879b6SRahul Joshi       rewriter.replaceOp(shuffleOp, llvmShuffleOp);
10203145427dSRiver Riddle       return success();
1021b36aaeafSAart Bik     }
1022b36aaeafSAart Bik 
10231c81adf3SAart Bik     // For all other cases, insert the individual values individually.
1024db6f8ebeSNicolas Vasilache     int64_t v1Dim = v1Type.getDimSize(0);
10255a8a159bSMehdi Amini     Type eltType;
10265550c821STres Popp     if (auto arrayType = dyn_cast<LLVM::LLVMArrayType>(llvmType))
10275a8a159bSMehdi Amini       eltType = arrayType.getElementType();
10285a8a159bSMehdi Amini     else
10295550c821STres Popp       eltType = cast<VectorType>(llvmType).getElementType();
1030e62a6956SRiver Riddle     Value insert = rewriter.create<LLVM::UndefOp>(loc, llvmType);
10311c81adf3SAart Bik     int64_t insPos = 0;
1032b4444dcaSBenjamin Maxwell     for (int64_t extPos : mask) {
10337c38fd60SJacques Pienaar       Value value = adaptor.getV1();
10341c81adf3SAart Bik       if (extPos >= v1Dim) {
10351c81adf3SAart Bik         extPos -= v1Dim;
10367c38fd60SJacques Pienaar         value = adaptor.getV2();
1037b36aaeafSAart Bik       }
1038dcec2ca5SChristian Sigg       Value extract = extractOne(rewriter, *getTypeConverter(), loc, value,
10395a8a159bSMehdi Amini                                  eltType, rank, extPos);
1040dcec2ca5SChristian Sigg       insert = insertOne(rewriter, *getTypeConverter(), loc, insert, extract,
10410f04384dSAlex Zinenko                          llvmType, rank, insPos++);
10421c81adf3SAart Bik     }
1043563879b6SRahul Joshi     rewriter.replaceOp(shuffleOp, insert);
10443145427dSRiver Riddle     return success();
1045b36aaeafSAart Bik   }
1046b36aaeafSAart Bik };
1047b36aaeafSAart Bik 
1048563879b6SRahul Joshi class VectorExtractElementOpConversion
1049563879b6SRahul Joshi     : public ConvertOpToLLVMPattern<vector::ExtractElementOp> {
1050cd5dab8aSAart Bik public:
1051563879b6SRahul Joshi   using ConvertOpToLLVMPattern<
1052563879b6SRahul Joshi       vector::ExtractElementOp>::ConvertOpToLLVMPattern;
1053cd5dab8aSAart Bik 
10543145427dSRiver Riddle   LogicalResult
1055ef976337SRiver Riddle   matchAndRewrite(vector::ExtractElementOp extractEltOp, OpAdaptor adaptor,
1056cd5dab8aSAart Bik                   ConversionPatternRewriter &rewriter) const override {
1057a1aad28dSLei Zhang     auto vectorType = extractEltOp.getSourceVectorType();
1058dcec2ca5SChristian Sigg     auto llvmType = typeConverter->convertType(vectorType.getElementType());
1059cd5dab8aSAart Bik 
1060cd5dab8aSAart Bik     // Bail if result type cannot be lowered.
1061cd5dab8aSAart Bik     if (!llvmType)
10623145427dSRiver Riddle       return failure();
1063cd5dab8aSAart Bik 
1064e7026abaSNicolas Vasilache     if (vectorType.getRank() == 0) {
1065e7026abaSNicolas Vasilache       Location loc = extractEltOp.getLoc();
1066e7026abaSNicolas Vasilache       auto idxType = rewriter.getIndexType();
1067e7026abaSNicolas Vasilache       auto zero = rewriter.create<LLVM::ConstantOp>(
1068e7026abaSNicolas Vasilache           loc, typeConverter->convertType(idxType),
1069e7026abaSNicolas Vasilache           rewriter.getIntegerAttr(idxType, 0));
1070e7026abaSNicolas Vasilache       rewriter.replaceOpWithNewOp<LLVM::ExtractElementOp>(
10717c38fd60SJacques Pienaar           extractEltOp, llvmType, adaptor.getVector(), zero);
1072e7026abaSNicolas Vasilache       return success();
1073e7026abaSNicolas Vasilache     }
1074e7026abaSNicolas Vasilache 
1075cd5dab8aSAart Bik     rewriter.replaceOpWithNewOp<LLVM::ExtractElementOp>(
10767c38fd60SJacques Pienaar         extractEltOp, llvmType, adaptor.getVector(), adaptor.getPosition());
10773145427dSRiver Riddle     return success();
1078cd5dab8aSAart Bik   }
1079cd5dab8aSAart Bik };
1080cd5dab8aSAart Bik 
1081563879b6SRahul Joshi class VectorExtractOpConversion
1082563879b6SRahul Joshi     : public ConvertOpToLLVMPattern<vector::ExtractOp> {
10835c0c51a9SNicolas Vasilache public:
1084563879b6SRahul Joshi   using ConvertOpToLLVMPattern<vector::ExtractOp>::ConvertOpToLLVMPattern;
10855c0c51a9SNicolas Vasilache 
10863145427dSRiver Riddle   LogicalResult
1087ef976337SRiver Riddle   matchAndRewrite(vector::ExtractOp extractOp, OpAdaptor adaptor,
10885c0c51a9SNicolas Vasilache                   ConversionPatternRewriter &rewriter) const override {
1089563879b6SRahul Joshi     auto loc = extractOp->getLoc();
10902bdf33ccSRiver Riddle     auto resultType = extractOp.getResult().getType();
1091dcec2ca5SChristian Sigg     auto llvmResultType = typeConverter->convertType(resultType);
10929826fe5cSAart Bik     // Bail if result type cannot be lowered.
10939826fe5cSAart Bik     if (!llvmResultType)
10943145427dSRiver Riddle       return failure();
10959826fe5cSAart Bik 
109642a6ad7bSDiego Caballero     SmallVector<OpFoldResult> positionVec = getMixedValues(
109742a6ad7bSDiego Caballero         adaptor.getStaticPosition(), adaptor.getDynamicPosition(), rewriter);
109898f6289aSDiego Caballero 
1099a8f92716SKunwar Grover     // The Vector -> LLVM lowering models N-D vectors as nested aggregates of
1100a8f92716SKunwar Grover     // 1-d vectors. This nesting is modeled using arrays. We do this conversion
1101a8f92716SKunwar Grover     // from a N-d vector extract to a nested aggregate vector extract in two
1102a8f92716SKunwar Grover     // steps:
1103a8f92716SKunwar Grover     //  - Extract a member from the nested aggregate. The result can be
1104a8f92716SKunwar Grover     //    a lower rank nested aggregate or a vector (1-D). This is done using
1105a8f92716SKunwar Grover     //    `llvm.extractvalue`.
1106a8f92716SKunwar Grover     //  - Extract a scalar out of the vector if needed. This is done using
1107a8f92716SKunwar Grover     //   `llvm.extractelement`.
1108a8f92716SKunwar Grover 
1109a8f92716SKunwar Grover     // Determine if we need to extract a member out of the aggregate. We
1110a8f92716SKunwar Grover     // always need to extract a member if the input rank >= 2.
1111a8f92716SKunwar Grover     bool extractsAggregate = extractOp.getSourceVectorType().getRank() >= 2;
1112a8f92716SKunwar Grover     // Determine if we need to extract a scalar as the result. We extract
1113a8f92716SKunwar Grover     // a scalar if the extract is full rank, i.e., the number of indices is
1114a8f92716SKunwar Grover     // equal to source vector rank.
1115a8f92716SKunwar Grover     bool extractsScalar = static_cast<int64_t>(positionVec.size()) ==
1116a8f92716SKunwar Grover                           extractOp.getSourceVectorType().getRank();
1117a8f92716SKunwar Grover 
1118a8f92716SKunwar Grover     // Since the LLVM type converter converts 0-d vectors to 1-d vectors, we
1119a8f92716SKunwar Grover     // need to add a position for this change.
1120a8f92716SKunwar Grover     if (extractOp.getSourceVectorType().getRank() == 0) {
1121a8f92716SKunwar Grover       Type idxType = typeConverter->convertType(rewriter.getIndexType());
1122a8f92716SKunwar Grover       positionVec.push_back(rewriter.getZeroAttr(idxType));
1123864adf39SMatthias Springer     }
1124864adf39SMatthias Springer 
11257c38fd60SJacques Pienaar     Value extracted = adaptor.getVector();
1126a8f92716SKunwar Grover     if (extractsAggregate) {
1127a8f92716SKunwar Grover       ArrayRef<OpFoldResult> position(positionVec);
1128a8f92716SKunwar Grover       if (extractsScalar) {
1129a8f92716SKunwar Grover         // If we are extracting a scalar from the extracted member, we drop
1130a8f92716SKunwar Grover         // the last index, which will be used to extract the scalar out of the
1131a8f92716SKunwar Grover         // vector.
1132a8f92716SKunwar Grover         position = position.drop_back();
1133a8f92716SKunwar Grover       }
1134a8f92716SKunwar Grover       // llvm.extractvalue does not support dynamic dimensions.
1135a8f92716SKunwar Grover       if (!llvm::all_of(position, llvm::IsaPred<Attribute>)) {
113698f6289aSDiego Caballero         return failure();
1137a8f92716SKunwar Grover       }
1138a8f92716SKunwar Grover       extracted = rewriter.create<LLVM::ExtractValueOp>(
1139a8f92716SKunwar Grover           loc, extracted, getAsIntegers(position));
11405c0c51a9SNicolas Vasilache     }
11415c0c51a9SNicolas Vasilache 
1142a8f92716SKunwar Grover     if (extractsScalar) {
1143a8f92716SKunwar Grover       extracted = rewriter.create<LLVM::ExtractElementOp>(
1144a8f92716SKunwar Grover           loc, extracted, getAsLLVMValue(rewriter, loc, positionVec.back()));
1145a8f92716SKunwar Grover     }
1146a8f92716SKunwar Grover 
1147a8f92716SKunwar Grover     rewriter.replaceOp(extractOp, extracted);
11483145427dSRiver Riddle     return success();
11495c0c51a9SNicolas Vasilache   }
11505c0c51a9SNicolas Vasilache };
11515c0c51a9SNicolas Vasilache 
1152681f929fSNicolas Vasilache /// Conversion pattern that turns a vector.fma on a 1-D vector
1153681f929fSNicolas Vasilache /// into an llvm.intr.fmuladd. This is a trivial 1-1 conversion.
1154681f929fSNicolas Vasilache /// This does not match vectors of n >= 2 rank.
1155681f929fSNicolas Vasilache ///
1156681f929fSNicolas Vasilache /// Example:
1157681f929fSNicolas Vasilache /// ```
1158681f929fSNicolas Vasilache ///  vector.fma %a, %a, %a : vector<8xf32>
1159681f929fSNicolas Vasilache /// ```
1160681f929fSNicolas Vasilache /// is converted to:
1161681f929fSNicolas Vasilache /// ```
11623bffe602SBenjamin Kramer ///  llvm.intr.fmuladd %va, %va, %va:
1163dd5165a9SAlex Zinenko ///    (!llvm."<8 x f32>">, !llvm<"<8 x f32>">, !llvm<"<8 x f32>">)
1164dd5165a9SAlex Zinenko ///    -> !llvm."<8 x f32>">
1165681f929fSNicolas Vasilache /// ```
1166563879b6SRahul Joshi class VectorFMAOp1DConversion : public ConvertOpToLLVMPattern<vector::FMAOp> {
1167681f929fSNicolas Vasilache public:
1168563879b6SRahul Joshi   using ConvertOpToLLVMPattern<vector::FMAOp>::ConvertOpToLLVMPattern;
1169681f929fSNicolas Vasilache 
11703145427dSRiver Riddle   LogicalResult
1171ef976337SRiver Riddle   matchAndRewrite(vector::FMAOp fmaOp, OpAdaptor adaptor,
1172681f929fSNicolas Vasilache                   ConversionPatternRewriter &rewriter) const override {
1173681f929fSNicolas Vasilache     VectorType vType = fmaOp.getVectorType();
1174ab45a432SMichal Terepeta     if (vType.getRank() > 1)
11753145427dSRiver Riddle       return failure();
11769452356dSDiego Caballero 
11777c38fd60SJacques Pienaar     rewriter.replaceOpWithNewOp<LLVM::FMulAddOp>(
11787c38fd60SJacques Pienaar         fmaOp, adaptor.getLhs(), adaptor.getRhs(), adaptor.getAcc());
11793145427dSRiver Riddle     return success();
1180681f929fSNicolas Vasilache   }
1181681f929fSNicolas Vasilache };
1182681f929fSNicolas Vasilache 
1183563879b6SRahul Joshi class VectorInsertElementOpConversion
1184563879b6SRahul Joshi     : public ConvertOpToLLVMPattern<vector::InsertElementOp> {
1185cd5dab8aSAart Bik public:
1186563879b6SRahul Joshi   using ConvertOpToLLVMPattern<vector::InsertElementOp>::ConvertOpToLLVMPattern;
1187cd5dab8aSAart Bik 
11883145427dSRiver Riddle   LogicalResult
1189ef976337SRiver Riddle   matchAndRewrite(vector::InsertElementOp insertEltOp, OpAdaptor adaptor,
1190cd5dab8aSAart Bik                   ConversionPatternRewriter &rewriter) const override {
1191cd5dab8aSAart Bik     auto vectorType = insertEltOp.getDestVectorType();
1192dcec2ca5SChristian Sigg     auto llvmType = typeConverter->convertType(vectorType);
1193cd5dab8aSAart Bik 
1194cd5dab8aSAart Bik     // Bail if result type cannot be lowered.
1195cd5dab8aSAart Bik     if (!llvmType)
11963145427dSRiver Riddle       return failure();
1197cd5dab8aSAart Bik 
11983ff4e5f2SNicolas Vasilache     if (vectorType.getRank() == 0) {
11993ff4e5f2SNicolas Vasilache       Location loc = insertEltOp.getLoc();
12003ff4e5f2SNicolas Vasilache       auto idxType = rewriter.getIndexType();
12013ff4e5f2SNicolas Vasilache       auto zero = rewriter.create<LLVM::ConstantOp>(
12023ff4e5f2SNicolas Vasilache           loc, typeConverter->convertType(idxType),
12033ff4e5f2SNicolas Vasilache           rewriter.getIntegerAttr(idxType, 0));
12043ff4e5f2SNicolas Vasilache       rewriter.replaceOpWithNewOp<LLVM::InsertElementOp>(
12057c38fd60SJacques Pienaar           insertEltOp, llvmType, adaptor.getDest(), adaptor.getSource(), zero);
12063ff4e5f2SNicolas Vasilache       return success();
12073ff4e5f2SNicolas Vasilache     }
12083ff4e5f2SNicolas Vasilache 
1209cd5dab8aSAart Bik     rewriter.replaceOpWithNewOp<LLVM::InsertElementOp>(
12107c38fd60SJacques Pienaar         insertEltOp, llvmType, adaptor.getDest(), adaptor.getSource(),
12117c38fd60SJacques Pienaar         adaptor.getPosition());
12123145427dSRiver Riddle     return success();
1213cd5dab8aSAart Bik   }
1214cd5dab8aSAart Bik };
1215cd5dab8aSAart Bik 
1216563879b6SRahul Joshi class VectorInsertOpConversion
1217563879b6SRahul Joshi     : public ConvertOpToLLVMPattern<vector::InsertOp> {
12189826fe5cSAart Bik public:
1219563879b6SRahul Joshi   using ConvertOpToLLVMPattern<vector::InsertOp>::ConvertOpToLLVMPattern;
12209826fe5cSAart Bik 
12213145427dSRiver Riddle   LogicalResult
1222ef976337SRiver Riddle   matchAndRewrite(vector::InsertOp insertOp, OpAdaptor adaptor,
12239826fe5cSAart Bik                   ConversionPatternRewriter &rewriter) const override {
1224563879b6SRahul Joshi     auto loc = insertOp->getLoc();
12259826fe5cSAart Bik     auto sourceType = insertOp.getSourceType();
12269826fe5cSAart Bik     auto destVectorType = insertOp.getDestVectorType();
1227dcec2ca5SChristian Sigg     auto llvmResultType = typeConverter->convertType(destVectorType);
12289826fe5cSAart Bik     // Bail if result type cannot be lowered.
12299826fe5cSAart Bik     if (!llvmResultType)
12303145427dSRiver Riddle       return failure();
12319826fe5cSAart Bik 
123242a6ad7bSDiego Caballero     SmallVector<OpFoldResult> positionVec = getMixedValues(
123342a6ad7bSDiego Caballero         adaptor.getStaticPosition(), adaptor.getDynamicPosition(), rewriter);
123498f6289aSDiego Caballero 
1235864adf39SMatthias Springer     // Overwrite entire vector with value. Should be handled by folder, but
1236864adf39SMatthias Springer     // just to be safe.
123798f6289aSDiego Caballero     ArrayRef<OpFoldResult> position(positionVec);
123898f6289aSDiego Caballero     if (position.empty()) {
12397c38fd60SJacques Pienaar       rewriter.replaceOp(insertOp, adaptor.getSource());
1240864adf39SMatthias Springer       return success();
1241864adf39SMatthias Springer     }
1242864adf39SMatthias Springer 
12439826fe5cSAart Bik     // One-shot insertion of a vector into an array (only requires insertvalue).
12445550c821STres Popp     if (isa<VectorType>(sourceType)) {
124598f6289aSDiego Caballero       if (insertOp.hasDynamicPosition())
124698f6289aSDiego Caballero         return failure();
124798f6289aSDiego Caballero 
1248e62a6956SRiver Riddle       Value inserted = rewriter.create<LLVM::InsertValueOp>(
124998f6289aSDiego Caballero           loc, adaptor.getDest(), adaptor.getSource(), getAsIntegers(position));
1250563879b6SRahul Joshi       rewriter.replaceOp(insertOp, inserted);
12513145427dSRiver Riddle       return success();
12529826fe5cSAart Bik     }
12539826fe5cSAart Bik 
12549826fe5cSAart Bik     // Potential extraction of 1-D vector from array.
12557c38fd60SJacques Pienaar     Value extracted = adaptor.getDest();
12569826fe5cSAart Bik     auto oneDVectorType = destVectorType;
125798f6289aSDiego Caballero     if (position.size() > 1) {
125898f6289aSDiego Caballero       if (insertOp.hasDynamicPosition())
125998f6289aSDiego Caballero         return failure();
126098f6289aSDiego Caballero 
12619826fe5cSAart Bik       oneDVectorType = reducedVectorTypeBack(destVectorType);
12629826fe5cSAart Bik       extracted = rewriter.create<LLVM::ExtractValueOp>(
126398f6289aSDiego Caballero           loc, extracted, getAsIntegers(position.drop_back()));
12649826fe5cSAart Bik     }
12659826fe5cSAart Bik 
12669826fe5cSAart Bik     // Insertion of an element into a 1-D LLVM vector.
1267e62a6956SRiver Riddle     Value inserted = rewriter.create<LLVM::InsertElementOp>(
1268dcec2ca5SChristian Sigg         loc, typeConverter->convertType(oneDVectorType), extracted,
126998f6289aSDiego Caballero         adaptor.getSource(), getAsLLVMValue(rewriter, loc, position.back()));
12709826fe5cSAart Bik 
12719826fe5cSAart Bik     // Potential insertion of resulting 1-D vector into array.
127298f6289aSDiego Caballero     if (position.size() > 1) {
127398f6289aSDiego Caballero       if (insertOp.hasDynamicPosition())
127498f6289aSDiego Caballero         return failure();
127598f6289aSDiego Caballero 
12767c38fd60SJacques Pienaar       inserted = rewriter.create<LLVM::InsertValueOp>(
127798f6289aSDiego Caballero           loc, adaptor.getDest(), inserted,
127898f6289aSDiego Caballero           getAsIntegers(position.drop_back()));
12799826fe5cSAart Bik     }
12809826fe5cSAart Bik 
1281563879b6SRahul Joshi     rewriter.replaceOp(insertOp, inserted);
12823145427dSRiver Riddle     return success();
12839826fe5cSAart Bik   }
12849826fe5cSAart Bik };
12859826fe5cSAart Bik 
1286aa9647e2SJavier Setoain /// Lower vector.scalable.insert ops to LLVM vector.insert
1287aa9647e2SJavier Setoain struct VectorScalableInsertOpLowering
1288aa9647e2SJavier Setoain     : public ConvertOpToLLVMPattern<vector::ScalableInsertOp> {
1289aa9647e2SJavier Setoain   using ConvertOpToLLVMPattern<
1290aa9647e2SJavier Setoain       vector::ScalableInsertOp>::ConvertOpToLLVMPattern;
1291aa9647e2SJavier Setoain 
1292aa9647e2SJavier Setoain   LogicalResult
1293aa9647e2SJavier Setoain   matchAndRewrite(vector::ScalableInsertOp insOp, OpAdaptor adaptor,
1294aa9647e2SJavier Setoain                   ConversionPatternRewriter &rewriter) const override {
1295aa9647e2SJavier Setoain     rewriter.replaceOpWithNewOp<LLVM::vector_insert>(
1296dbb86433SBenjamin Maxwell         insOp, adaptor.getDest(), adaptor.getSource(), adaptor.getPos());
1297aa9647e2SJavier Setoain     return success();
1298aa9647e2SJavier Setoain   }
1299aa9647e2SJavier Setoain };
1300aa9647e2SJavier Setoain 
1301aa9647e2SJavier Setoain /// Lower vector.scalable.extract ops to LLVM vector.extract
1302aa9647e2SJavier Setoain struct VectorScalableExtractOpLowering
1303aa9647e2SJavier Setoain     : public ConvertOpToLLVMPattern<vector::ScalableExtractOp> {
1304aa9647e2SJavier Setoain   using ConvertOpToLLVMPattern<
1305aa9647e2SJavier Setoain       vector::ScalableExtractOp>::ConvertOpToLLVMPattern;
1306aa9647e2SJavier Setoain 
1307aa9647e2SJavier Setoain   LogicalResult
1308aa9647e2SJavier Setoain   matchAndRewrite(vector::ScalableExtractOp extOp, OpAdaptor adaptor,
1309aa9647e2SJavier Setoain                   ConversionPatternRewriter &rewriter) const override {
1310aa9647e2SJavier Setoain     rewriter.replaceOpWithNewOp<LLVM::vector_extract>(
1311aa9647e2SJavier Setoain         extOp, typeConverter->convertType(extOp.getResultVectorType()),
1312aa9647e2SJavier Setoain         adaptor.getSource(), adaptor.getPos());
1313aa9647e2SJavier Setoain     return success();
1314aa9647e2SJavier Setoain   }
1315aa9647e2SJavier Setoain };
1316aa9647e2SJavier Setoain 
1317681f929fSNicolas Vasilache /// Rank reducing rewrite for n-D FMA into (n-1)-D FMA where n > 1.
1318681f929fSNicolas Vasilache ///
1319681f929fSNicolas Vasilache /// Example:
1320681f929fSNicolas Vasilache /// ```
1321681f929fSNicolas Vasilache ///   %d = vector.fma %a, %b, %c : vector<2x4xf32>
1322681f929fSNicolas Vasilache /// ```
1323681f929fSNicolas Vasilache /// is rewritten into:
1324681f929fSNicolas Vasilache /// ```
1325681f929fSNicolas Vasilache ///  %r = splat %f0: vector<2x4xf32>
1326681f929fSNicolas Vasilache ///  %va = vector.extractvalue %a[0] : vector<2x4xf32>
1327681f929fSNicolas Vasilache ///  %vb = vector.extractvalue %b[0] : vector<2x4xf32>
1328681f929fSNicolas Vasilache ///  %vc = vector.extractvalue %c[0] : vector<2x4xf32>
1329681f929fSNicolas Vasilache ///  %vd = vector.fma %va, %vb, %vc : vector<4xf32>
1330681f929fSNicolas Vasilache ///  %r2 = vector.insertvalue %vd, %r[0] : vector<4xf32> into vector<2x4xf32>
1331681f929fSNicolas Vasilache ///  %va2 = vector.extractvalue %a2[1] : vector<2x4xf32>
1332681f929fSNicolas Vasilache ///  %vb2 = vector.extractvalue %b2[1] : vector<2x4xf32>
1333681f929fSNicolas Vasilache ///  %vc2 = vector.extractvalue %c2[1] : vector<2x4xf32>
1334681f929fSNicolas Vasilache ///  %vd2 = vector.fma %va2, %vb2, %vc2 : vector<4xf32>
1335681f929fSNicolas Vasilache ///  %r3 = vector.insertvalue %vd2, %r2[1] : vector<4xf32> into vector<2x4xf32>
1336681f929fSNicolas Vasilache ///  // %r3 holds the final value.
1337681f929fSNicolas Vasilache /// ```
1338681f929fSNicolas Vasilache class VectorFMAOpNDRewritePattern : public OpRewritePattern<FMAOp> {
1339681f929fSNicolas Vasilache public:
1340681f929fSNicolas Vasilache   using OpRewritePattern<FMAOp>::OpRewritePattern;
1341681f929fSNicolas Vasilache 
1342ee80ffbfSNicolas Vasilache   void initialize() {
1343ee80ffbfSNicolas Vasilache     // This pattern recursively unpacks one dimension at a time. The recursion
1344ee80ffbfSNicolas Vasilache     // bounded as the rank is strictly decreasing.
1345ee80ffbfSNicolas Vasilache     setHasBoundedRewriteRecursion();
1346ee80ffbfSNicolas Vasilache   }
1347ee80ffbfSNicolas Vasilache 
13483145427dSRiver Riddle   LogicalResult matchAndRewrite(FMAOp op,
1349681f929fSNicolas Vasilache                                 PatternRewriter &rewriter) const override {
1350681f929fSNicolas Vasilache     auto vType = op.getVectorType();
1351681f929fSNicolas Vasilache     if (vType.getRank() < 2)
13523145427dSRiver Riddle       return failure();
1353681f929fSNicolas Vasilache 
1354681f929fSNicolas Vasilache     auto loc = op.getLoc();
1355681f929fSNicolas Vasilache     auto elemType = vType.getElementType();
1356a54f4eaeSMogball     Value zero = rewriter.create<arith::ConstantOp>(
1357a54f4eaeSMogball         loc, elemType, rewriter.getZeroAttr(elemType));
13586a8ba318SRiver Riddle     Value desc = rewriter.create<vector::SplatOp>(loc, vType, zero);
1359681f929fSNicolas Vasilache     for (int64_t i = 0, e = vType.getShape().front(); i != e; ++i) {
13607c38fd60SJacques Pienaar       Value extrLHS = rewriter.create<ExtractOp>(loc, op.getLhs(), i);
13617c38fd60SJacques Pienaar       Value extrRHS = rewriter.create<ExtractOp>(loc, op.getRhs(), i);
13627c38fd60SJacques Pienaar       Value extrACC = rewriter.create<ExtractOp>(loc, op.getAcc(), i);
1363681f929fSNicolas Vasilache       Value fma = rewriter.create<FMAOp>(loc, extrLHS, extrRHS, extrACC);
1364681f929fSNicolas Vasilache       desc = rewriter.create<InsertOp>(loc, fma, desc, i);
1365681f929fSNicolas Vasilache     }
1366681f929fSNicolas Vasilache     rewriter.replaceOp(op, desc);
13673145427dSRiver Riddle     return success();
1368681f929fSNicolas Vasilache   }
1369681f929fSNicolas Vasilache };
1370681f929fSNicolas Vasilache 
137130e6033bSNicolas Vasilache /// Returns the strides if the memory underlying `memRefType` has a contiguous
137230e6033bSNicolas Vasilache /// static layout.
1373a9a1b45bSKazu Hirata static std::optional<SmallVector<int64_t, 4>>
137430e6033bSNicolas Vasilache computeContiguousStrides(MemRefType memRefType) {
13752bf491c7SBenjamin Kramer   int64_t offset;
137630e6033bSNicolas Vasilache   SmallVector<int64_t, 4> strides;
13776aaa8f25SMatthias Springer   if (failed(memRefType.getStridesAndOffset(strides, offset)))
13781a36588eSKazu Hirata     return std::nullopt;
137930e6033bSNicolas Vasilache   if (!strides.empty() && strides.back() != 1)
13801a36588eSKazu Hirata     return std::nullopt;
138130e6033bSNicolas Vasilache   // If no layout or identity layout, this is contiguous by definition.
1382e41ebbecSVladislav Vinogradov   if (memRefType.getLayout().isIdentity())
138330e6033bSNicolas Vasilache     return strides;
138430e6033bSNicolas Vasilache 
138530e6033bSNicolas Vasilache   // Otherwise, we must determine contiguity form shapes. This can only ever
138630e6033bSNicolas Vasilache   // work in static cases because MemRefType is underspecified to represent
138730e6033bSNicolas Vasilache   // contiguous dynamic shapes in other ways than with just empty/identity
138830e6033bSNicolas Vasilache   // layout.
13892bf491c7SBenjamin Kramer   auto sizes = memRefType.getShape();
13905017b0f8SMatthias Springer   for (int index = 0, e = strides.size() - 1; index < e; ++index) {
139130e6033bSNicolas Vasilache     if (ShapedType::isDynamic(sizes[index + 1]) ||
1392399638f9SAliia Khasanova         ShapedType::isDynamic(strides[index]) ||
1393399638f9SAliia Khasanova         ShapedType::isDynamic(strides[index + 1]))
13941a36588eSKazu Hirata       return std::nullopt;
139530e6033bSNicolas Vasilache     if (strides[index] != strides[index + 1] * sizes[index + 1])
13961a36588eSKazu Hirata       return std::nullopt;
13972bf491c7SBenjamin Kramer   }
139830e6033bSNicolas Vasilache   return strides;
13992bf491c7SBenjamin Kramer }
14002bf491c7SBenjamin Kramer 
1401563879b6SRahul Joshi class VectorTypeCastOpConversion
1402563879b6SRahul Joshi     : public ConvertOpToLLVMPattern<vector::TypeCastOp> {
14035c0c51a9SNicolas Vasilache public:
1404563879b6SRahul Joshi   using ConvertOpToLLVMPattern<vector::TypeCastOp>::ConvertOpToLLVMPattern;
14055c0c51a9SNicolas Vasilache 
14063145427dSRiver Riddle   LogicalResult
1407ef976337SRiver Riddle   matchAndRewrite(vector::TypeCastOp castOp, OpAdaptor adaptor,
14085c0c51a9SNicolas Vasilache                   ConversionPatternRewriter &rewriter) const override {
1409563879b6SRahul Joshi     auto loc = castOp->getLoc();
14105c0c51a9SNicolas Vasilache     MemRefType sourceMemRefType =
14115550c821STres Popp         cast<MemRefType>(castOp.getOperand().getType());
14129eb3e564SChris Lattner     MemRefType targetMemRefType = castOp.getType();
14135c0c51a9SNicolas Vasilache 
14145c0c51a9SNicolas Vasilache     // Only static shape casts supported atm.
14155c0c51a9SNicolas Vasilache     if (!sourceMemRefType.hasStaticShape() ||
14165c0c51a9SNicolas Vasilache         !targetMemRefType.hasStaticShape())
14173145427dSRiver Riddle       return failure();
14185c0c51a9SNicolas Vasilache 
14195c0c51a9SNicolas Vasilache     auto llvmSourceDescriptorTy =
14205550c821STres Popp         dyn_cast<LLVM::LLVMStructType>(adaptor.getOperands()[0].getType());
14218de43b92SAlex Zinenko     if (!llvmSourceDescriptorTy)
14223145427dSRiver Riddle       return failure();
1423ef976337SRiver Riddle     MemRefDescriptor sourceMemRef(adaptor.getOperands()[0]);
14245c0c51a9SNicolas Vasilache 
14255550c821STres Popp     auto llvmTargetDescriptorTy = dyn_cast_or_null<LLVM::LLVMStructType>(
14265550c821STres Popp         typeConverter->convertType(targetMemRefType));
14278de43b92SAlex Zinenko     if (!llvmTargetDescriptorTy)
14283145427dSRiver Riddle       return failure();
14295c0c51a9SNicolas Vasilache 
143030e6033bSNicolas Vasilache     // Only contiguous source buffers supported atm.
143130e6033bSNicolas Vasilache     auto sourceStrides = computeContiguousStrides(sourceMemRefType);
143230e6033bSNicolas Vasilache     if (!sourceStrides)
143330e6033bSNicolas Vasilache       return failure();
143430e6033bSNicolas Vasilache     auto targetStrides = computeContiguousStrides(targetMemRefType);
143530e6033bSNicolas Vasilache     if (!targetStrides)
143630e6033bSNicolas Vasilache       return failure();
143730e6033bSNicolas Vasilache     // Only support static strides for now, regardless of contiguity.
1438399638f9SAliia Khasanova     if (llvm::any_of(*targetStrides, ShapedType::isDynamic))
14393145427dSRiver Riddle       return failure();
14405c0c51a9SNicolas Vasilache 
14412230bf99SAlex Zinenko     auto int64Ty = IntegerType::get(rewriter.getContext(), 64);
14425c0c51a9SNicolas Vasilache 
14435c0c51a9SNicolas Vasilache     // Create descriptor.
14445c0c51a9SNicolas Vasilache     auto desc = MemRefDescriptor::undef(rewriter, loc, llvmTargetDescriptorTy);
14455c0c51a9SNicolas Vasilache     // Set allocated ptr.
1446e62a6956SRiver Riddle     Value allocated = sourceMemRef.allocatedPtr(rewriter, loc);
14475c0c51a9SNicolas Vasilache     desc.setAllocatedPtr(rewriter, loc, allocated);
14484a2d4588SMarkus Böck 
14495c0c51a9SNicolas Vasilache     // Set aligned ptr.
1450e62a6956SRiver Riddle     Value ptr = sourceMemRef.alignedPtr(rewriter, loc);
14515c0c51a9SNicolas Vasilache     desc.setAlignedPtr(rewriter, loc, ptr);
14525c0c51a9SNicolas Vasilache     // Fill offset 0.
14535c0c51a9SNicolas Vasilache     auto attr = rewriter.getIntegerAttr(rewriter.getIndexType(), 0);
14545c0c51a9SNicolas Vasilache     auto zero = rewriter.create<LLVM::ConstantOp>(loc, int64Ty, attr);
14555c0c51a9SNicolas Vasilache     desc.setOffset(rewriter, loc, zero);
14565c0c51a9SNicolas Vasilache 
14575c0c51a9SNicolas Vasilache     // Fill size and stride descriptors in memref.
1458e4853be2SMehdi Amini     for (const auto &indexedSize :
1459e4853be2SMehdi Amini          llvm::enumerate(targetMemRefType.getShape())) {
14605c0c51a9SNicolas Vasilache       int64_t index = indexedSize.index();
14615c0c51a9SNicolas Vasilache       auto sizeAttr =
14625c0c51a9SNicolas Vasilache           rewriter.getIntegerAttr(rewriter.getIndexType(), indexedSize.value());
14635c0c51a9SNicolas Vasilache       auto size = rewriter.create<LLVM::ConstantOp>(loc, int64Ty, sizeAttr);
14645c0c51a9SNicolas Vasilache       desc.setSize(rewriter, loc, index, size);
146530e6033bSNicolas Vasilache       auto strideAttr = rewriter.getIntegerAttr(rewriter.getIndexType(),
146630e6033bSNicolas Vasilache                                                 (*targetStrides)[index]);
14675c0c51a9SNicolas Vasilache       auto stride = rewriter.create<LLVM::ConstantOp>(loc, int64Ty, strideAttr);
14685c0c51a9SNicolas Vasilache       desc.setStride(rewriter, loc, index, stride);
14695c0c51a9SNicolas Vasilache     }
14705c0c51a9SNicolas Vasilache 
1471563879b6SRahul Joshi     rewriter.replaceOp(castOp, {desc});
14723145427dSRiver Riddle     return success();
14735c0c51a9SNicolas Vasilache   }
14745c0c51a9SNicolas Vasilache };
14755c0c51a9SNicolas Vasilache 
1476a75a46dbSJavier Setoain /// Conversion pattern for a `vector.create_mask` (1-D scalable vectors only).
1477a75a46dbSJavier Setoain /// Non-scalable versions of this operation are handled in Vector Transforms.
14780693b9e9SMatthias Springer class VectorCreateMaskOpConversion
14790693b9e9SMatthias Springer     : public OpConversionPattern<vector::CreateMaskOp> {
1480a75a46dbSJavier Setoain public:
14810693b9e9SMatthias Springer   explicit VectorCreateMaskOpConversion(MLIRContext *context,
1482a75a46dbSJavier Setoain                                         bool enableIndexOpt)
14830693b9e9SMatthias Springer       : OpConversionPattern<vector::CreateMaskOp>(context),
14847bc8ad51SJavier Setoain         force32BitVectorIndices(enableIndexOpt) {}
1485a75a46dbSJavier Setoain 
14860693b9e9SMatthias Springer   LogicalResult
14870693b9e9SMatthias Springer   matchAndRewrite(vector::CreateMaskOp op, OpAdaptor adaptor,
14880693b9e9SMatthias Springer                   ConversionPatternRewriter &rewriter) const override {
1489a75a46dbSJavier Setoain     auto dstType = op.getType();
14905550c821STres Popp     if (dstType.getRank() != 1 || !cast<VectorType>(dstType).isScalable())
1491a75a46dbSJavier Setoain       return failure();
1492a75a46dbSJavier Setoain     IntegerType idxType =
14937bc8ad51SJavier Setoain         force32BitVectorIndices ? rewriter.getI32Type() : rewriter.getI64Type();
1494a75a46dbSJavier Setoain     auto loc = op->getLoc();
1495a75a46dbSJavier Setoain     Value indices = rewriter.create<LLVM::StepVectorOp>(
1496a75a46dbSJavier Setoain         loc, LLVM::getVectorType(idxType, dstType.getShape()[0],
1497a75a46dbSJavier Setoain                                  /*isScalable=*/true));
1498a75a46dbSJavier Setoain     auto bound = getValueOrCreateCastToIndexLike(rewriter, loc, idxType,
14990693b9e9SMatthias Springer                                                  adaptor.getOperands()[0]);
1500a75a46dbSJavier Setoain     Value bounds = rewriter.create<SplatOp>(loc, indices.getType(), bound);
1501a75a46dbSJavier Setoain     Value comp = rewriter.create<arith::CmpIOp>(loc, arith::CmpIPredicate::slt,
1502a75a46dbSJavier Setoain                                                 indices, bounds);
1503a75a46dbSJavier Setoain     rewriter.replaceOp(op, comp);
1504a75a46dbSJavier Setoain     return success();
1505a75a46dbSJavier Setoain   }
1506a75a46dbSJavier Setoain 
1507a75a46dbSJavier Setoain private:
15087bc8ad51SJavier Setoain   const bool force32BitVectorIndices;
1509a75a46dbSJavier Setoain };
1510a75a46dbSJavier Setoain 
1511563879b6SRahul Joshi class VectorPrintOpConversion : public ConvertOpToLLVMPattern<vector::PrintOp> {
1512d9b500d3SAart Bik public:
1513563879b6SRahul Joshi   using ConvertOpToLLVMPattern<vector::PrintOp>::ConvertOpToLLVMPattern;
1514d9b500d3SAart Bik 
1515f36e909dSBenjamin Maxwell   // Lowering implementation that relies on a small runtime support library,
1516f36e909dSBenjamin Maxwell   // which only needs to provide a few printing methods (single value for all
1517f36e909dSBenjamin Maxwell   // data types, opening/closing bracket, comma, newline). The lowering splits
1518f36e909dSBenjamin Maxwell   // the vector into elementary printing operations. The advantage of this
1519f36e909dSBenjamin Maxwell   // approach is that the library can remain unaware of all low-level
1520f36e909dSBenjamin Maxwell   // implementation details of vectors while still supporting output of any
1521f36e909dSBenjamin Maxwell   // shaped and dimensioned vector.
1522f36e909dSBenjamin Maxwell   //
1523f36e909dSBenjamin Maxwell   // Note: This lowering only handles scalars, n-D vectors are broken into
1524f36e909dSBenjamin Maxwell   // printing scalars in loops in VectorToSCF.
1525d9b500d3SAart Bik   //
15269db53a18SRiver Riddle   // TODO: rely solely on libc in future? something else?
1527d9b500d3SAart Bik   //
15283145427dSRiver Riddle   LogicalResult
1529ef976337SRiver Riddle   matchAndRewrite(vector::PrintOp printOp, OpAdaptor adaptor,
1530d9b500d3SAart Bik                   ConversionPatternRewriter &rewriter) const override {
1531f36e909dSBenjamin Maxwell     auto parent = printOp->getParentOfType<ModuleOp>();
1532f36e909dSBenjamin Maxwell     if (!parent)
1533f36e909dSBenjamin Maxwell       return failure();
1534490dae26SBenjamin Maxwell 
1535f36e909dSBenjamin Maxwell     auto loc = printOp->getLoc();
1536f36e909dSBenjamin Maxwell 
1537f36e909dSBenjamin Maxwell     if (auto value = adaptor.getSource()) {
1538f36e909dSBenjamin Maxwell       Type printType = printOp.getPrintType();
1539f36e909dSBenjamin Maxwell       if (isa<VectorType>(printType)) {
1540f36e909dSBenjamin Maxwell         // Vectors should be broken into elementary print ops in VectorToSCF.
1541f36e909dSBenjamin Maxwell         return failure();
1542f36e909dSBenjamin Maxwell       }
1543f36e909dSBenjamin Maxwell       if (failed(emitScalarPrint(rewriter, parent, loc, printType, value)))
1544f36e909dSBenjamin Maxwell         return failure();
1545f36e909dSBenjamin Maxwell     }
1546f36e909dSBenjamin Maxwell 
1547f36e909dSBenjamin Maxwell     auto punct = printOp.getPunctuation();
15483be3883eSBenjamin Maxwell     if (auto stringLiteral = printOp.getStringLiteral()) {
1549*e84f6b6aSLuohao Wang       auto createResult =
15503be3883eSBenjamin Maxwell           LLVM::createPrintStrCall(rewriter, loc, parent, "vector_print_str",
1551c1b8c6cfSAart Bik                                    *stringLiteral, *getTypeConverter(),
1552c1b8c6cfSAart Bik                                    /*addNewline=*/false);
1553*e84f6b6aSLuohao Wang       if (createResult.failed())
1554*e84f6b6aSLuohao Wang         return failure();
1555*e84f6b6aSLuohao Wang 
15563be3883eSBenjamin Maxwell     } else if (punct != PrintPunctuation::NoPunctuation) {
1557*e84f6b6aSLuohao Wang       FailureOr<LLVM::LLVMFuncOp> op = [&]() {
1558f36e909dSBenjamin Maxwell         switch (punct) {
1559f36e909dSBenjamin Maxwell         case PrintPunctuation::Close:
1560f36e909dSBenjamin Maxwell           return LLVM::lookupOrCreatePrintCloseFn(parent);
1561f36e909dSBenjamin Maxwell         case PrintPunctuation::Open:
1562f36e909dSBenjamin Maxwell           return LLVM::lookupOrCreatePrintOpenFn(parent);
1563f36e909dSBenjamin Maxwell         case PrintPunctuation::Comma:
1564f36e909dSBenjamin Maxwell           return LLVM::lookupOrCreatePrintCommaFn(parent);
1565f36e909dSBenjamin Maxwell         case PrintPunctuation::NewLine:
1566f36e909dSBenjamin Maxwell           return LLVM::lookupOrCreatePrintNewlineFn(parent);
1567f36e909dSBenjamin Maxwell         default:
1568f36e909dSBenjamin Maxwell           llvm_unreachable("unexpected punctuation");
1569f36e909dSBenjamin Maxwell         }
1570*e84f6b6aSLuohao Wang       }();
1571*e84f6b6aSLuohao Wang       if (failed(op))
1572*e84f6b6aSLuohao Wang         return failure();
1573*e84f6b6aSLuohao Wang       emitCall(rewriter, printOp->getLoc(), op.value());
1574f36e909dSBenjamin Maxwell     }
1575f36e909dSBenjamin Maxwell 
1576f36e909dSBenjamin Maxwell     rewriter.eraseOp(printOp);
1577f36e909dSBenjamin Maxwell     return success();
1578f36e909dSBenjamin Maxwell   }
1579f36e909dSBenjamin Maxwell 
1580f36e909dSBenjamin Maxwell private:
1581f36e909dSBenjamin Maxwell   enum class PrintConversion {
1582f36e909dSBenjamin Maxwell     // clang-format off
1583f36e909dSBenjamin Maxwell     None,
1584f36e909dSBenjamin Maxwell     ZeroExt64,
1585f36e909dSBenjamin Maxwell     SignExt64,
1586f36e909dSBenjamin Maxwell     Bitcast16
1587f36e909dSBenjamin Maxwell     // clang-format on
1588f36e909dSBenjamin Maxwell   };
1589f36e909dSBenjamin Maxwell 
1590f36e909dSBenjamin Maxwell   LogicalResult emitScalarPrint(ConversionPatternRewriter &rewriter,
1591f36e909dSBenjamin Maxwell                                 ModuleOp parent, Location loc, Type printType,
1592f36e909dSBenjamin Maxwell                                 Value value) const {
1593dcec2ca5SChristian Sigg     if (typeConverter->convertType(printType) == nullptr)
15943145427dSRiver Riddle       return failure();
1595d9b500d3SAart Bik 
1596b8880f5fSAart Bik     // Make sure element type has runtime support.
1597b8880f5fSAart Bik     PrintConversion conversion = PrintConversion::None;
1598*e84f6b6aSLuohao Wang     FailureOr<Operation *> printer;
1599f36e909dSBenjamin Maxwell     if (printType.isF32()) {
1600657f60a0SAart Bik       printer = LLVM::lookupOrCreatePrintF32Fn(parent);
1601f36e909dSBenjamin Maxwell     } else if (printType.isF64()) {
1602657f60a0SAart Bik       printer = LLVM::lookupOrCreatePrintF64Fn(parent);
1603f36e909dSBenjamin Maxwell     } else if (printType.isF16()) {
1604657f60a0SAart Bik       conversion = PrintConversion::Bitcast16; // bits!
1605657f60a0SAart Bik       printer = LLVM::lookupOrCreatePrintF16Fn(parent);
1606f36e909dSBenjamin Maxwell     } else if (printType.isBF16()) {
1607657f60a0SAart Bik       conversion = PrintConversion::Bitcast16; // bits!
1608657f60a0SAart Bik       printer = LLVM::lookupOrCreatePrintBF16Fn(parent);
1609f36e909dSBenjamin Maxwell     } else if (printType.isIndex()) {
1610657f60a0SAart Bik       printer = LLVM::lookupOrCreatePrintU64Fn(parent);
1611f36e909dSBenjamin Maxwell     } else if (auto intTy = dyn_cast<IntegerType>(printType)) {
1612b8880f5fSAart Bik       // Integers need a zero or sign extension on the operand
1613b8880f5fSAart Bik       // (depending on the source type) as well as a signed or
1614b8880f5fSAart Bik       // unsigned print method. Up to 64-bit is supported.
1615b8880f5fSAart Bik       unsigned width = intTy.getWidth();
1616b8880f5fSAart Bik       if (intTy.isUnsigned()) {
161754759cefSAart Bik         if (width <= 64) {
1618b8880f5fSAart Bik           if (width < 64)
1619b8880f5fSAart Bik             conversion = PrintConversion::ZeroExt64;
1620657f60a0SAart Bik           printer = LLVM::lookupOrCreatePrintU64Fn(parent);
1621b8880f5fSAart Bik         } else {
16223145427dSRiver Riddle           return failure();
1623b8880f5fSAart Bik         }
1624b8880f5fSAart Bik       } else {
1625b8880f5fSAart Bik         assert(intTy.isSignless() || intTy.isSigned());
162654759cefSAart Bik         if (width <= 64) {
1627b8880f5fSAart Bik           // Note that we *always* zero extend booleans (1-bit integers),
1628b8880f5fSAart Bik           // so that true/false is printed as 1/0 rather than -1/0.
1629b8880f5fSAart Bik           if (width == 1)
163054759cefSAart Bik             conversion = PrintConversion::ZeroExt64;
163154759cefSAart Bik           else if (width < 64)
1632b8880f5fSAart Bik             conversion = PrintConversion::SignExt64;
1633657f60a0SAart Bik           printer = LLVM::lookupOrCreatePrintI64Fn(parent);
1634b8880f5fSAart Bik         } else {
1635b8880f5fSAart Bik           return failure();
1636b8880f5fSAart Bik         }
1637b8880f5fSAart Bik       }
1638b8880f5fSAart Bik     } else {
1639b8880f5fSAart Bik       return failure();
1640b8880f5fSAart Bik     }
1641*e84f6b6aSLuohao Wang     if (failed(printer))
1642*e84f6b6aSLuohao Wang       return failure();
1643d9b500d3SAart Bik 
1644b8880f5fSAart Bik     switch (conversion) {
1645b8880f5fSAart Bik     case PrintConversion::ZeroExt64:
1646a54f4eaeSMogball       value = rewriter.create<arith::ExtUIOp>(
16473c69bc4dSRiver Riddle           loc, IntegerType::get(rewriter.getContext(), 64), value);
1648b8880f5fSAart Bik       break;
1649b8880f5fSAart Bik     case PrintConversion::SignExt64:
1650a54f4eaeSMogball       value = rewriter.create<arith::ExtSIOp>(
16513c69bc4dSRiver Riddle           loc, IntegerType::get(rewriter.getContext(), 64), value);
1652b8880f5fSAart Bik       break;
1653657f60a0SAart Bik     case PrintConversion::Bitcast16:
1654657f60a0SAart Bik       value = rewriter.create<LLVM::BitcastOp>(
1655657f60a0SAart Bik           loc, IntegerType::get(rewriter.getContext(), 16), value);
1656657f60a0SAart Bik       break;
1657b8880f5fSAart Bik     case PrintConversion::None:
1658b8880f5fSAart Bik       break;
1659c9eeeb38Saartbik     }
1660*e84f6b6aSLuohao Wang     emitCall(rewriter, loc, printer.value(), value);
1661f36e909dSBenjamin Maxwell     return success();
1662d9b500d3SAart Bik   }
1663d9b500d3SAart Bik 
1664d9b500d3SAart Bik   // Helper to emit a call.
1665d9b500d3SAart Bik   static void emitCall(ConversionPatternRewriter &rewriter, Location loc,
1666d9b500d3SAart Bik                        Operation *ref, ValueRange params = ValueRange()) {
1667faf1c224SChris Lattner     rewriter.create<LLVM::CallOp>(loc, TypeRange(), SymbolRefAttr::get(ref),
1668faf1c224SChris Lattner                                   params);
1669d9b500d3SAart Bik   }
1670d9b500d3SAart Bik };
1671d9b500d3SAart Bik 
16726a8ba318SRiver Riddle /// The Splat operation is lowered to an insertelement + a shufflevector
16736a8ba318SRiver Riddle /// operation. Splat to only 0-d and 1-d vector result types are lowered.
16746a8ba318SRiver Riddle struct VectorSplatOpLowering : public ConvertOpToLLVMPattern<vector::SplatOp> {
16756a8ba318SRiver Riddle   using ConvertOpToLLVMPattern<vector::SplatOp>::ConvertOpToLLVMPattern;
16766a8ba318SRiver Riddle 
16776a8ba318SRiver Riddle   LogicalResult
16786a8ba318SRiver Riddle   matchAndRewrite(vector::SplatOp splatOp, OpAdaptor adaptor,
16796a8ba318SRiver Riddle                   ConversionPatternRewriter &rewriter) const override {
16805550c821STres Popp     VectorType resultType = cast<VectorType>(splatOp.getType());
16816a8ba318SRiver Riddle     if (resultType.getRank() > 1)
16826a8ba318SRiver Riddle       return failure();
16836a8ba318SRiver Riddle 
16846a8ba318SRiver Riddle     // First insert it into an undef vector so we can shuffle it.
16856a8ba318SRiver Riddle     auto vectorType = typeConverter->convertType(splatOp.getType());
16866a8ba318SRiver Riddle     Value undef = rewriter.create<LLVM::UndefOp>(splatOp.getLoc(), vectorType);
16876a8ba318SRiver Riddle     auto zero = rewriter.create<LLVM::ConstantOp>(
16886a8ba318SRiver Riddle         splatOp.getLoc(),
16896a8ba318SRiver Riddle         typeConverter->convertType(rewriter.getIntegerType(32)),
16906a8ba318SRiver Riddle         rewriter.getZeroAttr(rewriter.getIntegerType(32)));
16916a8ba318SRiver Riddle 
16926a8ba318SRiver Riddle     // For 0-d vector, we simply do `insertelement`.
16936a8ba318SRiver Riddle     if (resultType.getRank() == 0) {
16946a8ba318SRiver Riddle       rewriter.replaceOpWithNewOp<LLVM::InsertElementOp>(
16957c38fd60SJacques Pienaar           splatOp, vectorType, undef, adaptor.getInput(), zero);
16966a8ba318SRiver Riddle       return success();
16976a8ba318SRiver Riddle     }
16986a8ba318SRiver Riddle 
16996a8ba318SRiver Riddle     // For 1-d vector, we additionally do a `vectorshuffle`.
17006a8ba318SRiver Riddle     auto v = rewriter.create<LLVM::InsertElementOp>(
17017c38fd60SJacques Pienaar         splatOp.getLoc(), vectorType, undef, adaptor.getInput(), zero);
17026a8ba318SRiver Riddle 
17035550c821STres Popp     int64_t width = cast<VectorType>(splatOp.getType()).getDimSize(0);
1704b2ccfb4dSJeff Niu     SmallVector<int32_t> zeroValues(width, 0);
17056a8ba318SRiver Riddle 
17066a8ba318SRiver Riddle     // Shuffle the value across the desired number of elements.
17076a8ba318SRiver Riddle     rewriter.replaceOpWithNewOp<LLVM::ShuffleVectorOp>(splatOp, v, undef,
1708b2ccfb4dSJeff Niu                                                        zeroValues);
17096a8ba318SRiver Riddle     return success();
17106a8ba318SRiver Riddle   }
17116a8ba318SRiver Riddle };
17126a8ba318SRiver Riddle 
17136a8ba318SRiver Riddle /// The Splat operation is lowered to an insertelement + a shufflevector
17146a8ba318SRiver Riddle /// operation. Splat to only 2+-d vector result types are lowered by the
17156a8ba318SRiver Riddle /// SplatNdOpLowering, the 1-d case is handled by SplatOpLowering.
17166a8ba318SRiver Riddle struct VectorSplatNdOpLowering : public ConvertOpToLLVMPattern<SplatOp> {
17176a8ba318SRiver Riddle   using ConvertOpToLLVMPattern<SplatOp>::ConvertOpToLLVMPattern;
17186a8ba318SRiver Riddle 
17196a8ba318SRiver Riddle   LogicalResult
17206a8ba318SRiver Riddle   matchAndRewrite(SplatOp splatOp, OpAdaptor adaptor,
17216a8ba318SRiver Riddle                   ConversionPatternRewriter &rewriter) const override {
17226a8ba318SRiver Riddle     VectorType resultType = splatOp.getType();
17236a8ba318SRiver Riddle     if (resultType.getRank() <= 1)
17246a8ba318SRiver Riddle       return failure();
17256a8ba318SRiver Riddle 
17266a8ba318SRiver Riddle     // First insert it into an undef vector so we can shuffle it.
17276a8ba318SRiver Riddle     auto loc = splatOp.getLoc();
17286a8ba318SRiver Riddle     auto vectorTypeInfo =
17296a8ba318SRiver Riddle         LLVM::detail::extractNDVectorTypeInfo(resultType, *getTypeConverter());
17306a8ba318SRiver Riddle     auto llvmNDVectorTy = vectorTypeInfo.llvmNDVectorTy;
17316a8ba318SRiver Riddle     auto llvm1DVectorTy = vectorTypeInfo.llvm1DVectorTy;
17326a8ba318SRiver Riddle     if (!llvmNDVectorTy || !llvm1DVectorTy)
17336a8ba318SRiver Riddle       return failure();
17346a8ba318SRiver Riddle 
17356a8ba318SRiver Riddle     // Construct returned value.
17366a8ba318SRiver Riddle     Value desc = rewriter.create<LLVM::UndefOp>(loc, llvmNDVectorTy);
17376a8ba318SRiver Riddle 
17386a8ba318SRiver Riddle     // Construct a 1-D vector with the splatted value that we insert in all the
17396a8ba318SRiver Riddle     // places within the returned descriptor.
17406a8ba318SRiver Riddle     Value vdesc = rewriter.create<LLVM::UndefOp>(loc, llvm1DVectorTy);
17416a8ba318SRiver Riddle     auto zero = rewriter.create<LLVM::ConstantOp>(
17426a8ba318SRiver Riddle         loc, typeConverter->convertType(rewriter.getIntegerType(32)),
17436a8ba318SRiver Riddle         rewriter.getZeroAttr(rewriter.getIntegerType(32)));
17446a8ba318SRiver Riddle     Value v = rewriter.create<LLVM::InsertElementOp>(loc, llvm1DVectorTy, vdesc,
17457c38fd60SJacques Pienaar                                                      adaptor.getInput(), zero);
17466a8ba318SRiver Riddle 
17476a8ba318SRiver Riddle     // Shuffle the value across the desired number of elements.
17486a8ba318SRiver Riddle     int64_t width = resultType.getDimSize(resultType.getRank() - 1);
1749b2ccfb4dSJeff Niu     SmallVector<int32_t> zeroValues(width, 0);
1750b2ccfb4dSJeff Niu     v = rewriter.create<LLVM::ShuffleVectorOp>(loc, v, v, zeroValues);
17516a8ba318SRiver Riddle 
17526a8ba318SRiver Riddle     // Iterate of linear index, convert to coords space and insert splatted 1-D
17536a8ba318SRiver Riddle     // vector in each position.
17545c5af910SJeff Niu     nDVectorIterate(vectorTypeInfo, rewriter, [&](ArrayRef<int64_t> position) {
17555c5af910SJeff Niu       desc = rewriter.create<LLVM::InsertValueOp>(loc, desc, v, position);
17566a8ba318SRiver Riddle     });
17576a8ba318SRiver Riddle     rewriter.replaceOp(splatOp, desc);
17586a8ba318SRiver Riddle     return success();
17596a8ba318SRiver Riddle   }
17606a8ba318SRiver Riddle };
17616a8ba318SRiver Riddle 
176279ce2c93SBenjamin Maxwell /// Conversion pattern for a `vector.interleave`.
176379ce2c93SBenjamin Maxwell /// This supports fixed-sized vectors and scalable vectors.
176479ce2c93SBenjamin Maxwell struct VectorInterleaveOpLowering
176579ce2c93SBenjamin Maxwell     : public ConvertOpToLLVMPattern<vector::InterleaveOp> {
176679ce2c93SBenjamin Maxwell   using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern;
176779ce2c93SBenjamin Maxwell 
176879ce2c93SBenjamin Maxwell   LogicalResult
176979ce2c93SBenjamin Maxwell   matchAndRewrite(vector::InterleaveOp interleaveOp, OpAdaptor adaptor,
177079ce2c93SBenjamin Maxwell                   ConversionPatternRewriter &rewriter) const override {
177179ce2c93SBenjamin Maxwell     VectorType resultType = interleaveOp.getResultVectorType();
177279ce2c93SBenjamin Maxwell     // n-D interleaves should have been lowered already.
177379ce2c93SBenjamin Maxwell     if (resultType.getRank() != 1)
177479ce2c93SBenjamin Maxwell       return rewriter.notifyMatchFailure(interleaveOp,
177579ce2c93SBenjamin Maxwell                                          "InterleaveOp not rank 1");
177679ce2c93SBenjamin Maxwell     // If the result is rank 1, then this directly maps to LLVM.
177779ce2c93SBenjamin Maxwell     if (resultType.isScalable()) {
1778bfc03171SMaciej Gabka       rewriter.replaceOpWithNewOp<LLVM::vector_interleave2>(
177979ce2c93SBenjamin Maxwell           interleaveOp, typeConverter->convertType(resultType),
178079ce2c93SBenjamin Maxwell           adaptor.getLhs(), adaptor.getRhs());
178179ce2c93SBenjamin Maxwell       return success();
178279ce2c93SBenjamin Maxwell     }
178379ce2c93SBenjamin Maxwell     // Lower fixed-size interleaves to a shufflevector. While the
178479ce2c93SBenjamin Maxwell     // vector.interleave2 intrinsic supports fixed and scalable vectors, the
178579ce2c93SBenjamin Maxwell     // langref still recommends fixed-vectors use shufflevector, see:
178679ce2c93SBenjamin Maxwell     // https://llvm.org/docs/LangRef.html#id876.
178779ce2c93SBenjamin Maxwell     int64_t resultVectorSize = resultType.getNumElements();
178879ce2c93SBenjamin Maxwell     SmallVector<int32_t> interleaveShuffleMask;
178979ce2c93SBenjamin Maxwell     interleaveShuffleMask.reserve(resultVectorSize);
179079ce2c93SBenjamin Maxwell     for (int i = 0, end = resultVectorSize / 2; i < end; ++i) {
179179ce2c93SBenjamin Maxwell       interleaveShuffleMask.push_back(i);
179279ce2c93SBenjamin Maxwell       interleaveShuffleMask.push_back((resultVectorSize / 2) + i);
179379ce2c93SBenjamin Maxwell     }
179479ce2c93SBenjamin Maxwell     rewriter.replaceOpWithNewOp<LLVM::ShuffleVectorOp>(
179579ce2c93SBenjamin Maxwell         interleaveOp, adaptor.getLhs(), adaptor.getRhs(),
179679ce2c93SBenjamin Maxwell         interleaveShuffleMask);
179779ce2c93SBenjamin Maxwell     return success();
179879ce2c93SBenjamin Maxwell   }
179979ce2c93SBenjamin Maxwell };
180079ce2c93SBenjamin Maxwell 
1801bc946f52SMubashar Ahmad /// Conversion pattern for a `vector.deinterleave`.
1802bc946f52SMubashar Ahmad /// This supports fixed-sized vectors and scalable vectors.
1803bc946f52SMubashar Ahmad struct VectorDeinterleaveOpLowering
1804bc946f52SMubashar Ahmad     : public ConvertOpToLLVMPattern<vector::DeinterleaveOp> {
1805bc946f52SMubashar Ahmad   using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern;
1806bc946f52SMubashar Ahmad 
1807bc946f52SMubashar Ahmad   LogicalResult
1808bc946f52SMubashar Ahmad   matchAndRewrite(vector::DeinterleaveOp deinterleaveOp, OpAdaptor adaptor,
1809bc946f52SMubashar Ahmad                   ConversionPatternRewriter &rewriter) const override {
1810bc946f52SMubashar Ahmad     VectorType resultType = deinterleaveOp.getResultVectorType();
1811bc946f52SMubashar Ahmad     VectorType sourceType = deinterleaveOp.getSourceVectorType();
1812bc946f52SMubashar Ahmad     auto loc = deinterleaveOp.getLoc();
1813bc946f52SMubashar Ahmad 
1814bc946f52SMubashar Ahmad     // Note: n-D deinterleave operations should be lowered to the 1-D before
1815bc946f52SMubashar Ahmad     // converting to LLVM.
1816bc946f52SMubashar Ahmad     if (resultType.getRank() != 1)
1817bc946f52SMubashar Ahmad       return rewriter.notifyMatchFailure(deinterleaveOp,
1818bc946f52SMubashar Ahmad                                          "DeinterleaveOp not rank 1");
1819bc946f52SMubashar Ahmad 
1820bc946f52SMubashar Ahmad     if (resultType.isScalable()) {
1821bc946f52SMubashar Ahmad       auto llvmTypeConverter = this->getTypeConverter();
1822bc946f52SMubashar Ahmad       auto deinterleaveResults = deinterleaveOp.getResultTypes();
1823bc946f52SMubashar Ahmad       auto packedOpResults =
1824bc946f52SMubashar Ahmad           llvmTypeConverter->packOperationResults(deinterleaveResults);
1825bc946f52SMubashar Ahmad       auto intrinsic = rewriter.create<LLVM::vector_deinterleave2>(
1826bc946f52SMubashar Ahmad           loc, packedOpResults, adaptor.getSource());
1827bc946f52SMubashar Ahmad 
1828bc946f52SMubashar Ahmad       auto evenResult = rewriter.create<LLVM::ExtractValueOp>(
1829bc946f52SMubashar Ahmad           loc, intrinsic->getResult(0), 0);
1830bc946f52SMubashar Ahmad       auto oddResult = rewriter.create<LLVM::ExtractValueOp>(
1831bc946f52SMubashar Ahmad           loc, intrinsic->getResult(0), 1);
1832bc946f52SMubashar Ahmad 
1833bc946f52SMubashar Ahmad       rewriter.replaceOp(deinterleaveOp, ValueRange{evenResult, oddResult});
1834bc946f52SMubashar Ahmad       return success();
1835bc946f52SMubashar Ahmad     }
1836bc946f52SMubashar Ahmad     // Lower fixed-size deinterleave to two shufflevectors. While the
1837bc946f52SMubashar Ahmad     // vector.deinterleave2 intrinsic supports fixed and scalable vectors, the
1838bc946f52SMubashar Ahmad     // langref still recommends fixed-vectors use shufflevector, see:
1839bc946f52SMubashar Ahmad     // https://llvm.org/docs/LangRef.html#id889.
1840bc946f52SMubashar Ahmad     int64_t resultVectorSize = resultType.getNumElements();
1841bc946f52SMubashar Ahmad     SmallVector<int32_t> evenShuffleMask;
1842bc946f52SMubashar Ahmad     SmallVector<int32_t> oddShuffleMask;
1843bc946f52SMubashar Ahmad 
1844bc946f52SMubashar Ahmad     evenShuffleMask.reserve(resultVectorSize);
1845bc946f52SMubashar Ahmad     oddShuffleMask.reserve(resultVectorSize);
1846bc946f52SMubashar Ahmad 
1847bc946f52SMubashar Ahmad     for (int i = 0; i < sourceType.getNumElements(); ++i) {
1848bc946f52SMubashar Ahmad       if (i % 2 == 0)
1849bc946f52SMubashar Ahmad         evenShuffleMask.push_back(i);
1850bc946f52SMubashar Ahmad       else
1851bc946f52SMubashar Ahmad         oddShuffleMask.push_back(i);
1852bc946f52SMubashar Ahmad     }
1853bc946f52SMubashar Ahmad 
1854bc946f52SMubashar Ahmad     auto poison = rewriter.create<LLVM::PoisonOp>(loc, sourceType);
1855bc946f52SMubashar Ahmad     auto evenShuffle = rewriter.create<LLVM::ShuffleVectorOp>(
1856bc946f52SMubashar Ahmad         loc, adaptor.getSource(), poison, evenShuffleMask);
1857bc946f52SMubashar Ahmad     auto oddShuffle = rewriter.create<LLVM::ShuffleVectorOp>(
1858bc946f52SMubashar Ahmad         loc, adaptor.getSource(), poison, oddShuffleMask);
1859bc946f52SMubashar Ahmad 
1860bc946f52SMubashar Ahmad     rewriter.replaceOp(deinterleaveOp, ValueRange{evenShuffle, oddShuffle});
1861bc946f52SMubashar Ahmad     return success();
1862bc946f52SMubashar Ahmad   }
1863bc946f52SMubashar Ahmad };
1864bc946f52SMubashar Ahmad 
1865c6ff2446SMatthias Springer /// Conversion pattern for a `vector.from_elements`.
1866c6ff2446SMatthias Springer struct VectorFromElementsLowering
1867c6ff2446SMatthias Springer     : public ConvertOpToLLVMPattern<vector::FromElementsOp> {
1868c6ff2446SMatthias Springer   using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern;
1869c6ff2446SMatthias Springer 
1870c6ff2446SMatthias Springer   LogicalResult
1871c6ff2446SMatthias Springer   matchAndRewrite(vector::FromElementsOp fromElementsOp, OpAdaptor adaptor,
1872c6ff2446SMatthias Springer                   ConversionPatternRewriter &rewriter) const override {
1873c6ff2446SMatthias Springer     Location loc = fromElementsOp.getLoc();
1874c6ff2446SMatthias Springer     VectorType vectorType = fromElementsOp.getType();
1875c6ff2446SMatthias Springer     // TODO: Multi-dimensional vectors lower to !llvm.array<... x vector<>>.
1876c6ff2446SMatthias Springer     // Such ops should be handled in the same way as vector.insert.
1877c6ff2446SMatthias Springer     if (vectorType.getRank() > 1)
1878c6ff2446SMatthias Springer       return rewriter.notifyMatchFailure(fromElementsOp,
1879c6ff2446SMatthias Springer                                          "rank > 1 vectors are not supported");
1880c6ff2446SMatthias Springer     Type llvmType = typeConverter->convertType(vectorType);
1881c6ff2446SMatthias Springer     Value result = rewriter.create<LLVM::UndefOp>(loc, llvmType);
1882c6ff2446SMatthias Springer     for (auto [idx, val] : llvm::enumerate(adaptor.getElements()))
1883c6ff2446SMatthias Springer       result = rewriter.create<vector::InsertOp>(loc, val, result, idx);
1884c6ff2446SMatthias Springer     rewriter.replaceOp(fromElementsOp, result);
1885c6ff2446SMatthias Springer     return success();
1886c6ff2446SMatthias Springer   }
1887c6ff2446SMatthias Springer };
1888c6ff2446SMatthias Springer 
188967b302c5SCullen Rhodes /// Conversion pattern for vector.step.
1890a6e72f93SManupa Karunaratne struct VectorScalableStepOpLowering
1891a6e72f93SManupa Karunaratne     : public ConvertOpToLLVMPattern<vector::StepOp> {
189267b302c5SCullen Rhodes   using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern;
189367b302c5SCullen Rhodes 
189467b302c5SCullen Rhodes   LogicalResult
189567b302c5SCullen Rhodes   matchAndRewrite(vector::StepOp stepOp, OpAdaptor adaptor,
189667b302c5SCullen Rhodes                   ConversionPatternRewriter &rewriter) const override {
1897a6e72f93SManupa Karunaratne     auto resultType = cast<VectorType>(stepOp.getType());
1898a6e72f93SManupa Karunaratne     if (!resultType.isScalable()) {
1899a6e72f93SManupa Karunaratne       return failure();
1900a6e72f93SManupa Karunaratne     }
190167b302c5SCullen Rhodes     Type llvmType = typeConverter->convertType(stepOp.getType());
190267b302c5SCullen Rhodes     rewriter.replaceOpWithNewOp<LLVM::StepVectorOp>(stepOp, llvmType);
190367b302c5SCullen Rhodes     return success();
190467b302c5SCullen Rhodes   }
190567b302c5SCullen Rhodes };
190667b302c5SCullen Rhodes 
1907df186507SBenjamin Kramer } // namespace
1908df186507SBenjamin Kramer 
19090693b9e9SMatthias Springer void mlir::vector::populateVectorRankReducingFMAPattern(
19100693b9e9SMatthias Springer     RewritePatternSet &patterns) {
19110693b9e9SMatthias Springer   patterns.add<VectorFMAOpNDRewritePattern>(patterns.getContext());
19120693b9e9SMatthias Springer }
19130693b9e9SMatthias Springer 
19145c0c51a9SNicolas Vasilache /// Populate the given list with patterns that convert from Vector to LLVM.
19157bc8ad51SJavier Setoain void mlir::populateVectorToLLVMConversionPatterns(
1916206fad0eSMatthias Springer     const LLVMTypeConverter &converter, RewritePatternSet &patterns,
19177bc8ad51SJavier Setoain     bool reassociateFPReductions, bool force32BitVectorIndices) {
19180693b9e9SMatthias Springer   // This function populates only ConversionPatterns, not RewritePatterns.
191965678d93SNicolas Vasilache   MLIRContext *ctx = converter.getDialect()->getContext();
1920dc4e913bSChris Lattner   patterns.add<VectorReductionOpConversion>(converter, reassociateFPReductions);
19210693b9e9SMatthias Springer   patterns.add<VectorCreateMaskOpConversion>(ctx, force32BitVectorIndices);
19224db0bd28SCullen Rhodes   patterns.add<VectorBitCastOpConversion, VectorShuffleOpConversion,
1923dc4e913bSChris Lattner                VectorExtractElementOpConversion, VectorExtractOpConversion,
1924c339f9e1SDiego Caballero                VectorFMAOp1DConversion, VectorInsertElementOpConversion,
1925c339f9e1SDiego Caballero                VectorInsertOpConversion, VectorPrintOpConversion,
1926c339f9e1SDiego Caballero                VectorTypeCastOpConversion, VectorScaleOpConversion,
19274db0bd28SCullen Rhodes                VectorLoadStoreConversion<vector::LoadOp>,
19284db0bd28SCullen Rhodes                VectorLoadStoreConversion<vector::MaskedLoadOp>,
19294db0bd28SCullen Rhodes                VectorLoadStoreConversion<vector::StoreOp>,
19304db0bd28SCullen Rhodes                VectorLoadStoreConversion<vector::MaskedStoreOp>,
1931dc4e913bSChris Lattner                VectorGatherOpConversion, VectorScatterOpConversion,
19326a8ba318SRiver Riddle                VectorExpandLoadOpConversion, VectorCompressStoreOpConversion,
1933aa9647e2SJavier Setoain                VectorSplatOpLowering, VectorSplatNdOpLowering,
1934e9b82a5cSDiego Caballero                VectorScalableInsertOpLowering, VectorScalableExtractOpLowering,
1935bc946f52SMubashar Ahmad                MaskedReductionOpConversion, VectorInterleaveOpLowering,
193667b302c5SCullen Rhodes                VectorDeinterleaveOpLowering, VectorFromElementsLowering,
1937a6e72f93SManupa Karunaratne                VectorScalableStepOpLowering>(converter);
19385c0c51a9SNicolas Vasilache }
19395c0c51a9SNicolas Vasilache 
194063b683a8SNicolas Vasilache void mlir::populateVectorToLLVMMatrixConversionPatterns(
1941206fad0eSMatthias Springer     const LLVMTypeConverter &converter, RewritePatternSet &patterns) {
1942dc4e913bSChris Lattner   patterns.add<VectorMatmulOpConversion>(converter);
1943dc4e913bSChris Lattner   patterns.add<VectorFlatTransposeOpConversion>(converter);
194463b683a8SNicolas Vasilache }
1945