15c0c51a9SNicolas Vasilache //===- VectorToLLVM.cpp - Conversion from Vector to the LLVM dialect ------===// 25c0c51a9SNicolas Vasilache // 330857107SMehdi Amini // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 456222a06SMehdi Amini // See https://llvm.org/LICENSE.txt for license information. 556222a06SMehdi Amini // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 65c0c51a9SNicolas Vasilache // 756222a06SMehdi Amini //===----------------------------------------------------------------------===// 85c0c51a9SNicolas Vasilache 965678d93SNicolas Vasilache #include "mlir/Conversion/VectorToLLVM/ConvertVectorToLLVM.h" 10870c1fd4SAlex Zinenko 111b8b5564SNicolas Vasilache #include "mlir/Conversion/ArithCommon/AttrToLLVMConverter.h" 123be3883eSBenjamin Maxwell #include "mlir/Conversion/LLVMCommon/PrintCallHelper.h" 13499abb24SKrzysztof Drewniak #include "mlir/Conversion/LLVMCommon/TypeConverter.h" 1475e5f0aaSAlex Zinenko #include "mlir/Conversion/LLVMCommon/VectorPattern.h" 15abc362a1SJakub Kuderski #include "mlir/Dialect/Arith/IR/Arith.h" 16abc362a1SJakub Kuderski #include "mlir/Dialect/Arith/Utils/Utils.h" 17e332c22cSNicolas Vasilache #include "mlir/Dialect/LLVMIR/FunctionCallUtils.h" 185c0c51a9SNicolas Vasilache #include "mlir/Dialect/LLVMIR/LLVMDialect.h" 19e2310704SJulian Gross #include "mlir/Dialect/MemRef/IR/MemRef.h" 208f5d5194SDaniil Dudkin #include "mlir/Dialect/Vector/IR/VectorOps.h" 21e9b82a5cSDiego Caballero #include "mlir/Dialect/Vector/Interfaces/MaskableOpInterface.h" 222bc4c3e9SNicolas Vasilache #include "mlir/Dialect/Vector/Transforms/LoweringPatterns.h" 2399ef9eebSMatthias Springer #include "mlir/Dialect/Vector/Transforms/VectorTransforms.h" 248f5d5194SDaniil Dudkin #include "mlir/IR/BuiltinAttributes.h" 258f5d5194SDaniil Dudkin #include "mlir/IR/BuiltinTypeInterfaces.h" 2609f7a55fSRiver Riddle #include "mlir/IR/BuiltinTypes.h" 278fe076ffSThomas Raoux #include "mlir/IR/TypeUtilities.h" 28929189a4SWilliam S. Moses #include "mlir/Target/LLVMIR/TypeToLLVM.h" 295c0c51a9SNicolas Vasilache #include "mlir/Transforms/DialectConversion.h" 308f5d5194SDaniil Dudkin #include "llvm/ADT/APFloat.h" 3169049988SMehdi Amini #include "llvm/Support/Casting.h" 32a9a1b45bSKazu Hirata #include <optional> 335c0c51a9SNicolas Vasilache 345c0c51a9SNicolas Vasilache using namespace mlir; 3565678d93SNicolas Vasilache using namespace mlir::vector; 365c0c51a9SNicolas Vasilache 379826fe5cSAart Bik // Helper to reduce vector type by *all* but one rank at back. 389826fe5cSAart Bik static VectorType reducedVectorTypeBack(VectorType tp) { 399826fe5cSAart Bik assert((tp.getRank() > 1) && "unlowerable vector type"); 40a4830d14SJavier Setoain return VectorType::get(tp.getShape().take_back(), tp.getElementType(), 41f22af204SAndrzej Warzynski tp.getScalableDims().take_back()); 429826fe5cSAart Bik } 439826fe5cSAart Bik 441c81adf3SAart Bik // Helper that picks the proper sequence for inserting. 45e62a6956SRiver Riddle static Value insertOne(ConversionPatternRewriter &rewriter, 46ce254598SMatthias Springer const LLVMTypeConverter &typeConverter, Location loc, 470f04384dSAlex Zinenko Value val1, Value val2, Type llvmType, int64_t rank, 480f04384dSAlex Zinenko int64_t pos) { 49e7026abaSNicolas Vasilache assert(rank > 0 && "0-D vector corner case should have been handled already"); 501c81adf3SAart Bik if (rank == 1) { 511c81adf3SAart Bik auto idxType = rewriter.getIndexType(); 521c81adf3SAart Bik auto constant = rewriter.create<LLVM::ConstantOp>( 530f04384dSAlex Zinenko loc, typeConverter.convertType(idxType), 541c81adf3SAart Bik rewriter.getIntegerAttr(idxType, pos)); 551c81adf3SAart Bik return rewriter.create<LLVM::InsertElementOp>(loc, llvmType, val1, val2, 561c81adf3SAart Bik constant); 571c81adf3SAart Bik } 585c5af910SJeff Niu return rewriter.create<LLVM::InsertValueOp>(loc, val1, val2, pos); 591c81adf3SAart Bik } 601c81adf3SAart Bik 611c81adf3SAart Bik // Helper that picks the proper sequence for extracting. 62e62a6956SRiver Riddle static Value extractOne(ConversionPatternRewriter &rewriter, 63ce254598SMatthias Springer const LLVMTypeConverter &typeConverter, Location loc, 640f04384dSAlex Zinenko Value val, Type llvmType, int64_t rank, int64_t pos) { 65cc311a15SMichal Terepeta if (rank <= 1) { 661c81adf3SAart Bik auto idxType = rewriter.getIndexType(); 671c81adf3SAart Bik auto constant = rewriter.create<LLVM::ConstantOp>( 680f04384dSAlex Zinenko loc, typeConverter.convertType(idxType), 691c81adf3SAart Bik rewriter.getIntegerAttr(idxType, pos)); 701c81adf3SAart Bik return rewriter.create<LLVM::ExtractElementOp>(loc, llvmType, val, 711c81adf3SAart Bik constant); 721c81adf3SAart Bik } 735c5af910SJeff Niu return rewriter.create<LLVM::ExtractValueOp>(loc, val, pos); 741c81adf3SAart Bik } 751c81adf3SAart Bik 7626c8f908SThomas Raoux // Helper that returns data layout alignment of a memref. 77ce254598SMatthias Springer LogicalResult getMemRefAlignment(const LLVMTypeConverter &typeConverter, 7826c8f908SThomas Raoux MemRefType memrefType, unsigned &align) { 7926c8f908SThomas Raoux Type elementTy = typeConverter.convertType(memrefType.getElementType()); 805f9e0466SNicolas Vasilache if (!elementTy) 815f9e0466SNicolas Vasilache return failure(); 825f9e0466SNicolas Vasilache 83b2ab375dSAlex Zinenko // TODO: this should use the MLIR data layout when it becomes available and 84b2ab375dSAlex Zinenko // stop depending on translation. 8587a89e0fSAlex Zinenko llvm::LLVMContext llvmContext; 8687a89e0fSAlex Zinenko align = LLVM::TypeToLLVMIRTranslator(llvmContext) 87c69c9e0fSAlex Zinenko .getPreferredAlignment(elementTy, typeConverter.getDataLayout()); 885f9e0466SNicolas Vasilache return success(); 895f9e0466SNicolas Vasilache } 905f9e0466SNicolas Vasilache 9178c49743SQuinn Dawkins // Check if the last stride is non-unit and has a valid memory space. 92499abb24SKrzysztof Drewniak static LogicalResult isMemRefTypeSupported(MemRefType memRefType, 93ce254598SMatthias Springer const LLVMTypeConverter &converter) { 946aaa8f25SMatthias Springer if (!memRefType.isLastDimUnitStride()) 95b1d26875SMatthias Springer return failure(); 9678c49743SQuinn Dawkins if (failed(converter.getMemRefAddressSpace(memRefType))) 97e8dcf5f8Saartbik return failure(); 9819dbb230Saartbik return success(); 9919dbb230Saartbik } 10019dbb230Saartbik 101f250b972SChe-Yu Wu // Add an index vector component to a base pointer. 102f250b972SChe-Yu Wu static Value getIndexedPtrs(ConversionPatternRewriter &rewriter, Location loc, 103ce254598SMatthias Springer const LLVMTypeConverter &typeConverter, 104f250b972SChe-Yu Wu MemRefType memRefType, Value llvmMemref, Value base, 1051e7d6d34SCullen Rhodes Value index, VectorType vectorType) { 106499abb24SKrzysztof Drewniak assert(succeeded(isMemRefTypeSupported(memRefType, typeConverter)) && 107f250b972SChe-Yu Wu "unsupported memref type"); 1081e7d6d34SCullen Rhodes assert(vectorType.getRank() == 1 && "expected a 1-d vector type"); 109f250b972SChe-Yu Wu auto pType = MemRefDescriptor(llvmMemref).getElementPtrType(); 1101e7d6d34SCullen Rhodes auto ptrsType = 1111e7d6d34SCullen Rhodes LLVM::getVectorType(pType, vectorType.getDimSize(0), 1121e7d6d34SCullen Rhodes /*isScalable=*/vectorType.getScalableDims()[0]); 1134a2d4588SMarkus Böck return rewriter.create<LLVM::GEPOp>( 1144a2d4588SMarkus Böck loc, ptrsType, typeConverter.convertType(memRefType.getElementType()), 1154a2d4588SMarkus Böck base, index); 116f250b972SChe-Yu Wu } 117f250b972SChe-Yu Wu 11898f6289aSDiego Caballero /// Convert `foldResult` into a Value. Integer attribute is converted to 11998f6289aSDiego Caballero /// an LLVM constant op. 12098f6289aSDiego Caballero static Value getAsLLVMValue(OpBuilder &builder, Location loc, 12198f6289aSDiego Caballero OpFoldResult foldResult) { 12298f6289aSDiego Caballero if (auto attr = foldResult.dyn_cast<Attribute>()) { 12398f6289aSDiego Caballero auto intAttr = cast<IntegerAttr>(attr); 12498f6289aSDiego Caballero return builder.create<LLVM::ConstantOp>(loc, intAttr).getResult(); 12598f6289aSDiego Caballero } 12698f6289aSDiego Caballero 127129ec845SKazu Hirata return cast<Value>(foldResult); 12898f6289aSDiego Caballero } 12998f6289aSDiego Caballero 13090c01357SBenjamin Kramer namespace { 131e83b7b99Saartbik 132a4830d14SJavier Setoain /// Trivial Vector to LLVM conversions 133a4830d14SJavier Setoain using VectorScaleOpConversion = 134a4830d14SJavier Setoain OneToOneConvertToLLVMPattern<vector::VectorScaleOp, LLVM::vscale>; 135a4830d14SJavier Setoain 136cf5c517cSDiego Caballero /// Conversion pattern for a vector.bitcast. 137cf5c517cSDiego Caballero class VectorBitCastOpConversion 138cf5c517cSDiego Caballero : public ConvertOpToLLVMPattern<vector::BitCastOp> { 139cf5c517cSDiego Caballero public: 140cf5c517cSDiego Caballero using ConvertOpToLLVMPattern<vector::BitCastOp>::ConvertOpToLLVMPattern; 141cf5c517cSDiego Caballero 142cf5c517cSDiego Caballero LogicalResult 143ef976337SRiver Riddle matchAndRewrite(vector::BitCastOp bitCastOp, OpAdaptor adaptor, 144cf5c517cSDiego Caballero ConversionPatternRewriter &rewriter) const override { 1451423e8bfSMichal Terepeta // Only 0-D and 1-D vectors can be lowered to LLVM. 1461423e8bfSMichal Terepeta VectorType resultTy = bitCastOp.getResultVectorType(); 1471423e8bfSMichal Terepeta if (resultTy.getRank() > 1) 148cf5c517cSDiego Caballero return failure(); 149cf5c517cSDiego Caballero Type newResultTy = typeConverter->convertType(resultTy); 150cf5c517cSDiego Caballero rewriter.replaceOpWithNewOp<LLVM::BitcastOp>(bitCastOp, newResultTy, 151ef976337SRiver Riddle adaptor.getOperands()[0]); 152cf5c517cSDiego Caballero return success(); 153cf5c517cSDiego Caballero } 154cf5c517cSDiego Caballero }; 155cf5c517cSDiego Caballero 15663b683a8SNicolas Vasilache /// Conversion pattern for a vector.matrix_multiply. 15763b683a8SNicolas Vasilache /// This is lowered directly to the proper llvm.intr.matrix.multiply. 158563879b6SRahul Joshi class VectorMatmulOpConversion 159563879b6SRahul Joshi : public ConvertOpToLLVMPattern<vector::MatmulOp> { 16063b683a8SNicolas Vasilache public: 161563879b6SRahul Joshi using ConvertOpToLLVMPattern<vector::MatmulOp>::ConvertOpToLLVMPattern; 16263b683a8SNicolas Vasilache 1633145427dSRiver Riddle LogicalResult 164ef976337SRiver Riddle matchAndRewrite(vector::MatmulOp matmulOp, OpAdaptor adaptor, 16563b683a8SNicolas Vasilache ConversionPatternRewriter &rewriter) const override { 16663b683a8SNicolas Vasilache rewriter.replaceOpWithNewOp<LLVM::MatrixMultiplyOp>( 1677c38fd60SJacques Pienaar matmulOp, typeConverter->convertType(matmulOp.getRes().getType()), 1687c38fd60SJacques Pienaar adaptor.getLhs(), adaptor.getRhs(), matmulOp.getLhsRows(), 1697c38fd60SJacques Pienaar matmulOp.getLhsColumns(), matmulOp.getRhsColumns()); 1703145427dSRiver Riddle return success(); 17163b683a8SNicolas Vasilache } 17263b683a8SNicolas Vasilache }; 17363b683a8SNicolas Vasilache 174c295a65dSaartbik /// Conversion pattern for a vector.flat_transpose. 175c295a65dSaartbik /// This is lowered directly to the proper llvm.intr.matrix.transpose. 176563879b6SRahul Joshi class VectorFlatTransposeOpConversion 177563879b6SRahul Joshi : public ConvertOpToLLVMPattern<vector::FlatTransposeOp> { 178c295a65dSaartbik public: 179563879b6SRahul Joshi using ConvertOpToLLVMPattern<vector::FlatTransposeOp>::ConvertOpToLLVMPattern; 180c295a65dSaartbik 181c295a65dSaartbik LogicalResult 182ef976337SRiver Riddle matchAndRewrite(vector::FlatTransposeOp transOp, OpAdaptor adaptor, 183c295a65dSaartbik ConversionPatternRewriter &rewriter) const override { 184c295a65dSaartbik rewriter.replaceOpWithNewOp<LLVM::MatrixTransposeOp>( 1857c38fd60SJacques Pienaar transOp, typeConverter->convertType(transOp.getRes().getType()), 1867c38fd60SJacques Pienaar adaptor.getMatrix(), transOp.getRows(), transOp.getColumns()); 187c295a65dSaartbik return success(); 188c295a65dSaartbik } 189c295a65dSaartbik }; 190c295a65dSaartbik 191ee66e43aSDiego Caballero /// Overloaded utility that replaces a vector.load, vector.store, 192ee66e43aSDiego Caballero /// vector.maskedload and vector.maskedstore with their respective LLVM 193ee66e43aSDiego Caballero /// couterparts. 194ee66e43aSDiego Caballero static void replaceLoadOrStoreOp(vector::LoadOp loadOp, 195ee66e43aSDiego Caballero vector::LoadOpAdaptor adaptor, 196ee66e43aSDiego Caballero VectorType vectorTy, Value ptr, unsigned align, 197ee66e43aSDiego Caballero ConversionPatternRewriter &rewriter) { 1985cfe24eeSKrzysztof Drewniak rewriter.replaceOpWithNewOp<LLVM::LoadOp>(loadOp, vectorTy, ptr, align, 1995cfe24eeSKrzysztof Drewniak /*volatile_=*/false, 2005cfe24eeSKrzysztof Drewniak loadOp.getNontemporal()); 20139379916Saartbik } 20239379916Saartbik 203ee66e43aSDiego Caballero static void replaceLoadOrStoreOp(vector::MaskedLoadOp loadOp, 204ee66e43aSDiego Caballero vector::MaskedLoadOpAdaptor adaptor, 205ee66e43aSDiego Caballero VectorType vectorTy, Value ptr, unsigned align, 206ee66e43aSDiego Caballero ConversionPatternRewriter &rewriter) { 207ee66e43aSDiego Caballero rewriter.replaceOpWithNewOp<LLVM::MaskedLoadOp>( 2087c38fd60SJacques Pienaar loadOp, vectorTy, ptr, adaptor.getMask(), adaptor.getPassThru(), align); 209ee66e43aSDiego Caballero } 210ee66e43aSDiego Caballero 211ee66e43aSDiego Caballero static void replaceLoadOrStoreOp(vector::StoreOp storeOp, 212ee66e43aSDiego Caballero vector::StoreOpAdaptor adaptor, 213ee66e43aSDiego Caballero VectorType vectorTy, Value ptr, unsigned align, 214ee66e43aSDiego Caballero ConversionPatternRewriter &rewriter) { 2157c38fd60SJacques Pienaar rewriter.replaceOpWithNewOp<LLVM::StoreOp>(storeOp, adaptor.getValueToStore(), 2165cfe24eeSKrzysztof Drewniak ptr, align, /*volatile_=*/false, 2175cfe24eeSKrzysztof Drewniak storeOp.getNontemporal()); 218ee66e43aSDiego Caballero } 219ee66e43aSDiego Caballero 220ee66e43aSDiego Caballero static void replaceLoadOrStoreOp(vector::MaskedStoreOp storeOp, 221ee66e43aSDiego Caballero vector::MaskedStoreOpAdaptor adaptor, 222ee66e43aSDiego Caballero VectorType vectorTy, Value ptr, unsigned align, 223ee66e43aSDiego Caballero ConversionPatternRewriter &rewriter) { 224ee66e43aSDiego Caballero rewriter.replaceOpWithNewOp<LLVM::MaskedStoreOp>( 2257c38fd60SJacques Pienaar storeOp, adaptor.getValueToStore(), ptr, adaptor.getMask(), align); 226ee66e43aSDiego Caballero } 227ee66e43aSDiego Caballero 228ee66e43aSDiego Caballero /// Conversion pattern for a vector.load, vector.store, vector.maskedload, and 229ee66e43aSDiego Caballero /// vector.maskedstore. 2304db0bd28SCullen Rhodes template <class LoadOrStoreOp> 231ee66e43aSDiego Caballero class VectorLoadStoreConversion : public ConvertOpToLLVMPattern<LoadOrStoreOp> { 23239379916Saartbik public: 233ee66e43aSDiego Caballero using ConvertOpToLLVMPattern<LoadOrStoreOp>::ConvertOpToLLVMPattern; 23439379916Saartbik 23539379916Saartbik LogicalResult 236ef976337SRiver Riddle matchAndRewrite(LoadOrStoreOp loadOrStoreOp, 237ef976337SRiver Riddle typename LoadOrStoreOp::Adaptor adaptor, 23839379916Saartbik ConversionPatternRewriter &rewriter) const override { 239ee66e43aSDiego Caballero // Only 1-D vectors can be lowered to LLVM. 240ee66e43aSDiego Caballero VectorType vectorTy = loadOrStoreOp.getVectorType(); 241ee66e43aSDiego Caballero if (vectorTy.getRank() > 1) 242ee66e43aSDiego Caballero return failure(); 243ee66e43aSDiego Caballero 244ee66e43aSDiego Caballero auto loc = loadOrStoreOp->getLoc(); 245ee66e43aSDiego Caballero MemRefType memRefTy = loadOrStoreOp.getMemRefType(); 24639379916Saartbik 24739379916Saartbik // Resolve alignment. 24839379916Saartbik unsigned align; 24973863648SStephen Neuendorffer if (failed(getMemRefAlignment(*this->getTypeConverter(), memRefTy, align))) 25039379916Saartbik return failure(); 25139379916Saartbik 252a57def30SAart Bik // Resolve address. 2535550c821STres Popp auto vtype = cast<VectorType>( 2545550c821STres Popp this->typeConverter->convertType(loadOrStoreOp.getVectorType())); 2557c38fd60SJacques Pienaar Value dataPtr = this->getStridedElementPtr(loc, memRefTy, adaptor.getBase(), 2567c38fd60SJacques Pienaar adaptor.getIndices(), rewriter); 257ceb4dc44SChristian Ulmann replaceLoadOrStoreOp(loadOrStoreOp, adaptor, vtype, dataPtr, align, 258ceb4dc44SChristian Ulmann rewriter); 25939379916Saartbik return success(); 26039379916Saartbik } 26139379916Saartbik }; 26239379916Saartbik 26319dbb230Saartbik /// Conversion pattern for a vector.gather. 264563879b6SRahul Joshi class VectorGatherOpConversion 265563879b6SRahul Joshi : public ConvertOpToLLVMPattern<vector::GatherOp> { 26619dbb230Saartbik public: 267563879b6SRahul Joshi using ConvertOpToLLVMPattern<vector::GatherOp>::ConvertOpToLLVMPattern; 26819dbb230Saartbik 26919dbb230Saartbik LogicalResult 270ef976337SRiver Riddle matchAndRewrite(vector::GatherOp gather, OpAdaptor adaptor, 27119dbb230Saartbik ConversionPatternRewriter &rewriter) const override { 2725550c821STres Popp MemRefType memRefType = dyn_cast<MemRefType>(gather.getBaseType()); 27366c2b768SJerry Wu assert(memRefType && "The base should be bufferized"); 27419dbb230Saartbik 275499abb24SKrzysztof Drewniak if (failed(isMemRefTypeSupported(memRefType, *this->getTypeConverter()))) 276f250b972SChe-Yu Wu return failure(); 277f250b972SChe-Yu Wu 278f250b972SChe-Yu Wu auto loc = gather->getLoc(); 279f250b972SChe-Yu Wu 28019dbb230Saartbik // Resolve alignment. 28119dbb230Saartbik unsigned align; 28273863648SStephen Neuendorffer if (failed(getMemRefAlignment(*getTypeConverter(), memRefType, align))) 28319dbb230Saartbik return failure(); 28419dbb230Saartbik 2857c38fd60SJacques Pienaar Value ptr = getStridedElementPtr(loc, memRefType, adaptor.getBase(), 2867c38fd60SJacques Pienaar adaptor.getIndices(), rewriter); 287f250b972SChe-Yu Wu Value base = adaptor.getBase(); 28819dbb230Saartbik 289f250b972SChe-Yu Wu auto llvmNDVectorTy = adaptor.getIndexVec().getType(); 290f250b972SChe-Yu Wu // Handle the simple case of 1-D vector. 2915550c821STres Popp if (!isa<LLVM::LLVMArrayType>(llvmNDVectorTy)) { 292f250b972SChe-Yu Wu auto vType = gather.getVectorType(); 293f250b972SChe-Yu Wu // Resolve address. 2941e7d6d34SCullen Rhodes Value ptrs = 2951e7d6d34SCullen Rhodes getIndexedPtrs(rewriter, loc, *this->getTypeConverter(), memRefType, 2961e7d6d34SCullen Rhodes base, ptr, adaptor.getIndexVec(), vType); 29719dbb230Saartbik // Replace with the gather intrinsic. 29819dbb230Saartbik rewriter.replaceOpWithNewOp<LLVM::masked_gather>( 2997c38fd60SJacques Pienaar gather, typeConverter->convertType(vType), ptrs, adaptor.getMask(), 3007c38fd60SJacques Pienaar adaptor.getPassThru(), rewriter.getI32IntegerAttr(align)); 30119dbb230Saartbik return success(); 30219dbb230Saartbik } 303f250b972SChe-Yu Wu 304ce254598SMatthias Springer const LLVMTypeConverter &typeConverter = *this->getTypeConverter(); 305499abb24SKrzysztof Drewniak auto callback = [align, memRefType, base, ptr, loc, &rewriter, 306499abb24SKrzysztof Drewniak &typeConverter](Type llvm1DVectorTy, 307499abb24SKrzysztof Drewniak ValueRange vectorOperands) { 308f250b972SChe-Yu Wu // Resolve address. 309f250b972SChe-Yu Wu Value ptrs = getIndexedPtrs( 310499abb24SKrzysztof Drewniak rewriter, loc, typeConverter, memRefType, base, ptr, 3111e7d6d34SCullen Rhodes /*index=*/vectorOperands[0], cast<VectorType>(llvm1DVectorTy)); 312f250b972SChe-Yu Wu // Create the gather intrinsic. 313f250b972SChe-Yu Wu return rewriter.create<LLVM::masked_gather>( 314f250b972SChe-Yu Wu loc, llvm1DVectorTy, ptrs, /*mask=*/vectorOperands[1], 315f250b972SChe-Yu Wu /*passThru=*/vectorOperands[2], rewriter.getI32IntegerAttr(align)); 316f250b972SChe-Yu Wu }; 317f250b972SChe-Yu Wu SmallVector<Value> vectorOperands = { 318f250b972SChe-Yu Wu adaptor.getIndexVec(), adaptor.getMask(), adaptor.getPassThru()}; 319f250b972SChe-Yu Wu return LLVM::detail::handleMultidimensionalVectors( 320f250b972SChe-Yu Wu gather, vectorOperands, *getTypeConverter(), callback, rewriter); 321f250b972SChe-Yu Wu } 32219dbb230Saartbik }; 32319dbb230Saartbik 32419dbb230Saartbik /// Conversion pattern for a vector.scatter. 325563879b6SRahul Joshi class VectorScatterOpConversion 326563879b6SRahul Joshi : public ConvertOpToLLVMPattern<vector::ScatterOp> { 32719dbb230Saartbik public: 328563879b6SRahul Joshi using ConvertOpToLLVMPattern<vector::ScatterOp>::ConvertOpToLLVMPattern; 32919dbb230Saartbik 33019dbb230Saartbik LogicalResult 331ef976337SRiver Riddle matchAndRewrite(vector::ScatterOp scatter, OpAdaptor adaptor, 33219dbb230Saartbik ConversionPatternRewriter &rewriter) const override { 333563879b6SRahul Joshi auto loc = scatter->getLoc(); 334df5ccf5aSAart Bik MemRefType memRefType = scatter.getMemRefType(); 33519dbb230Saartbik 336499abb24SKrzysztof Drewniak if (failed(isMemRefTypeSupported(memRefType, *this->getTypeConverter()))) 337f250b972SChe-Yu Wu return failure(); 338f250b972SChe-Yu Wu 33919dbb230Saartbik // Resolve alignment. 34019dbb230Saartbik unsigned align; 34173863648SStephen Neuendorffer if (failed(getMemRefAlignment(*getTypeConverter(), memRefType, align))) 34219dbb230Saartbik return failure(); 34319dbb230Saartbik 344df5ccf5aSAart Bik // Resolve address. 345df5ccf5aSAart Bik VectorType vType = scatter.getVectorType(); 3467c38fd60SJacques Pienaar Value ptr = getStridedElementPtr(loc, memRefType, adaptor.getBase(), 3477c38fd60SJacques Pienaar adaptor.getIndices(), rewriter); 3481e7d6d34SCullen Rhodes Value ptrs = 3491e7d6d34SCullen Rhodes getIndexedPtrs(rewriter, loc, *this->getTypeConverter(), memRefType, 3501e7d6d34SCullen Rhodes adaptor.getBase(), ptr, adaptor.getIndexVec(), vType); 35119dbb230Saartbik 35219dbb230Saartbik // Replace with the scatter intrinsic. 35319dbb230Saartbik rewriter.replaceOpWithNewOp<LLVM::masked_scatter>( 3547c38fd60SJacques Pienaar scatter, adaptor.getValueToStore(), ptrs, adaptor.getMask(), 35519dbb230Saartbik rewriter.getI32IntegerAttr(align)); 35619dbb230Saartbik return success(); 35719dbb230Saartbik } 35819dbb230Saartbik }; 35919dbb230Saartbik 360e8dcf5f8Saartbik /// Conversion pattern for a vector.expandload. 361563879b6SRahul Joshi class VectorExpandLoadOpConversion 362563879b6SRahul Joshi : public ConvertOpToLLVMPattern<vector::ExpandLoadOp> { 363e8dcf5f8Saartbik public: 364563879b6SRahul Joshi using ConvertOpToLLVMPattern<vector::ExpandLoadOp>::ConvertOpToLLVMPattern; 365e8dcf5f8Saartbik 366e8dcf5f8Saartbik LogicalResult 367ef976337SRiver Riddle matchAndRewrite(vector::ExpandLoadOp expand, OpAdaptor adaptor, 368e8dcf5f8Saartbik ConversionPatternRewriter &rewriter) const override { 369563879b6SRahul Joshi auto loc = expand->getLoc(); 370a57def30SAart Bik MemRefType memRefType = expand.getMemRefType(); 371e8dcf5f8Saartbik 372a57def30SAart Bik // Resolve address. 373656674a7SDiego Caballero auto vtype = typeConverter->convertType(expand.getVectorType()); 3747c38fd60SJacques Pienaar Value ptr = getStridedElementPtr(loc, memRefType, adaptor.getBase(), 3757c38fd60SJacques Pienaar adaptor.getIndices(), rewriter); 376e8dcf5f8Saartbik 377e8dcf5f8Saartbik rewriter.replaceOpWithNewOp<LLVM::masked_expandload>( 3787c38fd60SJacques Pienaar expand, vtype, ptr, adaptor.getMask(), adaptor.getPassThru()); 379e8dcf5f8Saartbik return success(); 380e8dcf5f8Saartbik } 381e8dcf5f8Saartbik }; 382e8dcf5f8Saartbik 383e8dcf5f8Saartbik /// Conversion pattern for a vector.compressstore. 384563879b6SRahul Joshi class VectorCompressStoreOpConversion 385563879b6SRahul Joshi : public ConvertOpToLLVMPattern<vector::CompressStoreOp> { 386e8dcf5f8Saartbik public: 387563879b6SRahul Joshi using ConvertOpToLLVMPattern<vector::CompressStoreOp>::ConvertOpToLLVMPattern; 388e8dcf5f8Saartbik 389e8dcf5f8Saartbik LogicalResult 390ef976337SRiver Riddle matchAndRewrite(vector::CompressStoreOp compress, OpAdaptor adaptor, 391e8dcf5f8Saartbik ConversionPatternRewriter &rewriter) const override { 392563879b6SRahul Joshi auto loc = compress->getLoc(); 393a57def30SAart Bik MemRefType memRefType = compress.getMemRefType(); 394e8dcf5f8Saartbik 395a57def30SAart Bik // Resolve address. 3967c38fd60SJacques Pienaar Value ptr = getStridedElementPtr(loc, memRefType, adaptor.getBase(), 3977c38fd60SJacques Pienaar adaptor.getIndices(), rewriter); 398e8dcf5f8Saartbik 399e8dcf5f8Saartbik rewriter.replaceOpWithNewOp<LLVM::masked_compressstore>( 4007c38fd60SJacques Pienaar compress, adaptor.getValueToStore(), ptr, adaptor.getMask()); 401e8dcf5f8Saartbik return success(); 402e8dcf5f8Saartbik } 403e8dcf5f8Saartbik }; 404e8dcf5f8Saartbik 405e9b82a5cSDiego Caballero /// Reduction neutral classes for overloading. 406e9b82a5cSDiego Caballero class ReductionNeutralZero {}; 407e9b82a5cSDiego Caballero class ReductionNeutralIntOne {}; 408e9b82a5cSDiego Caballero class ReductionNeutralFPOne {}; 409e9b82a5cSDiego Caballero class ReductionNeutralAllOnes {}; 410e9b82a5cSDiego Caballero class ReductionNeutralSIntMin {}; 411e9b82a5cSDiego Caballero class ReductionNeutralUIntMin {}; 412e9b82a5cSDiego Caballero class ReductionNeutralSIntMax {}; 413e9b82a5cSDiego Caballero class ReductionNeutralUIntMax {}; 414e9b82a5cSDiego Caballero class ReductionNeutralFPMin {}; 415e9b82a5cSDiego Caballero class ReductionNeutralFPMax {}; 416e9b82a5cSDiego Caballero 417e9b82a5cSDiego Caballero /// Create the reduction neutral zero value. 418e9b82a5cSDiego Caballero static Value createReductionNeutralValue(ReductionNeutralZero neutral, 419e9b82a5cSDiego Caballero ConversionPatternRewriter &rewriter, 420e9b82a5cSDiego Caballero Location loc, Type llvmType) { 421e9b82a5cSDiego Caballero return rewriter.create<LLVM::ConstantOp>(loc, llvmType, 422e9b82a5cSDiego Caballero rewriter.getZeroAttr(llvmType)); 423e9b82a5cSDiego Caballero } 424e9b82a5cSDiego Caballero 425e9b82a5cSDiego Caballero /// Create the reduction neutral integer one value. 426e9b82a5cSDiego Caballero static Value createReductionNeutralValue(ReductionNeutralIntOne neutral, 427e9b82a5cSDiego Caballero ConversionPatternRewriter &rewriter, 428e9b82a5cSDiego Caballero Location loc, Type llvmType) { 429e9b82a5cSDiego Caballero return rewriter.create<LLVM::ConstantOp>( 430e9b82a5cSDiego Caballero loc, llvmType, rewriter.getIntegerAttr(llvmType, 1)); 431e9b82a5cSDiego Caballero } 432e9b82a5cSDiego Caballero 433e9b82a5cSDiego Caballero /// Create the reduction neutral fp one value. 434e9b82a5cSDiego Caballero static Value createReductionNeutralValue(ReductionNeutralFPOne neutral, 435e9b82a5cSDiego Caballero ConversionPatternRewriter &rewriter, 436e9b82a5cSDiego Caballero Location loc, Type llvmType) { 437e9b82a5cSDiego Caballero return rewriter.create<LLVM::ConstantOp>( 438e9b82a5cSDiego Caballero loc, llvmType, rewriter.getFloatAttr(llvmType, 1.0)); 439e9b82a5cSDiego Caballero } 440e9b82a5cSDiego Caballero 441e9b82a5cSDiego Caballero /// Create the reduction neutral all-ones value. 442e9b82a5cSDiego Caballero static Value createReductionNeutralValue(ReductionNeutralAllOnes neutral, 443e9b82a5cSDiego Caballero ConversionPatternRewriter &rewriter, 444e9b82a5cSDiego Caballero Location loc, Type llvmType) { 445e9b82a5cSDiego Caballero return rewriter.create<LLVM::ConstantOp>( 446e9b82a5cSDiego Caballero loc, llvmType, 447e9b82a5cSDiego Caballero rewriter.getIntegerAttr( 448e9b82a5cSDiego Caballero llvmType, llvm::APInt::getAllOnes(llvmType.getIntOrFloatBitWidth()))); 449e9b82a5cSDiego Caballero } 450e9b82a5cSDiego Caballero 451e9b82a5cSDiego Caballero /// Create the reduction neutral signed int minimum value. 452e9b82a5cSDiego Caballero static Value createReductionNeutralValue(ReductionNeutralSIntMin neutral, 453e9b82a5cSDiego Caballero ConversionPatternRewriter &rewriter, 454e9b82a5cSDiego Caballero Location loc, Type llvmType) { 455e9b82a5cSDiego Caballero return rewriter.create<LLVM::ConstantOp>( 456e9b82a5cSDiego Caballero loc, llvmType, 457e9b82a5cSDiego Caballero rewriter.getIntegerAttr(llvmType, llvm::APInt::getSignedMinValue( 458e9b82a5cSDiego Caballero llvmType.getIntOrFloatBitWidth()))); 459e9b82a5cSDiego Caballero } 460e9b82a5cSDiego Caballero 461e9b82a5cSDiego Caballero /// Create the reduction neutral unsigned int minimum value. 462e9b82a5cSDiego Caballero static Value createReductionNeutralValue(ReductionNeutralUIntMin neutral, 463e9b82a5cSDiego Caballero ConversionPatternRewriter &rewriter, 464e9b82a5cSDiego Caballero Location loc, Type llvmType) { 465e9b82a5cSDiego Caballero return rewriter.create<LLVM::ConstantOp>( 466e9b82a5cSDiego Caballero loc, llvmType, 467e9b82a5cSDiego Caballero rewriter.getIntegerAttr(llvmType, llvm::APInt::getMinValue( 468e9b82a5cSDiego Caballero llvmType.getIntOrFloatBitWidth()))); 469e9b82a5cSDiego Caballero } 470e9b82a5cSDiego Caballero 471e9b82a5cSDiego Caballero /// Create the reduction neutral signed int maximum value. 472e9b82a5cSDiego Caballero static Value createReductionNeutralValue(ReductionNeutralSIntMax neutral, 473e9b82a5cSDiego Caballero ConversionPatternRewriter &rewriter, 474e9b82a5cSDiego Caballero Location loc, Type llvmType) { 475e9b82a5cSDiego Caballero return rewriter.create<LLVM::ConstantOp>( 476e9b82a5cSDiego Caballero loc, llvmType, 477e9b82a5cSDiego Caballero rewriter.getIntegerAttr(llvmType, llvm::APInt::getSignedMaxValue( 478e9b82a5cSDiego Caballero llvmType.getIntOrFloatBitWidth()))); 479e9b82a5cSDiego Caballero } 480e9b82a5cSDiego Caballero 481e9b82a5cSDiego Caballero /// Create the reduction neutral unsigned int maximum value. 482e9b82a5cSDiego Caballero static Value createReductionNeutralValue(ReductionNeutralUIntMax neutral, 483e9b82a5cSDiego Caballero ConversionPatternRewriter &rewriter, 484e9b82a5cSDiego Caballero Location loc, Type llvmType) { 485e9b82a5cSDiego Caballero return rewriter.create<LLVM::ConstantOp>( 486e9b82a5cSDiego Caballero loc, llvmType, 487e9b82a5cSDiego Caballero rewriter.getIntegerAttr(llvmType, llvm::APInt::getMaxValue( 488e9b82a5cSDiego Caballero llvmType.getIntOrFloatBitWidth()))); 489e9b82a5cSDiego Caballero } 490e9b82a5cSDiego Caballero 491e9b82a5cSDiego Caballero /// Create the reduction neutral fp minimum value. 492e9b82a5cSDiego Caballero static Value createReductionNeutralValue(ReductionNeutralFPMin neutral, 493e9b82a5cSDiego Caballero ConversionPatternRewriter &rewriter, 494e9b82a5cSDiego Caballero Location loc, Type llvmType) { 4955550c821STres Popp auto floatType = cast<FloatType>(llvmType); 496e9b82a5cSDiego Caballero return rewriter.create<LLVM::ConstantOp>( 497e9b82a5cSDiego Caballero loc, llvmType, 498e9b82a5cSDiego Caballero rewriter.getFloatAttr( 499e9b82a5cSDiego Caballero llvmType, llvm::APFloat::getQNaN(floatType.getFloatSemantics(), 500e9b82a5cSDiego Caballero /*Negative=*/false))); 501e9b82a5cSDiego Caballero } 502e9b82a5cSDiego Caballero 503e9b82a5cSDiego Caballero /// Create the reduction neutral fp maximum value. 504e9b82a5cSDiego Caballero static Value createReductionNeutralValue(ReductionNeutralFPMax neutral, 505e9b82a5cSDiego Caballero ConversionPatternRewriter &rewriter, 506e9b82a5cSDiego Caballero Location loc, Type llvmType) { 5075550c821STres Popp auto floatType = cast<FloatType>(llvmType); 508e9b82a5cSDiego Caballero return rewriter.create<LLVM::ConstantOp>( 509e9b82a5cSDiego Caballero loc, llvmType, 510e9b82a5cSDiego Caballero rewriter.getFloatAttr( 511e9b82a5cSDiego Caballero llvmType, llvm::APFloat::getQNaN(floatType.getFloatSemantics(), 512e9b82a5cSDiego Caballero /*Negative=*/true))); 513e9b82a5cSDiego Caballero } 514e9b82a5cSDiego Caballero 515e9b82a5cSDiego Caballero /// Returns `accumulator` if it has a valid value. Otherwise, creates and 516e9b82a5cSDiego Caballero /// returns a new accumulator value using `ReductionNeutral`. 517e9b82a5cSDiego Caballero template <class ReductionNeutral> 518e9b82a5cSDiego Caballero static Value getOrCreateAccumulator(ConversionPatternRewriter &rewriter, 519e9b82a5cSDiego Caballero Location loc, Type llvmType, 520e9b82a5cSDiego Caballero Value accumulator) { 521e9b82a5cSDiego Caballero if (accumulator) 522e9b82a5cSDiego Caballero return accumulator; 523e9b82a5cSDiego Caballero 524e9b82a5cSDiego Caballero return createReductionNeutralValue(ReductionNeutral(), rewriter, loc, 525e9b82a5cSDiego Caballero llvmType); 526e9b82a5cSDiego Caballero } 527e9b82a5cSDiego Caballero 528abcbbe71SZhaoshi Zheng /// Creates a value with the 1-D vector shape provided in `llvmType`. 529e9b82a5cSDiego Caballero /// This is used as effective vector length by some intrinsics supporting 530e9b82a5cSDiego Caballero /// dynamic vector lengths at runtime. 531e9b82a5cSDiego Caballero static Value createVectorLengthValue(ConversionPatternRewriter &rewriter, 532e9b82a5cSDiego Caballero Location loc, Type llvmType) { 533e9b82a5cSDiego Caballero VectorType vType = cast<VectorType>(llvmType); 534e9b82a5cSDiego Caballero auto vShape = vType.getShape(); 535e9b82a5cSDiego Caballero assert(vShape.size() == 1 && "Unexpected multi-dim vector type"); 536e9b82a5cSDiego Caballero 537abcbbe71SZhaoshi Zheng Value baseVecLength = rewriter.create<LLVM::ConstantOp>( 538e9b82a5cSDiego Caballero loc, rewriter.getI32Type(), 539e9b82a5cSDiego Caballero rewriter.getIntegerAttr(rewriter.getI32Type(), vShape[0])); 540abcbbe71SZhaoshi Zheng 541abcbbe71SZhaoshi Zheng if (!vType.getScalableDims()[0]) 542abcbbe71SZhaoshi Zheng return baseVecLength; 543abcbbe71SZhaoshi Zheng 544abcbbe71SZhaoshi Zheng // For a scalable vector type, create and return `vScale * baseVecLength`. 545abcbbe71SZhaoshi Zheng Value vScale = rewriter.create<vector::VectorScaleOp>(loc); 546abcbbe71SZhaoshi Zheng vScale = 547abcbbe71SZhaoshi Zheng rewriter.create<arith::IndexCastOp>(loc, rewriter.getI32Type(), vScale); 548abcbbe71SZhaoshi Zheng Value scalableVecLength = 549abcbbe71SZhaoshi Zheng rewriter.create<arith::MulIOp>(loc, baseVecLength, vScale); 550abcbbe71SZhaoshi Zheng return scalableVecLength; 551e9b82a5cSDiego Caballero } 552e9b82a5cSDiego Caballero 553fa596c69SMahesh Ravishankar /// Helper method to lower a `vector.reduction` op that performs an arithmetic 554fa596c69SMahesh Ravishankar /// operation like add,mul, etc.. `VectorOp` is the LLVM vector intrinsic to use 555fa596c69SMahesh Ravishankar /// and `ScalarOp` is the scalar operation used to add the accumulation value if 556fa596c69SMahesh Ravishankar /// non-null. 557e9b82a5cSDiego Caballero template <class LLVMRedIntrinOp, class ScalarOp> 558fa596c69SMahesh Ravishankar static Value createIntegerReductionArithmeticOpLowering( 559fa596c69SMahesh Ravishankar ConversionPatternRewriter &rewriter, Location loc, Type llvmType, 560fa596c69SMahesh Ravishankar Value vectorOperand, Value accumulator) { 561e9b82a5cSDiego Caballero 562e9b82a5cSDiego Caballero Value result = rewriter.create<LLVMRedIntrinOp>(loc, llvmType, vectorOperand); 563e9b82a5cSDiego Caballero 564fa596c69SMahesh Ravishankar if (accumulator) 565fa596c69SMahesh Ravishankar result = rewriter.create<ScalarOp>(loc, accumulator, result); 566fa596c69SMahesh Ravishankar return result; 567fa596c69SMahesh Ravishankar } 568fa596c69SMahesh Ravishankar 569fa596c69SMahesh Ravishankar /// Helper method to lower a `vector.reduction` operation that performs 570fa596c69SMahesh Ravishankar /// a comparison operation like `min`/`max`. `VectorOp` is the LLVM vector 571fa596c69SMahesh Ravishankar /// intrinsic to use and `predicate` is the predicate to use to compare+combine 572fa596c69SMahesh Ravishankar /// the accumulator value if non-null. 573e9b82a5cSDiego Caballero template <class LLVMRedIntrinOp> 574fa596c69SMahesh Ravishankar static Value createIntegerReductionComparisonOpLowering( 575fa596c69SMahesh Ravishankar ConversionPatternRewriter &rewriter, Location loc, Type llvmType, 576fa596c69SMahesh Ravishankar Value vectorOperand, Value accumulator, LLVM::ICmpPredicate predicate) { 577e9b82a5cSDiego Caballero Value result = rewriter.create<LLVMRedIntrinOp>(loc, llvmType, vectorOperand); 578fa596c69SMahesh Ravishankar if (accumulator) { 579fa596c69SMahesh Ravishankar Value cmp = 580fa596c69SMahesh Ravishankar rewriter.create<LLVM::ICmpOp>(loc, predicate, accumulator, result); 581fa596c69SMahesh Ravishankar result = rewriter.create<LLVM::SelectOp>(loc, cmp, accumulator, result); 582fa596c69SMahesh Ravishankar } 583fa596c69SMahesh Ravishankar return result; 584fa596c69SMahesh Ravishankar } 585fa596c69SMahesh Ravishankar 586dad9de0aSDaniil Dudkin namespace { 587dad9de0aSDaniil Dudkin template <typename Source> 588dad9de0aSDaniil Dudkin struct VectorToScalarMapper; 589dad9de0aSDaniil Dudkin template <> 590dad9de0aSDaniil Dudkin struct VectorToScalarMapper<LLVM::vector_reduce_fmaximum> { 591dad9de0aSDaniil Dudkin using Type = LLVM::MaximumOp; 592dad9de0aSDaniil Dudkin }; 593dad9de0aSDaniil Dudkin template <> 594dad9de0aSDaniil Dudkin struct VectorToScalarMapper<LLVM::vector_reduce_fminimum> { 595dad9de0aSDaniil Dudkin using Type = LLVM::MinimumOp; 596dad9de0aSDaniil Dudkin }; 597709b2742SDaniil Dudkin template <> 598709b2742SDaniil Dudkin struct VectorToScalarMapper<LLVM::vector_reduce_fmax> { 599709b2742SDaniil Dudkin using Type = LLVM::MaxNumOp; 600709b2742SDaniil Dudkin }; 601709b2742SDaniil Dudkin template <> 602709b2742SDaniil Dudkin struct VectorToScalarMapper<LLVM::vector_reduce_fmin> { 603709b2742SDaniil Dudkin using Type = LLVM::MinNumOp; 604709b2742SDaniil Dudkin }; 605dad9de0aSDaniil Dudkin } // namespace 6068fe076ffSThomas Raoux 607e9b82a5cSDiego Caballero template <class LLVMRedIntrinOp> 6081b8b5564SNicolas Vasilache static Value createFPReductionComparisonOpLowering( 6091b8b5564SNicolas Vasilache ConversionPatternRewriter &rewriter, Location loc, Type llvmType, 6101b8b5564SNicolas Vasilache Value vectorOperand, Value accumulator, LLVM::FastmathFlagsAttr fmf) { 6111b8b5564SNicolas Vasilache Value result = 6121b8b5564SNicolas Vasilache rewriter.create<LLVMRedIntrinOp>(loc, llvmType, vectorOperand, fmf); 613e9b82a5cSDiego Caballero 614dad9de0aSDaniil Dudkin if (accumulator) { 615dad9de0aSDaniil Dudkin result = 616dad9de0aSDaniil Dudkin rewriter.create<typename VectorToScalarMapper<LLVMRedIntrinOp>::Type>( 617dad9de0aSDaniil Dudkin loc, result, accumulator); 618dad9de0aSDaniil Dudkin } 619e9b82a5cSDiego Caballero 620e9b82a5cSDiego Caballero return result; 621e9b82a5cSDiego Caballero } 622e9b82a5cSDiego Caballero 6238f5d5194SDaniil Dudkin /// Reduction neutral classes for overloading 6248f5d5194SDaniil Dudkin class MaskNeutralFMaximum {}; 6258f5d5194SDaniil Dudkin class MaskNeutralFMinimum {}; 6268f5d5194SDaniil Dudkin 6278f5d5194SDaniil Dudkin /// Get the mask neutral floating point maximum value 6288f5d5194SDaniil Dudkin static llvm::APFloat 6298f5d5194SDaniil Dudkin getMaskNeutralValue(MaskNeutralFMaximum, 6308f5d5194SDaniil Dudkin const llvm::fltSemantics &floatSemantics) { 6318f5d5194SDaniil Dudkin return llvm::APFloat::getSmallest(floatSemantics, /*Negative=*/true); 6328f5d5194SDaniil Dudkin } 6338f5d5194SDaniil Dudkin /// Get the mask neutral floating point minimum value 6348f5d5194SDaniil Dudkin static llvm::APFloat 6358f5d5194SDaniil Dudkin getMaskNeutralValue(MaskNeutralFMinimum, 6368f5d5194SDaniil Dudkin const llvm::fltSemantics &floatSemantics) { 6378f5d5194SDaniil Dudkin return llvm::APFloat::getLargest(floatSemantics, /*Negative=*/false); 6388f5d5194SDaniil Dudkin } 6398f5d5194SDaniil Dudkin 6408f5d5194SDaniil Dudkin /// Create the mask neutral floating point MLIR vector constant 6418f5d5194SDaniil Dudkin template <typename MaskNeutral> 6428f5d5194SDaniil Dudkin static Value createMaskNeutralValue(ConversionPatternRewriter &rewriter, 6438f5d5194SDaniil Dudkin Location loc, Type llvmType, 6448f5d5194SDaniil Dudkin Type vectorType) { 6458f5d5194SDaniil Dudkin const auto &floatSemantics = cast<FloatType>(llvmType).getFloatSemantics(); 6468f5d5194SDaniil Dudkin auto value = getMaskNeutralValue(MaskNeutral{}, floatSemantics); 647a5757c5bSChristian Sigg auto denseValue = DenseElementsAttr::get(cast<ShapedType>(vectorType), value); 6488f5d5194SDaniil Dudkin return rewriter.create<LLVM::ConstantOp>(loc, vectorType, denseValue); 6498f5d5194SDaniil Dudkin } 6508f5d5194SDaniil Dudkin 6518f5d5194SDaniil Dudkin /// Lowers masked `fmaximum` and `fminimum` reductions using the non-masked 6528f5d5194SDaniil Dudkin /// intrinsics. It is a workaround to overcome the lack of masked intrinsics for 6538f5d5194SDaniil Dudkin /// `fmaximum`/`fminimum`. 6548f5d5194SDaniil Dudkin /// More information: https://github.com/llvm/llvm-project/issues/64940 6558f5d5194SDaniil Dudkin template <class LLVMRedIntrinOp, class MaskNeutral> 6561b8b5564SNicolas Vasilache static Value 6571b8b5564SNicolas Vasilache lowerMaskedReductionWithRegular(ConversionPatternRewriter &rewriter, 6581b8b5564SNicolas Vasilache Location loc, Type llvmType, 6591b8b5564SNicolas Vasilache Value vectorOperand, Value accumulator, 6601b8b5564SNicolas Vasilache Value mask, LLVM::FastmathFlagsAttr fmf) { 6618f5d5194SDaniil Dudkin const Value vectorMaskNeutral = createMaskNeutralValue<MaskNeutral>( 6628f5d5194SDaniil Dudkin rewriter, loc, llvmType, vectorOperand.getType()); 6638f5d5194SDaniil Dudkin const Value selectedVectorByMask = rewriter.create<LLVM::SelectOp>( 6648f5d5194SDaniil Dudkin loc, mask, vectorOperand, vectorMaskNeutral); 6658f5d5194SDaniil Dudkin return createFPReductionComparisonOpLowering<LLVMRedIntrinOp>( 6661b8b5564SNicolas Vasilache rewriter, loc, llvmType, selectedVectorByMask, accumulator, fmf); 6678f5d5194SDaniil Dudkin } 6688f5d5194SDaniil Dudkin 6691b8b5564SNicolas Vasilache template <class LLVMRedIntrinOp, class ReductionNeutral> 6701b8b5564SNicolas Vasilache static Value 6711b8b5564SNicolas Vasilache lowerReductionWithStartValue(ConversionPatternRewriter &rewriter, Location loc, 6721b8b5564SNicolas Vasilache Type llvmType, Value vectorOperand, 6731b8b5564SNicolas Vasilache Value accumulator, LLVM::FastmathFlagsAttr fmf) { 6741b8b5564SNicolas Vasilache accumulator = getOrCreateAccumulator<ReductionNeutral>(rewriter, loc, 6751b8b5564SNicolas Vasilache llvmType, accumulator); 6761b8b5564SNicolas Vasilache return rewriter.create<LLVMRedIntrinOp>(loc, llvmType, 6771b8b5564SNicolas Vasilache /*startValue=*/accumulator, 6781b8b5564SNicolas Vasilache vectorOperand, fmf); 6791b8b5564SNicolas Vasilache } 6801b8b5564SNicolas Vasilache 681922992a2SJay Foad /// Overloaded methods to lower a *predicated* reduction to an llvm intrinsic 6821b8b5564SNicolas Vasilache /// that requires a start value. This start value format spans across fp 6831b8b5564SNicolas Vasilache /// reductions without mask and all the masked reduction intrinsics. 684e9b82a5cSDiego Caballero template <class LLVMVPRedIntrinOp, class ReductionNeutral> 6851b8b5564SNicolas Vasilache static Value 6861b8b5564SNicolas Vasilache lowerPredicatedReductionWithStartValue(ConversionPatternRewriter &rewriter, 687e9b82a5cSDiego Caballero Location loc, Type llvmType, 6881b8b5564SNicolas Vasilache Value vectorOperand, Value accumulator) { 689e9b82a5cSDiego Caballero accumulator = getOrCreateAccumulator<ReductionNeutral>(rewriter, loc, 690e9b82a5cSDiego Caballero llvmType, accumulator); 691e9b82a5cSDiego Caballero return rewriter.create<LLVMVPRedIntrinOp>(loc, llvmType, 692e9b82a5cSDiego Caballero /*startValue=*/accumulator, 693e9b82a5cSDiego Caballero vectorOperand); 694e9b82a5cSDiego Caballero } 695e9b82a5cSDiego Caballero 696e9b82a5cSDiego Caballero template <class LLVMVPRedIntrinOp, class ReductionNeutral> 6971b8b5564SNicolas Vasilache static Value lowerPredicatedReductionWithStartValue( 6981b8b5564SNicolas Vasilache ConversionPatternRewriter &rewriter, Location loc, Type llvmType, 6991b8b5564SNicolas Vasilache Value vectorOperand, Value accumulator, Value mask) { 700e9b82a5cSDiego Caballero accumulator = getOrCreateAccumulator<ReductionNeutral>(rewriter, loc, 701e9b82a5cSDiego Caballero llvmType, accumulator); 702e9b82a5cSDiego Caballero Value vectorLength = 703e9b82a5cSDiego Caballero createVectorLengthValue(rewriter, loc, vectorOperand.getType()); 704e9b82a5cSDiego Caballero return rewriter.create<LLVMVPRedIntrinOp>(loc, llvmType, 705e9b82a5cSDiego Caballero /*startValue=*/accumulator, 706e9b82a5cSDiego Caballero vectorOperand, mask, vectorLength); 707e9b82a5cSDiego Caballero } 708e9b82a5cSDiego Caballero 709e9b82a5cSDiego Caballero template <class LLVMIntVPRedIntrinOp, class IntReductionNeutral, 710e9b82a5cSDiego Caballero class LLVMFPVPRedIntrinOp, class FPReductionNeutral> 7111b8b5564SNicolas Vasilache static Value lowerPredicatedReductionWithStartValue( 7121b8b5564SNicolas Vasilache ConversionPatternRewriter &rewriter, Location loc, Type llvmType, 7131b8b5564SNicolas Vasilache Value vectorOperand, Value accumulator, Value mask) { 714e9b82a5cSDiego Caballero if (llvmType.isIntOrIndex()) 7151b8b5564SNicolas Vasilache return lowerPredicatedReductionWithStartValue<LLVMIntVPRedIntrinOp, 716e9b82a5cSDiego Caballero IntReductionNeutral>( 717e9b82a5cSDiego Caballero rewriter, loc, llvmType, vectorOperand, accumulator, mask); 718e9b82a5cSDiego Caballero 719e9b82a5cSDiego Caballero // FP dispatch. 7201b8b5564SNicolas Vasilache return lowerPredicatedReductionWithStartValue<LLVMFPVPRedIntrinOp, 7211b8b5564SNicolas Vasilache FPReductionNeutral>( 722e9b82a5cSDiego Caballero rewriter, loc, llvmType, vectorOperand, accumulator, mask); 723e9b82a5cSDiego Caballero } 724e9b82a5cSDiego Caballero 72519dbb230Saartbik /// Conversion pattern for all vector reductions. 726563879b6SRahul Joshi class VectorReductionOpConversion 727563879b6SRahul Joshi : public ConvertOpToLLVMPattern<vector::ReductionOp> { 728e83b7b99Saartbik public: 729ce254598SMatthias Springer explicit VectorReductionOpConversion(const LLVMTypeConverter &typeConv, 730060c9dd1Saartbik bool reassociateFPRed) 731563879b6SRahul Joshi : ConvertOpToLLVMPattern<vector::ReductionOp>(typeConv), 732060c9dd1Saartbik reassociateFPReductions(reassociateFPRed) {} 733e83b7b99Saartbik 7343145427dSRiver Riddle LogicalResult 735ef976337SRiver Riddle matchAndRewrite(vector::ReductionOp reductionOp, OpAdaptor adaptor, 736e83b7b99Saartbik ConversionPatternRewriter &rewriter) const override { 7377c38fd60SJacques Pienaar auto kind = reductionOp.getKind(); 7387c38fd60SJacques Pienaar Type eltType = reductionOp.getDest().getType(); 739dcec2ca5SChristian Sigg Type llvmType = typeConverter->convertType(eltType); 740fa596c69SMahesh Ravishankar Value operand = adaptor.getVector(); 741fa596c69SMahesh Ravishankar Value acc = adaptor.getAcc(); 742fa596c69SMahesh Ravishankar Location loc = reductionOp.getLoc(); 743e9b82a5cSDiego Caballero 744e9628955SAart Bik if (eltType.isIntOrIndex()) { 745e83b7b99Saartbik // Integer reductions: add/mul/min/max/and/or/xor. 746fa596c69SMahesh Ravishankar Value result; 747fa596c69SMahesh Ravishankar switch (kind) { 748fa596c69SMahesh Ravishankar case vector::CombiningKind::ADD: 749fa596c69SMahesh Ravishankar result = 750fa596c69SMahesh Ravishankar createIntegerReductionArithmeticOpLowering<LLVM::vector_reduce_add, 751fa596c69SMahesh Ravishankar LLVM::AddOp>( 752fa596c69SMahesh Ravishankar rewriter, loc, llvmType, operand, acc); 753fa596c69SMahesh Ravishankar break; 754fa596c69SMahesh Ravishankar case vector::CombiningKind::MUL: 755fa596c69SMahesh Ravishankar result = 756fa596c69SMahesh Ravishankar createIntegerReductionArithmeticOpLowering<LLVM::vector_reduce_mul, 757fa596c69SMahesh Ravishankar LLVM::MulOp>( 758fa596c69SMahesh Ravishankar rewriter, loc, llvmType, operand, acc); 759fa596c69SMahesh Ravishankar break; 760fa596c69SMahesh Ravishankar case vector::CombiningKind::MINUI: 761fa596c69SMahesh Ravishankar result = createIntegerReductionComparisonOpLowering< 762fa596c69SMahesh Ravishankar LLVM::vector_reduce_umin>(rewriter, loc, llvmType, operand, acc, 763fa596c69SMahesh Ravishankar LLVM::ICmpPredicate::ule); 764fa596c69SMahesh Ravishankar break; 765fa596c69SMahesh Ravishankar case vector::CombiningKind::MINSI: 766fa596c69SMahesh Ravishankar result = createIntegerReductionComparisonOpLowering< 767fa596c69SMahesh Ravishankar LLVM::vector_reduce_smin>(rewriter, loc, llvmType, operand, acc, 768fa596c69SMahesh Ravishankar LLVM::ICmpPredicate::sle); 769fa596c69SMahesh Ravishankar break; 770fa596c69SMahesh Ravishankar case vector::CombiningKind::MAXUI: 771fa596c69SMahesh Ravishankar result = createIntegerReductionComparisonOpLowering< 772fa596c69SMahesh Ravishankar LLVM::vector_reduce_umax>(rewriter, loc, llvmType, operand, acc, 773fa596c69SMahesh Ravishankar LLVM::ICmpPredicate::uge); 774fa596c69SMahesh Ravishankar break; 775fa596c69SMahesh Ravishankar case vector::CombiningKind::MAXSI: 776fa596c69SMahesh Ravishankar result = createIntegerReductionComparisonOpLowering< 777fa596c69SMahesh Ravishankar LLVM::vector_reduce_smax>(rewriter, loc, llvmType, operand, acc, 778fa596c69SMahesh Ravishankar LLVM::ICmpPredicate::sge); 779fa596c69SMahesh Ravishankar break; 780fa596c69SMahesh Ravishankar case vector::CombiningKind::AND: 781fa596c69SMahesh Ravishankar result = 782fa596c69SMahesh Ravishankar createIntegerReductionArithmeticOpLowering<LLVM::vector_reduce_and, 783fa596c69SMahesh Ravishankar LLVM::AndOp>( 784fa596c69SMahesh Ravishankar rewriter, loc, llvmType, operand, acc); 785fa596c69SMahesh Ravishankar break; 786fa596c69SMahesh Ravishankar case vector::CombiningKind::OR: 787fa596c69SMahesh Ravishankar result = 788fa596c69SMahesh Ravishankar createIntegerReductionArithmeticOpLowering<LLVM::vector_reduce_or, 789fa596c69SMahesh Ravishankar LLVM::OrOp>( 790fa596c69SMahesh Ravishankar rewriter, loc, llvmType, operand, acc); 791fa596c69SMahesh Ravishankar break; 792fa596c69SMahesh Ravishankar case vector::CombiningKind::XOR: 793fa596c69SMahesh Ravishankar result = 794fa596c69SMahesh Ravishankar createIntegerReductionArithmeticOpLowering<LLVM::vector_reduce_xor, 795fa596c69SMahesh Ravishankar LLVM::XOrOp>( 796fa596c69SMahesh Ravishankar rewriter, loc, llvmType, operand, acc); 797fa596c69SMahesh Ravishankar break; 798fa596c69SMahesh Ravishankar default: 7993145427dSRiver Riddle return failure(); 800fa596c69SMahesh Ravishankar } 801fa596c69SMahesh Ravishankar rewriter.replaceOp(reductionOp, result); 802fa596c69SMahesh Ravishankar 8033145427dSRiver Riddle return success(); 804dcec2ca5SChristian Sigg } 805e83b7b99Saartbik 8065550c821STres Popp if (!isa<FloatType>(eltType)) 807dcec2ca5SChristian Sigg return failure(); 808dcec2ca5SChristian Sigg 8091b8b5564SNicolas Vasilache arith::FastMathFlagsAttr fMFAttr = reductionOp.getFastMathFlagsAttr(); 8101b8b5564SNicolas Vasilache LLVM::FastmathFlagsAttr fmf = LLVM::FastmathFlagsAttr::get( 8111b8b5564SNicolas Vasilache reductionOp.getContext(), 8121b8b5564SNicolas Vasilache convertArithFastMathFlagsToLLVM(fMFAttr.getValue())); 8131b8b5564SNicolas Vasilache fmf = LLVM::FastmathFlagsAttr::get( 8141b8b5564SNicolas Vasilache reductionOp.getContext(), 8151b8b5564SNicolas Vasilache fmf.getValue() | (reassociateFPReductions ? LLVM::FastmathFlags::reassoc 8161b8b5564SNicolas Vasilache : LLVM::FastmathFlags::none)); 8171b8b5564SNicolas Vasilache 818e83b7b99Saartbik // Floating-point reductions: add/mul/min/max 819e9b82a5cSDiego Caballero Value result; 820fe0bf7d4SMatthias Springer if (kind == vector::CombiningKind::ADD) { 821e9b82a5cSDiego Caballero result = lowerReductionWithStartValue<LLVM::vector_reduce_fadd, 822e9b82a5cSDiego Caballero ReductionNeutralZero>( 8231b8b5564SNicolas Vasilache rewriter, loc, llvmType, operand, acc, fmf); 824fe0bf7d4SMatthias Springer } else if (kind == vector::CombiningKind::MUL) { 825e9b82a5cSDiego Caballero result = lowerReductionWithStartValue<LLVM::vector_reduce_fmul, 826e9b82a5cSDiego Caballero ReductionNeutralFPOne>( 8271b8b5564SNicolas Vasilache rewriter, loc, llvmType, operand, acc, fmf); 8284a831250SDaniil Dudkin } else if (kind == vector::CombiningKind::MINIMUMF) { 829dad9de0aSDaniil Dudkin result = 830dad9de0aSDaniil Dudkin createFPReductionComparisonOpLowering<LLVM::vector_reduce_fminimum>( 8311b8b5564SNicolas Vasilache rewriter, loc, llvmType, operand, acc, fmf); 8324a831250SDaniil Dudkin } else if (kind == vector::CombiningKind::MAXIMUMF) { 833dad9de0aSDaniil Dudkin result = 834dad9de0aSDaniil Dudkin createFPReductionComparisonOpLowering<LLVM::vector_reduce_fmaximum>( 8351b8b5564SNicolas Vasilache rewriter, loc, llvmType, operand, acc, fmf); 836560564f5SJakub Kuderski } else if (kind == vector::CombiningKind::MINNUMF) { 837709b2742SDaniil Dudkin result = createFPReductionComparisonOpLowering<LLVM::vector_reduce_fmin>( 8381b8b5564SNicolas Vasilache rewriter, loc, llvmType, operand, acc, fmf); 839560564f5SJakub Kuderski } else if (kind == vector::CombiningKind::MAXNUMF) { 840709b2742SDaniil Dudkin result = createFPReductionComparisonOpLowering<LLVM::vector_reduce_fmax>( 8411b8b5564SNicolas Vasilache rewriter, loc, llvmType, operand, acc, fmf); 8428fe076ffSThomas Raoux } else 8433145427dSRiver Riddle return failure(); 8448fe076ffSThomas Raoux 845e9b82a5cSDiego Caballero rewriter.replaceOp(reductionOp, result); 8463145427dSRiver Riddle return success(); 847e83b7b99Saartbik } 848ceb1b327Saartbik 849ceb1b327Saartbik private: 850ceb1b327Saartbik const bool reassociateFPReductions; 851e83b7b99Saartbik }; 852e83b7b99Saartbik 853e9b82a5cSDiego Caballero /// Base class to convert a `vector.mask` operation while matching traits 854e9b82a5cSDiego Caballero /// of the maskable operation nested inside. A `VectorMaskOpConversionBase` 855e9b82a5cSDiego Caballero /// instance matches against a `vector.mask` operation. The `matchAndRewrite` 856e9b82a5cSDiego Caballero /// method performs a second match against the maskable operation `MaskedOp`. 857e9b82a5cSDiego Caballero /// Finally, it invokes the virtual method `matchAndRewriteMaskableOp` to be 858e9b82a5cSDiego Caballero /// implemented by the concrete conversion classes. This method can match 859e9b82a5cSDiego Caballero /// against specific traits of the `vector.mask` and the maskable operation. It 860e9b82a5cSDiego Caballero /// must replace the `vector.mask` operation. 861e9b82a5cSDiego Caballero template <class MaskedOp> 862e9b82a5cSDiego Caballero class VectorMaskOpConversionBase 863e9b82a5cSDiego Caballero : public ConvertOpToLLVMPattern<vector::MaskOp> { 864e9b82a5cSDiego Caballero public: 865e9b82a5cSDiego Caballero using ConvertOpToLLVMPattern<vector::MaskOp>::ConvertOpToLLVMPattern; 866e9b82a5cSDiego Caballero 867e9b82a5cSDiego Caballero LogicalResult 868e9b82a5cSDiego Caballero matchAndRewrite(vector::MaskOp maskOp, OpAdaptor adaptor, 8694eb1f1faSAdrian Kuegel ConversionPatternRewriter &rewriter) const final { 870e9b82a5cSDiego Caballero // Match against the maskable operation kind. 87169049988SMehdi Amini auto maskedOp = llvm::dyn_cast_or_null<MaskedOp>(maskOp.getMaskableOp()); 87269049988SMehdi Amini if (!maskedOp) 873e9b82a5cSDiego Caballero return failure(); 87469049988SMehdi Amini return matchAndRewriteMaskableOp(maskOp, maskedOp, rewriter); 875e9b82a5cSDiego Caballero } 876e9b82a5cSDiego Caballero 877e9b82a5cSDiego Caballero protected: 878e9b82a5cSDiego Caballero virtual LogicalResult 879e9b82a5cSDiego Caballero matchAndRewriteMaskableOp(vector::MaskOp maskOp, 880e9b82a5cSDiego Caballero vector::MaskableOpInterface maskableOp, 881e9b82a5cSDiego Caballero ConversionPatternRewriter &rewriter) const = 0; 882e9b82a5cSDiego Caballero }; 883e9b82a5cSDiego Caballero 884e9b82a5cSDiego Caballero class MaskedReductionOpConversion 885e9b82a5cSDiego Caballero : public VectorMaskOpConversionBase<vector::ReductionOp> { 886e9b82a5cSDiego Caballero 887e9b82a5cSDiego Caballero public: 888e9b82a5cSDiego Caballero using VectorMaskOpConversionBase< 889e9b82a5cSDiego Caballero vector::ReductionOp>::VectorMaskOpConversionBase; 890e9b82a5cSDiego Caballero 8914eb1f1faSAdrian Kuegel LogicalResult matchAndRewriteMaskableOp( 892e9b82a5cSDiego Caballero vector::MaskOp maskOp, MaskableOpInterface maskableOp, 893e9b82a5cSDiego Caballero ConversionPatternRewriter &rewriter) const override { 894e9b82a5cSDiego Caballero auto reductionOp = cast<ReductionOp>(maskableOp.getOperation()); 895e9b82a5cSDiego Caballero auto kind = reductionOp.getKind(); 896e9b82a5cSDiego Caballero Type eltType = reductionOp.getDest().getType(); 897e9b82a5cSDiego Caballero Type llvmType = typeConverter->convertType(eltType); 898e9b82a5cSDiego Caballero Value operand = reductionOp.getVector(); 899e9b82a5cSDiego Caballero Value acc = reductionOp.getAcc(); 900e9b82a5cSDiego Caballero Location loc = reductionOp.getLoc(); 901e9b82a5cSDiego Caballero 9021b8b5564SNicolas Vasilache arith::FastMathFlagsAttr fMFAttr = reductionOp.getFastMathFlagsAttr(); 9031b8b5564SNicolas Vasilache LLVM::FastmathFlagsAttr fmf = LLVM::FastmathFlagsAttr::get( 9041b8b5564SNicolas Vasilache reductionOp.getContext(), 9051b8b5564SNicolas Vasilache convertArithFastMathFlagsToLLVM(fMFAttr.getValue())); 9061b8b5564SNicolas Vasilache 907e9b82a5cSDiego Caballero Value result; 908e9b82a5cSDiego Caballero switch (kind) { 909e9b82a5cSDiego Caballero case vector::CombiningKind::ADD: 9101b8b5564SNicolas Vasilache result = lowerPredicatedReductionWithStartValue< 911e9b82a5cSDiego Caballero LLVM::VPReduceAddOp, ReductionNeutralZero, LLVM::VPReduceFAddOp, 912e9b82a5cSDiego Caballero ReductionNeutralZero>(rewriter, loc, llvmType, operand, acc, 913e9b82a5cSDiego Caballero maskOp.getMask()); 914e9b82a5cSDiego Caballero break; 915e9b82a5cSDiego Caballero case vector::CombiningKind::MUL: 9161b8b5564SNicolas Vasilache result = lowerPredicatedReductionWithStartValue< 917e9b82a5cSDiego Caballero LLVM::VPReduceMulOp, ReductionNeutralIntOne, LLVM::VPReduceFMulOp, 918e9b82a5cSDiego Caballero ReductionNeutralFPOne>(rewriter, loc, llvmType, operand, acc, 919e9b82a5cSDiego Caballero maskOp.getMask()); 920e9b82a5cSDiego Caballero break; 921e9b82a5cSDiego Caballero case vector::CombiningKind::MINUI: 9221b8b5564SNicolas Vasilache result = lowerPredicatedReductionWithStartValue<LLVM::VPReduceUMinOp, 923e9b82a5cSDiego Caballero ReductionNeutralUIntMax>( 924e9b82a5cSDiego Caballero rewriter, loc, llvmType, operand, acc, maskOp.getMask()); 925e9b82a5cSDiego Caballero break; 926e9b82a5cSDiego Caballero case vector::CombiningKind::MINSI: 9271b8b5564SNicolas Vasilache result = lowerPredicatedReductionWithStartValue<LLVM::VPReduceSMinOp, 928e9b82a5cSDiego Caballero ReductionNeutralSIntMax>( 929e9b82a5cSDiego Caballero rewriter, loc, llvmType, operand, acc, maskOp.getMask()); 930e9b82a5cSDiego Caballero break; 931e9b82a5cSDiego Caballero case vector::CombiningKind::MAXUI: 9321b8b5564SNicolas Vasilache result = lowerPredicatedReductionWithStartValue<LLVM::VPReduceUMaxOp, 933e9b82a5cSDiego Caballero ReductionNeutralUIntMin>( 934e9b82a5cSDiego Caballero rewriter, loc, llvmType, operand, acc, maskOp.getMask()); 935e9b82a5cSDiego Caballero break; 936e9b82a5cSDiego Caballero case vector::CombiningKind::MAXSI: 9371b8b5564SNicolas Vasilache result = lowerPredicatedReductionWithStartValue<LLVM::VPReduceSMaxOp, 938e9b82a5cSDiego Caballero ReductionNeutralSIntMin>( 939e9b82a5cSDiego Caballero rewriter, loc, llvmType, operand, acc, maskOp.getMask()); 940e9b82a5cSDiego Caballero break; 941e9b82a5cSDiego Caballero case vector::CombiningKind::AND: 9421b8b5564SNicolas Vasilache result = lowerPredicatedReductionWithStartValue<LLVM::VPReduceAndOp, 943e9b82a5cSDiego Caballero ReductionNeutralAllOnes>( 944e9b82a5cSDiego Caballero rewriter, loc, llvmType, operand, acc, maskOp.getMask()); 945e9b82a5cSDiego Caballero break; 946e9b82a5cSDiego Caballero case vector::CombiningKind::OR: 9471b8b5564SNicolas Vasilache result = lowerPredicatedReductionWithStartValue<LLVM::VPReduceOrOp, 948e9b82a5cSDiego Caballero ReductionNeutralZero>( 949e9b82a5cSDiego Caballero rewriter, loc, llvmType, operand, acc, maskOp.getMask()); 950e9b82a5cSDiego Caballero break; 951e9b82a5cSDiego Caballero case vector::CombiningKind::XOR: 9521b8b5564SNicolas Vasilache result = lowerPredicatedReductionWithStartValue<LLVM::VPReduceXorOp, 953e9b82a5cSDiego Caballero ReductionNeutralZero>( 954e9b82a5cSDiego Caballero rewriter, loc, llvmType, operand, acc, maskOp.getMask()); 955e9b82a5cSDiego Caballero break; 956560564f5SJakub Kuderski case vector::CombiningKind::MINNUMF: 9571b8b5564SNicolas Vasilache result = lowerPredicatedReductionWithStartValue<LLVM::VPReduceFMinOp, 958e9b82a5cSDiego Caballero ReductionNeutralFPMax>( 959e9b82a5cSDiego Caballero rewriter, loc, llvmType, operand, acc, maskOp.getMask()); 960e9b82a5cSDiego Caballero break; 961560564f5SJakub Kuderski case vector::CombiningKind::MAXNUMF: 9621b8b5564SNicolas Vasilache result = lowerPredicatedReductionWithStartValue<LLVM::VPReduceFMaxOp, 963e9b82a5cSDiego Caballero ReductionNeutralFPMin>( 964e9b82a5cSDiego Caballero rewriter, loc, llvmType, operand, acc, maskOp.getMask()); 965e9b82a5cSDiego Caballero break; 9668f5d5194SDaniil Dudkin case CombiningKind::MAXIMUMF: 9678f5d5194SDaniil Dudkin result = lowerMaskedReductionWithRegular<LLVM::vector_reduce_fmaximum, 9688f5d5194SDaniil Dudkin MaskNeutralFMaximum>( 9691b8b5564SNicolas Vasilache rewriter, loc, llvmType, operand, acc, maskOp.getMask(), fmf); 9708f5d5194SDaniil Dudkin break; 9718f5d5194SDaniil Dudkin case CombiningKind::MINIMUMF: 9728f5d5194SDaniil Dudkin result = lowerMaskedReductionWithRegular<LLVM::vector_reduce_fminimum, 9738f5d5194SDaniil Dudkin MaskNeutralFMinimum>( 9741b8b5564SNicolas Vasilache rewriter, loc, llvmType, operand, acc, maskOp.getMask(), fmf); 9758f5d5194SDaniil Dudkin break; 976e9b82a5cSDiego Caballero } 977e9b82a5cSDiego Caballero 978e9b82a5cSDiego Caballero // Replace `vector.mask` operation altogether. 979e9b82a5cSDiego Caballero rewriter.replaceOp(maskOp, result); 980e9b82a5cSDiego Caballero return success(); 981e9b82a5cSDiego Caballero } 982e9b82a5cSDiego Caballero }; 983e9b82a5cSDiego Caballero 984563879b6SRahul Joshi class VectorShuffleOpConversion 985563879b6SRahul Joshi : public ConvertOpToLLVMPattern<vector::ShuffleOp> { 9861c81adf3SAart Bik public: 987563879b6SRahul Joshi using ConvertOpToLLVMPattern<vector::ShuffleOp>::ConvertOpToLLVMPattern; 9881c81adf3SAart Bik 9893145427dSRiver Riddle LogicalResult 990ef976337SRiver Riddle matchAndRewrite(vector::ShuffleOp shuffleOp, OpAdaptor adaptor, 9911c81adf3SAart Bik ConversionPatternRewriter &rewriter) const override { 992563879b6SRahul Joshi auto loc = shuffleOp->getLoc(); 9931c81adf3SAart Bik auto v1Type = shuffleOp.getV1VectorType(); 9941c81adf3SAart Bik auto v2Type = shuffleOp.getV2VectorType(); 995a1aad28dSLei Zhang auto vectorType = shuffleOp.getResultVectorType(); 996dcec2ca5SChristian Sigg Type llvmType = typeConverter->convertType(vectorType); 997b4444dcaSBenjamin Maxwell ArrayRef<int64_t> mask = shuffleOp.getMask(); 9981c81adf3SAart Bik 9991c81adf3SAart Bik // Bail if result type cannot be lowered. 10001c81adf3SAart Bik if (!llvmType) 10013145427dSRiver Riddle return failure(); 10021c81adf3SAart Bik 10031c81adf3SAart Bik // Get rank and dimension sizes. 10041c81adf3SAart Bik int64_t rank = vectorType.getRank(); 1005fe09cee3SBenjamin Kramer #ifndef NDEBUG 1006db6f8ebeSNicolas Vasilache bool wellFormed0DCase = 1007db6f8ebeSNicolas Vasilache v1Type.getRank() == 0 && v2Type.getRank() == 0 && rank == 1; 1008db6f8ebeSNicolas Vasilache bool wellFormedNDCase = 1009db6f8ebeSNicolas Vasilache v1Type.getRank() == rank && v2Type.getRank() == rank; 1010db6f8ebeSNicolas Vasilache assert((wellFormed0DCase || wellFormedNDCase) && "op is not well-formed"); 1011fe09cee3SBenjamin Kramer #endif 10121c81adf3SAart Bik 1013db6f8ebeSNicolas Vasilache // For rank 0 and 1, where both operands have *exactly* the same vector 1014db6f8ebeSNicolas Vasilache // type, there is direct shuffle support in LLVM. Use it! 1015db6f8ebeSNicolas Vasilache if (rank <= 1 && v1Type == v2Type) { 1016563879b6SRahul Joshi Value llvmShuffleOp = rewriter.create<LLVM::ShuffleVectorOp>( 1017b2ccfb4dSJeff Niu loc, adaptor.getV1(), adaptor.getV2(), 1018b4444dcaSBenjamin Maxwell llvm::to_vector_of<int32_t>(mask)); 1019563879b6SRahul Joshi rewriter.replaceOp(shuffleOp, llvmShuffleOp); 10203145427dSRiver Riddle return success(); 1021b36aaeafSAart Bik } 1022b36aaeafSAart Bik 10231c81adf3SAart Bik // For all other cases, insert the individual values individually. 1024db6f8ebeSNicolas Vasilache int64_t v1Dim = v1Type.getDimSize(0); 10255a8a159bSMehdi Amini Type eltType; 10265550c821STres Popp if (auto arrayType = dyn_cast<LLVM::LLVMArrayType>(llvmType)) 10275a8a159bSMehdi Amini eltType = arrayType.getElementType(); 10285a8a159bSMehdi Amini else 10295550c821STres Popp eltType = cast<VectorType>(llvmType).getElementType(); 1030e62a6956SRiver Riddle Value insert = rewriter.create<LLVM::UndefOp>(loc, llvmType); 10311c81adf3SAart Bik int64_t insPos = 0; 1032b4444dcaSBenjamin Maxwell for (int64_t extPos : mask) { 10337c38fd60SJacques Pienaar Value value = adaptor.getV1(); 10341c81adf3SAart Bik if (extPos >= v1Dim) { 10351c81adf3SAart Bik extPos -= v1Dim; 10367c38fd60SJacques Pienaar value = adaptor.getV2(); 1037b36aaeafSAart Bik } 1038dcec2ca5SChristian Sigg Value extract = extractOne(rewriter, *getTypeConverter(), loc, value, 10395a8a159bSMehdi Amini eltType, rank, extPos); 1040dcec2ca5SChristian Sigg insert = insertOne(rewriter, *getTypeConverter(), loc, insert, extract, 10410f04384dSAlex Zinenko llvmType, rank, insPos++); 10421c81adf3SAart Bik } 1043563879b6SRahul Joshi rewriter.replaceOp(shuffleOp, insert); 10443145427dSRiver Riddle return success(); 1045b36aaeafSAart Bik } 1046b36aaeafSAart Bik }; 1047b36aaeafSAart Bik 1048563879b6SRahul Joshi class VectorExtractElementOpConversion 1049563879b6SRahul Joshi : public ConvertOpToLLVMPattern<vector::ExtractElementOp> { 1050cd5dab8aSAart Bik public: 1051563879b6SRahul Joshi using ConvertOpToLLVMPattern< 1052563879b6SRahul Joshi vector::ExtractElementOp>::ConvertOpToLLVMPattern; 1053cd5dab8aSAart Bik 10543145427dSRiver Riddle LogicalResult 1055ef976337SRiver Riddle matchAndRewrite(vector::ExtractElementOp extractEltOp, OpAdaptor adaptor, 1056cd5dab8aSAart Bik ConversionPatternRewriter &rewriter) const override { 1057a1aad28dSLei Zhang auto vectorType = extractEltOp.getSourceVectorType(); 1058dcec2ca5SChristian Sigg auto llvmType = typeConverter->convertType(vectorType.getElementType()); 1059cd5dab8aSAart Bik 1060cd5dab8aSAart Bik // Bail if result type cannot be lowered. 1061cd5dab8aSAart Bik if (!llvmType) 10623145427dSRiver Riddle return failure(); 1063cd5dab8aSAart Bik 1064e7026abaSNicolas Vasilache if (vectorType.getRank() == 0) { 1065e7026abaSNicolas Vasilache Location loc = extractEltOp.getLoc(); 1066e7026abaSNicolas Vasilache auto idxType = rewriter.getIndexType(); 1067e7026abaSNicolas Vasilache auto zero = rewriter.create<LLVM::ConstantOp>( 1068e7026abaSNicolas Vasilache loc, typeConverter->convertType(idxType), 1069e7026abaSNicolas Vasilache rewriter.getIntegerAttr(idxType, 0)); 1070e7026abaSNicolas Vasilache rewriter.replaceOpWithNewOp<LLVM::ExtractElementOp>( 10717c38fd60SJacques Pienaar extractEltOp, llvmType, adaptor.getVector(), zero); 1072e7026abaSNicolas Vasilache return success(); 1073e7026abaSNicolas Vasilache } 1074e7026abaSNicolas Vasilache 1075cd5dab8aSAart Bik rewriter.replaceOpWithNewOp<LLVM::ExtractElementOp>( 10767c38fd60SJacques Pienaar extractEltOp, llvmType, adaptor.getVector(), adaptor.getPosition()); 10773145427dSRiver Riddle return success(); 1078cd5dab8aSAart Bik } 1079cd5dab8aSAart Bik }; 1080cd5dab8aSAart Bik 1081563879b6SRahul Joshi class VectorExtractOpConversion 1082563879b6SRahul Joshi : public ConvertOpToLLVMPattern<vector::ExtractOp> { 10835c0c51a9SNicolas Vasilache public: 1084563879b6SRahul Joshi using ConvertOpToLLVMPattern<vector::ExtractOp>::ConvertOpToLLVMPattern; 10855c0c51a9SNicolas Vasilache 10863145427dSRiver Riddle LogicalResult 1087ef976337SRiver Riddle matchAndRewrite(vector::ExtractOp extractOp, OpAdaptor adaptor, 10885c0c51a9SNicolas Vasilache ConversionPatternRewriter &rewriter) const override { 1089563879b6SRahul Joshi auto loc = extractOp->getLoc(); 10902bdf33ccSRiver Riddle auto resultType = extractOp.getResult().getType(); 1091dcec2ca5SChristian Sigg auto llvmResultType = typeConverter->convertType(resultType); 10929826fe5cSAart Bik // Bail if result type cannot be lowered. 10939826fe5cSAart Bik if (!llvmResultType) 10943145427dSRiver Riddle return failure(); 10959826fe5cSAart Bik 109642a6ad7bSDiego Caballero SmallVector<OpFoldResult> positionVec = getMixedValues( 109742a6ad7bSDiego Caballero adaptor.getStaticPosition(), adaptor.getDynamicPosition(), rewriter); 109898f6289aSDiego Caballero 1099a8f92716SKunwar Grover // The Vector -> LLVM lowering models N-D vectors as nested aggregates of 1100a8f92716SKunwar Grover // 1-d vectors. This nesting is modeled using arrays. We do this conversion 1101a8f92716SKunwar Grover // from a N-d vector extract to a nested aggregate vector extract in two 1102a8f92716SKunwar Grover // steps: 1103a8f92716SKunwar Grover // - Extract a member from the nested aggregate. The result can be 1104a8f92716SKunwar Grover // a lower rank nested aggregate or a vector (1-D). This is done using 1105a8f92716SKunwar Grover // `llvm.extractvalue`. 1106a8f92716SKunwar Grover // - Extract a scalar out of the vector if needed. This is done using 1107a8f92716SKunwar Grover // `llvm.extractelement`. 1108a8f92716SKunwar Grover 1109a8f92716SKunwar Grover // Determine if we need to extract a member out of the aggregate. We 1110a8f92716SKunwar Grover // always need to extract a member if the input rank >= 2. 1111a8f92716SKunwar Grover bool extractsAggregate = extractOp.getSourceVectorType().getRank() >= 2; 1112a8f92716SKunwar Grover // Determine if we need to extract a scalar as the result. We extract 1113a8f92716SKunwar Grover // a scalar if the extract is full rank, i.e., the number of indices is 1114a8f92716SKunwar Grover // equal to source vector rank. 1115a8f92716SKunwar Grover bool extractsScalar = static_cast<int64_t>(positionVec.size()) == 1116a8f92716SKunwar Grover extractOp.getSourceVectorType().getRank(); 1117a8f92716SKunwar Grover 1118a8f92716SKunwar Grover // Since the LLVM type converter converts 0-d vectors to 1-d vectors, we 1119a8f92716SKunwar Grover // need to add a position for this change. 1120a8f92716SKunwar Grover if (extractOp.getSourceVectorType().getRank() == 0) { 1121a8f92716SKunwar Grover Type idxType = typeConverter->convertType(rewriter.getIndexType()); 1122a8f92716SKunwar Grover positionVec.push_back(rewriter.getZeroAttr(idxType)); 1123864adf39SMatthias Springer } 1124864adf39SMatthias Springer 11257c38fd60SJacques Pienaar Value extracted = adaptor.getVector(); 1126a8f92716SKunwar Grover if (extractsAggregate) { 1127a8f92716SKunwar Grover ArrayRef<OpFoldResult> position(positionVec); 1128a8f92716SKunwar Grover if (extractsScalar) { 1129a8f92716SKunwar Grover // If we are extracting a scalar from the extracted member, we drop 1130a8f92716SKunwar Grover // the last index, which will be used to extract the scalar out of the 1131a8f92716SKunwar Grover // vector. 1132a8f92716SKunwar Grover position = position.drop_back(); 1133a8f92716SKunwar Grover } 1134a8f92716SKunwar Grover // llvm.extractvalue does not support dynamic dimensions. 1135a8f92716SKunwar Grover if (!llvm::all_of(position, llvm::IsaPred<Attribute>)) { 113698f6289aSDiego Caballero return failure(); 1137a8f92716SKunwar Grover } 1138a8f92716SKunwar Grover extracted = rewriter.create<LLVM::ExtractValueOp>( 1139a8f92716SKunwar Grover loc, extracted, getAsIntegers(position)); 11405c0c51a9SNicolas Vasilache } 11415c0c51a9SNicolas Vasilache 1142a8f92716SKunwar Grover if (extractsScalar) { 1143a8f92716SKunwar Grover extracted = rewriter.create<LLVM::ExtractElementOp>( 1144a8f92716SKunwar Grover loc, extracted, getAsLLVMValue(rewriter, loc, positionVec.back())); 1145a8f92716SKunwar Grover } 1146a8f92716SKunwar Grover 1147a8f92716SKunwar Grover rewriter.replaceOp(extractOp, extracted); 11483145427dSRiver Riddle return success(); 11495c0c51a9SNicolas Vasilache } 11505c0c51a9SNicolas Vasilache }; 11515c0c51a9SNicolas Vasilache 1152681f929fSNicolas Vasilache /// Conversion pattern that turns a vector.fma on a 1-D vector 1153681f929fSNicolas Vasilache /// into an llvm.intr.fmuladd. This is a trivial 1-1 conversion. 1154681f929fSNicolas Vasilache /// This does not match vectors of n >= 2 rank. 1155681f929fSNicolas Vasilache /// 1156681f929fSNicolas Vasilache /// Example: 1157681f929fSNicolas Vasilache /// ``` 1158681f929fSNicolas Vasilache /// vector.fma %a, %a, %a : vector<8xf32> 1159681f929fSNicolas Vasilache /// ``` 1160681f929fSNicolas Vasilache /// is converted to: 1161681f929fSNicolas Vasilache /// ``` 11623bffe602SBenjamin Kramer /// llvm.intr.fmuladd %va, %va, %va: 1163dd5165a9SAlex Zinenko /// (!llvm."<8 x f32>">, !llvm<"<8 x f32>">, !llvm<"<8 x f32>">) 1164dd5165a9SAlex Zinenko /// -> !llvm."<8 x f32>"> 1165681f929fSNicolas Vasilache /// ``` 1166563879b6SRahul Joshi class VectorFMAOp1DConversion : public ConvertOpToLLVMPattern<vector::FMAOp> { 1167681f929fSNicolas Vasilache public: 1168563879b6SRahul Joshi using ConvertOpToLLVMPattern<vector::FMAOp>::ConvertOpToLLVMPattern; 1169681f929fSNicolas Vasilache 11703145427dSRiver Riddle LogicalResult 1171ef976337SRiver Riddle matchAndRewrite(vector::FMAOp fmaOp, OpAdaptor adaptor, 1172681f929fSNicolas Vasilache ConversionPatternRewriter &rewriter) const override { 1173681f929fSNicolas Vasilache VectorType vType = fmaOp.getVectorType(); 1174ab45a432SMichal Terepeta if (vType.getRank() > 1) 11753145427dSRiver Riddle return failure(); 11769452356dSDiego Caballero 11777c38fd60SJacques Pienaar rewriter.replaceOpWithNewOp<LLVM::FMulAddOp>( 11787c38fd60SJacques Pienaar fmaOp, adaptor.getLhs(), adaptor.getRhs(), adaptor.getAcc()); 11793145427dSRiver Riddle return success(); 1180681f929fSNicolas Vasilache } 1181681f929fSNicolas Vasilache }; 1182681f929fSNicolas Vasilache 1183563879b6SRahul Joshi class VectorInsertElementOpConversion 1184563879b6SRahul Joshi : public ConvertOpToLLVMPattern<vector::InsertElementOp> { 1185cd5dab8aSAart Bik public: 1186563879b6SRahul Joshi using ConvertOpToLLVMPattern<vector::InsertElementOp>::ConvertOpToLLVMPattern; 1187cd5dab8aSAart Bik 11883145427dSRiver Riddle LogicalResult 1189ef976337SRiver Riddle matchAndRewrite(vector::InsertElementOp insertEltOp, OpAdaptor adaptor, 1190cd5dab8aSAart Bik ConversionPatternRewriter &rewriter) const override { 1191cd5dab8aSAart Bik auto vectorType = insertEltOp.getDestVectorType(); 1192dcec2ca5SChristian Sigg auto llvmType = typeConverter->convertType(vectorType); 1193cd5dab8aSAart Bik 1194cd5dab8aSAart Bik // Bail if result type cannot be lowered. 1195cd5dab8aSAart Bik if (!llvmType) 11963145427dSRiver Riddle return failure(); 1197cd5dab8aSAart Bik 11983ff4e5f2SNicolas Vasilache if (vectorType.getRank() == 0) { 11993ff4e5f2SNicolas Vasilache Location loc = insertEltOp.getLoc(); 12003ff4e5f2SNicolas Vasilache auto idxType = rewriter.getIndexType(); 12013ff4e5f2SNicolas Vasilache auto zero = rewriter.create<LLVM::ConstantOp>( 12023ff4e5f2SNicolas Vasilache loc, typeConverter->convertType(idxType), 12033ff4e5f2SNicolas Vasilache rewriter.getIntegerAttr(idxType, 0)); 12043ff4e5f2SNicolas Vasilache rewriter.replaceOpWithNewOp<LLVM::InsertElementOp>( 12057c38fd60SJacques Pienaar insertEltOp, llvmType, adaptor.getDest(), adaptor.getSource(), zero); 12063ff4e5f2SNicolas Vasilache return success(); 12073ff4e5f2SNicolas Vasilache } 12083ff4e5f2SNicolas Vasilache 1209cd5dab8aSAart Bik rewriter.replaceOpWithNewOp<LLVM::InsertElementOp>( 12107c38fd60SJacques Pienaar insertEltOp, llvmType, adaptor.getDest(), adaptor.getSource(), 12117c38fd60SJacques Pienaar adaptor.getPosition()); 12123145427dSRiver Riddle return success(); 1213cd5dab8aSAart Bik } 1214cd5dab8aSAart Bik }; 1215cd5dab8aSAart Bik 1216563879b6SRahul Joshi class VectorInsertOpConversion 1217563879b6SRahul Joshi : public ConvertOpToLLVMPattern<vector::InsertOp> { 12189826fe5cSAart Bik public: 1219563879b6SRahul Joshi using ConvertOpToLLVMPattern<vector::InsertOp>::ConvertOpToLLVMPattern; 12209826fe5cSAart Bik 12213145427dSRiver Riddle LogicalResult 1222ef976337SRiver Riddle matchAndRewrite(vector::InsertOp insertOp, OpAdaptor adaptor, 12239826fe5cSAart Bik ConversionPatternRewriter &rewriter) const override { 1224563879b6SRahul Joshi auto loc = insertOp->getLoc(); 12259826fe5cSAart Bik auto sourceType = insertOp.getSourceType(); 12269826fe5cSAart Bik auto destVectorType = insertOp.getDestVectorType(); 1227dcec2ca5SChristian Sigg auto llvmResultType = typeConverter->convertType(destVectorType); 12289826fe5cSAart Bik // Bail if result type cannot be lowered. 12299826fe5cSAart Bik if (!llvmResultType) 12303145427dSRiver Riddle return failure(); 12319826fe5cSAart Bik 123242a6ad7bSDiego Caballero SmallVector<OpFoldResult> positionVec = getMixedValues( 123342a6ad7bSDiego Caballero adaptor.getStaticPosition(), adaptor.getDynamicPosition(), rewriter); 123498f6289aSDiego Caballero 1235864adf39SMatthias Springer // Overwrite entire vector with value. Should be handled by folder, but 1236864adf39SMatthias Springer // just to be safe. 123798f6289aSDiego Caballero ArrayRef<OpFoldResult> position(positionVec); 123898f6289aSDiego Caballero if (position.empty()) { 12397c38fd60SJacques Pienaar rewriter.replaceOp(insertOp, adaptor.getSource()); 1240864adf39SMatthias Springer return success(); 1241864adf39SMatthias Springer } 1242864adf39SMatthias Springer 12439826fe5cSAart Bik // One-shot insertion of a vector into an array (only requires insertvalue). 12445550c821STres Popp if (isa<VectorType>(sourceType)) { 124598f6289aSDiego Caballero if (insertOp.hasDynamicPosition()) 124698f6289aSDiego Caballero return failure(); 124798f6289aSDiego Caballero 1248e62a6956SRiver Riddle Value inserted = rewriter.create<LLVM::InsertValueOp>( 124998f6289aSDiego Caballero loc, adaptor.getDest(), adaptor.getSource(), getAsIntegers(position)); 1250563879b6SRahul Joshi rewriter.replaceOp(insertOp, inserted); 12513145427dSRiver Riddle return success(); 12529826fe5cSAart Bik } 12539826fe5cSAart Bik 12549826fe5cSAart Bik // Potential extraction of 1-D vector from array. 12557c38fd60SJacques Pienaar Value extracted = adaptor.getDest(); 12569826fe5cSAart Bik auto oneDVectorType = destVectorType; 125798f6289aSDiego Caballero if (position.size() > 1) { 125898f6289aSDiego Caballero if (insertOp.hasDynamicPosition()) 125998f6289aSDiego Caballero return failure(); 126098f6289aSDiego Caballero 12619826fe5cSAart Bik oneDVectorType = reducedVectorTypeBack(destVectorType); 12629826fe5cSAart Bik extracted = rewriter.create<LLVM::ExtractValueOp>( 126398f6289aSDiego Caballero loc, extracted, getAsIntegers(position.drop_back())); 12649826fe5cSAart Bik } 12659826fe5cSAart Bik 12669826fe5cSAart Bik // Insertion of an element into a 1-D LLVM vector. 1267e62a6956SRiver Riddle Value inserted = rewriter.create<LLVM::InsertElementOp>( 1268dcec2ca5SChristian Sigg loc, typeConverter->convertType(oneDVectorType), extracted, 126998f6289aSDiego Caballero adaptor.getSource(), getAsLLVMValue(rewriter, loc, position.back())); 12709826fe5cSAart Bik 12719826fe5cSAart Bik // Potential insertion of resulting 1-D vector into array. 127298f6289aSDiego Caballero if (position.size() > 1) { 127398f6289aSDiego Caballero if (insertOp.hasDynamicPosition()) 127498f6289aSDiego Caballero return failure(); 127598f6289aSDiego Caballero 12767c38fd60SJacques Pienaar inserted = rewriter.create<LLVM::InsertValueOp>( 127798f6289aSDiego Caballero loc, adaptor.getDest(), inserted, 127898f6289aSDiego Caballero getAsIntegers(position.drop_back())); 12799826fe5cSAart Bik } 12809826fe5cSAart Bik 1281563879b6SRahul Joshi rewriter.replaceOp(insertOp, inserted); 12823145427dSRiver Riddle return success(); 12839826fe5cSAart Bik } 12849826fe5cSAart Bik }; 12859826fe5cSAart Bik 1286aa9647e2SJavier Setoain /// Lower vector.scalable.insert ops to LLVM vector.insert 1287aa9647e2SJavier Setoain struct VectorScalableInsertOpLowering 1288aa9647e2SJavier Setoain : public ConvertOpToLLVMPattern<vector::ScalableInsertOp> { 1289aa9647e2SJavier Setoain using ConvertOpToLLVMPattern< 1290aa9647e2SJavier Setoain vector::ScalableInsertOp>::ConvertOpToLLVMPattern; 1291aa9647e2SJavier Setoain 1292aa9647e2SJavier Setoain LogicalResult 1293aa9647e2SJavier Setoain matchAndRewrite(vector::ScalableInsertOp insOp, OpAdaptor adaptor, 1294aa9647e2SJavier Setoain ConversionPatternRewriter &rewriter) const override { 1295aa9647e2SJavier Setoain rewriter.replaceOpWithNewOp<LLVM::vector_insert>( 1296dbb86433SBenjamin Maxwell insOp, adaptor.getDest(), adaptor.getSource(), adaptor.getPos()); 1297aa9647e2SJavier Setoain return success(); 1298aa9647e2SJavier Setoain } 1299aa9647e2SJavier Setoain }; 1300aa9647e2SJavier Setoain 1301aa9647e2SJavier Setoain /// Lower vector.scalable.extract ops to LLVM vector.extract 1302aa9647e2SJavier Setoain struct VectorScalableExtractOpLowering 1303aa9647e2SJavier Setoain : public ConvertOpToLLVMPattern<vector::ScalableExtractOp> { 1304aa9647e2SJavier Setoain using ConvertOpToLLVMPattern< 1305aa9647e2SJavier Setoain vector::ScalableExtractOp>::ConvertOpToLLVMPattern; 1306aa9647e2SJavier Setoain 1307aa9647e2SJavier Setoain LogicalResult 1308aa9647e2SJavier Setoain matchAndRewrite(vector::ScalableExtractOp extOp, OpAdaptor adaptor, 1309aa9647e2SJavier Setoain ConversionPatternRewriter &rewriter) const override { 1310aa9647e2SJavier Setoain rewriter.replaceOpWithNewOp<LLVM::vector_extract>( 1311aa9647e2SJavier Setoain extOp, typeConverter->convertType(extOp.getResultVectorType()), 1312aa9647e2SJavier Setoain adaptor.getSource(), adaptor.getPos()); 1313aa9647e2SJavier Setoain return success(); 1314aa9647e2SJavier Setoain } 1315aa9647e2SJavier Setoain }; 1316aa9647e2SJavier Setoain 1317681f929fSNicolas Vasilache /// Rank reducing rewrite for n-D FMA into (n-1)-D FMA where n > 1. 1318681f929fSNicolas Vasilache /// 1319681f929fSNicolas Vasilache /// Example: 1320681f929fSNicolas Vasilache /// ``` 1321681f929fSNicolas Vasilache /// %d = vector.fma %a, %b, %c : vector<2x4xf32> 1322681f929fSNicolas Vasilache /// ``` 1323681f929fSNicolas Vasilache /// is rewritten into: 1324681f929fSNicolas Vasilache /// ``` 1325681f929fSNicolas Vasilache /// %r = splat %f0: vector<2x4xf32> 1326681f929fSNicolas Vasilache /// %va = vector.extractvalue %a[0] : vector<2x4xf32> 1327681f929fSNicolas Vasilache /// %vb = vector.extractvalue %b[0] : vector<2x4xf32> 1328681f929fSNicolas Vasilache /// %vc = vector.extractvalue %c[0] : vector<2x4xf32> 1329681f929fSNicolas Vasilache /// %vd = vector.fma %va, %vb, %vc : vector<4xf32> 1330681f929fSNicolas Vasilache /// %r2 = vector.insertvalue %vd, %r[0] : vector<4xf32> into vector<2x4xf32> 1331681f929fSNicolas Vasilache /// %va2 = vector.extractvalue %a2[1] : vector<2x4xf32> 1332681f929fSNicolas Vasilache /// %vb2 = vector.extractvalue %b2[1] : vector<2x4xf32> 1333681f929fSNicolas Vasilache /// %vc2 = vector.extractvalue %c2[1] : vector<2x4xf32> 1334681f929fSNicolas Vasilache /// %vd2 = vector.fma %va2, %vb2, %vc2 : vector<4xf32> 1335681f929fSNicolas Vasilache /// %r3 = vector.insertvalue %vd2, %r2[1] : vector<4xf32> into vector<2x4xf32> 1336681f929fSNicolas Vasilache /// // %r3 holds the final value. 1337681f929fSNicolas Vasilache /// ``` 1338681f929fSNicolas Vasilache class VectorFMAOpNDRewritePattern : public OpRewritePattern<FMAOp> { 1339681f929fSNicolas Vasilache public: 1340681f929fSNicolas Vasilache using OpRewritePattern<FMAOp>::OpRewritePattern; 1341681f929fSNicolas Vasilache 1342ee80ffbfSNicolas Vasilache void initialize() { 1343ee80ffbfSNicolas Vasilache // This pattern recursively unpacks one dimension at a time. The recursion 1344ee80ffbfSNicolas Vasilache // bounded as the rank is strictly decreasing. 1345ee80ffbfSNicolas Vasilache setHasBoundedRewriteRecursion(); 1346ee80ffbfSNicolas Vasilache } 1347ee80ffbfSNicolas Vasilache 13483145427dSRiver Riddle LogicalResult matchAndRewrite(FMAOp op, 1349681f929fSNicolas Vasilache PatternRewriter &rewriter) const override { 1350681f929fSNicolas Vasilache auto vType = op.getVectorType(); 1351681f929fSNicolas Vasilache if (vType.getRank() < 2) 13523145427dSRiver Riddle return failure(); 1353681f929fSNicolas Vasilache 1354681f929fSNicolas Vasilache auto loc = op.getLoc(); 1355681f929fSNicolas Vasilache auto elemType = vType.getElementType(); 1356a54f4eaeSMogball Value zero = rewriter.create<arith::ConstantOp>( 1357a54f4eaeSMogball loc, elemType, rewriter.getZeroAttr(elemType)); 13586a8ba318SRiver Riddle Value desc = rewriter.create<vector::SplatOp>(loc, vType, zero); 1359681f929fSNicolas Vasilache for (int64_t i = 0, e = vType.getShape().front(); i != e; ++i) { 13607c38fd60SJacques Pienaar Value extrLHS = rewriter.create<ExtractOp>(loc, op.getLhs(), i); 13617c38fd60SJacques Pienaar Value extrRHS = rewriter.create<ExtractOp>(loc, op.getRhs(), i); 13627c38fd60SJacques Pienaar Value extrACC = rewriter.create<ExtractOp>(loc, op.getAcc(), i); 1363681f929fSNicolas Vasilache Value fma = rewriter.create<FMAOp>(loc, extrLHS, extrRHS, extrACC); 1364681f929fSNicolas Vasilache desc = rewriter.create<InsertOp>(loc, fma, desc, i); 1365681f929fSNicolas Vasilache } 1366681f929fSNicolas Vasilache rewriter.replaceOp(op, desc); 13673145427dSRiver Riddle return success(); 1368681f929fSNicolas Vasilache } 1369681f929fSNicolas Vasilache }; 1370681f929fSNicolas Vasilache 137130e6033bSNicolas Vasilache /// Returns the strides if the memory underlying `memRefType` has a contiguous 137230e6033bSNicolas Vasilache /// static layout. 1373a9a1b45bSKazu Hirata static std::optional<SmallVector<int64_t, 4>> 137430e6033bSNicolas Vasilache computeContiguousStrides(MemRefType memRefType) { 13752bf491c7SBenjamin Kramer int64_t offset; 137630e6033bSNicolas Vasilache SmallVector<int64_t, 4> strides; 13776aaa8f25SMatthias Springer if (failed(memRefType.getStridesAndOffset(strides, offset))) 13781a36588eSKazu Hirata return std::nullopt; 137930e6033bSNicolas Vasilache if (!strides.empty() && strides.back() != 1) 13801a36588eSKazu Hirata return std::nullopt; 138130e6033bSNicolas Vasilache // If no layout or identity layout, this is contiguous by definition. 1382e41ebbecSVladislav Vinogradov if (memRefType.getLayout().isIdentity()) 138330e6033bSNicolas Vasilache return strides; 138430e6033bSNicolas Vasilache 138530e6033bSNicolas Vasilache // Otherwise, we must determine contiguity form shapes. This can only ever 138630e6033bSNicolas Vasilache // work in static cases because MemRefType is underspecified to represent 138730e6033bSNicolas Vasilache // contiguous dynamic shapes in other ways than with just empty/identity 138830e6033bSNicolas Vasilache // layout. 13892bf491c7SBenjamin Kramer auto sizes = memRefType.getShape(); 13905017b0f8SMatthias Springer for (int index = 0, e = strides.size() - 1; index < e; ++index) { 139130e6033bSNicolas Vasilache if (ShapedType::isDynamic(sizes[index + 1]) || 1392399638f9SAliia Khasanova ShapedType::isDynamic(strides[index]) || 1393399638f9SAliia Khasanova ShapedType::isDynamic(strides[index + 1])) 13941a36588eSKazu Hirata return std::nullopt; 139530e6033bSNicolas Vasilache if (strides[index] != strides[index + 1] * sizes[index + 1]) 13961a36588eSKazu Hirata return std::nullopt; 13972bf491c7SBenjamin Kramer } 139830e6033bSNicolas Vasilache return strides; 13992bf491c7SBenjamin Kramer } 14002bf491c7SBenjamin Kramer 1401563879b6SRahul Joshi class VectorTypeCastOpConversion 1402563879b6SRahul Joshi : public ConvertOpToLLVMPattern<vector::TypeCastOp> { 14035c0c51a9SNicolas Vasilache public: 1404563879b6SRahul Joshi using ConvertOpToLLVMPattern<vector::TypeCastOp>::ConvertOpToLLVMPattern; 14055c0c51a9SNicolas Vasilache 14063145427dSRiver Riddle LogicalResult 1407ef976337SRiver Riddle matchAndRewrite(vector::TypeCastOp castOp, OpAdaptor adaptor, 14085c0c51a9SNicolas Vasilache ConversionPatternRewriter &rewriter) const override { 1409563879b6SRahul Joshi auto loc = castOp->getLoc(); 14105c0c51a9SNicolas Vasilache MemRefType sourceMemRefType = 14115550c821STres Popp cast<MemRefType>(castOp.getOperand().getType()); 14129eb3e564SChris Lattner MemRefType targetMemRefType = castOp.getType(); 14135c0c51a9SNicolas Vasilache 14145c0c51a9SNicolas Vasilache // Only static shape casts supported atm. 14155c0c51a9SNicolas Vasilache if (!sourceMemRefType.hasStaticShape() || 14165c0c51a9SNicolas Vasilache !targetMemRefType.hasStaticShape()) 14173145427dSRiver Riddle return failure(); 14185c0c51a9SNicolas Vasilache 14195c0c51a9SNicolas Vasilache auto llvmSourceDescriptorTy = 14205550c821STres Popp dyn_cast<LLVM::LLVMStructType>(adaptor.getOperands()[0].getType()); 14218de43b92SAlex Zinenko if (!llvmSourceDescriptorTy) 14223145427dSRiver Riddle return failure(); 1423ef976337SRiver Riddle MemRefDescriptor sourceMemRef(adaptor.getOperands()[0]); 14245c0c51a9SNicolas Vasilache 14255550c821STres Popp auto llvmTargetDescriptorTy = dyn_cast_or_null<LLVM::LLVMStructType>( 14265550c821STres Popp typeConverter->convertType(targetMemRefType)); 14278de43b92SAlex Zinenko if (!llvmTargetDescriptorTy) 14283145427dSRiver Riddle return failure(); 14295c0c51a9SNicolas Vasilache 143030e6033bSNicolas Vasilache // Only contiguous source buffers supported atm. 143130e6033bSNicolas Vasilache auto sourceStrides = computeContiguousStrides(sourceMemRefType); 143230e6033bSNicolas Vasilache if (!sourceStrides) 143330e6033bSNicolas Vasilache return failure(); 143430e6033bSNicolas Vasilache auto targetStrides = computeContiguousStrides(targetMemRefType); 143530e6033bSNicolas Vasilache if (!targetStrides) 143630e6033bSNicolas Vasilache return failure(); 143730e6033bSNicolas Vasilache // Only support static strides for now, regardless of contiguity. 1438399638f9SAliia Khasanova if (llvm::any_of(*targetStrides, ShapedType::isDynamic)) 14393145427dSRiver Riddle return failure(); 14405c0c51a9SNicolas Vasilache 14412230bf99SAlex Zinenko auto int64Ty = IntegerType::get(rewriter.getContext(), 64); 14425c0c51a9SNicolas Vasilache 14435c0c51a9SNicolas Vasilache // Create descriptor. 14445c0c51a9SNicolas Vasilache auto desc = MemRefDescriptor::undef(rewriter, loc, llvmTargetDescriptorTy); 14455c0c51a9SNicolas Vasilache // Set allocated ptr. 1446e62a6956SRiver Riddle Value allocated = sourceMemRef.allocatedPtr(rewriter, loc); 14475c0c51a9SNicolas Vasilache desc.setAllocatedPtr(rewriter, loc, allocated); 14484a2d4588SMarkus Böck 14495c0c51a9SNicolas Vasilache // Set aligned ptr. 1450e62a6956SRiver Riddle Value ptr = sourceMemRef.alignedPtr(rewriter, loc); 14515c0c51a9SNicolas Vasilache desc.setAlignedPtr(rewriter, loc, ptr); 14525c0c51a9SNicolas Vasilache // Fill offset 0. 14535c0c51a9SNicolas Vasilache auto attr = rewriter.getIntegerAttr(rewriter.getIndexType(), 0); 14545c0c51a9SNicolas Vasilache auto zero = rewriter.create<LLVM::ConstantOp>(loc, int64Ty, attr); 14555c0c51a9SNicolas Vasilache desc.setOffset(rewriter, loc, zero); 14565c0c51a9SNicolas Vasilache 14575c0c51a9SNicolas Vasilache // Fill size and stride descriptors in memref. 1458e4853be2SMehdi Amini for (const auto &indexedSize : 1459e4853be2SMehdi Amini llvm::enumerate(targetMemRefType.getShape())) { 14605c0c51a9SNicolas Vasilache int64_t index = indexedSize.index(); 14615c0c51a9SNicolas Vasilache auto sizeAttr = 14625c0c51a9SNicolas Vasilache rewriter.getIntegerAttr(rewriter.getIndexType(), indexedSize.value()); 14635c0c51a9SNicolas Vasilache auto size = rewriter.create<LLVM::ConstantOp>(loc, int64Ty, sizeAttr); 14645c0c51a9SNicolas Vasilache desc.setSize(rewriter, loc, index, size); 146530e6033bSNicolas Vasilache auto strideAttr = rewriter.getIntegerAttr(rewriter.getIndexType(), 146630e6033bSNicolas Vasilache (*targetStrides)[index]); 14675c0c51a9SNicolas Vasilache auto stride = rewriter.create<LLVM::ConstantOp>(loc, int64Ty, strideAttr); 14685c0c51a9SNicolas Vasilache desc.setStride(rewriter, loc, index, stride); 14695c0c51a9SNicolas Vasilache } 14705c0c51a9SNicolas Vasilache 1471563879b6SRahul Joshi rewriter.replaceOp(castOp, {desc}); 14723145427dSRiver Riddle return success(); 14735c0c51a9SNicolas Vasilache } 14745c0c51a9SNicolas Vasilache }; 14755c0c51a9SNicolas Vasilache 1476a75a46dbSJavier Setoain /// Conversion pattern for a `vector.create_mask` (1-D scalable vectors only). 1477a75a46dbSJavier Setoain /// Non-scalable versions of this operation are handled in Vector Transforms. 14780693b9e9SMatthias Springer class VectorCreateMaskOpConversion 14790693b9e9SMatthias Springer : public OpConversionPattern<vector::CreateMaskOp> { 1480a75a46dbSJavier Setoain public: 14810693b9e9SMatthias Springer explicit VectorCreateMaskOpConversion(MLIRContext *context, 1482a75a46dbSJavier Setoain bool enableIndexOpt) 14830693b9e9SMatthias Springer : OpConversionPattern<vector::CreateMaskOp>(context), 14847bc8ad51SJavier Setoain force32BitVectorIndices(enableIndexOpt) {} 1485a75a46dbSJavier Setoain 14860693b9e9SMatthias Springer LogicalResult 14870693b9e9SMatthias Springer matchAndRewrite(vector::CreateMaskOp op, OpAdaptor adaptor, 14880693b9e9SMatthias Springer ConversionPatternRewriter &rewriter) const override { 1489a75a46dbSJavier Setoain auto dstType = op.getType(); 14905550c821STres Popp if (dstType.getRank() != 1 || !cast<VectorType>(dstType).isScalable()) 1491a75a46dbSJavier Setoain return failure(); 1492a75a46dbSJavier Setoain IntegerType idxType = 14937bc8ad51SJavier Setoain force32BitVectorIndices ? rewriter.getI32Type() : rewriter.getI64Type(); 1494a75a46dbSJavier Setoain auto loc = op->getLoc(); 1495a75a46dbSJavier Setoain Value indices = rewriter.create<LLVM::StepVectorOp>( 1496a75a46dbSJavier Setoain loc, LLVM::getVectorType(idxType, dstType.getShape()[0], 1497a75a46dbSJavier Setoain /*isScalable=*/true)); 1498a75a46dbSJavier Setoain auto bound = getValueOrCreateCastToIndexLike(rewriter, loc, idxType, 14990693b9e9SMatthias Springer adaptor.getOperands()[0]); 1500a75a46dbSJavier Setoain Value bounds = rewriter.create<SplatOp>(loc, indices.getType(), bound); 1501a75a46dbSJavier Setoain Value comp = rewriter.create<arith::CmpIOp>(loc, arith::CmpIPredicate::slt, 1502a75a46dbSJavier Setoain indices, bounds); 1503a75a46dbSJavier Setoain rewriter.replaceOp(op, comp); 1504a75a46dbSJavier Setoain return success(); 1505a75a46dbSJavier Setoain } 1506a75a46dbSJavier Setoain 1507a75a46dbSJavier Setoain private: 15087bc8ad51SJavier Setoain const bool force32BitVectorIndices; 1509a75a46dbSJavier Setoain }; 1510a75a46dbSJavier Setoain 1511563879b6SRahul Joshi class VectorPrintOpConversion : public ConvertOpToLLVMPattern<vector::PrintOp> { 1512d9b500d3SAart Bik public: 1513563879b6SRahul Joshi using ConvertOpToLLVMPattern<vector::PrintOp>::ConvertOpToLLVMPattern; 1514d9b500d3SAart Bik 1515f36e909dSBenjamin Maxwell // Lowering implementation that relies on a small runtime support library, 1516f36e909dSBenjamin Maxwell // which only needs to provide a few printing methods (single value for all 1517f36e909dSBenjamin Maxwell // data types, opening/closing bracket, comma, newline). The lowering splits 1518f36e909dSBenjamin Maxwell // the vector into elementary printing operations. The advantage of this 1519f36e909dSBenjamin Maxwell // approach is that the library can remain unaware of all low-level 1520f36e909dSBenjamin Maxwell // implementation details of vectors while still supporting output of any 1521f36e909dSBenjamin Maxwell // shaped and dimensioned vector. 1522f36e909dSBenjamin Maxwell // 1523f36e909dSBenjamin Maxwell // Note: This lowering only handles scalars, n-D vectors are broken into 1524f36e909dSBenjamin Maxwell // printing scalars in loops in VectorToSCF. 1525d9b500d3SAart Bik // 15269db53a18SRiver Riddle // TODO: rely solely on libc in future? something else? 1527d9b500d3SAart Bik // 15283145427dSRiver Riddle LogicalResult 1529ef976337SRiver Riddle matchAndRewrite(vector::PrintOp printOp, OpAdaptor adaptor, 1530d9b500d3SAart Bik ConversionPatternRewriter &rewriter) const override { 1531f36e909dSBenjamin Maxwell auto parent = printOp->getParentOfType<ModuleOp>(); 1532f36e909dSBenjamin Maxwell if (!parent) 1533f36e909dSBenjamin Maxwell return failure(); 1534490dae26SBenjamin Maxwell 1535f36e909dSBenjamin Maxwell auto loc = printOp->getLoc(); 1536f36e909dSBenjamin Maxwell 1537f36e909dSBenjamin Maxwell if (auto value = adaptor.getSource()) { 1538f36e909dSBenjamin Maxwell Type printType = printOp.getPrintType(); 1539f36e909dSBenjamin Maxwell if (isa<VectorType>(printType)) { 1540f36e909dSBenjamin Maxwell // Vectors should be broken into elementary print ops in VectorToSCF. 1541f36e909dSBenjamin Maxwell return failure(); 1542f36e909dSBenjamin Maxwell } 1543f36e909dSBenjamin Maxwell if (failed(emitScalarPrint(rewriter, parent, loc, printType, value))) 1544f36e909dSBenjamin Maxwell return failure(); 1545f36e909dSBenjamin Maxwell } 1546f36e909dSBenjamin Maxwell 1547f36e909dSBenjamin Maxwell auto punct = printOp.getPunctuation(); 15483be3883eSBenjamin Maxwell if (auto stringLiteral = printOp.getStringLiteral()) { 1549*e84f6b6aSLuohao Wang auto createResult = 15503be3883eSBenjamin Maxwell LLVM::createPrintStrCall(rewriter, loc, parent, "vector_print_str", 1551c1b8c6cfSAart Bik *stringLiteral, *getTypeConverter(), 1552c1b8c6cfSAart Bik /*addNewline=*/false); 1553*e84f6b6aSLuohao Wang if (createResult.failed()) 1554*e84f6b6aSLuohao Wang return failure(); 1555*e84f6b6aSLuohao Wang 15563be3883eSBenjamin Maxwell } else if (punct != PrintPunctuation::NoPunctuation) { 1557*e84f6b6aSLuohao Wang FailureOr<LLVM::LLVMFuncOp> op = [&]() { 1558f36e909dSBenjamin Maxwell switch (punct) { 1559f36e909dSBenjamin Maxwell case PrintPunctuation::Close: 1560f36e909dSBenjamin Maxwell return LLVM::lookupOrCreatePrintCloseFn(parent); 1561f36e909dSBenjamin Maxwell case PrintPunctuation::Open: 1562f36e909dSBenjamin Maxwell return LLVM::lookupOrCreatePrintOpenFn(parent); 1563f36e909dSBenjamin Maxwell case PrintPunctuation::Comma: 1564f36e909dSBenjamin Maxwell return LLVM::lookupOrCreatePrintCommaFn(parent); 1565f36e909dSBenjamin Maxwell case PrintPunctuation::NewLine: 1566f36e909dSBenjamin Maxwell return LLVM::lookupOrCreatePrintNewlineFn(parent); 1567f36e909dSBenjamin Maxwell default: 1568f36e909dSBenjamin Maxwell llvm_unreachable("unexpected punctuation"); 1569f36e909dSBenjamin Maxwell } 1570*e84f6b6aSLuohao Wang }(); 1571*e84f6b6aSLuohao Wang if (failed(op)) 1572*e84f6b6aSLuohao Wang return failure(); 1573*e84f6b6aSLuohao Wang emitCall(rewriter, printOp->getLoc(), op.value()); 1574f36e909dSBenjamin Maxwell } 1575f36e909dSBenjamin Maxwell 1576f36e909dSBenjamin Maxwell rewriter.eraseOp(printOp); 1577f36e909dSBenjamin Maxwell return success(); 1578f36e909dSBenjamin Maxwell } 1579f36e909dSBenjamin Maxwell 1580f36e909dSBenjamin Maxwell private: 1581f36e909dSBenjamin Maxwell enum class PrintConversion { 1582f36e909dSBenjamin Maxwell // clang-format off 1583f36e909dSBenjamin Maxwell None, 1584f36e909dSBenjamin Maxwell ZeroExt64, 1585f36e909dSBenjamin Maxwell SignExt64, 1586f36e909dSBenjamin Maxwell Bitcast16 1587f36e909dSBenjamin Maxwell // clang-format on 1588f36e909dSBenjamin Maxwell }; 1589f36e909dSBenjamin Maxwell 1590f36e909dSBenjamin Maxwell LogicalResult emitScalarPrint(ConversionPatternRewriter &rewriter, 1591f36e909dSBenjamin Maxwell ModuleOp parent, Location loc, Type printType, 1592f36e909dSBenjamin Maxwell Value value) const { 1593dcec2ca5SChristian Sigg if (typeConverter->convertType(printType) == nullptr) 15943145427dSRiver Riddle return failure(); 1595d9b500d3SAart Bik 1596b8880f5fSAart Bik // Make sure element type has runtime support. 1597b8880f5fSAart Bik PrintConversion conversion = PrintConversion::None; 1598*e84f6b6aSLuohao Wang FailureOr<Operation *> printer; 1599f36e909dSBenjamin Maxwell if (printType.isF32()) { 1600657f60a0SAart Bik printer = LLVM::lookupOrCreatePrintF32Fn(parent); 1601f36e909dSBenjamin Maxwell } else if (printType.isF64()) { 1602657f60a0SAart Bik printer = LLVM::lookupOrCreatePrintF64Fn(parent); 1603f36e909dSBenjamin Maxwell } else if (printType.isF16()) { 1604657f60a0SAart Bik conversion = PrintConversion::Bitcast16; // bits! 1605657f60a0SAart Bik printer = LLVM::lookupOrCreatePrintF16Fn(parent); 1606f36e909dSBenjamin Maxwell } else if (printType.isBF16()) { 1607657f60a0SAart Bik conversion = PrintConversion::Bitcast16; // bits! 1608657f60a0SAart Bik printer = LLVM::lookupOrCreatePrintBF16Fn(parent); 1609f36e909dSBenjamin Maxwell } else if (printType.isIndex()) { 1610657f60a0SAart Bik printer = LLVM::lookupOrCreatePrintU64Fn(parent); 1611f36e909dSBenjamin Maxwell } else if (auto intTy = dyn_cast<IntegerType>(printType)) { 1612b8880f5fSAart Bik // Integers need a zero or sign extension on the operand 1613b8880f5fSAart Bik // (depending on the source type) as well as a signed or 1614b8880f5fSAart Bik // unsigned print method. Up to 64-bit is supported. 1615b8880f5fSAart Bik unsigned width = intTy.getWidth(); 1616b8880f5fSAart Bik if (intTy.isUnsigned()) { 161754759cefSAart Bik if (width <= 64) { 1618b8880f5fSAart Bik if (width < 64) 1619b8880f5fSAart Bik conversion = PrintConversion::ZeroExt64; 1620657f60a0SAart Bik printer = LLVM::lookupOrCreatePrintU64Fn(parent); 1621b8880f5fSAart Bik } else { 16223145427dSRiver Riddle return failure(); 1623b8880f5fSAart Bik } 1624b8880f5fSAart Bik } else { 1625b8880f5fSAart Bik assert(intTy.isSignless() || intTy.isSigned()); 162654759cefSAart Bik if (width <= 64) { 1627b8880f5fSAart Bik // Note that we *always* zero extend booleans (1-bit integers), 1628b8880f5fSAart Bik // so that true/false is printed as 1/0 rather than -1/0. 1629b8880f5fSAart Bik if (width == 1) 163054759cefSAart Bik conversion = PrintConversion::ZeroExt64; 163154759cefSAart Bik else if (width < 64) 1632b8880f5fSAart Bik conversion = PrintConversion::SignExt64; 1633657f60a0SAart Bik printer = LLVM::lookupOrCreatePrintI64Fn(parent); 1634b8880f5fSAart Bik } else { 1635b8880f5fSAart Bik return failure(); 1636b8880f5fSAart Bik } 1637b8880f5fSAart Bik } 1638b8880f5fSAart Bik } else { 1639b8880f5fSAart Bik return failure(); 1640b8880f5fSAart Bik } 1641*e84f6b6aSLuohao Wang if (failed(printer)) 1642*e84f6b6aSLuohao Wang return failure(); 1643d9b500d3SAart Bik 1644b8880f5fSAart Bik switch (conversion) { 1645b8880f5fSAart Bik case PrintConversion::ZeroExt64: 1646a54f4eaeSMogball value = rewriter.create<arith::ExtUIOp>( 16473c69bc4dSRiver Riddle loc, IntegerType::get(rewriter.getContext(), 64), value); 1648b8880f5fSAart Bik break; 1649b8880f5fSAart Bik case PrintConversion::SignExt64: 1650a54f4eaeSMogball value = rewriter.create<arith::ExtSIOp>( 16513c69bc4dSRiver Riddle loc, IntegerType::get(rewriter.getContext(), 64), value); 1652b8880f5fSAart Bik break; 1653657f60a0SAart Bik case PrintConversion::Bitcast16: 1654657f60a0SAart Bik value = rewriter.create<LLVM::BitcastOp>( 1655657f60a0SAart Bik loc, IntegerType::get(rewriter.getContext(), 16), value); 1656657f60a0SAart Bik break; 1657b8880f5fSAart Bik case PrintConversion::None: 1658b8880f5fSAart Bik break; 1659c9eeeb38Saartbik } 1660*e84f6b6aSLuohao Wang emitCall(rewriter, loc, printer.value(), value); 1661f36e909dSBenjamin Maxwell return success(); 1662d9b500d3SAart Bik } 1663d9b500d3SAart Bik 1664d9b500d3SAart Bik // Helper to emit a call. 1665d9b500d3SAart Bik static void emitCall(ConversionPatternRewriter &rewriter, Location loc, 1666d9b500d3SAart Bik Operation *ref, ValueRange params = ValueRange()) { 1667faf1c224SChris Lattner rewriter.create<LLVM::CallOp>(loc, TypeRange(), SymbolRefAttr::get(ref), 1668faf1c224SChris Lattner params); 1669d9b500d3SAart Bik } 1670d9b500d3SAart Bik }; 1671d9b500d3SAart Bik 16726a8ba318SRiver Riddle /// The Splat operation is lowered to an insertelement + a shufflevector 16736a8ba318SRiver Riddle /// operation. Splat to only 0-d and 1-d vector result types are lowered. 16746a8ba318SRiver Riddle struct VectorSplatOpLowering : public ConvertOpToLLVMPattern<vector::SplatOp> { 16756a8ba318SRiver Riddle using ConvertOpToLLVMPattern<vector::SplatOp>::ConvertOpToLLVMPattern; 16766a8ba318SRiver Riddle 16776a8ba318SRiver Riddle LogicalResult 16786a8ba318SRiver Riddle matchAndRewrite(vector::SplatOp splatOp, OpAdaptor adaptor, 16796a8ba318SRiver Riddle ConversionPatternRewriter &rewriter) const override { 16805550c821STres Popp VectorType resultType = cast<VectorType>(splatOp.getType()); 16816a8ba318SRiver Riddle if (resultType.getRank() > 1) 16826a8ba318SRiver Riddle return failure(); 16836a8ba318SRiver Riddle 16846a8ba318SRiver Riddle // First insert it into an undef vector so we can shuffle it. 16856a8ba318SRiver Riddle auto vectorType = typeConverter->convertType(splatOp.getType()); 16866a8ba318SRiver Riddle Value undef = rewriter.create<LLVM::UndefOp>(splatOp.getLoc(), vectorType); 16876a8ba318SRiver Riddle auto zero = rewriter.create<LLVM::ConstantOp>( 16886a8ba318SRiver Riddle splatOp.getLoc(), 16896a8ba318SRiver Riddle typeConverter->convertType(rewriter.getIntegerType(32)), 16906a8ba318SRiver Riddle rewriter.getZeroAttr(rewriter.getIntegerType(32))); 16916a8ba318SRiver Riddle 16926a8ba318SRiver Riddle // For 0-d vector, we simply do `insertelement`. 16936a8ba318SRiver Riddle if (resultType.getRank() == 0) { 16946a8ba318SRiver Riddle rewriter.replaceOpWithNewOp<LLVM::InsertElementOp>( 16957c38fd60SJacques Pienaar splatOp, vectorType, undef, adaptor.getInput(), zero); 16966a8ba318SRiver Riddle return success(); 16976a8ba318SRiver Riddle } 16986a8ba318SRiver Riddle 16996a8ba318SRiver Riddle // For 1-d vector, we additionally do a `vectorshuffle`. 17006a8ba318SRiver Riddle auto v = rewriter.create<LLVM::InsertElementOp>( 17017c38fd60SJacques Pienaar splatOp.getLoc(), vectorType, undef, adaptor.getInput(), zero); 17026a8ba318SRiver Riddle 17035550c821STres Popp int64_t width = cast<VectorType>(splatOp.getType()).getDimSize(0); 1704b2ccfb4dSJeff Niu SmallVector<int32_t> zeroValues(width, 0); 17056a8ba318SRiver Riddle 17066a8ba318SRiver Riddle // Shuffle the value across the desired number of elements. 17076a8ba318SRiver Riddle rewriter.replaceOpWithNewOp<LLVM::ShuffleVectorOp>(splatOp, v, undef, 1708b2ccfb4dSJeff Niu zeroValues); 17096a8ba318SRiver Riddle return success(); 17106a8ba318SRiver Riddle } 17116a8ba318SRiver Riddle }; 17126a8ba318SRiver Riddle 17136a8ba318SRiver Riddle /// The Splat operation is lowered to an insertelement + a shufflevector 17146a8ba318SRiver Riddle /// operation. Splat to only 2+-d vector result types are lowered by the 17156a8ba318SRiver Riddle /// SplatNdOpLowering, the 1-d case is handled by SplatOpLowering. 17166a8ba318SRiver Riddle struct VectorSplatNdOpLowering : public ConvertOpToLLVMPattern<SplatOp> { 17176a8ba318SRiver Riddle using ConvertOpToLLVMPattern<SplatOp>::ConvertOpToLLVMPattern; 17186a8ba318SRiver Riddle 17196a8ba318SRiver Riddle LogicalResult 17206a8ba318SRiver Riddle matchAndRewrite(SplatOp splatOp, OpAdaptor adaptor, 17216a8ba318SRiver Riddle ConversionPatternRewriter &rewriter) const override { 17226a8ba318SRiver Riddle VectorType resultType = splatOp.getType(); 17236a8ba318SRiver Riddle if (resultType.getRank() <= 1) 17246a8ba318SRiver Riddle return failure(); 17256a8ba318SRiver Riddle 17266a8ba318SRiver Riddle // First insert it into an undef vector so we can shuffle it. 17276a8ba318SRiver Riddle auto loc = splatOp.getLoc(); 17286a8ba318SRiver Riddle auto vectorTypeInfo = 17296a8ba318SRiver Riddle LLVM::detail::extractNDVectorTypeInfo(resultType, *getTypeConverter()); 17306a8ba318SRiver Riddle auto llvmNDVectorTy = vectorTypeInfo.llvmNDVectorTy; 17316a8ba318SRiver Riddle auto llvm1DVectorTy = vectorTypeInfo.llvm1DVectorTy; 17326a8ba318SRiver Riddle if (!llvmNDVectorTy || !llvm1DVectorTy) 17336a8ba318SRiver Riddle return failure(); 17346a8ba318SRiver Riddle 17356a8ba318SRiver Riddle // Construct returned value. 17366a8ba318SRiver Riddle Value desc = rewriter.create<LLVM::UndefOp>(loc, llvmNDVectorTy); 17376a8ba318SRiver Riddle 17386a8ba318SRiver Riddle // Construct a 1-D vector with the splatted value that we insert in all the 17396a8ba318SRiver Riddle // places within the returned descriptor. 17406a8ba318SRiver Riddle Value vdesc = rewriter.create<LLVM::UndefOp>(loc, llvm1DVectorTy); 17416a8ba318SRiver Riddle auto zero = rewriter.create<LLVM::ConstantOp>( 17426a8ba318SRiver Riddle loc, typeConverter->convertType(rewriter.getIntegerType(32)), 17436a8ba318SRiver Riddle rewriter.getZeroAttr(rewriter.getIntegerType(32))); 17446a8ba318SRiver Riddle Value v = rewriter.create<LLVM::InsertElementOp>(loc, llvm1DVectorTy, vdesc, 17457c38fd60SJacques Pienaar adaptor.getInput(), zero); 17466a8ba318SRiver Riddle 17476a8ba318SRiver Riddle // Shuffle the value across the desired number of elements. 17486a8ba318SRiver Riddle int64_t width = resultType.getDimSize(resultType.getRank() - 1); 1749b2ccfb4dSJeff Niu SmallVector<int32_t> zeroValues(width, 0); 1750b2ccfb4dSJeff Niu v = rewriter.create<LLVM::ShuffleVectorOp>(loc, v, v, zeroValues); 17516a8ba318SRiver Riddle 17526a8ba318SRiver Riddle // Iterate of linear index, convert to coords space and insert splatted 1-D 17536a8ba318SRiver Riddle // vector in each position. 17545c5af910SJeff Niu nDVectorIterate(vectorTypeInfo, rewriter, [&](ArrayRef<int64_t> position) { 17555c5af910SJeff Niu desc = rewriter.create<LLVM::InsertValueOp>(loc, desc, v, position); 17566a8ba318SRiver Riddle }); 17576a8ba318SRiver Riddle rewriter.replaceOp(splatOp, desc); 17586a8ba318SRiver Riddle return success(); 17596a8ba318SRiver Riddle } 17606a8ba318SRiver Riddle }; 17616a8ba318SRiver Riddle 176279ce2c93SBenjamin Maxwell /// Conversion pattern for a `vector.interleave`. 176379ce2c93SBenjamin Maxwell /// This supports fixed-sized vectors and scalable vectors. 176479ce2c93SBenjamin Maxwell struct VectorInterleaveOpLowering 176579ce2c93SBenjamin Maxwell : public ConvertOpToLLVMPattern<vector::InterleaveOp> { 176679ce2c93SBenjamin Maxwell using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern; 176779ce2c93SBenjamin Maxwell 176879ce2c93SBenjamin Maxwell LogicalResult 176979ce2c93SBenjamin Maxwell matchAndRewrite(vector::InterleaveOp interleaveOp, OpAdaptor adaptor, 177079ce2c93SBenjamin Maxwell ConversionPatternRewriter &rewriter) const override { 177179ce2c93SBenjamin Maxwell VectorType resultType = interleaveOp.getResultVectorType(); 177279ce2c93SBenjamin Maxwell // n-D interleaves should have been lowered already. 177379ce2c93SBenjamin Maxwell if (resultType.getRank() != 1) 177479ce2c93SBenjamin Maxwell return rewriter.notifyMatchFailure(interleaveOp, 177579ce2c93SBenjamin Maxwell "InterleaveOp not rank 1"); 177679ce2c93SBenjamin Maxwell // If the result is rank 1, then this directly maps to LLVM. 177779ce2c93SBenjamin Maxwell if (resultType.isScalable()) { 1778bfc03171SMaciej Gabka rewriter.replaceOpWithNewOp<LLVM::vector_interleave2>( 177979ce2c93SBenjamin Maxwell interleaveOp, typeConverter->convertType(resultType), 178079ce2c93SBenjamin Maxwell adaptor.getLhs(), adaptor.getRhs()); 178179ce2c93SBenjamin Maxwell return success(); 178279ce2c93SBenjamin Maxwell } 178379ce2c93SBenjamin Maxwell // Lower fixed-size interleaves to a shufflevector. While the 178479ce2c93SBenjamin Maxwell // vector.interleave2 intrinsic supports fixed and scalable vectors, the 178579ce2c93SBenjamin Maxwell // langref still recommends fixed-vectors use shufflevector, see: 178679ce2c93SBenjamin Maxwell // https://llvm.org/docs/LangRef.html#id876. 178779ce2c93SBenjamin Maxwell int64_t resultVectorSize = resultType.getNumElements(); 178879ce2c93SBenjamin Maxwell SmallVector<int32_t> interleaveShuffleMask; 178979ce2c93SBenjamin Maxwell interleaveShuffleMask.reserve(resultVectorSize); 179079ce2c93SBenjamin Maxwell for (int i = 0, end = resultVectorSize / 2; i < end; ++i) { 179179ce2c93SBenjamin Maxwell interleaveShuffleMask.push_back(i); 179279ce2c93SBenjamin Maxwell interleaveShuffleMask.push_back((resultVectorSize / 2) + i); 179379ce2c93SBenjamin Maxwell } 179479ce2c93SBenjamin Maxwell rewriter.replaceOpWithNewOp<LLVM::ShuffleVectorOp>( 179579ce2c93SBenjamin Maxwell interleaveOp, adaptor.getLhs(), adaptor.getRhs(), 179679ce2c93SBenjamin Maxwell interleaveShuffleMask); 179779ce2c93SBenjamin Maxwell return success(); 179879ce2c93SBenjamin Maxwell } 179979ce2c93SBenjamin Maxwell }; 180079ce2c93SBenjamin Maxwell 1801bc946f52SMubashar Ahmad /// Conversion pattern for a `vector.deinterleave`. 1802bc946f52SMubashar Ahmad /// This supports fixed-sized vectors and scalable vectors. 1803bc946f52SMubashar Ahmad struct VectorDeinterleaveOpLowering 1804bc946f52SMubashar Ahmad : public ConvertOpToLLVMPattern<vector::DeinterleaveOp> { 1805bc946f52SMubashar Ahmad using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern; 1806bc946f52SMubashar Ahmad 1807bc946f52SMubashar Ahmad LogicalResult 1808bc946f52SMubashar Ahmad matchAndRewrite(vector::DeinterleaveOp deinterleaveOp, OpAdaptor adaptor, 1809bc946f52SMubashar Ahmad ConversionPatternRewriter &rewriter) const override { 1810bc946f52SMubashar Ahmad VectorType resultType = deinterleaveOp.getResultVectorType(); 1811bc946f52SMubashar Ahmad VectorType sourceType = deinterleaveOp.getSourceVectorType(); 1812bc946f52SMubashar Ahmad auto loc = deinterleaveOp.getLoc(); 1813bc946f52SMubashar Ahmad 1814bc946f52SMubashar Ahmad // Note: n-D deinterleave operations should be lowered to the 1-D before 1815bc946f52SMubashar Ahmad // converting to LLVM. 1816bc946f52SMubashar Ahmad if (resultType.getRank() != 1) 1817bc946f52SMubashar Ahmad return rewriter.notifyMatchFailure(deinterleaveOp, 1818bc946f52SMubashar Ahmad "DeinterleaveOp not rank 1"); 1819bc946f52SMubashar Ahmad 1820bc946f52SMubashar Ahmad if (resultType.isScalable()) { 1821bc946f52SMubashar Ahmad auto llvmTypeConverter = this->getTypeConverter(); 1822bc946f52SMubashar Ahmad auto deinterleaveResults = deinterleaveOp.getResultTypes(); 1823bc946f52SMubashar Ahmad auto packedOpResults = 1824bc946f52SMubashar Ahmad llvmTypeConverter->packOperationResults(deinterleaveResults); 1825bc946f52SMubashar Ahmad auto intrinsic = rewriter.create<LLVM::vector_deinterleave2>( 1826bc946f52SMubashar Ahmad loc, packedOpResults, adaptor.getSource()); 1827bc946f52SMubashar Ahmad 1828bc946f52SMubashar Ahmad auto evenResult = rewriter.create<LLVM::ExtractValueOp>( 1829bc946f52SMubashar Ahmad loc, intrinsic->getResult(0), 0); 1830bc946f52SMubashar Ahmad auto oddResult = rewriter.create<LLVM::ExtractValueOp>( 1831bc946f52SMubashar Ahmad loc, intrinsic->getResult(0), 1); 1832bc946f52SMubashar Ahmad 1833bc946f52SMubashar Ahmad rewriter.replaceOp(deinterleaveOp, ValueRange{evenResult, oddResult}); 1834bc946f52SMubashar Ahmad return success(); 1835bc946f52SMubashar Ahmad } 1836bc946f52SMubashar Ahmad // Lower fixed-size deinterleave to two shufflevectors. While the 1837bc946f52SMubashar Ahmad // vector.deinterleave2 intrinsic supports fixed and scalable vectors, the 1838bc946f52SMubashar Ahmad // langref still recommends fixed-vectors use shufflevector, see: 1839bc946f52SMubashar Ahmad // https://llvm.org/docs/LangRef.html#id889. 1840bc946f52SMubashar Ahmad int64_t resultVectorSize = resultType.getNumElements(); 1841bc946f52SMubashar Ahmad SmallVector<int32_t> evenShuffleMask; 1842bc946f52SMubashar Ahmad SmallVector<int32_t> oddShuffleMask; 1843bc946f52SMubashar Ahmad 1844bc946f52SMubashar Ahmad evenShuffleMask.reserve(resultVectorSize); 1845bc946f52SMubashar Ahmad oddShuffleMask.reserve(resultVectorSize); 1846bc946f52SMubashar Ahmad 1847bc946f52SMubashar Ahmad for (int i = 0; i < sourceType.getNumElements(); ++i) { 1848bc946f52SMubashar Ahmad if (i % 2 == 0) 1849bc946f52SMubashar Ahmad evenShuffleMask.push_back(i); 1850bc946f52SMubashar Ahmad else 1851bc946f52SMubashar Ahmad oddShuffleMask.push_back(i); 1852bc946f52SMubashar Ahmad } 1853bc946f52SMubashar Ahmad 1854bc946f52SMubashar Ahmad auto poison = rewriter.create<LLVM::PoisonOp>(loc, sourceType); 1855bc946f52SMubashar Ahmad auto evenShuffle = rewriter.create<LLVM::ShuffleVectorOp>( 1856bc946f52SMubashar Ahmad loc, adaptor.getSource(), poison, evenShuffleMask); 1857bc946f52SMubashar Ahmad auto oddShuffle = rewriter.create<LLVM::ShuffleVectorOp>( 1858bc946f52SMubashar Ahmad loc, adaptor.getSource(), poison, oddShuffleMask); 1859bc946f52SMubashar Ahmad 1860bc946f52SMubashar Ahmad rewriter.replaceOp(deinterleaveOp, ValueRange{evenShuffle, oddShuffle}); 1861bc946f52SMubashar Ahmad return success(); 1862bc946f52SMubashar Ahmad } 1863bc946f52SMubashar Ahmad }; 1864bc946f52SMubashar Ahmad 1865c6ff2446SMatthias Springer /// Conversion pattern for a `vector.from_elements`. 1866c6ff2446SMatthias Springer struct VectorFromElementsLowering 1867c6ff2446SMatthias Springer : public ConvertOpToLLVMPattern<vector::FromElementsOp> { 1868c6ff2446SMatthias Springer using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern; 1869c6ff2446SMatthias Springer 1870c6ff2446SMatthias Springer LogicalResult 1871c6ff2446SMatthias Springer matchAndRewrite(vector::FromElementsOp fromElementsOp, OpAdaptor adaptor, 1872c6ff2446SMatthias Springer ConversionPatternRewriter &rewriter) const override { 1873c6ff2446SMatthias Springer Location loc = fromElementsOp.getLoc(); 1874c6ff2446SMatthias Springer VectorType vectorType = fromElementsOp.getType(); 1875c6ff2446SMatthias Springer // TODO: Multi-dimensional vectors lower to !llvm.array<... x vector<>>. 1876c6ff2446SMatthias Springer // Such ops should be handled in the same way as vector.insert. 1877c6ff2446SMatthias Springer if (vectorType.getRank() > 1) 1878c6ff2446SMatthias Springer return rewriter.notifyMatchFailure(fromElementsOp, 1879c6ff2446SMatthias Springer "rank > 1 vectors are not supported"); 1880c6ff2446SMatthias Springer Type llvmType = typeConverter->convertType(vectorType); 1881c6ff2446SMatthias Springer Value result = rewriter.create<LLVM::UndefOp>(loc, llvmType); 1882c6ff2446SMatthias Springer for (auto [idx, val] : llvm::enumerate(adaptor.getElements())) 1883c6ff2446SMatthias Springer result = rewriter.create<vector::InsertOp>(loc, val, result, idx); 1884c6ff2446SMatthias Springer rewriter.replaceOp(fromElementsOp, result); 1885c6ff2446SMatthias Springer return success(); 1886c6ff2446SMatthias Springer } 1887c6ff2446SMatthias Springer }; 1888c6ff2446SMatthias Springer 188967b302c5SCullen Rhodes /// Conversion pattern for vector.step. 1890a6e72f93SManupa Karunaratne struct VectorScalableStepOpLowering 1891a6e72f93SManupa Karunaratne : public ConvertOpToLLVMPattern<vector::StepOp> { 189267b302c5SCullen Rhodes using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern; 189367b302c5SCullen Rhodes 189467b302c5SCullen Rhodes LogicalResult 189567b302c5SCullen Rhodes matchAndRewrite(vector::StepOp stepOp, OpAdaptor adaptor, 189667b302c5SCullen Rhodes ConversionPatternRewriter &rewriter) const override { 1897a6e72f93SManupa Karunaratne auto resultType = cast<VectorType>(stepOp.getType()); 1898a6e72f93SManupa Karunaratne if (!resultType.isScalable()) { 1899a6e72f93SManupa Karunaratne return failure(); 1900a6e72f93SManupa Karunaratne } 190167b302c5SCullen Rhodes Type llvmType = typeConverter->convertType(stepOp.getType()); 190267b302c5SCullen Rhodes rewriter.replaceOpWithNewOp<LLVM::StepVectorOp>(stepOp, llvmType); 190367b302c5SCullen Rhodes return success(); 190467b302c5SCullen Rhodes } 190567b302c5SCullen Rhodes }; 190667b302c5SCullen Rhodes 1907df186507SBenjamin Kramer } // namespace 1908df186507SBenjamin Kramer 19090693b9e9SMatthias Springer void mlir::vector::populateVectorRankReducingFMAPattern( 19100693b9e9SMatthias Springer RewritePatternSet &patterns) { 19110693b9e9SMatthias Springer patterns.add<VectorFMAOpNDRewritePattern>(patterns.getContext()); 19120693b9e9SMatthias Springer } 19130693b9e9SMatthias Springer 19145c0c51a9SNicolas Vasilache /// Populate the given list with patterns that convert from Vector to LLVM. 19157bc8ad51SJavier Setoain void mlir::populateVectorToLLVMConversionPatterns( 1916206fad0eSMatthias Springer const LLVMTypeConverter &converter, RewritePatternSet &patterns, 19177bc8ad51SJavier Setoain bool reassociateFPReductions, bool force32BitVectorIndices) { 19180693b9e9SMatthias Springer // This function populates only ConversionPatterns, not RewritePatterns. 191965678d93SNicolas Vasilache MLIRContext *ctx = converter.getDialect()->getContext(); 1920dc4e913bSChris Lattner patterns.add<VectorReductionOpConversion>(converter, reassociateFPReductions); 19210693b9e9SMatthias Springer patterns.add<VectorCreateMaskOpConversion>(ctx, force32BitVectorIndices); 19224db0bd28SCullen Rhodes patterns.add<VectorBitCastOpConversion, VectorShuffleOpConversion, 1923dc4e913bSChris Lattner VectorExtractElementOpConversion, VectorExtractOpConversion, 1924c339f9e1SDiego Caballero VectorFMAOp1DConversion, VectorInsertElementOpConversion, 1925c339f9e1SDiego Caballero VectorInsertOpConversion, VectorPrintOpConversion, 1926c339f9e1SDiego Caballero VectorTypeCastOpConversion, VectorScaleOpConversion, 19274db0bd28SCullen Rhodes VectorLoadStoreConversion<vector::LoadOp>, 19284db0bd28SCullen Rhodes VectorLoadStoreConversion<vector::MaskedLoadOp>, 19294db0bd28SCullen Rhodes VectorLoadStoreConversion<vector::StoreOp>, 19304db0bd28SCullen Rhodes VectorLoadStoreConversion<vector::MaskedStoreOp>, 1931dc4e913bSChris Lattner VectorGatherOpConversion, VectorScatterOpConversion, 19326a8ba318SRiver Riddle VectorExpandLoadOpConversion, VectorCompressStoreOpConversion, 1933aa9647e2SJavier Setoain VectorSplatOpLowering, VectorSplatNdOpLowering, 1934e9b82a5cSDiego Caballero VectorScalableInsertOpLowering, VectorScalableExtractOpLowering, 1935bc946f52SMubashar Ahmad MaskedReductionOpConversion, VectorInterleaveOpLowering, 193667b302c5SCullen Rhodes VectorDeinterleaveOpLowering, VectorFromElementsLowering, 1937a6e72f93SManupa Karunaratne VectorScalableStepOpLowering>(converter); 19385c0c51a9SNicolas Vasilache } 19395c0c51a9SNicolas Vasilache 194063b683a8SNicolas Vasilache void mlir::populateVectorToLLVMMatrixConversionPatterns( 1941206fad0eSMatthias Springer const LLVMTypeConverter &converter, RewritePatternSet &patterns) { 1942dc4e913bSChris Lattner patterns.add<VectorMatmulOpConversion>(converter); 1943dc4e913bSChris Lattner patterns.add<VectorFlatTransposeOpConversion>(converter); 194463b683a8SNicolas Vasilache } 1945