1 //===- VectorToLLVM.cpp - Conversion from Vector to the LLVM dialect ------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 9 #include "mlir/Conversion/VectorToLLVM/ConvertVectorToLLVM.h" 10 11 #include "../PassDetail.h" 12 13 #include "mlir/Conversion/ArmSVEToLLVM/ArmSVEToLLVM.h" 14 #include "mlir/Conversion/StandardToLLVM/ConvertStandardToLLVM.h" 15 #include "mlir/Conversion/StandardToLLVM/ConvertStandardToLLVMPass.h" 16 #include "mlir/Dialect/AMX/AMXDialect.h" 17 #include "mlir/Dialect/AMX/Transforms.h" 18 #include "mlir/Dialect/AVX512/AVX512Dialect.h" 19 #include "mlir/Dialect/AVX512/Transforms.h" 20 #include "mlir/Dialect/ArmNeon/ArmNeonDialect.h" 21 #include "mlir/Dialect/ArmSVE/ArmSVEDialect.h" 22 #include "mlir/Dialect/LLVMIR/LLVMArmSVEDialect.h" 23 #include "mlir/Dialect/LLVMIR/LLVMDialect.h" 24 #include "mlir/Dialect/MemRef/IR/MemRef.h" 25 #include "mlir/Dialect/StandardOps/IR/Ops.h" 26 #include "mlir/Dialect/Vector/VectorOps.h" 27 #include "mlir/Transforms/GreedyPatternRewriteDriver.h" 28 29 using namespace mlir; 30 using namespace mlir::vector; 31 32 namespace { 33 struct LowerVectorToLLVMPass 34 : public ConvertVectorToLLVMBase<LowerVectorToLLVMPass> { 35 LowerVectorToLLVMPass(const LowerVectorToLLVMOptions &options) { 36 this->reassociateFPReductions = options.reassociateFPReductions; 37 this->enableIndexOptimizations = options.enableIndexOptimizations; 38 this->enableArmNeon = options.enableArmNeon; 39 this->enableArmSVE = options.enableArmSVE; 40 this->enableAMX = options.enableAMX; 41 this->enableAVX512 = options.enableAVX512; 42 } 43 // Override explicitly to allow conditional dialect dependence. 44 void getDependentDialects(DialectRegistry ®istry) const override { 45 registry.insert<LLVM::LLVMDialect>(); 46 registry.insert<memref::MemRefDialect>(); 47 if (enableArmNeon) 48 registry.insert<arm_neon::ArmNeonDialect>(); 49 if (enableArmSVE) 50 registry.insert<LLVM::LLVMArmSVEDialect>(); 51 if (enableAMX) 52 registry.insert<amx::AMXDialect>(); 53 if (enableAVX512) 54 registry.insert<avx512::AVX512Dialect>(); 55 } 56 void runOnOperation() override; 57 }; 58 } // namespace 59 60 void LowerVectorToLLVMPass::runOnOperation() { 61 // Perform progressive lowering of operations on slices and 62 // all contraction operations. Also applies folding and DCE. 63 { 64 RewritePatternSet patterns(&getContext()); 65 populateVectorToVectorCanonicalizationPatterns(patterns); 66 populateVectorSlicesLoweringPatterns(patterns); 67 populateVectorContractLoweringPatterns(patterns); 68 (void)applyPatternsAndFoldGreedily(getOperation(), std::move(patterns)); 69 } 70 71 // Convert to the LLVM IR dialect. 72 LLVMTypeConverter converter(&getContext()); 73 RewritePatternSet patterns(&getContext()); 74 populateVectorMaskMaterializationPatterns(patterns, enableIndexOptimizations); 75 populateVectorToLLVMMatrixConversionPatterns(converter, patterns); 76 populateVectorToLLVMConversionPatterns(converter, patterns, 77 reassociateFPReductions); 78 populateVectorToLLVMMatrixConversionPatterns(converter, patterns); 79 80 // Architecture specific augmentations. 81 LLVMConversionTarget target(getContext()); 82 target.addLegalOp<LLVM::DialectCastOp>(); 83 target.addLegalDialect<memref::MemRefDialect>(); 84 target.addLegalDialect<StandardOpsDialect>(); 85 target.addLegalOp<UnrealizedConversionCastOp>(); 86 if (enableArmNeon) { 87 // TODO: we may or may not want to include in-dialect lowering to 88 // LLVM-compatible operations here. So far, all operations in the dialect 89 // can be translated to LLVM IR so there is no conversion necessary. 90 target.addLegalDialect<arm_neon::ArmNeonDialect>(); 91 } 92 if (enableArmSVE) { 93 target.addLegalDialect<LLVM::LLVMArmSVEDialect>(); 94 target.addIllegalDialect<arm_sve::ArmSVEDialect>(); 95 auto hasScalableVectorType = [](TypeRange types) { 96 for (Type type : types) 97 if (type.isa<arm_sve::ScalableVectorType>()) 98 return true; 99 return false; 100 }; 101 // Remove any ArmSVE-specific types from function signatures and results. 102 populateFuncOpTypeConversionPattern(patterns, converter); 103 target.addDynamicallyLegalOp<FuncOp>([hasScalableVectorType](FuncOp op) { 104 return !hasScalableVectorType(op.getType().getInputs()) && 105 !hasScalableVectorType(op.getType().getResults()); 106 }); 107 target.addDynamicallyLegalOp<CallOp, CallIndirectOp, ReturnOp>( 108 [hasScalableVectorType](Operation *op) { 109 return !hasScalableVectorType(op->getOperandTypes()) && 110 !hasScalableVectorType(op->getResultTypes()); 111 }); 112 populateArmSVEToLLVMConversionPatterns(converter, patterns); 113 } 114 if (enableAMX) { 115 configureAMXLegalizeForExportTarget(target); 116 populateAMXLegalizeForLLVMExportPatterns(converter, patterns); 117 } 118 if (enableAVX512) { 119 configureAVX512LegalizeForExportTarget(target); 120 populateAVX512LegalizeForLLVMExportPatterns(converter, patterns); 121 } 122 123 if (failed( 124 applyPartialConversion(getOperation(), target, std::move(patterns)))) 125 signalPassFailure(); 126 } 127 128 std::unique_ptr<OperationPass<ModuleOp>> 129 mlir::createConvertVectorToLLVMPass(const LowerVectorToLLVMOptions &options) { 130 return std::make_unique<LowerVectorToLLVMPass>(options); 131 } 132