1//===--- arm_sve.td - ARM SVE compiler interface ------------------------===// 2// 3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4// See https://llvm.org/LICENSE.txt for license information. 5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6// 7//===----------------------------------------------------------------------===// 8// 9// This file defines the TableGen definitions from which the ARM SVE header 10// file will be generated. See: 11// 12// https://developer.arm.com/architectures/system-architectures/software-standards/acle 13// 14//===----------------------------------------------------------------------===// 15 16include "arm_sve_sme_incl.td" 17 18//////////////////////////////////////////////////////////////////////////////// 19// Loads 20 21// Load one vector (scalar base) 22def SVLD1 : MInst<"svld1[_{2}]", "dPc", "csilUcUsUiUlhfd", [IsLoad, VerifyRuntimeMode], MemEltTyDefault, "aarch64_sve_ld1">; 23def SVLD1SB : MInst<"svld1sb_{d}", "dPS", "silUsUiUl", [IsLoad, VerifyRuntimeMode], MemEltTyInt8, "aarch64_sve_ld1">; 24def SVLD1UB : MInst<"svld1ub_{d}", "dPW", "silUsUiUl", [IsLoad, IsZExtReturn, VerifyRuntimeMode], MemEltTyInt8, "aarch64_sve_ld1">; 25def SVLD1SH : MInst<"svld1sh_{d}", "dPT", "ilUiUl", [IsLoad, VerifyRuntimeMode], MemEltTyInt16, "aarch64_sve_ld1">; 26def SVLD1UH : MInst<"svld1uh_{d}", "dPX", "ilUiUl", [IsLoad, IsZExtReturn, VerifyRuntimeMode], MemEltTyInt16, "aarch64_sve_ld1">; 27def SVLD1SW : MInst<"svld1sw_{d}", "dPU", "lUl", [IsLoad, VerifyRuntimeMode], MemEltTyInt32, "aarch64_sve_ld1">; 28def SVLD1UW : MInst<"svld1uw_{d}", "dPY", "lUl", [IsLoad, IsZExtReturn, VerifyRuntimeMode], MemEltTyInt32, "aarch64_sve_ld1">; 29 30let SVETargetGuard = "sve,bf16", SMETargetGuard = "sme,bf16" in { 31 def SVLD1_BF : MInst<"svld1[_{2}]", "dPc", "b", [IsLoad, VerifyRuntimeMode], MemEltTyDefault, "aarch64_sve_ld1">; 32 def SVLD1_VNUM_BF : MInst<"svld1_vnum[_{2}]", "dPcl", "b", [IsLoad, VerifyRuntimeMode], MemEltTyDefault, "aarch64_sve_ld1">; 33} 34 35// Load one vector (scalar base, VL displacement) 36def SVLD1_VNUM : MInst<"svld1_vnum[_{2}]", "dPcl", "csilUcUsUiUlhfd", [IsLoad, VerifyRuntimeMode], MemEltTyDefault, "aarch64_sve_ld1">; 37def SVLD1SB_VNUM : MInst<"svld1sb_vnum_{d}", "dPSl", "silUsUiUl", [IsLoad, VerifyRuntimeMode], MemEltTyInt8, "aarch64_sve_ld1">; 38def SVLD1UB_VNUM : MInst<"svld1ub_vnum_{d}", "dPWl", "silUsUiUl", [IsLoad, IsZExtReturn, VerifyRuntimeMode], MemEltTyInt8, "aarch64_sve_ld1">; 39def SVLD1SH_VNUM : MInst<"svld1sh_vnum_{d}", "dPTl", "ilUiUl", [IsLoad, VerifyRuntimeMode], MemEltTyInt16, "aarch64_sve_ld1">; 40def SVLD1UH_VNUM : MInst<"svld1uh_vnum_{d}", "dPXl", "ilUiUl", [IsLoad, IsZExtReturn, VerifyRuntimeMode], MemEltTyInt16, "aarch64_sve_ld1">; 41def SVLD1SW_VNUM : MInst<"svld1sw_vnum_{d}", "dPUl", "lUl", [IsLoad, VerifyRuntimeMode], MemEltTyInt32, "aarch64_sve_ld1">; 42def SVLD1UW_VNUM : MInst<"svld1uw_vnum_{d}", "dPYl", "lUl", [IsLoad, IsZExtReturn, VerifyRuntimeMode], MemEltTyInt32, "aarch64_sve_ld1">; 43 44let SVETargetGuard = "sve", SMETargetGuard = InvalidMode in { 45// Load one vector (vector base) 46def SVLD1_GATHER_BASES_U : MInst<"svld1_gather[_{2}base]_{d}", "dPu", "ilUiUlfd", [IsGatherLoad], MemEltTyDefault, "aarch64_sve_ld1_gather_scalar_offset">; 47def SVLD1SB_GATHER_BASES_U : MInst<"svld1sb_gather[_{2}base]_{d}", "dPu", "ilUiUl", [IsGatherLoad], MemEltTyInt8, "aarch64_sve_ld1_gather_scalar_offset">; 48def SVLD1UB_GATHER_BASES_U : MInst<"svld1ub_gather[_{2}base]_{d}", "dPu", "ilUiUl", [IsGatherLoad, IsZExtReturn], MemEltTyInt8, "aarch64_sve_ld1_gather_scalar_offset">; 49def SVLD1SH_GATHER_BASES_U : MInst<"svld1sh_gather[_{2}base]_{d}", "dPu", "ilUiUl", [IsGatherLoad], MemEltTyInt16, "aarch64_sve_ld1_gather_scalar_offset">; 50def SVLD1UH_GATHER_BASES_U : MInst<"svld1uh_gather[_{2}base]_{d}", "dPu", "ilUiUl", [IsGatherLoad, IsZExtReturn], MemEltTyInt16, "aarch64_sve_ld1_gather_scalar_offset">; 51def SVLD1SW_GATHER_BASES_U : MInst<"svld1sw_gather[_{2}base]_{d}", "dPu", "lUl", [IsGatherLoad], MemEltTyInt32, "aarch64_sve_ld1_gather_scalar_offset">; 52def SVLD1UW_GATHER_BASES_U : MInst<"svld1uw_gather[_{2}base]_{d}", "dPu", "lUl", [IsGatherLoad, IsZExtReturn], MemEltTyInt32, "aarch64_sve_ld1_gather_scalar_offset">; 53 54// Load one vector (scalar base, signed vector offset in bytes) 55def SVLD1_GATHER_64B_OFFSETS_S : MInst<"svld1_gather_[{3}]offset[_{d}]", "dPcx", "lUld", [IsGatherLoad, IsByteIndexed], MemEltTyDefault, "aarch64_sve_ld1_gather">; 56def SVLD1SB_GATHER_64B_OFFSETS_S : MInst<"svld1sb_gather_[{3}]offset_{d}", "dPSx", "lUl", [IsGatherLoad, IsByteIndexed], MemEltTyInt8, "aarch64_sve_ld1_gather">; 57def SVLD1UB_GATHER_64B_OFFSETS_S : MInst<"svld1ub_gather_[{3}]offset_{d}", "dPWx", "lUl", [IsGatherLoad, IsByteIndexed, IsZExtReturn], MemEltTyInt8, "aarch64_sve_ld1_gather">; 58def SVLD1SH_GATHER_64B_OFFSETS_S : MInst<"svld1sh_gather_[{3}]offset_{d}", "dPTx", "lUl", [IsGatherLoad, IsByteIndexed], MemEltTyInt16, "aarch64_sve_ld1_gather">; 59def SVLD1UH_GATHER_64B_OFFSETS_S : MInst<"svld1uh_gather_[{3}]offset_{d}", "dPXx", "lUl", [IsGatherLoad, IsByteIndexed, IsZExtReturn], MemEltTyInt16, "aarch64_sve_ld1_gather">; 60def SVLD1SW_GATHER_64B_OFFSETS_S : MInst<"svld1sw_gather_[{3}]offset_{d}", "dPUx", "lUl", [IsGatherLoad, IsByteIndexed], MemEltTyInt32, "aarch64_sve_ld1_gather">; 61def SVLD1UW_GATHER_64B_OFFSETS_S : MInst<"svld1uw_gather_[{3}]offset_{d}", "dPYx", "lUl", [IsGatherLoad, IsByteIndexed, IsZExtReturn], MemEltTyInt32, "aarch64_sve_ld1_gather">; 62 63def SVLD1_GATHER_32B_OFFSETS_S : MInst<"svld1_gather_[{3}]offset[_{d}]", "dPcx", "iUif", [IsGatherLoad, IsByteIndexed], MemEltTyDefault, "aarch64_sve_ld1_gather_sxtw">; 64def SVLD1SB_GATHER_32B_OFFSETS_S : MInst<"svld1sb_gather_[{3}]offset_{d}", "dPSx", "iUi", [IsGatherLoad, IsByteIndexed], MemEltTyInt8, "aarch64_sve_ld1_gather_sxtw">; 65def SVLD1UB_GATHER_32B_OFFSETS_S : MInst<"svld1ub_gather_[{3}]offset_{d}", "dPWx", "iUi", [IsGatherLoad, IsByteIndexed, IsZExtReturn], MemEltTyInt8, "aarch64_sve_ld1_gather_sxtw">; 66def SVLD1SH_GATHER_32B_OFFSETS_S : MInst<"svld1sh_gather_[{3}]offset_{d}", "dPTx", "iUi", [IsGatherLoad, IsByteIndexed], MemEltTyInt16, "aarch64_sve_ld1_gather_sxtw">; 67def SVLD1UH_GATHER_32B_OFFSETS_S : MInst<"svld1uh_gather_[{3}]offset_{d}", "dPXx", "iUi", [IsGatherLoad, IsByteIndexed, IsZExtReturn], MemEltTyInt16, "aarch64_sve_ld1_gather_sxtw">; 68 69// Load one vector (scalar base, unsigned vector offset in bytes) 70def SVLD1_GATHER_64B_OFFSETS_U : MInst<"svld1_gather_[{3}]offset[_{d}]", "dPcu", "lUld", [IsGatherLoad, IsByteIndexed], MemEltTyDefault, "aarch64_sve_ld1_gather">; 71def SVLD1SB_GATHER_64B_OFFSETS_U : MInst<"svld1sb_gather_[{3}]offset_{d}", "dPSu", "lUl", [IsGatherLoad, IsByteIndexed], MemEltTyInt8, "aarch64_sve_ld1_gather">; 72def SVLD1UB_GATHER_64B_OFFSETS_U : MInst<"svld1ub_gather_[{3}]offset_{d}", "dPWu", "lUl", [IsGatherLoad, IsByteIndexed, IsZExtReturn], MemEltTyInt8, "aarch64_sve_ld1_gather">; 73def SVLD1SH_GATHER_64B_OFFSETS_U : MInst<"svld1sh_gather_[{3}]offset_{d}", "dPTu", "lUl", [IsGatherLoad, IsByteIndexed], MemEltTyInt16, "aarch64_sve_ld1_gather">; 74def SVLD1UH_GATHER_64B_OFFSETS_U : MInst<"svld1uh_gather_[{3}]offset_{d}", "dPXu", "lUl", [IsGatherLoad, IsByteIndexed, IsZExtReturn], MemEltTyInt16, "aarch64_sve_ld1_gather">; 75def SVLD1SW_GATHER_64B_OFFSETS_U : MInst<"svld1sw_gather_[{3}]offset_{d}", "dPUu", "lUl", [IsGatherLoad, IsByteIndexed], MemEltTyInt32, "aarch64_sve_ld1_gather">; 76def SVLD1UW_GATHER_64B_OFFSETS_U : MInst<"svld1uw_gather_[{3}]offset_{d}", "dPYu", "lUl", [IsGatherLoad, IsByteIndexed, IsZExtReturn], MemEltTyInt32, "aarch64_sve_ld1_gather">; 77 78def SVLD1_GATHER_32B_OFFSETS_U : MInst<"svld1_gather_[{3}]offset[_{d}]", "dPcu", "iUif", [IsGatherLoad, IsByteIndexed], MemEltTyDefault, "aarch64_sve_ld1_gather_uxtw">; 79def SVLD1SB_GATHER_32B_OFFSETS_U : MInst<"svld1sb_gather_[{3}]offset_{d}", "dPSu", "iUi", [IsGatherLoad, IsByteIndexed], MemEltTyInt8, "aarch64_sve_ld1_gather_uxtw">; 80def SVLD1UB_GATHER_32B_OFFSETS_U : MInst<"svld1ub_gather_[{3}]offset_{d}", "dPWu", "iUi", [IsGatherLoad, IsByteIndexed, IsZExtReturn], MemEltTyInt8, "aarch64_sve_ld1_gather_uxtw">; 81def SVLD1SH_GATHER_32B_OFFSETS_U : MInst<"svld1sh_gather_[{3}]offset_{d}", "dPTu", "iUi", [IsGatherLoad, IsByteIndexed], MemEltTyInt16, "aarch64_sve_ld1_gather_uxtw">; 82def SVLD1UH_GATHER_32B_OFFSETS_U : MInst<"svld1uh_gather_[{3}]offset_{d}", "dPXu", "iUi", [IsGatherLoad, IsByteIndexed, IsZExtReturn], MemEltTyInt16, "aarch64_sve_ld1_gather_uxtw">; 83 84// Load one vector (vector base, signed scalar offset in bytes) 85def SVLD1_GATHER_OFFSET_S : MInst<"svld1_gather[_{2}base]_offset_{d}", "dPul", "ilUiUlfd", [IsGatherLoad, IsByteIndexed], MemEltTyDefault, "aarch64_sve_ld1_gather_scalar_offset">; 86def SVLD1SB_GATHER_OFFSET_S : MInst<"svld1sb_gather[_{2}base]_offset_{d}", "dPul", "ilUiUl", [IsGatherLoad, IsByteIndexed], MemEltTyInt8, "aarch64_sve_ld1_gather_scalar_offset">; 87def SVLD1UB_GATHER_OFFSET_S : MInst<"svld1ub_gather[_{2}base]_offset_{d}", "dPul", "ilUiUl", [IsGatherLoad, IsByteIndexed, IsZExtReturn], MemEltTyInt8, "aarch64_sve_ld1_gather_scalar_offset">; 88def SVLD1SH_GATHER_OFFSET_S : MInst<"svld1sh_gather[_{2}base]_offset_{d}", "dPul", "ilUiUl", [IsGatherLoad, IsByteIndexed], MemEltTyInt16, "aarch64_sve_ld1_gather_scalar_offset">; 89def SVLD1UH_GATHER_OFFSET_S : MInst<"svld1uh_gather[_{2}base]_offset_{d}", "dPul", "ilUiUl", [IsGatherLoad, IsByteIndexed, IsZExtReturn], MemEltTyInt16, "aarch64_sve_ld1_gather_scalar_offset">; 90def SVLD1SW_GATHER_OFFSET_S : MInst<"svld1sw_gather[_{2}base]_offset_{d}", "dPul", "lUl", [IsGatherLoad, IsByteIndexed], MemEltTyInt32, "aarch64_sve_ld1_gather_scalar_offset">; 91def SVLD1UW_GATHER_OFFSET_S : MInst<"svld1uw_gather[_{2}base]_offset_{d}", "dPul", "lUl", [IsGatherLoad, IsByteIndexed, IsZExtReturn], MemEltTyInt32, "aarch64_sve_ld1_gather_scalar_offset">; 92 93// Load one vector (scalar base, signed vector index) 94def SVLD1_GATHER_64B_INDICES_S : MInst<"svld1_gather_[{3}]index[_{d}]", "dPcx", "lUld", [IsGatherLoad], MemEltTyDefault, "aarch64_sve_ld1_gather_index">; 95def SVLD1SH_GATHER_64B_INDICES_S : MInst<"svld1sh_gather_[{3}]index_{d}", "dPTx", "lUl", [IsGatherLoad], MemEltTyInt16, "aarch64_sve_ld1_gather_index">; 96def SVLD1UH_GATHER_64B_INDICES_S : MInst<"svld1uh_gather_[{3}]index_{d}", "dPXx", "lUl", [IsGatherLoad, IsZExtReturn], MemEltTyInt16, "aarch64_sve_ld1_gather_index">; 97def SVLD1SW_GATHER_64B_INDICES_S : MInst<"svld1sw_gather_[{3}]index_{d}", "dPUx", "lUl", [IsGatherLoad], MemEltTyInt32, "aarch64_sve_ld1_gather_index">; 98def SVLD1UW_GATHER_64B_INDICES_S : MInst<"svld1uw_gather_[{3}]index_{d}", "dPYx", "lUl", [IsGatherLoad, IsZExtReturn], MemEltTyInt32, "aarch64_sve_ld1_gather_index">; 99 100def SVLD1_GATHER_32B_INDICES_S : MInst<"svld1_gather_[{3}]index[_{d}]", "dPcx", "iUif", [IsGatherLoad], MemEltTyDefault, "aarch64_sve_ld1_gather_sxtw_index">; 101def SVLD1SH_GATHER_32B_INDICES_S : MInst<"svld1sh_gather_[{3}]index_{d}", "dPTx", "iUi", [IsGatherLoad], MemEltTyInt16, "aarch64_sve_ld1_gather_sxtw_index">; 102def SVLD1UH_GATHER_32B_INDICES_S : MInst<"svld1uh_gather_[{3}]index_{d}", "dPXx", "iUi", [IsGatherLoad, IsZExtReturn], MemEltTyInt16, "aarch64_sve_ld1_gather_sxtw_index">; 103 104// Load one vector (scalar base, unsigned vector index) 105def SVLD1_GATHER_64B_INDICES_U : MInst<"svld1_gather_[{3}]index[_{d}]", "dPcu", "lUld", [IsGatherLoad], MemEltTyDefault, "aarch64_sve_ld1_gather_index">; 106def SVLD1SH_GATHER_64B_INDICES_U : MInst<"svld1sh_gather_[{3}]index_{d}", "dPTu", "lUl", [IsGatherLoad], MemEltTyInt16, "aarch64_sve_ld1_gather_index">; 107def SVLD1UH_GATHER_64B_INDICES_U : MInst<"svld1uh_gather_[{3}]index_{d}", "dPXu", "lUl", [IsGatherLoad, IsZExtReturn], MemEltTyInt16, "aarch64_sve_ld1_gather_index">; 108def SVLD1SW_GATHER_64B_INDICES_U : MInst<"svld1sw_gather_[{3}]index_{d}", "dPUu", "lUl", [IsGatherLoad], MemEltTyInt32, "aarch64_sve_ld1_gather_index">; 109def SVLD1UW_GATHER_64B_INDICES_U : MInst<"svld1uw_gather_[{3}]index_{d}", "dPYu", "lUl", [IsGatherLoad, IsZExtReturn], MemEltTyInt32, "aarch64_sve_ld1_gather_index">; 110 111def SVLD1_GATHER_32B_INDICES_U : MInst<"svld1_gather_[{3}]index[_{d}]", "dPcu", "iUif", [IsGatherLoad], MemEltTyDefault, "aarch64_sve_ld1_gather_uxtw_index">; 112def SVLD1SH_GATHER_32B_INDICES_U : MInst<"svld1sh_gather_[{3}]index_{d}", "dPTu", "iUi", [IsGatherLoad], MemEltTyInt16, "aarch64_sve_ld1_gather_uxtw_index">; 113def SVLD1UH_GATHER_32B_INDICES_U : MInst<"svld1uh_gather_[{3}]index_{d}", "dPXu", "iUi", [IsGatherLoad, IsZExtReturn], MemEltTyInt16, "aarch64_sve_ld1_gather_uxtw_index">; 114 115// Load one vector (vector base, signed scalar index) 116def SVLD1_GATHER_INDEX_S : MInst<"svld1_gather[_{2}base]_index_{d}", "dPul", "ilUiUlfd", [IsGatherLoad], MemEltTyDefault, "aarch64_sve_ld1_gather_scalar_offset">; 117def SVLD1SH_GATHER_INDEX_S : MInst<"svld1sh_gather[_{2}base]_index_{d}", "dPul", "ilUiUl", [IsGatherLoad], MemEltTyInt16, "aarch64_sve_ld1_gather_scalar_offset">; 118def SVLD1UH_GATHER_INDEX_S : MInst<"svld1uh_gather[_{2}base]_index_{d}", "dPul", "ilUiUl", [IsGatherLoad, IsZExtReturn], MemEltTyInt16, "aarch64_sve_ld1_gather_scalar_offset">; 119def SVLD1SW_GATHER_INDEX_S : MInst<"svld1sw_gather[_{2}base]_index_{d}", "dPul", "lUl", [IsGatherLoad], MemEltTyInt32, "aarch64_sve_ld1_gather_scalar_offset">; 120def SVLD1UW_GATHER_INDEX_S : MInst<"svld1uw_gather[_{2}base]_index_{d}", "dPul", "lUl", [IsGatherLoad, IsZExtReturn], MemEltTyInt32, "aarch64_sve_ld1_gather_scalar_offset">; 121 122 123// First-faulting load one vector (scalar base) 124def SVLDFF1 : MInst<"svldff1[_{2}]", "dPc", "csilUcUsUiUlhfd", [IsLoad], MemEltTyDefault, "aarch64_sve_ldff1">; 125def SVLDFF1SB : MInst<"svldff1sb_{d}", "dPS", "silUsUiUl", [IsLoad], MemEltTyInt8, "aarch64_sve_ldff1">; 126def SVLDFF1UB : MInst<"svldff1ub_{d}", "dPW", "silUsUiUl", [IsLoad, IsZExtReturn], MemEltTyInt8, "aarch64_sve_ldff1">; 127def SVLDFF1SH : MInst<"svldff1sh_{d}", "dPT", "ilUiUl", [IsLoad], MemEltTyInt16, "aarch64_sve_ldff1">; 128def SVLDFF1UH : MInst<"svldff1uh_{d}", "dPX", "ilUiUl", [IsLoad, IsZExtReturn], MemEltTyInt16, "aarch64_sve_ldff1">; 129def SVLDFF1SW : MInst<"svldff1sw_{d}", "dPU", "lUl", [IsLoad], MemEltTyInt32, "aarch64_sve_ldff1">; 130def SVLDFF1UW : MInst<"svldff1uw_{d}", "dPY", "lUl", [IsLoad, IsZExtReturn], MemEltTyInt32, "aarch64_sve_ldff1">; 131 132// First-faulting load one vector (scalar base, VL displacement) 133def SVLDFF1_VNUM : MInst<"svldff1_vnum[_{2}]", "dPcl", "csilUcUsUiUlhfd", [IsLoad], MemEltTyDefault, "aarch64_sve_ldff1">; 134def SVLDFF1SB_VNUM : MInst<"svldff1sb_vnum_{d}", "dPSl", "silUsUiUl", [IsLoad], MemEltTyInt8, "aarch64_sve_ldff1">; 135def SVLDFF1UB_VNUM : MInst<"svldff1ub_vnum_{d}", "dPWl", "silUsUiUl", [IsLoad, IsZExtReturn], MemEltTyInt8, "aarch64_sve_ldff1">; 136def SVLDFF1SH_VNUM : MInst<"svldff1sh_vnum_{d}", "dPTl", "ilUiUl", [IsLoad], MemEltTyInt16, "aarch64_sve_ldff1">; 137def SVLDFF1UH_VNUM : MInst<"svldff1uh_vnum_{d}", "dPXl", "ilUiUl", [IsLoad, IsZExtReturn], MemEltTyInt16, "aarch64_sve_ldff1">; 138def SVLDFF1SW_VNUM : MInst<"svldff1sw_vnum_{d}", "dPUl", "lUl", [IsLoad], MemEltTyInt32, "aarch64_sve_ldff1">; 139def SVLDFF1UW_VNUM : MInst<"svldff1uw_vnum_{d}", "dPYl", "lUl", [IsLoad, IsZExtReturn], MemEltTyInt32, "aarch64_sve_ldff1">; 140} // let SVETargetGuard = "sve", SMETargetGuard = InvalidMode 141 142let SVETargetGuard = "sve,bf16", SMETargetGuard = InvalidMode in { 143 def SVLDFF1_BF : MInst<"svldff1[_{2}]", "dPc", "b", [IsLoad], MemEltTyDefault, "aarch64_sve_ldff1">; 144 def SVLDFF1_VNUM_BF : MInst<"svldff1_vnum[_{2}]", "dPcl", "b", [IsLoad], MemEltTyDefault, "aarch64_sve_ldff1">; 145} 146 147let SVETargetGuard = "sve", SMETargetGuard = InvalidMode in { 148// First-faulting load one vector (vector base) 149def SVLDFF1_GATHER_BASES_U : MInst<"svldff1_gather[_{2}base]_{d}", "dPu", "ilUiUlfd", [IsGatherLoad], MemEltTyDefault, "aarch64_sve_ldff1_gather_scalar_offset">; 150def SVLDFF1SB_GATHER_BASES_U : MInst<"svldff1sb_gather[_{2}base]_{d}", "dPu", "ilUiUl", [IsGatherLoad], MemEltTyInt8, "aarch64_sve_ldff1_gather_scalar_offset">; 151def SVLDFF1UB_GATHER_BASES_U : MInst<"svldff1ub_gather[_{2}base]_{d}", "dPu", "ilUiUl", [IsGatherLoad, IsZExtReturn], MemEltTyInt8, "aarch64_sve_ldff1_gather_scalar_offset">; 152def SVLDFF1SH_GATHER_BASES_U : MInst<"svldff1sh_gather[_{2}base]_{d}", "dPu", "ilUiUl", [IsGatherLoad], MemEltTyInt16, "aarch64_sve_ldff1_gather_scalar_offset">; 153def SVLDFF1UH_GATHER_BASES_U : MInst<"svldff1uh_gather[_{2}base]_{d}", "dPu", "ilUiUl", [IsGatherLoad, IsZExtReturn], MemEltTyInt16, "aarch64_sve_ldff1_gather_scalar_offset">; 154def SVLDFF1SW_GATHER_BASES_U : MInst<"svldff1sw_gather[_{2}base]_{d}", "dPu", "lUl", [IsGatherLoad], MemEltTyInt32, "aarch64_sve_ldff1_gather_scalar_offset">; 155def SVLDFF1UW_GATHER_BASES_U : MInst<"svldff1uw_gather[_{2}base]_{d}", "dPu", "lUl", [IsGatherLoad, IsZExtReturn], MemEltTyInt32, "aarch64_sve_ldff1_gather_scalar_offset">; 156 157// First-faulting load one vector (scalar base, signed vector offset in bytes) 158def SVLDFF1_GATHER_64B_OFFSETS_S : MInst<"svldff1_gather_[{3}]offset[_{d}]", "dPcx", "lUld", [IsGatherLoad, IsByteIndexed], MemEltTyDefault, "aarch64_sve_ldff1_gather">; 159def SVLDFF1SB_GATHER_64B_OFFSETS_S : MInst<"svldff1sb_gather_[{3}]offset_{d}", "dPSx", "lUl", [IsGatherLoad, IsByteIndexed], MemEltTyInt8, "aarch64_sve_ldff1_gather">; 160def SVLDFF1UB_GATHER_64B_OFFSETS_S : MInst<"svldff1ub_gather_[{3}]offset_{d}", "dPWx", "lUl", [IsGatherLoad, IsByteIndexed, IsZExtReturn], MemEltTyInt8, "aarch64_sve_ldff1_gather">; 161def SVLDFF1SH_GATHER_64B_OFFSETS_S : MInst<"svldff1sh_gather_[{3}]offset_{d}", "dPTx", "lUl", [IsGatherLoad, IsByteIndexed], MemEltTyInt16, "aarch64_sve_ldff1_gather">; 162def SVLDFF1UH_GATHER_64B_OFFSETS_S : MInst<"svldff1uh_gather_[{3}]offset_{d}", "dPXx", "lUl", [IsGatherLoad, IsByteIndexed, IsZExtReturn], MemEltTyInt16, "aarch64_sve_ldff1_gather">; 163def SVLDFF1SW_GATHER_64B_OFFSETS_S : MInst<"svldff1sw_gather_[{3}]offset_{d}", "dPUx", "lUl", [IsGatherLoad, IsByteIndexed], MemEltTyInt32, "aarch64_sve_ldff1_gather">; 164def SVLDFF1UW_GATHER_64B_OFFSETS_S : MInst<"svldff1uw_gather_[{3}]offset_{d}", "dPYx", "lUl", [IsGatherLoad, IsByteIndexed, IsZExtReturn], MemEltTyInt32, "aarch64_sve_ldff1_gather">; 165 166def SVLDFF1_GATHER_32B_OFFSETS_S : MInst<"svldff1_gather_[{3}]offset[_{d}]", "dPcx", "iUif", [IsGatherLoad, IsByteIndexed], MemEltTyDefault, "aarch64_sve_ldff1_gather_sxtw">; 167def SVLDFF1SB_GATHER_32B_OFFSETS_S : MInst<"svldff1sb_gather_[{3}]offset_{d}", "dPSx", "iUi", [IsGatherLoad, IsByteIndexed], MemEltTyInt8, "aarch64_sve_ldff1_gather_sxtw">; 168def SVLDFF1UB_GATHER_32B_OFFSETS_S : MInst<"svldff1ub_gather_[{3}]offset_{d}", "dPWx", "iUi", [IsGatherLoad, IsByteIndexed, IsZExtReturn], MemEltTyInt8, "aarch64_sve_ldff1_gather_sxtw">; 169def SVLDFF1SH_GATHER_32B_OFFSETS_S : MInst<"svldff1sh_gather_[{3}]offset_{d}", "dPTx", "iUi", [IsGatherLoad, IsByteIndexed], MemEltTyInt16, "aarch64_sve_ldff1_gather_sxtw">; 170def SVLDFF1UH_GATHER_32B_OFFSETS_S : MInst<"svldff1uh_gather_[{3}]offset_{d}", "dPXx", "iUi", [IsGatherLoad, IsByteIndexed, IsZExtReturn], MemEltTyInt16, "aarch64_sve_ldff1_gather_sxtw">; 171 172// First-faulting load one vector (scalar base, unsigned vector offset in bytes) 173def SVLDFF1_GATHER_64B_OFFSETS_U : MInst<"svldff1_gather_[{3}]offset[_{d}]", "dPcu", "lUld", [IsGatherLoad, IsByteIndexed], MemEltTyDefault, "aarch64_sve_ldff1_gather">; 174def SVLDFF1SB_GATHER_64B_OFFSETS_U : MInst<"svldff1sb_gather_[{3}]offset_{d}", "dPSu", "lUl", [IsGatherLoad, IsByteIndexed], MemEltTyInt8, "aarch64_sve_ldff1_gather">; 175def SVLDFF1UB_GATHER_64B_OFFSETS_U : MInst<"svldff1ub_gather_[{3}]offset_{d}", "dPWu", "lUl", [IsGatherLoad, IsByteIndexed, IsZExtReturn], MemEltTyInt8, "aarch64_sve_ldff1_gather">; 176def SVLDFF1SH_GATHER_64B_OFFSETS_U : MInst<"svldff1sh_gather_[{3}]offset_{d}", "dPTu", "lUl", [IsGatherLoad, IsByteIndexed], MemEltTyInt16, "aarch64_sve_ldff1_gather">; 177def SVLDFF1UH_GATHER_64B_OFFSETS_U : MInst<"svldff1uh_gather_[{3}]offset_{d}", "dPXu", "lUl", [IsGatherLoad, IsByteIndexed, IsZExtReturn], MemEltTyInt16, "aarch64_sve_ldff1_gather">; 178def SVLDFF1SW_GATHER_64B_OFFSETS_U : MInst<"svldff1sw_gather_[{3}]offset_{d}", "dPUu", "lUl", [IsGatherLoad, IsByteIndexed], MemEltTyInt32, "aarch64_sve_ldff1_gather">; 179def SVLDFF1UW_GATHER_64B_OFFSETS_U : MInst<"svldff1uw_gather_[{3}]offset_{d}", "dPYu", "lUl", [IsGatherLoad, IsByteIndexed, IsZExtReturn], MemEltTyInt32, "aarch64_sve_ldff1_gather">; 180 181def SVLDFF1_GATHER_32B_OFFSETS_U : MInst<"svldff1_gather_[{3}]offset[_{d}]", "dPcu", "iUif", [IsGatherLoad, IsByteIndexed], MemEltTyDefault, "aarch64_sve_ldff1_gather_uxtw">; 182def SVLDFF1SB_GATHER_32B_OFFSETS_U : MInst<"svldff1sb_gather_[{3}]offset_{d}", "dPSu", "iUi", [IsGatherLoad, IsByteIndexed], MemEltTyInt8, "aarch64_sve_ldff1_gather_uxtw">; 183def SVLDFF1UB_GATHER_32B_OFFSETS_U : MInst<"svldff1ub_gather_[{3}]offset_{d}", "dPWu", "iUi", [IsGatherLoad, IsByteIndexed, IsZExtReturn], MemEltTyInt8, "aarch64_sve_ldff1_gather_uxtw">; 184def SVLDFF1SH_GATHER_32B_OFFSETS_U : MInst<"svldff1sh_gather_[{3}]offset_{d}", "dPTu", "iUi", [IsGatherLoad, IsByteIndexed], MemEltTyInt16, "aarch64_sve_ldff1_gather_uxtw">; 185def SVLDFF1UH_GATHER_32B_OFFSETS_U : MInst<"svldff1uh_gather_[{3}]offset_{d}", "dPXu", "iUi", [IsGatherLoad, IsByteIndexed, IsZExtReturn], MemEltTyInt16, "aarch64_sve_ldff1_gather_uxtw">; 186 187// First-faulting load one vector (vector base, signed scalar offset in bytes) 188def SVLDFF1_GATHER_OFFSET_S : MInst<"svldff1_gather[_{2}base]_offset_{d}", "dPul", "ilUiUlfd", [IsGatherLoad, IsByteIndexed], MemEltTyDefault, "aarch64_sve_ldff1_gather_scalar_offset">; 189def SVLDFF1SB_GATHER_OFFSET_S : MInst<"svldff1sb_gather[_{2}base]_offset_{d}", "dPul", "ilUiUl", [IsGatherLoad, IsByteIndexed], MemEltTyInt8, "aarch64_sve_ldff1_gather_scalar_offset">; 190def SVLDFF1UB_GATHER_OFFSET_S : MInst<"svldff1ub_gather[_{2}base]_offset_{d}", "dPul", "ilUiUl", [IsGatherLoad, IsByteIndexed, IsZExtReturn], MemEltTyInt8, "aarch64_sve_ldff1_gather_scalar_offset">; 191def SVLDFF1SH_GATHER_OFFSET_S : MInst<"svldff1sh_gather[_{2}base]_offset_{d}", "dPul", "ilUiUl", [IsGatherLoad, IsByteIndexed], MemEltTyInt16, "aarch64_sve_ldff1_gather_scalar_offset">; 192def SVLDFF1UH_GATHER_OFFSET_S : MInst<"svldff1uh_gather[_{2}base]_offset_{d}", "dPul", "ilUiUl", [IsGatherLoad, IsByteIndexed, IsZExtReturn], MemEltTyInt16, "aarch64_sve_ldff1_gather_scalar_offset">; 193def SVLDFF1SW_GATHER_OFFSET_S : MInst<"svldff1sw_gather[_{2}base]_offset_{d}", "dPul", "lUl", [IsGatherLoad, IsByteIndexed], MemEltTyInt32, "aarch64_sve_ldff1_gather_scalar_offset">; 194def SVLDFF1UW_GATHER_OFFSET_S : MInst<"svldff1uw_gather[_{2}base]_offset_{d}", "dPul", "lUl", [IsGatherLoad, IsByteIndexed, IsZExtReturn], MemEltTyInt32, "aarch64_sve_ldff1_gather_scalar_offset">; 195 196// First-faulting load one vector (scalar base, signed vector index) 197def SVLDFF1_GATHER_64B_INDICES_S : MInst<"svldff1_gather_[{3}]index[_{d}]", "dPcx", "lUld", [IsGatherLoad], MemEltTyDefault, "aarch64_sve_ldff1_gather_index">; 198def SVLDFF1SH_GATHER_64B_INDICES_S : MInst<"svldff1sh_gather_[{3}]index_{d}", "dPTx", "lUl", [IsGatherLoad], MemEltTyInt16, "aarch64_sve_ldff1_gather_index">; 199def SVLDFF1UH_GATHER_64B_INDICES_S : MInst<"svldff1uh_gather_[{3}]index_{d}", "dPXx", "lUl", [IsGatherLoad, IsZExtReturn], MemEltTyInt16, "aarch64_sve_ldff1_gather_index">; 200def SVLDFF1SW_GATHER_64B_INDICES_S : MInst<"svldff1sw_gather_[{3}]index_{d}", "dPUx", "lUl", [IsGatherLoad], MemEltTyInt32, "aarch64_sve_ldff1_gather_index">; 201def SVLDFF1UW_GATHER_64B_INDICES_S : MInst<"svldff1uw_gather_[{3}]index_{d}", "dPYx", "lUl", [IsGatherLoad, IsZExtReturn], MemEltTyInt32, "aarch64_sve_ldff1_gather_index">; 202 203def SVLDFF1_GATHER_32B_INDICES_S : MInst<"svldff1_gather_[{3}]index[_{d}]", "dPcx", "iUif", [IsGatherLoad], MemEltTyDefault, "aarch64_sve_ldff1_gather_sxtw_index">; 204def SVLDFF1SH_GATHER_32B_INDICES_S : MInst<"svldff1sh_gather_[{3}]index_{d}", "dPTx", "iUi", [IsGatherLoad], MemEltTyInt16, "aarch64_sve_ldff1_gather_sxtw_index">; 205def SVLDFF1UH_GATHER_32B_INDICES_S : MInst<"svldff1uh_gather_[{3}]index_{d}", "dPXx", "iUi", [IsGatherLoad, IsZExtReturn], MemEltTyInt16, "aarch64_sve_ldff1_gather_sxtw_index">; 206 207// First-faulting load one vector (scalar base, unsigned vector index) 208def SVLDFF1_GATHER_64B_INDICES_U : MInst<"svldff1_gather_[{3}]index[_{d}]", "dPcu", "lUld", [IsGatherLoad], MemEltTyDefault, "aarch64_sve_ldff1_gather_index">; 209def SVLDFF1SH_GATHER_64B_INDICES_U : MInst<"svldff1sh_gather_[{3}]index_{d}", "dPTu", "lUl", [IsGatherLoad], MemEltTyInt16, "aarch64_sve_ldff1_gather_index">; 210def SVLDFF1UH_GATHER_64B_INDICES_U : MInst<"svldff1uh_gather_[{3}]index_{d}", "dPXu", "lUl", [IsGatherLoad, IsZExtReturn], MemEltTyInt16, "aarch64_sve_ldff1_gather_index">; 211def SVLDFF1SW_GATHER_64B_INDICES_U : MInst<"svldff1sw_gather_[{3}]index_{d}", "dPUu", "lUl", [IsGatherLoad], MemEltTyInt32, "aarch64_sve_ldff1_gather_index">; 212def SVLDFF1UW_GATHER_64B_INDICES_U : MInst<"svldff1uw_gather_[{3}]index_{d}", "dPYu", "lUl", [IsGatherLoad, IsZExtReturn], MemEltTyInt32, "aarch64_sve_ldff1_gather_index">; 213 214def SVLDFF1_GATHER_32B_INDICES_U : MInst<"svldff1_gather_[{3}]index[_{d}]", "dPcu", "iUif", [IsGatherLoad], MemEltTyDefault, "aarch64_sve_ldff1_gather_uxtw_index">; 215def SVLDFF1SH_GATHER_32B_INDICES_U : MInst<"svldff1sh_gather_[{3}]index_{d}", "dPTu", "iUi", [IsGatherLoad], MemEltTyInt16, "aarch64_sve_ldff1_gather_uxtw_index">; 216def SVLDFF1UH_GATHER_32B_INDICES_U : MInst<"svldff1uh_gather_[{3}]index_{d}", "dPXu", "iUi", [IsGatherLoad, IsZExtReturn], MemEltTyInt16, "aarch64_sve_ldff1_gather_uxtw_index">; 217 218// First-faulting load one vector (vector base, signed scalar index) 219def SVLDFF1_GATHER_INDEX_S : MInst<"svldff1_gather[_{2}base]_index_{d}", "dPul", "ilUiUlfd", [IsGatherLoad], MemEltTyDefault, "aarch64_sve_ldff1_gather_scalar_offset">; 220def SVLDFF1SH_GATHER_INDEX_S : MInst<"svldff1sh_gather[_{2}base]_index_{d}", "dPul", "ilUiUl", [IsGatherLoad], MemEltTyInt16, "aarch64_sve_ldff1_gather_scalar_offset">; 221def SVLDFF1UH_GATHER_INDEX_S : MInst<"svldff1uh_gather[_{2}base]_index_{d}", "dPul", "ilUiUl", [IsGatherLoad, IsZExtReturn], MemEltTyInt16, "aarch64_sve_ldff1_gather_scalar_offset">; 222def SVLDFF1SW_GATHER_INDEX_S : MInst<"svldff1sw_gather[_{2}base]_index_{d}", "dPul", "lUl", [IsGatherLoad], MemEltTyInt32, "aarch64_sve_ldff1_gather_scalar_offset">; 223def SVLDFF1UW_GATHER_INDEX_S : MInst<"svldff1uw_gather[_{2}base]_index_{d}", "dPul", "lUl", [IsGatherLoad, IsZExtReturn], MemEltTyInt32, "aarch64_sve_ldff1_gather_scalar_offset">; 224 225// Non-faulting load one vector (scalar base) 226def SVLDNF1 : MInst<"svldnf1[_{2}]", "dPc", "csilUcUsUiUlhfd", [IsLoad], MemEltTyDefault, "aarch64_sve_ldnf1">; 227def SVLDNF1SB : MInst<"svldnf1sb_{d}", "dPS", "silUsUiUl", [IsLoad], MemEltTyInt8, "aarch64_sve_ldnf1">; 228def SVLDNF1UB : MInst<"svldnf1ub_{d}", "dPW", "silUsUiUl", [IsLoad, IsZExtReturn], MemEltTyInt8, "aarch64_sve_ldnf1">; 229def SVLDNF1SH : MInst<"svldnf1sh_{d}", "dPT", "ilUiUl", [IsLoad], MemEltTyInt16, "aarch64_sve_ldnf1">; 230def SVLDNF1UH : MInst<"svldnf1uh_{d}", "dPX", "ilUiUl", [IsLoad, IsZExtReturn], MemEltTyInt16, "aarch64_sve_ldnf1">; 231def SVLDNF1SW : MInst<"svldnf1sw_{d}", "dPU", "lUl", [IsLoad], MemEltTyInt32, "aarch64_sve_ldnf1">; 232def SVLDNF1UW : MInst<"svldnf1uw_{d}", "dPY", "lUl", [IsLoad, IsZExtReturn], MemEltTyInt32, "aarch64_sve_ldnf1">; 233 234// Non-faulting load one vector (scalar base, VL displacement) 235def SVLDNF1_VNUM : MInst<"svldnf1_vnum[_{2}]", "dPcl", "csilUcUsUiUlhfd", [IsLoad], MemEltTyDefault, "aarch64_sve_ldnf1">; 236def SVLDNF1SB_VNUM : MInst<"svldnf1sb_vnum_{d}", "dPSl", "silUsUiUl", [IsLoad], MemEltTyInt8, "aarch64_sve_ldnf1">; 237def SVLDNF1UB_VNUM : MInst<"svldnf1ub_vnum_{d}", "dPWl", "silUsUiUl", [IsLoad, IsZExtReturn], MemEltTyInt8, "aarch64_sve_ldnf1">; 238def SVLDNF1SH_VNUM : MInst<"svldnf1sh_vnum_{d}", "dPTl", "ilUiUl", [IsLoad], MemEltTyInt16, "aarch64_sve_ldnf1">; 239def SVLDNF1UH_VNUM : MInst<"svldnf1uh_vnum_{d}", "dPXl", "ilUiUl", [IsLoad, IsZExtReturn], MemEltTyInt16, "aarch64_sve_ldnf1">; 240def SVLDNF1SW_VNUM : MInst<"svldnf1sw_vnum_{d}", "dPUl", "lUl", [IsLoad], MemEltTyInt32, "aarch64_sve_ldnf1">; 241def SVLDNF1UW_VNUM : MInst<"svldnf1uw_vnum_{d}", "dPYl", "lUl", [IsLoad, IsZExtReturn], MemEltTyInt32, "aarch64_sve_ldnf1">; 242} // let SVETargetGuard = "sve", SMETargetGuard = InvalidMode 243 244let SVETargetGuard = "sve,bf16", SMETargetGuard = InvalidMode in { 245 def SVLDNF1_BF : MInst<"svldnf1[_{2}]", "dPc", "b", [IsLoad], MemEltTyDefault, "aarch64_sve_ldnf1">; 246 def SVLDNF1_VNUM_BF : MInst<"svldnf1_vnum[_{2}]", "dPcl", "b", [IsLoad], MemEltTyDefault, "aarch64_sve_ldnf1">; 247} 248 249// Load one vector, unextended load, non-temporal (scalar base) 250def SVLDNT1 : MInst<"svldnt1[_{2}]", "dPc", "csilUcUsUiUlhfd", [IsLoad, VerifyRuntimeMode], MemEltTyDefault, "aarch64_sve_ldnt1">; 251 252// Load one vector, unextended load, non-temporal (scalar base, VL displacement) 253def SVLDNT1_VNUM : MInst<"svldnt1_vnum[_{2}]", "dPcl", "csilUcUsUiUlhfd", [IsLoad, VerifyRuntimeMode], MemEltTyDefault, "aarch64_sve_ldnt1">; 254 255let SVETargetGuard = "sve,bf16", SMETargetGuard = "sme,bf16" in { 256 def SVLDNT1_BF : MInst<"svldnt1[_{2}]", "dPc", "b", [IsLoad, VerifyRuntimeMode], MemEltTyDefault, "aarch64_sve_ldnt1">; 257 def SVLDNT1_VNUM_BF : MInst<"svldnt1_vnum[_{2}]", "dPcl", "b", [IsLoad, VerifyRuntimeMode], MemEltTyDefault, "aarch64_sve_ldnt1">; 258} 259 260// Load one quadword and replicate (scalar base) 261def SVLD1RQ : SInst<"svld1rq[_{2}]", "dPc", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_ld1rq", [VerifyRuntimeMode]>; 262 263let SVETargetGuard = "sve,bf16", SMETargetGuard = "sme,bf16" in { 264 def SVLD1RQ_BF : SInst<"svld1rq[_{2}]", "dPc", "b", MergeNone, "aarch64_sve_ld1rq", [VerifyRuntimeMode]>; 265} 266 267multiclass StructLoad<string name, string proto, string i> { 268 def : SInst<name, proto, "csilUcUsUiUlhfd", MergeNone, i, [IsStructLoad, VerifyRuntimeMode]>; 269 let SVETargetGuard = "sve,bf16", SMETargetGuard = "sme,bf16" in { 270 def: SInst<name, proto, "b", MergeNone, i, [IsStructLoad, VerifyRuntimeMode]>; 271 } 272} 273 274// Load N-element structure into N vectors (scalar base) 275defm SVLD2 : StructLoad<"svld2[_{2}]", "2Pc", "aarch64_sve_ld2_sret">; 276defm SVLD3 : StructLoad<"svld3[_{2}]", "3Pc", "aarch64_sve_ld3_sret">; 277defm SVLD4 : StructLoad<"svld4[_{2}]", "4Pc", "aarch64_sve_ld4_sret">; 278 279// Load N-element structure into N vectors (scalar base, VL displacement) 280defm SVLD2_VNUM : StructLoad<"svld2_vnum[_{2}]", "2Pcl", "aarch64_sve_ld2_sret">; 281defm SVLD3_VNUM : StructLoad<"svld3_vnum[_{2}]", "3Pcl", "aarch64_sve_ld3_sret">; 282defm SVLD4_VNUM : StructLoad<"svld4_vnum[_{2}]", "4Pcl", "aarch64_sve_ld4_sret">; 283 284// Load one octoword and replicate (scalar base) 285let SVETargetGuard = "sve,f64mm", SMETargetGuard = InvalidMode in { 286 def SVLD1RO : SInst<"svld1ro[_{2}]", "dPc", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_ld1ro">; 287} 288let SVETargetGuard = "sve,f64mm,bf16", SMETargetGuard = InvalidMode in { 289 def SVLD1RO_BF16 : SInst<"svld1ro[_{2}]", "dPc", "b", MergeNone, "aarch64_sve_ld1ro">; 290} 291 292let SVETargetGuard = "sve,bf16", SMETargetGuard = InvalidMode in { 293 def SVBFMMLA : SInst<"svbfmmla[_{0}]", "MMdd", "b", MergeNone, "aarch64_sve_bfmmla", [IsOverloadNone]>; 294} 295 296let SVETargetGuard = "sve,bf16", SMETargetGuard = "sme,bf16" in { 297 def SVBFDOT : SInst<"svbfdot[_{0}]", "MMdd", "b", MergeNone, "aarch64_sve_bfdot", [IsOverloadNone, VerifyRuntimeMode]>; 298 def SVBFMLALB : SInst<"svbfmlalb[_{0}]", "MMdd", "b", MergeNone, "aarch64_sve_bfmlalb", [IsOverloadNone, VerifyRuntimeMode]>; 299 def SVBFMLALT : SInst<"svbfmlalt[_{0}]", "MMdd", "b", MergeNone, "aarch64_sve_bfmlalt", [IsOverloadNone, VerifyRuntimeMode]>; 300 def SVBFDOT_N : SInst<"svbfdot[_n_{0}]", "MMda", "b", MergeNone, "aarch64_sve_bfdot", [IsOverloadNone, VerifyRuntimeMode]>; 301 def SVBFMLALB_N : SInst<"svbfmlalb[_n_{0}]", "MMda", "b", MergeNone, "aarch64_sve_bfmlalb", [IsOverloadNone, VerifyRuntimeMode]>; 302 def SVBFMLALT_N : SInst<"svbfmlalt[_n_{0}]", "MMda", "b", MergeNone, "aarch64_sve_bfmlalt", [IsOverloadNone, VerifyRuntimeMode]>; 303 def SVBFDOT_LANE : SInst<"svbfdot_lane[_{0}]", "MMddi", "b", MergeNone, "aarch64_sve_bfdot_lane_v2", [IsOverloadNone, VerifyRuntimeMode], [ImmCheck<3, ImmCheck0_3>]>; 304 def SVBFMLALB_LANE : SInst<"svbfmlalb_lane[_{0}]", "MMddi", "b", MergeNone, "aarch64_sve_bfmlalb_lane_v2", [IsOverloadNone, VerifyRuntimeMode], [ImmCheck<3, ImmCheck0_7>]>; 305 def SVBFMLALT_LANE : SInst<"svbfmlalt_lane[_{0}]", "MMddi", "b", MergeNone, "aarch64_sve_bfmlalt_lane_v2", [IsOverloadNone, VerifyRuntimeMode], [ImmCheck<3, ImmCheck0_7>]>; 306} // let SVETargetGuard = "sve,bf16", SMETargetGuard = "sme,bf16" 307 308let SVETargetGuard = "sve2p1", SMETargetGuard = InvalidMode in { 309 // Contiguous zero-extending load to quadword (single vector). 310 def SVLD1UWQ : MInst<"svld1uwq[_{d}]", "dPc", "iUif", [IsLoad], MemEltTyInt32, "aarch64_sve_ld1uwq">; 311 def SVLD1UWQ_VNUM : MInst<"svld1uwq_vnum[_{d}]", "dPcl", "iUif", [IsLoad], MemEltTyInt32, "aarch64_sve_ld1uwq">; 312 313 def SVLD1UDQ : MInst<"svld1udq[_{d}]", "dPc", "lUld", [IsLoad], MemEltTyInt64, "aarch64_sve_ld1udq">; 314 def SVLD1UDQ_VNUM : MInst<"svld1udq_vnum[_{d}]", "dPcl", "lUld", [IsLoad], MemEltTyInt64, "aarch64_sve_ld1udq">; 315 316 // Load one vector (vector base + scalar offset) 317 def SVLD1Q_GATHER_U64BASE_OFFSET : MInst<"svld1q_gather[_{2}base]_offset_{d}", "dPgl", "cUcsUsiUilUlfhdb", [IsGatherLoad, IsByteIndexed], MemEltTyDefault, "aarch64_sve_ld1q_gather_scalar_offset">; 318 def SVLD1Q_GATHER_U64BASE : MInst<"svld1q_gather[_{2}base]_{d}", "dPg", "cUcsUsiUilUlfhdb", [IsGatherLoad, IsByteIndexed], MemEltTyDefault, "aarch64_sve_ld1q_gather_scalar_offset">; 319 320 // Load one vector (scalar base + vector offset) 321 def SVLD1Q_GATHER_U64OFFSET : MInst<"svld1q_gather_[{3}]offset[_{d}]", "dPcg", "cUcsUsiUilUlfhdb", [IsGatherLoad, IsByteIndexed], MemEltTyDefault, "aarch64_sve_ld1q_gather_vector_offset">; 322 323 // Load N-element structure into N vectors (scalar base) 324 defm SVLD2Q : StructLoad<"svld2q[_{2}]", "2Pc", "aarch64_sve_ld2q_sret">; 325 defm SVLD3Q : StructLoad<"svld3q[_{2}]", "3Pc", "aarch64_sve_ld3q_sret">; 326 defm SVLD4Q : StructLoad<"svld4q[_{2}]", "4Pc", "aarch64_sve_ld4q_sret">; 327 328 // Load N-element structure into N vectors (scalar base, VL displacement) 329 defm SVLD2Q_VNUM : StructLoad<"svld2q_vnum[_{2}]", "2Pcl", "aarch64_sve_ld2q_sret">; 330 defm SVLD3Q_VNUM : StructLoad<"svld3q_vnum[_{2}]", "3Pcl", "aarch64_sve_ld3q_sret">; 331 defm SVLD4Q_VNUM : StructLoad<"svld4q_vnum[_{2}]", "4Pcl", "aarch64_sve_ld4q_sret">; 332 333 // Load quadwords (scalar base + vector index) 334 def SVLD1Q_GATHER_INDICES_U : MInst<"svld1q_gather_[{3}]index[_{d}]", "dPcg", "sUsiUilUlbhfd", [IsGatherLoad], MemEltTyDefault, "aarch64_sve_ld1q_gather_index">; 335 336 // Load quadwords (vector base + scalar index) 337 def SVLD1Q_GATHER_INDEX_S : MInst<"svld1q_gather[_{2}base]_index_{d}", "dPgl", "sUsiUilUlbhfd", [IsGatherLoad], MemEltTyDefault, "aarch64_sve_ld1q_gather_scalar_offset">; 338} 339 340//////////////////////////////////////////////////////////////////////////////// 341// Stores 342 343// Store one vector (scalar base) 344def SVST1 : MInst<"svst1[_{d}]", "vPpd", "csilUcUsUiUlhfd", [IsStore, VerifyRuntimeMode], MemEltTyDefault, "aarch64_sve_st1">; 345def SVST1B_S : MInst<"svst1b[_{d}]", "vPAd", "sil", [IsStore, VerifyRuntimeMode], MemEltTyInt8, "aarch64_sve_st1">; 346def SVST1B_U : MInst<"svst1b[_{d}]", "vPEd", "UsUiUl", [IsStore, VerifyRuntimeMode], MemEltTyInt8, "aarch64_sve_st1">; 347def SVST1H_S : MInst<"svst1h[_{d}]", "vPBd", "il", [IsStore, VerifyRuntimeMode], MemEltTyInt16, "aarch64_sve_st1">; 348def SVST1H_U : MInst<"svst1h[_{d}]", "vPFd", "UiUl", [IsStore, VerifyRuntimeMode], MemEltTyInt16, "aarch64_sve_st1">; 349def SVST1W_S : MInst<"svst1w[_{d}]", "vPCd", "l", [IsStore, VerifyRuntimeMode], MemEltTyInt32, "aarch64_sve_st1">; 350def SVST1W_U : MInst<"svst1w[_{d}]", "vPGd", "Ul", [IsStore, VerifyRuntimeMode], MemEltTyInt32, "aarch64_sve_st1">; 351 352// Store one vector (scalar base, VL displacement) 353def SVST1_VNUM : MInst<"svst1_vnum[_{d}]", "vPpld", "csilUcUsUiUlhfd", [IsStore, VerifyRuntimeMode], MemEltTyDefault, "aarch64_sve_st1">; 354def SVST1B_VNUM_S : MInst<"svst1b_vnum[_{d}]", "vPAld", "sil", [IsStore, VerifyRuntimeMode], MemEltTyInt8, "aarch64_sve_st1">; 355def SVST1B_VNUM_U : MInst<"svst1b_vnum[_{d}]", "vPEld", "UsUiUl", [IsStore, VerifyRuntimeMode], MemEltTyInt8, "aarch64_sve_st1">; 356def SVST1H_VNUM_S : MInst<"svst1h_vnum[_{d}]", "vPBld", "il", [IsStore, VerifyRuntimeMode], MemEltTyInt16, "aarch64_sve_st1">; 357def SVST1H_VNUM_U : MInst<"svst1h_vnum[_{d}]", "vPFld", "UiUl", [IsStore, VerifyRuntimeMode], MemEltTyInt16, "aarch64_sve_st1">; 358def SVST1W_VNUM_S : MInst<"svst1w_vnum[_{d}]", "vPCld", "l", [IsStore, VerifyRuntimeMode], MemEltTyInt32, "aarch64_sve_st1">; 359def SVST1W_VNUM_U : MInst<"svst1w_vnum[_{d}]", "vPGld", "Ul", [IsStore, VerifyRuntimeMode], MemEltTyInt32, "aarch64_sve_st1">; 360 361let SVETargetGuard = "sve,bf16", SMETargetGuard = "sme,bf16" in { 362 def SVST1_BF : MInst<"svst1[_{d}]", "vPpd", "b", [IsStore, VerifyRuntimeMode], MemEltTyDefault, "aarch64_sve_st1">; 363 def SVST1_VNUM_BF : MInst<"svst1_vnum[_{d}]", "vPpld", "b", [IsStore, VerifyRuntimeMode], MemEltTyDefault, "aarch64_sve_st1">; 364} 365 366let SVETargetGuard = "sve", SMETargetGuard = InvalidMode in { 367// Store one vector (vector base) 368def SVST1_SCATTER_BASES_U : MInst<"svst1_scatter[_{2}base_{d}]", "vPud", "ilUiUlfd", [IsScatterStore], MemEltTyDefault, "aarch64_sve_st1_scatter_scalar_offset">; 369def SVST1B_SCATTER_BASES_U : MInst<"svst1b_scatter[_{2}base_{d}]", "vPud", "ilUiUl", [IsScatterStore], MemEltTyInt8, "aarch64_sve_st1_scatter_scalar_offset">; 370def SVST1H_SCATTER_BASES_U : MInst<"svst1h_scatter[_{2}base_{d}]", "vPud", "ilUiUl", [IsScatterStore], MemEltTyInt16, "aarch64_sve_st1_scatter_scalar_offset">; 371def SVST1W_SCATTER_BASES_U : MInst<"svst1w_scatter[_{2}base_{d}]", "vPud", "lUl", [IsScatterStore], MemEltTyInt32, "aarch64_sve_st1_scatter_scalar_offset">; 372 373// Store one vector (scalar base, signed vector offset in bytes) 374def SVST1_SCATTER_64B_OFFSETS_S : MInst<"svst1_scatter_[{3}]offset[_{d}]", "vPpxd", "lUld", [IsScatterStore, IsByteIndexed], MemEltTyDefault, "aarch64_sve_st1_scatter">; 375def SVST1B_SCATTER_64B_OFFSETS_SS : MInst<"svst1b_scatter_[{3}]offset[_{d}]", "vPAxd", "l", [IsScatterStore, IsByteIndexed], MemEltTyInt8, "aarch64_sve_st1_scatter">; 376def SVST1B_SCATTER_64B_OFFSETS_SU : MInst<"svst1b_scatter_[{3}]offset[_{d}]", "vPExd", "Ul", [IsScatterStore, IsByteIndexed], MemEltTyInt8, "aarch64_sve_st1_scatter">; 377def SVST1H_SCATTER_64B_OFFSETS_SS : MInst<"svst1h_scatter_[{3}]offset[_{d}]", "vPBxd", "l", [IsScatterStore, IsByteIndexed], MemEltTyInt16, "aarch64_sve_st1_scatter">; 378def SVST1H_SCATTER_64B_OFFSETS_SU : MInst<"svst1h_scatter_[{3}]offset[_{d}]", "vPFxd", "Ul", [IsScatterStore, IsByteIndexed], MemEltTyInt16, "aarch64_sve_st1_scatter">; 379def SVST1W_SCATTER_64B_OFFSETS_SS : MInst<"svst1w_scatter_[{3}]offset[_{d}]", "vPCxd", "l", [IsScatterStore, IsByteIndexed], MemEltTyInt32, "aarch64_sve_st1_scatter">; 380def SVST1W_SCATTER_64B_OFFSETS_SU : MInst<"svst1w_scatter_[{3}]offset[_{d}]", "vPGxd", "Ul", [IsScatterStore, IsByteIndexed], MemEltTyInt32, "aarch64_sve_st1_scatter">; 381 382def SVST1_SCATTER_32B_OFFSETS_S : MInst<"svst1_scatter_[{3}]offset[_{d}]", "vPpxd", "iUif", [IsScatterStore, IsByteIndexed], MemEltTyDefault, "aarch64_sve_st1_scatter_sxtw">; 383def SVST1B_SCATTER_32B_OFFSETS_SS : MInst<"svst1b_scatter_[{3}]offset[_{d}]", "vPAxd", "i", [IsScatterStore, IsByteIndexed], MemEltTyInt8, "aarch64_sve_st1_scatter_sxtw">; 384def SVST1B_SCATTER_32B_OFFSETS_SU : MInst<"svst1b_scatter_[{3}]offset[_{d}]", "vPExd", "Ui", [IsScatterStore, IsByteIndexed], MemEltTyInt8, "aarch64_sve_st1_scatter_sxtw">; 385def SVST1H_SCATTER_32B_OFFSETS_SS : MInst<"svst1h_scatter_[{3}]offset[_{d}]", "vPBxd", "i", [IsScatterStore, IsByteIndexed], MemEltTyInt16, "aarch64_sve_st1_scatter_sxtw">; 386def SVST1H_SCATTER_32B_OFFSETS_SU : MInst<"svst1h_scatter_[{3}]offset[_{d}]", "vPFxd", "Ui", [IsScatterStore, IsByteIndexed], MemEltTyInt16, "aarch64_sve_st1_scatter_sxtw">; 387 388// Store one vector (scalar base, unsigned vector offset in bytes) 389def SVST1_SCATTER_64B_OFFSETS_U : MInst<"svst1_scatter_[{3}]offset[_{d}]", "vPpud", "lUld", [IsScatterStore, IsByteIndexed], MemEltTyDefault, "aarch64_sve_st1_scatter">; 390def SVST1B_SCATTER_64B_OFFSETS_US : MInst<"svst1b_scatter_[{3}]offset[_{d}]", "vPAud", "l", [IsScatterStore, IsByteIndexed], MemEltTyInt8, "aarch64_sve_st1_scatter">; 391def SVST1B_SCATTER_64B_OFFSETS_UU : MInst<"svst1b_scatter_[{3}]offset[_{d}]", "vPEud", "Ul", [IsScatterStore, IsByteIndexed], MemEltTyInt8, "aarch64_sve_st1_scatter">; 392def SVST1H_SCATTER_64B_OFFSETS_US : MInst<"svst1h_scatter_[{3}]offset[_{d}]", "vPBud", "l", [IsScatterStore, IsByteIndexed], MemEltTyInt16, "aarch64_sve_st1_scatter">; 393def SVST1H_SCATTER_64B_OFFSETS_UU : MInst<"svst1h_scatter_[{3}]offset[_{d}]", "vPFud", "Ul", [IsScatterStore, IsByteIndexed], MemEltTyInt16, "aarch64_sve_st1_scatter">; 394def SVST1W_SCATTER_64B_OFFSETS_US : MInst<"svst1w_scatter_[{3}]offset[_{d}]", "vPCud", "l", [IsScatterStore, IsByteIndexed], MemEltTyInt32, "aarch64_sve_st1_scatter">; 395def SVST1W_SCATTER_64B_OFFSETS_UU : MInst<"svst1w_scatter_[{3}]offset[_{d}]", "vPGud", "Ul", [IsScatterStore, IsByteIndexed], MemEltTyInt32, "aarch64_sve_st1_scatter">; 396 397def SVST1_SCATTER_32B_OFFSETS_U : MInst<"svst1_scatter_[{3}]offset[_{d}]", "vPpud", "iUif", [IsScatterStore, IsByteIndexed], MemEltTyDefault, "aarch64_sve_st1_scatter_uxtw">; 398def SVST1B_SCATTER_32B_OFFSETS_US : MInst<"svst1b_scatter_[{3}]offset[_{d}]", "vPAud", "i", [IsScatterStore, IsByteIndexed], MemEltTyInt8, "aarch64_sve_st1_scatter_uxtw">; 399def SVST1B_SCATTER_32B_OFFSETS_UU : MInst<"svst1b_scatter_[{3}]offset[_{d}]", "vPEud", "Ui", [IsScatterStore, IsByteIndexed], MemEltTyInt8, "aarch64_sve_st1_scatter_uxtw">; 400def SVST1H_SCATTER_32B_OFFSETS_US : MInst<"svst1h_scatter_[{3}]offset[_{d}]", "vPBud", "i", [IsScatterStore, IsByteIndexed], MemEltTyInt16, "aarch64_sve_st1_scatter_uxtw">; 401def SVST1H_SCATTER_32B_OFFSETS_UU : MInst<"svst1h_scatter_[{3}]offset[_{d}]", "vPFud", "Ui", [IsScatterStore, IsByteIndexed], MemEltTyInt16, "aarch64_sve_st1_scatter_uxtw">; 402 403// Store one vector (vector base, signed scalar offset in bytes) 404def SVST1_SCATTER_OFFSET_S : MInst<"svst1_scatter[_{2}base]_offset[_{d}]", "vPuld", "ilUiUlfd", [IsScatterStore, IsByteIndexed], MemEltTyDefault, "aarch64_sve_st1_scatter_scalar_offset">; 405def SVST1B_SCATTER_OFFSET_S : MInst<"svst1b_scatter[_{2}base]_offset[_{d}]", "vPuld", "ilUiUl", [IsScatterStore, IsByteIndexed], MemEltTyInt8, "aarch64_sve_st1_scatter_scalar_offset">; 406def SVST1H_SCATTER_OFFSET_S : MInst<"svst1h_scatter[_{2}base]_offset[_{d}]", "vPuld", "ilUiUl", [IsScatterStore, IsByteIndexed], MemEltTyInt16, "aarch64_sve_st1_scatter_scalar_offset">; 407def SVST1W_SCATTER_OFFSET_S : MInst<"svst1w_scatter[_{2}base]_offset[_{d}]", "vPuld", "lUl", [IsScatterStore, IsByteIndexed], MemEltTyInt32, "aarch64_sve_st1_scatter_scalar_offset">; 408 409// Store one vector (scalar base, signed vector index) 410def SVST1_SCATTER_64B_INDICES_S : MInst<"svst1_scatter_[{3}]index[_{d}]", "vPpxd", "lUld", [IsScatterStore], MemEltTyDefault, "aarch64_sve_st1_scatter_index">; 411def SVST1H_SCATTER_64B_INDICES_SS : MInst<"svst1h_scatter_[{3}]index[_{d}]", "vPBxd", "l", [IsScatterStore], MemEltTyInt16, "aarch64_sve_st1_scatter_index">; 412def SVST1H_SCATTER_64B_INDICES_SU : MInst<"svst1h_scatter_[{3}]index[_{d}]", "vPFxd", "Ul", [IsScatterStore], MemEltTyInt16, "aarch64_sve_st1_scatter_index">; 413def SVST1W_SCATTER_64B_INDICES_SS : MInst<"svst1w_scatter_[{3}]index[_{d}]", "vPCxd", "l", [IsScatterStore], MemEltTyInt32, "aarch64_sve_st1_scatter_index">; 414def SVST1W_SCATTER_64B_INDICES_SU : MInst<"svst1w_scatter_[{3}]index[_{d}]", "vPGxd", "Ul", [IsScatterStore], MemEltTyInt32, "aarch64_sve_st1_scatter_index">; 415 416def SVST1_SCATTER_32B_INDICES_S : MInst<"svst1_scatter_[{3}]index[_{d}]", "vPpxd", "iUif", [IsScatterStore], MemEltTyDefault, "aarch64_sve_st1_scatter_sxtw_index">; 417def SVST1H_SCATTER_32B_INDICES_SS : MInst<"svst1h_scatter_[{3}]index[_{d}]", "vPBxd", "i", [IsScatterStore], MemEltTyInt16, "aarch64_sve_st1_scatter_sxtw_index">; 418def SVST1H_SCATTER_32B_INDICES_SU : MInst<"svst1h_scatter_[{3}]index[_{d}]", "vPFxd", "Ui", [IsScatterStore], MemEltTyInt16, "aarch64_sve_st1_scatter_sxtw_index">; 419 420// Store one vector (scalar base, unsigned vector index) 421def SVST1_SCATTER_64B_INDICES_U : MInst<"svst1_scatter_[{3}]index[_{d}]", "vPpud", "lUld", [IsScatterStore], MemEltTyDefault, "aarch64_sve_st1_scatter_index">; 422def SVST1H_SCATTER_64B_INDICES_US : MInst<"svst1h_scatter_[{3}]index[_{d}]", "vPBud", "l", [IsScatterStore], MemEltTyInt16, "aarch64_sve_st1_scatter_index">; 423def SVST1H_SCATTER_64B_INDICES_UU : MInst<"svst1h_scatter_[{3}]index[_{d}]", "vPFud", "Ul", [IsScatterStore], MemEltTyInt16, "aarch64_sve_st1_scatter_index">; 424def SVST1W_SCATTER_64B_INDICES_US : MInst<"svst1w_scatter_[{3}]index[_{d}]", "vPCud", "l", [IsScatterStore], MemEltTyInt32, "aarch64_sve_st1_scatter_index">; 425def SVST1W_SCATTER_64B_INDICES_UU : MInst<"svst1w_scatter_[{3}]index[_{d}]", "vPGud", "Ul", [IsScatterStore], MemEltTyInt32, "aarch64_sve_st1_scatter_index">; 426 427def SVST1_SCATTER_32B_INDICES_U : MInst<"svst1_scatter_[{3}]index[_{d}]", "vPpud", "iUif", [IsScatterStore], MemEltTyDefault, "aarch64_sve_st1_scatter_uxtw_index">; 428def SVST1H_SCATTER_32B_INDICES_US : MInst<"svst1h_scatter_[{3}]index[_{d}]", "vPBud", "i", [IsScatterStore], MemEltTyInt16, "aarch64_sve_st1_scatter_uxtw_index">; 429def SVST1H_SCATTER_32B_INDICES_UU : MInst<"svst1h_scatter_[{3}]index[_{d}]", "vPFud", "Ui", [IsScatterStore], MemEltTyInt16, "aarch64_sve_st1_scatter_uxtw_index">; 430 431// Store one vector (vector base, signed scalar index) 432def SVST1_SCATTER_INDEX_S : MInst<"svst1_scatter[_{2}base]_index[_{d}]", "vPuld", "ilUiUlfd", [IsScatterStore], MemEltTyDefault, "aarch64_sve_st1_scatter_scalar_offset">; 433def SVST1H_SCATTER_INDEX_S : MInst<"svst1h_scatter[_{2}base]_index[_{d}]", "vPuld", "ilUiUl", [IsScatterStore], MemEltTyInt16, "aarch64_sve_st1_scatter_scalar_offset">; 434def SVST1W_SCATTER_INDEX_S : MInst<"svst1w_scatter[_{2}base]_index[_{d}]", "vPuld", "lUl", [IsScatterStore], MemEltTyInt32, "aarch64_sve_st1_scatter_scalar_offset">; 435} // let SVETargetGuard = "sve" 436 437multiclass StructStore<string name, string proto, string i> { 438 def : SInst<name, proto, "csilUcUsUiUlhfd", MergeNone, i, [IsStructStore, VerifyRuntimeMode]>; 439 let SVETargetGuard = "sve,bf16", SMETargetGuard = "sme,bf16" in { 440 def: SInst<name, proto, "b", MergeNone, i, [IsStructStore, VerifyRuntimeMode]>; 441 } 442} 443// Store N vectors into N-element structure (scalar base) 444defm SVST2 : StructStore<"svst2[_{d}]", "vPp2", "aarch64_sve_st2">; 445defm SVST3 : StructStore<"svst3[_{d}]", "vPp3", "aarch64_sve_st3">; 446defm SVST4 : StructStore<"svst4[_{d}]", "vPp4", "aarch64_sve_st4">; 447 448// Store N vectors into N-element structure (scalar base, VL displacement) 449defm SVST2_VNUM : StructStore<"svst2_vnum[_{d}]", "vPpl2", "aarch64_sve_st2">; 450defm SVST3_VNUM : StructStore<"svst3_vnum[_{d}]", "vPpl3", "aarch64_sve_st3">; 451defm SVST4_VNUM : StructStore<"svst4_vnum[_{d}]", "vPpl4", "aarch64_sve_st4">; 452 453// Store one vector, with no truncation, non-temporal (scalar base) 454def SVSTNT1 : MInst<"svstnt1[_{d}]", "vPpd", "csilUcUsUiUlhfd", [IsStore, VerifyRuntimeMode], MemEltTyDefault, "aarch64_sve_stnt1">; 455 456// Store one vector, with no truncation, non-temporal (scalar base, VL displacement) 457def SVSTNT1_VNUM : MInst<"svstnt1_vnum[_{d}]", "vPpld", "csilUcUsUiUlhfd", [IsStore, VerifyRuntimeMode], MemEltTyDefault, "aarch64_sve_stnt1">; 458 459let SVETargetGuard = "sve,bf16", SMETargetGuard = "sme,bf16" in { 460 def SVSTNT1_BF : MInst<"svstnt1[_{d}]", "vPpd", "b", [IsStore, VerifyRuntimeMode], MemEltTyDefault, "aarch64_sve_stnt1">; 461 def SVSTNT1_VNUM_BF : MInst<"svstnt1_vnum[_{d}]", "vPpld", "b", [IsStore, VerifyRuntimeMode], MemEltTyDefault, "aarch64_sve_stnt1">; 462} 463 464let SVETargetGuard = "sve2p1", SMETargetGuard = InvalidMode in { 465 // Contiguous truncating store from quadword (single vector). 466 def SVST1UWQ : MInst<"svst1wq[_{d}]", "vPpd", "iUif", [IsStore], MemEltTyInt32, "aarch64_sve_st1wq">; 467 def SVST1UWQ_VNUM : MInst<"svst1wq_vnum[_{d}]", "vPpld", "iUif", [IsStore], MemEltTyInt32, "aarch64_sve_st1wq">; 468 469 def SVST1UDQ : MInst<"svst1dq[_{d}]", "vPpd", "lUld", [IsStore], MemEltTyInt64, "aarch64_sve_st1dq">; 470 def SVST1UDQ_VNUM : MInst<"svst1dq_vnum[_{d}]", "vPpld", "lUld", [IsStore], MemEltTyInt64, "aarch64_sve_st1dq">; 471 472 // Store one vector (vector base + scalar offset) 473 def SVST1Q_SCATTER_U64BASE_OFFSET : MInst<"svst1q_scatter[_{2}base]_offset[_{d}]", "vPgld", "cUcsUsiUilUlfhdb", [IsScatterStore, IsByteIndexed], MemEltTyDefault, "aarch64_sve_st1q_scatter_scalar_offset">; 474 def SVST1Q_SCATTER_U64BASE : MInst<"svst1q_scatter[_{2}base][_{d}]", "vPgd", "cUcsUsiUilUlfhdb", [IsScatterStore, IsByteIndexed], MemEltTyDefault, "aarch64_sve_st1q_scatter_scalar_offset">; 475 476 // Store one vector (scalar base + vector offset) 477 def SVST1Q_SCATTER_OFFSETS_U : MInst<"svst1q_scatter_[{3}]offset[_{d}]", "vPpgd", "cUcsUsiUilUlfhdb", [IsScatterStore, IsByteIndexed], MemEltTyDefault, "aarch64_sve_st1q_scatter_vector_offset">; 478 def SVST1Q_SCATTER_OFFSETS_S : MInst<"svst1q_scatter_[{3}]offset[_{d}]", "vPp#d", "cUcsUsiUilUlfhdb", [IsScatterStore, IsByteIndexed], MemEltTyDefault, "aarch64_sve_st1q_scatter_vector_offset">; 479 480 // Store N vectors into N-element structure (scalar base) 481 defm SVST2Q : StructStore<"svst2q[_{d}]", "vPc2", "aarch64_sve_st2q">; 482 defm SVST3Q : StructStore<"svst3q[_{d}]", "vPc3", "aarch64_sve_st3q">; 483 defm SVST4Q : StructStore<"svst4q[_{d}]", "vPc4", "aarch64_sve_st4q">; 484 485 // Store N vectors into N-element structure (scalar base, VL displacement) 486 defm SVST2Q_VNUM : StructStore<"svst2q_vnum[_{d}]", "vPcl2", "aarch64_sve_st2q">; 487 defm SVST3Q_VNUM : StructStore<"svst3q_vnum[_{d}]", "vPcl3", "aarch64_sve_st3q">; 488 defm SVST4Q_VNUM : StructStore<"svst4q_vnum[_{d}]", "vPcl4", "aarch64_sve_st4q">; 489 490 // Scatter store quadwords (scalar base + vector index) 491 def SVST1Q_SCATTER_INDICES_U : MInst<"svst1q_scatter_[{3}]index[_{d}]", "vPpgd", "sUsiUilUlbhfd", [IsScatterStore], MemEltTyDefault, "aarch64_sve_st1q_scatter_index">; 492 def SVST1Q_SCATTER_INDICES_S : MInst<"svst1q_scatter_[{3}]index[_{d}]", "vPp#d", "sUsiUilUlbhfd", [IsScatterStore], MemEltTyDefault, "aarch64_sve_st1q_scatter_index">; 493 494 // Scatter store quadwords (vector base + scalar index) 495 def SVST1Q_SCATTER_INDEX_S : MInst<"svst1q_scatter[_{2}base]_index[_{d}]", "vPgld", "sUsiUilUlbhfd", [IsScatterStore], MemEltTyDefault, "aarch64_sve_st1q_scatter_scalar_offset">; 496} 497 498//////////////////////////////////////////////////////////////////////////////// 499// Prefetches 500 501// Prefetch (Scalar base) 502def SVPRFB : MInst<"svprfb", "vPQJ", "c", [IsPrefetch, VerifyRuntimeMode], MemEltTyInt8, "aarch64_sve_prf">; 503def SVPRFH : MInst<"svprfh", "vPQJ", "s", [IsPrefetch, VerifyRuntimeMode], MemEltTyInt16, "aarch64_sve_prf">; 504def SVPRFW : MInst<"svprfw", "vPQJ", "i", [IsPrefetch, VerifyRuntimeMode], MemEltTyInt32, "aarch64_sve_prf">; 505def SVPRFD : MInst<"svprfd", "vPQJ", "l", [IsPrefetch, VerifyRuntimeMode], MemEltTyInt64, "aarch64_sve_prf">; 506 507// Prefetch (Scalar base, VL displacement) 508def SVPRFB_VNUM : MInst<"svprfb_vnum", "vPQlJ", "c", [IsPrefetch, VerifyRuntimeMode], MemEltTyInt8, "aarch64_sve_prf">; 509def SVPRFH_VNUM : MInst<"svprfh_vnum", "vPQlJ", "s", [IsPrefetch, VerifyRuntimeMode], MemEltTyInt16, "aarch64_sve_prf">; 510def SVPRFW_VNUM : MInst<"svprfw_vnum", "vPQlJ", "i", [IsPrefetch, VerifyRuntimeMode], MemEltTyInt32, "aarch64_sve_prf">; 511def SVPRFD_VNUM : MInst<"svprfd_vnum", "vPQlJ", "l", [IsPrefetch, VerifyRuntimeMode], MemEltTyInt64, "aarch64_sve_prf">; 512 513let SVETargetGuard = "sve", SMETargetGuard = InvalidMode in { 514// Prefetch (Vector bases) 515def SVPRFB_GATHER_BASES : MInst<"svprfb_gather[_{2}base]", "vPdJ", "UiUl", [IsGatherPrefetch], MemEltTyInt8, "aarch64_sve_prfb_gather_scalar_offset">; 516def SVPRFH_GATHER_BASES : MInst<"svprfh_gather[_{2}base]", "vPdJ", "UiUl", [IsGatherPrefetch], MemEltTyInt16, "aarch64_sve_prfh_gather_scalar_offset">; 517def SVPRFW_GATHER_BASES : MInst<"svprfw_gather[_{2}base]", "vPdJ", "UiUl", [IsGatherPrefetch], MemEltTyInt32, "aarch64_sve_prfw_gather_scalar_offset">; 518def SVPRFD_GATHER_BASES : MInst<"svprfd_gather[_{2}base]", "vPdJ", "UiUl", [IsGatherPrefetch], MemEltTyInt64, "aarch64_sve_prfd_gather_scalar_offset">; 519 520// Prefetch (Scalar base, Vector offsets) 521def SVPRFB_GATHER_32B_OFFSETS_S : MInst<"svprfb_gather_[{3}]offset", "vPQdJ", "i", [IsGatherPrefetch], MemEltTyInt8, "aarch64_sve_prfb_gather_sxtw_index">; 522def SVPRFH_GATHER_32B_OFFSETS_S : MInst<"svprfh_gather_[{3}]index", "vPQdJ", "i", [IsGatherPrefetch], MemEltTyInt16, "aarch64_sve_prfh_gather_sxtw_index">; 523def SVPRFW_GATHER_32B_OFFSETS_S : MInst<"svprfw_gather_[{3}]index", "vPQdJ", "i", [IsGatherPrefetch], MemEltTyInt32, "aarch64_sve_prfw_gather_sxtw_index">; 524def SVPRFD_GATHER_32B_OFFSETS_S : MInst<"svprfd_gather_[{3}]index", "vPQdJ", "i", [IsGatherPrefetch], MemEltTyInt64, "aarch64_sve_prfd_gather_sxtw_index">; 525 526def SVPRFB_GATHER_64B_OFFSETS_S : MInst<"svprfb_gather_[{3}]offset", "vPQdJ", "l", [IsGatherPrefetch], MemEltTyInt8, "aarch64_sve_prfb_gather_index">; 527def SVPRFH_GATHER_64B_OFFSETS_S : MInst<"svprfh_gather_[{3}]index", "vPQdJ", "l", [IsGatherPrefetch], MemEltTyInt16, "aarch64_sve_prfh_gather_index">; 528def SVPRFW_GATHER_64B_OFFSETS_S : MInst<"svprfw_gather_[{3}]index", "vPQdJ", "l", [IsGatherPrefetch], MemEltTyInt32, "aarch64_sve_prfw_gather_index">; 529def SVPRFD_GATHER_64B_OFFSETS_S : MInst<"svprfd_gather_[{3}]index", "vPQdJ", "l", [IsGatherPrefetch], MemEltTyInt64, "aarch64_sve_prfd_gather_index">; 530 531def SVPRFB_GATHER_32B_OFFSETS_U : MInst<"svprfb_gather_[{3}]offset", "vPQdJ", "Ui", [IsGatherPrefetch], MemEltTyInt8, "aarch64_sve_prfb_gather_uxtw_index">; 532def SVPRFH_GATHER_32B_OFFSETS_U : MInst<"svprfh_gather_[{3}]index", "vPQdJ", "Ui", [IsGatherPrefetch], MemEltTyInt16, "aarch64_sve_prfh_gather_uxtw_index">; 533def SVPRFW_GATHER_32B_OFFSETS_U : MInst<"svprfw_gather_[{3}]index", "vPQdJ", "Ui", [IsGatherPrefetch], MemEltTyInt32, "aarch64_sve_prfw_gather_uxtw_index">; 534def SVPRFD_GATHER_32B_OFFSETS_U : MInst<"svprfd_gather_[{3}]index", "vPQdJ", "Ui", [IsGatherPrefetch], MemEltTyInt64, "aarch64_sve_prfd_gather_uxtw_index">; 535 536def SVPRFB_GATHER_64B_OFFSETS_U : MInst<"svprfb_gather_[{3}]offset", "vPQdJ", "Ul", [IsGatherPrefetch], MemEltTyInt8, "aarch64_sve_prfb_gather_index">; 537def SVPRFH_GATHER_64B_OFFSETS_U : MInst<"svprfh_gather_[{3}]index", "vPQdJ", "Ul", [IsGatherPrefetch], MemEltTyInt16, "aarch64_sve_prfh_gather_index">; 538def SVPRFW_GATHER_64B_OFFSETS_U : MInst<"svprfw_gather_[{3}]index", "vPQdJ", "Ul", [IsGatherPrefetch], MemEltTyInt32, "aarch64_sve_prfw_gather_index">; 539def SVPRFD_GATHER_64B_OFFSETS_U : MInst<"svprfd_gather_[{3}]index", "vPQdJ", "Ul", [IsGatherPrefetch], MemEltTyInt64, "aarch64_sve_prfd_gather_index">; 540 541// Prefetch (Vector bases, scalar offset) 542def SVPRFB_GATHER_BASES_OFFSET : MInst<"svprfb_gather[_{2}base]_offset", "vPdlJ", "UiUl", [IsGatherPrefetch], MemEltTyInt8, "aarch64_sve_prfb_gather_scalar_offset">; 543def SVPRFH_GATHER_BASES_OFFSET : MInst<"svprfh_gather[_{2}base]_index", "vPdlJ", "UiUl", [IsGatherPrefetch], MemEltTyInt16, "aarch64_sve_prfh_gather_scalar_offset">; 544def SVPRFW_GATHER_BASES_OFFSET : MInst<"svprfw_gather[_{2}base]_index", "vPdlJ", "UiUl", [IsGatherPrefetch], MemEltTyInt32, "aarch64_sve_prfw_gather_scalar_offset">; 545def SVPRFD_GATHER_BASES_OFFSET : MInst<"svprfd_gather[_{2}base]_index", "vPdlJ", "UiUl", [IsGatherPrefetch], MemEltTyInt64, "aarch64_sve_prfd_gather_scalar_offset">; 546} // let SVETargetGuard = "sve" 547 548//////////////////////////////////////////////////////////////////////////////// 549// Address calculations 550 551let SVETargetGuard = "sve", SMETargetGuard = InvalidMode in { 552def SVADRB : SInst<"svadrb[_{0}base]_[{2}]offset", "uud", "ilUiUl", MergeNone, "aarch64_sve_adrb">; 553def SVADRH : SInst<"svadrh[_{0}base]_[{2}]index", "uud", "ilUiUl", MergeNone, "aarch64_sve_adrh">; 554def SVADRW : SInst<"svadrw[_{0}base]_[{2}]index", "uud", "ilUiUl", MergeNone, "aarch64_sve_adrw">; 555def SVADRD : SInst<"svadrd[_{0}base]_[{2}]index", "uud", "ilUiUl", MergeNone, "aarch64_sve_adrd">; 556} // let SVETargetGuard = "sve" 557 558//////////////////////////////////////////////////////////////////////////////// 559// Scalar to vector 560 561def SVDUPQ_8 : SInst<"svdupq[_n]_{d}", "dssssssssssssssss", "cUc", MergeNone, "", [VerifyRuntimeMode]>; 562def SVDUPQ_16 : SInst<"svdupq[_n]_{d}", "dssssssss", "sUsh", MergeNone, "", [VerifyRuntimeMode]>; 563let SVETargetGuard = "sve,bf16", SMETargetGuard = "sme,bf16" in { 564 def SVDUPQ_BF16 : SInst<"svdupq[_n]_{d}", "dssssssss", "b", MergeNone, "", [VerifyRuntimeMode]>; 565} 566def SVDUPQ_32 : SInst<"svdupq[_n]_{d}", "dssss", "iUif", MergeNone, "", [VerifyRuntimeMode]>; 567def SVDUPQ_64 : SInst<"svdupq[_n]_{d}", "dss", "lUld", MergeNone, "", [VerifyRuntimeMode]>; 568 569multiclass svdup_base<string n, string p, MergeType mt, string i> { 570 def NAME : SInst<n, p, "csilUcUsUiUlhfd", mt, i, [VerifyRuntimeMode]>; 571 let SVETargetGuard = "sve,bf16", SMETargetGuard = "sme,bf16" in { 572 def _BF16: SInst<n, p, "b", mt, i, [VerifyRuntimeMode]>; 573 } 574} 575 576defm SVDUP : svdup_base<"svdup[_n]_{d}", "ds", MergeNone, "aarch64_sve_dup_x">; 577defm SVDUP_M : svdup_base<"svdup[_n]_{d}", "ddPs", MergeOp1, "aarch64_sve_dup">; 578defm SVDUP_X : svdup_base<"svdup[_n]_{d}", "dPs", MergeAnyExp, "aarch64_sve_dup">; 579defm SVDUP_Z : svdup_base<"svdup[_n]_{d}", "dPs", MergeZeroExp, "aarch64_sve_dup">; 580 581def SVINDEX : SInst<"svindex_{d}", "dss", "csilUcUsUiUl", MergeNone, "aarch64_sve_index", [VerifyRuntimeMode]>; 582 583// Integer arithmetic 584 585multiclass SInstZPZ<string name, string types, string intrinsic> { 586 def _M : SInst<name # "[_{d}]", "ddPd", types, MergeOp1, intrinsic, [VerifyRuntimeMode]>; 587 def _X : SInst<name # "[_{d}]", "dPd", types, MergeAnyExp, intrinsic, [VerifyRuntimeMode]>; 588 def _Z : SInst<name # "[_{d}]", "dPd", types, MergeZeroExp, intrinsic, [VerifyRuntimeMode]>; 589} 590 591defm SVABS : SInstZPZ<"svabs", "csil", "aarch64_sve_abs">; 592defm SVNEG : SInstZPZ<"svneg", "csil", "aarch64_sve_neg">; 593 594//------------------------------------------------------------------------------ 595 596multiclass SInstZPZZ<string name, string types, string m_intrinsic, string x_intrinsic, list<FlagType> flags=[]> { 597 def _M : SInst<name # "[_{d}]", "dPdd", types, MergeOp1, m_intrinsic, !listconcat(flags, [VerifyRuntimeMode])>; 598 def _X : SInst<name # "[_{d}]", "dPdd", types, MergeAny, x_intrinsic, !listconcat(flags, [VerifyRuntimeMode])>; 599 def _Z : SInst<name # "[_{d}]", "dPdd", types, MergeZero, m_intrinsic, !listconcat(flags, [VerifyRuntimeMode])>; 600 601 def _N_M : SInst<name # "[_n_{d}]", "dPda", types, MergeOp1, m_intrinsic, !listconcat(flags, [VerifyRuntimeMode])>; 602 def _N_X : SInst<name # "[_n_{d}]", "dPda", types, MergeAny, x_intrinsic, !listconcat(flags, [VerifyRuntimeMode])>; 603 def _N_Z : SInst<name # "[_n_{d}]", "dPda", types, MergeZero, m_intrinsic, !listconcat(flags, [VerifyRuntimeMode])>; 604} 605 606defm SVABD_S : SInstZPZZ<"svabd", "csil", "aarch64_sve_sabd", "aarch64_sve_sabd_u">; 607defm SVABD_U : SInstZPZZ<"svabd", "UcUsUiUl", "aarch64_sve_uabd", "aarch64_sve_uabd_u">; 608defm SVADD : SInstZPZZ<"svadd", "csilUcUsUiUl", "aarch64_sve_add", "aarch64_sve_add_u">; 609defm SVDIV_S : SInstZPZZ<"svdiv", "il", "aarch64_sve_sdiv", "aarch64_sve_sdiv_u">; 610defm SVDIV_U : SInstZPZZ<"svdiv", "UiUl", "aarch64_sve_udiv", "aarch64_sve_udiv_u">; 611defm SVDIVR_S : SInstZPZZ<"svdivr", "il", "aarch64_sve_sdivr", "aarch64_sve_sdiv_u", [ReverseMergeAnyBinOp]>; 612defm SVDIVR_U : SInstZPZZ<"svdivr", "UiUl", "aarch64_sve_udivr", "aarch64_sve_udiv_u", [ReverseMergeAnyBinOp]>; 613defm SVMAX_S : SInstZPZZ<"svmax", "csil", "aarch64_sve_smax", "aarch64_sve_smax_u">; 614defm SVMAX_U : SInstZPZZ<"svmax", "UcUsUiUl", "aarch64_sve_umax", "aarch64_sve_umax_u">; 615defm SVMIN_S : SInstZPZZ<"svmin", "csil", "aarch64_sve_smin", "aarch64_sve_smin_u">; 616defm SVMIN_U : SInstZPZZ<"svmin", "UcUsUiUl", "aarch64_sve_umin", "aarch64_sve_umin_u">; 617defm SVMUL : SInstZPZZ<"svmul", "csilUcUsUiUl", "aarch64_sve_mul", "aarch64_sve_mul_u">; 618defm SVMULH_S : SInstZPZZ<"svmulh", "csil", "aarch64_sve_smulh", "aarch64_sve_smulh_u">; 619defm SVMULH_U : SInstZPZZ<"svmulh", "UcUsUiUl", "aarch64_sve_umulh", "aarch64_sve_umulh_u">; 620defm SVSUB : SInstZPZZ<"svsub", "csilUcUsUiUl", "aarch64_sve_sub", "aarch64_sve_sub_u">; 621defm SVSUBR : SInstZPZZ<"svsubr", "csilUcUsUiUl", "aarch64_sve_subr", "aarch64_sve_sub_u", [ReverseMergeAnyBinOp]>; 622 623//------------------------------------------------------------------------------ 624 625multiclass SInstZPZZZ<string name, string types, string m_intrinsic, string x_intrinsic, list<FlagType> flags=[]> { 626 def _M : SInst<name # "[_{d}]", "dPddd", types, MergeOp1, m_intrinsic, flags>; 627 def _X : SInst<name # "[_{d}]", "dPddd", types, MergeAny, x_intrinsic, flags>; 628 def _Z : SInst<name # "[_{d}]", "dPddd", types, MergeZero, m_intrinsic, flags>; 629 630 def _N_M : SInst<name # "[_n_{d}]", "dPdda", types, MergeOp1, m_intrinsic, flags>; 631 def _N_X : SInst<name # "[_n_{d}]", "dPdda", types, MergeAny, x_intrinsic, flags>; 632 def _N_Z : SInst<name # "[_n_{d}]", "dPdda", types, MergeZero, m_intrinsic, flags>; 633} 634 635defm SVMAD : SInstZPZZZ<"svmad", "csilUcUsUiUl", "aarch64_sve_mad", "aarch64_sve_mla_u", [ReverseMergeAnyAccOp, VerifyRuntimeMode]>; 636defm SVMLA : SInstZPZZZ<"svmla", "csilUcUsUiUl", "aarch64_sve_mla", "aarch64_sve_mla_u", [VerifyRuntimeMode]>; 637defm SVMLS : SInstZPZZZ<"svmls", "csilUcUsUiUl", "aarch64_sve_mls", "aarch64_sve_mls_u", [VerifyRuntimeMode]>; 638defm SVMSB : SInstZPZZZ<"svmsb", "csilUcUsUiUl", "aarch64_sve_msb", "aarch64_sve_mls_u", [ReverseMergeAnyAccOp, VerifyRuntimeMode]>; 639 640//------------------------------------------------------------------------------ 641 642def SVDOT_S : SInst<"svdot[_{0}]", "ddqq", "il", MergeNone, "aarch64_sve_sdot", [VerifyRuntimeMode]>; 643def SVDOT_U : SInst<"svdot[_{0}]", "ddqq", "UiUl", MergeNone, "aarch64_sve_udot", [VerifyRuntimeMode]>; 644def SVQADD_S : SInst<"svqadd[_{d}]", "ddd", "csil", MergeNone, "aarch64_sve_sqadd_x", [VerifyRuntimeMode]>; 645def SVQADD_U : SInst<"svqadd[_{d}]", "ddd", "UcUsUiUl", MergeNone, "aarch64_sve_uqadd_x", [VerifyRuntimeMode]>; 646def SVQSUB_S : SInst<"svqsub[_{d}]", "ddd", "csil", MergeNone, "aarch64_sve_sqsub_x", [VerifyRuntimeMode]>; 647def SVQSUB_U : SInst<"svqsub[_{d}]", "ddd", "UcUsUiUl", MergeNone, "aarch64_sve_uqsub_x", [VerifyRuntimeMode]>; 648 649def SVDOT_N_S : SInst<"svdot[_n_{0}]", "ddqr", "il", MergeNone, "aarch64_sve_sdot", [VerifyRuntimeMode]>; 650def SVDOT_N_U : SInst<"svdot[_n_{0}]", "ddqr", "UiUl", MergeNone, "aarch64_sve_udot", [VerifyRuntimeMode]>; 651def SVQADD_N_S : SInst<"svqadd[_n_{d}]", "dda", "csil", MergeNone, "aarch64_sve_sqadd_x", [VerifyRuntimeMode]>; 652def SVQADD_N_U : SInst<"svqadd[_n_{d}]", "dda", "UcUsUiUl", MergeNone, "aarch64_sve_uqadd_x", [VerifyRuntimeMode]>; 653def SVQSUB_N_S : SInst<"svqsub[_n_{d}]", "dda", "csil", MergeNone, "aarch64_sve_sqsub_x", [VerifyRuntimeMode]>; 654def SVQSUB_N_U : SInst<"svqsub[_n_{d}]", "dda", "UcUsUiUl", MergeNone, "aarch64_sve_uqsub_x", [VerifyRuntimeMode]>; 655 656def SVDOT_LANE_S : SInst<"svdot_lane[_{d}]", "ddqqi", "il", MergeNone, "aarch64_sve_sdot_lane", [VerifyRuntimeMode], [ImmCheck<3, ImmCheckLaneIndexDot, 2>]>; 657def SVDOT_LANE_U : SInst<"svdot_lane[_{d}]", "ddqqi", "UiUl", MergeNone, "aarch64_sve_udot_lane", [VerifyRuntimeMode], [ImmCheck<3, ImmCheckLaneIndexDot, 2>]>; 658 659//////////////////////////////////////////////////////////////////////////////// 660// Logical operations 661 662defm SVAND : SInstZPZZ<"svand", "csilUcUsUiUl", "aarch64_sve_and", "aarch64_sve_and_u">; 663defm SVBIC : SInstZPZZ<"svbic", "csilUcUsUiUl", "aarch64_sve_bic", "aarch64_sve_bic_u">; 664defm SVEOR : SInstZPZZ<"sveor", "csilUcUsUiUl", "aarch64_sve_eor", "aarch64_sve_eor_u">; 665defm SVORR : SInstZPZZ<"svorr", "csilUcUsUiUl", "aarch64_sve_orr", "aarch64_sve_orr_u">; 666 667defm SVCNOT : SInstZPZ<"svcnot", "csilUcUsUiUl", "aarch64_sve_cnot">; 668defm SVNOT : SInstZPZ<"svnot", "csilUcUsUiUl", "aarch64_sve_not">; 669 670//////////////////////////////////////////////////////////////////////////////// 671// Shifts 672 673multiclass SInst_SHIFT<string name, string intrinsic, string ts, string wide_ts> { 674 def _M : SInst<name # "[_{d}]", "dPdu", ts, MergeOp1, intrinsic, [VerifyRuntimeMode]>; 675 def _X : SInst<name # "[_{d}]", "dPdu", ts, MergeAny, intrinsic # _u, [VerifyRuntimeMode]>; 676 def _Z : SInst<name # "[_{d}]", "dPdu", ts, MergeZero, intrinsic, [VerifyRuntimeMode]>; 677 678 def _N_M : SInst<name # "[_n_{d}]", "dPdL", ts, MergeOp1, intrinsic, [VerifyRuntimeMode]>; 679 def _N_X : SInst<name # "[_n_{d}]", "dPdL", ts, MergeAny, intrinsic # _u, [VerifyRuntimeMode]>; 680 def _N_Z : SInst<name # "[_n_{d}]", "dPdL", ts, MergeZero, intrinsic, [VerifyRuntimeMode]>; 681 682 def _WIDE_M : SInst<name # _wide # "[_{d}]", "dPdg", wide_ts, MergeOp1, intrinsic # _wide, [VerifyRuntimeMode]>; 683 def _WIDE_X : SInst<name # _wide # "[_{d}]", "dPdg", wide_ts, MergeAny, intrinsic # _wide, [VerifyRuntimeMode]>; 684 def _WIDE_Z : SInst<name # _wide # "[_{d}]", "dPdg", wide_ts, MergeZero, intrinsic # _wide, [VerifyRuntimeMode]>; 685 686 def _WIDE_N_M : SInst<name # _wide # "[_n_{d}]", "dPdf", wide_ts, MergeOp1, intrinsic # _wide, [VerifyRuntimeMode]>; 687 def _WIDE_N_X : SInst<name # _wide # "[_n_{d}]", "dPdf", wide_ts, MergeAny, intrinsic # _wide, [VerifyRuntimeMode]>; 688 def _WIDE_N_Z : SInst<name # _wide # "[_n_{d}]", "dPdf", wide_ts, MergeZero, intrinsic # _wide, [VerifyRuntimeMode]>; 689} 690 691defm SVASR : SInst_SHIFT<"svasr", "aarch64_sve_asr", "csil", "csi">; 692defm SVLSL : SInst_SHIFT<"svlsl", "aarch64_sve_lsl", "csilUcUsUiUl", "csiUcUsUi">; 693defm SVLSR : SInst_SHIFT<"svlsr", "aarch64_sve_lsr", "UcUsUiUl", "UcUsUi">; 694 695def SVASRD_M : SInst<"svasrd[_n_{d}]", "dPdi", "csil", MergeOp1, "aarch64_sve_asrd", [VerifyRuntimeMode], [ImmCheck<2, ImmCheckShiftRight, 1>]>; 696def SVASRD_X : SInst<"svasrd[_n_{d}]", "dPdi", "csil", MergeAny, "aarch64_sve_asrd", [VerifyRuntimeMode], [ImmCheck<2, ImmCheckShiftRight, 1>]>; 697def SVASRD_Z : SInst<"svasrd[_n_{d}]", "dPdi", "csil", MergeZero, "aarch64_sve_asrd", [VerifyRuntimeMode], [ImmCheck<2, ImmCheckShiftRight, 1>]>; 698 699def SVINSR : SInst<"svinsr[_n_{d}]", "dds", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_insr", [VerifyRuntimeMode]>; 700let SVETargetGuard = "sve,bf16", SMETargetGuard = "sme,bf16" in { 701 def SVINSR_BF16 : SInst<"svinsr[_n_{d}]", "dds", "b", MergeNone, "aarch64_sve_insr", [VerifyRuntimeMode]>; 702} 703 704//////////////////////////////////////////////////////////////////////////////// 705// Integer reductions 706 707def SVADDV_S : SInst<"svaddv[_{d}]", "lPd", "csil", MergeNone, "aarch64_sve_saddv", [VerifyRuntimeMode]>; 708def SVADDV_U : SInst<"svaddv[_{d}]", "nPd", "UcUsUiUl", MergeNone, "aarch64_sve_uaddv", [VerifyRuntimeMode]>; 709def SVANDV : SInst<"svandv[_{d}]", "sPd", "csilUcUsUiUl", MergeNone, "aarch64_sve_andv", [VerifyRuntimeMode]>; 710def SVEORV : SInst<"sveorv[_{d}]", "sPd", "csilUcUsUiUl", MergeNone, "aarch64_sve_eorv", [VerifyRuntimeMode]>; 711def SVMAXV_S : SInst<"svmaxv[_{d}]", "sPd", "csil", MergeNone, "aarch64_sve_smaxv", [VerifyRuntimeMode]>; 712def SVMAXV_U : SInst<"svmaxv[_{d}]", "sPd", "UcUsUiUl", MergeNone, "aarch64_sve_umaxv", [VerifyRuntimeMode]>; 713def SVMINV_S : SInst<"svminv[_{d}]", "sPd", "csil", MergeNone, "aarch64_sve_sminv", [VerifyRuntimeMode]>; 714def SVMINV_U : SInst<"svminv[_{d}]", "sPd", "UcUsUiUl", MergeNone, "aarch64_sve_uminv", [VerifyRuntimeMode]>; 715def SVORV : SInst<"svorv[_{d}]", "sPd", "csilUcUsUiUl", MergeNone, "aarch64_sve_orv", [VerifyRuntimeMode]>; 716 717//////////////////////////////////////////////////////////////////////////////// 718// Integer comparisons 719 720def SVCMPEQ : SInst<"svcmpeq[_{d}]", "PPdd", "csilUcUsUiUl", MergeNone, "aarch64_sve_cmpeq", [VerifyRuntimeMode]>; 721def SVCMPNE : SInst<"svcmpne[_{d}]", "PPdd", "csilUcUsUiUl", MergeNone, "aarch64_sve_cmpne", [VerifyRuntimeMode]>; 722def SVCMPGE : SInst<"svcmpge[_{d}]", "PPdd", "csil", MergeNone, "aarch64_sve_cmpge", [VerifyRuntimeMode]>; 723def SVCMPGT : SInst<"svcmpgt[_{d}]", "PPdd", "csil", MergeNone, "aarch64_sve_cmpgt", [VerifyRuntimeMode]>; 724def SVCMPLE : SInst<"svcmple[_{d}]", "PPdd", "csil", MergeNone, "aarch64_sve_cmpge", [ReverseCompare, VerifyRuntimeMode]>; 725def SVCMPLT : SInst<"svcmplt[_{d}]", "PPdd", "csil", MergeNone, "aarch64_sve_cmpgt", [ReverseCompare, VerifyRuntimeMode]>; 726def SVCMPHI : SInst<"svcmpgt[_{d}]", "PPdd", "UcUsUiUl", MergeNone, "aarch64_sve_cmphi", [VerifyRuntimeMode]>; 727def SVCMPHS : SInst<"svcmpge[_{d}]", "PPdd", "UcUsUiUl", MergeNone, "aarch64_sve_cmphs", [VerifyRuntimeMode]>; 728def SVCMPLO : SInst<"svcmplt[_{d}]", "PPdd", "UcUsUiUl", MergeNone, "aarch64_sve_cmphi", [ReverseCompare, VerifyRuntimeMode]>; 729def SVCMPLS : SInst<"svcmple[_{d}]", "PPdd", "UcUsUiUl", MergeNone, "aarch64_sve_cmphs", [ReverseCompare, VerifyRuntimeMode]>; 730 731def SVCMPEQ_N : SInst<"svcmpeq[_n_{d}]", "PPda", "csilUcUsUiUl", MergeNone, "aarch64_sve_cmpeq", [VerifyRuntimeMode]>; 732def SVCMPNE_N : SInst<"svcmpne[_n_{d}]", "PPda", "csilUcUsUiUl", MergeNone, "aarch64_sve_cmpne", [VerifyRuntimeMode]>; 733def SVCMPGE_N : SInst<"svcmpge[_n_{d}]", "PPda", "csil", MergeNone, "aarch64_sve_cmpge", [VerifyRuntimeMode]>; 734def SVCMPGT_N : SInst<"svcmpgt[_n_{d}]", "PPda", "csil", MergeNone, "aarch64_sve_cmpgt", [VerifyRuntimeMode]>; 735def SVCMPLE_N : SInst<"svcmple[_n_{d}]", "PPda", "csil", MergeNone, "aarch64_sve_cmpge", [ReverseCompare, VerifyRuntimeMode]>; 736def SVCMPLT_N : SInst<"svcmplt[_n_{d}]", "PPda", "csil", MergeNone, "aarch64_sve_cmpgt", [ReverseCompare, VerifyRuntimeMode]>; 737def SVCMPHS_N : SInst<"svcmpge[_n_{d}]", "PPda", "UcUsUiUl", MergeNone, "aarch64_sve_cmphs", [VerifyRuntimeMode]>; 738def SVCMPHI_N : SInst<"svcmpgt[_n_{d}]", "PPda", "UcUsUiUl", MergeNone, "aarch64_sve_cmphi", [VerifyRuntimeMode]>; 739def SVCMPLS_N : SInst<"svcmple[_n_{d}]", "PPda", "UcUsUiUl", MergeNone, "aarch64_sve_cmphs", [ReverseCompare, VerifyRuntimeMode]>; 740def SVCMPLO_N : SInst<"svcmplt[_n_{d}]", "PPda", "UcUsUiUl", MergeNone, "aarch64_sve_cmphi", [ReverseCompare, VerifyRuntimeMode]>; 741 742def SVCMPEQ_WIDE : SInst<"svcmpeq_wide[_{d}]", "PPdw", "csi", MergeNone, "aarch64_sve_cmpeq_wide", [VerifyRuntimeMode]>; 743def SVCMPNE_WIDE : SInst<"svcmpne_wide[_{d}]", "PPdw", "csi", MergeNone, "aarch64_sve_cmpne_wide", [VerifyRuntimeMode]>; 744def SVCMPGE_WIDE : SInst<"svcmpge_wide[_{d}]", "PPdw", "csi", MergeNone, "aarch64_sve_cmpge_wide", [VerifyRuntimeMode]>; 745def SVCMPGT_WIDE : SInst<"svcmpgt_wide[_{d}]", "PPdw", "csi", MergeNone, "aarch64_sve_cmpgt_wide", [VerifyRuntimeMode]>; 746def SVCMPLE_WIDE : SInst<"svcmple_wide[_{d}]", "PPdw", "csi", MergeNone, "aarch64_sve_cmple_wide", [VerifyRuntimeMode]>; 747def SVCMPLT_WIDE : SInst<"svcmplt_wide[_{d}]", "PPdw", "csi", MergeNone, "aarch64_sve_cmplt_wide", [VerifyRuntimeMode]>; 748def SVCMPHI_WIDE : SInst<"svcmpgt_wide[_{d}]", "PPdw", "UcUsUi", MergeNone, "aarch64_sve_cmphi_wide", [VerifyRuntimeMode]>; 749def SVCMPHS_WIDE : SInst<"svcmpge_wide[_{d}]", "PPdw", "UcUsUi", MergeNone, "aarch64_sve_cmphs_wide", [VerifyRuntimeMode]>; 750def SVCMPLO_WIDE : SInst<"svcmplt_wide[_{d}]", "PPdw", "UcUsUi", MergeNone, "aarch64_sve_cmplo_wide", [VerifyRuntimeMode]>; 751def SVCMPLS_WIDE : SInst<"svcmple_wide[_{d}]", "PPdw", "UcUsUi", MergeNone, "aarch64_sve_cmpls_wide", [VerifyRuntimeMode]>; 752 753def SVCMPEQ_WIDE_N : SInst<"svcmpeq_wide[_n_{d}]", "PPdj", "csi", MergeNone, "aarch64_sve_cmpeq_wide", [VerifyRuntimeMode]>; 754def SVCMPNE_WIDE_N : SInst<"svcmpne_wide[_n_{d}]", "PPdj", "csi", MergeNone, "aarch64_sve_cmpne_wide", [VerifyRuntimeMode]>; 755def SVCMPGE_WIDE_N : SInst<"svcmpge_wide[_n_{d}]", "PPdj", "csi", MergeNone, "aarch64_sve_cmpge_wide", [VerifyRuntimeMode]>; 756def SVCMPGT_WIDE_N : SInst<"svcmpgt_wide[_n_{d}]", "PPdj", "csi", MergeNone, "aarch64_sve_cmpgt_wide", [VerifyRuntimeMode]>; 757def SVCMPLE_WIDE_N : SInst<"svcmple_wide[_n_{d}]", "PPdj", "csi", MergeNone, "aarch64_sve_cmple_wide", [VerifyRuntimeMode]>; 758def SVCMPLT_WIDE_N : SInst<"svcmplt_wide[_n_{d}]", "PPdj", "csi", MergeNone, "aarch64_sve_cmplt_wide", [VerifyRuntimeMode]>; 759def SVCMPHS_WIDE_N : SInst<"svcmpge_wide[_n_{d}]", "PPdj", "UcUsUi", MergeNone, "aarch64_sve_cmphs_wide", [VerifyRuntimeMode]>; 760def SVCMPHI_WIDE_N : SInst<"svcmpgt_wide[_n_{d}]", "PPdj", "UcUsUi", MergeNone, "aarch64_sve_cmphi_wide", [VerifyRuntimeMode]>; 761def SVCMPLO_WIDE_N : SInst<"svcmplt_wide[_n_{d}]", "PPdj", "UcUsUi", MergeNone, "aarch64_sve_cmplo_wide", [VerifyRuntimeMode]>; 762def SVCMPLS_WIDE_N : SInst<"svcmple_wide[_n_{d}]", "PPdj", "UcUsUi", MergeNone, "aarch64_sve_cmpls_wide", [VerifyRuntimeMode]>; 763 764//////////////////////////////////////////////////////////////////////////////// 765// While comparisons 766 767def SVWHILELE_S32 : SInst<"svwhilele_{d}[_{1}]", "Pkk", "PcPsPiPl", MergeNone, "aarch64_sve_whilele", [IsOverloadWhileOrMultiVecCvt, VerifyRuntimeMode]>; 768def SVWHILELE_S64 : SInst<"svwhilele_{d}[_{1}]", "Pll", "PcPsPiPl", MergeNone, "aarch64_sve_whilele", [IsOverloadWhileOrMultiVecCvt, VerifyRuntimeMode]>; 769def SVWHILELO_U32 : SInst<"svwhilelt_{d}[_{1}]", "Pmm", "PcPsPiPl", MergeNone, "aarch64_sve_whilelo", [IsOverloadWhileOrMultiVecCvt, VerifyRuntimeMode]>; 770def SVWHILELO_U64 : SInst<"svwhilelt_{d}[_{1}]", "Pnn", "PcPsPiPl", MergeNone, "aarch64_sve_whilelo", [IsOverloadWhileOrMultiVecCvt, VerifyRuntimeMode]>; 771def SVWHILELS_U32 : SInst<"svwhilele_{d}[_{1}]", "Pmm", "PcPsPiPl", MergeNone, "aarch64_sve_whilels", [IsOverloadWhileOrMultiVecCvt, VerifyRuntimeMode]>; 772def SVWHILELS_U64 : SInst<"svwhilele_{d}[_{1}]", "Pnn", "PcPsPiPl", MergeNone, "aarch64_sve_whilels", [IsOverloadWhileOrMultiVecCvt, VerifyRuntimeMode]>; 773def SVWHILELT_S32 : SInst<"svwhilelt_{d}[_{1}]", "Pkk", "PcPsPiPl", MergeNone, "aarch64_sve_whilelt", [IsOverloadWhileOrMultiVecCvt, VerifyRuntimeMode]>; 774def SVWHILELT_S64 : SInst<"svwhilelt_{d}[_{1}]", "Pll", "PcPsPiPl", MergeNone, "aarch64_sve_whilelt", [IsOverloadWhileOrMultiVecCvt, VerifyRuntimeMode]>; 775 776//////////////////////////////////////////////////////////////////////////////// 777// Counting bit 778 779multiclass SInstCLS<string name, string types, string intrinsic, list<FlagType> flags=[]> { 780 def _M : SInst<name # "[_{d}]", "uuPd", types, MergeOp1, intrinsic, flags>; 781 def _X : SInst<name # "[_{d}]", "uPd", types, MergeAnyExp, intrinsic, flags>; 782 def _Z : SInst<name # "[_{d}]", "uPd", types, MergeZeroExp, intrinsic, flags>; 783} 784 785defm SVCLS : SInstCLS<"svcls", "csil", "aarch64_sve_cls", [VerifyRuntimeMode]>; 786defm SVCLZ : SInstCLS<"svclz", "csilUcUsUiUl", "aarch64_sve_clz", [VerifyRuntimeMode]>; 787defm SVCNT : SInstCLS<"svcnt", "csilUcUsUiUlhfd", "aarch64_sve_cnt", [VerifyRuntimeMode]>; 788 789let SVETargetGuard = "sve,bf16", SMETargetGuard = "sme,bf16" in { 790 defm SVCNT_BF16 : SInstCLS<"svcnt", "b", "aarch64_sve_cnt", [VerifyRuntimeMode]>; 791} 792 793//////////////////////////////////////////////////////////////////////////////// 794// Conversion 795 796defm SVEXTB_S : SInstZPZ<"svextb", "sil", "aarch64_sve_sxtb">; 797defm SVEXTB_U : SInstZPZ<"svextb", "UsUiUl", "aarch64_sve_uxtb">; 798defm SVEXTH_S : SInstZPZ<"svexth", "il", "aarch64_sve_sxth">; 799defm SVEXTH_U : SInstZPZ<"svexth", "UiUl", "aarch64_sve_uxth">; 800defm SVEXTW_S : SInstZPZ<"svextw", "l", "aarch64_sve_sxtw">; 801defm SVEXTW_U : SInstZPZ<"svextw", "Ul", "aarch64_sve_uxtw">; 802 803//////////////////////////////////////////////////////////////////////////////// 804// Reversal 805 806defm SVRBIT : SInstZPZ<"svrbit", "csilUcUsUiUl", "aarch64_sve_rbit">; 807defm SVREVB : SInstZPZ<"svrevb", "silUsUiUl", "aarch64_sve_revb">; 808defm SVREVH : SInstZPZ<"svrevh", "ilUiUl", "aarch64_sve_revh">; 809defm SVREVW : SInstZPZ<"svrevw", "lUl", "aarch64_sve_revw">; 810 811//////////////////////////////////////////////////////////////////////////////// 812// Floating-point arithmetic 813 814defm SVABS_F : SInstZPZ<"svabs", "hfd", "aarch64_sve_fabs">; 815defm SVNEG_F : SInstZPZ<"svneg", "hfd", "aarch64_sve_fneg">; 816 817defm SVABD_F : SInstZPZZ<"svabd", "hfd", "aarch64_sve_fabd", "aarch64_sve_fabd_u">; 818defm SVADD_F : SInstZPZZ<"svadd", "hfd", "aarch64_sve_fadd", "aarch64_sve_fadd_u">; 819defm SVDIV_F : SInstZPZZ<"svdiv", "hfd", "aarch64_sve_fdiv", "aarch64_sve_fdiv_u">; 820defm SVDIVR_F : SInstZPZZ<"svdivr", "hfd", "aarch64_sve_fdivr", "aarch64_sve_fdiv_u", [ReverseMergeAnyBinOp]>; 821defm SVMAX_F : SInstZPZZ<"svmax", "hfd", "aarch64_sve_fmax", "aarch64_sve_fmax_u">; 822defm SVMAXNM : SInstZPZZ<"svmaxnm","hfd", "aarch64_sve_fmaxnm", "aarch64_sve_fmaxnm_u">; 823defm SVMIN_F : SInstZPZZ<"svmin", "hfd", "aarch64_sve_fmin", "aarch64_sve_fmin_u">; 824defm SVMINNM : SInstZPZZ<"svminnm","hfd", "aarch64_sve_fminnm", "aarch64_sve_fminnm_u">; 825defm SVMUL_F : SInstZPZZ<"svmul", "hfd", "aarch64_sve_fmul", "aarch64_sve_fmul_u">; 826defm SVMULX : SInstZPZZ<"svmulx", "hfd", "aarch64_sve_fmulx", "aarch64_sve_fmulx_u">; 827defm SVSUB_F : SInstZPZZ<"svsub", "hfd", "aarch64_sve_fsub", "aarch64_sve_fsub_u">; 828defm SVSUBR_F : SInstZPZZ<"svsubr", "hfd", "aarch64_sve_fsubr", "aarch64_sve_fsub_u", [ReverseMergeAnyBinOp]>; 829 830defm SVRECPX : SInstZPZ<"svrecpx", "hfd", "aarch64_sve_frecpx">; 831defm SVRINTA : SInstZPZ<"svrinta", "hfd", "aarch64_sve_frinta">; 832defm SVRINTI : SInstZPZ<"svrinti", "hfd", "aarch64_sve_frinti">; 833defm SVRINTM : SInstZPZ<"svrintm", "hfd", "aarch64_sve_frintm">; 834defm SVRINTN : SInstZPZ<"svrintn", "hfd", "aarch64_sve_frintn">; 835defm SVRINTP : SInstZPZ<"svrintp", "hfd", "aarch64_sve_frintp">; 836defm SVRINTX : SInstZPZ<"svrintx", "hfd", "aarch64_sve_frintx">; 837defm SVRINTZ : SInstZPZ<"svrintz", "hfd", "aarch64_sve_frintz">; 838defm SVSQRT : SInstZPZ<"svsqrt", "hfd", "aarch64_sve_fsqrt">; 839 840let SVETargetGuard = "sve", SMETargetGuard = InvalidMode in { 841def SVEXPA : SInst<"svexpa[_{d}]", "du", "hfd", MergeNone, "aarch64_sve_fexpa_x">; 842def SVTMAD : SInst<"svtmad[_{d}]", "dddi", "hfd", MergeNone, "aarch64_sve_ftmad_x", [], [ImmCheck<2, ImmCheck0_7>]>; 843def SVTSMUL : SInst<"svtsmul[_{d}]", "ddu", "hfd", MergeNone, "aarch64_sve_ftsmul_x">; 844def SVTSSEL : SInst<"svtssel[_{d}]", "ddu", "hfd", MergeNone, "aarch64_sve_ftssel_x">; 845} 846 847def SVSCALE_M : SInst<"svscale[_{d}]", "dPdx", "hfd", MergeOp1, "aarch64_sve_fscale", [VerifyRuntimeMode]>; 848def SVSCALE_X : SInst<"svscale[_{d}]", "dPdx", "hfd", MergeAny, "aarch64_sve_fscale", [VerifyRuntimeMode]>; 849def SVSCALE_Z : SInst<"svscale[_{d}]", "dPdx", "hfd", MergeZero, "aarch64_sve_fscale", [VerifyRuntimeMode]>; 850 851def SVSCALE_N_M : SInst<"svscale[_n_{d}]", "dPdK", "hfd", MergeOp1, "aarch64_sve_fscale", [VerifyRuntimeMode]>; 852def SVSCALE_N_X : SInst<"svscale[_n_{d}]", "dPdK", "hfd", MergeAny, "aarch64_sve_fscale", [VerifyRuntimeMode]>; 853def SVSCALE_N_Z : SInst<"svscale[_n_{d}]", "dPdK", "hfd", MergeZero, "aarch64_sve_fscale", [VerifyRuntimeMode]>; 854 855defm SVMAD_F : SInstZPZZZ<"svmad", "hfd", "aarch64_sve_fmad", "aarch64_sve_fmla_u", [VerifyRuntimeMode, ReverseMergeAnyAccOp]>; 856defm SVMLA_F : SInstZPZZZ<"svmla", "hfd", "aarch64_sve_fmla", "aarch64_sve_fmla_u", [VerifyRuntimeMode]>; 857defm SVMLS_F : SInstZPZZZ<"svmls", "hfd", "aarch64_sve_fmls", "aarch64_sve_fmls_u", [VerifyRuntimeMode]>; 858defm SVMSB_F : SInstZPZZZ<"svmsb", "hfd", "aarch64_sve_fmsb", "aarch64_sve_fmls_u", [VerifyRuntimeMode, ReverseMergeAnyAccOp]>; 859defm SVNMAD_F : SInstZPZZZ<"svnmad", "hfd", "aarch64_sve_fnmad", "aarch64_sve_fnmla_u", [VerifyRuntimeMode, ReverseMergeAnyAccOp]>; 860defm SVNMLA_F : SInstZPZZZ<"svnmla", "hfd", "aarch64_sve_fnmla", "aarch64_sve_fnmla_u", [VerifyRuntimeMode]>; 861defm SVNMLS_F : SInstZPZZZ<"svnmls", "hfd", "aarch64_sve_fnmls", "aarch64_sve_fnmls_u", [VerifyRuntimeMode]>; 862defm SVNMSB_F : SInstZPZZZ<"svnmsb", "hfd", "aarch64_sve_fnmsb", "aarch64_sve_fnmls_u", [VerifyRuntimeMode, ReverseMergeAnyAccOp]>; 863 864def SVCADD_M : SInst<"svcadd[_{d}]", "dPddi", "hfd", MergeOp1, "aarch64_sve_fcadd", [VerifyRuntimeMode], [ImmCheck<3, ImmCheckComplexRot90_270>]>; 865def SVCADD_X : SInst<"svcadd[_{d}]", "dPddi", "hfd", MergeAny, "aarch64_sve_fcadd", [VerifyRuntimeMode], [ImmCheck<3, ImmCheckComplexRot90_270>]>; 866def SVCADD_Z : SInst<"svcadd[_{d}]", "dPddi", "hfd", MergeZero, "aarch64_sve_fcadd", [VerifyRuntimeMode], [ImmCheck<3, ImmCheckComplexRot90_270>]>; 867def SVCMLA_M : SInst<"svcmla[_{d}]", "dPdddi", "hfd", MergeOp1, "aarch64_sve_fcmla", [VerifyRuntimeMode], [ImmCheck<4, ImmCheckComplexRotAll90>]>; 868def SVCMLA_X : SInst<"svcmla[_{d}]", "dPdddi", "hfd", MergeAny, "aarch64_sve_fcmla", [VerifyRuntimeMode], [ImmCheck<4, ImmCheckComplexRotAll90>]>; 869def SVCMLA_Z : SInst<"svcmla[_{d}]", "dPdddi", "hfd", MergeZero, "aarch64_sve_fcmla", [VerifyRuntimeMode], [ImmCheck<4, ImmCheckComplexRotAll90>]>; 870 871def SVCMLA_LANE : SInst<"svcmla_lane[_{d}]", "ddddii", "hf", MergeNone, "aarch64_sve_fcmla_lane", [VerifyRuntimeMode], [ImmCheck<3, ImmCheckLaneIndexCompRotate, 2>, 872 ImmCheck<4, ImmCheckComplexRotAll90>]>; 873def SVMLA_LANE : SInst<"svmla_lane[_{d}]", "ddddi", "hfd", MergeNone, "aarch64_sve_fmla_lane", [VerifyRuntimeMode], [ImmCheck<3, ImmCheckLaneIndex, 2>]>; 874def SVMLS_LANE : SInst<"svmls_lane[_{d}]", "ddddi", "hfd", MergeNone, "aarch64_sve_fmls_lane", [VerifyRuntimeMode], [ImmCheck<3, ImmCheckLaneIndex, 2>]>; 875def SVMUL_LANE : SInst<"svmul_lane[_{d}]", "dddi", "hfd", MergeNone, "aarch64_sve_fmul_lane", [VerifyRuntimeMode], [ImmCheck<2, ImmCheckLaneIndex, 1>]>; 876 877def SVRECPE : SInst<"svrecpe[_{d}]", "dd", "hfd", MergeNone, "aarch64_sve_frecpe_x", [VerifyRuntimeMode]>; 878def SVRECPS : SInst<"svrecps[_{d}]", "ddd", "hfd", MergeNone, "aarch64_sve_frecps_x", [VerifyRuntimeMode]>; 879def SVRSQRTE : SInst<"svrsqrte[_{d}]", "dd", "hfd", MergeNone, "aarch64_sve_frsqrte_x", [VerifyRuntimeMode]>; 880def SVRSQRTS : SInst<"svrsqrts[_{d}]", "ddd", "hfd", MergeNone, "aarch64_sve_frsqrts_x", [VerifyRuntimeMode]>; 881 882//////////////////////////////////////////////////////////////////////////////// 883// Floating-point reductions 884 885let SVETargetGuard = "sve", SMETargetGuard = InvalidMode in { 886def SVFADDA : SInst<"svadda[_{d}]", "sPsd", "hfd", MergeNone, "aarch64_sve_fadda">; 887} 888def SVFADDV : SInst<"svaddv[_{d}]", "sPd", "hfd", MergeNone, "aarch64_sve_faddv", [VerifyRuntimeMode]>; 889def SVFMAXV : SInst<"svmaxv[_{d}]", "sPd", "hfd", MergeNone, "aarch64_sve_fmaxv", [VerifyRuntimeMode]>; 890def SVFMAXNMV : SInst<"svmaxnmv[_{d}]", "sPd", "hfd", MergeNone, "aarch64_sve_fmaxnmv", [VerifyRuntimeMode]>; 891def SVFMINV : SInst<"svminv[_{d}]", "sPd", "hfd", MergeNone, "aarch64_sve_fminv", [VerifyRuntimeMode]>; 892def SVFMINNMV : SInst<"svminnmv[_{d}]", "sPd", "hfd", MergeNone, "aarch64_sve_fminnmv", [VerifyRuntimeMode]>; 893 894//////////////////////////////////////////////////////////////////////////////// 895// Floating-point comparisons 896 897def SVACGE : SInst<"svacge[_{d}]", "PPdd", "hfd", MergeNone, "aarch64_sve_facge", [VerifyRuntimeMode]>; 898def SVACGT : SInst<"svacgt[_{d}]", "PPdd", "hfd", MergeNone, "aarch64_sve_facgt", [VerifyRuntimeMode]>; 899def SVACLE : SInst<"svacle[_{d}]", "PPdd", "hfd", MergeNone, "aarch64_sve_facge", [ReverseCompare, VerifyRuntimeMode]>; 900def SVACLT : SInst<"svaclt[_{d}]", "PPdd", "hfd", MergeNone, "aarch64_sve_facgt", [ReverseCompare, VerifyRuntimeMode]>; 901def SVCMPUO : SInst<"svcmpuo[_{d}]", "PPdd", "hfd", MergeNone, "aarch64_sve_fcmpuo", [VerifyRuntimeMode]>; 902 903def SVACGE_N : SInst<"svacge[_n_{d}]", "PPda", "hfd", MergeNone, "aarch64_sve_facge", [VerifyRuntimeMode]>; 904def SVACGT_N : SInst<"svacgt[_n_{d}]", "PPda", "hfd", MergeNone, "aarch64_sve_facgt", [VerifyRuntimeMode]>; 905def SVACLE_N : SInst<"svacle[_n_{d}]", "PPda", "hfd", MergeNone, "aarch64_sve_facge", [ReverseCompare, VerifyRuntimeMode]>; 906def SVACLT_N : SInst<"svaclt[_n_{d}]", "PPda", "hfd", MergeNone, "aarch64_sve_facgt", [ReverseCompare, VerifyRuntimeMode]>; 907def SVCMPUO_N : SInst<"svcmpuo[_n_{d}]", "PPda", "hfd", MergeNone, "aarch64_sve_fcmpuo", [VerifyRuntimeMode]>; 908 909def SVCMPEQ_F : SInst<"svcmpeq[_{d}]", "PPdd", "hfd", MergeNone, "aarch64_sve_fcmpeq", [VerifyRuntimeMode]>; 910def SVCMPNE_F : SInst<"svcmpne[_{d}]", "PPdd", "hfd", MergeNone, "aarch64_sve_fcmpne", [VerifyRuntimeMode]>; 911def SVCMPGE_F : SInst<"svcmpge[_{d}]", "PPdd", "hfd", MergeNone, "aarch64_sve_fcmpge", [VerifyRuntimeMode]>; 912def SVCMPGT_F : SInst<"svcmpgt[_{d}]", "PPdd", "hfd", MergeNone, "aarch64_sve_fcmpgt", [VerifyRuntimeMode]>; 913def SVCMPLE_F : SInst<"svcmple[_{d}]", "PPdd", "hfd", MergeNone, "aarch64_sve_fcmpge", [ReverseCompare, VerifyRuntimeMode]>; 914def SVCMPLT_F : SInst<"svcmplt[_{d}]", "PPdd", "hfd", MergeNone, "aarch64_sve_fcmpgt", [ReverseCompare, VerifyRuntimeMode]>; 915 916def SVCMPEQ_F_N : SInst<"svcmpeq[_n_{d}]", "PPda", "hfd", MergeNone, "aarch64_sve_fcmpeq", [VerifyRuntimeMode]>; 917def SVCMPNE_F_N : SInst<"svcmpne[_n_{d}]", "PPda", "hfd", MergeNone, "aarch64_sve_fcmpne", [VerifyRuntimeMode]>; 918def SVCMPGE_F_N : SInst<"svcmpge[_n_{d}]", "PPda", "hfd", MergeNone, "aarch64_sve_fcmpge", [VerifyRuntimeMode]>; 919def SVCMPGT_F_N : SInst<"svcmpgt[_n_{d}]", "PPda", "hfd", MergeNone, "aarch64_sve_fcmpgt", [VerifyRuntimeMode]>; 920def SVCMPLE_F_N : SInst<"svcmple[_n_{d}]", "PPda", "hfd", MergeNone, "aarch64_sve_fcmpge", [ReverseCompare, VerifyRuntimeMode]>; 921def SVCMPLT_F_N : SInst<"svcmplt[_n_{d}]", "PPda", "hfd", MergeNone, "aarch64_sve_fcmpgt", [ReverseCompare, VerifyRuntimeMode]>; 922 923//////////////////////////////////////////////////////////////////////////////// 924// Floating-point conversions 925 926multiclass SInstCvtMXZ< 927 string name, string m_types, string xz_types, string types, 928 string intrinsic, list<FlagType> flags = [IsOverloadNone]> { 929 def _M : SInst<name, m_types, types, MergeOp1, intrinsic, !listconcat(flags, [VerifyRuntimeMode])>; 930 def _X : SInst<name, xz_types, types, MergeAnyExp, intrinsic, !listconcat(flags, [VerifyRuntimeMode])>; 931 def _Z : SInst<name, xz_types, types, MergeZeroExp, intrinsic, !listconcat(flags, [VerifyRuntimeMode])>; 932} 933 934multiclass SInstCvtMX<string name, string m_types, string xz_types, 935 string types, string intrinsic, 936 list<FlagType> flags = [IsOverloadNone]> { 937 def _M : SInst<name, m_types, types, MergeOp1, intrinsic, !listconcat(flags, [VerifyRuntimeMode])>; 938 def _X : SInst<name, xz_types, types, MergeAnyExp, intrinsic, !listconcat(flags, [VerifyRuntimeMode])>; 939} 940 941// svcvt_s##_f16 942defm SVFCVTZS_S16_F16 : SInstCvtMXZ<"svcvt_s16[_f16]", "ddPO", "dPO", "s", "aarch64_sve_fcvtzs", [IsOverloadCvt]>; 943defm SVFCVTZS_S32_F16 : SInstCvtMXZ<"svcvt_s32[_f16]", "ddPO", "dPO", "i", "aarch64_sve_fcvtzs_i32f16">; 944defm SVFCVTZS_S64_F16 : SInstCvtMXZ<"svcvt_s64[_f16]", "ddPO", "dPO", "l", "aarch64_sve_fcvtzs_i64f16">; 945 946// svcvt_s##_f32 947defm SVFCVTZS_S32_F32 : SInstCvtMXZ<"svcvt_s32[_f32]", "ddPM", "dPM", "i", "aarch64_sve_fcvtzs", [IsOverloadCvt]>; 948defm SVFCVTZS_S64_F32 : SInstCvtMXZ<"svcvt_s64[_f32]", "ddPM", "dPM", "l", "aarch64_sve_fcvtzs_i64f32">; 949 950// svcvt_s##_f64 951defm SVFCVTZS_S32_F64 : SInstCvtMXZ<"svcvt_s32[_f64]", "ttPd", "tPd", "d", "aarch64_sve_fcvtzs_i32f64">; 952defm SVFCVTZS_S64_F64 : SInstCvtMXZ<"svcvt_s64[_f64]", "ddPN", "dPN", "l", "aarch64_sve_fcvtzs", [IsOverloadCvt]>; 953 954// svcvt_u##_f16 955defm SVFCVTZU_U16_F16 : SInstCvtMXZ<"svcvt_u16[_f16]", "ddPO", "dPO", "Us", "aarch64_sve_fcvtzu", [IsOverloadCvt]>; 956defm SVFCVTZU_U32_F16 : SInstCvtMXZ<"svcvt_u32[_f16]", "ddPO", "dPO", "Ui", "aarch64_sve_fcvtzu_i32f16">; 957defm SVFCVTZU_U64_F16 : SInstCvtMXZ<"svcvt_u64[_f16]", "ddPO", "dPO", "Ul", "aarch64_sve_fcvtzu_i64f16">; 958 959// svcvt_u##_f32 960defm SVFCVTZU_U32_F32 : SInstCvtMXZ<"svcvt_u32[_f32]", "ddPM", "dPM", "Ui", "aarch64_sve_fcvtzu", [IsOverloadCvt]>; 961defm SVFCVTZU_U64_F32 : SInstCvtMXZ<"svcvt_u64[_f32]", "ddPM", "dPM", "Ul", "aarch64_sve_fcvtzu_i64f32">; 962 963// svcvt_u##_f64 964defm SVFCVTZU_U32_F64 : SInstCvtMXZ<"svcvt_u32[_f64]", "zzPd", "zPd", "d", "aarch64_sve_fcvtzu_i32f64">; 965defm SVFCVTZU_U64_F64 : SInstCvtMXZ<"svcvt_u64[_f64]", "ddPN", "dPN", "Ul", "aarch64_sve_fcvtzu", [IsOverloadCvt]>; 966 967// svcvt_f16_s## 968defm SVFCVTZS_F16_S16 : SInstCvtMXZ<"svcvt_f16[_s16]", "OOPd", "OPd", "s", "aarch64_sve_scvtf", [IsOverloadCvt]>; 969defm SVFCVTZS_F16_S32 : SInstCvtMXZ<"svcvt_f16[_s32]", "OOPd", "OPd", "i", "aarch64_sve_scvtf_f16i32">; 970defm SVFCVTZS_F16_S64 : SInstCvtMXZ<"svcvt_f16[_s64]", "OOPd", "OPd", "l", "aarch64_sve_scvtf_f16i64">; 971 972// svcvt_f32_s## 973defm SVFCVTZS_F32_S32 : SInstCvtMXZ<"svcvt_f32[_s32]", "MMPd", "MPd", "i", "aarch64_sve_scvtf", [IsOverloadCvt]>; 974defm SVFCVTZS_F32_S64 : SInstCvtMXZ<"svcvt_f32[_s64]", "MMPd", "MPd", "l", "aarch64_sve_scvtf_f32i64">; 975 976// svcvt_f64_s## 977defm SVFCVTZS_F64_S32 : SInstCvtMXZ<"svcvt_f64[_s32]", "ddPt", "dPt", "d", "aarch64_sve_scvtf_f64i32">; 978defm SVFCVTZS_F64_S64 : SInstCvtMXZ<"svcvt_f64[_s64]", "NNPd", "NPd", "l", "aarch64_sve_scvtf", [IsOverloadCvt]>; 979 980// svcvt_f16_u## 981defm SVFCVTZU_F16_U16 : SInstCvtMXZ<"svcvt_f16[_u16]", "OOPd", "OPd", "Us", "aarch64_sve_ucvtf", [IsOverloadCvt]>; 982defm SVFCVTZU_F16_U32 : SInstCvtMXZ<"svcvt_f16[_u32]", "OOPd", "OPd", "Ui", "aarch64_sve_ucvtf_f16i32">; 983defm SVFCVTZU_F16_U64 : SInstCvtMXZ<"svcvt_f16[_u64]", "OOPd", "OPd", "Ul", "aarch64_sve_ucvtf_f16i64">; 984 985// svcvt_f32_u## 986defm SVFCVTZU_F32_U32 : SInstCvtMXZ<"svcvt_f32[_u32]", "MMPd", "MPd", "Ui", "aarch64_sve_ucvtf", [IsOverloadCvt]>; 987defm SVFCVTZU_F32_U64 : SInstCvtMXZ<"svcvt_f32[_u64]", "MMPd", "MPd", "Ul", "aarch64_sve_ucvtf_f32i64">; 988 989// svcvt_f64_u## 990defm SVFCVTZU_F64_U32 : SInstCvtMXZ<"svcvt_f64[_u32]", "ddPz", "dPz", "d", "aarch64_sve_ucvtf_f64i32">; 991defm SVFCVTZU_F64_U64 : SInstCvtMXZ<"svcvt_f64[_u64]", "NNPd", "NPd", "Ul", "aarch64_sve_ucvtf", [IsOverloadCvt]>; 992 993// svcvt_f16_f## 994defm SVFCVT_F16_F32 : SInstCvtMXZ<"svcvt_f16[_f32]", "OOPd", "OPd", "f", "aarch64_sve_fcvt_f16f32">; 995defm SVFCVT_F16_F64 : SInstCvtMXZ<"svcvt_f16[_f64]", "OOPd", "OPd", "d", "aarch64_sve_fcvt_f16f64">; 996 997// svcvt_f32_f## 998defm SVFCVT_F32_F16 : SInstCvtMXZ<"svcvt_f32[_f16]", "ddPO", "dPO", "f", "aarch64_sve_fcvt_f32f16">; 999defm SVFCVT_F32_F64 : SInstCvtMXZ<"svcvt_f32[_f64]", "MMPd", "MPd", "d", "aarch64_sve_fcvt_f32f64">; 1000 1001// svcvt_f64_f## 1002defm SVFCVT_F64_F16 : SInstCvtMXZ<"svcvt_f64[_f16]", "ddPO", "dPO", "d", "aarch64_sve_fcvt_f64f16">; 1003defm SVFCVT_F64_F32 : SInstCvtMXZ<"svcvt_f64[_f32]", "ddPM", "dPM", "d", "aarch64_sve_fcvt_f64f32">; 1004 1005let SVETargetGuard = "sve,bf16", SMETargetGuard = "sme,bf16" in { 1006defm SVCVT_BF16_F32 : SInstCvtMXZ<"svcvt_bf16[_f32]", "$$Pd", "$Pd", "f", "aarch64_sve_fcvt_bf16f32_v2">; 1007 1008def SVCVTNT_BF16_F32 : SInst<"svcvtnt_bf16[_f32]", "$$Pd", "f", MergeOp1, "aarch64_sve_fcvtnt_bf16f32_v2", [IsOverloadNone, VerifyRuntimeMode]>; 1009// SVCVTNT_X_BF16_F32 : Implemented as macro by SveEmitter.cpp 1010} 1011 1012let SVETargetGuard = "sve2", SMETargetGuard = "sme" in { 1013defm SVCVTLT_F32_F16 : SInstCvtMX<"svcvtlt_f32[_f16]", "ddPh", "dPh", "f", "aarch64_sve_fcvtlt_f32f16">; 1014defm SVCVTLT_F64_F32 : SInstCvtMX<"svcvtlt_f64[_f32]", "ddPh", "dPh", "d", "aarch64_sve_fcvtlt_f64f32">; 1015 1016defm SVCVTX_F32_F64 : SInstCvtMXZ<"svcvtx_f32[_f64]", "MMPd", "MPd", "d", "aarch64_sve_fcvtx_f32f64">; 1017 1018def SVCVTNT_F16_F32 : SInst<"svcvtnt_f16[_f32]", "hhPd", "f", MergeOp1, "aarch64_sve_fcvtnt_f16f32", [IsOverloadNone, VerifyRuntimeMode]>; 1019def SVCVTNT_F32_F64 : SInst<"svcvtnt_f32[_f64]", "hhPd", "d", MergeOp1, "aarch64_sve_fcvtnt_f32f64", [IsOverloadNone, VerifyRuntimeMode]>; 1020// SVCVTNT_X_F16_F32 : Implemented as macro by SveEmitter.cpp 1021// SVCVTNT_X_F32_F64 : Implemented as macro by SveEmitter.cpp 1022 1023def SVCVTXNT_F32_F64 : SInst<"svcvtxnt_f32[_f64]", "MMPd", "d", MergeOp1, "aarch64_sve_fcvtxnt_f32f64", [IsOverloadNone, VerifyRuntimeMode]>; 1024// SVCVTXNT_X_F32_F64 : Implemented as macro by SveEmitter.cpp 1025} 1026 1027//////////////////////////////////////////////////////////////////////////////// 1028// Permutations and selection 1029 1030multiclass SVEPerm<string name, string proto, string i> { 1031 def : SInst<name, proto, "csilUcUsUiUlhfd", MergeNone, i, [VerifyRuntimeMode]>; 1032 let SVETargetGuard = "sve,bf16", SMETargetGuard = "sme,bf16" in { 1033 def: SInst<name, proto, "b", MergeNone, i, [VerifyRuntimeMode]>; 1034 } 1035} 1036 1037defm SVCLASTA : SVEPerm<"svclasta[_{d}]", "dPdd", "aarch64_sve_clasta">; 1038defm SVCLASTA_N : SVEPerm<"svclasta[_n_{d}]", "sPsd", "aarch64_sve_clasta_n">; 1039defm SVCLASTB : SVEPerm<"svclastb[_{d}]", "dPdd", "aarch64_sve_clastb">; 1040defm SVCLASTB_N : SVEPerm<"svclastb[_n_{d}]", "sPsd", "aarch64_sve_clastb_n">; 1041 1042let SVETargetGuard = "sve", SMETargetGuard = InvalidMode in { 1043def SVCOMPACT : SInst<"svcompact[_{d}]", "dPd", "ilUiUlfd", MergeNone, "aarch64_sve_compact">; 1044} 1045 1046// Note: svdup_lane is implemented using the intrinsic for TBL to represent a 1047// splat of any possible lane. It is upto LLVM to pick a more efficient 1048// instruction such as DUP (indexed) if the lane index fits the range of the 1049// instruction's immediate. 1050def SVDUP_LANE : SInst<"svdup_lane[_{d}]", "ddL", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_tbl", [VerifyRuntimeMode]>; 1051let SVETargetGuard = "sve,bf16", SMETargetGuard = "sme,bf16" in { 1052def SVDUP_LANE_BF16 : 1053 SInst<"svdup_lane[_{d}]", "ddL", "b", MergeNone, "aarch64_sve_tbl", [VerifyRuntimeMode]>; 1054} 1055 1056def SVDUPQ_LANE : SInst<"svdupq_lane[_{d}]", "ddn", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_dupq_lane", [VerifyRuntimeMode]>; 1057let SVETargetGuard = "sve,bf16", SMETargetGuard = "sme,bf16" in { 1058 def SVDUPQ_LANE_BF16 : SInst<"svdupq_lane[_{d}]", "ddn", "b", MergeNone, "aarch64_sve_dupq_lane", [VerifyRuntimeMode]>; 1059} 1060def SVEXT : SInst<"svext[_{d}]", "dddi", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_ext", [VerifyRuntimeMode], [ImmCheck<2, ImmCheckExtract, 1>]>; 1061defm SVLASTA : SVEPerm<"svlasta[_{d}]", "sPd", "aarch64_sve_lasta">; 1062defm SVLASTB : SVEPerm<"svlastb[_{d}]", "sPd", "aarch64_sve_lastb">; 1063def SVREV : SInst<"svrev[_{d}]", "dd", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_rev", [VerifyRuntimeMode]>; 1064def SVSEL : SInst<"svsel[_{d}]", "dPdd", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_sel", [VerifyRuntimeMode]>; 1065def SVSPLICE : SInst<"svsplice[_{d}]", "dPdd", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_splice", [VerifyRuntimeMode]>; 1066def SVTBL : SInst<"svtbl[_{d}]", "ddu", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_tbl", [VerifyRuntimeMode]>; 1067 1068let SVETargetGuard = "sve,bf16", SMETargetGuard = "sme,bf16" in { 1069 def SVTBL_BF16 : SInst<"svtbl[_{d}]", "ddu", "b", MergeNone, "aarch64_sve_tbl", [VerifyRuntimeMode]>; 1070} 1071 1072def SVTRN1 : SInst<"svtrn1[_{d}]", "ddd", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_trn1", [VerifyRuntimeMode]>; 1073def SVTRN2 : SInst<"svtrn2[_{d}]", "ddd", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_trn2", [VerifyRuntimeMode]>; 1074def SVUNPKHI_S : SInst<"svunpkhi[_{d}]", "dh", "sil", MergeNone, "aarch64_sve_sunpkhi", [VerifyRuntimeMode]>; 1075def SVUNPKHI_U : SInst<"svunpkhi[_{d}]", "dh", "UsUiUl", MergeNone, "aarch64_sve_uunpkhi", [VerifyRuntimeMode]>; 1076def SVUNPKLO_S : SInst<"svunpklo[_{d}]", "dh", "sil", MergeNone, "aarch64_sve_sunpklo", [VerifyRuntimeMode]>; 1077def SVUNPKLO_U : SInst<"svunpklo[_{d}]", "dh", "UsUiUl", MergeNone, "aarch64_sve_uunpklo", [VerifyRuntimeMode]>; 1078def SVUZP1 : SInst<"svuzp1[_{d}]", "ddd", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_uzp1", [VerifyRuntimeMode]>; 1079def SVUZP2 : SInst<"svuzp2[_{d}]", "ddd", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_uzp2", [VerifyRuntimeMode]>; 1080def SVZIP1 : SInst<"svzip1[_{d}]", "ddd", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_zip1", [VerifyRuntimeMode]>; 1081def SVZIP2 : SInst<"svzip2[_{d}]", "ddd", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_zip2", [VerifyRuntimeMode]>; 1082 1083let SVETargetGuard = "sve,bf16", SMETargetGuard = "sme,bf16" in { 1084def SVEXT_BF16 : SInst<"svext[_{d}]", "dddi", "b", MergeNone, "aarch64_sve_ext", [VerifyRuntimeMode], [ImmCheck<2, ImmCheckExtract, 1>]>; 1085def SVREV_BF16 : SInst<"svrev[_{d}]", "dd", "b", MergeNone, "aarch64_sve_rev", [VerifyRuntimeMode]>; 1086def SVSEL_BF16 : SInst<"svsel[_{d}]", "dPdd", "b", MergeNone, "aarch64_sve_sel", [VerifyRuntimeMode]>; 1087def SVSPLICE_BF16 : SInst<"svsplice[_{d}]", "dPdd", "b", MergeNone, "aarch64_sve_splice", [VerifyRuntimeMode]>; 1088def SVTRN1_BF16 : SInst<"svtrn1[_{d}]", "ddd", "b", MergeNone, "aarch64_sve_trn1", [VerifyRuntimeMode]>; 1089def SVTRN2_BF16 : SInst<"svtrn2[_{d}]", "ddd", "b", MergeNone, "aarch64_sve_trn2", [VerifyRuntimeMode]>; 1090def SVUZP1_BF16 : SInst<"svuzp1[_{d}]", "ddd", "b", MergeNone, "aarch64_sve_uzp1", [VerifyRuntimeMode]>; 1091def SVUZP2_BF16 : SInst<"svuzp2[_{d}]", "ddd", "b", MergeNone, "aarch64_sve_uzp2", [VerifyRuntimeMode]>; 1092def SVZIP1_BF16 : SInst<"svzip1[_{d}]", "ddd", "b", MergeNone, "aarch64_sve_zip1", [VerifyRuntimeMode]>; 1093def SVZIP2_BF16 : SInst<"svzip2[_{d}]", "ddd", "b", MergeNone, "aarch64_sve_zip2", [VerifyRuntimeMode]>; 1094} 1095 1096def SVREV_B8 : SInst<"svrev_b8", "PP", "Pc", MergeNone, "aarch64_sve_rev", [VerifyRuntimeMode]>; 1097def SVREV_B16 : SInst<"svrev_b16", "PP", "Pc", MergeNone, "aarch64_sve_rev_b16", [IsOverloadNone, VerifyRuntimeMode]>; 1098def SVREV_B32 : SInst<"svrev_b32", "PP", "Pc", MergeNone, "aarch64_sve_rev_b32", [IsOverloadNone, VerifyRuntimeMode]>; 1099def SVREV_B64 : SInst<"svrev_b64", "PP", "Pc", MergeNone, "aarch64_sve_rev_b64", [IsOverloadNone, VerifyRuntimeMode]>; 1100def SVSEL_B : SInst<"svsel[_b]", "PPPP", "Pc", MergeNone, "aarch64_sve_sel", [VerifyRuntimeMode]>; 1101def SVTRN1_B8 : SInst<"svtrn1_b8", "PPP", "Pc", MergeNone, "aarch64_sve_trn1", [VerifyRuntimeMode]>; 1102def SVTRN1_B16 : SInst<"svtrn1_b16", "PPP", "Pc", MergeNone, "aarch64_sve_trn1_b16", [IsOverloadNone, VerifyRuntimeMode]>; 1103def SVTRN1_B32 : SInst<"svtrn1_b32", "PPP", "Pc", MergeNone, "aarch64_sve_trn1_b32", [IsOverloadNone, VerifyRuntimeMode]>; 1104def SVTRN1_B64 : SInst<"svtrn1_b64", "PPP", "Pc", MergeNone, "aarch64_sve_trn1_b64", [IsOverloadNone, VerifyRuntimeMode]>; 1105def SVTRN2_B8 : SInst<"svtrn2_b8", "PPP", "Pc", MergeNone, "aarch64_sve_trn2", [VerifyRuntimeMode]>; 1106def SVTRN2_B16 : SInst<"svtrn2_b16", "PPP", "Pc", MergeNone, "aarch64_sve_trn2_b16", [IsOverloadNone, VerifyRuntimeMode]>; 1107def SVTRN2_B32 : SInst<"svtrn2_b32", "PPP", "Pc", MergeNone, "aarch64_sve_trn2_b32", [IsOverloadNone, VerifyRuntimeMode]>; 1108def SVTRN2_B64 : SInst<"svtrn2_b64", "PPP", "Pc", MergeNone, "aarch64_sve_trn2_b64", [IsOverloadNone, VerifyRuntimeMode]>; 1109def SVPUNPKHI : SInst<"svunpkhi[_b]", "PP", "Pc", MergeNone, "aarch64_sve_punpkhi", [VerifyRuntimeMode]>; 1110def SVPUNPKLO : SInst<"svunpklo[_b]", "PP", "Pc", MergeNone, "aarch64_sve_punpklo", [VerifyRuntimeMode]>; 1111def SVUZP1_B8 : SInst<"svuzp1_b8", "PPP", "Pc", MergeNone, "aarch64_sve_uzp1", [VerifyRuntimeMode]>; 1112def SVUZP1_B16 : SInst<"svuzp1_b16", "PPP", "Pc", MergeNone, "aarch64_sve_uzp1_b16", [IsOverloadNone, VerifyRuntimeMode]>; 1113def SVUZP1_B32 : SInst<"svuzp1_b32", "PPP", "Pc", MergeNone, "aarch64_sve_uzp1_b32", [IsOverloadNone, VerifyRuntimeMode]>; 1114def SVUZP1_B64 : SInst<"svuzp1_b64", "PPP", "Pc", MergeNone, "aarch64_sve_uzp1_b64", [IsOverloadNone, VerifyRuntimeMode]>; 1115def SVUZP2_B8 : SInst<"svuzp2_b8", "PPP", "Pc", MergeNone, "aarch64_sve_uzp2", [VerifyRuntimeMode]>; 1116def SVUZP2_B16 : SInst<"svuzp2_b16", "PPP", "Pc", MergeNone, "aarch64_sve_uzp2_b16", [IsOverloadNone, VerifyRuntimeMode]>; 1117def SVUZP2_B32 : SInst<"svuzp2_b32", "PPP", "Pc", MergeNone, "aarch64_sve_uzp2_b32", [IsOverloadNone, VerifyRuntimeMode]>; 1118def SVUZP2_B64 : SInst<"svuzp2_b64", "PPP", "Pc", MergeNone, "aarch64_sve_uzp2_b64", [IsOverloadNone, VerifyRuntimeMode]>; 1119def SVZIP1_B8 : SInst<"svzip1_b8", "PPP", "Pc", MergeNone, "aarch64_sve_zip1", [VerifyRuntimeMode]>; 1120def SVZIP1_B16 : SInst<"svzip1_b16", "PPP", "Pc", MergeNone, "aarch64_sve_zip1_b16", [IsOverloadNone, VerifyRuntimeMode]>; 1121def SVZIP1_B32 : SInst<"svzip1_b32", "PPP", "Pc", MergeNone, "aarch64_sve_zip1_b32", [IsOverloadNone, VerifyRuntimeMode]>; 1122def SVZIP1_B64 : SInst<"svzip1_b64", "PPP", "Pc", MergeNone, "aarch64_sve_zip1_b64", [IsOverloadNone, VerifyRuntimeMode]>; 1123def SVZIP2_B : SInst<"svzip2_b8", "PPP", "Pc", MergeNone, "aarch64_sve_zip2", [VerifyRuntimeMode]>; 1124def SVZIP2_B16 : SInst<"svzip2_b16", "PPP", "Pc", MergeNone, "aarch64_sve_zip2_b16", [IsOverloadNone, VerifyRuntimeMode]>; 1125def SVZIP2_B32 : SInst<"svzip2_b32", "PPP", "Pc", MergeNone, "aarch64_sve_zip2_b32", [IsOverloadNone, VerifyRuntimeMode]>; 1126def SVZIP2_B64 : SInst<"svzip2_b64", "PPP", "Pc", MergeNone, "aarch64_sve_zip2_b64", [IsOverloadNone, VerifyRuntimeMode]>; 1127 1128//////////////////////////////////////////////////////////////////////////////// 1129// Predicate creation 1130 1131def SVPFALSE : SInst<"svpfalse[_b]", "Pv", "", MergeNone, "", [IsOverloadNone, VerifyRuntimeMode]>; 1132def SVPTRUE_PAT : SInst<"svptrue_pat_{d}", "PI", "PcPsPiPl", MergeNone, "aarch64_sve_ptrue", [VerifyRuntimeMode]>; 1133def SVPTRUE : SInst<"svptrue_{d}", "Pv", "PcPsPiPl", MergeNone, "aarch64_sve_ptrue", [IsAppendSVALL, VerifyRuntimeMode]>; 1134 1135def SVDUPQ_B8 : SInst<"svdupq[_n]_{d}", "Pssssssssssssssss", "Pc", MergeNone, "", [VerifyRuntimeMode]>; 1136def SVDUPQ_B16 : SInst<"svdupq[_n]_{d}", "Pssssssss", "Ps", MergeNone, "", [VerifyRuntimeMode]>; 1137def SVDUPQ_B32 : SInst<"svdupq[_n]_{d}", "Pssss", "Pi", MergeNone, "", [VerifyRuntimeMode]>; 1138def SVDUPQ_B64 : SInst<"svdupq[_n]_{d}", "Pss", "Pl", MergeNone, "", [VerifyRuntimeMode]>; 1139def SVDUP_N_B : SInst<"svdup[_n]_{d}", "Ps", "PcPsPiPl", MergeNone, "", [VerifyRuntimeMode]>; 1140 1141 1142//////////////////////////////////////////////////////////////////////////////// 1143// Predicate operations 1144 1145def SVAND_B_Z : SInst<"svand[_b]_z", "PPPP", "Pc", MergeNone, "aarch64_sve_and_z", [VerifyRuntimeMode]>; 1146def SVBIC_B_Z : SInst<"svbic[_b]_z", "PPPP", "Pc", MergeNone, "aarch64_sve_bic_z", [VerifyRuntimeMode]>; 1147def SVEOR_B_Z : SInst<"sveor[_b]_z", "PPPP", "Pc", MergeNone, "aarch64_sve_eor_z", [VerifyRuntimeMode]>; 1148def SVMOV_B_Z : SInst<"svmov[_b]_z", "PPP", "Pc", MergeNone, "", [VerifyRuntimeMode]>; // Uses custom expansion 1149def SVNAND_B_Z : SInst<"svnand[_b]_z", "PPPP", "Pc", MergeNone, "aarch64_sve_nand_z", [VerifyRuntimeMode]>; 1150def SVNOR_B_Z : SInst<"svnor[_b]_z", "PPPP", "Pc", MergeNone, "aarch64_sve_nor_z", [VerifyRuntimeMode]>; 1151def SVNOT_B_Z : SInst<"svnot[_b]_z", "PPP", "Pc", MergeNone, "", [VerifyRuntimeMode]>; // Uses custom expansion 1152def SVORN_B_Z : SInst<"svorn[_b]_z", "PPPP", "Pc", MergeNone, "aarch64_sve_orn_z", [VerifyRuntimeMode]>; 1153def SVORR_B_Z : SInst<"svorr[_b]_z", "PPPP", "Pc", MergeNone, "aarch64_sve_orr_z", [VerifyRuntimeMode]>; 1154 1155def SVBRKA : SInst<"svbrka[_b]_m", "PPPP", "Pc", MergeNone, "aarch64_sve_brka", [VerifyRuntimeMode]>; 1156def SVBRKA_Z : SInst<"svbrka[_b]_z", "PPP", "Pc", MergeNone, "aarch64_sve_brka_z", [VerifyRuntimeMode]>; 1157def SVBRKB : SInst<"svbrkb[_b]_m", "PPPP", "Pc", MergeNone, "aarch64_sve_brkb", [VerifyRuntimeMode]>; 1158def SVBRKB_Z : SInst<"svbrkb[_b]_z", "PPP", "Pc", MergeNone, "aarch64_sve_brkb_z", [VerifyRuntimeMode]>; 1159def SVBRKN_Z : SInst<"svbrkn[_b]_z", "PPPP", "Pc", MergeNone, "aarch64_sve_brkn_z", [VerifyRuntimeMode]>; 1160def SVBRKPA_Z : SInst<"svbrkpa[_b]_z", "PPPP", "Pc", MergeNone, "aarch64_sve_brkpa_z", [VerifyRuntimeMode]>; 1161def SVBRKPB_Z : SInst<"svbrkpb[_b]_z", "PPPP", "Pc", MergeNone, "aarch64_sve_brkpb_z", [VerifyRuntimeMode]>; 1162 1163def SVPFIRST : SInst<"svpfirst[_b]", "PPP", "Pc", MergeNone, "aarch64_sve_pfirst", [VerifyRuntimeMode]>; 1164def SVPNEXT : SInst<"svpnext_{d}", "PPP", "PcPsPiPl", MergeNone, "aarch64_sve_pnext", [VerifyRuntimeMode]>; 1165 1166//////////////////////////////////////////////////////////////////////////////// 1167// Testing predicates 1168 1169def SVPTEST_ANY : SInst<"svptest_any", "sPP", "Pc", MergeNone, "aarch64_sve_ptest_any", [VerifyRuntimeMode]>; 1170def SVPTEST_FIRST : SInst<"svptest_first", "sPP", "Pc", MergeNone, "aarch64_sve_ptest_first", [VerifyRuntimeMode]>; 1171def SVPTEST_LAST : SInst<"svptest_last", "sPP", "Pc", MergeNone, "aarch64_sve_ptest_last", [VerifyRuntimeMode]>; 1172 1173//////////////////////////////////////////////////////////////////////////////// 1174// FFR manipulation 1175 1176let SVETargetGuard = "sve", SMETargetGuard = InvalidMode in { 1177def SVRDFFR : SInst<"svrdffr", "Pv", "Pc", MergeNone, "", [IsOverloadNone]>; 1178def SVRDFFR_Z : SInst<"svrdffr_z", "PP", "Pc", MergeNone, "", [IsOverloadNone]>; 1179def SVSETFFR : SInst<"svsetffr", "vv", "", MergeNone, "", [IsOverloadNone]>; 1180def SVWRFFR : SInst<"svwrffr", "vP", "Pc", MergeNone, "", [IsOverloadNone]>; 1181} 1182 1183//////////////////////////////////////////////////////////////////////////////// 1184// Counting elements 1185 1186def SVCNTB_PAT : SInst<"svcntb_pat", "nI", "", MergeNone, "aarch64_sve_cntb", [IsOverloadNone, VerifyRuntimeMode]>; 1187def SVCNTH_PAT : SInst<"svcnth_pat", "nI", "", MergeNone, "aarch64_sve_cnth", [IsOverloadNone, VerifyRuntimeMode]>; 1188def SVCNTW_PAT : SInst<"svcntw_pat", "nI", "", MergeNone, "aarch64_sve_cntw", [IsOverloadNone, VerifyRuntimeMode]>; 1189def SVCNTD_PAT : SInst<"svcntd_pat", "nI", "", MergeNone, "aarch64_sve_cntd", [IsOverloadNone, VerifyRuntimeMode]>; 1190 1191def SVCNTB : SInst<"svcntb", "nv", "", MergeNone, "aarch64_sve_cntb", [IsAppendSVALL, IsOverloadNone, VerifyRuntimeMode]>; 1192def SVCNTH : SInst<"svcnth", "nv", "", MergeNone, "aarch64_sve_cnth", [IsAppendSVALL, IsOverloadNone, VerifyRuntimeMode]>; 1193def SVCNTW : SInst<"svcntw", "nv", "", MergeNone, "aarch64_sve_cntw", [IsAppendSVALL, IsOverloadNone, VerifyRuntimeMode]>; 1194def SVCNTD : SInst<"svcntd", "nv", "", MergeNone, "aarch64_sve_cntd", [IsAppendSVALL, IsOverloadNone, VerifyRuntimeMode]>; 1195 1196def SVCNTP : SInst<"svcntp_{d}", "nPP", "PcPsPiPl", MergeNone, "aarch64_sve_cntp", [VerifyRuntimeMode]>; 1197def SVLEN : SInst<"svlen[_{d}]", "nd", "csilUcUsUiUlhfd", MergeNone, "", [VerifyRuntimeMode]>; 1198 1199let SVETargetGuard = "sve,bf16", SMETargetGuard = "sme,bf16" in { 1200def SVLEN_BF16 : SInst<"svlen[_{d}]", "nd", "b", MergeNone, "", [VerifyRuntimeMode]>; 1201} 1202 1203//////////////////////////////////////////////////////////////////////////////// 1204// Saturating scalar arithmetic 1205 1206class sat_type<string u, string t> { string U = u; string T = t; } 1207def SignedByte : sat_type<"", "c">; 1208def SignedHalf : sat_type<"", "s">; 1209def SignedWord : sat_type<"", "i">; 1210def SignedDoubleWord : sat_type<"", "l">; 1211def UnsignedByte : sat_type<"U", "Uc">; 1212def UnsignedHalf : sat_type<"U", "Us">; 1213def UnsignedWord : sat_type<"U", "Ui">; 1214def UnsignedDoubleWord : sat_type<"U", "Ul">; 1215 1216multiclass SInst_SAT1<string name, string intrinsic, sat_type type> { 1217 def _N32 : SInst<name # "_pat[_n_{d}]", "ssIi", type.U # "i", MergeNone, intrinsic # "_n32", [IsOverloadNone, VerifyRuntimeMode], [ImmCheck<2, ImmCheck1_16>]>; 1218 def _N64 : SInst<name # "_pat[_n_{d}]", "ssIi", type.U # "l", MergeNone, intrinsic # "_n64", [IsOverloadNone, VerifyRuntimeMode], [ImmCheck<2, ImmCheck1_16>]>; 1219 def _N32_ALL : SInst<name # "[_n_{d}]", "ssi", type.U # "i", MergeNone, intrinsic # "_n32", [IsOverloadNone, IsInsertOp1SVALL, VerifyRuntimeMode], [ImmCheck<1, ImmCheck1_16>]>; 1220 def _N64_ALL : SInst<name # "[_n_{d}]", "ssi", type.U # "l", MergeNone, intrinsic # "_n64", [IsOverloadNone, IsInsertOp1SVALL, VerifyRuntimeMode], [ImmCheck<1, ImmCheck1_16>]>; 1221} 1222 1223multiclass SInst_SAT2<string name, string intrinsic, sat_type type> { 1224 def "" : SInst<name # "_pat[_{d}]", "ddIi", type.T, MergeNone, intrinsic, [VerifyRuntimeMode], [ImmCheck<2, ImmCheck1_16>]>; 1225 def _ALL : SInst<name # "[_{d}]", "ddi", type.T, MergeNone, intrinsic, [IsInsertOp1SVALL, VerifyRuntimeMode], [ImmCheck<1, ImmCheck1_16>]>; 1226 1227 def _N32 : SInst<name # "_pat[_n_{d}]", "ssIi", type.U # "i", MergeNone, intrinsic # "_n32", [IsOverloadNone, VerifyRuntimeMode], [ImmCheck<2, ImmCheck1_16>]>; 1228 def _N64 : SInst<name # "_pat[_n_{d}]", "ssIi", type.U # "l", MergeNone, intrinsic # "_n64", [IsOverloadNone, VerifyRuntimeMode], [ImmCheck<2, ImmCheck1_16>]>; 1229 def _N32_ALL : SInst<name # "[_n_{d}]", "ssi", type.U # "i", MergeNone, intrinsic # "_n32", [IsOverloadNone, IsInsertOp1SVALL, VerifyRuntimeMode], [ImmCheck<1, ImmCheck1_16>]>; 1230 def _N64_ALL : SInst<name # "[_n_{d}]", "ssi", type.U # "l", MergeNone, intrinsic # "_n64", [IsOverloadNone, IsInsertOp1SVALL, VerifyRuntimeMode], [ImmCheck<1, ImmCheck1_16>]>; 1231} 1232 1233defm SVQDECB_S : SInst_SAT1<"svqdecb", "aarch64_sve_sqdecb", SignedByte>; 1234defm SVQDECB_U : SInst_SAT1<"svqdecb", "aarch64_sve_uqdecb", UnsignedByte>; 1235defm SVQDECH_S : SInst_SAT2<"svqdech", "aarch64_sve_sqdech", SignedHalf>; 1236defm SVQDECH_U : SInst_SAT2<"svqdech", "aarch64_sve_uqdech", UnsignedHalf>; 1237defm SVQDECW_S : SInst_SAT2<"svqdecw", "aarch64_sve_sqdecw", SignedWord>; 1238defm SVQDECW_U : SInst_SAT2<"svqdecw", "aarch64_sve_uqdecw", UnsignedWord>; 1239defm SVQDECD_S : SInst_SAT2<"svqdecd", "aarch64_sve_sqdecd", SignedDoubleWord>; 1240defm SVQDECD_U : SInst_SAT2<"svqdecd", "aarch64_sve_uqdecd", UnsignedDoubleWord>; 1241 1242defm SVQINCB_S : SInst_SAT1<"svqincb", "aarch64_sve_sqincb", SignedByte>; 1243defm SVQINCB_U : SInst_SAT1<"svqincb", "aarch64_sve_uqincb", UnsignedByte>; 1244defm SVQINCH_S : SInst_SAT2<"svqinch", "aarch64_sve_sqinch", SignedHalf>; 1245defm SVQINCH_U : SInst_SAT2<"svqinch", "aarch64_sve_uqinch", UnsignedHalf>; 1246defm SVQINCW_S : SInst_SAT2<"svqincw", "aarch64_sve_sqincw", SignedWord>; 1247defm SVQINCW_U : SInst_SAT2<"svqincw", "aarch64_sve_uqincw", UnsignedWord>; 1248defm SVQINCD_S : SInst_SAT2<"svqincd", "aarch64_sve_sqincd", SignedDoubleWord>; 1249defm SVQINCD_U : SInst_SAT2<"svqincd", "aarch64_sve_uqincd", UnsignedDoubleWord>; 1250 1251def SVQDECP_S : SInst<"svqdecp[_{d}]", "ddP", "sil", MergeNone, "aarch64_sve_sqdecp", [VerifyRuntimeMode]>; 1252def SVQDECP_U : SInst<"svqdecp[_{d}]", "ddP", "UsUiUl", MergeNone, "aarch64_sve_uqdecp", [VerifyRuntimeMode]>; 1253def SVQINCP_S : SInst<"svqincp[_{d}]", "ddP", "sil", MergeNone, "aarch64_sve_sqincp", [VerifyRuntimeMode]>; 1254def SVQINCP_U : SInst<"svqincp[_{d}]", "ddP", "UsUiUl", MergeNone, "aarch64_sve_uqincp", [VerifyRuntimeMode]>; 1255 1256def SVQDECP_N_S32 : SInst<"svqdecp[_n_s32]_{d}", "kkP", "PcPsPiPl", MergeNone, "aarch64_sve_sqdecp_n32", [VerifyRuntimeMode]>; 1257def SVQDECP_N_S64 : SInst<"svqdecp[_n_s64]_{d}", "llP", "PcPsPiPl", MergeNone, "aarch64_sve_sqdecp_n64", [VerifyRuntimeMode]>; 1258def SVQDECP_N_U32 : SInst<"svqdecp[_n_u32]_{d}", "mmP", "PcPsPiPl", MergeNone, "aarch64_sve_uqdecp_n32", [VerifyRuntimeMode]>; 1259def SVQDECP_N_U64 : SInst<"svqdecp[_n_u64]_{d}", "nnP", "PcPsPiPl", MergeNone, "aarch64_sve_uqdecp_n64", [VerifyRuntimeMode]>; 1260def SVQINCP_N_S32 : SInst<"svqincp[_n_s32]_{d}", "kkP", "PcPsPiPl", MergeNone, "aarch64_sve_sqincp_n32", [VerifyRuntimeMode]>; 1261def SVQINCP_N_S64 : SInst<"svqincp[_n_s64]_{d}", "llP", "PcPsPiPl", MergeNone, "aarch64_sve_sqincp_n64", [VerifyRuntimeMode]>; 1262def SVQINCP_N_U32 : SInst<"svqincp[_n_u32]_{d}", "mmP", "PcPsPiPl", MergeNone, "aarch64_sve_uqincp_n32", [VerifyRuntimeMode]>; 1263def SVQINCP_N_U64 : SInst<"svqincp[_n_u64]_{d}", "nnP", "PcPsPiPl", MergeNone, "aarch64_sve_uqincp_n64", [VerifyRuntimeMode]>; 1264 1265let SVETargetGuard = "sve,i8mm", SMETargetGuard = InvalidMode in { 1266def SVMLLA_S32 : SInst<"svmmla[_s32]", "ddqq","i", MergeNone, "aarch64_sve_smmla">; 1267def SVMLLA_U32 : SInst<"svmmla[_u32]", "ddqq","Ui", MergeNone, "aarch64_sve_ummla">; 1268def SVUSMLLA_S32 : SInst<"svusmmla[_s32]", "ddbq","i", MergeNone, "aarch64_sve_usmmla">; 1269} 1270 1271let SVETargetGuard = "sve,i8mm", SMETargetGuard = "sme,i8mm"in { 1272def SVUSDOT_S : SInst<"svusdot[_s32]", "ddbq", "i", MergeNone, "aarch64_sve_usdot", [VerifyRuntimeMode]>; 1273def SVUSDOT_N_S : SInst<"svusdot[_n_s32]", "ddbr", "i", MergeNone, "aarch64_sve_usdot", [VerifyRuntimeMode]>; 1274def SVSUDOT_S : SInst<"svsudot[_s32]", "ddqb", "i", MergeNone, "aarch64_sve_usdot", [ReverseUSDOT, VerifyRuntimeMode]>; 1275def SVSUDOT_N_S : SInst<"svsudot[_n_s32]", "ddq@", "i", MergeNone, "aarch64_sve_usdot", [ReverseUSDOT, VerifyRuntimeMode]>; 1276 1277def SVUSDOT_LANE_S : SInst<"svusdot_lane[_s32]", "ddbqi", "i", MergeNone, "aarch64_sve_usdot_lane", [VerifyRuntimeMode], [ImmCheck<3, ImmCheckLaneIndexDot, 2>]>; 1278def SVSUDOT_LANE_S : SInst<"svsudot_lane[_s32]", "ddqbi", "i", MergeNone, "aarch64_sve_sudot_lane", [VerifyRuntimeMode], [ImmCheck<3, ImmCheckLaneIndexDot, 2>]>; 1279} 1280 1281let SVETargetGuard = "sve,f32mm", SMETargetGuard = InvalidMode in { 1282def SVMLLA_F32 : SInst<"svmmla[_f32]", "dddd","f", MergeNone, "aarch64_sve_fmmla">; 1283} 1284 1285let SVETargetGuard = "sve,f64mm", SMETargetGuard = InvalidMode in { 1286def SVMLLA_F64 : SInst<"svmmla[_f64]", "dddd","d", MergeNone, "aarch64_sve_fmmla">; 1287def SVTRN1Q : SInst<"svtrn1q[_{d}]", "ddd", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_trn1q">; 1288def SVTRN2Q : SInst<"svtrn2q[_{d}]", "ddd", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_trn2q">; 1289def SVUZP1Q : SInst<"svuzp1q[_{d}]", "ddd", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_uzp1q">; 1290def SVUZP2Q : SInst<"svuzp2q[_{d}]", "ddd", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_uzp2q">; 1291def SVZIP1Q : SInst<"svzip1q[_{d}]", "ddd", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_zip1q">; 1292def SVZIP2Q : SInst<"svzip2q[_{d}]", "ddd", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_zip2q">; 1293} 1294 1295let SVETargetGuard = "sve,bf16,f64mm", SMETargetGuard = InvalidMode in { 1296def SVTRN1Q_BF16 : SInst<"svtrn1q[_{d}]", "ddd", "b", MergeNone, "aarch64_sve_trn1q">; 1297def SVTRN2Q_BF16 : SInst<"svtrn2q[_{d}]", "ddd", "b", MergeNone, "aarch64_sve_trn2q">; 1298def SVUZP1Q_BF16 : SInst<"svuzp1q[_{d}]", "ddd", "b", MergeNone, "aarch64_sve_uzp1q">; 1299def SVUZP2Q_BF16 : SInst<"svuzp2q[_{d}]", "ddd", "b", MergeNone, "aarch64_sve_uzp2q">; 1300def SVZIP1Q_BF16 : SInst<"svzip1q[_{d}]", "ddd", "b", MergeNone, "aarch64_sve_zip1q">; 1301def SVZIP2Q_BF16 : SInst<"svzip2q[_{d}]", "ddd", "b", MergeNone, "aarch64_sve_zip2q">; 1302} 1303 1304//////////////////////////////////////////////////////////////////////////////// 1305// Vector creation 1306def SVUNDEF_1 : SInst<"svundef_{d}", "dv", "csilUcUsUiUlhfd", MergeNone, "", [IsUndef, VerifyRuntimeMode]>; 1307def SVUNDEF_2 : SInst<"svundef2_{d}", "2v", "csilUcUsUiUlhfd", MergeNone, "", [IsUndef, VerifyRuntimeMode]>; 1308def SVUNDEF_3 : SInst<"svundef3_{d}", "3v", "csilUcUsUiUlhfd", MergeNone, "", [IsUndef, VerifyRuntimeMode]>; 1309def SVUNDEF_4 : SInst<"svundef4_{d}", "4v", "csilUcUsUiUlhfd", MergeNone, "", [IsUndef, VerifyRuntimeMode]>; 1310 1311def SVCREATE_2 : SInst<"svcreate2[_{d}]", "2dd", "csilUcUsUiUlhfd", MergeNone, "", [IsTupleCreate, VerifyRuntimeMode]>; 1312def SVCREATE_3 : SInst<"svcreate3[_{d}]", "3ddd", "csilUcUsUiUlhfd", MergeNone, "", [IsTupleCreate, VerifyRuntimeMode]>; 1313def SVCREATE_4 : SInst<"svcreate4[_{d}]", "4dddd", "csilUcUsUiUlhfd", MergeNone, "", [IsTupleCreate, VerifyRuntimeMode]>; 1314 1315let SVETargetGuard = "sve,bf16", SMETargetGuard = "sme,bf16" in { 1316def SVUNDEF_1_BF16 : SInst<"svundef_{d}", "dv", "b", MergeNone, "", [IsUndef, VerifyRuntimeMode]>; 1317def SVUNDEF_2_BF16 : SInst<"svundef2_{d}", "2v", "b", MergeNone, "", [IsUndef, VerifyRuntimeMode]>; 1318def SVUNDEF_3_BF16 : SInst<"svundef3_{d}", "3v", "b", MergeNone, "", [IsUndef, VerifyRuntimeMode]>; 1319def SVUNDEF_4_BF16 : SInst<"svundef4_{d}", "4v", "b", MergeNone, "", [IsUndef, VerifyRuntimeMode]>; 1320 1321def SVCREATE_2_BF16 : SInst<"svcreate2[_{d}]", "2dd", "b", MergeNone, "", [IsTupleCreate, VerifyRuntimeMode]>; 1322def SVCREATE_3_BF16 : SInst<"svcreate3[_{d}]", "3ddd", "b", MergeNone, "", [IsTupleCreate, VerifyRuntimeMode]>; 1323def SVCREATE_4_BF16 : SInst<"svcreate4[_{d}]", "4dddd", "b", MergeNone, "", [IsTupleCreate, VerifyRuntimeMode]>; 1324} 1325 1326let SVETargetGuard = "sve2p1", SMETargetGuard = "sme2" in { 1327 def SVCREATE_2_B : SInst<"svcreate2[_b]", "2dd", "Pc", MergeNone, "", [IsTupleCreate, VerifyRuntimeMode]>; 1328 def SVCREATE_4_B : SInst<"svcreate4[_b]", "4dddd", "Pc", MergeNone, "", [IsTupleCreate, VerifyRuntimeMode]>; 1329} 1330 1331//////////////////////////////////////////////////////////////////////////////// 1332// Vector insertion and extraction 1333def SVGET_2 : SInst<"svget2[_{d}]", "d2i", "csilUcUsUiUlhfd", MergeNone, "", [IsTupleGet, VerifyRuntimeMode], [ImmCheck<1, ImmCheck0_1>]>; 1334def SVGET_3 : SInst<"svget3[_{d}]", "d3i", "csilUcUsUiUlhfd", MergeNone, "", [IsTupleGet, VerifyRuntimeMode], [ImmCheck<1, ImmCheck0_2>]>; 1335def SVGET_4 : SInst<"svget4[_{d}]", "d4i", "csilUcUsUiUlhfd", MergeNone, "", [IsTupleGet, VerifyRuntimeMode], [ImmCheck<1, ImmCheck0_3>]>; 1336 1337def SVSET_2 : SInst<"svset2[_{d}]", "22id", "csilUcUsUiUlhfd", MergeNone, "", [IsTupleSet, VerifyRuntimeMode], [ImmCheck<1, ImmCheck0_1>]>; 1338def SVSET_3 : SInst<"svset3[_{d}]", "33id", "csilUcUsUiUlhfd", MergeNone, "", [IsTupleSet, VerifyRuntimeMode], [ImmCheck<1, ImmCheck0_2>]>; 1339def SVSET_4 : SInst<"svset4[_{d}]", "44id", "csilUcUsUiUlhfd", MergeNone, "", [IsTupleSet, VerifyRuntimeMode], [ImmCheck<1, ImmCheck0_3>]>; 1340 1341let SVETargetGuard = "sve,bf16", SMETargetGuard = "sme,bf16" in { 1342def SVGET_2_BF16 : SInst<"svget2[_{d}]", "d2i", "b", MergeNone, "", [IsTupleGet, VerifyRuntimeMode], [ImmCheck<1, ImmCheck0_1>]>; 1343def SVGET_3_BF16 : SInst<"svget3[_{d}]", "d3i", "b", MergeNone, "", [IsTupleGet, VerifyRuntimeMode], [ImmCheck<1, ImmCheck0_2>]>; 1344def SVGET_4_BF16 : SInst<"svget4[_{d}]", "d4i", "b", MergeNone, "", [IsTupleGet, VerifyRuntimeMode], [ImmCheck<1, ImmCheck0_3>]>; 1345 1346def SVSET_2_BF16 : SInst<"svset2[_{d}]", "22id", "b", MergeNone, "", [IsTupleSet, VerifyRuntimeMode], [ImmCheck<1, ImmCheck0_1>]>; 1347def SVSET_3_BF16 : SInst<"svset3[_{d}]", "33id", "b", MergeNone, "", [IsTupleSet, VerifyRuntimeMode], [ImmCheck<1, ImmCheck0_2>]>; 1348def SVSET_4_BF16 : SInst<"svset4[_{d}]", "44id", "b", MergeNone, "", [IsTupleSet, VerifyRuntimeMode], [ImmCheck<1, ImmCheck0_3>]>; 1349} 1350 1351let SVETargetGuard = "sve2p1", SMETargetGuard = "sme2" in { 1352 def SVGET_2_B : SInst<"svget2[_b]", "d2i", "Pc", MergeNone, "", [IsTupleGet, VerifyRuntimeMode], [ImmCheck<1, ImmCheck0_1>]>; 1353 def SVGET_4_B : SInst<"svget4[_b]", "d4i", "Pc", MergeNone, "", [IsTupleGet, VerifyRuntimeMode], [ImmCheck<1, ImmCheck0_3>]>; 1354 1355 def SVSET_2_B : SInst<"svset2[_b]", "22id", "Pc", MergeNone, "", [IsTupleSet, VerifyRuntimeMode], [ImmCheck<1, ImmCheck0_1>]>; 1356 def SVSET_4_B : SInst<"svset4[_b]", "44id", "Pc", MergeNone, "", [IsTupleSet, VerifyRuntimeMode], [ImmCheck<1, ImmCheck0_3>]>; 1357} 1358 1359let SVETargetGuard = "sve2p1", SMETargetGuard = "sme2" in { 1360 def SVUNDEF_2_B: Inst<"svundef2_b", "2", "Pc", MergeNone, "", [IsUndef, VerifyRuntimeMode], []>; 1361 def SVUNDEF_4_B: Inst<"svundef4_b", "4", "Pc", MergeNone, "", [IsUndef, VerifyRuntimeMode], []>; 1362} 1363//////////////////////////////////////////////////////////////////////////////// 1364// SVE2 WhileGE/GT 1365let SVETargetGuard = "sve2", SMETargetGuard = "sme" in { 1366def SVWHILEGE_S32 : SInst<"svwhilege_{d}[_{1}]", "Pkk", "PcPsPiPl", MergeNone, "aarch64_sve_whilege", [IsOverloadWhileOrMultiVecCvt, VerifyRuntimeMode]>; 1367def SVWHILEGE_S64 : SInst<"svwhilege_{d}[_{1}]", "Pll", "PcPsPiPl", MergeNone, "aarch64_sve_whilege", [IsOverloadWhileOrMultiVecCvt, VerifyRuntimeMode]>; 1368def SVWHILEGT_S32 : SInst<"svwhilegt_{d}[_{1}]", "Pkk", "PcPsPiPl", MergeNone, "aarch64_sve_whilegt", [IsOverloadWhileOrMultiVecCvt, VerifyRuntimeMode]>; 1369def SVWHILEGT_S64 : SInst<"svwhilegt_{d}[_{1}]", "Pll", "PcPsPiPl", MergeNone, "aarch64_sve_whilegt", [IsOverloadWhileOrMultiVecCvt, VerifyRuntimeMode]>; 1370def SVWHILEHI_U32 : SInst<"svwhilegt_{d}[_{1}]", "Pmm", "PcPsPiPl", MergeNone, "aarch64_sve_whilehi", [IsOverloadWhileOrMultiVecCvt, VerifyRuntimeMode]>; 1371def SVWHILEHI_U64 : SInst<"svwhilegt_{d}[_{1}]", "Pnn", "PcPsPiPl", MergeNone, "aarch64_sve_whilehi", [IsOverloadWhileOrMultiVecCvt, VerifyRuntimeMode]>; 1372def SVWHILEHS_U32 : SInst<"svwhilege_{d}[_{1}]", "Pmm", "PcPsPiPl", MergeNone, "aarch64_sve_whilehs", [IsOverloadWhileOrMultiVecCvt, VerifyRuntimeMode]>; 1373def SVWHILEHS_U64 : SInst<"svwhilege_{d}[_{1}]", "Pnn", "PcPsPiPl", MergeNone, "aarch64_sve_whilehs", [IsOverloadWhileOrMultiVecCvt, VerifyRuntimeMode]>; 1374} 1375 1376let SVETargetGuard = "sve2p1", SMETargetGuard = "sme2" in { 1377 def SVWHILEGE_S64_X2 : SInst<"svwhilege_{d}[_{1}]_x2", "2ll", "PcPsPiPl", MergeNone, "aarch64_sve_whilege_x2", [VerifyRuntimeMode]>; 1378 def SVWHILEGT_S64_X2 : SInst<"svwhilegt_{d}[_{1}]_x2", "2ll", "PcPsPiPl", MergeNone, "aarch64_sve_whilegt_x2", [VerifyRuntimeMode]>; 1379 def SVWHILEHI_U64_X2 : SInst<"svwhilegt_{d}[_{1}]_x2", "2nn", "PcPsPiPl", MergeNone, "aarch64_sve_whilehi_x2", [VerifyRuntimeMode]>; 1380 def SVWHILEHS_U64_X2 : SInst<"svwhilege_{d}[_{1}]_x2", "2nn", "PcPsPiPl", MergeNone, "aarch64_sve_whilehs_x2", [VerifyRuntimeMode]>; 1381 def SVWHILELE_S64_X2 : SInst<"svwhilele_{d}[_{1}]_x2", "2ll", "PcPsPiPl", MergeNone, "aarch64_sve_whilele_x2", [VerifyRuntimeMode]>; 1382 def SVWHILELT_S64_X2 : SInst<"svwhilelt_{d}[_{1}]_x2", "2ll", "PcPsPiPl", MergeNone, "aarch64_sve_whilelt_x2", [VerifyRuntimeMode]>; 1383 def SVWHILELO_U64_X2 : SInst<"svwhilelt_{d}[_{1}]_x2", "2nn", "PcPsPiPl", MergeNone, "aarch64_sve_whilelo_x2", [VerifyRuntimeMode]>; 1384 def SVWHILELS_U64_X2 : SInst<"svwhilele_{d}[_{1}]_x2", "2nn", "PcPsPiPl", MergeNone, "aarch64_sve_whilels_x2", [VerifyRuntimeMode]>; 1385 1386} 1387 1388//////////////////////////////////////////////////////////////////////////////// 1389// SVE2 - Uniform DSP operations 1390 1391let SVETargetGuard = "sve2", SMETargetGuard = "sme" in { 1392defm SVQADD_S : SInstZPZZ<"svqadd", "csli", "aarch64_sve_sqadd", "aarch64_sve_sqadd">; 1393defm SVQADD_U : SInstZPZZ<"svqadd", "UcUsUiUl", "aarch64_sve_uqadd", "aarch64_sve_uqadd">; 1394defm SVHADD_S : SInstZPZZ<"svhadd", "csli", "aarch64_sve_shadd", "aarch64_sve_shadd">; 1395defm SVHADD_U : SInstZPZZ<"svhadd", "UcUsUiUl", "aarch64_sve_uhadd", "aarch64_sve_uhadd">; 1396defm SVRHADD_S : SInstZPZZ<"svrhadd", "csli", "aarch64_sve_srhadd", "aarch64_sve_srhadd">; 1397defm SVRHADD_U : SInstZPZZ<"svrhadd", "UcUsUiUl", "aarch64_sve_urhadd", "aarch64_sve_urhadd">; 1398 1399defm SVQSUB_S : SInstZPZZ<"svqsub", "csli", "aarch64_sve_sqsub", "aarch64_sve_sqsub_u">; 1400defm SVQSUB_U : SInstZPZZ<"svqsub", "UcUsUiUl", "aarch64_sve_uqsub", "aarch64_sve_uqsub_u">; 1401defm SVQSUBR_S : SInstZPZZ<"svqsubr", "csli", "aarch64_sve_sqsubr", "aarch64_sve_sqsub_u", [ReverseMergeAnyBinOp]>; 1402defm SVQSUBR_U : SInstZPZZ<"svqsubr", "UcUsUiUl", "aarch64_sve_uqsubr", "aarch64_sve_uqsub_u", [ReverseMergeAnyBinOp]>; 1403defm SVHSUB_S : SInstZPZZ<"svhsub", "csli", "aarch64_sve_shsub", "aarch64_sve_shsub">; 1404defm SVHSUB_U : SInstZPZZ<"svhsub", "UcUsUiUl", "aarch64_sve_uhsub", "aarch64_sve_uhsub">; 1405defm SVHSUBR_S : SInstZPZZ<"svhsubr", "csli", "aarch64_sve_shsubr", "aarch64_sve_shsubr">; 1406defm SVHSUBR_U : SInstZPZZ<"svhsubr", "UcUsUiUl", "aarch64_sve_uhsubr", "aarch64_sve_uhsubr">; 1407 1408defm SVQABS : SInstZPZ<"svqabs", "csil", "aarch64_sve_sqabs">; 1409defm SVQNEG : SInstZPZ<"svqneg", "csil", "aarch64_sve_sqneg">; 1410defm SVRECPE : SInstZPZ<"svrecpe", "Ui", "aarch64_sve_urecpe">; 1411defm SVRSQRTE : SInstZPZ<"svrsqrte", "Ui", "aarch64_sve_ursqrte">; 1412} 1413 1414//------------------------------------------------------------------------------ 1415 1416multiclass SInstZPZxZ<string name, string types, string pat_v, string pat_n, string intrinsic, list<FlagType> flags=[]> { 1417 def _M : SInst<name # "[_{d}]", pat_v, types, MergeOp1, intrinsic, flags>; 1418 def _X : SInst<name # "[_{d}]", pat_v, types, MergeAny, intrinsic, flags>; 1419 def _Z : SInst<name # "[_{d}]", pat_v, types, MergeZero, intrinsic, flags>; 1420 1421 def _N_M : SInst<name # "[_n_{d}]", pat_n, types, MergeOp1, intrinsic, flags>; 1422 def _N_X : SInst<name # "[_n_{d}]", pat_n, types, MergeAny, intrinsic, flags>; 1423 def _N_Z : SInst<name # "[_n_{d}]", pat_n, types, MergeZero, intrinsic, flags>; 1424} 1425 1426let SVETargetGuard = "sve2", SMETargetGuard = "sme" in { 1427defm SVQRSHL_S : SInstZPZxZ<"svqrshl", "csil", "dPdx", "dPdK", "aarch64_sve_sqrshl", [VerifyRuntimeMode]>; 1428defm SVQRSHL_U : SInstZPZxZ<"svqrshl", "UcUsUiUl", "dPdx", "dPdK", "aarch64_sve_uqrshl", [VerifyRuntimeMode]>; 1429defm SVQSHL_S : SInstZPZxZ<"svqshl", "csil", "dPdx", "dPdK", "aarch64_sve_sqshl", [VerifyRuntimeMode]>; 1430defm SVQSHL_U : SInstZPZxZ<"svqshl", "UcUsUiUl", "dPdx", "dPdK", "aarch64_sve_uqshl", [VerifyRuntimeMode]>; 1431defm SVRSHL_S : SInstZPZxZ<"svrshl", "csil", "dPdx", "dPdK", "aarch64_sve_srshl", [VerifyRuntimeMode]>; 1432defm SVRSHL_U : SInstZPZxZ<"svrshl", "UcUsUiUl", "dPdx", "dPdK", "aarch64_sve_urshl", [VerifyRuntimeMode]>; 1433defm SVSQADD : SInstZPZxZ<"svsqadd", "UcUsUiUl", "dPdx", "dPdK", "aarch64_sve_usqadd", [VerifyRuntimeMode]>; 1434defm SVUQADD : SInstZPZxZ<"svuqadd", "csil", "dPdu", "dPdL", "aarch64_sve_suqadd", [VerifyRuntimeMode]>; 1435 1436def SVABA_S : SInst<"svaba[_{d}]", "dddd", "csil" , MergeNone, "aarch64_sve_saba", [VerifyRuntimeMode]>; 1437def SVABA_U : SInst<"svaba[_{d}]", "dddd", "UcUsUiUl", MergeNone, "aarch64_sve_uaba", [VerifyRuntimeMode]>; 1438def SVQDMULH : SInst<"svqdmulh[_{d}]", "ddd", "csil", MergeNone, "aarch64_sve_sqdmulh", [VerifyRuntimeMode]>; 1439def SVQRDMULH : SInst<"svqrdmulh[_{d}]", "ddd", "csil", MergeNone, "aarch64_sve_sqrdmulh", [VerifyRuntimeMode]>; 1440def SVQRDMLAH : SInst<"svqrdmlah[_{d}]", "dddd", "csil", MergeNone, "aarch64_sve_sqrdmlah", [VerifyRuntimeMode]>; 1441def SVQRDMLSH : SInst<"svqrdmlsh[_{d}]", "dddd", "csil", MergeNone, "aarch64_sve_sqrdmlsh", [VerifyRuntimeMode]>; 1442 1443def SVABA_S_N : SInst<"svaba[_n_{d}]", "ddda", "csil", MergeNone, "aarch64_sve_saba", [VerifyRuntimeMode]>; 1444def SVABA_U_N : SInst<"svaba[_n_{d}]", "ddda", "UcUsUiUl", MergeNone, "aarch64_sve_uaba", [VerifyRuntimeMode]>; 1445def SVQDMULH_N : SInst<"svqdmulh[_n_{d}]", "dda", "csil", MergeNone, "aarch64_sve_sqdmulh", [VerifyRuntimeMode]>; 1446def SVQRDMULH_N : SInst<"svqrdmulh[_n_{d}]", "dda", "csil", MergeNone, "aarch64_sve_sqrdmulh", [VerifyRuntimeMode]>; 1447def SVQRDMLAH_N : SInst<"svqrdmlah[_n_{d}]", "ddda", "csil", MergeNone, "aarch64_sve_sqrdmlah", [VerifyRuntimeMode]>; 1448def SVQRDMLSH_N : SInst<"svqrdmlsh[_n_{d}]", "ddda", "csil", MergeNone, "aarch64_sve_sqrdmlsh", [VerifyRuntimeMode]>; 1449 1450def SVQDMULH_LANE : SInst<"svqdmulh_lane[_{d}]", "dddi", "sil", MergeNone, "aarch64_sve_sqdmulh_lane", [VerifyRuntimeMode], [ImmCheck<2, ImmCheckLaneIndex, 1>]>; 1451def SVQRDMULH_LANE : SInst<"svqrdmulh_lane[_{d}]", "dddi", "sil", MergeNone, "aarch64_sve_sqrdmulh_lane", [VerifyRuntimeMode], [ImmCheck<2, ImmCheckLaneIndex, 1>]>; 1452def SVQRDMLAH_LANE : SInst<"svqrdmlah_lane[_{d}]", "ddddi", "sil", MergeNone, "aarch64_sve_sqrdmlah_lane", [VerifyRuntimeMode], [ImmCheck<3, ImmCheckLaneIndex, 2>]>; 1453def SVQRDMLSH_LANE : SInst<"svqrdmlsh_lane[_{d}]", "ddddi", "sil", MergeNone, "aarch64_sve_sqrdmlsh_lane", [VerifyRuntimeMode], [ImmCheck<3, ImmCheckLaneIndex, 2>]>; 1454 1455def SVQSHLU_M : SInst<"svqshlu[_n_{d}]", "uPdi", "csil", MergeOp1, "aarch64_sve_sqshlu", [VerifyRuntimeMode], [ImmCheck<2, ImmCheckShiftLeft, 1>]>; 1456def SVQSHLU_X : SInst<"svqshlu[_n_{d}]", "uPdi", "csil", MergeAny, "aarch64_sve_sqshlu", [VerifyRuntimeMode], [ImmCheck<2, ImmCheckShiftLeft, 1>]>; 1457def SVQSHLU_Z : SInst<"svqshlu[_n_{d}]", "uPdi", "csil", MergeZero, "aarch64_sve_sqshlu", [VerifyRuntimeMode], [ImmCheck<2, ImmCheckShiftLeft, 1>]>; 1458def SVRSHR_M_S : SInst<"svrshr[_n_{d}]", "dPdi", "csil", MergeOp1, "aarch64_sve_srshr", [VerifyRuntimeMode], [ImmCheck<2, ImmCheckShiftRight, 1>]>; 1459def SVRSHR_M_U : SInst<"svrshr[_n_{d}]", "dPdi", "UcUsUiUl", MergeOp1, "aarch64_sve_urshr", [VerifyRuntimeMode], [ImmCheck<2, ImmCheckShiftRight, 1>]>; 1460def SVRSHR_X_S : SInst<"svrshr[_n_{d}]", "dPdi", "csil", MergeAny, "aarch64_sve_srshr", [VerifyRuntimeMode], [ImmCheck<2, ImmCheckShiftRight, 1>]>; 1461def SVRSHR_X_U : SInst<"svrshr[_n_{d}]", "dPdi", "UcUsUiUl", MergeAny, "aarch64_sve_urshr", [VerifyRuntimeMode], [ImmCheck<2, ImmCheckShiftRight, 1>]>; 1462def SVRSHR_Z_S : SInst<"svrshr[_n_{d}]", "dPdi", "csil", MergeZero, "aarch64_sve_srshr", [VerifyRuntimeMode], [ImmCheck<2, ImmCheckShiftRight, 1>]>; 1463def SVRSHR_Z_U : SInst<"svrshr[_n_{d}]", "dPdi", "UcUsUiUl", MergeZero, "aarch64_sve_urshr", [VerifyRuntimeMode], [ImmCheck<2, ImmCheckShiftRight, 1>]>; 1464def SVRSRA_S : SInst<"svrsra[_n_{d}]", "dddi", "csil", MergeNone, "aarch64_sve_srsra", [VerifyRuntimeMode], [ImmCheck<2, ImmCheckShiftRight, 1>]>; 1465def SVRSRA_U : SInst<"svrsra[_n_{d}]", "dddi", "UcUsUiUl", MergeNone, "aarch64_sve_ursra", [VerifyRuntimeMode], [ImmCheck<2, ImmCheckShiftRight, 1>]>; 1466def SVSLI : SInst<"svsli[_n_{d}]", "dddi", "csilUcUsUiUl", MergeNone, "aarch64_sve_sli", [VerifyRuntimeMode], [ImmCheck<2, ImmCheckShiftLeft, 1>]>; 1467def SVSRA_S : SInst<"svsra[_n_{d}]", "dddi", "csil", MergeNone, "aarch64_sve_ssra", [VerifyRuntimeMode], [ImmCheck<2, ImmCheckShiftRight, 1>]>; 1468def SVSRA_U : SInst<"svsra[_n_{d}]", "dddi", "UcUsUiUl", MergeNone, "aarch64_sve_usra", [VerifyRuntimeMode], [ImmCheck<2, ImmCheckShiftRight, 1>]>; 1469def SVSRI : SInst<"svsri[_n_{d}]", "dddi", "csilUcUsUiUl", MergeNone, "aarch64_sve_sri", [VerifyRuntimeMode], [ImmCheck<2, ImmCheckShiftRight, 1>]>; 1470} 1471 1472//////////////////////////////////////////////////////////////////////////////// 1473// SVE2 - Non-widening pairwise arithmetic 1474 1475multiclass SInstPairwise<string name, string types, string intrinsic, list<FlagType> flags=[]> { 1476 def _M : SInst<name # "[_{d}]", "dPdd", types, MergeOp1, intrinsic, flags>; 1477 def _X : SInst<name # "[_{d}]", "dPdd", types, MergeAny, intrinsic, flags>; 1478} 1479 1480let SVETargetGuard = "sve2", SMETargetGuard = "sme" in { 1481defm SVADDP : SInstPairwise<"svaddp", "csliUcUsUiUl", "aarch64_sve_addp", [VerifyRuntimeMode]>; 1482defm SVADDP_F : SInstPairwise<"svaddp", "hfd", "aarch64_sve_faddp", [VerifyRuntimeMode]>; 1483defm SVMAXNMP : SInstPairwise<"svmaxnmp", "hfd", "aarch64_sve_fmaxnmp", [VerifyRuntimeMode]>; 1484defm SVMAXP_F : SInstPairwise<"svmaxp", "hfd", "aarch64_sve_fmaxp", [VerifyRuntimeMode]>; 1485defm SVMAXP_S : SInstPairwise<"svmaxp", "csli", "aarch64_sve_smaxp", [VerifyRuntimeMode]>; 1486defm SVMAXP_U : SInstPairwise<"svmaxp", "UcUsUiUl", "aarch64_sve_umaxp", [VerifyRuntimeMode]>; 1487defm SVMINNMP : SInstPairwise<"svminnmp", "hfd", "aarch64_sve_fminnmp", [VerifyRuntimeMode]>; 1488defm SVMINP_F : SInstPairwise<"svminp", "hfd", "aarch64_sve_fminp", [VerifyRuntimeMode]>; 1489defm SVMINP_S : SInstPairwise<"svminp", "csli", "aarch64_sve_sminp", [VerifyRuntimeMode]>; 1490defm SVMINP_U : SInstPairwise<"svminp", "UcUsUiUl", "aarch64_sve_uminp", [VerifyRuntimeMode]>; 1491} 1492 1493//////////////////////////////////////////////////////////////////////////////// 1494// SVE2 - Widening pairwise arithmetic 1495 1496let SVETargetGuard = "sve2", SMETargetGuard = "sme" in { 1497def SVADALP_S_M : SInst<"svadalp[_{d}]", "dPdh", "sil", MergeOp1, "aarch64_sve_sadalp", [VerifyRuntimeMode]>; 1498def SVADALP_S_X : SInst<"svadalp[_{d}]", "dPdh", "sil", MergeAny, "aarch64_sve_sadalp", [VerifyRuntimeMode]>; 1499def SVADALP_S_Z : SInst<"svadalp[_{d}]", "dPdh", "sil", MergeZero, "aarch64_sve_sadalp", [VerifyRuntimeMode]>; 1500 1501def SVADALP_U_M : SInst<"svadalp[_{d}]", "dPdh", "UsUiUl", MergeOp1, "aarch64_sve_uadalp", [VerifyRuntimeMode]>; 1502def SVADALP_U_X : SInst<"svadalp[_{d}]", "dPdh", "UsUiUl", MergeAny, "aarch64_sve_uadalp", [VerifyRuntimeMode]>; 1503def SVADALP_U_Z : SInst<"svadalp[_{d}]", "dPdh", "UsUiUl", MergeZero, "aarch64_sve_uadalp", [VerifyRuntimeMode]>; 1504} 1505 1506//////////////////////////////////////////////////////////////////////////////// 1507// SVE2 - Bitwise ternary logical instructions 1508// 1509 1510let SVETargetGuard = "sve2", SMETargetGuard = "sme" in { 1511def SVBCAX : SInst<"svbcax[_{d}]", "dddd", "csilUcUsUiUl", MergeNone, "aarch64_sve_bcax", [VerifyRuntimeMode]>; 1512def SVBSL : SInst<"svbsl[_{d}]", "dddd", "csilUcUsUiUl", MergeNone, "aarch64_sve_bsl", [VerifyRuntimeMode]>; 1513def SVBSL1N : SInst<"svbsl1n[_{d}]", "dddd", "csilUcUsUiUl", MergeNone, "aarch64_sve_bsl1n", [VerifyRuntimeMode]>; 1514def SVBSL2N : SInst<"svbsl2n[_{d}]", "dddd", "csilUcUsUiUl", MergeNone, "aarch64_sve_bsl2n", [VerifyRuntimeMode]>; 1515def SVEOR3 : SInst<"sveor3[_{d}]", "dddd", "csilUcUsUiUl", MergeNone, "aarch64_sve_eor3", [VerifyRuntimeMode]>; 1516def SVNBSL : SInst<"svnbsl[_{d}]", "dddd", "csilUcUsUiUl", MergeNone, "aarch64_sve_nbsl", [VerifyRuntimeMode]>; 1517 1518def SVBCAX_N : SInst<"svbcax[_n_{d}]", "ddda", "csilUcUsUiUl", MergeNone, "aarch64_sve_bcax", [VerifyRuntimeMode]>; 1519def SVBSL_N : SInst<"svbsl[_n_{d}]", "ddda", "csilUcUsUiUl", MergeNone, "aarch64_sve_bsl", [VerifyRuntimeMode]>; 1520def SVBSL1N_N : SInst<"svbsl1n[_n_{d}]", "ddda", "csilUcUsUiUl", MergeNone, "aarch64_sve_bsl1n", [VerifyRuntimeMode]>; 1521def SVBSL2N_N : SInst<"svbsl2n[_n_{d}]", "ddda", "csilUcUsUiUl", MergeNone, "aarch64_sve_bsl2n", [VerifyRuntimeMode]>; 1522def SVEOR3_N : SInst<"sveor3[_n_{d}]", "ddda", "csilUcUsUiUl", MergeNone, "aarch64_sve_eor3", [VerifyRuntimeMode]>; 1523def SVNBSL_N : SInst<"svnbsl[_n_{d}]", "ddda", "csilUcUsUiUl", MergeNone, "aarch64_sve_nbsl", [VerifyRuntimeMode]>; 1524def SVXAR_N : SInst<"svxar[_n_{d}]", "dddi", "csilUcUsUiUl", MergeNone, "aarch64_sve_xar", [VerifyRuntimeMode], [ImmCheck<2, ImmCheckShiftRight, 1>]>; 1525} 1526 1527//////////////////////////////////////////////////////////////////////////////// 1528// SVE2 - Large integer arithmetic 1529 1530let SVETargetGuard = "sve2", SMETargetGuard = "sme" in { 1531def SVADCLB : SInst<"svadclb[_{d}]", "dddd", "UiUl", MergeNone, "aarch64_sve_adclb", [VerifyRuntimeMode]>; 1532def SVADCLT : SInst<"svadclt[_{d}]", "dddd", "UiUl", MergeNone, "aarch64_sve_adclt", [VerifyRuntimeMode]>; 1533def SVSBCLB : SInst<"svsbclb[_{d}]", "dddd", "UiUl", MergeNone, "aarch64_sve_sbclb", [VerifyRuntimeMode]>; 1534def SVSBCLT : SInst<"svsbclt[_{d}]", "dddd", "UiUl", MergeNone, "aarch64_sve_sbclt", [VerifyRuntimeMode]>; 1535 1536def SVADCLB_N : SInst<"svadclb[_n_{d}]", "ddda", "UiUl", MergeNone, "aarch64_sve_adclb", [VerifyRuntimeMode]>; 1537def SVADCLT_N : SInst<"svadclt[_n_{d}]", "ddda", "UiUl", MergeNone, "aarch64_sve_adclt", [VerifyRuntimeMode]>; 1538def SVSBCLB_N : SInst<"svsbclb[_n_{d}]", "ddda", "UiUl", MergeNone, "aarch64_sve_sbclb", [VerifyRuntimeMode]>; 1539def SVSBCLT_N : SInst<"svsbclt[_n_{d}]", "ddda", "UiUl", MergeNone, "aarch64_sve_sbclt", [VerifyRuntimeMode]>; 1540} 1541 1542//////////////////////////////////////////////////////////////////////////////// 1543// SVE2 - Multiplication by indexed elements 1544 1545let SVETargetGuard = "sve2", SMETargetGuard = "sme" in { 1546def SVMLA_LANE_2 : SInst<"svmla_lane[_{d}]", "ddddi", "silUsUiUl", MergeNone, "aarch64_sve_mla_lane", [VerifyRuntimeMode], [ImmCheck<3, ImmCheckLaneIndex, 2>]>; 1547def SVMLS_LANE_2 : SInst<"svmls_lane[_{d}]", "ddddi", "silUsUiUl", MergeNone, "aarch64_sve_mls_lane", [VerifyRuntimeMode], [ImmCheck<3, ImmCheckLaneIndex, 2>]>; 1548def SVMUL_LANE_2 : SInst<"svmul_lane[_{d}]", "dddi", "silUsUiUl", MergeNone, "aarch64_sve_mul_lane", [VerifyRuntimeMode], [ImmCheck<2, ImmCheckLaneIndex, 1>]>; 1549} 1550 1551//////////////////////////////////////////////////////////////////////////////// 1552// SVE2 - Uniform complex integer arithmetic 1553let SVETargetGuard = "sve2", SMETargetGuard = "sme" in { 1554def SVCADD : SInst<"svcadd[_{d}]", "dddi", "csilUcUsUiUl", MergeNone, "aarch64_sve_cadd_x", [VerifyRuntimeMode], [ImmCheck<2, ImmCheckComplexRot90_270>]>; 1555def SVSQCADD : SInst<"svqcadd[_{d}]", "dddi", "csil", MergeNone, "aarch64_sve_sqcadd_x", [VerifyRuntimeMode], [ImmCheck<2, ImmCheckComplexRot90_270>]>; 1556def SVCMLA : SInst<"svcmla[_{d}]", "ddddi", "csilUcUsUiUl", MergeNone, "aarch64_sve_cmla_x", [VerifyRuntimeMode], [ImmCheck<3, ImmCheckComplexRotAll90>]>; 1557def SVCMLA_LANE_X : SInst<"svcmla_lane[_{d}]", "ddddii", "siUsUi", MergeNone, "aarch64_sve_cmla_lane_x", [VerifyRuntimeMode], [ImmCheck<3, ImmCheckLaneIndexCompRotate, 2>, 1558 ImmCheck<4, ImmCheckComplexRotAll90>]>; 1559def SVSQRDCMLAH_X : SInst<"svqrdcmlah[_{d}]", "ddddi", "csil", MergeNone, "aarch64_sve_sqrdcmlah_x", [VerifyRuntimeMode], [ImmCheck<3, ImmCheckComplexRotAll90>]>; 1560def SVSQRDCMLAH_LANE_X : SInst<"svqrdcmlah_lane[_{d}]", "ddddii", "si", MergeNone, "aarch64_sve_sqrdcmlah_lane_x", [VerifyRuntimeMode], [ImmCheck<3, ImmCheckLaneIndexCompRotate, 2>, 1561 ImmCheck<4, ImmCheckComplexRotAll90>]>; 1562} 1563 1564//////////////////////////////////////////////////////////////////////////////// 1565// SVE2 - Widening DSP operations 1566 1567multiclass SInstWideDSPAcc<string name, string types, string intrinsic> { 1568 def : SInst<name # "[_{d}]", "ddhh", types, MergeNone, intrinsic, [VerifyRuntimeMode]>; 1569 def _N : SInst<name # "[_n_{d}]", "ddhR", types, MergeNone, intrinsic, [VerifyRuntimeMode]>; 1570} 1571 1572multiclass SInstWideDSPLong<string name, string types, string intrinsic> { 1573 def : SInst<name # "[_{d}]", "dhh", types, MergeNone, intrinsic, [VerifyRuntimeMode]>; 1574 def _N : SInst<name # "[_n_{d}]", "dhR", types, MergeNone, intrinsic, [VerifyRuntimeMode]>; 1575} 1576 1577multiclass SInstWideDSPWide<string name, string types, string intrinsic> { 1578 def : SInst<name # "[_{d}]", "ddh", types, MergeNone, intrinsic, [VerifyRuntimeMode]>; 1579 def _N : SInst<name # "[_n_{d}]", "ddR", types, MergeNone, intrinsic, [VerifyRuntimeMode]>; 1580} 1581 1582let SVETargetGuard = "sve2", SMETargetGuard = "sme" in { 1583defm SVABALB_S : SInstWideDSPAcc<"svabalb", "sil", "aarch64_sve_sabalb">; 1584defm SVABALB_U : SInstWideDSPAcc<"svabalb", "UsUiUl", "aarch64_sve_uabalb">; 1585defm SVABALT_S : SInstWideDSPAcc<"svabalt", "sil", "aarch64_sve_sabalt">; 1586defm SVABALT_U : SInstWideDSPAcc<"svabalt", "UsUiUl", "aarch64_sve_uabalt">; 1587defm SVMLALB_S : SInstWideDSPAcc<"svmlalb", "sil", "aarch64_sve_smlalb">; 1588defm SVMLALB_U : SInstWideDSPAcc<"svmlalb", "UsUiUl", "aarch64_sve_umlalb">; 1589defm SVMLALT_S : SInstWideDSPAcc<"svmlalt", "sil", "aarch64_sve_smlalt">; 1590defm SVMLALT_U : SInstWideDSPAcc<"svmlalt", "UsUiUl", "aarch64_sve_umlalt">; 1591defm SVMLSLB_S : SInstWideDSPAcc<"svmlslb", "sil", "aarch64_sve_smlslb">; 1592defm SVMLSLB_U : SInstWideDSPAcc<"svmlslb", "UsUiUl", "aarch64_sve_umlslb">; 1593defm SVMLSLT_S : SInstWideDSPAcc<"svmlslt", "sil", "aarch64_sve_smlslt">; 1594defm SVMLSLT_U : SInstWideDSPAcc<"svmlslt", "UsUiUl", "aarch64_sve_umlslt">; 1595defm SVQDMLALB : SInstWideDSPAcc<"svqdmlalb", "sil", "aarch64_sve_sqdmlalb">; 1596defm SVQDMLALT : SInstWideDSPAcc<"svqdmlalt", "sil", "aarch64_sve_sqdmlalt">; 1597defm SVQDMLSLB : SInstWideDSPAcc<"svqdmlslb", "sil", "aarch64_sve_sqdmlslb">; 1598defm SVQDMLSLT : SInstWideDSPAcc<"svqdmlslt", "sil", "aarch64_sve_sqdmlslt">; 1599 1600defm SVABDLB_S : SInstWideDSPLong<"svabdlb", "sil", "aarch64_sve_sabdlb">; 1601defm SVABDLB_U : SInstWideDSPLong<"svabdlb", "UsUiUl", "aarch64_sve_uabdlb">; 1602defm SVABDLT_S : SInstWideDSPLong<"svabdlt", "sil", "aarch64_sve_sabdlt">; 1603defm SVABDLT_U : SInstWideDSPLong<"svabdlt", "UsUiUl", "aarch64_sve_uabdlt">; 1604defm SVADDLB_S : SInstWideDSPLong<"svaddlb", "sil", "aarch64_sve_saddlb">; 1605defm SVADDLB_U : SInstWideDSPLong<"svaddlb", "UsUiUl", "aarch64_sve_uaddlb">; 1606defm SVADDLT_S : SInstWideDSPLong<"svaddlt", "sil", "aarch64_sve_saddlt">; 1607defm SVADDLT_U : SInstWideDSPLong<"svaddlt", "UsUiUl", "aarch64_sve_uaddlt">; 1608defm SVMULLB_S : SInstWideDSPLong<"svmullb", "sil", "aarch64_sve_smullb">; 1609defm SVMULLB_U : SInstWideDSPLong<"svmullb", "UsUiUl", "aarch64_sve_umullb">; 1610defm SVMULLT_S : SInstWideDSPLong<"svmullt", "sil", "aarch64_sve_smullt">; 1611defm SVMULLT_U : SInstWideDSPLong<"svmullt", "UsUiUl", "aarch64_sve_umullt">; 1612defm SVQDMULLB : SInstWideDSPLong<"svqdmullb", "sil", "aarch64_sve_sqdmullb">; 1613defm SVQDMULLT : SInstWideDSPLong<"svqdmullt", "sil", "aarch64_sve_sqdmullt">; 1614defm SVSUBLB_S : SInstWideDSPLong<"svsublb", "sil", "aarch64_sve_ssublb">; 1615defm SVSUBLB_U : SInstWideDSPLong<"svsublb", "UsUiUl", "aarch64_sve_usublb">; 1616defm SVSUBLT_S : SInstWideDSPLong<"svsublt", "sil", "aarch64_sve_ssublt">; 1617defm SVSUBLT_U : SInstWideDSPLong<"svsublt", "UsUiUl", "aarch64_sve_usublt">; 1618 1619defm SVADDWB_S : SInstWideDSPWide<"svaddwb", "sil", "aarch64_sve_saddwb">; 1620defm SVADDWB_U : SInstWideDSPWide<"svaddwb", "UsUiUl", "aarch64_sve_uaddwb">; 1621defm SVADDWT_S : SInstWideDSPWide<"svaddwt", "sil", "aarch64_sve_saddwt">; 1622defm SVADDWT_U : SInstWideDSPWide<"svaddwt", "UsUiUl", "aarch64_sve_uaddwt">; 1623defm SVSUBWB_S : SInstWideDSPWide<"svsubwb", "sil", "aarch64_sve_ssubwb">; 1624defm SVSUBWB_U : SInstWideDSPWide<"svsubwb", "UsUiUl", "aarch64_sve_usubwb">; 1625defm SVSUBWT_S : SInstWideDSPWide<"svsubwt", "sil", "aarch64_sve_ssubwt">; 1626defm SVSUBWT_U : SInstWideDSPWide<"svsubwt", "UsUiUl", "aarch64_sve_usubwt">; 1627 1628def SVSHLLB_S_N : SInst<"svshllb[_n_{d}]", "dhi", "sil", MergeNone, "aarch64_sve_sshllb", [VerifyRuntimeMode], [ImmCheck<1, ImmCheckShiftLeft, 0>]>; 1629def SVSHLLB_U_N : SInst<"svshllb[_n_{d}]", "dhi", "UsUiUl", MergeNone, "aarch64_sve_ushllb", [VerifyRuntimeMode], [ImmCheck<1, ImmCheckShiftLeft, 0>]>; 1630def SVSHLLT_S_N : SInst<"svshllt[_n_{d}]", "dhi", "sil", MergeNone, "aarch64_sve_sshllt", [VerifyRuntimeMode], [ImmCheck<1, ImmCheckShiftLeft, 0>]>; 1631def SVSHLLT_U_N : SInst<"svshllt[_n_{d}]", "dhi", "UsUiUl", MergeNone, "aarch64_sve_ushllt", [VerifyRuntimeMode], [ImmCheck<1, ImmCheckShiftLeft, 0>]>; 1632 1633def SVMOVLB_S_N : SInst<"svmovlb[_{d}]", "dh", "sil", MergeNone, "", [VerifyRuntimeMode]>; 1634def SVMOVLB_U_N : SInst<"svmovlb[_{d}]", "dh", "UsUiUl", MergeNone, "", [VerifyRuntimeMode]>; 1635def SVMOVLT_S_N : SInst<"svmovlt[_{d}]", "dh", "sil", MergeNone, "", [VerifyRuntimeMode]>; 1636def SVMOVLT_U_N : SInst<"svmovlt[_{d}]", "dh", "UsUiUl", MergeNone, "", [VerifyRuntimeMode]>; 1637 1638def SVMLALB_S_LANE : SInst<"svmlalb_lane[_{d}]", "ddhhi", "il", MergeNone, "aarch64_sve_smlalb_lane", [VerifyRuntimeMode], [ImmCheck<3, ImmCheckLaneIndex, 2>]>; 1639def SVMLALB_U_LANE : SInst<"svmlalb_lane[_{d}]", "ddhhi", "UiUl", MergeNone, "aarch64_sve_umlalb_lane", [VerifyRuntimeMode], [ImmCheck<3, ImmCheckLaneIndex, 2>]>; 1640def SVMLALT_S_LANE : SInst<"svmlalt_lane[_{d}]", "ddhhi", "il", MergeNone, "aarch64_sve_smlalt_lane", [VerifyRuntimeMode], [ImmCheck<3, ImmCheckLaneIndex, 2>]>; 1641def SVMLALT_U_LANE : SInst<"svmlalt_lane[_{d}]", "ddhhi", "UiUl", MergeNone, "aarch64_sve_umlalt_lane", [VerifyRuntimeMode], [ImmCheck<3, ImmCheckLaneIndex, 2>]>; 1642def SVMLSLB_S_LANE : SInst<"svmlslb_lane[_{d}]", "ddhhi", "il", MergeNone, "aarch64_sve_smlslb_lane", [VerifyRuntimeMode], [ImmCheck<3, ImmCheckLaneIndex, 2>]>; 1643def SVMLSLB_U_LANE : SInst<"svmlslb_lane[_{d}]", "ddhhi", "UiUl", MergeNone, "aarch64_sve_umlslb_lane", [VerifyRuntimeMode], [ImmCheck<3, ImmCheckLaneIndex, 2>]>; 1644def SVMLSLT_S_LANE : SInst<"svmlslt_lane[_{d}]", "ddhhi", "il", MergeNone, "aarch64_sve_smlslt_lane", [VerifyRuntimeMode], [ImmCheck<3, ImmCheckLaneIndex, 2>]>; 1645def SVMLSLT_U_LANE : SInst<"svmlslt_lane[_{d}]", "ddhhi", "UiUl", MergeNone, "aarch64_sve_umlslt_lane", [VerifyRuntimeMode], [ImmCheck<3, ImmCheckLaneIndex, 2>]>; 1646def SVMULLB_S_LANE : SInst<"svmullb_lane[_{d}]", "dhhi", "il", MergeNone, "aarch64_sve_smullb_lane", [VerifyRuntimeMode], [ImmCheck<2, ImmCheckLaneIndex, 1>]>; 1647def SVMULLB_U_LANE : SInst<"svmullb_lane[_{d}]", "dhhi", "UiUl", MergeNone, "aarch64_sve_umullb_lane", [VerifyRuntimeMode], [ImmCheck<2, ImmCheckLaneIndex, 1>]>; 1648def SVMULLT_S_LANE : SInst<"svmullt_lane[_{d}]", "dhhi", "il", MergeNone, "aarch64_sve_smullt_lane", [VerifyRuntimeMode], [ImmCheck<2, ImmCheckLaneIndex, 1>]>; 1649def SVMULLT_U_LANE : SInst<"svmullt_lane[_{d}]", "dhhi", "UiUl", MergeNone, "aarch64_sve_umullt_lane", [VerifyRuntimeMode], [ImmCheck<2, ImmCheckLaneIndex, 1>]>; 1650def SVQDMLALB_LANE : SInst<"svqdmlalb_lane[_{d}]", "ddhhi", "il", MergeNone, "aarch64_sve_sqdmlalb_lane", [VerifyRuntimeMode], [ImmCheck<3, ImmCheckLaneIndex, 2>]>; 1651def SVQDMLALT_LANE : SInst<"svqdmlalt_lane[_{d}]", "ddhhi", "il", MergeNone, "aarch64_sve_sqdmlalt_lane", [VerifyRuntimeMode], [ImmCheck<3, ImmCheckLaneIndex, 2>]>; 1652def SVQDMLSLB_LANE : SInst<"svqdmlslb_lane[_{d}]", "ddhhi", "il", MergeNone, "aarch64_sve_sqdmlslb_lane", [VerifyRuntimeMode], [ImmCheck<3, ImmCheckLaneIndex, 2>]>; 1653def SVQDMLSLT_LANE : SInst<"svqdmlslt_lane[_{d}]", "ddhhi", "il", MergeNone, "aarch64_sve_sqdmlslt_lane", [VerifyRuntimeMode], [ImmCheck<3, ImmCheckLaneIndex, 2>]>; 1654def SVQDMULLB_LANE : SInst<"svqdmullb_lane[_{d}]", "dhhi", "il", MergeNone, "aarch64_sve_sqdmullb_lane", [VerifyRuntimeMode], [ImmCheck<2, ImmCheckLaneIndex, 1>]>; 1655def SVQDMULLT_LANE : SInst<"svqdmullt_lane[_{d}]", "dhhi", "il", MergeNone, "aarch64_sve_sqdmullt_lane", [VerifyRuntimeMode], [ImmCheck<2, ImmCheckLaneIndex, 1>]>; 1656} 1657 1658//////////////////////////////////////////////////////////////////////////////// 1659// SVE2 - Narrowing DSP operations 1660 1661let SVETargetGuard = "sve2", SMETargetGuard = "sme" in { 1662def SVADDHNB : SInst<"svaddhnb[_{d}]", "hdd", "silUsUiUl", MergeNone, "aarch64_sve_addhnb", [VerifyRuntimeMode]>; 1663def SVADDHNT : SInst<"svaddhnt[_{d}]", "hhdd", "silUsUiUl", MergeNone, "aarch64_sve_addhnt", [VerifyRuntimeMode]>; 1664def SVRADDHNB : SInst<"svraddhnb[_{d}]", "hdd", "silUsUiUl", MergeNone, "aarch64_sve_raddhnb", [VerifyRuntimeMode]>; 1665def SVRADDHNT : SInst<"svraddhnt[_{d}]", "hhdd", "silUsUiUl", MergeNone, "aarch64_sve_raddhnt", [VerifyRuntimeMode]>; 1666def SVRSUBHNB : SInst<"svrsubhnb[_{d}]", "hdd", "silUsUiUl", MergeNone, "aarch64_sve_rsubhnb", [VerifyRuntimeMode]>; 1667def SVRSUBHNT : SInst<"svrsubhnt[_{d}]", "hhdd", "silUsUiUl", MergeNone, "aarch64_sve_rsubhnt", [VerifyRuntimeMode]>; 1668def SVSUBHNB : SInst<"svsubhnb[_{d}]", "hdd", "silUsUiUl", MergeNone, "aarch64_sve_subhnb", [VerifyRuntimeMode]>; 1669def SVSUBHNT : SInst<"svsubhnt[_{d}]", "hhdd", "silUsUiUl", MergeNone, "aarch64_sve_subhnt", [VerifyRuntimeMode]>; 1670 1671def SVADDHNB_N : SInst<"svaddhnb[_n_{d}]", "hda", "silUsUiUl", MergeNone, "aarch64_sve_addhnb", [VerifyRuntimeMode]>; 1672def SVADDHNT_N : SInst<"svaddhnt[_n_{d}]", "hhda", "silUsUiUl", MergeNone, "aarch64_sve_addhnt", [VerifyRuntimeMode]>; 1673def SVRADDHNB_N : SInst<"svraddhnb[_n_{d}]", "hda", "silUsUiUl", MergeNone, "aarch64_sve_raddhnb", [VerifyRuntimeMode]>; 1674def SVRADDHNT_N : SInst<"svraddhnt[_n_{d}]", "hhda", "silUsUiUl", MergeNone, "aarch64_sve_raddhnt", [VerifyRuntimeMode]>; 1675def SVRSUBHNB_N : SInst<"svrsubhnb[_n_{d}]", "hda", "silUsUiUl", MergeNone, "aarch64_sve_rsubhnb", [VerifyRuntimeMode]>; 1676def SVRSUBHNT_N : SInst<"svrsubhnt[_n_{d}]", "hhda", "silUsUiUl", MergeNone, "aarch64_sve_rsubhnt", [VerifyRuntimeMode]>; 1677def SVSUBHNB_N : SInst<"svsubhnb[_n_{d}]", "hda", "silUsUiUl", MergeNone, "aarch64_sve_subhnb", [VerifyRuntimeMode]>; 1678def SVSUBHNT_N : SInst<"svsubhnt[_n_{d}]", "hhda", "silUsUiUl", MergeNone, "aarch64_sve_subhnt", [VerifyRuntimeMode]>; 1679 1680def SVSHRNB : SInst<"svshrnb[_n_{d}]", "hdi", "silUsUiUl", MergeNone, "aarch64_sve_shrnb", [VerifyRuntimeMode], [ImmCheck<1, ImmCheckShiftRightNarrow, 0>]>; 1681def SVRSHRNB : SInst<"svrshrnb[_n_{d}]", "hdi", "silUsUiUl", MergeNone, "aarch64_sve_rshrnb", [VerifyRuntimeMode], [ImmCheck<1, ImmCheckShiftRightNarrow, 0>]>; 1682def SVQSHRUNB : SInst<"svqshrunb[_n_{d}]", "edi", "sil", MergeNone, "aarch64_sve_sqshrunb", [VerifyRuntimeMode], [ImmCheck<1, ImmCheckShiftRightNarrow, 0>]>; 1683def SVQRSHRUNB : SInst<"svqrshrunb[_n_{d}]", "edi", "sil", MergeNone, "aarch64_sve_sqrshrunb", [VerifyRuntimeMode], [ImmCheck<1, ImmCheckShiftRightNarrow, 0>]>; 1684def SVQSHRNB_S : SInst<"svqshrnb[_n_{d}]", "hdi", "sil", MergeNone, "aarch64_sve_sqshrnb", [VerifyRuntimeMode], [ImmCheck<1, ImmCheckShiftRightNarrow, 0>]>; 1685def SVQSHRNB_U : SInst<"svqshrnb[_n_{d}]", "hdi", "UsUiUl", MergeNone, "aarch64_sve_uqshrnb", [VerifyRuntimeMode], [ImmCheck<1, ImmCheckShiftRightNarrow, 0>]>; 1686def SVQRSHRNB_S : SInst<"svqrshrnb[_n_{d}]", "hdi", "sil", MergeNone, "aarch64_sve_sqrshrnb", [VerifyRuntimeMode], [ImmCheck<1, ImmCheckShiftRightNarrow, 0>]>; 1687def SVQRSHRNB_U : SInst<"svqrshrnb[_n_{d}]", "hdi", "UsUiUl", MergeNone, "aarch64_sve_uqrshrnb", [VerifyRuntimeMode], [ImmCheck<1, ImmCheckShiftRightNarrow, 0>]>; 1688 1689def SVSHRNT : SInst<"svshrnt[_n_{d}]", "hhdi", "silUsUiUl", MergeNone, "aarch64_sve_shrnt", [VerifyRuntimeMode], [ImmCheck<2, ImmCheckShiftRightNarrow, 1>]>; 1690def SVRSHRNT : SInst<"svrshrnt[_n_{d}]", "hhdi", "silUsUiUl", MergeNone, "aarch64_sve_rshrnt", [VerifyRuntimeMode], [ImmCheck<2, ImmCheckShiftRightNarrow, 1>]>; 1691def SVQSHRUNT : SInst<"svqshrunt[_n_{d}]", "eedi", "sil", MergeNone, "aarch64_sve_sqshrunt", [VerifyRuntimeMode], [ImmCheck<2, ImmCheckShiftRightNarrow, 1>]>; 1692def SVQRSHRUNT : SInst<"svqrshrunt[_n_{d}]", "eedi", "sil", MergeNone, "aarch64_sve_sqrshrunt", [VerifyRuntimeMode], [ImmCheck<2, ImmCheckShiftRightNarrow, 1>]>; 1693def SVQSHRNT_S : SInst<"svqshrnt[_n_{d}]", "hhdi", "sil", MergeNone, "aarch64_sve_sqshrnt", [VerifyRuntimeMode], [ImmCheck<2, ImmCheckShiftRightNarrow, 1>]>; 1694def SVQSHRNT_U : SInst<"svqshrnt[_n_{d}]", "hhdi", "UsUiUl", MergeNone, "aarch64_sve_uqshrnt", [VerifyRuntimeMode], [ImmCheck<2, ImmCheckShiftRightNarrow, 1>]>; 1695def SVQRSHRNT_S : SInst<"svqrshrnt[_n_{d}]", "hhdi", "sil", MergeNone, "aarch64_sve_sqrshrnt", [VerifyRuntimeMode], [ImmCheck<2, ImmCheckShiftRightNarrow, 1>]>; 1696def SVQRSHRNT_U : SInst<"svqrshrnt[_n_{d}]", "hhdi", "UsUiUl", MergeNone, "aarch64_sve_uqrshrnt", [VerifyRuntimeMode], [ImmCheck<2, ImmCheckShiftRightNarrow, 1>]>; 1697} 1698//////////////////////////////////////////////////////////////////////////////// 1699// SVE2 - Unary narrowing operations 1700 1701let SVETargetGuard = "sve2", SMETargetGuard = "sme" in { 1702def SVQXTNB_S : SInst<"svqxtnb[_{d}]", "hd", "sil", MergeNone, "aarch64_sve_sqxtnb", [VerifyRuntimeMode]>; 1703def SVQXTNB_U : SInst<"svqxtnb[_{d}]", "hd", "UsUiUl", MergeNone, "aarch64_sve_uqxtnb", [VerifyRuntimeMode]>; 1704def SVQXTUNB_S : SInst<"svqxtunb[_{d}]", "ed", "sil", MergeNone, "aarch64_sve_sqxtunb", [VerifyRuntimeMode]>; 1705 1706def SVQXTNT_S : SInst<"svqxtnt[_{d}]", "hhd", "sil", MergeNone, "aarch64_sve_sqxtnt", [VerifyRuntimeMode]>; 1707def SVQXTNT_U : SInst<"svqxtnt[_{d}]", "hhd", "UsUiUl", MergeNone, "aarch64_sve_uqxtnt", [VerifyRuntimeMode]>; 1708def SVQXTUNT_S : SInst<"svqxtunt[_{d}]", "eed", "sil", MergeNone, "aarch64_sve_sqxtunt", [VerifyRuntimeMode]>; 1709} 1710 1711//////////////////////////////////////////////////////////////////////////////// 1712// SVE2 - Widening complex integer arithmetic 1713 1714let SVETargetGuard = "sve2", SMETargetGuard = "sme" in { 1715defm SVADDLBT : SInstWideDSPLong<"svaddlbt", "sil", "aarch64_sve_saddlbt">; 1716defm SVSUBLBT : SInstWideDSPLong<"svsublbt", "sil", "aarch64_sve_ssublbt">; 1717defm SVSUBLTB : SInstWideDSPLong<"svsubltb", "sil", "aarch64_sve_ssubltb">; 1718 1719defm SVQDMLALBT : SInstWideDSPAcc<"svqdmlalbt", "sil", "aarch64_sve_sqdmlalbt">; 1720defm SVQDMLSLBT : SInstWideDSPAcc<"svqdmlslbt", "sil", "aarch64_sve_sqdmlslbt">; 1721} 1722 1723//////////////////////////////////////////////////////////////////////////////// 1724// SVE2 - Non-temporal gather/scatter 1725let SVETargetGuard = "sve2", SMETargetGuard = InvalidMode in { 1726// Non-temporal gather load one vector (vector base) 1727def SVLDNT1_GATHER_BASES_U : MInst<"svldnt1_gather[_{2}base]_{0}", "dPu", "ilUiUlfd", [IsGatherLoad], MemEltTyDefault, "aarch64_sve_ldnt1_gather_scalar_offset">; 1728def SVLDNT1SB_GATHER_BASES_U : MInst<"svldnt1sb_gather[_{2}base]_{0}", "dPu", "ilUiUl", [IsGatherLoad], MemEltTyInt8, "aarch64_sve_ldnt1_gather_scalar_offset">; 1729def SVLDNT1UB_GATHER_BASES_U : MInst<"svldnt1ub_gather[_{2}base]_{0}", "dPu", "ilUiUl", [IsGatherLoad, IsZExtReturn], MemEltTyInt8, "aarch64_sve_ldnt1_gather_scalar_offset">; 1730def SVLDNT1SH_GATHER_BASES_U : MInst<"svldnt1sh_gather[_{2}base]_{0}", "dPu", "ilUiUl", [IsGatherLoad], MemEltTyInt16, "aarch64_sve_ldnt1_gather_scalar_offset">; 1731def SVLDNT1UH_GATHER_BASES_U : MInst<"svldnt1uh_gather[_{2}base]_{0}", "dPu", "ilUiUl", [IsGatherLoad, IsZExtReturn], MemEltTyInt16, "aarch64_sve_ldnt1_gather_scalar_offset">; 1732def SVLDNT1SW_GATHER_BASES_U : MInst<"svldnt1sw_gather[_{2}base]_{0}", "dPu", "lUl", [IsGatherLoad], MemEltTyInt32, "aarch64_sve_ldnt1_gather_scalar_offset">; 1733def SVLDNT1UW_GATHER_BASES_U : MInst<"svldnt1uw_gather[_{2}base]_{0}", "dPu", "lUl", [IsGatherLoad, IsZExtReturn], MemEltTyInt32, "aarch64_sve_ldnt1_gather_scalar_offset">; 1734 1735// Non-temporal gather load one vector (scalar base, signed vector offset in bytes) 1736def SVLDNT1_GATHER_64B_OFFSETS_S : MInst<"svldnt1_gather_[{3}]offset[_{0}]", "dPcx", "lUld", [IsGatherLoad, IsByteIndexed], MemEltTyDefault, "aarch64_sve_ldnt1_gather">; 1737def SVLDNT1SB_GATHER_64B_OFFSETS_S : MInst<"svldnt1sb_gather_[{3}]offset_{0}", "dPSx", "lUl", [IsGatherLoad, IsByteIndexed], MemEltTyInt8, "aarch64_sve_ldnt1_gather">; 1738def SVLDNT1UB_GATHER_64B_OFFSETS_S : MInst<"svldnt1ub_gather_[{3}]offset_{0}", "dPWx", "lUl", [IsGatherLoad, IsByteIndexed, IsZExtReturn], MemEltTyInt8, "aarch64_sve_ldnt1_gather">; 1739def SVLDNT1SH_GATHER_64B_OFFSETS_S : MInst<"svldnt1sh_gather_[{3}]offset_{0}", "dPTx", "lUl", [IsGatherLoad, IsByteIndexed], MemEltTyInt16, "aarch64_sve_ldnt1_gather">; 1740def SVLDNT1UH_GATHER_64B_OFFSETS_S : MInst<"svldnt1uh_gather_[{3}]offset_{0}", "dPXx", "lUl", [IsGatherLoad, IsByteIndexed, IsZExtReturn], MemEltTyInt16, "aarch64_sve_ldnt1_gather">; 1741def SVLDNT1SW_GATHER_64B_OFFSETS_S : MInst<"svldnt1sw_gather_[{3}]offset_{0}", "dPUx", "lUl", [IsGatherLoad, IsByteIndexed], MemEltTyInt32, "aarch64_sve_ldnt1_gather">; 1742def SVLDNT1UW_GATHER_64B_OFFSETS_S : MInst<"svldnt1uw_gather_[{3}]offset_{0}", "dPYx", "lUl", [IsGatherLoad, IsByteIndexed, IsZExtReturn], MemEltTyInt32, "aarch64_sve_ldnt1_gather">; 1743 1744// Non-temporal gather load one vector (scalar base, unsigned vector offset in bytes) 1745def SVLDNT1_GATHER_64B_OFFSETS_U : MInst<"svldnt1_gather_[{3}]offset[_{0}]", "dPcu", "lUld", [IsGatherLoad, IsByteIndexed], MemEltTyDefault, "aarch64_sve_ldnt1_gather">; 1746def SVLDNT1SB_GATHER_64B_OFFSETS_U : MInst<"svldnt1sb_gather_[{3}]offset_{0}", "dPSu", "lUl", [IsGatherLoad, IsByteIndexed], MemEltTyInt8, "aarch64_sve_ldnt1_gather">; 1747def SVLDNT1UB_GATHER_64B_OFFSETS_U : MInst<"svldnt1ub_gather_[{3}]offset_{0}", "dPWu", "lUl", [IsGatherLoad, IsByteIndexed, IsZExtReturn], MemEltTyInt8, "aarch64_sve_ldnt1_gather">; 1748def SVLDNT1SH_GATHER_64B_OFFSETS_U : MInst<"svldnt1sh_gather_[{3}]offset_{0}", "dPTu", "lUl", [IsGatherLoad, IsByteIndexed], MemEltTyInt16, "aarch64_sve_ldnt1_gather">; 1749def SVLDNT1UH_GATHER_64B_OFFSETS_U : MInst<"svldnt1uh_gather_[{3}]offset_{0}", "dPXu", "lUl", [IsGatherLoad, IsByteIndexed, IsZExtReturn], MemEltTyInt16, "aarch64_sve_ldnt1_gather">; 1750def SVLDNT1SW_GATHER_64B_OFFSETS_U : MInst<"svldnt1sw_gather_[{3}]offset_{0}", "dPUu", "lUl", [IsGatherLoad, IsByteIndexed], MemEltTyInt32, "aarch64_sve_ldnt1_gather">; 1751def SVLDNT1UW_GATHER_64B_OFFSETS_U : MInst<"svldnt1uw_gather_[{3}]offset_{0}", "dPYu", "lUl", [IsGatherLoad, IsByteIndexed, IsZExtReturn], MemEltTyInt32, "aarch64_sve_ldnt1_gather">; 1752 1753def SVLDNT1_GATHER_32B_OFFSETS_U : MInst<"svldnt1_gather_[{3}]offset[_{0}]", "dPcu", "iUif", [IsGatherLoad, IsByteIndexed], MemEltTyDefault, "aarch64_sve_ldnt1_gather_uxtw">; 1754def SVLDNT1SB_GATHER_32B_OFFSETS_U : MInst<"svldnt1sb_gather_[{3}]offset_{0}", "dPSu", "iUi", [IsGatherLoad, IsByteIndexed], MemEltTyInt8, "aarch64_sve_ldnt1_gather_uxtw">; 1755def SVLDNT1UB_GATHER_32B_OFFSETS_U : MInst<"svldnt1ub_gather_[{3}]offset_{0}", "dPWu", "iUi", [IsGatherLoad, IsByteIndexed, IsZExtReturn], MemEltTyInt8, "aarch64_sve_ldnt1_gather_uxtw">; 1756def SVLDNT1SH_GATHER_32B_OFFSETS_U : MInst<"svldnt1sh_gather_[{3}]offset_{0}", "dPTu", "iUi", [IsGatherLoad, IsByteIndexed], MemEltTyInt16, "aarch64_sve_ldnt1_gather_uxtw">; 1757def SVLDNT1UH_GATHER_32B_OFFSETS_U : MInst<"svldnt1uh_gather_[{3}]offset_{0}", "dPXu", "iUi", [IsGatherLoad, IsByteIndexed, IsZExtReturn], MemEltTyInt16, "aarch64_sve_ldnt1_gather_uxtw">; 1758 1759// Non-temporal gather load one vector (vector base, scalar offset in bytes) 1760def SVLDNT1_GATHER_OFFSET_S : MInst<"svldnt1_gather[_{2}base]_offset_{0}", "dPul", "ilUiUlfd", [IsGatherLoad, IsByteIndexed], MemEltTyDefault, "aarch64_sve_ldnt1_gather_scalar_offset">; 1761def SVLDNT1SB_GATHER_OFFSET_S : MInst<"svldnt1sb_gather[_{2}base]_offset_{0}", "dPul", "ilUiUl", [IsGatherLoad, IsByteIndexed], MemEltTyInt8, "aarch64_sve_ldnt1_gather_scalar_offset">; 1762def SVLDNT1UB_GATHER_OFFSET_S : MInst<"svldnt1ub_gather[_{2}base]_offset_{0}", "dPul", "ilUiUl", [IsGatherLoad, IsByteIndexed, IsZExtReturn], MemEltTyInt8, "aarch64_sve_ldnt1_gather_scalar_offset">; 1763def SVLDNT1SH_GATHER_OFFSET_S : MInst<"svldnt1sh_gather[_{2}base]_offset_{0}", "dPul", "ilUiUl", [IsGatherLoad, IsByteIndexed], MemEltTyInt16, "aarch64_sve_ldnt1_gather_scalar_offset">; 1764def SVLDNT1UH_GATHER_OFFSET_S : MInst<"svldnt1uh_gather[_{2}base]_offset_{0}", "dPul", "ilUiUl", [IsGatherLoad, IsByteIndexed, IsZExtReturn], MemEltTyInt16, "aarch64_sve_ldnt1_gather_scalar_offset">; 1765def SVLDNT1SW_GATHER_OFFSET_S : MInst<"svldnt1sw_gather[_{2}base]_offset_{0}", "dPul", "lUl", [IsGatherLoad, IsByteIndexed], MemEltTyInt32, "aarch64_sve_ldnt1_gather_scalar_offset">; 1766def SVLDNT1UW_GATHER_OFFSET_S : MInst<"svldnt1uw_gather[_{2}base]_offset_{0}", "dPul", "lUl", [IsGatherLoad, IsByteIndexed, IsZExtReturn], MemEltTyInt32, "aarch64_sve_ldnt1_gather_scalar_offset">; 1767 1768// Non-temporal gather load one vector (scalar base, signed vector index) 1769def SVLDNT1_GATHER_64B_INDICES_S : MInst<"svldnt1_gather_[{3}]index[_{0}]", "dPcx", "lUld", [IsGatherLoad], MemEltTyDefault, "aarch64_sve_ldnt1_gather_index">; 1770def SVLDNT1SH_GATHER_64B_INDICES_S : MInst<"svldnt1sh_gather_[{3}]index_{0}", "dPTx", "lUl", [IsGatherLoad], MemEltTyInt16, "aarch64_sve_ldnt1_gather_index">; 1771def SVLDNT1UH_GATHER_64B_INDICES_S : MInst<"svldnt1uh_gather_[{3}]index_{0}", "dPXx", "lUl", [IsGatherLoad, IsZExtReturn], MemEltTyInt16, "aarch64_sve_ldnt1_gather_index">; 1772def SVLDNT1SW_GATHER_64B_INDICES_S : MInst<"svldnt1sw_gather_[{3}]index_{0}", "dPUx", "lUl", [IsGatherLoad], MemEltTyInt32, "aarch64_sve_ldnt1_gather_index">; 1773def SVLDNT1UW_GATHER_64B_INDICES_S : MInst<"svldnt1uw_gather_[{3}]index_{0}", "dPYx", "lUl", [IsGatherLoad, IsZExtReturn], MemEltTyInt32, "aarch64_sve_ldnt1_gather_index">; 1774 1775// Non temporal gather load one vector (scalar base, unsigned vector index) 1776def SVLDNT1_GATHER_64B_INDICES_U : MInst<"svldnt1_gather_[{3}]index[_{0}]", "dPcu", "lUld", [IsGatherLoad], MemEltTyDefault, "aarch64_sve_ldnt1_gather_index">; 1777def SVLDNT1SH_GATHER_64B_INDICES_U : MInst<"svldnt1sh_gather_[{3}]index_{0}", "dPTu", "lUl", [IsGatherLoad], MemEltTyInt16, "aarch64_sve_ldnt1_gather_index">; 1778def SVLDNT1UH_GATHER_64B_INDICES_U : MInst<"svldnt1uh_gather_[{3}]index_{0}", "dPXu", "lUl", [IsGatherLoad, IsZExtReturn], MemEltTyInt16, "aarch64_sve_ldnt1_gather_index">; 1779def SVLDNT1SW_GATHER_64B_INDICES_U : MInst<"svldnt1sw_gather_[{3}]index_{0}", "dPUu", "lUl", [IsGatherLoad], MemEltTyInt32, "aarch64_sve_ldnt1_gather_index">; 1780def SVLDNT1UW_GATHER_64B_INDICES_U : MInst<"svldnt1uw_gather_[{3}]index_{0}", "dPYu", "lUl", [IsGatherLoad, IsZExtReturn], MemEltTyInt32, "aarch64_sve_ldnt1_gather_index">; 1781 1782// Non-temporal gather load one vector (vector base, signed scalar index) 1783def SVLDNT1_GATHER_INDEX_S : MInst<"svldnt1_gather[_{2}base]_index_{0}", "dPul", "ilUiUlfd", [IsGatherLoad], MemEltTyDefault, "aarch64_sve_ldnt1_gather_scalar_offset">; 1784def SVLDNT1SH_GATHER_INDEX_S : MInst<"svldnt1sh_gather[_{2}base]_index_{0}", "dPul", "ilUiUl", [IsGatherLoad], MemEltTyInt16, "aarch64_sve_ldnt1_gather_scalar_offset">; 1785def SVLDNT1UH_GATHER_INDEX_S : MInst<"svldnt1uh_gather[_{2}base]_index_{0}", "dPul", "ilUiUl", [IsGatherLoad, IsZExtReturn], MemEltTyInt16, "aarch64_sve_ldnt1_gather_scalar_offset">; 1786def SVLDNT1SW_GATHER_INDEX_S : MInst<"svldnt1sw_gather[_{2}base]_index_{0}", "dPul", "lUl", [IsGatherLoad], MemEltTyInt32, "aarch64_sve_ldnt1_gather_scalar_offset">; 1787def SVLDNT1UW_GATHER_INDEX_S : MInst<"svldnt1uw_gather[_{2}base]_index_{0}", "dPul", "lUl", [IsGatherLoad, IsZExtReturn], MemEltTyInt32, "aarch64_sve_ldnt1_gather_scalar_offset">; 1788 1789// Non-temporal scatter store one vector (vector base) 1790def SVSTNT1_SCATTER_BASES_U : MInst<"svstnt1_scatter[_{2}base_{d}]", "vPud", "ilUiUlfd", [IsScatterStore], MemEltTyDefault, "aarch64_sve_stnt1_scatter_scalar_offset">; 1791def SVSTNT1B_SCATTER_BASES_U : MInst<"svstnt1b_scatter[_{2}base_{d}]", "vPud", "ilUiUl", [IsScatterStore], MemEltTyInt8, "aarch64_sve_stnt1_scatter_scalar_offset">; 1792def SVSTNT1H_SCATTER_BASES_U : MInst<"svstnt1h_scatter[_{2}base_{d}]", "vPud", "ilUiUl", [IsScatterStore], MemEltTyInt16, "aarch64_sve_stnt1_scatter_scalar_offset">; 1793def SVSTNT1W_SCATTER_BASES_U : MInst<"svstnt1w_scatter[_{2}base_{d}]", "vPud", "lUl", [IsScatterStore], MemEltTyInt32, "aarch64_sve_stnt1_scatter_scalar_offset">; 1794 1795// Non-temporal scatter store one vector (scalar base, signed vector offset in bytes) 1796def SVSTNT1_SCATTER_64B_OFFSETS_S : MInst<"svstnt1_scatter_[{3}]offset[_{d}]", "vPpxd", "lUld", [IsScatterStore, IsByteIndexed], MemEltTyDefault, "aarch64_sve_stnt1_scatter">; 1797def SVSTNT1B_SCATTER_64B_OFFSETS_SS : MInst<"svstnt1b_scatter_[{3}]offset[_{d}]", "vPAxd", "l", [IsScatterStore, IsByteIndexed], MemEltTyInt8, "aarch64_sve_stnt1_scatter">; 1798def SVSTNT1B_SCATTER_64B_OFFSETS_SU : MInst<"svstnt1b_scatter_[{3}]offset[_{d}]", "vPExd", "Ul", [IsScatterStore, IsByteIndexed], MemEltTyInt8, "aarch64_sve_stnt1_scatter">; 1799def SVSTNT1H_SCATTER_64B_OFFSETS_SS : MInst<"svstnt1h_scatter_[{3}]offset[_{d}]", "vPBxd", "l", [IsScatterStore, IsByteIndexed], MemEltTyInt16, "aarch64_sve_stnt1_scatter">; 1800def SVSTNT1H_SCATTER_64B_OFFSETS_SU : MInst<"svstnt1h_scatter_[{3}]offset[_{d}]", "vPFxd", "Ul", [IsScatterStore, IsByteIndexed], MemEltTyInt16, "aarch64_sve_stnt1_scatter">; 1801def SVSTNT1W_SCATTER_64B_OFFSETS_SS : MInst<"svstnt1w_scatter_[{3}]offset[_{d}]", "vPCxd", "l", [IsScatterStore, IsByteIndexed], MemEltTyInt32, "aarch64_sve_stnt1_scatter">; 1802def SVSTNT1W_SCATTER_64B_OFFSETS_SU : MInst<"svstnt1w_scatter_[{3}]offset[_{d}]", "vPGxd", "Ul", [IsScatterStore, IsByteIndexed], MemEltTyInt32, "aarch64_sve_stnt1_scatter">; 1803 1804// Non-temporal scatter store one vector (scalar base, unsigned vector offset in bytes) 1805def SVSTNT1_SCATTER_64B_OFFSETS_U : MInst<"svstnt1_scatter_[{3}]offset[_{d}]", "vPpud", "lUld", [IsScatterStore, IsByteIndexed], MemEltTyDefault, "aarch64_sve_stnt1_scatter">; 1806def SVSTNT1B_SCATTER_64B_OFFSETS_US : MInst<"svstnt1b_scatter_[{3}]offset[_{d}]", "vPAud", "l", [IsScatterStore, IsByteIndexed], MemEltTyInt8, "aarch64_sve_stnt1_scatter">; 1807def SVSTNT1B_SCATTER_64B_OFFSETS_UU : MInst<"svstnt1b_scatter_[{3}]offset[_{d}]", "vPEud", "Ul", [IsScatterStore, IsByteIndexed], MemEltTyInt8, "aarch64_sve_stnt1_scatter">; 1808def SVSTNT1H_SCATTER_64B_OFFSETS_US : MInst<"svstnt1h_scatter_[{3}]offset[_{d}]", "vPBud", "l", [IsScatterStore, IsByteIndexed], MemEltTyInt16, "aarch64_sve_stnt1_scatter">; 1809def SVSTNT1H_SCATTER_64B_OFFSETS_UU : MInst<"svstnt1h_scatter_[{3}]offset[_{d}]", "vPFud", "Ul", [IsScatterStore, IsByteIndexed], MemEltTyInt16, "aarch64_sve_stnt1_scatter">; 1810def SVSTNT1W_SCATTER_64B_OFFSETS_US : MInst<"svstnt1w_scatter_[{3}]offset[_{d}]", "vPCud", "l", [IsScatterStore, IsByteIndexed], MemEltTyInt32, "aarch64_sve_stnt1_scatter">; 1811def SVSTNT1W_SCATTER_64B_OFFSETS_UU : MInst<"svstnt1w_scatter_[{3}]offset[_{d}]", "vPGud", "Ul", [IsScatterStore, IsByteIndexed], MemEltTyInt32, "aarch64_sve_stnt1_scatter">; 1812 1813def SVSTNT1_SCATTER_32B_OFFSETS_U : MInst<"svstnt1_scatter_[{3}]offset[_{d}]", "vPpud", "iUif", [IsScatterStore, IsByteIndexed], MemEltTyDefault, "aarch64_sve_stnt1_scatter_uxtw">; 1814def SVSTNT1B_SCATTER_32B_OFFSETS_US : MInst<"svstnt1b_scatter_[{3}]offset[_{d}]", "vPAud", "i", [IsScatterStore, IsByteIndexed], MemEltTyInt8, "aarch64_sve_stnt1_scatter_uxtw">; 1815def SVSTNT1B_SCATTER_32B_OFFSETS_UU : MInst<"svstnt1b_scatter_[{3}]offset[_{d}]", "vPEud", "Ui", [IsScatterStore, IsByteIndexed], MemEltTyInt8, "aarch64_sve_stnt1_scatter_uxtw">; 1816def SVSTNT1H_SCATTER_32B_OFFSETS_US : MInst<"svstnt1h_scatter_[{3}]offset[_{d}]", "vPBud", "i", [IsScatterStore, IsByteIndexed], MemEltTyInt16, "aarch64_sve_stnt1_scatter_uxtw">; 1817def SVSTNT1H_SCATTER_32B_OFFSETS_UU : MInst<"svstnt1h_scatter_[{3}]offset[_{d}]", "vPFud", "Ui", [IsScatterStore, IsByteIndexed], MemEltTyInt16, "aarch64_sve_stnt1_scatter_uxtw">; 1818 1819// Non-temporal scatter store one vector (vector base, scalar offset in bytes) 1820def SVSTNT1_SCATTER_OFFSET_S : MInst<"svstnt1_scatter[_{2}base]_offset[_{d}]", "vPuld", "ilUiUlfd", [IsScatterStore, IsByteIndexed], MemEltTyDefault, "aarch64_sve_stnt1_scatter_scalar_offset">; 1821def SVSTNT1B_SCATTER_OFFSET_S : MInst<"svstnt1b_scatter[_{2}base]_offset[_{d}]", "vPuld", "ilUiUl", [IsScatterStore, IsByteIndexed], MemEltTyInt8, "aarch64_sve_stnt1_scatter_scalar_offset">; 1822def SVSTNT1H_SCATTER_OFFSET_S : MInst<"svstnt1h_scatter[_{2}base]_offset[_{d}]", "vPuld", "ilUiUl", [IsScatterStore, IsByteIndexed], MemEltTyInt16, "aarch64_sve_stnt1_scatter_scalar_offset">; 1823def SVSTNT1W_SCATTER_OFFSET_S : MInst<"svstnt1w_scatter[_{2}base]_offset[_{d}]", "vPuld", "lUl", [IsScatterStore, IsByteIndexed], MemEltTyInt32, "aarch64_sve_stnt1_scatter_scalar_offset">; 1824 1825// Non-temporal scatter store one vector (scalar base, signed vector index) 1826def SVSTNT1_SCATTER_INDICES_S : MInst<"svstnt1_scatter_[{3}]index[_{d}]", "vPpxd", "lUld", [IsScatterStore], MemEltTyDefault, "aarch64_sve_stnt1_scatter_index">; 1827def SVSTNT1H_SCATTER_INDICES_SS : MInst<"svstnt1h_scatter_[{3}]index[_{d}]", "vPBxd", "l", [IsScatterStore], MemEltTyInt16, "aarch64_sve_stnt1_scatter_index">; 1828def SVSTNT1H_SCATTER_INDICES_SU : MInst<"svstnt1h_scatter_[{3}]index[_{d}]", "vPFxd", "Ul", [IsScatterStore], MemEltTyInt16, "aarch64_sve_stnt1_scatter_index">; 1829def SVSTNT1W_SCATTER_INDICES_SS : MInst<"svstnt1w_scatter_[{3}]index[_{d}]", "vPCxd", "l", [IsScatterStore], MemEltTyInt32, "aarch64_sve_stnt1_scatter_index">; 1830def SVSTNT1W_SCATTER_INDICES_SU : MInst<"svstnt1w_scatter_[{3}]index[_{d}]", "vPGxd", "Ul", [IsScatterStore], MemEltTyInt32, "aarch64_sve_stnt1_scatter_index">; 1831 1832// Non-temporal scatter store one vector (scalar base, unsigned vector index) 1833def SVSTNT1_SCATTER_INDICES_U : MInst<"svstnt1_scatter_[{3}]index[_{d}]", "vPpud", "lUld", [IsScatterStore], MemEltTyDefault, "aarch64_sve_stnt1_scatter_index">; 1834def SVSTNT1H_SCATTER_INDICES_US : MInst<"svstnt1h_scatter_[{3}]index[_{d}]", "vPBud", "l", [IsScatterStore], MemEltTyInt16, "aarch64_sve_stnt1_scatter_index">; 1835def SVSTNT1H_SCATTER_INDICES_UU : MInst<"svstnt1h_scatter_[{3}]index[_{d}]", "vPFud", "Ul", [IsScatterStore], MemEltTyInt16, "aarch64_sve_stnt1_scatter_index">; 1836def SVSTNT1W_SCATTER_INDICES_US : MInst<"svstnt1w_scatter_[{3}]index[_{d}]", "vPCud", "l", [IsScatterStore], MemEltTyInt32, "aarch64_sve_stnt1_scatter_index">; 1837def SVSTNT1W_SCATTER_INDICES_UU : MInst<"svstnt1w_scatter_[{3}]index[_{d}]", "vPGud", "Ul", [IsScatterStore], MemEltTyInt32, "aarch64_sve_stnt1_scatter_index">; 1838 1839// Non-temporal scatter store one vector (vector base, signed scalar index) 1840def SVSTNT1_SCATTER_INDEX_S : MInst<"svstnt1_scatter[_{2}base]_index[_{d}]", "vPuld", "ilUiUlfd", [IsScatterStore], MemEltTyDefault, "aarch64_sve_stnt1_scatter_scalar_offset">; 1841def SVSTNT1H_SCATTER_INDEX_S : MInst<"svstnt1h_scatter[_{2}base]_index[_{d}]", "vPuld", "ilUiUl", [IsScatterStore], MemEltTyInt16, "aarch64_sve_stnt1_scatter_scalar_offset">; 1842def SVSTNT1W_SCATTER_INDEX_S : MInst<"svstnt1w_scatter[_{2}base]_index[_{d}]", "vPuld", "lUl", [IsScatterStore], MemEltTyInt32, "aarch64_sve_stnt1_scatter_scalar_offset">; 1843} 1844 1845//////////////////////////////////////////////////////////////////////////////// 1846// SVE2 - Polynomial arithmetic 1847 1848let SVETargetGuard = "sve2", SMETargetGuard = "sme" in { 1849def SVEORBT : SInst<"sveorbt[_{d}]", "dddd", "csilUcUsUiUl", MergeNone, "aarch64_sve_eorbt", [VerifyRuntimeMode]>; 1850def SVEORBT_N : SInst<"sveorbt[_n_{d}]", "ddda", "csilUcUsUiUl", MergeNone, "aarch64_sve_eorbt", [VerifyRuntimeMode]>; 1851def SVEORTB : SInst<"sveortb[_{d}]", "dddd", "csilUcUsUiUl", MergeNone, "aarch64_sve_eortb", [VerifyRuntimeMode]>; 1852def SVEORTB_N : SInst<"sveortb[_n_{d}]", "ddda", "csilUcUsUiUl", MergeNone, "aarch64_sve_eortb", [VerifyRuntimeMode]>; 1853def SVPMUL : SInst<"svpmul[_{d}]", "ddd", "Uc", MergeNone, "aarch64_sve_pmul", [VerifyRuntimeMode]>; 1854def SVPMUL_N : SInst<"svpmul[_n_{d}]", "dda", "Uc", MergeNone, "aarch64_sve_pmul", [VerifyRuntimeMode]>; 1855def SVPMULLB : SInst<"svpmullb[_{d}]", "dhh", "UsUl", MergeNone, "", [VerifyRuntimeMode]>; 1856def SVPMULLB_N : SInst<"svpmullb[_n_{d}]", "dhR", "UsUl", MergeNone, "", [VerifyRuntimeMode]>; 1857def SVPMULLB_PAIR : SInst<"svpmullb_pair[_{d}]", "ddd", "UcUi", MergeNone, "aarch64_sve_pmullb_pair", [VerifyRuntimeMode]>; 1858def SVPMULLB_PAIR_N : SInst<"svpmullb_pair[_n_{d}]", "dda", "UcUi", MergeNone, "aarch64_sve_pmullb_pair", [VerifyRuntimeMode]>; 1859def SVPMULLT : SInst<"svpmullt[_{d}]", "dhh", "UsUl", MergeNone, "", [VerifyRuntimeMode]>; 1860def SVPMULLT_N : SInst<"svpmullt[_n_{d}]", "dhR", "UsUl", MergeNone, "", [VerifyRuntimeMode]>; 1861def SVPMULLT_PAIR : SInst<"svpmullt_pair[_{d}]", "ddd", "UcUi", MergeNone, "aarch64_sve_pmullt_pair", [VerifyRuntimeMode]>; 1862def SVPMULLT_PAIR_N : SInst<"svpmullt_pair[_n_{d}]", "dda", "UcUi", MergeNone, "aarch64_sve_pmullt_pair", [VerifyRuntimeMode]>; 1863} 1864 1865//////////////////////////////////////////////////////////////////////////////// 1866// SVE2 - Complex integer dot product 1867 1868let SVETargetGuard = "sve2", SMETargetGuard = "sme" in { 1869def SVCDOT : SInst<"svcdot[_{d}]", "ddqqi", "il", MergeNone, "aarch64_sve_cdot", [VerifyRuntimeMode], [ImmCheck<3, ImmCheckComplexRotAll90>]>; 1870def SVCDOT_LANE : SInst<"svcdot_lane[_{d}]", "ddqqii", "il", MergeNone, "aarch64_sve_cdot_lane", [VerifyRuntimeMode], [ImmCheck<4, ImmCheckComplexRotAll90>, 1871 ImmCheck<3, ImmCheckLaneIndexDot, 2>]>; 1872} 1873 1874//////////////////////////////////////////////////////////////////////////////// 1875// SVE2 - Floating-point widening multiply-accumulate 1876 1877let SVETargetGuard = "sve2", SMETargetGuard = "sme" in { 1878def SVMLALB_F : SInst<"svmlalb[_{d}]", "ddhh", "f", MergeNone, "aarch64_sve_fmlalb", [VerifyRuntimeMode]>; 1879def SVMLALB_F_N : SInst<"svmlalb[_n_{d}]", "ddhR", "f", MergeNone, "aarch64_sve_fmlalb", [VerifyRuntimeMode]>; 1880def SVMLALB_F_LANE : SInst<"svmlalb_lane[_{d}]", "ddhhi", "f", MergeNone, "aarch64_sve_fmlalb_lane", [VerifyRuntimeMode], [ImmCheck<3, ImmCheckLaneIndex, 2>]>; 1881def SVMLALT_F : SInst<"svmlalt[_{d}]", "ddhh", "f", MergeNone, "aarch64_sve_fmlalt", [VerifyRuntimeMode]>; 1882def SVMLALT_F_N : SInst<"svmlalt[_n_{d}]", "ddhR", "f", MergeNone, "aarch64_sve_fmlalt", [VerifyRuntimeMode]>; 1883def SVMLALT_F_LANE : SInst<"svmlalt_lane[_{d}]", "ddhhi", "f", MergeNone, "aarch64_sve_fmlalt_lane", [VerifyRuntimeMode], [ImmCheck<3, ImmCheckLaneIndex, 2>]>; 1884def SVMLSLB_F : SInst<"svmlslb[_{d}]", "ddhh", "f", MergeNone, "aarch64_sve_fmlslb", [VerifyRuntimeMode]>; 1885def SVMLSLB_F_N : SInst<"svmlslb[_n_{d}]", "ddhR", "f", MergeNone, "aarch64_sve_fmlslb", [VerifyRuntimeMode]>; 1886def SVMLSLB_F_LANE : SInst<"svmlslb_lane[_{d}]", "ddhhi", "f", MergeNone, "aarch64_sve_fmlslb_lane", [VerifyRuntimeMode], [ImmCheck<3, ImmCheckLaneIndex, 2>]>; 1887def SVMLSLT_F : SInst<"svmlslt[_{d}]", "ddhh", "f", MergeNone, "aarch64_sve_fmlslt", [VerifyRuntimeMode]>; 1888def SVMLSLT_F_N : SInst<"svmlslt[_n_{d}]", "ddhR", "f", MergeNone, "aarch64_sve_fmlslt", [VerifyRuntimeMode]>; 1889def SVMLSLT_F_LANE : SInst<"svmlslt_lane[_{d}]", "ddhhi", "f", MergeNone, "aarch64_sve_fmlslt_lane", [VerifyRuntimeMode], [ImmCheck<3, ImmCheckLaneIndex, 2>]>; 1890} 1891 1892//////////////////////////////////////////////////////////////////////////////// 1893// SVE2 - Floating-point integer binary logarithm 1894 1895let SVETargetGuard = "sve2", SMETargetGuard = "sme" in { 1896def SVLOGB_M : SInst<"svlogb[_{d}]", "xxPd", "hfd", MergeOp1, "aarch64_sve_flogb", [VerifyRuntimeMode]>; 1897def SVLOGB_X : SInst<"svlogb[_{d}]", "xPd", "hfd", MergeAnyExp, "aarch64_sve_flogb", [VerifyRuntimeMode]>; 1898def SVLOGB_Z : SInst<"svlogb[_{d}]", "xPd", "hfd", MergeZeroExp, "aarch64_sve_flogb", [VerifyRuntimeMode]>; 1899} 1900 1901//////////////////////////////////////////////////////////////////////////////// 1902// SVE2 - Vector Histogram count 1903 1904let SVETargetGuard = "sve2", SMETargetGuard = InvalidMode in { 1905def SVHISTCNT : SInst<"svhistcnt[_{d}]_z", "uPdd", "ilUiUl", MergeNone, "aarch64_sve_histcnt">; 1906def SVHISTSEG : SInst<"svhistseg[_{d}]", "udd", "cUc", MergeNone, "aarch64_sve_histseg">; 1907} 1908 1909//////////////////////////////////////////////////////////////////////////////// 1910// SVE2 - Character match 1911 1912let SVETargetGuard = "sve2", SMETargetGuard = InvalidMode in { 1913def SVMATCH : SInst<"svmatch[_{d}]", "PPdd", "csUcUs", MergeNone, "aarch64_sve_match">; 1914def SVNMATCH : SInst<"svnmatch[_{d}]", "PPdd", "csUcUs", MergeNone, "aarch64_sve_nmatch">; 1915} 1916 1917//////////////////////////////////////////////////////////////////////////////// 1918// SVE2 - Contiguous conflict detection 1919let SVETargetGuard = "sve2", SMETargetGuard = "sme" in { 1920def SVWHILERW_B : SInst<"svwhilerw[_{1}]", "Pcc", "cUc", MergeNone, "aarch64_sve_whilerw_b", [IsOverloadWhileRW, VerifyRuntimeMode]>; 1921def SVWHILERW_H : SInst<"svwhilerw[_{1}]", "Pcc", "sUsh", MergeNone, "aarch64_sve_whilerw_h", [IsOverloadWhileRW, VerifyRuntimeMode]>; 1922def SVWHILERW_S : SInst<"svwhilerw[_{1}]", "Pcc", "iUif", MergeNone, "aarch64_sve_whilerw_s", [IsOverloadWhileRW, VerifyRuntimeMode]>; 1923def SVWHILERW_D : SInst<"svwhilerw[_{1}]", "Pcc", "lUld", MergeNone, "aarch64_sve_whilerw_d", [IsOverloadWhileRW, VerifyRuntimeMode]>; 1924 1925def SVWHILEWR_B : SInst<"svwhilewr[_{1}]", "Pcc", "cUc", MergeNone, "aarch64_sve_whilewr_b", [IsOverloadWhileRW, VerifyRuntimeMode]>; 1926def SVWHILEWR_H : SInst<"svwhilewr[_{1}]", "Pcc", "sUsh", MergeNone, "aarch64_sve_whilewr_h", [IsOverloadWhileRW, VerifyRuntimeMode]>; 1927def SVWHILEWR_S : SInst<"svwhilewr[_{1}]", "Pcc", "iUif", MergeNone, "aarch64_sve_whilewr_s", [IsOverloadWhileRW, VerifyRuntimeMode]>; 1928def SVWHILEWR_D : SInst<"svwhilewr[_{1}]", "Pcc", "lUld", MergeNone, "aarch64_sve_whilewr_d", [IsOverloadWhileRW, VerifyRuntimeMode]>; 1929} 1930 1931let SVETargetGuard = "sve2,bf16", SMETargetGuard = "sme,bf16" in { 1932def SVWHILERW_H_BF16 : SInst<"svwhilerw[_{1}]", "Pcc", "b", MergeNone, "aarch64_sve_whilerw_h", [IsOverloadWhileRW, VerifyRuntimeMode]>; 1933def SVWHILEWR_H_BF16 : SInst<"svwhilewr[_{1}]", "Pcc", "b", MergeNone, "aarch64_sve_whilewr_h", [IsOverloadWhileRW, VerifyRuntimeMode]>; 1934} 1935 1936//////////////////////////////////////////////////////////////////////////////// 1937// SVE2 - Extended table lookup/permute 1938let SVETargetGuard = "sve2", SMETargetGuard = "sme" in { 1939def SVTBL2 : SInst<"svtbl2[_{d}]", "d2u", "csilUcUsUiUlhfd", MergeNone, "", [VerifyRuntimeMode]>; 1940def SVTBX : SInst<"svtbx[_{d}]", "dddu", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_tbx", [VerifyRuntimeMode]>; 1941} 1942 1943let SVETargetGuard = "sve2,bf16", SMETargetGuard = "sme,bf16" in { 1944def SVTBL2_BF16 : SInst<"svtbl2[_{d}]", "d2u", "b", MergeNone, "", [VerifyRuntimeMode]>; 1945def SVTBX_BF16 : SInst<"svtbx[_{d}]", "dddu", "b", MergeNone, "aarch64_sve_tbx", [VerifyRuntimeMode]>; 1946} 1947 1948//////////////////////////////////////////////////////////////////////////////// 1949// SVE2 - Lookup table 1950let SVETargetGuard = "sve2,lut", SMETargetGuard = "sme2,lut" in { 1951 def SVLUTI2_B : SInst<"svluti2_lane[_{d}]", "dd[i", "cUc", MergeNone, "aarch64_sve_luti2_lane", [VerifyRuntimeMode], [ImmCheck<2, ImmCheck0_3>]>; 1952 def SVLUTI2_H : SInst<"svluti2_lane[_{d}]", "dd[i", "sUsh", MergeNone, "aarch64_sve_luti2_lane", [VerifyRuntimeMode], [ImmCheck<2, ImmCheck0_7>]>; 1953 1954 def SVLUTI4_B : SInst<"svluti4_lane[_{d}]", "dd[i", "cUc", MergeNone, "aarch64_sve_luti4_lane", [VerifyRuntimeMode], [ImmCheck<2, ImmCheck0_1>]>; 1955 def SVLUTI4_H : SInst<"svluti4_lane[_{d}]", "dd[i", "sUsh", MergeNone, "aarch64_sve_luti4_lane", [VerifyRuntimeMode], [ImmCheck<2, ImmCheck0_3>]>; 1956 1957 def SVLUTI4_x2 : SInst<"svluti4_lane[_{d}_x2]", "d2.d[i", "sUsh", MergeNone, "aarch64_sve_luti4_lane_x2", [VerifyRuntimeMode], [ImmCheck<2, ImmCheck0_3>]>; 1958} 1959 1960let SVETargetGuard = "sve2,lut,bf16", SMETargetGuard = "sme2,lut,bf16" in { 1961 def SVLUTI2_BF16 : SInst<"svluti2_lane[_{d}]", "dd[i", "b", MergeNone, "aarch64_sve_luti2_lane", [ VerifyRuntimeMode], [ImmCheck<2, ImmCheck0_7>]>; 1962 def SVLUTI4_BF16 : SInst<"svluti4_lane[_{d}]", "dd[i", "b", MergeNone, "aarch64_sve_luti4_lane", [ VerifyRuntimeMode], [ImmCheck<2, ImmCheck0_3>]>; 1963 def SVLUTI4_BF16_x2 : SInst<"svluti4_lane[_{d}_x2]", "d2.d[i", "b", MergeNone, "aarch64_sve_luti4_lane_x2", [ VerifyRuntimeMode], [ImmCheck<2, ImmCheck0_3>]>; 1964} 1965 1966//////////////////////////////////////////////////////////////////////////////// 1967// SVE2 - Optional 1968 1969let SVETargetGuard = "sve2,sve-aes", SMETargetGuard = InvalidMode in { 1970def SVAESD : SInst<"svaesd[_{d}]", "ddd", "Uc", MergeNone, "aarch64_sve_aesd", [IsOverloadNone]>; 1971def SVAESIMC : SInst<"svaesimc[_{d}]", "dd", "Uc", MergeNone, "aarch64_sve_aesimc", [IsOverloadNone]>; 1972def SVAESE : SInst<"svaese[_{d}]", "ddd", "Uc", MergeNone, "aarch64_sve_aese", [IsOverloadNone]>; 1973def SVAESMC : SInst<"svaesmc[_{d}]", "dd", "Uc", MergeNone, "aarch64_sve_aesmc", [IsOverloadNone]>; 1974 1975def SVPMULLB_PAIR_U64 : SInst<"svpmullb_pair[_{d}]", "ddd", "Ul", MergeNone, "aarch64_sve_pmullb_pair">; 1976def SVPMULLB_PAIR_N_U64 : SInst<"svpmullb_pair[_n_{d}]", "dda", "Ul", MergeNone, "aarch64_sve_pmullb_pair">; 1977 1978def SVPMULLT_PAIR_U64 : SInst<"svpmullt_pair[_{d}]", "ddd", "Ul", MergeNone, "aarch64_sve_pmullt_pair">; 1979def SVPMULLT_PAIR_N_U64 : SInst<"svpmullt_pair[_n_{d}]", "dda", "Ul", MergeNone, "aarch64_sve_pmullt_pair">; 1980} 1981 1982let SVETargetGuard = "sve2-sha3", SMETargetGuard = InvalidMode in { // FIXME: valid from sme2p1. 1983def SVRAX1 : SInst<"svrax1[_{d}]", "ddd", "lUl", MergeNone, "aarch64_sve_rax1", [IsOverloadNone]>; 1984} 1985 1986let SVETargetGuard = "sve2-sm4", SMETargetGuard = InvalidMode in { 1987def SVSM4E : SInst<"svsm4e[_{d}]", "ddd", "Ui", MergeNone, "aarch64_sve_sm4e", [IsOverloadNone]>; 1988def SVSM4EKEY : SInst<"svsm4ekey[_{d}]", "ddd", "Ui", MergeNone, "aarch64_sve_sm4ekey", [IsOverloadNone]>; 1989} 1990 1991let SVETargetGuard = "sve2,sve-bitperm", SMETargetGuard = InvalidMode in { 1992def SVBDEP : SInst<"svbdep[_{d}]", "ddd", "UcUsUiUl", MergeNone, "aarch64_sve_bdep_x">; 1993def SVBDEP_N : SInst<"svbdep[_n_{d}]", "dda", "UcUsUiUl", MergeNone, "aarch64_sve_bdep_x">; 1994def SVBEXT : SInst<"svbext[_{d}]", "ddd", "UcUsUiUl", MergeNone, "aarch64_sve_bext_x">; 1995def SVBEXT_N : SInst<"svbext[_n_{d}]", "dda", "UcUsUiUl", MergeNone, "aarch64_sve_bext_x">; 1996def SVBGRP : SInst<"svbgrp[_{d}]", "ddd", "UcUsUiUl", MergeNone, "aarch64_sve_bgrp_x">; 1997def SVBGRP_N : SInst<"svbgrp[_n_{d}]", "dda", "UcUsUiUl", MergeNone, "aarch64_sve_bgrp_x">; 1998} 1999 2000let SVETargetGuard = "sve2p1", SMETargetGuard = "sme" in { 2001def SVPSEL_B : SInst<"svpsel_lane_b8", "PPPm", "Pc", MergeNone, "", [VerifyRuntimeMode], []>; 2002def SVPSEL_H : SInst<"svpsel_lane_b16", "PPPm", "Ps", MergeNone, "", [VerifyRuntimeMode], []>; 2003def SVPSEL_S : SInst<"svpsel_lane_b32", "PPPm", "Pi", MergeNone, "", [VerifyRuntimeMode], []>; 2004def SVPSEL_D : SInst<"svpsel_lane_b64", "PPPm", "Pl", MergeNone, "", [VerifyRuntimeMode], []>; 2005} 2006 2007// Standalone sve2.1 builtins 2008let SVETargetGuard = "sve2p1", SMETargetGuard = InvalidMode in { 2009def SVORQV : SInst<"svorqv[_{d}]", "{Pd", "csilUcUsUiUl", MergeNone, "aarch64_sve_orqv", [IsReductionQV]>; 2010def SVEORQV : SInst<"sveorqv[_{d}]", "{Pd", "csilUcUsUiUl", MergeNone, "aarch64_sve_eorqv", [IsReductionQV]>; 2011def SVADDQV : SInst<"svaddqv[_{d}]", "{Pd", "csilUcUsUiUl", MergeNone, "aarch64_sve_addqv", [IsReductionQV]>; 2012def SVANDQV : SInst<"svandqv[_{d}]", "{Pd", "csilUcUsUiUl", MergeNone, "aarch64_sve_andqv", [IsReductionQV]>; 2013def SVSMAXQV : SInst<"svmaxqv[_{d}]", "{Pd", "csil", MergeNone, "aarch64_sve_smaxqv", [IsReductionQV]>; 2014def SVUMAXQV : SInst<"svmaxqv[_{d}]", "{Pd", "UcUsUiUl", MergeNone, "aarch64_sve_umaxqv", [IsReductionQV]>; 2015def SVSMINQV : SInst<"svminqv[_{d}]", "{Pd", "csil", MergeNone, "aarch64_sve_sminqv", [IsReductionQV]>; 2016def SVUMINQV : SInst<"svminqv[_{d}]", "{Pd", "UcUsUiUl", MergeNone, "aarch64_sve_uminqv", [IsReductionQV]>; 2017 2018def SVFADDQV : SInst<"svaddqv[_{d}]", "{Pd", "hfd", MergeNone, "aarch64_sve_faddqv", [IsReductionQV]>; 2019def SVFMAXNMQV : SInst<"svmaxnmqv[_{d}]", "{Pd", "hfd", MergeNone, "aarch64_sve_fmaxnmqv", [IsReductionQV]>; 2020def SVFMINNMQV : SInst<"svminnmqv[_{d}]", "{Pd", "hfd", MergeNone, "aarch64_sve_fminnmqv", [IsReductionQV]>; 2021def SVFMAXQV : SInst<"svmaxqv[_{d}]", "{Pd", "hfd", MergeNone, "aarch64_sve_fmaxqv", [IsReductionQV]>; 2022def SVFMINQV : SInst<"svminqv[_{d}]", "{Pd", "hfd", MergeNone, "aarch64_sve_fminqv", [IsReductionQV]>; 2023} 2024 2025let SVETargetGuard = "sve2p1", SMETargetGuard = "sme2" in { 2026def SVPEXT_SINGLE : SInst<"svpext_lane_{d}", "P}i", "QcQsQiQl", MergeNone, "aarch64_sve_pext", [VerifyRuntimeMode], [ImmCheck<1, ImmCheck0_3>]>; 2027def SVPEXT_X2 : SInst<"svpext_lane_{d}_x2", "2.P}i", "QcQsQiQl", MergeNone, "aarch64_sve_pext_x2", [VerifyRuntimeMode], [ImmCheck<1, ImmCheck0_1>]>; 2028 2029def SVPSEL_COUNT_ALIAS_B : SInst<"svpsel_lane_c8", "}}Pm", "Pc", MergeNone, "", [VerifyRuntimeMode], []>; 2030def SVPSEL_COUNT_ALIAS_H : SInst<"svpsel_lane_c16", "}}Pm", "Ps", MergeNone, "", [VerifyRuntimeMode], []>; 2031def SVPSEL_COUNT_ALIAS_S : SInst<"svpsel_lane_c32", "}}Pm", "Pi", MergeNone, "", [VerifyRuntimeMode], []>; 2032def SVPSEL_COUNT_ALIAS_D : SInst<"svpsel_lane_c64", "}}Pm", "Pl", MergeNone, "", [VerifyRuntimeMode], []>; 2033 2034def SVWHILEGE_COUNT : SInst<"svwhilege_{d}[_{1}]", "}lli", "QcQsQiQl", MergeNone, "aarch64_sve_whilege_{d}", [IsOverloadNone, VerifyRuntimeMode], [ImmCheck<2, ImmCheck2_4_Mul2>]>; 2035def SVWHILEGT_COUNT : SInst<"svwhilegt_{d}[_{1}]", "}lli", "QcQsQiQl", MergeNone, "aarch64_sve_whilegt_{d}", [IsOverloadNone, VerifyRuntimeMode], [ImmCheck<2, ImmCheck2_4_Mul2>]>; 2036def SVWHILELE_COUNT : SInst<"svwhilele_{d}[_{1}]", "}lli", "QcQsQiQl", MergeNone, "aarch64_sve_whilele_{d}", [IsOverloadNone, VerifyRuntimeMode], [ImmCheck<2, ImmCheck2_4_Mul2>]>; 2037def SVWHILELT_COUNT : SInst<"svwhilelt_{d}[_{1}]", "}lli", "QcQsQiQl", MergeNone, "aarch64_sve_whilelt_{d}", [IsOverloadNone, VerifyRuntimeMode], [ImmCheck<2, ImmCheck2_4_Mul2>]>; 2038def SVWHILELO_COUNT : SInst<"svwhilelt_{d}[_{1}]", "}nni", "QcQsQiQl", MergeNone, "aarch64_sve_whilelo_{d}", [IsOverloadNone, VerifyRuntimeMode], [ImmCheck<2, ImmCheck2_4_Mul2>]>; 2039def SVWHILELS_COUNT : SInst<"svwhilele_{d}[_{1}]", "}nni", "QcQsQiQl", MergeNone, "aarch64_sve_whilels_{d}", [IsOverloadNone, VerifyRuntimeMode], [ImmCheck<2, ImmCheck2_4_Mul2>]>; 2040def SVWHILEHI_COUNT : SInst<"svwhilegt_{d}[_{1}]", "}nni", "QcQsQiQl", MergeNone, "aarch64_sve_whilehi_{d}", [IsOverloadNone, VerifyRuntimeMode], [ImmCheck<2, ImmCheck2_4_Mul2>]>; 2041def SVWHILEHS_COUNT : SInst<"svwhilege_{d}[_{1}]", "}nni", "QcQsQiQl", MergeNone, "aarch64_sve_whilehs_{d}", [IsOverloadNone, VerifyRuntimeMode], [ImmCheck<2, ImmCheck2_4_Mul2>]>; 2042} 2043 2044multiclass MultiVecLoad<string i> { 2045 def SV # NAME # B_X2 : MInst<"sv" # i # "[_{2}]_x2", "2}c", "cUc", [IsStructLoad, VerifyRuntimeMode], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">; 2046 def SV # NAME # H_X2 : MInst<"sv" # i # "[_{2}]_x2", "2}c", "sUshb", [IsStructLoad, VerifyRuntimeMode], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">; 2047 def SV # NAME # W_X2 : MInst<"sv" # i # "[_{2}]_x2", "2}c", "iUif", [IsStructLoad, VerifyRuntimeMode], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">; 2048 def SV # NAME # D_X2 : MInst<"sv" # i # "[_{2}]_x2", "2}c", "lUld", [IsStructLoad, VerifyRuntimeMode], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">; 2049 def SV # NAME # B_X4 : MInst<"sv" # i # "[_{2}]_x4", "4}c", "cUc", [IsStructLoad, VerifyRuntimeMode], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">; 2050 def SV # NAME # H_X4 : MInst<"sv" # i # "[_{2}]_x4", "4}c", "sUshb", [IsStructLoad, VerifyRuntimeMode], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">; 2051 def SV # NAME # W_X4 : MInst<"sv" # i # "[_{2}]_x4", "4}c", "iUif", [IsStructLoad, VerifyRuntimeMode], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">; 2052 def SV # NAME # D_X4 : MInst<"sv" # i # "[_{2}]_x4", "4}c", "lUld", [IsStructLoad, VerifyRuntimeMode], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">; 2053 2054 def SV # NAME # B_VNUM_X2 : MInst<"sv" # i # "_vnum" # "[_{2}]_x2", "2}cl", "cUc", [IsStructLoad, VerifyRuntimeMode], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">; 2055 def SV # NAME # H_VNUM_X2 : MInst<"sv" # i # "_vnum" # "[_{2}]_x2", "2}cl", "sUshb", [IsStructLoad, VerifyRuntimeMode], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">; 2056 def SV # NAME # W_VNUM_X2 : MInst<"sv" # i # "_vnum" # "[_{2}]_x2", "2}cl", "iUif", [IsStructLoad, VerifyRuntimeMode], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">; 2057 def SV # NAME # D_VNUM_X2 : MInst<"sv" # i # "_vnum" # "[_{2}]_x2", "2}cl", "lUld", [IsStructLoad, VerifyRuntimeMode], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">; 2058 def SV # NAME # B_VNUM_X4 : MInst<"sv" # i # "_vnum" # "[_{2}]_x4", "4}cl", "cUc", [IsStructLoad, VerifyRuntimeMode], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">; 2059 def SV # NAME # H_VNUM_X4 : MInst<"sv" # i # "_vnum" # "[_{2}]_x4", "4}cl", "sUshb", [IsStructLoad, VerifyRuntimeMode], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">; 2060 def SV # NAME # W_VNUM_X4 : MInst<"sv" # i # "_vnum" # "[_{2}]_x4", "4}cl", "iUif", [IsStructLoad, VerifyRuntimeMode], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">; 2061 def SV # NAME # D_VNUM_X4 : MInst<"sv" # i # "_vnum" # "[_{2}]_x4", "4}cl", "lUld", [IsStructLoad, VerifyRuntimeMode], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">; 2062} 2063 2064let SVETargetGuard = "sve2p1", SMETargetGuard = "sme2" in { 2065 defm LD1 : MultiVecLoad<"ld1">; 2066 defm LDNT1 : MultiVecLoad<"ldnt1">; 2067} 2068 2069multiclass MultiVecStore<string i> { 2070 def SV # NAME # B_X2 : MInst<"sv" # i # "[_{2}_x2]", "v}p2", "cUc", [IsStructStore, VerifyRuntimeMode], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">; 2071 def SV # NAME # H_X2 : MInst<"sv" # i # "[_{2}_x2]", "v}p2", "sUshb", [IsStructStore, VerifyRuntimeMode], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">; 2072 def SV # NAME # W_X2 : MInst<"sv" # i # "[_{2}_x2]", "v}p2", "iUif", [IsStructStore, VerifyRuntimeMode], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">; 2073 def SV # NAME # D_X2 : MInst<"sv" # i # "[_{2}_x2]", "v}p2", "lUld", [IsStructStore, VerifyRuntimeMode], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">; 2074 def SV # NAME # B_X4 : MInst<"sv" # i # "[_{2}_x4]", "v}p4", "cUc", [IsStructStore, VerifyRuntimeMode], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">; 2075 def SV # NAME # H_X4 : MInst<"sv" # i # "[_{2}_x4]", "v}p4", "sUshb", [IsStructStore, VerifyRuntimeMode], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">; 2076 def SV # NAME # W_X4 : MInst<"sv" # i # "[_{2}_x4]", "v}p4", "iUif", [IsStructStore, VerifyRuntimeMode], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">; 2077 def SV # NAME # D_X4 : MInst<"sv" # i # "[_{2}_x4]", "v}p4", "lUld", [IsStructStore, VerifyRuntimeMode], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">; 2078 2079 def SV # NAME # B_VNUM_X2 : MInst<"sv" # i # "_vnum" # "[_{2}_x2]", "v}pl2", "cUc", [IsStructStore, VerifyRuntimeMode], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">; 2080 def SV # NAME # H_VNUM_X2 : MInst<"sv" # i # "_vnum" # "[_{2}_x2]", "v}pl2", "sUshb", [IsStructStore, VerifyRuntimeMode], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">; 2081 def SV # NAME # W_VNUM_X2 : MInst<"sv" # i # "_vnum" # "[_{2}_x2]", "v}pl2", "iUif", [IsStructStore, VerifyRuntimeMode], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">; 2082 def SV # NAME # D_VNUM_X2 : MInst<"sv" # i # "_vnum" # "[_{2}_x2]", "v}pl2", "lUld", [IsStructStore, VerifyRuntimeMode], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">; 2083 def SV # NAME # B_VNUM_X4 : MInst<"sv" # i # "_vnum" # "[_{2}_x4]", "v}pl4", "cUc", [IsStructStore, VerifyRuntimeMode], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">; 2084 def SV # NAME # H_VNUM_X4 : MInst<"sv" # i # "_vnum" # "[_{2}_x4]", "v}pl4", "sUshb", [IsStructStore, VerifyRuntimeMode], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">; 2085 def SV # NAME # W_VNUM_X4 : MInst<"sv" # i # "_vnum" # "[_{2}_x4]", "v}pl4", "iUif", [IsStructStore, VerifyRuntimeMode], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">; 2086 def SV # NAME # D_VNUM_X4 : MInst<"sv" # i # "_vnum" # "[_{2}_x4]", "v}pl4", "lUld", [IsStructStore, VerifyRuntimeMode], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">; 2087} 2088 2089let SVETargetGuard = "sve2p1", SMETargetGuard = "sme2" in { 2090 defm ST1 : MultiVecStore<"st1">; 2091 defm STNT1 : MultiVecStore<"stnt1">; 2092} 2093 2094let SVETargetGuard = "sve2p1", SMETargetGuard = "sme2" in { 2095def SVDOT_X2_S : SInst<"svdot[_{d}_{2}]", "ddhh", "i", MergeNone, "aarch64_sve_sdot_x2", [VerifyRuntimeMode], []>; 2096def SVDOT_X2_U : SInst<"svdot[_{d}_{2}]", "ddhh", "Ui", MergeNone, "aarch64_sve_udot_x2", [VerifyRuntimeMode], []>; 2097def SVDOT_X2_F : SInst<"svdot[_{d}_{2}]", "ddhh", "f", MergeNone, "aarch64_sve_fdot_x2", [VerifyRuntimeMode], []>; 2098def SVDOT_LANE_X2_S : SInst<"svdot_lane[_{d}_{2}]", "ddhhi", "i", MergeNone, "aarch64_sve_sdot_lane_x2", [VerifyRuntimeMode], [ImmCheck<3, ImmCheck0_3>]>; 2099def SVDOT_LANE_X2_U : SInst<"svdot_lane[_{d}_{2}]", "ddhhi", "Ui", MergeNone, "aarch64_sve_udot_lane_x2", [VerifyRuntimeMode], [ImmCheck<3, ImmCheck0_3>]>; 2100def SVDOT_LANE_X2_F : SInst<"svdot_lane[_{d}_{2}]", "ddhhi", "f", MergeNone, "aarch64_sve_fdot_lane_x2", [VerifyRuntimeMode], [ImmCheck<3, ImmCheck0_3>]>; 2101} 2102 2103let SVETargetGuard = "sve2p1", SMETargetGuard = "sme" in { 2104def SVSCLAMP : SInst<"svclamp[_{d}]", "dddd", "csil", MergeNone, "aarch64_sve_sclamp", [VerifyRuntimeMode], []>; 2105def SVUCLAMP : SInst<"svclamp[_{d}]", "dddd", "UcUsUiUl", MergeNone, "aarch64_sve_uclamp", [VerifyRuntimeMode], []>; 2106 2107defm SVREVD : SInstZPZ<"svrevd", "csilUcUsUiUlbhfd", "aarch64_sve_revd">; 2108} 2109 2110let SVETargetGuard = "sve2p1", SMETargetGuard = "sme2" in { 2111 def SVPTRUE_COUNT : SInst<"svptrue_{d}", "}v", "QcQsQiQl", MergeNone, "aarch64_sve_ptrue_{d}", [IsOverloadNone, VerifyRuntimeMode], []>; 2112 2113 def SVPFALSE_COUNT_ALIAS : SInst<"svpfalse_c", "}v", "", MergeNone, "", [IsOverloadNone, VerifyRuntimeMode]>; 2114 2115 def SVFCLAMP : SInst<"svclamp[_{d}]", "dddd", "hfd", MergeNone, "aarch64_sve_fclamp", [VerifyRuntimeMode], []>; 2116 def SVCNTP_COUNT : SInst<"svcntp_{d}", "n}i", "QcQsQiQl", MergeNone, "aarch64_sve_cntp_{d}", [IsOverloadNone, VerifyRuntimeMode], [ImmCheck<1, ImmCheck2_4_Mul2>]>; 2117} 2118 2119let SVETargetGuard = "sve2,sve-b16b16", SMETargetGuard = "sme2,sve-b16b16" in { 2120defm SVMUL_BF : SInstZPZZ<"svmul", "b", "aarch64_sve_fmul", "aarch64_sve_fmul_u", [VerifyRuntimeMode]>; 2121defm SVADD_BF : SInstZPZZ<"svadd", "b", "aarch64_sve_fadd", "aarch64_sve_fadd_u", [VerifyRuntimeMode]>; 2122defm SVSUB_BF : SInstZPZZ<"svsub", "b", "aarch64_sve_fsub", "aarch64_sve_fsub_u", [VerifyRuntimeMode]>; 2123defm SVMAXNM_BF : SInstZPZZ<"svmaxnm","b", "aarch64_sve_fmaxnm", "aarch64_sve_fmaxnm_u", [VerifyRuntimeMode]>; 2124defm SVMINNM_BF : SInstZPZZ<"svminnm","b", "aarch64_sve_fminnm", "aarch64_sve_fminnm_u", [VerifyRuntimeMode]>; 2125defm SVMAX_BF : SInstZPZZ<"svmax", "b", "aarch64_sve_fmax", "aarch64_sve_fmax_u", [VerifyRuntimeMode]>; 2126defm SVMIN_BF : SInstZPZZ<"svmin", "b", "aarch64_sve_fmin", "aarch64_sve_fmin_u", [VerifyRuntimeMode]>; 2127defm SVMLA_BF : SInstZPZZZ<"svmla", "b", "aarch64_sve_fmla", "aarch64_sve_fmla_u", [VerifyRuntimeMode]>; 2128defm SVMLS_BF : SInstZPZZZ<"svmls", "b", "aarch64_sve_fmls", "aarch64_sve_fmls_u", [VerifyRuntimeMode]>; 2129def SVMLA_LANE_BF : SInst<"svmla_lane[_{d}]", "ddddi", "b", MergeNone, "aarch64_sve_fmla_lane", [VerifyRuntimeMode], [ImmCheck<3, ImmCheckLaneIndex, 2>]>; 2130def SVMLS_LANE_BF : SInst<"svmls_lane[_{d}]", "ddddi", "b", MergeNone, "aarch64_sve_fmls_lane", [VerifyRuntimeMode], [ImmCheck<3, ImmCheckLaneIndex, 2>]>; 2131def SVMUL_LANE_BF : SInst<"svmul_lane[_{d}]", "dddi", "b", MergeNone, "aarch64_sve_fmul_lane", [VerifyRuntimeMode], [ImmCheck<2, ImmCheckLaneIndex, 1>]>; 2132def SVFCLAMP_BF : SInst<"svclamp[_{d}]", "dddd", "b", MergeNone, "aarch64_sve_fclamp", [VerifyRuntimeMode], []>; 2133} 2134 2135// SME2 2136 2137// SME intrinsics which operate only on vectors and do not require ZA should be added here, 2138// as they could possibly become SVE instructions in the future. 2139 2140multiclass MinMaxIntr<string i, string zm, string mul, string t> { 2141 def SVS # NAME : SInst<"sv" # i # "[" # zm # "_{d}_" # mul # "]", t, "csil", MergeNone, "aarch64_sve_s" # i # zm # "_" # mul, [IsStreaming], []>; 2142 def SVU # NAME : SInst<"sv" # i # "[" # zm # "_{d}_" # mul # "]", t, "UcUsUiUl", MergeNone, "aarch64_sve_u" # i # zm # "_" # mul, [IsStreaming], []>; 2143 def SVF # NAME : SInst<"sv" # i # "[" # zm # "_{d}_" # mul # "]", t, "hfd", MergeNone, "aarch64_sve_f" # i # zm # "_" # mul, [IsStreaming], []>; 2144} 2145 2146let SVETargetGuard = InvalidMode, SMETargetGuard = "sme2" in { 2147// == SMAX / UMAX / FMAX == 2148 defm MAX_SINGLE_X2 : MinMaxIntr<"max", "_single", "x2", "22d">; 2149 defm MAX_MULTI_X2 : MinMaxIntr<"max", "", "x2", "222">; 2150 defm MAX_SINGLE_X4 : MinMaxIntr<"max", "_single", "x4", "44d">; 2151 defm MAX_MULTI_X4 : MinMaxIntr<"max", "", "x4", "444">; 2152 2153// == SMIN / UMIN / FMIN == 2154 defm MIN_SINGLE_X2 : MinMaxIntr<"min", "_single", "x2", "22d">; 2155 defm MIN_MULTI_X2 : MinMaxIntr<"min", "", "x2", "222">; 2156 defm MIN_SINGLE_X4 : MinMaxIntr<"min", "_single", "x4", "44d">; 2157 defm MIN_MULTI_X4 : MinMaxIntr<"min", "", "x4", "444">; 2158} 2159 2160multiclass SInstMinMaxByVector<string name> { 2161 def NAME # _SINGLE_X2 : SInst<"sv" # name # "nm[_single_{d}_x2]", "22d", "hfd", MergeNone, "aarch64_sve_f" # name # "nm_single_x2", [IsStreaming], []>; 2162 def NAME # _SINGLE_X4 : SInst<"sv" # name # "nm[_single_{d}_x4]", "44d", "hfd", MergeNone, "aarch64_sve_f" # name # "nm_single_x4", [IsStreaming], []>; 2163 2164 def NAME # _X2 : SInst<"sv" # name # "nm[_{d}_x2]", "222", "hfd", MergeNone, "aarch64_sve_f" # name # "nm_x2", [IsStreaming], []>; 2165 def NAME # _X4 : SInst<"sv" # name # "nm[_{d}_x4]", "444", "hfd", MergeNone, "aarch64_sve_f" # name # "nm_x4", [IsStreaming], []>; 2166} 2167 2168let SVETargetGuard = InvalidMode, SMETargetGuard = "sme2" in { 2169// == FMINNM / FMAXNM == 2170 defm SVMINNM : SInstMinMaxByVector<"min">; 2171 defm SVMAXNM : SInstMinMaxByVector<"max">; 2172} 2173 2174let SVETargetGuard = InvalidMode, SMETargetGuard = "sme2" in { 2175 // FRINTA / FRINTM / FRINTN / FRINTP 2176 def SVRINTA_X2 : SInst<"svrinta[_{d}_x2]", "22", "f", MergeNone, "aarch64_sve_frinta_x2", [IsStreaming], []>; 2177 def SVRINTA_X4 : SInst<"svrinta[_{d}_x4]", "44", "f", MergeNone, "aarch64_sve_frinta_x4", [IsStreaming], []>; 2178 2179 def SVRINTM_X2 : SInst<"svrintm[_{d}_x2]", "22", "f", MergeNone, "aarch64_sve_frintm_x2", [IsStreaming], []>; 2180 def SVRINTM_X4 : SInst<"svrintm[_{d}_x4]", "44", "f", MergeNone, "aarch64_sve_frintm_x4", [IsStreaming], []>; 2181 2182 def SVRINTN_X2 : SInst<"svrintn[_{d}_x2]", "22", "f", MergeNone, "aarch64_sve_frintn_x2", [IsStreaming], []>; 2183 def SVRINTN_X4 : SInst<"svrintn[_{d}_x4]", "44", "f", MergeNone, "aarch64_sve_frintn_x4", [IsStreaming], []>; 2184 2185 def SVRINTP_X2 : SInst<"svrintp[_{d}_x2]", "22", "f", MergeNone, "aarch64_sve_frintp_x2", [IsStreaming], []>; 2186 def SVRINTP_X4 : SInst<"svrintp[_{d}_x4]", "44", "f", MergeNone, "aarch64_sve_frintp_x4", [IsStreaming], []>; 2187} 2188 2189let SVETargetGuard = InvalidMode, SMETargetGuard = "sme2" in { 2190 def SVSCLAMP_X2 : SInst<"svclamp[_single_{d}_x2]", "22dd", "csil", MergeNone, "aarch64_sve_sclamp_single_x2", [IsStreaming], []>; 2191 def SVUCLAMP_X2 : SInst<"svclamp[_single_{d}_x2]", "22dd", "UcUsUiUl", MergeNone, "aarch64_sve_uclamp_single_x2", [IsStreaming], []>; 2192 def SVFCLAMP_X2 : SInst<"svclamp[_single_{d}_x2]", "22dd", "hfd", MergeNone, "aarch64_sve_fclamp_single_x2", [IsStreaming], []>; 2193 2194 def SVSCLAMP_X4 : SInst<"svclamp[_single_{d}_x4]", "44dd", "csil", MergeNone, "aarch64_sve_sclamp_single_x4", [IsStreaming], []>; 2195 def SVUCLAMP_X4 : SInst<"svclamp[_single_{d}_x4]", "44dd", "UcUsUiUl", MergeNone, "aarch64_sve_uclamp_single_x4", [IsStreaming], []>; 2196 def SVFCLAMP_X4 : SInst<"svclamp[_single_{d}_x4]", "44dd", "hfd", MergeNone, "aarch64_sve_fclamp_single_x4", [IsStreaming], []>; 2197} 2198 2199multiclass BfSingleMultiVector<string name> { 2200 def NAME # _SINGLE_X2 : SInst<"sv" # name # "[_single_{d}_x2]", "22d", "b", MergeNone, "aarch64_sve_f" # name # "_single_x2", [IsStreaming], []>; 2201 def NAME # _SINGLE_X4 : SInst<"sv" # name # "[_single_{d}_x4]", "44d", "b", MergeNone, "aarch64_sve_f" # name # "_single_x4", [IsStreaming], []>; 2202 2203 def NAME # _X2 : SInst<"sv" # name # "[_{d}_x2]", "222", "b", MergeNone, "aarch64_sve_f" # name # "_x2", [IsStreaming], []>; 2204 def NAME # _X4 : SInst<"sv" # name # "[_{d}_x4]", "444", "b", MergeNone, "aarch64_sve_f" # name # "_x4", [IsStreaming], []>; 2205} 2206 2207let SVETargetGuard = InvalidMode, SMETargetGuard = "sme2,sve-b16b16"in { 2208 def SVBFCLAMP_X2 : SInst<"svclamp[_single_{d}_x2]", "22dd", "b", MergeNone, "aarch64_sve_bfclamp_single_x2", [IsStreaming], []>; 2209 def SVBFCLAMP_X4 : SInst<"svclamp[_single_{d}_x4]", "44dd", "b", MergeNone, "aarch64_sve_bfclamp_single_x4", [IsStreaming], []>; 2210 2211 // bfmin, bfmax (single, multi) 2212 defm SVBFMIN : BfSingleMultiVector<"min">; 2213 defm SVBFMAX : BfSingleMultiVector<"max">; 2214 2215 // bfminnm, bfmaxnm (single, multi) 2216 defm SVBFMINNM : BfSingleMultiVector<"minnm">; 2217 defm SVBFMAXNM : BfSingleMultiVector<"maxnm">; 2218} 2219 2220let SVETargetGuard = InvalidMode, SMETargetGuard = "sme2" in { 2221// == ADD (vectors) == 2222 def SVADD_SINGLE_X2 : SInst<"svadd[_single_{d}_x2]", "22d", "cUcsUsiUilUl", MergeNone, "aarch64_sve_add_single_x2", [IsStreaming], []>; 2223 def SVADD_SINGLE_X4 : SInst<"svadd[_single_{d}_x4]", "44d", "cUcsUsiUilUl", MergeNone, "aarch64_sve_add_single_x4", [IsStreaming], []>; 2224 2225 // 2-way and 4-way selects 2226 def SVSEL_X2 : SInst<"svsel[_{d}_x2]", "2}22", "cUcsUsiUilUlbhfd", MergeNone, "aarch64_sve_sel_x2", [IsStreaming], []>; 2227 def SVSEL_X4 : SInst<"svsel[_{d}_x4]", "4}44", "cUcsUsiUilUlbhfd", MergeNone, "aarch64_sve_sel_x4", [IsStreaming], []>; 2228 2229 // SRSHL / URSHL 2230 def SVSRSHL_SINGLE_X2 : SInst<"svrshl[_single_{d}_x2]", "22d", "csil", MergeNone, "aarch64_sve_srshl_single_x2", [IsStreaming], []>; 2231 def SVURSHL_SINGLE_X2 : SInst<"svrshl[_single_{d}_x2]", "22d", "UcUsUiUl", MergeNone, "aarch64_sve_urshl_single_x2", [IsStreaming], []>; 2232 def SVSRSHL_SINGLE_X4 : SInst<"svrshl[_single_{d}_x4]", "44d", "csil", MergeNone, "aarch64_sve_srshl_single_x4", [IsStreaming], []>; 2233 def SVURSHL_SINGLE_X4 : SInst<"svrshl[_single_{d}_x4]", "44d", "UcUsUiUl", MergeNone, "aarch64_sve_urshl_single_x4", [IsStreaming], []>; 2234 2235 def SVSRSHL_X2 : SInst<"svrshl[_{d}_x2]", "222", "csil", MergeNone, "aarch64_sve_srshl_x2", [IsStreaming], []>; 2236 def SVURSHL_X2 : SInst<"svrshl[_{d}_x2]", "222", "UcUsUiUl", MergeNone, "aarch64_sve_urshl_x2", [IsStreaming], []>; 2237 def SVSRSHL_X4 : SInst<"svrshl[_{d}_x4]", "444", "csil", MergeNone, "aarch64_sve_srshl_x4", [IsStreaming], []>; 2238 def SVURSHL_X4 : SInst<"svrshl[_{d}_x4]", "444", "UcUsUiUl", MergeNone, "aarch64_sve_urshl_x4", [IsStreaming], []>; 2239 2240 def SVQRSHRN_X4 : SInst<"svqrshrn[_n]_{0}[_{d}_x4]", "q4i", "il", MergeNone, "aarch64_sve_sqrshrn_x4", [IsStreaming], [ImmCheck<1, ImmCheckShiftRight, 0>]>; 2241 def SVUQRSHRN_X4 : SInst<"svqrshrn[_n]_{0}[_{d}_x4]", "b4i", "UiUl", MergeNone, "aarch64_sve_uqrshrn_x4", [IsStreaming], [ImmCheck<1, ImmCheckShiftRight, 0>]>; 2242 2243 // SQRSHR / UQRSHR 2244 def SVQRSHR_X2 : SInst<"svqrshr[_n]_{0}[_{d}_x2]", "h2i", "i", MergeNone, "aarch64_sve_sqrshr_x2", [IsStreaming], [ImmCheck<1, ImmCheck1_16>]>; 2245 def SVUQRSHR_X2 : SInst<"svqrshr[_n]_{0}[_{d}_x2]", "e2i", "Ui", MergeNone, "aarch64_sve_uqrshr_x2", [IsStreaming], [ImmCheck<1, ImmCheck1_16>]>; 2246 def SVQRSHR_X4 : SInst<"svqrshr[_n]_{0}[_{d}_x4]", "q4i", "il", MergeNone, "aarch64_sve_sqrshr_x4", [IsStreaming], [ImmCheck<1, ImmCheckShiftRight, 0>]>; 2247 def SVUQRSHR_X4 : SInst<"svqrshr[_n]_{0}[_{d}_x4]", "b4i", "UiUl", MergeNone, "aarch64_sve_uqrshr_x4", [IsStreaming], [ImmCheck<1, ImmCheckShiftRight, 0>]>; 2248 2249 // SQRSHRU 2250 def SVSQRSHRU_X2 : SInst<"svqrshru[_n]_{0}[_{d}_x2]", "e2i", "i", MergeNone, "aarch64_sve_sqrshru_x2", [IsStreaming], [ImmCheck<1, ImmCheck1_16>]>; 2251 def SVSQRSHRU_X4 : SInst<"svqrshru[_n]_{0}[_{d}_x4]", "b4i", "il", MergeNone, "aarch64_sve_sqrshru_x4", [IsStreaming], [ImmCheck<1, ImmCheckShiftRight, 0>]>; 2252 2253 def SVSQRSHRUN_X4 : SInst<"svqrshrun[_n]_{0}[_{d}_x4]", "b4i", "il", MergeNone, "aarch64_sve_sqrshrun_x4", [IsStreaming], [ImmCheck<1, ImmCheckShiftRight, 0>]>; 2254 2255 // SQDMULH 2256 def SVSQDMULH_SINGLE_X2 : SInst<"svqdmulh[_single_{d}_x2]", "22d", "csil", MergeNone, "aarch64_sve_sqdmulh_single_vgx2", [IsStreaming], []>; 2257 def SVSQDMULH_SINGLE_X4 : SInst<"svqdmulh[_single_{d}_x4]", "44d", "csil", MergeNone, "aarch64_sve_sqdmulh_single_vgx4", [IsStreaming], []>; 2258 def SVSQDMULH_X2 : SInst<"svqdmulh[_{d}_x2]", "222", "csil", MergeNone, "aarch64_sve_sqdmulh_vgx2", [IsStreaming], []>; 2259 def SVSQDMULH_X4 : SInst<"svqdmulh[_{d}_x4]", "444", "csil", MergeNone, "aarch64_sve_sqdmulh_vgx4", [IsStreaming], []>; 2260} 2261 2262let SVETargetGuard = InvalidMode, SMETargetGuard = "sme2,faminmax" in { 2263 def FAMIN_X2 : Inst<"svamin[_{d}_x2]", "222", "hfd", MergeNone, "aarch64_sme_famin_x2", [IsStreaming], []>; 2264 def FAMAX_X2 : Inst<"svamax[_{d}_x2]", "222", "hfd", MergeNone, "aarch64_sme_famax_x2", [IsStreaming], []>; 2265 def FAMIN_X4 : Inst<"svamin[_{d}_x4]", "444", "hfd", MergeNone, "aarch64_sme_famin_x4", [IsStreaming], []>; 2266 def FAMAX_X4 : Inst<"svamax[_{d}_x4]", "444", "hfd", MergeNone, "aarch64_sme_famax_x4", [IsStreaming], []>; 2267} 2268 2269let SVETargetGuard = "sve2p1", SMETargetGuard = "sme2" in { 2270 def REINTERPRET_SVBOOL_TO_SVCOUNT : Inst<"svreinterpret[_c]", "}P", "Pc", MergeNone, "", [VerifyRuntimeMode], []>; 2271 def REINTERPRET_SVCOUNT_TO_SVBOOL : Inst<"svreinterpret[_b]", "P}", "Pc", MergeNone, "", [VerifyRuntimeMode], []>; 2272 2273 // SQRSHRN / UQRSHRN 2274 def SVQRSHRN_X2 : SInst<"svqrshrn[_n]_{0}[_{d}_x2]", "h2i", "i", MergeNone, "aarch64_sve_sqrshrn_x2", [VerifyRuntimeMode], [ImmCheck<1, ImmCheck1_16>]>; 2275 def SVUQRSHRN_X2 : SInst<"svqrshrn[_n]_{0}[_{d}_x2]", "e2i", "Ui", MergeNone, "aarch64_sve_uqrshrn_x2", [VerifyRuntimeMode], [ImmCheck<1, ImmCheck1_16>]>; 2276 2277 // SQRSHRUN 2278 def SVSQRSHRUN_X2 : SInst<"svqrshrun[_n]_{0}[_{d}_x2]", "e2i", "i", MergeNone, "aarch64_sve_sqrshrun_x2", [VerifyRuntimeMode], [ImmCheck<1, ImmCheck1_16>]>; 2279} 2280 2281let SVETargetGuard = "sve2p1", SMETargetGuard = InvalidMode in { 2282 // ZIPQ1, ZIPQ2, UZPQ1, UZPQ2 2283 def SVZIPQ1 : SInst<"svzipq1[_{d}]", "ddd", "cUcsUsiUilUlbhfdm", MergeNone, "aarch64_sve_zipq1", [], []>; 2284 def SVZIPQ2 : SInst<"svzipq2[_{d}]", "ddd", "cUcsUsiUilUlbhfdm", MergeNone, "aarch64_sve_zipq2", [], []>; 2285 def SVUZPQ1 : SInst<"svuzpq1[_{d}]", "ddd", "cUcsUsiUilUlbhfdm", MergeNone, "aarch64_sve_uzpq1", [], []>; 2286 def SVUZPQ2 : SInst<"svuzpq2[_{d}]", "ddd", "cUcsUsiUilUlbhfdm", MergeNone, "aarch64_sve_uzpq2", [], []>; 2287 // TBLQ, TBXQ 2288 def SVTBLQ : SInst<"svtblq[_{d}]", "ddu", "cUcsUsiUilUlbhfdm", MergeNone, "aarch64_sve_tblq">; 2289 def SVTBXQ : SInst<"svtbxq[_{d}]", "dddu", "cUcsUsiUilUlbhfdm", MergeNone, "aarch64_sve_tbxq">; 2290 // EXTQ 2291 def EXTQ : SInst<"svextq[_{d}]", "dddk", "cUcsUsiUilUlbhfdm", MergeNone, "aarch64_sve_extq", [], [ImmCheck<2, ImmCheckLaneIndex, 0>]>; 2292 2293 // PMOV 2294 // Move to Pred 2295 multiclass PMOV_TO_PRED<string name, string types, string intrinsic, list<FlagType> flags=[], ImmCheckType immCh > { 2296 def _LANE : Inst<name # "_lane[_{d}]", "Pdi", types, MergeNone, intrinsic, flags, [ImmCheck<1, immCh>]>; 2297 def _LANE_ZERO : SInst<name # "[_{d}]", "Pd", types, MergeNone, intrinsic # "_zero", flags, []>; 2298 } 2299 defm SVPMOV_B_TO_PRED : PMOV_TO_PRED<"svpmov", "cUc", "aarch64_sve_pmov_to_pred_lane", [], ImmCheck0_0>; 2300 defm SVPMOV_H_TO_PRED : PMOV_TO_PRED<"svpmov", "sUs", "aarch64_sve_pmov_to_pred_lane", [], ImmCheck0_1>; 2301 defm SVPMOV_S_TO_PRED : PMOV_TO_PRED<"svpmov", "iUi", "aarch64_sve_pmov_to_pred_lane", [], ImmCheck0_3>; 2302 defm SVPMOV_D_TO_PRED : PMOV_TO_PRED<"svpmov", "lUl", "aarch64_sve_pmov_to_pred_lane", [], ImmCheck0_7>; 2303 2304 // Move to Vector 2305 multiclass PMOV_TO_VEC<string name, string types, string intrinsic, list<FlagType> flags=[], ImmCheckType immCh > { 2306 def _M : SInst<name # "_lane[_{d}]", "ddPi", types, MergeOp1, intrinsic # "_merging", flags, [ImmCheck<2, immCh>]>; 2307 def _Z : SInst<name # "_{d}_z", "dP", types, MergeNone, intrinsic # "_zeroing", flags, []>; 2308 } 2309 def SVPMOV_TO_VEC_LANE_B : SInst<"svpmov_{d}_z", "dP", "cUc", MergeNone, "aarch64_sve_pmov_to_vector_lane_zeroing", [], []>; 2310 defm SVPMOV_TO_VEC_LANE_H : PMOV_TO_VEC<"svpmov", "sUs", "aarch64_sve_pmov_to_vector_lane", [], ImmCheck1_1>; 2311 defm SVPMOV_TO_VEC_LANE_S : PMOV_TO_VEC<"svpmov", "iUi", "aarch64_sve_pmov_to_vector_lane", [], ImmCheck1_3>; 2312 defm SVPMOV_TO_VEC_LANE_D : PMOV_TO_VEC<"svpmov", "lUl", "aarch64_sve_pmov_to_vector_lane" ,[], ImmCheck1_7>; 2313} 2314 2315let SVETargetGuard = "sve2p1", SMETargetGuard = "sme2p1" in { 2316 // DUPQ 2317 def SVDUP_LANEQ_B : SInst<"svdup_laneq[_{d}]", "ddi", "cUcm", MergeNone, "aarch64_sve_dup_laneq", [VerifyRuntimeMode], [ImmCheck<1, ImmCheck0_15>]>; 2318 def SVDUP_LANEQ_H : SInst<"svdup_laneq[_{d}]", "ddi", "sUsh", MergeNone, "aarch64_sve_dup_laneq", [VerifyRuntimeMode], [ImmCheck<1, ImmCheck0_7>]>; 2319 def SVDUP_LANEQ_S : SInst<"svdup_laneq[_{d}]", "ddi", "iUif", MergeNone, "aarch64_sve_dup_laneq", [VerifyRuntimeMode], [ImmCheck<1, ImmCheck0_3>]>; 2320 def SVDUP_LANEQ_D : SInst<"svdup_laneq[_{d}]", "ddi", "lUld", MergeNone, "aarch64_sve_dup_laneq", [VerifyRuntimeMode], [ImmCheck<1, ImmCheck0_1>]>; 2321} 2322 2323let SVETargetGuard = "sve2p1,bf16", SMETargetGuard = "sme2p1,bf16" in { 2324 def SVDUP_LANEQ_BF16 : SInst<"svdup_laneq[_{d}]", "ddi", "b", MergeNone, "aarch64_sve_dup_laneq", [VerifyRuntimeMode], [ImmCheck<1, ImmCheck0_7>]>; 2325} 2326 2327// 2328// Multi-vector convert to/from floating-point. 2329// 2330let SVETargetGuard = InvalidMode, SMETargetGuard = "sme2" in { 2331 def SVCVT_F16_X2 : SInst<"svcvt_f16[_f32_x2]", "h2", "f", MergeNone, "aarch64_sve_fcvt_x2", [IsStreaming],[]>; 2332 def SVCVT_BF16_X2 : SInst<"svcvt_bf16[_f32_x2]", "$2", "f", MergeNone, "aarch64_sve_bfcvt_x2", [IsOverloadNone, IsStreaming],[]>; 2333 2334 def SVCVT_F32_U32_X2 : SInst<"svcvt_{d}[_u32_x2]", "2.d2.u", "f", MergeNone, "aarch64_sve_ucvtf_x2", [IsStreaming, IsOverloadWhileOrMultiVecCvt], []>; 2335 def SVCVT_U32_F32_X2 : SInst<"svcvt_{d}[_f32_x2]", "2.d2.M", "Ui", MergeNone, "aarch64_sve_fcvtzu_x2", [IsStreaming, IsOverloadWhileOrMultiVecCvt], []>; 2336 def SVCVT_F32_S32_X2 : SInst<"svcvt_{d}[_s32_x2]", "2.d2.x", "f", MergeNone, "aarch64_sve_scvtf_x2", [IsStreaming, IsOverloadWhileOrMultiVecCvt], []>; 2337 def SVCVT_S32_F32_X2 : SInst<"svcvt_{d}[_f32_x2]", "2.d2.M", "i", MergeNone, "aarch64_sve_fcvtzs_x2", [IsStreaming, IsOverloadWhileOrMultiVecCvt], []>; 2338 2339 def SVCVT_F32_U32_X4 : SInst<"svcvt_{d}[_u32_x4]", "4.d4.u", "f", MergeNone, "aarch64_sve_ucvtf_x4", [IsStreaming, IsOverloadWhileOrMultiVecCvt], []>; 2340 def SVCVT_U32_F32_X4 : SInst<"svcvt_{d}[_f32_x4]", "4.d4.M", "Ui", MergeNone, "aarch64_sve_fcvtzu_x4", [IsStreaming, IsOverloadWhileOrMultiVecCvt], []>; 2341 def SVCVT_F32_S32_X4 : SInst<"svcvt_{d}[_s32_x4]", "4.d4.x", "f", MergeNone, "aarch64_sve_scvtf_x4", [IsStreaming, IsOverloadWhileOrMultiVecCvt], []>; 2342 def SVCVT_S32_F32_X4 : SInst<"svcvt_{d}[_f32_x4]", "4.d4.M", "i", MergeNone, "aarch64_sve_fcvtzs_x4", [IsStreaming, IsOverloadWhileOrMultiVecCvt], []>; 2343} 2344 2345let SVETargetGuard = InvalidMode, SMETargetGuard = "sme-f16f16" in { 2346 def SVCVT_F32_X2 : SInst<"svcvt_{d}[_f16_x2]", "2h", "f", MergeNone, "aarch64_sve_fcvt_widen_x2", [ IsStreaming],[]>; 2347} 2348 2349// 2350// Multi-vector floating-point convert from single-precision to interleaved half-precision/BFloat16 2351// 2352let SVETargetGuard = InvalidMode, SMETargetGuard = "sme2" in { 2353 def SVCVTN_F16_X2 : SInst<"svcvtn_f16[_f32_x2]", "h2", "f", MergeNone, "aarch64_sve_fcvtn_x2", [IsStreaming],[]>; 2354 def SVCVTN_BF16_X2 : SInst<"svcvtn_bf16[_f32_x2]", "$2", "f", MergeNone, "aarch64_sve_bfcvtn_x2", [IsOverloadNone, IsStreaming],[]>; 2355} 2356 2357// 2358//Multi-vector floating-point convert from half-precision to deinterleaved single-precision. 2359// 2360let SVETargetGuard = InvalidMode, SMETargetGuard = "sme-f16f16" in { 2361 def SVCVTL_F32_X2 : SInst<"svcvtl_f32[_f16_x2]", "2h", "f", MergeNone, "aarch64_sve_fcvtl_widen_x2", [ IsStreaming],[]>; 2362} 2363 2364// 2365// Multi-vector saturating extract narrow 2366// 2367let SVETargetGuard = InvalidMode, SMETargetGuard = "sme2" in { 2368 def SVQCVT_S16_S32_X2 : SInst<"svqcvt_s16[_{d}_x2]", "h2.d", "i", MergeNone, "aarch64_sve_sqcvt_x2", [IsStreaming], []>; 2369 def SVQCVT_U16_U32_X2 : SInst<"svqcvt_u16[_{d}_x2]", "e2.d", "Ui", MergeNone, "aarch64_sve_uqcvt_x2", [IsStreaming], []>; 2370 def SVQCVT_U16_S32_X2 : SInst<"svqcvt_u16[_{d}_x2]", "e2.d", "i", MergeNone, "aarch64_sve_sqcvtu_x2", [IsStreaming], []>; 2371 2372 def SVQCVT_S8_S32_X4 : SInst<"svqcvt_s8[_{d}_x4]", "q4.d", "i", MergeNone, "aarch64_sve_sqcvt_x4", [IsStreaming], []>; 2373 def SVQCVT_U8_U32_X4 : SInst<"svqcvt_u8[_{d}_x4]", "b4.d", "Ui", MergeNone, "aarch64_sve_uqcvt_x4", [IsStreaming], []>; 2374 def SVQCVT_U8_S32_X4 : SInst<"svqcvt_u8[_{d}_x4]", "b4.d", "i", MergeNone, "aarch64_sve_sqcvtu_x4", [IsStreaming], []>; 2375 2376 def SVQCVT_S16_S64_X4 : SInst<"svqcvt_s16[_{d}_x4]", "q4.d", "l", MergeNone, "aarch64_sve_sqcvt_x4", [IsStreaming], []>; 2377 def SVQCVT_U16_U64_X4 : SInst<"svqcvt_u16[_{d}_x4]", "b4.d", "Ul", MergeNone, "aarch64_sve_uqcvt_x4", [IsStreaming], []>; 2378 def SVQCVT_U16_S64_X4 : SInst<"svqcvt_u16[_{d}_x4]", "b4.d", "l", MergeNone, "aarch64_sve_sqcvtu_x4", [IsStreaming], []>; 2379} 2380 2381// 2382// Multi-vector saturating extract narrow and interleave 2383// 2384let SVETargetGuard = "sve2p1", SMETargetGuard = "sme2" in { 2385 def SVQCVTN_S16_S32_X2 : SInst<"svqcvtn_s16[_{d}_x2]", "h2.d", "i", MergeNone, "aarch64_sve_sqcvtn_x2", [VerifyRuntimeMode], []>; 2386 def SVQCVTN_U16_U32_X2 : SInst<"svqcvtn_u16[_{d}_x2]", "e2.d", "Ui", MergeNone, "aarch64_sve_uqcvtn_x2", [VerifyRuntimeMode], []>; 2387 def SVQCVTN_U16_S32_X2 : SInst<"svqcvtn_u16[_{d}_x2]", "e2.d", "i", MergeNone, "aarch64_sve_sqcvtun_x2", [VerifyRuntimeMode], []>; 2388} 2389 2390let SVETargetGuard = InvalidMode, SMETargetGuard = "sme2" in { 2391 def SVQCVTN_S8_S32_X4 : SInst<"svqcvtn_s8[_{d}_x4]", "q4.d", "i", MergeNone, "aarch64_sve_sqcvtn_x4", [IsStreaming], []>; 2392 def SVQCVTN_U8_U32_X4 : SInst<"svqcvtn_u8[_{d}_x4]", "b4.d", "Ui", MergeNone, "aarch64_sve_uqcvtn_x4", [IsStreaming], []>; 2393 def SVQCVTN_U8_S32_X4 : SInst<"svqcvtn_u8[_{d}_x4]", "b4.d", "i", MergeNone, "aarch64_sve_sqcvtun_x4", [IsStreaming], []>; 2394 2395 def SVQCVTN_S16_S64_X4 : SInst<"svqcvtn_s16[_{d}_x4]", "q4.d", "l", MergeNone, "aarch64_sve_sqcvtn_x4", [IsStreaming], []>; 2396 def SVQCVTN_U16_U64_X4 : SInst<"svqcvtn_u16[_{d}_x4]", "b4.d", "Ul", MergeNone, "aarch64_sve_uqcvtn_x4", [IsStreaming], []>; 2397 def SVQCVTN_U16_S64_X4 : SInst<"svqcvtn_u16[_{d}_x4]", "b4.d", "l", MergeNone, "aarch64_sve_sqcvtun_x4", [IsStreaming], []>; 2398} 2399 2400// 2401// Multi-vector zip/unzip 2402// 2403 2404let SVETargetGuard = InvalidMode, SMETargetGuard = "sme2" in { 2405 def SVZIP_X2 : SInst<"svzip[_{d}_x2]", "22", "cUcsUsiUilUlbhfd", MergeNone, "aarch64_sve_zip_x2", [IsStreaming], []>; 2406 def SVZIPQ_X2 : SInst<"svzipq[_{d}_x2]", "22", "cUcsUsiUilUlbhfd", MergeNone, "aarch64_sve_zipq_x2", [IsStreaming], []>; 2407 def SVZIP_X4 : SInst<"svzip[_{d}_x4]", "44", "cUcsUsiUilUlbhfd", MergeNone, "aarch64_sve_zip_x4", [IsStreaming], []>; 2408 def SVZIPQ_X4 : SInst<"svzipq[_{d}_x4]", "44", "cUcsUsiUilUlbhfd", MergeNone, "aarch64_sve_zipq_x4", [IsStreaming], []>; 2409 2410 def SVUZP_X2 : SInst<"svuzp[_{d}_x2]", "22", "cUcsUsiUilUlbhfd", MergeNone, "aarch64_sve_uzp_x2", [IsStreaming], []>; 2411 def SVUZPQ_X2 : SInst<"svuzpq[_{d}_x2]", "22", "cUcsUsiUilUlbhfd", MergeNone, "aarch64_sve_uzpq_x2", [IsStreaming], []>; 2412 def SVUZP_X4 : SInst<"svuzp[_{d}_x4]", "44", "cUcsUsiUilUlbhfd", MergeNone, "aarch64_sve_uzp_x4", [IsStreaming], []>; 2413 def SVUZPQ_X4 : SInst<"svuzpq[_{d}_x4]", "44", "cUcsUsiUilUlbhfd", MergeNone, "aarch64_sve_uzpq_x4", [IsStreaming], []>; 2414} 2415 2416// 2417// Multi-vector unpack 2418// 2419 2420let SVETargetGuard = InvalidMode, SMETargetGuard = "sme2" in { 2421 def SVSUNPK_X2 : SInst<"svunpk_{d}[_{1}_x2]", "2h", "sil", MergeNone, "aarch64_sve_sunpk_x2", [IsStreaming], []>; 2422 def SVUUNPK_X2 : SInst<"svunpk_{d}[_{1}_x2]", "2h", "UsUiUl", MergeNone, "aarch64_sve_uunpk_x2", [IsStreaming], []>; 2423 def SVSUNPK_X4 : SInst<"svunpk_{d}[_{3}_x4]", "42.h", "sil", MergeNone, "aarch64_sve_sunpk_x4", [IsStreaming], []>; 2424 def SVUUNPK_X4 : SInst<"svunpk_{d}[_{3}_x4]", "42.h", "UsUiUl", MergeNone, "aarch64_sve_uunpk_x4", [IsStreaming], []>; 2425} 2426 2427let SVETargetGuard = InvalidMode, SMETargetGuard = "sme2,fp8" in { 2428 // Multi-vector scaling 2429 def FSCALE_SINGLE_X2 : Inst<"svscale[_single_{d}_x2]", "22x", "fhd", MergeNone, "aarch64_sme_fp8_scale_single_x2", [IsStreaming],[]>; 2430 def FSCALE_SINGLE_X4 : Inst<"svscale[_single_{d}_x4]", "44x", "fhd", MergeNone, "aarch64_sme_fp8_scale_single_x4", [IsStreaming],[]>; 2431 def FSCALE_X2 : Inst<"svscale[_{d}_x2]", "222.x", "fhd", MergeNone, "aarch64_sme_fp8_scale_x2", [IsStreaming],[]>; 2432 def FSCALE_X4 : Inst<"svscale[_{d}_x4]", "444.x", "fhd", MergeNone, "aarch64_sme_fp8_scale_x4", [IsStreaming],[]>; 2433 2434 // Convert from FP8 to half-precision/BFloat16 multi-vector 2435 def SVF1CVT_X2 : Inst<"svcvt1_{d}[_mf8]_x2_fpm", "2~>", "bh", MergeNone, "aarch64_sve_fp8_cvt1_x2", [IsStreaming, SetsFPMR], []>; 2436 def SVF2CVT_X2 : Inst<"svcvt2_{d}[_mf8]_x2_fpm", "2~>", "bh", MergeNone, "aarch64_sve_fp8_cvt2_x2", [IsStreaming, SetsFPMR], []>; 2437 2438 // Convert from FP8 to deinterleaved half-precision/BFloat16 multi-vector 2439 def SVF1CVTL_X2 : Inst<"svcvtl1_{d}[_mf8]_x2_fpm", "2~>", "bh", MergeNone, "aarch64_sve_fp8_cvtl1_x2", [IsStreaming, SetsFPMR], []>; 2440 def SVF2CVTL_X2 : Inst<"svcvtl2_{d}[_mf8]_x2_fpm", "2~>", "bh", MergeNone, "aarch64_sve_fp8_cvtl2_x2", [IsStreaming, SetsFPMR], []>; 2441 2442 // Convert from single/half/bfloat multivector to FP8 2443 def SVFCVT_X2 : Inst<"svcvt_mf8[_{d}_x2]_fpm", "~2>", "bh", MergeNone, "aarch64_sve_fp8_cvt_x2", [IsStreaming, SetsFPMR], []>; 2444 def SVFCVT_X4 : Inst<"svcvt_mf8[_{d}_x4]_fpm", "~4>", "f", MergeNone, "aarch64_sve_fp8_cvt_x4", [IsOverloadNone, IsStreaming, SetsFPMR], []>; 2445 // interleaved 2446 def SVFCVTN_X4 : Inst<"svcvtn_mf8[_{d}_x4]_fpm", "~4>", "f", MergeNone, "aarch64_sve_fp8_cvtn_x4", [IsOverloadNone, IsStreaming, SetsFPMR], []>; 2447} 2448 2449let SVETargetGuard = "sve2p1", SMETargetGuard = "sme2" in { 2450// == BFloat16 multiply-subtract == 2451 def SVBFMLSLB : SInst<"svbfmlslb[_{d}]", "dd$$", "f", MergeNone, "aarch64_sve_bfmlslb", [IsOverloadNone, VerifyRuntimeMode], []>; 2452 def SVBFMLSLT : SInst<"svbfmlslt[_{d}]", "dd$$", "f", MergeNone, "aarch64_sve_bfmlslt", [IsOverloadNone, VerifyRuntimeMode], []>; 2453 2454 def SVBFMLSLB_LANE : SInst<"svbfmlslb_lane[_{d}]", "dd$$i", "f", MergeNone, "aarch64_sve_bfmlslb_lane", [IsOverloadNone, VerifyRuntimeMode], [ImmCheck<3, ImmCheck0_7>]>; 2455 def SVBFMLSLT_LANE : SInst<"svbfmlslt_lane[_{d}]", "dd$$i", "f", MergeNone, "aarch64_sve_bfmlslt_lane", [IsOverloadNone, VerifyRuntimeMode], [ImmCheck<3, ImmCheck0_7>]>; 2456} 2457 2458let SVETargetGuard = "sve2,faminmax", SMETargetGuard = "sme2,faminmax" in { 2459 defm SVAMIN : SInstZPZZ<"svamin", "hfd", "aarch64_sve_famin", "aarch64_sve_famin_u">; 2460 defm SVAMAX : SInstZPZZ<"svamax", "hfd", "aarch64_sve_famax", "aarch64_sve_famax_u">; 2461} 2462 2463let SVETargetGuard = "sve2,fp8", SMETargetGuard = "sme2,fp8" in { 2464 // SVE FP8 widening conversions 2465 2466 // 8-bit floating-point convert to BFloat16/Float16 2467 def SVF1CVT : SInst<"svcvt1_{d}[_mf8]_fpm", "d~>", "bh", MergeNone, "aarch64_sve_fp8_cvt1", [VerifyRuntimeMode, SetsFPMR]>; 2468 def SVF2CVT : SInst<"svcvt2_{d}[_mf8]_fpm", "d~>", "bh", MergeNone, "aarch64_sve_fp8_cvt2", [VerifyRuntimeMode, SetsFPMR]>; 2469 2470 // 8-bit floating-point convert to BFloat16/Float16 (top) 2471 def SVF1CVTLT : SInst<"svcvtlt1_{d}[_mf8]_fpm", "d~>", "bh", MergeNone, "aarch64_sve_fp8_cvtlt1", [VerifyRuntimeMode, SetsFPMR]>; 2472 def SVF2CVTLT : SInst<"svcvtlt2_{d}[_mf8]_fpm", "d~>", "bh", MergeNone, "aarch64_sve_fp8_cvtlt2", [VerifyRuntimeMode, SetsFPMR]>; 2473 2474 // BFloat16/Float16 convert, narrow and interleave to 8-bit floating-point 2475 def SVFCVTN : SInst<"svcvtn_mf8[_{d}_x2]_fpm", "~2>", "bh", MergeNone, "aarch64_sve_fp8_cvtn", [VerifyRuntimeMode, SetsFPMR]>; 2476 2477 // Single-precision convert, narrow and interleave to 8-bit floating-point (top and bottom) 2478 def SVFCVTNB : SInst<"svcvtnb_mf8[_f32_x2]_fpm", "~2>", "f", MergeNone, "aarch64_sve_fp8_cvtnb", [VerifyRuntimeMode, SetsFPMR]>; 2479 def SVFCVTNT : SInst<"svcvtnt_mf8[_f32_x2]_fpm", "~~2>", "f", MergeNone, "aarch64_sve_fp8_cvtnt", [VerifyRuntimeMode, SetsFPMR]>; 2480} 2481 2482let SVETargetGuard = "sve2,fp8dot2", SMETargetGuard ="sme,ssve-fp8dot2" in { 2483 // 8-bit floating-point dot product to half-precision (vectors) 2484 def SVFDOT_2WAY : SInst<"svdot[_f16_mf8]_fpm", "dd~~>", "h", MergeNone, "aarch64_sve_fp8_fdot", [VerifyRuntimeMode, SetsFPMR]>; 2485 def SVFDOT_N_2WAY : SInst<"svdot[_n_f16_mf8]_fpm", "dd~!>", "h", MergeNone, "aarch64_sve_fp8_fdot", [VerifyRuntimeMode, SetsFPMR]>; 2486 2487 // 8-bit floating-point dot product to half-precision (indexed) 2488 def SVFDOT_LANE_2WAY : SInst<"svdot_lane[_f16_mf8]_fpm", "dd~~i>", "h", MergeNone, "aarch64_sve_fp8_fdot_lane", [VerifyRuntimeMode, SetsFPMR], [ImmCheck<3, ImmCheck0_7>]>; 2489} 2490 2491let SVETargetGuard = "sve2,fp8dot4", SMETargetGuard ="sme,ssve-fp8dot4" in { 2492 // 8-bit floating-point dot product to single-precision (vectors) 2493 def SVFDOT_4WAY : SInst<"svdot[_f32_mf8]_fpm", "dd~~>", "f", MergeNone, "aarch64_sve_fp8_fdot", [VerifyRuntimeMode, SetsFPMR]>; 2494 def SVFDOT_N_4WAY : SInst<"svdot[_n_f32_mf8]_fpm", "dd~!>", "f", MergeNone, "aarch64_sve_fp8_fdot", [VerifyRuntimeMode, SetsFPMR]>; 2495 2496 // 8-bit floating-point dot product to single-precision (indexed) 2497 def SVFDOT_LANE_4WAY : SInst<"svdot_lane[_f32_mf8]_fpm", "dd~~i>", "f", MergeNone, "aarch64_sve_fp8_fdot_lane", [VerifyRuntimeMode, SetsFPMR], [ImmCheck<3, ImmCheck0_3>]>; 2498} 2499 2500let SVETargetGuard = "sve2,fp8fma", SMETargetGuard = "sme,ssve-fp8fma" in { 2501 // 8-bit floating-point multiply-add long to half-precision (bottom) 2502 def SVFMLALB : SInst<"svmlalb[_f16_mf8]_fpm", "dd~~>", "h", MergeNone, "aarch64_sve_fp8_fmlalb", [VerifyRuntimeMode, SetsFPMR]>; 2503 def SVFMLALB_N : SInst<"svmlalb[_n_f16_mf8]_fpm", "dd~!>", "h", MergeNone, "aarch64_sve_fp8_fmlalb", [VerifyRuntimeMode, SetsFPMR]>; 2504 2505 // 8-bit floating-point multiply-add long to ha_fpmlf-precision (bottom, indexed) 2506 def SVFMLALB_LANE : SInst<"svmlalb_lane[_f16_mf8]_fpm", "dd~~i>", "h", MergeNone, "aarch64_sve_fp8_fmlalb_lane", [VerifyRuntimeMode, SetsFPMR], [ImmCheck<3, ImmCheck0_15>]>; 2507 2508 // 8-bit floating-point multiply-add long to half-precision (top) 2509 def SVFMLALT : SInst<"svmlalt[_f16_mf8]_fpm", "dd~~>", "h", MergeNone, "aarch64_sve_fp8_fmlalt", [VerifyRuntimeMode, SetsFPMR]>; 2510 def SVFMLALT_N : SInst<"svmlalt[_n_f16_mf8]_fpm", "dd~!>", "h", MergeNone, "aarch64_sve_fp8_fmlalt", [VerifyRuntimeMode, SetsFPMR]>; 2511 2512 // 8-bit floating-point multiply-add long to half-precision (top, indexed) 2513 def SVFMLALT_LANE : SInst<"svmlalt_lane[_f16_mf8]_fpm", "dd~~i>", "h", MergeNone, "aarch64_sve_fp8_fmlalt_lane", [VerifyRuntimeMode, SetsFPMR], [ImmCheck<3, ImmCheck0_15>]>; 2514 2515 // 8-bit floating-point multiply-add long long to single-precision (all top/bottom variants) 2516 def SVFMLALLBB : SInst<"svmlallbb[_f32_mf8]_fpm", "dd~~>", "f", MergeNone, "aarch64_sve_fp8_fmlallbb", [VerifyRuntimeMode, SetsFPMR]>; 2517 def SVFMLALLBB_N : SInst<"svmlallbb[_n_f32_mf8]_fpm", "dd~!>", "f", MergeNone, "aarch64_sve_fp8_fmlallbb", [VerifyRuntimeMode, SetsFPMR]>; 2518 def SVFMLALLBT : SInst<"svmlallbt[_f32_mf8]_fpm", "dd~~>", "f", MergeNone, "aarch64_sve_fp8_fmlallbt", [VerifyRuntimeMode, SetsFPMR]>; 2519 def SVFMLALLBT_N : SInst<"svmlallbt[_n_f32_mf8]_fpm", "dd~!>", "f", MergeNone, "aarch64_sve_fp8_fmlallbt", [VerifyRuntimeMode, SetsFPMR]>; 2520 def SVFMLALLTB : SInst<"svmlalltb[_f32_mf8]_fpm", "dd~~>", "f", MergeNone, "aarch64_sve_fp8_fmlalltb", [VerifyRuntimeMode, SetsFPMR]>; 2521 def SVFMLALLTB_N : SInst<"svmlalltb[_n_f32_mf8]_fpm", "dd~!>", "f", MergeNone, "aarch64_sve_fp8_fmlalltb", [VerifyRuntimeMode, SetsFPMR]>; 2522 def SVFMLALLTT : SInst<"svmlalltt[_f32_mf8]_fpm", "dd~~>", "f", MergeNone, "aarch64_sve_fp8_fmlalltt", [VerifyRuntimeMode, SetsFPMR]>; 2523 def SVFMLALLTT_N : SInst<"svmlalltt[_n_f32_mf8]_fpm", "dd~!>", "f", MergeNone, "aarch64_sve_fp8_fmlalltt", [VerifyRuntimeMode, SetsFPMR]>; 2524 2525 // 8-bit floating-point multiply-add long long to single-precision (indexed, all top/bottom variants) 2526 def SVFMLALLBB_LANE : SInst<"svmlallbb_lane[_f32_mf8]_fpm", "dd~~i>", "f", MergeNone, "aarch64_sve_fp8_fmlallbb_lane", [VerifyRuntimeMode, SetsFPMR], [ImmCheck<3, ImmCheck0_7>]>; 2527 def SVFMLALLBT_LANE : SInst<"svmlallbt_lane[_f32_mf8]_fpm", "dd~~i>", "f", MergeNone, "aarch64_sve_fp8_fmlallbt_lane", [VerifyRuntimeMode, SetsFPMR], [ImmCheck<3, ImmCheck0_7>]>; 2528 def SVFMLALLTB_LANE : SInst<"svmlalltb_lane[_f32_mf8]_fpm", "dd~~i>", "f", MergeNone, "aarch64_sve_fp8_fmlalltb_lane", [VerifyRuntimeMode, SetsFPMR], [ImmCheck<3, ImmCheck0_7>]>; 2529 def SVFMLALLTT_LANE : SInst<"svmlalltt_lane[_f32_mf8]_fpm", "dd~~i>", "f", MergeNone, "aarch64_sve_fp8_fmlalltt_lane", [VerifyRuntimeMode, SetsFPMR], [ImmCheck<3, ImmCheck0_7>]>; 2530} 2531