1//=- AArch64Combine.td - Define AArch64 Combine Rules ---------*-tablegen -*-=// 2// 3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4// See https://llvm.org/LICENSE.txt for license information. 5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6// 7//===----------------------------------------------------------------------===// 8// 9// 10//===----------------------------------------------------------------------===// 11 12include "llvm/Target/GlobalISel/Combine.td" 13 14def fconstant_to_constant : GICombineRule< 15 (defs root:$root), 16 (match (G_FCONSTANT $dst, $src):$root, 17 [{ return matchFConstantToConstant(*${root}, MRI); }]), 18 (apply [{ applyFConstantToConstant(*${root}); }])>; 19 20def icmp_redundant_trunc_matchdata : GIDefMatchData<"Register">; 21def icmp_redundant_trunc : GICombineRule< 22 (defs root:$root, icmp_redundant_trunc_matchdata:$matchinfo), 23 (match (G_ICMP $dst, $tst, $src1, $src2):$root, 24 [{ return matchICmpRedundantTrunc(*${root}, MRI, Helper.getKnownBits(), ${matchinfo}); }]), 25 (apply [{ applyICmpRedundantTrunc(*${root}, MRI, B, Observer, ${matchinfo}); }])>; 26 27// AArch64-specific offset folding for G_GLOBAL_VALUE. 28def fold_global_offset_matchdata : GIDefMatchData<"std::pair<uint64_t, uint64_t>">; 29def fold_global_offset : GICombineRule< 30 (defs root:$root, fold_global_offset_matchdata:$matchinfo), 31 (match (wip_match_opcode G_GLOBAL_VALUE):$root, 32 [{ return matchFoldGlobalOffset(*${root}, MRI, ${matchinfo}); }]), 33 (apply [{ applyFoldGlobalOffset(*${root}, MRI, B, Observer, ${matchinfo});}]) 34>; 35 36// Boolean: 0 = G_ZEXT, 1 = G_SEXT 37def ext_addv_to_udot_addv_matchinfo : GIDefMatchData<"std::tuple<Register, Register, bool>">; 38let Predicates = [HasDotProd] in { 39def ext_addv_to_udot_addv : GICombineRule< 40 (defs root:$root, ext_addv_to_udot_addv_matchinfo:$matchinfo), 41 (match (wip_match_opcode G_VECREDUCE_ADD):$root, 42 [{ return matchExtAddvToUdotAddv(*${root}, MRI, STI, ${matchinfo}); }]), 43 (apply [{ applyExtAddvToUdotAddv(*${root}, MRI, B, Observer, STI, ${matchinfo}); }]) 44>; 45} 46 47def ext_uaddv_to_uaddlv_matchinfo : GIDefMatchData<"std::pair<Register, bool>">; 48def ext_uaddv_to_uaddlv : GICombineRule< 49 (defs root:$root, ext_uaddv_to_uaddlv_matchinfo:$matchinfo), 50 (match (wip_match_opcode G_VECREDUCE_ADD):$root, 51 [{ return matchExtUaddvToUaddlv(*${root}, MRI, ${matchinfo}); }]), 52 (apply [{ applyExtUaddvToUaddlv(*${root}, MRI, B, Observer, ${matchinfo}); }]) 53>; 54 55class push_opcode_through_ext<Instruction opcode, Instruction extOpcode> : GICombineRule < 56 (defs root:$root), 57 (match (extOpcode $ext1, $src1):$ExtMI, 58 (extOpcode $ext2, $src2), 59 (opcode $dst, $ext1, $ext2):$root, 60 [{ return matchPushAddSubExt(*${root}, MRI, ${dst}.getReg(), ${src1}.getReg(), ${src2}.getReg()); }]), 61 (apply [{ applyPushAddSubExt(*${root}, MRI, B, ${ExtMI}->getOpcode() == TargetOpcode::G_SEXT, ${dst}.getReg(), ${src1}.getReg(), ${src2}.getReg()); }])>; 62 63def push_sub_through_zext : push_opcode_through_ext<G_SUB, G_ZEXT>; 64def push_add_through_zext : push_opcode_through_ext<G_ADD, G_ZEXT>; 65def push_sub_through_sext : push_opcode_through_ext<G_SUB, G_SEXT>; 66def push_add_through_sext : push_opcode_through_ext<G_ADD, G_SEXT>; 67 68def AArch64PreLegalizerCombiner: GICombiner< 69 "AArch64PreLegalizerCombinerImpl", [all_combines, 70 fconstant_to_constant, 71 icmp_redundant_trunc, 72 fold_global_offset, 73 shuffle_to_extract, 74 ext_addv_to_udot_addv, 75 ext_uaddv_to_uaddlv, 76 push_sub_through_zext, 77 push_add_through_zext, 78 push_sub_through_sext, 79 push_add_through_sext]> { 80 let CombineAllMethodName = "tryCombineAllImpl"; 81} 82 83def AArch64O0PreLegalizerCombiner: GICombiner< 84 "AArch64O0PreLegalizerCombinerImpl", [optnone_combines]> { 85 let CombineAllMethodName = "tryCombineAllImpl"; 86} 87 88// Matchdata for combines which replace a G_SHUFFLE_VECTOR with a 89// target-specific opcode. 90def shuffle_matchdata : GIDefMatchData<"ShuffleVectorPseudo">; 91 92def rev : GICombineRule< 93 (defs root:$root, shuffle_matchdata:$matchinfo), 94 (match (wip_match_opcode G_SHUFFLE_VECTOR):$root, 95 [{ return matchREV(*${root}, MRI, ${matchinfo}); }]), 96 (apply [{ applyShuffleVectorPseudo(*${root}, ${matchinfo}); }]) 97>; 98 99def zip : GICombineRule< 100 (defs root:$root, shuffle_matchdata:$matchinfo), 101 (match (wip_match_opcode G_SHUFFLE_VECTOR):$root, 102 [{ return matchZip(*${root}, MRI, ${matchinfo}); }]), 103 (apply [{ applyShuffleVectorPseudo(*${root}, ${matchinfo}); }]) 104>; 105 106def uzp : GICombineRule< 107 (defs root:$root, shuffle_matchdata:$matchinfo), 108 (match (wip_match_opcode G_SHUFFLE_VECTOR):$root, 109 [{ return matchUZP(*${root}, MRI, ${matchinfo}); }]), 110 (apply [{ applyShuffleVectorPseudo(*${root}, ${matchinfo}); }]) 111>; 112 113def dup: GICombineRule < 114 (defs root:$root, shuffle_matchdata:$matchinfo), 115 (match (wip_match_opcode G_SHUFFLE_VECTOR):$root, 116 [{ return matchDup(*${root}, MRI, ${matchinfo}); }]), 117 (apply [{ applyShuffleVectorPseudo(*${root}, ${matchinfo}); }]) 118>; 119 120def trn : GICombineRule< 121 (defs root:$root, shuffle_matchdata:$matchinfo), 122 (match (wip_match_opcode G_SHUFFLE_VECTOR):$root, 123 [{ return matchTRN(*${root}, MRI, ${matchinfo}); }]), 124 (apply [{ applyShuffleVectorPseudo(*${root}, ${matchinfo}); }]) 125>; 126 127def ext: GICombineRule < 128 (defs root:$root, shuffle_matchdata:$matchinfo), 129 (match (wip_match_opcode G_SHUFFLE_VECTOR):$root, 130 [{ return matchEXT(*${root}, MRI, ${matchinfo}); }]), 131 (apply [{ applyEXT(*${root}, ${matchinfo}); }]) 132>; 133 134def fullrev: GICombineRule < 135 (defs root:$root, shuffle_matchdata:$matchinfo), 136 (match (G_IMPLICIT_DEF $src2), 137 (G_SHUFFLE_VECTOR $src, $src1, $src2, $mask):$root, 138 [{ return ShuffleVectorInst::isReverseMask(${mask}.getShuffleMask(), 139 ${mask}.getShuffleMask().size()); }]), 140 (apply [{ applyFullRev(*${root}, MRI); }]) 141>; 142 143def insertelt_nonconst: GICombineRule < 144 (defs root:$root, shuffle_matchdata:$matchinfo), 145 (match (wip_match_opcode G_INSERT_VECTOR_ELT):$root, 146 [{ return matchNonConstInsert(*${root}, MRI); }]), 147 (apply [{ applyNonConstInsert(*${root}, MRI, B); }]) 148>; 149 150def shuf_to_ins_matchdata : GIDefMatchData<"std::tuple<Register, int, Register, int>">; 151def shuf_to_ins: GICombineRule < 152 (defs root:$root, shuf_to_ins_matchdata:$matchinfo), 153 (match (wip_match_opcode G_SHUFFLE_VECTOR):$root, 154 [{ return matchINS(*${root}, MRI, ${matchinfo}); }]), 155 (apply [{ applyINS(*${root}, MRI, B, ${matchinfo}); }]) 156>; 157 158def vashr_vlshr_imm_matchdata : GIDefMatchData<"int64_t">; 159def vashr_vlshr_imm : GICombineRule< 160 (defs root:$root, vashr_vlshr_imm_matchdata:$matchinfo), 161 (match (wip_match_opcode G_ASHR, G_LSHR):$root, 162 [{ return matchVAshrLshrImm(*${root}, MRI, ${matchinfo}); }]), 163 (apply [{ applyVAshrLshrImm(*${root}, MRI, ${matchinfo}); }]) 164>; 165 166def form_duplane_matchdata : 167 GIDefMatchData<"std::pair<unsigned, int>">; 168def form_duplane : GICombineRule < 169 (defs root:$root, form_duplane_matchdata:$matchinfo), 170 (match (wip_match_opcode G_SHUFFLE_VECTOR):$root, 171 [{ return matchDupLane(*${root}, MRI, ${matchinfo}); }]), 172 (apply [{ applyDupLane(*${root}, MRI, B, ${matchinfo}); }]) 173>; 174 175def shuffle_vector_lowering : GICombineGroup<[dup, rev, ext, zip, uzp, trn, fullrev, 176 form_duplane, shuf_to_ins]>; 177 178// Turn G_UNMERGE_VALUES -> G_EXTRACT_VECTOR_ELT's 179def vector_unmerge_lowering : GICombineRule < 180 (defs root:$root), 181 (match (wip_match_opcode G_UNMERGE_VALUES):$root, 182 [{ return matchScalarizeVectorUnmerge(*${root}, MRI); }]), 183 (apply [{ applyScalarizeVectorUnmerge(*${root}, MRI, B); }]) 184>; 185 186def adjust_icmp_imm_matchdata : 187 GIDefMatchData<"std::pair<uint64_t, CmpInst::Predicate>">; 188def adjust_icmp_imm : GICombineRule < 189 (defs root:$root, adjust_icmp_imm_matchdata:$matchinfo), 190 (match (G_ICMP $dst, $tst, $src1, $src2):$root, 191 [{ return matchAdjustICmpImmAndPred(*${root}, MRI, ${matchinfo}); }]), 192 (apply [{ applyAdjustICmpImmAndPred(*${root}, ${matchinfo}, B, Observer); }]) 193>; 194 195def swap_icmp_operands : GICombineRule < 196 (defs root:$root), 197 (match (G_ICMP $dst, $tst, $src1, $src2):$root, 198 [{ return trySwapICmpOperands(*${root}, MRI); }]), 199 (apply [{ applySwapICmpOperands(*${root}, Observer); }]) 200>; 201 202def icmp_lowering : GICombineGroup<[adjust_icmp_imm, swap_icmp_operands]>; 203 204def extractvecelt_pairwise_add_matchdata : GIDefMatchData<"std::tuple<unsigned, LLT, Register>">; 205def extractvecelt_pairwise_add : GICombineRule< 206 (defs root:$root, extractvecelt_pairwise_add_matchdata:$matchinfo), 207 (match (wip_match_opcode G_EXTRACT_VECTOR_ELT):$root, 208 [{ return matchExtractVecEltPairwiseAdd(*${root}, MRI, ${matchinfo}); }]), 209 (apply [{ applyExtractVecEltPairwiseAdd(*${root}, MRI, B, ${matchinfo}); }]) 210>; 211 212def mul_const_matchdata : GIDefMatchData<"std::function<void(MachineIRBuilder&, Register)>">; 213def mul_const : GICombineRule< 214 (defs root:$root, mul_const_matchdata:$matchinfo), 215 (match (wip_match_opcode G_MUL):$root, 216 [{ return matchAArch64MulConstCombine(*${root}, MRI, ${matchinfo}); }]), 217 (apply [{ applyAArch64MulConstCombine(*${root}, MRI, B, ${matchinfo}); }]) 218>; 219 220def lower_mull : GICombineRule< 221 (defs root:$root), 222 (match (wip_match_opcode G_MUL):$root, 223 [{ return matchExtMulToMULL(*${root}, MRI); }]), 224 (apply [{ applyExtMulToMULL(*${root}, MRI, B, Observer); }]) 225>; 226 227def build_vector_to_dup : GICombineRule< 228 (defs root:$root), 229 (match (wip_match_opcode G_BUILD_VECTOR):$root, 230 [{ return matchBuildVectorToDup(*${root}, MRI); }]), 231 (apply [{ applyBuildVectorToDup(*${root}, MRI, B); }]) 232>; 233 234def build_vector_to_vector_insert : GICombineRule< 235 (defs root:$root), 236 (match (G_BUILD_VECTOR $dst, GIVariadic<>:$unused):$root, 237 [{ return matchLowerBuildToInsertVecElt(*${root}, MRI); }]), 238 (apply [{ applyLowerBuildToInsertVecElt(*${root}, MRI, B); }]) 239>; 240 241def build_vector_lowering : GICombineGroup<[build_vector_to_dup, 242 build_vector_to_vector_insert]>; 243 244def lower_vector_fcmp : GICombineRule< 245 (defs root:$root), 246 (match (G_FCMP $dst, $tst, $src1, $src2):$root, 247 [{ return matchLowerVectorFCMP(*${root}, MRI, B); }]), 248 (apply [{ applyLowerVectorFCMP(*${root}, MRI, B); }])>; 249 250def form_truncstore_matchdata : GIDefMatchData<"Register">; 251def form_truncstore : GICombineRule< 252 (defs root:$root, form_truncstore_matchdata:$matchinfo), 253 (match (G_STORE $src, $addr):$root, 254 [{ return matchFormTruncstore(*${root}, MRI, ${matchinfo}); }]), 255 (apply [{ applyFormTruncstore(*${root}, MRI, B, Observer, ${matchinfo}); }]) 256>; 257 258def fold_merge_to_zext : GICombineRule< 259 (defs root:$d), 260 (match (wip_match_opcode G_MERGE_VALUES):$d, 261 [{ return matchFoldMergeToZext(*${d}, MRI); }]), 262 (apply [{ applyFoldMergeToZext(*${d}, MRI, B, Observer); }]) 263>; 264 265def mutate_anyext_to_zext : GICombineRule< 266 (defs root:$d), 267 (match (wip_match_opcode G_ANYEXT):$d, 268 [{ return matchMutateAnyExtToZExt(*${d}, MRI); }]), 269 (apply [{ applyMutateAnyExtToZExt(*${d}, MRI, B, Observer); }]) 270>; 271 272def split_store_zero_128 : GICombineRule< 273 (defs root:$d), 274 (match (G_STORE $src, $addr):$d, 275 [{ return matchSplitStoreZero128(*${d}, MRI); }]), 276 (apply [{ applySplitStoreZero128(*${d}, MRI, B, Observer); }]) 277>; 278 279def vector_sext_inreg_to_shift : GICombineRule< 280 (defs root:$d), 281 (match (wip_match_opcode G_SEXT_INREG):$d, 282 [{ return matchVectorSextInReg(*${d}, MRI); }]), 283 (apply [{ applyVectorSextInReg(*${d}, MRI, B, Observer); }]) 284>; 285 286def unmerge_ext_to_unmerge_matchdata : GIDefMatchData<"Register">; 287def unmerge_ext_to_unmerge : GICombineRule< 288 (defs root:$d, unmerge_ext_to_unmerge_matchdata:$matchinfo), 289 (match (wip_match_opcode G_UNMERGE_VALUES):$d, 290 [{ return matchUnmergeExtToUnmerge(*${d}, MRI, ${matchinfo}); }]), 291 (apply [{ applyUnmergeExtToUnmerge(*${d}, MRI, B, Observer, ${matchinfo}); }]) 292>; 293 294def regtriple_matchdata : GIDefMatchData<"std::tuple<Register, Register, Register>">; 295def or_to_bsp: GICombineRule < 296 (defs root:$root, regtriple_matchdata:$matchinfo), 297 (match (G_OR $dst, $src1, $src2):$root, 298 [{ return matchOrToBSP(*${root}, MRI, ${matchinfo}); }]), 299 (apply [{ applyOrToBSP(*${root}, MRI, B, ${matchinfo}); }]) 300>; 301 302// Combines Mul(And(Srl(X, 15), 0x10001), 0xffff) into CMLTz 303def combine_mul_cmlt : GICombineRule< 304 (defs root:$root, register_matchinfo:$matchinfo), 305 (match (wip_match_opcode G_MUL):$root, 306 [{ return matchCombineMulCMLT(*${root}, MRI, ${matchinfo}); }]), 307 (apply [{ applyCombineMulCMLT(*${root}, MRI, B, ${matchinfo}); }]) 308>; 309 310// Post-legalization combines which should happen at all optimization levels. 311// (E.g. ones that facilitate matching for the selector) For example, matching 312// pseudos. 313def AArch64PostLegalizerLowering 314 : GICombiner<"AArch64PostLegalizerLoweringImpl", 315 [shuffle_vector_lowering, vashr_vlshr_imm, 316 icmp_lowering, build_vector_lowering, 317 lower_vector_fcmp, form_truncstore, 318 vector_sext_inreg_to_shift, 319 unmerge_ext_to_unmerge, lower_mull, 320 vector_unmerge_lowering, insertelt_nonconst]> { 321} 322 323// Post-legalization combines which are primarily optimizations. 324def AArch64PostLegalizerCombiner 325 : GICombiner<"AArch64PostLegalizerCombinerImpl", 326 [copy_prop, cast_of_cast_combines, buildvector_of_truncate, 327 integer_of_truncate, mutate_anyext_to_zext, 328 combines_for_extload, combine_indexed_load_store, sext_trunc_sextload, 329 hoist_logic_op_with_same_opcode_hands, 330 redundant_and, xor_of_and_with_same_reg, 331 extractvecelt_pairwise_add, redundant_or, 332 mul_const, redundant_sext_inreg, 333 form_bitfield_extract, rotate_out_of_range, 334 icmp_to_true_false_known_bits, overflow_combines, 335 select_combines, fold_merge_to_zext, merge_combines, 336 constant_fold_binops, identity_combines, 337 ptr_add_immed_chain, overlapping_and, 338 split_store_zero_128, undef_combines, 339 select_to_minmax, or_to_bsp, combine_concat_vector, 340 commute_constant_to_rhs, 341 push_freeze_to_prevent_poison_from_propagating, 342 combine_mul_cmlt, combine_use_vector_truncate]> { 343} 344