1 //===- PPCMacroFusion.cpp - PowerPC Macro Fusion --------------------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 /// \file This file contains the PowerPC implementation of the DAG scheduling 10 /// mutation to pair instructions back to back. 11 // 12 //===----------------------------------------------------------------------===// 13 14 #include "PPC.h" 15 #include "PPCSubtarget.h" 16 #include "llvm/ADT/DenseSet.h" 17 #include "llvm/CodeGen/MacroFusion.h" 18 #include "llvm/CodeGen/ScheduleDAGMutation.h" 19 #include <optional> 20 21 using namespace llvm; 22 namespace { 23 24 class FusionFeature { 25 public: 26 typedef SmallDenseSet<unsigned> FusionOpSet; 27 28 enum FusionKind { 29 #define FUSION_KIND(KIND) FK_##KIND 30 #define FUSION_FEATURE(KIND, HAS_FEATURE, DEP_OP_IDX, OPSET1, OPSET2) \ 31 FUSION_KIND(KIND), 32 #include "PPCMacroFusion.def" 33 FUSION_KIND(END) 34 }; 35 private: 36 // Each fusion feature is assigned with one fusion kind. All the 37 // instructions with the same fusion kind have the same fusion characteristic. 38 FusionKind Kd; 39 // True if this feature is enabled. 40 bool Supported; 41 // li rx, si 42 // load rt, ra, rx 43 // The dependent operand index in the second op(load). And the negative means 44 // it could be any one. 45 int DepOpIdx; 46 // The first fusion op set. 47 FusionOpSet OpSet1; 48 // The second fusion op set. 49 FusionOpSet OpSet2; 50 public: 51 FusionFeature(FusionKind Kind, bool HasFeature, int Index, 52 const FusionOpSet &First, const FusionOpSet &Second) : 53 Kd(Kind), Supported(HasFeature), DepOpIdx(Index), OpSet1(First), 54 OpSet2(Second) {} 55 56 bool hasOp1(unsigned Opc) const { return OpSet1.contains(Opc); } 57 bool hasOp2(unsigned Opc) const { return OpSet2.contains(Opc); } 58 bool isSupported() const { return Supported; } 59 std::optional<unsigned> depOpIdx() const { 60 if (DepOpIdx < 0) 61 return None; 62 return DepOpIdx; 63 } 64 65 FusionKind getKind() const { return Kd; } 66 }; 67 68 static bool matchingRegOps(const MachineInstr &FirstMI, 69 int FirstMIOpIndex, 70 const MachineInstr &SecondMI, 71 int SecondMIOpIndex) { 72 const MachineOperand &Op1 = FirstMI.getOperand(FirstMIOpIndex); 73 const MachineOperand &Op2 = SecondMI.getOperand(SecondMIOpIndex); 74 if (!Op1.isReg() || !Op2.isReg()) 75 return false; 76 77 return Op1.getReg() == Op2.getReg(); 78 } 79 80 static bool matchingImmOps(const MachineInstr &MI, 81 int MIOpIndex, 82 int64_t Expect, 83 unsigned ExtendFrom = 64) { 84 const MachineOperand &Op = MI.getOperand(MIOpIndex); 85 if (!Op.isImm()) 86 return false; 87 int64_t Imm = Op.getImm(); 88 if (ExtendFrom < 64) 89 Imm = SignExtend64(Imm, ExtendFrom); 90 return Imm == Expect; 91 } 92 93 // Return true if the FirstMI meets the constraints of SecondMI according to 94 // fusion specification. 95 static bool checkOpConstraints(FusionFeature::FusionKind Kd, 96 const MachineInstr &FirstMI, 97 const MachineInstr &SecondMI) { 98 switch (Kd) { 99 // The hardware didn't require any specific check for the fused instructions' 100 // operands. Therefore, return true to indicate that, it is fusable. 101 default: return true; 102 // [addi rt,ra,si - lxvd2x xt,ra,rb] etc. 103 case FusionFeature::FK_AddiLoad: { 104 // lxvd2x(ra) cannot be zero 105 const MachineOperand &RA = SecondMI.getOperand(1); 106 if (!RA.isReg()) 107 return true; 108 109 return Register::isVirtualRegister(RA.getReg()) || 110 (RA.getReg() != PPC::ZERO && RA.getReg() != PPC::ZERO8); 111 } 112 // [addis rt,ra,si - ld rt,ds(ra)] etc. 113 case FusionFeature::FK_AddisLoad: { 114 const MachineOperand &RT = SecondMI.getOperand(0); 115 if (!RT.isReg()) 116 return true; 117 118 // Only check it for non-virtual register. 119 if (!Register::isVirtualRegister(RT.getReg())) 120 // addis(rt) = ld(ra) = ld(rt) 121 // ld(rt) cannot be zero 122 if (!matchingRegOps(SecondMI, 0, SecondMI, 2) || 123 (RT.getReg() == PPC::ZERO || RT.getReg() == PPC::ZERO8)) 124 return false; 125 126 // addis(si) first 12 bits must be all 1s or all 0s 127 const MachineOperand &SI = FirstMI.getOperand(2); 128 if (!SI.isImm()) 129 return true; 130 int64_t Imm = SI.getImm(); 131 if (((Imm & 0xFFF0) != 0) && ((Imm & 0xFFF0) != 0xFFF0)) 132 return false; 133 134 // If si = 1111111111110000 and the msb of the d/ds field of the load equals 135 // 1, then fusion does not occur. 136 if ((Imm & 0xFFF0) == 0xFFF0) { 137 const MachineOperand &D = SecondMI.getOperand(1); 138 if (!D.isImm()) 139 return true; 140 141 // 14 bit for DS field, while 16 bit for D field. 142 int MSB = 15; 143 if (SecondMI.getOpcode() == PPC::LD) 144 MSB = 13; 145 146 return (D.getImm() & (1ULL << MSB)) == 0; 147 } 148 return true; 149 } 150 151 case FusionFeature::FK_SldiAdd: 152 return (matchingImmOps(FirstMI, 2, 3) && matchingImmOps(FirstMI, 3, 60)) || 153 (matchingImmOps(FirstMI, 2, 6) && matchingImmOps(FirstMI, 3, 57)); 154 155 // rldicl rx, ra, 1, 0 - xor 156 case FusionFeature::FK_RotateLeftXor: 157 return matchingImmOps(FirstMI, 2, 1) && matchingImmOps(FirstMI, 3, 0); 158 159 // rldicr rx, ra, 1, 63 - xor 160 case FusionFeature::FK_RotateRightXor: 161 return matchingImmOps(FirstMI, 2, 1) && matchingImmOps(FirstMI, 3, 63); 162 163 // We actually use CMPW* and CMPD*, 'l' doesn't exist as an operand in instr. 164 165 // { lbz,lbzx,lhz,lhzx,lwz,lwzx } - cmpi 0,1,rx,{ 0,1,-1 } 166 // { lbz,lbzx,lhz,lhzx,lwz,lwzx } - cmpli 0,L,rx,{ 0,1 } 167 case FusionFeature::FK_LoadCmp1: 168 // { ld,ldx } - cmpi 0,1,rx,{ 0,1,-1 } 169 // { ld,ldx } - cmpli 0,1,rx,{ 0,1 } 170 case FusionFeature::FK_LoadCmp2: { 171 const MachineOperand &BT = SecondMI.getOperand(0); 172 if (!BT.isReg() || 173 (!Register::isVirtualRegister(BT.getReg()) && BT.getReg() != PPC::CR0)) 174 return false; 175 if (SecondMI.getOpcode() == PPC::CMPDI && 176 matchingImmOps(SecondMI, 2, -1, 16)) 177 return true; 178 return matchingImmOps(SecondMI, 2, 0) || matchingImmOps(SecondMI, 2, 1); 179 } 180 181 // { lha,lhax,lwa,lwax } - cmpi 0,L,rx,{ 0,1,-1 } 182 case FusionFeature::FK_LoadCmp3: { 183 const MachineOperand &BT = SecondMI.getOperand(0); 184 if (!BT.isReg() || 185 (!Register::isVirtualRegister(BT.getReg()) && BT.getReg() != PPC::CR0)) 186 return false; 187 return matchingImmOps(SecondMI, 2, 0) || matchingImmOps(SecondMI, 2, 1) || 188 matchingImmOps(SecondMI, 2, -1, 16); 189 } 190 191 // mtctr - { bcctr,bcctrl } 192 case FusionFeature::FK_ZeroMoveCTR: 193 // ( mtctr rx ) is alias of ( mtspr 9, rx ) 194 return (FirstMI.getOpcode() != PPC::MTSPR && 195 FirstMI.getOpcode() != PPC::MTSPR8) || 196 matchingImmOps(FirstMI, 0, 9); 197 198 // mtlr - { bclr,bclrl } 199 case FusionFeature::FK_ZeroMoveLR: 200 // ( mtlr rx ) is alias of ( mtspr 8, rx ) 201 return (FirstMI.getOpcode() != PPC::MTSPR && 202 FirstMI.getOpcode() != PPC::MTSPR8) || 203 matchingImmOps(FirstMI, 0, 8); 204 205 // addis rx,ra,si - addi rt,rx,SI, SI >= 0 206 case FusionFeature::FK_AddisAddi: { 207 const MachineOperand &RA = FirstMI.getOperand(1); 208 const MachineOperand &SI = SecondMI.getOperand(2); 209 if (!SI.isImm() || !RA.isReg()) 210 return false; 211 if (RA.getReg() == PPC::ZERO || RA.getReg() == PPC::ZERO8) 212 return false; 213 return SignExtend64(SI.getImm(), 16) >= 0; 214 } 215 216 // addi rx,ra,si - addis rt,rx,SI, ra > 0, SI >= 2 217 case FusionFeature::FK_AddiAddis: { 218 const MachineOperand &RA = FirstMI.getOperand(1); 219 const MachineOperand &SI = FirstMI.getOperand(2); 220 if (!SI.isImm() || !RA.isReg()) 221 return false; 222 if (RA.getReg() == PPC::ZERO || RA.getReg() == PPC::ZERO8) 223 return false; 224 int64_t ExtendedSI = SignExtend64(SI.getImm(), 16); 225 return ExtendedSI >= 2; 226 } 227 } 228 229 llvm_unreachable("All the cases should have been handled"); 230 return true; 231 } 232 233 /// Check if the instr pair, FirstMI and SecondMI, should be fused together. 234 /// Given SecondMI, when FirstMI is unspecified, then check if SecondMI may be 235 /// part of a fused pair at all. 236 static bool shouldScheduleAdjacent(const TargetInstrInfo &TII, 237 const TargetSubtargetInfo &TSI, 238 const MachineInstr *FirstMI, 239 const MachineInstr &SecondMI) { 240 // We use the PPC namespace to avoid the need to prefix opcodes with PPC:: in 241 // the def file. 242 using namespace PPC; 243 244 const PPCSubtarget &ST = static_cast<const PPCSubtarget&>(TSI); 245 static const FusionFeature FusionFeatures[] = { 246 #define FUSION_FEATURE(KIND, HAS_FEATURE, DEP_OP_IDX, OPSET1, OPSET2) { \ 247 FusionFeature::FUSION_KIND(KIND), ST.HAS_FEATURE(), DEP_OP_IDX, { OPSET1 },\ 248 { OPSET2 } }, 249 #include "PPCMacroFusion.def" 250 }; 251 #undef FUSION_KIND 252 253 for (auto &Feature : FusionFeatures) { 254 // Skip if the feature is not supported. 255 if (!Feature.isSupported()) 256 continue; 257 258 // Only when the SecondMI is fusable, we are starting to look for the 259 // fusable FirstMI. 260 if (Feature.hasOp2(SecondMI.getOpcode())) { 261 // If FirstMI == nullptr, that means, we're only checking whether SecondMI 262 // can be fused at all. 263 if (!FirstMI) 264 return true; 265 266 // Checking if the FirstMI is fusable with the SecondMI. 267 if (!Feature.hasOp1(FirstMI->getOpcode())) 268 continue; 269 270 auto DepOpIdx = Feature.depOpIdx(); 271 if (DepOpIdx) { 272 // Checking if the result of the FirstMI is the desired operand of the 273 // SecondMI if the DepOpIdx is set. Otherwise, ignore it. 274 if (!matchingRegOps(*FirstMI, 0, SecondMI, *DepOpIdx)) 275 return false; 276 } 277 278 // Checking more on the instruction operands. 279 if (checkOpConstraints(Feature.getKind(), *FirstMI, SecondMI)) 280 return true; 281 } 282 } 283 284 return false; 285 } 286 287 } // end anonymous namespace 288 289 namespace llvm { 290 291 std::unique_ptr<ScheduleDAGMutation> createPowerPCMacroFusionDAGMutation () { 292 return createMacroFusionDAGMutation(shouldScheduleAdjacent); 293 } 294 295 } // end namespace llvm 296