1 //===-- R600EmitClauseMarkers.cpp - Emit CF_ALU ---------------------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 /// \file 10 /// Add CF_ALU. R600 Alu instructions are grouped in clause which can hold 11 /// 128 Alu instructions ; these instructions can access up to 4 prefetched 12 /// 4 lines of 16 registers from constant buffers. Such ALU clauses are 13 /// initiated by CF_ALU instructions. 14 //===----------------------------------------------------------------------===// 15 16 #include "MCTargetDesc/R600MCTargetDesc.h" 17 #include "R600.h" 18 #include "R600Defines.h" 19 #include "R600Subtarget.h" 20 #include "llvm/CodeGen/MachineFunctionPass.h" 21 22 using namespace llvm; 23 24 namespace llvm { 25 26 void initializeR600EmitClauseMarkersPass(PassRegistry&); 27 28 } // end namespace llvm 29 30 namespace { 31 32 class R600EmitClauseMarkers : public MachineFunctionPass { 33 private: 34 const R600InstrInfo *TII = nullptr; 35 int Address = 0; 36 37 unsigned OccupiedDwords(MachineInstr &MI) const { 38 switch (MI.getOpcode()) { 39 case R600::INTERP_PAIR_XY: 40 case R600::INTERP_PAIR_ZW: 41 case R600::INTERP_VEC_LOAD: 42 case R600::DOT_4: 43 return 4; 44 case R600::KILL: 45 return 0; 46 default: 47 break; 48 } 49 50 // These will be expanded to two ALU instructions in the 51 // ExpandSpecialInstructions pass. 52 if (TII->isLDSRetInstr(MI.getOpcode())) 53 return 2; 54 55 if (TII->isVector(MI) || TII->isCubeOp(MI.getOpcode()) || 56 TII->isReductionOp(MI.getOpcode())) 57 return 4; 58 59 unsigned NumLiteral = 0; 60 for (MachineInstr::mop_iterator It = MI.operands_begin(), 61 E = MI.operands_end(); 62 It != E; ++It) { 63 MachineOperand &MO = *It; 64 if (MO.isReg() && MO.getReg() == R600::ALU_LITERAL_X) 65 ++NumLiteral; 66 } 67 return 1 + NumLiteral; 68 } 69 70 bool isALU(const MachineInstr &MI) const { 71 if (TII->isALUInstr(MI.getOpcode())) 72 return true; 73 if (TII->isVector(MI) || TII->isCubeOp(MI.getOpcode())) 74 return true; 75 switch (MI.getOpcode()) { 76 case R600::PRED_X: 77 case R600::INTERP_PAIR_XY: 78 case R600::INTERP_PAIR_ZW: 79 case R600::INTERP_VEC_LOAD: 80 case R600::COPY: 81 case R600::DOT_4: 82 return true; 83 default: 84 return false; 85 } 86 } 87 88 bool IsTrivialInst(MachineInstr &MI) const { 89 switch (MI.getOpcode()) { 90 case R600::KILL: 91 case R600::RETURN: 92 case R600::IMPLICIT_DEF: 93 return true; 94 default: 95 return false; 96 } 97 } 98 99 std::pair<unsigned, unsigned> getAccessedBankLine(unsigned Sel) const { 100 // Sel is (512 + (kc_bank << 12) + ConstIndex) << 2 101 // (See also R600ISelLowering.cpp) 102 // ConstIndex value is in [0, 4095]; 103 return std::pair<unsigned, unsigned>( 104 ((Sel >> 2) - 512) >> 12, // KC_BANK 105 // Line Number of ConstIndex 106 // A line contains 16 constant registers however KCX bank can lock 107 // two line at the same time ; thus we want to get an even line number. 108 // Line number can be retrieved with (>>4), using (>>5) <<1 generates 109 // an even number. 110 ((((Sel >> 2) - 512) & 4095) >> 5) << 1); 111 } 112 113 bool 114 SubstituteKCacheBank(MachineInstr &MI, 115 std::vector<std::pair<unsigned, unsigned>> &CachedConsts, 116 bool UpdateInstr = true) const { 117 std::vector<std::pair<unsigned, unsigned>> UsedKCache; 118 119 if (!TII->isALUInstr(MI.getOpcode()) && MI.getOpcode() != R600::DOT_4) 120 return true; 121 122 const SmallVectorImpl<std::pair<MachineOperand *, int64_t>> &Consts = 123 TII->getSrcs(MI); 124 assert( 125 (TII->isALUInstr(MI.getOpcode()) || MI.getOpcode() == R600::DOT_4) && 126 "Can't assign Const"); 127 for (auto &[Op, Sel] : Consts) { 128 if (Op->getReg() != R600::ALU_CONST) 129 continue; 130 unsigned Chan = Sel & 3, Index = ((Sel >> 2) - 512) & 31; 131 unsigned KCacheIndex = Index * 4 + Chan; 132 const std::pair<unsigned, unsigned> &BankLine = getAccessedBankLine(Sel); 133 if (CachedConsts.empty()) { 134 CachedConsts.push_back(BankLine); 135 UsedKCache.emplace_back(0, KCacheIndex); 136 continue; 137 } 138 if (CachedConsts[0] == BankLine) { 139 UsedKCache.emplace_back(0, KCacheIndex); 140 continue; 141 } 142 if (CachedConsts.size() == 1) { 143 CachedConsts.push_back(BankLine); 144 UsedKCache.emplace_back(1, KCacheIndex); 145 continue; 146 } 147 if (CachedConsts[1] == BankLine) { 148 UsedKCache.emplace_back(1, KCacheIndex); 149 continue; 150 } 151 return false; 152 } 153 154 if (!UpdateInstr) 155 return true; 156 157 unsigned j = 0; 158 for (auto &[Op, Sel] : Consts) { 159 if (Op->getReg() != R600::ALU_CONST) 160 continue; 161 switch (UsedKCache[j].first) { 162 case 0: 163 Op->setReg(R600::R600_KC0RegClass.getRegister(UsedKCache[j].second)); 164 break; 165 case 1: 166 Op->setReg(R600::R600_KC1RegClass.getRegister(UsedKCache[j].second)); 167 break; 168 default: 169 llvm_unreachable("Wrong Cache Line"); 170 } 171 j++; 172 } 173 return true; 174 } 175 176 bool canClauseLocalKillFitInClause( 177 unsigned AluInstCount, 178 std::vector<std::pair<unsigned, unsigned>> KCacheBanks, 179 MachineBasicBlock::iterator Def, 180 MachineBasicBlock::iterator BBEnd) { 181 const R600RegisterInfo &TRI = TII->getRegisterInfo(); 182 //TODO: change this to defs? 183 for (MachineOperand &MO : Def->all_defs()) { 184 if (TRI.isPhysRegLiveAcrossClauses(MO.getReg())) 185 continue; 186 187 // Def defines a clause local register, so check that its use will fit 188 // in the clause. 189 unsigned LastUseCount = 0; 190 for (MachineBasicBlock::iterator UseI = Def; UseI != BBEnd; ++UseI) { 191 AluInstCount += OccupiedDwords(*UseI); 192 // Make sure we won't need to end the clause due to KCache limitations. 193 if (!SubstituteKCacheBank(*UseI, KCacheBanks, false)) 194 return false; 195 196 // We have reached the maximum instruction limit before finding the 197 // use that kills this register, so we cannot use this def in the 198 // current clause. 199 if (AluInstCount >= TII->getMaxAlusPerClause()) 200 return false; 201 202 // TODO: Is this true? kill flag appears to work OK below 203 // Register kill flags have been cleared by the time we get to this 204 // pass, but it is safe to assume that all uses of this register 205 // occur in the same basic block as its definition, because 206 // it is illegal for the scheduler to schedule them in 207 // different blocks. 208 if (UseI->readsRegister(MO.getReg(), &TRI)) 209 LastUseCount = AluInstCount; 210 211 // Exit early if the current use kills the register 212 if (UseI != Def && UseI->killsRegister(MO.getReg(), &TRI)) 213 break; 214 } 215 if (LastUseCount) 216 return LastUseCount <= TII->getMaxAlusPerClause(); 217 llvm_unreachable("Clause local register live at end of clause."); 218 } 219 return true; 220 } 221 222 MachineBasicBlock::iterator 223 MakeALUClause(MachineBasicBlock &MBB, MachineBasicBlock::iterator I) { 224 MachineBasicBlock::iterator ClauseHead = I; 225 std::vector<std::pair<unsigned, unsigned>> KCacheBanks; 226 bool PushBeforeModifier = false; 227 unsigned AluInstCount = 0; 228 for (MachineBasicBlock::iterator E = MBB.end(); I != E; ++I) { 229 if (IsTrivialInst(*I)) 230 continue; 231 if (!isALU(*I)) 232 break; 233 if (AluInstCount > TII->getMaxAlusPerClause()) 234 break; 235 if (I->getOpcode() == R600::PRED_X) { 236 // We put PRED_X in its own clause to ensure that ifcvt won't create 237 // clauses with more than 128 insts. 238 // IfCvt is indeed checking that "then" and "else" branches of an if 239 // statement have less than ~60 insts thus converted clauses can't be 240 // bigger than ~121 insts (predicate setter needs to be in the same 241 // clause as predicated alus). 242 if (AluInstCount > 0) 243 break; 244 if (TII->getFlagOp(*I).getImm() & MO_FLAG_PUSH) 245 PushBeforeModifier = true; 246 AluInstCount ++; 247 continue; 248 } 249 // XXX: GROUP_BARRIER instructions cannot be in the same ALU clause as: 250 // 251 // * KILL or INTERP instructions 252 // * Any instruction that sets UPDATE_EXEC_MASK or UPDATE_PRED bits 253 // * Uses waterfalling (i.e. INDEX_MODE = AR.X) 254 // 255 // XXX: These checks have not been implemented yet. 256 if (TII->mustBeLastInClause(I->getOpcode())) { 257 I++; 258 break; 259 } 260 261 // If this instruction defines a clause local register, make sure 262 // its use can fit in this clause. 263 if (!canClauseLocalKillFitInClause(AluInstCount, KCacheBanks, I, E)) 264 break; 265 266 if (!SubstituteKCacheBank(*I, KCacheBanks)) 267 break; 268 AluInstCount += OccupiedDwords(*I); 269 } 270 unsigned Opcode = PushBeforeModifier ? 271 R600::CF_ALU_PUSH_BEFORE : R600::CF_ALU; 272 BuildMI(MBB, ClauseHead, MBB.findDebugLoc(ClauseHead), TII->get(Opcode)) 273 // We don't use the ADDR field until R600ControlFlowFinalizer pass, where 274 // it is safe to assume it is 0. However if we always put 0 here, the ifcvt 275 // pass may assume that identical ALU clause starter at the beginning of a 276 // true and false branch can be factorized which is not the case. 277 .addImm(Address++) // ADDR 278 .addImm(KCacheBanks.empty()?0:KCacheBanks[0].first) // KB0 279 .addImm((KCacheBanks.size() < 2)?0:KCacheBanks[1].first) // KB1 280 .addImm(KCacheBanks.empty()?0:2) // KM0 281 .addImm((KCacheBanks.size() < 2)?0:2) // KM1 282 .addImm(KCacheBanks.empty()?0:KCacheBanks[0].second) // KLINE0 283 .addImm((KCacheBanks.size() < 2)?0:KCacheBanks[1].second) // KLINE1 284 .addImm(AluInstCount) // COUNT 285 .addImm(1); // Enabled 286 return I; 287 } 288 289 public: 290 static char ID; 291 292 R600EmitClauseMarkers() : MachineFunctionPass(ID) { 293 initializeR600EmitClauseMarkersPass(*PassRegistry::getPassRegistry()); 294 } 295 296 bool runOnMachineFunction(MachineFunction &MF) override { 297 const R600Subtarget &ST = MF.getSubtarget<R600Subtarget>(); 298 TII = ST.getInstrInfo(); 299 300 for (MachineBasicBlock &MBB : MF) { 301 MachineBasicBlock::iterator I = MBB.begin(); 302 if (I != MBB.end() && I->getOpcode() == R600::CF_ALU) 303 continue; // BB was already parsed 304 for (MachineBasicBlock::iterator E = MBB.end(); I != E;) { 305 if (isALU(*I)) { 306 auto next = MakeALUClause(MBB, I); 307 assert(next != I); 308 I = next; 309 } else 310 ++I; 311 } 312 } 313 return false; 314 } 315 316 StringRef getPassName() const override { 317 return "R600 Emit Clause Markers Pass"; 318 } 319 }; 320 321 char R600EmitClauseMarkers::ID = 0; 322 323 } // end anonymous namespace 324 325 INITIALIZE_PASS_BEGIN(R600EmitClauseMarkers, "emitclausemarkers", 326 "R600 Emit Clause Markers", false, false) 327 INITIALIZE_PASS_END(R600EmitClauseMarkers, "emitclausemarkers", 328 "R600 Emit Clause Markers", false, false) 329 330 FunctionPass *llvm::createR600EmitClauseMarkers() { 331 return new R600EmitClauseMarkers(); 332 } 333