1 //===- SIFixSGPRCopies.cpp - Remove potential VGPR => SGPR copies ---------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 /// \file 10 /// Copies from VGPR to SGPR registers are illegal and the register coalescer 11 /// will sometimes generate these illegal copies in situations like this: 12 /// 13 /// Register Class <vsrc> is the union of <vgpr> and <sgpr> 14 /// 15 /// BB0: 16 /// %0 <sgpr> = SCALAR_INST 17 /// %1 <vsrc> = COPY %0 <sgpr> 18 /// ... 19 /// BRANCH %cond BB1, BB2 20 /// BB1: 21 /// %2 <vgpr> = VECTOR_INST 22 /// %3 <vsrc> = COPY %2 <vgpr> 23 /// BB2: 24 /// %4 <vsrc> = PHI %1 <vsrc>, <%bb.0>, %3 <vrsc>, <%bb.1> 25 /// %5 <vgpr> = VECTOR_INST %4 <vsrc> 26 /// 27 /// 28 /// The coalescer will begin at BB0 and eliminate its copy, then the resulting 29 /// code will look like this: 30 /// 31 /// BB0: 32 /// %0 <sgpr> = SCALAR_INST 33 /// ... 34 /// BRANCH %cond BB1, BB2 35 /// BB1: 36 /// %2 <vgpr> = VECTOR_INST 37 /// %3 <vsrc> = COPY %2 <vgpr> 38 /// BB2: 39 /// %4 <sgpr> = PHI %0 <sgpr>, <%bb.0>, %3 <vsrc>, <%bb.1> 40 /// %5 <vgpr> = VECTOR_INST %4 <sgpr> 41 /// 42 /// Now that the result of the PHI instruction is an SGPR, the register 43 /// allocator is now forced to constrain the register class of %3 to 44 /// <sgpr> so we end up with final code like this: 45 /// 46 /// BB0: 47 /// %0 <sgpr> = SCALAR_INST 48 /// ... 49 /// BRANCH %cond BB1, BB2 50 /// BB1: 51 /// %2 <vgpr> = VECTOR_INST 52 /// %3 <sgpr> = COPY %2 <vgpr> 53 /// BB2: 54 /// %4 <sgpr> = PHI %0 <sgpr>, <%bb.0>, %3 <sgpr>, <%bb.1> 55 /// %5 <vgpr> = VECTOR_INST %4 <sgpr> 56 /// 57 /// Now this code contains an illegal copy from a VGPR to an SGPR. 58 /// 59 /// In order to avoid this problem, this pass searches for PHI instructions 60 /// which define a <vsrc> register and constrains its definition class to 61 /// <vgpr> if the user of the PHI's definition register is a vector instruction. 62 /// If the PHI's definition class is constrained to <vgpr> then the coalescer 63 /// will be unable to perform the COPY removal from the above example which 64 /// ultimately led to the creation of an illegal COPY. 65 //===----------------------------------------------------------------------===// 66 67 #include "AMDGPU.h" 68 #include "AMDGPUSubtarget.h" 69 #include "SIInstrInfo.h" 70 #include "SIRegisterInfo.h" 71 #include "MCTargetDesc/AMDGPUMCTargetDesc.h" 72 #include "llvm/ADT/DenseSet.h" 73 #include "llvm/ADT/STLExtras.h" 74 #include "llvm/ADT/SmallSet.h" 75 #include "llvm/ADT/SmallVector.h" 76 #include "llvm/CodeGen/MachineBasicBlock.h" 77 #include "llvm/CodeGen/MachineDominators.h" 78 #include "llvm/CodeGen/MachineFunction.h" 79 #include "llvm/CodeGen/MachineFunctionPass.h" 80 #include "llvm/CodeGen/MachineInstr.h" 81 #include "llvm/CodeGen/MachineInstrBuilder.h" 82 #include "llvm/CodeGen/MachineOperand.h" 83 #include "llvm/CodeGen/MachineRegisterInfo.h" 84 #include "llvm/CodeGen/TargetRegisterInfo.h" 85 #include "llvm/Pass.h" 86 #include "llvm/Support/CodeGen.h" 87 #include "llvm/Support/CommandLine.h" 88 #include "llvm/Support/Debug.h" 89 #include "llvm/Support/raw_ostream.h" 90 #include "llvm/Target/TargetMachine.h" 91 #include <cassert> 92 #include <cstdint> 93 #include <iterator> 94 #include <list> 95 #include <map> 96 #include <tuple> 97 #include <utility> 98 99 using namespace llvm; 100 101 #define DEBUG_TYPE "si-fix-sgpr-copies" 102 103 static cl::opt<bool> EnableM0Merge( 104 "amdgpu-enable-merge-m0", 105 cl::desc("Merge and hoist M0 initializations"), 106 cl::init(true)); 107 108 namespace { 109 110 class SIFixSGPRCopies : public MachineFunctionPass { 111 MachineDominatorTree *MDT; 112 113 public: 114 static char ID; 115 116 MachineRegisterInfo *MRI; 117 const SIRegisterInfo *TRI; 118 const SIInstrInfo *TII; 119 120 SIFixSGPRCopies() : MachineFunctionPass(ID) {} 121 122 bool runOnMachineFunction(MachineFunction &MF) override; 123 124 void processPHINode(MachineInstr &MI); 125 126 StringRef getPassName() const override { return "SI Fix SGPR copies"; } 127 128 void getAnalysisUsage(AnalysisUsage &AU) const override { 129 AU.addRequired<MachineDominatorTree>(); 130 AU.addPreserved<MachineDominatorTree>(); 131 AU.setPreservesCFG(); 132 MachineFunctionPass::getAnalysisUsage(AU); 133 } 134 }; 135 136 } // end anonymous namespace 137 138 INITIALIZE_PASS_BEGIN(SIFixSGPRCopies, DEBUG_TYPE, 139 "SI Fix SGPR copies", false, false) 140 INITIALIZE_PASS_DEPENDENCY(MachineDominatorTree) 141 INITIALIZE_PASS_END(SIFixSGPRCopies, DEBUG_TYPE, 142 "SI Fix SGPR copies", false, false) 143 144 char SIFixSGPRCopies::ID = 0; 145 146 char &llvm::SIFixSGPRCopiesID = SIFixSGPRCopies::ID; 147 148 FunctionPass *llvm::createSIFixSGPRCopiesPass() { 149 return new SIFixSGPRCopies(); 150 } 151 152 static bool hasVectorOperands(const MachineInstr &MI, 153 const SIRegisterInfo *TRI) { 154 const MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo(); 155 for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) { 156 if (!MI.getOperand(i).isReg() || 157 !Register::isVirtualRegister(MI.getOperand(i).getReg())) 158 continue; 159 160 if (TRI->hasVectorRegisters(MRI.getRegClass(MI.getOperand(i).getReg()))) 161 return true; 162 } 163 return false; 164 } 165 166 static std::pair<const TargetRegisterClass *, const TargetRegisterClass *> 167 getCopyRegClasses(const MachineInstr &Copy, 168 const SIRegisterInfo &TRI, 169 const MachineRegisterInfo &MRI) { 170 Register DstReg = Copy.getOperand(0).getReg(); 171 Register SrcReg = Copy.getOperand(1).getReg(); 172 173 const TargetRegisterClass *SrcRC = Register::isVirtualRegister(SrcReg) 174 ? MRI.getRegClass(SrcReg) 175 : TRI.getPhysRegClass(SrcReg); 176 177 // We don't really care about the subregister here. 178 // SrcRC = TRI.getSubRegClass(SrcRC, Copy.getOperand(1).getSubReg()); 179 180 const TargetRegisterClass *DstRC = Register::isVirtualRegister(DstReg) 181 ? MRI.getRegClass(DstReg) 182 : TRI.getPhysRegClass(DstReg); 183 184 return std::make_pair(SrcRC, DstRC); 185 } 186 187 static bool isVGPRToSGPRCopy(const TargetRegisterClass *SrcRC, 188 const TargetRegisterClass *DstRC, 189 const SIRegisterInfo &TRI) { 190 return SrcRC != &AMDGPU::VReg_1RegClass && TRI.isSGPRClass(DstRC) && 191 TRI.hasVectorRegisters(SrcRC); 192 } 193 194 static bool isSGPRToVGPRCopy(const TargetRegisterClass *SrcRC, 195 const TargetRegisterClass *DstRC, 196 const SIRegisterInfo &TRI) { 197 return DstRC != &AMDGPU::VReg_1RegClass && TRI.isSGPRClass(SrcRC) && 198 TRI.hasVectorRegisters(DstRC); 199 } 200 201 static bool tryChangeVGPRtoSGPRinCopy(MachineInstr &MI, 202 const SIRegisterInfo *TRI, 203 const SIInstrInfo *TII) { 204 MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo(); 205 auto &Src = MI.getOperand(1); 206 Register DstReg = MI.getOperand(0).getReg(); 207 Register SrcReg = Src.getReg(); 208 if (!Register::isVirtualRegister(SrcReg) || 209 !Register::isVirtualRegister(DstReg)) 210 return false; 211 212 for (const auto &MO : MRI.reg_nodbg_operands(DstReg)) { 213 const auto *UseMI = MO.getParent(); 214 if (UseMI == &MI) 215 continue; 216 if (MO.isDef() || UseMI->getParent() != MI.getParent() || 217 UseMI->getOpcode() <= TargetOpcode::GENERIC_OP_END || 218 !TII->isOperandLegal(*UseMI, UseMI->getOperandNo(&MO), &Src)) 219 return false; 220 } 221 // Change VGPR to SGPR destination. 222 MRI.setRegClass(DstReg, TRI->getEquivalentSGPRClass(MRI.getRegClass(DstReg))); 223 return true; 224 } 225 226 // Distribute an SGPR->VGPR copy of a REG_SEQUENCE into a VGPR REG_SEQUENCE. 227 // 228 // SGPRx = ... 229 // SGPRy = REG_SEQUENCE SGPRx, sub0 ... 230 // VGPRz = COPY SGPRy 231 // 232 // ==> 233 // 234 // VGPRx = COPY SGPRx 235 // VGPRz = REG_SEQUENCE VGPRx, sub0 236 // 237 // This exposes immediate folding opportunities when materializing 64-bit 238 // immediates. 239 static bool foldVGPRCopyIntoRegSequence(MachineInstr &MI, 240 const SIRegisterInfo *TRI, 241 const SIInstrInfo *TII, 242 MachineRegisterInfo &MRI) { 243 assert(MI.isRegSequence()); 244 245 Register DstReg = MI.getOperand(0).getReg(); 246 if (!TRI->isSGPRClass(MRI.getRegClass(DstReg))) 247 return false; 248 249 if (!MRI.hasOneUse(DstReg)) 250 return false; 251 252 MachineInstr &CopyUse = *MRI.use_instr_begin(DstReg); 253 if (!CopyUse.isCopy()) 254 return false; 255 256 // It is illegal to have vreg inputs to a physreg defining reg_sequence. 257 if (Register::isPhysicalRegister(CopyUse.getOperand(0).getReg())) 258 return false; 259 260 const TargetRegisterClass *SrcRC, *DstRC; 261 std::tie(SrcRC, DstRC) = getCopyRegClasses(CopyUse, *TRI, MRI); 262 263 if (!isSGPRToVGPRCopy(SrcRC, DstRC, *TRI)) 264 return false; 265 266 if (tryChangeVGPRtoSGPRinCopy(CopyUse, TRI, TII)) 267 return true; 268 269 // TODO: Could have multiple extracts? 270 unsigned SubReg = CopyUse.getOperand(1).getSubReg(); 271 if (SubReg != AMDGPU::NoSubRegister) 272 return false; 273 274 MRI.setRegClass(DstReg, DstRC); 275 276 // SGPRx = ... 277 // SGPRy = REG_SEQUENCE SGPRx, sub0 ... 278 // VGPRz = COPY SGPRy 279 280 // => 281 // VGPRx = COPY SGPRx 282 // VGPRz = REG_SEQUENCE VGPRx, sub0 283 284 MI.getOperand(0).setReg(CopyUse.getOperand(0).getReg()); 285 bool IsAGPR = TRI->hasAGPRs(DstRC); 286 287 for (unsigned I = 1, N = MI.getNumOperands(); I != N; I += 2) { 288 Register SrcReg = MI.getOperand(I).getReg(); 289 unsigned SrcSubReg = MI.getOperand(I).getSubReg(); 290 291 const TargetRegisterClass *SrcRC = MRI.getRegClass(SrcReg); 292 assert(TRI->isSGPRClass(SrcRC) && 293 "Expected SGPR REG_SEQUENCE to only have SGPR inputs"); 294 295 SrcRC = TRI->getSubRegClass(SrcRC, SrcSubReg); 296 const TargetRegisterClass *NewSrcRC = TRI->getEquivalentVGPRClass(SrcRC); 297 298 Register TmpReg = MRI.createVirtualRegister(NewSrcRC); 299 300 BuildMI(*MI.getParent(), &MI, MI.getDebugLoc(), TII->get(AMDGPU::COPY), 301 TmpReg) 302 .add(MI.getOperand(I)); 303 304 if (IsAGPR) { 305 const TargetRegisterClass *NewSrcRC = TRI->getEquivalentAGPRClass(SrcRC); 306 Register TmpAReg = MRI.createVirtualRegister(NewSrcRC); 307 unsigned Opc = NewSrcRC == &AMDGPU::AGPR_32RegClass ? 308 AMDGPU::V_ACCVGPR_WRITE_B32 : AMDGPU::COPY; 309 BuildMI(*MI.getParent(), &MI, MI.getDebugLoc(), TII->get(Opc), 310 TmpAReg) 311 .addReg(TmpReg, RegState::Kill); 312 TmpReg = TmpAReg; 313 } 314 315 MI.getOperand(I).setReg(TmpReg); 316 } 317 318 CopyUse.eraseFromParent(); 319 return true; 320 } 321 322 static bool isSafeToFoldImmIntoCopy(const MachineInstr *Copy, 323 const MachineInstr *MoveImm, 324 const SIInstrInfo *TII, 325 unsigned &SMovOp, 326 int64_t &Imm) { 327 if (Copy->getOpcode() != AMDGPU::COPY) 328 return false; 329 330 if (!MoveImm->isMoveImmediate()) 331 return false; 332 333 const MachineOperand *ImmOp = 334 TII->getNamedOperand(*MoveImm, AMDGPU::OpName::src0); 335 if (!ImmOp->isImm()) 336 return false; 337 338 // FIXME: Handle copies with sub-regs. 339 if (Copy->getOperand(0).getSubReg()) 340 return false; 341 342 switch (MoveImm->getOpcode()) { 343 default: 344 return false; 345 case AMDGPU::V_MOV_B32_e32: 346 SMovOp = AMDGPU::S_MOV_B32; 347 break; 348 case AMDGPU::V_MOV_B64_PSEUDO: 349 SMovOp = AMDGPU::S_MOV_B64; 350 break; 351 } 352 Imm = ImmOp->getImm(); 353 return true; 354 } 355 356 template <class UnaryPredicate> 357 bool searchPredecessors(const MachineBasicBlock *MBB, 358 const MachineBasicBlock *CutOff, 359 UnaryPredicate Predicate) { 360 if (MBB == CutOff) 361 return false; 362 363 DenseSet<const MachineBasicBlock *> Visited; 364 SmallVector<MachineBasicBlock *, 4> Worklist(MBB->pred_begin(), 365 MBB->pred_end()); 366 367 while (!Worklist.empty()) { 368 MachineBasicBlock *MBB = Worklist.pop_back_val(); 369 370 if (!Visited.insert(MBB).second) 371 continue; 372 if (MBB == CutOff) 373 continue; 374 if (Predicate(MBB)) 375 return true; 376 377 Worklist.append(MBB->pred_begin(), MBB->pred_end()); 378 } 379 380 return false; 381 } 382 383 // Checks if there is potential path From instruction To instruction. 384 // If CutOff is specified and it sits in between of that path we ignore 385 // a higher portion of the path and report it is not reachable. 386 static bool isReachable(const MachineInstr *From, 387 const MachineInstr *To, 388 const MachineBasicBlock *CutOff, 389 MachineDominatorTree &MDT) { 390 // If either From block dominates To block or instructions are in the same 391 // block and From is higher. 392 if (MDT.dominates(From, To)) 393 return true; 394 395 const MachineBasicBlock *MBBFrom = From->getParent(); 396 const MachineBasicBlock *MBBTo = To->getParent(); 397 if (MBBFrom == MBBTo) 398 return false; 399 400 // Instructions are in different blocks, do predecessor search. 401 // We should almost never get here since we do not usually produce M0 stores 402 // other than -1. 403 return searchPredecessors(MBBTo, CutOff, [MBBFrom] 404 (const MachineBasicBlock *MBB) { return MBB == MBBFrom; }); 405 } 406 407 // Return the first non-prologue instruction in the block. 408 static MachineBasicBlock::iterator 409 getFirstNonPrologue(MachineBasicBlock *MBB, const TargetInstrInfo *TII) { 410 MachineBasicBlock::iterator I = MBB->getFirstNonPHI(); 411 while (I != MBB->end() && TII->isBasicBlockPrologue(*I)) 412 ++I; 413 414 return I; 415 } 416 417 // Hoist and merge identical SGPR initializations into a common predecessor. 418 // This is intended to combine M0 initializations, but can work with any 419 // SGPR. A VGPR cannot be processed since we cannot guarantee vector 420 // executioon. 421 static bool hoistAndMergeSGPRInits(unsigned Reg, 422 const MachineRegisterInfo &MRI, 423 const TargetRegisterInfo *TRI, 424 MachineDominatorTree &MDT, 425 const TargetInstrInfo *TII) { 426 // List of inits by immediate value. 427 using InitListMap = std::map<unsigned, std::list<MachineInstr *>>; 428 InitListMap Inits; 429 // List of clobbering instructions. 430 SmallVector<MachineInstr*, 8> Clobbers; 431 // List of instructions marked for deletion. 432 SmallSet<MachineInstr*, 8> MergedInstrs; 433 434 bool Changed = false; 435 436 for (auto &MI : MRI.def_instructions(Reg)) { 437 MachineOperand *Imm = nullptr; 438 for (auto &MO : MI.operands()) { 439 if ((MO.isReg() && ((MO.isDef() && MO.getReg() != Reg) || !MO.isDef())) || 440 (!MO.isImm() && !MO.isReg()) || (MO.isImm() && Imm)) { 441 Imm = nullptr; 442 break; 443 } else if (MO.isImm()) 444 Imm = &MO; 445 } 446 if (Imm) 447 Inits[Imm->getImm()].push_front(&MI); 448 else 449 Clobbers.push_back(&MI); 450 } 451 452 for (auto &Init : Inits) { 453 auto &Defs = Init.second; 454 455 for (auto I1 = Defs.begin(), E = Defs.end(); I1 != E; ) { 456 MachineInstr *MI1 = *I1; 457 458 for (auto I2 = std::next(I1); I2 != E; ) { 459 MachineInstr *MI2 = *I2; 460 461 // Check any possible interference 462 auto interferes = [&](MachineBasicBlock::iterator From, 463 MachineBasicBlock::iterator To) -> bool { 464 465 assert(MDT.dominates(&*To, &*From)); 466 467 auto interferes = [&MDT, From, To](MachineInstr* &Clobber) -> bool { 468 const MachineBasicBlock *MBBFrom = From->getParent(); 469 const MachineBasicBlock *MBBTo = To->getParent(); 470 bool MayClobberFrom = isReachable(Clobber, &*From, MBBTo, MDT); 471 bool MayClobberTo = isReachable(Clobber, &*To, MBBTo, MDT); 472 if (!MayClobberFrom && !MayClobberTo) 473 return false; 474 if ((MayClobberFrom && !MayClobberTo) || 475 (!MayClobberFrom && MayClobberTo)) 476 return true; 477 // Both can clobber, this is not an interference only if both are 478 // dominated by Clobber and belong to the same block or if Clobber 479 // properly dominates To, given that To >> From, so it dominates 480 // both and located in a common dominator. 481 return !((MBBFrom == MBBTo && 482 MDT.dominates(Clobber, &*From) && 483 MDT.dominates(Clobber, &*To)) || 484 MDT.properlyDominates(Clobber->getParent(), MBBTo)); 485 }; 486 487 return (llvm::any_of(Clobbers, interferes)) || 488 (llvm::any_of(Inits, [&](InitListMap::value_type &C) { 489 return C.first != Init.first && 490 llvm::any_of(C.second, interferes); 491 })); 492 }; 493 494 if (MDT.dominates(MI1, MI2)) { 495 if (!interferes(MI2, MI1)) { 496 LLVM_DEBUG(dbgs() 497 << "Erasing from " 498 << printMBBReference(*MI2->getParent()) << " " << *MI2); 499 MergedInstrs.insert(MI2); 500 Changed = true; 501 ++I2; 502 continue; 503 } 504 } else if (MDT.dominates(MI2, MI1)) { 505 if (!interferes(MI1, MI2)) { 506 LLVM_DEBUG(dbgs() 507 << "Erasing from " 508 << printMBBReference(*MI1->getParent()) << " " << *MI1); 509 MergedInstrs.insert(MI1); 510 Changed = true; 511 ++I1; 512 break; 513 } 514 } else { 515 auto *MBB = MDT.findNearestCommonDominator(MI1->getParent(), 516 MI2->getParent()); 517 if (!MBB) { 518 ++I2; 519 continue; 520 } 521 522 MachineBasicBlock::iterator I = getFirstNonPrologue(MBB, TII); 523 if (!interferes(MI1, I) && !interferes(MI2, I)) { 524 LLVM_DEBUG(dbgs() 525 << "Erasing from " 526 << printMBBReference(*MI1->getParent()) << " " << *MI1 527 << "and moving from " 528 << printMBBReference(*MI2->getParent()) << " to " 529 << printMBBReference(*I->getParent()) << " " << *MI2); 530 I->getParent()->splice(I, MI2->getParent(), MI2); 531 MergedInstrs.insert(MI1); 532 Changed = true; 533 ++I1; 534 break; 535 } 536 } 537 ++I2; 538 } 539 ++I1; 540 } 541 } 542 543 // Remove initializations that were merged into another. 544 for (auto &Init : Inits) { 545 auto &Defs = Init.second; 546 auto I = Defs.begin(); 547 while (I != Defs.end()) { 548 if (MergedInstrs.count(*I)) { 549 (*I)->eraseFromParent(); 550 I = Defs.erase(I); 551 } else 552 ++I; 553 } 554 } 555 556 // Try to schedule SGPR initializations as early as possible in the MBB. 557 for (auto &Init : Inits) { 558 auto &Defs = Init.second; 559 for (auto MI : Defs) { 560 auto MBB = MI->getParent(); 561 MachineInstr &BoundaryMI = *getFirstNonPrologue(MBB, TII); 562 MachineBasicBlock::reverse_iterator B(BoundaryMI); 563 // Check if B should actually be a boundary. If not set the previous 564 // instruction as the boundary instead. 565 if (!TII->isBasicBlockPrologue(*B)) 566 B++; 567 568 auto R = std::next(MI->getReverseIterator()); 569 const unsigned Threshold = 50; 570 // Search until B or Threshold for a place to insert the initialization. 571 for (unsigned I = 0; R != B && I < Threshold; ++R, ++I) 572 if (R->readsRegister(Reg, TRI) || R->definesRegister(Reg, TRI) || 573 TII->isSchedulingBoundary(*R, MBB, *MBB->getParent())) 574 break; 575 576 // Move to directly after R. 577 if (&*--R != MI) 578 MBB->splice(*R, MBB, MI); 579 } 580 } 581 582 if (Changed) 583 MRI.clearKillFlags(Reg); 584 585 return Changed; 586 } 587 588 bool SIFixSGPRCopies::runOnMachineFunction(MachineFunction &MF) { 589 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>(); 590 MRI = &MF.getRegInfo(); 591 TRI = ST.getRegisterInfo(); 592 TII = ST.getInstrInfo(); 593 MDT = &getAnalysis<MachineDominatorTree>(); 594 595 SmallVector<MachineInstr *, 16> Worklist; 596 597 for (MachineFunction::iterator BI = MF.begin(), BE = MF.end(); 598 BI != BE; ++BI) { 599 MachineBasicBlock &MBB = *BI; 600 for (MachineBasicBlock::iterator I = MBB.begin(), E = MBB.end(); 601 I != E; ++I) { 602 MachineInstr &MI = *I; 603 604 switch (MI.getOpcode()) { 605 default: 606 continue; 607 case AMDGPU::COPY: 608 case AMDGPU::WQM: 609 case AMDGPU::SOFT_WQM: 610 case AMDGPU::WWM: { 611 Register DstReg = MI.getOperand(0).getReg(); 612 613 const TargetRegisterClass *SrcRC, *DstRC; 614 std::tie(SrcRC, DstRC) = getCopyRegClasses(MI, *TRI, *MRI); 615 616 if (!Register::isVirtualRegister(DstReg)) { 617 // If the destination register is a physical register there isn't 618 // really much we can do to fix this. 619 // Some special instructions use M0 as an input. Some even only use 620 // the first lane. Insert a readfirstlane and hope for the best. 621 if (DstReg == AMDGPU::M0 && TRI->hasVectorRegisters(SrcRC)) { 622 Register TmpReg 623 = MRI->createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass); 624 625 BuildMI(MBB, MI, MI.getDebugLoc(), 626 TII->get(AMDGPU::V_READFIRSTLANE_B32), TmpReg) 627 .add(MI.getOperand(1)); 628 MI.getOperand(1).setReg(TmpReg); 629 } 630 631 continue; 632 } 633 634 if (isVGPRToSGPRCopy(SrcRC, DstRC, *TRI)) { 635 Register SrcReg = MI.getOperand(1).getReg(); 636 if (!Register::isVirtualRegister(SrcReg)) { 637 TII->moveToVALU(MI, MDT); 638 break; 639 } 640 641 MachineInstr *DefMI = MRI->getVRegDef(SrcReg); 642 unsigned SMovOp; 643 int64_t Imm; 644 // If we are just copying an immediate, we can replace the copy with 645 // s_mov_b32. 646 if (isSafeToFoldImmIntoCopy(&MI, DefMI, TII, SMovOp, Imm)) { 647 MI.getOperand(1).ChangeToImmediate(Imm); 648 MI.addImplicitDefUseOperands(MF); 649 MI.setDesc(TII->get(SMovOp)); 650 break; 651 } 652 TII->moveToVALU(MI, MDT); 653 } else if (isSGPRToVGPRCopy(SrcRC, DstRC, *TRI)) { 654 tryChangeVGPRtoSGPRinCopy(MI, TRI, TII); 655 } 656 657 break; 658 } 659 case AMDGPU::PHI: { 660 processPHINode(MI); 661 break; 662 } 663 case AMDGPU::REG_SEQUENCE: 664 if (TRI->hasVectorRegisters(TII->getOpRegClass(MI, 0)) || 665 !hasVectorOperands(MI, TRI)) { 666 foldVGPRCopyIntoRegSequence(MI, TRI, TII, *MRI); 667 continue; 668 } 669 670 LLVM_DEBUG(dbgs() << "Fixing REG_SEQUENCE: " << MI); 671 672 TII->moveToVALU(MI, MDT); 673 break; 674 case AMDGPU::INSERT_SUBREG: { 675 const TargetRegisterClass *DstRC, *Src0RC, *Src1RC; 676 DstRC = MRI->getRegClass(MI.getOperand(0).getReg()); 677 Src0RC = MRI->getRegClass(MI.getOperand(1).getReg()); 678 Src1RC = MRI->getRegClass(MI.getOperand(2).getReg()); 679 if (TRI->isSGPRClass(DstRC) && 680 (TRI->hasVectorRegisters(Src0RC) || 681 TRI->hasVectorRegisters(Src1RC))) { 682 LLVM_DEBUG(dbgs() << " Fixing INSERT_SUBREG: " << MI); 683 TII->moveToVALU(MI, MDT); 684 } 685 break; 686 } 687 case AMDGPU::V_WRITELANE_B32: { 688 // Some architectures allow more than one constant bus access without 689 // SGPR restriction 690 if (ST.getConstantBusLimit(MI.getOpcode()) != 1) 691 break; 692 693 // Writelane is special in that it can use SGPR and M0 (which would 694 // normally count as using the constant bus twice - but in this case it 695 // is allowed since the lane selector doesn't count as a use of the 696 // constant bus). However, it is still required to abide by the 1 SGPR 697 // rule. Apply a fix here as we might have multiple SGPRs after 698 // legalizing VGPRs to SGPRs 699 int Src0Idx = 700 AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::src0); 701 int Src1Idx = 702 AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::src1); 703 MachineOperand &Src0 = MI.getOperand(Src0Idx); 704 MachineOperand &Src1 = MI.getOperand(Src1Idx); 705 706 // Check to see if the instruction violates the 1 SGPR rule 707 if ((Src0.isReg() && TRI->isSGPRReg(*MRI, Src0.getReg()) && 708 Src0.getReg() != AMDGPU::M0) && 709 (Src1.isReg() && TRI->isSGPRReg(*MRI, Src1.getReg()) && 710 Src1.getReg() != AMDGPU::M0)) { 711 712 // Check for trivially easy constant prop into one of the operands 713 // If this is the case then perform the operation now to resolve SGPR 714 // issue. If we don't do that here we will always insert a mov to m0 715 // that can't be resolved in later operand folding pass 716 bool Resolved = false; 717 for (MachineOperand *MO : {&Src0, &Src1}) { 718 if (Register::isVirtualRegister(MO->getReg())) { 719 MachineInstr *DefMI = MRI->getVRegDef(MO->getReg()); 720 if (DefMI && TII->isFoldableCopy(*DefMI)) { 721 const MachineOperand &Def = DefMI->getOperand(0); 722 if (Def.isReg() && 723 MO->getReg() == Def.getReg() && 724 MO->getSubReg() == Def.getSubReg()) { 725 const MachineOperand &Copied = DefMI->getOperand(1); 726 if (Copied.isImm() && 727 TII->isInlineConstant(APInt(64, Copied.getImm(), true))) { 728 MO->ChangeToImmediate(Copied.getImm()); 729 Resolved = true; 730 break; 731 } 732 } 733 } 734 } 735 } 736 737 if (!Resolved) { 738 // Haven't managed to resolve by replacing an SGPR with an immediate 739 // Move src1 to be in M0 740 BuildMI(*MI.getParent(), MI, MI.getDebugLoc(), 741 TII->get(AMDGPU::COPY), AMDGPU::M0) 742 .add(Src1); 743 Src1.ChangeToRegister(AMDGPU::M0, false); 744 } 745 } 746 break; 747 } 748 } 749 } 750 } 751 752 if (MF.getTarget().getOptLevel() > CodeGenOpt::None && EnableM0Merge) 753 hoistAndMergeSGPRInits(AMDGPU::M0, *MRI, TRI, *MDT, TII); 754 755 return true; 756 } 757 758 void SIFixSGPRCopies::processPHINode(MachineInstr &MI) { 759 unsigned numVGPRUses = 0; 760 bool AllAGPRUses = true; 761 SetVector<const MachineInstr *> worklist; 762 SmallSet<const MachineInstr *, 4> Visited; 763 worklist.insert(&MI); 764 Visited.insert(&MI); 765 while (!worklist.empty()) { 766 const MachineInstr *Instr = worklist.pop_back_val(); 767 unsigned Reg = Instr->getOperand(0).getReg(); 768 for (const auto &Use : MRI->use_operands(Reg)) { 769 const MachineInstr *UseMI = Use.getParent(); 770 AllAGPRUses &= (UseMI->isCopy() && 771 TRI->isAGPR(*MRI, UseMI->getOperand(0).getReg())) || 772 TRI->isAGPR(*MRI, Use.getReg()); 773 if (UseMI->isCopy() || UseMI->isRegSequence()) { 774 if (UseMI->isCopy() && 775 UseMI->getOperand(0).getReg().isPhysical() && 776 !TRI->isSGPRReg(*MRI, UseMI->getOperand(0).getReg())) { 777 numVGPRUses++; 778 } 779 if (Visited.insert(UseMI).second) 780 worklist.insert(UseMI); 781 782 continue; 783 } 784 785 if (UseMI->isPHI()) { 786 const TargetRegisterClass *UseRC = MRI->getRegClass(Use.getReg()); 787 if (!TRI->isSGPRReg(*MRI, Use.getReg()) && 788 UseRC != &AMDGPU::VReg_1RegClass) 789 numVGPRUses++; 790 continue; 791 } 792 793 const TargetRegisterClass *OpRC = 794 TII->getOpRegClass(*UseMI, UseMI->getOperandNo(&Use)); 795 if (!TRI->isSGPRClass(OpRC) && OpRC != &AMDGPU::VS_32RegClass && 796 OpRC != &AMDGPU::VS_64RegClass) { 797 numVGPRUses++; 798 } 799 } 800 } 801 802 Register PHIRes = MI.getOperand(0).getReg(); 803 const TargetRegisterClass *RC0 = MRI->getRegClass(PHIRes); 804 if (AllAGPRUses && numVGPRUses && !TRI->hasAGPRs(RC0)) { 805 LLVM_DEBUG(dbgs() << "Moving PHI to AGPR: " << MI); 806 MRI->setRegClass(PHIRes, TRI->getEquivalentAGPRClass(RC0)); 807 } 808 809 bool hasVGPRInput = false; 810 for (unsigned i = 1; i < MI.getNumOperands(); i += 2) { 811 unsigned InputReg = MI.getOperand(i).getReg(); 812 MachineInstr *Def = MRI->getVRegDef(InputReg); 813 if (TRI->isVectorRegister(*MRI, InputReg)) { 814 if (Def->isCopy()) { 815 unsigned SrcReg = Def->getOperand(1).getReg(); 816 const TargetRegisterClass *RC = 817 TRI->getRegClassForReg(*MRI, SrcReg); 818 if (TRI->isSGPRClass(RC)) 819 continue; 820 } 821 hasVGPRInput = true; 822 break; 823 } 824 else if (Def->isCopy() && 825 TRI->isVectorRegister(*MRI, Def->getOperand(1).getReg())) { 826 hasVGPRInput = true; 827 break; 828 } 829 } 830 831 if ((!TRI->isVectorRegister(*MRI, PHIRes) && 832 RC0 != &AMDGPU::VReg_1RegClass) && 833 (hasVGPRInput || numVGPRUses > 1)) { 834 LLVM_DEBUG(dbgs() << "Fixing PHI: " << MI); 835 TII->moveToVALU(MI); 836 } 837 else { 838 LLVM_DEBUG(dbgs() << "Legalizing PHI: " << MI); 839 TII->legalizeOperands(MI, MDT); 840 } 841 842 } 843